2023-08-27T21:31:54.575 INFO:root:teuthology version: 0.0.1.dev185+g449a1bc 2023-08-27T21:31:54.576 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --description upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} --archive /home/teuthworker/archive/teuthology-2023-08-26_14:23:13-upgrade:pacific-x-reef-distro-default-smithi/7381797 --name teuthology-2023-08-26_14:23:13-upgrade:pacific-x-reef-distro-default-smithi --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2023-08-26_14:23:13-upgrade:pacific-x-reef-distro-default-smithi/7381797/orig.config.yaml 2023-08-27T21:31:54.635 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-08-27T21:31:54.723 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-08-26_14:23:13-upgrade:pacific-x-reef-distro-default-smithi/7381797 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' first_in_suite: false job_id: '7381797' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2023-08-26_14:23:13-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: ubuntu os_version: '20.04' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 kernel: hwe: true workunit: branch: reef sha1: 81fef490342f7b9544e89fe32a9e51929064494d owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 2117 sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_81fef490342f7b9544e89fe32a9e51929064494d/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: 81fef490342f7b9544e89fe32a9e51929064494d targets: smithi006.front.sepia.ceph.com: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw smithi130.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM= tasks: - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 449a1bc2027504e7b3c3d7b30fa4178906581da7 timestamp: 2023-08-26_14:23:13 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.29856 2023-08-27T21:31:54.724 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_81fef490342f7b9544e89fe32a9e51929064494d/qa; will attempt to use it 2023-08-27T21:31:54.725 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_81fef490342f7b9544e89fe32a9e51929064494d/qa/tasks 2023-08-27T21:31:54.725 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-08-27T21:31:54.726 INFO:teuthology.task.internal:Checking packages... 2023-08-27T21:31:54.749 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash '9cf208207dfe23ffccb33308b22e51570cb45e51' 2023-08-27T21:31:54.750 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-08-27T21:31:54.750 INFO:teuthology.packaging:ref: None 2023-08-27T21:31:54.750 INFO:teuthology.packaging:tag: None 2023-08-27T21:31:54.750 INFO:teuthology.packaging:branch: reef 2023-08-27T21:31:54.750 INFO:teuthology.packaging:sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T21:31:54.750 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=reef 2023-08-27T21:31:55.085 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-191-g02042ef2-1focal 2023-08-27T21:31:55.085 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-08-27T21:31:55.091 INFO:teuthology.task.internal:no buildpackages task found 2023-08-27T21:31:55.091 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-08-27T21:31:55.109 INFO:teuthology.task.internal:Saving configuration 2023-08-27T21:31:55.130 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-08-27T21:31:55.136 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-08-27T21:31:55.161 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi006.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-08-26_14:23:13-upgrade:pacific-x-reef-distro-default-smithi/7381797', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-08-27 21:26:05.022085', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw'} 2023-08-27T21:31:55.183 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi130.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-08-26_14:23:13-upgrade:pacific-x-reef-distro-default-smithi/7381797', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-08-27 21:26:05.020078', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw'} 2023-08-27T21:31:55.183 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-08-27T21:31:55.189 INFO:teuthology.task.internal:roles: ubuntu@smithi006.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-08-27T21:31:55.189 INFO:teuthology.task.internal:roles: ubuntu@smithi130.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-08-27T21:31:55.189 INFO:teuthology.run_tasks:Running task console_log... 2023-08-27T21:31:55.260 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fa4f3caa040>, signals=[15]) 2023-08-27T21:31:55.261 INFO:teuthology.run_tasks:Running task internal.connect... 2023-08-27T21:31:55.267 INFO:teuthology.task.internal:Opening connections... 2023-08-27T21:31:55.267 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi006.front.sepia.ceph.com 2023-08-27T21:31:55.268 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi006.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T21:31:55.319 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi130.front.sepia.ceph.com 2023-08-27T21:31:55.319 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T21:31:55.364 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-08-27T21:31:55.370 DEBUG:teuthology.orchestra.run.smithi006:> uname -m 2023-08-27T21:31:55.404 INFO:teuthology.orchestra.run.smithi006.stdout:x86_64 2023-08-27T21:31:55.404 DEBUG:teuthology.orchestra.run.smithi006:> cat /etc/os-release 2023-08-27T21:31:55.454 INFO:teuthology.orchestra.run.smithi006.stdout:NAME="Ubuntu" 2023-08-27T21:31:55.454 INFO:teuthology.orchestra.run.smithi006.stdout:VERSION="20.04.4 LTS (Focal Fossa)" 2023-08-27T21:31:55.454 INFO:teuthology.orchestra.run.smithi006.stdout:ID=ubuntu 2023-08-27T21:31:55.454 INFO:teuthology.orchestra.run.smithi006.stdout:ID_LIKE=debian 2023-08-27T21:31:55.454 INFO:teuthology.orchestra.run.smithi006.stdout:PRETTY_NAME="Ubuntu 20.04.4 LTS" 2023-08-27T21:31:55.454 INFO:teuthology.orchestra.run.smithi006.stdout:VERSION_ID="20.04" 2023-08-27T21:31:55.454 INFO:teuthology.orchestra.run.smithi006.stdout:HOME_URL="https://www.ubuntu.com/" 2023-08-27T21:31:55.454 INFO:teuthology.orchestra.run.smithi006.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2023-08-27T21:31:55.455 INFO:teuthology.orchestra.run.smithi006.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2023-08-27T21:31:55.455 INFO:teuthology.orchestra.run.smithi006.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2023-08-27T21:31:55.455 INFO:teuthology.orchestra.run.smithi006.stdout:VERSION_CODENAME=focal 2023-08-27T21:31:55.455 INFO:teuthology.orchestra.run.smithi006.stdout:UBUNTU_CODENAME=focal 2023-08-27T21:31:55.455 INFO:teuthology.lock.ops:Updating smithi006.front.sepia.ceph.com on lock server 2023-08-27T21:31:55.474 DEBUG:teuthology.orchestra.run.smithi130:> uname -m 2023-08-27T21:31:55.481 INFO:teuthology.orchestra.run.smithi130.stdout:x86_64 2023-08-27T21:31:55.481 DEBUG:teuthology.orchestra.run.smithi130:> cat /etc/os-release 2023-08-27T21:31:55.528 INFO:teuthology.orchestra.run.smithi130.stdout:NAME="Ubuntu" 2023-08-27T21:31:55.528 INFO:teuthology.orchestra.run.smithi130.stdout:VERSION="20.04.4 LTS (Focal Fossa)" 2023-08-27T21:31:55.528 INFO:teuthology.orchestra.run.smithi130.stdout:ID=ubuntu 2023-08-27T21:31:55.528 INFO:teuthology.orchestra.run.smithi130.stdout:ID_LIKE=debian 2023-08-27T21:31:55.528 INFO:teuthology.orchestra.run.smithi130.stdout:PRETTY_NAME="Ubuntu 20.04.4 LTS" 2023-08-27T21:31:55.528 INFO:teuthology.orchestra.run.smithi130.stdout:VERSION_ID="20.04" 2023-08-27T21:31:55.528 INFO:teuthology.orchestra.run.smithi130.stdout:HOME_URL="https://www.ubuntu.com/" 2023-08-27T21:31:55.528 INFO:teuthology.orchestra.run.smithi130.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2023-08-27T21:31:55.528 INFO:teuthology.orchestra.run.smithi130.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2023-08-27T21:31:55.529 INFO:teuthology.orchestra.run.smithi130.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2023-08-27T21:31:55.529 INFO:teuthology.orchestra.run.smithi130.stdout:VERSION_CODENAME=focal 2023-08-27T21:31:55.529 INFO:teuthology.orchestra.run.smithi130.stdout:UBUNTU_CODENAME=focal 2023-08-27T21:31:55.529 INFO:teuthology.lock.ops:Updating smithi130.front.sepia.ceph.com on lock server 2023-08-27T21:31:55.558 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-08-27T21:31:55.571 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-08-27T21:31:55.578 INFO:teuthology.task.internal:Checking for old test directory... 2023-08-27T21:31:55.579 DEBUG:teuthology.orchestra.run.smithi006:> test '!' -e /home/ubuntu/cephtest 2023-08-27T21:31:55.582 DEBUG:teuthology.orchestra.run.smithi130:> test '!' -e /home/ubuntu/cephtest 2023-08-27T21:31:55.587 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-08-27T21:31:55.594 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-08-27T21:31:55.594 DEBUG:teuthology.orchestra.run.smithi006:> test -z $(ls -A /var/lib/ceph) 2023-08-27T21:31:55.596 DEBUG:teuthology.orchestra.run.smithi130:> test -z $(ls -A /var/lib/ceph) 2023-08-27T21:31:55.611 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-08-27T21:31:55.663 INFO:teuthology.run_tasks:Running task kernel... 2023-08-27T21:31:55.679 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-08-27T21:31:55.679 INFO:teuthology.task.kernel:normalize config orig: {'hwe': True} 2023-08-27T21:31:55.679 DEBUG:teuthology.task.kernel:normalized overrides {'mon.a': {'hwe': True}, 'mon.c': {'hwe': True}, 'mgr.y': {'hwe': True}, 'osd.0': {'hwe': True}, 'osd.1': {'hwe': True}, 'osd.2': {'hwe': True}, 'osd.3': {'hwe': True}, 'client.0': {'hwe': True}, 'node-exporter.a': {'hwe': True}, 'alertmanager.a': {'hwe': True}, 'mon.b': {'hwe': True}, 'mgr.x': {'hwe': True}, 'osd.4': {'hwe': True}, 'osd.5': {'hwe': True}, 'osd.6': {'hwe': True}, 'osd.7': {'hwe': True}, 'client.1': {'hwe': True}, 'prometheus.a': {'hwe': True}, 'grafana.a': {'hwe': True}, 'node-exporter.b': {'hwe': True}} 2023-08-27T21:31:55.679 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro', 'hwe': True}, 'mon.b': {'kdb': True, 'sha1': 'distro', 'hwe': True}}, timeout 300 2023-08-27T21:31:55.680 DEBUG:teuthology.orchestra.run.smithi006:> test -f /run/.containerenv -o -f /.dockerenv 2023-08-27T21:31:55.680 DEBUG:teuthology.orchestra.run.smithi130:> test -f /run/.containerenv -o -f /.dockerenv 2023-08-27T21:31:55.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-08-27T21:31:55.686 DEBUG:teuthology.orchestra.run.smithi006:> uname -r 2023-08-27T21:31:55.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-08-27T21:31:55.687 DEBUG:teuthology.orchestra.run.smithi130:> uname -r 2023-08-27T21:31:55.692 INFO:teuthology.orchestra.run.smithi006.stdout:5.4.0-124-generic 2023-08-27T21:31:55.693 INFO:teuthology.task.kernel:Running kernel on smithi006: 5.4.0-124-generic 2023-08-27T21:31:55.693 DEBUG:teuthology.orchestra.run.smithi006:> sudo apt-get clean 2023-08-27T21:31:55.694 INFO:teuthology.orchestra.run.smithi130.stdout:5.4.0-124-generic 2023-08-27T21:31:55.694 INFO:teuthology.task.kernel:Running kernel on smithi130: 5.4.0-124-generic 2023-08-27T21:31:55.694 DEBUG:teuthology.orchestra.run.smithi130:> sudo apt-get clean 2023-08-27T21:31:55.798 DEBUG:teuthology.orchestra.run.smithi006:> sudo apt-get update 2023-08-27T21:31:55.848 DEBUG:teuthology.orchestra.run.smithi130:> sudo apt-get update 2023-08-27T21:31:55.984 INFO:teuthology.orchestra.run.smithi006.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-08-27T21:31:55.992 INFO:teuthology.orchestra.run.smithi006.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-08-27T21:31:56.017 INFO:teuthology.orchestra.run.smithi130.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-08-27T21:31:56.017 INFO:teuthology.orchestra.run.smithi130.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-08-27T21:31:56.017 INFO:teuthology.orchestra.run.smithi130.stdout:Get:3 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-08-27T21:31:56.052 INFO:teuthology.orchestra.run.smithi006.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-08-27T21:31:56.075 INFO:teuthology.orchestra.run.smithi130.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-08-27T21:31:56.365 INFO:teuthology.orchestra.run.smithi006.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [2,753 kB] 2023-08-27T21:31:56.395 INFO:teuthology.orchestra.run.smithi130.stdout:Get:5 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,365 kB] 2023-08-27T21:31:56.497 INFO:teuthology.orchestra.run.smithi006.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [864 kB] 2023-08-27T21:31:56.513 INFO:teuthology.orchestra.run.smithi006.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [456 kB] 2023-08-27T21:31:56.517 INFO:teuthology.orchestra.run.smithi006.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [17.0 kB] 2023-08-27T21:31:56.517 INFO:teuthology.orchestra.run.smithi006.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [2,185 kB] 2023-08-27T21:31:56.532 INFO:teuthology.orchestra.run.smithi006.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [33.7 kB] 2023-08-27T21:31:56.532 INFO:teuthology.orchestra.run.smithi006.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [306 kB] 2023-08-27T21:31:56.533 INFO:teuthology.orchestra.run.smithi130.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [864 kB] 2023-08-27T21:31:56.535 INFO:teuthology.orchestra.run.smithi006.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [576 B] 2023-08-27T21:31:56.535 INFO:teuthology.orchestra.run.smithi006.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [744 kB] 2023-08-27T21:31:56.535 INFO:teuthology.orchestra.run.smithi130.stdout:Get:7 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [632 kB] 2023-08-27T21:31:56.540 INFO:teuthology.orchestra.run.smithi006.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,102 kB] 2023-08-27T21:31:56.545 INFO:teuthology.orchestra.run.smithi130.stdout:Get:8 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [373 kB] 2023-08-27T21:31:56.547 INFO:teuthology.orchestra.run.smithi006.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [263 kB] 2023-08-27T21:31:56.548 INFO:teuthology.orchestra.run.smithi130.stdout:Get:9 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [13.0 kB] 2023-08-27T21:31:56.548 INFO:teuthology.orchestra.run.smithi130.stdout:Get:10 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [2,070 kB] 2023-08-27T21:31:56.549 INFO:teuthology.orchestra.run.smithi006.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [25.4 kB] 2023-08-27T21:31:56.550 INFO:teuthology.orchestra.run.smithi006.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [25.8 kB] 2023-08-27T21:31:56.550 INFO:teuthology.orchestra.run.smithi006.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,436 B] 2023-08-27T21:31:56.563 INFO:teuthology.orchestra.run.smithi130.stdout:Get:11 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [32.3 kB] 2023-08-27T21:31:56.563 INFO:teuthology.orchestra.run.smithi130.stdout:Get:12 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [289 kB] 2023-08-27T21:31:56.564 INFO:teuthology.orchestra.run.smithi006.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,484 B] 2023-08-27T21:31:56.565 INFO:teuthology.orchestra.run.smithi006.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [620 B] 2023-08-27T21:31:56.565 INFO:teuthology.orchestra.run.smithi006.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [45.7 kB] 2023-08-27T21:31:56.565 INFO:teuthology.orchestra.run.smithi006.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [36.1 kB] 2023-08-27T21:31:56.566 INFO:teuthology.orchestra.run.smithi006.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 c-n-f Metadata [1,420 B] 2023-08-27T21:31:56.567 INFO:teuthology.orchestra.run.smithi130.stdout:Get:13 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [580 B] 2023-08-27T21:31:56.567 INFO:teuthology.orchestra.run.smithi130.stdout:Get:14 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [610 kB] 2023-08-27T21:31:56.572 INFO:teuthology.orchestra.run.smithi130.stdout:Get:15 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [869 kB] 2023-08-27T21:31:56.581 INFO:teuthology.orchestra.run.smithi006.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [13.8 kB] 2023-08-27T21:31:56.582 INFO:teuthology.orchestra.run.smithi006.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [25.0 kB] 2023-08-27T21:31:56.582 INFO:teuthology.orchestra.run.smithi006.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [880 B] 2023-08-27T21:31:56.607 INFO:teuthology.orchestra.run.smithi130.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [2,753 kB] 2023-08-27T21:31:56.612 INFO:teuthology.orchestra.run.smithi130.stdout:Get:17 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [182 kB] 2023-08-27T21:31:56.650 INFO:teuthology.orchestra.run.smithi130.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [456 kB] 2023-08-27T21:31:56.654 INFO:teuthology.orchestra.run.smithi130.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [17.0 kB] 2023-08-27T21:31:56.654 INFO:teuthology.orchestra.run.smithi130.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [2,185 kB] 2023-08-27T21:31:56.669 INFO:teuthology.orchestra.run.smithi130.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [33.7 kB] 2023-08-27T21:31:56.669 INFO:teuthology.orchestra.run.smithi130.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [306 kB] 2023-08-27T21:31:56.671 INFO:teuthology.orchestra.run.smithi130.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [576 B] 2023-08-27T21:31:56.671 INFO:teuthology.orchestra.run.smithi130.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,102 kB] 2023-08-27T21:31:56.678 INFO:teuthology.orchestra.run.smithi130.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [744 kB] 2023-08-27T21:31:56.683 INFO:teuthology.orchestra.run.smithi130.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [263 kB] 2023-08-27T21:31:56.685 INFO:teuthology.orchestra.run.smithi130.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [25.4 kB] 2023-08-27T21:31:56.685 INFO:teuthology.orchestra.run.smithi130.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [25.8 kB] 2023-08-27T21:31:56.686 INFO:teuthology.orchestra.run.smithi130.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,436 B] 2023-08-27T21:31:56.697 INFO:teuthology.orchestra.run.smithi130.stdout:Get:30 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [18.9 kB] 2023-08-27T21:31:56.698 INFO:teuthology.orchestra.run.smithi130.stdout:Get:31 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,196 B] 2023-08-27T21:31:56.699 INFO:teuthology.orchestra.run.smithi130.stdout:Get:32 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [23.6 kB] 2023-08-27T21:31:56.700 INFO:teuthology.orchestra.run.smithi130.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,484 B] 2023-08-27T21:31:56.701 INFO:teuthology.orchestra.run.smithi130.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [620 B] 2023-08-27T21:31:56.701 INFO:teuthology.orchestra.run.smithi130.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [36.1 kB] 2023-08-27T21:31:56.701 INFO:teuthology.orchestra.run.smithi130.stdout:Get:36 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,504 B] 2023-08-27T21:31:56.701 INFO:teuthology.orchestra.run.smithi130.stdout:Get:37 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [548 B] 2023-08-27T21:31:56.702 INFO:teuthology.orchestra.run.smithi130.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [45.7 kB] 2023-08-27T21:31:56.702 INFO:teuthology.orchestra.run.smithi130.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 c-n-f Metadata [1,420 B] 2023-08-27T21:31:56.702 INFO:teuthology.orchestra.run.smithi130.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [13.8 kB] 2023-08-27T21:31:56.719 INFO:teuthology.orchestra.run.smithi130.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [25.0 kB] 2023-08-27T21:31:56.719 INFO:teuthology.orchestra.run.smithi130.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [880 B] 2023-08-27T21:32:00.317 INFO:teuthology.orchestra.run.smithi130.stdout:Fetched 16.7 MB in 2s (6,807 kB/s) 2023-08-27T21:32:01.786 INFO:teuthology.orchestra.run.smithi130.stdout:Reading package lists... 2023-08-27T21:32:01.813 DEBUG:teuthology.orchestra.run.smithi130:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-08-27T21:32:01.881 INFO:teuthology.orchestra.run.smithi130.stdout:Reading package lists... 2023-08-27T21:32:02.060 INFO:teuthology.orchestra.run.smithi130.stdout:Building dependency tree... 2023-08-27T21:32:02.061 INFO:teuthology.orchestra.run.smithi130.stdout:Reading state information... 2023-08-27T21:32:02.222 INFO:teuthology.orchestra.run.smithi130.stdout:The following additional packages will be installed: 2023-08-27T21:32:02.222 INFO:teuthology.orchestra.run.smithi130.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-08-27T21:32:02.223 INFO:teuthology.orchestra.run.smithi130.stdout: libusbmuxd6 linux-image-5.15.0-79-generic linux-modules-5.15.0-79-generic 2023-08-27T21:32:02.223 INFO:teuthology.orchestra.run.smithi130.stdout: linux-modules-extra-5.15.0-79-generic thermald upower usbmuxd 2023-08-27T21:32:02.224 INFO:teuthology.orchestra.run.smithi130.stdout:Suggested packages: 2023-08-27T21:32:02.224 INFO:teuthology.orchestra.run.smithi130.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.15-source-5.15.0 2023-08-27T21:32:02.224 INFO:teuthology.orchestra.run.smithi130.stdout: linux-hwe-5.15-tools linux-headers-5.15.0-79-generic 2023-08-27T21:32:02.277 INFO:teuthology.orchestra.run.smithi130.stdout:The following NEW packages will be installed: 2023-08-27T21:32:02.277 INFO:teuthology.orchestra.run.smithi130.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-08-27T21:32:02.278 INFO:teuthology.orchestra.run.smithi130.stdout: libusbmuxd6 linux-image-5.15.0-79-generic linux-image-generic-hwe-20.04 2023-08-27T21:32:02.278 INFO:teuthology.orchestra.run.smithi130.stdout: linux-modules-5.15.0-79-generic linux-modules-extra-5.15.0-79-generic 2023-08-27T21:32:02.278 INFO:teuthology.orchestra.run.smithi130.stdout: thermald upower usbmuxd 2023-08-27T21:32:02.464 INFO:teuthology.orchestra.run.smithi130.stdout:0 upgraded, 13 newly installed, 0 to remove and 261 not upgraded. 2023-08-27T21:32:02.464 INFO:teuthology.orchestra.run.smithi130.stdout:Need to get 90.9 MB of archives. 2023-08-27T21:32:02.464 INFO:teuthology.orchestra.run.smithi130.stdout:After this operation, 497 MB of additional disk space will be used. 2023-08-27T21:32:02.464 INFO:teuthology.orchestra.run.smithi130.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2023-08-27T21:32:02.735 INFO:teuthology.orchestra.run.smithi130.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2023-08-27T21:32:02.764 INFO:teuthology.orchestra.run.smithi130.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2023-08-27T21:32:02.782 INFO:teuthology.orchestra.run.smithi130.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2023-08-27T21:32:02.855 INFO:teuthology.orchestra.run.smithi130.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2023-08-27T21:32:02.891 INFO:teuthology.orchestra.run.smithi130.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.15.0-79-generic amd64 5.15.0-79.86~20.04.2 [20.9 MB] 2023-08-27T21:32:03.779 INFO:teuthology.orchestra.run.smithi130.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.15.0-79-generic amd64 5.15.0-79.86~20.04.2 [11.4 MB] 2023-08-27T21:32:04.087 INFO:teuthology.orchestra.run.smithi130.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.15.0-79-generic amd64 5.15.0-79.86~20.04.2 [58.0 MB] 2023-08-27T21:32:05.249 INFO:teuthology.orchestra.run.smithi006.stdout:Get:26 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-08-27T21:32:05.476 INFO:teuthology.orchestra.run.smithi006.stdout:Get:27 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [632 kB] 2023-08-27T21:32:05.544 INFO:teuthology.orchestra.run.smithi006.stdout:Get:28 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,365 kB] 2023-08-27T21:32:05.583 INFO:teuthology.orchestra.run.smithi006.stdout:Get:29 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [373 kB] 2023-08-27T21:32:05.586 INFO:teuthology.orchestra.run.smithi006.stdout:Get:30 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [13.0 kB] 2023-08-27T21:32:05.587 INFO:teuthology.orchestra.run.smithi006.stdout:Get:31 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [32.3 kB] 2023-08-27T21:32:05.587 INFO:teuthology.orchestra.run.smithi006.stdout:Get:32 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [2,070 kB] 2023-08-27T21:32:05.601 INFO:teuthology.orchestra.run.smithi006.stdout:Get:33 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [289 kB] 2023-08-27T21:32:05.603 INFO:teuthology.orchestra.run.smithi006.stdout:Get:34 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [580 B] 2023-08-27T21:32:05.603 INFO:teuthology.orchestra.run.smithi006.stdout:Get:35 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [610 kB] 2023-08-27T21:32:05.607 INFO:teuthology.orchestra.run.smithi006.stdout:Get:36 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [869 kB] 2023-08-27T21:32:05.612 INFO:teuthology.orchestra.run.smithi006.stdout:Get:37 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [182 kB] 2023-08-27T21:32:05.614 INFO:teuthology.orchestra.run.smithi006.stdout:Get:38 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [18.9 kB] 2023-08-27T21:32:05.618 INFO:teuthology.orchestra.run.smithi006.stdout:Get:39 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [23.6 kB] 2023-08-27T21:32:05.619 INFO:teuthology.orchestra.run.smithi006.stdout:Get:40 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,196 B] 2023-08-27T21:32:05.619 INFO:teuthology.orchestra.run.smithi006.stdout:Get:41 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,504 B] 2023-08-27T21:32:05.619 INFO:teuthology.orchestra.run.smithi006.stdout:Get:42 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [548 B] 2023-08-27T21:32:05.994 INFO:teuthology.orchestra.run.smithi130.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.15.0.79.86~20.04.39 [2,696 B] 2023-08-27T21:32:05.994 INFO:teuthology.orchestra.run.smithi130.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.2 [31.6 kB] 2023-08-27T21:32:05.995 INFO:teuthology.orchestra.run.smithi130.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2023-08-27T21:32:06.164 INFO:teuthology.orchestra.run.smithi130.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2023-08-27T21:32:06.507 INFO:teuthology.orchestra.run.smithi130.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2023-08-27T21:32:07.237 INFO:teuthology.orchestra.run.smithi130.stdout:Fetched 90.9 MB in 4s (21.4 MB/s) 2023-08-27T21:32:07.433 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2023-08-27T21:32:08.387 INFO:teuthology.orchestra.run.smithi006.stdout:Fetched 16.7 MB in 11s (1,585 kB/s) 2023-08-27T21:32:09.141 INFO:teuthology.orchestra.run.smithi130.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 104373 files and directories currently installed.) 2023-08-27T21:32:09.146 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2023-08-27T21:32:09.280 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-08-27T21:32:09.616 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package libplist3:amd64. 2023-08-27T21:32:09.631 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2023-08-27T21:32:09.675 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2023-08-27T21:32:09.829 INFO:teuthology.orchestra.run.smithi006.stdout:Reading package lists... 2023-08-27T21:32:09.856 DEBUG:teuthology.orchestra.run.smithi006:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-08-27T21:32:09.929 INFO:teuthology.orchestra.run.smithi006.stdout:Reading package lists... 2023-08-27T21:32:10.011 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2023-08-27T21:32:10.026 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2023-08-27T21:32:10.069 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2023-08-27T21:32:10.132 INFO:teuthology.orchestra.run.smithi006.stdout:Building dependency tree... 2023-08-27T21:32:10.133 INFO:teuthology.orchestra.run.smithi006.stdout:Reading state information... 2023-08-27T21:32:10.309 INFO:teuthology.orchestra.run.smithi006.stdout:The following additional packages will be installed: 2023-08-27T21:32:10.309 INFO:teuthology.orchestra.run.smithi006.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-08-27T21:32:10.309 INFO:teuthology.orchestra.run.smithi006.stdout: libusbmuxd6 linux-image-5.15.0-79-generic linux-modules-5.15.0-79-generic 2023-08-27T21:32:10.309 INFO:teuthology.orchestra.run.smithi006.stdout: linux-modules-extra-5.15.0-79-generic thermald upower usbmuxd 2023-08-27T21:32:10.310 INFO:teuthology.orchestra.run.smithi006.stdout:Suggested packages: 2023-08-27T21:32:10.311 INFO:teuthology.orchestra.run.smithi006.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.15-source-5.15.0 2023-08-27T21:32:10.311 INFO:teuthology.orchestra.run.smithi006.stdout: linux-hwe-5.15-tools linux-headers-5.15.0-79-generic 2023-08-27T21:32:10.360 INFO:teuthology.orchestra.run.smithi006.stdout:The following NEW packages will be installed: 2023-08-27T21:32:10.361 INFO:teuthology.orchestra.run.smithi006.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-08-27T21:32:10.361 INFO:teuthology.orchestra.run.smithi006.stdout: libusbmuxd6 linux-image-5.15.0-79-generic linux-image-generic-hwe-20.04 2023-08-27T21:32:10.361 INFO:teuthology.orchestra.run.smithi006.stdout: linux-modules-5.15.0-79-generic linux-modules-extra-5.15.0-79-generic 2023-08-27T21:32:10.361 INFO:teuthology.orchestra.run.smithi006.stdout: thermald upower usbmuxd 2023-08-27T21:32:10.414 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2023-08-27T21:32:10.428 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2023-08-27T21:32:10.472 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-08-27T21:32:10.545 INFO:teuthology.orchestra.run.smithi006.stdout:0 upgraded, 13 newly installed, 0 to remove and 261 not upgraded. 2023-08-27T21:32:10.545 INFO:teuthology.orchestra.run.smithi006.stdout:Need to get 90.9 MB of archives. 2023-08-27T21:32:10.545 INFO:teuthology.orchestra.run.smithi006.stdout:After this operation, 497 MB of additional disk space will be used. 2023-08-27T21:32:10.545 INFO:teuthology.orchestra.run.smithi006.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2023-08-27T21:32:10.783 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package libupower-glib3:amd64. 2023-08-27T21:32:10.796 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2023-08-27T21:32:10.816 INFO:teuthology.orchestra.run.smithi006.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2023-08-27T21:32:10.825 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2023-08-27T21:32:10.844 INFO:teuthology.orchestra.run.smithi006.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2023-08-27T21:32:10.862 INFO:teuthology.orchestra.run.smithi006.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2023-08-27T21:32:10.936 INFO:teuthology.orchestra.run.smithi006.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2023-08-27T21:32:10.972 INFO:teuthology.orchestra.run.smithi006.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.15.0-79-generic amd64 5.15.0-79.86~20.04.2 [20.9 MB] 2023-08-27T21:32:11.170 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package linux-modules-5.15.0-79-generic. 2023-08-27T21:32:11.185 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../05-linux-modules-5.15.0-79-generic_5.15.0-79.86~20.04.2_amd64.deb ... 2023-08-27T21:32:11.228 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking linux-modules-5.15.0-79-generic (5.15.0-79.86~20.04.2) ... 2023-08-27T21:32:11.844 INFO:teuthology.orchestra.run.smithi006.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.15.0-79-generic amd64 5.15.0-79.86~20.04.2 [11.4 MB] 2023-08-27T21:32:12.152 INFO:teuthology.orchestra.run.smithi006.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.15.0-79-generic amd64 5.15.0-79.86~20.04.2 [58.0 MB] 2023-08-27T21:32:13.770 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package linux-image-5.15.0-79-generic. 2023-08-27T21:32:13.784 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../06-linux-image-5.15.0-79-generic_5.15.0-79.86~20.04.2_amd64.deb ... 2023-08-27T21:32:13.884 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking linux-image-5.15.0-79-generic (5.15.0-79.86~20.04.2) ... 2023-08-27T21:32:14.045 INFO:teuthology.orchestra.run.smithi006.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.15.0.79.86~20.04.39 [2,696 B] 2023-08-27T21:32:14.045 INFO:teuthology.orchestra.run.smithi006.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.2 [31.6 kB] 2023-08-27T21:32:14.046 INFO:teuthology.orchestra.run.smithi006.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2023-08-27T21:32:14.216 INFO:teuthology.orchestra.run.smithi006.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2023-08-27T21:32:14.432 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-79-generic. 2023-08-27T21:32:14.445 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../07-linux-modules-extra-5.15.0-79-generic_5.15.0-79.86~20.04.2_amd64.deb ... 2023-08-27T21:32:14.507 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking linux-modules-extra-5.15.0-79-generic (5.15.0-79.86~20.04.2) ... 2023-08-27T21:32:14.559 INFO:teuthology.orchestra.run.smithi006.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2023-08-27T21:32:15.273 INFO:teuthology.orchestra.run.smithi006.stdout:Fetched 90.9 MB in 4s (21.6 MB/s) 2023-08-27T21:32:15.487 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2023-08-27T21:32:17.182 INFO:teuthology.orchestra.run.smithi006.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 104373 files and directories currently installed.) 2023-08-27T21:32:17.187 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2023-08-27T21:32:17.360 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-08-27T21:32:17.847 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package libplist3:amd64. 2023-08-27T21:32:17.861 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2023-08-27T21:32:17.905 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2023-08-27T21:32:18.241 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2023-08-27T21:32:18.256 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2023-08-27T21:32:18.308 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2023-08-27T21:32:18.661 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2023-08-27T21:32:18.676 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2023-08-27T21:32:18.719 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-08-27T21:32:19.080 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package libupower-glib3:amd64. 2023-08-27T21:32:19.096 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2023-08-27T21:32:19.139 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2023-08-27T21:32:19.666 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package linux-modules-5.15.0-79-generic. 2023-08-27T21:32:19.681 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../05-linux-modules-5.15.0-79-generic_5.15.0-79.86~20.04.2_amd64.deb ... 2023-08-27T21:32:19.725 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking linux-modules-5.15.0-79-generic (5.15.0-79.86~20.04.2) ... 2023-08-27T21:32:21.483 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2023-08-27T21:32:21.500 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.15.0.79.86~20.04.39_amd64.deb ... 2023-08-27T21:32:21.541 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking linux-image-generic-hwe-20.04 (5.15.0.79.86~20.04.39) ... 2023-08-27T21:32:21.877 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package libevdev2:amd64. 2023-08-27T21:32:21.891 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.2_amd64.deb ... 2023-08-27T21:32:21.935 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-08-27T21:32:22.243 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package linux-image-5.15.0-79-generic. 2023-08-27T21:32:22.259 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../06-linux-image-5.15.0-79-generic_5.15.0-79.86~20.04.2_amd64.deb ... 2023-08-27T21:32:22.330 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package thermald. 2023-08-27T21:32:22.344 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2023-08-27T21:32:22.381 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking linux-image-5.15.0-79-generic (5.15.0-79.86~20.04.2) ... 2023-08-27T21:32:22.388 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2023-08-27T21:32:22.943 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package upower. 2023-08-27T21:32:22.957 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2023-08-27T21:32:22.987 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-79-generic. 2023-08-27T21:32:23.003 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../07-linux-modules-extra-5.15.0-79-generic_5.15.0-79.86~20.04.2_amd64.deb ... 2023-08-27T21:32:23.021 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking upower (0.99.11-1build2) ... 2023-08-27T21:32:23.046 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking linux-modules-extra-5.15.0-79-generic (5.15.0-79.86~20.04.2) ... 2023-08-27T21:32:23.422 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package usbmuxd. 2023-08-27T21:32:23.439 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2023-08-27T21:32:23.480 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-08-27T21:32:23.805 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2023-08-27T21:32:23.926 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-08-27T21:32:24.069 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2023-08-27T21:32:24.195 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2023-08-27T21:32:24.321 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-08-27T21:32:24.464 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-08-27T21:32:24.590 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up upower (0.99.11-1build2) ... 2023-08-27T21:32:25.146 INFO:teuthology.orchestra.run.smithi130.stdout:upower.service is a disabled or a static unit, not starting it. 2023-08-27T21:32:25.202 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-08-27T21:32:25.409 INFO:teuthology.orchestra.run.smithi130.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2023-08-27T21:32:25.419 INFO:teuthology.orchestra.run.smithi130.stdout:Adding system user `usbmux' (UID 114) ... 2023-08-27T21:32:25.419 INFO:teuthology.orchestra.run.smithi130.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2023-08-27T21:32:25.953 INFO:teuthology.orchestra.run.smithi130.stdout:Not creating home directory `/var/lib/usbmux'. 2023-08-27T21:32:26.174 INFO:teuthology.orchestra.run.smithi130.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2023-08-27T21:32:26.233 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2023-08-27T21:32:26.544 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2023-08-27T21:32:26.544 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2023-08-27T21:32:27.013 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up linux-image-5.15.0-79-generic (5.15.0-79.86~20.04.2) ... 2023-08-27T21:32:28.907 INFO:teuthology.orchestra.run.smithi130.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-79-generic 2023-08-27T21:32:28.907 INFO:teuthology.orchestra.run.smithi130.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-79-generic 2023-08-27T21:32:29.081 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up linux-modules-5.15.0-79-generic (5.15.0-79.86~20.04.2) ... 2023-08-27T21:32:30.450 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up linux-modules-extra-5.15.0-79-generic (5.15.0-79.86~20.04.2) ... 2023-08-27T21:32:30.706 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2023-08-27T21:32:30.721 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.15.0.79.86~20.04.39_amd64.deb ... 2023-08-27T21:32:30.764 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking linux-image-generic-hwe-20.04 (5.15.0.79.86~20.04.39) ... 2023-08-27T21:32:31.092 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package libevdev2:amd64. 2023-08-27T21:32:31.108 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.2_amd64.deb ... 2023-08-27T21:32:31.150 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-08-27T21:32:31.553 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package thermald. 2023-08-27T21:32:31.567 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2023-08-27T21:32:31.612 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2023-08-27T21:32:31.876 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up linux-image-generic-hwe-20.04 (5.15.0.79.86~20.04.39) ... 2023-08-27T21:32:32.001 INFO:teuthology.orchestra.run.smithi130.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-08-27T21:32:32.192 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package upower. 2023-08-27T21:32:32.206 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2023-08-27T21:32:32.268 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking upower (0.99.11-1build2) ... 2023-08-27T21:32:32.299 INFO:teuthology.orchestra.run.smithi130.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-08-27T21:32:32.696 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package usbmuxd. 2023-08-27T21:32:32.710 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2023-08-27T21:32:32.754 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-08-27T21:32:33.079 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2023-08-27T21:32:33.200 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-08-27T21:32:33.335 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2023-08-27T21:32:33.461 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2023-08-27T21:32:33.612 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-08-27T21:32:33.747 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-08-27T21:32:33.852 INFO:teuthology.orchestra.run.smithi130.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.2) ... 2023-08-27T21:32:33.873 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up upower (0.99.11-1build2) ... 2023-08-27T21:32:33.960 INFO:teuthology.orchestra.run.smithi130.stdout:Processing triggers for linux-image-5.15.0-79-generic (5.15.0-79.86~20.04.2) ... 2023-08-27T21:32:34.034 INFO:teuthology.orchestra.run.smithi130.stdout:/etc/kernel/postinst.d/initramfs-tools: 2023-08-27T21:32:34.034 INFO:teuthology.orchestra.run.smithi130.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-79-generic 2023-08-27T21:32:34.435 INFO:teuthology.orchestra.run.smithi006.stdout:upower.service is a disabled or a static unit, not starting it. 2023-08-27T21:32:34.493 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-08-27T21:32:34.683 INFO:teuthology.orchestra.run.smithi006.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2023-08-27T21:32:34.696 INFO:teuthology.orchestra.run.smithi006.stdout:Adding system user `usbmux' (UID 114) ... 2023-08-27T21:32:34.696 INFO:teuthology.orchestra.run.smithi006.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2023-08-27T21:32:35.194 INFO:teuthology.orchestra.run.smithi006.stdout:Not creating home directory `/var/lib/usbmux'. 2023-08-27T21:32:35.413 INFO:teuthology.orchestra.run.smithi006.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2023-08-27T21:32:35.450 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2023-08-27T21:32:35.736 INFO:teuthology.orchestra.run.smithi006.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2023-08-27T21:32:35.736 INFO:teuthology.orchestra.run.smithi006.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2023-08-27T21:32:36.179 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up linux-image-5.15.0-79-generic (5.15.0-79.86~20.04.2) ... 2023-08-27T21:32:37.543 INFO:teuthology.orchestra.run.smithi006.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-79-generic 2023-08-27T21:32:37.543 INFO:teuthology.orchestra.run.smithi006.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-79-generic 2023-08-27T21:32:37.715 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up linux-modules-5.15.0-79-generic (5.15.0-79.86~20.04.2) ... 2023-08-27T21:32:39.150 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up linux-modules-extra-5.15.0-79-generic (5.15.0-79.86~20.04.2) ... 2023-08-27T21:32:41.007 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up linux-image-generic-hwe-20.04 (5.15.0.79.86~20.04.39) ... 2023-08-27T21:32:41.878 INFO:teuthology.orchestra.run.smithi006.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-08-27T21:32:42.225 INFO:teuthology.orchestra.run.smithi006.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-08-27T21:32:43.144 INFO:teuthology.orchestra.run.smithi006.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.2) ... 2023-08-27T21:32:43.253 INFO:teuthology.orchestra.run.smithi006.stdout:Processing triggers for linux-image-5.15.0-79-generic (5.15.0-79.86~20.04.2) ... 2023-08-27T21:32:43.327 INFO:teuthology.orchestra.run.smithi006.stdout:/etc/kernel/postinst.d/initramfs-tools: 2023-08-27T21:32:43.327 INFO:teuthology.orchestra.run.smithi006.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-79-generic 2023-08-27T21:32:59.422 INFO:teuthology.orchestra.run.smithi130.stdout:/etc/kernel/postinst.d/zz-update-grub: 2023-08-27T21:32:59.422 INFO:teuthology.orchestra.run.smithi130.stdout:Sourcing file `/etc/default/grub' 2023-08-27T21:32:59.435 INFO:teuthology.orchestra.run.smithi130.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-08-27T21:32:59.460 INFO:teuthology.orchestra.run.smithi130.stdout:Generating grub configuration file ... 2023-08-27T21:32:59.785 INFO:teuthology.orchestra.run.smithi130.stdout:Found linux image: /boot/vmlinuz-5.15.0-79-generic 2023-08-27T21:32:59.809 INFO:teuthology.orchestra.run.smithi130.stdout:Found initrd image: /boot/initrd.img-5.15.0-79-generic 2023-08-27T21:33:00.228 INFO:teuthology.orchestra.run.smithi130.stdout:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-08-27T21:33:00.242 INFO:teuthology.orchestra.run.smithi130.stdout:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-08-27T21:33:02.311 INFO:teuthology.orchestra.run.smithi130.stdout:done 2023-08-27T21:33:02.727 DEBUG:teuthology.orchestra.run.smithi130:> dpkg -s linux-image-generic-hwe-20.04 2023-08-27T21:33:02.782 INFO:teuthology.orchestra.run.smithi130.stdout:Package: linux-image-generic-hwe-20.04 2023-08-27T21:33:02.782 INFO:teuthology.orchestra.run.smithi130.stdout:Status: install ok installed 2023-08-27T21:33:02.782 INFO:teuthology.orchestra.run.smithi130.stdout:Priority: optional 2023-08-27T21:33:02.782 INFO:teuthology.orchestra.run.smithi130.stdout:Section: kernel 2023-08-27T21:33:02.782 INFO:teuthology.orchestra.run.smithi130.stdout:Installed-Size: 20 2023-08-27T21:33:02.782 INFO:teuthology.orchestra.run.smithi130.stdout:Maintainer: Ubuntu Kernel Team 2023-08-27T21:33:02.783 INFO:teuthology.orchestra.run.smithi130.stdout:Architecture: amd64 2023-08-27T21:33:02.783 INFO:teuthology.orchestra.run.smithi130.stdout:Source: linux-meta-hwe-5.15 2023-08-27T21:33:02.783 INFO:teuthology.orchestra.run.smithi130.stdout:Version: 5.15.0.79.86~20.04.39 2023-08-27T21:33:02.783 INFO:teuthology.orchestra.run.smithi130.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-79), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-08-27T21:33:02.783 INFO:teuthology.orchestra.run.smithi130.stdout:Depends: linux-image-5.15.0-79-generic, linux-modules-extra-5.15.0-79-generic, linux-firmware, intel-microcode, amd64-microcode 2023-08-27T21:33:02.783 INFO:teuthology.orchestra.run.smithi130.stdout:Recommends: thermald 2023-08-27T21:33:02.783 INFO:teuthology.orchestra.run.smithi130.stdout:Description: Generic Linux kernel image 2023-08-27T21:33:02.783 INFO:teuthology.orchestra.run.smithi130.stdout: This package will always depend on the latest generic kernel image 2023-08-27T21:33:02.783 INFO:teuthology.orchestra.run.smithi130.stdout: available. 2023-08-27T21:33:02.783 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-124-generic Expected: 5.15.0-79-generic 2023-08-27T21:33:02.784 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-08-27T21:33:02.784 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-08-27T21:33:02.784 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2023-08-27T21:33:02.784 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi130.front.sepia.ceph.com, path=None, version=distro) 2023-08-27T21:33:02.784 DEBUG:teuthology.orchestra.run.smithi130:> sudo apt-get clean 2023-08-27T21:33:02.867 DEBUG:teuthology.orchestra.run.smithi130:> sudo apt-get update 2023-08-27T21:33:03.018 INFO:teuthology.orchestra.run.smithi130.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-08-27T21:33:03.018 INFO:teuthology.orchestra.run.smithi130.stdout:Hit:2 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-08-27T21:33:03.024 INFO:teuthology.orchestra.run.smithi130.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-08-27T21:33:03.042 INFO:teuthology.orchestra.run.smithi130.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-08-27T21:33:04.727 INFO:teuthology.orchestra.run.smithi130.stdout:Reading package lists... 2023-08-27T21:33:04.754 DEBUG:teuthology.orchestra.run.smithi130:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-08-27T21:33:04.821 INFO:teuthology.orchestra.run.smithi130.stdout:Reading package lists... 2023-08-27T21:33:05.019 INFO:teuthology.orchestra.run.smithi130.stdout:Building dependency tree... 2023-08-27T21:33:05.019 INFO:teuthology.orchestra.run.smithi130.stdout:Reading state information... 2023-08-27T21:33:05.234 INFO:teuthology.orchestra.run.smithi130.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.15.0.79.86~20.04.39). 2023-08-27T21:33:05.234 INFO:teuthology.orchestra.run.smithi130.stdout:0 upgraded, 0 newly installed, 0 to remove and 261 not upgraded. 2023-08-27T21:33:05.236 DEBUG:teuthology.orchestra.run.smithi130:> dpkg -s linux-image-generic-hwe-20.04 2023-08-27T21:33:05.255 INFO:teuthology.orchestra.run.smithi130.stdout:Package: linux-image-generic-hwe-20.04 2023-08-27T21:33:05.255 INFO:teuthology.orchestra.run.smithi130.stdout:Status: install ok installed 2023-08-27T21:33:05.256 INFO:teuthology.orchestra.run.smithi130.stdout:Priority: optional 2023-08-27T21:33:05.256 INFO:teuthology.orchestra.run.smithi130.stdout:Section: kernel 2023-08-27T21:33:05.256 INFO:teuthology.orchestra.run.smithi130.stdout:Installed-Size: 20 2023-08-27T21:33:05.256 INFO:teuthology.orchestra.run.smithi130.stdout:Maintainer: Ubuntu Kernel Team 2023-08-27T21:33:05.256 INFO:teuthology.orchestra.run.smithi130.stdout:Architecture: amd64 2023-08-27T21:33:05.256 INFO:teuthology.orchestra.run.smithi130.stdout:Source: linux-meta-hwe-5.15 2023-08-27T21:33:05.256 INFO:teuthology.orchestra.run.smithi130.stdout:Version: 5.15.0.79.86~20.04.39 2023-08-27T21:33:05.256 INFO:teuthology.orchestra.run.smithi130.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-79), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-08-27T21:33:05.256 INFO:teuthology.orchestra.run.smithi130.stdout:Depends: linux-image-5.15.0-79-generic, linux-modules-extra-5.15.0-79-generic, linux-firmware, intel-microcode, amd64-microcode 2023-08-27T21:33:05.256 INFO:teuthology.orchestra.run.smithi130.stdout:Recommends: thermald 2023-08-27T21:33:05.256 INFO:teuthology.orchestra.run.smithi130.stdout:Description: Generic Linux kernel image 2023-08-27T21:33:05.256 INFO:teuthology.orchestra.run.smithi130.stdout: This package will always depend on the latest generic kernel image 2023-08-27T21:33:05.257 INFO:teuthology.orchestra.run.smithi130.stdout: available. 2023-08-27T21:33:05.257 DEBUG:teuthology.orchestra.run.smithi130:> mktemp 2023-08-27T21:33:05.261 INFO:teuthology.orchestra.run.smithi130.stdout:/tmp/tmp.t7zc6hesVx 2023-08-27T21:33:05.261 DEBUG:teuthology.orchestra.run.smithi130:> sudo cp /boot/grub/grub.cfg /tmp/tmp.t7zc6hesVx 2023-08-27T21:33:05.274 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod 0666 /tmp/tmp.t7zc6hesVx 2023-08-27T21:33:05.311 DEBUG:teuthology.orchestra.remote:smithi130:/tmp/tmp.t7zc6hesVx is 9KB 2023-08-27T21:33:05.325 DEBUG:teuthology.orchestra.run.smithi130:> rm -fr /tmp/tmp.t7zc6hesVx 2023-08-27T21:33:05.330 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2023-08-27T21:33:05.386 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-08-27T21:33:05.386 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/grub.d/01_ceph_kernel 2023-08-27T21:33:05.386 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2023-08-27T21:33:05.417 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-79-generic 2023-08-27T21:33:05.417 DEBUG:teuthology.orchestra.run.smithi130:> sudo update-grub 2023-08-27T21:33:06.725 INFO:teuthology.orchestra.run.smithi130.stderr:Sourcing file `/etc/default/grub' 2023-08-27T21:33:06.731 INFO:teuthology.orchestra.run.smithi130.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-08-27T21:33:06.745 INFO:teuthology.orchestra.run.smithi130.stderr:Generating grub configuration file ... 2023-08-27T21:33:06.990 INFO:teuthology.orchestra.run.smithi130.stderr:Found linux image: /boot/vmlinuz-5.15.0-79-generic 2023-08-27T21:33:07.001 INFO:teuthology.orchestra.run.smithi130.stderr:Found initrd image: /boot/initrd.img-5.15.0-79-generic 2023-08-27T21:33:07.370 INFO:teuthology.orchestra.run.smithi130.stderr:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-08-27T21:33:07.381 INFO:teuthology.orchestra.run.smithi130.stderr:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-08-27T21:33:08.431 INFO:teuthology.orchestra.run.smithi130.stderr:done 2023-08-27T21:33:08.433 DEBUG:teuthology.orchestra.run.smithi130:> sudo shutdown -r now 2023-08-27T21:33:09.534 INFO:teuthology.orchestra.run.smithi006.stdout:/etc/kernel/postinst.d/zz-update-grub: 2023-08-27T21:33:09.534 INFO:teuthology.orchestra.run.smithi006.stdout:Sourcing file `/etc/default/grub' 2023-08-27T21:33:09.543 INFO:teuthology.orchestra.run.smithi006.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-08-27T21:33:09.573 INFO:teuthology.orchestra.run.smithi006.stdout:Generating grub configuration file ... 2023-08-27T21:33:09.977 INFO:teuthology.orchestra.run.smithi006.stdout:Found linux image: /boot/vmlinuz-5.15.0-79-generic 2023-08-27T21:33:09.998 INFO:teuthology.orchestra.run.smithi006.stdout:Found initrd image: /boot/initrd.img-5.15.0-79-generic 2023-08-27T21:33:10.434 INFO:teuthology.orchestra.run.smithi006.stdout:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-08-27T21:33:10.445 INFO:teuthology.orchestra.run.smithi006.stdout:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-08-27T21:33:12.480 INFO:teuthology.orchestra.run.smithi006.stdout:done 2023-08-27T21:33:12.973 DEBUG:teuthology.orchestra.run.smithi006:> dpkg -s linux-image-generic-hwe-20.04 2023-08-27T21:33:13.015 INFO:teuthology.orchestra.run.smithi006.stdout:Package: linux-image-generic-hwe-20.04 2023-08-27T21:33:13.016 INFO:teuthology.orchestra.run.smithi006.stdout:Status: install ok installed 2023-08-27T21:33:13.016 INFO:teuthology.orchestra.run.smithi006.stdout:Priority: optional 2023-08-27T21:33:13.016 INFO:teuthology.orchestra.run.smithi006.stdout:Section: kernel 2023-08-27T21:33:13.016 INFO:teuthology.orchestra.run.smithi006.stdout:Installed-Size: 20 2023-08-27T21:33:13.016 INFO:teuthology.orchestra.run.smithi006.stdout:Maintainer: Ubuntu Kernel Team 2023-08-27T21:33:13.016 INFO:teuthology.orchestra.run.smithi006.stdout:Architecture: amd64 2023-08-27T21:33:13.016 INFO:teuthology.orchestra.run.smithi006.stdout:Source: linux-meta-hwe-5.15 2023-08-27T21:33:13.016 INFO:teuthology.orchestra.run.smithi006.stdout:Version: 5.15.0.79.86~20.04.39 2023-08-27T21:33:13.017 INFO:teuthology.orchestra.run.smithi006.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-79), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-08-27T21:33:13.017 INFO:teuthology.orchestra.run.smithi006.stdout:Depends: linux-image-5.15.0-79-generic, linux-modules-extra-5.15.0-79-generic, linux-firmware, intel-microcode, amd64-microcode 2023-08-27T21:33:13.017 INFO:teuthology.orchestra.run.smithi006.stdout:Recommends: thermald 2023-08-27T21:33:13.017 INFO:teuthology.orchestra.run.smithi006.stdout:Description: Generic Linux kernel image 2023-08-27T21:33:13.017 INFO:teuthology.orchestra.run.smithi006.stdout: This package will always depend on the latest generic kernel image 2023-08-27T21:33:13.017 INFO:teuthology.orchestra.run.smithi006.stdout: available. 2023-08-27T21:33:13.017 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-124-generic Expected: 5.15.0-79-generic 2023-08-27T21:33:13.017 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-08-27T21:33:13.017 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-08-27T21:33:13.018 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2023-08-27T21:33:13.018 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi006.front.sepia.ceph.com, path=None, version=distro) 2023-08-27T21:33:13.018 DEBUG:teuthology.orchestra.run.smithi006:> sudo apt-get clean 2023-08-27T21:33:13.142 DEBUG:teuthology.orchestra.run.smithi006:> sudo apt-get update 2023-08-27T21:33:13.275 INFO:teuthology.orchestra.run.smithi006.stdout:Hit:1 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-08-27T21:33:13.392 INFO:teuthology.orchestra.run.smithi006.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2023-08-27T21:33:13.474 INFO:teuthology.orchestra.run.smithi006.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-08-27T21:33:13.568 INFO:teuthology.orchestra.run.smithi006.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-08-27T21:33:15.063 INFO:teuthology.orchestra.run.smithi006.stdout:Reading package lists... 2023-08-27T21:33:15.088 DEBUG:teuthology.orchestra.run.smithi006:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-08-27T21:33:15.161 INFO:teuthology.orchestra.run.smithi006.stdout:Reading package lists... 2023-08-27T21:33:15.344 INFO:teuthology.orchestra.run.smithi006.stdout:Building dependency tree... 2023-08-27T21:33:15.344 INFO:teuthology.orchestra.run.smithi006.stdout:Reading state information... 2023-08-27T21:33:15.561 INFO:teuthology.orchestra.run.smithi006.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.15.0.79.86~20.04.39). 2023-08-27T21:33:15.562 INFO:teuthology.orchestra.run.smithi006.stdout:0 upgraded, 0 newly installed, 0 to remove and 261 not upgraded. 2023-08-27T21:33:15.563 DEBUG:teuthology.orchestra.run.smithi006:> dpkg -s linux-image-generic-hwe-20.04 2023-08-27T21:33:15.584 INFO:teuthology.orchestra.run.smithi006.stdout:Package: linux-image-generic-hwe-20.04 2023-08-27T21:33:15.585 INFO:teuthology.orchestra.run.smithi006.stdout:Status: install ok installed 2023-08-27T21:33:15.585 INFO:teuthology.orchestra.run.smithi006.stdout:Priority: optional 2023-08-27T21:33:15.585 INFO:teuthology.orchestra.run.smithi006.stdout:Section: kernel 2023-08-27T21:33:15.585 INFO:teuthology.orchestra.run.smithi006.stdout:Installed-Size: 20 2023-08-27T21:33:15.585 INFO:teuthology.orchestra.run.smithi006.stdout:Maintainer: Ubuntu Kernel Team 2023-08-27T21:33:15.585 INFO:teuthology.orchestra.run.smithi006.stdout:Architecture: amd64 2023-08-27T21:33:15.585 INFO:teuthology.orchestra.run.smithi006.stdout:Source: linux-meta-hwe-5.15 2023-08-27T21:33:15.585 INFO:teuthology.orchestra.run.smithi006.stdout:Version: 5.15.0.79.86~20.04.39 2023-08-27T21:33:15.585 INFO:teuthology.orchestra.run.smithi006.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-79), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-08-27T21:33:15.585 INFO:teuthology.orchestra.run.smithi006.stdout:Depends: linux-image-5.15.0-79-generic, linux-modules-extra-5.15.0-79-generic, linux-firmware, intel-microcode, amd64-microcode 2023-08-27T21:33:15.585 INFO:teuthology.orchestra.run.smithi006.stdout:Recommends: thermald 2023-08-27T21:33:15.586 INFO:teuthology.orchestra.run.smithi006.stdout:Description: Generic Linux kernel image 2023-08-27T21:33:15.586 INFO:teuthology.orchestra.run.smithi006.stdout: This package will always depend on the latest generic kernel image 2023-08-27T21:33:15.586 INFO:teuthology.orchestra.run.smithi006.stdout: available. 2023-08-27T21:33:15.587 DEBUG:teuthology.orchestra.run.smithi006:> mktemp 2023-08-27T21:33:15.636 INFO:teuthology.orchestra.run.smithi006.stdout:/tmp/tmp.wMmv23NPCl 2023-08-27T21:33:15.636 DEBUG:teuthology.orchestra.run.smithi006:> sudo cp /boot/grub/grub.cfg /tmp/tmp.wMmv23NPCl 2023-08-27T21:33:15.649 DEBUG:teuthology.orchestra.run.smithi006:> sudo chmod 0666 /tmp/tmp.wMmv23NPCl 2023-08-27T21:33:15.730 DEBUG:teuthology.orchestra.remote:smithi006:/tmp/tmp.wMmv23NPCl is 9KB 2023-08-27T21:33:15.744 DEBUG:teuthology.orchestra.run.smithi006:> rm -fr /tmp/tmp.wMmv23NPCl 2023-08-27T21:33:15.750 DEBUG:teuthology.orchestra.run.smithi006:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2023-08-27T21:33:15.764 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2023-08-27T21:33:15.764 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd of=/etc/grub.d/01_ceph_kernel 2023-08-27T21:33:15.764 DEBUG:teuthology.orchestra.run.smithi006:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2023-08-27T21:33:15.804 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-79-generic 2023-08-27T21:33:15.804 DEBUG:teuthology.orchestra.run.smithi006:> sudo update-grub 2023-08-27T21:33:17.096 INFO:teuthology.orchestra.run.smithi006.stderr:Sourcing file `/etc/default/grub' 2023-08-27T21:33:17.101 INFO:teuthology.orchestra.run.smithi006.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-08-27T21:33:17.113 INFO:teuthology.orchestra.run.smithi006.stderr:Generating grub configuration file ... 2023-08-27T21:33:17.426 INFO:teuthology.orchestra.run.smithi006.stderr:Found linux image: /boot/vmlinuz-5.15.0-79-generic 2023-08-27T21:33:17.438 INFO:teuthology.orchestra.run.smithi006.stderr:Found initrd image: /boot/initrd.img-5.15.0-79-generic 2023-08-27T21:33:17.892 INFO:teuthology.orchestra.run.smithi006.stderr:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-08-27T21:33:17.904 INFO:teuthology.orchestra.run.smithi006.stderr:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-08-27T21:33:18.942 INFO:teuthology.orchestra.run.smithi006.stderr:done 2023-08-27T21:33:18.944 DEBUG:teuthology.orchestra.run.smithi006:> sudo shutdown -r now 2023-08-27T21:33:38.455 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2023-08-27T21:33:38.455 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi130.front.sepia.ceph.com' 2023-08-27T21:33:38.456 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T21:33:48.955 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2023-08-27T21:33:48.955 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi006.front.sepia.ceph.com' 2023-08-27T21:33:48.955 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi006.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T21:33:56.975 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.130 2023-08-27T21:34:05.983 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi130.front.sepia.ceph.com' 2023-08-27T21:34:05.983 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T21:34:07.467 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.6 2023-08-27T21:34:09.035 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.130 2023-08-27T21:34:16.475 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi006.front.sepia.ceph.com' 2023-08-27T21:34:16.475 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi006.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T21:34:21.040 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi130.front.sepia.ceph.com' 2023-08-27T21:34:21.041 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T21:34:24.107 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.130 2023-08-27T21:34:39.112 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi130.front.sepia.ceph.com' 2023-08-27T21:34:39.113 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T21:34:42.187 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.130 2023-08-27T21:35:00.195 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi130.front.sepia.ceph.com' 2023-08-27T21:35:00.196 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T21:35:00.421 DEBUG:teuthology.orchestra.run.smithi130:> true 2023-08-27T21:35:00.933 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi130.front.sepia.ceph.com' 2023-08-27T21:35:00.934 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.15.0-79-generic"... 2023-08-27T21:35:00.934 DEBUG:teuthology.orchestra.run.smithi130:> uname -r 2023-08-27T21:35:00.980 INFO:teuthology.orchestra.run.smithi130.stdout:5.15.0-79-generic 2023-08-27T21:35:00.981 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-79-generic vs 5.15.0-79-generic 2023-08-27T21:35:00.981 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-08-27T21:35:00.981 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2023-08-27T21:35:01.982 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-08-27T21:35:01.983 DEBUG:teuthology.orchestra.run.smithi130:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-08-27T21:35:02.063 INFO:teuthology.orchestra.run.smithi130.stdout:ttyS1 2023-08-27T21:35:02.093 DEBUG:teuthology.parallel:result is None 2023-08-27T21:35:16.477 DEBUG:teuthology.orchestra.remote:timed out 2023-08-27T21:35:28.478 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi006.front.sepia.ceph.com' 2023-08-27T21:35:28.478 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi006.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T21:35:28.719 DEBUG:teuthology.orchestra.run.smithi006:> true 2023-08-27T21:35:29.318 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi006.front.sepia.ceph.com' 2023-08-27T21:35:29.318 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.15.0-79-generic"... 2023-08-27T21:35:29.319 DEBUG:teuthology.orchestra.run.smithi006:> uname -r 2023-08-27T21:35:29.367 INFO:teuthology.orchestra.run.smithi006.stdout:5.15.0-79-generic 2023-08-27T21:35:29.368 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-79-generic vs 5.15.0-79-generic 2023-08-27T21:35:29.368 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-08-27T21:35:29.368 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2023-08-27T21:35:30.369 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-08-27T21:35:30.369 DEBUG:teuthology.orchestra.run.smithi006:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-08-27T21:35:30.444 INFO:teuthology.orchestra.run.smithi006.stdout:ttyS1 2023-08-27T21:35:30.460 DEBUG:teuthology.parallel:result is None 2023-08-27T21:35:30.460 INFO:teuthology.run_tasks:Running task internal.base... 2023-08-27T21:35:30.468 INFO:teuthology.task.internal:Creating test directory... 2023-08-27T21:35:30.468 DEBUG:teuthology.orchestra.run.smithi006:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-08-27T21:35:30.470 DEBUG:teuthology.orchestra.run.smithi130:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-08-27T21:35:30.478 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-08-27T21:35:30.486 INFO:teuthology.run_tasks:Running task internal.archive... 2023-08-27T21:35:30.498 INFO:teuthology.task.internal:Creating archive directory... 2023-08-27T21:35:30.498 DEBUG:teuthology.orchestra.run.smithi006:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-08-27T21:35:30.519 DEBUG:teuthology.orchestra.run.smithi130:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-08-27T21:35:30.591 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-08-27T21:35:30.598 INFO:teuthology.task.internal:Enabling coredump saving... 2023-08-27T21:35:30.598 DEBUG:teuthology.orchestra.run.smithi006:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-08-27T21:35:30.600 DEBUG:teuthology.orchestra.run.smithi130:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-08-27T21:35:30.615 INFO:teuthology.orchestra.run.smithi006.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-08-27T21:35:30.619 INFO:teuthology.orchestra.run.smithi130.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-08-27T21:35:30.621 INFO:teuthology.orchestra.run.smithi006.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-08-27T21:35:30.626 INFO:teuthology.orchestra.run.smithi130.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-08-27T21:35:30.628 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-08-27T21:35:30.636 INFO:teuthology.task.internal:Configuring sudo... 2023-08-27T21:35:30.637 DEBUG:teuthology.orchestra.run.smithi006:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-08-27T21:35:30.671 DEBUG:teuthology.orchestra.run.smithi130:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-08-27T21:35:30.687 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-08-27T21:35:30.697 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-08-27T21:35:30.697 DEBUG:teuthology.orchestra.run.smithi006:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-08-27T21:35:30.731 DEBUG:teuthology.orchestra.run.smithi130:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-08-27T21:35:30.739 DEBUG:teuthology.orchestra.run.smithi006:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-08-27T21:35:30.784 DEBUG:teuthology.orchestra.run.smithi006:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-08-27T21:35:30.833 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2023-08-27T21:35:30.833 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-08-27T21:35:30.902 DEBUG:teuthology.orchestra.run.smithi130:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-08-27T21:35:30.909 DEBUG:teuthology.orchestra.run.smithi130:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-08-27T21:35:30.958 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-08-27T21:35:30.958 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-08-27T21:35:31.019 DEBUG:teuthology.orchestra.run.smithi006:> sudo service rsyslog restart 2023-08-27T21:35:31.021 DEBUG:teuthology.orchestra.run.smithi130:> sudo service rsyslog restart 2023-08-27T21:35:31.110 INFO:teuthology.run_tasks:Running task internal.timer... 2023-08-27T21:35:31.118 INFO:teuthology.task.internal:Starting timer... 2023-08-27T21:35:31.118 INFO:teuthology.run_tasks:Running task pcp... 2023-08-27T21:35:31.130 INFO:teuthology.run_tasks:Running task selinux... 2023-08-27T21:35:31.142 DEBUG:teuthology.task.selinux:Excluding smithi006: OS 'ubuntu' does not support SELinux 2023-08-27T21:35:31.142 DEBUG:teuthology.task.selinux:Excluding smithi130: OS 'ubuntu' does not support SELinux 2023-08-27T21:35:31.142 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-08-27T21:35:31.142 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2023-08-27T21:35:31.142 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-08-27T21:35:31.142 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-08-27T21:35:31.154 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-08-27T21:35:31.158 INFO:teuthology.repo_utils:Fetching main from origin 2023-08-27T21:35:31.289 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-08-27T21:35:31.304 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-08-27T21:35:31.305 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi006.front.sepia.ceph.com,smithi130.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-08-27T21:41:10.159 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi006.front.sepia.ceph.com'), Remote(name='ubuntu@smithi130.front.sepia.ceph.com')] 2023-08-27T21:41:10.159 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi006.front.sepia.ceph.com' 2023-08-27T21:41:10.160 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi006.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T21:41:10.238 DEBUG:teuthology.orchestra.run.smithi006:> true 2023-08-27T21:41:10.360 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi006.front.sepia.ceph.com' 2023-08-27T21:41:10.361 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi130.front.sepia.ceph.com' 2023-08-27T21:41:10.361 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T21:41:10.441 DEBUG:teuthology.orchestra.run.smithi130:> true 2023-08-27T21:41:10.560 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi130.front.sepia.ceph.com' 2023-08-27T21:41:10.561 INFO:teuthology.run_tasks:Running task clock... 2023-08-27T21:41:10.570 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-08-27T21:41:10.570 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-08-27T21:41:10.571 DEBUG:teuthology.orchestra.run.smithi006:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-08-27T21:41:10.573 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-08-27T21:41:10.573 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-08-27T21:41:10.602 INFO:teuthology.orchestra.run.smithi006.stdout:27 Aug 21:41:10 ntpd[22469]: ntpd 4.2.8p12@1.3728-o (1): Starting 2023-08-27T21:41:10.603 INFO:teuthology.orchestra.run.smithi006.stdout:27 Aug 21:41:10 ntpd[22469]: Command line: ntpd -gq 2023-08-27T21:41:10.603 INFO:teuthology.orchestra.run.smithi006.stdout:27 Aug 21:41:10 ntpd[22469]: proto: precision = 0.098 usec (-23) 2023-08-27T21:41:10.603 INFO:teuthology.orchestra.run.smithi006.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-08-27T21:41:10.604 INFO:teuthology.orchestra.run.smithi006.stdout:27 Aug 21:41:10 ntpd[22469]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-08-27T21:41:10.604 INFO:teuthology.orchestra.run.smithi006.stderr:restrict ::: KOD does nothing without LIMITED. 2023-08-27T21:41:10.605 INFO:teuthology.orchestra.run.smithi006.stdout:27 Aug 21:41:10 ntpd[22469]: restrict ::: KOD does nothing without LIMITED. 2023-08-27T21:41:10.605 INFO:teuthology.orchestra.run.smithi006.stdout:27 Aug 21:41:10 ntpd[22469]: Listen and drop on 0 v6wildcard [::]:123 2023-08-27T21:41:10.605 INFO:teuthology.orchestra.run.smithi006.stdout:27 Aug 21:41:10 ntpd[22469]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2023-08-27T21:41:10.605 INFO:teuthology.orchestra.run.smithi006.stdout:27 Aug 21:41:10 ntpd[22469]: Listen normally on 2 lo 127.0.0.1:123 2023-08-27T21:41:10.605 INFO:teuthology.orchestra.run.smithi006.stdout:27 Aug 21:41:10 ntpd[22469]: Listen normally on 3 ens1f0 172.21.15.6:123 2023-08-27T21:41:10.605 INFO:teuthology.orchestra.run.smithi006.stdout:27 Aug 21:41:10 ntpd[22469]: Listen normally on 4 lo [::1]:123 2023-08-27T21:41:10.605 INFO:teuthology.orchestra.run.smithi006.stdout:27 Aug 21:41:10 ntpd[22469]: Listen normally on 5 ens1f0 [fe80::ec4:7aff:febd:15e4%4]:123 2023-08-27T21:41:10.606 INFO:teuthology.orchestra.run.smithi006.stdout:27 Aug 21:41:10 ntpd[22469]: Listening on routing socket on fd #22 for interface updates 2023-08-27T21:41:10.636 INFO:teuthology.orchestra.run.smithi130.stdout:27 Aug 21:41:10 ntpd[22507]: ntpd 4.2.8p12@1.3728-o (1): Starting 2023-08-27T21:41:10.637 INFO:teuthology.orchestra.run.smithi130.stdout:27 Aug 21:41:10 ntpd[22507]: Command line: ntpd -gq 2023-08-27T21:41:10.637 INFO:teuthology.orchestra.run.smithi130.stdout:27 Aug 21:41:10 ntpd[22507]: proto: precision = 0.078 usec (-24) 2023-08-27T21:41:10.638 INFO:teuthology.orchestra.run.smithi130.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-08-27T21:41:10.638 INFO:teuthology.orchestra.run.smithi130.stdout:27 Aug 21:41:10 ntpd[22507]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-08-27T21:41:10.639 INFO:teuthology.orchestra.run.smithi130.stderr:restrict ::: KOD does nothing without LIMITED. 2023-08-27T21:41:10.640 INFO:teuthology.orchestra.run.smithi130.stdout:27 Aug 21:41:10 ntpd[22507]: restrict ::: KOD does nothing without LIMITED. 2023-08-27T21:41:10.640 INFO:teuthology.orchestra.run.smithi130.stdout:27 Aug 21:41:10 ntpd[22507]: Listen and drop on 0 v6wildcard [::]:123 2023-08-27T21:41:10.640 INFO:teuthology.orchestra.run.smithi130.stdout:27 Aug 21:41:10 ntpd[22507]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2023-08-27T21:41:10.641 INFO:teuthology.orchestra.run.smithi130.stdout:27 Aug 21:41:10 ntpd[22507]: Listen normally on 2 lo 127.0.0.1:123 2023-08-27T21:41:10.641 INFO:teuthology.orchestra.run.smithi130.stdout:27 Aug 21:41:10 ntpd[22507]: Listen normally on 3 enp3s0f1 172.21.15.130:123 2023-08-27T21:41:10.641 INFO:teuthology.orchestra.run.smithi130.stdout:27 Aug 21:41:10 ntpd[22507]: Listen normally on 4 lo [::1]:123 2023-08-27T21:41:10.641 INFO:teuthology.orchestra.run.smithi130.stdout:27 Aug 21:41:10 ntpd[22507]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:7f37%5]:123 2023-08-27T21:41:10.641 INFO:teuthology.orchestra.run.smithi130.stdout:27 Aug 21:41:10 ntpd[22507]: Listening on routing socket on fd #22 for interface updates 2023-08-27T21:41:11.603 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:11 ntpd[22469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:11.603 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:11 ntpd[22469]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:11.603 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:11 ntpd[22469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:11.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:11 ntpd[22469]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:11.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:11 ntpd[22469]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-08-27T21:41:11.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:11 ntpd[22469]: can't open /var/log/ntpstats/peerstats.20230827: Permission denied 2023-08-27T21:41:11.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:11 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:11.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:11 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:11.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:11 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:11.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:11 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:11.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:11 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:11.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:11 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:11.639 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:11 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:11.639 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:11 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:11.639 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:11 ntpd[22507]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-08-27T21:41:11.639 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:11 ntpd[22507]: can't open /var/log/ntpstats/peerstats.20230827: Permission denied 2023-08-27T21:41:11.639 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:11 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:11.639 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:11 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:11.639 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:11 ntpd[22507]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-08-27T21:41:11.639 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:11 ntpd[22507]: can't open /var/log/ntpstats/peerstats.20230827: Permission denied 2023-08-27T21:41:12.603 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:12 ntpd[22469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:12.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:12 ntpd[22469]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:12.605 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:12 ntpd[22469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:12.605 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:12 ntpd[22469]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:12.605 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:12 ntpd[22469]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-08-27T21:41:12.605 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:12 ntpd[22469]: can't open /var/log/ntpstats/peerstats.20230827: Permission denied 2023-08-27T21:41:13.603 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:13 ntpd[22469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:13.603 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:13 ntpd[22469]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:13.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:13 ntpd[22469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:13.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:13 ntpd[22469]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:13.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:13 ntpd[22469]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-08-27T21:41:13.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:13 ntpd[22469]: can't open /var/log/ntpstats/peerstats.20230827: Permission denied 2023-08-27T21:41:13.637 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:13 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:13.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:13 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:13.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:13 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:13.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:13 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:13.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:13 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:13.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:13 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:13.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:13 ntpd[22507]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-08-27T21:41:13.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:13 ntpd[22507]: can't open /var/log/ntpstats/peerstats.20230827: Permission denied 2023-08-27T21:41:13.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:13 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:13.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:13 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:13.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:13 ntpd[22507]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-08-27T21:41:13.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:13 ntpd[22507]: can't open /var/log/ntpstats/peerstats.20230827: Permission denied 2023-08-27T21:41:14.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:14 ntpd[22469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:14.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:14 ntpd[22469]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:14.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:14 ntpd[22469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:14.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:14 ntpd[22469]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:14.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:14 ntpd[22469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:14.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:14 ntpd[22469]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:14.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:14 ntpd[22469]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-08-27T21:41:14.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:14 ntpd[22469]: can't open /var/log/ntpstats/peerstats.20230827: Permission denied 2023-08-27T21:41:14.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:14 ntpd[22469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:14.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:14 ntpd[22469]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:14.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:14 ntpd[22469]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-08-27T21:41:14.605 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:14 ntpd[22469]: can't open /var/log/ntpstats/peerstats.20230827: Permission denied 2023-08-27T21:41:14.637 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:14 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:14.637 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:14 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:14.637 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:14 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:14.637 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:14 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:14.637 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:14 ntpd[22507]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-08-27T21:41:14.637 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:14 ntpd[22507]: can't open /var/log/ntpstats/peerstats.20230827: Permission denied 2023-08-27T21:41:15.603 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:15 ntpd[22469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:15.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:15 ntpd[22469]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:15.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:15 ntpd[22469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:15.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:15 ntpd[22469]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:15.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:15 ntpd[22469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:15.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:15 ntpd[22469]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:15.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:15 ntpd[22469]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-08-27T21:41:15.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:15 ntpd[22469]: can't open /var/log/ntpstats/peerstats.20230827: Permission denied 2023-08-27T21:41:15.637 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:15 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:15.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:15 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:15.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:15 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:15.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:15 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:15.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:15 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:15.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:15 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:15.639 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:15 ntpd[22507]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-08-27T21:41:15.639 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:15 ntpd[22507]: can't open /var/log/ntpstats/peerstats.20230827: Permission denied 2023-08-27T21:41:15.639 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:15 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:15.639 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:15 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:15.639 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:15 ntpd[22507]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-08-27T21:41:15.639 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:15 ntpd[22507]: can't open /var/log/ntpstats/peerstats.20230827: Permission denied 2023-08-27T21:41:16.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:16 ntpd[22469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:16.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:16 ntpd[22469]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:16.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:16 ntpd[22469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:16.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:16 ntpd[22469]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:16.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:16 ntpd[22469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:16.605 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:16 ntpd[22469]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:16.605 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:16 ntpd[22469]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-08-27T21:41:16.605 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:16 ntpd[22469]: can't open /var/log/ntpstats/peerstats.20230827: Permission denied 2023-08-27T21:41:16.605 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:16 ntpd[22469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:16.605 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:16 ntpd[22469]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:16.605 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:16 ntpd[22469]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-08-27T21:41:16.605 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:16 ntpd[22469]: can't open /var/log/ntpstats/peerstats.20230827: Permission denied 2023-08-27T21:41:16.637 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:16 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:16.637 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:16 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:16.637 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:16 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:16.637 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:16 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:16.637 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:16 ntpd[22507]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-08-27T21:41:16.637 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:16 ntpd[22507]: can't open /var/log/ntpstats/peerstats.20230827: Permission denied 2023-08-27T21:41:17.603 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:17 ntpd[22469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:17.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:17 ntpd[22469]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:17.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:17 ntpd[22469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:17.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:17 ntpd[22469]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:17.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:17 ntpd[22469]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-08-27T21:41:17.604 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:17 ntpd[22469]: can't open /var/log/ntpstats/peerstats.20230827: Permission denied 2023-08-27T21:41:17.604 INFO:teuthology.orchestra.run.smithi006.stdout:27 Aug 21:41:17 ntpd[22469]: ntpd: time slew +0.000392 s 2023-08-27T21:41:17.604 INFO:teuthology.orchestra.run.smithi006.stdout:ntpd: time slew +0.000392s 2023-08-27T21:41:17.605 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:17 ntpd[22469]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2023-08-27T21:41:17.605 INFO:teuthology.orchestra.run.smithi006.stderr:27 Aug 21:41:17 ntpd[22469]: can't open /var/log/ntpstats/loopstats.20230827: Permission denied 2023-08-27T21:41:17.637 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:17 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:17.637 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:17 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:17.637 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:17 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:17.637 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:17 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:17.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:17 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-08-27T21:41:17.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:17 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20230827: Permission denied 2023-08-27T21:41:17.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:17 ntpd[22507]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-08-27T21:41:17.638 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:17 ntpd[22507]: can't open /var/log/ntpstats/peerstats.20230827: Permission denied 2023-08-27T21:41:17.638 INFO:teuthology.orchestra.run.smithi130.stdout:27 Aug 21:41:17 ntpd[22507]: ntpd: time slew +0.003779 s 2023-08-27T21:41:17.638 INFO:teuthology.orchestra.run.smithi130.stdout:ntpd: time slew +0.003779s 2023-08-27T21:41:17.639 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:17 ntpd[22507]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2023-08-27T21:41:17.639 INFO:teuthology.orchestra.run.smithi130.stderr:27 Aug 21:41:17 ntpd[22507]: can't open /var/log/ntpstats/loopstats.20230827: Permission denied 2023-08-27T21:41:17.666 INFO:teuthology.orchestra.run.smithi006.stdout: remote refid st t when poll reach delay offset jitter 2023-08-27T21:41:17.666 INFO:teuthology.orchestra.run.smithi006.stdout:============================================================================== 2023-08-27T21:41:17.666 INFO:teuthology.orchestra.run.smithi006.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-08-27T21:41:17.666 INFO:teuthology.orchestra.run.smithi006.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-08-27T21:41:17.666 INFO:teuthology.orchestra.run.smithi006.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-08-27T21:41:17.666 INFO:teuthology.orchestra.run.smithi006.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-08-27T21:41:17.756 INFO:teuthology.orchestra.run.smithi130.stdout: remote refid st t when poll reach delay offset jitter 2023-08-27T21:41:17.756 INFO:teuthology.orchestra.run.smithi130.stdout:============================================================================== 2023-08-27T21:41:17.756 INFO:teuthology.orchestra.run.smithi130.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-08-27T21:41:17.756 INFO:teuthology.orchestra.run.smithi130.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-08-27T21:41:17.756 INFO:teuthology.orchestra.run.smithi130.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-08-27T21:41:17.757 INFO:teuthology.orchestra.run.smithi130.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-08-27T21:41:17.757 INFO:teuthology.run_tasks:Running task install... 2023-08-27T21:41:17.767 DEBUG:teuthology.task.install:project ceph 2023-08-27T21:41:17.767 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '9cf208207dfe23ffccb33308b22e51570cb45e51'}} 2023-08-27T21:41:17.767 DEBUG:teuthology.task.install:config {'branch': 'pacific', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '9cf208207dfe23ffccb33308b22e51570cb45e51'} 2023-08-27T21:41:17.767 INFO:teuthology.task.install:Using flavor: default 2023-08-27T21:41:17.773 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-08-27T21:41:17.773 INFO:teuthology.task.install:extra packages: [] 2023-08-27T21:41:17.773 DEBUG:teuthology.orchestra.run.smithi006:> sudo apt-key list | grep Ceph 2023-08-27T21:41:17.774 DEBUG:teuthology.orchestra.run.smithi130:> sudo apt-key list | grep Ceph 2023-08-27T21:41:17.869 INFO:teuthology.orchestra.run.smithi006.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2023-08-27T21:41:17.904 INFO:teuthology.orchestra.run.smithi130.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2023-08-27T21:41:17.932 INFO:teuthology.orchestra.run.smithi006.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2023-08-27T21:41:17.933 INFO:teuthology.orchestra.run.smithi006.stdout:uid [ unknown] Ceph.com (release key) 2023-08-27T21:41:17.934 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2023-08-27T21:41:17.935 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-08-27T21:41:17.935 INFO:teuthology.packaging:ref: None 2023-08-27T21:41:17.935 INFO:teuthology.packaging:tag: None 2023-08-27T21:41:17.935 INFO:teuthology.packaging:branch: pacific 2023-08-27T21:41:17.935 INFO:teuthology.packaging:sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T21:41:17.935 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-08-27T21:41:17.970 INFO:teuthology.orchestra.run.smithi130.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2023-08-27T21:41:17.970 INFO:teuthology.orchestra.run.smithi130.stdout:uid [ unknown] Ceph.com (release key) 2023-08-27T21:41:17.971 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2023-08-27T21:41:17.971 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-08-27T21:41:17.971 INFO:teuthology.packaging:ref: None 2023-08-27T21:41:17.971 INFO:teuthology.packaging:tag: None 2023-08-27T21:41:17.971 INFO:teuthology.packaging:branch: pacific 2023-08-27T21:41:17.971 INFO:teuthology.packaging:sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T21:41:17.971 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-08-27T21:41:18.157 INFO:teuthology.task.install.deb:Pulling from https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default/ 2023-08-27T21:41:18.157 INFO:teuthology.task.install.deb:Package version is 16.2.13-565-g595355f4-1focal 2023-08-27T21:41:18.297 INFO:teuthology.task.install.deb:Pulling from https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default/ 2023-08-27T21:41:18.297 INFO:teuthology.task.install.deb:Package version is 16.2.13-565-g595355f4-1focal 2023-08-27T21:41:18.332 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2023-08-27T21:41:18.333 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2023-08-27T21:41:18.350 DEBUG:teuthology.orchestra.run.smithi006:> sudo apt-get update 2023-08-27T21:41:18.462 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-08-27T21:41:18.462 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2023-08-27T21:41:18.477 DEBUG:teuthology.orchestra.run.smithi130:> sudo apt-get update 2023-08-27T21:41:18.537 INFO:teuthology.orchestra.run.smithi006.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-08-27T21:41:18.544 INFO:teuthology.orchestra.run.smithi006.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-08-27T21:41:18.562 INFO:teuthology.orchestra.run.smithi006.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-08-27T21:41:18.651 INFO:teuthology.orchestra.run.smithi006.stdout:Ign:4 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal InRelease 2023-08-27T21:41:18.656 INFO:teuthology.orchestra.run.smithi006.stdout:Get:5 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-08-27T21:41:18.662 INFO:teuthology.orchestra.run.smithi130.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-08-27T21:41:18.663 INFO:teuthology.orchestra.run.smithi130.stdout:Get:2 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-08-27T21:41:18.669 INFO:teuthology.orchestra.run.smithi130.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-08-27T21:41:18.687 INFO:teuthology.orchestra.run.smithi130.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-08-27T21:41:18.687 INFO:teuthology.orchestra.run.smithi006.stdout:Get:6 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal Release [7,680 B] 2023-08-27T21:41:18.717 INFO:teuthology.orchestra.run.smithi130.stdout:Ign:5 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal InRelease 2023-08-27T21:41:18.726 INFO:teuthology.orchestra.run.smithi006.stdout:Ign:7 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal Release.gpg 2023-08-27T21:41:18.739 INFO:teuthology.orchestra.run.smithi130.stdout:Get:6 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal Release [7,680 B] 2023-08-27T21:41:18.762 INFO:teuthology.orchestra.run.smithi006.stdout:Get:8 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 Packages [17.2 kB] 2023-08-27T21:41:18.763 INFO:teuthology.orchestra.run.smithi130.stdout:Ign:7 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal Release.gpg 2023-08-27T21:41:18.784 INFO:teuthology.orchestra.run.smithi130.stdout:Get:8 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 Packages [17.2 kB] 2023-08-27T21:41:18.817 INFO:teuthology.orchestra.run.smithi006.stdout:Get:9 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main i386 Packages [3,103 B] 2023-08-27T21:41:18.833 INFO:teuthology.orchestra.run.smithi130.stdout:Get:9 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main i386 Packages [3,103 B] 2023-08-27T21:41:19.195 INFO:teuthology.orchestra.run.smithi006.stdout:Fetched 250 kB in 1s (393 kB/s) 2023-08-27T21:41:19.284 INFO:teuthology.orchestra.run.smithi130.stdout:Fetched 250 kB in 1s (415 kB/s) 2023-08-27T21:41:20.257 INFO:teuthology.orchestra.run.smithi006.stdout:Reading package lists... 2023-08-27T21:41:20.286 DEBUG:teuthology.orchestra.run.smithi006:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=16.2.13-565-g595355f4-1focal cephadm=16.2.13-565-g595355f4-1focal ceph-mds=16.2.13-565-g595355f4-1focal ceph-mgr=16.2.13-565-g595355f4-1focal ceph-common=16.2.13-565-g595355f4-1focal ceph-fuse=16.2.13-565-g595355f4-1focal ceph-test=16.2.13-565-g595355f4-1focal radosgw=16.2.13-565-g595355f4-1focal python3-rados=16.2.13-565-g595355f4-1focal python3-rgw=16.2.13-565-g595355f4-1focal python3-cephfs=16.2.13-565-g595355f4-1focal python3-rbd=16.2.13-565-g595355f4-1focal libcephfs2=16.2.13-565-g595355f4-1focal libcephfs-dev=16.2.13-565-g595355f4-1focal librados2=16.2.13-565-g595355f4-1focal librbd1=16.2.13-565-g595355f4-1focal rbd-fuse=16.2.13-565-g595355f4-1focal 2023-08-27T21:41:20.308 INFO:teuthology.orchestra.run.smithi130.stdout:Reading package lists... 2023-08-27T21:41:20.336 DEBUG:teuthology.orchestra.run.smithi130:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=16.2.13-565-g595355f4-1focal cephadm=16.2.13-565-g595355f4-1focal ceph-mds=16.2.13-565-g595355f4-1focal ceph-mgr=16.2.13-565-g595355f4-1focal ceph-common=16.2.13-565-g595355f4-1focal ceph-fuse=16.2.13-565-g595355f4-1focal ceph-test=16.2.13-565-g595355f4-1focal radosgw=16.2.13-565-g595355f4-1focal python3-rados=16.2.13-565-g595355f4-1focal python3-rgw=16.2.13-565-g595355f4-1focal python3-cephfs=16.2.13-565-g595355f4-1focal python3-rbd=16.2.13-565-g595355f4-1focal libcephfs2=16.2.13-565-g595355f4-1focal libcephfs-dev=16.2.13-565-g595355f4-1focal librados2=16.2.13-565-g595355f4-1focal librbd1=16.2.13-565-g595355f4-1focal rbd-fuse=16.2.13-565-g595355f4-1focal 2023-08-27T21:41:20.358 INFO:teuthology.orchestra.run.smithi006.stdout:Reading package lists... 2023-08-27T21:41:20.408 INFO:teuthology.orchestra.run.smithi130.stdout:Reading package lists... 2023-08-27T21:41:20.561 INFO:teuthology.orchestra.run.smithi006.stdout:Building dependency tree... 2023-08-27T21:41:20.562 INFO:teuthology.orchestra.run.smithi006.stdout:Reading state information... 2023-08-27T21:41:20.606 INFO:teuthology.orchestra.run.smithi130.stdout:Building dependency tree... 2023-08-27T21:41:20.607 INFO:teuthology.orchestra.run.smithi130.stdout:Reading state information... 2023-08-27T21:41:20.741 INFO:teuthology.orchestra.run.smithi006.stdout:The following packages were automatically installed and are no longer required: 2023-08-27T21:41:20.742 INFO:teuthology.orchestra.run.smithi006.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2023-08-27T21:41:20.742 INFO:teuthology.orchestra.run.smithi006.stdout:Use 'sudo apt autoremove' to remove them. 2023-08-27T21:41:20.742 INFO:teuthology.orchestra.run.smithi006.stdout:The following additional packages will be installed: 2023-08-27T21:41:20.742 INFO:teuthology.orchestra.run.smithi006.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2023-08-27T21:41:20.743 INFO:teuthology.orchestra.run.smithi006.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq libjaeger 2023-08-27T21:41:20.743 INFO:teuthology.orchestra.run.smithi006.stdout: libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 libleveldb1d 2023-08-27T21:41:20.743 INFO:teuthology.orchestra.run.smithi006.stdout: liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 libradosstriper1 2023-08-27T21:41:20.743 INFO:teuthology.orchestra.run.smithi006.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli python-pastedeploy-tpl 2023-08-27T21:41:20.743 INFO:teuthology.orchestra.run.smithi006.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2023-08-27T21:41:20.744 INFO:teuthology.orchestra.run.smithi006.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common 2023-08-27T21:41:20.744 INFO:teuthology.orchestra.run.smithi006.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2023-08-27T21:41:20.744 INFO:teuthology.orchestra.run.smithi006.stdout: python3-html5lib python3-jinja2 python3-joblib python3-jwt 2023-08-27T21:41:20.744 INFO:teuthology.orchestra.run.smithi006.stdout: python3-kubernetes python3-logutils python3-lxml python3-mako 2023-08-27T21:41:20.744 INFO:teuthology.orchestra.run.smithi006.stdout: python3-markupsafe python3-oauthlib python3-openssl python3-packaging 2023-08-27T21:41:20.744 INFO:teuthology.orchestra.run.smithi006.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2023-08-27T21:41:20.744 INFO:teuthology.orchestra.run.smithi006.stdout: python3-pluggy python3-prettytable python3-psutil python3-py python3-pyasn1 2023-08-27T21:41:20.744 INFO:teuthology.orchestra.run.smithi006.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2023-08-27T21:41:20.744 INFO:teuthology.orchestra.run.smithi006.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2023-08-27T21:41:20.744 INFO:teuthology.orchestra.run.smithi006.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2023-08-27T21:41:20.745 INFO:teuthology.orchestra.run.smithi006.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2023-08-27T21:41:20.745 INFO:teuthology.orchestra.run.smithi006.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2023-08-27T21:41:20.745 INFO:teuthology.orchestra.run.smithi006.stdout: python3-websocket python3-webtest python3-werkzeug socat xmlstarlet 2023-08-27T21:41:20.745 INFO:teuthology.orchestra.run.smithi006.stdout:Suggested packages: 2023-08-27T21:41:20.745 INFO:teuthology.orchestra.run.smithi006.stdout: python3-influxdb python-attr-doc python-blinker-doc python-cryptography-doc 2023-08-27T21:41:20.745 INFO:teuthology.orchestra.run.smithi006.stdout: python3-cryptography-vectors python3-genshi python-jinja2-doc python3-crypto 2023-08-27T21:41:20.745 INFO:teuthology.orchestra.run.smithi006.stdout: python3-lxml-dbg python-lxml-doc python3-beaker python-mako-doc 2023-08-27T21:41:20.745 INFO:teuthology.orchestra.run.smithi006.stdout: python-openssl-doc python3-openssl-dbg httpd-wsgi libapache2-mod-python 2023-08-27T21:41:20.746 INFO:teuthology.orchestra.run.smithi006.stdout: libapache2-mod-scgi libjs-mochikit python-pecan-doc python-psutil-doc 2023-08-27T21:41:20.746 INFO:teuthology.orchestra.run.smithi006.stdout: subversion python-pyinotify-doc python-scipy-doc python3-dap 2023-08-27T21:41:20.746 INFO:teuthology.orchestra.run.smithi006.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2023-08-27T21:41:20.746 INFO:teuthology.orchestra.run.smithi006.stdout: python-webtest-doc python-werkzeug-doc python3-termcolor python3-watchdog 2023-08-27T21:41:20.746 INFO:teuthology.orchestra.run.smithi006.stdout:Recommended packages: 2023-08-27T21:41:20.746 INFO:teuthology.orchestra.run.smithi006.stdout: btrfs-tools 2023-08-27T21:41:20.782 INFO:teuthology.orchestra.run.smithi130.stdout:The following packages were automatically installed and are no longer required: 2023-08-27T21:41:20.783 INFO:teuthology.orchestra.run.smithi130.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2023-08-27T21:41:20.783 INFO:teuthology.orchestra.run.smithi130.stdout:Use 'sudo apt autoremove' to remove them. 2023-08-27T21:41:20.783 INFO:teuthology.orchestra.run.smithi130.stdout:The following additional packages will be installed: 2023-08-27T21:41:20.783 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2023-08-27T21:41:20.784 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq libjaeger 2023-08-27T21:41:20.784 INFO:teuthology.orchestra.run.smithi130.stdout: libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 libleveldb1d 2023-08-27T21:41:20.784 INFO:teuthology.orchestra.run.smithi130.stdout: liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 libradosstriper1 2023-08-27T21:41:20.784 INFO:teuthology.orchestra.run.smithi130.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli python-pastedeploy-tpl 2023-08-27T21:41:20.784 INFO:teuthology.orchestra.run.smithi130.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2023-08-27T21:41:20.784 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common 2023-08-27T21:41:20.784 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2023-08-27T21:41:20.784 INFO:teuthology.orchestra.run.smithi130.stdout: python3-html5lib python3-jinja2 python3-joblib python3-jwt 2023-08-27T21:41:20.784 INFO:teuthology.orchestra.run.smithi130.stdout: python3-kubernetes python3-logutils python3-lxml python3-mako 2023-08-27T21:41:20.784 INFO:teuthology.orchestra.run.smithi130.stdout: python3-markupsafe python3-oauthlib python3-openssl python3-packaging 2023-08-27T21:41:20.785 INFO:teuthology.orchestra.run.smithi130.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2023-08-27T21:41:20.785 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pluggy python3-prettytable python3-psutil python3-py python3-pyasn1 2023-08-27T21:41:20.785 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2023-08-27T21:41:20.785 INFO:teuthology.orchestra.run.smithi130.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2023-08-27T21:41:20.785 INFO:teuthology.orchestra.run.smithi130.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2023-08-27T21:41:20.785 INFO:teuthology.orchestra.run.smithi130.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2023-08-27T21:41:20.785 INFO:teuthology.orchestra.run.smithi130.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2023-08-27T21:41:20.785 INFO:teuthology.orchestra.run.smithi130.stdout: python3-websocket python3-webtest python3-werkzeug socat xmlstarlet 2023-08-27T21:41:20.785 INFO:teuthology.orchestra.run.smithi130.stdout:Suggested packages: 2023-08-27T21:41:20.786 INFO:teuthology.orchestra.run.smithi130.stdout: python3-influxdb python-attr-doc python-blinker-doc python-cryptography-doc 2023-08-27T21:41:20.786 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cryptography-vectors python3-genshi python-jinja2-doc python3-crypto 2023-08-27T21:41:20.786 INFO:teuthology.orchestra.run.smithi130.stdout: python3-lxml-dbg python-lxml-doc python3-beaker python-mako-doc 2023-08-27T21:41:20.786 INFO:teuthology.orchestra.run.smithi130.stdout: python-openssl-doc python3-openssl-dbg httpd-wsgi libapache2-mod-python 2023-08-27T21:41:20.786 INFO:teuthology.orchestra.run.smithi130.stdout: libapache2-mod-scgi libjs-mochikit python-pecan-doc python-psutil-doc 2023-08-27T21:41:20.786 INFO:teuthology.orchestra.run.smithi130.stdout: subversion python-pyinotify-doc python-scipy-doc python3-dap 2023-08-27T21:41:20.786 INFO:teuthology.orchestra.run.smithi130.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2023-08-27T21:41:20.786 INFO:teuthology.orchestra.run.smithi130.stdout: python-webtest-doc python-werkzeug-doc python3-termcolor python3-watchdog 2023-08-27T21:41:20.786 INFO:teuthology.orchestra.run.smithi130.stdout:Recommended packages: 2023-08-27T21:41:20.786 INFO:teuthology.orchestra.run.smithi130.stdout: btrfs-tools 2023-08-27T21:41:20.864 INFO:teuthology.orchestra.run.smithi006.stdout:The following NEW packages will be installed: 2023-08-27T21:41:20.864 INFO:teuthology.orchestra.run.smithi006.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2023-08-27T21:41:20.865 INFO:teuthology.orchestra.run.smithi006.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2023-08-27T21:41:20.865 INFO:teuthology.orchestra.run.smithi006.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test cephadm jq libcephfs-dev 2023-08-27T21:41:20.865 INFO:teuthology.orchestra.run.smithi006.stdout: libcephfs2 libjaeger libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 2023-08-27T21:41:20.865 INFO:teuthology.orchestra.run.smithi006.stdout: libleveldb1d liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 2023-08-27T21:41:20.866 INFO:teuthology.orchestra.run.smithi006.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli 2023-08-27T21:41:20.866 INFO:teuthology.orchestra.run.smithi006.stdout: python-pastedeploy-tpl python3-atomicwrites python3-attr python3-bcrypt 2023-08-27T21:41:20.866 INFO:teuthology.orchestra.run.smithi006.stdout: python3-blinker python3-bs4 python3-cachetools python3-ceph-argparse 2023-08-27T21:41:20.866 INFO:teuthology.orchestra.run.smithi006.stdout: python3-ceph-common python3-cephfs python3-cherrypy3 python3-cryptography 2023-08-27T21:41:20.866 INFO:teuthology.orchestra.run.smithi006.stdout: python3-decorator python3-google-auth python3-html5lib python3-jinja2 2023-08-27T21:41:20.866 INFO:teuthology.orchestra.run.smithi006.stdout: python3-joblib python3-jwt python3-kubernetes python3-logutils python3-lxml 2023-08-27T21:41:20.866 INFO:teuthology.orchestra.run.smithi006.stdout: python3-mako python3-markupsafe python3-oauthlib python3-openssl 2023-08-27T21:41:20.867 INFO:teuthology.orchestra.run.smithi006.stdout: python3-packaging python3-paste python3-pastedeploy python3-pastescript 2023-08-27T21:41:20.867 INFO:teuthology.orchestra.run.smithi006.stdout: python3-pecan python3-pluggy python3-prettytable python3-psutil python3-py 2023-08-27T21:41:20.867 INFO:teuthology.orchestra.run.smithi006.stdout: python3-pyasn1 python3-pyasn1-modules python3-pyinotify python3-pytest 2023-08-27T21:41:20.867 INFO:teuthology.orchestra.run.smithi006.stdout: python3-rados python3-rbd python3-repoze.lru python3-requests-oauthlib 2023-08-27T21:41:20.867 INFO:teuthology.orchestra.run.smithi006.stdout: python3-rgw python3-routes python3-rsa python3-scipy python3-simplegeneric 2023-08-27T21:41:20.867 INFO:teuthology.orchestra.run.smithi006.stdout: python3-simplejson python3-singledispatch python3-sklearn 2023-08-27T21:41:20.867 INFO:teuthology.orchestra.run.smithi006.stdout: python3-sklearn-lib python3-soupsieve python3-tempita python3-waitress 2023-08-27T21:41:20.867 INFO:teuthology.orchestra.run.smithi006.stdout: python3-wcwidth python3-webencodings python3-webob python3-websocket 2023-08-27T21:41:20.867 INFO:teuthology.orchestra.run.smithi006.stdout: python3-webtest python3-werkzeug radosgw rbd-fuse socat xmlstarlet 2023-08-27T21:41:20.868 INFO:teuthology.orchestra.run.smithi006.stdout:The following packages will be upgraded: 2023-08-27T21:41:20.868 INFO:teuthology.orchestra.run.smithi006.stdout: librados2 librbd1 2023-08-27T21:41:20.895 INFO:teuthology.orchestra.run.smithi130.stdout:The following NEW packages will be installed: 2023-08-27T21:41:20.895 INFO:teuthology.orchestra.run.smithi130.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2023-08-27T21:41:20.895 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2023-08-27T21:41:20.895 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test cephadm jq libcephfs-dev 2023-08-27T21:41:20.896 INFO:teuthology.orchestra.run.smithi130.stdout: libcephfs2 libjaeger libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 2023-08-27T21:41:20.896 INFO:teuthology.orchestra.run.smithi130.stdout: libleveldb1d liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 2023-08-27T21:41:20.896 INFO:teuthology.orchestra.run.smithi130.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli 2023-08-27T21:41:20.896 INFO:teuthology.orchestra.run.smithi130.stdout: python-pastedeploy-tpl python3-atomicwrites python3-attr python3-bcrypt 2023-08-27T21:41:20.896 INFO:teuthology.orchestra.run.smithi130.stdout: python3-blinker python3-bs4 python3-cachetools python3-ceph-argparse 2023-08-27T21:41:20.897 INFO:teuthology.orchestra.run.smithi130.stdout: python3-ceph-common python3-cephfs python3-cherrypy3 python3-cryptography 2023-08-27T21:41:20.897 INFO:teuthology.orchestra.run.smithi130.stdout: python3-decorator python3-google-auth python3-html5lib python3-jinja2 2023-08-27T21:41:20.897 INFO:teuthology.orchestra.run.smithi130.stdout: python3-joblib python3-jwt python3-kubernetes python3-logutils python3-lxml 2023-08-27T21:41:20.897 INFO:teuthology.orchestra.run.smithi130.stdout: python3-mako python3-markupsafe python3-oauthlib python3-openssl 2023-08-27T21:41:20.897 INFO:teuthology.orchestra.run.smithi130.stdout: python3-packaging python3-paste python3-pastedeploy python3-pastescript 2023-08-27T21:41:20.897 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pecan python3-pluggy python3-prettytable python3-psutil python3-py 2023-08-27T21:41:20.897 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyasn1 python3-pyasn1-modules python3-pyinotify python3-pytest 2023-08-27T21:41:20.898 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rados python3-rbd python3-repoze.lru python3-requests-oauthlib 2023-08-27T21:41:20.898 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rgw python3-routes python3-rsa python3-scipy python3-simplegeneric 2023-08-27T21:41:20.898 INFO:teuthology.orchestra.run.smithi130.stdout: python3-simplejson python3-singledispatch python3-sklearn 2023-08-27T21:41:20.898 INFO:teuthology.orchestra.run.smithi130.stdout: python3-sklearn-lib python3-soupsieve python3-tempita python3-waitress 2023-08-27T21:41:20.898 INFO:teuthology.orchestra.run.smithi130.stdout: python3-wcwidth python3-webencodings python3-webob python3-websocket 2023-08-27T21:41:20.898 INFO:teuthology.orchestra.run.smithi130.stdout: python3-webtest python3-werkzeug radosgw rbd-fuse socat xmlstarlet 2023-08-27T21:41:20.899 INFO:teuthology.orchestra.run.smithi130.stdout:The following packages will be upgraded: 2023-08-27T21:41:20.899 INFO:teuthology.orchestra.run.smithi130.stdout: librados2 librbd1 2023-08-27T21:41:21.026 INFO:teuthology.orchestra.run.smithi130.stdout:2 upgraded, 98 newly installed, 0 to remove and 243 not upgraded. 2023-08-27T21:41:21.026 INFO:teuthology.orchestra.run.smithi130.stdout:Need to get 173 MB of archives. 2023-08-27T21:41:21.026 INFO:teuthology.orchestra.run.smithi130.stdout:After this operation, 804 MB of additional disk space will be used. 2023-08-27T21:41:21.026 INFO:teuthology.orchestra.run.smithi130.stdout:Get:1 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 libjaeger amd64 16.2.13-565-g595355f4-1focal [3,948 B] 2023-08-27T21:41:21.055 INFO:teuthology.orchestra.run.smithi006.stdout:2 upgraded, 98 newly installed, 0 to remove and 243 not upgraded. 2023-08-27T21:41:21.055 INFO:teuthology.orchestra.run.smithi006.stdout:Need to get 173 MB of archives. 2023-08-27T21:41:21.055 INFO:teuthology.orchestra.run.smithi006.stdout:After this operation, 804 MB of additional disk space will be used. 2023-08-27T21:41:21.055 INFO:teuthology.orchestra.run.smithi006.stdout:Get:1 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 libjaeger amd64 16.2.13-565-g595355f4-1focal [3,948 B] 2023-08-27T21:41:21.060 INFO:teuthology.orchestra.run.smithi006.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2023-08-27T21:41:21.069 INFO:teuthology.orchestra.run.smithi130.stdout:Get:2 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 16.2.13-565-g595355f4-1focal [3,383 kB] 2023-08-27T21:41:21.090 INFO:teuthology.orchestra.run.smithi130.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2023-08-27T21:41:21.129 INFO:teuthology.orchestra.run.smithi006.stdout:Get:3 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 16.2.13-565-g595355f4-1focal [3,383 kB] 2023-08-27T21:41:21.281 INFO:teuthology.orchestra.run.smithi130.stdout:Get:4 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 16.2.13-565-g595355f4-1focal [3,391 kB] 2023-08-27T21:41:21.444 INFO:teuthology.orchestra.run.smithi130.stdout:Get:5 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 16.2.13-565-g595355f4-1focal [728 kB] 2023-08-27T21:41:21.457 INFO:teuthology.orchestra.run.smithi006.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2023-08-27T21:41:21.464 INFO:teuthology.orchestra.run.smithi006.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2023-08-27T21:41:21.479 INFO:teuthology.orchestra.run.smithi130.stdout:Get:6 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 16.2.13-565-g595355f4-1focal [414 kB] 2023-08-27T21:41:21.487 INFO:teuthology.orchestra.run.smithi130.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2023-08-27T21:41:21.494 INFO:teuthology.orchestra.run.smithi130.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2023-08-27T21:41:21.501 INFO:teuthology.orchestra.run.smithi130.stdout:Get:9 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 16.2.13-565-g595355f4-1focal [22.2 kB] 2023-08-27T21:41:21.503 INFO:teuthology.orchestra.run.smithi130.stdout:Get:10 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 16.2.13-565-g595355f4-1focal [212 kB] 2023-08-27T21:41:21.509 INFO:teuthology.orchestra.run.smithi006.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2023-08-27T21:41:21.511 INFO:teuthology.orchestra.run.smithi130.stdout:Get:11 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 16.2.13-565-g595355f4-1focal [38.4 kB] 2023-08-27T21:41:21.513 INFO:teuthology.orchestra.run.smithi130.stdout:Get:12 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 16.2.13-565-g595355f4-1focal [386 kB] 2023-08-27T21:41:21.522 INFO:teuthology.orchestra.run.smithi006.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2023-08-27T21:41:21.533 INFO:teuthology.orchestra.run.smithi130.stdout:Get:13 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 16.2.13-565-g595355f4-1focal [3,686 kB] 2023-08-27T21:41:21.538 INFO:teuthology.orchestra.run.smithi130.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2023-08-27T21:41:21.552 INFO:teuthology.orchestra.run.smithi130.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2023-08-27T21:41:21.555 INFO:teuthology.orchestra.run.smithi006.stdout:Get:8 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 16.2.13-565-g595355f4-1focal [3,391 kB] 2023-08-27T21:41:21.649 INFO:teuthology.orchestra.run.smithi006.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal/main amd64 libleveldb1d amd64 1.22-3ubuntu2 [139 kB] 2023-08-27T21:41:21.671 INFO:teuthology.orchestra.run.smithi006.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2023-08-27T21:41:21.678 INFO:teuthology.orchestra.run.smithi006.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2023-08-27T21:41:21.678 INFO:teuthology.orchestra.run.smithi130.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal/main amd64 libleveldb1d amd64 1.22-3ubuntu2 [139 kB] 2023-08-27T21:41:21.701 INFO:teuthology.orchestra.run.smithi130.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2023-08-27T21:41:21.708 INFO:teuthology.orchestra.run.smithi130.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2023-08-27T21:41:21.709 INFO:teuthology.orchestra.run.smithi006.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2023-08-27T21:41:21.714 INFO:teuthology.orchestra.run.smithi006.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2023-08-27T21:41:21.714 INFO:teuthology.orchestra.run.smithi130.stdout:Get:19 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 16.2.13-565-g595355f4-1focal [116 kB] 2023-08-27T21:41:21.716 INFO:teuthology.orchestra.run.smithi006.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2023-08-27T21:41:21.719 INFO:teuthology.orchestra.run.smithi130.stdout:Get:20 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 16.2.13-565-g595355f4-1focal [449 kB] 2023-08-27T21:41:21.739 INFO:teuthology.orchestra.run.smithi130.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2023-08-27T21:41:21.742 INFO:teuthology.orchestra.run.smithi130.stdout:Get:22 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 16.2.13-565-g595355f4-1focal [22.6 MB] 2023-08-27T21:41:21.743 INFO:teuthology.orchestra.run.smithi130.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2023-08-27T21:41:21.746 INFO:teuthology.orchestra.run.smithi130.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2023-08-27T21:41:21.884 INFO:teuthology.orchestra.run.smithi006.stdout:Get:15 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 16.2.13-565-g595355f4-1focal [728 kB] 2023-08-27T21:41:21.896 INFO:teuthology.orchestra.run.smithi006.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-jwt all 1.7.1-2ubuntu2.1 [18.0 kB] 2023-08-27T21:41:21.926 INFO:teuthology.orchestra.run.smithi130.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-jwt all 1.7.1-2ubuntu2.1 [18.0 kB] 2023-08-27T21:41:21.956 INFO:teuthology.orchestra.run.smithi006.stdout:Get:17 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 16.2.13-565-g595355f4-1focal [414 kB] 2023-08-27T21:41:21.997 INFO:teuthology.orchestra.run.smithi006.stdout:Get:18 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 16.2.13-565-g595355f4-1focal [22.2 kB] 2023-08-27T21:41:22.000 INFO:teuthology.orchestra.run.smithi006.stdout:Get:19 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 16.2.13-565-g595355f4-1focal [212 kB] 2023-08-27T21:41:22.020 INFO:teuthology.orchestra.run.smithi006.stdout:Get:20 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 16.2.13-565-g595355f4-1focal [38.4 kB] 2023-08-27T21:41:22.022 INFO:teuthology.orchestra.run.smithi006.stdout:Get:21 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 16.2.13-565-g595355f4-1focal [386 kB] 2023-08-27T21:41:22.060 INFO:teuthology.orchestra.run.smithi006.stdout:Get:22 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 16.2.13-565-g595355f4-1focal [3,686 kB] 2023-08-27T21:41:22.064 INFO:teuthology.orchestra.run.smithi006.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2023-08-27T21:41:22.094 INFO:teuthology.orchestra.run.smithi130.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2023-08-27T21:41:22.094 INFO:teuthology.orchestra.run.smithi006.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2023-08-27T21:41:22.118 INFO:teuthology.orchestra.run.smithi006.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2.1 [59.2 kB] 2023-08-27T21:41:22.123 INFO:teuthology.orchestra.run.smithi130.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2023-08-27T21:41:22.148 INFO:teuthology.orchestra.run.smithi130.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2.1 [59.2 kB] 2023-08-27T21:41:22.235 INFO:teuthology.orchestra.run.smithi006.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2023-08-27T21:41:22.247 INFO:teuthology.orchestra.run.smithi006.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2023-08-27T21:41:22.252 INFO:teuthology.orchestra.run.smithi006.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2023-08-27T21:41:22.264 INFO:teuthology.orchestra.run.smithi130.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2023-08-27T21:41:22.277 INFO:teuthology.orchestra.run.smithi130.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2023-08-27T21:41:22.281 INFO:teuthology.orchestra.run.smithi130.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2023-08-27T21:41:22.319 INFO:teuthology.orchestra.run.smithi006.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2023-08-27T21:41:22.335 INFO:teuthology.orchestra.run.smithi006.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2023-08-27T21:41:22.348 INFO:teuthology.orchestra.run.smithi130.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2023-08-27T21:41:22.364 INFO:teuthology.orchestra.run.smithi130.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2023-08-27T21:41:22.369 INFO:teuthology.orchestra.run.smithi006.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-waitress all 1.4.1-1ubuntu0.1 [81.4 kB] 2023-08-27T21:41:22.396 INFO:teuthology.orchestra.run.smithi006.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2023-08-27T21:41:22.399 INFO:teuthology.orchestra.run.smithi130.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-waitress all 1.4.1-1ubuntu0.1 [81.4 kB] 2023-08-27T21:41:22.426 INFO:teuthology.orchestra.run.smithi130.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2023-08-27T21:41:22.501 INFO:teuthology.orchestra.run.smithi006.stdout:Get:33 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 16.2.13-565-g595355f4-1focal [116 kB] 2023-08-27T21:41:22.503 INFO:teuthology.orchestra.run.smithi006.stdout:Get:34 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 16.2.13-565-g595355f4-1focal [449 kB] 2023-08-27T21:41:22.535 INFO:teuthology.orchestra.run.smithi006.stdout:Get:35 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 16.2.13-565-g595355f4-1focal [22.6 MB] 2023-08-27T21:41:22.565 INFO:teuthology.orchestra.run.smithi006.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2023-08-27T21:41:22.594 INFO:teuthology.orchestra.run.smithi130.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2023-08-27T21:41:22.931 INFO:teuthology.orchestra.run.smithi130.stdout:Get:37 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 16.2.13-565-g595355f4-1focal [5,914 kB] 2023-08-27T21:41:23.092 INFO:teuthology.orchestra.run.smithi006.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2023-08-27T21:41:23.093 INFO:teuthology.orchestra.run.smithi006.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2023-08-27T21:41:23.096 INFO:teuthology.orchestra.run.smithi006.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2023-08-27T21:41:23.099 INFO:teuthology.orchestra.run.smithi006.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2023-08-27T21:41:23.111 INFO:teuthology.orchestra.run.smithi006.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-werkzeug all 0.16.1+dfsg1-2ubuntu0.1 [183 kB] 2023-08-27T21:41:23.119 INFO:teuthology.orchestra.run.smithi130.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2023-08-27T21:41:23.119 INFO:teuthology.orchestra.run.smithi130.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2023-08-27T21:41:23.122 INFO:teuthology.orchestra.run.smithi130.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2023-08-27T21:41:23.126 INFO:teuthology.orchestra.run.smithi130.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2023-08-27T21:41:23.134 INFO:teuthology.orchestra.run.smithi006.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jinja2 all 2.10.1-2 [95.5 kB] 2023-08-27T21:41:23.137 INFO:teuthology.orchestra.run.smithi130.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-werkzeug all 0.16.1+dfsg1-2ubuntu0.1 [183 kB] 2023-08-27T21:41:23.143 INFO:teuthology.orchestra.run.smithi130.stdout:Get:43 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 16.2.13-565-g595355f4-1focal [204 kB] 2023-08-27T21:41:23.146 INFO:teuthology.orchestra.run.smithi006.stdout:Get:43 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2023-08-27T21:41:23.154 INFO:teuthology.orchestra.run.smithi006.stdout:Get:44 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2023-08-27T21:41:23.155 INFO:teuthology.orchestra.run.smithi130.stdout:Get:44 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 16.2.13-565-g595355f4-1focal [126 kB] 2023-08-27T21:41:23.157 INFO:teuthology.orchestra.run.smithi130.stdout:Get:45 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 16.2.13-565-g595355f4-1focal [1,533 kB] 2023-08-27T21:41:23.161 INFO:teuthology.orchestra.run.smithi130.stdout:Get:46 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jinja2 all 2.10.1-2 [95.5 kB] 2023-08-27T21:41:23.173 INFO:teuthology.orchestra.run.smithi130.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2023-08-27T21:41:23.174 INFO:teuthology.orchestra.run.smithi130.stdout:Get:48 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2023-08-27T21:41:23.182 INFO:teuthology.orchestra.run.smithi006.stdout:Get:45 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2023-08-27T21:41:23.194 INFO:teuthology.orchestra.run.smithi006.stdout:Get:46 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2023-08-27T21:41:23.208 INFO:teuthology.orchestra.run.smithi130.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2023-08-27T21:41:23.221 INFO:teuthology.orchestra.run.smithi130.stdout:Get:50 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2023-08-27T21:41:23.233 INFO:teuthology.orchestra.run.smithi130.stdout:Get:51 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 16.2.13-565-g595355f4-1focal [6,997 kB] 2023-08-27T21:41:23.370 INFO:teuthology.orchestra.run.smithi006.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2023-08-27T21:41:23.396 INFO:teuthology.orchestra.run.smithi130.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2023-08-27T21:41:23.454 INFO:teuthology.orchestra.run.smithi006.stdout:Get:48 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2023-08-27T21:41:23.480 INFO:teuthology.orchestra.run.smithi130.stdout:Get:53 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2023-08-27T21:41:23.538 INFO:teuthology.orchestra.run.smithi006.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2023-08-27T21:41:23.565 INFO:teuthology.orchestra.run.smithi130.stdout:Get:54 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2023-08-27T21:41:23.575 INFO:teuthology.orchestra.run.smithi130.stdout:Get:55 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 16.2.13-565-g595355f4-1focal [26.2 MB] 2023-08-27T21:41:24.339 INFO:teuthology.orchestra.run.smithi006.stdout:Get:50 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2023-08-27T21:41:24.355 INFO:teuthology.orchestra.run.smithi006.stdout:Get:51 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2023-08-27T21:41:24.357 INFO:teuthology.orchestra.run.smithi006.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2023-08-27T21:41:24.359 INFO:teuthology.orchestra.run.smithi130.stdout:Get:56 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2023-08-27T21:41:24.375 INFO:teuthology.orchestra.run.smithi130.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2023-08-27T21:41:24.376 INFO:teuthology.orchestra.run.smithi130.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2023-08-27T21:41:24.431 INFO:teuthology.orchestra.run.smithi006.stdout:Get:53 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2023-08-27T21:41:24.431 INFO:teuthology.orchestra.run.smithi006.stdout:Get:54 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2023-08-27T21:41:24.432 INFO:teuthology.orchestra.run.smithi006.stdout:Get:55 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2023-08-27T21:41:24.433 INFO:teuthology.orchestra.run.smithi006.stdout:Get:56 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2023-08-27T21:41:24.433 INFO:teuthology.orchestra.run.smithi006.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2023-08-27T21:41:24.456 INFO:teuthology.orchestra.run.smithi130.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2023-08-27T21:41:24.456 INFO:teuthology.orchestra.run.smithi130.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2023-08-27T21:41:24.457 INFO:teuthology.orchestra.run.smithi130.stdout:Get:61 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2023-08-27T21:41:24.458 INFO:teuthology.orchestra.run.smithi130.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2023-08-27T21:41:24.458 INFO:teuthology.orchestra.run.smithi130.stdout:Get:63 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2023-08-27T21:41:24.598 INFO:teuthology.orchestra.run.smithi006.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2023-08-27T21:41:24.622 INFO:teuthology.orchestra.run.smithi130.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2023-08-27T21:41:24.681 INFO:teuthology.orchestra.run.smithi006.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2023-08-27T21:41:24.705 INFO:teuthology.orchestra.run.smithi130.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2023-08-27T21:41:24.785 INFO:teuthology.orchestra.run.smithi006.stdout:Get:60 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 16.2.13-565-g595355f4-1focal [5,914 kB] 2023-08-27T21:41:24.860 INFO:teuthology.orchestra.run.smithi006.stdout:Get:61 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2023-08-27T21:41:24.876 INFO:teuthology.orchestra.run.smithi006.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2023-08-27T21:41:24.909 INFO:teuthology.orchestra.run.smithi006.stdout:Get:63 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2023-08-27T21:41:24.935 INFO:teuthology.orchestra.run.smithi130.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2023-08-27T21:41:24.935 INFO:teuthology.orchestra.run.smithi130.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2023-08-27T21:41:24.966 INFO:teuthology.orchestra.run.smithi130.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2023-08-27T21:41:24.966 INFO:teuthology.orchestra.run.smithi130.stdout:Get:69 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 16.2.13-565-g595355f4-1focal [4,044 B] 2023-08-27T21:41:24.967 INFO:teuthology.orchestra.run.smithi130.stdout:Get:70 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 16.2.13-565-g595355f4-1focal [834 kB] 2023-08-27T21:41:24.973 INFO:teuthology.orchestra.run.smithi130.stdout:Get:71 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 16.2.13-565-g595355f4-1focal [2,363 kB] 2023-08-27T21:41:25.031 INFO:teuthology.orchestra.run.smithi130.stdout:Get:72 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 16.2.13-565-g595355f4-1focal [76.4 kB] 2023-08-27T21:41:25.033 INFO:teuthology.orchestra.run.smithi130.stdout:Get:73 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-cephadm all 16.2.13-565-g595355f4-1focal [100 kB] 2023-08-27T21:41:25.037 INFO:teuthology.orchestra.run.smithi130.stdout:Get:74 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 16.2.13-565-g595355f4-1focal [2,823 kB] 2023-08-27T21:41:25.039 INFO:teuthology.orchestra.run.smithi006.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2023-08-27T21:41:25.062 INFO:teuthology.orchestra.run.smithi130.stdout:Get:75 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2023-08-27T21:41:25.075 INFO:teuthology.orchestra.run.smithi006.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2023-08-27T21:41:25.098 INFO:teuthology.orchestra.run.smithi130.stdout:Get:76 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2023-08-27T21:41:25.099 INFO:teuthology.orchestra.run.smithi006.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2023-08-27T21:41:25.107 INFO:teuthology.orchestra.run.smithi006.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2023-08-27T21:41:25.121 INFO:teuthology.orchestra.run.smithi130.stdout:Get:77 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2023-08-27T21:41:25.129 INFO:teuthology.orchestra.run.smithi130.stdout:Get:78 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2023-08-27T21:41:25.151 INFO:teuthology.orchestra.run.smithi006.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2023-08-27T21:41:25.172 INFO:teuthology.orchestra.run.smithi130.stdout:Get:79 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2023-08-27T21:41:25.177 INFO:teuthology.orchestra.run.smithi006.stdout:Get:69 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2023-08-27T21:41:25.178 INFO:teuthology.orchestra.run.smithi130.stdout:Get:80 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 16.2.13-565-g595355f4-1focal [7,891 kB] 2023-08-27T21:41:25.199 INFO:teuthology.orchestra.run.smithi130.stdout:Get:81 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2023-08-27T21:41:25.282 INFO:teuthology.orchestra.run.smithi006.stdout:Get:70 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 16.2.13-565-g595355f4-1focal [204 kB] 2023-08-27T21:41:25.299 INFO:teuthology.orchestra.run.smithi006.stdout:Get:71 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 16.2.13-565-g595355f4-1focal [126 kB] 2023-08-27T21:41:25.326 INFO:teuthology.orchestra.run.smithi006.stdout:Get:72 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 16.2.13-565-g595355f4-1focal [1,533 kB] 2023-08-27T21:41:25.364 INFO:teuthology.orchestra.run.smithi006.stdout:Get:73 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2023-08-27T21:41:25.385 INFO:teuthology.orchestra.run.smithi130.stdout:Get:82 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2023-08-27T21:41:25.448 INFO:teuthology.orchestra.run.smithi006.stdout:Get:74 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2023-08-27T21:41:25.462 INFO:teuthology.orchestra.run.smithi006.stdout:Get:75 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 16.2.13-565-g595355f4-1focal [6,997 kB] 2023-08-27T21:41:25.468 INFO:teuthology.orchestra.run.smithi130.stdout:Get:83 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2023-08-27T21:41:25.550 INFO:teuthology.orchestra.run.smithi006.stdout:Get:76 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2023-08-27T21:41:25.566 INFO:teuthology.orchestra.run.smithi130.stdout:Get:84 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 16.2.13-565-g595355f4-1focal [4,156 B] 2023-08-27T21:41:25.566 INFO:teuthology.orchestra.run.smithi130.stdout:Get:85 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 16.2.13-565-g595355f4-1focal [49.3 MB] 2023-08-27T21:41:25.569 INFO:teuthology.orchestra.run.smithi130.stdout:Get:86 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2023-08-27T21:41:25.573 INFO:teuthology.orchestra.run.smithi006.stdout:Get:77 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2023-08-27T21:41:25.593 INFO:teuthology.orchestra.run.smithi130.stdout:Get:87 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2023-08-27T21:41:25.689 INFO:teuthology.orchestra.run.smithi006.stdout:Get:78 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2023-08-27T21:41:25.709 INFO:teuthology.orchestra.run.smithi130.stdout:Get:88 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2023-08-27T21:41:25.976 INFO:teuthology.orchestra.run.smithi006.stdout:Get:79 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2023-08-27T21:41:25.977 INFO:teuthology.orchestra.run.smithi006.stdout:Get:80 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2023-08-27T21:41:25.982 INFO:teuthology.orchestra.run.smithi006.stdout:Get:81 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2023-08-27T21:41:25.983 INFO:teuthology.orchestra.run.smithi006.stdout:Get:82 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2023-08-27T21:41:25.994 INFO:teuthology.orchestra.run.smithi130.stdout:Get:89 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2023-08-27T21:41:25.995 INFO:teuthology.orchestra.run.smithi006.stdout:Get:83 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2023-08-27T21:41:25.995 INFO:teuthology.orchestra.run.smithi130.stdout:Get:90 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2023-08-27T21:41:25.998 INFO:teuthology.orchestra.run.smithi006.stdout:Get:84 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2023-08-27T21:41:25.999 INFO:teuthology.orchestra.run.smithi130.stdout:Get:91 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2023-08-27T21:41:26.000 INFO:teuthology.orchestra.run.smithi130.stdout:Get:92 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2023-08-27T21:41:26.010 INFO:teuthology.orchestra.run.smithi130.stdout:Get:93 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2023-08-27T21:41:26.015 INFO:teuthology.orchestra.run.smithi130.stdout:Get:94 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2023-08-27T21:41:26.146 INFO:teuthology.orchestra.run.smithi006.stdout:Get:85 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 16.2.13-565-g595355f4-1focal [26.2 MB] 2023-08-27T21:41:26.162 INFO:teuthology.orchestra.run.smithi006.stdout:Get:86 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2023-08-27T21:41:26.180 INFO:teuthology.orchestra.run.smithi130.stdout:Get:95 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2023-08-27T21:41:26.328 INFO:teuthology.orchestra.run.smithi006.stdout:Get:87 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2023-08-27T21:41:26.345 INFO:teuthology.orchestra.run.smithi130.stdout:Get:96 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2023-08-27T21:41:26.580 INFO:teuthology.orchestra.run.smithi006.stdout:Get:88 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2023-08-27T21:41:26.597 INFO:teuthology.orchestra.run.smithi130.stdout:Get:97 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2023-08-27T21:41:28.461 INFO:teuthology.orchestra.run.smithi006.stdout:Get:89 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 16.2.13-565-g595355f4-1focal [4,044 B] 2023-08-27T21:41:28.462 INFO:teuthology.orchestra.run.smithi006.stdout:Get:90 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 16.2.13-565-g595355f4-1focal [834 kB] 2023-08-27T21:41:28.468 INFO:teuthology.orchestra.run.smithi006.stdout:Get:91 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 16.2.13-565-g595355f4-1focal [2,363 kB] 2023-08-27T21:41:28.470 INFO:teuthology.orchestra.run.smithi130.stdout:Get:98 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 16.2.13-565-g595355f4-1focal [21.5 kB] 2023-08-27T21:41:28.470 INFO:teuthology.orchestra.run.smithi130.stdout:Get:99 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 16.2.13-565-g595355f4-1focal [11.3 MB] 2023-08-27T21:41:28.568 INFO:teuthology.orchestra.run.smithi006.stdout:Get:92 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 16.2.13-565-g595355f4-1focal [76.4 kB] 2023-08-27T21:41:28.570 INFO:teuthology.orchestra.run.smithi006.stdout:Get:93 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-cephadm all 16.2.13-565-g595355f4-1focal [100 kB] 2023-08-27T21:41:28.573 INFO:teuthology.orchestra.run.smithi006.stdout:Get:94 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 16.2.13-565-g595355f4-1focal [2,823 kB] 2023-08-27T21:41:28.802 INFO:teuthology.orchestra.run.smithi006.stdout:Get:95 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 16.2.13-565-g595355f4-1focal [7,891 kB] 2023-08-27T21:41:28.888 INFO:teuthology.orchestra.run.smithi130.stdout:Get:100 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 16.2.13-565-g595355f4-1focal [78.2 kB] 2023-08-27T21:41:29.139 INFO:teuthology.orchestra.run.smithi006.stdout:Get:96 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 16.2.13-565-g595355f4-1focal [4,156 B] 2023-08-27T21:41:29.139 INFO:teuthology.orchestra.run.smithi006.stdout:Get:97 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 16.2.13-565-g595355f4-1focal [49.3 MB] 2023-08-27T21:41:29.527 INFO:teuthology.orchestra.run.smithi130.stdout:Fetched 173 MB in 8s (21.7 MB/s) 2023-08-27T21:41:29.615 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package libjaeger. 2023-08-27T21:41:29.662 INFO:teuthology.orchestra.run.smithi130.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 112644 files and directories currently installed.) 2023-08-27T21:41:29.666 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../00-libjaeger_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:29.697 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking libjaeger (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:29.966 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2023-08-27T21:41:29.981 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../01-liblttng-ust0_2.11.0-1_amd64.deb ... 2023-08-27T21:41:30.066 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2023-08-27T21:41:30.410 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../02-librbd1_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:30.538 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking librbd1 (16.2.13-565-g595355f4-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-08-27T21:41:31.167 INFO:teuthology.orchestra.run.smithi006.stdout:Get:98 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 16.2.13-565-g595355f4-1focal [21.5 kB] 2023-08-27T21:41:31.167 INFO:teuthology.orchestra.run.smithi006.stdout:Get:99 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 16.2.13-565-g595355f4-1focal [11.3 MB] 2023-08-27T21:41:31.179 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../03-librados2_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:31.308 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking librados2 (16.2.13-565-g595355f4-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-08-27T21:41:31.459 INFO:teuthology.orchestra.run.smithi006.stdout:Get:100 https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 16.2.13-565-g595355f4-1focal [78.2 kB] 2023-08-27T21:41:31.978 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package libcephfs2. 2023-08-27T21:41:31.994 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../04-libcephfs2_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:32.036 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking libcephfs2 (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:32.121 INFO:teuthology.orchestra.run.smithi006.stdout:Fetched 173 MB in 11s (16.4 MB/s) 2023-08-27T21:41:32.215 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package libjaeger. 2023-08-27T21:41:32.262 INFO:teuthology.orchestra.run.smithi006.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 112644 files and directories currently installed.) 2023-08-27T21:41:32.265 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../00-libjaeger_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:32.296 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking libjaeger (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:32.364 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-rados. 2023-08-27T21:41:32.379 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../05-python3-rados_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:32.422 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-rados (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:32.641 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2023-08-27T21:41:32.654 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../01-liblttng-ust0_2.11.0-1_amd64.deb ... 2023-08-27T21:41:32.733 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2023-08-27T21:41:32.767 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-ceph-argparse. 2023-08-27T21:41:32.780 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../06-python3-ceph-argparse_16.2.13-565-g595355f4-1focal_all.deb ... 2023-08-27T21:41:32.825 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-ceph-argparse (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:33.074 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../02-librbd1_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:33.111 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-cephfs. 2023-08-27T21:41:33.126 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../07-python3-cephfs_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:33.169 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-cephfs (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:33.212 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking librbd1 (16.2.13-565-g595355f4-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-08-27T21:41:33.530 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-ceph-common. 2023-08-27T21:41:33.546 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../08-python3-ceph-common_16.2.13-565-g595355f4-1focal_all.deb ... 2023-08-27T21:41:33.588 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-ceph-common (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:33.867 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-prettytable. 2023-08-27T21:41:33.881 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../09-python3-prettytable_0.7.2-5_all.deb ... 2023-08-27T21:41:33.896 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../03-librados2_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:33.917 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2023-08-27T21:41:34.016 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking librados2 (16.2.13-565-g595355f4-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-08-27T21:41:34.161 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-rbd. 2023-08-27T21:41:34.175 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../10-python3-rbd_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:34.220 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-rbd (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:34.557 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2023-08-27T21:41:34.571 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../11-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2023-08-27T21:41:34.610 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-08-27T21:41:34.679 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package libcephfs2. 2023-08-27T21:41:34.693 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../04-libcephfs2_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:34.729 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking libcephfs2 (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:34.960 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package librabbitmq4:amd64. 2023-08-27T21:41:34.973 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../12-librabbitmq4_0.10.0-1_amd64.deb ... 2023-08-27T21:41:35.010 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2023-08-27T21:41:35.065 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-rados. 2023-08-27T21:41:35.078 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../05-python3-rados_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:35.114 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-rados (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:35.351 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package librdkafka1:amd64. 2023-08-27T21:41:35.365 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../13-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2023-08-27T21:41:35.427 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-08-27T21:41:35.476 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-ceph-argparse. 2023-08-27T21:41:35.489 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../06-python3-ceph-argparse_16.2.13-565-g595355f4-1focal_all.deb ... 2023-08-27T21:41:35.534 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-ceph-argparse (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:35.771 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package librgw2. 2023-08-27T21:41:35.785 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../14-librgw2_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:35.829 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking librgw2 (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:35.836 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-cephfs. 2023-08-27T21:41:35.850 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../07-python3-cephfs_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:35.903 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-cephfs (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:36.223 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-ceph-common. 2023-08-27T21:41:36.239 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../08-python3-ceph-common_16.2.13-565-g595355f4-1focal_all.deb ... 2023-08-27T21:41:36.281 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-ceph-common (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:36.432 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-rgw. 2023-08-27T21:41:36.447 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../15-python3-rgw_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:36.490 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-rgw (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:36.601 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-prettytable. 2023-08-27T21:41:36.616 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../09-python3-prettytable_0.7.2-5_all.deb ... 2023-08-27T21:41:36.659 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2023-08-27T21:41:36.844 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package libleveldb1d:amd64. 2023-08-27T21:41:36.858 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../16-libleveldb1d_1.22-3ubuntu2_amd64.deb ... 2023-08-27T21:41:36.902 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-08-27T21:41:36.922 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-rbd. 2023-08-27T21:41:36.935 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../10-python3-rbd_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:36.966 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-rbd (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:37.247 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package liboath0. 2023-08-27T21:41:37.260 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../17-liboath0_2.6.1-1.3_amd64.deb ... 2023-08-27T21:41:37.305 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2023-08-27T21:41:37.567 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2023-08-27T21:41:37.580 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../11-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2023-08-27T21:41:37.607 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package libradosstriper1. 2023-08-27T21:41:37.618 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-08-27T21:41:37.621 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../18-libradosstriper1_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:37.666 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking libradosstriper1 (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:37.959 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package librabbitmq4:amd64. 2023-08-27T21:41:37.972 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../12-librabbitmq4_0.10.0-1_amd64.deb ... 2023-08-27T21:41:38.017 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2023-08-27T21:41:38.128 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package ceph-common. 2023-08-27T21:41:38.141 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../19-ceph-common_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:38.186 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking ceph-common (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:38.470 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package librdkafka1:amd64. 2023-08-27T21:41:38.484 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../13-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2023-08-27T21:41:38.528 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-08-27T21:41:38.882 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package librgw2. 2023-08-27T21:41:38.896 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../14-librgw2_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:38.940 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking librgw2 (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:39.518 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-rgw. 2023-08-27T21:41:39.531 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../15-python3-rgw_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:39.593 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-rgw (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:39.921 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package libleveldb1d:amd64. 2023-08-27T21:41:39.934 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../16-libleveldb1d_1.22-3ubuntu2_amd64.deb ... 2023-08-27T21:41:39.987 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-08-27T21:41:40.332 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package liboath0. 2023-08-27T21:41:40.346 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../17-liboath0_2.6.1-1.3_amd64.deb ... 2023-08-27T21:41:40.390 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2023-08-27T21:41:40.450 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package ceph-base. 2023-08-27T21:41:40.464 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../20-ceph-base_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:40.526 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking ceph-base (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:40.710 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package libradosstriper1. 2023-08-27T21:41:40.724 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../18-libradosstriper1_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:40.768 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking libradosstriper1 (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:41.206 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package ceph-common. 2023-08-27T21:41:41.219 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../19-ceph-common_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:41.263 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking ceph-common (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:41.513 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-cryptography. 2023-08-27T21:41:41.527 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../21-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2023-08-27T21:41:41.571 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2023-08-27T21:41:41.966 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-openssl. 2023-08-27T21:41:41.982 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../22-python3-openssl_19.0.0-1build1_all.deb ... 2023-08-27T21:41:42.024 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2023-08-27T21:41:42.303 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2023-08-27T21:41:42.318 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../23-ceph-mgr-modules-core_16.2.13-565-g595355f4-1focal_all.deb ... 2023-08-27T21:41:42.352 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking ceph-mgr-modules-core (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:42.725 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-bcrypt. 2023-08-27T21:41:42.739 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../24-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2023-08-27T21:41:42.783 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2023-08-27T21:41:43.086 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-cherrypy3. 2023-08-27T21:41:43.571 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../25-python3-cherrypy3_8.9.1-6_all.deb ... 2023-08-27T21:41:43.572 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2023-08-27T21:41:43.572 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-jwt. 2023-08-27T21:41:43.572 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../26-python3-jwt_1.7.1-2ubuntu2.1_all.deb ... 2023-08-27T21:41:43.572 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2.1) ... 2023-08-27T21:41:43.811 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package ceph-base. 2023-08-27T21:41:43.825 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../20-ceph-base_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:43.876 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-logutils. 2023-08-27T21:41:43.878 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking ceph-base (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:43.890 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../27-python3-logutils_0.3.3-7_all.deb ... 2023-08-27T21:41:44.209 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-logutils (0.3.3-7) ... 2023-08-27T21:41:44.512 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-markupsafe. 2023-08-27T21:41:44.525 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../28-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2023-08-27T21:41:44.563 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2023-08-27T21:41:44.848 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-mako. 2023-08-27T21:41:44.849 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-cryptography. 2023-08-27T21:41:44.863 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../29-python3-mako_1.1.0+ds1-1ubuntu2.1_all.deb ... 2023-08-27T21:41:44.863 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../21-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2023-08-27T21:41:44.907 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2023-08-27T21:41:44.939 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-08-27T21:41:45.268 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-simplegeneric. 2023-08-27T21:41:45.281 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../30-python3-simplegeneric_0.8.1-3_all.deb ... 2023-08-27T21:41:45.326 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2023-08-27T21:41:45.344 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-openssl. 2023-08-27T21:41:45.357 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../22-python3-openssl_19.0.0-1build1_all.deb ... 2023-08-27T21:41:45.410 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2023-08-27T21:41:45.645 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-singledispatch. 2023-08-27T21:41:45.659 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../31-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2023-08-27T21:41:45.695 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2023-08-27T21:41:45.731 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2023-08-27T21:41:45.747 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../23-ceph-mgr-modules-core_16.2.13-565-g595355f4-1focal_all.deb ... 2023-08-27T21:41:45.797 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking ceph-mgr-modules-core (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:45.998 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-webob. 2023-08-27T21:41:46.014 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../32-python3-webob_1%3a1.8.5-2_all.deb ... 2023-08-27T21:41:46.048 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2023-08-27T21:41:46.211 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-bcrypt. 2023-08-27T21:41:46.225 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../24-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2023-08-27T21:41:46.277 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2023-08-27T21:41:46.359 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-soupsieve. 2023-08-27T21:41:46.372 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../33-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2023-08-27T21:41:46.409 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2023-08-27T21:41:46.655 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-cherrypy3. 2023-08-27T21:41:46.671 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../25-python3-cherrypy3_8.9.1-6_all.deb ... 2023-08-27T21:41:46.712 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-bs4. 2023-08-27T21:41:46.722 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2023-08-27T21:41:46.725 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../34-python3-bs4_4.8.2-1_all.deb ... 2023-08-27T21:41:46.770 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2023-08-27T21:41:47.123 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-waitress. 2023-08-27T21:41:47.138 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../35-python3-waitress_1.4.1-1ubuntu0.1_all.deb ... 2023-08-27T21:41:47.142 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-jwt. 2023-08-27T21:41:47.156 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../26-python3-jwt_1.7.1-2ubuntu2.1_all.deb ... 2023-08-27T21:41:47.184 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-waitress (1.4.1-1ubuntu0.1) ... 2023-08-27T21:41:47.208 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2.1) ... 2023-08-27T21:41:47.510 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-tempita. 2023-08-27T21:41:47.524 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../36-python3-tempita_0.5.2-6_all.deb ... 2023-08-27T21:41:47.568 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-tempita (0.5.2-6) ... 2023-08-27T21:41:47.570 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-logutils. 2023-08-27T21:41:47.584 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../27-python3-logutils_0.3.3-7_all.deb ... 2023-08-27T21:41:47.636 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-logutils (0.3.3-7) ... 2023-08-27T21:41:47.871 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-paste. 2023-08-27T21:41:47.886 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../37-python3-paste_3.4.0+dfsg1-1_all.deb ... 2023-08-27T21:41:47.929 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2023-08-27T21:41:47.956 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-markupsafe. 2023-08-27T21:41:47.970 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../28-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2023-08-27T21:41:48.014 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2023-08-27T21:41:48.240 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2023-08-27T21:41:48.255 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../38-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2023-08-27T21:41:48.298 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2023-08-27T21:41:48.301 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-mako. 2023-08-27T21:41:48.315 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../29-python3-mako_1.1.0+ds1-1ubuntu2.1_all.deb ... 2023-08-27T21:41:48.359 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-08-27T21:41:48.602 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-pastedeploy. 2023-08-27T21:41:48.616 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../39-python3-pastedeploy_2.1.0-1_all.deb ... 2023-08-27T21:41:48.660 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2023-08-27T21:41:48.696 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-simplegeneric. 2023-08-27T21:41:48.710 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../30-python3-simplegeneric_0.8.1-3_all.deb ... 2023-08-27T21:41:48.754 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2023-08-27T21:41:49.007 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-webtest. 2023-08-27T21:41:49.022 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../40-python3-webtest_2.0.32-2_all.deb ... 2023-08-27T21:41:49.063 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-webtest (2.0.32-2) ... 2023-08-27T21:41:49.074 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-singledispatch. 2023-08-27T21:41:49.088 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../31-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2023-08-27T21:41:49.132 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2023-08-27T21:41:49.358 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-pecan. 2023-08-27T21:41:49.370 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../41-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2023-08-27T21:41:49.407 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2023-08-27T21:41:49.410 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-webob. 2023-08-27T21:41:49.424 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../32-python3-webob_1%3a1.8.5-2_all.deb ... 2023-08-27T21:41:49.468 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2023-08-27T21:41:49.745 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-werkzeug. 2023-08-27T21:41:49.753 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../42-python3-werkzeug_0.16.1+dfsg1-2ubuntu0.1_all.deb ... 2023-08-27T21:41:49.787 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-soupsieve. 2023-08-27T21:41:49.795 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-08-27T21:41:49.796 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../33-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2023-08-27T21:41:49.837 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2023-08-27T21:41:50.148 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2023-08-27T21:41:50.148 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-bs4. 2023-08-27T21:41:50.157 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../43-libsqlite3-mod-ceph_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:50.159 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../34-python3-bs4_4.8.2-1_all.deb ... 2023-08-27T21:41:50.198 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking libsqlite3-mod-ceph (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:50.198 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2023-08-27T21:41:50.500 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package ceph-mgr. 2023-08-27T21:41:50.509 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../44-ceph-mgr_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:50.559 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking ceph-mgr (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:50.577 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-waitress. 2023-08-27T21:41:50.590 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../35-python3-waitress_1.4.1-1ubuntu0.1_all.deb ... 2023-08-27T21:41:50.638 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-waitress (1.4.1-1ubuntu0.1) ... 2023-08-27T21:41:50.955 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-tempita. 2023-08-27T21:41:50.968 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../36-python3-tempita_0.5.2-6_all.deb ... 2023-08-27T21:41:51.013 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-tempita (0.5.2-6) ... 2023-08-27T21:41:51.037 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package ceph-mon. 2023-08-27T21:41:51.051 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../45-ceph-mon_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:51.096 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking ceph-mon (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:51.324 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-paste. 2023-08-27T21:41:51.338 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../37-python3-paste_3.4.0+dfsg1-1_all.deb ... 2023-08-27T21:41:51.382 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2023-08-27T21:41:51.702 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2023-08-27T21:41:51.717 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../38-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2023-08-27T21:41:51.760 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2023-08-27T21:41:52.063 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-pastedeploy. 2023-08-27T21:41:52.078 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../39-python3-pastedeploy_2.1.0-1_all.deb ... 2023-08-27T21:41:52.122 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2023-08-27T21:41:52.182 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package ceph-osd. 2023-08-27T21:41:52.198 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../46-ceph-osd_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:52.240 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking ceph-osd (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:52.383 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-webtest. 2023-08-27T21:41:52.399 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../40-python3-webtest_2.0.32-2_all.deb ... 2023-08-27T21:41:52.441 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-webtest (2.0.32-2) ... 2023-08-27T21:41:53.587 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-pecan. 2023-08-27T21:41:53.601 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../41-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2023-08-27T21:41:53.645 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2023-08-27T21:41:54.033 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-werkzeug. 2023-08-27T21:41:54.047 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../42-python3-werkzeug_0.16.1+dfsg1-2ubuntu0.1_all.deb ... 2023-08-27T21:41:54.092 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-08-27T21:41:54.445 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2023-08-27T21:41:54.458 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../43-libsqlite3-mod-ceph_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:54.503 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking libsqlite3-mod-ceph (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:54.864 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package ceph-mgr. 2023-08-27T21:41:54.880 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../44-ceph-mgr_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:54.922 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking ceph-mgr (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:55.313 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package ceph. 2023-08-27T21:41:55.326 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../47-ceph_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:55.371 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking ceph (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:55.426 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package ceph-mon. 2023-08-27T21:41:55.440 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../45-ceph-mon_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:55.484 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking ceph-mon (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:55.690 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package ceph-fuse. 2023-08-27T21:41:55.704 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../48-ceph-fuse_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:55.748 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking ceph-fuse (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:56.168 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package ceph-mds. 2023-08-27T21:41:56.182 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../49-ceph-mds_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:56.226 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking ceph-mds (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:56.654 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package ceph-osd. 2023-08-27T21:41:56.668 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../46-ceph-osd_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:56.712 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking ceph-osd (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:56.771 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package cephadm. 2023-08-27T21:41:56.786 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../50-cephadm_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:56.829 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking cephadm (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:57.124 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-jinja2. 2023-08-27T21:41:57.137 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../51-python3-jinja2_2.10.1-2_all.deb ... 2023-08-27T21:41:57.182 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-jinja2 (2.10.1-2) ... 2023-08-27T21:41:57.502 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2023-08-27T21:41:57.516 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../52-ceph-mgr-cephadm_16.2.13-565-g595355f4-1focal_all.deb ... 2023-08-27T21:41:57.560 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking ceph-mgr-cephadm (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:57.897 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-repoze.lru. 2023-08-27T21:41:57.910 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../53-python3-repoze.lru_0.7-2_all.deb ... 2023-08-27T21:41:57.954 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2023-08-27T21:41:58.207 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package libjs-underscore. 2023-08-27T21:41:58.223 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../54-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2023-08-27T21:41:58.265 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-08-27T21:41:58.526 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package libjs-sphinxdoc. 2023-08-27T21:41:58.542 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../55-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2023-08-27T21:41:58.593 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-08-27T21:41:58.895 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-routes. 2023-08-27T21:41:58.909 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../56-python3-routes_2.4.1-2_all.deb ... 2023-08-27T21:41:58.953 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-routes (2.4.1-2) ... 2023-08-27T21:41:59.282 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2023-08-27T21:41:59.295 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../57-ceph-mgr-dashboard_16.2.13-565-g595355f4-1focal_all.deb ... 2023-08-27T21:41:59.340 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking ceph-mgr-dashboard (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:41:59.526 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package ceph. 2023-08-27T21:41:59.539 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../47-ceph_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:41:59.702 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking ceph (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:00.012 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package ceph-fuse. 2023-08-27T21:42:00.026 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../48-ceph-fuse_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:42:00.070 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking ceph-fuse (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:00.489 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-decorator. 2023-08-27T21:42:00.490 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package ceph-mds. 2023-08-27T21:42:00.504 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../58-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2023-08-27T21:42:00.504 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../49-ceph-mds_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:42:00.547 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2023-08-27T21:42:00.549 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking ceph-mds (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:00.850 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2023-08-27T21:42:00.863 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../59-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2023-08-27T21:42:00.908 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-08-27T21:42:01.093 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package cephadm. 2023-08-27T21:42:01.103 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../50-cephadm_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:42:01.143 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking cephadm (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:01.221 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-scipy. 2023-08-27T21:42:01.235 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../60-python3-scipy_1.3.3-3build1_amd64.deb ... 2023-08-27T21:42:01.279 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2023-08-27T21:42:01.455 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-jinja2. 2023-08-27T21:42:01.469 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../51-python3-jinja2_2.10.1-2_all.deb ... 2023-08-27T21:42:01.513 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-jinja2 (2.10.1-2) ... 2023-08-27T21:42:01.841 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2023-08-27T21:42:01.855 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../52-ceph-mgr-cephadm_16.2.13-565-g595355f4-1focal_all.deb ... 2023-08-27T21:42:01.899 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking ceph-mgr-cephadm (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:02.244 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-repoze.lru. 2023-08-27T21:42:02.260 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../53-python3-repoze.lru_0.7-2_all.deb ... 2023-08-27T21:42:02.302 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2023-08-27T21:42:02.538 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package libjs-underscore. 2023-08-27T21:42:02.552 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../54-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2023-08-27T21:42:02.571 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-sklearn-lib. 2023-08-27T21:42:02.587 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../61-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2023-08-27T21:42:02.596 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-08-27T21:42:02.629 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-08-27T21:42:02.882 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package libjs-sphinxdoc. 2023-08-27T21:42:02.896 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../55-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2023-08-27T21:42:02.940 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-08-27T21:42:03.141 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-joblib. 2023-08-27T21:42:03.155 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../62-python3-joblib_0.14.0-3_all.deb ... 2023-08-27T21:42:03.199 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-joblib (0.14.0-3) ... 2023-08-27T21:42:03.243 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-routes. 2023-08-27T21:42:03.259 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../56-python3-routes_2.4.1-2_all.deb ... 2023-08-27T21:42:03.301 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-routes (2.4.1-2) ... 2023-08-27T21:42:03.560 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-sklearn. 2023-08-27T21:42:03.574 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../63-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2023-08-27T21:42:03.618 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-08-27T21:42:03.659 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2023-08-27T21:42:03.675 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../57-ceph-mgr-dashboard_16.2.13-565-g595355f4-1focal_all.deb ... 2023-08-27T21:42:03.730 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking ceph-mgr-dashboard (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:04.359 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2023-08-27T21:42:04.373 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../64-ceph-mgr-diskprediction-local_16.2.13-565-g595355f4-1focal_all.deb ... 2023-08-27T21:42:04.417 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking ceph-mgr-diskprediction-local (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:04.805 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-decorator. 2023-08-27T21:42:04.819 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../58-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2023-08-27T21:42:04.862 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2023-08-27T21:42:05.165 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2023-08-27T21:42:05.179 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../59-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2023-08-27T21:42:05.224 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-08-27T21:42:05.504 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-cachetools. 2023-08-27T21:42:05.517 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../65-python3-cachetools_4.0.0-1_all.deb ... 2023-08-27T21:42:05.562 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2023-08-27T21:42:05.585 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-scipy. 2023-08-27T21:42:05.600 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../60-python3-scipy_1.3.3-3build1_amd64.deb ... 2023-08-27T21:42:05.643 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2023-08-27T21:42:05.848 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-pyasn1. 2023-08-27T21:42:05.863 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../66-python3-pyasn1_0.4.2-3build1_all.deb ... 2023-08-27T21:42:05.922 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2023-08-27T21:42:06.250 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-pyasn1-modules. 2023-08-27T21:42:06.266 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../67-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2023-08-27T21:42:06.308 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-08-27T21:42:06.619 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-rsa. 2023-08-27T21:42:06.633 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../68-python3-rsa_4.0-3_all.deb ... 2023-08-27T21:42:06.677 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-rsa (4.0-3) ... 2023-08-27T21:42:06.911 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-sklearn-lib. 2023-08-27T21:42:06.925 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../61-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2023-08-27T21:42:06.968 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-08-27T21:42:06.972 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-google-auth. 2023-08-27T21:42:06.987 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../69-python3-google-auth_1.5.1-2_all.deb ... 2023-08-27T21:42:07.030 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2023-08-27T21:42:07.350 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-blinker. 2023-08-27T21:42:07.364 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../70-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2023-08-27T21:42:07.408 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-08-27T21:42:07.505 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-joblib. 2023-08-27T21:42:07.519 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../62-python3-joblib_0.14.0-3_all.deb ... 2023-08-27T21:42:07.564 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-joblib (0.14.0-3) ... 2023-08-27T21:42:07.710 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-oauthlib. 2023-08-27T21:42:07.726 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../71-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2023-08-27T21:42:07.768 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2023-08-27T21:42:07.919 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-sklearn. 2023-08-27T21:42:07.933 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../63-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2023-08-27T21:42:07.977 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-08-27T21:42:08.114 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-requests-oauthlib. 2023-08-27T21:42:08.128 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../72-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2023-08-27T21:42:08.172 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-08-27T21:42:08.483 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-websocket. 2023-08-27T21:42:08.498 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../73-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2023-08-27T21:42:08.541 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2023-08-27T21:42:08.761 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2023-08-27T21:42:08.774 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../64-ceph-mgr-diskprediction-local_16.2.13-565-g595355f4-1focal_all.deb ... 2023-08-27T21:42:08.826 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking ceph-mgr-diskprediction-local (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:08.861 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-kubernetes. 2023-08-27T21:42:08.874 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../74-python3-kubernetes_7.0.0~a1-2_all.deb ... 2023-08-27T21:42:08.918 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2023-08-27T21:42:09.482 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2023-08-27T21:42:09.496 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../75-ceph-mgr-k8sevents_16.2.13-565-g595355f4-1focal_all.deb ... 2023-08-27T21:42:09.539 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking ceph-mgr-k8sevents (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:09.917 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package libonig5:amd64. 2023-08-27T21:42:09.931 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../76-libonig5_6.9.4-1_amd64.deb ... 2023-08-27T21:42:09.975 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2023-08-27T21:42:10.013 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-cachetools. 2023-08-27T21:42:10.027 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../65-python3-cachetools_4.0.0-1_all.deb ... 2023-08-27T21:42:10.071 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2023-08-27T21:42:10.328 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package libjq1:amd64. 2023-08-27T21:42:10.341 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../77-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-08-27T21:42:10.374 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-pyasn1. 2023-08-27T21:42:10.385 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-08-27T21:42:10.389 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../66-python3-pyasn1_0.4.2-3build1_all.deb ... 2023-08-27T21:42:10.432 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2023-08-27T21:42:10.655 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package jq. 2023-08-27T21:42:10.671 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../78-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-08-27T21:42:10.713 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2023-08-27T21:42:10.769 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-pyasn1-modules. 2023-08-27T21:42:10.783 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../67-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2023-08-27T21:42:10.827 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-08-27T21:42:10.966 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package socat. 2023-08-27T21:42:10.979 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../79-socat_1.7.3.3-2_amd64.deb ... 2023-08-27T21:42:11.024 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking socat (1.7.3.3-2) ... 2023-08-27T21:42:11.146 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-rsa. 2023-08-27T21:42:11.161 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../68-python3-rsa_4.0-3_all.deb ... 2023-08-27T21:42:11.205 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-rsa (4.0-3) ... 2023-08-27T21:42:11.368 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package xmlstarlet. 2023-08-27T21:42:11.382 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../80-xmlstarlet_1.6.1-2build1_amd64.deb ... 2023-08-27T21:42:11.426 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2023-08-27T21:42:11.515 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-google-auth. 2023-08-27T21:42:11.532 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../69-python3-google-auth_1.5.1-2_all.deb ... 2023-08-27T21:42:11.574 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2023-08-27T21:42:11.874 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package ceph-test. 2023-08-27T21:42:11.888 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../81-ceph-test_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:42:11.932 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking ceph-test (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:11.935 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-blinker. 2023-08-27T21:42:11.952 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../70-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2023-08-27T21:42:12.002 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-08-27T21:42:12.355 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-oauthlib. 2023-08-27T21:42:12.369 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../71-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2023-08-27T21:42:12.421 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2023-08-27T21:42:12.791 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-requests-oauthlib. 2023-08-27T21:42:12.805 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../72-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2023-08-27T21:42:12.849 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-08-27T21:42:13.170 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-websocket. 2023-08-27T21:42:13.183 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../73-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2023-08-27T21:42:13.227 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2023-08-27T21:42:13.546 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-kubernetes. 2023-08-27T21:42:13.561 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../74-python3-kubernetes_7.0.0~a1-2_all.deb ... 2023-08-27T21:42:13.604 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2023-08-27T21:42:14.477 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2023-08-27T21:42:15.093 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../75-ceph-mgr-k8sevents_16.2.13-565-g595355f4-1focal_all.deb ... 2023-08-27T21:42:15.093 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking ceph-mgr-k8sevents (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:15.093 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package libonig5:amd64. 2023-08-27T21:42:15.093 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../76-libonig5_6.9.4-1_amd64.deb ... 2023-08-27T21:42:15.093 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2023-08-27T21:42:15.273 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package libjq1:amd64. 2023-08-27T21:42:15.287 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../77-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-08-27T21:42:15.331 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-08-27T21:42:15.642 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package jq. 2023-08-27T21:42:15.656 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../78-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-08-27T21:42:15.700 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2023-08-27T21:42:15.953 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package socat. 2023-08-27T21:42:15.966 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../79-socat_1.7.3.3-2_amd64.deb ... 2023-08-27T21:42:16.010 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking socat (1.7.3.3-2) ... 2023-08-27T21:42:16.261 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package libcephfs-dev. 2023-08-27T21:42:16.276 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../82-libcephfs-dev_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:42:16.335 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking libcephfs-dev (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:16.355 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package xmlstarlet. 2023-08-27T21:42:16.369 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../80-xmlstarlet_1.6.1-2build1_amd64.deb ... 2023-08-27T21:42:16.413 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2023-08-27T21:42:16.747 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package nvme-cli. 2023-08-27T21:42:16.763 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../83-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2023-08-27T21:42:16.821 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2023-08-27T21:42:16.862 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package ceph-test. 2023-08-27T21:42:16.877 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../81-ceph-test_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:42:16.920 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking ceph-test (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:17.198 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-atomicwrites. 2023-08-27T21:42:17.207 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../84-python3-atomicwrites_1.1.5-2build1_all.deb ... 2023-08-27T21:42:17.248 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2023-08-27T21:42:17.550 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-attr. 2023-08-27T21:42:17.567 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../85-python3-attr_19.3.0-2_all.deb ... 2023-08-27T21:42:17.609 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-attr (19.3.0-2) ... 2023-08-27T21:42:17.928 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-webencodings. 2023-08-27T21:42:17.944 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../86-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2023-08-27T21:42:17.986 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2023-08-27T21:42:18.281 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-html5lib. 2023-08-27T21:42:18.296 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../87-python3-html5lib_1.0.1-2_all.deb ... 2023-08-27T21:42:18.339 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2023-08-27T21:42:18.683 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-lxml:amd64. 2023-08-27T21:42:18.692 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../88-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2023-08-27T21:42:18.733 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-08-27T21:42:19.203 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-packaging. 2023-08-27T21:42:19.218 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../89-python3-packaging_20.3-1_all.deb ... 2023-08-27T21:42:19.261 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-packaging (20.3-1) ... 2023-08-27T21:42:19.597 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-pastescript. 2023-08-27T21:42:19.612 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../90-python3-pastescript_2.0.2-4_all.deb ... 2023-08-27T21:42:19.655 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2023-08-27T21:42:19.979 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-pluggy. 2023-08-27T21:42:19.995 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../91-python3-pluggy_0.13.0-2_all.deb ... 2023-08-27T21:42:20.045 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2023-08-27T21:42:20.365 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-psutil. 2023-08-27T21:42:20.379 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../92-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2023-08-27T21:42:20.423 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2023-08-27T21:42:20.765 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-py. 2023-08-27T21:42:20.779 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../93-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2023-08-27T21:42:20.823 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2023-08-27T21:42:21.143 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-pyinotify. 2023-08-27T21:42:21.159 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../94-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2023-08-27T21:42:21.201 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-08-27T21:42:21.212 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package libcephfs-dev. 2023-08-27T21:42:21.228 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../82-libcephfs-dev_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:42:21.270 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking libcephfs-dev (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:21.495 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-wcwidth. 2023-08-27T21:42:21.511 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../95-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2023-08-27T21:42:21.553 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2023-08-27T21:42:21.682 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package nvme-cli. 2023-08-27T21:42:21.696 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../83-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2023-08-27T21:42:21.757 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2023-08-27T21:42:21.839 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-pytest. 2023-08-27T21:42:21.855 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../96-python3-pytest_4.6.9-1_all.deb ... 2023-08-27T21:42:21.897 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-pytest (4.6.9-1) ... 2023-08-27T21:42:22.139 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-atomicwrites. 2023-08-27T21:42:22.153 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../84-python3-atomicwrites_1.1.5-2build1_all.deb ... 2023-08-27T21:42:22.198 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2023-08-27T21:42:22.235 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package python3-simplejson. 2023-08-27T21:42:22.251 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../97-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2023-08-27T21:42:22.293 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2023-08-27T21:42:22.499 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-attr. 2023-08-27T21:42:22.514 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../85-python3-attr_19.3.0-2_all.deb ... 2023-08-27T21:42:22.557 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-attr (19.3.0-2) ... 2023-08-27T21:42:22.704 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package radosgw. 2023-08-27T21:42:22.718 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../98-radosgw_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:42:22.762 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking radosgw (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:23.019 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-webencodings. 2023-08-27T21:42:23.033 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../86-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2023-08-27T21:42:23.077 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2023-08-27T21:42:23.380 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-html5lib. 2023-08-27T21:42:23.396 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../87-python3-html5lib_1.0.1-2_all.deb ... 2023-08-27T21:42:23.438 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2023-08-27T21:42:23.808 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-lxml:amd64. 2023-08-27T21:42:23.824 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../88-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2023-08-27T21:42:23.866 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-08-27T21:42:23.974 INFO:teuthology.orchestra.run.smithi130.stdout:Selecting previously unselected package rbd-fuse. 2023-08-27T21:42:23.987 INFO:teuthology.orchestra.run.smithi130.stdout:Preparing to unpack .../99-rbd-fuse_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:42:24.032 INFO:teuthology.orchestra.run.smithi130.stdout:Unpacking rbd-fuse (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:24.329 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-packaging. 2023-08-27T21:42:24.340 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-attr (19.3.0-2) ... 2023-08-27T21:42:24.345 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../89-python3-packaging_20.3-1_all.deb ... 2023-08-27T21:42:24.389 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-packaging (20.3-1) ... 2023-08-27T21:42:24.636 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-08-27T21:42:24.684 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-pastescript. 2023-08-27T21:42:24.698 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../90-python3-pastescript_2.0.2-4_all.deb ... 2023-08-27T21:42:24.762 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2023-08-27T21:42:24.766 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2023-08-27T21:42:24.888 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2023-08-27T21:42:25.081 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up cephadm (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:25.103 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-pluggy. 2023-08-27T21:42:25.113 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../91-python3-pluggy_0.13.0-2_all.deb ... 2023-08-27T21:42:25.153 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2023-08-27T21:42:25.472 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-psutil. 2023-08-27T21:42:25.488 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../92-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2023-08-27T21:42:25.530 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2023-08-27T21:42:25.692 INFO:teuthology.orchestra.run.smithi130.stdout:Adding system user cephadm....done 2023-08-27T21:42:25.853 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-waitress (1.4.1-1ubuntu0.1) ... 2023-08-27T21:42:25.866 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-py. 2023-08-27T21:42:25.880 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../93-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2023-08-27T21:42:25.924 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2023-08-27T21:42:26.205 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-repoze.lru (0.7-2) ... 2023-08-27T21:42:26.287 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-pyinotify. 2023-08-27T21:42:26.300 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../94-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2023-08-27T21:42:26.344 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-08-27T21:42:26.497 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up liboath0 (2.6.1-1.3) ... 2023-08-27T21:42:26.623 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2023-08-27T21:42:26.688 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-wcwidth. 2023-08-27T21:42:26.702 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../95-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2023-08-27T21:42:26.747 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2023-08-27T21:42:26.941 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-joblib (0.14.0-3) ... 2023-08-27T21:42:27.049 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-pytest. 2023-08-27T21:42:27.065 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../96-python3-pytest_4.6.9-1_all.deb ... 2023-08-27T21:42:27.108 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-pytest (4.6.9-1) ... 2023-08-27T21:42:27.326 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-jwt (1.7.1-2ubuntu2.1) ... 2023-08-27T21:42:27.453 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package python3-simplejson. 2023-08-27T21:42:27.469 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../97-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2023-08-27T21:42:27.511 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2023-08-27T21:42:27.619 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-cachetools (4.0.0-1) ... 2023-08-27T21:42:27.903 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2023-08-27T21:42:27.948 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package radosgw. 2023-08-27T21:42:27.962 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../98-radosgw_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:42:28.005 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking radosgw (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:28.171 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-08-27T21:42:28.464 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-ceph-argparse (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:28.766 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-08-27T21:42:28.892 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2023-08-27T21:42:29.185 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2023-08-27T21:42:29.200 INFO:teuthology.orchestra.run.smithi006.stdout:Selecting previously unselected package rbd-fuse. 2023-08-27T21:42:29.215 INFO:teuthology.orchestra.run.smithi006.stdout:Preparing to unpack .../99-rbd-fuse_16.2.13-565-g595355f4-1focal_amd64.deb ... 2023-08-27T21:42:29.259 INFO:teuthology.orchestra.run.smithi006.stdout:Unpacking rbd-fuse (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:29.494 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2023-08-27T21:42:29.567 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-attr (19.3.0-2) ... 2023-08-27T21:42:29.853 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2023-08-27T21:42:29.871 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-08-27T21:42:29.998 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2023-08-27T21:42:30.121 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2023-08-27T21:42:30.123 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2023-08-27T21:42:30.333 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up cephadm (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:30.481 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2023-08-27T21:42:30.799 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up libjaeger (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:30.925 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-jinja2 (2.10.1-2) ... 2023-08-27T21:42:30.978 INFO:teuthology.orchestra.run.smithi006.stdout:Adding system user cephadm....done 2023-08-27T21:42:31.131 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-waitress (1.4.1-1ubuntu0.1) ... 2023-08-27T21:42:31.284 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-tempita (0.5.2-6) ... 2023-08-27T21:42:31.491 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-repoze.lru (0.7-2) ... 2023-08-27T21:42:31.576 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-packaging (20.3-1) ... 2023-08-27T21:42:31.783 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up liboath0 (2.6.1-1.3) ... 2023-08-27T21:42:31.869 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2023-08-27T21:42:31.934 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2023-08-27T21:42:31.978 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2023-08-27T21:42:32.271 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2023-08-27T21:42:32.286 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-joblib (0.14.0-3) ... 2023-08-27T21:42:32.673 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-08-27T21:42:32.680 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-jwt (1.7.1-2ubuntu2.1) ... 2023-08-27T21:42:32.974 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-cachetools (4.0.0-1) ... 2023-08-27T21:42:33.041 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2023-08-27T21:42:33.267 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2023-08-27T21:42:33.401 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up socat (1.7.3.3-2) ... 2023-08-27T21:42:33.527 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-ceph-common (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:33.584 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-08-27T21:42:33.836 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-html5lib (1.0.1-2) ... 2023-08-27T21:42:33.878 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-ceph-argparse (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:34.170 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-08-27T21:42:34.179 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-08-27T21:42:34.304 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2023-08-27T21:42:34.305 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2023-08-27T21:42:34.431 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-pluggy (0.13.0-2) ... 2023-08-27T21:42:34.580 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2023-08-27T21:42:34.707 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-08-27T21:42:34.914 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2023-08-27T21:42:35.017 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-08-27T21:42:35.143 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2023-08-27T21:42:35.283 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2023-08-27T21:42:35.462 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2023-08-27T21:42:35.559 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2023-08-27T21:42:35.737 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-logutils (0.3.3-7) ... 2023-08-27T21:42:35.902 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2023-08-27T21:42:36.021 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2023-08-27T21:42:36.148 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2023-08-27T21:42:36.177 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up libjaeger (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:36.303 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-jinja2 (2.10.1-2) ... 2023-08-27T21:42:36.423 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-prettytable (0.7.2-5) ... 2023-08-27T21:42:36.646 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-tempita (0.5.2-6) ... 2023-08-27T21:42:36.708 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2023-08-27T21:42:36.955 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-packaging (20.3-1) ... 2023-08-27T21:42:36.992 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2023-08-27T21:42:37.256 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2023-08-27T21:42:37.362 INFO:teuthology.orchestra.run.smithi130.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2023-08-27T21:42:37.382 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2023-08-27T21:42:37.619 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2023-08-27T21:42:37.683 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2023-08-27T21:42:37.746 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-08-27T21:42:38.064 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2023-08-27T21:42:38.093 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-08-27T21:42:38.407 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-08-27T21:42:38.470 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2023-08-27T21:42:38.533 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-08-27T21:42:38.675 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-webob (1:1.8.5-2) ... 2023-08-27T21:42:38.830 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up socat (1.7.3.3-2) ... 2023-08-27T21:42:38.981 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-ceph-common (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:39.001 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-08-27T21:42:39.294 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up librados2 (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:39.299 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-html5lib (1.0.1-2) ... 2023-08-27T21:42:39.420 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2023-08-27T21:42:39.642 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-08-27T21:42:39.768 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2023-08-27T21:42:39.893 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-pluggy (0.13.0-2) ... 2023-08-27T21:42:40.203 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-08-27T21:42:40.546 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-08-27T21:42:40.681 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2023-08-27T21:42:40.901 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up librgw2 (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:41.015 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2023-08-27T21:42:41.244 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up libsqlite3-mod-ceph (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:41.307 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-logutils (0.3.3-7) ... 2023-08-27T21:42:41.370 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-08-27T21:42:41.487 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-pytest (4.6.9-1) ... 2023-08-27T21:42:41.600 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2023-08-27T21:42:41.726 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2023-08-27T21:42:41.865 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2023-08-27T21:42:42.027 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-prettytable (0.7.2-5) ... 2023-08-27T21:42:42.168 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2023-08-27T21:42:42.320 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2023-08-27T21:42:42.452 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-bs4 (4.8.2-1) ... 2023-08-27T21:42:42.630 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2023-08-27T21:42:42.812 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up libcephfs2 (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:42.940 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2023-08-27T21:42:43.016 INFO:teuthology.orchestra.run.smithi006.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2023-08-27T21:42:43.266 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2023-08-27T21:42:43.275 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up libradosstriper1 (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:43.392 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-08-27T21:42:43.401 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-08-27T21:42:43.718 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2023-08-27T21:42:43.744 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-08-27T21:42:44.097 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-08-27T21:42:44.229 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-08-27T21:42:44.355 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-webob (1:1.8.5-2) ... 2023-08-27T21:42:44.681 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-08-27T21:42:44.833 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up librbd1 (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:44.959 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-08-27T21:42:45.084 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2023-08-27T21:42:45.210 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-rsa (4.0-3) ... 2023-08-27T21:42:45.440 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up librados2 (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:45.503 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-webtest (2.0.32-2) ... 2023-08-27T21:42:45.567 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2023-08-27T21:42:45.787 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up ceph-mgr-modules-core (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:45.899 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up ceph-fuse (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:46.116 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-08-27T21:42:46.116 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-08-27T21:42:46.701 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up libcephfs-dev (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:46.827 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-08-27T21:42:47.081 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up librgw2 (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:47.112 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-pastescript (2.0.2-4) ... 2023-08-27T21:42:47.199 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up libsqlite3-mod-ceph (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:47.325 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-08-27T21:42:47.421 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2023-08-27T21:42:47.454 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-pytest (4.6.9-1) ... 2023-08-27T21:42:47.789 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-rados (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:47.839 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2023-08-27T21:42:47.923 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-rbd (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:48.051 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up rbd-fuse (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:48.132 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2023-08-27T21:42:48.177 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-routes (2.4.1-2) ... 2023-08-27T21:42:48.416 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-bs4 (4.8.2-1) ... 2023-08-27T21:42:48.470 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-google-auth (1.5.1-2) ... 2023-08-27T21:42:48.759 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up libcephfs2 (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:48.856 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-rgw (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:48.885 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2023-08-27T21:42:48.982 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-cephfs (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:49.229 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up libradosstriper1 (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:49.250 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2023-08-27T21:42:49.355 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-08-27T21:42:49.697 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-08-27T21:42:50.203 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up ceph-common (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:50.747 INFO:teuthology.orchestra.run.smithi130.stdout:Adding group ceph....done 2023-08-27T21:42:50.839 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up librbd1 (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:50.965 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-08-27T21:42:51.082 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2023-08-27T21:42:51.133 INFO:teuthology.orchestra.run.smithi130.stdout:Adding system user ceph....done 2023-08-27T21:42:51.217 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-rsa (4.0-3) ... 2023-08-27T21:42:51.291 INFO:teuthology.orchestra.run.smithi130.stdout:Setting system user ceph properties....done 2023-08-27T21:42:51.312 INFO:teuthology.orchestra.run.smithi130.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2023-08-27T21:42:51.459 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2023-08-27T21:42:51.539 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-webtest (2.0.32-2) ... 2023-08-27T21:42:51.786 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2023-08-27T21:42:51.839 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up ceph-mgr-modules-core (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:51.973 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up ceph-fuse (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:52.206 INFO:teuthology.orchestra.run.smithi006.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-08-27T21:42:52.206 INFO:teuthology.orchestra.run.smithi006.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-08-27T21:42:52.246 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up ceph-test (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:52.372 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up radosgw (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:52.636 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-08-27T21:42:52.636 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-08-27T21:42:52.683 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up libcephfs-dev (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:52.809 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-08-27T21:42:53.127 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-pastescript (2.0.2-4) ... 2023-08-27T21:42:53.298 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up ceph-base (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:53.453 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2023-08-27T21:42:53.678 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2023-08-27T21:42:53.816 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-rados (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:53.950 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-rbd (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:54.076 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up rbd-fuse (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:54.126 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up ceph-mds (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:54.202 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-routes (2.4.1-2) ... 2023-08-27T21:42:54.349 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-08-27T21:42:54.349 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-08-27T21:42:54.511 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-google-auth (1.5.1-2) ... 2023-08-27T21:42:54.811 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up ceph-mgr (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:54.898 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-rgw (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:55.024 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-cephfs (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:55.032 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-08-27T21:42:55.032 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-08-27T21:42:55.317 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2023-08-27T21:42:55.489 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up ceph-osd (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:55.987 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-08-27T21:42:55.987 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-08-27T21:42:56.287 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up ceph-common (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:56.407 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up ceph-mgr-k8sevents (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:56.533 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up ceph-mgr-diskprediction-local (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:56.667 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up ceph-mon (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:56.823 INFO:teuthology.orchestra.run.smithi006.stdout:Adding group ceph....done 2023-08-27T21:42:56.869 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-08-27T21:42:56.869 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-08-27T21:42:57.225 INFO:teuthology.orchestra.run.smithi006.stdout:Adding system user ceph....done 2023-08-27T21:42:57.294 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up ceph-mgr-cephadm (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:57.401 INFO:teuthology.orchestra.run.smithi006.stdout:Setting system user ceph properties....done 2023-08-27T21:42:57.411 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up ceph (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:57.420 INFO:teuthology.orchestra.run.smithi006.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2023-08-27T21:42:57.540 INFO:teuthology.orchestra.run.smithi130.stdout:Setting up ceph-mgr-dashboard (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:57.575 INFO:teuthology.orchestra.run.smithi006.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2023-08-27T21:42:57.684 INFO:teuthology.orchestra.run.smithi130.stdout:Processing triggers for systemd (245.4-4ubuntu3.17) ... 2023-08-27T21:42:57.897 INFO:teuthology.orchestra.run.smithi006.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2023-08-27T21:42:58.018 INFO:teuthology.orchestra.run.smithi130.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-08-27T21:42:58.372 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up ceph-test (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:58.515 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up radosgw (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:58.781 INFO:teuthology.orchestra.run.smithi006.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-08-27T21:42:58.781 INFO:teuthology.orchestra.run.smithi006.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-08-27T21:42:59.459 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up ceph-base (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:42:59.836 INFO:teuthology.orchestra.run.smithi006.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2023-08-27T21:43:00.312 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up ceph-mds (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:43:00.549 INFO:teuthology.orchestra.run.smithi006.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-08-27T21:43:00.549 INFO:teuthology.orchestra.run.smithi006.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-08-27T21:43:01.031 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up ceph-mgr (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:43:01.087 INFO:teuthology.orchestra.run.smithi130.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-08-27T21:43:01.263 INFO:teuthology.orchestra.run.smithi006.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-08-27T21:43:01.264 INFO:teuthology.orchestra.run.smithi006.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-08-27T21:43:01.531 INFO:teuthology.orchestra.run.smithi130.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2023-08-27T21:43:01.535 DEBUG:teuthology.parallel:result is None 2023-08-27T21:43:01.716 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up ceph-osd (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:43:02.221 INFO:teuthology.orchestra.run.smithi006.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-08-27T21:43:02.221 INFO:teuthology.orchestra.run.smithi006.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-08-27T21:43:02.667 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up ceph-mgr-k8sevents (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:43:02.793 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up ceph-mgr-diskprediction-local (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:43:02.936 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up ceph-mon (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:43:03.160 INFO:teuthology.orchestra.run.smithi006.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-08-27T21:43:03.160 INFO:teuthology.orchestra.run.smithi006.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-08-27T21:43:03.640 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up ceph-mgr-cephadm (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:43:03.775 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up ceph (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:43:03.901 INFO:teuthology.orchestra.run.smithi006.stdout:Setting up ceph-mgr-dashboard (16.2.13-565-g595355f4-1focal) ... 2023-08-27T21:43:04.036 INFO:teuthology.orchestra.run.smithi006.stdout:Processing triggers for systemd (245.4-4ubuntu3.17) ... 2023-08-27T21:43:04.345 INFO:teuthology.orchestra.run.smithi006.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-08-27T21:43:07.458 INFO:teuthology.orchestra.run.smithi006.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-08-27T21:43:07.903 INFO:teuthology.orchestra.run.smithi006.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2023-08-27T21:43:07.905 DEBUG:teuthology.parallel:result is None 2023-08-27T21:43:07.906 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-08-27T21:43:07.906 INFO:teuthology.packaging:ref: None 2023-08-27T21:43:07.906 INFO:teuthology.packaging:tag: None 2023-08-27T21:43:07.906 INFO:teuthology.packaging:branch: pacific 2023-08-27T21:43:07.906 INFO:teuthology.packaging:sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T21:43:07.906 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-08-27T21:43:08.126 DEBUG:teuthology.orchestra.run.smithi006:> dpkg-query -W -f '${Version}' ceph 2023-08-27T21:43:08.149 INFO:teuthology.orchestra.run.smithi006.stdout:16.2.13-565-g595355f4-1focal 2023-08-27T21:43:08.149 INFO:teuthology.packaging:The installed version of ceph is 16.2.13-565-g595355f4-1focal 2023-08-27T21:43:08.150 INFO:teuthology.task.install:The correct ceph version 16.2.13-565-g595355f4-1focal is installed. 2023-08-27T21:43:08.150 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-08-27T21:43:08.150 INFO:teuthology.packaging:ref: None 2023-08-27T21:43:08.150 INFO:teuthology.packaging:tag: None 2023-08-27T21:43:08.150 INFO:teuthology.packaging:branch: pacific 2023-08-27T21:43:08.150 INFO:teuthology.packaging:sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T21:43:08.150 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-08-27T21:43:08.355 DEBUG:teuthology.orchestra.run.smithi130:> dpkg-query -W -f '${Version}' ceph 2023-08-27T21:43:08.379 INFO:teuthology.orchestra.run.smithi130.stdout:16.2.13-565-g595355f4-1focal 2023-08-27T21:43:08.379 INFO:teuthology.packaging:The installed version of ceph is 16.2.13-565-g595355f4-1focal 2023-08-27T21:43:08.379 INFO:teuthology.task.install:The correct ceph version 16.2.13-565-g595355f4-1focal is installed. 2023-08-27T21:43:08.380 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-08-27T21:43:08.380 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2023-08-27T21:43:08.380 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-08-27T21:43:08.396 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-08-27T21:43:08.396 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-08-27T21:43:08.436 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-08-27T21:43:08.436 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2023-08-27T21:43:08.436 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd of=/usr/bin/daemon-helper 2023-08-27T21:43:08.453 DEBUG:teuthology.orchestra.run.smithi006:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-08-27T21:43:08.511 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-08-27T21:43:08.511 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/usr/bin/daemon-helper 2023-08-27T21:43:08.526 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-08-27T21:43:08.585 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-08-27T21:43:08.585 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2023-08-27T21:43:08.585 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd of=/usr/bin/adjust-ulimits 2023-08-27T21:43:08.602 DEBUG:teuthology.orchestra.run.smithi006:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-08-27T21:43:08.659 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-08-27T21:43:08.659 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/usr/bin/adjust-ulimits 2023-08-27T21:43:08.674 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-08-27T21:43:08.733 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-08-27T21:43:08.733 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2023-08-27T21:43:08.733 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd of=/usr/bin/stdin-killer 2023-08-27T21:43:08.750 DEBUG:teuthology.orchestra.run.smithi006:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-08-27T21:43:08.805 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-08-27T21:43:08.805 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/usr/bin/stdin-killer 2023-08-27T21:43:08.822 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-08-27T21:43:08.879 INFO:teuthology.run_tasks:Running task cephadm... 2023-08-27T21:43:08.960 INFO:tasks.cephadm:Config: {'cephadm_branch': 'pacific', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:pacific', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '9cf208207dfe23ffccb33308b22e51570cb45e51'} 2023-08-27T21:43:08.961 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:pacific 2023-08-27T21:43:08.961 INFO:tasks.cephadm:Cluster fsid is b696f380-4522-11ee-817b-1b7a5f1ba315 2023-08-27T21:43:08.961 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-08-27T21:43:08.961 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.6', 'mon.c': '[v2:172.21.15.6:3301,v1:172.21.15.6:6790]', 'mon.b': '172.21.15.130'} 2023-08-27T21:43:08.962 INFO:tasks.cephadm:First mon is mon.a on smithi006 2023-08-27T21:43:08.962 INFO:tasks.cephadm:First mgr is y 2023-08-27T21:43:08.962 INFO:tasks.cephadm:Normalizing hostnames... 2023-08-27T21:43:08.962 DEBUG:teuthology.orchestra.run.smithi006:> sudo hostname $(hostname -s) 2023-08-27T21:43:08.979 DEBUG:teuthology.orchestra.run.smithi130:> sudo hostname $(hostname -s) 2023-08-27T21:43:08.994 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref pacific)... 2023-08-27T21:43:08.995 DEBUG:teuthology.orchestra.run.smithi006:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-08-27T21:43:09.183 INFO:teuthology.orchestra.run.smithi006.stdout:-rw-rw-r-- 1 ubuntu ubuntu 350873 Aug 27 21:43 /home/ubuntu/cephtest/cephadm 2023-08-27T21:43:09.183 DEBUG:teuthology.orchestra.run.smithi130:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-08-27T21:43:09.339 INFO:teuthology.orchestra.run.smithi130.stdout:-rw-rw-r-- 1 ubuntu ubuntu 350873 Aug 27 21:43 /home/ubuntu/cephtest/cephadm 2023-08-27T21:43:09.340 DEBUG:teuthology.orchestra.run.smithi006:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-08-27T21:43:09.349 DEBUG:teuthology.orchestra.run.smithi130:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-08-27T21:43:09.367 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:pacific on all hosts... 2023-08-27T21:43:09.367 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-08-27T21:43:09.393 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-08-27T21:43:09.661 INFO:teuthology.orchestra.run.smithi006.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-08-27T21:43:09.671 INFO:teuthology.orchestra.run.smithi130.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-08-27T21:43:35.650 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2023-08-27T21:43:35.651 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_version": "ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)", 2023-08-27T21:43:35.651 INFO:teuthology.orchestra.run.smithi130.stdout: "image_id": "4099d2e00414f822b7771da6dc5d588eeda3dc16e3b39be700f46017c3047641", 2023-08-27T21:43:35.651 INFO:teuthology.orchestra.run.smithi130.stdout: "repo_digests": [ 2023-08-27T21:43:35.651 INFO:teuthology.orchestra.run.smithi130.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:e2028333a2006b532dd5e4caab422f750a165963636ee26acf78c4fa6593c513" 2023-08-27T21:43:35.651 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2023-08-27T21:43:35.651 INFO:teuthology.orchestra.run.smithi130.stdout:} 2023-08-27T21:43:36.825 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2023-08-27T21:43:36.825 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph_version": "ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)", 2023-08-27T21:43:36.825 INFO:teuthology.orchestra.run.smithi006.stdout: "image_id": "4099d2e00414f822b7771da6dc5d588eeda3dc16e3b39be700f46017c3047641", 2023-08-27T21:43:36.825 INFO:teuthology.orchestra.run.smithi006.stdout: "repo_digests": [ 2023-08-27T21:43:36.825 INFO:teuthology.orchestra.run.smithi006.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:e2028333a2006b532dd5e4caab422f750a165963636ee26acf78c4fa6593c513" 2023-08-27T21:43:36.825 INFO:teuthology.orchestra.run.smithi006.stdout: ] 2023-08-27T21:43:36.825 INFO:teuthology.orchestra.run.smithi006.stdout:} 2023-08-27T21:43:36.850 DEBUG:teuthology.orchestra.run.smithi006:> sudo mkdir -p /etc/ceph 2023-08-27T21:43:36.864 DEBUG:teuthology.orchestra.run.smithi130:> sudo mkdir -p /etc/ceph 2023-08-27T21:43:36.881 DEBUG:teuthology.orchestra.run.smithi006:> sudo chmod 777 /etc/ceph 2023-08-27T21:43:36.923 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod 777 /etc/ceph 2023-08-27T21:43:36.938 INFO:tasks.cephadm:Writing seed config... 2023-08-27T21:43:36.939 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-08-27T21:43:36.939 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-08-27T21:43:36.939 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-08-27T21:43:36.939 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-08-27T21:43:36.939 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-08-27T21:43:36.939 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2023-08-27T21:43:36.939 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-08-27T21:43:36.939 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-08-27T21:43:36.939 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-08-27T21:43:36.939 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-08-27T21:43:36.940 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-08-27T21:43:36.940 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2023-08-27T21:43:36.940 DEBUG:teuthology.orchestra.run.smithi006:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-08-27T21:43:36.970 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = b696f380-4522-11ee-817b-1b7a5f1ba315 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-08-27T21:43:36.970 DEBUG:teuthology.orchestra.run.smithi006:mon.a> sudo journalctl -f -n 0 -u ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@mon.a.service 2023-08-27T21:43:37.013 DEBUG:teuthology.orchestra.run.smithi006:mgr.y> sudo journalctl -f -n 0 -u ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@mgr.y.service 2023-08-27T21:43:37.057 INFO:tasks.cephadm:Bootstrapping... 2023-08-27T21:43:37.057 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific -v bootstrap --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.6 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-08-27T21:43:37.266 INFO:teuthology.orchestra.run.smithi006.stderr:-------------------------------------------------------------------------------- 2023-08-27T21:43:37.266 INFO:teuthology.orchestra.run.smithi006.stderr:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:pacific', '-v', 'bootstrap', '--fsid', 'b696f380-4522-11ee-817b-1b7a5f1ba315', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.6', '--skip-admin-label'] 2023-08-27T21:43:37.266 INFO:teuthology.orchestra.run.smithi006.stderr:Verifying podman|docker is present... 2023-08-27T21:43:37.266 INFO:teuthology.orchestra.run.smithi006.stderr:Verifying lvm2 is present... 2023-08-27T21:43:37.266 INFO:teuthology.orchestra.run.smithi006.stderr:Verifying time synchronization is in place... 2023-08-27T21:43:37.272 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-08-27T21:43:37.277 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: inactive 2023-08-27T21:43:37.284 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Failed to get unit file state for chronyd.service: No such file or directory 2023-08-27T21:43:37.290 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: inactive 2023-08-27T21:43:37.296 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: masked 2023-08-27T21:43:37.302 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: inactive 2023-08-27T21:43:37.310 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Failed to get unit file state for ntpd.service: No such file or directory 2023-08-27T21:43:37.315 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: inactive 2023-08-27T21:43:37.320 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: enabled 2023-08-27T21:43:37.326 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: active 2023-08-27T21:43:37.327 INFO:teuthology.orchestra.run.smithi006.stderr:Unit ntp.service is enabled and running 2023-08-27T21:43:37.328 INFO:teuthology.orchestra.run.smithi006.stderr:Repeating the final host check... 2023-08-27T21:43:37.328 INFO:teuthology.orchestra.run.smithi006.stderr:docker (/usr/bin/docker) is present 2023-08-27T21:43:37.328 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl is present 2023-08-27T21:43:37.328 INFO:teuthology.orchestra.run.smithi006.stderr:lvcreate is present 2023-08-27T21:43:37.334 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-08-27T21:43:37.338 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: inactive 2023-08-27T21:43:37.345 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Failed to get unit file state for chronyd.service: No such file or directory 2023-08-27T21:43:37.351 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: inactive 2023-08-27T21:43:37.358 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: masked 2023-08-27T21:43:37.363 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: inactive 2023-08-27T21:43:37.368 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Failed to get unit file state for ntpd.service: No such file or directory 2023-08-27T21:43:37.374 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: inactive 2023-08-27T21:43:37.380 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: enabled 2023-08-27T21:43:37.386 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: active 2023-08-27T21:43:37.388 INFO:teuthology.orchestra.run.smithi006.stderr:Unit ntp.service is enabled and running 2023-08-27T21:43:37.388 INFO:teuthology.orchestra.run.smithi006.stderr:Host looks OK 2023-08-27T21:43:37.388 INFO:teuthology.orchestra.run.smithi006.stderr:Cluster fsid: b696f380-4522-11ee-817b-1b7a5f1ba315 2023-08-27T21:43:37.388 INFO:teuthology.orchestra.run.smithi006.stderr:Acquiring lock 140683307487184 on /run/cephadm/b696f380-4522-11ee-817b-1b7a5f1ba315.lock 2023-08-27T21:43:37.388 INFO:teuthology.orchestra.run.smithi006.stderr:Lock 140683307487184 acquired on /run/cephadm/b696f380-4522-11ee-817b-1b7a5f1ba315.lock 2023-08-27T21:43:37.388 INFO:teuthology.orchestra.run.smithi006.stderr:Verifying IP 172.21.15.6 port 3300 ... 2023-08-27T21:43:37.389 INFO:teuthology.orchestra.run.smithi006.stderr:Verifying IP 172.21.15.6 port 6789 ... 2023-08-27T21:43:37.389 INFO:teuthology.orchestra.run.smithi006.stderr:Base mon IP(s) is [172.21.15.6:3300, 172.21.15.6:6789], mon addrv is [v2:172.21.15.6:3300,v1:172.21.15.6:6789] 2023-08-27T21:43:37.393 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/sbin/ip: default via 172.21.15.254 dev ens1f0 2023-08-27T21:43:37.393 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/sbin/ip: 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 2023-08-27T21:43:37.393 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.6 2023-08-27T21:43:37.397 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2023-08-27T21:43:37.397 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2023-08-27T21:43:37.397 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1578sec hoplimit 64 pref medium 2023-08-27T21:43:37.401 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-08-27T21:43:37.401 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/sbin/ip: inet6 ::1/128 scope host 2023-08-27T21:43:37.401 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/sbin/ip: valid_lft forever preferred_lft forever 2023-08-27T21:43:37.402 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2023-08-27T21:43:37.402 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/sbin/ip: inet6 fe80::ec4:7aff:febd:15e4/64 scope link 2023-08-27T21:43:37.402 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/sbin/ip: valid_lft forever preferred_lft forever 2023-08-27T21:43:37.403 INFO:teuthology.orchestra.run.smithi006.stderr:Mon IP `172.21.15.6` is in CIDR network `172.21.0.0/20` 2023-08-27T21:43:37.403 INFO:teuthology.orchestra.run.smithi006.stderr:Mon IP `172.21.15.6` is in CIDR network `172.21.0.0/20` 2023-08-27T21:43:37.403 INFO:teuthology.orchestra.run.smithi006.stderr:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-08-27T21:43:37.404 INFO:teuthology.orchestra.run.smithi006.stderr:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-08-27T21:43:37.404 INFO:teuthology.orchestra.run.smithi006.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-08-27T21:43:37.510 INFO:journalctl@ceph.mgr.y.smithi006.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-08-27T21:43:37.521 INFO:journalctl@ceph.mon.a.smithi006.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-08-27T21:43:37.624 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/docker: pacific: Pulling from ceph-ci/ceph 2023-08-27T21:43:37.662 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/docker: Digest: sha256:e2028333a2006b532dd5e4caab422f750a165963636ee26acf78c4fa6593c513 2023-08-27T21:43:37.663 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/docker: Status: Image is up to date for quay.ceph.io/ceph-ci/ceph:pacific 2023-08-27T21:43:37.695 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/docker: quay.ceph.io/ceph-ci/ceph:pacific 2023-08-27T21:43:38.798 INFO:teuthology.orchestra.run.smithi006.stderr:ceph: ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable) 2023-08-27T21:43:39.355 INFO:teuthology.orchestra.run.smithi006.stderr:Ceph version: ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable) 2023-08-27T21:43:39.355 INFO:teuthology.orchestra.run.smithi006.stderr:Extracting ceph user uid/gid from container image... 2023-08-27T21:43:40.230 INFO:teuthology.orchestra.run.smithi006.stderr:stat: 167 167 2023-08-27T21:43:40.827 INFO:teuthology.orchestra.run.smithi006.stderr:Creating initial keys... 2023-08-27T21:43:41.733 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-authtool: AQCNw+tk7pl3KxAAgsyZSg58SK5L5T1mEt+fvg== 2023-08-27T21:43:43.212 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-authtool: AQCPw+tkP1xlDBAAlSnmBCIBJKfgrfptEOR+VQ== 2023-08-27T21:43:44.713 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-authtool: AQCQw+tkLjJHKhAA3CvpA8E7dXG7XVFYDo8Phg== 2023-08-27T21:43:45.385 INFO:teuthology.orchestra.run.smithi006.stderr:Creating initial monmap... 2023-08-27T21:43:46.277 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2023-08-27T21:43:46.277 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to b696f380-4522-11ee-817b-1b7a5f1ba315 2023-08-27T21:43:46.277 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-08-27T21:43:46.831 INFO:teuthology.orchestra.run.smithi006.stderr:monmaptool for a [v2:172.21.15.6:3300,v1:172.21.15.6:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-08-27T21:43:46.831 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/monmaptool: set fsid to b696f380-4522-11ee-817b-1b7a5f1ba315 2023-08-27T21:43:46.831 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-08-27T21:43:46.831 INFO:teuthology.orchestra.run.smithi006.stderr: 2023-08-27T21:43:46.831 INFO:teuthology.orchestra.run.smithi006.stderr:Creating mon... 2023-08-27T21:43:47.724 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.716+0000 7f73609b7880 0 set uid:gid to 167:167 (ceph:ceph) 2023-08-27T21:43:47.725 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.716+0000 7f73609b7880 1 imported monmap: 2023-08-27T21:43:47.725 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: epoch 0 2023-08-27T21:43:47.725 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: fsid b696f380-4522-11ee-817b-1b7a5f1ba315 2023-08-27T21:43:47.726 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: last_changed 2023-08-27T21:43:46.272349+0000 2023-08-27T21:43:47.726 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: created 2023-08-27T21:43:46.272349+0000 2023-08-27T21:43:47.726 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: min_mon_release 0 (unknown) 2023-08-27T21:43:47.726 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: election_strategy: 1 2023-08-27T21:43:47.727 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: 0: [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] mon.a 2023-08-27T21:43:47.727 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: 2023-08-27T21:43:47.727 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.716+0000 7f73609b7880 0 /usr/bin/ceph-mon: set fsid to b696f380-4522-11ee-817b-1b7a5f1ba315 2023-08-27T21:43:47.728 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: RocksDB version: 6.8.1 2023-08-27T21:43:47.728 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: 2023-08-27T21:43:47.728 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-08-27T21:43:47.728 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Compile date Aug 22 2023 2023-08-27T21:43:47.728 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: DB SUMMARY 2023-08-27T21:43:47.728 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: 2023-08-27T21:43:47.728 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2023-08-27T21:43:47.728 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: 2023-08-27T21:43:47.736 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2023-08-27T21:43:47.736 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: 2023-08-27T21:43:47.736 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.error_if_exists: 0 2023-08-27T21:43:47.736 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.create_if_missing: 1 2023-08-27T21:43:47.736 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.paranoid_checks: 1 2023-08-27T21:43:47.736 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.env: 0x5633aafc2080 2023-08-27T21:43:47.736 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.fs: Posix File System 2023-08-27T21:43:47.736 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.info_log: 0x5633ac579aa0 2023-08-27T21:43:47.736 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_file_opening_threads: 16 2023-08-27T21:43:47.736 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.statistics: (nil) 2023-08-27T21:43:47.737 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.use_fsync: 0 2023-08-27T21:43:47.737 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_log_file_size: 0 2023-08-27T21:43:47.737 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-08-27T21:43:47.737 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-08-27T21:43:47.737 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.keep_log_file_num: 1000 2023-08-27T21:43:47.737 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.recycle_log_file_num: 0 2023-08-27T21:43:47.737 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.allow_fallocate: 1 2023-08-27T21:43:47.737 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.allow_mmap_reads: 0 2023-08-27T21:43:47.737 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.allow_mmap_writes: 0 2023-08-27T21:43:47.737 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.use_direct_reads: 0 2023-08-27T21:43:47.737 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-08-27T21:43:47.737 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.create_missing_column_families: 0 2023-08-27T21:43:47.738 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.db_log_dir: 2023-08-27T21:43:47.738 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-08-27T21:43:47.738 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-08-27T21:43:47.738 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_subcompactions: 1 2023-08-27T21:43:47.738 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_background_flushes: -1 2023-08-27T21:43:47.738 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-08-27T21:43:47.738 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-08-27T21:43:47.738 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-08-27T21:43:47.738 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-08-27T21:43:47.738 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-08-27T21:43:47.738 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.advise_random_on_open: 1 2023-08-27T21:43:47.738 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.db_write_buffer_size: 0 2023-08-27T21:43:47.738 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.write_buffer_manager: 0x5633ac581890 2023-08-27T21:43:47.739 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-08-27T21:43:47.739 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-08-27T21:43:47.739 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-08-27T21:43:47.739 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-08-27T21:43:47.739 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.rate_limiter: (nil) 2023-08-27T21:43:47.739 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-08-27T21:43:47.739 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.wal_recovery_mode: 2 2023-08-27T21:43:47.739 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.enable_thread_tracking: 0 2023-08-27T21:43:47.739 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.enable_pipelined_write: 0 2023-08-27T21:43:47.739 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.unordered_write: 0 2023-08-27T21:43:47.739 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-08-27T21:43:47.739 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-08-27T21:43:47.740 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-08-27T21:43:47.740 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-08-27T21:43:47.740 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.row_cache: None 2023-08-27T21:43:47.740 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.wal_filter: None 2023-08-27T21:43:47.740 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-08-27T21:43:47.740 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.allow_ingest_behind: 0 2023-08-27T21:43:47.740 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.preserve_deletes: 0 2023-08-27T21:43:47.740 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.two_write_queues: 0 2023-08-27T21:43:47.740 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.manual_wal_flush: 0 2023-08-27T21:43:47.741 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.atomic_flush: 0 2023-08-27T21:43:47.741 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-08-27T21:43:47.741 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-08-27T21:43:47.741 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-08-27T21:43:47.741 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.log_readahead_size: 0 2023-08-27T21:43:47.741 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-08-27T21:43:47.741 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_background_jobs: 2 2023-08-27T21:43:47.741 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_background_compactions: -1 2023-08-27T21:43:47.741 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-08-27T21:43:47.741 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-08-27T21:43:47.741 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-08-27T21:43:47.741 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_total_wal_size: 0 2023-08-27T21:43:47.741 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-08-27T21:43:47.742 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-08-27T21:43:47.742 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-08-27T21:43:47.742 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-08-27T21:43:47.742 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_open_files: -1 2023-08-27T21:43:47.742 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.bytes_per_sync: 0 2023-08-27T21:43:47.742 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-08-27T21:43:47.759 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-08-27T21:43:47.759 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.compaction_readahead_size: 0 2023-08-27T21:43:47.759 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Compression algorithms supported: 2023-08-27T21:43:47.759 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-08-27T21:43:47.759 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: kZSTD supported: 0 2023-08-27T21:43:47.759 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: kXpressCompression supported: 0 2023-08-27T21:43:47.759 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: kLZ4HCCompression supported: 1 2023-08-27T21:43:47.759 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: kLZ4Compression supported: 1 2023-08-27T21:43:47.759 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: kBZip2Compression supported: 0 2023-08-27T21:43:47.759 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: kZlibCompression supported: 1 2023-08-27T21:43:47.760 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: kSnappyCompression supported: 1 2023-08-27T21:43:47.760 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-08-27T21:43:47.760 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: [db_impl/db_impl_open.cc:273] Creating manifest 1 2023-08-27T21:43:47.760 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: 2023-08-27T21:43:47.760 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2023-08-27T21:43:47.760 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: 2023-08-27T21:43:47.760 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-08-27T21:43:47.760 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: 2023-08-27T21:43:47.760 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-08-27T21:43:47.760 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.merge_operator: 2023-08-27T21:43:47.760 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.compaction_filter: None 2023-08-27T21:43:47.760 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.compaction_filter_factory: None 2023-08-27T21:43:47.760 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-08-27T21:43:47.761 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-08-27T21:43:47.761 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5633ac48fd20) 2023-08-27T21:43:47.761 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks: 1 2023-08-27T21:43:47.761 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks_with_high_priority: 0 2023-08-27T21:43:47.761 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: pin_l0_filter_and_index_blocks_in_cache: 0 2023-08-27T21:43:47.761 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: pin_top_level_index_and_filter: 1 2023-08-27T21:43:47.761 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: index_type: 0 2023-08-27T21:43:47.761 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: data_block_index_type: 0 2023-08-27T21:43:47.761 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: index_shortening: 1 2023-08-27T21:43:47.761 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: data_block_hash_table_util_ratio: 0.750000 2023-08-27T21:43:47.761 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: hash_index_allow_collision: 1 2023-08-27T21:43:47.761 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: checksum: 1 2023-08-27T21:43:47.761 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: no_block_cache: 0 2023-08-27T21:43:47.762 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: block_cache: 0x5633ac4c6d10 2023-08-27T21:43:47.762 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: block_cache_name: BinnedLRUCache 2023-08-27T21:43:47.762 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: block_cache_options: 2023-08-27T21:43:47.762 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: capacity : 536870912 2023-08-27T21:43:47.762 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: num_shard_bits : 4 2023-08-27T21:43:47.762 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: strict_capacity_limit : 0 2023-08-27T21:43:47.762 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: high_pri_pool_ratio: 0.000 2023-08-27T21:43:47.762 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: block_cache_compressed: (nil) 2023-08-27T21:43:47.762 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: persistent_cache: (nil) 2023-08-27T21:43:47.762 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: block_size: 4096 2023-08-27T21:43:47.762 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: block_size_deviation: 10 2023-08-27T21:43:47.762 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: block_restart_interval: 16 2023-08-27T21:43:47.763 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: index_block_restart_interval: 1 2023-08-27T21:43:47.763 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: metadata_block_size: 4096 2023-08-27T21:43:47.763 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: partition_filters: 0 2023-08-27T21:43:47.763 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: use_delta_encoding: 1 2023-08-27T21:43:47.763 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: filter_policy: rocksdb.BuiltinBloomFilter 2023-08-27T21:43:47.763 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: whole_key_filtering: 1 2023-08-27T21:43:47.763 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: verify_compression: 0 2023-08-27T21:43:47.763 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: read_amp_bytes_per_bit: 0 2023-08-27T21:43:47.763 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: format_version: 2 2023-08-27T21:43:47.763 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: enable_index_compression: 1 2023-08-27T21:43:47.763 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: block_align: 0 2023-08-27T21:43:47.763 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: 2023-08-27T21:43:47.763 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.write_buffer_size: 33554432 2023-08-27T21:43:47.764 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_write_buffer_number: 2 2023-08-27T21:43:47.764 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.compression: NoCompression 2023-08-27T21:43:47.764 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.bottommost_compression: Disabled 2023-08-27T21:43:47.764 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.prefix_extractor: nullptr 2023-08-27T21:43:47.764 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-08-27T21:43:47.764 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.num_levels: 7 2023-08-27T21:43:47.764 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-08-27T21:43:47.764 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-08-27T21:43:47.764 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-08-27T21:43:47.764 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-08-27T21:43:47.764 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-08-27T21:43:47.764 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-08-27T21:43:47.765 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-08-27T21:43:47.765 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-08-27T21:43:47.765 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-08-27T21:43:47.765 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-08-27T21:43:47.765 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.compression_opts.level: 32767 2023-08-27T21:43:47.765 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.compression_opts.strategy: 0 2023-08-27T21:43:47.765 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-08-27T21:43:47.765 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-08-27T21:43:47.765 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.compression_opts.enabled: false 2023-08-27T21:43:47.765 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-08-27T21:43:47.765 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-08-27T21:43:47.765 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-08-27T21:43:47.766 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.target_file_size_base: 67108864 2023-08-27T21:43:47.766 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-08-27T21:43:47.766 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-08-27T21:43:47.766 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-08-27T21:43:47.766 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-08-27T21:43:47.766 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-08-27T21:43:47.766 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-08-27T21:43:47.766 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-08-27T21:43:47.766 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-08-27T21:43:47.766 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-08-27T21:43:47.766 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-08-27T21:43:47.766 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-08-27T21:43:47.766 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-08-27T21:43:47.767 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-08-27T21:43:47.767 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.arena_block_size: 4194304 2023-08-27T21:43:47.767 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-08-27T21:43:47.767 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-08-27T21:43:47.767 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-08-27T21:43:47.767 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.disable_auto_compactions: 0 2023-08-27T21:43:47.767 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-08-27T21:43:47.767 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-08-27T21:43:47.767 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-08-27T21:43:47.767 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-08-27T21:43:47.767 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-08-27T21:43:47.767 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-08-27T21:43:47.768 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-08-27T21:43:47.768 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-08-27T21:43:47.768 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-08-27T21:43:47.768 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-08-27T21:43:47.768 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.table_properties_collectors: 2023-08-27T21:43:47.768 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.inplace_update_support: 0 2023-08-27T21:43:47.768 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-08-27T21:43:47.768 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-08-27T21:43:47.768 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-08-27T21:43:47.768 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-08-27T21:43:47.768 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.bloom_locality: 0 2023-08-27T21:43:47.768 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.max_successive_merges: 0 2023-08-27T21:43:47.769 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-08-27T21:43:47.769 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.paranoid_file_checks: 0 2023-08-27T21:43:47.769 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.force_consistency_checks: 0 2023-08-27T21:43:47.769 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.report_bg_io_stats: 0 2023-08-27T21:43:47.769 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.ttl: 2592000 2023-08-27T21:43:47.769 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-08-27T21:43:47.769 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-08-27T21:43:47.769 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: 2023-08-27T21:43:47.769 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.720+0000 7f73609b7880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-08-27T21:43:47.769 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: 2023-08-27T21:43:47.769 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.724+0000 7f73609b7880 4 rocksdb: DB pointer 0x5633ac58f800 2023-08-27T21:43:47.769 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.724+0000 7f734983f700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-08-27T21:43:47.769 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.724+0000 7f734983f700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-08-27T21:43:47.770 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: ** DB Stats ** 2023-08-27T21:43:47.770 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-08-27T21:43:47.770 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-08-27T21:43:47.770 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-08-27T21:43:47.770 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-27T21:43:47.770 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-08-27T21:43:47.770 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-08-27T21:43:47.770 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-27T21:43:47.770 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: 2023-08-27T21:43:47.770 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-08-27T21:43:47.770 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-27T21:43:47.770 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-27T21:43:47.771 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-08-27T21:43:47.771 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-08-27T21:43:47.771 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: 2023-08-27T21:43:47.771 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-08-27T21:43:47.771 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-27T21:43:47.771 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-27T21:43:47.771 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-08-27T21:43:47.771 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-08-27T21:43:47.771 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-08-27T21:43:47.771 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-08-27T21:43:47.771 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-08-27T21:43:47.771 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-08-27T21:43:47.772 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-27T21:43:47.772 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-27T21:43:47.772 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-08-27T21:43:47.772 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: 2023-08-27T21:43:47.772 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-08-27T21:43:47.772 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: 2023-08-27T21:43:47.772 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-08-27T21:43:47.772 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-27T21:43:47.772 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-27T21:43:47.772 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-08-27T21:43:47.772 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-08-27T21:43:47.772 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: 2023-08-27T21:43:47.772 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-08-27T21:43:47.773 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-27T21:43:47.773 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-27T21:43:47.773 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-08-27T21:43:47.773 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-08-27T21:43:47.773 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-08-27T21:43:47.773 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-08-27T21:43:47.773 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-08-27T21:43:47.773 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-08-27T21:43:47.773 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-27T21:43:47.773 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-27T21:43:47.773 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-08-27T21:43:47.773 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: 2023-08-27T21:43:47.774 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-08-27T21:43:47.774 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: 2023-08-27T21:43:47.774 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.724+0000 7f73609b7880 4 rocksdb: [db_impl/db_impl.cc:397] Shutdown: canceling all background work 2023-08-27T21:43:47.774 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.724+0000 7f73609b7880 4 rocksdb: [db_impl/db_impl.cc:573] Shutdown complete 2023-08-27T21:43:47.774 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-mon: debug 2023-08-27T21:43:47.724+0000 7f73609b7880 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2023-08-27T21:43:48.353 INFO:teuthology.orchestra.run.smithi006.stderr:create mon.a on 2023-08-27T21:43:48.378 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-08-27T21:43:48.608 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /etc/systemd/system/ceph.target. 2023-08-27T21:43:48.817 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-b696f380-4522-11ee-817b-1b7a5f1ba315.target -> /etc/systemd/system/ceph-b696f380-4522-11ee-817b-1b7a5f1ba315.target. 2023-08-27T21:43:48.817 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-b696f380-4522-11ee-817b-1b7a5f1ba315.target -> /etc/systemd/system/ceph-b696f380-4522-11ee-817b-1b7a5f1ba315.target. 2023-08-27T21:43:49.263 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Failed to reset failed state of unit ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@mon.a.service: Unit ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@mon.a.service not loaded. 2023-08-27T21:43:49.273 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Created symlink /etc/systemd/system/ceph-b696f380-4522-11ee-817b-1b7a5f1ba315.target.wants/ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@mon.a.service -> /etc/systemd/system/ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@.service. 2023-08-27T21:43:49.478 INFO:teuthology.orchestra.run.smithi006.stderr:firewalld does not appear to be present 2023-08-27T21:43:49.479 INFO:teuthology.orchestra.run.smithi006.stderr:Not possible to enable service . firewalld.service is not available 2023-08-27T21:43:49.479 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for mon to start... 2023-08-27T21:43:49.479 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for mon... 2023-08-27T21:43:51.196 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: cluster: 2023-08-27T21:43:51.196 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: id: b696f380-4522-11ee-817b-1b7a5f1ba315 2023-08-27T21:43:51.196 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: health: HEALTH_OK 2023-08-27T21:43:51.197 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2023-08-27T21:43:51.197 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: services: 2023-08-27T21:43:51.197 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.173755s) 2023-08-27T21:43:51.198 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mgr: no daemons active 2023-08-27T21:43:51.198 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2023-08-27T21:43:51.198 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2023-08-27T21:43:51.198 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: data: 2023-08-27T21:43:51.198 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2023-08-27T21:43:51.198 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2023-08-27T21:43:51.199 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2023-08-27T21:43:51.199 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: pgs: 2023-08-27T21:43:51.199 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2023-08-27T21:43:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:51 smithi006 bash[26747]: cluster 2023-08-27T21:43:50.989824+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-08-27T21:43:51.788 INFO:teuthology.orchestra.run.smithi006.stderr:mon is available 2023-08-27T21:43:51.788 INFO:teuthology.orchestra.run.smithi006.stderr:Assimilating anything we can from ceph.conf... 2023-08-27T21:43:52.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:52 smithi006 bash[26747]: cluster 2023-08-27T21:43:51.017646+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-08-27T21:43:52.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:52 smithi006 bash[26747]: cluster 2023-08-27T21:43:51.018014+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0]} removed_ranks: {} 2023-08-27T21:43:52.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:52 smithi006 bash[26747]: cluster 2023-08-27T21:43:51.026113+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2023-08-27T21:43:52.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:52 smithi006 bash[26747]: cluster 2023-08-27T21:43:51.029852+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-08-27T21:43:52.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:52 smithi006 bash[26747]: cluster 2023-08-27T21:43:51.030390+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2023-08-27T21:43:52.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:52 smithi006 bash[26747]: audit 2023-08-27T21:43:51.191310+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.6:0/241626155' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-08-27T21:43:52.928 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2023-08-27T21:43:52.928 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: [global] 2023-08-27T21:43:52.929 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: fsid = b696f380-4522-11ee-817b-1b7a5f1ba315 2023-08-27T21:43:52.929 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: log_to_journald = false 2023-08-27T21:43:52.929 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.6:3300,v1:172.21.15.6:6789] 2023-08-27T21:43:52.929 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-08-27T21:43:52.929 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-08-27T21:43:52.929 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-08-27T21:43:52.929 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-08-27T21:43:52.929 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2023-08-27T21:43:52.929 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: [mgr] 2023-08-27T21:43:52.929 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-08-27T21:43:52.929 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2023-08-27T21:43:52.929 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: [osd] 2023-08-27T21:43:52.929 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-08-27T21:43:52.930 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-08-27T21:43:53.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:53 smithi006 bash[26747]: audit 2023-08-27T21:43:52.915398+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.6:0/3719127515' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-08-27T21:43:53.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:53 smithi006 bash[26747]: audit 2023-08-27T21:43:52.922401+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.6:0/3719127515' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-08-27T21:43:53.494 INFO:teuthology.orchestra.run.smithi006.stderr:Generating new minimal ceph.conf... 2023-08-27T21:43:54.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:54 smithi006 bash[26747]: audit 2023-08-27T21:43:54.537527+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.6:0/3710419064' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:43:55.192 INFO:teuthology.orchestra.run.smithi006.stderr:Restarting the monitor... 2023-08-27T21:43:55.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:55 smithi006 systemd[1]: Stopping Ceph mon.a for b696f380-4522-11ee-817b-1b7a5f1ba315... 2023-08-27T21:43:55.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:55 smithi006 bash[27144]: Error response from daemon: No such container: ceph-b696f380-4522-11ee-817b-1b7a5f1ba315-mon.a 2023-08-27T21:43:55.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:55 smithi006 bash[26747]: debug 2023-08-27T21:43:55.404+0000 7f3c5f5dc700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-08-27T21:43:55.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:55 smithi006 bash[26747]: debug 2023-08-27T21:43:55.404+0000 7f3c5f5dc700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-08-27T21:43:56.092 INFO:teuthology.orchestra.run.smithi006.stderr:Setting mon public_network to 172.21.0.0/20 2023-08-27T21:43:56.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:55 smithi006 bash[27185]: ceph-b696f380-4522-11ee-817b-1b7a5f1ba315-mon-a 2023-08-27T21:43:56.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:56 smithi006 systemd[1]: ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@mon.a.service: Succeeded. 2023-08-27T21:43:56.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:56 smithi006 systemd[1]: Stopped Ceph mon.a for b696f380-4522-11ee-817b-1b7a5f1ba315. 2023-08-27T21:43:56.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:56 smithi006 systemd[1]: Started Ceph mon.a for b696f380-4522-11ee-817b-1b7a5f1ba315. 2023-08-27T21:43:57.384 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.268+0000 7f97d6407880 0 set uid:gid to 167:167 (ceph:ceph) 2023-08-27T21:43:57.385 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.268+0000 7f97d6407880 0 ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable), process ceph-mon, pid 7 2023-08-27T21:43:57.385 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.268+0000 7f97d6407880 0 pidfile_write: ignore empty --pid-file 2023-08-27T21:43:57.385 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 0 load: jerasure load: lrc load: isa 2023-08-27T21:43:57.385 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: RocksDB version: 6.8.1 2023-08-27T21:43:57.386 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-08-27T21:43:57.386 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Compile date Aug 22 2023 2023-08-27T21:43:57.386 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: DB SUMMARY 2023-08-27T21:43:57.386 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: CURRENT file: CURRENT 2023-08-27T21:43:57.386 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: IDENTITY file: IDENTITY 2023-08-27T21:43:57.386 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 131 Bytes 2023-08-27T21:43:57.386 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000004.sst 2023-08-27T21:43:57.386 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000006.log size: 86673 ; 2023-08-27T21:43:57.386 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.error_if_exists: 0 2023-08-27T21:43:57.386 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.create_if_missing: 0 2023-08-27T21:43:57.386 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.paranoid_checks: 1 2023-08-27T21:43:57.387 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.env: 0x558f201c2080 2023-08-27T21:43:57.387 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.fs: Posix File System 2023-08-27T21:43:57.387 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.info_log: 0x558f22617a80 2023-08-27T21:43:57.387 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_file_opening_threads: 16 2023-08-27T21:43:57.387 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.statistics: (nil) 2023-08-27T21:43:57.387 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.use_fsync: 0 2023-08-27T21:43:57.387 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_log_file_size: 0 2023-08-27T21:43:57.387 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-08-27T21:43:57.387 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-08-27T21:43:57.387 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.keep_log_file_num: 1000 2023-08-27T21:43:57.387 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.recycle_log_file_num: 0 2023-08-27T21:43:57.389 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.allow_fallocate: 1 2023-08-27T21:43:57.389 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.allow_mmap_reads: 0 2023-08-27T21:43:57.389 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.allow_mmap_writes: 0 2023-08-27T21:43:57.389 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.use_direct_reads: 0 2023-08-27T21:43:57.389 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-08-27T21:43:57.389 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.create_missing_column_families: 0 2023-08-27T21:43:57.389 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.db_log_dir: 2023-08-27T21:43:57.389 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-08-27T21:43:57.389 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-08-27T21:43:57.390 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_subcompactions: 1 2023-08-27T21:43:57.390 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_background_flushes: -1 2023-08-27T21:43:57.390 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-08-27T21:43:57.390 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-08-27T21:43:57.390 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-08-27T21:43:57.390 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-08-27T21:43:57.390 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-08-27T21:43:57.390 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.advise_random_on_open: 1 2023-08-27T21:43:57.390 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.db_write_buffer_size: 0 2023-08-27T21:43:57.390 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.write_buffer_manager: 0x558f23314330 2023-08-27T21:43:57.390 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-08-27T21:43:57.390 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-08-27T21:43:57.391 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-08-27T21:43:57.391 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-08-27T21:43:57.391 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.rate_limiter: (nil) 2023-08-27T21:43:57.391 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-08-27T21:43:57.391 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.wal_recovery_mode: 2 2023-08-27T21:43:57.392 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.enable_thread_tracking: 0 2023-08-27T21:43:57.392 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.enable_pipelined_write: 0 2023-08-27T21:43:57.392 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.unordered_write: 0 2023-08-27T21:43:57.392 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-08-27T21:43:57.392 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-08-27T21:43:57.392 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-08-27T21:43:57.392 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-08-27T21:43:57.393 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.row_cache: None 2023-08-27T21:43:57.393 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.wal_filter: None 2023-08-27T21:43:57.393 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-08-27T21:43:57.393 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.allow_ingest_behind: 0 2023-08-27T21:43:57.393 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.preserve_deletes: 0 2023-08-27T21:43:57.393 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.two_write_queues: 0 2023-08-27T21:43:57.394 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.manual_wal_flush: 0 2023-08-27T21:43:57.394 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.atomic_flush: 0 2023-08-27T21:43:57.394 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-08-27T21:43:57.394 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-08-27T21:43:57.394 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-08-27T21:43:57.394 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.log_readahead_size: 0 2023-08-27T21:43:57.394 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-08-27T21:43:57.395 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_background_jobs: 2 2023-08-27T21:43:57.395 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_background_compactions: -1 2023-08-27T21:43:57.395 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-08-27T21:43:57.395 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-08-27T21:43:57.395 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-08-27T21:43:57.395 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_total_wal_size: 0 2023-08-27T21:43:57.395 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-08-27T21:43:57.396 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-08-27T21:43:57.396 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-08-27T21:43:57.396 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-08-27T21:43:57.396 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_open_files: -1 2023-08-27T21:43:57.396 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.bytes_per_sync: 0 2023-08-27T21:43:57.396 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-08-27T21:43:57.396 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-08-27T21:43:57.397 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.compaction_readahead_size: 0 2023-08-27T21:43:57.397 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Compression algorithms supported: 2023-08-27T21:43:57.397 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-08-27T21:43:57.397 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: kZSTD supported: 0 2023-08-27T21:43:57.397 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: kXpressCompression supported: 0 2023-08-27T21:43:57.397 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: kLZ4HCCompression supported: 1 2023-08-27T21:43:57.397 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: kLZ4Compression supported: 1 2023-08-27T21:43:57.398 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: kBZip2Compression supported: 0 2023-08-27T21:43:57.398 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: kZlibCompression supported: 1 2023-08-27T21:43:57.398 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: kSnappyCompression supported: 1 2023-08-27T21:43:57.398 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-08-27T21:43:57.398 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 2023-08-27T21:43:57.398 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-08-27T21:43:57.398 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-08-27T21:43:57.399 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.merge_operator: 2023-08-27T21:43:57.399 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.compaction_filter: None 2023-08-27T21:43:57.399 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.compaction_filter_factory: None 2023-08-27T21:43:57.399 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-08-27T21:43:57.400 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-08-27T21:43:57.400 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558f225c5d18) 2023-08-27T21:43:57.400 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: cache_index_and_filter_blocks: 1 2023-08-27T21:43:57.400 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: cache_index_and_filter_blocks_with_high_priority: 0 2023-08-27T21:43:57.400 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-08-27T21:43:57.401 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: pin_top_level_index_and_filter: 1 2023-08-27T21:43:57.401 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: index_type: 0 2023-08-27T21:43:57.401 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: data_block_index_type: 0 2023-08-27T21:43:57.401 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: index_shortening: 1 2023-08-27T21:43:57.401 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: data_block_hash_table_util_ratio: 0.750000 2023-08-27T21:43:57.401 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: hash_index_allow_collision: 1 2023-08-27T21:43:57.401 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: checksum: 1 2023-08-27T21:43:57.401 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: no_block_cache: 0 2023-08-27T21:43:57.402 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: block_cache: 0x558f225fcf10 2023-08-27T21:43:57.402 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: block_cache_name: BinnedLRUCache 2023-08-27T21:43:57.402 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: block_cache_options: 2023-08-27T21:43:57.402 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: capacity : 536870912 2023-08-27T21:43:57.402 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: num_shard_bits : 4 2023-08-27T21:43:57.402 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: strict_capacity_limit : 0 2023-08-27T21:43:57.402 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: high_pri_pool_ratio: 0.000 2023-08-27T21:43:57.402 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: block_cache_compressed: (nil) 2023-08-27T21:43:57.402 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: persistent_cache: (nil) 2023-08-27T21:43:57.402 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: block_size: 4096 2023-08-27T21:43:57.403 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: block_size_deviation: 10 2023-08-27T21:43:57.403 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: block_restart_interval: 16 2023-08-27T21:43:57.403 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: index_block_restart_interval: 1 2023-08-27T21:43:57.403 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: metadata_block_size: 4096 2023-08-27T21:43:57.403 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: partition_filters: 0 2023-08-27T21:43:57.403 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: use_delta_encoding: 1 2023-08-27T21:43:57.404 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: filter_policy: rocksdb.BuiltinBloomFilter 2023-08-27T21:43:57.404 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: whole_key_filtering: 1 2023-08-27T21:43:57.404 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: verify_compression: 0 2023-08-27T21:43:57.404 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: read_amp_bytes_per_bit: 0 2023-08-27T21:43:57.404 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: format_version: 2 2023-08-27T21:43:57.404 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: enable_index_compression: 1 2023-08-27T21:43:57.404 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: block_align: 0 2023-08-27T21:43:57.405 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.write_buffer_size: 33554432 2023-08-27T21:43:57.405 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_write_buffer_number: 2 2023-08-27T21:43:57.405 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.compression: NoCompression 2023-08-27T21:43:57.405 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.bottommost_compression: Disabled 2023-08-27T21:43:57.405 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.prefix_extractor: nullptr 2023-08-27T21:43:57.405 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-08-27T21:43:57.406 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.num_levels: 7 2023-08-27T21:43:57.406 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-08-27T21:43:57.406 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-08-27T21:43:57.406 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-08-27T21:43:57.406 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-08-27T21:43:57.406 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-08-27T21:43:57.406 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-08-27T21:43:57.407 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-08-27T21:43:57.407 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-08-27T21:43:57.407 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-08-27T21:43:57.407 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-08-27T21:43:57.407 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.compression_opts.level: 32767 2023-08-27T21:43:57.407 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.compression_opts.strategy: 0 2023-08-27T21:43:57.407 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-08-27T21:43:57.408 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-08-27T21:43:57.408 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.compression_opts.enabled: false 2023-08-27T21:43:57.408 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-08-27T21:43:57.408 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-08-27T21:43:57.408 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-08-27T21:43:57.408 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.target_file_size_base: 67108864 2023-08-27T21:43:57.408 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-08-27T21:43:57.409 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-08-27T21:43:57.409 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-08-27T21:43:57.409 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-08-27T21:43:57.409 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-08-27T21:43:57.409 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-08-27T21:43:57.409 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-08-27T21:43:57.410 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-08-27T21:43:57.410 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-08-27T21:43:57.410 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-08-27T21:43:57.410 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-08-27T21:43:57.410 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-08-27T21:43:57.410 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-08-27T21:43:57.410 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.arena_block_size: 4194304 2023-08-27T21:43:57.411 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-08-27T21:43:57.411 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-08-27T21:43:57.411 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-08-27T21:43:57.411 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.disable_auto_compactions: 0 2023-08-27T21:43:57.411 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-08-27T21:43:57.411 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-08-27T21:43:57.411 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-08-27T21:43:57.411 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-08-27T21:43:57.411 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-08-27T21:43:57.412 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-08-27T21:43:57.412 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-08-27T21:43:57.412 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-08-27T21:43:57.412 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-08-27T21:43:57.412 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-08-27T21:43:57.412 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.table_properties_collectors: 2023-08-27T21:43:57.412 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.inplace_update_support: 0 2023-08-27T21:43:57.412 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-08-27T21:43:57.412 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-08-27T21:43:57.412 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-08-27T21:43:57.413 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-08-27T21:43:57.413 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.bloom_locality: 0 2023-08-27T21:43:57.413 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.max_successive_merges: 0 2023-08-27T21:43:57.413 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-08-27T21:43:57.413 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.paranoid_file_checks: 0 2023-08-27T21:43:57.413 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.force_consistency_checks: 0 2023-08-27T21:43:57.413 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.report_bg_io_stats: 0 2023-08-27T21:43:57.413 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.ttl: 2592000 2023-08-27T21:43:57.413 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-08-27T21:43:57.413 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-08-27T21:43:57.413 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 4 2023-08-27T21:43:57.414 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1693172637276698, "job": 1, "event": "recovery_started", "log_files": [6]} 2023-08-27T21:43:57.414 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.272+0000 7f97d6407880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #6 mode 2 2023-08-27T21:43:57.414 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.276+0000 7f97d6407880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-08-27T21:43:57.414 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.276+0000 7f97d6407880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1693172637278177, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 83273, "table_properties": {"data_size": 81546, "index_size": 332, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 581, "raw_key_size": 9829, "raw_average_key_size": 47, "raw_value_size": 75867, "raw_average_value_size": 368, "num_data_blocks": 11, "num_entries": 206, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1693172637, "oldest_key_time": 3, "file_creation_time": 0}} 2023-08-27T21:43:57.414 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.276+0000 7f97d6407880 4 rocksdb: [version_set.cc:3826] Creating manifest 8 2023-08-27T21:43:57.414 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.276+0000 7f97d6407880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1693172637278916, "job": 1, "event": "recovery_finished"} 2023-08-27T21:43:57.414 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.276+0000 7f97d6407880 4 rocksdb: DB pointer 0x558f226c5800 2023-08-27T21:43:57.414 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.276+0000 7f97bd20e700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-08-27T21:43:57.414 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.280+0000 7f97bd20e700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-08-27T21:43:57.414 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: ** DB Stats ** 2023-08-27T21:43:57.414 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Uptime(secs): 0.0 total, 0.0 interval 2023-08-27T21:43:57.414 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-08-27T21:43:57.415 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-08-27T21:43:57.415 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-27T21:43:57.415 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-08-27T21:43:57.415 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-08-27T21:43:57.415 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-27T21:43:57.415 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: ** Compaction Stats [default] ** 2023-08-27T21:43:57.415 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-27T21:43:57.415 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-27T21:43:57.415 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: L0 2/0 82.98 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 93.3 0.00 0.00 1 0.001 0 0 2023-08-27T21:43:57.416 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Sum 2/0 82.98 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 93.3 0.00 0.00 1 0.001 0 0 2023-08-27T21:43:57.416 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 93.3 0.00 0.00 1 0.001 0 0 2023-08-27T21:43:57.416 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: ** Compaction Stats [default] ** 2023-08-27T21:43:57.416 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-27T21:43:57.416 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-27T21:43:57.416 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 93.3 0.00 0.00 1 0.001 0 0 2023-08-27T21:43:57.416 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Uptime(secs): 0.0 total, 0.0 interval 2023-08-27T21:43:57.417 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Flush(GB): cumulative 0.000, interval 0.000 2023-08-27T21:43:57.417 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: AddFile(GB): cumulative 0.000, interval 0.000 2023-08-27T21:43:57.417 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: AddFile(Total Files): cumulative 0, interval 0 2023-08-27T21:43:57.417 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: AddFile(L0 Files): cumulative 0, interval 0 2023-08-27T21:43:57.417 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: AddFile(Keys): cumulative 0, interval 0 2023-08-27T21:43:57.417 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Cumulative compaction: 0.00 GB write, 14.87 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-27T21:43:57.417 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Interval compaction: 0.00 GB write, 14.87 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-27T21:43:57.418 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-08-27T21:43:57.418 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: ** File Read Latency Histogram By Level [default] ** 2023-08-27T21:43:57.418 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: ** Compaction Stats [default] ** 2023-08-27T21:43:57.418 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-27T21:43:57.418 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-27T21:43:57.418 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: L0 2/0 82.98 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 93.3 0.00 0.00 1 0.001 0 0 2023-08-27T21:43:57.418 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Sum 2/0 82.98 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 93.3 0.00 0.00 1 0.001 0 0 2023-08-27T21:43:57.419 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-08-27T21:43:57.419 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: ** Compaction Stats [default] ** 2023-08-27T21:43:57.420 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-27T21:43:57.420 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-27T21:43:57.420 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 93.3 0.00 0.00 1 0.001 0 0 2023-08-27T21:43:57.420 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Uptime(secs): 0.0 total, 0.0 interval 2023-08-27T21:43:57.420 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Flush(GB): cumulative 0.000, interval 0.000 2023-08-27T21:43:57.420 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: AddFile(GB): cumulative 0.000, interval 0.000 2023-08-27T21:43:57.420 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: AddFile(Total Files): cumulative 0, interval 0 2023-08-27T21:43:57.421 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: AddFile(L0 Files): cumulative 0, interval 0 2023-08-27T21:43:57.421 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: AddFile(Keys): cumulative 0, interval 0 2023-08-27T21:43:57.421 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Cumulative compaction: 0.00 GB write, 14.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-27T21:43:57.421 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-27T21:43:57.421 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-08-27T21:43:57.421 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: ** File Read Latency Histogram By Level [default] ** 2023-08-27T21:43:57.422 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.280+0000 7f97d6407880 0 starting mon.a rank 0 at public addrs [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] at bind addrs [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid b696f380-4522-11ee-817b-1b7a5f1ba315 2023-08-27T21:43:57.422 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.280+0000 7f97d6407880 1 mon.a@-1(???) e1 preinit fsid b696f380-4522-11ee-817b-1b7a5f1ba315 2023-08-27T21:43:57.422 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.280+0000 7f97d6407880 0 mon.a@-1(???).mds e1 new map 2023-08-27T21:43:57.422 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.280+0000 7f97d6407880 0 mon.a@-1(???).mds e1 print_map 2023-08-27T21:43:57.422 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: e1 2023-08-27T21:43:57.422 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: enable_multiple, ever_enabled_multiple: 1,1 2023-08-27T21:43:57.423 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-08-27T21:43:57.423 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: legacy client fscid: -1 2023-08-27T21:43:57.423 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: 2023-08-27T21:43:57.423 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: No filesystems configured 2023-08-27T21:43:57.423 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.280+0000 7f97d6407880 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-08-27T21:43:57.423 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.280+0000 7f97d6407880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-08-27T21:43:57.423 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.280+0000 7f97d6407880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-08-27T21:43:57.424 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.280+0000 7f97d6407880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-08-27T21:43:57.424 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: audit 2023-08-27T21:43:54.537527+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.6:0/3710419064' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:43:57.424 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: debug 2023-08-27T21:43:57.280+0000 7f97d6407880 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-08-27T21:43:57.424 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: cluster 2023-08-27T21:43:57.305345+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-08-27T21:43:57.424 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: cluster 2023-08-27T21:43:57.305431+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0]} removed_ranks: {} 2023-08-27T21:43:57.424 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: cluster 2023-08-27T21:43:57.325750+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2023-08-27T21:43:57.425 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: cluster 2023-08-27T21:43:57.325804+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-08-27T21:43:57.425 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:57 smithi006 bash[27292]: cluster 2023-08-27T21:43:57.326327+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2023-08-27T21:43:58.154 INFO:teuthology.orchestra.run.smithi006.stderr:Wrote config to /etc/ceph/ceph.conf 2023-08-27T21:43:58.154 INFO:teuthology.orchestra.run.smithi006.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-08-27T21:43:58.154 INFO:teuthology.orchestra.run.smithi006.stderr:Creating mgr... 2023-08-27T21:43:58.155 INFO:teuthology.orchestra.run.smithi006.stderr:Verifying port 9283 ... 2023-08-27T21:43:58.383 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Failed to reset failed state of unit ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@mgr.y.service: Unit ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@mgr.y.service not loaded. 2023-08-27T21:43:58.392 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Created symlink /etc/systemd/system/ceph-b696f380-4522-11ee-817b-1b7a5f1ba315.target.wants/ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@mgr.y.service -> /etc/systemd/system/ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@.service. 2023-08-27T21:43:58.613 INFO:teuthology.orchestra.run.smithi006.stderr:firewalld does not appear to be present 2023-08-27T21:43:58.613 INFO:teuthology.orchestra.run.smithi006.stderr:Not possible to enable service . firewalld.service is not available 2023-08-27T21:43:58.614 INFO:teuthology.orchestra.run.smithi006.stderr:firewalld does not appear to be present 2023-08-27T21:43:58.614 INFO:teuthology.orchestra.run.smithi006.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2023-08-27T21:43:58.614 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for mgr to start... 2023-08-27T21:43:58.614 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for mgr... 2023-08-27T21:43:58.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:43:58 smithi006 bash[27292]: audit 2023-08-27T21:43:57.542820+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.6:0/2589029084' entity='client.admin' 2023-08-27T21:44:00.317 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2023-08-27T21:44:00.318 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2023-08-27T21:44:00.318 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsid": "b696f380-4522-11ee-817b-1b7a5f1ba315", 2023-08-27T21:44:00.318 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "health": { 2023-08-27T21:44:00.318 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-08-27T21:44:00.318 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "checks": {}, 2023-08-27T21:44:00.318 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mutes": [] 2023-08-27T21:44:00.318 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2023-08-27T21:44:00.322 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-08-27T21:44:00.322 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum": [ 2023-08-27T21:44:00.322 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 0 2023-08-27T21:44:00.322 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2023-08-27T21:44:00.322 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_names": [ 2023-08-27T21:44:00.322 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "a" 2023-08-27T21:44:00.322 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2023-08-27T21:44:00.322 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_age": 3, 2023-08-27T21:44:00.322 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "monmap": { 2023-08-27T21:44:00.322 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-27T21:44:00.323 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-08-27T21:44:00.323 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_mons": 1 2023-08-27T21:44:00.323 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2023-08-27T21:44:00.323 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osdmap": { 2023-08-27T21:44:00.323 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-27T21:44:00.323 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_osds": 0, 2023-08-27T21:44:00.323 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-08-27T21:44:00.323 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-08-27T21:44:00.323 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-08-27T21:44:00.323 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-08-27T21:44:00.323 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-08-27T21:44:00.324 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2023-08-27T21:44:00.324 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgmap": { 2023-08-27T21:44:00.324 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-08-27T21:44:00.324 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-08-27T21:44:00.324 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pools": 0, 2023-08-27T21:44:00.324 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_objects": 0, 2023-08-27T21:44:00.324 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-08-27T21:44:00.324 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-08-27T21:44:00.324 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-08-27T21:44:00.324 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_total": 0 2023-08-27T21:44:00.324 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2023-08-27T21:44:00.324 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsmap": { 2023-08-27T21:44:00.324 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-27T21:44:00.325 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "by_rank": [], 2023-08-27T21:44:00.325 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "up:standby": 0 2023-08-27T21:44:00.327 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2023-08-27T21:44:00.327 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mgrmap": { 2023-08-27T21:44:00.327 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "available": false, 2023-08-27T21:44:00.327 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-08-27T21:44:00.328 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modules": [ 2023-08-27T21:44:00.328 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "iostat", 2023-08-27T21:44:00.328 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "nfs", 2023-08-27T21:44:00.328 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "restful" 2023-08-27T21:44:00.328 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2023-08-27T21:44:00.328 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2023-08-27T21:44:00.328 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2023-08-27T21:44:00.328 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "servicemap": { 2023-08-27T21:44:00.328 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-27T21:44:00.328 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modified": "2023-08-27T21:43:51.022117+0000", 2023-08-27T21:44:00.328 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2023-08-27T21:44:00.328 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2023-08-27T21:44:00.329 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "progress_events": {} 2023-08-27T21:44:00.329 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2023-08-27T21:44:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:00 smithi006 bash[27292]: audit 2023-08-27T21:44:00.312290+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.6:0/4016914622' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-08-27T21:44:00.719 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:00 smithi006 bash[27566]: debug 2023-08-27T21:44:00.348+0000 7fa518b27680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-27T21:44:00.945 INFO:teuthology.orchestra.run.smithi006.stderr:mgr not available, waiting (1/15)... 2023-08-27T21:44:01.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:00 smithi006 bash[27566]: debug 2023-08-27T21:44:00.832+0000 7fa518b27680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-27T21:44:02.157 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:01 smithi006 bash[27566]: debug 2023-08-27T21:44:01.792+0000 7fa518b27680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-27T21:44:02.157 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:01 smithi006 bash[27566]: debug 2023-08-27T21:44:01.892+0000 7fa518b27680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-27T21:44:02.468 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:02 smithi006 bash[27566]: debug 2023-08-27T21:44:02.152+0000 7fa518b27680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-27T21:44:03.617 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:03 smithi006 bash[27566]: debug 2023-08-27T21:44:03.340+0000 7fa518b27680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-27T21:44:03.617 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:03 smithi006 bash[27566]: debug 2023-08-27T21:44:03.508+0000 7fa518b27680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-27T21:44:03.617 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:03 smithi006 bash[27566]: debug 2023-08-27T21:44:03.612+0000 7fa518b27680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-27T21:44:03.868 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:03 smithi006 bash[27566]: debug 2023-08-27T21:44:03.728+0000 7fa518b27680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-27T21:44:04.201 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2023-08-27T21:44:04.201 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2023-08-27T21:44:04.201 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsid": "b696f380-4522-11ee-817b-1b7a5f1ba315", 2023-08-27T21:44:04.203 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "health": { 2023-08-27T21:44:04.204 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-08-27T21:44:04.204 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "checks": {}, 2023-08-27T21:44:04.204 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mutes": [] 2023-08-27T21:44:04.204 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2023-08-27T21:44:04.204 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-08-27T21:44:04.204 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum": [ 2023-08-27T21:44:04.204 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 0 2023-08-27T21:44:04.204 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2023-08-27T21:44:04.204 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_names": [ 2023-08-27T21:44:04.204 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "a" 2023-08-27T21:44:04.204 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2023-08-27T21:44:04.205 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_age": 6, 2023-08-27T21:44:04.205 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "monmap": { 2023-08-27T21:44:04.205 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-27T21:44:04.205 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-08-27T21:44:04.205 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_mons": 1 2023-08-27T21:44:04.205 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2023-08-27T21:44:04.205 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osdmap": { 2023-08-27T21:44:04.205 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-27T21:44:04.205 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_osds": 0, 2023-08-27T21:44:04.205 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-08-27T21:44:04.209 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-08-27T21:44:04.209 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-08-27T21:44:04.209 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-08-27T21:44:04.209 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-08-27T21:44:04.209 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2023-08-27T21:44:04.209 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgmap": { 2023-08-27T21:44:04.209 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-08-27T21:44:04.210 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-08-27T21:44:04.210 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pools": 0, 2023-08-27T21:44:04.210 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_objects": 0, 2023-08-27T21:44:04.210 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-08-27T21:44:04.210 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-08-27T21:44:04.210 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-08-27T21:44:04.210 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_total": 0 2023-08-27T21:44:04.210 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2023-08-27T21:44:04.210 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsmap": { 2023-08-27T21:44:04.210 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-27T21:44:04.210 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "by_rank": [], 2023-08-27T21:44:04.210 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "up:standby": 0 2023-08-27T21:44:04.211 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2023-08-27T21:44:04.211 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mgrmap": { 2023-08-27T21:44:04.211 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "available": false, 2023-08-27T21:44:04.211 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-08-27T21:44:04.211 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modules": [ 2023-08-27T21:44:04.211 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "iostat", 2023-08-27T21:44:04.211 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "nfs", 2023-08-27T21:44:04.211 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "restful" 2023-08-27T21:44:04.211 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2023-08-27T21:44:04.211 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2023-08-27T21:44:04.211 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2023-08-27T21:44:04.212 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "servicemap": { 2023-08-27T21:44:04.212 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-27T21:44:04.212 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modified": "2023-08-27T21:43:51.022117+0000", 2023-08-27T21:44:04.212 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2023-08-27T21:44:04.212 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2023-08-27T21:44:04.212 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "progress_events": {} 2023-08-27T21:44:04.212 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2023-08-27T21:44:04.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:03 smithi006 bash[27566]: debug 2023-08-27T21:44:03.960+0000 7fa518b27680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-27T21:44:04.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:04 smithi006 bash[27292]: audit 2023-08-27T21:44:04.196138+0000 mon.a (mon.0) 8 : audit [DBG] from='client.? 172.21.15.6:0/3522008335' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-08-27T21:44:04.719 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:04 smithi006 bash[27566]: debug 2023-08-27T21:44:04.344+0000 7fa518b27680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-27T21:44:04.719 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:04 smithi006 bash[27566]: debug 2023-08-27T21:44:04.464+0000 7fa518b27680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-27T21:44:04.877 INFO:teuthology.orchestra.run.smithi006.stderr:mgr not available, waiting (2/15)... 2023-08-27T21:44:05.718 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:05 smithi006 bash[27566]: debug 2023-08-27T21:44:05.372+0000 7fa518b27680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-27T21:44:05.719 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:05 smithi006 bash[27566]: debug 2023-08-27T21:44:05.480+0000 7fa518b27680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-27T21:44:05.719 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:05 smithi006 bash[27566]: debug 2023-08-27T21:44:05.600+0000 7fa518b27680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-27T21:44:06.130 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:05 smithi006 bash[27566]: debug 2023-08-27T21:44:05.832+0000 7fa518b27680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-27T21:44:06.469 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:06 smithi006 bash[27566]: debug 2023-08-27T21:44:06.124+0000 7fa518b27680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-27T21:44:06.469 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:06 smithi006 bash[27566]: debug 2023-08-27T21:44:06.296+0000 7fa518b27680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-27T21:44:06.813 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:06 smithi006 bash[27566]: debug 2023-08-27T21:44:06.476+0000 7fa518b27680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-27T21:44:06.813 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:06 smithi006 bash[27566]: debug 2023-08-27T21:44:06.696+0000 7fa518b27680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-27T21:44:07.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:06 smithi006 bash[27566]: debug 2023-08-27T21:44:06.808+0000 7fa518b27680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-27T21:44:07.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:06 smithi006 bash[27292]: cluster 2023-08-27T21:44:06.831057+0000 mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2023-08-27T21:44:07.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:06 smithi006 bash[27292]: cluster 2023-08-27T21:44:06.837803+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00694606s) 2023-08-27T21:44:07.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:06 smithi006 bash[27292]: audit 2023-08-27T21:44:06.840172+0000 mon.a (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.6:0/3899887083' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T21:44:07.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:06 smithi006 bash[27292]: audit 2023-08-27T21:44:06.840638+0000 mon.a (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.6:0/3899887083' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T21:44:07.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:06 smithi006 bash[27292]: audit 2023-08-27T21:44:06.840935+0000 mon.a (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.6:0/3899887083' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T21:44:07.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:06 smithi006 bash[27292]: audit 2023-08-27T21:44:06.841197+0000 mon.a (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.6:0/3899887083' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T21:44:07.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:06 smithi006 bash[27292]: audit 2023-08-27T21:44:06.841480+0000 mon.a (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.6:0/3899887083' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T21:44:07.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:06 smithi006 bash[27292]: cluster 2023-08-27T21:44:06.845504+0000 mon.a (mon.0) 16 : cluster [INF] Manager daemon y is now available 2023-08-27T21:44:07.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:06 smithi006 bash[27292]: audit 2023-08-27T21:44:06.854744+0000 mon.a (mon.0) 17 : audit [INF] from='mgr.14100 172.21.15.6:0/3899887083' entity='mgr.y' 2023-08-27T21:44:07.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:06 smithi006 bash[27292]: audit 2023-08-27T21:44:06.863435+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.6:0/3899887083' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T21:44:07.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:06 smithi006 bash[27292]: audit 2023-08-27T21:44:06.867849+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.14100 172.21.15.6:0/3899887083' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T21:44:07.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:06 smithi006 bash[27292]: audit 2023-08-27T21:44:06.873046+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.6:0/3899887083' entity='mgr.y' 2023-08-27T21:44:07.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:06 smithi006 bash[27292]: audit 2023-08-27T21:44:06.875886+0000 mon.a (mon.0) 21 : audit [INF] from='mgr.14100 172.21.15.6:0/3899887083' entity='mgr.y' 2023-08-27T21:44:08.519 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2023-08-27T21:44:08.519 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2023-08-27T21:44:08.519 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsid": "b696f380-4522-11ee-817b-1b7a5f1ba315", 2023-08-27T21:44:08.520 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "health": { 2023-08-27T21:44:08.521 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-08-27T21:44:08.521 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "checks": {}, 2023-08-27T21:44:08.521 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mutes": [] 2023-08-27T21:44:08.521 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2023-08-27T21:44:08.521 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-08-27T21:44:08.521 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum": [ 2023-08-27T21:44:08.521 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 0 2023-08-27T21:44:08.521 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2023-08-27T21:44:08.523 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_names": [ 2023-08-27T21:44:08.523 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "a" 2023-08-27T21:44:08.523 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2023-08-27T21:44:08.523 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_age": 11, 2023-08-27T21:44:08.523 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "monmap": { 2023-08-27T21:44:08.523 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-27T21:44:08.523 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-08-27T21:44:08.523 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_mons": 1 2023-08-27T21:44:08.523 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2023-08-27T21:44:08.523 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osdmap": { 2023-08-27T21:44:08.524 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-27T21:44:08.524 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_osds": 0, 2023-08-27T21:44:08.524 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-08-27T21:44:08.527 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-08-27T21:44:08.527 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-08-27T21:44:08.527 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-08-27T21:44:08.527 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-08-27T21:44:08.528 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2023-08-27T21:44:08.528 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgmap": { 2023-08-27T21:44:08.528 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-08-27T21:44:08.528 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-08-27T21:44:08.528 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pools": 0, 2023-08-27T21:44:08.528 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_objects": 0, 2023-08-27T21:44:08.528 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-08-27T21:44:08.528 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-08-27T21:44:08.528 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-08-27T21:44:08.528 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_total": 0 2023-08-27T21:44:08.528 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2023-08-27T21:44:08.528 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsmap": { 2023-08-27T21:44:08.529 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-27T21:44:08.529 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "by_rank": [], 2023-08-27T21:44:08.529 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "up:standby": 0 2023-08-27T21:44:08.529 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2023-08-27T21:44:08.529 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mgrmap": { 2023-08-27T21:44:08.529 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "available": true, 2023-08-27T21:44:08.529 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-08-27T21:44:08.529 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modules": [ 2023-08-27T21:44:08.529 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "iostat", 2023-08-27T21:44:08.529 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "nfs", 2023-08-27T21:44:08.529 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "restful" 2023-08-27T21:44:08.529 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2023-08-27T21:44:08.529 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2023-08-27T21:44:08.530 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2023-08-27T21:44:08.530 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "servicemap": { 2023-08-27T21:44:08.530 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-27T21:44:08.530 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modified": "2023-08-27T21:43:51.022117+0000", 2023-08-27T21:44:08.530 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2023-08-27T21:44:08.530 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2023-08-27T21:44:08.530 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "progress_events": {} 2023-08-27T21:44:08.530 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2023-08-27T21:44:08.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:08 smithi006 bash[27292]: cluster 2023-08-27T21:44:07.843069+0000 mon.a (mon.0) 22 : cluster [DBG] mgrmap e3: y(active, since 1.0122s) 2023-08-27T21:44:08.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:08 smithi006 bash[27292]: audit 2023-08-27T21:44:08.514205+0000 mon.a (mon.0) 23 : audit [DBG] from='client.? 172.21.15.6:0/1745713714' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-08-27T21:44:09.174 INFO:teuthology.orchestra.run.smithi006.stderr:mgr is available 2023-08-27T21:44:10.600 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2023-08-27T21:44:10.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: [global] 2023-08-27T21:44:10.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: fsid = b696f380-4522-11ee-817b-1b7a5f1ba315 2023-08-27T21:44:10.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: log_to_journald = false 2023-08-27T21:44:10.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-08-27T21:44:10.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-08-27T21:44:10.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-08-27T21:44:10.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-08-27T21:44:10.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2023-08-27T21:44:10.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: [mgr] 2023-08-27T21:44:10.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-08-27T21:44:10.601 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2023-08-27T21:44:10.602 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: [osd] 2023-08-27T21:44:10.602 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-08-27T21:44:10.602 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-08-27T21:44:10.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:10 smithi006 bash[27292]: cluster 2023-08-27T21:44:09.842972+0000 mon.a (mon.0) 24 : cluster [DBG] mgrmap e4: y(active, since 3s) 2023-08-27T21:44:10.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:10 smithi006 bash[27292]: audit 2023-08-27T21:44:10.593772+0000 mon.a (mon.0) 25 : audit [INF] from='client.? 172.21.15.6:0/1839295035' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-08-27T21:44:11.215 INFO:teuthology.orchestra.run.smithi006.stderr:Enabling cephadm module... 2023-08-27T21:44:13.637 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:13 smithi006 bash[27292]: audit 2023-08-27T21:44:12.735628+0000 mon.a (mon.0) 26 : audit [INF] from='client.? 172.21.15.6:0/2953280016' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-08-27T21:44:13.637 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:13 smithi006 bash[27566]: ignoring --setuser ceph since I am not root 2023-08-27T21:44:13.637 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:13 smithi006 bash[27566]: ignoring --setgroup ceph since I am not root 2023-08-27T21:44:13.638 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:13 smithi006 bash[27566]: debug 2023-08-27T21:44:13.515+0000 7f577d7ef680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-27T21:44:13.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:13 smithi006 bash[27566]: debug 2023-08-27T21:44:13.631+0000 7f577d7ef680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-27T21:44:14.302 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:14 smithi006 bash[27566]: debug 2023-08-27T21:44:14.107+0000 7f577d7ef680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-27T21:44:14.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:14 smithi006 bash[27292]: audit 2023-08-27T21:44:13.302042+0000 mon.a (mon.0) 27 : audit [INF] from='client.? 172.21.15.6:0/2953280016' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-08-27T21:44:14.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:14 smithi006 bash[27292]: cluster 2023-08-27T21:44:13.302091+0000 mon.a (mon.0) 28 : cluster [DBG] mgrmap e5: y(active, since 6s) 2023-08-27T21:44:15.178 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:15 smithi006 bash[27566]: debug 2023-08-27T21:44:15.067+0000 7f577d7ef680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-27T21:44:15.457 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:15 smithi006 bash[27566]: debug 2023-08-27T21:44:15.171+0000 7f577d7ef680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-27T21:44:15.562 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2023-08-27T21:44:15.562 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 5, 2023-08-27T21:44:15.562 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "available": true, 2023-08-27T21:44:15.562 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "active_name": "y", 2023-08-27T21:44:15.563 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_standby": 0 2023-08-27T21:44:15.563 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2023-08-27T21:44:15.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:15 smithi006 bash[27292]: audit 2023-08-27T21:44:15.557161+0000 mon.a (mon.0) 29 : audit [DBG] from='client.? 172.21.15.6:0/2948612897' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-08-27T21:44:15.719 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:15 smithi006 bash[27566]: debug 2023-08-27T21:44:15.451+0000 7f577d7ef680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-27T21:44:16.159 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for the mgr to restart... 2023-08-27T21:44:16.160 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for mgr epoch 5... 2023-08-27T21:44:16.916 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:16 smithi006 bash[27566]: debug 2023-08-27T21:44:16.651+0000 7f577d7ef680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-27T21:44:16.916 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:16 smithi006 bash[27566]: debug 2023-08-27T21:44:16.819+0000 7f577d7ef680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-27T21:44:17.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:16 smithi006 bash[27566]: debug 2023-08-27T21:44:16.923+0000 7f577d7ef680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-27T21:44:17.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:17 smithi006 bash[27566]: debug 2023-08-27T21:44:17.035+0000 7f577d7ef680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-27T21:44:17.655 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:17 smithi006 bash[27566]: debug 2023-08-27T21:44:17.263+0000 7f577d7ef680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-27T21:44:17.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:17 smithi006 bash[27566]: debug 2023-08-27T21:44:17.647+0000 7f577d7ef680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-27T21:44:17.969 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:17 smithi006 bash[27566]: debug 2023-08-27T21:44:17.775+0000 7f577d7ef680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-27T21:44:18.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:18 smithi006 bash[27566]: debug 2023-08-27T21:44:18.667+0000 7f577d7ef680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-27T21:44:18.969 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:18 smithi006 bash[27566]: debug 2023-08-27T21:44:18.775+0000 7f577d7ef680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-27T21:44:18.969 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:18 smithi006 bash[27566]: debug 2023-08-27T21:44:18.895+0000 7f577d7ef680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-27T21:44:19.433 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:19 smithi006 bash[27566]: debug 2023-08-27T21:44:19.123+0000 7f577d7ef680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-27T21:44:19.718 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:19 smithi006 bash[27566]: debug 2023-08-27T21:44:19.427+0000 7f577d7ef680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-27T21:44:19.719 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:19 smithi006 bash[27566]: debug 2023-08-27T21:44:19.595+0000 7f577d7ef680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-27T21:44:20.116 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:19 smithi006 bash[27566]: debug 2023-08-27T21:44:19.771+0000 7f577d7ef680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-27T21:44:20.116 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:19 smithi006 bash[27566]: debug 2023-08-27T21:44:19.995+0000 7f577d7ef680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-27T21:44:20.469 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:20 smithi006 bash[27566]: debug 2023-08-27T21:44:20.107+0000 7f577d7ef680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-27T21:44:20.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:20 smithi006 bash[27292]: cluster 2023-08-27T21:44:20.130780+0000 mon.a (mon.0) 30 : cluster [INF] Active manager daemon y restarted 2023-08-27T21:44:20.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:20 smithi006 bash[27292]: cluster 2023-08-27T21:44:20.131212+0000 mon.a (mon.0) 31 : cluster [INF] Activating manager daemon y 2023-08-27T21:44:20.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:20 smithi006 bash[27292]: cluster 2023-08-27T21:44:20.137049+0000 mon.a (mon.0) 32 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2023-08-27T21:44:20.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:20 smithi006 bash[27292]: cluster 2023-08-27T21:44:20.137257+0000 mon.a (mon.0) 33 : cluster [DBG] mgrmap e6: y(active, starting, since 0.0061939s) 2023-08-27T21:44:20.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:20 smithi006 bash[27292]: audit 2023-08-27T21:44:20.139522+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T21:44:20.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:20 smithi006 bash[27292]: audit 2023-08-27T21:44:20.141145+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T21:44:20.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:20 smithi006 bash[27292]: audit 2023-08-27T21:44:20.142436+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T21:44:20.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:20 smithi006 bash[27292]: audit 2023-08-27T21:44:20.142837+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T21:44:20.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:20 smithi006 bash[27292]: audit 2023-08-27T21:44:20.143205+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T21:44:20.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:20 smithi006 bash[27292]: cluster 2023-08-27T21:44:20.147023+0000 mon.a (mon.0) 39 : cluster [INF] Manager daemon y is now available 2023-08-27T21:44:20.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:20 smithi006 bash[27292]: audit 2023-08-27T21:44:20.167213+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' 2023-08-27T21:44:21.159 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2023-08-27T21:44:21.160 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2023-08-27T21:44:21.160 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "initialized": true 2023-08-27T21:44:21.160 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2023-08-27T21:44:21.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:21 smithi006 bash[27292]: cephadm 2023-08-27T21:44:20.162354+0000 mgr.y (mgr.14116) 1 : cephadm [INF] Found migration_current of "None". Setting to last migration. 2023-08-27T21:44:21.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:21 smithi006 bash[27292]: audit 2023-08-27T21:44:20.292903+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' 2023-08-27T21:44:21.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:21 smithi006 bash[27292]: audit 2023-08-27T21:44:20.294940+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:44:21.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:21 smithi006 bash[27292]: audit 2023-08-27T21:44:20.296787+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:44:21.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:21 smithi006 bash[27292]: audit 2023-08-27T21:44:20.299667+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:44:21.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:21 smithi006 bash[27292]: audit 2023-08-27T21:44:20.325471+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T21:44:21.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:21 smithi006 bash[27292]: audit 2023-08-27T21:44:20.330636+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T21:44:21.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:21 smithi006 bash[27292]: cluster 2023-08-27T21:44:21.141489+0000 mon.a (mon.0) 47 : cluster [DBG] mgrmap e7: y(active, since 1.01042s) 2023-08-27T21:44:21.875 INFO:teuthology.orchestra.run.smithi006.stderr:mgr epoch 5 is available 2023-08-27T21:44:21.876 INFO:teuthology.orchestra.run.smithi006.stderr:Setting orchestrator backend to cephadm... 2023-08-27T21:44:22.600 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:22 smithi006 bash[27292]: audit 2023-08-27T21:44:21.141282+0000 mgr.y (mgr.14116) 2 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-08-27T21:44:22.600 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:22 smithi006 bash[27292]: audit 2023-08-27T21:44:21.154723+0000 mgr.y (mgr.14116) 3 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-08-27T21:44:23.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:23 smithi006 bash[27292]: cluster 2023-08-27T21:44:22.297151+0000 mon.a (mon.0) 48 : cluster [DBG] mgrmap e8: y(active, since 2s) 2023-08-27T21:44:23.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:23 smithi006 bash[27292]: audit 2023-08-27T21:44:23.247445+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' 2023-08-27T21:44:23.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:23 smithi006 bash[27292]: audit 2023-08-27T21:44:23.277265+0000 mon.a (mon.0) 50 : audit [DBG] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:44:24.574 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:24 smithi006 bash[27292]: audit 2023-08-27T21:44:23.237662+0000 mgr.y (mgr.14116) 4 : audit [DBG] from='client.14126 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:44:25.160 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: value unchanged 2023-08-27T21:44:25.498 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:25 smithi006 bash[27292]: audit 2023-08-27T21:44:25.155421+0000 mgr.y (mgr.14116) 5 : audit [DBG] from='client.14128 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:44:25.837 INFO:teuthology.orchestra.run.smithi006.stderr:Generating ssh key... 2023-08-27T21:44:27.719 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:27 smithi006 bash[27566]: Generating public/private rsa key pair. 2023-08-27T21:44:27.719 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:27 smithi006 bash[27566]: Your identification has been saved in /tmp/tmplrcapzmg/key. 2023-08-27T21:44:27.719 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:27 smithi006 bash[27566]: Your public key has been saved in /tmp/tmplrcapzmg/key.pub. 2023-08-27T21:44:27.719 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:27 smithi006 bash[27566]: The key fingerprint is: 2023-08-27T21:44:27.719 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:27 smithi006 bash[27566]: SHA256:LTjONbqvkmw3Kd2idvY+ID1W/n3EGcXdYx/zuFTzL/A ceph-b696f380-4522-11ee-817b-1b7a5f1ba315 2023-08-27T21:44:27.720 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:27 smithi006 bash[27566]: The key's randomart image is: 2023-08-27T21:44:27.720 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:27 smithi006 bash[27566]: +---[RSA 3072]----+ 2023-08-27T21:44:27.720 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:27 smithi006 bash[27566]: | .o| 2023-08-27T21:44:27.720 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:27 smithi006 bash[27566]: | **| 2023-08-27T21:44:27.720 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:27 smithi006 bash[27566]: | .+O| 2023-08-27T21:44:27.720 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:27 smithi006 bash[27566]: | ... . + +| 2023-08-27T21:44:27.720 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:27 smithi006 bash[27566]: | .ooS . = +.| 2023-08-27T21:44:27.720 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:27 smithi006 bash[27566]: | .o=+.o E .| 2023-08-27T21:44:27.720 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:27 smithi006 bash[27566]: | . =+= . . . . | 2023-08-27T21:44:27.720 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:27 smithi006 bash[27566]: | B O.o . . . | 2023-08-27T21:44:27.720 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:27 smithi006 bash[27566]: | o.BoB+. . | 2023-08-27T21:44:27.720 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:27 smithi006 bash[27566]: +----[SHA256]-----+ 2023-08-27T21:44:28.709 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:28 smithi006 bash[27292]: audit 2023-08-27T21:44:27.222368+0000 mgr.y (mgr.14116) 6 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:44:28.709 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:28 smithi006 bash[27292]: cephadm 2023-08-27T21:44:27.222797+0000 mgr.y (mgr.14116) 7 : cephadm [INF] Generating ssh key... 2023-08-27T21:44:28.709 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:28 smithi006 bash[27292]: audit 2023-08-27T21:44:27.282288+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' 2023-08-27T21:44:28.709 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:28 smithi006 bash[27292]: audit 2023-08-27T21:44:27.284889+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' 2023-08-27T21:44:29.310 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDmFR8M0V6PSsoPZhPk6xXAkNA1NmERbxMo1uWV+YAglLb4dNf/+TdsRyK+6fXFS+XDHuJq+iShFiUY5ia5G2srevBJ40OnqNjSh3NcBQN3taOJR+eNL0UBdDNCvVNXMkfWpTZNvYupvaPtYI/VI6yPNDys7eGzhB57jkzKaBaMvvnNVhIF7iiA01DD2nzHdcBtvX8j8ps/JulyDOE1hr62dPI4qR3sDoQgGhxOiErxzl49xldSsa2wpnOiFhMlKWhldXemLm2V1ADhZvGmpuv9EbpO/qAULvnuog0vvaMJqtDEh+3VrOFDX9+8wJ7CY7S1felSWX17p8H0jGE/FFARuKDS2FsubWuVBvWrxZa5EK4sib4jBHTs74tS2mmbjLNLFqSTGrzs6GvpPE/Xc81cIJ/15czhKAfiqeFLeG167Gb6GbSEnnaeyelZQpeo6ypfF7w2RZtbU64U2QSzGPa6fuRiXW6Cx73gZfmtmpqs9HdH0tYMra5rDnuiUvksHFU= ceph-b696f380-4522-11ee-817b-1b7a5f1ba315 2023-08-27T21:44:29.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:29 smithi006 bash[27292]: audit 2023-08-27T21:44:29.305158+0000 mgr.y (mgr.14116) 8 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:44:29.968 INFO:teuthology.orchestra.run.smithi006.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-08-27T21:44:29.968 INFO:teuthology.orchestra.run.smithi006.stderr:Adding key to root@localhost authorized_keys... 2023-08-27T21:44:29.969 INFO:teuthology.orchestra.run.smithi006.stderr:Adding host smithi006... 2023-08-27T21:44:31.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:31 smithi006 bash[27292]: audit 2023-08-27T21:44:31.316880+0000 mgr.y (mgr.14116) 9 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi006", "addr": "172.21.15.6", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:44:31.719 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:31 smithi006 bash[27566]: Warning: Permanently added '172.21.15.6' (ECDSA) to the list of known hosts. 2023-08-27T21:44:32.167 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Added host 'smithi006' with addr '172.21.15.6' 2023-08-27T21:44:33.046 INFO:teuthology.orchestra.run.smithi006.stderr:Deploying unmanaged mon service... 2023-08-27T21:44:33.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:33 smithi006 bash[27292]: audit 2023-08-27T21:44:32.160967+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' 2023-08-27T21:44:33.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:33 smithi006 bash[27292]: cephadm 2023-08-27T21:44:32.161617+0000 mgr.y (mgr.14116) 10 : cephadm [INF] Added host smithi006 2023-08-27T21:44:33.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:33 smithi006 bash[27292]: audit 2023-08-27T21:44:32.180893+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:44:34.977 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Scheduled mon update... 2023-08-27T21:44:35.765 INFO:teuthology.orchestra.run.smithi006.stderr:Deploying unmanaged mgr service... 2023-08-27T21:44:36.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:35 smithi006 bash[27292]: audit 2023-08-27T21:44:34.967871+0000 mgr.y (mgr.14116) 11 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:44:36.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:35 smithi006 bash[27292]: cephadm 2023-08-27T21:44:34.969422+0000 mgr.y (mgr.14116) 12 : cephadm [INF] Saving service mon spec with placement count:5 2023-08-27T21:44:36.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:35 smithi006 bash[27292]: audit 2023-08-27T21:44:34.972691+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' 2023-08-27T21:44:36.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:35 smithi006 bash[27292]: audit 2023-08-27T21:44:35.527649+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' 2023-08-27T21:44:36.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:35 smithi006 bash[27292]: audit 2023-08-27T21:44:35.794900+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' 2023-08-27T21:44:37.205 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Scheduled mgr update... 2023-08-27T21:44:38.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:38 smithi006 bash[27292]: audit 2023-08-27T21:44:37.192433+0000 mgr.y (mgr.14116) 13 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:44:38.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:38 smithi006 bash[27292]: cephadm 2023-08-27T21:44:37.194959+0000 mgr.y (mgr.14116) 14 : cephadm [INF] Saving service mgr spec with placement count:2 2023-08-27T21:44:38.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:38 smithi006 bash[27292]: audit 2023-08-27T21:44:37.199542+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' 2023-08-27T21:44:40.657 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:40 smithi006 bash[27292]: audit 2023-08-27T21:44:39.272531+0000 mon.a (mon.0) 59 : audit [INF] from='client.? 172.21.15.6:0/435559103' entity='client.admin' 2023-08-27T21:44:41.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:41 smithi006 bash[27292]: cluster 2023-08-27T21:44:40.144367+0000 mgr.y (mgr.14116) 15 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:44:41.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:41 smithi006 bash[27292]: audit 2023-08-27T21:44:40.893339+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.14116 172.21.15.6:0/1232639812' entity='mgr.y' 2023-08-27T21:44:41.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:41 smithi006 bash[27292]: audit 2023-08-27T21:44:41.267503+0000 mon.a (mon.0) 61 : audit [INF] from='client.? 172.21.15.6:0/3529849566' entity='client.admin' 2023-08-27T21:44:41.963 INFO:teuthology.orchestra.run.smithi006.stderr:Enabling the dashboard module... 2023-08-27T21:44:43.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:43 smithi006 bash[27292]: cluster 2023-08-27T21:44:42.144922+0000 mgr.y (mgr.14116) 16 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:44:44.624 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:44 smithi006 bash[27292]: audit 2023-08-27T21:44:43.446246+0000 mon.a (mon.0) 62 : audit [INF] from='client.? 172.21.15.6:0/1366986363' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-08-27T21:44:44.624 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:44 smithi006 bash[27566]: ignoring --setuser ceph since I am not root 2023-08-27T21:44:44.624 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:44 smithi006 bash[27566]: ignoring --setgroup ceph since I am not root 2023-08-27T21:44:44.624 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:44 smithi006 bash[27566]: debug 2023-08-27T21:44:44.516+0000 7f14a709e680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-27T21:44:44.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:44 smithi006 bash[27566]: debug 2023-08-27T21:44:44.620+0000 7f14a709e680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-27T21:44:45.311 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:45 smithi006 bash[27566]: debug 2023-08-27T21:44:45.076+0000 7f14a709e680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-27T21:44:45.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:45 smithi006 bash[27292]: audit 2023-08-27T21:44:44.310216+0000 mon.a (mon.0) 63 : audit [INF] from='client.? 172.21.15.6:0/1366986363' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-08-27T21:44:45.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:45 smithi006 bash[27292]: cluster 2023-08-27T21:44:44.310326+0000 mon.a (mon.0) 64 : cluster [DBG] mgrmap e9: y(active, since 24s) 2023-08-27T21:44:46.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:46 smithi006 bash[27566]: debug 2023-08-27T21:44:46.036+0000 7f14a709e680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-27T21:44:46.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:46 smithi006 bash[27566]: debug 2023-08-27T21:44:46.140+0000 7f14a709e680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-27T21:44:46.589 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2023-08-27T21:44:46.589 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 9, 2023-08-27T21:44:46.589 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "available": true, 2023-08-27T21:44:46.589 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "active_name": "y", 2023-08-27T21:44:46.589 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_standby": 0 2023-08-27T21:44:46.589 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2023-08-27T21:44:46.717 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:46 smithi006 bash[27566]: debug 2023-08-27T21:44:46.416+0000 7f14a709e680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-27T21:44:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:46 smithi006 bash[27292]: audit 2023-08-27T21:44:46.584666+0000 mon.a (mon.0) 65 : audit [DBG] from='client.? 172.21.15.6:0/4050524052' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-08-27T21:44:47.220 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for the mgr to restart... 2023-08-27T21:44:47.221 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for mgr epoch 9... 2023-08-27T21:44:47.891 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:47 smithi006 bash[27566]: debug 2023-08-27T21:44:47.608+0000 7f14a709e680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-27T21:44:47.891 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:47 smithi006 bash[27566]: debug 2023-08-27T21:44:47.788+0000 7f14a709e680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-27T21:44:47.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:47 smithi006 bash[27566]: debug 2023-08-27T21:44:47.884+0000 7f14a709e680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-27T21:44:48.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:48 smithi006 bash[27566]: debug 2023-08-27T21:44:48.008+0000 7f14a709e680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-27T21:44:48.613 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:48 smithi006 bash[27566]: debug 2023-08-27T21:44:48.236+0000 7f14a709e680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-27T21:44:48.882 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:48 smithi006 bash[27566]: debug 2023-08-27T21:44:48.608+0000 7f14a709e680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-27T21:44:48.882 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:48 smithi006 bash[27566]: debug 2023-08-27T21:44:48.732+0000 7f14a709e680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-27T21:44:49.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:49 smithi006 bash[27566]: debug 2023-08-27T21:44:49.632+0000 7f14a709e680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-27T21:44:49.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:49 smithi006 bash[27566]: debug 2023-08-27T21:44:49.744+0000 7f14a709e680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-27T21:44:49.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:49 smithi006 bash[27566]: debug 2023-08-27T21:44:49.864+0000 7f14a709e680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-27T21:44:50.403 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:50 smithi006 bash[27566]: debug 2023-08-27T21:44:50.096+0000 7f14a709e680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-27T21:44:50.717 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:50 smithi006 bash[27566]: debug 2023-08-27T21:44:50.396+0000 7f14a709e680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-27T21:44:50.717 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:50 smithi006 bash[27566]: debug 2023-08-27T21:44:50.568+0000 7f14a709e680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-27T21:44:51.094 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:50 smithi006 bash[27566]: debug 2023-08-27T21:44:50.748+0000 7f14a709e680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-27T21:44:51.094 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:50 smithi006 bash[27566]: debug 2023-08-27T21:44:50.976+0000 7f14a709e680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-27T21:44:51.094 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:51 smithi006 bash[27566]: debug 2023-08-27T21:44:51.088+0000 7f14a709e680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-27T21:44:51.348 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:51 smithi006 bash[27292]: cluster 2023-08-27T21:44:51.111326+0000 mon.a (mon.0) 66 : cluster [INF] Active manager daemon y restarted 2023-08-27T21:44:51.349 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:51 smithi006 bash[27292]: cluster 2023-08-27T21:44:51.111766+0000 mon.a (mon.0) 67 : cluster [INF] Activating manager daemon y 2023-08-27T21:44:51.349 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:51 smithi006 bash[27292]: cluster 2023-08-27T21:44:51.118567+0000 mon.a (mon.0) 68 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2023-08-27T21:44:51.349 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:51 smithi006 bash[27292]: cluster 2023-08-27T21:44:51.118731+0000 mon.a (mon.0) 69 : cluster [DBG] mgrmap e10: y(active, starting, since 0.00713966s) 2023-08-27T21:44:51.349 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:51 smithi006 bash[27292]: audit 2023-08-27T21:44:51.119850+0000 mon.a (mon.0) 70 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T21:44:51.349 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:51 smithi006 bash[27292]: audit 2023-08-27T21:44:51.121557+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T21:44:51.349 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:51 smithi006 bash[27292]: audit 2023-08-27T21:44:51.122343+0000 mon.a (mon.0) 72 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T21:44:51.349 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:51 smithi006 bash[27292]: audit 2023-08-27T21:44:51.122565+0000 mon.a (mon.0) 73 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T21:44:51.349 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:51 smithi006 bash[27292]: audit 2023-08-27T21:44:51.122726+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T21:44:51.349 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:51 smithi006 bash[27292]: cluster 2023-08-27T21:44:51.125927+0000 mon.a (mon.0) 75 : cluster [INF] Manager daemon y is now available 2023-08-27T21:44:51.717 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:44:51 smithi006 bash[27566]: Warning: Permanently added '172.21.15.6' (ECDSA) to the list of known hosts. 2023-08-27T21:44:52.141 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2023-08-27T21:44:52.141 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2023-08-27T21:44:52.141 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "initialized": true 2023-08-27T21:44:52.141 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2023-08-27T21:44:52.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:52 smithi006 bash[27292]: audit 2023-08-27T21:44:51.270554+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:44:52.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:52 smithi006 bash[27292]: audit 2023-08-27T21:44:51.274326+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:44:52.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:52 smithi006 bash[27292]: audit 2023-08-27T21:44:51.311516+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T21:44:52.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:52 smithi006 bash[27292]: audit 2023-08-27T21:44:51.317897+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T21:44:52.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:52 smithi006 bash[27292]: cluster 2023-08-27T21:44:52.122102+0000 mon.a (mon.0) 80 : cluster [DBG] mgrmap e11: y(active, since 1.0105s) 2023-08-27T21:44:52.779 INFO:teuthology.orchestra.run.smithi006.stderr:mgr epoch 9 is available 2023-08-27T21:44:52.779 INFO:teuthology.orchestra.run.smithi006.stderr:Generating a dashboard self-signed certificate... 2023-08-27T21:44:53.573 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:53 smithi006 bash[27292]: audit 2023-08-27T21:44:52.121957+0000 mgr.y (mgr.14146) 1 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-08-27T21:44:53.573 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:53 smithi006 bash[27292]: audit 2023-08-27T21:44:52.137811+0000 mgr.y (mgr.14146) 2 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-08-27T21:44:54.359 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Self-signed certificate created 2023-08-27T21:44:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:54 smithi006 bash[27292]: cluster 2023-08-27T21:44:53.306392+0000 mon.a (mon.0) 81 : cluster [DBG] mgrmap e12: y(active, since 2s) 2023-08-27T21:44:55.021 INFO:teuthology.orchestra.run.smithi006.stderr:Creating initial admin user... 2023-08-27T21:44:55.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:55 smithi006 bash[27292]: audit 2023-08-27T21:44:54.184724+0000 mgr.y (mgr.14146) 3 : audit [DBG] from='client.14156 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:44:55.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:55 smithi006 bash[27292]: audit 2023-08-27T21:44:54.349682+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:44:55.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:55 smithi006 bash[27292]: audit 2023-08-27T21:44:54.355317+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:44:56.753 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$H0iElvNAHUcN9sL9h7SGXe7W4m0O2QWXDzJpHND/J/rzT/tNLYrY6", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1693172696, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-08-27T21:44:57.394 INFO:teuthology.orchestra.run.smithi006.stderr:Fetching dashboard port number... 2023-08-27T21:44:58.181 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:57 smithi006 bash[27292]: audit 2023-08-27T21:44:56.491097+0000 mgr.y (mgr.14146) 4 : audit [DBG] from='client.14158 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:44:58.181 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:57 smithi006 bash[27292]: audit 2023-08-27T21:44:56.750165+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:44:58.774 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 8443 2023-08-27T21:44:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:58 smithi006 bash[27292]: cluster 2023-08-27T21:44:57.754566+0000 mon.a (mon.0) 85 : cluster [DBG] mgrmap e13: y(active, since 6s) 2023-08-27T21:44:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:58 smithi006 bash[27292]: audit 2023-08-27T21:44:58.195049+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:44:59.415 INFO:teuthology.orchestra.run.smithi006.stderr:firewalld does not appear to be present 2023-08-27T21:44:59.415 INFO:teuthology.orchestra.run.smithi006.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2023-08-27T21:44:59.417 INFO:teuthology.orchestra.run.smithi006.stderr:Ceph Dashboard is now available at: 2023-08-27T21:44:59.417 INFO:teuthology.orchestra.run.smithi006.stderr: 2023-08-27T21:44:59.417 INFO:teuthology.orchestra.run.smithi006.stderr: URL: https://smithi006.front.sepia.ceph.com:8443/ 2023-08-27T21:44:59.417 INFO:teuthology.orchestra.run.smithi006.stderr: User: admin 2023-08-27T21:44:59.418 INFO:teuthology.orchestra.run.smithi006.stderr: Password: mskgjym26w 2023-08-27T21:44:59.418 INFO:teuthology.orchestra.run.smithi006.stderr: 2023-08-27T21:44:59.418 INFO:teuthology.orchestra.run.smithi006.stderr:Enabling autotune for osd_memory_target 2023-08-27T21:45:00.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:44:59 smithi006 bash[27292]: audit 2023-08-27T21:44:58.771513+0000 mon.a (mon.0) 87 : audit [DBG] from='client.? 172.21.15.6:0/560639137' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-08-27T21:45:03.045 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2023-08-27T21:45:03.709 INFO:teuthology.orchestra.run.smithi006.stderr:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-08-27T21:45:03.709 INFO:teuthology.orchestra.run.smithi006.stderr: 2023-08-27T21:45:03.709 INFO:teuthology.orchestra.run.smithi006.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-08-27T21:45:03.709 INFO:teuthology.orchestra.run.smithi006.stderr: 2023-08-27T21:45:03.709 INFO:teuthology.orchestra.run.smithi006.stderr:Or, if you are only running a single cluster on this host: 2023-08-27T21:45:03.709 INFO:teuthology.orchestra.run.smithi006.stderr: 2023-08-27T21:45:03.709 INFO:teuthology.orchestra.run.smithi006.stderr: sudo /home/ubuntu/cephtest/cephadm shell 2023-08-27T21:45:03.710 INFO:teuthology.orchestra.run.smithi006.stderr: 2023-08-27T21:45:03.710 INFO:teuthology.orchestra.run.smithi006.stderr:Please consider enabling telemetry to help improve Ceph: 2023-08-27T21:45:03.710 INFO:teuthology.orchestra.run.smithi006.stderr: 2023-08-27T21:45:03.710 INFO:teuthology.orchestra.run.smithi006.stderr: ceph telemetry on 2023-08-27T21:45:03.710 INFO:teuthology.orchestra.run.smithi006.stderr: 2023-08-27T21:45:03.710 INFO:teuthology.orchestra.run.smithi006.stderr:For more information see: 2023-08-27T21:45:03.710 INFO:teuthology.orchestra.run.smithi006.stderr: 2023-08-27T21:45:03.710 INFO:teuthology.orchestra.run.smithi006.stderr: https://docs.ceph.com/en/pacific/mgr/telemetry/ 2023-08-27T21:45:03.710 INFO:teuthology.orchestra.run.smithi006.stderr: 2023-08-27T21:45:03.710 INFO:teuthology.orchestra.run.smithi006.stderr:Bootstrap complete. 2023-08-27T21:45:03.744 INFO:tasks.cephadm:Fetching config... 2023-08-27T21:45:03.744 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2023-08-27T21:45:03.745 DEBUG:teuthology.orchestra.run.smithi006:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-08-27T21:45:03.751 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-08-27T21:45:03.751 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2023-08-27T21:45:03.751 DEBUG:teuthology.orchestra.run.smithi006:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-08-27T21:45:03.799 INFO:tasks.cephadm:Fetching mon keyring... 2023-08-27T21:45:03.799 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2023-08-27T21:45:03.799 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd if=/var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/keyring of=/dev/stdout 2023-08-27T21:45:03.851 INFO:tasks.cephadm:Fetching pub ssh key... 2023-08-27T21:45:03.851 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2023-08-27T21:45:03.852 DEBUG:teuthology.orchestra.run.smithi006:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-08-27T21:45:03.898 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-08-27T21:45:03.898 DEBUG:teuthology.orchestra.run.smithi006:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDmFR8M0V6PSsoPZhPk6xXAkNA1NmERbxMo1uWV+YAglLb4dNf/+TdsRyK+6fXFS+XDHuJq+iShFiUY5ia5G2srevBJ40OnqNjSh3NcBQN3taOJR+eNL0UBdDNCvVNXMkfWpTZNvYupvaPtYI/VI6yPNDys7eGzhB57jkzKaBaMvvnNVhIF7iiA01DD2nzHdcBtvX8j8ps/JulyDOE1hr62dPI4qR3sDoQgGhxOiErxzl49xldSsa2wpnOiFhMlKWhldXemLm2V1ADhZvGmpuv9EbpO/qAULvnuog0vvaMJqtDEh+3VrOFDX9+8wJ7CY7S1felSWX17p8H0jGE/FFARuKDS2FsubWuVBvWrxZa5EK4sib4jBHTs74tS2mmbjLNLFqSTGrzs6GvpPE/Xc81cIJ/15czhKAfiqeFLeG167Gb6GbSEnnaeyelZQpeo6ypfF7w2RZtbU64U2QSzGPa6fuRiXW6Cx73gZfmtmpqs9HdH0tYMra5rDnuiUvksHFU= ceph-b696f380-4522-11ee-817b-1b7a5f1ba315' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-08-27T21:45:03.967 INFO:teuthology.orchestra.run.smithi006.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDmFR8M0V6PSsoPZhPk6xXAkNA1NmERbxMo1uWV+YAglLb4dNf/+TdsRyK+6fXFS+XDHuJq+iShFiUY5ia5G2srevBJ40OnqNjSh3NcBQN3taOJR+eNL0UBdDNCvVNXMkfWpTZNvYupvaPtYI/VI6yPNDys7eGzhB57jkzKaBaMvvnNVhIF7iiA01DD2nzHdcBtvX8j8ps/JulyDOE1hr62dPI4qR3sDoQgGhxOiErxzl49xldSsa2wpnOiFhMlKWhldXemLm2V1ADhZvGmpuv9EbpO/qAULvnuog0vvaMJqtDEh+3VrOFDX9+8wJ7CY7S1felSWX17p8H0jGE/FFARuKDS2FsubWuVBvWrxZa5EK4sib4jBHTs74tS2mmbjLNLFqSTGrzs6GvpPE/Xc81cIJ/15czhKAfiqeFLeG167Gb6GbSEnnaeyelZQpeo6ypfF7w2RZtbU64U2QSzGPa6fuRiXW6Cx73gZfmtmpqs9HdH0tYMra5rDnuiUvksHFU= ceph-b696f380-4522-11ee-817b-1b7a5f1ba315 2023-08-27T21:45:03.976 DEBUG:teuthology.orchestra.run.smithi130:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDmFR8M0V6PSsoPZhPk6xXAkNA1NmERbxMo1uWV+YAglLb4dNf/+TdsRyK+6fXFS+XDHuJq+iShFiUY5ia5G2srevBJ40OnqNjSh3NcBQN3taOJR+eNL0UBdDNCvVNXMkfWpTZNvYupvaPtYI/VI6yPNDys7eGzhB57jkzKaBaMvvnNVhIF7iiA01DD2nzHdcBtvX8j8ps/JulyDOE1hr62dPI4qR3sDoQgGhxOiErxzl49xldSsa2wpnOiFhMlKWhldXemLm2V1ADhZvGmpuv9EbpO/qAULvnuog0vvaMJqtDEh+3VrOFDX9+8wJ7CY7S1felSWX17p8H0jGE/FFARuKDS2FsubWuVBvWrxZa5EK4sib4jBHTs74tS2mmbjLNLFqSTGrzs6GvpPE/Xc81cIJ/15czhKAfiqeFLeG167Gb6GbSEnnaeyelZQpeo6ypfF7w2RZtbU64U2QSzGPa6fuRiXW6Cx73gZfmtmpqs9HdH0tYMra5rDnuiUvksHFU= ceph-b696f380-4522-11ee-817b-1b7a5f1ba315' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-08-27T21:45:03.998 INFO:teuthology.orchestra.run.smithi130.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDmFR8M0V6PSsoPZhPk6xXAkNA1NmERbxMo1uWV+YAglLb4dNf/+TdsRyK+6fXFS+XDHuJq+iShFiUY5ia5G2srevBJ40OnqNjSh3NcBQN3taOJR+eNL0UBdDNCvVNXMkfWpTZNvYupvaPtYI/VI6yPNDys7eGzhB57jkzKaBaMvvnNVhIF7iiA01DD2nzHdcBtvX8j8ps/JulyDOE1hr62dPI4qR3sDoQgGhxOiErxzl49xldSsa2wpnOiFhMlKWhldXemLm2V1ADhZvGmpuv9EbpO/qAULvnuog0vvaMJqtDEh+3VrOFDX9+8wJ7CY7S1felSWX17p8H0jGE/FFARuKDS2FsubWuVBvWrxZa5EK4sib4jBHTs74tS2mmbjLNLFqSTGrzs6GvpPE/Xc81cIJ/15czhKAfiqeFLeG167Gb6GbSEnnaeyelZQpeo6ypfF7w2RZtbU64U2QSzGPa6fuRiXW6Cx73gZfmtmpqs9HdH0tYMra5rDnuiUvksHFU= ceph-b696f380-4522-11ee-817b-1b7a5f1ba315 2023-08-27T21:45:04.008 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-08-27T21:45:04.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:04 smithi006 bash[27292]: audit 2023-08-27T21:45:03.039132+0000 mon.a (mon.0) 88 : audit [INF] from='client.? 172.21.15.6:0/3804358072' entity='client.admin' 2023-08-27T21:45:10.327 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-08-27T21:45:10.328 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-08-27T21:45:10.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:10 smithi006 bash[27292]: audit 2023-08-27T21:45:09.403150+0000 mon.a (mon.0) 89 : audit [INF] from='client.? 172.21.15.6:0/2328577621' entity='client.admin' 2023-08-27T21:45:12.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:12 smithi006 bash[27292]: cluster 2023-08-27T21:45:11.124418+0000 mgr.y (mgr.14146) 5 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:12.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:12 smithi006 bash[27292]: audit 2023-08-27T21:45:11.159147+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:12.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:12 smithi006 bash[27292]: audit 2023-08-27T21:45:11.463302+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi006", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:45:12.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:12 smithi006 bash[27292]: audit 2023-08-27T21:45:11.468594+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:12.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:12 smithi006 bash[27292]: audit 2023-08-27T21:45:11.471005+0000 mon.a (mon.0) 93 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:45:12.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:12 smithi006 bash[27292]: audit 2023-08-27T21:45:11.477697+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:14.382 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi130 2023-08-27T21:45:14.382 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-08-27T21:45:14.382 DEBUG:teuthology.orchestra.run.smithi130:> dd of=/etc/ceph/ceph.conf 2023-08-27T21:45:14.390 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-08-27T21:45:14.390 DEBUG:teuthology.orchestra.run.smithi130:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-08-27T21:45:14.441 INFO:tasks.cephadm:Adding host smithi130 to orchestrator... 2023-08-27T21:45:14.441 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph orch host add smithi130 2023-08-27T21:45:14.655 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:14 smithi006 bash[27292]: cluster 2023-08-27T21:45:13.124872+0000 mgr.y (mgr.14146) 6 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:14.655 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:14 smithi006 bash[27292]: audit 2023-08-27T21:45:13.682402+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:14.655 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:14 smithi006 bash[27292]: audit 2023-08-27T21:45:13.698563+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:45:14.655 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:14 smithi006 bash[27292]: audit 2023-08-27T21:45:13.701737+0000 mon.a (mon.0) 97 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:14.656 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:14 smithi006 bash[27292]: audit 2023-08-27T21:45:13.703292+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:45:14.656 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:14 smithi006 bash[27292]: audit 2023-08-27T21:45:13.754591+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:14.656 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:14 smithi006 bash[27292]: audit 2023-08-27T21:45:13.757567+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:15.701 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:15 smithi006 bash[27292]: cephadm 2023-08-27T21:45:13.705666+0000 mgr.y (mgr.14146) 8 : cephadm [INF] Updating smithi006:/etc/ceph/ceph.conf 2023-08-27T21:45:15.701 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:15 smithi006 bash[27292]: cephadm 2023-08-27T21:45:13.726416+0000 mgr.y (mgr.14146) 9 : cephadm [INF] Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2023-08-27T21:45:16.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:16 smithi006 bash[27292]: cluster 2023-08-27T21:45:15.125257+0000 mgr.y (mgr.14146) 10 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:18.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:45:17 smithi006 bash[27566]: Warning: Permanently added 'smithi130,172.21.15.130' (ECDSA) to the list of known hosts. 2023-08-27T21:45:18.627 INFO:teuthology.orchestra.run.smithi006.stdout:Added host 'smithi130' with addr '172.21.15.130' 2023-08-27T21:45:18.684 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:18 smithi006 bash[27292]: cluster 2023-08-27T21:45:17.125757+0000 mgr.y (mgr.14146) 11 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:18.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:45:18 smithi006 bash[27566]: Warning: Permanently added '172.21.15.130' (ECDSA) to the list of known hosts. 2023-08-27T21:45:19.337 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph orch host ls --format=json 2023-08-27T21:45:19.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:19 smithi006 bash[27292]: audit 2023-08-27T21:45:17.693224+0000 mgr.y (mgr.14146) 12 : audit [DBG] from='client.14170 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi130", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:45:19.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:19 smithi006 bash[27292]: audit 2023-08-27T21:45:18.624593+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:19.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:19 smithi006 bash[27292]: audit 2023-08-27T21:45:18.640538+0000 mon.a (mon.0) 102 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:45:19.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:19 smithi006 bash[27292]: audit 2023-08-27T21:45:19.194940+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:20.657 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:20 smithi006 bash[27292]: cephadm 2023-08-27T21:45:18.625414+0000 mgr.y (mgr.14146) 13 : cephadm [INF] Added host smithi130 2023-08-27T21:45:20.657 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:20 smithi006 bash[27292]: cluster 2023-08-27T21:45:19.126374+0000 mgr.y (mgr.14146) 14 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:21.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:21 smithi006 bash[27292]: audit 2023-08-27T21:45:20.460717+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:21.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:21 smithi006 bash[27292]: cluster 2023-08-27T21:45:21.126961+0000 mgr.y (mgr.14146) 15 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:22.607 INFO:teuthology.orchestra.run.smithi006.stdout: 2023-08-27T21:45:22.608 INFO:teuthology.orchestra.run.smithi006.stdout:[{"addr": "172.21.15.6", "hostname": "smithi006", "labels": [], "status": ""}, {"addr": "172.21.15.130", "hostname": "smithi130", "labels": [], "status": ""}] 2023-08-27T21:45:23.240 INFO:tasks.cephadm:Setting crush tunables to default 2023-08-27T21:45:23.240 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd crush tunables default 2023-08-27T21:45:24.598 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:24 smithi006 bash[27292]: audit 2023-08-27T21:45:22.604660+0000 mgr.y (mgr.14146) 16 : audit [DBG] from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T21:45:24.598 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:24 smithi006 bash[27292]: cluster 2023-08-27T21:45:23.127484+0000 mgr.y (mgr.14146) 17 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:26.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:26 smithi006 bash[27292]: cluster 2023-08-27T21:45:25.127944+0000 mgr.y (mgr.14146) 18 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:26.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:26 smithi006 bash[27292]: audit 2023-08-27T21:45:26.123129+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:27.450 INFO:teuthology.orchestra.run.smithi006.stderr:adjusted tunables profile to default 2023-08-27T21:45:27.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:27 smithi006 bash[27292]: audit 2023-08-27T21:45:26.440936+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:27.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:27 smithi006 bash[27292]: audit 2023-08-27T21:45:26.442225+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi130", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:45:27.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:27 smithi006 bash[27292]: audit 2023-08-27T21:45:26.443803+0000 mon.a (mon.0) 108 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:27.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:27 smithi006 bash[27292]: audit 2023-08-27T21:45:26.445290+0000 mon.a (mon.0) 109 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:45:27.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:27 smithi006 bash[27292]: cephadm 2023-08-27T21:45:26.447107+0000 mgr.y (mgr.14146) 19 : cephadm [INF] Updating smithi130:/etc/ceph/ceph.conf 2023-08-27T21:45:27.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:27 smithi006 bash[27292]: audit 2023-08-27T21:45:26.454036+0000 mon.a (mon.0) 110 : audit [INF] from='client.? 172.21.15.6:0/2817221047' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-08-27T21:45:27.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:27 smithi006 bash[27292]: cephadm 2023-08-27T21:45:26.464384+0000 mgr.y (mgr.14146) 20 : cephadm [INF] Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2023-08-27T21:45:27.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:27 smithi006 bash[27292]: audit 2023-08-27T21:45:26.494679+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:27.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:27 smithi006 bash[27292]: audit 2023-08-27T21:45:26.500406+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:27.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:27 smithi006 bash[27292]: cluster 2023-08-27T21:45:27.128411+0000 mgr.y (mgr.14146) 21 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:28.142 INFO:tasks.cephadm:Adding mon.a on smithi006 2023-08-27T21:45:28.142 INFO:tasks.cephadm:Adding mon.c on smithi006 2023-08-27T21:45:28.142 INFO:tasks.cephadm:Adding mon.b on smithi130 2023-08-27T21:45:28.143 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph orch apply mon '3;smithi006:172.21.15.6=a;smithi006:[v2:172.21.15.6:3301,v1:172.21.15.6:6790]=c;smithi130:172.21.15.130=b' 2023-08-27T21:45:28.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:28 smithi006 bash[27292]: audit 2023-08-27T21:45:27.441657+0000 mon.a (mon.0) 113 : audit [INF] from='client.? 172.21.15.6:0/2817221047' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-08-27T21:45:28.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:28 smithi006 bash[27292]: cluster 2023-08-27T21:45:27.441847+0000 mon.a (mon.0) 114 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-08-27T21:45:29.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:29 smithi006 bash[27292]: cluster 2023-08-27T21:45:29.128904+0000 mgr.y (mgr.14146) 22 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:31.375 INFO:teuthology.orchestra.run.smithi130.stdout:Scheduled mon update... 2023-08-27T21:45:32.408 DEBUG:teuthology.orchestra.run.smithi006:mon.c> sudo journalctl -f -n 0 -u ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@mon.c.service 2023-08-27T21:45:32.411 DEBUG:teuthology.orchestra.run.smithi130:mon.b> sudo journalctl -f -n 0 -u ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@mon.b.service 2023-08-27T21:45:32.413 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-08-27T21:45:32.414 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph mon dump -f json 2023-08-27T21:45:32.662 INFO:journalctl@ceph.mon.c.smithi006.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-08-27T21:45:32.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:32 smithi006 bash[27292]: cluster 2023-08-27T21:45:31.129267+0000 mgr.y (mgr.14146) 23 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:32.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:32 smithi006 bash[27292]: audit 2023-08-27T21:45:31.359384+0000 mgr.y (mgr.14146) 24 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi006:172.21.15.6=a;smithi006:[v2:172.21.15.6:3301,v1:172.21.15.6:6790]=c;smithi130:172.21.15.130=b", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:45:32.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:32 smithi006 bash[27292]: cephadm 2023-08-27T21:45:31.365511+0000 mgr.y (mgr.14146) 25 : cephadm [INF] Saving service mon spec with placement smithi006:172.21.15.6=a;smithi006:[v2:172.21.15.6:3301,v1:172.21.15.6:6790]=c;smithi130:172.21.15.130=b;count:3 2023-08-27T21:45:32.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:32 smithi006 bash[27292]: audit 2023-08-27T21:45:31.373149+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:32.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:32 smithi006 bash[27292]: audit 2023-08-27T21:45:31.378676+0000 mon.a (mon.0) 116 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:45:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:32 smithi006 bash[27292]: audit 2023-08-27T21:45:31.381555+0000 mon.a (mon.0) 117 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:32 smithi006 bash[27292]: audit 2023-08-27T21:45:31.383176+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:45:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:32 smithi006 bash[27292]: audit 2023-08-27T21:45:31.391303+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:32 smithi006 bash[27292]: audit 2023-08-27T21:45:31.394676+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T21:45:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:32 smithi006 bash[27292]: audit 2023-08-27T21:45:31.395976+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:33.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:33 smithi006 bash[27292]: cephadm 2023-08-27T21:45:31.397175+0000 mgr.y (mgr.14146) 26 : cephadm [INF] Deploying daemon mon.b on smithi130 2023-08-27T21:45:33.805 INFO:journalctl@ceph.mon.b.smithi130.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-08-27T21:45:34.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:34 smithi006 bash[27292]: cluster 2023-08-27T21:45:33.129731+0000 mgr.y (mgr.14146) 27 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:35.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:35 smithi006 bash[27292]: cluster 2023-08-27T21:45:35.130241+0000 mgr.y (mgr.14146) 28 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:37.911 INFO:teuthology.orchestra.run.smithi130.stdout: 2023-08-27T21:45:37.911 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":1,"fsid":"b696f380-4522-11ee-817b-1b7a5f1ba315","modified":"2023-08-27T21:43:46.272349Z","created":"2023-08-27T21:43:46.272349Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-08-27T21:45:37.914 INFO:teuthology.orchestra.run.smithi130.stderr:dumped monmap epoch 1 2023-08-27T21:45:38.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:38 smithi006 bash[27292]: cluster 2023-08-27T21:45:37.130727+0000 mgr.y (mgr.14146) 29 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:38.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:38 smithi006 bash[27292]: audit 2023-08-27T21:45:37.909272+0000 mon.a (mon.0) 122 : audit [DBG] from='client.? 172.21.15.130:0/3699301285' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-08-27T21:45:39.549 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-08-27T21:45:39.550 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph mon dump -f json 2023-08-27T21:45:39.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:39 smithi006 bash[27292]: audit 2023-08-27T21:45:38.440000+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:39.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:39 smithi006 bash[27292]: audit 2023-08-27T21:45:38.441750+0000 mon.a (mon.0) 124 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T21:45:39.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:39 smithi006 bash[27292]: audit 2023-08-27T21:45:38.443575+0000 mon.a (mon.0) 125 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:39.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:39 smithi006 bash[27292]: cephadm 2023-08-27T21:45:38.444906+0000 mgr.y (mgr.14146) 30 : cephadm [INF] Deploying daemon mon.c on smithi006 2023-08-27T21:45:39.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:39 smithi006 bash[27292]: cluster 2023-08-27T21:45:39.131202+0000 mgr.y (mgr.14146) 31 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:39.763 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:39 smithi130 bash[27579]: debug 2023-08-27T21:45:39.677+0000 7f28008f4700 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-08-27T21:45:43.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:43 smithi006 bash[33918]: debug 2023-08-27T21:45:43.781+0000 7f6f14de1700 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-08-27T21:45:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:44 smithi130 bash[27579]: audit 2023-08-27T21:45:39.692394+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T21:45:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:44 smithi130 bash[27579]: audit 2023-08-27T21:45:39.692586+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:45:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:44 smithi130 bash[27579]: cluster 2023-08-27T21:45:39.692768+0000 mon.a (mon.0) 129 : cluster [INF] mon.a calling monitor election 2023-08-27T21:45:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:44 smithi130 bash[27579]: audit 2023-08-27T21:45:40.688957+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:45:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:44 smithi130 bash[27579]: cluster 2023-08-27T21:45:41.131645+0000 mgr.y (mgr.14146) 32 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:44 smithi130 bash[27579]: audit 2023-08-27T21:45:41.688346+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:45:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:44 smithi130 bash[27579]: cluster 2023-08-27T21:45:41.691523+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-08-27T21:45:45.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:44 smithi130 bash[27579]: audit 2023-08-27T21:45:42.688730+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:45:45.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:44 smithi130 bash[27579]: cluster 2023-08-27T21:45:43.132088+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:45.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:44 smithi130 bash[27579]: audit 2023-08-27T21:45:43.688897+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:45:45.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:44 smithi130 bash[27579]: audit 2023-08-27T21:45:43.790860+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:45.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:44 smithi130 bash[27579]: audit 2023-08-27T21:45:44.689411+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:45:45.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:44 smithi130 bash[27579]: cluster 2023-08-27T21:45:44.736218+0000 mon.a (mon.0) 136 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-08-27T21:45:45.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:44 smithi130 bash[27579]: cluster 2023-08-27T21:45:44.741890+0000 mon.a (mon.0) 137 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0],b=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0]} removed_ranks: {} 2023-08-27T21:45:45.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:44 smithi130 bash[27579]: cluster 2023-08-27T21:45:44.761781+0000 mon.a (mon.0) 138 : cluster [DBG] fsmap 2023-08-27T21:45:45.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:44 smithi130 bash[27579]: cluster 2023-08-27T21:45:44.761843+0000 mon.a (mon.0) 139 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-08-27T21:45:45.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:44 smithi130 bash[27579]: cluster 2023-08-27T21:45:44.762187+0000 mon.a (mon.0) 140 : cluster [DBG] mgrmap e13: y(active, since 53s) 2023-08-27T21:45:45.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:44 smithi130 bash[27579]: cluster 2023-08-27T21:45:44.769161+0000 mon.a (mon.0) 141 : cluster [INF] overall HEALTH_OK 2023-08-27T21:45:45.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:44 smithi130 bash[27579]: audit 2023-08-27T21:45:44.773578+0000 mon.a (mon.0) 142 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:45.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:44 smithi130 bash[27579]: audit 2023-08-27T21:45:44.775212+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:45:45.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:44 smithi130 bash[27579]: audit 2023-08-27T21:45:44.790744+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:45.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:44 smithi006 bash[27292]: audit 2023-08-27T21:45:39.692394+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T21:45:45.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:44 smithi006 bash[27292]: audit 2023-08-27T21:45:39.692586+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:45:45.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:44 smithi006 bash[27292]: cluster 2023-08-27T21:45:39.692768+0000 mon.a (mon.0) 129 : cluster [INF] mon.a calling monitor election 2023-08-27T21:45:45.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:44 smithi006 bash[27292]: audit 2023-08-27T21:45:40.688957+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:45:45.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:44 smithi006 bash[27292]: cluster 2023-08-27T21:45:41.131645+0000 mgr.y (mgr.14146) 32 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:44 smithi006 bash[27292]: audit 2023-08-27T21:45:41.688346+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:45:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:44 smithi006 bash[27292]: cluster 2023-08-27T21:45:41.691523+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-08-27T21:45:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:44 smithi006 bash[27292]: audit 2023-08-27T21:45:42.688730+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:45:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:44 smithi006 bash[27292]: cluster 2023-08-27T21:45:43.132088+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:44 smithi006 bash[27292]: audit 2023-08-27T21:45:43.688897+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:45:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:44 smithi006 bash[27292]: audit 2023-08-27T21:45:43.790860+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:44 smithi006 bash[27292]: audit 2023-08-27T21:45:44.689411+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:45:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:44 smithi006 bash[27292]: cluster 2023-08-27T21:45:44.736218+0000 mon.a (mon.0) 136 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-08-27T21:45:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:44 smithi006 bash[27292]: cluster 2023-08-27T21:45:44.741890+0000 mon.a (mon.0) 137 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0],b=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0]} removed_ranks: {} 2023-08-27T21:45:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:44 smithi006 bash[27292]: cluster 2023-08-27T21:45:44.761781+0000 mon.a (mon.0) 138 : cluster [DBG] fsmap 2023-08-27T21:45:45.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:44 smithi006 bash[27292]: cluster 2023-08-27T21:45:44.761843+0000 mon.a (mon.0) 139 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-08-27T21:45:45.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:44 smithi006 bash[27292]: cluster 2023-08-27T21:45:44.762187+0000 mon.a (mon.0) 140 : cluster [DBG] mgrmap e13: y(active, since 53s) 2023-08-27T21:45:45.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:44 smithi006 bash[27292]: cluster 2023-08-27T21:45:44.769161+0000 mon.a (mon.0) 141 : cluster [INF] overall HEALTH_OK 2023-08-27T21:45:45.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:44 smithi006 bash[27292]: audit 2023-08-27T21:45:44.773578+0000 mon.a (mon.0) 142 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:45.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:44 smithi006 bash[27292]: audit 2023-08-27T21:45:44.775212+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:45:45.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:44 smithi006 bash[27292]: audit 2023-08-27T21:45:44.790744+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:50.844 INFO:teuthology.orchestra.run.smithi130.stdout: 2023-08-27T21:45:50.845 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":3,"fsid":"b696f380-4522-11ee-817b-1b7a5f1ba315","modified":"2023-08-27T21:45:45.791311Z","created":"2023-08-27T21:43:46.272349Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3301","nonce":0},{"type":"v1","addr":"172.21.15.6:6790","nonce":0}]},"addr":"172.21.15.6:6790/0","public_addr":"172.21.15.6:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-08-27T21:45:50.850 INFO:teuthology.orchestra.run.smithi130.stderr:dumped monmap epoch 3 2023-08-27T21:45:51.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: audit 2023-08-27T21:45:45.797263+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T21:45:51.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: cluster 2023-08-27T21:45:45.797807+0000 mon.a (mon.0) 148 : cluster [INF] mon.a calling monitor election 2023-08-27T21:45:51.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: cluster 2023-08-27T21:45:45.798398+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-08-27T21:45:51.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: audit 2023-08-27T21:45:45.818438+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:45:51.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: audit 2023-08-27T21:45:45.818946+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:51.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: audit 2023-08-27T21:45:45.971186+0000 mon.a (mon.0) 151 : audit [DBG] from='client.? 172.21.15.130:0/2283875694' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-08-27T21:45:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: audit 2023-08-27T21:45:46.791108+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: cluster 2023-08-27T21:45:47.132992+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: audit 2023-08-27T21:45:47.791561+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: audit 2023-08-27T21:45:48.791691+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: cluster 2023-08-27T21:45:49.133533+0000 mgr.y (mgr.14146) 36 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: audit 2023-08-27T21:45:49.791398+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: audit 2023-08-27T21:45:50.791565+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: cluster 2023-08-27T21:45:50.818391+0000 mon.a (mon.0) 157 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-08-27T21:45:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: cluster 2023-08-27T21:45:50.823927+0000 mon.a (mon.0) 158 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0],b=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],c=[v2:172.21.15.6:3301/0,v1:172.21.15.6:6790/0]} removed_ranks: {} 2023-08-27T21:45:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: cluster 2023-08-27T21:45:50.841780+0000 mon.a (mon.0) 159 : cluster [DBG] fsmap 2023-08-27T21:45:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: cluster 2023-08-27T21:45:50.841838+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-08-27T21:45:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: cluster 2023-08-27T21:45:50.842178+0000 mon.a (mon.0) 161 : cluster [DBG] mgrmap e13: y(active, since 59s) 2023-08-27T21:45:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: cluster 2023-08-27T21:45:50.842499+0000 mon.a (mon.0) 162 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-08-27T21:45:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: cluster 2023-08-27T21:45:50.849553+0000 mon.a (mon.0) 163 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-08-27T21:45:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: cluster 2023-08-27T21:45:50.849617+0000 mon.a (mon.0) 164 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-08-27T21:45:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: cluster 2023-08-27T21:45:50.849657+0000 mon.a (mon.0) 165 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.6:3301/0,v1:172.21.15.6:6790/0] is down (out of quorum) 2023-08-27T21:45:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: audit 2023-08-27T21:45:50.856240+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: audit 2023-08-27T21:45:50.863714+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: audit 2023-08-27T21:45:50.871912+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: audit 2023-08-27T21:45:50.873170+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:51.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:50 smithi006 bash[27292]: audit 2023-08-27T21:45:50.874743+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:45:51.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: audit 2023-08-27T21:45:45.797263+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T21:45:51.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: cluster 2023-08-27T21:45:45.797807+0000 mon.a (mon.0) 148 : cluster [INF] mon.a calling monitor election 2023-08-27T21:45:51.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: cluster 2023-08-27T21:45:45.798398+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-08-27T21:45:51.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: audit 2023-08-27T21:45:45.818438+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:45:51.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: audit 2023-08-27T21:45:45.818946+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:51.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: audit 2023-08-27T21:45:45.971186+0000 mon.a (mon.0) 151 : audit [DBG] from='client.? 172.21.15.130:0/2283875694' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-08-27T21:45:51.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: audit 2023-08-27T21:45:46.791108+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:51.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: cluster 2023-08-27T21:45:47.132992+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:51.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: audit 2023-08-27T21:45:47.791561+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:51.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: audit 2023-08-27T21:45:48.791691+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:51.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: cluster 2023-08-27T21:45:49.133533+0000 mgr.y (mgr.14146) 36 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:51.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: audit 2023-08-27T21:45:49.791398+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:51.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: audit 2023-08-27T21:45:50.791565+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:51.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: cluster 2023-08-27T21:45:50.818391+0000 mon.a (mon.0) 157 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-08-27T21:45:51.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: cluster 2023-08-27T21:45:50.823927+0000 mon.a (mon.0) 158 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0],b=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],c=[v2:172.21.15.6:3301/0,v1:172.21.15.6:6790/0]} removed_ranks: {} 2023-08-27T21:45:51.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: cluster 2023-08-27T21:45:50.841780+0000 mon.a (mon.0) 159 : cluster [DBG] fsmap 2023-08-27T21:45:51.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: cluster 2023-08-27T21:45:50.841838+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-08-27T21:45:51.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: cluster 2023-08-27T21:45:50.842178+0000 mon.a (mon.0) 161 : cluster [DBG] mgrmap e13: y(active, since 59s) 2023-08-27T21:45:51.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: cluster 2023-08-27T21:45:50.842499+0000 mon.a (mon.0) 162 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-08-27T21:45:51.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: cluster 2023-08-27T21:45:50.849553+0000 mon.a (mon.0) 163 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-08-27T21:45:51.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: cluster 2023-08-27T21:45:50.849617+0000 mon.a (mon.0) 164 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-08-27T21:45:51.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: cluster 2023-08-27T21:45:50.849657+0000 mon.a (mon.0) 165 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.6:3301/0,v1:172.21.15.6:6790/0] is down (out of quorum) 2023-08-27T21:45:51.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: audit 2023-08-27T21:45:50.856240+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:51.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: audit 2023-08-27T21:45:50.863714+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:51.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: audit 2023-08-27T21:45:50.871912+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:51.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: audit 2023-08-27T21:45:50.873170+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:51.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:50 smithi130 bash[27579]: audit 2023-08-27T21:45:50.874743+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:45:51.638 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-08-27T21:45:51.638 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph config generate-minimal-conf 2023-08-27T21:45:52.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:51 smithi006 bash[27292]: cephadm 2023-08-27T21:45:50.876854+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Updating smithi006:/etc/ceph/ceph.conf 2023-08-27T21:45:52.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:51 smithi006 bash[27292]: cephadm 2023-08-27T21:45:50.877513+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Updating smithi130:/etc/ceph/ceph.conf 2023-08-27T21:45:52.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:51 smithi006 bash[27292]: audit 2023-08-27T21:45:50.908932+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:52.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:51 smithi006 bash[27292]: audit 2023-08-27T21:45:50.911775+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:52.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:51 smithi006 bash[27292]: audit 2023-08-27T21:45:50.915948+0000 mon.a (mon.0) 173 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:52.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:51 smithi006 bash[27292]: cephadm 2023-08-27T21:45:50.917819+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-08-27T21:45:52.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:51 smithi006 bash[27292]: audit 2023-08-27T21:45:50.918240+0000 mon.a (mon.0) 174 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T21:45:52.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:51 smithi006 bash[27292]: audit 2023-08-27T21:45:50.919595+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-27T21:45:52.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:51 smithi006 bash[27292]: audit 2023-08-27T21:45:50.920891+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:52.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:51 smithi006 bash[27292]: cephadm 2023-08-27T21:45:50.922002+0000 mgr.y (mgr.14146) 40 : cephadm [INF] Reconfiguring daemon mon.a on smithi006 2023-08-27T21:45:52.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:51 smithi006 bash[27292]: cluster 2023-08-27T21:45:51.133890+0000 mgr.y (mgr.14146) 41 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:52.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:51 smithi006 bash[27292]: audit 2023-08-27T21:45:51.791354+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:51 smithi130 bash[27579]: cephadm 2023-08-27T21:45:50.876854+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Updating smithi006:/etc/ceph/ceph.conf 2023-08-27T21:45:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:51 smithi130 bash[27579]: cephadm 2023-08-27T21:45:50.877513+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Updating smithi130:/etc/ceph/ceph.conf 2023-08-27T21:45:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:51 smithi130 bash[27579]: audit 2023-08-27T21:45:50.908932+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:51 smithi130 bash[27579]: audit 2023-08-27T21:45:50.911775+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:51 smithi130 bash[27579]: audit 2023-08-27T21:45:50.915948+0000 mon.a (mon.0) 173 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:51 smithi130 bash[27579]: cephadm 2023-08-27T21:45:50.917819+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-08-27T21:45:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:51 smithi130 bash[27579]: audit 2023-08-27T21:45:50.918240+0000 mon.a (mon.0) 174 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T21:45:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:51 smithi130 bash[27579]: audit 2023-08-27T21:45:50.919595+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-27T21:45:52.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:51 smithi130 bash[27579]: audit 2023-08-27T21:45:50.920891+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:52.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:51 smithi130 bash[27579]: cephadm 2023-08-27T21:45:50.922002+0000 mgr.y (mgr.14146) 40 : cephadm [INF] Reconfiguring daemon mon.a on smithi006 2023-08-27T21:45:52.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:51 smithi130 bash[27579]: cluster 2023-08-27T21:45:51.133890+0000 mgr.y (mgr.14146) 41 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:52.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:51 smithi130 bash[27579]: audit 2023-08-27T21:45:51.791354+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:52.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:52 smithi006 bash[33918]: cephadm 2023-08-27T21:45:50.876854+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Updating smithi006:/etc/ceph/ceph.conf 2023-08-27T21:45:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:52 smithi006 bash[33918]: cephadm 2023-08-27T21:45:50.877513+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Updating smithi130:/etc/ceph/ceph.conf 2023-08-27T21:45:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:52 smithi006 bash[33918]: audit 2023-08-27T21:45:50.908932+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:52 smithi006 bash[33918]: audit 2023-08-27T21:45:50.911775+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:52 smithi006 bash[33918]: audit 2023-08-27T21:45:50.915948+0000 mon.a (mon.0) 173 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:52 smithi006 bash[33918]: cephadm 2023-08-27T21:45:50.917819+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-08-27T21:45:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:52 smithi006 bash[33918]: audit 2023-08-27T21:45:50.918240+0000 mon.a (mon.0) 174 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T21:45:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:52 smithi006 bash[33918]: audit 2023-08-27T21:45:50.919595+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-27T21:45:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:52 smithi006 bash[33918]: audit 2023-08-27T21:45:50.920891+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:52 smithi006 bash[33918]: cephadm 2023-08-27T21:45:50.922002+0000 mgr.y (mgr.14146) 40 : cephadm [INF] Reconfiguring daemon mon.a on smithi006 2023-08-27T21:45:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:52 smithi006 bash[33918]: cluster 2023-08-27T21:45:51.133890+0000 mgr.y (mgr.14146) 41 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:52 smithi006 bash[33918]: audit 2023-08-27T21:45:51.791354+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:53.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:53 smithi130 bash[27579]: cluster 2023-08-27T21:45:47.793410+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-08-27T21:45:53.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:53 smithi130 bash[27579]: cluster 2023-08-27T21:45:52.823850+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2023-08-27T21:45:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:53 smithi130 bash[27579]: cluster 2023-08-27T21:45:52.824465+0000 mon.a (mon.0) 179 : cluster [INF] mon.a calling monitor election 2023-08-27T21:45:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:53 smithi130 bash[27579]: cluster 2023-08-27T21:45:52.824572+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2023-08-27T21:45:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:53 smithi130 bash[27579]: cluster 2023-08-27T21:45:52.846916+0000 mon.a (mon.0) 180 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-08-27T21:45:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:53 smithi130 bash[27579]: cluster 2023-08-27T21:45:52.851189+0000 mon.a (mon.0) 181 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0],b=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],c=[v2:172.21.15.6:3301/0,v1:172.21.15.6:6790/0]} removed_ranks: {} 2023-08-27T21:45:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:53 smithi130 bash[27579]: cluster 2023-08-27T21:45:52.851724+0000 mon.a (mon.0) 182 : cluster [DBG] fsmap 2023-08-27T21:45:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:53 smithi130 bash[27579]: cluster 2023-08-27T21:45:52.851783+0000 mon.a (mon.0) 183 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-08-27T21:45:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:53 smithi130 bash[27579]: cluster 2023-08-27T21:45:52.852025+0000 mon.a (mon.0) 184 : cluster [DBG] mgrmap e13: y(active, since 61s) 2023-08-27T21:45:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:53 smithi130 bash[27579]: cluster 2023-08-27T21:45:52.852229+0000 mon.a (mon.0) 185 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-08-27T21:45:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:53 smithi130 bash[27579]: cluster 2023-08-27T21:45:52.852269+0000 mon.a (mon.0) 186 : cluster [INF] Cluster is now healthy 2023-08-27T21:45:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:53 smithi130 bash[27579]: cluster 2023-08-27T21:45:52.873190+0000 mon.a (mon.0) 187 : cluster [INF] overall HEALTH_OK 2023-08-27T21:45:53.658 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[33918]: cluster 2023-08-27T21:45:47.793410+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-08-27T21:45:53.658 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[33918]: cluster 2023-08-27T21:45:52.823850+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2023-08-27T21:45:53.658 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[33918]: cluster 2023-08-27T21:45:52.824465+0000 mon.a (mon.0) 179 : cluster [INF] mon.a calling monitor election 2023-08-27T21:45:53.659 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[33918]: cluster 2023-08-27T21:45:52.824572+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2023-08-27T21:45:53.659 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[33918]: cluster 2023-08-27T21:45:52.846916+0000 mon.a (mon.0) 180 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-08-27T21:45:53.659 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[33918]: cluster 2023-08-27T21:45:52.851189+0000 mon.a (mon.0) 181 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0],b=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],c=[v2:172.21.15.6:3301/0,v1:172.21.15.6:6790/0]} removed_ranks: {} 2023-08-27T21:45:53.659 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[33918]: cluster 2023-08-27T21:45:52.851724+0000 mon.a (mon.0) 182 : cluster [DBG] fsmap 2023-08-27T21:45:53.659 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[33918]: cluster 2023-08-27T21:45:52.851783+0000 mon.a (mon.0) 183 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-08-27T21:45:53.659 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[33918]: cluster 2023-08-27T21:45:52.852025+0000 mon.a (mon.0) 184 : cluster [DBG] mgrmap e13: y(active, since 61s) 2023-08-27T21:45:53.660 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[33918]: cluster 2023-08-27T21:45:52.852229+0000 mon.a (mon.0) 185 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-08-27T21:45:53.660 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[33918]: cluster 2023-08-27T21:45:52.852269+0000 mon.a (mon.0) 186 : cluster [INF] Cluster is now healthy 2023-08-27T21:45:53.660 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[33918]: cluster 2023-08-27T21:45:52.873190+0000 mon.a (mon.0) 187 : cluster [INF] overall HEALTH_OK 2023-08-27T21:45:53.660 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[27292]: cluster 2023-08-27T21:45:47.793410+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-08-27T21:45:53.660 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[27292]: cluster 2023-08-27T21:45:52.823850+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2023-08-27T21:45:53.660 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[27292]: cluster 2023-08-27T21:45:52.824465+0000 mon.a (mon.0) 179 : cluster [INF] mon.a calling monitor election 2023-08-27T21:45:53.660 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[27292]: cluster 2023-08-27T21:45:52.824572+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2023-08-27T21:45:53.660 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[27292]: cluster 2023-08-27T21:45:52.846916+0000 mon.a (mon.0) 180 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-08-27T21:45:53.660 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[27292]: cluster 2023-08-27T21:45:52.851189+0000 mon.a (mon.0) 181 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0],b=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],c=[v2:172.21.15.6:3301/0,v1:172.21.15.6:6790/0]} removed_ranks: {} 2023-08-27T21:45:53.660 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[27292]: cluster 2023-08-27T21:45:52.851724+0000 mon.a (mon.0) 182 : cluster [DBG] fsmap 2023-08-27T21:45:53.661 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[27292]: cluster 2023-08-27T21:45:52.851783+0000 mon.a (mon.0) 183 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-08-27T21:45:53.661 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[27292]: cluster 2023-08-27T21:45:52.852025+0000 mon.a (mon.0) 184 : cluster [DBG] mgrmap e13: y(active, since 61s) 2023-08-27T21:45:53.661 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[27292]: cluster 2023-08-27T21:45:52.852229+0000 mon.a (mon.0) 185 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-08-27T21:45:53.661 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[27292]: cluster 2023-08-27T21:45:52.852269+0000 mon.a (mon.0) 186 : cluster [INF] Cluster is now healthy 2023-08-27T21:45:53.661 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:53 smithi006 bash[27292]: cluster 2023-08-27T21:45:52.873190+0000 mon.a (mon.0) 187 : cluster [INF] overall HEALTH_OK 2023-08-27T21:45:54.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:54 smithi130 bash[27579]: cluster 2023-08-27T21:45:53.134332+0000 mgr.y (mgr.14146) 42 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:54.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:54 smithi130 bash[27579]: audit 2023-08-27T21:45:53.730694+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:54 smithi130 bash[27579]: audit 2023-08-27T21:45:53.732084+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T21:45:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:54 smithi130 bash[27579]: audit 2023-08-27T21:45:53.733289+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-27T21:45:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:54 smithi130 bash[27579]: audit 2023-08-27T21:45:53.734471+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:54 smithi130 bash[27579]: audit 2023-08-27T21:45:53.791468+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:54.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:54 smithi006 bash[27292]: cluster 2023-08-27T21:45:53.134332+0000 mgr.y (mgr.14146) 42 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:54.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:54 smithi006 bash[27292]: audit 2023-08-27T21:45:53.730694+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:54.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:54 smithi006 bash[27292]: audit 2023-08-27T21:45:53.732084+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T21:45:54.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:54 smithi006 bash[27292]: audit 2023-08-27T21:45:53.733289+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-27T21:45:54.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:54 smithi006 bash[27292]: audit 2023-08-27T21:45:53.734471+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:54.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:54 smithi006 bash[27292]: audit 2023-08-27T21:45:53.791468+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:54.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:54 smithi006 bash[33918]: cluster 2023-08-27T21:45:53.134332+0000 mgr.y (mgr.14146) 42 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:54.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:54 smithi006 bash[33918]: audit 2023-08-27T21:45:53.730694+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:54.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:54 smithi006 bash[33918]: audit 2023-08-27T21:45:53.732084+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T21:45:54.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:54 smithi006 bash[33918]: audit 2023-08-27T21:45:53.733289+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-27T21:45:54.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:54 smithi006 bash[33918]: audit 2023-08-27T21:45:53.734471+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:54.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:54 smithi006 bash[33918]: audit 2023-08-27T21:45:53.791468+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:45:55.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:55 smithi130 bash[27579]: cephadm 2023-08-27T21:45:53.731678+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-08-27T21:45:55.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:55 smithi130 bash[27579]: cephadm 2023-08-27T21:45:53.735493+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring daemon mon.c on smithi006 2023-08-27T21:45:55.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:55 smithi006 bash[27292]: cephadm 2023-08-27T21:45:53.731678+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-08-27T21:45:55.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:55 smithi006 bash[27292]: cephadm 2023-08-27T21:45:53.735493+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring daemon mon.c on smithi006 2023-08-27T21:45:55.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:55 smithi006 bash[33918]: cephadm 2023-08-27T21:45:53.731678+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-08-27T21:45:55.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:55 smithi006 bash[33918]: cephadm 2023-08-27T21:45:53.735493+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring daemon mon.c on smithi006 2023-08-27T21:45:56.575 INFO:teuthology.orchestra.run.smithi006.stdout:# minimal ceph.conf for b696f380-4522-11ee-817b-1b7a5f1ba315 2023-08-27T21:45:56.575 INFO:teuthology.orchestra.run.smithi006.stdout:[global] 2023-08-27T21:45:56.576 INFO:teuthology.orchestra.run.smithi006.stdout: fsid = b696f380-4522-11ee-817b-1b7a5f1ba315 2023-08-27T21:45:56.576 INFO:teuthology.orchestra.run.smithi006.stdout: mon_host = [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] [v2:172.21.15.6:3301/0,v1:172.21.15.6:6790/0] 2023-08-27T21:45:56.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:56 smithi130 bash[27579]: cluster 2023-08-27T21:45:55.134786+0000 mgr.y (mgr.14146) 45 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:56.620 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:56 smithi006 bash[27292]: cluster 2023-08-27T21:45:55.134786+0000 mgr.y (mgr.14146) 45 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:56.620 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:56 smithi006 bash[33918]: cluster 2023-08-27T21:45:55.134786+0000 mgr.y (mgr.14146) 45 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:57.558 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-08-27T21:45:57.559 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2023-08-27T21:45:57.559 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd of=/etc/ceph/ceph.conf 2023-08-27T21:45:57.573 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2023-08-27T21:45:57.574 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-08-27T21:45:57.589 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:57 smithi130 bash[27579]: audit 2023-08-27T21:45:56.574624+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.6:0/1097881054' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:57.633 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:57 smithi006 bash[27292]: audit 2023-08-27T21:45:56.574624+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.6:0/1097881054' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:57.633 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:57 smithi006 bash[33918]: audit 2023-08-27T21:45:56.574624+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.6:0/1097881054' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:57.636 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-08-27T21:45:57.637 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/ceph/ceph.conf 2023-08-27T21:45:57.786 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-08-27T21:45:57.786 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-08-27T21:45:57.799 INFO:tasks.cephadm:Adding mgr.y on smithi006 2023-08-27T21:45:57.799 INFO:tasks.cephadm:Adding mgr.x on smithi130 2023-08-27T21:45:57.800 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph orch apply mgr '2;smithi006=y;smithi130=x' 2023-08-27T21:45:58.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:58 smithi130 bash[27579]: cluster 2023-08-27T21:45:57.135227+0000 mgr.y (mgr.14146) 46 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:58 smithi130 bash[27579]: audit 2023-08-27T21:45:57.365624+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:58 smithi130 bash[27579]: cephadm 2023-08-27T21:45:57.366551+0000 mgr.y (mgr.14146) 47 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-08-27T21:45:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:58 smithi130 bash[27579]: audit 2023-08-27T21:45:57.366962+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T21:45:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:58 smithi130 bash[27579]: audit 2023-08-27T21:45:57.368104+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-27T21:45:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:58 smithi130 bash[27579]: audit 2023-08-27T21:45:57.369303+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:45:58 smithi130 bash[27579]: cephadm 2023-08-27T21:45:57.370419+0000 mgr.y (mgr.14146) 48 : cephadm [INF] Reconfiguring daemon mon.b on smithi130 2023-08-27T21:45:58.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:58 smithi006 bash[27292]: cluster 2023-08-27T21:45:57.135227+0000 mgr.y (mgr.14146) 46 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:58.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:58 smithi006 bash[27292]: audit 2023-08-27T21:45:57.365624+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:58.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:58 smithi006 bash[27292]: cephadm 2023-08-27T21:45:57.366551+0000 mgr.y (mgr.14146) 47 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-08-27T21:45:58.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:58 smithi006 bash[27292]: audit 2023-08-27T21:45:57.366962+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T21:45:58.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:58 smithi006 bash[27292]: audit 2023-08-27T21:45:57.368104+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-27T21:45:58.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:58 smithi006 bash[27292]: audit 2023-08-27T21:45:57.369303+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:58.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:45:58 smithi006 bash[27292]: cephadm 2023-08-27T21:45:57.370419+0000 mgr.y (mgr.14146) 48 : cephadm [INF] Reconfiguring daemon mon.b on smithi130 2023-08-27T21:45:58.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:58 smithi006 bash[33918]: cluster 2023-08-27T21:45:57.135227+0000 mgr.y (mgr.14146) 46 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:45:58.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:58 smithi006 bash[33918]: audit 2023-08-27T21:45:57.365624+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:45:58.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:58 smithi006 bash[33918]: cephadm 2023-08-27T21:45:57.366551+0000 mgr.y (mgr.14146) 47 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-08-27T21:45:58.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:58 smithi006 bash[33918]: audit 2023-08-27T21:45:57.366962+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T21:45:58.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:58 smithi006 bash[33918]: audit 2023-08-27T21:45:57.368104+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-27T21:45:58.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:58 smithi006 bash[33918]: audit 2023-08-27T21:45:57.369303+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:45:58 smithi006 bash[33918]: cephadm 2023-08-27T21:45:57.370419+0000 mgr.y (mgr.14146) 48 : cephadm [INF] Reconfiguring daemon mon.b on smithi130 2023-08-27T21:46:00.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:00 smithi130 bash[27579]: cluster 2023-08-27T21:45:59.135712+0000 mgr.y (mgr.14146) 49 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:00.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:00 smithi130 bash[27579]: audit 2023-08-27T21:46:00.141469+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:00.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:00 smithi130 bash[27579]: audit 2023-08-27T21:46:00.143149+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:46:00.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:00 smithi130 bash[27579]: audit 2023-08-27T21:46:00.145608+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:00.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:00 smithi130 bash[27579]: audit 2023-08-27T21:46:00.146736+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:46:00.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:00 smithi130 bash[27579]: audit 2023-08-27T21:46:00.152229+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:00 smithi006 bash[27292]: cluster 2023-08-27T21:45:59.135712+0000 mgr.y (mgr.14146) 49 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:00 smithi006 bash[27292]: audit 2023-08-27T21:46:00.141469+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:00.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:00 smithi006 bash[27292]: audit 2023-08-27T21:46:00.143149+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:46:00.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:00 smithi006 bash[27292]: audit 2023-08-27T21:46:00.145608+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:00.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:00 smithi006 bash[27292]: audit 2023-08-27T21:46:00.146736+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:46:00.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:00 smithi006 bash[27292]: audit 2023-08-27T21:46:00.152229+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:00.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:00 smithi006 bash[33918]: cluster 2023-08-27T21:45:59.135712+0000 mgr.y (mgr.14146) 49 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:00.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:00 smithi006 bash[33918]: audit 2023-08-27T21:46:00.141469+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:00.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:00 smithi006 bash[33918]: audit 2023-08-27T21:46:00.143149+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:46:00.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:00 smithi006 bash[33918]: audit 2023-08-27T21:46:00.145608+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:00.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:00 smithi006 bash[33918]: audit 2023-08-27T21:46:00.146736+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:46:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:00 smithi006 bash[33918]: audit 2023-08-27T21:46:00.152229+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:01.890 INFO:teuthology.orchestra.run.smithi130.stdout:Scheduled mgr update... 2023-08-27T21:46:02.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:02 smithi130 bash[27579]: cluster 2023-08-27T21:46:01.136162+0000 mgr.y (mgr.14146) 50 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:02 smithi130 bash[27579]: audit 2023-08-27T21:46:01.889377+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:02 smithi130 bash[27579]: audit 2023-08-27T21:46:01.923660+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:46:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:02 smithi130 bash[27579]: audit 2023-08-27T21:46:01.927429+0000 mon.a (mon.0) 204 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:02 smithi130 bash[27579]: audit 2023-08-27T21:46:01.929163+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:46:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:02 smithi130 bash[27579]: audit 2023-08-27T21:46:01.936755+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:02 smithi130 bash[27579]: audit 2023-08-27T21:46:01.939169+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-27T21:46:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:02 smithi130 bash[27579]: audit 2023-08-27T21:46:01.943159+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-08-27T21:46:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:02 smithi130 bash[27579]: audit 2023-08-27T21:46:01.944462+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-27T21:46:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:02 smithi130 bash[27579]: audit 2023-08-27T21:46:01.946054+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:02.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:02 smithi006 bash[33918]: cluster 2023-08-27T21:46:01.136162+0000 mgr.y (mgr.14146) 50 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:02.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:02 smithi006 bash[33918]: audit 2023-08-27T21:46:01.889377+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:02.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:02 smithi006 bash[33918]: audit 2023-08-27T21:46:01.923660+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:46:02.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:02 smithi006 bash[33918]: audit 2023-08-27T21:46:01.927429+0000 mon.a (mon.0) 204 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:02.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:02 smithi006 bash[33918]: audit 2023-08-27T21:46:01.929163+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:46:02.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:02 smithi006 bash[33918]: audit 2023-08-27T21:46:01.936755+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:02.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:02 smithi006 bash[33918]: audit 2023-08-27T21:46:01.939169+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-27T21:46:02.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:02 smithi006 bash[33918]: audit 2023-08-27T21:46:01.943159+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-08-27T21:46:02.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:02 smithi006 bash[33918]: audit 2023-08-27T21:46:01.944462+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-27T21:46:02.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:02 smithi006 bash[33918]: audit 2023-08-27T21:46:01.946054+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:02.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:02 smithi006 bash[27292]: cluster 2023-08-27T21:46:01.136162+0000 mgr.y (mgr.14146) 50 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:02.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:02 smithi006 bash[27292]: audit 2023-08-27T21:46:01.889377+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:02 smithi006 bash[27292]: audit 2023-08-27T21:46:01.923660+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:46:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:02 smithi006 bash[27292]: audit 2023-08-27T21:46:01.927429+0000 mon.a (mon.0) 204 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:02 smithi006 bash[27292]: audit 2023-08-27T21:46:01.929163+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:46:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:02 smithi006 bash[27292]: audit 2023-08-27T21:46:01.936755+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:02 smithi006 bash[27292]: audit 2023-08-27T21:46:01.939169+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-27T21:46:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:02 smithi006 bash[27292]: audit 2023-08-27T21:46:01.943159+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-08-27T21:46:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:02 smithi006 bash[27292]: audit 2023-08-27T21:46:01.944462+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-27T21:46:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:02 smithi006 bash[27292]: audit 2023-08-27T21:46:01.946054+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:02.988 DEBUG:teuthology.orchestra.run.smithi130:mgr.x> sudo journalctl -f -n 0 -u ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@mgr.x.service 2023-08-27T21:46:02.990 INFO:tasks.cephadm:Deploying OSDs... 2023-08-27T21:46:02.990 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2023-08-27T21:46:02.990 DEBUG:teuthology.orchestra.run.smithi006:> dd if=/scratch_devs of=/dev/stdout 2023-08-27T21:46:02.998 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-08-27T21:46:02.998 DEBUG:teuthology.orchestra.run.smithi006:> stat /dev/vg_nvme/lv_1 2023-08-27T21:46:03.051 INFO:teuthology.orchestra.run.smithi006.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-08-27T21:46:03.051 INFO:teuthology.orchestra.run.smithi006.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-27T21:46:03.052 INFO:teuthology.orchestra.run.smithi006.stdout:Device: 5h/5d Inode: 492 Links: 1 2023-08-27T21:46:03.052 INFO:teuthology.orchestra.run.smithi006.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-27T21:46:03.052 INFO:teuthology.orchestra.run.smithi006.stdout:Access: 2023-08-27 21:39:37.505819906 +0000 2023-08-27T21:46:03.052 INFO:teuthology.orchestra.run.smithi006.stdout:Modify: 2023-08-27 21:39:37.181827379 +0000 2023-08-27T21:46:03.052 INFO:teuthology.orchestra.run.smithi006.stdout:Change: 2023-08-27 21:39:37.181827379 +0000 2023-08-27T21:46:03.052 INFO:teuthology.orchestra.run.smithi006.stdout: Birth: - 2023-08-27T21:46:03.052 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-08-27T21:46:03.107 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records in 2023-08-27T21:46:03.107 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records out 2023-08-27T21:46:03.107 INFO:teuthology.orchestra.run.smithi006.stderr:512 bytes copied, 0.000403153 s, 1.3 MB/s 2023-08-27T21:46:03.108 DEBUG:teuthology.orchestra.run.smithi006:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-08-27T21:46:03.161 DEBUG:teuthology.orchestra.run.smithi006:> stat /dev/vg_nvme/lv_2 2023-08-27T21:46:03.211 INFO:teuthology.orchestra.run.smithi006.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-08-27T21:46:03.211 INFO:teuthology.orchestra.run.smithi006.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-27T21:46:03.211 INFO:teuthology.orchestra.run.smithi006.stdout:Device: 5h/5d Inode: 498 Links: 1 2023-08-27T21:46:03.212 INFO:teuthology.orchestra.run.smithi006.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-27T21:46:03.212 INFO:teuthology.orchestra.run.smithi006.stdout:Access: 2023-08-27 21:39:38.033807731 +0000 2023-08-27T21:46:03.212 INFO:teuthology.orchestra.run.smithi006.stdout:Modify: 2023-08-27 21:39:37.717815017 +0000 2023-08-27T21:46:03.212 INFO:teuthology.orchestra.run.smithi006.stdout:Change: 2023-08-27 21:39:37.717815017 +0000 2023-08-27T21:46:03.212 INFO:teuthology.orchestra.run.smithi006.stdout: Birth: - 2023-08-27T21:46:03.212 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-08-27T21:46:03.265 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records in 2023-08-27T21:46:03.265 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records out 2023-08-27T21:46:03.265 INFO:teuthology.orchestra.run.smithi006.stderr:512 bytes copied, 0.000413191 s, 1.2 MB/s 2023-08-27T21:46:03.266 DEBUG:teuthology.orchestra.run.smithi006:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-08-27T21:46:03.321 DEBUG:teuthology.orchestra.run.smithi006:> stat /dev/vg_nvme/lv_3 2023-08-27T21:46:03.371 INFO:teuthology.orchestra.run.smithi006.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-08-27T21:46:03.371 INFO:teuthology.orchestra.run.smithi006.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-27T21:46:03.371 INFO:teuthology.orchestra.run.smithi006.stdout:Device: 5h/5d Inode: 504 Links: 1 2023-08-27T21:46:03.371 INFO:teuthology.orchestra.run.smithi006.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-27T21:46:03.371 INFO:teuthology.orchestra.run.smithi006.stdout:Access: 2023-08-27 21:39:38.269802289 +0000 2023-08-27T21:46:03.372 INFO:teuthology.orchestra.run.smithi006.stdout:Modify: 2023-08-27 21:39:38.265802380 +0000 2023-08-27T21:46:03.372 INFO:teuthology.orchestra.run.smithi006.stdout:Change: 2023-08-27 21:39:38.265802380 +0000 2023-08-27T21:46:03.372 INFO:teuthology.orchestra.run.smithi006.stdout: Birth: - 2023-08-27T21:46:03.372 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-08-27T21:46:03.427 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records in 2023-08-27T21:46:03.427 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records out 2023-08-27T21:46:03.427 INFO:teuthology.orchestra.run.smithi006.stderr:512 bytes copied, 0.000420487 s, 1.2 MB/s 2023-08-27T21:46:03.428 DEBUG:teuthology.orchestra.run.smithi006:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-08-27T21:46:03.481 DEBUG:teuthology.orchestra.run.smithi006:> stat /dev/vg_nvme/lv_4 2023-08-27T21:46:03.496 INFO:journalctl@ceph.mgr.x.smithi130.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-08-27T21:46:03.531 INFO:teuthology.orchestra.run.smithi006.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-08-27T21:46:03.531 INFO:teuthology.orchestra.run.smithi006.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-27T21:46:03.531 INFO:teuthology.orchestra.run.smithi006.stdout:Device: 5h/5d Inode: 508 Links: 1 2023-08-27T21:46:03.531 INFO:teuthology.orchestra.run.smithi006.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-27T21:46:03.531 INFO:teuthology.orchestra.run.smithi006.stdout:Access: 2023-08-27 21:39:39.133782363 +0000 2023-08-27T21:46:03.531 INFO:teuthology.orchestra.run.smithi006.stdout:Modify: 2023-08-27 21:39:38.817789650 +0000 2023-08-27T21:46:03.531 INFO:teuthology.orchestra.run.smithi006.stdout:Change: 2023-08-27 21:39:38.817789650 +0000 2023-08-27T21:46:03.531 INFO:teuthology.orchestra.run.smithi006.stdout: Birth: - 2023-08-27T21:46:03.532 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-08-27T21:46:03.585 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records in 2023-08-27T21:46:03.585 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records out 2023-08-27T21:46:03.585 INFO:teuthology.orchestra.run.smithi006.stderr:512 bytes copied, 0.000421491 s, 1.2 MB/s 2023-08-27T21:46:03.587 DEBUG:teuthology.orchestra.run.smithi006:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-08-27T21:46:03.637 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-08-27T21:46:03.637 DEBUG:teuthology.orchestra.run.smithi130:> dd if=/scratch_devs of=/dev/stdout 2023-08-27T21:46:03.645 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-08-27T21:46:03.645 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_1 2023-08-27T21:46:03.678 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:03 smithi130 bash[27579]: audit 2023-08-27T21:46:01.879414+0000 mgr.y (mgr.14146) 51 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi006=y;smithi130=x", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:46:03.678 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:03 smithi130 bash[27579]: cephadm 2023-08-27T21:46:01.882903+0000 mgr.y (mgr.14146) 52 : cephadm [INF] Saving service mgr spec with placement smithi006=y;smithi130=x;count:2 2023-08-27T21:46:03.678 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:03 smithi130 bash[27579]: cephadm 2023-08-27T21:46:01.947245+0000 mgr.y (mgr.14146) 53 : cephadm [INF] Deploying daemon mgr.x on smithi130 2023-08-27T21:46:03.684 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-08-27T21:46:03.685 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-27T21:46:03.686 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 5h/5d Inode: 489 Links: 1 2023-08-27T21:46:03.686 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-27T21:46:03.686 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2023-08-27 21:39:39.054294170 +0000 2023-08-27T21:46:03.686 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2023-08-27 21:39:38.730301642 +0000 2023-08-27T21:46:03.686 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2023-08-27 21:39:38.730301642 +0000 2023-08-27T21:46:03.686 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: - 2023-08-27T21:46:03.686 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-08-27T21:46:03.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:03 smithi006 bash[27292]: audit 2023-08-27T21:46:01.879414+0000 mgr.y (mgr.14146) 51 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi006=y;smithi130=x", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:46:03.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:03 smithi006 bash[27292]: cephadm 2023-08-27T21:46:01.882903+0000 mgr.y (mgr.14146) 52 : cephadm [INF] Saving service mgr spec with placement smithi006=y;smithi130=x;count:2 2023-08-27T21:46:03.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:03 smithi006 bash[27292]: cephadm 2023-08-27T21:46:01.947245+0000 mgr.y (mgr.14146) 53 : cephadm [INF] Deploying daemon mgr.x on smithi130 2023-08-27T21:46:03.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:03 smithi006 bash[33918]: audit 2023-08-27T21:46:01.879414+0000 mgr.y (mgr.14146) 51 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi006=y;smithi130=x", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:46:03.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:03 smithi006 bash[33918]: cephadm 2023-08-27T21:46:01.882903+0000 mgr.y (mgr.14146) 52 : cephadm [INF] Saving service mgr spec with placement smithi006=y;smithi130=x;count:2 2023-08-27T21:46:03.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:03 smithi006 bash[33918]: cephadm 2023-08-27T21:46:01.947245+0000 mgr.y (mgr.14146) 53 : cephadm [INF] Deploying daemon mgr.x on smithi130 2023-08-27T21:46:03.740 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2023-08-27T21:46:03.740 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2023-08-27T21:46:03.740 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.000370336 s, 1.4 MB/s 2023-08-27T21:46:03.741 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-08-27T21:46:03.791 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_2 2023-08-27T21:46:03.842 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-08-27T21:46:03.842 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-27T21:46:03.842 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 5h/5d Inode: 497 Links: 1 2023-08-27T21:46:03.842 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-27T21:46:03.842 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2023-08-27 21:39:39.258289465 +0000 2023-08-27T21:46:03.842 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2023-08-27 21:39:39.254289558 +0000 2023-08-27T21:46:03.843 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2023-08-27 21:39:39.254289558 +0000 2023-08-27T21:46:03.843 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: - 2023-08-27T21:46:03.843 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-08-27T21:46:03.897 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2023-08-27T21:46:03.897 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2023-08-27T21:46:03.898 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.000408241 s, 1.3 MB/s 2023-08-27T21:46:03.899 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-08-27T21:46:03.951 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_3 2023-08-27T21:46:04.001 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-08-27T21:46:04.001 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-27T21:46:04.001 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 5h/5d Inode: 501 Links: 1 2023-08-27T21:46:04.001 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-27T21:46:04.001 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2023-08-27 21:39:40.106269910 +0000 2023-08-27T21:46:04.001 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2023-08-27 21:39:39.810276736 +0000 2023-08-27T21:46:04.001 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2023-08-27 21:39:39.810276736 +0000 2023-08-27T21:46:04.002 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: - 2023-08-27T21:46:04.002 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-08-27T21:46:04.055 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2023-08-27T21:46:04.055 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2023-08-27T21:46:04.055 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.00040475 s, 1.3 MB/s 2023-08-27T21:46:04.056 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-08-27T21:46:04.108 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_4 2023-08-27T21:46:04.158 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-08-27T21:46:04.158 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-27T21:46:04.158 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 5h/5d Inode: 508 Links: 1 2023-08-27T21:46:04.158 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-27T21:46:04.158 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2023-08-27 21:39:40.642257549 +0000 2023-08-27T21:46:04.158 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2023-08-27 21:39:40.318265021 +0000 2023-08-27T21:46:04.158 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2023-08-27 21:39:40.318265021 +0000 2023-08-27T21:46:04.158 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: - 2023-08-27T21:46:04.158 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-08-27T21:46:04.211 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2023-08-27T21:46:04.211 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2023-08-27T21:46:04.212 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.000428691 s, 1.2 MB/s 2023-08-27T21:46:04.213 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-08-27T21:46:04.263 INFO:tasks.cephadm:Deploying osd.0 on smithi006 with /dev/vg_nvme/lv_4... 2023-08-27T21:46:04.263 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-08-27T21:46:04.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:04 smithi130 bash[27579]: cluster 2023-08-27T21:46:03.136609+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:04.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:04 smithi006 bash[27292]: cluster 2023-08-27T21:46:03.136609+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:04.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:04 smithi006 bash[33918]: cluster 2023-08-27T21:46:03.136609+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:06.025 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:05 smithi130 bash[27579]: audit 2023-08-27T21:46:04.807591+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:06.025 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:05 smithi130 bash[27579]: audit 2023-08-27T21:46:04.810163+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:46:06.025 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:05 smithi130 bash[27579]: cluster 2023-08-27T21:46:05.137116+0000 mgr.y (mgr.14146) 55 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:06.025 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:05 smithi130 bash[28467]: debug 2023-08-27T21:46:05.857+0000 7f531ff7e700 1 -- 172.21.15.130:0/3549578183 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562cf1c0a4e0 con 0x562cf1cfe800 2023-08-27T21:46:06.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:05 smithi006 bash[27292]: audit 2023-08-27T21:46:04.807591+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:06.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:05 smithi006 bash[27292]: audit 2023-08-27T21:46:04.810163+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:46:06.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:05 smithi006 bash[27292]: cluster 2023-08-27T21:46:05.137116+0000 mgr.y (mgr.14146) 55 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:06.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:05 smithi006 bash[33918]: audit 2023-08-27T21:46:04.807591+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:06.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:05 smithi006 bash[33918]: audit 2023-08-27T21:46:04.810163+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:46:06.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:05 smithi006 bash[33918]: cluster 2023-08-27T21:46:05.137116+0000 mgr.y (mgr.14146) 55 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:06.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:06 smithi130 bash[28467]: debug 2023-08-27T21:46:06.021+0000 7f5330565680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-27T21:46:06.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:06 smithi130 bash[28467]: debug 2023-08-27T21:46:06.125+0000 7f5330565680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-27T21:46:06.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:06 smithi130 bash[28467]: debug 2023-08-27T21:46:06.589+0000 7f5330565680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-27T21:46:07.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:06 smithi006 bash[27292]: audit 2023-08-27T21:46:05.867723+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:07.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:06 smithi006 bash[33918]: audit 2023-08-27T21:46:05.867723+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:07.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:06 smithi130 bash[27579]: audit 2023-08-27T21:46:05.867723+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:07.862 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:07 smithi130 bash[28467]: debug 2023-08-27T21:46:07.501+0000 7f5330565680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-27T21:46:07.862 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:07 smithi130 bash[28467]: debug 2023-08-27T21:46:07.605+0000 7f5330565680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-27T21:46:08.013 INFO:teuthology.orchestra.run.smithi006.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-08-27T21:46:08.014 INFO:teuthology.orchestra.run.smithi006.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-08-27T21:46:08.014 INFO:teuthology.orchestra.run.smithi006.stderr: stderr: 10+0 records in 2023-08-27T21:46:08.014 INFO:teuthology.orchestra.run.smithi006.stderr:10+0 records out 2023-08-27T21:46:08.014 INFO:teuthology.orchestra.run.smithi006.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0145134 s, 722 MB/s 2023-08-27T21:46:08.014 INFO:teuthology.orchestra.run.smithi006.stderr:--> Zapping successful for: 2023-08-27T21:46:08.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:07 smithi130 bash[28467]: debug 2023-08-27T21:46:07.857+0000 7f5330565680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-27T21:46:08.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:08 smithi006 bash[27292]: cluster 2023-08-27T21:46:07.137519+0000 mgr.y (mgr.14146) 56 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:08.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:08 smithi006 bash[33918]: cluster 2023-08-27T21:46:07.137519+0000 mgr.y (mgr.14146) 56 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:08.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:08 smithi130 bash[27579]: cluster 2023-08-27T21:46:07.137519+0000 mgr.y (mgr.14146) 56 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:08.763 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph orch daemon add osd smithi006:vg_nvme/lv_4 2023-08-27T21:46:09.267 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:09 smithi130 bash[28467]: debug 2023-08-27T21:46:09.005+0000 7f5330565680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-27T21:46:09.267 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:09 smithi130 bash[28467]: debug 2023-08-27T21:46:09.165+0000 7f5330565680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-27T21:46:09.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:09 smithi130 bash[28467]: debug 2023-08-27T21:46:09.261+0000 7f5330565680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-27T21:46:09.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:09 smithi130 bash[28467]: debug 2023-08-27T21:46:09.377+0000 7f5330565680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-27T21:46:10.011 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:09 smithi130 bash[28467]: debug 2023-08-27T21:46:09.625+0000 7f5330565680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-27T21:46:10.325 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:10 smithi130 bash[28467]: debug 2023-08-27T21:46:10.009+0000 7f5330565680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-27T21:46:10.325 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:10 smithi130 bash[28467]: debug 2023-08-27T21:46:10.141+0000 7f5330565680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-27T21:46:10.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:10 smithi006 bash[27292]: cluster 2023-08-27T21:46:09.137965+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:10.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:10 smithi006 bash[27292]: audit 2023-08-27T21:46:09.856273+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:10.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:10 smithi006 bash[27292]: audit 2023-08-27T21:46:09.857505+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:10.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:10 smithi006 bash[27292]: audit 2023-08-27T21:46:09.858701+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:46:10.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:10 smithi006 bash[27292]: audit 2023-08-27T21:46:09.864297+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:10.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:10 smithi006 bash[27292]: audit 2023-08-27T21:46:09.867273+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-27T21:46:10.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:10 smithi006 bash[27292]: audit 2023-08-27T21:46:09.868555+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-27T21:46:10.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:10 smithi006 bash[27292]: audit 2023-08-27T21:46:09.869624+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:10.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:10 smithi006 bash[33918]: cluster 2023-08-27T21:46:09.137965+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:10.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:10 smithi006 bash[33918]: audit 2023-08-27T21:46:09.856273+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:10.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:10 smithi006 bash[33918]: audit 2023-08-27T21:46:09.857505+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:10.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:10 smithi006 bash[33918]: audit 2023-08-27T21:46:09.858701+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:46:10.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:10 smithi006 bash[33918]: audit 2023-08-27T21:46:09.864297+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:10.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:10 smithi006 bash[33918]: audit 2023-08-27T21:46:09.867273+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-27T21:46:10.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:10 smithi006 bash[33918]: audit 2023-08-27T21:46:09.868555+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-27T21:46:10.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:10 smithi006 bash[33918]: audit 2023-08-27T21:46:09.869624+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:10 smithi130 bash[27579]: cluster 2023-08-27T21:46:09.137965+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:10 smithi130 bash[27579]: audit 2023-08-27T21:46:09.856273+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:10 smithi130 bash[27579]: audit 2023-08-27T21:46:09.857505+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:10 smithi130 bash[27579]: audit 2023-08-27T21:46:09.858701+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:46:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:10 smithi130 bash[27579]: audit 2023-08-27T21:46:09.864297+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:10.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:10 smithi130 bash[27579]: audit 2023-08-27T21:46:09.867273+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-27T21:46:10.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:10 smithi130 bash[27579]: audit 2023-08-27T21:46:09.868555+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-27T21:46:10.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:10 smithi130 bash[27579]: audit 2023-08-27T21:46:09.869624+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:11.327 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:11 smithi130 bash[28467]: debug 2023-08-27T21:46:11.037+0000 7f5330565680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-27T21:46:11.327 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:11 smithi130 bash[28467]: debug 2023-08-27T21:46:11.141+0000 7f5330565680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-27T21:46:11.327 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:11 smithi130 bash[28467]: debug 2023-08-27T21:46:11.265+0000 7f5330565680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-27T21:46:11.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:11 smithi130 bash[28467]: debug 2023-08-27T21:46:11.493+0000 7f5330565680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-27T21:46:11.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:11 smithi130 bash[27579]: cephadm 2023-08-27T21:46:09.866893+0000 mgr.y (mgr.14146) 58 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-08-27T21:46:11.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:11 smithi130 bash[27579]: cephadm 2023-08-27T21:46:09.870543+0000 mgr.y (mgr.14146) 59 : cephadm [INF] Reconfiguring daemon mgr.y on smithi006 2023-08-27T21:46:11.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:11 smithi006 bash[33918]: cephadm 2023-08-27T21:46:09.866893+0000 mgr.y (mgr.14146) 58 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-08-27T21:46:11.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:11 smithi006 bash[33918]: cephadm 2023-08-27T21:46:09.870543+0000 mgr.y (mgr.14146) 59 : cephadm [INF] Reconfiguring daemon mgr.y on smithi006 2023-08-27T21:46:11.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:11 smithi006 bash[27292]: cephadm 2023-08-27T21:46:09.866893+0000 mgr.y (mgr.14146) 58 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-08-27T21:46:11.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:11 smithi006 bash[27292]: cephadm 2023-08-27T21:46:09.870543+0000 mgr.y (mgr.14146) 59 : cephadm [INF] Reconfiguring daemon mgr.y on smithi006 2023-08-27T21:46:12.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:11 smithi130 bash[28467]: debug 2023-08-27T21:46:11.781+0000 7f5330565680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-27T21:46:12.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:11 smithi130 bash[28467]: debug 2023-08-27T21:46:11.945+0000 7f5330565680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-27T21:46:12.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:12 smithi130 bash[27579]: cluster 2023-08-27T21:46:11.138392+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:12 smithi130 bash[28467]: debug 2023-08-27T21:46:12.113+0000 7f5330565680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-27T21:46:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:12 smithi130 bash[28467]: debug 2023-08-27T21:46:12.325+0000 7f5330565680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-27T21:46:12.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:12 smithi006 bash[27292]: cluster 2023-08-27T21:46:11.138392+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:12.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:12 smithi006 bash[33918]: cluster 2023-08-27T21:46:11.138392+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:12.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:46:12 smithi130 bash[28467]: debug 2023-08-27T21:46:12.429+0000 7f5330565680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-27T21:46:13.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:13 smithi006 bash[33918]: cluster 2023-08-27T21:46:12.459177+0000 mon.a (mon.0) 221 : cluster [DBG] Standby manager daemon x started 2023-08-27T21:46:13.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:13 smithi006 bash[33918]: audit 2023-08-27T21:46:12.461106+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.130:0/666830340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-27T21:46:13.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:13 smithi006 bash[33918]: audit 2023-08-27T21:46:12.461932+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.130:0/666830340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T21:46:13.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:13 smithi006 bash[33918]: audit 2023-08-27T21:46:12.464008+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.130:0/666830340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-27T21:46:13.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:13 smithi006 bash[33918]: audit 2023-08-27T21:46:12.464557+0000 mon.b (mon.1) 7 : audit [DBG] from='mgr.? 172.21.15.130:0/666830340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T21:46:13.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:13 smithi006 bash[33918]: audit 2023-08-27T21:46:13.090877+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:13.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:13 smithi006 bash[33918]: audit 2023-08-27T21:46:13.092310+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:46:13.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:13 smithi006 bash[27292]: cluster 2023-08-27T21:46:12.459177+0000 mon.a (mon.0) 221 : cluster [DBG] Standby manager daemon x started 2023-08-27T21:46:13.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:13 smithi006 bash[27292]: audit 2023-08-27T21:46:12.461106+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.130:0/666830340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-27T21:46:13.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:13 smithi006 bash[27292]: audit 2023-08-27T21:46:12.461932+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.130:0/666830340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T21:46:13.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:13 smithi006 bash[27292]: audit 2023-08-27T21:46:12.464008+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.130:0/666830340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-27T21:46:13.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:13 smithi006 bash[27292]: audit 2023-08-27T21:46:12.464557+0000 mon.b (mon.1) 7 : audit [DBG] from='mgr.? 172.21.15.130:0/666830340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T21:46:13.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:13 smithi006 bash[27292]: audit 2023-08-27T21:46:13.090877+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:13.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:13 smithi006 bash[27292]: audit 2023-08-27T21:46:13.092310+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:46:13.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:13 smithi130 bash[27579]: cluster 2023-08-27T21:46:12.459177+0000 mon.a (mon.0) 221 : cluster [DBG] Standby manager daemon x started 2023-08-27T21:46:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:13 smithi130 bash[27579]: audit 2023-08-27T21:46:12.461106+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.130:0/666830340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-27T21:46:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:13 smithi130 bash[27579]: audit 2023-08-27T21:46:12.461932+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.130:0/666830340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T21:46:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:13 smithi130 bash[27579]: audit 2023-08-27T21:46:12.464008+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.130:0/666830340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-27T21:46:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:13 smithi130 bash[27579]: audit 2023-08-27T21:46:12.464557+0000 mon.b (mon.1) 7 : audit [DBG] from='mgr.? 172.21.15.130:0/666830340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T21:46:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:13 smithi130 bash[27579]: audit 2023-08-27T21:46:13.090877+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:13 smithi130 bash[27579]: audit 2023-08-27T21:46:13.092310+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:46:14.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:14 smithi130 bash[27579]: cluster 2023-08-27T21:46:13.138813+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:14 smithi130 bash[27579]: cluster 2023-08-27T21:46:13.347299+0000 mon.a (mon.0) 224 : cluster [DBG] mgrmap e14: y(active, since 82s), standbys: x 2023-08-27T21:46:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:14 smithi130 bash[27579]: audit 2023-08-27T21:46:13.347781+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-27T21:46:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:14 smithi130 bash[27579]: audit 2023-08-27T21:46:13.376902+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:14 smithi130 bash[27579]: audit 2023-08-27T21:46:13.378103+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:46:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:14 smithi130 bash[27579]: audit 2023-08-27T21:46:13.387004+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:14 smithi130 bash[27579]: audit 2023-08-27T21:46:13.559712+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:46:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:14 smithi130 bash[27579]: audit 2023-08-27T21:46:13.563111+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:46:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:14 smithi130 bash[27579]: audit 2023-08-27T21:46:13.564526+0000 mon.a (mon.0) 231 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:14.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:14 smithi006 bash[27292]: cluster 2023-08-27T21:46:13.138813+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:14.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:14 smithi006 bash[27292]: cluster 2023-08-27T21:46:13.347299+0000 mon.a (mon.0) 224 : cluster [DBG] mgrmap e14: y(active, since 82s), standbys: x 2023-08-27T21:46:14.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:14 smithi006 bash[27292]: audit 2023-08-27T21:46:13.347781+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-27T21:46:14.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:14 smithi006 bash[27292]: audit 2023-08-27T21:46:13.376902+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:14.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:14 smithi006 bash[27292]: audit 2023-08-27T21:46:13.378103+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:46:14.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:14 smithi006 bash[27292]: audit 2023-08-27T21:46:13.387004+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:14.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:14 smithi006 bash[27292]: audit 2023-08-27T21:46:13.559712+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:46:14.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:14 smithi006 bash[27292]: audit 2023-08-27T21:46:13.563111+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:46:14.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:14 smithi006 bash[27292]: audit 2023-08-27T21:46:13.564526+0000 mon.a (mon.0) 231 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:14.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:14 smithi006 bash[33918]: cluster 2023-08-27T21:46:13.138813+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:14.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:14 smithi006 bash[33918]: cluster 2023-08-27T21:46:13.347299+0000 mon.a (mon.0) 224 : cluster [DBG] mgrmap e14: y(active, since 82s), standbys: x 2023-08-27T21:46:14.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:14 smithi006 bash[33918]: audit 2023-08-27T21:46:13.347781+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-27T21:46:14.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:14 smithi006 bash[33918]: audit 2023-08-27T21:46:13.376902+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:14.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:14 smithi006 bash[33918]: audit 2023-08-27T21:46:13.378103+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:46:14.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:14 smithi006 bash[33918]: audit 2023-08-27T21:46:13.387004+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:14.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:14 smithi006 bash[33918]: audit 2023-08-27T21:46:13.559712+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:46:14.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:14 smithi006 bash[33918]: audit 2023-08-27T21:46:13.563111+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:46:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:14 smithi006 bash[33918]: audit 2023-08-27T21:46:13.564526+0000 mon.a (mon.0) 231 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:15.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:15 smithi130 bash[27579]: audit 2023-08-27T21:46:13.552345+0000 mgr.y (mgr.14146) 62 : audit [DBG] from='client.14199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:46:15.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:15 smithi006 bash[27292]: audit 2023-08-27T21:46:13.552345+0000 mgr.y (mgr.14146) 62 : audit [DBG] from='client.14199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:46:15.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:15 smithi006 bash[33918]: audit 2023-08-27T21:46:13.552345+0000 mgr.y (mgr.14146) 62 : audit [DBG] from='client.14199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:46:16.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:16 smithi130 bash[27579]: cluster 2023-08-27T21:46:15.139320+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:16 smithi006 bash[27292]: cluster 2023-08-27T21:46:15.139320+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:16 smithi006 bash[33918]: cluster 2023-08-27T21:46:15.139320+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:18.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:18 smithi130 bash[27579]: cluster 2023-08-27T21:46:17.139692+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:18.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:18 smithi006 bash[33918]: cluster 2023-08-27T21:46:17.139692+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:18.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:18 smithi006 bash[27292]: cluster 2023-08-27T21:46:17.139692+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:20.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:20 smithi006 bash[27292]: cluster 2023-08-27T21:46:19.140202+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:20.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:20 smithi006 bash[33918]: cluster 2023-08-27T21:46:19.140202+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:20.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:20 smithi130 bash[27579]: cluster 2023-08-27T21:46:19.140202+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:22.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:22 smithi006 bash[27292]: cluster 2023-08-27T21:46:21.140658+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:22.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:22 smithi006 bash[27292]: audit 2023-08-27T21:46:21.992156+0000 mon.a (mon.0) 232 : audit [INF] from='client.? 172.21.15.6:0/1168997108' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "226c7900-24be-48c6-98fa-97893e22d328"}]: dispatch 2023-08-27T21:46:22.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:22 smithi006 bash[27292]: audit 2023-08-27T21:46:22.001165+0000 mon.a (mon.0) 233 : audit [INF] from='client.? 172.21.15.6:0/1168997108' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "226c7900-24be-48c6-98fa-97893e22d328"}]': finished 2023-08-27T21:46:22.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:22 smithi006 bash[27292]: cluster 2023-08-27T21:46:22.001274+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-08-27T21:46:22.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:22 smithi006 bash[27292]: audit 2023-08-27T21:46:22.001457+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:46:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:22 smithi006 bash[33918]: cluster 2023-08-27T21:46:21.140658+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:22 smithi006 bash[33918]: audit 2023-08-27T21:46:21.992156+0000 mon.a (mon.0) 232 : audit [INF] from='client.? 172.21.15.6:0/1168997108' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "226c7900-24be-48c6-98fa-97893e22d328"}]: dispatch 2023-08-27T21:46:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:22 smithi006 bash[33918]: audit 2023-08-27T21:46:22.001165+0000 mon.a (mon.0) 233 : audit [INF] from='client.? 172.21.15.6:0/1168997108' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "226c7900-24be-48c6-98fa-97893e22d328"}]': finished 2023-08-27T21:46:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:22 smithi006 bash[33918]: cluster 2023-08-27T21:46:22.001274+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-08-27T21:46:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:22 smithi006 bash[33918]: audit 2023-08-27T21:46:22.001457+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:46:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:22 smithi130 bash[27579]: cluster 2023-08-27T21:46:21.140658+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:22 smithi130 bash[27579]: audit 2023-08-27T21:46:21.992156+0000 mon.a (mon.0) 232 : audit [INF] from='client.? 172.21.15.6:0/1168997108' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "226c7900-24be-48c6-98fa-97893e22d328"}]: dispatch 2023-08-27T21:46:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:22 smithi130 bash[27579]: audit 2023-08-27T21:46:22.001165+0000 mon.a (mon.0) 233 : audit [INF] from='client.? 172.21.15.6:0/1168997108' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "226c7900-24be-48c6-98fa-97893e22d328"}]': finished 2023-08-27T21:46:22.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:22 smithi130 bash[27579]: cluster 2023-08-27T21:46:22.001274+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-08-27T21:46:22.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:22 smithi130 bash[27579]: audit 2023-08-27T21:46:22.001457+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:46:23.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:23 smithi006 bash[27292]: audit 2023-08-27T21:46:22.737641+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.6:0/1668823560' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:46:23.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:23 smithi006 bash[33918]: audit 2023-08-27T21:46:22.737641+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.6:0/1668823560' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:46:23.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:23 smithi130 bash[27579]: audit 2023-08-27T21:46:22.737641+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.6:0/1668823560' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:46:24.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:24 smithi006 bash[27292]: cluster 2023-08-27T21:46:23.141169+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:24.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:24 smithi006 bash[33918]: cluster 2023-08-27T21:46:23.141169+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:24 smithi130 bash[27579]: cluster 2023-08-27T21:46:23.141169+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:25.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:25 smithi006 bash[27292]: cluster 2023-08-27T21:46:25.141671+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:25.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:25 smithi006 bash[33918]: cluster 2023-08-27T21:46:25.141671+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:25.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:25 smithi130 bash[27579]: cluster 2023-08-27T21:46:25.141671+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:28.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:28 smithi130 bash[27579]: cluster 2023-08-27T21:46:27.142204+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:28.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:28 smithi006 bash[27292]: cluster 2023-08-27T21:46:27.142204+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:28 smithi006 bash[33918]: cluster 2023-08-27T21:46:27.142204+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:30.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:30 smithi130 bash[27579]: cluster 2023-08-27T21:46:29.142670+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:30.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:30 smithi006 bash[27292]: cluster 2023-08-27T21:46:29.142670+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:30 smithi006 bash[33918]: cluster 2023-08-27T21:46:29.142670+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:32.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:32 smithi006 bash[33918]: cluster 2023-08-27T21:46:31.143141+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:32.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:32 smithi006 bash[27292]: cluster 2023-08-27T21:46:31.143141+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:32.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:32 smithi130 bash[27579]: cluster 2023-08-27T21:46:31.143141+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:34.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:34 smithi130 bash[27579]: cluster 2023-08-27T21:46:33.143663+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:34.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:34 smithi006 bash[33918]: cluster 2023-08-27T21:46:33.143663+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:34.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:34 smithi006 bash[27292]: cluster 2023-08-27T21:46:33.143663+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:35.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:35 smithi006 bash[27292]: audit 2023-08-27T21:46:34.878348+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-08-27T21:46:35.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:35 smithi006 bash[27292]: audit 2023-08-27T21:46:34.879716+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:35.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:35 smithi006 bash[33918]: audit 2023-08-27T21:46:34.878348+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-08-27T21:46:35.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:35 smithi006 bash[33918]: audit 2023-08-27T21:46:34.879716+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:35.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:35 smithi130 bash[27579]: audit 2023-08-27T21:46:34.878348+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-08-27T21:46:35.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:35 smithi130 bash[27579]: audit 2023-08-27T21:46:34.879716+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:36.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:36 smithi130 bash[27579]: cephadm 2023-08-27T21:46:34.880734+0000 mgr.y (mgr.14146) 73 : cephadm [INF] Deploying daemon osd.0 on smithi006 2023-08-27T21:46:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:36 smithi130 bash[27579]: cluster 2023-08-27T21:46:35.144095+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:36.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:36 smithi006 bash[27292]: cephadm 2023-08-27T21:46:34.880734+0000 mgr.y (mgr.14146) 73 : cephadm [INF] Deploying daemon osd.0 on smithi006 2023-08-27T21:46:36.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:36 smithi006 bash[27292]: cluster 2023-08-27T21:46:35.144095+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:36.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:36 smithi006 bash[33918]: cephadm 2023-08-27T21:46:34.880734+0000 mgr.y (mgr.14146) 73 : cephadm [INF] Deploying daemon osd.0 on smithi006 2023-08-27T21:46:36.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:36 smithi006 bash[33918]: cluster 2023-08-27T21:46:35.144095+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:38.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:38 smithi130 bash[27579]: cluster 2023-08-27T21:46:37.144629+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:38 smithi006 bash[33918]: cluster 2023-08-27T21:46:37.144629+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:38 smithi006 bash[27292]: cluster 2023-08-27T21:46:37.144629+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:40 smithi130 bash[27579]: cluster 2023-08-27T21:46:39.145027+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:40 smithi130 bash[27579]: audit 2023-08-27T21:46:39.331905+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:40 smithi130 bash[27579]: audit 2023-08-27T21:46:39.373469+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:46:40.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:40 smithi006 bash[27292]: cluster 2023-08-27T21:46:39.145027+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:40 smithi006 bash[27292]: audit 2023-08-27T21:46:39.331905+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:40 smithi006 bash[27292]: audit 2023-08-27T21:46:39.373469+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:46:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:40 smithi006 bash[33918]: cluster 2023-08-27T21:46:39.145027+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:40 smithi006 bash[33918]: audit 2023-08-27T21:46:39.331905+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:40 smithi006 bash[33918]: audit 2023-08-27T21:46:39.373469+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:46:42.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:42 smithi130 bash[27579]: cluster 2023-08-27T21:46:41.145447+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:42.703 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:42 smithi006 bash[27292]: cluster 2023-08-27T21:46:41.145447+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:42.703 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:42 smithi006 bash[33918]: cluster 2023-08-27T21:46:41.145447+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:44.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:44 smithi130 bash[27579]: cluster 2023-08-27T21:46:43.145896+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:44.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:44 smithi006 bash[27292]: cluster 2023-08-27T21:46:43.145896+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:44.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:44 smithi006 bash[33918]: cluster 2023-08-27T21:46:43.145896+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:46.611 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:46 smithi006 bash[27292]: cluster 2023-08-27T21:46:45.146340+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:46.611 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:46 smithi006 bash[27292]: audit 2023-08-27T21:46:45.767366+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:46.612 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:46 smithi006 bash[27292]: audit 2023-08-27T21:46:45.768581+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:46.612 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:46 smithi006 bash[27292]: audit 2023-08-27T21:46:45.769650+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:46:46.612 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:46 smithi006 bash[27292]: audit 2023-08-27T21:46:45.778835+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:46.612 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:46 smithi006 bash[33918]: cluster 2023-08-27T21:46:45.146340+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:46.612 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:46 smithi006 bash[33918]: audit 2023-08-27T21:46:45.767366+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:46.612 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:46 smithi006 bash[33918]: audit 2023-08-27T21:46:45.768581+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:46.612 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:46 smithi006 bash[33918]: audit 2023-08-27T21:46:45.769650+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:46:46.612 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:46 smithi006 bash[33918]: audit 2023-08-27T21:46:45.778835+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:46.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:46 smithi130 bash[27579]: cluster 2023-08-27T21:46:45.146340+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:46 smithi130 bash[27579]: audit 2023-08-27T21:46:45.767366+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:46 smithi130 bash[27579]: audit 2023-08-27T21:46:45.768581+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:46 smithi130 bash[27579]: audit 2023-08-27T21:46:45.769650+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:46:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:46 smithi130 bash[27579]: audit 2023-08-27T21:46:45.778835+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:47.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:47 smithi130 bash[27579]: audit 2023-08-27T21:46:46.392621+0000 mon.a (mon.0) 244 : audit [INF] from='osd.0 [v2:172.21.15.6:6802/224143379,v1:172.21.15.6:6803/224143379]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-08-27T21:46:47.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:47 smithi006 bash[27292]: audit 2023-08-27T21:46:46.392621+0000 mon.a (mon.0) 244 : audit [INF] from='osd.0 [v2:172.21.15.6:6802/224143379,v1:172.21.15.6:6803/224143379]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-08-27T21:46:47.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:47 smithi006 bash[33918]: audit 2023-08-27T21:46:46.392621+0000 mon.a (mon.0) 244 : audit [INF] from='osd.0 [v2:172.21.15.6:6802/224143379,v1:172.21.15.6:6803/224143379]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-08-27T21:46:48.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:48 smithi130 bash[27579]: cluster 2023-08-27T21:46:47.146778+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:48 smithi130 bash[27579]: audit 2023-08-27T21:46:47.358270+0000 mon.a (mon.0) 245 : audit [INF] from='osd.0 [v2:172.21.15.6:6802/224143379,v1:172.21.15.6:6803/224143379]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-08-27T21:46:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:48 smithi130 bash[27579]: cluster 2023-08-27T21:46:47.358352+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-08-27T21:46:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:48 smithi130 bash[27579]: audit 2023-08-27T21:46:47.358563+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:46:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:48 smithi130 bash[27579]: audit 2023-08-27T21:46:47.359574+0000 mon.a (mon.0) 248 : audit [INF] from='osd.0 [v2:172.21.15.6:6802/224143379,v1:172.21.15.6:6803/224143379]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2023-08-27T21:46:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:48 smithi006 bash[27292]: cluster 2023-08-27T21:46:47.146778+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:48 smithi006 bash[27292]: audit 2023-08-27T21:46:47.358270+0000 mon.a (mon.0) 245 : audit [INF] from='osd.0 [v2:172.21.15.6:6802/224143379,v1:172.21.15.6:6803/224143379]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-08-27T21:46:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:48 smithi006 bash[27292]: cluster 2023-08-27T21:46:47.358352+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-08-27T21:46:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:48 smithi006 bash[27292]: audit 2023-08-27T21:46:47.358563+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:46:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:48 smithi006 bash[27292]: audit 2023-08-27T21:46:47.359574+0000 mon.a (mon.0) 248 : audit [INF] from='osd.0 [v2:172.21.15.6:6802/224143379,v1:172.21.15.6:6803/224143379]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2023-08-27T21:46:48.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:48 smithi006 bash[33918]: cluster 2023-08-27T21:46:47.146778+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:48.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:48 smithi006 bash[33918]: audit 2023-08-27T21:46:47.358270+0000 mon.a (mon.0) 245 : audit [INF] from='osd.0 [v2:172.21.15.6:6802/224143379,v1:172.21.15.6:6803/224143379]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-08-27T21:46:48.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:48 smithi006 bash[33918]: cluster 2023-08-27T21:46:47.358352+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-08-27T21:46:48.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:48 smithi006 bash[33918]: audit 2023-08-27T21:46:47.358563+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:46:48.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:48 smithi006 bash[33918]: audit 2023-08-27T21:46:47.359574+0000 mon.a (mon.0) 248 : audit [INF] from='osd.0 [v2:172.21.15.6:6802/224143379,v1:172.21.15.6:6803/224143379]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2023-08-27T21:46:49.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:49 smithi006 bash[27292]: audit 2023-08-27T21:46:48.363293+0000 mon.a (mon.0) 249 : audit [INF] from='osd.0 [v2:172.21.15.6:6802/224143379,v1:172.21.15.6:6803/224143379]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2023-08-27T21:46:49.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:49 smithi006 bash[27292]: cluster 2023-08-27T21:46:48.363503+0000 mon.a (mon.0) 250 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-08-27T21:46:49.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:49 smithi006 bash[27292]: audit 2023-08-27T21:46:48.364303+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:46:49.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:49 smithi006 bash[27292]: audit 2023-08-27T21:46:48.370769+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:46:49.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:49 smithi006 bash[33918]: audit 2023-08-27T21:46:48.363293+0000 mon.a (mon.0) 249 : audit [INF] from='osd.0 [v2:172.21.15.6:6802/224143379,v1:172.21.15.6:6803/224143379]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2023-08-27T21:46:49.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:49 smithi006 bash[33918]: cluster 2023-08-27T21:46:48.363503+0000 mon.a (mon.0) 250 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-08-27T21:46:49.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:49 smithi006 bash[33918]: audit 2023-08-27T21:46:48.364303+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:46:49.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:49 smithi006 bash[33918]: audit 2023-08-27T21:46:48.370769+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:46:49.727 INFO:teuthology.orchestra.run.smithi006.stdout:Created osd(s) 0 on host 'smithi006' 2023-08-27T21:46:49.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:49 smithi130 bash[27579]: audit 2023-08-27T21:46:48.363293+0000 mon.a (mon.0) 249 : audit [INF] from='osd.0 [v2:172.21.15.6:6802/224143379,v1:172.21.15.6:6803/224143379]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2023-08-27T21:46:49.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:49 smithi130 bash[27579]: cluster 2023-08-27T21:46:48.363503+0000 mon.a (mon.0) 250 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-08-27T21:46:49.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:49 smithi130 bash[27579]: audit 2023-08-27T21:46:48.364303+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:46:49.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:49 smithi130 bash[27579]: audit 2023-08-27T21:46:48.370769+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:46:50.407 DEBUG:teuthology.orchestra.run.smithi006:osd.0> sudo journalctl -f -n 0 -u ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@osd.0.service 2023-08-27T21:46:50.410 INFO:tasks.cephadm:Deploying osd.1 on smithi006 with /dev/vg_nvme/lv_3... 2023-08-27T21:46:50.410 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-08-27T21:46:50.618 INFO:journalctl@ceph.osd.0.smithi006.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-08-27T21:46:50.660 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:50 smithi006 bash[33918]: cluster 2023-08-27T21:46:47.390653+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:46:50.660 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:50 smithi006 bash[33918]: cluster 2023-08-27T21:46:47.390739+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:46:50.660 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:50 smithi006 bash[33918]: cluster 2023-08-27T21:46:49.147207+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:50.660 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:50 smithi006 bash[33918]: cluster 2023-08-27T21:46:49.371967+0000 mon.a (mon.0) 253 : cluster [INF] osd.0 [v2:172.21.15.6:6802/224143379,v1:172.21.15.6:6803/224143379] boot 2023-08-27T21:46:50.660 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:50 smithi006 bash[33918]: cluster 2023-08-27T21:46:49.371999+0000 mon.a (mon.0) 254 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-08-27T21:46:50.660 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:50 smithi006 bash[33918]: audit 2023-08-27T21:46:49.372582+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:46:50.660 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:50 smithi006 bash[33918]: audit 2023-08-27T21:46:49.726166+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:50.661 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:50 smithi006 bash[33918]: audit 2023-08-27T21:46:49.757067+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:46:50.661 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:50 smithi006 bash[27292]: cluster 2023-08-27T21:46:47.390653+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:46:50.661 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:50 smithi006 bash[27292]: cluster 2023-08-27T21:46:47.390739+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:46:50.661 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:50 smithi006 bash[27292]: cluster 2023-08-27T21:46:49.147207+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:50.661 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:50 smithi006 bash[27292]: cluster 2023-08-27T21:46:49.371967+0000 mon.a (mon.0) 253 : cluster [INF] osd.0 [v2:172.21.15.6:6802/224143379,v1:172.21.15.6:6803/224143379] boot 2023-08-27T21:46:50.661 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:50 smithi006 bash[27292]: cluster 2023-08-27T21:46:49.371999+0000 mon.a (mon.0) 254 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-08-27T21:46:50.661 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:50 smithi006 bash[27292]: audit 2023-08-27T21:46:49.372582+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:46:50.661 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:50 smithi006 bash[27292]: audit 2023-08-27T21:46:49.726166+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:50.661 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:50 smithi006 bash[27292]: audit 2023-08-27T21:46:49.757067+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:46:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:50 smithi130 bash[27579]: cluster 2023-08-27T21:46:47.390653+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:46:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:50 smithi130 bash[27579]: cluster 2023-08-27T21:46:47.390739+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:46:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:50 smithi130 bash[27579]: cluster 2023-08-27T21:46:49.147207+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T21:46:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:50 smithi130 bash[27579]: cluster 2023-08-27T21:46:49.371967+0000 mon.a (mon.0) 253 : cluster [INF] osd.0 [v2:172.21.15.6:6802/224143379,v1:172.21.15.6:6803/224143379] boot 2023-08-27T21:46:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:50 smithi130 bash[27579]: cluster 2023-08-27T21:46:49.371999+0000 mon.a (mon.0) 254 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-08-27T21:46:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:50 smithi130 bash[27579]: audit 2023-08-27T21:46:49.372582+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:46:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:50 smithi130 bash[27579]: audit 2023-08-27T21:46:49.726166+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:46:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:50 smithi130 bash[27579]: audit 2023-08-27T21:46:49.757067+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:46:51.662 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:51 smithi006 bash[33918]: cluster 2023-08-27T21:46:50.388130+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-08-27T21:46:51.663 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:51 smithi006 bash[27292]: cluster 2023-08-27T21:46:50.388130+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-08-27T21:46:51.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:51 smithi130 bash[27579]: cluster 2023-08-27T21:46:50.388130+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-08-27T21:46:52.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:52 smithi006 bash[27292]: cluster 2023-08-27T21:46:51.147596+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:46:52.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:52 smithi006 bash[33918]: cluster 2023-08-27T21:46:51.147596+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:46:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:52 smithi130 bash[27579]: cluster 2023-08-27T21:46:51.147596+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:46:53.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:53 smithi006 bash[27292]: cluster 2023-08-27T21:46:53.148046+0000 mgr.y (mgr.14146) 83 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:46:53.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:53 smithi006 bash[33918]: cluster 2023-08-27T21:46:53.148046+0000 mgr.y (mgr.14146) 83 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:46:53.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:53 smithi130 bash[27579]: cluster 2023-08-27T21:46:53.148046+0000 mgr.y (mgr.14146) 83 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:46:54.176 INFO:teuthology.orchestra.run.smithi006.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-08-27T21:46:54.176 INFO:teuthology.orchestra.run.smithi006.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-08-27T21:46:54.176 INFO:teuthology.orchestra.run.smithi006.stderr: stderr: 10+0 records in 2023-08-27T21:46:54.176 INFO:teuthology.orchestra.run.smithi006.stderr:10+0 records out 2023-08-27T21:46:54.176 INFO:teuthology.orchestra.run.smithi006.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0200091 s, 524 MB/s 2023-08-27T21:46:54.176 INFO:teuthology.orchestra.run.smithi006.stderr:--> Zapping successful for: 2023-08-27T21:46:54.852 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph orch daemon add osd smithi006:vg_nvme/lv_3 2023-08-27T21:46:56.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:56 smithi006 bash[33918]: cluster 2023-08-27T21:46:55.148599+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:46:56.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:56 smithi006 bash[27292]: cluster 2023-08-27T21:46:55.148599+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:46:56.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:56 smithi130 bash[27579]: cluster 2023-08-27T21:46:55.148599+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:46:58.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:46:58 smithi130 bash[27579]: cluster 2023-08-27T21:46:57.149060+0000 mgr.y (mgr.14146) 85 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:46:58.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:46:58 smithi006 bash[27292]: cluster 2023-08-27T21:46:57.149060+0000 mgr.y (mgr.14146) 85 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:46:58.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:46:58 smithi006 bash[33918]: cluster 2023-08-27T21:46:57.149060+0000 mgr.y (mgr.14146) 85 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:00.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:00 smithi130 bash[27579]: cluster 2023-08-27T21:46:59.149412+0000 mgr.y (mgr.14146) 86 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:00 smithi130 bash[27579]: audit 2023-08-27T21:46:59.903108+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:47:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:00 smithi130 bash[27579]: audit 2023-08-27T21:46:59.907981+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:47:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:00 smithi130 bash[27579]: audit 2023-08-27T21:46:59.909156+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:47:00.654 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:00 smithi006 bash[33918]: cluster 2023-08-27T21:46:59.149412+0000 mgr.y (mgr.14146) 86 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:00.654 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:00 smithi006 bash[33918]: audit 2023-08-27T21:46:59.903108+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:47:00.654 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:00 smithi006 bash[33918]: audit 2023-08-27T21:46:59.907981+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:47:00.654 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:00 smithi006 bash[33918]: audit 2023-08-27T21:46:59.909156+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:47:00.654 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:00 smithi006 bash[27292]: cluster 2023-08-27T21:46:59.149412+0000 mgr.y (mgr.14146) 86 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:00.654 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:00 smithi006 bash[27292]: audit 2023-08-27T21:46:59.903108+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:47:00.654 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:00 smithi006 bash[27292]: audit 2023-08-27T21:46:59.907981+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:47:00.655 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:00 smithi006 bash[27292]: audit 2023-08-27T21:46:59.909156+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:47:01.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:01 smithi130 bash[27579]: audit 2023-08-27T21:46:59.897652+0000 mgr.y (mgr.14146) 87 : audit [DBG] from='client.14226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:47:01.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:01 smithi006 bash[27292]: audit 2023-08-27T21:46:59.897652+0000 mgr.y (mgr.14146) 87 : audit [DBG] from='client.14226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:47:01.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:01 smithi006 bash[33918]: audit 2023-08-27T21:46:59.897652+0000 mgr.y (mgr.14146) 87 : audit [DBG] from='client.14226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:47:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:02 smithi130 bash[27579]: cluster 2023-08-27T21:47:01.149878+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:02 smithi130 bash[27579]: cephadm 2023-08-27T21:47:01.358708+0000 mgr.y (mgr.14146) 89 : cephadm [INF] Detected new or changed devices on smithi006 2023-08-27T21:47:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:02 smithi130 bash[27579]: audit 2023-08-27T21:47:01.367018+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:02 smithi130 bash[27579]: audit 2023-08-27T21:47:01.368318+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:47:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:02 smithi130 bash[27579]: cephadm 2023-08-27T21:47:01.369255+0000 mgr.y (mgr.14146) 90 : cephadm [INF] Adjusting osd_memory_target on smithi006 to 16245M 2023-08-27T21:47:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:02 smithi130 bash[27579]: audit 2023-08-27T21:47:01.374408+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:02 smithi130 bash[27579]: audit 2023-08-27T21:47:01.376109+0000 mon.a (mon.0) 265 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:47:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:02 smithi130 bash[27579]: audit 2023-08-27T21:47:01.377353+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:47:02.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:02 smithi130 bash[27579]: audit 2023-08-27T21:47:01.383272+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:02.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:02 smithi006 bash[27292]: cluster 2023-08-27T21:47:01.149878+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:02.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:02 smithi006 bash[27292]: cephadm 2023-08-27T21:47:01.358708+0000 mgr.y (mgr.14146) 89 : cephadm [INF] Detected new or changed devices on smithi006 2023-08-27T21:47:02.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:02 smithi006 bash[27292]: audit 2023-08-27T21:47:01.367018+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:02.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:02 smithi006 bash[27292]: audit 2023-08-27T21:47:01.368318+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:47:02.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:02 smithi006 bash[27292]: cephadm 2023-08-27T21:47:01.369255+0000 mgr.y (mgr.14146) 90 : cephadm [INF] Adjusting osd_memory_target on smithi006 to 16245M 2023-08-27T21:47:02.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:02 smithi006 bash[27292]: audit 2023-08-27T21:47:01.374408+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:02.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:02 smithi006 bash[27292]: audit 2023-08-27T21:47:01.376109+0000 mon.a (mon.0) 265 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:47:02.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:02 smithi006 bash[27292]: audit 2023-08-27T21:47:01.377353+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:47:02.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:02 smithi006 bash[27292]: audit 2023-08-27T21:47:01.383272+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:02.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:02 smithi006 bash[33918]: cluster 2023-08-27T21:47:01.149878+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:02.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:02 smithi006 bash[33918]: cephadm 2023-08-27T21:47:01.358708+0000 mgr.y (mgr.14146) 89 : cephadm [INF] Detected new or changed devices on smithi006 2023-08-27T21:47:02.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:02 smithi006 bash[33918]: audit 2023-08-27T21:47:01.367018+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:02.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:02 smithi006 bash[33918]: audit 2023-08-27T21:47:01.368318+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:47:02.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:02 smithi006 bash[33918]: cephadm 2023-08-27T21:47:01.369255+0000 mgr.y (mgr.14146) 90 : cephadm [INF] Adjusting osd_memory_target on smithi006 to 16245M 2023-08-27T21:47:02.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:02 smithi006 bash[33918]: audit 2023-08-27T21:47:01.374408+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:02.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:02 smithi006 bash[33918]: audit 2023-08-27T21:47:01.376109+0000 mon.a (mon.0) 265 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:47:02.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:02 smithi006 bash[33918]: audit 2023-08-27T21:47:01.377353+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:47:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:02 smithi006 bash[33918]: audit 2023-08-27T21:47:01.383272+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:04.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:04 smithi006 bash[27292]: cluster 2023-08-27T21:47:03.150422+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:04.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:04 smithi006 bash[33918]: cluster 2023-08-27T21:47:03.150422+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:04 smithi130 bash[27579]: cluster 2023-08-27T21:47:03.150422+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:06 smithi006 bash[27292]: cluster 2023-08-27T21:47:05.151018+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:06.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:06 smithi006 bash[33918]: cluster 2023-08-27T21:47:05.151018+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:06 smithi130 bash[27579]: cluster 2023-08-27T21:47:05.151018+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:08 smithi006 bash[27292]: cluster 2023-08-27T21:47:07.151591+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:08.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:08 smithi006 bash[33918]: cluster 2023-08-27T21:47:07.151591+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:08 smithi130 bash[27579]: cluster 2023-08-27T21:47:07.151591+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:09.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:09 smithi006 bash[27292]: audit 2023-08-27T21:47:08.549257+0000 mon.a (mon.0) 268 : audit [INF] from='client.? 172.21.15.6:0/4229413952' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55ceda65-564a-4b00-89c2-718610d59762"}]: dispatch 2023-08-27T21:47:09.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:09 smithi006 bash[27292]: audit 2023-08-27T21:47:08.555298+0000 mon.a (mon.0) 269 : audit [INF] from='client.? 172.21.15.6:0/4229413952' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "55ceda65-564a-4b00-89c2-718610d59762"}]': finished 2023-08-27T21:47:09.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:09 smithi006 bash[27292]: cluster 2023-08-27T21:47:08.555359+0000 mon.a (mon.0) 270 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-08-27T21:47:09.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:09 smithi006 bash[27292]: audit 2023-08-27T21:47:08.555599+0000 mon.a (mon.0) 271 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:47:09.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:09 smithi006 bash[27292]: audit 2023-08-27T21:47:09.293073+0000 mon.c (mon.2) 5 : audit [DBG] from='client.? 172.21.15.6:0/2806289290' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:47:09.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:09 smithi006 bash[33918]: audit 2023-08-27T21:47:08.549257+0000 mon.a (mon.0) 268 : audit [INF] from='client.? 172.21.15.6:0/4229413952' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55ceda65-564a-4b00-89c2-718610d59762"}]: dispatch 2023-08-27T21:47:09.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:09 smithi006 bash[33918]: audit 2023-08-27T21:47:08.555298+0000 mon.a (mon.0) 269 : audit [INF] from='client.? 172.21.15.6:0/4229413952' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "55ceda65-564a-4b00-89c2-718610d59762"}]': finished 2023-08-27T21:47:09.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:09 smithi006 bash[33918]: cluster 2023-08-27T21:47:08.555359+0000 mon.a (mon.0) 270 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-08-27T21:47:09.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:09 smithi006 bash[33918]: audit 2023-08-27T21:47:08.555599+0000 mon.a (mon.0) 271 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:47:09.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:09 smithi006 bash[33918]: audit 2023-08-27T21:47:09.293073+0000 mon.c (mon.2) 5 : audit [DBG] from='client.? 172.21.15.6:0/2806289290' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:47:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:09 smithi130 bash[27579]: audit 2023-08-27T21:47:08.549257+0000 mon.a (mon.0) 268 : audit [INF] from='client.? 172.21.15.6:0/4229413952' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55ceda65-564a-4b00-89c2-718610d59762"}]: dispatch 2023-08-27T21:47:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:09 smithi130 bash[27579]: audit 2023-08-27T21:47:08.555298+0000 mon.a (mon.0) 269 : audit [INF] from='client.? 172.21.15.6:0/4229413952' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "55ceda65-564a-4b00-89c2-718610d59762"}]': finished 2023-08-27T21:47:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:09 smithi130 bash[27579]: cluster 2023-08-27T21:47:08.555359+0000 mon.a (mon.0) 270 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-08-27T21:47:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:09 smithi130 bash[27579]: audit 2023-08-27T21:47:08.555599+0000 mon.a (mon.0) 271 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:47:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:09 smithi130 bash[27579]: audit 2023-08-27T21:47:09.293073+0000 mon.c (mon.2) 5 : audit [DBG] from='client.? 172.21.15.6:0/2806289290' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:47:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:10 smithi006 bash[27292]: cluster 2023-08-27T21:47:09.152063+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:10.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:10 smithi006 bash[33918]: cluster 2023-08-27T21:47:09.152063+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:10 smithi130 bash[27579]: cluster 2023-08-27T21:47:09.152063+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:12.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:12 smithi006 bash[27292]: cluster 2023-08-27T21:47:11.152565+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:12.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:12 smithi006 bash[33918]: cluster 2023-08-27T21:47:11.152565+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:12 smithi130 bash[27579]: cluster 2023-08-27T21:47:11.152565+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:14.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:14 smithi006 bash[27292]: cluster 2023-08-27T21:47:13.153056+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:14.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:14 smithi006 bash[33918]: cluster 2023-08-27T21:47:13.153056+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:14 smithi130 bash[27579]: cluster 2023-08-27T21:47:13.153056+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:15.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:15 smithi006 bash[27292]: cluster 2023-08-27T21:47:15.153565+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:15.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:15 smithi006 bash[33918]: cluster 2023-08-27T21:47:15.153565+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:15.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:15 smithi130 bash[27579]: cluster 2023-08-27T21:47:15.153565+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:18.568 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:18 smithi006 bash[27292]: cluster 2023-08-27T21:47:17.154054+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:18.569 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:18 smithi006 bash[33918]: cluster 2023-08-27T21:47:17.154054+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:18 smithi130 bash[27579]: cluster 2023-08-27T21:47:17.154054+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:20.607 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:20 smithi006 bash[27292]: cluster 2023-08-27T21:47:19.154428+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:20.608 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:20 smithi006 bash[33918]: cluster 2023-08-27T21:47:19.154428+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:20.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:20 smithi130 bash[27579]: cluster 2023-08-27T21:47:19.154428+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:21.613 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:21 smithi006 bash[27292]: audit 2023-08-27T21:47:21.288808+0000 mon.a (mon.0) 272 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-08-27T21:47:21.613 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:21 smithi006 bash[27292]: audit 2023-08-27T21:47:21.290020+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:47:21.613 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:21 smithi006 bash[33918]: audit 2023-08-27T21:47:21.288808+0000 mon.a (mon.0) 272 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-08-27T21:47:21.613 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:21 smithi006 bash[33918]: audit 2023-08-27T21:47:21.290020+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:47:21.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:21 smithi130 bash[27579]: audit 2023-08-27T21:47:21.288808+0000 mon.a (mon.0) 272 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-08-27T21:47:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:21 smithi130 bash[27579]: audit 2023-08-27T21:47:21.290020+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:47:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:22 smithi130 bash[27579]: cluster 2023-08-27T21:47:21.154728+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:22 smithi130 bash[27579]: cephadm 2023-08-27T21:47:21.291141+0000 mgr.y (mgr.14146) 101 : cephadm [INF] Deploying daemon osd.1 on smithi006 2023-08-27T21:47:22.680 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:22 smithi006 bash[33918]: cluster 2023-08-27T21:47:21.154728+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:22.680 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:22 smithi006 bash[33918]: cephadm 2023-08-27T21:47:21.291141+0000 mgr.y (mgr.14146) 101 : cephadm [INF] Deploying daemon osd.1 on smithi006 2023-08-27T21:47:22.680 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:22 smithi006 bash[27292]: cluster 2023-08-27T21:47:21.154728+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:22.680 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:22 smithi006 bash[27292]: cephadm 2023-08-27T21:47:21.291141+0000 mgr.y (mgr.14146) 101 : cephadm [INF] Deploying daemon osd.1 on smithi006 2023-08-27T21:47:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:24 smithi006 bash[27292]: cluster 2023-08-27T21:47:23.155179+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:24.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:24 smithi006 bash[33918]: cluster 2023-08-27T21:47:23.155179+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:24.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:24 smithi130 bash[27579]: cluster 2023-08-27T21:47:23.155179+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:26 smithi130 bash[27579]: cluster 2023-08-27T21:47:25.155658+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:26 smithi130 bash[27579]: audit 2023-08-27T21:47:25.809122+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:26 smithi130 bash[27579]: audit 2023-08-27T21:47:25.815208+0000 mon.a (mon.0) 275 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:47:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:26 smithi006 bash[27292]: cluster 2023-08-27T21:47:25.155658+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:26 smithi006 bash[27292]: audit 2023-08-27T21:47:25.809122+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:26 smithi006 bash[27292]: audit 2023-08-27T21:47:25.815208+0000 mon.a (mon.0) 275 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:47:26.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:26 smithi006 bash[33918]: cluster 2023-08-27T21:47:25.155658+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:26.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:26 smithi006 bash[33918]: audit 2023-08-27T21:47:25.809122+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:26.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:26 smithi006 bash[33918]: audit 2023-08-27T21:47:25.815208+0000 mon.a (mon.0) 275 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:47:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:28 smithi006 bash[27292]: cluster 2023-08-27T21:47:27.155964+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:28 smithi006 bash[33918]: cluster 2023-08-27T21:47:27.155964+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:28 smithi130 bash[27579]: cluster 2023-08-27T21:47:27.155964+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:30 smithi006 bash[27292]: cluster 2023-08-27T21:47:29.156367+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:30 smithi006 bash[33918]: cluster 2023-08-27T21:47:29.156367+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:30 smithi130 bash[27579]: cluster 2023-08-27T21:47:29.156367+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:32.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:32 smithi006 bash[27292]: cluster 2023-08-27T21:47:31.156891+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:32.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:32 smithi006 bash[27292]: audit 2023-08-27T21:47:32.220345+0000 mon.a (mon.0) 276 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:32.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:32 smithi006 bash[33918]: cluster 2023-08-27T21:47:31.156891+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:32.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:32 smithi006 bash[33918]: audit 2023-08-27T21:47:32.220345+0000 mon.a (mon.0) 276 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:32 smithi130 bash[27579]: cluster 2023-08-27T21:47:31.156891+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:32 smithi130 bash[27579]: audit 2023-08-27T21:47:32.220345+0000 mon.a (mon.0) 276 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:33.646 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:33 smithi006 bash[27292]: audit 2023-08-27T21:47:32.503408+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:47:33.646 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:33 smithi006 bash[27292]: audit 2023-08-27T21:47:32.504483+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:47:33.646 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:33 smithi006 bash[27292]: audit 2023-08-27T21:47:32.515433+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:33.646 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:33 smithi006 bash[27292]: audit 2023-08-27T21:47:32.797073+0000 mon.a (mon.0) 280 : audit [INF] from='osd.1 [v2:172.21.15.6:6810/2798606708,v1:172.21.15.6:6811/2798606708]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-08-27T21:47:33.647 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:33 smithi006 bash[33918]: audit 2023-08-27T21:47:32.503408+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:47:33.647 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:33 smithi006 bash[33918]: audit 2023-08-27T21:47:32.504483+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:47:33.647 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:33 smithi006 bash[33918]: audit 2023-08-27T21:47:32.515433+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:33.647 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:33 smithi006 bash[33918]: audit 2023-08-27T21:47:32.797073+0000 mon.a (mon.0) 280 : audit [INF] from='osd.1 [v2:172.21.15.6:6810/2798606708,v1:172.21.15.6:6811/2798606708]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-08-27T21:47:33.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:33 smithi130 bash[27579]: audit 2023-08-27T21:47:32.503408+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:47:33.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:33 smithi130 bash[27579]: audit 2023-08-27T21:47:32.504483+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:47:33.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:33 smithi130 bash[27579]: audit 2023-08-27T21:47:32.515433+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:33.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:33 smithi130 bash[27579]: audit 2023-08-27T21:47:32.797073+0000 mon.a (mon.0) 280 : audit [INF] from='osd.1 [v2:172.21.15.6:6810/2798606708,v1:172.21.15.6:6811/2798606708]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-08-27T21:47:34.675 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:34 smithi006 bash[27292]: cluster 2023-08-27T21:47:33.157340+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:34.675 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:34 smithi006 bash[27292]: audit 2023-08-27T21:47:33.519670+0000 mon.a (mon.0) 281 : audit [INF] from='osd.1 [v2:172.21.15.6:6810/2798606708,v1:172.21.15.6:6811/2798606708]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-08-27T21:47:34.675 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:34 smithi006 bash[27292]: cluster 2023-08-27T21:47:33.519781+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-08-27T21:47:34.675 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:34 smithi006 bash[27292]: audit 2023-08-27T21:47:33.521484+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:47:34.675 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:34 smithi006 bash[27292]: audit 2023-08-27T21:47:33.522029+0000 mon.a (mon.0) 284 : audit [INF] from='osd.1 [v2:172.21.15.6:6810/2798606708,v1:172.21.15.6:6811/2798606708]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2023-08-27T21:47:34.675 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:34 smithi006 bash[33918]: cluster 2023-08-27T21:47:33.157340+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:34.675 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:34 smithi006 bash[33918]: audit 2023-08-27T21:47:33.519670+0000 mon.a (mon.0) 281 : audit [INF] from='osd.1 [v2:172.21.15.6:6810/2798606708,v1:172.21.15.6:6811/2798606708]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-08-27T21:47:34.675 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:34 smithi006 bash[33918]: cluster 2023-08-27T21:47:33.519781+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-08-27T21:47:34.676 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:34 smithi006 bash[33918]: audit 2023-08-27T21:47:33.521484+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:47:34.676 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:34 smithi006 bash[33918]: audit 2023-08-27T21:47:33.522029+0000 mon.a (mon.0) 284 : audit [INF] from='osd.1 [v2:172.21.15.6:6810/2798606708,v1:172.21.15.6:6811/2798606708]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2023-08-27T21:47:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:34 smithi130 bash[27579]: cluster 2023-08-27T21:47:33.157340+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:34 smithi130 bash[27579]: audit 2023-08-27T21:47:33.519670+0000 mon.a (mon.0) 281 : audit [INF] from='osd.1 [v2:172.21.15.6:6810/2798606708,v1:172.21.15.6:6811/2798606708]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-08-27T21:47:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:34 smithi130 bash[27579]: cluster 2023-08-27T21:47:33.519781+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-08-27T21:47:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:34 smithi130 bash[27579]: audit 2023-08-27T21:47:33.521484+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:47:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:34 smithi130 bash[27579]: audit 2023-08-27T21:47:33.522029+0000 mon.a (mon.0) 284 : audit [INF] from='osd.1 [v2:172.21.15.6:6810/2798606708,v1:172.21.15.6:6811/2798606708]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2023-08-27T21:47:35.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:35 smithi006 bash[33918]: audit 2023-08-27T21:47:34.521273+0000 mon.a (mon.0) 285 : audit [INF] from='osd.1 [v2:172.21.15.6:6810/2798606708,v1:172.21.15.6:6811/2798606708]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2023-08-27T21:47:35.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:35 smithi006 bash[33918]: cluster 2023-08-27T21:47:34.521365+0000 mon.a (mon.0) 286 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-08-27T21:47:35.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:35 smithi006 bash[33918]: audit 2023-08-27T21:47:34.521553+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:47:35.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:35 smithi006 bash[33918]: audit 2023-08-27T21:47:34.526022+0000 mon.a (mon.0) 288 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:47:35.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:35 smithi006 bash[33918]: cluster 2023-08-27T21:47:35.157903+0000 mgr.y (mgr.14146) 108 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:35.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:35 smithi006 bash[27292]: audit 2023-08-27T21:47:34.521273+0000 mon.a (mon.0) 285 : audit [INF] from='osd.1 [v2:172.21.15.6:6810/2798606708,v1:172.21.15.6:6811/2798606708]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2023-08-27T21:47:35.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:35 smithi006 bash[27292]: cluster 2023-08-27T21:47:34.521365+0000 mon.a (mon.0) 286 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-08-27T21:47:35.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:35 smithi006 bash[27292]: audit 2023-08-27T21:47:34.521553+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:47:35.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:35 smithi006 bash[27292]: audit 2023-08-27T21:47:34.526022+0000 mon.a (mon.0) 288 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:47:35.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:35 smithi006 bash[27292]: cluster 2023-08-27T21:47:35.157903+0000 mgr.y (mgr.14146) 108 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:35.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:35 smithi130 bash[27579]: audit 2023-08-27T21:47:34.521273+0000 mon.a (mon.0) 285 : audit [INF] from='osd.1 [v2:172.21.15.6:6810/2798606708,v1:172.21.15.6:6811/2798606708]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2023-08-27T21:47:35.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:35 smithi130 bash[27579]: cluster 2023-08-27T21:47:34.521365+0000 mon.a (mon.0) 286 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-08-27T21:47:35.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:35 smithi130 bash[27579]: audit 2023-08-27T21:47:34.521553+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:47:35.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:35 smithi130 bash[27579]: audit 2023-08-27T21:47:34.526022+0000 mon.a (mon.0) 288 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:47:35.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:35 smithi130 bash[27579]: cluster 2023-08-27T21:47:35.157903+0000 mgr.y (mgr.14146) 108 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-27T21:47:35.924 INFO:teuthology.orchestra.run.smithi006.stdout:Created osd(s) 1 on host 'smithi006' 2023-08-27T21:47:36.609 DEBUG:teuthology.orchestra.run.smithi006:osd.1> sudo journalctl -f -n 0 -u ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@osd.1.service 2023-08-27T21:47:36.611 INFO:tasks.cephadm:Deploying osd.2 on smithi006 with /dev/vg_nvme/lv_2... 2023-08-27T21:47:36.611 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-08-27T21:47:36.806 INFO:journalctl@ceph.osd.1.smithi006.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-08-27T21:47:36.867 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:36 smithi006 bash[33918]: cluster 2023-08-27T21:47:33.822509+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:47:36.867 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:36 smithi006 bash[33918]: cluster 2023-08-27T21:47:33.822599+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:47:36.867 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:36 smithi006 bash[33918]: cluster 2023-08-27T21:47:35.523118+0000 mon.a (mon.0) 289 : cluster [INF] osd.1 [v2:172.21.15.6:6810/2798606708,v1:172.21.15.6:6811/2798606708] boot 2023-08-27T21:47:36.867 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:36 smithi006 bash[33918]: cluster 2023-08-27T21:47:35.523157+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-08-27T21:47:36.867 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:36 smithi006 bash[33918]: audit 2023-08-27T21:47:35.523601+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:47:36.868 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:36 smithi006 bash[33918]: audit 2023-08-27T21:47:35.922648+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:36.868 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:36 smithi006 bash[33918]: audit 2023-08-27T21:47:35.941756+0000 mon.a (mon.0) 293 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:47:36.868 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:36 smithi006 bash[27292]: cluster 2023-08-27T21:47:33.822509+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:47:36.868 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:36 smithi006 bash[27292]: cluster 2023-08-27T21:47:33.822599+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:47:36.868 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:36 smithi006 bash[27292]: cluster 2023-08-27T21:47:35.523118+0000 mon.a (mon.0) 289 : cluster [INF] osd.1 [v2:172.21.15.6:6810/2798606708,v1:172.21.15.6:6811/2798606708] boot 2023-08-27T21:47:36.868 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:36 smithi006 bash[27292]: cluster 2023-08-27T21:47:35.523157+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-08-27T21:47:36.868 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:36 smithi006 bash[27292]: audit 2023-08-27T21:47:35.523601+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:47:36.868 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:36 smithi006 bash[27292]: audit 2023-08-27T21:47:35.922648+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:36.868 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:36 smithi006 bash[27292]: audit 2023-08-27T21:47:35.941756+0000 mon.a (mon.0) 293 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:47:36.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:36 smithi130 bash[27579]: cluster 2023-08-27T21:47:33.822509+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:47:36.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:36 smithi130 bash[27579]: cluster 2023-08-27T21:47:33.822599+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:47:36.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:36 smithi130 bash[27579]: cluster 2023-08-27T21:47:35.523118+0000 mon.a (mon.0) 289 : cluster [INF] osd.1 [v2:172.21.15.6:6810/2798606708,v1:172.21.15.6:6811/2798606708] boot 2023-08-27T21:47:36.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:36 smithi130 bash[27579]: cluster 2023-08-27T21:47:35.523157+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-08-27T21:47:36.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:36 smithi130 bash[27579]: audit 2023-08-27T21:47:35.523601+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:47:36.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:36 smithi130 bash[27579]: audit 2023-08-27T21:47:35.922648+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:36.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:36 smithi130 bash[27579]: audit 2023-08-27T21:47:35.941756+0000 mon.a (mon.0) 293 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:47:37.816 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:37 smithi006 bash[27292]: cluster 2023-08-27T21:47:36.536344+0000 mon.a (mon.0) 294 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-08-27T21:47:37.817 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:37 smithi006 bash[27292]: cluster 2023-08-27T21:47:37.158264+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:37.818 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:37 smithi006 bash[33918]: cluster 2023-08-27T21:47:36.536344+0000 mon.a (mon.0) 294 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-08-27T21:47:37.818 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:37 smithi006 bash[33918]: cluster 2023-08-27T21:47:37.158264+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:37.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:37 smithi130 bash[27579]: cluster 2023-08-27T21:47:36.536344+0000 mon.a (mon.0) 294 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-08-27T21:47:37.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:37 smithi130 bash[27579]: cluster 2023-08-27T21:47:37.158264+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:40.257 INFO:teuthology.orchestra.run.smithi006.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-08-27T21:47:40.257 INFO:teuthology.orchestra.run.smithi006.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-08-27T21:47:40.258 INFO:teuthology.orchestra.run.smithi006.stderr: stderr: 10+0 records in 2023-08-27T21:47:40.258 INFO:teuthology.orchestra.run.smithi006.stderr:10+0 records out 2023-08-27T21:47:40.258 INFO:teuthology.orchestra.run.smithi006.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0167715 s, 625 MB/s 2023-08-27T21:47:40.258 INFO:teuthology.orchestra.run.smithi006.stderr:--> Zapping successful for: 2023-08-27T21:47:40.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:40 smithi006 bash[27292]: cluster 2023-08-27T21:47:39.158798+0000 mgr.y (mgr.14146) 110 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:40 smithi006 bash[33918]: cluster 2023-08-27T21:47:39.158798+0000 mgr.y (mgr.14146) 110 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:40 smithi130 bash[27579]: cluster 2023-08-27T21:47:39.158798+0000 mgr.y (mgr.14146) 110 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:41.042 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph orch daemon add osd smithi006:vg_nvme/lv_2 2023-08-27T21:47:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:42 smithi130 bash[27579]: cluster 2023-08-27T21:47:41.159419+0000 mgr.y (mgr.14146) 111 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:42.632 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:42 smithi006 bash[27292]: cluster 2023-08-27T21:47:41.159419+0000 mgr.y (mgr.14146) 111 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:42.633 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:42 smithi006 bash[33918]: cluster 2023-08-27T21:47:41.159419+0000 mgr.y (mgr.14146) 111 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:44 smithi130 bash[27579]: cluster 2023-08-27T21:47:43.159975+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:44.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:44 smithi006 bash[27292]: cluster 2023-08-27T21:47:43.159975+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:44.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:44 smithi006 bash[33918]: cluster 2023-08-27T21:47:43.159975+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:46 smithi130 bash[27579]: cluster 2023-08-27T21:47:45.160438+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:46 smithi130 bash[27579]: audit 2023-08-27T21:47:45.675982+0000 mon.a (mon.0) 295 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:47:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:46 smithi130 bash[27579]: audit 2023-08-27T21:47:45.680612+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:47:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:46 smithi130 bash[27579]: audit 2023-08-27T21:47:45.681810+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:47:46.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:46 smithi006 bash[27292]: cluster 2023-08-27T21:47:45.160438+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:46.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:46 smithi006 bash[27292]: audit 2023-08-27T21:47:45.675982+0000 mon.a (mon.0) 295 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:47:46.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:46 smithi006 bash[27292]: audit 2023-08-27T21:47:45.680612+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:47:46.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:46 smithi006 bash[27292]: audit 2023-08-27T21:47:45.681810+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:47:46.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:46 smithi006 bash[33918]: cluster 2023-08-27T21:47:45.160438+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:46.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:46 smithi006 bash[33918]: audit 2023-08-27T21:47:45.675982+0000 mon.a (mon.0) 295 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:47:46.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:46 smithi006 bash[33918]: audit 2023-08-27T21:47:45.680612+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:47:46.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:46 smithi006 bash[33918]: audit 2023-08-27T21:47:45.681810+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:47:47.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:47 smithi130 bash[27579]: audit 2023-08-27T21:47:45.670469+0000 mgr.y (mgr.14146) 114 : audit [DBG] from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:47:47.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:47 smithi130 bash[27579]: audit 2023-08-27T21:47:47.288203+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:47.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:47 smithi130 bash[27579]: audit 2023-08-27T21:47:47.290056+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:47:47.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:47 smithi130 bash[27579]: audit 2023-08-27T21:47:47.291639+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:47:47.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:47 smithi130 bash[27579]: audit 2023-08-27T21:47:47.297502+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:47.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:47 smithi130 bash[27579]: audit 2023-08-27T21:47:47.298966+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:47:47.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:47 smithi130 bash[27579]: audit 2023-08-27T21:47:47.300023+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:47:47.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:47 smithi130 bash[27579]: audit 2023-08-27T21:47:47.305727+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:47.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:47 smithi006 bash[27292]: audit 2023-08-27T21:47:45.670469+0000 mgr.y (mgr.14146) 114 : audit [DBG] from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:47:47.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:47 smithi006 bash[27292]: audit 2023-08-27T21:47:47.288203+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:47.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:47 smithi006 bash[27292]: audit 2023-08-27T21:47:47.290056+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:47:47.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:47 smithi006 bash[27292]: audit 2023-08-27T21:47:47.291639+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:47:47.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:47 smithi006 bash[27292]: audit 2023-08-27T21:47:47.297502+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:47.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:47 smithi006 bash[27292]: audit 2023-08-27T21:47:47.298966+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:47:47.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:47 smithi006 bash[27292]: audit 2023-08-27T21:47:47.300023+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:47:47.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:47 smithi006 bash[27292]: audit 2023-08-27T21:47:47.305727+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:47.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:47 smithi006 bash[33918]: audit 2023-08-27T21:47:45.670469+0000 mgr.y (mgr.14146) 114 : audit [DBG] from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:47:47.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:47 smithi006 bash[33918]: audit 2023-08-27T21:47:47.288203+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:47.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:47 smithi006 bash[33918]: audit 2023-08-27T21:47:47.290056+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:47:47.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:47 smithi006 bash[33918]: audit 2023-08-27T21:47:47.291639+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:47:47.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:47 smithi006 bash[33918]: audit 2023-08-27T21:47:47.297502+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:47.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:47 smithi006 bash[33918]: audit 2023-08-27T21:47:47.298966+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:47:47.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:47 smithi006 bash[33918]: audit 2023-08-27T21:47:47.300023+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:47:47.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:47 smithi006 bash[33918]: audit 2023-08-27T21:47:47.305727+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:47:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:48 smithi130 bash[27579]: cluster 2023-08-27T21:47:47.160773+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:48 smithi130 bash[27579]: cephadm 2023-08-27T21:47:47.279153+0000 mgr.y (mgr.14146) 116 : cephadm [INF] Detected new or changed devices on smithi006 2023-08-27T21:47:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:48 smithi130 bash[27579]: cephadm 2023-08-27T21:47:47.292643+0000 mgr.y (mgr.14146) 117 : cephadm [INF] Adjusting osd_memory_target on smithi006 to 8122M 2023-08-27T21:47:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:48 smithi006 bash[27292]: cluster 2023-08-27T21:47:47.160773+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:48.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:48 smithi006 bash[27292]: cephadm 2023-08-27T21:47:47.279153+0000 mgr.y (mgr.14146) 116 : cephadm [INF] Detected new or changed devices on smithi006 2023-08-27T21:47:48.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:48 smithi006 bash[27292]: cephadm 2023-08-27T21:47:47.292643+0000 mgr.y (mgr.14146) 117 : cephadm [INF] Adjusting osd_memory_target on smithi006 to 8122M 2023-08-27T21:47:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:48 smithi006 bash[33918]: cluster 2023-08-27T21:47:47.160773+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:48 smithi006 bash[33918]: cephadm 2023-08-27T21:47:47.279153+0000 mgr.y (mgr.14146) 116 : cephadm [INF] Detected new or changed devices on smithi006 2023-08-27T21:47:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:48 smithi006 bash[33918]: cephadm 2023-08-27T21:47:47.292643+0000 mgr.y (mgr.14146) 117 : cephadm [INF] Adjusting osd_memory_target on smithi006 to 8122M 2023-08-27T21:47:50.637 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:50 smithi006 bash[27292]: cluster 2023-08-27T21:47:49.161207+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:50.637 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:50 smithi006 bash[33918]: cluster 2023-08-27T21:47:49.161207+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:50 smithi130 bash[27579]: cluster 2023-08-27T21:47:49.161207+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:52.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:52 smithi006 bash[27292]: cluster 2023-08-27T21:47:51.161841+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:52.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:52 smithi006 bash[33918]: cluster 2023-08-27T21:47:51.161841+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:52 smithi130 bash[27579]: cluster 2023-08-27T21:47:51.161841+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:54.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:54 smithi006 bash[27292]: cluster 2023-08-27T21:47:53.162364+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:54.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:54 smithi006 bash[27292]: audit 2023-08-27T21:47:54.295174+0000 mon.a (mon.0) 305 : audit [INF] from='client.? 172.21.15.6:0/2716784904' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b027a54f-cf0e-4547-a530-f70e146cb386"}]: dispatch 2023-08-27T21:47:54.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:54 smithi006 bash[27292]: audit 2023-08-27T21:47:54.301523+0000 mon.a (mon.0) 306 : audit [INF] from='client.? 172.21.15.6:0/2716784904' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b027a54f-cf0e-4547-a530-f70e146cb386"}]': finished 2023-08-27T21:47:54.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:54 smithi006 bash[27292]: cluster 2023-08-27T21:47:54.301592+0000 mon.a (mon.0) 307 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-08-27T21:47:54.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:54 smithi006 bash[27292]: audit 2023-08-27T21:47:54.301875+0000 mon.a (mon.0) 308 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:47:54.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:54 smithi006 bash[33918]: cluster 2023-08-27T21:47:53.162364+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:54.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:54 smithi006 bash[33918]: audit 2023-08-27T21:47:54.295174+0000 mon.a (mon.0) 305 : audit [INF] from='client.? 172.21.15.6:0/2716784904' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b027a54f-cf0e-4547-a530-f70e146cb386"}]: dispatch 2023-08-27T21:47:54.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:54 smithi006 bash[33918]: audit 2023-08-27T21:47:54.301523+0000 mon.a (mon.0) 306 : audit [INF] from='client.? 172.21.15.6:0/2716784904' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b027a54f-cf0e-4547-a530-f70e146cb386"}]': finished 2023-08-27T21:47:54.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:54 smithi006 bash[33918]: cluster 2023-08-27T21:47:54.301592+0000 mon.a (mon.0) 307 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-08-27T21:47:54.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:54 smithi006 bash[33918]: audit 2023-08-27T21:47:54.301875+0000 mon.a (mon.0) 308 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:47:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:54 smithi130 bash[27579]: cluster 2023-08-27T21:47:53.162364+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:54 smithi130 bash[27579]: audit 2023-08-27T21:47:54.295174+0000 mon.a (mon.0) 305 : audit [INF] from='client.? 172.21.15.6:0/2716784904' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b027a54f-cf0e-4547-a530-f70e146cb386"}]: dispatch 2023-08-27T21:47:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:54 smithi130 bash[27579]: audit 2023-08-27T21:47:54.301523+0000 mon.a (mon.0) 306 : audit [INF] from='client.? 172.21.15.6:0/2716784904' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b027a54f-cf0e-4547-a530-f70e146cb386"}]': finished 2023-08-27T21:47:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:54 smithi130 bash[27579]: cluster 2023-08-27T21:47:54.301592+0000 mon.a (mon.0) 307 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-08-27T21:47:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:54 smithi130 bash[27579]: audit 2023-08-27T21:47:54.301875+0000 mon.a (mon.0) 308 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:47:55.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:55 smithi006 bash[27292]: audit 2023-08-27T21:47:55.042574+0000 mon.c (mon.2) 6 : audit [DBG] from='client.? 172.21.15.6:0/3376362062' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:47:55.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:55 smithi006 bash[33918]: audit 2023-08-27T21:47:55.042574+0000 mon.c (mon.2) 6 : audit [DBG] from='client.? 172.21.15.6:0/3376362062' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:47:55.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:55 smithi130 bash[27579]: audit 2023-08-27T21:47:55.042574+0000 mon.c (mon.2) 6 : audit [DBG] from='client.? 172.21.15.6:0/3376362062' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:47:56.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:56 smithi006 bash[27292]: cluster 2023-08-27T21:47:55.162875+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:56.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:56 smithi006 bash[33918]: cluster 2023-08-27T21:47:55.162875+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:56 smithi130 bash[27579]: cluster 2023-08-27T21:47:55.162875+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:58.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:47:58 smithi006 bash[27292]: cluster 2023-08-27T21:47:57.163353+0000 mgr.y (mgr.14146) 122 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:58.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:47:58 smithi006 bash[33918]: cluster 2023-08-27T21:47:57.163353+0000 mgr.y (mgr.14146) 122 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:47:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:47:58 smithi130 bash[27579]: cluster 2023-08-27T21:47:57.163353+0000 mgr.y (mgr.14146) 122 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:00 smithi006 bash[27292]: cluster 2023-08-27T21:47:59.163799+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:00.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:00 smithi006 bash[33918]: cluster 2023-08-27T21:47:59.163799+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:00 smithi130 bash[27579]: cluster 2023-08-27T21:47:59.163799+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:01.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:01 smithi006 bash[27292]: cluster 2023-08-27T21:48:01.164355+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:01.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:01 smithi006 bash[33918]: cluster 2023-08-27T21:48:01.164355+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:01.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:01 smithi130 bash[27579]: cluster 2023-08-27T21:48:01.164355+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:04 smithi130 bash[27579]: cluster 2023-08-27T21:48:03.164887+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:04.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:04 smithi006 bash[27292]: cluster 2023-08-27T21:48:03.164887+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:04.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:04 smithi006 bash[33918]: cluster 2023-08-27T21:48:03.164887+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:06 smithi130 bash[27579]: cluster 2023-08-27T21:48:05.165367+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:06 smithi006 bash[27292]: cluster 2023-08-27T21:48:05.165367+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:06.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:06 smithi006 bash[33918]: cluster 2023-08-27T21:48:05.165367+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:07.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:07 smithi130 bash[27579]: audit 2023-08-27T21:48:07.046897+0000 mon.a (mon.0) 309 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-08-27T21:48:07.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:07 smithi130 bash[27579]: audit 2023-08-27T21:48:07.048283+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:48:07.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:07 smithi006 bash[27292]: audit 2023-08-27T21:48:07.046897+0000 mon.a (mon.0) 309 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-08-27T21:48:07.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:07 smithi006 bash[27292]: audit 2023-08-27T21:48:07.048283+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:48:07.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:07 smithi006 bash[33918]: audit 2023-08-27T21:48:07.046897+0000 mon.a (mon.0) 309 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-08-27T21:48:07.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:07 smithi006 bash[33918]: audit 2023-08-27T21:48:07.048283+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:48:08.561 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:08 smithi006 bash[27292]: cephadm 2023-08-27T21:48:07.049492+0000 mgr.y (mgr.14146) 127 : cephadm [INF] Deploying daemon osd.2 on smithi006 2023-08-27T21:48:08.562 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:08 smithi006 bash[27292]: cluster 2023-08-27T21:48:07.165744+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:08.562 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:08 smithi006 bash[33918]: cephadm 2023-08-27T21:48:07.049492+0000 mgr.y (mgr.14146) 127 : cephadm [INF] Deploying daemon osd.2 on smithi006 2023-08-27T21:48:08.562 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:08 smithi006 bash[33918]: cluster 2023-08-27T21:48:07.165744+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:08 smithi130 bash[27579]: cephadm 2023-08-27T21:48:07.049492+0000 mgr.y (mgr.14146) 127 : cephadm [INF] Deploying daemon osd.2 on smithi006 2023-08-27T21:48:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:08 smithi130 bash[27579]: cluster 2023-08-27T21:48:07.165744+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:10.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:10 smithi006 bash[33918]: cluster 2023-08-27T21:48:09.166224+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:10.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:10 smithi006 bash[27292]: cluster 2023-08-27T21:48:09.166224+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:10 smithi130 bash[27579]: cluster 2023-08-27T21:48:09.166224+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:12.632 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:12 smithi006 bash[27292]: cluster 2023-08-27T21:48:11.166640+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v105: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:12.632 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:12 smithi006 bash[27292]: audit 2023-08-27T21:48:11.592891+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:12.632 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:12 smithi006 bash[27292]: audit 2023-08-27T21:48:11.635967+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:48:12.632 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:12 smithi006 bash[33918]: cluster 2023-08-27T21:48:11.166640+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v105: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:12.632 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:12 smithi006 bash[33918]: audit 2023-08-27T21:48:11.592891+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:12.632 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:12 smithi006 bash[33918]: audit 2023-08-27T21:48:11.635967+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:48:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:12 smithi130 bash[27579]: cluster 2023-08-27T21:48:11.166640+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v105: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:12 smithi130 bash[27579]: audit 2023-08-27T21:48:11.592891+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:12 smithi130 bash[27579]: audit 2023-08-27T21:48:11.635967+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:48:14.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:14 smithi006 bash[27292]: cluster 2023-08-27T21:48:13.166997+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:14.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:14 smithi006 bash[33918]: cluster 2023-08-27T21:48:13.166997+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:14 smithi130 bash[27579]: cluster 2023-08-27T21:48:13.166997+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:16 smithi006 bash[27292]: cluster 2023-08-27T21:48:15.167462+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:16.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:16 smithi006 bash[33918]: cluster 2023-08-27T21:48:15.167462+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:16 smithi130 bash[27579]: cluster 2023-08-27T21:48:15.167462+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:18.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:18 smithi006 bash[33918]: cluster 2023-08-27T21:48:17.167962+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:18.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:18 smithi006 bash[33918]: audit 2023-08-27T21:48:18.081410+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:18.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:18 smithi006 bash[33918]: audit 2023-08-27T21:48:18.082590+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:48:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:18 smithi006 bash[33918]: audit 2023-08-27T21:48:18.083927+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:48:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:18 smithi006 bash[33918]: audit 2023-08-27T21:48:18.090648+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:18.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:18 smithi006 bash[27292]: cluster 2023-08-27T21:48:17.167962+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:18 smithi006 bash[27292]: audit 2023-08-27T21:48:18.081410+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:18 smithi006 bash[27292]: audit 2023-08-27T21:48:18.082590+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:48:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:18 smithi006 bash[27292]: audit 2023-08-27T21:48:18.083927+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:48:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:18 smithi006 bash[27292]: audit 2023-08-27T21:48:18.090648+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:18 smithi130 bash[27579]: cluster 2023-08-27T21:48:17.167962+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:18 smithi130 bash[27579]: audit 2023-08-27T21:48:18.081410+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:18 smithi130 bash[27579]: audit 2023-08-27T21:48:18.082590+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:48:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:18 smithi130 bash[27579]: audit 2023-08-27T21:48:18.083927+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:48:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:18 smithi130 bash[27579]: audit 2023-08-27T21:48:18.090648+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:19.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:19 smithi006 bash[27292]: audit 2023-08-27T21:48:18.523677+0000 mon.c (mon.2) 7 : audit [INF] from='osd.2 [v2:172.21.15.6:6818/3315375049,v1:172.21.15.6:6819/3315375049]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-08-27T21:48:19.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:19 smithi006 bash[27292]: audit 2023-08-27T21:48:18.524299+0000 mon.a (mon.0) 317 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-08-27T21:48:19.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:19 smithi006 bash[33918]: audit 2023-08-27T21:48:18.523677+0000 mon.c (mon.2) 7 : audit [INF] from='osd.2 [v2:172.21.15.6:6818/3315375049,v1:172.21.15.6:6819/3315375049]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-08-27T21:48:19.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:19 smithi006 bash[33918]: audit 2023-08-27T21:48:18.524299+0000 mon.a (mon.0) 317 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-08-27T21:48:19.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:19 smithi130 bash[27579]: audit 2023-08-27T21:48:18.523677+0000 mon.c (mon.2) 7 : audit [INF] from='osd.2 [v2:172.21.15.6:6818/3315375049,v1:172.21.15.6:6819/3315375049]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-08-27T21:48:19.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:19 smithi130 bash[27579]: audit 2023-08-27T21:48:18.524299+0000 mon.a (mon.0) 317 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-08-27T21:48:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:20 smithi006 bash[27292]: cluster 2023-08-27T21:48:19.168421+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:20 smithi006 bash[27292]: audit 2023-08-27T21:48:19.400400+0000 mon.a (mon.0) 318 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-08-27T21:48:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:20 smithi006 bash[27292]: cluster 2023-08-27T21:48:19.400764+0000 mon.a (mon.0) 319 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-08-27T21:48:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:20 smithi006 bash[27292]: audit 2023-08-27T21:48:19.401197+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:48:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:20 smithi006 bash[27292]: audit 2023-08-27T21:48:19.403907+0000 mon.c (mon.2) 8 : audit [INF] from='osd.2 [v2:172.21.15.6:6818/3315375049,v1:172.21.15.6:6819/3315375049]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2023-08-27T21:48:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:20 smithi006 bash[27292]: audit 2023-08-27T21:48:19.404460+0000 mon.a (mon.0) 321 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2023-08-27T21:48:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:20 smithi006 bash[33918]: cluster 2023-08-27T21:48:19.168421+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:20 smithi006 bash[33918]: audit 2023-08-27T21:48:19.400400+0000 mon.a (mon.0) 318 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-08-27T21:48:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:20 smithi006 bash[33918]: cluster 2023-08-27T21:48:19.400764+0000 mon.a (mon.0) 319 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-08-27T21:48:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:20 smithi006 bash[33918]: audit 2023-08-27T21:48:19.401197+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:48:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:20 smithi006 bash[33918]: audit 2023-08-27T21:48:19.403907+0000 mon.c (mon.2) 8 : audit [INF] from='osd.2 [v2:172.21.15.6:6818/3315375049,v1:172.21.15.6:6819/3315375049]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2023-08-27T21:48:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:20 smithi006 bash[33918]: audit 2023-08-27T21:48:19.404460+0000 mon.a (mon.0) 321 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2023-08-27T21:48:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:20 smithi130 bash[27579]: cluster 2023-08-27T21:48:19.168421+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:20 smithi130 bash[27579]: audit 2023-08-27T21:48:19.400400+0000 mon.a (mon.0) 318 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-08-27T21:48:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:20 smithi130 bash[27579]: cluster 2023-08-27T21:48:19.400764+0000 mon.a (mon.0) 319 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-08-27T21:48:20.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:20 smithi130 bash[27579]: audit 2023-08-27T21:48:19.401197+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:48:20.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:20 smithi130 bash[27579]: audit 2023-08-27T21:48:19.403907+0000 mon.c (mon.2) 8 : audit [INF] from='osd.2 [v2:172.21.15.6:6818/3315375049,v1:172.21.15.6:6819/3315375049]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2023-08-27T21:48:20.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:20 smithi130 bash[27579]: audit 2023-08-27T21:48:19.404460+0000 mon.a (mon.0) 321 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2023-08-27T21:48:21.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:21 smithi006 bash[27292]: audit 2023-08-27T21:48:20.406666+0000 mon.a (mon.0) 322 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2023-08-27T21:48:21.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:21 smithi006 bash[27292]: cluster 2023-08-27T21:48:20.406771+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-08-27T21:48:21.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:21 smithi006 bash[27292]: audit 2023-08-27T21:48:20.407374+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:48:21.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:21 smithi006 bash[27292]: audit 2023-08-27T21:48:20.413772+0000 mon.a (mon.0) 325 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:48:21.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:21 smithi006 bash[27292]: cluster 2023-08-27T21:48:21.168962+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:21.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:21 smithi006 bash[33918]: audit 2023-08-27T21:48:20.406666+0000 mon.a (mon.0) 322 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2023-08-27T21:48:21.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:21 smithi006 bash[33918]: cluster 2023-08-27T21:48:20.406771+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-08-27T21:48:21.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:21 smithi006 bash[33918]: audit 2023-08-27T21:48:20.407374+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:48:21.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:21 smithi006 bash[33918]: audit 2023-08-27T21:48:20.413772+0000 mon.a (mon.0) 325 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:48:21.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:21 smithi006 bash[33918]: cluster 2023-08-27T21:48:21.168962+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:21.727 INFO:teuthology.orchestra.run.smithi006.stdout:Created osd(s) 2 on host 'smithi006' 2023-08-27T21:48:21.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:21 smithi130 bash[27579]: audit 2023-08-27T21:48:20.406666+0000 mon.a (mon.0) 322 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2023-08-27T21:48:21.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:21 smithi130 bash[27579]: cluster 2023-08-27T21:48:20.406771+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-08-27T21:48:21.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:21 smithi130 bash[27579]: audit 2023-08-27T21:48:20.407374+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:48:21.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:21 smithi130 bash[27579]: audit 2023-08-27T21:48:20.413772+0000 mon.a (mon.0) 325 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:48:21.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:21 smithi130 bash[27579]: cluster 2023-08-27T21:48:21.168962+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-27T21:48:22.402 DEBUG:teuthology.orchestra.run.smithi006:osd.2> sudo journalctl -f -n 0 -u ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@osd.2.service 2023-08-27T21:48:22.405 INFO:tasks.cephadm:Deploying osd.3 on smithi006 with /dev/vg_nvme/lv_1... 2023-08-27T21:48:22.405 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-08-27T21:48:22.443 INFO:journalctl@ceph.osd.2.smithi006.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-08-27T21:48:22.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:22 smithi006 bash[27292]: cluster 2023-08-27T21:48:19.486207+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:48:22.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:22 smithi006 bash[27292]: cluster 2023-08-27T21:48:19.486334+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:48:22.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:22 smithi006 bash[27292]: audit 2023-08-27T21:48:21.413931+0000 mon.a (mon.0) 326 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:48:22.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:22 smithi006 bash[27292]: cluster 2023-08-27T21:48:21.418933+0000 mon.a (mon.0) 327 : cluster [INF] osd.2 [v2:172.21.15.6:6818/3315375049,v1:172.21.15.6:6819/3315375049] boot 2023-08-27T21:48:22.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:22 smithi006 bash[27292]: cluster 2023-08-27T21:48:21.418967+0000 mon.a (mon.0) 328 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-08-27T21:48:22.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:22 smithi006 bash[27292]: audit 2023-08-27T21:48:21.419177+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:48:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:22 smithi006 bash[27292]: audit 2023-08-27T21:48:21.717602+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:48:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:22 smithi006 bash[27292]: audit 2023-08-27T21:48:21.724864+0000 mon.a (mon.0) 331 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:22 smithi006 bash[27292]: audit 2023-08-27T21:48:21.728532+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-08-27T21:48:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:22 smithi006 bash[33918]: cluster 2023-08-27T21:48:19.486207+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:48:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:22 smithi006 bash[33918]: cluster 2023-08-27T21:48:19.486334+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:48:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:22 smithi006 bash[33918]: audit 2023-08-27T21:48:21.413931+0000 mon.a (mon.0) 326 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:48:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:22 smithi006 bash[33918]: cluster 2023-08-27T21:48:21.418933+0000 mon.a (mon.0) 327 : cluster [INF] osd.2 [v2:172.21.15.6:6818/3315375049,v1:172.21.15.6:6819/3315375049] boot 2023-08-27T21:48:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:22 smithi006 bash[33918]: cluster 2023-08-27T21:48:21.418967+0000 mon.a (mon.0) 328 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-08-27T21:48:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:22 smithi006 bash[33918]: audit 2023-08-27T21:48:21.419177+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:48:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:22 smithi006 bash[33918]: audit 2023-08-27T21:48:21.717602+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:48:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:22 smithi006 bash[33918]: audit 2023-08-27T21:48:21.724864+0000 mon.a (mon.0) 331 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:22 smithi006 bash[33918]: audit 2023-08-27T21:48:21.728532+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-08-27T21:48:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:22 smithi130 bash[27579]: cluster 2023-08-27T21:48:19.486207+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:48:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:22 smithi130 bash[27579]: cluster 2023-08-27T21:48:19.486334+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:48:22.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:22 smithi130 bash[27579]: audit 2023-08-27T21:48:21.413931+0000 mon.a (mon.0) 326 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:48:22.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:22 smithi130 bash[27579]: cluster 2023-08-27T21:48:21.418933+0000 mon.a (mon.0) 327 : cluster [INF] osd.2 [v2:172.21.15.6:6818/3315375049,v1:172.21.15.6:6819/3315375049] boot 2023-08-27T21:48:22.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:22 smithi130 bash[27579]: cluster 2023-08-27T21:48:21.418967+0000 mon.a (mon.0) 328 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-08-27T21:48:22.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:22 smithi130 bash[27579]: audit 2023-08-27T21:48:21.419177+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:48:22.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:22 smithi130 bash[27579]: audit 2023-08-27T21:48:21.717602+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:48:22.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:22 smithi130 bash[27579]: audit 2023-08-27T21:48:21.724864+0000 mon.a (mon.0) 331 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:22.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:22 smithi130 bash[27579]: audit 2023-08-27T21:48:21.728532+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-08-27T21:48:23.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:23 smithi006 bash[27292]: audit 2023-08-27T21:48:22.432066+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-08-27T21:48:23.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:23 smithi006 bash[27292]: cluster 2023-08-27T21:48:22.432129+0000 mon.a (mon.0) 334 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-08-27T21:48:23.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:23 smithi006 bash[27292]: audit 2023-08-27T21:48:22.433110+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-08-27T21:48:23.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:23 smithi006 bash[27292]: cluster 2023-08-27T21:48:23.169418+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v115: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:23.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:23 smithi006 bash[33918]: audit 2023-08-27T21:48:22.432066+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-08-27T21:48:23.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:23 smithi006 bash[33918]: cluster 2023-08-27T21:48:22.432129+0000 mon.a (mon.0) 334 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-08-27T21:48:23.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:23 smithi006 bash[33918]: audit 2023-08-27T21:48:22.433110+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-08-27T21:48:23.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:23 smithi006 bash[33918]: cluster 2023-08-27T21:48:23.169418+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v115: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:23.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:23 smithi130 bash[27579]: audit 2023-08-27T21:48:22.432066+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-08-27T21:48:23.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:23 smithi130 bash[27579]: cluster 2023-08-27T21:48:22.432129+0000 mon.a (mon.0) 334 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-08-27T21:48:23.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:23 smithi130 bash[27579]: audit 2023-08-27T21:48:22.433110+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-08-27T21:48:23.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:23 smithi130 bash[27579]: cluster 2023-08-27T21:48:23.169418+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v115: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:24.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:24 smithi006 bash[27292]: audit 2023-08-27T21:48:23.444315+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-08-27T21:48:24.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:24 smithi006 bash[27292]: cluster 2023-08-27T21:48:23.444474+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-08-27T21:48:24.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:24 smithi006 bash[33918]: audit 2023-08-27T21:48:23.444315+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-08-27T21:48:24.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:24 smithi006 bash[33918]: cluster 2023-08-27T21:48:23.444474+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-08-27T21:48:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:24 smithi130 bash[27579]: audit 2023-08-27T21:48:23.444315+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-08-27T21:48:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:24 smithi130 bash[27579]: cluster 2023-08-27T21:48:23.444474+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-08-27T21:48:25.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:25 smithi006 bash[33918]: cluster 2023-08-27T21:48:24.444794+0000 mon.a (mon.0) 338 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-08-27T21:48:25.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:25 smithi006 bash[33918]: cluster 2023-08-27T21:48:25.170087+0000 mgr.y (mgr.14146) 137 : cluster [DBG] pgmap v118: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:25.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:25 smithi006 bash[27292]: cluster 2023-08-27T21:48:24.444794+0000 mon.a (mon.0) 338 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-08-27T21:48:25.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:25 smithi006 bash[27292]: cluster 2023-08-27T21:48:25.170087+0000 mgr.y (mgr.14146) 137 : cluster [DBG] pgmap v118: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:25.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:25 smithi130 bash[27579]: cluster 2023-08-27T21:48:24.444794+0000 mon.a (mon.0) 338 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-08-27T21:48:25.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:25 smithi130 bash[27579]: cluster 2023-08-27T21:48:25.170087+0000 mgr.y (mgr.14146) 137 : cluster [DBG] pgmap v118: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:26.138 INFO:teuthology.orchestra.run.smithi006.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-08-27T21:48:26.138 INFO:teuthology.orchestra.run.smithi006.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-08-27T21:48:26.138 INFO:teuthology.orchestra.run.smithi006.stderr: stderr: 10+0 records in 2023-08-27T21:48:26.138 INFO:teuthology.orchestra.run.smithi006.stderr:10+0 records out 2023-08-27T21:48:26.138 INFO:teuthology.orchestra.run.smithi006.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0181617 s, 577 MB/s 2023-08-27T21:48:26.138 INFO:teuthology.orchestra.run.smithi006.stderr: stderr: 2023-08-27T21:48:26.139 INFO:teuthology.orchestra.run.smithi006.stderr:--> Zapping successful for: 2023-08-27T21:48:26.839 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph orch daemon add osd smithi006:vg_nvme/lv_1 2023-08-27T21:48:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:28 smithi130 bash[27579]: cluster 2023-08-27T21:48:27.170802+0000 mgr.y (mgr.14146) 138 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:28.672 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:28 smithi006 bash[27292]: cluster 2023-08-27T21:48:27.170802+0000 mgr.y (mgr.14146) 138 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:28.673 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:28 smithi006 bash[33918]: cluster 2023-08-27T21:48:27.170802+0000 mgr.y (mgr.14146) 138 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:30 smithi130 bash[27579]: cluster 2023-08-27T21:48:29.171206+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:30 smithi006 bash[27292]: cluster 2023-08-27T21:48:29.171206+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:30.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:30 smithi006 bash[33918]: cluster 2023-08-27T21:48:29.171206+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:32.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:32 smithi006 bash[27292]: cluster 2023-08-27T21:48:31.171560+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:32.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:32 smithi006 bash[27292]: audit 2023-08-27T21:48:31.802219+0000 mon.a (mon.0) 339 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:48:32.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:32 smithi006 bash[27292]: audit 2023-08-27T21:48:31.807559+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:48:32.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:32 smithi006 bash[27292]: audit 2023-08-27T21:48:31.809007+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:48:32.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:32 smithi006 bash[33918]: cluster 2023-08-27T21:48:31.171560+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:32.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:32 smithi006 bash[33918]: audit 2023-08-27T21:48:31.802219+0000 mon.a (mon.0) 339 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:48:32.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:32 smithi006 bash[33918]: audit 2023-08-27T21:48:31.807559+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:48:32.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:32 smithi006 bash[33918]: audit 2023-08-27T21:48:31.809007+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:48:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:32 smithi130 bash[27579]: cluster 2023-08-27T21:48:31.171560+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:32.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:32 smithi130 bash[27579]: audit 2023-08-27T21:48:31.802219+0000 mon.a (mon.0) 339 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:48:32.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:32 smithi130 bash[27579]: audit 2023-08-27T21:48:31.807559+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:48:32.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:32 smithi130 bash[27579]: audit 2023-08-27T21:48:31.809007+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:48:33.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:33 smithi006 bash[27292]: audit 2023-08-27T21:48:31.795935+0000 mgr.y (mgr.14146) 141 : audit [DBG] from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:48:33.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:33 smithi006 bash[27292]: audit 2023-08-27T21:48:33.133755+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:33.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:33 smithi006 bash[33918]: audit 2023-08-27T21:48:31.795935+0000 mgr.y (mgr.14146) 141 : audit [DBG] from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:48:33.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:33 smithi006 bash[33918]: audit 2023-08-27T21:48:33.133755+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:33.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:33 smithi130 bash[27579]: audit 2023-08-27T21:48:31.795935+0000 mgr.y (mgr.14146) 141 : audit [DBG] from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:48:33.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:33 smithi130 bash[27579]: audit 2023-08-27T21:48:33.133755+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:34 smithi006 bash[27292]: cephadm 2023-08-27T21:48:33.124436+0000 mgr.y (mgr.14146) 142 : cephadm [INF] Detected new or changed devices on smithi006 2023-08-27T21:48:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:34 smithi006 bash[27292]: cluster 2023-08-27T21:48:33.172080+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:34 smithi006 bash[27292]: audit 2023-08-27T21:48:33.432590+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:48:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:34 smithi006 bash[27292]: audit 2023-08-27T21:48:33.434125+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:48:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:34 smithi006 bash[27292]: audit 2023-08-27T21:48:33.435554+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:48:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:34 smithi006 bash[27292]: audit 2023-08-27T21:48:33.443908+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:34 smithi006 bash[27292]: audit 2023-08-27T21:48:33.445699+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:48:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:34 smithi006 bash[27292]: audit 2023-08-27T21:48:33.446874+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:48:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:34 smithi006 bash[27292]: audit 2023-08-27T21:48:33.454977+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:34 smithi006 bash[33918]: cephadm 2023-08-27T21:48:33.124436+0000 mgr.y (mgr.14146) 142 : cephadm [INF] Detected new or changed devices on smithi006 2023-08-27T21:48:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:34 smithi006 bash[33918]: cluster 2023-08-27T21:48:33.172080+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:34 smithi006 bash[33918]: audit 2023-08-27T21:48:33.432590+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:48:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:34 smithi006 bash[33918]: audit 2023-08-27T21:48:33.434125+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:48:34.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:34 smithi006 bash[33918]: audit 2023-08-27T21:48:33.435554+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:48:34.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:34 smithi006 bash[33918]: audit 2023-08-27T21:48:33.443908+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:34.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:34 smithi006 bash[33918]: audit 2023-08-27T21:48:33.445699+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:48:34.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:34 smithi006 bash[33918]: audit 2023-08-27T21:48:33.446874+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:48:34.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:34 smithi006 bash[33918]: audit 2023-08-27T21:48:33.454977+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:34 smithi130 bash[27579]: cephadm 2023-08-27T21:48:33.124436+0000 mgr.y (mgr.14146) 142 : cephadm [INF] Detected new or changed devices on smithi006 2023-08-27T21:48:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:34 smithi130 bash[27579]: cluster 2023-08-27T21:48:33.172080+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:34 smithi130 bash[27579]: audit 2023-08-27T21:48:33.432590+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:48:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:34 smithi130 bash[27579]: audit 2023-08-27T21:48:33.434125+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:48:34.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:34 smithi130 bash[27579]: audit 2023-08-27T21:48:33.435554+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:48:34.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:34 smithi130 bash[27579]: audit 2023-08-27T21:48:33.443908+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:34.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:34 smithi130 bash[27579]: audit 2023-08-27T21:48:33.445699+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:48:34.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:34 smithi130 bash[27579]: audit 2023-08-27T21:48:33.446874+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:48:34.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:34 smithi130 bash[27579]: audit 2023-08-27T21:48:33.454977+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:35.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:35 smithi006 bash[27292]: cephadm 2023-08-27T21:48:33.436700+0000 mgr.y (mgr.14146) 144 : cephadm [INF] Adjusting osd_memory_target on smithi006 to 5415M 2023-08-27T21:48:35.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:35 smithi006 bash[33918]: cephadm 2023-08-27T21:48:33.436700+0000 mgr.y (mgr.14146) 144 : cephadm [INF] Adjusting osd_memory_target on smithi006 to 5415M 2023-08-27T21:48:35.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:35 smithi130 bash[27579]: cephadm 2023-08-27T21:48:33.436700+0000 mgr.y (mgr.14146) 144 : cephadm [INF] Adjusting osd_memory_target on smithi006 to 5415M 2023-08-27T21:48:36.641 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:36 smithi006 bash[27292]: cluster 2023-08-27T21:48:35.172616+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:36.642 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:36 smithi006 bash[33918]: cluster 2023-08-27T21:48:35.172616+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:36 smithi130 bash[27579]: cluster 2023-08-27T21:48:35.172616+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:38.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:38 smithi006 bash[27292]: cluster 2023-08-27T21:48:37.173222+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:38.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:38 smithi006 bash[33918]: cluster 2023-08-27T21:48:37.173222+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:38 smithi130 bash[27579]: cluster 2023-08-27T21:48:37.173222+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:40.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:40 smithi006 bash[27292]: cluster 2023-08-27T21:48:39.173735+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:40.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:40 smithi006 bash[27292]: audit 2023-08-27T21:48:40.368757+0000 mon.a (mon.0) 350 : audit [INF] from='client.? 172.21.15.6:0/2715690455' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "74cd432a-aa78-4538-81bf-a4b209d60f31"}]: dispatch 2023-08-27T21:48:40.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:40 smithi006 bash[27292]: audit 2023-08-27T21:48:40.374892+0000 mon.a (mon.0) 351 : audit [INF] from='client.? 172.21.15.6:0/2715690455' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "74cd432a-aa78-4538-81bf-a4b209d60f31"}]': finished 2023-08-27T21:48:40.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:40 smithi006 bash[27292]: cluster 2023-08-27T21:48:40.375009+0000 mon.a (mon.0) 352 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-08-27T21:48:40.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:40 smithi006 bash[27292]: audit 2023-08-27T21:48:40.375271+0000 mon.a (mon.0) 353 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:48:40.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:40 smithi006 bash[33918]: cluster 2023-08-27T21:48:39.173735+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:40 smithi006 bash[33918]: audit 2023-08-27T21:48:40.368757+0000 mon.a (mon.0) 350 : audit [INF] from='client.? 172.21.15.6:0/2715690455' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "74cd432a-aa78-4538-81bf-a4b209d60f31"}]: dispatch 2023-08-27T21:48:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:40 smithi006 bash[33918]: audit 2023-08-27T21:48:40.374892+0000 mon.a (mon.0) 351 : audit [INF] from='client.? 172.21.15.6:0/2715690455' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "74cd432a-aa78-4538-81bf-a4b209d60f31"}]': finished 2023-08-27T21:48:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:40 smithi006 bash[33918]: cluster 2023-08-27T21:48:40.375009+0000 mon.a (mon.0) 352 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-08-27T21:48:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:40 smithi006 bash[33918]: audit 2023-08-27T21:48:40.375271+0000 mon.a (mon.0) 353 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:48:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:40 smithi130 bash[27579]: cluster 2023-08-27T21:48:39.173735+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:40 smithi130 bash[27579]: audit 2023-08-27T21:48:40.368757+0000 mon.a (mon.0) 350 : audit [INF] from='client.? 172.21.15.6:0/2715690455' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "74cd432a-aa78-4538-81bf-a4b209d60f31"}]: dispatch 2023-08-27T21:48:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:40 smithi130 bash[27579]: audit 2023-08-27T21:48:40.374892+0000 mon.a (mon.0) 351 : audit [INF] from='client.? 172.21.15.6:0/2715690455' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "74cd432a-aa78-4538-81bf-a4b209d60f31"}]': finished 2023-08-27T21:48:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:40 smithi130 bash[27579]: cluster 2023-08-27T21:48:40.375009+0000 mon.a (mon.0) 352 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-08-27T21:48:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:40 smithi130 bash[27579]: audit 2023-08-27T21:48:40.375271+0000 mon.a (mon.0) 353 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:48:41.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:41 smithi006 bash[27292]: audit 2023-08-27T21:48:41.093019+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.6:0/2376668329' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:48:41.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:41 smithi006 bash[33918]: audit 2023-08-27T21:48:41.093019+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.6:0/2376668329' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:48:41.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:41 smithi130 bash[27579]: audit 2023-08-27T21:48:41.093019+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.6:0/2376668329' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:48:42.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:42 smithi006 bash[33918]: cluster 2023-08-27T21:48:41.174253+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:42.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:42 smithi006 bash[27292]: cluster 2023-08-27T21:48:41.174253+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:42 smithi130 bash[27579]: cluster 2023-08-27T21:48:41.174253+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:43.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:43 smithi006 bash[33918]: cluster 2023-08-27T21:48:43.174793+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:43.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:43 smithi006 bash[27292]: cluster 2023-08-27T21:48:43.174793+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:43.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:43 smithi130 bash[27579]: cluster 2023-08-27T21:48:43.174793+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:46 smithi130 bash[27579]: cluster 2023-08-27T21:48:45.175325+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:46.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:46 smithi006 bash[27292]: cluster 2023-08-27T21:48:45.175325+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:46.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:46 smithi006 bash[33918]: cluster 2023-08-27T21:48:45.175325+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:48 smithi130 bash[27579]: cluster 2023-08-27T21:48:47.175878+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:48.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:48 smithi006 bash[27292]: cluster 2023-08-27T21:48:47.175878+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:48.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:48 smithi006 bash[33918]: cluster 2023-08-27T21:48:47.175878+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:50.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:50 smithi006 bash[27292]: cluster 2023-08-27T21:48:49.176481+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:50.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:50 smithi006 bash[33918]: cluster 2023-08-27T21:48:49.176481+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:50 smithi130 bash[27579]: cluster 2023-08-27T21:48:49.176481+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:52.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:52 smithi006 bash[27292]: cluster 2023-08-27T21:48:51.177091+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:52.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:52 smithi006 bash[33918]: cluster 2023-08-27T21:48:51.177091+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:52 smithi130 bash[27579]: cluster 2023-08-27T21:48:51.177091+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:53.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:53 smithi006 bash[33918]: audit 2023-08-27T21:48:53.062035+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-08-27T21:48:53.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:53 smithi006 bash[33918]: audit 2023-08-27T21:48:53.063531+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:48:53.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:53 smithi006 bash[27292]: audit 2023-08-27T21:48:53.062035+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-08-27T21:48:53.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:53 smithi006 bash[27292]: audit 2023-08-27T21:48:53.063531+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:48:53.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:53 smithi130 bash[27579]: audit 2023-08-27T21:48:53.062035+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-08-27T21:48:53.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:53 smithi130 bash[27579]: audit 2023-08-27T21:48:53.063531+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:48:54.590 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:54 smithi006 bash[33918]: cephadm 2023-08-27T21:48:53.064695+0000 mgr.y (mgr.14146) 154 : cephadm [INF] Deploying daemon osd.3 on smithi006 2023-08-27T21:48:54.590 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:54 smithi006 bash[33918]: cluster 2023-08-27T21:48:53.177589+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:54.590 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:54 smithi006 bash[27292]: cephadm 2023-08-27T21:48:53.064695+0000 mgr.y (mgr.14146) 154 : cephadm [INF] Deploying daemon osd.3 on smithi006 2023-08-27T21:48:54.590 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:54 smithi006 bash[27292]: cluster 2023-08-27T21:48:53.177589+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:54 smithi130 bash[27579]: cephadm 2023-08-27T21:48:53.064695+0000 mgr.y (mgr.14146) 154 : cephadm [INF] Deploying daemon osd.3 on smithi006 2023-08-27T21:48:54.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:54 smithi130 bash[27579]: cluster 2023-08-27T21:48:53.177589+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:56.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:56 smithi006 bash[27292]: cluster 2023-08-27T21:48:55.178126+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:56.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:56 smithi006 bash[33918]: cluster 2023-08-27T21:48:55.178126+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:56.872 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:56 smithi130 bash[27579]: cluster 2023-08-27T21:48:55.178126+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:58.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:58 smithi006 bash[27292]: cluster 2023-08-27T21:48:57.178672+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:58.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:58 smithi006 bash[27292]: audit 2023-08-27T21:48:57.616927+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:58.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:48:58 smithi006 bash[27292]: audit 2023-08-27T21:48:57.633921+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:48:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:58 smithi006 bash[33918]: cluster 2023-08-27T21:48:57.178672+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:58 smithi006 bash[33918]: audit 2023-08-27T21:48:57.616927+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:48:58 smithi006 bash[33918]: audit 2023-08-27T21:48:57.633921+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:48:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:58 smithi130 bash[27579]: cluster 2023-08-27T21:48:57.178672+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:48:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:58 smithi130 bash[27579]: audit 2023-08-27T21:48:57.616927+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:48:58.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:48:58 smithi130 bash[27579]: audit 2023-08-27T21:48:57.633921+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:49:00.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:00 smithi006 bash[33918]: cluster 2023-08-27T21:48:59.179209+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:49:00.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:00 smithi006 bash[27292]: cluster 2023-08-27T21:48:59.179209+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:49:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:00 smithi130 bash[27579]: cluster 2023-08-27T21:48:59.179209+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:49:02.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:02 smithi006 bash[27292]: cluster 2023-08-27T21:49:01.179786+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:49:02.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:02 smithi006 bash[33918]: cluster 2023-08-27T21:49:01.179786+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:49:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:02 smithi130 bash[27579]: cluster 2023-08-27T21:49:01.179786+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:49:04.683 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:04 smithi006 bash[33918]: cluster 2023-08-27T21:49:03.180403+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:49:04.683 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:04 smithi006 bash[33918]: audit 2023-08-27T21:49:04.023391+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:04.683 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:04 smithi006 bash[33918]: audit 2023-08-27T21:49:04.024462+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:49:04.683 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:04 smithi006 bash[33918]: audit 2023-08-27T21:49:04.025450+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:49:04.683 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:04 smithi006 bash[33918]: audit 2023-08-27T21:49:04.032224+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:04.683 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:04 smithi006 bash[27292]: cluster 2023-08-27T21:49:03.180403+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:49:04.683 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:04 smithi006 bash[27292]: audit 2023-08-27T21:49:04.023391+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:04.684 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:04 smithi006 bash[27292]: audit 2023-08-27T21:49:04.024462+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:49:04.684 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:04 smithi006 bash[27292]: audit 2023-08-27T21:49:04.025450+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:49:04.684 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:04 smithi006 bash[27292]: audit 2023-08-27T21:49:04.032224+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:04 smithi130 bash[27579]: cluster 2023-08-27T21:49:03.180403+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:49:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:04 smithi130 bash[27579]: audit 2023-08-27T21:49:04.023391+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:04 smithi130 bash[27579]: audit 2023-08-27T21:49:04.024462+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:49:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:04 smithi130 bash[27579]: audit 2023-08-27T21:49:04.025450+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:49:04.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:04 smithi130 bash[27579]: audit 2023-08-27T21:49:04.032224+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:05.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:05 smithi006 bash[27292]: audit 2023-08-27T21:49:04.547917+0000 mon.a (mon.0) 362 : audit [INF] from='osd.3 [v2:172.21.15.6:6826/1657061419,v1:172.21.15.6:6827/1657061419]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-08-27T21:49:05.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:05 smithi006 bash[27292]: cluster 2023-08-27T21:49:05.181014+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:49:05.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:05 smithi006 bash[33918]: audit 2023-08-27T21:49:04.547917+0000 mon.a (mon.0) 362 : audit [INF] from='osd.3 [v2:172.21.15.6:6826/1657061419,v1:172.21.15.6:6827/1657061419]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-08-27T21:49:05.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:05 smithi006 bash[33918]: cluster 2023-08-27T21:49:05.181014+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:49:05.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:05 smithi130 bash[27579]: audit 2023-08-27T21:49:04.547917+0000 mon.a (mon.0) 362 : audit [INF] from='osd.3 [v2:172.21.15.6:6826/1657061419,v1:172.21.15.6:6827/1657061419]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-08-27T21:49:05.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:05 smithi130 bash[27579]: cluster 2023-08-27T21:49:05.181014+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:49:06.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:06 smithi006 bash[27292]: audit 2023-08-27T21:49:05.425933+0000 mon.a (mon.0) 363 : audit [INF] from='osd.3 [v2:172.21.15.6:6826/1657061419,v1:172.21.15.6:6827/1657061419]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-08-27T21:49:06.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:06 smithi006 bash[27292]: cluster 2023-08-27T21:49:05.426064+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-08-27T21:49:06.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:06 smithi006 bash[27292]: audit 2023-08-27T21:49:05.426280+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:49:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:06 smithi006 bash[27292]: audit 2023-08-27T21:49:05.426765+0000 mon.a (mon.0) 366 : audit [INF] from='osd.3 [v2:172.21.15.6:6826/1657061419,v1:172.21.15.6:6827/1657061419]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2023-08-27T21:49:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:06 smithi006 bash[33918]: audit 2023-08-27T21:49:05.425933+0000 mon.a (mon.0) 363 : audit [INF] from='osd.3 [v2:172.21.15.6:6826/1657061419,v1:172.21.15.6:6827/1657061419]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-08-27T21:49:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:06 smithi006 bash[33918]: cluster 2023-08-27T21:49:05.426064+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-08-27T21:49:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:06 smithi006 bash[33918]: audit 2023-08-27T21:49:05.426280+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:49:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:06 smithi006 bash[33918]: audit 2023-08-27T21:49:05.426765+0000 mon.a (mon.0) 366 : audit [INF] from='osd.3 [v2:172.21.15.6:6826/1657061419,v1:172.21.15.6:6827/1657061419]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2023-08-27T21:49:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:06 smithi130 bash[27579]: audit 2023-08-27T21:49:05.425933+0000 mon.a (mon.0) 363 : audit [INF] from='osd.3 [v2:172.21.15.6:6826/1657061419,v1:172.21.15.6:6827/1657061419]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-08-27T21:49:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:06 smithi130 bash[27579]: cluster 2023-08-27T21:49:05.426064+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-08-27T21:49:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:06 smithi130 bash[27579]: audit 2023-08-27T21:49:05.426280+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:49:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:06 smithi130 bash[27579]: audit 2023-08-27T21:49:05.426765+0000 mon.a (mon.0) 366 : audit [INF] from='osd.3 [v2:172.21.15.6:6826/1657061419,v1:172.21.15.6:6827/1657061419]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2023-08-27T21:49:07.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:07 smithi006 bash[27292]: audit 2023-08-27T21:49:06.430541+0000 mon.a (mon.0) 367 : audit [INF] from='osd.3 [v2:172.21.15.6:6826/1657061419,v1:172.21.15.6:6827/1657061419]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2023-08-27T21:49:07.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:07 smithi006 bash[27292]: cluster 2023-08-27T21:49:06.430751+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-08-27T21:49:07.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:07 smithi006 bash[27292]: audit 2023-08-27T21:49:06.431442+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:49:07.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:07 smithi006 bash[27292]: audit 2023-08-27T21:49:06.437343+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:49:07.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:07 smithi006 bash[27292]: cluster 2023-08-27T21:49:07.181452+0000 mgr.y (mgr.14146) 162 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:49:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:07 smithi006 bash[33918]: audit 2023-08-27T21:49:06.430541+0000 mon.a (mon.0) 367 : audit [INF] from='osd.3 [v2:172.21.15.6:6826/1657061419,v1:172.21.15.6:6827/1657061419]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2023-08-27T21:49:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:07 smithi006 bash[33918]: cluster 2023-08-27T21:49:06.430751+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-08-27T21:49:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:07 smithi006 bash[33918]: audit 2023-08-27T21:49:06.431442+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:49:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:07 smithi006 bash[33918]: audit 2023-08-27T21:49:06.437343+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:49:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:07 smithi006 bash[33918]: cluster 2023-08-27T21:49:07.181452+0000 mgr.y (mgr.14146) 162 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:49:07.827 INFO:teuthology.orchestra.run.smithi006.stdout:Created osd(s) 3 on host 'smithi006' 2023-08-27T21:49:07.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:07 smithi130 bash[27579]: audit 2023-08-27T21:49:06.430541+0000 mon.a (mon.0) 367 : audit [INF] from='osd.3 [v2:172.21.15.6:6826/1657061419,v1:172.21.15.6:6827/1657061419]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2023-08-27T21:49:07.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:07 smithi130 bash[27579]: cluster 2023-08-27T21:49:06.430751+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-08-27T21:49:07.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:07 smithi130 bash[27579]: audit 2023-08-27T21:49:06.431442+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:49:07.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:07 smithi130 bash[27579]: audit 2023-08-27T21:49:06.437343+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:49:07.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:07 smithi130 bash[27579]: cluster 2023-08-27T21:49:07.181452+0000 mgr.y (mgr.14146) 162 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-27T21:49:08.580 DEBUG:teuthology.orchestra.run.smithi006:osd.3> sudo journalctl -f -n 0 -u ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@osd.3.service 2023-08-27T21:49:08.583 INFO:tasks.cephadm:Deploying osd.4 on smithi130 with /dev/vg_nvme/lv_4... 2023-08-27T21:49:08.583 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-08-27T21:49:08.618 INFO:journalctl@ceph.osd.3.smithi006.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-08-27T21:49:08.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:08 smithi006 bash[27292]: cluster 2023-08-27T21:49:05.565600+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:49:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:08 smithi006 bash[27292]: cluster 2023-08-27T21:49:05.565696+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:49:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:08 smithi006 bash[27292]: audit 2023-08-27T21:49:07.439671+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:49:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:08 smithi006 bash[27292]: cluster 2023-08-27T21:49:07.446077+0000 mon.a (mon.0) 372 : cluster [INF] osd.3 [v2:172.21.15.6:6826/1657061419,v1:172.21.15.6:6827/1657061419] boot 2023-08-27T21:49:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:08 smithi006 bash[27292]: cluster 2023-08-27T21:49:07.446121+0000 mon.a (mon.0) 373 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-08-27T21:49:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:08 smithi006 bash[27292]: audit 2023-08-27T21:49:07.446944+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:49:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:08 smithi006 bash[27292]: audit 2023-08-27T21:49:07.824648+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:08 smithi006 bash[27292]: audit 2023-08-27T21:49:07.859627+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:49:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:08 smithi006 bash[33918]: cluster 2023-08-27T21:49:05.565600+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:49:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:08 smithi006 bash[33918]: cluster 2023-08-27T21:49:05.565696+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:49:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:08 smithi006 bash[33918]: audit 2023-08-27T21:49:07.439671+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:49:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:08 smithi006 bash[33918]: cluster 2023-08-27T21:49:07.446077+0000 mon.a (mon.0) 372 : cluster [INF] osd.3 [v2:172.21.15.6:6826/1657061419,v1:172.21.15.6:6827/1657061419] boot 2023-08-27T21:49:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:08 smithi006 bash[33918]: cluster 2023-08-27T21:49:07.446121+0000 mon.a (mon.0) 373 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-08-27T21:49:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:08 smithi006 bash[33918]: audit 2023-08-27T21:49:07.446944+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:49:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:08 smithi006 bash[33918]: audit 2023-08-27T21:49:07.824648+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:08 smithi006 bash[33918]: audit 2023-08-27T21:49:07.859627+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:49:08.784 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:08 smithi130 bash[27579]: cluster 2023-08-27T21:49:05.565600+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:49:08.784 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:08 smithi130 bash[27579]: cluster 2023-08-27T21:49:05.565696+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:49:08.784 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:08 smithi130 bash[27579]: audit 2023-08-27T21:49:07.439671+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:49:08.784 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:08 smithi130 bash[27579]: cluster 2023-08-27T21:49:07.446077+0000 mon.a (mon.0) 372 : cluster [INF] osd.3 [v2:172.21.15.6:6826/1657061419,v1:172.21.15.6:6827/1657061419] boot 2023-08-27T21:49:08.784 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:08 smithi130 bash[27579]: cluster 2023-08-27T21:49:07.446121+0000 mon.a (mon.0) 373 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-08-27T21:49:08.784 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:08 smithi130 bash[27579]: audit 2023-08-27T21:49:07.446944+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:49:08.785 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:08 smithi130 bash[27579]: audit 2023-08-27T21:49:07.824648+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:08.785 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:08 smithi130 bash[27579]: audit 2023-08-27T21:49:07.859627+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:49:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:09 smithi006 bash[27292]: cluster 2023-08-27T21:49:08.446872+0000 mon.a (mon.0) 377 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-08-27T21:49:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:09 smithi006 bash[27292]: cluster 2023-08-27T21:49:09.181986+0000 mgr.y (mgr.14146) 163 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:09.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:09 smithi006 bash[33918]: cluster 2023-08-27T21:49:08.446872+0000 mon.a (mon.0) 377 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-08-27T21:49:09.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:09 smithi006 bash[33918]: cluster 2023-08-27T21:49:09.181986+0000 mgr.y (mgr.14146) 163 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:09 smithi130 bash[27579]: cluster 2023-08-27T21:49:08.446872+0000 mon.a (mon.0) 377 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-08-27T21:49:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:09 smithi130 bash[27579]: cluster 2023-08-27T21:49:09.181986+0000 mgr.y (mgr.14146) 163 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:12.420 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-08-27T21:49:12.420 INFO:teuthology.orchestra.run.smithi130.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-08-27T21:49:12.420 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10+0 records in 2023-08-27T21:49:12.420 INFO:teuthology.orchestra.run.smithi130.stderr:10+0 records out 2023-08-27T21:49:12.421 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0170362 s, 615 MB/s 2023-08-27T21:49:12.421 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping successful for: 2023-08-27T21:49:12.494 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:12 smithi130 bash[27579]: cluster 2023-08-27T21:49:11.182548+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:12.691 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:12 smithi006 bash[33918]: cluster 2023-08-27T21:49:11.182548+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:12.691 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:12 smithi006 bash[27292]: cluster 2023-08-27T21:49:11.182548+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:13.117 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph orch daemon add osd smithi130:vg_nvme/lv_4 2023-08-27T21:49:14.485 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:14 smithi130 bash[27579]: cluster 2023-08-27T21:49:13.183090+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:14.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:14 smithi006 bash[27292]: cluster 2023-08-27T21:49:13.183090+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:14 smithi006 bash[33918]: cluster 2023-08-27T21:49:13.183090+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:16.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:16 smithi006 bash[27292]: cluster 2023-08-27T21:49:15.183633+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:16 smithi006 bash[27292]: audit 2023-08-27T21:49:16.253803+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:49:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:16 smithi006 bash[27292]: audit 2023-08-27T21:49:16.258934+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:49:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:16 smithi006 bash[27292]: audit 2023-08-27T21:49:16.260085+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:49:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:16 smithi006 bash[33918]: cluster 2023-08-27T21:49:15.183633+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:16 smithi006 bash[33918]: audit 2023-08-27T21:49:16.253803+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:49:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:16 smithi006 bash[33918]: audit 2023-08-27T21:49:16.258934+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:49:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:16 smithi006 bash[33918]: audit 2023-08-27T21:49:16.260085+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:49:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:16 smithi130 bash[27579]: cluster 2023-08-27T21:49:15.183633+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:16.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:16 smithi130 bash[27579]: audit 2023-08-27T21:49:16.253803+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:49:16.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:16 smithi130 bash[27579]: audit 2023-08-27T21:49:16.258934+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:49:16.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:16 smithi130 bash[27579]: audit 2023-08-27T21:49:16.260085+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:49:17.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:17 smithi006 bash[27292]: audit 2023-08-27T21:49:16.248116+0000 mgr.y (mgr.14146) 167 : audit [DBG] from='client.24217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:49:17.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:17 smithi006 bash[27292]: audit 2023-08-27T21:49:17.274474+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:17.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:17 smithi006 bash[27292]: audit 2023-08-27T21:49:17.276160+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:49:17.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:17 smithi006 bash[27292]: audit 2023-08-27T21:49:17.277412+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:49:17.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:17 smithi006 bash[27292]: audit 2023-08-27T21:49:17.278536+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:49:17.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:17 smithi006 bash[27292]: audit 2023-08-27T21:49:17.279561+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:49:17.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:17 smithi006 bash[27292]: audit 2023-08-27T21:49:17.284979+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:17.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:17 smithi006 bash[27292]: audit 2023-08-27T21:49:17.286725+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:49:17.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:17 smithi006 bash[27292]: audit 2023-08-27T21:49:17.287766+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:49:17.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:17 smithi006 bash[27292]: audit 2023-08-27T21:49:17.294904+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:17.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:17 smithi006 bash[33918]: audit 2023-08-27T21:49:16.248116+0000 mgr.y (mgr.14146) 167 : audit [DBG] from='client.24217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:49:17.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:17 smithi006 bash[33918]: audit 2023-08-27T21:49:17.274474+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:17.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:17 smithi006 bash[33918]: audit 2023-08-27T21:49:17.276160+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:49:17.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:17 smithi006 bash[33918]: audit 2023-08-27T21:49:17.277412+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:49:17.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:17 smithi006 bash[33918]: audit 2023-08-27T21:49:17.278536+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:49:17.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:17 smithi006 bash[33918]: audit 2023-08-27T21:49:17.279561+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:49:17.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:17 smithi006 bash[33918]: audit 2023-08-27T21:49:17.284979+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:17.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:17 smithi006 bash[33918]: audit 2023-08-27T21:49:17.286725+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:49:17.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:17 smithi006 bash[33918]: audit 2023-08-27T21:49:17.287766+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:49:17.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:17 smithi006 bash[33918]: audit 2023-08-27T21:49:17.294904+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:17.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:17 smithi130 bash[27579]: audit 2023-08-27T21:49:16.248116+0000 mgr.y (mgr.14146) 167 : audit [DBG] from='client.24217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:49:17.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:17 smithi130 bash[27579]: audit 2023-08-27T21:49:17.274474+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:17.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:17 smithi130 bash[27579]: audit 2023-08-27T21:49:17.276160+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:49:17.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:17 smithi130 bash[27579]: audit 2023-08-27T21:49:17.277412+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:49:17.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:17 smithi130 bash[27579]: audit 2023-08-27T21:49:17.278536+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:49:17.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:17 smithi130 bash[27579]: audit 2023-08-27T21:49:17.279561+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:49:17.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:17 smithi130 bash[27579]: audit 2023-08-27T21:49:17.284979+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:17.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:17 smithi130 bash[27579]: audit 2023-08-27T21:49:17.286725+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:49:17.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:17 smithi130 bash[27579]: audit 2023-08-27T21:49:17.287766+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:49:17.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:17 smithi130 bash[27579]: audit 2023-08-27T21:49:17.294904+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:18.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:18 smithi006 bash[27292]: cluster 2023-08-27T21:49:17.184229+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:18.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:18 smithi006 bash[27292]: cephadm 2023-08-27T21:49:17.264336+0000 mgr.y (mgr.14146) 169 : cephadm [INF] Detected new or changed devices on smithi006 2023-08-27T21:49:18.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:18 smithi006 bash[27292]: cephadm 2023-08-27T21:49:17.280331+0000 mgr.y (mgr.14146) 170 : cephadm [INF] Adjusting osd_memory_target on smithi006 to 4061M 2023-08-27T21:49:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:18 smithi006 bash[33918]: cluster 2023-08-27T21:49:17.184229+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:18 smithi006 bash[33918]: cephadm 2023-08-27T21:49:17.264336+0000 mgr.y (mgr.14146) 169 : cephadm [INF] Detected new or changed devices on smithi006 2023-08-27T21:49:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:18 smithi006 bash[33918]: cephadm 2023-08-27T21:49:17.280331+0000 mgr.y (mgr.14146) 170 : cephadm [INF] Adjusting osd_memory_target on smithi006 to 4061M 2023-08-27T21:49:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:18 smithi130 bash[27579]: cluster 2023-08-27T21:49:17.184229+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:18.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:18 smithi130 bash[27579]: cephadm 2023-08-27T21:49:17.264336+0000 mgr.y (mgr.14146) 169 : cephadm [INF] Detected new or changed devices on smithi006 2023-08-27T21:49:18.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:18 smithi130 bash[27579]: cephadm 2023-08-27T21:49:17.280331+0000 mgr.y (mgr.14146) 170 : cephadm [INF] Adjusting osd_memory_target on smithi006 to 4061M 2023-08-27T21:49:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:20 smithi006 bash[27292]: cluster 2023-08-27T21:49:19.184928+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:20.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:20 smithi006 bash[33918]: cluster 2023-08-27T21:49:19.184928+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:20 smithi130 bash[27579]: cluster 2023-08-27T21:49:19.184928+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:22.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:22 smithi006 bash[27292]: cluster 2023-08-27T21:49:21.185602+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:22 smithi006 bash[33918]: cluster 2023-08-27T21:49:21.185602+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:22 smithi130 bash[27579]: cluster 2023-08-27T21:49:21.185602+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:24.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:24 smithi006 bash[27292]: cluster 2023-08-27T21:49:23.186320+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:24.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:24 smithi006 bash[33918]: cluster 2023-08-27T21:49:23.186320+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:24 smithi130 bash[27579]: cluster 2023-08-27T21:49:23.186320+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:25.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:25 smithi006 bash[33918]: audit 2023-08-27T21:49:24.736382+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.130:0/3722501718' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5710ffb1-d311-45a8-99bf-d01a7bb65cd1"}]: dispatch 2023-08-27T21:49:25.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:25 smithi006 bash[33918]: audit 2023-08-27T21:49:24.737312+0000 mon.a (mon.0) 390 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5710ffb1-d311-45a8-99bf-d01a7bb65cd1"}]: dispatch 2023-08-27T21:49:25.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:25 smithi006 bash[33918]: audit 2023-08-27T21:49:24.746053+0000 mon.a (mon.0) 391 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5710ffb1-d311-45a8-99bf-d01a7bb65cd1"}]': finished 2023-08-27T21:49:25.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:25 smithi006 bash[33918]: cluster 2023-08-27T21:49:24.746151+0000 mon.a (mon.0) 392 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-08-27T21:49:25.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:25 smithi006 bash[33918]: audit 2023-08-27T21:49:24.746411+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:49:25.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:25 smithi006 bash[27292]: audit 2023-08-27T21:49:24.736382+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.130:0/3722501718' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5710ffb1-d311-45a8-99bf-d01a7bb65cd1"}]: dispatch 2023-08-27T21:49:25.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:25 smithi006 bash[27292]: audit 2023-08-27T21:49:24.737312+0000 mon.a (mon.0) 390 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5710ffb1-d311-45a8-99bf-d01a7bb65cd1"}]: dispatch 2023-08-27T21:49:25.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:25 smithi006 bash[27292]: audit 2023-08-27T21:49:24.746053+0000 mon.a (mon.0) 391 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5710ffb1-d311-45a8-99bf-d01a7bb65cd1"}]': finished 2023-08-27T21:49:25.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:25 smithi006 bash[27292]: cluster 2023-08-27T21:49:24.746151+0000 mon.a (mon.0) 392 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-08-27T21:49:25.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:25 smithi006 bash[27292]: audit 2023-08-27T21:49:24.746411+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:49:25.851 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:25 smithi130 bash[27579]: audit 2023-08-27T21:49:24.736382+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.130:0/3722501718' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5710ffb1-d311-45a8-99bf-d01a7bb65cd1"}]: dispatch 2023-08-27T21:49:25.851 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:25 smithi130 bash[27579]: audit 2023-08-27T21:49:24.737312+0000 mon.a (mon.0) 390 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5710ffb1-d311-45a8-99bf-d01a7bb65cd1"}]: dispatch 2023-08-27T21:49:25.851 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:25 smithi130 bash[27579]: audit 2023-08-27T21:49:24.746053+0000 mon.a (mon.0) 391 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5710ffb1-d311-45a8-99bf-d01a7bb65cd1"}]': finished 2023-08-27T21:49:25.851 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:25 smithi130 bash[27579]: cluster 2023-08-27T21:49:24.746151+0000 mon.a (mon.0) 392 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-08-27T21:49:25.851 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:25 smithi130 bash[27579]: audit 2023-08-27T21:49:24.746411+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:49:26.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:26 smithi006 bash[27292]: cluster 2023-08-27T21:49:25.186873+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:26 smithi006 bash[27292]: audit 2023-08-27T21:49:25.441944+0000 mon.b (mon.1) 9 : audit [DBG] from='client.? 172.21.15.130:0/2819507546' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:49:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:26 smithi006 bash[33918]: cluster 2023-08-27T21:49:25.186873+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:26 smithi006 bash[33918]: audit 2023-08-27T21:49:25.441944+0000 mon.b (mon.1) 9 : audit [DBG] from='client.? 172.21.15.130:0/2819507546' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:49:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:26 smithi130 bash[27579]: cluster 2023-08-27T21:49:25.186873+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:26.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:26 smithi130 bash[27579]: audit 2023-08-27T21:49:25.441944+0000 mon.b (mon.1) 9 : audit [DBG] from='client.? 172.21.15.130:0/2819507546' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:49:28.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:28 smithi006 bash[27292]: cluster 2023-08-27T21:49:27.187515+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:28 smithi006 bash[33918]: cluster 2023-08-27T21:49:27.187515+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:28 smithi130 bash[27579]: cluster 2023-08-27T21:49:27.187515+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:30 smithi006 bash[27292]: cluster 2023-08-27T21:49:29.188099+0000 mgr.y (mgr.14146) 176 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:30 smithi006 bash[33918]: cluster 2023-08-27T21:49:29.188099+0000 mgr.y (mgr.14146) 176 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:30 smithi130 bash[27579]: cluster 2023-08-27T21:49:29.188099+0000 mgr.y (mgr.14146) 176 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:31.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:31 smithi130 bash[27579]: cluster 2023-08-27T21:49:31.188629+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:31.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:31 smithi006 bash[27292]: cluster 2023-08-27T21:49:31.188629+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:31.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:31 smithi006 bash[33918]: cluster 2023-08-27T21:49:31.188629+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:34.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:34 smithi006 bash[27292]: cluster 2023-08-27T21:49:33.189266+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:34 smithi006 bash[33918]: cluster 2023-08-27T21:49:33.189266+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:34.756 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:34 smithi130 bash[27579]: cluster 2023-08-27T21:49:33.189266+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:36 smithi130 bash[27579]: cluster 2023-08-27T21:49:35.189831+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:36.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:36 smithi006 bash[33918]: cluster 2023-08-27T21:49:35.189831+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:36 smithi006 bash[27292]: cluster 2023-08-27T21:49:35.189831+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:38.655 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:38 smithi130 bash[27579]: cluster 2023-08-27T21:49:37.190351+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:38.655 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:38 smithi130 bash[27579]: audit 2023-08-27T21:49:37.413579+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-08-27T21:49:38.655 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:38 smithi130 bash[27579]: audit 2023-08-27T21:49:37.415212+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:49:38.655 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:38 smithi130 bash[27579]: cephadm 2023-08-27T21:49:37.416618+0000 mgr.y (mgr.14146) 181 : cephadm [INF] Deploying daemon osd.4 on smithi130 2023-08-27T21:49:38.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:38 smithi006 bash[27292]: cluster 2023-08-27T21:49:37.190351+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:38.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:38 smithi006 bash[27292]: audit 2023-08-27T21:49:37.413579+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-08-27T21:49:38.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:38 smithi006 bash[27292]: audit 2023-08-27T21:49:37.415212+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:49:38.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:38 smithi006 bash[27292]: cephadm 2023-08-27T21:49:37.416618+0000 mgr.y (mgr.14146) 181 : cephadm [INF] Deploying daemon osd.4 on smithi130 2023-08-27T21:49:38.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:38 smithi006 bash[33918]: cluster 2023-08-27T21:49:37.190351+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:38.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:38 smithi006 bash[33918]: audit 2023-08-27T21:49:37.413579+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-08-27T21:49:38.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:38 smithi006 bash[33918]: audit 2023-08-27T21:49:37.415212+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:49:38.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:38 smithi006 bash[33918]: cephadm 2023-08-27T21:49:37.416618+0000 mgr.y (mgr.14146) 181 : cephadm [INF] Deploying daemon osd.4 on smithi130 2023-08-27T21:49:40.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:40 smithi130 bash[27579]: cluster 2023-08-27T21:49:39.190858+0000 mgr.y (mgr.14146) 182 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:40 smithi006 bash[27292]: cluster 2023-08-27T21:49:39.190858+0000 mgr.y (mgr.14146) 182 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:40 smithi006 bash[33918]: cluster 2023-08-27T21:49:39.190858+0000 mgr.y (mgr.14146) 182 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:42 smithi006 bash[27292]: cluster 2023-08-27T21:49:41.191403+0000 mgr.y (mgr.14146) 183 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:42 smithi006 bash[27292]: audit 2023-08-27T21:49:41.801613+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:42 smithi006 bash[27292]: audit 2023-08-27T21:49:41.829395+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:49:42.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:42 smithi006 bash[33918]: cluster 2023-08-27T21:49:41.191403+0000 mgr.y (mgr.14146) 183 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:42.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:42 smithi006 bash[33918]: audit 2023-08-27T21:49:41.801613+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:42.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:42 smithi006 bash[33918]: audit 2023-08-27T21:49:41.829395+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:49:42.766 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:42 smithi130 bash[27579]: cluster 2023-08-27T21:49:41.191403+0000 mgr.y (mgr.14146) 183 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:42.766 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:42 smithi130 bash[27579]: audit 2023-08-27T21:49:41.801613+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:42.766 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:42 smithi130 bash[27579]: audit 2023-08-27T21:49:41.829395+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:49:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:44 smithi130 bash[27579]: cluster 2023-08-27T21:49:43.192044+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:44.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:44 smithi006 bash[27292]: cluster 2023-08-27T21:49:43.192044+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:44.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:44 smithi006 bash[33918]: cluster 2023-08-27T21:49:43.192044+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:46 smithi006 bash[27292]: cluster 2023-08-27T21:49:45.192589+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:46 smithi006 bash[33918]: cluster 2023-08-27T21:49:45.192589+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:46 smithi130 bash[27579]: cluster 2023-08-27T21:49:45.192589+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:48 smithi006 bash[27292]: cluster 2023-08-27T21:49:47.193262+0000 mgr.y (mgr.14146) 186 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:48 smithi006 bash[27292]: audit 2023-08-27T21:49:47.982116+0000 mon.a (mon.0) 398 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:48 smithi006 bash[27292]: audit 2023-08-27T21:49:48.275166+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:49:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:48 smithi006 bash[27292]: audit 2023-08-27T21:49:48.277018+0000 mon.a (mon.0) 400 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:49:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:48 smithi006 bash[27292]: audit 2023-08-27T21:49:48.286523+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:48 smithi006 bash[33918]: cluster 2023-08-27T21:49:47.193262+0000 mgr.y (mgr.14146) 186 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:48 smithi006 bash[33918]: audit 2023-08-27T21:49:47.982116+0000 mon.a (mon.0) 398 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:48 smithi006 bash[33918]: audit 2023-08-27T21:49:48.275166+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:49:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:48 smithi006 bash[33918]: audit 2023-08-27T21:49:48.277018+0000 mon.a (mon.0) 400 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:49:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:48 smithi006 bash[33918]: audit 2023-08-27T21:49:48.286523+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:48.721 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:48 smithi130 bash[27579]: cluster 2023-08-27T21:49:47.193262+0000 mgr.y (mgr.14146) 186 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:48.721 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:48 smithi130 bash[27579]: audit 2023-08-27T21:49:47.982116+0000 mon.a (mon.0) 398 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:48.721 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:48 smithi130 bash[27579]: audit 2023-08-27T21:49:48.275166+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:49:48.722 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:48 smithi130 bash[27579]: audit 2023-08-27T21:49:48.277018+0000 mon.a (mon.0) 400 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:49:48.722 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:48 smithi130 bash[27579]: audit 2023-08-27T21:49:48.286523+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:49.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:49 smithi006 bash[27292]: audit 2023-08-27T21:49:48.724566+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.130:6800/576819502,v1:172.21.15.130:6801/576819502]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-08-27T21:49:49.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:49 smithi006 bash[27292]: audit 2023-08-27T21:49:48.725115+0000 mon.a (mon.0) 402 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-08-27T21:49:49.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:49 smithi006 bash[33918]: audit 2023-08-27T21:49:48.724566+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.130:6800/576819502,v1:172.21.15.130:6801/576819502]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-08-27T21:49:49.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:49 smithi006 bash[33918]: audit 2023-08-27T21:49:48.725115+0000 mon.a (mon.0) 402 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-08-27T21:49:49.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:49 smithi130 bash[27579]: audit 2023-08-27T21:49:48.724566+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.130:6800/576819502,v1:172.21.15.130:6801/576819502]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-08-27T21:49:49.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:49 smithi130 bash[27579]: audit 2023-08-27T21:49:48.725115+0000 mon.a (mon.0) 402 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-08-27T21:49:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:50 smithi006 bash[27292]: cluster 2023-08-27T21:49:49.193858+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:50 smithi006 bash[27292]: audit 2023-08-27T21:49:49.422740+0000 mon.a (mon.0) 403 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-08-27T21:49:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:50 smithi006 bash[27292]: cluster 2023-08-27T21:49:49.422860+0000 mon.a (mon.0) 404 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-08-27T21:49:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:50 smithi006 bash[27292]: audit 2023-08-27T21:49:49.423043+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:49:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:50 smithi006 bash[27292]: audit 2023-08-27T21:49:49.424087+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.130:6800/576819502,v1:172.21.15.130:6801/576819502]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:49:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:50 smithi006 bash[27292]: audit 2023-08-27T21:49:49.424426+0000 mon.a (mon.0) 406 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:49:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:50 smithi006 bash[33918]: cluster 2023-08-27T21:49:49.193858+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:50 smithi006 bash[33918]: audit 2023-08-27T21:49:49.422740+0000 mon.a (mon.0) 403 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-08-27T21:49:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:50 smithi006 bash[33918]: cluster 2023-08-27T21:49:49.422860+0000 mon.a (mon.0) 404 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-08-27T21:49:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:50 smithi006 bash[33918]: audit 2023-08-27T21:49:49.423043+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:49:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:50 smithi006 bash[33918]: audit 2023-08-27T21:49:49.424087+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.130:6800/576819502,v1:172.21.15.130:6801/576819502]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:49:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:50 smithi006 bash[33918]: audit 2023-08-27T21:49:49.424426+0000 mon.a (mon.0) 406 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:49:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:50 smithi130 bash[27579]: cluster 2023-08-27T21:49:49.193858+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:50.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:50 smithi130 bash[27579]: audit 2023-08-27T21:49:49.422740+0000 mon.a (mon.0) 403 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-08-27T21:49:50.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:50 smithi130 bash[27579]: cluster 2023-08-27T21:49:49.422860+0000 mon.a (mon.0) 404 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-08-27T21:49:50.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:50 smithi130 bash[27579]: audit 2023-08-27T21:49:49.423043+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:49:50.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:50 smithi130 bash[27579]: audit 2023-08-27T21:49:49.424087+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.130:6800/576819502,v1:172.21.15.130:6801/576819502]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:49:50.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:50 smithi130 bash[27579]: audit 2023-08-27T21:49:49.424426+0000 mon.a (mon.0) 406 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:49:51.654 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 4 on host 'smithi130' 2023-08-27T21:49:51.712 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:51 smithi130 bash[27579]: audit 2023-08-27T21:49:50.429470+0000 mon.a (mon.0) 407 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-08-27T21:49:51.712 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:51 smithi130 bash[27579]: cluster 2023-08-27T21:49:50.429620+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-08-27T21:49:51.712 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:51 smithi130 bash[27579]: audit 2023-08-27T21:49:50.430149+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:49:51.712 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:51 smithi130 bash[27579]: audit 2023-08-27T21:49:50.436434+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:49:51.712 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:51 smithi130 bash[27579]: cluster 2023-08-27T21:49:51.194395+0000 mgr.y (mgr.14146) 188 : cluster [DBG] pgmap v169: 1 pgs: 1 peering; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:51.713 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:51 smithi130 bash[27579]: cluster 2023-08-27T21:49:51.429779+0000 mon.a (mon.0) 411 : cluster [INF] osd.4 [v2:172.21.15.130:6800/576819502,v1:172.21.15.130:6801/576819502] boot 2023-08-27T21:49:51.713 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:51 smithi130 bash[27579]: cluster 2023-08-27T21:49:51.429888+0000 mon.a (mon.0) 412 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-08-27T21:49:51.713 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:51 smithi130 bash[27579]: audit 2023-08-27T21:49:51.430238+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:49:51.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:51 smithi006 bash[27292]: audit 2023-08-27T21:49:50.429470+0000 mon.a (mon.0) 407 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-08-27T21:49:51.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:51 smithi006 bash[27292]: cluster 2023-08-27T21:49:50.429620+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-08-27T21:49:51.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:51 smithi006 bash[27292]: audit 2023-08-27T21:49:50.430149+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:49:51.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:51 smithi006 bash[27292]: audit 2023-08-27T21:49:50.436434+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:49:51.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:51 smithi006 bash[27292]: cluster 2023-08-27T21:49:51.194395+0000 mgr.y (mgr.14146) 188 : cluster [DBG] pgmap v169: 1 pgs: 1 peering; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:51.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:51 smithi006 bash[27292]: cluster 2023-08-27T21:49:51.429779+0000 mon.a (mon.0) 411 : cluster [INF] osd.4 [v2:172.21.15.130:6800/576819502,v1:172.21.15.130:6801/576819502] boot 2023-08-27T21:49:51.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:51 smithi006 bash[27292]: cluster 2023-08-27T21:49:51.429888+0000 mon.a (mon.0) 412 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-08-27T21:49:51.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:51 smithi006 bash[27292]: audit 2023-08-27T21:49:51.430238+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:49:51.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:51 smithi006 bash[33918]: audit 2023-08-27T21:49:50.429470+0000 mon.a (mon.0) 407 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-08-27T21:49:51.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:51 smithi006 bash[33918]: cluster 2023-08-27T21:49:50.429620+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-08-27T21:49:51.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:51 smithi006 bash[33918]: audit 2023-08-27T21:49:50.430149+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:49:51.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:51 smithi006 bash[33918]: audit 2023-08-27T21:49:50.436434+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:49:51.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:51 smithi006 bash[33918]: cluster 2023-08-27T21:49:51.194395+0000 mgr.y (mgr.14146) 188 : cluster [DBG] pgmap v169: 1 pgs: 1 peering; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-27T21:49:51.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:51 smithi006 bash[33918]: cluster 2023-08-27T21:49:51.429779+0000 mon.a (mon.0) 411 : cluster [INF] osd.4 [v2:172.21.15.130:6800/576819502,v1:172.21.15.130:6801/576819502] boot 2023-08-27T21:49:51.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:51 smithi006 bash[33918]: cluster 2023-08-27T21:49:51.429888+0000 mon.a (mon.0) 412 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-08-27T21:49:51.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:51 smithi006 bash[33918]: audit 2023-08-27T21:49:51.430238+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:49:52.358 DEBUG:teuthology.orchestra.run.smithi130:osd.4> sudo journalctl -f -n 0 -u ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@osd.4.service 2023-08-27T21:49:52.360 INFO:tasks.cephadm:Deploying osd.5 on smithi130 with /dev/vg_nvme/lv_3... 2023-08-27T21:49:52.360 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-08-27T21:49:52.552 INFO:journalctl@ceph.osd.4.smithi130.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-08-27T21:49:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:52 smithi130 bash[27579]: cluster 2023-08-27T21:49:49.732348+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:49:52.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:52 smithi130 bash[27579]: cluster 2023-08-27T21:49:49.732440+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:49:52.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:52 smithi130 bash[27579]: audit 2023-08-27T21:49:51.650808+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:52.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:52 smithi130 bash[27579]: audit 2023-08-27T21:49:51.664618+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:49:52.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:52 smithi130 bash[27579]: cluster 2023-08-27T21:49:52.384908+0000 mon.a (mon.0) 416 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-08-27T21:49:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:52 smithi006 bash[27292]: cluster 2023-08-27T21:49:49.732348+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:49:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:52 smithi006 bash[27292]: cluster 2023-08-27T21:49:49.732440+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:49:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:52 smithi006 bash[27292]: audit 2023-08-27T21:49:51.650808+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:52 smithi006 bash[27292]: audit 2023-08-27T21:49:51.664618+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:49:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:52 smithi006 bash[27292]: cluster 2023-08-27T21:49:52.384908+0000 mon.a (mon.0) 416 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-08-27T21:49:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:52 smithi006 bash[33918]: cluster 2023-08-27T21:49:49.732348+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:49:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:52 smithi006 bash[33918]: cluster 2023-08-27T21:49:49.732440+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:49:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:52 smithi006 bash[33918]: audit 2023-08-27T21:49:51.650808+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:49:52.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:52 smithi006 bash[33918]: audit 2023-08-27T21:49:51.664618+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:49:52.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:52 smithi006 bash[33918]: cluster 2023-08-27T21:49:52.384908+0000 mon.a (mon.0) 416 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-08-27T21:49:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:54 smithi006 bash[27292]: cluster 2023-08-27T21:49:53.194945+0000 mgr.y (mgr.14146) 189 : cluster [DBG] pgmap v172: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:49:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:54 smithi006 bash[27292]: cluster 2023-08-27T21:49:53.392636+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-08-27T21:49:54.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:54 smithi006 bash[33918]: cluster 2023-08-27T21:49:53.194945+0000 mgr.y (mgr.14146) 189 : cluster [DBG] pgmap v172: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:49:54.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:54 smithi006 bash[33918]: cluster 2023-08-27T21:49:53.392636+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-08-27T21:49:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:54 smithi130 bash[27579]: cluster 2023-08-27T21:49:53.194945+0000 mgr.y (mgr.14146) 189 : cluster [DBG] pgmap v172: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:49:54.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:54 smithi130 bash[27579]: cluster 2023-08-27T21:49:53.392636+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-08-27T21:49:56.157 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-08-27T21:49:56.158 INFO:teuthology.orchestra.run.smithi130.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-08-27T21:49:56.158 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10+0 records in 2023-08-27T21:49:56.158 INFO:teuthology.orchestra.run.smithi130.stderr:10+0 records out 2023-08-27T21:49:56.158 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0156388 s, 670 MB/s 2023-08-27T21:49:56.158 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping successful for: 2023-08-27T21:49:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:56 smithi130 bash[27579]: cluster 2023-08-27T21:49:55.195496+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v174: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:49:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:56 smithi006 bash[27292]: cluster 2023-08-27T21:49:55.195496+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v174: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:49:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:56 smithi006 bash[33918]: cluster 2023-08-27T21:49:55.195496+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v174: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:49:56.914 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph orch daemon add osd smithi130:vg_nvme/lv_3 2023-08-27T21:49:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:49:58 smithi006 bash[27292]: cluster 2023-08-27T21:49:57.196062+0000 mgr.y (mgr.14146) 191 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:49:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:49:58 smithi006 bash[33918]: cluster 2023-08-27T21:49:57.196062+0000 mgr.y (mgr.14146) 191 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:49:58.719 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:49:58 smithi130 bash[27579]: cluster 2023-08-27T21:49:57.196062+0000 mgr.y (mgr.14146) 191 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:00 smithi006 bash[27292]: cluster 2023-08-27T21:49:59.196621+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:00 smithi006 bash[27292]: cluster 2023-08-27T21:50:00.000156+0000 mon.a (mon.0) 418 : cluster [INF] overall HEALTH_OK 2023-08-27T21:50:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:00 smithi006 bash[33918]: cluster 2023-08-27T21:49:59.196621+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:00 smithi006 bash[33918]: cluster 2023-08-27T21:50:00.000156+0000 mon.a (mon.0) 418 : cluster [INF] overall HEALTH_OK 2023-08-27T21:50:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:00 smithi130 bash[27579]: cluster 2023-08-27T21:49:59.196621+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:00 smithi130 bash[27579]: cluster 2023-08-27T21:50:00.000156+0000 mon.a (mon.0) 418 : cluster [INF] overall HEALTH_OK 2023-08-27T21:50:02.538 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:02 smithi130 bash[27579]: cluster 2023-08-27T21:50:01.197192+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:02.538 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:02 smithi130 bash[27579]: audit 2023-08-27T21:50:01.824533+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:50:02.539 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:02 smithi130 bash[27579]: audit 2023-08-27T21:50:01.830207+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:50:02.539 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:02 smithi130 bash[27579]: audit 2023-08-27T21:50:01.831656+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:50:02.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:02 smithi006 bash[27292]: cluster 2023-08-27T21:50:01.197192+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:02.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:02 smithi006 bash[27292]: audit 2023-08-27T21:50:01.824533+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:50:02.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:02 smithi006 bash[27292]: audit 2023-08-27T21:50:01.830207+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:50:02.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:02 smithi006 bash[27292]: audit 2023-08-27T21:50:01.831656+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:50:02.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:02 smithi006 bash[33918]: cluster 2023-08-27T21:50:01.197192+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:02.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:02 smithi006 bash[33918]: audit 2023-08-27T21:50:01.824533+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:50:02.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:02 smithi006 bash[33918]: audit 2023-08-27T21:50:01.830207+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:50:02.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:02 smithi006 bash[33918]: audit 2023-08-27T21:50:01.831656+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:50:03.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:03 smithi006 bash[27292]: audit 2023-08-27T21:50:01.817573+0000 mgr.y (mgr.14146) 194 : audit [DBG] from='client.24244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:50:03.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:03 smithi006 bash[27292]: audit 2023-08-27T21:50:03.259745+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:03.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:03 smithi006 bash[27292]: audit 2023-08-27T21:50:03.261354+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:50:03.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:03 smithi006 bash[27292]: audit 2023-08-27T21:50:03.269457+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:03.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:03 smithi006 bash[27292]: audit 2023-08-27T21:50:03.271149+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:50:03.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:03 smithi006 bash[27292]: audit 2023-08-27T21:50:03.272449+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:50:03.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:03 smithi006 bash[27292]: audit 2023-08-27T21:50:03.281784+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:03.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:03 smithi006 bash[33918]: audit 2023-08-27T21:50:01.817573+0000 mgr.y (mgr.14146) 194 : audit [DBG] from='client.24244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:50:03.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:03 smithi006 bash[33918]: audit 2023-08-27T21:50:03.259745+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:03.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:03 smithi006 bash[33918]: audit 2023-08-27T21:50:03.261354+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:50:03.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:03 smithi006 bash[33918]: audit 2023-08-27T21:50:03.269457+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:03.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:03 smithi006 bash[33918]: audit 2023-08-27T21:50:03.271149+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:50:03.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:03 smithi006 bash[33918]: audit 2023-08-27T21:50:03.272449+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:50:03.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:03 smithi006 bash[33918]: audit 2023-08-27T21:50:03.281784+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:03.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:03 smithi130 bash[27579]: audit 2023-08-27T21:50:01.817573+0000 mgr.y (mgr.14146) 194 : audit [DBG] from='client.24244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:50:03.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:03 smithi130 bash[27579]: audit 2023-08-27T21:50:03.259745+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:03.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:03 smithi130 bash[27579]: audit 2023-08-27T21:50:03.261354+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:50:03.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:03 smithi130 bash[27579]: audit 2023-08-27T21:50:03.269457+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:03.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:03 smithi130 bash[27579]: audit 2023-08-27T21:50:03.271149+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:50:03.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:03 smithi130 bash[27579]: audit 2023-08-27T21:50:03.272449+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:50:03.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:03 smithi130 bash[27579]: audit 2023-08-27T21:50:03.281784+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:04.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:04 smithi006 bash[27292]: cluster 2023-08-27T21:50:03.197698+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:04.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:04 smithi006 bash[27292]: cephadm 2023-08-27T21:50:03.250270+0000 mgr.y (mgr.14146) 196 : cephadm [INF] Detected new or changed devices on smithi130 2023-08-27T21:50:04.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:04 smithi006 bash[27292]: cephadm 2023-08-27T21:50:03.262412+0000 mgr.y (mgr.14146) 197 : cephadm [INF] Adjusting osd_memory_target on smithi130 to 17265M 2023-08-27T21:50:04.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:04 smithi006 bash[33918]: cluster 2023-08-27T21:50:03.197698+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:04.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:04 smithi006 bash[33918]: cephadm 2023-08-27T21:50:03.250270+0000 mgr.y (mgr.14146) 196 : cephadm [INF] Detected new or changed devices on smithi130 2023-08-27T21:50:04.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:04 smithi006 bash[33918]: cephadm 2023-08-27T21:50:03.262412+0000 mgr.y (mgr.14146) 197 : cephadm [INF] Adjusting osd_memory_target on smithi130 to 17265M 2023-08-27T21:50:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:04 smithi130 bash[27579]: cluster 2023-08-27T21:50:03.197698+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:04.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:04 smithi130 bash[27579]: cephadm 2023-08-27T21:50:03.250270+0000 mgr.y (mgr.14146) 196 : cephadm [INF] Detected new or changed devices on smithi130 2023-08-27T21:50:04.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:04 smithi130 bash[27579]: cephadm 2023-08-27T21:50:03.262412+0000 mgr.y (mgr.14146) 197 : cephadm [INF] Adjusting osd_memory_target on smithi130 to 17265M 2023-08-27T21:50:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:06 smithi006 bash[27292]: cluster 2023-08-27T21:50:05.198313+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:06 smithi006 bash[33918]: cluster 2023-08-27T21:50:05.198313+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:06 smithi130 bash[27579]: cluster 2023-08-27T21:50:05.198313+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:08 smithi006 bash[27292]: cluster 2023-08-27T21:50:07.198937+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:08 smithi006 bash[33918]: cluster 2023-08-27T21:50:07.198937+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:08.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:08 smithi130 bash[27579]: cluster 2023-08-27T21:50:07.198937+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:09 smithi006 bash[27292]: cluster 2023-08-27T21:50:09.199652+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:09.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:09 smithi006 bash[33918]: cluster 2023-08-27T21:50:09.199652+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:09 smithi130 bash[27579]: cluster 2023-08-27T21:50:09.199652+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:10 smithi006 bash[27292]: audit 2023-08-27T21:50:10.375831+0000 mon.b (mon.1) 12 : audit [INF] from='client.? 172.21.15.130:0/840497060' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0b6bba7-d752-4a95-927b-8a378bcd77b4"}]: dispatch 2023-08-27T21:50:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:10 smithi006 bash[27292]: audit 2023-08-27T21:50:10.376518+0000 mon.a (mon.0) 428 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0b6bba7-d752-4a95-927b-8a378bcd77b4"}]: dispatch 2023-08-27T21:50:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:10 smithi006 bash[27292]: audit 2023-08-27T21:50:10.384393+0000 mon.a (mon.0) 429 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c0b6bba7-d752-4a95-927b-8a378bcd77b4"}]': finished 2023-08-27T21:50:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:10 smithi006 bash[27292]: cluster 2023-08-27T21:50:10.384568+0000 mon.a (mon.0) 430 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-08-27T21:50:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:10 smithi006 bash[27292]: audit 2023-08-27T21:50:10.384826+0000 mon.a (mon.0) 431 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:50:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:10 smithi006 bash[33918]: audit 2023-08-27T21:50:10.375831+0000 mon.b (mon.1) 12 : audit [INF] from='client.? 172.21.15.130:0/840497060' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0b6bba7-d752-4a95-927b-8a378bcd77b4"}]: dispatch 2023-08-27T21:50:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:10 smithi006 bash[33918]: audit 2023-08-27T21:50:10.376518+0000 mon.a (mon.0) 428 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0b6bba7-d752-4a95-927b-8a378bcd77b4"}]: dispatch 2023-08-27T21:50:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:10 smithi006 bash[33918]: audit 2023-08-27T21:50:10.384393+0000 mon.a (mon.0) 429 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c0b6bba7-d752-4a95-927b-8a378bcd77b4"}]': finished 2023-08-27T21:50:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:10 smithi006 bash[33918]: cluster 2023-08-27T21:50:10.384568+0000 mon.a (mon.0) 430 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-08-27T21:50:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:10 smithi006 bash[33918]: audit 2023-08-27T21:50:10.384826+0000 mon.a (mon.0) 431 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:50:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:10 smithi130 bash[27579]: audit 2023-08-27T21:50:10.375831+0000 mon.b (mon.1) 12 : audit [INF] from='client.? 172.21.15.130:0/840497060' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0b6bba7-d752-4a95-927b-8a378bcd77b4"}]: dispatch 2023-08-27T21:50:10.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:10 smithi130 bash[27579]: audit 2023-08-27T21:50:10.376518+0000 mon.a (mon.0) 428 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0b6bba7-d752-4a95-927b-8a378bcd77b4"}]: dispatch 2023-08-27T21:50:10.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:10 smithi130 bash[27579]: audit 2023-08-27T21:50:10.384393+0000 mon.a (mon.0) 429 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c0b6bba7-d752-4a95-927b-8a378bcd77b4"}]': finished 2023-08-27T21:50:10.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:10 smithi130 bash[27579]: cluster 2023-08-27T21:50:10.384568+0000 mon.a (mon.0) 430 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-08-27T21:50:10.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:10 smithi130 bash[27579]: audit 2023-08-27T21:50:10.384826+0000 mon.a (mon.0) 431 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:50:11.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:11 smithi006 bash[27292]: audit 2023-08-27T21:50:11.102682+0000 mon.b (mon.1) 13 : audit [DBG] from='client.? 172.21.15.130:0/2315082184' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:50:11.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:11 smithi006 bash[27292]: cluster 2023-08-27T21:50:11.200331+0000 mgr.y (mgr.14146) 201 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:11.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:11 smithi006 bash[33918]: audit 2023-08-27T21:50:11.102682+0000 mon.b (mon.1) 13 : audit [DBG] from='client.? 172.21.15.130:0/2315082184' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:50:11.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:11 smithi006 bash[33918]: cluster 2023-08-27T21:50:11.200331+0000 mgr.y (mgr.14146) 201 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:11.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:11 smithi130 bash[27579]: audit 2023-08-27T21:50:11.102682+0000 mon.b (mon.1) 13 : audit [DBG] from='client.? 172.21.15.130:0/2315082184' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:50:11.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:11 smithi130 bash[27579]: cluster 2023-08-27T21:50:11.200331+0000 mgr.y (mgr.14146) 201 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:14.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:14 smithi006 bash[27292]: cluster 2023-08-27T21:50:13.201066+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:14 smithi006 bash[33918]: cluster 2023-08-27T21:50:13.201066+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:14.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:14 smithi130 bash[27579]: cluster 2023-08-27T21:50:13.201066+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:16 smithi006 bash[27292]: cluster 2023-08-27T21:50:15.201729+0000 mgr.y (mgr.14146) 203 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:16 smithi006 bash[33918]: cluster 2023-08-27T21:50:15.201729+0000 mgr.y (mgr.14146) 203 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:16 smithi130 bash[27579]: cluster 2023-08-27T21:50:15.201729+0000 mgr.y (mgr.14146) 203 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:18.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:18 smithi006 bash[27292]: cluster 2023-08-27T21:50:17.202314+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:18.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:18 smithi006 bash[33918]: cluster 2023-08-27T21:50:17.202314+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:18 smithi130 bash[27579]: cluster 2023-08-27T21:50:17.202314+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:20 smithi006 bash[27292]: cluster 2023-08-27T21:50:19.202874+0000 mgr.y (mgr.14146) 205 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:20 smithi006 bash[33918]: cluster 2023-08-27T21:50:19.202874+0000 mgr.y (mgr.14146) 205 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:20 smithi130 bash[27579]: cluster 2023-08-27T21:50:19.202874+0000 mgr.y (mgr.14146) 205 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:22 smithi006 bash[27292]: cluster 2023-08-27T21:50:21.203443+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:22 smithi006 bash[33918]: cluster 2023-08-27T21:50:21.203443+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:22.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:22 smithi130 bash[27579]: cluster 2023-08-27T21:50:21.203443+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:23.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:23 smithi130 bash[27579]: audit 2023-08-27T21:50:23.027985+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-08-27T21:50:23.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:23 smithi130 bash[27579]: audit 2023-08-27T21:50:23.029843+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:50:23.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:23 smithi006 bash[27292]: audit 2023-08-27T21:50:23.027985+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-08-27T21:50:23.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:23 smithi006 bash[27292]: audit 2023-08-27T21:50:23.029843+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:50:23.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:23 smithi006 bash[33918]: audit 2023-08-27T21:50:23.027985+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-08-27T21:50:23.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:23 smithi006 bash[33918]: audit 2023-08-27T21:50:23.029843+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:50:24.541 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:24 smithi130 bash[27579]: cephadm 2023-08-27T21:50:23.031359+0000 mgr.y (mgr.14146) 207 : cephadm [INF] Deploying daemon osd.5 on smithi130 2023-08-27T21:50:24.541 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:24 smithi130 bash[27579]: cluster 2023-08-27T21:50:23.203935+0000 mgr.y (mgr.14146) 208 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:24.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:24 smithi006 bash[27292]: cephadm 2023-08-27T21:50:23.031359+0000 mgr.y (mgr.14146) 207 : cephadm [INF] Deploying daemon osd.5 on smithi130 2023-08-27T21:50:24.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:24 smithi006 bash[27292]: cluster 2023-08-27T21:50:23.203935+0000 mgr.y (mgr.14146) 208 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:24.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:24 smithi006 bash[33918]: cephadm 2023-08-27T21:50:23.031359+0000 mgr.y (mgr.14146) 207 : cephadm [INF] Deploying daemon osd.5 on smithi130 2023-08-27T21:50:24.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:24 smithi006 bash[33918]: cluster 2023-08-27T21:50:23.203935+0000 mgr.y (mgr.14146) 208 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:26.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:26 smithi130 bash[27579]: cluster 2023-08-27T21:50:25.204466+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:26.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:26 smithi006 bash[27292]: cluster 2023-08-27T21:50:25.204466+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:26.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:26 smithi006 bash[33918]: cluster 2023-08-27T21:50:25.204466+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:28 smithi006 bash[33918]: cluster 2023-08-27T21:50:27.205049+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:28 smithi006 bash[33918]: audit 2023-08-27T21:50:27.448239+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:28.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:28 smithi006 bash[33918]: audit 2023-08-27T21:50:27.469923+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:50:28.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:28 smithi006 bash[27292]: cluster 2023-08-27T21:50:27.205049+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:28.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:28 smithi006 bash[27292]: audit 2023-08-27T21:50:27.448239+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:28.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:28 smithi006 bash[27292]: audit 2023-08-27T21:50:27.469923+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:50:28.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:28 smithi130 bash[27579]: cluster 2023-08-27T21:50:27.205049+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:28.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:28 smithi130 bash[27579]: audit 2023-08-27T21:50:27.448239+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:28.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:28 smithi130 bash[27579]: audit 2023-08-27T21:50:27.469923+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:50:30.699 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:30 smithi130 bash[27579]: cluster 2023-08-27T21:50:29.205702+0000 mgr.y (mgr.14146) 211 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:30 smithi006 bash[27292]: cluster 2023-08-27T21:50:29.205702+0000 mgr.y (mgr.14146) 211 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:30 smithi006 bash[33918]: cluster 2023-08-27T21:50:29.205702+0000 mgr.y (mgr.14146) 211 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:31.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:31 smithi130 bash[27579]: cluster 2023-08-27T21:50:31.206304+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:31.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:31 smithi006 bash[27292]: cluster 2023-08-27T21:50:31.206304+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:31.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:31 smithi006 bash[33918]: cluster 2023-08-27T21:50:31.206304+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:34.471 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:34 smithi130 bash[27579]: cluster 2023-08-27T21:50:33.206943+0000 mgr.y (mgr.14146) 213 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:34.471 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:34 smithi130 bash[27579]: audit 2023-08-27T21:50:33.618774+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:34.471 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:34 smithi130 bash[27579]: audit 2023-08-27T21:50:33.620247+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:50:34.471 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:34 smithi130 bash[27579]: audit 2023-08-27T21:50:33.621693+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:50:34.472 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:34 smithi130 bash[27579]: audit 2023-08-27T21:50:33.627898+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:34.472 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:34 smithi130 bash[27579]: audit 2023-08-27T21:50:34.212461+0000 mon.b (mon.1) 14 : audit [INF] from='osd.5 [v2:172.21.15.130:6808/169159537,v1:172.21.15.130:6809/169159537]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-08-27T21:50:34.472 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:34 smithi130 bash[27579]: audit 2023-08-27T21:50:34.212796+0000 mon.a (mon.0) 440 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-08-27T21:50:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:34 smithi006 bash[27292]: cluster 2023-08-27T21:50:33.206943+0000 mgr.y (mgr.14146) 213 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:34 smithi006 bash[27292]: audit 2023-08-27T21:50:33.618774+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:34 smithi006 bash[27292]: audit 2023-08-27T21:50:33.620247+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:50:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:34 smithi006 bash[27292]: audit 2023-08-27T21:50:33.621693+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:50:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:34 smithi006 bash[27292]: audit 2023-08-27T21:50:33.627898+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:34 smithi006 bash[27292]: audit 2023-08-27T21:50:34.212461+0000 mon.b (mon.1) 14 : audit [INF] from='osd.5 [v2:172.21.15.130:6808/169159537,v1:172.21.15.130:6809/169159537]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-08-27T21:50:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:34 smithi006 bash[27292]: audit 2023-08-27T21:50:34.212796+0000 mon.a (mon.0) 440 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-08-27T21:50:34.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:34 smithi006 bash[33918]: cluster 2023-08-27T21:50:33.206943+0000 mgr.y (mgr.14146) 213 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:34.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:34 smithi006 bash[33918]: audit 2023-08-27T21:50:33.618774+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:34.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:34 smithi006 bash[33918]: audit 2023-08-27T21:50:33.620247+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:50:34.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:34 smithi006 bash[33918]: audit 2023-08-27T21:50:33.621693+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:50:34.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:34 smithi006 bash[33918]: audit 2023-08-27T21:50:33.627898+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:34.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:34 smithi006 bash[33918]: audit 2023-08-27T21:50:34.212461+0000 mon.b (mon.1) 14 : audit [INF] from='osd.5 [v2:172.21.15.130:6808/169159537,v1:172.21.15.130:6809/169159537]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-08-27T21:50:34.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:34 smithi006 bash[33918]: audit 2023-08-27T21:50:34.212796+0000 mon.a (mon.0) 440 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-08-27T21:50:35.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:35 smithi006 bash[27292]: audit 2023-08-27T21:50:34.635216+0000 mon.a (mon.0) 441 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-08-27T21:50:35.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:35 smithi006 bash[27292]: cluster 2023-08-27T21:50:34.635390+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-08-27T21:50:35.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:35 smithi006 bash[27292]: audit 2023-08-27T21:50:34.635644+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:50:35.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:35 smithi006 bash[27292]: audit 2023-08-27T21:50:34.636870+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.130:6808/169159537,v1:172.21.15.130:6809/169159537]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:50:35.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:35 smithi006 bash[27292]: audit 2023-08-27T21:50:34.637147+0000 mon.a (mon.0) 444 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:50:35.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:35 smithi006 bash[27292]: cluster 2023-08-27T21:50:35.207516+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:35.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:35 smithi006 bash[33918]: audit 2023-08-27T21:50:34.635216+0000 mon.a (mon.0) 441 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-08-27T21:50:35.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:35 smithi006 bash[33918]: cluster 2023-08-27T21:50:34.635390+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-08-27T21:50:35.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:35 smithi006 bash[33918]: audit 2023-08-27T21:50:34.635644+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:50:35.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:35 smithi006 bash[33918]: audit 2023-08-27T21:50:34.636870+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.130:6808/169159537,v1:172.21.15.130:6809/169159537]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:50:35.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:35 smithi006 bash[33918]: audit 2023-08-27T21:50:34.637147+0000 mon.a (mon.0) 444 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:50:35.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:35 smithi006 bash[33918]: cluster 2023-08-27T21:50:35.207516+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:36.074 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:35 smithi130 bash[27579]: audit 2023-08-27T21:50:34.635216+0000 mon.a (mon.0) 441 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-08-27T21:50:36.074 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:35 smithi130 bash[27579]: cluster 2023-08-27T21:50:34.635390+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-08-27T21:50:36.074 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:35 smithi130 bash[27579]: audit 2023-08-27T21:50:34.635644+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:50:36.074 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:35 smithi130 bash[27579]: audit 2023-08-27T21:50:34.636870+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.130:6808/169159537,v1:172.21.15.130:6809/169159537]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:50:36.074 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:35 smithi130 bash[27579]: audit 2023-08-27T21:50:34.637147+0000 mon.a (mon.0) 444 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:50:36.075 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:35 smithi130 bash[27579]: cluster 2023-08-27T21:50:35.207516+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-27T21:50:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:36 smithi006 bash[27292]: audit 2023-08-27T21:50:35.642835+0000 mon.a (mon.0) 445 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-08-27T21:50:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:36 smithi006 bash[27292]: cluster 2023-08-27T21:50:35.642956+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-08-27T21:50:36.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:36 smithi006 bash[27292]: audit 2023-08-27T21:50:35.643657+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:50:36.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:36 smithi006 bash[27292]: audit 2023-08-27T21:50:35.649482+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:50:36.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:36 smithi006 bash[27292]: cluster 2023-08-27T21:50:36.641688+0000 mon.a (mon.0) 449 : cluster [INF] osd.5 [v2:172.21.15.130:6808/169159537,v1:172.21.15.130:6809/169159537] boot 2023-08-27T21:50:36.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:36 smithi006 bash[27292]: cluster 2023-08-27T21:50:36.641845+0000 mon.a (mon.0) 450 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-08-27T21:50:36.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:36 smithi006 bash[27292]: audit 2023-08-27T21:50:36.642235+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:50:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:36 smithi006 bash[33918]: audit 2023-08-27T21:50:35.642835+0000 mon.a (mon.0) 445 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-08-27T21:50:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:36 smithi006 bash[33918]: cluster 2023-08-27T21:50:35.642956+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-08-27T21:50:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:36 smithi006 bash[33918]: audit 2023-08-27T21:50:35.643657+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:50:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:36 smithi006 bash[33918]: audit 2023-08-27T21:50:35.649482+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:50:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:36 smithi006 bash[33918]: cluster 2023-08-27T21:50:36.641688+0000 mon.a (mon.0) 449 : cluster [INF] osd.5 [v2:172.21.15.130:6808/169159537,v1:172.21.15.130:6809/169159537] boot 2023-08-27T21:50:36.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:36 smithi006 bash[33918]: cluster 2023-08-27T21:50:36.641845+0000 mon.a (mon.0) 450 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-08-27T21:50:36.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:36 smithi006 bash[33918]: audit 2023-08-27T21:50:36.642235+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:50:37.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:36 smithi130 bash[27579]: audit 2023-08-27T21:50:35.642835+0000 mon.a (mon.0) 445 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-08-27T21:50:37.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:36 smithi130 bash[27579]: cluster 2023-08-27T21:50:35.642956+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-08-27T21:50:37.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:36 smithi130 bash[27579]: audit 2023-08-27T21:50:35.643657+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:50:37.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:36 smithi130 bash[27579]: audit 2023-08-27T21:50:35.649482+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:50:37.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:36 smithi130 bash[27579]: cluster 2023-08-27T21:50:36.641688+0000 mon.a (mon.0) 449 : cluster [INF] osd.5 [v2:172.21.15.130:6808/169159537,v1:172.21.15.130:6809/169159537] boot 2023-08-27T21:50:37.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:36 smithi130 bash[27579]: cluster 2023-08-27T21:50:36.641845+0000 mon.a (mon.0) 450 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-08-27T21:50:37.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:36 smithi130 bash[27579]: audit 2023-08-27T21:50:36.642235+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:50:37.344 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 5 on host 'smithi130' 2023-08-27T21:50:37.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:37 smithi130 bash[27579]: cluster 2023-08-27T21:50:35.236761+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:50:37.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:37 smithi130 bash[27579]: cluster 2023-08-27T21:50:35.236862+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:50:37.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:37 smithi130 bash[27579]: cluster 2023-08-27T21:50:37.208086+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:37.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:37 smithi130 bash[27579]: audit 2023-08-27T21:50:37.340792+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:37.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:37 smithi130 bash[27579]: audit 2023-08-27T21:50:37.356367+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:50:37.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:37 smithi130 bash[27579]: cluster 2023-08-27T21:50:37.395810+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-08-27T21:50:37.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:37 smithi006 bash[27292]: cluster 2023-08-27T21:50:35.236761+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:50:37.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:37 smithi006 bash[27292]: cluster 2023-08-27T21:50:35.236862+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:50:37.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:37 smithi006 bash[27292]: cluster 2023-08-27T21:50:37.208086+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:37.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:37 smithi006 bash[27292]: audit 2023-08-27T21:50:37.340792+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:37.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:37 smithi006 bash[27292]: audit 2023-08-27T21:50:37.356367+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:50:37.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:37 smithi006 bash[27292]: cluster 2023-08-27T21:50:37.395810+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-08-27T21:50:37.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:37 smithi006 bash[33918]: cluster 2023-08-27T21:50:35.236761+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:50:37.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:37 smithi006 bash[33918]: cluster 2023-08-27T21:50:35.236862+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:50:37.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:37 smithi006 bash[33918]: cluster 2023-08-27T21:50:37.208086+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:37.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:37 smithi006 bash[33918]: audit 2023-08-27T21:50:37.340792+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:37.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:37 smithi006 bash[33918]: audit 2023-08-27T21:50:37.356367+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:50:37.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:37 smithi006 bash[33918]: cluster 2023-08-27T21:50:37.395810+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-08-27T21:50:38.031 DEBUG:teuthology.orchestra.run.smithi130:osd.5> sudo journalctl -f -n 0 -u ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@osd.5.service 2023-08-27T21:50:38.034 INFO:tasks.cephadm:Deploying osd.6 on smithi130 with /dev/vg_nvme/lv_2... 2023-08-27T21:50:38.034 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-08-27T21:50:38.281 INFO:journalctl@ceph.osd.5.smithi130.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-08-27T21:50:39.513 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:39 smithi130 bash[27579]: cluster 2023-08-27T21:50:38.406850+0000 mon.a (mon.0) 455 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-08-27T21:50:39.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:39 smithi006 bash[27292]: cluster 2023-08-27T21:50:38.406850+0000 mon.a (mon.0) 455 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-08-27T21:50:39.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:39 smithi006 bash[33918]: cluster 2023-08-27T21:50:38.406850+0000 mon.a (mon.0) 455 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-08-27T21:50:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:40 smithi006 bash[33918]: cluster 2023-08-27T21:50:39.208767+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:40 smithi006 bash[27292]: cluster 2023-08-27T21:50:39.208767+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:40.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:40 smithi130 bash[27579]: cluster 2023-08-27T21:50:39.208767+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:41.667 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-08-27T21:50:41.668 INFO:teuthology.orchestra.run.smithi130.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-08-27T21:50:41.668 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10+0 records in 2023-08-27T21:50:41.668 INFO:teuthology.orchestra.run.smithi130.stderr:10+0 records out 2023-08-27T21:50:41.668 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0173209 s, 605 MB/s 2023-08-27T21:50:41.668 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping successful for: 2023-08-27T21:50:42.486 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph orch daemon add osd smithi130:vg_nvme/lv_2 2023-08-27T21:50:42.692 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:42 smithi130 bash[27579]: cluster 2023-08-27T21:50:41.209532+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:42 smithi006 bash[27292]: cluster 2023-08-27T21:50:41.209532+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:42.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:42 smithi006 bash[33918]: cluster 2023-08-27T21:50:41.209532+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:44.554 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:44 smithi130 bash[27579]: cluster 2023-08-27T21:50:43.210160+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:44.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:44 smithi006 bash[27292]: cluster 2023-08-27T21:50:43.210160+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:44.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:44 smithi006 bash[33918]: cluster 2023-08-27T21:50:43.210160+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:46 smithi006 bash[33918]: cluster 2023-08-27T21:50:45.210703+0000 mgr.y (mgr.14146) 219 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:46 smithi006 bash[27292]: cluster 2023-08-27T21:50:45.210703+0000 mgr.y (mgr.14146) 219 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:46.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:46 smithi130 bash[27579]: cluster 2023-08-27T21:50:45.210703+0000 mgr.y (mgr.14146) 219 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:47.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:47 smithi006 bash[27292]: audit 2023-08-27T21:50:47.350119+0000 mon.a (mon.0) 456 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:50:47.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:47 smithi006 bash[27292]: audit 2023-08-27T21:50:47.354116+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:50:47.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:47 smithi006 bash[27292]: audit 2023-08-27T21:50:47.355685+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:50:47.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:47 smithi006 bash[33918]: audit 2023-08-27T21:50:47.350119+0000 mon.a (mon.0) 456 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:50:47.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:47 smithi006 bash[33918]: audit 2023-08-27T21:50:47.354116+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:50:47.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:47 smithi006 bash[33918]: audit 2023-08-27T21:50:47.355685+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:50:47.767 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:47 smithi130 bash[27579]: audit 2023-08-27T21:50:47.350119+0000 mon.a (mon.0) 456 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:50:47.767 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:47 smithi130 bash[27579]: audit 2023-08-27T21:50:47.354116+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:50:47.767 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:47 smithi130 bash[27579]: audit 2023-08-27T21:50:47.355685+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:50:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:48 smithi006 bash[27292]: cluster 2023-08-27T21:50:47.211206+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:48 smithi006 bash[27292]: audit 2023-08-27T21:50:47.343713+0000 mgr.y (mgr.14146) 221 : audit [DBG] from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:50:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:48 smithi006 bash[33918]: cluster 2023-08-27T21:50:47.211206+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:48 smithi006 bash[33918]: audit 2023-08-27T21:50:47.343713+0000 mgr.y (mgr.14146) 221 : audit [DBG] from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:50:48.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:48 smithi130 bash[27579]: cluster 2023-08-27T21:50:47.211206+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:48.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:48 smithi130 bash[27579]: audit 2023-08-27T21:50:47.343713+0000 mgr.y (mgr.14146) 221 : audit [DBG] from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:50:49.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:49 smithi006 bash[27292]: cephadm 2023-08-27T21:50:48.697948+0000 mgr.y (mgr.14146) 222 : cephadm [INF] Detected new or changed devices on smithi130 2023-08-27T21:50:49.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:49 smithi006 bash[27292]: audit 2023-08-27T21:50:48.706841+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:49.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:49 smithi006 bash[27292]: audit 2023-08-27T21:50:48.992213+0000 mon.a (mon.0) 460 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:50:49.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:49 smithi006 bash[27292]: audit 2023-08-27T21:50:48.994179+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:50:49.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:49 smithi006 bash[27292]: cephadm 2023-08-27T21:50:48.995662+0000 mgr.y (mgr.14146) 223 : cephadm [INF] Adjusting osd_memory_target on smithi130 to 8632M 2023-08-27T21:50:49.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:49 smithi006 bash[27292]: audit 2023-08-27T21:50:49.004271+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:49.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:49 smithi006 bash[27292]: audit 2023-08-27T21:50:49.006044+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:50:49.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:49 smithi006 bash[27292]: audit 2023-08-27T21:50:49.007454+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:50:49.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:49 smithi006 bash[27292]: audit 2023-08-27T21:50:49.015759+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:49.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:49 smithi006 bash[27292]: cluster 2023-08-27T21:50:49.211749+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:49.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:49 smithi006 bash[33918]: cephadm 2023-08-27T21:50:48.697948+0000 mgr.y (mgr.14146) 222 : cephadm [INF] Detected new or changed devices on smithi130 2023-08-27T21:50:49.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:49 smithi006 bash[33918]: audit 2023-08-27T21:50:48.706841+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:49.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:49 smithi006 bash[33918]: audit 2023-08-27T21:50:48.992213+0000 mon.a (mon.0) 460 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:50:49.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:49 smithi006 bash[33918]: audit 2023-08-27T21:50:48.994179+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:50:49.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:49 smithi006 bash[33918]: cephadm 2023-08-27T21:50:48.995662+0000 mgr.y (mgr.14146) 223 : cephadm [INF] Adjusting osd_memory_target on smithi130 to 8632M 2023-08-27T21:50:49.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:49 smithi006 bash[33918]: audit 2023-08-27T21:50:49.004271+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:49.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:49 smithi006 bash[33918]: audit 2023-08-27T21:50:49.006044+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:50:49.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:49 smithi006 bash[33918]: audit 2023-08-27T21:50:49.007454+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:50:49.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:49 smithi006 bash[33918]: audit 2023-08-27T21:50:49.015759+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:49.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:49 smithi006 bash[33918]: cluster 2023-08-27T21:50:49.211749+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:50.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:49 smithi130 bash[27579]: cephadm 2023-08-27T21:50:48.697948+0000 mgr.y (mgr.14146) 222 : cephadm [INF] Detected new or changed devices on smithi130 2023-08-27T21:50:50.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:49 smithi130 bash[27579]: audit 2023-08-27T21:50:48.706841+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:50.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:49 smithi130 bash[27579]: audit 2023-08-27T21:50:48.992213+0000 mon.a (mon.0) 460 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:50:50.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:49 smithi130 bash[27579]: audit 2023-08-27T21:50:48.994179+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:50:50.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:49 smithi130 bash[27579]: cephadm 2023-08-27T21:50:48.995662+0000 mgr.y (mgr.14146) 223 : cephadm [INF] Adjusting osd_memory_target on smithi130 to 8632M 2023-08-27T21:50:50.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:49 smithi130 bash[27579]: audit 2023-08-27T21:50:49.004271+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:50.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:49 smithi130 bash[27579]: audit 2023-08-27T21:50:49.006044+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:50:50.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:49 smithi130 bash[27579]: audit 2023-08-27T21:50:49.007454+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:50:50.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:49 smithi130 bash[27579]: audit 2023-08-27T21:50:49.015759+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:50:50.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:49 smithi130 bash[27579]: cluster 2023-08-27T21:50:49.211749+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:52.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:52 smithi130 bash[27579]: cluster 2023-08-27T21:50:51.212287+0000 mgr.y (mgr.14146) 225 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:52.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:52 smithi006 bash[27292]: cluster 2023-08-27T21:50:51.212287+0000 mgr.y (mgr.14146) 225 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:52 smithi006 bash[33918]: cluster 2023-08-27T21:50:51.212287+0000 mgr.y (mgr.14146) 225 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:54 smithi006 bash[27292]: cluster 2023-08-27T21:50:53.212960+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:54.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:54 smithi006 bash[33918]: cluster 2023-08-27T21:50:53.212960+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:54.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:54 smithi130 bash[27579]: cluster 2023-08-27T21:50:53.212960+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:56 smithi006 bash[27292]: cluster 2023-08-27T21:50:55.213520+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:56 smithi006 bash[27292]: audit 2023-08-27T21:50:55.852201+0000 mon.b (mon.1) 16 : audit [INF] from='client.? 172.21.15.130:0/3702620442' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fa4c8b52-d88a-427b-bac6-a1abaa0e92e6"}]: dispatch 2023-08-27T21:50:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:56 smithi006 bash[27292]: audit 2023-08-27T21:50:55.852748+0000 mon.a (mon.0) 466 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fa4c8b52-d88a-427b-bac6-a1abaa0e92e6"}]: dispatch 2023-08-27T21:50:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:56 smithi006 bash[27292]: audit 2023-08-27T21:50:55.860674+0000 mon.a (mon.0) 467 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fa4c8b52-d88a-427b-bac6-a1abaa0e92e6"}]': finished 2023-08-27T21:50:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:56 smithi006 bash[27292]: cluster 2023-08-27T21:50:55.860946+0000 mon.a (mon.0) 468 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-08-27T21:50:56.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:56 smithi006 bash[27292]: audit 2023-08-27T21:50:55.861199+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:50:56.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:56 smithi006 bash[33918]: cluster 2023-08-27T21:50:55.213520+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:56.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:56 smithi006 bash[33918]: audit 2023-08-27T21:50:55.852201+0000 mon.b (mon.1) 16 : audit [INF] from='client.? 172.21.15.130:0/3702620442' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fa4c8b52-d88a-427b-bac6-a1abaa0e92e6"}]: dispatch 2023-08-27T21:50:56.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:56 smithi006 bash[33918]: audit 2023-08-27T21:50:55.852748+0000 mon.a (mon.0) 466 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fa4c8b52-d88a-427b-bac6-a1abaa0e92e6"}]: dispatch 2023-08-27T21:50:56.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:56 smithi006 bash[33918]: audit 2023-08-27T21:50:55.860674+0000 mon.a (mon.0) 467 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fa4c8b52-d88a-427b-bac6-a1abaa0e92e6"}]': finished 2023-08-27T21:50:56.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:56 smithi006 bash[33918]: cluster 2023-08-27T21:50:55.860946+0000 mon.a (mon.0) 468 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-08-27T21:50:56.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:56 smithi006 bash[33918]: audit 2023-08-27T21:50:55.861199+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:50:56.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:56 smithi130 bash[27579]: cluster 2023-08-27T21:50:55.213520+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:56.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:56 smithi130 bash[27579]: audit 2023-08-27T21:50:55.852201+0000 mon.b (mon.1) 16 : audit [INF] from='client.? 172.21.15.130:0/3702620442' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fa4c8b52-d88a-427b-bac6-a1abaa0e92e6"}]: dispatch 2023-08-27T21:50:56.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:56 smithi130 bash[27579]: audit 2023-08-27T21:50:55.852748+0000 mon.a (mon.0) 466 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fa4c8b52-d88a-427b-bac6-a1abaa0e92e6"}]: dispatch 2023-08-27T21:50:56.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:56 smithi130 bash[27579]: audit 2023-08-27T21:50:55.860674+0000 mon.a (mon.0) 467 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fa4c8b52-d88a-427b-bac6-a1abaa0e92e6"}]': finished 2023-08-27T21:50:56.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:56 smithi130 bash[27579]: cluster 2023-08-27T21:50:55.860946+0000 mon.a (mon.0) 468 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-08-27T21:50:56.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:56 smithi130 bash[27579]: audit 2023-08-27T21:50:55.861199+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:50:57.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:57 smithi006 bash[33918]: audit 2023-08-27T21:50:56.579640+0000 mon.b (mon.1) 17 : audit [DBG] from='client.? 172.21.15.130:0/947817852' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:50:57.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:57 smithi006 bash[27292]: audit 2023-08-27T21:50:56.579640+0000 mon.b (mon.1) 17 : audit [DBG] from='client.? 172.21.15.130:0/947817852' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:50:57.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:57 smithi130 bash[27579]: audit 2023-08-27T21:50:56.579640+0000 mon.b (mon.1) 17 : audit [DBG] from='client.? 172.21.15.130:0/947817852' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:50:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:50:58 smithi006 bash[27292]: cluster 2023-08-27T21:50:57.214067+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:58.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:50:58 smithi006 bash[33918]: cluster 2023-08-27T21:50:57.214067+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:50:58.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:50:58 smithi130 bash[27579]: cluster 2023-08-27T21:50:57.214067+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:00 smithi006 bash[33918]: cluster 2023-08-27T21:50:59.214613+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:00 smithi006 bash[27292]: cluster 2023-08-27T21:50:59.214613+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:00.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:00 smithi130 bash[27579]: cluster 2023-08-27T21:50:59.214613+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:02 smithi006 bash[27292]: cluster 2023-08-27T21:51:01.215136+0000 mgr.y (mgr.14146) 230 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:02 smithi006 bash[33918]: cluster 2023-08-27T21:51:01.215136+0000 mgr.y (mgr.14146) 230 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:02.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:02 smithi130 bash[27579]: cluster 2023-08-27T21:51:01.215136+0000 mgr.y (mgr.14146) 230 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:04.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:04 smithi006 bash[27292]: cluster 2023-08-27T21:51:03.215669+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:04.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:04 smithi006 bash[33918]: cluster 2023-08-27T21:51:03.215669+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:04.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:04 smithi130 bash[27579]: cluster 2023-08-27T21:51:03.215669+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:06 smithi006 bash[27292]: cluster 2023-08-27T21:51:05.216242+0000 mgr.y (mgr.14146) 232 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:06 smithi006 bash[33918]: cluster 2023-08-27T21:51:05.216242+0000 mgr.y (mgr.14146) 232 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:06 smithi130 bash[27579]: cluster 2023-08-27T21:51:05.216242+0000 mgr.y (mgr.14146) 232 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:08 smithi006 bash[27292]: cluster 2023-08-27T21:51:07.216796+0000 mgr.y (mgr.14146) 233 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:08 smithi006 bash[33918]: cluster 2023-08-27T21:51:07.216796+0000 mgr.y (mgr.14146) 233 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:08.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:08 smithi130 bash[27579]: cluster 2023-08-27T21:51:07.216796+0000 mgr.y (mgr.14146) 233 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:09.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:09 smithi006 bash[27292]: audit 2023-08-27T21:51:08.629624+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-08-27T21:51:09.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:09 smithi006 bash[27292]: audit 2023-08-27T21:51:08.631633+0000 mon.a (mon.0) 471 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:51:09.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:09 smithi006 bash[33918]: audit 2023-08-27T21:51:08.629624+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-08-27T21:51:09.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:09 smithi006 bash[33918]: audit 2023-08-27T21:51:08.631633+0000 mon.a (mon.0) 471 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:51:09.768 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:09 smithi130 bash[27579]: audit 2023-08-27T21:51:08.629624+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-08-27T21:51:09.768 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:09 smithi130 bash[27579]: audit 2023-08-27T21:51:08.631633+0000 mon.a (mon.0) 471 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:51:10.710 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:10 smithi130 bash[27579]: cephadm 2023-08-27T21:51:08.633272+0000 mgr.y (mgr.14146) 234 : cephadm [INF] Deploying daemon osd.6 on smithi130 2023-08-27T21:51:10.710 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:10 smithi130 bash[27579]: cluster 2023-08-27T21:51:09.217322+0000 mgr.y (mgr.14146) 235 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:10 smithi006 bash[27292]: cephadm 2023-08-27T21:51:08.633272+0000 mgr.y (mgr.14146) 234 : cephadm [INF] Deploying daemon osd.6 on smithi130 2023-08-27T21:51:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:10 smithi006 bash[27292]: cluster 2023-08-27T21:51:09.217322+0000 mgr.y (mgr.14146) 235 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:10 smithi006 bash[33918]: cephadm 2023-08-27T21:51:08.633272+0000 mgr.y (mgr.14146) 234 : cephadm [INF] Deploying daemon osd.6 on smithi130 2023-08-27T21:51:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:10 smithi006 bash[33918]: cluster 2023-08-27T21:51:09.217322+0000 mgr.y (mgr.14146) 235 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:11.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:11 smithi006 bash[27292]: cluster 2023-08-27T21:51:11.217868+0000 mgr.y (mgr.14146) 236 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:11.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:11 smithi006 bash[33918]: cluster 2023-08-27T21:51:11.217868+0000 mgr.y (mgr.14146) 236 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:11.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:11 smithi130 bash[27579]: cluster 2023-08-27T21:51:11.217868+0000 mgr.y (mgr.14146) 236 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:14.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:14 smithi130 bash[27579]: cluster 2023-08-27T21:51:13.218438+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:14.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:14 smithi130 bash[27579]: audit 2023-08-27T21:51:13.289851+0000 mon.a (mon.0) 472 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:51:14.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:14 smithi130 bash[27579]: audit 2023-08-27T21:51:13.305212+0000 mon.a (mon.0) 473 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:51:14.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:14 smithi006 bash[27292]: cluster 2023-08-27T21:51:13.218438+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:14.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:14 smithi006 bash[27292]: audit 2023-08-27T21:51:13.289851+0000 mon.a (mon.0) 472 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:51:14.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:14 smithi006 bash[27292]: audit 2023-08-27T21:51:13.305212+0000 mon.a (mon.0) 473 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:51:14.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:14 smithi006 bash[33918]: cluster 2023-08-27T21:51:13.218438+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:14.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:14 smithi006 bash[33918]: audit 2023-08-27T21:51:13.289851+0000 mon.a (mon.0) 472 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:51:14.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:14 smithi006 bash[33918]: audit 2023-08-27T21:51:13.305212+0000 mon.a (mon.0) 473 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:51:16.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:16 smithi130 bash[27579]: cluster 2023-08-27T21:51:15.218966+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:16 smithi006 bash[27292]: cluster 2023-08-27T21:51:15.218966+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:16 smithi006 bash[33918]: cluster 2023-08-27T21:51:15.218966+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:18 smithi006 bash[27292]: cluster 2023-08-27T21:51:17.219522+0000 mgr.y (mgr.14146) 239 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:18.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:18 smithi006 bash[33918]: cluster 2023-08-27T21:51:17.219522+0000 mgr.y (mgr.14146) 239 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:18.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:18 smithi130 bash[27579]: cluster 2023-08-27T21:51:17.219522+0000 mgr.y (mgr.14146) 239 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:20 smithi006 bash[27292]: cluster 2023-08-27T21:51:19.220155+0000 mgr.y (mgr.14146) 240 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:20 smithi006 bash[27292]: audit 2023-08-27T21:51:19.625656+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:51:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:20 smithi006 bash[27292]: audit 2023-08-27T21:51:19.627022+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:51:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:20 smithi006 bash[27292]: audit 2023-08-27T21:51:19.628434+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:51:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:20 smithi006 bash[27292]: audit 2023-08-27T21:51:19.635346+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:51:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:20 smithi006 bash[27292]: audit 2023-08-27T21:51:20.324764+0000 mon.b (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.130:6816/1800436766,v1:172.21.15.130:6817/1800436766]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-08-27T21:51:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:20 smithi006 bash[27292]: audit 2023-08-27T21:51:20.324956+0000 mon.a (mon.0) 478 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-08-27T21:51:20.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:20 smithi006 bash[33918]: cluster 2023-08-27T21:51:19.220155+0000 mgr.y (mgr.14146) 240 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:20.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:20 smithi006 bash[33918]: audit 2023-08-27T21:51:19.625656+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:51:20.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:20 smithi006 bash[33918]: audit 2023-08-27T21:51:19.627022+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:51:20.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:20 smithi006 bash[33918]: audit 2023-08-27T21:51:19.628434+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:51:20.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:20 smithi006 bash[33918]: audit 2023-08-27T21:51:19.635346+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:51:20.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:20 smithi006 bash[33918]: audit 2023-08-27T21:51:20.324764+0000 mon.b (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.130:6816/1800436766,v1:172.21.15.130:6817/1800436766]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-08-27T21:51:20.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:20 smithi006 bash[33918]: audit 2023-08-27T21:51:20.324956+0000 mon.a (mon.0) 478 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-08-27T21:51:20.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:20 smithi130 bash[27579]: cluster 2023-08-27T21:51:19.220155+0000 mgr.y (mgr.14146) 240 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:20.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:20 smithi130 bash[27579]: audit 2023-08-27T21:51:19.625656+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:51:20.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:20 smithi130 bash[27579]: audit 2023-08-27T21:51:19.627022+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:51:20.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:20 smithi130 bash[27579]: audit 2023-08-27T21:51:19.628434+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:51:20.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:20 smithi130 bash[27579]: audit 2023-08-27T21:51:19.635346+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:51:20.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:20 smithi130 bash[27579]: audit 2023-08-27T21:51:20.324764+0000 mon.b (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.130:6816/1800436766,v1:172.21.15.130:6817/1800436766]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-08-27T21:51:20.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:20 smithi130 bash[27579]: audit 2023-08-27T21:51:20.324956+0000 mon.a (mon.0) 478 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-08-27T21:51:21.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:21 smithi006 bash[27292]: audit 2023-08-27T21:51:20.639762+0000 mon.a (mon.0) 479 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-08-27T21:51:21.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:21 smithi006 bash[27292]: cluster 2023-08-27T21:51:20.639906+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-08-27T21:51:21.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:21 smithi006 bash[27292]: audit 2023-08-27T21:51:20.642353+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:51:21.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:21 smithi006 bash[27292]: audit 2023-08-27T21:51:20.643753+0000 mon.a (mon.0) 482 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:51:21.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:21 smithi006 bash[27292]: audit 2023-08-27T21:51:20.644023+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.130:6816/1800436766,v1:172.21.15.130:6817/1800436766]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:51:21.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:21 smithi006 bash[27292]: cluster 2023-08-27T21:51:21.220823+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:21.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:21 smithi006 bash[33918]: audit 2023-08-27T21:51:20.639762+0000 mon.a (mon.0) 479 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-08-27T21:51:21.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:21 smithi006 bash[33918]: cluster 2023-08-27T21:51:20.639906+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-08-27T21:51:21.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:21 smithi006 bash[33918]: audit 2023-08-27T21:51:20.642353+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:51:21.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:21 smithi006 bash[33918]: audit 2023-08-27T21:51:20.643753+0000 mon.a (mon.0) 482 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:51:21.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:21 smithi006 bash[33918]: audit 2023-08-27T21:51:20.644023+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.130:6816/1800436766,v1:172.21.15.130:6817/1800436766]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:51:21.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:21 smithi006 bash[33918]: cluster 2023-08-27T21:51:21.220823+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:21.984 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:21 smithi130 bash[27579]: audit 2023-08-27T21:51:20.639762+0000 mon.a (mon.0) 479 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-08-27T21:51:21.984 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:21 smithi130 bash[27579]: cluster 2023-08-27T21:51:20.639906+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-08-27T21:51:21.984 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:21 smithi130 bash[27579]: audit 2023-08-27T21:51:20.642353+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:51:21.985 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:21 smithi130 bash[27579]: audit 2023-08-27T21:51:20.643753+0000 mon.a (mon.0) 482 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:51:21.985 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:21 smithi130 bash[27579]: audit 2023-08-27T21:51:20.644023+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.130:6816/1800436766,v1:172.21.15.130:6817/1800436766]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:51:21.985 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:21 smithi130 bash[27579]: cluster 2023-08-27T21:51:21.220823+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-27T21:51:22.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:22 smithi006 bash[33918]: audit 2023-08-27T21:51:21.643912+0000 mon.a (mon.0) 483 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-08-27T21:51:22.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:22 smithi006 bash[33918]: cluster 2023-08-27T21:51:21.644147+0000 mon.a (mon.0) 484 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-08-27T21:51:22.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:22 smithi006 bash[33918]: audit 2023-08-27T21:51:21.644997+0000 mon.a (mon.0) 485 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:51:22.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:22 smithi006 bash[33918]: audit 2023-08-27T21:51:21.651839+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:51:22.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:22 smithi006 bash[33918]: cluster 2023-08-27T21:51:22.410844+0000 mon.a (mon.0) 487 : cluster [INF] osd.6 [v2:172.21.15.130:6816/1800436766,v1:172.21.15.130:6817/1800436766] boot 2023-08-27T21:51:22.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:22 smithi006 bash[33918]: cluster 2023-08-27T21:51:22.410979+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-08-27T21:51:22.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:22 smithi006 bash[33918]: audit 2023-08-27T21:51:22.411715+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:51:22.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:22 smithi006 bash[27292]: audit 2023-08-27T21:51:21.643912+0000 mon.a (mon.0) 483 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-08-27T21:51:22.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:22 smithi006 bash[27292]: cluster 2023-08-27T21:51:21.644147+0000 mon.a (mon.0) 484 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-08-27T21:51:22.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:22 smithi006 bash[27292]: audit 2023-08-27T21:51:21.644997+0000 mon.a (mon.0) 485 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:51:22.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:22 smithi006 bash[27292]: audit 2023-08-27T21:51:21.651839+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:51:22.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:22 smithi006 bash[27292]: cluster 2023-08-27T21:51:22.410844+0000 mon.a (mon.0) 487 : cluster [INF] osd.6 [v2:172.21.15.130:6816/1800436766,v1:172.21.15.130:6817/1800436766] boot 2023-08-27T21:51:22.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:22 smithi006 bash[27292]: cluster 2023-08-27T21:51:22.410979+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-08-27T21:51:22.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:22 smithi006 bash[27292]: audit 2023-08-27T21:51:22.411715+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:51:23.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:22 smithi130 bash[27579]: audit 2023-08-27T21:51:21.643912+0000 mon.a (mon.0) 483 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-08-27T21:51:23.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:22 smithi130 bash[27579]: cluster 2023-08-27T21:51:21.644147+0000 mon.a (mon.0) 484 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-08-27T21:51:23.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:22 smithi130 bash[27579]: audit 2023-08-27T21:51:21.644997+0000 mon.a (mon.0) 485 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:51:23.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:22 smithi130 bash[27579]: audit 2023-08-27T21:51:21.651839+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:51:23.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:22 smithi130 bash[27579]: cluster 2023-08-27T21:51:22.410844+0000 mon.a (mon.0) 487 : cluster [INF] osd.6 [v2:172.21.15.130:6816/1800436766,v1:172.21.15.130:6817/1800436766] boot 2023-08-27T21:51:23.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:22 smithi130 bash[27579]: cluster 2023-08-27T21:51:22.410979+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-08-27T21:51:23.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:22 smithi130 bash[27579]: audit 2023-08-27T21:51:22.411715+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:51:23.358 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 6 on host 'smithi130' 2023-08-27T21:51:23.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:23 smithi130 bash[27579]: cluster 2023-08-27T21:51:21.327245+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:51:23.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:23 smithi130 bash[27579]: cluster 2023-08-27T21:51:21.327316+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:51:23.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:23 smithi130 bash[27579]: cluster 2023-08-27T21:51:23.221415+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:23.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:23 smithi130 bash[27579]: audit 2023-08-27T21:51:23.354158+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:51:23.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:23 smithi130 bash[27579]: audit 2023-08-27T21:51:23.363158+0000 mon.a (mon.0) 491 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:51:23.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:23 smithi130 bash[27579]: cluster 2023-08-27T21:51:23.414101+0000 mon.a (mon.0) 492 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-08-27T21:51:23.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:23 smithi006 bash[27292]: cluster 2023-08-27T21:51:21.327245+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:51:23.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:23 smithi006 bash[27292]: cluster 2023-08-27T21:51:21.327316+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:51:23.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:23 smithi006 bash[27292]: cluster 2023-08-27T21:51:23.221415+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:23.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:23 smithi006 bash[27292]: audit 2023-08-27T21:51:23.354158+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:51:23.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:23 smithi006 bash[27292]: audit 2023-08-27T21:51:23.363158+0000 mon.a (mon.0) 491 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:51:23.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:23 smithi006 bash[27292]: cluster 2023-08-27T21:51:23.414101+0000 mon.a (mon.0) 492 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-08-27T21:51:23.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:23 smithi006 bash[33918]: cluster 2023-08-27T21:51:21.327245+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:51:23.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:23 smithi006 bash[33918]: cluster 2023-08-27T21:51:21.327316+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:51:23.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:23 smithi006 bash[33918]: cluster 2023-08-27T21:51:23.221415+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:23.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:23 smithi006 bash[33918]: audit 2023-08-27T21:51:23.354158+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:51:23.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:23 smithi006 bash[33918]: audit 2023-08-27T21:51:23.363158+0000 mon.a (mon.0) 491 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:51:23.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:23 smithi006 bash[33918]: cluster 2023-08-27T21:51:23.414101+0000 mon.a (mon.0) 492 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-08-27T21:51:24.046 DEBUG:teuthology.orchestra.run.smithi130:osd.6> sudo journalctl -f -n 0 -u ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@osd.6.service 2023-08-27T21:51:24.049 INFO:tasks.cephadm:Deploying osd.7 on smithi130 with /dev/vg_nvme/lv_1... 2023-08-27T21:51:24.049 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-08-27T21:51:24.577 INFO:journalctl@ceph.osd.6.smithi130.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-08-27T21:51:25.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:25 smithi130 bash[27579]: cluster 2023-08-27T21:51:24.418449+0000 mon.a (mon.0) 493 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-08-27T21:51:25.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:25 smithi006 bash[33918]: cluster 2023-08-27T21:51:24.418449+0000 mon.a (mon.0) 493 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-08-27T21:51:25.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:25 smithi006 bash[27292]: cluster 2023-08-27T21:51:24.418449+0000 mon.a (mon.0) 493 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-08-27T21:51:26.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:26 smithi130 bash[27579]: cluster 2023-08-27T21:51:25.222065+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:26 smithi006 bash[27292]: cluster 2023-08-27T21:51:25.222065+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:26 smithi006 bash[33918]: cluster 2023-08-27T21:51:25.222065+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:27.744 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-08-27T21:51:27.744 INFO:teuthology.orchestra.run.smithi130.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-08-27T21:51:27.744 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10+0 records in 2023-08-27T21:51:27.744 INFO:teuthology.orchestra.run.smithi130.stderr:10+0 records out 2023-08-27T21:51:27.744 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0173263 s, 605 MB/s 2023-08-27T21:51:27.744 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping successful for: 2023-08-27T21:51:28.477 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph orch daemon add osd smithi130:vg_nvme/lv_1 2023-08-27T21:51:28.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:28 smithi006 bash[33918]: cluster 2023-08-27T21:51:27.222586+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:28.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:28 smithi006 bash[27292]: cluster 2023-08-27T21:51:27.222586+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:28.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:28 smithi130 bash[27579]: cluster 2023-08-27T21:51:27.222586+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:30.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:30 smithi006 bash[27292]: cluster 2023-08-27T21:51:29.223119+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:30.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:30 smithi006 bash[33918]: cluster 2023-08-27T21:51:29.223119+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:30 smithi130 bash[27579]: cluster 2023-08-27T21:51:29.223119+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:32 smithi006 bash[33918]: cluster 2023-08-27T21:51:31.223666+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:32 smithi006 bash[27292]: cluster 2023-08-27T21:51:31.223666+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:32.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:32 smithi130 bash[27579]: cluster 2023-08-27T21:51:31.223666+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:34.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:34 smithi130 bash[27579]: cluster 2023-08-27T21:51:33.224352+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:34.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:34 smithi130 bash[27579]: audit 2023-08-27T21:51:33.570531+0000 mon.a (mon.0) 494 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:51:34.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:34 smithi130 bash[27579]: audit 2023-08-27T21:51:33.575820+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:51:34.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:34 smithi130 bash[27579]: audit 2023-08-27T21:51:33.577138+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:51:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:34 smithi006 bash[27292]: cluster 2023-08-27T21:51:33.224352+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:34 smithi006 bash[27292]: audit 2023-08-27T21:51:33.570531+0000 mon.a (mon.0) 494 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:51:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:34 smithi006 bash[27292]: audit 2023-08-27T21:51:33.575820+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:51:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:34 smithi006 bash[27292]: audit 2023-08-27T21:51:33.577138+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:51:34.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:34 smithi006 bash[33918]: cluster 2023-08-27T21:51:33.224352+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:34.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:34 smithi006 bash[33918]: audit 2023-08-27T21:51:33.570531+0000 mon.a (mon.0) 494 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T21:51:34.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:34 smithi006 bash[33918]: audit 2023-08-27T21:51:33.575820+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T21:51:34.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:34 smithi006 bash[33918]: audit 2023-08-27T21:51:33.577138+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:51:35.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[27292]: audit 2023-08-27T21:51:33.563539+0000 mgr.y (mgr.14146) 248 : audit [DBG] from='client.24298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:51:35.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[27292]: cephadm 2023-08-27T21:51:35.164991+0000 mgr.y (mgr.14146) 249 : cephadm [INF] Detected new or changed devices on smithi130 2023-08-27T21:51:35.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[27292]: audit 2023-08-27T21:51:35.175136+0000 mon.a (mon.0) 497 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:51:35.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[27292]: audit 2023-08-27T21:51:35.177047+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:51:35.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[27292]: audit 2023-08-27T21:51:35.178556+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:51:35.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[27292]: audit 2023-08-27T21:51:35.179977+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:51:35.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[27292]: cephadm 2023-08-27T21:51:35.180991+0000 mgr.y (mgr.14146) 250 : cephadm [INF] Adjusting osd_memory_target on smithi130 to 5755M 2023-08-27T21:51:35.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[27292]: audit 2023-08-27T21:51:35.186907+0000 mon.a (mon.0) 501 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:51:35.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[27292]: audit 2023-08-27T21:51:35.188542+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:51:35.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[27292]: audit 2023-08-27T21:51:35.190076+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:51:35.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[27292]: audit 2023-08-27T21:51:35.198610+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:51:35.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[27292]: cluster 2023-08-27T21:51:35.224868+0000 mgr.y (mgr.14146) 251 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:35.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[33918]: audit 2023-08-27T21:51:33.563539+0000 mgr.y (mgr.14146) 248 : audit [DBG] from='client.24298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:51:35.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[33918]: cephadm 2023-08-27T21:51:35.164991+0000 mgr.y (mgr.14146) 249 : cephadm [INF] Detected new or changed devices on smithi130 2023-08-27T21:51:35.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[33918]: audit 2023-08-27T21:51:35.175136+0000 mon.a (mon.0) 497 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:51:35.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[33918]: audit 2023-08-27T21:51:35.177047+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:51:35.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[33918]: audit 2023-08-27T21:51:35.178556+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:51:35.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[33918]: audit 2023-08-27T21:51:35.179977+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:51:35.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[33918]: cephadm 2023-08-27T21:51:35.180991+0000 mgr.y (mgr.14146) 250 : cephadm [INF] Adjusting osd_memory_target on smithi130 to 5755M 2023-08-27T21:51:35.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[33918]: audit 2023-08-27T21:51:35.186907+0000 mon.a (mon.0) 501 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:51:35.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[33918]: audit 2023-08-27T21:51:35.188542+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:51:35.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[33918]: audit 2023-08-27T21:51:35.190076+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:51:35.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[33918]: audit 2023-08-27T21:51:35.198610+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:51:35.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:35 smithi006 bash[33918]: cluster 2023-08-27T21:51:35.224868+0000 mgr.y (mgr.14146) 251 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:35.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:35 smithi130 bash[27579]: audit 2023-08-27T21:51:33.563539+0000 mgr.y (mgr.14146) 248 : audit [DBG] from='client.24298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:51:35.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:35 smithi130 bash[27579]: cephadm 2023-08-27T21:51:35.164991+0000 mgr.y (mgr.14146) 249 : cephadm [INF] Detected new or changed devices on smithi130 2023-08-27T21:51:35.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:35 smithi130 bash[27579]: audit 2023-08-27T21:51:35.175136+0000 mon.a (mon.0) 497 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:51:35.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:35 smithi130 bash[27579]: audit 2023-08-27T21:51:35.177047+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:51:35.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:35 smithi130 bash[27579]: audit 2023-08-27T21:51:35.178556+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:51:35.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:35 smithi130 bash[27579]: audit 2023-08-27T21:51:35.179977+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:51:35.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:35 smithi130 bash[27579]: cephadm 2023-08-27T21:51:35.180991+0000 mgr.y (mgr.14146) 250 : cephadm [INF] Adjusting osd_memory_target on smithi130 to 5755M 2023-08-27T21:51:35.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:35 smithi130 bash[27579]: audit 2023-08-27T21:51:35.186907+0000 mon.a (mon.0) 501 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:51:35.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:35 smithi130 bash[27579]: audit 2023-08-27T21:51:35.188542+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:51:35.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:35 smithi130 bash[27579]: audit 2023-08-27T21:51:35.190076+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:51:35.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:35 smithi130 bash[27579]: audit 2023-08-27T21:51:35.198610+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:51:35.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:35 smithi130 bash[27579]: cluster 2023-08-27T21:51:35.224868+0000 mgr.y (mgr.14146) 251 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:38.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:38 smithi006 bash[27292]: cluster 2023-08-27T21:51:37.225414+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:38.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:38 smithi006 bash[33918]: cluster 2023-08-27T21:51:37.225414+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:38.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:38 smithi130 bash[27579]: cluster 2023-08-27T21:51:37.225414+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:40 smithi006 bash[27292]: cluster 2023-08-27T21:51:39.225994+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v238: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:40 smithi006 bash[33918]: cluster 2023-08-27T21:51:39.225994+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v238: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:40.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:40 smithi130 bash[27579]: cluster 2023-08-27T21:51:39.225994+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v238: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:42.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:42 smithi006 bash[27292]: cluster 2023-08-27T21:51:41.226537+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:42.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:42 smithi006 bash[27292]: audit 2023-08-27T21:51:42.236352+0000 mon.a (mon.0) 505 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ac7e358c-d0fd-4b6c-a2b5-7006c5779948"}]: dispatch 2023-08-27T21:51:42.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:42 smithi006 bash[27292]: audit 2023-08-27T21:51:42.236508+0000 mon.b (mon.1) 20 : audit [INF] from='client.? 172.21.15.130:0/298762327' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ac7e358c-d0fd-4b6c-a2b5-7006c5779948"}]: dispatch 2023-08-27T21:51:42.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:42 smithi006 bash[27292]: audit 2023-08-27T21:51:42.243499+0000 mon.a (mon.0) 506 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ac7e358c-d0fd-4b6c-a2b5-7006c5779948"}]': finished 2023-08-27T21:51:42.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:42 smithi006 bash[27292]: cluster 2023-08-27T21:51:42.243600+0000 mon.a (mon.0) 507 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-08-27T21:51:42.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:42 smithi006 bash[27292]: audit 2023-08-27T21:51:42.243895+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:51:42.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:42 smithi006 bash[33918]: cluster 2023-08-27T21:51:41.226537+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:42.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:42 smithi006 bash[33918]: audit 2023-08-27T21:51:42.236352+0000 mon.a (mon.0) 505 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ac7e358c-d0fd-4b6c-a2b5-7006c5779948"}]: dispatch 2023-08-27T21:51:42.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:42 smithi006 bash[33918]: audit 2023-08-27T21:51:42.236508+0000 mon.b (mon.1) 20 : audit [INF] from='client.? 172.21.15.130:0/298762327' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ac7e358c-d0fd-4b6c-a2b5-7006c5779948"}]: dispatch 2023-08-27T21:51:42.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:42 smithi006 bash[33918]: audit 2023-08-27T21:51:42.243499+0000 mon.a (mon.0) 506 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ac7e358c-d0fd-4b6c-a2b5-7006c5779948"}]': finished 2023-08-27T21:51:42.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:42 smithi006 bash[33918]: cluster 2023-08-27T21:51:42.243600+0000 mon.a (mon.0) 507 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-08-27T21:51:42.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:42 smithi006 bash[33918]: audit 2023-08-27T21:51:42.243895+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:51:42.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:42 smithi130 bash[27579]: cluster 2023-08-27T21:51:41.226537+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:42.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:42 smithi130 bash[27579]: audit 2023-08-27T21:51:42.236352+0000 mon.a (mon.0) 505 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ac7e358c-d0fd-4b6c-a2b5-7006c5779948"}]: dispatch 2023-08-27T21:51:42.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:42 smithi130 bash[27579]: audit 2023-08-27T21:51:42.236508+0000 mon.b (mon.1) 20 : audit [INF] from='client.? 172.21.15.130:0/298762327' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ac7e358c-d0fd-4b6c-a2b5-7006c5779948"}]: dispatch 2023-08-27T21:51:42.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:42 smithi130 bash[27579]: audit 2023-08-27T21:51:42.243499+0000 mon.a (mon.0) 506 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ac7e358c-d0fd-4b6c-a2b5-7006c5779948"}]': finished 2023-08-27T21:51:42.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:42 smithi130 bash[27579]: cluster 2023-08-27T21:51:42.243600+0000 mon.a (mon.0) 507 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-08-27T21:51:42.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:42 smithi130 bash[27579]: audit 2023-08-27T21:51:42.243895+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:51:43.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:43 smithi006 bash[27292]: audit 2023-08-27T21:51:42.960076+0000 mon.b (mon.1) 21 : audit [DBG] from='client.? 172.21.15.130:0/2001152858' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:51:43.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:43 smithi006 bash[33918]: audit 2023-08-27T21:51:42.960076+0000 mon.b (mon.1) 21 : audit [DBG] from='client.? 172.21.15.130:0/2001152858' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:51:43.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:43 smithi130 bash[27579]: audit 2023-08-27T21:51:42.960076+0000 mon.b (mon.1) 21 : audit [DBG] from='client.? 172.21.15.130:0/2001152858' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T21:51:44.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:44 smithi006 bash[27292]: cluster 2023-08-27T21:51:43.227110+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:44.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:44 smithi006 bash[33918]: cluster 2023-08-27T21:51:43.227110+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:44.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:44 smithi130 bash[27579]: cluster 2023-08-27T21:51:43.227110+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:46 smithi006 bash[27292]: cluster 2023-08-27T21:51:45.227691+0000 mgr.y (mgr.14146) 256 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:46.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:46 smithi006 bash[33918]: cluster 2023-08-27T21:51:45.227691+0000 mgr.y (mgr.14146) 256 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:46.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:46 smithi130 bash[27579]: cluster 2023-08-27T21:51:45.227691+0000 mgr.y (mgr.14146) 256 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:48.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:48 smithi006 bash[27292]: cluster 2023-08-27T21:51:47.228235+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:48 smithi006 bash[33918]: cluster 2023-08-27T21:51:47.228235+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:48.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:48 smithi130 bash[27579]: cluster 2023-08-27T21:51:47.228235+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:50 smithi006 bash[27292]: cluster 2023-08-27T21:51:49.228809+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:50 smithi006 bash[33918]: cluster 2023-08-27T21:51:49.228809+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:50.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:50 smithi130 bash[27579]: cluster 2023-08-27T21:51:49.228809+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:51.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:51 smithi006 bash[27292]: cluster 2023-08-27T21:51:51.229391+0000 mgr.y (mgr.14146) 259 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:51.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:51 smithi006 bash[33918]: cluster 2023-08-27T21:51:51.229391+0000 mgr.y (mgr.14146) 259 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:51.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:51 smithi130 bash[27579]: cluster 2023-08-27T21:51:51.229391+0000 mgr.y (mgr.14146) 259 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:54.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:54 smithi006 bash[27292]: cluster 2023-08-27T21:51:53.229934+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:54.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:54 smithi006 bash[33918]: cluster 2023-08-27T21:51:53.229934+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:54.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:54 smithi130 bash[27579]: cluster 2023-08-27T21:51:53.229934+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:55.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:55 smithi006 bash[27292]: audit 2023-08-27T21:51:55.132744+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-08-27T21:51:55.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:55 smithi006 bash[27292]: audit 2023-08-27T21:51:55.134956+0000 mon.a (mon.0) 510 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:51:55.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:55 smithi006 bash[33918]: audit 2023-08-27T21:51:55.132744+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-08-27T21:51:55.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:55 smithi006 bash[33918]: audit 2023-08-27T21:51:55.134956+0000 mon.a (mon.0) 510 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:51:55.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:55 smithi130 bash[27579]: audit 2023-08-27T21:51:55.132744+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-08-27T21:51:55.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:55 smithi130 bash[27579]: audit 2023-08-27T21:51:55.134956+0000 mon.a (mon.0) 510 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:51:56.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:56 smithi130 bash[27579]: cephadm 2023-08-27T21:51:55.136636+0000 mgr.y (mgr.14146) 261 : cephadm [INF] Deploying daemon osd.7 on smithi130 2023-08-27T21:51:56.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:56 smithi130 bash[27579]: cluster 2023-08-27T21:51:55.230460+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:56 smithi006 bash[27292]: cephadm 2023-08-27T21:51:55.136636+0000 mgr.y (mgr.14146) 261 : cephadm [INF] Deploying daemon osd.7 on smithi130 2023-08-27T21:51:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:56 smithi006 bash[27292]: cluster 2023-08-27T21:51:55.230460+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:56.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:56 smithi006 bash[33918]: cephadm 2023-08-27T21:51:55.136636+0000 mgr.y (mgr.14146) 261 : cephadm [INF] Deploying daemon osd.7 on smithi130 2023-08-27T21:51:56.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:56 smithi006 bash[33918]: cluster 2023-08-27T21:51:55.230460+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:58.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:58 smithi006 bash[27292]: cluster 2023-08-27T21:51:57.231036+0000 mgr.y (mgr.14146) 263 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:58.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:58 smithi006 bash[33918]: cluster 2023-08-27T21:51:57.231036+0000 mgr.y (mgr.14146) 263 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:58.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:58 smithi130 bash[27579]: cluster 2023-08-27T21:51:57.231036+0000 mgr.y (mgr.14146) 263 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:59.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:51:59 smithi130 bash[27579]: cluster 2023-08-27T21:51:59.231541+0000 mgr.y (mgr.14146) 264 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:59.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:51:59 smithi006 bash[27292]: cluster 2023-08-27T21:51:59.231541+0000 mgr.y (mgr.14146) 264 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:51:59.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:51:59 smithi006 bash[33918]: cluster 2023-08-27T21:51:59.231541+0000 mgr.y (mgr.14146) 264 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:52:01.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:00 smithi130 bash[27579]: audit 2023-08-27T21:51:59.793401+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:01.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:00 smithi130 bash[27579]: audit 2023-08-27T21:51:59.793822+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:52:01.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:00 smithi006 bash[27292]: audit 2023-08-27T21:51:59.793401+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:01.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:00 smithi006 bash[27292]: audit 2023-08-27T21:51:59.793822+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:52:01.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:00 smithi006 bash[33918]: audit 2023-08-27T21:51:59.793401+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:01.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:00 smithi006 bash[33918]: audit 2023-08-27T21:51:59.793822+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:52:02.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:01 smithi130 bash[27579]: cluster 2023-08-27T21:52:01.232068+0000 mgr.y (mgr.14146) 265 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:52:02.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:01 smithi006 bash[27292]: cluster 2023-08-27T21:52:01.232068+0000 mgr.y (mgr.14146) 265 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:52:02.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:01 smithi006 bash[33918]: cluster 2023-08-27T21:52:01.232068+0000 mgr.y (mgr.14146) 265 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:52:04.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:04 smithi006 bash[33918]: cluster 2023-08-27T21:52:03.232604+0000 mgr.y (mgr.14146) 266 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:52:04.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:04 smithi006 bash[27292]: cluster 2023-08-27T21:52:03.232604+0000 mgr.y (mgr.14146) 266 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:52:04.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:04 smithi130 bash[27579]: cluster 2023-08-27T21:52:03.232604+0000 mgr.y (mgr.14146) 266 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:52:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:06 smithi006 bash[27292]: cluster 2023-08-27T21:52:05.233194+0000 mgr.y (mgr.14146) 267 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:52:06.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:06 smithi006 bash[27292]: audit 2023-08-27T21:52:06.131306+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:06.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:06 smithi006 bash[27292]: audit 2023-08-27T21:52:06.390565+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:52:06.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:06 smithi006 bash[27292]: audit 2023-08-27T21:52:06.392394+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:52:06.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:06 smithi006 bash[27292]: audit 2023-08-27T21:52:06.400399+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:06.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:06 smithi006 bash[33918]: cluster 2023-08-27T21:52:05.233194+0000 mgr.y (mgr.14146) 267 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:52:06.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:06 smithi006 bash[33918]: audit 2023-08-27T21:52:06.131306+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:06.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:06 smithi006 bash[33918]: audit 2023-08-27T21:52:06.390565+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:52:06.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:06 smithi006 bash[33918]: audit 2023-08-27T21:52:06.392394+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:52:06.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:06 smithi006 bash[33918]: audit 2023-08-27T21:52:06.400399+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:06.798 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:06 smithi130 bash[27579]: cluster 2023-08-27T21:52:05.233194+0000 mgr.y (mgr.14146) 267 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:52:06.798 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:06 smithi130 bash[27579]: audit 2023-08-27T21:52:06.131306+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:06.798 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:06 smithi130 bash[27579]: audit 2023-08-27T21:52:06.390565+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:52:06.798 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:06 smithi130 bash[27579]: audit 2023-08-27T21:52:06.392394+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:52:06.799 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:06 smithi130 bash[27579]: audit 2023-08-27T21:52:06.400399+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:07.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:07 smithi130 bash[27579]: audit 2023-08-27T21:52:06.800268+0000 mon.a (mon.0) 517 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-08-27T21:52:07.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:07 smithi130 bash[27579]: audit 2023-08-27T21:52:06.800679+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.130:6824/2598634202,v1:172.21.15.130:6825/2598634202]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-08-27T21:52:07.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:07 smithi006 bash[33918]: audit 2023-08-27T21:52:06.800268+0000 mon.a (mon.0) 517 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-08-27T21:52:07.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:07 smithi006 bash[33918]: audit 2023-08-27T21:52:06.800679+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.130:6824/2598634202,v1:172.21.15.130:6825/2598634202]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-08-27T21:52:07.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:07 smithi006 bash[27292]: audit 2023-08-27T21:52:06.800268+0000 mon.a (mon.0) 517 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-08-27T21:52:07.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:07 smithi006 bash[27292]: audit 2023-08-27T21:52:06.800679+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.130:6824/2598634202,v1:172.21.15.130:6825/2598634202]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-08-27T21:52:08.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:08 smithi006 bash[27292]: cluster 2023-08-27T21:52:07.233759+0000 mgr.y (mgr.14146) 268 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:52:08.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:08 smithi006 bash[27292]: audit 2023-08-27T21:52:07.449067+0000 mon.a (mon.0) 518 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-08-27T21:52:08.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:08 smithi006 bash[27292]: cluster 2023-08-27T21:52:07.449156+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-08-27T21:52:08.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:08 smithi006 bash[27292]: audit 2023-08-27T21:52:07.449316+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:52:08.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:08 smithi006 bash[27292]: audit 2023-08-27T21:52:07.450752+0000 mon.a (mon.0) 521 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:52:08.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:08 smithi006 bash[27292]: audit 2023-08-27T21:52:07.451248+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.130:6824/2598634202,v1:172.21.15.130:6825/2598634202]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:52:08.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:08 smithi006 bash[33918]: cluster 2023-08-27T21:52:07.233759+0000 mgr.y (mgr.14146) 268 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:52:08.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:08 smithi006 bash[33918]: audit 2023-08-27T21:52:07.449067+0000 mon.a (mon.0) 518 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-08-27T21:52:08.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:08 smithi006 bash[33918]: cluster 2023-08-27T21:52:07.449156+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-08-27T21:52:08.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:08 smithi006 bash[33918]: audit 2023-08-27T21:52:07.449316+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:52:08.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:08 smithi006 bash[33918]: audit 2023-08-27T21:52:07.450752+0000 mon.a (mon.0) 521 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:52:08.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:08 smithi006 bash[33918]: audit 2023-08-27T21:52:07.451248+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.130:6824/2598634202,v1:172.21.15.130:6825/2598634202]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:52:08.794 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:08 smithi130 bash[27579]: cluster 2023-08-27T21:52:07.233759+0000 mgr.y (mgr.14146) 268 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:52:08.794 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:08 smithi130 bash[27579]: audit 2023-08-27T21:52:07.449067+0000 mon.a (mon.0) 518 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-08-27T21:52:08.794 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:08 smithi130 bash[27579]: cluster 2023-08-27T21:52:07.449156+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-08-27T21:52:08.794 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:08 smithi130 bash[27579]: audit 2023-08-27T21:52:07.449316+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:52:08.795 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:08 smithi130 bash[27579]: audit 2023-08-27T21:52:07.450752+0000 mon.a (mon.0) 521 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:52:08.795 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:08 smithi130 bash[27579]: audit 2023-08-27T21:52:07.451248+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.130:6824/2598634202,v1:172.21.15.130:6825/2598634202]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-08-27T21:52:09.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:09 smithi006 bash[27292]: audit 2023-08-27T21:52:08.448073+0000 mon.a (mon.0) 522 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-08-27T21:52:09.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:09 smithi006 bash[27292]: cluster 2023-08-27T21:52:08.448213+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-08-27T21:52:09.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:09 smithi006 bash[27292]: audit 2023-08-27T21:52:08.448959+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:52:09.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:09 smithi006 bash[27292]: audit 2023-08-27T21:52:08.457157+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:52:09.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:09 smithi006 bash[27292]: cluster 2023-08-27T21:52:09.234310+0000 mgr.y (mgr.14146) 269 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:52:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:09 smithi006 bash[33918]: audit 2023-08-27T21:52:08.448073+0000 mon.a (mon.0) 522 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-08-27T21:52:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:09 smithi006 bash[33918]: cluster 2023-08-27T21:52:08.448213+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-08-27T21:52:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:09 smithi006 bash[33918]: audit 2023-08-27T21:52:08.448959+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:52:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:09 smithi006 bash[33918]: audit 2023-08-27T21:52:08.457157+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:52:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:09 smithi006 bash[33918]: cluster 2023-08-27T21:52:09.234310+0000 mgr.y (mgr.14146) 269 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:52:09.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:09 smithi130 bash[27579]: audit 2023-08-27T21:52:08.448073+0000 mon.a (mon.0) 522 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-08-27T21:52:09.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:09 smithi130 bash[27579]: cluster 2023-08-27T21:52:08.448213+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-08-27T21:52:09.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:09 smithi130 bash[27579]: audit 2023-08-27T21:52:08.448959+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:52:09.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:09 smithi130 bash[27579]: audit 2023-08-27T21:52:08.457157+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:52:09.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:09 smithi130 bash[27579]: cluster 2023-08-27T21:52:09.234310+0000 mgr.y (mgr.14146) 269 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-27T21:52:10.105 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 7 on host 'smithi130' 2023-08-27T21:52:10.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:10 smithi130 bash[27579]: cluster 2023-08-27T21:52:07.813375+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:52:10.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:10 smithi130 bash[27579]: cluster 2023-08-27T21:52:07.813487+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:52:10.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:10 smithi130 bash[27579]: cluster 2023-08-27T21:52:09.459217+0000 mon.a (mon.0) 526 : cluster [INF] osd.7 [v2:172.21.15.130:6824/2598634202,v1:172.21.15.130:6825/2598634202] boot 2023-08-27T21:52:10.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:10 smithi130 bash[27579]: cluster 2023-08-27T21:52:09.459387+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-08-27T21:52:10.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:10 smithi130 bash[27579]: audit 2023-08-27T21:52:09.460262+0000 mon.a (mon.0) 528 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:52:10.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:10 smithi130 bash[27579]: audit 2023-08-27T21:52:10.100143+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:10.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:10 smithi130 bash[27579]: audit 2023-08-27T21:52:10.128789+0000 mon.a (mon.0) 530 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:52:10.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:10 smithi130 bash[27579]: cluster 2023-08-27T21:52:10.461533+0000 mon.a (mon.0) 531 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-08-27T21:52:10.869 DEBUG:teuthology.orchestra.run.smithi130:osd.7> sudo journalctl -f -n 0 -u ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@osd.7.service 2023-08-27T21:52:10.914 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-08-27T21:52:10.915 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd stat -f json 2023-08-27T21:52:10.926 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:10 smithi006 bash[33918]: cluster 2023-08-27T21:52:07.813375+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:52:10.927 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:10 smithi006 bash[33918]: cluster 2023-08-27T21:52:07.813487+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:52:10.927 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:10 smithi006 bash[33918]: cluster 2023-08-27T21:52:09.459217+0000 mon.a (mon.0) 526 : cluster [INF] osd.7 [v2:172.21.15.130:6824/2598634202,v1:172.21.15.130:6825/2598634202] boot 2023-08-27T21:52:10.927 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:10 smithi006 bash[33918]: cluster 2023-08-27T21:52:09.459387+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-08-27T21:52:10.927 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:10 smithi006 bash[33918]: audit 2023-08-27T21:52:09.460262+0000 mon.a (mon.0) 528 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:52:10.927 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:10 smithi006 bash[33918]: audit 2023-08-27T21:52:10.100143+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:10.927 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:10 smithi006 bash[33918]: audit 2023-08-27T21:52:10.128789+0000 mon.a (mon.0) 530 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:52:10.927 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:10 smithi006 bash[33918]: cluster 2023-08-27T21:52:10.461533+0000 mon.a (mon.0) 531 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-08-27T21:52:10.927 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:10 smithi006 bash[27292]: cluster 2023-08-27T21:52:07.813375+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-27T21:52:10.927 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:10 smithi006 bash[27292]: cluster 2023-08-27T21:52:07.813487+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-27T21:52:10.928 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:10 smithi006 bash[27292]: cluster 2023-08-27T21:52:09.459217+0000 mon.a (mon.0) 526 : cluster [INF] osd.7 [v2:172.21.15.130:6824/2598634202,v1:172.21.15.130:6825/2598634202] boot 2023-08-27T21:52:10.928 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:10 smithi006 bash[27292]: cluster 2023-08-27T21:52:09.459387+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-08-27T21:52:10.928 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:10 smithi006 bash[27292]: audit 2023-08-27T21:52:09.460262+0000 mon.a (mon.0) 528 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:52:10.928 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:10 smithi006 bash[27292]: audit 2023-08-27T21:52:10.100143+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:10.928 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:10 smithi006 bash[27292]: audit 2023-08-27T21:52:10.128789+0000 mon.a (mon.0) 530 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:52:10.928 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:10 smithi006 bash[27292]: cluster 2023-08-27T21:52:10.461533+0000 mon.a (mon.0) 531 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-08-27T21:52:11.140 INFO:journalctl@ceph.osd.7.smithi130.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-08-27T21:52:11.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:11 smithi006 bash[27292]: cluster 2023-08-27T21:52:11.234858+0000 mgr.y (mgr.14146) 270 : cluster [DBG] pgmap v259: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:11.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:11 smithi006 bash[27292]: cluster 2023-08-27T21:52:11.461722+0000 mon.a (mon.0) 532 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-08-27T21:52:11.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:11 smithi006 bash[33918]: cluster 2023-08-27T21:52:11.234858+0000 mgr.y (mgr.14146) 270 : cluster [DBG] pgmap v259: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:11.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:11 smithi006 bash[33918]: cluster 2023-08-27T21:52:11.461722+0000 mon.a (mon.0) 532 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-08-27T21:52:11.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:11 smithi130 bash[27579]: cluster 2023-08-27T21:52:11.234858+0000 mgr.y (mgr.14146) 270 : cluster [DBG] pgmap v259: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:11.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:11 smithi130 bash[27579]: cluster 2023-08-27T21:52:11.461722+0000 mon.a (mon.0) 532 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-08-27T21:52:12.690 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:12 smithi006 bash[33918]: cluster 2023-08-27T21:52:11.464821+0000 mon.a (mon.0) 533 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-08-27T21:52:12.690 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:12 smithi006 bash[27292]: cluster 2023-08-27T21:52:11.464821+0000 mon.a (mon.0) 533 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-08-27T21:52:12.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:12 smithi130 bash[27579]: cluster 2023-08-27T21:52:11.464821+0000 mon.a (mon.0) 533 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-08-27T21:52:13.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:13 smithi130 bash[27579]: cluster 2023-08-27T21:52:13.235611+0000 mgr.y (mgr.14146) 271 : cluster [DBG] pgmap v261: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:13.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:13 smithi006 bash[27292]: cluster 2023-08-27T21:52:13.235611+0000 mgr.y (mgr.14146) 271 : cluster [DBG] pgmap v261: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:13.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:13 smithi006 bash[33918]: cluster 2023-08-27T21:52:13.235611+0000 mgr.y (mgr.14146) 271 : cluster [DBG] pgmap v261: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:14.456 INFO:teuthology.orchestra.run.smithi006.stdout: 2023-08-27T21:52:14.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:14 smithi130 bash[27579]: audit 2023-08-27T21:52:14.451695+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.6:0/1639708961' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-08-27T21:52:14.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:14 smithi006 bash[33918]: audit 2023-08-27T21:52:14.451695+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.6:0/1639708961' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-08-27T21:52:14.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:14 smithi006 bash[27292]: audit 2023-08-27T21:52:14.451695+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.6:0/1639708961' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-08-27T21:52:15.237 INFO:teuthology.orchestra.run.smithi006.stdout:{"epoch":50,"num_osds":8,"num_up_osds":8,"osd_up_since":1693173129,"num_in_osds":8,"osd_in_since":1693173102,"num_remapped_pgs":0} 2023-08-27T21:52:15.237 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd dump --format=json 2023-08-27T21:52:15.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:15 smithi130 bash[27579]: cluster 2023-08-27T21:52:15.236268+0000 mgr.y (mgr.14146) 272 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:15.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:15 smithi006 bash[27292]: cluster 2023-08-27T21:52:15.236268+0000 mgr.y (mgr.14146) 272 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:15.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:15 smithi006 bash[33918]: cluster 2023-08-27T21:52:15.236268+0000 mgr.y (mgr.14146) 272 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:16.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:16 smithi130 bash[27579]: cluster 2023-08-27T21:52:15.491149+0000 mon.a (mon.0) 534 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-08-27T21:52:16.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:16 smithi130 bash[27579]: cluster 2023-08-27T21:52:15.491195+0000 mon.a (mon.0) 535 : cluster [INF] Cluster is now healthy 2023-08-27T21:52:16.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:16 smithi006 bash[27292]: cluster 2023-08-27T21:52:15.491149+0000 mon.a (mon.0) 534 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-08-27T21:52:16.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:16 smithi006 bash[27292]: cluster 2023-08-27T21:52:15.491195+0000 mon.a (mon.0) 535 : cluster [INF] Cluster is now healthy 2023-08-27T21:52:16.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:16 smithi006 bash[33918]: cluster 2023-08-27T21:52:15.491149+0000 mon.a (mon.0) 534 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-08-27T21:52:16.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:16 smithi006 bash[33918]: cluster 2023-08-27T21:52:15.491195+0000 mon.a (mon.0) 535 : cluster [INF] Cluster is now healthy 2023-08-27T21:52:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:18 smithi006 bash[27292]: cluster 2023-08-27T21:52:17.236911+0000 mgr.y (mgr.14146) 273 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:18.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:18 smithi006 bash[33918]: cluster 2023-08-27T21:52:17.236911+0000 mgr.y (mgr.14146) 273 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:18.840 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:18 smithi130 bash[27579]: cluster 2023-08-27T21:52:17.236911+0000 mgr.y (mgr.14146) 273 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:20.078 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:52:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:20 smithi006 bash[27292]: cluster 2023-08-27T21:52:19.237475+0000 mgr.y (mgr.14146) 274 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:20.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:20 smithi006 bash[27292]: audit 2023-08-27T21:52:19.763545+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:20.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:20 smithi006 bash[27292]: audit 2023-08-27T21:52:19.765480+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:52:20.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:20 smithi006 bash[27292]: audit 2023-08-27T21:52:19.766779+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:52:20.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:20 smithi006 bash[27292]: audit 2023-08-27T21:52:19.768053+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:52:20.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:20 smithi006 bash[27292]: audit 2023-08-27T21:52:19.769402+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:52:20.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:20 smithi006 bash[27292]: audit 2023-08-27T21:52:19.777185+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:20.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:20 smithi006 bash[27292]: audit 2023-08-27T21:52:19.778808+0000 mon.a (mon.0) 542 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:52:20.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:20 smithi006 bash[27292]: audit 2023-08-27T21:52:19.780049+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:52:20.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:20 smithi006 bash[27292]: audit 2023-08-27T21:52:19.786147+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:20.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:20 smithi006 bash[33918]: cluster 2023-08-27T21:52:19.237475+0000 mgr.y (mgr.14146) 274 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:20.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:20 smithi006 bash[33918]: audit 2023-08-27T21:52:19.763545+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:20.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:20 smithi006 bash[33918]: audit 2023-08-27T21:52:19.765480+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:52:20.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:20 smithi006 bash[33918]: audit 2023-08-27T21:52:19.766779+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:52:20.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:20 smithi006 bash[33918]: audit 2023-08-27T21:52:19.768053+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:52:20.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:20 smithi006 bash[33918]: audit 2023-08-27T21:52:19.769402+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:52:20.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:20 smithi006 bash[33918]: audit 2023-08-27T21:52:19.777185+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:20.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:20 smithi006 bash[33918]: audit 2023-08-27T21:52:19.778808+0000 mon.a (mon.0) 542 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:52:20.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:20 smithi006 bash[33918]: audit 2023-08-27T21:52:19.780049+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:52:20.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:20 smithi006 bash[33918]: audit 2023-08-27T21:52:19.786147+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:20.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:20 smithi130 bash[27579]: cluster 2023-08-27T21:52:19.237475+0000 mgr.y (mgr.14146) 274 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:20.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:20 smithi130 bash[27579]: audit 2023-08-27T21:52:19.763545+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:20.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:20 smithi130 bash[27579]: audit 2023-08-27T21:52:19.765480+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:52:20.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:20 smithi130 bash[27579]: audit 2023-08-27T21:52:19.766779+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:52:20.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:20 smithi130 bash[27579]: audit 2023-08-27T21:52:19.768053+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:52:20.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:20 smithi130 bash[27579]: audit 2023-08-27T21:52:19.769402+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:52:20.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:20 smithi130 bash[27579]: audit 2023-08-27T21:52:19.777185+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:20.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:20 smithi130 bash[27579]: audit 2023-08-27T21:52:19.778808+0000 mon.a (mon.0) 542 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:52:20.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:20 smithi130 bash[27579]: audit 2023-08-27T21:52:19.780049+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:52:20.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:20 smithi130 bash[27579]: audit 2023-08-27T21:52:19.786147+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:21.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:21 smithi006 bash[27292]: cephadm 2023-08-27T21:52:19.753398+0000 mgr.y (mgr.14146) 275 : cephadm [INF] Detected new or changed devices on smithi130 2023-08-27T21:52:21.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:21 smithi006 bash[27292]: cephadm 2023-08-27T21:52:19.770387+0000 mgr.y (mgr.14146) 276 : cephadm [INF] Adjusting osd_memory_target on smithi130 to 4316M 2023-08-27T21:52:21.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:21 smithi006 bash[33918]: cephadm 2023-08-27T21:52:19.753398+0000 mgr.y (mgr.14146) 275 : cephadm [INF] Detected new or changed devices on smithi130 2023-08-27T21:52:21.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:21 smithi006 bash[33918]: cephadm 2023-08-27T21:52:19.770387+0000 mgr.y (mgr.14146) 276 : cephadm [INF] Adjusting osd_memory_target on smithi130 to 4316M 2023-08-27T21:52:21.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:21 smithi130 bash[27579]: cephadm 2023-08-27T21:52:19.753398+0000 mgr.y (mgr.14146) 275 : cephadm [INF] Detected new or changed devices on smithi130 2023-08-27T21:52:21.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:21 smithi130 bash[27579]: cephadm 2023-08-27T21:52:19.770387+0000 mgr.y (mgr.14146) 276 : cephadm [INF] Adjusting osd_memory_target on smithi130 to 4316M 2023-08-27T21:52:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:22 smithi006 bash[27292]: cluster 2023-08-27T21:52:21.238050+0000 mgr.y (mgr.14146) 277 : cluster [DBG] pgmap v265: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:22.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:22 smithi006 bash[33918]: cluster 2023-08-27T21:52:21.238050+0000 mgr.y (mgr.14146) 277 : cluster [DBG] pgmap v265: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:22.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:22 smithi130 bash[27579]: cluster 2023-08-27T21:52:21.238050+0000 mgr.y (mgr.14146) 277 : cluster [DBG] pgmap v265: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:23.303 INFO:teuthology.orchestra.run.smithi006.stdout: 2023-08-27T21:52:23.304 INFO:teuthology.orchestra.run.smithi006.stdout:{"epoch":50,"fsid":"b696f380-4522-11ee-817b-1b7a5f1ba315","created":"2023-08-27T21:43:51.020683+0000","modified":"2023-08-27T21:52:11.455770+0000","last_up_change":"2023-08-27T21:52:09.451350+0000","last_in_change":"2023-08-27T21:51:42.237602+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-08-27T21:48:22.209678+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"226c7900-24be-48c6-98fa-97893e22d328","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6802","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6803","nonce":224143379}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6804","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6805","nonce":224143379}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6808","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6809","nonce":224143379}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6806","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6807","nonce":224143379}]},"public_addr":"172.21.15.6:6803/224143379","cluster_addr":"172.21.15.6:6805/224143379","heartbeat_back_addr":"172.21.15.6:6809/224143379","heartbeat_front_addr":"172.21.15.6:6807/224143379","state":["exists","up"]},{"osd":1,"uuid":"55ceda65-564a-4b00-89c2-718610d59762","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6810","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6811","nonce":2798606708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6812","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6813","nonce":2798606708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6816","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6817","nonce":2798606708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6814","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6815","nonce":2798606708}]},"public_addr":"172.21.15.6:6811/2798606708","cluster_addr":"172.21.15.6:6813/2798606708","heartbeat_back_addr":"172.21.15.6:6817/2798606708","heartbeat_front_addr":"172.21.15.6:6815/2798606708","state":["exists","up"]},{"osd":2,"uuid":"b027a54f-cf0e-4547-a530-f70e146cb386","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6818","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6819","nonce":3315375049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6820","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6821","nonce":3315375049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6824","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6825","nonce":3315375049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6822","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6823","nonce":3315375049}]},"public_addr":"172.21.15.6:6819/3315375049","cluster_addr":"172.21.15.6:6821/3315375049","heartbeat_back_addr":"172.21.15.6:6825/3315375049","heartbeat_front_addr":"172.21.15.6:6823/3315375049","state":["exists","up"]},{"osd":3,"uuid":"74cd432a-aa78-4538-81bf-a4b209d60f31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6826","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6827","nonce":1657061419}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6828","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6829","nonce":1657061419}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6832","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6833","nonce":1657061419}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6830","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6831","nonce":1657061419}]},"public_addr":"172.21.15.6:6827/1657061419","cluster_addr":"172.21.15.6:6829/1657061419","heartbeat_back_addr":"172.21.15.6:6833/1657061419","heartbeat_front_addr":"172.21.15.6:6831/1657061419","state":["exists","up"]},{"osd":4,"uuid":"5710ffb1-d311-45a8-99bf-d01a7bb65cd1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6801","nonce":576819502}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6803","nonce":576819502}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6807","nonce":576819502}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6805","nonce":576819502}]},"public_addr":"172.21.15.130:6801/576819502","cluster_addr":"172.21.15.130:6803/576819502","heartbeat_back_addr":"172.21.15.130:6807/576819502","heartbeat_front_addr":"172.21.15.130:6805/576819502","state":["exists","up"]},{"osd":5,"uuid":"c0b6bba7-d752-4a95-927b-8a378bcd77b4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6809","nonce":169159537}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6811","nonce":169159537}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6815","nonce":169159537}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6813","nonce":169159537}]},"public_addr":"172.21.15.130:6809/169159537","cluster_addr":"172.21.15.130:6811/169159537","heartbeat_back_addr":"172.21.15.130:6815/169159537","heartbeat_front_addr":"172.21.15.130:6813/169159537","state":["exists","up"]},{"osd":6,"uuid":"fa4c8b52-d88a-427b-bac6-a1abaa0e92e6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6817","nonce":1800436766}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6819","nonce":1800436766}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6823","nonce":1800436766}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6821","nonce":1800436766}]},"public_addr":"172.21.15.130:6817/1800436766","cluster_addr":"172.21.15.130:6819/1800436766","heartbeat_back_addr":"172.21.15.130:6823/1800436766","heartbeat_front_addr":"172.21.15.130:6821/1800436766","state":["exists","up"]},{"osd":7,"uuid":"ac7e358c-d0fd-4b6c-a2b5-7006c5779948","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6825","nonce":2598634202}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6827","nonce":2598634202}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6831","nonce":2598634202}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6829","nonce":2598634202}]},"public_addr":"172.21.15.130:6825/2598634202","cluster_addr":"172.21.15.130:6827/2598634202","heartbeat_back_addr":"172.21.15.130:6831/2598634202","heartbeat_front_addr":"172.21.15.130:6829/2598634202","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:46:47.390747+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:47:33.822605+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:48:19.486341+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:49:05.565702+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:49:49.732445+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:50:35.236868+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:51:21.327320+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:52:07.813491+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.6:0/626906238":"2023-08-28T21:44:51.111365+0000","172.21.15.6:0/1056882729":"2023-08-28T21:44:51.111365+0000","172.21.15.6:6800/1902722306":"2023-08-28T21:44:51.111365+0000","172.21.15.6:0/3288657189":"2023-08-28T21:44:20.130814+0000","172.21.15.6:6800/3591185797":"2023-08-28T21:44:20.130814+0000","172.21.15.6:6801/1902722306":"2023-08-28T21:44:51.111365+0000","172.21.15.6:6801/3591185797":"2023-08-28T21:44:20.130814+0000","172.21.15.6:0/3143467986":"2023-08-28T21:44:20.130814+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-08-27T21:52:23.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:23 smithi006 bash[27292]: audit 2023-08-27T21:52:23.298150+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.6:0/437326607' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T21:52:23.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:23 smithi006 bash[33918]: audit 2023-08-27T21:52:23.298150+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.6:0/437326607' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T21:52:23.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:23 smithi130 bash[27579]: audit 2023-08-27T21:52:23.298150+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.6:0/437326607' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T21:52:23.995 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2023-08-27T21:48:22.209678+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2023-08-27T21:52:23.995 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd pool get device_health_metrics pg_num 2023-08-27T21:52:24.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:24 smithi006 bash[27292]: cluster 2023-08-27T21:52:23.238570+0000 mgr.y (mgr.14146) 278 : cluster [DBG] pgmap v266: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:24.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:24 smithi006 bash[33918]: cluster 2023-08-27T21:52:23.238570+0000 mgr.y (mgr.14146) 278 : cluster [DBG] pgmap v266: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:24.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:24 smithi130 bash[27579]: cluster 2023-08-27T21:52:23.238570+0000 mgr.y (mgr.14146) 278 : cluster [DBG] pgmap v266: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:25.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:25 smithi130 bash[27579]: cluster 2023-08-27T21:52:25.239201+0000 mgr.y (mgr.14146) 279 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:25.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:25 smithi006 bash[27292]: cluster 2023-08-27T21:52:25.239201+0000 mgr.y (mgr.14146) 279 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:25.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:25 smithi006 bash[33918]: cluster 2023-08-27T21:52:25.239201+0000 mgr.y (mgr.14146) 279 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:28.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:28 smithi006 bash[27292]: cluster 2023-08-27T21:52:27.239771+0000 mgr.y (mgr.14146) 280 : cluster [DBG] pgmap v268: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:28.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:28 smithi006 bash[33918]: cluster 2023-08-27T21:52:27.239771+0000 mgr.y (mgr.14146) 280 : cluster [DBG] pgmap v268: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:28.783 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:52:28.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:28 smithi130 bash[27579]: cluster 2023-08-27T21:52:27.239771+0000 mgr.y (mgr.14146) 280 : cluster [DBG] pgmap v268: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:30.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:30 smithi006 bash[33918]: cluster 2023-08-27T21:52:29.240396+0000 mgr.y (mgr.14146) 281 : cluster [DBG] pgmap v269: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:30.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:30 smithi006 bash[27292]: cluster 2023-08-27T21:52:29.240396+0000 mgr.y (mgr.14146) 281 : cluster [DBG] pgmap v269: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:30.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:30 smithi130 bash[27579]: cluster 2023-08-27T21:52:29.240396+0000 mgr.y (mgr.14146) 281 : cluster [DBG] pgmap v269: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:32.147 INFO:teuthology.orchestra.run.smithi006.stdout:pg_num: 1 2023-08-27T21:52:32.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:32 smithi006 bash[27292]: cluster 2023-08-27T21:52:31.241024+0000 mgr.y (mgr.14146) 282 : cluster [DBG] pgmap v270: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:32.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:32 smithi006 bash[27292]: audit 2023-08-27T21:52:32.142724+0000 mon.a (mon.0) 545 : audit [DBG] from='client.? 172.21.15.6:0/2253365294' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-08-27T21:52:32.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:32 smithi006 bash[33918]: cluster 2023-08-27T21:52:31.241024+0000 mgr.y (mgr.14146) 282 : cluster [DBG] pgmap v270: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:32.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:32 smithi006 bash[33918]: audit 2023-08-27T21:52:32.142724+0000 mon.a (mon.0) 545 : audit [DBG] from='client.? 172.21.15.6:0/2253365294' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-08-27T21:52:32.860 INFO:tasks.cephadm:Adding prometheus.a on smithi130 2023-08-27T21:52:32.860 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph orch apply prometheus '1;smithi130=a' 2023-08-27T21:52:32.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:32 smithi130 bash[27579]: cluster 2023-08-27T21:52:31.241024+0000 mgr.y (mgr.14146) 282 : cluster [DBG] pgmap v270: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:32.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:32 smithi130 bash[27579]: audit 2023-08-27T21:52:32.142724+0000 mon.a (mon.0) 545 : audit [DBG] from='client.? 172.21.15.6:0/2253365294' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-08-27T21:52:34.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:34 smithi006 bash[27292]: cluster 2023-08-27T21:52:33.241555+0000 mgr.y (mgr.14146) 283 : cluster [DBG] pgmap v271: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:34.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:34 smithi006 bash[33918]: cluster 2023-08-27T21:52:33.241555+0000 mgr.y (mgr.14146) 283 : cluster [DBG] pgmap v271: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:34.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:34 smithi130 bash[27579]: cluster 2023-08-27T21:52:33.241555+0000 mgr.y (mgr.14146) 283 : cluster [DBG] pgmap v271: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:36.318 INFO:teuthology.orchestra.run.smithi130.stdout:Scheduled prometheus update... 2023-08-27T21:52:36.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:36 smithi006 bash[27292]: cluster 2023-08-27T21:52:35.242089+0000 mgr.y (mgr.14146) 284 : cluster [DBG] pgmap v272: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:36.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:36 smithi006 bash[27292]: audit 2023-08-27T21:52:36.313907+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:36.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:36 smithi006 bash[27292]: audit 2023-08-27T21:52:36.348330+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:52:36.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:36 smithi006 bash[33918]: cluster 2023-08-27T21:52:35.242089+0000 mgr.y (mgr.14146) 284 : cluster [DBG] pgmap v272: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:36.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:36 smithi006 bash[33918]: audit 2023-08-27T21:52:36.313907+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:36.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:36 smithi006 bash[33918]: audit 2023-08-27T21:52:36.348330+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:52:36.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:36 smithi130 bash[27579]: cluster 2023-08-27T21:52:35.242089+0000 mgr.y (mgr.14146) 284 : cluster [DBG] pgmap v272: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:36.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:36 smithi130 bash[27579]: audit 2023-08-27T21:52:36.313907+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:36.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:36 smithi130 bash[27579]: audit 2023-08-27T21:52:36.348330+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:52:37.051 DEBUG:teuthology.orchestra.run.smithi130:prometheus.a> sudo journalctl -f -n 0 -u ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@prometheus.a.service 2023-08-27T21:52:37.094 INFO:tasks.cephadm:Adding node-exporter.a on smithi006 2023-08-27T21:52:37.095 INFO:tasks.cephadm:Adding node-exporter.b on smithi130 2023-08-27T21:52:37.095 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph orch apply node-exporter '2;smithi006=a;smithi130=b' 2023-08-27T21:52:37.136 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-08-27T21:52:37.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:37 smithi130 bash[27579]: audit 2023-08-27T21:52:36.303908+0000 mgr.y (mgr.14146) 285 : audit [DBG] from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi130=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:52:37.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:37 smithi130 bash[27579]: cephadm 2023-08-27T21:52:36.307350+0000 mgr.y (mgr.14146) 286 : cephadm [INF] Saving service prometheus spec with placement smithi130=a;count:1 2023-08-27T21:52:37.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:37 smithi130 bash[27579]: audit 2023-08-27T21:52:36.650823+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:52:37.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:37 smithi130 bash[27579]: audit 2023-08-27T21:52:36.652534+0000 mon.a (mon.0) 549 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:52:37.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:37 smithi130 bash[27579]: audit 2023-08-27T21:52:36.660029+0000 mon.a (mon.0) 550 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:37.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:37 smithi130 bash[27579]: audit 2023-08-27T21:52:36.667357+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-08-27T21:52:37.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:37 smithi006 bash[27292]: audit 2023-08-27T21:52:36.303908+0000 mgr.y (mgr.14146) 285 : audit [DBG] from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi130=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:52:37.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:37 smithi006 bash[27292]: cephadm 2023-08-27T21:52:36.307350+0000 mgr.y (mgr.14146) 286 : cephadm [INF] Saving service prometheus spec with placement smithi130=a;count:1 2023-08-27T21:52:37.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:37 smithi006 bash[27292]: audit 2023-08-27T21:52:36.650823+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:52:37.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:37 smithi006 bash[27292]: audit 2023-08-27T21:52:36.652534+0000 mon.a (mon.0) 549 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:52:37.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:37 smithi006 bash[27292]: audit 2023-08-27T21:52:36.660029+0000 mon.a (mon.0) 550 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:37.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:37 smithi006 bash[27292]: audit 2023-08-27T21:52:36.667357+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-08-27T21:52:37.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:37 smithi006 bash[33918]: audit 2023-08-27T21:52:36.303908+0000 mgr.y (mgr.14146) 285 : audit [DBG] from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi130=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:52:37.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:37 smithi006 bash[33918]: cephadm 2023-08-27T21:52:36.307350+0000 mgr.y (mgr.14146) 286 : cephadm [INF] Saving service prometheus spec with placement smithi130=a;count:1 2023-08-27T21:52:37.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:37 smithi006 bash[33918]: audit 2023-08-27T21:52:36.650823+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:52:37.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:37 smithi006 bash[33918]: audit 2023-08-27T21:52:36.652534+0000 mon.a (mon.0) 549 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:52:37.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:37 smithi006 bash[33918]: audit 2023-08-27T21:52:36.660029+0000 mon.a (mon.0) 550 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' 2023-08-27T21:52:37.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:37 smithi006 bash[33918]: audit 2023-08-27T21:52:36.667357+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-08-27T21:52:37.969 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:37 smithi006 bash[27566]: ignoring --setuser ceph since I am not root 2023-08-27T21:52:37.969 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:37 smithi006 bash[27566]: ignoring --setgroup ceph since I am not root 2023-08-27T21:52:37.969 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:37 smithi006 bash[27566]: debug 2023-08-27T21:52:37.718+0000 7f903457c700 1 -- 172.21.15.6:0/3284534810 <== mon.0 v2:172.21.15.6:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55edcdbac4e0 con 0x55edcdca1000 2023-08-27T21:52:37.969 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:37 smithi006 bash[27566]: debug 2023-08-27T21:52:37.874+0000 7f9044745680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-27T21:52:37.987 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:37 smithi130 bash[28467]: ignoring --setuser ceph since I am not root 2023-08-27T21:52:37.987 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:37 smithi130 bash[28467]: ignoring --setgroup ceph since I am not root 2023-08-27T21:52:37.987 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:37 smithi130 bash[28467]: debug 2023-08-27T21:52:37.716+0000 7efd46fdf700 1 -- 172.21.15.130:0/2354997313 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5587f67ba4e0 con 0x5587f68af000 2023-08-27T21:52:37.987 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:37 smithi130 bash[28467]: debug 2023-08-27T21:52:37.876+0000 7efd571a8680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-27T21:52:38.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:37 smithi130 bash[28467]: debug 2023-08-27T21:52:37.980+0000 7efd571a8680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-27T21:52:38.461 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:37 smithi006 bash[27566]: debug 2023-08-27T21:52:37.982+0000 7f9044745680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-27T21:52:38.718 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:38 smithi006 bash[27566]: debug 2023-08-27T21:52:38.458+0000 7f9044745680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-27T21:52:38.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:38 smithi006 bash[27292]: cluster 2023-08-27T21:52:37.242763+0000 mgr.y (mgr.14146) 287 : cluster [DBG] pgmap v273: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:38.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:38 smithi006 bash[27292]: audit 2023-08-27T21:52:37.672789+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-08-27T21:52:38.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:38 smithi006 bash[27292]: cluster 2023-08-27T21:52:37.672887+0000 mon.a (mon.0) 553 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-08-27T21:52:38.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:38 smithi006 bash[33918]: cluster 2023-08-27T21:52:37.242763+0000 mgr.y (mgr.14146) 287 : cluster [DBG] pgmap v273: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:38.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:38 smithi006 bash[33918]: audit 2023-08-27T21:52:37.672789+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-08-27T21:52:38.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:38 smithi006 bash[33918]: cluster 2023-08-27T21:52:37.672887+0000 mon.a (mon.0) 553 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-08-27T21:52:38.805 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:38 smithi130 bash[27579]: cluster 2023-08-27T21:52:37.242763+0000 mgr.y (mgr.14146) 287 : cluster [DBG] pgmap v273: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:38.805 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:38 smithi130 bash[27579]: audit 2023-08-27T21:52:37.672789+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14146 172.21.15.6:0/787491961' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-08-27T21:52:38.806 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:38 smithi130 bash[27579]: cluster 2023-08-27T21:52:37.672887+0000 mon.a (mon.0) 553 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-08-27T21:52:38.806 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:38 smithi130 bash[28467]: debug 2023-08-27T21:52:38.468+0000 7efd571a8680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-27T21:52:39.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:39 smithi130 bash[28467]: debug 2023-08-27T21:52:39.416+0000 7efd571a8680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-27T21:52:39.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:39 smithi130 bash[28467]: debug 2023-08-27T21:52:39.520+0000 7efd571a8680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-27T21:52:39.718 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:39 smithi006 bash[27566]: debug 2023-08-27T21:52:39.410+0000 7f9044745680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-27T21:52:39.719 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:39 smithi006 bash[27566]: debug 2023-08-27T21:52:39.510+0000 7f9044745680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-27T21:52:40.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:39 smithi130 bash[28467]: debug 2023-08-27T21:52:39.796+0000 7efd571a8680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-27T21:52:40.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:39 smithi006 bash[27566]: debug 2023-08-27T21:52:39.774+0000 7f9044745680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-27T21:52:41.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:40 smithi006 bash[27566]: debug 2023-08-27T21:52:40.958+0000 7f9044745680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-27T21:52:41.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:41 smithi006 bash[27566]: debug 2023-08-27T21:52:41.130+0000 7f9044745680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-27T21:52:41.247 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:40 smithi130 bash[28467]: debug 2023-08-27T21:52:40.976+0000 7efd571a8680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-27T21:52:41.248 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:41 smithi130 bash[28467]: debug 2023-08-27T21:52:41.144+0000 7efd571a8680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-27T21:52:41.583 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:41 smithi130 bash[28467]: debug 2023-08-27T21:52:41.240+0000 7efd571a8680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-27T21:52:41.583 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:41 smithi130 bash[28467]: debug 2023-08-27T21:52:41.352+0000 7efd571a8680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-27T21:52:41.594 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:41 smithi006 bash[27566]: debug 2023-08-27T21:52:41.238+0000 7f9044745680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-27T21:52:41.595 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:41 smithi006 bash[27566]: debug 2023-08-27T21:52:41.358+0000 7f9044745680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-27T21:52:41.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:41 smithi130 bash[28467]: debug 2023-08-27T21:52:41.576+0000 7efd571a8680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-27T21:52:41.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:41 smithi006 bash[27566]: debug 2023-08-27T21:52:41.586+0000 7f9044745680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-27T21:52:42.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:41 smithi130 bash[28467]: debug 2023-08-27T21:52:41.940+0000 7efd571a8680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-27T21:52:42.371 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:42 smithi130 bash[28467]: debug 2023-08-27T21:52:42.064+0000 7efd571a8680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-27T21:52:42.468 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:41 smithi006 bash[27566]: debug 2023-08-27T21:52:41.970+0000 7f9044745680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-27T21:52:42.469 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:42 smithi006 bash[27566]: debug 2023-08-27T21:52:42.098+0000 7f9044745680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-27T21:52:43.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:42 smithi130 bash[28467]: debug 2023-08-27T21:52:42.939+0000 7efd571a8680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-27T21:52:43.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:43 smithi130 bash[28467]: debug 2023-08-27T21:52:43.051+0000 7efd571a8680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-27T21:52:43.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:43 smithi130 bash[28467]: debug 2023-08-27T21:52:43.167+0000 7efd571a8680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-27T21:52:43.468 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:43 smithi006 bash[27566]: debug 2023-08-27T21:52:43.022+0000 7f9044745680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-27T21:52:43.469 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:43 smithi006 bash[27566]: debug 2023-08-27T21:52:43.134+0000 7f9044745680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-27T21:52:43.469 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:43 smithi006 bash[27566]: debug 2023-08-27T21:52:43.258+0000 7f9044745680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-27T21:52:43.694 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:43 smithi130 bash[28467]: debug 2023-08-27T21:52:43.391+0000 7efd571a8680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-27T21:52:43.804 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:43 smithi006 bash[27566]: debug 2023-08-27T21:52:43.490+0000 7f9044745680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-27T21:52:44.032 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:43 smithi130 bash[28467]: debug 2023-08-27T21:52:43.683+0000 7efd571a8680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-27T21:52:44.033 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:43 smithi130 bash[28467]: debug 2023-08-27T21:52:43.851+0000 7efd571a8680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-27T21:52:44.188 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:43 smithi006 bash[27566]: debug 2023-08-27T21:52:43.794+0000 7f9044745680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-27T21:52:44.188 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:43 smithi006 bash[27566]: debug 2023-08-27T21:52:43.982+0000 7f9044745680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-27T21:52:44.356 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:44 smithi130 bash[28467]: debug 2023-08-27T21:52:44.023+0000 7efd571a8680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-27T21:52:44.356 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:44 smithi130 bash[28467]: debug 2023-08-27T21:52:44.239+0000 7efd571a8680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-27T21:52:44.440 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[27566]: debug 2023-08-27T21:52:44.178+0000 7f9044745680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-27T21:52:44.440 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[27566]: debug 2023-08-27T21:52:44.422+0000 7f9044745680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-27T21:52:44.442 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[27292]: cluster 2023-08-27T21:52:44.372671+0000 mon.a (mon.0) 554 : cluster [DBG] Standby manager daemon x restarted 2023-08-27T21:52:44.442 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[27292]: cluster 2023-08-27T21:52:44.372859+0000 mon.a (mon.0) 555 : cluster [DBG] Standby manager daemon x started 2023-08-27T21:52:44.442 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[27292]: audit 2023-08-27T21:52:44.378455+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.130:0/313841783' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-27T21:52:44.442 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[27292]: audit 2023-08-27T21:52:44.380615+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.130:0/313841783' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T21:52:44.442 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[27292]: audit 2023-08-27T21:52:44.385175+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.130:0/313841783' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-27T21:52:44.442 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[27292]: audit 2023-08-27T21:52:44.385682+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.130:0/313841783' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T21:52:44.442 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[33918]: cluster 2023-08-27T21:52:44.372671+0000 mon.a (mon.0) 554 : cluster [DBG] Standby manager daemon x restarted 2023-08-27T21:52:44.442 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[33918]: cluster 2023-08-27T21:52:44.372859+0000 mon.a (mon.0) 555 : cluster [DBG] Standby manager daemon x started 2023-08-27T21:52:44.442 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[33918]: audit 2023-08-27T21:52:44.378455+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.130:0/313841783' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-27T21:52:44.443 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[33918]: audit 2023-08-27T21:52:44.380615+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.130:0/313841783' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T21:52:44.443 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[33918]: audit 2023-08-27T21:52:44.385175+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.130:0/313841783' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-27T21:52:44.443 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[33918]: audit 2023-08-27T21:52:44.385682+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.130:0/313841783' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T21:52:44.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:44 smithi130 bash[28467]: debug 2023-08-27T21:52:44.347+0000 7efd571a8680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-27T21:52:44.620 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:44 smithi130 bash[28467]: [27/Aug/2023:21:52:44] ENGINE Bus STARTING 2023-08-27T21:52:44.620 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:44 smithi130 bash[28467]: CherryPy Checker: 2023-08-27T21:52:44.620 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:44 smithi130 bash[28467]: The Application mounted at '' has an empty config. 2023-08-27T21:52:44.620 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:44 smithi130 bash[28467]: [27/Aug/2023:21:52:44] ENGINE Serving on http://:::9283 2023-08-27T21:52:44.620 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:52:44 smithi130 bash[28467]: [27/Aug/2023:21:52:44] ENGINE Bus STARTED 2023-08-27T21:52:44.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:44 smithi130 bash[27579]: cluster 2023-08-27T21:52:44.372671+0000 mon.a (mon.0) 554 : cluster [DBG] Standby manager daemon x restarted 2023-08-27T21:52:44.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:44 smithi130 bash[27579]: cluster 2023-08-27T21:52:44.372859+0000 mon.a (mon.0) 555 : cluster [DBG] Standby manager daemon x started 2023-08-27T21:52:44.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:44 smithi130 bash[27579]: audit 2023-08-27T21:52:44.378455+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.130:0/313841783' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-27T21:52:44.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:44 smithi130 bash[27579]: audit 2023-08-27T21:52:44.380615+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.130:0/313841783' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T21:52:44.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:44 smithi130 bash[27579]: audit 2023-08-27T21:52:44.385175+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.130:0/313841783' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-27T21:52:44.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:44 smithi130 bash[27579]: audit 2023-08-27T21:52:44.385682+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.130:0/313841783' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T21:52:44.719 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[27566]: debug 2023-08-27T21:52:44.542+0000 7f9044745680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-27T21:52:45.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[27566]: [27/Aug/2023:21:52:44] ENGINE Bus STARTING 2023-08-27T21:52:45.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[27566]: Warning: Permanently added '172.21.15.6' (ECDSA) to the list of known hosts. 2023-08-27T21:52:45.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[27566]: Warning: Permanently added '172.21.15.130' (ECDSA) to the list of known hosts. 2023-08-27T21:52:45.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[27566]: CherryPy Checker: 2023-08-27T21:52:45.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[27566]: The Application mounted at '' has an empty config. 2023-08-27T21:52:45.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[27566]: [27/Aug/2023:21:52:44] ENGINE Serving on http://:::9283 2023-08-27T21:52:45.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:52:44 smithi006 bash[27566]: [27/Aug/2023:21:52:44] ENGINE Bus STARTED 2023-08-27T21:52:45.606 INFO:teuthology.orchestra.run.smithi130.stdout:Scheduled node-exporter update... 2023-08-27T21:52:45.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: cluster 2023-08-27T21:52:44.434507+0000 mon.a (mon.0) 556 : cluster [DBG] mgrmap e16: y(active, since 7m), standbys: x 2023-08-27T21:52:45.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: cluster 2023-08-27T21:52:44.567194+0000 mon.a (mon.0) 557 : cluster [INF] Active manager daemon y restarted 2023-08-27T21:52:45.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: cluster 2023-08-27T21:52:44.568120+0000 mon.a (mon.0) 558 : cluster [INF] Activating manager daemon y 2023-08-27T21:52:45.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: cluster 2023-08-27T21:52:44.580217+0000 mon.a (mon.0) 559 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-08-27T21:52:45.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: cluster 2023-08-27T21:52:44.580586+0000 mon.a (mon.0) 560 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0127968s), standbys: x 2023-08-27T21:52:45.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: audit 2023-08-27T21:52:44.581987+0000 mon.a (mon.0) 561 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T21:52:45.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: audit 2023-08-27T21:52:44.582271+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:52:45.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: audit 2023-08-27T21:52:44.582455+0000 mon.a (mon.0) 563 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:52:45.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: audit 2023-08-27T21:52:44.583844+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T21:52:45.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: audit 2023-08-27T21:52:44.584047+0000 mon.a (mon.0) 565 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-27T21:52:45.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: audit 2023-08-27T21:52:44.584309+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:52:45.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: audit 2023-08-27T21:52:44.584669+0000 mon.a (mon.0) 567 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:52:45.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: audit 2023-08-27T21:52:44.585128+0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:52:45.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: audit 2023-08-27T21:52:44.585558+0000 mon.a (mon.0) 569 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:52:45.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: audit 2023-08-27T21:52:44.586024+0000 mon.a (mon.0) 570 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:52:45.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: audit 2023-08-27T21:52:44.586326+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:52:45.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: audit 2023-08-27T21:52:44.586634+0000 mon.a (mon.0) 572 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:52:45.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: audit 2023-08-27T21:52:44.586998+0000 mon.a (mon.0) 573 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:52:45.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: audit 2023-08-27T21:52:44.587311+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T21:52:45.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: audit 2023-08-27T21:52:44.587541+0000 mon.a (mon.0) 575 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T21:52:45.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: audit 2023-08-27T21:52:44.588136+0000 mon.a (mon.0) 576 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T21:52:45.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: cluster 2023-08-27T21:52:44.592300+0000 mon.a (mon.0) 577 : cluster [INF] Manager daemon y is now available 2023-08-27T21:52:45.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: audit 2023-08-27T21:52:44.738362+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:52:45.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: audit 2023-08-27T21:52:44.744385+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:52:45.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: audit 2023-08-27T21:52:44.794795+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:45.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: audit 2023-08-27T21:52:44.821535+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T21:52:45.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[27292]: audit 2023-08-27T21:52:44.833150+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T21:52:45.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: cluster 2023-08-27T21:52:44.434507+0000 mon.a (mon.0) 556 : cluster [DBG] mgrmap e16: y(active, since 7m), standbys: x 2023-08-27T21:52:45.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: cluster 2023-08-27T21:52:44.567194+0000 mon.a (mon.0) 557 : cluster [INF] Active manager daemon y restarted 2023-08-27T21:52:45.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: cluster 2023-08-27T21:52:44.568120+0000 mon.a (mon.0) 558 : cluster [INF] Activating manager daemon y 2023-08-27T21:52:45.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: cluster 2023-08-27T21:52:44.580217+0000 mon.a (mon.0) 559 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-08-27T21:52:45.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: cluster 2023-08-27T21:52:44.580586+0000 mon.a (mon.0) 560 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0127968s), standbys: x 2023-08-27T21:52:45.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: audit 2023-08-27T21:52:44.581987+0000 mon.a (mon.0) 561 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T21:52:45.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: audit 2023-08-27T21:52:44.582271+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:52:45.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: audit 2023-08-27T21:52:44.582455+0000 mon.a (mon.0) 563 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:52:45.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: audit 2023-08-27T21:52:44.583844+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T21:52:45.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: audit 2023-08-27T21:52:44.584047+0000 mon.a (mon.0) 565 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-27T21:52:45.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: audit 2023-08-27T21:52:44.584309+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:52:45.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: audit 2023-08-27T21:52:44.584669+0000 mon.a (mon.0) 567 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:52:45.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: audit 2023-08-27T21:52:44.585128+0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:52:45.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: audit 2023-08-27T21:52:44.585558+0000 mon.a (mon.0) 569 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:52:45.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: audit 2023-08-27T21:52:44.586024+0000 mon.a (mon.0) 570 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:52:45.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: audit 2023-08-27T21:52:44.586326+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:52:45.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: audit 2023-08-27T21:52:44.586634+0000 mon.a (mon.0) 572 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:52:45.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: audit 2023-08-27T21:52:44.586998+0000 mon.a (mon.0) 573 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:52:45.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: audit 2023-08-27T21:52:44.587311+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T21:52:45.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: audit 2023-08-27T21:52:44.587541+0000 mon.a (mon.0) 575 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T21:52:45.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: audit 2023-08-27T21:52:44.588136+0000 mon.a (mon.0) 576 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T21:52:45.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: cluster 2023-08-27T21:52:44.592300+0000 mon.a (mon.0) 577 : cluster [INF] Manager daemon y is now available 2023-08-27T21:52:45.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: audit 2023-08-27T21:52:44.738362+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:52:45.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: audit 2023-08-27T21:52:44.744385+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:52:45.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: audit 2023-08-27T21:52:44.794795+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:45.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: audit 2023-08-27T21:52:44.821535+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T21:52:45.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:45 smithi006 bash[33918]: audit 2023-08-27T21:52:44.833150+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T21:52:45.812 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: cluster 2023-08-27T21:52:44.434507+0000 mon.a (mon.0) 556 : cluster [DBG] mgrmap e16: y(active, since 7m), standbys: x 2023-08-27T21:52:45.812 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: cluster 2023-08-27T21:52:44.567194+0000 mon.a (mon.0) 557 : cluster [INF] Active manager daemon y restarted 2023-08-27T21:52:45.812 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: cluster 2023-08-27T21:52:44.568120+0000 mon.a (mon.0) 558 : cluster [INF] Activating manager daemon y 2023-08-27T21:52:45.812 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: cluster 2023-08-27T21:52:44.580217+0000 mon.a (mon.0) 559 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-08-27T21:52:45.813 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: cluster 2023-08-27T21:52:44.580586+0000 mon.a (mon.0) 560 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0127968s), standbys: x 2023-08-27T21:52:45.813 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: audit 2023-08-27T21:52:44.581987+0000 mon.a (mon.0) 561 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T21:52:45.813 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: audit 2023-08-27T21:52:44.582271+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:52:45.813 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: audit 2023-08-27T21:52:44.582455+0000 mon.a (mon.0) 563 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:52:45.813 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: audit 2023-08-27T21:52:44.583844+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T21:52:45.813 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: audit 2023-08-27T21:52:44.584047+0000 mon.a (mon.0) 565 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-27T21:52:45.813 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: audit 2023-08-27T21:52:44.584309+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:52:45.813 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: audit 2023-08-27T21:52:44.584669+0000 mon.a (mon.0) 567 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:52:45.813 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: audit 2023-08-27T21:52:44.585128+0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:52:45.813 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: audit 2023-08-27T21:52:44.585558+0000 mon.a (mon.0) 569 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:52:45.813 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: audit 2023-08-27T21:52:44.586024+0000 mon.a (mon.0) 570 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:52:45.814 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: audit 2023-08-27T21:52:44.586326+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:52:45.814 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: audit 2023-08-27T21:52:44.586634+0000 mon.a (mon.0) 572 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:52:45.814 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: audit 2023-08-27T21:52:44.586998+0000 mon.a (mon.0) 573 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:52:45.814 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: audit 2023-08-27T21:52:44.587311+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T21:52:45.814 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: audit 2023-08-27T21:52:44.587541+0000 mon.a (mon.0) 575 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T21:52:45.814 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: audit 2023-08-27T21:52:44.588136+0000 mon.a (mon.0) 576 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T21:52:45.814 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: cluster 2023-08-27T21:52:44.592300+0000 mon.a (mon.0) 577 : cluster [INF] Manager daemon y is now available 2023-08-27T21:52:45.814 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: audit 2023-08-27T21:52:44.738362+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:52:45.814 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: audit 2023-08-27T21:52:44.744385+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:52:45.814 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: audit 2023-08-27T21:52:44.794795+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:45.814 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: audit 2023-08-27T21:52:44.821535+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T21:52:45.815 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:45 smithi130 bash[27579]: audit 2023-08-27T21:52:44.833150+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T21:52:46.429 DEBUG:teuthology.orchestra.run.smithi006:node-exporter.a> sudo journalctl -f -n 0 -u ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@node-exporter.a.service 2023-08-27T21:52:46.432 DEBUG:teuthology.orchestra.run.smithi130:node-exporter.b> sudo journalctl -f -n 0 -u ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@node-exporter.b.service 2023-08-27T21:52:46.434 INFO:tasks.cephadm:Adding alertmanager.a on smithi006 2023-08-27T21:52:46.434 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph orch apply alertmanager '1;smithi006=a' 2023-08-27T21:52:46.506 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-08-27T21:52:46.554 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-08-27T21:52:46.680 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:46 smithi130 bash[27579]: cluster 2023-08-27T21:52:45.587585+0000 mon.a (mon.0) 583 : cluster [DBG] mgrmap e18: y(active, since 1.01978s), standbys: x 2023-08-27T21:52:46.680 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:46 smithi130 bash[27579]: cephadm 2023-08-27T21:52:45.597289+0000 mgr.y (mgr.14430) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi006=a;smithi130=b;count:2 2023-08-27T21:52:46.680 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:46 smithi130 bash[27579]: audit 2023-08-27T21:52:45.601267+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:46.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:46 smithi006 bash[27292]: cluster 2023-08-27T21:52:45.587585+0000 mon.a (mon.0) 583 : cluster [DBG] mgrmap e18: y(active, since 1.01978s), standbys: x 2023-08-27T21:52:46.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:46 smithi006 bash[27292]: cephadm 2023-08-27T21:52:45.597289+0000 mgr.y (mgr.14430) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi006=a;smithi130=b;count:2 2023-08-27T21:52:46.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:46 smithi006 bash[27292]: audit 2023-08-27T21:52:45.601267+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:46.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:46 smithi006 bash[33918]: cluster 2023-08-27T21:52:45.587585+0000 mon.a (mon.0) 583 : cluster [DBG] mgrmap e18: y(active, since 1.01978s), standbys: x 2023-08-27T21:52:46.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:46 smithi006 bash[33918]: cephadm 2023-08-27T21:52:45.597289+0000 mgr.y (mgr.14430) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi006=a;smithi130=b;count:2 2023-08-27T21:52:46.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:46 smithi006 bash[33918]: audit 2023-08-27T21:52:45.601267+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:47.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:47 smithi130 bash[27579]: cluster 2023-08-27T21:52:46.586186+0000 mgr.y (mgr.14430) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:47.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:47 smithi006 bash[33918]: cluster 2023-08-27T21:52:46.586186+0000 mgr.y (mgr.14430) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:47.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:47 smithi006 bash[27292]: cluster 2023-08-27T21:52:46.586186+0000 mgr.y (mgr.14430) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:48.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:48 smithi130 bash[27579]: cluster 2023-08-27T21:52:47.595460+0000 mon.a (mon.0) 585 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2023-08-27T21:52:48.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:48 smithi006 bash[27292]: cluster 2023-08-27T21:52:47.595460+0000 mon.a (mon.0) 585 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2023-08-27T21:52:48.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:48 smithi006 bash[33918]: cluster 2023-08-27T21:52:47.595460+0000 mon.a (mon.0) 585 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2023-08-27T21:52:49.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:49 smithi130 bash[27579]: cluster 2023-08-27T21:52:48.586841+0000 mgr.y (mgr.14430) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:49.879 INFO:teuthology.orchestra.run.smithi130.stdout:Scheduled alertmanager update... 2023-08-27T21:52:49.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:49 smithi006 bash[33918]: cluster 2023-08-27T21:52:48.586841+0000 mgr.y (mgr.14430) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:49.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:49 smithi006 bash[27292]: cluster 2023-08-27T21:52:48.586841+0000 mgr.y (mgr.14430) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:50.637 DEBUG:teuthology.orchestra.run.smithi006:alertmanager.a> sudo journalctl -f -n 0 -u ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@alertmanager.a.service 2023-08-27T21:52:50.647 INFO:tasks.cephadm:Adding grafana.a on smithi130 2023-08-27T21:52:50.647 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph orch apply grafana '1;smithi130=a' 2023-08-27T21:52:50.796 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-08-27T21:52:50.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:50 smithi006 bash[27292]: audit 2023-08-27T21:52:49.864634+0000 mgr.y (mgr.14430) 6 : audit [DBG] from='client.24370 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi006=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:52:50.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:50 smithi006 bash[27292]: cephadm 2023-08-27T21:52:49.867958+0000 mgr.y (mgr.14430) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi006=a;count:1 2023-08-27T21:52:50.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:50 smithi006 bash[27292]: audit 2023-08-27T21:52:49.874598+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:50.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:50 smithi006 bash[27292]: audit 2023-08-27T21:52:50.333917+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:50.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:50 smithi006 bash[27292]: audit 2023-08-27T21:52:50.530438+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:50.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:50 smithi006 bash[27292]: cluster 2023-08-27T21:52:50.587601+0000 mgr.y (mgr.14430) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:50.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:50 smithi006 bash[27292]: audit 2023-08-27T21:52:50.830829+0000 mon.a (mon.0) 589 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:52:50.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:50 smithi006 bash[27292]: audit 2023-08-27T21:52:50.832305+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:52:50.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:50 smithi006 bash[33918]: audit 2023-08-27T21:52:49.864634+0000 mgr.y (mgr.14430) 6 : audit [DBG] from='client.24370 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi006=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:52:50.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:50 smithi006 bash[33918]: cephadm 2023-08-27T21:52:49.867958+0000 mgr.y (mgr.14430) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi006=a;count:1 2023-08-27T21:52:50.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:50 smithi006 bash[33918]: audit 2023-08-27T21:52:49.874598+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:50.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:50 smithi006 bash[33918]: audit 2023-08-27T21:52:50.333917+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:50.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:50 smithi006 bash[33918]: audit 2023-08-27T21:52:50.530438+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:50.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:50 smithi006 bash[33918]: cluster 2023-08-27T21:52:50.587601+0000 mgr.y (mgr.14430) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:50.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:50 smithi006 bash[33918]: audit 2023-08-27T21:52:50.830829+0000 mon.a (mon.0) 589 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:52:50.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:50 smithi006 bash[33918]: audit 2023-08-27T21:52:50.832305+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:52:50.977 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:50 smithi130 bash[27579]: audit 2023-08-27T21:52:49.864634+0000 mgr.y (mgr.14430) 6 : audit [DBG] from='client.24370 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi006=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:52:50.977 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:50 smithi130 bash[27579]: cephadm 2023-08-27T21:52:49.867958+0000 mgr.y (mgr.14430) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi006=a;count:1 2023-08-27T21:52:50.977 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:50 smithi130 bash[27579]: audit 2023-08-27T21:52:49.874598+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:51.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:50 smithi130 bash[27579]: audit 2023-08-27T21:52:50.333917+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:51.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:50 smithi130 bash[27579]: audit 2023-08-27T21:52:50.530438+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:51.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:50 smithi130 bash[27579]: cluster 2023-08-27T21:52:50.587601+0000 mgr.y (mgr.14430) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:51.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:50 smithi130 bash[27579]: audit 2023-08-27T21:52:50.830829+0000 mon.a (mon.0) 589 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:52:51.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:50 smithi130 bash[27579]: audit 2023-08-27T21:52:50.832305+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:52:52.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:51 smithi006 bash[33918]: cephadm 2023-08-27T21:52:50.834448+0000 mgr.y (mgr.14430) 9 : cephadm [INF] Updating smithi006:/etc/ceph/ceph.conf 2023-08-27T21:52:52.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:51 smithi006 bash[33918]: cephadm 2023-08-27T21:52:50.835002+0000 mgr.y (mgr.14430) 10 : cephadm [INF] Updating smithi130:/etc/ceph/ceph.conf 2023-08-27T21:52:52.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:51 smithi006 bash[33918]: cephadm 2023-08-27T21:52:50.855738+0000 mgr.y (mgr.14430) 11 : cephadm [INF] Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2023-08-27T21:52:52.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:51 smithi006 bash[33918]: cephadm 2023-08-27T21:52:50.856426+0000 mgr.y (mgr.14430) 12 : cephadm [INF] Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2023-08-27T21:52:52.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:51 smithi006 bash[33918]: audit 2023-08-27T21:52:50.889800+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:52.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:51 smithi006 bash[33918]: audit 2023-08-27T21:52:51.015739+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:52.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:51 smithi006 bash[33918]: audit 2023-08-27T21:52:51.021841+0000 mon.a (mon.0) 593 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:52.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:51 smithi006 bash[33918]: cephadm 2023-08-27T21:52:51.045245+0000 mgr.y (mgr.14430) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi130 2023-08-27T21:52:52.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:51 smithi006 bash[27292]: cephadm 2023-08-27T21:52:50.834448+0000 mgr.y (mgr.14430) 9 : cephadm [INF] Updating smithi006:/etc/ceph/ceph.conf 2023-08-27T21:52:52.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:51 smithi006 bash[27292]: cephadm 2023-08-27T21:52:50.835002+0000 mgr.y (mgr.14430) 10 : cephadm [INF] Updating smithi130:/etc/ceph/ceph.conf 2023-08-27T21:52:52.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:51 smithi006 bash[27292]: cephadm 2023-08-27T21:52:50.855738+0000 mgr.y (mgr.14430) 11 : cephadm [INF] Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2023-08-27T21:52:52.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:51 smithi006 bash[27292]: cephadm 2023-08-27T21:52:50.856426+0000 mgr.y (mgr.14430) 12 : cephadm [INF] Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2023-08-27T21:52:52.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:51 smithi006 bash[27292]: audit 2023-08-27T21:52:50.889800+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:52.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:51 smithi006 bash[27292]: audit 2023-08-27T21:52:51.015739+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:52.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:51 smithi006 bash[27292]: audit 2023-08-27T21:52:51.021841+0000 mon.a (mon.0) 593 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:52.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:51 smithi006 bash[27292]: cephadm 2023-08-27T21:52:51.045245+0000 mgr.y (mgr.14430) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi130 2023-08-27T21:52:52.276 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:51 smithi130 bash[27579]: cephadm 2023-08-27T21:52:50.834448+0000 mgr.y (mgr.14430) 9 : cephadm [INF] Updating smithi006:/etc/ceph/ceph.conf 2023-08-27T21:52:52.276 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:51 smithi130 bash[27579]: cephadm 2023-08-27T21:52:50.835002+0000 mgr.y (mgr.14430) 10 : cephadm [INF] Updating smithi130:/etc/ceph/ceph.conf 2023-08-27T21:52:52.276 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:51 smithi130 bash[27579]: cephadm 2023-08-27T21:52:50.855738+0000 mgr.y (mgr.14430) 11 : cephadm [INF] Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2023-08-27T21:52:52.276 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:51 smithi130 bash[27579]: cephadm 2023-08-27T21:52:50.856426+0000 mgr.y (mgr.14430) 12 : cephadm [INF] Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2023-08-27T21:52:52.276 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:51 smithi130 bash[27579]: audit 2023-08-27T21:52:50.889800+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:52.277 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:51 smithi130 bash[27579]: audit 2023-08-27T21:52:51.015739+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:52.277 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:51 smithi130 bash[27579]: audit 2023-08-27T21:52:51.021841+0000 mon.a (mon.0) 593 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:52.277 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:51 smithi130 bash[27579]: cephadm 2023-08-27T21:52:51.045245+0000 mgr.y (mgr.14430) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi130 2023-08-27T21:52:53.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:53 smithi006 bash[33918]: cluster 2023-08-27T21:52:52.588432+0000 mgr.y (mgr.14430) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:53.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:53 smithi006 bash[27292]: cluster 2023-08-27T21:52:52.588432+0000 mgr.y (mgr.14430) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:53.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:53 smithi130 bash[27579]: cluster 2023-08-27T21:52:52.588432+0000 mgr.y (mgr.14430) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:55.860 INFO:teuthology.orchestra.run.smithi130.stdout:Scheduled grafana update... 2023-08-27T21:52:55.912 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:55 smithi130 bash[27579]: cluster 2023-08-27T21:52:54.589067+0000 mgr.y (mgr.14430) 15 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:55.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:55 smithi006 bash[27292]: cluster 2023-08-27T21:52:54.589067+0000 mgr.y (mgr.14430) 15 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:55.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:55 smithi006 bash[33918]: cluster 2023-08-27T21:52:54.589067+0000 mgr.y (mgr.14430) 15 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:57.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:56 smithi130 bash[27579]: audit 2023-08-27T21:52:55.845383+0000 mgr.y (mgr.14430) 16 : audit [DBG] from='client.24376 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi130=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:52:57.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:56 smithi130 bash[27579]: cephadm 2023-08-27T21:52:55.848536+0000 mgr.y (mgr.14430) 17 : cephadm [INF] Saving service grafana spec with placement smithi130=a;count:1 2023-08-27T21:52:57.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:56 smithi130 bash[27579]: audit 2023-08-27T21:52:55.855109+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:57.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:56 smithi130 bash[27579]: cluster 2023-08-27T21:52:56.589702+0000 mgr.y (mgr.14430) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:57.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:56 smithi006 bash[27292]: audit 2023-08-27T21:52:55.845383+0000 mgr.y (mgr.14430) 16 : audit [DBG] from='client.24376 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi130=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:52:57.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:56 smithi006 bash[27292]: cephadm 2023-08-27T21:52:55.848536+0000 mgr.y (mgr.14430) 17 : cephadm [INF] Saving service grafana spec with placement smithi130=a;count:1 2023-08-27T21:52:57.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:56 smithi006 bash[27292]: audit 2023-08-27T21:52:55.855109+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:57.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:56 smithi006 bash[27292]: cluster 2023-08-27T21:52:56.589702+0000 mgr.y (mgr.14430) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:57.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:56 smithi006 bash[33918]: audit 2023-08-27T21:52:55.845383+0000 mgr.y (mgr.14430) 16 : audit [DBG] from='client.24376 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi130=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:52:57.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:56 smithi006 bash[33918]: cephadm 2023-08-27T21:52:55.848536+0000 mgr.y (mgr.14430) 17 : cephadm [INF] Saving service grafana spec with placement smithi130=a;count:1 2023-08-27T21:52:57.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:56 smithi006 bash[33918]: audit 2023-08-27T21:52:55.855109+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:52:57.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:56 smithi006 bash[33918]: cluster 2023-08-27T21:52:56.589702+0000 mgr.y (mgr.14430) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:58.288 DEBUG:teuthology.orchestra.run.smithi130:grafana.a> sudo journalctl -f -n 0 -u ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@grafana.a.service 2023-08-27T21:52:58.291 INFO:tasks.cephadm:Setting up client nodes... 2023-08-27T21:52:58.292 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-08-27T21:52:58.485 INFO:journalctl@ceph.grafana.a.smithi130.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-08-27T21:52:59.818 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:52:59 smithi006 bash[27292]: cluster 2023-08-27T21:52:58.590157+0000 mgr.y (mgr.14430) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:52:59.819 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:52:59 smithi006 bash[33918]: cluster 2023-08-27T21:52:58.590157+0000 mgr.y (mgr.14430) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:00.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:52:59 smithi130 bash[27579]: cluster 2023-08-27T21:52:58.590157+0000 mgr.y (mgr.14430) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:01.779 INFO:teuthology.orchestra.run.smithi006.stdout:[client.0] 2023-08-27T21:53:01.779 INFO:teuthology.orchestra.run.smithi006.stdout: key = AQC9xetk153ULRAArDsiWBCu4wZHhmgPTXWrBw== 2023-08-27T21:53:01.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:01 smithi006 bash[27292]: cluster 2023-08-27T21:53:00.590512+0000 mgr.y (mgr.14430) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:01.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:01 smithi006 bash[33918]: cluster 2023-08-27T21:53:00.590512+0000 mgr.y (mgr.14430) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:02.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:01 smithi130 bash[27579]: cluster 2023-08-27T21:53:00.590512+0000 mgr.y (mgr.14430) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:02.528 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2023-08-27T21:53:02.528 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-08-27T21:53:02.528 DEBUG:teuthology.orchestra.run.smithi006:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-08-27T21:53:02.551 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-08-27T21:53:02.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:02 smithi130 bash[27579]: audit 2023-08-27T21:53:01.767764+0000 mon.c (mon.2) 12 : audit [INF] from='client.? 172.21.15.6:0/2972034111' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T21:53:02.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:02 smithi130 bash[27579]: audit 2023-08-27T21:53:01.768555+0000 mon.a (mon.0) 595 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T21:53:02.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:02 smithi130 bash[27579]: audit 2023-08-27T21:53:01.773399+0000 mon.a (mon.0) 596 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-08-27T21:53:02.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:02 smithi006 bash[27292]: audit 2023-08-27T21:53:01.767764+0000 mon.c (mon.2) 12 : audit [INF] from='client.? 172.21.15.6:0/2972034111' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T21:53:02.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:02 smithi006 bash[27292]: audit 2023-08-27T21:53:01.768555+0000 mon.a (mon.0) 595 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T21:53:02.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:02 smithi006 bash[27292]: audit 2023-08-27T21:53:01.773399+0000 mon.a (mon.0) 596 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-08-27T21:53:02.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:02 smithi006 bash[33918]: audit 2023-08-27T21:53:01.767764+0000 mon.c (mon.2) 12 : audit [INF] from='client.? 172.21.15.6:0/2972034111' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T21:53:02.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:02 smithi006 bash[33918]: audit 2023-08-27T21:53:01.768555+0000 mon.a (mon.0) 595 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T21:53:02.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:02 smithi006 bash[33918]: audit 2023-08-27T21:53:01.773399+0000 mon.a (mon.0) 596 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-08-27T21:53:03.916 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:03 smithi130 bash[27579]: cluster 2023-08-27T21:53:02.591104+0000 mgr.y (mgr.14430) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:03.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:03 smithi006 bash[33918]: cluster 2023-08-27T21:53:02.591104+0000 mgr.y (mgr.14430) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:03.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:03 smithi006 bash[27292]: cluster 2023-08-27T21:53:02.591104+0000 mgr.y (mgr.14430) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:05.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:05 smithi130 bash[27579]: cluster 2023-08-27T21:53:04.591707+0000 mgr.y (mgr.14430) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:05.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:05 smithi006 bash[33918]: cluster 2023-08-27T21:53:04.591707+0000 mgr.y (mgr.14430) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:05.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:05 smithi006 bash[27292]: cluster 2023-08-27T21:53:04.591707+0000 mgr.y (mgr.14430) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:06.811 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:06 smithi006 systemd[1]: Started Ceph node-exporter.a for b696f380-4522-11ee-817b-1b7a5f1ba315. 2023-08-27T21:53:07.219 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:06 smithi006 bash[52721]: Unable to find image 'quay.io/prometheus/node-exporter:v1.3.1' locally 2023-08-27T21:53:07.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:07 smithi006 bash[27292]: audit 2023-08-27T21:53:06.014461+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:07.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:07 smithi006 bash[27292]: cephadm 2023-08-27T21:53:06.018182+0000 mgr.y (mgr.14430) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi006 2023-08-27T21:53:07.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:07 smithi006 bash[27292]: cluster 2023-08-27T21:53:06.592129+0000 mgr.y (mgr.14430) 24 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:07.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:07 smithi006 bash[27292]: audit 2023-08-27T21:53:06.777315+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:07.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:07 smithi006 bash[33918]: audit 2023-08-27T21:53:06.014461+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:07.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:07 smithi006 bash[33918]: cephadm 2023-08-27T21:53:06.018182+0000 mgr.y (mgr.14430) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi006 2023-08-27T21:53:07.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:07 smithi006 bash[33918]: cluster 2023-08-27T21:53:06.592129+0000 mgr.y (mgr.14430) 24 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:07.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:07 smithi006 bash[33918]: audit 2023-08-27T21:53:06.777315+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:07.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:07 smithi130 bash[27579]: audit 2023-08-27T21:53:06.014461+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:07.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:07 smithi130 bash[27579]: cephadm 2023-08-27T21:53:06.018182+0000 mgr.y (mgr.14430) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi006 2023-08-27T21:53:07.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:07 smithi130 bash[27579]: cluster 2023-08-27T21:53:06.592129+0000 mgr.y (mgr.14430) 24 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:07.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:07 smithi130 bash[27579]: audit 2023-08-27T21:53:06.777315+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:07.627 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:07 smithi130 systemd[1]: Started Ceph node-exporter.b for b696f380-4522-11ee-817b-1b7a5f1ba315. 2023-08-27T21:53:07.627 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:07 smithi130 bash[47380]: Unable to find image 'quay.io/prometheus/node-exporter:v1.3.1' locally 2023-08-27T21:53:07.700 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:07 smithi006 bash[52721]: v1.3.1: Pulling from prometheus/node-exporter 2023-08-27T21:53:07.700 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:07 smithi006 bash[52721]: aa2a8d90b84c: Pulling fs layer 2023-08-27T21:53:07.701 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:07 smithi006 bash[52721]: b45d31ee2d7f: Pulling fs layer 2023-08-27T21:53:07.701 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:07 smithi006 bash[52721]: b5db1e299295: Pulling fs layer 2023-08-27T21:53:07.889 INFO:teuthology.orchestra.run.smithi130.stdout:[client.1] 2023-08-27T21:53:07.889 INFO:teuthology.orchestra.run.smithi130.stdout: key = AQDDxetkhFEFNBAAqO9E0RkAbnmLWy2wYdpxsA== 2023-08-27T21:53:07.922 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:53:07 smithi130 bash[47189]: ts=2023-08-27T21:53:07.663Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-08-27T21:53:07.969 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:07 smithi006 bash[52721]: aa2a8d90b84c: Download complete 2023-08-27T21:53:07.969 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:07 smithi006 bash[52721]: b45d31ee2d7f: Verifying Checksum 2023-08-27T21:53:07.969 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:07 smithi006 bash[52721]: b45d31ee2d7f: Download complete 2023-08-27T21:53:07.969 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:07 smithi006 bash[52721]: aa2a8d90b84c: Pull complete 2023-08-27T21:53:08.370 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:08 smithi130 bash[47380]: v1.3.1: Pulling from prometheus/node-exporter 2023-08-27T21:53:08.469 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:08 smithi006 bash[52721]: b5db1e299295: Verifying Checksum 2023-08-27T21:53:08.469 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:08 smithi006 bash[52721]: b5db1e299295: Download complete 2023-08-27T21:53:08.469 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:08 smithi006 bash[52721]: b45d31ee2d7f: Pull complete 2023-08-27T21:53:08.804 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:08 smithi006 bash[27292]: cephadm 2023-08-27T21:53:06.778511+0000 mgr.y (mgr.14430) 25 : cephadm [INF] Deploying daemon node-exporter.b on smithi130 2023-08-27T21:53:08.804 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:08 smithi006 bash[27292]: audit 2023-08-27T21:53:07.499958+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:08.804 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:08 smithi006 bash[27292]: cephadm 2023-08-27T21:53:07.510315+0000 mgr.y (mgr.14430) 26 : cephadm [INF] Deploying daemon alertmanager.a on smithi006 2023-08-27T21:53:08.804 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:08 smithi006 bash[27292]: audit 2023-08-27T21:53:07.872288+0000 mon.a (mon.0) 600 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T21:53:08.804 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:08 smithi006 bash[27292]: audit 2023-08-27T21:53:07.872746+0000 mon.b (mon.1) 28 : audit [INF] from='client.? 172.21.15.130:0/1906025339' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T21:53:08.805 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:08 smithi006 bash[27292]: audit 2023-08-27T21:53:07.883736+0000 mon.a (mon.0) 601 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-08-27T21:53:08.805 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:08 smithi006 bash[33918]: cephadm 2023-08-27T21:53:06.778511+0000 mgr.y (mgr.14430) 25 : cephadm [INF] Deploying daemon node-exporter.b on smithi130 2023-08-27T21:53:08.805 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:08 smithi006 bash[33918]: audit 2023-08-27T21:53:07.499958+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:08.805 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:08 smithi006 bash[33918]: cephadm 2023-08-27T21:53:07.510315+0000 mgr.y (mgr.14430) 26 : cephadm [INF] Deploying daemon alertmanager.a on smithi006 2023-08-27T21:53:08.805 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:08 smithi006 bash[33918]: audit 2023-08-27T21:53:07.872288+0000 mon.a (mon.0) 600 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T21:53:08.805 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:08 smithi006 bash[33918]: audit 2023-08-27T21:53:07.872746+0000 mon.b (mon.1) 28 : audit [INF] from='client.? 172.21.15.130:0/1906025339' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T21:53:08.806 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:08 smithi006 bash[33918]: audit 2023-08-27T21:53:07.883736+0000 mon.a (mon.0) 601 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-08-27T21:53:08.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:08 smithi130 bash[27579]: cephadm 2023-08-27T21:53:06.778511+0000 mgr.y (mgr.14430) 25 : cephadm [INF] Deploying daemon node-exporter.b on smithi130 2023-08-27T21:53:08.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:08 smithi130 bash[27579]: audit 2023-08-27T21:53:07.499958+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:08.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:08 smithi130 bash[27579]: cephadm 2023-08-27T21:53:07.510315+0000 mgr.y (mgr.14430) 26 : cephadm [INF] Deploying daemon alertmanager.a on smithi006 2023-08-27T21:53:08.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:08 smithi130 bash[27579]: audit 2023-08-27T21:53:07.872288+0000 mon.a (mon.0) 600 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T21:53:08.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:08 smithi130 bash[27579]: audit 2023-08-27T21:53:07.872746+0000 mon.b (mon.1) 28 : audit [INF] from='client.? 172.21.15.130:0/1906025339' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T21:53:08.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:08 smithi130 bash[27579]: audit 2023-08-27T21:53:07.883736+0000 mon.a (mon.0) 601 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-08-27T21:53:09.049 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-08-27T21:53:09.049 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-08-27T21:53:09.049 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-08-27T21:53:09.068 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-08-27T21:53:09.068 DEBUG:teuthology.orchestra.run.smithi006:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-08-27T21:53:09.088 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:08 smithi006 bash[52721]: b5db1e299295: Pull complete 2023-08-27T21:53:09.088 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:09 smithi006 bash[52721]: Digest: sha256:f2269e73124dd0f60a7d19a2ce1264d33d08a985aed0ee6b0b89d0be470592cd 2023-08-27T21:53:09.088 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:09 smithi006 bash[52721]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.3.1 2023-08-27T21:53:09.141 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:08 smithi130 bash[47380]: aa2a8d90b84c: Pulling fs layer 2023-08-27T21:53:09.141 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:08 smithi130 bash[47380]: b45d31ee2d7f: Pulling fs layer 2023-08-27T21:53:09.141 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:08 smithi130 bash[47380]: b5db1e299295: Pulling fs layer 2023-08-27T21:53:09.141 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:09 smithi130 bash[47380]: b45d31ee2d7f: Download complete 2023-08-27T21:53:09.141 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:09 smithi130 bash[47380]: b5db1e299295: Verifying Checksum 2023-08-27T21:53:09.415 INFO:teuthology.misc.health.smithi006.stdout: 2023-08-27T21:53:09.415 INFO:teuthology.misc.health.smithi006.stdout:{"epoch":51,"fsid":"b696f380-4522-11ee-817b-1b7a5f1ba315","created":"2023-08-27T21:43:51.020683+0000","modified":"2023-08-27T21:52:44.567392+0000","last_up_change":"2023-08-27T21:52:09.451350+0000","last_in_change":"2023-08-27T21:51:42.237602+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-08-27T21:48:22.209678+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"226c7900-24be-48c6-98fa-97893e22d328","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6802","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6803","nonce":224143379}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6804","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6805","nonce":224143379}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6808","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6809","nonce":224143379}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6806","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6807","nonce":224143379}]},"public_addr":"172.21.15.6:6803/224143379","cluster_addr":"172.21.15.6:6805/224143379","heartbeat_back_addr":"172.21.15.6:6809/224143379","heartbeat_front_addr":"172.21.15.6:6807/224143379","state":["exists","up"]},{"osd":1,"uuid":"55ceda65-564a-4b00-89c2-718610d59762","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6810","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6811","nonce":2798606708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6812","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6813","nonce":2798606708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6816","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6817","nonce":2798606708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6814","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6815","nonce":2798606708}]},"public_addr":"172.21.15.6:6811/2798606708","cluster_addr":"172.21.15.6:6813/2798606708","heartbeat_back_addr":"172.21.15.6:6817/2798606708","heartbeat_front_addr":"172.21.15.6:6815/2798606708","state":["exists","up"]},{"osd":2,"uuid":"b027a54f-cf0e-4547-a530-f70e146cb386","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6818","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6819","nonce":3315375049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6820","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6821","nonce":3315375049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6824","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6825","nonce":3315375049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6822","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6823","nonce":3315375049}]},"public_addr":"172.21.15.6:6819/3315375049","cluster_addr":"172.21.15.6:6821/3315375049","heartbeat_back_addr":"172.21.15.6:6825/3315375049","heartbeat_front_addr":"172.21.15.6:6823/3315375049","state":["exists","up"]},{"osd":3,"uuid":"74cd432a-aa78-4538-81bf-a4b209d60f31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6826","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6827","nonce":1657061419}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6828","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6829","nonce":1657061419}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6832","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6833","nonce":1657061419}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6830","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6831","nonce":1657061419}]},"public_addr":"172.21.15.6:6827/1657061419","cluster_addr":"172.21.15.6:6829/1657061419","heartbeat_back_addr":"172.21.15.6:6833/1657061419","heartbeat_front_addr":"172.21.15.6:6831/1657061419","state":["exists","up"]},{"osd":4,"uuid":"5710ffb1-d311-45a8-99bf-d01a7bb65cd1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6801","nonce":576819502}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6803","nonce":576819502}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6807","nonce":576819502}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6805","nonce":576819502}]},"public_addr":"172.21.15.130:6801/576819502","cluster_addr":"172.21.15.130:6803/576819502","heartbeat_back_addr":"172.21.15.130:6807/576819502","heartbeat_front_addr":"172.21.15.130:6805/576819502","state":["exists","up"]},{"osd":5,"uuid":"c0b6bba7-d752-4a95-927b-8a378bcd77b4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6809","nonce":169159537}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6811","nonce":169159537}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6815","nonce":169159537}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6813","nonce":169159537}]},"public_addr":"172.21.15.130:6809/169159537","cluster_addr":"172.21.15.130:6811/169159537","heartbeat_back_addr":"172.21.15.130:6815/169159537","heartbeat_front_addr":"172.21.15.130:6813/169159537","state":["exists","up"]},{"osd":6,"uuid":"fa4c8b52-d88a-427b-bac6-a1abaa0e92e6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6817","nonce":1800436766}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6819","nonce":1800436766}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6823","nonce":1800436766}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6821","nonce":1800436766}]},"public_addr":"172.21.15.130:6817/1800436766","cluster_addr":"172.21.15.130:6819/1800436766","heartbeat_back_addr":"172.21.15.130:6823/1800436766","heartbeat_front_addr":"172.21.15.130:6821/1800436766","state":["exists","up"]},{"osd":7,"uuid":"ac7e358c-d0fd-4b6c-a2b5-7006c5779948","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6825","nonce":2598634202}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6827","nonce":2598634202}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6831","nonce":2598634202}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6829","nonce":2598634202}]},"public_addr":"172.21.15.130:6825/2598634202","cluster_addr":"172.21.15.130:6827/2598634202","heartbeat_back_addr":"172.21.15.130:6831/2598634202","heartbeat_front_addr":"172.21.15.130:6829/2598634202","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:46:47.390747+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:47:33.822605+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:48:19.486341+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:49:05.565702+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:49:49.732445+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:50:35.236868+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:51:21.327320+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:52:07.813491+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.6:0/2156177013":"2023-08-28T21:52:44.567266+0000","172.21.15.6:0/4156113241":"2023-08-28T21:52:44.567266+0000","172.21.15.6:0/626906238":"2023-08-28T21:44:51.111365+0000","172.21.15.6:0/1056882729":"2023-08-28T21:44:51.111365+0000","172.21.15.6:6800/1902722306":"2023-08-28T21:44:51.111365+0000","172.21.15.6:0/3288657189":"2023-08-28T21:44:20.130814+0000","172.21.15.6:6800/3591185797":"2023-08-28T21:44:20.130814+0000","172.21.15.6:6800/2215677068":"2023-08-28T21:52:44.567266+0000","172.21.15.6:6801/1902722306":"2023-08-28T21:44:51.111365+0000","172.21.15.6:6801/3591185797":"2023-08-28T21:44:20.130814+0000","172.21.15.6:6801/2215677068":"2023-08-28T21:52:44.567266+0000","172.21.15.6:0/3143467986":"2023-08-28T21:44:20.130814+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-08-27T21:53:09.430 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-08-27T21:53:09.430 INFO:tasks.cephadm:Creating RBD pool 2023-08-27T21:53:09.430 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-08-27T21:53:09.513 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:09 smithi130 bash[47380]: b5db1e299295: Download complete 2023-08-27T21:53:09.514 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:09 smithi130 bash[47380]: aa2a8d90b84c: Pull complete 2023-08-27T21:53:09.869 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:09 smithi130 bash[47380]: b45d31ee2d7f: Pull complete 2023-08-27T21:53:09.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:09 smithi130 bash[27579]: cluster 2023-08-27T21:53:08.592695+0000 mgr.y (mgr.14430) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:09.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:09 smithi130 bash[27579]: audit 2023-08-27T21:53:09.410477+0000 mon.a (mon.0) 602 : audit [DBG] from='client.? 172.21.15.6:0/2817048098' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T21:53:09.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:09 smithi006 bash[27292]: cluster 2023-08-27T21:53:08.592695+0000 mgr.y (mgr.14430) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:09.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:09 smithi006 bash[27292]: audit 2023-08-27T21:53:09.410477+0000 mon.a (mon.0) 602 : audit [DBG] from='client.? 172.21.15.6:0/2817048098' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T21:53:09.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:09 smithi006 bash[33918]: cluster 2023-08-27T21:53:08.592695+0000 mgr.y (mgr.14430) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:09.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:09 smithi006 bash[33918]: audit 2023-08-27T21:53:09.410477+0000 mon.a (mon.0) 602 : audit [DBG] from='client.? 172.21.15.6:0/2817048098' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T21:53:10.196 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:09 smithi130 bash[47380]: b5db1e299295: Pull complete 2023-08-27T21:53:10.197 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:10 smithi130 bash[47380]: Digest: sha256:f2269e73124dd0f60a7d19a2ce1264d33d08a985aed0ee6b0b89d0be470592cd 2023-08-27T21:53:10.197 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:10 smithi130 bash[47380]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.3.1 2023-08-27T21:53:11.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:10 smithi130 bash[27579]: audit 2023-08-27T21:53:09.808498+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:11.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:10 smithi130 bash[27579]: cluster 2023-08-27T21:53:10.593315+0000 mgr.y (mgr.14430) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:11.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:10 smithi006 bash[33918]: audit 2023-08-27T21:53:09.808498+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:11.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:10 smithi006 bash[33918]: cluster 2023-08-27T21:53:10.593315+0000 mgr.y (mgr.14430) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:11.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:10 smithi006 bash[27292]: audit 2023-08-27T21:53:09.808498+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:11.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:10 smithi006 bash[27292]: cluster 2023-08-27T21:53:10.593315+0000 mgr.y (mgr.14430) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:11.871 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.639Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-08-27T21:53:11.871 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.639Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-08-27T21:53:11.871 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.639Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-08-27T21:53:11.871 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.639Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-08-27T21:53:11.871 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-08-27T21:53:11.871 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=arp 2023-08-27T21:53:11.871 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=bcache 2023-08-27T21:53:11.871 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=bonding 2023-08-27T21:53:11.871 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=btrfs 2023-08-27T21:53:11.871 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=conntrack 2023-08-27T21:53:11.872 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=cpu 2023-08-27T21:53:11.872 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-08-27T21:53:11.872 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=diskstats 2023-08-27T21:53:11.872 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=dmi 2023-08-27T21:53:11.872 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=edac 2023-08-27T21:53:11.872 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=entropy 2023-08-27T21:53:11.872 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-08-27T21:53:11.872 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=filefd 2023-08-27T21:53:11.872 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=filesystem 2023-08-27T21:53:11.872 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=hwmon 2023-08-27T21:53:11.872 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=infiniband 2023-08-27T21:53:11.872 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=ipvs 2023-08-27T21:53:11.873 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=loadavg 2023-08-27T21:53:11.873 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=mdadm 2023-08-27T21:53:11.873 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=meminfo 2023-08-27T21:53:11.873 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=netclass 2023-08-27T21:53:11.873 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=netdev 2023-08-27T21:53:11.873 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=netstat 2023-08-27T21:53:11.873 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=nfs 2023-08-27T21:53:11.873 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=nfsd 2023-08-27T21:53:11.873 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=nvme 2023-08-27T21:53:11.873 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=os 2023-08-27T21:53:11.874 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-08-27T21:53:11.874 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=pressure 2023-08-27T21:53:11.874 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.641Z caller=node_exporter.go:115 level=info collector=rapl 2023-08-27T21:53:11.874 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.642Z caller=node_exporter.go:115 level=info collector=schedstat 2023-08-27T21:53:11.874 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.642Z caller=node_exporter.go:115 level=info collector=sockstat 2023-08-27T21:53:11.874 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.642Z caller=node_exporter.go:115 level=info collector=softnet 2023-08-27T21:53:11.874 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.642Z caller=node_exporter.go:115 level=info collector=stat 2023-08-27T21:53:11.874 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.642Z caller=node_exporter.go:115 level=info collector=tapestats 2023-08-27T21:53:11.874 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.642Z caller=node_exporter.go:115 level=info collector=textfile 2023-08-27T21:53:11.874 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.642Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-08-27T21:53:11.874 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.642Z caller=node_exporter.go:115 level=info collector=time 2023-08-27T21:53:11.874 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.642Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-08-27T21:53:11.875 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.642Z caller=node_exporter.go:115 level=info collector=uname 2023-08-27T21:53:11.875 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.642Z caller=node_exporter.go:115 level=info collector=vmstat 2023-08-27T21:53:11.875 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.642Z caller=node_exporter.go:115 level=info collector=xfs 2023-08-27T21:53:11.875 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.642Z caller=node_exporter.go:115 level=info collector=zfs 2023-08-27T21:53:11.875 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.642Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-08-27T21:53:11.875 INFO:journalctl@ceph.node-exporter.b.smithi130.stdout:Aug 27 21:53:11 smithi130 bash[47380]: ts=2023-08-27T21:53:11.642Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-08-27T21:53:12.182 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.032Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-08-27T21:53:12.182 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.032Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-08-27T21:53:12.182 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.032Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-08-27T21:53:12.182 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.032Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-08-27T21:53:12.182 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.034Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-08-27T21:53:12.182 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.034Z caller=node_exporter.go:115 level=info collector=arp 2023-08-27T21:53:12.182 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.034Z caller=node_exporter.go:115 level=info collector=bcache 2023-08-27T21:53:12.182 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.034Z caller=node_exporter.go:115 level=info collector=bonding 2023-08-27T21:53:12.183 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.034Z caller=node_exporter.go:115 level=info collector=btrfs 2023-08-27T21:53:12.183 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.034Z caller=node_exporter.go:115 level=info collector=conntrack 2023-08-27T21:53:12.183 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.034Z caller=node_exporter.go:115 level=info collector=cpu 2023-08-27T21:53:12.183 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.034Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-08-27T21:53:12.183 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.034Z caller=node_exporter.go:115 level=info collector=diskstats 2023-08-27T21:53:12.183 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.034Z caller=node_exporter.go:115 level=info collector=dmi 2023-08-27T21:53:12.183 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.034Z caller=node_exporter.go:115 level=info collector=edac 2023-08-27T21:53:12.183 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.034Z caller=node_exporter.go:115 level=info collector=entropy 2023-08-27T21:53:12.183 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.034Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-08-27T21:53:12.183 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.034Z caller=node_exporter.go:115 level=info collector=filefd 2023-08-27T21:53:12.183 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.034Z caller=node_exporter.go:115 level=info collector=filesystem 2023-08-27T21:53:12.183 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.034Z caller=node_exporter.go:115 level=info collector=hwmon 2023-08-27T21:53:12.184 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.034Z caller=node_exporter.go:115 level=info collector=infiniband 2023-08-27T21:53:12.184 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.034Z caller=node_exporter.go:115 level=info collector=ipvs 2023-08-27T21:53:12.184 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.034Z caller=node_exporter.go:115 level=info collector=loadavg 2023-08-27T21:53:12.184 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.034Z caller=node_exporter.go:115 level=info collector=mdadm 2023-08-27T21:53:12.184 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.034Z caller=node_exporter.go:115 level=info collector=meminfo 2023-08-27T21:53:12.184 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=netclass 2023-08-27T21:53:12.184 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=netdev 2023-08-27T21:53:12.184 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=netstat 2023-08-27T21:53:12.184 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=nfs 2023-08-27T21:53:12.184 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=nfsd 2023-08-27T21:53:12.184 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=nvme 2023-08-27T21:53:12.184 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=os 2023-08-27T21:53:12.184 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-08-27T21:53:12.185 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=pressure 2023-08-27T21:53:12.185 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=rapl 2023-08-27T21:53:12.185 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=schedstat 2023-08-27T21:53:12.185 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=sockstat 2023-08-27T21:53:12.185 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=softnet 2023-08-27T21:53:12.185 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=stat 2023-08-27T21:53:12.185 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=tapestats 2023-08-27T21:53:12.185 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=textfile 2023-08-27T21:53:12.185 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-08-27T21:53:12.185 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=time 2023-08-27T21:53:12.185 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-08-27T21:53:12.185 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=uname 2023-08-27T21:53:12.185 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=vmstat 2023-08-27T21:53:12.186 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=xfs 2023-08-27T21:53:12.186 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.035Z caller=node_exporter.go:115 level=info collector=zfs 2023-08-27T21:53:12.186 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.036Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-08-27T21:53:12.186 INFO:journalctl@ceph.node-exporter.a.smithi006.stdout:Aug 27 21:53:12 smithi006 bash[52721]: ts=2023-08-27T21:53:12.036Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-08-27T21:53:13.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:13 smithi006 bash[33918]: cluster 2023-08-27T21:53:12.594011+0000 mgr.y (mgr.14430) 29 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:13.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:13 smithi006 bash[27292]: cluster 2023-08-27T21:53:12.594011+0000 mgr.y (mgr.14430) 29 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:13.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:13 smithi130 bash[27579]: cluster 2023-08-27T21:53:12.594011+0000 mgr.y (mgr.14430) 29 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:16.038 INFO:teuthology.orchestra.run.smithi006.stderr:pool 'rbd' created 2023-08-27T21:53:16.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:15 smithi006 bash[27292]: cluster 2023-08-27T21:53:14.594519+0000 mgr.y (mgr.14430) 30 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:16.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:15 smithi006 bash[27292]: audit 2023-08-27T21:53:15.511817+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.6:0/2659697690' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-08-27T21:53:16.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:15 smithi006 bash[27292]: audit 2023-08-27T21:53:15.512385+0000 mon.a (mon.0) 604 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-08-27T21:53:16.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:15 smithi006 bash[33918]: cluster 2023-08-27T21:53:14.594519+0000 mgr.y (mgr.14430) 30 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:16.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:15 smithi006 bash[33918]: audit 2023-08-27T21:53:15.511817+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.6:0/2659697690' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-08-27T21:53:16.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:15 smithi006 bash[33918]: audit 2023-08-27T21:53:15.512385+0000 mon.a (mon.0) 604 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-08-27T21:53:16.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:15 smithi130 bash[27579]: cluster 2023-08-27T21:53:14.594519+0000 mgr.y (mgr.14430) 30 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:16.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:15 smithi130 bash[27579]: audit 2023-08-27T21:53:15.511817+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.6:0/2659697690' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-08-27T21:53:16.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:15 smithi130 bash[27579]: audit 2023-08-27T21:53:15.512385+0000 mon.a (mon.0) 604 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-08-27T21:53:16.959 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-08-27T21:53:17.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:17 smithi006 bash[33918]: audit 2023-08-27T21:53:16.030029+0000 mon.a (mon.0) 605 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-08-27T21:53:17.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:17 smithi006 bash[33918]: cluster 2023-08-27T21:53:16.030165+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-08-27T21:53:17.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:17 smithi006 bash[33918]: audit 2023-08-27T21:53:16.030729+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-08-27T21:53:17.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:17 smithi006 bash[33918]: audit 2023-08-27T21:53:16.031743+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-08-27T21:53:17.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:17 smithi006 bash[33918]: cluster 2023-08-27T21:53:16.595153+0000 mgr.y (mgr.14430) 31 : cluster [DBG] pgmap v20: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:17.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:17 smithi006 bash[27292]: audit 2023-08-27T21:53:16.030029+0000 mon.a (mon.0) 605 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-08-27T21:53:17.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:17 smithi006 bash[27292]: cluster 2023-08-27T21:53:16.030165+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-08-27T21:53:17.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:17 smithi006 bash[27292]: audit 2023-08-27T21:53:16.030729+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-08-27T21:53:17.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:17 smithi006 bash[27292]: audit 2023-08-27T21:53:16.031743+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-08-27T21:53:17.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:17 smithi006 bash[27292]: cluster 2023-08-27T21:53:16.595153+0000 mgr.y (mgr.14430) 31 : cluster [DBG] pgmap v20: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:17.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:53:17 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:53:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:53:17.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:17 smithi130 bash[27579]: audit 2023-08-27T21:53:16.030029+0000 mon.a (mon.0) 605 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-08-27T21:53:17.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:17 smithi130 bash[27579]: cluster 2023-08-27T21:53:16.030165+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-08-27T21:53:17.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:17 smithi130 bash[27579]: audit 2023-08-27T21:53:16.030729+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-08-27T21:53:17.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:17 smithi130 bash[27579]: audit 2023-08-27T21:53:16.031743+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-08-27T21:53:17.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:17 smithi130 bash[27579]: cluster 2023-08-27T21:53:16.595153+0000 mgr.y (mgr.14430) 31 : cluster [DBG] pgmap v20: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:18.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:18 smithi130 bash[27579]: audit 2023-08-27T21:53:17.035867+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-08-27T21:53:18.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:18 smithi130 bash[27579]: cluster 2023-08-27T21:53:17.036144+0000 mon.a (mon.0) 610 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-08-27T21:53:18.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:18 smithi006 bash[27292]: audit 2023-08-27T21:53:17.035867+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-08-27T21:53:18.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:18 smithi006 bash[27292]: cluster 2023-08-27T21:53:17.036144+0000 mon.a (mon.0) 610 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-08-27T21:53:18.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:18 smithi006 bash[33918]: audit 2023-08-27T21:53:17.035867+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-08-27T21:53:18.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:18 smithi006 bash[33918]: cluster 2023-08-27T21:53:17.036144+0000 mon.a (mon.0) 610 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-08-27T21:53:19.063 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:19 smithi006 bash[33918]: cluster 2023-08-27T21:53:18.041975+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-08-27T21:53:19.063 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:19 smithi006 bash[33918]: cluster 2023-08-27T21:53:18.595754+0000 mgr.y (mgr.14430) 32 : cluster [DBG] pgmap v23: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:19.063 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:19 smithi006 bash[27292]: cluster 2023-08-27T21:53:18.041975+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-08-27T21:53:19.064 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:19 smithi006 bash[27292]: cluster 2023-08-27T21:53:18.595754+0000 mgr.y (mgr.14430) 32 : cluster [DBG] pgmap v23: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:19.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:19 smithi130 bash[27579]: cluster 2023-08-27T21:53:18.041975+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-08-27T21:53:19.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:19 smithi130 bash[27579]: cluster 2023-08-27T21:53:18.595754+0000 mgr.y (mgr.14430) 32 : cluster [DBG] pgmap v23: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:20.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:20 smithi006 bash[27292]: audit 2023-08-27T21:53:19.355627+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:20.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:20 smithi006 bash[27292]: audit 2023-08-27T21:53:19.357593+0000 mon.a (mon.0) 613 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:53:20.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:20 smithi006 bash[27292]: audit 2023-08-27T21:53:19.819104+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:20.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:20 smithi130 bash[27579]: audit 2023-08-27T21:53:19.355627+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:20.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:20 smithi130 bash[27579]: audit 2023-08-27T21:53:19.357593+0000 mon.a (mon.0) 613 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:53:20.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:20 smithi130 bash[27579]: audit 2023-08-27T21:53:19.819104+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:20.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:20 smithi006 bash[33918]: audit 2023-08-27T21:53:19.355627+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:20.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:20 smithi006 bash[33918]: audit 2023-08-27T21:53:19.357593+0000 mon.a (mon.0) 613 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:53:20.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:20 smithi006 bash[33918]: audit 2023-08-27T21:53:19.819104+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:21.229 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:53:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:53:20] "GET /metrics HTTP/1.1" 200 153265 "" "Prometheus/2.33.4" 2023-08-27T21:53:21.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:21 smithi130 bash[27579]: cluster 2023-08-27T21:53:20.596335+0000 mgr.y (mgr.14430) 33 : cluster [DBG] pgmap v24: 9 pgs: 2 unknown, 7 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:21.621 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:21 smithi006 bash[27292]: cluster 2023-08-27T21:53:20.596335+0000 mgr.y (mgr.14430) 33 : cluster [DBG] pgmap v24: 9 pgs: 2 unknown, 7 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:21.621 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:21 smithi006 bash[33918]: cluster 2023-08-27T21:53:20.596335+0000 mgr.y (mgr.14430) 33 : cluster [DBG] pgmap v24: 9 pgs: 2 unknown, 7 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:21.621 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:53:21 smithi006 bash[53461]: level=info ts=2023-08-27T21:53:21.511Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-08-27T21:53:22.385 INFO:teuthology.orchestra.run.smithi006.stderr:enabled application 'rbd' on pool 'rbd' 2023-08-27T21:53:22.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:22 smithi006 bash[33918]: audit 2023-08-27T21:53:22.052915+0000 mon.a (mon.0) 615 : audit [INF] from='client.? 172.21.15.6:0/1847895545' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T21:53:22.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:22 smithi006 bash[27292]: audit 2023-08-27T21:53:22.052915+0000 mon.a (mon.0) 615 : audit [INF] from='client.? 172.21.15.6:0/1847895545' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T21:53:22.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:22 smithi130 bash[27579]: audit 2023-08-27T21:53:22.052915+0000 mon.a (mon.0) 615 : audit [INF] from='client.? 172.21.15.6:0/1847895545' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T21:53:23.195 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-08-27T21:53:23.195 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-08-27T21:53:23.195 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph mgr dump --format=json 2023-08-27T21:53:23.460 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:23 smithi006 bash[27292]: audit 2023-08-27T21:53:22.376350+0000 mon.a (mon.0) 616 : audit [INF] from='client.? 172.21.15.6:0/1847895545' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-08-27T21:53:23.460 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:23 smithi006 bash[27292]: cluster 2023-08-27T21:53:22.376431+0000 mon.a (mon.0) 617 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-08-27T21:53:23.460 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:23 smithi006 bash[27292]: cluster 2023-08-27T21:53:22.596963+0000 mgr.y (mgr.14430) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:23.460 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:23 smithi006 bash[33918]: audit 2023-08-27T21:53:22.376350+0000 mon.a (mon.0) 616 : audit [INF] from='client.? 172.21.15.6:0/1847895545' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-08-27T21:53:23.461 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:23 smithi006 bash[33918]: cluster 2023-08-27T21:53:22.376431+0000 mon.a (mon.0) 617 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-08-27T21:53:23.461 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:23 smithi006 bash[33918]: cluster 2023-08-27T21:53:22.596963+0000 mgr.y (mgr.14430) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:23.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:53:23 smithi006 bash[53461]: level=info ts=2023-08-27T21:53:23.455Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000914241s 2023-08-27T21:53:23.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:23 smithi130 bash[27579]: audit 2023-08-27T21:53:22.376350+0000 mon.a (mon.0) 616 : audit [INF] from='client.? 172.21.15.6:0/1847895545' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-08-27T21:53:23.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:23 smithi130 bash[27579]: cluster 2023-08-27T21:53:22.376431+0000 mon.a (mon.0) 617 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-08-27T21:53:23.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:23 smithi130 bash[27579]: cluster 2023-08-27T21:53:22.596963+0000 mgr.y (mgr.14430) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:25.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:25 smithi006 bash[27292]: cluster 2023-08-27T21:53:24.597510+0000 mgr.y (mgr.14430) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:25.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:25 smithi006 bash[27292]: audit 2023-08-27T21:53:25.139892+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:25.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:25 smithi006 bash[33918]: cluster 2023-08-27T21:53:24.597510+0000 mgr.y (mgr.14430) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:25.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:25 smithi006 bash[33918]: audit 2023-08-27T21:53:25.139892+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:26.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:25 smithi130 bash[27579]: cluster 2023-08-27T21:53:24.597510+0000 mgr.y (mgr.14430) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:26.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:25 smithi130 bash[27579]: audit 2023-08-27T21:53:25.139892+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:27.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:53:26 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:53:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:53:27.470 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:53:26 smithi006 bash[27566]: [27/Aug/2023:21:53:26] ENGINE Bus STOPPING 2023-08-27T21:53:27.470 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[27566]: [27/Aug/2023:21:53:27] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-27T21:53:27.470 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[27566]: [27/Aug/2023:21:53:27] ENGINE Bus STOPPED 2023-08-27T21:53:27.470 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[27566]: [27/Aug/2023:21:53:27] ENGINE Bus STARTING 2023-08-27T21:53:27.470 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[27566]: [27/Aug/2023:21:53:27] ENGINE Serving on http://:::9283 2023-08-27T21:53:27.471 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[27566]: [27/Aug/2023:21:53:27] ENGINE Bus STARTED 2023-08-27T21:53:27.762 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:53:27.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[27292]: cluster 2023-08-27T21:53:26.598188+0000 mgr.y (mgr.14430) 36 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:27.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[27292]: audit 2023-08-27T21:53:26.783254+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:27.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[27292]: audit 2023-08-27T21:53:26.784916+0000 mon.a (mon.0) 620 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:53:27.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[27292]: audit 2023-08-27T21:53:26.786478+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:53:27.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[27292]: audit 2023-08-27T21:53:26.794386+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:27.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[27292]: audit 2023-08-27T21:53:26.949497+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:27.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[27292]: audit 2023-08-27T21:53:26.954909+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:27.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[27292]: audit 2023-08-27T21:53:26.961025+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-08-27T21:53:27.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[27292]: audit 2023-08-27T21:53:26.967513+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:27.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[33918]: cluster 2023-08-27T21:53:26.598188+0000 mgr.y (mgr.14430) 36 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:27.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[33918]: audit 2023-08-27T21:53:26.783254+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:27.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[33918]: audit 2023-08-27T21:53:26.784916+0000 mon.a (mon.0) 620 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:53:27.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[33918]: audit 2023-08-27T21:53:26.786478+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:53:27.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[33918]: audit 2023-08-27T21:53:26.794386+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:27.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[33918]: audit 2023-08-27T21:53:26.949497+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:27.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[33918]: audit 2023-08-27T21:53:26.954909+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:27.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[33918]: audit 2023-08-27T21:53:26.961025+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-08-27T21:53:27.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:27 smithi006 bash[33918]: audit 2023-08-27T21:53:26.967513+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:28.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:27 smithi130 bash[27579]: cluster 2023-08-27T21:53:26.598188+0000 mgr.y (mgr.14430) 36 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:28.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:27 smithi130 bash[27579]: audit 2023-08-27T21:53:26.783254+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:28.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:27 smithi130 bash[27579]: audit 2023-08-27T21:53:26.784916+0000 mon.a (mon.0) 620 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:53:28.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:27 smithi130 bash[27579]: audit 2023-08-27T21:53:26.786478+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:53:28.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:27 smithi130 bash[27579]: audit 2023-08-27T21:53:26.794386+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:28.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:27 smithi130 bash[27579]: audit 2023-08-27T21:53:26.949497+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:28.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:27 smithi130 bash[27579]: audit 2023-08-27T21:53:26.954909+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:28.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:27 smithi130 bash[27579]: audit 2023-08-27T21:53:26.961025+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-08-27T21:53:28.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:27 smithi130 bash[27579]: audit 2023-08-27T21:53:26.967513+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:28.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:28 smithi006 bash[33918]: audit 2023-08-27T21:53:26.961657+0000 mgr.y (mgr.14430) 37 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-08-27T21:53:28.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:28 smithi006 bash[33918]: cephadm 2023-08-27T21:53:26.975321+0000 mgr.y (mgr.14430) 38 : cephadm [INF] Deploying daemon grafana.a on smithi130 2023-08-27T21:53:28.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:28 smithi006 bash[27292]: audit 2023-08-27T21:53:26.961657+0000 mgr.y (mgr.14430) 37 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-08-27T21:53:28.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:28 smithi006 bash[27292]: cephadm 2023-08-27T21:53:26.975321+0000 mgr.y (mgr.14430) 38 : cephadm [INF] Deploying daemon grafana.a on smithi130 2023-08-27T21:53:29.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:28 smithi130 bash[27579]: audit 2023-08-27T21:53:26.961657+0000 mgr.y (mgr.14430) 37 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-08-27T21:53:29.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:28 smithi130 bash[27579]: cephadm 2023-08-27T21:53:26.975321+0000 mgr.y (mgr.14430) 38 : cephadm [INF] Deploying daemon grafana.a on smithi130 2023-08-27T21:53:29.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:29 smithi006 bash[27292]: cluster 2023-08-27T21:53:28.598836+0000 mgr.y (mgr.14430) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:29.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:29 smithi006 bash[33918]: cluster 2023-08-27T21:53:28.598836+0000 mgr.y (mgr.14430) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:30.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:29 smithi130 bash[27579]: cluster 2023-08-27T21:53:28.598836+0000 mgr.y (mgr.14430) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:30.969 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:53:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:53:30] "GET /metrics HTTP/1.1" 200 155649 "" "Prometheus/2.33.4" 2023-08-27T21:53:31.187 INFO:teuthology.orchestra.run.smithi006.stdout: 2023-08-27T21:53:31.679 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:53:31 smithi006 bash[53461]: level=info ts=2023-08-27T21:53:31.459Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.004561977s 2023-08-27T21:53:31.854 INFO:teuthology.orchestra.run.smithi006.stdout:{"epoch":19,"active_gid":14430,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6800","nonce":3377670591},{"type":"v1","addr":"172.21.15.6:6801","nonce":3377670591}]},"active_addr":"172.21.15.6:6801/3377670591","active_change":"2023-08-27T21:52:44.567775+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24349,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.6:8443/","prometheus":"http://172.21.15.6:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.6:0","nonce":4243266477}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.6:0","nonce":647135083}]}]} 2023-08-27T21:53:31.858 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-08-27T21:53:31.859 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-08-27T21:53:31.859 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd dump --format=json 2023-08-27T21:53:31.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:31 smithi006 bash[27292]: cluster 2023-08-27T21:53:30.599291+0000 mgr.y (mgr.14430) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:31.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:31 smithi006 bash[27292]: audit 2023-08-27T21:53:31.174573+0000 mon.a (mon.0) 627 : audit [DBG] from='client.? 172.21.15.6:0/1461435732' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-08-27T21:53:31.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:31 smithi006 bash[33918]: cluster 2023-08-27T21:53:30.599291+0000 mgr.y (mgr.14430) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:31.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:31 smithi006 bash[33918]: audit 2023-08-27T21:53:31.174573+0000 mon.a (mon.0) 627 : audit [DBG] from='client.? 172.21.15.6:0/1461435732' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-08-27T21:53:32.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:31 smithi130 bash[27579]: cluster 2023-08-27T21:53:30.599291+0000 mgr.y (mgr.14430) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:32.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:31 smithi130 bash[27579]: audit 2023-08-27T21:53:31.174573+0000 mon.a (mon.0) 627 : audit [DBG] from='client.? 172.21.15.6:0/1461435732' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-08-27T21:53:33.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:33 smithi006 bash[27292]: cluster 2023-08-27T21:53:32.599882+0000 mgr.y (mgr.14430) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:33.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:33 smithi006 bash[33918]: cluster 2023-08-27T21:53:32.599882+0000 mgr.y (mgr.14430) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:33.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:33 smithi130 bash[27579]: cluster 2023-08-27T21:53:32.599882+0000 mgr.y (mgr.14430) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:35.949 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:35 smithi130 bash[27579]: cluster 2023-08-27T21:53:34.600488+0000 mgr.y (mgr.14430) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:35.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:35 smithi006 bash[27292]: cluster 2023-08-27T21:53:34.600488+0000 mgr.y (mgr.14430) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:35.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:35 smithi006 bash[33918]: cluster 2023-08-27T21:53:34.600488+0000 mgr.y (mgr.14430) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:36.674 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:53:37.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:53:36 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:53:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:53:37.921 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:37 smithi006 bash[33918]: cluster 2023-08-27T21:53:36.601257+0000 mgr.y (mgr.14430) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:37.921 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:37 smithi006 bash[27292]: cluster 2023-08-27T21:53:36.601257+0000 mgr.y (mgr.14430) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:38.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:37 smithi130 bash[27579]: cluster 2023-08-27T21:53:36.601257+0000 mgr.y (mgr.14430) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:39.885 INFO:teuthology.orchestra.run.smithi006.stdout: 2023-08-27T21:53:39.886 INFO:teuthology.orchestra.run.smithi006.stdout:{"epoch":55,"fsid":"b696f380-4522-11ee-817b-1b7a5f1ba315","created":"2023-08-27T21:43:51.020683+0000","modified":"2023-08-27T21:53:22.364540+0000","last_up_change":"2023-08-27T21:52:09.451350+0000","last_in_change":"2023-08-27T21:51:42.237602+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-08-27T21:48:22.209678+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-08-27T21:53:15.972053+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"226c7900-24be-48c6-98fa-97893e22d328","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6802","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6803","nonce":224143379}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6804","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6805","nonce":224143379}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6808","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6809","nonce":224143379}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6806","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6807","nonce":224143379}]},"public_addr":"172.21.15.6:6803/224143379","cluster_addr":"172.21.15.6:6805/224143379","heartbeat_back_addr":"172.21.15.6:6809/224143379","heartbeat_front_addr":"172.21.15.6:6807/224143379","state":["exists","up"]},{"osd":1,"uuid":"55ceda65-564a-4b00-89c2-718610d59762","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6810","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6811","nonce":2798606708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6812","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6813","nonce":2798606708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6816","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6817","nonce":2798606708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6814","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6815","nonce":2798606708}]},"public_addr":"172.21.15.6:6811/2798606708","cluster_addr":"172.21.15.6:6813/2798606708","heartbeat_back_addr":"172.21.15.6:6817/2798606708","heartbeat_front_addr":"172.21.15.6:6815/2798606708","state":["exists","up"]},{"osd":2,"uuid":"b027a54f-cf0e-4547-a530-f70e146cb386","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6818","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6819","nonce":3315375049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6820","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6821","nonce":3315375049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6824","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6825","nonce":3315375049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6822","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6823","nonce":3315375049}]},"public_addr":"172.21.15.6:6819/3315375049","cluster_addr":"172.21.15.6:6821/3315375049","heartbeat_back_addr":"172.21.15.6:6825/3315375049","heartbeat_front_addr":"172.21.15.6:6823/3315375049","state":["exists","up"]},{"osd":3,"uuid":"74cd432a-aa78-4538-81bf-a4b209d60f31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6826","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6827","nonce":1657061419}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6828","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6829","nonce":1657061419}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6832","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6833","nonce":1657061419}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6830","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6831","nonce":1657061419}]},"public_addr":"172.21.15.6:6827/1657061419","cluster_addr":"172.21.15.6:6829/1657061419","heartbeat_back_addr":"172.21.15.6:6833/1657061419","heartbeat_front_addr":"172.21.15.6:6831/1657061419","state":["exists","up"]},{"osd":4,"uuid":"5710ffb1-d311-45a8-99bf-d01a7bb65cd1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6801","nonce":576819502}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6803","nonce":576819502}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6807","nonce":576819502}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6805","nonce":576819502}]},"public_addr":"172.21.15.130:6801/576819502","cluster_addr":"172.21.15.130:6803/576819502","heartbeat_back_addr":"172.21.15.130:6807/576819502","heartbeat_front_addr":"172.21.15.130:6805/576819502","state":["exists","up"]},{"osd":5,"uuid":"c0b6bba7-d752-4a95-927b-8a378bcd77b4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6809","nonce":169159537}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6811","nonce":169159537}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6815","nonce":169159537}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6813","nonce":169159537}]},"public_addr":"172.21.15.130:6809/169159537","cluster_addr":"172.21.15.130:6811/169159537","heartbeat_back_addr":"172.21.15.130:6815/169159537","heartbeat_front_addr":"172.21.15.130:6813/169159537","state":["exists","up"]},{"osd":6,"uuid":"fa4c8b52-d88a-427b-bac6-a1abaa0e92e6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6817","nonce":1800436766}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6819","nonce":1800436766}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6823","nonce":1800436766}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6821","nonce":1800436766}]},"public_addr":"172.21.15.130:6817/1800436766","cluster_addr":"172.21.15.130:6819/1800436766","heartbeat_back_addr":"172.21.15.130:6823/1800436766","heartbeat_front_addr":"172.21.15.130:6821/1800436766","state":["exists","up"]},{"osd":7,"uuid":"ac7e358c-d0fd-4b6c-a2b5-7006c5779948","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6825","nonce":2598634202}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6827","nonce":2598634202}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6831","nonce":2598634202}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6829","nonce":2598634202}]},"public_addr":"172.21.15.130:6825/2598634202","cluster_addr":"172.21.15.130:6827/2598634202","heartbeat_back_addr":"172.21.15.130:6831/2598634202","heartbeat_front_addr":"172.21.15.130:6829/2598634202","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:46:47.390747+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:47:33.822605+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:48:19.486341+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:49:05.565702+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:49:49.732445+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:50:35.236868+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:51:21.327320+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:52:07.813491+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.6:0/2156177013":"2023-08-28T21:52:44.567266+0000","172.21.15.6:0/4156113241":"2023-08-28T21:52:44.567266+0000","172.21.15.6:0/626906238":"2023-08-28T21:44:51.111365+0000","172.21.15.6:0/1056882729":"2023-08-28T21:44:51.111365+0000","172.21.15.6:6800/1902722306":"2023-08-28T21:44:51.111365+0000","172.21.15.6:0/3288657189":"2023-08-28T21:44:20.130814+0000","172.21.15.6:6800/3591185797":"2023-08-28T21:44:20.130814+0000","172.21.15.6:6800/2215677068":"2023-08-28T21:52:44.567266+0000","172.21.15.6:6801/1902722306":"2023-08-28T21:44:51.111365+0000","172.21.15.6:6801/3591185797":"2023-08-28T21:44:20.130814+0000","172.21.15.6:6801/2215677068":"2023-08-28T21:52:44.567266+0000","172.21.15.6:0/3143467986":"2023-08-28T21:44:20.130814+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-08-27T21:53:39.935 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:39 smithi006 bash[33918]: cluster 2023-08-27T21:53:38.601774+0000 mgr.y (mgr.14430) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:39.936 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:39 smithi006 bash[27292]: cluster 2023-08-27T21:53:38.601774+0000 mgr.y (mgr.14430) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:40.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:39 smithi130 bash[27579]: cluster 2023-08-27T21:53:38.601774+0000 mgr.y (mgr.14430) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:40.603 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-08-27T21:53:40.604 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd dump --format=json 2023-08-27T21:53:40.969 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:53:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:53:40] "GET /metrics HTTP/1.1" 200 155649 "" "Prometheus/2.33.4" 2023-08-27T21:53:40.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:40 smithi006 bash[33918]: audit 2023-08-27T21:53:39.879006+0000 mon.a (mon.0) 628 : audit [DBG] from='client.? 172.21.15.6:0/1480383147' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T21:53:40.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:40 smithi006 bash[27292]: audit 2023-08-27T21:53:39.879006+0000 mon.a (mon.0) 628 : audit [DBG] from='client.? 172.21.15.6:0/1480383147' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T21:53:41.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:40 smithi130 bash[27579]: audit 2023-08-27T21:53:39.879006+0000 mon.a (mon.0) 628 : audit [DBG] from='client.? 172.21.15.6:0/1480383147' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T21:53:41.851 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:41 smithi130 bash[27579]: cluster 2023-08-27T21:53:40.602404+0000 mgr.y (mgr.14430) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:41.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:41 smithi006 bash[33918]: cluster 2023-08-27T21:53:40.602404+0000 mgr.y (mgr.14430) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:41.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:41 smithi006 bash[27292]: cluster 2023-08-27T21:53:40.602404+0000 mgr.y (mgr.14430) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:43.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:43 smithi130 bash[27579]: cluster 2023-08-27T21:53:42.603023+0000 mgr.y (mgr.14430) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:43.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:43 smithi006 bash[33918]: cluster 2023-08-27T21:53:42.603023+0000 mgr.y (mgr.14430) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:43.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:43 smithi006 bash[27292]: cluster 2023-08-27T21:53:42.603023+0000 mgr.y (mgr.14430) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:44.806 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:44 smithi130 systemd[1]: Started Ceph grafana.a for b696f380-4522-11ee-817b-1b7a5f1ba315. 2023-08-27T21:53:45.412 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:53:45.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:45 smithi006 bash[33918]: cluster 2023-08-27T21:53:44.603566+0000 mgr.y (mgr.14430) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:45.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:45 smithi006 bash[33918]: audit 2023-08-27T21:53:44.762683+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:45.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:45 smithi006 bash[33918]: audit 2023-08-27T21:53:44.765035+0000 mon.a (mon.0) 630 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:53:45.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:45 smithi006 bash[33918]: audit 2023-08-27T21:53:44.841818+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:45.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:45 smithi006 bash[27292]: cluster 2023-08-27T21:53:44.603566+0000 mgr.y (mgr.14430) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:45.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:45 smithi006 bash[27292]: audit 2023-08-27T21:53:44.762683+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:45.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:45 smithi006 bash[27292]: audit 2023-08-27T21:53:44.765035+0000 mon.a (mon.0) 630 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:53:45.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:45 smithi006 bash[27292]: audit 2023-08-27T21:53:44.841818+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:45.955 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[27579]: cluster 2023-08-27T21:53:44.603566+0000 mgr.y (mgr.14430) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:45.955 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[27579]: audit 2023-08-27T21:53:44.762683+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:45.955 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[27579]: audit 2023-08-27T21:53:44.765035+0000 mon.a (mon.0) 630 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:53:45.956 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[27579]: audit 2023-08-27T21:53:44.841818+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:46.225 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[48144]: t=2023-08-27T21:53:45+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-08-27T21:53:46.225 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[48144]: t=2023-08-27T21:53:45+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-08-27T21:53:46.225 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[48144]: t=2023-08-27T21:53:45+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-08-27T21:53:46.225 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[48144]: t=2023-08-27T21:53:45+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-08-27T21:53:46.225 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[48144]: t=2023-08-27T21:53:45+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-08-27T21:53:46.226 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[48144]: t=2023-08-27T21:53:45+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-08-27T21:53:46.226 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[48144]: t=2023-08-27T21:53:45+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-08-27T21:53:46.226 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[48144]: t=2023-08-27T21:53:45+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-08-27T21:53:46.226 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[48144]: t=2023-08-27T21:53:45+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-08-27T21:53:46.226 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[48144]: t=2023-08-27T21:53:45+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-08-27T21:53:46.226 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[48144]: t=2023-08-27T21:53:45+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-08-27T21:53:46.226 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[48144]: t=2023-08-27T21:53:45+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-08-27T21:53:46.227 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[48144]: t=2023-08-27T21:53:45+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-08-27T21:53:46.227 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[48144]: t=2023-08-27T21:53:45+0000 lvl=info msg="App mode production" logger=settings 2023-08-27T21:53:46.227 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[48144]: t=2023-08-27T21:53:45+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-08-27T21:53:46.228 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[48144]: t=2023-08-27T21:53:45+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-08-27T21:53:46.228 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[48144]: t=2023-08-27T21:53:45+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-08-27T21:53:46.228 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[48144]: t=2023-08-27T21:53:45+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2023-08-27T21:53:46.228 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:45 smithi130 bash[48144]: t=2023-08-27T21:53:45+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2023-08-27T21:53:46.228 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2023-08-27T21:53:46.228 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2023-08-27T21:53:46.229 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2023-08-27T21:53:46.229 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2023-08-27T21:53:46.229 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2023-08-27T21:53:46.229 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2023-08-27T21:53:46.229 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2023-08-27T21:53:46.229 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2023-08-27T21:53:46.501 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-08-27T21:53:46.502 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2023-08-27T21:53:46.502 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2023-08-27T21:53:46.502 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2023-08-27T21:53:46.502 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2023-08-27T21:53:46.502 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2023-08-27T21:53:46.502 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2023-08-27T21:53:46.502 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2023-08-27T21:53:46.502 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2023-08-27T21:53:46.753 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2023-08-27T21:53:46.754 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2023-08-27T21:53:46.754 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2023-08-27T21:53:46.754 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2023-08-27T21:53:46.754 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2023-08-27T21:53:46.754 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2023-08-27T21:53:46.754 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2023-08-27T21:53:46.754 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2023-08-27T21:53:47.023 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:53:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:53:47.023 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2023-08-27T21:53:47.023 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2023-08-27T21:53:47.023 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-08-27T21:53:47.023 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2023-08-27T21:53:47.023 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:46 smithi130 bash[48144]: t=2023-08-27T21:53:46+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2023-08-27T21:53:47.275 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2023-08-27T21:53:47.275 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2023-08-27T21:53:47.275 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2023-08-27T21:53:47.276 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2023-08-27T21:53:47.276 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2023-08-27T21:53:47.276 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2023-08-27T21:53:47.276 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2023-08-27T21:53:47.276 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2023-08-27T21:53:47.276 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2023-08-27T21:53:47.561 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2023-08-27T21:53:47.561 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2023-08-27T21:53:47.562 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2023-08-27T21:53:47.562 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2023-08-27T21:53:47.562 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2023-08-27T21:53:47.562 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2023-08-27T21:53:47.562 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2023-08-27T21:53:47.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[27579]: cluster 2023-08-27T21:53:46.604386+0000 mgr.y (mgr.14430) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:47.864 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2023-08-27T21:53:47.864 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2023-08-27T21:53:47.865 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2023-08-27T21:53:47.865 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2023-08-27T21:53:47.865 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2023-08-27T21:53:47.865 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2023-08-27T21:53:47.865 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-08-27T21:53:47.866 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2023-08-27T21:53:47.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:47 smithi006 bash[33918]: cluster 2023-08-27T21:53:46.604386+0000 mgr.y (mgr.14430) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:47.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:47 smithi006 bash[27292]: cluster 2023-08-27T21:53:46.604386+0000 mgr.y (mgr.14430) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:48.120 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2023-08-27T21:53:48.120 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2023-08-27T21:53:48.120 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2023-08-27T21:53:48.120 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2023-08-27T21:53:48.121 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:47 smithi130 bash[48144]: t=2023-08-27T21:53:47+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2023-08-27T21:53:48.121 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2023-08-27T21:53:48.121 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2023-08-27T21:53:48.121 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2023-08-27T21:53:48.385 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2023-08-27T21:53:48.385 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2023-08-27T21:53:48.386 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2023-08-27T21:53:48.386 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2023-08-27T21:53:48.386 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2023-08-27T21:53:48.386 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2023-08-27T21:53:48.386 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2023-08-27T21:53:48.386 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2023-08-27T21:53:48.665 INFO:teuthology.orchestra.run.smithi006.stdout: 2023-08-27T21:53:48.665 INFO:teuthology.orchestra.run.smithi006.stdout:{"epoch":55,"fsid":"b696f380-4522-11ee-817b-1b7a5f1ba315","created":"2023-08-27T21:43:51.020683+0000","modified":"2023-08-27T21:53:22.364540+0000","last_up_change":"2023-08-27T21:52:09.451350+0000","last_in_change":"2023-08-27T21:51:42.237602+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-08-27T21:48:22.209678+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-08-27T21:53:15.972053+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"226c7900-24be-48c6-98fa-97893e22d328","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6802","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6803","nonce":224143379}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6804","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6805","nonce":224143379}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6808","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6809","nonce":224143379}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6806","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6807","nonce":224143379}]},"public_addr":"172.21.15.6:6803/224143379","cluster_addr":"172.21.15.6:6805/224143379","heartbeat_back_addr":"172.21.15.6:6809/224143379","heartbeat_front_addr":"172.21.15.6:6807/224143379","state":["exists","up"]},{"osd":1,"uuid":"55ceda65-564a-4b00-89c2-718610d59762","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6810","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6811","nonce":2798606708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6812","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6813","nonce":2798606708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6816","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6817","nonce":2798606708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6814","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6815","nonce":2798606708}]},"public_addr":"172.21.15.6:6811/2798606708","cluster_addr":"172.21.15.6:6813/2798606708","heartbeat_back_addr":"172.21.15.6:6817/2798606708","heartbeat_front_addr":"172.21.15.6:6815/2798606708","state":["exists","up"]},{"osd":2,"uuid":"b027a54f-cf0e-4547-a530-f70e146cb386","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6818","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6819","nonce":3315375049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6820","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6821","nonce":3315375049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6824","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6825","nonce":3315375049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6822","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6823","nonce":3315375049}]},"public_addr":"172.21.15.6:6819/3315375049","cluster_addr":"172.21.15.6:6821/3315375049","heartbeat_back_addr":"172.21.15.6:6825/3315375049","heartbeat_front_addr":"172.21.15.6:6823/3315375049","state":["exists","up"]},{"osd":3,"uuid":"74cd432a-aa78-4538-81bf-a4b209d60f31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6826","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6827","nonce":1657061419}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6828","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6829","nonce":1657061419}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6832","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6833","nonce":1657061419}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6830","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6831","nonce":1657061419}]},"public_addr":"172.21.15.6:6827/1657061419","cluster_addr":"172.21.15.6:6829/1657061419","heartbeat_back_addr":"172.21.15.6:6833/1657061419","heartbeat_front_addr":"172.21.15.6:6831/1657061419","state":["exists","up"]},{"osd":4,"uuid":"5710ffb1-d311-45a8-99bf-d01a7bb65cd1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6801","nonce":576819502}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6803","nonce":576819502}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6807","nonce":576819502}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6805","nonce":576819502}]},"public_addr":"172.21.15.130:6801/576819502","cluster_addr":"172.21.15.130:6803/576819502","heartbeat_back_addr":"172.21.15.130:6807/576819502","heartbeat_front_addr":"172.21.15.130:6805/576819502","state":["exists","up"]},{"osd":5,"uuid":"c0b6bba7-d752-4a95-927b-8a378bcd77b4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6809","nonce":169159537}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6811","nonce":169159537}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6815","nonce":169159537}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6813","nonce":169159537}]},"public_addr":"172.21.15.130:6809/169159537","cluster_addr":"172.21.15.130:6811/169159537","heartbeat_back_addr":"172.21.15.130:6815/169159537","heartbeat_front_addr":"172.21.15.130:6813/169159537","state":["exists","up"]},{"osd":6,"uuid":"fa4c8b52-d88a-427b-bac6-a1abaa0e92e6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6817","nonce":1800436766}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6819","nonce":1800436766}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6823","nonce":1800436766}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6821","nonce":1800436766}]},"public_addr":"172.21.15.130:6817/1800436766","cluster_addr":"172.21.15.130:6819/1800436766","heartbeat_back_addr":"172.21.15.130:6823/1800436766","heartbeat_front_addr":"172.21.15.130:6821/1800436766","state":["exists","up"]},{"osd":7,"uuid":"ac7e358c-d0fd-4b6c-a2b5-7006c5779948","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6825","nonce":2598634202}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6827","nonce":2598634202}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6831","nonce":2598634202}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6829","nonce":2598634202}]},"public_addr":"172.21.15.130:6825/2598634202","cluster_addr":"172.21.15.130:6827/2598634202","heartbeat_back_addr":"172.21.15.130:6831/2598634202","heartbeat_front_addr":"172.21.15.130:6829/2598634202","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:46:47.390747+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:47:33.822605+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:48:19.486341+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:49:05.565702+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:49:49.732445+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:50:35.236868+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:51:21.327320+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:52:07.813491+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.6:0/2156177013":"2023-08-28T21:52:44.567266+0000","172.21.15.6:0/4156113241":"2023-08-28T21:52:44.567266+0000","172.21.15.6:0/626906238":"2023-08-28T21:44:51.111365+0000","172.21.15.6:0/1056882729":"2023-08-28T21:44:51.111365+0000","172.21.15.6:6800/1902722306":"2023-08-28T21:44:51.111365+0000","172.21.15.6:0/3288657189":"2023-08-28T21:44:20.130814+0000","172.21.15.6:6800/3591185797":"2023-08-28T21:44:20.130814+0000","172.21.15.6:6800/2215677068":"2023-08-28T21:52:44.567266+0000","172.21.15.6:6801/1902722306":"2023-08-28T21:44:51.111365+0000","172.21.15.6:6801/3591185797":"2023-08-28T21:44:20.130814+0000","172.21.15.6:6801/2215677068":"2023-08-28T21:52:44.567266+0000","172.21.15.6:0/3143467986":"2023-08-28T21:44:20.130814+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-08-27T21:53:48.677 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2023-08-27T21:53:48.677 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2023-08-27T21:53:48.677 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2023-08-27T21:53:48.677 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2023-08-27T21:53:48.677 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2023-08-27T21:53:48.678 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2023-08-27T21:53:48.678 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2023-08-27T21:53:48.678 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2023-08-27T21:53:48.678 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2023-08-27T21:53:48.941 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[27579]: audit 2023-08-27T21:53:48.658408+0000 mon.a (mon.0) 632 : audit [DBG] from='client.? 172.21.15.6:0/3175468663' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T21:53:48.941 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-08-27T21:53:48.942 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2023-08-27T21:53:48.942 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-08-27T21:53:48.942 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-08-27T21:53:48.942 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2023-08-27T21:53:48.943 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2023-08-27T21:53:48.943 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2023-08-27T21:53:48.943 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2023-08-27T21:53:48.943 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2023-08-27T21:53:48.943 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2023-08-27T21:53:48.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:48 smithi006 bash[33918]: audit 2023-08-27T21:53:48.658408+0000 mon.a (mon.0) 632 : audit [DBG] from='client.? 172.21.15.6:0/3175468663' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T21:53:48.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:48 smithi006 bash[27292]: audit 2023-08-27T21:53:48.658408+0000 mon.a (mon.0) 632 : audit [DBG] from='client.? 172.21.15.6:0/3175468663' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T21:53:49.193 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:48 smithi130 bash[48144]: t=2023-08-27T21:53:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2023-08-27T21:53:49.194 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2023-08-27T21:53:49.194 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2023-08-27T21:53:49.194 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2023-08-27T21:53:49.194 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2023-08-27T21:53:49.194 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2023-08-27T21:53:49.194 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2023-08-27T21:53:49.194 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2023-08-27T21:53:49.397 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph tell osd.0 flush_pg_stats 2023-08-27T21:53:49.398 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph tell osd.1 flush_pg_stats 2023-08-27T21:53:49.398 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph tell osd.2 flush_pg_stats 2023-08-27T21:53:49.399 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph tell osd.3 flush_pg_stats 2023-08-27T21:53:49.399 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph tell osd.4 flush_pg_stats 2023-08-27T21:53:49.400 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph tell osd.5 flush_pg_stats 2023-08-27T21:53:49.400 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph tell osd.6 flush_pg_stats 2023-08-27T21:53:49.401 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph tell osd.7 flush_pg_stats 2023-08-27T21:53:49.445 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2023-08-27T21:53:49.445 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2023-08-27T21:53:49.445 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-08-27T21:53:49.446 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2023-08-27T21:53:49.446 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2023-08-27T21:53:49.446 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2023-08-27T21:53:49.446 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2023-08-27T21:53:49.446 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2023-08-27T21:53:49.723 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[27579]: cluster 2023-08-27T21:53:48.604984+0000 mgr.y (mgr.14430) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:49.724 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2023-08-27T21:53:49.724 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2023-08-27T21:53:49.724 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2023-08-27T21:53:49.724 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2023-08-27T21:53:49.724 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2023-08-27T21:53:49.724 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2023-08-27T21:53:49.724 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2023-08-27T21:53:49.724 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2023-08-27T21:53:49.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:49 smithi006 bash[27292]: cluster 2023-08-27T21:53:48.604984+0000 mgr.y (mgr.14430) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:49.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:49 smithi006 bash[33918]: cluster 2023-08-27T21:53:48.604984+0000 mgr.y (mgr.14430) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:50.001 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2023-08-27T21:53:50.001 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2023-08-27T21:53:50.001 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2023-08-27T21:53:50.001 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2023-08-27T21:53:50.001 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2023-08-27T21:53:50.001 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2023-08-27T21:53:50.253 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:49 smithi130 bash[48144]: t=2023-08-27T21:53:49+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2023-08-27T21:53:50.253 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2023-08-27T21:53:50.254 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2023-08-27T21:53:50.254 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2023-08-27T21:53:50.254 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2023-08-27T21:53:50.254 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2023-08-27T21:53:50.254 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2023-08-27T21:53:50.539 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2023-08-27T21:53:50.540 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2023-08-27T21:53:50.540 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2023-08-27T21:53:50.540 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2023-08-27T21:53:50.540 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2023-08-27T21:53:50.540 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2023-08-27T21:53:50.809 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2023-08-27T21:53:50.809 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-08-27T21:53:50.809 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2023-08-27T21:53:50.809 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2023-08-27T21:53:50.809 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2023-08-27T21:53:50.809 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2023-08-27T21:53:50.810 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2023-08-27T21:53:51.078 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-08-27T21:53:51.078 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2023-08-27T21:53:51.078 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2023-08-27T21:53:51.078 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2023-08-27T21:53:51.078 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:50 smithi130 bash[48144]: t=2023-08-27T21:53:50+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2023-08-27T21:53:51.078 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-08-27T21:53:51.078 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2023-08-27T21:53:51.078 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2023-08-27T21:53:51.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:53:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:53:50] "GET /metrics HTTP/1.1" 200 155649 "" "Prometheus/2.33.4" 2023-08-27T21:53:51.370 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2023-08-27T21:53:51.370 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2023-08-27T21:53:51.370 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2023-08-27T21:53:51.370 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2023-08-27T21:53:51.370 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2023-08-27T21:53:51.370 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2023-08-27T21:53:51.370 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2023-08-27T21:53:51.642 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2023-08-27T21:53:51.642 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2023-08-27T21:53:51.642 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2023-08-27T21:53:51.642 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2023-08-27T21:53:51.642 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2023-08-27T21:53:51.642 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2023-08-27T21:53:51.642 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2023-08-27T21:53:51.643 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2023-08-27T21:53:51.643 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2023-08-27T21:53:51.643 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2023-08-27T21:53:51.643 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2023-08-27T21:53:51.700 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:51 smithi006 bash[27292]: cluster 2023-08-27T21:53:50.605455+0000 mgr.y (mgr.14430) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:51.700 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:51 smithi006 bash[27292]: audit 2023-08-27T21:53:50.691678+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:51.700 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:51 smithi006 bash[27292]: audit 2023-08-27T21:53:50.900581+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:51.700 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:51 smithi006 bash[27292]: audit 2023-08-27T21:53:51.197917+0000 mon.a (mon.0) 635 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:53:51.700 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:51 smithi006 bash[27292]: audit 2023-08-27T21:53:51.199647+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:53:51.701 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:51 smithi006 bash[27292]: audit 2023-08-27T21:53:51.208227+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:51.911 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[27579]: cluster 2023-08-27T21:53:50.605455+0000 mgr.y (mgr.14430) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:51.911 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[27579]: audit 2023-08-27T21:53:50.691678+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:51.911 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[27579]: audit 2023-08-27T21:53:50.900581+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:51.911 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[27579]: audit 2023-08-27T21:53:51.197917+0000 mon.a (mon.0) 635 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:53:51.911 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[27579]: audit 2023-08-27T21:53:51.199647+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:53:51.911 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[27579]: audit 2023-08-27T21:53:51.208227+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:51.911 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2023-08-27T21:53:51.912 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-08-27T21:53:51.912 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-08-27T21:53:51.913 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-08-27T21:53:51.913 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2023-08-27T21:53:51.913 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-08-27T21:53:51.913 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2023-08-27T21:53:51.913 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2023-08-27T21:53:51.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:51 smithi006 bash[33918]: cluster 2023-08-27T21:53:50.605455+0000 mgr.y (mgr.14430) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:51.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:51 smithi006 bash[33918]: audit 2023-08-27T21:53:50.691678+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:51.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:51 smithi006 bash[33918]: audit 2023-08-27T21:53:50.900581+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:51.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:51 smithi006 bash[33918]: audit 2023-08-27T21:53:51.197917+0000 mon.a (mon.0) 635 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:53:51.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:51 smithi006 bash[33918]: audit 2023-08-27T21:53:51.199647+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:53:51.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:51 smithi006 bash[33918]: audit 2023-08-27T21:53:51.208227+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:53:52.188 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2023-08-27T21:53:52.188 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:51 smithi130 bash[48144]: t=2023-08-27T21:53:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2023-08-27T21:53:52.189 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2023-08-27T21:53:52.189 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2023-08-27T21:53:52.189 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2023-08-27T21:53:52.189 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2023-08-27T21:53:52.189 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2023-08-27T21:53:52.189 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2023-08-27T21:53:52.189 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2023-08-27T21:53:52.450 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2023-08-27T21:53:52.450 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2023-08-27T21:53:52.450 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2023-08-27T21:53:52.450 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-08-27T21:53:52.450 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2023-08-27T21:53:52.450 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2023-08-27T21:53:52.450 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2023-08-27T21:53:52.450 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2023-08-27T21:53:52.451 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2023-08-27T21:53:52.701 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2023-08-27T21:53:52.701 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2023-08-27T21:53:52.701 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2023-08-27T21:53:52.701 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2023-08-27T21:53:52.701 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2023-08-27T21:53:52.701 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2023-08-27T21:53:52.701 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2023-08-27T21:53:52.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:52 smithi006 bash[27292]: cephadm 2023-08-27T21:53:51.213488+0000 mgr.y (mgr.14430) 51 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-08-27T21:53:52.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:52 smithi006 bash[27292]: cephadm 2023-08-27T21:53:51.217429+0000 mgr.y (mgr.14430) 52 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi006 2023-08-27T21:53:52.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:52 smithi006 bash[33918]: cephadm 2023-08-27T21:53:51.213488+0000 mgr.y (mgr.14430) 51 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-08-27T21:53:52.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:52 smithi006 bash[33918]: cephadm 2023-08-27T21:53:51.217429+0000 mgr.y (mgr.14430) 52 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi006 2023-08-27T21:53:52.979 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[27579]: cephadm 2023-08-27T21:53:51.213488+0000 mgr.y (mgr.14430) 51 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-08-27T21:53:52.980 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[27579]: cephadm 2023-08-27T21:53:51.217429+0000 mgr.y (mgr.14430) 52 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi006 2023-08-27T21:53:52.980 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2023-08-27T21:53:52.980 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2023-08-27T21:53:52.980 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2023-08-27T21:53:52.980 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2023-08-27T21:53:52.980 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2023-08-27T21:53:52.980 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2023-08-27T21:53:52.980 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2023-08-27T21:53:52.980 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2023-08-27T21:53:52.981 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:52 smithi130 bash[48144]: t=2023-08-27T21:53:52+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2023-08-27T21:53:53.240 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2023-08-27T21:53:53.241 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-08-27T21:53:53.241 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-08-27T21:53:53.241 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2023-08-27T21:53:53.241 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-08-27T21:53:53.241 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2023-08-27T21:53:53.241 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2023-08-27T21:53:53.241 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2023-08-27T21:53:53.493 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2023-08-27T21:53:53.493 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2023-08-27T21:53:53.493 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2023-08-27T21:53:53.493 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2023-08-27T21:53:53.494 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2023-08-27T21:53:53.494 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2023-08-27T21:53:53.494 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2023-08-27T21:53:53.494 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2023-08-27T21:53:53.745 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[27579]: cluster 2023-08-27T21:53:52.605991+0000 mgr.y (mgr.14430) 53 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:53.746 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2023-08-27T21:53:53.746 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2023-08-27T21:53:53.746 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-08-27T21:53:53.746 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-08-27T21:53:53.746 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2023-08-27T21:53:53.746 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2023-08-27T21:53:53.746 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2023-08-27T21:53:53.747 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2023-08-27T21:53:53.820 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:53 smithi006 bash[33918]: cluster 2023-08-27T21:53:52.605991+0000 mgr.y (mgr.14430) 53 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:53.820 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:53 smithi006 bash[27292]: cluster 2023-08-27T21:53:52.605991+0000 mgr.y (mgr.14430) 53 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:54.006 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2023-08-27T21:53:54.006 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2023-08-27T21:53:54.006 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-08-27T21:53:54.006 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2023-08-27T21:53:54.007 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2023-08-27T21:53:54.007 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2023-08-27T21:53:54.007 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:53 smithi130 bash[48144]: t=2023-08-27T21:53:53+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2023-08-27T21:53:54.012 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:53:54.013 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:53:54.015 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:53:54.016 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:53:54.023 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:53:54.023 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:53:54.024 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:53:54.026 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:53:54.267 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2023-08-27T21:53:54.267 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2023-08-27T21:53:54.267 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2023-08-27T21:53:54.267 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2023-08-27T21:53:54.267 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2023-08-27T21:53:54.267 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2023-08-27T21:53:54.268 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2023-08-27T21:53:54.268 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2023-08-27T21:53:54.268 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2023-08-27T21:53:54.519 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2023-08-27T21:53:54.519 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2023-08-27T21:53:54.519 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2023-08-27T21:53:54.520 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2023-08-27T21:53:54.520 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2023-08-27T21:53:54.520 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2023-08-27T21:53:54.520 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2023-08-27T21:53:54.520 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2023-08-27T21:53:54.788 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2023-08-27T21:53:54.789 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2023-08-27T21:53:54.789 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2023-08-27T21:53:54.789 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2023-08-27T21:53:54.789 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2023-08-27T21:53:54.789 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2023-08-27T21:53:54.789 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2023-08-27T21:53:54.789 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-08-27T21:53:54.789 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2023-08-27T21:53:54.789 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2023-08-27T21:53:55.050 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2023-08-27T21:53:55.050 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2023-08-27T21:53:55.050 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2023-08-27T21:53:55.050 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-08-27T21:53:55.050 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2023-08-27T21:53:55.050 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:54 smithi130 bash[48144]: t=2023-08-27T21:53:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2023-08-27T21:53:55.050 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2023-08-27T21:53:55.050 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2023-08-27T21:53:55.302 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2023-08-27T21:53:55.302 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2023-08-27T21:53:55.302 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2023-08-27T21:53:55.302 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2023-08-27T21:53:55.303 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2023-08-27T21:53:55.303 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2023-08-27T21:53:55.303 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2023-08-27T21:53:55.303 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2023-08-27T21:53:55.566 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2023-08-27T21:53:55.566 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2023-08-27T21:53:55.566 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2023-08-27T21:53:55.566 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2023-08-27T21:53:55.566 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2023-08-27T21:53:55.566 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2023-08-27T21:53:55.566 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2023-08-27T21:53:55.566 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2023-08-27T21:53:55.567 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2023-08-27T21:53:55.815 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[27579]: cluster 2023-08-27T21:53:54.606554+0000 mgr.y (mgr.14430) 54 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:55.816 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2023-08-27T21:53:55.816 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2023-08-27T21:53:55.816 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2023-08-27T21:53:55.816 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2023-08-27T21:53:55.816 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2023-08-27T21:53:55.816 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2023-08-27T21:53:55.816 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2023-08-27T21:53:55.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:55 smithi006 bash[27292]: cluster 2023-08-27T21:53:54.606554+0000 mgr.y (mgr.14430) 54 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:55.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:55 smithi006 bash[33918]: cluster 2023-08-27T21:53:54.606554+0000 mgr.y (mgr.14430) 54 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:56.067 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2023-08-27T21:53:56.068 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2023-08-27T21:53:56.068 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-08-27T21:53:56.068 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-08-27T21:53:56.068 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2023-08-27T21:53:56.068 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:55 smithi130 bash[48144]: t=2023-08-27T21:53:55+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2023-08-27T21:53:56.068 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2023-08-27T21:53:56.337 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-08-27T21:53:56.337 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2023-08-27T21:53:56.337 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2023-08-27T21:53:56.337 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-08-27T21:53:56.337 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2023-08-27T21:53:56.337 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2023-08-27T21:53:56.337 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2023-08-27T21:53:56.337 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-08-27T21:53:56.337 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2023-08-27T21:53:56.606 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2023-08-27T21:53:56.606 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2023-08-27T21:53:56.606 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2023-08-27T21:53:56.606 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-08-27T21:53:56.606 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2023-08-27T21:53:56.606 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2023-08-27T21:53:56.607 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2023-08-27T21:53:56.607 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2023-08-27T21:53:56.607 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2023-08-27T21:53:56.870 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-08-27T21:53:56.870 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2023-08-27T21:53:56.870 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2023-08-27T21:53:56.870 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-08-27T21:53:56.870 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2023-08-27T21:53:56.870 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-08-27T21:53:56.871 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-08-27T21:53:56.871 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2023-08-27T21:53:57.144 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:53:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:53:57.145 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2023-08-27T21:53:57.145 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2023-08-27T21:53:57.145 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2023-08-27T21:53:57.145 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:56 smithi130 bash[48144]: t=2023-08-27T21:53:56+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2023-08-27T21:53:57.145 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2023-08-27T21:53:57.146 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-08-27T21:53:57.146 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2023-08-27T21:53:57.146 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2023-08-27T21:53:57.146 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2023-08-27T21:53:57.399 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2023-08-27T21:53:57.400 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2023-08-27T21:53:57.400 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2023-08-27T21:53:57.400 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2023-08-27T21:53:57.400 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2023-08-27T21:53:57.400 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2023-08-27T21:53:57.400 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2023-08-27T21:53:57.400 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2023-08-27T21:53:57.677 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2023-08-27T21:53:57.677 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2023-08-27T21:53:57.677 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2023-08-27T21:53:57.677 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2023-08-27T21:53:57.677 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2023-08-27T21:53:57.678 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2023-08-27T21:53:57.678 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2023-08-27T21:53:57.678 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2023-08-27T21:53:57.678 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2023-08-27T21:53:57.945 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[27579]: cluster 2023-08-27T21:53:56.607135+0000 mgr.y (mgr.14430) 55 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:57.946 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2023-08-27T21:53:57.946 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2023-08-27T21:53:57.946 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2023-08-27T21:53:57.946 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2023-08-27T21:53:57.946 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2023-08-27T21:53:57.946 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2023-08-27T21:53:57.946 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2023-08-27T21:53:57.947 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2023-08-27T21:53:57.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:57 smithi006 bash[33918]: cluster 2023-08-27T21:53:56.607135+0000 mgr.y (mgr.14430) 55 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:57.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:57 smithi006 bash[27292]: cluster 2023-08-27T21:53:56.607135+0000 mgr.y (mgr.14430) 55 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:58.224 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:57 smithi130 bash[48144]: t=2023-08-27T21:53:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2023-08-27T21:53:58.224 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2023-08-27T21:53:58.224 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2023-08-27T21:53:58.224 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2023-08-27T21:53:58.224 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2023-08-27T21:53:58.224 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2023-08-27T21:53:58.224 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2023-08-27T21:53:58.224 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2023-08-27T21:53:58.224 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2023-08-27T21:53:58.478 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2023-08-27T21:53:58.478 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2023-08-27T21:53:58.478 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2023-08-27T21:53:58.478 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2023-08-27T21:53:58.478 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2023-08-27T21:53:58.478 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2023-08-27T21:53:58.478 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=12.49390073s 2023-08-27T21:53:58.478 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-08-27T21:53:58.478 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-08-27T21:53:58.518 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:53:58 smithi006 systemd[1]: Stopping Ceph alertmanager.a for b696f380-4522-11ee-817b-1b7a5f1ba315... 2023-08-27T21:53:58.518 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:53:58 smithi006 bash[55657]: Error response from daemon: No such container: ceph-b696f380-4522-11ee-817b-1b7a5f1ba315-alertmanager.a 2023-08-27T21:53:58.779 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-08-27T21:53:58.779 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-08-27T21:53:58.779 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-08-27T21:53:58.779 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-08-27T21:53:58.779 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-08-27T21:53:58.780 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-08-27T21:53:58.780 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-08-27T21:53:58.780 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-08-27T21:53:58.780 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 21:53:58 smithi130 bash[48144]: t=2023-08-27T21:53:58+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-08-27T21:53:59.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:53:59 smithi006 bash[33918]: cluster 2023-08-27T21:53:58.607553+0000 mgr.y (mgr.14430) 56 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:53:59.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:53:59 smithi006 bash[27292]: cluster 2023-08-27T21:53:58.607553+0000 mgr.y (mgr.14430) 56 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:00.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:53:59 smithi130 bash[27579]: cluster 2023-08-27T21:53:58.607553+0000 mgr.y (mgr.14430) 56 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:00.969 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:54:00] "GET /metrics HTTP/1.1" 200 155644 "" "Prometheus/2.33.4" 2023-08-27T21:54:00.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:54:00 smithi006 bash[53461]: level=info ts=2023-08-27T21:54:00.579Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-08-27T21:54:01.987 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:01 smithi006 bash[27292]: cluster 2023-08-27T21:54:00.608163+0000 mgr.y (mgr.14430) 57 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:01.987 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:01 smithi006 bash[33918]: cluster 2023-08-27T21:54:00.608163+0000 mgr.y (mgr.14430) 57 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:02.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:01 smithi130 bash[27579]: cluster 2023-08-27T21:54:00.608163+0000 mgr.y (mgr.14430) 57 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:03.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:03 smithi006 bash[27292]: cluster 2023-08-27T21:54:02.608721+0000 mgr.y (mgr.14430) 58 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:03.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:03 smithi006 bash[33918]: cluster 2023-08-27T21:54:02.608721+0000 mgr.y (mgr.14430) 58 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:03.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:03 smithi130 bash[27579]: cluster 2023-08-27T21:54:02.608721+0000 mgr.y (mgr.14430) 58 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:05.669 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:54:05 smithi006 bash[55915]: ceph-b696f380-4522-11ee-817b-1b7a5f1ba315-alertmanager-a 2023-08-27T21:54:05.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:05 smithi006 bash[27292]: cluster 2023-08-27T21:54:04.609247+0000 mgr.y (mgr.14430) 59 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:05.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:05 smithi006 bash[33918]: cluster 2023-08-27T21:54:04.609247+0000 mgr.y (mgr.14430) 59 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:06.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:05 smithi130 bash[27579]: cluster 2023-08-27T21:54:04.609247+0000 mgr.y (mgr.14430) 59 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:06.556 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:54:06 smithi006 systemd[1]: ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@alertmanager.a.service: Succeeded. 2023-08-27T21:54:06.556 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:54:06 smithi006 systemd[1]: Stopped Ceph alertmanager.a for b696f380-4522-11ee-817b-1b7a5f1ba315. 2023-08-27T21:54:06.557 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:54:06 smithi006 systemd[1]: Started Ceph alertmanager.a for b696f380-4522-11ee-817b-1b7a5f1ba315. 2023-08-27T21:54:07.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:54:06 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:54:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:54:07.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:07 smithi130 bash[27579]: audit 2023-08-27T21:54:06.176082+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:07.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:07 smithi130 bash[27579]: cephadm 2023-08-27T21:54:06.177560+0000 mgr.y (mgr.14430) 60 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-08-27T21:54:07.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:07 smithi130 bash[27579]: cephadm 2023-08-27T21:54:06.181699+0000 mgr.y (mgr.14430) 61 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi130 2023-08-27T21:54:07.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:07 smithi130 bash[27579]: cluster 2023-08-27T21:54:06.609949+0000 mgr.y (mgr.14430) 62 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:07.381 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:07 smithi006 bash[33918]: audit 2023-08-27T21:54:06.176082+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:07.381 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:07 smithi006 bash[33918]: cephadm 2023-08-27T21:54:06.177560+0000 mgr.y (mgr.14430) 60 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-08-27T21:54:07.381 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:07 smithi006 bash[33918]: cephadm 2023-08-27T21:54:06.181699+0000 mgr.y (mgr.14430) 61 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi130 2023-08-27T21:54:07.381 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:07 smithi006 bash[33918]: cluster 2023-08-27T21:54:06.609949+0000 mgr.y (mgr.14430) 62 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:07.381 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:07 smithi006 bash[27292]: audit 2023-08-27T21:54:06.176082+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:07.382 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:07 smithi006 bash[27292]: cephadm 2023-08-27T21:54:06.177560+0000 mgr.y (mgr.14430) 60 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-08-27T21:54:07.382 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:07 smithi006 bash[27292]: cephadm 2023-08-27T21:54:06.181699+0000 mgr.y (mgr.14430) 61 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi130 2023-08-27T21:54:07.382 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:07 smithi006 bash[27292]: cluster 2023-08-27T21:54:06.609949+0000 mgr.y (mgr.14430) 62 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:09.645 INFO:teuthology.orchestra.run.smithi006.stdout:55834574928 2023-08-27T21:54:09.645 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.1 2023-08-27T21:54:09.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:09 smithi130 bash[27579]: cluster 2023-08-27T21:54:08.610588+0000 mgr.y (mgr.14430) 63 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:09.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:09 smithi006 bash[33918]: cluster 2023-08-27T21:54:08.610588+0000 mgr.y (mgr.14430) 63 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:09.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:09 smithi006 bash[27292]: cluster 2023-08-27T21:54:08.610588+0000 mgr.y (mgr.14430) 63 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:10.332 INFO:teuthology.orchestra.run.smithi006.stdout:180388626467 2023-08-27T21:54:10.333 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.6 2023-08-27T21:54:10.748 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:10 smithi130 systemd[1]: Stopping Ceph prometheus.a for b696f380-4522-11ee-817b-1b7a5f1ba315... 2023-08-27T21:54:10.749 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:10 smithi130 bash[48772]: Error response from daemon: No such container: ceph-b696f380-4522-11ee-817b-1b7a5f1ba315-prometheus.a 2023-08-27T21:54:10.749 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:10 smithi130 bash[47189]: ts=2023-08-27T21:54:10.675Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-08-27T21:54:10.749 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:10 smithi130 bash[47189]: ts=2023-08-27T21:54:10.675Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-08-27T21:54:10.749 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:10 smithi130 bash[47189]: ts=2023-08-27T21:54:10.675Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-08-27T21:54:10.749 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:10 smithi130 bash[47189]: ts=2023-08-27T21:54:10.675Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-08-27T21:54:10.749 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:10 smithi130 bash[47189]: ts=2023-08-27T21:54:10.675Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-08-27T21:54:10.749 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:10 smithi130 bash[47189]: ts=2023-08-27T21:54:10.675Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-08-27T21:54:10.749 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:10 smithi130 bash[47189]: ts=2023-08-27T21:54:10.676Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-08-27T21:54:10.749 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:10 smithi130 bash[47189]: ts=2023-08-27T21:54:10.676Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-08-27T21:54:10.749 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:10 smithi130 bash[47189]: ts=2023-08-27T21:54:10.676Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-08-27T21:54:10.749 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:10 smithi130 bash[47189]: ts=2023-08-27T21:54:10.676Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-08-27T21:54:10.750 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:10 smithi130 bash[47189]: ts=2023-08-27T21:54:10.676Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-08-27T21:54:10.750 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:10 smithi130 bash[47189]: ts=2023-08-27T21:54:10.677Z caller=main.go:1066 level=info msg="See you next time!" 2023-08-27T21:54:11.620 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:11 smithi130 bash[48810]: ceph-b696f380-4522-11ee-817b-1b7a5f1ba315-prometheus-a 2023-08-27T21:54:11.620 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:11 smithi130 systemd[1]: ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@prometheus.a.service: Succeeded. 2023-08-27T21:54:11.620 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:11 smithi130 systemd[1]: Stopped Ceph prometheus.a for b696f380-4522-11ee-817b-1b7a5f1ba315. 2023-08-27T21:54:11.620 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:11 smithi130 systemd[1]: Started Ceph prometheus.a for b696f380-4522-11ee-817b-1b7a5f1ba315. 2023-08-27T21:54:11.772 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27566]: [27/Aug/2023:21:54:11] ENGINE Bus STOPPING 2023-08-27T21:54:11.772 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27566]: [27/Aug/2023:21:54:11] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-27T21:54:11.772 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27566]: [27/Aug/2023:21:54:11] ENGINE Bus STOPPED 2023-08-27T21:54:11.772 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27566]: [27/Aug/2023:21:54:11] ENGINE Bus STARTING 2023-08-27T21:54:11.772 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27566]: [27/Aug/2023:21:54:11] ENGINE Serving on http://:::9283 2023-08-27T21:54:11.772 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27566]: [27/Aug/2023:21:54:11] ENGINE Bus STARTED 2023-08-27T21:54:11.773 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27566]: [27/Aug/2023:21:54:11] ENGINE Bus STOPPING 2023-08-27T21:54:11.773 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27566]: [27/Aug/2023:21:54:11] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-27T21:54:11.773 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27566]: [27/Aug/2023:21:54:11] ENGINE Bus STOPPED 2023-08-27T21:54:11.773 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27566]: [27/Aug/2023:21:54:11] ENGINE Bus STARTING 2023-08-27T21:54:11.773 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27292]: cluster 2023-08-27T21:54:10.611064+0000 mgr.y (mgr.14430) 64 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:11.773 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27292]: audit 2023-08-27T21:54:11.478557+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:11.773 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27292]: audit 2023-08-27T21:54:11.482785+0000 mon.a (mon.0) 640 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-27T21:54:11.773 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27292]: audit 2023-08-27T21:54:11.484871+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi006.front.sepia.ceph.com:9093"}]: dispatch 2023-08-27T21:54:11.773 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27292]: audit 2023-08-27T21:54:11.491303+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:11.774 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27292]: audit 2023-08-27T21:54:11.497065+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-27T21:54:11.774 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27292]: audit 2023-08-27T21:54:11.619510+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi130.front.sepia.ceph.com:9095"}]: dispatch 2023-08-27T21:54:11.774 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27292]: audit 2023-08-27T21:54:11.628050+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:11.774 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27292]: audit 2023-08-27T21:54:11.633453+0000 mon.a (mon.0) 646 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-27T21:54:11.774 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[33918]: cluster 2023-08-27T21:54:10.611064+0000 mgr.y (mgr.14430) 64 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:11.774 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[33918]: audit 2023-08-27T21:54:11.478557+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:11.774 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[33918]: audit 2023-08-27T21:54:11.482785+0000 mon.a (mon.0) 640 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-27T21:54:11.774 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[33918]: audit 2023-08-27T21:54:11.484871+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi006.front.sepia.ceph.com:9093"}]: dispatch 2023-08-27T21:54:11.774 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[33918]: audit 2023-08-27T21:54:11.491303+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:11.774 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[33918]: audit 2023-08-27T21:54:11.497065+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-27T21:54:11.775 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[33918]: audit 2023-08-27T21:54:11.619510+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi130.front.sepia.ceph.com:9095"}]: dispatch 2023-08-27T21:54:11.775 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[33918]: audit 2023-08-27T21:54:11.628050+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:11.775 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[33918]: audit 2023-08-27T21:54:11.633453+0000 mon.a (mon.0) 646 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-27T21:54:11.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:11 smithi130 bash[27579]: cluster 2023-08-27T21:54:10.611064+0000 mgr.y (mgr.14430) 64 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:11.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:11 smithi130 bash[27579]: audit 2023-08-27T21:54:11.478557+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:11.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:11 smithi130 bash[27579]: audit 2023-08-27T21:54:11.482785+0000 mon.a (mon.0) 640 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-27T21:54:11.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:11 smithi130 bash[27579]: audit 2023-08-27T21:54:11.484871+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi006.front.sepia.ceph.com:9093"}]: dispatch 2023-08-27T21:54:11.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:11 smithi130 bash[27579]: audit 2023-08-27T21:54:11.491303+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:11.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:11 smithi130 bash[27579]: audit 2023-08-27T21:54:11.497065+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-27T21:54:11.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:11 smithi130 bash[27579]: audit 2023-08-27T21:54:11.619510+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi130.front.sepia.ceph.com:9095"}]: dispatch 2023-08-27T21:54:11.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:11 smithi130 bash[27579]: audit 2023-08-27T21:54:11.628050+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:11.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:11 smithi130 bash[27579]: audit 2023-08-27T21:54:11.633453+0000 mon.a (mon.0) 646 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-27T21:54:12.183 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27566]: [27/Aug/2023:21:54:11] ENGINE Serving on http://:::9283 2023-08-27T21:54:12.183 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27566]: [27/Aug/2023:21:54:11] ENGINE Bus STARTED 2023-08-27T21:54:12.183 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27566]: [27/Aug/2023:21:54:11] ENGINE Bus STOPPING 2023-08-27T21:54:12.183 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27566]: [27/Aug/2023:21:54:11] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-27T21:54:12.183 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27566]: [27/Aug/2023:21:54:11] ENGINE Bus STOPPED 2023-08-27T21:54:12.183 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27566]: [27/Aug/2023:21:54:11] ENGINE Bus STARTING 2023-08-27T21:54:12.183 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27566]: [27/Aug/2023:21:54:11] ENGINE Serving on http://:::9283 2023-08-27T21:54:12.184 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:11 smithi006 bash[27566]: [27/Aug/2023:21:54:11] ENGINE Bus STARTED 2023-08-27T21:54:12.460 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[56298]: level=info ts=2023-08-27T21:54:12.290Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-08-27T21:54:12.460 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[56298]: level=info ts=2023-08-27T21:54:12.290Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-08-27T21:54:12.460 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[56298]: level=info ts=2023-08-27T21:54:12.293Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.6 port=9094 2023-08-27T21:54:12.460 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[56298]: level=info ts=2023-08-27T21:54:12.295Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-08-27T21:54:12.460 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[56298]: level=info ts=2023-08-27T21:54:12.332Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-08-27T21:54:12.460 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[56298]: level=info ts=2023-08-27T21:54:12.333Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-08-27T21:54:12.460 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[56298]: level=info ts=2023-08-27T21:54:12.336Z caller=main.go:518 msg=Listening address=:9093 2023-08-27T21:54:12.460 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[56298]: level=info ts=2023-08-27T21:54:12.336Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-08-27T21:54:12.746 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[27292]: audit 2023-08-27T21:54:11.483549+0000 mgr.y (mgr.14430) 65 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-27T21:54:12.746 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[27292]: audit 2023-08-27T21:54:11.485262+0000 mgr.y (mgr.14430) 66 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi006.front.sepia.ceph.com:9093"}]: dispatch 2023-08-27T21:54:12.746 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[27292]: audit 2023-08-27T21:54:11.497678+0000 mgr.y (mgr.14430) 67 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-27T21:54:12.746 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[27292]: audit 2023-08-27T21:54:11.620205+0000 mgr.y (mgr.14430) 68 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi130.front.sepia.ceph.com:9095"}]: dispatch 2023-08-27T21:54:12.747 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[27292]: audit 2023-08-27T21:54:11.634099+0000 mgr.y (mgr.14430) 69 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-27T21:54:12.747 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[27292]: audit 2023-08-27T21:54:11.760280+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi130.front.sepia.ceph.com:3000"}]: dispatch 2023-08-27T21:54:12.747 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[27292]: audit 2023-08-27T21:54:11.770940+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:12.863 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[27579]: audit 2023-08-27T21:54:11.483549+0000 mgr.y (mgr.14430) 65 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-27T21:54:12.863 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[27579]: audit 2023-08-27T21:54:11.485262+0000 mgr.y (mgr.14430) 66 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi006.front.sepia.ceph.com:9093"}]: dispatch 2023-08-27T21:54:12.863 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[27579]: audit 2023-08-27T21:54:11.497678+0000 mgr.y (mgr.14430) 67 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-27T21:54:12.863 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[27579]: audit 2023-08-27T21:54:11.620205+0000 mgr.y (mgr.14430) 68 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi130.front.sepia.ceph.com:9095"}]: dispatch 2023-08-27T21:54:12.863 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[27579]: audit 2023-08-27T21:54:11.634099+0000 mgr.y (mgr.14430) 69 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-27T21:54:12.863 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[27579]: audit 2023-08-27T21:54:11.760280+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi130.front.sepia.ceph.com:3000"}]: dispatch 2023-08-27T21:54:12.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[27579]: audit 2023-08-27T21:54:11.770940+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:12.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[27579]: audit 2023-08-27T21:54:11.774072+0000 mon.a (mon.0) 649 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:54:12.864 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[48907]: ts=2023-08-27T21:54:12.727Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-08-27T21:54:12.864 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[48907]: ts=2023-08-27T21:54:12.727Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-08-27T21:54:12.864 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[48907]: ts=2023-08-27T21:54:12.727Z caller=main.go:518 level=info host_details="(Linux 5.15.0-79-generic #86~20.04.2-Ubuntu SMP Mon Jul 17 23:27:17 UTC 2023 x86_64 smithi130 (none))" 2023-08-27T21:54:12.864 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[48907]: ts=2023-08-27T21:54:12.727Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-08-27T21:54:12.864 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[48907]: ts=2023-08-27T21:54:12.727Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-08-27T21:54:12.864 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[48907]: ts=2023-08-27T21:54:12.730Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-08-27T21:54:12.864 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[48907]: ts=2023-08-27T21:54:12.730Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-08-27T21:54:12.865 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[48907]: ts=2023-08-27T21:54:12.732Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-08-27T21:54:12.865 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[48907]: ts=2023-08-27T21:54:12.736Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-08-27T21:54:12.865 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[48907]: ts=2023-08-27T21:54:12.736Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.638µs 2023-08-27T21:54:12.865 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[48907]: ts=2023-08-27T21:54:12.736Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-08-27T21:54:12.865 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[48907]: ts=2023-08-27T21:54:12.859Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-08-27T21:54:13.120 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[48907]: ts=2023-08-27T21:54:12.860Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-08-27T21:54:13.120 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[48907]: ts=2023-08-27T21:54:12.860Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=106.271µs wal_replay_duration=124.082473ms total_replay_duration=124.241645ms 2023-08-27T21:54:13.120 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[48907]: ts=2023-08-27T21:54:12.863Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-08-27T21:54:13.121 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[48907]: ts=2023-08-27T21:54:12.863Z caller=main.go:947 level=info msg="TSDB started" 2023-08-27T21:54:13.121 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[48907]: ts=2023-08-27T21:54:12.863Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-08-27T21:54:13.121 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[48907]: ts=2023-08-27T21:54:12.893Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=29.893581ms db_storage=1.358µs remote_storage=2.186µs web_handler=652ns query_engine=972ns scrape=437.422µs scrape_sd=54.417µs notify=56.063µs notify_sd=18.645µs rules=28.835484ms 2023-08-27T21:54:13.121 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 27 21:54:12 smithi130 bash[48907]: ts=2023-08-27T21:54:12.893Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-08-27T21:54:13.198 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[27292]: audit 2023-08-27T21:54:11.774072+0000 mon.a (mon.0) 649 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:54:13.198 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[33918]: audit 2023-08-27T21:54:11.483549+0000 mgr.y (mgr.14430) 65 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-27T21:54:13.199 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[33918]: audit 2023-08-27T21:54:11.485262+0000 mgr.y (mgr.14430) 66 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi006.front.sepia.ceph.com:9093"}]: dispatch 2023-08-27T21:54:13.199 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[33918]: audit 2023-08-27T21:54:11.497678+0000 mgr.y (mgr.14430) 67 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-27T21:54:13.199 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[33918]: audit 2023-08-27T21:54:11.620205+0000 mgr.y (mgr.14430) 68 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi130.front.sepia.ceph.com:9095"}]: dispatch 2023-08-27T21:54:13.199 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[33918]: audit 2023-08-27T21:54:11.634099+0000 mgr.y (mgr.14430) 69 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-27T21:54:13.199 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[33918]: audit 2023-08-27T21:54:11.760280+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi130.front.sepia.ceph.com:3000"}]: dispatch 2023-08-27T21:54:13.199 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[33918]: audit 2023-08-27T21:54:11.770940+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:13.199 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:12 smithi006 bash[33918]: audit 2023-08-27T21:54:11.774072+0000 mon.a (mon.0) 649 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:54:13.411 INFO:teuthology.orchestra.run.smithi006.stdout:34359738457 2023-08-27T21:54:13.411 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.0 2023-08-27T21:54:13.432 INFO:teuthology.orchestra.run.smithi006.stdout:77309411398 2023-08-27T21:54:13.432 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.2 2023-08-27T21:54:13.733 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:13 smithi006 bash[33918]: audit 2023-08-27T21:54:11.761268+0000 mgr.y (mgr.14430) 70 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi130.front.sepia.ceph.com:3000"}]: dispatch 2023-08-27T21:54:13.733 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:13 smithi006 bash[33918]: cluster 2023-08-27T21:54:12.611660+0000 mgr.y (mgr.14430) 71 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:13.733 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:13 smithi006 bash[27292]: audit 2023-08-27T21:54:11.761268+0000 mgr.y (mgr.14430) 70 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi130.front.sepia.ceph.com:3000"}]: dispatch 2023-08-27T21:54:13.734 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:13 smithi006 bash[27292]: cluster 2023-08-27T21:54:12.611660+0000 mgr.y (mgr.14430) 71 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:13.772 INFO:teuthology.orchestra.run.smithi006.stdout:206158430234 2023-08-27T21:54:13.773 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.7 2023-08-27T21:54:14.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:13 smithi130 bash[27579]: audit 2023-08-27T21:54:11.761268+0000 mgr.y (mgr.14430) 70 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi130.front.sepia.ceph.com:3000"}]: dispatch 2023-08-27T21:54:14.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:13 smithi130 bash[27579]: cluster 2023-08-27T21:54:12.611660+0000 mgr.y (mgr.14430) 71 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:14.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:54:14 smithi006 bash[56298]: level=info ts=2023-08-27T21:54:14.295Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000319382s 2023-08-27T21:54:15.345 INFO:teuthology.orchestra.run.smithi006.stdout:128849018934 2023-08-27T21:54:15.346 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.4 2023-08-27T21:54:15.584 INFO:teuthology.orchestra.run.smithi006.stdout:154618822701 2023-08-27T21:54:15.585 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.5 2023-08-27T21:54:15.632 INFO:teuthology.orchestra.run.smithi006.stdout:107374182462 2023-08-27T21:54:15.632 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.3 2023-08-27T21:54:15.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:15 smithi006 bash[27292]: cluster 2023-08-27T21:54:14.612202+0000 mgr.y (mgr.14430) 72 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:15.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:15 smithi006 bash[33918]: cluster 2023-08-27T21:54:14.612202+0000 mgr.y (mgr.14430) 72 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:16.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:15 smithi130 bash[27579]: cluster 2023-08-27T21:54:14.612202+0000 mgr.y (mgr.14430) 72 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:17.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:17 smithi006 bash[33918]: cluster 2023-08-27T21:54:16.612777+0000 mgr.y (mgr.14430) 73 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:17.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:17 smithi006 bash[27292]: cluster 2023-08-27T21:54:16.612777+0000 mgr.y (mgr.14430) 73 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:18.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:17 smithi130 bash[27579]: cluster 2023-08-27T21:54:16.612777+0000 mgr.y (mgr.14430) 73 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:19.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:18 smithi130 bash[27579]: audit 2023-08-27T21:54:17.770481+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:19.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:18 smithi130 bash[27579]: cluster 2023-08-27T21:54:18.613324+0000 mgr.y (mgr.14430) 74 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:19.189 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:54:19.190 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:54:19.192 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:54:19.192 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:54:19.193 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:54:19.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:18 smithi006 bash[27292]: audit 2023-08-27T21:54:17.770481+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:19.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:18 smithi006 bash[27292]: cluster 2023-08-27T21:54:18.613324+0000 mgr.y (mgr.14430) 74 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:19.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:18 smithi006 bash[33918]: audit 2023-08-27T21:54:17.770481+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:19.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:18 smithi006 bash[33918]: cluster 2023-08-27T21:54:18.613324+0000 mgr.y (mgr.14430) 74 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:20.207 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:54:20.208 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:54:20.208 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:54:21.100 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:54:20] "GET /metrics HTTP/1.1" 200 155640 "" "Prometheus/2.33.4" 2023-08-27T21:54:21.101 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:21 smithi006 bash[27292]: audit 2023-08-27T21:54:20.094923+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:21.102 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:21 smithi006 bash[27292]: audit 2023-08-27T21:54:20.095880+0000 mon.a (mon.0) 652 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:54:21.102 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:21 smithi006 bash[27292]: audit 2023-08-27T21:54:20.096828+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:54:21.102 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:21 smithi006 bash[27292]: audit 2023-08-27T21:54:20.101902+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:21.102 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:21 smithi006 bash[27292]: audit 2023-08-27T21:54:20.108299+0000 mon.a (mon.0) 655 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:54:21.102 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:21 smithi006 bash[27292]: cluster 2023-08-27T21:54:20.613757+0000 mgr.y (mgr.14430) 75 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:21.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:21 smithi130 bash[27579]: audit 2023-08-27T21:54:20.094923+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:21.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:21 smithi130 bash[27579]: audit 2023-08-27T21:54:20.095880+0000 mon.a (mon.0) 652 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:54:21.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:21 smithi130 bash[27579]: audit 2023-08-27T21:54:20.096828+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:54:21.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:21 smithi130 bash[27579]: audit 2023-08-27T21:54:20.101902+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:21.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:21 smithi130 bash[27579]: audit 2023-08-27T21:54:20.108299+0000 mon.a (mon.0) 655 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:54:21.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:21 smithi130 bash[27579]: cluster 2023-08-27T21:54:20.613757+0000 mgr.y (mgr.14430) 75 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:21.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:21 smithi006 bash[33918]: audit 2023-08-27T21:54:20.094923+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:21.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:21 smithi006 bash[33918]: audit 2023-08-27T21:54:20.095880+0000 mon.a (mon.0) 652 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:54:21.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:21 smithi006 bash[33918]: audit 2023-08-27T21:54:20.096828+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:54:21.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:21 smithi006 bash[33918]: audit 2023-08-27T21:54:20.101902+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:21.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:21 smithi006 bash[33918]: audit 2023-08-27T21:54:20.108299+0000 mon.a (mon.0) 655 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:54:21.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:21 smithi006 bash[33918]: cluster 2023-08-27T21:54:20.613757+0000 mgr.y (mgr.14430) 75 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:22.402 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:54:22 smithi006 bash[56298]: level=info ts=2023-08-27T21:54:22.298Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002886997s 2023-08-27T21:54:23.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:23 smithi130 bash[27579]: cluster 2023-08-27T21:54:22.614168+0000 mgr.y (mgr.14430) 76 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:23.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:23 smithi006 bash[33918]: cluster 2023-08-27T21:54:22.614168+0000 mgr.y (mgr.14430) 76 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:23.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:23 smithi006 bash[27292]: cluster 2023-08-27T21:54:22.614168+0000 mgr.y (mgr.14430) 76 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:25.893 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:25 smithi006 bash[33918]: cluster 2023-08-27T21:54:24.614527+0000 mgr.y (mgr.14430) 77 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:25.893 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:25 smithi006 bash[27292]: cluster 2023-08-27T21:54:24.614527+0000 mgr.y (mgr.14430) 77 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:26.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:25 smithi130 bash[27579]: cluster 2023-08-27T21:54:24.614527+0000 mgr.y (mgr.14430) 77 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:27.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:27 smithi006 bash[33918]: audit 2023-08-27T21:54:26.104009+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:27.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:27 smithi006 bash[33918]: audit 2023-08-27T21:54:26.105247+0000 mon.a (mon.0) 657 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:54:27.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:27 smithi006 bash[33918]: audit 2023-08-27T21:54:26.106286+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:54:27.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:27 smithi006 bash[33918]: audit 2023-08-27T21:54:26.111675+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:27.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:27 smithi006 bash[33918]: cluster 2023-08-27T21:54:26.615174+0000 mgr.y (mgr.14430) 78 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:27.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:27 smithi006 bash[27292]: audit 2023-08-27T21:54:26.104009+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:27.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:27 smithi006 bash[27292]: audit 2023-08-27T21:54:26.105247+0000 mon.a (mon.0) 657 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:54:27.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:27 smithi006 bash[27292]: audit 2023-08-27T21:54:26.106286+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:54:27.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:27 smithi006 bash[27292]: audit 2023-08-27T21:54:26.111675+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:27 smithi006 bash[27292]: cluster 2023-08-27T21:54:26.615174+0000 mgr.y (mgr.14430) 78 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:27.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:54:26 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:54:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:54:27.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:27 smithi130 bash[27579]: audit 2023-08-27T21:54:26.104009+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:27.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:27 smithi130 bash[27579]: audit 2023-08-27T21:54:26.105247+0000 mon.a (mon.0) 657 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:54:27.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:27 smithi130 bash[27579]: audit 2023-08-27T21:54:26.106286+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:54:27.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:27 smithi130 bash[27579]: audit 2023-08-27T21:54:26.111675+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:54:27.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:27 smithi130 bash[27579]: cluster 2023-08-27T21:54:26.615174+0000 mgr.y (mgr.14430) 78 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:29.930 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:29 smithi006 bash[33918]: cluster 2023-08-27T21:54:28.615688+0000 mgr.y (mgr.14430) 79 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:29.931 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:29 smithi006 bash[27292]: cluster 2023-08-27T21:54:28.615688+0000 mgr.y (mgr.14430) 79 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:30.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:29 smithi130 bash[27579]: cluster 2023-08-27T21:54:28.615688+0000 mgr.y (mgr.14430) 79 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:30.249 INFO:teuthology.orchestra.run.smithi006.stdout:55834574932 2023-08-27T21:54:30.990 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:30 smithi006 bash[27292]: audit 2023-08-27T21:54:30.244112+0000 mon.a (mon.0) 660 : audit [DBG] from='client.? 172.21.15.6:0/2563198117' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-08-27T21:54:30.990 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:30 smithi006 bash[33918]: audit 2023-08-27T21:54:30.244112+0000 mon.a (mon.0) 660 : audit [DBG] from='client.? 172.21.15.6:0/2563198117' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-08-27T21:54:30.991 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:54:30] "GET /metrics HTTP/1.1" 200 155647 "" "Prometheus/2.33.4" 2023-08-27T21:54:31.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:30 smithi130 bash[27579]: audit 2023-08-27T21:54:30.244112+0000 mon.a (mon.0) 660 : audit [DBG] from='client.? 172.21.15.6:0/2563198117' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-08-27T21:54:31.458 INFO:teuthology.orchestra.run.smithi006.stdout:34359738461 2023-08-27T21:54:31.950 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:31 smithi006 bash[27292]: cluster 2023-08-27T21:54:30.616298+0000 mgr.y (mgr.14430) 80 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:31.950 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:31 smithi006 bash[27292]: audit 2023-08-27T21:54:31.453739+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.6:0/2689373579' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-08-27T21:54:31.950 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:31 smithi006 bash[33918]: cluster 2023-08-27T21:54:30.616298+0000 mgr.y (mgr.14430) 80 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:31.951 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:31 smithi006 bash[33918]: audit 2023-08-27T21:54:31.453739+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.6:0/2689373579' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-08-27T21:54:32.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:31 smithi130 bash[27579]: cluster 2023-08-27T21:54:30.616298+0000 mgr.y (mgr.14430) 80 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:32.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:31 smithi130 bash[27579]: audit 2023-08-27T21:54:31.453739+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.6:0/2689373579' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-08-27T21:54:32.894 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574928 got 55834574932 for osd.1 2023-08-27T21:54:32.894 DEBUG:teuthology.parallel:result is None 2023-08-27T21:54:33.309 INFO:teuthology.orchestra.run.smithi006.stdout:180388626472 2023-08-27T21:54:33.614 INFO:teuthology.orchestra.run.smithi006.stdout:77309411403 2023-08-27T21:54:33.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:33 smithi006 bash[27292]: cluster 2023-08-27T21:54:32.616896+0000 mgr.y (mgr.14430) 81 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:33.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:33 smithi006 bash[27292]: audit 2023-08-27T21:54:33.304669+0000 mon.a (mon.0) 661 : audit [DBG] from='client.? 172.21.15.6:0/1463798371' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-08-27T21:54:33.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:33 smithi006 bash[33918]: cluster 2023-08-27T21:54:32.616896+0000 mgr.y (mgr.14430) 81 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:33.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:33 smithi006 bash[33918]: audit 2023-08-27T21:54:33.304669+0000 mon.a (mon.0) 661 : audit [DBG] from='client.? 172.21.15.6:0/1463798371' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-08-27T21:54:33.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:33 smithi130 bash[27579]: cluster 2023-08-27T21:54:32.616896+0000 mgr.y (mgr.14430) 81 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:33.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:33 smithi130 bash[27579]: audit 2023-08-27T21:54:33.304669+0000 mon.a (mon.0) 661 : audit [DBG] from='client.? 172.21.15.6:0/1463798371' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-08-27T21:54:34.263 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738457 got 34359738461 for osd.0 2023-08-27T21:54:34.264 DEBUG:teuthology.parallel:result is None 2023-08-27T21:54:34.438 INFO:teuthology.orchestra.run.smithi006.stdout:154618822705 2023-08-27T21:54:34.729 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:34 smithi006 bash[33918]: audit 2023-08-27T21:54:33.609807+0000 mon.a (mon.0) 662 : audit [DBG] from='client.? 172.21.15.6:0/15996963' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-08-27T21:54:34.729 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:34 smithi006 bash[33918]: audit 2023-08-27T21:54:34.433294+0000 mon.a (mon.0) 663 : audit [DBG] from='client.? 172.21.15.6:0/3754553065' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-08-27T21:54:34.730 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:34 smithi006 bash[27292]: audit 2023-08-27T21:54:33.609807+0000 mon.a (mon.0) 662 : audit [DBG] from='client.? 172.21.15.6:0/15996963' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-08-27T21:54:34.730 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:34 smithi006 bash[27292]: audit 2023-08-27T21:54:34.433294+0000 mon.a (mon.0) 663 : audit [DBG] from='client.? 172.21.15.6:0/3754553065' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-08-27T21:54:34.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:34 smithi130 bash[27579]: audit 2023-08-27T21:54:33.609807+0000 mon.a (mon.0) 662 : audit [DBG] from='client.? 172.21.15.6:0/15996963' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-08-27T21:54:34.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:34 smithi130 bash[27579]: audit 2023-08-27T21:54:34.433294+0000 mon.a (mon.0) 663 : audit [DBG] from='client.? 172.21.15.6:0/3754553065' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-08-27T21:54:35.388 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411398 got 77309411403 for osd.2 2023-08-27T21:54:35.388 DEBUG:teuthology.parallel:result is None 2023-08-27T21:54:35.568 INFO:teuthology.orchestra.run.smithi006.stdout:128849018938 2023-08-27T21:54:35.759 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:35 smithi006 bash[33918]: cluster 2023-08-27T21:54:34.617379+0000 mgr.y (mgr.14430) 82 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:35.759 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:35 smithi006 bash[27292]: cluster 2023-08-27T21:54:34.617379+0000 mgr.y (mgr.14430) 82 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:35.810 INFO:teuthology.orchestra.run.smithi006.stdout:206158430238 2023-08-27T21:54:35.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:35 smithi130 bash[27579]: cluster 2023-08-27T21:54:34.617379+0000 mgr.y (mgr.14430) 82 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:36.128 INFO:teuthology.orchestra.run.smithi006.stdout:107374182466 2023-08-27T21:54:36.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:36 smithi006 bash[27292]: audit 2023-08-27T21:54:35.563564+0000 mon.a (mon.0) 664 : audit [DBG] from='client.? 172.21.15.6:0/3104888634' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-08-27T21:54:36.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:36 smithi006 bash[27292]: audit 2023-08-27T21:54:35.805173+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.6:0/2108803974' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-08-27T21:54:36.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:36 smithi006 bash[27292]: audit 2023-08-27T21:54:36.123699+0000 mon.a (mon.0) 665 : audit [DBG] from='client.? 172.21.15.6:0/3320575758' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-08-27T21:54:36.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:36 smithi006 bash[33918]: audit 2023-08-27T21:54:35.563564+0000 mon.a (mon.0) 664 : audit [DBG] from='client.? 172.21.15.6:0/3104888634' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-08-27T21:54:36.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:36 smithi006 bash[33918]: audit 2023-08-27T21:54:35.805173+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.6:0/2108803974' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-08-27T21:54:36.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:36 smithi006 bash[33918]: audit 2023-08-27T21:54:36.123699+0000 mon.a (mon.0) 665 : audit [DBG] from='client.? 172.21.15.6:0/3320575758' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-08-27T21:54:36.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:36 smithi130 bash[27579]: audit 2023-08-27T21:54:35.563564+0000 mon.a (mon.0) 664 : audit [DBG] from='client.? 172.21.15.6:0/3104888634' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-08-27T21:54:36.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:36 smithi130 bash[27579]: audit 2023-08-27T21:54:35.805173+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.6:0/2108803974' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-08-27T21:54:36.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:36 smithi130 bash[27579]: audit 2023-08-27T21:54:36.123699+0000 mon.a (mon.0) 665 : audit [DBG] from='client.? 172.21.15.6:0/3320575758' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-08-27T21:54:37.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:54:36 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:54:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:54:37.521 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626467 got 180388626472 for osd.6 2023-08-27T21:54:37.521 DEBUG:teuthology.parallel:result is None 2023-08-27T21:54:37.613 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:37 smithi006 bash[33918]: cluster 2023-08-27T21:54:36.617857+0000 mgr.y (mgr.14430) 83 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:37.614 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:37 smithi006 bash[27292]: cluster 2023-08-27T21:54:36.617857+0000 mgr.y (mgr.14430) 83 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:37.830 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822701 got 154618822705 for osd.5 2023-08-27T21:54:37.830 DEBUG:teuthology.parallel:result is None 2023-08-27T21:54:37.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:37 smithi130 bash[27579]: cluster 2023-08-27T21:54:36.617857+0000 mgr.y (mgr.14430) 83 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:37.937 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018934 got 128849018938 for osd.4 2023-08-27T21:54:37.938 DEBUG:teuthology.parallel:result is None 2023-08-27T21:54:38.370 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430234 got 206158430238 for osd.7 2023-08-27T21:54:38.371 DEBUG:teuthology.parallel:result is None 2023-08-27T21:54:38.500 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182462 got 107374182466 for osd.3 2023-08-27T21:54:38.500 DEBUG:teuthology.parallel:result is None 2023-08-27T21:54:38.501 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-08-27T21:54:38.501 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-27T21:54:39.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:39 smithi006 bash[27292]: cluster 2023-08-27T21:54:38.618349+0000 mgr.y (mgr.14430) 84 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:39.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:39 smithi006 bash[33918]: cluster 2023-08-27T21:54:38.618349+0000 mgr.y (mgr.14430) 84 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:40.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:39 smithi130 bash[27579]: cluster 2023-08-27T21:54:38.618349+0000 mgr.y (mgr.14430) 84 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:41.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:54:40] "GET /metrics HTTP/1.1" 200 155647 "" "Prometheus/2.33.4" 2023-08-27T21:54:41.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:41 smithi006 bash[27292]: cluster 2023-08-27T21:54:40.618984+0000 mgr.y (mgr.14430) 85 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:41.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:41 smithi006 bash[33918]: cluster 2023-08-27T21:54:40.618984+0000 mgr.y (mgr.14430) 85 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:42.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:41 smithi130 bash[27579]: cluster 2023-08-27T21:54:40.618984+0000 mgr.y (mgr.14430) 85 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:43.311 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:54:43.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:43 smithi006 bash[33918]: cluster 2023-08-27T21:54:42.619588+0000 mgr.y (mgr.14430) 86 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:43.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:43 smithi006 bash[27292]: cluster 2023-08-27T21:54:42.619588+0000 mgr.y (mgr.14430) 86 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:43.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:43 smithi130 bash[27579]: cluster 2023-08-27T21:54:42.619588+0000 mgr.y (mgr.14430) 86 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:45.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:45 smithi006 bash[27292]: cluster 2023-08-27T21:54:44.620125+0000 mgr.y (mgr.14430) 87 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:45.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:45 smithi006 bash[33918]: cluster 2023-08-27T21:54:44.620125+0000 mgr.y (mgr.14430) 87 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:46.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:45 smithi130 bash[27579]: cluster 2023-08-27T21:54:44.620125+0000 mgr.y (mgr.14430) 87 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:46.698 INFO:teuthology.orchestra.run.smithi006.stdout: 2023-08-27T21:54:46.701 INFO:teuthology.orchestra.run.smithi006.stderr:dumped all 2023-08-27T21:54:47.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:54:46 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:54:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:54:47.381 INFO:teuthology.orchestra.run.smithi006.stdout:{"pg_ready":true,"pg_map":{"version":68,"stamp":"2023-08-27T21:54:46.620383+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380968,"kb_used_data":4200,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580248,"statfs":{"total":767960285184,"available":765522173952,"internally_reserved":0,"allocated":4300800,"data_stored":1762230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-27T21:53:18.989798+0000","last_change":"2023-08-27T21:53:17.043470+0000","last_active":"2023-08-27T21:53:18.989798+0000","last_peered":"2023-08-27T21:53:18.989798+0000","last_clean":"2023-08-27T21:53:18.989798+0000","last_became_active":"2023-08-27T21:53:17.042696+0000","last_became_peered":"2023-08-27T21:53:17.042696+0000","last_unstale":"2023-08-27T21:53:18.989798+0000","last_undegraded":"2023-08-27T21:53:18.989798+0000","last_fullsized":"2023-08-27T21:53:18.989798+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-27T21:53:18.063987+0000","last_change":"2023-08-27T21:53:17.187485+0000","last_active":"2023-08-27T21:53:18.063987+0000","last_peered":"2023-08-27T21:53:18.063987+0000","last_clean":"2023-08-27T21:53:18.063987+0000","last_became_active":"2023-08-27T21:53:17.187094+0000","last_became_peered":"2023-08-27T21:53:17.187094+0000","last_unstale":"2023-08-27T21:53:18.063987+0000","last_undegraded":"2023-08-27T21:53:18.063987+0000","last_fullsized":"2023-08-27T21:53:18.063987+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-27T21:53:18.989950+0000","last_change":"2023-08-27T21:53:17.816851+0000","last_active":"2023-08-27T21:53:18.989950+0000","last_peered":"2023-08-27T21:53:18.989950+0000","last_clean":"2023-08-27T21:53:18.989950+0000","last_became_active":"2023-08-27T21:53:17.815511+0000","last_became_peered":"2023-08-27T21:53:17.815511+0000","last_unstale":"2023-08-27T21:53:18.989950+0000","last_undegraded":"2023-08-27T21:53:18.989950+0000","last_fullsized":"2023-08-27T21:53:18.989950+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-27T21:53:18.989527+0000","last_change":"2023-08-27T21:53:17.815961+0000","last_active":"2023-08-27T21:53:18.989527+0000","last_peered":"2023-08-27T21:53:18.989527+0000","last_clean":"2023-08-27T21:53:18.989527+0000","last_became_active":"2023-08-27T21:53:17.815034+0000","last_became_peered":"2023-08-27T21:53:17.815034+0000","last_unstale":"2023-08-27T21:53:18.989527+0000","last_undegraded":"2023-08-27T21:53:18.989527+0000","last_fullsized":"2023-08-27T21:53:18.989527+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-27T21:53:18.989856+0000","last_change":"2023-08-27T21:52:11.532578+0000","last_active":"2023-08-27T21:53:18.989856+0000","last_peered":"2023-08-27T21:53:18.989856+0000","last_clean":"2023-08-27T21:53:18.989856+0000","last_became_active":"2023-08-27T21:52:11.531739+0000","last_became_peered":"2023-08-27T21:52:11.531739+0000","last_unstale":"2023-08-27T21:53:18.989856+0000","last_undegraded":"2023-08-27T21:53:18.989856+0000","last_fullsized":"2023-08-27T21:53:18.989856+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:48:22.416466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:48:22.416466+0000","last_clean_scrub_stamp":"2023-08-27T21:48:22.416466+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-27T21:53:18.989404+0000","last_change":"2023-08-27T21:53:17.043700+0000","last_active":"2023-08-27T21:53:18.989404+0000","last_peered":"2023-08-27T21:53:18.989404+0000","last_clean":"2023-08-27T21:53:18.989404+0000","last_became_active":"2023-08-27T21:53:17.043332+0000","last_became_peered":"2023-08-27T21:53:17.043332+0000","last_unstale":"2023-08-27T21:53:18.989404+0000","last_undegraded":"2023-08-27T21:53:18.989404+0000","last_fullsized":"2023-08-27T21:53:18.989404+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-27T21:53:18.041502+0000","last_change":"2023-08-27T21:53:18.019051+0000","last_active":"2023-08-27T21:53:18.041502+0000","last_peered":"2023-08-27T21:53:18.041502+0000","last_clean":"2023-08-27T21:53:18.041502+0000","last_became_active":"2023-08-27T21:53:18.018616+0000","last_became_peered":"2023-08-27T21:53:18.018616+0000","last_unstale":"2023-08-27T21:53:18.041502+0000","last_undegraded":"2023-08-27T21:53:18.041502+0000","last_fullsized":"2023-08-27T21:53:18.041502+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-27T21:53:18.989409+0000","last_change":"2023-08-27T21:53:17.043427+0000","last_active":"2023-08-27T21:53:18.989409+0000","last_peered":"2023-08-27T21:53:18.989409+0000","last_clean":"2023-08-27T21:53:18.989409+0000","last_became_active":"2023-08-27T21:53:17.043025+0000","last_became_peered":"2023-08-27T21:53:17.043025+0000","last_unstale":"2023-08-27T21:53:18.989409+0000","last_undegraded":"2023-08-27T21:53:18.989409+0000","last_fullsized":"2023-08-27T21:53:18.989409+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-27T21:53:18.064019+0000","last_change":"2023-08-27T21:53:17.813356+0000","last_active":"2023-08-27T21:53:18.064019+0000","last_peered":"2023-08-27T21:53:18.064019+0000","last_clean":"2023-08-27T21:53:18.064019+0000","last_became_active":"2023-08-27T21:53:17.812986+0000","last_became_peered":"2023-08-27T21:53:17.812986+0000","last_unstale":"2023-08-27T21:53:18.064019+0000","last_undegraded":"2023-08-27T21:53:18.064019+0000","last_fullsized":"2023-08-27T21:53:18.064019+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430241,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:54:16 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":0.89200000000000002,"15min":0.88800000000000001},"min":{"1min":0.60199999999999998,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":1.774,"5min":1.774,"15min":1.774},"last":0.88},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.878,"15min":0.877},"min":{"1min":0.626,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":1.3600000000000001,"5min":1.3600000000000001,"15min":1.3600000000000001},"last":1.202}]},{"osd":1,"last update":"Sun Aug 27 21:54:16 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.89300000000000002,"15min":0.88800000000000001},"min":{"1min":0.57499999999999996,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.696,"5min":1.696,"15min":1.696},"last":1.113},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.92300000000000004,"15min":0.92700000000000005},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.4390000000000001,"5min":1.4390000000000001,"15min":1.4390000000000001},"last":1.147}]},{"osd":2,"last update":"Sun Aug 27 21:54:16 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.93200000000000005,"15min":0.93300000000000005},"min":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.8580000000000001,"5min":1.8580000000000001,"15min":1.8580000000000001},"last":1.0529999999999999},{"interface":"front","average":{"1min":0.97799999999999998,"5min":0.91200000000000003,"15min":0.90100000000000002},"min":{"1min":0.754,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.8009999999999999,"5min":1.8009999999999999,"15min":1.8009999999999999},"last":0.95499999999999996}]},{"osd":3,"last update":"Sun Aug 27 21:54:16 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.94499999999999995,"15min":0.94699999999999995},"min":{"1min":0.66100000000000003,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.931,"5min":1.931,"15min":1.931},"last":0.98799999999999999},{"interface":"front","average":{"1min":1.01,"5min":0.95499999999999996,"15min":0.94599999999999995},"min":{"1min":0.64300000000000002,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.913,"5min":1.913,"15min":1.913},"last":0.84499999999999997}]},{"osd":4,"last update":"Sun Aug 27 21:54:16 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":0.88200000000000001,"15min":0.86699999999999999},"min":{"1min":0.54700000000000004,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.889,"5min":1.889,"15min":1.889},"last":0.92900000000000005},{"interface":"front","average":{"1min":0.95599999999999996,"5min":0.90200000000000002,"15min":0.89300000000000002},"min":{"1min":0.60499999999999998,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":2.0049999999999999,"5min":2.0049999999999999,"15min":2.0049999999999999},"last":1.1299999999999999}]},{"osd":5,"last update":"Sun Aug 27 21:54:16 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.90100000000000002,"15min":0.89500000000000002},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":2.0259999999999998,"5min":2.0259999999999998,"15min":2.0259999999999998},"last":1.022},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.0369999999999999,"15min":1.032},"min":{"1min":0.69999999999999996,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":2.1059999999999999,"5min":2.1059999999999999,"15min":2.1059999999999999},"last":0.751}]},{"osd":6,"last update":"Sun Aug 27 21:54:16 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.0149999999999999,"15min":1.0049999999999999},"min":{"1min":0.68300000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.5389999999999999,"5min":1.5389999999999999,"15min":1.5389999999999999},"last":1.0780000000000001},{"interface":"front","average":{"1min":1.093,"5min":1.0529999999999999,"15min":1.046},"min":{"1min":0.72999999999999998,"5min":0.66300000000000003,"15min":0.66300000000000003},"max":{"1min":2.0870000000000002,"5min":2.0870000000000002,"15min":2.0870000000000002},"last":1.177}]}]},{"osd":6,"up_from":42,"seq":180388626475,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297616,"kb_used_data":520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447536,"statfs":{"total":95995035648,"available":95690276864,"internally_reserved":0,"allocated":532480,"data_stored":219885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":0.83699999999999997,"15min":0.85599999999999998},"min":{"1min":0.61099999999999999,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.1140000000000001,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":0.753},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.85999999999999999,"15min":0.83799999999999997},"min":{"1min":0.59499999999999997,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.2649999999999999,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":1.0649999999999999}]},{"osd":1,"last update":"Sun Aug 27 21:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":0.92500000000000004,"15min":0.94499999999999995},"min":{"1min":0.63400000000000001,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.1910000000000001,"5min":1.629,"15min":1.629},"last":0.88200000000000001},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.877,"15min":0.85499999999999998},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.226,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":1.1479999999999999}]},{"osd":2,"last update":"Sun Aug 27 21:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.91200000000000003,"15min":0.90800000000000003},"min":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.196,"5min":1.196,"15min":1.196},"last":0.93799999999999994},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.90600000000000003,"15min":0.92300000000000004},"min":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.2330000000000001,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":0.84999999999999998}]},{"osd":3,"last update":"Sun Aug 27 21:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.95499999999999996,"15min":0.98199999999999998},"min":{"1min":0.66200000000000003,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.6719999999999999,"5min":1.752,"15min":1.752},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.96199999999999997,"15min":0.97499999999999998},"min":{"1min":0.69699999999999995,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.179,"5min":1.546,"15min":1.546},"last":1.2150000000000001}]},{"osd":4,"last update":"Sun Aug 27 21:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.83599999999999997,"15min":0.81200000000000006},"min":{"1min":0.48899999999999999,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.3100000000000001,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":0.92200000000000004},{"interface":"front","average":{"1min":1.0389999999999999,"5min":0.89300000000000002,"15min":0.86199999999999999},"min":{"1min":0.70899999999999996,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.24,"5min":1.24,"15min":1.24},"last":1.0449999999999999}]},{"osd":5,"last update":"Sun Aug 27 21:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.88400000000000001,"15min":0.86499999999999999},"min":{"1min":0.64600000000000002,"5min":0.437,"15min":0.437},"max":{"1min":1.2050000000000001,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":0.97199999999999998},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.94099999999999995,"15min":0.93600000000000005},"min":{"1min":0.65600000000000003,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.236,"5min":1.845,"15min":1.845},"last":0.97999999999999998}]},{"osd":7,"last update":"Sun Aug 27 21:54:14 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":0.96999999999999997,"15min":0.96699999999999997},"min":{"1min":0.52700000000000002,"5min":0.5,"15min":0.5},"max":{"1min":1.2110000000000001,"5min":1.5049999999999999,"15min":1.5049999999999999},"last":1.0069999999999999},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.016,"15min":0.999},"min":{"1min":0.77100000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.381,"5min":1.46,"15min":1.46},"last":1.5529999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574936,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297616,"kb_used_data":520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447536,"statfs":{"total":95995035648,"available":95690276864,"internally_reserved":0,"allocated":532480,"data_stored":219885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:53:50 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":0.70599999999999996,"15min":0.56000000000000005},"min":{"1min":0.51200000000000001,"5min":0.38200000000000001,"15min":0.34799999999999998},"max":{"1min":1.214,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.82799999999999996,"5min":0.79900000000000004,"15min":0.61699999999999999},"min":{"1min":0.56899999999999995,"5min":0.40400000000000003,"15min":0.39500000000000002},"max":{"1min":1.115,"5min":1.2370000000000001,"15min":1.2370000000000001},"last":0.83999999999999997}]},{"osd":2,"last update":"Sun Aug 27 21:54:32 2023","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":0.77300000000000002,"15min":0.69199999999999995},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.3600000000000001,"5min":1.3600000000000001,"15min":1.3600000000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.80300000000000005,"15min":0.69499999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.1060000000000001,"5min":1.395,"15min":1.395},"last":0.79600000000000004}]},{"osd":3,"last update":"Sun Aug 27 21:54:19 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.81100000000000005,"15min":0.79500000000000004},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.089,"5min":1.2410000000000001,"15min":1.2410000000000001},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.79800000000000004,"5min":0.88200000000000001,"15min":0.84999999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.2749999999999999,"5min":1.351,"15min":1.351},"last":0.53300000000000003}]},{"osd":4,"last update":"Sun Aug 27 21:53:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.0309999999999999,"15min":1.0169999999999999},"min":{"1min":0.73399999999999999,"5min":0.69399999999999995,"15min":0.69399999999999995},"max":{"1min":1.3220000000000001,"5min":1.673,"15min":1.673},"last":0.77300000000000002},{"interface":"front","average":{"1min":1.032,"5min":1.05,"15min":1.0069999999999999},"min":{"1min":0.83099999999999996,"5min":0.75900000000000001,"15min":0.75900000000000001},"max":{"1min":1.21,"5min":1.673,"15min":1.673},"last":0.88900000000000001}]},{"osd":5,"last update":"Sun Aug 27 21:54:40 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.075,"15min":1.0669999999999999},"min":{"1min":0.76600000000000001,"5min":0.69299999999999995,"15min":0.69299999999999995},"max":{"1min":1.663,"5min":1.663,"15min":1.663},"last":0.65700000000000003},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.095,"15min":1.121},"min":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"max":{"1min":1.861,"5min":1.861,"15min":1.861},"last":0.93300000000000005}]},{"osd":6,"last update":"Sun Aug 27 21:54:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":1.1379999999999999,"15min":1.151},"min":{"1min":0.75,"5min":0.75,"15min":0.75},"max":{"1min":1.821,"5min":1.821,"15min":1.821},"last":0.96499999999999997},{"interface":"front","average":{"1min":1.097,"5min":1.1890000000000001,"15min":1.2230000000000001},"min":{"1min":0.77400000000000002,"5min":0.746,"15min":0.746},"max":{"1min":1.7,"5min":1.7110000000000001,"15min":1.7110000000000001},"last":0.82999999999999996}]},{"osd":7,"last update":"Sun Aug 27 21:54:10 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":1.21,"15min":1.22},"min":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.4470000000000001,"5min":1.597,"15min":1.597},"last":0.92600000000000005},{"interface":"front","average":{"1min":1.137,"5min":1.173,"15min":1.179},"min":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"max":{"1min":1.4259999999999999,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":0.86699999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738464,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Aug 27 21:53:43 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.69099999999999995,"15min":0.57199999999999995},"min":{"1min":0.44400000000000001,"5min":0.39000000000000001,"15min":0.31},"max":{"1min":1.115,"5min":1.177,"15min":1.177},"last":0.81699999999999995},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.75900000000000001,"15min":0.621},"min":{"1min":0.45300000000000001,"5min":0.40999999999999998,"15min":0.34499999999999997},"max":{"1min":1.091,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.77100000000000002}]},{"osd":2,"last update":"Sun Aug 27 21:54:27 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":0.76700000000000002,"15min":0.67500000000000004},"min":{"1min":0.42999999999999999,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.1619999999999999,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":1.048},{"interface":"front","average":{"1min":0.78700000000000003,"5min":0.82999999999999996,"15min":0.71899999999999997},"min":{"1min":0.53900000000000003,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.999,"5min":1.25,"15min":1.25},"last":0.86399999999999999}]},{"osd":3,"last update":"Sun Aug 27 21:54:15 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":0.86599999999999999,"15min":0.83799999999999997},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.024,"5min":1.359,"15min":1.359},"last":0.92300000000000004},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.78400000000000003,"15min":0.78100000000000003},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.0309999999999999,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":0.79600000000000004}]},{"osd":4,"last update":"Sun Aug 27 21:53:58 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.96799999999999997,"15min":0.94799999999999995},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.2090000000000001,"5min":1.5860000000000001,"15min":1.5860000000000001},"last":1.1579999999999999},{"interface":"front","average":{"1min":1.012,"5min":1.0309999999999999,"15min":1.012},"min":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"max":{"1min":1.26,"5min":1.6299999999999999,"15min":1.6299999999999999},"last":1.123}]},{"osd":5,"last update":"Sun Aug 27 21:53:43 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.96099999999999997,"15min":0.96799999999999997},"min":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.1759999999999999,"5min":1.4950000000000001,"15min":1.4950000000000001},"last":1.0900000000000001},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.0589999999999999,"15min":1.05},"min":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"max":{"1min":1.325,"5min":1.448,"15min":1.448},"last":1.3819999999999999}]},{"osd":6,"last update":"Sun Aug 27 21:54:27 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":1.1020000000000001,"15min":1.095},"min":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"max":{"1min":1.3999999999999999,"5min":1.651,"15min":1.651},"last":1.236},{"interface":"front","average":{"1min":1.119,"5min":1.1459999999999999,"15min":1.155},"min":{"1min":0.80000000000000004,"5min":0.628,"15min":0.628},"max":{"1min":1.355,"5min":1.5640000000000001,"15min":1.5640000000000001},"last":1.367}]},{"osd":7,"last update":"Sun Aug 27 21:54:15 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":1.097,"15min":1.1080000000000001},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.286,"5min":1.5780000000000001,"15min":1.5780000000000001},"last":1.4139999999999999},{"interface":"front","average":{"1min":1.083,"5min":1.2430000000000001,"15min":1.2689999999999999},"min":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"max":{"1min":1.397,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":1.482}]}]},{"osd":2,"up_from":18,"seq":77309411406,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:54:33 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":0.69599999999999995,"15min":0.628},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.372},"max":{"1min":1.885,"5min":1.885,"15min":1.885},"last":0.503},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.77100000000000002,"15min":0.64500000000000002},"min":{"1min":0.40899999999999997,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.718,"5min":1.718,"15min":1.718},"last":0.45000000000000001}]},{"osd":1,"last update":"Sun Aug 27 21:54:33 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.79000000000000004,"15min":0.68600000000000005},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.492,"5min":2.4769999999999999,"15min":2.4769999999999999},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.79300000000000004,"15min":0.67100000000000004},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.439},"max":{"1min":1.6080000000000001,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":0.61799999999999999}]},{"osd":3,"last update":"Sun Aug 27 21:54:15 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.82199999999999995,"15min":0.77600000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.5509999999999999,"5min":1.5509999999999999,"15min":1.5509999999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.82799999999999996,"5min":0.871,"15min":0.83299999999999996},"min":{"1min":0.53900000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.4359999999999999,"5min":1.55,"15min":1.55},"last":0.91200000000000003}]},{"osd":4,"last update":"Sun Aug 27 21:54:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":0.99299999999999999,"15min":0.97499999999999998},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.8580000000000001,"5min":1.8580000000000001,"15min":1.8580000000000001},"last":0.88800000000000001},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.02,"15min":0.96799999999999997},"min":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.921,"5min":1.921,"15min":1.921},"last":0.76300000000000001}]},{"osd":5,"last update":"Sun Aug 27 21:54:43 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.0589999999999999,"15min":1.0309999999999999},"min":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"max":{"1min":1.845,"5min":1.845,"15min":1.845},"last":0.73699999999999999},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.0940000000000001,"15min":1.069},"min":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.909,"5min":2.0089999999999999,"15min":2.0089999999999999},"last":1.2110000000000001}]},{"osd":6,"last update":"Sun Aug 27 21:54:25 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":1.196,"15min":1.2},"min":{"1min":0.84199999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.8879999999999999,"5min":2.0960000000000001,"15min":2.0960000000000001},"last":1.1180000000000001},{"interface":"front","average":{"1min":1.1679999999999999,"5min":1.1639999999999999,"15min":1.1779999999999999},"min":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.78,"5min":1.78,"15min":1.78},"last":0.997}]},{"osd":7,"last update":"Sun Aug 27 21:54:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":1.218,"15min":1.24},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.804,"5min":1.804,"15min":1.804},"last":0.94399999999999995},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.1850000000000001,"15min":1.1850000000000001},"min":{"1min":0.71199999999999997,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.758,"5min":1.758,"15min":1.758},"last":0.78600000000000003}]}]},{"osd":3,"up_from":25,"seq":107374182469,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:54:20 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.69899999999999995,"15min":0.67900000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.0069999999999999,"5min":1.1439999999999999,"15min":1.1439999999999999},"last":0.84999999999999998},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.74199999999999999,"15min":0.73999999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.113,"5min":1.224,"15min":1.224},"last":0.73899999999999999}]},{"osd":1,"last update":"Sun Aug 27 21:54:20 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.77000000000000002,"15min":0.76300000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.0660000000000001,"5min":1.111,"15min":1.111},"last":1.071},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.80900000000000005,"15min":0.73799999999999999},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":2.883,"5min":2.883,"15min":2.883},"last":1.0249999999999999}]},{"osd":2,"last update":"Sun Aug 27 21:54:20 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.88500000000000001,"15min":0.86899999999999999},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.1439999999999999,"5min":1.151,"15min":1.151},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.871,"5min":0.89500000000000002,"15min":0.88900000000000001},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":3.319,"5min":3.319,"15min":3.319},"last":0.52400000000000002}]},{"osd":4,"last update":"Sun Aug 27 21:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.012,"15min":0.99299999999999999},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":3.5939999999999999,"5min":3.5939999999999999,"15min":3.5939999999999999},"last":1},{"interface":"front","average":{"1min":1.0740000000000001,"5min":1.026,"15min":1.0109999999999999},"min":{"1min":0.751,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.3899999999999999,"5min":1.4410000000000001,"15min":1.4410000000000001},"last":1.1000000000000001}]},{"osd":5,"last update":"Sun Aug 27 21:54:44 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":1.042,"15min":1.044},"min":{"1min":0.622,"5min":0.622,"15min":0.622},"max":{"1min":1.4039999999999999,"5min":3.5670000000000002,"15min":3.5670000000000002},"last":0.92800000000000005},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.0469999999999999,"15min":1.038},"min":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.4890000000000001,"5min":1.5369999999999999,"15min":1.5369999999999999},"last":0.94799999999999995}]},{"osd":6,"last update":"Sun Aug 27 21:54:24 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.097,"15min":1.0960000000000001},"min":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"max":{"1min":1.4079999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":1.147},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.113,"15min":1.117},"min":{"1min":0.751,"5min":0.751,"15min":0.751},"max":{"1min":1.359,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":1.1699999999999999}]},{"osd":7,"last update":"Sun Aug 27 21:54:11 2023","interfaces":[{"interface":"back","average":{"1min":1.2370000000000001,"5min":1.1479999999999999,"15min":1.133},"min":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"max":{"1min":3.5270000000000001,"5min":3.5270000000000001,"15min":3.5270000000000001},"last":1.054},{"interface":"front","average":{"1min":1.171,"5min":1.23,"15min":1.24},"min":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.4850000000000001,"5min":1.5900000000000001,"15min":1.5900000000000001},"last":1.1950000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018941,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:54:00 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.879,"15min":0.82499999999999996},"min":{"1min":0.61499999999999999,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.302,"5min":3.383,"15min":3.383},"last":0.91000000000000003},{"interface":"front","average":{"1min":0.82599999999999996,"5min":0.82699999999999996,"15min":0.82999999999999996},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.1499999999999999,"5min":1.2010000000000001,"15min":1.2010000000000001},"last":1.03}]},{"osd":1,"last update":"Sun Aug 27 21:54:00 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.82499999999999996,"15min":0.79700000000000004},"min":{"1min":0.56899999999999995,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.1619999999999999,"5min":2.1349999999999998,"15min":2.1349999999999998},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.93400000000000005,"15min":0.88600000000000001},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.345,"5min":3.3340000000000001,"15min":3.3340000000000001},"last":1.1919999999999999}]},{"osd":2,"last update":"Sun Aug 27 21:54:00 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.89400000000000002,"15min":0.85199999999999998},"min":{"1min":0.64600000000000002,"5min":0.62,"15min":0.62},"max":{"1min":1.7430000000000001,"5min":3.4260000000000002,"15min":3.4260000000000002},"last":0.998},{"interface":"front","average":{"1min":0.83299999999999996,"5min":0.85299999999999998,"15min":0.86299999999999999},"min":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.1060000000000001,"5min":1.149,"15min":1.149},"last":0.73199999999999998}]},{"osd":3,"last update":"Sun Aug 27 21:54:00 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.86299999999999999,"15min":0.873},"min":{"1min":0.67900000000000005,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.1759999999999999,"5min":1.1759999999999999,"15min":1.1759999999999999},"last":0.94699999999999995},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.90600000000000003,"15min":0.91400000000000003},"min":{"1min":0.69499999999999995,"5min":0.629,"15min":0.629},"max":{"1min":1.236,"5min":1.236,"15min":1.236},"last":0.93600000000000005}]},{"osd":5,"last update":"Sun Aug 27 21:53:42 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":0.89400000000000002,"15min":0.91800000000000004},"min":{"1min":0.42099999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.6679999999999999,"5min":3.2200000000000002,"15min":3.2200000000000002},"last":1.0600000000000001},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.80500000000000005,"15min":0.81699999999999995},"min":{"1min":0.41699999999999998,"5min":0.378,"15min":0.378},"max":{"1min":1.23,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":0.82499999999999996}]},{"osd":6,"last update":"Sun Aug 27 21:54:28 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":0.82899999999999996,"15min":0.81000000000000005},"min":{"1min":0.55800000000000005,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.3280000000000001,"5min":1.5109999999999999,"15min":1.5109999999999999},"last":0.84199999999999997},{"interface":"front","average":{"1min":1.006,"5min":0.94099999999999995,"15min":0.92300000000000004},"min":{"1min":0.625,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.583,"5min":3.278,"15min":3.278},"last":1.1080000000000001}]},{"osd":7,"last update":"Sun Aug 27 21:54:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":0.94199999999999995,"15min":0.92400000000000004},"min":{"1min":0.68200000000000005,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.637,"5min":1.637,"15min":1.637},"last":1.1499999999999999},{"interface":"front","average":{"1min":1.0089999999999999,"5min":0.90700000000000003,"15min":0.89000000000000001},"min":{"1min":0.58899999999999997,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.5449999999999999,"5min":1.5449999999999999,"15min":1.5449999999999999},"last":0.89600000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822708,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297616,"kb_used_data":520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447536,"statfs":{"total":95995035648,"available":95690276864,"internally_reserved":0,"allocated":532480,"data_stored":219885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:54:45 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":0.84299999999999997,"15min":0.877},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.3320000000000001,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.82799999999999996,"5min":0.85199999999999998,"15min":0.86399999999999999},"min":{"1min":0.58699999999999997,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.2050000000000001,"5min":1.2050000000000001,"15min":1.2050000000000001},"last":0.81000000000000005}]},{"osd":1,"last update":"Sun Aug 27 21:54:45 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":0.89200000000000002,"15min":0.92000000000000004},"min":{"1min":0.53000000000000003,"5min":0.504,"15min":0.504},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.89100000000000001,"5min":0.88400000000000001,"15min":0.88600000000000001},"min":{"1min":0.61599999999999999,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.605,"5min":1.605,"15min":1.605},"last":0.94199999999999995}]},{"osd":2,"last update":"Sun Aug 27 21:54:45 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.91300000000000003,"15min":0.90500000000000003},"min":{"1min":0.67900000000000005,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.2350000000000001,"5min":1.7609999999999999,"15min":1.7609999999999999},"last":1.1659999999999999},{"interface":"front","average":{"1min":0.84499999999999997,"5min":0.89000000000000001,"15min":0.90700000000000003},"min":{"1min":0.65200000000000002,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.0840000000000001,"5min":1.264,"15min":1.264},"last":0.88}]},{"osd":3,"last update":"Sun Aug 27 21:54:45 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.93999999999999995,"15min":0.93300000000000005},"min":{"1min":0.621,"5min":0.503,"15min":0.503},"max":{"1min":1.1859999999999999,"5min":1.587,"15min":1.587},"last":0.92100000000000004},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.94999999999999996,"15min":0.94799999999999995},"min":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.2450000000000001,"5min":1.506,"15min":1.506},"last":0.98899999999999999}]},{"osd":4,"last update":"Sun Aug 27 21:54:45 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.83099999999999996,"15min":0.78000000000000003},"min":{"1min":0.61299999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.6919999999999999,"5min":1.6919999999999999,"15min":1.6919999999999999},"last":1.0609999999999999},{"interface":"front","average":{"1min":1.0489999999999999,"5min":0.88,"15min":0.79100000000000004},"min":{"1min":0.77900000000000003,"5min":0.442,"15min":0.442},"max":{"1min":1.3180000000000001,"5min":1.538,"15min":1.538},"last":1.1040000000000001}]},{"osd":6,"last update":"Sun Aug 27 21:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.94599999999999995,"15min":0.94699999999999995},"min":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.4119999999999999,"5min":1.7050000000000001,"15min":1.7050000000000001},"last":1.026},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.95999999999999996,"15min":0.96999999999999997},"min":{"1min":0.55600000000000005,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.2529999999999999,"5min":1.4370000000000001,"15min":1.4370000000000001},"last":1.0820000000000001}]},{"osd":7,"last update":"Sun Aug 27 21:54:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.038,"15min":1.036},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.498,"5min":1.498,"15min":1.498},"last":1.1910000000000001},{"interface":"front","average":{"1min":1.04,"5min":1.0009999999999999,"15min":0.995},"min":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.4990000000000001,"5min":1.611,"15min":1.611},"last":1.133}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-27T21:54:47.384 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-27T21:54:47.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:47 smithi006 bash[27292]: cluster 2023-08-27T21:54:46.620642+0000 mgr.y (mgr.14430) 88 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:47.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:47 smithi006 bash[27292]: audit 2023-08-27T21:54:46.685291+0000 mgr.y (mgr.14430) 89 : audit [DBG] from='client.14610 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T21:54:47.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:47 smithi006 bash[33918]: cluster 2023-08-27T21:54:46.620642+0000 mgr.y (mgr.14430) 88 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:47.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:47 smithi006 bash[33918]: audit 2023-08-27T21:54:46.685291+0000 mgr.y (mgr.14430) 89 : audit [DBG] from='client.14610 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T21:54:48.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:47 smithi130 bash[27579]: cluster 2023-08-27T21:54:46.620642+0000 mgr.y (mgr.14430) 88 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:48.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:47 smithi130 bash[27579]: audit 2023-08-27T21:54:46.685291+0000 mgr.y (mgr.14430) 89 : audit [DBG] from='client.14610 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T21:54:49.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:49 smithi006 bash[27292]: cluster 2023-08-27T21:54:48.621155+0000 mgr.y (mgr.14430) 90 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:49.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:49 smithi006 bash[33918]: cluster 2023-08-27T21:54:48.621155+0000 mgr.y (mgr.14430) 90 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:50.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:49 smithi130 bash[27579]: cluster 2023-08-27T21:54:48.621155+0000 mgr.y (mgr.14430) 90 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:51.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:54:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:54:50] "GET /metrics HTTP/1.1" 200 155658 "" "Prometheus/2.33.4" 2023-08-27T21:54:51.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:51 smithi006 bash[27292]: cluster 2023-08-27T21:54:50.621773+0000 mgr.y (mgr.14430) 91 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:51.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:51 smithi006 bash[33918]: cluster 2023-08-27T21:54:50.621773+0000 mgr.y (mgr.14430) 91 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:52.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:51 smithi130 bash[27579]: cluster 2023-08-27T21:54:50.621773+0000 mgr.y (mgr.14430) 91 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:52.201 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:54:53.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:53 smithi130 bash[27579]: cluster 2023-08-27T21:54:52.622455+0000 mgr.y (mgr.14430) 92 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:53.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:53 smithi006 bash[27292]: cluster 2023-08-27T21:54:52.622455+0000 mgr.y (mgr.14430) 92 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:53.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:53 smithi006 bash[33918]: cluster 2023-08-27T21:54:52.622455+0000 mgr.y (mgr.14430) 92 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:55.337 INFO:teuthology.orchestra.run.smithi006.stdout: 2023-08-27T21:54:55.339 INFO:teuthology.orchestra.run.smithi006.stderr:dumped all 2023-08-27T21:54:55.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:55 smithi006 bash[33918]: cluster 2023-08-27T21:54:54.623083+0000 mgr.y (mgr.14430) 93 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:55.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:55 smithi006 bash[27292]: cluster 2023-08-27T21:54:54.623083+0000 mgr.y (mgr.14430) 93 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:56.082 INFO:teuthology.orchestra.run.smithi006.stdout:{"pg_ready":true,"pg_map":{"version":72,"stamp":"2023-08-27T21:54:54.622816+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380968,"kb_used_data":4200,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580248,"statfs":{"total":767960285184,"available":765522173952,"internally_reserved":0,"allocated":4300800,"data_stored":1762230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-27T21:53:18.989798+0000","last_change":"2023-08-27T21:53:17.043470+0000","last_active":"2023-08-27T21:53:18.989798+0000","last_peered":"2023-08-27T21:53:18.989798+0000","last_clean":"2023-08-27T21:53:18.989798+0000","last_became_active":"2023-08-27T21:53:17.042696+0000","last_became_peered":"2023-08-27T21:53:17.042696+0000","last_unstale":"2023-08-27T21:53:18.989798+0000","last_undegraded":"2023-08-27T21:53:18.989798+0000","last_fullsized":"2023-08-27T21:53:18.989798+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-27T21:53:18.063987+0000","last_change":"2023-08-27T21:53:17.187485+0000","last_active":"2023-08-27T21:53:18.063987+0000","last_peered":"2023-08-27T21:53:18.063987+0000","last_clean":"2023-08-27T21:53:18.063987+0000","last_became_active":"2023-08-27T21:53:17.187094+0000","last_became_peered":"2023-08-27T21:53:17.187094+0000","last_unstale":"2023-08-27T21:53:18.063987+0000","last_undegraded":"2023-08-27T21:53:18.063987+0000","last_fullsized":"2023-08-27T21:53:18.063987+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-27T21:53:18.989950+0000","last_change":"2023-08-27T21:53:17.816851+0000","last_active":"2023-08-27T21:53:18.989950+0000","last_peered":"2023-08-27T21:53:18.989950+0000","last_clean":"2023-08-27T21:53:18.989950+0000","last_became_active":"2023-08-27T21:53:17.815511+0000","last_became_peered":"2023-08-27T21:53:17.815511+0000","last_unstale":"2023-08-27T21:53:18.989950+0000","last_undegraded":"2023-08-27T21:53:18.989950+0000","last_fullsized":"2023-08-27T21:53:18.989950+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-27T21:53:18.989527+0000","last_change":"2023-08-27T21:53:17.815961+0000","last_active":"2023-08-27T21:53:18.989527+0000","last_peered":"2023-08-27T21:53:18.989527+0000","last_clean":"2023-08-27T21:53:18.989527+0000","last_became_active":"2023-08-27T21:53:17.815034+0000","last_became_peered":"2023-08-27T21:53:17.815034+0000","last_unstale":"2023-08-27T21:53:18.989527+0000","last_undegraded":"2023-08-27T21:53:18.989527+0000","last_fullsized":"2023-08-27T21:53:18.989527+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-27T21:53:18.989856+0000","last_change":"2023-08-27T21:52:11.532578+0000","last_active":"2023-08-27T21:53:18.989856+0000","last_peered":"2023-08-27T21:53:18.989856+0000","last_clean":"2023-08-27T21:53:18.989856+0000","last_became_active":"2023-08-27T21:52:11.531739+0000","last_became_peered":"2023-08-27T21:52:11.531739+0000","last_unstale":"2023-08-27T21:53:18.989856+0000","last_undegraded":"2023-08-27T21:53:18.989856+0000","last_fullsized":"2023-08-27T21:53:18.989856+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:48:22.416466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:48:22.416466+0000","last_clean_scrub_stamp":"2023-08-27T21:48:22.416466+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-27T21:53:18.989404+0000","last_change":"2023-08-27T21:53:17.043700+0000","last_active":"2023-08-27T21:53:18.989404+0000","last_peered":"2023-08-27T21:53:18.989404+0000","last_clean":"2023-08-27T21:53:18.989404+0000","last_became_active":"2023-08-27T21:53:17.043332+0000","last_became_peered":"2023-08-27T21:53:17.043332+0000","last_unstale":"2023-08-27T21:53:18.989404+0000","last_undegraded":"2023-08-27T21:53:18.989404+0000","last_fullsized":"2023-08-27T21:53:18.989404+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-27T21:53:18.041502+0000","last_change":"2023-08-27T21:53:18.019051+0000","last_active":"2023-08-27T21:53:18.041502+0000","last_peered":"2023-08-27T21:53:18.041502+0000","last_clean":"2023-08-27T21:53:18.041502+0000","last_became_active":"2023-08-27T21:53:18.018616+0000","last_became_peered":"2023-08-27T21:53:18.018616+0000","last_unstale":"2023-08-27T21:53:18.041502+0000","last_undegraded":"2023-08-27T21:53:18.041502+0000","last_fullsized":"2023-08-27T21:53:18.041502+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-27T21:53:18.989409+0000","last_change":"2023-08-27T21:53:17.043427+0000","last_active":"2023-08-27T21:53:18.989409+0000","last_peered":"2023-08-27T21:53:18.989409+0000","last_clean":"2023-08-27T21:53:18.989409+0000","last_became_active":"2023-08-27T21:53:17.043025+0000","last_became_peered":"2023-08-27T21:53:17.043025+0000","last_unstale":"2023-08-27T21:53:18.989409+0000","last_undegraded":"2023-08-27T21:53:18.989409+0000","last_fullsized":"2023-08-27T21:53:18.989409+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-27T21:53:18.064019+0000","last_change":"2023-08-27T21:53:17.813356+0000","last_active":"2023-08-27T21:53:18.064019+0000","last_peered":"2023-08-27T21:53:18.064019+0000","last_clean":"2023-08-27T21:53:18.064019+0000","last_became_active":"2023-08-27T21:53:17.812986+0000","last_became_peered":"2023-08-27T21:53:17.812986+0000","last_unstale":"2023-08-27T21:53:18.064019+0000","last_undegraded":"2023-08-27T21:53:18.064019+0000","last_fullsized":"2023-08-27T21:53:18.064019+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430242,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:54:16 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":0.89200000000000002,"15min":0.88800000000000001},"min":{"1min":0.60199999999999998,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":1.774,"5min":1.774,"15min":1.774},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.878,"15min":0.877},"min":{"1min":0.626,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":1.3600000000000001,"5min":1.3600000000000001,"15min":1.3600000000000001},"last":1.0800000000000001}]},{"osd":1,"last update":"Sun Aug 27 21:54:16 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.89300000000000002,"15min":0.88800000000000001},"min":{"1min":0.57499999999999996,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.696,"5min":1.696,"15min":1.696},"last":0.96099999999999997},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.92300000000000004,"15min":0.92700000000000005},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.4390000000000001,"5min":1.4390000000000001,"15min":1.4390000000000001},"last":0.91400000000000003}]},{"osd":2,"last update":"Sun Aug 27 21:54:16 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.93200000000000005,"15min":0.93300000000000005},"min":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.8580000000000001,"5min":1.8580000000000001,"15min":1.8580000000000001},"last":0.99399999999999999},{"interface":"front","average":{"1min":0.97799999999999998,"5min":0.91200000000000003,"15min":0.90100000000000002},"min":{"1min":0.754,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.8009999999999999,"5min":1.8009999999999999,"15min":1.8009999999999999},"last":0.84499999999999997}]},{"osd":3,"last update":"Sun Aug 27 21:54:16 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.94499999999999995,"15min":0.94699999999999995},"min":{"1min":0.66100000000000003,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.931,"5min":1.931,"15min":1.931},"last":0.70099999999999996},{"interface":"front","average":{"1min":1.01,"5min":0.95499999999999996,"15min":0.94599999999999995},"min":{"1min":0.64300000000000002,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.913,"5min":1.913,"15min":1.913},"last":1.0429999999999999}]},{"osd":4,"last update":"Sun Aug 27 21:54:16 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":0.88200000000000001,"15min":0.86699999999999999},"min":{"1min":0.54700000000000004,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.889,"5min":1.889,"15min":1.889},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.95599999999999996,"5min":0.90200000000000002,"15min":0.89300000000000002},"min":{"1min":0.60499999999999998,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":2.0049999999999999,"5min":2.0049999999999999,"15min":2.0049999999999999},"last":0.93400000000000005}]},{"osd":5,"last update":"Sun Aug 27 21:54:16 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.90100000000000002,"15min":0.89500000000000002},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":2.0259999999999998,"5min":2.0259999999999998,"15min":2.0259999999999998},"last":0.73799999999999999},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.0369999999999999,"15min":1.032},"min":{"1min":0.69999999999999996,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":2.1059999999999999,"5min":2.1059999999999999,"15min":2.1059999999999999},"last":1.125}]},{"osd":6,"last update":"Sun Aug 27 21:54:16 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.0149999999999999,"15min":1.0049999999999999},"min":{"1min":0.68300000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.5389999999999999,"5min":1.5389999999999999,"15min":1.5389999999999999},"last":1.0129999999999999},{"interface":"front","average":{"1min":1.093,"5min":1.0529999999999999,"15min":1.046},"min":{"1min":0.72999999999999998,"5min":0.66300000000000003,"15min":0.66300000000000003},"max":{"1min":2.0870000000000002,"5min":2.0870000000000002,"15min":2.0870000000000002},"last":0.89200000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626476,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297616,"kb_used_data":520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447536,"statfs":{"total":95995035648,"available":95690276864,"internally_reserved":0,"allocated":532480,"data_stored":219885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":0.83699999999999997,"15min":0.85599999999999998},"min":{"1min":0.61099999999999999,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.1140000000000001,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":0.96799999999999997},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.85999999999999999,"15min":0.83799999999999997},"min":{"1min":0.59499999999999997,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.2649999999999999,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":1.141}]},{"osd":1,"last update":"Sun Aug 27 21:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":0.92500000000000004,"15min":0.94499999999999995},"min":{"1min":0.63400000000000001,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.1910000000000001,"5min":1.629,"15min":1.629},"last":0.91600000000000004},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.877,"15min":0.85499999999999998},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.226,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":1.0329999999999999}]},{"osd":2,"last update":"Sun Aug 27 21:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.91200000000000003,"15min":0.90800000000000003},"min":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.196,"5min":1.196,"15min":1.196},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.90600000000000003,"15min":0.92300000000000004},"min":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.2330000000000001,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":1.27}]},{"osd":3,"last update":"Sun Aug 27 21:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.95499999999999996,"15min":0.98199999999999998},"min":{"1min":0.66200000000000003,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.6719999999999999,"5min":1.752,"15min":1.752},"last":1.0609999999999999},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.96199999999999997,"15min":0.97499999999999998},"min":{"1min":0.69699999999999995,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.179,"5min":1.546,"15min":1.546},"last":0.94499999999999995}]},{"osd":4,"last update":"Sun Aug 27 21:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.83599999999999997,"15min":0.81200000000000006},"min":{"1min":0.48899999999999999,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.3100000000000001,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":1.101},{"interface":"front","average":{"1min":1.0389999999999999,"5min":0.89300000000000002,"15min":0.86199999999999999},"min":{"1min":0.70899999999999996,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.24,"5min":1.24,"15min":1.24},"last":1.0129999999999999}]},{"osd":5,"last update":"Sun Aug 27 21:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.88400000000000001,"15min":0.86499999999999999},"min":{"1min":0.64600000000000002,"5min":0.437,"15min":0.437},"max":{"1min":1.2050000000000001,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":1.194},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.94099999999999995,"15min":0.93600000000000005},"min":{"1min":0.65600000000000003,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.236,"5min":1.845,"15min":1.845},"last":1.1659999999999999}]},{"osd":7,"last update":"Sun Aug 27 21:54:14 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":0.96999999999999997,"15min":0.96699999999999997},"min":{"1min":0.52700000000000002,"5min":0.5,"15min":0.5},"max":{"1min":1.2110000000000001,"5min":1.5049999999999999,"15min":1.5049999999999999},"last":1.341},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.016,"15min":0.999},"min":{"1min":0.77100000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.381,"5min":1.46,"15min":1.46},"last":1.222}]}]},{"osd":1,"up_from":13,"seq":55834574937,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297616,"kb_used_data":520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447536,"statfs":{"total":95995035648,"available":95690276864,"internally_reserved":0,"allocated":532480,"data_stored":219885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:53:50 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":0.70599999999999996,"15min":0.56000000000000005},"min":{"1min":0.51200000000000001,"5min":0.38200000000000001,"15min":0.34799999999999998},"max":{"1min":1.214,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.82799999999999996,"5min":0.79900000000000004,"15min":0.61699999999999999},"min":{"1min":0.56899999999999995,"5min":0.40400000000000003,"15min":0.39500000000000002},"max":{"1min":1.115,"5min":1.2370000000000001,"15min":1.2370000000000001},"last":0.69999999999999996}]},{"osd":2,"last update":"Sun Aug 27 21:54:32 2023","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":0.77300000000000002,"15min":0.69199999999999995},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.3600000000000001,"5min":1.3600000000000001,"15min":1.3600000000000001},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.80300000000000005,"15min":0.69499999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.1060000000000001,"5min":1.395,"15min":1.395},"last":0.98399999999999999}]},{"osd":3,"last update":"Sun Aug 27 21:54:19 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.81100000000000005,"15min":0.79500000000000004},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.089,"5min":1.2410000000000001,"15min":1.2410000000000001},"last":0.81200000000000006},{"interface":"front","average":{"1min":0.79800000000000004,"5min":0.88200000000000001,"15min":0.84999999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.2749999999999999,"5min":1.351,"15min":1.351},"last":1.1240000000000001}]},{"osd":4,"last update":"Sun Aug 27 21:53:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.0309999999999999,"15min":1.0169999999999999},"min":{"1min":0.73399999999999999,"5min":0.69399999999999995,"15min":0.69399999999999995},"max":{"1min":1.3220000000000001,"5min":1.673,"15min":1.673},"last":1.323},{"interface":"front","average":{"1min":1.032,"5min":1.05,"15min":1.0069999999999999},"min":{"1min":0.83099999999999996,"5min":0.75900000000000001,"15min":0.75900000000000001},"max":{"1min":1.21,"5min":1.673,"15min":1.673},"last":1.0820000000000001}]},{"osd":5,"last update":"Sun Aug 27 21:54:40 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.075,"15min":1.0669999999999999},"min":{"1min":0.76600000000000001,"5min":0.69299999999999995,"15min":0.69299999999999995},"max":{"1min":1.663,"5min":1.663,"15min":1.663},"last":1.157},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.095,"15min":1.121},"min":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"max":{"1min":1.861,"5min":1.861,"15min":1.861},"last":1.387}]},{"osd":6,"last update":"Sun Aug 27 21:54:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":1.1379999999999999,"15min":1.151},"min":{"1min":0.75,"5min":0.75,"15min":0.75},"max":{"1min":1.821,"5min":1.821,"15min":1.821},"last":1.4299999999999999},{"interface":"front","average":{"1min":1.097,"5min":1.1890000000000001,"15min":1.2230000000000001},"min":{"1min":0.77400000000000002,"5min":0.746,"15min":0.746},"max":{"1min":1.7,"5min":1.7110000000000001,"15min":1.7110000000000001},"last":1.2270000000000001}]},{"osd":7,"last update":"Sun Aug 27 21:54:10 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":1.21,"15min":1.22},"min":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.4470000000000001,"5min":1.597,"15min":1.597},"last":1.282},{"interface":"front","average":{"1min":1.137,"5min":1.173,"15min":1.179},"min":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"max":{"1min":1.4259999999999999,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":1.486}]}]},{"osd":0,"up_from":8,"seq":34359738465,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Aug 27 21:54:46 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.70099999999999996,"15min":0.58299999999999996},"min":{"1min":0.317,"5min":0.317,"15min":0.31},"max":{"1min":0.97399999999999998,"5min":1.177,"15min":1.177},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.76900000000000002,"15min":0.63100000000000001},"min":{"1min":0.48299999999999998,"5min":0.45300000000000001,"15min":0.34499999999999997},"max":{"1min":0.92600000000000005,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.73799999999999999}]},{"osd":2,"last update":"Sun Aug 27 21:54:27 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":0.76700000000000002,"15min":0.67500000000000004},"min":{"1min":0.42999999999999999,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.1619999999999999,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.84399999999999997},{"interface":"front","average":{"1min":0.78700000000000003,"5min":0.82999999999999996,"15min":0.71899999999999997},"min":{"1min":0.53900000000000003,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.999,"5min":1.25,"15min":1.25},"last":0.53300000000000003}]},{"osd":3,"last update":"Sun Aug 27 21:54:15 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":0.86599999999999999,"15min":0.83799999999999997},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.024,"5min":1.359,"15min":1.359},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.78400000000000003,"15min":0.78100000000000003},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.0309999999999999,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":0.67800000000000005}]},{"osd":4,"last update":"Sun Aug 27 21:53:58 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.96799999999999997,"15min":0.94799999999999995},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.2090000000000001,"5min":1.5860000000000001,"15min":1.5860000000000001},"last":1.0980000000000001},{"interface":"front","average":{"1min":1.012,"5min":1.0309999999999999,"15min":1.012},"min":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"max":{"1min":1.26,"5min":1.6299999999999999,"15min":1.6299999999999999},"last":1.0289999999999999}]},{"osd":5,"last update":"Sun Aug 27 21:54:46 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.95399999999999996,"15min":0.96599999999999997},"min":{"1min":0.72599999999999998,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.2110000000000001,"5min":1.4950000000000001,"15min":1.4950000000000001},"last":0.86599999999999999},{"interface":"front","average":{"1min":1.016,"5min":1.0529999999999999,"15min":1.048},"min":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.3819999999999999,"5min":1.448,"15min":1.448},"last":0.98199999999999998}]},{"osd":6,"last update":"Sun Aug 27 21:54:27 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":1.1020000000000001,"15min":1.095},"min":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"max":{"1min":1.3999999999999999,"5min":1.651,"15min":1.651},"last":1.0529999999999999},{"interface":"front","average":{"1min":1.119,"5min":1.1459999999999999,"15min":1.155},"min":{"1min":0.80000000000000004,"5min":0.628,"15min":0.628},"max":{"1min":1.355,"5min":1.5640000000000001,"15min":1.5640000000000001},"last":1.0049999999999999}]},{"osd":7,"last update":"Sun Aug 27 21:54:15 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":1.097,"15min":1.1080000000000001},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.286,"5min":1.5780000000000001,"15min":1.5780000000000001},"last":0.89400000000000002},{"interface":"front","average":{"1min":1.083,"5min":1.2430000000000001,"15min":1.2689999999999999},"min":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"max":{"1min":1.397,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":1.214}]}]},{"osd":2,"up_from":18,"seq":77309411407,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:54:33 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":0.69599999999999995,"15min":0.628},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.372},"max":{"1min":1.885,"5min":1.885,"15min":1.885},"last":1.3109999999999999},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.77100000000000002,"15min":0.64500000000000002},"min":{"1min":0.40899999999999997,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.718,"5min":1.718,"15min":1.718},"last":0.72699999999999998}]},{"osd":1,"last update":"Sun Aug 27 21:54:33 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.79000000000000004,"15min":0.68600000000000005},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.492,"5min":2.4769999999999999,"15min":2.4769999999999999},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.79300000000000004,"15min":0.67100000000000004},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.439},"max":{"1min":1.6080000000000001,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":0.89300000000000002}]},{"osd":3,"last update":"Sun Aug 27 21:54:15 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.82199999999999995,"15min":0.77600000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.5509999999999999,"5min":1.5509999999999999,"15min":1.5509999999999999},"last":0.85099999999999998},{"interface":"front","average":{"1min":0.82799999999999996,"5min":0.871,"15min":0.83299999999999996},"min":{"1min":0.53900000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.4359999999999999,"5min":1.55,"15min":1.55},"last":0.93799999999999994}]},{"osd":4,"last update":"Sun Aug 27 21:54:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":0.99299999999999999,"15min":0.97499999999999998},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.8580000000000001,"5min":1.8580000000000001,"15min":1.8580000000000001},"last":1.048},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.02,"15min":0.96799999999999997},"min":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.921,"5min":1.921,"15min":1.921},"last":1.006}]},{"osd":5,"last update":"Sun Aug 27 21:54:43 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.0589999999999999,"15min":1.0309999999999999},"min":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"max":{"1min":1.845,"5min":1.845,"15min":1.845},"last":1.1910000000000001},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.0940000000000001,"15min":1.069},"min":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.909,"5min":2.0089999999999999,"15min":2.0089999999999999},"last":1.079}]},{"osd":6,"last update":"Sun Aug 27 21:54:25 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":1.196,"15min":1.2},"min":{"1min":0.84199999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.8879999999999999,"5min":2.0960000000000001,"15min":2.0960000000000001},"last":1.1639999999999999},{"interface":"front","average":{"1min":1.1679999999999999,"5min":1.1639999999999999,"15min":1.1779999999999999},"min":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.78,"5min":1.78,"15min":1.78},"last":1.1459999999999999}]},{"osd":7,"last update":"Sun Aug 27 21:54:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":1.218,"15min":1.24},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.804,"5min":1.804,"15min":1.804},"last":1.4350000000000001},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.1850000000000001,"15min":1.1850000000000001},"min":{"1min":0.71199999999999997,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.758,"5min":1.758,"15min":1.758},"last":1.2470000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182470,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:54:20 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.69899999999999995,"15min":0.67900000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.0069999999999999,"5min":1.1439999999999999,"15min":1.1439999999999999},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.74199999999999999,"15min":0.73999999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.113,"5min":1.224,"15min":1.224},"last":0.77400000000000002}]},{"osd":1,"last update":"Sun Aug 27 21:54:20 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.77000000000000002,"15min":0.76300000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.0660000000000001,"5min":1.111,"15min":1.111},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.80900000000000005,"15min":0.73799999999999999},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":2.883,"5min":2.883,"15min":2.883},"last":0.55800000000000005}]},{"osd":2,"last update":"Sun Aug 27 21:54:20 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.88500000000000001,"15min":0.86899999999999999},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.1439999999999999,"5min":1.151,"15min":1.151},"last":0.91600000000000004},{"interface":"front","average":{"1min":0.871,"5min":0.89500000000000002,"15min":0.88900000000000001},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":3.319,"5min":3.319,"15min":3.319},"last":1.1200000000000001}]},{"osd":4,"last update":"Sun Aug 27 21:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.012,"15min":0.99299999999999999},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":3.5939999999999999,"5min":3.5939999999999999,"15min":3.5939999999999999},"last":1.008},{"interface":"front","average":{"1min":1.0740000000000001,"5min":1.026,"15min":1.0109999999999999},"min":{"1min":0.751,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.3899999999999999,"5min":1.4410000000000001,"15min":1.4410000000000001},"last":0.97799999999999998}]},{"osd":5,"last update":"Sun Aug 27 21:54:44 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":1.042,"15min":1.044},"min":{"1min":0.622,"5min":0.622,"15min":0.622},"max":{"1min":1.4039999999999999,"5min":3.5670000000000002,"15min":3.5670000000000002},"last":1.1830000000000001},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.0469999999999999,"15min":1.038},"min":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.4890000000000001,"5min":1.5369999999999999,"15min":1.5369999999999999},"last":0.96599999999999997}]},{"osd":6,"last update":"Sun Aug 27 21:54:24 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.097,"15min":1.0960000000000001},"min":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"max":{"1min":1.4079999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":1.0960000000000001},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.113,"15min":1.117},"min":{"1min":0.751,"5min":0.751,"15min":0.751},"max":{"1min":1.359,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":0.95299999999999996}]},{"osd":7,"last update":"Sun Aug 27 21:54:11 2023","interfaces":[{"interface":"back","average":{"1min":1.2370000000000001,"5min":1.1479999999999999,"15min":1.133},"min":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"max":{"1min":3.5270000000000001,"5min":3.5270000000000001,"15min":3.5270000000000001},"last":1.2330000000000001},{"interface":"front","average":{"1min":1.171,"5min":1.23,"15min":1.24},"min":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.4850000000000001,"5min":1.5900000000000001,"15min":1.5900000000000001},"last":1.214}]}]},{"osd":4,"up_from":30,"seq":128849018942,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:54:00 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.879,"15min":0.82499999999999996},"min":{"1min":0.61499999999999999,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.302,"5min":3.383,"15min":3.383},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.82599999999999996,"5min":0.82699999999999996,"15min":0.82999999999999996},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.1499999999999999,"5min":1.2010000000000001,"15min":1.2010000000000001},"last":0.54300000000000004}]},{"osd":1,"last update":"Sun Aug 27 21:54:00 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.82499999999999996,"15min":0.79700000000000004},"min":{"1min":0.56899999999999995,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.1619999999999999,"5min":2.1349999999999998,"15min":2.1349999999999998},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.93400000000000005,"15min":0.88600000000000001},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.345,"5min":3.3340000000000001,"15min":3.3340000000000001},"last":0.70799999999999996}]},{"osd":2,"last update":"Sun Aug 27 21:54:00 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.89400000000000002,"15min":0.85199999999999998},"min":{"1min":0.64600000000000002,"5min":0.62,"15min":0.62},"max":{"1min":1.7430000000000001,"5min":3.4260000000000002,"15min":3.4260000000000002},"last":1.0640000000000001},{"interface":"front","average":{"1min":0.83299999999999996,"5min":0.85299999999999998,"15min":0.86299999999999999},"min":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.1060000000000001,"5min":1.149,"15min":1.149},"last":0.95099999999999996}]},{"osd":3,"last update":"Sun Aug 27 21:54:00 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.86299999999999999,"15min":0.873},"min":{"1min":0.67900000000000005,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.1759999999999999,"5min":1.1759999999999999,"15min":1.1759999999999999},"last":1.038},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.90600000000000003,"15min":0.91400000000000003},"min":{"1min":0.69499999999999995,"5min":0.629,"15min":0.629},"max":{"1min":1.236,"5min":1.236,"15min":1.236},"last":0.90100000000000002}]},{"osd":5,"last update":"Sun Aug 27 21:54:46 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.88900000000000001,"15min":0.91600000000000004},"min":{"1min":0.57299999999999995,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.3200000000000001,"5min":3.2200000000000002,"15min":3.2200000000000002},"last":1.143},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.82999999999999996,"15min":0.82599999999999996},"min":{"1min":0.55300000000000005,"5min":0.378,"15min":0.378},"max":{"1min":1.282,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":1.0940000000000001}]},{"osd":6,"last update":"Sun Aug 27 21:54:28 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":0.82899999999999996,"15min":0.81000000000000005},"min":{"1min":0.55800000000000005,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.3280000000000001,"5min":1.5109999999999999,"15min":1.5109999999999999},"last":1.016},{"interface":"front","average":{"1min":1.006,"5min":0.94099999999999995,"15min":0.92300000000000004},"min":{"1min":0.625,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.583,"5min":3.278,"15min":3.278},"last":1.1990000000000001}]},{"osd":7,"last update":"Sun Aug 27 21:54:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":0.94199999999999995,"15min":0.92400000000000004},"min":{"1min":0.68200000000000005,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.637,"5min":1.637,"15min":1.637},"last":1.2509999999999999},{"interface":"front","average":{"1min":1.0089999999999999,"5min":0.90700000000000003,"15min":0.89000000000000001},"min":{"1min":0.58899999999999997,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.5449999999999999,"5min":1.5449999999999999,"15min":1.5449999999999999},"last":1.103}]}]},{"osd":5,"up_from":36,"seq":154618822709,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297616,"kb_used_data":520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447536,"statfs":{"total":95995035648,"available":95690276864,"internally_reserved":0,"allocated":532480,"data_stored":219885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:54:45 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":0.84299999999999997,"15min":0.877},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.3320000000000001,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":1.149},{"interface":"front","average":{"1min":0.82799999999999996,"5min":0.85199999999999998,"15min":0.86399999999999999},"min":{"1min":0.58699999999999997,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.2050000000000001,"5min":1.2050000000000001,"15min":1.2050000000000001},"last":0.80000000000000004}]},{"osd":1,"last update":"Sun Aug 27 21:54:45 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":0.89200000000000002,"15min":0.92000000000000004},"min":{"1min":0.53000000000000003,"5min":0.504,"15min":0.504},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.94899999999999995},{"interface":"front","average":{"1min":0.89100000000000001,"5min":0.88400000000000001,"15min":0.88600000000000001},"min":{"1min":0.61599999999999999,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.605,"5min":1.605,"15min":1.605},"last":1.1859999999999999}]},{"osd":2,"last update":"Sun Aug 27 21:54:45 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.91300000000000003,"15min":0.90500000000000003},"min":{"1min":0.67900000000000005,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.2350000000000001,"5min":1.7609999999999999,"15min":1.7609999999999999},"last":0.85999999999999999},{"interface":"front","average":{"1min":0.84499999999999997,"5min":0.89000000000000001,"15min":0.90700000000000003},"min":{"1min":0.65200000000000002,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.0840000000000001,"5min":1.264,"15min":1.264},"last":1.0640000000000001}]},{"osd":3,"last update":"Sun Aug 27 21:54:45 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.93999999999999995,"15min":0.93300000000000005},"min":{"1min":0.621,"5min":0.503,"15min":0.503},"max":{"1min":1.1859999999999999,"5min":1.587,"15min":1.587},"last":1.246},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.94999999999999996,"15min":0.94799999999999995},"min":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.2450000000000001,"5min":1.506,"15min":1.506},"last":1.2170000000000001}]},{"osd":4,"last update":"Sun Aug 27 21:54:45 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.83099999999999996,"15min":0.78000000000000003},"min":{"1min":0.61299999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.6919999999999999,"5min":1.6919999999999999,"15min":1.6919999999999999},"last":1.0009999999999999},{"interface":"front","average":{"1min":1.0489999999999999,"5min":0.88,"15min":0.79100000000000004},"min":{"1min":0.77900000000000003,"5min":0.442,"15min":0.442},"max":{"1min":1.3180000000000001,"5min":1.538,"15min":1.538},"last":1.333}]},{"osd":6,"last update":"Sun Aug 27 21:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.94599999999999995,"15min":0.94699999999999995},"min":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.4119999999999999,"5min":1.7050000000000001,"15min":1.7050000000000001},"last":1.0229999999999999},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.95999999999999996,"15min":0.96999999999999997},"min":{"1min":0.55600000000000005,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.2529999999999999,"5min":1.4370000000000001,"15min":1.4370000000000001},"last":1.3029999999999999}]},{"osd":7,"last update":"Sun Aug 27 21:54:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.038,"15min":1.036},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.498,"5min":1.498,"15min":1.498},"last":1.3839999999999999},{"interface":"front","average":{"1min":1.04,"5min":1.0009999999999999,"15min":0.995},"min":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.4990000000000001,"5min":1.611,"15min":1.611},"last":1.0940000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-27T21:54:56.087 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-08-27T21:54:56.088 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-08-27T21:54:56.088 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-08-27T21:54:56.088 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph health --format=json 2023-08-27T21:54:56.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:55 smithi130 bash[27579]: cluster 2023-08-27T21:54:54.623083+0000 mgr.y (mgr.14430) 93 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:56.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:56 smithi006 bash[27292]: audit 2023-08-27T21:54:55.320653+0000 mgr.y (mgr.14430) 94 : audit [DBG] from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T21:54:56.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:56 smithi006 bash[33918]: audit 2023-08-27T21:54:55.320653+0000 mgr.y (mgr.14430) 94 : audit [DBG] from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T21:54:57.001 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:56 smithi130 bash[27579]: audit 2023-08-27T21:54:55.320653+0000 mgr.y (mgr.14430) 94 : audit [DBG] from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T21:54:57.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:54:56 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:54:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:54:57.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:57 smithi006 bash[27292]: cluster 2023-08-27T21:54:56.623802+0000 mgr.y (mgr.14430) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:57.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:57 smithi006 bash[33918]: cluster 2023-08-27T21:54:56.623802+0000 mgr.y (mgr.14430) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:58.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:57 smithi130 bash[27579]: cluster 2023-08-27T21:54:56.623802+0000 mgr.y (mgr.14430) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:59.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:54:59 smithi006 bash[33918]: cluster 2023-08-27T21:54:58.624396+0000 mgr.y (mgr.14430) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:54:59.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:54:59 smithi006 bash[27292]: cluster 2023-08-27T21:54:58.624396+0000 mgr.y (mgr.14430) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:00.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:54:59 smithi130 bash[27579]: cluster 2023-08-27T21:54:58.624396+0000 mgr.y (mgr.14430) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:00.901 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:55:01.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:55:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:55:00] "GET /metrics HTTP/1.1" 200 155668 "" "Prometheus/2.33.4" 2023-08-27T21:55:01.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:01 smithi006 bash[33918]: cluster 2023-08-27T21:55:00.625078+0000 mgr.y (mgr.14430) 97 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:01.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:01 smithi006 bash[27292]: cluster 2023-08-27T21:55:00.625078+0000 mgr.y (mgr.14430) 97 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:02.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:01 smithi130 bash[27579]: cluster 2023-08-27T21:55:00.625078+0000 mgr.y (mgr.14430) 97 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:03.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:03 smithi130 bash[27579]: cluster 2023-08-27T21:55:02.625649+0000 mgr.y (mgr.14430) 98 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:03.940 INFO:teuthology.orchestra.run.smithi006.stdout: 2023-08-27T21:55:03.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:03 smithi006 bash[33918]: cluster 2023-08-27T21:55:02.625649+0000 mgr.y (mgr.14430) 98 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:03.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:03 smithi006 bash[27292]: cluster 2023-08-27T21:55:02.625649+0000 mgr.y (mgr.14430) 98 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:04.647 INFO:teuthology.orchestra.run.smithi006.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-08-27T21:55:04.647 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-08-27T21:55:04.647 INFO:tasks.cephadm:Setup complete, yielding 2023-08-27T21:55:04.648 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-08-27T21:55:04.657 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi006.front.sepia.ceph.com 2023-08-27T21:55:04.657 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- bash -c 'ceph fs volume create foo' 2023-08-27T21:55:04.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:04 smithi130 bash[27579]: audit 2023-08-27T21:55:03.935521+0000 mon.a (mon.0) 666 : audit [DBG] from='client.? 172.21.15.6:0/2028215198' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-08-27T21:55:04.872 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:04 smithi006 bash[33918]: audit 2023-08-27T21:55:03.935521+0000 mon.a (mon.0) 666 : audit [DBG] from='client.? 172.21.15.6:0/2028215198' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-08-27T21:55:04.872 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:04 smithi006 bash[27292]: audit 2023-08-27T21:55:03.935521+0000 mon.a (mon.0) 666 : audit [DBG] from='client.? 172.21.15.6:0/2028215198' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-08-27T21:55:05.849 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:05 smithi006 bash[33918]: cluster 2023-08-27T21:55:04.626192+0000 mgr.y (mgr.14430) 99 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:05.849 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:05 smithi006 bash[27292]: cluster 2023-08-27T21:55:04.626192+0000 mgr.y (mgr.14430) 99 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:05.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:05 smithi130 bash[27579]: cluster 2023-08-27T21:55:04.626192+0000 mgr.y (mgr.14430) 99 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:07.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:55:06 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:55:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:55:07.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:07 smithi006 bash[27292]: cluster 2023-08-27T21:55:06.626809+0000 mgr.y (mgr.14430) 100 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:07.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:07 smithi006 bash[33918]: cluster 2023-08-27T21:55:06.626809+0000 mgr.y (mgr.14430) 100 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:08.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:07 smithi130 bash[27579]: cluster 2023-08-27T21:55:06.626809+0000 mgr.y (mgr.14430) 100 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:08.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:08 smithi006 bash[27292]: audit 2023-08-27T21:55:08.069641+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-08-27T21:55:08.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:08 smithi006 bash[33918]: audit 2023-08-27T21:55:08.069641+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-08-27T21:55:09.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:08 smithi130 bash[27579]: audit 2023-08-27T21:55:08.069641+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-08-27T21:55:09.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:09 smithi006 bash[27292]: audit 2023-08-27T21:55:08.068476+0000 mgr.y (mgr.14430) 101 : audit [DBG] from='client.14628 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:55:09.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:09 smithi006 bash[27292]: cluster 2023-08-27T21:55:08.627327+0000 mgr.y (mgr.14430) 102 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:09.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:09 smithi006 bash[27292]: audit 2023-08-27T21:55:08.703568+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-08-27T21:55:09.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:09 smithi006 bash[27292]: cluster 2023-08-27T21:55:08.703671+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-08-27T21:55:09.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:09 smithi006 bash[27292]: audit 2023-08-27T21:55:08.705702+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-08-27T21:55:09.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:09 smithi006 bash[27292]: debug 2023-08-27T21:55:09.724+0000 7f97bfa13700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-08-27T21:55:09.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:09 smithi006 bash[33918]: audit 2023-08-27T21:55:08.068476+0000 mgr.y (mgr.14430) 101 : audit [DBG] from='client.14628 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:55:09.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:09 smithi006 bash[33918]: cluster 2023-08-27T21:55:08.627327+0000 mgr.y (mgr.14430) 102 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:09.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:09 smithi006 bash[33918]: audit 2023-08-27T21:55:08.703568+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-08-27T21:55:09.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:09 smithi006 bash[33918]: cluster 2023-08-27T21:55:08.703671+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-08-27T21:55:09.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:09 smithi006 bash[33918]: audit 2023-08-27T21:55:08.705702+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-08-27T21:55:10.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:09 smithi130 bash[27579]: audit 2023-08-27T21:55:08.068476+0000 mgr.y (mgr.14430) 101 : audit [DBG] from='client.14628 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:55:10.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:09 smithi130 bash[27579]: cluster 2023-08-27T21:55:08.627327+0000 mgr.y (mgr.14430) 102 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:10.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:09 smithi130 bash[27579]: audit 2023-08-27T21:55:08.703568+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-08-27T21:55:10.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:09 smithi130 bash[27579]: cluster 2023-08-27T21:55:08.703671+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-08-27T21:55:10.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:09 smithi130 bash[27579]: audit 2023-08-27T21:55:08.705702+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-08-27T21:55:10.447 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-08-27T21:55:10.969 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:55:10] "GET /metrics HTTP/1.1" 200 155668 "" "Prometheus/2.33.4" 2023-08-27T21:55:10.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[27292]: audit 2023-08-27T21:55:09.714783+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-08-27T21:55:10.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[27292]: cluster 2023-08-27T21:55:09.715059+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-08-27T21:55:10.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[27292]: audit 2023-08-27T21:55:09.725919+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-08-27T21:55:10.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[27292]: cluster 2023-08-27T21:55:09.726850+0000 mon.a (mon.0) 674 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-08-27T21:55:10.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[27292]: cluster 2023-08-27T21:55:09.726893+0000 mon.a (mon.0) 675 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-08-27T21:55:10.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[27292]: cluster 2023-08-27T21:55:09.737904+0000 mon.a (mon.0) 676 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-08-27T21:55:10.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[27292]: audit 2023-08-27T21:55:09.738246+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-08-27T21:55:10.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[27292]: cluster 2023-08-27T21:55:09.738516+0000 mon.a (mon.0) 678 : cluster [DBG] fsmap foo:0 2023-08-27T21:55:10.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[27292]: cephadm 2023-08-27T21:55:09.742163+0000 mgr.y (mgr.14430) 103 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-08-27T21:55:10.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[27292]: audit 2023-08-27T21:55:09.752582+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:10.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[27292]: audit 2023-08-27T21:55:09.799828+0000 mon.a (mon.0) 680 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:55:10.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[27292]: audit 2023-08-27T21:55:10.058314+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:10.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[27292]: audit 2023-08-27T21:55:10.329617+0000 mon.a (mon.0) 682 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:55:10.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[27292]: audit 2023-08-27T21:55:10.332823+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:55:10.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[27292]: audit 2023-08-27T21:55:10.341346+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:10.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[27292]: audit 2023-08-27T21:55:10.350863+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:10.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[27292]: audit 2023-08-27T21:55:10.352148+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi006.bzplue", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-27T21:55:10.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[27292]: audit 2023-08-27T21:55:10.356795+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi006.bzplue", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-08-27T21:55:10.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[27292]: audit 2023-08-27T21:55:10.358484+0000 mon.a (mon.0) 688 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:55:10.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[33918]: audit 2023-08-27T21:55:09.714783+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-08-27T21:55:10.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[33918]: cluster 2023-08-27T21:55:09.715059+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-08-27T21:55:10.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[33918]: audit 2023-08-27T21:55:09.725919+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-08-27T21:55:10.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[33918]: cluster 2023-08-27T21:55:09.726850+0000 mon.a (mon.0) 674 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-08-27T21:55:10.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[33918]: cluster 2023-08-27T21:55:09.726893+0000 mon.a (mon.0) 675 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-08-27T21:55:10.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[33918]: cluster 2023-08-27T21:55:09.737904+0000 mon.a (mon.0) 676 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-08-27T21:55:10.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[33918]: audit 2023-08-27T21:55:09.738246+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-08-27T21:55:10.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[33918]: cluster 2023-08-27T21:55:09.738516+0000 mon.a (mon.0) 678 : cluster [DBG] fsmap foo:0 2023-08-27T21:55:10.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[33918]: cephadm 2023-08-27T21:55:09.742163+0000 mgr.y (mgr.14430) 103 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-08-27T21:55:10.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[33918]: audit 2023-08-27T21:55:09.752582+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:10.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[33918]: audit 2023-08-27T21:55:09.799828+0000 mon.a (mon.0) 680 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:55:10.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[33918]: audit 2023-08-27T21:55:10.058314+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:10.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[33918]: audit 2023-08-27T21:55:10.329617+0000 mon.a (mon.0) 682 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:55:10.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[33918]: audit 2023-08-27T21:55:10.332823+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:55:10.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[33918]: audit 2023-08-27T21:55:10.341346+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:10.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[33918]: audit 2023-08-27T21:55:10.350863+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:10.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[33918]: audit 2023-08-27T21:55:10.352148+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi006.bzplue", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-27T21:55:10.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[33918]: audit 2023-08-27T21:55:10.356795+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi006.bzplue", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-08-27T21:55:10.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:10 smithi006 bash[33918]: audit 2023-08-27T21:55:10.358484+0000 mon.a (mon.0) 688 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:55:11.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:10 smithi130 bash[27579]: audit 2023-08-27T21:55:09.714783+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-08-27T21:55:11.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:10 smithi130 bash[27579]: cluster 2023-08-27T21:55:09.715059+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-08-27T21:55:11.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:10 smithi130 bash[27579]: audit 2023-08-27T21:55:09.725919+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-08-27T21:55:11.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:10 smithi130 bash[27579]: cluster 2023-08-27T21:55:09.726850+0000 mon.a (mon.0) 674 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-08-27T21:55:11.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:10 smithi130 bash[27579]: cluster 2023-08-27T21:55:09.726893+0000 mon.a (mon.0) 675 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-08-27T21:55:11.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:10 smithi130 bash[27579]: cluster 2023-08-27T21:55:09.737904+0000 mon.a (mon.0) 676 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-08-27T21:55:11.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:10 smithi130 bash[27579]: audit 2023-08-27T21:55:09.738246+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-08-27T21:55:11.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:10 smithi130 bash[27579]: cluster 2023-08-27T21:55:09.738516+0000 mon.a (mon.0) 678 : cluster [DBG] fsmap foo:0 2023-08-27T21:55:11.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:10 smithi130 bash[27579]: cephadm 2023-08-27T21:55:09.742163+0000 mgr.y (mgr.14430) 103 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-08-27T21:55:11.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:10 smithi130 bash[27579]: audit 2023-08-27T21:55:09.752582+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:11.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:10 smithi130 bash[27579]: audit 2023-08-27T21:55:09.799828+0000 mon.a (mon.0) 680 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:55:11.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:10 smithi130 bash[27579]: audit 2023-08-27T21:55:10.058314+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:11.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:10 smithi130 bash[27579]: audit 2023-08-27T21:55:10.329617+0000 mon.a (mon.0) 682 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:55:11.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:10 smithi130 bash[27579]: audit 2023-08-27T21:55:10.332823+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:55:11.122 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:10 smithi130 bash[27579]: audit 2023-08-27T21:55:10.341346+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:11.122 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:10 smithi130 bash[27579]: audit 2023-08-27T21:55:10.350863+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:11.122 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:10 smithi130 bash[27579]: audit 2023-08-27T21:55:10.352148+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi006.bzplue", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-27T21:55:11.122 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:10 smithi130 bash[27579]: audit 2023-08-27T21:55:10.356795+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi006.bzplue", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-08-27T21:55:11.122 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:10 smithi130 bash[27579]: audit 2023-08-27T21:55:10.358484+0000 mon.a (mon.0) 688 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:55:12.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:11 smithi130 bash[27579]: cephadm 2023-08-27T21:55:10.359602+0000 mgr.y (mgr.14430) 104 : cephadm [INF] Deploying daemon mds.foo.smithi006.bzplue on smithi006 2023-08-27T21:55:12.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:11 smithi130 bash[27579]: cluster 2023-08-27T21:55:10.628235+0000 mgr.y (mgr.14430) 105 : cluster [DBG] pgmap v83: 73 pgs: 19 creating+peering, 54 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:12.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:11 smithi130 bash[27579]: cluster 2023-08-27T21:55:10.738624+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-08-27T21:55:12.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:11 smithi006 bash[27292]: cephadm 2023-08-27T21:55:10.359602+0000 mgr.y (mgr.14430) 104 : cephadm [INF] Deploying daemon mds.foo.smithi006.bzplue on smithi006 2023-08-27T21:55:12.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:11 smithi006 bash[27292]: cluster 2023-08-27T21:55:10.628235+0000 mgr.y (mgr.14430) 105 : cluster [DBG] pgmap v83: 73 pgs: 19 creating+peering, 54 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:12.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:11 smithi006 bash[27292]: cluster 2023-08-27T21:55:10.738624+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-08-27T21:55:12.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:11 smithi006 bash[33918]: cephadm 2023-08-27T21:55:10.359602+0000 mgr.y (mgr.14430) 104 : cephadm [INF] Deploying daemon mds.foo.smithi006.bzplue on smithi006 2023-08-27T21:55:12.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:11 smithi006 bash[33918]: cluster 2023-08-27T21:55:10.628235+0000 mgr.y (mgr.14430) 105 : cluster [DBG] pgmap v83: 73 pgs: 19 creating+peering, 54 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:12.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:11 smithi006 bash[33918]: cluster 2023-08-27T21:55:10.738624+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-08-27T21:55:12.846 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:12 smithi006 bash[27292]: cluster 2023-08-27T21:55:11.741678+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-08-27T21:55:12.847 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:12 smithi006 bash[33918]: cluster 2023-08-27T21:55:11.741678+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-08-27T21:55:13.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:12 smithi130 bash[27579]: cluster 2023-08-27T21:55:11.741678+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-08-27T21:55:13.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:13 smithi006 bash[33918]: cluster 2023-08-27T21:55:12.628941+0000 mgr.y (mgr.14430) 106 : cluster [DBG] pgmap v86: 73 pgs: 19 creating+peering, 54 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:13.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:13 smithi006 bash[27292]: cluster 2023-08-27T21:55:12.628941+0000 mgr.y (mgr.14430) 106 : cluster [DBG] pgmap v86: 73 pgs: 19 creating+peering, 54 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:14.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:13 smithi130 bash[27579]: cluster 2023-08-27T21:55:12.628941+0000 mgr.y (mgr.14430) 106 : cluster [DBG] pgmap v86: 73 pgs: 19 creating+peering, 54 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:15.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:15 smithi130 bash[27579]: audit 2023-08-27T21:55:14.096541+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:15.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:15 smithi130 bash[27579]: audit 2023-08-27T21:55:14.098277+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi130.qmrguj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-27T21:55:15.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:15 smithi130 bash[27579]: audit 2023-08-27T21:55:14.103332+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi130.qmrguj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-08-27T21:55:15.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:15 smithi130 bash[27579]: audit 2023-08-27T21:55:14.104482+0000 mon.a (mon.0) 694 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:55:15.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:15 smithi130 bash[27579]: cephadm 2023-08-27T21:55:14.105385+0000 mgr.y (mgr.14430) 107 : cephadm [INF] Deploying daemon mds.foo.smithi130.qmrguj on smithi130 2023-08-27T21:55:15.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:15 smithi130 bash[27579]: cluster 2023-08-27T21:55:14.629631+0000 mgr.y (mgr.14430) 108 : cluster [DBG] pgmap v87: 73 pgs: 19 creating+peering, 54 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:15.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:15 smithi130 bash[27579]: audit 2023-08-27T21:55:14.826934+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:15.395 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:15 smithi006 bash[27292]: audit 2023-08-27T21:55:14.096541+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:15.395 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:15 smithi006 bash[27292]: audit 2023-08-27T21:55:14.098277+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi130.qmrguj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-27T21:55:15.395 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:15 smithi006 bash[27292]: audit 2023-08-27T21:55:14.103332+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi130.qmrguj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-08-27T21:55:15.395 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:15 smithi006 bash[27292]: audit 2023-08-27T21:55:14.104482+0000 mon.a (mon.0) 694 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:55:15.395 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:15 smithi006 bash[27292]: cephadm 2023-08-27T21:55:14.105385+0000 mgr.y (mgr.14430) 107 : cephadm [INF] Deploying daemon mds.foo.smithi130.qmrguj on smithi130 2023-08-27T21:55:15.396 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:15 smithi006 bash[27292]: cluster 2023-08-27T21:55:14.629631+0000 mgr.y (mgr.14430) 108 : cluster [DBG] pgmap v87: 73 pgs: 19 creating+peering, 54 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:15.396 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:15 smithi006 bash[27292]: audit 2023-08-27T21:55:14.826934+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:15.396 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:15 smithi006 bash[33918]: audit 2023-08-27T21:55:14.096541+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:15.396 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:15 smithi006 bash[33918]: audit 2023-08-27T21:55:14.098277+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi130.qmrguj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-27T21:55:15.396 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:15 smithi006 bash[33918]: audit 2023-08-27T21:55:14.103332+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi130.qmrguj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-08-27T21:55:15.397 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:15 smithi006 bash[33918]: audit 2023-08-27T21:55:14.104482+0000 mon.a (mon.0) 694 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:55:15.397 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:15 smithi006 bash[33918]: cephadm 2023-08-27T21:55:14.105385+0000 mgr.y (mgr.14430) 107 : cephadm [INF] Deploying daemon mds.foo.smithi130.qmrguj on smithi130 2023-08-27T21:55:15.397 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:15 smithi006 bash[33918]: cluster 2023-08-27T21:55:14.629631+0000 mgr.y (mgr.14430) 108 : cluster [DBG] pgmap v87: 73 pgs: 19 creating+peering, 54 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:15.397 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:15 smithi006 bash[33918]: audit 2023-08-27T21:55:14.826934+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:16.479 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-08-27T21:55:17.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:55:16 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:55:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:55:17.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:17 smithi130 bash[27579]: cluster 2023-08-27T21:55:16.112267+0000 mon.a (mon.0) 696 : cluster [DBG] mds.? [v2:172.21.15.6:6834/360157245,v1:172.21.15.6:6835/360157245] up:boot 2023-08-27T21:55:17.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:17 smithi130 bash[27579]: cluster 2023-08-27T21:55:16.112329+0000 mon.a (mon.0) 697 : cluster [INF] daemon mds.foo.smithi006.bzplue assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-08-27T21:55:17.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:17 smithi130 bash[27579]: cluster 2023-08-27T21:55:16.112391+0000 mon.a (mon.0) 698 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-08-27T21:55:17.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:17 smithi130 bash[27579]: cluster 2023-08-27T21:55:16.112420+0000 mon.a (mon.0) 699 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-08-27T21:55:17.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:17 smithi130 bash[27579]: cluster 2023-08-27T21:55:16.112447+0000 mon.a (mon.0) 700 : cluster [INF] Cluster is now healthy 2023-08-27T21:55:17.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:17 smithi130 bash[27579]: cluster 2023-08-27T21:55:16.112713+0000 mon.a (mon.0) 701 : cluster [DBG] fsmap foo:0 1 up:standby 2023-08-27T21:55:17.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:17 smithi130 bash[27579]: audit 2023-08-27T21:55:16.112984+0000 mon.a (mon.0) 702 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi006.bzplue"}]: dispatch 2023-08-27T21:55:17.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:17 smithi130 bash[27579]: cluster 2023-08-27T21:55:16.117993+0000 mon.a (mon.0) 703 : cluster [DBG] fsmap foo:1 {0=foo.smithi006.bzplue=up:creating} 2023-08-27T21:55:17.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:17 smithi130 bash[27579]: cluster 2023-08-27T21:55:16.136643+0000 mon.a (mon.0) 704 : cluster [INF] daemon mds.foo.smithi006.bzplue is now active in filesystem foo as rank 0 2023-08-27T21:55:17.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:17 smithi130 bash[27579]: cluster 2023-08-27T21:55:16.630654+0000 mgr.y (mgr.14430) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:17.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:17 smithi130 bash[27579]: audit 2023-08-27T21:55:16.671568+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:17.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:17 smithi130 bash[27579]: audit 2023-08-27T21:55:16.673804+0000 mon.a (mon.0) 706 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:55:17.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[27292]: cluster 2023-08-27T21:55:16.112267+0000 mon.a (mon.0) 696 : cluster [DBG] mds.? [v2:172.21.15.6:6834/360157245,v1:172.21.15.6:6835/360157245] up:boot 2023-08-27T21:55:17.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[27292]: cluster 2023-08-27T21:55:16.112329+0000 mon.a (mon.0) 697 : cluster [INF] daemon mds.foo.smithi006.bzplue assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-08-27T21:55:17.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[27292]: cluster 2023-08-27T21:55:16.112391+0000 mon.a (mon.0) 698 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-08-27T21:55:17.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[27292]: cluster 2023-08-27T21:55:16.112420+0000 mon.a (mon.0) 699 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-08-27T21:55:17.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[27292]: cluster 2023-08-27T21:55:16.112447+0000 mon.a (mon.0) 700 : cluster [INF] Cluster is now healthy 2023-08-27T21:55:17.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[27292]: cluster 2023-08-27T21:55:16.112713+0000 mon.a (mon.0) 701 : cluster [DBG] fsmap foo:0 1 up:standby 2023-08-27T21:55:17.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[27292]: audit 2023-08-27T21:55:16.112984+0000 mon.a (mon.0) 702 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi006.bzplue"}]: dispatch 2023-08-27T21:55:17.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[27292]: cluster 2023-08-27T21:55:16.117993+0000 mon.a (mon.0) 703 : cluster [DBG] fsmap foo:1 {0=foo.smithi006.bzplue=up:creating} 2023-08-27T21:55:17.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[27292]: cluster 2023-08-27T21:55:16.136643+0000 mon.a (mon.0) 704 : cluster [INF] daemon mds.foo.smithi006.bzplue is now active in filesystem foo as rank 0 2023-08-27T21:55:17.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[27292]: cluster 2023-08-27T21:55:16.630654+0000 mgr.y (mgr.14430) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:17.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[27292]: audit 2023-08-27T21:55:16.671568+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:17.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[27292]: audit 2023-08-27T21:55:16.673804+0000 mon.a (mon.0) 706 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:55:17.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[33918]: cluster 2023-08-27T21:55:16.112267+0000 mon.a (mon.0) 696 : cluster [DBG] mds.? [v2:172.21.15.6:6834/360157245,v1:172.21.15.6:6835/360157245] up:boot 2023-08-27T21:55:17.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[33918]: cluster 2023-08-27T21:55:16.112329+0000 mon.a (mon.0) 697 : cluster [INF] daemon mds.foo.smithi006.bzplue assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-08-27T21:55:17.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[33918]: cluster 2023-08-27T21:55:16.112391+0000 mon.a (mon.0) 698 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-08-27T21:55:17.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[33918]: cluster 2023-08-27T21:55:16.112420+0000 mon.a (mon.0) 699 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-08-27T21:55:17.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[33918]: cluster 2023-08-27T21:55:16.112447+0000 mon.a (mon.0) 700 : cluster [INF] Cluster is now healthy 2023-08-27T21:55:17.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[33918]: cluster 2023-08-27T21:55:16.112713+0000 mon.a (mon.0) 701 : cluster [DBG] fsmap foo:0 1 up:standby 2023-08-27T21:55:17.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[33918]: audit 2023-08-27T21:55:16.112984+0000 mon.a (mon.0) 702 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi006.bzplue"}]: dispatch 2023-08-27T21:55:17.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[33918]: cluster 2023-08-27T21:55:16.117993+0000 mon.a (mon.0) 703 : cluster [DBG] fsmap foo:1 {0=foo.smithi006.bzplue=up:creating} 2023-08-27T21:55:17.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[33918]: cluster 2023-08-27T21:55:16.136643+0000 mon.a (mon.0) 704 : cluster [INF] daemon mds.foo.smithi006.bzplue is now active in filesystem foo as rank 0 2023-08-27T21:55:17.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[33918]: cluster 2023-08-27T21:55:16.630654+0000 mgr.y (mgr.14430) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:17.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[33918]: audit 2023-08-27T21:55:16.671568+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:17.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:17 smithi006 bash[33918]: audit 2023-08-27T21:55:16.673804+0000 mon.a (mon.0) 706 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:55:18.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:18 smithi006 bash[27292]: cluster 2023-08-27T21:55:17.123947+0000 mon.a (mon.0) 707 : cluster [DBG] mds.? [v2:172.21.15.6:6834/360157245,v1:172.21.15.6:6835/360157245] up:active 2023-08-27T21:55:18.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:18 smithi006 bash[27292]: cluster 2023-08-27T21:55:17.124012+0000 mon.a (mon.0) 708 : cluster [DBG] fsmap foo:1 {0=foo.smithi006.bzplue=up:active} 2023-08-27T21:55:18.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:18 smithi006 bash[33918]: cluster 2023-08-27T21:55:17.123947+0000 mon.a (mon.0) 707 : cluster [DBG] mds.? [v2:172.21.15.6:6834/360157245,v1:172.21.15.6:6835/360157245] up:active 2023-08-27T21:55:18.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:18 smithi006 bash[33918]: cluster 2023-08-27T21:55:17.124012+0000 mon.a (mon.0) 708 : cluster [DBG] fsmap foo:1 {0=foo.smithi006.bzplue=up:active} 2023-08-27T21:55:18.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:18 smithi130 bash[27579]: cluster 2023-08-27T21:55:17.123947+0000 mon.a (mon.0) 707 : cluster [DBG] mds.? [v2:172.21.15.6:6834/360157245,v1:172.21.15.6:6835/360157245] up:active 2023-08-27T21:55:18.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:18 smithi130 bash[27579]: cluster 2023-08-27T21:55:17.124012+0000 mon.a (mon.0) 708 : cluster [DBG] fsmap foo:1 {0=foo.smithi006.bzplue=up:active} 2023-08-27T21:55:19.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:19 smithi130 bash[27579]: cluster 2023-08-27T21:55:18.484743+0000 mon.a (mon.0) 709 : cluster [DBG] mds.? [v2:172.21.15.130:6832/2883196664,v1:172.21.15.130:6833/2883196664] up:boot 2023-08-27T21:55:19.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:19 smithi130 bash[27579]: cluster 2023-08-27T21:55:18.485086+0000 mon.a (mon.0) 710 : cluster [DBG] fsmap foo:1 {0=foo.smithi006.bzplue=up:active} 1 up:standby 2023-08-27T21:55:19.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:19 smithi130 bash[27579]: audit 2023-08-27T21:55:18.485345+0000 mon.a (mon.0) 711 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi130.qmrguj"}]: dispatch 2023-08-27T21:55:19.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:19 smithi130 bash[27579]: cluster 2023-08-27T21:55:18.490998+0000 mon.a (mon.0) 712 : cluster [DBG] fsmap foo:1 {0=foo.smithi006.bzplue=up:active} 1 up:standby 2023-08-27T21:55:19.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:19 smithi130 bash[27579]: cluster 2023-08-27T21:55:18.631514+0000 mgr.y (mgr.14430) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:19.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:19 smithi006 bash[27292]: cluster 2023-08-27T21:55:18.484743+0000 mon.a (mon.0) 709 : cluster [DBG] mds.? [v2:172.21.15.130:6832/2883196664,v1:172.21.15.130:6833/2883196664] up:boot 2023-08-27T21:55:19.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:19 smithi006 bash[27292]: cluster 2023-08-27T21:55:18.485086+0000 mon.a (mon.0) 710 : cluster [DBG] fsmap foo:1 {0=foo.smithi006.bzplue=up:active} 1 up:standby 2023-08-27T21:55:19.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:19 smithi006 bash[27292]: audit 2023-08-27T21:55:18.485345+0000 mon.a (mon.0) 711 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi130.qmrguj"}]: dispatch 2023-08-27T21:55:19.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:19 smithi006 bash[27292]: cluster 2023-08-27T21:55:18.490998+0000 mon.a (mon.0) 712 : cluster [DBG] fsmap foo:1 {0=foo.smithi006.bzplue=up:active} 1 up:standby 2023-08-27T21:55:19.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:19 smithi006 bash[27292]: cluster 2023-08-27T21:55:18.631514+0000 mgr.y (mgr.14430) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:19.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:19 smithi006 bash[33918]: cluster 2023-08-27T21:55:18.484743+0000 mon.a (mon.0) 709 : cluster [DBG] mds.? [v2:172.21.15.130:6832/2883196664,v1:172.21.15.130:6833/2883196664] up:boot 2023-08-27T21:55:19.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:19 smithi006 bash[33918]: cluster 2023-08-27T21:55:18.485086+0000 mon.a (mon.0) 710 : cluster [DBG] fsmap foo:1 {0=foo.smithi006.bzplue=up:active} 1 up:standby 2023-08-27T21:55:19.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:19 smithi006 bash[33918]: audit 2023-08-27T21:55:18.485345+0000 mon.a (mon.0) 711 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi130.qmrguj"}]: dispatch 2023-08-27T21:55:19.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:19 smithi006 bash[33918]: cluster 2023-08-27T21:55:18.490998+0000 mon.a (mon.0) 712 : cluster [DBG] fsmap foo:1 {0=foo.smithi006.bzplue=up:active} 1 up:standby 2023-08-27T21:55:19.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:19 smithi006 bash[33918]: cluster 2023-08-27T21:55:18.631514+0000 mgr.y (mgr.14430) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:20.707 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-08-27T21:55:20.717 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-08-27T21:55:20.717 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-08-27T21:55:20.718 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph mgr dump --format=json 2023-08-27T21:55:20.969 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:55:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:55:20] "GET /metrics HTTP/1.1" 200 160748 "" "Prometheus/2.33.4" 2023-08-27T21:55:20.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:20 smithi006 bash[33918]: audit 2023-08-27T21:55:19.905189+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:20.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:20 smithi006 bash[33918]: cluster 2023-08-27T21:55:20.632524+0000 mgr.y (mgr.14430) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:20.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:20 smithi006 bash[27292]: audit 2023-08-27T21:55:19.905189+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:20.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:20 smithi006 bash[27292]: cluster 2023-08-27T21:55:20.632524+0000 mgr.y (mgr.14430) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:21.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:20 smithi130 bash[27579]: audit 2023-08-27T21:55:19.905189+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:21.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:20 smithi130 bash[27579]: cluster 2023-08-27T21:55:20.632524+0000 mgr.y (mgr.14430) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:23.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:23 smithi130 bash[27579]: audit 2023-08-27T21:55:22.566923+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:23.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:23 smithi130 bash[27579]: cluster 2023-08-27T21:55:22.633228+0000 mgr.y (mgr.14430) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:23.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:23 smithi006 bash[27292]: audit 2023-08-27T21:55:22.566923+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:23.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:23 smithi006 bash[27292]: cluster 2023-08-27T21:55:22.633228+0000 mgr.y (mgr.14430) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:23.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:23 smithi006 bash[33918]: audit 2023-08-27T21:55:22.566923+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:23.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:23 smithi006 bash[33918]: cluster 2023-08-27T21:55:22.633228+0000 mgr.y (mgr.14430) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:24.561 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:55:24.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:24 smithi006 bash[33918]: audit 2023-08-27T21:55:23.645623+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:24.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:24 smithi006 bash[33918]: audit 2023-08-27T21:55:23.647510+0000 mon.a (mon.0) 716 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:55:24.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:24 smithi006 bash[33918]: audit 2023-08-27T21:55:23.648997+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:55:24.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:24 smithi006 bash[33918]: audit 2023-08-27T21:55:23.656403+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:24.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:24 smithi006 bash[33918]: audit 2023-08-27T21:55:23.663615+0000 mon.a (mon.0) 719 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:55:24.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:24 smithi006 bash[33918]: audit 2023-08-27T21:55:23.933618+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:24.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:24 smithi006 bash[33918]: audit 2023-08-27T21:55:23.935505+0000 mon.a (mon.0) 721 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:55:24.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:24 smithi006 bash[33918]: audit 2023-08-27T21:55:23.936731+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:55:24.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:24 smithi006 bash[33918]: audit 2023-08-27T21:55:23.943837+0000 mon.a (mon.0) 723 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:24.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:24 smithi006 bash[27292]: audit 2023-08-27T21:55:23.645623+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:24.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:24 smithi006 bash[27292]: audit 2023-08-27T21:55:23.647510+0000 mon.a (mon.0) 716 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:55:24.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:24 smithi006 bash[27292]: audit 2023-08-27T21:55:23.648997+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:55:24.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:24 smithi006 bash[27292]: audit 2023-08-27T21:55:23.656403+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:24.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:24 smithi006 bash[27292]: audit 2023-08-27T21:55:23.663615+0000 mon.a (mon.0) 719 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:55:24.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:24 smithi006 bash[27292]: audit 2023-08-27T21:55:23.933618+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:24.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:24 smithi006 bash[27292]: audit 2023-08-27T21:55:23.935505+0000 mon.a (mon.0) 721 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:55:24.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:24 smithi006 bash[27292]: audit 2023-08-27T21:55:23.936731+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:55:24.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:24 smithi006 bash[27292]: audit 2023-08-27T21:55:23.943837+0000 mon.a (mon.0) 723 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:25.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:24 smithi130 bash[27579]: audit 2023-08-27T21:55:23.645623+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:25.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:24 smithi130 bash[27579]: audit 2023-08-27T21:55:23.647510+0000 mon.a (mon.0) 716 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:55:25.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:24 smithi130 bash[27579]: audit 2023-08-27T21:55:23.648997+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:55:25.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:24 smithi130 bash[27579]: audit 2023-08-27T21:55:23.656403+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:25.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:24 smithi130 bash[27579]: audit 2023-08-27T21:55:23.663615+0000 mon.a (mon.0) 719 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:55:25.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:24 smithi130 bash[27579]: audit 2023-08-27T21:55:23.933618+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:25.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:24 smithi130 bash[27579]: audit 2023-08-27T21:55:23.935505+0000 mon.a (mon.0) 721 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:55:25.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:24 smithi130 bash[27579]: audit 2023-08-27T21:55:23.936731+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:55:25.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:24 smithi130 bash[27579]: audit 2023-08-27T21:55:23.943837+0000 mon.a (mon.0) 723 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:25.840 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:25 smithi006 bash[33918]: cluster 2023-08-27T21:55:24.634002+0000 mgr.y (mgr.14430) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:25.841 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:25 smithi006 bash[27292]: cluster 2023-08-27T21:55:24.634002+0000 mgr.y (mgr.14430) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:26.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:25 smithi130 bash[27579]: cluster 2023-08-27T21:55:24.634002+0000 mgr.y (mgr.14430) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:27.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:55:26 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:55:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:55:27.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:27 smithi006 bash[33918]: cluster 2023-08-27T21:55:26.635191+0000 mgr.y (mgr.14430) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-27T21:55:27.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:27 smithi006 bash[27292]: cluster 2023-08-27T21:55:26.635191+0000 mgr.y (mgr.14430) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-27T21:55:27.950 INFO:teuthology.orchestra.run.smithi006.stdout: 2023-08-27T21:55:28.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:27 smithi130 bash[27579]: cluster 2023-08-27T21:55:26.635191+0000 mgr.y (mgr.14430) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-27T21:55:28.751 INFO:teuthology.orchestra.run.smithi006.stdout:{"epoch":19,"active_gid":14430,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6800","nonce":3377670591},{"type":"v1","addr":"172.21.15.6:6801","nonce":3377670591}]},"active_addr":"172.21.15.6:6801/3377670591","active_change":"2023-08-27T21:52:44.567775+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24349,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.6:8443/","prometheus":"http://172.21.15.6:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.6:0","nonce":4243266477}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.6:0","nonce":647135083}]}]} 2023-08-27T21:55:28.755 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-08-27T21:55:28.756 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-08-27T21:55:28.756 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd dump --format=json 2023-08-27T21:55:28.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:28 smithi006 bash[27292]: audit 2023-08-27T21:55:27.936799+0000 mon.a (mon.0) 724 : audit [DBG] from='client.? 172.21.15.6:0/97301125' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-08-27T21:55:28.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:28 smithi006 bash[33918]: audit 2023-08-27T21:55:27.936799+0000 mon.a (mon.0) 724 : audit [DBG] from='client.? 172.21.15.6:0/97301125' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-08-27T21:55:29.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:28 smithi130 bash[27579]: audit 2023-08-27T21:55:27.936799+0000 mon.a (mon.0) 724 : audit [DBG] from='client.? 172.21.15.6:0/97301125' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-08-27T21:55:29.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:29 smithi006 bash[27292]: cluster 2023-08-27T21:55:28.635857+0000 mgr.y (mgr.14430) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-27T21:55:29.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:29 smithi006 bash[33918]: cluster 2023-08-27T21:55:28.635857+0000 mgr.y (mgr.14430) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-27T21:55:30.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:29 smithi130 bash[27579]: cluster 2023-08-27T21:55:28.635857+0000 mgr.y (mgr.14430) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-27T21:55:31.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:30 smithi130 bash[27579]: audit 2023-08-27T21:55:29.828018+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:31.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:30 smithi130 bash[27579]: audit 2023-08-27T21:55:29.928376+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:31.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:30 smithi130 bash[27579]: cluster 2023-08-27T21:55:30.637038+0000 mgr.y (mgr.14430) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-27T21:55:31.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:55:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:55:30] "GET /metrics HTTP/1.1" 200 191490 "" "Prometheus/2.33.4" 2023-08-27T21:55:31.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:30 smithi006 bash[27292]: audit 2023-08-27T21:55:29.828018+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:31.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:30 smithi006 bash[27292]: audit 2023-08-27T21:55:29.928376+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:31.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:30 smithi006 bash[27292]: cluster 2023-08-27T21:55:30.637038+0000 mgr.y (mgr.14430) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-27T21:55:31.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:30 smithi006 bash[33918]: audit 2023-08-27T21:55:29.828018+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:31.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:30 smithi006 bash[33918]: audit 2023-08-27T21:55:29.928376+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:55:31.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:30 smithi006 bash[33918]: cluster 2023-08-27T21:55:30.637038+0000 mgr.y (mgr.14430) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-27T21:55:33.582 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:55:33.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:33 smithi130 bash[27579]: cluster 2023-08-27T21:55:32.637616+0000 mgr.y (mgr.14430) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-27T21:55:33.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:33 smithi006 bash[27292]: cluster 2023-08-27T21:55:32.637616+0000 mgr.y (mgr.14430) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-27T21:55:33.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:33 smithi006 bash[33918]: cluster 2023-08-27T21:55:32.637616+0000 mgr.y (mgr.14430) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-27T21:55:35.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:35 smithi006 bash[33918]: cluster 2023-08-27T21:55:34.638143+0000 mgr.y (mgr.14430) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-27T21:55:35.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:35 smithi006 bash[27292]: cluster 2023-08-27T21:55:34.638143+0000 mgr.y (mgr.14430) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-27T21:55:36.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:35 smithi130 bash[27579]: cluster 2023-08-27T21:55:34.638143+0000 mgr.y (mgr.14430) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-27T21:55:36.834 INFO:teuthology.orchestra.run.smithi006.stdout: 2023-08-27T21:55:36.834 INFO:teuthology.orchestra.run.smithi006.stdout:{"epoch":60,"fsid":"b696f380-4522-11ee-817b-1b7a5f1ba315","created":"2023-08-27T21:43:51.020683+0000","modified":"2023-08-27T21:55:11.732618+0000","last_up_change":"2023-08-27T21:52:09.451350+0000","last_in_change":"2023-08-27T21:51:42.237602+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-08-27T21:48:22.209678+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-08-27T21:53:15.972053+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-08-27T21:55:08.528709+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-08-27T21:55:09.165422+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"226c7900-24be-48c6-98fa-97893e22d328","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6802","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6803","nonce":224143379}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6804","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6805","nonce":224143379}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6808","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6809","nonce":224143379}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6806","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6807","nonce":224143379}]},"public_addr":"172.21.15.6:6803/224143379","cluster_addr":"172.21.15.6:6805/224143379","heartbeat_back_addr":"172.21.15.6:6809/224143379","heartbeat_front_addr":"172.21.15.6:6807/224143379","state":["exists","up"]},{"osd":1,"uuid":"55ceda65-564a-4b00-89c2-718610d59762","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6810","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6811","nonce":2798606708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6812","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6813","nonce":2798606708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6816","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6817","nonce":2798606708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6814","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6815","nonce":2798606708}]},"public_addr":"172.21.15.6:6811/2798606708","cluster_addr":"172.21.15.6:6813/2798606708","heartbeat_back_addr":"172.21.15.6:6817/2798606708","heartbeat_front_addr":"172.21.15.6:6815/2798606708","state":["exists","up"]},{"osd":2,"uuid":"b027a54f-cf0e-4547-a530-f70e146cb386","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6818","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6819","nonce":3315375049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6820","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6821","nonce":3315375049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6824","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6825","nonce":3315375049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6822","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6823","nonce":3315375049}]},"public_addr":"172.21.15.6:6819/3315375049","cluster_addr":"172.21.15.6:6821/3315375049","heartbeat_back_addr":"172.21.15.6:6825/3315375049","heartbeat_front_addr":"172.21.15.6:6823/3315375049","state":["exists","up"]},{"osd":3,"uuid":"74cd432a-aa78-4538-81bf-a4b209d60f31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6826","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6827","nonce":1657061419}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6828","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6829","nonce":1657061419}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6832","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6833","nonce":1657061419}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6830","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6831","nonce":1657061419}]},"public_addr":"172.21.15.6:6827/1657061419","cluster_addr":"172.21.15.6:6829/1657061419","heartbeat_back_addr":"172.21.15.6:6833/1657061419","heartbeat_front_addr":"172.21.15.6:6831/1657061419","state":["exists","up"]},{"osd":4,"uuid":"5710ffb1-d311-45a8-99bf-d01a7bb65cd1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6801","nonce":576819502}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6803","nonce":576819502}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6807","nonce":576819502}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6805","nonce":576819502}]},"public_addr":"172.21.15.130:6801/576819502","cluster_addr":"172.21.15.130:6803/576819502","heartbeat_back_addr":"172.21.15.130:6807/576819502","heartbeat_front_addr":"172.21.15.130:6805/576819502","state":["exists","up"]},{"osd":5,"uuid":"c0b6bba7-d752-4a95-927b-8a378bcd77b4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6809","nonce":169159537}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6811","nonce":169159537}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6815","nonce":169159537}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6813","nonce":169159537}]},"public_addr":"172.21.15.130:6809/169159537","cluster_addr":"172.21.15.130:6811/169159537","heartbeat_back_addr":"172.21.15.130:6815/169159537","heartbeat_front_addr":"172.21.15.130:6813/169159537","state":["exists","up"]},{"osd":6,"uuid":"fa4c8b52-d88a-427b-bac6-a1abaa0e92e6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6817","nonce":1800436766}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6819","nonce":1800436766}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6823","nonce":1800436766}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6821","nonce":1800436766}]},"public_addr":"172.21.15.130:6817/1800436766","cluster_addr":"172.21.15.130:6819/1800436766","heartbeat_back_addr":"172.21.15.130:6823/1800436766","heartbeat_front_addr":"172.21.15.130:6821/1800436766","state":["exists","up"]},{"osd":7,"uuid":"ac7e358c-d0fd-4b6c-a2b5-7006c5779948","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6825","nonce":2598634202}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6827","nonce":2598634202}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6831","nonce":2598634202}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6829","nonce":2598634202}]},"public_addr":"172.21.15.130:6825/2598634202","cluster_addr":"172.21.15.130:6827/2598634202","heartbeat_back_addr":"172.21.15.130:6831/2598634202","heartbeat_front_addr":"172.21.15.130:6829/2598634202","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:46:47.390747+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:47:33.822605+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:48:19.486341+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:49:05.565702+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:49:49.732445+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:50:35.236868+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:51:21.327320+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:52:07.813491+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.6:0/2156177013":"2023-08-28T21:52:44.567266+0000","172.21.15.6:0/4156113241":"2023-08-28T21:52:44.567266+0000","172.21.15.6:0/626906238":"2023-08-28T21:44:51.111365+0000","172.21.15.6:0/1056882729":"2023-08-28T21:44:51.111365+0000","172.21.15.6:6800/1902722306":"2023-08-28T21:44:51.111365+0000","172.21.15.6:0/3288657189":"2023-08-28T21:44:20.130814+0000","172.21.15.6:6800/3591185797":"2023-08-28T21:44:20.130814+0000","172.21.15.6:6800/2215677068":"2023-08-28T21:52:44.567266+0000","172.21.15.6:6801/1902722306":"2023-08-28T21:44:51.111365+0000","172.21.15.6:6801/3591185797":"2023-08-28T21:44:20.130814+0000","172.21.15.6:6801/2215677068":"2023-08-28T21:52:44.567266+0000","172.21.15.6:0/3143467986":"2023-08-28T21:44:20.130814+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-08-27T21:55:37.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:55:36 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:55:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:55:37.561 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-08-27T21:55:37.561 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd dump --format=json 2023-08-27T21:55:37.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:37 smithi006 bash[33918]: cluster 2023-08-27T21:55:36.638926+0000 mgr.y (mgr.14430) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-27T21:55:37.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:37 smithi006 bash[33918]: audit 2023-08-27T21:55:36.826618+0000 mon.a (mon.0) 727 : audit [DBG] from='client.? 172.21.15.6:0/146677259' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T21:55:37.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:37 smithi006 bash[27292]: cluster 2023-08-27T21:55:36.638926+0000 mgr.y (mgr.14430) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-27T21:55:37.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:37 smithi006 bash[27292]: audit 2023-08-27T21:55:36.826618+0000 mon.a (mon.0) 727 : audit [DBG] from='client.? 172.21.15.6:0/146677259' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T21:55:38.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:37 smithi130 bash[27579]: cluster 2023-08-27T21:55:36.638926+0000 mgr.y (mgr.14430) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-27T21:55:38.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:37 smithi130 bash[27579]: audit 2023-08-27T21:55:36.826618+0000 mon.a (mon.0) 727 : audit [DBG] from='client.? 172.21.15.6:0/146677259' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T21:55:39.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:39 smithi006 bash[27292]: cluster 2023-08-27T21:55:38.639527+0000 mgr.y (mgr.14430) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:39.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:39 smithi006 bash[33918]: cluster 2023-08-27T21:55:38.639527+0000 mgr.y (mgr.14430) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:40.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:39 smithi130 bash[27579]: cluster 2023-08-27T21:55:38.639527+0000 mgr.y (mgr.14430) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:41.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:55:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:55:40] "GET /metrics HTTP/1.1" 200 191490 "" "Prometheus/2.33.4" 2023-08-27T21:55:41.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:41 smithi006 bash[33918]: cluster 2023-08-27T21:55:40.640547+0000 mgr.y (mgr.14430) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:41.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:41 smithi006 bash[27292]: cluster 2023-08-27T21:55:40.640547+0000 mgr.y (mgr.14430) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:42.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:41 smithi130 bash[27579]: cluster 2023-08-27T21:55:40.640547+0000 mgr.y (mgr.14430) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:42.385 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:55:43.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:43 smithi006 bash[27292]: cluster 2023-08-27T21:55:42.641079+0000 mgr.y (mgr.14430) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:43.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:43 smithi006 bash[33918]: cluster 2023-08-27T21:55:42.641079+0000 mgr.y (mgr.14430) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:43.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:43 smithi130 bash[27579]: cluster 2023-08-27T21:55:42.641079+0000 mgr.y (mgr.14430) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:44.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:44 smithi006 bash[33918]: audit 2023-08-27T21:55:44.609003+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-08-27T21:55:44.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:44 smithi006 bash[33918]: audit 2023-08-27T21:55:44.609696+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-08-27T21:55:44.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:44 smithi006 bash[33918]: audit 2023-08-27T21:55:44.610195+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-08-27T21:55:44.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:44 smithi006 bash[27292]: audit 2023-08-27T21:55:44.609003+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-08-27T21:55:44.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:44 smithi006 bash[27292]: audit 2023-08-27T21:55:44.609696+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-08-27T21:55:44.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:44 smithi006 bash[27292]: audit 2023-08-27T21:55:44.610195+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-08-27T21:55:45.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:44 smithi130 bash[27579]: audit 2023-08-27T21:55:44.609003+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-08-27T21:55:45.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:44 smithi130 bash[27579]: audit 2023-08-27T21:55:44.609696+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-08-27T21:55:45.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:44 smithi130 bash[27579]: audit 2023-08-27T21:55:44.610195+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-08-27T21:55:45.808 INFO:teuthology.orchestra.run.smithi006.stdout: 2023-08-27T21:55:45.808 INFO:teuthology.orchestra.run.smithi006.stdout:{"epoch":62,"fsid":"b696f380-4522-11ee-817b-1b7a5f1ba315","created":"2023-08-27T21:43:51.020683+0000","modified":"2023-08-27T21:55:45.670012+0000","last_up_change":"2023-08-27T21:52:09.451350+0000","last_in_change":"2023-08-27T21:51:42.237602+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-08-27T21:48:22.209678+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-08-27T21:53:15.972053+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-08-27T21:55:08.528709+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-08-27T21:55:09.165422+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"226c7900-24be-48c6-98fa-97893e22d328","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6802","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6803","nonce":224143379}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6804","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6805","nonce":224143379}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6808","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6809","nonce":224143379}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6806","nonce":224143379},{"type":"v1","addr":"172.21.15.6:6807","nonce":224143379}]},"public_addr":"172.21.15.6:6803/224143379","cluster_addr":"172.21.15.6:6805/224143379","heartbeat_back_addr":"172.21.15.6:6809/224143379","heartbeat_front_addr":"172.21.15.6:6807/224143379","state":["exists","up"]},{"osd":1,"uuid":"55ceda65-564a-4b00-89c2-718610d59762","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6810","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6811","nonce":2798606708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6812","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6813","nonce":2798606708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6816","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6817","nonce":2798606708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6814","nonce":2798606708},{"type":"v1","addr":"172.21.15.6:6815","nonce":2798606708}]},"public_addr":"172.21.15.6:6811/2798606708","cluster_addr":"172.21.15.6:6813/2798606708","heartbeat_back_addr":"172.21.15.6:6817/2798606708","heartbeat_front_addr":"172.21.15.6:6815/2798606708","state":["exists","up"]},{"osd":2,"uuid":"b027a54f-cf0e-4547-a530-f70e146cb386","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6818","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6819","nonce":3315375049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6820","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6821","nonce":3315375049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6824","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6825","nonce":3315375049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6822","nonce":3315375049},{"type":"v1","addr":"172.21.15.6:6823","nonce":3315375049}]},"public_addr":"172.21.15.6:6819/3315375049","cluster_addr":"172.21.15.6:6821/3315375049","heartbeat_back_addr":"172.21.15.6:6825/3315375049","heartbeat_front_addr":"172.21.15.6:6823/3315375049","state":["exists","up"]},{"osd":3,"uuid":"74cd432a-aa78-4538-81bf-a4b209d60f31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6826","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6827","nonce":1657061419}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6828","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6829","nonce":1657061419}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6832","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6833","nonce":1657061419}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6830","nonce":1657061419},{"type":"v1","addr":"172.21.15.6:6831","nonce":1657061419}]},"public_addr":"172.21.15.6:6827/1657061419","cluster_addr":"172.21.15.6:6829/1657061419","heartbeat_back_addr":"172.21.15.6:6833/1657061419","heartbeat_front_addr":"172.21.15.6:6831/1657061419","state":["exists","up"]},{"osd":4,"uuid":"5710ffb1-d311-45a8-99bf-d01a7bb65cd1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6801","nonce":576819502}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6803","nonce":576819502}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6807","nonce":576819502}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":576819502},{"type":"v1","addr":"172.21.15.130:6805","nonce":576819502}]},"public_addr":"172.21.15.130:6801/576819502","cluster_addr":"172.21.15.130:6803/576819502","heartbeat_back_addr":"172.21.15.130:6807/576819502","heartbeat_front_addr":"172.21.15.130:6805/576819502","state":["exists","up"]},{"osd":5,"uuid":"c0b6bba7-d752-4a95-927b-8a378bcd77b4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6809","nonce":169159537}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6811","nonce":169159537}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6815","nonce":169159537}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":169159537},{"type":"v1","addr":"172.21.15.130:6813","nonce":169159537}]},"public_addr":"172.21.15.130:6809/169159537","cluster_addr":"172.21.15.130:6811/169159537","heartbeat_back_addr":"172.21.15.130:6815/169159537","heartbeat_front_addr":"172.21.15.130:6813/169159537","state":["exists","up"]},{"osd":6,"uuid":"fa4c8b52-d88a-427b-bac6-a1abaa0e92e6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6817","nonce":1800436766}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6819","nonce":1800436766}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6823","nonce":1800436766}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1800436766},{"type":"v1","addr":"172.21.15.130:6821","nonce":1800436766}]},"public_addr":"172.21.15.130:6817/1800436766","cluster_addr":"172.21.15.130:6819/1800436766","heartbeat_back_addr":"172.21.15.130:6823/1800436766","heartbeat_front_addr":"172.21.15.130:6821/1800436766","state":["exists","up"]},{"osd":7,"uuid":"ac7e358c-d0fd-4b6c-a2b5-7006c5779948","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6825","nonce":2598634202}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6827","nonce":2598634202}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6831","nonce":2598634202}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2598634202},{"type":"v1","addr":"172.21.15.130:6829","nonce":2598634202}]},"public_addr":"172.21.15.130:6825/2598634202","cluster_addr":"172.21.15.130:6827/2598634202","heartbeat_back_addr":"172.21.15.130:6831/2598634202","heartbeat_front_addr":"172.21.15.130:6829/2598634202","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:46:47.390747+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:47:33.822605+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:48:19.486341+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:49:05.565702+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:49:49.732445+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:50:35.236868+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:51:21.327320+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T21:52:07.813491+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.12","mappings":[{"from":1,"to":5}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.6:0/2156177013":"2023-08-28T21:52:44.567266+0000","172.21.15.6:0/4156113241":"2023-08-28T21:52:44.567266+0000","172.21.15.6:0/626906238":"2023-08-28T21:44:51.111365+0000","172.21.15.6:0/1056882729":"2023-08-28T21:44:51.111365+0000","172.21.15.6:6800/1902722306":"2023-08-28T21:44:51.111365+0000","172.21.15.6:0/3288657189":"2023-08-28T21:44:20.130814+0000","172.21.15.6:6800/3591185797":"2023-08-28T21:44:20.130814+0000","172.21.15.6:6800/2215677068":"2023-08-28T21:52:44.567266+0000","172.21.15.6:6801/1902722306":"2023-08-28T21:44:51.111365+0000","172.21.15.6:6801/3591185797":"2023-08-28T21:44:20.130814+0000","172.21.15.6:6801/2215677068":"2023-08-28T21:52:44.567266+0000","172.21.15.6:0/3143467986":"2023-08-28T21:44:20.130814+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-08-27T21:55:45.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:45 smithi006 bash[27292]: cluster 2023-08-27T21:55:44.641616+0000 mgr.y (mgr.14430) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:45.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:45 smithi006 bash[27292]: audit 2023-08-27T21:55:44.665505+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-08-27T21:55:45.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:45 smithi006 bash[27292]: audit 2023-08-27T21:55:44.665579+0000 mon.a (mon.0) 732 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-08-27T21:55:45.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:45 smithi006 bash[27292]: audit 2023-08-27T21:55:44.665661+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-08-27T21:55:45.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:45 smithi006 bash[27292]: cluster 2023-08-27T21:55:44.665734+0000 mon.a (mon.0) 734 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-08-27T21:55:45.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:45 smithi006 bash[33918]: cluster 2023-08-27T21:55:44.641616+0000 mgr.y (mgr.14430) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:45.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:45 smithi006 bash[33918]: audit 2023-08-27T21:55:44.665505+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-08-27T21:55:45.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:45 smithi006 bash[33918]: audit 2023-08-27T21:55:44.665579+0000 mon.a (mon.0) 732 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-08-27T21:55:45.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:45 smithi006 bash[33918]: audit 2023-08-27T21:55:44.665661+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-08-27T21:55:45.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:45 smithi006 bash[33918]: cluster 2023-08-27T21:55:44.665734+0000 mon.a (mon.0) 734 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-08-27T21:55:46.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:45 smithi130 bash[27579]: cluster 2023-08-27T21:55:44.641616+0000 mgr.y (mgr.14430) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:46.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:45 smithi130 bash[27579]: audit 2023-08-27T21:55:44.665505+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-08-27T21:55:46.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:45 smithi130 bash[27579]: audit 2023-08-27T21:55:44.665579+0000 mon.a (mon.0) 732 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-08-27T21:55:46.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:45 smithi130 bash[27579]: audit 2023-08-27T21:55:44.665661+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-08-27T21:55:46.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:45 smithi130 bash[27579]: cluster 2023-08-27T21:55:44.665734+0000 mon.a (mon.0) 734 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-08-27T21:55:46.582 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph tell osd.0 flush_pg_stats 2023-08-27T21:55:46.582 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph tell osd.1 flush_pg_stats 2023-08-27T21:55:46.583 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph tell osd.2 flush_pg_stats 2023-08-27T21:55:46.583 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph tell osd.3 flush_pg_stats 2023-08-27T21:55:46.584 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph tell osd.4 flush_pg_stats 2023-08-27T21:55:46.584 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph tell osd.5 flush_pg_stats 2023-08-27T21:55:46.584 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph tell osd.6 flush_pg_stats 2023-08-27T21:55:46.585 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph tell osd.7 flush_pg_stats 2023-08-27T21:55:46.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:46 smithi006 bash[33918]: cluster 2023-08-27T21:55:45.683658+0000 mon.a (mon.0) 735 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-08-27T21:55:46.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:46 smithi006 bash[33918]: audit 2023-08-27T21:55:45.800550+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.6:0/219936778' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T21:55:46.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:46 smithi006 bash[27292]: cluster 2023-08-27T21:55:45.683658+0000 mon.a (mon.0) 735 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-08-27T21:55:46.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:46 smithi006 bash[27292]: audit 2023-08-27T21:55:45.800550+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.6:0/219936778' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T21:55:47.001 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:46 smithi130 bash[27579]: cluster 2023-08-27T21:55:45.683658+0000 mon.a (mon.0) 735 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-08-27T21:55:47.001 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:46 smithi130 bash[27579]: audit 2023-08-27T21:55:45.800550+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.6:0/219936778' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T21:55:47.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:55:46 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:55:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:55:47.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:47 smithi006 bash[33918]: cluster 2023-08-27T21:55:46.642134+0000 mgr.y (mgr.14430) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:47.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:47 smithi006 bash[27292]: cluster 2023-08-27T21:55:46.642134+0000 mgr.y (mgr.14430) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:48.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:47 smithi130 bash[27579]: cluster 2023-08-27T21:55:46.642134+0000 mgr.y (mgr.14430) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:49.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:49 smithi006 bash[27292]: cluster 2023-08-27T21:55:48.642748+0000 mgr.y (mgr.14430) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:49.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:49 smithi006 bash[33918]: cluster 2023-08-27T21:55:48.642748+0000 mgr.y (mgr.14430) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:50.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:49 smithi130 bash[27579]: cluster 2023-08-27T21:55:48.642748+0000 mgr.y (mgr.14430) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:51.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:55:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:55:50] "GET /metrics HTTP/1.1" 200 191483 "" "Prometheus/2.33.4" 2023-08-27T21:55:51.529 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:55:51.530 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:55:51.530 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:55:51.531 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:55:51.533 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:55:51.533 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:55:51.533 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:55:51.537 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:55:51.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:51 smithi006 bash[33918]: cluster 2023-08-27T21:55:50.643688+0000 mgr.y (mgr.14430) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:51.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:51 smithi006 bash[27292]: cluster 2023-08-27T21:55:50.643688+0000 mgr.y (mgr.14430) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:52.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:51 smithi130 bash[27579]: cluster 2023-08-27T21:55:50.643688+0000 mgr.y (mgr.14430) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:53.660 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:53 smithi006 bash[27292]: cluster 2023-08-27T21:55:52.644338+0000 mgr.y (mgr.14430) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:53.660 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:53 smithi006 bash[33918]: cluster 2023-08-27T21:55:52.644338+0000 mgr.y (mgr.14430) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:53.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:53 smithi130 bash[27579]: cluster 2023-08-27T21:55:52.644338+0000 mgr.y (mgr.14430) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:56.050 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:55 smithi006 bash[33918]: cluster 2023-08-27T21:55:54.645103+0000 mgr.y (mgr.14430) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:56.051 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:55 smithi006 bash[27292]: cluster 2023-08-27T21:55:54.645103+0000 mgr.y (mgr.14430) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:56.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:55 smithi130 bash[27579]: cluster 2023-08-27T21:55:54.645103+0000 mgr.y (mgr.14430) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:57.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:55:56 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:55:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:55:57.735 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:57 smithi006 bash[27292]: cluster 2023-08-27T21:55:56.646283+0000 mgr.y (mgr.14430) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:57.736 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:57 smithi006 bash[33918]: cluster 2023-08-27T21:55:56.646283+0000 mgr.y (mgr.14430) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:58.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:57 smithi130 bash[27579]: cluster 2023-08-27T21:55:56.646283+0000 mgr.y (mgr.14430) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:59.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:55:59 smithi006 bash[27292]: cluster 2023-08-27T21:55:58.646894+0000 mgr.y (mgr.14430) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:55:59.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:55:59 smithi006 bash[33918]: cluster 2023-08-27T21:55:58.646894+0000 mgr.y (mgr.14430) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:00.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:55:59 smithi130 bash[27579]: cluster 2023-08-27T21:55:58.646894+0000 mgr.y (mgr.14430) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:00.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:56:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:56:00] "GET /metrics HTTP/1.1" 200 191522 "" "Prometheus/2.33.4" 2023-08-27T21:56:02.057 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:01 smithi006 bash[27292]: cluster 2023-08-27T21:56:00.647686+0000 mgr.y (mgr.14430) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:02.057 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:01 smithi006 bash[33918]: cluster 2023-08-27T21:56:00.647686+0000 mgr.y (mgr.14430) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:02.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:01 smithi130 bash[27579]: cluster 2023-08-27T21:56:00.647686+0000 mgr.y (mgr.14430) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:03.847 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:03 smithi006 bash[33918]: cluster 2023-08-27T21:56:02.648064+0000 mgr.y (mgr.14430) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:03.847 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:03 smithi006 bash[27292]: cluster 2023-08-27T21:56:02.648064+0000 mgr.y (mgr.14430) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:03.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:03 smithi130 bash[27579]: cluster 2023-08-27T21:56:02.648064+0000 mgr.y (mgr.14430) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:04.824 INFO:teuthology.orchestra.run.smithi006.stdout:55834574952 2023-08-27T21:56:04.825 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.1 2023-08-27T21:56:05.317 INFO:teuthology.orchestra.run.smithi006.stdout:180388626491 2023-08-27T21:56:05.318 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.6 2023-08-27T21:56:05.730 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:05 smithi006 bash[27292]: cluster 2023-08-27T21:56:04.648490+0000 mgr.y (mgr.14430) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:05.730 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:05 smithi006 bash[33918]: cluster 2023-08-27T21:56:04.648490+0000 mgr.y (mgr.14430) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:06.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:05 smithi130 bash[27579]: cluster 2023-08-27T21:56:04.648490+0000 mgr.y (mgr.14430) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:07.195 INFO:teuthology.orchestra.run.smithi006.stdout:206158430257 2023-08-27T21:56:07.196 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.7 2023-08-27T21:56:07.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:56:06 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:56:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:56:07.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:07 smithi006 bash[27292]: cluster 2023-08-27T21:56:06.649625+0000 mgr.y (mgr.14430) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:07.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:07 smithi006 bash[33918]: cluster 2023-08-27T21:56:06.649625+0000 mgr.y (mgr.14430) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:08.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:07 smithi130 bash[27579]: cluster 2023-08-27T21:56:06.649625+0000 mgr.y (mgr.14430) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:08.336 INFO:teuthology.orchestra.run.smithi006.stdout:107374182485 2023-08-27T21:56:08.336 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.3 2023-08-27T21:56:08.472 INFO:teuthology.orchestra.run.smithi006.stdout:128849018958 2023-08-27T21:56:08.472 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.4 2023-08-27T21:56:09.315 INFO:teuthology.orchestra.run.smithi006.stdout:154618822725 2023-08-27T21:56:09.316 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.5 2023-08-27T21:56:09.364 INFO:teuthology.orchestra.run.smithi006.stdout:34359738481 2023-08-27T21:56:09.365 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.0 2023-08-27T21:56:09.434 INFO:teuthology.orchestra.run.smithi006.stdout:77309411423 2023-08-27T21:56:09.435 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.2 2023-08-27T21:56:09.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:09 smithi006 bash[33918]: cluster 2023-08-27T21:56:08.650152+0000 mgr.y (mgr.14430) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:10.145 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:09 smithi006 bash[27292]: cluster 2023-08-27T21:56:08.650152+0000 mgr.y (mgr.14430) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:10.146 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:09 smithi130 bash[27579]: cluster 2023-08-27T21:56:08.650152+0000 mgr.y (mgr.14430) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:11.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:56:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:56:10] "GET /metrics HTTP/1.1" 200 191522 "" "Prometheus/2.33.4" 2023-08-27T21:56:11.903 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:56:11.904 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:56:12.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:11 smithi130 bash[27579]: cluster 2023-08-27T21:56:10.651170+0000 mgr.y (mgr.14430) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:12.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:11 smithi006 bash[33918]: cluster 2023-08-27T21:56:10.651170+0000 mgr.y (mgr.14430) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:12.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:11 smithi006 bash[27292]: cluster 2023-08-27T21:56:10.651170+0000 mgr.y (mgr.14430) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:12.924 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:56:12.925 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:56:12.925 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:56:13.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:13 smithi130 bash[27579]: cluster 2023-08-27T21:56:12.651799+0000 mgr.y (mgr.14430) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:13.946 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:56:13.948 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:56:13.949 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:56:13.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:13 smithi006 bash[33918]: cluster 2023-08-27T21:56:12.651799+0000 mgr.y (mgr.14430) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:13.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:13 smithi006 bash[27292]: cluster 2023-08-27T21:56:12.651799+0000 mgr.y (mgr.14430) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:15.748 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:15 smithi006 bash[33918]: cluster 2023-08-27T21:56:14.652474+0000 mgr.y (mgr.14430) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:15.748 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:15 smithi006 bash[27292]: cluster 2023-08-27T21:56:14.652474+0000 mgr.y (mgr.14430) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:16.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:15 smithi130 bash[27579]: cluster 2023-08-27T21:56:14.652474+0000 mgr.y (mgr.14430) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:17.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:56:16 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:56:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:56:17.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:17 smithi006 bash[27292]: cluster 2023-08-27T21:56:16.653727+0000 mgr.y (mgr.14430) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:17.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:17 smithi006 bash[33918]: cluster 2023-08-27T21:56:16.653727+0000 mgr.y (mgr.14430) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:18.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:17 smithi130 bash[27579]: cluster 2023-08-27T21:56:16.653727+0000 mgr.y (mgr.14430) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:19.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:19 smithi006 bash[33918]: cluster 2023-08-27T21:56:18.654402+0000 mgr.y (mgr.14430) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:19.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:19 smithi006 bash[27292]: cluster 2023-08-27T21:56:18.654402+0000 mgr.y (mgr.14430) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:20.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:19 smithi130 bash[27579]: cluster 2023-08-27T21:56:18.654402+0000 mgr.y (mgr.14430) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:56:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:56:20] "GET /metrics HTTP/1.1" 200 191532 "" "Prometheus/2.33.4" 2023-08-27T21:56:21.858 INFO:teuthology.orchestra.run.smithi006.stdout:55834574956 2023-08-27T21:56:22.111 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:21 smithi006 bash[27292]: cluster 2023-08-27T21:56:20.655479+0000 mgr.y (mgr.14430) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:22.111 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:21 smithi006 bash[33918]: cluster 2023-08-27T21:56:20.655479+0000 mgr.y (mgr.14430) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:22.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:21 smithi130 bash[27579]: cluster 2023-08-27T21:56:20.655479+0000 mgr.y (mgr.14430) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:22.986 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:22 smithi006 bash[33918]: audit 2023-08-27T21:56:21.853147+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.6:0/730264486' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-08-27T21:56:22.987 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:22 smithi006 bash[27292]: audit 2023-08-27T21:56:21.853147+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.6:0/730264486' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-08-27T21:56:23.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:22 smithi130 bash[27579]: audit 2023-08-27T21:56:21.853147+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.6:0/730264486' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-08-27T21:56:24.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:23 smithi130 bash[27579]: cluster 2023-08-27T21:56:22.656115+0000 mgr.y (mgr.14430) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:24.253 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:23 smithi006 bash[27292]: cluster 2023-08-27T21:56:22.656115+0000 mgr.y (mgr.14430) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:24.254 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:23 smithi006 bash[33918]: cluster 2023-08-27T21:56:22.656115+0000 mgr.y (mgr.14430) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:24.956 INFO:teuthology.orchestra.run.smithi006.stdout:180388626495 2023-08-27T21:56:25.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:24 smithi130 bash[27579]: audit 2023-08-27T21:56:23.951330+0000 mon.a (mon.0) 736 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:56:25.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:24 smithi130 bash[27579]: audit 2023-08-27T21:56:24.239762+0000 mon.a (mon.0) 737 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:56:25.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:24 smithi130 bash[27579]: audit 2023-08-27T21:56:24.240776+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:56:25.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:24 smithi130 bash[27579]: audit 2023-08-27T21:56:24.247494+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:56:25.157 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:24 smithi006 bash[33918]: audit 2023-08-27T21:56:23.951330+0000 mon.a (mon.0) 736 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:56:25.158 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:24 smithi006 bash[33918]: audit 2023-08-27T21:56:24.239762+0000 mon.a (mon.0) 737 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:56:25.158 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:24 smithi006 bash[33918]: audit 2023-08-27T21:56:24.240776+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:56:25.158 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:24 smithi006 bash[33918]: audit 2023-08-27T21:56:24.247494+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:56:25.158 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:24 smithi006 bash[27292]: audit 2023-08-27T21:56:23.951330+0000 mon.a (mon.0) 736 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:56:25.158 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:24 smithi006 bash[27292]: audit 2023-08-27T21:56:24.239762+0000 mon.a (mon.0) 737 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:56:25.158 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:24 smithi006 bash[27292]: audit 2023-08-27T21:56:24.240776+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:56:25.158 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:24 smithi006 bash[27292]: audit 2023-08-27T21:56:24.247494+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:56:26.050 INFO:teuthology.orchestra.run.smithi006.stdout:206158430261 2023-08-27T21:56:26.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:25 smithi130 bash[27579]: cluster 2023-08-27T21:56:24.656602+0000 mgr.y (mgr.14430) 143 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:26.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:25 smithi130 bash[27579]: audit 2023-08-27T21:56:24.951815+0000 mon.a (mon.0) 740 : audit [DBG] from='client.? 172.21.15.6:0/1057004499' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-08-27T21:56:26.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:25 smithi006 bash[27292]: cluster 2023-08-27T21:56:24.656602+0000 mgr.y (mgr.14430) 143 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:26.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:25 smithi006 bash[27292]: audit 2023-08-27T21:56:24.951815+0000 mon.a (mon.0) 740 : audit [DBG] from='client.? 172.21.15.6:0/1057004499' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-08-27T21:56:26.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:25 smithi006 bash[33918]: cluster 2023-08-27T21:56:24.656602+0000 mgr.y (mgr.14430) 143 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:26.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:25 smithi006 bash[33918]: audit 2023-08-27T21:56:24.951815+0000 mon.a (mon.0) 740 : audit [DBG] from='client.? 172.21.15.6:0/1057004499' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-08-27T21:56:26.569 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574952 got 55834574956 for osd.1 2023-08-27T21:56:26.570 DEBUG:teuthology.parallel:result is None 2023-08-27T21:56:26.894 INFO:teuthology.orchestra.run.smithi006.stdout:128849018962 2023-08-27T21:56:26.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:26 smithi006 bash[27292]: audit 2023-08-27T21:56:26.044994+0000 mon.a (mon.0) 741 : audit [DBG] from='client.? 172.21.15.6:0/686963811' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-08-27T21:56:26.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:26 smithi006 bash[33918]: audit 2023-08-27T21:56:26.044994+0000 mon.a (mon.0) 741 : audit [DBG] from='client.? 172.21.15.6:0/686963811' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-08-27T21:56:27.001 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:26 smithi130 bash[27579]: audit 2023-08-27T21:56:26.044994+0000 mon.a (mon.0) 741 : audit [DBG] from='client.? 172.21.15.6:0/686963811' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-08-27T21:56:27.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:56:26 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:56:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:56:28.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:27 smithi130 bash[27579]: cluster 2023-08-27T21:56:26.657166+0000 mgr.y (mgr.14430) 144 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:28.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:27 smithi130 bash[27579]: audit 2023-08-27T21:56:26.888945+0000 mon.a (mon.0) 742 : audit [DBG] from='client.? 172.21.15.6:0/2300067252' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-08-27T21:56:28.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:27 smithi006 bash[27292]: cluster 2023-08-27T21:56:26.657166+0000 mgr.y (mgr.14430) 144 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:28.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:27 smithi006 bash[27292]: audit 2023-08-27T21:56:26.888945+0000 mon.a (mon.0) 742 : audit [DBG] from='client.? 172.21.15.6:0/2300067252' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-08-27T21:56:28.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:27 smithi006 bash[33918]: cluster 2023-08-27T21:56:26.657166+0000 mgr.y (mgr.14430) 144 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:28.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:27 smithi006 bash[33918]: audit 2023-08-27T21:56:26.888945+0000 mon.a (mon.0) 742 : audit [DBG] from='client.? 172.21.15.6:0/2300067252' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-08-27T21:56:28.343 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626491 got 180388626495 for osd.6 2023-08-27T21:56:28.343 DEBUG:teuthology.parallel:result is None 2023-08-27T21:56:28.831 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430257 got 206158430261 for osd.7 2023-08-27T21:56:28.831 DEBUG:teuthology.parallel:result is None 2023-08-27T21:56:29.375 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018958 got 128849018962 for osd.4 2023-08-27T21:56:29.375 DEBUG:teuthology.parallel:result is None 2023-08-27T21:56:29.824 INFO:teuthology.orchestra.run.smithi006.stdout:107374182490 2023-08-27T21:56:30.083 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:29 smithi006 bash[27292]: cluster 2023-08-27T21:56:28.657706+0000 mgr.y (mgr.14430) 145 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:30.083 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:29 smithi006 bash[33918]: cluster 2023-08-27T21:56:28.657706+0000 mgr.y (mgr.14430) 145 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:30.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:29 smithi130 bash[27579]: cluster 2023-08-27T21:56:28.657706+0000 mgr.y (mgr.14430) 145 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:30.199 INFO:teuthology.orchestra.run.smithi006.stdout:77309411427 2023-08-27T21:56:30.352 INFO:teuthology.orchestra.run.smithi006.stdout:154618822729 2023-08-27T21:56:30.401 INFO:teuthology.orchestra.run.smithi006.stdout:34359738485 2023-08-27T21:56:30.890 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182485 got 107374182490 for osd.3 2023-08-27T21:56:30.890 DEBUG:teuthology.parallel:result is None 2023-08-27T21:56:31.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:30 smithi130 bash[27579]: audit 2023-08-27T21:56:29.819056+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.6:0/2394978552' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-08-27T21:56:31.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:30 smithi130 bash[27579]: audit 2023-08-27T21:56:30.193899+0000 mon.a (mon.0) 743 : audit [DBG] from='client.? 172.21.15.6:0/3708515891' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-08-27T21:56:31.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:30 smithi130 bash[27579]: audit 2023-08-27T21:56:30.347458+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.6:0/3859432562' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-08-27T21:56:31.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:30 smithi130 bash[27579]: audit 2023-08-27T21:56:30.395944+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.6:0/2079948340' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-08-27T21:56:31.142 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:30 smithi006 bash[27292]: audit 2023-08-27T21:56:29.819056+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.6:0/2394978552' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-08-27T21:56:31.142 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:30 smithi006 bash[27292]: audit 2023-08-27T21:56:30.193899+0000 mon.a (mon.0) 743 : audit [DBG] from='client.? 172.21.15.6:0/3708515891' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-08-27T21:56:31.142 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:30 smithi006 bash[27292]: audit 2023-08-27T21:56:30.347458+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.6:0/3859432562' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-08-27T21:56:31.142 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:30 smithi006 bash[27292]: audit 2023-08-27T21:56:30.395944+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.6:0/2079948340' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-08-27T21:56:31.142 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:56:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:56:30] "GET /metrics HTTP/1.1" 200 191530 "" "Prometheus/2.33.4" 2023-08-27T21:56:31.143 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:30 smithi006 bash[33918]: audit 2023-08-27T21:56:29.819056+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.6:0/2394978552' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-08-27T21:56:31.143 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:30 smithi006 bash[33918]: audit 2023-08-27T21:56:30.193899+0000 mon.a (mon.0) 743 : audit [DBG] from='client.? 172.21.15.6:0/3708515891' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-08-27T21:56:31.143 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:30 smithi006 bash[33918]: audit 2023-08-27T21:56:30.347458+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.6:0/3859432562' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-08-27T21:56:31.143 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:30 smithi006 bash[33918]: audit 2023-08-27T21:56:30.395944+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.6:0/2079948340' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-08-27T21:56:32.055 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:31 smithi006 bash[27292]: cluster 2023-08-27T21:56:30.658682+0000 mgr.y (mgr.14430) 146 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:32.056 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:31 smithi006 bash[33918]: cluster 2023-08-27T21:56:30.658682+0000 mgr.y (mgr.14430) 146 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:32.056 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822725 got 154618822729 for osd.5 2023-08-27T21:56:32.057 DEBUG:teuthology.parallel:result is None 2023-08-27T21:56:32.070 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738481 got 34359738485 for osd.0 2023-08-27T21:56:32.070 DEBUG:teuthology.parallel:result is None 2023-08-27T21:56:32.091 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411423 got 77309411427 for osd.2 2023-08-27T21:56:32.092 DEBUG:teuthology.parallel:result is None 2023-08-27T21:56:32.092 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-08-27T21:56:32.092 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-27T21:56:32.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:31 smithi130 bash[27579]: cluster 2023-08-27T21:56:30.658682+0000 mgr.y (mgr.14430) 146 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:33.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:33 smithi130 bash[27579]: cluster 2023-08-27T21:56:32.659328+0000 mgr.y (mgr.14430) 147 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:33.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:33 smithi006 bash[27292]: cluster 2023-08-27T21:56:32.659328+0000 mgr.y (mgr.14430) 147 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:33.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:33 smithi006 bash[33918]: cluster 2023-08-27T21:56:32.659328+0000 mgr.y (mgr.14430) 147 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:36.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:35 smithi130 bash[27579]: cluster 2023-08-27T21:56:34.659986+0000 mgr.y (mgr.14430) 148 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:36.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:35 smithi006 bash[33918]: cluster 2023-08-27T21:56:34.659986+0000 mgr.y (mgr.14430) 148 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:36.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:35 smithi006 bash[27292]: cluster 2023-08-27T21:56:34.659986+0000 mgr.y (mgr.14430) 148 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:36.960 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:56:37.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:56:36 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:56:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:56:38.002 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:37 smithi006 bash[33918]: cluster 2023-08-27T21:56:36.661244+0000 mgr.y (mgr.14430) 149 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:38.002 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:37 smithi006 bash[27292]: cluster 2023-08-27T21:56:36.661244+0000 mgr.y (mgr.14430) 149 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:38.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:37 smithi130 bash[27579]: cluster 2023-08-27T21:56:36.661244+0000 mgr.y (mgr.14430) 149 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:39.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:39 smithi006 bash[33918]: cluster 2023-08-27T21:56:38.661951+0000 mgr.y (mgr.14430) 150 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:39.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:39 smithi006 bash[27292]: cluster 2023-08-27T21:56:38.661951+0000 mgr.y (mgr.14430) 150 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:40.027 INFO:teuthology.orchestra.run.smithi006.stdout: 2023-08-27T21:56:40.028 INFO:teuthology.orchestra.run.smithi006.stderr:dumped all 2023-08-27T21:56:40.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:39 smithi130 bash[27579]: cluster 2023-08-27T21:56:38.661951+0000 mgr.y (mgr.14430) 150 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:40.669 INFO:teuthology.orchestra.run.smithi006.stdout:{"pg_ready":true,"pg_map":{"version":131,"stamp":"2023-08-27T21:56:38.661597+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381736,"kb_used_data":4968,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579480,"statfs":{"total":767960285184,"available":765521387520,"internally_reserved":0,"allocated":5087232,"data_stored":2164612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004804"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675155+0000","last_change":"2023-08-27T21:55:10.887197+0000","last_active":"2023-08-27T21:55:44.675155+0000","last_peered":"2023-08-27T21:55:44.675155+0000","last_clean":"2023-08-27T21:55:44.675155+0000","last_became_active":"2023-08-27T21:55:10.886486+0000","last_became_peered":"2023-08-27T21:55:10.886486+0000","last_unstale":"2023-08-27T21:55:44.675155+0000","last_undegraded":"2023-08-27T21:55:44.675155+0000","last_fullsized":"2023-08-27T21:55:44.675155+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.664184+0000","last_change":"2023-08-27T21:55:09.729805+0000","last_active":"2023-08-27T21:55:44.664184+0000","last_peered":"2023-08-27T21:55:44.664184+0000","last_clean":"2023-08-27T21:55:44.664184+0000","last_became_active":"2023-08-27T21:55:09.729492+0000","last_became_peered":"2023-08-27T21:55:09.729492+0000","last_unstale":"2023-08-27T21:55:44.664184+0000","last_undegraded":"2023-08-27T21:55:44.664184+0000","last_fullsized":"2023-08-27T21:55:44.664184+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676124+0000","last_change":"2023-08-27T21:55:09.751511+0000","last_active":"2023-08-27T21:55:44.676124+0000","last_peered":"2023-08-27T21:55:44.676124+0000","last_clean":"2023-08-27T21:55:44.676124+0000","last_became_active":"2023-08-27T21:55:09.751175+0000","last_became_peered":"2023-08-27T21:55:09.751175+0000","last_unstale":"2023-08-27T21:55:44.676124+0000","last_undegraded":"2023-08-27T21:55:44.676124+0000","last_fullsized":"2023-08-27T21:55:44.676124+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676575+0000","last_change":"2023-08-27T21:55:09.725081+0000","last_active":"2023-08-27T21:55:44.676575+0000","last_peered":"2023-08-27T21:55:44.676575+0000","last_clean":"2023-08-27T21:55:44.676575+0000","last_became_active":"2023-08-27T21:55:09.724636+0000","last_became_peered":"2023-08-27T21:55:09.724636+0000","last_unstale":"2023-08-27T21:55:44.676575+0000","last_undegraded":"2023-08-27T21:55:44.676575+0000","last_fullsized":"2023-08-27T21:55:44.676575+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675495+0000","last_change":"2023-08-27T21:55:10.753222+0000","last_active":"2023-08-27T21:55:44.675495+0000","last_peered":"2023-08-27T21:55:44.675495+0000","last_clean":"2023-08-27T21:55:44.675495+0000","last_became_active":"2023-08-27T21:55:10.752892+0000","last_became_peered":"2023-08-27T21:55:10.752892+0000","last_unstale":"2023-08-27T21:55:44.675495+0000","last_undegraded":"2023-08-27T21:55:44.675495+0000","last_fullsized":"2023-08-27T21:55:44.675495+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.667135+0000","last_change":"2023-08-27T21:55:09.737892+0000","last_active":"2023-08-27T21:55:44.667135+0000","last_peered":"2023-08-27T21:55:44.667135+0000","last_clean":"2023-08-27T21:55:44.667135+0000","last_became_active":"2023-08-27T21:55:09.737576+0000","last_became_peered":"2023-08-27T21:55:09.737576+0000","last_unstale":"2023-08-27T21:55:44.667135+0000","last_undegraded":"2023-08-27T21:55:44.667135+0000","last_fullsized":"2023-08-27T21:55:44.667135+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-08-27T21:55:45.731265+0000","last_change":"2023-08-27T21:55:45.731265+0000","last_active":"2023-08-27T21:55:45.731265+0000","last_peered":"2023-08-27T21:55:45.731265+0000","last_clean":"2023-08-27T21:55:45.731265+0000","last_became_active":"2023-08-27T21:55:45.730248+0000","last_became_peered":"2023-08-27T21:55:45.730248+0000","last_unstale":"2023-08-27T21:55:45.731265+0000","last_undegraded":"2023-08-27T21:55:45.731265+0000","last_fullsized":"2023-08-27T21:55:45.731265+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.666985+0000","last_change":"2023-08-27T21:55:09.744050+0000","last_active":"2023-08-27T21:55:44.666985+0000","last_peered":"2023-08-27T21:55:44.666985+0000","last_clean":"2023-08-27T21:55:44.666985+0000","last_became_active":"2023-08-27T21:55:09.743737+0000","last_became_peered":"2023-08-27T21:55:09.743737+0000","last_unstale":"2023-08-27T21:55:44.666985+0000","last_undegraded":"2023-08-27T21:55:44.666985+0000","last_fullsized":"2023-08-27T21:55:44.666985+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.664476+0000","last_change":"2023-08-27T21:55:09.745987+0000","last_active":"2023-08-27T21:55:44.664476+0000","last_peered":"2023-08-27T21:55:44.664476+0000","last_clean":"2023-08-27T21:55:44.664476+0000","last_became_active":"2023-08-27T21:55:09.744651+0000","last_became_peered":"2023-08-27T21:55:09.744651+0000","last_unstale":"2023-08-27T21:55:44.664476+0000","last_undegraded":"2023-08-27T21:55:44.664476+0000","last_fullsized":"2023-08-27T21:55:44.664476+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.664248+0000","last_change":"2023-08-27T21:55:09.745955+0000","last_active":"2023-08-27T21:55:44.664248+0000","last_peered":"2023-08-27T21:55:44.664248+0000","last_clean":"2023-08-27T21:55:44.664248+0000","last_became_active":"2023-08-27T21:55:09.745552+0000","last_became_peered":"2023-08-27T21:55:09.745552+0000","last_unstale":"2023-08-27T21:55:44.664248+0000","last_undegraded":"2023-08-27T21:55:44.664248+0000","last_fullsized":"2023-08-27T21:55:44.664248+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675558+0000","last_change":"2023-08-27T21:55:10.751504+0000","last_active":"2023-08-27T21:55:44.675558+0000","last_peered":"2023-08-27T21:55:44.675558+0000","last_clean":"2023-08-27T21:55:44.675558+0000","last_became_active":"2023-08-27T21:55:10.751164+0000","last_became_peered":"2023-08-27T21:55:10.751164+0000","last_unstale":"2023-08-27T21:55:44.675558+0000","last_undegraded":"2023-08-27T21:55:44.675558+0000","last_fullsized":"2023-08-27T21:55:44.675558+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.668799+0000","last_change":"2023-08-27T21:55:09.738206+0000","last_active":"2023-08-27T21:55:44.668799+0000","last_peered":"2023-08-27T21:55:44.668799+0000","last_clean":"2023-08-27T21:55:44.668799+0000","last_became_active":"2023-08-27T21:55:09.737859+0000","last_became_peered":"2023-08-27T21:55:09.737859+0000","last_unstale":"2023-08-27T21:55:44.668799+0000","last_undegraded":"2023-08-27T21:55:44.668799+0000","last_fullsized":"2023-08-27T21:55:44.668799+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.671207+0000","last_change":"2023-08-27T21:53:17.043470+0000","last_active":"2023-08-27T21:55:44.671207+0000","last_peered":"2023-08-27T21:55:44.671207+0000","last_clean":"2023-08-27T21:55:44.671207+0000","last_became_active":"2023-08-27T21:53:17.042696+0000","last_became_peered":"2023-08-27T21:53:17.042696+0000","last_unstale":"2023-08-27T21:55:44.671207+0000","last_undegraded":"2023-08-27T21:55:44.671207+0000","last_fullsized":"2023-08-27T21:55:44.671207+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675899+0000","last_change":"2023-08-27T21:55:10.193189+0000","last_active":"2023-08-27T21:55:44.675899+0000","last_peered":"2023-08-27T21:55:44.675899+0000","last_clean":"2023-08-27T21:55:44.675899+0000","last_became_active":"2023-08-27T21:55:10.192728+0000","last_became_peered":"2023-08-27T21:55:10.192728+0000","last_unstale":"2023-08-27T21:55:44.675899+0000","last_undegraded":"2023-08-27T21:55:44.675899+0000","last_fullsized":"2023-08-27T21:55:44.675899+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.672807+0000","last_change":"2023-08-27T21:53:17.187485+0000","last_active":"2023-08-27T21:55:44.672807+0000","last_peered":"2023-08-27T21:55:44.672807+0000","last_clean":"2023-08-27T21:55:44.672807+0000","last_became_active":"2023-08-27T21:53:17.187094+0000","last_became_peered":"2023-08-27T21:53:17.187094+0000","last_unstale":"2023-08-27T21:55:44.672807+0000","last_undegraded":"2023-08-27T21:55:44.672807+0000","last_fullsized":"2023-08-27T21:55:44.672807+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676664+0000","last_change":"2023-08-27T21:55:09.727972+0000","last_active":"2023-08-27T21:55:44.676664+0000","last_peered":"2023-08-27T21:55:44.676664+0000","last_clean":"2023-08-27T21:55:44.676664+0000","last_became_active":"2023-08-27T21:55:09.727636+0000","last_became_peered":"2023-08-27T21:55:09.727636+0000","last_unstale":"2023-08-27T21:55:44.676664+0000","last_undegraded":"2023-08-27T21:55:44.676664+0000","last_fullsized":"2023-08-27T21:55:44.676664+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.664229+0000","last_change":"2023-08-27T21:55:10.886864+0000","last_active":"2023-08-27T21:55:44.664229+0000","last_peered":"2023-08-27T21:55:44.664229+0000","last_clean":"2023-08-27T21:55:44.664229+0000","last_became_active":"2023-08-27T21:55:10.886272+0000","last_became_peered":"2023-08-27T21:55:10.886272+0000","last_unstale":"2023-08-27T21:55:44.664229+0000","last_undegraded":"2023-08-27T21:55:44.664229+0000","last_fullsized":"2023-08-27T21:55:44.664229+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.671429+0000","last_change":"2023-08-27T21:55:09.736041+0000","last_active":"2023-08-27T21:55:44.671429+0000","last_peered":"2023-08-27T21:55:44.671429+0000","last_clean":"2023-08-27T21:55:44.671429+0000","last_became_active":"2023-08-27T21:55:09.735653+0000","last_became_peered":"2023-08-27T21:55:09.735653+0000","last_unstale":"2023-08-27T21:55:44.671429+0000","last_undegraded":"2023-08-27T21:55:44.671429+0000","last_fullsized":"2023-08-27T21:55:44.671429+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.729582+0000","last_change":"2023-08-27T21:53:17.816851+0000","last_active":"2023-08-27T21:55:44.729582+0000","last_peered":"2023-08-27T21:55:44.729582+0000","last_clean":"2023-08-27T21:55:44.729582+0000","last_became_active":"2023-08-27T21:53:17.815511+0000","last_became_peered":"2023-08-27T21:53:17.815511+0000","last_unstale":"2023-08-27T21:55:44.729582+0000","last_undegraded":"2023-08-27T21:55:44.729582+0000","last_fullsized":"2023-08-27T21:55:44.729582+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-08-27T21:55:45.686659+0000","last_change":"2023-08-27T21:55:45.686659+0000","last_active":"2023-08-27T21:55:45.686659+0000","last_peered":"2023-08-27T21:55:45.686659+0000","last_clean":"2023-08-27T21:55:45.686659+0000","last_became_active":"2023-08-27T21:55:45.686225+0000","last_became_peered":"2023-08-27T21:55:45.686225+0000","last_unstale":"2023-08-27T21:55:45.686659+0000","last_undegraded":"2023-08-27T21:55:45.686659+0000","last_fullsized":"2023-08-27T21:55:45.686659+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.663847+0000","last_change":"2023-08-27T21:55:09.745846+0000","last_active":"2023-08-27T21:55:44.663847+0000","last_peered":"2023-08-27T21:55:44.663847+0000","last_clean":"2023-08-27T21:55:44.663847+0000","last_became_active":"2023-08-27T21:55:09.745486+0000","last_became_peered":"2023-08-27T21:55:09.745486+0000","last_unstale":"2023-08-27T21:55:44.663847+0000","last_undegraded":"2023-08-27T21:55:44.663847+0000","last_fullsized":"2023-08-27T21:55:44.663847+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.729272+0000","last_change":"2023-08-27T21:53:17.815961+0000","last_active":"2023-08-27T21:55:44.729272+0000","last_peered":"2023-08-27T21:55:44.729272+0000","last_clean":"2023-08-27T21:55:44.729272+0000","last_became_active":"2023-08-27T21:53:17.815034+0000","last_became_peered":"2023-08-27T21:53:17.815034+0000","last_unstale":"2023-08-27T21:55:44.729272+0000","last_undegraded":"2023-08-27T21:55:44.729272+0000","last_fullsized":"2023-08-27T21:55:44.729272+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.670832+0000","last_change":"2023-08-27T21:55:10.891543+0000","last_active":"2023-08-27T21:55:44.670832+0000","last_peered":"2023-08-27T21:55:44.670832+0000","last_clean":"2023-08-27T21:55:44.670832+0000","last_became_active":"2023-08-27T21:55:10.891155+0000","last_became_peered":"2023-08-27T21:55:10.891155+0000","last_unstale":"2023-08-27T21:55:44.670832+0000","last_undegraded":"2023-08-27T21:55:44.670832+0000","last_fullsized":"2023-08-27T21:55:44.670832+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676207+0000","last_change":"2023-08-27T21:55:09.728432+0000","last_active":"2023-08-27T21:55:44.676207+0000","last_peered":"2023-08-27T21:55:44.676207+0000","last_clean":"2023-08-27T21:55:44.676207+0000","last_became_active":"2023-08-27T21:55:09.728135+0000","last_became_peered":"2023-08-27T21:55:09.728135+0000","last_unstale":"2023-08-27T21:55:44.676207+0000","last_undegraded":"2023-08-27T21:55:44.676207+0000","last_fullsized":"2023-08-27T21:55:44.676207+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.729052+0000","last_change":"2023-08-27T21:52:11.532578+0000","last_active":"2023-08-27T21:55:44.729052+0000","last_peered":"2023-08-27T21:55:44.729052+0000","last_clean":"2023-08-27T21:55:44.729052+0000","last_became_active":"2023-08-27T21:52:11.531739+0000","last_became_peered":"2023-08-27T21:52:11.531739+0000","last_unstale":"2023-08-27T21:55:44.729052+0000","last_undegraded":"2023-08-27T21:55:44.729052+0000","last_fullsized":"2023-08-27T21:55:44.729052+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:48:22.416466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:48:22.416466+0000","last_clean_scrub_stamp":"2023-08-27T21:48:22.416466+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.666628+0000","last_change":"2023-08-27T21:53:17.043700+0000","last_active":"2023-08-27T21:55:44.666628+0000","last_peered":"2023-08-27T21:55:44.666628+0000","last_clean":"2023-08-27T21:55:44.666628+0000","last_became_active":"2023-08-27T21:53:17.043332+0000","last_became_peered":"2023-08-27T21:53:17.043332+0000","last_unstale":"2023-08-27T21:55:44.666628+0000","last_undegraded":"2023-08-27T21:55:44.666628+0000","last_fullsized":"2023-08-27T21:55:44.666628+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.672796+0000","last_change":"2023-08-27T21:55:10.172334+0000","last_active":"2023-08-27T21:55:44.672796+0000","last_peered":"2023-08-27T21:55:44.672796+0000","last_clean":"2023-08-27T21:55:44.672796+0000","last_became_active":"2023-08-27T21:55:10.171993+0000","last_became_peered":"2023-08-27T21:55:10.171993+0000","last_unstale":"2023-08-27T21:55:44.672796+0000","last_undegraded":"2023-08-27T21:55:44.672796+0000","last_fullsized":"2023-08-27T21:55:44.672796+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.670878+0000","last_change":"2023-08-27T21:55:09.735816+0000","last_active":"2023-08-27T21:55:44.670878+0000","last_peered":"2023-08-27T21:55:44.670878+0000","last_clean":"2023-08-27T21:55:44.670878+0000","last_became_active":"2023-08-27T21:55:09.735470+0000","last_became_peered":"2023-08-27T21:55:09.735470+0000","last_unstale":"2023-08-27T21:55:44.670878+0000","last_undegraded":"2023-08-27T21:55:44.670878+0000","last_fullsized":"2023-08-27T21:55:44.670878+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.664745+0000","last_change":"2023-08-27T21:53:18.019051+0000","last_active":"2023-08-27T21:55:44.664745+0000","last_peered":"2023-08-27T21:55:44.664745+0000","last_clean":"2023-08-27T21:55:44.664745+0000","last_became_active":"2023-08-27T21:53:18.018616+0000","last_became_peered":"2023-08-27T21:53:18.018616+0000","last_unstale":"2023-08-27T21:55:44.664745+0000","last_undegraded":"2023-08-27T21:55:44.664745+0000","last_fullsized":"2023-08-27T21:55:44.664745+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.671300+0000","last_change":"2023-08-27T21:55:09.750817+0000","last_active":"2023-08-27T21:55:44.671300+0000","last_peered":"2023-08-27T21:55:44.671300+0000","last_clean":"2023-08-27T21:55:44.671300+0000","last_became_active":"2023-08-27T21:55:09.750495+0000","last_became_peered":"2023-08-27T21:55:09.750495+0000","last_unstale":"2023-08-27T21:55:44.671300+0000","last_undegraded":"2023-08-27T21:55:44.671300+0000","last_fullsized":"2023-08-27T21:55:44.671300+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675578+0000","last_change":"2023-08-27T21:55:09.738412+0000","last_active":"2023-08-27T21:55:44.675578+0000","last_peered":"2023-08-27T21:55:44.675578+0000","last_clean":"2023-08-27T21:55:44.675578+0000","last_became_active":"2023-08-27T21:55:09.736674+0000","last_became_peered":"2023-08-27T21:55:09.736674+0000","last_unstale":"2023-08-27T21:55:44.675578+0000","last_undegraded":"2023-08-27T21:55:44.675578+0000","last_fullsized":"2023-08-27T21:55:44.675578+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.666459+0000","last_change":"2023-08-27T21:53:17.043427+0000","last_active":"2023-08-27T21:55:44.666459+0000","last_peered":"2023-08-27T21:55:44.666459+0000","last_clean":"2023-08-27T21:55:44.666459+0000","last_became_active":"2023-08-27T21:53:17.043025+0000","last_became_peered":"2023-08-27T21:53:17.043025+0000","last_unstale":"2023-08-27T21:55:44.666459+0000","last_undegraded":"2023-08-27T21:55:44.666459+0000","last_fullsized":"2023-08-27T21:55:44.666459+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.671135+0000","last_change":"2023-08-27T21:55:10.172077+0000","last_active":"2023-08-27T21:55:44.671135+0000","last_peered":"2023-08-27T21:55:44.671135+0000","last_clean":"2023-08-27T21:55:44.671135+0000","last_became_active":"2023-08-27T21:55:10.171767+0000","last_became_peered":"2023-08-27T21:55:10.171767+0000","last_unstale":"2023-08-27T21:55:44.671135+0000","last_undegraded":"2023-08-27T21:55:44.671135+0000","last_fullsized":"2023-08-27T21:55:44.671135+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.667080+0000","last_change":"2023-08-27T21:55:09.739638+0000","last_active":"2023-08-27T21:55:44.667080+0000","last_peered":"2023-08-27T21:55:44.667080+0000","last_clean":"2023-08-27T21:55:44.667080+0000","last_became_active":"2023-08-27T21:55:09.739305+0000","last_became_peered":"2023-08-27T21:55:09.739305+0000","last_unstale":"2023-08-27T21:55:44.667080+0000","last_undegraded":"2023-08-27T21:55:44.667080+0000","last_fullsized":"2023-08-27T21:55:44.667080+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.671180+0000","last_change":"2023-08-27T21:53:17.813356+0000","last_active":"2023-08-27T21:55:44.671180+0000","last_peered":"2023-08-27T21:55:44.671180+0000","last_clean":"2023-08-27T21:55:44.671180+0000","last_became_active":"2023-08-27T21:53:17.812986+0000","last_became_peered":"2023-08-27T21:53:17.812986+0000","last_unstale":"2023-08-27T21:55:44.671180+0000","last_undegraded":"2023-08-27T21:55:44.671180+0000","last_fullsized":"2023-08-27T21:55:44.671180+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.672913+0000","last_change":"2023-08-27T21:55:09.750798+0000","last_active":"2023-08-27T21:55:44.672913+0000","last_peered":"2023-08-27T21:55:44.672913+0000","last_clean":"2023-08-27T21:55:44.672913+0000","last_became_active":"2023-08-27T21:55:09.750478+0000","last_became_peered":"2023-08-27T21:55:09.750478+0000","last_unstale":"2023-08-27T21:55:44.672913+0000","last_undegraded":"2023-08-27T21:55:44.672913+0000","last_fullsized":"2023-08-27T21:55:44.672913+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675959+0000","last_change":"2023-08-27T21:55:09.721249+0000","last_active":"2023-08-27T21:55:44.675959+0000","last_peered":"2023-08-27T21:55:44.675959+0000","last_clean":"2023-08-27T21:55:44.675959+0000","last_became_active":"2023-08-27T21:55:09.720806+0000","last_became_peered":"2023-08-27T21:55:09.720806+0000","last_unstale":"2023-08-27T21:55:44.675959+0000","last_undegraded":"2023-08-27T21:55:44.675959+0000","last_fullsized":"2023-08-27T21:55:44.675959+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676048+0000","last_change":"2023-08-27T21:55:10.887095+0000","last_active":"2023-08-27T21:55:44.676048+0000","last_peered":"2023-08-27T21:55:44.676048+0000","last_clean":"2023-08-27T21:55:44.676048+0000","last_became_active":"2023-08-27T21:55:10.886294+0000","last_became_peered":"2023-08-27T21:55:10.886294+0000","last_unstale":"2023-08-27T21:55:44.676048+0000","last_undegraded":"2023-08-27T21:55:44.676048+0000","last_fullsized":"2023-08-27T21:55:44.676048+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675445+0000","last_change":"2023-08-27T21:55:09.731403+0000","last_active":"2023-08-27T21:55:44.675445+0000","last_peered":"2023-08-27T21:55:44.675445+0000","last_clean":"2023-08-27T21:55:44.675445+0000","last_became_active":"2023-08-27T21:55:09.730437+0000","last_became_peered":"2023-08-27T21:55:09.730437+0000","last_unstale":"2023-08-27T21:55:44.675445+0000","last_undegraded":"2023-08-27T21:55:44.675445+0000","last_fullsized":"2023-08-27T21:55:44.675445+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676049+0000","last_change":"2023-08-27T21:55:10.887669+0000","last_active":"2023-08-27T21:55:44.676049+0000","last_peered":"2023-08-27T21:55:44.676049+0000","last_clean":"2023-08-27T21:55:44.676049+0000","last_became_active":"2023-08-27T21:55:10.887069+0000","last_became_peered":"2023-08-27T21:55:10.887069+0000","last_unstale":"2023-08-27T21:55:44.676049+0000","last_undegraded":"2023-08-27T21:55:44.676049+0000","last_fullsized":"2023-08-27T21:55:44.676049+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.671341+0000","last_change":"2023-08-27T21:55:09.744571+0000","last_active":"2023-08-27T21:55:44.671341+0000","last_peered":"2023-08-27T21:55:44.671341+0000","last_clean":"2023-08-27T21:55:44.671341+0000","last_became_active":"2023-08-27T21:55:09.743932+0000","last_became_peered":"2023-08-27T21:55:09.743932+0000","last_unstale":"2023-08-27T21:55:44.671341+0000","last_undegraded":"2023-08-27T21:55:44.671341+0000","last_fullsized":"2023-08-27T21:55:44.671341+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675533+0000","last_change":"2023-08-27T21:55:10.750933+0000","last_active":"2023-08-27T21:55:44.675533+0000","last_peered":"2023-08-27T21:55:44.675533+0000","last_clean":"2023-08-27T21:55:44.675533+0000","last_became_active":"2023-08-27T21:55:10.750599+0000","last_became_peered":"2023-08-27T21:55:10.750599+0000","last_unstale":"2023-08-27T21:55:44.675533+0000","last_undegraded":"2023-08-27T21:55:44.675533+0000","last_fullsized":"2023-08-27T21:55:44.675533+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676646+0000","last_change":"2023-08-27T21:55:09.727541+0000","last_active":"2023-08-27T21:55:44.676646+0000","last_peered":"2023-08-27T21:55:44.676646+0000","last_clean":"2023-08-27T21:55:44.676646+0000","last_became_active":"2023-08-27T21:55:09.727136+0000","last_became_peered":"2023-08-27T21:55:09.727136+0000","last_unstale":"2023-08-27T21:55:44.676646+0000","last_undegraded":"2023-08-27T21:55:44.676646+0000","last_fullsized":"2023-08-27T21:55:44.676646+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.664327+0000","last_change":"2023-08-27T21:55:10.888097+0000","last_active":"2023-08-27T21:55:44.664327+0000","last_peered":"2023-08-27T21:55:44.664327+0000","last_clean":"2023-08-27T21:55:44.664327+0000","last_became_active":"2023-08-27T21:55:10.887680+0000","last_became_peered":"2023-08-27T21:55:10.887680+0000","last_unstale":"2023-08-27T21:55:44.664327+0000","last_undegraded":"2023-08-27T21:55:44.664327+0000","last_fullsized":"2023-08-27T21:55:44.664327+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.666839+0000","last_change":"2023-08-27T21:55:09.738841+0000","last_active":"2023-08-27T21:55:44.666839+0000","last_peered":"2023-08-27T21:55:44.666839+0000","last_clean":"2023-08-27T21:55:44.666839+0000","last_became_active":"2023-08-27T21:55:09.738487+0000","last_became_peered":"2023-08-27T21:55:09.738487+0000","last_unstale":"2023-08-27T21:55:44.666839+0000","last_undegraded":"2023-08-27T21:55:44.666839+0000","last_fullsized":"2023-08-27T21:55:44.666839+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.671211+0000","last_change":"2023-08-27T21:55:10.889355+0000","last_active":"2023-08-27T21:55:44.671211+0000","last_peered":"2023-08-27T21:55:44.671211+0000","last_clean":"2023-08-27T21:55:44.671211+0000","last_became_active":"2023-08-27T21:55:10.888510+0000","last_became_peered":"2023-08-27T21:55:10.888510+0000","last_unstale":"2023-08-27T21:55:44.671211+0000","last_undegraded":"2023-08-27T21:55:44.671211+0000","last_fullsized":"2023-08-27T21:55:44.671211+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.729785+0000","last_change":"2023-08-27T21:55:09.731816+0000","last_active":"2023-08-27T21:55:44.729785+0000","last_peered":"2023-08-27T21:55:44.729785+0000","last_clean":"2023-08-27T21:55:44.729785+0000","last_became_active":"2023-08-27T21:55:09.731471+0000","last_became_peered":"2023-08-27T21:55:09.731471+0000","last_unstale":"2023-08-27T21:55:44.729785+0000","last_undegraded":"2023-08-27T21:55:44.729785+0000","last_fullsized":"2023-08-27T21:55:44.729785+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675278+0000","last_change":"2023-08-27T21:55:10.887722+0000","last_active":"2023-08-27T21:55:44.675278+0000","last_peered":"2023-08-27T21:55:44.675278+0000","last_clean":"2023-08-27T21:55:44.675278+0000","last_became_active":"2023-08-27T21:55:10.887139+0000","last_became_peered":"2023-08-27T21:55:10.887139+0000","last_unstale":"2023-08-27T21:55:44.675278+0000","last_undegraded":"2023-08-27T21:55:44.675278+0000","last_fullsized":"2023-08-27T21:55:44.675278+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.729706+0000","last_change":"2023-08-27T21:55:09.733184+0000","last_active":"2023-08-27T21:55:44.729706+0000","last_peered":"2023-08-27T21:55:44.729706+0000","last_clean":"2023-08-27T21:55:44.729706+0000","last_became_active":"2023-08-27T21:55:09.732871+0000","last_became_peered":"2023-08-27T21:55:09.732871+0000","last_unstale":"2023-08-27T21:55:44.729706+0000","last_undegraded":"2023-08-27T21:55:44.729706+0000","last_fullsized":"2023-08-27T21:55:44.729706+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.666560+0000","last_change":"2023-08-27T21:55:10.886227+0000","last_active":"2023-08-27T21:55:44.666560+0000","last_peered":"2023-08-27T21:55:44.666560+0000","last_clean":"2023-08-27T21:55:44.666560+0000","last_became_active":"2023-08-27T21:55:10.885594+0000","last_became_peered":"2023-08-27T21:55:10.885594+0000","last_unstale":"2023-08-27T21:55:44.666560+0000","last_undegraded":"2023-08-27T21:55:44.666560+0000","last_fullsized":"2023-08-27T21:55:44.666560+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.729834+0000","last_change":"2023-08-27T21:55:09.734175+0000","last_active":"2023-08-27T21:55:44.729834+0000","last_peered":"2023-08-27T21:55:44.729834+0000","last_clean":"2023-08-27T21:55:44.729834+0000","last_became_active":"2023-08-27T21:55:09.733839+0000","last_became_peered":"2023-08-27T21:55:09.733839+0000","last_unstale":"2023-08-27T21:55:44.729834+0000","last_undegraded":"2023-08-27T21:55:44.729834+0000","last_fullsized":"2023-08-27T21:55:44.729834+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676490+0000","last_change":"2023-08-27T21:55:10.193452+0000","last_active":"2023-08-27T21:55:44.676490+0000","last_peered":"2023-08-27T21:55:44.676490+0000","last_clean":"2023-08-27T21:55:44.676490+0000","last_became_active":"2023-08-27T21:55:10.192817+0000","last_became_peered":"2023-08-27T21:55:10.192817+0000","last_unstale":"2023-08-27T21:55:44.676490+0000","last_undegraded":"2023-08-27T21:55:44.676490+0000","last_fullsized":"2023-08-27T21:55:44.676490+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.670492+0000","last_change":"2023-08-27T21:55:09.729934+0000","last_active":"2023-08-27T21:55:44.670492+0000","last_peered":"2023-08-27T21:55:44.670492+0000","last_clean":"2023-08-27T21:55:44.670492+0000","last_became_active":"2023-08-27T21:55:09.729533+0000","last_became_peered":"2023-08-27T21:55:09.729533+0000","last_unstale":"2023-08-27T21:55:44.670492+0000","last_undegraded":"2023-08-27T21:55:44.670492+0000","last_fullsized":"2023-08-27T21:55:44.670492+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.668595+0000","last_change":"2023-08-27T21:55:10.886329+0000","last_active":"2023-08-27T21:55:44.668595+0000","last_peered":"2023-08-27T21:55:44.668595+0000","last_clean":"2023-08-27T21:55:44.668595+0000","last_became_active":"2023-08-27T21:55:10.885846+0000","last_became_peered":"2023-08-27T21:55:10.885846+0000","last_unstale":"2023-08-27T21:55:44.668595+0000","last_undegraded":"2023-08-27T21:55:44.668595+0000","last_fullsized":"2023-08-27T21:55:44.668595+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.728871+0000","last_change":"2023-08-27T21:55:09.734222+0000","last_active":"2023-08-27T21:55:44.728871+0000","last_peered":"2023-08-27T21:55:44.728871+0000","last_clean":"2023-08-27T21:55:44.728871+0000","last_became_active":"2023-08-27T21:55:09.733665+0000","last_became_peered":"2023-08-27T21:55:09.733665+0000","last_unstale":"2023-08-27T21:55:44.728871+0000","last_undegraded":"2023-08-27T21:55:44.728871+0000","last_fullsized":"2023-08-27T21:55:44.728871+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.666928+0000","last_change":"2023-08-27T21:55:10.888185+0000","last_active":"2023-08-27T21:55:44.666928+0000","last_peered":"2023-08-27T21:55:44.666928+0000","last_clean":"2023-08-27T21:55:44.666928+0000","last_became_active":"2023-08-27T21:55:10.887602+0000","last_became_peered":"2023-08-27T21:55:10.887602+0000","last_unstale":"2023-08-27T21:55:44.666928+0000","last_undegraded":"2023-08-27T21:55:44.666928+0000","last_fullsized":"2023-08-27T21:55:44.666928+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.665392+0000","last_change":"2023-08-27T21:55:09.731963+0000","last_active":"2023-08-27T21:55:44.665392+0000","last_peered":"2023-08-27T21:55:44.665392+0000","last_clean":"2023-08-27T21:55:44.665392+0000","last_became_active":"2023-08-27T21:55:09.731543+0000","last_became_peered":"2023-08-27T21:55:09.731543+0000","last_unstale":"2023-08-27T21:55:44.665392+0000","last_undegraded":"2023-08-27T21:55:44.665392+0000","last_fullsized":"2023-08-27T21:55:44.665392+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676415+0000","last_change":"2023-08-27T21:55:10.193438+0000","last_active":"2023-08-27T21:55:44.676415+0000","last_peered":"2023-08-27T21:55:44.676415+0000","last_clean":"2023-08-27T21:55:44.676415+0000","last_became_active":"2023-08-27T21:55:10.192753+0000","last_became_peered":"2023-08-27T21:55:10.192753+0000","last_unstale":"2023-08-27T21:55:44.676415+0000","last_undegraded":"2023-08-27T21:55:44.676415+0000","last_fullsized":"2023-08-27T21:55:44.676415+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.729314+0000","last_change":"2023-08-27T21:55:09.734742+0000","last_active":"2023-08-27T21:55:44.729314+0000","last_peered":"2023-08-27T21:55:44.729314+0000","last_clean":"2023-08-27T21:55:44.729314+0000","last_became_active":"2023-08-27T21:55:09.734348+0000","last_became_peered":"2023-08-27T21:55:09.734348+0000","last_unstale":"2023-08-27T21:55:44.729314+0000","last_undegraded":"2023-08-27T21:55:44.729314+0000","last_fullsized":"2023-08-27T21:55:44.729314+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675678+0000","last_change":"2023-08-27T21:55:10.889499+0000","last_active":"2023-08-27T21:55:44.675678+0000","last_peered":"2023-08-27T21:55:44.675678+0000","last_clean":"2023-08-27T21:55:44.675678+0000","last_became_active":"2023-08-27T21:55:10.889097+0000","last_became_peered":"2023-08-27T21:55:10.889097+0000","last_unstale":"2023-08-27T21:55:44.675678+0000","last_undegraded":"2023-08-27T21:55:44.675678+0000","last_fullsized":"2023-08-27T21:55:44.675678+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":20,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675807+0000","last_change":"2023-08-27T21:55:09.734659+0000","last_active":"2023-08-27T21:55:44.675807+0000","last_peered":"2023-08-27T21:55:44.675807+0000","last_clean":"2023-08-27T21:55:44.675807+0000","last_became_active":"2023-08-27T21:55:09.734326+0000","last_became_peered":"2023-08-27T21:55:09.734326+0000","last_unstale":"2023-08-27T21:55:44.675807+0000","last_undegraded":"2023-08-27T21:55:44.675807+0000","last_fullsized":"2023-08-27T21:55:44.675807+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":10,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-08-27T21:55:45.730896+0000","last_change":"2023-08-27T21:55:45.730896+0000","last_active":"2023-08-27T21:55:45.730896+0000","last_peered":"2023-08-27T21:55:45.730896+0000","last_clean":"2023-08-27T21:55:45.730896+0000","last_became_active":"2023-08-27T21:55:45.730202+0000","last_became_peered":"2023-08-27T21:55:45.730202+0000","last_unstale":"2023-08-27T21:55:45.730896+0000","last_undegraded":"2023-08-27T21:55:45.730896+0000","last_fullsized":"2023-08-27T21:55:45.730896+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.729620+0000","last_change":"2023-08-27T21:55:09.735453+0000","last_active":"2023-08-27T21:55:44.729620+0000","last_peered":"2023-08-27T21:55:44.729620+0000","last_clean":"2023-08-27T21:55:44.729620+0000","last_became_active":"2023-08-27T21:55:09.735100+0000","last_became_peered":"2023-08-27T21:55:09.735100+0000","last_unstale":"2023-08-27T21:55:44.729620+0000","last_undegraded":"2023-08-27T21:55:44.729620+0000","last_fullsized":"2023-08-27T21:55:44.729620+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676137+0000","last_change":"2023-08-27T21:55:10.226811+0000","last_active":"2023-08-27T21:55:44.676137+0000","last_peered":"2023-08-27T21:55:44.676137+0000","last_clean":"2023-08-27T21:55:44.676137+0000","last_became_active":"2023-08-27T21:55:10.226484+0000","last_became_peered":"2023-08-27T21:55:10.226484+0000","last_unstale":"2023-08-27T21:55:44.676137+0000","last_undegraded":"2023-08-27T21:55:44.676137+0000","last_fullsized":"2023-08-27T21:55:44.676137+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.666763+0000","last_change":"2023-08-27T21:55:09.744095+0000","last_active":"2023-08-27T21:55:44.666763+0000","last_peered":"2023-08-27T21:55:44.666763+0000","last_clean":"2023-08-27T21:55:44.666763+0000","last_became_active":"2023-08-27T21:55:09.743755+0000","last_became_peered":"2023-08-27T21:55:09.743755+0000","last_unstale":"2023-08-27T21:55:44.666763+0000","last_undegraded":"2023-08-27T21:55:44.666763+0000","last_fullsized":"2023-08-27T21:55:44.666763+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675974+0000","last_change":"2023-08-27T21:55:10.227656+0000","last_active":"2023-08-27T21:55:44.675974+0000","last_peered":"2023-08-27T21:55:44.675974+0000","last_clean":"2023-08-27T21:55:44.675974+0000","last_became_active":"2023-08-27T21:55:10.227206+0000","last_became_peered":"2023-08-27T21:55:10.227206+0000","last_unstale":"2023-08-27T21:55:44.675974+0000","last_undegraded":"2023-08-27T21:55:44.675974+0000","last_fullsized":"2023-08-27T21:55:44.675974+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.664033+0000","last_change":"2023-08-27T21:55:09.728718+0000","last_active":"2023-08-27T21:55:44.664033+0000","last_peered":"2023-08-27T21:55:44.664033+0000","last_clean":"2023-08-27T21:55:44.664033+0000","last_became_active":"2023-08-27T21:55:09.728363+0000","last_became_peered":"2023-08-27T21:55:09.728363+0000","last_unstale":"2023-08-27T21:55:44.664033+0000","last_undegraded":"2023-08-27T21:55:44.664033+0000","last_fullsized":"2023-08-27T21:55:44.664033+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.671008+0000","last_change":"2023-08-27T21:55:10.891642+0000","last_active":"2023-08-27T21:55:44.671008+0000","last_peered":"2023-08-27T21:55:44.671008+0000","last_clean":"2023-08-27T21:55:44.671008+0000","last_became_active":"2023-08-27T21:55:10.890828+0000","last_became_peered":"2023-08-27T21:55:10.890828+0000","last_unstale":"2023-08-27T21:55:44.671008+0000","last_undegraded":"2023-08-27T21:55:44.671008+0000","last_fullsized":"2023-08-27T21:55:44.671008+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676081+0000","last_change":"2023-08-27T21:55:09.722569+0000","last_active":"2023-08-27T21:55:44.676081+0000","last_peered":"2023-08-27T21:55:44.676081+0000","last_clean":"2023-08-27T21:55:44.676081+0000","last_became_active":"2023-08-27T21:55:09.722184+0000","last_became_peered":"2023-08-27T21:55:09.722184+0000","last_unstale":"2023-08-27T21:55:44.676081+0000","last_undegraded":"2023-08-27T21:55:44.676081+0000","last_fullsized":"2023-08-27T21:55:44.676081+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.668689+0000","last_change":"2023-08-27T21:55:10.886745+0000","last_active":"2023-08-27T21:55:44.668689+0000","last_peered":"2023-08-27T21:55:44.668689+0000","last_clean":"2023-08-27T21:55:44.668689+0000","last_became_active":"2023-08-27T21:55:10.886364+0000","last_became_peered":"2023-08-27T21:55:10.886364+0000","last_unstale":"2023-08-27T21:55:44.668689+0000","last_undegraded":"2023-08-27T21:55:44.668689+0000","last_fullsized":"2023-08-27T21:55:44.668689+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.669523+0000","last_change":"2023-08-27T21:55:09.741809+0000","last_active":"2023-08-27T21:55:44.669523+0000","last_peered":"2023-08-27T21:55:44.669523+0000","last_clean":"2023-08-27T21:55:44.669523+0000","last_became_active":"2023-08-27T21:55:09.741221+0000","last_became_peered":"2023-08-27T21:55:09.741221+0000","last_unstale":"2023-08-27T21:55:44.669523+0000","last_undegraded":"2023-08-27T21:55:44.669523+0000","last_fullsized":"2023-08-27T21:55:44.669523+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676459+0000","last_change":"2023-08-27T21:55:09.751600+0000","last_active":"2023-08-27T21:55:44.676459+0000","last_peered":"2023-08-27T21:55:44.676459+0000","last_clean":"2023-08-27T21:55:44.676459+0000","last_became_active":"2023-08-27T21:55:09.751332+0000","last_became_peered":"2023-08-27T21:55:09.751332+0000","last_unstale":"2023-08-27T21:55:44.676459+0000","last_undegraded":"2023-08-27T21:55:44.676459+0000","last_fullsized":"2023-08-27T21:55:44.676459+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675384+0000","last_change":"2023-08-27T21:55:09.746051+0000","last_active":"2023-08-27T21:55:44.675384+0000","last_peered":"2023-08-27T21:55:44.675384+0000","last_clean":"2023-08-27T21:55:44.675384+0000","last_became_active":"2023-08-27T21:55:09.745813+0000","last_became_peered":"2023-08-27T21:55:09.745813+0000","last_unstale":"2023-08-27T21:55:44.675384+0000","last_undegraded":"2023-08-27T21:55:44.675384+0000","last_fullsized":"2023-08-27T21:55:44.675384+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430264,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297736,"kb_used_data":640,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447416,"statfs":{"total":95995035648,"available":95690153984,"internally_reserved":0,"allocated":655360,"data_stored":271700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:56:20 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":0.89300000000000002,"15min":0.88900000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.2090000000000001,"5min":1.774,"15min":1.774},"last":1.0389999999999999},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.88200000000000001,"15min":0.878},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.1739999999999999,"5min":1.3600000000000001,"15min":1.3600000000000001},"last":0.71099999999999997}]},{"osd":1,"last update":"Sun Aug 27 21:56:20 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.90300000000000002,"15min":0.89100000000000001},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.1919999999999999,"5min":1.696,"15min":1.696},"last":1.1120000000000001},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.91100000000000003,"15min":0.92300000000000004},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.2350000000000001,"5min":1.4390000000000001,"15min":1.4390000000000001},"last":0.94599999999999995}]},{"osd":2,"last update":"Sun Aug 27 21:56:20 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.93899999999999995,"15min":0.93500000000000005},"min":{"1min":0.64500000000000002,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.2450000000000001,"5min":1.8580000000000001,"15min":1.8580000000000001},"last":0.98099999999999998},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.91200000000000003,"15min":0.90100000000000002},"min":{"1min":0.63,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.143,"5min":1.8009999999999999,"15min":1.8009999999999999},"last":0.84699999999999998}]},{"osd":3,"last update":"Sun Aug 27 21:56:20 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.93600000000000005,"15min":0.94399999999999995},"min":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.2170000000000001,"5min":1.931,"15min":1.931},"last":0.81699999999999995},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.97399999999999998,"15min":0.95199999999999996},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.244,"5min":1.913,"15min":1.913},"last":1.0069999999999999}]},{"osd":4,"last update":"Sun Aug 27 21:56:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.89900000000000002,"15min":0.873},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.1870000000000001,"5min":1.889,"15min":1.889},"last":0.86299999999999999},{"interface":"front","average":{"1min":0.97699999999999998,"5min":0.93200000000000005,"15min":0.90300000000000002},"min":{"1min":0.59899999999999998,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.276,"5min":2.0049999999999999,"15min":2.0049999999999999},"last":1.0780000000000001}]},{"osd":5,"last update":"Sun Aug 27 21:56:20 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.91800000000000004,"15min":0.90000000000000002},"min":{"1min":0.56599999999999995,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.1890000000000001,"5min":2.0259999999999998,"15min":2.0259999999999998},"last":0.77200000000000002},{"interface":"front","average":{"1min":1.0249999999999999,"5min":1.024,"15min":1.028},"min":{"1min":0.61699999999999999,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.329,"5min":2.1059999999999999,"15min":2.1059999999999999},"last":1.0509999999999999}]},{"osd":6,"last update":"Sun Aug 27 21:56:20 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.036,"15min":1.012},"min":{"1min":0.57899999999999996,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.3149999999999999,"5min":1.5389999999999999,"15min":1.5389999999999999},"last":1.167},{"interface":"front","average":{"1min":1.125,"5min":1.071,"15min":1.052},"min":{"1min":0.75900000000000001,"5min":0.66300000000000003,"15min":0.66300000000000003},"max":{"1min":1.3759999999999999,"5min":2.0870000000000002,"15min":2.0870000000000002},"last":1.1339999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626498,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:56:27 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.83699999999999997,"15min":0.85599999999999998},"min":{"1min":0.60499999999999998,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.2849999999999999,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":1.1479999999999999},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.90100000000000002,"15min":0.85199999999999998},"min":{"1min":0.54700000000000004,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.254,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":0.85299999999999998}]},{"osd":1,"last update":"Sun Aug 27 21:56:27 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.89200000000000002,"15min":0.93400000000000005},"min":{"1min":0.61599999999999999,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.1080000000000001,"5min":1.629,"15min":1.629},"last":0.81399999999999995},{"interface":"front","average":{"1min":0.88,"5min":0.90700000000000003,"15min":0.86499999999999999},"min":{"1min":0.66300000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.1319999999999999,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":0.93200000000000005}]},{"osd":2,"last update":"Sun Aug 27 21:56:27 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.94299999999999995,"15min":0.91900000000000004},"min":{"1min":0.63800000000000001,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.3720000000000001,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":1.1200000000000001},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.91700000000000004,"15min":0.92700000000000005},"min":{"1min":0.60999999999999999,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.2370000000000001,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":1.2669999999999999}]},{"osd":3,"last update":"Sun Aug 27 21:56:27 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.93200000000000005,"15min":0.97399999999999998},"min":{"1min":0.624,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.125,"5min":1.752,"15min":1.752},"last":0.90600000000000003},{"interface":"front","average":{"1min":1.0409999999999999,"5min":0.97899999999999998,"15min":0.98099999999999998},"min":{"1min":0.79500000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.3420000000000001,"5min":1.546,"15min":1.546},"last":1.0129999999999999}]},{"osd":4,"last update":"Sun Aug 27 21:56:27 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.90200000000000002,"15min":0.83399999999999996},"min":{"1min":0.65400000000000003,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.2769999999999999,"5min":1.446,"15min":1.446},"last":1.0489999999999999},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.95199999999999996,"15min":0.88200000000000001},"min":{"1min":0.69799999999999995,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.2629999999999999,"5min":1.296,"15min":1.296},"last":1.0800000000000001}]},{"osd":5,"last update":"Sun Aug 27 21:56:27 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":0.96899999999999997,"15min":0.89300000000000002},"min":{"1min":0.73999999999999999,"5min":0.437,"15min":0.437},"max":{"1min":1.208,"5min":1.7170000000000001,"15min":1.7170000000000001},"last":1.202},{"interface":"front","average":{"1min":1.089,"5min":0.998,"15min":0.95499999999999996},"min":{"1min":0.72199999999999998,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.369,"5min":1.845,"15min":1.845},"last":1.167}]},{"osd":7,"last update":"Sun Aug 27 21:56:16 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.03,"15min":0.98699999999999999},"min":{"1min":0.78400000000000003,"5min":0.5,"15min":0.5},"max":{"1min":1.3740000000000001,"5min":1.5049999999999999,"15min":1.5049999999999999},"last":1.319},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.085,"15min":1.022},"min":{"1min":0.95799999999999996,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.4670000000000001,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":1.2210000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574959,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297700,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447452,"statfs":{"total":95995035648,"available":95690190848,"internally_reserved":0,"allocated":618496,"data_stored":269318,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:55:54 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.73799999999999999,"15min":0.58799999999999997},"min":{"1min":0.40600000000000003,"5min":0.35599999999999998,"15min":0.34799999999999998},"max":{"1min":1.669,"5min":1.669,"15min":1.669},"last":1.02},{"interface":"front","average":{"1min":0.82099999999999995,"5min":0.81000000000000005,"15min":0.64800000000000002},"min":{"1min":0.441,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.8779999999999999,"5min":1.8779999999999999,"15min":1.8779999999999999},"last":0.755}]},{"osd":2,"last update":"Sun Aug 27 21:55:34 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.76100000000000001,"15min":0.69499999999999995},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.90400000000000003,"5min":1.3600000000000001,"15min":1.3600000000000001},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.80800000000000005,"15min":0.70599999999999996},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.107,"5min":1.395,"15min":1.395},"last":0.79400000000000004}]},{"osd":3,"last update":"Sun Aug 27 21:56:20 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.78000000000000003,"15min":0.79100000000000004},"min":{"1min":0.496,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.742,"5min":1.742,"15min":1.742},"last":0.92500000000000004},{"interface":"front","average":{"1min":0.81399999999999995,"5min":0.84299999999999997,"15min":0.84299999999999997},"min":{"1min":0.47599999999999998,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.1499999999999999,"5min":1.351,"15min":1.351},"last":1.3169999999999999}]},{"osd":4,"last update":"Sun Aug 27 21:56:02 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.0229999999999999,"15min":1.0149999999999999},"min":{"1min":0.629,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.45,"5min":1.6779999999999999,"15min":1.6779999999999999},"last":1.21},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.081,"15min":1.0169999999999999},"min":{"1min":0.69499999999999995,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.851,"5min":1.851,"15min":1.851},"last":1.121}]},{"osd":5,"last update":"Sun Aug 27 21:55:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":1.075,"15min":1.0669999999999999},"min":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.762,"5min":1.762,"15min":1.762},"last":1.284},{"interface":"front","average":{"1min":1.111,"5min":1.0900000000000001,"15min":1.1200000000000001},"min":{"1min":0.78600000000000003,"5min":0.66600000000000004,"15min":0.66600000000000004},"max":{"1min":1.5669999999999999,"5min":1.861,"15min":1.861},"last":1.258}]},{"osd":6,"last update":"Sun Aug 27 21:55:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":1.1220000000000001,"15min":1.1459999999999999},"min":{"1min":0.86299999999999999,"5min":0.75,"15min":0.75},"max":{"1min":1.6220000000000001,"5min":1.821,"15min":1.821},"last":1.3759999999999999},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.159,"15min":1.2130000000000001},"min":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"max":{"1min":1.516,"5min":1.7110000000000001,"15min":1.7110000000000001},"last":1.4359999999999999}]},{"osd":7,"last update":"Sun Aug 27 21:56:20 2023","interfaces":[{"interface":"back","average":{"1min":1.216,"5min":1.1830000000000001,"15min":1.2110000000000001},"min":{"1min":0.80400000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.665,"5min":1.665,"15min":1.665},"last":1.51},{"interface":"front","average":{"1min":1.1459999999999999,"5min":1.1519999999999999,"15min":1.1719999999999999},"min":{"1min":0.73799999999999999,"5min":0.69099999999999995,"15min":0.69099999999999995},"max":{"1min":1.5389999999999999,"5min":1.7130000000000001,"15min":1.7130000000000001},"last":1.4810000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738487,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Aug 27 21:55:49 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.69799999999999995,"15min":0.59299999999999997},"min":{"1min":0.45000000000000001,"5min":0.317,"15min":0.31},"max":{"1min":0.96399999999999997,"5min":1.177,"15min":1.177},"last":0.92000000000000004},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.77200000000000002,"15min":0.64500000000000002},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.34499999999999997},"max":{"1min":1.0680000000000001,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.872}]},{"osd":2,"last update":"Sun Aug 27 21:56:28 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.749,"15min":0.68500000000000005},"min":{"1min":0.45300000000000001,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.98099999999999998,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.754},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.77200000000000002,"15min":0.72199999999999998},"min":{"1min":0.42899999999999999,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.01,"5min":1.25,"15min":1.25},"last":1.0780000000000001}]},{"osd":3,"last update":"Sun Aug 27 21:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":0.82799999999999996,"15min":0.82899999999999996},"min":{"1min":0.45000000000000001,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.161,"5min":1.359,"15min":1.359},"last":1.0149999999999999},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.752,"15min":0.77300000000000002},"min":{"1min":0.45200000000000001,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.94199999999999995,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":1.1040000000000001}]},{"osd":4,"last update":"Sun Aug 27 21:56:03 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":0.98299999999999998,"15min":0.95299999999999996},"min":{"1min":0.73699999999999999,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.407,"5min":1.5860000000000001,"15min":1.5860000000000001},"last":1.036},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.028,"15min":1.0109999999999999},"min":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":1.5760000000000001,"5min":1.6299999999999999,"15min":1.6299999999999999},"last":1.179}]},{"osd":5,"last update":"Sun Aug 27 21:55:49 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":0.97499999999999998,"15min":0.97299999999999998},"min":{"1min":0.82699999999999996,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.591,"5min":1.591,"15min":1.591},"last":1.1699999999999999},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.0740000000000001,"15min":1.0549999999999999},"min":{"1min":0.70799999999999996,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.6060000000000001,"5min":1.6060000000000001,"15min":1.6060000000000001},"last":1.214}]},{"osd":6,"last update":"Sun Aug 27 21:56:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.1040000000000001,"15min":1.0960000000000001},"min":{"1min":0.71699999999999997,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.73,"5min":1.8069999999999999,"15min":1.8069999999999999},"last":1.0960000000000001},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.1040000000000001,"15min":1.141},"min":{"1min":0.68600000000000005,"5min":0.628,"15min":0.628},"max":{"1min":1.5189999999999999,"5min":1.5640000000000001,"15min":1.5640000000000001},"last":1.1299999999999999}]},{"osd":7,"last update":"Sun Aug 27 21:56:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":1.085,"15min":1.1040000000000001},"min":{"1min":0.71999999999999997,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.6879999999999999,"5min":1.6879999999999999,"15min":1.6879999999999999},"last":0.95799999999999996},{"interface":"front","average":{"1min":1.038,"5min":1.1839999999999999,"15min":1.25},"min":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"max":{"1min":1.5449999999999999,"5min":1.647,"15min":1.647},"last":1.1499999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411429,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:55:36 2023","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":0.71899999999999997,"15min":0.63800000000000001},"min":{"1min":0.503,"5min":0.38300000000000001,"15min":0.372},"max":{"1min":1.3109999999999999,"5min":1.885,"15min":1.885},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.77700000000000002,"15min":0.65700000000000003},"min":{"1min":0.45000000000000001,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.244,"5min":1.718,"15min":1.718},"last":0.61499999999999999}]},{"osd":1,"last update":"Sun Aug 27 21:55:36 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":0.79400000000000004,"15min":0.69599999999999995},"min":{"1min":0.39300000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.29,"5min":2.4769999999999999,"15min":2.4769999999999999},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.82199999999999995,"5min":0.81000000000000005,"15min":0.68500000000000005},"min":{"1min":0.51800000000000002,"5min":0.44900000000000001,"15min":0.439},"max":{"1min":1.409,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":0.748}]},{"osd":3,"last update":"Sun Aug 27 21:56:16 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":0.81899999999999995,"15min":0.78100000000000003},"min":{"1min":0.40899999999999997,"5min":0.372,"15min":0.372},"max":{"1min":1.268,"5min":1.5569999999999999,"15min":1.5569999999999999},"last":0.995},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.872,"15min":0.83699999999999997},"min":{"1min":0.44800000000000001,"5min":0.441,"15min":0.441},"max":{"1min":1.133,"5min":1.7649999999999999,"15min":1.7649999999999999},"last":0.66400000000000003}]},{"osd":4,"last update":"Sun Aug 27 21:56:07 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.0229999999999999,"15min":0.98499999999999999},"min":{"1min":0.69199999999999995,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.7310000000000001,"5min":1.8580000000000001,"15min":1.8580000000000001},"last":0.83699999999999997},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.073,"15min":0.98599999999999999},"min":{"1min":0.60399999999999998,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.4970000000000001,"5min":1.921,"15min":1.921},"last":1.028}]},{"osd":5,"last update":"Sun Aug 27 21:55:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1579999999999999,"5min":1.087,"15min":1.0409999999999999},"min":{"1min":0.74099999999999999,"5min":0.67300000000000004,"15min":0.67300000000000004},"max":{"1min":1.51,"5min":1.845,"15min":1.845},"last":0.86899999999999999},{"interface":"front","average":{"1min":1.095,"5min":1.1020000000000001,"15min":1.0720000000000001},"min":{"1min":0.78600000000000003,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.2789999999999999,"5min":2.0089999999999999,"15min":2.0089999999999999},"last":1.0589999999999999}]},{"osd":6,"last update":"Sun Aug 27 21:56:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.153,"15min":1.1850000000000001},"min":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.55,"5min":2.0960000000000001,"15min":2.0960000000000001},"last":0.97199999999999998},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.137,"15min":1.169},"min":{"1min":0.63100000000000001,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.806,"5min":1.806,"15min":1.806},"last":1.0900000000000001}]},{"osd":7,"last update":"Sun Aug 27 21:56:16 2023","interfaces":[{"interface":"back","average":{"1min":1.216,"5min":1.1890000000000001,"15min":1.23},"min":{"1min":0.68000000000000005,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.518,"5min":1.804,"15min":1.804},"last":1.1259999999999999},{"interface":"front","average":{"1min":1.2569999999999999,"5min":1.1970000000000001,"15min":1.1890000000000001},"min":{"1min":0.86899999999999999,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.8500000000000001,"5min":1.8500000000000001,"15min":1.8500000000000001},"last":1.139}]}]},{"osd":3,"up_from":25,"seq":107374182492,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":270038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:56:22 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.69699999999999995,"15min":0.68100000000000005},"min":{"1min":0.45600000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.0089999999999999,"5min":1.1439999999999999,"15min":1.1439999999999999},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.72599999999999998,"15min":0.73499999999999999},"min":{"1min":0.501,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.72299999999999998}]},{"osd":1,"last update":"Sun Aug 27 21:56:22 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":0.76000000000000001,"15min":0.75900000000000001},"min":{"1min":0.46600000000000003,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.0940000000000001,"5min":1.111,"15min":1.111},"last":0.92000000000000004},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.80400000000000005,"15min":0.74299999999999999},"min":{"1min":0.53100000000000003,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.98699999999999999,"5min":2.883,"15min":2.883},"last":0.871}]},{"osd":2,"last update":"Sun Aug 27 21:56:22 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.88100000000000001,"15min":0.86699999999999999},"min":{"1min":0.50700000000000001,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.179,"5min":1.179,"15min":1.179},"last":0.95599999999999996},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.85399999999999998,"15min":0.876},"min":{"1min":0.42499999999999999,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.1719999999999999,"5min":3.319,"15min":3.319},"last":0.98699999999999999}]},{"osd":4,"last update":"Sun Aug 27 21:56:03 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.98999999999999999,"15min":0.98599999999999999},"min":{"1min":0.64400000000000002,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.3320000000000001,"5min":3.5939999999999999,"15min":3.5939999999999999},"last":1.0740000000000001},{"interface":"front","average":{"1min":1.071,"5min":1.0329999999999999,"15min":1.0129999999999999},"min":{"1min":0.69499999999999995,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.4670000000000001,"5min":1.679,"15min":1.679},"last":1.292}]},{"osd":5,"last update":"Sun Aug 27 21:55:44 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.036,"15min":1.042},"min":{"1min":0.746,"5min":0.622,"15min":0.622},"max":{"1min":1.488,"5min":3.5670000000000002,"15min":3.5670000000000002},"last":1.2609999999999999},{"interface":"front","average":{"1min":1.0660000000000001,"5min":1.054,"15min":1.04},"min":{"1min":0.875,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.371,"5min":1.5369999999999999,"15min":1.5369999999999999},"last":1.327}]},{"osd":6,"last update":"Sun Aug 27 21:56:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.0880000000000001,"15min":1.093},"min":{"1min":0.81299999999999994,"5min":0.69799999999999995,"15min":0.69799999999999995},"max":{"1min":1.532,"5min":1.532,"15min":1.532},"last":1.3959999999999999},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.1200000000000001,"15min":1.119},"min":{"1min":0.76100000000000001,"5min":0.66400000000000003,"15min":0.66400000000000003},"max":{"1min":1.5289999999999999,"5min":1.5960000000000001,"15min":1.5960000000000001},"last":1.198}]},{"osd":7,"last update":"Sun Aug 27 21:56:15 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.115,"15min":1.1220000000000001},"min":{"1min":0.81799999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.4039999999999999,"5min":3.5270000000000001,"15min":3.5270000000000001},"last":1.369},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.1819999999999999,"15min":1.224},"min":{"1min":0.81299999999999994,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.383,"5min":1.643,"15min":1.643},"last":1.4490000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018964,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297732,"kb_used_data":636,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447420,"statfs":{"total":95995035648,"available":95690158080,"internally_reserved":0,"allocated":651264,"data_stored":271630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:56:03 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.90400000000000003,"15min":0.83299999999999996},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.742,"5min":3.383,"15min":3.383},"last":0.90600000000000003},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.85399999999999998,"15min":0.83899999999999997},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.425,"5min":1.425,"15min":1.425},"last":1.153}]},{"osd":1,"last update":"Sun Aug 27 21:56:03 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":0.86299999999999999,"15min":0.81000000000000005},"min":{"1min":0.60199999999999998,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.2,"5min":2.1349999999999998,"15min":2.1349999999999998},"last":1.022},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.97899999999999998,"15min":0.90100000000000002},"min":{"1min":0.63900000000000001,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.6040000000000001,"5min":3.3340000000000001,"15min":3.3340000000000001},"last":0.83399999999999996}]},{"osd":2,"last update":"Sun Aug 27 21:56:03 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":0.97099999999999997,"15min":0.878},"min":{"1min":0.63200000000000001,"5min":0.62,"15min":0.62},"max":{"1min":1.871,"5min":3.4260000000000002,"15min":3.4260000000000002},"last":1.071},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.88700000000000001,"15min":0.874},"min":{"1min":0.68200000000000005,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.4530000000000001,"5min":1.4530000000000001,"15min":1.4530000000000001},"last":0.874}]},{"osd":3,"last update":"Sun Aug 27 21:56:03 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.89900000000000002,"15min":0.88500000000000001},"min":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.5209999999999999,"5min":1.5209999999999999,"15min":1.5209999999999999},"last":1.0569999999999999},{"interface":"front","average":{"1min":1.0469999999999999,"5min":0.93999999999999995,"15min":0.92600000000000005},"min":{"1min":0.61299999999999999,"5min":0.501,"15min":0.501},"max":{"1min":1.5529999999999999,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":1.1240000000000001}]},{"osd":5,"last update":"Sun Aug 27 21:55:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.89400000000000002,"15min":0.91800000000000004},"min":{"1min":0.58699999999999997,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.4019999999999999,"5min":3.2200000000000002,"15min":3.2200000000000002},"last":0.745},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.85899999999999999,"15min":0.83499999999999996},"min":{"1min":0.56599999999999995,"5min":0.378,"15min":0.378},"max":{"1min":1.353,"5min":1.353,"15min":1.353},"last":0.95499999999999996}]},{"osd":6,"last update":"Sun Aug 27 21:56:31 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":0.91600000000000004,"15min":0.83899999999999997},"min":{"1min":0.73799999999999999,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.679,"5min":1.679,"15min":1.679},"last":0.92500000000000004},{"interface":"front","average":{"1min":1.2290000000000001,"5min":1.042,"15min":0.95599999999999996},"min":{"1min":0.89600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":2.0630000000000002,"5min":3.278,"15min":3.278},"last":0.97999999999999998}]},{"osd":7,"last update":"Sun Aug 27 21:56:18 2023","interfaces":[{"interface":"back","average":{"1min":1.2390000000000001,"5min":1.044,"15min":0.95799999999999996},"min":{"1min":0.77300000000000002,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.9590000000000001,"5min":1.9590000000000001,"15min":1.9590000000000001},"last":1.1850000000000001},{"interface":"front","average":{"1min":1.2130000000000001,"5min":1.0109999999999999,"15min":0.92500000000000004},"min":{"1min":0.84599999999999997,"5min":0.443,"15min":0.443},"max":{"1min":2.2509999999999999,"5min":2.2509999999999999,"15min":2.2509999999999999},"last":1.222}]}]},{"osd":5,"up_from":36,"seq":154618822731,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":269942,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:55:46 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.84999999999999998,"15min":0.879},"min":{"1min":0.626,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.2450000000000001,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":1.1399999999999999},{"interface":"front","average":{"1min":0.92500000000000004,"5min":0.86299999999999999,"15min":0.86699999999999999},"min":{"1min":0.63,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.137,"5min":1.2050000000000001,"15min":1.2050000000000001},"last":0.91600000000000004}]},{"osd":1,"last update":"Sun Aug 27 21:55:46 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":0.89200000000000002,"15min":0.92000000000000004},"min":{"1min":0.71899999999999997,"5min":0.504,"15min":0.504},"max":{"1min":1.135,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.96799999999999997},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.90300000000000002,"15min":0.89300000000000002},"min":{"1min":0.67900000000000005,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.2170000000000001,"5min":1.605,"15min":1.605},"last":0.99299999999999999}]},{"osd":2,"last update":"Sun Aug 27 21:55:46 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.91000000000000003,"15min":0.90400000000000003},"min":{"1min":0.65100000000000002,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.153,"5min":1.7609999999999999,"15min":1.7609999999999999},"last":1.022},{"interface":"front","average":{"1min":0.95799999999999996,"5min":0.89800000000000002,"15min":0.91000000000000003},"min":{"1min":0.64000000000000001,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.177,"5min":1.264,"15min":1.264},"last":1.0489999999999999}]},{"osd":3,"last update":"Sun Aug 27 21:55:46 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.93999999999999995,"15min":0.93300000000000005},"min":{"1min":0.67200000000000004,"5min":0.503,"15min":0.503},"max":{"1min":1.246,"5min":1.587,"15min":1.587},"last":1.091},{"interface":"front","average":{"1min":1.0349999999999999,"5min":0.96699999999999997,"15min":0.95299999999999996},"min":{"1min":0.69999999999999996,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.331,"5min":1.506,"15min":1.506},"last":1.115}]},{"osd":4,"last update":"Sun Aug 27 21:55:46 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.85099999999999998,"15min":0.78700000000000003},"min":{"1min":0.65900000000000003,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.085,"5min":1.6919999999999999,"15min":1.6919999999999999},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.92500000000000004,"15min":0.80600000000000005},"min":{"1min":0.64400000000000002,"5min":0.442,"15min":0.442},"max":{"1min":1.333,"5min":1.538,"15min":1.538},"last":0.94799999999999995}]},{"osd":6,"last update":"Sun Aug 27 21:56:27 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":0.96899999999999997,"15min":0.95499999999999996},"min":{"1min":0.78400000000000003,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.1870000000000001,"5min":1.7050000000000001,"15min":1.7050000000000001},"last":0.88800000000000001},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.98399999999999999,"15min":0.97799999999999998},"min":{"1min":0.71799999999999997,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.264,"5min":1.4370000000000001,"15min":1.4370000000000001},"last":1.1619999999999999}]},{"osd":7,"last update":"Sun Aug 27 21:56:19 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":1.0449999999999999,"15min":1.0389999999999999},"min":{"1min":0.71199999999999997,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.2450000000000001,"5min":1.498,"15min":1.498},"last":1.204},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.0269999999999999,"15min":1.0029999999999999},"min":{"1min":0.75900000000000001,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.2789999999999999,"5min":1.8340000000000001,"15min":1.8340000000000001},"last":1.0740000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-27T21:56:40.675 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-27T21:56:40.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:56:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:56:40] "GET /metrics HTTP/1.1" 200 191530 "" "Prometheus/2.33.4" 2023-08-27T21:56:42.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:41 smithi130 bash[27579]: audit 2023-08-27T21:56:40.003847+0000 mgr.y (mgr.14430) 151 : audit [DBG] from='client.24677 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T21:56:42.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:41 smithi130 bash[27579]: cluster 2023-08-27T21:56:40.663141+0000 mgr.y (mgr.14430) 152 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:42.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:41 smithi006 bash[33918]: audit 2023-08-27T21:56:40.003847+0000 mgr.y (mgr.14430) 151 : audit [DBG] from='client.24677 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T21:56:42.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:41 smithi006 bash[33918]: cluster 2023-08-27T21:56:40.663141+0000 mgr.y (mgr.14430) 152 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:42.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:41 smithi006 bash[27292]: audit 2023-08-27T21:56:40.003847+0000 mgr.y (mgr.14430) 151 : audit [DBG] from='client.24677 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T21:56:42.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:41 smithi006 bash[27292]: cluster 2023-08-27T21:56:40.663141+0000 mgr.y (mgr.14430) 152 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:43.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:43 smithi130 bash[27579]: cluster 2023-08-27T21:56:42.663840+0000 mgr.y (mgr.14430) 153 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:43.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:43 smithi006 bash[33918]: cluster 2023-08-27T21:56:42.663840+0000 mgr.y (mgr.14430) 153 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:43.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:43 smithi006 bash[27292]: cluster 2023-08-27T21:56:42.663840+0000 mgr.y (mgr.14430) 153 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:45.542 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:56:45.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:45 smithi006 bash[33918]: cluster 2023-08-27T21:56:44.664437+0000 mgr.y (mgr.14430) 154 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:45.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:45 smithi006 bash[27292]: cluster 2023-08-27T21:56:44.664437+0000 mgr.y (mgr.14430) 154 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:46.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:45 smithi130 bash[27579]: cluster 2023-08-27T21:56:44.664437+0000 mgr.y (mgr.14430) 154 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:47.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:56:46 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:56:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:56:48.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:47 smithi130 bash[27579]: cluster 2023-08-27T21:56:46.665581+0000 mgr.y (mgr.14430) 155 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:48.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:47 smithi006 bash[27292]: cluster 2023-08-27T21:56:46.665581+0000 mgr.y (mgr.14430) 155 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:48.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:47 smithi006 bash[33918]: cluster 2023-08-27T21:56:46.665581+0000 mgr.y (mgr.14430) 155 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:48.651 INFO:teuthology.orchestra.run.smithi006.stdout: 2023-08-27T21:56:48.655 INFO:teuthology.orchestra.run.smithi006.stderr:dumped all 2023-08-27T21:56:49.376 INFO:teuthology.orchestra.run.smithi006.stdout:{"pg_ready":true,"pg_map":{"version":135,"stamp":"2023-08-27T21:56:46.664682+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381736,"kb_used_data":4968,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579480,"statfs":{"total":767960285184,"available":765521387520,"internally_reserved":0,"allocated":5087232,"data_stored":2164612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005033"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675155+0000","last_change":"2023-08-27T21:55:10.887197+0000","last_active":"2023-08-27T21:55:44.675155+0000","last_peered":"2023-08-27T21:55:44.675155+0000","last_clean":"2023-08-27T21:55:44.675155+0000","last_became_active":"2023-08-27T21:55:10.886486+0000","last_became_peered":"2023-08-27T21:55:10.886486+0000","last_unstale":"2023-08-27T21:55:44.675155+0000","last_undegraded":"2023-08-27T21:55:44.675155+0000","last_fullsized":"2023-08-27T21:55:44.675155+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.664184+0000","last_change":"2023-08-27T21:55:09.729805+0000","last_active":"2023-08-27T21:55:44.664184+0000","last_peered":"2023-08-27T21:55:44.664184+0000","last_clean":"2023-08-27T21:55:44.664184+0000","last_became_active":"2023-08-27T21:55:09.729492+0000","last_became_peered":"2023-08-27T21:55:09.729492+0000","last_unstale":"2023-08-27T21:55:44.664184+0000","last_undegraded":"2023-08-27T21:55:44.664184+0000","last_fullsized":"2023-08-27T21:55:44.664184+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676124+0000","last_change":"2023-08-27T21:55:09.751511+0000","last_active":"2023-08-27T21:55:44.676124+0000","last_peered":"2023-08-27T21:55:44.676124+0000","last_clean":"2023-08-27T21:55:44.676124+0000","last_became_active":"2023-08-27T21:55:09.751175+0000","last_became_peered":"2023-08-27T21:55:09.751175+0000","last_unstale":"2023-08-27T21:55:44.676124+0000","last_undegraded":"2023-08-27T21:55:44.676124+0000","last_fullsized":"2023-08-27T21:55:44.676124+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676575+0000","last_change":"2023-08-27T21:55:09.725081+0000","last_active":"2023-08-27T21:55:44.676575+0000","last_peered":"2023-08-27T21:55:44.676575+0000","last_clean":"2023-08-27T21:55:44.676575+0000","last_became_active":"2023-08-27T21:55:09.724636+0000","last_became_peered":"2023-08-27T21:55:09.724636+0000","last_unstale":"2023-08-27T21:55:44.676575+0000","last_undegraded":"2023-08-27T21:55:44.676575+0000","last_fullsized":"2023-08-27T21:55:44.676575+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675495+0000","last_change":"2023-08-27T21:55:10.753222+0000","last_active":"2023-08-27T21:55:44.675495+0000","last_peered":"2023-08-27T21:55:44.675495+0000","last_clean":"2023-08-27T21:55:44.675495+0000","last_became_active":"2023-08-27T21:55:10.752892+0000","last_became_peered":"2023-08-27T21:55:10.752892+0000","last_unstale":"2023-08-27T21:55:44.675495+0000","last_undegraded":"2023-08-27T21:55:44.675495+0000","last_fullsized":"2023-08-27T21:55:44.675495+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.667135+0000","last_change":"2023-08-27T21:55:09.737892+0000","last_active":"2023-08-27T21:55:44.667135+0000","last_peered":"2023-08-27T21:55:44.667135+0000","last_clean":"2023-08-27T21:55:44.667135+0000","last_became_active":"2023-08-27T21:55:09.737576+0000","last_became_peered":"2023-08-27T21:55:09.737576+0000","last_unstale":"2023-08-27T21:55:44.667135+0000","last_undegraded":"2023-08-27T21:55:44.667135+0000","last_fullsized":"2023-08-27T21:55:44.667135+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-08-27T21:55:45.731265+0000","last_change":"2023-08-27T21:55:45.731265+0000","last_active":"2023-08-27T21:55:45.731265+0000","last_peered":"2023-08-27T21:55:45.731265+0000","last_clean":"2023-08-27T21:55:45.731265+0000","last_became_active":"2023-08-27T21:55:45.730248+0000","last_became_peered":"2023-08-27T21:55:45.730248+0000","last_unstale":"2023-08-27T21:55:45.731265+0000","last_undegraded":"2023-08-27T21:55:45.731265+0000","last_fullsized":"2023-08-27T21:55:45.731265+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.666985+0000","last_change":"2023-08-27T21:55:09.744050+0000","last_active":"2023-08-27T21:55:44.666985+0000","last_peered":"2023-08-27T21:55:44.666985+0000","last_clean":"2023-08-27T21:55:44.666985+0000","last_became_active":"2023-08-27T21:55:09.743737+0000","last_became_peered":"2023-08-27T21:55:09.743737+0000","last_unstale":"2023-08-27T21:55:44.666985+0000","last_undegraded":"2023-08-27T21:55:44.666985+0000","last_fullsized":"2023-08-27T21:55:44.666985+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.664476+0000","last_change":"2023-08-27T21:55:09.745987+0000","last_active":"2023-08-27T21:55:44.664476+0000","last_peered":"2023-08-27T21:55:44.664476+0000","last_clean":"2023-08-27T21:55:44.664476+0000","last_became_active":"2023-08-27T21:55:09.744651+0000","last_became_peered":"2023-08-27T21:55:09.744651+0000","last_unstale":"2023-08-27T21:55:44.664476+0000","last_undegraded":"2023-08-27T21:55:44.664476+0000","last_fullsized":"2023-08-27T21:55:44.664476+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.664248+0000","last_change":"2023-08-27T21:55:09.745955+0000","last_active":"2023-08-27T21:55:44.664248+0000","last_peered":"2023-08-27T21:55:44.664248+0000","last_clean":"2023-08-27T21:55:44.664248+0000","last_became_active":"2023-08-27T21:55:09.745552+0000","last_became_peered":"2023-08-27T21:55:09.745552+0000","last_unstale":"2023-08-27T21:55:44.664248+0000","last_undegraded":"2023-08-27T21:55:44.664248+0000","last_fullsized":"2023-08-27T21:55:44.664248+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675558+0000","last_change":"2023-08-27T21:55:10.751504+0000","last_active":"2023-08-27T21:55:44.675558+0000","last_peered":"2023-08-27T21:55:44.675558+0000","last_clean":"2023-08-27T21:55:44.675558+0000","last_became_active":"2023-08-27T21:55:10.751164+0000","last_became_peered":"2023-08-27T21:55:10.751164+0000","last_unstale":"2023-08-27T21:55:44.675558+0000","last_undegraded":"2023-08-27T21:55:44.675558+0000","last_fullsized":"2023-08-27T21:55:44.675558+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.668799+0000","last_change":"2023-08-27T21:55:09.738206+0000","last_active":"2023-08-27T21:55:44.668799+0000","last_peered":"2023-08-27T21:55:44.668799+0000","last_clean":"2023-08-27T21:55:44.668799+0000","last_became_active":"2023-08-27T21:55:09.737859+0000","last_became_peered":"2023-08-27T21:55:09.737859+0000","last_unstale":"2023-08-27T21:55:44.668799+0000","last_undegraded":"2023-08-27T21:55:44.668799+0000","last_fullsized":"2023-08-27T21:55:44.668799+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.671207+0000","last_change":"2023-08-27T21:53:17.043470+0000","last_active":"2023-08-27T21:55:44.671207+0000","last_peered":"2023-08-27T21:55:44.671207+0000","last_clean":"2023-08-27T21:55:44.671207+0000","last_became_active":"2023-08-27T21:53:17.042696+0000","last_became_peered":"2023-08-27T21:53:17.042696+0000","last_unstale":"2023-08-27T21:55:44.671207+0000","last_undegraded":"2023-08-27T21:55:44.671207+0000","last_fullsized":"2023-08-27T21:55:44.671207+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675899+0000","last_change":"2023-08-27T21:55:10.193189+0000","last_active":"2023-08-27T21:55:44.675899+0000","last_peered":"2023-08-27T21:55:44.675899+0000","last_clean":"2023-08-27T21:55:44.675899+0000","last_became_active":"2023-08-27T21:55:10.192728+0000","last_became_peered":"2023-08-27T21:55:10.192728+0000","last_unstale":"2023-08-27T21:55:44.675899+0000","last_undegraded":"2023-08-27T21:55:44.675899+0000","last_fullsized":"2023-08-27T21:55:44.675899+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.672807+0000","last_change":"2023-08-27T21:53:17.187485+0000","last_active":"2023-08-27T21:55:44.672807+0000","last_peered":"2023-08-27T21:55:44.672807+0000","last_clean":"2023-08-27T21:55:44.672807+0000","last_became_active":"2023-08-27T21:53:17.187094+0000","last_became_peered":"2023-08-27T21:53:17.187094+0000","last_unstale":"2023-08-27T21:55:44.672807+0000","last_undegraded":"2023-08-27T21:55:44.672807+0000","last_fullsized":"2023-08-27T21:55:44.672807+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676664+0000","last_change":"2023-08-27T21:55:09.727972+0000","last_active":"2023-08-27T21:55:44.676664+0000","last_peered":"2023-08-27T21:55:44.676664+0000","last_clean":"2023-08-27T21:55:44.676664+0000","last_became_active":"2023-08-27T21:55:09.727636+0000","last_became_peered":"2023-08-27T21:55:09.727636+0000","last_unstale":"2023-08-27T21:55:44.676664+0000","last_undegraded":"2023-08-27T21:55:44.676664+0000","last_fullsized":"2023-08-27T21:55:44.676664+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.664229+0000","last_change":"2023-08-27T21:55:10.886864+0000","last_active":"2023-08-27T21:55:44.664229+0000","last_peered":"2023-08-27T21:55:44.664229+0000","last_clean":"2023-08-27T21:55:44.664229+0000","last_became_active":"2023-08-27T21:55:10.886272+0000","last_became_peered":"2023-08-27T21:55:10.886272+0000","last_unstale":"2023-08-27T21:55:44.664229+0000","last_undegraded":"2023-08-27T21:55:44.664229+0000","last_fullsized":"2023-08-27T21:55:44.664229+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.671429+0000","last_change":"2023-08-27T21:55:09.736041+0000","last_active":"2023-08-27T21:55:44.671429+0000","last_peered":"2023-08-27T21:55:44.671429+0000","last_clean":"2023-08-27T21:55:44.671429+0000","last_became_active":"2023-08-27T21:55:09.735653+0000","last_became_peered":"2023-08-27T21:55:09.735653+0000","last_unstale":"2023-08-27T21:55:44.671429+0000","last_undegraded":"2023-08-27T21:55:44.671429+0000","last_fullsized":"2023-08-27T21:55:44.671429+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.729582+0000","last_change":"2023-08-27T21:53:17.816851+0000","last_active":"2023-08-27T21:55:44.729582+0000","last_peered":"2023-08-27T21:55:44.729582+0000","last_clean":"2023-08-27T21:55:44.729582+0000","last_became_active":"2023-08-27T21:53:17.815511+0000","last_became_peered":"2023-08-27T21:53:17.815511+0000","last_unstale":"2023-08-27T21:55:44.729582+0000","last_undegraded":"2023-08-27T21:55:44.729582+0000","last_fullsized":"2023-08-27T21:55:44.729582+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-08-27T21:55:45.686659+0000","last_change":"2023-08-27T21:55:45.686659+0000","last_active":"2023-08-27T21:55:45.686659+0000","last_peered":"2023-08-27T21:55:45.686659+0000","last_clean":"2023-08-27T21:55:45.686659+0000","last_became_active":"2023-08-27T21:55:45.686225+0000","last_became_peered":"2023-08-27T21:55:45.686225+0000","last_unstale":"2023-08-27T21:55:45.686659+0000","last_undegraded":"2023-08-27T21:55:45.686659+0000","last_fullsized":"2023-08-27T21:55:45.686659+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.663847+0000","last_change":"2023-08-27T21:55:09.745846+0000","last_active":"2023-08-27T21:55:44.663847+0000","last_peered":"2023-08-27T21:55:44.663847+0000","last_clean":"2023-08-27T21:55:44.663847+0000","last_became_active":"2023-08-27T21:55:09.745486+0000","last_became_peered":"2023-08-27T21:55:09.745486+0000","last_unstale":"2023-08-27T21:55:44.663847+0000","last_undegraded":"2023-08-27T21:55:44.663847+0000","last_fullsized":"2023-08-27T21:55:44.663847+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.729272+0000","last_change":"2023-08-27T21:53:17.815961+0000","last_active":"2023-08-27T21:55:44.729272+0000","last_peered":"2023-08-27T21:55:44.729272+0000","last_clean":"2023-08-27T21:55:44.729272+0000","last_became_active":"2023-08-27T21:53:17.815034+0000","last_became_peered":"2023-08-27T21:53:17.815034+0000","last_unstale":"2023-08-27T21:55:44.729272+0000","last_undegraded":"2023-08-27T21:55:44.729272+0000","last_fullsized":"2023-08-27T21:55:44.729272+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.670832+0000","last_change":"2023-08-27T21:55:10.891543+0000","last_active":"2023-08-27T21:55:44.670832+0000","last_peered":"2023-08-27T21:55:44.670832+0000","last_clean":"2023-08-27T21:55:44.670832+0000","last_became_active":"2023-08-27T21:55:10.891155+0000","last_became_peered":"2023-08-27T21:55:10.891155+0000","last_unstale":"2023-08-27T21:55:44.670832+0000","last_undegraded":"2023-08-27T21:55:44.670832+0000","last_fullsized":"2023-08-27T21:55:44.670832+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676207+0000","last_change":"2023-08-27T21:55:09.728432+0000","last_active":"2023-08-27T21:55:44.676207+0000","last_peered":"2023-08-27T21:55:44.676207+0000","last_clean":"2023-08-27T21:55:44.676207+0000","last_became_active":"2023-08-27T21:55:09.728135+0000","last_became_peered":"2023-08-27T21:55:09.728135+0000","last_unstale":"2023-08-27T21:55:44.676207+0000","last_undegraded":"2023-08-27T21:55:44.676207+0000","last_fullsized":"2023-08-27T21:55:44.676207+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.729052+0000","last_change":"2023-08-27T21:52:11.532578+0000","last_active":"2023-08-27T21:55:44.729052+0000","last_peered":"2023-08-27T21:55:44.729052+0000","last_clean":"2023-08-27T21:55:44.729052+0000","last_became_active":"2023-08-27T21:52:11.531739+0000","last_became_peered":"2023-08-27T21:52:11.531739+0000","last_unstale":"2023-08-27T21:55:44.729052+0000","last_undegraded":"2023-08-27T21:55:44.729052+0000","last_fullsized":"2023-08-27T21:55:44.729052+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:48:22.416466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:48:22.416466+0000","last_clean_scrub_stamp":"2023-08-27T21:48:22.416466+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.666628+0000","last_change":"2023-08-27T21:53:17.043700+0000","last_active":"2023-08-27T21:55:44.666628+0000","last_peered":"2023-08-27T21:55:44.666628+0000","last_clean":"2023-08-27T21:55:44.666628+0000","last_became_active":"2023-08-27T21:53:17.043332+0000","last_became_peered":"2023-08-27T21:53:17.043332+0000","last_unstale":"2023-08-27T21:55:44.666628+0000","last_undegraded":"2023-08-27T21:55:44.666628+0000","last_fullsized":"2023-08-27T21:55:44.666628+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.672796+0000","last_change":"2023-08-27T21:55:10.172334+0000","last_active":"2023-08-27T21:55:44.672796+0000","last_peered":"2023-08-27T21:55:44.672796+0000","last_clean":"2023-08-27T21:55:44.672796+0000","last_became_active":"2023-08-27T21:55:10.171993+0000","last_became_peered":"2023-08-27T21:55:10.171993+0000","last_unstale":"2023-08-27T21:55:44.672796+0000","last_undegraded":"2023-08-27T21:55:44.672796+0000","last_fullsized":"2023-08-27T21:55:44.672796+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.670878+0000","last_change":"2023-08-27T21:55:09.735816+0000","last_active":"2023-08-27T21:55:44.670878+0000","last_peered":"2023-08-27T21:55:44.670878+0000","last_clean":"2023-08-27T21:55:44.670878+0000","last_became_active":"2023-08-27T21:55:09.735470+0000","last_became_peered":"2023-08-27T21:55:09.735470+0000","last_unstale":"2023-08-27T21:55:44.670878+0000","last_undegraded":"2023-08-27T21:55:44.670878+0000","last_fullsized":"2023-08-27T21:55:44.670878+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.664745+0000","last_change":"2023-08-27T21:53:18.019051+0000","last_active":"2023-08-27T21:55:44.664745+0000","last_peered":"2023-08-27T21:55:44.664745+0000","last_clean":"2023-08-27T21:55:44.664745+0000","last_became_active":"2023-08-27T21:53:18.018616+0000","last_became_peered":"2023-08-27T21:53:18.018616+0000","last_unstale":"2023-08-27T21:55:44.664745+0000","last_undegraded":"2023-08-27T21:55:44.664745+0000","last_fullsized":"2023-08-27T21:55:44.664745+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.671300+0000","last_change":"2023-08-27T21:55:09.750817+0000","last_active":"2023-08-27T21:55:44.671300+0000","last_peered":"2023-08-27T21:55:44.671300+0000","last_clean":"2023-08-27T21:55:44.671300+0000","last_became_active":"2023-08-27T21:55:09.750495+0000","last_became_peered":"2023-08-27T21:55:09.750495+0000","last_unstale":"2023-08-27T21:55:44.671300+0000","last_undegraded":"2023-08-27T21:55:44.671300+0000","last_fullsized":"2023-08-27T21:55:44.671300+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675578+0000","last_change":"2023-08-27T21:55:09.738412+0000","last_active":"2023-08-27T21:55:44.675578+0000","last_peered":"2023-08-27T21:55:44.675578+0000","last_clean":"2023-08-27T21:55:44.675578+0000","last_became_active":"2023-08-27T21:55:09.736674+0000","last_became_peered":"2023-08-27T21:55:09.736674+0000","last_unstale":"2023-08-27T21:55:44.675578+0000","last_undegraded":"2023-08-27T21:55:44.675578+0000","last_fullsized":"2023-08-27T21:55:44.675578+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.666459+0000","last_change":"2023-08-27T21:53:17.043427+0000","last_active":"2023-08-27T21:55:44.666459+0000","last_peered":"2023-08-27T21:55:44.666459+0000","last_clean":"2023-08-27T21:55:44.666459+0000","last_became_active":"2023-08-27T21:53:17.043025+0000","last_became_peered":"2023-08-27T21:53:17.043025+0000","last_unstale":"2023-08-27T21:55:44.666459+0000","last_undegraded":"2023-08-27T21:55:44.666459+0000","last_fullsized":"2023-08-27T21:55:44.666459+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.671135+0000","last_change":"2023-08-27T21:55:10.172077+0000","last_active":"2023-08-27T21:55:44.671135+0000","last_peered":"2023-08-27T21:55:44.671135+0000","last_clean":"2023-08-27T21:55:44.671135+0000","last_became_active":"2023-08-27T21:55:10.171767+0000","last_became_peered":"2023-08-27T21:55:10.171767+0000","last_unstale":"2023-08-27T21:55:44.671135+0000","last_undegraded":"2023-08-27T21:55:44.671135+0000","last_fullsized":"2023-08-27T21:55:44.671135+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.667080+0000","last_change":"2023-08-27T21:55:09.739638+0000","last_active":"2023-08-27T21:55:44.667080+0000","last_peered":"2023-08-27T21:55:44.667080+0000","last_clean":"2023-08-27T21:55:44.667080+0000","last_became_active":"2023-08-27T21:55:09.739305+0000","last_became_peered":"2023-08-27T21:55:09.739305+0000","last_unstale":"2023-08-27T21:55:44.667080+0000","last_undegraded":"2023-08-27T21:55:44.667080+0000","last_fullsized":"2023-08-27T21:55:44.667080+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.671180+0000","last_change":"2023-08-27T21:53:17.813356+0000","last_active":"2023-08-27T21:55:44.671180+0000","last_peered":"2023-08-27T21:55:44.671180+0000","last_clean":"2023-08-27T21:55:44.671180+0000","last_became_active":"2023-08-27T21:53:17.812986+0000","last_became_peered":"2023-08-27T21:53:17.812986+0000","last_unstale":"2023-08-27T21:55:44.671180+0000","last_undegraded":"2023-08-27T21:55:44.671180+0000","last_fullsized":"2023-08-27T21:55:44.671180+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:53:16.022279+0000","last_clean_scrub_stamp":"2023-08-27T21:53:16.022279+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.672913+0000","last_change":"2023-08-27T21:55:09.750798+0000","last_active":"2023-08-27T21:55:44.672913+0000","last_peered":"2023-08-27T21:55:44.672913+0000","last_clean":"2023-08-27T21:55:44.672913+0000","last_became_active":"2023-08-27T21:55:09.750478+0000","last_became_peered":"2023-08-27T21:55:09.750478+0000","last_unstale":"2023-08-27T21:55:44.672913+0000","last_undegraded":"2023-08-27T21:55:44.672913+0000","last_fullsized":"2023-08-27T21:55:44.672913+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675959+0000","last_change":"2023-08-27T21:55:09.721249+0000","last_active":"2023-08-27T21:55:44.675959+0000","last_peered":"2023-08-27T21:55:44.675959+0000","last_clean":"2023-08-27T21:55:44.675959+0000","last_became_active":"2023-08-27T21:55:09.720806+0000","last_became_peered":"2023-08-27T21:55:09.720806+0000","last_unstale":"2023-08-27T21:55:44.675959+0000","last_undegraded":"2023-08-27T21:55:44.675959+0000","last_fullsized":"2023-08-27T21:55:44.675959+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676048+0000","last_change":"2023-08-27T21:55:10.887095+0000","last_active":"2023-08-27T21:55:44.676048+0000","last_peered":"2023-08-27T21:55:44.676048+0000","last_clean":"2023-08-27T21:55:44.676048+0000","last_became_active":"2023-08-27T21:55:10.886294+0000","last_became_peered":"2023-08-27T21:55:10.886294+0000","last_unstale":"2023-08-27T21:55:44.676048+0000","last_undegraded":"2023-08-27T21:55:44.676048+0000","last_fullsized":"2023-08-27T21:55:44.676048+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675445+0000","last_change":"2023-08-27T21:55:09.731403+0000","last_active":"2023-08-27T21:55:44.675445+0000","last_peered":"2023-08-27T21:55:44.675445+0000","last_clean":"2023-08-27T21:55:44.675445+0000","last_became_active":"2023-08-27T21:55:09.730437+0000","last_became_peered":"2023-08-27T21:55:09.730437+0000","last_unstale":"2023-08-27T21:55:44.675445+0000","last_undegraded":"2023-08-27T21:55:44.675445+0000","last_fullsized":"2023-08-27T21:55:44.675445+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676049+0000","last_change":"2023-08-27T21:55:10.887669+0000","last_active":"2023-08-27T21:55:44.676049+0000","last_peered":"2023-08-27T21:55:44.676049+0000","last_clean":"2023-08-27T21:55:44.676049+0000","last_became_active":"2023-08-27T21:55:10.887069+0000","last_became_peered":"2023-08-27T21:55:10.887069+0000","last_unstale":"2023-08-27T21:55:44.676049+0000","last_undegraded":"2023-08-27T21:55:44.676049+0000","last_fullsized":"2023-08-27T21:55:44.676049+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.671341+0000","last_change":"2023-08-27T21:55:09.744571+0000","last_active":"2023-08-27T21:55:44.671341+0000","last_peered":"2023-08-27T21:55:44.671341+0000","last_clean":"2023-08-27T21:55:44.671341+0000","last_became_active":"2023-08-27T21:55:09.743932+0000","last_became_peered":"2023-08-27T21:55:09.743932+0000","last_unstale":"2023-08-27T21:55:44.671341+0000","last_undegraded":"2023-08-27T21:55:44.671341+0000","last_fullsized":"2023-08-27T21:55:44.671341+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675533+0000","last_change":"2023-08-27T21:55:10.750933+0000","last_active":"2023-08-27T21:55:44.675533+0000","last_peered":"2023-08-27T21:55:44.675533+0000","last_clean":"2023-08-27T21:55:44.675533+0000","last_became_active":"2023-08-27T21:55:10.750599+0000","last_became_peered":"2023-08-27T21:55:10.750599+0000","last_unstale":"2023-08-27T21:55:44.675533+0000","last_undegraded":"2023-08-27T21:55:44.675533+0000","last_fullsized":"2023-08-27T21:55:44.675533+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676646+0000","last_change":"2023-08-27T21:55:09.727541+0000","last_active":"2023-08-27T21:55:44.676646+0000","last_peered":"2023-08-27T21:55:44.676646+0000","last_clean":"2023-08-27T21:55:44.676646+0000","last_became_active":"2023-08-27T21:55:09.727136+0000","last_became_peered":"2023-08-27T21:55:09.727136+0000","last_unstale":"2023-08-27T21:55:44.676646+0000","last_undegraded":"2023-08-27T21:55:44.676646+0000","last_fullsized":"2023-08-27T21:55:44.676646+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.664327+0000","last_change":"2023-08-27T21:55:10.888097+0000","last_active":"2023-08-27T21:55:44.664327+0000","last_peered":"2023-08-27T21:55:44.664327+0000","last_clean":"2023-08-27T21:55:44.664327+0000","last_became_active":"2023-08-27T21:55:10.887680+0000","last_became_peered":"2023-08-27T21:55:10.887680+0000","last_unstale":"2023-08-27T21:55:44.664327+0000","last_undegraded":"2023-08-27T21:55:44.664327+0000","last_fullsized":"2023-08-27T21:55:44.664327+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.666839+0000","last_change":"2023-08-27T21:55:09.738841+0000","last_active":"2023-08-27T21:55:44.666839+0000","last_peered":"2023-08-27T21:55:44.666839+0000","last_clean":"2023-08-27T21:55:44.666839+0000","last_became_active":"2023-08-27T21:55:09.738487+0000","last_became_peered":"2023-08-27T21:55:09.738487+0000","last_unstale":"2023-08-27T21:55:44.666839+0000","last_undegraded":"2023-08-27T21:55:44.666839+0000","last_fullsized":"2023-08-27T21:55:44.666839+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.671211+0000","last_change":"2023-08-27T21:55:10.889355+0000","last_active":"2023-08-27T21:55:44.671211+0000","last_peered":"2023-08-27T21:55:44.671211+0000","last_clean":"2023-08-27T21:55:44.671211+0000","last_became_active":"2023-08-27T21:55:10.888510+0000","last_became_peered":"2023-08-27T21:55:10.888510+0000","last_unstale":"2023-08-27T21:55:44.671211+0000","last_undegraded":"2023-08-27T21:55:44.671211+0000","last_fullsized":"2023-08-27T21:55:44.671211+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.729785+0000","last_change":"2023-08-27T21:55:09.731816+0000","last_active":"2023-08-27T21:55:44.729785+0000","last_peered":"2023-08-27T21:55:44.729785+0000","last_clean":"2023-08-27T21:55:44.729785+0000","last_became_active":"2023-08-27T21:55:09.731471+0000","last_became_peered":"2023-08-27T21:55:09.731471+0000","last_unstale":"2023-08-27T21:55:44.729785+0000","last_undegraded":"2023-08-27T21:55:44.729785+0000","last_fullsized":"2023-08-27T21:55:44.729785+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675278+0000","last_change":"2023-08-27T21:55:10.887722+0000","last_active":"2023-08-27T21:55:44.675278+0000","last_peered":"2023-08-27T21:55:44.675278+0000","last_clean":"2023-08-27T21:55:44.675278+0000","last_became_active":"2023-08-27T21:55:10.887139+0000","last_became_peered":"2023-08-27T21:55:10.887139+0000","last_unstale":"2023-08-27T21:55:44.675278+0000","last_undegraded":"2023-08-27T21:55:44.675278+0000","last_fullsized":"2023-08-27T21:55:44.675278+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.729706+0000","last_change":"2023-08-27T21:55:09.733184+0000","last_active":"2023-08-27T21:55:44.729706+0000","last_peered":"2023-08-27T21:55:44.729706+0000","last_clean":"2023-08-27T21:55:44.729706+0000","last_became_active":"2023-08-27T21:55:09.732871+0000","last_became_peered":"2023-08-27T21:55:09.732871+0000","last_unstale":"2023-08-27T21:55:44.729706+0000","last_undegraded":"2023-08-27T21:55:44.729706+0000","last_fullsized":"2023-08-27T21:55:44.729706+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.666560+0000","last_change":"2023-08-27T21:55:10.886227+0000","last_active":"2023-08-27T21:55:44.666560+0000","last_peered":"2023-08-27T21:55:44.666560+0000","last_clean":"2023-08-27T21:55:44.666560+0000","last_became_active":"2023-08-27T21:55:10.885594+0000","last_became_peered":"2023-08-27T21:55:10.885594+0000","last_unstale":"2023-08-27T21:55:44.666560+0000","last_undegraded":"2023-08-27T21:55:44.666560+0000","last_fullsized":"2023-08-27T21:55:44.666560+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.729834+0000","last_change":"2023-08-27T21:55:09.734175+0000","last_active":"2023-08-27T21:55:44.729834+0000","last_peered":"2023-08-27T21:55:44.729834+0000","last_clean":"2023-08-27T21:55:44.729834+0000","last_became_active":"2023-08-27T21:55:09.733839+0000","last_became_peered":"2023-08-27T21:55:09.733839+0000","last_unstale":"2023-08-27T21:55:44.729834+0000","last_undegraded":"2023-08-27T21:55:44.729834+0000","last_fullsized":"2023-08-27T21:55:44.729834+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676490+0000","last_change":"2023-08-27T21:55:10.193452+0000","last_active":"2023-08-27T21:55:44.676490+0000","last_peered":"2023-08-27T21:55:44.676490+0000","last_clean":"2023-08-27T21:55:44.676490+0000","last_became_active":"2023-08-27T21:55:10.192817+0000","last_became_peered":"2023-08-27T21:55:10.192817+0000","last_unstale":"2023-08-27T21:55:44.676490+0000","last_undegraded":"2023-08-27T21:55:44.676490+0000","last_fullsized":"2023-08-27T21:55:44.676490+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.670492+0000","last_change":"2023-08-27T21:55:09.729934+0000","last_active":"2023-08-27T21:55:44.670492+0000","last_peered":"2023-08-27T21:55:44.670492+0000","last_clean":"2023-08-27T21:55:44.670492+0000","last_became_active":"2023-08-27T21:55:09.729533+0000","last_became_peered":"2023-08-27T21:55:09.729533+0000","last_unstale":"2023-08-27T21:55:44.670492+0000","last_undegraded":"2023-08-27T21:55:44.670492+0000","last_fullsized":"2023-08-27T21:55:44.670492+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.668595+0000","last_change":"2023-08-27T21:55:10.886329+0000","last_active":"2023-08-27T21:55:44.668595+0000","last_peered":"2023-08-27T21:55:44.668595+0000","last_clean":"2023-08-27T21:55:44.668595+0000","last_became_active":"2023-08-27T21:55:10.885846+0000","last_became_peered":"2023-08-27T21:55:10.885846+0000","last_unstale":"2023-08-27T21:55:44.668595+0000","last_undegraded":"2023-08-27T21:55:44.668595+0000","last_fullsized":"2023-08-27T21:55:44.668595+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.728871+0000","last_change":"2023-08-27T21:55:09.734222+0000","last_active":"2023-08-27T21:55:44.728871+0000","last_peered":"2023-08-27T21:55:44.728871+0000","last_clean":"2023-08-27T21:55:44.728871+0000","last_became_active":"2023-08-27T21:55:09.733665+0000","last_became_peered":"2023-08-27T21:55:09.733665+0000","last_unstale":"2023-08-27T21:55:44.728871+0000","last_undegraded":"2023-08-27T21:55:44.728871+0000","last_fullsized":"2023-08-27T21:55:44.728871+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.666928+0000","last_change":"2023-08-27T21:55:10.888185+0000","last_active":"2023-08-27T21:55:44.666928+0000","last_peered":"2023-08-27T21:55:44.666928+0000","last_clean":"2023-08-27T21:55:44.666928+0000","last_became_active":"2023-08-27T21:55:10.887602+0000","last_became_peered":"2023-08-27T21:55:10.887602+0000","last_unstale":"2023-08-27T21:55:44.666928+0000","last_undegraded":"2023-08-27T21:55:44.666928+0000","last_fullsized":"2023-08-27T21:55:44.666928+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.665392+0000","last_change":"2023-08-27T21:55:09.731963+0000","last_active":"2023-08-27T21:55:44.665392+0000","last_peered":"2023-08-27T21:55:44.665392+0000","last_clean":"2023-08-27T21:55:44.665392+0000","last_became_active":"2023-08-27T21:55:09.731543+0000","last_became_peered":"2023-08-27T21:55:09.731543+0000","last_unstale":"2023-08-27T21:55:44.665392+0000","last_undegraded":"2023-08-27T21:55:44.665392+0000","last_fullsized":"2023-08-27T21:55:44.665392+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676415+0000","last_change":"2023-08-27T21:55:10.193438+0000","last_active":"2023-08-27T21:55:44.676415+0000","last_peered":"2023-08-27T21:55:44.676415+0000","last_clean":"2023-08-27T21:55:44.676415+0000","last_became_active":"2023-08-27T21:55:10.192753+0000","last_became_peered":"2023-08-27T21:55:10.192753+0000","last_unstale":"2023-08-27T21:55:44.676415+0000","last_undegraded":"2023-08-27T21:55:44.676415+0000","last_fullsized":"2023-08-27T21:55:44.676415+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.729314+0000","last_change":"2023-08-27T21:55:09.734742+0000","last_active":"2023-08-27T21:55:44.729314+0000","last_peered":"2023-08-27T21:55:44.729314+0000","last_clean":"2023-08-27T21:55:44.729314+0000","last_became_active":"2023-08-27T21:55:09.734348+0000","last_became_peered":"2023-08-27T21:55:09.734348+0000","last_unstale":"2023-08-27T21:55:44.729314+0000","last_undegraded":"2023-08-27T21:55:44.729314+0000","last_fullsized":"2023-08-27T21:55:44.729314+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675678+0000","last_change":"2023-08-27T21:55:10.889499+0000","last_active":"2023-08-27T21:55:44.675678+0000","last_peered":"2023-08-27T21:55:44.675678+0000","last_clean":"2023-08-27T21:55:44.675678+0000","last_became_active":"2023-08-27T21:55:10.889097+0000","last_became_peered":"2023-08-27T21:55:10.889097+0000","last_unstale":"2023-08-27T21:55:44.675678+0000","last_undegraded":"2023-08-27T21:55:44.675678+0000","last_fullsized":"2023-08-27T21:55:44.675678+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":20,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675807+0000","last_change":"2023-08-27T21:55:09.734659+0000","last_active":"2023-08-27T21:55:44.675807+0000","last_peered":"2023-08-27T21:55:44.675807+0000","last_clean":"2023-08-27T21:55:44.675807+0000","last_became_active":"2023-08-27T21:55:09.734326+0000","last_became_peered":"2023-08-27T21:55:09.734326+0000","last_unstale":"2023-08-27T21:55:44.675807+0000","last_undegraded":"2023-08-27T21:55:44.675807+0000","last_fullsized":"2023-08-27T21:55:44.675807+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":10,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-08-27T21:55:45.730896+0000","last_change":"2023-08-27T21:55:45.730896+0000","last_active":"2023-08-27T21:55:45.730896+0000","last_peered":"2023-08-27T21:55:45.730896+0000","last_clean":"2023-08-27T21:55:45.730896+0000","last_became_active":"2023-08-27T21:55:45.730202+0000","last_became_peered":"2023-08-27T21:55:45.730202+0000","last_unstale":"2023-08-27T21:55:45.730896+0000","last_undegraded":"2023-08-27T21:55:45.730896+0000","last_fullsized":"2023-08-27T21:55:45.730896+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.729620+0000","last_change":"2023-08-27T21:55:09.735453+0000","last_active":"2023-08-27T21:55:44.729620+0000","last_peered":"2023-08-27T21:55:44.729620+0000","last_clean":"2023-08-27T21:55:44.729620+0000","last_became_active":"2023-08-27T21:55:09.735100+0000","last_became_peered":"2023-08-27T21:55:09.735100+0000","last_unstale":"2023-08-27T21:55:44.729620+0000","last_undegraded":"2023-08-27T21:55:44.729620+0000","last_fullsized":"2023-08-27T21:55:44.729620+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676137+0000","last_change":"2023-08-27T21:55:10.226811+0000","last_active":"2023-08-27T21:55:44.676137+0000","last_peered":"2023-08-27T21:55:44.676137+0000","last_clean":"2023-08-27T21:55:44.676137+0000","last_became_active":"2023-08-27T21:55:10.226484+0000","last_became_peered":"2023-08-27T21:55:10.226484+0000","last_unstale":"2023-08-27T21:55:44.676137+0000","last_undegraded":"2023-08-27T21:55:44.676137+0000","last_fullsized":"2023-08-27T21:55:44.676137+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.666763+0000","last_change":"2023-08-27T21:55:09.744095+0000","last_active":"2023-08-27T21:55:44.666763+0000","last_peered":"2023-08-27T21:55:44.666763+0000","last_clean":"2023-08-27T21:55:44.666763+0000","last_became_active":"2023-08-27T21:55:09.743755+0000","last_became_peered":"2023-08-27T21:55:09.743755+0000","last_unstale":"2023-08-27T21:55:44.666763+0000","last_undegraded":"2023-08-27T21:55:44.666763+0000","last_fullsized":"2023-08-27T21:55:44.666763+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675974+0000","last_change":"2023-08-27T21:55:10.227656+0000","last_active":"2023-08-27T21:55:44.675974+0000","last_peered":"2023-08-27T21:55:44.675974+0000","last_clean":"2023-08-27T21:55:44.675974+0000","last_became_active":"2023-08-27T21:55:10.227206+0000","last_became_peered":"2023-08-27T21:55:10.227206+0000","last_unstale":"2023-08-27T21:55:44.675974+0000","last_undegraded":"2023-08-27T21:55:44.675974+0000","last_fullsized":"2023-08-27T21:55:44.675974+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.664033+0000","last_change":"2023-08-27T21:55:09.728718+0000","last_active":"2023-08-27T21:55:44.664033+0000","last_peered":"2023-08-27T21:55:44.664033+0000","last_clean":"2023-08-27T21:55:44.664033+0000","last_became_active":"2023-08-27T21:55:09.728363+0000","last_became_peered":"2023-08-27T21:55:09.728363+0000","last_unstale":"2023-08-27T21:55:44.664033+0000","last_undegraded":"2023-08-27T21:55:44.664033+0000","last_fullsized":"2023-08-27T21:55:44.664033+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.671008+0000","last_change":"2023-08-27T21:55:10.891642+0000","last_active":"2023-08-27T21:55:44.671008+0000","last_peered":"2023-08-27T21:55:44.671008+0000","last_clean":"2023-08-27T21:55:44.671008+0000","last_became_active":"2023-08-27T21:55:10.890828+0000","last_became_peered":"2023-08-27T21:55:10.890828+0000","last_unstale":"2023-08-27T21:55:44.671008+0000","last_undegraded":"2023-08-27T21:55:44.671008+0000","last_fullsized":"2023-08-27T21:55:44.671008+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676081+0000","last_change":"2023-08-27T21:55:09.722569+0000","last_active":"2023-08-27T21:55:44.676081+0000","last_peered":"2023-08-27T21:55:44.676081+0000","last_clean":"2023-08-27T21:55:44.676081+0000","last_became_active":"2023-08-27T21:55:09.722184+0000","last_became_peered":"2023-08-27T21:55:09.722184+0000","last_unstale":"2023-08-27T21:55:44.676081+0000","last_undegraded":"2023-08-27T21:55:44.676081+0000","last_fullsized":"2023-08-27T21:55:44.676081+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.668689+0000","last_change":"2023-08-27T21:55:10.886745+0000","last_active":"2023-08-27T21:55:44.668689+0000","last_peered":"2023-08-27T21:55:44.668689+0000","last_clean":"2023-08-27T21:55:44.668689+0000","last_became_active":"2023-08-27T21:55:10.886364+0000","last_became_peered":"2023-08-27T21:55:10.886364+0000","last_unstale":"2023-08-27T21:55:44.668689+0000","last_undegraded":"2023-08-27T21:55:44.668689+0000","last_fullsized":"2023-08-27T21:55:44.668689+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.669523+0000","last_change":"2023-08-27T21:55:09.741809+0000","last_active":"2023-08-27T21:55:44.669523+0000","last_peered":"2023-08-27T21:55:44.669523+0000","last_clean":"2023-08-27T21:55:44.669523+0000","last_became_active":"2023-08-27T21:55:09.741221+0000","last_became_peered":"2023-08-27T21:55:09.741221+0000","last_unstale":"2023-08-27T21:55:44.669523+0000","last_undegraded":"2023-08-27T21:55:44.669523+0000","last_fullsized":"2023-08-27T21:55:44.669523+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.676459+0000","last_change":"2023-08-27T21:55:09.751600+0000","last_active":"2023-08-27T21:55:44.676459+0000","last_peered":"2023-08-27T21:55:44.676459+0000","last_clean":"2023-08-27T21:55:44.676459+0000","last_became_active":"2023-08-27T21:55:09.751332+0000","last_became_peered":"2023-08-27T21:55:09.751332+0000","last_unstale":"2023-08-27T21:55:44.676459+0000","last_undegraded":"2023-08-27T21:55:44.676459+0000","last_fullsized":"2023-08-27T21:55:44.676459+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:09.700295+0000","last_clean_scrub_stamp":"2023-08-27T21:55:09.700295+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-08-27T21:55:44.675384+0000","last_change":"2023-08-27T21:55:09.746051+0000","last_active":"2023-08-27T21:55:44.675384+0000","last_peered":"2023-08-27T21:55:44.675384+0000","last_clean":"2023-08-27T21:55:44.675384+0000","last_became_active":"2023-08-27T21:55:09.745813+0000","last_became_peered":"2023-08-27T21:55:09.745813+0000","last_unstale":"2023-08-27T21:55:44.675384+0000","last_undegraded":"2023-08-27T21:55:44.675384+0000","last_fullsized":"2023-08-27T21:55:44.675384+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T21:55:08.684812+0000","last_clean_scrub_stamp":"2023-08-27T21:55:08.684812+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430266,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297736,"kb_used_data":640,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447416,"statfs":{"total":95995035648,"available":95690153984,"internally_reserved":0,"allocated":655360,"data_stored":271700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:56:20 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":0.89300000000000002,"15min":0.88900000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.2090000000000001,"5min":1.774,"15min":1.774},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.88200000000000001,"15min":0.878},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.1739999999999999,"5min":1.3600000000000001,"15min":1.3600000000000001},"last":1.0409999999999999}]},{"osd":1,"last update":"Sun Aug 27 21:56:20 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.90300000000000002,"15min":0.89100000000000001},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.1919999999999999,"5min":1.696,"15min":1.696},"last":1.1579999999999999},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.91100000000000003,"15min":0.92300000000000004},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.2350000000000001,"5min":1.4390000000000001,"15min":1.4390000000000001},"last":0.90400000000000003}]},{"osd":2,"last update":"Sun Aug 27 21:56:20 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.93899999999999995,"15min":0.93500000000000005},"min":{"1min":0.64500000000000002,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.2450000000000001,"5min":1.8580000000000001,"15min":1.8580000000000001},"last":0.96299999999999997},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.91200000000000003,"15min":0.90100000000000002},"min":{"1min":0.63,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.143,"5min":1.8009999999999999,"15min":1.8009999999999999},"last":0.876}]},{"osd":3,"last update":"Sun Aug 27 21:56:20 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.93600000000000005,"15min":0.94399999999999995},"min":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.2170000000000001,"5min":1.931,"15min":1.931},"last":0.84899999999999998},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.97399999999999998,"15min":0.95199999999999996},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.244,"5min":1.913,"15min":1.913},"last":1.0029999999999999}]},{"osd":4,"last update":"Sun Aug 27 21:56:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.89900000000000002,"15min":0.873},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.1870000000000001,"5min":1.889,"15min":1.889},"last":0.98899999999999999},{"interface":"front","average":{"1min":0.97699999999999998,"5min":0.93200000000000005,"15min":0.90300000000000002},"min":{"1min":0.59899999999999998,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.276,"5min":2.0049999999999999,"15min":2.0049999999999999},"last":1.0940000000000001}]},{"osd":5,"last update":"Sun Aug 27 21:56:20 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.91800000000000004,"15min":0.90000000000000002},"min":{"1min":0.56599999999999995,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.1890000000000001,"5min":2.0259999999999998,"15min":2.0259999999999998},"last":1.073},{"interface":"front","average":{"1min":1.0249999999999999,"5min":1.024,"15min":1.028},"min":{"1min":0.61699999999999999,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.329,"5min":2.1059999999999999,"15min":2.1059999999999999},"last":1.1319999999999999}]},{"osd":6,"last update":"Sun Aug 27 21:56:20 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.036,"15min":1.012},"min":{"1min":0.57899999999999996,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.3149999999999999,"5min":1.5389999999999999,"15min":1.5389999999999999},"last":1.2130000000000001},{"interface":"front","average":{"1min":1.125,"5min":1.071,"15min":1.052},"min":{"1min":0.75900000000000001,"5min":0.66300000000000003,"15min":0.66300000000000003},"max":{"1min":1.3759999999999999,"5min":2.0870000000000002,"15min":2.0870000000000002},"last":1.1899999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626500,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:56:27 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.83699999999999997,"15min":0.85599999999999998},"min":{"1min":0.60499999999999998,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.2849999999999999,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":1.0449999999999999},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.90100000000000002,"15min":0.85199999999999998},"min":{"1min":0.54700000000000004,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.254,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":1.0549999999999999}]},{"osd":1,"last update":"Sun Aug 27 21:56:27 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.89200000000000002,"15min":0.93400000000000005},"min":{"1min":0.61599999999999999,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.1080000000000001,"5min":1.629,"15min":1.629},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.88,"5min":0.90700000000000003,"15min":0.86499999999999999},"min":{"1min":0.66300000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.1319999999999999,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":0.97999999999999998}]},{"osd":2,"last update":"Sun Aug 27 21:56:27 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.94299999999999995,"15min":0.91900000000000004},"min":{"1min":0.63800000000000001,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.3720000000000001,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":1.1160000000000001},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.91700000000000004,"15min":0.92700000000000005},"min":{"1min":0.60999999999999999,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.2370000000000001,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":0.875}]},{"osd":3,"last update":"Sun Aug 27 21:56:27 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.93200000000000005,"15min":0.97399999999999998},"min":{"1min":0.624,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.125,"5min":1.752,"15min":1.752},"last":1.087},{"interface":"front","average":{"1min":1.0409999999999999,"5min":0.97899999999999998,"15min":0.98099999999999998},"min":{"1min":0.79500000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.3420000000000001,"5min":1.546,"15min":1.546},"last":1.1639999999999999}]},{"osd":4,"last update":"Sun Aug 27 21:56:27 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.90200000000000002,"15min":0.83399999999999996},"min":{"1min":0.65400000000000003,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.2769999999999999,"5min":1.446,"15min":1.446},"last":0.752},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.95199999999999996,"15min":0.88200000000000001},"min":{"1min":0.69799999999999995,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.2629999999999999,"5min":1.296,"15min":1.296},"last":1.016}]},{"osd":5,"last update":"Sun Aug 27 21:56:27 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":0.96899999999999997,"15min":0.89300000000000002},"min":{"1min":0.73999999999999999,"5min":0.437,"15min":0.437},"max":{"1min":1.208,"5min":1.7170000000000001,"15min":1.7170000000000001},"last":0.94699999999999995},{"interface":"front","average":{"1min":1.089,"5min":0.998,"15min":0.95499999999999996},"min":{"1min":0.72199999999999998,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.369,"5min":1.845,"15min":1.845},"last":0.93300000000000005}]},{"osd":7,"last update":"Sun Aug 27 21:56:16 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.03,"15min":0.98699999999999999},"min":{"1min":0.78400000000000003,"5min":0.5,"15min":0.5},"max":{"1min":1.3740000000000001,"5min":1.5049999999999999,"15min":1.5049999999999999},"last":1.0069999999999999},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.085,"15min":1.022},"min":{"1min":0.95799999999999996,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.4670000000000001,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":1.214}]}]},{"osd":1,"up_from":13,"seq":55834574961,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297700,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447452,"statfs":{"total":95995035648,"available":95690190848,"internally_reserved":0,"allocated":618496,"data_stored":269318,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:55:54 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.73799999999999999,"15min":0.58799999999999997},"min":{"1min":0.40600000000000003,"5min":0.35599999999999998,"15min":0.34799999999999998},"max":{"1min":1.669,"5min":1.669,"15min":1.669},"last":0.85699999999999998},{"interface":"front","average":{"1min":0.82099999999999995,"5min":0.81000000000000005,"15min":0.64800000000000002},"min":{"1min":0.441,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.8779999999999999,"5min":1.8779999999999999,"15min":1.8779999999999999},"last":1.022}]},{"osd":2,"last update":"Sun Aug 27 21:56:35 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.71899999999999997,"15min":0.69399999999999995},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":1.0720000000000001,"5min":1.3600000000000001,"15min":1.3600000000000001},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.78700000000000003,"15min":0.71299999999999997},"min":{"1min":0.34699999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.353,"5min":1.395,"15min":1.395},"last":1.119}]},{"osd":3,"last update":"Sun Aug 27 21:56:20 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.78000000000000003,"15min":0.79100000000000004},"min":{"1min":0.496,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.742,"5min":1.742,"15min":1.742},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.81399999999999995,"5min":0.84299999999999997,"15min":0.84299999999999997},"min":{"1min":0.47599999999999998,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.1499999999999999,"5min":1.351,"15min":1.351},"last":0.97899999999999998}]},{"osd":4,"last update":"Sun Aug 27 21:56:02 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.0229999999999999,"15min":1.0149999999999999},"min":{"1min":0.629,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.45,"5min":1.6779999999999999,"15min":1.6779999999999999},"last":1.399},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.081,"15min":1.0169999999999999},"min":{"1min":0.69499999999999995,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.851,"5min":1.851,"15min":1.851},"last":1.544}]},{"osd":5,"last update":"Sun Aug 27 21:56:45 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":1.089,"15min":1.071},"min":{"1min":0.65900000000000003,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.6020000000000001,"5min":1.762,"15min":1.762},"last":1.6020000000000001},{"interface":"front","average":{"1min":1.093,"5min":1.0820000000000001,"15min":1.117},"min":{"1min":0.745,"5min":0.66600000000000004,"15min":0.66600000000000004},"max":{"1min":1.8169999999999999,"5min":1.861,"15min":1.861},"last":1.462}]},{"osd":6,"last update":"Sun Aug 27 21:56:35 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.115,"15min":1.143},"min":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.581,"5min":1.821,"15min":1.821},"last":1.575},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.133,"15min":1.204},"min":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.4359999999999999,"5min":1.7110000000000001,"15min":1.7110000000000001},"last":1.651}]},{"osd":7,"last update":"Sun Aug 27 21:56:20 2023","interfaces":[{"interface":"back","average":{"1min":1.216,"5min":1.1830000000000001,"15min":1.2110000000000001},"min":{"1min":0.80400000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.665,"5min":1.665,"15min":1.665},"last":1.4390000000000001},{"interface":"front","average":{"1min":1.1459999999999999,"5min":1.1519999999999999,"15min":1.1719999999999999},"min":{"1min":0.73799999999999999,"5min":0.69099999999999995,"15min":0.69099999999999995},"max":{"1min":1.5389999999999999,"5min":1.7130000000000001,"15min":1.7130000000000001},"last":1.667}]}]},{"osd":0,"up_from":8,"seq":34359738489,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Aug 27 21:55:49 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.69799999999999995,"15min":0.59299999999999997},"min":{"1min":0.45000000000000001,"5min":0.317,"15min":0.31},"max":{"1min":0.96399999999999997,"5min":1.177,"15min":1.177},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.77200000000000002,"15min":0.64500000000000002},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.34499999999999997},"max":{"1min":1.0680000000000001,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.98399999999999999}]},{"osd":2,"last update":"Sun Aug 27 21:56:28 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.749,"15min":0.68500000000000005},"min":{"1min":0.45300000000000001,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.98099999999999998,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.94999999999999996},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.77200000000000002,"15min":0.72199999999999998},"min":{"1min":0.42899999999999999,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.01,"5min":1.25,"15min":1.25},"last":0.81200000000000006}]},{"osd":3,"last update":"Sun Aug 27 21:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":0.82799999999999996,"15min":0.82899999999999996},"min":{"1min":0.45000000000000001,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.161,"5min":1.359,"15min":1.359},"last":0.86899999999999999},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.752,"15min":0.77300000000000002},"min":{"1min":0.45200000000000001,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.94199999999999995,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":0.65700000000000003}]},{"osd":4,"last update":"Sun Aug 27 21:56:03 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":0.98299999999999998,"15min":0.95299999999999996},"min":{"1min":0.73699999999999999,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.407,"5min":1.5860000000000001,"15min":1.5860000000000001},"last":1.1120000000000001},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.028,"15min":1.0109999999999999},"min":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":1.5760000000000001,"5min":1.6299999999999999,"15min":1.6299999999999999},"last":1.1299999999999999}]},{"osd":5,"last update":"Sun Aug 27 21:55:49 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":0.97499999999999998,"15min":0.97299999999999998},"min":{"1min":0.82699999999999996,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.591,"5min":1.591,"15min":1.591},"last":0.84999999999999998},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.0740000000000001,"15min":1.0549999999999999},"min":{"1min":0.70799999999999996,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.6060000000000001,"5min":1.6060000000000001,"15min":1.6060000000000001},"last":1.2110000000000001}]},{"osd":6,"last update":"Sun Aug 27 21:56:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.1040000000000001,"15min":1.0960000000000001},"min":{"1min":0.71699999999999997,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.73,"5min":1.8069999999999999,"15min":1.8069999999999999},"last":1.165},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.1040000000000001,"15min":1.141},"min":{"1min":0.68600000000000005,"5min":0.628,"15min":0.628},"max":{"1min":1.5189999999999999,"5min":1.5640000000000001,"15min":1.5640000000000001},"last":1.363}]},{"osd":7,"last update":"Sun Aug 27 21:56:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":1.085,"15min":1.1040000000000001},"min":{"1min":0.71999999999999997,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.6879999999999999,"5min":1.6879999999999999,"15min":1.6879999999999999},"last":1.0049999999999999},{"interface":"front","average":{"1min":1.038,"5min":1.1839999999999999,"15min":1.25},"min":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"max":{"1min":1.5449999999999999,"5min":1.647,"15min":1.647},"last":1.2549999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411431,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:56:37 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.71499999999999997,"15min":0.64200000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.113,"5min":1.885,"15min":1.885},"last":0.621},{"interface":"front","average":{"1min":0.70199999999999996,"5min":0.76000000000000001,"15min":0.66500000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.121,"5min":1.718,"15min":1.718},"last":0.70899999999999996}]},{"osd":1,"last update":"Sun Aug 27 21:56:37 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.77100000000000002,"15min":0.69699999999999995},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.995,"5min":2.4769999999999999,"15min":2.4769999999999999},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.80000000000000004,"15min":0.69299999999999995},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.439},"max":{"1min":1.242,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":0.84899999999999998}]},{"osd":3,"last update":"Sun Aug 27 21:56:16 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":0.81899999999999995,"15min":0.78100000000000003},"min":{"1min":0.40899999999999997,"5min":0.372,"15min":0.372},"max":{"1min":1.268,"5min":1.5569999999999999,"15min":1.5569999999999999},"last":0.82199999999999995},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.872,"15min":0.83699999999999997},"min":{"1min":0.44800000000000001,"5min":0.441,"15min":0.441},"max":{"1min":1.133,"5min":1.7649999999999999,"15min":1.7649999999999999},"last":0.997}]},{"osd":4,"last update":"Sun Aug 27 21:56:07 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.0229999999999999,"15min":0.98499999999999999},"min":{"1min":0.69199999999999995,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.7310000000000001,"5min":1.8580000000000001,"15min":1.8580000000000001},"last":0.95599999999999996},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.073,"15min":0.98599999999999999},"min":{"1min":0.60399999999999998,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.4970000000000001,"5min":1.921,"15min":1.921},"last":0.96799999999999997}]},{"osd":5,"last update":"Sun Aug 27 21:55:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1579999999999999,"5min":1.087,"15min":1.0409999999999999},"min":{"1min":0.74099999999999999,"5min":0.67300000000000004,"15min":0.67300000000000004},"max":{"1min":1.51,"5min":1.845,"15min":1.845},"last":1.0249999999999999},{"interface":"front","average":{"1min":1.095,"5min":1.1020000000000001,"15min":1.0720000000000001},"min":{"1min":0.78600000000000003,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.2789999999999999,"5min":2.0089999999999999,"15min":2.0089999999999999},"last":0.91000000000000003}]},{"osd":6,"last update":"Sun Aug 27 21:56:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.153,"15min":1.1850000000000001},"min":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.55,"5min":2.0960000000000001,"15min":2.0960000000000001},"last":1.0620000000000001},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.137,"15min":1.169},"min":{"1min":0.63100000000000001,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.806,"5min":1.806,"15min":1.806},"last":1.1319999999999999}]},{"osd":7,"last update":"Sun Aug 27 21:56:16 2023","interfaces":[{"interface":"back","average":{"1min":1.216,"5min":1.1890000000000001,"15min":1.23},"min":{"1min":0.68000000000000005,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.518,"5min":1.804,"15min":1.804},"last":1.0800000000000001},{"interface":"front","average":{"1min":1.2569999999999999,"5min":1.1970000000000001,"15min":1.1890000000000001},"min":{"1min":0.86899999999999999,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.8500000000000001,"5min":1.8500000000000001,"15min":1.8500000000000001},"last":1.1830000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182494,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":270038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:56:22 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.69699999999999995,"15min":0.68100000000000005},"min":{"1min":0.45600000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.0089999999999999,"5min":1.1439999999999999,"15min":1.1439999999999999},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.72599999999999998,"15min":0.73499999999999999},"min":{"1min":0.501,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.69199999999999995}]},{"osd":1,"last update":"Sun Aug 27 21:56:22 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":0.76000000000000001,"15min":0.75900000000000001},"min":{"1min":0.46600000000000003,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.0940000000000001,"5min":1.111,"15min":1.111},"last":0.90500000000000003},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.80400000000000005,"15min":0.74299999999999999},"min":{"1min":0.53100000000000003,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.98699999999999999,"5min":2.883,"15min":2.883},"last":0.72499999999999998}]},{"osd":2,"last update":"Sun Aug 27 21:56:22 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.88100000000000001,"15min":0.86699999999999999},"min":{"1min":0.50700000000000001,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.179,"5min":1.179,"15min":1.179},"last":0.81699999999999995},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.85399999999999998,"15min":0.876},"min":{"1min":0.42499999999999999,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.1719999999999999,"5min":3.319,"15min":3.319},"last":0.93799999999999994}]},{"osd":4,"last update":"Sun Aug 27 21:56:03 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.98999999999999999,"15min":0.98599999999999999},"min":{"1min":0.64400000000000002,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.3320000000000001,"5min":3.5939999999999999,"15min":3.5939999999999999},"last":1.1599999999999999},{"interface":"front","average":{"1min":1.071,"5min":1.0329999999999999,"15min":1.0129999999999999},"min":{"1min":0.69499999999999995,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.4670000000000001,"5min":1.679,"15min":1.679},"last":1.3700000000000001}]},{"osd":5,"last update":"Sun Aug 27 21:56:45 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.032,"15min":1.0409999999999999},"min":{"1min":0.70799999999999996,"5min":0.622,"15min":0.622},"max":{"1min":1.601,"5min":3.5670000000000002,"15min":3.5670000000000002},"last":1.0780000000000001},{"interface":"front","average":{"1min":1.0529999999999999,"5min":1.0580000000000001,"15min":1.042},"min":{"1min":0.77600000000000002,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.3859999999999999,"5min":1.5369999999999999,"15min":1.5369999999999999},"last":1.091}]},{"osd":6,"last update":"Sun Aug 27 21:56:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.0880000000000001,"15min":1.093},"min":{"1min":0.81299999999999994,"5min":0.69799999999999995,"15min":0.69799999999999995},"max":{"1min":1.532,"5min":1.532,"15min":1.532},"last":1.1899999999999999},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.1200000000000001,"15min":1.119},"min":{"1min":0.76100000000000001,"5min":0.66400000000000003,"15min":0.66400000000000003},"max":{"1min":1.5289999999999999,"5min":1.5960000000000001,"15min":1.5960000000000001},"last":1.298}]},{"osd":7,"last update":"Sun Aug 27 21:56:15 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.115,"15min":1.1220000000000001},"min":{"1min":0.81799999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.4039999999999999,"5min":3.5270000000000001,"15min":3.5270000000000001},"last":1.222},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.1819999999999999,"15min":1.224},"min":{"1min":0.81299999999999994,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.383,"5min":1.643,"15min":1.643},"last":1.24}]}]},{"osd":4,"up_from":30,"seq":128849018966,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297732,"kb_used_data":636,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447420,"statfs":{"total":95995035648,"available":95690158080,"internally_reserved":0,"allocated":651264,"data_stored":271630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:56:03 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.90400000000000003,"15min":0.83299999999999996},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.742,"5min":3.383,"15min":3.383},"last":1.1240000000000001},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.85399999999999998,"15min":0.83899999999999997},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.425,"5min":1.425,"15min":1.425},"last":0.95499999999999996}]},{"osd":1,"last update":"Sun Aug 27 21:56:03 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":0.86299999999999999,"15min":0.81000000000000005},"min":{"1min":0.60199999999999998,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.2,"5min":2.1349999999999998,"15min":2.1349999999999998},"last":0.84399999999999997},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.97899999999999998,"15min":0.90100000000000002},"min":{"1min":0.63900000000000001,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.6040000000000001,"5min":3.3340000000000001,"15min":3.3340000000000001},"last":1.2829999999999999}]},{"osd":2,"last update":"Sun Aug 27 21:56:03 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":0.97099999999999997,"15min":0.878},"min":{"1min":0.63200000000000001,"5min":0.62,"15min":0.62},"max":{"1min":1.871,"5min":3.4260000000000002,"15min":3.4260000000000002},"last":1.1899999999999999},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.88700000000000001,"15min":0.874},"min":{"1min":0.68200000000000005,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.4530000000000001,"5min":1.4530000000000001,"15min":1.4530000000000001},"last":1.056}]},{"osd":3,"last update":"Sun Aug 27 21:56:03 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.89900000000000002,"15min":0.88500000000000001},"min":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.5209999999999999,"5min":1.5209999999999999,"15min":1.5209999999999999},"last":0.89000000000000001},{"interface":"front","average":{"1min":1.0469999999999999,"5min":0.93999999999999995,"15min":0.92600000000000005},"min":{"1min":0.61299999999999999,"5min":0.501,"15min":0.501},"max":{"1min":1.5529999999999999,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":0.92200000000000004}]},{"osd":5,"last update":"Sun Aug 27 21:55:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.89400000000000002,"15min":0.91800000000000004},"min":{"1min":0.58699999999999997,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.4019999999999999,"5min":3.2200000000000002,"15min":3.2200000000000002},"last":1.24},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.85899999999999999,"15min":0.83499999999999996},"min":{"1min":0.56599999999999995,"5min":0.378,"15min":0.378},"max":{"1min":1.353,"5min":1.353,"15min":1.353},"last":0.77900000000000003}]},{"osd":6,"last update":"Sun Aug 27 21:56:31 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":0.91600000000000004,"15min":0.83899999999999997},"min":{"1min":0.73799999999999999,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.679,"5min":1.679,"15min":1.679},"last":1.0840000000000001},{"interface":"front","average":{"1min":1.2290000000000001,"5min":1.042,"15min":0.95599999999999996},"min":{"1min":0.89600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":2.0630000000000002,"5min":3.278,"15min":3.278},"last":1.0289999999999999}]},{"osd":7,"last update":"Sun Aug 27 21:56:18 2023","interfaces":[{"interface":"back","average":{"1min":1.2390000000000001,"5min":1.044,"15min":0.95799999999999996},"min":{"1min":0.77300000000000002,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.9590000000000001,"5min":1.9590000000000001,"15min":1.9590000000000001},"last":1.325},{"interface":"front","average":{"1min":1.2130000000000001,"5min":1.0109999999999999,"15min":0.92500000000000004},"min":{"1min":0.84599999999999997,"5min":0.443,"15min":0.443},"max":{"1min":2.2509999999999999,"5min":2.2509999999999999,"15min":2.2509999999999999},"last":1.0009999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822733,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":269942,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 21:55:46 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.84999999999999998,"15min":0.879},"min":{"1min":0.626,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.2450000000000001,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":0.94099999999999995},{"interface":"front","average":{"1min":0.92500000000000004,"5min":0.86299999999999999,"15min":0.86699999999999999},"min":{"1min":0.63,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.137,"5min":1.2050000000000001,"15min":1.2050000000000001},"last":0.72599999999999998}]},{"osd":1,"last update":"Sun Aug 27 21:55:46 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":0.89200000000000002,"15min":0.92000000000000004},"min":{"1min":0.71899999999999997,"5min":0.504,"15min":0.504},"max":{"1min":1.135,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.99199999999999999},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.90300000000000002,"15min":0.89300000000000002},"min":{"1min":0.67900000000000005,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.2170000000000001,"5min":1.605,"15min":1.605},"last":0.877}]},{"osd":2,"last update":"Sun Aug 27 21:55:46 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.91000000000000003,"15min":0.90400000000000003},"min":{"1min":0.65100000000000002,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.153,"5min":1.7609999999999999,"15min":1.7609999999999999},"last":1.2010000000000001},{"interface":"front","average":{"1min":0.95799999999999996,"5min":0.89800000000000002,"15min":0.91000000000000003},"min":{"1min":0.64000000000000001,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.177,"5min":1.264,"15min":1.264},"last":0.90400000000000003}]},{"osd":3,"last update":"Sun Aug 27 21:55:46 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.93999999999999995,"15min":0.93300000000000005},"min":{"1min":0.67200000000000004,"5min":0.503,"15min":0.503},"max":{"1min":1.246,"5min":1.587,"15min":1.587},"last":0.82799999999999996},{"interface":"front","average":{"1min":1.0349999999999999,"5min":0.96699999999999997,"15min":0.95299999999999996},"min":{"1min":0.69999999999999996,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.331,"5min":1.506,"15min":1.506},"last":1.083}]},{"osd":4,"last update":"Sun Aug 27 21:55:46 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.85099999999999998,"15min":0.78700000000000003},"min":{"1min":0.65900000000000003,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.085,"5min":1.6919999999999999,"15min":1.6919999999999999},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.92500000000000004,"15min":0.80600000000000005},"min":{"1min":0.64400000000000002,"5min":0.442,"15min":0.442},"max":{"1min":1.333,"5min":1.538,"15min":1.538},"last":1.1379999999999999}]},{"osd":6,"last update":"Sun Aug 27 21:56:27 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":0.96899999999999997,"15min":0.95499999999999996},"min":{"1min":0.78400000000000003,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.1870000000000001,"5min":1.7050000000000001,"15min":1.7050000000000001},"last":1.0329999999999999},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.98399999999999999,"15min":0.97799999999999998},"min":{"1min":0.71799999999999997,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.264,"5min":1.4370000000000001,"15min":1.4370000000000001},"last":1.159}]},{"osd":7,"last update":"Sun Aug 27 21:56:19 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":1.0449999999999999,"15min":1.0389999999999999},"min":{"1min":0.71199999999999997,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.2450000000000001,"5min":1.498,"15min":1.498},"last":1.228},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.0269999999999999,"15min":1.0029999999999999},"min":{"1min":0.75900000000000001,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.2789999999999999,"5min":1.8340000000000001,"15min":1.8340000000000001},"last":1.1120000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-27T21:56:49.382 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-08-27T21:56:49.382 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-08-27T21:56:49.382 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-08-27T21:56:49.383 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph health --format=json 2023-08-27T21:56:50.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:49 smithi130 bash[27579]: audit 2023-08-27T21:56:48.628128+0000 mgr.y (mgr.14430) 156 : audit [DBG] from='client.14775 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T21:56:50.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:49 smithi130 bash[27579]: cluster 2023-08-27T21:56:48.666188+0000 mgr.y (mgr.14430) 157 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:50.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:49 smithi006 bash[33918]: audit 2023-08-27T21:56:48.628128+0000 mgr.y (mgr.14430) 156 : audit [DBG] from='client.14775 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T21:56:50.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:49 smithi006 bash[33918]: cluster 2023-08-27T21:56:48.666188+0000 mgr.y (mgr.14430) 157 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:50.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:49 smithi006 bash[27292]: audit 2023-08-27T21:56:48.628128+0000 mgr.y (mgr.14430) 156 : audit [DBG] from='client.14775 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T21:56:50.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:49 smithi006 bash[27292]: cluster 2023-08-27T21:56:48.666188+0000 mgr.y (mgr.14430) 157 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:51.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:56:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:56:50] "GET /metrics HTTP/1.1" 200 191531 "" "Prometheus/2.33.4" 2023-08-27T21:56:52.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:51 smithi130 bash[27579]: cluster 2023-08-27T21:56:50.666976+0000 mgr.y (mgr.14430) 158 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:52.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:51 smithi006 bash[27292]: cluster 2023-08-27T21:56:50.666976+0000 mgr.y (mgr.14430) 158 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:52.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:51 smithi006 bash[33918]: cluster 2023-08-27T21:56:50.666976+0000 mgr.y (mgr.14430) 158 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:53.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:53 smithi130 bash[27579]: cluster 2023-08-27T21:56:52.667596+0000 mgr.y (mgr.14430) 159 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:53.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:53 smithi006 bash[27292]: cluster 2023-08-27T21:56:52.667596+0000 mgr.y (mgr.14430) 159 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:53.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:53 smithi006 bash[33918]: cluster 2023-08-27T21:56:52.667596+0000 mgr.y (mgr.14430) 159 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:54.251 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:56:55.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:55 smithi006 bash[33918]: cluster 2023-08-27T21:56:54.668226+0000 mgr.y (mgr.14430) 160 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:55.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:55 smithi006 bash[27292]: cluster 2023-08-27T21:56:54.668226+0000 mgr.y (mgr.14430) 160 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:56.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:55 smithi130 bash[27579]: cluster 2023-08-27T21:56:54.668226+0000 mgr.y (mgr.14430) 160 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:57.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:56:56 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:56:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:56:57.481 INFO:teuthology.orchestra.run.smithi006.stdout: 2023-08-27T21:56:57.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:57 smithi006 bash[27292]: cluster 2023-08-27T21:56:56.668817+0000 mgr.y (mgr.14430) 161 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:57.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:57 smithi006 bash[27292]: audit 2023-08-27T21:56:57.475567+0000 mon.a (mon.0) 744 : audit [DBG] from='client.? 172.21.15.6:0/2460594773' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-08-27T21:56:57.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:57 smithi006 bash[33918]: cluster 2023-08-27T21:56:56.668817+0000 mgr.y (mgr.14430) 161 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:57.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:57 smithi006 bash[33918]: audit 2023-08-27T21:56:57.475567+0000 mon.a (mon.0) 744 : audit [DBG] from='client.? 172.21.15.6:0/2460594773' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-08-27T21:56:58.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:57 smithi130 bash[27579]: cluster 2023-08-27T21:56:56.668817+0000 mgr.y (mgr.14430) 161 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:56:58.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:57 smithi130 bash[27579]: audit 2023-08-27T21:56:57.475567+0000 mon.a (mon.0) 744 : audit [DBG] from='client.? 172.21.15.6:0/2460594773' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-08-27T21:56:58.282 INFO:teuthology.orchestra.run.smithi006.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-08-27T21:56:58.282 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-08-27T21:56:58.283 INFO:teuthology.run_tasks:Running task print... 2023-08-27T21:56:58.299 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-08-27T21:56:58.300 INFO:teuthology.run_tasks:Running task parallel... 2023-08-27T21:56:58.314 INFO:teuthology.task.parallel:starting parallel... 2023-08-27T21:56:58.314 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-08-27T21:56:58.315 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-08-27T21:56:58.316 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-08-27T21:56:58.317 INFO:tasks.radosbench:Beginning radosbench... 2023-08-27T21:56:58.317 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-08-27T21:56:58.317 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd pool create unique_pool_0 16 2023-08-27T21:56:58.318 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-08-27T21:56:58.318 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-08-27T21:56:58.318 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi006.front.sepia.ceph.com 2023-08-27T21:56:58.318 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-08-27T21:57:00.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:56:59 smithi130 bash[27579]: cluster 2023-08-27T21:56:58.669391+0000 mgr.y (mgr.14430) 162 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:00.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:56:59 smithi006 bash[27292]: cluster 2023-08-27T21:56:58.669391+0000 mgr.y (mgr.14430) 162 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:00.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:56:59 smithi006 bash[33918]: cluster 2023-08-27T21:56:58.669391+0000 mgr.y (mgr.14430) 162 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:01.168 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:57:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:57:00] "GET /metrics HTTP/1.1" 200 191527 "" "Prometheus/2.33.4" 2023-08-27T21:57:01.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:57:01 smithi006 bash[27566]: [27/Aug/2023:21:57:01] ENGINE Bus STOPPING 2023-08-27T21:57:01.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:57:01 smithi006 bash[27566]: [27/Aug/2023:21:57:01] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-27T21:57:01.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:57:01 smithi006 bash[27566]: [27/Aug/2023:21:57:01] ENGINE Bus STOPPED 2023-08-27T21:57:01.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:57:01 smithi006 bash[27566]: [27/Aug/2023:21:57:01] ENGINE Bus STARTING 2023-08-27T21:57:01.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:57:01 smithi006 bash[27566]: [27/Aug/2023:21:57:01] ENGINE Serving on http://:::9283 2023-08-27T21:57:01.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:57:01 smithi006 bash[27566]: [27/Aug/2023:21:57:01] ENGINE Bus STARTED 2023-08-27T21:57:01.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:01 smithi006 bash[33918]: cluster 2023-08-27T21:57:00.670432+0000 mgr.y (mgr.14430) 163 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:01.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:01 smithi006 bash[33918]: audit 2023-08-27T21:57:01.570838+0000 mon.a (mon.0) 745 : audit [INF] from='client.? 172.21.15.6:0/3928088430' entity='client.admin' 2023-08-27T21:57:01.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:01 smithi006 bash[33918]: audit 2023-08-27T21:57:01.622755+0000 mon.a (mon.0) 746 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:57:01.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:01 smithi006 bash[27292]: cluster 2023-08-27T21:57:00.670432+0000 mgr.y (mgr.14430) 163 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:01.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:01 smithi006 bash[27292]: audit 2023-08-27T21:57:01.570838+0000 mon.a (mon.0) 745 : audit [INF] from='client.? 172.21.15.6:0/3928088430' entity='client.admin' 2023-08-27T21:57:01.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:01 smithi006 bash[27292]: audit 2023-08-27T21:57:01.622755+0000 mon.a (mon.0) 746 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:57:02.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:01 smithi130 bash[27579]: cluster 2023-08-27T21:57:00.670432+0000 mgr.y (mgr.14430) 163 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:02.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:01 smithi130 bash[27579]: audit 2023-08-27T21:57:01.570838+0000 mon.a (mon.0) 745 : audit [INF] from='client.? 172.21.15.6:0/3928088430' entity='client.admin' 2023-08-27T21:57:02.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:01 smithi130 bash[27579]: audit 2023-08-27T21:57:01.622755+0000 mon.a (mon.0) 746 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:57:02.360 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph config set global log_to_journald false --force' 2023-08-27T21:57:03.125 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:57:03.866 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:03 smithi006 bash[33918]: cluster 2023-08-27T21:57:02.671085+0000 mgr.y (mgr.14430) 164 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:03.866 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:03 smithi006 bash[27292]: cluster 2023-08-27T21:57:02.671085+0000 mgr.y (mgr.14430) 164 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:03.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:03 smithi130 bash[27579]: cluster 2023-08-27T21:57:02.671085+0000 mgr.y (mgr.14430) 164 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:06.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:05 smithi130 bash[27579]: cluster 2023-08-27T21:57:04.671697+0000 mgr.y (mgr.14430) 165 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:06.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:05 smithi006 bash[27292]: cluster 2023-08-27T21:57:04.671697+0000 mgr.y (mgr.14430) 165 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:06.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:05 smithi006 bash[33918]: cluster 2023-08-27T21:57:04.671697+0000 mgr.y (mgr.14430) 165 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:07.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:57:06 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:57:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:57:07.397 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:57:07 smithi006 bash[27566]: [27/Aug/2023:21:57:07] ENGINE Bus STOPPING 2023-08-27T21:57:07.397 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:57:07 smithi006 bash[27566]: [27/Aug/2023:21:57:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-27T21:57:07.397 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:57:07 smithi006 bash[27566]: [27/Aug/2023:21:57:07] ENGINE Bus STOPPED 2023-08-27T21:57:07.397 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:57:07 smithi006 bash[27566]: [27/Aug/2023:21:57:07] ENGINE Bus STARTING 2023-08-27T21:57:07.720 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:57:07 smithi006 bash[27566]: [27/Aug/2023:21:57:07] ENGINE Serving on http://:::9283 2023-08-27T21:57:07.720 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:57:07 smithi006 bash[27566]: [27/Aug/2023:21:57:07] ENGINE Bus STARTED 2023-08-27T21:57:07.969 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-08-27T21:57:08.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:07 smithi006 bash[27292]: cluster 2023-08-27T21:57:06.672363+0000 mgr.y (mgr.14430) 166 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:08.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:07 smithi006 bash[27292]: audit 2023-08-27T21:57:07.196176+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:08.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:07 smithi006 bash[27292]: audit 2023-08-27T21:57:07.266935+0000 mon.a (mon.0) 748 : audit [INF] from='client.? 172.21.15.6:0/3335330305' entity='client.admin' 2023-08-27T21:57:08.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:07 smithi006 bash[33918]: cluster 2023-08-27T21:57:06.672363+0000 mgr.y (mgr.14430) 166 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:08.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:07 smithi006 bash[33918]: audit 2023-08-27T21:57:07.196176+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:08.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:07 smithi006 bash[33918]: audit 2023-08-27T21:57:07.266935+0000 mon.a (mon.0) 748 : audit [INF] from='client.? 172.21.15.6:0/3335330305' entity='client.admin' 2023-08-27T21:57:08.283 INFO:teuthology.orchestra.run.smithi006.stderr:pool 'unique_pool_0' created 2023-08-27T21:57:08.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:07 smithi130 bash[27579]: cluster 2023-08-27T21:57:06.672363+0000 mgr.y (mgr.14430) 166 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:08.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:07 smithi130 bash[27579]: audit 2023-08-27T21:57:07.196176+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:08.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:07 smithi130 bash[27579]: audit 2023-08-27T21:57:07.266935+0000 mon.a (mon.0) 748 : audit [INF] from='client.? 172.21.15.6:0/3335330305' entity='client.admin' 2023-08-27T21:57:09.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[27292]: audit 2023-08-27T21:57:07.465127+0000 mon.a (mon.0) 749 : audit [INF] from='client.? 172.21.15.6:0/2970965090' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-08-27T21:57:09.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[27292]: audit 2023-08-27T21:57:07.934954+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:09.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[27292]: audit 2023-08-27T21:57:07.936569+0000 mon.a (mon.0) 751 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:57:09.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[27292]: audit 2023-08-27T21:57:07.938078+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:57:09.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[27292]: audit 2023-08-27T21:57:07.963124+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:09.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[27292]: audit 2023-08-27T21:57:07.969811+0000 mon.a (mon.0) 754 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:57:09.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[27292]: audit 2023-08-27T21:57:07.972329+0000 mon.a (mon.0) 755 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:57:09.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[27292]: audit 2023-08-27T21:57:07.973504+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:57:09.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[27292]: audit 2023-08-27T21:57:07.982263+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:09.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[27292]: audit 2023-08-27T21:57:08.269161+0000 mon.a (mon.0) 758 : audit [INF] from='client.? 172.21.15.6:0/2970965090' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-08-27T21:57:09.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[27292]: cluster 2023-08-27T21:57:08.269293+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-08-27T21:57:09.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[27292]: cluster 2023-08-27T21:57:08.673026+0000 mgr.y (mgr.14430) 167 : cluster [DBG] pgmap v147: 89 pgs: 16 unknown, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[33918]: audit 2023-08-27T21:57:07.465127+0000 mon.a (mon.0) 749 : audit [INF] from='client.? 172.21.15.6:0/2970965090' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-08-27T21:57:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[33918]: audit 2023-08-27T21:57:07.934954+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[33918]: audit 2023-08-27T21:57:07.936569+0000 mon.a (mon.0) 751 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:57:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[33918]: audit 2023-08-27T21:57:07.938078+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:57:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[33918]: audit 2023-08-27T21:57:07.963124+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[33918]: audit 2023-08-27T21:57:07.969811+0000 mon.a (mon.0) 754 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:57:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[33918]: audit 2023-08-27T21:57:07.972329+0000 mon.a (mon.0) 755 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:57:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[33918]: audit 2023-08-27T21:57:07.973504+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:57:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[33918]: audit 2023-08-27T21:57:07.982263+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:09.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[33918]: audit 2023-08-27T21:57:08.269161+0000 mon.a (mon.0) 758 : audit [INF] from='client.? 172.21.15.6:0/2970965090' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-08-27T21:57:09.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[33918]: cluster 2023-08-27T21:57:08.269293+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-08-27T21:57:09.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:08 smithi006 bash[33918]: cluster 2023-08-27T21:57:08.673026+0000 mgr.y (mgr.14430) 167 : cluster [DBG] pgmap v147: 89 pgs: 16 unknown, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:09.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:08 smithi130 bash[27579]: audit 2023-08-27T21:57:07.465127+0000 mon.a (mon.0) 749 : audit [INF] from='client.? 172.21.15.6:0/2970965090' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-08-27T21:57:09.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:08 smithi130 bash[27579]: audit 2023-08-27T21:57:07.934954+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:09.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:08 smithi130 bash[27579]: audit 2023-08-27T21:57:07.936569+0000 mon.a (mon.0) 751 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:57:09.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:08 smithi130 bash[27579]: audit 2023-08-27T21:57:07.938078+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:57:09.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:08 smithi130 bash[27579]: audit 2023-08-27T21:57:07.963124+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:09.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:08 smithi130 bash[27579]: audit 2023-08-27T21:57:07.969811+0000 mon.a (mon.0) 754 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:57:09.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:08 smithi130 bash[27579]: audit 2023-08-27T21:57:07.972329+0000 mon.a (mon.0) 755 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:57:09.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:08 smithi130 bash[27579]: audit 2023-08-27T21:57:07.973504+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:57:09.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:08 smithi130 bash[27579]: audit 2023-08-27T21:57:07.982263+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:09.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:08 smithi130 bash[27579]: audit 2023-08-27T21:57:08.269161+0000 mon.a (mon.0) 758 : audit [INF] from='client.? 172.21.15.6:0/2970965090' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-08-27T21:57:09.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:08 smithi130 bash[27579]: cluster 2023-08-27T21:57:08.269293+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-08-27T21:57:09.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:08 smithi130 bash[27579]: cluster 2023-08-27T21:57:08.673026+0000 mgr.y (mgr.14430) 167 : cluster [DBG] pgmap v147: 89 pgs: 16 unknown, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:09.638 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-08-27T21:57:10.492 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:10 smithi006 bash[27292]: cluster 2023-08-27T21:57:09.275325+0000 mon.a (mon.0) 760 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-08-27T21:57:10.492 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:10 smithi006 bash[33918]: cluster 2023-08-27T21:57:09.275325+0000 mon.a (mon.0) 760 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-08-27T21:57:10.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:10 smithi130 bash[27579]: cluster 2023-08-27T21:57:09.275325+0000 mon.a (mon.0) 760 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-08-27T21:57:11.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:57:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:57:10] "GET /metrics HTTP/1.1" 200 191527 "" "Prometheus/2.33.4" 2023-08-27T21:57:11.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:11 smithi130 bash[27579]: cluster 2023-08-27T21:57:10.276042+0000 mon.a (mon.0) 761 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-08-27T21:57:11.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:11 smithi130 bash[27579]: cluster 2023-08-27T21:57:10.674040+0000 mgr.y (mgr.14430) 168 : cluster [DBG] pgmap v150: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:11.705 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:11 smithi006 bash[33918]: cluster 2023-08-27T21:57:10.276042+0000 mon.a (mon.0) 761 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-08-27T21:57:11.705 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:11 smithi006 bash[33918]: cluster 2023-08-27T21:57:10.674040+0000 mgr.y (mgr.14430) 168 : cluster [DBG] pgmap v150: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:11.705 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:11 smithi006 bash[27292]: cluster 2023-08-27T21:57:10.276042+0000 mon.a (mon.0) 761 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-08-27T21:57:11.706 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:11 smithi006 bash[27292]: cluster 2023-08-27T21:57:10.674040+0000 mgr.y (mgr.14430) 168 : cluster [DBG] pgmap v150: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:12.191 INFO:teuthology.orchestra.run.smithi006.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T21:57:13.147 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph orch ps' 2023-08-27T21:57:13.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:13 smithi006 bash[27292]: audit 2023-08-27T21:57:12.177014+0000 mgr.y (mgr.14430) 169 : audit [DBG] from='client.14805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:57:13.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:13 smithi006 bash[27292]: cephadm 2023-08-27T21:57:12.178657+0000 mgr.y (mgr.14430) 170 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T21:57:13.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:13 smithi006 bash[27292]: audit 2023-08-27T21:57:12.185494+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:13.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:13 smithi006 bash[27292]: audit 2023-08-27T21:57:12.213712+0000 mon.a (mon.0) 763 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:57:13.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:13 smithi006 bash[27292]: audit 2023-08-27T21:57:12.215972+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:57:13.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:13 smithi006 bash[27292]: audit 2023-08-27T21:57:12.216964+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:57:13.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:13 smithi006 bash[27292]: audit 2023-08-27T21:57:12.224625+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:13.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:13 smithi006 bash[27292]: cephadm 2023-08-27T21:57:12.230891+0000 mgr.y (mgr.14430) 171 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T21:57:13.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:13 smithi006 bash[27292]: cluster 2023-08-27T21:57:12.674611+0000 mgr.y (mgr.14430) 172 : cluster [DBG] pgmap v151: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:13.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:13 smithi006 bash[33918]: audit 2023-08-27T21:57:12.177014+0000 mgr.y (mgr.14430) 169 : audit [DBG] from='client.14805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:57:13.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:13 smithi006 bash[33918]: cephadm 2023-08-27T21:57:12.178657+0000 mgr.y (mgr.14430) 170 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T21:57:13.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:13 smithi006 bash[33918]: audit 2023-08-27T21:57:12.185494+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:13.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:13 smithi006 bash[33918]: audit 2023-08-27T21:57:12.213712+0000 mon.a (mon.0) 763 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:57:13.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:13 smithi006 bash[33918]: audit 2023-08-27T21:57:12.215972+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:57:13.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:13 smithi006 bash[33918]: audit 2023-08-27T21:57:12.216964+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:57:13.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:13 smithi006 bash[33918]: audit 2023-08-27T21:57:12.224625+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:13.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:13 smithi006 bash[33918]: cephadm 2023-08-27T21:57:12.230891+0000 mgr.y (mgr.14430) 171 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T21:57:13.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:13 smithi006 bash[33918]: cluster 2023-08-27T21:57:12.674611+0000 mgr.y (mgr.14430) 172 : cluster [DBG] pgmap v151: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:13.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:13 smithi130 bash[27579]: audit 2023-08-27T21:57:12.177014+0000 mgr.y (mgr.14430) 169 : audit [DBG] from='client.14805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:57:13.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:13 smithi130 bash[27579]: cephadm 2023-08-27T21:57:12.178657+0000 mgr.y (mgr.14430) 170 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T21:57:13.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:13 smithi130 bash[27579]: audit 2023-08-27T21:57:12.185494+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:13.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:13 smithi130 bash[27579]: audit 2023-08-27T21:57:12.213712+0000 mon.a (mon.0) 763 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:57:13.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:13 smithi130 bash[27579]: audit 2023-08-27T21:57:12.215972+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:57:13.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:13 smithi130 bash[27579]: audit 2023-08-27T21:57:12.216964+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:57:13.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:13 smithi130 bash[27579]: audit 2023-08-27T21:57:12.224625+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:13.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:13 smithi130 bash[27579]: cephadm 2023-08-27T21:57:12.230891+0000 mgr.y (mgr.14430) 171 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T21:57:13.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:13 smithi130 bash[27579]: cluster 2023-08-27T21:57:12.674611+0000 mgr.y (mgr.14430) 172 : cluster [DBG] pgmap v151: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:15.661 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T21:57:15.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:15 smithi006 bash[27292]: cluster 2023-08-27T21:57:14.675081+0000 mgr.y (mgr.14430) 173 : cluster [DBG] pgmap v152: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:15.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:15 smithi006 bash[33918]: cluster 2023-08-27T21:57:14.675081+0000 mgr.y (mgr.14430) 173 : cluster [DBG] pgmap v152: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:16.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:15 smithi130 bash[27579]: cluster 2023-08-27T21:57:14.675081+0000 mgr.y (mgr.14430) 173 : cluster [DBG] pgmap v152: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:17.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:57:16 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:57:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:57:18.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:17 smithi130 bash[27579]: cluster 2023-08-27T21:57:16.676296+0000 mgr.y (mgr.14430) 174 : cluster [DBG] pgmap v153: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:18.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:17 smithi006 bash[27292]: cluster 2023-08-27T21:57:16.676296+0000 mgr.y (mgr.14430) 174 : cluster [DBG] pgmap v153: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:18.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:17 smithi006 bash[33918]: cluster 2023-08-27T21:57:16.676296+0000 mgr.y (mgr.14430) 174 : cluster [DBG] pgmap v153: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:19.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:18 smithi006 bash[27292]: cluster 2023-08-27T21:57:18.676743+0000 mgr.y (mgr.14430) 175 : cluster [DBG] pgmap v154: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:19.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:18 smithi006 bash[33918]: cluster 2023-08-27T21:57:18.676743+0000 mgr.y (mgr.14430) 175 : cluster [DBG] pgmap v154: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:19.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:18 smithi130 bash[27579]: cluster 2023-08-27T21:57:18.676743+0000 mgr.y (mgr.14430) 175 : cluster [DBG] pgmap v154: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:21.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:57:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:57:20] "GET /metrics HTTP/1.1" 200 193925 "" "Prometheus/2.33.4" 2023-08-27T21:57:22.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:21 smithi130 bash[27579]: cluster 2023-08-27T21:57:20.677426+0000 mgr.y (mgr.14430) 176 : cluster [DBG] pgmap v155: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:22.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:21 smithi006 bash[27292]: cluster 2023-08-27T21:57:20.677426+0000 mgr.y (mgr.14430) 176 : cluster [DBG] pgmap v155: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:22.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:21 smithi006 bash[33918]: cluster 2023-08-27T21:57:20.677426+0000 mgr.y (mgr.14430) 176 : cluster [DBG] pgmap v155: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:24.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:23 smithi130 bash[27579]: cluster 2023-08-27T21:57:22.677874+0000 mgr.y (mgr.14430) 177 : cluster [DBG] pgmap v156: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:24.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:23 smithi006 bash[27292]: cluster 2023-08-27T21:57:22.677874+0000 mgr.y (mgr.14430) 177 : cluster [DBG] pgmap v156: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:24.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:23 smithi006 bash[33918]: cluster 2023-08-27T21:57:22.677874+0000 mgr.y (mgr.14430) 177 : cluster [DBG] pgmap v156: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:26.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:25 smithi130 bash[27579]: cluster 2023-08-27T21:57:24.678393+0000 mgr.y (mgr.14430) 178 : cluster [DBG] pgmap v157: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:26.232 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:25 smithi006 bash[33918]: cluster 2023-08-27T21:57:24.678393+0000 mgr.y (mgr.14430) 178 : cluster [DBG] pgmap v157: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:26.232 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:25 smithi006 bash[27292]: cluster 2023-08-27T21:57:24.678393+0000 mgr.y (mgr.14430) 178 : cluster [DBG] pgmap v157: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:27.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:57:26 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:57:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:57:28.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:27 smithi130 bash[27579]: cluster 2023-08-27T21:57:26.679393+0000 mgr.y (mgr.14430) 179 : cluster [DBG] pgmap v158: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:28.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:27 smithi006 bash[33918]: cluster 2023-08-27T21:57:26.679393+0000 mgr.y (mgr.14430) 179 : cluster [DBG] pgmap v158: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:28.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:27 smithi006 bash[27292]: cluster 2023-08-27T21:57:26.679393+0000 mgr.y (mgr.14430) 179 : cluster [DBG] pgmap v158: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:29.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:28 smithi006 bash[33918]: cluster 2023-08-27T21:57:28.680052+0000 mgr.y (mgr.14430) 180 : cluster [DBG] pgmap v159: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:29.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:28 smithi006 bash[27292]: cluster 2023-08-27T21:57:28.680052+0000 mgr.y (mgr.14430) 180 : cluster [DBG] pgmap v159: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:29.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:28 smithi130 bash[27579]: cluster 2023-08-27T21:57:28.680052+0000 mgr.y (mgr.14430) 180 : cluster [DBG] pgmap v159: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:31.161 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:57:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:57:30] "GET /metrics HTTP/1.1" 200 193920 "" "Prometheus/2.33.4" 2023-08-27T21:57:32.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:31 smithi130 bash[27579]: cluster 2023-08-27T21:57:30.681100+0000 mgr.y (mgr.14430) 181 : cluster [DBG] pgmap v160: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:32.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:31 smithi006 bash[33918]: cluster 2023-08-27T21:57:30.681100+0000 mgr.y (mgr.14430) 181 : cluster [DBG] pgmap v160: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:32.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:31 smithi006 bash[27292]: cluster 2023-08-27T21:57:30.681100+0000 mgr.y (mgr.14430) 181 : cluster [DBG] pgmap v160: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:33.776 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-08-27T21:57:33.777 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.a smithi006 *:9093,9094 running (3m) 26s ago 4m 12.9M - 0.23.0 ba2b418f427c 4923e642bd06 2023-08-27T21:57:33.777 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.a smithi130 *:3000 running (3m) 26s ago 3m 43.9M - 8.3.5 dad864ee21e9 34693aa2d9b4 2023-08-27T21:57:33.777 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foo.smithi006.bzplue smithi006 running (2m) 26s ago 2m 13.2M - 16.2.13-565-g595355f4 4099d2e00414 80bd87f6a82e 2023-08-27T21:57:33.777 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foo.smithi130.qmrguj smithi130 running (2m) 26s ago 2m 11.4M - 16.2.13-565-g595355f4 4099d2e00414 50fe6c812436 2023-08-27T21:57:33.777 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.x smithi130 *:8443 running (11m) 26s ago 11m 375M - 16.2.13-565-g595355f4 4099d2e00414 34f4750f5be8 2023-08-27T21:57:33.777 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.y smithi006 *:9283 running (13m) 26s ago 13m 426M - 16.2.13-565-g595355f4 4099d2e00414 166210c5975b 2023-08-27T21:57:33.777 INFO:teuthology.orchestra.run.smithi006.stdout:mon.a smithi006 running (13m) 26s ago 13m 51.5M 2048M 16.2.13-565-g595355f4 4099d2e00414 27ce6f52a384 2023-08-27T21:57:33.777 INFO:teuthology.orchestra.run.smithi006.stdout:mon.b smithi130 running (11m) 26s ago 11m 45.6M 2048M 16.2.13-565-g595355f4 4099d2e00414 0858e5854770 2023-08-27T21:57:33.777 INFO:teuthology.orchestra.run.smithi006.stdout:mon.c smithi006 running (11m) 26s ago 11m 44.3M 2048M 16.2.13-565-g595355f4 4099d2e00414 398d4f4de897 2023-08-27T21:57:33.777 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.a smithi006 *:9100 running (4m) 26s ago 4m 9455k - 1.3.1 1dbe0e931976 6d7afc4bda6d 2023-08-27T21:57:33.777 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.b smithi130 *:9100 running (4m) 26s ago 4m 9228k - 1.3.1 1dbe0e931976 268455e5c610 2023-08-27T21:57:33.778 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (10m) 26s ago 10m 38.8M 4061M 16.2.13-565-g595355f4 4099d2e00414 c0cabbdbdfd7 2023-08-27T21:57:33.778 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (10m) 26s ago 10m 38.8M 4061M 16.2.13-565-g595355f4 4099d2e00414 253d8e8df45d 2023-08-27T21:57:33.778 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (9m) 26s ago 9m 37.3M 4061M 16.2.13-565-g595355f4 4099d2e00414 c9bd522d044e 2023-08-27T21:57:33.778 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (8m) 26s ago 8m 36.9M 4061M 16.2.13-565-g595355f4 4099d2e00414 28b97ad36f98 2023-08-27T21:57:33.778 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi130 running (7m) 26s ago 7m 37.5M 4316M 16.2.13-565-g595355f4 4099d2e00414 748a464896a0 2023-08-27T21:57:33.778 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi130 running (7m) 26s ago 7m 38.0M 4316M 16.2.13-565-g595355f4 4099d2e00414 1b14aad2f694 2023-08-27T21:57:33.778 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi130 running (6m) 26s ago 6m 37.4M 4316M 16.2.13-565-g595355f4 4099d2e00414 7142e53b64e4 2023-08-27T21:57:33.778 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi130 running (5m) 26s ago 5m 40.3M 4316M 16.2.13-565-g595355f4 4099d2e00414 50ee72d76aad 2023-08-27T21:57:33.778 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.a smithi130 *:9095 running (3m) 26s ago 4m 51.2M - 2.33.4 514e6a882f6e bcdeee5b85f1 2023-08-27T21:57:33.867 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:33 smithi006 bash[27292]: cluster 2023-08-27T21:57:32.681633+0000 mgr.y (mgr.14430) 182 : cluster [DBG] pgmap v161: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:33.868 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:33 smithi006 bash[27292]: audit 2023-08-27T21:57:33.737715+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:34.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:33 smithi130 bash[27579]: cluster 2023-08-27T21:57:32.681633+0000 mgr.y (mgr.14430) 182 : cluster [DBG] pgmap v161: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:34.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:33 smithi130 bash[27579]: audit 2023-08-27T21:57:33.737715+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:34.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:33 smithi006 bash[33918]: cluster 2023-08-27T21:57:32.681633+0000 mgr.y (mgr.14430) 182 : cluster [DBG] pgmap v161: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:34.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:33 smithi006 bash[33918]: audit 2023-08-27T21:57:33.737715+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:35.035 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:34 smithi006 bash[33918]: cephadm 2023-08-27T21:57:33.738173+0000 mgr.y (mgr.14430) 183 : cephadm [INF] Upgrade: Target is version 18.2.0-187-g9cf20820 (unknown) 2023-08-27T21:57:35.035 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:34 smithi006 bash[33918]: cephadm 2023-08-27T21:57:33.738257+0000 mgr.y (mgr.14430) 184 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8'] 2023-08-27T21:57:35.036 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:34 smithi006 bash[33918]: audit 2023-08-27T21:57:33.739519+0000 mon.a (mon.0) 768 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:57:35.036 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:34 smithi006 bash[33918]: audit 2023-08-27T21:57:33.750511+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:35.036 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:34 smithi006 bash[33918]: cephadm 2023-08-27T21:57:33.751463+0000 mgr.y (mgr.14430) 185 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-08-27T21:57:35.036 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:34 smithi006 bash[33918]: audit 2023-08-27T21:57:33.755754+0000 mgr.y (mgr.14430) 186 : audit [DBG] from='client.24716 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:57:35.036 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:34 smithi006 bash[27292]: cephadm 2023-08-27T21:57:33.738173+0000 mgr.y (mgr.14430) 183 : cephadm [INF] Upgrade: Target is version 18.2.0-187-g9cf20820 (unknown) 2023-08-27T21:57:35.036 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:34 smithi006 bash[27292]: cephadm 2023-08-27T21:57:33.738257+0000 mgr.y (mgr.14430) 184 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8'] 2023-08-27T21:57:35.036 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:34 smithi006 bash[27292]: audit 2023-08-27T21:57:33.739519+0000 mon.a (mon.0) 768 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:57:35.036 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:34 smithi006 bash[27292]: audit 2023-08-27T21:57:33.750511+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:35.036 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:34 smithi006 bash[27292]: cephadm 2023-08-27T21:57:33.751463+0000 mgr.y (mgr.14430) 185 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-08-27T21:57:35.037 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:34 smithi006 bash[27292]: audit 2023-08-27T21:57:33.755754+0000 mgr.y (mgr.14430) 186 : audit [DBG] from='client.24716 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:57:35.037 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'echo wait for minority of mons to upgrade' 2023-08-27T21:57:35.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:34 smithi130 bash[27579]: cephadm 2023-08-27T21:57:33.738173+0000 mgr.y (mgr.14430) 183 : cephadm [INF] Upgrade: Target is version 18.2.0-187-g9cf20820 (unknown) 2023-08-27T21:57:35.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:34 smithi130 bash[27579]: cephadm 2023-08-27T21:57:33.738257+0000 mgr.y (mgr.14430) 184 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8'] 2023-08-27T21:57:35.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:34 smithi130 bash[27579]: audit 2023-08-27T21:57:33.739519+0000 mon.a (mon.0) 768 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:57:35.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:34 smithi130 bash[27579]: audit 2023-08-27T21:57:33.750511+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:57:35.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:34 smithi130 bash[27579]: cephadm 2023-08-27T21:57:33.751463+0000 mgr.y (mgr.14430) 185 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-08-27T21:57:35.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:34 smithi130 bash[27579]: audit 2023-08-27T21:57:33.755754+0000 mgr.y (mgr.14430) 186 : audit [DBG] from='client.24716 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T21:57:35.777 INFO:teuthology.orchestra.run.smithi006.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-08-27T21:57:36.062 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:35 smithi006 bash[33918]: cephadm 2023-08-27T21:57:33.983162+0000 mgr.y (mgr.14430) 187 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 on smithi130 2023-08-27T21:57:36.062 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:35 smithi006 bash[33918]: cluster 2023-08-27T21:57:34.682276+0000 mgr.y (mgr.14430) 188 : cluster [DBG] pgmap v162: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:36.062 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:35 smithi006 bash[33918]: audit 2023-08-27T21:57:35.610341+0000 mon.c (mon.2) 21 : audit [INF] from='client.? 172.21.15.6:0/1153185701' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T21:57:36.063 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:35 smithi006 bash[33918]: audit 2023-08-27T21:57:35.610868+0000 mon.a (mon.0) 770 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T21:57:36.063 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:35 smithi006 bash[27292]: cephadm 2023-08-27T21:57:33.983162+0000 mgr.y (mgr.14430) 187 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 on smithi130 2023-08-27T21:57:36.063 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:35 smithi006 bash[27292]: cluster 2023-08-27T21:57:34.682276+0000 mgr.y (mgr.14430) 188 : cluster [DBG] pgmap v162: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:36.063 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:35 smithi006 bash[27292]: audit 2023-08-27T21:57:35.610341+0000 mon.c (mon.2) 21 : audit [INF] from='client.? 172.21.15.6:0/1153185701' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T21:57:36.063 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:35 smithi006 bash[27292]: audit 2023-08-27T21:57:35.610868+0000 mon.a (mon.0) 770 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T21:57:36.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:35 smithi130 bash[27579]: cephadm 2023-08-27T21:57:33.983162+0000 mgr.y (mgr.14430) 187 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 on smithi130 2023-08-27T21:57:36.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:35 smithi130 bash[27579]: cluster 2023-08-27T21:57:34.682276+0000 mgr.y (mgr.14430) 188 : cluster [DBG] pgmap v162: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:36.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:35 smithi130 bash[27579]: audit 2023-08-27T21:57:35.610341+0000 mon.c (mon.2) 21 : audit [INF] from='client.? 172.21.15.6:0/1153185701' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T21:57:36.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:35 smithi130 bash[27579]: audit 2023-08-27T21:57:35.610868+0000 mon.a (mon.0) 770 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T21:57:37.064 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:36 smithi006 bash[33918]: audit 2023-08-27T21:57:35.767351+0000 mon.a (mon.0) 771 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-27T21:57:37.064 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:36 smithi006 bash[33918]: cluster 2023-08-27T21:57:35.767547+0000 mon.a (mon.0) 772 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-08-27T21:57:37.064 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:36 smithi006 bash[27292]: audit 2023-08-27T21:57:35.767351+0000 mon.a (mon.0) 771 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-27T21:57:37.065 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:36 smithi006 bash[27292]: cluster 2023-08-27T21:57:35.767547+0000 mon.a (mon.0) 772 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-08-27T21:57:37.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:36 smithi130 bash[27579]: audit 2023-08-27T21:57:35.767351+0000 mon.a (mon.0) 771 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-27T21:57:37.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:36 smithi130 bash[27579]: cluster 2023-08-27T21:57:35.767547+0000 mon.a (mon.0) 772 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-08-27T21:57:37.120 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:57:36 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:57:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:57:37.844 DEBUG:teuthology.orchestra.run.smithi006:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-08-27T21:57:37.846 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-08-27T21:57:37.847 INFO:teuthology.orchestra.run:waiting for 3000 2023-08-27T21:57:37.893 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:hints = 1 2023-08-27T21:57:37.893 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-08-27T21:57:37.893 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Object prefix: benchmark_data_smithi006_67820 2023-08-27T21:57:37.894 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-27T21:57:37.894 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 0 0 0 0 0 0 - 0 2023-08-27T21:57:38.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:37 smithi130 bash[27579]: cluster 2023-08-27T21:57:36.682922+0000 mgr.y (mgr.14430) 189 : cluster [DBG] pgmap v164: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:38.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:37 smithi006 bash[33918]: cluster 2023-08-27T21:57:36.682922+0000 mgr.y (mgr.14430) 189 : cluster [DBG] pgmap v164: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:38.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:37 smithi006 bash[27292]: cluster 2023-08-27T21:57:36.682922+0000 mgr.y (mgr.14430) 189 : cluster [DBG] pgmap v164: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:38.894 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 1 16 2688 2672 166.99 167 0.00329246 0.005965 2023-08-27T21:57:39.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:38 smithi006 bash[33918]: cluster 2023-08-27T21:57:38.683369+0000 mgr.y (mgr.14430) 190 : cluster [DBG] pgmap v165: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:39.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:38 smithi006 bash[27292]: cluster 2023-08-27T21:57:38.683369+0000 mgr.y (mgr.14430) 190 : cluster [DBG] pgmap v165: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:39.317 INFO:teuthology.orchestra.run.smithi006.stdout:wait for minority of mons to upgrade 2023-08-27T21:57:39.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:38 smithi130 bash[27579]: cluster 2023-08-27T21:57:38.683369+0000 mgr.y (mgr.14430) 190 : cluster [DBG] pgmap v165: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T21:57:39.894 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 2 16 5248 5232 163.487 160 0.00368082 0.006099 2023-08-27T21:57:40.588 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-08-27T21:57:40.894 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 3 16 7586 7570 157.692 146.125 0.00527262 0.00633141 2023-08-27T21:57:40.904 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:57:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:57:40] "GET /metrics HTTP/1.1" 200 193920 "" "Prometheus/2.33.4" 2023-08-27T21:57:41.895 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 4 16 9714 9698 151.516 133 0.0062035 0.00659104 2023-08-27T21:57:42.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:41 smithi130 bash[27579]: cluster 2023-08-27T21:57:40.684440+0000 mgr.y (mgr.14430) 191 : cluster [DBG] pgmap v166: 89 pgs: 89 active+clean; 436 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s 2023-08-27T21:57:42.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:41 smithi006 bash[27292]: cluster 2023-08-27T21:57:40.684440+0000 mgr.y (mgr.14430) 191 : cluster [DBG] pgmap v166: 89 pgs: 89 active+clean; 436 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s 2023-08-27T21:57:42.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:41 smithi006 bash[33918]: cluster 2023-08-27T21:57:40.684440+0000 mgr.y (mgr.14430) 191 : cluster [DBG] pgmap v166: 89 pgs: 89 active+clean; 436 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s 2023-08-27T21:57:42.894 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 5 16 11904 11888 148.585 136.875 0.00962082 0.00671943 2023-08-27T21:57:43.894 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 6 16 13573 13557 141.205 104.312 0.00355381 0.00707039 2023-08-27T21:57:44.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:43 smithi130 bash[27579]: cluster 2023-08-27T21:57:42.684918+0000 mgr.y (mgr.14430) 192 : cluster [DBG] pgmap v167: 89 pgs: 89 active+clean; 436 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s 2023-08-27T21:57:44.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:43 smithi006 bash[33918]: cluster 2023-08-27T21:57:42.684918+0000 mgr.y (mgr.14430) 192 : cluster [DBG] pgmap v167: 89 pgs: 89 active+clean; 436 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s 2023-08-27T21:57:44.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:43 smithi006 bash[27292]: cluster 2023-08-27T21:57:42.684918+0000 mgr.y (mgr.14430) 192 : cluster [DBG] pgmap v167: 89 pgs: 89 active+clean; 436 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s 2023-08-27T21:57:44.895 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 7 16 15591 15575 139.049 126.125 0.0163504 0.00718334 2023-08-27T21:57:45.894 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 8 16 17461 17445 136.276 116.875 0.0140572 0.00732943 2023-08-27T21:57:46.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:45 smithi130 bash[27579]: cluster 2023-08-27T21:57:44.685451+0000 mgr.y (mgr.14430) 193 : cluster [DBG] pgmap v168: 89 pgs: 89 active+clean; 436 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s 2023-08-27T21:57:46.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:45 smithi006 bash[33918]: cluster 2023-08-27T21:57:44.685451+0000 mgr.y (mgr.14430) 193 : cluster [DBG] pgmap v168: 89 pgs: 89 active+clean; 436 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s 2023-08-27T21:57:46.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:45 smithi006 bash[27292]: cluster 2023-08-27T21:57:44.685451+0000 mgr.y (mgr.14430) 193 : cluster [DBG] pgmap v168: 89 pgs: 89 active+clean; 436 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s 2023-08-27T21:57:46.895 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 9 16 19316 19300 134.014 115.938 0.00330967 0.00745237 2023-08-27T21:57:47.120 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:57:46 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:57:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:57:47.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:46 smithi130 bash[27579]: audit 2023-08-27T21:57:45.752952+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.6:0/631895032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:47.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:46 smithi006 bash[27292]: audit 2023-08-27T21:57:45.752952+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.6:0/631895032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:47.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:46 smithi006 bash[33918]: audit 2023-08-27T21:57:45.752952+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.6:0/631895032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:47.895 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 10 16 21143 21127 132.031 114.188 0.00466464 0.00756671 2023-08-27T21:57:48.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:47 smithi130 bash[27579]: cluster 2023-08-27T21:57:46.686262+0000 mgr.y (mgr.14430) 194 : cluster [DBG] pgmap v169: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-08-27T21:57:48.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:47 smithi006 bash[27292]: cluster 2023-08-27T21:57:46.686262+0000 mgr.y (mgr.14430) 194 : cluster [DBG] pgmap v169: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-08-27T21:57:48.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:47 smithi006 bash[33918]: cluster 2023-08-27T21:57:46.686262+0000 mgr.y (mgr.14430) 194 : cluster [DBG] pgmap v169: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-08-27T21:57:48.895 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 11 16 23097 23081 131.129 122.125 0.00464025 0.00761915 2023-08-27T21:57:49.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:48 smithi006 bash[33918]: audit 2023-08-27T21:57:48.408488+0000 mon.a (mon.0) 773 : audit [DBG] from='client.? 172.21.15.6:0/932471301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:49.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:48 smithi006 bash[33918]: cluster 2023-08-27T21:57:48.686734+0000 mgr.y (mgr.14430) 195 : cluster [DBG] pgmap v170: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-08-27T21:57:49.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:48 smithi006 bash[27292]: audit 2023-08-27T21:57:48.408488+0000 mon.a (mon.0) 773 : audit [DBG] from='client.? 172.21.15.6:0/932471301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:49.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:48 smithi006 bash[27292]: cluster 2023-08-27T21:57:48.686734+0000 mgr.y (mgr.14430) 195 : cluster [DBG] pgmap v170: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-08-27T21:57:49.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:48 smithi130 bash[27579]: audit 2023-08-27T21:57:48.408488+0000 mon.a (mon.0) 773 : audit [DBG] from='client.? 172.21.15.6:0/932471301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:49.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:48 smithi130 bash[27579]: cluster 2023-08-27T21:57:48.686734+0000 mgr.y (mgr.14430) 195 : cluster [DBG] pgmap v170: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-08-27T21:57:49.895 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 12 15 25123 25108 130.758 126.688 0.00987591 0.00764175 2023-08-27T21:57:50.895 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 13 16 27065 27049 130.031 121.312 0.00461774 0.00768404 2023-08-27T21:57:51.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:57:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:57:50] "GET /metrics HTTP/1.1" 200 194437 "" "Prometheus/2.33.4" 2023-08-27T21:57:51.895 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 14 16 29039 29023 129.554 123.375 0.00969061 0.00771294 2023-08-27T21:57:52.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:51 smithi130 bash[27579]: cluster 2023-08-27T21:57:50.687560+0000 mgr.y (mgr.14430) 196 : cluster [DBG] pgmap v171: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-08-27T21:57:52.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:51 smithi130 bash[27579]: audit 2023-08-27T21:57:51.068408+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.6:0/644027636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:52.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:51 smithi006 bash[33918]: cluster 2023-08-27T21:57:50.687560+0000 mgr.y (mgr.14430) 196 : cluster [DBG] pgmap v171: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-08-27T21:57:52.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:51 smithi006 bash[33918]: audit 2023-08-27T21:57:51.068408+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.6:0/644027636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:52.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:51 smithi006 bash[27292]: cluster 2023-08-27T21:57:50.687560+0000 mgr.y (mgr.14430) 196 : cluster [DBG] pgmap v171: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-08-27T21:57:52.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:51 smithi006 bash[27292]: audit 2023-08-27T21:57:51.068408+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.6:0/644027636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:52.895 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 15 16 31003 30987 129.1 122.75 0.00600349 0.00773876 2023-08-27T21:57:53.895 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 16 16 32718 32702 127.729 107.188 0.0020647 0.00780267 2023-08-27T21:57:54.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:53 smithi130 bash[27579]: cluster 2023-08-27T21:57:52.688108+0000 mgr.y (mgr.14430) 197 : cluster [DBG] pgmap v172: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-08-27T21:57:54.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:53 smithi130 bash[27579]: audit 2023-08-27T21:57:53.714911+0000 mon.a (mon.0) 774 : audit [DBG] from='client.? 172.21.15.6:0/789789717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:54.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:53 smithi006 bash[27292]: cluster 2023-08-27T21:57:52.688108+0000 mgr.y (mgr.14430) 197 : cluster [DBG] pgmap v172: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-08-27T21:57:54.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:53 smithi006 bash[27292]: audit 2023-08-27T21:57:53.714911+0000 mon.a (mon.0) 774 : audit [DBG] from='client.? 172.21.15.6:0/789789717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:54.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:53 smithi006 bash[33918]: cluster 2023-08-27T21:57:52.688108+0000 mgr.y (mgr.14430) 197 : cluster [DBG] pgmap v172: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-08-27T21:57:54.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:53 smithi006 bash[33918]: audit 2023-08-27T21:57:53.714911+0000 mon.a (mon.0) 774 : audit [DBG] from='client.? 172.21.15.6:0/789789717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:54.895 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 17 15 34515 34500 126.825 112.375 0.0105857 0.00787895 2023-08-27T21:57:55.896 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 18 16 36315 36299 126.025 112.438 0.00853981 0.00792816 2023-08-27T21:57:56.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:55 smithi130 bash[27579]: cluster 2023-08-27T21:57:54.688514+0000 mgr.y (mgr.14430) 198 : cluster [DBG] pgmap v173: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-08-27T21:57:56.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:55 smithi006 bash[27292]: cluster 2023-08-27T21:57:54.688514+0000 mgr.y (mgr.14430) 198 : cluster [DBG] pgmap v173: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-08-27T21:57:56.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:55 smithi006 bash[33918]: cluster 2023-08-27T21:57:54.688514+0000 mgr.y (mgr.14430) 198 : cluster [DBG] pgmap v173: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-08-27T21:57:56.896 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 19 16 38005 37989 124.951 105.625 0.00838543 0.00799785 2023-08-27T21:57:57.120 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:57:56 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:57:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:57:57.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:56 smithi130 bash[27579]: audit 2023-08-27T21:57:56.366827+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.6:0/455851607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:57.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:56 smithi006 bash[33918]: audit 2023-08-27T21:57:56.366827+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.6:0/455851607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:57.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:56 smithi006 bash[27292]: audit 2023-08-27T21:57:56.366827+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.6:0/455851607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:57.896 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:2023-08-27T21:57:57.890927+0000 min lat: 0.00143029 max lat: 0.168897 avg lat: 0.0080564 2023-08-27T21:57:57.896 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-27T21:57:57.896 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 20 16 39709 39693 124.028 106.5 0.00722138 0.0080564 2023-08-27T21:57:58.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:57 smithi130 bash[27579]: cluster 2023-08-27T21:57:56.689359+0000 mgr.y (mgr.14430) 199 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 150 MiB/s wr, 2.41k op/s 2023-08-27T21:57:58.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:57 smithi006 bash[27292]: cluster 2023-08-27T21:57:56.689359+0000 mgr.y (mgr.14430) 199 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 150 MiB/s wr, 2.41k op/s 2023-08-27T21:57:58.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:57 smithi006 bash[33918]: cluster 2023-08-27T21:57:56.689359+0000 mgr.y (mgr.14430) 199 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 150 MiB/s wr, 2.41k op/s 2023-08-27T21:57:58.896 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 21 16 41312 41296 122.893 100.188 0.00983216 0.00813075 2023-08-27T21:57:59.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:58 smithi006 bash[33918]: cluster 2023-08-27T21:57:58.689894+0000 mgr.y (mgr.14430) 200 : cluster [DBG] pgmap v175: 89 pgs: 89 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-08-27T21:57:59.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:58 smithi006 bash[27292]: cluster 2023-08-27T21:57:58.689894+0000 mgr.y (mgr.14430) 200 : cluster [DBG] pgmap v175: 89 pgs: 89 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-08-27T21:57:59.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:58 smithi130 bash[27579]: cluster 2023-08-27T21:57:58.689894+0000 mgr.y (mgr.14430) 200 : cluster [DBG] pgmap v175: 89 pgs: 89 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-08-27T21:57:59.896 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 22 16 42913 42897 121.854 100.062 0.00380633 0.00820069 2023-08-27T21:58:00.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:57:59 smithi006 bash[27292]: audit 2023-08-27T21:57:59.040850+0000 mon.a (mon.0) 775 : audit [DBG] from='client.? 172.21.15.6:0/3543439297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:00.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:57:59 smithi006 bash[33918]: audit 2023-08-27T21:57:59.040850+0000 mon.a (mon.0) 775 : audit [DBG] from='client.? 172.21.15.6:0/3543439297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:00.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:57:59 smithi130 bash[27579]: audit 2023-08-27T21:57:59.040850+0000 mon.a (mon.0) 775 : audit [DBG] from='client.? 172.21.15.6:0/3543439297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:00.896 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 23 16 44446 44430 120.722 95.8125 0.0101653 0.00827714 2023-08-27T21:58:01.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:00 smithi006 bash[27292]: cluster 2023-08-27T21:58:00.690786+0000 mgr.y (mgr.14430) 201 : cluster [DBG] pgmap v176: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-08-27T21:58:01.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:58:00] "GET /metrics HTTP/1.1" 200 194750 "" "Prometheus/2.33.4" 2023-08-27T21:58:01.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:00 smithi006 bash[33918]: cluster 2023-08-27T21:58:00.690786+0000 mgr.y (mgr.14430) 201 : cluster [DBG] pgmap v176: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-08-27T21:58:01.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:00 smithi130 bash[27579]: cluster 2023-08-27T21:58:00.690786+0000 mgr.y (mgr.14430) 201 : cluster [DBG] pgmap v176: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-08-27T21:58:01.896 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 24 16 45920 45904 119.53 92.125 0.0185453 0.00836033 2023-08-27T21:58:02.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:01 smithi130 bash[27579]: audit 2023-08-27T21:58:01.708822+0000 mon.b (mon.1) 29 : audit [DBG] from='client.? 172.21.15.6:0/3103464595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:02.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:01 smithi006 bash[27292]: audit 2023-08-27T21:58:01.708822+0000 mon.b (mon.1) 29 : audit [DBG] from='client.? 172.21.15.6:0/3103464595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:02.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:01 smithi006 bash[33918]: audit 2023-08-27T21:58:01.708822+0000 mon.b (mon.1) 29 : audit [DBG] from='client.? 172.21.15.6:0/3103464595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:02.896 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 25 16 47441 47425 118.551 95.0625 0.00676639 0.00842944 2023-08-27T21:58:03.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:02 smithi130 bash[27579]: cluster 2023-08-27T21:58:02.691237+0000 mgr.y (mgr.14430) 202 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-08-27T21:58:03.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:02 smithi006 bash[27292]: cluster 2023-08-27T21:58:02.691237+0000 mgr.y (mgr.14430) 202 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-08-27T21:58:03.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:02 smithi006 bash[33918]: cluster 2023-08-27T21:58:02.691237+0000 mgr.y (mgr.14430) 202 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-08-27T21:58:03.896 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 26 16 48948 48932 117.613 94.1875 0.00475408 0.0084959 2023-08-27T21:58:04.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:04 smithi006 bash[27292]: audit 2023-08-27T21:58:04.385171+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.6:0/4155254103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:04.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:04 smithi006 bash[33918]: audit 2023-08-27T21:58:04.385171+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.6:0/4155254103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:04.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:04 smithi130 bash[27579]: audit 2023-08-27T21:58:04.385171+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.6:0/4155254103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:04.896 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 27 16 50373 50357 116.556 89.0625 0.00519581 0.00857332 2023-08-27T21:58:05.500 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:05 smithi130 bash[27579]: cluster 2023-08-27T21:58:04.691716+0000 mgr.y (mgr.14430) 203 : cluster [DBG] pgmap v178: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-08-27T21:58:05.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:05 smithi006 bash[27292]: cluster 2023-08-27T21:58:04.691716+0000 mgr.y (mgr.14430) 203 : cluster [DBG] pgmap v178: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-08-27T21:58:05.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:05 smithi006 bash[33918]: cluster 2023-08-27T21:58:04.691716+0000 mgr.y (mgr.14430) 203 : cluster [DBG] pgmap v178: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-08-27T21:58:05.896 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 28 16 51744 51728 115.453 85.6875 0.00895343 0.00865469 2023-08-27T21:58:06.897 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 29 16 53067 53051 114.323 82.6875 0.0692502 0.00874002 2023-08-27T21:58:07.120 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:06 smithi130 bash[28467]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:58:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:58:07.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:07 smithi130 bash[27579]: cephadm 2023-08-27T21:58:06.495612+0000 mgr.y (mgr.14430) 204 : cephadm [INF] Upgrade: Updating mgr.x 2023-08-27T21:58:07.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:07 smithi130 bash[27579]: audit 2023-08-27T21:58:06.508850+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:58:07.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:07 smithi130 bash[27579]: audit 2023-08-27T21:58:06.511583+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-27T21:58:07.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:07 smithi130 bash[27579]: audit 2023-08-27T21:58:06.512812+0000 mon.a (mon.0) 778 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-27T21:58:07.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:07 smithi130 bash[27579]: audit 2023-08-27T21:58:06.516737+0000 mon.a (mon.0) 779 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:58:07.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:07 smithi130 bash[27579]: cephadm 2023-08-27T21:58:06.517735+0000 mgr.y (mgr.14430) 205 : cephadm [INF] Deploying daemon mgr.x on smithi130 2023-08-27T21:58:07.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:07 smithi130 bash[27579]: cluster 2023-08-27T21:58:06.692498+0000 mgr.y (mgr.14430) 206 : cluster [DBG] pgmap v179: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-08-27T21:58:07.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:07 smithi130 bash[27579]: audit 2023-08-27T21:58:07.038576+0000 mon.a (mon.0) 780 : audit [DBG] from='client.? 172.21.15.6:0/2187641710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:07.897 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 30 16 54465 54449 113.424 87.375 0.00926103 0.00881118 2023-08-27T21:58:07.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:07 smithi006 bash[33918]: cephadm 2023-08-27T21:58:06.495612+0000 mgr.y (mgr.14430) 204 : cephadm [INF] Upgrade: Updating mgr.x 2023-08-27T21:58:07.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:07 smithi006 bash[33918]: audit 2023-08-27T21:58:06.508850+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:58:07.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:07 smithi006 bash[33918]: audit 2023-08-27T21:58:06.511583+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-27T21:58:07.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:07 smithi006 bash[33918]: audit 2023-08-27T21:58:06.512812+0000 mon.a (mon.0) 778 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-27T21:58:07.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:07 smithi006 bash[33918]: audit 2023-08-27T21:58:06.516737+0000 mon.a (mon.0) 779 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:58:07.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:07 smithi006 bash[33918]: cephadm 2023-08-27T21:58:06.517735+0000 mgr.y (mgr.14430) 205 : cephadm [INF] Deploying daemon mgr.x on smithi130 2023-08-27T21:58:07.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:07 smithi006 bash[33918]: cluster 2023-08-27T21:58:06.692498+0000 mgr.y (mgr.14430) 206 : cluster [DBG] pgmap v179: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-08-27T21:58:07.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:07 smithi006 bash[33918]: audit 2023-08-27T21:58:07.038576+0000 mon.a (mon.0) 780 : audit [DBG] from='client.? 172.21.15.6:0/2187641710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:07.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:07 smithi006 bash[27292]: cephadm 2023-08-27T21:58:06.495612+0000 mgr.y (mgr.14430) 204 : cephadm [INF] Upgrade: Updating mgr.x 2023-08-27T21:58:07.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:07 smithi006 bash[27292]: audit 2023-08-27T21:58:06.508850+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:58:07.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:07 smithi006 bash[27292]: audit 2023-08-27T21:58:06.511583+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-27T21:58:07.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:07 smithi006 bash[27292]: audit 2023-08-27T21:58:06.512812+0000 mon.a (mon.0) 778 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-27T21:58:07.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:07 smithi006 bash[27292]: audit 2023-08-27T21:58:06.516737+0000 mon.a (mon.0) 779 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:58:07.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:07 smithi006 bash[27292]: cephadm 2023-08-27T21:58:06.517735+0000 mgr.y (mgr.14430) 205 : cephadm [INF] Deploying daemon mgr.x on smithi130 2023-08-27T21:58:07.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:07 smithi006 bash[27292]: cluster 2023-08-27T21:58:06.692498+0000 mgr.y (mgr.14430) 206 : cluster [DBG] pgmap v179: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-08-27T21:58:07.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:07 smithi006 bash[27292]: audit 2023-08-27T21:58:07.038576+0000 mon.a (mon.0) 780 : audit [DBG] from='client.? 172.21.15.6:0/2187641710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:08.897 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 31 16 55822 55806 112.501 84.8125 0.00965737 0.00888239 2023-08-27T21:58:09.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:08 smithi006 bash[33918]: cluster 2023-08-27T21:58:08.692919+0000 mgr.y (mgr.14430) 207 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-08-27T21:58:09.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:08 smithi006 bash[27292]: cluster 2023-08-27T21:58:08.692919+0000 mgr.y (mgr.14430) 207 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-08-27T21:58:09.362 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:08 smithi130 bash[27579]: cluster 2023-08-27T21:58:08.692919+0000 mgr.y (mgr.14430) 207 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-08-27T21:58:09.897 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 32 16 57034 57018 111.352 75.75 0.00274518 0.00897243 2023-08-27T21:58:09.934 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:09 smithi130 systemd[1]: Stopping Ceph mgr.x for b696f380-4522-11ee-817b-1b7a5f1ba315... 2023-08-27T21:58:09.934 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:09 smithi130 bash[51079]: Error response from daemon: No such container: ceph-b696f380-4522-11ee-817b-1b7a5f1ba315-mgr.x 2023-08-27T21:58:10.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:09 smithi006 bash[33918]: audit 2023-08-27T21:58:09.639363+0000 mon.a (mon.0) 781 : audit [DBG] from='client.? 172.21.15.6:0/122147946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:10.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:09 smithi006 bash[27292]: audit 2023-08-27T21:58:09.639363+0000 mon.a (mon.0) 781 : audit [DBG] from='client.? 172.21.15.6:0/122147946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:10.311 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:09 smithi130 bash[27579]: audit 2023-08-27T21:58:09.639363+0000 mon.a (mon.0) 781 : audit [DBG] from='client.? 172.21.15.6:0/122147946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:10.897 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 33 16 58213 58197 110.211 73.6875 0.00951969 0.00906753 2023-08-27T21:58:11.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:10 smithi006 bash[27292]: cluster 2023-08-27T21:58:10.693574+0000 mgr.y (mgr.14430) 208 : cluster [DBG] pgmap v181: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-08-27T21:58:11.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:58:10] "GET /metrics HTTP/1.1" 200 194750 "" "Prometheus/2.33.4" 2023-08-27T21:58:11.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:10 smithi006 bash[33918]: cluster 2023-08-27T21:58:10.693574+0000 mgr.y (mgr.14430) 208 : cluster [DBG] pgmap v181: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-08-27T21:58:11.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:10 smithi130 bash[27579]: cluster 2023-08-27T21:58:10.693574+0000 mgr.y (mgr.14430) 208 : cluster [DBG] pgmap v181: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-08-27T21:58:11.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:10 smithi130 bash[51144]: ceph-b696f380-4522-11ee-817b-1b7a5f1ba315-mgr-x 2023-08-27T21:58:11.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:10 smithi130 systemd[1]: ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-08-27T21:58:11.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:11 smithi130 systemd[1]: ceph-b696f380-4522-11ee-817b-1b7a5f1ba315@mgr.x.service: Failed with result 'exit-code'. 2023-08-27T21:58:11.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:11 smithi130 systemd[1]: Stopped Ceph mgr.x for b696f380-4522-11ee-817b-1b7a5f1ba315. 2023-08-27T21:58:11.870 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:11 smithi130 systemd[1]: Started Ceph mgr.x for b696f380-4522-11ee-817b-1b7a5f1ba315. 2023-08-27T21:58:11.897 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 34 16 59363 59347 109.083 71.875 0.0141371 0.00916109 2023-08-27T21:58:12.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:12 smithi006 bash[33918]: audit 2023-08-27T21:58:11.424129+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:58:12.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:12 smithi006 bash[33918]: audit 2023-08-27T21:58:11.425903+0000 mon.a (mon.0) 783 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:58:12.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:12 smithi006 bash[33918]: audit 2023-08-27T21:58:12.300825+0000 mon.a (mon.0) 784 : audit [DBG] from='client.? 172.21.15.6:0/1078390230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:12.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:12 smithi006 bash[27292]: audit 2023-08-27T21:58:11.424129+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:58:12.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:12 smithi006 bash[27292]: audit 2023-08-27T21:58:11.425903+0000 mon.a (mon.0) 783 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:58:12.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:12 smithi006 bash[27292]: audit 2023-08-27T21:58:12.300825+0000 mon.a (mon.0) 784 : audit [DBG] from='client.? 172.21.15.6:0/1078390230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:12.853 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:12 smithi130 bash[27579]: audit 2023-08-27T21:58:11.424129+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:58:12.853 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:12 smithi130 bash[27579]: audit 2023-08-27T21:58:11.425903+0000 mon.a (mon.0) 783 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:58:12.854 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:12 smithi130 bash[27579]: audit 2023-08-27T21:58:12.300825+0000 mon.a (mon.0) 784 : audit [DBG] from='client.? 172.21.15.6:0/1078390230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:12.897 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 35 16 60612 60596 108.197 78.0625 0.0119931 0.0092371 2023-08-27T21:58:13.120 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:12 smithi130 bash[51302]: debug 2023-08-27T21:58:12.977+0000 7f7c92d23700 1 -- 172.21.15.130:0/2915505899 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557d8bdc31e0 con 0x557d8cb42800 2023-08-27T21:58:13.620 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:13 smithi130 bash[51302]: debug 2023-08-27T21:58:13.237+0000 7f7c9ce71200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-27T21:58:13.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:13 smithi130 bash[27579]: cluster 2023-08-27T21:58:12.695098+0000 mgr.y (mgr.14430) 209 : cluster [DBG] pgmap v182: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-08-27T21:58:13.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:13 smithi006 bash[27292]: cluster 2023-08-27T21:58:12.695098+0000 mgr.y (mgr.14430) 209 : cluster [DBG] pgmap v182: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-08-27T21:58:13.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:13 smithi006 bash[33918]: cluster 2023-08-27T21:58:12.695098+0000 mgr.y (mgr.14430) 209 : cluster [DBG] pgmap v182: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-08-27T21:58:13.897 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 36 16 61734 61718 107.139 70.125 0.0108823 0.00932705 2023-08-27T21:58:14.898 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 37 16 62854 62838 106.135 70 0.0176763 0.00941504 2023-08-27T21:58:14.937 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:14 smithi130 bash[51302]: debug 2023-08-27T21:58:14.633+0000 7f7c9ce71200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-08-27T21:58:14.937 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:14 smithi130 bash[51302]: debug 2023-08-27T21:58:14.785+0000 7f7c9ce71200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-27T21:58:14.938 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:14 smithi130 bash[51302]: debug 2023-08-27T21:58:14.929+0000 7f7c9ce71200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-27T21:58:15.314 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:15 smithi130 bash[51302]: debug 2023-08-27T21:58:15.309+0000 7f7c9ce71200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-27T21:58:15.898 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 38 16 63999 63983 105.225 71.5625 0.0198877 0.00949726 2023-08-27T21:58:16.095 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:15 smithi130 bash[27579]: cluster 2023-08-27T21:58:14.695566+0000 mgr.y (mgr.14430) 210 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-08-27T21:58:16.095 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:15 smithi130 bash[27579]: audit 2023-08-27T21:58:14.929848+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.6:0/1489934572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:16.096 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:15 smithi130 bash[51302]: debug 2023-08-27T21:58:15.845+0000 7f7c9ce71200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-27T21:58:16.096 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:16 smithi130 bash[51302]: debug 2023-08-27T21:58:16.089+0000 7f7c9ce71200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-27T21:58:16.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:15 smithi006 bash[33918]: cluster 2023-08-27T21:58:14.695566+0000 mgr.y (mgr.14430) 210 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-08-27T21:58:16.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:15 smithi006 bash[33918]: audit 2023-08-27T21:58:14.929848+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.6:0/1489934572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:16.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:15 smithi006 bash[27292]: cluster 2023-08-27T21:58:14.695566+0000 mgr.y (mgr.14430) 210 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-08-27T21:58:16.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:15 smithi006 bash[27292]: audit 2023-08-27T21:58:14.929848+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.6:0/1489934572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:16.870 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:16 smithi130 bash[51302]: debug 2023-08-27T21:58:16.417+0000 7f7c9ce71200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-27T21:58:16.898 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 39 16 65091 65075 104.277 68.25 0.0119495 0.00958336 2023-08-27T21:58:17.898 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:2023-08-27T21:58:17.892813+0000 min lat: 0.00143029 max lat: 0.168897 avg lat: 0.009692 2023-08-27T21:58:17.898 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-27T21:58:17.898 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 40 16 66009 65993 103.104 57.375 0.0217724 0.009692 2023-08-27T21:58:18.016 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:17 smithi130 bash[27579]: cluster 2023-08-27T21:58:16.696408+0000 mgr.y (mgr.14430) 211 : cluster [DBG] pgmap v184: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 102 MiB/s wr, 1.64k op/s 2023-08-27T21:58:18.017 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:17 smithi130 bash[27579]: audit 2023-08-27T21:58:17.532721+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:58:18.017 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:17 smithi130 bash[27579]: audit 2023-08-27T21:58:17.601170+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.6:0/4244160607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:18.017 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:18 smithi130 bash[51302]: debug 2023-08-27T21:58:18.009+0000 7f7c9ce71200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-27T21:58:18.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:17 smithi006 bash[33918]: cluster 2023-08-27T21:58:16.696408+0000 mgr.y (mgr.14430) 211 : cluster [DBG] pgmap v184: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 102 MiB/s wr, 1.64k op/s 2023-08-27T21:58:18.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:17 smithi006 bash[33918]: audit 2023-08-27T21:58:17.532721+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:58:18.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:17 smithi006 bash[33918]: audit 2023-08-27T21:58:17.601170+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.6:0/4244160607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:18.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:17 smithi006 bash[27292]: cluster 2023-08-27T21:58:16.696408+0000 mgr.y (mgr.14430) 211 : cluster [DBG] pgmap v184: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 102 MiB/s wr, 1.64k op/s 2023-08-27T21:58:18.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:17 smithi006 bash[27292]: audit 2023-08-27T21:58:17.532721+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:58:18.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:17 smithi006 bash[27292]: audit 2023-08-27T21:58:17.601170+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.6:0/4244160607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:18.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:18 smithi130 bash[51302]: debug 2023-08-27T21:58:18.201+0000 7f7c9ce71200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-08-27T21:58:18.870 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:18 smithi130 bash[51302]: debug 2023-08-27T21:58:18.397+0000 7f7c9ce71200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-27T21:58:18.898 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 41 16 66934 66918 101.999 57.8125 0.00647732 0.00979786 2023-08-27T21:58:19.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:18 smithi006 bash[27292]: cluster 2023-08-27T21:58:18.696811+0000 mgr.y (mgr.14430) 212 : cluster [DBG] pgmap v185: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-08-27T21:58:19.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:18 smithi006 bash[33918]: cluster 2023-08-27T21:58:18.696811+0000 mgr.y (mgr.14430) 212 : cluster [DBG] pgmap v185: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-08-27T21:58:19.223 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:18 smithi130 bash[27579]: cluster 2023-08-27T21:58:18.696811+0000 mgr.y (mgr.14430) 212 : cluster [DBG] pgmap v185: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-08-27T21:58:19.224 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:19 smithi130 bash[51302]: debug 2023-08-27T21:58:19.217+0000 7f7c9ce71200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-08-27T21:58:19.575 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:19 smithi130 bash[51302]: debug 2023-08-27T21:58:19.569+0000 7f7c9ce71200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-27T21:58:19.898 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 42 16 67887 67871 100.989 59.5625 0.0102745 0.00989545 2023-08-27T21:58:19.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:19 smithi006 bash[27566]: debug 2023-08-27T21:58:19.585+0000 7f9031d77700 -1 mgr handle_mgr_map I was active but no longer am 2023-08-27T21:58:19.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:19 smithi006 bash[27566]: ignoring --setuser ceph since I am not root 2023-08-27T21:58:19.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:19 smithi006 bash[27566]: ignoring --setgroup ceph since I am not root 2023-08-27T21:58:19.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:19 smithi006 bash[27566]: debug 2023-08-27T21:58:19.665+0000 7fa22590e700 1 -- 172.21.15.6:0/2244944353 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56364a2e64e0 con 0x56364a3da800 2023-08-27T21:58:20.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:19 smithi006 bash[27566]: debug 2023-08-27T21:58:19.957+0000 7fa235ad7680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-27T21:58:20.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:20 smithi006 bash[27566]: debug 2023-08-27T21:58:20.125+0000 7fa235ad7680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-27T21:58:20.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:20 smithi006 bash[33918]: audit 2023-08-27T21:58:19.188969+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:58:20.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:20 smithi006 bash[33918]: audit 2023-08-27T21:58:19.552884+0000 mon.a (mon.0) 787 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:58:20.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:20 smithi006 bash[33918]: audit 2023-08-27T21:58:19.554557+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:58:20.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:20 smithi006 bash[33918]: audit 2023-08-27T21:58:19.562020+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:58:20.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:20 smithi006 bash[33918]: audit 2023-08-27T21:58:19.571579+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:58:20.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:20 smithi006 bash[33918]: audit 2023-08-27T21:58:19.581758+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-08-27T21:58:20.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:20 smithi006 bash[33918]: cluster 2023-08-27T21:58:19.594728+0000 mon.a (mon.0) 792 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-08-27T21:58:20.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:20 smithi006 bash[33918]: audit 2023-08-27T21:58:19.596000+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-08-27T21:58:20.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:20 smithi006 bash[33918]: cluster 2023-08-27T21:58:19.596185+0000 mon.a (mon.0) 794 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0137529s) 2023-08-27T21:58:20.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:20 smithi006 bash[27292]: audit 2023-08-27T21:58:19.188969+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:58:20.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:20 smithi006 bash[27292]: audit 2023-08-27T21:58:19.552884+0000 mon.a (mon.0) 787 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:58:20.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:20 smithi006 bash[27292]: audit 2023-08-27T21:58:19.554557+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:58:20.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:20 smithi006 bash[27292]: audit 2023-08-27T21:58:19.562020+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:58:20.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:20 smithi006 bash[27292]: audit 2023-08-27T21:58:19.571579+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:58:20.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:20 smithi006 bash[27292]: audit 2023-08-27T21:58:19.581758+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-08-27T21:58:20.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:20 smithi006 bash[27292]: cluster 2023-08-27T21:58:19.594728+0000 mon.a (mon.0) 792 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-08-27T21:58:20.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:20 smithi006 bash[27292]: audit 2023-08-27T21:58:19.596000+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-08-27T21:58:20.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:20 smithi006 bash[27292]: cluster 2023-08-27T21:58:19.596185+0000 mon.a (mon.0) 794 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0137529s) 2023-08-27T21:58:20.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:20 smithi130 bash[27579]: audit 2023-08-27T21:58:19.188969+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:58:20.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:20 smithi130 bash[27579]: audit 2023-08-27T21:58:19.552884+0000 mon.a (mon.0) 787 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:58:20.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:20 smithi130 bash[27579]: audit 2023-08-27T21:58:19.554557+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:58:20.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:20 smithi130 bash[27579]: audit 2023-08-27T21:58:19.562020+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' 2023-08-27T21:58:20.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:20 smithi130 bash[27579]: audit 2023-08-27T21:58:19.571579+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:58:20.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:20 smithi130 bash[27579]: audit 2023-08-27T21:58:19.581758+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-08-27T21:58:20.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:20 smithi130 bash[27579]: cluster 2023-08-27T21:58:19.594728+0000 mon.a (mon.0) 792 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-08-27T21:58:20.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:20 smithi130 bash[27579]: audit 2023-08-27T21:58:19.596000+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.14430 172.21.15.6:0/175266859' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-08-27T21:58:20.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:20 smithi130 bash[27579]: cluster 2023-08-27T21:58:19.596185+0000 mon.a (mon.0) 794 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0137529s) 2023-08-27T21:58:20.621 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:20 smithi130 bash[51302]: debug 2023-08-27T21:58:20.217+0000 7f7c9ce71200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-27T21:58:20.898 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 43 16 68872 68856 100.072 61.5625 0.00957569 0.00998582 2023-08-27T21:58:21.197 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:20 smithi006 bash[27566]: debug 2023-08-27T21:58:20.853+0000 7fa235ad7680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-27T21:58:21.199 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:20 smithi130 bash[51302]: debug 2023-08-27T21:58:20.933+0000 7f7c9ce71200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-27T21:58:21.199 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:21 smithi130 bash[27579]: audit 2023-08-27T21:58:20.267791+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.6:0/3317062945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:21.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:21 smithi006 bash[33918]: audit 2023-08-27T21:58:20.267791+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.6:0/3317062945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:21.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:21 smithi006 bash[27292]: audit 2023-08-27T21:58:20.267791+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.6:0/3317062945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:21.898 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 44 16 69923 69907 99.29 65.6875 0.0177258 0.0100657 2023-08-27T21:58:22.120 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:21 smithi130 bash[51302]: debug 2023-08-27T21:58:21.721+0000 7f7c9ce71200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-27T21:58:22.120 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:21 smithi130 bash[51302]: debug 2023-08-27T21:58:21.865+0000 7f7c9ce71200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-27T21:58:22.620 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:22 smithi130 bash[51302]: debug 2023-08-27T21:58:22.333+0000 7f7c9ce71200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-27T21:58:22.620 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:22 smithi130 bash[51302]: debug 2023-08-27T21:58:22.485+0000 7f7c9ce71200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-27T21:58:22.720 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:22 smithi006 bash[27566]: debug 2023-08-27T21:58:22.349+0000 7fa235ad7680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-27T21:58:22.720 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:22 smithi006 bash[27566]: debug 2023-08-27T21:58:22.509+0000 7fa235ad7680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-27T21:58:22.898 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 45 16 70919 70903 98.4668 62.25 0.0254782 0.0101487 2023-08-27T21:58:22.963 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:22 smithi130 bash[51302]: debug 2023-08-27T21:58:22.693+0000 7f7c9ce71200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-27T21:58:22.963 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:22 smithi130 bash[51302]: debug 2023-08-27T21:58:22.845+0000 7f7c9ce71200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-27T21:58:23.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:22 smithi006 bash[27292]: audit 2023-08-27T21:58:22.904396+0000 mon.a (mon.0) 795 : audit [DBG] from='client.? 172.21.15.6:0/3856274970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:23.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:22 smithi006 bash[27566]: debug 2023-08-27T21:58:22.889+0000 7fa235ad7680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-27T21:58:23.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:22 smithi006 bash[33918]: audit 2023-08-27T21:58:22.904396+0000 mon.a (mon.0) 795 : audit [DBG] from='client.? 172.21.15.6:0/3856274970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:23.315 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:22 smithi130 bash[27579]: audit 2023-08-27T21:58:22.904396+0000 mon.a (mon.0) 795 : audit [DBG] from='client.? 172.21.15.6:0/3856274970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:23.315 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:23 smithi130 bash[51302]: debug 2023-08-27T21:58:23.025+0000 7f7c9ce71200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-27T21:58:23.315 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:23 smithi130 bash[51302]: debug 2023-08-27T21:58:23.309+0000 7f7c9ce71200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-27T21:58:23.620 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:23 smithi130 bash[51302]: debug 2023-08-27T21:58:23.493+0000 7f7c9ce71200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-27T21:58:23.898 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 46 16 71954 71938 97.7323 64.6875 0.0195979 0.0102256 2023-08-27T21:58:24.835 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:24 smithi006 bash[27566]: debug 2023-08-27T21:58:24.561+0000 7fa235ad7680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-27T21:58:24.835 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:24 smithi006 bash[27566]: debug 2023-08-27T21:58:24.825+0000 7fa235ad7680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-27T21:58:24.899 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 47 15 72941 72926 96.9666 61.75 0.0478335 0.0103054 2023-08-27T21:58:25.179 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:24 smithi006 bash[27566]: debug 2023-08-27T21:58:24.981+0000 7fa235ad7680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-27T21:58:25.470 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27566]: debug 2023-08-27T21:58:25.165+0000 7fa235ad7680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-27T21:58:25.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: cluster 2023-08-27T21:58:25.346471+0000 mon.a (mon.0) 796 : cluster [INF] Active manager daemon x restarted 2023-08-27T21:58:25.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: cluster 2023-08-27T21:58:25.347044+0000 mon.a (mon.0) 797 : cluster [INF] Activating manager daemon x 2023-08-27T21:58:25.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: audit 2023-08-27T21:58:25.352240+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.? 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-27T21:58:25.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: cluster 2023-08-27T21:58:25.361543+0000 mon.a (mon.0) 798 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-08-27T21:58:25.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: cluster 2023-08-27T21:58:25.362213+0000 mon.a (mon.0) 799 : cluster [DBG] mgrmap e21: x(active, starting, since 0.0154424s) 2023-08-27T21:58:25.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: audit 2023-08-27T21:58:25.369902+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-27T21:58:25.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: audit 2023-08-27T21:58:25.370407+0000 mon.c (mon.2) 30 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T21:58:25.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: audit 2023-08-27T21:58:25.370971+0000 mon.c (mon.2) 31 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:58:25.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: audit 2023-08-27T21:58:25.376496+0000 mon.c (mon.2) 32 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:58:25.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: audit 2023-08-27T21:58:25.377467+0000 mon.c (mon.2) 33 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi130.qmrguj"}]: dispatch 2023-08-27T21:58:25.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: audit 2023-08-27T21:58:25.379002+0000 mon.c (mon.2) 34 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi006.bzplue"}]: dispatch 2023-08-27T21:58:25.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: audit 2023-08-27T21:58:25.380406+0000 mon.c (mon.2) 35 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-27T21:58:25.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: audit 2023-08-27T21:58:25.381473+0000 mon.c (mon.2) 36 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:58:25.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: audit 2023-08-27T21:58:25.382652+0000 mon.c (mon.2) 37 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:58:25.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: audit 2023-08-27T21:58:25.383833+0000 mon.c (mon.2) 38 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:58:25.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: audit 2023-08-27T21:58:25.385037+0000 mon.c (mon.2) 39 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:58:25.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: audit 2023-08-27T21:58:25.385866+0000 mon.c (mon.2) 40 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:58:25.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: audit 2023-08-27T21:58:25.386698+0000 mon.c (mon.2) 41 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:58:25.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: audit 2023-08-27T21:58:25.387554+0000 mon.c (mon.2) 42 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:58:25.474 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: audit 2023-08-27T21:58:25.389265+0000 mon.c (mon.2) 43 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:58:25.474 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: audit 2023-08-27T21:58:25.390543+0000 mon.c (mon.2) 44 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T21:58:25.474 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: audit 2023-08-27T21:58:25.391872+0000 mon.c (mon.2) 45 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T21:58:25.474 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: audit 2023-08-27T21:58:25.393855+0000 mon.c (mon.2) 46 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T21:58:25.474 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27292]: audit 2023-08-27T21:58:25.394842+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T21:58:25.475 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: cluster 2023-08-27T21:58:25.346471+0000 mon.a (mon.0) 796 : cluster [INF] Active manager daemon x restarted 2023-08-27T21:58:25.475 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: cluster 2023-08-27T21:58:25.347044+0000 mon.a (mon.0) 797 : cluster [INF] Activating manager daemon x 2023-08-27T21:58:25.475 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: audit 2023-08-27T21:58:25.352240+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.? 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-27T21:58:25.475 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: cluster 2023-08-27T21:58:25.361543+0000 mon.a (mon.0) 798 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-08-27T21:58:25.475 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: cluster 2023-08-27T21:58:25.362213+0000 mon.a (mon.0) 799 : cluster [DBG] mgrmap e21: x(active, starting, since 0.0154424s) 2023-08-27T21:58:25.476 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: audit 2023-08-27T21:58:25.369902+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-27T21:58:25.476 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: audit 2023-08-27T21:58:25.370407+0000 mon.c (mon.2) 30 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T21:58:25.476 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: audit 2023-08-27T21:58:25.370971+0000 mon.c (mon.2) 31 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:58:25.476 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: audit 2023-08-27T21:58:25.376496+0000 mon.c (mon.2) 32 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:58:25.476 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: audit 2023-08-27T21:58:25.377467+0000 mon.c (mon.2) 33 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi130.qmrguj"}]: dispatch 2023-08-27T21:58:25.477 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: audit 2023-08-27T21:58:25.379002+0000 mon.c (mon.2) 34 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi006.bzplue"}]: dispatch 2023-08-27T21:58:25.477 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: audit 2023-08-27T21:58:25.380406+0000 mon.c (mon.2) 35 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-27T21:58:25.477 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: audit 2023-08-27T21:58:25.381473+0000 mon.c (mon.2) 36 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:58:25.477 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: audit 2023-08-27T21:58:25.382652+0000 mon.c (mon.2) 37 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:58:25.477 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: audit 2023-08-27T21:58:25.383833+0000 mon.c (mon.2) 38 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:58:25.477 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: audit 2023-08-27T21:58:25.385037+0000 mon.c (mon.2) 39 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:58:25.477 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: audit 2023-08-27T21:58:25.385866+0000 mon.c (mon.2) 40 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:58:25.477 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: audit 2023-08-27T21:58:25.386698+0000 mon.c (mon.2) 41 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:58:25.478 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: audit 2023-08-27T21:58:25.387554+0000 mon.c (mon.2) 42 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:58:25.478 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: audit 2023-08-27T21:58:25.389265+0000 mon.c (mon.2) 43 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:58:25.478 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: audit 2023-08-27T21:58:25.390543+0000 mon.c (mon.2) 44 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T21:58:25.478 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: audit 2023-08-27T21:58:25.391872+0000 mon.c (mon.2) 45 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T21:58:25.478 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: audit 2023-08-27T21:58:25.393855+0000 mon.c (mon.2) 46 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T21:58:25.478 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[33918]: audit 2023-08-27T21:58:25.394842+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T21:58:25.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: cluster 2023-08-27T21:58:25.346471+0000 mon.a (mon.0) 796 : cluster [INF] Active manager daemon x restarted 2023-08-27T21:58:25.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: cluster 2023-08-27T21:58:25.347044+0000 mon.a (mon.0) 797 : cluster [INF] Activating manager daemon x 2023-08-27T21:58:25.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: audit 2023-08-27T21:58:25.352240+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.? 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-27T21:58:25.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: cluster 2023-08-27T21:58:25.361543+0000 mon.a (mon.0) 798 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-08-27T21:58:25.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: cluster 2023-08-27T21:58:25.362213+0000 mon.a (mon.0) 799 : cluster [DBG] mgrmap e21: x(active, starting, since 0.0154424s) 2023-08-27T21:58:25.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: audit 2023-08-27T21:58:25.369902+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-27T21:58:25.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: audit 2023-08-27T21:58:25.370407+0000 mon.c (mon.2) 30 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T21:58:25.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: audit 2023-08-27T21:58:25.370971+0000 mon.c (mon.2) 31 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:58:25.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: audit 2023-08-27T21:58:25.376496+0000 mon.c (mon.2) 32 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:58:25.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: audit 2023-08-27T21:58:25.377467+0000 mon.c (mon.2) 33 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi130.qmrguj"}]: dispatch 2023-08-27T21:58:25.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: audit 2023-08-27T21:58:25.379002+0000 mon.c (mon.2) 34 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi006.bzplue"}]: dispatch 2023-08-27T21:58:25.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: audit 2023-08-27T21:58:25.380406+0000 mon.c (mon.2) 35 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-27T21:58:25.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: audit 2023-08-27T21:58:25.381473+0000 mon.c (mon.2) 36 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:58:25.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: audit 2023-08-27T21:58:25.382652+0000 mon.c (mon.2) 37 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:58:25.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: audit 2023-08-27T21:58:25.383833+0000 mon.c (mon.2) 38 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:58:25.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: audit 2023-08-27T21:58:25.385037+0000 mon.c (mon.2) 39 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:58:25.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: audit 2023-08-27T21:58:25.385866+0000 mon.c (mon.2) 40 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:58:25.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: audit 2023-08-27T21:58:25.386698+0000 mon.c (mon.2) 41 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:58:25.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: audit 2023-08-27T21:58:25.387554+0000 mon.c (mon.2) 42 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:58:25.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: audit 2023-08-27T21:58:25.389265+0000 mon.c (mon.2) 43 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:58:25.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: audit 2023-08-27T21:58:25.390543+0000 mon.c (mon.2) 44 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T21:58:25.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: audit 2023-08-27T21:58:25.391872+0000 mon.c (mon.2) 45 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T21:58:25.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: audit 2023-08-27T21:58:25.393855+0000 mon.c (mon.2) 46 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T21:58:25.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[27579]: audit 2023-08-27T21:58:25.394842+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T21:58:25.623 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[51302]: debug 2023-08-27T21:58:25.325+0000 7f7c9ce71200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-27T21:58:25.623 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[51302]: [27/Aug/2023:21:58:25] ENGINE Bus STARTING 2023-08-27T21:58:25.623 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[51302]: CherryPy Checker: 2023-08-27T21:58:25.623 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[51302]: The Application mounted at '' has an empty config. 2023-08-27T21:58:25.623 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[51302]: [27/Aug/2023:21:58:25] ENGINE Serving on http://:::9283 2023-08-27T21:58:25.623 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[51302]: [27/Aug/2023:21:58:25] ENGINE Bus STARTED 2023-08-27T21:58:25.623 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[51302]: [27/Aug/2023:21:58:25] ENGINE Bus STOPPING 2023-08-27T21:58:25.623 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[51302]: [27/Aug/2023:21:58:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-27T21:58:25.623 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 21:58:25 smithi130 bash[51302]: [27/Aug/2023:21:58:25] ENGINE Bus STOPPED 2023-08-27T21:58:25.899 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 48 16 73905 73889 96.2002 60.1875 0.0156361 0.0103886 2023-08-27T21:58:25.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:25 smithi006 bash[27566]: debug 2023-08-27T21:58:25.513+0000 7fa235ad7680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-27T21:58:26.412 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:26 smithi006 bash[27566]: debug 2023-08-27T21:58:26.097+0000 7fa235ad7680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-27T21:58:26.412 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:26 smithi006 bash[27566]: debug 2023-08-27T21:58:26.285+0000 7fa235ad7680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-27T21:58:26.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:26 smithi006 bash[27292]: audit 2023-08-27T21:58:25.397986+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-27T21:58:26.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:26 smithi006 bash[27292]: audit 2023-08-27T21:58:25.400553+0000 mon.c (mon.2) 49 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T21:58:26.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:26 smithi006 bash[27292]: audit 2023-08-27T21:58:25.603754+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.6:0/3899902336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:26.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:26 smithi006 bash[33918]: audit 2023-08-27T21:58:25.397986+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-27T21:58:26.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:26 smithi006 bash[33918]: audit 2023-08-27T21:58:25.400553+0000 mon.c (mon.2) 49 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T21:58:26.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:26 smithi006 bash[33918]: audit 2023-08-27T21:58:25.603754+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.6:0/3899902336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:26.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:26 smithi130 bash[27579]: audit 2023-08-27T21:58:25.397986+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-27T21:58:26.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:26 smithi130 bash[27579]: audit 2023-08-27T21:58:25.400553+0000 mon.c (mon.2) 49 : audit [DBG] from='mgr.24664 172.21.15.130:0/1330069309' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T21:58:26.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:26 smithi130 bash[27579]: audit 2023-08-27T21:58:25.603754+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.6:0/3899902336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:26.899 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 49 16 74946 74930 95.5646 65.0625 0.0227781 0.0104583 2023-08-27T21:58:27.899 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 50 16 75947 75931 94.9045 62.5625 0.0137995 0.0105296 2023-08-27T21:58:27.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:27 smithi006 bash[27566]: debug 2023-08-27T21:58:27.697+0000 7fa235ad7680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-27T21:58:27.971 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:27 smithi006 bash[27566]: debug 2023-08-27T21:58:27.856+0000 7fa235ad7680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-27T21:58:28.420 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:28 smithi006 bash[27566]: debug 2023-08-27T21:58:28.052+0000 7fa235ad7680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-27T21:58:28.420 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:28 smithi006 bash[27566]: debug 2023-08-27T21:58:28.408+0000 7fa235ad7680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-27T21:58:28.899 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 51 16 76940 76924 94.2604 62.0625 0.0146698 0.0106031 2023-08-27T21:58:29.087 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:28 smithi006 bash[27566]: debug 2023-08-27T21:58:28.820+0000 7fa235ad7680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-27T21:58:29.087 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:28 smithi006 bash[33918]: audit 2023-08-27T21:58:28.296871+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.6:0/3080272368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:29.087 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:28 smithi006 bash[27292]: audit 2023-08-27T21:58:28.296871+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.6:0/3080272368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:29.355 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:29 smithi006 bash[27566]: debug 2023-08-27T21:58:29.072+0000 7fa235ad7680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-27T21:58:29.355 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:29 smithi006 bash[27566]: debug 2023-08-27T21:58:29.340+0000 7fa235ad7680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-27T21:58:29.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:28 smithi130 bash[27579]: audit 2023-08-27T21:58:28.296871+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.6:0/3080272368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:29.899 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 52 16 77963 77947 93.6772 63.9375 0.01238 0.0106684 2023-08-27T21:58:29.963 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:29 smithi006 bash[27292]: cluster 2023-08-27T21:58:29.882401+0000 mon.a (mon.0) 800 : cluster [DBG] Standby manager daemon y started 2023-08-27T21:58:29.963 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:29 smithi006 bash[27292]: audit 2023-08-27T21:58:29.885481+0000 mon.a (mon.0) 801 : audit [DBG] from='mgr.? 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-08-27T21:58:29.963 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:29 smithi006 bash[27292]: audit 2023-08-27T21:58:29.886301+0000 mon.a (mon.0) 802 : audit [DBG] from='mgr.? 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T21:58:29.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:29 smithi006 bash[27292]: audit 2023-08-27T21:58:29.891169+0000 mon.a (mon.0) 803 : audit [DBG] from='mgr.? 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-08-27T21:58:29.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:29 smithi006 bash[27292]: audit 2023-08-27T21:58:29.894523+0000 mon.a (mon.0) 804 : audit [DBG] from='mgr.? 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T21:58:29.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:29 smithi006 bash[27566]: debug 2023-08-27T21:58:29.688+0000 7fa235ad7680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-27T21:58:29.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:29 smithi006 bash[27566]: debug 2023-08-27T21:58:29.856+0000 7fa235ad7680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-27T21:58:29.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:29 smithi006 bash[27566]: [27/Aug/2023:21:58:29] ENGINE Bus STARTING 2023-08-27T21:58:29.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:29 smithi006 bash[33918]: cluster 2023-08-27T21:58:29.882401+0000 mon.a (mon.0) 800 : cluster [DBG] Standby manager daemon y started 2023-08-27T21:58:29.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:29 smithi006 bash[33918]: audit 2023-08-27T21:58:29.885481+0000 mon.a (mon.0) 801 : audit [DBG] from='mgr.? 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-08-27T21:58:29.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:29 smithi006 bash[33918]: audit 2023-08-27T21:58:29.886301+0000 mon.a (mon.0) 802 : audit [DBG] from='mgr.? 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T21:58:29.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:29 smithi006 bash[33918]: audit 2023-08-27T21:58:29.891169+0000 mon.a (mon.0) 803 : audit [DBG] from='mgr.? 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-08-27T21:58:29.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:29 smithi006 bash[33918]: audit 2023-08-27T21:58:29.894523+0000 mon.a (mon.0) 804 : audit [DBG] from='mgr.? 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T21:58:29.999 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:29 smithi006 bash[27566]: CherryPy Checker: 2023-08-27T21:58:29.999 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:29 smithi006 bash[27566]: The Application mounted at '' has an empty config. 2023-08-27T21:58:30.000 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:29 smithi006 bash[27566]: [27/Aug/2023:21:58:29] ENGINE Serving on http://:::9283 2023-08-27T21:58:30.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:29 smithi006 bash[27566]: [27/Aug/2023:21:58:29] ENGINE Bus STARTED 2023-08-27T21:58:30.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:29 smithi130 bash[27579]: cluster 2023-08-27T21:58:29.882401+0000 mon.a (mon.0) 800 : cluster [DBG] Standby manager daemon y started 2023-08-27T21:58:30.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:29 smithi130 bash[27579]: audit 2023-08-27T21:58:29.885481+0000 mon.a (mon.0) 801 : audit [DBG] from='mgr.? 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-08-27T21:58:30.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:29 smithi130 bash[27579]: audit 2023-08-27T21:58:29.886301+0000 mon.a (mon.0) 802 : audit [DBG] from='mgr.? 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T21:58:30.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:29 smithi130 bash[27579]: audit 2023-08-27T21:58:29.891169+0000 mon.a (mon.0) 803 : audit [DBG] from='mgr.? 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-08-27T21:58:30.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:29 smithi130 bash[27579]: audit 2023-08-27T21:58:29.894523+0000 mon.a (mon.0) 804 : audit [DBG] from='mgr.? 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T21:58:30.899 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 53 16 78999 78983 93.1312 64.75 0.0120938 0.0107317 2023-08-27T21:58:31.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:30 smithi006 bash[27292]: cluster 2023-08-27T21:58:29.971543+0000 mon.a (mon.0) 805 : cluster [DBG] mgrmap e22: x(active, starting, since 4s), standbys: y 2023-08-27T21:58:31.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:30 smithi006 bash[27292]: audit 2023-08-27T21:58:30.924117+0000 mon.c (mon.2) 52 : audit [DBG] from='client.? 172.21.15.6:0/518024235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:31.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:58:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:58:31.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:30 smithi006 bash[33918]: cluster 2023-08-27T21:58:29.971543+0000 mon.a (mon.0) 805 : cluster [DBG] mgrmap e22: x(active, starting, since 4s), standbys: y 2023-08-27T21:58:31.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:30 smithi006 bash[33918]: audit 2023-08-27T21:58:30.924117+0000 mon.c (mon.2) 52 : audit [DBG] from='client.? 172.21.15.6:0/518024235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:31.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:30 smithi130 bash[27579]: cluster 2023-08-27T21:58:29.971543+0000 mon.a (mon.0) 805 : cluster [DBG] mgrmap e22: x(active, starting, since 4s), standbys: y 2023-08-27T21:58:31.377 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:30 smithi130 bash[27579]: audit 2023-08-27T21:58:30.924117+0000 mon.c (mon.2) 52 : audit [DBG] from='client.? 172.21.15.6:0/518024235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:31.899 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 54 16 80090 80074 92.6692 68.1875 0.00619361 0.0107856 2023-08-27T21:58:32.899 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 55 16 81141 81125 92.1785 65.6875 0.018997 0.0108429 2023-08-27T21:58:33.900 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 56 16 82168 82152 91.6785 64.1875 0.0139492 0.010902 2023-08-27T21:58:34.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:33 smithi006 bash[33918]: audit 2023-08-27T21:58:33.549028+0000 mon.a (mon.0) 806 : audit [DBG] from='client.? 172.21.15.6:0/2960252848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:34.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:33 smithi006 bash[27292]: audit 2023-08-27T21:58:33.549028+0000 mon.a (mon.0) 806 : audit [DBG] from='client.? 172.21.15.6:0/2960252848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:34.372 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:33 smithi130 bash[27579]: audit 2023-08-27T21:58:33.549028+0000 mon.a (mon.0) 806 : audit [DBG] from='client.? 172.21.15.6:0/2960252848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:34.900 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 57 16 83241 83225 91.2466 67.0625 0.0171728 0.0109531 2023-08-27T21:58:35.900 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 58 16 84298 84282 90.8122 66.0625 0.0102498 0.0110057 2023-08-27T21:58:36.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:36 smithi130 bash[27579]: audit 2023-08-27T21:58:36.192038+0000 mon.b (mon.1) 31 : audit [DBG] from='client.? 172.21.15.6:0/1886862851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:36.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:36 smithi006 bash[33918]: audit 2023-08-27T21:58:36.192038+0000 mon.b (mon.1) 31 : audit [DBG] from='client.? 172.21.15.6:0/1886862851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:36.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:36 smithi006 bash[27292]: audit 2023-08-27T21:58:36.192038+0000 mon.b (mon.1) 31 : audit [DBG] from='client.? 172.21.15.6:0/1886862851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:36.900 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 59 16 85327 85311 90.363 64.3125 0.00781119 0.0110603 2023-08-27T21:58:37.900 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:2023-08-27T21:58:37.894759+0000 min lat: 0.00143029 max lat: 0.168897 avg lat: 0.0111127 2023-08-27T21:58:37.900 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-27T21:58:37.901 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 60 16 86366 86350 89.9391 64.9375 0.0141155 0.0111127 2023-08-27T21:58:38.900 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 61 16 87342 87326 89.4646 61 0.0197406 0.0111715 2023-08-27T21:58:39.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:38 smithi006 bash[33918]: audit 2023-08-27T21:58:38.869520+0000 mon.a (mon.0) 807 : audit [DBG] from='client.? 172.21.15.6:0/17707218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:39.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:38 smithi006 bash[27292]: audit 2023-08-27T21:58:38.869520+0000 mon.a (mon.0) 807 : audit [DBG] from='client.? 172.21.15.6:0/17707218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:39.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:38 smithi130 bash[27579]: audit 2023-08-27T21:58:38.869520+0000 mon.a (mon.0) 807 : audit [DBG] from='client.? 172.21.15.6:0/17707218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:39.900 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 62 16 88384 88368 89.0719 65.125 0.0178608 0.0112203 2023-08-27T21:58:40.900 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 63 16 89376 89360 88.6421 62 0.0176483 0.0112759 2023-08-27T21:58:41.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:58:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:58:41.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:41 smithi130 bash[27579]: audit 2023-08-27T21:58:41.541837+0000 mon.a (mon.0) 808 : audit [DBG] from='client.? 172.21.15.6:0/2974356092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:41.900 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 64 16 90338 90322 88.1964 60.125 0.00852829 0.0113321 2023-08-27T21:58:41.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:41 smithi006 bash[33918]: audit 2023-08-27T21:58:41.541837+0000 mon.a (mon.0) 808 : audit [DBG] from='client.? 172.21.15.6:0/2974356092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:41.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:41 smithi006 bash[27292]: audit 2023-08-27T21:58:41.541837+0000 mon.a (mon.0) 808 : audit [DBG] from='client.? 172.21.15.6:0/2974356092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:42.900 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 65 16 91328 91312 87.7914 61.875 0.0229291 0.0113845 2023-08-27T21:58:43.900 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 66 16 92302 92286 87.3835 60.875 0.0212933 0.0114374 2023-08-27T21:58:44.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:44 smithi130 bash[27579]: audit 2023-08-27T21:58:44.174425+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.6:0/586343107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:44.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:44 smithi006 bash[33918]: audit 2023-08-27T21:58:44.174425+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.6:0/586343107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:44.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:44 smithi006 bash[27292]: audit 2023-08-27T21:58:44.174425+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.6:0/586343107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:44.901 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 67 16 93286 93270 86.9971 61.5 0.0130982 0.0114885 2023-08-27T21:58:45.901 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 68 16 94259 94243 86.6119 60.8125 0.021798 0.0115401 2023-08-27T21:58:46.901 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 69 16 95199 95183 86.2081 58.75 0.0161433 0.0115939 2023-08-27T21:58:47.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:46 smithi006 bash[33918]: audit 2023-08-27T21:58:46.813010+0000 mon.a (mon.0) 809 : audit [DBG] from='client.? 172.21.15.6:0/1784746072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:47.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:46 smithi006 bash[27292]: audit 2023-08-27T21:58:46.813010+0000 mon.a (mon.0) 809 : audit [DBG] from='client.? 172.21.15.6:0/1784746072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:47.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:46 smithi130 bash[27579]: audit 2023-08-27T21:58:46.813010+0000 mon.a (mon.0) 809 : audit [DBG] from='client.? 172.21.15.6:0/1784746072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:47.901 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 70 16 96155 96139 85.83 59.75 0.00841744 0.0116447 2023-08-27T21:58:48.901 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 71 16 97095 97079 85.4485 58.75 0.0171633 0.011696 2023-08-27T21:58:49.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:49 smithi130 bash[27579]: audit 2023-08-27T21:58:49.464748+0000 mon.a (mon.0) 810 : audit [DBG] from='client.? 172.21.15.6:0/3558213347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:49.902 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 72 16 98012 97996 85.0577 57.3125 0.0160908 0.0117505 2023-08-27T21:58:49.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:49 smithi006 bash[33918]: audit 2023-08-27T21:58:49.464748+0000 mon.a (mon.0) 810 : audit [DBG] from='client.? 172.21.15.6:0/3558213347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:49.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:49 smithi006 bash[27292]: audit 2023-08-27T21:58:49.464748+0000 mon.a (mon.0) 810 : audit [DBG] from='client.? 172.21.15.6:0/3558213347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:50.901 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 73 16 98944 98928 84.6903 58.25 0.0106967 0.0118016 2023-08-27T21:58:51.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:58:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-27T21:58:51.901 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 74 16 99861 99845 84.3203 57.3125 0.0127303 0.0118541 2023-08-27T21:58:52.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:52 smithi006 bash[33918]: audit 2023-08-27T21:58:52.121459+0000 mon.a (mon.0) 811 : audit [DBG] from='client.? 172.21.15.6:0/268694020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:52.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:52 smithi006 bash[27292]: audit 2023-08-27T21:58:52.121459+0000 mon.a (mon.0) 811 : audit [DBG] from='client.? 172.21.15.6:0/268694020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:52.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:52 smithi130 bash[27579]: audit 2023-08-27T21:58:52.121459+0000 mon.a (mon.0) 811 : audit [DBG] from='client.? 172.21.15.6:0/268694020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:52.901 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 75 16 100765 100749 83.9493 56.5 0.0125666 0.0119058 2023-08-27T21:58:53.902 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 76 16 101674 101658 83.5921 56.8125 0.0100909 0.0119574 2023-08-27T21:58:54.902 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 77 16 102533 102517 83.2037 53.6875 0.015639 0.0120128 2023-08-27T21:58:55.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:54 smithi130 bash[27579]: audit 2023-08-27T21:58:54.774136+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.6:0/58958212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:55.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:54 smithi006 bash[33918]: audit 2023-08-27T21:58:54.774136+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.6:0/58958212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:55.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:54 smithi006 bash[27292]: audit 2023-08-27T21:58:54.774136+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.6:0/58958212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:55.902 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 78 15 103446 103431 82.8693 57.125 0.0271953 0.0120616 2023-08-27T21:58:56.902 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 79 16 104326 104310 82.5156 54.9375 0.0156654 0.0121117 2023-08-27T21:58:57.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:57 smithi130 bash[27579]: audit 2023-08-27T21:58:57.436760+0000 mon.a (mon.0) 812 : audit [DBG] from='client.? 172.21.15.6:0/4244406522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:57.902 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:2023-08-27T21:58:57.896747+0000 min lat: 0.00143029 max lat: 0.168897 avg lat: 0.0121663 2023-08-27T21:58:57.902 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-27T21:58:57.902 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 80 16 105186 105170 82.156 53.75 0.00492684 0.0121663 2023-08-27T21:58:57.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:57 smithi006 bash[33918]: audit 2023-08-27T21:58:57.436760+0000 mon.a (mon.0) 812 : audit [DBG] from='client.? 172.21.15.6:0/4244406522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:57.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:57 smithi006 bash[27292]: audit 2023-08-27T21:58:57.436760+0000 mon.a (mon.0) 812 : audit [DBG] from='client.? 172.21.15.6:0/4244406522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:58.327 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:57 smithi006 bash[27566]: [27/Aug/2023:21:58:57] ENGINE Bus STOPPING 2023-08-27T21:58:58.327 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27566]: [27/Aug/2023:21:58:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-27T21:58:58.327 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27566]: [27/Aug/2023:21:58:58] ENGINE Bus STOPPED 2023-08-27T21:58:58.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: cluster 2023-08-27T21:58:57.950988+0000 mon.a (mon.0) 813 : cluster [INF] Manager daemon x is unresponsive, replacing it with standby daemon y 2023-08-27T21:58:58.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: cluster 2023-08-27T21:58:57.961894+0000 mon.a (mon.0) 814 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-08-27T21:58:58.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: cluster 2023-08-27T21:58:57.962913+0000 mon.a (mon.0) 815 : cluster [DBG] mgrmap e23: y(active, starting, since 0.0119703s) 2023-08-27T21:58:58.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:57.968352+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T21:58:58.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:57.968617+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:58:58.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:57.968854+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:58:58.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:57.970890+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi130.qmrguj"}]: dispatch 2023-08-27T21:58:58.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:57.971068+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi006.bzplue"}]: dispatch 2023-08-27T21:58:58.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:57.971267+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T21:58:58.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:57.973506+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:58:58.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:57.973674+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:58:58.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:57.973817+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:58:58.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:57.973978+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:58:58.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:57.974117+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:58:58.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:57.984874+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:58:58.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:57.985635+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:58:58.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:57.986275+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:58:58.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:57.987050+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T21:58:58.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:57.987741+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T21:58:58.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:57.988831+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T21:58:58.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: cluster 2023-08-27T21:58:58.029289+0000 mon.a (mon.0) 816 : cluster [INF] Manager daemon y is now available 2023-08-27T21:58:58.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:58.247544+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:58:58.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:58.250932+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:58:58.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:58.408037+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T21:58:58.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:58.410304+0000 mon.b (mon.1) 51 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T21:58:58.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:58.478187+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T21:58:58.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:58 smithi130 bash[27579]: audit 2023-08-27T21:58:58.480292+0000 mon.b (mon.1) 52 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T21:58:58.721 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27566]: Warning: Permanently added '172.21.15.6' (ECDSA) to the list of known hosts. 2023-08-27T21:58:58.721 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27566]: Warning: Permanently added '172.21.15.130' (ECDSA) to the list of known hosts. 2023-08-27T21:58:58.721 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27566]: [27/Aug/2023:21:58:58] ENGINE Bus STARTING 2023-08-27T21:58:58.721 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27566]: CherryPy Checker: 2023-08-27T21:58:58.721 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27566]: The Application mounted at '' has an empty config. 2023-08-27T21:58:58.721 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27566]: [27/Aug/2023:21:58:58] ENGINE Serving on http://:::9283 2023-08-27T21:58:58.721 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27566]: [27/Aug/2023:21:58:58] ENGINE Bus STARTED 2023-08-27T21:58:58.721 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27566]: debug 2023-08-27T21:58:58.512+0000 7fa19aec7700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-27T21:58:58.722 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27566]: debug 2023-08-27T21:58:58.512+0000 7fa19aec7700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-27T21:58:58.722 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27566]: debug 2023-08-27T21:58:58.512+0000 7fa19aec7700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-27T21:58:58.722 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27566]: debug 2023-08-27T21:58:58.512+0000 7fa19aec7700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-27T21:58:58.722 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27566]: debug 2023-08-27T21:58:58.512+0000 7fa19aec7700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-27T21:58:58.722 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27566]: debug 2023-08-27T21:58:58.516+0000 7fa19eecf700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-27T21:58:58.722 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27566]: debug 2023-08-27T21:58:58.516+0000 7fa19eecf700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-27T21:58:58.722 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27566]: debug 2023-08-27T21:58:58.516+0000 7fa19eecf700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-27T21:58:58.722 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27566]: debug 2023-08-27T21:58:58.516+0000 7fa19eecf700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-27T21:58:58.722 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27566]: debug 2023-08-27T21:58:58.516+0000 7fa19eecf700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-27T21:58:58.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: cluster 2023-08-27T21:58:57.950988+0000 mon.a (mon.0) 813 : cluster [INF] Manager daemon x is unresponsive, replacing it with standby daemon y 2023-08-27T21:58:58.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: cluster 2023-08-27T21:58:57.961894+0000 mon.a (mon.0) 814 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-08-27T21:58:58.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: cluster 2023-08-27T21:58:57.962913+0000 mon.a (mon.0) 815 : cluster [DBG] mgrmap e23: y(active, starting, since 0.0119703s) 2023-08-27T21:58:58.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:57.968352+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T21:58:58.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:57.968617+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:58:58.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:57.968854+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:58:58.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:57.970890+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi130.qmrguj"}]: dispatch 2023-08-27T21:58:58.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:57.971068+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi006.bzplue"}]: dispatch 2023-08-27T21:58:58.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:57.971267+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T21:58:58.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:57.973506+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:58:58.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:57.973674+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:58:58.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:57.973817+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:58:58.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:57.973978+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:58:58.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:57.974117+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:58:58.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:57.984874+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:58:58.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:57.985635+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:58:58.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:57.986275+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:58:58.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:57.987050+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T21:58:58.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:57.987741+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T21:58:58.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:57.988831+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T21:58:58.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: cluster 2023-08-27T21:58:58.029289+0000 mon.a (mon.0) 816 : cluster [INF] Manager daemon y is now available 2023-08-27T21:58:58.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:58.247544+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:58:58.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:58.250932+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:58:58.725 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:58.408037+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T21:58:58.725 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:58.410304+0000 mon.b (mon.1) 51 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T21:58:58.725 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:58.478187+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T21:58:58.725 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[33918]: audit 2023-08-27T21:58:58.480292+0000 mon.b (mon.1) 52 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T21:58:58.725 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: cluster 2023-08-27T21:58:57.950988+0000 mon.a (mon.0) 813 : cluster [INF] Manager daemon x is unresponsive, replacing it with standby daemon y 2023-08-27T21:58:58.725 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: cluster 2023-08-27T21:58:57.961894+0000 mon.a (mon.0) 814 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-08-27T21:58:58.725 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: cluster 2023-08-27T21:58:57.962913+0000 mon.a (mon.0) 815 : cluster [DBG] mgrmap e23: y(active, starting, since 0.0119703s) 2023-08-27T21:58:58.725 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:57.968352+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T21:58:58.725 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:57.968617+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T21:58:58.726 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:57.968854+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T21:58:58.726 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:57.970890+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi130.qmrguj"}]: dispatch 2023-08-27T21:58:58.726 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:57.971068+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi006.bzplue"}]: dispatch 2023-08-27T21:58:58.726 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:57.971267+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T21:58:58.726 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:57.973506+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T21:58:58.726 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:57.973674+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T21:58:58.726 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:57.973817+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T21:58:58.726 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:57.973978+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T21:58:58.726 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:57.974117+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T21:58:58.726 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:57.984874+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T21:58:58.726 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:57.985635+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T21:58:58.726 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:57.986275+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T21:58:58.726 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:57.987050+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T21:58:58.727 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:57.987741+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T21:58:58.727 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:57.988831+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T21:58:58.727 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: cluster 2023-08-27T21:58:58.029289+0000 mon.a (mon.0) 816 : cluster [INF] Manager daemon y is now available 2023-08-27T21:58:58.727 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:58.247544+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:58:58.727 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:58.250932+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:58:58.727 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:58.408037+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T21:58:58.727 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:58.410304+0000 mon.b (mon.1) 51 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T21:58:58.727 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:58.478187+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T21:58:58.727 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:58 smithi006 bash[27292]: audit 2023-08-27T21:58:58.480292+0000 mon.b (mon.1) 52 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T21:58:58.902 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 81 16 105967 105951 81.7443 48.8125 0.0212728 0.0122261 2023-08-27T21:58:59.902 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 82 16 106812 106796 81.3914 52.8125 0.0187845 0.0122799 2023-08-27T21:59:00.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:59 smithi130 bash[27579]: cluster 2023-08-27T21:58:58.978040+0000 mon.a (mon.0) 819 : cluster [DBG] mgrmap e24: y(active, since 1.02709s) 2023-08-27T21:59:00.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:58:59 smithi130 bash[27579]: cluster 2023-08-27T21:58:59.014982+0000 mgr.y (mgr.14901) 1 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail 2023-08-27T21:59:00.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:59 smithi006 bash[33918]: cluster 2023-08-27T21:58:58.978040+0000 mon.a (mon.0) 819 : cluster [DBG] mgrmap e24: y(active, since 1.02709s) 2023-08-27T21:59:00.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:58:59 smithi006 bash[33918]: cluster 2023-08-27T21:58:59.014982+0000 mgr.y (mgr.14901) 1 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail 2023-08-27T21:59:00.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:59 smithi006 bash[27292]: cluster 2023-08-27T21:58:58.978040+0000 mon.a (mon.0) 819 : cluster [DBG] mgrmap e24: y(active, since 1.02709s) 2023-08-27T21:59:00.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:58:59 smithi006 bash[27292]: cluster 2023-08-27T21:58:59.014982+0000 mgr.y (mgr.14901) 1 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail 2023-08-27T21:59:00.902 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 83 16 107662 107646 81.0508 53.125 0.031833 0.0123323 2023-08-27T21:59:01.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:00 smithi006 bash[27292]: cluster 2023-08-27T21:58:59.974343+0000 mgr.y (mgr.14901) 2 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail 2023-08-27T21:59:01.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:00 smithi006 bash[27292]: audit 2023-08-27T21:59:00.121283+0000 mon.a (mon.0) 820 : audit [DBG] from='client.? 172.21.15.6:0/1985024746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:01.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:59:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:59:00] "GET /metrics HTTP/1.1" 200 32761 "" "Prometheus/2.33.4" 2023-08-27T21:59:01.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:00 smithi006 bash[33918]: cluster 2023-08-27T21:58:59.974343+0000 mgr.y (mgr.14901) 2 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail 2023-08-27T21:59:01.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:00 smithi006 bash[33918]: audit 2023-08-27T21:59:00.121283+0000 mon.a (mon.0) 820 : audit [DBG] from='client.? 172.21.15.6:0/1985024746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:01.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:00 smithi130 bash[27579]: cluster 2023-08-27T21:58:59.974343+0000 mgr.y (mgr.14901) 2 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail 2023-08-27T21:59:01.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:00 smithi130 bash[27579]: audit 2023-08-27T21:59:00.121283+0000 mon.a (mon.0) 820 : audit [DBG] from='client.? 172.21.15.6:0/1985024746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:01.902 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 84 16 108509 108493 80.716 52.9375 0.010728 0.0123829 2023-08-27T21:59:02.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:02 smithi130 bash[27579]: cluster 2023-08-27T21:59:01.003735+0000 mon.a (mon.0) 821 : cluster [DBG] mgrmap e25: y(active, since 3s) 2023-08-27T21:59:02.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:02 smithi006 bash[27292]: cluster 2023-08-27T21:59:01.003735+0000 mon.a (mon.0) 821 : cluster [DBG] mgrmap e25: y(active, since 3s) 2023-08-27T21:59:02.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:02 smithi006 bash[33918]: cluster 2023-08-27T21:59:01.003735+0000 mon.a (mon.0) 821 : cluster [DBG] mgrmap e25: y(active, since 3s) 2023-08-27T21:59:02.902 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 85 16 109334 109318 80.373 51.5625 0.0139559 0.0124359 2023-08-27T21:59:03.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:03 smithi130 bash[27579]: cluster 2023-08-27T21:59:01.974845+0000 mgr.y (mgr.14901) 3 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail 2023-08-27T21:59:03.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:03 smithi130 bash[27579]: audit 2023-08-27T21:59:02.784968+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.6:0/1477031245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:03.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:03 smithi006 bash[27292]: cluster 2023-08-27T21:59:01.974845+0000 mgr.y (mgr.14901) 3 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail 2023-08-27T21:59:03.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:03 smithi006 bash[27292]: audit 2023-08-27T21:59:02.784968+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.6:0/1477031245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:03.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:03 smithi006 bash[33918]: cluster 2023-08-27T21:59:01.974845+0000 mgr.y (mgr.14901) 3 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail 2023-08-27T21:59:03.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:03 smithi006 bash[33918]: audit 2023-08-27T21:59:02.784968+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.6:0/1477031245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:03.903 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 86 16 110160 110144 80.038 51.625 0.0218902 0.0124874 2023-08-27T21:59:04.904 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 87 16 110886 110870 79.6396 45.375 0.0116881 0.0125505 2023-08-27T21:59:05.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:05 smithi130 bash[27579]: cluster 2023-08-27T21:59:03.975314+0000 mgr.y (mgr.14901) 4 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail 2023-08-27T21:59:05.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:05 smithi130 bash[27579]: audit 2023-08-27T21:59:04.983985+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:05.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:05 smithi006 bash[27292]: cluster 2023-08-27T21:59:03.975314+0000 mgr.y (mgr.14901) 4 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail 2023-08-27T21:59:05.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:05 smithi006 bash[27292]: audit 2023-08-27T21:59:04.983985+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:05.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:05 smithi006 bash[33918]: cluster 2023-08-27T21:59:03.975314+0000 mgr.y (mgr.14901) 4 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail 2023-08-27T21:59:05.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:05 smithi006 bash[33918]: audit 2023-08-27T21:59:04.983985+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:05.903 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 88 16 111615 111599 79.2523 45.5625 0.01563 0.0126117 2023-08-27T21:59:06.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.363142+0000 mon.b (mon.1) 53 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.363156+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:06.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.365960+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.374081+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.376510+0000 mon.b (mon.1) 54 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.376521+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.379001+0000 mon.b (mon.1) 55 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.382089+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: cephadm 2023-08-27T21:59:05.383618+0000 mgr.y (mgr.14901) 5 : cephadm [INF] Adjusting osd_memory_target on smithi006 to 2525M 2023-08-27T21:59:06.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.383802+0000 mon.b (mon.1) 56 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.400815+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:06.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.464961+0000 mon.a (mon.0) 829 : audit [DBG] from='client.? 172.21.15.6:0/2825765035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:06.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.710355+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.713009+0000 mon.b (mon.1) 57 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.716544+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.719176+0000 mon.b (mon.1) 58 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.722670+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.725086+0000 mon.b (mon.1) 59 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.727711+0000 mon.b (mon.1) 60 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.728150+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.738521+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:06.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.746902+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:59:06.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.753066+0000 mon.b (mon.1) 62 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:59:06.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.806782+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:06.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.814844+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:06.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.828072+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:06.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:06 smithi130 bash[27579]: audit 2023-08-27T21:59:05.842879+0000 mon.b (mon.1) 63 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:59:06.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.363142+0000 mon.b (mon.1) 53 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.363156+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:06.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.365960+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.374081+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.376510+0000 mon.b (mon.1) 54 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.376521+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.379001+0000 mon.b (mon.1) 55 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.382089+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: cephadm 2023-08-27T21:59:05.383618+0000 mgr.y (mgr.14901) 5 : cephadm [INF] Adjusting osd_memory_target on smithi006 to 2525M 2023-08-27T21:59:06.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.383802+0000 mon.b (mon.1) 56 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.400815+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:06.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.464961+0000 mon.a (mon.0) 829 : audit [DBG] from='client.? 172.21.15.6:0/2825765035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:06.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.710355+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.713009+0000 mon.b (mon.1) 57 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.716544+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.719176+0000 mon.b (mon.1) 58 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.722670+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.725086+0000 mon.b (mon.1) 59 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.727711+0000 mon.b (mon.1) 60 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.728150+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.738521+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:06.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.746902+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:59:06.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.753066+0000 mon.b (mon.1) 62 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:59:06.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.806782+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:06.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.814844+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:06.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.828072+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:06.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[33918]: audit 2023-08-27T21:59:05.842879+0000 mon.b (mon.1) 63 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:59:06.724 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.363142+0000 mon.b (mon.1) 53 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.724 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.363156+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:06.724 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.365960+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.724 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.374081+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.724 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.376510+0000 mon.b (mon.1) 54 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.724 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.376521+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.724 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.379001+0000 mon.b (mon.1) 55 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.724 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.382089+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.724 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: cephadm 2023-08-27T21:59:05.383618+0000 mgr.y (mgr.14901) 5 : cephadm [INF] Adjusting osd_memory_target on smithi006 to 2525M 2023-08-27T21:59:06.725 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.383802+0000 mon.b (mon.1) 56 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.725 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.400815+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:06.725 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.464961+0000 mon.a (mon.0) 829 : audit [DBG] from='client.? 172.21.15.6:0/2825765035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:06.725 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.710355+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.725 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.713009+0000 mon.b (mon.1) 57 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.725 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.716544+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.725 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.719176+0000 mon.b (mon.1) 58 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.725 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.722670+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.725 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.725086+0000 mon.b (mon.1) 59 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.725 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.727711+0000 mon.b (mon.1) 60 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.725 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.728150+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-27T21:59:06.725 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.738521+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:06.725 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.746902+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:59:06.726 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.753066+0000 mon.b (mon.1) 62 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:59:06.726 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.806782+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:06.726 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.814844+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:06.726 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.828072+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:06.726 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:06 smithi006 bash[27292]: audit 2023-08-27T21:59:05.842879+0000 mon.b (mon.1) 63 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:59:06.903 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 89 16 112462 112446 78.9566 52.9375 0.022614 0.0126588 2023-08-27T21:59:07.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:07 smithi006 bash[33918]: cephadm 2023-08-27T21:59:05.729716+0000 mgr.y (mgr.14901) 6 : cephadm [INF] Adjusting osd_memory_target on smithi130 to 2524M 2023-08-27T21:59:07.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:07 smithi006 bash[33918]: cephadm 2023-08-27T21:59:05.752117+0000 mgr.y (mgr.14901) 7 : cephadm [INF] Updating smithi006:/etc/ceph/ceph.conf 2023-08-27T21:59:07.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:07 smithi006 bash[33918]: cephadm 2023-08-27T21:59:05.752630+0000 mgr.y (mgr.14901) 8 : cephadm [INF] Updating smithi130:/etc/ceph/ceph.conf 2023-08-27T21:59:07.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:07 smithi006 bash[33918]: cephadm 2023-08-27T21:59:05.770738+0000 mgr.y (mgr.14901) 9 : cephadm [INF] Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2023-08-27T21:59:07.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:07 smithi006 bash[33918]: cephadm 2023-08-27T21:59:05.778320+0000 mgr.y (mgr.14901) 10 : cephadm [INF] Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2023-08-27T21:59:07.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:07 smithi006 bash[33918]: cephadm 2023-08-27T21:59:05.847179+0000 mgr.y (mgr.14901) 11 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-08-27T21:59:07.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:07 smithi006 bash[33918]: cephadm 2023-08-27T21:59:05.850399+0000 mgr.y (mgr.14901) 12 : cephadm [INF] Failed to find standby mgr for failover. Retrying in 2 seconds 2023-08-27T21:59:07.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:07 smithi006 bash[33918]: cluster 2023-08-27T21:59:05.975927+0000 mgr.y (mgr.14901) 13 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 602 op/s 2023-08-27T21:59:07.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:07 smithi006 bash[27292]: cephadm 2023-08-27T21:59:05.729716+0000 mgr.y (mgr.14901) 6 : cephadm [INF] Adjusting osd_memory_target on smithi130 to 2524M 2023-08-27T21:59:07.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:07 smithi006 bash[27292]: cephadm 2023-08-27T21:59:05.752117+0000 mgr.y (mgr.14901) 7 : cephadm [INF] Updating smithi006:/etc/ceph/ceph.conf 2023-08-27T21:59:07.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:07 smithi006 bash[27292]: cephadm 2023-08-27T21:59:05.752630+0000 mgr.y (mgr.14901) 8 : cephadm [INF] Updating smithi130:/etc/ceph/ceph.conf 2023-08-27T21:59:07.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:07 smithi006 bash[27292]: cephadm 2023-08-27T21:59:05.770738+0000 mgr.y (mgr.14901) 9 : cephadm [INF] Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2023-08-27T21:59:07.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:07 smithi006 bash[27292]: cephadm 2023-08-27T21:59:05.778320+0000 mgr.y (mgr.14901) 10 : cephadm [INF] Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2023-08-27T21:59:07.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:07 smithi006 bash[27292]: cephadm 2023-08-27T21:59:05.847179+0000 mgr.y (mgr.14901) 11 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-08-27T21:59:07.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:07 smithi006 bash[27292]: cephadm 2023-08-27T21:59:05.850399+0000 mgr.y (mgr.14901) 12 : cephadm [INF] Failed to find standby mgr for failover. Retrying in 2 seconds 2023-08-27T21:59:07.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:07 smithi006 bash[27292]: cluster 2023-08-27T21:59:05.975927+0000 mgr.y (mgr.14901) 13 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 602 op/s 2023-08-27T21:59:07.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:07 smithi130 bash[27579]: cephadm 2023-08-27T21:59:05.729716+0000 mgr.y (mgr.14901) 6 : cephadm [INF] Adjusting osd_memory_target on smithi130 to 2524M 2023-08-27T21:59:07.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:07 smithi130 bash[27579]: cephadm 2023-08-27T21:59:05.752117+0000 mgr.y (mgr.14901) 7 : cephadm [INF] Updating smithi006:/etc/ceph/ceph.conf 2023-08-27T21:59:07.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:07 smithi130 bash[27579]: cephadm 2023-08-27T21:59:05.752630+0000 mgr.y (mgr.14901) 8 : cephadm [INF] Updating smithi130:/etc/ceph/ceph.conf 2023-08-27T21:59:07.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:07 smithi130 bash[27579]: cephadm 2023-08-27T21:59:05.770738+0000 mgr.y (mgr.14901) 9 : cephadm [INF] Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2023-08-27T21:59:07.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:07 smithi130 bash[27579]: cephadm 2023-08-27T21:59:05.778320+0000 mgr.y (mgr.14901) 10 : cephadm [INF] Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2023-08-27T21:59:07.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:07 smithi130 bash[27579]: cephadm 2023-08-27T21:59:05.847179+0000 mgr.y (mgr.14901) 11 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-08-27T21:59:07.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:07 smithi130 bash[27579]: cephadm 2023-08-27T21:59:05.850399+0000 mgr.y (mgr.14901) 12 : cephadm [INF] Failed to find standby mgr for failover. Retrying in 2 seconds 2023-08-27T21:59:07.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:07 smithi130 bash[27579]: cluster 2023-08-27T21:59:05.975927+0000 mgr.y (mgr.14901) 13 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 602 op/s 2023-08-27T21:59:07.904 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Total time run: 90.0077 2023-08-27T21:59:07.904 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Total writes made: 113276 2023-08-27T21:59:07.904 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Write size: 65536 2023-08-27T21:59:07.904 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Object size: 65536 2023-08-27T21:59:07.904 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Bandwidth (MB/sec): 78.6571 2023-08-27T21:59:07.904 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Stddev Bandwidth: 28.1671 2023-08-27T21:59:07.904 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Max bandwidth (MB/sec): 167 2023-08-27T21:59:07.904 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Min bandwidth (MB/sec): 45.375 2023-08-27T21:59:07.904 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Average IOPS: 1258 2023-08-27T21:59:07.905 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Stddev IOPS: 450.674 2023-08-27T21:59:07.905 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Max IOPS: 2672 2023-08-27T21:59:07.905 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Min IOPS: 726 2023-08-27T21:59:07.905 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Average Latency(s): 0.0127086 2023-08-27T21:59:07.905 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Stddev Latency(s): 0.00783826 2023-08-27T21:59:07.905 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Max latency(s): 0.168897 2023-08-27T21:59:07.905 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Min latency(s): 0.00143029 2023-08-27T21:59:07.911 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Cleaning up (deleting benchmark objects) 2023-08-27T21:59:09.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:08 smithi130 bash[27579]: cephadm 2023-08-27T21:59:07.854664+0000 mgr.y (mgr.14901) 14 : cephadm [INF] Failed to find standby mgr for failover. Retrying in 8 seconds 2023-08-27T21:59:09.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:08 smithi130 bash[27579]: cluster 2023-08-27T21:59:07.976352+0000 mgr.y (mgr.14901) 15 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 29 MiB/s wr, 468 op/s 2023-08-27T21:59:09.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:08 smithi130 bash[27579]: audit 2023-08-27T21:59:08.138491+0000 mon.b (mon.1) 64 : audit [DBG] from='client.? 172.21.15.6:0/2249247443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:09.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:08 smithi006 bash[33918]: cephadm 2023-08-27T21:59:07.854664+0000 mgr.y (mgr.14901) 14 : cephadm [INF] Failed to find standby mgr for failover. Retrying in 8 seconds 2023-08-27T21:59:09.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:08 smithi006 bash[33918]: cluster 2023-08-27T21:59:07.976352+0000 mgr.y (mgr.14901) 15 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 29 MiB/s wr, 468 op/s 2023-08-27T21:59:09.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:08 smithi006 bash[33918]: audit 2023-08-27T21:59:08.138491+0000 mon.b (mon.1) 64 : audit [DBG] from='client.? 172.21.15.6:0/2249247443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:09.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:08 smithi006 bash[27292]: cephadm 2023-08-27T21:59:07.854664+0000 mgr.y (mgr.14901) 14 : cephadm [INF] Failed to find standby mgr for failover. Retrying in 8 seconds 2023-08-27T21:59:09.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:08 smithi006 bash[27292]: cluster 2023-08-27T21:59:07.976352+0000 mgr.y (mgr.14901) 15 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 29 MiB/s wr, 468 op/s 2023-08-27T21:59:09.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:08 smithi006 bash[27292]: audit 2023-08-27T21:59:08.138491+0000 mon.b (mon.1) 64 : audit [DBG] from='client.? 172.21.15.6:0/2249247443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:11.047 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:59:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:59:10] "GET /metrics HTTP/1.1" 200 32761 "" "Prometheus/2.33.4" 2023-08-27T21:59:11.047 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:11 smithi006 bash[27292]: cluster 2023-08-27T21:59:09.977116+0000 mgr.y (mgr.14901) 16 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 93 B/s rd, 41 MiB/s wr, 750 op/s 2023-08-27T21:59:11.047 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:11 smithi006 bash[27292]: audit 2023-08-27T21:59:10.758842+0000 mon.a (mon.0) 838 : audit [DBG] from='client.? 172.21.15.6:0/3596779817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:11.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:11 smithi130 bash[27579]: cluster 2023-08-27T21:59:09.977116+0000 mgr.y (mgr.14901) 16 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 93 B/s rd, 41 MiB/s wr, 750 op/s 2023-08-27T21:59:11.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:11 smithi130 bash[27579]: audit 2023-08-27T21:59:10.758842+0000 mon.a (mon.0) 838 : audit [DBG] from='client.? 172.21.15.6:0/3596779817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:11.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:11 smithi006 bash[33918]: cluster 2023-08-27T21:59:09.977116+0000 mgr.y (mgr.14901) 16 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 93 B/s rd, 41 MiB/s wr, 750 op/s 2023-08-27T21:59:11.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:11 smithi006 bash[33918]: audit 2023-08-27T21:59:10.758842+0000 mon.a (mon.0) 838 : audit [DBG] from='client.? 172.21.15.6:0/3596779817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:13.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:13 smithi130 bash[27579]: cluster 2023-08-27T21:59:11.977574+0000 mgr.y (mgr.14901) 17 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 685 op/s 2023-08-27T21:59:13.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:13 smithi006 bash[27292]: cluster 2023-08-27T21:59:11.977574+0000 mgr.y (mgr.14901) 17 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 685 op/s 2023-08-27T21:59:13.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:13 smithi006 bash[33918]: cluster 2023-08-27T21:59:11.977574+0000 mgr.y (mgr.14901) 17 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 685 op/s 2023-08-27T21:59:14.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:14 smithi130 bash[27579]: audit 2023-08-27T21:59:13.412261+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.6:0/2172593626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:14.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:14 smithi006 bash[33918]: audit 2023-08-27T21:59:13.412261+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.6:0/2172593626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:14.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:14 smithi006 bash[27292]: audit 2023-08-27T21:59:13.412261+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.6:0/2172593626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:15.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:15 smithi130 bash[27579]: cluster 2023-08-27T21:59:13.977962+0000 mgr.y (mgr.14901) 18 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 685 op/s 2023-08-27T21:59:15.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:15 smithi006 bash[27292]: cluster 2023-08-27T21:59:13.977962+0000 mgr.y (mgr.14901) 18 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 685 op/s 2023-08-27T21:59:15.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:15 smithi006 bash[33918]: cluster 2023-08-27T21:59:13.977962+0000 mgr.y (mgr.14901) 18 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 685 op/s 2023-08-27T21:59:17.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:17 smithi130 bash[27579]: cephadm 2023-08-27T21:59:15.863001+0000 mgr.y (mgr.14901) 19 : cephadm [INF] Failed to find standby mgr for failover. Retrying in 15 seconds 2023-08-27T21:59:17.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:17 smithi130 bash[27579]: cluster 2023-08-27T21:59:15.978657+0000 mgr.y (mgr.14901) 20 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 1.04k op/s 2023-08-27T21:59:17.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:17 smithi130 bash[27579]: audit 2023-08-27T21:59:16.085014+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.6:0/116156874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:17.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:17 smithi006 bash[27292]: cephadm 2023-08-27T21:59:15.863001+0000 mgr.y (mgr.14901) 19 : cephadm [INF] Failed to find standby mgr for failover. Retrying in 15 seconds 2023-08-27T21:59:17.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:17 smithi006 bash[27292]: cluster 2023-08-27T21:59:15.978657+0000 mgr.y (mgr.14901) 20 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 1.04k op/s 2023-08-27T21:59:17.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:17 smithi006 bash[27292]: audit 2023-08-27T21:59:16.085014+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.6:0/116156874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:17.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:17 smithi006 bash[33918]: cephadm 2023-08-27T21:59:15.863001+0000 mgr.y (mgr.14901) 19 : cephadm [INF] Failed to find standby mgr for failover. Retrying in 15 seconds 2023-08-27T21:59:17.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:17 smithi006 bash[33918]: cluster 2023-08-27T21:59:15.978657+0000 mgr.y (mgr.14901) 20 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 1.04k op/s 2023-08-27T21:59:17.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:17 smithi006 bash[33918]: audit 2023-08-27T21:59:16.085014+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.6:0/116156874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:19.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:18 smithi006 bash[33918]: cluster 2023-08-27T21:59:17.979216+0000 mgr.y (mgr.14901) 21 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 693 op/s 2023-08-27T21:59:19.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:18 smithi006 bash[33918]: audit 2023-08-27T21:59:18.733611+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.6:0/411970969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:19.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:18 smithi006 bash[27292]: cluster 2023-08-27T21:59:17.979216+0000 mgr.y (mgr.14901) 21 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 693 op/s 2023-08-27T21:59:19.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:18 smithi006 bash[27292]: audit 2023-08-27T21:59:18.733611+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.6:0/411970969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:19.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:18 smithi130 bash[27579]: cluster 2023-08-27T21:59:17.979216+0000 mgr.y (mgr.14901) 21 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 693 op/s 2023-08-27T21:59:19.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:18 smithi130 bash[27579]: audit 2023-08-27T21:59:18.733611+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.6:0/411970969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:21.045 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:59:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:59:20] "GET /metrics HTTP/1.1" 200 194698 "" "Prometheus/2.33.4" 2023-08-27T21:59:21.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:21 smithi130 bash[27579]: cluster 2023-08-27T21:59:19.979912+0000 mgr.y (mgr.14901) 22 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 1.05k op/s 2023-08-27T21:59:21.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:21 smithi006 bash[27292]: cluster 2023-08-27T21:59:19.979912+0000 mgr.y (mgr.14901) 22 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 1.05k op/s 2023-08-27T21:59:21.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:21 smithi006 bash[33918]: cluster 2023-08-27T21:59:19.979912+0000 mgr.y (mgr.14901) 22 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 1.05k op/s 2023-08-27T21:59:22.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:22 smithi130 bash[27579]: audit 2023-08-27T21:59:21.368461+0000 mon.a (mon.0) 839 : audit [DBG] from='client.? 172.21.15.6:0/90346344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:22.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:22 smithi006 bash[33918]: audit 2023-08-27T21:59:21.368461+0000 mon.a (mon.0) 839 : audit [DBG] from='client.? 172.21.15.6:0/90346344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:22.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:22 smithi006 bash[27292]: audit 2023-08-27T21:59:21.368461+0000 mon.a (mon.0) 839 : audit [DBG] from='client.? 172.21.15.6:0/90346344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:23.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:23 smithi130 bash[27579]: cluster 2023-08-27T21:59:21.980447+0000 mgr.y (mgr.14901) 23 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-08-27T21:59:23.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:23 smithi006 bash[33918]: cluster 2023-08-27T21:59:21.980447+0000 mgr.y (mgr.14901) 23 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-08-27T21:59:23.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:23 smithi006 bash[27292]: cluster 2023-08-27T21:59:21.980447+0000 mgr.y (mgr.14901) 23 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-08-27T21:59:24.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:24 smithi130 bash[27579]: audit 2023-08-27T21:59:24.041643+0000 mon.b (mon.1) 65 : audit [DBG] from='client.? 172.21.15.6:0/454051141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:24.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:24 smithi006 bash[27292]: audit 2023-08-27T21:59:24.041643+0000 mon.b (mon.1) 65 : audit [DBG] from='client.? 172.21.15.6:0/454051141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:24.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:24 smithi006 bash[33918]: audit 2023-08-27T21:59:24.041643+0000 mon.b (mon.1) 65 : audit [DBG] from='client.? 172.21.15.6:0/454051141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:25.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:25 smithi130 bash[27579]: cluster 2023-08-27T21:59:23.981072+0000 mgr.y (mgr.14901) 24 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-08-27T21:59:25.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:25 smithi006 bash[27292]: cluster 2023-08-27T21:59:23.981072+0000 mgr.y (mgr.14901) 24 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-08-27T21:59:25.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:25 smithi006 bash[33918]: cluster 2023-08-27T21:59:23.981072+0000 mgr.y (mgr.14901) 24 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-08-27T21:59:27.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:27 smithi130 bash[27579]: cluster 2023-08-27T21:59:25.981703+0000 mgr.y (mgr.14901) 25 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2023-08-27T21:59:27.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:27 smithi130 bash[27579]: audit 2023-08-27T21:59:26.679139+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.6:0/1529332716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:27.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:27 smithi006 bash[27292]: cluster 2023-08-27T21:59:25.981703+0000 mgr.y (mgr.14901) 25 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2023-08-27T21:59:27.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:27 smithi006 bash[27292]: audit 2023-08-27T21:59:26.679139+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.6:0/1529332716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:27.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:27 smithi006 bash[33918]: cluster 2023-08-27T21:59:25.981703+0000 mgr.y (mgr.14901) 25 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2023-08-27T21:59:27.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:27 smithi006 bash[33918]: audit 2023-08-27T21:59:26.679139+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.6:0/1529332716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:29.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:28 smithi130 bash[27579]: cluster 2023-08-27T21:59:27.982178+0000 mgr.y (mgr.14901) 26 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-08-27T21:59:29.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:28 smithi006 bash[27292]: cluster 2023-08-27T21:59:27.982178+0000 mgr.y (mgr.14901) 26 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-08-27T21:59:29.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:28 smithi006 bash[33918]: cluster 2023-08-27T21:59:27.982178+0000 mgr.y (mgr.14901) 26 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-08-27T21:59:30.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:29 smithi130 bash[27579]: audit 2023-08-27T21:59:29.306496+0000 mon.a (mon.0) 840 : audit [DBG] from='client.? 172.21.15.6:0/3586186978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:30.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:29 smithi006 bash[27292]: audit 2023-08-27T21:59:29.306496+0000 mon.a (mon.0) 840 : audit [DBG] from='client.? 172.21.15.6:0/3586186978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:30.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:29 smithi006 bash[33918]: audit 2023-08-27T21:59:29.306496+0000 mon.a (mon.0) 840 : audit [DBG] from='client.? 172.21.15.6:0/3586186978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:31.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:59:30] "GET /metrics HTTP/1.1" 200 194669 "" "Prometheus/2.33.4" 2023-08-27T21:59:31.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[27566]: debug 2023-08-27T21:59:30.875+0000 7fa1be48e700 -1 log_channel(cephadm) log [ERR] : Upgrade: Paused due to UPGRADE_NO_STANDBY_MGR: Upgrade: Need standby mgr daemon 2023-08-27T21:59:31.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[27292]: cluster 2023-08-27T21:59:29.982791+0000 mgr.y (mgr.14901) 27 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2023-08-27T21:59:31.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[27292]: audit 2023-08-27T21:59:30.889865+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:31.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[27292]: audit 2023-08-27T21:59:30.896603+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-08-27T21:59:31.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[27292]: audit 2023-08-27T21:59:30.897891+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:59:31.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[27292]: audit 2023-08-27T21:59:30.899861+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:59:31.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[27292]: audit 2023-08-27T21:59:30.900919+0000 mon.b (mon.1) 69 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:59:31.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[27292]: audit 2023-08-27T21:59:30.908317+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:31.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[27292]: audit 2023-08-27T21:59:30.927867+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:59:31.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[27292]: audit 2023-08-27T21:59:30.930111+0000 mon.b (mon.1) 71 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:59:31.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[27292]: audit 2023-08-27T21:59:30.931192+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:59:31.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[27292]: audit 2023-08-27T21:59:30.934520+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:31.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[33918]: cluster 2023-08-27T21:59:29.982791+0000 mgr.y (mgr.14901) 27 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2023-08-27T21:59:31.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[33918]: audit 2023-08-27T21:59:30.889865+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:31.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[33918]: audit 2023-08-27T21:59:30.896603+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-08-27T21:59:31.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[33918]: audit 2023-08-27T21:59:30.897891+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:59:31.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[33918]: audit 2023-08-27T21:59:30.899861+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:59:31.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[33918]: audit 2023-08-27T21:59:30.900919+0000 mon.b (mon.1) 69 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:59:31.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[33918]: audit 2023-08-27T21:59:30.908317+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:31.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[33918]: audit 2023-08-27T21:59:30.927867+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:59:31.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[33918]: audit 2023-08-27T21:59:30.930111+0000 mon.b (mon.1) 71 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:59:31.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[33918]: audit 2023-08-27T21:59:30.931192+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:59:31.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:30 smithi006 bash[33918]: audit 2023-08-27T21:59:30.934520+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:31.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:30 smithi130 bash[27579]: cluster 2023-08-27T21:59:29.982791+0000 mgr.y (mgr.14901) 27 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2023-08-27T21:59:31.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:30 smithi130 bash[27579]: audit 2023-08-27T21:59:30.889865+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:31.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:30 smithi130 bash[27579]: audit 2023-08-27T21:59:30.896603+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-08-27T21:59:31.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:30 smithi130 bash[27579]: audit 2023-08-27T21:59:30.897891+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:59:31.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:30 smithi130 bash[27579]: audit 2023-08-27T21:59:30.899861+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:59:31.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:30 smithi130 bash[27579]: audit 2023-08-27T21:59:30.900919+0000 mon.b (mon.1) 69 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:59:31.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:30 smithi130 bash[27579]: audit 2023-08-27T21:59:30.908317+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:31.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:30 smithi130 bash[27579]: audit 2023-08-27T21:59:30.927867+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:59:31.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:30 smithi130 bash[27579]: audit 2023-08-27T21:59:30.930111+0000 mon.b (mon.1) 71 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:59:31.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:30 smithi130 bash[27579]: audit 2023-08-27T21:59:30.931192+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:59:31.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:30 smithi130 bash[27579]: audit 2023-08-27T21:59:30.934520+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:31.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:59:31 smithi006 bash[56298]: level=warn ts=2023-08-27T21:59:31.624Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T21:59:32.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:31 smithi130 bash[27579]: cephadm 2023-08-27T21:59:30.879436+0000 mgr.y (mgr.14901) 28 : cephadm [ERR] Upgrade: Paused due to UPGRADE_NO_STANDBY_MGR: Upgrade: Need standby mgr daemon 2023-08-27T21:59:32.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:31 smithi130 bash[27579]: cephadm 2023-08-27T21:59:30.892140+0000 mgr.y (mgr.14901) 29 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-08-27T21:59:32.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:31 smithi130 bash[27579]: cluster 2023-08-27T21:59:30.893215+0000 mgr.y (mgr.14901) 30 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 744 op/s 2023-08-27T21:59:32.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:31 smithi130 bash[27579]: cluster 2023-08-27T21:59:30.975442+0000 mon.a (mon.0) 844 : cluster [WRN] Health check failed: Upgrade: Need standby mgr daemon (UPGRADE_NO_STANDBY_MGR) 2023-08-27T21:59:32.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:31 smithi130 bash[27579]: audit 2023-08-27T21:59:31.969152+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.6:0/1405050488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:32.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:31 smithi006 bash[33918]: cephadm 2023-08-27T21:59:30.879436+0000 mgr.y (mgr.14901) 28 : cephadm [ERR] Upgrade: Paused due to UPGRADE_NO_STANDBY_MGR: Upgrade: Need standby mgr daemon 2023-08-27T21:59:32.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:31 smithi006 bash[33918]: cephadm 2023-08-27T21:59:30.892140+0000 mgr.y (mgr.14901) 29 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-08-27T21:59:32.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:31 smithi006 bash[33918]: cluster 2023-08-27T21:59:30.893215+0000 mgr.y (mgr.14901) 30 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 744 op/s 2023-08-27T21:59:32.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:31 smithi006 bash[33918]: cluster 2023-08-27T21:59:30.975442+0000 mon.a (mon.0) 844 : cluster [WRN] Health check failed: Upgrade: Need standby mgr daemon (UPGRADE_NO_STANDBY_MGR) 2023-08-27T21:59:32.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:31 smithi006 bash[33918]: audit 2023-08-27T21:59:31.969152+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.6:0/1405050488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:32.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:32 smithi006 bash[27292]: cephadm 2023-08-27T21:59:30.879436+0000 mgr.y (mgr.14901) 28 : cephadm [ERR] Upgrade: Paused due to UPGRADE_NO_STANDBY_MGR: Upgrade: Need standby mgr daemon 2023-08-27T21:59:32.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:32 smithi006 bash[27292]: cephadm 2023-08-27T21:59:30.892140+0000 mgr.y (mgr.14901) 29 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-08-27T21:59:32.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:32 smithi006 bash[27292]: cluster 2023-08-27T21:59:30.893215+0000 mgr.y (mgr.14901) 30 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 744 op/s 2023-08-27T21:59:32.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:32 smithi006 bash[27292]: cluster 2023-08-27T21:59:30.975442+0000 mon.a (mon.0) 844 : cluster [WRN] Health check failed: Upgrade: Need standby mgr daemon (UPGRADE_NO_STANDBY_MGR) 2023-08-27T21:59:32.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:32 smithi006 bash[27292]: audit 2023-08-27T21:59:31.969152+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.6:0/1405050488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:59:33 smithi006 bash[56298]: level=warn ts=2023-08-27T21:59:33.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T21:59:34.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:34 smithi130 bash[27579]: cluster 2023-08-27T21:59:32.893871+0000 mgr.y (mgr.14901) 31 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 744 op/s 2023-08-27T21:59:34.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:33 smithi006 bash[27292]: cluster 2023-08-27T21:59:32.893871+0000 mgr.y (mgr.14901) 31 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 744 op/s 2023-08-27T21:59:34.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:33 smithi006 bash[33918]: cluster 2023-08-27T21:59:32.893871+0000 mgr.y (mgr.14901) 31 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 744 op/s 2023-08-27T21:59:35.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:35 smithi130 bash[27579]: audit 2023-08-27T21:59:34.611999+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.6:0/2123285042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:35.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:35 smithi006 bash[27292]: audit 2023-08-27T21:59:34.611999+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.6:0/2123285042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:35.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:35 smithi006 bash[33918]: audit 2023-08-27T21:59:34.611999+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.6:0/2123285042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:36.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:36 smithi130 bash[27579]: cluster 2023-08-27T21:59:34.894785+0000 mgr.y (mgr.14901) 32 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2023-08-27T21:59:36.474 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:36 smithi006 bash[27292]: cluster 2023-08-27T21:59:34.894785+0000 mgr.y (mgr.14901) 32 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2023-08-27T21:59:36.477 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:36 smithi006 bash[33918]: cluster 2023-08-27T21:59:34.894785+0000 mgr.y (mgr.14901) 32 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2023-08-27T21:59:38.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:38 smithi130 bash[27579]: cluster 2023-08-27T21:59:36.895354+0000 mgr.y (mgr.14901) 33 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-08-27T21:59:38.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:38 smithi130 bash[27579]: audit 2023-08-27T21:59:37.240815+0000 mon.a (mon.0) 845 : audit [DBG] from='client.? 172.21.15.6:0/3901759093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:38.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:38 smithi006 bash[33918]: cluster 2023-08-27T21:59:36.895354+0000 mgr.y (mgr.14901) 33 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-08-27T21:59:38.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:38 smithi006 bash[33918]: audit 2023-08-27T21:59:37.240815+0000 mon.a (mon.0) 845 : audit [DBG] from='client.? 172.21.15.6:0/3901759093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:38.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:38 smithi006 bash[27292]: cluster 2023-08-27T21:59:36.895354+0000 mgr.y (mgr.14901) 33 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-08-27T21:59:38.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:38 smithi006 bash[27292]: audit 2023-08-27T21:59:37.240815+0000 mon.a (mon.0) 845 : audit [DBG] from='client.? 172.21.15.6:0/3901759093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:40.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:40 smithi130 bash[27579]: cluster 2023-08-27T21:59:38.895870+0000 mgr.y (mgr.14901) 34 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-08-27T21:59:40.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:40 smithi130 bash[27579]: audit 2023-08-27T21:59:39.859843+0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.6:0/3329036835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:40.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:40 smithi006 bash[33918]: cluster 2023-08-27T21:59:38.895870+0000 mgr.y (mgr.14901) 34 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-08-27T21:59:40.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:40 smithi006 bash[33918]: audit 2023-08-27T21:59:39.859843+0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.6:0/3329036835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:40.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:40 smithi006 bash[27292]: cluster 2023-08-27T21:59:38.895870+0000 mgr.y (mgr.14901) 34 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-08-27T21:59:40.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:40 smithi006 bash[27292]: audit 2023-08-27T21:59:39.859843+0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.6:0/3329036835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:41.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:59:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:59:40] "GET /metrics HTTP/1.1" 200 194669 "" "Prometheus/2.33.4" 2023-08-27T21:59:41.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:59:41 smithi006 bash[56298]: level=error ts=2023-08-27T21:59:41.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T21:59:41.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:59:41 smithi006 bash[56298]: level=warn ts=2023-08-27T21:59:41.623Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T21:59:42.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:42 smithi130 bash[27579]: cluster 2023-08-27T21:59:40.896623+0000 mgr.y (mgr.14901) 35 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-08-27T21:59:42.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:42 smithi006 bash[27292]: cluster 2023-08-27T21:59:40.896623+0000 mgr.y (mgr.14901) 35 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-08-27T21:59:42.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:42 smithi006 bash[33918]: cluster 2023-08-27T21:59:40.896623+0000 mgr.y (mgr.14901) 35 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-08-27T21:59:43.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:43 smithi130 bash[27579]: audit 2023-08-27T21:59:42.470131+0000 mon.a (mon.0) 846 : audit [DBG] from='client.? 172.21.15.6:0/3808407291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:43.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:43 smithi006 bash[33918]: audit 2023-08-27T21:59:42.470131+0000 mon.a (mon.0) 846 : audit [DBG] from='client.? 172.21.15.6:0/3808407291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:43.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:43 smithi006 bash[27292]: audit 2023-08-27T21:59:42.470131+0000 mon.a (mon.0) 846 : audit [DBG] from='client.? 172.21.15.6:0/3808407291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:43.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:59:43 smithi006 bash[56298]: level=error ts=2023-08-27T21:59:43.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T21:59:43.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:59:43 smithi006 bash[56298]: level=warn ts=2023-08-27T21:59:43.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T21:59:44.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:44 smithi130 bash[27579]: cluster 2023-08-27T21:59:42.897181+0000 mgr.y (mgr.14901) 36 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-08-27T21:59:44.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:44 smithi130 bash[27579]: audit 2023-08-27T21:59:43.354551+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:44.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:44 smithi006 bash[27292]: cluster 2023-08-27T21:59:42.897181+0000 mgr.y (mgr.14901) 36 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-08-27T21:59:44.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:44 smithi006 bash[27292]: audit 2023-08-27T21:59:43.354551+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:44.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:44 smithi006 bash[33918]: cluster 2023-08-27T21:59:42.897181+0000 mgr.y (mgr.14901) 36 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-08-27T21:59:44.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:44 smithi006 bash[33918]: audit 2023-08-27T21:59:43.354551+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T21:59:45.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:45 smithi006 bash[27292]: audit 2023-08-27T21:59:45.114065+0000 mon.a (mon.0) 848 : audit [DBG] from='client.? 172.21.15.6:0/2782660836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:45.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:45 smithi006 bash[33918]: audit 2023-08-27T21:59:45.114065+0000 mon.a (mon.0) 848 : audit [DBG] from='client.? 172.21.15.6:0/2782660836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:45.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:45 smithi130 bash[27579]: audit 2023-08-27T21:59:45.114065+0000 mon.a (mon.0) 848 : audit [DBG] from='client.? 172.21.15.6:0/2782660836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:46.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:46 smithi006 bash[33918]: cluster 2023-08-27T21:59:44.897958+0000 mgr.y (mgr.14901) 37 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2023-08-27T21:59:46.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:46 smithi006 bash[27292]: cluster 2023-08-27T21:59:44.897958+0000 mgr.y (mgr.14901) 37 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2023-08-27T21:59:46.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:46 smithi130 bash[27579]: cluster 2023-08-27T21:59:44.897958+0000 mgr.y (mgr.14901) 37 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2023-08-27T21:59:48.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:48 smithi006 bash[27292]: cluster 2023-08-27T21:59:46.898394+0000 mgr.y (mgr.14901) 38 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-08-27T21:59:48.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:48 smithi006 bash[27292]: audit 2023-08-27T21:59:47.757432+0000 mon.a (mon.0) 849 : audit [DBG] from='client.? 172.21.15.6:0/3747340731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:48.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:48 smithi006 bash[33918]: cluster 2023-08-27T21:59:46.898394+0000 mgr.y (mgr.14901) 38 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-08-27T21:59:48.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:48 smithi006 bash[33918]: audit 2023-08-27T21:59:47.757432+0000 mon.a (mon.0) 849 : audit [DBG] from='client.? 172.21.15.6:0/3747340731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:48.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:48 smithi130 bash[27579]: cluster 2023-08-27T21:59:46.898394+0000 mgr.y (mgr.14901) 38 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-08-27T21:59:48.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:48 smithi130 bash[27579]: audit 2023-08-27T21:59:47.757432+0000 mon.a (mon.0) 849 : audit [DBG] from='client.? 172.21.15.6:0/3747340731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:50.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:50 smithi006 bash[33918]: cluster 2023-08-27T21:59:48.898784+0000 mgr.y (mgr.14901) 39 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-08-27T21:59:50.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:50 smithi006 bash[27292]: cluster 2023-08-27T21:59:48.898784+0000 mgr.y (mgr.14901) 39 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-08-27T21:59:50.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:50 smithi130 bash[27579]: cluster 2023-08-27T21:59:48.898784+0000 mgr.y (mgr.14901) 39 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-08-27T21:59:51.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 21:59:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:21:59:50] "GET /metrics HTTP/1.1" 200 194757 "" "Prometheus/2.33.4" 2023-08-27T21:59:51.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:51 smithi006 bash[27292]: audit 2023-08-27T21:59:50.421964+0000 mon.c (mon.2) 63 : audit [DBG] from='client.? 172.21.15.6:0/1697410835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:51.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:51 smithi006 bash[33918]: audit 2023-08-27T21:59:50.421964+0000 mon.c (mon.2) 63 : audit [DBG] from='client.? 172.21.15.6:0/1697410835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:51.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:59:51 smithi006 bash[56298]: level=error ts=2023-08-27T21:59:51.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T21:59:51.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:59:51 smithi006 bash[56298]: level=warn ts=2023-08-27T21:59:51.623Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T21:59:51.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:51 smithi130 bash[27579]: audit 2023-08-27T21:59:50.421964+0000 mon.c (mon.2) 63 : audit [DBG] from='client.? 172.21.15.6:0/1697410835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:52.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:52 smithi006 bash[33918]: cluster 2023-08-27T21:59:50.901630+0000 mgr.y (mgr.14901) 40 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 949 op/s 2023-08-27T21:59:52.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:52 smithi006 bash[27292]: cluster 2023-08-27T21:59:50.901630+0000 mgr.y (mgr.14901) 40 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 949 op/s 2023-08-27T21:59:52.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:52 smithi130 bash[27579]: cluster 2023-08-27T21:59:50.901630+0000 mgr.y (mgr.14901) 40 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 949 op/s 2023-08-27T21:59:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:59:53 smithi006 bash[56298]: level=error ts=2023-08-27T21:59:53.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T21:59:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 21:59:53 smithi006 bash[56298]: level=warn ts=2023-08-27T21:59:53.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T21:59:54.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:53 smithi130 bash[27579]: cluster 2023-08-27T21:59:52.902112+0000 mgr.y (mgr.14901) 41 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-08-27T21:59:54.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:53 smithi130 bash[27579]: audit 2023-08-27T21:59:53.069658+0000 mon.c (mon.2) 64 : audit [DBG] from='client.? 172.21.15.6:0/1502785754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:54.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:53 smithi006 bash[33918]: cluster 2023-08-27T21:59:52.902112+0000 mgr.y (mgr.14901) 41 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-08-27T21:59:54.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:53 smithi006 bash[33918]: audit 2023-08-27T21:59:53.069658+0000 mon.c (mon.2) 64 : audit [DBG] from='client.? 172.21.15.6:0/1502785754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:54.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:53 smithi006 bash[27292]: cluster 2023-08-27T21:59:52.902112+0000 mgr.y (mgr.14901) 41 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-08-27T21:59:54.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:53 smithi006 bash[27292]: audit 2023-08-27T21:59:53.069658+0000 mon.c (mon.2) 64 : audit [DBG] from='client.? 172.21.15.6:0/1502785754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:56.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:55 smithi130 bash[27579]: cluster 2023-08-27T21:59:54.902861+0000 mgr.y (mgr.14901) 42 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2023-08-27T21:59:56.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:55 smithi130 bash[27579]: audit 2023-08-27T21:59:55.731184+0000 mon.b (mon.1) 73 : audit [DBG] from='client.? 172.21.15.6:0/3877993611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:56.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:55 smithi006 bash[33918]: cluster 2023-08-27T21:59:54.902861+0000 mgr.y (mgr.14901) 42 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2023-08-27T21:59:56.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:55 smithi006 bash[33918]: audit 2023-08-27T21:59:55.731184+0000 mon.b (mon.1) 73 : audit [DBG] from='client.? 172.21.15.6:0/3877993611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:56.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:55 smithi006 bash[27292]: cluster 2023-08-27T21:59:54.902861+0000 mgr.y (mgr.14901) 42 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2023-08-27T21:59:56.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:55 smithi006 bash[27292]: audit 2023-08-27T21:59:55.731184+0000 mon.b (mon.1) 73 : audit [DBG] from='client.? 172.21.15.6:0/3877993611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:58.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:57 smithi130 bash[27579]: cluster 2023-08-27T21:59:56.903292+0000 mgr.y (mgr.14901) 43 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-08-27T21:59:58.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:57 smithi006 bash[27292]: cluster 2023-08-27T21:59:56.903292+0000 mgr.y (mgr.14901) 43 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-08-27T21:59:58.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:58 smithi006 bash[33918]: cluster 2023-08-27T21:59:56.903292+0000 mgr.y (mgr.14901) 43 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-08-27T21:59:59.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 21:59:59 smithi130 bash[27579]: audit 2023-08-27T21:59:58.360622+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.6:0/3980022024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:59.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 21:59:59 smithi006 bash[27292]: audit 2023-08-27T21:59:58.360622+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.6:0/3980022024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:59.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 21:59:59 smithi006 bash[33918]: audit 2023-08-27T21:59:58.360622+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.6:0/3980022024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:00.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:00 smithi130 bash[27579]: cluster 2023-08-27T21:59:58.903910+0000 mgr.y (mgr.14901) 44 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-08-27T22:00:00.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:00 smithi130 bash[27579]: cluster 2023-08-27T22:00:00.000133+0000 mon.a (mon.0) 850 : cluster [WRN] Health detail: HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:00:00.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:00 smithi130 bash[27579]: cluster 2023-08-27T22:00:00.000196+0000 mon.a (mon.0) 851 : cluster [WRN] [WRN] UPGRADE_NO_STANDBY_MGR: Upgrade: Need standby mgr daemon 2023-08-27T22:00:00.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:00 smithi130 bash[27579]: cluster 2023-08-27T22:00:00.000223+0000 mon.a (mon.0) 852 : cluster [WRN] The upgrade process needs to upgrade the mgr, but it needs at least one standby to proceed. 2023-08-27T22:00:00.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:00 smithi006 bash[27292]: cluster 2023-08-27T21:59:58.903910+0000 mgr.y (mgr.14901) 44 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-08-27T22:00:00.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:00 smithi006 bash[27292]: cluster 2023-08-27T22:00:00.000133+0000 mon.a (mon.0) 850 : cluster [WRN] Health detail: HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:00:00.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:00 smithi006 bash[27292]: cluster 2023-08-27T22:00:00.000196+0000 mon.a (mon.0) 851 : cluster [WRN] [WRN] UPGRADE_NO_STANDBY_MGR: Upgrade: Need standby mgr daemon 2023-08-27T22:00:00.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:00 smithi006 bash[27292]: cluster 2023-08-27T22:00:00.000223+0000 mon.a (mon.0) 852 : cluster [WRN] The upgrade process needs to upgrade the mgr, but it needs at least one standby to proceed. 2023-08-27T22:00:00.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:00 smithi006 bash[33918]: cluster 2023-08-27T21:59:58.903910+0000 mgr.y (mgr.14901) 44 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-08-27T22:00:00.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:00 smithi006 bash[33918]: cluster 2023-08-27T22:00:00.000133+0000 mon.a (mon.0) 850 : cluster [WRN] Health detail: HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:00:00.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:00 smithi006 bash[33918]: cluster 2023-08-27T22:00:00.000196+0000 mon.a (mon.0) 851 : cluster [WRN] [WRN] UPGRADE_NO_STANDBY_MGR: Upgrade: Need standby mgr daemon 2023-08-27T22:00:00.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:00 smithi006 bash[33918]: cluster 2023-08-27T22:00:00.000223+0000 mon.a (mon.0) 852 : cluster [WRN] The upgrade process needs to upgrade the mgr, but it needs at least one standby to proceed. 2023-08-27T22:00:01.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:00:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:00:00] "GET /metrics HTTP/1.1" 200 194775 "" "Prometheus/2.33.4" 2023-08-27T22:00:01.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:00:01.623Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:01.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:00:01.625Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:02.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:02 smithi130 bash[27579]: cluster 2023-08-27T22:00:00.904704+0000 mgr.y (mgr.14901) 45 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 924 op/s 2023-08-27T22:00:02.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:02 smithi130 bash[27579]: audit 2023-08-27T22:00:01.030733+0000 mon.b (mon.1) 74 : audit [DBG] from='client.? 172.21.15.6:0/1616178136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:02.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:02 smithi006 bash[33918]: cluster 2023-08-27T22:00:00.904704+0000 mgr.y (mgr.14901) 45 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 924 op/s 2023-08-27T22:00:02.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:02 smithi006 bash[33918]: audit 2023-08-27T22:00:01.030733+0000 mon.b (mon.1) 74 : audit [DBG] from='client.? 172.21.15.6:0/1616178136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:02.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:02 smithi006 bash[27292]: cluster 2023-08-27T22:00:00.904704+0000 mgr.y (mgr.14901) 45 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 924 op/s 2023-08-27T22:00:02.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:02 smithi006 bash[27292]: audit 2023-08-27T22:00:01.030733+0000 mon.b (mon.1) 74 : audit [DBG] from='client.? 172.21.15.6:0/1616178136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:00:03.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:00:03.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:04.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:04 smithi130 bash[27579]: cluster 2023-08-27T22:00:02.905241+0000 mgr.y (mgr.14901) 46 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-08-27T22:00:04.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:04 smithi130 bash[27579]: audit 2023-08-27T22:00:03.637487+0000 mon.c (mon.2) 66 : audit [DBG] from='client.? 172.21.15.6:0/1730388000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:04.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:04 smithi006 bash[33918]: cluster 2023-08-27T22:00:02.905241+0000 mgr.y (mgr.14901) 46 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-08-27T22:00:04.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:04 smithi006 bash[33918]: audit 2023-08-27T22:00:03.637487+0000 mon.c (mon.2) 66 : audit [DBG] from='client.? 172.21.15.6:0/1730388000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:04.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:04 smithi006 bash[27292]: cluster 2023-08-27T22:00:02.905241+0000 mgr.y (mgr.14901) 46 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-08-27T22:00:04.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:04 smithi006 bash[27292]: audit 2023-08-27T22:00:03.637487+0000 mon.c (mon.2) 66 : audit [DBG] from='client.? 172.21.15.6:0/1730388000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:06.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:06 smithi130 bash[27579]: cluster 2023-08-27T22:00:04.906017+0000 mgr.y (mgr.14901) 47 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 912 op/s 2023-08-27T22:00:06.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:06 smithi006 bash[33918]: cluster 2023-08-27T22:00:04.906017+0000 mgr.y (mgr.14901) 47 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 912 op/s 2023-08-27T22:00:06.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:06 smithi006 bash[27292]: cluster 2023-08-27T22:00:04.906017+0000 mgr.y (mgr.14901) 47 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 912 op/s 2023-08-27T22:00:07.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:07 smithi130 bash[27579]: audit 2023-08-27T22:00:06.285730+0000 mon.c (mon.2) 67 : audit [DBG] from='client.? 172.21.15.6:0/710518986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:07.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:07 smithi006 bash[27292]: audit 2023-08-27T22:00:06.285730+0000 mon.c (mon.2) 67 : audit [DBG] from='client.? 172.21.15.6:0/710518986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:07.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:07 smithi006 bash[33918]: audit 2023-08-27T22:00:06.285730+0000 mon.c (mon.2) 67 : audit [DBG] from='client.? 172.21.15.6:0/710518986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:08.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:08 smithi130 bash[27579]: cluster 2023-08-27T22:00:06.906462+0000 mgr.y (mgr.14901) 48 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-08-27T22:00:08.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:08 smithi006 bash[27292]: cluster 2023-08-27T22:00:06.906462+0000 mgr.y (mgr.14901) 48 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-08-27T22:00:08.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:08 smithi006 bash[33918]: cluster 2023-08-27T22:00:06.906462+0000 mgr.y (mgr.14901) 48 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-08-27T22:00:09.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:09 smithi130 bash[27579]: audit 2023-08-27T22:00:08.909009+0000 mon.a (mon.0) 853 : audit [DBG] from='client.? 172.21.15.6:0/645419134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:09.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:09 smithi006 bash[27292]: audit 2023-08-27T22:00:08.909009+0000 mon.a (mon.0) 853 : audit [DBG] from='client.? 172.21.15.6:0/645419134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:09.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:09 smithi006 bash[33918]: audit 2023-08-27T22:00:08.909009+0000 mon.a (mon.0) 853 : audit [DBG] from='client.? 172.21.15.6:0/645419134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:10.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:10 smithi130 bash[27579]: cluster 2023-08-27T22:00:08.906941+0000 mgr.y (mgr.14901) 49 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-08-27T22:00:10.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:10 smithi006 bash[33918]: cluster 2023-08-27T22:00:08.906941+0000 mgr.y (mgr.14901) 49 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-08-27T22:00:10.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:10 smithi006 bash[27292]: cluster 2023-08-27T22:00:08.906941+0000 mgr.y (mgr.14901) 49 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-08-27T22:00:11.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:00:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:00:10] "GET /metrics HTTP/1.1" 200 194775 "" "Prometheus/2.33.4" 2023-08-27T22:00:11.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:00:11.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:11.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:00:11.625Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:12.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:12 smithi130 bash[27579]: cluster 2023-08-27T22:00:10.907607+0000 mgr.y (mgr.14901) 50 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-08-27T22:00:12.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:12 smithi130 bash[27579]: audit 2023-08-27T22:00:11.571342+0000 mon.c (mon.2) 68 : audit [DBG] from='client.? 172.21.15.6:0/3462386255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:12.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:12 smithi006 bash[33918]: cluster 2023-08-27T22:00:10.907607+0000 mgr.y (mgr.14901) 50 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-08-27T22:00:12.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:12 smithi006 bash[33918]: audit 2023-08-27T22:00:11.571342+0000 mon.c (mon.2) 68 : audit [DBG] from='client.? 172.21.15.6:0/3462386255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:12.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:12 smithi006 bash[27292]: cluster 2023-08-27T22:00:10.907607+0000 mgr.y (mgr.14901) 50 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-08-27T22:00:12.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:12 smithi006 bash[27292]: audit 2023-08-27T22:00:11.571342+0000 mon.c (mon.2) 68 : audit [DBG] from='client.? 172.21.15.6:0/3462386255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:00:13.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:00:13.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:14.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:14 smithi130 bash[27579]: cluster 2023-08-27T22:00:12.910900+0000 mgr.y (mgr.14901) 51 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-08-27T22:00:14.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:14 smithi006 bash[33918]: cluster 2023-08-27T22:00:12.910900+0000 mgr.y (mgr.14901) 51 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-08-27T22:00:14.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:14 smithi006 bash[27292]: cluster 2023-08-27T22:00:12.910900+0000 mgr.y (mgr.14901) 51 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-08-27T22:00:15.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:15 smithi130 bash[27579]: audit 2023-08-27T22:00:14.212351+0000 mon.b (mon.1) 75 : audit [DBG] from='client.? 172.21.15.6:0/718194824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:15.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:15 smithi006 bash[33918]: audit 2023-08-27T22:00:14.212351+0000 mon.b (mon.1) 75 : audit [DBG] from='client.? 172.21.15.6:0/718194824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:15.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:15 smithi006 bash[27292]: audit 2023-08-27T22:00:14.212351+0000 mon.b (mon.1) 75 : audit [DBG] from='client.? 172.21.15.6:0/718194824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:16.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:16 smithi130 bash[27579]: cluster 2023-08-27T22:00:14.911653+0000 mgr.y (mgr.14901) 52 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2023-08-27T22:00:16.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:16 smithi006 bash[27292]: cluster 2023-08-27T22:00:14.911653+0000 mgr.y (mgr.14901) 52 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2023-08-27T22:00:16.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:16 smithi006 bash[33918]: cluster 2023-08-27T22:00:14.911653+0000 mgr.y (mgr.14901) 52 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2023-08-27T22:00:17.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:17 smithi130 bash[27579]: audit 2023-08-27T22:00:16.836912+0000 mon.a (mon.0) 854 : audit [DBG] from='client.? 172.21.15.6:0/2055941640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:17.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:17 smithi006 bash[27292]: audit 2023-08-27T22:00:16.836912+0000 mon.a (mon.0) 854 : audit [DBG] from='client.? 172.21.15.6:0/2055941640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:17.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:17 smithi006 bash[33918]: audit 2023-08-27T22:00:16.836912+0000 mon.a (mon.0) 854 : audit [DBG] from='client.? 172.21.15.6:0/2055941640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:18.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:18 smithi130 bash[27579]: cluster 2023-08-27T22:00:16.912092+0000 mgr.y (mgr.14901) 53 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-08-27T22:00:18.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:18 smithi006 bash[33918]: cluster 2023-08-27T22:00:16.912092+0000 mgr.y (mgr.14901) 53 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-08-27T22:00:18.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:18 smithi006 bash[27292]: cluster 2023-08-27T22:00:16.912092+0000 mgr.y (mgr.14901) 53 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-08-27T22:00:20.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:20 smithi130 bash[27579]: cluster 2023-08-27T22:00:18.912527+0000 mgr.y (mgr.14901) 54 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-08-27T22:00:20.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:20 smithi130 bash[27579]: audit 2023-08-27T22:00:19.508439+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.6:0/2467271249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:20.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:20 smithi006 bash[33918]: cluster 2023-08-27T22:00:18.912527+0000 mgr.y (mgr.14901) 54 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-08-27T22:00:20.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:20 smithi006 bash[33918]: audit 2023-08-27T22:00:19.508439+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.6:0/2467271249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:20.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:20 smithi006 bash[27292]: cluster 2023-08-27T22:00:18.912527+0000 mgr.y (mgr.14901) 54 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-08-27T22:00:20.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:20 smithi006 bash[27292]: audit 2023-08-27T22:00:19.508439+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.6:0/2467271249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:21.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:00:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:00:20] "GET /metrics HTTP/1.1" 200 194792 "" "Prometheus/2.33.4" 2023-08-27T22:00:21.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:00:21.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:21.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:00:21.625Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:22.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:22 smithi006 bash[27292]: cluster 2023-08-27T22:00:20.913283+0000 mgr.y (mgr.14901) 55 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2023-08-27T22:00:22.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:22 smithi006 bash[33918]: cluster 2023-08-27T22:00:20.913283+0000 mgr.y (mgr.14901) 55 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2023-08-27T22:00:22.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:22 smithi130 bash[27579]: cluster 2023-08-27T22:00:20.913283+0000 mgr.y (mgr.14901) 55 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2023-08-27T22:00:23.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:23 smithi006 bash[27292]: audit 2023-08-27T22:00:22.152886+0000 mon.a (mon.0) 855 : audit [DBG] from='client.? 172.21.15.6:0/3401554798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:23.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:23 smithi006 bash[33918]: audit 2023-08-27T22:00:22.152886+0000 mon.a (mon.0) 855 : audit [DBG] from='client.? 172.21.15.6:0/3401554798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:23.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:23 smithi130 bash[27579]: audit 2023-08-27T22:00:22.152886+0000 mon.a (mon.0) 855 : audit [DBG] from='client.? 172.21.15.6:0/3401554798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:00:23.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:00:23.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:24.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:24 smithi006 bash[27292]: cluster 2023-08-27T22:00:22.913676+0000 mgr.y (mgr.14901) 56 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-08-27T22:00:24.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:24 smithi006 bash[33918]: cluster 2023-08-27T22:00:22.913676+0000 mgr.y (mgr.14901) 56 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-08-27T22:00:24.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:24 smithi130 bash[27579]: cluster 2023-08-27T22:00:22.913676+0000 mgr.y (mgr.14901) 56 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-08-27T22:00:25.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:25 smithi006 bash[33918]: audit 2023-08-27T22:00:24.773674+0000 mon.a (mon.0) 856 : audit [DBG] from='client.? 172.21.15.6:0/2204145141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:25.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:25 smithi006 bash[27292]: audit 2023-08-27T22:00:24.773674+0000 mon.a (mon.0) 856 : audit [DBG] from='client.? 172.21.15.6:0/2204145141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:25.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:25 smithi130 bash[27579]: audit 2023-08-27T22:00:24.773674+0000 mon.a (mon.0) 856 : audit [DBG] from='client.? 172.21.15.6:0/2204145141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:26.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:26 smithi006 bash[33918]: cluster 2023-08-27T22:00:24.914390+0000 mgr.y (mgr.14901) 57 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 892 op/s 2023-08-27T22:00:26.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:26 smithi006 bash[27292]: cluster 2023-08-27T22:00:24.914390+0000 mgr.y (mgr.14901) 57 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 892 op/s 2023-08-27T22:00:26.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:26 smithi130 bash[27579]: cluster 2023-08-27T22:00:24.914390+0000 mgr.y (mgr.14901) 57 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 892 op/s 2023-08-27T22:00:28.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:28 smithi006 bash[33918]: cluster 2023-08-27T22:00:26.914899+0000 mgr.y (mgr.14901) 58 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-08-27T22:00:28.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:28 smithi006 bash[33918]: audit 2023-08-27T22:00:27.422788+0000 mon.c (mon.2) 70 : audit [DBG] from='client.? 172.21.15.6:0/436347325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:28.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:28 smithi006 bash[27292]: cluster 2023-08-27T22:00:26.914899+0000 mgr.y (mgr.14901) 58 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-08-27T22:00:28.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:28 smithi006 bash[27292]: audit 2023-08-27T22:00:27.422788+0000 mon.c (mon.2) 70 : audit [DBG] from='client.? 172.21.15.6:0/436347325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:28.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:28 smithi130 bash[27579]: cluster 2023-08-27T22:00:26.914899+0000 mgr.y (mgr.14901) 58 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-08-27T22:00:28.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:28 smithi130 bash[27579]: audit 2023-08-27T22:00:27.422788+0000 mon.c (mon.2) 70 : audit [DBG] from='client.? 172.21.15.6:0/436347325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:30.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:30 smithi006 bash[27292]: cluster 2023-08-27T22:00:28.915261+0000 mgr.y (mgr.14901) 59 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-08-27T22:00:30.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:30 smithi006 bash[27292]: cluster 2023-08-27T22:00:29.161465+0000 mon.a (mon.0) 857 : cluster [DBG] mgrmap e26: y(active, since 91s) 2023-08-27T22:00:30.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:30 smithi006 bash[27292]: audit 2023-08-27T22:00:30.069015+0000 mon.b (mon.1) 76 : audit [DBG] from='client.? 172.21.15.6:0/1535817035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:30.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:30 smithi006 bash[33918]: cluster 2023-08-27T22:00:28.915261+0000 mgr.y (mgr.14901) 59 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-08-27T22:00:30.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:30 smithi006 bash[33918]: cluster 2023-08-27T22:00:29.161465+0000 mon.a (mon.0) 857 : cluster [DBG] mgrmap e26: y(active, since 91s) 2023-08-27T22:00:30.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:30 smithi006 bash[33918]: audit 2023-08-27T22:00:30.069015+0000 mon.b (mon.1) 76 : audit [DBG] from='client.? 172.21.15.6:0/1535817035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:30.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:30 smithi130 bash[27579]: cluster 2023-08-27T22:00:28.915261+0000 mgr.y (mgr.14901) 59 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-08-27T22:00:30.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:30 smithi130 bash[27579]: cluster 2023-08-27T22:00:29.161465+0000 mon.a (mon.0) 857 : cluster [DBG] mgrmap e26: y(active, since 91s) 2023-08-27T22:00:30.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:30 smithi130 bash[27579]: audit 2023-08-27T22:00:30.069015+0000 mon.b (mon.1) 76 : audit [DBG] from='client.? 172.21.15.6:0/1535817035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:31.176 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:00:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:00:30] "GET /metrics HTTP/1.1" 200 194812 "" "Prometheus/2.33.4" 2023-08-27T22:00:31.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:31 smithi006 bash[27292]: audit 2023-08-27T22:00:30.958276+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:00:31.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:31 smithi006 bash[33918]: audit 2023-08-27T22:00:30.958276+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:00:31.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:31 smithi130 bash[27579]: audit 2023-08-27T22:00:30.958276+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:00:31.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:00:31.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:31.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:00:31.626Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:32.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:32 smithi006 bash[33918]: cluster 2023-08-27T22:00:30.916275+0000 mgr.y (mgr.14901) 60 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 255 B/s wr, 890 op/s 2023-08-27T22:00:32.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:32 smithi006 bash[27292]: cluster 2023-08-27T22:00:30.916275+0000 mgr.y (mgr.14901) 60 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 255 B/s wr, 890 op/s 2023-08-27T22:00:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:32 smithi130 bash[27579]: cluster 2023-08-27T22:00:30.916275+0000 mgr.y (mgr.14901) 60 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 255 B/s wr, 890 op/s 2023-08-27T22:00:33.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:33 smithi006 bash[27292]: audit 2023-08-27T22:00:32.730679+0000 mon.a (mon.0) 858 : audit [DBG] from='client.? 172.21.15.6:0/664289906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:33.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:33 smithi006 bash[33918]: audit 2023-08-27T22:00:32.730679+0000 mon.a (mon.0) 858 : audit [DBG] from='client.? 172.21.15.6:0/664289906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:33.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:33 smithi130 bash[27579]: audit 2023-08-27T22:00:32.730679+0000 mon.a (mon.0) 858 : audit [DBG] from='client.? 172.21.15.6:0/664289906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:00:33.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:00:33.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:34.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:34 smithi006 bash[33918]: cluster 2023-08-27T22:00:32.916709+0000 mgr.y (mgr.14901) 61 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 255 B/s wr, 594 op/s 2023-08-27T22:00:34.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:34 smithi006 bash[27292]: cluster 2023-08-27T22:00:32.916709+0000 mgr.y (mgr.14901) 61 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 255 B/s wr, 594 op/s 2023-08-27T22:00:34.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:34 smithi130 bash[27579]: cluster 2023-08-27T22:00:32.916709+0000 mgr.y (mgr.14901) 61 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 255 B/s wr, 594 op/s 2023-08-27T22:00:36.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:36 smithi006 bash[27292]: cluster 2023-08-27T22:00:34.917454+0000 mgr.y (mgr.14901) 62 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 255 B/s wr, 884 op/s 2023-08-27T22:00:36.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:36 smithi006 bash[27292]: audit 2023-08-27T22:00:35.411192+0000 mon.a (mon.0) 859 : audit [DBG] from='client.? 172.21.15.6:0/2684110079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:36.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:36 smithi006 bash[33918]: cluster 2023-08-27T22:00:34.917454+0000 mgr.y (mgr.14901) 62 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 255 B/s wr, 884 op/s 2023-08-27T22:00:36.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:36 smithi006 bash[33918]: audit 2023-08-27T22:00:35.411192+0000 mon.a (mon.0) 859 : audit [DBG] from='client.? 172.21.15.6:0/2684110079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:36.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:36 smithi130 bash[27579]: cluster 2023-08-27T22:00:34.917454+0000 mgr.y (mgr.14901) 62 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 255 B/s wr, 884 op/s 2023-08-27T22:00:36.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:36 smithi130 bash[27579]: audit 2023-08-27T22:00:35.411192+0000 mon.a (mon.0) 859 : audit [DBG] from='client.? 172.21.15.6:0/2684110079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:38.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:38 smithi130 bash[27579]: cluster 2023-08-27T22:00:36.917821+0000 mgr.y (mgr.14901) 63 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 255 B/s wr, 589 op/s 2023-08-27T22:00:38.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:38 smithi130 bash[27579]: audit 2023-08-27T22:00:37.063389+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:00:38.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:38 smithi130 bash[27579]: audit 2023-08-27T22:00:37.622208+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:00:38.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:38 smithi130 bash[27579]: audit 2023-08-27T22:00:37.968241+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:00:38.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:38 smithi130 bash[27579]: audit 2023-08-27T22:00:37.970232+0000 mon.b (mon.1) 79 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:00:38.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:38 smithi130 bash[27579]: audit 2023-08-27T22:00:37.977574+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:00:38.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:38 smithi130 bash[27579]: audit 2023-08-27T22:00:38.040017+0000 mon.a (mon.0) 863 : audit [DBG] from='client.? 172.21.15.6:0/3642221943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:38.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:38 smithi006 bash[27292]: cluster 2023-08-27T22:00:36.917821+0000 mgr.y (mgr.14901) 63 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 255 B/s wr, 589 op/s 2023-08-27T22:00:38.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:38 smithi006 bash[27292]: audit 2023-08-27T22:00:37.063389+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:00:38.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:38 smithi006 bash[27292]: audit 2023-08-27T22:00:37.622208+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:00:38.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:38 smithi006 bash[27292]: audit 2023-08-27T22:00:37.968241+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:00:38.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:38 smithi006 bash[27292]: audit 2023-08-27T22:00:37.970232+0000 mon.b (mon.1) 79 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:00:38.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:38 smithi006 bash[27292]: audit 2023-08-27T22:00:37.977574+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:00:38.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:38 smithi006 bash[27292]: audit 2023-08-27T22:00:38.040017+0000 mon.a (mon.0) 863 : audit [DBG] from='client.? 172.21.15.6:0/3642221943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:38.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:38 smithi006 bash[33918]: cluster 2023-08-27T22:00:36.917821+0000 mgr.y (mgr.14901) 63 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 255 B/s wr, 589 op/s 2023-08-27T22:00:38.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:38 smithi006 bash[33918]: audit 2023-08-27T22:00:37.063389+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:00:38.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:38 smithi006 bash[33918]: audit 2023-08-27T22:00:37.622208+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:00:38.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:38 smithi006 bash[33918]: audit 2023-08-27T22:00:37.968241+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:00:38.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:38 smithi006 bash[33918]: audit 2023-08-27T22:00:37.970232+0000 mon.b (mon.1) 79 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:00:38.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:38 smithi006 bash[33918]: audit 2023-08-27T22:00:37.977574+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:00:38.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:38 smithi006 bash[33918]: audit 2023-08-27T22:00:38.040017+0000 mon.a (mon.0) 863 : audit [DBG] from='client.? 172.21.15.6:0/3642221943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:40.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:40 smithi130 bash[27579]: cluster 2023-08-27T22:00:38.918256+0000 mgr.y (mgr.14901) 64 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 255 B/s wr, 589 op/s 2023-08-27T22:00:40.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:40 smithi006 bash[27292]: cluster 2023-08-27T22:00:38.918256+0000 mgr.y (mgr.14901) 64 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 255 B/s wr, 589 op/s 2023-08-27T22:00:40.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:40 smithi006 bash[33918]: cluster 2023-08-27T22:00:38.918256+0000 mgr.y (mgr.14901) 64 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 255 B/s wr, 589 op/s 2023-08-27T22:00:41.092 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:41 smithi006 bash[27292]: audit 2023-08-27T22:00:40.668277+0000 mon.c (mon.2) 71 : audit [DBG] from='client.? 172.21.15.6:0/2545946419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:41.092 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:00:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:00:40] "GET /metrics HTTP/1.1" 200 194812 "" "Prometheus/2.33.4" 2023-08-27T22:00:41.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:41 smithi130 bash[27579]: audit 2023-08-27T22:00:40.668277+0000 mon.c (mon.2) 71 : audit [DBG] from='client.? 172.21.15.6:0/2545946419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:41.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:41 smithi006 bash[33918]: audit 2023-08-27T22:00:40.668277+0000 mon.c (mon.2) 71 : audit [DBG] from='client.? 172.21.15.6:0/2545946419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:41.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:00:41.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:41.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:00:41.627Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:42.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:42 smithi130 bash[27579]: cluster 2023-08-27T22:00:40.921526+0000 mgr.y (mgr.14901) 65 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 869 op/s 2023-08-27T22:00:42.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:42 smithi006 bash[27292]: cluster 2023-08-27T22:00:40.921526+0000 mgr.y (mgr.14901) 65 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 869 op/s 2023-08-27T22:00:42.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:42 smithi006 bash[33918]: cluster 2023-08-27T22:00:40.921526+0000 mgr.y (mgr.14901) 65 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 869 op/s 2023-08-27T22:00:43.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:00:43.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:00:43.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:44.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:44 smithi130 bash[27579]: cluster 2023-08-27T22:00:42.922005+0000 mgr.y (mgr.14901) 66 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-08-27T22:00:44.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:44 smithi130 bash[27579]: audit 2023-08-27T22:00:43.294837+0000 mon.a (mon.0) 864 : audit [DBG] from='client.? 172.21.15.6:0/638991535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:44.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:44 smithi006 bash[33918]: cluster 2023-08-27T22:00:42.922005+0000 mgr.y (mgr.14901) 66 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-08-27T22:00:44.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:44 smithi006 bash[33918]: audit 2023-08-27T22:00:43.294837+0000 mon.a (mon.0) 864 : audit [DBG] from='client.? 172.21.15.6:0/638991535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:44.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:44 smithi006 bash[27292]: cluster 2023-08-27T22:00:42.922005+0000 mgr.y (mgr.14901) 66 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-08-27T22:00:44.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:44 smithi006 bash[27292]: audit 2023-08-27T22:00:43.294837+0000 mon.a (mon.0) 864 : audit [DBG] from='client.? 172.21.15.6:0/638991535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:46.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:46 smithi130 bash[27579]: cluster 2023-08-27T22:00:44.922750+0000 mgr.y (mgr.14901) 67 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2023-08-27T22:00:46.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:46 smithi130 bash[27579]: audit 2023-08-27T22:00:45.916685+0000 mon.a (mon.0) 865 : audit [DBG] from='client.? 172.21.15.6:0/4028645371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:46.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:46 smithi006 bash[33918]: cluster 2023-08-27T22:00:44.922750+0000 mgr.y (mgr.14901) 67 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2023-08-27T22:00:46.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:46 smithi006 bash[33918]: audit 2023-08-27T22:00:45.916685+0000 mon.a (mon.0) 865 : audit [DBG] from='client.? 172.21.15.6:0/4028645371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:46.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:46 smithi006 bash[27292]: cluster 2023-08-27T22:00:44.922750+0000 mgr.y (mgr.14901) 67 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2023-08-27T22:00:46.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:46 smithi006 bash[27292]: audit 2023-08-27T22:00:45.916685+0000 mon.a (mon.0) 865 : audit [DBG] from='client.? 172.21.15.6:0/4028645371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:48.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:48 smithi006 bash[33918]: cluster 2023-08-27T22:00:46.923232+0000 mgr.y (mgr.14901) 68 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-08-27T22:00:48.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:48 smithi006 bash[27292]: cluster 2023-08-27T22:00:46.923232+0000 mgr.y (mgr.14901) 68 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-08-27T22:00:48.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:48 smithi130 bash[27579]: cluster 2023-08-27T22:00:46.923232+0000 mgr.y (mgr.14901) 68 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-08-27T22:00:49.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:49 smithi006 bash[33918]: audit 2023-08-27T22:00:48.555705+0000 mon.a (mon.0) 866 : audit [DBG] from='client.? 172.21.15.6:0/4040336599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:49.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:49 smithi006 bash[27292]: audit 2023-08-27T22:00:48.555705+0000 mon.a (mon.0) 866 : audit [DBG] from='client.? 172.21.15.6:0/4040336599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:49.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:49 smithi130 bash[27579]: audit 2023-08-27T22:00:48.555705+0000 mon.a (mon.0) 866 : audit [DBG] from='client.? 172.21.15.6:0/4040336599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:50.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:50 smithi006 bash[33918]: cluster 2023-08-27T22:00:48.923699+0000 mgr.y (mgr.14901) 69 : cluster [DBG] pgmap v59: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-08-27T22:00:50.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:50 smithi006 bash[27292]: cluster 2023-08-27T22:00:48.923699+0000 mgr.y (mgr.14901) 69 : cluster [DBG] pgmap v59: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-08-27T22:00:50.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:50 smithi130 bash[27579]: cluster 2023-08-27T22:00:48.923699+0000 mgr.y (mgr.14901) 69 : cluster [DBG] pgmap v59: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-08-27T22:00:51.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:00:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:00:50] "GET /metrics HTTP/1.1" 200 194815 "" "Prometheus/2.33.4" 2023-08-27T22:00:51.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:00:51.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:51.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:00:51.628Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:52.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:52 smithi006 bash[27292]: cluster 2023-08-27T22:00:50.924422+0000 mgr.y (mgr.14901) 70 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2023-08-27T22:00:52.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:52 smithi006 bash[27292]: audit 2023-08-27T22:00:51.192803+0000 mon.a (mon.0) 867 : audit [DBG] from='client.? 172.21.15.6:0/642143098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:52.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:52 smithi006 bash[33918]: cluster 2023-08-27T22:00:50.924422+0000 mgr.y (mgr.14901) 70 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2023-08-27T22:00:52.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:52 smithi006 bash[33918]: audit 2023-08-27T22:00:51.192803+0000 mon.a (mon.0) 867 : audit [DBG] from='client.? 172.21.15.6:0/642143098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:52.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:52 smithi130 bash[27579]: cluster 2023-08-27T22:00:50.924422+0000 mgr.y (mgr.14901) 70 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2023-08-27T22:00:52.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:52 smithi130 bash[27579]: audit 2023-08-27T22:00:51.192803+0000 mon.a (mon.0) 867 : audit [DBG] from='client.? 172.21.15.6:0/642143098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:53.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:00:53.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:53.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:00:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:00:53.523Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:00:54.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:54 smithi006 bash[27292]: cluster 2023-08-27T22:00:52.924878+0000 mgr.y (mgr.14901) 71 : cluster [DBG] pgmap v61: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-08-27T22:00:54.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:54 smithi006 bash[27292]: audit 2023-08-27T22:00:53.801568+0000 mon.c (mon.2) 72 : audit [DBG] from='client.? 172.21.15.6:0/1206695448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:54.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:54 smithi006 bash[33918]: cluster 2023-08-27T22:00:52.924878+0000 mgr.y (mgr.14901) 71 : cluster [DBG] pgmap v61: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-08-27T22:00:54.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:54 smithi006 bash[33918]: audit 2023-08-27T22:00:53.801568+0000 mon.c (mon.2) 72 : audit [DBG] from='client.? 172.21.15.6:0/1206695448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:54.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:54 smithi130 bash[27579]: cluster 2023-08-27T22:00:52.924878+0000 mgr.y (mgr.14901) 71 : cluster [DBG] pgmap v61: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-08-27T22:00:54.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:54 smithi130 bash[27579]: audit 2023-08-27T22:00:53.801568+0000 mon.c (mon.2) 72 : audit [DBG] from='client.? 172.21.15.6:0/1206695448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:56.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:56 smithi006 bash[27292]: cluster 2023-08-27T22:00:54.925560+0000 mgr.y (mgr.14901) 72 : cluster [DBG] pgmap v62: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-08-27T22:00:56.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:56 smithi006 bash[33918]: cluster 2023-08-27T22:00:54.925560+0000 mgr.y (mgr.14901) 72 : cluster [DBG] pgmap v62: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-08-27T22:00:56.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:56 smithi130 bash[27579]: cluster 2023-08-27T22:00:54.925560+0000 mgr.y (mgr.14901) 72 : cluster [DBG] pgmap v62: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-08-27T22:00:57.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:57 smithi006 bash[27292]: audit 2023-08-27T22:00:56.426263+0000 mon.a (mon.0) 868 : audit [DBG] from='client.? 172.21.15.6:0/22636016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:57.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:57 smithi006 bash[33918]: audit 2023-08-27T22:00:56.426263+0000 mon.a (mon.0) 868 : audit [DBG] from='client.? 172.21.15.6:0/22636016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:57.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:57 smithi130 bash[27579]: audit 2023-08-27T22:00:56.426263+0000 mon.a (mon.0) 868 : audit [DBG] from='client.? 172.21.15.6:0/22636016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:58.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:58 smithi006 bash[27292]: cluster 2023-08-27T22:00:56.925926+0000 mgr.y (mgr.14901) 73 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-08-27T22:00:58.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:58 smithi006 bash[33918]: cluster 2023-08-27T22:00:56.925926+0000 mgr.y (mgr.14901) 73 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-08-27T22:00:58.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:58 smithi130 bash[27579]: cluster 2023-08-27T22:00:56.925926+0000 mgr.y (mgr.14901) 73 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-08-27T22:00:59.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:00:59 smithi006 bash[27292]: audit 2023-08-27T22:00:59.083632+0000 mon.b (mon.1) 80 : audit [DBG] from='client.? 172.21.15.6:0/1564603788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:59.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:00:59 smithi006 bash[33918]: audit 2023-08-27T22:00:59.083632+0000 mon.b (mon.1) 80 : audit [DBG] from='client.? 172.21.15.6:0/1564603788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:59.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:00:59 smithi130 bash[27579]: audit 2023-08-27T22:00:59.083632+0000 mon.b (mon.1) 80 : audit [DBG] from='client.? 172.21.15.6:0/1564603788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:00.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:00 smithi006 bash[27292]: cluster 2023-08-27T22:00:58.926346+0000 mgr.y (mgr.14901) 74 : cluster [DBG] pgmap v64: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-08-27T22:01:00.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:00 smithi006 bash[33918]: cluster 2023-08-27T22:00:58.926346+0000 mgr.y (mgr.14901) 74 : cluster [DBG] pgmap v64: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-08-27T22:01:00.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:00 smithi130 bash[27579]: cluster 2023-08-27T22:00:58.926346+0000 mgr.y (mgr.14901) 74 : cluster [DBG] pgmap v64: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-08-27T22:01:01.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:01:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:01:00] "GET /metrics HTTP/1.1" 200 194809 "" "Prometheus/2.33.4" 2023-08-27T22:01:01.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:01:01.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:01.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:01:01.628Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:02.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:02 smithi006 bash[33918]: cluster 2023-08-27T22:01:00.927006+0000 mgr.y (mgr.14901) 75 : cluster [DBG] pgmap v65: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2023-08-27T22:01:02.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:02 smithi006 bash[33918]: audit 2023-08-27T22:01:01.738414+0000 mon.b (mon.1) 81 : audit [DBG] from='client.? 172.21.15.6:0/2693461942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:02.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:02 smithi006 bash[27292]: cluster 2023-08-27T22:01:00.927006+0000 mgr.y (mgr.14901) 75 : cluster [DBG] pgmap v65: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2023-08-27T22:01:02.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:02 smithi006 bash[27292]: audit 2023-08-27T22:01:01.738414+0000 mon.b (mon.1) 81 : audit [DBG] from='client.? 172.21.15.6:0/2693461942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:02.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:02 smithi130 bash[27579]: cluster 2023-08-27T22:01:00.927006+0000 mgr.y (mgr.14901) 75 : cluster [DBG] pgmap v65: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2023-08-27T22:01:02.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:02 smithi130 bash[27579]: audit 2023-08-27T22:01:01.738414+0000 mon.b (mon.1) 81 : audit [DBG] from='client.? 172.21.15.6:0/2693461942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:01:03.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:01:03.524Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:04.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:04 smithi006 bash[27292]: cluster 2023-08-27T22:01:02.927429+0000 mgr.y (mgr.14901) 76 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-08-27T22:01:04.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:04 smithi006 bash[33918]: cluster 2023-08-27T22:01:02.927429+0000 mgr.y (mgr.14901) 76 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-08-27T22:01:04.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:04 smithi130 bash[27579]: cluster 2023-08-27T22:01:02.927429+0000 mgr.y (mgr.14901) 76 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-08-27T22:01:05.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:05 smithi006 bash[33918]: audit 2023-08-27T22:01:04.436292+0000 mon.a (mon.0) 869 : audit [DBG] from='client.? 172.21.15.6:0/2039999088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:05.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:05 smithi006 bash[27292]: audit 2023-08-27T22:01:04.436292+0000 mon.a (mon.0) 869 : audit [DBG] from='client.? 172.21.15.6:0/2039999088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:05.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:05 smithi130 bash[27579]: audit 2023-08-27T22:01:04.436292+0000 mon.a (mon.0) 869 : audit [DBG] from='client.? 172.21.15.6:0/2039999088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:06.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:06 smithi006 bash[33918]: cluster 2023-08-27T22:01:04.928161+0000 mgr.y (mgr.14901) 77 : cluster [DBG] pgmap v67: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2023-08-27T22:01:06.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:06 smithi006 bash[27292]: cluster 2023-08-27T22:01:04.928161+0000 mgr.y (mgr.14901) 77 : cluster [DBG] pgmap v67: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2023-08-27T22:01:06.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:06 smithi130 bash[27579]: cluster 2023-08-27T22:01:04.928161+0000 mgr.y (mgr.14901) 77 : cluster [DBG] pgmap v67: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2023-08-27T22:01:07.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:07 smithi130 bash[27579]: audit 2023-08-27T22:01:07.090376+0000 mon.a (mon.0) 870 : audit [DBG] from='client.? 172.21.15.6:0/3313339986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:07.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:07 smithi006 bash[27292]: audit 2023-08-27T22:01:07.090376+0000 mon.a (mon.0) 870 : audit [DBG] from='client.? 172.21.15.6:0/3313339986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:07.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:07 smithi006 bash[33918]: audit 2023-08-27T22:01:07.090376+0000 mon.a (mon.0) 870 : audit [DBG] from='client.? 172.21.15.6:0/3313339986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:08.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:08 smithi130 bash[27579]: cluster 2023-08-27T22:01:06.928644+0000 mgr.y (mgr.14901) 78 : cluster [DBG] pgmap v68: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-08-27T22:01:08.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:08 smithi006 bash[27292]: cluster 2023-08-27T22:01:06.928644+0000 mgr.y (mgr.14901) 78 : cluster [DBG] pgmap v68: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-08-27T22:01:08.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:08 smithi006 bash[33918]: cluster 2023-08-27T22:01:06.928644+0000 mgr.y (mgr.14901) 78 : cluster [DBG] pgmap v68: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-08-27T22:01:10.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:10 smithi130 bash[27579]: cluster 2023-08-27T22:01:08.929083+0000 mgr.y (mgr.14901) 79 : cluster [DBG] pgmap v69: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-08-27T22:01:10.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:10 smithi130 bash[27579]: audit 2023-08-27T22:01:09.738388+0000 mon.c (mon.2) 73 : audit [DBG] from='client.? 172.21.15.6:0/1874255174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:10.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:10 smithi006 bash[33918]: cluster 2023-08-27T22:01:08.929083+0000 mgr.y (mgr.14901) 79 : cluster [DBG] pgmap v69: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-08-27T22:01:10.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:10 smithi006 bash[33918]: audit 2023-08-27T22:01:09.738388+0000 mon.c (mon.2) 73 : audit [DBG] from='client.? 172.21.15.6:0/1874255174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:10.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:10 smithi006 bash[27292]: cluster 2023-08-27T22:01:08.929083+0000 mgr.y (mgr.14901) 79 : cluster [DBG] pgmap v69: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-08-27T22:01:10.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:10 smithi006 bash[27292]: audit 2023-08-27T22:01:09.738388+0000 mon.c (mon.2) 73 : audit [DBG] from='client.? 172.21.15.6:0/1874255174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:11.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:01:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:01:10] "GET /metrics HTTP/1.1" 200 194809 "" "Prometheus/2.33.4" 2023-08-27T22:01:11.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:01:11.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:11.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:01:11.630Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:12.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:12 smithi130 bash[27579]: cluster 2023-08-27T22:01:10.929967+0000 mgr.y (mgr.14901) 80 : cluster [DBG] pgmap v70: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-08-27T22:01:12.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:12 smithi006 bash[27292]: cluster 2023-08-27T22:01:10.929967+0000 mgr.y (mgr.14901) 80 : cluster [DBG] pgmap v70: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-08-27T22:01:12.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:12 smithi006 bash[33918]: cluster 2023-08-27T22:01:10.929967+0000 mgr.y (mgr.14901) 80 : cluster [DBG] pgmap v70: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-08-27T22:01:13.530 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:13 smithi006 bash[27292]: audit 2023-08-27T22:01:12.386768+0000 mon.c (mon.2) 74 : audit [DBG] from='client.? 172.21.15.6:0/3375711966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:13.530 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:13 smithi006 bash[33918]: audit 2023-08-27T22:01:12.386768+0000 mon.c (mon.2) 74 : audit [DBG] from='client.? 172.21.15.6:0/3375711966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:13.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:13 smithi130 bash[27579]: audit 2023-08-27T22:01:12.386768+0000 mon.c (mon.2) 74 : audit [DBG] from='client.? 172.21.15.6:0/3375711966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:01:13.523Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:01:13.524Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:14.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:14 smithi130 bash[27579]: cluster 2023-08-27T22:01:12.930387+0000 mgr.y (mgr.14901) 81 : cluster [DBG] pgmap v71: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2023-08-27T22:01:14.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:14 smithi006 bash[33918]: cluster 2023-08-27T22:01:12.930387+0000 mgr.y (mgr.14901) 81 : cluster [DBG] pgmap v71: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2023-08-27T22:01:14.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:14 smithi006 bash[27292]: cluster 2023-08-27T22:01:12.930387+0000 mgr.y (mgr.14901) 81 : cluster [DBG] pgmap v71: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2023-08-27T22:01:15.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:15 smithi130 bash[27579]: audit 2023-08-27T22:01:15.032710+0000 mon.c (mon.2) 75 : audit [DBG] from='client.? 172.21.15.6:0/3755614919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:15.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:15 smithi006 bash[27292]: audit 2023-08-27T22:01:15.032710+0000 mon.c (mon.2) 75 : audit [DBG] from='client.? 172.21.15.6:0/3755614919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:15.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:15 smithi006 bash[33918]: audit 2023-08-27T22:01:15.032710+0000 mon.c (mon.2) 75 : audit [DBG] from='client.? 172.21.15.6:0/3755614919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:16.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:16 smithi130 bash[27579]: cluster 2023-08-27T22:01:14.931210+0000 mgr.y (mgr.14901) 82 : cluster [DBG] pgmap v72: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 882 op/s 2023-08-27T22:01:16.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:16 smithi006 bash[33918]: cluster 2023-08-27T22:01:14.931210+0000 mgr.y (mgr.14901) 82 : cluster [DBG] pgmap v72: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 882 op/s 2023-08-27T22:01:16.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:16 smithi006 bash[27292]: cluster 2023-08-27T22:01:14.931210+0000 mgr.y (mgr.14901) 82 : cluster [DBG] pgmap v72: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 882 op/s 2023-08-27T22:01:18.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:18 smithi130 bash[27579]: cluster 2023-08-27T22:01:16.931620+0000 mgr.y (mgr.14901) 83 : cluster [DBG] pgmap v73: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-08-27T22:01:18.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:18 smithi130 bash[27579]: audit 2023-08-27T22:01:17.687249+0000 mon.c (mon.2) 76 : audit [DBG] from='client.? 172.21.15.6:0/1801521357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:18.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:18 smithi006 bash[33918]: cluster 2023-08-27T22:01:16.931620+0000 mgr.y (mgr.14901) 83 : cluster [DBG] pgmap v73: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-08-27T22:01:18.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:18 smithi006 bash[33918]: audit 2023-08-27T22:01:17.687249+0000 mon.c (mon.2) 76 : audit [DBG] from='client.? 172.21.15.6:0/1801521357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:18.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:18 smithi006 bash[27292]: cluster 2023-08-27T22:01:16.931620+0000 mgr.y (mgr.14901) 83 : cluster [DBG] pgmap v73: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-08-27T22:01:18.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:18 smithi006 bash[27292]: audit 2023-08-27T22:01:17.687249+0000 mon.c (mon.2) 76 : audit [DBG] from='client.? 172.21.15.6:0/1801521357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:20.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:20 smithi130 bash[27579]: cluster 2023-08-27T22:01:18.932050+0000 mgr.y (mgr.14901) 84 : cluster [DBG] pgmap v74: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-08-27T22:01:20.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:20 smithi006 bash[33918]: cluster 2023-08-27T22:01:18.932050+0000 mgr.y (mgr.14901) 84 : cluster [DBG] pgmap v74: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-08-27T22:01:20.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:20 smithi006 bash[27292]: cluster 2023-08-27T22:01:18.932050+0000 mgr.y (mgr.14901) 84 : cluster [DBG] pgmap v74: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-08-27T22:01:21.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:01:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:01:20] "GET /metrics HTTP/1.1" 200 194834 "" "Prometheus/2.33.4" 2023-08-27T22:01:21.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:21 smithi130 bash[27579]: audit 2023-08-27T22:01:20.342250+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.6:0/3014353979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:21.637 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:21 smithi006 bash[27292]: audit 2023-08-27T22:01:20.342250+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.6:0/3014353979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:21.637 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:01:21.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:21.637 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:21 smithi006 bash[33918]: audit 2023-08-27T22:01:20.342250+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.6:0/3014353979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:21.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:01:21.632Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:22.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:22 smithi130 bash[27579]: cluster 2023-08-27T22:01:20.932624+0000 mgr.y (mgr.14901) 85 : cluster [DBG] pgmap v75: 89 pgs: 89 active+clean; 1022 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-08-27T22:01:22.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:22 smithi006 bash[27292]: cluster 2023-08-27T22:01:20.932624+0000 mgr.y (mgr.14901) 85 : cluster [DBG] pgmap v75: 89 pgs: 89 active+clean; 1022 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-08-27T22:01:22.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:22 smithi006 bash[33918]: cluster 2023-08-27T22:01:20.932624+0000 mgr.y (mgr.14901) 85 : cluster [DBG] pgmap v75: 89 pgs: 89 active+clean; 1022 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-08-27T22:01:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:01:23.523Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:01:23.525Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:24.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:23 smithi130 bash[27579]: cluster 2023-08-27T22:01:22.933053+0000 mgr.y (mgr.14901) 86 : cluster [DBG] pgmap v76: 89 pgs: 89 active+clean; 1022 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-08-27T22:01:24.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:23 smithi130 bash[27579]: audit 2023-08-27T22:01:22.996135+0000 mon.c (mon.2) 77 : audit [DBG] from='client.? 172.21.15.6:0/2335586245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:24.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:23 smithi006 bash[27292]: cluster 2023-08-27T22:01:22.933053+0000 mgr.y (mgr.14901) 86 : cluster [DBG] pgmap v76: 89 pgs: 89 active+clean; 1022 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-08-27T22:01:24.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:23 smithi006 bash[27292]: audit 2023-08-27T22:01:22.996135+0000 mon.c (mon.2) 77 : audit [DBG] from='client.? 172.21.15.6:0/2335586245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:24.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:23 smithi006 bash[33918]: cluster 2023-08-27T22:01:22.933053+0000 mgr.y (mgr.14901) 86 : cluster [DBG] pgmap v76: 89 pgs: 89 active+clean; 1022 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-08-27T22:01:24.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:23 smithi006 bash[33918]: audit 2023-08-27T22:01:22.996135+0000 mon.c (mon.2) 77 : audit [DBG] from='client.? 172.21.15.6:0/2335586245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:26.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:26 smithi130 bash[27579]: cluster 2023-08-27T22:01:24.933783+0000 mgr.y (mgr.14901) 87 : cluster [DBG] pgmap v77: 89 pgs: 89 active+clean; 804 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 880 op/s 2023-08-27T22:01:26.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:26 smithi130 bash[27579]: audit 2023-08-27T22:01:25.631753+0000 mon.c (mon.2) 78 : audit [DBG] from='client.? 172.21.15.6:0/109164142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:26.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:25 smithi006 bash[27292]: cluster 2023-08-27T22:01:24.933783+0000 mgr.y (mgr.14901) 87 : cluster [DBG] pgmap v77: 89 pgs: 89 active+clean; 804 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 880 op/s 2023-08-27T22:01:26.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:25 smithi006 bash[27292]: audit 2023-08-27T22:01:25.631753+0000 mon.c (mon.2) 78 : audit [DBG] from='client.? 172.21.15.6:0/109164142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:26.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:25 smithi006 bash[33918]: cluster 2023-08-27T22:01:24.933783+0000 mgr.y (mgr.14901) 87 : cluster [DBG] pgmap v77: 89 pgs: 89 active+clean; 804 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 880 op/s 2023-08-27T22:01:26.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:25 smithi006 bash[33918]: audit 2023-08-27T22:01:25.631753+0000 mon.c (mon.2) 78 : audit [DBG] from='client.? 172.21.15.6:0/109164142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:28.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:28 smithi130 bash[27579]: cluster 2023-08-27T22:01:26.934276+0000 mgr.y (mgr.14901) 88 : cluster [DBG] pgmap v78: 89 pgs: 89 active+clean; 804 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-08-27T22:01:28.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:28 smithi006 bash[33918]: cluster 2023-08-27T22:01:26.934276+0000 mgr.y (mgr.14901) 88 : cluster [DBG] pgmap v78: 89 pgs: 89 active+clean; 804 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-08-27T22:01:28.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:28 smithi006 bash[27292]: cluster 2023-08-27T22:01:26.934276+0000 mgr.y (mgr.14901) 88 : cluster [DBG] pgmap v78: 89 pgs: 89 active+clean; 804 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-08-27T22:01:29.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:29 smithi130 bash[27579]: audit 2023-08-27T22:01:28.286343+0000 mon.a (mon.0) 872 : audit [DBG] from='client.? 172.21.15.6:0/3453166549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:29.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:29 smithi006 bash[27292]: audit 2023-08-27T22:01:28.286343+0000 mon.a (mon.0) 872 : audit [DBG] from='client.? 172.21.15.6:0/3453166549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:29.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:29 smithi006 bash[33918]: audit 2023-08-27T22:01:28.286343+0000 mon.a (mon.0) 872 : audit [DBG] from='client.? 172.21.15.6:0/3453166549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:30.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:30 smithi130 bash[27579]: cluster 2023-08-27T22:01:28.934707+0000 mgr.y (mgr.14901) 89 : cluster [DBG] pgmap v79: 89 pgs: 89 active+clean; 804 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-08-27T22:01:30.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:30 smithi006 bash[27292]: cluster 2023-08-27T22:01:28.934707+0000 mgr.y (mgr.14901) 89 : cluster [DBG] pgmap v79: 89 pgs: 89 active+clean; 804 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-08-27T22:01:30.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:30 smithi006 bash[33918]: cluster 2023-08-27T22:01:28.934707+0000 mgr.y (mgr.14901) 89 : cluster [DBG] pgmap v79: 89 pgs: 89 active+clean; 804 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-08-27T22:01:31.032 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:01:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:01:30] "GET /metrics HTTP/1.1" 200 194839 "" "Prometheus/2.33.4" 2023-08-27T22:01:31.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:31 smithi130 bash[27579]: audit 2023-08-27T22:01:30.926751+0000 mon.c (mon.2) 79 : audit [DBG] from='client.? 172.21.15.6:0/383920186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:31.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:31 smithi006 bash[33918]: audit 2023-08-27T22:01:30.926751+0000 mon.c (mon.2) 79 : audit [DBG] from='client.? 172.21.15.6:0/383920186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:31.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:31 smithi006 bash[27292]: audit 2023-08-27T22:01:30.926751+0000 mon.c (mon.2) 79 : audit [DBG] from='client.? 172.21.15.6:0/383920186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:31.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:01:31.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:31.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:01:31.631Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:32.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:32 smithi130 bash[27579]: cluster 2023-08-27T22:01:30.935491+0000 mgr.y (mgr.14901) 90 : cluster [DBG] pgmap v80: 89 pgs: 89 active+clean; 585 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2023-08-27T22:01:32.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:32 smithi006 bash[33918]: cluster 2023-08-27T22:01:30.935491+0000 mgr.y (mgr.14901) 90 : cluster [DBG] pgmap v80: 89 pgs: 89 active+clean; 585 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2023-08-27T22:01:32.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:32 smithi006 bash[27292]: cluster 2023-08-27T22:01:30.935491+0000 mgr.y (mgr.14901) 90 : cluster [DBG] pgmap v80: 89 pgs: 89 active+clean; 585 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2023-08-27T22:01:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:01:33.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:01:33.525Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:34.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:34 smithi130 bash[27579]: cluster 2023-08-27T22:01:32.936081+0000 mgr.y (mgr.14901) 91 : cluster [DBG] pgmap v81: 89 pgs: 89 active+clean; 585 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-08-27T22:01:34.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:34 smithi130 bash[27579]: audit 2023-08-27T22:01:33.595862+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.6:0/462759169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:34.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:34 smithi006 bash[33918]: cluster 2023-08-27T22:01:32.936081+0000 mgr.y (mgr.14901) 91 : cluster [DBG] pgmap v81: 89 pgs: 89 active+clean; 585 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-08-27T22:01:34.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:34 smithi006 bash[33918]: audit 2023-08-27T22:01:33.595862+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.6:0/462759169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:34.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:34 smithi006 bash[27292]: cluster 2023-08-27T22:01:32.936081+0000 mgr.y (mgr.14901) 91 : cluster [DBG] pgmap v81: 89 pgs: 89 active+clean; 585 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-08-27T22:01:34.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:34 smithi006 bash[27292]: audit 2023-08-27T22:01:33.595862+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.6:0/462759169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:36.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:36 smithi130 bash[27579]: cluster 2023-08-27T22:01:34.936816+0000 mgr.y (mgr.14901) 92 : cluster [DBG] pgmap v82: 89 pgs: 89 active+clean; 368 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 871 op/s 2023-08-27T22:01:36.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:36 smithi006 bash[33918]: cluster 2023-08-27T22:01:34.936816+0000 mgr.y (mgr.14901) 92 : cluster [DBG] pgmap v82: 89 pgs: 89 active+clean; 368 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 871 op/s 2023-08-27T22:01:36.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:36 smithi006 bash[27292]: cluster 2023-08-27T22:01:34.936816+0000 mgr.y (mgr.14901) 92 : cluster [DBG] pgmap v82: 89 pgs: 89 active+clean; 368 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 871 op/s 2023-08-27T22:01:37.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:37 smithi130 bash[27579]: audit 2023-08-27T22:01:36.275378+0000 mon.c (mon.2) 81 : audit [DBG] from='client.? 172.21.15.6:0/2204604539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:37.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:37 smithi006 bash[27292]: audit 2023-08-27T22:01:36.275378+0000 mon.c (mon.2) 81 : audit [DBG] from='client.? 172.21.15.6:0/2204604539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:37.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:37 smithi006 bash[33918]: audit 2023-08-27T22:01:36.275378+0000 mon.c (mon.2) 81 : audit [DBG] from='client.? 172.21.15.6:0/2204604539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:38.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:38 smithi130 bash[27579]: cluster 2023-08-27T22:01:36.937272+0000 mgr.y (mgr.14901) 93 : cluster [DBG] pgmap v83: 89 pgs: 89 active+clean; 368 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-08-27T22:01:38.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:38 smithi130 bash[27579]: audit 2023-08-27T22:01:37.994576+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:01:38.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:38 smithi006 bash[27292]: cluster 2023-08-27T22:01:36.937272+0000 mgr.y (mgr.14901) 93 : cluster [DBG] pgmap v83: 89 pgs: 89 active+clean; 368 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-08-27T22:01:38.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:38 smithi006 bash[27292]: audit 2023-08-27T22:01:37.994576+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:01:38.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:38 smithi006 bash[33918]: cluster 2023-08-27T22:01:36.937272+0000 mgr.y (mgr.14901) 93 : cluster [DBG] pgmap v83: 89 pgs: 89 active+clean; 368 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-08-27T22:01:38.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:38 smithi006 bash[33918]: audit 2023-08-27T22:01:37.994576+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:01:39.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:39 smithi130 bash[27579]: audit 2023-08-27T22:01:38.902262+0000 mon.c (mon.2) 82 : audit [DBG] from='client.? 172.21.15.6:0/158026209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:39.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:39 smithi006 bash[27292]: audit 2023-08-27T22:01:38.902262+0000 mon.c (mon.2) 82 : audit [DBG] from='client.? 172.21.15.6:0/158026209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:39.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:39 smithi006 bash[33918]: audit 2023-08-27T22:01:38.902262+0000 mon.c (mon.2) 82 : audit [DBG] from='client.? 172.21.15.6:0/158026209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:40.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:40 smithi130 bash[27579]: cluster 2023-08-27T22:01:38.937806+0000 mgr.y (mgr.14901) 94 : cluster [DBG] pgmap v84: 89 pgs: 89 active+clean; 368 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-08-27T22:01:40.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:40 smithi006 bash[33918]: cluster 2023-08-27T22:01:38.937806+0000 mgr.y (mgr.14901) 94 : cluster [DBG] pgmap v84: 89 pgs: 89 active+clean; 368 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-08-27T22:01:40.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:40 smithi006 bash[27292]: cluster 2023-08-27T22:01:38.937806+0000 mgr.y (mgr.14901) 94 : cluster [DBG] pgmap v84: 89 pgs: 89 active+clean; 368 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-08-27T22:01:41.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:01:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:01:40] "GET /metrics HTTP/1.1" 200 194839 "" "Prometheus/2.33.4" 2023-08-27T22:01:41.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:01:41.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:41.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:01:41.641Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:42.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:42 smithi130 bash[27579]: cluster 2023-08-27T22:01:40.938534+0000 mgr.y (mgr.14901) 95 : cluster [DBG] pgmap v85: 89 pgs: 89 active+clean; 148 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2023-08-27T22:01:42.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:42 smithi130 bash[27579]: audit 2023-08-27T22:01:41.575686+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.6:0/1521496944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:42.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:42 smithi006 bash[27292]: cluster 2023-08-27T22:01:40.938534+0000 mgr.y (mgr.14901) 95 : cluster [DBG] pgmap v85: 89 pgs: 89 active+clean; 148 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2023-08-27T22:01:42.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:42 smithi006 bash[27292]: audit 2023-08-27T22:01:41.575686+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.6:0/1521496944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:42.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:42 smithi006 bash[33918]: cluster 2023-08-27T22:01:40.938534+0000 mgr.y (mgr.14901) 95 : cluster [DBG] pgmap v85: 89 pgs: 89 active+clean; 148 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2023-08-27T22:01:42.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:42 smithi006 bash[33918]: audit 2023-08-27T22:01:41.575686+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.6:0/1521496944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:42.721 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Removed 113276 objects 2023-08-27T22:01:42.721 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Clean up completed and total clean up time :154.808 2023-08-27T22:01:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:01:43.525Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:01:43.527Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:44.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:44 smithi130 bash[27579]: cluster 2023-08-27T22:01:42.939096+0000 mgr.y (mgr.14901) 96 : cluster [DBG] pgmap v86: 89 pgs: 89 active+clean; 148 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-08-27T22:01:44.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:44 smithi130 bash[27579]: audit 2023-08-27T22:01:43.909278+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:01:44.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:44 smithi130 bash[27579]: audit 2023-08-27T22:01:44.027327+0000 mon.c (mon.2) 84 : audit [DBG] from='client.? 172.21.15.6:0/2342006758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:44.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:44 smithi006 bash[27292]: cluster 2023-08-27T22:01:42.939096+0000 mgr.y (mgr.14901) 96 : cluster [DBG] pgmap v86: 89 pgs: 89 active+clean; 148 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-08-27T22:01:44.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:44 smithi006 bash[27292]: audit 2023-08-27T22:01:43.909278+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:01:44.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:44 smithi006 bash[27292]: audit 2023-08-27T22:01:44.027327+0000 mon.c (mon.2) 84 : audit [DBG] from='client.? 172.21.15.6:0/2342006758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:44.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:44 smithi006 bash[33918]: cluster 2023-08-27T22:01:42.939096+0000 mgr.y (mgr.14901) 96 : cluster [DBG] pgmap v86: 89 pgs: 89 active+clean; 148 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-08-27T22:01:44.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:44 smithi006 bash[33918]: audit 2023-08-27T22:01:43.909278+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:01:44.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:44 smithi006 bash[33918]: audit 2023-08-27T22:01:44.027327+0000 mon.c (mon.2) 84 : audit [DBG] from='client.? 172.21.15.6:0/2342006758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:45.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:45 smithi130 bash[27579]: audit 2023-08-27T22:01:44.309950+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:01:45.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:45 smithi130 bash[27579]: audit 2023-08-27T22:01:44.614906+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:01:45.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:45 smithi130 bash[27579]: audit 2023-08-27T22:01:44.617109+0000 mon.b (mon.1) 84 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:01:45.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:45 smithi130 bash[27579]: audit 2023-08-27T22:01:44.623902+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:01:45.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:45 smithi006 bash[27292]: audit 2023-08-27T22:01:44.309950+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:01:45.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:45 smithi006 bash[27292]: audit 2023-08-27T22:01:44.614906+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:01:45.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:45 smithi006 bash[27292]: audit 2023-08-27T22:01:44.617109+0000 mon.b (mon.1) 84 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:01:45.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:45 smithi006 bash[27292]: audit 2023-08-27T22:01:44.623902+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:01:45.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:45 smithi006 bash[33918]: audit 2023-08-27T22:01:44.309950+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:01:45.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:45 smithi006 bash[33918]: audit 2023-08-27T22:01:44.614906+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:01:45.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:45 smithi006 bash[33918]: audit 2023-08-27T22:01:44.617109+0000 mon.b (mon.1) 84 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:01:45.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:45 smithi006 bash[33918]: audit 2023-08-27T22:01:44.623902+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:01:46.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:46 smithi006 bash[27292]: cluster 2023-08-27T22:01:44.940140+0000 mgr.y (mgr.14901) 97 : cluster [DBG] pgmap v87: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2023-08-27T22:01:46.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:46 smithi006 bash[27292]: audit 2023-08-27T22:01:46.441917+0000 mon.a (mon.0) 876 : audit [DBG] from='client.? 172.21.15.6:0/1655197040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:46.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:46 smithi006 bash[33918]: cluster 2023-08-27T22:01:44.940140+0000 mgr.y (mgr.14901) 97 : cluster [DBG] pgmap v87: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2023-08-27T22:01:46.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:46 smithi006 bash[33918]: audit 2023-08-27T22:01:46.441917+0000 mon.a (mon.0) 876 : audit [DBG] from='client.? 172.21.15.6:0/1655197040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:47.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:46 smithi130 bash[27579]: cluster 2023-08-27T22:01:44.940140+0000 mgr.y (mgr.14901) 97 : cluster [DBG] pgmap v87: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2023-08-27T22:01:47.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:46 smithi130 bash[27579]: audit 2023-08-27T22:01:46.441917+0000 mon.a (mon.0) 876 : audit [DBG] from='client.? 172.21.15.6:0/1655197040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:48.002 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-08-27T22:01:48.003 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-08-27T22:01:48.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:48 smithi006 bash[27292]: cluster 2023-08-27T22:01:46.940817+0000 mgr.y (mgr.14901) 98 : cluster [DBG] pgmap v88: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 490 op/s 2023-08-27T22:01:48.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:48 smithi006 bash[33918]: cluster 2023-08-27T22:01:46.940817+0000 mgr.y (mgr.14901) 98 : cluster [DBG] pgmap v88: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 490 op/s 2023-08-27T22:01:49.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:48 smithi130 bash[27579]: cluster 2023-08-27T22:01:46.940817+0000 mgr.y (mgr.14901) 98 : cluster [DBG] pgmap v88: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 490 op/s 2023-08-27T22:01:49.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:49 smithi006 bash[27292]: audit 2023-08-27T22:01:48.864054+0000 mon.c (mon.2) 85 : audit [DBG] from='client.? 172.21.15.6:0/3576719793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:49.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:49 smithi006 bash[33918]: audit 2023-08-27T22:01:48.864054+0000 mon.c (mon.2) 85 : audit [DBG] from='client.? 172.21.15.6:0/3576719793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:50.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:49 smithi130 bash[27579]: audit 2023-08-27T22:01:48.864054+0000 mon.c (mon.2) 85 : audit [DBG] from='client.? 172.21.15.6:0/3576719793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:50.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:50 smithi006 bash[33918]: cluster 2023-08-27T22:01:48.941470+0000 mgr.y (mgr.14901) 99 : cluster [DBG] pgmap v89: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 490 op/s 2023-08-27T22:01:50.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:50 smithi006 bash[27292]: cluster 2023-08-27T22:01:48.941470+0000 mgr.y (mgr.14901) 99 : cluster [DBG] pgmap v89: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 490 op/s 2023-08-27T22:01:50.973 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:01:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:01:50] "GET /metrics HTTP/1.1" 200 194837 "" "Prometheus/2.33.4" 2023-08-27T22:01:51.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:50 smithi130 bash[27579]: cluster 2023-08-27T22:01:48.941470+0000 mgr.y (mgr.14901) 99 : cluster [DBG] pgmap v89: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 490 op/s 2023-08-27T22:01:51.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:51 smithi006 bash[27292]: audit 2023-08-27T22:01:51.284723+0000 mon.c (mon.2) 86 : audit [DBG] from='client.? 172.21.15.6:0/2930425904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:51.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:51 smithi006 bash[33918]: audit 2023-08-27T22:01:51.284723+0000 mon.c (mon.2) 86 : audit [DBG] from='client.? 172.21.15.6:0/2930425904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:51.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:01:51.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:51.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:01:51.634Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:52.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:51 smithi130 bash[27579]: audit 2023-08-27T22:01:51.284723+0000 mon.c (mon.2) 86 : audit [DBG] from='client.? 172.21.15.6:0/2930425904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:52.830 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T22:01:53.114 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:52 smithi006 bash[27292]: cluster 2023-08-27T22:01:50.942170+0000 mgr.y (mgr.14901) 100 : cluster [DBG] pgmap v90: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 490 op/s 2023-08-27T22:01:53.114 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:52 smithi006 bash[33918]: cluster 2023-08-27T22:01:50.942170+0000 mgr.y (mgr.14901) 100 : cluster [DBG] pgmap v90: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 490 op/s 2023-08-27T22:01:53.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:52 smithi130 bash[27579]: cluster 2023-08-27T22:01:50.942170+0000 mgr.y (mgr.14901) 100 : cluster [DBG] pgmap v90: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 490 op/s 2023-08-27T22:01:53.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:01:53.525Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:53.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:01:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:01:53.527Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:01:54.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:54 smithi006 bash[27292]: cluster 2023-08-27T22:01:52.942818+0000 mgr.y (mgr.14901) 101 : cluster [DBG] pgmap v91: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 196 op/s 2023-08-27T22:01:54.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:54 smithi006 bash[27292]: audit 2023-08-27T22:01:53.688714+0000 mon.a (mon.0) 877 : audit [DBG] from='client.? 172.21.15.6:0/1055478087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:54.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:54 smithi006 bash[33918]: cluster 2023-08-27T22:01:52.942818+0000 mgr.y (mgr.14901) 101 : cluster [DBG] pgmap v91: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 196 op/s 2023-08-27T22:01:54.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:54 smithi006 bash[33918]: audit 2023-08-27T22:01:53.688714+0000 mon.a (mon.0) 877 : audit [DBG] from='client.? 172.21.15.6:0/1055478087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:55.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:54 smithi130 bash[27579]: cluster 2023-08-27T22:01:52.942818+0000 mgr.y (mgr.14901) 101 : cluster [DBG] pgmap v91: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 196 op/s 2023-08-27T22:01:55.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:54 smithi130 bash[27579]: audit 2023-08-27T22:01:53.688714+0000 mon.a (mon.0) 877 : audit [DBG] from='client.? 172.21.15.6:0/1055478087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:56.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:56 smithi006 bash[27292]: cluster 2023-08-27T22:01:54.944081+0000 mgr.y (mgr.14901) 102 : cluster [DBG] pgmap v92: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 196 op/s 2023-08-27T22:01:56.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:56 smithi006 bash[27292]: audit 2023-08-27T22:01:56.096650+0000 mon.c (mon.2) 87 : audit [DBG] from='client.? 172.21.15.6:0/1677416205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:56.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:56 smithi006 bash[33918]: cluster 2023-08-27T22:01:54.944081+0000 mgr.y (mgr.14901) 102 : cluster [DBG] pgmap v92: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 196 op/s 2023-08-27T22:01:56.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:56 smithi006 bash[33918]: audit 2023-08-27T22:01:56.096650+0000 mon.c (mon.2) 87 : audit [DBG] from='client.? 172.21.15.6:0/1677416205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:57.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:56 smithi130 bash[27579]: cluster 2023-08-27T22:01:54.944081+0000 mgr.y (mgr.14901) 102 : cluster [DBG] pgmap v92: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 196 op/s 2023-08-27T22:01:57.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:56 smithi130 bash[27579]: audit 2023-08-27T22:01:56.096650+0000 mon.c (mon.2) 87 : audit [DBG] from='client.? 172.21.15.6:0/1677416205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:58.694 INFO:teuthology.orchestra.run.smithi006.stderr:pool 'unique_pool_0' removed 2023-08-27T22:01:58.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:58 smithi006 bash[33918]: cluster 2023-08-27T22:01:56.945065+0000 mgr.y (mgr.14901) 103 : cluster [DBG] pgmap v93: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:01:58.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:58 smithi006 bash[33918]: audit 2023-08-27T22:01:57.939179+0000 mon.a (mon.0) 878 : audit [INF] from='client.? 172.21.15.6:0/1707178764' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-27T22:01:58.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:58 smithi006 bash[33918]: audit 2023-08-27T22:01:58.514979+0000 mon.c (mon.2) 88 : audit [DBG] from='client.? 172.21.15.6:0/2020214089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:58.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:58 smithi006 bash[27292]: cluster 2023-08-27T22:01:56.945065+0000 mgr.y (mgr.14901) 103 : cluster [DBG] pgmap v93: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:01:58.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:58 smithi006 bash[27292]: audit 2023-08-27T22:01:57.939179+0000 mon.a (mon.0) 878 : audit [INF] from='client.? 172.21.15.6:0/1707178764' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-27T22:01:58.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:58 smithi006 bash[27292]: audit 2023-08-27T22:01:58.514979+0000 mon.c (mon.2) 88 : audit [DBG] from='client.? 172.21.15.6:0/2020214089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:59.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:58 smithi130 bash[27579]: cluster 2023-08-27T22:01:56.945065+0000 mgr.y (mgr.14901) 103 : cluster [DBG] pgmap v93: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:01:59.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:58 smithi130 bash[27579]: audit 2023-08-27T22:01:57.939179+0000 mon.a (mon.0) 878 : audit [INF] from='client.? 172.21.15.6:0/1707178764' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-27T22:01:59.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:58 smithi130 bash[27579]: audit 2023-08-27T22:01:58.514979+0000 mon.c (mon.2) 88 : audit [DBG] from='client.? 172.21.15.6:0/2020214089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:59.437 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-08-27T22:01:59.438 INFO:tasks.radosbench:Beginning radosbench... 2023-08-27T22:01:59.438 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-08-27T22:01:59.438 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd pool create unique_pool_1 16 2023-08-27T22:01:59.695 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:59 smithi006 bash[27292]: audit 2023-08-27T22:01:58.670387+0000 mon.a (mon.0) 879 : audit [INF] from='client.? 172.21.15.6:0/1707178764' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-08-27T22:01:59.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:59 smithi006 bash[27292]: cluster 2023-08-27T22:01:58.670494+0000 mon.a (mon.0) 880 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-08-27T22:01:59.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:59 smithi006 bash[27292]: audit 2023-08-27T22:01:58.672044+0000 mon.a (mon.0) 881 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-08-27T22:01:59.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:59 smithi006 bash[27292]: audit 2023-08-27T22:01:58.676259+0000 mon.b (mon.1) 85 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-08-27T22:01:59.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:59 smithi006 bash[27292]: audit 2023-08-27T22:01:58.684377+0000 mon.a (mon.0) 882 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-08-27T22:01:59.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:01:59 smithi006 bash[27292]: audit 2023-08-27T22:01:58.687368+0000 mon.b (mon.1) 86 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-08-27T22:01:59.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:59 smithi006 bash[33918]: audit 2023-08-27T22:01:58.670387+0000 mon.a (mon.0) 879 : audit [INF] from='client.? 172.21.15.6:0/1707178764' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-08-27T22:01:59.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:59 smithi006 bash[33918]: cluster 2023-08-27T22:01:58.670494+0000 mon.a (mon.0) 880 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-08-27T22:01:59.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:59 smithi006 bash[33918]: audit 2023-08-27T22:01:58.672044+0000 mon.a (mon.0) 881 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-08-27T22:01:59.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:59 smithi006 bash[33918]: audit 2023-08-27T22:01:58.676259+0000 mon.b (mon.1) 85 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-08-27T22:01:59.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:59 smithi006 bash[33918]: audit 2023-08-27T22:01:58.684377+0000 mon.a (mon.0) 882 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-08-27T22:01:59.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:01:59 smithi006 bash[33918]: audit 2023-08-27T22:01:58.687368+0000 mon.b (mon.1) 86 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-08-27T22:02:00.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:59 smithi130 bash[27579]: audit 2023-08-27T22:01:58.670387+0000 mon.a (mon.0) 879 : audit [INF] from='client.? 172.21.15.6:0/1707178764' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-08-27T22:02:00.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:59 smithi130 bash[27579]: cluster 2023-08-27T22:01:58.670494+0000 mon.a (mon.0) 880 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-08-27T22:02:00.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:59 smithi130 bash[27579]: audit 2023-08-27T22:01:58.672044+0000 mon.a (mon.0) 881 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-08-27T22:02:00.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:59 smithi130 bash[27579]: audit 2023-08-27T22:01:58.676259+0000 mon.b (mon.1) 85 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-08-27T22:02:00.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:59 smithi130 bash[27579]: audit 2023-08-27T22:01:58.684377+0000 mon.a (mon.0) 882 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-08-27T22:02:00.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:01:59 smithi130 bash[27579]: audit 2023-08-27T22:01:58.687368+0000 mon.b (mon.1) 86 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-08-27T22:02:00.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:02:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:02:00] "GET /metrics HTTP/1.1" 200 194794 "" "Prometheus/2.33.4" 2023-08-27T22:02:00.974 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:00 smithi006 bash[27292]: cluster 2023-08-27T22:01:58.945705+0000 mgr.y (mgr.14901) 104 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:00.974 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:00 smithi006 bash[27292]: audit 2023-08-27T22:01:59.689546+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-08-27T22:02:00.974 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:00 smithi006 bash[27292]: cluster 2023-08-27T22:01:59.689713+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-08-27T22:02:00.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:00 smithi006 bash[33918]: cluster 2023-08-27T22:01:58.945705+0000 mgr.y (mgr.14901) 104 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:00.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:00 smithi006 bash[33918]: audit 2023-08-27T22:01:59.689546+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-08-27T22:02:00.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:00 smithi006 bash[33918]: cluster 2023-08-27T22:01:59.689713+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-08-27T22:02:01.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:00 smithi130 bash[27579]: cluster 2023-08-27T22:01:58.945705+0000 mgr.y (mgr.14901) 104 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:01.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:00 smithi130 bash[27579]: audit 2023-08-27T22:01:59.689546+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.14901 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-08-27T22:02:01.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:00 smithi130 bash[27579]: cluster 2023-08-27T22:01:59.689713+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-08-27T22:02:01.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:01 smithi006 bash[27292]: audit 2023-08-27T22:02:00.940807+0000 mon.a (mon.0) 885 : audit [DBG] from='client.? 172.21.15.6:0/329517856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:01.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:01 smithi006 bash[33918]: audit 2023-08-27T22:02:00.940807+0000 mon.a (mon.0) 885 : audit [DBG] from='client.? 172.21.15.6:0/329517856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:01.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:02:01.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:01.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:02:01.635Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:02.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:01 smithi130 bash[27579]: audit 2023-08-27T22:02:00.940807+0000 mon.a (mon.0) 885 : audit [DBG] from='client.? 172.21.15.6:0/329517856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:02.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:02 smithi006 bash[27292]: cluster 2023-08-27T22:02:00.946119+0000 mgr.y (mgr.14901) 105 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:02.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:02 smithi006 bash[33918]: cluster 2023-08-27T22:02:00.946119+0000 mgr.y (mgr.14901) 105 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:03.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:02 smithi130 bash[27579]: cluster 2023-08-27T22:02:00.946119+0000 mgr.y (mgr.14901) 105 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:02:03.526Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:03.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:02:03.528Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:04.264 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T22:02:04.304 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:03 smithi006 bash[33918]: cluster 2023-08-27T22:02:02.946699+0000 mgr.y (mgr.14901) 106 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:04.304 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:03 smithi006 bash[33918]: audit 2023-08-27T22:02:03.355179+0000 mon.c (mon.2) 89 : audit [DBG] from='client.? 172.21.15.6:0/3485900470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:04.304 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:03 smithi006 bash[27292]: cluster 2023-08-27T22:02:02.946699+0000 mgr.y (mgr.14901) 106 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:04.304 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:03 smithi006 bash[27292]: audit 2023-08-27T22:02:03.355179+0000 mon.c (mon.2) 89 : audit [DBG] from='client.? 172.21.15.6:0/3485900470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:04.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:03 smithi130 bash[27579]: cluster 2023-08-27T22:02:02.946699+0000 mgr.y (mgr.14901) 106 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:04.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:03 smithi130 bash[27579]: audit 2023-08-27T22:02:03.355179+0000 mon.c (mon.2) 89 : audit [DBG] from='client.? 172.21.15.6:0/3485900470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:06.274 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:05 smithi006 bash[33918]: cluster 2023-08-27T22:02:04.947830+0000 mgr.y (mgr.14901) 107 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:06.274 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:05 smithi006 bash[33918]: audit 2023-08-27T22:02:05.775641+0000 mon.a (mon.0) 886 : audit [DBG] from='client.? 172.21.15.6:0/794026482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:06.274 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:05 smithi006 bash[27292]: cluster 2023-08-27T22:02:04.947830+0000 mgr.y (mgr.14901) 107 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:06.274 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:05 smithi006 bash[27292]: audit 2023-08-27T22:02:05.775641+0000 mon.a (mon.0) 886 : audit [DBG] from='client.? 172.21.15.6:0/794026482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:06.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:06 smithi130 bash[27579]: cluster 2023-08-27T22:02:04.947830+0000 mgr.y (mgr.14901) 107 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:06.383 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:06 smithi130 bash[27579]: audit 2023-08-27T22:02:05.775641+0000 mon.a (mon.0) 886 : audit [DBG] from='client.? 172.21.15.6:0/794026482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:08.205 INFO:teuthology.orchestra.run.smithi006.stderr:pool 'unique_pool_1' created 2023-08-27T22:02:08.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:08 smithi006 bash[33918]: cluster 2023-08-27T22:02:06.948384+0000 mgr.y (mgr.14901) 108 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:08.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:08 smithi006 bash[33918]: audit 2023-08-27T22:02:07.664782+0000 mon.a (mon.0) 887 : audit [INF] from='client.? 172.21.15.6:0/1501279375' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-08-27T22:02:08.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:08 smithi006 bash[27292]: cluster 2023-08-27T22:02:06.948384+0000 mgr.y (mgr.14901) 108 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:08.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:08 smithi006 bash[27292]: audit 2023-08-27T22:02:07.664782+0000 mon.a (mon.0) 887 : audit [INF] from='client.? 172.21.15.6:0/1501279375' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-08-27T22:02:08.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:08 smithi130 bash[27579]: cluster 2023-08-27T22:02:06.948384+0000 mgr.y (mgr.14901) 108 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:08.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:08 smithi130 bash[27579]: audit 2023-08-27T22:02:07.664782+0000 mon.a (mon.0) 887 : audit [INF] from='client.? 172.21.15.6:0/1501279375' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-08-27T22:02:08.938 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-08-27T22:02:09.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:09 smithi006 bash[33918]: audit 2023-08-27T22:02:08.189263+0000 mon.a (mon.0) 888 : audit [INF] from='client.? 172.21.15.6:0/1501279375' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-08-27T22:02:09.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:09 smithi006 bash[33918]: cluster 2023-08-27T22:02:08.189380+0000 mon.a (mon.0) 889 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-08-27T22:02:09.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:09 smithi006 bash[33918]: audit 2023-08-27T22:02:08.195928+0000 mon.b (mon.1) 87 : audit [DBG] from='client.? 172.21.15.6:0/3638104623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:09.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:09 smithi006 bash[33918]: cluster 2023-08-27T22:02:09.192832+0000 mon.a (mon.0) 890 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-08-27T22:02:09.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:09 smithi006 bash[27292]: audit 2023-08-27T22:02:08.189263+0000 mon.a (mon.0) 888 : audit [INF] from='client.? 172.21.15.6:0/1501279375' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-08-27T22:02:09.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:09 smithi006 bash[27292]: cluster 2023-08-27T22:02:08.189380+0000 mon.a (mon.0) 889 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-08-27T22:02:09.474 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:09 smithi006 bash[27292]: audit 2023-08-27T22:02:08.195928+0000 mon.b (mon.1) 87 : audit [DBG] from='client.? 172.21.15.6:0/3638104623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:09.474 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:09 smithi006 bash[27292]: cluster 2023-08-27T22:02:09.192832+0000 mon.a (mon.0) 890 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-08-27T22:02:09.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:09 smithi130 bash[27579]: audit 2023-08-27T22:02:08.189263+0000 mon.a (mon.0) 888 : audit [INF] from='client.? 172.21.15.6:0/1501279375' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-08-27T22:02:09.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:09 smithi130 bash[27579]: cluster 2023-08-27T22:02:08.189380+0000 mon.a (mon.0) 889 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-08-27T22:02:09.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:09 smithi130 bash[27579]: audit 2023-08-27T22:02:08.195928+0000 mon.b (mon.1) 87 : audit [DBG] from='client.? 172.21.15.6:0/3638104623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:09.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:09 smithi130 bash[27579]: cluster 2023-08-27T22:02:09.192832+0000 mon.a (mon.0) 890 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-08-27T22:02:10.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:10 smithi130 bash[27579]: cluster 2023-08-27T22:02:08.949066+0000 mgr.y (mgr.14901) 109 : cluster [DBG] pgmap v102: 89 pgs: 16 unknown, 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:10.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:10 smithi130 bash[27579]: cluster 2023-08-27T22:02:10.196984+0000 mon.a (mon.0) 891 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-08-27T22:02:10.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:10 smithi006 bash[27292]: cluster 2023-08-27T22:02:08.949066+0000 mgr.y (mgr.14901) 109 : cluster [DBG] pgmap v102: 89 pgs: 16 unknown, 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:10.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:10 smithi006 bash[27292]: cluster 2023-08-27T22:02:10.196984+0000 mon.a (mon.0) 891 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-08-27T22:02:10.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:10 smithi006 bash[33918]: cluster 2023-08-27T22:02:08.949066+0000 mgr.y (mgr.14901) 109 : cluster [DBG] pgmap v102: 89 pgs: 16 unknown, 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:10.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:10 smithi006 bash[33918]: cluster 2023-08-27T22:02:10.196984+0000 mon.a (mon.0) 891 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-08-27T22:02:11.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:02:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:02:10] "GET /metrics HTTP/1.1" 200 194794 "" "Prometheus/2.33.4" 2023-08-27T22:02:11.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:11 smithi130 bash[27579]: audit 2023-08-27T22:02:10.613863+0000 mon.a (mon.0) 892 : audit [DBG] from='client.? 172.21.15.6:0/1397391469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:11.642 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:11 smithi006 bash[27292]: audit 2023-08-27T22:02:10.613863+0000 mon.a (mon.0) 892 : audit [DBG] from='client.? 172.21.15.6:0/1397391469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:11.642 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:11 smithi006 bash[33918]: audit 2023-08-27T22:02:10.613863+0000 mon.a (mon.0) 892 : audit [DBG] from='client.? 172.21.15.6:0/1397391469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:11.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:02:11.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:11.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:02:11.636Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:12.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:12 smithi130 bash[27579]: cluster 2023-08-27T22:02:10.949808+0000 mgr.y (mgr.14901) 110 : cluster [DBG] pgmap v105: 89 pgs: 6 creating+peering, 83 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:12.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:12 smithi006 bash[33918]: cluster 2023-08-27T22:02:10.949808+0000 mgr.y (mgr.14901) 110 : cluster [DBG] pgmap v105: 89 pgs: 6 creating+peering, 83 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:12.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:12 smithi006 bash[27292]: cluster 2023-08-27T22:02:10.949808+0000 mgr.y (mgr.14901) 110 : cluster [DBG] pgmap v105: 89 pgs: 6 creating+peering, 83 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:13.535 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:13 smithi006 bash[33918]: audit 2023-08-27T22:02:13.025280+0000 mon.a (mon.0) 893 : audit [DBG] from='client.? 172.21.15.6:0/2279006644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:13.536 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:13 smithi006 bash[27292]: audit 2023-08-27T22:02:13.025280+0000 mon.a (mon.0) 893 : audit [DBG] from='client.? 172.21.15.6:0/2279006644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:13.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:13 smithi130 bash[27579]: audit 2023-08-27T22:02:13.025280+0000 mon.a (mon.0) 893 : audit [DBG] from='client.? 172.21.15.6:0/2279006644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:13.774 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T22:02:13.810 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:02:13.526Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:13.810 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:02:13.529Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:14.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:14 smithi130 bash[27579]: cluster 2023-08-27T22:02:12.950443+0000 mgr.y (mgr.14901) 111 : cluster [DBG] pgmap v106: 89 pgs: 6 creating+peering, 83 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:14.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:14 smithi006 bash[33918]: cluster 2023-08-27T22:02:12.950443+0000 mgr.y (mgr.14901) 111 : cluster [DBG] pgmap v106: 89 pgs: 6 creating+peering, 83 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:14.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:14 smithi006 bash[27292]: cluster 2023-08-27T22:02:12.950443+0000 mgr.y (mgr.14901) 111 : cluster [DBG] pgmap v106: 89 pgs: 6 creating+peering, 83 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:16.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:16 smithi130 bash[27579]: cluster 2023-08-27T22:02:14.951743+0000 mgr.y (mgr.14901) 112 : cluster [DBG] pgmap v107: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:16.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:16 smithi130 bash[27579]: audit 2023-08-27T22:02:15.454075+0000 mon.c (mon.2) 90 : audit [DBG] from='client.? 172.21.15.6:0/2161385127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:16.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:16 smithi006 bash[27292]: cluster 2023-08-27T22:02:14.951743+0000 mgr.y (mgr.14901) 112 : cluster [DBG] pgmap v107: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:16.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:16 smithi006 bash[27292]: audit 2023-08-27T22:02:15.454075+0000 mon.c (mon.2) 90 : audit [DBG] from='client.? 172.21.15.6:0/2161385127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:16.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:16 smithi006 bash[33918]: cluster 2023-08-27T22:02:14.951743+0000 mgr.y (mgr.14901) 112 : cluster [DBG] pgmap v107: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:16.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:16 smithi006 bash[33918]: audit 2023-08-27T22:02:15.454075+0000 mon.c (mon.2) 90 : audit [DBG] from='client.? 172.21.15.6:0/2161385127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:18.279 INFO:teuthology.orchestra.run.smithi006.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-08-27T22:02:18.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:18 smithi130 bash[27579]: cluster 2023-08-27T22:02:16.952496+0000 mgr.y (mgr.14901) 113 : cluster [DBG] pgmap v108: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:18.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:18 smithi130 bash[27579]: audit 2023-08-27T22:02:17.459578+0000 mon.a (mon.0) 894 : audit [INF] from='client.? 172.21.15.6:0/2463029362' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T22:02:18.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:18 smithi130 bash[27579]: audit 2023-08-27T22:02:17.857849+0000 mon.c (mon.2) 91 : audit [DBG] from='client.? 172.21.15.6:0/2509632413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:18.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:18 smithi006 bash[27292]: cluster 2023-08-27T22:02:16.952496+0000 mgr.y (mgr.14901) 113 : cluster [DBG] pgmap v108: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:18.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:18 smithi006 bash[27292]: audit 2023-08-27T22:02:17.459578+0000 mon.a (mon.0) 894 : audit [INF] from='client.? 172.21.15.6:0/2463029362' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T22:02:18.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:18 smithi006 bash[27292]: audit 2023-08-27T22:02:17.857849+0000 mon.c (mon.2) 91 : audit [DBG] from='client.? 172.21.15.6:0/2509632413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:18.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:18 smithi006 bash[33918]: cluster 2023-08-27T22:02:16.952496+0000 mgr.y (mgr.14901) 113 : cluster [DBG] pgmap v108: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:18.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:18 smithi006 bash[33918]: audit 2023-08-27T22:02:17.459578+0000 mon.a (mon.0) 894 : audit [INF] from='client.? 172.21.15.6:0/2463029362' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T22:02:18.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:18 smithi006 bash[33918]: audit 2023-08-27T22:02:17.857849+0000 mon.c (mon.2) 91 : audit [DBG] from='client.? 172.21.15.6:0/2509632413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:19.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:19 smithi130 bash[27579]: audit 2023-08-27T22:02:18.266277+0000 mon.a (mon.0) 895 : audit [INF] from='client.? 172.21.15.6:0/2463029362' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-27T22:02:19.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:19 smithi130 bash[27579]: cluster 2023-08-27T22:02:18.266377+0000 mon.a (mon.0) 896 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-08-27T22:02:19.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:19 smithi130 bash[27579]: audit 2023-08-27T22:02:18.493162+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:02:19.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:19 smithi006 bash[27292]: audit 2023-08-27T22:02:18.266277+0000 mon.a (mon.0) 895 : audit [INF] from='client.? 172.21.15.6:0/2463029362' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-27T22:02:19.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:19 smithi006 bash[27292]: cluster 2023-08-27T22:02:18.266377+0000 mon.a (mon.0) 896 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-08-27T22:02:19.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:19 smithi006 bash[27292]: audit 2023-08-27T22:02:18.493162+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:02:19.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:19 smithi006 bash[33918]: audit 2023-08-27T22:02:18.266277+0000 mon.a (mon.0) 895 : audit [INF] from='client.? 172.21.15.6:0/2463029362' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-27T22:02:19.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:19 smithi006 bash[33918]: cluster 2023-08-27T22:02:18.266377+0000 mon.a (mon.0) 896 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-08-27T22:02:19.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:19 smithi006 bash[33918]: audit 2023-08-27T22:02:18.493162+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:02:20.013 DEBUG:teuthology.orchestra.run.smithi006:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-08-27T22:02:20.016 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-08-27T22:02:20.016 INFO:teuthology.orchestra.run:waiting for 3000 2023-08-27T22:02:20.056 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:hints = 1 2023-08-27T22:02:20.056 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-08-27T22:02:20.057 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Object prefix: benchmark_data_smithi006_72898 2023-08-27T22:02:20.057 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-27T22:02:20.057 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 0 0 0 0 0 0 - 0 2023-08-27T22:02:20.754 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:20 smithi006 bash[33918]: cluster 2023-08-27T22:02:18.953221+0000 mgr.y (mgr.14901) 114 : cluster [DBG] pgmap v110: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:20.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:20 smithi006 bash[27292]: cluster 2023-08-27T22:02:18.953221+0000 mgr.y (mgr.14901) 114 : cluster [DBG] pgmap v110: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:20.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:20 smithi130 bash[27579]: cluster 2023-08-27T22:02:18.953221+0000 mgr.y (mgr.14901) 114 : cluster [DBG] pgmap v110: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:21.056 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 1 16 2872 2856 178.49 178.5 0.00564935 0.00558436 2023-08-27T22:02:21.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:02:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:02:20] "GET /metrics HTTP/1.1" 200 194889 "" "Prometheus/2.33.4" 2023-08-27T22:02:21.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:21 smithi130 bash[27579]: audit 2023-08-27T22:02:20.519554+0000 mon.a (mon.0) 898 : audit [DBG] from='client.? 172.21.15.6:0/3506296403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:21.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:21 smithi006 bash[27292]: audit 2023-08-27T22:02:20.519554+0000 mon.a (mon.0) 898 : audit [DBG] from='client.? 172.21.15.6:0/3506296403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:21.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:21 smithi006 bash[33918]: audit 2023-08-27T22:02:20.519554+0000 mon.a (mon.0) 898 : audit [DBG] from='client.? 172.21.15.6:0/3506296403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:21.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:02:21.635Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:21.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:02:21.636Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:22.056 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 2 16 5723 5707 178.33 178.188 0.00636742 0.00558723 2023-08-27T22:02:22.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:22 smithi130 bash[27579]: cluster 2023-08-27T22:02:20.954060+0000 mgr.y (mgr.14901) 115 : cluster [DBG] pgmap v111: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:22.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:22 smithi006 bash[27292]: cluster 2023-08-27T22:02:20.954060+0000 mgr.y (mgr.14901) 115 : cluster [DBG] pgmap v111: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:22.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:22 smithi006 bash[33918]: cluster 2023-08-27T22:02:20.954060+0000 mgr.y (mgr.14901) 115 : cluster [DBG] pgmap v111: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:23.056 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 3 16 8272 8256 171.985 159.312 0.00920735 0.00580191 2023-08-27T22:02:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:02:23.527Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:23.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:02:23.529Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:24.056 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 4 16 10741 10725 167.563 154.312 0.0116417 0.00595881 2023-08-27T22:02:24.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:24 smithi006 bash[27292]: cluster 2023-08-27T22:02:22.954531+0000 mgr.y (mgr.14901) 116 : cluster [DBG] pgmap v112: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:24.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:24 smithi006 bash[27292]: audit 2023-08-27T22:02:23.270683+0000 mon.b (mon.1) 88 : audit [DBG] from='client.? 172.21.15.6:0/2394903998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:24.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:24 smithi006 bash[33918]: cluster 2023-08-27T22:02:22.954531+0000 mgr.y (mgr.14901) 116 : cluster [DBG] pgmap v112: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:24.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:24 smithi006 bash[33918]: audit 2023-08-27T22:02:23.270683+0000 mon.b (mon.1) 88 : audit [DBG] from='client.? 172.21.15.6:0/2394903998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:24.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:24 smithi130 bash[27579]: cluster 2023-08-27T22:02:22.954531+0000 mgr.y (mgr.14901) 116 : cluster [DBG] pgmap v112: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:02:24.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:24 smithi130 bash[27579]: audit 2023-08-27T22:02:23.270683+0000 mon.b (mon.1) 88 : audit [DBG] from='client.? 172.21.15.6:0/2394903998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:25.056 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 5 16 13124 13108 163.835 148.938 0.0056226 0.00609331 2023-08-27T22:02:26.057 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 6 16 15262 15246 158.798 133.625 0.00458503 0.00628945 2023-08-27T22:02:26.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:26 smithi006 bash[27292]: cluster 2023-08-27T22:02:24.955982+0000 mgr.y (mgr.14901) 117 : cluster [DBG] pgmap v113: 89 pgs: 89 active+clean; 710 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-08-27T22:02:26.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:26 smithi006 bash[27292]: audit 2023-08-27T22:02:25.960779+0000 mon.c (mon.2) 92 : audit [DBG] from='client.? 172.21.15.6:0/2915451806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:26.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:26 smithi006 bash[33918]: cluster 2023-08-27T22:02:24.955982+0000 mgr.y (mgr.14901) 117 : cluster [DBG] pgmap v113: 89 pgs: 89 active+clean; 710 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-08-27T22:02:26.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:26 smithi006 bash[33918]: audit 2023-08-27T22:02:25.960779+0000 mon.c (mon.2) 92 : audit [DBG] from='client.? 172.21.15.6:0/2915451806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:26.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:26 smithi130 bash[27579]: cluster 2023-08-27T22:02:24.955982+0000 mgr.y (mgr.14901) 117 : cluster [DBG] pgmap v113: 89 pgs: 89 active+clean; 710 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-08-27T22:02:26.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:26 smithi130 bash[27579]: audit 2023-08-27T22:02:25.960779+0000 mon.c (mon.2) 92 : audit [DBG] from='client.? 172.21.15.6:0/2915451806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:27.057 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 7 16 17504 17488 156.128 140.125 0.0075025 0.00639704 2023-08-27T22:02:28.057 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 8 16 19620 19604 153.142 132.25 0.00988542 0.00652232 2023-08-27T22:02:28.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:28 smithi006 bash[27292]: cluster 2023-08-27T22:02:26.956492+0000 mgr.y (mgr.14901) 118 : cluster [DBG] pgmap v114: 89 pgs: 89 active+clean; 710 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-08-27T22:02:28.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:28 smithi006 bash[33918]: cluster 2023-08-27T22:02:26.956492+0000 mgr.y (mgr.14901) 118 : cluster [DBG] pgmap v114: 89 pgs: 89 active+clean; 710 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-08-27T22:02:28.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:28 smithi130 bash[27579]: cluster 2023-08-27T22:02:26.956492+0000 mgr.y (mgr.14901) 118 : cluster [DBG] pgmap v114: 89 pgs: 89 active+clean; 710 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-08-27T22:02:29.057 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 9 16 21526 21510 149.361 119.125 0.0121337 0.00668794 2023-08-27T22:02:29.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:29 smithi006 bash[27292]: audit 2023-08-27T22:02:28.628888+0000 mon.b (mon.1) 89 : audit [DBG] from='client.? 172.21.15.6:0/435618594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:29.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:29 smithi006 bash[33918]: audit 2023-08-27T22:02:28.628888+0000 mon.b (mon.1) 89 : audit [DBG] from='client.? 172.21.15.6:0/435618594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:29.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:29 smithi130 bash[27579]: audit 2023-08-27T22:02:28.628888+0000 mon.b (mon.1) 89 : audit [DBG] from='client.? 172.21.15.6:0/435618594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:30.057 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 10 16 23545 23529 147.042 126.188 0.00316026 0.00679259 2023-08-27T22:02:30.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:30 smithi006 bash[27292]: cluster 2023-08-27T22:02:28.957023+0000 mgr.y (mgr.14901) 119 : cluster [DBG] pgmap v115: 89 pgs: 89 active+clean; 710 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-08-27T22:02:30.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:30 smithi006 bash[33918]: cluster 2023-08-27T22:02:28.957023+0000 mgr.y (mgr.14901) 119 : cluster [DBG] pgmap v115: 89 pgs: 89 active+clean; 710 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-08-27T22:02:30.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:30 smithi130 bash[27579]: cluster 2023-08-27T22:02:28.957023+0000 mgr.y (mgr.14901) 119 : cluster [DBG] pgmap v115: 89 pgs: 89 active+clean; 710 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-08-27T22:02:31.057 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 11 16 25491 25475 144.73 121.625 0.0168694 0.00690205 2023-08-27T22:02:31.223 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:02:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:02:30] "GET /metrics HTTP/1.1" 200 194961 "" "Prometheus/2.33.4" 2023-08-27T22:02:31.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:02:31.637Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:31.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:02:31.638Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:32.057 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 12 16 27384 27368 142.528 118.312 0.00755693 0.00700917 2023-08-27T22:02:32.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:32 smithi006 bash[33918]: cluster 2023-08-27T22:02:30.957826+0000 mgr.y (mgr.14901) 120 : cluster [DBG] pgmap v116: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-08-27T22:02:32.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:32 smithi006 bash[33918]: audit 2023-08-27T22:02:31.287485+0000 mon.a (mon.0) 899 : audit [DBG] from='client.? 172.21.15.6:0/1526819998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:32.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:32 smithi006 bash[27292]: cluster 2023-08-27T22:02:30.957826+0000 mgr.y (mgr.14901) 120 : cluster [DBG] pgmap v116: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-08-27T22:02:32.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:32 smithi006 bash[27292]: audit 2023-08-27T22:02:31.287485+0000 mon.a (mon.0) 899 : audit [DBG] from='client.? 172.21.15.6:0/1526819998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:32 smithi130 bash[27579]: cluster 2023-08-27T22:02:30.957826+0000 mgr.y (mgr.14901) 120 : cluster [DBG] pgmap v116: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-08-27T22:02:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:32 smithi130 bash[27579]: audit 2023-08-27T22:02:31.287485+0000 mon.a (mon.0) 899 : audit [DBG] from='client.? 172.21.15.6:0/1526819998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:33.057 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 13 16 29120 29104 139.91 108.5 0.00595803 0.00714111 2023-08-27T22:02:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:02:33.528Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:33.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:02:33.529Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:34.057 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 14 16 30721 30705 137.063 100.062 0.0100196 0.00728878 2023-08-27T22:02:34.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:34 smithi006 bash[27292]: cluster 2023-08-27T22:02:32.958416+0000 mgr.y (mgr.14901) 121 : cluster [DBG] pgmap v117: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-08-27T22:02:34.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:34 smithi006 bash[27292]: audit 2023-08-27T22:02:33.981619+0000 mon.b (mon.1) 90 : audit [DBG] from='client.? 172.21.15.6:0/2905168665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:34.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:34 smithi006 bash[33918]: cluster 2023-08-27T22:02:32.958416+0000 mgr.y (mgr.14901) 121 : cluster [DBG] pgmap v117: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-08-27T22:02:34.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:34 smithi006 bash[33918]: audit 2023-08-27T22:02:33.981619+0000 mon.b (mon.1) 90 : audit [DBG] from='client.? 172.21.15.6:0/2905168665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:34.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:34 smithi130 bash[27579]: cluster 2023-08-27T22:02:32.958416+0000 mgr.y (mgr.14901) 121 : cluster [DBG] pgmap v117: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-08-27T22:02:34.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:34 smithi130 bash[27579]: audit 2023-08-27T22:02:33.981619+0000 mon.b (mon.1) 90 : audit [DBG] from='client.? 172.21.15.6:0/2905168665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:35.058 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 15 16 32443 32427 135.1 107.625 0.00352798 0.00739473 2023-08-27T22:02:36.058 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 16 16 34128 34112 133.237 105.312 0.0121834 0.00749749 2023-08-27T22:02:36.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:36 smithi006 bash[33918]: cluster 2023-08-27T22:02:34.959303+0000 mgr.y (mgr.14901) 122 : cluster [DBG] pgmap v118: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 162 MiB/s wr, 2.59k op/s 2023-08-27T22:02:36.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:36 smithi006 bash[27292]: cluster 2023-08-27T22:02:34.959303+0000 mgr.y (mgr.14901) 122 : cluster [DBG] pgmap v118: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 162 MiB/s wr, 2.59k op/s 2023-08-27T22:02:36.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:36 smithi130 bash[27579]: cluster 2023-08-27T22:02:34.959303+0000 mgr.y (mgr.14901) 122 : cluster [DBG] pgmap v118: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 162 MiB/s wr, 2.59k op/s 2023-08-27T22:02:37.058 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 17 16 35646 35630 130.98 94.875 0.00807228 0.00762765 2023-08-27T22:02:37.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:37 smithi006 bash[33918]: audit 2023-08-27T22:02:36.669908+0000 mon.b (mon.1) 91 : audit [DBG] from='client.? 172.21.15.6:0/1514591160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:37.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:37 smithi006 bash[27292]: audit 2023-08-27T22:02:36.669908+0000 mon.b (mon.1) 91 : audit [DBG] from='client.? 172.21.15.6:0/1514591160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:37.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:37 smithi130 bash[27579]: audit 2023-08-27T22:02:36.669908+0000 mon.b (mon.1) 91 : audit [DBG] from='client.? 172.21.15.6:0/1514591160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:38.058 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 18 16 37248 37232 129.265 100.125 0.0102111 0.00772977 2023-08-27T22:02:38.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:38 smithi006 bash[33918]: cluster 2023-08-27T22:02:36.959801+0000 mgr.y (mgr.14901) 123 : cluster [DBG] pgmap v119: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-08-27T22:02:38.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:38 smithi006 bash[27292]: cluster 2023-08-27T22:02:36.959801+0000 mgr.y (mgr.14901) 123 : cluster [DBG] pgmap v119: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-08-27T22:02:38.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:38 smithi130 bash[27579]: cluster 2023-08-27T22:02:36.959801+0000 mgr.y (mgr.14901) 123 : cluster [DBG] pgmap v119: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-08-27T22:02:39.058 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 19 16 38748 38732 127.396 93.75 0.0183047 0.00784246 2023-08-27T22:02:40.058 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:2023-08-27T22:02:40.050795+0000 min lat: 0.00155442 max lat: 0.0658797 avg lat: 0.00794527 2023-08-27T22:02:40.058 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-27T22:02:40.059 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 20 16 40259 40243 125.747 94.4375 0.0136817 0.00794527 2023-08-27T22:02:40.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:40 smithi006 bash[27292]: cluster 2023-08-27T22:02:38.960906+0000 mgr.y (mgr.14901) 124 : cluster [DBG] pgmap v120: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-08-27T22:02:40.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:40 smithi006 bash[27292]: audit 2023-08-27T22:02:39.348288+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.6:0/901377869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:40.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:40 smithi006 bash[33918]: cluster 2023-08-27T22:02:38.960906+0000 mgr.y (mgr.14901) 124 : cluster [DBG] pgmap v120: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-08-27T22:02:40.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:40 smithi006 bash[33918]: audit 2023-08-27T22:02:39.348288+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.6:0/901377869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:40.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:40 smithi130 bash[27579]: cluster 2023-08-27T22:02:38.960906+0000 mgr.y (mgr.14901) 124 : cluster [DBG] pgmap v120: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-08-27T22:02:40.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:40 smithi130 bash[27579]: audit 2023-08-27T22:02:39.348288+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.6:0/901377869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:41.058 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 21 15 41768 41753 124.253 94.375 0.00875883 0.00804276 2023-08-27T22:02:41.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:02:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:02:40] "GET /metrics HTTP/1.1" 200 194961 "" "Prometheus/2.33.4" 2023-08-27T22:02:41.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:02:41.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:41.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:02:41.639Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:42.058 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 22 16 43179 43163 122.61 88.125 0.0134526 0.00815017 2023-08-27T22:02:42.473 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:42 smithi006 bash[56298]: level=warn ts=2023-08-27T22:02:42.311Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:42.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:42 smithi006 bash[33918]: cluster 2023-08-27T22:02:40.961738+0000 mgr.y (mgr.14901) 125 : cluster [DBG] pgmap v121: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2023-08-27T22:02:42.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:42 smithi006 bash[33918]: audit 2023-08-27T22:02:42.077364+0000 mon.a (mon.0) 900 : audit [DBG] from='client.? 172.21.15.6:0/568740734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:42.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:42 smithi006 bash[27292]: cluster 2023-08-27T22:02:40.961738+0000 mgr.y (mgr.14901) 125 : cluster [DBG] pgmap v121: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2023-08-27T22:02:42.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:42 smithi006 bash[27292]: audit 2023-08-27T22:02:42.077364+0000 mon.a (mon.0) 900 : audit [DBG] from='client.? 172.21.15.6:0/568740734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:42.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:42 smithi130 bash[27579]: cluster 2023-08-27T22:02:40.961738+0000 mgr.y (mgr.14901) 125 : cluster [DBG] pgmap v121: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2023-08-27T22:02:42.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:42 smithi130 bash[27579]: audit 2023-08-27T22:02:42.077364+0000 mon.a (mon.0) 900 : audit [DBG] from='client.? 172.21.15.6:0/568740734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:43.058 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 23 16 44637 44621 121.241 91.125 0.0115143 0.00824191 2023-08-27T22:02:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:02:43.530Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:43.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:02:43.532Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:44.059 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 24 16 46067 46051 119.913 89.375 0.0128338 0.00833262 2023-08-27T22:02:44.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:44 smithi006 bash[33918]: cluster 2023-08-27T22:02:42.962212+0000 mgr.y (mgr.14901) 126 : cluster [DBG] pgmap v122: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-08-27T22:02:44.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:44 smithi006 bash[27292]: cluster 2023-08-27T22:02:42.962212+0000 mgr.y (mgr.14901) 126 : cluster [DBG] pgmap v122: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-08-27T22:02:44.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:44 smithi130 bash[27579]: cluster 2023-08-27T22:02:42.962212+0000 mgr.y (mgr.14901) 126 : cluster [DBG] pgmap v122: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-08-27T22:02:45.059 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 25 16 47319 47303 118.246 78.25 0.00935283 0.0084503 2023-08-27T22:02:45.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:45 smithi006 bash[33918]: audit 2023-08-27T22:02:44.639185+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:02:45.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:45 smithi006 bash[33918]: audit 2023-08-27T22:02:44.736998+0000 mon.b (mon.1) 93 : audit [DBG] from='client.? 172.21.15.6:0/2397053110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:45.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:45 smithi006 bash[27292]: audit 2023-08-27T22:02:44.639185+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:02:45.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:45 smithi006 bash[27292]: audit 2023-08-27T22:02:44.736998+0000 mon.b (mon.1) 93 : audit [DBG] from='client.? 172.21.15.6:0/2397053110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:45.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:45 smithi130 bash[27579]: audit 2023-08-27T22:02:44.639185+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:02:45.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:45 smithi130 bash[27579]: audit 2023-08-27T22:02:44.736998+0000 mon.b (mon.1) 93 : audit [DBG] from='client.? 172.21.15.6:0/2397053110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:46.059 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 26 16 48706 48690 117.032 86.6875 0.0233829 0.00853871 2023-08-27T22:02:46.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:46 smithi130 bash[27579]: cluster 2023-08-27T22:02:44.962966+0000 mgr.y (mgr.14901) 127 : cluster [DBG] pgmap v123: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-08-27T22:02:46.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:46 smithi006 bash[27292]: cluster 2023-08-27T22:02:44.962966+0000 mgr.y (mgr.14901) 127 : cluster [DBG] pgmap v123: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-08-27T22:02:46.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:46 smithi006 bash[33918]: cluster 2023-08-27T22:02:44.962966+0000 mgr.y (mgr.14901) 127 : cluster [DBG] pgmap v123: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-08-27T22:02:47.059 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 27 16 50027 50011 115.755 82.5625 0.0119844 0.00863076 2023-08-27T22:02:48.059 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 28 16 51287 51271 114.433 78.75 0.011279 0.00873198 2023-08-27T22:02:48.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:48 smithi130 bash[27579]: cluster 2023-08-27T22:02:46.965155+0000 mgr.y (mgr.14901) 128 : cluster [DBG] pgmap v124: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-08-27T22:02:48.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:48 smithi130 bash[27579]: audit 2023-08-27T22:02:47.408655+0000 mon.a (mon.0) 901 : audit [DBG] from='client.? 172.21.15.6:0/4121429519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:48.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:48 smithi006 bash[33918]: cluster 2023-08-27T22:02:46.965155+0000 mgr.y (mgr.14901) 128 : cluster [DBG] pgmap v124: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-08-27T22:02:48.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:48 smithi006 bash[33918]: audit 2023-08-27T22:02:47.408655+0000 mon.a (mon.0) 901 : audit [DBG] from='client.? 172.21.15.6:0/4121429519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:48.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:48 smithi006 bash[27292]: cluster 2023-08-27T22:02:46.965155+0000 mgr.y (mgr.14901) 128 : cluster [DBG] pgmap v124: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-08-27T22:02:48.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:48 smithi006 bash[27292]: audit 2023-08-27T22:02:47.408655+0000 mon.a (mon.0) 901 : audit [DBG] from='client.? 172.21.15.6:0/4121429519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:49.059 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 29 15 52482 52467 113.064 74.75 0.00742305 0.00883904 2023-08-27T22:02:50.064 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 30 16 53588 53572 111.596 69.0625 0.0373668 0.00895425 2023-08-27T22:02:50.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:50 smithi130 bash[27579]: cluster 2023-08-27T22:02:48.965610+0000 mgr.y (mgr.14901) 129 : cluster [DBG] pgmap v125: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-08-27T22:02:50.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:50 smithi130 bash[27579]: audit 2023-08-27T22:02:50.098803+0000 mon.c (mon.2) 94 : audit [DBG] from='client.? 172.21.15.6:0/3418365139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:50.659 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:50 smithi006 bash[27292]: cluster 2023-08-27T22:02:48.965610+0000 mgr.y (mgr.14901) 129 : cluster [DBG] pgmap v125: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-08-27T22:02:50.660 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:50 smithi006 bash[27292]: audit 2023-08-27T22:02:50.098803+0000 mon.c (mon.2) 94 : audit [DBG] from='client.? 172.21.15.6:0/3418365139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:50.660 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:50 smithi006 bash[33918]: cluster 2023-08-27T22:02:48.965610+0000 mgr.y (mgr.14901) 129 : cluster [DBG] pgmap v125: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-08-27T22:02:50.660 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:50 smithi006 bash[33918]: audit 2023-08-27T22:02:50.098803+0000 mon.c (mon.2) 94 : audit [DBG] from='client.? 172.21.15.6:0/3418365139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:50.973 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:02:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:02:50] "GET /metrics HTTP/1.1" 200 194988 "" "Prometheus/2.33.4" 2023-08-27T22:02:51.060 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 31 16 54580 54564 109.996 62 0.00720569 0.00908466 2023-08-27T22:02:51.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:51 smithi006 bash[27292]: audit 2023-08-27T22:02:50.834760+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:02:51.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:51 smithi006 bash[27292]: cluster 2023-08-27T22:02:50.966399+0000 mgr.y (mgr.14901) 130 : cluster [DBG] pgmap v126: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-08-27T22:02:51.974 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:51 smithi006 bash[27292]: audit 2023-08-27T22:02:51.047758+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:02:51.974 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:51 smithi006 bash[27292]: audit 2023-08-27T22:02:51.392593+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:02:51.974 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:51 smithi006 bash[27292]: audit 2023-08-27T22:02:51.394136+0000 mon.b (mon.1) 95 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:02:51.974 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:51 smithi006 bash[27292]: audit 2023-08-27T22:02:51.396176+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:02:51.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:51 smithi006 bash[33918]: audit 2023-08-27T22:02:50.834760+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:02:51.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:51 smithi006 bash[33918]: cluster 2023-08-27T22:02:50.966399+0000 mgr.y (mgr.14901) 130 : cluster [DBG] pgmap v126: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-08-27T22:02:51.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:51 smithi006 bash[33918]: audit 2023-08-27T22:02:51.047758+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:02:51.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:51 smithi006 bash[33918]: audit 2023-08-27T22:02:51.392593+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:02:51.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:51 smithi006 bash[33918]: audit 2023-08-27T22:02:51.394136+0000 mon.b (mon.1) 95 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:02:51.975 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:51 smithi006 bash[33918]: audit 2023-08-27T22:02:51.396176+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:02:51.975 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:02:51.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:51.975 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:02:51.639Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:52.060 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 32 16 55825 55809 108.99 77.8125 0.00630436 0.0091697 2023-08-27T22:02:52.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:51 smithi130 bash[27579]: audit 2023-08-27T22:02:50.834760+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:02:52.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:51 smithi130 bash[27579]: cluster 2023-08-27T22:02:50.966399+0000 mgr.y (mgr.14901) 130 : cluster [DBG] pgmap v126: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-08-27T22:02:52.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:51 smithi130 bash[27579]: audit 2023-08-27T22:02:51.047758+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:02:52.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:51 smithi130 bash[27579]: audit 2023-08-27T22:02:51.392593+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:02:52.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:51 smithi130 bash[27579]: audit 2023-08-27T22:02:51.394136+0000 mon.b (mon.1) 95 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:02:52.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:51 smithi130 bash[27579]: audit 2023-08-27T22:02:51.396176+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:02:52.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:52 smithi006 bash[56298]: level=error ts=2023-08-27T22:02:52.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:52.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:52 smithi006 bash[56298]: level=warn ts=2023-08-27T22:02:52.312Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:53.060 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 33 16 56976 56960 107.867 71.9375 0.0090382 0.00926337 2023-08-27T22:02:53.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:52 smithi130 bash[27579]: audit 2023-08-27T22:02:52.799435+0000 mon.b (mon.1) 96 : audit [DBG] from='client.? 172.21.15.6:0/2694831182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:53.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:52 smithi006 bash[33918]: audit 2023-08-27T22:02:52.799435+0000 mon.b (mon.1) 96 : audit [DBG] from='client.? 172.21.15.6:0/2694831182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:53.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:52 smithi006 bash[27292]: audit 2023-08-27T22:02:52.799435+0000 mon.b (mon.1) 96 : audit [DBG] from='client.? 172.21.15.6:0/2694831182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:53.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:02:53.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:53.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:02:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:02:53.533Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:02:54.060 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 34 16 58163 58147 106.876 74.1875 0.0174842 0.00935064 2023-08-27T22:02:54.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:54 smithi006 bash[33918]: cluster 2023-08-27T22:02:52.966859+0000 mgr.y (mgr.14901) 131 : cluster [DBG] pgmap v127: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-08-27T22:02:54.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:54 smithi006 bash[27292]: cluster 2023-08-27T22:02:52.966859+0000 mgr.y (mgr.14901) 131 : cluster [DBG] pgmap v127: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-08-27T22:02:54.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:54 smithi130 bash[27579]: cluster 2023-08-27T22:02:52.966859+0000 mgr.y (mgr.14901) 131 : cluster [DBG] pgmap v127: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-08-27T22:02:55.060 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 35 16 59340 59324 105.924 73.5625 0.0153982 0.0094337 2023-08-27T22:02:56.061 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 36 16 60500 60484 104.995 72.5 0.00814369 0.00951765 2023-08-27T22:02:56.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:56 smithi006 bash[33918]: cluster 2023-08-27T22:02:54.967642+0000 mgr.y (mgr.14901) 132 : cluster [DBG] pgmap v128: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-08-27T22:02:56.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:56 smithi006 bash[33918]: audit 2023-08-27T22:02:55.493609+0000 mon.b (mon.1) 97 : audit [DBG] from='client.? 172.21.15.6:0/4006254847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:56.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:56 smithi006 bash[27292]: cluster 2023-08-27T22:02:54.967642+0000 mgr.y (mgr.14901) 132 : cluster [DBG] pgmap v128: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-08-27T22:02:56.474 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:56 smithi006 bash[27292]: audit 2023-08-27T22:02:55.493609+0000 mon.b (mon.1) 97 : audit [DBG] from='client.? 172.21.15.6:0/4006254847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:56.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:56 smithi130 bash[27579]: cluster 2023-08-27T22:02:54.967642+0000 mgr.y (mgr.14901) 132 : cluster [DBG] pgmap v128: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-08-27T22:02:56.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:56 smithi130 bash[27579]: audit 2023-08-27T22:02:55.493609+0000 mon.b (mon.1) 97 : audit [DBG] from='client.? 172.21.15.6:0/4006254847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:57.061 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 37 16 61712 61696 104.205 75.75 0.0107691 0.0095909 2023-08-27T22:02:58.061 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 38 16 62811 62795 103.27 68.6875 0.0175377 0.00967658 2023-08-27T22:02:58.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:58 smithi006 bash[33918]: cluster 2023-08-27T22:02:56.968075+0000 mgr.y (mgr.14901) 133 : cluster [DBG] pgmap v129: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-08-27T22:02:58.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:02:58 smithi006 bash[33918]: audit 2023-08-27T22:02:58.143666+0000 mon.a (mon.0) 905 : audit [DBG] from='client.? 172.21.15.6:0/4038189349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:58.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:58 smithi006 bash[27292]: cluster 2023-08-27T22:02:56.968075+0000 mgr.y (mgr.14901) 133 : cluster [DBG] pgmap v129: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-08-27T22:02:58.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:02:58 smithi006 bash[27292]: audit 2023-08-27T22:02:58.143666+0000 mon.a (mon.0) 905 : audit [DBG] from='client.? 172.21.15.6:0/4038189349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:58.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:58 smithi130 bash[27579]: cluster 2023-08-27T22:02:56.968075+0000 mgr.y (mgr.14901) 133 : cluster [DBG] pgmap v129: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-08-27T22:02:58.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:02:58 smithi130 bash[27579]: audit 2023-08-27T22:02:58.143666+0000 mon.a (mon.0) 905 : audit [DBG] from='client.? 172.21.15.6:0/4038189349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:59.061 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 39 16 63940 63924 102.431 70.5625 0.0251563 0.00975642 2023-08-27T22:03:00.061 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:2023-08-27T22:03:00.053222+0000 min lat: 0.00155442 max lat: 0.0695041 avg lat: 0.00983132 2023-08-27T22:03:00.061 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-27T22:03:00.062 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 40 16 65082 65066 101.655 71.375 0.0211336 0.00983132 2023-08-27T22:03:00.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:00 smithi006 bash[27292]: cluster 2023-08-27T22:02:58.968536+0000 mgr.y (mgr.14901) 134 : cluster [DBG] pgmap v130: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-08-27T22:03:00.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:00 smithi006 bash[33918]: cluster 2023-08-27T22:02:58.968536+0000 mgr.y (mgr.14901) 134 : cluster [DBG] pgmap v130: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-08-27T22:03:00.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:00 smithi130 bash[27579]: cluster 2023-08-27T22:02:58.968536+0000 mgr.y (mgr.14901) 134 : cluster [DBG] pgmap v130: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-08-27T22:03:01.061 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 41 16 66187 66171 100.859 69.0625 0.0160278 0.00990836 2023-08-27T22:03:01.199 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:03:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:03:00] "GET /metrics HTTP/1.1" 200 195004 "" "Prometheus/2.33.4" 2023-08-27T22:03:01.199 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:01 smithi006 bash[27292]: audit 2023-08-27T22:03:00.869274+0000 mon.b (mon.1) 98 : audit [DBG] from='client.? 172.21.15.6:0/1839053244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:01.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:01 smithi006 bash[33918]: audit 2023-08-27T22:03:00.869274+0000 mon.b (mon.1) 98 : audit [DBG] from='client.? 172.21.15.6:0/1839053244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:01.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:01 smithi130 bash[27579]: audit 2023-08-27T22:03:00.869274+0000 mon.b (mon.1) 98 : audit [DBG] from='client.? 172.21.15.6:0/1839053244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:01.974 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:03:01.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:01.974 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:03:01.640Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:02.061 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 42 16 67359 67343 100.202 73.25 0.00712222 0.00997368 2023-08-27T22:03:02.473 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:02 smithi006 bash[56298]: level=error ts=2023-08-27T22:03:02.311Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:02.473 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:02 smithi006 bash[56298]: level=warn ts=2023-08-27T22:03:02.313Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:02.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:02 smithi006 bash[33918]: cluster 2023-08-27T22:03:00.969391+0000 mgr.y (mgr.14901) 135 : cluster [DBG] pgmap v131: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-08-27T22:03:02.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:02 smithi006 bash[27292]: cluster 2023-08-27T22:03:00.969391+0000 mgr.y (mgr.14901) 135 : cluster [DBG] pgmap v131: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-08-27T22:03:02.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:02 smithi130 bash[27579]: cluster 2023-08-27T22:03:00.969391+0000 mgr.y (mgr.14901) 135 : cluster [DBG] pgmap v131: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-08-27T22:03:03.061 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 43 16 68528 68512 99.5707 73.0625 0.00600799 0.0100365 2023-08-27T22:03:03.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:03:03.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:03.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:03:03.534Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:04.061 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 44 16 69621 69605 98.8601 68.3125 0.0220389 0.0101093 2023-08-27T22:03:04.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:04 smithi006 bash[27292]: cluster 2023-08-27T22:03:02.969898+0000 mgr.y (mgr.14901) 136 : cluster [DBG] pgmap v132: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s 2023-08-27T22:03:04.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:04 smithi006 bash[27292]: audit 2023-08-27T22:03:03.548072+0000 mon.c (mon.2) 95 : audit [DBG] from='client.? 172.21.15.6:0/2798915878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:04.474 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:04 smithi006 bash[33918]: cluster 2023-08-27T22:03:02.969898+0000 mgr.y (mgr.14901) 136 : cluster [DBG] pgmap v132: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s 2023-08-27T22:03:04.474 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:04 smithi006 bash[33918]: audit 2023-08-27T22:03:03.548072+0000 mon.c (mon.2) 95 : audit [DBG] from='client.? 172.21.15.6:0/2798915878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:04.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:04 smithi130 bash[27579]: cluster 2023-08-27T22:03:02.969898+0000 mgr.y (mgr.14901) 136 : cluster [DBG] pgmap v132: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s 2023-08-27T22:03:04.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:04 smithi130 bash[27579]: audit 2023-08-27T22:03:03.548072+0000 mon.c (mon.2) 95 : audit [DBG] from='client.? 172.21.15.6:0/2798915878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:05.061 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 45 16 70705 70689 98.1686 67.75 0.0168142 0.0101808 2023-08-27T22:03:06.061 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 46 16 71732 71716 97.4298 64.1875 0.0231599 0.0102583 2023-08-27T22:03:06.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:06 smithi006 bash[33918]: cluster 2023-08-27T22:03:04.970659+0000 mgr.y (mgr.14901) 137 : cluster [DBG] pgmap v133: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-08-27T22:03:06.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:06 smithi006 bash[27292]: cluster 2023-08-27T22:03:04.970659+0000 mgr.y (mgr.14901) 137 : cluster [DBG] pgmap v133: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-08-27T22:03:06.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:06 smithi130 bash[27579]: cluster 2023-08-27T22:03:04.970659+0000 mgr.y (mgr.14901) 137 : cluster [DBG] pgmap v133: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-08-27T22:03:07.062 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 47 16 72782 72766 96.7529 65.625 0.0134789 0.0103297 2023-08-27T22:03:07.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:07 smithi130 bash[27579]: audit 2023-08-27T22:03:06.233187+0000 mon.a (mon.0) 906 : audit [DBG] from='client.? 172.21.15.6:0/3781581749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:07.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:07 smithi006 bash[33918]: audit 2023-08-27T22:03:06.233187+0000 mon.a (mon.0) 906 : audit [DBG] from='client.? 172.21.15.6:0/3781581749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:07.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:07 smithi006 bash[27292]: audit 2023-08-27T22:03:06.233187+0000 mon.a (mon.0) 906 : audit [DBG] from='client.? 172.21.15.6:0/3781581749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:08.062 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 48 16 73843 73827 96.1186 66.3125 0.00844058 0.0103978 2023-08-27T22:03:08.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:08 smithi130 bash[27579]: cluster 2023-08-27T22:03:06.971151+0000 mgr.y (mgr.14901) 138 : cluster [DBG] pgmap v134: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2023-08-27T22:03:08.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:08 smithi006 bash[27292]: cluster 2023-08-27T22:03:06.971151+0000 mgr.y (mgr.14901) 138 : cluster [DBG] pgmap v134: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2023-08-27T22:03:08.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:08 smithi006 bash[33918]: cluster 2023-08-27T22:03:06.971151+0000 mgr.y (mgr.14901) 138 : cluster [DBG] pgmap v134: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2023-08-27T22:03:09.062 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 49 16 74815 74799 95.3967 60.75 0.00995924 0.0104761 2023-08-27T22:03:09.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:09 smithi130 bash[27579]: audit 2023-08-27T22:03:08.958213+0000 mon.c (mon.2) 96 : audit [DBG] from='client.? 172.21.15.6:0/3015925141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:09.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:09 smithi006 bash[27292]: audit 2023-08-27T22:03:08.958213+0000 mon.c (mon.2) 96 : audit [DBG] from='client.? 172.21.15.6:0/3015925141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:09.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:09 smithi006 bash[33918]: audit 2023-08-27T22:03:08.958213+0000 mon.c (mon.2) 96 : audit [DBG] from='client.? 172.21.15.6:0/3015925141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:10.062 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 50 16 75861 75845 94.7962 65.375 0.0145047 0.0105425 2023-08-27T22:03:10.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:10 smithi130 bash[27579]: cluster 2023-08-27T22:03:08.971636+0000 mgr.y (mgr.14901) 139 : cluster [DBG] pgmap v135: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2023-08-27T22:03:10.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:10 smithi006 bash[27292]: cluster 2023-08-27T22:03:08.971636+0000 mgr.y (mgr.14901) 139 : cluster [DBG] pgmap v135: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2023-08-27T22:03:10.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:10 smithi006 bash[33918]: cluster 2023-08-27T22:03:08.971636+0000 mgr.y (mgr.14901) 139 : cluster [DBG] pgmap v135: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2023-08-27T22:03:11.062 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 51 16 76872 76856 94.1763 63.1875 0.0116707 0.0106115 2023-08-27T22:03:11.223 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:03:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:03:10] "GET /metrics HTTP/1.1" 200 195004 "" "Prometheus/2.33.4" 2023-08-27T22:03:11.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:03:11.640Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:11.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:03:11.641Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:12.062 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 52 16 77834 77818 93.5214 60.125 0.0239381 0.0106865 2023-08-27T22:03:12.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:12 smithi130 bash[27579]: cluster 2023-08-27T22:03:10.972411+0000 mgr.y (mgr.14901) 140 : cluster [DBG] pgmap v136: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-08-27T22:03:12.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:12 smithi130 bash[27579]: audit 2023-08-27T22:03:11.616901+0000 mon.c (mon.2) 97 : audit [DBG] from='client.? 172.21.15.6:0/3081334454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:12.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:12 smithi006 bash[27292]: cluster 2023-08-27T22:03:10.972411+0000 mgr.y (mgr.14901) 140 : cluster [DBG] pgmap v136: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-08-27T22:03:12.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:12 smithi006 bash[27292]: audit 2023-08-27T22:03:11.616901+0000 mon.c (mon.2) 97 : audit [DBG] from='client.? 172.21.15.6:0/3081334454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:12.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:12 smithi006 bash[33918]: cluster 2023-08-27T22:03:10.972411+0000 mgr.y (mgr.14901) 140 : cluster [DBG] pgmap v136: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-08-27T22:03:12.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:12 smithi006 bash[33918]: audit 2023-08-27T22:03:11.616901+0000 mon.c (mon.2) 97 : audit [DBG] from='client.? 172.21.15.6:0/3081334454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:12.724 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:12 smithi006 bash[56298]: level=error ts=2023-08-27T22:03:12.312Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:12.724 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:12 smithi006 bash[56298]: level=warn ts=2023-08-27T22:03:12.313Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:13.062 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 53 16 78850 78834 92.9548 63.5 0.0132779 0.0107515 2023-08-27T22:03:13.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:03:13.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:13.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:03:13.534Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:14.062 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 54 16 79811 79795 92.3456 60.0625 0.00877873 0.010823 2023-08-27T22:03:14.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:14 smithi130 bash[27579]: cluster 2023-08-27T22:03:12.973214+0000 mgr.y (mgr.14901) 141 : cluster [DBG] pgmap v137: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s 2023-08-27T22:03:14.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:14 smithi006 bash[27292]: cluster 2023-08-27T22:03:12.973214+0000 mgr.y (mgr.14901) 141 : cluster [DBG] pgmap v137: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s 2023-08-27T22:03:14.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:14 smithi006 bash[33918]: cluster 2023-08-27T22:03:12.973214+0000 mgr.y (mgr.14901) 141 : cluster [DBG] pgmap v137: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s 2023-08-27T22:03:15.062 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 55 16 80788 80772 91.7767 61.0625 0.0109515 0.0108882 2023-08-27T22:03:15.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:15 smithi130 bash[27579]: audit 2023-08-27T22:03:14.256710+0000 mon.c (mon.2) 98 : audit [DBG] from='client.? 172.21.15.6:0/749338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:15.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:15 smithi006 bash[27292]: audit 2023-08-27T22:03:14.256710+0000 mon.c (mon.2) 98 : audit [DBG] from='client.? 172.21.15.6:0/749338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:15.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:15 smithi006 bash[33918]: audit 2023-08-27T22:03:14.256710+0000 mon.c (mon.2) 98 : audit [DBG] from='client.? 172.21.15.6:0/749338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:16.063 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 56 16 81788 81772 91.2538 62.5 0.0164858 0.0109526 2023-08-27T22:03:16.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:16 smithi130 bash[27579]: cluster 2023-08-27T22:03:14.974020+0000 mgr.y (mgr.14901) 142 : cluster [DBG] pgmap v138: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-08-27T22:03:16.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:16 smithi006 bash[33918]: cluster 2023-08-27T22:03:14.974020+0000 mgr.y (mgr.14901) 142 : cluster [DBG] pgmap v138: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-08-27T22:03:16.724 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:16 smithi006 bash[27292]: cluster 2023-08-27T22:03:14.974020+0000 mgr.y (mgr.14901) 142 : cluster [DBG] pgmap v138: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-08-27T22:03:17.063 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 57 16 82706 82690 90.6593 57.375 0.0205523 0.0110241 2023-08-27T22:03:17.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:17 smithi130 bash[27579]: audit 2023-08-27T22:03:16.938930+0000 mon.a (mon.0) 907 : audit [DBG] from='client.? 172.21.15.6:0/3316859383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:17.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:17 smithi006 bash[27292]: audit 2023-08-27T22:03:16.938930+0000 mon.a (mon.0) 907 : audit [DBG] from='client.? 172.21.15.6:0/3316859383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:17.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:17 smithi006 bash[33918]: audit 2023-08-27T22:03:16.938930+0000 mon.a (mon.0) 907 : audit [DBG] from='client.? 172.21.15.6:0/3316859383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:18.063 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 58 16 83681 83665 90.1468 60.9375 0.0392384 0.0110866 2023-08-27T22:03:18.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:18 smithi130 bash[27579]: cluster 2023-08-27T22:03:16.974454+0000 mgr.y (mgr.14901) 143 : cluster [DBG] pgmap v139: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2023-08-27T22:03:18.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:18 smithi006 bash[27292]: cluster 2023-08-27T22:03:16.974454+0000 mgr.y (mgr.14901) 143 : cluster [DBG] pgmap v139: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2023-08-27T22:03:18.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:18 smithi006 bash[33918]: cluster 2023-08-27T22:03:16.974454+0000 mgr.y (mgr.14901) 143 : cluster [DBG] pgmap v139: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2023-08-27T22:03:19.063 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 59 16 84639 84623 89.6336 59.875 0.00417522 0.0111502 2023-08-27T22:03:20.063 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:2023-08-27T22:03:20.055162+0000 min lat: 0.00155442 max lat: 0.0744202 avg lat: 0.0112221 2023-08-27T22:03:20.063 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-27T22:03:20.063 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 60 16 85518 85502 89.0553 54.9375 0.010158 0.0112221 2023-08-27T22:03:20.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:20 smithi130 bash[27579]: cluster 2023-08-27T22:03:18.974873+0000 mgr.y (mgr.14901) 144 : cluster [DBG] pgmap v140: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2023-08-27T22:03:20.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:20 smithi130 bash[27579]: audit 2023-08-27T22:03:19.632349+0000 mon.a (mon.0) 908 : audit [DBG] from='client.? 172.21.15.6:0/1981213294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:20.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:20 smithi006 bash[27292]: cluster 2023-08-27T22:03:18.974873+0000 mgr.y (mgr.14901) 144 : cluster [DBG] pgmap v140: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2023-08-27T22:03:20.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:20 smithi006 bash[27292]: audit 2023-08-27T22:03:19.632349+0000 mon.a (mon.0) 908 : audit [DBG] from='client.? 172.21.15.6:0/1981213294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:20.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:20 smithi006 bash[33918]: cluster 2023-08-27T22:03:18.974873+0000 mgr.y (mgr.14901) 144 : cluster [DBG] pgmap v140: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2023-08-27T22:03:20.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:20 smithi006 bash[33918]: audit 2023-08-27T22:03:19.632349+0000 mon.a (mon.0) 908 : audit [DBG] from='client.? 172.21.15.6:0/1981213294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:21.063 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 61 16 86486 86470 88.587 60.5 0.0134111 0.0112828 2023-08-27T22:03:21.223 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:03:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:03:20] "GET /metrics HTTP/1.1" 200 195015 "" "Prometheus/2.33.4" 2023-08-27T22:03:21.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:03:21.640Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:21.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:03:21.643Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:22.063 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 62 16 87391 87375 88.0704 56.5625 0.0067674 0.0113454 2023-08-27T22:03:22.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:22 smithi130 bash[27579]: cluster 2023-08-27T22:03:20.975519+0000 mgr.y (mgr.14901) 145 : cluster [DBG] pgmap v141: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-08-27T22:03:22.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:22 smithi006 bash[27292]: cluster 2023-08-27T22:03:20.975519+0000 mgr.y (mgr.14901) 145 : cluster [DBG] pgmap v141: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-08-27T22:03:22.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:22 smithi006 bash[33918]: cluster 2023-08-27T22:03:20.975519+0000 mgr.y (mgr.14901) 145 : cluster [DBG] pgmap v141: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-08-27T22:03:22.724 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:22 smithi006 bash[56298]: level=error ts=2023-08-27T22:03:22.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:22.724 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:22 smithi006 bash[56298]: level=warn ts=2023-08-27T22:03:22.314Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:23.063 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 63 16 88311 88295 87.5851 57.5 0.0113884 0.0114115 2023-08-27T22:03:23.542 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:23 smithi006 bash[27292]: audit 2023-08-27T22:03:22.324914+0000 mon.b (mon.1) 99 : audit [DBG] from='client.? 172.21.15.6:0/2551429056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:23.542 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:23 smithi006 bash[33918]: audit 2023-08-27T22:03:22.324914+0000 mon.b (mon.1) 99 : audit [DBG] from='client.? 172.21.15.6:0/2551429056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:23.543 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:03:23.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:23.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:23 smithi130 bash[27579]: audit 2023-08-27T22:03:22.324914+0000 mon.b (mon.1) 99 : audit [DBG] from='client.? 172.21.15.6:0/2551429056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:23.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:03:23.535Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:24.063 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 64 16 89249 89233 87.1325 58.625 0.0104265 0.0114707 2023-08-27T22:03:24.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:24 smithi130 bash[27579]: cluster 2023-08-27T22:03:22.975902+0000 mgr.y (mgr.14901) 146 : cluster [DBG] pgmap v142: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-08-27T22:03:24.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:24 smithi006 bash[33918]: cluster 2023-08-27T22:03:22.975902+0000 mgr.y (mgr.14901) 146 : cluster [DBG] pgmap v142: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-08-27T22:03:24.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:24 smithi006 bash[27292]: cluster 2023-08-27T22:03:22.975902+0000 mgr.y (mgr.14901) 146 : cluster [DBG] pgmap v142: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-08-27T22:03:25.064 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 65 16 90106 90090 86.616 53.5625 0.0206143 0.0115394 2023-08-27T22:03:25.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:25 smithi130 bash[27579]: audit 2023-08-27T22:03:25.058216+0000 mon.c (mon.2) 99 : audit [DBG] from='client.? 172.21.15.6:0/2354882075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:25.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:25 smithi006 bash[27292]: audit 2023-08-27T22:03:25.058216+0000 mon.c (mon.2) 99 : audit [DBG] from='client.? 172.21.15.6:0/2354882075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:25.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:25 smithi006 bash[33918]: audit 2023-08-27T22:03:25.058216+0000 mon.c (mon.2) 99 : audit [DBG] from='client.? 172.21.15.6:0/2354882075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:26.063 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 66 16 91052 91036 86.1994 59.125 0.00843011 0.0115944 2023-08-27T22:03:26.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:26 smithi130 bash[27579]: cluster 2023-08-27T22:03:24.976674+0000 mgr.y (mgr.14901) 147 : cluster [DBG] pgmap v143: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-08-27T22:03:26.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:26 smithi006 bash[27292]: cluster 2023-08-27T22:03:24.976674+0000 mgr.y (mgr.14901) 147 : cluster [DBG] pgmap v143: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-08-27T22:03:26.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:26 smithi006 bash[33918]: cluster 2023-08-27T22:03:24.976674+0000 mgr.y (mgr.14901) 147 : cluster [DBG] pgmap v143: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-08-27T22:03:27.063 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 67 16 91966 91950 85.7653 57.125 0.0249928 0.0116526 2023-08-27T22:03:28.064 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 68 16 92800 92784 85.2706 52.125 0.0137794 0.0117212 2023-08-27T22:03:28.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:28 smithi130 bash[27579]: cluster 2023-08-27T22:03:26.977127+0000 mgr.y (mgr.14901) 148 : cluster [DBG] pgmap v144: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2023-08-27T22:03:28.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:28 smithi130 bash[27579]: audit 2023-08-27T22:03:27.748482+0000 mon.b (mon.1) 100 : audit [DBG] from='client.? 172.21.15.6:0/430614717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:28.724 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:28 smithi006 bash[27292]: cluster 2023-08-27T22:03:26.977127+0000 mgr.y (mgr.14901) 148 : cluster [DBG] pgmap v144: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2023-08-27T22:03:28.724 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:28 smithi006 bash[27292]: audit 2023-08-27T22:03:27.748482+0000 mon.b (mon.1) 100 : audit [DBG] from='client.? 172.21.15.6:0/430614717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:28.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:28 smithi006 bash[33918]: cluster 2023-08-27T22:03:26.977127+0000 mgr.y (mgr.14901) 148 : cluster [DBG] pgmap v144: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2023-08-27T22:03:28.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:28 smithi006 bash[33918]: audit 2023-08-27T22:03:27.748482+0000 mon.b (mon.1) 100 : audit [DBG] from='client.? 172.21.15.6:0/430614717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:29.063 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 69 16 93716 93700 84.8644 57.25 0.0157485 0.0117769 2023-08-27T22:03:30.064 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 70 16 94589 94573 84.4314 54.5625 0.00989156 0.0118368 2023-08-27T22:03:30.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:30 smithi130 bash[27579]: cluster 2023-08-27T22:03:28.977625+0000 mgr.y (mgr.14901) 149 : cluster [DBG] pgmap v145: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2023-08-27T22:03:30.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:30 smithi006 bash[33918]: cluster 2023-08-27T22:03:28.977625+0000 mgr.y (mgr.14901) 149 : cluster [DBG] pgmap v145: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2023-08-27T22:03:30.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:30 smithi006 bash[27292]: cluster 2023-08-27T22:03:28.977625+0000 mgr.y (mgr.14901) 149 : cluster [DBG] pgmap v145: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2023-08-27T22:03:31.064 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 71 16 95446 95430 83.9966 53.5625 0.0659754 0.0118983 2023-08-27T22:03:31.223 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:03:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:03:30] "GET /metrics HTTP/1.1" 200 195022 "" "Prometheus/2.33.4" 2023-08-27T22:03:31.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:31 smithi130 bash[27579]: audit 2023-08-27T22:03:30.414656+0000 mon.b (mon.1) 101 : audit [DBG] from='client.? 172.21.15.6:0/3613102306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:31.650 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:31 smithi006 bash[27292]: audit 2023-08-27T22:03:30.414656+0000 mon.b (mon.1) 101 : audit [DBG] from='client.? 172.21.15.6:0/3613102306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:31.651 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:31 smithi006 bash[33918]: audit 2023-08-27T22:03:30.414656+0000 mon.b (mon.1) 101 : audit [DBG] from='client.? 172.21.15.6:0/3613102306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:31.651 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:03:31.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:31.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:03:31.644Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:32.064 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 72 16 96336 96320 83.6025 55.625 0.00941239 0.0119551 2023-08-27T22:03:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:32 smithi130 bash[27579]: cluster 2023-08-27T22:03:30.978482+0000 mgr.y (mgr.14901) 150 : cluster [DBG] pgmap v146: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-08-27T22:03:32.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:32 smithi006 bash[27292]: cluster 2023-08-27T22:03:30.978482+0000 mgr.y (mgr.14901) 150 : cluster [DBG] pgmap v146: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-08-27T22:03:32.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:32 smithi006 bash[33918]: cluster 2023-08-27T22:03:30.978482+0000 mgr.y (mgr.14901) 150 : cluster [DBG] pgmap v146: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-08-27T22:03:32.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:32 smithi006 bash[56298]: level=error ts=2023-08-27T22:03:32.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:32.724 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:32 smithi006 bash[56298]: level=warn ts=2023-08-27T22:03:32.315Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:33.064 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 73 16 97166 97150 83.1678 51.875 0.0131583 0.0120167 2023-08-27T22:03:33.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:33 smithi130 bash[27579]: audit 2023-08-27T22:03:33.101684+0000 mon.c (mon.2) 100 : audit [DBG] from='client.? 172.21.15.6:0/2326131902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:33.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:33 smithi006 bash[27292]: audit 2023-08-27T22:03:33.101684+0000 mon.c (mon.2) 100 : audit [DBG] from='client.? 172.21.15.6:0/2326131902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:33.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:33 smithi006 bash[33918]: audit 2023-08-27T22:03:33.101684+0000 mon.c (mon.2) 100 : audit [DBG] from='client.? 172.21.15.6:0/2326131902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:33.724 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:03:33.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:33.724 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:03:33.535Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:34.064 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 74 16 98055 98039 82.7947 55.5625 0.00939143 0.012071 2023-08-27T22:03:34.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:34 smithi130 bash[27579]: cluster 2023-08-27T22:03:32.978934+0000 mgr.y (mgr.14901) 151 : cluster [DBG] pgmap v147: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 47 MiB/s wr, 758 op/s 2023-08-27T22:03:34.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:34 smithi006 bash[27292]: cluster 2023-08-27T22:03:32.978934+0000 mgr.y (mgr.14901) 151 : cluster [DBG] pgmap v147: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 47 MiB/s wr, 758 op/s 2023-08-27T22:03:34.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:34 smithi006 bash[33918]: cluster 2023-08-27T22:03:32.978934+0000 mgr.y (mgr.14901) 151 : cluster [DBG] pgmap v147: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 47 MiB/s wr, 758 op/s 2023-08-27T22:03:35.064 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 75 16 98932 98916 82.4215 54.8125 0.0150711 0.0121269 2023-08-27T22:03:36.064 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 76 16 99732 99716 81.9948 50 0.0128565 0.0121896 2023-08-27T22:03:36.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:36 smithi130 bash[27579]: cluster 2023-08-27T22:03:34.979681+0000 mgr.y (mgr.14901) 152 : cluster [DBG] pgmap v148: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-08-27T22:03:36.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:36 smithi130 bash[27579]: audit 2023-08-27T22:03:35.794514+0000 mon.a (mon.0) 909 : audit [DBG] from='client.? 172.21.15.6:0/2256509553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:36.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:36 smithi006 bash[27292]: cluster 2023-08-27T22:03:34.979681+0000 mgr.y (mgr.14901) 152 : cluster [DBG] pgmap v148: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-08-27T22:03:36.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:36 smithi006 bash[27292]: audit 2023-08-27T22:03:35.794514+0000 mon.a (mon.0) 909 : audit [DBG] from='client.? 172.21.15.6:0/2256509553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:36.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:36 smithi006 bash[33918]: cluster 2023-08-27T22:03:34.979681+0000 mgr.y (mgr.14901) 152 : cluster [DBG] pgmap v148: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-08-27T22:03:36.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:36 smithi006 bash[33918]: audit 2023-08-27T22:03:35.794514+0000 mon.a (mon.0) 909 : audit [DBG] from='client.? 172.21.15.6:0/2256509553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:37.064 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 77 16 100607 100591 81.6401 54.6875 0.0165135 0.012242 2023-08-27T22:03:38.065 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 78 16 101456 101440 81.2737 53.0625 0.0148352 0.012297 2023-08-27T22:03:38.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:38 smithi130 bash[27579]: cluster 2023-08-27T22:03:36.980184+0000 mgr.y (mgr.14901) 153 : cluster [DBG] pgmap v149: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 733 op/s 2023-08-27T22:03:38.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:38 smithi006 bash[27292]: cluster 2023-08-27T22:03:36.980184+0000 mgr.y (mgr.14901) 153 : cluster [DBG] pgmap v149: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 733 op/s 2023-08-27T22:03:38.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:38 smithi006 bash[33918]: cluster 2023-08-27T22:03:36.980184+0000 mgr.y (mgr.14901) 153 : cluster [DBG] pgmap v149: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 733 op/s 2023-08-27T22:03:39.064 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 79 16 102293 102277 80.9071 52.3125 0.0137702 0.0123537 2023-08-27T22:03:39.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:39 smithi130 bash[27579]: audit 2023-08-27T22:03:38.465169+0000 mon.c (mon.2) 101 : audit [DBG] from='client.? 172.21.15.6:0/1537684732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:39.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:39 smithi006 bash[33918]: audit 2023-08-27T22:03:38.465169+0000 mon.c (mon.2) 101 : audit [DBG] from='client.? 172.21.15.6:0/1537684732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:39.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:39 smithi006 bash[27292]: audit 2023-08-27T22:03:38.465169+0000 mon.c (mon.2) 101 : audit [DBG] from='client.? 172.21.15.6:0/1537684732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:40.064 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:2023-08-27T22:03:40.057053+0000 min lat: 0.00155442 max lat: 0.085191 avg lat: 0.0124047 2023-08-27T22:03:40.064 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-27T22:03:40.065 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 80 16 103155 103139 80.5691 53.875 0.00688103 0.0124047 2023-08-27T22:03:40.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:40 smithi006 bash[33918]: cluster 2023-08-27T22:03:38.980731+0000 mgr.y (mgr.14901) 154 : cluster [DBG] pgmap v150: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 733 op/s 2023-08-27T22:03:40.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:40 smithi006 bash[27292]: cluster 2023-08-27T22:03:38.980731+0000 mgr.y (mgr.14901) 154 : cluster [DBG] pgmap v150: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 733 op/s 2023-08-27T22:03:40.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:40 smithi130 bash[27579]: cluster 2023-08-27T22:03:38.980731+0000 mgr.y (mgr.14901) 154 : cluster [DBG] pgmap v150: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 733 op/s 2023-08-27T22:03:41.064 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 81 16 103940 103924 80.1801 49.0625 0.00960569 0.0124658 2023-08-27T22:03:41.223 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:03:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:03:40] "GET /metrics HTTP/1.1" 200 195022 "" "Prometheus/2.33.4" 2023-08-27T22:03:41.652 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:41 smithi006 bash[27292]: audit 2023-08-27T22:03:41.153458+0000 mon.c (mon.2) 102 : audit [DBG] from='client.? 172.21.15.6:0/1655968700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:41.652 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:41 smithi006 bash[33918]: audit 2023-08-27T22:03:41.153458+0000 mon.c (mon.2) 102 : audit [DBG] from='client.? 172.21.15.6:0/1655968700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:41.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:41 smithi130 bash[27579]: audit 2023-08-27T22:03:41.153458+0000 mon.c (mon.2) 102 : audit [DBG] from='client.? 172.21.15.6:0/1655968700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:41.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:03:41.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:41.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:03:41.645Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:42.065 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 82 16 104781 104765 79.8432 52.5625 0.0158793 0.0125185 2023-08-27T22:03:42.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:42 smithi006 bash[56298]: level=error ts=2023-08-27T22:03:42.314Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:42.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:42 smithi006 bash[56298]: level=warn ts=2023-08-27T22:03:42.316Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:42.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:42 smithi006 bash[33918]: cluster 2023-08-27T22:03:40.981465+0000 mgr.y (mgr.14901) 155 : cluster [DBG] pgmap v151: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-08-27T22:03:42.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:42 smithi006 bash[27292]: cluster 2023-08-27T22:03:40.981465+0000 mgr.y (mgr.14901) 155 : cluster [DBG] pgmap v151: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-08-27T22:03:42.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:42 smithi130 bash[27579]: cluster 2023-08-27T22:03:40.981465+0000 mgr.y (mgr.14901) 155 : cluster [DBG] pgmap v151: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-08-27T22:03:43.065 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 83 16 105630 105614 79.5205 53.0625 0.0311614 0.0125691 2023-08-27T22:03:43.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:03:43.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:43.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:03:43.537Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:44.065 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 84 16 106365 106349 79.1207 45.9375 0.0137525 0.012633 2023-08-27T22:03:44.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:44 smithi006 bash[27292]: cluster 2023-08-27T22:03:42.981928+0000 mgr.y (mgr.14901) 156 : cluster [DBG] pgmap v152: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 713 op/s 2023-08-27T22:03:44.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:44 smithi006 bash[27292]: audit 2023-08-27T22:03:43.823134+0000 mon.a (mon.0) 910 : audit [DBG] from='client.? 172.21.15.6:0/3376724462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:44.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:44 smithi006 bash[33918]: cluster 2023-08-27T22:03:42.981928+0000 mgr.y (mgr.14901) 156 : cluster [DBG] pgmap v152: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 713 op/s 2023-08-27T22:03:44.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:44 smithi006 bash[33918]: audit 2023-08-27T22:03:43.823134+0000 mon.a (mon.0) 910 : audit [DBG] from='client.? 172.21.15.6:0/3376724462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:44.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:44 smithi130 bash[27579]: cluster 2023-08-27T22:03:42.981928+0000 mgr.y (mgr.14901) 156 : cluster [DBG] pgmap v152: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 713 op/s 2023-08-27T22:03:44.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:44 smithi130 bash[27579]: audit 2023-08-27T22:03:43.823134+0000 mon.a (mon.0) 910 : audit [DBG] from='client.? 172.21.15.6:0/3376724462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:45.065 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 85 16 107190 107174 78.7964 51.5625 0.0126451 0.0126848 2023-08-27T22:03:46.065 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 86 16 108002 107986 78.4702 50.75 0.039645 0.0127361 2023-08-27T22:03:46.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:46 smithi006 bash[27292]: cluster 2023-08-27T22:03:44.982764+0000 mgr.y (mgr.14901) 157 : cluster [DBG] pgmap v153: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-08-27T22:03:46.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:46 smithi006 bash[33918]: cluster 2023-08-27T22:03:44.982764+0000 mgr.y (mgr.14901) 157 : cluster [DBG] pgmap v153: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-08-27T22:03:46.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:46 smithi130 bash[27579]: cluster 2023-08-27T22:03:44.982764+0000 mgr.y (mgr.14901) 157 : cluster [DBG] pgmap v153: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-08-27T22:03:47.065 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 87 16 108781 108765 78.1278 48.6875 0.0103326 0.0127929 2023-08-27T22:03:47.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:47 smithi006 bash[27292]: audit 2023-08-27T22:03:46.509391+0000 mon.a (mon.0) 911 : audit [DBG] from='client.? 172.21.15.6:0/619027732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:47.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:47 smithi006 bash[33918]: audit 2023-08-27T22:03:46.509391+0000 mon.a (mon.0) 911 : audit [DBG] from='client.? 172.21.15.6:0/619027732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:47.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:47 smithi130 bash[27579]: audit 2023-08-27T22:03:46.509391+0000 mon.a (mon.0) 911 : audit [DBG] from='client.? 172.21.15.6:0/619027732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:48.065 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 88 16 109585 109569 77.811 50.25 0.0223472 0.0128453 2023-08-27T22:03:48.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:48 smithi006 bash[33918]: cluster 2023-08-27T22:03:46.983270+0000 mgr.y (mgr.14901) 158 : cluster [DBG] pgmap v154: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s 2023-08-27T22:03:48.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:48 smithi006 bash[27292]: cluster 2023-08-27T22:03:46.983270+0000 mgr.y (mgr.14901) 158 : cluster [DBG] pgmap v154: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s 2023-08-27T22:03:48.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:48 smithi130 bash[27579]: cluster 2023-08-27T22:03:46.983270+0000 mgr.y (mgr.14901) 158 : cluster [DBG] pgmap v154: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s 2023-08-27T22:03:49.065 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 89 16 110347 110331 77.4718 47.625 0.0221139 0.0129019 2023-08-27T22:03:49.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:49 smithi006 bash[27292]: audit 2023-08-27T22:03:49.189011+0000 mon.a (mon.0) 912 : audit [DBG] from='client.? 172.21.15.6:0/3037487773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:49.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:49 smithi006 bash[33918]: audit 2023-08-27T22:03:49.189011+0000 mon.a (mon.0) 912 : audit [DBG] from='client.? 172.21.15.6:0/3037487773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:49.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:49 smithi130 bash[27579]: audit 2023-08-27T22:03:49.189011+0000 mon.a (mon.0) 912 : audit [DBG] from='client.? 172.21.15.6:0/3037487773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:50.065 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 90 5 111139 111134 77.1685 50.1875 0.023397 0.0129526 2023-08-27T22:03:50.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:50 smithi006 bash[27292]: cluster 2023-08-27T22:03:48.983793+0000 mgr.y (mgr.14901) 159 : cluster [DBG] pgmap v155: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s 2023-08-27T22:03:50.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:50 smithi006 bash[33918]: cluster 2023-08-27T22:03:48.983793+0000 mgr.y (mgr.14901) 159 : cluster [DBG] pgmap v155: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s 2023-08-27T22:03:50.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:50 smithi130 bash[27579]: cluster 2023-08-27T22:03:48.983793+0000 mgr.y (mgr.14901) 159 : cluster [DBG] pgmap v155: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s 2023-08-27T22:03:51.065 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Total time run: 90.0098 2023-08-27T22:03:51.066 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Total writes made: 111139 2023-08-27T22:03:51.066 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Write size: 65536 2023-08-27T22:03:51.066 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Object size: 65536 2023-08-27T22:03:51.066 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Bandwidth (MB/sec): 77.1715 2023-08-27T22:03:51.066 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Stddev Bandwidth: 30.6623 2023-08-27T22:03:51.066 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Max bandwidth (MB/sec): 178.5 2023-08-27T22:03:51.066 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Min bandwidth (MB/sec): 45.9375 2023-08-27T22:03:51.066 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Average IOPS: 1234 2023-08-27T22:03:51.066 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Stddev IOPS: 490.597 2023-08-27T22:03:51.066 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Max IOPS: 2856 2023-08-27T22:03:51.067 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Min IOPS: 735 2023-08-27T22:03:51.067 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Average Latency(s): 0.0129529 2023-08-27T22:03:51.067 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Stddev Latency(s): 0.00823835 2023-08-27T22:03:51.067 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Max latency(s): 0.0997105 2023-08-27T22:03:51.067 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Min latency(s): 0.00155442 2023-08-27T22:03:51.077 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Cleaning up (deleting benchmark objects) 2023-08-27T22:03:51.223 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:03:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:03:50] "GET /metrics HTTP/1.1" 200 195031 "" "Prometheus/2.33.4" 2023-08-27T22:03:51.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:51 smithi006 bash[27292]: audit 2023-08-27T22:03:51.411082+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:03:51.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:51 smithi006 bash[33918]: audit 2023-08-27T22:03:51.411082+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:03:51.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:03:51.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:51.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:03:51.646Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:51.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:51 smithi130 bash[27579]: audit 2023-08-27T22:03:51.411082+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:03:52.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:52 smithi006 bash[27292]: cluster 2023-08-27T22:03:50.984908+0000 mgr.y (mgr.14901) 160 : cluster [DBG] pgmap v156: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-08-27T22:03:52.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:52 smithi006 bash[27292]: audit 2023-08-27T22:03:51.887546+0000 mon.c (mon.2) 103 : audit [DBG] from='client.? 172.21.15.6:0/2470295242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:52.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:52 smithi006 bash[33918]: cluster 2023-08-27T22:03:50.984908+0000 mgr.y (mgr.14901) 160 : cluster [DBG] pgmap v156: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-08-27T22:03:52.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:52 smithi006 bash[33918]: audit 2023-08-27T22:03:51.887546+0000 mon.c (mon.2) 103 : audit [DBG] from='client.? 172.21.15.6:0/2470295242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:52.724 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:52 smithi006 bash[56298]: level=error ts=2023-08-27T22:03:52.315Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:52.724 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:52 smithi006 bash[56298]: level=warn ts=2023-08-27T22:03:52.317Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:52.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:52 smithi130 bash[27579]: cluster 2023-08-27T22:03:50.984908+0000 mgr.y (mgr.14901) 160 : cluster [DBG] pgmap v156: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-08-27T22:03:52.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:52 smithi130 bash[27579]: audit 2023-08-27T22:03:51.887546+0000 mon.c (mon.2) 103 : audit [DBG] from='client.? 172.21.15.6:0/2470295242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:53.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:03:53.536Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:53.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:03:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:03:53.544Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:03:54.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:54 smithi006 bash[33918]: cluster 2023-08-27T22:03:52.985399+0000 mgr.y (mgr.14901) 161 : cluster [DBG] pgmap v157: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 42 MiB/s wr, 669 op/s 2023-08-27T22:03:54.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:54 smithi006 bash[27292]: cluster 2023-08-27T22:03:52.985399+0000 mgr.y (mgr.14901) 161 : cluster [DBG] pgmap v157: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 42 MiB/s wr, 669 op/s 2023-08-27T22:03:54.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:54 smithi130 bash[27579]: cluster 2023-08-27T22:03:52.985399+0000 mgr.y (mgr.14901) 161 : cluster [DBG] pgmap v157: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 42 MiB/s wr, 669 op/s 2023-08-27T22:03:55.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:55 smithi006 bash[27292]: audit 2023-08-27T22:03:54.575051+0000 mon.b (mon.1) 103 : audit [DBG] from='client.? 172.21.15.6:0/1762328870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:55.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:55 smithi006 bash[33918]: audit 2023-08-27T22:03:54.575051+0000 mon.b (mon.1) 103 : audit [DBG] from='client.? 172.21.15.6:0/1762328870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:55.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:55 smithi130 bash[27579]: audit 2023-08-27T22:03:54.575051+0000 mon.b (mon.1) 103 : audit [DBG] from='client.? 172.21.15.6:0/1762328870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:56.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:56 smithi006 bash[27292]: cluster 2023-08-27T22:03:54.986155+0000 mgr.y (mgr.14901) 162 : cluster [DBG] pgmap v158: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 934 op/s 2023-08-27T22:03:56.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:56 smithi006 bash[33918]: cluster 2023-08-27T22:03:54.986155+0000 mgr.y (mgr.14901) 162 : cluster [DBG] pgmap v158: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 934 op/s 2023-08-27T22:03:56.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:56 smithi130 bash[27579]: cluster 2023-08-27T22:03:54.986155+0000 mgr.y (mgr.14901) 162 : cluster [DBG] pgmap v158: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 934 op/s 2023-08-27T22:03:57.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:57 smithi006 bash[33918]: audit 2023-08-27T22:03:57.297864+0000 mon.a (mon.0) 913 : audit [DBG] from='client.? 172.21.15.6:0/3112941229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:57.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:57 smithi006 bash[27292]: audit 2023-08-27T22:03:57.297864+0000 mon.a (mon.0) 913 : audit [DBG] from='client.? 172.21.15.6:0/3112941229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:57.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:57 smithi130 bash[27579]: audit 2023-08-27T22:03:57.297864+0000 mon.a (mon.0) 913 : audit [DBG] from='client.? 172.21.15.6:0/3112941229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:58.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:58 smithi006 bash[27292]: cluster 2023-08-27T22:03:56.989171+0000 mgr.y (mgr.14901) 163 : cluster [DBG] pgmap v159: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 24 MiB/s wr, 597 op/s 2023-08-27T22:03:58.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:58 smithi006 bash[27292]: audit 2023-08-27T22:03:57.630576+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:03:58.974 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:58 smithi006 bash[27292]: audit 2023-08-27T22:03:57.846645+0000 mon.a (mon.0) 915 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:03:58.974 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:58 smithi006 bash[27292]: audit 2023-08-27T22:03:58.173426+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:03:58.974 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:58 smithi006 bash[27292]: audit 2023-08-27T22:03:58.174511+0000 mon.b (mon.1) 105 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:03:58.974 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:03:58 smithi006 bash[27292]: audit 2023-08-27T22:03:58.180768+0000 mon.a (mon.0) 916 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:03:58.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:58 smithi006 bash[33918]: cluster 2023-08-27T22:03:56.989171+0000 mgr.y (mgr.14901) 163 : cluster [DBG] pgmap v159: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 24 MiB/s wr, 597 op/s 2023-08-27T22:03:58.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:58 smithi006 bash[33918]: audit 2023-08-27T22:03:57.630576+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:03:58.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:58 smithi006 bash[33918]: audit 2023-08-27T22:03:57.846645+0000 mon.a (mon.0) 915 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:03:58.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:58 smithi006 bash[33918]: audit 2023-08-27T22:03:58.173426+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:03:58.975 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:58 smithi006 bash[33918]: audit 2023-08-27T22:03:58.174511+0000 mon.b (mon.1) 105 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:03:58.975 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:03:58 smithi006 bash[33918]: audit 2023-08-27T22:03:58.180768+0000 mon.a (mon.0) 916 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:03:59.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:58 smithi130 bash[27579]: cluster 2023-08-27T22:03:56.989171+0000 mgr.y (mgr.14901) 163 : cluster [DBG] pgmap v159: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 24 MiB/s wr, 597 op/s 2023-08-27T22:03:59.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:58 smithi130 bash[27579]: audit 2023-08-27T22:03:57.630576+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:03:59.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:58 smithi130 bash[27579]: audit 2023-08-27T22:03:57.846645+0000 mon.a (mon.0) 915 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:03:59.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:58 smithi130 bash[27579]: audit 2023-08-27T22:03:58.173426+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:03:59.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:58 smithi130 bash[27579]: audit 2023-08-27T22:03:58.174511+0000 mon.b (mon.1) 105 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:03:59.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:03:58 smithi130 bash[27579]: audit 2023-08-27T22:03:58.180768+0000 mon.a (mon.0) 916 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:04:00.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:04:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:04:00] "GET /metrics HTTP/1.1" 200 195062 "" "Prometheus/2.33.4" 2023-08-27T22:04:00.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:00 smithi006 bash[27292]: cluster 2023-08-27T22:03:58.989642+0000 mgr.y (mgr.14901) 164 : cluster [DBG] pgmap v160: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 24 MiB/s wr, 597 op/s 2023-08-27T22:04:00.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:00 smithi006 bash[27292]: audit 2023-08-27T22:04:00.026295+0000 mon.a (mon.0) 917 : audit [DBG] from='client.? 172.21.15.6:0/1484893840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:00.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:00 smithi006 bash[33918]: cluster 2023-08-27T22:03:58.989642+0000 mgr.y (mgr.14901) 164 : cluster [DBG] pgmap v160: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 24 MiB/s wr, 597 op/s 2023-08-27T22:04:00.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:00 smithi006 bash[33918]: audit 2023-08-27T22:04:00.026295+0000 mon.a (mon.0) 917 : audit [DBG] from='client.? 172.21.15.6:0/1484893840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:01.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:00 smithi130 bash[27579]: cluster 2023-08-27T22:03:58.989642+0000 mgr.y (mgr.14901) 164 : cluster [DBG] pgmap v160: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 24 MiB/s wr, 597 op/s 2023-08-27T22:04:01.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:00 smithi130 bash[27579]: audit 2023-08-27T22:04:00.026295+0000 mon.a (mon.0) 917 : audit [DBG] from='client.? 172.21.15.6:0/1484893840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:01.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:04:01.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:01.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:04:01.646Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:02.658 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:02 smithi006 bash[56298]: level=error ts=2023-08-27T22:04:02.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:02.658 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:02 smithi006 bash[56298]: level=warn ts=2023-08-27T22:04:02.318Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:02.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:02 smithi006 bash[33918]: cluster 2023-08-27T22:04:00.990394+0000 mgr.y (mgr.14901) 165 : cluster [DBG] pgmap v161: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 24 MiB/s wr, 908 op/s 2023-08-27T22:04:02.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:02 smithi006 bash[27292]: cluster 2023-08-27T22:04:00.990394+0000 mgr.y (mgr.14901) 165 : cluster [DBG] pgmap v161: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 24 MiB/s wr, 908 op/s 2023-08-27T22:04:03.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:02 smithi130 bash[27579]: cluster 2023-08-27T22:04:00.990394+0000 mgr.y (mgr.14901) 165 : cluster [DBG] pgmap v161: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 24 MiB/s wr, 908 op/s 2023-08-27T22:04:03.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:03 smithi006 bash[27292]: audit 2023-08-27T22:04:02.761073+0000 mon.b (mon.1) 106 : audit [DBG] from='client.? 172.21.15.6:0/2026199346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:03.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:03 smithi006 bash[33918]: audit 2023-08-27T22:04:02.761073+0000 mon.b (mon.1) 106 : audit [DBG] from='client.? 172.21.15.6:0/2026199346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:03.974 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:04:03.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:03.974 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:04:03.539Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:04.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:03 smithi130 bash[27579]: audit 2023-08-27T22:04:02.761073+0000 mon.b (mon.1) 106 : audit [DBG] from='client.? 172.21.15.6:0/2026199346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:04.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:04 smithi006 bash[33918]: cluster 2023-08-27T22:04:02.990874+0000 mgr.y (mgr.14901) 166 : cluster [DBG] pgmap v162: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 2.9 MiB/s wr, 576 op/s 2023-08-27T22:04:04.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:04 smithi006 bash[27292]: cluster 2023-08-27T22:04:02.990874+0000 mgr.y (mgr.14901) 166 : cluster [DBG] pgmap v162: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 2.9 MiB/s wr, 576 op/s 2023-08-27T22:04:05.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:04 smithi130 bash[27579]: cluster 2023-08-27T22:04:02.990874+0000 mgr.y (mgr.14901) 166 : cluster [DBG] pgmap v162: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 2.9 MiB/s wr, 576 op/s 2023-08-27T22:04:05.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:05 smithi006 bash[33918]: audit 2023-08-27T22:04:05.448553+0000 mon.b (mon.1) 107 : audit [DBG] from='client.? 172.21.15.6:0/2458994271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:05.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:05 smithi006 bash[27292]: audit 2023-08-27T22:04:05.448553+0000 mon.b (mon.1) 107 : audit [DBG] from='client.? 172.21.15.6:0/2458994271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:06.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:05 smithi130 bash[27579]: audit 2023-08-27T22:04:05.448553+0000 mon.b (mon.1) 107 : audit [DBG] from='client.? 172.21.15.6:0/2458994271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:06.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:06 smithi006 bash[27292]: cluster 2023-08-27T22:04:04.991656+0000 mgr.y (mgr.14901) 167 : cluster [DBG] pgmap v163: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 2.9 MiB/s wr, 901 op/s 2023-08-27T22:04:06.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:06 smithi006 bash[33918]: cluster 2023-08-27T22:04:04.991656+0000 mgr.y (mgr.14901) 167 : cluster [DBG] pgmap v163: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 2.9 MiB/s wr, 901 op/s 2023-08-27T22:04:07.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:06 smithi130 bash[27579]: cluster 2023-08-27T22:04:04.991656+0000 mgr.y (mgr.14901) 167 : cluster [DBG] pgmap v163: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 2.9 MiB/s wr, 901 op/s 2023-08-27T22:04:08.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:08 smithi006 bash[27292]: cluster 2023-08-27T22:04:06.992128+0000 mgr.y (mgr.14901) 168 : cluster [DBG] pgmap v164: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-08-27T22:04:08.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:08 smithi006 bash[27292]: audit 2023-08-27T22:04:08.117450+0000 mon.a (mon.0) 918 : audit [DBG] from='client.? 172.21.15.6:0/1159435128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:08.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:08 smithi006 bash[33918]: cluster 2023-08-27T22:04:06.992128+0000 mgr.y (mgr.14901) 168 : cluster [DBG] pgmap v164: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-08-27T22:04:08.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:08 smithi006 bash[33918]: audit 2023-08-27T22:04:08.117450+0000 mon.a (mon.0) 918 : audit [DBG] from='client.? 172.21.15.6:0/1159435128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:09.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:08 smithi130 bash[27579]: cluster 2023-08-27T22:04:06.992128+0000 mgr.y (mgr.14901) 168 : cluster [DBG] pgmap v164: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-08-27T22:04:09.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:08 smithi130 bash[27579]: audit 2023-08-27T22:04:08.117450+0000 mon.a (mon.0) 918 : audit [DBG] from='client.? 172.21.15.6:0/1159435128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:11.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:10 smithi130 bash[27579]: cluster 2023-08-27T22:04:08.992571+0000 mgr.y (mgr.14901) 169 : cluster [DBG] pgmap v165: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-08-27T22:04:11.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:10 smithi006 bash[27292]: cluster 2023-08-27T22:04:08.992571+0000 mgr.y (mgr.14901) 169 : cluster [DBG] pgmap v165: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-08-27T22:04:11.223 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:04:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:04:10] "GET /metrics HTTP/1.1" 200 195062 "" "Prometheus/2.33.4" 2023-08-27T22:04:11.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:10 smithi006 bash[33918]: cluster 2023-08-27T22:04:08.992571+0000 mgr.y (mgr.14901) 169 : cluster [DBG] pgmap v165: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-08-27T22:04:11.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:11 smithi006 bash[27292]: audit 2023-08-27T22:04:10.786685+0000 mon.c (mon.2) 104 : audit [DBG] from='client.? 172.21.15.6:0/3960405827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:11.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:11 smithi006 bash[33918]: audit 2023-08-27T22:04:10.786685+0000 mon.c (mon.2) 104 : audit [DBG] from='client.? 172.21.15.6:0/3960405827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:11.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:04:11.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:11.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:04:11.650Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:12.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:11 smithi130 bash[27579]: audit 2023-08-27T22:04:10.786685+0000 mon.c (mon.2) 104 : audit [DBG] from='client.? 172.21.15.6:0/3960405827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:12.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:12 smithi006 bash[56298]: level=error ts=2023-08-27T22:04:12.318Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:12.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:12 smithi006 bash[56298]: level=warn ts=2023-08-27T22:04:12.319Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:13.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:12 smithi130 bash[27579]: cluster 2023-08-27T22:04:10.995990+0000 mgr.y (mgr.14901) 170 : cluster [DBG] pgmap v166: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 952 op/s 2023-08-27T22:04:13.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:12 smithi006 bash[27292]: cluster 2023-08-27T22:04:10.995990+0000 mgr.y (mgr.14901) 170 : cluster [DBG] pgmap v166: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 952 op/s 2023-08-27T22:04:13.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:12 smithi006 bash[33918]: cluster 2023-08-27T22:04:10.995990+0000 mgr.y (mgr.14901) 170 : cluster [DBG] pgmap v166: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 952 op/s 2023-08-27T22:04:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:04:13.538Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:13.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:04:13.540Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:14.475 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:14 smithi006 bash[27292]: cluster 2023-08-27T22:04:12.996398+0000 mgr.y (mgr.14901) 171 : cluster [DBG] pgmap v167: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-08-27T22:04:14.475 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:14 smithi006 bash[27292]: audit 2023-08-27T22:04:13.491274+0000 mon.a (mon.0) 919 : audit [DBG] from='client.? 172.21.15.6:0/2028115087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:14.476 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:14 smithi006 bash[33918]: cluster 2023-08-27T22:04:12.996398+0000 mgr.y (mgr.14901) 171 : cluster [DBG] pgmap v167: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-08-27T22:04:14.476 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:14 smithi006 bash[33918]: audit 2023-08-27T22:04:13.491274+0000 mon.a (mon.0) 919 : audit [DBG] from='client.? 172.21.15.6:0/2028115087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:14.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:14 smithi130 bash[27579]: cluster 2023-08-27T22:04:12.996398+0000 mgr.y (mgr.14901) 171 : cluster [DBG] pgmap v167: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-08-27T22:04:14.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:14 smithi130 bash[27579]: audit 2023-08-27T22:04:13.491274+0000 mon.a (mon.0) 919 : audit [DBG] from='client.? 172.21.15.6:0/2028115087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:16.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:16 smithi006 bash[27292]: cluster 2023-08-27T22:04:14.997210+0000 mgr.y (mgr.14901) 172 : cluster [DBG] pgmap v168: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2023-08-27T22:04:16.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:16 smithi006 bash[33918]: cluster 2023-08-27T22:04:14.997210+0000 mgr.y (mgr.14901) 172 : cluster [DBG] pgmap v168: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2023-08-27T22:04:16.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:16 smithi130 bash[27579]: cluster 2023-08-27T22:04:14.997210+0000 mgr.y (mgr.14901) 172 : cluster [DBG] pgmap v168: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2023-08-27T22:04:17.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:17 smithi006 bash[27292]: audit 2023-08-27T22:04:16.215814+0000 mon.a (mon.0) 920 : audit [DBG] from='client.? 172.21.15.6:0/3999080532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:17.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:17 smithi006 bash[33918]: audit 2023-08-27T22:04:16.215814+0000 mon.a (mon.0) 920 : audit [DBG] from='client.? 172.21.15.6:0/3999080532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:17.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:17 smithi130 bash[27579]: audit 2023-08-27T22:04:16.215814+0000 mon.a (mon.0) 920 : audit [DBG] from='client.? 172.21.15.6:0/3999080532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:18.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:18 smithi006 bash[27292]: cluster 2023-08-27T22:04:16.997730+0000 mgr.y (mgr.14901) 173 : cluster [DBG] pgmap v169: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-08-27T22:04:18.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:18 smithi006 bash[33918]: cluster 2023-08-27T22:04:16.997730+0000 mgr.y (mgr.14901) 173 : cluster [DBG] pgmap v169: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-08-27T22:04:18.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:18 smithi130 bash[27579]: cluster 2023-08-27T22:04:16.997730+0000 mgr.y (mgr.14901) 173 : cluster [DBG] pgmap v169: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-08-27T22:04:19.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:19 smithi006 bash[27292]: audit 2023-08-27T22:04:18.900926+0000 mon.b (mon.1) 108 : audit [DBG] from='client.? 172.21.15.6:0/3389336410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:19.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:19 smithi006 bash[33918]: audit 2023-08-27T22:04:18.900926+0000 mon.b (mon.1) 108 : audit [DBG] from='client.? 172.21.15.6:0/3389336410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:19.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:19 smithi130 bash[27579]: audit 2023-08-27T22:04:18.900926+0000 mon.b (mon.1) 108 : audit [DBG] from='client.? 172.21.15.6:0/3389336410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:20.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:20 smithi006 bash[27292]: cluster 2023-08-27T22:04:18.998171+0000 mgr.y (mgr.14901) 174 : cluster [DBG] pgmap v170: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-08-27T22:04:20.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:20 smithi006 bash[33918]: cluster 2023-08-27T22:04:18.998171+0000 mgr.y (mgr.14901) 174 : cluster [DBG] pgmap v170: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-08-27T22:04:20.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:20 smithi130 bash[27579]: cluster 2023-08-27T22:04:18.998171+0000 mgr.y (mgr.14901) 174 : cluster [DBG] pgmap v170: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-08-27T22:04:21.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:04:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:04:20] "GET /metrics HTTP/1.1" 200 195059 "" "Prometheus/2.33.4" 2023-08-27T22:04:21.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:04:21.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:21.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:04:21.648Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:22.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:22 smithi130 bash[27579]: cluster 2023-08-27T22:04:20.998896+0000 mgr.y (mgr.14901) 175 : cluster [DBG] pgmap v171: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 948 op/s 2023-08-27T22:04:22.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:22 smithi130 bash[27579]: audit 2023-08-27T22:04:21.584004+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.6:0/1258316083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:22.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:22 smithi006 bash[27292]: cluster 2023-08-27T22:04:20.998896+0000 mgr.y (mgr.14901) 175 : cluster [DBG] pgmap v171: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 948 op/s 2023-08-27T22:04:22.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:22 smithi006 bash[27292]: audit 2023-08-27T22:04:21.584004+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.6:0/1258316083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:22.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:22 smithi006 bash[33918]: cluster 2023-08-27T22:04:20.998896+0000 mgr.y (mgr.14901) 175 : cluster [DBG] pgmap v171: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 948 op/s 2023-08-27T22:04:22.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:22 smithi006 bash[33918]: audit 2023-08-27T22:04:21.584004+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.6:0/1258316083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:22.724 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:22 smithi006 bash[56298]: level=error ts=2023-08-27T22:04:22.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:22.724 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:22 smithi006 bash[56298]: level=warn ts=2023-08-27T22:04:22.324Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:04:23.538Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:23.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:04:23.542Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:24.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:24 smithi130 bash[27579]: cluster 2023-08-27T22:04:22.999360+0000 mgr.y (mgr.14901) 176 : cluster [DBG] pgmap v172: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-08-27T22:04:24.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:24 smithi006 bash[27292]: cluster 2023-08-27T22:04:22.999360+0000 mgr.y (mgr.14901) 176 : cluster [DBG] pgmap v172: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-08-27T22:04:24.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:24 smithi006 bash[33918]: cluster 2023-08-27T22:04:22.999360+0000 mgr.y (mgr.14901) 176 : cluster [DBG] pgmap v172: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-08-27T22:04:25.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:25 smithi130 bash[27579]: audit 2023-08-27T22:04:24.256669+0000 mon.a (mon.0) 921 : audit [DBG] from='client.? 172.21.15.6:0/2423502486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:25.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:25 smithi006 bash[27292]: audit 2023-08-27T22:04:24.256669+0000 mon.a (mon.0) 921 : audit [DBG] from='client.? 172.21.15.6:0/2423502486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:25.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:25 smithi006 bash[33918]: audit 2023-08-27T22:04:24.256669+0000 mon.a (mon.0) 921 : audit [DBG] from='client.? 172.21.15.6:0/2423502486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:26.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:26 smithi130 bash[27579]: cluster 2023-08-27T22:04:25.000029+0000 mgr.y (mgr.14901) 177 : cluster [DBG] pgmap v173: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2023-08-27T22:04:26.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:26 smithi006 bash[27292]: cluster 2023-08-27T22:04:25.000029+0000 mgr.y (mgr.14901) 177 : cluster [DBG] pgmap v173: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2023-08-27T22:04:26.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:26 smithi006 bash[33918]: cluster 2023-08-27T22:04:25.000029+0000 mgr.y (mgr.14901) 177 : cluster [DBG] pgmap v173: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2023-08-27T22:04:27.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:27 smithi130 bash[27579]: audit 2023-08-27T22:04:26.972570+0000 mon.a (mon.0) 922 : audit [DBG] from='client.? 172.21.15.6:0/2800214843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:27.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:27 smithi006 bash[27292]: audit 2023-08-27T22:04:26.972570+0000 mon.a (mon.0) 922 : audit [DBG] from='client.? 172.21.15.6:0/2800214843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:27.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:27 smithi006 bash[33918]: audit 2023-08-27T22:04:26.972570+0000 mon.a (mon.0) 922 : audit [DBG] from='client.? 172.21.15.6:0/2800214843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:28.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:28 smithi130 bash[27579]: cluster 2023-08-27T22:04:27.000517+0000 mgr.y (mgr.14901) 178 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-08-27T22:04:28.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:28 smithi006 bash[27292]: cluster 2023-08-27T22:04:27.000517+0000 mgr.y (mgr.14901) 178 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-08-27T22:04:28.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:28 smithi006 bash[33918]: cluster 2023-08-27T22:04:27.000517+0000 mgr.y (mgr.14901) 178 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-08-27T22:04:30.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:30 smithi130 bash[27579]: cluster 2023-08-27T22:04:29.001020+0000 mgr.y (mgr.14901) 179 : cluster [DBG] pgmap v175: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-08-27T22:04:30.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:30 smithi130 bash[27579]: audit 2023-08-27T22:04:29.675313+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.6:0/481253874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:30.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:30 smithi006 bash[27292]: cluster 2023-08-27T22:04:29.001020+0000 mgr.y (mgr.14901) 179 : cluster [DBG] pgmap v175: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-08-27T22:04:30.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:30 smithi006 bash[27292]: audit 2023-08-27T22:04:29.675313+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.6:0/481253874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:30.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:30 smithi006 bash[33918]: cluster 2023-08-27T22:04:29.001020+0000 mgr.y (mgr.14901) 179 : cluster [DBG] pgmap v175: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-08-27T22:04:30.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:30 smithi006 bash[33918]: audit 2023-08-27T22:04:29.675313+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.6:0/481253874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:31.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:04:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:04:30] "GET /metrics HTTP/1.1" 200 195074 "" "Prometheus/2.33.4" 2023-08-27T22:04:31.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:04:31.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:31.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:04:31.649Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:32 smithi130 bash[27579]: cluster 2023-08-27T22:04:31.001779+0000 mgr.y (mgr.14901) 180 : cluster [DBG] pgmap v176: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2023-08-27T22:04:32.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:32 smithi006 bash[56298]: level=error ts=2023-08-27T22:04:32.320Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:32.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:32 smithi006 bash[56298]: level=warn ts=2023-08-27T22:04:32.321Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:32.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:32 smithi006 bash[33918]: cluster 2023-08-27T22:04:31.001779+0000 mgr.y (mgr.14901) 180 : cluster [DBG] pgmap v176: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2023-08-27T22:04:32.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:32 smithi006 bash[27292]: cluster 2023-08-27T22:04:31.001779+0000 mgr.y (mgr.14901) 180 : cluster [DBG] pgmap v176: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2023-08-27T22:04:33.548 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:33 smithi006 bash[33918]: audit 2023-08-27T22:04:32.347815+0000 mon.a (mon.0) 923 : audit [DBG] from='client.? 172.21.15.6:0/502517724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:33.548 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:33 smithi006 bash[27292]: audit 2023-08-27T22:04:32.347815+0000 mon.a (mon.0) 923 : audit [DBG] from='client.? 172.21.15.6:0/502517724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:33.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:33 smithi130 bash[27579]: audit 2023-08-27T22:04:32.347815+0000 mon.a (mon.0) 923 : audit [DBG] from='client.? 172.21.15.6:0/502517724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:04:33.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:33.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:04:33.541Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:34.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:34 smithi130 bash[27579]: cluster 2023-08-27T22:04:33.002225+0000 mgr.y (mgr.14901) 181 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-08-27T22:04:34.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:34 smithi006 bash[27292]: cluster 2023-08-27T22:04:33.002225+0000 mgr.y (mgr.14901) 181 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-08-27T22:04:34.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:34 smithi006 bash[33918]: cluster 2023-08-27T22:04:33.002225+0000 mgr.y (mgr.14901) 181 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-08-27T22:04:35.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:35 smithi130 bash[27579]: audit 2023-08-27T22:04:35.025271+0000 mon.b (mon.1) 109 : audit [DBG] from='client.? 172.21.15.6:0/4144255622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:35.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:35 smithi006 bash[33918]: audit 2023-08-27T22:04:35.025271+0000 mon.b (mon.1) 109 : audit [DBG] from='client.? 172.21.15.6:0/4144255622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:35.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:35 smithi006 bash[27292]: audit 2023-08-27T22:04:35.025271+0000 mon.b (mon.1) 109 : audit [DBG] from='client.? 172.21.15.6:0/4144255622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:36.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:36 smithi130 bash[27579]: cluster 2023-08-27T22:04:35.002999+0000 mgr.y (mgr.14901) 182 : cluster [DBG] pgmap v178: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-08-27T22:04:36.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:36 smithi006 bash[33918]: cluster 2023-08-27T22:04:35.002999+0000 mgr.y (mgr.14901) 182 : cluster [DBG] pgmap v178: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-08-27T22:04:36.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:36 smithi006 bash[27292]: cluster 2023-08-27T22:04:35.002999+0000 mgr.y (mgr.14901) 182 : cluster [DBG] pgmap v178: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-08-27T22:04:38.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:38 smithi130 bash[27579]: cluster 2023-08-27T22:04:37.003440+0000 mgr.y (mgr.14901) 183 : cluster [DBG] pgmap v179: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-08-27T22:04:38.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:38 smithi130 bash[27579]: audit 2023-08-27T22:04:37.718962+0000 mon.a (mon.0) 924 : audit [DBG] from='client.? 172.21.15.6:0/3663825004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:38.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:38 smithi006 bash[27292]: cluster 2023-08-27T22:04:37.003440+0000 mgr.y (mgr.14901) 183 : cluster [DBG] pgmap v179: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-08-27T22:04:38.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:38 smithi006 bash[27292]: audit 2023-08-27T22:04:37.718962+0000 mon.a (mon.0) 924 : audit [DBG] from='client.? 172.21.15.6:0/3663825004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:38.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:38 smithi006 bash[33918]: cluster 2023-08-27T22:04:37.003440+0000 mgr.y (mgr.14901) 183 : cluster [DBG] pgmap v179: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-08-27T22:04:38.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:38 smithi006 bash[33918]: audit 2023-08-27T22:04:37.718962+0000 mon.a (mon.0) 924 : audit [DBG] from='client.? 172.21.15.6:0/3663825004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:40.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:40 smithi130 bash[27579]: cluster 2023-08-27T22:04:39.003915+0000 mgr.y (mgr.14901) 184 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-08-27T22:04:40.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:40 smithi006 bash[27292]: cluster 2023-08-27T22:04:39.003915+0000 mgr.y (mgr.14901) 184 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-08-27T22:04:40.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:40 smithi006 bash[33918]: cluster 2023-08-27T22:04:39.003915+0000 mgr.y (mgr.14901) 184 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-08-27T22:04:41.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:04:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:04:40] "GET /metrics HTTP/1.1" 200 195074 "" "Prometheus/2.33.4" 2023-08-27T22:04:41.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:41 smithi130 bash[27579]: audit 2023-08-27T22:04:40.386473+0000 mon.b (mon.1) 110 : audit [DBG] from='client.? 172.21.15.6:0/1110384031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:41.656 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:04:41.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:41.657 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:41 smithi006 bash[33918]: audit 2023-08-27T22:04:40.386473+0000 mon.b (mon.1) 110 : audit [DBG] from='client.? 172.21.15.6:0/1110384031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:41.657 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:41 smithi006 bash[27292]: audit 2023-08-27T22:04:40.386473+0000 mon.b (mon.1) 110 : audit [DBG] from='client.? 172.21.15.6:0/1110384031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:41.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:04:41.649Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:42.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:42 smithi130 bash[27579]: cluster 2023-08-27T22:04:41.004686+0000 mgr.y (mgr.14901) 185 : cluster [DBG] pgmap v181: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-08-27T22:04:42.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:42 smithi006 bash[27292]: cluster 2023-08-27T22:04:41.004686+0000 mgr.y (mgr.14901) 185 : cluster [DBG] pgmap v181: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-08-27T22:04:42.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:42 smithi006 bash[33918]: cluster 2023-08-27T22:04:41.004686+0000 mgr.y (mgr.14901) 185 : cluster [DBG] pgmap v181: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-08-27T22:04:42.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:42 smithi006 bash[56298]: level=error ts=2023-08-27T22:04:42.320Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:42.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:42 smithi006 bash[56298]: level=warn ts=2023-08-27T22:04:42.322Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:43.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:43 smithi130 bash[27579]: audit 2023-08-27T22:04:43.072591+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.6:0/3151323263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:43.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:43 smithi006 bash[27292]: audit 2023-08-27T22:04:43.072591+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.6:0/3151323263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:43.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:43 smithi006 bash[33918]: audit 2023-08-27T22:04:43.072591+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.6:0/3151323263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:43.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:04:43.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:43.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:04:43.541Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:44.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:44 smithi130 bash[27579]: cluster 2023-08-27T22:04:43.005176+0000 mgr.y (mgr.14901) 186 : cluster [DBG] pgmap v182: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-08-27T22:04:44.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:44 smithi006 bash[27292]: cluster 2023-08-27T22:04:43.005176+0000 mgr.y (mgr.14901) 186 : cluster [DBG] pgmap v182: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-08-27T22:04:44.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:44 smithi006 bash[33918]: cluster 2023-08-27T22:04:43.005176+0000 mgr.y (mgr.14901) 186 : cluster [DBG] pgmap v182: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-08-27T22:04:46.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:46 smithi130 bash[27579]: cluster 2023-08-27T22:04:45.005985+0000 mgr.y (mgr.14901) 187 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2023-08-27T22:04:46.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:46 smithi130 bash[27579]: audit 2023-08-27T22:04:45.759551+0000 mon.c (mon.2) 108 : audit [DBG] from='client.? 172.21.15.6:0/130626814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:46.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:46 smithi006 bash[27292]: cluster 2023-08-27T22:04:45.005985+0000 mgr.y (mgr.14901) 187 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2023-08-27T22:04:46.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:46 smithi006 bash[27292]: audit 2023-08-27T22:04:45.759551+0000 mon.c (mon.2) 108 : audit [DBG] from='client.? 172.21.15.6:0/130626814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:46.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:46 smithi006 bash[33918]: cluster 2023-08-27T22:04:45.005985+0000 mgr.y (mgr.14901) 187 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2023-08-27T22:04:46.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:46 smithi006 bash[33918]: audit 2023-08-27T22:04:45.759551+0000 mon.c (mon.2) 108 : audit [DBG] from='client.? 172.21.15.6:0/130626814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:48.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:48 smithi130 bash[27579]: cluster 2023-08-27T22:04:47.006406+0000 mgr.y (mgr.14901) 188 : cluster [DBG] pgmap v184: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-08-27T22:04:48.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:48 smithi006 bash[27292]: cluster 2023-08-27T22:04:47.006406+0000 mgr.y (mgr.14901) 188 : cluster [DBG] pgmap v184: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-08-27T22:04:48.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:48 smithi006 bash[33918]: cluster 2023-08-27T22:04:47.006406+0000 mgr.y (mgr.14901) 188 : cluster [DBG] pgmap v184: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-08-27T22:04:49.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:49 smithi130 bash[27579]: audit 2023-08-27T22:04:48.454691+0000 mon.a (mon.0) 925 : audit [DBG] from='client.? 172.21.15.6:0/2478154118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:49.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:49 smithi006 bash[27292]: audit 2023-08-27T22:04:48.454691+0000 mon.a (mon.0) 925 : audit [DBG] from='client.? 172.21.15.6:0/2478154118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:49.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:49 smithi006 bash[33918]: audit 2023-08-27T22:04:48.454691+0000 mon.a (mon.0) 925 : audit [DBG] from='client.? 172.21.15.6:0/2478154118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:50.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:50 smithi130 bash[27579]: cluster 2023-08-27T22:04:49.006832+0000 mgr.y (mgr.14901) 189 : cluster [DBG] pgmap v185: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-08-27T22:04:50.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:50 smithi006 bash[27292]: cluster 2023-08-27T22:04:49.006832+0000 mgr.y (mgr.14901) 189 : cluster [DBG] pgmap v185: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-08-27T22:04:50.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:50 smithi006 bash[33918]: cluster 2023-08-27T22:04:49.006832+0000 mgr.y (mgr.14901) 189 : cluster [DBG] pgmap v185: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-08-27T22:04:51.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:04:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:04:50] "GET /metrics HTTP/1.1" 200 195070 "" "Prometheus/2.33.4" 2023-08-27T22:04:51.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:51 smithi130 bash[27579]: audit 2023-08-27T22:04:51.127014+0000 mon.a (mon.0) 926 : audit [DBG] from='client.? 172.21.15.6:0/3580330563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:51.657 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:51 smithi006 bash[33918]: audit 2023-08-27T22:04:51.127014+0000 mon.a (mon.0) 926 : audit [DBG] from='client.? 172.21.15.6:0/3580330563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:51.658 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:51 smithi006 bash[27292]: audit 2023-08-27T22:04:51.127014+0000 mon.a (mon.0) 926 : audit [DBG] from='client.? 172.21.15.6:0/3580330563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:51.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:04:51.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:51.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:04:51.651Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:52.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:52 smithi006 bash[27292]: cluster 2023-08-27T22:04:51.008210+0000 mgr.y (mgr.14901) 190 : cluster [DBG] pgmap v186: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-08-27T22:04:52.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:52 smithi006 bash[33918]: cluster 2023-08-27T22:04:51.008210+0000 mgr.y (mgr.14901) 190 : cluster [DBG] pgmap v186: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-08-27T22:04:52.724 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:52 smithi006 bash[56298]: level=error ts=2023-08-27T22:04:52.321Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:52.724 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:52 smithi006 bash[56298]: level=warn ts=2023-08-27T22:04:52.323Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:52.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:52 smithi130 bash[27579]: cluster 2023-08-27T22:04:51.008210+0000 mgr.y (mgr.14901) 190 : cluster [DBG] pgmap v186: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-08-27T22:04:53.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:04:53.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:53.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:04:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:04:53.542Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:04:54.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:54 smithi006 bash[27292]: cluster 2023-08-27T22:04:53.008861+0000 mgr.y (mgr.14901) 191 : cluster [DBG] pgmap v187: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 595 op/s 2023-08-27T22:04:54.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:54 smithi006 bash[27292]: audit 2023-08-27T22:04:53.824651+0000 mon.b (mon.1) 111 : audit [DBG] from='client.? 172.21.15.6:0/597120320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:54.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:54 smithi006 bash[33918]: cluster 2023-08-27T22:04:53.008861+0000 mgr.y (mgr.14901) 191 : cluster [DBG] pgmap v187: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 595 op/s 2023-08-27T22:04:54.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:54 smithi006 bash[33918]: audit 2023-08-27T22:04:53.824651+0000 mon.b (mon.1) 111 : audit [DBG] from='client.? 172.21.15.6:0/597120320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:54.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:54 smithi130 bash[27579]: cluster 2023-08-27T22:04:53.008861+0000 mgr.y (mgr.14901) 191 : cluster [DBG] pgmap v187: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 595 op/s 2023-08-27T22:04:54.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:54 smithi130 bash[27579]: audit 2023-08-27T22:04:53.824651+0000 mon.b (mon.1) 111 : audit [DBG] from='client.? 172.21.15.6:0/597120320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:56.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:56 smithi006 bash[27292]: cluster 2023-08-27T22:04:55.009685+0000 mgr.y (mgr.14901) 192 : cluster [DBG] pgmap v188: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2023-08-27T22:04:56.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:56 smithi006 bash[33918]: cluster 2023-08-27T22:04:55.009685+0000 mgr.y (mgr.14901) 192 : cluster [DBG] pgmap v188: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2023-08-27T22:04:56.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:56 smithi130 bash[27579]: cluster 2023-08-27T22:04:55.009685+0000 mgr.y (mgr.14901) 192 : cluster [DBG] pgmap v188: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2023-08-27T22:04:57.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:57 smithi006 bash[33918]: audit 2023-08-27T22:04:56.507893+0000 mon.a (mon.0) 927 : audit [DBG] from='client.? 172.21.15.6:0/591969136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:57.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:57 smithi006 bash[27292]: audit 2023-08-27T22:04:56.507893+0000 mon.a (mon.0) 927 : audit [DBG] from='client.? 172.21.15.6:0/591969136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:57.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:57 smithi130 bash[27579]: audit 2023-08-27T22:04:56.507893+0000 mon.a (mon.0) 927 : audit [DBG] from='client.? 172.21.15.6:0/591969136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:58.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:58 smithi006 bash[27292]: cluster 2023-08-27T22:04:57.010119+0000 mgr.y (mgr.14901) 193 : cluster [DBG] pgmap v189: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-08-27T22:04:58.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:58 smithi006 bash[27292]: audit 2023-08-27T22:04:58.196266+0000 mon.b (mon.1) 112 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:04:58.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:58 smithi006 bash[33918]: cluster 2023-08-27T22:04:57.010119+0000 mgr.y (mgr.14901) 193 : cluster [DBG] pgmap v189: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-08-27T22:04:58.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:58 smithi006 bash[33918]: audit 2023-08-27T22:04:58.196266+0000 mon.b (mon.1) 112 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:04:58.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:58 smithi130 bash[27579]: cluster 2023-08-27T22:04:57.010119+0000 mgr.y (mgr.14901) 193 : cluster [DBG] pgmap v189: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-08-27T22:04:58.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:58 smithi130 bash[27579]: audit 2023-08-27T22:04:58.196266+0000 mon.b (mon.1) 112 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:04:59.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:04:59 smithi006 bash[27292]: audit 2023-08-27T22:04:59.193175+0000 mon.a (mon.0) 928 : audit [DBG] from='client.? 172.21.15.6:0/2034853888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:59.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:04:59 smithi006 bash[33918]: audit 2023-08-27T22:04:59.193175+0000 mon.a (mon.0) 928 : audit [DBG] from='client.? 172.21.15.6:0/2034853888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:59.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:04:59 smithi130 bash[27579]: audit 2023-08-27T22:04:59.193175+0000 mon.a (mon.0) 928 : audit [DBG] from='client.? 172.21.15.6:0/2034853888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:00.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:00 smithi006 bash[27292]: cluster 2023-08-27T22:04:59.010581+0000 mgr.y (mgr.14901) 194 : cluster [DBG] pgmap v190: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-08-27T22:05:00.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:00 smithi006 bash[33918]: cluster 2023-08-27T22:04:59.010581+0000 mgr.y (mgr.14901) 194 : cluster [DBG] pgmap v190: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-08-27T22:05:00.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:00 smithi130 bash[27579]: cluster 2023-08-27T22:04:59.010581+0000 mgr.y (mgr.14901) 194 : cluster [DBG] pgmap v190: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-08-27T22:05:01.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:05:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:05:00] "GET /metrics HTTP/1.1" 200 195075 "" "Prometheus/2.33.4" 2023-08-27T22:05:01.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:05:01.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:01.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:05:01.650Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:02.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:02 smithi006 bash[27292]: cluster 2023-08-27T22:05:01.011376+0000 mgr.y (mgr.14901) 195 : cluster [DBG] pgmap v191: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2023-08-27T22:05:02.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:02 smithi006 bash[27292]: audit 2023-08-27T22:05:01.894299+0000 mon.a (mon.0) 929 : audit [DBG] from='client.? 172.21.15.6:0/3890344222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:02.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:02 smithi006 bash[56298]: level=error ts=2023-08-27T22:05:02.322Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:02.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:02 smithi006 bash[56298]: level=warn ts=2023-08-27T22:05:02.323Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:02.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:02 smithi006 bash[33918]: cluster 2023-08-27T22:05:01.011376+0000 mgr.y (mgr.14901) 195 : cluster [DBG] pgmap v191: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2023-08-27T22:05:02.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:02 smithi006 bash[33918]: audit 2023-08-27T22:05:01.894299+0000 mon.a (mon.0) 929 : audit [DBG] from='client.? 172.21.15.6:0/3890344222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:02.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:02 smithi130 bash[27579]: cluster 2023-08-27T22:05:01.011376+0000 mgr.y (mgr.14901) 195 : cluster [DBG] pgmap v191: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2023-08-27T22:05:02.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:02 smithi130 bash[27579]: audit 2023-08-27T22:05:01.894299+0000 mon.a (mon.0) 929 : audit [DBG] from='client.? 172.21.15.6:0/3890344222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:05:03.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:05:03.542Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:04.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:04 smithi006 bash[27292]: cluster 2023-08-27T22:05:03.011849+0000 mgr.y (mgr.14901) 196 : cluster [DBG] pgmap v192: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 574 op/s 2023-08-27T22:05:04.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:04 smithi006 bash[27292]: audit 2023-08-27T22:05:04.410399+0000 mon.a (mon.0) 930 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:05:04.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:04 smithi006 bash[33918]: cluster 2023-08-27T22:05:03.011849+0000 mgr.y (mgr.14901) 196 : cluster [DBG] pgmap v192: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 574 op/s 2023-08-27T22:05:04.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:04 smithi006 bash[33918]: audit 2023-08-27T22:05:04.410399+0000 mon.a (mon.0) 930 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:05:04.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:04 smithi130 bash[27579]: cluster 2023-08-27T22:05:03.011849+0000 mgr.y (mgr.14901) 196 : cluster [DBG] pgmap v192: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 574 op/s 2023-08-27T22:05:04.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:04 smithi130 bash[27579]: audit 2023-08-27T22:05:04.410399+0000 mon.a (mon.0) 930 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:05:05.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:05 smithi130 bash[27579]: audit 2023-08-27T22:05:04.582102+0000 mon.a (mon.0) 931 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:05:05.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:05 smithi130 bash[27579]: audit 2023-08-27T22:05:04.630555+0000 mon.a (mon.0) 932 : audit [DBG] from='client.? 172.21.15.6:0/2143408087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:05.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:05 smithi130 bash[27579]: audit 2023-08-27T22:05:04.929169+0000 mon.b (mon.1) 113 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:05:05.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:05 smithi130 bash[27579]: audit 2023-08-27T22:05:04.930898+0000 mon.b (mon.1) 114 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:05:05.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:05 smithi130 bash[27579]: audit 2023-08-27T22:05:04.933157+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:05:05.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:05 smithi006 bash[33918]: audit 2023-08-27T22:05:04.582102+0000 mon.a (mon.0) 931 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:05:05.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:05 smithi006 bash[33918]: audit 2023-08-27T22:05:04.630555+0000 mon.a (mon.0) 932 : audit [DBG] from='client.? 172.21.15.6:0/2143408087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:05.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:05 smithi006 bash[33918]: audit 2023-08-27T22:05:04.929169+0000 mon.b (mon.1) 113 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:05:05.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:05 smithi006 bash[33918]: audit 2023-08-27T22:05:04.930898+0000 mon.b (mon.1) 114 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:05:05.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:05 smithi006 bash[33918]: audit 2023-08-27T22:05:04.933157+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:05:05.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:05 smithi006 bash[27292]: audit 2023-08-27T22:05:04.582102+0000 mon.a (mon.0) 931 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:05:05.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:05 smithi006 bash[27292]: audit 2023-08-27T22:05:04.630555+0000 mon.a (mon.0) 932 : audit [DBG] from='client.? 172.21.15.6:0/2143408087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:05.974 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:05 smithi006 bash[27292]: audit 2023-08-27T22:05:04.929169+0000 mon.b (mon.1) 113 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:05:05.974 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:05 smithi006 bash[27292]: audit 2023-08-27T22:05:04.930898+0000 mon.b (mon.1) 114 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:05:05.974 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:05 smithi006 bash[27292]: audit 2023-08-27T22:05:04.933157+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:05:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:06 smithi130 bash[27579]: cluster 2023-08-27T22:05:05.017338+0000 mgr.y (mgr.14901) 197 : cluster [DBG] pgmap v193: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-08-27T22:05:06.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:06 smithi006 bash[27292]: cluster 2023-08-27T22:05:05.017338+0000 mgr.y (mgr.14901) 197 : cluster [DBG] pgmap v193: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-08-27T22:05:06.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:06 smithi006 bash[33918]: cluster 2023-08-27T22:05:05.017338+0000 mgr.y (mgr.14901) 197 : cluster [DBG] pgmap v193: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-08-27T22:05:07.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:07 smithi130 bash[27579]: audit 2023-08-27T22:05:07.309928+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.6:0/992972047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:07.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:07 smithi006 bash[33918]: audit 2023-08-27T22:05:07.309928+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.6:0/992972047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:07.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:07 smithi006 bash[27292]: audit 2023-08-27T22:05:07.309928+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.6:0/992972047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:08.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:08 smithi130 bash[27579]: cluster 2023-08-27T22:05:07.017890+0000 mgr.y (mgr.14901) 198 : cluster [DBG] pgmap v194: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 555 op/s 2023-08-27T22:05:08.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:08 smithi006 bash[33918]: cluster 2023-08-27T22:05:07.017890+0000 mgr.y (mgr.14901) 198 : cluster [DBG] pgmap v194: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 555 op/s 2023-08-27T22:05:08.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:08 smithi006 bash[27292]: cluster 2023-08-27T22:05:07.017890+0000 mgr.y (mgr.14901) 198 : cluster [DBG] pgmap v194: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 555 op/s 2023-08-27T22:05:10.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:10 smithi130 bash[27579]: cluster 2023-08-27T22:05:09.018295+0000 mgr.y (mgr.14901) 199 : cluster [DBG] pgmap v195: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 555 op/s 2023-08-27T22:05:10.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:10 smithi130 bash[27579]: audit 2023-08-27T22:05:09.985794+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.6:0/1782810654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:10.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:05:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:05:10] "GET /metrics HTTP/1.1" 200 195075 "" "Prometheus/2.33.4" 2023-08-27T22:05:10.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:10 smithi006 bash[33918]: cluster 2023-08-27T22:05:09.018295+0000 mgr.y (mgr.14901) 199 : cluster [DBG] pgmap v195: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 555 op/s 2023-08-27T22:05:10.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:10 smithi006 bash[33918]: audit 2023-08-27T22:05:09.985794+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.6:0/1782810654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:10.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:10 smithi006 bash[27292]: cluster 2023-08-27T22:05:09.018295+0000 mgr.y (mgr.14901) 199 : cluster [DBG] pgmap v195: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 555 op/s 2023-08-27T22:05:10.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:10 smithi006 bash[27292]: audit 2023-08-27T22:05:09.985794+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.6:0/1782810654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:11.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:05:11.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:11.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:05:11.651Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:12.618 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:12 smithi006 bash[56298]: level=error ts=2023-08-27T22:05:12.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:12.618 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:12 smithi006 bash[56298]: level=warn ts=2023-08-27T22:05:12.324Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:12.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:12 smithi130 bash[27579]: cluster 2023-08-27T22:05:11.018894+0000 mgr.y (mgr.14901) 200 : cluster [DBG] pgmap v196: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 838 op/s 2023-08-27T22:05:12.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:12 smithi006 bash[27292]: cluster 2023-08-27T22:05:11.018894+0000 mgr.y (mgr.14901) 200 : cluster [DBG] pgmap v196: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 838 op/s 2023-08-27T22:05:12.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:12 smithi006 bash[33918]: cluster 2023-08-27T22:05:11.018894+0000 mgr.y (mgr.14901) 200 : cluster [DBG] pgmap v196: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 838 op/s 2023-08-27T22:05:13.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:13 smithi006 bash[27292]: audit 2023-08-27T22:05:12.685080+0000 mon.a (mon.0) 934 : audit [DBG] from='client.? 172.21.15.6:0/898976789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:13.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:13 smithi006 bash[33918]: audit 2023-08-27T22:05:12.685080+0000 mon.a (mon.0) 934 : audit [DBG] from='client.? 172.21.15.6:0/898976789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:13.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:05:13.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:13.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:05:13.543Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:14.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:13 smithi130 bash[27579]: audit 2023-08-27T22:05:12.685080+0000 mon.a (mon.0) 934 : audit [DBG] from='client.? 172.21.15.6:0/898976789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:14.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:14 smithi006 bash[27292]: cluster 2023-08-27T22:05:13.019328+0000 mgr.y (mgr.14901) 201 : cluster [DBG] pgmap v197: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2023-08-27T22:05:14.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:14 smithi006 bash[33918]: cluster 2023-08-27T22:05:13.019328+0000 mgr.y (mgr.14901) 201 : cluster [DBG] pgmap v197: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2023-08-27T22:05:15.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:14 smithi130 bash[27579]: cluster 2023-08-27T22:05:13.019328+0000 mgr.y (mgr.14901) 201 : cluster [DBG] pgmap v197: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2023-08-27T22:05:15.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:15 smithi006 bash[27292]: audit 2023-08-27T22:05:15.359609+0000 mon.c (mon.2) 111 : audit [DBG] from='client.? 172.21.15.6:0/1992675722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:15.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:15 smithi006 bash[33918]: audit 2023-08-27T22:05:15.359609+0000 mon.c (mon.2) 111 : audit [DBG] from='client.? 172.21.15.6:0/1992675722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:16.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:15 smithi130 bash[27579]: audit 2023-08-27T22:05:15.359609+0000 mon.c (mon.2) 111 : audit [DBG] from='client.? 172.21.15.6:0/1992675722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:16.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:16 smithi006 bash[27292]: cluster 2023-08-27T22:05:15.020085+0000 mgr.y (mgr.14901) 202 : cluster [DBG] pgmap v198: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 838 op/s 2023-08-27T22:05:16.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:16 smithi006 bash[33918]: cluster 2023-08-27T22:05:15.020085+0000 mgr.y (mgr.14901) 202 : cluster [DBG] pgmap v198: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 838 op/s 2023-08-27T22:05:17.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:16 smithi130 bash[27579]: cluster 2023-08-27T22:05:15.020085+0000 mgr.y (mgr.14901) 202 : cluster [DBG] pgmap v198: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 838 op/s 2023-08-27T22:05:19.006 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:18 smithi006 bash[27292]: cluster 2023-08-27T22:05:17.020465+0000 mgr.y (mgr.14901) 203 : cluster [DBG] pgmap v199: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-08-27T22:05:19.007 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:18 smithi006 bash[27292]: audit 2023-08-27T22:05:18.036668+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.6:0/1222876380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:19.007 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:18 smithi006 bash[33918]: cluster 2023-08-27T22:05:17.020465+0000 mgr.y (mgr.14901) 203 : cluster [DBG] pgmap v199: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-08-27T22:05:19.007 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:18 smithi006 bash[33918]: audit 2023-08-27T22:05:18.036668+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.6:0/1222876380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:19.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:18 smithi130 bash[27579]: cluster 2023-08-27T22:05:17.020465+0000 mgr.y (mgr.14901) 203 : cluster [DBG] pgmap v199: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-08-27T22:05:19.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:18 smithi130 bash[27579]: audit 2023-08-27T22:05:18.036668+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.6:0/1222876380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:20.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:20 smithi006 bash[27292]: cluster 2023-08-27T22:05:19.020893+0000 mgr.y (mgr.14901) 204 : cluster [DBG] pgmap v200: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-08-27T22:05:20.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:05:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:05:20] "GET /metrics HTTP/1.1" 200 195065 "" "Prometheus/2.33.4" 2023-08-27T22:05:20.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:20 smithi006 bash[33918]: cluster 2023-08-27T22:05:19.020893+0000 mgr.y (mgr.14901) 204 : cluster [DBG] pgmap v200: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-08-27T22:05:21.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:20 smithi130 bash[27579]: cluster 2023-08-27T22:05:19.020893+0000 mgr.y (mgr.14901) 204 : cluster [DBG] pgmap v200: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-08-27T22:05:21.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:21 smithi006 bash[27292]: audit 2023-08-27T22:05:20.717265+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.6:0/3787865763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:21.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:21 smithi006 bash[33918]: audit 2023-08-27T22:05:20.717265+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.6:0/3787865763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:21.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:05:21.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:21.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:05:21.655Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:22.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:21 smithi130 bash[27579]: audit 2023-08-27T22:05:20.717265+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.6:0/3787865763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:22.685 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:22 smithi006 bash[56298]: level=error ts=2023-08-27T22:05:22.324Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:22.685 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:22 smithi006 bash[56298]: level=warn ts=2023-08-27T22:05:22.325Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:22.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:22 smithi006 bash[27292]: cluster 2023-08-27T22:05:21.022571+0000 mgr.y (mgr.14901) 205 : cluster [DBG] pgmap v201: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2023-08-27T22:05:22.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:22 smithi006 bash[33918]: cluster 2023-08-27T22:05:21.022571+0000 mgr.y (mgr.14901) 205 : cluster [DBG] pgmap v201: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2023-08-27T22:05:23.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:22 smithi130 bash[27579]: cluster 2023-08-27T22:05:21.022571+0000 mgr.y (mgr.14901) 205 : cluster [DBG] pgmap v201: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2023-08-27T22:05:23.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:05:23.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:23.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:05:23.544Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:24.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:24 smithi006 bash[27292]: cluster 2023-08-27T22:05:23.023003+0000 mgr.y (mgr.14901) 206 : cluster [DBG] pgmap v202: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-08-27T22:05:24.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:24 smithi006 bash[27292]: audit 2023-08-27T22:05:23.396955+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.6:0/5581115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:24.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:24 smithi006 bash[33918]: cluster 2023-08-27T22:05:23.023003+0000 mgr.y (mgr.14901) 206 : cluster [DBG] pgmap v202: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-08-27T22:05:24.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:24 smithi006 bash[33918]: audit 2023-08-27T22:05:23.396955+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.6:0/5581115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:24.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:24 smithi130 bash[27579]: cluster 2023-08-27T22:05:23.023003+0000 mgr.y (mgr.14901) 206 : cluster [DBG] pgmap v202: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-08-27T22:05:24.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:24 smithi130 bash[27579]: audit 2023-08-27T22:05:23.396955+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.6:0/5581115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:26.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:26 smithi006 bash[27292]: cluster 2023-08-27T22:05:25.023758+0000 mgr.y (mgr.14901) 207 : cluster [DBG] pgmap v203: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2023-08-27T22:05:26.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:26 smithi006 bash[27292]: audit 2023-08-27T22:05:26.060875+0000 mon.b (mon.1) 115 : audit [DBG] from='client.? 172.21.15.6:0/1039943357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:26.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:26 smithi006 bash[33918]: cluster 2023-08-27T22:05:25.023758+0000 mgr.y (mgr.14901) 207 : cluster [DBG] pgmap v203: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2023-08-27T22:05:26.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:26 smithi006 bash[33918]: audit 2023-08-27T22:05:26.060875+0000 mon.b (mon.1) 115 : audit [DBG] from='client.? 172.21.15.6:0/1039943357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:26.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:26 smithi130 bash[27579]: cluster 2023-08-27T22:05:25.023758+0000 mgr.y (mgr.14901) 207 : cluster [DBG] pgmap v203: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2023-08-27T22:05:26.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:26 smithi130 bash[27579]: audit 2023-08-27T22:05:26.060875+0000 mon.b (mon.1) 115 : audit [DBG] from='client.? 172.21.15.6:0/1039943357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:28.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:28 smithi006 bash[27292]: cluster 2023-08-27T22:05:27.024247+0000 mgr.y (mgr.14901) 208 : cluster [DBG] pgmap v204: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-08-27T22:05:28.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:28 smithi006 bash[33918]: cluster 2023-08-27T22:05:27.024247+0000 mgr.y (mgr.14901) 208 : cluster [DBG] pgmap v204: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-08-27T22:05:28.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:28 smithi130 bash[27579]: cluster 2023-08-27T22:05:27.024247+0000 mgr.y (mgr.14901) 208 : cluster [DBG] pgmap v204: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-08-27T22:05:29.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:29 smithi006 bash[27292]: audit 2023-08-27T22:05:28.752960+0000 mon.b (mon.1) 116 : audit [DBG] from='client.? 172.21.15.6:0/1585404094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:29.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:29 smithi006 bash[33918]: audit 2023-08-27T22:05:28.752960+0000 mon.b (mon.1) 116 : audit [DBG] from='client.? 172.21.15.6:0/1585404094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:29.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:29 smithi130 bash[27579]: audit 2023-08-27T22:05:28.752960+0000 mon.b (mon.1) 116 : audit [DBG] from='client.? 172.21.15.6:0/1585404094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:30.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:30 smithi130 bash[27579]: cluster 2023-08-27T22:05:29.024702+0000 mgr.y (mgr.14901) 209 : cluster [DBG] pgmap v205: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-08-27T22:05:30.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:30 smithi006 bash[33918]: cluster 2023-08-27T22:05:29.024702+0000 mgr.y (mgr.14901) 209 : cluster [DBG] pgmap v205: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-08-27T22:05:30.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:30 smithi006 bash[27292]: cluster 2023-08-27T22:05:29.024702+0000 mgr.y (mgr.14901) 209 : cluster [DBG] pgmap v205: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-08-27T22:05:31.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:05:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:05:30] "GET /metrics HTTP/1.1" 200 195077 "" "Prometheus/2.33.4" 2023-08-27T22:05:31.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:05:31.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:31.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:05:31.652Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:32 smithi130 bash[27579]: cluster 2023-08-27T22:05:31.025528+0000 mgr.y (mgr.14901) 210 : cluster [DBG] pgmap v206: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2023-08-27T22:05:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:32 smithi130 bash[27579]: audit 2023-08-27T22:05:31.438743+0000 mon.a (mon.0) 935 : audit [DBG] from='client.? 172.21.15.6:0/1113902517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:32.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:32 smithi006 bash[27292]: cluster 2023-08-27T22:05:31.025528+0000 mgr.y (mgr.14901) 210 : cluster [DBG] pgmap v206: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2023-08-27T22:05:32.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:32 smithi006 bash[27292]: audit 2023-08-27T22:05:31.438743+0000 mon.a (mon.0) 935 : audit [DBG] from='client.? 172.21.15.6:0/1113902517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:32.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:32 smithi006 bash[33918]: cluster 2023-08-27T22:05:31.025528+0000 mgr.y (mgr.14901) 210 : cluster [DBG] pgmap v206: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2023-08-27T22:05:32.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:32 smithi006 bash[33918]: audit 2023-08-27T22:05:31.438743+0000 mon.a (mon.0) 935 : audit [DBG] from='client.? 172.21.15.6:0/1113902517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:32.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:32 smithi006 bash[56298]: level=error ts=2023-08-27T22:05:32.325Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:32.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:32 smithi006 bash[56298]: level=warn ts=2023-08-27T22:05:32.326Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:05:33.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:05:33.544Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:34.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:34 smithi130 bash[27579]: cluster 2023-08-27T22:05:33.025900+0000 mgr.y (mgr.14901) 211 : cluster [DBG] pgmap v207: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-08-27T22:05:34.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:34 smithi130 bash[27579]: audit 2023-08-27T22:05:34.123242+0000 mon.a (mon.0) 936 : audit [DBG] from='client.? 172.21.15.6:0/2251716325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:34.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:34 smithi006 bash[27292]: cluster 2023-08-27T22:05:33.025900+0000 mgr.y (mgr.14901) 211 : cluster [DBG] pgmap v207: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-08-27T22:05:34.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:34 smithi006 bash[27292]: audit 2023-08-27T22:05:34.123242+0000 mon.a (mon.0) 936 : audit [DBG] from='client.? 172.21.15.6:0/2251716325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:34.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:34 smithi006 bash[33918]: cluster 2023-08-27T22:05:33.025900+0000 mgr.y (mgr.14901) 211 : cluster [DBG] pgmap v207: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-08-27T22:05:34.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:34 smithi006 bash[33918]: audit 2023-08-27T22:05:34.123242+0000 mon.a (mon.0) 936 : audit [DBG] from='client.? 172.21.15.6:0/2251716325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:36.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:36 smithi130 bash[27579]: cluster 2023-08-27T22:05:35.026635+0000 mgr.y (mgr.14901) 212 : cluster [DBG] pgmap v208: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 859 op/s 2023-08-27T22:05:36.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:36 smithi006 bash[27292]: cluster 2023-08-27T22:05:35.026635+0000 mgr.y (mgr.14901) 212 : cluster [DBG] pgmap v208: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 859 op/s 2023-08-27T22:05:36.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:36 smithi006 bash[33918]: cluster 2023-08-27T22:05:35.026635+0000 mgr.y (mgr.14901) 212 : cluster [DBG] pgmap v208: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 859 op/s 2023-08-27T22:05:37.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:37 smithi130 bash[27579]: audit 2023-08-27T22:05:36.796744+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.6:0/3044196236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:37.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:37 smithi006 bash[27292]: audit 2023-08-27T22:05:36.796744+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.6:0/3044196236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:37.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:37 smithi006 bash[33918]: audit 2023-08-27T22:05:36.796744+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.6:0/3044196236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:38.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:38 smithi130 bash[27579]: cluster 2023-08-27T22:05:37.027068+0000 mgr.y (mgr.14901) 213 : cluster [DBG] pgmap v209: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-08-27T22:05:38.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:38 smithi006 bash[27292]: cluster 2023-08-27T22:05:37.027068+0000 mgr.y (mgr.14901) 213 : cluster [DBG] pgmap v209: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-08-27T22:05:38.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:38 smithi006 bash[33918]: cluster 2023-08-27T22:05:37.027068+0000 mgr.y (mgr.14901) 213 : cluster [DBG] pgmap v209: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-08-27T22:05:40.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:40 smithi130 bash[27579]: cluster 2023-08-27T22:05:39.027480+0000 mgr.y (mgr.14901) 214 : cluster [DBG] pgmap v210: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-08-27T22:05:40.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:40 smithi130 bash[27579]: audit 2023-08-27T22:05:39.501440+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.6:0/3304143113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:40.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:40 smithi006 bash[27292]: cluster 2023-08-27T22:05:39.027480+0000 mgr.y (mgr.14901) 214 : cluster [DBG] pgmap v210: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-08-27T22:05:40.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:40 smithi006 bash[27292]: audit 2023-08-27T22:05:39.501440+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.6:0/3304143113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:40.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:40 smithi006 bash[33918]: cluster 2023-08-27T22:05:39.027480+0000 mgr.y (mgr.14901) 214 : cluster [DBG] pgmap v210: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-08-27T22:05:40.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:40 smithi006 bash[33918]: audit 2023-08-27T22:05:39.501440+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.6:0/3304143113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:41.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:05:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:05:40] "GET /metrics HTTP/1.1" 200 195077 "" "Prometheus/2.33.4" 2023-08-27T22:05:41.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:05:41.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:41.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:05:41.653Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:42.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:42 smithi130 bash[27579]: cluster 2023-08-27T22:05:41.028248+0000 mgr.y (mgr.14901) 215 : cluster [DBG] pgmap v211: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2023-08-27T22:05:42.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:42 smithi130 bash[27579]: audit 2023-08-27T22:05:42.178231+0000 mon.b (mon.1) 117 : audit [DBG] from='client.? 172.21.15.6:0/3363520845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:42.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:42 smithi006 bash[27292]: cluster 2023-08-27T22:05:41.028248+0000 mgr.y (mgr.14901) 215 : cluster [DBG] pgmap v211: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2023-08-27T22:05:42.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:42 smithi006 bash[27292]: audit 2023-08-27T22:05:42.178231+0000 mon.b (mon.1) 117 : audit [DBG] from='client.? 172.21.15.6:0/3363520845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:42.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:42 smithi006 bash[33918]: cluster 2023-08-27T22:05:41.028248+0000 mgr.y (mgr.14901) 215 : cluster [DBG] pgmap v211: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2023-08-27T22:05:42.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:42 smithi006 bash[33918]: audit 2023-08-27T22:05:42.178231+0000 mon.b (mon.1) 117 : audit [DBG] from='client.? 172.21.15.6:0/3363520845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:42.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:42 smithi006 bash[56298]: level=error ts=2023-08-27T22:05:42.325Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:42.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:42 smithi006 bash[56298]: level=warn ts=2023-08-27T22:05:42.327Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:05:43.543Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:05:43.545Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:44.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:44 smithi130 bash[27579]: cluster 2023-08-27T22:05:43.028814+0000 mgr.y (mgr.14901) 216 : cluster [DBG] pgmap v212: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-08-27T22:05:44.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:44 smithi006 bash[27292]: cluster 2023-08-27T22:05:43.028814+0000 mgr.y (mgr.14901) 216 : cluster [DBG] pgmap v212: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-08-27T22:05:44.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:44 smithi006 bash[33918]: cluster 2023-08-27T22:05:43.028814+0000 mgr.y (mgr.14901) 216 : cluster [DBG] pgmap v212: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-08-27T22:05:45.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:45 smithi130 bash[27579]: audit 2023-08-27T22:05:44.880311+0000 mon.a (mon.0) 937 : audit [DBG] from='client.? 172.21.15.6:0/944510480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:45.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:45 smithi006 bash[27292]: audit 2023-08-27T22:05:44.880311+0000 mon.a (mon.0) 937 : audit [DBG] from='client.? 172.21.15.6:0/944510480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:45.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:45 smithi006 bash[33918]: audit 2023-08-27T22:05:44.880311+0000 mon.a (mon.0) 937 : audit [DBG] from='client.? 172.21.15.6:0/944510480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:46.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:46 smithi130 bash[27579]: cluster 2023-08-27T22:05:45.029758+0000 mgr.y (mgr.14901) 217 : cluster [DBG] pgmap v213: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2023-08-27T22:05:46.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:46 smithi006 bash[27292]: cluster 2023-08-27T22:05:45.029758+0000 mgr.y (mgr.14901) 217 : cluster [DBG] pgmap v213: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2023-08-27T22:05:46.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:46 smithi006 bash[33918]: cluster 2023-08-27T22:05:45.029758+0000 mgr.y (mgr.14901) 217 : cluster [DBG] pgmap v213: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2023-08-27T22:05:48.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:48 smithi130 bash[27579]: cluster 2023-08-27T22:05:47.030206+0000 mgr.y (mgr.14901) 218 : cluster [DBG] pgmap v214: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-08-27T22:05:48.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:48 smithi130 bash[27579]: audit 2023-08-27T22:05:47.570124+0000 mon.b (mon.1) 118 : audit [DBG] from='client.? 172.21.15.6:0/2800444641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:48.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:48 smithi006 bash[33918]: cluster 2023-08-27T22:05:47.030206+0000 mgr.y (mgr.14901) 218 : cluster [DBG] pgmap v214: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-08-27T22:05:48.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:48 smithi006 bash[33918]: audit 2023-08-27T22:05:47.570124+0000 mon.b (mon.1) 118 : audit [DBG] from='client.? 172.21.15.6:0/2800444641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:48.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:48 smithi006 bash[27292]: cluster 2023-08-27T22:05:47.030206+0000 mgr.y (mgr.14901) 218 : cluster [DBG] pgmap v214: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-08-27T22:05:48.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:48 smithi006 bash[27292]: audit 2023-08-27T22:05:47.570124+0000 mon.b (mon.1) 118 : audit [DBG] from='client.? 172.21.15.6:0/2800444641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:50.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:50 smithi130 bash[27579]: cluster 2023-08-27T22:05:49.030659+0000 mgr.y (mgr.14901) 219 : cluster [DBG] pgmap v215: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-08-27T22:05:50.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:50 smithi130 bash[27579]: audit 2023-08-27T22:05:50.230263+0000 mon.a (mon.0) 938 : audit [DBG] from='client.? 172.21.15.6:0/4236562560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:50.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:50 smithi006 bash[33918]: cluster 2023-08-27T22:05:49.030659+0000 mgr.y (mgr.14901) 219 : cluster [DBG] pgmap v215: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-08-27T22:05:50.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:50 smithi006 bash[33918]: audit 2023-08-27T22:05:50.230263+0000 mon.a (mon.0) 938 : audit [DBG] from='client.? 172.21.15.6:0/4236562560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:50.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:50 smithi006 bash[27292]: cluster 2023-08-27T22:05:49.030659+0000 mgr.y (mgr.14901) 219 : cluster [DBG] pgmap v215: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-08-27T22:05:50.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:50 smithi006 bash[27292]: audit 2023-08-27T22:05:50.230263+0000 mon.a (mon.0) 938 : audit [DBG] from='client.? 172.21.15.6:0/4236562560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:51.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:05:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:05:50] "GET /metrics HTTP/1.1" 200 195079 "" "Prometheus/2.33.4" 2023-08-27T22:05:51.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:05:51.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:51.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:05:51.654Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:52.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:52 smithi130 bash[27579]: cluster 2023-08-27T22:05:51.031445+0000 mgr.y (mgr.14901) 220 : cluster [DBG] pgmap v216: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2023-08-27T22:05:52.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:52 smithi006 bash[27292]: cluster 2023-08-27T22:05:51.031445+0000 mgr.y (mgr.14901) 220 : cluster [DBG] pgmap v216: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2023-08-27T22:05:52.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:52 smithi006 bash[33918]: cluster 2023-08-27T22:05:51.031445+0000 mgr.y (mgr.14901) 220 : cluster [DBG] pgmap v216: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2023-08-27T22:05:52.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:52 smithi006 bash[56298]: level=error ts=2023-08-27T22:05:52.327Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:53.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:53 smithi130 bash[27579]: audit 2023-08-27T22:05:52.891960+0000 mon.c (mon.2) 117 : audit [DBG] from='client.? 172.21.15.6:0/1897506932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:53.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:53 smithi006 bash[27292]: audit 2023-08-27T22:05:52.891960+0000 mon.c (mon.2) 117 : audit [DBG] from='client.? 172.21.15.6:0/1897506932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:53.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:53 smithi006 bash[33918]: audit 2023-08-27T22:05:52.891960+0000 mon.c (mon.2) 117 : audit [DBG] from='client.? 172.21.15.6:0/1897506932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:53.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:05:53.544Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:53.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:05:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:05:53.546Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:05:54.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:54 smithi130 bash[27579]: cluster 2023-08-27T22:05:53.031825+0000 mgr.y (mgr.14901) 221 : cluster [DBG] pgmap v217: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-08-27T22:05:54.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:54 smithi006 bash[27292]: cluster 2023-08-27T22:05:53.031825+0000 mgr.y (mgr.14901) 221 : cluster [DBG] pgmap v217: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-08-27T22:05:54.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:54 smithi006 bash[33918]: cluster 2023-08-27T22:05:53.031825+0000 mgr.y (mgr.14901) 221 : cluster [DBG] pgmap v217: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-08-27T22:05:56.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:56 smithi130 bash[27579]: cluster 2023-08-27T22:05:55.032565+0000 mgr.y (mgr.14901) 222 : cluster [DBG] pgmap v218: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 859 op/s 2023-08-27T22:05:56.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:56 smithi130 bash[27579]: audit 2023-08-27T22:05:55.561165+0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.6:0/4212117955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:56.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:56 smithi006 bash[27292]: cluster 2023-08-27T22:05:55.032565+0000 mgr.y (mgr.14901) 222 : cluster [DBG] pgmap v218: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 859 op/s 2023-08-27T22:05:56.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:56 smithi006 bash[27292]: audit 2023-08-27T22:05:55.561165+0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.6:0/4212117955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:56.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:56 smithi006 bash[33918]: cluster 2023-08-27T22:05:55.032565+0000 mgr.y (mgr.14901) 222 : cluster [DBG] pgmap v218: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 859 op/s 2023-08-27T22:05:56.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:56 smithi006 bash[33918]: audit 2023-08-27T22:05:55.561165+0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.6:0/4212117955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:58.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:58 smithi130 bash[27579]: cluster 2023-08-27T22:05:57.033014+0000 mgr.y (mgr.14901) 223 : cluster [DBG] pgmap v219: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-08-27T22:05:58.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:05:58 smithi130 bash[27579]: audit 2023-08-27T22:05:58.260647+0000 mon.a (mon.0) 939 : audit [DBG] from='client.? 172.21.15.6:0/594568930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:58.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:58 smithi006 bash[33918]: cluster 2023-08-27T22:05:57.033014+0000 mgr.y (mgr.14901) 223 : cluster [DBG] pgmap v219: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-08-27T22:05:58.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:05:58 smithi006 bash[33918]: audit 2023-08-27T22:05:58.260647+0000 mon.a (mon.0) 939 : audit [DBG] from='client.? 172.21.15.6:0/594568930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:58.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:58 smithi006 bash[27292]: cluster 2023-08-27T22:05:57.033014+0000 mgr.y (mgr.14901) 223 : cluster [DBG] pgmap v219: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-08-27T22:05:58.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:05:58 smithi006 bash[27292]: audit 2023-08-27T22:05:58.260647+0000 mon.a (mon.0) 939 : audit [DBG] from='client.? 172.21.15.6:0/594568930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:00.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:00 smithi130 bash[27579]: cluster 2023-08-27T22:05:59.033449+0000 mgr.y (mgr.14901) 224 : cluster [DBG] pgmap v220: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-08-27T22:06:00.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:00 smithi006 bash[27292]: cluster 2023-08-27T22:05:59.033449+0000 mgr.y (mgr.14901) 224 : cluster [DBG] pgmap v220: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-08-27T22:06:00.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:00 smithi006 bash[33918]: cluster 2023-08-27T22:05:59.033449+0000 mgr.y (mgr.14901) 224 : cluster [DBG] pgmap v220: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-08-27T22:06:01.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:06:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:06:00] "GET /metrics HTTP/1.1" 200 195074 "" "Prometheus/2.33.4" 2023-08-27T22:06:01.660 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:01 smithi006 bash[33918]: audit 2023-08-27T22:06:00.963662+0000 mon.c (mon.2) 119 : audit [DBG] from='client.? 172.21.15.6:0/2735854837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:01.660 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:01 smithi006 bash[27292]: audit 2023-08-27T22:06:00.963662+0000 mon.c (mon.2) 119 : audit [DBG] from='client.? 172.21.15.6:0/2735854837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:01.661 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:06:01.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:01.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:01 smithi130 bash[27579]: audit 2023-08-27T22:06:00.963662+0000 mon.c (mon.2) 119 : audit [DBG] from='client.? 172.21.15.6:0/2735854837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:01.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:06:01.654Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:02.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:02 smithi006 bash[33918]: cluster 2023-08-27T22:06:01.034188+0000 mgr.y (mgr.14901) 225 : cluster [DBG] pgmap v221: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2023-08-27T22:06:02.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:02 smithi006 bash[27292]: cluster 2023-08-27T22:06:01.034188+0000 mgr.y (mgr.14901) 225 : cluster [DBG] pgmap v221: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2023-08-27T22:06:02.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:02 smithi130 bash[27579]: cluster 2023-08-27T22:06:01.034188+0000 mgr.y (mgr.14901) 225 : cluster [DBG] pgmap v221: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2023-08-27T22:06:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:06:03.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:06:03.547Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:04.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:04 smithi006 bash[27292]: cluster 2023-08-27T22:06:03.034644+0000 mgr.y (mgr.14901) 226 : cluster [DBG] pgmap v222: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 569 op/s 2023-08-27T22:06:04.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:04 smithi006 bash[27292]: audit 2023-08-27T22:06:03.646770+0000 mon.a (mon.0) 940 : audit [DBG] from='client.? 172.21.15.6:0/119789623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:04.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:04 smithi006 bash[33918]: cluster 2023-08-27T22:06:03.034644+0000 mgr.y (mgr.14901) 226 : cluster [DBG] pgmap v222: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 569 op/s 2023-08-27T22:06:04.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:04 smithi006 bash[33918]: audit 2023-08-27T22:06:03.646770+0000 mon.a (mon.0) 940 : audit [DBG] from='client.? 172.21.15.6:0/119789623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:04.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:04 smithi130 bash[27579]: cluster 2023-08-27T22:06:03.034644+0000 mgr.y (mgr.14901) 226 : cluster [DBG] pgmap v222: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 569 op/s 2023-08-27T22:06:04.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:04 smithi130 bash[27579]: audit 2023-08-27T22:06:03.646770+0000 mon.a (mon.0) 940 : audit [DBG] from='client.? 172.21.15.6:0/119789623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:05.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:05 smithi006 bash[33918]: audit 2023-08-27T22:06:04.945659+0000 mon.b (mon.1) 119 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:06:05.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:05 smithi006 bash[33918]: audit 2023-08-27T22:06:05.258622+0000 mon.a (mon.0) 941 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:06:05.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:05 smithi006 bash[33918]: audit 2023-08-27T22:06:05.305123+0000 mon.a (mon.0) 942 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:06:05.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:05 smithi006 bash[27292]: audit 2023-08-27T22:06:04.945659+0000 mon.b (mon.1) 119 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:06:05.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:05 smithi006 bash[27292]: audit 2023-08-27T22:06:05.258622+0000 mon.a (mon.0) 941 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:06:05.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:05 smithi006 bash[27292]: audit 2023-08-27T22:06:05.305123+0000 mon.a (mon.0) 942 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:06:05.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:05 smithi130 bash[27579]: audit 2023-08-27T22:06:04.945659+0000 mon.b (mon.1) 119 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:06:05.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:05 smithi130 bash[27579]: audit 2023-08-27T22:06:05.258622+0000 mon.a (mon.0) 941 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:06:05.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:05 smithi130 bash[27579]: audit 2023-08-27T22:06:05.305123+0000 mon.a (mon.0) 942 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:06:06.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:06 smithi006 bash[27292]: cluster 2023-08-27T22:06:05.035434+0000 mgr.y (mgr.14901) 227 : cluster [DBG] pgmap v223: 89 pgs: 89 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2023-08-27T22:06:06.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:06 smithi006 bash[27292]: audit 2023-08-27T22:06:06.346289+0000 mon.c (mon.2) 120 : audit [DBG] from='client.? 172.21.15.6:0/1791770246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:06.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:06 smithi006 bash[33918]: cluster 2023-08-27T22:06:05.035434+0000 mgr.y (mgr.14901) 227 : cluster [DBG] pgmap v223: 89 pgs: 89 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2023-08-27T22:06:06.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:06 smithi006 bash[33918]: audit 2023-08-27T22:06:06.346289+0000 mon.c (mon.2) 120 : audit [DBG] from='client.? 172.21.15.6:0/1791770246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:06 smithi130 bash[27579]: cluster 2023-08-27T22:06:05.035434+0000 mgr.y (mgr.14901) 227 : cluster [DBG] pgmap v223: 89 pgs: 89 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2023-08-27T22:06:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:06 smithi130 bash[27579]: audit 2023-08-27T22:06:06.346289+0000 mon.c (mon.2) 120 : audit [DBG] from='client.? 172.21.15.6:0/1791770246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:08.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:08 smithi006 bash[27292]: cluster 2023-08-27T22:06:07.036063+0000 mgr.y (mgr.14901) 228 : cluster [DBG] pgmap v224: 89 pgs: 89 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-08-27T22:06:08.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:08 smithi006 bash[33918]: cluster 2023-08-27T22:06:07.036063+0000 mgr.y (mgr.14901) 228 : cluster [DBG] pgmap v224: 89 pgs: 89 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-08-27T22:06:08.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:08 smithi130 bash[27579]: cluster 2023-08-27T22:06:07.036063+0000 mgr.y (mgr.14901) 228 : cluster [DBG] pgmap v224: 89 pgs: 89 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-08-27T22:06:09.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:09 smithi006 bash[27292]: audit 2023-08-27T22:06:09.053470+0000 mon.a (mon.0) 943 : audit [DBG] from='client.? 172.21.15.6:0/4040705935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:09.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:09 smithi006 bash[33918]: audit 2023-08-27T22:06:09.053470+0000 mon.a (mon.0) 943 : audit [DBG] from='client.? 172.21.15.6:0/4040705935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:09.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:09 smithi130 bash[27579]: audit 2023-08-27T22:06:09.053470+0000 mon.a (mon.0) 943 : audit [DBG] from='client.? 172.21.15.6:0/4040705935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:10.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:10 smithi006 bash[27292]: cluster 2023-08-27T22:06:09.036537+0000 mgr.y (mgr.14901) 229 : cluster [DBG] pgmap v225: 89 pgs: 89 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-08-27T22:06:10.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:10 smithi006 bash[33918]: cluster 2023-08-27T22:06:09.036537+0000 mgr.y (mgr.14901) 229 : cluster [DBG] pgmap v225: 89 pgs: 89 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-08-27T22:06:10.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:10 smithi130 bash[27579]: cluster 2023-08-27T22:06:09.036537+0000 mgr.y (mgr.14901) 229 : cluster [DBG] pgmap v225: 89 pgs: 89 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-08-27T22:06:11.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:06:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:06:10] "GET /metrics HTTP/1.1" 200 195074 "" "Prometheus/2.33.4" 2023-08-27T22:06:11.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:06:11.653Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:11.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:06:11.655Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:12.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:12 smithi130 bash[27579]: cluster 2023-08-27T22:06:11.039906+0000 mgr.y (mgr.14901) 230 : cluster [DBG] pgmap v226: 89 pgs: 89 active+clean; 832 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-08-27T22:06:12.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:12 smithi130 bash[27579]: audit 2023-08-27T22:06:11.497660+0000 mon.a (mon.0) 944 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:06:12.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:12 smithi130 bash[27579]: audit 2023-08-27T22:06:11.666750+0000 mon.a (mon.0) 945 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:06:12.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:12 smithi130 bash[27579]: audit 2023-08-27T22:06:11.817146+0000 mon.b (mon.1) 120 : audit [DBG] from='client.? 172.21.15.6:0/634779396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:12.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:12 smithi130 bash[27579]: audit 2023-08-27T22:06:11.986648+0000 mon.b (mon.1) 121 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:06:12.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:12 smithi130 bash[27579]: audit 2023-08-27T22:06:11.989681+0000 mon.b (mon.1) 122 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:06:12.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:12 smithi130 bash[27579]: audit 2023-08-27T22:06:11.994669+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:06:12.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:12 smithi006 bash[27292]: cluster 2023-08-27T22:06:11.039906+0000 mgr.y (mgr.14901) 230 : cluster [DBG] pgmap v226: 89 pgs: 89 active+clean; 832 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-08-27T22:06:12.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:12 smithi006 bash[27292]: audit 2023-08-27T22:06:11.497660+0000 mon.a (mon.0) 944 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:06:12.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:12 smithi006 bash[27292]: audit 2023-08-27T22:06:11.666750+0000 mon.a (mon.0) 945 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:06:12.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:12 smithi006 bash[27292]: audit 2023-08-27T22:06:11.817146+0000 mon.b (mon.1) 120 : audit [DBG] from='client.? 172.21.15.6:0/634779396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:12.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:12 smithi006 bash[27292]: audit 2023-08-27T22:06:11.986648+0000 mon.b (mon.1) 121 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:06:12.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:12 smithi006 bash[27292]: audit 2023-08-27T22:06:11.989681+0000 mon.b (mon.1) 122 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:06:12.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:12 smithi006 bash[27292]: audit 2023-08-27T22:06:11.994669+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:06:12.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:12 smithi006 bash[33918]: cluster 2023-08-27T22:06:11.039906+0000 mgr.y (mgr.14901) 230 : cluster [DBG] pgmap v226: 89 pgs: 89 active+clean; 832 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-08-27T22:06:12.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:12 smithi006 bash[33918]: audit 2023-08-27T22:06:11.497660+0000 mon.a (mon.0) 944 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:06:12.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:12 smithi006 bash[33918]: audit 2023-08-27T22:06:11.666750+0000 mon.a (mon.0) 945 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:06:12.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:12 smithi006 bash[33918]: audit 2023-08-27T22:06:11.817146+0000 mon.b (mon.1) 120 : audit [DBG] from='client.? 172.21.15.6:0/634779396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:12.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:12 smithi006 bash[33918]: audit 2023-08-27T22:06:11.986648+0000 mon.b (mon.1) 121 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:06:12.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:12 smithi006 bash[33918]: audit 2023-08-27T22:06:11.989681+0000 mon.b (mon.1) 122 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:06:12.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:12 smithi006 bash[33918]: audit 2023-08-27T22:06:11.994669+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:06:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:06:13.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:06:13.547Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:14.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:14 smithi130 bash[27579]: cluster 2023-08-27T22:06:13.040366+0000 mgr.y (mgr.14901) 231 : cluster [DBG] pgmap v227: 89 pgs: 89 active+clean; 832 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2023-08-27T22:06:14.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:14 smithi130 bash[27579]: audit 2023-08-27T22:06:14.493677+0000 mon.a (mon.0) 947 : audit [DBG] from='client.? 172.21.15.6:0/2923301775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:14.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:14 smithi006 bash[33918]: cluster 2023-08-27T22:06:13.040366+0000 mgr.y (mgr.14901) 231 : cluster [DBG] pgmap v227: 89 pgs: 89 active+clean; 832 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2023-08-27T22:06:14.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:14 smithi006 bash[33918]: audit 2023-08-27T22:06:14.493677+0000 mon.a (mon.0) 947 : audit [DBG] from='client.? 172.21.15.6:0/2923301775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:14.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:14 smithi006 bash[27292]: cluster 2023-08-27T22:06:13.040366+0000 mgr.y (mgr.14901) 231 : cluster [DBG] pgmap v227: 89 pgs: 89 active+clean; 832 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2023-08-27T22:06:14.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:14 smithi006 bash[27292]: audit 2023-08-27T22:06:14.493677+0000 mon.a (mon.0) 947 : audit [DBG] from='client.? 172.21.15.6:0/2923301775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:16.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:16 smithi130 bash[27579]: cluster 2023-08-27T22:06:15.041146+0000 mgr.y (mgr.14901) 232 : cluster [DBG] pgmap v228: 89 pgs: 89 active+clean; 631 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2023-08-27T22:06:16.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:16 smithi006 bash[33918]: cluster 2023-08-27T22:06:15.041146+0000 mgr.y (mgr.14901) 232 : cluster [DBG] pgmap v228: 89 pgs: 89 active+clean; 631 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2023-08-27T22:06:16.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:16 smithi006 bash[27292]: cluster 2023-08-27T22:06:15.041146+0000 mgr.y (mgr.14901) 232 : cluster [DBG] pgmap v228: 89 pgs: 89 active+clean; 631 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2023-08-27T22:06:17.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:17 smithi130 bash[27579]: audit 2023-08-27T22:06:17.191371+0000 mon.b (mon.1) 123 : audit [DBG] from='client.? 172.21.15.6:0/1875451284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:17.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:17 smithi006 bash[33918]: audit 2023-08-27T22:06:17.191371+0000 mon.b (mon.1) 123 : audit [DBG] from='client.? 172.21.15.6:0/1875451284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:17.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:17 smithi006 bash[27292]: audit 2023-08-27T22:06:17.191371+0000 mon.b (mon.1) 123 : audit [DBG] from='client.? 172.21.15.6:0/1875451284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:18.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:18 smithi130 bash[27579]: cluster 2023-08-27T22:06:17.041620+0000 mgr.y (mgr.14901) 233 : cluster [DBG] pgmap v229: 89 pgs: 89 active+clean; 631 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 545 op/s 2023-08-27T22:06:18.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:18 smithi006 bash[33918]: cluster 2023-08-27T22:06:17.041620+0000 mgr.y (mgr.14901) 233 : cluster [DBG] pgmap v229: 89 pgs: 89 active+clean; 631 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 545 op/s 2023-08-27T22:06:18.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:18 smithi006 bash[27292]: cluster 2023-08-27T22:06:17.041620+0000 mgr.y (mgr.14901) 233 : cluster [DBG] pgmap v229: 89 pgs: 89 active+clean; 631 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 545 op/s 2023-08-27T22:06:20.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:20 smithi130 bash[27579]: cluster 2023-08-27T22:06:19.042069+0000 mgr.y (mgr.14901) 234 : cluster [DBG] pgmap v230: 89 pgs: 89 active+clean; 631 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 545 op/s 2023-08-27T22:06:20.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:20 smithi130 bash[27579]: audit 2023-08-27T22:06:19.911610+0000 mon.c (mon.2) 121 : audit [DBG] from='client.? 172.21.15.6:0/2214233163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:20.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:20 smithi006 bash[27292]: cluster 2023-08-27T22:06:19.042069+0000 mgr.y (mgr.14901) 234 : cluster [DBG] pgmap v230: 89 pgs: 89 active+clean; 631 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 545 op/s 2023-08-27T22:06:20.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:20 smithi006 bash[27292]: audit 2023-08-27T22:06:19.911610+0000 mon.c (mon.2) 121 : audit [DBG] from='client.? 172.21.15.6:0/2214233163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:20.973 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:06:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:06:20] "GET /metrics HTTP/1.1" 200 195078 "" "Prometheus/2.33.4" 2023-08-27T22:06:20.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:20 smithi006 bash[33918]: cluster 2023-08-27T22:06:19.042069+0000 mgr.y (mgr.14901) 234 : cluster [DBG] pgmap v230: 89 pgs: 89 active+clean; 631 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 545 op/s 2023-08-27T22:06:20.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:20 smithi006 bash[33918]: audit 2023-08-27T22:06:19.911610+0000 mon.c (mon.2) 121 : audit [DBG] from='client.? 172.21.15.6:0/2214233163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:21.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:06:21.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:21.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:06:21.655Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:22.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:22 smithi130 bash[27579]: cluster 2023-08-27T22:06:21.042695+0000 mgr.y (mgr.14901) 235 : cluster [DBG] pgmap v231: 89 pgs: 89 active+clean; 417 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2023-08-27T22:06:22.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:22 smithi006 bash[27292]: cluster 2023-08-27T22:06:21.042695+0000 mgr.y (mgr.14901) 235 : cluster [DBG] pgmap v231: 89 pgs: 89 active+clean; 417 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2023-08-27T22:06:22.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:22 smithi006 bash[33918]: cluster 2023-08-27T22:06:21.042695+0000 mgr.y (mgr.14901) 235 : cluster [DBG] pgmap v231: 89 pgs: 89 active+clean; 417 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2023-08-27T22:06:23.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:23 smithi130 bash[27579]: audit 2023-08-27T22:06:22.575273+0000 mon.c (mon.2) 122 : audit [DBG] from='client.? 172.21.15.6:0/577376005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:23.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:23 smithi006 bash[27292]: audit 2023-08-27T22:06:22.575273+0000 mon.c (mon.2) 122 : audit [DBG] from='client.? 172.21.15.6:0/577376005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:23.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:23 smithi006 bash[33918]: audit 2023-08-27T22:06:22.575273+0000 mon.c (mon.2) 122 : audit [DBG] from='client.? 172.21.15.6:0/577376005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:06:23.546Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:23.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:06:23.548Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:24.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:24 smithi130 bash[27579]: cluster 2023-08-27T22:06:23.043777+0000 mgr.y (mgr.14901) 236 : cluster [DBG] pgmap v232: 89 pgs: 89 active+clean; 417 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2023-08-27T22:06:24.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:24 smithi006 bash[27292]: cluster 2023-08-27T22:06:23.043777+0000 mgr.y (mgr.14901) 236 : cluster [DBG] pgmap v232: 89 pgs: 89 active+clean; 417 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2023-08-27T22:06:24.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:24 smithi006 bash[33918]: cluster 2023-08-27T22:06:23.043777+0000 mgr.y (mgr.14901) 236 : cluster [DBG] pgmap v232: 89 pgs: 89 active+clean; 417 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2023-08-27T22:06:25.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:25 smithi130 bash[27579]: audit 2023-08-27T22:06:25.262357+0000 mon.c (mon.2) 123 : audit [DBG] from='client.? 172.21.15.6:0/3078265762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:25.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:25 smithi006 bash[27292]: audit 2023-08-27T22:06:25.262357+0000 mon.c (mon.2) 123 : audit [DBG] from='client.? 172.21.15.6:0/3078265762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:25.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:25 smithi006 bash[33918]: audit 2023-08-27T22:06:25.262357+0000 mon.c (mon.2) 123 : audit [DBG] from='client.? 172.21.15.6:0/3078265762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:26.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:26 smithi130 bash[27579]: cluster 2023-08-27T22:06:25.044555+0000 mgr.y (mgr.14901) 237 : cluster [DBG] pgmap v233: 89 pgs: 89 active+clean; 203 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2023-08-27T22:06:26.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:26 smithi006 bash[27292]: cluster 2023-08-27T22:06:25.044555+0000 mgr.y (mgr.14901) 237 : cluster [DBG] pgmap v233: 89 pgs: 89 active+clean; 203 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2023-08-27T22:06:26.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:26 smithi006 bash[33918]: cluster 2023-08-27T22:06:25.044555+0000 mgr.y (mgr.14901) 237 : cluster [DBG] pgmap v233: 89 pgs: 89 active+clean; 203 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2023-08-27T22:06:28.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:28 smithi130 bash[27579]: cluster 2023-08-27T22:06:27.045006+0000 mgr.y (mgr.14901) 238 : cluster [DBG] pgmap v234: 89 pgs: 89 active+clean; 203 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-08-27T22:06:28.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:28 smithi130 bash[27579]: audit 2023-08-27T22:06:27.893902+0000 mon.a (mon.0) 948 : audit [DBG] from='client.? 172.21.15.6:0/1740307598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:28.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:28 smithi006 bash[33918]: cluster 2023-08-27T22:06:27.045006+0000 mgr.y (mgr.14901) 238 : cluster [DBG] pgmap v234: 89 pgs: 89 active+clean; 203 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-08-27T22:06:28.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:28 smithi006 bash[33918]: audit 2023-08-27T22:06:27.893902+0000 mon.a (mon.0) 948 : audit [DBG] from='client.? 172.21.15.6:0/1740307598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:28.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:28 smithi006 bash[27292]: cluster 2023-08-27T22:06:27.045006+0000 mgr.y (mgr.14901) 238 : cluster [DBG] pgmap v234: 89 pgs: 89 active+clean; 203 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-08-27T22:06:28.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:28 smithi006 bash[27292]: audit 2023-08-27T22:06:27.893902+0000 mon.a (mon.0) 948 : audit [DBG] from='client.? 172.21.15.6:0/1740307598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:29.226 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Removed 111139 objects 2023-08-27T22:06:29.226 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Clean up completed and total clean up time :158.149 2023-08-27T22:06:30.171 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-08-27T22:06:30.171 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-08-27T22:06:30.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:30 smithi130 bash[27579]: cluster 2023-08-27T22:06:29.045431+0000 mgr.y (mgr.14901) 239 : cluster [DBG] pgmap v235: 89 pgs: 89 active+clean; 203 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-08-27T22:06:30.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:30 smithi130 bash[27579]: audit 2023-08-27T22:06:30.318085+0000 mon.a (mon.0) 949 : audit [DBG] from='client.? 172.21.15.6:0/424370663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:30.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:06:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:06:30] "GET /metrics HTTP/1.1" 200 195078 "" "Prometheus/2.33.4" 2023-08-27T22:06:30.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:30 smithi006 bash[27292]: cluster 2023-08-27T22:06:29.045431+0000 mgr.y (mgr.14901) 239 : cluster [DBG] pgmap v235: 89 pgs: 89 active+clean; 203 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-08-27T22:06:30.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:30 smithi006 bash[27292]: audit 2023-08-27T22:06:30.318085+0000 mon.a (mon.0) 949 : audit [DBG] from='client.? 172.21.15.6:0/424370663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:30.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:30 smithi006 bash[33918]: cluster 2023-08-27T22:06:29.045431+0000 mgr.y (mgr.14901) 239 : cluster [DBG] pgmap v235: 89 pgs: 89 active+clean; 203 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-08-27T22:06:30.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:30 smithi006 bash[33918]: audit 2023-08-27T22:06:30.318085+0000 mon.a (mon.0) 949 : audit [DBG] from='client.? 172.21.15.6:0/424370663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:31.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:06:31.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:31.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:06:31.658Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:32.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:32 smithi130 bash[27579]: cluster 2023-08-27T22:06:31.046629+0000 mgr.y (mgr.14901) 240 : cluster [DBG] pgmap v236: 89 pgs: 89 active+clean; 3.3 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 840 op/s 2023-08-27T22:06:32.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:32 smithi006 bash[33918]: cluster 2023-08-27T22:06:31.046629+0000 mgr.y (mgr.14901) 240 : cluster [DBG] pgmap v236: 89 pgs: 89 active+clean; 3.3 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 840 op/s 2023-08-27T22:06:32.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:32 smithi006 bash[27292]: cluster 2023-08-27T22:06:31.046629+0000 mgr.y (mgr.14901) 240 : cluster [DBG] pgmap v236: 89 pgs: 89 active+clean; 3.3 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 840 op/s 2023-08-27T22:06:33.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:33 smithi130 bash[27579]: audit 2023-08-27T22:06:32.744080+0000 mon.a (mon.0) 950 : audit [DBG] from='client.? 172.21.15.6:0/2456674561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:33.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:33 smithi006 bash[27292]: audit 2023-08-27T22:06:32.744080+0000 mon.a (mon.0) 950 : audit [DBG] from='client.? 172.21.15.6:0/2456674561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:33.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:33 smithi006 bash[33918]: audit 2023-08-27T22:06:32.744080+0000 mon.a (mon.0) 950 : audit [DBG] from='client.? 172.21.15.6:0/2456674561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:06:33.547Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:06:33.550Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:34.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:34 smithi130 bash[27579]: cluster 2023-08-27T22:06:33.047464+0000 mgr.y (mgr.14901) 241 : cluster [DBG] pgmap v237: 89 pgs: 89 active+clean; 3.3 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 555 op/s 2023-08-27T22:06:34.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:34 smithi006 bash[33918]: cluster 2023-08-27T22:06:33.047464+0000 mgr.y (mgr.14901) 241 : cluster [DBG] pgmap v237: 89 pgs: 89 active+clean; 3.3 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 555 op/s 2023-08-27T22:06:34.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:34 smithi006 bash[27292]: cluster 2023-08-27T22:06:33.047464+0000 mgr.y (mgr.14901) 241 : cluster [DBG] pgmap v237: 89 pgs: 89 active+clean; 3.3 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 555 op/s 2023-08-27T22:06:34.985 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T22:06:35.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:35 smithi130 bash[27579]: audit 2023-08-27T22:06:35.164604+0000 mon.a (mon.0) 951 : audit [DBG] from='client.? 172.21.15.6:0/3896312658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:35.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:35 smithi006 bash[27292]: audit 2023-08-27T22:06:35.164604+0000 mon.a (mon.0) 951 : audit [DBG] from='client.? 172.21.15.6:0/3896312658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:35.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:35 smithi006 bash[33918]: audit 2023-08-27T22:06:35.164604+0000 mon.a (mon.0) 951 : audit [DBG] from='client.? 172.21.15.6:0/3896312658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:36.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:36 smithi006 bash[27292]: cluster 2023-08-27T22:06:35.048305+0000 mgr.y (mgr.14901) 242 : cluster [DBG] pgmap v238: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 555 op/s 2023-08-27T22:06:36.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:36 smithi006 bash[33918]: cluster 2023-08-27T22:06:35.048305+0000 mgr.y (mgr.14901) 242 : cluster [DBG] pgmap v238: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 555 op/s 2023-08-27T22:06:36.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:36 smithi130 bash[27579]: cluster 2023-08-27T22:06:35.048305+0000 mgr.y (mgr.14901) 242 : cluster [DBG] pgmap v238: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 555 op/s 2023-08-27T22:06:37.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:37 smithi006 bash[27292]: audit 2023-08-27T22:06:37.565214+0000 mon.a (mon.0) 952 : audit [DBG] from='client.? 172.21.15.6:0/1290462335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:37.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:37 smithi006 bash[33918]: audit 2023-08-27T22:06:37.565214+0000 mon.a (mon.0) 952 : audit [DBG] from='client.? 172.21.15.6:0/1290462335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:37.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:37 smithi130 bash[27579]: audit 2023-08-27T22:06:37.565214+0000 mon.a (mon.0) 952 : audit [DBG] from='client.? 172.21.15.6:0/1290462335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:38.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:38 smithi006 bash[27292]: cluster 2023-08-27T22:06:37.048939+0000 mgr.y (mgr.14901) 243 : cluster [DBG] pgmap v239: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 270 op/s 2023-08-27T22:06:38.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:38 smithi006 bash[33918]: cluster 2023-08-27T22:06:37.048939+0000 mgr.y (mgr.14901) 243 : cluster [DBG] pgmap v239: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 270 op/s 2023-08-27T22:06:39.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:38 smithi130 bash[27579]: cluster 2023-08-27T22:06:37.048939+0000 mgr.y (mgr.14901) 243 : cluster [DBG] pgmap v239: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 270 op/s 2023-08-27T22:06:39.639 INFO:teuthology.orchestra.run.smithi006.stderr:pool 'unique_pool_1' removed 2023-08-27T22:06:39.974 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:39 smithi006 bash[27292]: audit 2023-08-27T22:06:38.938647+0000 mon.c (mon.2) 124 : audit [INF] from='client.? 172.21.15.6:0/3625802897' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-27T22:06:39.974 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:39 smithi006 bash[27292]: audit 2023-08-27T22:06:38.939486+0000 mon.a (mon.0) 953 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-27T22:06:39.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:39 smithi006 bash[33918]: audit 2023-08-27T22:06:38.938647+0000 mon.c (mon.2) 124 : audit [INF] from='client.? 172.21.15.6:0/3625802897' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-27T22:06:39.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:39 smithi006 bash[33918]: audit 2023-08-27T22:06:38.939486+0000 mon.a (mon.0) 953 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-27T22:06:40.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:39 smithi130 bash[27579]: audit 2023-08-27T22:06:38.938647+0000 mon.c (mon.2) 124 : audit [INF] from='client.? 172.21.15.6:0/3625802897' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-27T22:06:40.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:39 smithi130 bash[27579]: audit 2023-08-27T22:06:38.939486+0000 mon.a (mon.0) 953 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-27T22:06:40.482 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-08-27T22:06:40.482 INFO:tasks.radosbench:Beginning radosbench... 2023-08-27T22:06:40.482 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-08-27T22:06:40.483 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd pool create unique_pool_2 16 2023-08-27T22:06:40.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:40 smithi006 bash[33918]: cluster 2023-08-27T22:06:39.049510+0000 mgr.y (mgr.14901) 244 : cluster [DBG] pgmap v240: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 270 op/s 2023-08-27T22:06:40.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:40 smithi006 bash[33918]: audit 2023-08-27T22:06:39.624718+0000 mon.a (mon.0) 954 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-08-27T22:06:40.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:40 smithi006 bash[33918]: cluster 2023-08-27T22:06:39.624811+0000 mon.a (mon.0) 955 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-08-27T22:06:40.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:40 smithi006 bash[33918]: audit 2023-08-27T22:06:40.024026+0000 mon.a (mon.0) 956 : audit [DBG] from='client.? 172.21.15.6:0/1876939468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:40.756 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:40 smithi006 bash[27292]: cluster 2023-08-27T22:06:39.049510+0000 mgr.y (mgr.14901) 244 : cluster [DBG] pgmap v240: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 270 op/s 2023-08-27T22:06:40.756 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:40 smithi006 bash[27292]: audit 2023-08-27T22:06:39.624718+0000 mon.a (mon.0) 954 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-08-27T22:06:40.756 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:40 smithi006 bash[27292]: cluster 2023-08-27T22:06:39.624811+0000 mon.a (mon.0) 955 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-08-27T22:06:40.756 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:40 smithi006 bash[27292]: audit 2023-08-27T22:06:40.024026+0000 mon.a (mon.0) 956 : audit [DBG] from='client.? 172.21.15.6:0/1876939468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:41.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:40 smithi130 bash[27579]: cluster 2023-08-27T22:06:39.049510+0000 mgr.y (mgr.14901) 244 : cluster [DBG] pgmap v240: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 270 op/s 2023-08-27T22:06:41.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:40 smithi130 bash[27579]: audit 2023-08-27T22:06:39.624718+0000 mon.a (mon.0) 954 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-08-27T22:06:41.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:40 smithi130 bash[27579]: cluster 2023-08-27T22:06:39.624811+0000 mon.a (mon.0) 955 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-08-27T22:06:41.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:40 smithi130 bash[27579]: audit 2023-08-27T22:06:40.024026+0000 mon.a (mon.0) 956 : audit [DBG] from='client.? 172.21.15.6:0/1876939468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:41.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:06:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:06:40] "GET /metrics HTTP/1.1" 200 195078 "" "Prometheus/2.33.4" 2023-08-27T22:06:41.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:06:41.656Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:41.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:06:41.658Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:42.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:42 smithi006 bash[27292]: cluster 2023-08-27T22:06:41.050384+0000 mgr.y (mgr.14901) 245 : cluster [DBG] pgmap v242: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:42.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:42 smithi006 bash[27292]: audit 2023-08-27T22:06:42.436872+0000 mon.a (mon.0) 957 : audit [DBG] from='client.? 172.21.15.6:0/2497659794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:42.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:42 smithi006 bash[33918]: cluster 2023-08-27T22:06:41.050384+0000 mgr.y (mgr.14901) 245 : cluster [DBG] pgmap v242: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:42.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:42 smithi006 bash[33918]: audit 2023-08-27T22:06:42.436872+0000 mon.a (mon.0) 957 : audit [DBG] from='client.? 172.21.15.6:0/2497659794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:43.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:42 smithi130 bash[27579]: cluster 2023-08-27T22:06:41.050384+0000 mgr.y (mgr.14901) 245 : cluster [DBG] pgmap v242: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:43.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:42 smithi130 bash[27579]: audit 2023-08-27T22:06:42.436872+0000 mon.a (mon.0) 957 : audit [DBG] from='client.? 172.21.15.6:0/2497659794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:43.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:06:43.548Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:06:43.550Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:44.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:44 smithi006 bash[27292]: cluster 2023-08-27T22:06:43.051031+0000 mgr.y (mgr.14901) 246 : cluster [DBG] pgmap v243: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:44.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:44 smithi006 bash[33918]: cluster 2023-08-27T22:06:43.051031+0000 mgr.y (mgr.14901) 246 : cluster [DBG] pgmap v243: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:45.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:44 smithi130 bash[27579]: cluster 2023-08-27T22:06:43.051031+0000 mgr.y (mgr.14901) 246 : cluster [DBG] pgmap v243: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:45.317 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T22:06:45.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:45 smithi006 bash[27292]: audit 2023-08-27T22:06:44.864866+0000 mon.a (mon.0) 958 : audit [DBG] from='client.? 172.21.15.6:0/2637556487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:45.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:45 smithi006 bash[33918]: audit 2023-08-27T22:06:44.864866+0000 mon.a (mon.0) 958 : audit [DBG] from='client.? 172.21.15.6:0/2637556487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:46.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:45 smithi130 bash[27579]: audit 2023-08-27T22:06:44.864866+0000 mon.a (mon.0) 958 : audit [DBG] from='client.? 172.21.15.6:0/2637556487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:46.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:46 smithi006 bash[33918]: cluster 2023-08-27T22:06:45.052058+0000 mgr.y (mgr.14901) 247 : cluster [DBG] pgmap v244: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:46.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:46 smithi006 bash[27292]: cluster 2023-08-27T22:06:45.052058+0000 mgr.y (mgr.14901) 247 : cluster [DBG] pgmap v244: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:47.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:46 smithi130 bash[27579]: cluster 2023-08-27T22:06:45.052058+0000 mgr.y (mgr.14901) 247 : cluster [DBG] pgmap v244: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:47.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:47 smithi006 bash[33918]: audit 2023-08-27T22:06:47.276791+0000 mon.c (mon.2) 125 : audit [DBG] from='client.? 172.21.15.6:0/1179670426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:47.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:47 smithi006 bash[27292]: audit 2023-08-27T22:06:47.276791+0000 mon.c (mon.2) 125 : audit [DBG] from='client.? 172.21.15.6:0/1179670426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:48.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:47 smithi130 bash[27579]: audit 2023-08-27T22:06:47.276791+0000 mon.c (mon.2) 125 : audit [DBG] from='client.? 172.21.15.6:0/1179670426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:48.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:48 smithi006 bash[33918]: cluster 2023-08-27T22:06:47.052454+0000 mgr.y (mgr.14901) 248 : cluster [DBG] pgmap v245: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:48.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:48 smithi006 bash[27292]: cluster 2023-08-27T22:06:47.052454+0000 mgr.y (mgr.14901) 248 : cluster [DBG] pgmap v245: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:49.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:48 smithi130 bash[27579]: cluster 2023-08-27T22:06:47.052454+0000 mgr.y (mgr.14901) 248 : cluster [DBG] pgmap v245: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:49.689 INFO:teuthology.orchestra.run.smithi006.stderr:pool 'unique_pool_2' created 2023-08-27T22:06:49.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:49 smithi006 bash[33918]: audit 2023-08-27T22:06:48.950097+0000 mon.c (mon.2) 126 : audit [INF] from='client.? 172.21.15.6:0/450544225' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-08-27T22:06:49.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:49 smithi006 bash[33918]: audit 2023-08-27T22:06:48.950866+0000 mon.a (mon.0) 959 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-08-27T22:06:49.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:49 smithi006 bash[27292]: audit 2023-08-27T22:06:48.950097+0000 mon.c (mon.2) 126 : audit [INF] from='client.? 172.21.15.6:0/450544225' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-08-27T22:06:49.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:49 smithi006 bash[27292]: audit 2023-08-27T22:06:48.950866+0000 mon.a (mon.0) 959 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-08-27T22:06:50.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:49 smithi130 bash[27579]: audit 2023-08-27T22:06:48.950097+0000 mon.c (mon.2) 126 : audit [INF] from='client.? 172.21.15.6:0/450544225' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-08-27T22:06:50.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:49 smithi130 bash[27579]: audit 2023-08-27T22:06:48.950866+0000 mon.a (mon.0) 959 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-08-27T22:06:50.445 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-08-27T22:06:50.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:50 smithi006 bash[33918]: cluster 2023-08-27T22:06:49.053062+0000 mgr.y (mgr.14901) 249 : cluster [DBG] pgmap v246: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:50.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:50 smithi006 bash[33918]: audit 2023-08-27T22:06:49.672022+0000 mon.a (mon.0) 960 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-08-27T22:06:50.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:50 smithi006 bash[33918]: cluster 2023-08-27T22:06:49.672122+0000 mon.a (mon.0) 961 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-08-27T22:06:50.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:50 smithi006 bash[33918]: audit 2023-08-27T22:06:49.684924+0000 mon.a (mon.0) 962 : audit [DBG] from='client.? 172.21.15.6:0/1809740790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:50.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:50 smithi006 bash[27292]: cluster 2023-08-27T22:06:49.053062+0000 mgr.y (mgr.14901) 249 : cluster [DBG] pgmap v246: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:50.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:50 smithi006 bash[27292]: audit 2023-08-27T22:06:49.672022+0000 mon.a (mon.0) 960 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-08-27T22:06:50.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:50 smithi006 bash[27292]: cluster 2023-08-27T22:06:49.672122+0000 mon.a (mon.0) 961 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-08-27T22:06:50.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:50 smithi006 bash[27292]: audit 2023-08-27T22:06:49.684924+0000 mon.a (mon.0) 962 : audit [DBG] from='client.? 172.21.15.6:0/1809740790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:51.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:50 smithi130 bash[27579]: cluster 2023-08-27T22:06:49.053062+0000 mgr.y (mgr.14901) 249 : cluster [DBG] pgmap v246: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:51.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:50 smithi130 bash[27579]: audit 2023-08-27T22:06:49.672022+0000 mon.a (mon.0) 960 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-08-27T22:06:51.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:50 smithi130 bash[27579]: cluster 2023-08-27T22:06:49.672122+0000 mon.a (mon.0) 961 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-08-27T22:06:51.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:50 smithi130 bash[27579]: audit 2023-08-27T22:06:49.684924+0000 mon.a (mon.0) 962 : audit [DBG] from='client.? 172.21.15.6:0/1809740790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:51.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:06:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:06:50] "GET /metrics HTTP/1.1" 200 192738 "" "Prometheus/2.33.4" 2023-08-27T22:06:51.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:51 smithi006 bash[33918]: cluster 2023-08-27T22:06:50.677630+0000 mon.a (mon.0) 963 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-08-27T22:06:51.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:06:51.656Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:51.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:06:51.658Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:51.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:51 smithi006 bash[27292]: cluster 2023-08-27T22:06:50.677630+0000 mon.a (mon.0) 963 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-08-27T22:06:52.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:51 smithi130 bash[27579]: cluster 2023-08-27T22:06:50.677630+0000 mon.a (mon.0) 963 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-08-27T22:06:52.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:52 smithi006 bash[27292]: cluster 2023-08-27T22:06:51.053714+0000 mgr.y (mgr.14901) 250 : cluster [DBG] pgmap v249: 89 pgs: 9 creating+peering, 7 unknown, 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:52.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:52 smithi006 bash[27292]: cluster 2023-08-27T22:06:51.687317+0000 mon.a (mon.0) 964 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-08-27T22:06:52.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:52 smithi006 bash[27292]: audit 2023-08-27T22:06:52.104877+0000 mon.a (mon.0) 965 : audit [DBG] from='client.? 172.21.15.6:0/3369871164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:52.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:52 smithi006 bash[33918]: cluster 2023-08-27T22:06:51.053714+0000 mgr.y (mgr.14901) 250 : cluster [DBG] pgmap v249: 89 pgs: 9 creating+peering, 7 unknown, 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:52.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:52 smithi006 bash[33918]: cluster 2023-08-27T22:06:51.687317+0000 mon.a (mon.0) 964 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-08-27T22:06:52.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:52 smithi006 bash[33918]: audit 2023-08-27T22:06:52.104877+0000 mon.a (mon.0) 965 : audit [DBG] from='client.? 172.21.15.6:0/3369871164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:53.122 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:52 smithi130 bash[27579]: cluster 2023-08-27T22:06:51.053714+0000 mgr.y (mgr.14901) 250 : cluster [DBG] pgmap v249: 89 pgs: 9 creating+peering, 7 unknown, 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:53.122 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:52 smithi130 bash[27579]: cluster 2023-08-27T22:06:51.687317+0000 mon.a (mon.0) 964 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-08-27T22:06:53.122 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:52 smithi130 bash[27579]: audit 2023-08-27T22:06:52.104877+0000 mon.a (mon.0) 965 : audit [DBG] from='client.? 172.21.15.6:0/3369871164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:06:53.549Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:06:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:06:53.551Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:06:54.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:54 smithi006 bash[33918]: cluster 2023-08-27T22:06:53.054410+0000 mgr.y (mgr.14901) 251 : cluster [DBG] pgmap v251: 89 pgs: 9 creating+peering, 7 unknown, 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:54.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:54 smithi006 bash[33918]: audit 2023-08-27T22:06:54.519036+0000 mon.a (mon.0) 966 : audit [DBG] from='client.? 172.21.15.6:0/131638566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:54.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:54 smithi006 bash[27292]: cluster 2023-08-27T22:06:53.054410+0000 mgr.y (mgr.14901) 251 : cluster [DBG] pgmap v251: 89 pgs: 9 creating+peering, 7 unknown, 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:54.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:54 smithi006 bash[27292]: audit 2023-08-27T22:06:54.519036+0000 mon.a (mon.0) 966 : audit [DBG] from='client.? 172.21.15.6:0/131638566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:55.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:54 smithi130 bash[27579]: cluster 2023-08-27T22:06:53.054410+0000 mgr.y (mgr.14901) 251 : cluster [DBG] pgmap v251: 89 pgs: 9 creating+peering, 7 unknown, 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:55.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:54 smithi130 bash[27579]: audit 2023-08-27T22:06:54.519036+0000 mon.a (mon.0) 966 : audit [DBG] from='client.? 172.21.15.6:0/131638566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:55.271 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T22:06:56.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:56 smithi006 bash[27292]: cluster 2023-08-27T22:06:55.055625+0000 mgr.y (mgr.14901) 252 : cluster [DBG] pgmap v252: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:56.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:56 smithi006 bash[33918]: cluster 2023-08-27T22:06:55.055625+0000 mgr.y (mgr.14901) 252 : cluster [DBG] pgmap v252: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:57.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:56 smithi130 bash[27579]: cluster 2023-08-27T22:06:55.055625+0000 mgr.y (mgr.14901) 252 : cluster [DBG] pgmap v252: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:57.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:57 smithi006 bash[33918]: audit 2023-08-27T22:06:56.928345+0000 mon.c (mon.2) 127 : audit [DBG] from='client.? 172.21.15.6:0/2098324446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:57.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:57 smithi006 bash[27292]: audit 2023-08-27T22:06:56.928345+0000 mon.c (mon.2) 127 : audit [DBG] from='client.? 172.21.15.6:0/2098324446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:58.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:57 smithi130 bash[27579]: audit 2023-08-27T22:06:56.928345+0000 mon.c (mon.2) 127 : audit [DBG] from='client.? 172.21.15.6:0/2098324446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:59.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:58 smithi130 bash[27579]: cluster 2023-08-27T22:06:57.056139+0000 mgr.y (mgr.14901) 253 : cluster [DBG] pgmap v253: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:59.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:58 smithi006 bash[27292]: cluster 2023-08-27T22:06:57.056139+0000 mgr.y (mgr.14901) 253 : cluster [DBG] pgmap v253: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:59.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:58 smithi006 bash[33918]: cluster 2023-08-27T22:06:57.056139+0000 mgr.y (mgr.14901) 253 : cluster [DBG] pgmap v253: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:06:59.743 INFO:teuthology.orchestra.run.smithi006.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-08-27T22:07:00.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:59 smithi130 bash[27579]: audit 2023-08-27T22:06:58.811992+0000 mon.c (mon.2) 128 : audit [INF] from='client.? 172.21.15.6:0/2012591864' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T22:07:00.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:59 smithi130 bash[27579]: audit 2023-08-27T22:06:58.812781+0000 mon.a (mon.0) 967 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T22:07:00.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:06:59 smithi130 bash[27579]: audit 2023-08-27T22:06:59.343764+0000 mon.c (mon.2) 129 : audit [DBG] from='client.? 172.21.15.6:0/3904782578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:00.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:59 smithi006 bash[27292]: audit 2023-08-27T22:06:58.811992+0000 mon.c (mon.2) 128 : audit [INF] from='client.? 172.21.15.6:0/2012591864' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T22:07:00.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:59 smithi006 bash[27292]: audit 2023-08-27T22:06:58.812781+0000 mon.a (mon.0) 967 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T22:07:00.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:06:59 smithi006 bash[27292]: audit 2023-08-27T22:06:59.343764+0000 mon.c (mon.2) 129 : audit [DBG] from='client.? 172.21.15.6:0/3904782578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:00.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:59 smithi006 bash[33918]: audit 2023-08-27T22:06:58.811992+0000 mon.c (mon.2) 128 : audit [INF] from='client.? 172.21.15.6:0/2012591864' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T22:07:00.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:59 smithi006 bash[33918]: audit 2023-08-27T22:06:58.812781+0000 mon.a (mon.0) 967 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T22:07:00.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:06:59 smithi006 bash[33918]: audit 2023-08-27T22:06:59.343764+0000 mon.c (mon.2) 129 : audit [DBG] from='client.? 172.21.15.6:0/3904782578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:00.971 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:07:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:07:00] "GET /metrics HTTP/1.1" 200 195129 "" "Prometheus/2.33.4" 2023-08-27T22:07:00.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:00 smithi006 bash[27292]: cluster 2023-08-27T22:06:59.056499+0000 mgr.y (mgr.14901) 254 : cluster [DBG] pgmap v254: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:07:00.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:00 smithi006 bash[27292]: audit 2023-08-27T22:06:59.729821+0000 mon.a (mon.0) 968 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-27T22:07:00.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:00 smithi006 bash[27292]: cluster 2023-08-27T22:06:59.729914+0000 mon.a (mon.0) 969 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-08-27T22:07:00.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:00 smithi006 bash[33918]: cluster 2023-08-27T22:06:59.056499+0000 mgr.y (mgr.14901) 254 : cluster [DBG] pgmap v254: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:07:00.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:00 smithi006 bash[33918]: audit 2023-08-27T22:06:59.729821+0000 mon.a (mon.0) 968 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-27T22:07:00.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:00 smithi006 bash[33918]: cluster 2023-08-27T22:06:59.729914+0000 mon.a (mon.0) 969 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-08-27T22:07:01.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:00 smithi130 bash[27579]: cluster 2023-08-27T22:06:59.056499+0000 mgr.y (mgr.14901) 254 : cluster [DBG] pgmap v254: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:07:01.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:00 smithi130 bash[27579]: audit 2023-08-27T22:06:59.729821+0000 mon.a (mon.0) 968 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-27T22:07:01.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:00 smithi130 bash[27579]: cluster 2023-08-27T22:06:59.729914+0000 mon.a (mon.0) 969 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-08-27T22:07:01.552 DEBUG:teuthology.orchestra.run.smithi006:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-08-27T22:07:01.555 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-08-27T22:07:01.555 INFO:teuthology.orchestra.run:waiting for 3000 2023-08-27T22:07:01.597 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:hints = 1 2023-08-27T22:07:01.597 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-08-27T22:07:01.597 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Object prefix: benchmark_data_smithi006_77448 2023-08-27T22:07:01.598 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-27T22:07:01.598 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 0 0 0 0 0 0 - 0 2023-08-27T22:07:01.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:07:01.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:01.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:07:01.658Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:02.598 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 1 16 2679 2663 166.427 166.438 0.00338938 0.0059778 2023-08-27T22:07:03.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:02 smithi130 bash[27579]: cluster 2023-08-27T22:07:01.057395+0000 mgr.y (mgr.14901) 255 : cluster [DBG] pgmap v256: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:07:03.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:02 smithi130 bash[27579]: audit 2023-08-27T22:07:01.926359+0000 mon.a (mon.0) 970 : audit [DBG] from='client.? 172.21.15.6:0/1896565437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:03.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:02 smithi006 bash[33918]: cluster 2023-08-27T22:07:01.057395+0000 mgr.y (mgr.14901) 255 : cluster [DBG] pgmap v256: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:07:03.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:02 smithi006 bash[33918]: audit 2023-08-27T22:07:01.926359+0000 mon.a (mon.0) 970 : audit [DBG] from='client.? 172.21.15.6:0/1896565437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:03.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:02 smithi006 bash[27292]: cluster 2023-08-27T22:07:01.057395+0000 mgr.y (mgr.14901) 255 : cluster [DBG] pgmap v256: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:07:03.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:02 smithi006 bash[27292]: audit 2023-08-27T22:07:01.926359+0000 mon.a (mon.0) 970 : audit [DBG] from='client.? 172.21.15.6:0/1896565437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:03.598 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 2 16 5316 5300 165.612 164.812 0.0088785 0.00602724 2023-08-27T22:07:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:07:03.549Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:07:03.551Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:04.598 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 3 16 7646 7630 158.945 145.625 0.00537468 0.00627834 2023-08-27T22:07:05.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:04 smithi130 bash[27579]: cluster 2023-08-27T22:07:03.057833+0000 mgr.y (mgr.14901) 256 : cluster [DBG] pgmap v257: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:07:05.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:04 smithi130 bash[27579]: audit 2023-08-27T22:07:04.645916+0000 mon.c (mon.2) 130 : audit [DBG] from='client.? 172.21.15.6:0/1788463274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:05.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:04 smithi006 bash[33918]: cluster 2023-08-27T22:07:03.057833+0000 mgr.y (mgr.14901) 256 : cluster [DBG] pgmap v257: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:07:05.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:04 smithi006 bash[33918]: audit 2023-08-27T22:07:04.645916+0000 mon.c (mon.2) 130 : audit [DBG] from='client.? 172.21.15.6:0/1788463274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:05.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:04 smithi006 bash[27292]: cluster 2023-08-27T22:07:03.057833+0000 mgr.y (mgr.14901) 256 : cluster [DBG] pgmap v257: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:07:05.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:04 smithi006 bash[27292]: audit 2023-08-27T22:07:04.645916+0000 mon.c (mon.2) 130 : audit [DBG] from='client.? 172.21.15.6:0/1788463274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:05.598 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 4 16 10040 10024 156.611 149.625 0.00673878 0.00637409 2023-08-27T22:07:06.598 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 5 16 12334 12318 153.961 143.375 0.00409013 0.00648443 2023-08-27T22:07:07.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:06 smithi130 bash[27579]: cluster 2023-08-27T22:07:05.058714+0000 mgr.y (mgr.14901) 257 : cluster [DBG] pgmap v258: 89 pgs: 89 active+clean; 480 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 48 MiB/s wr, 767 op/s 2023-08-27T22:07:07.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:06 smithi006 bash[27292]: cluster 2023-08-27T22:07:05.058714+0000 mgr.y (mgr.14901) 257 : cluster [DBG] pgmap v258: 89 pgs: 89 active+clean; 480 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 48 MiB/s wr, 767 op/s 2023-08-27T22:07:07.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:06 smithi006 bash[33918]: cluster 2023-08-27T22:07:05.058714+0000 mgr.y (mgr.14901) 257 : cluster [DBG] pgmap v258: 89 pgs: 89 active+clean; 480 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 48 MiB/s wr, 767 op/s 2023-08-27T22:07:07.598 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 6 16 14340 14324 149.194 125.375 0.00258061 0.00669425 2023-08-27T22:07:08.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:07 smithi130 bash[27579]: audit 2023-08-27T22:07:07.402571+0000 mon.c (mon.2) 131 : audit [DBG] from='client.? 172.21.15.6:0/255813939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:08.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:07 smithi006 bash[33918]: audit 2023-08-27T22:07:07.402571+0000 mon.c (mon.2) 131 : audit [DBG] from='client.? 172.21.15.6:0/255813939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:08.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:07 smithi006 bash[27292]: audit 2023-08-27T22:07:07.402571+0000 mon.c (mon.2) 131 : audit [DBG] from='client.? 172.21.15.6:0/255813939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:08.598 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 7 16 16480 16464 146.986 133.75 0.00547884 0.00679468 2023-08-27T22:07:09.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:08 smithi130 bash[27579]: cluster 2023-08-27T22:07:07.059227+0000 mgr.y (mgr.14901) 258 : cluster [DBG] pgmap v259: 89 pgs: 89 active+clean; 480 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 48 MiB/s wr, 767 op/s 2023-08-27T22:07:09.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:08 smithi006 bash[27292]: cluster 2023-08-27T22:07:07.059227+0000 mgr.y (mgr.14901) 258 : cluster [DBG] pgmap v259: 89 pgs: 89 active+clean; 480 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 48 MiB/s wr, 767 op/s 2023-08-27T22:07:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:08 smithi006 bash[33918]: cluster 2023-08-27T22:07:07.059227+0000 mgr.y (mgr.14901) 258 : cluster [DBG] pgmap v259: 89 pgs: 89 active+clean; 480 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 48 MiB/s wr, 767 op/s 2023-08-27T22:07:09.598 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 8 16 18450 18434 144.002 123.125 0.0101523 0.00693696 2023-08-27T22:07:10.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:09 smithi130 bash[27579]: audit 2023-08-27T22:07:08.836037+0000 mon.a (mon.0) 971 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:07:10.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:09 smithi130 bash[27579]: cluster 2023-08-27T22:07:09.059669+0000 mgr.y (mgr.14901) 259 : cluster [DBG] pgmap v260: 89 pgs: 89 active+clean; 480 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 48 MiB/s wr, 767 op/s 2023-08-27T22:07:10.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:09 smithi006 bash[33918]: audit 2023-08-27T22:07:08.836037+0000 mon.a (mon.0) 971 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:07:10.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:09 smithi006 bash[33918]: cluster 2023-08-27T22:07:09.059669+0000 mgr.y (mgr.14901) 259 : cluster [DBG] pgmap v260: 89 pgs: 89 active+clean; 480 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 48 MiB/s wr, 767 op/s 2023-08-27T22:07:10.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:09 smithi006 bash[27292]: audit 2023-08-27T22:07:08.836037+0000 mon.a (mon.0) 971 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:07:10.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:09 smithi006 bash[27292]: cluster 2023-08-27T22:07:09.059669+0000 mgr.y (mgr.14901) 259 : cluster [DBG] pgmap v260: 89 pgs: 89 active+clean; 480 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 48 MiB/s wr, 767 op/s 2023-08-27T22:07:10.598 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 9 16 20256 20240 140.542 112.875 0.00591991 0.00710706 2023-08-27T22:07:11.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:10 smithi130 bash[27579]: audit 2023-08-27T22:07:10.103367+0000 mon.b (mon.1) 124 : audit [DBG] from='client.? 172.21.15.6:0/646522834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:11.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:10 smithi006 bash[33918]: audit 2023-08-27T22:07:10.103367+0000 mon.b (mon.1) 124 : audit [DBG] from='client.? 172.21.15.6:0/646522834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:11.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:07:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:07:10] "GET /metrics HTTP/1.1" 200 195129 "" "Prometheus/2.33.4" 2023-08-27T22:07:11.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:10 smithi006 bash[27292]: audit 2023-08-27T22:07:10.103367+0000 mon.b (mon.1) 124 : audit [DBG] from='client.? 172.21.15.6:0/646522834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:11.598 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 10 16 22171 22155 138.456 119.688 0.00616225 0.00721512 2023-08-27T22:07:11.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:11 smithi006 bash[27292]: cluster 2023-08-27T22:07:11.061078+0000 mgr.y (mgr.14901) 260 : cluster [DBG] pgmap v261: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-08-27T22:07:11.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:11 smithi006 bash[33918]: cluster 2023-08-27T22:07:11.061078+0000 mgr.y (mgr.14901) 260 : cluster [DBG] pgmap v261: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-08-27T22:07:11.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:07:11.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:11.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:07:11.659Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:12.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:11 smithi130 bash[27579]: cluster 2023-08-27T22:07:11.061078+0000 mgr.y (mgr.14901) 260 : cluster [DBG] pgmap v261: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-08-27T22:07:12.599 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 11 16 23921 23905 135.811 109.375 0.0145634 0.00735404 2023-08-27T22:07:13.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:12 smithi130 bash[27579]: audit 2023-08-27T22:07:12.005326+0000 mon.b (mon.1) 125 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:07:13.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:12 smithi130 bash[27579]: audit 2023-08-27T22:07:12.738938+0000 mon.a (mon.0) 972 : audit [DBG] from='client.? 172.21.15.6:0/4236015452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:13.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:12 smithi006 bash[27292]: audit 2023-08-27T22:07:12.005326+0000 mon.b (mon.1) 125 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:07:13.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:12 smithi006 bash[27292]: audit 2023-08-27T22:07:12.738938+0000 mon.a (mon.0) 972 : audit [DBG] from='client.? 172.21.15.6:0/4236015452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:13.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:12 smithi006 bash[33918]: audit 2023-08-27T22:07:12.005326+0000 mon.b (mon.1) 125 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:07:13.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:12 smithi006 bash[33918]: audit 2023-08-27T22:07:12.738938+0000 mon.a (mon.0) 972 : audit [DBG] from='client.? 172.21.15.6:0/4236015452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:13.600 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 12 16 25631 25615 133.389 106.875 0.00869381 0.00748981 2023-08-27T22:07:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:07:13.550Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:07:13.552Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:14.600 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 13 16 27392 27376 131.594 110.062 0.00338385 0.00759169 2023-08-27T22:07:14.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:14 smithi130 bash[27579]: cluster 2023-08-27T22:07:13.061765+0000 mgr.y (mgr.14901) 261 : cluster [DBG] pgmap v262: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-08-27T22:07:14.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:14 smithi006 bash[33918]: cluster 2023-08-27T22:07:13.061765+0000 mgr.y (mgr.14901) 261 : cluster [DBG] pgmap v262: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-08-27T22:07:14.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:14 smithi006 bash[27292]: cluster 2023-08-27T22:07:13.061765+0000 mgr.y (mgr.14901) 261 : cluster [DBG] pgmap v262: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-08-27T22:07:15.600 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 14 16 28958 28942 129.185 97.875 0.00785108 0.00773369 2023-08-27T22:07:16.600 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 15 16 30609 30593 127.451 103.188 0.0138739 0.00783818 2023-08-27T22:07:16.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:16 smithi130 bash[27579]: cluster 2023-08-27T22:07:15.062653+0000 mgr.y (mgr.14901) 262 : cluster [DBG] pgmap v263: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-08-27T22:07:16.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:16 smithi130 bash[27579]: audit 2023-08-27T22:07:15.466465+0000 mon.c (mon.2) 132 : audit [DBG] from='client.? 172.21.15.6:0/2079972141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:16.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:16 smithi006 bash[27292]: cluster 2023-08-27T22:07:15.062653+0000 mgr.y (mgr.14901) 262 : cluster [DBG] pgmap v263: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-08-27T22:07:16.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:16 smithi006 bash[27292]: audit 2023-08-27T22:07:15.466465+0000 mon.c (mon.2) 132 : audit [DBG] from='client.? 172.21.15.6:0/2079972141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:16.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:16 smithi006 bash[33918]: cluster 2023-08-27T22:07:15.062653+0000 mgr.y (mgr.14901) 262 : cluster [DBG] pgmap v263: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-08-27T22:07:16.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:16 smithi006 bash[33918]: audit 2023-08-27T22:07:15.466465+0000 mon.c (mon.2) 132 : audit [DBG] from='client.? 172.21.15.6:0/2079972141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:17.600 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 16 16 32035 32019 125.053 89.125 0.0136925 0.0079902 2023-08-27T22:07:18.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:18 smithi130 bash[27579]: cluster 2023-08-27T22:07:17.063155+0000 mgr.y (mgr.14901) 263 : cluster [DBG] pgmap v264: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-08-27T22:07:18.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:18 smithi130 bash[27579]: audit 2023-08-27T22:07:18.165431+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:07:18.600 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 17 16 33237 33221 122.116 75.125 0.0106229 0.00818194 2023-08-27T22:07:18.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:18 smithi006 bash[33918]: cluster 2023-08-27T22:07:17.063155+0000 mgr.y (mgr.14901) 263 : cluster [DBG] pgmap v264: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-08-27T22:07:18.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:18 smithi006 bash[33918]: audit 2023-08-27T22:07:18.165431+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:07:18.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:18 smithi006 bash[27292]: cluster 2023-08-27T22:07:17.063155+0000 mgr.y (mgr.14901) 263 : cluster [DBG] pgmap v264: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-08-27T22:07:18.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:18 smithi006 bash[27292]: audit 2023-08-27T22:07:18.165431+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:07:19.600 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 18 16 34793 34777 120.734 97.25 0.00804361 0.00827532 2023-08-27T22:07:19.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:19 smithi130 bash[27579]: audit 2023-08-27T22:07:18.255380+0000 mon.c (mon.2) 133 : audit [DBG] from='client.? 172.21.15.6:0/1192990166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:19.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:19 smithi130 bash[27579]: audit 2023-08-27T22:07:18.377308+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:07:19.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:19 smithi130 bash[27579]: audit 2023-08-27T22:07:18.714639+0000 mon.b (mon.1) 126 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:07:19.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:19 smithi130 bash[27579]: audit 2023-08-27T22:07:18.715849+0000 mon.b (mon.1) 127 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:07:19.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:19 smithi130 bash[27579]: audit 2023-08-27T22:07:18.719074+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:07:19.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:19 smithi006 bash[33918]: audit 2023-08-27T22:07:18.255380+0000 mon.c (mon.2) 133 : audit [DBG] from='client.? 172.21.15.6:0/1192990166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:19.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:19 smithi006 bash[33918]: audit 2023-08-27T22:07:18.377308+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:07:19.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:19 smithi006 bash[33918]: audit 2023-08-27T22:07:18.714639+0000 mon.b (mon.1) 126 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:07:19.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:19 smithi006 bash[33918]: audit 2023-08-27T22:07:18.715849+0000 mon.b (mon.1) 127 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:07:19.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:19 smithi006 bash[33918]: audit 2023-08-27T22:07:18.719074+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:07:19.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:19 smithi006 bash[27292]: audit 2023-08-27T22:07:18.255380+0000 mon.c (mon.2) 133 : audit [DBG] from='client.? 172.21.15.6:0/1192990166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:19.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:19 smithi006 bash[27292]: audit 2023-08-27T22:07:18.377308+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:07:19.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:19 smithi006 bash[27292]: audit 2023-08-27T22:07:18.714639+0000 mon.b (mon.1) 126 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:07:19.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:19 smithi006 bash[27292]: audit 2023-08-27T22:07:18.715849+0000 mon.b (mon.1) 127 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:07:19.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:19 smithi006 bash[27292]: audit 2023-08-27T22:07:18.719074+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:07:20.600 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 19 16 36212 36196 119.047 88.6875 0.0139194 0.0083918 2023-08-27T22:07:21.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:20 smithi130 bash[27579]: cluster 2023-08-27T22:07:19.063615+0000 mgr.y (mgr.14901) 264 : cluster [DBG] pgmap v265: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-08-27T22:07:21.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:20 smithi006 bash[27292]: cluster 2023-08-27T22:07:19.063615+0000 mgr.y (mgr.14901) 264 : cluster [DBG] pgmap v265: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-08-27T22:07:21.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:07:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:07:20] "GET /metrics HTTP/1.1" 200 195192 "" "Prometheus/2.33.4" 2023-08-27T22:07:21.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:20 smithi006 bash[33918]: cluster 2023-08-27T22:07:19.063615+0000 mgr.y (mgr.14901) 264 : cluster [DBG] pgmap v265: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-08-27T22:07:21.601 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:2023-08-27T22:07:21.594827+0000 min lat: 0.00150916 max lat: 0.0720339 avg lat: 0.00849736 2023-08-27T22:07:21.601 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-27T22:07:21.601 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 20 16 37650 37634 117.588 89.875 0.00486823 0.00849736 2023-08-27T22:07:21.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:07:21.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:21.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:07:21.660Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:21.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:21 smithi006 bash[33918]: audit 2023-08-27T22:07:20.959538+0000 mon.a (mon.0) 976 : audit [DBG] from='client.? 172.21.15.6:0/365361190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:21.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:21 smithi006 bash[27292]: audit 2023-08-27T22:07:20.959538+0000 mon.a (mon.0) 976 : audit [DBG] from='client.? 172.21.15.6:0/365361190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:22.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:21 smithi130 bash[27579]: audit 2023-08-27T22:07:20.959538+0000 mon.a (mon.0) 976 : audit [DBG] from='client.? 172.21.15.6:0/365361190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:22.601 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 21 16 39109 39093 116.331 91.1875 0.0114784 0.00858915 2023-08-27T22:07:23.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:22 smithi130 bash[27579]: cluster 2023-08-27T22:07:21.064425+0000 mgr.y (mgr.14901) 265 : cluster [DBG] pgmap v266: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-08-27T22:07:23.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:22 smithi006 bash[33918]: cluster 2023-08-27T22:07:21.064425+0000 mgr.y (mgr.14901) 265 : cluster [DBG] pgmap v266: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-08-27T22:07:23.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:22 smithi006 bash[27292]: cluster 2023-08-27T22:07:21.064425+0000 mgr.y (mgr.14901) 265 : cluster [DBG] pgmap v266: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-08-27T22:07:23.601 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 22 16 40450 40434 114.852 83.8125 0.0164015 0.00870026 2023-08-27T22:07:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:07:23.551Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:07:23.553Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:24.601 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 23 16 41836 41820 113.625 86.625 0.00796647 0.00879474 2023-08-27T22:07:24.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:24 smithi130 bash[27579]: cluster 2023-08-27T22:07:23.065138+0000 mgr.y (mgr.14901) 266 : cluster [DBG] pgmap v267: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-08-27T22:07:24.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:24 smithi130 bash[27579]: audit 2023-08-27T22:07:23.679792+0000 mon.c (mon.2) 134 : audit [DBG] from='client.? 172.21.15.6:0/2080281609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:24.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:24 smithi006 bash[33918]: cluster 2023-08-27T22:07:23.065138+0000 mgr.y (mgr.14901) 266 : cluster [DBG] pgmap v267: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-08-27T22:07:24.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:24 smithi006 bash[33918]: audit 2023-08-27T22:07:23.679792+0000 mon.c (mon.2) 134 : audit [DBG] from='client.? 172.21.15.6:0/2080281609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:24.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:24 smithi006 bash[27292]: cluster 2023-08-27T22:07:23.065138+0000 mgr.y (mgr.14901) 266 : cluster [DBG] pgmap v267: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-08-27T22:07:24.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:24 smithi006 bash[27292]: audit 2023-08-27T22:07:23.679792+0000 mon.c (mon.2) 134 : audit [DBG] from='client.? 172.21.15.6:0/2080281609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:25.601 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 24 16 43233 43217 112.528 87.3125 0.0188374 0.00887961 2023-08-27T22:07:26.601 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 25 16 44514 44498 111.229 80.0625 0.0194961 0.00898429 2023-08-27T22:07:26.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:26 smithi130 bash[27579]: cluster 2023-08-27T22:07:25.068008+0000 mgr.y (mgr.14901) 267 : cluster [DBG] pgmap v268: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-08-27T22:07:26.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:26 smithi006 bash[33918]: cluster 2023-08-27T22:07:25.068008+0000 mgr.y (mgr.14901) 267 : cluster [DBG] pgmap v268: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-08-27T22:07:26.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:26 smithi006 bash[27292]: cluster 2023-08-27T22:07:25.068008+0000 mgr.y (mgr.14901) 267 : cluster [DBG] pgmap v268: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-08-27T22:07:27.601 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 26 16 45857 45841 110.179 83.9375 0.00927003 0.00906947 2023-08-27T22:07:27.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:27 smithi130 bash[27579]: audit 2023-08-27T22:07:26.389267+0000 mon.a (mon.0) 977 : audit [DBG] from='client.? 172.21.15.6:0/2972226630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:27.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:27 smithi006 bash[27292]: audit 2023-08-27T22:07:26.389267+0000 mon.a (mon.0) 977 : audit [DBG] from='client.? 172.21.15.6:0/2972226630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:27.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:27 smithi006 bash[33918]: audit 2023-08-27T22:07:26.389267+0000 mon.a (mon.0) 977 : audit [DBG] from='client.? 172.21.15.6:0/2972226630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:28.601 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 27 16 47140 47124 109.068 80.1875 0.0144787 0.00916224 2023-08-27T22:07:28.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:28 smithi130 bash[27579]: cluster 2023-08-27T22:07:27.068400+0000 mgr.y (mgr.14901) 268 : cluster [DBG] pgmap v269: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-08-27T22:07:28.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:28 smithi006 bash[27292]: cluster 2023-08-27T22:07:27.068400+0000 mgr.y (mgr.14901) 268 : cluster [DBG] pgmap v269: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-08-27T22:07:28.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:28 smithi006 bash[33918]: cluster 2023-08-27T22:07:27.068400+0000 mgr.y (mgr.14901) 268 : cluster [DBG] pgmap v269: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-08-27T22:07:29.601 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 28 15 48357 48342 107.891 76.125 0.0148977 0.00926122 2023-08-27T22:07:29.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:29 smithi130 bash[27579]: audit 2023-08-27T22:07:29.097620+0000 mon.c (mon.2) 135 : audit [DBG] from='client.? 172.21.15.6:0/1755356889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:29.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:29 smithi006 bash[33918]: audit 2023-08-27T22:07:29.097620+0000 mon.c (mon.2) 135 : audit [DBG] from='client.? 172.21.15.6:0/1755356889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:29.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:29 smithi006 bash[27292]: audit 2023-08-27T22:07:29.097620+0000 mon.c (mon.2) 135 : audit [DBG] from='client.? 172.21.15.6:0/1755356889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:30.601 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 29 16 49646 49630 106.947 80.5 0.00926439 0.0093435 2023-08-27T22:07:30.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:30 smithi130 bash[27579]: cluster 2023-08-27T22:07:29.068879+0000 mgr.y (mgr.14901) 269 : cluster [DBG] pgmap v270: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-08-27T22:07:30.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:30 smithi006 bash[27292]: cluster 2023-08-27T22:07:29.068879+0000 mgr.y (mgr.14901) 269 : cluster [DBG] pgmap v270: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-08-27T22:07:30.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:30 smithi006 bash[33918]: cluster 2023-08-27T22:07:29.068879+0000 mgr.y (mgr.14901) 269 : cluster [DBG] pgmap v270: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-08-27T22:07:31.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:07:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:07:30] "GET /metrics HTTP/1.1" 200 195211 "" "Prometheus/2.33.4" 2023-08-27T22:07:31.601 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 30 16 50809 50793 105.805 72.6875 0.00376878 0.00944295 2023-08-27T22:07:31.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:07:31.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:31.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:07:31.660Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:32.601 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 31 16 52007 51991 104.807 74.875 0.00933033 0.00953465 2023-08-27T22:07:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:32 smithi130 bash[27579]: cluster 2023-08-27T22:07:31.069667+0000 mgr.y (mgr.14901) 270 : cluster [DBG] pgmap v271: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-08-27T22:07:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:32 smithi130 bash[27579]: audit 2023-08-27T22:07:31.789334+0000 mon.c (mon.2) 136 : audit [DBG] from='client.? 172.21.15.6:0/1996118029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:32.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:32 smithi006 bash[56298]: level=warn ts=2023-08-27T22:07:32.313Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:32.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:32 smithi006 bash[33918]: cluster 2023-08-27T22:07:31.069667+0000 mgr.y (mgr.14901) 270 : cluster [DBG] pgmap v271: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-08-27T22:07:32.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:32 smithi006 bash[33918]: audit 2023-08-27T22:07:31.789334+0000 mon.c (mon.2) 136 : audit [DBG] from='client.? 172.21.15.6:0/1996118029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:32.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:32 smithi006 bash[27292]: cluster 2023-08-27T22:07:31.069667+0000 mgr.y (mgr.14901) 270 : cluster [DBG] pgmap v271: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-08-27T22:07:32.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:32 smithi006 bash[27292]: audit 2023-08-27T22:07:31.789334+0000 mon.c (mon.2) 136 : audit [DBG] from='client.? 172.21.15.6:0/1996118029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:33.601 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 32 16 53218 53202 103.896 75.6875 0.00719304 0.00961762 2023-08-27T22:07:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:07:33.552Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:07:33.553Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:34.601 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 33 16 54331 54315 102.856 69.5625 0.00759316 0.00971552 2023-08-27T22:07:34.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:34 smithi130 bash[27579]: cluster 2023-08-27T22:07:33.070079+0000 mgr.y (mgr.14901) 271 : cluster [DBG] pgmap v272: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2023-08-27T22:07:34.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:34 smithi006 bash[27292]: cluster 2023-08-27T22:07:33.070079+0000 mgr.y (mgr.14901) 271 : cluster [DBG] pgmap v272: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2023-08-27T22:07:34.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:34 smithi006 bash[33918]: cluster 2023-08-27T22:07:33.070079+0000 mgr.y (mgr.14901) 271 : cluster [DBG] pgmap v272: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2023-08-27T22:07:35.602 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 34 16 55515 55499 102.007 74 0.00374407 0.00979615 2023-08-27T22:07:35.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:35 smithi130 bash[27579]: audit 2023-08-27T22:07:34.497797+0000 mon.b (mon.1) 128 : audit [DBG] from='client.? 172.21.15.6:0/3132833974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:35.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:35 smithi006 bash[27292]: audit 2023-08-27T22:07:34.497797+0000 mon.b (mon.1) 128 : audit [DBG] from='client.? 172.21.15.6:0/3132833974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:35.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:35 smithi006 bash[33918]: audit 2023-08-27T22:07:34.497797+0000 mon.b (mon.1) 128 : audit [DBG] from='client.? 172.21.15.6:0/3132833974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:36.602 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 35 16 56695 56679 101.199 73.75 0.0163746 0.00987549 2023-08-27T22:07:36.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:36 smithi130 bash[27579]: cluster 2023-08-27T22:07:35.070876+0000 mgr.y (mgr.14901) 272 : cluster [DBG] pgmap v273: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-08-27T22:07:36.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:36 smithi006 bash[33918]: cluster 2023-08-27T22:07:35.070876+0000 mgr.y (mgr.14901) 272 : cluster [DBG] pgmap v273: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-08-27T22:07:36.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:36 smithi006 bash[27292]: cluster 2023-08-27T22:07:35.070876+0000 mgr.y (mgr.14901) 272 : cluster [DBG] pgmap v273: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-08-27T22:07:37.602 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 36 16 57753 57737 100.225 66.125 0.0046275 0.00996979 2023-08-27T22:07:37.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:37 smithi130 bash[27579]: audit 2023-08-27T22:07:37.224550+0000 mon.b (mon.1) 129 : audit [DBG] from='client.? 172.21.15.6:0/1851354561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:37.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:37 smithi006 bash[27292]: audit 2023-08-27T22:07:37.224550+0000 mon.b (mon.1) 129 : audit [DBG] from='client.? 172.21.15.6:0/1851354561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:37.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:37 smithi006 bash[33918]: audit 2023-08-27T22:07:37.224550+0000 mon.b (mon.1) 129 : audit [DBG] from='client.? 172.21.15.6:0/1851354561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:38.602 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 37 16 58906 58890 99.4637 72.0625 0.00868823 0.0100464 2023-08-27T22:07:38.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:38 smithi130 bash[27579]: cluster 2023-08-27T22:07:37.071379+0000 mgr.y (mgr.14901) 273 : cluster [DBG] pgmap v274: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-08-27T22:07:38.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:38 smithi006 bash[27292]: cluster 2023-08-27T22:07:37.071379+0000 mgr.y (mgr.14901) 273 : cluster [DBG] pgmap v274: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-08-27T22:07:38.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:38 smithi006 bash[33918]: cluster 2023-08-27T22:07:37.071379+0000 mgr.y (mgr.14901) 273 : cluster [DBG] pgmap v274: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-08-27T22:07:39.602 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 38 16 59995 59979 98.6371 68.0625 0.00537453 0.0101319 2023-08-27T22:07:40.602 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 39 16 61078 61062 97.8434 67.6875 0.0191138 0.0102143 2023-08-27T22:07:40.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:40 smithi130 bash[27579]: cluster 2023-08-27T22:07:39.071889+0000 mgr.y (mgr.14901) 274 : cluster [DBG] pgmap v275: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-08-27T22:07:40.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:40 smithi130 bash[27579]: audit 2023-08-27T22:07:39.917256+0000 mon.b (mon.1) 130 : audit [DBG] from='client.? 172.21.15.6:0/1821781961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:40.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:40 smithi006 bash[33918]: cluster 2023-08-27T22:07:39.071889+0000 mgr.y (mgr.14901) 274 : cluster [DBG] pgmap v275: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-08-27T22:07:40.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:40 smithi006 bash[33918]: audit 2023-08-27T22:07:39.917256+0000 mon.b (mon.1) 130 : audit [DBG] from='client.? 172.21.15.6:0/1821781961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:40.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:40 smithi006 bash[27292]: cluster 2023-08-27T22:07:39.071889+0000 mgr.y (mgr.14901) 274 : cluster [DBG] pgmap v275: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-08-27T22:07:40.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:40 smithi006 bash[27292]: audit 2023-08-27T22:07:39.917256+0000 mon.b (mon.1) 130 : audit [DBG] from='client.? 172.21.15.6:0/1821781961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:41.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:07:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:07:40] "GET /metrics HTTP/1.1" 200 195211 "" "Prometheus/2.33.4" 2023-08-27T22:07:41.602 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:2023-08-27T22:07:41.596815+0000 min lat: 0.00150916 max lat: 0.0720339 avg lat: 0.0102888 2023-08-27T22:07:41.602 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-27T22:07:41.603 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 40 16 62184 62168 97.1253 69.125 0.00955249 0.0102888 2023-08-27T22:07:41.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:07:41.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:41.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:07:41.660Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:42.602 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 41 16 63229 63213 96.3492 65.3125 0.0120392 0.0103719 2023-08-27T22:07:42.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:42 smithi130 bash[27579]: cluster 2023-08-27T22:07:41.072725+0000 mgr.y (mgr.14901) 275 : cluster [DBG] pgmap v276: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 95 MiB/s wr, 1.51k op/s 2023-08-27T22:07:42.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:42 smithi006 bash[27292]: cluster 2023-08-27T22:07:41.072725+0000 mgr.y (mgr.14901) 275 : cluster [DBG] pgmap v276: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 95 MiB/s wr, 1.51k op/s 2023-08-27T22:07:42.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:42 smithi006 bash[33918]: cluster 2023-08-27T22:07:41.072725+0000 mgr.y (mgr.14901) 275 : cluster [DBG] pgmap v276: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 95 MiB/s wr, 1.51k op/s 2023-08-27T22:07:42.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:42 smithi006 bash[56298]: level=error ts=2023-08-27T22:07:42.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:42.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:42 smithi006 bash[56298]: level=warn ts=2023-08-27T22:07:42.315Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:43.602 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 42 16 64359 64343 95.7366 70.625 0.0120454 0.0104385 2023-08-27T22:07:43.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:43 smithi130 bash[27579]: audit 2023-08-27T22:07:42.575017+0000 mon.b (mon.1) 131 : audit [DBG] from='client.? 172.21.15.6:0/4002469418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:43.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:43 smithi006 bash[27292]: audit 2023-08-27T22:07:42.575017+0000 mon.b (mon.1) 131 : audit [DBG] from='client.? 172.21.15.6:0/4002469418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:43.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:43 smithi006 bash[33918]: audit 2023-08-27T22:07:42.575017+0000 mon.b (mon.1) 131 : audit [DBG] from='client.? 172.21.15.6:0/4002469418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:43.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:07:43.552Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:43.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:07:43.554Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:44.603 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 43 16 65519 65503 95.196 72.5 0.0128417 0.0104972 2023-08-27T22:07:44.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:44 smithi130 bash[27579]: cluster 2023-08-27T22:07:43.073160+0000 mgr.y (mgr.14901) 276 : cluster [DBG] pgmap v277: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2023-08-27T22:07:44.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:44 smithi006 bash[27292]: cluster 2023-08-27T22:07:43.073160+0000 mgr.y (mgr.14901) 276 : cluster [DBG] pgmap v277: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2023-08-27T22:07:44.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:44 smithi006 bash[33918]: cluster 2023-08-27T22:07:43.073160+0000 mgr.y (mgr.14901) 276 : cluster [DBG] pgmap v277: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2023-08-27T22:07:45.603 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 44 16 66512 66496 94.4429 62.0625 0.00285786 0.0105718 2023-08-27T22:07:45.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:45 smithi130 bash[27579]: audit 2023-08-27T22:07:45.226718+0000 mon.a (mon.0) 978 : audit [DBG] from='client.? 172.21.15.6:0/3887728329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:45.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:45 smithi006 bash[27292]: audit 2023-08-27T22:07:45.226718+0000 mon.a (mon.0) 978 : audit [DBG] from='client.? 172.21.15.6:0/3887728329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:45.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:45 smithi006 bash[33918]: audit 2023-08-27T22:07:45.226718+0000 mon.a (mon.0) 978 : audit [DBG] from='client.? 172.21.15.6:0/3887728329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:46.603 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 45 16 67572 67556 93.8163 66.25 0.00493604 0.0106519 2023-08-27T22:07:46.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:46 smithi130 bash[27579]: cluster 2023-08-27T22:07:45.073973+0000 mgr.y (mgr.14901) 277 : cluster [DBG] pgmap v278: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-08-27T22:07:46.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:46 smithi006 bash[27292]: cluster 2023-08-27T22:07:45.073973+0000 mgr.y (mgr.14901) 277 : cluster [DBG] pgmap v278: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-08-27T22:07:46.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:46 smithi006 bash[33918]: cluster 2023-08-27T22:07:45.073973+0000 mgr.y (mgr.14901) 277 : cluster [DBG] pgmap v278: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-08-27T22:07:47.603 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 46 16 68569 68553 93.1313 62.3125 0.0247533 0.0107304 2023-08-27T22:07:48.603 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 47 16 69538 69522 92.4382 60.5625 0.0104734 0.010812 2023-08-27T22:07:48.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:48 smithi130 bash[27579]: cluster 2023-08-27T22:07:47.074414+0000 mgr.y (mgr.14901) 278 : cluster [DBG] pgmap v279: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 929 op/s 2023-08-27T22:07:48.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:48 smithi130 bash[27579]: audit 2023-08-27T22:07:47.946797+0000 mon.b (mon.1) 132 : audit [DBG] from='client.? 172.21.15.6:0/3870557512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:48.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:48 smithi006 bash[33918]: cluster 2023-08-27T22:07:47.074414+0000 mgr.y (mgr.14901) 278 : cluster [DBG] pgmap v279: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 929 op/s 2023-08-27T22:07:48.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:48 smithi006 bash[33918]: audit 2023-08-27T22:07:47.946797+0000 mon.b (mon.1) 132 : audit [DBG] from='client.? 172.21.15.6:0/3870557512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:48.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:48 smithi006 bash[27292]: cluster 2023-08-27T22:07:47.074414+0000 mgr.y (mgr.14901) 278 : cluster [DBG] pgmap v279: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 929 op/s 2023-08-27T22:07:48.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:48 smithi006 bash[27292]: audit 2023-08-27T22:07:47.946797+0000 mon.b (mon.1) 132 : audit [DBG] from='client.? 172.21.15.6:0/3870557512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:49.603 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 48 16 70556 70540 91.8378 63.625 0.00945879 0.0108821 2023-08-27T22:07:50.603 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 49 16 71493 71477 91.1587 58.5625 0.0124204 0.0109637 2023-08-27T22:07:50.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:50 smithi130 bash[27579]: cluster 2023-08-27T22:07:49.074904+0000 mgr.y (mgr.14901) 279 : cluster [DBG] pgmap v280: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 929 op/s 2023-08-27T22:07:50.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:50 smithi006 bash[27292]: cluster 2023-08-27T22:07:49.074904+0000 mgr.y (mgr.14901) 279 : cluster [DBG] pgmap v280: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 929 op/s 2023-08-27T22:07:50.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:50 smithi006 bash[33918]: cluster 2023-08-27T22:07:49.074904+0000 mgr.y (mgr.14901) 279 : cluster [DBG] pgmap v280: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 929 op/s 2023-08-27T22:07:51.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:07:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:07:50] "GET /metrics HTTP/1.1" 200 195218 "" "Prometheus/2.33.4" 2023-08-27T22:07:51.603 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 50 16 72479 72463 90.5679 61.625 0.0135136 0.0110352 2023-08-27T22:07:51.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:51 smithi130 bash[27579]: audit 2023-08-27T22:07:50.669523+0000 mon.a (mon.0) 979 : audit [DBG] from='client.? 172.21.15.6:0/653689344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:51.667 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:51 smithi006 bash[27292]: audit 2023-08-27T22:07:50.669523+0000 mon.a (mon.0) 979 : audit [DBG] from='client.? 172.21.15.6:0/653689344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:51.667 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:51 smithi006 bash[33918]: audit 2023-08-27T22:07:50.669523+0000 mon.a (mon.0) 979 : audit [DBG] from='client.? 172.21.15.6:0/653689344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:51.667 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:07:51.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:51.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:07:51.661Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:52.603 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 51 16 73467 73451 90.0027 61.75 0.0128273 0.0111036 2023-08-27T22:07:52.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:52 smithi130 bash[27579]: cluster 2023-08-27T22:07:51.075715+0000 mgr.y (mgr.14901) 280 : cluster [DBG] pgmap v281: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-08-27T22:07:52.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:52 smithi006 bash[27292]: cluster 2023-08-27T22:07:51.075715+0000 mgr.y (mgr.14901) 280 : cluster [DBG] pgmap v281: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-08-27T22:07:52.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:52 smithi006 bash[33918]: cluster 2023-08-27T22:07:51.075715+0000 mgr.y (mgr.14901) 280 : cluster [DBG] pgmap v281: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-08-27T22:07:52.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:52 smithi006 bash[56298]: level=error ts=2023-08-27T22:07:52.314Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:52.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:52 smithi006 bash[56298]: level=warn ts=2023-08-27T22:07:52.315Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:53.603 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 52 16 74375 74359 89.3632 56.75 0.0154681 0.011183 2023-08-27T22:07:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:07:53.553Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:07:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:07:53.554Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:07:54.603 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 53 16 75348 75332 88.8244 60.8125 0.0204142 0.0112524 2023-08-27T22:07:54.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:54 smithi130 bash[27579]: cluster 2023-08-27T22:07:53.076181+0000 mgr.y (mgr.14901) 281 : cluster [DBG] pgmap v282: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s 2023-08-27T22:07:54.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:54 smithi130 bash[27579]: audit 2023-08-27T22:07:53.378603+0000 mon.c (mon.2) 137 : audit [DBG] from='client.? 172.21.15.6:0/3433339179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:54.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:54 smithi006 bash[27292]: cluster 2023-08-27T22:07:53.076181+0000 mgr.y (mgr.14901) 281 : cluster [DBG] pgmap v282: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s 2023-08-27T22:07:54.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:54 smithi006 bash[27292]: audit 2023-08-27T22:07:53.378603+0000 mon.c (mon.2) 137 : audit [DBG] from='client.? 172.21.15.6:0/3433339179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:54.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:54 smithi006 bash[33918]: cluster 2023-08-27T22:07:53.076181+0000 mgr.y (mgr.14901) 281 : cluster [DBG] pgmap v282: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s 2023-08-27T22:07:54.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:54 smithi006 bash[33918]: audit 2023-08-27T22:07:53.378603+0000 mon.c (mon.2) 137 : audit [DBG] from='client.? 172.21.15.6:0/3433339179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:55.604 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 54 16 76298 76282 88.2789 59.375 0.00803347 0.0113215 2023-08-27T22:07:56.603 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 55 16 77212 77196 87.7124 57.125 0.0098989 0.0113946 2023-08-27T22:07:56.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:56 smithi006 bash[27292]: cluster 2023-08-27T22:07:55.076945+0000 mgr.y (mgr.14901) 282 : cluster [DBG] pgmap v283: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-08-27T22:07:56.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:56 smithi006 bash[27292]: audit 2023-08-27T22:07:56.069745+0000 mon.b (mon.1) 133 : audit [DBG] from='client.? 172.21.15.6:0/2789930118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:56.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:56 smithi006 bash[33918]: cluster 2023-08-27T22:07:55.076945+0000 mgr.y (mgr.14901) 282 : cluster [DBG] pgmap v283: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-08-27T22:07:56.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:56 smithi006 bash[33918]: audit 2023-08-27T22:07:56.069745+0000 mon.b (mon.1) 133 : audit [DBG] from='client.? 172.21.15.6:0/2789930118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:56.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:56 smithi130 bash[27579]: cluster 2023-08-27T22:07:55.076945+0000 mgr.y (mgr.14901) 282 : cluster [DBG] pgmap v283: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-08-27T22:07:56.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:56 smithi130 bash[27579]: audit 2023-08-27T22:07:56.069745+0000 mon.b (mon.1) 133 : audit [DBG] from='client.? 172.21.15.6:0/2789930118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:57.603 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 56 16 78186 78170 87.2331 60.875 0.00533973 0.0114575 2023-08-27T22:07:58.604 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 57 16 79054 79038 86.6543 54.25 0.0188909 0.0115329 2023-08-27T22:07:58.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:58 smithi006 bash[27292]: cluster 2023-08-27T22:07:57.077316+0000 mgr.y (mgr.14901) 283 : cluster [DBG] pgmap v284: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2023-08-27T22:07:58.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:58 smithi006 bash[33918]: cluster 2023-08-27T22:07:57.077316+0000 mgr.y (mgr.14901) 283 : cluster [DBG] pgmap v284: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2023-08-27T22:07:58.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:58 smithi130 bash[27579]: cluster 2023-08-27T22:07:57.077316+0000 mgr.y (mgr.14901) 283 : cluster [DBG] pgmap v284: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2023-08-27T22:07:59.604 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 58 16 79975 79959 86.1527 57.5625 0.0141734 0.011601 2023-08-27T22:07:59.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:07:59 smithi006 bash[27292]: audit 2023-08-27T22:07:58.745545+0000 mon.a (mon.0) 980 : audit [DBG] from='client.? 172.21.15.6:0/4254250063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:59.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:07:59 smithi006 bash[33918]: audit 2023-08-27T22:07:58.745545+0000 mon.a (mon.0) 980 : audit [DBG] from='client.? 172.21.15.6:0/4254250063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:59.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:07:59 smithi130 bash[27579]: audit 2023-08-27T22:07:58.745545+0000 mon.a (mon.0) 980 : audit [DBG] from='client.? 172.21.15.6:0/4254250063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:00.604 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 59 16 80907 80891 85.6797 58.25 0.0269789 0.0116635 2023-08-27T22:08:00.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:00 smithi006 bash[27292]: cluster 2023-08-27T22:07:59.077770+0000 mgr.y (mgr.14901) 284 : cluster [DBG] pgmap v285: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2023-08-27T22:08:00.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:00 smithi006 bash[33918]: cluster 2023-08-27T22:07:59.077770+0000 mgr.y (mgr.14901) 284 : cluster [DBG] pgmap v285: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2023-08-27T22:08:00.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:00 smithi130 bash[27579]: cluster 2023-08-27T22:07:59.077770+0000 mgr.y (mgr.14901) 284 : cluster [DBG] pgmap v285: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2023-08-27T22:08:01.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:08:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:08:00] "GET /metrics HTTP/1.1" 200 195211 "" "Prometheus/2.33.4" 2023-08-27T22:08:01.604 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:2023-08-27T22:08:01.598723+0000 min lat: 0.00150916 max lat: 0.091257 avg lat: 0.0117364 2023-08-27T22:08:01.604 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-27T22:08:01.604 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 60 16 81778 81762 85.1589 54.4375 0.00664512 0.0117364 2023-08-27T22:08:01.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:08:01.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:01.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:08:01.662Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:02.604 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 61 16 82691 82675 84.6981 57.0625 0.0168771 0.0117994 2023-08-27T22:08:02.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:02 smithi006 bash[56298]: level=error ts=2023-08-27T22:08:02.318Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:02.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:02 smithi006 bash[56298]: level=warn ts=2023-08-27T22:08:02.319Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:02.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:02 smithi006 bash[33918]: cluster 2023-08-27T22:08:01.078555+0000 mgr.y (mgr.14901) 285 : cluster [DBG] pgmap v286: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-08-27T22:08:02.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:02 smithi006 bash[33918]: audit 2023-08-27T22:08:01.445127+0000 mon.a (mon.0) 981 : audit [DBG] from='client.? 172.21.15.6:0/2518993223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:02.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:02 smithi006 bash[27292]: cluster 2023-08-27T22:08:01.078555+0000 mgr.y (mgr.14901) 285 : cluster [DBG] pgmap v286: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-08-27T22:08:02.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:02 smithi006 bash[27292]: audit 2023-08-27T22:08:01.445127+0000 mon.a (mon.0) 981 : audit [DBG] from='client.? 172.21.15.6:0/2518993223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:02.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:02 smithi130 bash[27579]: cluster 2023-08-27T22:08:01.078555+0000 mgr.y (mgr.14901) 285 : cluster [DBG] pgmap v286: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-08-27T22:08:02.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:02 smithi130 bash[27579]: audit 2023-08-27T22:08:01.445127+0000 mon.a (mon.0) 981 : audit [DBG] from='client.? 172.21.15.6:0/2518993223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:03.604 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 62 16 83596 83580 84.2443 56.5625 0.0262076 0.0118636 2023-08-27T22:08:03.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:08:03.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:03.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:08:03.556Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:04.604 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 63 16 84455 84439 83.7592 53.6875 0.00897111 0.0119324 2023-08-27T22:08:04.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:04 smithi006 bash[27292]: cluster 2023-08-27T22:08:03.079016+0000 mgr.y (mgr.14901) 286 : cluster [DBG] pgmap v287: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2023-08-27T22:08:04.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:04 smithi006 bash[27292]: audit 2023-08-27T22:08:04.134113+0000 mon.b (mon.1) 134 : audit [DBG] from='client.? 172.21.15.6:0/4136459560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:04.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:04 smithi006 bash[33918]: cluster 2023-08-27T22:08:03.079016+0000 mgr.y (mgr.14901) 286 : cluster [DBG] pgmap v287: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2023-08-27T22:08:04.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:04 smithi006 bash[33918]: audit 2023-08-27T22:08:04.134113+0000 mon.b (mon.1) 134 : audit [DBG] from='client.? 172.21.15.6:0/4136459560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:04.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:04 smithi130 bash[27579]: cluster 2023-08-27T22:08:03.079016+0000 mgr.y (mgr.14901) 286 : cluster [DBG] pgmap v287: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2023-08-27T22:08:04.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:04 smithi130 bash[27579]: audit 2023-08-27T22:08:04.134113+0000 mon.b (mon.1) 134 : audit [DBG] from='client.? 172.21.15.6:0/4136459560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:05.604 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 64 16 85361 85345 83.3351 56.625 0.00991551 0.0119933 2023-08-27T22:08:06.604 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 65 16 86228 86212 82.8866 54.1875 0.0113321 0.0120571 2023-08-27T22:08:06.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:06 smithi006 bash[27292]: cluster 2023-08-27T22:08:05.079819+0000 mgr.y (mgr.14901) 287 : cluster [DBG] pgmap v288: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-08-27T22:08:06.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:06 smithi006 bash[33918]: cluster 2023-08-27T22:08:05.079819+0000 mgr.y (mgr.14901) 287 : cluster [DBG] pgmap v288: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-08-27T22:08:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:06 smithi130 bash[27579]: cluster 2023-08-27T22:08:05.079819+0000 mgr.y (mgr.14901) 287 : cluster [DBG] pgmap v288: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-08-27T22:08:07.604 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 66 16 87078 87062 82.4356 53.125 0.0165094 0.0121242 2023-08-27T22:08:07.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:07 smithi006 bash[33918]: audit 2023-08-27T22:08:06.827746+0000 mon.a (mon.0) 982 : audit [DBG] from='client.? 172.21.15.6:0/1162409084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:07.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:07 smithi006 bash[27292]: audit 2023-08-27T22:08:06.827746+0000 mon.a (mon.0) 982 : audit [DBG] from='client.? 172.21.15.6:0/1162409084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:07.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:07 smithi130 bash[27579]: audit 2023-08-27T22:08:06.827746+0000 mon.a (mon.0) 982 : audit [DBG] from='client.? 172.21.15.6:0/1162409084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:08.604 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 67 16 87976 87960 82.0429 56.125 0.0271066 0.012183 2023-08-27T22:08:08.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:08 smithi006 bash[27292]: cluster 2023-08-27T22:08:07.080372+0000 mgr.y (mgr.14901) 288 : cluster [DBG] pgmap v289: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 761 op/s 2023-08-27T22:08:08.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:08 smithi006 bash[33918]: cluster 2023-08-27T22:08:07.080372+0000 mgr.y (mgr.14901) 288 : cluster [DBG] pgmap v289: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 761 op/s 2023-08-27T22:08:08.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:08 smithi130 bash[27579]: cluster 2023-08-27T22:08:07.080372+0000 mgr.y (mgr.14901) 288 : cluster [DBG] pgmap v289: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 761 op/s 2023-08-27T22:08:09.606 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 68 16 88807 88791 81.6001 51.9375 0.0132686 0.0122486 2023-08-27T22:08:10.605 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 69 16 89683 89667 81.2109 54.75 0.0177476 0.012307 2023-08-27T22:08:10.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:10 smithi006 bash[27292]: cluster 2023-08-27T22:08:09.080874+0000 mgr.y (mgr.14901) 289 : cluster [DBG] pgmap v290: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 761 op/s 2023-08-27T22:08:10.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:10 smithi006 bash[27292]: audit 2023-08-27T22:08:09.505001+0000 mon.a (mon.0) 983 : audit [DBG] from='client.? 172.21.15.6:0/1563250154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:10.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:10 smithi006 bash[33918]: cluster 2023-08-27T22:08:09.080874+0000 mgr.y (mgr.14901) 289 : cluster [DBG] pgmap v290: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 761 op/s 2023-08-27T22:08:10.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:10 smithi006 bash[33918]: audit 2023-08-27T22:08:09.505001+0000 mon.a (mon.0) 983 : audit [DBG] from='client.? 172.21.15.6:0/1563250154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:10.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:10 smithi130 bash[27579]: cluster 2023-08-27T22:08:09.080874+0000 mgr.y (mgr.14901) 289 : cluster [DBG] pgmap v290: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 761 op/s 2023-08-27T22:08:10.910 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:10 smithi130 bash[27579]: audit 2023-08-27T22:08:09.505001+0000 mon.a (mon.0) 983 : audit [DBG] from='client.? 172.21.15.6:0/1563250154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:11.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:08:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:08:10] "GET /metrics HTTP/1.1" 200 195211 "" "Prometheus/2.33.4" 2023-08-27T22:08:11.605 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 70 16 90532 90516 80.8087 53.0625 0.0399763 0.0123681 2023-08-27T22:08:11.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:08:11.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:11.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:08:11.663Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:12.605 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 71 16 91330 91314 80.3729 49.875 0.0375291 0.0124354 2023-08-27T22:08:12.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:12 smithi006 bash[27292]: cluster 2023-08-27T22:08:11.081543+0000 mgr.y (mgr.14901) 290 : cluster [DBG] pgmap v291: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-08-27T22:08:12.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:12 smithi006 bash[27292]: audit 2023-08-27T22:08:12.174782+0000 mon.c (mon.2) 138 : audit [DBG] from='client.? 172.21.15.6:0/3886337737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:12.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:12 smithi006 bash[33918]: cluster 2023-08-27T22:08:11.081543+0000 mgr.y (mgr.14901) 290 : cluster [DBG] pgmap v291: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-08-27T22:08:12.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:12 smithi006 bash[33918]: audit 2023-08-27T22:08:12.174782+0000 mon.c (mon.2) 138 : audit [DBG] from='client.? 172.21.15.6:0/3886337737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:12.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:12 smithi006 bash[56298]: level=error ts=2023-08-27T22:08:12.321Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:12.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:12 smithi006 bash[56298]: level=warn ts=2023-08-27T22:08:12.325Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:12.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:12 smithi130 bash[27579]: cluster 2023-08-27T22:08:11.081543+0000 mgr.y (mgr.14901) 290 : cluster [DBG] pgmap v291: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-08-27T22:08:12.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:12 smithi130 bash[27579]: audit 2023-08-27T22:08:12.174782+0000 mon.c (mon.2) 138 : audit [DBG] from='client.? 172.21.15.6:0/3886337737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:13.605 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 72 16 92179 92163 79.9935 53.0625 0.0138322 0.0124949 2023-08-27T22:08:13.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:08:13.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:13.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:08:13.557Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:14.605 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 73 16 92992 92976 79.5937 50.8125 0.00801091 0.0125573 2023-08-27T22:08:14.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:14 smithi006 bash[27292]: cluster 2023-08-27T22:08:13.082014+0000 mgr.y (mgr.14901) 291 : cluster [DBG] pgmap v292: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 46 MiB/s wr, 735 op/s 2023-08-27T22:08:14.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:14 smithi006 bash[33918]: cluster 2023-08-27T22:08:13.082014+0000 mgr.y (mgr.14901) 291 : cluster [DBG] pgmap v292: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 46 MiB/s wr, 735 op/s 2023-08-27T22:08:14.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:14 smithi130 bash[27579]: cluster 2023-08-27T22:08:13.082014+0000 mgr.y (mgr.14901) 291 : cluster [DBG] pgmap v292: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 46 MiB/s wr, 735 op/s 2023-08-27T22:08:15.605 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 74 16 93827 93811 79.2233 52.1875 0.00573277 0.012616 2023-08-27T22:08:15.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:15 smithi006 bash[27292]: audit 2023-08-27T22:08:14.859823+0000 mon.a (mon.0) 984 : audit [DBG] from='client.? 172.21.15.6:0/2347204969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:15.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:15 smithi006 bash[33918]: audit 2023-08-27T22:08:14.859823+0000 mon.a (mon.0) 984 : audit [DBG] from='client.? 172.21.15.6:0/2347204969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:15.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:15 smithi130 bash[27579]: audit 2023-08-27T22:08:14.859823+0000 mon.a (mon.0) 984 : audit [DBG] from='client.? 172.21.15.6:0/2347204969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:16.605 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 75 16 94670 94654 78.8694 52.6875 0.0149249 0.0126732 2023-08-27T22:08:16.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:16 smithi006 bash[27292]: cluster 2023-08-27T22:08:15.082851+0000 mgr.y (mgr.14901) 292 : cluster [DBG] pgmap v293: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-08-27T22:08:16.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:16 smithi006 bash[33918]: cluster 2023-08-27T22:08:15.082851+0000 mgr.y (mgr.14901) 292 : cluster [DBG] pgmap v293: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-08-27T22:08:16.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:16 smithi130 bash[27579]: cluster 2023-08-27T22:08:15.082851+0000 mgr.y (mgr.14901) 292 : cluster [DBG] pgmap v293: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-08-27T22:08:17.605 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 76 16 95448 95432 78.4714 48.625 0.0196255 0.012737 2023-08-27T22:08:18.605 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 77 16 96308 96292 78.1503 53.75 0.00617207 0.0127883 2023-08-27T22:08:18.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:18 smithi006 bash[33918]: cluster 2023-08-27T22:08:17.084538+0000 mgr.y (mgr.14901) 293 : cluster [DBG] pgmap v294: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s 2023-08-27T22:08:18.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:18 smithi006 bash[33918]: audit 2023-08-27T22:08:17.588170+0000 mon.c (mon.2) 139 : audit [DBG] from='client.? 172.21.15.6:0/3883472308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:18.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:18 smithi006 bash[27292]: cluster 2023-08-27T22:08:17.084538+0000 mgr.y (mgr.14901) 293 : cluster [DBG] pgmap v294: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s 2023-08-27T22:08:18.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:18 smithi006 bash[27292]: audit 2023-08-27T22:08:17.588170+0000 mon.c (mon.2) 139 : audit [DBG] from='client.? 172.21.15.6:0/3883472308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:18.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:18 smithi130 bash[27579]: cluster 2023-08-27T22:08:17.084538+0000 mgr.y (mgr.14901) 293 : cluster [DBG] pgmap v294: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s 2023-08-27T22:08:18.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:18 smithi130 bash[27579]: audit 2023-08-27T22:08:17.588170+0000 mon.c (mon.2) 139 : audit [DBG] from='client.? 172.21.15.6:0/3883472308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:19.605 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 78 16 97142 97126 77.8166 52.125 0.0230194 0.0128444 2023-08-27T22:08:19.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:19 smithi130 bash[27579]: audit 2023-08-27T22:08:18.733512+0000 mon.b (mon.1) 135 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:08:19.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:19 smithi006 bash[33918]: audit 2023-08-27T22:08:18.733512+0000 mon.b (mon.1) 135 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:08:19.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:19 smithi006 bash[27292]: audit 2023-08-27T22:08:18.733512+0000 mon.b (mon.1) 135 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:08:20.605 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 79 16 97943 97927 77.4652 50.0625 0.00706379 0.012903 2023-08-27T22:08:20.760 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:20 smithi006 bash[27292]: cluster 2023-08-27T22:08:19.084981+0000 mgr.y (mgr.14901) 294 : cluster [DBG] pgmap v295: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s 2023-08-27T22:08:20.760 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:20 smithi006 bash[27292]: audit 2023-08-27T22:08:20.209713+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.6:0/2922244381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:20.761 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:08:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:08:20] "GET /metrics HTTP/1.1" 200 195213 "" "Prometheus/2.33.4" 2023-08-27T22:08:20.761 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:20 smithi006 bash[33918]: cluster 2023-08-27T22:08:19.084981+0000 mgr.y (mgr.14901) 294 : cluster [DBG] pgmap v295: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s 2023-08-27T22:08:20.761 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:20 smithi006 bash[33918]: audit 2023-08-27T22:08:20.209713+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.6:0/2922244381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:20.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:20 smithi130 bash[27579]: cluster 2023-08-27T22:08:19.084981+0000 mgr.y (mgr.14901) 294 : cluster [DBG] pgmap v295: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s 2023-08-27T22:08:20.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:20 smithi130 bash[27579]: audit 2023-08-27T22:08:20.209713+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.6:0/2922244381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:21.607 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:2023-08-27T22:08:21.601336+0000 min lat: 0.00150916 max lat: 0.173904 avg lat: 0.0129661 2023-08-27T22:08:21.607 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-27T22:08:21.607 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 80 16 98689 98673 77.0791 46.625 0.0273196 0.0129661 2023-08-27T22:08:21.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:08:21.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:21.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:08:21.665Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:22.606 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 81 16 99510 99494 76.7609 51.3125 0.0105764 0.0130219 2023-08-27T22:08:22.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:22 smithi006 bash[27292]: cluster 2023-08-27T22:08:21.085679+0000 mgr.y (mgr.14901) 295 : cluster [DBG] pgmap v296: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-08-27T22:08:22.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:22 smithi006 bash[33918]: cluster 2023-08-27T22:08:21.085679+0000 mgr.y (mgr.14901) 295 : cluster [DBG] pgmap v296: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-08-27T22:08:22.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:22 smithi006 bash[56298]: level=error ts=2023-08-27T22:08:22.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:22.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:22 smithi006 bash[56298]: level=warn ts=2023-08-27T22:08:22.325Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:22.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:22 smithi130 bash[27579]: cluster 2023-08-27T22:08:21.085679+0000 mgr.y (mgr.14901) 295 : cluster [DBG] pgmap v296: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-08-27T22:08:23.606 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 82 16 100316 100300 76.4391 50.375 0.0116281 0.0130765 2023-08-27T22:08:23.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:23 smithi130 bash[27579]: audit 2023-08-27T22:08:22.842078+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.6:0/4129974333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:23.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:23 smithi006 bash[27292]: audit 2023-08-27T22:08:22.842078+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.6:0/4129974333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:23.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:23 smithi006 bash[33918]: audit 2023-08-27T22:08:22.842078+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.6:0/4129974333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:08:23.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:08:23.558Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:24.614 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 83 16 101004 100988 76.0295 43 0.0336486 0.0131449 2023-08-27T22:08:24.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:24 smithi130 bash[27579]: cluster 2023-08-27T22:08:23.086039+0000 mgr.y (mgr.14901) 296 : cluster [DBG] pgmap v297: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 43 MiB/s wr, 692 op/s 2023-08-27T22:08:24.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:24 smithi006 bash[27292]: cluster 2023-08-27T22:08:23.086039+0000 mgr.y (mgr.14901) 296 : cluster [DBG] pgmap v297: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 43 MiB/s wr, 692 op/s 2023-08-27T22:08:24.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:24 smithi006 bash[33918]: cluster 2023-08-27T22:08:23.086039+0000 mgr.y (mgr.14901) 296 : cluster [DBG] pgmap v297: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 43 MiB/s wr, 692 op/s 2023-08-27T22:08:25.614 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 84 16 101673 101657 75.6222 41.8125 0.0254949 0.0132171 2023-08-27T22:08:26.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:25 smithi006 bash[27292]: audit 2023-08-27T22:08:24.914047+0000 mon.a (mon.0) 985 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:08:26.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:25 smithi006 bash[27292]: cluster 2023-08-27T22:08:25.086795+0000 mgr.y (mgr.14901) 297 : cluster [DBG] pgmap v298: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-08-27T22:08:26.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:25 smithi006 bash[27292]: audit 2023-08-27T22:08:25.169053+0000 mon.a (mon.0) 986 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:08:26.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:25 smithi006 bash[27292]: audit 2023-08-27T22:08:25.477073+0000 mon.b (mon.1) 136 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:08:26.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:25 smithi006 bash[27292]: audit 2023-08-27T22:08:25.478416+0000 mon.b (mon.1) 137 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:08:26.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:25 smithi006 bash[27292]: audit 2023-08-27T22:08:25.485882+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:08:26.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:25 smithi006 bash[27292]: audit 2023-08-27T22:08:25.595678+0000 mon.c (mon.2) 142 : audit [DBG] from='client.? 172.21.15.6:0/309138594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:26.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:25 smithi006 bash[33918]: audit 2023-08-27T22:08:24.914047+0000 mon.a (mon.0) 985 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:08:26.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:25 smithi006 bash[33918]: cluster 2023-08-27T22:08:25.086795+0000 mgr.y (mgr.14901) 297 : cluster [DBG] pgmap v298: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-08-27T22:08:26.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:25 smithi006 bash[33918]: audit 2023-08-27T22:08:25.169053+0000 mon.a (mon.0) 986 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:08:26.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:25 smithi006 bash[33918]: audit 2023-08-27T22:08:25.477073+0000 mon.b (mon.1) 136 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:08:26.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:25 smithi006 bash[33918]: audit 2023-08-27T22:08:25.478416+0000 mon.b (mon.1) 137 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:08:26.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:25 smithi006 bash[33918]: audit 2023-08-27T22:08:25.485882+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:08:26.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:25 smithi006 bash[33918]: audit 2023-08-27T22:08:25.595678+0000 mon.c (mon.2) 142 : audit [DBG] from='client.? 172.21.15.6:0/309138594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:26.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:25 smithi130 bash[27579]: audit 2023-08-27T22:08:24.914047+0000 mon.a (mon.0) 985 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:08:26.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:25 smithi130 bash[27579]: cluster 2023-08-27T22:08:25.086795+0000 mgr.y (mgr.14901) 297 : cluster [DBG] pgmap v298: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-08-27T22:08:26.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:25 smithi130 bash[27579]: audit 2023-08-27T22:08:25.169053+0000 mon.a (mon.0) 986 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:08:26.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:25 smithi130 bash[27579]: audit 2023-08-27T22:08:25.477073+0000 mon.b (mon.1) 136 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:08:26.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:25 smithi130 bash[27579]: audit 2023-08-27T22:08:25.478416+0000 mon.b (mon.1) 137 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:08:26.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:25 smithi130 bash[27579]: audit 2023-08-27T22:08:25.485882+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:08:26.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:25 smithi130 bash[27579]: audit 2023-08-27T22:08:25.595678+0000 mon.c (mon.2) 142 : audit [DBG] from='client.? 172.21.15.6:0/309138594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:26.614 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 85 16 102458 102442 75.3097 49.0625 0.017214 0.0132723 2023-08-27T22:08:27.614 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 86 16 103239 103223 75.0016 48.8125 0.0199645 0.0133266 2023-08-27T22:08:28.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:28 smithi006 bash[33918]: cluster 2023-08-27T22:08:27.087244+0000 mgr.y (mgr.14901) 298 : cluster [DBG] pgmap v299: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 667 op/s 2023-08-27T22:08:28.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:28 smithi006 bash[27292]: cluster 2023-08-27T22:08:27.087244+0000 mgr.y (mgr.14901) 298 : cluster [DBG] pgmap v299: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 667 op/s 2023-08-27T22:08:28.614 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 87 16 103945 103929 74.6467 44.125 0.0143202 0.0133902 2023-08-27T22:08:28.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:28 smithi130 bash[27579]: cluster 2023-08-27T22:08:27.087244+0000 mgr.y (mgr.14901) 298 : cluster [DBG] pgmap v299: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 667 op/s 2023-08-27T22:08:29.614 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 88 16 104678 104662 74.319 45.8125 0.0118205 0.0134493 2023-08-27T22:08:29.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:29 smithi130 bash[27579]: audit 2023-08-27T22:08:28.339208+0000 mon.b (mon.1) 138 : audit [DBG] from='client.? 172.21.15.6:0/2112259778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:29.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:29 smithi006 bash[33918]: audit 2023-08-27T22:08:28.339208+0000 mon.b (mon.1) 138 : audit [DBG] from='client.? 172.21.15.6:0/2112259778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:29.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:29 smithi006 bash[27292]: audit 2023-08-27T22:08:28.339208+0000 mon.b (mon.1) 138 : audit [DBG] from='client.? 172.21.15.6:0/2112259778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:30.614 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 89 16 105425 105409 74.0085 46.6875 0.0230707 0.0135056 2023-08-27T22:08:30.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:30 smithi130 bash[27579]: cluster 2023-08-27T22:08:29.087725+0000 mgr.y (mgr.14901) 299 : cluster [DBG] pgmap v300: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 667 op/s 2023-08-27T22:08:30.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:30 smithi006 bash[27292]: cluster 2023-08-27T22:08:29.087725+0000 mgr.y (mgr.14901) 299 : cluster [DBG] pgmap v300: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 667 op/s 2023-08-27T22:08:30.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:30 smithi006 bash[33918]: cluster 2023-08-27T22:08:29.087725+0000 mgr.y (mgr.14901) 299 : cluster [DBG] pgmap v300: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 667 op/s 2023-08-27T22:08:31.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:08:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:08:30] "GET /metrics HTTP/1.1" 200 195231 "" "Prometheus/2.33.4" 2023-08-27T22:08:31.614 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Total time run: 90.0111 2023-08-27T22:08:31.615 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Total writes made: 106141 2023-08-27T22:08:31.615 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Write size: 65536 2023-08-27T22:08:31.615 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Object size: 65536 2023-08-27T22:08:31.615 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Bandwidth (MB/sec): 73.6999 2023-08-27T22:08:31.615 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Stddev Bandwidth: 28.5341 2023-08-27T22:08:31.615 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Max bandwidth (MB/sec): 166.438 2023-08-27T22:08:31.615 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Min bandwidth (MB/sec): 41.8125 2023-08-27T22:08:31.615 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Average IOPS: 1179 2023-08-27T22:08:31.615 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Stddev IOPS: 456.545 2023-08-27T22:08:31.615 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Max IOPS: 2663 2023-08-27T22:08:31.615 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Min IOPS: 669 2023-08-27T22:08:31.616 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Average Latency(s): 0.0135634 2023-08-27T22:08:31.616 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Stddev Latency(s): 0.00879576 2023-08-27T22:08:31.616 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Max latency(s): 0.173904 2023-08-27T22:08:31.616 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Min latency(s): 0.00150916 2023-08-27T22:08:31.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:31 smithi130 bash[27579]: audit 2023-08-27T22:08:31.050639+0000 mon.a (mon.0) 988 : audit [DBG] from='client.? 172.21.15.6:0/497979803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:31.622 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Cleaning up (deleting benchmark objects) 2023-08-27T22:08:31.672 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:31 smithi006 bash[27292]: audit 2023-08-27T22:08:31.050639+0000 mon.a (mon.0) 988 : audit [DBG] from='client.? 172.21.15.6:0/497979803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:31.672 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:31 smithi006 bash[33918]: audit 2023-08-27T22:08:31.050639+0000 mon.a (mon.0) 988 : audit [DBG] from='client.? 172.21.15.6:0/497979803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:31.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:08:31.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:31.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:08:31.667Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:32 smithi130 bash[27579]: cluster 2023-08-27T22:08:31.088489+0000 mgr.y (mgr.14901) 300 : cluster [DBG] pgmap v301: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s 2023-08-27T22:08:32.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:32 smithi006 bash[27292]: cluster 2023-08-27T22:08:31.088489+0000 mgr.y (mgr.14901) 300 : cluster [DBG] pgmap v301: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s 2023-08-27T22:08:32.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:32 smithi006 bash[33918]: cluster 2023-08-27T22:08:31.088489+0000 mgr.y (mgr.14901) 300 : cluster [DBG] pgmap v301: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s 2023-08-27T22:08:32.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:32 smithi006 bash[56298]: level=error ts=2023-08-27T22:08:32.324Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:32.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:32 smithi006 bash[56298]: level=warn ts=2023-08-27T22:08:32.326Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:33.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:08:33.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:33.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:08:33.558Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:34.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:34 smithi130 bash[27579]: cluster 2023-08-27T22:08:33.088957+0000 mgr.y (mgr.14901) 301 : cluster [DBG] pgmap v302: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 39 MiB/s wr, 628 op/s 2023-08-27T22:08:34.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:34 smithi130 bash[27579]: audit 2023-08-27T22:08:33.721831+0000 mon.b (mon.1) 139 : audit [DBG] from='client.? 172.21.15.6:0/19715985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:34.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:34 smithi006 bash[33918]: cluster 2023-08-27T22:08:33.088957+0000 mgr.y (mgr.14901) 301 : cluster [DBG] pgmap v302: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 39 MiB/s wr, 628 op/s 2023-08-27T22:08:34.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:34 smithi006 bash[33918]: audit 2023-08-27T22:08:33.721831+0000 mon.b (mon.1) 139 : audit [DBG] from='client.? 172.21.15.6:0/19715985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:34.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:34 smithi006 bash[27292]: cluster 2023-08-27T22:08:33.088957+0000 mgr.y (mgr.14901) 301 : cluster [DBG] pgmap v302: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 39 MiB/s wr, 628 op/s 2023-08-27T22:08:34.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:34 smithi006 bash[27292]: audit 2023-08-27T22:08:33.721831+0000 mon.b (mon.1) 139 : audit [DBG] from='client.? 172.21.15.6:0/19715985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:36.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:36 smithi130 bash[27579]: cluster 2023-08-27T22:08:35.089923+0000 mgr.y (mgr.14901) 302 : cluster [DBG] pgmap v303: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 47 MiB/s wr, 949 op/s 2023-08-27T22:08:36.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:36 smithi006 bash[27292]: cluster 2023-08-27T22:08:35.089923+0000 mgr.y (mgr.14901) 302 : cluster [DBG] pgmap v303: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 47 MiB/s wr, 949 op/s 2023-08-27T22:08:36.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:36 smithi006 bash[33918]: cluster 2023-08-27T22:08:35.089923+0000 mgr.y (mgr.14901) 302 : cluster [DBG] pgmap v303: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 47 MiB/s wr, 949 op/s 2023-08-27T22:08:37.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:37 smithi130 bash[27579]: audit 2023-08-27T22:08:36.400861+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.6:0/903767880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:37.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:37 smithi006 bash[33918]: audit 2023-08-27T22:08:36.400861+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.6:0/903767880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:37.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:37 smithi006 bash[27292]: audit 2023-08-27T22:08:36.400861+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.6:0/903767880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:38.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:38 smithi130 bash[27579]: cluster 2023-08-27T22:08:37.090439+0000 mgr.y (mgr.14901) 303 : cluster [DBG] pgmap v304: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 626 op/s 2023-08-27T22:08:38.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:38 smithi006 bash[27292]: cluster 2023-08-27T22:08:37.090439+0000 mgr.y (mgr.14901) 303 : cluster [DBG] pgmap v304: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 626 op/s 2023-08-27T22:08:38.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:38 smithi006 bash[33918]: cluster 2023-08-27T22:08:37.090439+0000 mgr.y (mgr.14901) 303 : cluster [DBG] pgmap v304: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 626 op/s 2023-08-27T22:08:39.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:39 smithi130 bash[27579]: audit 2023-08-27T22:08:39.070669+0000 mon.a (mon.0) 989 : audit [DBG] from='client.? 172.21.15.6:0/3923183371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:39.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:39 smithi006 bash[27292]: audit 2023-08-27T22:08:39.070669+0000 mon.a (mon.0) 989 : audit [DBG] from='client.? 172.21.15.6:0/3923183371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:39.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:39 smithi006 bash[33918]: audit 2023-08-27T22:08:39.070669+0000 mon.a (mon.0) 989 : audit [DBG] from='client.? 172.21.15.6:0/3923183371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:40.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:40 smithi130 bash[27579]: cluster 2023-08-27T22:08:39.090876+0000 mgr.y (mgr.14901) 304 : cluster [DBG] pgmap v305: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 626 op/s 2023-08-27T22:08:40.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:40 smithi006 bash[27292]: cluster 2023-08-27T22:08:39.090876+0000 mgr.y (mgr.14901) 304 : cluster [DBG] pgmap v305: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 626 op/s 2023-08-27T22:08:40.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:40 smithi006 bash[33918]: cluster 2023-08-27T22:08:39.090876+0000 mgr.y (mgr.14901) 304 : cluster [DBG] pgmap v305: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 626 op/s 2023-08-27T22:08:41.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:08:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:08:40] "GET /metrics HTTP/1.1" 200 195231 "" "Prometheus/2.33.4" 2023-08-27T22:08:41.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:08:41.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:41.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:08:41.668Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:42.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:42 smithi130 bash[27579]: cluster 2023-08-27T22:08:41.091715+0000 mgr.y (mgr.14901) 305 : cluster [DBG] pgmap v306: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 944 op/s 2023-08-27T22:08:42.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:42 smithi130 bash[27579]: audit 2023-08-27T22:08:41.754025+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.6:0/3099383110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:42.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:42 smithi006 bash[56298]: level=error ts=2023-08-27T22:08:42.325Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:42.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:42 smithi006 bash[56298]: level=warn ts=2023-08-27T22:08:42.327Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:42.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:42 smithi006 bash[27292]: cluster 2023-08-27T22:08:41.091715+0000 mgr.y (mgr.14901) 305 : cluster [DBG] pgmap v306: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 944 op/s 2023-08-27T22:08:42.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:42 smithi006 bash[27292]: audit 2023-08-27T22:08:41.754025+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.6:0/3099383110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:42.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:42 smithi006 bash[33918]: cluster 2023-08-27T22:08:41.091715+0000 mgr.y (mgr.14901) 305 : cluster [DBG] pgmap v306: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 944 op/s 2023-08-27T22:08:42.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:42 smithi006 bash[33918]: audit 2023-08-27T22:08:41.754025+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.6:0/3099383110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:43.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:08:43.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:43.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:08:43.558Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:44.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:44 smithi130 bash[27579]: cluster 2023-08-27T22:08:43.092184+0000 mgr.y (mgr.14901) 306 : cluster [DBG] pgmap v307: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 7.4 MiB/s wr, 638 op/s 2023-08-27T22:08:44.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:44 smithi006 bash[33918]: cluster 2023-08-27T22:08:43.092184+0000 mgr.y (mgr.14901) 306 : cluster [DBG] pgmap v307: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 7.4 MiB/s wr, 638 op/s 2023-08-27T22:08:44.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:44 smithi006 bash[27292]: cluster 2023-08-27T22:08:43.092184+0000 mgr.y (mgr.14901) 306 : cluster [DBG] pgmap v307: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 7.4 MiB/s wr, 638 op/s 2023-08-27T22:08:45.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:45 smithi130 bash[27579]: audit 2023-08-27T22:08:44.413987+0000 mon.c (mon.2) 145 : audit [DBG] from='client.? 172.21.15.6:0/3836521733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:45.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:45 smithi006 bash[27292]: audit 2023-08-27T22:08:44.413987+0000 mon.c (mon.2) 145 : audit [DBG] from='client.? 172.21.15.6:0/3836521733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:45.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:45 smithi006 bash[33918]: audit 2023-08-27T22:08:44.413987+0000 mon.c (mon.2) 145 : audit [DBG] from='client.? 172.21.15.6:0/3836521733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:46.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:46 smithi130 bash[27579]: cluster 2023-08-27T22:08:45.092973+0000 mgr.y (mgr.14901) 307 : cluster [DBG] pgmap v308: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 7.4 MiB/s wr, 949 op/s 2023-08-27T22:08:46.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:46 smithi006 bash[27292]: cluster 2023-08-27T22:08:45.092973+0000 mgr.y (mgr.14901) 307 : cluster [DBG] pgmap v308: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 7.4 MiB/s wr, 949 op/s 2023-08-27T22:08:46.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:46 smithi006 bash[33918]: cluster 2023-08-27T22:08:45.092973+0000 mgr.y (mgr.14901) 307 : cluster [DBG] pgmap v308: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 7.4 MiB/s wr, 949 op/s 2023-08-27T22:08:47.622 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:47 smithi130 bash[27579]: audit 2023-08-27T22:08:47.111737+0000 mon.a (mon.0) 990 : audit [DBG] from='client.? 172.21.15.6:0/2127929226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:47.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:47 smithi006 bash[33918]: audit 2023-08-27T22:08:47.111737+0000 mon.a (mon.0) 990 : audit [DBG] from='client.? 172.21.15.6:0/2127929226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:47.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:47 smithi006 bash[27292]: audit 2023-08-27T22:08:47.111737+0000 mon.a (mon.0) 990 : audit [DBG] from='client.? 172.21.15.6:0/2127929226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:48.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:48 smithi006 bash[27292]: cluster 2023-08-27T22:08:47.093417+0000 mgr.y (mgr.14901) 308 : cluster [DBG] pgmap v309: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-08-27T22:08:48.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:48 smithi006 bash[33918]: cluster 2023-08-27T22:08:47.093417+0000 mgr.y (mgr.14901) 308 : cluster [DBG] pgmap v309: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-08-27T22:08:48.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:48 smithi130 bash[27579]: cluster 2023-08-27T22:08:47.093417+0000 mgr.y (mgr.14901) 308 : cluster [DBG] pgmap v309: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-08-27T22:08:50.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:50 smithi006 bash[27292]: cluster 2023-08-27T22:08:49.093864+0000 mgr.y (mgr.14901) 309 : cluster [DBG] pgmap v310: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-08-27T22:08:50.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:50 smithi006 bash[27292]: audit 2023-08-27T22:08:49.756591+0000 mon.a (mon.0) 991 : audit [DBG] from='client.? 172.21.15.6:0/3762059801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:50.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:50 smithi006 bash[33918]: cluster 2023-08-27T22:08:49.093864+0000 mgr.y (mgr.14901) 309 : cluster [DBG] pgmap v310: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-08-27T22:08:50.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:50 smithi006 bash[33918]: audit 2023-08-27T22:08:49.756591+0000 mon.a (mon.0) 991 : audit [DBG] from='client.? 172.21.15.6:0/3762059801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:50.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:50 smithi130 bash[27579]: cluster 2023-08-27T22:08:49.093864+0000 mgr.y (mgr.14901) 309 : cluster [DBG] pgmap v310: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-08-27T22:08:50.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:50 smithi130 bash[27579]: audit 2023-08-27T22:08:49.756591+0000 mon.a (mon.0) 991 : audit [DBG] from='client.? 172.21.15.6:0/3762059801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:51.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:08:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:08:50] "GET /metrics HTTP/1.1" 200 195256 "" "Prometheus/2.33.4" 2023-08-27T22:08:51.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:08:51.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:51.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:08:51.669Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:52.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:52 smithi006 bash[27292]: cluster 2023-08-27T22:08:51.094632+0000 mgr.y (mgr.14901) 310 : cluster [DBG] pgmap v311: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 934 op/s 2023-08-27T22:08:52.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:52 smithi006 bash[33918]: cluster 2023-08-27T22:08:51.094632+0000 mgr.y (mgr.14901) 310 : cluster [DBG] pgmap v311: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 934 op/s 2023-08-27T22:08:52.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:52 smithi006 bash[56298]: level=error ts=2023-08-27T22:08:52.327Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:52.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:52 smithi006 bash[56298]: level=warn ts=2023-08-27T22:08:52.329Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:52.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:52 smithi130 bash[27579]: cluster 2023-08-27T22:08:51.094632+0000 mgr.y (mgr.14901) 310 : cluster [DBG] pgmap v311: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 934 op/s 2023-08-27T22:08:53.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:53 smithi006 bash[27292]: audit 2023-08-27T22:08:52.421237+0000 mon.c (mon.2) 146 : audit [DBG] from='client.? 172.21.15.6:0/1384068007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:53.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:53 smithi006 bash[33918]: audit 2023-08-27T22:08:52.421237+0000 mon.c (mon.2) 146 : audit [DBG] from='client.? 172.21.15.6:0/1384068007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:53.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:08:53.558Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:53.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:08:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:08:53.559Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:08:53.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:53 smithi130 bash[27579]: audit 2023-08-27T22:08:52.421237+0000 mon.c (mon.2) 146 : audit [DBG] from='client.? 172.21.15.6:0/1384068007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:54.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:54 smithi006 bash[27292]: cluster 2023-08-27T22:08:53.097185+0000 mgr.y (mgr.14901) 311 : cluster [DBG] pgmap v312: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-08-27T22:08:54.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:54 smithi006 bash[33918]: cluster 2023-08-27T22:08:53.097185+0000 mgr.y (mgr.14901) 311 : cluster [DBG] pgmap v312: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-08-27T22:08:54.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:54 smithi130 bash[27579]: cluster 2023-08-27T22:08:53.097185+0000 mgr.y (mgr.14901) 311 : cluster [DBG] pgmap v312: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-08-27T22:08:55.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:55 smithi006 bash[27292]: audit 2023-08-27T22:08:55.132149+0000 mon.a (mon.0) 992 : audit [DBG] from='client.? 172.21.15.6:0/1334823760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:55.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:55 smithi006 bash[33918]: audit 2023-08-27T22:08:55.132149+0000 mon.a (mon.0) 992 : audit [DBG] from='client.? 172.21.15.6:0/1334823760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:55.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:55 smithi130 bash[27579]: audit 2023-08-27T22:08:55.132149+0000 mon.a (mon.0) 992 : audit [DBG] from='client.? 172.21.15.6:0/1334823760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:56.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:56 smithi006 bash[27292]: cluster 2023-08-27T22:08:55.097916+0000 mgr.y (mgr.14901) 312 : cluster [DBG] pgmap v313: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 922 op/s 2023-08-27T22:08:56.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:56 smithi006 bash[33918]: cluster 2023-08-27T22:08:55.097916+0000 mgr.y (mgr.14901) 312 : cluster [DBG] pgmap v313: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 922 op/s 2023-08-27T22:08:56.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:56 smithi130 bash[27579]: cluster 2023-08-27T22:08:55.097916+0000 mgr.y (mgr.14901) 312 : cluster [DBG] pgmap v313: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 922 op/s 2023-08-27T22:08:58.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:58 smithi006 bash[27292]: cluster 2023-08-27T22:08:57.098374+0000 mgr.y (mgr.14901) 313 : cluster [DBG] pgmap v314: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-08-27T22:08:58.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:08:58 smithi006 bash[27292]: audit 2023-08-27T22:08:57.828754+0000 mon.a (mon.0) 993 : audit [DBG] from='client.? 172.21.15.6:0/3351212461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:58.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:58 smithi006 bash[33918]: cluster 2023-08-27T22:08:57.098374+0000 mgr.y (mgr.14901) 313 : cluster [DBG] pgmap v314: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-08-27T22:08:58.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:08:58 smithi006 bash[33918]: audit 2023-08-27T22:08:57.828754+0000 mon.a (mon.0) 993 : audit [DBG] from='client.? 172.21.15.6:0/3351212461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:58.869 INFO:journalctl@ceph.grafana.a.smithi130.stdout:Aug 27 22:08:58 smithi130 bash[48144]: t=2023-08-27T22:08:58+0000 lvl=info msg="Database locked, sleeping then retrying" logger=sqlstore error="database is locked" retry=0 2023-08-27T22:08:58.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:58 smithi130 bash[27579]: cluster 2023-08-27T22:08:57.098374+0000 mgr.y (mgr.14901) 313 : cluster [DBG] pgmap v314: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-08-27T22:08:58.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:08:58 smithi130 bash[27579]: audit 2023-08-27T22:08:57.828754+0000 mon.a (mon.0) 993 : audit [DBG] from='client.? 172.21.15.6:0/3351212461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:00.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:00 smithi006 bash[27292]: cluster 2023-08-27T22:08:59.098808+0000 mgr.y (mgr.14901) 314 : cluster [DBG] pgmap v315: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-08-27T22:09:00.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:00 smithi006 bash[33918]: cluster 2023-08-27T22:08:59.098808+0000 mgr.y (mgr.14901) 314 : cluster [DBG] pgmap v315: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-08-27T22:09:00.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:00 smithi130 bash[27579]: cluster 2023-08-27T22:08:59.098808+0000 mgr.y (mgr.14901) 314 : cluster [DBG] pgmap v315: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-08-27T22:09:01.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:09:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:09:00] "GET /metrics HTTP/1.1" 200 195270 "" "Prometheus/2.33.4" 2023-08-27T22:09:01.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:01 smithi006 bash[27292]: audit 2023-08-27T22:09:00.559974+0000 mon.a (mon.0) 994 : audit [DBG] from='client.? 172.21.15.6:0/4059175677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:01.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:01 smithi006 bash[33918]: audit 2023-08-27T22:09:00.559974+0000 mon.a (mon.0) 994 : audit [DBG] from='client.? 172.21.15.6:0/4059175677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:01.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:09:01.669Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:01.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:09:01.670Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:01.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:01 smithi130 bash[27579]: audit 2023-08-27T22:09:00.559974+0000 mon.a (mon.0) 994 : audit [DBG] from='client.? 172.21.15.6:0/4059175677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:02.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:02 smithi006 bash[27292]: cluster 2023-08-27T22:09:01.099569+0000 mgr.y (mgr.14901) 315 : cluster [DBG] pgmap v316: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-08-27T22:09:02.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:02 smithi006 bash[33918]: cluster 2023-08-27T22:09:01.099569+0000 mgr.y (mgr.14901) 315 : cluster [DBG] pgmap v316: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-08-27T22:09:02.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:02 smithi006 bash[56298]: level=error ts=2023-08-27T22:09:02.329Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:02.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:02 smithi006 bash[56298]: level=warn ts=2023-08-27T22:09:02.333Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:02.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:02 smithi130 bash[27579]: cluster 2023-08-27T22:09:01.099569+0000 mgr.y (mgr.14901) 315 : cluster [DBG] pgmap v316: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-08-27T22:09:03.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:09:03.558Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:03.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:09:03.567Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:04.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:04 smithi130 bash[27579]: cluster 2023-08-27T22:09:03.100018+0000 mgr.y (mgr.14901) 316 : cluster [DBG] pgmap v317: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-08-27T22:09:04.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:04 smithi130 bash[27579]: audit 2023-08-27T22:09:03.280202+0000 mon.b (mon.1) 140 : audit [DBG] from='client.? 172.21.15.6:0/25702591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:04.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:04 smithi006 bash[27292]: cluster 2023-08-27T22:09:03.100018+0000 mgr.y (mgr.14901) 316 : cluster [DBG] pgmap v317: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-08-27T22:09:04.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:04 smithi006 bash[27292]: audit 2023-08-27T22:09:03.280202+0000 mon.b (mon.1) 140 : audit [DBG] from='client.? 172.21.15.6:0/25702591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:04.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:04 smithi006 bash[33918]: cluster 2023-08-27T22:09:03.100018+0000 mgr.y (mgr.14901) 316 : cluster [DBG] pgmap v317: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-08-27T22:09:04.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:04 smithi006 bash[33918]: audit 2023-08-27T22:09:03.280202+0000 mon.b (mon.1) 140 : audit [DBG] from='client.? 172.21.15.6:0/25702591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:06.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:06 smithi130 bash[27579]: cluster 2023-08-27T22:09:05.100734+0000 mgr.y (mgr.14901) 317 : cluster [DBG] pgmap v318: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-08-27T22:09:06.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:06 smithi130 bash[27579]: audit 2023-08-27T22:09:05.939214+0000 mon.c (mon.2) 147 : audit [DBG] from='client.? 172.21.15.6:0/1672474059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:06.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:06 smithi006 bash[27292]: cluster 2023-08-27T22:09:05.100734+0000 mgr.y (mgr.14901) 317 : cluster [DBG] pgmap v318: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-08-27T22:09:06.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:06 smithi006 bash[27292]: audit 2023-08-27T22:09:05.939214+0000 mon.c (mon.2) 147 : audit [DBG] from='client.? 172.21.15.6:0/1672474059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:06.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:06 smithi006 bash[33918]: cluster 2023-08-27T22:09:05.100734+0000 mgr.y (mgr.14901) 317 : cluster [DBG] pgmap v318: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-08-27T22:09:06.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:06 smithi006 bash[33918]: audit 2023-08-27T22:09:05.939214+0000 mon.c (mon.2) 147 : audit [DBG] from='client.? 172.21.15.6:0/1672474059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:08.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:08 smithi130 bash[27579]: cluster 2023-08-27T22:09:07.101168+0000 mgr.y (mgr.14901) 318 : cluster [DBG] pgmap v319: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-08-27T22:09:08.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:08 smithi006 bash[27292]: cluster 2023-08-27T22:09:07.101168+0000 mgr.y (mgr.14901) 318 : cluster [DBG] pgmap v319: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-08-27T22:09:08.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:08 smithi006 bash[33918]: cluster 2023-08-27T22:09:07.101168+0000 mgr.y (mgr.14901) 318 : cluster [DBG] pgmap v319: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-08-27T22:09:09.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:09 smithi130 bash[27579]: audit 2023-08-27T22:09:08.641711+0000 mon.c (mon.2) 148 : audit [DBG] from='client.? 172.21.15.6:0/3116409789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:09.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:09 smithi006 bash[33918]: audit 2023-08-27T22:09:08.641711+0000 mon.c (mon.2) 148 : audit [DBG] from='client.? 172.21.15.6:0/3116409789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:09.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:09 smithi006 bash[27292]: audit 2023-08-27T22:09:08.641711+0000 mon.c (mon.2) 148 : audit [DBG] from='client.? 172.21.15.6:0/3116409789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:10.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:10 smithi130 bash[27579]: cluster 2023-08-27T22:09:09.101611+0000 mgr.y (mgr.14901) 319 : cluster [DBG] pgmap v320: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-08-27T22:09:10.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:10 smithi006 bash[33918]: cluster 2023-08-27T22:09:09.101611+0000 mgr.y (mgr.14901) 319 : cluster [DBG] pgmap v320: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-08-27T22:09:10.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:10 smithi006 bash[27292]: cluster 2023-08-27T22:09:09.101611+0000 mgr.y (mgr.14901) 319 : cluster [DBG] pgmap v320: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-08-27T22:09:11.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:09:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:09:10] "GET /metrics HTTP/1.1" 200 195270 "" "Prometheus/2.33.4" 2023-08-27T22:09:11.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:09:11.671Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:11.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:09:11.673Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:12.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:12 smithi130 bash[27579]: cluster 2023-08-27T22:09:11.102362+0000 mgr.y (mgr.14901) 320 : cluster [DBG] pgmap v321: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2023-08-27T22:09:12.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:12 smithi130 bash[27579]: audit 2023-08-27T22:09:11.311434+0000 mon.c (mon.2) 149 : audit [DBG] from='client.? 172.21.15.6:0/1777924783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:12.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:12 smithi006 bash[27292]: cluster 2023-08-27T22:09:11.102362+0000 mgr.y (mgr.14901) 320 : cluster [DBG] pgmap v321: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2023-08-27T22:09:12.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:12 smithi006 bash[27292]: audit 2023-08-27T22:09:11.311434+0000 mon.c (mon.2) 149 : audit [DBG] from='client.? 172.21.15.6:0/1777924783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:12.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:12 smithi006 bash[33918]: cluster 2023-08-27T22:09:11.102362+0000 mgr.y (mgr.14901) 320 : cluster [DBG] pgmap v321: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2023-08-27T22:09:12.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:12 smithi006 bash[33918]: audit 2023-08-27T22:09:11.311434+0000 mon.c (mon.2) 149 : audit [DBG] from='client.? 172.21.15.6:0/1777924783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:12.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:12 smithi006 bash[56298]: level=error ts=2023-08-27T22:09:12.329Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:12.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:12 smithi006 bash[56298]: level=warn ts=2023-08-27T22:09:12.330Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:13.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:09:13.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:13.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:09:13.566Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:14.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:14 smithi130 bash[27579]: cluster 2023-08-27T22:09:13.102985+0000 mgr.y (mgr.14901) 321 : cluster [DBG] pgmap v322: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-08-27T22:09:14.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:14 smithi130 bash[27579]: audit 2023-08-27T22:09:14.045411+0000 mon.c (mon.2) 150 : audit [DBG] from='client.? 172.21.15.6:0/667801433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:14.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:14 smithi006 bash[27292]: cluster 2023-08-27T22:09:13.102985+0000 mgr.y (mgr.14901) 321 : cluster [DBG] pgmap v322: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-08-27T22:09:14.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:14 smithi006 bash[27292]: audit 2023-08-27T22:09:14.045411+0000 mon.c (mon.2) 150 : audit [DBG] from='client.? 172.21.15.6:0/667801433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:14.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:14 smithi006 bash[33918]: cluster 2023-08-27T22:09:13.102985+0000 mgr.y (mgr.14901) 321 : cluster [DBG] pgmap v322: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-08-27T22:09:14.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:14 smithi006 bash[33918]: audit 2023-08-27T22:09:14.045411+0000 mon.c (mon.2) 150 : audit [DBG] from='client.? 172.21.15.6:0/667801433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:16.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:16 smithi130 bash[27579]: cluster 2023-08-27T22:09:15.103623+0000 mgr.y (mgr.14901) 322 : cluster [DBG] pgmap v323: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2023-08-27T22:09:16.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:16 smithi006 bash[27292]: cluster 2023-08-27T22:09:15.103623+0000 mgr.y (mgr.14901) 322 : cluster [DBG] pgmap v323: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2023-08-27T22:09:16.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:16 smithi006 bash[33918]: cluster 2023-08-27T22:09:15.103623+0000 mgr.y (mgr.14901) 322 : cluster [DBG] pgmap v323: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2023-08-27T22:09:17.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:17 smithi130 bash[27579]: audit 2023-08-27T22:09:16.772219+0000 mon.a (mon.0) 995 : audit [DBG] from='client.? 172.21.15.6:0/2252707137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:17.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:17 smithi006 bash[27292]: audit 2023-08-27T22:09:16.772219+0000 mon.a (mon.0) 995 : audit [DBG] from='client.? 172.21.15.6:0/2252707137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:17.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:17 smithi006 bash[33918]: audit 2023-08-27T22:09:16.772219+0000 mon.a (mon.0) 995 : audit [DBG] from='client.? 172.21.15.6:0/2252707137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:18.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:18 smithi130 bash[27579]: cluster 2023-08-27T22:09:17.104121+0000 mgr.y (mgr.14901) 323 : cluster [DBG] pgmap v324: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-08-27T22:09:18.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:18 smithi006 bash[27292]: cluster 2023-08-27T22:09:17.104121+0000 mgr.y (mgr.14901) 323 : cluster [DBG] pgmap v324: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-08-27T22:09:18.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:18 smithi006 bash[33918]: cluster 2023-08-27T22:09:17.104121+0000 mgr.y (mgr.14901) 323 : cluster [DBG] pgmap v324: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-08-27T22:09:20.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:20 smithi130 bash[27579]: cluster 2023-08-27T22:09:19.104565+0000 mgr.y (mgr.14901) 324 : cluster [DBG] pgmap v325: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-08-27T22:09:20.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:20 smithi130 bash[27579]: audit 2023-08-27T22:09:19.432210+0000 mon.c (mon.2) 151 : audit [DBG] from='client.? 172.21.15.6:0/2277832475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:20.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:20 smithi006 bash[27292]: cluster 2023-08-27T22:09:19.104565+0000 mgr.y (mgr.14901) 324 : cluster [DBG] pgmap v325: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-08-27T22:09:20.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:20 smithi006 bash[27292]: audit 2023-08-27T22:09:19.432210+0000 mon.c (mon.2) 151 : audit [DBG] from='client.? 172.21.15.6:0/2277832475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:20.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:20 smithi006 bash[33918]: cluster 2023-08-27T22:09:19.104565+0000 mgr.y (mgr.14901) 324 : cluster [DBG] pgmap v325: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-08-27T22:09:20.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:20 smithi006 bash[33918]: audit 2023-08-27T22:09:19.432210+0000 mon.c (mon.2) 151 : audit [DBG] from='client.? 172.21.15.6:0/2277832475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:21.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:09:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:09:20] "GET /metrics HTTP/1.1" 200 195255 "" "Prometheus/2.33.4" 2023-08-27T22:09:21.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:09:21.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:21.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:09:21.673Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:22.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:22 smithi130 bash[27579]: cluster 2023-08-27T22:09:21.105338+0000 mgr.y (mgr.14901) 325 : cluster [DBG] pgmap v326: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-08-27T22:09:22.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:22 smithi130 bash[27579]: audit 2023-08-27T22:09:22.087183+0000 mon.b (mon.1) 141 : audit [DBG] from='client.? 172.21.15.6:0/4294757621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:22.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:22 smithi006 bash[27292]: cluster 2023-08-27T22:09:21.105338+0000 mgr.y (mgr.14901) 325 : cluster [DBG] pgmap v326: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-08-27T22:09:22.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:22 smithi006 bash[27292]: audit 2023-08-27T22:09:22.087183+0000 mon.b (mon.1) 141 : audit [DBG] from='client.? 172.21.15.6:0/4294757621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:22.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:22 smithi006 bash[33918]: cluster 2023-08-27T22:09:21.105338+0000 mgr.y (mgr.14901) 325 : cluster [DBG] pgmap v326: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-08-27T22:09:22.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:22 smithi006 bash[33918]: audit 2023-08-27T22:09:22.087183+0000 mon.b (mon.1) 141 : audit [DBG] from='client.? 172.21.15.6:0/4294757621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:22.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:22 smithi006 bash[56298]: level=error ts=2023-08-27T22:09:22.330Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:23.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:09:23.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:23.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:09:23.567Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:24.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:24 smithi130 bash[27579]: cluster 2023-08-27T22:09:23.105787+0000 mgr.y (mgr.14901) 326 : cluster [DBG] pgmap v327: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 558 op/s 2023-08-27T22:09:24.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:24 smithi006 bash[27292]: cluster 2023-08-27T22:09:23.105787+0000 mgr.y (mgr.14901) 326 : cluster [DBG] pgmap v327: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 558 op/s 2023-08-27T22:09:24.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:24 smithi006 bash[33918]: cluster 2023-08-27T22:09:23.105787+0000 mgr.y (mgr.14901) 326 : cluster [DBG] pgmap v327: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 558 op/s 2023-08-27T22:09:25.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:25 smithi130 bash[27579]: audit 2023-08-27T22:09:24.778491+0000 mon.c (mon.2) 152 : audit [DBG] from='client.? 172.21.15.6:0/1127635768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:25.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:25 smithi006 bash[27292]: audit 2023-08-27T22:09:24.778491+0000 mon.c (mon.2) 152 : audit [DBG] from='client.? 172.21.15.6:0/1127635768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:25.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:25 smithi006 bash[33918]: audit 2023-08-27T22:09:24.778491+0000 mon.c (mon.2) 152 : audit [DBG] from='client.? 172.21.15.6:0/1127635768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:26.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:26 smithi130 bash[27579]: cluster 2023-08-27T22:09:25.106593+0000 mgr.y (mgr.14901) 327 : cluster [DBG] pgmap v328: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2023-08-27T22:09:26.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:26 smithi130 bash[27579]: audit 2023-08-27T22:09:25.497793+0000 mon.b (mon.1) 142 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:09:26.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:26 smithi006 bash[27292]: cluster 2023-08-27T22:09:25.106593+0000 mgr.y (mgr.14901) 327 : cluster [DBG] pgmap v328: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2023-08-27T22:09:26.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:26 smithi006 bash[27292]: audit 2023-08-27T22:09:25.497793+0000 mon.b (mon.1) 142 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:09:26.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:26 smithi006 bash[33918]: cluster 2023-08-27T22:09:25.106593+0000 mgr.y (mgr.14901) 327 : cluster [DBG] pgmap v328: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2023-08-27T22:09:26.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:26 smithi006 bash[33918]: audit 2023-08-27T22:09:25.497793+0000 mon.b (mon.1) 142 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:09:28.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:28 smithi130 bash[27579]: cluster 2023-08-27T22:09:27.107096+0000 mgr.y (mgr.14901) 328 : cluster [DBG] pgmap v329: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-08-27T22:09:28.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:28 smithi130 bash[27579]: audit 2023-08-27T22:09:27.445158+0000 mon.c (mon.2) 153 : audit [DBG] from='client.? 172.21.15.6:0/1917542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:28.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:28 smithi006 bash[27292]: cluster 2023-08-27T22:09:27.107096+0000 mgr.y (mgr.14901) 328 : cluster [DBG] pgmap v329: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-08-27T22:09:28.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:28 smithi006 bash[27292]: audit 2023-08-27T22:09:27.445158+0000 mon.c (mon.2) 153 : audit [DBG] from='client.? 172.21.15.6:0/1917542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:28.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:28 smithi006 bash[33918]: cluster 2023-08-27T22:09:27.107096+0000 mgr.y (mgr.14901) 328 : cluster [DBG] pgmap v329: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-08-27T22:09:28.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:28 smithi006 bash[33918]: audit 2023-08-27T22:09:27.445158+0000 mon.c (mon.2) 153 : audit [DBG] from='client.? 172.21.15.6:0/1917542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:30.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:30 smithi130 bash[27579]: cluster 2023-08-27T22:09:29.107475+0000 mgr.y (mgr.14901) 329 : cluster [DBG] pgmap v330: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 558 op/s 2023-08-27T22:09:30.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:30 smithi130 bash[27579]: audit 2023-08-27T22:09:30.138188+0000 mon.a (mon.0) 996 : audit [DBG] from='client.? 172.21.15.6:0/1519058673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:30.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:30 smithi006 bash[27292]: cluster 2023-08-27T22:09:29.107475+0000 mgr.y (mgr.14901) 329 : cluster [DBG] pgmap v330: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 558 op/s 2023-08-27T22:09:30.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:30 smithi006 bash[27292]: audit 2023-08-27T22:09:30.138188+0000 mon.a (mon.0) 996 : audit [DBG] from='client.? 172.21.15.6:0/1519058673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:30.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:30 smithi006 bash[33918]: cluster 2023-08-27T22:09:29.107475+0000 mgr.y (mgr.14901) 329 : cluster [DBG] pgmap v330: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 558 op/s 2023-08-27T22:09:30.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:30 smithi006 bash[33918]: audit 2023-08-27T22:09:30.138188+0000 mon.a (mon.0) 996 : audit [DBG] from='client.? 172.21.15.6:0/1519058673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:31.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:09:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:09:30] "GET /metrics HTTP/1.1" 200 195273 "" "Prometheus/2.33.4" 2023-08-27T22:09:31.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:09:31.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:31.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:09:31.674Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:33.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:32 smithi130 bash[27579]: cluster 2023-08-27T22:09:31.108300+0000 mgr.y (mgr.14901) 330 : cluster [DBG] pgmap v331: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 828 op/s 2023-08-27T22:09:33.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:32 smithi130 bash[27579]: audit 2023-08-27T22:09:31.757838+0000 mon.a (mon.0) 997 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:09:33.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:32 smithi130 bash[27579]: audit 2023-08-27T22:09:31.875653+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:09:33.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:32 smithi130 bash[27579]: audit 2023-08-27T22:09:32.198008+0000 mon.b (mon.1) 143 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:09:33.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:32 smithi130 bash[27579]: audit 2023-08-27T22:09:32.199269+0000 mon.b (mon.1) 144 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:09:33.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:32 smithi130 bash[27579]: audit 2023-08-27T22:09:32.209922+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:09:33.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:32 smithi006 bash[33918]: cluster 2023-08-27T22:09:31.108300+0000 mgr.y (mgr.14901) 330 : cluster [DBG] pgmap v331: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 828 op/s 2023-08-27T22:09:33.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:32 smithi006 bash[33918]: audit 2023-08-27T22:09:31.757838+0000 mon.a (mon.0) 997 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:09:33.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:32 smithi006 bash[33918]: audit 2023-08-27T22:09:31.875653+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:09:33.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:32 smithi006 bash[33918]: audit 2023-08-27T22:09:32.198008+0000 mon.b (mon.1) 143 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:09:33.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:32 smithi006 bash[33918]: audit 2023-08-27T22:09:32.199269+0000 mon.b (mon.1) 144 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:09:33.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:32 smithi006 bash[33918]: audit 2023-08-27T22:09:32.209922+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:09:33.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:32 smithi006 bash[27292]: cluster 2023-08-27T22:09:31.108300+0000 mgr.y (mgr.14901) 330 : cluster [DBG] pgmap v331: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 828 op/s 2023-08-27T22:09:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:32 smithi006 bash[27292]: audit 2023-08-27T22:09:31.757838+0000 mon.a (mon.0) 997 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:09:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:32 smithi006 bash[27292]: audit 2023-08-27T22:09:31.875653+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:09:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:32 smithi006 bash[27292]: audit 2023-08-27T22:09:32.198008+0000 mon.b (mon.1) 143 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:09:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:32 smithi006 bash[27292]: audit 2023-08-27T22:09:32.199269+0000 mon.b (mon.1) 144 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:09:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:32 smithi006 bash[27292]: audit 2023-08-27T22:09:32.209922+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:09:33.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:33 smithi006 bash[27292]: audit 2023-08-27T22:09:32.847509+0000 mon.b (mon.1) 145 : audit [DBG] from='client.? 172.21.15.6:0/2415120340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:33.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:33 smithi006 bash[33918]: audit 2023-08-27T22:09:32.847509+0000 mon.b (mon.1) 145 : audit [DBG] from='client.? 172.21.15.6:0/2415120340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:09:33.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:09:33.567Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:34.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:33 smithi130 bash[27579]: audit 2023-08-27T22:09:32.847509+0000 mon.b (mon.1) 145 : audit [DBG] from='client.? 172.21.15.6:0/2415120340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:35.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:34 smithi130 bash[27579]: cluster 2023-08-27T22:09:33.109046+0000 mgr.y (mgr.14901) 331 : cluster [DBG] pgmap v332: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-08-27T22:09:35.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:34 smithi006 bash[27292]: cluster 2023-08-27T22:09:33.109046+0000 mgr.y (mgr.14901) 331 : cluster [DBG] pgmap v332: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-08-27T22:09:35.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:34 smithi006 bash[33918]: cluster 2023-08-27T22:09:33.109046+0000 mgr.y (mgr.14901) 331 : cluster [DBG] pgmap v332: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-08-27T22:09:36.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:35 smithi130 bash[27579]: audit 2023-08-27T22:09:35.522078+0000 mon.b (mon.1) 146 : audit [DBG] from='client.? 172.21.15.6:0/670182605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:36.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:35 smithi006 bash[27292]: audit 2023-08-27T22:09:35.522078+0000 mon.b (mon.1) 146 : audit [DBG] from='client.? 172.21.15.6:0/670182605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:36.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:35 smithi006 bash[33918]: audit 2023-08-27T22:09:35.522078+0000 mon.b (mon.1) 146 : audit [DBG] from='client.? 172.21.15.6:0/670182605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:37.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:36 smithi130 bash[27579]: cluster 2023-08-27T22:09:35.109903+0000 mgr.y (mgr.14901) 332 : cluster [DBG] pgmap v333: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2023-08-27T22:09:37.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:36 smithi006 bash[33918]: cluster 2023-08-27T22:09:35.109903+0000 mgr.y (mgr.14901) 332 : cluster [DBG] pgmap v333: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2023-08-27T22:09:37.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:36 smithi006 bash[27292]: cluster 2023-08-27T22:09:35.109903+0000 mgr.y (mgr.14901) 332 : cluster [DBG] pgmap v333: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2023-08-27T22:09:39.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:38 smithi130 bash[27579]: cluster 2023-08-27T22:09:37.111527+0000 mgr.y (mgr.14901) 333 : cluster [DBG] pgmap v334: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-08-27T22:09:39.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:38 smithi130 bash[27579]: audit 2023-08-27T22:09:38.240328+0000 mon.c (mon.2) 154 : audit [DBG] from='client.? 172.21.15.6:0/350740790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:39.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:38 smithi006 bash[27292]: cluster 2023-08-27T22:09:37.111527+0000 mgr.y (mgr.14901) 333 : cluster [DBG] pgmap v334: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-08-27T22:09:39.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:38 smithi006 bash[27292]: audit 2023-08-27T22:09:38.240328+0000 mon.c (mon.2) 154 : audit [DBG] from='client.? 172.21.15.6:0/350740790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:39.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:38 smithi006 bash[33918]: cluster 2023-08-27T22:09:37.111527+0000 mgr.y (mgr.14901) 333 : cluster [DBG] pgmap v334: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-08-27T22:09:39.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:38 smithi006 bash[33918]: audit 2023-08-27T22:09:38.240328+0000 mon.c (mon.2) 154 : audit [DBG] from='client.? 172.21.15.6:0/350740790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:41.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:40 smithi130 bash[27579]: cluster 2023-08-27T22:09:39.111984+0000 mgr.y (mgr.14901) 334 : cluster [DBG] pgmap v335: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 533 op/s 2023-08-27T22:09:41.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:40 smithi006 bash[27292]: cluster 2023-08-27T22:09:39.111984+0000 mgr.y (mgr.14901) 334 : cluster [DBG] pgmap v335: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 533 op/s 2023-08-27T22:09:41.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:09:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:09:40] "GET /metrics HTTP/1.1" 200 195273 "" "Prometheus/2.33.4" 2023-08-27T22:09:41.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:40 smithi006 bash[33918]: cluster 2023-08-27T22:09:39.111984+0000 mgr.y (mgr.14901) 334 : cluster [DBG] pgmap v335: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 533 op/s 2023-08-27T22:09:41.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:41 smithi006 bash[27292]: audit 2023-08-27T22:09:40.890598+0000 mon.c (mon.2) 155 : audit [DBG] from='client.? 172.21.15.6:0/2493510416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:41.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:41 smithi006 bash[33918]: audit 2023-08-27T22:09:40.890598+0000 mon.c (mon.2) 155 : audit [DBG] from='client.? 172.21.15.6:0/2493510416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:41.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:09:41.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:41.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:09:41.675Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:42.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:41 smithi130 bash[27579]: audit 2023-08-27T22:09:40.890598+0000 mon.c (mon.2) 155 : audit [DBG] from='client.? 172.21.15.6:0/2493510416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:43.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:42 smithi130 bash[27579]: cluster 2023-08-27T22:09:41.112718+0000 mgr.y (mgr.14901) 335 : cluster [DBG] pgmap v336: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-08-27T22:09:43.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:42 smithi006 bash[27292]: cluster 2023-08-27T22:09:41.112718+0000 mgr.y (mgr.14901) 335 : cluster [DBG] pgmap v336: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-08-27T22:09:43.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:42 smithi006 bash[33918]: cluster 2023-08-27T22:09:41.112718+0000 mgr.y (mgr.14901) 335 : cluster [DBG] pgmap v336: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-08-27T22:09:43.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:09:43.568Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:43.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:09:43.569Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:44.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:44 smithi130 bash[27579]: cluster 2023-08-27T22:09:43.113177+0000 mgr.y (mgr.14901) 336 : cluster [DBG] pgmap v337: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 533 op/s 2023-08-27T22:09:44.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:44 smithi130 bash[27579]: audit 2023-08-27T22:09:43.612300+0000 mon.a (mon.0) 1000 : audit [DBG] from='client.? 172.21.15.6:0/982362239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:44.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:44 smithi006 bash[33918]: cluster 2023-08-27T22:09:43.113177+0000 mgr.y (mgr.14901) 336 : cluster [DBG] pgmap v337: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 533 op/s 2023-08-27T22:09:44.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:44 smithi006 bash[33918]: audit 2023-08-27T22:09:43.612300+0000 mon.a (mon.0) 1000 : audit [DBG] from='client.? 172.21.15.6:0/982362239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:44.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:44 smithi006 bash[27292]: cluster 2023-08-27T22:09:43.113177+0000 mgr.y (mgr.14901) 336 : cluster [DBG] pgmap v337: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 533 op/s 2023-08-27T22:09:44.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:44 smithi006 bash[27292]: audit 2023-08-27T22:09:43.612300+0000 mon.a (mon.0) 1000 : audit [DBG] from='client.? 172.21.15.6:0/982362239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:46.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:46 smithi130 bash[27579]: cluster 2023-08-27T22:09:45.113924+0000 mgr.y (mgr.14901) 337 : cluster [DBG] pgmap v338: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-08-27T22:09:46.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:46 smithi006 bash[27292]: cluster 2023-08-27T22:09:45.113924+0000 mgr.y (mgr.14901) 337 : cluster [DBG] pgmap v338: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-08-27T22:09:46.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:46 smithi006 bash[33918]: cluster 2023-08-27T22:09:45.113924+0000 mgr.y (mgr.14901) 337 : cluster [DBG] pgmap v338: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-08-27T22:09:47.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:47 smithi130 bash[27579]: audit 2023-08-27T22:09:46.329956+0000 mon.a (mon.0) 1001 : audit [DBG] from='client.? 172.21.15.6:0/3735730193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:47.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:47 smithi006 bash[27292]: audit 2023-08-27T22:09:46.329956+0000 mon.a (mon.0) 1001 : audit [DBG] from='client.? 172.21.15.6:0/3735730193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:47.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:47 smithi006 bash[33918]: audit 2023-08-27T22:09:46.329956+0000 mon.a (mon.0) 1001 : audit [DBG] from='client.? 172.21.15.6:0/3735730193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:48.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:48 smithi130 bash[27579]: cluster 2023-08-27T22:09:47.114452+0000 mgr.y (mgr.14901) 338 : cluster [DBG] pgmap v339: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-08-27T22:09:48.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:48 smithi006 bash[27292]: cluster 2023-08-27T22:09:47.114452+0000 mgr.y (mgr.14901) 338 : cluster [DBG] pgmap v339: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-08-27T22:09:48.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:48 smithi006 bash[33918]: cluster 2023-08-27T22:09:47.114452+0000 mgr.y (mgr.14901) 338 : cluster [DBG] pgmap v339: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-08-27T22:09:49.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:49 smithi130 bash[27579]: audit 2023-08-27T22:09:49.004518+0000 mon.a (mon.0) 1002 : audit [DBG] from='client.? 172.21.15.6:0/883044311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:49.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:49 smithi006 bash[27292]: audit 2023-08-27T22:09:49.004518+0000 mon.a (mon.0) 1002 : audit [DBG] from='client.? 172.21.15.6:0/883044311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:49.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:49 smithi006 bash[33918]: audit 2023-08-27T22:09:49.004518+0000 mon.a (mon.0) 1002 : audit [DBG] from='client.? 172.21.15.6:0/883044311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:50.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:50 smithi130 bash[27579]: cluster 2023-08-27T22:09:49.114903+0000 mgr.y (mgr.14901) 339 : cluster [DBG] pgmap v340: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 539 op/s 2023-08-27T22:09:50.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:50 smithi006 bash[27292]: cluster 2023-08-27T22:09:49.114903+0000 mgr.y (mgr.14901) 339 : cluster [DBG] pgmap v340: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 539 op/s 2023-08-27T22:09:50.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:50 smithi006 bash[33918]: cluster 2023-08-27T22:09:49.114903+0000 mgr.y (mgr.14901) 339 : cluster [DBG] pgmap v340: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 539 op/s 2023-08-27T22:09:51.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:09:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:09:50] "GET /metrics HTTP/1.1" 200 195255 "" "Prometheus/2.33.4" 2023-08-27T22:09:51.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:09:51.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:51.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:09:51.675Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:52.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:52 smithi130 bash[27579]: cluster 2023-08-27T22:09:51.115654+0000 mgr.y (mgr.14901) 340 : cluster [DBG] pgmap v341: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2023-08-27T22:09:52.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:52 smithi130 bash[27579]: audit 2023-08-27T22:09:51.694938+0000 mon.a (mon.0) 1003 : audit [DBG] from='client.? 172.21.15.6:0/2754079614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:52.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:52 smithi006 bash[27292]: cluster 2023-08-27T22:09:51.115654+0000 mgr.y (mgr.14901) 340 : cluster [DBG] pgmap v341: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2023-08-27T22:09:52.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:52 smithi006 bash[27292]: audit 2023-08-27T22:09:51.694938+0000 mon.a (mon.0) 1003 : audit [DBG] from='client.? 172.21.15.6:0/2754079614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:52.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:52 smithi006 bash[33918]: cluster 2023-08-27T22:09:51.115654+0000 mgr.y (mgr.14901) 340 : cluster [DBG] pgmap v341: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2023-08-27T22:09:52.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:52 smithi006 bash[33918]: audit 2023-08-27T22:09:51.694938+0000 mon.a (mon.0) 1003 : audit [DBG] from='client.? 172.21.15.6:0/2754079614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:53.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:09:53.568Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:53.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:09:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:09:53.570Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:09:54.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:54 smithi130 bash[27579]: cluster 2023-08-27T22:09:53.116118+0000 mgr.y (mgr.14901) 341 : cluster [DBG] pgmap v342: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 544 op/s 2023-08-27T22:09:54.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:54 smithi006 bash[27292]: cluster 2023-08-27T22:09:53.116118+0000 mgr.y (mgr.14901) 341 : cluster [DBG] pgmap v342: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 544 op/s 2023-08-27T22:09:54.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:54 smithi006 bash[33918]: cluster 2023-08-27T22:09:53.116118+0000 mgr.y (mgr.14901) 341 : cluster [DBG] pgmap v342: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 544 op/s 2023-08-27T22:09:55.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:55 smithi130 bash[27579]: audit 2023-08-27T22:09:54.407362+0000 mon.a (mon.0) 1004 : audit [DBG] from='client.? 172.21.15.6:0/2698466367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:55.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:55 smithi006 bash[27292]: audit 2023-08-27T22:09:54.407362+0000 mon.a (mon.0) 1004 : audit [DBG] from='client.? 172.21.15.6:0/2698466367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:55.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:55 smithi006 bash[33918]: audit 2023-08-27T22:09:54.407362+0000 mon.a (mon.0) 1004 : audit [DBG] from='client.? 172.21.15.6:0/2698466367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:56.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:56 smithi130 bash[27579]: cluster 2023-08-27T22:09:55.116811+0000 mgr.y (mgr.14901) 342 : cluster [DBG] pgmap v343: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-08-27T22:09:56.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:56 smithi006 bash[27292]: cluster 2023-08-27T22:09:55.116811+0000 mgr.y (mgr.14901) 342 : cluster [DBG] pgmap v343: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-08-27T22:09:56.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:56 smithi006 bash[33918]: cluster 2023-08-27T22:09:55.116811+0000 mgr.y (mgr.14901) 342 : cluster [DBG] pgmap v343: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-08-27T22:09:57.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:57 smithi130 bash[27579]: audit 2023-08-27T22:09:57.046866+0000 mon.a (mon.0) 1005 : audit [DBG] from='client.? 172.21.15.6:0/2310071514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:57.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:57 smithi006 bash[27292]: audit 2023-08-27T22:09:57.046866+0000 mon.a (mon.0) 1005 : audit [DBG] from='client.? 172.21.15.6:0/2310071514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:57.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:57 smithi006 bash[33918]: audit 2023-08-27T22:09:57.046866+0000 mon.a (mon.0) 1005 : audit [DBG] from='client.? 172.21.15.6:0/2310071514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:58.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:09:58 smithi130 bash[27579]: cluster 2023-08-27T22:09:57.117495+0000 mgr.y (mgr.14901) 343 : cluster [DBG] pgmap v344: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-08-27T22:09:58.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:09:58 smithi006 bash[27292]: cluster 2023-08-27T22:09:57.117495+0000 mgr.y (mgr.14901) 343 : cluster [DBG] pgmap v344: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-08-27T22:09:58.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:09:58 smithi006 bash[33918]: cluster 2023-08-27T22:09:57.117495+0000 mgr.y (mgr.14901) 343 : cluster [DBG] pgmap v344: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-08-27T22:10:00.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:00 smithi130 bash[27579]: cluster 2023-08-27T22:09:59.117985+0000 mgr.y (mgr.14901) 344 : cluster [DBG] pgmap v345: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 540 op/s 2023-08-27T22:10:00.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:00 smithi130 bash[27579]: audit 2023-08-27T22:09:59.735169+0000 mon.b (mon.1) 147 : audit [DBG] from='client.? 172.21.15.6:0/2569589793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:00.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:00 smithi130 bash[27579]: cluster 2023-08-27T22:10:00.000147+0000 mon.a (mon.0) 1006 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:10:00.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:00 smithi006 bash[27292]: cluster 2023-08-27T22:09:59.117985+0000 mgr.y (mgr.14901) 344 : cluster [DBG] pgmap v345: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 540 op/s 2023-08-27T22:10:00.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:00 smithi006 bash[27292]: audit 2023-08-27T22:09:59.735169+0000 mon.b (mon.1) 147 : audit [DBG] from='client.? 172.21.15.6:0/2569589793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:00.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:00 smithi006 bash[27292]: cluster 2023-08-27T22:10:00.000147+0000 mon.a (mon.0) 1006 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:10:00.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:00 smithi006 bash[33918]: cluster 2023-08-27T22:09:59.117985+0000 mgr.y (mgr.14901) 344 : cluster [DBG] pgmap v345: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 540 op/s 2023-08-27T22:10:00.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:00 smithi006 bash[33918]: audit 2023-08-27T22:09:59.735169+0000 mon.b (mon.1) 147 : audit [DBG] from='client.? 172.21.15.6:0/2569589793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:00.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:00 smithi006 bash[33918]: cluster 2023-08-27T22:10:00.000147+0000 mon.a (mon.0) 1006 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:10:01.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:10:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:10:00] "GET /metrics HTTP/1.1" 200 195268 "" "Prometheus/2.33.4" 2023-08-27T22:10:01.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:10:01.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:01.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:10:01.676Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:02.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:02 smithi130 bash[27579]: cluster 2023-08-27T22:10:01.118775+0000 mgr.y (mgr.14901) 345 : cluster [DBG] pgmap v346: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-08-27T22:10:02.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:02 smithi006 bash[27292]: cluster 2023-08-27T22:10:01.118775+0000 mgr.y (mgr.14901) 345 : cluster [DBG] pgmap v346: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-08-27T22:10:02.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:02 smithi006 bash[33918]: cluster 2023-08-27T22:10:01.118775+0000 mgr.y (mgr.14901) 345 : cluster [DBG] pgmap v346: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-08-27T22:10:03.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:03 smithi130 bash[27579]: audit 2023-08-27T22:10:02.389274+0000 mon.c (mon.2) 156 : audit [DBG] from='client.? 172.21.15.6:0/2579753889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:03.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:10:03.572Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:03.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:10:03.574Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:03.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:03 smithi006 bash[33918]: audit 2023-08-27T22:10:02.389274+0000 mon.c (mon.2) 156 : audit [DBG] from='client.? 172.21.15.6:0/2579753889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:03.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:03 smithi006 bash[27292]: audit 2023-08-27T22:10:02.389274+0000 mon.c (mon.2) 156 : audit [DBG] from='client.? 172.21.15.6:0/2579753889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:04.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:04 smithi130 bash[27579]: cluster 2023-08-27T22:10:03.119150+0000 mgr.y (mgr.14901) 346 : cluster [DBG] pgmap v347: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 527 op/s 2023-08-27T22:10:04.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:04 smithi006 bash[27292]: cluster 2023-08-27T22:10:03.119150+0000 mgr.y (mgr.14901) 346 : cluster [DBG] pgmap v347: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 527 op/s 2023-08-27T22:10:04.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:04 smithi006 bash[33918]: cluster 2023-08-27T22:10:03.119150+0000 mgr.y (mgr.14901) 346 : cluster [DBG] pgmap v347: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 527 op/s 2023-08-27T22:10:05.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:05 smithi006 bash[27292]: audit 2023-08-27T22:10:05.064488+0000 mon.c (mon.2) 157 : audit [DBG] from='client.? 172.21.15.6:0/755887088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:05.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:05 smithi006 bash[33918]: audit 2023-08-27T22:10:05.064488+0000 mon.c (mon.2) 157 : audit [DBG] from='client.? 172.21.15.6:0/755887088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:05.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:05 smithi130 bash[27579]: audit 2023-08-27T22:10:05.064488+0000 mon.c (mon.2) 157 : audit [DBG] from='client.? 172.21.15.6:0/755887088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:06.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:06 smithi006 bash[33918]: cluster 2023-08-27T22:10:05.119839+0000 mgr.y (mgr.14901) 347 : cluster [DBG] pgmap v348: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-08-27T22:10:06.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:06 smithi006 bash[27292]: cluster 2023-08-27T22:10:05.119839+0000 mgr.y (mgr.14901) 347 : cluster [DBG] pgmap v348: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-08-27T22:10:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:06 smithi130 bash[27579]: cluster 2023-08-27T22:10:05.119839+0000 mgr.y (mgr.14901) 347 : cluster [DBG] pgmap v348: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-08-27T22:10:08.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:08 smithi006 bash[27292]: cluster 2023-08-27T22:10:07.120416+0000 mgr.y (mgr.14901) 348 : cluster [DBG] pgmap v349: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-08-27T22:10:08.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:08 smithi006 bash[27292]: audit 2023-08-27T22:10:07.740092+0000 mon.a (mon.0) 1007 : audit [DBG] from='client.? 172.21.15.6:0/1291843454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:08.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:08 smithi006 bash[33918]: cluster 2023-08-27T22:10:07.120416+0000 mgr.y (mgr.14901) 348 : cluster [DBG] pgmap v349: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-08-27T22:10:08.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:08 smithi006 bash[33918]: audit 2023-08-27T22:10:07.740092+0000 mon.a (mon.0) 1007 : audit [DBG] from='client.? 172.21.15.6:0/1291843454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:08.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:08 smithi130 bash[27579]: cluster 2023-08-27T22:10:07.120416+0000 mgr.y (mgr.14901) 348 : cluster [DBG] pgmap v349: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-08-27T22:10:08.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:08 smithi130 bash[27579]: audit 2023-08-27T22:10:07.740092+0000 mon.a (mon.0) 1007 : audit [DBG] from='client.? 172.21.15.6:0/1291843454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:10.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:10 smithi006 bash[27292]: cluster 2023-08-27T22:10:09.120907+0000 mgr.y (mgr.14901) 349 : cluster [DBG] pgmap v350: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 528 op/s 2023-08-27T22:10:10.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:10 smithi006 bash[33918]: cluster 2023-08-27T22:10:09.120907+0000 mgr.y (mgr.14901) 349 : cluster [DBG] pgmap v350: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 528 op/s 2023-08-27T22:10:10.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:10 smithi130 bash[27579]: cluster 2023-08-27T22:10:09.120907+0000 mgr.y (mgr.14901) 349 : cluster [DBG] pgmap v350: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 528 op/s 2023-08-27T22:10:11.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:10:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:10:10] "GET /metrics HTTP/1.1" 200 195268 "" "Prometheus/2.33.4" 2023-08-27T22:10:11.681 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:11 smithi006 bash[27292]: audit 2023-08-27T22:10:10.451782+0000 mon.c (mon.2) 158 : audit [DBG] from='client.? 172.21.15.6:0/796992041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:11.681 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:11 smithi006 bash[33918]: audit 2023-08-27T22:10:10.451782+0000 mon.c (mon.2) 158 : audit [DBG] from='client.? 172.21.15.6:0/796992041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:11.682 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:10:11.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:11.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:11 smithi130 bash[27579]: audit 2023-08-27T22:10:10.451782+0000 mon.c (mon.2) 158 : audit [DBG] from='client.? 172.21.15.6:0/796992041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:11.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:10:11.676Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:12.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:12 smithi006 bash[27292]: cluster 2023-08-27T22:10:11.121810+0000 mgr.y (mgr.14901) 350 : cluster [DBG] pgmap v351: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 796 op/s 2023-08-27T22:10:12.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:12 smithi006 bash[33918]: cluster 2023-08-27T22:10:11.121810+0000 mgr.y (mgr.14901) 350 : cluster [DBG] pgmap v351: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 796 op/s 2023-08-27T22:10:12.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:12 smithi130 bash[27579]: cluster 2023-08-27T22:10:11.121810+0000 mgr.y (mgr.14901) 350 : cluster [DBG] pgmap v351: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 796 op/s 2023-08-27T22:10:13.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:13 smithi006 bash[27292]: audit 2023-08-27T22:10:13.145334+0000 mon.c (mon.2) 159 : audit [DBG] from='client.? 172.21.15.6:0/3375928750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:13.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:13 smithi006 bash[33918]: audit 2023-08-27T22:10:13.145334+0000 mon.c (mon.2) 159 : audit [DBG] from='client.? 172.21.15.6:0/3375928750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:13.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:10:13.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:13.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:10:13.575Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:13.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:13 smithi130 bash[27579]: audit 2023-08-27T22:10:13.145334+0000 mon.c (mon.2) 159 : audit [DBG] from='client.? 172.21.15.6:0/3375928750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:14.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:14 smithi006 bash[33918]: cluster 2023-08-27T22:10:13.122240+0000 mgr.y (mgr.14901) 351 : cluster [DBG] pgmap v352: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 534 op/s 2023-08-27T22:10:14.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:14 smithi006 bash[27292]: cluster 2023-08-27T22:10:13.122240+0000 mgr.y (mgr.14901) 351 : cluster [DBG] pgmap v352: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 534 op/s 2023-08-27T22:10:14.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:14 smithi130 bash[27579]: cluster 2023-08-27T22:10:13.122240+0000 mgr.y (mgr.14901) 351 : cluster [DBG] pgmap v352: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 534 op/s 2023-08-27T22:10:16.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:16 smithi006 bash[27292]: cluster 2023-08-27T22:10:15.123497+0000 mgr.y (mgr.14901) 352 : cluster [DBG] pgmap v353: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-08-27T22:10:16.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:16 smithi006 bash[27292]: audit 2023-08-27T22:10:15.839201+0000 mon.c (mon.2) 160 : audit [DBG] from='client.? 172.21.15.6:0/3402176969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:16.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:16 smithi006 bash[33918]: cluster 2023-08-27T22:10:15.123497+0000 mgr.y (mgr.14901) 352 : cluster [DBG] pgmap v353: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-08-27T22:10:16.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:16 smithi006 bash[33918]: audit 2023-08-27T22:10:15.839201+0000 mon.c (mon.2) 160 : audit [DBG] from='client.? 172.21.15.6:0/3402176969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:16.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:16 smithi130 bash[27579]: cluster 2023-08-27T22:10:15.123497+0000 mgr.y (mgr.14901) 352 : cluster [DBG] pgmap v353: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-08-27T22:10:16.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:16 smithi130 bash[27579]: audit 2023-08-27T22:10:15.839201+0000 mon.c (mon.2) 160 : audit [DBG] from='client.? 172.21.15.6:0/3402176969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:18.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:18 smithi006 bash[27292]: cluster 2023-08-27T22:10:17.126518+0000 mgr.y (mgr.14901) 353 : cluster [DBG] pgmap v354: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-08-27T22:10:18.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:18 smithi006 bash[33918]: cluster 2023-08-27T22:10:17.126518+0000 mgr.y (mgr.14901) 353 : cluster [DBG] pgmap v354: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-08-27T22:10:18.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:18 smithi130 bash[27579]: cluster 2023-08-27T22:10:17.126518+0000 mgr.y (mgr.14901) 353 : cluster [DBG] pgmap v354: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-08-27T22:10:19.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:19 smithi006 bash[27292]: audit 2023-08-27T22:10:18.523346+0000 mon.a (mon.0) 1008 : audit [DBG] from='client.? 172.21.15.6:0/1765785263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:19.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:19 smithi006 bash[33918]: audit 2023-08-27T22:10:18.523346+0000 mon.a (mon.0) 1008 : audit [DBG] from='client.? 172.21.15.6:0/1765785263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:19.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:19 smithi130 bash[27579]: audit 2023-08-27T22:10:18.523346+0000 mon.a (mon.0) 1008 : audit [DBG] from='client.? 172.21.15.6:0/1765785263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:20.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:20 smithi006 bash[27292]: cluster 2023-08-27T22:10:19.126959+0000 mgr.y (mgr.14901) 354 : cluster [DBG] pgmap v355: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2023-08-27T22:10:20.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:20 smithi006 bash[33918]: cluster 2023-08-27T22:10:19.126959+0000 mgr.y (mgr.14901) 354 : cluster [DBG] pgmap v355: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2023-08-27T22:10:20.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:20 smithi130 bash[27579]: cluster 2023-08-27T22:10:19.126959+0000 mgr.y (mgr.14901) 354 : cluster [DBG] pgmap v355: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2023-08-27T22:10:21.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:10:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:10:20] "GET /metrics HTTP/1.1" 200 195264 "" "Prometheus/2.33.4" 2023-08-27T22:10:21.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:21 smithi006 bash[27292]: audit 2023-08-27T22:10:21.207655+0000 mon.a (mon.0) 1009 : audit [DBG] from='client.? 172.21.15.6:0/59499376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:21.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:21 smithi006 bash[33918]: audit 2023-08-27T22:10:21.207655+0000 mon.a (mon.0) 1009 : audit [DBG] from='client.? 172.21.15.6:0/59499376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:21.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:10:21.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:21.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:10:21.677Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:21.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:21 smithi130 bash[27579]: audit 2023-08-27T22:10:21.207655+0000 mon.a (mon.0) 1009 : audit [DBG] from='client.? 172.21.15.6:0/59499376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:22.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:22 smithi130 bash[27579]: cluster 2023-08-27T22:10:21.127766+0000 mgr.y (mgr.14901) 355 : cluster [DBG] pgmap v356: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 803 op/s 2023-08-27T22:10:22.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:22 smithi006 bash[27292]: cluster 2023-08-27T22:10:21.127766+0000 mgr.y (mgr.14901) 355 : cluster [DBG] pgmap v356: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 803 op/s 2023-08-27T22:10:22.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:22 smithi006 bash[33918]: cluster 2023-08-27T22:10:21.127766+0000 mgr.y (mgr.14901) 355 : cluster [DBG] pgmap v356: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 803 op/s 2023-08-27T22:10:23.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:10:23.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:23.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:10:23.578Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:24.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:24 smithi130 bash[27579]: cluster 2023-08-27T22:10:23.128206+0000 mgr.y (mgr.14901) 356 : cluster [DBG] pgmap v357: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 535 op/s 2023-08-27T22:10:24.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:24 smithi130 bash[27579]: audit 2023-08-27T22:10:23.876734+0000 mon.a (mon.0) 1010 : audit [DBG] from='client.? 172.21.15.6:0/249312402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:24.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:24 smithi006 bash[27292]: cluster 2023-08-27T22:10:23.128206+0000 mgr.y (mgr.14901) 356 : cluster [DBG] pgmap v357: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 535 op/s 2023-08-27T22:10:24.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:24 smithi006 bash[27292]: audit 2023-08-27T22:10:23.876734+0000 mon.a (mon.0) 1010 : audit [DBG] from='client.? 172.21.15.6:0/249312402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:24.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:24 smithi006 bash[33918]: cluster 2023-08-27T22:10:23.128206+0000 mgr.y (mgr.14901) 356 : cluster [DBG] pgmap v357: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 535 op/s 2023-08-27T22:10:24.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:24 smithi006 bash[33918]: audit 2023-08-27T22:10:23.876734+0000 mon.a (mon.0) 1010 : audit [DBG] from='client.? 172.21.15.6:0/249312402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:26.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:26 smithi130 bash[27579]: cluster 2023-08-27T22:10:25.128805+0000 mgr.y (mgr.14901) 357 : cluster [DBG] pgmap v358: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2023-08-27T22:10:26.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:26 smithi006 bash[27292]: cluster 2023-08-27T22:10:25.128805+0000 mgr.y (mgr.14901) 357 : cluster [DBG] pgmap v358: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2023-08-27T22:10:26.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:26 smithi006 bash[33918]: cluster 2023-08-27T22:10:25.128805+0000 mgr.y (mgr.14901) 357 : cluster [DBG] pgmap v358: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2023-08-27T22:10:27.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:27 smithi130 bash[27579]: audit 2023-08-27T22:10:26.597358+0000 mon.a (mon.0) 1011 : audit [DBG] from='client.? 172.21.15.6:0/2413879173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:27.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:27 smithi006 bash[27292]: audit 2023-08-27T22:10:26.597358+0000 mon.a (mon.0) 1011 : audit [DBG] from='client.? 172.21.15.6:0/2413879173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:27.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:27 smithi006 bash[33918]: audit 2023-08-27T22:10:26.597358+0000 mon.a (mon.0) 1011 : audit [DBG] from='client.? 172.21.15.6:0/2413879173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:28.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:28 smithi130 bash[27579]: cluster 2023-08-27T22:10:27.129601+0000 mgr.y (mgr.14901) 358 : cluster [DBG] pgmap v359: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-08-27T22:10:28.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:28 smithi006 bash[27292]: cluster 2023-08-27T22:10:27.129601+0000 mgr.y (mgr.14901) 358 : cluster [DBG] pgmap v359: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-08-27T22:10:28.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:28 smithi006 bash[33918]: cluster 2023-08-27T22:10:27.129601+0000 mgr.y (mgr.14901) 358 : cluster [DBG] pgmap v359: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-08-27T22:10:29.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:29 smithi130 bash[27579]: audit 2023-08-27T22:10:29.284202+0000 mon.a (mon.0) 1012 : audit [DBG] from='client.? 172.21.15.6:0/1519708220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:29.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:29 smithi006 bash[27292]: audit 2023-08-27T22:10:29.284202+0000 mon.a (mon.0) 1012 : audit [DBG] from='client.? 172.21.15.6:0/1519708220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:29.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:29 smithi006 bash[33918]: audit 2023-08-27T22:10:29.284202+0000 mon.a (mon.0) 1012 : audit [DBG] from='client.? 172.21.15.6:0/1519708220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:30.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:30 smithi130 bash[27579]: cluster 2023-08-27T22:10:29.130074+0000 mgr.y (mgr.14901) 359 : cluster [DBG] pgmap v360: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 540 op/s 2023-08-27T22:10:30.969 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:10:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:10:30] "GET /metrics HTTP/1.1" 200 195271 "" "Prometheus/2.33.4" 2023-08-27T22:10:30.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:30 smithi006 bash[33918]: cluster 2023-08-27T22:10:29.130074+0000 mgr.y (mgr.14901) 359 : cluster [DBG] pgmap v360: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 540 op/s 2023-08-27T22:10:30.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:30 smithi006 bash[27292]: cluster 2023-08-27T22:10:29.130074+0000 mgr.y (mgr.14901) 359 : cluster [DBG] pgmap v360: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 540 op/s 2023-08-27T22:10:31.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:10:31.676Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:31.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:10:31.678Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:32.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:32 smithi130 bash[27579]: cluster 2023-08-27T22:10:31.130836+0000 mgr.y (mgr.14901) 360 : cluster [DBG] pgmap v361: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2023-08-27T22:10:32.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:32 smithi130 bash[27579]: audit 2023-08-27T22:10:31.934746+0000 mon.c (mon.2) 161 : audit [DBG] from='client.? 172.21.15.6:0/557703222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:32.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:32 smithi130 bash[27579]: audit 2023-08-27T22:10:32.222106+0000 mon.b (mon.1) 148 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:10:32.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:32 smithi006 bash[33918]: cluster 2023-08-27T22:10:31.130836+0000 mgr.y (mgr.14901) 360 : cluster [DBG] pgmap v361: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2023-08-27T22:10:32.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:32 smithi006 bash[33918]: audit 2023-08-27T22:10:31.934746+0000 mon.c (mon.2) 161 : audit [DBG] from='client.? 172.21.15.6:0/557703222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:32.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:32 smithi006 bash[33918]: audit 2023-08-27T22:10:32.222106+0000 mon.b (mon.1) 148 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:10:32.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:32 smithi006 bash[27292]: cluster 2023-08-27T22:10:31.130836+0000 mgr.y (mgr.14901) 360 : cluster [DBG] pgmap v361: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2023-08-27T22:10:32.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:32 smithi006 bash[27292]: audit 2023-08-27T22:10:31.934746+0000 mon.c (mon.2) 161 : audit [DBG] from='client.? 172.21.15.6:0/557703222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:32.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:32 smithi006 bash[27292]: audit 2023-08-27T22:10:32.222106+0000 mon.b (mon.1) 148 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:10:33.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:10:33.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:33.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:10:33.576Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:34.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:34 smithi130 bash[27579]: cluster 2023-08-27T22:10:33.131317+0000 mgr.y (mgr.14901) 361 : cluster [DBG] pgmap v362: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 540 op/s 2023-08-27T22:10:34.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:34 smithi006 bash[33918]: cluster 2023-08-27T22:10:33.131317+0000 mgr.y (mgr.14901) 361 : cluster [DBG] pgmap v362: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 540 op/s 2023-08-27T22:10:34.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:34 smithi006 bash[27292]: cluster 2023-08-27T22:10:33.131317+0000 mgr.y (mgr.14901) 361 : cluster [DBG] pgmap v362: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 540 op/s 2023-08-27T22:10:35.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:35 smithi130 bash[27579]: audit 2023-08-27T22:10:34.598582+0000 mon.a (mon.0) 1013 : audit [DBG] from='client.? 172.21.15.6:0/4059748109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:35.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:35 smithi006 bash[27292]: audit 2023-08-27T22:10:34.598582+0000 mon.a (mon.0) 1013 : audit [DBG] from='client.? 172.21.15.6:0/4059748109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:35.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:35 smithi006 bash[33918]: audit 2023-08-27T22:10:34.598582+0000 mon.a (mon.0) 1013 : audit [DBG] from='client.? 172.21.15.6:0/4059748109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:36.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:36 smithi130 bash[27579]: cluster 2023-08-27T22:10:35.132043+0000 mgr.y (mgr.14901) 362 : cluster [DBG] pgmap v363: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-08-27T22:10:36.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:36 smithi006 bash[27292]: cluster 2023-08-27T22:10:35.132043+0000 mgr.y (mgr.14901) 362 : cluster [DBG] pgmap v363: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-08-27T22:10:36.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:36 smithi006 bash[33918]: cluster 2023-08-27T22:10:35.132043+0000 mgr.y (mgr.14901) 362 : cluster [DBG] pgmap v363: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-08-27T22:10:37.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:37 smithi130 bash[27579]: audit 2023-08-27T22:10:37.319385+0000 mon.c (mon.2) 162 : audit [DBG] from='client.? 172.21.15.6:0/505243036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:37.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:37 smithi006 bash[27292]: audit 2023-08-27T22:10:37.319385+0000 mon.c (mon.2) 162 : audit [DBG] from='client.? 172.21.15.6:0/505243036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:37.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:37 smithi006 bash[33918]: audit 2023-08-27T22:10:37.319385+0000 mon.c (mon.2) 162 : audit [DBG] from='client.? 172.21.15.6:0/505243036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:38.561 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:38 smithi006 bash[27292]: cluster 2023-08-27T22:10:37.136807+0000 mgr.y (mgr.14901) 363 : cluster [DBG] pgmap v364: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-08-27T22:10:38.561 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:38 smithi006 bash[27292]: audit 2023-08-27T22:10:38.385570+0000 mon.a (mon.0) 1014 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:10:38.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:38 smithi130 bash[27579]: cluster 2023-08-27T22:10:37.136807+0000 mgr.y (mgr.14901) 363 : cluster [DBG] pgmap v364: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-08-27T22:10:38.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:38 smithi130 bash[27579]: audit 2023-08-27T22:10:38.385570+0000 mon.a (mon.0) 1014 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:10:38.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:38 smithi006 bash[33918]: cluster 2023-08-27T22:10:37.136807+0000 mgr.y (mgr.14901) 363 : cluster [DBG] pgmap v364: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-08-27T22:10:38.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:38 smithi006 bash[33918]: audit 2023-08-27T22:10:38.385570+0000 mon.a (mon.0) 1014 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:10:39.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:39 smithi130 bash[27579]: audit 2023-08-27T22:10:38.607845+0000 mon.a (mon.0) 1015 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:10:39.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:39 smithi130 bash[27579]: audit 2023-08-27T22:10:38.935334+0000 mon.b (mon.1) 149 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:10:39.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:39 smithi130 bash[27579]: audit 2023-08-27T22:10:38.936384+0000 mon.b (mon.1) 150 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:10:39.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:39 smithi130 bash[27579]: audit 2023-08-27T22:10:38.943612+0000 mon.a (mon.0) 1016 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:10:39.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:39 smithi006 bash[27292]: audit 2023-08-27T22:10:38.607845+0000 mon.a (mon.0) 1015 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:10:39.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:39 smithi006 bash[27292]: audit 2023-08-27T22:10:38.935334+0000 mon.b (mon.1) 149 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:10:39.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:39 smithi006 bash[27292]: audit 2023-08-27T22:10:38.936384+0000 mon.b (mon.1) 150 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:10:39.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:39 smithi006 bash[27292]: audit 2023-08-27T22:10:38.943612+0000 mon.a (mon.0) 1016 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:10:39.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:39 smithi006 bash[33918]: audit 2023-08-27T22:10:38.607845+0000 mon.a (mon.0) 1015 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:10:39.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:39 smithi006 bash[33918]: audit 2023-08-27T22:10:38.935334+0000 mon.b (mon.1) 149 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:10:39.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:39 smithi006 bash[33918]: audit 2023-08-27T22:10:38.936384+0000 mon.b (mon.1) 150 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:10:39.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:39 smithi006 bash[33918]: audit 2023-08-27T22:10:38.943612+0000 mon.a (mon.0) 1016 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:10:40.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:40 smithi130 bash[27579]: cluster 2023-08-27T22:10:39.137268+0000 mgr.y (mgr.14901) 364 : cluster [DBG] pgmap v365: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 535 op/s 2023-08-27T22:10:40.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:40 smithi130 bash[27579]: audit 2023-08-27T22:10:40.054240+0000 mon.a (mon.0) 1017 : audit [DBG] from='client.? 172.21.15.6:0/1399689882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:40.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:40 smithi006 bash[27292]: cluster 2023-08-27T22:10:39.137268+0000 mgr.y (mgr.14901) 364 : cluster [DBG] pgmap v365: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 535 op/s 2023-08-27T22:10:40.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:40 smithi006 bash[27292]: audit 2023-08-27T22:10:40.054240+0000 mon.a (mon.0) 1017 : audit [DBG] from='client.? 172.21.15.6:0/1399689882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:40.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:10:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:10:40] "GET /metrics HTTP/1.1" 200 195271 "" "Prometheus/2.33.4" 2023-08-27T22:10:40.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:40 smithi006 bash[33918]: cluster 2023-08-27T22:10:39.137268+0000 mgr.y (mgr.14901) 364 : cluster [DBG] pgmap v365: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 535 op/s 2023-08-27T22:10:40.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:40 smithi006 bash[33918]: audit 2023-08-27T22:10:40.054240+0000 mon.a (mon.0) 1017 : audit [DBG] from='client.? 172.21.15.6:0/1399689882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:41.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:10:41.679Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:41.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:10:41.680Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:42.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:42 smithi006 bash[27292]: cluster 2023-08-27T22:10:41.138034+0000 mgr.y (mgr.14901) 365 : cluster [DBG] pgmap v366: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2023-08-27T22:10:42.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:42 smithi006 bash[33918]: cluster 2023-08-27T22:10:41.138034+0000 mgr.y (mgr.14901) 365 : cluster [DBG] pgmap v366: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2023-08-27T22:10:43.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:42 smithi130 bash[27579]: cluster 2023-08-27T22:10:41.138034+0000 mgr.y (mgr.14901) 365 : cluster [DBG] pgmap v366: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2023-08-27T22:10:43.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:43 smithi006 bash[33918]: audit 2023-08-27T22:10:42.745350+0000 mon.a (mon.0) 1018 : audit [DBG] from='client.? 172.21.15.6:0/4178550268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:43.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:10:43.578Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:43.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:10:43.580Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:43.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:43 smithi006 bash[27292]: audit 2023-08-27T22:10:42.745350+0000 mon.a (mon.0) 1018 : audit [DBG] from='client.? 172.21.15.6:0/4178550268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:44.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:43 smithi130 bash[27579]: audit 2023-08-27T22:10:42.745350+0000 mon.a (mon.0) 1018 : audit [DBG] from='client.? 172.21.15.6:0/4178550268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:44.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:44 smithi006 bash[27292]: cluster 2023-08-27T22:10:43.138481+0000 mgr.y (mgr.14901) 366 : cluster [DBG] pgmap v367: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 518 op/s 2023-08-27T22:10:44.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:44 smithi006 bash[33918]: cluster 2023-08-27T22:10:43.138481+0000 mgr.y (mgr.14901) 366 : cluster [DBG] pgmap v367: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 518 op/s 2023-08-27T22:10:45.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:44 smithi130 bash[27579]: cluster 2023-08-27T22:10:43.138481+0000 mgr.y (mgr.14901) 366 : cluster [DBG] pgmap v367: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 518 op/s 2023-08-27T22:10:45.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:45 smithi006 bash[27292]: audit 2023-08-27T22:10:45.439295+0000 mon.a (mon.0) 1019 : audit [DBG] from='client.? 172.21.15.6:0/3070411330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:45.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:45 smithi006 bash[33918]: audit 2023-08-27T22:10:45.439295+0000 mon.a (mon.0) 1019 : audit [DBG] from='client.? 172.21.15.6:0/3070411330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:46.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:45 smithi130 bash[27579]: audit 2023-08-27T22:10:45.439295+0000 mon.a (mon.0) 1019 : audit [DBG] from='client.? 172.21.15.6:0/3070411330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:46.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:46 smithi006 bash[27292]: cluster 2023-08-27T22:10:45.139029+0000 mgr.y (mgr.14901) 367 : cluster [DBG] pgmap v368: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-08-27T22:10:46.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:46 smithi006 bash[33918]: cluster 2023-08-27T22:10:45.139029+0000 mgr.y (mgr.14901) 367 : cluster [DBG] pgmap v368: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-08-27T22:10:47.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:46 smithi130 bash[27579]: cluster 2023-08-27T22:10:45.139029+0000 mgr.y (mgr.14901) 367 : cluster [DBG] pgmap v368: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-08-27T22:10:48.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:48 smithi006 bash[27292]: cluster 2023-08-27T22:10:47.139706+0000 mgr.y (mgr.14901) 368 : cluster [DBG] pgmap v369: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-08-27T22:10:48.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:48 smithi006 bash[27292]: audit 2023-08-27T22:10:48.119273+0000 mon.a (mon.0) 1020 : audit [DBG] from='client.? 172.21.15.6:0/3872400778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:48.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:48 smithi006 bash[33918]: cluster 2023-08-27T22:10:47.139706+0000 mgr.y (mgr.14901) 368 : cluster [DBG] pgmap v369: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-08-27T22:10:48.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:48 smithi006 bash[33918]: audit 2023-08-27T22:10:48.119273+0000 mon.a (mon.0) 1020 : audit [DBG] from='client.? 172.21.15.6:0/3872400778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:49.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:48 smithi130 bash[27579]: cluster 2023-08-27T22:10:47.139706+0000 mgr.y (mgr.14901) 368 : cluster [DBG] pgmap v369: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-08-27T22:10:49.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:48 smithi130 bash[27579]: audit 2023-08-27T22:10:48.119273+0000 mon.a (mon.0) 1020 : audit [DBG] from='client.? 172.21.15.6:0/3872400778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:50.969 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:10:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:10:50] "GET /metrics HTTP/1.1" 200 195263 "" "Prometheus/2.33.4" 2023-08-27T22:10:50.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:50 smithi006 bash[27292]: cluster 2023-08-27T22:10:49.141250+0000 mgr.y (mgr.14901) 369 : cluster [DBG] pgmap v370: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 519 op/s 2023-08-27T22:10:50.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:50 smithi006 bash[33918]: cluster 2023-08-27T22:10:49.141250+0000 mgr.y (mgr.14901) 369 : cluster [DBG] pgmap v370: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 519 op/s 2023-08-27T22:10:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:50 smithi130 bash[27579]: cluster 2023-08-27T22:10:49.141250+0000 mgr.y (mgr.14901) 369 : cluster [DBG] pgmap v370: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 519 op/s 2023-08-27T22:10:51.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:51 smithi006 bash[27292]: audit 2023-08-27T22:10:50.829479+0000 mon.b (mon.1) 151 : audit [DBG] from='client.? 172.21.15.6:0/936115706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:51.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:51 smithi006 bash[33918]: audit 2023-08-27T22:10:50.829479+0000 mon.b (mon.1) 151 : audit [DBG] from='client.? 172.21.15.6:0/936115706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:51.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:10:51.679Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:51.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:10:51.682Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:52.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:51 smithi130 bash[27579]: audit 2023-08-27T22:10:50.829479+0000 mon.b (mon.1) 151 : audit [DBG] from='client.? 172.21.15.6:0/936115706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:52.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:52 smithi006 bash[27292]: cluster 2023-08-27T22:10:51.142001+0000 mgr.y (mgr.14901) 370 : cluster [DBG] pgmap v371: 89 pgs: 89 active+clean; 863 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2023-08-27T22:10:52.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:52 smithi006 bash[33918]: cluster 2023-08-27T22:10:51.142001+0000 mgr.y (mgr.14901) 370 : cluster [DBG] pgmap v371: 89 pgs: 89 active+clean; 863 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2023-08-27T22:10:53.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:52 smithi130 bash[27579]: cluster 2023-08-27T22:10:51.142001+0000 mgr.y (mgr.14901) 370 : cluster [DBG] pgmap v371: 89 pgs: 89 active+clean; 863 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2023-08-27T22:10:53.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:10:53.580Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:53.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:10:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:10:53.582Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:10:54.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:54 smithi130 bash[27579]: cluster 2023-08-27T22:10:53.142447+0000 mgr.y (mgr.14901) 371 : cluster [DBG] pgmap v372: 89 pgs: 89 active+clean; 863 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 538 op/s 2023-08-27T22:10:54.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:54 smithi130 bash[27579]: audit 2023-08-27T22:10:53.500186+0000 mon.c (mon.2) 163 : audit [DBG] from='client.? 172.21.15.6:0/2511983184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:54.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:54 smithi006 bash[27292]: cluster 2023-08-27T22:10:53.142447+0000 mgr.y (mgr.14901) 371 : cluster [DBG] pgmap v372: 89 pgs: 89 active+clean; 863 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 538 op/s 2023-08-27T22:10:54.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:54 smithi006 bash[27292]: audit 2023-08-27T22:10:53.500186+0000 mon.c (mon.2) 163 : audit [DBG] from='client.? 172.21.15.6:0/2511983184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:54.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:54 smithi006 bash[33918]: cluster 2023-08-27T22:10:53.142447+0000 mgr.y (mgr.14901) 371 : cluster [DBG] pgmap v372: 89 pgs: 89 active+clean; 863 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 538 op/s 2023-08-27T22:10:54.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:54 smithi006 bash[33918]: audit 2023-08-27T22:10:53.500186+0000 mon.c (mon.2) 163 : audit [DBG] from='client.? 172.21.15.6:0/2511983184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:56.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:56 smithi130 bash[27579]: cluster 2023-08-27T22:10:55.143139+0000 mgr.y (mgr.14901) 372 : cluster [DBG] pgmap v373: 89 pgs: 89 active+clean; 774 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-08-27T22:10:56.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:56 smithi130 bash[27579]: audit 2023-08-27T22:10:56.191967+0000 mon.c (mon.2) 164 : audit [DBG] from='client.? 172.21.15.6:0/4081364292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:56.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:56 smithi006 bash[27292]: cluster 2023-08-27T22:10:55.143139+0000 mgr.y (mgr.14901) 372 : cluster [DBG] pgmap v373: 89 pgs: 89 active+clean; 774 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-08-27T22:10:56.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:56 smithi006 bash[27292]: audit 2023-08-27T22:10:56.191967+0000 mon.c (mon.2) 164 : audit [DBG] from='client.? 172.21.15.6:0/4081364292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:56.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:56 smithi006 bash[33918]: cluster 2023-08-27T22:10:55.143139+0000 mgr.y (mgr.14901) 372 : cluster [DBG] pgmap v373: 89 pgs: 89 active+clean; 774 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-08-27T22:10:56.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:56 smithi006 bash[33918]: audit 2023-08-27T22:10:56.191967+0000 mon.c (mon.2) 164 : audit [DBG] from='client.? 172.21.15.6:0/4081364292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:58.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:58 smithi130 bash[27579]: cluster 2023-08-27T22:10:57.143843+0000 mgr.y (mgr.14901) 373 : cluster [DBG] pgmap v374: 89 pgs: 89 active+clean; 661 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-08-27T22:10:58.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:58 smithi006 bash[27292]: cluster 2023-08-27T22:10:57.143843+0000 mgr.y (mgr.14901) 373 : cluster [DBG] pgmap v374: 89 pgs: 89 active+clean; 661 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-08-27T22:10:58.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:58 smithi006 bash[33918]: cluster 2023-08-27T22:10:57.143843+0000 mgr.y (mgr.14901) 373 : cluster [DBG] pgmap v374: 89 pgs: 89 active+clean; 661 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-08-27T22:10:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:10:59 smithi130 bash[27579]: audit 2023-08-27T22:10:58.932447+0000 mon.c (mon.2) 165 : audit [DBG] from='client.? 172.21.15.6:0/2892457450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:59.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:10:59 smithi006 bash[27292]: audit 2023-08-27T22:10:58.932447+0000 mon.c (mon.2) 165 : audit [DBG] from='client.? 172.21.15.6:0/2892457450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:59.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:10:59 smithi006 bash[33918]: audit 2023-08-27T22:10:58.932447+0000 mon.c (mon.2) 165 : audit [DBG] from='client.? 172.21.15.6:0/2892457450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:00.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:00 smithi130 bash[27579]: cluster 2023-08-27T22:10:59.148176+0000 mgr.y (mgr.14901) 374 : cluster [DBG] pgmap v375: 89 pgs: 89 active+clean; 661 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 544 op/s 2023-08-27T22:11:00.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:00 smithi006 bash[27292]: cluster 2023-08-27T22:10:59.148176+0000 mgr.y (mgr.14901) 374 : cluster [DBG] pgmap v375: 89 pgs: 89 active+clean; 661 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 544 op/s 2023-08-27T22:11:00.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:00 smithi006 bash[33918]: cluster 2023-08-27T22:10:59.148176+0000 mgr.y (mgr.14901) 374 : cluster [DBG] pgmap v375: 89 pgs: 89 active+clean; 661 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 544 op/s 2023-08-27T22:11:01.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:11:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:11:00] "GET /metrics HTTP/1.1" 200 195265 "" "Prometheus/2.33.4" 2023-08-27T22:11:01.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:11:01.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:01.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:11:01.682Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:02.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:02 smithi130 bash[27579]: cluster 2023-08-27T22:11:01.149137+0000 mgr.y (mgr.14901) 375 : cluster [DBG] pgmap v376: 89 pgs: 89 active+clean; 465 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-08-27T22:11:02.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:02 smithi130 bash[27579]: audit 2023-08-27T22:11:01.626449+0000 mon.a (mon.0) 1021 : audit [DBG] from='client.? 172.21.15.6:0/147295011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:02.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:02 smithi006 bash[27292]: cluster 2023-08-27T22:11:01.149137+0000 mgr.y (mgr.14901) 375 : cluster [DBG] pgmap v376: 89 pgs: 89 active+clean; 465 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-08-27T22:11:02.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:02 smithi006 bash[27292]: audit 2023-08-27T22:11:01.626449+0000 mon.a (mon.0) 1021 : audit [DBG] from='client.? 172.21.15.6:0/147295011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:02.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:02 smithi006 bash[33918]: cluster 2023-08-27T22:11:01.149137+0000 mgr.y (mgr.14901) 375 : cluster [DBG] pgmap v376: 89 pgs: 89 active+clean; 465 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-08-27T22:11:02.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:02 smithi006 bash[33918]: audit 2023-08-27T22:11:01.626449+0000 mon.a (mon.0) 1021 : audit [DBG] from='client.? 172.21.15.6:0/147295011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:03.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:11:03.581Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:03.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:11:03.584Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:04.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:04 smithi130 bash[27579]: cluster 2023-08-27T22:11:03.149586+0000 mgr.y (mgr.14901) 376 : cluster [DBG] pgmap v377: 89 pgs: 89 active+clean; 465 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 531 op/s 2023-08-27T22:11:04.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:04 smithi130 bash[27579]: audit 2023-08-27T22:11:04.287983+0000 mon.c (mon.2) 166 : audit [DBG] from='client.? 172.21.15.6:0/257216675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:04.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:04 smithi006 bash[27292]: cluster 2023-08-27T22:11:03.149586+0000 mgr.y (mgr.14901) 376 : cluster [DBG] pgmap v377: 89 pgs: 89 active+clean; 465 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 531 op/s 2023-08-27T22:11:04.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:04 smithi006 bash[27292]: audit 2023-08-27T22:11:04.287983+0000 mon.c (mon.2) 166 : audit [DBG] from='client.? 172.21.15.6:0/257216675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:04.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:04 smithi006 bash[33918]: cluster 2023-08-27T22:11:03.149586+0000 mgr.y (mgr.14901) 376 : cluster [DBG] pgmap v377: 89 pgs: 89 active+clean; 465 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 531 op/s 2023-08-27T22:11:04.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:04 smithi006 bash[33918]: audit 2023-08-27T22:11:04.287983+0000 mon.c (mon.2) 166 : audit [DBG] from='client.? 172.21.15.6:0/257216675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:06.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:06 smithi130 bash[27579]: cluster 2023-08-27T22:11:05.153213+0000 mgr.y (mgr.14901) 377 : cluster [DBG] pgmap v378: 89 pgs: 89 active+clean; 378 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-08-27T22:11:06.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:06 smithi006 bash[27292]: cluster 2023-08-27T22:11:05.153213+0000 mgr.y (mgr.14901) 377 : cluster [DBG] pgmap v378: 89 pgs: 89 active+clean; 378 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-08-27T22:11:06.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:06 smithi006 bash[33918]: cluster 2023-08-27T22:11:05.153213+0000 mgr.y (mgr.14901) 377 : cluster [DBG] pgmap v378: 89 pgs: 89 active+clean; 378 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-08-27T22:11:07.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:07 smithi130 bash[27579]: audit 2023-08-27T22:11:07.022708+0000 mon.a (mon.0) 1022 : audit [DBG] from='client.? 172.21.15.6:0/1341289482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:07.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:07 smithi006 bash[27292]: audit 2023-08-27T22:11:07.022708+0000 mon.a (mon.0) 1022 : audit [DBG] from='client.? 172.21.15.6:0/1341289482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:07.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:07 smithi006 bash[33918]: audit 2023-08-27T22:11:07.022708+0000 mon.a (mon.0) 1022 : audit [DBG] from='client.? 172.21.15.6:0/1341289482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:08.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:08 smithi130 bash[27579]: cluster 2023-08-27T22:11:07.153933+0000 mgr.y (mgr.14901) 378 : cluster [DBG] pgmap v379: 89 pgs: 89 active+clean; 267 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-08-27T22:11:08.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:08 smithi006 bash[27292]: cluster 2023-08-27T22:11:07.153933+0000 mgr.y (mgr.14901) 378 : cluster [DBG] pgmap v379: 89 pgs: 89 active+clean; 267 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-08-27T22:11:08.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:08 smithi006 bash[33918]: cluster 2023-08-27T22:11:07.153933+0000 mgr.y (mgr.14901) 378 : cluster [DBG] pgmap v379: 89 pgs: 89 active+clean; 267 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-08-27T22:11:10.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:10 smithi130 bash[27579]: cluster 2023-08-27T22:11:09.154405+0000 mgr.y (mgr.14901) 379 : cluster [DBG] pgmap v380: 89 pgs: 89 active+clean; 267 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 524 op/s 2023-08-27T22:11:10.626 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:10 smithi130 bash[27579]: audit 2023-08-27T22:11:09.757228+0000 mon.c (mon.2) 167 : audit [DBG] from='client.? 172.21.15.6:0/1329024596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:10.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:10 smithi006 bash[27292]: cluster 2023-08-27T22:11:09.154405+0000 mgr.y (mgr.14901) 379 : cluster [DBG] pgmap v380: 89 pgs: 89 active+clean; 267 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 524 op/s 2023-08-27T22:11:10.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:10 smithi006 bash[27292]: audit 2023-08-27T22:11:09.757228+0000 mon.c (mon.2) 167 : audit [DBG] from='client.? 172.21.15.6:0/1329024596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:10.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:10 smithi006 bash[33918]: cluster 2023-08-27T22:11:09.154405+0000 mgr.y (mgr.14901) 379 : cluster [DBG] pgmap v380: 89 pgs: 89 active+clean; 267 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 524 op/s 2023-08-27T22:11:10.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:10 smithi006 bash[33918]: audit 2023-08-27T22:11:09.757228+0000 mon.c (mon.2) 167 : audit [DBG] from='client.? 172.21.15.6:0/1329024596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:11.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:11:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:11:10] "GET /metrics HTTP/1.1" 200 195265 "" "Prometheus/2.33.4" 2023-08-27T22:11:11.732 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Removed 106141 objects 2023-08-27T22:11:11.733 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Clean up completed and total clean up time :160.111 2023-08-27T22:11:11.741 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-08-27T22:11:11.743 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-08-27T22:11:11.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:11:11.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:11.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:11:11.684Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:12.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:12 smithi130 bash[27579]: cluster 2023-08-27T22:11:11.155145+0000 mgr.y (mgr.14901) 380 : cluster [DBG] pgmap v381: 89 pgs: 89 active+clean; 70 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2023-08-27T22:11:12.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:12 smithi130 bash[27579]: audit 2023-08-27T22:11:12.180620+0000 mon.a (mon.0) 1023 : audit [DBG] from='client.? 172.21.15.6:0/1602919898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:12.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:12 smithi006 bash[27292]: cluster 2023-08-27T22:11:11.155145+0000 mgr.y (mgr.14901) 380 : cluster [DBG] pgmap v381: 89 pgs: 89 active+clean; 70 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2023-08-27T22:11:12.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:12 smithi006 bash[27292]: audit 2023-08-27T22:11:12.180620+0000 mon.a (mon.0) 1023 : audit [DBG] from='client.? 172.21.15.6:0/1602919898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:12.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:12 smithi006 bash[33918]: cluster 2023-08-27T22:11:11.155145+0000 mgr.y (mgr.14901) 380 : cluster [DBG] pgmap v381: 89 pgs: 89 active+clean; 70 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2023-08-27T22:11:12.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:12 smithi006 bash[33918]: audit 2023-08-27T22:11:12.180620+0000 mon.a (mon.0) 1023 : audit [DBG] from='client.? 172.21.15.6:0/1602919898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:13.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:11:13.583Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:13.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:11:13.586Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:14.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:14 smithi006 bash[33918]: cluster 2023-08-27T22:11:13.155808+0000 mgr.y (mgr.14901) 381 : cluster [DBG] pgmap v382: 89 pgs: 89 active+clean; 70 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 526 op/s 2023-08-27T22:11:14.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:14 smithi006 bash[27292]: cluster 2023-08-27T22:11:13.155808+0000 mgr.y (mgr.14901) 381 : cluster [DBG] pgmap v382: 89 pgs: 89 active+clean; 70 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 526 op/s 2023-08-27T22:11:14.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:14 smithi130 bash[27579]: cluster 2023-08-27T22:11:13.155808+0000 mgr.y (mgr.14901) 381 : cluster [DBG] pgmap v382: 89 pgs: 89 active+clean; 70 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 526 op/s 2023-08-27T22:11:15.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:15 smithi006 bash[33918]: audit 2023-08-27T22:11:14.601442+0000 mon.c (mon.2) 168 : audit [DBG] from='client.? 172.21.15.6:0/1143285714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:15.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:15 smithi006 bash[27292]: audit 2023-08-27T22:11:14.601442+0000 mon.c (mon.2) 168 : audit [DBG] from='client.? 172.21.15.6:0/1143285714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:15.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:15 smithi130 bash[27579]: audit 2023-08-27T22:11:14.601442+0000 mon.c (mon.2) 168 : audit [DBG] from='client.? 172.21.15.6:0/1143285714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:16.576 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T22:11:16.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:16 smithi006 bash[33918]: cluster 2023-08-27T22:11:15.156909+0000 mgr.y (mgr.14901) 382 : cluster [DBG] pgmap v383: 89 pgs: 89 active+clean; 31 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-08-27T22:11:16.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:16 smithi006 bash[27292]: cluster 2023-08-27T22:11:15.156909+0000 mgr.y (mgr.14901) 382 : cluster [DBG] pgmap v383: 89 pgs: 89 active+clean; 31 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-08-27T22:11:16.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:16 smithi130 bash[27579]: cluster 2023-08-27T22:11:15.156909+0000 mgr.y (mgr.14901) 382 : cluster [DBG] pgmap v383: 89 pgs: 89 active+clean; 31 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-08-27T22:11:17.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:17 smithi006 bash[27292]: audit 2023-08-27T22:11:17.015375+0000 mon.a (mon.0) 1024 : audit [DBG] from='client.? 172.21.15.6:0/2515490289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:17.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:17 smithi006 bash[33918]: audit 2023-08-27T22:11:17.015375+0000 mon.a (mon.0) 1024 : audit [DBG] from='client.? 172.21.15.6:0/2515490289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:17.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:17 smithi130 bash[27579]: audit 2023-08-27T22:11:17.015375+0000 mon.a (mon.0) 1024 : audit [DBG] from='client.? 172.21.15.6:0/2515490289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:18.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:18 smithi006 bash[33918]: cluster 2023-08-27T22:11:17.157859+0000 mgr.y (mgr.14901) 383 : cluster [DBG] pgmap v384: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 504 op/s 2023-08-27T22:11:18.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:18 smithi006 bash[27292]: cluster 2023-08-27T22:11:17.157859+0000 mgr.y (mgr.14901) 383 : cluster [DBG] pgmap v384: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 504 op/s 2023-08-27T22:11:18.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:18 smithi130 bash[27579]: cluster 2023-08-27T22:11:17.157859+0000 mgr.y (mgr.14901) 383 : cluster [DBG] pgmap v384: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 504 op/s 2023-08-27T22:11:20.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:20 smithi006 bash[33918]: cluster 2023-08-27T22:11:19.158305+0000 mgr.y (mgr.14901) 384 : cluster [DBG] pgmap v385: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 356 op/s 2023-08-27T22:11:20.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:20 smithi006 bash[33918]: audit 2023-08-27T22:11:19.431765+0000 mon.c (mon.2) 169 : audit [DBG] from='client.? 172.21.15.6:0/3233077113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:20.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:20 smithi006 bash[27292]: cluster 2023-08-27T22:11:19.158305+0000 mgr.y (mgr.14901) 384 : cluster [DBG] pgmap v385: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 356 op/s 2023-08-27T22:11:20.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:20 smithi006 bash[27292]: audit 2023-08-27T22:11:19.431765+0000 mon.c (mon.2) 169 : audit [DBG] from='client.? 172.21.15.6:0/3233077113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:20.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:20 smithi130 bash[27579]: cluster 2023-08-27T22:11:19.158305+0000 mgr.y (mgr.14901) 384 : cluster [DBG] pgmap v385: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 356 op/s 2023-08-27T22:11:20.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:20 smithi130 bash[27579]: audit 2023-08-27T22:11:19.431765+0000 mon.c (mon.2) 169 : audit [DBG] from='client.? 172.21.15.6:0/3233077113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:21.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:11:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:11:20] "GET /metrics HTTP/1.1" 200 195274 "" "Prometheus/2.33.4" 2023-08-27T22:11:21.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:11:21.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:21.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:11:21.684Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:22.446 INFO:teuthology.orchestra.run.smithi006.stderr:pool 'unique_pool_2' removed 2023-08-27T22:11:22.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:22 smithi006 bash[33918]: cluster 2023-08-27T22:11:21.159515+0000 mgr.y (mgr.14901) 385 : cluster [DBG] pgmap v386: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 356 op/s 2023-08-27T22:11:22.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:22 smithi006 bash[33918]: audit 2023-08-27T22:11:21.726435+0000 mon.a (mon.0) 1025 : audit [INF] from='client.? 172.21.15.6:0/3998245681' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-27T22:11:22.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:22 smithi006 bash[33918]: audit 2023-08-27T22:11:21.837717+0000 mon.c (mon.2) 170 : audit [DBG] from='client.? 172.21.15.6:0/168078790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:22.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:22 smithi006 bash[27292]: cluster 2023-08-27T22:11:21.159515+0000 mgr.y (mgr.14901) 385 : cluster [DBG] pgmap v386: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 356 op/s 2023-08-27T22:11:22.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:22 smithi006 bash[27292]: audit 2023-08-27T22:11:21.726435+0000 mon.a (mon.0) 1025 : audit [INF] from='client.? 172.21.15.6:0/3998245681' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-27T22:11:22.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:22 smithi006 bash[27292]: audit 2023-08-27T22:11:21.837717+0000 mon.c (mon.2) 170 : audit [DBG] from='client.? 172.21.15.6:0/168078790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:22.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:22 smithi130 bash[27579]: cluster 2023-08-27T22:11:21.159515+0000 mgr.y (mgr.14901) 385 : cluster [DBG] pgmap v386: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 356 op/s 2023-08-27T22:11:22.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:22 smithi130 bash[27579]: audit 2023-08-27T22:11:21.726435+0000 mon.a (mon.0) 1025 : audit [INF] from='client.? 172.21.15.6:0/3998245681' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-27T22:11:22.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:22 smithi130 bash[27579]: audit 2023-08-27T22:11:21.837717+0000 mon.c (mon.2) 170 : audit [DBG] from='client.? 172.21.15.6:0/168078790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:23.374 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-08-27T22:11:23.374 INFO:tasks.radosbench:Beginning radosbench... 2023-08-27T22:11:23.375 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-08-27T22:11:23.375 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd pool create unique_pool_3 16 2023-08-27T22:11:23.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:23 smithi006 bash[27292]: audit 2023-08-27T22:11:22.421204+0000 mon.a (mon.0) 1026 : audit [INF] from='client.? 172.21.15.6:0/3998245681' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-08-27T22:11:23.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:23 smithi006 bash[27292]: cluster 2023-08-27T22:11:22.421311+0000 mon.a (mon.0) 1027 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-08-27T22:11:23.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:23 smithi006 bash[33918]: audit 2023-08-27T22:11:22.421204+0000 mon.a (mon.0) 1026 : audit [INF] from='client.? 172.21.15.6:0/3998245681' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-08-27T22:11:23.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:23 smithi006 bash[33918]: cluster 2023-08-27T22:11:22.421311+0000 mon.a (mon.0) 1027 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-08-27T22:11:23.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:11:23.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:23.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:11:23.586Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:23.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:23 smithi130 bash[27579]: audit 2023-08-27T22:11:22.421204+0000 mon.a (mon.0) 1026 : audit [INF] from='client.? 172.21.15.6:0/3998245681' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-08-27T22:11:23.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:23 smithi130 bash[27579]: cluster 2023-08-27T22:11:22.421311+0000 mon.a (mon.0) 1027 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-08-27T22:11:24.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:24 smithi006 bash[27292]: cluster 2023-08-27T22:11:23.160013+0000 mgr.y (mgr.14901) 386 : cluster [DBG] pgmap v388: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:24.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:24 smithi006 bash[27292]: audit 2023-08-27T22:11:24.246545+0000 mon.c (mon.2) 171 : audit [DBG] from='client.? 172.21.15.6:0/158833747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:24.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:24 smithi006 bash[33918]: cluster 2023-08-27T22:11:23.160013+0000 mgr.y (mgr.14901) 386 : cluster [DBG] pgmap v388: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:24.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:24 smithi006 bash[33918]: audit 2023-08-27T22:11:24.246545+0000 mon.c (mon.2) 171 : audit [DBG] from='client.? 172.21.15.6:0/158833747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:24.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:24 smithi130 bash[27579]: cluster 2023-08-27T22:11:23.160013+0000 mgr.y (mgr.14901) 386 : cluster [DBG] pgmap v388: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:24.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:24 smithi130 bash[27579]: audit 2023-08-27T22:11:24.246545+0000 mon.c (mon.2) 171 : audit [DBG] from='client.? 172.21.15.6:0/158833747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:26.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:26 smithi006 bash[33918]: cluster 2023-08-27T22:11:25.160924+0000 mgr.y (mgr.14901) 387 : cluster [DBG] pgmap v389: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:26.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:26 smithi006 bash[27292]: cluster 2023-08-27T22:11:25.160924+0000 mgr.y (mgr.14901) 387 : cluster [DBG] pgmap v389: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:26.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:26 smithi130 bash[27579]: cluster 2023-08-27T22:11:25.160924+0000 mgr.y (mgr.14901) 387 : cluster [DBG] pgmap v389: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:27.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:27 smithi006 bash[27292]: audit 2023-08-27T22:11:26.672183+0000 mon.c (mon.2) 172 : audit [DBG] from='client.? 172.21.15.6:0/1579360109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:27.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:27 smithi006 bash[33918]: audit 2023-08-27T22:11:26.672183+0000 mon.c (mon.2) 172 : audit [DBG] from='client.? 172.21.15.6:0/1579360109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:27.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:27 smithi130 bash[27579]: audit 2023-08-27T22:11:26.672183+0000 mon.c (mon.2) 172 : audit [DBG] from='client.? 172.21.15.6:0/1579360109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:28.214 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T22:11:28.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:28 smithi006 bash[33918]: cluster 2023-08-27T22:11:27.161772+0000 mgr.y (mgr.14901) 388 : cluster [DBG] pgmap v390: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:28.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:28 smithi006 bash[27292]: cluster 2023-08-27T22:11:27.161772+0000 mgr.y (mgr.14901) 388 : cluster [DBG] pgmap v390: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:28.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:28 smithi130 bash[27579]: cluster 2023-08-27T22:11:27.161772+0000 mgr.y (mgr.14901) 388 : cluster [DBG] pgmap v390: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:29.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:29 smithi006 bash[27292]: audit 2023-08-27T22:11:29.079880+0000 mon.a (mon.0) 1028 : audit [DBG] from='client.? 172.21.15.6:0/3497909541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:29.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:29 smithi006 bash[33918]: audit 2023-08-27T22:11:29.079880+0000 mon.a (mon.0) 1028 : audit [DBG] from='client.? 172.21.15.6:0/3497909541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:29.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:29 smithi130 bash[27579]: audit 2023-08-27T22:11:29.079880+0000 mon.a (mon.0) 1028 : audit [DBG] from='client.? 172.21.15.6:0/3497909541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:30.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:30 smithi006 bash[27292]: cluster 2023-08-27T22:11:29.162497+0000 mgr.y (mgr.14901) 389 : cluster [DBG] pgmap v391: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:30.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:30 smithi006 bash[33918]: cluster 2023-08-27T22:11:29.162497+0000 mgr.y (mgr.14901) 389 : cluster [DBG] pgmap v391: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:30.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:30 smithi130 bash[27579]: cluster 2023-08-27T22:11:29.162497+0000 mgr.y (mgr.14901) 389 : cluster [DBG] pgmap v391: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:31.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:11:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:11:30] "GET /metrics HTTP/1.1" 200 192928 "" "Prometheus/2.33.4" 2023-08-27T22:11:31.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:11:31.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:31.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:11:31.685Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:32.487 INFO:teuthology.orchestra.run.smithi006.stderr:pool 'unique_pool_3' created 2023-08-27T22:11:32.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:32 smithi006 bash[27292]: cluster 2023-08-27T22:11:31.163522+0000 mgr.y (mgr.14901) 390 : cluster [DBG] pgmap v392: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:32.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:32 smithi006 bash[27292]: audit 2023-08-27T22:11:31.502519+0000 mon.a (mon.0) 1029 : audit [DBG] from='client.? 172.21.15.6:0/415102703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:32.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:32 smithi006 bash[27292]: audit 2023-08-27T22:11:31.795005+0000 mon.a (mon.0) 1030 : audit [INF] from='client.? 172.21.15.6:0/1799713212' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-08-27T22:11:32.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:32 smithi006 bash[33918]: cluster 2023-08-27T22:11:31.163522+0000 mgr.y (mgr.14901) 390 : cluster [DBG] pgmap v392: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:32.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:32 smithi006 bash[33918]: audit 2023-08-27T22:11:31.502519+0000 mon.a (mon.0) 1029 : audit [DBG] from='client.? 172.21.15.6:0/415102703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:32.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:32 smithi006 bash[33918]: audit 2023-08-27T22:11:31.795005+0000 mon.a (mon.0) 1030 : audit [INF] from='client.? 172.21.15.6:0/1799713212' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-08-27T22:11:32.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:32 smithi130 bash[27579]: cluster 2023-08-27T22:11:31.163522+0000 mgr.y (mgr.14901) 390 : cluster [DBG] pgmap v392: 73 pgs: 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:32.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:32 smithi130 bash[27579]: audit 2023-08-27T22:11:31.502519+0000 mon.a (mon.0) 1029 : audit [DBG] from='client.? 172.21.15.6:0/415102703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:32.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:32 smithi130 bash[27579]: audit 2023-08-27T22:11:31.795005+0000 mon.a (mon.0) 1030 : audit [INF] from='client.? 172.21.15.6:0/1799713212' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-08-27T22:11:33.361 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-08-27T22:11:33.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:33 smithi006 bash[33918]: audit 2023-08-27T22:11:32.473177+0000 mon.a (mon.0) 1031 : audit [INF] from='client.? 172.21.15.6:0/1799713212' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-08-27T22:11:33.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:33 smithi006 bash[33918]: cluster 2023-08-27T22:11:32.473270+0000 mon.a (mon.0) 1032 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-08-27T22:11:33.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:33 smithi006 bash[27292]: audit 2023-08-27T22:11:32.473177+0000 mon.a (mon.0) 1031 : audit [INF] from='client.? 172.21.15.6:0/1799713212' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-08-27T22:11:33.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:33 smithi006 bash[27292]: cluster 2023-08-27T22:11:32.473270+0000 mon.a (mon.0) 1032 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-08-27T22:11:33.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:11:33.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:33.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:11:33.586Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:33.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:33 smithi130 bash[27579]: audit 2023-08-27T22:11:32.473177+0000 mon.a (mon.0) 1031 : audit [INF] from='client.? 172.21.15.6:0/1799713212' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-08-27T22:11:33.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:33 smithi130 bash[27579]: cluster 2023-08-27T22:11:32.473270+0000 mon.a (mon.0) 1032 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-08-27T22:11:34.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:34 smithi130 bash[27579]: cluster 2023-08-27T22:11:33.164219+0000 mgr.y (mgr.14901) 391 : cluster [DBG] pgmap v394: 89 pgs: 16 unknown, 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:34.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:34 smithi130 bash[27579]: cluster 2023-08-27T22:11:33.476167+0000 mon.a (mon.0) 1033 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-08-27T22:11:34.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:34 smithi130 bash[27579]: audit 2023-08-27T22:11:33.916646+0000 mon.c (mon.2) 173 : audit [DBG] from='client.? 172.21.15.6:0/3287369285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:34.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:34 smithi006 bash[33918]: cluster 2023-08-27T22:11:33.164219+0000 mgr.y (mgr.14901) 391 : cluster [DBG] pgmap v394: 89 pgs: 16 unknown, 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:34.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:34 smithi006 bash[33918]: cluster 2023-08-27T22:11:33.476167+0000 mon.a (mon.0) 1033 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-08-27T22:11:34.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:34 smithi006 bash[33918]: audit 2023-08-27T22:11:33.916646+0000 mon.c (mon.2) 173 : audit [DBG] from='client.? 172.21.15.6:0/3287369285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:34.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:34 smithi006 bash[27292]: cluster 2023-08-27T22:11:33.164219+0000 mgr.y (mgr.14901) 391 : cluster [DBG] pgmap v394: 89 pgs: 16 unknown, 73 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:34.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:34 smithi006 bash[27292]: cluster 2023-08-27T22:11:33.476167+0000 mon.a (mon.0) 1033 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-08-27T22:11:34.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:34 smithi006 bash[27292]: audit 2023-08-27T22:11:33.916646+0000 mon.c (mon.2) 173 : audit [DBG] from='client.? 172.21.15.6:0/3287369285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:35.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:35 smithi130 bash[27579]: cluster 2023-08-27T22:11:34.488105+0000 mon.a (mon.0) 1034 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-08-27T22:11:35.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:35 smithi006 bash[27292]: cluster 2023-08-27T22:11:34.488105+0000 mon.a (mon.0) 1034 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-08-27T22:11:35.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:35 smithi006 bash[33918]: cluster 2023-08-27T22:11:34.488105+0000 mon.a (mon.0) 1034 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-08-27T22:11:36.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:36 smithi130 bash[27579]: cluster 2023-08-27T22:11:35.165284+0000 mgr.y (mgr.14901) 392 : cluster [DBG] pgmap v397: 89 pgs: 6 unknown, 83 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:36.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:36 smithi130 bash[27579]: audit 2023-08-27T22:11:36.341254+0000 mon.a (mon.0) 1035 : audit [DBG] from='client.? 172.21.15.6:0/2860580935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:36.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:36 smithi006 bash[27292]: cluster 2023-08-27T22:11:35.165284+0000 mgr.y (mgr.14901) 392 : cluster [DBG] pgmap v397: 89 pgs: 6 unknown, 83 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:36.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:36 smithi006 bash[27292]: audit 2023-08-27T22:11:36.341254+0000 mon.a (mon.0) 1035 : audit [DBG] from='client.? 172.21.15.6:0/2860580935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:36.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:36 smithi006 bash[33918]: cluster 2023-08-27T22:11:35.165284+0000 mgr.y (mgr.14901) 392 : cluster [DBG] pgmap v397: 89 pgs: 6 unknown, 83 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:36.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:36 smithi006 bash[33918]: audit 2023-08-27T22:11:36.341254+0000 mon.a (mon.0) 1035 : audit [DBG] from='client.? 172.21.15.6:0/2860580935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:38.207 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T22:11:38.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:38 smithi130 bash[27579]: cluster 2023-08-27T22:11:37.166275+0000 mgr.y (mgr.14901) 393 : cluster [DBG] pgmap v398: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:38.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:38 smithi006 bash[27292]: cluster 2023-08-27T22:11:37.166275+0000 mgr.y (mgr.14901) 393 : cluster [DBG] pgmap v398: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:38.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:38 smithi006 bash[33918]: cluster 2023-08-27T22:11:37.166275+0000 mgr.y (mgr.14901) 393 : cluster [DBG] pgmap v398: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:39.706 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:39 smithi006 bash[27292]: audit 2023-08-27T22:11:38.746332+0000 mon.c (mon.2) 174 : audit [DBG] from='client.? 172.21.15.6:0/2318842754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:39.706 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:39 smithi006 bash[27292]: audit 2023-08-27T22:11:38.955803+0000 mon.b (mon.1) 152 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:11:39.706 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:39 smithi006 bash[33918]: audit 2023-08-27T22:11:38.746332+0000 mon.c (mon.2) 174 : audit [DBG] from='client.? 172.21.15.6:0/2318842754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:39.706 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:39 smithi006 bash[33918]: audit 2023-08-27T22:11:38.955803+0000 mon.b (mon.1) 152 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:11:39.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:39 smithi130 bash[27579]: audit 2023-08-27T22:11:38.746332+0000 mon.c (mon.2) 174 : audit [DBG] from='client.? 172.21.15.6:0/2318842754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:39.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:39 smithi130 bash[27579]: audit 2023-08-27T22:11:38.955803+0000 mon.b (mon.1) 152 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:11:40.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:40 smithi006 bash[33918]: cluster 2023-08-27T22:11:39.166968+0000 mgr.y (mgr.14901) 394 : cluster [DBG] pgmap v399: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:40.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:40 smithi006 bash[27292]: cluster 2023-08-27T22:11:39.166968+0000 mgr.y (mgr.14901) 394 : cluster [DBG] pgmap v399: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:40.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:40 smithi130 bash[27579]: cluster 2023-08-27T22:11:39.166968+0000 mgr.y (mgr.14901) 394 : cluster [DBG] pgmap v399: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:41.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:11:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:11:40] "GET /metrics HTTP/1.1" 200 192928 "" "Prometheus/2.33.4" 2023-08-27T22:11:41.688 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:41 smithi006 bash[33918]: audit 2023-08-27T22:11:41.149177+0000 mon.c (mon.2) 175 : audit [DBG] from='client.? 172.21.15.6:0/64818221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:41.688 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:41 smithi006 bash[27292]: audit 2023-08-27T22:11:41.149177+0000 mon.c (mon.2) 175 : audit [DBG] from='client.? 172.21.15.6:0/64818221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:41.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:41 smithi130 bash[27579]: audit 2023-08-27T22:11:41.149177+0000 mon.c (mon.2) 175 : audit [DBG] from='client.? 172.21.15.6:0/64818221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:41.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:11:41.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:41.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:11:41.685Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:42.556 INFO:teuthology.orchestra.run.smithi006.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-08-27T22:11:42.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:42 smithi130 bash[27579]: cluster 2023-08-27T22:11:41.167690+0000 mgr.y (mgr.14901) 395 : cluster [DBG] pgmap v400: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:42.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:42 smithi130 bash[27579]: audit 2023-08-27T22:11:41.829368+0000 mon.c (mon.2) 176 : audit [INF] from='client.? 172.21.15.6:0/803398613' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T22:11:42.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:42 smithi130 bash[27579]: audit 2023-08-27T22:11:41.830009+0000 mon.a (mon.0) 1036 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T22:11:42.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:42 smithi006 bash[27292]: cluster 2023-08-27T22:11:41.167690+0000 mgr.y (mgr.14901) 395 : cluster [DBG] pgmap v400: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:42.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:42 smithi006 bash[27292]: audit 2023-08-27T22:11:41.829368+0000 mon.c (mon.2) 176 : audit [INF] from='client.? 172.21.15.6:0/803398613' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T22:11:42.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:42 smithi006 bash[27292]: audit 2023-08-27T22:11:41.830009+0000 mon.a (mon.0) 1036 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T22:11:42.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:42 smithi006 bash[33918]: cluster 2023-08-27T22:11:41.167690+0000 mgr.y (mgr.14901) 395 : cluster [DBG] pgmap v400: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:42.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:42 smithi006 bash[33918]: audit 2023-08-27T22:11:41.829368+0000 mon.c (mon.2) 176 : audit [INF] from='client.? 172.21.15.6:0/803398613' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T22:11:42.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:42 smithi006 bash[33918]: audit 2023-08-27T22:11:41.830009+0000 mon.a (mon.0) 1036 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T22:11:43.593 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:11:43.586Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:43.593 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:43 smithi006 bash[33918]: audit 2023-08-27T22:11:42.545346+0000 mon.a (mon.0) 1037 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-27T22:11:43.593 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:43 smithi006 bash[33918]: cluster 2023-08-27T22:11:42.545449+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2023-08-27T22:11:43.593 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:43 smithi006 bash[27292]: audit 2023-08-27T22:11:42.545346+0000 mon.a (mon.0) 1037 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-27T22:11:43.593 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:43 smithi006 bash[27292]: cluster 2023-08-27T22:11:42.545449+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2023-08-27T22:11:43.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:43 smithi130 bash[27579]: audit 2023-08-27T22:11:42.545346+0000 mon.a (mon.0) 1037 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-27T22:11:43.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:43 smithi130 bash[27579]: cluster 2023-08-27T22:11:42.545449+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2023-08-27T22:11:43.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:11:43.587Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:44.343 DEBUG:teuthology.orchestra.run.smithi006:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-08-27T22:11:44.346 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-08-27T22:11:44.346 INFO:teuthology.orchestra.run:waiting for 3000 2023-08-27T22:11:44.392 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:hints = 1 2023-08-27T22:11:44.393 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-08-27T22:11:44.393 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Object prefix: benchmark_data_smithi006_82185 2023-08-27T22:11:44.393 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-27T22:11:44.393 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 0 0 0 0 0 0 - 0 2023-08-27T22:11:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:44 smithi130 bash[27579]: cluster 2023-08-27T22:11:43.168292+0000 mgr.y (mgr.14901) 396 : cluster [DBG] pgmap v402: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:44.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:44 smithi130 bash[27579]: audit 2023-08-27T22:11:43.573887+0000 mon.c (mon.2) 177 : audit [DBG] from='client.? 172.21.15.6:0/538302666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:44.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:44 smithi130 bash[27579]: audit 2023-08-27T22:11:44.011314+0000 mon.a (mon.0) 1039 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:11:44.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:44 smithi006 bash[27292]: cluster 2023-08-27T22:11:43.168292+0000 mgr.y (mgr.14901) 396 : cluster [DBG] pgmap v402: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:44.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:44 smithi006 bash[27292]: audit 2023-08-27T22:11:43.573887+0000 mon.c (mon.2) 177 : audit [DBG] from='client.? 172.21.15.6:0/538302666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:44.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:44 smithi006 bash[27292]: audit 2023-08-27T22:11:44.011314+0000 mon.a (mon.0) 1039 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:11:44.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:44 smithi006 bash[33918]: cluster 2023-08-27T22:11:43.168292+0000 mgr.y (mgr.14901) 396 : cluster [DBG] pgmap v402: 89 pgs: 89 active+clean; 3.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-27T22:11:44.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:44 smithi006 bash[33918]: audit 2023-08-27T22:11:43.573887+0000 mon.c (mon.2) 177 : audit [DBG] from='client.? 172.21.15.6:0/538302666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:44.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:44 smithi006 bash[33918]: audit 2023-08-27T22:11:44.011314+0000 mon.a (mon.0) 1039 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:11:45.393 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 1 16 2383 2367 147.929 147.938 0.0059004 0.00673298 2023-08-27T22:11:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:45 smithi130 bash[27579]: audit 2023-08-27T22:11:44.964293+0000 mon.a (mon.0) 1040 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:11:46.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:45 smithi130 bash[27579]: cluster 2023-08-27T22:11:45.169018+0000 mgr.y (mgr.14901) 397 : cluster [DBG] pgmap v403: 89 pgs: 89 active+clean; 29 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s wr, 46 op/s 2023-08-27T22:11:46.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:45 smithi130 bash[27579]: audit 2023-08-27T22:11:45.366722+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:11:46.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:45 smithi130 bash[27579]: audit 2023-08-27T22:11:45.734765+0000 mon.b (mon.1) 153 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:11:46.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:45 smithi130 bash[27579]: audit 2023-08-27T22:11:45.736295+0000 mon.b (mon.1) 154 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:11:46.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:45 smithi130 bash[27579]: audit 2023-08-27T22:11:45.744285+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:11:46.393 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 2 16 4925 4909 153.394 158.875 0.00432427 0.00650327 2023-08-27T22:11:46.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:45 smithi006 bash[27292]: audit 2023-08-27T22:11:44.964293+0000 mon.a (mon.0) 1040 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:11:46.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:45 smithi006 bash[27292]: cluster 2023-08-27T22:11:45.169018+0000 mgr.y (mgr.14901) 397 : cluster [DBG] pgmap v403: 89 pgs: 89 active+clean; 29 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s wr, 46 op/s 2023-08-27T22:11:46.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:45 smithi006 bash[27292]: audit 2023-08-27T22:11:45.366722+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:11:46.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:45 smithi006 bash[27292]: audit 2023-08-27T22:11:45.734765+0000 mon.b (mon.1) 153 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:11:46.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:45 smithi006 bash[27292]: audit 2023-08-27T22:11:45.736295+0000 mon.b (mon.1) 154 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:11:46.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:45 smithi006 bash[27292]: audit 2023-08-27T22:11:45.744285+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:11:46.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:45 smithi006 bash[33918]: audit 2023-08-27T22:11:44.964293+0000 mon.a (mon.0) 1040 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:11:46.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:45 smithi006 bash[33918]: cluster 2023-08-27T22:11:45.169018+0000 mgr.y (mgr.14901) 397 : cluster [DBG] pgmap v403: 89 pgs: 89 active+clean; 29 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s wr, 46 op/s 2023-08-27T22:11:46.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:45 smithi006 bash[33918]: audit 2023-08-27T22:11:45.366722+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:11:46.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:45 smithi006 bash[33918]: audit 2023-08-27T22:11:45.734765+0000 mon.b (mon.1) 153 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:11:46.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:45 smithi006 bash[33918]: audit 2023-08-27T22:11:45.736295+0000 mon.b (mon.1) 154 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:11:46.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:45 smithi006 bash[33918]: audit 2023-08-27T22:11:45.744285+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:11:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:46 smithi130 bash[27579]: audit 2023-08-27T22:11:46.318224+0000 mon.c (mon.2) 178 : audit [DBG] from='client.? 172.21.15.6:0/4190248477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:47.393 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 3 16 7665 7649 159.341 171.25 0.00459523 0.00626482 2023-08-27T22:11:47.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:46 smithi006 bash[27292]: audit 2023-08-27T22:11:46.318224+0000 mon.c (mon.2) 178 : audit [DBG] from='client.? 172.21.15.6:0/4190248477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:47.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:46 smithi006 bash[33918]: audit 2023-08-27T22:11:46.318224+0000 mon.c (mon.2) 178 : audit [DBG] from='client.? 172.21.15.6:0/4190248477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:48.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:47 smithi130 bash[27579]: cluster 2023-08-27T22:11:47.170188+0000 mgr.y (mgr.14901) 398 : cluster [DBG] pgmap v404: 89 pgs: 89 active+clean; 86 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s wr, 138 op/s 2023-08-27T22:11:48.393 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 4 16 10213 10197 159.314 159.25 0.0150367 0.00626683 2023-08-27T22:11:48.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:47 smithi006 bash[27292]: cluster 2023-08-27T22:11:47.170188+0000 mgr.y (mgr.14901) 398 : cluster [DBG] pgmap v404: 89 pgs: 89 active+clean; 86 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s wr, 138 op/s 2023-08-27T22:11:48.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:47 smithi006 bash[33918]: cluster 2023-08-27T22:11:47.170188+0000 mgr.y (mgr.14901) 398 : cluster [DBG] pgmap v404: 89 pgs: 89 active+clean; 86 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s wr, 138 op/s 2023-08-27T22:11:49.394 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 5 16 12487 12471 155.873 142.125 0.005705 0.00640713 2023-08-27T22:11:49.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:49 smithi130 bash[27579]: audit 2023-08-27T22:11:49.003833+0000 mon.c (mon.2) 179 : audit [DBG] from='client.? 172.21.15.6:0/4230132126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:49.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:49 smithi006 bash[27292]: audit 2023-08-27T22:11:49.003833+0000 mon.c (mon.2) 179 : audit [DBG] from='client.? 172.21.15.6:0/4230132126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:49.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:49 smithi006 bash[33918]: audit 2023-08-27T22:11:49.003833+0000 mon.c (mon.2) 179 : audit [DBG] from='client.? 172.21.15.6:0/4230132126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:50.394 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 6 16 14817 14801 154.163 145.625 0.00764557 0.00647854 2023-08-27T22:11:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:50 smithi130 bash[27579]: cluster 2023-08-27T22:11:49.170683+0000 mgr.y (mgr.14901) 399 : cluster [DBG] pgmap v405: 89 pgs: 89 active+clean; 86 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s wr, 138 op/s 2023-08-27T22:11:50.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:50 smithi006 bash[27292]: cluster 2023-08-27T22:11:49.170683+0000 mgr.y (mgr.14901) 399 : cluster [DBG] pgmap v405: 89 pgs: 89 active+clean; 86 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s wr, 138 op/s 2023-08-27T22:11:50.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:50 smithi006 bash[33918]: cluster 2023-08-27T22:11:49.170683+0000 mgr.y (mgr.14901) 399 : cluster [DBG] pgmap v405: 89 pgs: 89 active+clean; 86 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s wr, 138 op/s 2023-08-27T22:11:51.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:11:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:11:50] "GET /metrics HTTP/1.1" 200 195315 "" "Prometheus/2.33.4" 2023-08-27T22:11:51.394 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 7 16 16974 16958 151.397 134.812 0.004556 0.00659664 2023-08-27T22:11:51.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:11:51.684Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:51.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:11:51.687Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:52.394 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 8 16 19063 19047 148.791 130.562 0.00432869 0.00671339 2023-08-27T22:11:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:52 smithi130 bash[27579]: cluster 2023-08-27T22:11:51.171486+0000 mgr.y (mgr.14901) 400 : cluster [DBG] pgmap v406: 89 pgs: 89 active+clean; 865 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-08-27T22:11:52.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:52 smithi130 bash[27579]: audit 2023-08-27T22:11:51.701011+0000 mon.a (mon.0) 1043 : audit [DBG] from='client.? 172.21.15.6:0/510218964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:52.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:52 smithi006 bash[27292]: cluster 2023-08-27T22:11:51.171486+0000 mgr.y (mgr.14901) 400 : cluster [DBG] pgmap v406: 89 pgs: 89 active+clean; 865 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-08-27T22:11:52.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:52 smithi006 bash[27292]: audit 2023-08-27T22:11:51.701011+0000 mon.a (mon.0) 1043 : audit [DBG] from='client.? 172.21.15.6:0/510218964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:52.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:52 smithi006 bash[33918]: cluster 2023-08-27T22:11:51.171486+0000 mgr.y (mgr.14901) 400 : cluster [DBG] pgmap v406: 89 pgs: 89 active+clean; 865 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-08-27T22:11:52.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:52 smithi006 bash[33918]: audit 2023-08-27T22:11:51.701011+0000 mon.a (mon.0) 1043 : audit [DBG] from='client.? 172.21.15.6:0/510218964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:53.394 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 9 16 21082 21066 146.278 126.188 0.00267571 0.00682966 2023-08-27T22:11:53.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:11:53.586Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:53.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:11:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:11:53.588Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:11:54.394 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 10 16 22981 22965 143.518 118.688 0.00449535 0.00696072 2023-08-27T22:11:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:54 smithi130 bash[27579]: cluster 2023-08-27T22:11:53.171935+0000 mgr.y (mgr.14901) 401 : cluster [DBG] pgmap v407: 89 pgs: 89 active+clean; 865 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-08-27T22:11:54.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:54 smithi006 bash[33918]: cluster 2023-08-27T22:11:53.171935+0000 mgr.y (mgr.14901) 401 : cluster [DBG] pgmap v407: 89 pgs: 89 active+clean; 865 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-08-27T22:11:54.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:54 smithi006 bash[27292]: cluster 2023-08-27T22:11:53.171935+0000 mgr.y (mgr.14901) 401 : cluster [DBG] pgmap v407: 89 pgs: 89 active+clean; 865 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-08-27T22:11:55.394 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 11 15 25035 25020 142.146 128.438 0.0097666 0.00702766 2023-08-27T22:11:55.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:55 smithi130 bash[27579]: audit 2023-08-27T22:11:54.407815+0000 mon.c (mon.2) 180 : audit [DBG] from='client.? 172.21.15.6:0/120885558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:55.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:55 smithi006 bash[33918]: audit 2023-08-27T22:11:54.407815+0000 mon.c (mon.2) 180 : audit [DBG] from='client.? 172.21.15.6:0/120885558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:55.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:55 smithi006 bash[27292]: audit 2023-08-27T22:11:54.407815+0000 mon.c (mon.2) 180 : audit [DBG] from='client.? 172.21.15.6:0/120885558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:56.394 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 12 16 26991 26975 140.482 122.188 0.00673545 0.00711174 2023-08-27T22:11:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:56 smithi130 bash[27579]: cluster 2023-08-27T22:11:55.172690+0000 mgr.y (mgr.14901) 402 : cluster [DBG] pgmap v408: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-08-27T22:11:56.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:56 smithi006 bash[27292]: cluster 2023-08-27T22:11:55.172690+0000 mgr.y (mgr.14901) 402 : cluster [DBG] pgmap v408: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-08-27T22:11:56.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:56 smithi006 bash[33918]: cluster 2023-08-27T22:11:55.172690+0000 mgr.y (mgr.14901) 402 : cluster [DBG] pgmap v408: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-08-27T22:11:57.394 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 13 16 28545 28529 137.146 97.125 0.00752079 0.00728505 2023-08-27T22:11:57.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:57 smithi130 bash[27579]: audit 2023-08-27T22:11:57.087495+0000 mon.c (mon.2) 181 : audit [DBG] from='client.? 172.21.15.6:0/306383811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:57.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:57 smithi006 bash[33918]: audit 2023-08-27T22:11:57.087495+0000 mon.c (mon.2) 181 : audit [DBG] from='client.? 172.21.15.6:0/306383811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:57.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:57 smithi006 bash[27292]: audit 2023-08-27T22:11:57.087495+0000 mon.c (mon.2) 181 : audit [DBG] from='client.? 172.21.15.6:0/306383811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:58.395 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 14 16 30333 30317 135.331 111.75 0.00491583 0.00738021 2023-08-27T22:11:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:11:58 smithi130 bash[27579]: cluster 2023-08-27T22:11:57.173524+0000 mgr.y (mgr.14901) 403 : cluster [DBG] pgmap v409: 89 pgs: 89 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-08-27T22:11:58.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:11:58 smithi006 bash[33918]: cluster 2023-08-27T22:11:57.173524+0000 mgr.y (mgr.14901) 403 : cluster [DBG] pgmap v409: 89 pgs: 89 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-08-27T22:11:58.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:11:58 smithi006 bash[27292]: cluster 2023-08-27T22:11:57.173524+0000 mgr.y (mgr.14901) 403 : cluster [DBG] pgmap v409: 89 pgs: 89 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-08-27T22:11:59.395 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 15 16 32051 32035 133.467 107.375 0.0102568 0.00748672 2023-08-27T22:12:00.395 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 16 16 33625 33609 131.273 98.375 0.00658063 0.00761073 2023-08-27T22:12:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:00 smithi130 bash[27579]: cluster 2023-08-27T22:11:59.174143+0000 mgr.y (mgr.14901) 404 : cluster [DBG] pgmap v410: 89 pgs: 89 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-08-27T22:12:00.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:00 smithi130 bash[27579]: audit 2023-08-27T22:11:59.760977+0000 mon.c (mon.2) 182 : audit [DBG] from='client.? 172.21.15.6:0/1216748985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:00.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:00 smithi006 bash[33918]: cluster 2023-08-27T22:11:59.174143+0000 mgr.y (mgr.14901) 404 : cluster [DBG] pgmap v410: 89 pgs: 89 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-08-27T22:12:00.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:00 smithi006 bash[33918]: audit 2023-08-27T22:11:59.760977+0000 mon.c (mon.2) 182 : audit [DBG] from='client.? 172.21.15.6:0/1216748985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:00.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:00 smithi006 bash[27292]: cluster 2023-08-27T22:11:59.174143+0000 mgr.y (mgr.14901) 404 : cluster [DBG] pgmap v410: 89 pgs: 89 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-08-27T22:12:00.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:00 smithi006 bash[27292]: audit 2023-08-27T22:11:59.760977+0000 mon.c (mon.2) 182 : audit [DBG] from='client.? 172.21.15.6:0/1216748985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:01.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:12:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:12:00] "GET /metrics HTTP/1.1" 200 195398 "" "Prometheus/2.33.4" 2023-08-27T22:12:01.395 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 17 15 35283 35268 129.65 103.688 0.0205836 0.00770659 2023-08-27T22:12:01.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:12:01.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:01.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:12:01.687Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:02.395 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 18 16 36802 36786 127.717 94.875 0.0103066 0.00782233 2023-08-27T22:12:02.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:02 smithi006 bash[33918]: cluster 2023-08-27T22:12:01.174891+0000 mgr.y (mgr.14901) 405 : cluster [DBG] pgmap v411: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 164 MiB/s wr, 2.63k op/s 2023-08-27T22:12:02.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:02 smithi006 bash[27292]: cluster 2023-08-27T22:12:01.174891+0000 mgr.y (mgr.14901) 405 : cluster [DBG] pgmap v411: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 164 MiB/s wr, 2.63k op/s 2023-08-27T22:12:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:02 smithi130 bash[27579]: cluster 2023-08-27T22:12:01.174891+0000 mgr.y (mgr.14901) 405 : cluster [DBG] pgmap v411: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 164 MiB/s wr, 2.63k op/s 2023-08-27T22:12:03.395 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 19 16 38374 38358 126.166 98.25 0.0139696 0.00791986 2023-08-27T22:12:03.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:03 smithi006 bash[33918]: audit 2023-08-27T22:12:02.478249+0000 mon.c (mon.2) 183 : audit [DBG] from='client.? 172.21.15.6:0/1221336748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:03.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:12:03.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:03.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:12:03.588Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:03.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:03 smithi006 bash[27292]: audit 2023-08-27T22:12:02.478249+0000 mon.c (mon.2) 183 : audit [DBG] from='client.? 172.21.15.6:0/1221336748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:03.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:03 smithi130 bash[27579]: audit 2023-08-27T22:12:02.478249+0000 mon.c (mon.2) 183 : audit [DBG] from='client.? 172.21.15.6:0/1221336748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:04.395 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:2023-08-27T22:12:04.391487+0000 min lat: 0.00134927 max lat: 0.088481 avg lat: 0.00800721 2023-08-27T22:12:04.395 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-27T22:12:04.396 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 20 16 39944 39928 124.763 98.125 0.00401338 0.00800721 2023-08-27T22:12:04.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:04 smithi006 bash[33918]: cluster 2023-08-27T22:12:03.175362+0000 mgr.y (mgr.14901) 406 : cluster [DBG] pgmap v412: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-08-27T22:12:04.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:04 smithi006 bash[27292]: cluster 2023-08-27T22:12:03.175362+0000 mgr.y (mgr.14901) 406 : cluster [DBG] pgmap v412: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-08-27T22:12:04.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:04 smithi130 bash[27579]: cluster 2023-08-27T22:12:03.175362+0000 mgr.y (mgr.14901) 406 : cluster [DBG] pgmap v412: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-08-27T22:12:05.395 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 21 16 41378 41362 123.09 89.625 0.0261369 0.00811567 2023-08-27T22:12:05.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:05 smithi006 bash[33918]: audit 2023-08-27T22:12:05.160587+0000 mon.c (mon.2) 184 : audit [DBG] from='client.? 172.21.15.6:0/93322521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:05.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:05 smithi006 bash[27292]: audit 2023-08-27T22:12:05.160587+0000 mon.c (mon.2) 184 : audit [DBG] from='client.? 172.21.15.6:0/93322521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:05.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:05 smithi130 bash[27579]: audit 2023-08-27T22:12:05.160587+0000 mon.c (mon.2) 184 : audit [DBG] from='client.? 172.21.15.6:0/93322521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:06.396 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 22 16 42900 42884 121.818 95.125 0.0184197 0.00820296 2023-08-27T22:12:06.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:06 smithi006 bash[33918]: cluster 2023-08-27T22:12:05.176079+0000 mgr.y (mgr.14901) 407 : cluster [DBG] pgmap v413: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-08-27T22:12:06.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:06 smithi006 bash[27292]: cluster 2023-08-27T22:12:05.176079+0000 mgr.y (mgr.14901) 407 : cluster [DBG] pgmap v413: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-08-27T22:12:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:06 smithi130 bash[27579]: cluster 2023-08-27T22:12:05.176079+0000 mgr.y (mgr.14901) 407 : cluster [DBG] pgmap v413: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-08-27T22:12:07.396 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 23 16 44340 44324 120.434 90 0.0159279 0.00829644 2023-08-27T22:12:08.396 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 24 16 45693 45677 118.939 84.5625 0.0155808 0.00840022 2023-08-27T22:12:08.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:08 smithi006 bash[33918]: cluster 2023-08-27T22:12:07.176732+0000 mgr.y (mgr.14901) 408 : cluster [DBG] pgmap v414: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-08-27T22:12:08.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:08 smithi006 bash[33918]: audit 2023-08-27T22:12:07.847074+0000 mon.c (mon.2) 185 : audit [DBG] from='client.? 172.21.15.6:0/659659769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:08.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:08 smithi006 bash[27292]: cluster 2023-08-27T22:12:07.176732+0000 mgr.y (mgr.14901) 408 : cluster [DBG] pgmap v414: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-08-27T22:12:08.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:08 smithi006 bash[27292]: audit 2023-08-27T22:12:07.847074+0000 mon.c (mon.2) 185 : audit [DBG] from='client.? 172.21.15.6:0/659659769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:08 smithi130 bash[27579]: cluster 2023-08-27T22:12:07.176732+0000 mgr.y (mgr.14901) 408 : cluster [DBG] pgmap v414: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-08-27T22:12:08.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:08 smithi130 bash[27579]: audit 2023-08-27T22:12:07.847074+0000 mon.c (mon.2) 185 : audit [DBG] from='client.? 172.21.15.6:0/659659769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:09.396 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 25 16 47121 47105 117.751 89.25 0.00831928 0.00848671 2023-08-27T22:12:10.396 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 26 16 48438 48422 116.388 82.3125 0.00915643 0.0085856 2023-08-27T22:12:10.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:10 smithi006 bash[27292]: cluster 2023-08-27T22:12:09.177189+0000 mgr.y (mgr.14901) 409 : cluster [DBG] pgmap v415: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-08-27T22:12:10.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:10 smithi006 bash[33918]: cluster 2023-08-27T22:12:09.177189+0000 mgr.y (mgr.14901) 409 : cluster [DBG] pgmap v415: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-08-27T22:12:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:10 smithi130 bash[27579]: cluster 2023-08-27T22:12:09.177189+0000 mgr.y (mgr.14901) 409 : cluster [DBG] pgmap v415: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-08-27T22:12:11.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:12:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:12:10] "GET /metrics HTTP/1.1" 200 195398 "" "Prometheus/2.33.4" 2023-08-27T22:12:11.439 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 27 16 49775 49759 115.172 83.5625 0.00852408 0.00867593 2023-08-27T22:12:11.692 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:11 smithi006 bash[27292]: audit 2023-08-27T22:12:10.534717+0000 mon.a (mon.0) 1044 : audit [DBG] from='client.? 172.21.15.6:0/265961240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:12.318 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:11 smithi006 bash[33918]: audit 2023-08-27T22:12:10.534717+0000 mon.a (mon.0) 1044 : audit [DBG] from='client.? 172.21.15.6:0/265961240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:12.320 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:12:11.686Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:12.320 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:12:11.687Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:12.321 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:11 smithi130 bash[27579]: audit 2023-08-27T22:12:10.534717+0000 mon.a (mon.0) 1044 : audit [DBG] from='client.? 172.21.15.6:0/265961240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:12.396 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 28 16 51139 51123 114.103 85.25 0.0221895 0.00875734 2023-08-27T22:12:12.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:12 smithi006 bash[56298]: level=warn ts=2023-08-27T22:12:12.317Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:13.052 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:12 smithi006 bash[33918]: cluster 2023-08-27T22:12:11.178041+0000 mgr.y (mgr.14901) 410 : cluster [DBG] pgmap v416: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-08-27T22:12:13.052 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:12 smithi006 bash[27292]: cluster 2023-08-27T22:12:11.178041+0000 mgr.y (mgr.14901) 410 : cluster [DBG] pgmap v416: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-08-27T22:12:13.052 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:12 smithi130 bash[27579]: cluster 2023-08-27T22:12:11.178041+0000 mgr.y (mgr.14901) 410 : cluster [DBG] pgmap v416: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-08-27T22:12:13.396 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 29 16 52365 52349 112.81 76.625 0.00382013 0.00885592 2023-08-27T22:12:13.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:12:13.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:13.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:12:13.589Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:13.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:13 smithi006 bash[33918]: audit 2023-08-27T22:12:13.225651+0000 mon.c (mon.2) 186 : audit [DBG] from='client.? 172.21.15.6:0/2058857036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:13.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:13 smithi006 bash[27292]: audit 2023-08-27T22:12:13.225651+0000 mon.c (mon.2) 186 : audit [DBG] from='client.? 172.21.15.6:0/2058857036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:13.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:13 smithi130 bash[27579]: audit 2023-08-27T22:12:13.225651+0000 mon.c (mon.2) 186 : audit [DBG] from='client.? 172.21.15.6:0/2058857036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:14.396 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 30 16 53675 53659 111.779 81.875 0.00820727 0.00893972 2023-08-27T22:12:14.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:14 smithi006 bash[33918]: cluster 2023-08-27T22:12:13.179728+0000 mgr.y (mgr.14901) 411 : cluster [DBG] pgmap v417: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-08-27T22:12:14.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:14 smithi006 bash[27292]: cluster 2023-08-27T22:12:13.179728+0000 mgr.y (mgr.14901) 411 : cluster [DBG] pgmap v417: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-08-27T22:12:14.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:14 smithi130 bash[27579]: cluster 2023-08-27T22:12:13.179728+0000 mgr.y (mgr.14901) 411 : cluster [DBG] pgmap v417: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-08-27T22:12:15.396 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 31 16 54946 54930 110.735 79.4375 0.00348198 0.00902298 2023-08-27T22:12:16.396 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 32 16 56163 56147 109.652 76.0625 0.00742025 0.0091133 2023-08-27T22:12:16.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:16 smithi006 bash[27292]: cluster 2023-08-27T22:12:15.180422+0000 mgr.y (mgr.14901) 412 : cluster [DBG] pgmap v418: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-08-27T22:12:16.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:16 smithi006 bash[27292]: audit 2023-08-27T22:12:15.864962+0000 mon.c (mon.2) 187 : audit [DBG] from='client.? 172.21.15.6:0/3768470860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:16.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:16 smithi006 bash[33918]: cluster 2023-08-27T22:12:15.180422+0000 mgr.y (mgr.14901) 412 : cluster [DBG] pgmap v418: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-08-27T22:12:16.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:16 smithi006 bash[33918]: audit 2023-08-27T22:12:15.864962+0000 mon.c (mon.2) 187 : audit [DBG] from='client.? 172.21.15.6:0/3768470860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:16 smithi130 bash[27579]: cluster 2023-08-27T22:12:15.180422+0000 mgr.y (mgr.14901) 412 : cluster [DBG] pgmap v418: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-08-27T22:12:16.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:16 smithi130 bash[27579]: audit 2023-08-27T22:12:15.864962+0000 mon.c (mon.2) 187 : audit [DBG] from='client.? 172.21.15.6:0/3768470860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:17.396 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 33 16 57403 57387 108.677 77.5 0.00865398 0.00919491 2023-08-27T22:12:18.396 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 34 16 58587 58571 107.657 74 0.0134392 0.00928109 2023-08-27T22:12:18.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:18 smithi006 bash[27292]: cluster 2023-08-27T22:12:17.181098+0000 mgr.y (mgr.14901) 413 : cluster [DBG] pgmap v419: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-08-27T22:12:18.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:18 smithi006 bash[33918]: cluster 2023-08-27T22:12:17.181098+0000 mgr.y (mgr.14901) 413 : cluster [DBG] pgmap v419: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-08-27T22:12:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:18 smithi130 bash[27579]: cluster 2023-08-27T22:12:17.181098+0000 mgr.y (mgr.14901) 413 : cluster [DBG] pgmap v419: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-08-27T22:12:19.396 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 35 16 59782 59766 106.715 74.6875 0.0131588 0.00936469 2023-08-27T22:12:19.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:19 smithi006 bash[27292]: audit 2023-08-27T22:12:18.565183+0000 mon.b (mon.1) 155 : audit [DBG] from='client.? 172.21.15.6:0/1189779030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:19.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:19 smithi006 bash[33918]: audit 2023-08-27T22:12:18.565183+0000 mon.b (mon.1) 155 : audit [DBG] from='client.? 172.21.15.6:0/1189779030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:19.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:19 smithi130 bash[27579]: audit 2023-08-27T22:12:18.565183+0000 mon.b (mon.1) 155 : audit [DBG] from='client.? 172.21.15.6:0/1189779030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:20.397 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 36 16 60961 60945 105.797 73.6875 0.0116898 0.00944549 2023-08-27T22:12:20.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:20 smithi006 bash[33918]: cluster 2023-08-27T22:12:19.181521+0000 mgr.y (mgr.14901) 414 : cluster [DBG] pgmap v420: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-08-27T22:12:20.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:20 smithi006 bash[27292]: cluster 2023-08-27T22:12:19.181521+0000 mgr.y (mgr.14901) 414 : cluster [DBG] pgmap v420: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-08-27T22:12:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:20 smithi130 bash[27579]: cluster 2023-08-27T22:12:19.181521+0000 mgr.y (mgr.14901) 414 : cluster [DBG] pgmap v420: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-08-27T22:12:21.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:12:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:12:20] "GET /metrics HTTP/1.1" 200 195401 "" "Prometheus/2.33.4" 2023-08-27T22:12:21.397 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 37 16 62074 62058 104.818 69.5625 0.00500475 0.00953312 2023-08-27T22:12:21.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:21 smithi130 bash[27579]: audit 2023-08-27T22:12:21.233255+0000 mon.c (mon.2) 188 : audit [DBG] from='client.? 172.21.15.6:0/1354733109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:21.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:12:21.687Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:21.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:12:21.688Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:21.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:21 smithi006 bash[33918]: audit 2023-08-27T22:12:21.233255+0000 mon.c (mon.2) 188 : audit [DBG] from='client.? 172.21.15.6:0/1354733109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:21.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:21 smithi006 bash[27292]: audit 2023-08-27T22:12:21.233255+0000 mon.c (mon.2) 188 : audit [DBG] from='client.? 172.21.15.6:0/1354733109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:22.397 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 38 16 63250 63234 103.993 73.5 0.00679241 0.00960979 2023-08-27T22:12:22.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:22 smithi006 bash[56298]: level=error ts=2023-08-27T22:12:22.316Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:22.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:22 smithi006 bash[56298]: level=warn ts=2023-08-27T22:12:22.318Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:22.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:22 smithi006 bash[33918]: cluster 2023-08-27T22:12:21.182446+0000 mgr.y (mgr.14901) 415 : cluster [DBG] pgmap v421: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-08-27T22:12:22.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:22 smithi006 bash[27292]: cluster 2023-08-27T22:12:21.182446+0000 mgr.y (mgr.14901) 415 : cluster [DBG] pgmap v421: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-08-27T22:12:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:22 smithi130 bash[27579]: cluster 2023-08-27T22:12:21.182446+0000 mgr.y (mgr.14901) 415 : cluster [DBG] pgmap v421: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-08-27T22:12:23.397 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 39 16 64398 64382 103.166 71.75 0.0115319 0.00968684 2023-08-27T22:12:24.010 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:12:23.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:24.010 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:12:23.590Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:24.397 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:2023-08-27T22:12:24.393479+0000 min lat: 0.00134927 max lat: 0.088481 avg lat: 0.00977204 2023-08-27T22:12:24.397 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-27T22:12:24.397 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 40 16 65474 65458 102.268 67.25 0.0143732 0.00977204 2023-08-27T22:12:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:24 smithi130 bash[27579]: cluster 2023-08-27T22:12:23.183056+0000 mgr.y (mgr.14901) 416 : cluster [DBG] pgmap v422: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-08-27T22:12:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:24 smithi130 bash[27579]: audit 2023-08-27T22:12:23.956706+0000 mon.c (mon.2) 189 : audit [DBG] from='client.? 172.21.15.6:0/1866752986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:24.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:24 smithi006 bash[27292]: cluster 2023-08-27T22:12:23.183056+0000 mgr.y (mgr.14901) 416 : cluster [DBG] pgmap v422: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-08-27T22:12:24.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:24 smithi006 bash[27292]: audit 2023-08-27T22:12:23.956706+0000 mon.c (mon.2) 189 : audit [DBG] from='client.? 172.21.15.6:0/1866752986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:24.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:24 smithi006 bash[33918]: cluster 2023-08-27T22:12:23.183056+0000 mgr.y (mgr.14901) 416 : cluster [DBG] pgmap v422: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-08-27T22:12:24.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:24 smithi006 bash[33918]: audit 2023-08-27T22:12:23.956706+0000 mon.c (mon.2) 189 : audit [DBG] from='client.? 172.21.15.6:0/1866752986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:25.397 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 41 16 66654 66638 101.573 73.75 0.00669188 0.00983872 2023-08-27T22:12:26.397 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 42 16 67760 67744 100.8 69.125 0.0173916 0.00991424 2023-08-27T22:12:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:26 smithi130 bash[27579]: cluster 2023-08-27T22:12:25.183760+0000 mgr.y (mgr.14901) 417 : cluster [DBG] pgmap v423: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-08-27T22:12:26.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:26 smithi006 bash[27292]: cluster 2023-08-27T22:12:25.183760+0000 mgr.y (mgr.14901) 417 : cluster [DBG] pgmap v423: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-08-27T22:12:26.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:26 smithi006 bash[33918]: cluster 2023-08-27T22:12:25.183760+0000 mgr.y (mgr.14901) 417 : cluster [DBG] pgmap v423: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-08-27T22:12:27.398 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 43 16 68926 68910 100.15 72.875 0.0107624 0.00997811 2023-08-27T22:12:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:27 smithi130 bash[27579]: audit 2023-08-27T22:12:26.618581+0000 mon.c (mon.2) 190 : audit [DBG] from='client.? 172.21.15.6:0/26439705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:27.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:27 smithi006 bash[33918]: audit 2023-08-27T22:12:26.618581+0000 mon.c (mon.2) 190 : audit [DBG] from='client.? 172.21.15.6:0/26439705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:27.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:27 smithi006 bash[27292]: audit 2023-08-27T22:12:26.618581+0000 mon.c (mon.2) 190 : audit [DBG] from='client.? 172.21.15.6:0/26439705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:28.397 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 44 16 70042 70026 99.4591 69.75 0.0109667 0.010048 2023-08-27T22:12:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:28 smithi130 bash[27579]: cluster 2023-08-27T22:12:27.184230+0000 mgr.y (mgr.14901) 418 : cluster [DBG] pgmap v424: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-08-27T22:12:28.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:28 smithi006 bash[27292]: cluster 2023-08-27T22:12:27.184230+0000 mgr.y (mgr.14901) 418 : cluster [DBG] pgmap v424: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-08-27T22:12:28.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:28 smithi006 bash[33918]: cluster 2023-08-27T22:12:27.184230+0000 mgr.y (mgr.14901) 418 : cluster [DBG] pgmap v424: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-08-27T22:12:29.398 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 45 16 71073 71057 98.6807 64.4375 0.0107185 0.0101285 2023-08-27T22:12:29.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:29 smithi130 bash[27579]: audit 2023-08-27T22:12:29.288616+0000 mon.a (mon.0) 1045 : audit [DBG] from='client.? 172.21.15.6:0/1034070820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:29.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:29 smithi006 bash[27292]: audit 2023-08-27T22:12:29.288616+0000 mon.a (mon.0) 1045 : audit [DBG] from='client.? 172.21.15.6:0/1034070820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:29.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:29 smithi006 bash[33918]: audit 2023-08-27T22:12:29.288616+0000 mon.a (mon.0) 1045 : audit [DBG] from='client.? 172.21.15.6:0/1034070820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:30.398 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 46 16 72139 72123 97.9837 66.625 0.0454921 0.0101999 2023-08-27T22:12:30.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:30 smithi130 bash[27579]: cluster 2023-08-27T22:12:29.184772+0000 mgr.y (mgr.14901) 419 : cluster [DBG] pgmap v425: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s 2023-08-27T22:12:30.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:30 smithi006 bash[27292]: cluster 2023-08-27T22:12:29.184772+0000 mgr.y (mgr.14901) 419 : cluster [DBG] pgmap v425: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s 2023-08-27T22:12:30.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:12:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:12:30] "GET /metrics HTTP/1.1" 200 195404 "" "Prometheus/2.33.4" 2023-08-27T22:12:30.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:30 smithi006 bash[33918]: cluster 2023-08-27T22:12:29.184772+0000 mgr.y (mgr.14901) 419 : cluster [DBG] pgmap v425: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s 2023-08-27T22:12:31.398 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 47 16 73189 73173 97.2951 65.625 0.0117155 0.010271 2023-08-27T22:12:31.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:12:31.688Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:31.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:12:31.689Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:32.398 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 48 16 74205 74189 96.591 63.5 0.0158805 0.0103476 2023-08-27T22:12:32.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:32 smithi006 bash[27292]: cluster 2023-08-27T22:12:31.185708+0000 mgr.y (mgr.14901) 420 : cluster [DBG] pgmap v426: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-08-27T22:12:32.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:32 smithi006 bash[27292]: audit 2023-08-27T22:12:31.940947+0000 mon.c (mon.2) 191 : audit [DBG] from='client.? 172.21.15.6:0/613916171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:32.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:32 smithi006 bash[33918]: cluster 2023-08-27T22:12:31.185708+0000 mgr.y (mgr.14901) 420 : cluster [DBG] pgmap v426: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-08-27T22:12:32.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:32 smithi006 bash[33918]: audit 2023-08-27T22:12:31.940947+0000 mon.c (mon.2) 191 : audit [DBG] from='client.? 172.21.15.6:0/613916171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:32.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:32 smithi006 bash[56298]: level=error ts=2023-08-27T22:12:32.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:32.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:32 smithi006 bash[56298]: level=warn ts=2023-08-27T22:12:32.318Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:32 smithi130 bash[27579]: cluster 2023-08-27T22:12:31.185708+0000 mgr.y (mgr.14901) 420 : cluster [DBG] pgmap v426: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-08-27T22:12:32.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:32 smithi130 bash[27579]: audit 2023-08-27T22:12:31.940947+0000 mon.c (mon.2) 191 : audit [DBG] from='client.? 172.21.15.6:0/613916171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:33.398 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 49 16 75270 75254 95.978 66.5625 0.0230498 0.0104131 2023-08-27T22:12:33.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:12:33.589Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:33.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:12:33.591Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:34.398 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 50 16 76279 76263 95.3195 63.0625 0.0072053 0.0104849 2023-08-27T22:12:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:34 smithi130 bash[27579]: cluster 2023-08-27T22:12:33.186167+0000 mgr.y (mgr.14901) 421 : cluster [DBG] pgmap v427: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s 2023-08-27T22:12:34.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:34 smithi006 bash[27292]: cluster 2023-08-27T22:12:33.186167+0000 mgr.y (mgr.14901) 421 : cluster [DBG] pgmap v427: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s 2023-08-27T22:12:34.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:34 smithi006 bash[33918]: cluster 2023-08-27T22:12:33.186167+0000 mgr.y (mgr.14901) 421 : cluster [DBG] pgmap v427: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s 2023-08-27T22:12:35.398 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 51 16 77290 77274 94.6894 63.1875 0.0159936 0.0105548 2023-08-27T22:12:35.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:35 smithi130 bash[27579]: audit 2023-08-27T22:12:34.632531+0000 mon.a (mon.0) 1046 : audit [DBG] from='client.? 172.21.15.6:0/3488566335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:35.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:35 smithi006 bash[33918]: audit 2023-08-27T22:12:34.632531+0000 mon.a (mon.0) 1046 : audit [DBG] from='client.? 172.21.15.6:0/3488566335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:35.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:35 smithi006 bash[27292]: audit 2023-08-27T22:12:34.632531+0000 mon.a (mon.0) 1046 : audit [DBG] from='client.? 172.21.15.6:0/3488566335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:36.398 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 52 16 78317 78301 94.1027 64.1875 0.00722623 0.0106211 2023-08-27T22:12:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:36 smithi130 bash[27579]: cluster 2023-08-27T22:12:35.186841+0000 mgr.y (mgr.14901) 422 : cluster [DBG] pgmap v428: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-08-27T22:12:36.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:36 smithi006 bash[27292]: cluster 2023-08-27T22:12:35.186841+0000 mgr.y (mgr.14901) 422 : cluster [DBG] pgmap v428: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-08-27T22:12:36.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:36 smithi006 bash[33918]: cluster 2023-08-27T22:12:35.186841+0000 mgr.y (mgr.14901) 422 : cluster [DBG] pgmap v428: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-08-27T22:12:37.398 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 53 16 79247 79231 93.4238 58.125 0.0132895 0.0106976 2023-08-27T22:12:37.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:37 smithi130 bash[27579]: audit 2023-08-27T22:12:37.315632+0000 mon.c (mon.2) 192 : audit [DBG] from='client.? 172.21.15.6:0/4152230072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:37.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:37 smithi006 bash[27292]: audit 2023-08-27T22:12:37.315632+0000 mon.c (mon.2) 192 : audit [DBG] from='client.? 172.21.15.6:0/4152230072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:37.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:37 smithi006 bash[33918]: audit 2023-08-27T22:12:37.315632+0000 mon.c (mon.2) 192 : audit [DBG] from='client.? 172.21.15.6:0/4152230072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:38.398 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 54 16 80258 80242 92.8638 63.1875 0.0136596 0.0107627 2023-08-27T22:12:38.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:38 smithi130 bash[27579]: cluster 2023-08-27T22:12:37.187438+0000 mgr.y (mgr.14901) 423 : cluster [DBG] pgmap v429: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-08-27T22:12:38.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:38 smithi006 bash[33918]: cluster 2023-08-27T22:12:37.187438+0000 mgr.y (mgr.14901) 423 : cluster [DBG] pgmap v429: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-08-27T22:12:38.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:38 smithi006 bash[27292]: cluster 2023-08-27T22:12:37.187438+0000 mgr.y (mgr.14901) 423 : cluster [DBG] pgmap v429: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-08-27T22:12:39.398 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 55 16 81253 81237 92.3059 62.1875 0.0214321 0.0108265 2023-08-27T22:12:40.399 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 56 16 82188 82172 91.7011 58.4375 0.0082736 0.0108982 2023-08-27T22:12:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:40 smithi130 bash[27579]: cluster 2023-08-27T22:12:39.187905+0000 mgr.y (mgr.14901) 424 : cluster [DBG] pgmap v430: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 891 op/s 2023-08-27T22:12:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:40 smithi130 bash[27579]: audit 2023-08-27T22:12:39.976130+0000 mon.a (mon.0) 1047 : audit [DBG] from='client.? 172.21.15.6:0/3288572524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:40.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:40 smithi006 bash[27292]: cluster 2023-08-27T22:12:39.187905+0000 mgr.y (mgr.14901) 424 : cluster [DBG] pgmap v430: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 891 op/s 2023-08-27T22:12:40.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:40 smithi006 bash[27292]: audit 2023-08-27T22:12:39.976130+0000 mon.a (mon.0) 1047 : audit [DBG] from='client.? 172.21.15.6:0/3288572524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:40.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:12:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:12:40] "GET /metrics HTTP/1.1" 200 195404 "" "Prometheus/2.33.4" 2023-08-27T22:12:40.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:40 smithi006 bash[33918]: cluster 2023-08-27T22:12:39.187905+0000 mgr.y (mgr.14901) 424 : cluster [DBG] pgmap v430: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 891 op/s 2023-08-27T22:12:40.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:40 smithi006 bash[33918]: audit 2023-08-27T22:12:39.976130+0000 mon.a (mon.0) 1047 : audit [DBG] from='client.? 172.21.15.6:0/3288572524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:41.399 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 57 16 83185 83169 91.1854 62.3125 0.005643 0.010961 2023-08-27T22:12:41.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:12:41.688Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:41.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:12:41.689Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:42.399 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 58 16 84131 84115 90.6325 59.125 0.0104901 0.0110273 2023-08-27T22:12:42.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:42 smithi006 bash[27292]: cluster 2023-08-27T22:12:41.188587+0000 mgr.y (mgr.14901) 425 : cluster [DBG] pgmap v431: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-08-27T22:12:42.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:42 smithi006 bash[33918]: cluster 2023-08-27T22:12:41.188587+0000 mgr.y (mgr.14901) 425 : cluster [DBG] pgmap v431: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-08-27T22:12:42.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:42 smithi006 bash[56298]: level=error ts=2023-08-27T22:12:42.318Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:42.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:42 smithi006 bash[56298]: level=warn ts=2023-08-27T22:12:42.319Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:42 smithi130 bash[27579]: cluster 2023-08-27T22:12:41.188587+0000 mgr.y (mgr.14901) 425 : cluster [DBG] pgmap v431: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-08-27T22:12:43.400 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 59 16 85042 85026 90.0613 56.9375 0.0184613 0.0110971 2023-08-27T22:12:43.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:43 smithi130 bash[27579]: audit 2023-08-27T22:12:42.686008+0000 mon.a (mon.0) 1048 : audit [DBG] from='client.? 172.21.15.6:0/4085723488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:43.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:43 smithi006 bash[27292]: audit 2023-08-27T22:12:42.686008+0000 mon.a (mon.0) 1048 : audit [DBG] from='client.? 172.21.15.6:0/4085723488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:43.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:43 smithi006 bash[33918]: audit 2023-08-27T22:12:42.686008+0000 mon.a (mon.0) 1048 : audit [DBG] from='client.? 172.21.15.6:0/4085723488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:43.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:12:43.589Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:43.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:12:43.591Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:44.399 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:2023-08-27T22:12:44.395371+0000 min lat: 0.00134927 max lat: 0.088481 avg lat: 0.011161 2023-08-27T22:12:44.497 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-27T22:12:44.497 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 60 16 85993 85977 89.5508 59.4375 0.0195571 0.011161 2023-08-27T22:12:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:44 smithi130 bash[27579]: cluster 2023-08-27T22:12:43.189043+0000 mgr.y (mgr.14901) 426 : cluster [DBG] pgmap v432: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s 2023-08-27T22:12:44.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:44 smithi006 bash[27292]: cluster 2023-08-27T22:12:43.189043+0000 mgr.y (mgr.14901) 426 : cluster [DBG] pgmap v432: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s 2023-08-27T22:12:44.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:44 smithi006 bash[33918]: cluster 2023-08-27T22:12:43.189043+0000 mgr.y (mgr.14901) 426 : cluster [DBG] pgmap v432: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s 2023-08-27T22:12:45.399 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 61 16 86873 86857 88.9843 55 0.0136204 0.0112313 2023-08-27T22:12:45.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:45 smithi130 bash[27579]: audit 2023-08-27T22:12:45.353352+0000 mon.c (mon.2) 193 : audit [DBG] from='client.? 172.21.15.6:0/3318747010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:45.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:45 smithi006 bash[27292]: audit 2023-08-27T22:12:45.353352+0000 mon.c (mon.2) 193 : audit [DBG] from='client.? 172.21.15.6:0/3318747010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:45.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:45 smithi006 bash[33918]: audit 2023-08-27T22:12:45.353352+0000 mon.c (mon.2) 193 : audit [DBG] from='client.? 172.21.15.6:0/3318747010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:46.399 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 62 16 87785 87769 88.4683 57 0.00881021 0.0112981 2023-08-27T22:12:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:46 smithi130 bash[27579]: cluster 2023-08-27T22:12:45.189722+0000 mgr.y (mgr.14901) 427 : cluster [DBG] pgmap v433: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-08-27T22:12:46.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:46 smithi130 bash[27579]: audit 2023-08-27T22:12:45.759685+0000 mon.b (mon.1) 156 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:12:46.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:46 smithi006 bash[27292]: cluster 2023-08-27T22:12:45.189722+0000 mgr.y (mgr.14901) 427 : cluster [DBG] pgmap v433: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-08-27T22:12:46.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:46 smithi006 bash[27292]: audit 2023-08-27T22:12:45.759685+0000 mon.b (mon.1) 156 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:12:46.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:46 smithi006 bash[33918]: cluster 2023-08-27T22:12:45.189722+0000 mgr.y (mgr.14901) 427 : cluster [DBG] pgmap v433: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-08-27T22:12:46.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:46 smithi006 bash[33918]: audit 2023-08-27T22:12:45.759685+0000 mon.b (mon.1) 156 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:12:47.399 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 63 16 88725 88709 87.9965 58.75 0.0172879 0.011358 2023-08-27T22:12:48.399 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 64 16 89564 89548 87.4408 52.4375 0.0169047 0.0114305 2023-08-27T22:12:48.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:48 smithi130 bash[27579]: cluster 2023-08-27T22:12:47.190343+0000 mgr.y (mgr.14901) 428 : cluster [DBG] pgmap v434: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2023-08-27T22:12:48.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:48 smithi130 bash[27579]: audit 2023-08-27T22:12:48.046106+0000 mon.a (mon.0) 1049 : audit [DBG] from='client.? 172.21.15.6:0/1497247648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:48.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:48 smithi006 bash[27292]: cluster 2023-08-27T22:12:47.190343+0000 mgr.y (mgr.14901) 428 : cluster [DBG] pgmap v434: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2023-08-27T22:12:48.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:48 smithi006 bash[27292]: audit 2023-08-27T22:12:48.046106+0000 mon.a (mon.0) 1049 : audit [DBG] from='client.? 172.21.15.6:0/1497247648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:48.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:48 smithi006 bash[33918]: cluster 2023-08-27T22:12:47.190343+0000 mgr.y (mgr.14901) 428 : cluster [DBG] pgmap v434: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2023-08-27T22:12:48.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:48 smithi006 bash[33918]: audit 2023-08-27T22:12:48.046106+0000 mon.a (mon.0) 1049 : audit [DBG] from='client.? 172.21.15.6:0/1497247648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:49.399 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 65 16 90493 90477 86.9888 58.0625 0.0144113 0.0114898 2023-08-27T22:12:50.400 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 66 16 91390 91374 86.5201 56.0625 0.0124789 0.0115517 2023-08-27T22:12:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:50 smithi130 bash[27579]: cluster 2023-08-27T22:12:49.190799+0000 mgr.y (mgr.14901) 429 : cluster [DBG] pgmap v435: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-08-27T22:12:50.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:50 smithi006 bash[27292]: cluster 2023-08-27T22:12:49.190799+0000 mgr.y (mgr.14901) 429 : cluster [DBG] pgmap v435: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-08-27T22:12:50.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:12:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:12:50] "GET /metrics HTTP/1.1" 200 195411 "" "Prometheus/2.33.4" 2023-08-27T22:12:50.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:50 smithi006 bash[33918]: cluster 2023-08-27T22:12:49.190799+0000 mgr.y (mgr.14901) 429 : cluster [DBG] pgmap v435: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-08-27T22:12:51.403 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 67 16 92155 92139 85.9394 47.8125 0.0140892 0.0116294 2023-08-27T22:12:51.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:51 smithi006 bash[27292]: audit 2023-08-27T22:12:50.768754+0000 mon.a (mon.0) 1050 : audit [DBG] from='client.? 172.21.15.6:0/3636079184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:51.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:51 smithi006 bash[33918]: audit 2023-08-27T22:12:50.768754+0000 mon.a (mon.0) 1050 : audit [DBG] from='client.? 172.21.15.6:0/3636079184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:51.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:12:51.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:51.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:12:51.692Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:52.055 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:51 smithi130 bash[27579]: audit 2023-08-27T22:12:50.768754+0000 mon.a (mon.0) 1050 : audit [DBG] from='client.? 172.21.15.6:0/3636079184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:52.402 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 68 16 92917 92901 85.3759 47.625 0.0309307 0.0117066 2023-08-27T22:12:52.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:52 smithi006 bash[56298]: level=error ts=2023-08-27T22:12:52.318Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:52.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:52 smithi006 bash[56298]: level=warn ts=2023-08-27T22:12:52.320Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:53.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:52 smithi006 bash[27292]: cluster 2023-08-27T22:12:51.191587+0000 mgr.y (mgr.14901) 430 : cluster [DBG] pgmap v436: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-08-27T22:12:53.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:52 smithi006 bash[27292]: audit 2023-08-27T22:12:51.892470+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:12:53.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:52 smithi006 bash[27292]: audit 2023-08-27T22:12:52.240058+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:12:53.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:52 smithi006 bash[27292]: audit 2023-08-27T22:12:52.579711+0000 mon.b (mon.1) 157 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:12:53.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:52 smithi006 bash[27292]: audit 2023-08-27T22:12:52.584889+0000 mon.b (mon.1) 158 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:12:53.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:52 smithi006 bash[27292]: audit 2023-08-27T22:12:52.589838+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:12:53.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:52 smithi006 bash[33918]: cluster 2023-08-27T22:12:51.191587+0000 mgr.y (mgr.14901) 430 : cluster [DBG] pgmap v436: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-08-27T22:12:53.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:52 smithi006 bash[33918]: audit 2023-08-27T22:12:51.892470+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:12:53.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:52 smithi006 bash[33918]: audit 2023-08-27T22:12:52.240058+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:12:53.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:52 smithi006 bash[33918]: audit 2023-08-27T22:12:52.579711+0000 mon.b (mon.1) 157 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:12:53.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:52 smithi006 bash[33918]: audit 2023-08-27T22:12:52.584889+0000 mon.b (mon.1) 158 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:12:53.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:52 smithi006 bash[33918]: audit 2023-08-27T22:12:52.589838+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:12:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:52 smithi130 bash[27579]: cluster 2023-08-27T22:12:51.191587+0000 mgr.y (mgr.14901) 430 : cluster [DBG] pgmap v436: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-08-27T22:12:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:52 smithi130 bash[27579]: audit 2023-08-27T22:12:51.892470+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:12:53.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:52 smithi130 bash[27579]: audit 2023-08-27T22:12:52.240058+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:12:53.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:52 smithi130 bash[27579]: audit 2023-08-27T22:12:52.579711+0000 mon.b (mon.1) 157 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:12:53.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:52 smithi130 bash[27579]: audit 2023-08-27T22:12:52.584889+0000 mon.b (mon.1) 158 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:12:53.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:52 smithi130 bash[27579]: audit 2023-08-27T22:12:52.589838+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:12:53.402 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 69 16 93757 93741 84.8994 52.5 0.00887059 0.0117732 2023-08-27T22:12:53.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:12:53.590Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:53.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:12:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:12:53.591Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:12:54.402 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 70 16 94645 94629 84.4793 55.5 0.0206517 0.0118305 2023-08-27T22:12:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:54 smithi130 bash[27579]: cluster 2023-08-27T22:12:53.192031+0000 mgr.y (mgr.14901) 431 : cluster [DBG] pgmap v437: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s 2023-08-27T22:12:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:54 smithi130 bash[27579]: audit 2023-08-27T22:12:53.446596+0000 mon.a (mon.0) 1054 : audit [DBG] from='client.? 172.21.15.6:0/3139499026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:54.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:54 smithi006 bash[33918]: cluster 2023-08-27T22:12:53.192031+0000 mgr.y (mgr.14901) 431 : cluster [DBG] pgmap v437: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s 2023-08-27T22:12:54.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:54 smithi006 bash[33918]: audit 2023-08-27T22:12:53.446596+0000 mon.a (mon.0) 1054 : audit [DBG] from='client.? 172.21.15.6:0/3139499026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:54.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:54 smithi006 bash[27292]: cluster 2023-08-27T22:12:53.192031+0000 mgr.y (mgr.14901) 431 : cluster [DBG] pgmap v437: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s 2023-08-27T22:12:54.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:54 smithi006 bash[27292]: audit 2023-08-27T22:12:53.446596+0000 mon.a (mon.0) 1054 : audit [DBG] from='client.? 172.21.15.6:0/3139499026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:55.402 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 71 16 95550 95534 84.0861 56.5625 0.0166968 0.0118862 2023-08-27T22:12:56.402 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 72 16 96390 96374 83.6473 52.5 0.00744018 0.0119488 2023-08-27T22:12:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:56 smithi130 bash[27579]: cluster 2023-08-27T22:12:55.192966+0000 mgr.y (mgr.14901) 432 : cluster [DBG] pgmap v438: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2023-08-27T22:12:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:56 smithi130 bash[27579]: audit 2023-08-27T22:12:56.124009+0000 mon.c (mon.2) 194 : audit [DBG] from='client.? 172.21.15.6:0/4082354658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:56.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:56 smithi006 bash[33918]: cluster 2023-08-27T22:12:55.192966+0000 mgr.y (mgr.14901) 432 : cluster [DBG] pgmap v438: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2023-08-27T22:12:56.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:56 smithi006 bash[33918]: audit 2023-08-27T22:12:56.124009+0000 mon.c (mon.2) 194 : audit [DBG] from='client.? 172.21.15.6:0/4082354658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:56.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:56 smithi006 bash[27292]: cluster 2023-08-27T22:12:55.192966+0000 mgr.y (mgr.14901) 432 : cluster [DBG] pgmap v438: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2023-08-27T22:12:56.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:56 smithi006 bash[27292]: audit 2023-08-27T22:12:56.124009+0000 mon.c (mon.2) 194 : audit [DBG] from='client.? 172.21.15.6:0/4082354658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:57.403 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 73 16 97283 97267 83.2659 55.8125 0.0161659 0.0120031 2023-08-27T22:12:58.403 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 74 16 98143 98127 82.867 53.75 0.0104428 0.0120605 2023-08-27T22:12:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:58 smithi130 bash[27579]: cluster 2023-08-27T22:12:57.193586+0000 mgr.y (mgr.14901) 433 : cluster [DBG] pgmap v439: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s 2023-08-27T22:12:58.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:58 smithi006 bash[27292]: cluster 2023-08-27T22:12:57.193586+0000 mgr.y (mgr.14901) 433 : cluster [DBG] pgmap v439: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s 2023-08-27T22:12:58.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:58 smithi006 bash[33918]: cluster 2023-08-27T22:12:57.193586+0000 mgr.y (mgr.14901) 433 : cluster [DBG] pgmap v439: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s 2023-08-27T22:12:59.403 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 75 16 98990 98974 82.4679 52.9375 0.00803998 0.0121202 2023-08-27T22:12:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:12:59 smithi130 bash[27579]: audit 2023-08-27T22:12:58.805056+0000 mon.a (mon.0) 1055 : audit [DBG] from='client.? 172.21.15.6:0/348996286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:59.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:12:59 smithi006 bash[33918]: audit 2023-08-27T22:12:58.805056+0000 mon.a (mon.0) 1055 : audit [DBG] from='client.? 172.21.15.6:0/348996286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:59.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:12:59 smithi006 bash[27292]: audit 2023-08-27T22:12:58.805056+0000 mon.a (mon.0) 1055 : audit [DBG] from='client.? 172.21.15.6:0/348996286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:00.403 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 76 16 99842 99826 82.0834 53.25 0.0425986 0.0121767 2023-08-27T22:13:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:00 smithi130 bash[27579]: cluster 2023-08-27T22:12:59.194131+0000 mgr.y (mgr.14901) 434 : cluster [DBG] pgmap v440: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 721 op/s 2023-08-27T22:13:00.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:00 smithi006 bash[27292]: cluster 2023-08-27T22:12:59.194131+0000 mgr.y (mgr.14901) 434 : cluster [DBG] pgmap v440: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 721 op/s 2023-08-27T22:13:00.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:00 smithi006 bash[33918]: cluster 2023-08-27T22:12:59.194131+0000 mgr.y (mgr.14901) 434 : cluster [DBG] pgmap v440: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 721 op/s 2023-08-27T22:13:01.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:13:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:13:00] "GET /metrics HTTP/1.1" 200 195419 "" "Prometheus/2.33.4" 2023-08-27T22:13:01.403 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 77 16 100665 100649 81.6854 51.4375 0.00784433 0.0122353 2023-08-27T22:13:01.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:13:01.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:01.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:13:01.691Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:02.403 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 78 16 101556 101540 81.352 55.6875 0.00811085 0.0122863 2023-08-27T22:13:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:02 smithi130 bash[27579]: cluster 2023-08-27T22:13:01.194966+0000 mgr.y (mgr.14901) 435 : cluster [DBG] pgmap v441: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-08-27T22:13:02.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:02 smithi130 bash[27579]: audit 2023-08-27T22:13:01.477325+0000 mon.c (mon.2) 195 : audit [DBG] from='client.? 172.21.15.6:0/1248756814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:02.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:02 smithi006 bash[56298]: level=error ts=2023-08-27T22:13:02.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:02.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:02 smithi006 bash[56298]: level=warn ts=2023-08-27T22:13:02.320Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:02.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:02 smithi006 bash[33918]: cluster 2023-08-27T22:13:01.194966+0000 mgr.y (mgr.14901) 435 : cluster [DBG] pgmap v441: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-08-27T22:13:02.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:02 smithi006 bash[33918]: audit 2023-08-27T22:13:01.477325+0000 mon.c (mon.2) 195 : audit [DBG] from='client.? 172.21.15.6:0/1248756814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:02.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:02 smithi006 bash[27292]: cluster 2023-08-27T22:13:01.194966+0000 mgr.y (mgr.14901) 435 : cluster [DBG] pgmap v441: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-08-27T22:13:02.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:02 smithi006 bash[27292]: audit 2023-08-27T22:13:01.477325+0000 mon.c (mon.2) 195 : audit [DBG] from='client.? 172.21.15.6:0/1248756814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:03.403 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 79 16 102463 102447 81.0397 56.6875 0.0337297 0.012333 2023-08-27T22:13:03.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:13:03.591Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:03.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:13:03.592Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:04.404 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:2023-08-27T22:13:04.399591+0000 min lat: 0.00134927 max lat: 0.088481 avg lat: 0.0123968 2023-08-27T22:13:04.404 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-27T22:13:04.404 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 80 16 103229 103213 80.6251 47.875 0.0105889 0.0123968 2023-08-27T22:13:04.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:04 smithi006 bash[27292]: cluster 2023-08-27T22:13:03.195346+0000 mgr.y (mgr.14901) 436 : cluster [DBG] pgmap v442: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s 2023-08-27T22:13:04.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:04 smithi006 bash[27292]: audit 2023-08-27T22:13:04.160992+0000 mon.a (mon.0) 1056 : audit [DBG] from='client.? 172.21.15.6:0/476228671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:04.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:04 smithi006 bash[33918]: cluster 2023-08-27T22:13:03.195346+0000 mgr.y (mgr.14901) 436 : cluster [DBG] pgmap v442: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s 2023-08-27T22:13:04.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:04 smithi006 bash[33918]: audit 2023-08-27T22:13:04.160992+0000 mon.a (mon.0) 1056 : audit [DBG] from='client.? 172.21.15.6:0/476228671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:04 smithi130 bash[27579]: cluster 2023-08-27T22:13:03.195346+0000 mgr.y (mgr.14901) 436 : cluster [DBG] pgmap v442: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s 2023-08-27T22:13:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:04 smithi130 bash[27579]: audit 2023-08-27T22:13:04.160992+0000 mon.a (mon.0) 1056 : audit [DBG] from='client.? 172.21.15.6:0/476228671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:05.403 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 81 16 104057 104041 80.2686 51.75 0.00980759 0.0124519 2023-08-27T22:13:06.403 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 82 16 104872 104856 79.9108 50.9375 0.0258304 0.0125064 2023-08-27T22:13:06.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:06 smithi006 bash[27292]: cluster 2023-08-27T22:13:05.196018+0000 mgr.y (mgr.14901) 437 : cluster [DBG] pgmap v443: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s 2023-08-27T22:13:06.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:06 smithi006 bash[33918]: cluster 2023-08-27T22:13:05.196018+0000 mgr.y (mgr.14901) 437 : cluster [DBG] pgmap v443: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s 2023-08-27T22:13:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:06 smithi130 bash[27579]: cluster 2023-08-27T22:13:05.196018+0000 mgr.y (mgr.14901) 437 : cluster [DBG] pgmap v443: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s 2023-08-27T22:13:07.403 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 83 16 105663 105647 79.5436 49.4375 0.017616 0.0125657 2023-08-27T22:13:07.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:07 smithi006 bash[33918]: audit 2023-08-27T22:13:06.849723+0000 mon.b (mon.1) 159 : audit [DBG] from='client.? 172.21.15.6:0/3237362075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:07.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:07 smithi006 bash[27292]: audit 2023-08-27T22:13:06.849723+0000 mon.b (mon.1) 159 : audit [DBG] from='client.? 172.21.15.6:0/3237362075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:07.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:07 smithi130 bash[27579]: audit 2023-08-27T22:13:06.849723+0000 mon.b (mon.1) 159 : audit [DBG] from='client.? 172.21.15.6:0/3237362075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:08.403 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 84 16 106499 106483 79.2187 52.25 0.0134072 0.0126173 2023-08-27T22:13:08.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:08 smithi006 bash[33918]: cluster 2023-08-27T22:13:07.196508+0000 mgr.y (mgr.14901) 438 : cluster [DBG] pgmap v444: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s 2023-08-27T22:13:08.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:08 smithi006 bash[27292]: cluster 2023-08-27T22:13:07.196508+0000 mgr.y (mgr.14901) 438 : cluster [DBG] pgmap v444: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s 2023-08-27T22:13:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:08 smithi130 bash[27579]: cluster 2023-08-27T22:13:07.196508+0000 mgr.y (mgr.14901) 438 : cluster [DBG] pgmap v444: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s 2023-08-27T22:13:09.404 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 85 16 107264 107248 78.8491 47.8125 0.010987 0.0126753 2023-08-27T22:13:10.404 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 86 16 108059 108043 78.51 49.6875 0.00867274 0.0127305 2023-08-27T22:13:10.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:10 smithi006 bash[27292]: cluster 2023-08-27T22:13:09.197063+0000 mgr.y (mgr.14901) 439 : cluster [DBG] pgmap v445: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 712 op/s 2023-08-27T22:13:10.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:10 smithi006 bash[27292]: audit 2023-08-27T22:13:09.519233+0000 mon.a (mon.0) 1057 : audit [DBG] from='client.? 172.21.15.6:0/3404343643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:10.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:10 smithi006 bash[33918]: cluster 2023-08-27T22:13:09.197063+0000 mgr.y (mgr.14901) 439 : cluster [DBG] pgmap v445: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 712 op/s 2023-08-27T22:13:10.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:10 smithi006 bash[33918]: audit 2023-08-27T22:13:09.519233+0000 mon.a (mon.0) 1057 : audit [DBG] from='client.? 172.21.15.6:0/3404343643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:10 smithi130 bash[27579]: cluster 2023-08-27T22:13:09.197063+0000 mgr.y (mgr.14901) 439 : cluster [DBG] pgmap v445: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 712 op/s 2023-08-27T22:13:10.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:10 smithi130 bash[27579]: audit 2023-08-27T22:13:09.519233+0000 mon.a (mon.0) 1057 : audit [DBG] from='client.? 172.21.15.6:0/3404343643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:11.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:13:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:13:10] "GET /metrics HTTP/1.1" 200 195419 "" "Prometheus/2.33.4" 2023-08-27T22:13:11.404 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 87 16 108891 108875 78.2052 52 0.0292959 0.0127806 2023-08-27T22:13:11.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:13:11.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:11.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:13:11.692Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:12.404 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 88 16 109667 109651 77.8676 48.5 0.0131036 0.012836 2023-08-27T22:13:12.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:12 smithi006 bash[27292]: cluster 2023-08-27T22:13:11.197779+0000 mgr.y (mgr.14901) 440 : cluster [DBG] pgmap v446: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-08-27T22:13:12.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:12 smithi006 bash[27292]: audit 2023-08-27T22:13:12.207119+0000 mon.a (mon.0) 1058 : audit [DBG] from='client.? 172.21.15.6:0/1921702384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:12.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:12 smithi006 bash[33918]: cluster 2023-08-27T22:13:11.197779+0000 mgr.y (mgr.14901) 440 : cluster [DBG] pgmap v446: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-08-27T22:13:12.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:12 smithi006 bash[33918]: audit 2023-08-27T22:13:12.207119+0000 mon.a (mon.0) 1058 : audit [DBG] from='client.? 172.21.15.6:0/1921702384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:12.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:12 smithi006 bash[56298]: level=error ts=2023-08-27T22:13:12.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:12.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:12 smithi006 bash[56298]: level=warn ts=2023-08-27T22:13:12.321Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:12 smithi130 bash[27579]: cluster 2023-08-27T22:13:11.197779+0000 mgr.y (mgr.14901) 440 : cluster [DBG] pgmap v446: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-08-27T22:13:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:12 smithi130 bash[27579]: audit 2023-08-27T22:13:12.207119+0000 mon.a (mon.0) 1058 : audit [DBG] from='client.? 172.21.15.6:0/1921702384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:13.404 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 89 16 110470 110454 77.5566 50.1875 0.0299356 0.0128868 2023-08-27T22:13:13.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:13:13.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:13.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:13:13.593Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:14.404 INFO:tasks.radosbench.radosbench.0.smithi006.stdout: 90 7 111254 111247 77.2455 49.5625 0.0121798 0.0129391 2023-08-27T22:13:14.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:14 smithi006 bash[33918]: cluster 2023-08-27T22:13:13.198183+0000 mgr.y (mgr.14901) 441 : cluster [DBG] pgmap v447: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 43 MiB/s wr, 687 op/s 2023-08-27T22:13:14.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:14 smithi006 bash[27292]: cluster 2023-08-27T22:13:13.198183+0000 mgr.y (mgr.14901) 441 : cluster [DBG] pgmap v447: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 43 MiB/s wr, 687 op/s 2023-08-27T22:13:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:14 smithi130 bash[27579]: cluster 2023-08-27T22:13:13.198183+0000 mgr.y (mgr.14901) 441 : cluster [DBG] pgmap v447: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 43 MiB/s wr, 687 op/s 2023-08-27T22:13:15.405 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Total time run: 90.0231 2023-08-27T22:13:15.405 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Total writes made: 111254 2023-08-27T22:13:15.405 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Write size: 65536 2023-08-27T22:13:15.406 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Object size: 65536 2023-08-27T22:13:15.406 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Bandwidth (MB/sec): 77.2399 2023-08-27T22:13:15.406 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Stddev Bandwidth: 29.8688 2023-08-27T22:13:15.406 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Max bandwidth (MB/sec): 171.25 2023-08-27T22:13:15.406 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Min bandwidth (MB/sec): 47.625 2023-08-27T22:13:15.406 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Average IOPS: 1235 2023-08-27T22:13:15.406 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Stddev IOPS: 477.9 2023-08-27T22:13:15.406 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Max IOPS: 2740 2023-08-27T22:13:15.406 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Min IOPS: 762 2023-08-27T22:13:15.407 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Average Latency(s): 0.0129402 2023-08-27T22:13:15.407 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Stddev Latency(s): 0.00803307 2023-08-27T22:13:15.407 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Max latency(s): 0.088481 2023-08-27T22:13:15.407 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Min latency(s): 0.00134927 2023-08-27T22:13:15.416 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Cleaning up (deleting benchmark objects) 2023-08-27T22:13:15.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:15 smithi006 bash[33918]: audit 2023-08-27T22:13:14.679343+0000 mon.a (mon.0) 1059 : audit [DBG] from='client.? 172.21.15.6:0/889454602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:15.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:15 smithi006 bash[27292]: audit 2023-08-27T22:13:14.679343+0000 mon.a (mon.0) 1059 : audit [DBG] from='client.? 172.21.15.6:0/889454602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:15.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:15 smithi130 bash[27579]: audit 2023-08-27T22:13:14.679343+0000 mon.a (mon.0) 1059 : audit [DBG] from='client.? 172.21.15.6:0/889454602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:16.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:16 smithi006 bash[33918]: cluster 2023-08-27T22:13:15.199190+0000 mgr.y (mgr.14901) 442 : cluster [DBG] pgmap v448: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s 2023-08-27T22:13:16.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:16 smithi006 bash[27292]: cluster 2023-08-27T22:13:15.199190+0000 mgr.y (mgr.14901) 442 : cluster [DBG] pgmap v448: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s 2023-08-27T22:13:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:16 smithi130 bash[27579]: cluster 2023-08-27T22:13:15.199190+0000 mgr.y (mgr.14901) 442 : cluster [DBG] pgmap v448: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s 2023-08-27T22:13:17.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:17 smithi006 bash[27292]: audit 2023-08-27T22:13:17.331768+0000 mon.c (mon.2) 196 : audit [DBG] from='client.? 172.21.15.6:0/1399669319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:17.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:17 smithi006 bash[33918]: audit 2023-08-27T22:13:17.331768+0000 mon.c (mon.2) 196 : audit [DBG] from='client.? 172.21.15.6:0/1399669319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:17.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:17 smithi130 bash[27579]: audit 2023-08-27T22:13:17.331768+0000 mon.c (mon.2) 196 : audit [DBG] from='client.? 172.21.15.6:0/1399669319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:18.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:18 smithi006 bash[27292]: cluster 2023-08-27T22:13:17.199847+0000 mgr.y (mgr.14901) 443 : cluster [DBG] pgmap v449: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 757 op/s 2023-08-27T22:13:18.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:18 smithi006 bash[33918]: cluster 2023-08-27T22:13:17.199847+0000 mgr.y (mgr.14901) 443 : cluster [DBG] pgmap v449: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 757 op/s 2023-08-27T22:13:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:18 smithi130 bash[27579]: cluster 2023-08-27T22:13:17.199847+0000 mgr.y (mgr.14901) 443 : cluster [DBG] pgmap v449: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 757 op/s 2023-08-27T22:13:20.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:20 smithi006 bash[33918]: cluster 2023-08-27T22:13:19.200300+0000 mgr.y (mgr.14901) 444 : cluster [DBG] pgmap v450: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 39 MiB/s wr, 626 op/s 2023-08-27T22:13:20.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:20 smithi006 bash[33918]: audit 2023-08-27T22:13:19.998687+0000 mon.c (mon.2) 197 : audit [DBG] from='client.? 172.21.15.6:0/2369477699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:20.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:20 smithi006 bash[27292]: cluster 2023-08-27T22:13:19.200300+0000 mgr.y (mgr.14901) 444 : cluster [DBG] pgmap v450: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 39 MiB/s wr, 626 op/s 2023-08-27T22:13:20.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:20 smithi006 bash[27292]: audit 2023-08-27T22:13:19.998687+0000 mon.c (mon.2) 197 : audit [DBG] from='client.? 172.21.15.6:0/2369477699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:20 smithi130 bash[27579]: cluster 2023-08-27T22:13:19.200300+0000 mgr.y (mgr.14901) 444 : cluster [DBG] pgmap v450: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 39 MiB/s wr, 626 op/s 2023-08-27T22:13:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:20 smithi130 bash[27579]: audit 2023-08-27T22:13:19.998687+0000 mon.c (mon.2) 197 : audit [DBG] from='client.? 172.21.15.6:0/2369477699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:21.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:13:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:13:20] "GET /metrics HTTP/1.1" 200 195426 "" "Prometheus/2.33.4" 2023-08-27T22:13:21.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:13:21.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:21.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:13:21.692Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:22.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:22 smithi006 bash[27292]: cluster 2023-08-27T22:13:21.201186+0000 mgr.y (mgr.14901) 445 : cluster [DBG] pgmap v451: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 936 op/s 2023-08-27T22:13:22.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:22 smithi006 bash[33918]: cluster 2023-08-27T22:13:21.201186+0000 mgr.y (mgr.14901) 445 : cluster [DBG] pgmap v451: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 936 op/s 2023-08-27T22:13:22.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:22 smithi006 bash[56298]: level=error ts=2023-08-27T22:13:22.320Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:22.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:22 smithi006 bash[56298]: level=warn ts=2023-08-27T22:13:22.325Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:22 smithi130 bash[27579]: cluster 2023-08-27T22:13:21.201186+0000 mgr.y (mgr.14901) 445 : cluster [DBG] pgmap v451: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 936 op/s 2023-08-27T22:13:23.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:13:23.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:23.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:13:23.593Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:23.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:23 smithi006 bash[33918]: audit 2023-08-27T22:13:22.671054+0000 mon.c (mon.2) 198 : audit [DBG] from='client.? 172.21.15.6:0/1442887440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:23.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:23 smithi006 bash[27292]: audit 2023-08-27T22:13:22.671054+0000 mon.c (mon.2) 198 : audit [DBG] from='client.? 172.21.15.6:0/1442887440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:23.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:23 smithi130 bash[27579]: audit 2023-08-27T22:13:22.671054+0000 mon.c (mon.2) 198 : audit [DBG] from='client.? 172.21.15.6:0/1442887440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:24.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:24 smithi006 bash[33918]: cluster 2023-08-27T22:13:23.201615+0000 mgr.y (mgr.14901) 446 : cluster [DBG] pgmap v452: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 602 op/s 2023-08-27T22:13:24.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:24 smithi006 bash[27292]: cluster 2023-08-27T22:13:23.201615+0000 mgr.y (mgr.14901) 446 : cluster [DBG] pgmap v452: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 602 op/s 2023-08-27T22:13:24.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:24 smithi130 bash[27579]: cluster 2023-08-27T22:13:23.201615+0000 mgr.y (mgr.14901) 446 : cluster [DBG] pgmap v452: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 602 op/s 2023-08-27T22:13:25.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:25 smithi130 bash[27579]: audit 2023-08-27T22:13:25.339056+0000 mon.b (mon.1) 160 : audit [DBG] from='client.? 172.21.15.6:0/1704267493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:25.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:25 smithi006 bash[33918]: audit 2023-08-27T22:13:25.339056+0000 mon.b (mon.1) 160 : audit [DBG] from='client.? 172.21.15.6:0/1704267493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:25.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:25 smithi006 bash[27292]: audit 2023-08-27T22:13:25.339056+0000 mon.b (mon.1) 160 : audit [DBG] from='client.? 172.21.15.6:0/1704267493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:26.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:26 smithi130 bash[27579]: cluster 2023-08-27T22:13:25.202310+0000 mgr.y (mgr.14901) 447 : cluster [DBG] pgmap v453: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 809 op/s 2023-08-27T22:13:26.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:26 smithi006 bash[33918]: cluster 2023-08-27T22:13:25.202310+0000 mgr.y (mgr.14901) 447 : cluster [DBG] pgmap v453: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 809 op/s 2023-08-27T22:13:26.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:26 smithi006 bash[27292]: cluster 2023-08-27T22:13:25.202310+0000 mgr.y (mgr.14901) 447 : cluster [DBG] pgmap v453: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 809 op/s 2023-08-27T22:13:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:28 smithi130 bash[27579]: cluster 2023-08-27T22:13:27.206230+0000 mgr.y (mgr.14901) 448 : cluster [DBG] pgmap v454: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 5.9 MiB/s wr, 735 op/s 2023-08-27T22:13:28.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:28 smithi130 bash[27579]: audit 2023-08-27T22:13:28.007291+0000 mon.c (mon.2) 199 : audit [DBG] from='client.? 172.21.15.6:0/763483843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:28.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:28 smithi006 bash[33918]: cluster 2023-08-27T22:13:27.206230+0000 mgr.y (mgr.14901) 448 : cluster [DBG] pgmap v454: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 5.9 MiB/s wr, 735 op/s 2023-08-27T22:13:28.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:28 smithi006 bash[33918]: audit 2023-08-27T22:13:28.007291+0000 mon.c (mon.2) 199 : audit [DBG] from='client.? 172.21.15.6:0/763483843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:28.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:28 smithi006 bash[27292]: cluster 2023-08-27T22:13:27.206230+0000 mgr.y (mgr.14901) 448 : cluster [DBG] pgmap v454: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 5.9 MiB/s wr, 735 op/s 2023-08-27T22:13:28.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:28 smithi006 bash[27292]: audit 2023-08-27T22:13:28.007291+0000 mon.c (mon.2) 199 : audit [DBG] from='client.? 172.21.15.6:0/763483843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:30.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:30 smithi006 bash[33918]: cluster 2023-08-27T22:13:29.206727+0000 mgr.y (mgr.14901) 449 : cluster [DBG] pgmap v455: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 640 op/s 2023-08-27T22:13:30.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:30 smithi006 bash[27292]: cluster 2023-08-27T22:13:29.206727+0000 mgr.y (mgr.14901) 449 : cluster [DBG] pgmap v455: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 640 op/s 2023-08-27T22:13:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:30 smithi130 bash[27579]: cluster 2023-08-27T22:13:29.206727+0000 mgr.y (mgr.14901) 449 : cluster [DBG] pgmap v455: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 640 op/s 2023-08-27T22:13:31.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:13:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:13:30] "GET /metrics HTTP/1.1" 200 195427 "" "Prometheus/2.33.4" 2023-08-27T22:13:31.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:31 smithi130 bash[27579]: audit 2023-08-27T22:13:30.737090+0000 mon.b (mon.1) 161 : audit [DBG] from='client.? 172.21.15.6:0/4100108146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:31.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:31 smithi006 bash[27292]: audit 2023-08-27T22:13:30.737090+0000 mon.b (mon.1) 161 : audit [DBG] from='client.? 172.21.15.6:0/4100108146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:31.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:31 smithi006 bash[33918]: audit 2023-08-27T22:13:30.737090+0000 mon.b (mon.1) 161 : audit [DBG] from='client.? 172.21.15.6:0/4100108146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:31.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:13:31.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:31.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:13:31.693Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:32.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:32 smithi006 bash[27292]: cluster 2023-08-27T22:13:31.207511+0000 mgr.y (mgr.14901) 450 : cluster [DBG] pgmap v456: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 960 op/s 2023-08-27T22:13:32.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:32 smithi006 bash[33918]: cluster 2023-08-27T22:13:31.207511+0000 mgr.y (mgr.14901) 450 : cluster [DBG] pgmap v456: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 960 op/s 2023-08-27T22:13:32.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:32 smithi006 bash[56298]: level=error ts=2023-08-27T22:13:32.320Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:32.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:32 smithi006 bash[56298]: level=warn ts=2023-08-27T22:13:32.322Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:32 smithi130 bash[27579]: cluster 2023-08-27T22:13:31.207511+0000 mgr.y (mgr.14901) 450 : cluster [DBG] pgmap v456: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 960 op/s 2023-08-27T22:13:33.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:13:33.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:33.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:13:33.594Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:34 smithi130 bash[27579]: cluster 2023-08-27T22:13:33.209220+0000 mgr.y (mgr.14901) 451 : cluster [DBG] pgmap v457: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-08-27T22:13:34.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:34 smithi130 bash[27579]: audit 2023-08-27T22:13:33.402547+0000 mon.a (mon.0) 1060 : audit [DBG] from='client.? 172.21.15.6:0/3354632210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:34.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:34 smithi006 bash[33918]: cluster 2023-08-27T22:13:33.209220+0000 mgr.y (mgr.14901) 451 : cluster [DBG] pgmap v457: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-08-27T22:13:34.751 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:34 smithi006 bash[33918]: audit 2023-08-27T22:13:33.402547+0000 mon.a (mon.0) 1060 : audit [DBG] from='client.? 172.21.15.6:0/3354632210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:34.751 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:34 smithi006 bash[27292]: cluster 2023-08-27T22:13:33.209220+0000 mgr.y (mgr.14901) 451 : cluster [DBG] pgmap v457: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-08-27T22:13:34.751 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:34 smithi006 bash[27292]: audit 2023-08-27T22:13:33.402547+0000 mon.a (mon.0) 1060 : audit [DBG] from='client.? 172.21.15.6:0/3354632210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:36.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:36 smithi130 bash[27579]: cluster 2023-08-27T22:13:35.211198+0000 mgr.y (mgr.14901) 452 : cluster [DBG] pgmap v458: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2023-08-27T22:13:36.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:36 smithi130 bash[27579]: audit 2023-08-27T22:13:36.058798+0000 mon.a (mon.0) 1061 : audit [DBG] from='client.? 172.21.15.6:0/3126113096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:36.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:36 smithi006 bash[33918]: cluster 2023-08-27T22:13:35.211198+0000 mgr.y (mgr.14901) 452 : cluster [DBG] pgmap v458: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2023-08-27T22:13:36.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:36 smithi006 bash[33918]: audit 2023-08-27T22:13:36.058798+0000 mon.a (mon.0) 1061 : audit [DBG] from='client.? 172.21.15.6:0/3126113096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:36.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:36 smithi006 bash[27292]: cluster 2023-08-27T22:13:35.211198+0000 mgr.y (mgr.14901) 452 : cluster [DBG] pgmap v458: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2023-08-27T22:13:36.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:36 smithi006 bash[27292]: audit 2023-08-27T22:13:36.058798+0000 mon.a (mon.0) 1061 : audit [DBG] from='client.? 172.21.15.6:0/3126113096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:38 smithi130 bash[27579]: cluster 2023-08-27T22:13:37.211897+0000 mgr.y (mgr.14901) 453 : cluster [DBG] pgmap v459: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2023-08-27T22:13:38.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:38 smithi006 bash[27292]: cluster 2023-08-27T22:13:37.211897+0000 mgr.y (mgr.14901) 453 : cluster [DBG] pgmap v459: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2023-08-27T22:13:38.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:38 smithi006 bash[33918]: cluster 2023-08-27T22:13:37.211897+0000 mgr.y (mgr.14901) 453 : cluster [DBG] pgmap v459: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2023-08-27T22:13:39.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:39 smithi130 bash[27579]: audit 2023-08-27T22:13:38.724373+0000 mon.c (mon.2) 200 : audit [DBG] from='client.? 172.21.15.6:0/256361389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:39.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:39 smithi006 bash[33918]: audit 2023-08-27T22:13:38.724373+0000 mon.c (mon.2) 200 : audit [DBG] from='client.? 172.21.15.6:0/256361389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:39.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:39 smithi006 bash[27292]: audit 2023-08-27T22:13:38.724373+0000 mon.c (mon.2) 200 : audit [DBG] from='client.? 172.21.15.6:0/256361389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:40 smithi130 bash[27579]: cluster 2023-08-27T22:13:39.212350+0000 mgr.y (mgr.14901) 454 : cluster [DBG] pgmap v460: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-08-27T22:13:40.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:40 smithi006 bash[27292]: cluster 2023-08-27T22:13:39.212350+0000 mgr.y (mgr.14901) 454 : cluster [DBG] pgmap v460: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-08-27T22:13:40.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:40 smithi006 bash[33918]: cluster 2023-08-27T22:13:39.212350+0000 mgr.y (mgr.14901) 454 : cluster [DBG] pgmap v460: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-08-27T22:13:41.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:13:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:13:40] "GET /metrics HTTP/1.1" 200 195427 "" "Prometheus/2.33.4" 2023-08-27T22:13:41.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:13:41.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:41.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:13:41.694Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:42.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:42 smithi006 bash[27292]: cluster 2023-08-27T22:13:41.213234+0000 mgr.y (mgr.14901) 455 : cluster [DBG] pgmap v461: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 948 op/s 2023-08-27T22:13:42.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:42 smithi006 bash[27292]: audit 2023-08-27T22:13:41.427213+0000 mon.c (mon.2) 201 : audit [DBG] from='client.? 172.21.15.6:0/3118507768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:42.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:42 smithi006 bash[33918]: cluster 2023-08-27T22:13:41.213234+0000 mgr.y (mgr.14901) 455 : cluster [DBG] pgmap v461: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 948 op/s 2023-08-27T22:13:42.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:42 smithi006 bash[33918]: audit 2023-08-27T22:13:41.427213+0000 mon.c (mon.2) 201 : audit [DBG] from='client.? 172.21.15.6:0/3118507768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:42.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:42 smithi006 bash[56298]: level=error ts=2023-08-27T22:13:42.321Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:42.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:42 smithi006 bash[56298]: level=warn ts=2023-08-27T22:13:42.323Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:42 smithi130 bash[27579]: cluster 2023-08-27T22:13:41.213234+0000 mgr.y (mgr.14901) 455 : cluster [DBG] pgmap v461: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 948 op/s 2023-08-27T22:13:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:42 smithi130 bash[27579]: audit 2023-08-27T22:13:41.427213+0000 mon.c (mon.2) 201 : audit [DBG] from='client.? 172.21.15.6:0/3118507768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:43.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:13:43.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:43.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:13:43.595Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:44.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:44 smithi006 bash[27292]: cluster 2023-08-27T22:13:43.213694+0000 mgr.y (mgr.14901) 456 : cluster [DBG] pgmap v462: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-08-27T22:13:44.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:44 smithi006 bash[27292]: audit 2023-08-27T22:13:44.113002+0000 mon.c (mon.2) 202 : audit [DBG] from='client.? 172.21.15.6:0/2678110586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:44.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:44 smithi006 bash[33918]: cluster 2023-08-27T22:13:43.213694+0000 mgr.y (mgr.14901) 456 : cluster [DBG] pgmap v462: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-08-27T22:13:44.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:44 smithi006 bash[33918]: audit 2023-08-27T22:13:44.113002+0000 mon.c (mon.2) 202 : audit [DBG] from='client.? 172.21.15.6:0/2678110586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:44 smithi130 bash[27579]: cluster 2023-08-27T22:13:43.213694+0000 mgr.y (mgr.14901) 456 : cluster [DBG] pgmap v462: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-08-27T22:13:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:44 smithi130 bash[27579]: audit 2023-08-27T22:13:44.113002+0000 mon.c (mon.2) 202 : audit [DBG] from='client.? 172.21.15.6:0/2678110586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:46.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:46 smithi006 bash[33918]: cluster 2023-08-27T22:13:45.214378+0000 mgr.y (mgr.14901) 457 : cluster [DBG] pgmap v463: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2023-08-27T22:13:46.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:46 smithi006 bash[27292]: cluster 2023-08-27T22:13:45.214378+0000 mgr.y (mgr.14901) 457 : cluster [DBG] pgmap v463: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2023-08-27T22:13:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:46 smithi130 bash[27579]: cluster 2023-08-27T22:13:45.214378+0000 mgr.y (mgr.14901) 457 : cluster [DBG] pgmap v463: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2023-08-27T22:13:47.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:47 smithi006 bash[33918]: audit 2023-08-27T22:13:46.801837+0000 mon.a (mon.0) 1062 : audit [DBG] from='client.? 172.21.15.6:0/697907756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:47.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:47 smithi006 bash[27292]: audit 2023-08-27T22:13:46.801837+0000 mon.a (mon.0) 1062 : audit [DBG] from='client.? 172.21.15.6:0/697907756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:47.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:47 smithi130 bash[27579]: audit 2023-08-27T22:13:46.801837+0000 mon.a (mon.0) 1062 : audit [DBG] from='client.? 172.21.15.6:0/697907756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:48.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:48 smithi006 bash[33918]: cluster 2023-08-27T22:13:47.216342+0000 mgr.y (mgr.14901) 458 : cluster [DBG] pgmap v464: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-08-27T22:13:48.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:48 smithi006 bash[27292]: cluster 2023-08-27T22:13:47.216342+0000 mgr.y (mgr.14901) 458 : cluster [DBG] pgmap v464: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-08-27T22:13:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:48 smithi130 bash[27579]: cluster 2023-08-27T22:13:47.216342+0000 mgr.y (mgr.14901) 458 : cluster [DBG] pgmap v464: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-08-27T22:13:50.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:50 smithi006 bash[27292]: cluster 2023-08-27T22:13:49.216783+0000 mgr.y (mgr.14901) 459 : cluster [DBG] pgmap v465: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-08-27T22:13:50.863 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:50 smithi006 bash[27292]: audit 2023-08-27T22:13:49.474718+0000 mon.c (mon.2) 203 : audit [DBG] from='client.? 172.21.15.6:0/71718232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:50.863 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:50 smithi006 bash[33918]: cluster 2023-08-27T22:13:49.216783+0000 mgr.y (mgr.14901) 459 : cluster [DBG] pgmap v465: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-08-27T22:13:50.863 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:50 smithi006 bash[33918]: audit 2023-08-27T22:13:49.474718+0000 mon.c (mon.2) 203 : audit [DBG] from='client.? 172.21.15.6:0/71718232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:50 smithi130 bash[27579]: cluster 2023-08-27T22:13:49.216783+0000 mgr.y (mgr.14901) 459 : cluster [DBG] pgmap v465: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-08-27T22:13:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:50 smithi130 bash[27579]: audit 2023-08-27T22:13:49.474718+0000 mon.c (mon.2) 203 : audit [DBG] from='client.? 172.21.15.6:0/71718232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:51.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:13:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:13:50] "GET /metrics HTTP/1.1" 200 195427 "" "Prometheus/2.33.4" 2023-08-27T22:13:51.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:13:51.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:51.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:13:51.696Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:52.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:52 smithi006 bash[27292]: cluster 2023-08-27T22:13:51.218545+0000 mgr.y (mgr.14901) 460 : cluster [DBG] pgmap v466: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 915 op/s 2023-08-27T22:13:52.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:52 smithi006 bash[27292]: audit 2023-08-27T22:13:52.118804+0000 mon.c (mon.2) 204 : audit [DBG] from='client.? 172.21.15.6:0/1607105996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:52.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:52 smithi006 bash[33918]: cluster 2023-08-27T22:13:51.218545+0000 mgr.y (mgr.14901) 460 : cluster [DBG] pgmap v466: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 915 op/s 2023-08-27T22:13:52.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:52 smithi006 bash[33918]: audit 2023-08-27T22:13:52.118804+0000 mon.c (mon.2) 204 : audit [DBG] from='client.? 172.21.15.6:0/1607105996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:52.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:52 smithi006 bash[56298]: level=error ts=2023-08-27T22:13:52.322Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:52.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:52 smithi006 bash[56298]: level=warn ts=2023-08-27T22:13:52.323Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:52 smithi130 bash[27579]: cluster 2023-08-27T22:13:51.218545+0000 mgr.y (mgr.14901) 460 : cluster [DBG] pgmap v466: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 915 op/s 2023-08-27T22:13:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:52 smithi130 bash[27579]: audit 2023-08-27T22:13:52.118804+0000 mon.c (mon.2) 204 : audit [DBG] from='client.? 172.21.15.6:0/1607105996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:53.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:53 smithi006 bash[27292]: audit 2023-08-27T22:13:52.606418+0000 mon.b (mon.1) 162 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:13:53.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:53 smithi006 bash[33918]: audit 2023-08-27T22:13:52.606418+0000 mon.b (mon.1) 162 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:13:53.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:13:53.594Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:53.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:13:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:13:53.596Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:13:53.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:53 smithi130 bash[27579]: audit 2023-08-27T22:13:52.606418+0000 mon.b (mon.1) 162 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:13:54.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:54 smithi006 bash[27292]: cluster 2023-08-27T22:13:53.218977+0000 mgr.y (mgr.14901) 461 : cluster [DBG] pgmap v467: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-08-27T22:13:54.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:54 smithi006 bash[33918]: cluster 2023-08-27T22:13:53.218977+0000 mgr.y (mgr.14901) 461 : cluster [DBG] pgmap v467: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-08-27T22:13:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:54 smithi130 bash[27579]: cluster 2023-08-27T22:13:53.218977+0000 mgr.y (mgr.14901) 461 : cluster [DBG] pgmap v467: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-08-27T22:13:55.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:55 smithi006 bash[33918]: audit 2023-08-27T22:13:54.792554+0000 mon.b (mon.1) 163 : audit [DBG] from='client.? 172.21.15.6:0/1461767371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:55.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:55 smithi006 bash[27292]: audit 2023-08-27T22:13:54.792554+0000 mon.b (mon.1) 163 : audit [DBG] from='client.? 172.21.15.6:0/1461767371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:55.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:55 smithi130 bash[27579]: audit 2023-08-27T22:13:54.792554+0000 mon.b (mon.1) 163 : audit [DBG] from='client.? 172.21.15.6:0/1461767371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:56.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:56 smithi006 bash[33918]: cluster 2023-08-27T22:13:55.219614+0000 mgr.y (mgr.14901) 462 : cluster [DBG] pgmap v468: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2023-08-27T22:13:56.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:56 smithi006 bash[27292]: cluster 2023-08-27T22:13:55.219614+0000 mgr.y (mgr.14901) 462 : cluster [DBG] pgmap v468: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2023-08-27T22:13:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:56 smithi130 bash[27579]: cluster 2023-08-27T22:13:55.219614+0000 mgr.y (mgr.14901) 462 : cluster [DBG] pgmap v468: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2023-08-27T22:13:58.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:58 smithi006 bash[27292]: cluster 2023-08-27T22:13:57.220094+0000 mgr.y (mgr.14901) 463 : cluster [DBG] pgmap v469: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-08-27T22:13:58.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:58 smithi006 bash[27292]: audit 2023-08-27T22:13:57.445507+0000 mon.c (mon.2) 205 : audit [DBG] from='client.? 172.21.15.6:0/399790194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:58.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:58 smithi006 bash[33918]: cluster 2023-08-27T22:13:57.220094+0000 mgr.y (mgr.14901) 463 : cluster [DBG] pgmap v469: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-08-27T22:13:58.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:58 smithi006 bash[33918]: audit 2023-08-27T22:13:57.445507+0000 mon.c (mon.2) 205 : audit [DBG] from='client.? 172.21.15.6:0/399790194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:58 smithi130 bash[27579]: cluster 2023-08-27T22:13:57.220094+0000 mgr.y (mgr.14901) 463 : cluster [DBG] pgmap v469: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-08-27T22:13:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:58 smithi130 bash[27579]: audit 2023-08-27T22:13:57.445507+0000 mon.c (mon.2) 205 : audit [DBG] from='client.? 172.21.15.6:0/399790194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:59 smithi130 bash[27579]: audit 2023-08-27T22:13:58.724669+0000 mon.a (mon.0) 1063 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:14:00.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:59 smithi130 bash[27579]: audit 2023-08-27T22:13:59.182066+0000 mon.a (mon.0) 1064 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:14:00.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:59 smithi130 bash[27579]: audit 2023-08-27T22:13:59.517910+0000 mon.b (mon.1) 164 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:14:00.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:59 smithi130 bash[27579]: audit 2023-08-27T22:13:59.519494+0000 mon.b (mon.1) 165 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:14:00.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:13:59 smithi130 bash[27579]: audit 2023-08-27T22:13:59.527351+0000 mon.a (mon.0) 1065 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:14:00.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:59 smithi006 bash[27292]: audit 2023-08-27T22:13:58.724669+0000 mon.a (mon.0) 1063 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:14:00.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:59 smithi006 bash[27292]: audit 2023-08-27T22:13:59.182066+0000 mon.a (mon.0) 1064 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:14:00.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:59 smithi006 bash[27292]: audit 2023-08-27T22:13:59.517910+0000 mon.b (mon.1) 164 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:14:00.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:59 smithi006 bash[27292]: audit 2023-08-27T22:13:59.519494+0000 mon.b (mon.1) 165 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:14:00.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:13:59 smithi006 bash[27292]: audit 2023-08-27T22:13:59.527351+0000 mon.a (mon.0) 1065 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:14:00.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:59 smithi006 bash[33918]: audit 2023-08-27T22:13:58.724669+0000 mon.a (mon.0) 1063 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:14:00.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:59 smithi006 bash[33918]: audit 2023-08-27T22:13:59.182066+0000 mon.a (mon.0) 1064 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:14:00.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:59 smithi006 bash[33918]: audit 2023-08-27T22:13:59.517910+0000 mon.b (mon.1) 164 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:14:00.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:59 smithi006 bash[33918]: audit 2023-08-27T22:13:59.519494+0000 mon.b (mon.1) 165 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:14:00.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:13:59 smithi006 bash[33918]: audit 2023-08-27T22:13:59.527351+0000 mon.a (mon.0) 1065 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:14:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:00 smithi130 bash[27579]: cluster 2023-08-27T22:13:59.220703+0000 mgr.y (mgr.14901) 464 : cluster [DBG] pgmap v470: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-08-27T22:14:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:00 smithi130 bash[27579]: audit 2023-08-27T22:14:00.142325+0000 mon.c (mon.2) 206 : audit [DBG] from='client.? 172.21.15.6:0/92242814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:01.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:00 smithi006 bash[27292]: cluster 2023-08-27T22:13:59.220703+0000 mgr.y (mgr.14901) 464 : cluster [DBG] pgmap v470: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-08-27T22:14:01.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:00 smithi006 bash[27292]: audit 2023-08-27T22:14:00.142325+0000 mon.c (mon.2) 206 : audit [DBG] from='client.? 172.21.15.6:0/92242814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:01.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:14:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:14:00] "GET /metrics HTTP/1.1" 200 195433 "" "Prometheus/2.33.4" 2023-08-27T22:14:01.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:00 smithi006 bash[33918]: cluster 2023-08-27T22:13:59.220703+0000 mgr.y (mgr.14901) 464 : cluster [DBG] pgmap v470: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-08-27T22:14:01.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:00 smithi006 bash[33918]: audit 2023-08-27T22:14:00.142325+0000 mon.c (mon.2) 206 : audit [DBG] from='client.? 172.21.15.6:0/92242814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:01.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:14:01.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:01.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:14:01.696Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:02.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:02 smithi006 bash[56298]: level=error ts=2023-08-27T22:14:02.324Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:02 smithi130 bash[27579]: cluster 2023-08-27T22:14:01.221434+0000 mgr.y (mgr.14901) 465 : cluster [DBG] pgmap v471: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2023-08-27T22:14:03.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:02 smithi006 bash[33918]: cluster 2023-08-27T22:14:01.221434+0000 mgr.y (mgr.14901) 465 : cluster [DBG] pgmap v471: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2023-08-27T22:14:03.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:02 smithi006 bash[27292]: cluster 2023-08-27T22:14:01.221434+0000 mgr.y (mgr.14901) 465 : cluster [DBG] pgmap v471: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2023-08-27T22:14:03.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:03 smithi006 bash[27292]: audit 2023-08-27T22:14:02.825197+0000 mon.c (mon.2) 207 : audit [DBG] from='client.? 172.21.15.6:0/447282255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:03.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:03 smithi006 bash[33918]: audit 2023-08-27T22:14:02.825197+0000 mon.c (mon.2) 207 : audit [DBG] from='client.? 172.21.15.6:0/447282255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:03.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:14:03.595Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:03.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:14:03.596Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:04.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:03 smithi130 bash[27579]: audit 2023-08-27T22:14:02.825197+0000 mon.c (mon.2) 207 : audit [DBG] from='client.? 172.21.15.6:0/447282255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:04 smithi130 bash[27579]: cluster 2023-08-27T22:14:03.221909+0000 mgr.y (mgr.14901) 466 : cluster [DBG] pgmap v472: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 559 op/s 2023-08-27T22:14:05.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:04 smithi006 bash[27292]: cluster 2023-08-27T22:14:03.221909+0000 mgr.y (mgr.14901) 466 : cluster [DBG] pgmap v472: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 559 op/s 2023-08-27T22:14:05.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:04 smithi006 bash[33918]: cluster 2023-08-27T22:14:03.221909+0000 mgr.y (mgr.14901) 466 : cluster [DBG] pgmap v472: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 559 op/s 2023-08-27T22:14:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:05 smithi130 bash[27579]: audit 2023-08-27T22:14:05.476922+0000 mon.a (mon.0) 1066 : audit [DBG] from='client.? 172.21.15.6:0/3613188410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:06.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:05 smithi006 bash[27292]: audit 2023-08-27T22:14:05.476922+0000 mon.a (mon.0) 1066 : audit [DBG] from='client.? 172.21.15.6:0/3613188410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:06.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:05 smithi006 bash[33918]: audit 2023-08-27T22:14:05.476922+0000 mon.a (mon.0) 1066 : audit [DBG] from='client.? 172.21.15.6:0/3613188410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:06 smithi130 bash[27579]: cluster 2023-08-27T22:14:05.225442+0000 mgr.y (mgr.14901) 467 : cluster [DBG] pgmap v473: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2023-08-27T22:14:07.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:06 smithi006 bash[27292]: cluster 2023-08-27T22:14:05.225442+0000 mgr.y (mgr.14901) 467 : cluster [DBG] pgmap v473: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2023-08-27T22:14:07.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:06 smithi006 bash[33918]: cluster 2023-08-27T22:14:05.225442+0000 mgr.y (mgr.14901) 467 : cluster [DBG] pgmap v473: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2023-08-27T22:14:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:08 smithi130 bash[27579]: cluster 2023-08-27T22:14:07.225963+0000 mgr.y (mgr.14901) 468 : cluster [DBG] pgmap v474: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-08-27T22:14:09.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:08 smithi130 bash[27579]: audit 2023-08-27T22:14:08.154884+0000 mon.a (mon.0) 1067 : audit [DBG] from='client.? 172.21.15.6:0/3478499743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:09.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:08 smithi006 bash[27292]: cluster 2023-08-27T22:14:07.225963+0000 mgr.y (mgr.14901) 468 : cluster [DBG] pgmap v474: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-08-27T22:14:09.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:08 smithi006 bash[27292]: audit 2023-08-27T22:14:08.154884+0000 mon.a (mon.0) 1067 : audit [DBG] from='client.? 172.21.15.6:0/3478499743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:09.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:08 smithi006 bash[33918]: cluster 2023-08-27T22:14:07.225963+0000 mgr.y (mgr.14901) 468 : cluster [DBG] pgmap v474: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-08-27T22:14:09.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:08 smithi006 bash[33918]: audit 2023-08-27T22:14:08.154884+0000 mon.a (mon.0) 1067 : audit [DBG] from='client.? 172.21.15.6:0/3478499743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:10 smithi130 bash[27579]: cluster 2023-08-27T22:14:09.226532+0000 mgr.y (mgr.14901) 469 : cluster [DBG] pgmap v475: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-08-27T22:14:11.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:10 smithi006 bash[27292]: cluster 2023-08-27T22:14:09.226532+0000 mgr.y (mgr.14901) 469 : cluster [DBG] pgmap v475: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-08-27T22:14:11.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:14:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:14:10] "GET /metrics HTTP/1.1" 200 195433 "" "Prometheus/2.33.4" 2023-08-27T22:14:11.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:10 smithi006 bash[33918]: cluster 2023-08-27T22:14:09.226532+0000 mgr.y (mgr.14901) 469 : cluster [DBG] pgmap v475: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-08-27T22:14:11.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:11 smithi006 bash[27292]: audit 2023-08-27T22:14:10.821132+0000 mon.c (mon.2) 208 : audit [DBG] from='client.? 172.21.15.6:0/130926252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:11.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:11 smithi006 bash[33918]: audit 2023-08-27T22:14:10.821132+0000 mon.c (mon.2) 208 : audit [DBG] from='client.? 172.21.15.6:0/130926252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:11.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:14:11.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:11.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:14:11.697Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:12.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:11 smithi130 bash[27579]: audit 2023-08-27T22:14:10.821132+0000 mon.c (mon.2) 208 : audit [DBG] from='client.? 172.21.15.6:0/130926252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:12 smithi130 bash[27579]: cluster 2023-08-27T22:14:11.227094+0000 mgr.y (mgr.14901) 470 : cluster [DBG] pgmap v476: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 797 op/s 2023-08-27T22:14:13.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:12 smithi006 bash[27292]: cluster 2023-08-27T22:14:11.227094+0000 mgr.y (mgr.14901) 470 : cluster [DBG] pgmap v476: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 797 op/s 2023-08-27T22:14:13.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:12 smithi006 bash[33918]: cluster 2023-08-27T22:14:11.227094+0000 mgr.y (mgr.14901) 470 : cluster [DBG] pgmap v476: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 797 op/s 2023-08-27T22:14:13.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:14:13.596Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:13.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:14:13.597Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:14 smithi130 bash[27579]: cluster 2023-08-27T22:14:13.227505+0000 mgr.y (mgr.14901) 471 : cluster [DBG] pgmap v477: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 584 op/s 2023-08-27T22:14:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:14 smithi130 bash[27579]: audit 2023-08-27T22:14:13.474823+0000 mon.b (mon.1) 166 : audit [DBG] from='client.? 172.21.15.6:0/4086466501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:14.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:14 smithi006 bash[33918]: cluster 2023-08-27T22:14:13.227505+0000 mgr.y (mgr.14901) 471 : cluster [DBG] pgmap v477: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 584 op/s 2023-08-27T22:14:14.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:14 smithi006 bash[33918]: audit 2023-08-27T22:14:13.474823+0000 mon.b (mon.1) 166 : audit [DBG] from='client.? 172.21.15.6:0/4086466501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:14.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:14 smithi006 bash[27292]: cluster 2023-08-27T22:14:13.227505+0000 mgr.y (mgr.14901) 471 : cluster [DBG] pgmap v477: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 584 op/s 2023-08-27T22:14:14.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:14 smithi006 bash[27292]: audit 2023-08-27T22:14:13.474823+0000 mon.b (mon.1) 166 : audit [DBG] from='client.? 172.21.15.6:0/4086466501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:16 smithi130 bash[27579]: cluster 2023-08-27T22:14:15.228300+0000 mgr.y (mgr.14901) 472 : cluster [DBG] pgmap v478: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2023-08-27T22:14:16.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:16 smithi130 bash[27579]: audit 2023-08-27T22:14:16.106260+0000 mon.c (mon.2) 209 : audit [DBG] from='client.? 172.21.15.6:0/2898020453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:16.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:16 smithi006 bash[27292]: cluster 2023-08-27T22:14:15.228300+0000 mgr.y (mgr.14901) 472 : cluster [DBG] pgmap v478: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2023-08-27T22:14:16.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:16 smithi006 bash[27292]: audit 2023-08-27T22:14:16.106260+0000 mon.c (mon.2) 209 : audit [DBG] from='client.? 172.21.15.6:0/2898020453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:16.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:16 smithi006 bash[33918]: cluster 2023-08-27T22:14:15.228300+0000 mgr.y (mgr.14901) 472 : cluster [DBG] pgmap v478: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2023-08-27T22:14:16.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:16 smithi006 bash[33918]: audit 2023-08-27T22:14:16.106260+0000 mon.c (mon.2) 209 : audit [DBG] from='client.? 172.21.15.6:0/2898020453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:18 smithi130 bash[27579]: cluster 2023-08-27T22:14:17.228824+0000 mgr.y (mgr.14901) 473 : cluster [DBG] pgmap v479: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-08-27T22:14:18.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:18 smithi006 bash[27292]: cluster 2023-08-27T22:14:17.228824+0000 mgr.y (mgr.14901) 473 : cluster [DBG] pgmap v479: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-08-27T22:14:18.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:18 smithi006 bash[33918]: cluster 2023-08-27T22:14:17.228824+0000 mgr.y (mgr.14901) 473 : cluster [DBG] pgmap v479: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-08-27T22:14:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:19 smithi130 bash[27579]: audit 2023-08-27T22:14:18.758758+0000 mon.c (mon.2) 210 : audit [DBG] from='client.? 172.21.15.6:0/1013302482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:19.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:19 smithi006 bash[27292]: audit 2023-08-27T22:14:18.758758+0000 mon.c (mon.2) 210 : audit [DBG] from='client.? 172.21.15.6:0/1013302482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:19.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:19 smithi006 bash[33918]: audit 2023-08-27T22:14:18.758758+0000 mon.c (mon.2) 210 : audit [DBG] from='client.? 172.21.15.6:0/1013302482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:20.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:20 smithi006 bash[33918]: cluster 2023-08-27T22:14:19.229538+0000 mgr.y (mgr.14901) 474 : cluster [DBG] pgmap v480: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-08-27T22:14:20.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:20 smithi006 bash[27292]: cluster 2023-08-27T22:14:19.229538+0000 mgr.y (mgr.14901) 474 : cluster [DBG] pgmap v480: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-08-27T22:14:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:20 smithi130 bash[27579]: cluster 2023-08-27T22:14:19.229538+0000 mgr.y (mgr.14901) 474 : cluster [DBG] pgmap v480: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-08-27T22:14:21.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:14:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:14:20] "GET /metrics HTTP/1.1" 200 195437 "" "Prometheus/2.33.4" 2023-08-27T22:14:21.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:14:21.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:21.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:14:21.697Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:22.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:22 smithi006 bash[27292]: cluster 2023-08-27T22:14:21.230205+0000 mgr.y (mgr.14901) 475 : cluster [DBG] pgmap v481: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2023-08-27T22:14:22.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:22 smithi006 bash[27292]: audit 2023-08-27T22:14:21.433708+0000 mon.b (mon.1) 167 : audit [DBG] from='client.? 172.21.15.6:0/1083241764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:22.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:22 smithi006 bash[33918]: cluster 2023-08-27T22:14:21.230205+0000 mgr.y (mgr.14901) 475 : cluster [DBG] pgmap v481: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2023-08-27T22:14:22.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:22 smithi006 bash[33918]: audit 2023-08-27T22:14:21.433708+0000 mon.b (mon.1) 167 : audit [DBG] from='client.? 172.21.15.6:0/1083241764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:22 smithi130 bash[27579]: cluster 2023-08-27T22:14:21.230205+0000 mgr.y (mgr.14901) 475 : cluster [DBG] pgmap v481: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2023-08-27T22:14:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:22 smithi130 bash[27579]: audit 2023-08-27T22:14:21.433708+0000 mon.b (mon.1) 167 : audit [DBG] from='client.? 172.21.15.6:0/1083241764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:23.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:14:23.597Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:23.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:14:23.599Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:24.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:24 smithi006 bash[27292]: cluster 2023-08-27T22:14:23.230648+0000 mgr.y (mgr.14901) 476 : cluster [DBG] pgmap v482: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-08-27T22:14:24.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:24 smithi006 bash[27292]: audit 2023-08-27T22:14:24.093470+0000 mon.a (mon.0) 1068 : audit [DBG] from='client.? 172.21.15.6:0/3497832895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:24.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:24 smithi006 bash[33918]: cluster 2023-08-27T22:14:23.230648+0000 mgr.y (mgr.14901) 476 : cluster [DBG] pgmap v482: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-08-27T22:14:24.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:24 smithi006 bash[33918]: audit 2023-08-27T22:14:24.093470+0000 mon.a (mon.0) 1068 : audit [DBG] from='client.? 172.21.15.6:0/3497832895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:24 smithi130 bash[27579]: cluster 2023-08-27T22:14:23.230648+0000 mgr.y (mgr.14901) 476 : cluster [DBG] pgmap v482: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-08-27T22:14:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:24 smithi130 bash[27579]: audit 2023-08-27T22:14:24.093470+0000 mon.a (mon.0) 1068 : audit [DBG] from='client.? 172.21.15.6:0/3497832895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:26.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:26 smithi006 bash[27292]: cluster 2023-08-27T22:14:25.231367+0000 mgr.y (mgr.14901) 477 : cluster [DBG] pgmap v483: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2023-08-27T22:14:26.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:26 smithi006 bash[33918]: cluster 2023-08-27T22:14:25.231367+0000 mgr.y (mgr.14901) 477 : cluster [DBG] pgmap v483: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2023-08-27T22:14:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:26 smithi130 bash[27579]: cluster 2023-08-27T22:14:25.231367+0000 mgr.y (mgr.14901) 477 : cluster [DBG] pgmap v483: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2023-08-27T22:14:27.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:27 smithi006 bash[27292]: audit 2023-08-27T22:14:26.788946+0000 mon.c (mon.2) 211 : audit [DBG] from='client.? 172.21.15.6:0/1278427980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:27.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:27 smithi006 bash[33918]: audit 2023-08-27T22:14:26.788946+0000 mon.c (mon.2) 211 : audit [DBG] from='client.? 172.21.15.6:0/1278427980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:27 smithi130 bash[27579]: audit 2023-08-27T22:14:26.788946+0000 mon.c (mon.2) 211 : audit [DBG] from='client.? 172.21.15.6:0/1278427980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:28.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:28 smithi006 bash[27292]: cluster 2023-08-27T22:14:27.231855+0000 mgr.y (mgr.14901) 478 : cluster [DBG] pgmap v484: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-08-27T22:14:28.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:28 smithi006 bash[33918]: cluster 2023-08-27T22:14:27.231855+0000 mgr.y (mgr.14901) 478 : cluster [DBG] pgmap v484: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-08-27T22:14:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:28 smithi130 bash[27579]: cluster 2023-08-27T22:14:27.231855+0000 mgr.y (mgr.14901) 478 : cluster [DBG] pgmap v484: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-08-27T22:14:30.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:30 smithi006 bash[27292]: cluster 2023-08-27T22:14:29.232446+0000 mgr.y (mgr.14901) 479 : cluster [DBG] pgmap v485: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-08-27T22:14:30.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:30 smithi006 bash[27292]: audit 2023-08-27T22:14:29.499260+0000 mon.a (mon.0) 1069 : audit [DBG] from='client.? 172.21.15.6:0/1336629070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:30.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:30 smithi006 bash[33918]: cluster 2023-08-27T22:14:29.232446+0000 mgr.y (mgr.14901) 479 : cluster [DBG] pgmap v485: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-08-27T22:14:30.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:30 smithi006 bash[33918]: audit 2023-08-27T22:14:29.499260+0000 mon.a (mon.0) 1069 : audit [DBG] from='client.? 172.21.15.6:0/1336629070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:30 smithi130 bash[27579]: cluster 2023-08-27T22:14:29.232446+0000 mgr.y (mgr.14901) 479 : cluster [DBG] pgmap v485: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-08-27T22:14:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:30 smithi130 bash[27579]: audit 2023-08-27T22:14:29.499260+0000 mon.a (mon.0) 1069 : audit [DBG] from='client.? 172.21.15.6:0/1336629070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:31.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:14:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:14:30] "GET /metrics HTTP/1.1" 200 195434 "" "Prometheus/2.33.4" 2023-08-27T22:14:31.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:14:31.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:31.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:14:31.697Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:32.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:32 smithi006 bash[27292]: cluster 2023-08-27T22:14:31.233214+0000 mgr.y (mgr.14901) 480 : cluster [DBG] pgmap v486: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2023-08-27T22:14:32.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:32 smithi006 bash[27292]: audit 2023-08-27T22:14:32.195695+0000 mon.a (mon.0) 1070 : audit [DBG] from='client.? 172.21.15.6:0/1834152990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:32.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:32 smithi006 bash[33918]: cluster 2023-08-27T22:14:31.233214+0000 mgr.y (mgr.14901) 480 : cluster [DBG] pgmap v486: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2023-08-27T22:14:32.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:32 smithi006 bash[33918]: audit 2023-08-27T22:14:32.195695+0000 mon.a (mon.0) 1070 : audit [DBG] from='client.? 172.21.15.6:0/1834152990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:32 smithi130 bash[27579]: cluster 2023-08-27T22:14:31.233214+0000 mgr.y (mgr.14901) 480 : cluster [DBG] pgmap v486: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2023-08-27T22:14:32.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:32 smithi130 bash[27579]: audit 2023-08-27T22:14:32.195695+0000 mon.a (mon.0) 1070 : audit [DBG] from='client.? 172.21.15.6:0/1834152990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:33.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:14:33.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:33.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:14:33.599Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:34.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:34 smithi006 bash[33918]: cluster 2023-08-27T22:14:33.233693+0000 mgr.y (mgr.14901) 481 : cluster [DBG] pgmap v487: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-08-27T22:14:34.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:34 smithi006 bash[27292]: cluster 2023-08-27T22:14:33.233693+0000 mgr.y (mgr.14901) 481 : cluster [DBG] pgmap v487: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-08-27T22:14:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:34 smithi130 bash[27579]: cluster 2023-08-27T22:14:33.233693+0000 mgr.y (mgr.14901) 481 : cluster [DBG] pgmap v487: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-08-27T22:14:35.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:35 smithi006 bash[27292]: audit 2023-08-27T22:14:34.857086+0000 mon.a (mon.0) 1071 : audit [DBG] from='client.? 172.21.15.6:0/1163327247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:35.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:35 smithi006 bash[33918]: audit 2023-08-27T22:14:34.857086+0000 mon.a (mon.0) 1071 : audit [DBG] from='client.? 172.21.15.6:0/1163327247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:35.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:35 smithi130 bash[27579]: audit 2023-08-27T22:14:34.857086+0000 mon.a (mon.0) 1071 : audit [DBG] from='client.? 172.21.15.6:0/1163327247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:36.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:36 smithi006 bash[27292]: cluster 2023-08-27T22:14:35.234457+0000 mgr.y (mgr.14901) 482 : cluster [DBG] pgmap v488: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2023-08-27T22:14:36.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:36 smithi006 bash[33918]: cluster 2023-08-27T22:14:35.234457+0000 mgr.y (mgr.14901) 482 : cluster [DBG] pgmap v488: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2023-08-27T22:14:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:36 smithi130 bash[27579]: cluster 2023-08-27T22:14:35.234457+0000 mgr.y (mgr.14901) 482 : cluster [DBG] pgmap v488: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2023-08-27T22:14:38.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:38 smithi006 bash[27292]: cluster 2023-08-27T22:14:37.234912+0000 mgr.y (mgr.14901) 483 : cluster [DBG] pgmap v489: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-08-27T22:14:38.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:38 smithi006 bash[27292]: audit 2023-08-27T22:14:37.519777+0000 mon.a (mon.0) 1072 : audit [DBG] from='client.? 172.21.15.6:0/760191469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:38.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:38 smithi006 bash[33918]: cluster 2023-08-27T22:14:37.234912+0000 mgr.y (mgr.14901) 483 : cluster [DBG] pgmap v489: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-08-27T22:14:38.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:38 smithi006 bash[33918]: audit 2023-08-27T22:14:37.519777+0000 mon.a (mon.0) 1072 : audit [DBG] from='client.? 172.21.15.6:0/760191469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:38 smithi130 bash[27579]: cluster 2023-08-27T22:14:37.234912+0000 mgr.y (mgr.14901) 483 : cluster [DBG] pgmap v489: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-08-27T22:14:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:38 smithi130 bash[27579]: audit 2023-08-27T22:14:37.519777+0000 mon.a (mon.0) 1072 : audit [DBG] from='client.? 172.21.15.6:0/760191469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:40.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:40 smithi006 bash[27292]: cluster 2023-08-27T22:14:39.235538+0000 mgr.y (mgr.14901) 484 : cluster [DBG] pgmap v490: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-08-27T22:14:40.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:40 smithi006 bash[27292]: audit 2023-08-27T22:14:40.187524+0000 mon.a (mon.0) 1073 : audit [DBG] from='client.? 172.21.15.6:0/3810448486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:40.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:40 smithi006 bash[33918]: cluster 2023-08-27T22:14:39.235538+0000 mgr.y (mgr.14901) 484 : cluster [DBG] pgmap v490: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-08-27T22:14:40.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:40 smithi006 bash[33918]: audit 2023-08-27T22:14:40.187524+0000 mon.a (mon.0) 1073 : audit [DBG] from='client.? 172.21.15.6:0/3810448486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:40 smithi130 bash[27579]: cluster 2023-08-27T22:14:39.235538+0000 mgr.y (mgr.14901) 484 : cluster [DBG] pgmap v490: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-08-27T22:14:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:40 smithi130 bash[27579]: audit 2023-08-27T22:14:40.187524+0000 mon.a (mon.0) 1073 : audit [DBG] from='client.? 172.21.15.6:0/3810448486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:41.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:14:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:14:40] "GET /metrics HTTP/1.1" 200 195434 "" "Prometheus/2.33.4" 2023-08-27T22:14:41.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:14:41.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:41.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:14:41.702Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:42.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:42 smithi006 bash[27292]: cluster 2023-08-27T22:14:41.236265+0000 mgr.y (mgr.14901) 485 : cluster [DBG] pgmap v491: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2023-08-27T22:14:42.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:42 smithi006 bash[33918]: cluster 2023-08-27T22:14:41.236265+0000 mgr.y (mgr.14901) 485 : cluster [DBG] pgmap v491: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2023-08-27T22:14:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:42 smithi130 bash[27579]: cluster 2023-08-27T22:14:41.236265+0000 mgr.y (mgr.14901) 485 : cluster [DBG] pgmap v491: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2023-08-27T22:14:43.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:43 smithi130 bash[27579]: audit 2023-08-27T22:14:42.859315+0000 mon.c (mon.2) 212 : audit [DBG] from='client.? 172.21.15.6:0/86440242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:43.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:14:43.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:43.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:14:43.600Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:43.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:43 smithi006 bash[33918]: audit 2023-08-27T22:14:42.859315+0000 mon.c (mon.2) 212 : audit [DBG] from='client.? 172.21.15.6:0/86440242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:43.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:43 smithi006 bash[27292]: audit 2023-08-27T22:14:42.859315+0000 mon.c (mon.2) 212 : audit [DBG] from='client.? 172.21.15.6:0/86440242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:44 smithi130 bash[27579]: cluster 2023-08-27T22:14:43.236731+0000 mgr.y (mgr.14901) 486 : cluster [DBG] pgmap v492: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-08-27T22:14:44.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:44 smithi006 bash[33918]: cluster 2023-08-27T22:14:43.236731+0000 mgr.y (mgr.14901) 486 : cluster [DBG] pgmap v492: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-08-27T22:14:44.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:44 smithi006 bash[27292]: cluster 2023-08-27T22:14:43.236731+0000 mgr.y (mgr.14901) 486 : cluster [DBG] pgmap v492: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-08-27T22:14:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:46 smithi130 bash[27579]: cluster 2023-08-27T22:14:45.237427+0000 mgr.y (mgr.14901) 487 : cluster [DBG] pgmap v493: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2023-08-27T22:14:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:46 smithi130 bash[27579]: audit 2023-08-27T22:14:45.527983+0000 mon.a (mon.0) 1074 : audit [DBG] from='client.? 172.21.15.6:0/2990536262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:46.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:46 smithi006 bash[33918]: cluster 2023-08-27T22:14:45.237427+0000 mgr.y (mgr.14901) 487 : cluster [DBG] pgmap v493: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2023-08-27T22:14:46.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:46 smithi006 bash[33918]: audit 2023-08-27T22:14:45.527983+0000 mon.a (mon.0) 1074 : audit [DBG] from='client.? 172.21.15.6:0/2990536262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:46.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:46 smithi006 bash[27292]: cluster 2023-08-27T22:14:45.237427+0000 mgr.y (mgr.14901) 487 : cluster [DBG] pgmap v493: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2023-08-27T22:14:46.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:46 smithi006 bash[27292]: audit 2023-08-27T22:14:45.527983+0000 mon.a (mon.0) 1074 : audit [DBG] from='client.? 172.21.15.6:0/2990536262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:48 smithi130 bash[27579]: cluster 2023-08-27T22:14:47.237958+0000 mgr.y (mgr.14901) 488 : cluster [DBG] pgmap v494: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2023-08-27T22:14:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:48 smithi130 bash[27579]: audit 2023-08-27T22:14:48.182044+0000 mon.c (mon.2) 213 : audit [DBG] from='client.? 172.21.15.6:0/4209010083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:48.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:48 smithi006 bash[27292]: cluster 2023-08-27T22:14:47.237958+0000 mgr.y (mgr.14901) 488 : cluster [DBG] pgmap v494: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2023-08-27T22:14:48.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:48 smithi006 bash[27292]: audit 2023-08-27T22:14:48.182044+0000 mon.c (mon.2) 213 : audit [DBG] from='client.? 172.21.15.6:0/4209010083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:48.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:48 smithi006 bash[33918]: cluster 2023-08-27T22:14:47.237958+0000 mgr.y (mgr.14901) 488 : cluster [DBG] pgmap v494: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2023-08-27T22:14:48.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:48 smithi006 bash[33918]: audit 2023-08-27T22:14:48.182044+0000 mon.c (mon.2) 213 : audit [DBG] from='client.? 172.21.15.6:0/4209010083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:50.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:50 smithi006 bash[27292]: cluster 2023-08-27T22:14:49.238591+0000 mgr.y (mgr.14901) 489 : cluster [DBG] pgmap v495: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2023-08-27T22:14:50.756 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:14:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:14:50] "GET /metrics HTTP/1.1" 200 195430 "" "Prometheus/2.33.4" 2023-08-27T22:14:50.756 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:50 smithi006 bash[33918]: cluster 2023-08-27T22:14:49.238591+0000 mgr.y (mgr.14901) 489 : cluster [DBG] pgmap v495: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2023-08-27T22:14:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:50 smithi130 bash[27579]: cluster 2023-08-27T22:14:49.238591+0000 mgr.y (mgr.14901) 489 : cluster [DBG] pgmap v495: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2023-08-27T22:14:51.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:51 smithi130 bash[27579]: audit 2023-08-27T22:14:50.832267+0000 mon.c (mon.2) 214 : audit [DBG] from='client.? 172.21.15.6:0/3827850707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:51.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:51 smithi006 bash[27292]: audit 2023-08-27T22:14:50.832267+0000 mon.c (mon.2) 214 : audit [DBG] from='client.? 172.21.15.6:0/3827850707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:51.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:51 smithi006 bash[33918]: audit 2023-08-27T22:14:50.832267+0000 mon.c (mon.2) 214 : audit [DBG] from='client.? 172.21.15.6:0/3827850707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:51.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:14:51.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:51.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:14:51.701Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:52 smithi130 bash[27579]: cluster 2023-08-27T22:14:51.239274+0000 mgr.y (mgr.14901) 490 : cluster [DBG] pgmap v496: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2023-08-27T22:14:52.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:52 smithi006 bash[27292]: cluster 2023-08-27T22:14:51.239274+0000 mgr.y (mgr.14901) 490 : cluster [DBG] pgmap v496: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2023-08-27T22:14:52.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:52 smithi006 bash[33918]: cluster 2023-08-27T22:14:51.239274+0000 mgr.y (mgr.14901) 490 : cluster [DBG] pgmap v496: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2023-08-27T22:14:53.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:14:53.599Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:53.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:14:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:14:53.600Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:14:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:54 smithi130 bash[27579]: cluster 2023-08-27T22:14:53.239724+0000 mgr.y (mgr.14901) 491 : cluster [DBG] pgmap v497: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-08-27T22:14:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:54 smithi130 bash[27579]: audit 2023-08-27T22:14:53.520631+0000 mon.a (mon.0) 1075 : audit [DBG] from='client.? 172.21.15.6:0/985566455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:54.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:54 smithi006 bash[27292]: cluster 2023-08-27T22:14:53.239724+0000 mgr.y (mgr.14901) 491 : cluster [DBG] pgmap v497: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-08-27T22:14:54.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:54 smithi006 bash[27292]: audit 2023-08-27T22:14:53.520631+0000 mon.a (mon.0) 1075 : audit [DBG] from='client.? 172.21.15.6:0/985566455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:54.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:54 smithi006 bash[33918]: cluster 2023-08-27T22:14:53.239724+0000 mgr.y (mgr.14901) 491 : cluster [DBG] pgmap v497: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-08-27T22:14:54.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:54 smithi006 bash[33918]: audit 2023-08-27T22:14:53.520631+0000 mon.a (mon.0) 1075 : audit [DBG] from='client.? 172.21.15.6:0/985566455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:56 smithi130 bash[27579]: cluster 2023-08-27T22:14:55.240439+0000 mgr.y (mgr.14901) 492 : cluster [DBG] pgmap v498: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 817 op/s 2023-08-27T22:14:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:56 smithi130 bash[27579]: audit 2023-08-27T22:14:56.227839+0000 mon.a (mon.0) 1076 : audit [DBG] from='client.? 172.21.15.6:0/3320258696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:56.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:56 smithi006 bash[33918]: cluster 2023-08-27T22:14:55.240439+0000 mgr.y (mgr.14901) 492 : cluster [DBG] pgmap v498: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 817 op/s 2023-08-27T22:14:56.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:56 smithi006 bash[33918]: audit 2023-08-27T22:14:56.227839+0000 mon.a (mon.0) 1076 : audit [DBG] from='client.? 172.21.15.6:0/3320258696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:56.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:56 smithi006 bash[27292]: cluster 2023-08-27T22:14:55.240439+0000 mgr.y (mgr.14901) 492 : cluster [DBG] pgmap v498: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 817 op/s 2023-08-27T22:14:56.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:56 smithi006 bash[27292]: audit 2023-08-27T22:14:56.227839+0000 mon.a (mon.0) 1076 : audit [DBG] from='client.? 172.21.15.6:0/3320258696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:58 smithi130 bash[27579]: cluster 2023-08-27T22:14:57.240941+0000 mgr.y (mgr.14901) 493 : cluster [DBG] pgmap v499: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-08-27T22:14:58.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:58 smithi006 bash[27292]: cluster 2023-08-27T22:14:57.240941+0000 mgr.y (mgr.14901) 493 : cluster [DBG] pgmap v499: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-08-27T22:14:58.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:58 smithi006 bash[33918]: cluster 2023-08-27T22:14:57.240941+0000 mgr.y (mgr.14901) 493 : cluster [DBG] pgmap v499: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-08-27T22:14:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:14:59 smithi130 bash[27579]: audit 2023-08-27T22:14:58.951389+0000 mon.a (mon.0) 1077 : audit [DBG] from='client.? 172.21.15.6:0/736675606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:59.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:14:59 smithi006 bash[27292]: audit 2023-08-27T22:14:58.951389+0000 mon.a (mon.0) 1077 : audit [DBG] from='client.? 172.21.15.6:0/736675606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:59.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:14:59 smithi006 bash[33918]: audit 2023-08-27T22:14:58.951389+0000 mon.a (mon.0) 1077 : audit [DBG] from='client.? 172.21.15.6:0/736675606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:00 smithi130 bash[27579]: cluster 2023-08-27T22:14:59.241499+0000 mgr.y (mgr.14901) 494 : cluster [DBG] pgmap v500: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-08-27T22:15:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:00 smithi130 bash[27579]: audit 2023-08-27T22:14:59.540538+0000 mon.b (mon.1) 168 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:15:00.969 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:15:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:15:00] "GET /metrics HTTP/1.1" 200 195443 "" "Prometheus/2.33.4" 2023-08-27T22:15:00.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:00 smithi006 bash[27292]: cluster 2023-08-27T22:14:59.241499+0000 mgr.y (mgr.14901) 494 : cluster [DBG] pgmap v500: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-08-27T22:15:00.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:00 smithi006 bash[27292]: audit 2023-08-27T22:14:59.540538+0000 mon.b (mon.1) 168 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:15:00.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:00 smithi006 bash[33918]: cluster 2023-08-27T22:14:59.241499+0000 mgr.y (mgr.14901) 494 : cluster [DBG] pgmap v500: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-08-27T22:15:00.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:00 smithi006 bash[33918]: audit 2023-08-27T22:14:59.540538+0000 mon.b (mon.1) 168 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:15:01.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:15:01.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:01.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:15:01.711Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:02 smithi130 bash[27579]: cluster 2023-08-27T22:15:01.242203+0000 mgr.y (mgr.14901) 495 : cluster [DBG] pgmap v501: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2023-08-27T22:15:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:02 smithi130 bash[27579]: audit 2023-08-27T22:15:01.617495+0000 mon.a (mon.0) 1078 : audit [DBG] from='client.? 172.21.15.6:0/2243113594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:02.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:02 smithi006 bash[33918]: cluster 2023-08-27T22:15:01.242203+0000 mgr.y (mgr.14901) 495 : cluster [DBG] pgmap v501: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2023-08-27T22:15:02.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:02 smithi006 bash[33918]: audit 2023-08-27T22:15:01.617495+0000 mon.a (mon.0) 1078 : audit [DBG] from='client.? 172.21.15.6:0/2243113594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:02.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:02 smithi006 bash[27292]: cluster 2023-08-27T22:15:01.242203+0000 mgr.y (mgr.14901) 495 : cluster [DBG] pgmap v501: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2023-08-27T22:15:02.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:02 smithi006 bash[27292]: audit 2023-08-27T22:15:01.617495+0000 mon.a (mon.0) 1078 : audit [DBG] from='client.? 172.21.15.6:0/2243113594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:03.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:15:03.599Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:04.268 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:15:03.601Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:04.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:04 smithi130 bash[27579]: cluster 2023-08-27T22:15:03.242669+0000 mgr.y (mgr.14901) 496 : cluster [DBG] pgmap v502: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-08-27T22:15:04.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:04 smithi130 bash[27579]: audit 2023-08-27T22:15:04.284277+0000 mon.b (mon.1) 169 : audit [DBG] from='client.? 172.21.15.6:0/2060318541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:04.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:04 smithi006 bash[33918]: cluster 2023-08-27T22:15:03.242669+0000 mgr.y (mgr.14901) 496 : cluster [DBG] pgmap v502: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-08-27T22:15:04.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:04 smithi006 bash[33918]: audit 2023-08-27T22:15:04.284277+0000 mon.b (mon.1) 169 : audit [DBG] from='client.? 172.21.15.6:0/2060318541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:04.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:04 smithi006 bash[27292]: cluster 2023-08-27T22:15:03.242669+0000 mgr.y (mgr.14901) 496 : cluster [DBG] pgmap v502: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-08-27T22:15:04.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:04 smithi006 bash[27292]: audit 2023-08-27T22:15:04.284277+0000 mon.b (mon.1) 169 : audit [DBG] from='client.? 172.21.15.6:0/2060318541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:06.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:06 smithi006 bash[27292]: cluster 2023-08-27T22:15:05.243422+0000 mgr.y (mgr.14901) 497 : cluster [DBG] pgmap v503: 89 pgs: 89 active+clean; 2.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-08-27T22:15:06.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:06 smithi006 bash[27292]: audit 2023-08-27T22:15:05.638815+0000 mon.a (mon.0) 1079 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:15:06.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:06 smithi006 bash[27292]: audit 2023-08-27T22:15:06.079230+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:15:06.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:06 smithi006 bash[27292]: audit 2023-08-27T22:15:06.418726+0000 mon.b (mon.1) 170 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:15:06.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:06 smithi006 bash[27292]: audit 2023-08-27T22:15:06.419778+0000 mon.b (mon.1) 171 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:15:06.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:06 smithi006 bash[27292]: audit 2023-08-27T22:15:06.427283+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:15:06.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:06 smithi006 bash[33918]: cluster 2023-08-27T22:15:05.243422+0000 mgr.y (mgr.14901) 497 : cluster [DBG] pgmap v503: 89 pgs: 89 active+clean; 2.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-08-27T22:15:06.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:06 smithi006 bash[33918]: audit 2023-08-27T22:15:05.638815+0000 mon.a (mon.0) 1079 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:15:06.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:06 smithi006 bash[33918]: audit 2023-08-27T22:15:06.079230+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:15:06.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:06 smithi006 bash[33918]: audit 2023-08-27T22:15:06.418726+0000 mon.b (mon.1) 170 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:15:06.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:06 smithi006 bash[33918]: audit 2023-08-27T22:15:06.419778+0000 mon.b (mon.1) 171 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:15:06.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:06 smithi006 bash[33918]: audit 2023-08-27T22:15:06.427283+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:15:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:06 smithi130 bash[27579]: cluster 2023-08-27T22:15:05.243422+0000 mgr.y (mgr.14901) 497 : cluster [DBG] pgmap v503: 89 pgs: 89 active+clean; 2.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-08-27T22:15:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:06 smithi130 bash[27579]: audit 2023-08-27T22:15:05.638815+0000 mon.a (mon.0) 1079 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:15:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:06 smithi130 bash[27579]: audit 2023-08-27T22:15:06.079230+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:15:07.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:06 smithi130 bash[27579]: audit 2023-08-27T22:15:06.418726+0000 mon.b (mon.1) 170 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:15:07.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:06 smithi130 bash[27579]: audit 2023-08-27T22:15:06.419778+0000 mon.b (mon.1) 171 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:15:07.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:06 smithi130 bash[27579]: audit 2023-08-27T22:15:06.427283+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:15:07.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:07 smithi006 bash[33918]: audit 2023-08-27T22:15:06.969497+0000 mon.a (mon.0) 1082 : audit [DBG] from='client.? 172.21.15.6:0/143900646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:07.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:07 smithi006 bash[27292]: audit 2023-08-27T22:15:06.969497+0000 mon.a (mon.0) 1082 : audit [DBG] from='client.? 172.21.15.6:0/143900646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:08.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:07 smithi130 bash[27579]: audit 2023-08-27T22:15:06.969497+0000 mon.a (mon.0) 1082 : audit [DBG] from='client.? 172.21.15.6:0/143900646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:08.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:08 smithi006 bash[27292]: cluster 2023-08-27T22:15:07.243999+0000 mgr.y (mgr.14901) 498 : cluster [DBG] pgmap v504: 89 pgs: 89 active+clean; 2.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-08-27T22:15:08.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:08 smithi006 bash[33918]: cluster 2023-08-27T22:15:07.243999+0000 mgr.y (mgr.14901) 498 : cluster [DBG] pgmap v504: 89 pgs: 89 active+clean; 2.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-08-27T22:15:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:08 smithi130 bash[27579]: cluster 2023-08-27T22:15:07.243999+0000 mgr.y (mgr.14901) 498 : cluster [DBG] pgmap v504: 89 pgs: 89 active+clean; 2.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-08-27T22:15:09.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:09 smithi006 bash[33918]: audit 2023-08-27T22:15:09.620000+0000 mon.b (mon.1) 172 : audit [DBG] from='client.? 172.21.15.6:0/4234360113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:09.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:09 smithi006 bash[27292]: audit 2023-08-27T22:15:09.620000+0000 mon.b (mon.1) 172 : audit [DBG] from='client.? 172.21.15.6:0/4234360113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:10.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:09 smithi130 bash[27579]: audit 2023-08-27T22:15:09.620000+0000 mon.b (mon.1) 172 : audit [DBG] from='client.? 172.21.15.6:0/4234360113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:10.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:15:10.143Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:10.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:10 smithi006 bash[27292]: cluster 2023-08-27T22:15:09.244579+0000 mgr.y (mgr.14901) 499 : cluster [DBG] pgmap v505: 89 pgs: 89 active+clean; 2.0 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-08-27T22:15:10.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:15:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:15:10] "GET /metrics HTTP/1.1" 200 195443 "" "Prometheus/2.33.4" 2023-08-27T22:15:10.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:10 smithi006 bash[33918]: cluster 2023-08-27T22:15:09.244579+0000 mgr.y (mgr.14901) 499 : cluster [DBG] pgmap v505: 89 pgs: 89 active+clean; 2.0 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-08-27T22:15:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:10 smithi130 bash[27579]: cluster 2023-08-27T22:15:09.244579+0000 mgr.y (mgr.14901) 499 : cluster [DBG] pgmap v505: 89 pgs: 89 active+clean; 2.0 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-08-27T22:15:11.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:15:11.705Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:11.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:15:11.707Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:12.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:12 smithi006 bash[33918]: cluster 2023-08-27T22:15:11.245261+0000 mgr.y (mgr.14901) 500 : cluster [DBG] pgmap v506: 89 pgs: 89 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 774 op/s 2023-08-27T22:15:12.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:12 smithi006 bash[33918]: audit 2023-08-27T22:15:12.289643+0000 mon.a (mon.0) 1083 : audit [DBG] from='client.? 172.21.15.6:0/2855662105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:12.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:12 smithi006 bash[27292]: cluster 2023-08-27T22:15:11.245261+0000 mgr.y (mgr.14901) 500 : cluster [DBG] pgmap v506: 89 pgs: 89 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 774 op/s 2023-08-27T22:15:12.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:12 smithi006 bash[27292]: audit 2023-08-27T22:15:12.289643+0000 mon.a (mon.0) 1083 : audit [DBG] from='client.? 172.21.15.6:0/2855662105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:12 smithi130 bash[27579]: cluster 2023-08-27T22:15:11.245261+0000 mgr.y (mgr.14901) 500 : cluster [DBG] pgmap v506: 89 pgs: 89 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 774 op/s 2023-08-27T22:15:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:12 smithi130 bash[27579]: audit 2023-08-27T22:15:12.289643+0000 mon.a (mon.0) 1083 : audit [DBG] from='client.? 172.21.15.6:0/2855662105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:13.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:15:13.600Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:13.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:15:13.601Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:14.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:14 smithi006 bash[27292]: cluster 2023-08-27T22:15:13.245725+0000 mgr.y (mgr.14901) 501 : cluster [DBG] pgmap v507: 89 pgs: 89 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-08-27T22:15:14.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:14 smithi006 bash[33918]: cluster 2023-08-27T22:15:13.245725+0000 mgr.y (mgr.14901) 501 : cluster [DBG] pgmap v507: 89 pgs: 89 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-08-27T22:15:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:14 smithi130 bash[27579]: cluster 2023-08-27T22:15:13.245725+0000 mgr.y (mgr.14901) 501 : cluster [DBG] pgmap v507: 89 pgs: 89 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-08-27T22:15:15.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:15 smithi006 bash[27292]: audit 2023-08-27T22:15:14.963355+0000 mon.b (mon.1) 173 : audit [DBG] from='client.? 172.21.15.6:0/4246265842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:15.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:15 smithi006 bash[33918]: audit 2023-08-27T22:15:14.963355+0000 mon.b (mon.1) 173 : audit [DBG] from='client.? 172.21.15.6:0/4246265842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:16.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:15 smithi130 bash[27579]: audit 2023-08-27T22:15:14.963355+0000 mon.b (mon.1) 173 : audit [DBG] from='client.? 172.21.15.6:0/4246265842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:16.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:16 smithi006 bash[27292]: cluster 2023-08-27T22:15:15.246440+0000 mgr.y (mgr.14901) 502 : cluster [DBG] pgmap v508: 89 pgs: 89 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2023-08-27T22:15:16.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:16 smithi006 bash[33918]: cluster 2023-08-27T22:15:15.246440+0000 mgr.y (mgr.14901) 502 : cluster [DBG] pgmap v508: 89 pgs: 89 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2023-08-27T22:15:17.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:16 smithi130 bash[27579]: cluster 2023-08-27T22:15:15.246440+0000 mgr.y (mgr.14901) 502 : cluster [DBG] pgmap v508: 89 pgs: 89 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2023-08-27T22:15:17.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:17 smithi006 bash[33918]: audit 2023-08-27T22:15:17.604703+0000 mon.a (mon.0) 1084 : audit [DBG] from='client.? 172.21.15.6:0/1333195312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:17.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:17 smithi006 bash[27292]: audit 2023-08-27T22:15:17.604703+0000 mon.a (mon.0) 1084 : audit [DBG] from='client.? 172.21.15.6:0/1333195312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:18.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:17 smithi130 bash[27579]: audit 2023-08-27T22:15:17.604703+0000 mon.a (mon.0) 1084 : audit [DBG] from='client.? 172.21.15.6:0/1333195312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:18 smithi130 bash[27579]: cluster 2023-08-27T22:15:17.246964+0000 mgr.y (mgr.14901) 503 : cluster [DBG] pgmap v509: 89 pgs: 89 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-08-27T22:15:19.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:18 smithi006 bash[27292]: cluster 2023-08-27T22:15:17.246964+0000 mgr.y (mgr.14901) 503 : cluster [DBG] pgmap v509: 89 pgs: 89 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-08-27T22:15:19.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:18 smithi006 bash[33918]: cluster 2023-08-27T22:15:17.246964+0000 mgr.y (mgr.14901) 503 : cluster [DBG] pgmap v509: 89 pgs: 89 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-08-27T22:15:20.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:15:20.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:20.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:15:20.144Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:20 smithi130 bash[27579]: cluster 2023-08-27T22:15:19.247524+0000 mgr.y (mgr.14901) 504 : cluster [DBG] pgmap v510: 89 pgs: 89 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-08-27T22:15:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:20 smithi130 bash[27579]: audit 2023-08-27T22:15:20.275160+0000 mon.b (mon.1) 174 : audit [DBG] from='client.? 172.21.15.6:0/1529544743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:21.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:20 smithi006 bash[27292]: cluster 2023-08-27T22:15:19.247524+0000 mgr.y (mgr.14901) 504 : cluster [DBG] pgmap v510: 89 pgs: 89 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-08-27T22:15:21.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:20 smithi006 bash[27292]: audit 2023-08-27T22:15:20.275160+0000 mon.b (mon.1) 174 : audit [DBG] from='client.? 172.21.15.6:0/1529544743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:21.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:15:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:15:20] "GET /metrics HTTP/1.1" 200 195439 "" "Prometheus/2.33.4" 2023-08-27T22:15:21.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:20 smithi006 bash[33918]: cluster 2023-08-27T22:15:19.247524+0000 mgr.y (mgr.14901) 504 : cluster [DBG] pgmap v510: 89 pgs: 89 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-08-27T22:15:21.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:20 smithi006 bash[33918]: audit 2023-08-27T22:15:20.275160+0000 mon.b (mon.1) 174 : audit [DBG] from='client.? 172.21.15.6:0/1529544743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:21.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:15:21.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:21.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:15:21.708Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:22 smithi130 bash[27579]: cluster 2023-08-27T22:15:21.248213+0000 mgr.y (mgr.14901) 505 : cluster [DBG] pgmap v511: 89 pgs: 89 active+clean; 1.4 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2023-08-27T22:15:23.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:22 smithi006 bash[33918]: cluster 2023-08-27T22:15:21.248213+0000 mgr.y (mgr.14901) 505 : cluster [DBG] pgmap v511: 89 pgs: 89 active+clean; 1.4 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2023-08-27T22:15:23.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:22 smithi006 bash[27292]: cluster 2023-08-27T22:15:21.248213+0000 mgr.y (mgr.14901) 505 : cluster [DBG] pgmap v511: 89 pgs: 89 active+clean; 1.4 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2023-08-27T22:15:23.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:15:23.601Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:23.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:15:23.603Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:23.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:23 smithi006 bash[33918]: audit 2023-08-27T22:15:22.956583+0000 mon.b (mon.1) 175 : audit [DBG] from='client.? 172.21.15.6:0/1369804974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:23.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:23 smithi006 bash[27292]: audit 2023-08-27T22:15:22.956583+0000 mon.b (mon.1) 175 : audit [DBG] from='client.? 172.21.15.6:0/1369804974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:24.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:23 smithi130 bash[27579]: audit 2023-08-27T22:15:22.956583+0000 mon.b (mon.1) 175 : audit [DBG] from='client.? 172.21.15.6:0/1369804974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:24 smithi130 bash[27579]: cluster 2023-08-27T22:15:23.248722+0000 mgr.y (mgr.14901) 506 : cluster [DBG] pgmap v512: 89 pgs: 89 active+clean; 1.4 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-08-27T22:15:25.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:24 smithi006 bash[27292]: cluster 2023-08-27T22:15:23.248722+0000 mgr.y (mgr.14901) 506 : cluster [DBG] pgmap v512: 89 pgs: 89 active+clean; 1.4 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-08-27T22:15:25.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:24 smithi006 bash[33918]: cluster 2023-08-27T22:15:23.248722+0000 mgr.y (mgr.14901) 506 : cluster [DBG] pgmap v512: 89 pgs: 89 active+clean; 1.4 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-08-27T22:15:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:25 smithi130 bash[27579]: audit 2023-08-27T22:15:25.614722+0000 mon.c (mon.2) 215 : audit [DBG] from='client.? 172.21.15.6:0/1416733633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:26.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:25 smithi006 bash[33918]: audit 2023-08-27T22:15:25.614722+0000 mon.c (mon.2) 215 : audit [DBG] from='client.? 172.21.15.6:0/1416733633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:26.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:25 smithi006 bash[27292]: audit 2023-08-27T22:15:25.614722+0000 mon.c (mon.2) 215 : audit [DBG] from='client.? 172.21.15.6:0/1416733633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:26 smithi130 bash[27579]: cluster 2023-08-27T22:15:25.249460+0000 mgr.y (mgr.14901) 507 : cluster [DBG] pgmap v513: 89 pgs: 89 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2023-08-27T22:15:27.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:26 smithi006 bash[27292]: cluster 2023-08-27T22:15:25.249460+0000 mgr.y (mgr.14901) 507 : cluster [DBG] pgmap v513: 89 pgs: 89 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2023-08-27T22:15:27.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:26 smithi006 bash[33918]: cluster 2023-08-27T22:15:25.249460+0000 mgr.y (mgr.14901) 507 : cluster [DBG] pgmap v513: 89 pgs: 89 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2023-08-27T22:15:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:28 smithi130 bash[27579]: cluster 2023-08-27T22:15:27.252982+0000 mgr.y (mgr.14901) 508 : cluster [DBG] pgmap v514: 89 pgs: 89 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-08-27T22:15:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:28 smithi130 bash[27579]: audit 2023-08-27T22:15:28.262926+0000 mon.c (mon.2) 216 : audit [DBG] from='client.? 172.21.15.6:0/3874652074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:29.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:28 smithi006 bash[33918]: cluster 2023-08-27T22:15:27.252982+0000 mgr.y (mgr.14901) 508 : cluster [DBG] pgmap v514: 89 pgs: 89 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-08-27T22:15:29.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:28 smithi006 bash[33918]: audit 2023-08-27T22:15:28.262926+0000 mon.c (mon.2) 216 : audit [DBG] from='client.? 172.21.15.6:0/3874652074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:29.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:28 smithi006 bash[27292]: cluster 2023-08-27T22:15:27.252982+0000 mgr.y (mgr.14901) 508 : cluster [DBG] pgmap v514: 89 pgs: 89 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-08-27T22:15:29.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:28 smithi006 bash[27292]: audit 2023-08-27T22:15:28.262926+0000 mon.c (mon.2) 216 : audit [DBG] from='client.? 172.21.15.6:0/3874652074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:30.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:15:30.143Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:30.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:15:30.145Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:31.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:30 smithi130 bash[27579]: cluster 2023-08-27T22:15:29.254117+0000 mgr.y (mgr.14901) 509 : cluster [DBG] pgmap v515: 89 pgs: 89 active+clean; 1.1 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-08-27T22:15:31.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:30 smithi006 bash[27292]: cluster 2023-08-27T22:15:29.254117+0000 mgr.y (mgr.14901) 509 : cluster [DBG] pgmap v515: 89 pgs: 89 active+clean; 1.1 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-08-27T22:15:31.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:15:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:15:30] "GET /metrics HTTP/1.1" 200 195431 "" "Prometheus/2.33.4" 2023-08-27T22:15:31.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:30 smithi006 bash[33918]: cluster 2023-08-27T22:15:29.254117+0000 mgr.y (mgr.14901) 509 : cluster [DBG] pgmap v515: 89 pgs: 89 active+clean; 1.1 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-08-27T22:15:31.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:15:31.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:31.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:15:31.709Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:31.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:31 smithi006 bash[33918]: audit 2023-08-27T22:15:30.955210+0000 mon.b (mon.1) 176 : audit [DBG] from='client.? 172.21.15.6:0/2839964580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:31.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:31 smithi006 bash[27292]: audit 2023-08-27T22:15:30.955210+0000 mon.b (mon.1) 176 : audit [DBG] from='client.? 172.21.15.6:0/2839964580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:32.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:31 smithi130 bash[27579]: audit 2023-08-27T22:15:30.955210+0000 mon.b (mon.1) 176 : audit [DBG] from='client.? 172.21.15.6:0/2839964580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:32 smithi130 bash[27579]: cluster 2023-08-27T22:15:31.254831+0000 mgr.y (mgr.14901) 510 : cluster [DBG] pgmap v516: 89 pgs: 89 active+clean; 974 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2023-08-27T22:15:33.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:32 smithi006 bash[33918]: cluster 2023-08-27T22:15:31.254831+0000 mgr.y (mgr.14901) 510 : cluster [DBG] pgmap v516: 89 pgs: 89 active+clean; 974 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2023-08-27T22:15:33.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:32 smithi006 bash[27292]: cluster 2023-08-27T22:15:31.254831+0000 mgr.y (mgr.14901) 510 : cluster [DBG] pgmap v516: 89 pgs: 89 active+clean; 974 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2023-08-27T22:15:33.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:15:33.602Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:33.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:15:33.603Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:34.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:34 smithi006 bash[33918]: cluster 2023-08-27T22:15:33.255287+0000 mgr.y (mgr.14901) 511 : cluster [DBG] pgmap v517: 89 pgs: 89 active+clean; 974 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-08-27T22:15:35.806 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:34 smithi006 bash[33918]: audit 2023-08-27T22:15:33.621694+0000 mon.a (mon.0) 1085 : audit [DBG] from='client.? 172.21.15.6:0/3925542370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:35.806 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:34 smithi006 bash[27292]: cluster 2023-08-27T22:15:33.255287+0000 mgr.y (mgr.14901) 511 : cluster [DBG] pgmap v517: 89 pgs: 89 active+clean; 974 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-08-27T22:15:35.806 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:34 smithi006 bash[27292]: audit 2023-08-27T22:15:33.621694+0000 mon.a (mon.0) 1085 : audit [DBG] from='client.? 172.21.15.6:0/3925542370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:35.806 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:34 smithi130 bash[27579]: cluster 2023-08-27T22:15:33.255287+0000 mgr.y (mgr.14901) 511 : cluster [DBG] pgmap v517: 89 pgs: 89 active+clean; 974 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-08-27T22:15:35.807 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:34 smithi130 bash[27579]: audit 2023-08-27T22:15:33.621694+0000 mon.a (mon.0) 1085 : audit [DBG] from='client.? 172.21.15.6:0/3925542370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:36.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:36 smithi006 bash[27292]: cluster 2023-08-27T22:15:35.256031+0000 mgr.y (mgr.14901) 512 : cluster [DBG] pgmap v518: 89 pgs: 89 active+clean; 795 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2023-08-27T22:15:36.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:36 smithi006 bash[27292]: audit 2023-08-27T22:15:36.293902+0000 mon.a (mon.0) 1086 : audit [DBG] from='client.? 172.21.15.6:0/1621882082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:36.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:36 smithi006 bash[33918]: cluster 2023-08-27T22:15:35.256031+0000 mgr.y (mgr.14901) 512 : cluster [DBG] pgmap v518: 89 pgs: 89 active+clean; 795 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2023-08-27T22:15:36.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:36 smithi006 bash[33918]: audit 2023-08-27T22:15:36.293902+0000 mon.a (mon.0) 1086 : audit [DBG] from='client.? 172.21.15.6:0/1621882082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:36 smithi130 bash[27579]: cluster 2023-08-27T22:15:35.256031+0000 mgr.y (mgr.14901) 512 : cluster [DBG] pgmap v518: 89 pgs: 89 active+clean; 795 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2023-08-27T22:15:37.041 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:36 smithi130 bash[27579]: audit 2023-08-27T22:15:36.293902+0000 mon.a (mon.0) 1086 : audit [DBG] from='client.? 172.21.15.6:0/1621882082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:38.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:38 smithi006 bash[33918]: cluster 2023-08-27T22:15:37.256565+0000 mgr.y (mgr.14901) 513 : cluster [DBG] pgmap v519: 89 pgs: 89 active+clean; 754 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-08-27T22:15:38.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:38 smithi006 bash[27292]: cluster 2023-08-27T22:15:37.256565+0000 mgr.y (mgr.14901) 513 : cluster [DBG] pgmap v519: 89 pgs: 89 active+clean; 754 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-08-27T22:15:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:38 smithi130 bash[27579]: cluster 2023-08-27T22:15:37.256565+0000 mgr.y (mgr.14901) 513 : cluster [DBG] pgmap v519: 89 pgs: 89 active+clean; 754 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-08-27T22:15:39.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:39 smithi006 bash[33918]: audit 2023-08-27T22:15:38.987787+0000 mon.b (mon.1) 177 : audit [DBG] from='client.? 172.21.15.6:0/438194207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:39.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:39 smithi006 bash[27292]: audit 2023-08-27T22:15:38.987787+0000 mon.b (mon.1) 177 : audit [DBG] from='client.? 172.21.15.6:0/438194207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:39.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:39 smithi130 bash[27579]: audit 2023-08-27T22:15:38.987787+0000 mon.b (mon.1) 177 : audit [DBG] from='client.? 172.21.15.6:0/438194207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:40.403 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:15:40.143Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:40.404 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:15:40.145Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:40.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:40 smithi006 bash[27292]: cluster 2023-08-27T22:15:39.257143+0000 mgr.y (mgr.14901) 514 : cluster [DBG] pgmap v520: 89 pgs: 89 active+clean; 740 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-08-27T22:15:40.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:40 smithi006 bash[33918]: cluster 2023-08-27T22:15:39.257143+0000 mgr.y (mgr.14901) 514 : cluster [DBG] pgmap v520: 89 pgs: 89 active+clean; 740 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-08-27T22:15:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:40 smithi130 bash[27579]: cluster 2023-08-27T22:15:39.257143+0000 mgr.y (mgr.14901) 514 : cluster [DBG] pgmap v520: 89 pgs: 89 active+clean; 740 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-08-27T22:15:41.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:15:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:15:40] "GET /metrics HTTP/1.1" 200 195431 "" "Prometheus/2.33.4" 2023-08-27T22:15:41.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:15:41.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:41.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:15:41.710Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:42.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:42 smithi006 bash[27292]: cluster 2023-08-27T22:15:41.257853+0000 mgr.y (mgr.14901) 515 : cluster [DBG] pgmap v521: 89 pgs: 89 active+clean; 539 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 846 op/s 2023-08-27T22:15:42.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:42 smithi006 bash[27292]: audit 2023-08-27T22:15:41.664324+0000 mon.a (mon.0) 1087 : audit [DBG] from='client.? 172.21.15.6:0/3350004994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:42.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:42 smithi006 bash[33918]: cluster 2023-08-27T22:15:41.257853+0000 mgr.y (mgr.14901) 515 : cluster [DBG] pgmap v521: 89 pgs: 89 active+clean; 539 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 846 op/s 2023-08-27T22:15:42.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:42 smithi006 bash[33918]: audit 2023-08-27T22:15:41.664324+0000 mon.a (mon.0) 1087 : audit [DBG] from='client.? 172.21.15.6:0/3350004994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:42 smithi130 bash[27579]: cluster 2023-08-27T22:15:41.257853+0000 mgr.y (mgr.14901) 515 : cluster [DBG] pgmap v521: 89 pgs: 89 active+clean; 539 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 846 op/s 2023-08-27T22:15:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:42 smithi130 bash[27579]: audit 2023-08-27T22:15:41.664324+0000 mon.a (mon.0) 1087 : audit [DBG] from='client.? 172.21.15.6:0/3350004994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:43.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:15:43.605Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:43.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:15:43.606Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:44.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:44 smithi006 bash[27292]: cluster 2023-08-27T22:15:43.258301+0000 mgr.y (mgr.14901) 516 : cluster [DBG] pgmap v522: 89 pgs: 89 active+clean; 539 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-08-27T22:15:44.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:44 smithi006 bash[27292]: audit 2023-08-27T22:15:44.300714+0000 mon.c (mon.2) 217 : audit [DBG] from='client.? 172.21.15.6:0/2581974848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:44.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:44 smithi006 bash[33918]: cluster 2023-08-27T22:15:43.258301+0000 mgr.y (mgr.14901) 516 : cluster [DBG] pgmap v522: 89 pgs: 89 active+clean; 539 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-08-27T22:15:44.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:44 smithi006 bash[33918]: audit 2023-08-27T22:15:44.300714+0000 mon.c (mon.2) 217 : audit [DBG] from='client.? 172.21.15.6:0/2581974848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:44 smithi130 bash[27579]: cluster 2023-08-27T22:15:43.258301+0000 mgr.y (mgr.14901) 516 : cluster [DBG] pgmap v522: 89 pgs: 89 active+clean; 539 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-08-27T22:15:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:44 smithi130 bash[27579]: audit 2023-08-27T22:15:44.300714+0000 mon.c (mon.2) 217 : audit [DBG] from='client.? 172.21.15.6:0/2581974848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:46.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:46 smithi006 bash[27292]: cluster 2023-08-27T22:15:45.258930+0000 mgr.y (mgr.14901) 517 : cluster [DBG] pgmap v523: 89 pgs: 89 active+clean; 367 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2023-08-27T22:15:46.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:46 smithi006 bash[33918]: cluster 2023-08-27T22:15:45.258930+0000 mgr.y (mgr.14901) 517 : cluster [DBG] pgmap v523: 89 pgs: 89 active+clean; 367 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2023-08-27T22:15:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:46 smithi130 bash[27579]: cluster 2023-08-27T22:15:45.258930+0000 mgr.y (mgr.14901) 517 : cluster [DBG] pgmap v523: 89 pgs: 89 active+clean; 367 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2023-08-27T22:15:47.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:47 smithi006 bash[27292]: audit 2023-08-27T22:15:46.989276+0000 mon.a (mon.0) 1088 : audit [DBG] from='client.? 172.21.15.6:0/3406836538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:47.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:47 smithi006 bash[33918]: audit 2023-08-27T22:15:46.989276+0000 mon.a (mon.0) 1088 : audit [DBG] from='client.? 172.21.15.6:0/3406836538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:47.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:47 smithi130 bash[27579]: audit 2023-08-27T22:15:46.989276+0000 mon.a (mon.0) 1088 : audit [DBG] from='client.? 172.21.15.6:0/3406836538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:48.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:48 smithi006 bash[27292]: cluster 2023-08-27T22:15:47.259516+0000 mgr.y (mgr.14901) 518 : cluster [DBG] pgmap v524: 89 pgs: 89 active+clean; 325 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-08-27T22:15:48.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:48 smithi006 bash[33918]: cluster 2023-08-27T22:15:47.259516+0000 mgr.y (mgr.14901) 518 : cluster [DBG] pgmap v524: 89 pgs: 89 active+clean; 325 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-08-27T22:15:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:48 smithi130 bash[27579]: cluster 2023-08-27T22:15:47.259516+0000 mgr.y (mgr.14901) 518 : cluster [DBG] pgmap v524: 89 pgs: 89 active+clean; 325 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-08-27T22:15:50.433 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:15:50.144Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:50.433 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:15:50.146Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:50.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:50 smithi006 bash[27292]: cluster 2023-08-27T22:15:49.260040+0000 mgr.y (mgr.14901) 519 : cluster [DBG] pgmap v525: 89 pgs: 89 active+clean; 310 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-08-27T22:15:50.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:50 smithi006 bash[27292]: audit 2023-08-27T22:15:49.642321+0000 mon.a (mon.0) 1089 : audit [DBG] from='client.? 172.21.15.6:0/1446183958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:50.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:50 smithi006 bash[33918]: cluster 2023-08-27T22:15:49.260040+0000 mgr.y (mgr.14901) 519 : cluster [DBG] pgmap v525: 89 pgs: 89 active+clean; 310 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-08-27T22:15:50.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:50 smithi006 bash[33918]: audit 2023-08-27T22:15:49.642321+0000 mon.a (mon.0) 1089 : audit [DBG] from='client.? 172.21.15.6:0/1446183958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:50 smithi130 bash[27579]: cluster 2023-08-27T22:15:49.260040+0000 mgr.y (mgr.14901) 519 : cluster [DBG] pgmap v525: 89 pgs: 89 active+clean; 310 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-08-27T22:15:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:50 smithi130 bash[27579]: audit 2023-08-27T22:15:49.642321+0000 mon.a (mon.0) 1089 : audit [DBG] from='client.? 172.21.15.6:0/1446183958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:51.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:15:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:15:50] "GET /metrics HTTP/1.1" 200 195429 "" "Prometheus/2.33.4" 2023-08-27T22:15:51.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:15:51.709Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:51.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:15:51.711Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:52.430 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Removed 111254 objects 2023-08-27T22:15:52.430 INFO:tasks.radosbench.radosbench.0.smithi006.stdout:Clean up completed and total clean up time :157.013 2023-08-27T22:15:52.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:52 smithi006 bash[27292]: cluster 2023-08-27T22:15:51.260742+0000 mgr.y (mgr.14901) 520 : cluster [DBG] pgmap v526: 89 pgs: 89 active+clean; 105 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 846 op/s 2023-08-27T22:15:52.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:52 smithi006 bash[27292]: audit 2023-08-27T22:15:52.348766+0000 mon.a (mon.0) 1090 : audit [DBG] from='client.? 172.21.15.6:0/4119148014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:52.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:52 smithi006 bash[33918]: cluster 2023-08-27T22:15:51.260742+0000 mgr.y (mgr.14901) 520 : cluster [DBG] pgmap v526: 89 pgs: 89 active+clean; 105 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 846 op/s 2023-08-27T22:15:52.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:52 smithi006 bash[33918]: audit 2023-08-27T22:15:52.348766+0000 mon.a (mon.0) 1090 : audit [DBG] from='client.? 172.21.15.6:0/4119148014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:52 smithi130 bash[27579]: cluster 2023-08-27T22:15:51.260742+0000 mgr.y (mgr.14901) 520 : cluster [DBG] pgmap v526: 89 pgs: 89 active+clean; 105 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 846 op/s 2023-08-27T22:15:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:52 smithi130 bash[27579]: audit 2023-08-27T22:15:52.348766+0000 mon.a (mon.0) 1090 : audit [DBG] from='client.? 172.21.15.6:0/4119148014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:53.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:15:53.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:53.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:15:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:15:53.608Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:15:54.504 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-08-27T22:15:54.505 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b696f380-4522-11ee-817b-1b7a5f1ba315 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-08-27T22:15:54.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:54 smithi006 bash[27292]: cluster 2023-08-27T22:15:53.261330+0000 mgr.y (mgr.14901) 521 : cluster [DBG] pgmap v527: 89 pgs: 89 active+clean; 105 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-08-27T22:15:54.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:54 smithi006 bash[33918]: cluster 2023-08-27T22:15:53.261330+0000 mgr.y (mgr.14901) 521 : cluster [DBG] pgmap v527: 89 pgs: 89 active+clean; 105 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-08-27T22:15:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:54 smithi130 bash[27579]: cluster 2023-08-27T22:15:53.261330+0000 mgr.y (mgr.14901) 521 : cluster [DBG] pgmap v527: 89 pgs: 89 active+clean; 105 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-08-27T22:15:55.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:55 smithi006 bash[27292]: audit 2023-08-27T22:15:54.777103+0000 mon.a (mon.0) 1091 : audit [DBG] from='client.? 172.21.15.6:0/138783816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:55.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:55 smithi006 bash[33918]: audit 2023-08-27T22:15:54.777103+0000 mon.a (mon.0) 1091 : audit [DBG] from='client.? 172.21.15.6:0/138783816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:55.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:55 smithi130 bash[27579]: audit 2023-08-27T22:15:54.777103+0000 mon.a (mon.0) 1091 : audit [DBG] from='client.? 172.21.15.6:0/138783816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:56.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:56 smithi006 bash[27292]: cluster 2023-08-27T22:15:55.262449+0000 mgr.y (mgr.14901) 522 : cluster [DBG] pgmap v528: 89 pgs: 89 active+clean; 13 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-08-27T22:15:56.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:56 smithi006 bash[33918]: cluster 2023-08-27T22:15:55.262449+0000 mgr.y (mgr.14901) 522 : cluster [DBG] pgmap v528: 89 pgs: 89 active+clean; 13 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-08-27T22:15:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:56 smithi130 bash[27579]: cluster 2023-08-27T22:15:55.262449+0000 mgr.y (mgr.14901) 522 : cluster [DBG] pgmap v528: 89 pgs: 89 active+clean; 13 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-08-27T22:15:57.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:57 smithi006 bash[33918]: audit 2023-08-27T22:15:57.189471+0000 mon.c (mon.2) 218 : audit [DBG] from='client.? 172.21.15.6:0/1241914221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:57.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:57 smithi006 bash[27292]: audit 2023-08-27T22:15:57.189471+0000 mon.c (mon.2) 218 : audit [DBG] from='client.? 172.21.15.6:0/1241914221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:57.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:57 smithi130 bash[27579]: audit 2023-08-27T22:15:57.189471+0000 mon.c (mon.2) 218 : audit [DBG] from='client.? 172.21.15.6:0/1241914221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:15:58 smithi130 bash[27579]: cluster 2023-08-27T22:15:57.263221+0000 mgr.y (mgr.14901) 523 : cluster [DBG] pgmap v529: 89 pgs: 89 active+clean; 3.3 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 489 op/s 2023-08-27T22:15:58.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:15:58 smithi006 bash[33918]: cluster 2023-08-27T22:15:57.263221+0000 mgr.y (mgr.14901) 523 : cluster [DBG] pgmap v529: 89 pgs: 89 active+clean; 3.3 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 489 op/s 2023-08-27T22:15:58.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:15:58 smithi006 bash[27292]: cluster 2023-08-27T22:15:57.263221+0000 mgr.y (mgr.14901) 523 : cluster [DBG] pgmap v529: 89 pgs: 89 active+clean; 3.3 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 489 op/s 2023-08-27T22:15:59.325 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/b696f380-4522-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T22:16:00.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:16:00.145Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:00.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:16:00.146Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:00.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:00 smithi006 bash[33918]: cluster 2023-08-27T22:15:59.264102+0000 mgr.y (mgr.14901) 524 : cluster [DBG] pgmap v530: 89 pgs: 89 active+clean; 3.3 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 433 op/s 2023-08-27T22:16:00.756 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:00 smithi006 bash[33918]: audit 2023-08-27T22:15:59.602034+0000 mon.c (mon.2) 219 : audit [DBG] from='client.? 172.21.15.6:0/3300942174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:00.756 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:00 smithi006 bash[27292]: cluster 2023-08-27T22:15:59.264102+0000 mgr.y (mgr.14901) 524 : cluster [DBG] pgmap v530: 89 pgs: 89 active+clean; 3.3 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 433 op/s 2023-08-27T22:16:00.756 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:00 smithi006 bash[27292]: audit 2023-08-27T22:15:59.602034+0000 mon.c (mon.2) 219 : audit [DBG] from='client.? 172.21.15.6:0/3300942174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:00 smithi130 bash[27579]: cluster 2023-08-27T22:15:59.264102+0000 mgr.y (mgr.14901) 524 : cluster [DBG] pgmap v530: 89 pgs: 89 active+clean; 3.3 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 433 op/s 2023-08-27T22:16:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:00 smithi130 bash[27579]: audit 2023-08-27T22:15:59.602034+0000 mon.c (mon.2) 219 : audit [DBG] from='client.? 172.21.15.6:0/3300942174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:01.180 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:16:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:16:00] "GET /metrics HTTP/1.1" 200 195385 "" "Prometheus/2.33.4" 2023-08-27T22:16:01.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:16:01.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:01.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:16:01.712Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:02 smithi130 bash[27579]: cluster 2023-08-27T22:16:01.265015+0000 mgr.y (mgr.14901) 525 : cluster [DBG] pgmap v531: 89 pgs: 89 active+clean; 3.3 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 413 op/s 2023-08-27T22:16:02.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:02 smithi130 bash[27579]: audit 2023-08-27T22:16:02.003297+0000 mon.c (mon.2) 220 : audit [DBG] from='client.? 172.21.15.6:0/2994183059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:02.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:02 smithi006 bash[27292]: cluster 2023-08-27T22:16:01.265015+0000 mgr.y (mgr.14901) 525 : cluster [DBG] pgmap v531: 89 pgs: 89 active+clean; 3.3 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 413 op/s 2023-08-27T22:16:02.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:02 smithi006 bash[27292]: audit 2023-08-27T22:16:02.003297+0000 mon.c (mon.2) 220 : audit [DBG] from='client.? 172.21.15.6:0/2994183059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:02.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:02 smithi006 bash[33918]: cluster 2023-08-27T22:16:01.265015+0000 mgr.y (mgr.14901) 525 : cluster [DBG] pgmap v531: 89 pgs: 89 active+clean; 3.3 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 413 op/s 2023-08-27T22:16:02.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:02 smithi006 bash[33918]: audit 2023-08-27T22:16:02.003297+0000 mon.c (mon.2) 220 : audit [DBG] from='client.? 172.21.15.6:0/2994183059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:03.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:16:03.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:03.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:16:03.608Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:04.511 INFO:teuthology.orchestra.run.smithi006.stderr:pool 'unique_pool_3' removed 2023-08-27T22:16:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:04 smithi130 bash[27579]: cluster 2023-08-27T22:16:03.265444+0000 mgr.y (mgr.14901) 526 : cluster [DBG] pgmap v532: 89 pgs: 89 active+clean; 3.3 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 139 op/s 2023-08-27T22:16:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:04 smithi130 bash[27579]: audit 2023-08-27T22:16:03.513285+0000 mon.c (mon.2) 221 : audit [INF] from='client.? 172.21.15.6:0/1911114054' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-27T22:16:04.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:04 smithi130 bash[27579]: audit 2023-08-27T22:16:03.513938+0000 mon.a (mon.0) 1092 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-27T22:16:04.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:04 smithi130 bash[27579]: audit 2023-08-27T22:16:04.420789+0000 mon.a (mon.0) 1093 : audit [DBG] from='client.? 172.21.15.6:0/2118973678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:04.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:04 smithi006 bash[27292]: cluster 2023-08-27T22:16:03.265444+0000 mgr.y (mgr.14901) 526 : cluster [DBG] pgmap v532: 89 pgs: 89 active+clean; 3.3 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 139 op/s 2023-08-27T22:16:04.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:04 smithi006 bash[27292]: audit 2023-08-27T22:16:03.513285+0000 mon.c (mon.2) 221 : audit [INF] from='client.? 172.21.15.6:0/1911114054' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-27T22:16:04.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:04 smithi006 bash[27292]: audit 2023-08-27T22:16:03.513938+0000 mon.a (mon.0) 1092 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-27T22:16:04.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:04 smithi006 bash[27292]: audit 2023-08-27T22:16:04.420789+0000 mon.a (mon.0) 1093 : audit [DBG] from='client.? 172.21.15.6:0/2118973678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:04.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:04 smithi006 bash[33918]: cluster 2023-08-27T22:16:03.265444+0000 mgr.y (mgr.14901) 526 : cluster [DBG] pgmap v532: 89 pgs: 89 active+clean; 3.3 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 139 op/s 2023-08-27T22:16:04.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:04 smithi006 bash[33918]: audit 2023-08-27T22:16:03.513285+0000 mon.c (mon.2) 221 : audit [INF] from='client.? 172.21.15.6:0/1911114054' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-27T22:16:04.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:04 smithi006 bash[33918]: audit 2023-08-27T22:16:03.513938+0000 mon.a (mon.0) 1092 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-27T22:16:04.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:04 smithi006 bash[33918]: audit 2023-08-27T22:16:04.420789+0000 mon.a (mon.0) 1093 : audit [DBG] from='client.? 172.21.15.6:0/2118973678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:05.423 INFO:teuthology.task.sequential:In sequential, running task print... 2023-08-27T22:16:05.424 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-08-27T22:16:05.424 DEBUG:teuthology.parallel:result is None 2023-08-27T22:16:05.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:05 smithi006 bash[33918]: audit 2023-08-27T22:16:04.487359+0000 mon.a (mon.0) 1094 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-08-27T22:16:05.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:05 smithi006 bash[33918]: cluster 2023-08-27T22:16:04.487456+0000 mon.a (mon.0) 1095 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2023-08-27T22:16:05.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:05 smithi006 bash[27292]: audit 2023-08-27T22:16:04.487359+0000 mon.a (mon.0) 1094 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-08-27T22:16:05.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:05 smithi006 bash[27292]: cluster 2023-08-27T22:16:04.487456+0000 mon.a (mon.0) 1095 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2023-08-27T22:16:05.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:05 smithi130 bash[27579]: audit 2023-08-27T22:16:04.487359+0000 mon.a (mon.0) 1094 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-08-27T22:16:05.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:05 smithi130 bash[27579]: cluster 2023-08-27T22:16:04.487456+0000 mon.a (mon.0) 1095 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2023-08-27T22:16:06.873 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:06 smithi130 bash[27579]: cluster 2023-08-27T22:16:05.266223+0000 mgr.y (mgr.14901) 527 : cluster [DBG] pgmap v534: 73 pgs: 73 active+clean; 3.3 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-08-27T22:16:06.873 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:06 smithi130 bash[27579]: audit 2023-08-27T22:16:06.439633+0000 mon.b (mon.1) 178 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:16:06.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:06 smithi006 bash[33918]: cluster 2023-08-27T22:16:05.266223+0000 mgr.y (mgr.14901) 527 : cluster [DBG] pgmap v534: 73 pgs: 73 active+clean; 3.3 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-08-27T22:16:06.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:06 smithi006 bash[33918]: audit 2023-08-27T22:16:06.439633+0000 mon.b (mon.1) 178 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:16:06.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:06 smithi006 bash[27292]: cluster 2023-08-27T22:16:05.266223+0000 mgr.y (mgr.14901) 527 : cluster [DBG] pgmap v534: 73 pgs: 73 active+clean; 3.3 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-08-27T22:16:06.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:06 smithi006 bash[27292]: audit 2023-08-27T22:16:06.439633+0000 mon.b (mon.1) 178 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:16:08.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:07 smithi130 bash[27579]: audit 2023-08-27T22:16:06.717637+0000 mon.a (mon.0) 1096 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:16:08.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:07 smithi130 bash[27579]: audit 2023-08-27T22:16:06.722751+0000 mon.a (mon.0) 1097 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:16:08.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:07 smithi130 bash[27579]: audit 2023-08-27T22:16:06.865303+0000 mon.c (mon.2) 222 : audit [DBG] from='client.? 172.21.15.6:0/4171272014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:08.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:07 smithi006 bash[33918]: audit 2023-08-27T22:16:06.717637+0000 mon.a (mon.0) 1096 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:16:08.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:07 smithi006 bash[33918]: audit 2023-08-27T22:16:06.722751+0000 mon.a (mon.0) 1097 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:16:08.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:07 smithi006 bash[33918]: audit 2023-08-27T22:16:06.865303+0000 mon.c (mon.2) 222 : audit [DBG] from='client.? 172.21.15.6:0/4171272014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:08.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:07 smithi006 bash[27292]: audit 2023-08-27T22:16:06.717637+0000 mon.a (mon.0) 1096 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:16:08.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:07 smithi006 bash[27292]: audit 2023-08-27T22:16:06.722751+0000 mon.a (mon.0) 1097 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:16:08.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:07 smithi006 bash[27292]: audit 2023-08-27T22:16:06.865303+0000 mon.c (mon.2) 222 : audit [DBG] from='client.? 172.21.15.6:0/4171272014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:08 smithi130 bash[27579]: cluster 2023-08-27T22:16:07.266934+0000 mgr.y (mgr.14901) 528 : cluster [DBG] pgmap v535: 73 pgs: 73 active+clean; 3.3 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-08-27T22:16:09.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:08 smithi006 bash[33918]: cluster 2023-08-27T22:16:07.266934+0000 mgr.y (mgr.14901) 528 : cluster [DBG] pgmap v535: 73 pgs: 73 active+clean; 3.3 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-08-27T22:16:09.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:08 smithi006 bash[27292]: cluster 2023-08-27T22:16:07.266934+0000 mgr.y (mgr.14901) 528 : cluster [DBG] pgmap v535: 73 pgs: 73 active+clean; 3.3 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-08-27T22:16:10.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:09 smithi130 bash[27579]: audit 2023-08-27T22:16:09.295114+0000 mon.a (mon.0) 1098 : audit [DBG] from='client.? 172.21.15.6:0/3616116096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:10.151 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:09 smithi006 bash[27292]: audit 2023-08-27T22:16:09.295114+0000 mon.a (mon.0) 1098 : audit [DBG] from='client.? 172.21.15.6:0/3616116096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:10.151 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:09 smithi006 bash[33918]: audit 2023-08-27T22:16:09.295114+0000 mon.a (mon.0) 1098 : audit [DBG] from='client.? 172.21.15.6:0/3616116096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:10.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:16:10.145Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:10.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:16:10.147Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:10 smithi130 bash[27579]: cluster 2023-08-27T22:16:09.267631+0000 mgr.y (mgr.14901) 529 : cluster [DBG] pgmap v536: 73 pgs: 73 active+clean; 3.3 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-08-27T22:16:11.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:10 smithi006 bash[27292]: cluster 2023-08-27T22:16:09.267631+0000 mgr.y (mgr.14901) 529 : cluster [DBG] pgmap v536: 73 pgs: 73 active+clean; 3.3 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-08-27T22:16:11.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:16:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:16:10] "GET /metrics HTTP/1.1" 200 195385 "" "Prometheus/2.33.4" 2023-08-27T22:16:11.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:10 smithi006 bash[33918]: cluster 2023-08-27T22:16:09.267631+0000 mgr.y (mgr.14901) 529 : cluster [DBG] pgmap v536: 73 pgs: 73 active+clean; 3.3 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-08-27T22:16:11.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:16:11.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:11.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:16:11.713Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:11.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:11 smithi006 bash[27292]: audit 2023-08-27T22:16:11.719524+0000 mon.a (mon.0) 1099 : audit [DBG] from='client.? 172.21.15.6:0/1081605278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:11.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:11 smithi006 bash[33918]: audit 2023-08-27T22:16:11.719524+0000 mon.a (mon.0) 1099 : audit [DBG] from='client.? 172.21.15.6:0/1081605278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:12.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:11 smithi130 bash[27579]: audit 2023-08-27T22:16:11.719524+0000 mon.a (mon.0) 1099 : audit [DBG] from='client.? 172.21.15.6:0/1081605278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:12 smithi130 bash[27579]: cluster 2023-08-27T22:16:11.268685+0000 mgr.y (mgr.14901) 530 : cluster [DBG] pgmap v537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:12 smithi130 bash[27579]: audit 2023-08-27T22:16:12.593422+0000 mon.a (mon.0) 1100 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:16:13.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:12 smithi006 bash[27292]: cluster 2023-08-27T22:16:11.268685+0000 mgr.y (mgr.14901) 530 : cluster [DBG] pgmap v537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:13.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:12 smithi006 bash[27292]: audit 2023-08-27T22:16:12.593422+0000 mon.a (mon.0) 1100 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:16:13.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:12 smithi006 bash[33918]: cluster 2023-08-27T22:16:11.268685+0000 mgr.y (mgr.14901) 530 : cluster [DBG] pgmap v537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:13.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:12 smithi006 bash[33918]: audit 2023-08-27T22:16:12.593422+0000 mon.a (mon.0) 1100 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:16:13.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:16:13.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:13.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:16:13.609Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:14.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:13 smithi130 bash[27579]: audit 2023-08-27T22:16:12.981864+0000 mon.a (mon.0) 1101 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:16:14.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:13 smithi130 bash[27579]: cluster 2023-08-27T22:16:13.269430+0000 mgr.y (mgr.14901) 531 : cluster [DBG] pgmap v538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:14.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:13 smithi130 bash[27579]: audit 2023-08-27T22:16:13.280354+0000 mon.b (mon.1) 179 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:16:14.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:13 smithi130 bash[27579]: audit 2023-08-27T22:16:13.282690+0000 mon.b (mon.1) 180 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:16:14.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:13 smithi130 bash[27579]: audit 2023-08-27T22:16:13.290753+0000 mon.a (mon.0) 1102 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:16:14.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:13 smithi006 bash[27292]: audit 2023-08-27T22:16:12.981864+0000 mon.a (mon.0) 1101 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:16:14.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:13 smithi006 bash[27292]: cluster 2023-08-27T22:16:13.269430+0000 mgr.y (mgr.14901) 531 : cluster [DBG] pgmap v538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:14.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:13 smithi006 bash[27292]: audit 2023-08-27T22:16:13.280354+0000 mon.b (mon.1) 179 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:16:14.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:13 smithi006 bash[27292]: audit 2023-08-27T22:16:13.282690+0000 mon.b (mon.1) 180 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:16:14.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:13 smithi006 bash[27292]: audit 2023-08-27T22:16:13.290753+0000 mon.a (mon.0) 1102 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:16:14.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:13 smithi006 bash[33918]: audit 2023-08-27T22:16:12.981864+0000 mon.a (mon.0) 1101 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:16:14.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:13 smithi006 bash[33918]: cluster 2023-08-27T22:16:13.269430+0000 mgr.y (mgr.14901) 531 : cluster [DBG] pgmap v538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:14.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:13 smithi006 bash[33918]: audit 2023-08-27T22:16:13.280354+0000 mon.b (mon.1) 179 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:16:14.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:13 smithi006 bash[33918]: audit 2023-08-27T22:16:13.282690+0000 mon.b (mon.1) 180 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:16:14.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:13 smithi006 bash[33918]: audit 2023-08-27T22:16:13.290753+0000 mon.a (mon.0) 1102 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:16:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:14 smithi130 bash[27579]: audit 2023-08-27T22:16:14.141470+0000 mon.a (mon.0) 1103 : audit [DBG] from='client.? 172.21.15.6:0/2813401832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:15.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:14 smithi006 bash[33918]: audit 2023-08-27T22:16:14.141470+0000 mon.a (mon.0) 1103 : audit [DBG] from='client.? 172.21.15.6:0/2813401832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:15.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:14 smithi006 bash[27292]: audit 2023-08-27T22:16:14.141470+0000 mon.a (mon.0) 1103 : audit [DBG] from='client.? 172.21.15.6:0/2813401832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:16 smithi130 bash[27579]: cluster 2023-08-27T22:16:15.270495+0000 mgr.y (mgr.14901) 532 : cluster [DBG] pgmap v539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:16.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:15 smithi006 bash[27292]: cluster 2023-08-27T22:16:15.270495+0000 mgr.y (mgr.14901) 532 : cluster [DBG] pgmap v539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:16.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:15 smithi006 bash[33918]: cluster 2023-08-27T22:16:15.270495+0000 mgr.y (mgr.14901) 532 : cluster [DBG] pgmap v539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:17.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:17 smithi130 bash[27579]: audit 2023-08-27T22:16:16.554745+0000 mon.a (mon.0) 1104 : audit [DBG] from='client.? 172.21.15.6:0/3426899840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:17.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:16 smithi006 bash[27292]: audit 2023-08-27T22:16:16.554745+0000 mon.a (mon.0) 1104 : audit [DBG] from='client.? 172.21.15.6:0/3426899840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:17.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:16 smithi006 bash[33918]: audit 2023-08-27T22:16:16.554745+0000 mon.a (mon.0) 1104 : audit [DBG] from='client.? 172.21.15.6:0/3426899840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:18 smithi130 bash[27579]: cluster 2023-08-27T22:16:17.271287+0000 mgr.y (mgr.14901) 533 : cluster [DBG] pgmap v540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:18.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:18 smithi006 bash[27292]: cluster 2023-08-27T22:16:17.271287+0000 mgr.y (mgr.14901) 533 : cluster [DBG] pgmap v540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:18.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:18 smithi006 bash[33918]: cluster 2023-08-27T22:16:17.271287+0000 mgr.y (mgr.14901) 533 : cluster [DBG] pgmap v540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:19.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:19 smithi006 bash[33918]: audit 2023-08-27T22:16:18.976588+0000 mon.a (mon.0) 1105 : audit [DBG] from='client.? 172.21.15.6:0/2758639832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:19.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:19 smithi006 bash[27292]: audit 2023-08-27T22:16:18.976588+0000 mon.a (mon.0) 1105 : audit [DBG] from='client.? 172.21.15.6:0/2758639832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:19.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:19 smithi130 bash[27579]: audit 2023-08-27T22:16:18.976588+0000 mon.a (mon.0) 1105 : audit [DBG] from='client.? 172.21.15.6:0/2758639832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:20.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:20 smithi006 bash[33918]: cluster 2023-08-27T22:16:19.272081+0000 mgr.y (mgr.14901) 534 : cluster [DBG] pgmap v541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:20.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:16:20.145Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:20.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:16:20.147Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:20.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:20 smithi006 bash[27292]: cluster 2023-08-27T22:16:19.272081+0000 mgr.y (mgr.14901) 534 : cluster [DBG] pgmap v541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:20 smithi130 bash[27579]: cluster 2023-08-27T22:16:19.272081+0000 mgr.y (mgr.14901) 534 : cluster [DBG] pgmap v541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:21.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:16:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:16:20] "GET /metrics HTTP/1.1" 200 193079 "" "Prometheus/2.33.4" 2023-08-27T22:16:21.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:16:21.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:21.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:16:21.713Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:22.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:22 smithi006 bash[27292]: cluster 2023-08-27T22:16:21.273091+0000 mgr.y (mgr.14901) 535 : cluster [DBG] pgmap v542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:22.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:22 smithi006 bash[27292]: audit 2023-08-27T22:16:21.399304+0000 mon.c (mon.2) 223 : audit [DBG] from='client.? 172.21.15.6:0/1744296443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:22.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:22 smithi006 bash[33918]: cluster 2023-08-27T22:16:21.273091+0000 mgr.y (mgr.14901) 535 : cluster [DBG] pgmap v542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:22.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:22 smithi006 bash[33918]: audit 2023-08-27T22:16:21.399304+0000 mon.c (mon.2) 223 : audit [DBG] from='client.? 172.21.15.6:0/1744296443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:22 smithi130 bash[27579]: cluster 2023-08-27T22:16:21.273091+0000 mgr.y (mgr.14901) 535 : cluster [DBG] pgmap v542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:22 smithi130 bash[27579]: audit 2023-08-27T22:16:21.399304+0000 mon.c (mon.2) 223 : audit [DBG] from='client.? 172.21.15.6:0/1744296443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:23.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:16:23.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:23.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:16:23.610Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:24.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:24 smithi006 bash[33918]: cluster 2023-08-27T22:16:23.273747+0000 mgr.y (mgr.14901) 536 : cluster [DBG] pgmap v543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:24.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:24 smithi006 bash[33918]: audit 2023-08-27T22:16:23.817957+0000 mon.a (mon.0) 1106 : audit [DBG] from='client.? 172.21.15.6:0/3277828651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:24.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:24 smithi006 bash[27292]: cluster 2023-08-27T22:16:23.273747+0000 mgr.y (mgr.14901) 536 : cluster [DBG] pgmap v543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:24.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:24 smithi006 bash[27292]: audit 2023-08-27T22:16:23.817957+0000 mon.a (mon.0) 1106 : audit [DBG] from='client.? 172.21.15.6:0/3277828651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:24 smithi130 bash[27579]: cluster 2023-08-27T22:16:23.273747+0000 mgr.y (mgr.14901) 536 : cluster [DBG] pgmap v543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:24.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:24 smithi130 bash[27579]: audit 2023-08-27T22:16:23.817957+0000 mon.a (mon.0) 1106 : audit [DBG] from='client.? 172.21.15.6:0/3277828651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:26.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:26 smithi006 bash[33918]: cluster 2023-08-27T22:16:25.274891+0000 mgr.y (mgr.14901) 537 : cluster [DBG] pgmap v544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:26.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:26 smithi006 bash[33918]: audit 2023-08-27T22:16:26.240902+0000 mon.c (mon.2) 224 : audit [DBG] from='client.? 172.21.15.6:0/1891783929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:26.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:26 smithi006 bash[27292]: cluster 2023-08-27T22:16:25.274891+0000 mgr.y (mgr.14901) 537 : cluster [DBG] pgmap v544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:26.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:26 smithi006 bash[27292]: audit 2023-08-27T22:16:26.240902+0000 mon.c (mon.2) 224 : audit [DBG] from='client.? 172.21.15.6:0/1891783929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:26 smithi130 bash[27579]: cluster 2023-08-27T22:16:25.274891+0000 mgr.y (mgr.14901) 537 : cluster [DBG] pgmap v544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:26 smithi130 bash[27579]: audit 2023-08-27T22:16:26.240902+0000 mon.c (mon.2) 224 : audit [DBG] from='client.? 172.21.15.6:0/1891783929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:28.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:28 smithi006 bash[27292]: cluster 2023-08-27T22:16:27.275712+0000 mgr.y (mgr.14901) 538 : cluster [DBG] pgmap v545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:28.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:28 smithi006 bash[33918]: cluster 2023-08-27T22:16:27.275712+0000 mgr.y (mgr.14901) 538 : cluster [DBG] pgmap v545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:28 smithi130 bash[27579]: cluster 2023-08-27T22:16:27.275712+0000 mgr.y (mgr.14901) 538 : cluster [DBG] pgmap v545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:29.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:29 smithi006 bash[33918]: audit 2023-08-27T22:16:28.655802+0000 mon.c (mon.2) 225 : audit [DBG] from='client.? 172.21.15.6:0/2140678757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:29.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:29 smithi006 bash[27292]: audit 2023-08-27T22:16:28.655802+0000 mon.c (mon.2) 225 : audit [DBG] from='client.? 172.21.15.6:0/2140678757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:29.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:29 smithi130 bash[27579]: audit 2023-08-27T22:16:28.655802+0000 mon.c (mon.2) 225 : audit [DBG] from='client.? 172.21.15.6:0/2140678757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:30.421 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:16:30.146Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:30.421 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:16:30.148Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:30.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:30 smithi006 bash[27292]: cluster 2023-08-27T22:16:29.276492+0000 mgr.y (mgr.14901) 539 : cluster [DBG] pgmap v546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:30.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:30 smithi006 bash[33918]: cluster 2023-08-27T22:16:29.276492+0000 mgr.y (mgr.14901) 539 : cluster [DBG] pgmap v546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:30 smithi130 bash[27579]: cluster 2023-08-27T22:16:29.276492+0000 mgr.y (mgr.14901) 539 : cluster [DBG] pgmap v546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:31.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:16:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:16:30] "GET /metrics HTTP/1.1" 200 193066 "" "Prometheus/2.33.4" 2023-08-27T22:16:31.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:31 smithi006 bash[33918]: audit 2023-08-27T22:16:31.065435+0000 mon.a (mon.0) 1107 : audit [DBG] from='client.? 172.21.15.6:0/1328558787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:31.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:31 smithi006 bash[27292]: audit 2023-08-27T22:16:31.065435+0000 mon.a (mon.0) 1107 : audit [DBG] from='client.? 172.21.15.6:0/1328558787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:31.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:31 smithi130 bash[27579]: audit 2023-08-27T22:16:31.065435+0000 mon.a (mon.0) 1107 : audit [DBG] from='client.? 172.21.15.6:0/1328558787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:31.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:16:31.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:31.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:16:31.714Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:32.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:32 smithi006 bash[27292]: cluster 2023-08-27T22:16:31.277548+0000 mgr.y (mgr.14901) 540 : cluster [DBG] pgmap v547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:32.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:32 smithi006 bash[33918]: cluster 2023-08-27T22:16:31.277548+0000 mgr.y (mgr.14901) 540 : cluster [DBG] pgmap v547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:32 smithi130 bash[27579]: cluster 2023-08-27T22:16:31.277548+0000 mgr.y (mgr.14901) 540 : cluster [DBG] pgmap v547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:33.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:16:33.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:33.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:16:33.610Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:34.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:34 smithi006 bash[33918]: cluster 2023-08-27T22:16:33.278117+0000 mgr.y (mgr.14901) 541 : cluster [DBG] pgmap v548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:34.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:34 smithi006 bash[33918]: audit 2023-08-27T22:16:33.486837+0000 mon.a (mon.0) 1108 : audit [DBG] from='client.? 172.21.15.6:0/3761197452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:34.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:34 smithi006 bash[27292]: cluster 2023-08-27T22:16:33.278117+0000 mgr.y (mgr.14901) 541 : cluster [DBG] pgmap v548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:34.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:34 smithi006 bash[27292]: audit 2023-08-27T22:16:33.486837+0000 mon.a (mon.0) 1108 : audit [DBG] from='client.? 172.21.15.6:0/3761197452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:34 smithi130 bash[27579]: cluster 2023-08-27T22:16:33.278117+0000 mgr.y (mgr.14901) 541 : cluster [DBG] pgmap v548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:34.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:34 smithi130 bash[27579]: audit 2023-08-27T22:16:33.486837+0000 mon.a (mon.0) 1108 : audit [DBG] from='client.? 172.21.15.6:0/3761197452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:36.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:36 smithi006 bash[33918]: cluster 2023-08-27T22:16:35.279177+0000 mgr.y (mgr.14901) 542 : cluster [DBG] pgmap v549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:36.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:36 smithi006 bash[33918]: audit 2023-08-27T22:16:35.902345+0000 mon.a (mon.0) 1109 : audit [DBG] from='client.? 172.21.15.6:0/2914441021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:36.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:36 smithi006 bash[27292]: cluster 2023-08-27T22:16:35.279177+0000 mgr.y (mgr.14901) 542 : cluster [DBG] pgmap v549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:36.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:36 smithi006 bash[27292]: audit 2023-08-27T22:16:35.902345+0000 mon.a (mon.0) 1109 : audit [DBG] from='client.? 172.21.15.6:0/2914441021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:36 smithi130 bash[27579]: cluster 2023-08-27T22:16:35.279177+0000 mgr.y (mgr.14901) 542 : cluster [DBG] pgmap v549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:36 smithi130 bash[27579]: audit 2023-08-27T22:16:35.902345+0000 mon.a (mon.0) 1109 : audit [DBG] from='client.? 172.21.15.6:0/2914441021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:38.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:38 smithi006 bash[27292]: cluster 2023-08-27T22:16:37.279966+0000 mgr.y (mgr.14901) 543 : cluster [DBG] pgmap v550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:38.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:38 smithi006 bash[27292]: audit 2023-08-27T22:16:38.320491+0000 mon.c (mon.2) 226 : audit [DBG] from='client.? 172.21.15.6:0/4047977105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:38.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:38 smithi006 bash[33918]: cluster 2023-08-27T22:16:37.279966+0000 mgr.y (mgr.14901) 543 : cluster [DBG] pgmap v550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:38.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:38 smithi006 bash[33918]: audit 2023-08-27T22:16:38.320491+0000 mon.c (mon.2) 226 : audit [DBG] from='client.? 172.21.15.6:0/4047977105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:38 smithi130 bash[27579]: cluster 2023-08-27T22:16:37.279966+0000 mgr.y (mgr.14901) 543 : cluster [DBG] pgmap v550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:38 smithi130 bash[27579]: audit 2023-08-27T22:16:38.320491+0000 mon.c (mon.2) 226 : audit [DBG] from='client.? 172.21.15.6:0/4047977105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:40.453 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:16:40.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:40.454 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:16:40.150Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:40.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:40 smithi006 bash[33918]: cluster 2023-08-27T22:16:39.280864+0000 mgr.y (mgr.14901) 544 : cluster [DBG] pgmap v551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:40.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:40 smithi006 bash[27292]: cluster 2023-08-27T22:16:39.280864+0000 mgr.y (mgr.14901) 544 : cluster [DBG] pgmap v551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:40 smithi130 bash[27579]: cluster 2023-08-27T22:16:39.280864+0000 mgr.y (mgr.14901) 544 : cluster [DBG] pgmap v551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:41.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:16:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:16:40] "GET /metrics HTTP/1.1" 200 193066 "" "Prometheus/2.33.4" 2023-08-27T22:16:41.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:41 smithi006 bash[27292]: audit 2023-08-27T22:16:40.730820+0000 mon.a (mon.0) 1110 : audit [DBG] from='client.? 172.21.15.6:0/2246884532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:41.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:41 smithi006 bash[33918]: audit 2023-08-27T22:16:40.730820+0000 mon.a (mon.0) 1110 : audit [DBG] from='client.? 172.21.15.6:0/2246884532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:41.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:41 smithi130 bash[27579]: audit 2023-08-27T22:16:40.730820+0000 mon.a (mon.0) 1110 : audit [DBG] from='client.? 172.21.15.6:0/2246884532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:41.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:16:41.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:41.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:16:41.714Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:42.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:42 smithi006 bash[27292]: cluster 2023-08-27T22:16:41.282009+0000 mgr.y (mgr.14901) 545 : cluster [DBG] pgmap v552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:42.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:42 smithi006 bash[33918]: cluster 2023-08-27T22:16:41.282009+0000 mgr.y (mgr.14901) 545 : cluster [DBG] pgmap v552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:42.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:42 smithi130 bash[27579]: cluster 2023-08-27T22:16:41.282009+0000 mgr.y (mgr.14901) 545 : cluster [DBG] pgmap v552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:43.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:43 smithi130 bash[27579]: audit 2023-08-27T22:16:43.144854+0000 mon.c (mon.2) 227 : audit [DBG] from='client.? 172.21.15.6:0/252762806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:43.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:43 smithi006 bash[33918]: audit 2023-08-27T22:16:43.144854+0000 mon.c (mon.2) 227 : audit [DBG] from='client.? 172.21.15.6:0/252762806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:43.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:43 smithi006 bash[27292]: audit 2023-08-27T22:16:43.144854+0000 mon.c (mon.2) 227 : audit [DBG] from='client.? 172.21.15.6:0/252762806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:43.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:16:43.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:43.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:16:43.611Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:44 smithi130 bash[27579]: cluster 2023-08-27T22:16:43.282663+0000 mgr.y (mgr.14901) 546 : cluster [DBG] pgmap v553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:44.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:44 smithi006 bash[33918]: cluster 2023-08-27T22:16:43.282663+0000 mgr.y (mgr.14901) 546 : cluster [DBG] pgmap v553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:44.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:44 smithi006 bash[27292]: cluster 2023-08-27T22:16:43.282663+0000 mgr.y (mgr.14901) 546 : cluster [DBG] pgmap v553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:46 smithi130 bash[27579]: cluster 2023-08-27T22:16:45.283611+0000 mgr.y (mgr.14901) 547 : cluster [DBG] pgmap v554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:46.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:46 smithi130 bash[27579]: audit 2023-08-27T22:16:45.577371+0000 mon.a (mon.0) 1111 : audit [DBG] from='client.? 172.21.15.6:0/902580977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:46.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:46 smithi006 bash[33918]: cluster 2023-08-27T22:16:45.283611+0000 mgr.y (mgr.14901) 547 : cluster [DBG] pgmap v554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:46.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:46 smithi006 bash[33918]: audit 2023-08-27T22:16:45.577371+0000 mon.a (mon.0) 1111 : audit [DBG] from='client.? 172.21.15.6:0/902580977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:46.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:46 smithi006 bash[27292]: cluster 2023-08-27T22:16:45.283611+0000 mgr.y (mgr.14901) 547 : cluster [DBG] pgmap v554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:46.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:46 smithi006 bash[27292]: audit 2023-08-27T22:16:45.577371+0000 mon.a (mon.0) 1111 : audit [DBG] from='client.? 172.21.15.6:0/902580977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:48.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:48 smithi130 bash[27579]: cluster 2023-08-27T22:16:47.284360+0000 mgr.y (mgr.14901) 548 : cluster [DBG] pgmap v555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:48.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:48 smithi130 bash[27579]: audit 2023-08-27T22:16:47.990933+0000 mon.c (mon.2) 228 : audit [DBG] from='client.? 172.21.15.6:0/1855256987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:48.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:48 smithi006 bash[33918]: cluster 2023-08-27T22:16:47.284360+0000 mgr.y (mgr.14901) 548 : cluster [DBG] pgmap v555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:48.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:48 smithi006 bash[33918]: audit 2023-08-27T22:16:47.990933+0000 mon.c (mon.2) 228 : audit [DBG] from='client.? 172.21.15.6:0/1855256987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:48.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:48 smithi006 bash[27292]: cluster 2023-08-27T22:16:47.284360+0000 mgr.y (mgr.14901) 548 : cluster [DBG] pgmap v555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:48.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:48 smithi006 bash[27292]: audit 2023-08-27T22:16:47.990933+0000 mon.c (mon.2) 228 : audit [DBG] from='client.? 172.21.15.6:0/1855256987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:50.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:16:50.148Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:50.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:16:50.149Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:50.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:50 smithi006 bash[33918]: cluster 2023-08-27T22:16:49.285158+0000 mgr.y (mgr.14901) 549 : cluster [DBG] pgmap v556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:50.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:50 smithi006 bash[33918]: audit 2023-08-27T22:16:50.413313+0000 mon.c (mon.2) 229 : audit [DBG] from='client.? 172.21.15.6:0/968397838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:50.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:50 smithi006 bash[27292]: cluster 2023-08-27T22:16:49.285158+0000 mgr.y (mgr.14901) 549 : cluster [DBG] pgmap v556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:50.756 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:50 smithi006 bash[27292]: audit 2023-08-27T22:16:50.413313+0000 mon.c (mon.2) 229 : audit [DBG] from='client.? 172.21.15.6:0/968397838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:50.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:50 smithi130 bash[27579]: cluster 2023-08-27T22:16:49.285158+0000 mgr.y (mgr.14901) 549 : cluster [DBG] pgmap v556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:50.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:50 smithi130 bash[27579]: audit 2023-08-27T22:16:50.413313+0000 mon.c (mon.2) 229 : audit [DBG] from='client.? 172.21.15.6:0/968397838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:51.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:16:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:16:50] "GET /metrics HTTP/1.1" 200 193068 "" "Prometheus/2.33.4" 2023-08-27T22:16:51.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:16:51.713Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:51.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:16:51.715Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:52 smithi130 bash[27579]: cluster 2023-08-27T22:16:51.286168+0000 mgr.y (mgr.14901) 550 : cluster [DBG] pgmap v557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:52.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:52 smithi006 bash[27292]: cluster 2023-08-27T22:16:51.286168+0000 mgr.y (mgr.14901) 550 : cluster [DBG] pgmap v557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:52.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:52 smithi006 bash[33918]: cluster 2023-08-27T22:16:51.286168+0000 mgr.y (mgr.14901) 550 : cluster [DBG] pgmap v557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:53.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:53 smithi130 bash[27579]: audit 2023-08-27T22:16:52.830104+0000 mon.a (mon.0) 1112 : audit [DBG] from='client.? 172.21.15.6:0/51097094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:53.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:53 smithi006 bash[27292]: audit 2023-08-27T22:16:52.830104+0000 mon.a (mon.0) 1112 : audit [DBG] from='client.? 172.21.15.6:0/51097094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:53.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:53 smithi006 bash[33918]: audit 2023-08-27T22:16:52.830104+0000 mon.a (mon.0) 1112 : audit [DBG] from='client.? 172.21.15.6:0/51097094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:53.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:16:53.609Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:53.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:16:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:16:53.611Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:16:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:54 smithi130 bash[27579]: cluster 2023-08-27T22:16:53.286828+0000 mgr.y (mgr.14901) 551 : cluster [DBG] pgmap v558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:54.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:54 smithi006 bash[33918]: cluster 2023-08-27T22:16:53.286828+0000 mgr.y (mgr.14901) 551 : cluster [DBG] pgmap v558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:54.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:54 smithi006 bash[27292]: cluster 2023-08-27T22:16:53.286828+0000 mgr.y (mgr.14901) 551 : cluster [DBG] pgmap v558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:55.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:55 smithi130 bash[27579]: audit 2023-08-27T22:16:55.250255+0000 mon.a (mon.0) 1113 : audit [DBG] from='client.? 172.21.15.6:0/2063735121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:55.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:55 smithi006 bash[33918]: audit 2023-08-27T22:16:55.250255+0000 mon.a (mon.0) 1113 : audit [DBG] from='client.? 172.21.15.6:0/2063735121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:55.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:55 smithi006 bash[27292]: audit 2023-08-27T22:16:55.250255+0000 mon.a (mon.0) 1113 : audit [DBG] from='client.? 172.21.15.6:0/2063735121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:56 smithi130 bash[27579]: cluster 2023-08-27T22:16:55.287839+0000 mgr.y (mgr.14901) 552 : cluster [DBG] pgmap v559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:56.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:56 smithi006 bash[27292]: cluster 2023-08-27T22:16:55.287839+0000 mgr.y (mgr.14901) 552 : cluster [DBG] pgmap v559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:56.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:56 smithi006 bash[33918]: cluster 2023-08-27T22:16:55.287839+0000 mgr.y (mgr.14901) 552 : cluster [DBG] pgmap v559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:58 smithi130 bash[27579]: cluster 2023-08-27T22:16:57.288550+0000 mgr.y (mgr.14901) 553 : cluster [DBG] pgmap v560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:16:58 smithi130 bash[27579]: audit 2023-08-27T22:16:57.664159+0000 mon.a (mon.0) 1114 : audit [DBG] from='client.? 172.21.15.6:0/1400174667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:58.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:58 smithi006 bash[27292]: cluster 2023-08-27T22:16:57.288550+0000 mgr.y (mgr.14901) 553 : cluster [DBG] pgmap v560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:58.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:16:58 smithi006 bash[27292]: audit 2023-08-27T22:16:57.664159+0000 mon.a (mon.0) 1114 : audit [DBG] from='client.? 172.21.15.6:0/1400174667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:58.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:58 smithi006 bash[33918]: cluster 2023-08-27T22:16:57.288550+0000 mgr.y (mgr.14901) 553 : cluster [DBG] pgmap v560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:16:58.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:16:58 smithi006 bash[33918]: audit 2023-08-27T22:16:57.664159+0000 mon.a (mon.0) 1114 : audit [DBG] from='client.? 172.21.15.6:0/1400174667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:00.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:17:00.149Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:00.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:17:00.151Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:00.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:00 smithi130 bash[27579]: cluster 2023-08-27T22:16:59.289337+0000 mgr.y (mgr.14901) 554 : cluster [DBG] pgmap v561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:00.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:00 smithi130 bash[27579]: audit 2023-08-27T22:17:00.078986+0000 mon.c (mon.2) 230 : audit [DBG] from='client.? 172.21.15.6:0/927786515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:00.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:17:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:17:00] "GET /metrics HTTP/1.1" 200 193066 "" "Prometheus/2.33.4" 2023-08-27T22:17:00.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:00 smithi006 bash[27292]: cluster 2023-08-27T22:16:59.289337+0000 mgr.y (mgr.14901) 554 : cluster [DBG] pgmap v561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:00.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:00 smithi006 bash[27292]: audit 2023-08-27T22:17:00.078986+0000 mon.c (mon.2) 230 : audit [DBG] from='client.? 172.21.15.6:0/927786515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:00.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:00 smithi006 bash[33918]: cluster 2023-08-27T22:16:59.289337+0000 mgr.y (mgr.14901) 554 : cluster [DBG] pgmap v561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:00.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:00 smithi006 bash[33918]: audit 2023-08-27T22:17:00.078986+0000 mon.c (mon.2) 230 : audit [DBG] from='client.? 172.21.15.6:0/927786515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:02.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:17:01.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:02.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:17:01.716Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:02 smithi130 bash[27579]: cluster 2023-08-27T22:17:01.290362+0000 mgr.y (mgr.14901) 555 : cluster [DBG] pgmap v562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:02 smithi130 bash[27579]: audit 2023-08-27T22:17:02.498490+0000 mon.c (mon.2) 231 : audit [DBG] from='client.? 172.21.15.6:0/1538701696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:02.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:02 smithi006 bash[33918]: cluster 2023-08-27T22:17:01.290362+0000 mgr.y (mgr.14901) 555 : cluster [DBG] pgmap v562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:02.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:02 smithi006 bash[33918]: audit 2023-08-27T22:17:02.498490+0000 mon.c (mon.2) 231 : audit [DBG] from='client.? 172.21.15.6:0/1538701696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:02.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:02 smithi006 bash[27292]: cluster 2023-08-27T22:17:01.290362+0000 mgr.y (mgr.14901) 555 : cluster [DBG] pgmap v562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:02.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:02 smithi006 bash[27292]: audit 2023-08-27T22:17:02.498490+0000 mon.c (mon.2) 231 : audit [DBG] from='client.? 172.21.15.6:0/1538701696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:17:03.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:17:03.612Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:04 smithi130 bash[27579]: cluster 2023-08-27T22:17:03.291166+0000 mgr.y (mgr.14901) 556 : cluster [DBG] pgmap v563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:04.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:04 smithi006 bash[27292]: cluster 2023-08-27T22:17:03.291166+0000 mgr.y (mgr.14901) 556 : cluster [DBG] pgmap v563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:04.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:04 smithi006 bash[33918]: cluster 2023-08-27T22:17:03.291166+0000 mgr.y (mgr.14901) 556 : cluster [DBG] pgmap v563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:05.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:05 smithi130 bash[27579]: audit 2023-08-27T22:17:04.909539+0000 mon.c (mon.2) 232 : audit [DBG] from='client.? 172.21.15.6:0/2554899728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:05.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:05 smithi006 bash[27292]: audit 2023-08-27T22:17:04.909539+0000 mon.c (mon.2) 232 : audit [DBG] from='client.? 172.21.15.6:0/2554899728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:05.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:05 smithi006 bash[33918]: audit 2023-08-27T22:17:04.909539+0000 mon.c (mon.2) 232 : audit [DBG] from='client.? 172.21.15.6:0/2554899728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:06 smithi130 bash[27579]: cluster 2023-08-27T22:17:05.292251+0000 mgr.y (mgr.14901) 557 : cluster [DBG] pgmap v564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:06.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:06 smithi006 bash[27292]: cluster 2023-08-27T22:17:05.292251+0000 mgr.y (mgr.14901) 557 : cluster [DBG] pgmap v564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:06.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:06 smithi006 bash[33918]: cluster 2023-08-27T22:17:05.292251+0000 mgr.y (mgr.14901) 557 : cluster [DBG] pgmap v564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:07.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:07 smithi130 bash[27579]: audit 2023-08-27T22:17:07.322186+0000 mon.c (mon.2) 233 : audit [DBG] from='client.? 172.21.15.6:0/396908944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:07.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:07 smithi006 bash[33918]: audit 2023-08-27T22:17:07.322186+0000 mon.c (mon.2) 233 : audit [DBG] from='client.? 172.21.15.6:0/396908944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:07.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:07 smithi006 bash[27292]: audit 2023-08-27T22:17:07.322186+0000 mon.c (mon.2) 233 : audit [DBG] from='client.? 172.21.15.6:0/396908944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:08 smithi130 bash[27579]: cluster 2023-08-27T22:17:07.292981+0000 mgr.y (mgr.14901) 558 : cluster [DBG] pgmap v565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:08.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:08 smithi006 bash[27292]: cluster 2023-08-27T22:17:07.292981+0000 mgr.y (mgr.14901) 558 : cluster [DBG] pgmap v565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:08.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:08 smithi006 bash[33918]: cluster 2023-08-27T22:17:07.292981+0000 mgr.y (mgr.14901) 558 : cluster [DBG] pgmap v565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:10.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:17:10.150Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:10.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:17:10.152Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:10 smithi130 bash[27579]: cluster 2023-08-27T22:17:09.293721+0000 mgr.y (mgr.14901) 559 : cluster [DBG] pgmap v566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:10 smithi130 bash[27579]: audit 2023-08-27T22:17:09.737220+0000 mon.a (mon.0) 1115 : audit [DBG] from='client.? 172.21.15.6:0/1789683288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:10.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:17:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:17:10] "GET /metrics HTTP/1.1" 200 193066 "" "Prometheus/2.33.4" 2023-08-27T22:17:10.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:10 smithi006 bash[27292]: cluster 2023-08-27T22:17:09.293721+0000 mgr.y (mgr.14901) 559 : cluster [DBG] pgmap v566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:10.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:10 smithi006 bash[27292]: audit 2023-08-27T22:17:09.737220+0000 mon.a (mon.0) 1115 : audit [DBG] from='client.? 172.21.15.6:0/1789683288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:10.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:10 smithi006 bash[33918]: cluster 2023-08-27T22:17:09.293721+0000 mgr.y (mgr.14901) 559 : cluster [DBG] pgmap v566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:10.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:10 smithi006 bash[33918]: audit 2023-08-27T22:17:09.737220+0000 mon.a (mon.0) 1115 : audit [DBG] from='client.? 172.21.15.6:0/1789683288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:12.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:17:11.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:12.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:17:11.717Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:12 smithi130 bash[27579]: cluster 2023-08-27T22:17:11.294755+0000 mgr.y (mgr.14901) 560 : cluster [DBG] pgmap v567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:12 smithi130 bash[27579]: audit 2023-08-27T22:17:12.160916+0000 mon.a (mon.0) 1116 : audit [DBG] from='client.? 172.21.15.6:0/2223427223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:12.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:12 smithi006 bash[27292]: cluster 2023-08-27T22:17:11.294755+0000 mgr.y (mgr.14901) 560 : cluster [DBG] pgmap v567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:12.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:12 smithi006 bash[27292]: audit 2023-08-27T22:17:12.160916+0000 mon.a (mon.0) 1116 : audit [DBG] from='client.? 172.21.15.6:0/2223427223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:12.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:12 smithi006 bash[33918]: cluster 2023-08-27T22:17:11.294755+0000 mgr.y (mgr.14901) 560 : cluster [DBG] pgmap v567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:12.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:12 smithi006 bash[33918]: audit 2023-08-27T22:17:12.160916+0000 mon.a (mon.0) 1116 : audit [DBG] from='client.? 172.21.15.6:0/2223427223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:13.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:13 smithi130 bash[27579]: audit 2023-08-27T22:17:13.303859+0000 mon.b (mon.1) 181 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:17:13.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:13 smithi006 bash[27292]: audit 2023-08-27T22:17:13.303859+0000 mon.b (mon.1) 181 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:17:13.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:13 smithi006 bash[33918]: audit 2023-08-27T22:17:13.303859+0000 mon.b (mon.1) 181 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:17:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:17:13.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:17:13.612Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:14 smithi130 bash[27579]: cluster 2023-08-27T22:17:13.295391+0000 mgr.y (mgr.14901) 561 : cluster [DBG] pgmap v568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:14.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:14 smithi006 bash[27292]: cluster 2023-08-27T22:17:13.295391+0000 mgr.y (mgr.14901) 561 : cluster [DBG] pgmap v568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:14.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:14 smithi006 bash[33918]: cluster 2023-08-27T22:17:13.295391+0000 mgr.y (mgr.14901) 561 : cluster [DBG] pgmap v568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:15.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:15 smithi130 bash[27579]: audit 2023-08-27T22:17:14.581286+0000 mon.c (mon.2) 234 : audit [DBG] from='client.? 172.21.15.6:0/2826546127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:15.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:15 smithi006 bash[27292]: audit 2023-08-27T22:17:14.581286+0000 mon.c (mon.2) 234 : audit [DBG] from='client.? 172.21.15.6:0/2826546127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:15.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:15 smithi006 bash[33918]: audit 2023-08-27T22:17:14.581286+0000 mon.c (mon.2) 234 : audit [DBG] from='client.? 172.21.15.6:0/2826546127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:16 smithi130 bash[27579]: cluster 2023-08-27T22:17:15.296394+0000 mgr.y (mgr.14901) 562 : cluster [DBG] pgmap v569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:16.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:16 smithi006 bash[33918]: cluster 2023-08-27T22:17:15.296394+0000 mgr.y (mgr.14901) 562 : cluster [DBG] pgmap v569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:16.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:16 smithi006 bash[27292]: cluster 2023-08-27T22:17:15.296394+0000 mgr.y (mgr.14901) 562 : cluster [DBG] pgmap v569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:17.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:17 smithi130 bash[27579]: audit 2023-08-27T22:17:17.000314+0000 mon.a (mon.0) 1117 : audit [DBG] from='client.? 172.21.15.6:0/1050067362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:17.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:17 smithi006 bash[33918]: audit 2023-08-27T22:17:17.000314+0000 mon.a (mon.0) 1117 : audit [DBG] from='client.? 172.21.15.6:0/1050067362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:17.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:17 smithi006 bash[27292]: audit 2023-08-27T22:17:17.000314+0000 mon.a (mon.0) 1117 : audit [DBG] from='client.? 172.21.15.6:0/1050067362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:18 smithi130 bash[27579]: cluster 2023-08-27T22:17:17.297222+0000 mgr.y (mgr.14901) 563 : cluster [DBG] pgmap v570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:18.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:18 smithi006 bash[27292]: cluster 2023-08-27T22:17:17.297222+0000 mgr.y (mgr.14901) 563 : cluster [DBG] pgmap v570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:18.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:18 smithi006 bash[33918]: cluster 2023-08-27T22:17:17.297222+0000 mgr.y (mgr.14901) 563 : cluster [DBG] pgmap v570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:20.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:17:20.151Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:20.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:17:20.153Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:20.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:20 smithi006 bash[33918]: audit 2023-08-27T22:17:19.193560+0000 mon.a (mon.0) 1118 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:17:20.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:20 smithi006 bash[33918]: cluster 2023-08-27T22:17:19.297981+0000 mgr.y (mgr.14901) 564 : cluster [DBG] pgmap v571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:20.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:20 smithi006 bash[33918]: audit 2023-08-27T22:17:19.419922+0000 mon.a (mon.0) 1119 : audit [DBG] from='client.? 172.21.15.6:0/252761091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:20.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:20 smithi006 bash[33918]: audit 2023-08-27T22:17:19.704096+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:17:20.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:20 smithi006 bash[33918]: audit 2023-08-27T22:17:19.986684+0000 mon.b (mon.1) 182 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:17:20.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:20 smithi006 bash[33918]: audit 2023-08-27T22:17:19.989201+0000 mon.b (mon.1) 183 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:17:20.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:20 smithi006 bash[33918]: audit 2023-08-27T22:17:19.995645+0000 mon.a (mon.0) 1121 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:17:20.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:20 smithi006 bash[27292]: audit 2023-08-27T22:17:19.193560+0000 mon.a (mon.0) 1118 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:17:20.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:20 smithi006 bash[27292]: cluster 2023-08-27T22:17:19.297981+0000 mgr.y (mgr.14901) 564 : cluster [DBG] pgmap v571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:20.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:20 smithi006 bash[27292]: audit 2023-08-27T22:17:19.419922+0000 mon.a (mon.0) 1119 : audit [DBG] from='client.? 172.21.15.6:0/252761091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:20.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:20 smithi006 bash[27292]: audit 2023-08-27T22:17:19.704096+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:17:20.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:20 smithi006 bash[27292]: audit 2023-08-27T22:17:19.986684+0000 mon.b (mon.1) 182 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:17:20.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:20 smithi006 bash[27292]: audit 2023-08-27T22:17:19.989201+0000 mon.b (mon.1) 183 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:17:20.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:20 smithi006 bash[27292]: audit 2023-08-27T22:17:19.995645+0000 mon.a (mon.0) 1121 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:17:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:20 smithi130 bash[27579]: audit 2023-08-27T22:17:19.193560+0000 mon.a (mon.0) 1118 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:17:20.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:20 smithi130 bash[27579]: cluster 2023-08-27T22:17:19.297981+0000 mgr.y (mgr.14901) 564 : cluster [DBG] pgmap v571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:20.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:20 smithi130 bash[27579]: audit 2023-08-27T22:17:19.419922+0000 mon.a (mon.0) 1119 : audit [DBG] from='client.? 172.21.15.6:0/252761091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:20.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:20 smithi130 bash[27579]: audit 2023-08-27T22:17:19.704096+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:17:20.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:20 smithi130 bash[27579]: audit 2023-08-27T22:17:19.986684+0000 mon.b (mon.1) 182 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:17:20.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:20 smithi130 bash[27579]: audit 2023-08-27T22:17:19.989201+0000 mon.b (mon.1) 183 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:17:20.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:20 smithi130 bash[27579]: audit 2023-08-27T22:17:19.995645+0000 mon.a (mon.0) 1121 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:17:21.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:17:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:17:20] "GET /metrics HTTP/1.1" 200 193066 "" "Prometheus/2.33.4" 2023-08-27T22:17:22.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:17:21.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:17:21.718Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:22 smithi130 bash[27579]: cluster 2023-08-27T22:17:21.299113+0000 mgr.y (mgr.14901) 565 : cluster [DBG] pgmap v572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:22.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:22 smithi130 bash[27579]: audit 2023-08-27T22:17:21.834032+0000 mon.c (mon.2) 235 : audit [DBG] from='client.? 172.21.15.6:0/3798614499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:22.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:22 smithi006 bash[33918]: cluster 2023-08-27T22:17:21.299113+0000 mgr.y (mgr.14901) 565 : cluster [DBG] pgmap v572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:22.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:22 smithi006 bash[33918]: audit 2023-08-27T22:17:21.834032+0000 mon.c (mon.2) 235 : audit [DBG] from='client.? 172.21.15.6:0/3798614499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:22.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:22 smithi006 bash[27292]: cluster 2023-08-27T22:17:21.299113+0000 mgr.y (mgr.14901) 565 : cluster [DBG] pgmap v572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:22.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:22 smithi006 bash[27292]: audit 2023-08-27T22:17:21.834032+0000 mon.c (mon.2) 235 : audit [DBG] from='client.? 172.21.15.6:0/3798614499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:23.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:17:23.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:23.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:17:23.613Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:24.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:24 smithi006 bash[27292]: cluster 2023-08-27T22:17:23.299913+0000 mgr.y (mgr.14901) 566 : cluster [DBG] pgmap v573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:24.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:24 smithi006 bash[27292]: audit 2023-08-27T22:17:24.245154+0000 mon.a (mon.0) 1122 : audit [DBG] from='client.? 172.21.15.6:0/705987795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:24.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:24 smithi006 bash[33918]: cluster 2023-08-27T22:17:23.299913+0000 mgr.y (mgr.14901) 566 : cluster [DBG] pgmap v573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:24.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:24 smithi006 bash[33918]: audit 2023-08-27T22:17:24.245154+0000 mon.a (mon.0) 1122 : audit [DBG] from='client.? 172.21.15.6:0/705987795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:24 smithi130 bash[27579]: cluster 2023-08-27T22:17:23.299913+0000 mgr.y (mgr.14901) 566 : cluster [DBG] pgmap v573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:24 smithi130 bash[27579]: audit 2023-08-27T22:17:24.245154+0000 mon.a (mon.0) 1122 : audit [DBG] from='client.? 172.21.15.6:0/705987795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:26.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:26 smithi006 bash[27292]: cluster 2023-08-27T22:17:25.300988+0000 mgr.y (mgr.14901) 567 : cluster [DBG] pgmap v574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:26.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:26 smithi006 bash[33918]: cluster 2023-08-27T22:17:25.300988+0000 mgr.y (mgr.14901) 567 : cluster [DBG] pgmap v574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:26.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:26 smithi130 bash[27579]: cluster 2023-08-27T22:17:25.300988+0000 mgr.y (mgr.14901) 567 : cluster [DBG] pgmap v574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:27.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:27 smithi006 bash[33918]: audit 2023-08-27T22:17:26.661339+0000 mon.a (mon.0) 1123 : audit [DBG] from='client.? 172.21.15.6:0/4146851484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:27.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:27 smithi006 bash[27292]: audit 2023-08-27T22:17:26.661339+0000 mon.a (mon.0) 1123 : audit [DBG] from='client.? 172.21.15.6:0/4146851484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:27 smithi130 bash[27579]: audit 2023-08-27T22:17:26.661339+0000 mon.a (mon.0) 1123 : audit [DBG] from='client.? 172.21.15.6:0/4146851484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:28.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:28 smithi006 bash[33918]: cluster 2023-08-27T22:17:27.301708+0000 mgr.y (mgr.14901) 568 : cluster [DBG] pgmap v575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:28.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:28 smithi006 bash[27292]: cluster 2023-08-27T22:17:27.301708+0000 mgr.y (mgr.14901) 568 : cluster [DBG] pgmap v575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:28 smithi130 bash[27579]: cluster 2023-08-27T22:17:27.301708+0000 mgr.y (mgr.14901) 568 : cluster [DBG] pgmap v575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:29.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:29 smithi006 bash[27292]: audit 2023-08-27T22:17:29.085080+0000 mon.a (mon.0) 1124 : audit [DBG] from='client.? 172.21.15.6:0/3204455779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:29.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:29 smithi006 bash[33918]: audit 2023-08-27T22:17:29.085080+0000 mon.a (mon.0) 1124 : audit [DBG] from='client.? 172.21.15.6:0/3204455779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:29.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:29 smithi130 bash[27579]: audit 2023-08-27T22:17:29.085080+0000 mon.a (mon.0) 1124 : audit [DBG] from='client.? 172.21.15.6:0/3204455779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:30.433 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:17:30.152Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:30.434 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:17:30.154Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:30.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:30 smithi006 bash[27292]: cluster 2023-08-27T22:17:29.302467+0000 mgr.y (mgr.14901) 569 : cluster [DBG] pgmap v576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:30.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:30 smithi006 bash[33918]: cluster 2023-08-27T22:17:29.302467+0000 mgr.y (mgr.14901) 569 : cluster [DBG] pgmap v576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:30 smithi130 bash[27579]: cluster 2023-08-27T22:17:29.302467+0000 mgr.y (mgr.14901) 569 : cluster [DBG] pgmap v576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:31.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:17:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:17:30] "GET /metrics HTTP/1.1" 200 193067 "" "Prometheus/2.33.4" 2023-08-27T22:17:32.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:17:31.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:32.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:17:31.719Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:32.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:32 smithi006 bash[33918]: cluster 2023-08-27T22:17:31.303459+0000 mgr.y (mgr.14901) 570 : cluster [DBG] pgmap v577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:32.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:32 smithi006 bash[33918]: audit 2023-08-27T22:17:31.507289+0000 mon.a (mon.0) 1125 : audit [DBG] from='client.? 172.21.15.6:0/3928034267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:32.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:32 smithi006 bash[27292]: cluster 2023-08-27T22:17:31.303459+0000 mgr.y (mgr.14901) 570 : cluster [DBG] pgmap v577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:32.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:32 smithi006 bash[27292]: audit 2023-08-27T22:17:31.507289+0000 mon.a (mon.0) 1125 : audit [DBG] from='client.? 172.21.15.6:0/3928034267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:32 smithi130 bash[27579]: cluster 2023-08-27T22:17:31.303459+0000 mgr.y (mgr.14901) 570 : cluster [DBG] pgmap v577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:32.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:32 smithi130 bash[27579]: audit 2023-08-27T22:17:31.507289+0000 mon.a (mon.0) 1125 : audit [DBG] from='client.? 172.21.15.6:0/3928034267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:33.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:17:33.611Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:33.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:17:33.614Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:34.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:34 smithi006 bash[33918]: cluster 2023-08-27T22:17:33.304108+0000 mgr.y (mgr.14901) 571 : cluster [DBG] pgmap v578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:34.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:34 smithi006 bash[33918]: audit 2023-08-27T22:17:33.929499+0000 mon.c (mon.2) 236 : audit [DBG] from='client.? 172.21.15.6:0/193225324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:34.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:34 smithi006 bash[27292]: cluster 2023-08-27T22:17:33.304108+0000 mgr.y (mgr.14901) 571 : cluster [DBG] pgmap v578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:34.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:34 smithi006 bash[27292]: audit 2023-08-27T22:17:33.929499+0000 mon.c (mon.2) 236 : audit [DBG] from='client.? 172.21.15.6:0/193225324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:34 smithi130 bash[27579]: cluster 2023-08-27T22:17:33.304108+0000 mgr.y (mgr.14901) 571 : cluster [DBG] pgmap v578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:34.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:34 smithi130 bash[27579]: audit 2023-08-27T22:17:33.929499+0000 mon.c (mon.2) 236 : audit [DBG] from='client.? 172.21.15.6:0/193225324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:36.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:36 smithi006 bash[33918]: cluster 2023-08-27T22:17:35.305216+0000 mgr.y (mgr.14901) 572 : cluster [DBG] pgmap v579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:36.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:36 smithi006 bash[33918]: audit 2023-08-27T22:17:36.349608+0000 mon.c (mon.2) 237 : audit [DBG] from='client.? 172.21.15.6:0/3659641360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:36.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:36 smithi006 bash[27292]: cluster 2023-08-27T22:17:35.305216+0000 mgr.y (mgr.14901) 572 : cluster [DBG] pgmap v579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:36.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:36 smithi006 bash[27292]: audit 2023-08-27T22:17:36.349608+0000 mon.c (mon.2) 237 : audit [DBG] from='client.? 172.21.15.6:0/3659641360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:36 smithi130 bash[27579]: cluster 2023-08-27T22:17:35.305216+0000 mgr.y (mgr.14901) 572 : cluster [DBG] pgmap v579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:36 smithi130 bash[27579]: audit 2023-08-27T22:17:36.349608+0000 mon.c (mon.2) 237 : audit [DBG] from='client.? 172.21.15.6:0/3659641360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:38.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:38 smithi006 bash[33918]: cluster 2023-08-27T22:17:37.306018+0000 mgr.y (mgr.14901) 573 : cluster [DBG] pgmap v580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:38.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:38 smithi006 bash[27292]: cluster 2023-08-27T22:17:37.306018+0000 mgr.y (mgr.14901) 573 : cluster [DBG] pgmap v580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:38 smithi130 bash[27579]: cluster 2023-08-27T22:17:37.306018+0000 mgr.y (mgr.14901) 573 : cluster [DBG] pgmap v580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:39.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:39 smithi006 bash[33918]: audit 2023-08-27T22:17:38.764952+0000 mon.c (mon.2) 238 : audit [DBG] from='client.? 172.21.15.6:0/2048193935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:39.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:39 smithi006 bash[27292]: audit 2023-08-27T22:17:38.764952+0000 mon.c (mon.2) 238 : audit [DBG] from='client.? 172.21.15.6:0/2048193935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:39.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:39 smithi130 bash[27579]: audit 2023-08-27T22:17:38.764952+0000 mon.c (mon.2) 238 : audit [DBG] from='client.? 172.21.15.6:0/2048193935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:40.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:17:40.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:40.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:17:40.155Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:40.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:40 smithi006 bash[27292]: cluster 2023-08-27T22:17:39.306752+0000 mgr.y (mgr.14901) 574 : cluster [DBG] pgmap v581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:40.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:40 smithi006 bash[33918]: cluster 2023-08-27T22:17:39.306752+0000 mgr.y (mgr.14901) 574 : cluster [DBG] pgmap v581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:40 smithi130 bash[27579]: cluster 2023-08-27T22:17:39.306752+0000 mgr.y (mgr.14901) 574 : cluster [DBG] pgmap v581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:41.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:17:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:17:40] "GET /metrics HTTP/1.1" 200 193067 "" "Prometheus/2.33.4" 2023-08-27T22:17:41.726 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:41 smithi006 bash[33918]: audit 2023-08-27T22:17:41.181813+0000 mon.c (mon.2) 239 : audit [DBG] from='client.? 172.21.15.6:0/1366485478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:41.727 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:17:41.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:41.727 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:41 smithi006 bash[27292]: audit 2023-08-27T22:17:41.181813+0000 mon.c (mon.2) 239 : audit [DBG] from='client.? 172.21.15.6:0/1366485478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:41.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:41 smithi130 bash[27579]: audit 2023-08-27T22:17:41.181813+0000 mon.c (mon.2) 239 : audit [DBG] from='client.? 172.21.15.6:0/1366485478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:42.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:17:41.720Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:42 smithi130 bash[27579]: cluster 2023-08-27T22:17:41.307831+0000 mgr.y (mgr.14901) 575 : cluster [DBG] pgmap v582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:42.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:42 smithi006 bash[33918]: cluster 2023-08-27T22:17:41.307831+0000 mgr.y (mgr.14901) 575 : cluster [DBG] pgmap v582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:42.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:42 smithi006 bash[27292]: cluster 2023-08-27T22:17:41.307831+0000 mgr.y (mgr.14901) 575 : cluster [DBG] pgmap v582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:43.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:17:43.612Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:43.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:17:43.615Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:44 smithi130 bash[27579]: cluster 2023-08-27T22:17:43.308219+0000 mgr.y (mgr.14901) 576 : cluster [DBG] pgmap v583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:44.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:44 smithi130 bash[27579]: audit 2023-08-27T22:17:43.605268+0000 mon.a (mon.0) 1126 : audit [DBG] from='client.? 172.21.15.6:0/2141942075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:44.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:44 smithi006 bash[33918]: cluster 2023-08-27T22:17:43.308219+0000 mgr.y (mgr.14901) 576 : cluster [DBG] pgmap v583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:44.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:44 smithi006 bash[33918]: audit 2023-08-27T22:17:43.605268+0000 mon.a (mon.0) 1126 : audit [DBG] from='client.? 172.21.15.6:0/2141942075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:44.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:44 smithi006 bash[27292]: cluster 2023-08-27T22:17:43.308219+0000 mgr.y (mgr.14901) 576 : cluster [DBG] pgmap v583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:44.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:44 smithi006 bash[27292]: audit 2023-08-27T22:17:43.605268+0000 mon.a (mon.0) 1126 : audit [DBG] from='client.? 172.21.15.6:0/2141942075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:46 smithi130 bash[27579]: cluster 2023-08-27T22:17:45.309272+0000 mgr.y (mgr.14901) 577 : cluster [DBG] pgmap v584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:46.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:46 smithi130 bash[27579]: audit 2023-08-27T22:17:46.024688+0000 mon.a (mon.0) 1127 : audit [DBG] from='client.? 172.21.15.6:0/3003972488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:46.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:46 smithi006 bash[27292]: cluster 2023-08-27T22:17:45.309272+0000 mgr.y (mgr.14901) 577 : cluster [DBG] pgmap v584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:46.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:46 smithi006 bash[27292]: audit 2023-08-27T22:17:46.024688+0000 mon.a (mon.0) 1127 : audit [DBG] from='client.? 172.21.15.6:0/3003972488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:46.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:46 smithi006 bash[33918]: cluster 2023-08-27T22:17:45.309272+0000 mgr.y (mgr.14901) 577 : cluster [DBG] pgmap v584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:46.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:46 smithi006 bash[33918]: audit 2023-08-27T22:17:46.024688+0000 mon.a (mon.0) 1127 : audit [DBG] from='client.? 172.21.15.6:0/3003972488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:48 smithi130 bash[27579]: cluster 2023-08-27T22:17:47.310044+0000 mgr.y (mgr.14901) 578 : cluster [DBG] pgmap v585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:48.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:48 smithi130 bash[27579]: audit 2023-08-27T22:17:48.446013+0000 mon.c (mon.2) 240 : audit [DBG] from='client.? 172.21.15.6:0/3958471217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:48.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:48 smithi006 bash[27292]: cluster 2023-08-27T22:17:47.310044+0000 mgr.y (mgr.14901) 578 : cluster [DBG] pgmap v585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:48.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:48 smithi006 bash[27292]: audit 2023-08-27T22:17:48.446013+0000 mon.c (mon.2) 240 : audit [DBG] from='client.? 172.21.15.6:0/3958471217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:48.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:48 smithi006 bash[33918]: cluster 2023-08-27T22:17:47.310044+0000 mgr.y (mgr.14901) 578 : cluster [DBG] pgmap v585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:48.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:48 smithi006 bash[33918]: audit 2023-08-27T22:17:48.446013+0000 mon.c (mon.2) 240 : audit [DBG] from='client.? 172.21.15.6:0/3958471217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:50.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:17:50.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:50.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:17:50.155Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:50.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:50 smithi006 bash[27292]: cluster 2023-08-27T22:17:49.310814+0000 mgr.y (mgr.14901) 579 : cluster [DBG] pgmap v586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:50.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:50 smithi006 bash[33918]: cluster 2023-08-27T22:17:49.310814+0000 mgr.y (mgr.14901) 579 : cluster [DBG] pgmap v586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:50 smithi130 bash[27579]: cluster 2023-08-27T22:17:49.310814+0000 mgr.y (mgr.14901) 579 : cluster [DBG] pgmap v586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:51.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:17:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:17:50] "GET /metrics HTTP/1.1" 200 193065 "" "Prometheus/2.33.4" 2023-08-27T22:17:51.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:51 smithi130 bash[27579]: audit 2023-08-27T22:17:50.860330+0000 mon.c (mon.2) 241 : audit [DBG] from='client.? 172.21.15.6:0/3621944902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:51.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:51 smithi006 bash[33918]: audit 2023-08-27T22:17:50.860330+0000 mon.c (mon.2) 241 : audit [DBG] from='client.? 172.21.15.6:0/3621944902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:51.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:17:51.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:51.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:17:51.721Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:51.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:51 smithi006 bash[27292]: audit 2023-08-27T22:17:50.860330+0000 mon.c (mon.2) 241 : audit [DBG] from='client.? 172.21.15.6:0/3621944902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:52 smithi130 bash[27579]: cluster 2023-08-27T22:17:51.311865+0000 mgr.y (mgr.14901) 580 : cluster [DBG] pgmap v587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:52.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:52 smithi006 bash[27292]: cluster 2023-08-27T22:17:51.311865+0000 mgr.y (mgr.14901) 580 : cluster [DBG] pgmap v587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:52.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:52 smithi006 bash[33918]: cluster 2023-08-27T22:17:51.311865+0000 mgr.y (mgr.14901) 580 : cluster [DBG] pgmap v587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:53.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:53 smithi130 bash[27579]: audit 2023-08-27T22:17:53.278480+0000 mon.c (mon.2) 242 : audit [DBG] from='client.? 172.21.15.6:0/12733470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:53.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:53 smithi006 bash[27292]: audit 2023-08-27T22:17:53.278480+0000 mon.c (mon.2) 242 : audit [DBG] from='client.? 172.21.15.6:0/12733470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:53.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:53 smithi006 bash[33918]: audit 2023-08-27T22:17:53.278480+0000 mon.c (mon.2) 242 : audit [DBG] from='client.? 172.21.15.6:0/12733470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:17:53.613Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:17:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:17:53.615Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:17:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:54 smithi130 bash[27579]: cluster 2023-08-27T22:17:53.312503+0000 mgr.y (mgr.14901) 581 : cluster [DBG] pgmap v588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:54.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:54 smithi006 bash[27292]: cluster 2023-08-27T22:17:53.312503+0000 mgr.y (mgr.14901) 581 : cluster [DBG] pgmap v588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:54.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:54 smithi006 bash[33918]: cluster 2023-08-27T22:17:53.312503+0000 mgr.y (mgr.14901) 581 : cluster [DBG] pgmap v588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:56 smithi130 bash[27579]: cluster 2023-08-27T22:17:55.313386+0000 mgr.y (mgr.14901) 582 : cluster [DBG] pgmap v589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:56.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:56 smithi130 bash[27579]: audit 2023-08-27T22:17:55.689601+0000 mon.c (mon.2) 243 : audit [DBG] from='client.? 172.21.15.6:0/1745870119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:56.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:56 smithi006 bash[27292]: cluster 2023-08-27T22:17:55.313386+0000 mgr.y (mgr.14901) 582 : cluster [DBG] pgmap v589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:56.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:56 smithi006 bash[27292]: audit 2023-08-27T22:17:55.689601+0000 mon.c (mon.2) 243 : audit [DBG] from='client.? 172.21.15.6:0/1745870119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:56.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:56 smithi006 bash[33918]: cluster 2023-08-27T22:17:55.313386+0000 mgr.y (mgr.14901) 582 : cluster [DBG] pgmap v589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:56.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:56 smithi006 bash[33918]: audit 2023-08-27T22:17:55.689601+0000 mon.c (mon.2) 243 : audit [DBG] from='client.? 172.21.15.6:0/1745870119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:58 smithi130 bash[27579]: cluster 2023-08-27T22:17:57.314105+0000 mgr.y (mgr.14901) 583 : cluster [DBG] pgmap v590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:58.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:17:58 smithi130 bash[27579]: audit 2023-08-27T22:17:58.113676+0000 mon.a (mon.0) 1128 : audit [DBG] from='client.? 172.21.15.6:0/1514130132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:58.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:58 smithi006 bash[27292]: cluster 2023-08-27T22:17:57.314105+0000 mgr.y (mgr.14901) 583 : cluster [DBG] pgmap v590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:58.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:17:58 smithi006 bash[27292]: audit 2023-08-27T22:17:58.113676+0000 mon.a (mon.0) 1128 : audit [DBG] from='client.? 172.21.15.6:0/1514130132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:58.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:58 smithi006 bash[33918]: cluster 2023-08-27T22:17:57.314105+0000 mgr.y (mgr.14901) 583 : cluster [DBG] pgmap v590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:17:58.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:17:58 smithi006 bash[33918]: audit 2023-08-27T22:17:58.113676+0000 mon.a (mon.0) 1128 : audit [DBG] from='client.? 172.21.15.6:0/1514130132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:00.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:18:00.154Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:00.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:18:00.156Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:00.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:00 smithi130 bash[27579]: cluster 2023-08-27T22:17:59.314852+0000 mgr.y (mgr.14901) 584 : cluster [DBG] pgmap v591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:00.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:18:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:18:00] "GET /metrics HTTP/1.1" 200 193064 "" "Prometheus/2.33.4" 2023-08-27T22:18:00.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:00 smithi006 bash[33918]: cluster 2023-08-27T22:17:59.314852+0000 mgr.y (mgr.14901) 584 : cluster [DBG] pgmap v591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:00.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:00 smithi006 bash[27292]: cluster 2023-08-27T22:17:59.314852+0000 mgr.y (mgr.14901) 584 : cluster [DBG] pgmap v591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:01.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:01 smithi130 bash[27579]: audit 2023-08-27T22:18:00.535617+0000 mon.a (mon.0) 1129 : audit [DBG] from='client.? 172.21.15.6:0/17028625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:01.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:01 smithi006 bash[33918]: audit 2023-08-27T22:18:00.535617+0000 mon.a (mon.0) 1129 : audit [DBG] from='client.? 172.21.15.6:0/17028625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:01.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:01 smithi006 bash[27292]: audit 2023-08-27T22:18:00.535617+0000 mon.a (mon.0) 1129 : audit [DBG] from='client.? 172.21.15.6:0/17028625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:01.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:18:01.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:01.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:18:01.722Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:02 smithi130 bash[27579]: cluster 2023-08-27T22:18:01.315888+0000 mgr.y (mgr.14901) 585 : cluster [DBG] pgmap v592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:02.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:02 smithi006 bash[33918]: cluster 2023-08-27T22:18:01.315888+0000 mgr.y (mgr.14901) 585 : cluster [DBG] pgmap v592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:02.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:02 smithi006 bash[27292]: cluster 2023-08-27T22:18:01.315888+0000 mgr.y (mgr.14901) 585 : cluster [DBG] pgmap v592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:03.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:03 smithi130 bash[27579]: audit 2023-08-27T22:18:02.950150+0000 mon.c (mon.2) 244 : audit [DBG] from='client.? 172.21.15.6:0/3563972974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:03.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:03 smithi006 bash[27292]: audit 2023-08-27T22:18:02.950150+0000 mon.c (mon.2) 244 : audit [DBG] from='client.? 172.21.15.6:0/3563972974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:03.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:03 smithi006 bash[33918]: audit 2023-08-27T22:18:02.950150+0000 mon.c (mon.2) 244 : audit [DBG] from='client.? 172.21.15.6:0/3563972974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:18:03.613Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:18:03.615Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:04 smithi130 bash[27579]: cluster 2023-08-27T22:18:03.316533+0000 mgr.y (mgr.14901) 586 : cluster [DBG] pgmap v593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:04.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:04 smithi006 bash[33918]: cluster 2023-08-27T22:18:03.316533+0000 mgr.y (mgr.14901) 586 : cluster [DBG] pgmap v593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:04.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:04 smithi006 bash[27292]: cluster 2023-08-27T22:18:03.316533+0000 mgr.y (mgr.14901) 586 : cluster [DBG] pgmap v593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:05.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:05 smithi130 bash[27579]: audit 2023-08-27T22:18:05.361340+0000 mon.c (mon.2) 245 : audit [DBG] from='client.? 172.21.15.6:0/3220151546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:05.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:05 smithi006 bash[33918]: audit 2023-08-27T22:18:05.361340+0000 mon.c (mon.2) 245 : audit [DBG] from='client.? 172.21.15.6:0/3220151546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:05.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:05 smithi006 bash[27292]: audit 2023-08-27T22:18:05.361340+0000 mon.c (mon.2) 245 : audit [DBG] from='client.? 172.21.15.6:0/3220151546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:06 smithi130 bash[27579]: cluster 2023-08-27T22:18:05.317538+0000 mgr.y (mgr.14901) 587 : cluster [DBG] pgmap v594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:06.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:06 smithi006 bash[27292]: cluster 2023-08-27T22:18:05.317538+0000 mgr.y (mgr.14901) 587 : cluster [DBG] pgmap v594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:06.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:06 smithi006 bash[33918]: cluster 2023-08-27T22:18:05.317538+0000 mgr.y (mgr.14901) 587 : cluster [DBG] pgmap v594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:08.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:08 smithi130 bash[27579]: cluster 2023-08-27T22:18:07.318301+0000 mgr.y (mgr.14901) 588 : cluster [DBG] pgmap v595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:08.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:08 smithi130 bash[27579]: audit 2023-08-27T22:18:07.774729+0000 mon.c (mon.2) 246 : audit [DBG] from='client.? 172.21.15.6:0/1525203250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:08.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:08 smithi006 bash[33918]: cluster 2023-08-27T22:18:07.318301+0000 mgr.y (mgr.14901) 588 : cluster [DBG] pgmap v595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:08.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:08 smithi006 bash[33918]: audit 2023-08-27T22:18:07.774729+0000 mon.c (mon.2) 246 : audit [DBG] from='client.? 172.21.15.6:0/1525203250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:08.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:08 smithi006 bash[27292]: cluster 2023-08-27T22:18:07.318301+0000 mgr.y (mgr.14901) 588 : cluster [DBG] pgmap v595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:08.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:08 smithi006 bash[27292]: audit 2023-08-27T22:18:07.774729+0000 mon.c (mon.2) 246 : audit [DBG] from='client.? 172.21.15.6:0/1525203250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:10.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:18:10.155Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:10.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:18:10.157Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:10.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:10 smithi130 bash[27579]: cluster 2023-08-27T22:18:09.319067+0000 mgr.y (mgr.14901) 589 : cluster [DBG] pgmap v596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:10.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:10 smithi130 bash[27579]: audit 2023-08-27T22:18:10.194949+0000 mon.c (mon.2) 247 : audit [DBG] from='client.? 172.21.15.6:0/88249379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:10.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:18:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:18:10] "GET /metrics HTTP/1.1" 200 193064 "" "Prometheus/2.33.4" 2023-08-27T22:18:10.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:10 smithi006 bash[27292]: cluster 2023-08-27T22:18:09.319067+0000 mgr.y (mgr.14901) 589 : cluster [DBG] pgmap v596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:10.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:10 smithi006 bash[27292]: audit 2023-08-27T22:18:10.194949+0000 mon.c (mon.2) 247 : audit [DBG] from='client.? 172.21.15.6:0/88249379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:10.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:10 smithi006 bash[33918]: cluster 2023-08-27T22:18:09.319067+0000 mgr.y (mgr.14901) 589 : cluster [DBG] pgmap v596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:10.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:10 smithi006 bash[33918]: audit 2023-08-27T22:18:10.194949+0000 mon.c (mon.2) 247 : audit [DBG] from='client.? 172.21.15.6:0/88249379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:12.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:18:11.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:12.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:18:11.723Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:12 smithi130 bash[27579]: cluster 2023-08-27T22:18:11.320074+0000 mgr.y (mgr.14901) 590 : cluster [DBG] pgmap v597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:12.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:12 smithi006 bash[27292]: cluster 2023-08-27T22:18:11.320074+0000 mgr.y (mgr.14901) 590 : cluster [DBG] pgmap v597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:12.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:12 smithi006 bash[33918]: cluster 2023-08-27T22:18:11.320074+0000 mgr.y (mgr.14901) 590 : cluster [DBG] pgmap v597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:13.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:13 smithi130 bash[27579]: audit 2023-08-27T22:18:12.614359+0000 mon.c (mon.2) 248 : audit [DBG] from='client.? 172.21.15.6:0/170453766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:13.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:13 smithi006 bash[27292]: audit 2023-08-27T22:18:12.614359+0000 mon.c (mon.2) 248 : audit [DBG] from='client.? 172.21.15.6:0/170453766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:13.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:13 smithi006 bash[33918]: audit 2023-08-27T22:18:12.614359+0000 mon.c (mon.2) 248 : audit [DBG] from='client.? 172.21.15.6:0/170453766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:18:13.614Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:18:13.616Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:14 smithi130 bash[27579]: cluster 2023-08-27T22:18:13.320748+0000 mgr.y (mgr.14901) 591 : cluster [DBG] pgmap v598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:14.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:14 smithi006 bash[27292]: cluster 2023-08-27T22:18:13.320748+0000 mgr.y (mgr.14901) 591 : cluster [DBG] pgmap v598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:14.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:14 smithi006 bash[33918]: cluster 2023-08-27T22:18:13.320748+0000 mgr.y (mgr.14901) 591 : cluster [DBG] pgmap v598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:15.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:15 smithi130 bash[27579]: audit 2023-08-27T22:18:15.025666+0000 mon.a (mon.0) 1130 : audit [DBG] from='client.? 172.21.15.6:0/523116458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:15.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:15 smithi006 bash[27292]: audit 2023-08-27T22:18:15.025666+0000 mon.a (mon.0) 1130 : audit [DBG] from='client.? 172.21.15.6:0/523116458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:15.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:15 smithi006 bash[33918]: audit 2023-08-27T22:18:15.025666+0000 mon.a (mon.0) 1130 : audit [DBG] from='client.? 172.21.15.6:0/523116458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:16 smithi130 bash[27579]: cluster 2023-08-27T22:18:15.321688+0000 mgr.y (mgr.14901) 592 : cluster [DBG] pgmap v599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:16.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:16 smithi006 bash[27292]: cluster 2023-08-27T22:18:15.321688+0000 mgr.y (mgr.14901) 592 : cluster [DBG] pgmap v599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:16.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:16 smithi006 bash[33918]: cluster 2023-08-27T22:18:15.321688+0000 mgr.y (mgr.14901) 592 : cluster [DBG] pgmap v599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:17.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:17 smithi006 bash[27292]: audit 2023-08-27T22:18:17.445838+0000 mon.c (mon.2) 249 : audit [DBG] from='client.? 172.21.15.6:0/2654109766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:17.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:17 smithi006 bash[33918]: audit 2023-08-27T22:18:17.445838+0000 mon.c (mon.2) 249 : audit [DBG] from='client.? 172.21.15.6:0/2654109766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:18.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:17 smithi130 bash[27579]: audit 2023-08-27T22:18:17.445838+0000 mon.c (mon.2) 249 : audit [DBG] from='client.? 172.21.15.6:0/2654109766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:18.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:18 smithi006 bash[27292]: cluster 2023-08-27T22:18:17.322365+0000 mgr.y (mgr.14901) 593 : cluster [DBG] pgmap v600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:18.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:18 smithi006 bash[33918]: cluster 2023-08-27T22:18:17.322365+0000 mgr.y (mgr.14901) 593 : cluster [DBG] pgmap v600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:18 smithi130 bash[27579]: cluster 2023-08-27T22:18:17.322365+0000 mgr.y (mgr.14901) 593 : cluster [DBG] pgmap v600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:20.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:18:20.156Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:20.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:18:20.158Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:20.971 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:18:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:18:20] "GET /metrics HTTP/1.1" 200 193066 "" "Prometheus/2.33.4" 2023-08-27T22:18:20.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:20 smithi006 bash[33918]: cluster 2023-08-27T22:18:19.323083+0000 mgr.y (mgr.14901) 594 : cluster [DBG] pgmap v601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:20.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:20 smithi006 bash[33918]: audit 2023-08-27T22:18:19.863856+0000 mon.c (mon.2) 250 : audit [DBG] from='client.? 172.21.15.6:0/1595735786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:20.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:20 smithi006 bash[33918]: audit 2023-08-27T22:18:20.009531+0000 mon.b (mon.1) 184 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:18:20.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:20 smithi006 bash[27292]: cluster 2023-08-27T22:18:19.323083+0000 mgr.y (mgr.14901) 594 : cluster [DBG] pgmap v601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:20.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:20 smithi006 bash[27292]: audit 2023-08-27T22:18:19.863856+0000 mon.c (mon.2) 250 : audit [DBG] from='client.? 172.21.15.6:0/1595735786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:20.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:20 smithi006 bash[27292]: audit 2023-08-27T22:18:20.009531+0000 mon.b (mon.1) 184 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:18:21.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:20 smithi130 bash[27579]: cluster 2023-08-27T22:18:19.323083+0000 mgr.y (mgr.14901) 594 : cluster [DBG] pgmap v601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:21.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:20 smithi130 bash[27579]: audit 2023-08-27T22:18:19.863856+0000 mon.c (mon.2) 250 : audit [DBG] from='client.? 172.21.15.6:0/1595735786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:21.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:20 smithi130 bash[27579]: audit 2023-08-27T22:18:20.009531+0000 mon.b (mon.1) 184 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:18:22.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:18:21.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:22.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:18:21.723Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:22.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:22 smithi006 bash[27292]: cluster 2023-08-27T22:18:21.324135+0000 mgr.y (mgr.14901) 595 : cluster [DBG] pgmap v602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:23.006 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:22 smithi006 bash[27292]: audit 2023-08-27T22:18:22.284495+0000 mon.c (mon.2) 251 : audit [DBG] from='client.? 172.21.15.6:0/1981441904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:23.006 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:22 smithi006 bash[33918]: cluster 2023-08-27T22:18:21.324135+0000 mgr.y (mgr.14901) 595 : cluster [DBG] pgmap v602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:23.006 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:22 smithi006 bash[33918]: audit 2023-08-27T22:18:22.284495+0000 mon.c (mon.2) 251 : audit [DBG] from='client.? 172.21.15.6:0/1981441904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:22 smithi130 bash[27579]: cluster 2023-08-27T22:18:21.324135+0000 mgr.y (mgr.14901) 595 : cluster [DBG] pgmap v602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:23.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:22 smithi130 bash[27579]: audit 2023-08-27T22:18:22.284495+0000 mon.c (mon.2) 251 : audit [DBG] from='client.? 172.21.15.6:0/1981441904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:23.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:18:23.615Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:23.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:18:23.617Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:24.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:24 smithi006 bash[33918]: cluster 2023-08-27T22:18:23.324962+0000 mgr.y (mgr.14901) 596 : cluster [DBG] pgmap v603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:24.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:24 smithi006 bash[27292]: cluster 2023-08-27T22:18:23.324962+0000 mgr.y (mgr.14901) 596 : cluster [DBG] pgmap v603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:24 smithi130 bash[27579]: cluster 2023-08-27T22:18:23.324962+0000 mgr.y (mgr.14901) 596 : cluster [DBG] pgmap v603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:25.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:25 smithi006 bash[33918]: audit 2023-08-27T22:18:24.705058+0000 mon.a (mon.0) 1131 : audit [DBG] from='client.? 172.21.15.6:0/488389897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:25.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:25 smithi006 bash[27292]: audit 2023-08-27T22:18:24.705058+0000 mon.a (mon.0) 1131 : audit [DBG] from='client.? 172.21.15.6:0/488389897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:25 smithi130 bash[27579]: audit 2023-08-27T22:18:24.705058+0000 mon.a (mon.0) 1131 : audit [DBG] from='client.? 172.21.15.6:0/488389897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:27.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:26 smithi006 bash[33918]: cluster 2023-08-27T22:18:25.325955+0000 mgr.y (mgr.14901) 597 : cluster [DBG] pgmap v604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:27.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:26 smithi006 bash[33918]: audit 2023-08-27T22:18:25.935973+0000 mon.a (mon.0) 1132 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:18:27.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:26 smithi006 bash[33918]: audit 2023-08-27T22:18:26.325477+0000 mon.a (mon.0) 1133 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:18:27.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:26 smithi006 bash[33918]: audit 2023-08-27T22:18:26.591394+0000 mon.b (mon.1) 185 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:18:27.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:26 smithi006 bash[33918]: audit 2023-08-27T22:18:26.593453+0000 mon.b (mon.1) 186 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:18:27.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:26 smithi006 bash[33918]: audit 2023-08-27T22:18:26.599239+0000 mon.a (mon.0) 1134 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:18:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:26 smithi006 bash[27292]: cluster 2023-08-27T22:18:25.325955+0000 mgr.y (mgr.14901) 597 : cluster [DBG] pgmap v604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:26 smithi006 bash[27292]: audit 2023-08-27T22:18:25.935973+0000 mon.a (mon.0) 1132 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:18:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:26 smithi006 bash[27292]: audit 2023-08-27T22:18:26.325477+0000 mon.a (mon.0) 1133 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:18:27.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:26 smithi006 bash[27292]: audit 2023-08-27T22:18:26.591394+0000 mon.b (mon.1) 185 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:18:27.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:26 smithi006 bash[27292]: audit 2023-08-27T22:18:26.593453+0000 mon.b (mon.1) 186 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:18:27.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:26 smithi006 bash[27292]: audit 2023-08-27T22:18:26.599239+0000 mon.a (mon.0) 1134 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:18:27.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:26 smithi130 bash[27579]: cluster 2023-08-27T22:18:25.325955+0000 mgr.y (mgr.14901) 597 : cluster [DBG] pgmap v604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:27.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:26 smithi130 bash[27579]: audit 2023-08-27T22:18:25.935973+0000 mon.a (mon.0) 1132 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:18:27.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:26 smithi130 bash[27579]: audit 2023-08-27T22:18:26.325477+0000 mon.a (mon.0) 1133 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:18:27.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:26 smithi130 bash[27579]: audit 2023-08-27T22:18:26.591394+0000 mon.b (mon.1) 185 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:18:27.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:26 smithi130 bash[27579]: audit 2023-08-27T22:18:26.593453+0000 mon.b (mon.1) 186 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:18:27.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:26 smithi130 bash[27579]: audit 2023-08-27T22:18:26.599239+0000 mon.a (mon.0) 1134 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:18:28.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:27 smithi006 bash[27292]: audit 2023-08-27T22:18:27.124193+0000 mon.c (mon.2) 252 : audit [DBG] from='client.? 172.21.15.6:0/804064491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:28.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:27 smithi006 bash[33918]: audit 2023-08-27T22:18:27.124193+0000 mon.c (mon.2) 252 : audit [DBG] from='client.? 172.21.15.6:0/804064491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:28.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:27 smithi130 bash[27579]: audit 2023-08-27T22:18:27.124193+0000 mon.c (mon.2) 252 : audit [DBG] from='client.? 172.21.15.6:0/804064491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:29.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:28 smithi006 bash[27292]: cluster 2023-08-27T22:18:27.326726+0000 mgr.y (mgr.14901) 598 : cluster [DBG] pgmap v605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:29.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:28 smithi006 bash[33918]: cluster 2023-08-27T22:18:27.326726+0000 mgr.y (mgr.14901) 598 : cluster [DBG] pgmap v605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:29.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:28 smithi130 bash[27579]: cluster 2023-08-27T22:18:27.326726+0000 mgr.y (mgr.14901) 598 : cluster [DBG] pgmap v605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:30.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:29 smithi006 bash[27292]: audit 2023-08-27T22:18:29.543570+0000 mon.c (mon.2) 253 : audit [DBG] from='client.? 172.21.15.6:0/669807258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:30.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:29 smithi006 bash[33918]: audit 2023-08-27T22:18:29.543570+0000 mon.c (mon.2) 253 : audit [DBG] from='client.? 172.21.15.6:0/669807258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:30.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:18:30.157Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:30.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:18:30.159Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:29 smithi130 bash[27579]: audit 2023-08-27T22:18:29.543570+0000 mon.c (mon.2) 253 : audit [DBG] from='client.? 172.21.15.6:0/669807258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:31.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:18:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:18:30] "GET /metrics HTTP/1.1" 200 193068 "" "Prometheus/2.33.4" 2023-08-27T22:18:31.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:30 smithi006 bash[33918]: cluster 2023-08-27T22:18:29.327460+0000 mgr.y (mgr.14901) 599 : cluster [DBG] pgmap v606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:31.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:30 smithi006 bash[27292]: cluster 2023-08-27T22:18:29.327460+0000 mgr.y (mgr.14901) 599 : cluster [DBG] pgmap v606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:31.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:30 smithi130 bash[27579]: cluster 2023-08-27T22:18:29.327460+0000 mgr.y (mgr.14901) 599 : cluster [DBG] pgmap v606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:32.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:18:31.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:18:31.723Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:33.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:32 smithi006 bash[27292]: cluster 2023-08-27T22:18:31.328410+0000 mgr.y (mgr.14901) 600 : cluster [DBG] pgmap v607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:32 smithi006 bash[27292]: audit 2023-08-27T22:18:31.967187+0000 mon.c (mon.2) 254 : audit [DBG] from='client.? 172.21.15.6:0/4198204987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:33.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:32 smithi006 bash[33918]: cluster 2023-08-27T22:18:31.328410+0000 mgr.y (mgr.14901) 600 : cluster [DBG] pgmap v607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:33.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:32 smithi006 bash[33918]: audit 2023-08-27T22:18:31.967187+0000 mon.c (mon.2) 254 : audit [DBG] from='client.? 172.21.15.6:0/4198204987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:32 smithi130 bash[27579]: cluster 2023-08-27T22:18:31.328410+0000 mgr.y (mgr.14901) 600 : cluster [DBG] pgmap v607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:33.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:32 smithi130 bash[27579]: audit 2023-08-27T22:18:31.967187+0000 mon.c (mon.2) 254 : audit [DBG] from='client.? 172.21.15.6:0/4198204987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:18:33.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:18:33.618Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:35.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:34 smithi130 bash[27579]: cluster 2023-08-27T22:18:33.329083+0000 mgr.y (mgr.14901) 601 : cluster [DBG] pgmap v608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:35.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:34 smithi130 bash[27579]: audit 2023-08-27T22:18:34.384514+0000 mon.a (mon.0) 1135 : audit [DBG] from='client.? 172.21.15.6:0/1941719632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:35.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:34 smithi006 bash[27292]: cluster 2023-08-27T22:18:33.329083+0000 mgr.y (mgr.14901) 601 : cluster [DBG] pgmap v608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:35.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:34 smithi006 bash[27292]: audit 2023-08-27T22:18:34.384514+0000 mon.a (mon.0) 1135 : audit [DBG] from='client.? 172.21.15.6:0/1941719632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:35.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:34 smithi006 bash[33918]: cluster 2023-08-27T22:18:33.329083+0000 mgr.y (mgr.14901) 601 : cluster [DBG] pgmap v608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:35.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:34 smithi006 bash[33918]: audit 2023-08-27T22:18:34.384514+0000 mon.a (mon.0) 1135 : audit [DBG] from='client.? 172.21.15.6:0/1941719632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:37.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:36 smithi130 bash[27579]: cluster 2023-08-27T22:18:35.330113+0000 mgr.y (mgr.14901) 602 : cluster [DBG] pgmap v609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:37.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:36 smithi130 bash[27579]: audit 2023-08-27T22:18:36.803546+0000 mon.a (mon.0) 1136 : audit [DBG] from='client.? 172.21.15.6:0/2650927405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:37.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:36 smithi006 bash[27292]: cluster 2023-08-27T22:18:35.330113+0000 mgr.y (mgr.14901) 602 : cluster [DBG] pgmap v609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:37.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:36 smithi006 bash[27292]: audit 2023-08-27T22:18:36.803546+0000 mon.a (mon.0) 1136 : audit [DBG] from='client.? 172.21.15.6:0/2650927405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:37.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:36 smithi006 bash[33918]: cluster 2023-08-27T22:18:35.330113+0000 mgr.y (mgr.14901) 602 : cluster [DBG] pgmap v609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:37.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:36 smithi006 bash[33918]: audit 2023-08-27T22:18:36.803546+0000 mon.a (mon.0) 1136 : audit [DBG] from='client.? 172.21.15.6:0/2650927405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:38 smithi130 bash[27579]: cluster 2023-08-27T22:18:37.330867+0000 mgr.y (mgr.14901) 603 : cluster [DBG] pgmap v610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:39.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:38 smithi006 bash[27292]: cluster 2023-08-27T22:18:37.330867+0000 mgr.y (mgr.14901) 603 : cluster [DBG] pgmap v610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:39.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:38 smithi006 bash[33918]: cluster 2023-08-27T22:18:37.330867+0000 mgr.y (mgr.14901) 603 : cluster [DBG] pgmap v610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:40.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:39 smithi130 bash[27579]: audit 2023-08-27T22:18:39.217918+0000 mon.a (mon.0) 1137 : audit [DBG] from='client.? 172.21.15.6:0/4255929223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:40.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:39 smithi006 bash[33918]: audit 2023-08-27T22:18:39.217918+0000 mon.a (mon.0) 1137 : audit [DBG] from='client.? 172.21.15.6:0/4255929223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:40.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:18:40.157Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:40.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:18:40.160Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:40.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:39 smithi006 bash[27292]: audit 2023-08-27T22:18:39.217918+0000 mon.a (mon.0) 1137 : audit [DBG] from='client.? 172.21.15.6:0/4255929223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:41.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:18:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:18:40] "GET /metrics HTTP/1.1" 200 193068 "" "Prometheus/2.33.4" 2023-08-27T22:18:41.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:40 smithi006 bash[33918]: cluster 2023-08-27T22:18:39.331648+0000 mgr.y (mgr.14901) 604 : cluster [DBG] pgmap v611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:41.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:40 smithi006 bash[27292]: cluster 2023-08-27T22:18:39.331648+0000 mgr.y (mgr.14901) 604 : cluster [DBG] pgmap v611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:41.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:40 smithi130 bash[27579]: cluster 2023-08-27T22:18:39.331648+0000 mgr.y (mgr.14901) 604 : cluster [DBG] pgmap v611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:42.006 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:18:41.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:42.006 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:18:41.724Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:42 smithi130 bash[27579]: cluster 2023-08-27T22:18:41.332696+0000 mgr.y (mgr.14901) 605 : cluster [DBG] pgmap v612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:42.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:42 smithi130 bash[27579]: audit 2023-08-27T22:18:41.631162+0000 mon.c (mon.2) 255 : audit [DBG] from='client.? 172.21.15.6:0/1024377548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:42.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:42 smithi006 bash[33918]: cluster 2023-08-27T22:18:41.332696+0000 mgr.y (mgr.14901) 605 : cluster [DBG] pgmap v612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:42.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:42 smithi006 bash[33918]: audit 2023-08-27T22:18:41.631162+0000 mon.c (mon.2) 255 : audit [DBG] from='client.? 172.21.15.6:0/1024377548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:42.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:42 smithi006 bash[27292]: cluster 2023-08-27T22:18:41.332696+0000 mgr.y (mgr.14901) 605 : cluster [DBG] pgmap v612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:42.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:42 smithi006 bash[27292]: audit 2023-08-27T22:18:41.631162+0000 mon.c (mon.2) 255 : audit [DBG] from='client.? 172.21.15.6:0/1024377548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:43.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:18:43.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:43.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:18:43.618Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:44.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:44 smithi006 bash[27292]: cluster 2023-08-27T22:18:43.333412+0000 mgr.y (mgr.14901) 606 : cluster [DBG] pgmap v613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:44.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:44 smithi006 bash[27292]: audit 2023-08-27T22:18:44.043751+0000 mon.c (mon.2) 256 : audit [DBG] from='client.? 172.21.15.6:0/2894614644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:44.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:44 smithi006 bash[33918]: cluster 2023-08-27T22:18:43.333412+0000 mgr.y (mgr.14901) 606 : cluster [DBG] pgmap v613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:44.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:44 smithi006 bash[33918]: audit 2023-08-27T22:18:44.043751+0000 mon.c (mon.2) 256 : audit [DBG] from='client.? 172.21.15.6:0/2894614644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:44.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:44 smithi130 bash[27579]: cluster 2023-08-27T22:18:43.333412+0000 mgr.y (mgr.14901) 606 : cluster [DBG] pgmap v613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:44.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:44 smithi130 bash[27579]: audit 2023-08-27T22:18:44.043751+0000 mon.c (mon.2) 256 : audit [DBG] from='client.? 172.21.15.6:0/2894614644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:46.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:46 smithi006 bash[33918]: cluster 2023-08-27T22:18:45.334462+0000 mgr.y (mgr.14901) 607 : cluster [DBG] pgmap v614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:46.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:46 smithi006 bash[27292]: cluster 2023-08-27T22:18:45.334462+0000 mgr.y (mgr.14901) 607 : cluster [DBG] pgmap v614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:46 smithi130 bash[27579]: cluster 2023-08-27T22:18:45.334462+0000 mgr.y (mgr.14901) 607 : cluster [DBG] pgmap v614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:47.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:47 smithi006 bash[33918]: audit 2023-08-27T22:18:46.458791+0000 mon.a (mon.0) 1138 : audit [DBG] from='client.? 172.21.15.6:0/920947582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:47.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:47 smithi006 bash[27292]: audit 2023-08-27T22:18:46.458791+0000 mon.a (mon.0) 1138 : audit [DBG] from='client.? 172.21.15.6:0/920947582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:47.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:47 smithi130 bash[27579]: audit 2023-08-27T22:18:46.458791+0000 mon.a (mon.0) 1138 : audit [DBG] from='client.? 172.21.15.6:0/920947582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:48.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:48 smithi006 bash[33918]: cluster 2023-08-27T22:18:47.335195+0000 mgr.y (mgr.14901) 608 : cluster [DBG] pgmap v615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:48.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:48 smithi006 bash[27292]: cluster 2023-08-27T22:18:47.335195+0000 mgr.y (mgr.14901) 608 : cluster [DBG] pgmap v615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:48 smithi130 bash[27579]: cluster 2023-08-27T22:18:47.335195+0000 mgr.y (mgr.14901) 608 : cluster [DBG] pgmap v615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:49.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:49 smithi006 bash[33918]: audit 2023-08-27T22:18:48.873039+0000 mon.a (mon.0) 1139 : audit [DBG] from='client.? 172.21.15.6:0/4250208100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:49.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:49 smithi006 bash[27292]: audit 2023-08-27T22:18:48.873039+0000 mon.a (mon.0) 1139 : audit [DBG] from='client.? 172.21.15.6:0/4250208100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:49.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:49 smithi130 bash[27579]: audit 2023-08-27T22:18:48.873039+0000 mon.a (mon.0) 1139 : audit [DBG] from='client.? 172.21.15.6:0/4250208100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:50.443 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:18:50.158Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:50.443 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:18:50.160Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:50.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:50 smithi006 bash[33918]: cluster 2023-08-27T22:18:49.335975+0000 mgr.y (mgr.14901) 609 : cluster [DBG] pgmap v616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:50.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:50 smithi006 bash[27292]: cluster 2023-08-27T22:18:49.335975+0000 mgr.y (mgr.14901) 609 : cluster [DBG] pgmap v616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:50.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:50 smithi130 bash[27579]: cluster 2023-08-27T22:18:49.335975+0000 mgr.y (mgr.14901) 609 : cluster [DBG] pgmap v616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:51.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:18:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:18:50] "GET /metrics HTTP/1.1" 200 193065 "" "Prometheus/2.33.4" 2023-08-27T22:18:51.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:51 smithi006 bash[27292]: audit 2023-08-27T22:18:51.285757+0000 mon.a (mon.0) 1140 : audit [DBG] from='client.? 172.21.15.6:0/2681917551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:51.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:51 smithi006 bash[33918]: audit 2023-08-27T22:18:51.285757+0000 mon.a (mon.0) 1140 : audit [DBG] from='client.? 172.21.15.6:0/2681917551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:51.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:51 smithi130 bash[27579]: audit 2023-08-27T22:18:51.285757+0000 mon.a (mon.0) 1140 : audit [DBG] from='client.? 172.21.15.6:0/2681917551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:52.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:18:51.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:18:51.724Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:52.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:52 smithi006 bash[33918]: cluster 2023-08-27T22:18:51.336954+0000 mgr.y (mgr.14901) 610 : cluster [DBG] pgmap v617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:52.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:52 smithi006 bash[27292]: cluster 2023-08-27T22:18:51.336954+0000 mgr.y (mgr.14901) 610 : cluster [DBG] pgmap v617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:52 smithi130 bash[27579]: cluster 2023-08-27T22:18:51.336954+0000 mgr.y (mgr.14901) 610 : cluster [DBG] pgmap v617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:18:53.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:54.013 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:18:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:18:53.619Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:18:54.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:54 smithi006 bash[33918]: cluster 2023-08-27T22:18:53.337605+0000 mgr.y (mgr.14901) 611 : cluster [DBG] pgmap v618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:54.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:54 smithi006 bash[33918]: audit 2023-08-27T22:18:53.698126+0000 mon.a (mon.0) 1141 : audit [DBG] from='client.? 172.21.15.6:0/1618148947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:54.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:54 smithi006 bash[27292]: cluster 2023-08-27T22:18:53.337605+0000 mgr.y (mgr.14901) 611 : cluster [DBG] pgmap v618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:54.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:54 smithi006 bash[27292]: audit 2023-08-27T22:18:53.698126+0000 mon.a (mon.0) 1141 : audit [DBG] from='client.? 172.21.15.6:0/1618148947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:54 smithi130 bash[27579]: cluster 2023-08-27T22:18:53.337605+0000 mgr.y (mgr.14901) 611 : cluster [DBG] pgmap v618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:54.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:54 smithi130 bash[27579]: audit 2023-08-27T22:18:53.698126+0000 mon.a (mon.0) 1141 : audit [DBG] from='client.? 172.21.15.6:0/1618148947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:56.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:56 smithi006 bash[33918]: cluster 2023-08-27T22:18:55.338658+0000 mgr.y (mgr.14901) 612 : cluster [DBG] pgmap v619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:56.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:56 smithi006 bash[33918]: audit 2023-08-27T22:18:56.112330+0000 mon.a (mon.0) 1142 : audit [DBG] from='client.? 172.21.15.6:0/3453248129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:56.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:56 smithi006 bash[27292]: cluster 2023-08-27T22:18:55.338658+0000 mgr.y (mgr.14901) 612 : cluster [DBG] pgmap v619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:56.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:56 smithi006 bash[27292]: audit 2023-08-27T22:18:56.112330+0000 mon.a (mon.0) 1142 : audit [DBG] from='client.? 172.21.15.6:0/3453248129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:56.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:56 smithi130 bash[27579]: cluster 2023-08-27T22:18:55.338658+0000 mgr.y (mgr.14901) 612 : cluster [DBG] pgmap v619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:56.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:56 smithi130 bash[27579]: audit 2023-08-27T22:18:56.112330+0000 mon.a (mon.0) 1142 : audit [DBG] from='client.? 172.21.15.6:0/3453248129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:58.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:58 smithi006 bash[33918]: cluster 2023-08-27T22:18:57.339426+0000 mgr.y (mgr.14901) 613 : cluster [DBG] pgmap v620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:58.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:58 smithi006 bash[27292]: cluster 2023-08-27T22:18:57.339426+0000 mgr.y (mgr.14901) 613 : cluster [DBG] pgmap v620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:58 smithi130 bash[27579]: cluster 2023-08-27T22:18:57.339426+0000 mgr.y (mgr.14901) 613 : cluster [DBG] pgmap v620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:18:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:18:59 smithi130 bash[27579]: audit 2023-08-27T22:18:58.531754+0000 mon.c (mon.2) 257 : audit [DBG] from='client.? 172.21.15.6:0/3918788885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:59.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:18:59 smithi006 bash[33918]: audit 2023-08-27T22:18:58.531754+0000 mon.c (mon.2) 257 : audit [DBG] from='client.? 172.21.15.6:0/3918788885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:59.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:18:59 smithi006 bash[27292]: audit 2023-08-27T22:18:58.531754+0000 mon.c (mon.2) 257 : audit [DBG] from='client.? 172.21.15.6:0/3918788885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:00.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:19:00.159Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:00.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:19:00.161Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:00.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:00 smithi006 bash[27292]: cluster 2023-08-27T22:18:59.340309+0000 mgr.y (mgr.14901) 614 : cluster [DBG] pgmap v621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:00.756 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:00 smithi006 bash[33918]: cluster 2023-08-27T22:18:59.340309+0000 mgr.y (mgr.14901) 614 : cluster [DBG] pgmap v621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:00 smithi130 bash[27579]: cluster 2023-08-27T22:18:59.340309+0000 mgr.y (mgr.14901) 614 : cluster [DBG] pgmap v621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:01.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:19:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:19:00] "GET /metrics HTTP/1.1" 200 193066 "" "Prometheus/2.33.4" 2023-08-27T22:19:01.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:01 smithi130 bash[27579]: audit 2023-08-27T22:19:00.951422+0000 mon.c (mon.2) 258 : audit [DBG] from='client.? 172.21.15.6:0/3669420212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:01.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:01 smithi006 bash[33918]: audit 2023-08-27T22:19:00.951422+0000 mon.c (mon.2) 258 : audit [DBG] from='client.? 172.21.15.6:0/3669420212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:01.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:01 smithi006 bash[27292]: audit 2023-08-27T22:19:00.951422+0000 mon.c (mon.2) 258 : audit [DBG] from='client.? 172.21.15.6:0/3669420212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:01.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:19:01.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:01.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:19:01.725Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:02 smithi130 bash[27579]: cluster 2023-08-27T22:19:01.341435+0000 mgr.y (mgr.14901) 615 : cluster [DBG] pgmap v622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:02.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:02 smithi006 bash[27292]: cluster 2023-08-27T22:19:01.341435+0000 mgr.y (mgr.14901) 615 : cluster [DBG] pgmap v622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:02.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:02 smithi006 bash[33918]: cluster 2023-08-27T22:19:01.341435+0000 mgr.y (mgr.14901) 615 : cluster [DBG] pgmap v622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:03.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:03 smithi130 bash[27579]: audit 2023-08-27T22:19:03.368959+0000 mon.a (mon.0) 1143 : audit [DBG] from='client.? 172.21.15.6:0/2897446604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:03.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:03 smithi006 bash[27292]: audit 2023-08-27T22:19:03.368959+0000 mon.a (mon.0) 1143 : audit [DBG] from='client.? 172.21.15.6:0/2897446604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:03.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:03 smithi006 bash[33918]: audit 2023-08-27T22:19:03.368959+0000 mon.a (mon.0) 1143 : audit [DBG] from='client.? 172.21.15.6:0/2897446604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:19:03.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:19:03.621Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:04 smithi130 bash[27579]: cluster 2023-08-27T22:19:03.342191+0000 mgr.y (mgr.14901) 616 : cluster [DBG] pgmap v623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:04.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:04 smithi006 bash[27292]: cluster 2023-08-27T22:19:03.342191+0000 mgr.y (mgr.14901) 616 : cluster [DBG] pgmap v623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:04.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:04 smithi006 bash[33918]: cluster 2023-08-27T22:19:03.342191+0000 mgr.y (mgr.14901) 616 : cluster [DBG] pgmap v623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:06 smithi130 bash[27579]: cluster 2023-08-27T22:19:05.343364+0000 mgr.y (mgr.14901) 617 : cluster [DBG] pgmap v624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:06 smithi130 bash[27579]: audit 2023-08-27T22:19:05.781982+0000 mon.c (mon.2) 259 : audit [DBG] from='client.? 172.21.15.6:0/2511750342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:06.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:06 smithi006 bash[27292]: cluster 2023-08-27T22:19:05.343364+0000 mgr.y (mgr.14901) 617 : cluster [DBG] pgmap v624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:06.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:06 smithi006 bash[27292]: audit 2023-08-27T22:19:05.781982+0000 mon.c (mon.2) 259 : audit [DBG] from='client.? 172.21.15.6:0/2511750342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:06.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:06 smithi006 bash[33918]: cluster 2023-08-27T22:19:05.343364+0000 mgr.y (mgr.14901) 617 : cluster [DBG] pgmap v624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:06.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:06 smithi006 bash[33918]: audit 2023-08-27T22:19:05.781982+0000 mon.c (mon.2) 259 : audit [DBG] from='client.? 172.21.15.6:0/2511750342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:08.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:08 smithi130 bash[27579]: cluster 2023-08-27T22:19:07.344260+0000 mgr.y (mgr.14901) 618 : cluster [DBG] pgmap v625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:08.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:08 smithi130 bash[27579]: audit 2023-08-27T22:19:08.200968+0000 mon.c (mon.2) 260 : audit [DBG] from='client.? 172.21.15.6:0/4271243912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:08.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:08 smithi006 bash[27292]: cluster 2023-08-27T22:19:07.344260+0000 mgr.y (mgr.14901) 618 : cluster [DBG] pgmap v625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:08.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:08 smithi006 bash[27292]: audit 2023-08-27T22:19:08.200968+0000 mon.c (mon.2) 260 : audit [DBG] from='client.? 172.21.15.6:0/4271243912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:08.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:08 smithi006 bash[33918]: cluster 2023-08-27T22:19:07.344260+0000 mgr.y (mgr.14901) 618 : cluster [DBG] pgmap v625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:08.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:08 smithi006 bash[33918]: audit 2023-08-27T22:19:08.200968+0000 mon.c (mon.2) 260 : audit [DBG] from='client.? 172.21.15.6:0/4271243912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:10.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:19:10.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:10.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:19:10.163Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:10 smithi130 bash[27579]: cluster 2023-08-27T22:19:09.345112+0000 mgr.y (mgr.14901) 619 : cluster [DBG] pgmap v626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:10.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:10 smithi006 bash[27292]: cluster 2023-08-27T22:19:09.345112+0000 mgr.y (mgr.14901) 619 : cluster [DBG] pgmap v626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:10.971 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:19:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:19:10] "GET /metrics HTTP/1.1" 200 193066 "" "Prometheus/2.33.4" 2023-08-27T22:19:10.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:10 smithi006 bash[33918]: cluster 2023-08-27T22:19:09.345112+0000 mgr.y (mgr.14901) 619 : cluster [DBG] pgmap v626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:11.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:11 smithi130 bash[27579]: audit 2023-08-27T22:19:10.623776+0000 mon.a (mon.0) 1144 : audit [DBG] from='client.? 172.21.15.6:0/2076630686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:11.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:11 smithi006 bash[33918]: audit 2023-08-27T22:19:10.623776+0000 mon.a (mon.0) 1144 : audit [DBG] from='client.? 172.21.15.6:0/2076630686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:11.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:11 smithi006 bash[27292]: audit 2023-08-27T22:19:10.623776+0000 mon.a (mon.0) 1144 : audit [DBG] from='client.? 172.21.15.6:0/2076630686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:11.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:19:11.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:11.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:19:11.726Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:12.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:12 smithi130 bash[27579]: cluster 2023-08-27T22:19:11.346075+0000 mgr.y (mgr.14901) 620 : cluster [DBG] pgmap v627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:12.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:12 smithi006 bash[33918]: cluster 2023-08-27T22:19:11.346075+0000 mgr.y (mgr.14901) 620 : cluster [DBG] pgmap v627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:12.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:12 smithi006 bash[27292]: cluster 2023-08-27T22:19:11.346075+0000 mgr.y (mgr.14901) 620 : cluster [DBG] pgmap v627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:13.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:13 smithi130 bash[27579]: audit 2023-08-27T22:19:13.033228+0000 mon.a (mon.0) 1145 : audit [DBG] from='client.? 172.21.15.6:0/1056379898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:13.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:13 smithi006 bash[27292]: audit 2023-08-27T22:19:13.033228+0000 mon.a (mon.0) 1145 : audit [DBG] from='client.? 172.21.15.6:0/1056379898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:13.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:13 smithi006 bash[33918]: audit 2023-08-27T22:19:13.033228+0000 mon.a (mon.0) 1145 : audit [DBG] from='client.? 172.21.15.6:0/1056379898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:19:13.619Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:19:13.621Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:14.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:14 smithi130 bash[27579]: cluster 2023-08-27T22:19:13.346737+0000 mgr.y (mgr.14901) 621 : cluster [DBG] pgmap v628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:14.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:14 smithi006 bash[27292]: cluster 2023-08-27T22:19:13.346737+0000 mgr.y (mgr.14901) 621 : cluster [DBG] pgmap v628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:14.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:14 smithi006 bash[33918]: cluster 2023-08-27T22:19:13.346737+0000 mgr.y (mgr.14901) 621 : cluster [DBG] pgmap v628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:15.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:15 smithi130 bash[27579]: audit 2023-08-27T22:19:15.459271+0000 mon.a (mon.0) 1146 : audit [DBG] from='client.? 172.21.15.6:0/18109330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:15.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:15 smithi006 bash[33918]: audit 2023-08-27T22:19:15.459271+0000 mon.a (mon.0) 1146 : audit [DBG] from='client.? 172.21.15.6:0/18109330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:15.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:15 smithi006 bash[27292]: audit 2023-08-27T22:19:15.459271+0000 mon.a (mon.0) 1146 : audit [DBG] from='client.? 172.21.15.6:0/18109330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:16 smithi130 bash[27579]: cluster 2023-08-27T22:19:15.347739+0000 mgr.y (mgr.14901) 622 : cluster [DBG] pgmap v629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:16.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:16 smithi006 bash[27292]: cluster 2023-08-27T22:19:15.347739+0000 mgr.y (mgr.14901) 622 : cluster [DBG] pgmap v629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:16.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:16 smithi006 bash[33918]: cluster 2023-08-27T22:19:15.347739+0000 mgr.y (mgr.14901) 622 : cluster [DBG] pgmap v629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:18.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:18 smithi130 bash[27579]: cluster 2023-08-27T22:19:17.348412+0000 mgr.y (mgr.14901) 623 : cluster [DBG] pgmap v630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:18.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:18 smithi130 bash[27579]: audit 2023-08-27T22:19:17.874158+0000 mon.a (mon.0) 1147 : audit [DBG] from='client.? 172.21.15.6:0/1443798896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:18.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:18 smithi006 bash[27292]: cluster 2023-08-27T22:19:17.348412+0000 mgr.y (mgr.14901) 623 : cluster [DBG] pgmap v630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:18.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:18 smithi006 bash[27292]: audit 2023-08-27T22:19:17.874158+0000 mon.a (mon.0) 1147 : audit [DBG] from='client.? 172.21.15.6:0/1443798896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:18.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:18 smithi006 bash[33918]: cluster 2023-08-27T22:19:17.348412+0000 mgr.y (mgr.14901) 623 : cluster [DBG] pgmap v630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:18.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:18 smithi006 bash[33918]: audit 2023-08-27T22:19:17.874158+0000 mon.a (mon.0) 1147 : audit [DBG] from='client.? 172.21.15.6:0/1443798896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:20.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:19:20.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:20.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:19:20.163Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:20.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:20 smithi130 bash[27579]: cluster 2023-08-27T22:19:19.349214+0000 mgr.y (mgr.14901) 624 : cluster [DBG] pgmap v631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:20.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:20 smithi130 bash[27579]: audit 2023-08-27T22:19:20.290320+0000 mon.c (mon.2) 261 : audit [DBG] from='client.? 172.21.15.6:0/243746605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:20.970 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:19:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:19:20] "GET /metrics HTTP/1.1" 200 193066 "" "Prometheus/2.33.4" 2023-08-27T22:19:20.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:20 smithi006 bash[27292]: cluster 2023-08-27T22:19:19.349214+0000 mgr.y (mgr.14901) 624 : cluster [DBG] pgmap v631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:20.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:20 smithi006 bash[27292]: audit 2023-08-27T22:19:20.290320+0000 mon.c (mon.2) 261 : audit [DBG] from='client.? 172.21.15.6:0/243746605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:20.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:20 smithi006 bash[33918]: cluster 2023-08-27T22:19:19.349214+0000 mgr.y (mgr.14901) 624 : cluster [DBG] pgmap v631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:20.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:20 smithi006 bash[33918]: audit 2023-08-27T22:19:20.290320+0000 mon.c (mon.2) 261 : audit [DBG] from='client.? 172.21.15.6:0/243746605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:22.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:19:21.725Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:19:21.727Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:22 smithi130 bash[27579]: cluster 2023-08-27T22:19:21.350122+0000 mgr.y (mgr.14901) 625 : cluster [DBG] pgmap v632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:22.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:22 smithi006 bash[33918]: cluster 2023-08-27T22:19:21.350122+0000 mgr.y (mgr.14901) 625 : cluster [DBG] pgmap v632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:22.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:22 smithi006 bash[27292]: cluster 2023-08-27T22:19:21.350122+0000 mgr.y (mgr.14901) 625 : cluster [DBG] pgmap v632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:23.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:23 smithi130 bash[27579]: audit 2023-08-27T22:19:22.714294+0000 mon.a (mon.0) 1148 : audit [DBG] from='client.? 172.21.15.6:0/2151332967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:23.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:23 smithi006 bash[33918]: audit 2023-08-27T22:19:22.714294+0000 mon.a (mon.0) 1148 : audit [DBG] from='client.? 172.21.15.6:0/2151332967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:23.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:23 smithi006 bash[27292]: audit 2023-08-27T22:19:22.714294+0000 mon.a (mon.0) 1148 : audit [DBG] from='client.? 172.21.15.6:0/2151332967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:19:23.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:19:23.621Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:24 smithi130 bash[27579]: cluster 2023-08-27T22:19:23.350782+0000 mgr.y (mgr.14901) 626 : cluster [DBG] pgmap v633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:24.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:24 smithi006 bash[33918]: cluster 2023-08-27T22:19:23.350782+0000 mgr.y (mgr.14901) 626 : cluster [DBG] pgmap v633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:24.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:24 smithi006 bash[27292]: cluster 2023-08-27T22:19:23.350782+0000 mgr.y (mgr.14901) 626 : cluster [DBG] pgmap v633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:25.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:25 smithi130 bash[27579]: audit 2023-08-27T22:19:25.126357+0000 mon.a (mon.0) 1149 : audit [DBG] from='client.? 172.21.15.6:0/3431463636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:25.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:25 smithi006 bash[27292]: audit 2023-08-27T22:19:25.126357+0000 mon.a (mon.0) 1149 : audit [DBG] from='client.? 172.21.15.6:0/3431463636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:25.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:25 smithi006 bash[33918]: audit 2023-08-27T22:19:25.126357+0000 mon.a (mon.0) 1149 : audit [DBG] from='client.? 172.21.15.6:0/3431463636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:26.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:26 smithi130 bash[27579]: cluster 2023-08-27T22:19:25.351835+0000 mgr.y (mgr.14901) 627 : cluster [DBG] pgmap v634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:26.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:26 smithi006 bash[33918]: cluster 2023-08-27T22:19:25.351835+0000 mgr.y (mgr.14901) 627 : cluster [DBG] pgmap v634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:26.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:26 smithi006 bash[27292]: cluster 2023-08-27T22:19:25.351835+0000 mgr.y (mgr.14901) 627 : cluster [DBG] pgmap v634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:27.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:27 smithi130 bash[27579]: audit 2023-08-27T22:19:26.612672+0000 mon.b (mon.1) 187 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:19:27.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:27 smithi130 bash[27579]: audit 2023-08-27T22:19:27.540400+0000 mon.a (mon.0) 1150 : audit [DBG] from='client.? 172.21.15.6:0/2149825912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:27.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:27 smithi006 bash[27292]: audit 2023-08-27T22:19:26.612672+0000 mon.b (mon.1) 187 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:19:27.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:27 smithi006 bash[27292]: audit 2023-08-27T22:19:27.540400+0000 mon.a (mon.0) 1150 : audit [DBG] from='client.? 172.21.15.6:0/2149825912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:27.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:27 smithi006 bash[33918]: audit 2023-08-27T22:19:26.612672+0000 mon.b (mon.1) 187 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:19:27.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:27 smithi006 bash[33918]: audit 2023-08-27T22:19:27.540400+0000 mon.a (mon.0) 1150 : audit [DBG] from='client.? 172.21.15.6:0/2149825912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:28 smithi130 bash[27579]: cluster 2023-08-27T22:19:27.352576+0000 mgr.y (mgr.14901) 628 : cluster [DBG] pgmap v635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:28.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:28 smithi006 bash[33918]: cluster 2023-08-27T22:19:27.352576+0000 mgr.y (mgr.14901) 628 : cluster [DBG] pgmap v635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:28.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:28 smithi006 bash[27292]: cluster 2023-08-27T22:19:27.352576+0000 mgr.y (mgr.14901) 628 : cluster [DBG] pgmap v635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:30.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:19:30.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:30.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:19:30.163Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:30.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:30 smithi130 bash[27579]: cluster 2023-08-27T22:19:29.353497+0000 mgr.y (mgr.14901) 629 : cluster [DBG] pgmap v636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:30.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:30 smithi130 bash[27579]: audit 2023-08-27T22:19:29.958499+0000 mon.c (mon.2) 262 : audit [DBG] from='client.? 172.21.15.6:0/2981844759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:30.971 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:19:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:19:30] "GET /metrics HTTP/1.1" 200 193066 "" "Prometheus/2.33.4" 2023-08-27T22:19:30.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:30 smithi006 bash[27292]: cluster 2023-08-27T22:19:29.353497+0000 mgr.y (mgr.14901) 629 : cluster [DBG] pgmap v636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:30.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:30 smithi006 bash[27292]: audit 2023-08-27T22:19:29.958499+0000 mon.c (mon.2) 262 : audit [DBG] from='client.? 172.21.15.6:0/2981844759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:30.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:30 smithi006 bash[33918]: cluster 2023-08-27T22:19:29.353497+0000 mgr.y (mgr.14901) 629 : cluster [DBG] pgmap v636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:30.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:30 smithi006 bash[33918]: audit 2023-08-27T22:19:29.958499+0000 mon.c (mon.2) 262 : audit [DBG] from='client.? 172.21.15.6:0/2981844759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:19:31.725Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:19:31.727Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:32.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:32 smithi006 bash[27292]: cluster 2023-08-27T22:19:31.354489+0000 mgr.y (mgr.14901) 630 : cluster [DBG] pgmap v637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:32.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:32 smithi006 bash[27292]: audit 2023-08-27T22:19:32.402543+0000 mon.a (mon.0) 1151 : audit [DBG] from='client.? 172.21.15.6:0/218268136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:32.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:32 smithi006 bash[27292]: audit 2023-08-27T22:19:32.468567+0000 mon.a (mon.0) 1152 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:19:32.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:32 smithi006 bash[33918]: cluster 2023-08-27T22:19:31.354489+0000 mgr.y (mgr.14901) 630 : cluster [DBG] pgmap v637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:32.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:32 smithi006 bash[33918]: audit 2023-08-27T22:19:32.402543+0000 mon.a (mon.0) 1151 : audit [DBG] from='client.? 172.21.15.6:0/218268136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:32.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:32 smithi006 bash[33918]: audit 2023-08-27T22:19:32.468567+0000 mon.a (mon.0) 1152 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:19:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:32 smithi130 bash[27579]: cluster 2023-08-27T22:19:31.354489+0000 mgr.y (mgr.14901) 630 : cluster [DBG] pgmap v637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:32 smithi130 bash[27579]: audit 2023-08-27T22:19:32.402543+0000 mon.a (mon.0) 1151 : audit [DBG] from='client.? 172.21.15.6:0/218268136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:33.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:32 smithi130 bash[27579]: audit 2023-08-27T22:19:32.468567+0000 mon.a (mon.0) 1152 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:19:33.919 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:19:33.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:33.919 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:19:33.623Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:34.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:33 smithi006 bash[33918]: audit 2023-08-27T22:19:32.907992+0000 mon.a (mon.0) 1153 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:19:34.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:33 smithi006 bash[27292]: audit 2023-08-27T22:19:32.907992+0000 mon.a (mon.0) 1153 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:19:34.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:33 smithi130 bash[27579]: audit 2023-08-27T22:19:32.907992+0000 mon.a (mon.0) 1153 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:19:35.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:34 smithi006 bash[33918]: cluster 2023-08-27T22:19:33.355146+0000 mgr.y (mgr.14901) 631 : cluster [DBG] pgmap v638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:35.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:34 smithi006 bash[33918]: audit 2023-08-27T22:19:34.821972+0000 mon.c (mon.2) 263 : audit [DBG] from='client.? 172.21.15.6:0/2951658173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:35.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:34 smithi006 bash[27292]: cluster 2023-08-27T22:19:33.355146+0000 mgr.y (mgr.14901) 631 : cluster [DBG] pgmap v638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:35.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:34 smithi006 bash[27292]: audit 2023-08-27T22:19:34.821972+0000 mon.c (mon.2) 263 : audit [DBG] from='client.? 172.21.15.6:0/2951658173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:34 smithi130 bash[27579]: cluster 2023-08-27T22:19:33.355146+0000 mgr.y (mgr.14901) 631 : cluster [DBG] pgmap v638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:35.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:34 smithi130 bash[27579]: audit 2023-08-27T22:19:34.821972+0000 mon.c (mon.2) 263 : audit [DBG] from='client.? 172.21.15.6:0/2951658173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:37.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:36 smithi006 bash[33918]: cluster 2023-08-27T22:19:35.356309+0000 mgr.y (mgr.14901) 632 : cluster [DBG] pgmap v639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:37.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:36 smithi006 bash[27292]: cluster 2023-08-27T22:19:35.356309+0000 mgr.y (mgr.14901) 632 : cluster [DBG] pgmap v639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:37.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:36 smithi130 bash[27579]: cluster 2023-08-27T22:19:35.356309+0000 mgr.y (mgr.14901) 632 : cluster [DBG] pgmap v639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:38.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:37 smithi006 bash[27292]: audit 2023-08-27T22:19:37.235372+0000 mon.c (mon.2) 264 : audit [DBG] from='client.? 172.21.15.6:0/3325818023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:38.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:37 smithi006 bash[33918]: audit 2023-08-27T22:19:37.235372+0000 mon.c (mon.2) 264 : audit [DBG] from='client.? 172.21.15.6:0/3325818023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:38.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:37 smithi130 bash[27579]: audit 2023-08-27T22:19:37.235372+0000 mon.c (mon.2) 264 : audit [DBG] from='client.? 172.21.15.6:0/3325818023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:39.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:38 smithi006 bash[27292]: cluster 2023-08-27T22:19:37.357138+0000 mgr.y (mgr.14901) 633 : cluster [DBG] pgmap v640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:39.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:38 smithi006 bash[33918]: cluster 2023-08-27T22:19:37.357138+0000 mgr.y (mgr.14901) 633 : cluster [DBG] pgmap v640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:38 smithi130 bash[27579]: cluster 2023-08-27T22:19:37.357138+0000 mgr.y (mgr.14901) 633 : cluster [DBG] pgmap v640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:40.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:39 smithi006 bash[33918]: audit 2023-08-27T22:19:39.641698+0000 mon.a (mon.0) 1154 : audit [DBG] from='client.? 172.21.15.6:0/3291181775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:40.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:39 smithi006 bash[27292]: audit 2023-08-27T22:19:39.641698+0000 mon.a (mon.0) 1154 : audit [DBG] from='client.? 172.21.15.6:0/3291181775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:40.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:19:40.162Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:40.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:19:40.164Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:39 smithi130 bash[27579]: audit 2023-08-27T22:19:39.641698+0000 mon.a (mon.0) 1154 : audit [DBG] from='client.? 172.21.15.6:0/3291181775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:40.951 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:19:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:19:40] "GET /metrics HTTP/1.1" 200 193066 "" "Prometheus/2.33.4" 2023-08-27T22:19:41.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:40 smithi006 bash[27292]: cluster 2023-08-27T22:19:39.357625+0000 mgr.y (mgr.14901) 634 : cluster [DBG] pgmap v641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:41.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:40 smithi006 bash[33918]: cluster 2023-08-27T22:19:39.357625+0000 mgr.y (mgr.14901) 634 : cluster [DBG] pgmap v641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:41.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:40 smithi130 bash[27579]: cluster 2023-08-27T22:19:39.357625+0000 mgr.y (mgr.14901) 634 : cluster [DBG] pgmap v641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:41.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:19:41.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:41.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:19:41.728Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:43.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:42 smithi006 bash[33918]: cluster 2023-08-27T22:19:41.358518+0000 mgr.y (mgr.14901) 635 : cluster [DBG] pgmap v642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:43.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:42 smithi006 bash[33918]: audit 2023-08-27T22:19:42.039808+0000 mon.a (mon.0) 1155 : audit [DBG] from='client.? 172.21.15.6:0/4228586460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:43.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:42 smithi006 bash[33918]: audit 2023-08-27T22:19:42.395293+0000 mon.a (mon.0) 1156 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:19:43.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:42 smithi006 bash[33918]: audit 2023-08-27T22:19:42.702691+0000 mon.b (mon.1) 188 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:19:43.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:42 smithi006 bash[33918]: audit 2023-08-27T22:19:42.704823+0000 mon.b (mon.1) 189 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:19:43.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:42 smithi006 bash[33918]: audit 2023-08-27T22:19:42.712901+0000 mon.a (mon.0) 1157 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:19:43.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:42 smithi006 bash[27292]: cluster 2023-08-27T22:19:41.358518+0000 mgr.y (mgr.14901) 635 : cluster [DBG] pgmap v642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:43.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:42 smithi006 bash[27292]: audit 2023-08-27T22:19:42.039808+0000 mon.a (mon.0) 1155 : audit [DBG] from='client.? 172.21.15.6:0/4228586460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:43.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:42 smithi006 bash[27292]: audit 2023-08-27T22:19:42.395293+0000 mon.a (mon.0) 1156 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:19:43.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:42 smithi006 bash[27292]: audit 2023-08-27T22:19:42.702691+0000 mon.b (mon.1) 188 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:19:43.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:42 smithi006 bash[27292]: audit 2023-08-27T22:19:42.704823+0000 mon.b (mon.1) 189 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:19:43.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:42 smithi006 bash[27292]: audit 2023-08-27T22:19:42.712901+0000 mon.a (mon.0) 1157 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:19:43.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:42 smithi130 bash[27579]: cluster 2023-08-27T22:19:41.358518+0000 mgr.y (mgr.14901) 635 : cluster [DBG] pgmap v642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:43.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:42 smithi130 bash[27579]: audit 2023-08-27T22:19:42.039808+0000 mon.a (mon.0) 1155 : audit [DBG] from='client.? 172.21.15.6:0/4228586460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:43.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:42 smithi130 bash[27579]: audit 2023-08-27T22:19:42.395293+0000 mon.a (mon.0) 1156 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:19:43.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:42 smithi130 bash[27579]: audit 2023-08-27T22:19:42.702691+0000 mon.b (mon.1) 188 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:19:43.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:42 smithi130 bash[27579]: audit 2023-08-27T22:19:42.704823+0000 mon.b (mon.1) 189 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:19:43.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:42 smithi130 bash[27579]: audit 2023-08-27T22:19:42.712901+0000 mon.a (mon.0) 1157 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:19:43.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:19:43.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:43.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:19:43.624Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:45.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:44 smithi006 bash[33918]: cluster 2023-08-27T22:19:43.359160+0000 mgr.y (mgr.14901) 636 : cluster [DBG] pgmap v643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:45.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:44 smithi006 bash[33918]: audit 2023-08-27T22:19:44.453121+0000 mon.a (mon.0) 1158 : audit [DBG] from='client.? 172.21.15.6:0/2829815227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:45.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:44 smithi006 bash[27292]: cluster 2023-08-27T22:19:43.359160+0000 mgr.y (mgr.14901) 636 : cluster [DBG] pgmap v643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:45.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:44 smithi006 bash[27292]: audit 2023-08-27T22:19:44.453121+0000 mon.a (mon.0) 1158 : audit [DBG] from='client.? 172.21.15.6:0/2829815227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:44 smithi130 bash[27579]: cluster 2023-08-27T22:19:43.359160+0000 mgr.y (mgr.14901) 636 : cluster [DBG] pgmap v643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:45.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:44 smithi130 bash[27579]: audit 2023-08-27T22:19:44.453121+0000 mon.a (mon.0) 1158 : audit [DBG] from='client.? 172.21.15.6:0/2829815227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:47.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:46 smithi006 bash[33918]: cluster 2023-08-27T22:19:45.360216+0000 mgr.y (mgr.14901) 637 : cluster [DBG] pgmap v644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:47.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:46 smithi006 bash[33918]: audit 2023-08-27T22:19:46.867382+0000 mon.c (mon.2) 265 : audit [DBG] from='client.? 172.21.15.6:0/665005545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:47.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:46 smithi006 bash[27292]: cluster 2023-08-27T22:19:45.360216+0000 mgr.y (mgr.14901) 637 : cluster [DBG] pgmap v644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:47.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:46 smithi006 bash[27292]: audit 2023-08-27T22:19:46.867382+0000 mon.c (mon.2) 265 : audit [DBG] from='client.? 172.21.15.6:0/665005545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:46 smithi130 bash[27579]: cluster 2023-08-27T22:19:45.360216+0000 mgr.y (mgr.14901) 637 : cluster [DBG] pgmap v644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:46 smithi130 bash[27579]: audit 2023-08-27T22:19:46.867382+0000 mon.c (mon.2) 265 : audit [DBG] from='client.? 172.21.15.6:0/665005545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:49.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:48 smithi130 bash[27579]: cluster 2023-08-27T22:19:47.361089+0000 mgr.y (mgr.14901) 638 : cluster [DBG] pgmap v645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:49.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:48 smithi006 bash[33918]: cluster 2023-08-27T22:19:47.361089+0000 mgr.y (mgr.14901) 638 : cluster [DBG] pgmap v645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:49.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:48 smithi006 bash[27292]: cluster 2023-08-27T22:19:47.361089+0000 mgr.y (mgr.14901) 638 : cluster [DBG] pgmap v645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:50.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:49 smithi130 bash[27579]: audit 2023-08-27T22:19:49.284041+0000 mon.a (mon.0) 1159 : audit [DBG] from='client.? 172.21.15.6:0/3830835177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:50.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:49 smithi006 bash[27292]: audit 2023-08-27T22:19:49.284041+0000 mon.a (mon.0) 1159 : audit [DBG] from='client.? 172.21.15.6:0/3830835177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:50.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:49 smithi006 bash[33918]: audit 2023-08-27T22:19:49.284041+0000 mon.a (mon.0) 1159 : audit [DBG] from='client.? 172.21.15.6:0/3830835177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:50.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:19:50.163Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:50.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:19:50.165Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:51.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:19:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:19:50] "GET /metrics HTTP/1.1" 200 193067 "" "Prometheus/2.33.4" 2023-08-27T22:19:51.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:50 smithi006 bash[27292]: cluster 2023-08-27T22:19:49.362025+0000 mgr.y (mgr.14901) 639 : cluster [DBG] pgmap v646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:51.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:50 smithi006 bash[33918]: cluster 2023-08-27T22:19:49.362025+0000 mgr.y (mgr.14901) 639 : cluster [DBG] pgmap v646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:51.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:50 smithi130 bash[27579]: cluster 2023-08-27T22:19:49.362025+0000 mgr.y (mgr.14901) 639 : cluster [DBG] pgmap v646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:52.000 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:19:51.727Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:52.000 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:19:51.729Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:51 smithi130 bash[27579]: audit 2023-08-27T22:19:51.703607+0000 mon.a (mon.0) 1160 : audit [DBG] from='client.? 172.21.15.6:0/1283897079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:52.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:51 smithi006 bash[33918]: audit 2023-08-27T22:19:51.703607+0000 mon.a (mon.0) 1160 : audit [DBG] from='client.? 172.21.15.6:0/1283897079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:52.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:51 smithi006 bash[27292]: audit 2023-08-27T22:19:51.703607+0000 mon.a (mon.0) 1160 : audit [DBG] from='client.? 172.21.15.6:0/1283897079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:53 smithi130 bash[27579]: cluster 2023-08-27T22:19:51.363145+0000 mgr.y (mgr.14901) 640 : cluster [DBG] pgmap v647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:53.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:52 smithi006 bash[27292]: cluster 2023-08-27T22:19:51.363145+0000 mgr.y (mgr.14901) 640 : cluster [DBG] pgmap v647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:53.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:52 smithi006 bash[33918]: cluster 2023-08-27T22:19:51.363145+0000 mgr.y (mgr.14901) 640 : cluster [DBG] pgmap v647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:19:53.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:19:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:19:53.624Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:19:54.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:54 smithi006 bash[27292]: cluster 2023-08-27T22:19:53.363854+0000 mgr.y (mgr.14901) 641 : cluster [DBG] pgmap v648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:54.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:54 smithi006 bash[27292]: audit 2023-08-27T22:19:54.122204+0000 mon.c (mon.2) 266 : audit [DBG] from='client.? 172.21.15.6:0/111191967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:54.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:54 smithi006 bash[33918]: cluster 2023-08-27T22:19:53.363854+0000 mgr.y (mgr.14901) 641 : cluster [DBG] pgmap v648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:54.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:54 smithi006 bash[33918]: audit 2023-08-27T22:19:54.122204+0000 mon.c (mon.2) 266 : audit [DBG] from='client.? 172.21.15.6:0/111191967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:54 smithi130 bash[27579]: cluster 2023-08-27T22:19:53.363854+0000 mgr.y (mgr.14901) 641 : cluster [DBG] pgmap v648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:54.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:54 smithi130 bash[27579]: audit 2023-08-27T22:19:54.122204+0000 mon.c (mon.2) 266 : audit [DBG] from='client.? 172.21.15.6:0/111191967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:56.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:56 smithi006 bash[27292]: cluster 2023-08-27T22:19:55.365018+0000 mgr.y (mgr.14901) 642 : cluster [DBG] pgmap v649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:56.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:56 smithi006 bash[33918]: cluster 2023-08-27T22:19:55.365018+0000 mgr.y (mgr.14901) 642 : cluster [DBG] pgmap v649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:56 smithi130 bash[27579]: cluster 2023-08-27T22:19:55.365018+0000 mgr.y (mgr.14901) 642 : cluster [DBG] pgmap v649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:57.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:57 smithi006 bash[33918]: audit 2023-08-27T22:19:56.541293+0000 mon.c (mon.2) 267 : audit [DBG] from='client.? 172.21.15.6:0/1019821581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:57.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:57 smithi006 bash[27292]: audit 2023-08-27T22:19:56.541293+0000 mon.c (mon.2) 267 : audit [DBG] from='client.? 172.21.15.6:0/1019821581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:57.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:57 smithi130 bash[27579]: audit 2023-08-27T22:19:56.541293+0000 mon.c (mon.2) 267 : audit [DBG] from='client.? 172.21.15.6:0/1019821581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:58.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:58 smithi006 bash[27292]: cluster 2023-08-27T22:19:57.365946+0000 mgr.y (mgr.14901) 643 : cluster [DBG] pgmap v650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:58.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:58 smithi006 bash[33918]: cluster 2023-08-27T22:19:57.365946+0000 mgr.y (mgr.14901) 643 : cluster [DBG] pgmap v650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:58 smithi130 bash[27579]: cluster 2023-08-27T22:19:57.365946+0000 mgr.y (mgr.14901) 643 : cluster [DBG] pgmap v650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:19:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:19:59 smithi130 bash[27579]: audit 2023-08-27T22:19:58.959004+0000 mon.c (mon.2) 268 : audit [DBG] from='client.? 172.21.15.6:0/3718952178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:59.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:19:59 smithi006 bash[27292]: audit 2023-08-27T22:19:58.959004+0000 mon.c (mon.2) 268 : audit [DBG] from='client.? 172.21.15.6:0/3718952178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:59.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:19:59 smithi006 bash[33918]: audit 2023-08-27T22:19:58.959004+0000 mon.c (mon.2) 268 : audit [DBG] from='client.? 172.21.15.6:0/3718952178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:00.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:20:00.164Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:00.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:20:00.165Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:00.756 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:00 smithi006 bash[27292]: cluster 2023-08-27T22:19:59.366797+0000 mgr.y (mgr.14901) 644 : cluster [DBG] pgmap v651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:00.756 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:00 smithi006 bash[27292]: cluster 2023-08-27T22:20:00.000171+0000 mon.a (mon.0) 1161 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:20:00.756 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:00 smithi006 bash[33918]: cluster 2023-08-27T22:19:59.366797+0000 mgr.y (mgr.14901) 644 : cluster [DBG] pgmap v651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:00.756 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:00 smithi006 bash[33918]: cluster 2023-08-27T22:20:00.000171+0000 mon.a (mon.0) 1161 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:20:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:00 smithi130 bash[27579]: cluster 2023-08-27T22:19:59.366797+0000 mgr.y (mgr.14901) 644 : cluster [DBG] pgmap v651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:00 smithi130 bash[27579]: cluster 2023-08-27T22:20:00.000171+0000 mon.a (mon.0) 1161 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:20:01.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:20:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:20:00] "GET /metrics HTTP/1.1" 200 193067 "" "Prometheus/2.33.4" 2023-08-27T22:20:01.737 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:01 smithi006 bash[33918]: audit 2023-08-27T22:20:01.374176+0000 mon.a (mon.0) 1162 : audit [DBG] from='client.? 172.21.15.6:0/2295647989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:01.737 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:01 smithi006 bash[27292]: audit 2023-08-27T22:20:01.374176+0000 mon.a (mon.0) 1162 : audit [DBG] from='client.? 172.21.15.6:0/2295647989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:01.737 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:20:01.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:01.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:01 smithi130 bash[27579]: audit 2023-08-27T22:20:01.374176+0000 mon.a (mon.0) 1162 : audit [DBG] from='client.? 172.21.15.6:0/2295647989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:02 smithi130 bash[27579]: cluster 2023-08-27T22:20:01.367434+0000 mgr.y (mgr.14901) 645 : cluster [DBG] pgmap v652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:02.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:02 smithi006 bash[33918]: cluster 2023-08-27T22:20:01.367434+0000 mgr.y (mgr.14901) 645 : cluster [DBG] pgmap v652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:02.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:20:01.730Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:02.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:02 smithi006 bash[27292]: cluster 2023-08-27T22:20:01.367434+0000 mgr.y (mgr.14901) 645 : cluster [DBG] pgmap v652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:20:03.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:20:03.624Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:04 smithi130 bash[27579]: cluster 2023-08-27T22:20:03.368043+0000 mgr.y (mgr.14901) 646 : cluster [DBG] pgmap v653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:04 smithi130 bash[27579]: audit 2023-08-27T22:20:03.786630+0000 mon.c (mon.2) 269 : audit [DBG] from='client.? 172.21.15.6:0/3908267678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:04.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:04 smithi006 bash[27292]: cluster 2023-08-27T22:20:03.368043+0000 mgr.y (mgr.14901) 646 : cluster [DBG] pgmap v653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:04.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:04 smithi006 bash[27292]: audit 2023-08-27T22:20:03.786630+0000 mon.c (mon.2) 269 : audit [DBG] from='client.? 172.21.15.6:0/3908267678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:04.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:04 smithi006 bash[33918]: cluster 2023-08-27T22:20:03.368043+0000 mgr.y (mgr.14901) 646 : cluster [DBG] pgmap v653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:04.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:04 smithi006 bash[33918]: audit 2023-08-27T22:20:03.786630+0000 mon.c (mon.2) 269 : audit [DBG] from='client.? 172.21.15.6:0/3908267678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:06 smithi130 bash[27579]: cluster 2023-08-27T22:20:05.369154+0000 mgr.y (mgr.14901) 647 : cluster [DBG] pgmap v654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:06 smithi130 bash[27579]: audit 2023-08-27T22:20:06.207572+0000 mon.c (mon.2) 270 : audit [DBG] from='client.? 172.21.15.6:0/1488108592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:06.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:06 smithi006 bash[33918]: cluster 2023-08-27T22:20:05.369154+0000 mgr.y (mgr.14901) 647 : cluster [DBG] pgmap v654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:06.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:06 smithi006 bash[33918]: audit 2023-08-27T22:20:06.207572+0000 mon.c (mon.2) 270 : audit [DBG] from='client.? 172.21.15.6:0/1488108592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:06.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:06 smithi006 bash[27292]: cluster 2023-08-27T22:20:05.369154+0000 mgr.y (mgr.14901) 647 : cluster [DBG] pgmap v654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:06.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:06 smithi006 bash[27292]: audit 2023-08-27T22:20:06.207572+0000 mon.c (mon.2) 270 : audit [DBG] from='client.? 172.21.15.6:0/1488108592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:08 smithi130 bash[27579]: cluster 2023-08-27T22:20:07.369865+0000 mgr.y (mgr.14901) 648 : cluster [DBG] pgmap v655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:08.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:08 smithi006 bash[27292]: cluster 2023-08-27T22:20:07.369865+0000 mgr.y (mgr.14901) 648 : cluster [DBG] pgmap v655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:08.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:08 smithi006 bash[33918]: cluster 2023-08-27T22:20:07.369865+0000 mgr.y (mgr.14901) 648 : cluster [DBG] pgmap v655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:09 smithi130 bash[27579]: audit 2023-08-27T22:20:08.630408+0000 mon.c (mon.2) 271 : audit [DBG] from='client.? 172.21.15.6:0/57040841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:09.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:09 smithi006 bash[27292]: audit 2023-08-27T22:20:08.630408+0000 mon.c (mon.2) 271 : audit [DBG] from='client.? 172.21.15.6:0/57040841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:09.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:09 smithi006 bash[33918]: audit 2023-08-27T22:20:08.630408+0000 mon.c (mon.2) 271 : audit [DBG] from='client.? 172.21.15.6:0/57040841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:10.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:20:10.164Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:10.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:20:10.166Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:10 smithi130 bash[27579]: cluster 2023-08-27T22:20:09.370690+0000 mgr.y (mgr.14901) 649 : cluster [DBG] pgmap v656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:10.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:10 smithi006 bash[27292]: cluster 2023-08-27T22:20:09.370690+0000 mgr.y (mgr.14901) 649 : cluster [DBG] pgmap v656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:10.971 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:20:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:20:10] "GET /metrics HTTP/1.1" 200 193067 "" "Prometheus/2.33.4" 2023-08-27T22:20:10.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:10 smithi006 bash[33918]: cluster 2023-08-27T22:20:09.370690+0000 mgr.y (mgr.14901) 649 : cluster [DBG] pgmap v656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:11.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:11 smithi130 bash[27579]: audit 2023-08-27T22:20:11.054992+0000 mon.c (mon.2) 272 : audit [DBG] from='client.? 172.21.15.6:0/6335066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:11.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:11 smithi006 bash[27292]: audit 2023-08-27T22:20:11.054992+0000 mon.c (mon.2) 272 : audit [DBG] from='client.? 172.21.15.6:0/6335066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:11.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:11 smithi006 bash[33918]: audit 2023-08-27T22:20:11.054992+0000 mon.c (mon.2) 272 : audit [DBG] from='client.? 172.21.15.6:0/6335066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:11.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:20:11.730Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:11.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:20:11.732Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:12.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:12 smithi130 bash[27579]: cluster 2023-08-27T22:20:11.371799+0000 mgr.y (mgr.14901) 650 : cluster [DBG] pgmap v657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:12.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:12 smithi006 bash[27292]: cluster 2023-08-27T22:20:11.371799+0000 mgr.y (mgr.14901) 650 : cluster [DBG] pgmap v657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:12.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:12 smithi006 bash[33918]: cluster 2023-08-27T22:20:11.371799+0000 mgr.y (mgr.14901) 650 : cluster [DBG] pgmap v657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:20:13.623Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:20:13.625Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:14.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:14 smithi006 bash[33918]: cluster 2023-08-27T22:20:13.372577+0000 mgr.y (mgr.14901) 651 : cluster [DBG] pgmap v658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:14.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:14 smithi006 bash[33918]: audit 2023-08-27T22:20:13.476500+0000 mon.c (mon.2) 273 : audit [DBG] from='client.? 172.21.15.6:0/78872976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:14.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:14 smithi006 bash[27292]: cluster 2023-08-27T22:20:13.372577+0000 mgr.y (mgr.14901) 651 : cluster [DBG] pgmap v658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:14.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:14 smithi006 bash[27292]: audit 2023-08-27T22:20:13.476500+0000 mon.c (mon.2) 273 : audit [DBG] from='client.? 172.21.15.6:0/78872976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:14 smithi130 bash[27579]: cluster 2023-08-27T22:20:13.372577+0000 mgr.y (mgr.14901) 651 : cluster [DBG] pgmap v658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:14 smithi130 bash[27579]: audit 2023-08-27T22:20:13.476500+0000 mon.c (mon.2) 273 : audit [DBG] from='client.? 172.21.15.6:0/78872976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:16.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:16 smithi006 bash[27292]: cluster 2023-08-27T22:20:15.373581+0000 mgr.y (mgr.14901) 652 : cluster [DBG] pgmap v659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:16.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:16 smithi006 bash[27292]: audit 2023-08-27T22:20:15.894570+0000 mon.c (mon.2) 274 : audit [DBG] from='client.? 172.21.15.6:0/4170617495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:16.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:16 smithi006 bash[33918]: cluster 2023-08-27T22:20:15.373581+0000 mgr.y (mgr.14901) 652 : cluster [DBG] pgmap v659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:16.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:16 smithi006 bash[33918]: audit 2023-08-27T22:20:15.894570+0000 mon.c (mon.2) 274 : audit [DBG] from='client.? 172.21.15.6:0/4170617495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:16.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:16 smithi130 bash[27579]: cluster 2023-08-27T22:20:15.373581+0000 mgr.y (mgr.14901) 652 : cluster [DBG] pgmap v659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:16.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:16 smithi130 bash[27579]: audit 2023-08-27T22:20:15.894570+0000 mon.c (mon.2) 274 : audit [DBG] from='client.? 172.21.15.6:0/4170617495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:18.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:18 smithi006 bash[33918]: cluster 2023-08-27T22:20:17.374302+0000 mgr.y (mgr.14901) 653 : cluster [DBG] pgmap v660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:18.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:18 smithi006 bash[33918]: audit 2023-08-27T22:20:18.301295+0000 mon.a (mon.0) 1163 : audit [DBG] from='client.? 172.21.15.6:0/1653737691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:18.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:18 smithi006 bash[27292]: cluster 2023-08-27T22:20:17.374302+0000 mgr.y (mgr.14901) 653 : cluster [DBG] pgmap v660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:18.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:18 smithi006 bash[27292]: audit 2023-08-27T22:20:18.301295+0000 mon.a (mon.0) 1163 : audit [DBG] from='client.? 172.21.15.6:0/1653737691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:18.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:18 smithi130 bash[27579]: cluster 2023-08-27T22:20:17.374302+0000 mgr.y (mgr.14901) 653 : cluster [DBG] pgmap v660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:18.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:18 smithi130 bash[27579]: audit 2023-08-27T22:20:18.301295+0000 mon.a (mon.0) 1163 : audit [DBG] from='client.? 172.21.15.6:0/1653737691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:20.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:20:20.165Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:20.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:20:20.167Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:20.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:20 smithi006 bash[33918]: cluster 2023-08-27T22:20:19.375179+0000 mgr.y (mgr.14901) 654 : cluster [DBG] pgmap v661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:20.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:20 smithi006 bash[27292]: cluster 2023-08-27T22:20:19.375179+0000 mgr.y (mgr.14901) 654 : cluster [DBG] pgmap v661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:20 smithi130 bash[27579]: cluster 2023-08-27T22:20:19.375179+0000 mgr.y (mgr.14901) 654 : cluster [DBG] pgmap v661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:21.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:20:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:20:20] "GET /metrics HTTP/1.1" 200 193069 "" "Prometheus/2.33.4" 2023-08-27T22:20:21.737 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:21 smithi006 bash[27292]: audit 2023-08-27T22:20:20.719324+0000 mon.c (mon.2) 275 : audit [DBG] from='client.? 172.21.15.6:0/80620023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:21.737 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:21 smithi006 bash[33918]: audit 2023-08-27T22:20:20.719324+0000 mon.c (mon.2) 275 : audit [DBG] from='client.? 172.21.15.6:0/80620023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:21.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:21 smithi130 bash[27579]: audit 2023-08-27T22:20:20.719324+0000 mon.c (mon.2) 275 : audit [DBG] from='client.? 172.21.15.6:0/80620023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:20:21.730Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:20:21.732Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:22 smithi130 bash[27579]: cluster 2023-08-27T22:20:21.376126+0000 mgr.y (mgr.14901) 655 : cluster [DBG] pgmap v662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:22.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:22 smithi006 bash[27292]: cluster 2023-08-27T22:20:21.376126+0000 mgr.y (mgr.14901) 655 : cluster [DBG] pgmap v662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:22.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:22 smithi006 bash[33918]: cluster 2023-08-27T22:20:21.376126+0000 mgr.y (mgr.14901) 655 : cluster [DBG] pgmap v662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:23.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:23 smithi130 bash[27579]: audit 2023-08-27T22:20:23.138882+0000 mon.a (mon.0) 1164 : audit [DBG] from='client.? 172.21.15.6:0/3064920276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:23.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:23 smithi006 bash[27292]: audit 2023-08-27T22:20:23.138882+0000 mon.a (mon.0) 1164 : audit [DBG] from='client.? 172.21.15.6:0/3064920276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:23.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:23 smithi006 bash[33918]: audit 2023-08-27T22:20:23.138882+0000 mon.a (mon.0) 1164 : audit [DBG] from='client.? 172.21.15.6:0/3064920276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:20:23.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:20:23.627Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:24 smithi130 bash[27579]: cluster 2023-08-27T22:20:23.376733+0000 mgr.y (mgr.14901) 656 : cluster [DBG] pgmap v663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:24.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:24 smithi006 bash[33918]: cluster 2023-08-27T22:20:23.376733+0000 mgr.y (mgr.14901) 656 : cluster [DBG] pgmap v663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:24.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:24 smithi006 bash[27292]: cluster 2023-08-27T22:20:23.376733+0000 mgr.y (mgr.14901) 656 : cluster [DBG] pgmap v663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:26 smithi130 bash[27579]: cluster 2023-08-27T22:20:25.377664+0000 mgr.y (mgr.14901) 657 : cluster [DBG] pgmap v664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:26.887 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:26 smithi130 bash[27579]: audit 2023-08-27T22:20:25.559442+0000 mon.c (mon.2) 276 : audit [DBG] from='client.? 172.21.15.6:0/2418897371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:26.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:26 smithi006 bash[27292]: cluster 2023-08-27T22:20:25.377664+0000 mgr.y (mgr.14901) 657 : cluster [DBG] pgmap v664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:26.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:26 smithi006 bash[27292]: audit 2023-08-27T22:20:25.559442+0000 mon.c (mon.2) 276 : audit [DBG] from='client.? 172.21.15.6:0/2418897371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:26.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:26 smithi006 bash[33918]: cluster 2023-08-27T22:20:25.377664+0000 mgr.y (mgr.14901) 657 : cluster [DBG] pgmap v664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:26.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:26 smithi006 bash[33918]: audit 2023-08-27T22:20:25.559442+0000 mon.c (mon.2) 276 : audit [DBG] from='client.? 172.21.15.6:0/2418897371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:28 smithi130 bash[27579]: cluster 2023-08-27T22:20:27.378394+0000 mgr.y (mgr.14901) 658 : cluster [DBG] pgmap v665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:28 smithi130 bash[27579]: audit 2023-08-27T22:20:27.969616+0000 mon.a (mon.0) 1165 : audit [DBG] from='client.? 172.21.15.6:0/2890151488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:28.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:28 smithi006 bash[27292]: cluster 2023-08-27T22:20:27.378394+0000 mgr.y (mgr.14901) 658 : cluster [DBG] pgmap v665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:28.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:28 smithi006 bash[27292]: audit 2023-08-27T22:20:27.969616+0000 mon.a (mon.0) 1165 : audit [DBG] from='client.? 172.21.15.6:0/2890151488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:28.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:28 smithi006 bash[33918]: cluster 2023-08-27T22:20:27.378394+0000 mgr.y (mgr.14901) 658 : cluster [DBG] pgmap v665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:28.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:28 smithi006 bash[33918]: audit 2023-08-27T22:20:27.969616+0000 mon.a (mon.0) 1165 : audit [DBG] from='client.? 172.21.15.6:0/2890151488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:30.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:20:30.166Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:30.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:20:30.168Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:30 smithi130 bash[27579]: cluster 2023-08-27T22:20:29.379210+0000 mgr.y (mgr.14901) 659 : cluster [DBG] pgmap v666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:30 smithi130 bash[27579]: audit 2023-08-27T22:20:30.408616+0000 mon.a (mon.0) 1166 : audit [DBG] from='client.? 172.21.15.6:0/2494179281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:30.971 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:20:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:20:30] "GET /metrics HTTP/1.1" 200 193069 "" "Prometheus/2.33.4" 2023-08-27T22:20:30.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:30 smithi006 bash[27292]: cluster 2023-08-27T22:20:29.379210+0000 mgr.y (mgr.14901) 659 : cluster [DBG] pgmap v666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:30.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:30 smithi006 bash[27292]: audit 2023-08-27T22:20:30.408616+0000 mon.a (mon.0) 1166 : audit [DBG] from='client.? 172.21.15.6:0/2494179281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:30.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:30 smithi006 bash[33918]: cluster 2023-08-27T22:20:29.379210+0000 mgr.y (mgr.14901) 659 : cluster [DBG] pgmap v666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:30.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:30 smithi006 bash[33918]: audit 2023-08-27T22:20:30.408616+0000 mon.a (mon.0) 1166 : audit [DBG] from='client.? 172.21.15.6:0/2494179281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:20:31.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:20:31.733Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:32 smithi130 bash[27579]: cluster 2023-08-27T22:20:31.380127+0000 mgr.y (mgr.14901) 660 : cluster [DBG] pgmap v667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:32.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:32 smithi006 bash[33918]: cluster 2023-08-27T22:20:31.380127+0000 mgr.y (mgr.14901) 660 : cluster [DBG] pgmap v667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:32.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:32 smithi006 bash[27292]: cluster 2023-08-27T22:20:31.380127+0000 mgr.y (mgr.14901) 660 : cluster [DBG] pgmap v667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:33.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:33 smithi130 bash[27579]: audit 2023-08-27T22:20:32.827493+0000 mon.a (mon.0) 1167 : audit [DBG] from='client.? 172.21.15.6:0/1236731743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:33.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:33 smithi006 bash[27292]: audit 2023-08-27T22:20:32.827493+0000 mon.a (mon.0) 1167 : audit [DBG] from='client.? 172.21.15.6:0/1236731743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:33.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:33 smithi006 bash[33918]: audit 2023-08-27T22:20:32.827493+0000 mon.a (mon.0) 1167 : audit [DBG] from='client.? 172.21.15.6:0/1236731743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:20:33.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:20:33.627Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:34 smithi130 bash[27579]: cluster 2023-08-27T22:20:33.380771+0000 mgr.y (mgr.14901) 661 : cluster [DBG] pgmap v668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:34.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:34 smithi006 bash[33918]: cluster 2023-08-27T22:20:33.380771+0000 mgr.y (mgr.14901) 661 : cluster [DBG] pgmap v668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:34.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:34 smithi006 bash[27292]: cluster 2023-08-27T22:20:33.380771+0000 mgr.y (mgr.14901) 661 : cluster [DBG] pgmap v668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:35.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:35 smithi130 bash[27579]: audit 2023-08-27T22:20:35.242336+0000 mon.c (mon.2) 277 : audit [DBG] from='client.? 172.21.15.6:0/1862532722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:35.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:35 smithi006 bash[27292]: audit 2023-08-27T22:20:35.242336+0000 mon.c (mon.2) 277 : audit [DBG] from='client.? 172.21.15.6:0/1862532722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:35.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:35 smithi006 bash[33918]: audit 2023-08-27T22:20:35.242336+0000 mon.c (mon.2) 277 : audit [DBG] from='client.? 172.21.15.6:0/1862532722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:36 smithi130 bash[27579]: cluster 2023-08-27T22:20:35.381920+0000 mgr.y (mgr.14901) 662 : cluster [DBG] pgmap v669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:36.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:36 smithi006 bash[27292]: cluster 2023-08-27T22:20:35.381920+0000 mgr.y (mgr.14901) 662 : cluster [DBG] pgmap v669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:36.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:36 smithi006 bash[33918]: cluster 2023-08-27T22:20:35.381920+0000 mgr.y (mgr.14901) 662 : cluster [DBG] pgmap v669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:38 smithi130 bash[27579]: cluster 2023-08-27T22:20:37.382439+0000 mgr.y (mgr.14901) 663 : cluster [DBG] pgmap v670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:38 smithi130 bash[27579]: audit 2023-08-27T22:20:37.657427+0000 mon.c (mon.2) 278 : audit [DBG] from='client.? 172.21.15.6:0/596551106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:38.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:38 smithi006 bash[33918]: cluster 2023-08-27T22:20:37.382439+0000 mgr.y (mgr.14901) 663 : cluster [DBG] pgmap v670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:38.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:38 smithi006 bash[33918]: audit 2023-08-27T22:20:37.657427+0000 mon.c (mon.2) 278 : audit [DBG] from='client.? 172.21.15.6:0/596551106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:38.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:38 smithi006 bash[27292]: cluster 2023-08-27T22:20:37.382439+0000 mgr.y (mgr.14901) 663 : cluster [DBG] pgmap v670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:38.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:38 smithi006 bash[27292]: audit 2023-08-27T22:20:37.657427+0000 mon.c (mon.2) 278 : audit [DBG] from='client.? 172.21.15.6:0/596551106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:40.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:20:40.167Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:40.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:20:40.168Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:40 smithi130 bash[27579]: cluster 2023-08-27T22:20:39.383358+0000 mgr.y (mgr.14901) 664 : cluster [DBG] pgmap v671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:40 smithi130 bash[27579]: audit 2023-08-27T22:20:40.068373+0000 mon.c (mon.2) 279 : audit [DBG] from='client.? 172.21.15.6:0/1466397020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:40.971 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:20:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:20:40] "GET /metrics HTTP/1.1" 200 193069 "" "Prometheus/2.33.4" 2023-08-27T22:20:40.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:40 smithi006 bash[27292]: cluster 2023-08-27T22:20:39.383358+0000 mgr.y (mgr.14901) 664 : cluster [DBG] pgmap v671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:40.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:40 smithi006 bash[27292]: audit 2023-08-27T22:20:40.068373+0000 mon.c (mon.2) 279 : audit [DBG] from='client.? 172.21.15.6:0/1466397020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:40.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:40 smithi006 bash[33918]: cluster 2023-08-27T22:20:39.383358+0000 mgr.y (mgr.14901) 664 : cluster [DBG] pgmap v671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:40.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:40 smithi006 bash[33918]: audit 2023-08-27T22:20:40.068373+0000 mon.c (mon.2) 279 : audit [DBG] from='client.? 172.21.15.6:0/1466397020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:20:41.732Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:20:41.734Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:42 smithi130 bash[27579]: cluster 2023-08-27T22:20:41.384378+0000 mgr.y (mgr.14901) 665 : cluster [DBG] pgmap v672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:42 smithi130 bash[27579]: audit 2023-08-27T22:20:42.481140+0000 mon.c (mon.2) 280 : audit [DBG] from='client.? 172.21.15.6:0/893580217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:42.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:42 smithi006 bash[33918]: cluster 2023-08-27T22:20:41.384378+0000 mgr.y (mgr.14901) 665 : cluster [DBG] pgmap v672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:42.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:42 smithi006 bash[33918]: audit 2023-08-27T22:20:42.481140+0000 mon.c (mon.2) 280 : audit [DBG] from='client.? 172.21.15.6:0/893580217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:42.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:42 smithi006 bash[27292]: cluster 2023-08-27T22:20:41.384378+0000 mgr.y (mgr.14901) 665 : cluster [DBG] pgmap v672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:42.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:42 smithi006 bash[27292]: audit 2023-08-27T22:20:42.481140+0000 mon.c (mon.2) 280 : audit [DBG] from='client.? 172.21.15.6:0/893580217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:43.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:43 smithi130 bash[27579]: audit 2023-08-27T22:20:42.726230+0000 mon.b (mon.1) 190 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:20:44.001 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:43 smithi006 bash[27292]: audit 2023-08-27T22:20:42.726230+0000 mon.b (mon.1) 190 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:20:44.002 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:20:43.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:44.002 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:20:43.627Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:44.002 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:43 smithi006 bash[33918]: audit 2023-08-27T22:20:42.726230+0000 mon.b (mon.1) 190 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:20:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:44 smithi130 bash[27579]: cluster 2023-08-27T22:20:43.387381+0000 mgr.y (mgr.14901) 666 : cluster [DBG] pgmap v673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:44.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:44 smithi006 bash[27292]: cluster 2023-08-27T22:20:43.387381+0000 mgr.y (mgr.14901) 666 : cluster [DBG] pgmap v673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:44.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:44 smithi006 bash[33918]: cluster 2023-08-27T22:20:43.387381+0000 mgr.y (mgr.14901) 666 : cluster [DBG] pgmap v673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:45.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:45 smithi130 bash[27579]: audit 2023-08-27T22:20:44.895043+0000 mon.c (mon.2) 281 : audit [DBG] from='client.? 172.21.15.6:0/1986996955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:45.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:45 smithi006 bash[33918]: audit 2023-08-27T22:20:44.895043+0000 mon.c (mon.2) 281 : audit [DBG] from='client.? 172.21.15.6:0/1986996955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:45.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:45 smithi006 bash[27292]: audit 2023-08-27T22:20:44.895043+0000 mon.c (mon.2) 281 : audit [DBG] from='client.? 172.21.15.6:0/1986996955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:46 smithi130 bash[27579]: cluster 2023-08-27T22:20:45.388416+0000 mgr.y (mgr.14901) 667 : cluster [DBG] pgmap v674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:46.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:46 smithi006 bash[33918]: cluster 2023-08-27T22:20:45.388416+0000 mgr.y (mgr.14901) 667 : cluster [DBG] pgmap v674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:46.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:46 smithi006 bash[27292]: cluster 2023-08-27T22:20:45.388416+0000 mgr.y (mgr.14901) 667 : cluster [DBG] pgmap v674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:47.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:47 smithi130 bash[27579]: audit 2023-08-27T22:20:47.306348+0000 mon.c (mon.2) 282 : audit [DBG] from='client.? 172.21.15.6:0/1696817859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:47.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:47 smithi006 bash[33918]: audit 2023-08-27T22:20:47.306348+0000 mon.c (mon.2) 282 : audit [DBG] from='client.? 172.21.15.6:0/1696817859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:47.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:47 smithi006 bash[27292]: audit 2023-08-27T22:20:47.306348+0000 mon.c (mon.2) 282 : audit [DBG] from='client.? 172.21.15.6:0/1696817859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:48 smithi130 bash[27579]: cluster 2023-08-27T22:20:47.389172+0000 mgr.y (mgr.14901) 668 : cluster [DBG] pgmap v675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:48.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:48 smithi006 bash[27292]: cluster 2023-08-27T22:20:47.389172+0000 mgr.y (mgr.14901) 668 : cluster [DBG] pgmap v675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:48.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:48 smithi006 bash[33918]: cluster 2023-08-27T22:20:47.389172+0000 mgr.y (mgr.14901) 668 : cluster [DBG] pgmap v675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:49.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:49 smithi006 bash[27292]: audit 2023-08-27T22:20:48.634233+0000 mon.a (mon.0) 1168 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:20:49.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:49 smithi006 bash[27292]: audit 2023-08-27T22:20:49.032160+0000 mon.a (mon.0) 1169 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:20:49.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:49 smithi006 bash[27292]: audit 2023-08-27T22:20:49.326515+0000 mon.b (mon.1) 191 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:20:49.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:49 smithi006 bash[27292]: audit 2023-08-27T22:20:49.328829+0000 mon.b (mon.1) 192 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:20:49.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:49 smithi006 bash[27292]: audit 2023-08-27T22:20:49.332651+0000 mon.a (mon.0) 1170 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:20:49.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:49 smithi006 bash[33918]: audit 2023-08-27T22:20:48.634233+0000 mon.a (mon.0) 1168 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:20:49.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:49 smithi006 bash[33918]: audit 2023-08-27T22:20:49.032160+0000 mon.a (mon.0) 1169 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:20:49.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:49 smithi006 bash[33918]: audit 2023-08-27T22:20:49.326515+0000 mon.b (mon.1) 191 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:20:49.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:49 smithi006 bash[33918]: audit 2023-08-27T22:20:49.328829+0000 mon.b (mon.1) 192 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:20:49.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:49 smithi006 bash[33918]: audit 2023-08-27T22:20:49.332651+0000 mon.a (mon.0) 1170 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:20:50.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:49 smithi130 bash[27579]: audit 2023-08-27T22:20:48.634233+0000 mon.a (mon.0) 1168 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:20:50.287 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:49 smithi130 bash[27579]: audit 2023-08-27T22:20:49.032160+0000 mon.a (mon.0) 1169 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:20:50.287 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:49 smithi130 bash[27579]: audit 2023-08-27T22:20:49.326515+0000 mon.b (mon.1) 191 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:20:50.287 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:49 smithi130 bash[27579]: audit 2023-08-27T22:20:49.328829+0000 mon.b (mon.1) 192 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:20:50.287 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:49 smithi130 bash[27579]: audit 2023-08-27T22:20:49.332651+0000 mon.a (mon.0) 1170 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:20:50.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:20:50.167Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:50.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:20:50.169Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:50.971 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:20:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:20:50] "GET /metrics HTTP/1.1" 200 193067 "" "Prometheus/2.33.4" 2023-08-27T22:20:50.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:50 smithi006 bash[33918]: cluster 2023-08-27T22:20:49.389955+0000 mgr.y (mgr.14901) 669 : cluster [DBG] pgmap v676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:50.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:50 smithi006 bash[33918]: audit 2023-08-27T22:20:49.725618+0000 mon.c (mon.2) 283 : audit [DBG] from='client.? 172.21.15.6:0/3308337632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:50.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:50 smithi006 bash[27292]: cluster 2023-08-27T22:20:49.389955+0000 mgr.y (mgr.14901) 669 : cluster [DBG] pgmap v676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:50.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:50 smithi006 bash[27292]: audit 2023-08-27T22:20:49.725618+0000 mon.c (mon.2) 283 : audit [DBG] from='client.? 172.21.15.6:0/3308337632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:50 smithi130 bash[27579]: cluster 2023-08-27T22:20:49.389955+0000 mgr.y (mgr.14901) 669 : cluster [DBG] pgmap v676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:50 smithi130 bash[27579]: audit 2023-08-27T22:20:49.725618+0000 mon.c (mon.2) 283 : audit [DBG] from='client.? 172.21.15.6:0/3308337632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:20:51.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:20:51.734Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:52.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:52 smithi006 bash[33918]: cluster 2023-08-27T22:20:51.390972+0000 mgr.y (mgr.14901) 670 : cluster [DBG] pgmap v677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:52.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:52 smithi006 bash[33918]: audit 2023-08-27T22:20:52.135358+0000 mon.c (mon.2) 284 : audit [DBG] from='client.? 172.21.15.6:0/389290800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:52.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:52 smithi006 bash[27292]: cluster 2023-08-27T22:20:51.390972+0000 mgr.y (mgr.14901) 670 : cluster [DBG] pgmap v677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:52.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:52 smithi006 bash[27292]: audit 2023-08-27T22:20:52.135358+0000 mon.c (mon.2) 284 : audit [DBG] from='client.? 172.21.15.6:0/389290800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:52 smithi130 bash[27579]: cluster 2023-08-27T22:20:51.390972+0000 mgr.y (mgr.14901) 670 : cluster [DBG] pgmap v677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:52 smithi130 bash[27579]: audit 2023-08-27T22:20:52.135358+0000 mon.c (mon.2) 284 : audit [DBG] from='client.? 172.21.15.6:0/389290800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:20:53.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:20:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:20:53.628Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:20:54.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:54 smithi006 bash[27292]: cluster 2023-08-27T22:20:53.391624+0000 mgr.y (mgr.14901) 671 : cluster [DBG] pgmap v678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:54.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:54 smithi006 bash[27292]: audit 2023-08-27T22:20:54.557605+0000 mon.c (mon.2) 285 : audit [DBG] from='client.? 172.21.15.6:0/3895359692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:54.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:54 smithi006 bash[33918]: cluster 2023-08-27T22:20:53.391624+0000 mgr.y (mgr.14901) 671 : cluster [DBG] pgmap v678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:54.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:54 smithi006 bash[33918]: audit 2023-08-27T22:20:54.557605+0000 mon.c (mon.2) 285 : audit [DBG] from='client.? 172.21.15.6:0/3895359692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:54 smithi130 bash[27579]: cluster 2023-08-27T22:20:53.391624+0000 mgr.y (mgr.14901) 671 : cluster [DBG] pgmap v678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:54 smithi130 bash[27579]: audit 2023-08-27T22:20:54.557605+0000 mon.c (mon.2) 285 : audit [DBG] from='client.? 172.21.15.6:0/3895359692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:56.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:56 smithi006 bash[33918]: cluster 2023-08-27T22:20:55.392672+0000 mgr.y (mgr.14901) 672 : cluster [DBG] pgmap v679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:56.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:56 smithi006 bash[27292]: cluster 2023-08-27T22:20:55.392672+0000 mgr.y (mgr.14901) 672 : cluster [DBG] pgmap v679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:57.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:56 smithi130 bash[27579]: cluster 2023-08-27T22:20:55.392672+0000 mgr.y (mgr.14901) 672 : cluster [DBG] pgmap v679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:57.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:57 smithi006 bash[27292]: audit 2023-08-27T22:20:56.975458+0000 mon.c (mon.2) 286 : audit [DBG] from='client.? 172.21.15.6:0/1346576532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:57.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:57 smithi006 bash[33918]: audit 2023-08-27T22:20:56.975458+0000 mon.c (mon.2) 286 : audit [DBG] from='client.? 172.21.15.6:0/1346576532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:58.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:57 smithi130 bash[27579]: audit 2023-08-27T22:20:56.975458+0000 mon.c (mon.2) 286 : audit [DBG] from='client.? 172.21.15.6:0/1346576532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:58.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:58 smithi006 bash[33918]: cluster 2023-08-27T22:20:57.393394+0000 mgr.y (mgr.14901) 673 : cluster [DBG] pgmap v680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:58.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:58 smithi006 bash[27292]: cluster 2023-08-27T22:20:57.393394+0000 mgr.y (mgr.14901) 673 : cluster [DBG] pgmap v680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:58 smithi130 bash[27579]: cluster 2023-08-27T22:20:57.393394+0000 mgr.y (mgr.14901) 673 : cluster [DBG] pgmap v680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:20:59.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:20:59 smithi006 bash[27292]: audit 2023-08-27T22:20:59.397458+0000 mon.c (mon.2) 287 : audit [DBG] from='client.? 172.21.15.6:0/4055428623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:59.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:20:59 smithi006 bash[33918]: audit 2023-08-27T22:20:59.397458+0000 mon.c (mon.2) 287 : audit [DBG] from='client.? 172.21.15.6:0/4055428623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:20:59 smithi130 bash[27579]: audit 2023-08-27T22:20:59.397458+0000 mon.c (mon.2) 287 : audit [DBG] from='client.? 172.21.15.6:0/4055428623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:00.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:21:00.168Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:00.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:21:00.170Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:00.971 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:21:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:21:00] "GET /metrics HTTP/1.1" 200 193069 "" "Prometheus/2.33.4" 2023-08-27T22:21:00.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:00 smithi006 bash[33918]: cluster 2023-08-27T22:20:59.394152+0000 mgr.y (mgr.14901) 674 : cluster [DBG] pgmap v681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:00.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:00 smithi006 bash[27292]: cluster 2023-08-27T22:20:59.394152+0000 mgr.y (mgr.14901) 674 : cluster [DBG] pgmap v681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:00 smithi130 bash[27579]: cluster 2023-08-27T22:20:59.394152+0000 mgr.y (mgr.14901) 674 : cluster [DBG] pgmap v681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:21:01.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:21:01.735Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:02.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:02 smithi006 bash[27292]: cluster 2023-08-27T22:21:01.395082+0000 mgr.y (mgr.14901) 675 : cluster [DBG] pgmap v682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:02.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:02 smithi006 bash[27292]: audit 2023-08-27T22:21:01.819043+0000 mon.a (mon.0) 1171 : audit [DBG] from='client.? 172.21.15.6:0/1844876784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:02.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:02 smithi006 bash[33918]: cluster 2023-08-27T22:21:01.395082+0000 mgr.y (mgr.14901) 675 : cluster [DBG] pgmap v682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:02.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:02 smithi006 bash[33918]: audit 2023-08-27T22:21:01.819043+0000 mon.a (mon.0) 1171 : audit [DBG] from='client.? 172.21.15.6:0/1844876784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:02 smithi130 bash[27579]: cluster 2023-08-27T22:21:01.395082+0000 mgr.y (mgr.14901) 675 : cluster [DBG] pgmap v682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:02 smithi130 bash[27579]: audit 2023-08-27T22:21:01.819043+0000 mon.a (mon.0) 1171 : audit [DBG] from='client.? 172.21.15.6:0/1844876784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:21:03.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:21:03.628Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:04.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:04 smithi006 bash[27292]: cluster 2023-08-27T22:21:03.395891+0000 mgr.y (mgr.14901) 676 : cluster [DBG] pgmap v683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:05.066 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:04 smithi006 bash[27292]: audit 2023-08-27T22:21:04.232461+0000 mon.a (mon.0) 1172 : audit [DBG] from='client.? 172.21.15.6:0/2092019506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:05.066 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:04 smithi006 bash[33918]: cluster 2023-08-27T22:21:03.395891+0000 mgr.y (mgr.14901) 676 : cluster [DBG] pgmap v683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:05.067 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:04 smithi006 bash[33918]: audit 2023-08-27T22:21:04.232461+0000 mon.a (mon.0) 1172 : audit [DBG] from='client.? 172.21.15.6:0/2092019506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:04 smithi130 bash[27579]: cluster 2023-08-27T22:21:03.395891+0000 mgr.y (mgr.14901) 676 : cluster [DBG] pgmap v683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:05.147 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:04 smithi130 bash[27579]: audit 2023-08-27T22:21:04.232461+0000 mon.a (mon.0) 1172 : audit [DBG] from='client.? 172.21.15.6:0/2092019506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:06.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:06 smithi006 bash[27292]: cluster 2023-08-27T22:21:05.397128+0000 mgr.y (mgr.14901) 677 : cluster [DBG] pgmap v684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:06.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:06 smithi006 bash[27292]: audit 2023-08-27T22:21:06.644324+0000 mon.a (mon.0) 1173 : audit [DBG] from='client.? 172.21.15.6:0/2892031544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:06.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:06 smithi006 bash[33918]: cluster 2023-08-27T22:21:05.397128+0000 mgr.y (mgr.14901) 677 : cluster [DBG] pgmap v684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:06.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:06 smithi006 bash[33918]: audit 2023-08-27T22:21:06.644324+0000 mon.a (mon.0) 1173 : audit [DBG] from='client.? 172.21.15.6:0/2892031544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:06 smithi130 bash[27579]: cluster 2023-08-27T22:21:05.397128+0000 mgr.y (mgr.14901) 677 : cluster [DBG] pgmap v684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:06 smithi130 bash[27579]: audit 2023-08-27T22:21:06.644324+0000 mon.a (mon.0) 1173 : audit [DBG] from='client.? 172.21.15.6:0/2892031544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:08.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:08 smithi006 bash[27292]: cluster 2023-08-27T22:21:07.397929+0000 mgr.y (mgr.14901) 678 : cluster [DBG] pgmap v685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:08.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:08 smithi006 bash[33918]: cluster 2023-08-27T22:21:07.397929+0000 mgr.y (mgr.14901) 678 : cluster [DBG] pgmap v685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:08 smithi130 bash[27579]: cluster 2023-08-27T22:21:07.397929+0000 mgr.y (mgr.14901) 678 : cluster [DBG] pgmap v685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:09.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:09 smithi006 bash[27292]: audit 2023-08-27T22:21:09.059462+0000 mon.c (mon.2) 288 : audit [DBG] from='client.? 172.21.15.6:0/4011764538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:09.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:09 smithi006 bash[33918]: audit 2023-08-27T22:21:09.059462+0000 mon.c (mon.2) 288 : audit [DBG] from='client.? 172.21.15.6:0/4011764538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:10.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:09 smithi130 bash[27579]: audit 2023-08-27T22:21:09.059462+0000 mon.c (mon.2) 288 : audit [DBG] from='client.? 172.21.15.6:0/4011764538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:10.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:21:10.169Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:10.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:21:10.171Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:10.971 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:21:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:21:10] "GET /metrics HTTP/1.1" 200 193069 "" "Prometheus/2.33.4" 2023-08-27T22:21:10.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:10 smithi006 bash[27292]: cluster 2023-08-27T22:21:09.398723+0000 mgr.y (mgr.14901) 679 : cluster [DBG] pgmap v686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:10.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:10 smithi006 bash[33918]: cluster 2023-08-27T22:21:09.398723+0000 mgr.y (mgr.14901) 679 : cluster [DBG] pgmap v686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:10 smithi130 bash[27579]: cluster 2023-08-27T22:21:09.398723+0000 mgr.y (mgr.14901) 679 : cluster [DBG] pgmap v686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:12.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:11 smithi130 bash[27579]: audit 2023-08-27T22:21:11.477962+0000 mon.c (mon.2) 289 : audit [DBG] from='client.? 172.21.15.6:0/1687827297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:12.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:11 smithi006 bash[27292]: audit 2023-08-27T22:21:11.477962+0000 mon.c (mon.2) 289 : audit [DBG] from='client.? 172.21.15.6:0/1687827297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:12.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:11 smithi006 bash[33918]: audit 2023-08-27T22:21:11.477962+0000 mon.c (mon.2) 289 : audit [DBG] from='client.? 172.21.15.6:0/1687827297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:21:11.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:21:11.736Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:12 smithi130 bash[27579]: cluster 2023-08-27T22:21:11.399674+0000 mgr.y (mgr.14901) 680 : cluster [DBG] pgmap v687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:13.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:12 smithi006 bash[27292]: cluster 2023-08-27T22:21:11.399674+0000 mgr.y (mgr.14901) 680 : cluster [DBG] pgmap v687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:13.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:12 smithi006 bash[33918]: cluster 2023-08-27T22:21:11.399674+0000 mgr.y (mgr.14901) 680 : cluster [DBG] pgmap v687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:21:13.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:21:13.629Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:14 smithi130 bash[27579]: cluster 2023-08-27T22:21:13.400335+0000 mgr.y (mgr.14901) 681 : cluster [DBG] pgmap v688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:14 smithi130 bash[27579]: audit 2023-08-27T22:21:13.898515+0000 mon.a (mon.0) 1174 : audit [DBG] from='client.? 172.21.15.6:0/1309277761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:15.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:14 smithi006 bash[33918]: cluster 2023-08-27T22:21:13.400335+0000 mgr.y (mgr.14901) 681 : cluster [DBG] pgmap v688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:15.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:14 smithi006 bash[33918]: audit 2023-08-27T22:21:13.898515+0000 mon.a (mon.0) 1174 : audit [DBG] from='client.? 172.21.15.6:0/1309277761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:15.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:14 smithi006 bash[27292]: cluster 2023-08-27T22:21:13.400335+0000 mgr.y (mgr.14901) 681 : cluster [DBG] pgmap v688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:15.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:14 smithi006 bash[27292]: audit 2023-08-27T22:21:13.898515+0000 mon.a (mon.0) 1174 : audit [DBG] from='client.? 172.21.15.6:0/1309277761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:17.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:16 smithi130 bash[27579]: cluster 2023-08-27T22:21:15.401355+0000 mgr.y (mgr.14901) 682 : cluster [DBG] pgmap v689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:17.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:16 smithi130 bash[27579]: audit 2023-08-27T22:21:16.317405+0000 mon.a (mon.0) 1175 : audit [DBG] from='client.? 172.21.15.6:0/3432050452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:17.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:16 smithi006 bash[33918]: cluster 2023-08-27T22:21:15.401355+0000 mgr.y (mgr.14901) 682 : cluster [DBG] pgmap v689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:17.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:16 smithi006 bash[33918]: audit 2023-08-27T22:21:16.317405+0000 mon.a (mon.0) 1175 : audit [DBG] from='client.? 172.21.15.6:0/3432050452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:17.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:16 smithi006 bash[27292]: cluster 2023-08-27T22:21:15.401355+0000 mgr.y (mgr.14901) 682 : cluster [DBG] pgmap v689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:17.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:16 smithi006 bash[27292]: audit 2023-08-27T22:21:16.317405+0000 mon.a (mon.0) 1175 : audit [DBG] from='client.? 172.21.15.6:0/3432050452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:18 smithi130 bash[27579]: cluster 2023-08-27T22:21:17.402098+0000 mgr.y (mgr.14901) 683 : cluster [DBG] pgmap v690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:19.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:18 smithi006 bash[27292]: cluster 2023-08-27T22:21:17.402098+0000 mgr.y (mgr.14901) 683 : cluster [DBG] pgmap v690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:19.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:18 smithi006 bash[33918]: cluster 2023-08-27T22:21:17.402098+0000 mgr.y (mgr.14901) 683 : cluster [DBG] pgmap v690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:20.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:19 smithi130 bash[27579]: audit 2023-08-27T22:21:18.736518+0000 mon.c (mon.2) 290 : audit [DBG] from='client.? 172.21.15.6:0/3070582546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:20.177 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:19 smithi006 bash[33918]: audit 2023-08-27T22:21:18.736518+0000 mon.c (mon.2) 290 : audit [DBG] from='client.? 172.21.15.6:0/3070582546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:20.178 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:19 smithi006 bash[27292]: audit 2023-08-27T22:21:18.736518+0000 mon.c (mon.2) 290 : audit [DBG] from='client.? 172.21.15.6:0/3070582546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:20.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:21:20.170Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:20.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:21:20.172Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:20 smithi130 bash[27579]: cluster 2023-08-27T22:21:19.402920+0000 mgr.y (mgr.14901) 684 : cluster [DBG] pgmap v691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:21.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:21:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:21:20] "GET /metrics HTTP/1.1" 200 193068 "" "Prometheus/2.33.4" 2023-08-27T22:21:21.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:20 smithi006 bash[33918]: cluster 2023-08-27T22:21:19.402920+0000 mgr.y (mgr.14901) 684 : cluster [DBG] pgmap v691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:21.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:20 smithi006 bash[27292]: cluster 2023-08-27T22:21:19.402920+0000 mgr.y (mgr.14901) 684 : cluster [DBG] pgmap v691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:22.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:21 smithi130 bash[27579]: audit 2023-08-27T22:21:21.157739+0000 mon.a (mon.0) 1176 : audit [DBG] from='client.? 172.21.15.6:0/471304656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:22.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:21 smithi006 bash[33918]: audit 2023-08-27T22:21:21.157739+0000 mon.a (mon.0) 1176 : audit [DBG] from='client.? 172.21.15.6:0/471304656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:21:21.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:21:21.736Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:22.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:21 smithi006 bash[27292]: audit 2023-08-27T22:21:21.157739+0000 mon.a (mon.0) 1176 : audit [DBG] from='client.? 172.21.15.6:0/471304656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:22 smithi130 bash[27579]: cluster 2023-08-27T22:21:21.403935+0000 mgr.y (mgr.14901) 685 : cluster [DBG] pgmap v692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:23.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:22 smithi006 bash[27292]: cluster 2023-08-27T22:21:21.403935+0000 mgr.y (mgr.14901) 685 : cluster [DBG] pgmap v692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:23.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:22 smithi006 bash[33918]: cluster 2023-08-27T22:21:21.403935+0000 mgr.y (mgr.14901) 685 : cluster [DBG] pgmap v692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:21:23.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:21:23.630Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:24 smithi130 bash[27579]: cluster 2023-08-27T22:21:23.404549+0000 mgr.y (mgr.14901) 686 : cluster [DBG] pgmap v693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:24 smithi130 bash[27579]: audit 2023-08-27T22:21:23.576818+0000 mon.c (mon.2) 291 : audit [DBG] from='client.? 172.21.15.6:0/4233834243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:24.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:24 smithi006 bash[27292]: cluster 2023-08-27T22:21:23.404549+0000 mgr.y (mgr.14901) 686 : cluster [DBG] pgmap v693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:24.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:24 smithi006 bash[27292]: audit 2023-08-27T22:21:23.576818+0000 mon.c (mon.2) 291 : audit [DBG] from='client.? 172.21.15.6:0/4233834243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:24.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:24 smithi006 bash[33918]: cluster 2023-08-27T22:21:23.404549+0000 mgr.y (mgr.14901) 686 : cluster [DBG] pgmap v693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:24.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:24 smithi006 bash[33918]: audit 2023-08-27T22:21:23.576818+0000 mon.c (mon.2) 291 : audit [DBG] from='client.? 172.21.15.6:0/4233834243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:26 smithi130 bash[27579]: cluster 2023-08-27T22:21:25.405643+0000 mgr.y (mgr.14901) 687 : cluster [DBG] pgmap v694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:26 smithi130 bash[27579]: audit 2023-08-27T22:21:25.994035+0000 mon.c (mon.2) 292 : audit [DBG] from='client.? 172.21.15.6:0/1932855876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:26.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:26 smithi006 bash[27292]: cluster 2023-08-27T22:21:25.405643+0000 mgr.y (mgr.14901) 687 : cluster [DBG] pgmap v694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:26.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:26 smithi006 bash[27292]: audit 2023-08-27T22:21:25.994035+0000 mon.c (mon.2) 292 : audit [DBG] from='client.? 172.21.15.6:0/1932855876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:26.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:26 smithi006 bash[33918]: cluster 2023-08-27T22:21:25.405643+0000 mgr.y (mgr.14901) 687 : cluster [DBG] pgmap v694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:26.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:26 smithi006 bash[33918]: audit 2023-08-27T22:21:25.994035+0000 mon.c (mon.2) 292 : audit [DBG] from='client.? 172.21.15.6:0/1932855876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:28 smithi130 bash[27579]: cluster 2023-08-27T22:21:27.406421+0000 mgr.y (mgr.14901) 688 : cluster [DBG] pgmap v695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:28 smithi130 bash[27579]: audit 2023-08-27T22:21:28.410019+0000 mon.a (mon.0) 1177 : audit [DBG] from='client.? 172.21.15.6:0/725304656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:28.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:28 smithi006 bash[33918]: cluster 2023-08-27T22:21:27.406421+0000 mgr.y (mgr.14901) 688 : cluster [DBG] pgmap v695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:28.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:28 smithi006 bash[33918]: audit 2023-08-27T22:21:28.410019+0000 mon.a (mon.0) 1177 : audit [DBG] from='client.? 172.21.15.6:0/725304656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:28.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:28 smithi006 bash[27292]: cluster 2023-08-27T22:21:27.406421+0000 mgr.y (mgr.14901) 688 : cluster [DBG] pgmap v695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:28.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:28 smithi006 bash[27292]: audit 2023-08-27T22:21:28.410019+0000 mon.a (mon.0) 1177 : audit [DBG] from='client.? 172.21.15.6:0/725304656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:30.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:21:30.171Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:30.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:21:30.173Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:30.754 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:30 smithi006 bash[33918]: cluster 2023-08-27T22:21:29.407240+0000 mgr.y (mgr.14901) 689 : cluster [DBG] pgmap v696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:30.754 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:30 smithi006 bash[27292]: cluster 2023-08-27T22:21:29.407240+0000 mgr.y (mgr.14901) 689 : cluster [DBG] pgmap v696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:30 smithi130 bash[27579]: cluster 2023-08-27T22:21:29.407240+0000 mgr.y (mgr.14901) 689 : cluster [DBG] pgmap v696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:31.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:21:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:21:30] "GET /metrics HTTP/1.1" 200 193066 "" "Prometheus/2.33.4" 2023-08-27T22:21:31.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:31 smithi130 bash[27579]: audit 2023-08-27T22:21:30.823811+0000 mon.a (mon.0) 1178 : audit [DBG] from='client.? 172.21.15.6:0/3025128949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:31.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:31 smithi006 bash[33918]: audit 2023-08-27T22:21:30.823811+0000 mon.a (mon.0) 1178 : audit [DBG] from='client.? 172.21.15.6:0/3025128949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:31.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:31 smithi006 bash[27292]: audit 2023-08-27T22:21:30.823811+0000 mon.a (mon.0) 1178 : audit [DBG] from='client.? 172.21.15.6:0/3025128949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:31.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:21:31.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:31.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:21:31.738Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:32 smithi130 bash[27579]: cluster 2023-08-27T22:21:31.408193+0000 mgr.y (mgr.14901) 690 : cluster [DBG] pgmap v697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:32.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:32 smithi006 bash[27292]: cluster 2023-08-27T22:21:31.408193+0000 mgr.y (mgr.14901) 690 : cluster [DBG] pgmap v697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:32.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:32 smithi006 bash[33918]: cluster 2023-08-27T22:21:31.408193+0000 mgr.y (mgr.14901) 690 : cluster [DBG] pgmap v697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:33.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:33 smithi130 bash[27579]: audit 2023-08-27T22:21:33.241714+0000 mon.a (mon.0) 1179 : audit [DBG] from='client.? 172.21.15.6:0/1327730178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:33.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:33 smithi006 bash[27292]: audit 2023-08-27T22:21:33.241714+0000 mon.a (mon.0) 1179 : audit [DBG] from='client.? 172.21.15.6:0/1327730178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:33.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:33 smithi006 bash[33918]: audit 2023-08-27T22:21:33.241714+0000 mon.a (mon.0) 1179 : audit [DBG] from='client.? 172.21.15.6:0/1327730178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:21:33.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:21:33.630Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:34 smithi130 bash[27579]: cluster 2023-08-27T22:21:33.408857+0000 mgr.y (mgr.14901) 691 : cluster [DBG] pgmap v698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:34.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:34 smithi006 bash[27292]: cluster 2023-08-27T22:21:33.408857+0000 mgr.y (mgr.14901) 691 : cluster [DBG] pgmap v698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:34.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:34 smithi006 bash[33918]: cluster 2023-08-27T22:21:33.408857+0000 mgr.y (mgr.14901) 691 : cluster [DBG] pgmap v698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:36 smithi130 bash[27579]: cluster 2023-08-27T22:21:35.409878+0000 mgr.y (mgr.14901) 692 : cluster [DBG] pgmap v699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:36 smithi130 bash[27579]: audit 2023-08-27T22:21:35.656577+0000 mon.a (mon.0) 1180 : audit [DBG] from='client.? 172.21.15.6:0/4116431038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:36.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:36 smithi006 bash[27292]: cluster 2023-08-27T22:21:35.409878+0000 mgr.y (mgr.14901) 692 : cluster [DBG] pgmap v699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:37.050 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:36 smithi006 bash[27292]: audit 2023-08-27T22:21:35.656577+0000 mon.a (mon.0) 1180 : audit [DBG] from='client.? 172.21.15.6:0/4116431038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:37.050 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:36 smithi006 bash[33918]: cluster 2023-08-27T22:21:35.409878+0000 mgr.y (mgr.14901) 692 : cluster [DBG] pgmap v699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:37.050 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:36 smithi006 bash[33918]: audit 2023-08-27T22:21:35.656577+0000 mon.a (mon.0) 1180 : audit [DBG] from='client.? 172.21.15.6:0/4116431038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:38 smithi130 bash[27579]: cluster 2023-08-27T22:21:37.410578+0000 mgr.y (mgr.14901) 693 : cluster [DBG] pgmap v700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:38 smithi130 bash[27579]: audit 2023-08-27T22:21:38.072151+0000 mon.a (mon.0) 1181 : audit [DBG] from='client.? 172.21.15.6:0/2073346226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:38.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:38 smithi006 bash[27292]: cluster 2023-08-27T22:21:37.410578+0000 mgr.y (mgr.14901) 693 : cluster [DBG] pgmap v700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:38.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:38 smithi006 bash[27292]: audit 2023-08-27T22:21:38.072151+0000 mon.a (mon.0) 1181 : audit [DBG] from='client.? 172.21.15.6:0/2073346226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:38.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:38 smithi006 bash[33918]: cluster 2023-08-27T22:21:37.410578+0000 mgr.y (mgr.14901) 693 : cluster [DBG] pgmap v700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:38.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:38 smithi006 bash[33918]: audit 2023-08-27T22:21:38.072151+0000 mon.a (mon.0) 1181 : audit [DBG] from='client.? 172.21.15.6:0/2073346226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:40.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:21:40.171Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:40.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:21:40.173Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:40 smithi130 bash[27579]: cluster 2023-08-27T22:21:39.411439+0000 mgr.y (mgr.14901) 694 : cluster [DBG] pgmap v701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:40 smithi130 bash[27579]: audit 2023-08-27T22:21:40.492123+0000 mon.a (mon.0) 1182 : audit [DBG] from='client.? 172.21.15.6:0/370313344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:40.971 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:21:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:21:40] "GET /metrics HTTP/1.1" 200 193066 "" "Prometheus/2.33.4" 2023-08-27T22:21:40.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:40 smithi006 bash[27292]: cluster 2023-08-27T22:21:39.411439+0000 mgr.y (mgr.14901) 694 : cluster [DBG] pgmap v701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:40.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:40 smithi006 bash[27292]: audit 2023-08-27T22:21:40.492123+0000 mon.a (mon.0) 1182 : audit [DBG] from='client.? 172.21.15.6:0/370313344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:40.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:40 smithi006 bash[33918]: cluster 2023-08-27T22:21:39.411439+0000 mgr.y (mgr.14901) 694 : cluster [DBG] pgmap v701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:40.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:40 smithi006 bash[33918]: audit 2023-08-27T22:21:40.492123+0000 mon.a (mon.0) 1182 : audit [DBG] from='client.? 172.21.15.6:0/370313344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:21:41.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:21:41.738Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:42 smithi130 bash[27579]: cluster 2023-08-27T22:21:41.412439+0000 mgr.y (mgr.14901) 695 : cluster [DBG] pgmap v702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:42.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:42 smithi006 bash[27292]: cluster 2023-08-27T22:21:41.412439+0000 mgr.y (mgr.14901) 695 : cluster [DBG] pgmap v702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:42.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:42 smithi006 bash[33918]: cluster 2023-08-27T22:21:41.412439+0000 mgr.y (mgr.14901) 695 : cluster [DBG] pgmap v702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:43.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:43 smithi130 bash[27579]: audit 2023-08-27T22:21:42.912779+0000 mon.c (mon.2) 293 : audit [DBG] from='client.? 172.21.15.6:0/332129856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:43.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:43 smithi006 bash[33918]: audit 2023-08-27T22:21:42.912779+0000 mon.c (mon.2) 293 : audit [DBG] from='client.? 172.21.15.6:0/332129856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:43.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:43 smithi006 bash[27292]: audit 2023-08-27T22:21:42.912779+0000 mon.c (mon.2) 293 : audit [DBG] from='client.? 172.21.15.6:0/332129856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:21:43.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:43.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:21:43.631Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:44 smithi130 bash[27579]: cluster 2023-08-27T22:21:43.413040+0000 mgr.y (mgr.14901) 696 : cluster [DBG] pgmap v703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:44.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:44 smithi006 bash[27292]: cluster 2023-08-27T22:21:43.413040+0000 mgr.y (mgr.14901) 696 : cluster [DBG] pgmap v703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:44.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:44 smithi006 bash[33918]: cluster 2023-08-27T22:21:43.413040+0000 mgr.y (mgr.14901) 696 : cluster [DBG] pgmap v703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:45.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:45 smithi130 bash[27579]: audit 2023-08-27T22:21:45.327131+0000 mon.c (mon.2) 294 : audit [DBG] from='client.? 172.21.15.6:0/2348865151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:45.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:45 smithi006 bash[33918]: audit 2023-08-27T22:21:45.327131+0000 mon.c (mon.2) 294 : audit [DBG] from='client.? 172.21.15.6:0/2348865151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:45.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:45 smithi006 bash[27292]: audit 2023-08-27T22:21:45.327131+0000 mon.c (mon.2) 294 : audit [DBG] from='client.? 172.21.15.6:0/2348865151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:46 smithi130 bash[27579]: cluster 2023-08-27T22:21:45.414061+0000 mgr.y (mgr.14901) 697 : cluster [DBG] pgmap v704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:46.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:46 smithi006 bash[27292]: cluster 2023-08-27T22:21:45.414061+0000 mgr.y (mgr.14901) 697 : cluster [DBG] pgmap v704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:46.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:46 smithi006 bash[33918]: cluster 2023-08-27T22:21:45.414061+0000 mgr.y (mgr.14901) 697 : cluster [DBG] pgmap v704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:48 smithi130 bash[27579]: cluster 2023-08-27T22:21:47.414808+0000 mgr.y (mgr.14901) 698 : cluster [DBG] pgmap v705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:48 smithi130 bash[27579]: audit 2023-08-27T22:21:47.738839+0000 mon.a (mon.0) 1183 : audit [DBG] from='client.? 172.21.15.6:0/2627037736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:48.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:48 smithi006 bash[33918]: cluster 2023-08-27T22:21:47.414808+0000 mgr.y (mgr.14901) 698 : cluster [DBG] pgmap v705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:48.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:48 smithi006 bash[33918]: audit 2023-08-27T22:21:47.738839+0000 mon.a (mon.0) 1183 : audit [DBG] from='client.? 172.21.15.6:0/2627037736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:48.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:48 smithi006 bash[27292]: cluster 2023-08-27T22:21:47.414808+0000 mgr.y (mgr.14901) 698 : cluster [DBG] pgmap v705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:48.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:48 smithi006 bash[27292]: audit 2023-08-27T22:21:47.738839+0000 mon.a (mon.0) 1183 : audit [DBG] from='client.? 172.21.15.6:0/2627037736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:49.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:49 smithi130 bash[27579]: audit 2023-08-27T22:21:49.347815+0000 mon.b (mon.1) 193 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:21:49.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:49 smithi006 bash[27292]: audit 2023-08-27T22:21:49.347815+0000 mon.b (mon.1) 193 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:21:49.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:49 smithi006 bash[33918]: audit 2023-08-27T22:21:49.347815+0000 mon.b (mon.1) 193 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:21:50.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:21:50.172Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:50.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:21:50.174Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:50 smithi130 bash[27579]: cluster 2023-08-27T22:21:49.415652+0000 mgr.y (mgr.14901) 699 : cluster [DBG] pgmap v706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:50 smithi130 bash[27579]: audit 2023-08-27T22:21:50.163677+0000 mon.a (mon.0) 1184 : audit [DBG] from='client.? 172.21.15.6:0/2586711951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:50.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:50 smithi006 bash[27292]: cluster 2023-08-27T22:21:49.415652+0000 mgr.y (mgr.14901) 699 : cluster [DBG] pgmap v706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:50.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:50 smithi006 bash[27292]: audit 2023-08-27T22:21:50.163677+0000 mon.a (mon.0) 1184 : audit [DBG] from='client.? 172.21.15.6:0/2586711951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:50.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:21:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:21:50] "GET /metrics HTTP/1.1" 200 193067 "" "Prometheus/2.33.4" 2023-08-27T22:21:50.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:50 smithi006 bash[33918]: cluster 2023-08-27T22:21:49.415652+0000 mgr.y (mgr.14901) 699 : cluster [DBG] pgmap v706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:50.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:50 smithi006 bash[33918]: audit 2023-08-27T22:21:50.163677+0000 mon.a (mon.0) 1184 : audit [DBG] from='client.? 172.21.15.6:0/2586711951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:21:51.737Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:21:51.739Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:52 smithi130 bash[27579]: cluster 2023-08-27T22:21:51.416688+0000 mgr.y (mgr.14901) 700 : cluster [DBG] pgmap v707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:52 smithi130 bash[27579]: audit 2023-08-27T22:21:52.573479+0000 mon.a (mon.0) 1185 : audit [DBG] from='client.? 172.21.15.6:0/551090528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:52.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:52 smithi006 bash[33918]: cluster 2023-08-27T22:21:51.416688+0000 mgr.y (mgr.14901) 700 : cluster [DBG] pgmap v707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:53.003 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:52 smithi006 bash[33918]: audit 2023-08-27T22:21:52.573479+0000 mon.a (mon.0) 1185 : audit [DBG] from='client.? 172.21.15.6:0/551090528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:53.003 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:52 smithi006 bash[27292]: cluster 2023-08-27T22:21:51.416688+0000 mgr.y (mgr.14901) 700 : cluster [DBG] pgmap v707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:53.003 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:52 smithi006 bash[27292]: audit 2023-08-27T22:21:52.573479+0000 mon.a (mon.0) 1185 : audit [DBG] from='client.? 172.21.15.6:0/551090528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:53.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:21:53.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:53.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:21:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:21:53.633Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:21:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:54 smithi130 bash[27579]: cluster 2023-08-27T22:21:53.417353+0000 mgr.y (mgr.14901) 701 : cluster [DBG] pgmap v708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:54.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:54 smithi006 bash[33918]: cluster 2023-08-27T22:21:53.417353+0000 mgr.y (mgr.14901) 701 : cluster [DBG] pgmap v708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:54.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:54 smithi006 bash[27292]: cluster 2023-08-27T22:21:53.417353+0000 mgr.y (mgr.14901) 701 : cluster [DBG] pgmap v708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:55.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:55 smithi130 bash[27579]: audit 2023-08-27T22:21:55.013335+0000 mon.a (mon.0) 1186 : audit [DBG] from='client.? 172.21.15.6:0/1844120503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:55.872 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:55 smithi130 bash[27579]: audit 2023-08-27T22:21:55.212563+0000 mon.a (mon.0) 1187 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:21:55.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:55 smithi006 bash[33918]: audit 2023-08-27T22:21:55.013335+0000 mon.a (mon.0) 1186 : audit [DBG] from='client.? 172.21.15.6:0/1844120503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:55.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:55 smithi006 bash[33918]: audit 2023-08-27T22:21:55.212563+0000 mon.a (mon.0) 1187 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:21:55.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:55 smithi006 bash[27292]: audit 2023-08-27T22:21:55.013335+0000 mon.a (mon.0) 1186 : audit [DBG] from='client.? 172.21.15.6:0/1844120503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:55.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:55 smithi006 bash[27292]: audit 2023-08-27T22:21:55.212563+0000 mon.a (mon.0) 1187 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:21:56.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:56 smithi006 bash[27292]: cluster 2023-08-27T22:21:55.418420+0000 mgr.y (mgr.14901) 702 : cluster [DBG] pgmap v709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:56.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:56 smithi006 bash[27292]: audit 2023-08-27T22:21:55.698677+0000 mon.a (mon.0) 1188 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:21:56.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:56 smithi006 bash[27292]: audit 2023-08-27T22:21:55.999315+0000 mon.b (mon.1) 194 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:21:56.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:56 smithi006 bash[27292]: audit 2023-08-27T22:21:56.001384+0000 mon.b (mon.1) 195 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:21:56.974 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:56 smithi006 bash[27292]: audit 2023-08-27T22:21:56.005357+0000 mon.a (mon.0) 1189 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:21:56.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:56 smithi006 bash[33918]: cluster 2023-08-27T22:21:55.418420+0000 mgr.y (mgr.14901) 702 : cluster [DBG] pgmap v709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:56.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:56 smithi006 bash[33918]: audit 2023-08-27T22:21:55.698677+0000 mon.a (mon.0) 1188 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:21:56.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:56 smithi006 bash[33918]: audit 2023-08-27T22:21:55.999315+0000 mon.b (mon.1) 194 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:21:56.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:56 smithi006 bash[33918]: audit 2023-08-27T22:21:56.001384+0000 mon.b (mon.1) 195 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:21:56.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:56 smithi006 bash[33918]: audit 2023-08-27T22:21:56.005357+0000 mon.a (mon.0) 1189 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:21:57.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:56 smithi130 bash[27579]: cluster 2023-08-27T22:21:55.418420+0000 mgr.y (mgr.14901) 702 : cluster [DBG] pgmap v709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:57.281 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:56 smithi130 bash[27579]: audit 2023-08-27T22:21:55.698677+0000 mon.a (mon.0) 1188 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:21:57.282 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:56 smithi130 bash[27579]: audit 2023-08-27T22:21:55.999315+0000 mon.b (mon.1) 194 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:21:57.282 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:56 smithi130 bash[27579]: audit 2023-08-27T22:21:56.001384+0000 mon.b (mon.1) 195 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:21:57.283 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:56 smithi130 bash[27579]: audit 2023-08-27T22:21:56.005357+0000 mon.a (mon.0) 1189 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:21:57.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:57 smithi006 bash[27292]: audit 2023-08-27T22:21:57.428237+0000 mon.a (mon.0) 1190 : audit [DBG] from='client.? 172.21.15.6:0/1493737180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:57.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:57 smithi006 bash[33918]: audit 2023-08-27T22:21:57.428237+0000 mon.a (mon.0) 1190 : audit [DBG] from='client.? 172.21.15.6:0/1493737180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:58.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:57 smithi130 bash[27579]: audit 2023-08-27T22:21:57.428237+0000 mon.a (mon.0) 1190 : audit [DBG] from='client.? 172.21.15.6:0/1493737180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:58.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:21:58 smithi006 bash[33918]: cluster 2023-08-27T22:21:57.419150+0000 mgr.y (mgr.14901) 703 : cluster [DBG] pgmap v710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:58.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:21:58 smithi006 bash[27292]: cluster 2023-08-27T22:21:57.419150+0000 mgr.y (mgr.14901) 703 : cluster [DBG] pgmap v710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:21:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:21:58 smithi130 bash[27579]: cluster 2023-08-27T22:21:57.419150+0000 mgr.y (mgr.14901) 703 : cluster [DBG] pgmap v710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:00.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:22:00.172Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:00.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:22:00.174Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:00.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:00 smithi006 bash[33918]: cluster 2023-08-27T22:21:59.419970+0000 mgr.y (mgr.14901) 704 : cluster [DBG] pgmap v711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:00.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:00 smithi006 bash[33918]: audit 2023-08-27T22:21:59.834838+0000 mon.c (mon.2) 295 : audit [DBG] from='client.? 172.21.15.6:0/2525111862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:00.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:00 smithi006 bash[27292]: cluster 2023-08-27T22:21:59.419970+0000 mgr.y (mgr.14901) 704 : cluster [DBG] pgmap v711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:00.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:00 smithi006 bash[27292]: audit 2023-08-27T22:21:59.834838+0000 mon.c (mon.2) 295 : audit [DBG] from='client.? 172.21.15.6:0/2525111862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:00.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:22:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:22:00] "GET /metrics HTTP/1.1" 200 193070 "" "Prometheus/2.33.4" 2023-08-27T22:22:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:00 smithi130 bash[27579]: cluster 2023-08-27T22:21:59.419970+0000 mgr.y (mgr.14901) 704 : cluster [DBG] pgmap v711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:00 smithi130 bash[27579]: audit 2023-08-27T22:21:59.834838+0000 mon.c (mon.2) 295 : audit [DBG] from='client.? 172.21.15.6:0/2525111862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:22:01.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:22:01.740Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:02 smithi130 bash[27579]: cluster 2023-08-27T22:22:01.420959+0000 mgr.y (mgr.14901) 705 : cluster [DBG] pgmap v712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:02 smithi130 bash[27579]: audit 2023-08-27T22:22:02.251237+0000 mon.c (mon.2) 296 : audit [DBG] from='client.? 172.21.15.6:0/4031748950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:03.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:02 smithi006 bash[27292]: cluster 2023-08-27T22:22:01.420959+0000 mgr.y (mgr.14901) 705 : cluster [DBG] pgmap v712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:03.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:02 smithi006 bash[27292]: audit 2023-08-27T22:22:02.251237+0000 mon.c (mon.2) 296 : audit [DBG] from='client.? 172.21.15.6:0/4031748950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:03.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:02 smithi006 bash[33918]: cluster 2023-08-27T22:22:01.420959+0000 mgr.y (mgr.14901) 705 : cluster [DBG] pgmap v712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:03.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:02 smithi006 bash[33918]: audit 2023-08-27T22:22:02.251237+0000 mon.c (mon.2) 296 : audit [DBG] from='client.? 172.21.15.6:0/4031748950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:22:03.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:22:03.634Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:04 smithi130 bash[27579]: cluster 2023-08-27T22:22:03.421592+0000 mgr.y (mgr.14901) 706 : cluster [DBG] pgmap v713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:04 smithi130 bash[27579]: audit 2023-08-27T22:22:04.665514+0000 mon.c (mon.2) 297 : audit [DBG] from='client.? 172.21.15.6:0/1138518507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:05.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:04 smithi006 bash[33918]: cluster 2023-08-27T22:22:03.421592+0000 mgr.y (mgr.14901) 706 : cluster [DBG] pgmap v713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:05.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:04 smithi006 bash[33918]: audit 2023-08-27T22:22:04.665514+0000 mon.c (mon.2) 297 : audit [DBG] from='client.? 172.21.15.6:0/1138518507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:05.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:04 smithi006 bash[27292]: cluster 2023-08-27T22:22:03.421592+0000 mgr.y (mgr.14901) 706 : cluster [DBG] pgmap v713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:05.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:04 smithi006 bash[27292]: audit 2023-08-27T22:22:04.665514+0000 mon.c (mon.2) 297 : audit [DBG] from='client.? 172.21.15.6:0/1138518507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:06 smithi130 bash[27579]: cluster 2023-08-27T22:22:05.422634+0000 mgr.y (mgr.14901) 707 : cluster [DBG] pgmap v714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:07.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:06 smithi006 bash[27292]: cluster 2023-08-27T22:22:05.422634+0000 mgr.y (mgr.14901) 707 : cluster [DBG] pgmap v714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:07.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:06 smithi006 bash[33918]: cluster 2023-08-27T22:22:05.422634+0000 mgr.y (mgr.14901) 707 : cluster [DBG] pgmap v714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:08.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:07 smithi130 bash[27579]: audit 2023-08-27T22:22:07.088294+0000 mon.c (mon.2) 298 : audit [DBG] from='client.? 172.21.15.6:0/882021382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:08.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:07 smithi006 bash[27292]: audit 2023-08-27T22:22:07.088294+0000 mon.c (mon.2) 298 : audit [DBG] from='client.? 172.21.15.6:0/882021382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:08.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:07 smithi006 bash[33918]: audit 2023-08-27T22:22:07.088294+0000 mon.c (mon.2) 298 : audit [DBG] from='client.? 172.21.15.6:0/882021382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:08 smithi130 bash[27579]: cluster 2023-08-27T22:22:07.423406+0000 mgr.y (mgr.14901) 708 : cluster [DBG] pgmap v715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:08 smithi006 bash[33918]: cluster 2023-08-27T22:22:07.423406+0000 mgr.y (mgr.14901) 708 : cluster [DBG] pgmap v715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:09.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:08 smithi006 bash[27292]: cluster 2023-08-27T22:22:07.423406+0000 mgr.y (mgr.14901) 708 : cluster [DBG] pgmap v715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:10.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:09 smithi130 bash[27579]: audit 2023-08-27T22:22:09.497430+0000 mon.c (mon.2) 299 : audit [DBG] from='client.? 172.21.15.6:0/480308399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:10.181 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:09 smithi006 bash[33918]: audit 2023-08-27T22:22:09.497430+0000 mon.c (mon.2) 299 : audit [DBG] from='client.? 172.21.15.6:0/480308399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:10.181 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:09 smithi006 bash[27292]: audit 2023-08-27T22:22:09.497430+0000 mon.c (mon.2) 299 : audit [DBG] from='client.? 172.21.15.6:0/480308399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:10.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:22:10.174Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:10.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:22:10.176Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:10 smithi130 bash[27579]: cluster 2023-08-27T22:22:09.424246+0000 mgr.y (mgr.14901) 709 : cluster [DBG] pgmap v716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:11.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:22:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:22:10] "GET /metrics HTTP/1.1" 200 193070 "" "Prometheus/2.33.4" 2023-08-27T22:22:11.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:10 smithi006 bash[27292]: cluster 2023-08-27T22:22:09.424246+0000 mgr.y (mgr.14901) 709 : cluster [DBG] pgmap v716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:11.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:10 smithi006 bash[33918]: cluster 2023-08-27T22:22:09.424246+0000 mgr.y (mgr.14901) 709 : cluster [DBG] pgmap v716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:22:11.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:22:11.740Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:12 smithi130 bash[27579]: cluster 2023-08-27T22:22:11.425234+0000 mgr.y (mgr.14901) 710 : cluster [DBG] pgmap v717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:12 smithi130 bash[27579]: audit 2023-08-27T22:22:11.921157+0000 mon.c (mon.2) 300 : audit [DBG] from='client.? 172.21.15.6:0/4240757653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:13.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:12 smithi006 bash[27292]: cluster 2023-08-27T22:22:11.425234+0000 mgr.y (mgr.14901) 710 : cluster [DBG] pgmap v717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:13.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:12 smithi006 bash[27292]: audit 2023-08-27T22:22:11.921157+0000 mon.c (mon.2) 300 : audit [DBG] from='client.? 172.21.15.6:0/4240757653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:13.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:12 smithi006 bash[33918]: cluster 2023-08-27T22:22:11.425234+0000 mgr.y (mgr.14901) 710 : cluster [DBG] pgmap v717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:13.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:12 smithi006 bash[33918]: audit 2023-08-27T22:22:11.921157+0000 mon.c (mon.2) 300 : audit [DBG] from='client.? 172.21.15.6:0/4240757653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:22:13.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:22:13.633Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:14 smithi130 bash[27579]: cluster 2023-08-27T22:22:13.425816+0000 mgr.y (mgr.14901) 711 : cluster [DBG] pgmap v718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:14 smithi130 bash[27579]: audit 2023-08-27T22:22:14.335231+0000 mon.a (mon.0) 1191 : audit [DBG] from='client.? 172.21.15.6:0/2220176993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:15.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:14 smithi006 bash[33918]: cluster 2023-08-27T22:22:13.425816+0000 mgr.y (mgr.14901) 711 : cluster [DBG] pgmap v718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:15.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:14 smithi006 bash[33918]: audit 2023-08-27T22:22:14.335231+0000 mon.a (mon.0) 1191 : audit [DBG] from='client.? 172.21.15.6:0/2220176993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:15.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:14 smithi006 bash[27292]: cluster 2023-08-27T22:22:13.425816+0000 mgr.y (mgr.14901) 711 : cluster [DBG] pgmap v718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:15.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:14 smithi006 bash[27292]: audit 2023-08-27T22:22:14.335231+0000 mon.a (mon.0) 1191 : audit [DBG] from='client.? 172.21.15.6:0/2220176993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:17.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:16 smithi130 bash[27579]: cluster 2023-08-27T22:22:15.426984+0000 mgr.y (mgr.14901) 712 : cluster [DBG] pgmap v719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:17.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:16 smithi130 bash[27579]: audit 2023-08-27T22:22:16.745221+0000 mon.a (mon.0) 1192 : audit [DBG] from='client.? 172.21.15.6:0/3987136700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:17.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:16 smithi006 bash[33918]: cluster 2023-08-27T22:22:15.426984+0000 mgr.y (mgr.14901) 712 : cluster [DBG] pgmap v719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:17.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:16 smithi006 bash[33918]: audit 2023-08-27T22:22:16.745221+0000 mon.a (mon.0) 1192 : audit [DBG] from='client.? 172.21.15.6:0/3987136700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:17.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:16 smithi006 bash[27292]: cluster 2023-08-27T22:22:15.426984+0000 mgr.y (mgr.14901) 712 : cluster [DBG] pgmap v719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:17.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:16 smithi006 bash[27292]: audit 2023-08-27T22:22:16.745221+0000 mon.a (mon.0) 1192 : audit [DBG] from='client.? 172.21.15.6:0/3987136700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:18 smithi130 bash[27579]: cluster 2023-08-27T22:22:17.427855+0000 mgr.y (mgr.14901) 713 : cluster [DBG] pgmap v720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:19.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:18 smithi006 bash[33918]: cluster 2023-08-27T22:22:17.427855+0000 mgr.y (mgr.14901) 713 : cluster [DBG] pgmap v720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:19.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:18 smithi006 bash[27292]: cluster 2023-08-27T22:22:17.427855+0000 mgr.y (mgr.14901) 713 : cluster [DBG] pgmap v720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:20.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:19 smithi130 bash[27579]: audit 2023-08-27T22:22:19.160635+0000 mon.a (mon.0) 1193 : audit [DBG] from='client.? 172.21.15.6:0/1173010348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:20.182 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:19 smithi006 bash[27292]: audit 2023-08-27T22:22:19.160635+0000 mon.a (mon.0) 1193 : audit [DBG] from='client.? 172.21.15.6:0/1173010348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:20.183 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:19 smithi006 bash[33918]: audit 2023-08-27T22:22:19.160635+0000 mon.a (mon.0) 1193 : audit [DBG] from='client.? 172.21.15.6:0/1173010348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:20.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:22:20.175Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:20.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:22:20.177Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:20 smithi130 bash[27579]: cluster 2023-08-27T22:22:19.428745+0000 mgr.y (mgr.14901) 714 : cluster [DBG] pgmap v721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:21.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:22:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:22:20] "GET /metrics HTTP/1.1" 200 193062 "" "Prometheus/2.33.4" 2023-08-27T22:22:21.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:20 smithi006 bash[33918]: cluster 2023-08-27T22:22:19.428745+0000 mgr.y (mgr.14901) 714 : cluster [DBG] pgmap v721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:21.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:20 smithi006 bash[27292]: cluster 2023-08-27T22:22:19.428745+0000 mgr.y (mgr.14901) 714 : cluster [DBG] pgmap v721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:22.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:21 smithi130 bash[27579]: audit 2023-08-27T22:22:21.579155+0000 mon.c (mon.2) 301 : audit [DBG] from='client.? 172.21.15.6:0/395259761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:22.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:21 smithi006 bash[33918]: audit 2023-08-27T22:22:21.579155+0000 mon.c (mon.2) 301 : audit [DBG] from='client.? 172.21.15.6:0/395259761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:22.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:21 smithi006 bash[27292]: audit 2023-08-27T22:22:21.579155+0000 mon.c (mon.2) 301 : audit [DBG] from='client.? 172.21.15.6:0/395259761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:22:21.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:22:21.742Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:22 smithi130 bash[27579]: cluster 2023-08-27T22:22:21.429755+0000 mgr.y (mgr.14901) 715 : cluster [DBG] pgmap v722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:23.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:22 smithi006 bash[27292]: cluster 2023-08-27T22:22:21.429755+0000 mgr.y (mgr.14901) 715 : cluster [DBG] pgmap v722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:23.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:22 smithi006 bash[33918]: cluster 2023-08-27T22:22:21.429755+0000 mgr.y (mgr.14901) 715 : cluster [DBG] pgmap v722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:22:23.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:22:23.635Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:24 smithi130 bash[27579]: cluster 2023-08-27T22:22:23.430476+0000 mgr.y (mgr.14901) 716 : cluster [DBG] pgmap v723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:24 smithi130 bash[27579]: audit 2023-08-27T22:22:24.000376+0000 mon.c (mon.2) 302 : audit [DBG] from='client.? 172.21.15.6:0/3397380591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:25.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:24 smithi006 bash[27292]: cluster 2023-08-27T22:22:23.430476+0000 mgr.y (mgr.14901) 716 : cluster [DBG] pgmap v723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:25.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:24 smithi006 bash[27292]: audit 2023-08-27T22:22:24.000376+0000 mon.c (mon.2) 302 : audit [DBG] from='client.? 172.21.15.6:0/3397380591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:25.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:24 smithi006 bash[33918]: cluster 2023-08-27T22:22:23.430476+0000 mgr.y (mgr.14901) 716 : cluster [DBG] pgmap v723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:25.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:24 smithi006 bash[33918]: audit 2023-08-27T22:22:24.000376+0000 mon.c (mon.2) 302 : audit [DBG] from='client.? 172.21.15.6:0/3397380591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:26 smithi130 bash[27579]: cluster 2023-08-27T22:22:25.431571+0000 mgr.y (mgr.14901) 717 : cluster [DBG] pgmap v724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:26 smithi130 bash[27579]: audit 2023-08-27T22:22:26.415902+0000 mon.c (mon.2) 303 : audit [DBG] from='client.? 172.21.15.6:0/332463738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:27.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:26 smithi006 bash[27292]: cluster 2023-08-27T22:22:25.431571+0000 mgr.y (mgr.14901) 717 : cluster [DBG] pgmap v724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:27.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:26 smithi006 bash[27292]: audit 2023-08-27T22:22:26.415902+0000 mon.c (mon.2) 303 : audit [DBG] from='client.? 172.21.15.6:0/332463738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:27.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:26 smithi006 bash[33918]: cluster 2023-08-27T22:22:25.431571+0000 mgr.y (mgr.14901) 717 : cluster [DBG] pgmap v724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:27.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:26 smithi006 bash[33918]: audit 2023-08-27T22:22:26.415902+0000 mon.c (mon.2) 303 : audit [DBG] from='client.? 172.21.15.6:0/332463738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:28 smithi130 bash[27579]: cluster 2023-08-27T22:22:27.432363+0000 mgr.y (mgr.14901) 718 : cluster [DBG] pgmap v725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:29.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:28 smithi006 bash[33918]: cluster 2023-08-27T22:22:27.432363+0000 mgr.y (mgr.14901) 718 : cluster [DBG] pgmap v725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:29.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:28 smithi006 bash[27292]: cluster 2023-08-27T22:22:27.432363+0000 mgr.y (mgr.14901) 718 : cluster [DBG] pgmap v725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:30.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:29 smithi130 bash[27579]: audit 2023-08-27T22:22:28.829269+0000 mon.a (mon.0) 1194 : audit [DBG] from='client.? 172.21.15.6:0/409309144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:30.183 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:29 smithi006 bash[33918]: audit 2023-08-27T22:22:28.829269+0000 mon.a (mon.0) 1194 : audit [DBG] from='client.? 172.21.15.6:0/409309144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:30.184 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:29 smithi006 bash[27292]: audit 2023-08-27T22:22:28.829269+0000 mon.a (mon.0) 1194 : audit [DBG] from='client.? 172.21.15.6:0/409309144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:30.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:22:30.176Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:30.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:22:30.178Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:31.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:30 smithi130 bash[27579]: cluster 2023-08-27T22:22:29.433233+0000 mgr.y (mgr.14901) 719 : cluster [DBG] pgmap v726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:31.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:22:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:22:30] "GET /metrics HTTP/1.1" 200 193066 "" "Prometheus/2.33.4" 2023-08-27T22:22:31.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:30 smithi006 bash[33918]: cluster 2023-08-27T22:22:29.433233+0000 mgr.y (mgr.14901) 719 : cluster [DBG] pgmap v726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:31.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:30 smithi006 bash[27292]: cluster 2023-08-27T22:22:29.433233+0000 mgr.y (mgr.14901) 719 : cluster [DBG] pgmap v726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:32.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:31 smithi130 bash[27579]: audit 2023-08-27T22:22:31.241310+0000 mon.c (mon.2) 304 : audit [DBG] from='client.? 172.21.15.6:0/2419095919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:32.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:31 smithi006 bash[27292]: audit 2023-08-27T22:22:31.241310+0000 mon.c (mon.2) 304 : audit [DBG] from='client.? 172.21.15.6:0/2419095919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:32.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:31 smithi006 bash[33918]: audit 2023-08-27T22:22:31.241310+0000 mon.c (mon.2) 304 : audit [DBG] from='client.? 172.21.15.6:0/2419095919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:22:31.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:22:31.742Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:32 smithi130 bash[27579]: cluster 2023-08-27T22:22:31.434194+0000 mgr.y (mgr.14901) 720 : cluster [DBG] pgmap v727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:33.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:32 smithi006 bash[33918]: cluster 2023-08-27T22:22:31.434194+0000 mgr.y (mgr.14901) 720 : cluster [DBG] pgmap v727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:33.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:32 smithi006 bash[27292]: cluster 2023-08-27T22:22:31.434194+0000 mgr.y (mgr.14901) 720 : cluster [DBG] pgmap v727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:22:33.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:22:33.636Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:34 smithi130 bash[27579]: cluster 2023-08-27T22:22:33.434836+0000 mgr.y (mgr.14901) 721 : cluster [DBG] pgmap v728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:34 smithi130 bash[27579]: audit 2023-08-27T22:22:33.662053+0000 mon.c (mon.2) 305 : audit [DBG] from='client.? 172.21.15.6:0/2720448394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:34.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:34 smithi006 bash[27292]: cluster 2023-08-27T22:22:33.434836+0000 mgr.y (mgr.14901) 721 : cluster [DBG] pgmap v728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:34.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:34 smithi006 bash[27292]: audit 2023-08-27T22:22:33.662053+0000 mon.c (mon.2) 305 : audit [DBG] from='client.? 172.21.15.6:0/2720448394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:34.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:34 smithi006 bash[33918]: cluster 2023-08-27T22:22:33.434836+0000 mgr.y (mgr.14901) 721 : cluster [DBG] pgmap v728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:34.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:34 smithi006 bash[33918]: audit 2023-08-27T22:22:33.662053+0000 mon.c (mon.2) 305 : audit [DBG] from='client.? 172.21.15.6:0/2720448394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:36 smithi130 bash[27579]: cluster 2023-08-27T22:22:35.435905+0000 mgr.y (mgr.14901) 722 : cluster [DBG] pgmap v729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:36 smithi130 bash[27579]: audit 2023-08-27T22:22:36.066420+0000 mon.c (mon.2) 306 : audit [DBG] from='client.? 172.21.15.6:0/3854156791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:36.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:36 smithi006 bash[27292]: cluster 2023-08-27T22:22:35.435905+0000 mgr.y (mgr.14901) 722 : cluster [DBG] pgmap v729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:36.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:36 smithi006 bash[27292]: audit 2023-08-27T22:22:36.066420+0000 mon.c (mon.2) 306 : audit [DBG] from='client.? 172.21.15.6:0/3854156791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:36.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:36 smithi006 bash[33918]: cluster 2023-08-27T22:22:35.435905+0000 mgr.y (mgr.14901) 722 : cluster [DBG] pgmap v729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:36.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:36 smithi006 bash[33918]: audit 2023-08-27T22:22:36.066420+0000 mon.c (mon.2) 306 : audit [DBG] from='client.? 172.21.15.6:0/3854156791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:38 smithi130 bash[27579]: cluster 2023-08-27T22:22:37.436754+0000 mgr.y (mgr.14901) 723 : cluster [DBG] pgmap v730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:38 smithi130 bash[27579]: audit 2023-08-27T22:22:38.476232+0000 mon.c (mon.2) 307 : audit [DBG] from='client.? 172.21.15.6:0/3120429434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:38.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:38 smithi006 bash[27292]: cluster 2023-08-27T22:22:37.436754+0000 mgr.y (mgr.14901) 723 : cluster [DBG] pgmap v730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:38.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:38 smithi006 bash[27292]: audit 2023-08-27T22:22:38.476232+0000 mon.c (mon.2) 307 : audit [DBG] from='client.? 172.21.15.6:0/3120429434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:38.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:38 smithi006 bash[33918]: cluster 2023-08-27T22:22:37.436754+0000 mgr.y (mgr.14901) 723 : cluster [DBG] pgmap v730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:38.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:38 smithi006 bash[33918]: audit 2023-08-27T22:22:38.476232+0000 mon.c (mon.2) 307 : audit [DBG] from='client.? 172.21.15.6:0/3120429434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:40.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:22:40.177Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:40.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:22:40.179Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:40 smithi130 bash[27579]: cluster 2023-08-27T22:22:39.437605+0000 mgr.y (mgr.14901) 724 : cluster [DBG] pgmap v731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:40.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:40 smithi006 bash[27292]: cluster 2023-08-27T22:22:39.437605+0000 mgr.y (mgr.14901) 724 : cluster [DBG] pgmap v731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:40.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:22:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:22:40] "GET /metrics HTTP/1.1" 200 193066 "" "Prometheus/2.33.4" 2023-08-27T22:22:40.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:40 smithi006 bash[33918]: cluster 2023-08-27T22:22:39.437605+0000 mgr.y (mgr.14901) 724 : cluster [DBG] pgmap v731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:41.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:41 smithi130 bash[27579]: audit 2023-08-27T22:22:40.892283+0000 mon.c (mon.2) 308 : audit [DBG] from='client.? 172.21.15.6:0/2482476331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:41.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:41 smithi006 bash[33918]: audit 2023-08-27T22:22:40.892283+0000 mon.c (mon.2) 308 : audit [DBG] from='client.? 172.21.15.6:0/2482476331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:41.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:41 smithi006 bash[27292]: audit 2023-08-27T22:22:40.892283+0000 mon.c (mon.2) 308 : audit [DBG] from='client.? 172.21.15.6:0/2482476331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:41.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:22:41.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:41.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:22:41.743Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:42 smithi130 bash[27579]: cluster 2023-08-27T22:22:41.438667+0000 mgr.y (mgr.14901) 725 : cluster [DBG] pgmap v732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:42.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:42 smithi006 bash[27292]: cluster 2023-08-27T22:22:41.438667+0000 mgr.y (mgr.14901) 725 : cluster [DBG] pgmap v732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:42.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:42 smithi006 bash[33918]: cluster 2023-08-27T22:22:41.438667+0000 mgr.y (mgr.14901) 725 : cluster [DBG] pgmap v732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:43.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:43 smithi130 bash[27579]: audit 2023-08-27T22:22:43.302286+0000 mon.c (mon.2) 309 : audit [DBG] from='client.? 172.21.15.6:0/360178419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:43.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:43 smithi006 bash[27292]: audit 2023-08-27T22:22:43.302286+0000 mon.c (mon.2) 309 : audit [DBG] from='client.? 172.21.15.6:0/360178419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:43.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:43 smithi006 bash[33918]: audit 2023-08-27T22:22:43.302286+0000 mon.c (mon.2) 309 : audit [DBG] from='client.? 172.21.15.6:0/360178419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:43.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:22:43.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:43.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:22:43.637Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:44 smithi130 bash[27579]: cluster 2023-08-27T22:22:43.439305+0000 mgr.y (mgr.14901) 726 : cluster [DBG] pgmap v733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:45.888 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:44 smithi006 bash[27292]: cluster 2023-08-27T22:22:43.439305+0000 mgr.y (mgr.14901) 726 : cluster [DBG] pgmap v733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:45.889 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:44 smithi006 bash[33918]: cluster 2023-08-27T22:22:43.439305+0000 mgr.y (mgr.14901) 726 : cluster [DBG] pgmap v733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:46 smithi130 bash[27579]: cluster 2023-08-27T22:22:45.440325+0000 mgr.y (mgr.14901) 727 : cluster [DBG] pgmap v734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:46 smithi130 bash[27579]: audit 2023-08-27T22:22:45.717489+0000 mon.a (mon.0) 1195 : audit [DBG] from='client.? 172.21.15.6:0/3338439176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:46.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:46 smithi006 bash[27292]: cluster 2023-08-27T22:22:45.440325+0000 mgr.y (mgr.14901) 727 : cluster [DBG] pgmap v734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:46.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:46 smithi006 bash[27292]: audit 2023-08-27T22:22:45.717489+0000 mon.a (mon.0) 1195 : audit [DBG] from='client.? 172.21.15.6:0/3338439176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:46.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:46 smithi006 bash[33918]: cluster 2023-08-27T22:22:45.440325+0000 mgr.y (mgr.14901) 727 : cluster [DBG] pgmap v734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:46.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:46 smithi006 bash[33918]: audit 2023-08-27T22:22:45.717489+0000 mon.a (mon.0) 1195 : audit [DBG] from='client.? 172.21.15.6:0/3338439176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:48 smithi130 bash[27579]: cluster 2023-08-27T22:22:47.441199+0000 mgr.y (mgr.14901) 728 : cluster [DBG] pgmap v735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:48 smithi130 bash[27579]: audit 2023-08-27T22:22:48.133119+0000 mon.c (mon.2) 310 : audit [DBG] from='client.? 172.21.15.6:0/1131652854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:48.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:48 smithi006 bash[27292]: cluster 2023-08-27T22:22:47.441199+0000 mgr.y (mgr.14901) 728 : cluster [DBG] pgmap v735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:48.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:48 smithi006 bash[27292]: audit 2023-08-27T22:22:48.133119+0000 mon.c (mon.2) 310 : audit [DBG] from='client.? 172.21.15.6:0/1131652854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:48.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:48 smithi006 bash[33918]: cluster 2023-08-27T22:22:47.441199+0000 mgr.y (mgr.14901) 728 : cluster [DBG] pgmap v735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:48.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:48 smithi006 bash[33918]: audit 2023-08-27T22:22:48.133119+0000 mon.c (mon.2) 310 : audit [DBG] from='client.? 172.21.15.6:0/1131652854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:50.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:22:50.178Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:50.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:22:50.180Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:50 smithi130 bash[27579]: cluster 2023-08-27T22:22:49.442042+0000 mgr.y (mgr.14901) 729 : cluster [DBG] pgmap v736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:50.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:50 smithi006 bash[27292]: cluster 2023-08-27T22:22:49.442042+0000 mgr.y (mgr.14901) 729 : cluster [DBG] pgmap v736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:50.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:22:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:22:50] "GET /metrics HTTP/1.1" 200 193066 "" "Prometheus/2.33.4" 2023-08-27T22:22:50.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:50 smithi006 bash[33918]: cluster 2023-08-27T22:22:49.442042+0000 mgr.y (mgr.14901) 729 : cluster [DBG] pgmap v736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:51.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:51 smithi130 bash[27579]: audit 2023-08-27T22:22:50.565925+0000 mon.c (mon.2) 311 : audit [DBG] from='client.? 172.21.15.6:0/4006592897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:51.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:51 smithi006 bash[27292]: audit 2023-08-27T22:22:50.565925+0000 mon.c (mon.2) 311 : audit [DBG] from='client.? 172.21.15.6:0/4006592897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:51.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:51 smithi006 bash[33918]: audit 2023-08-27T22:22:50.565925+0000 mon.c (mon.2) 311 : audit [DBG] from='client.? 172.21.15.6:0/4006592897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:51.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:22:51.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:51.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:22:51.743Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:52 smithi130 bash[27579]: cluster 2023-08-27T22:22:51.443020+0000 mgr.y (mgr.14901) 730 : cluster [DBG] pgmap v737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:52.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:52 smithi006 bash[27292]: cluster 2023-08-27T22:22:51.443020+0000 mgr.y (mgr.14901) 730 : cluster [DBG] pgmap v737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:52.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:52 smithi006 bash[33918]: cluster 2023-08-27T22:22:51.443020+0000 mgr.y (mgr.14901) 730 : cluster [DBG] pgmap v737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:53.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:53 smithi130 bash[27579]: audit 2023-08-27T22:22:52.982301+0000 mon.c (mon.2) 312 : audit [DBG] from='client.? 172.21.15.6:0/942864556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:53.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:53 smithi006 bash[27292]: audit 2023-08-27T22:22:52.982301+0000 mon.c (mon.2) 312 : audit [DBG] from='client.? 172.21.15.6:0/942864556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:53.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:53 smithi006 bash[33918]: audit 2023-08-27T22:22:52.982301+0000 mon.c (mon.2) 312 : audit [DBG] from='client.? 172.21.15.6:0/942864556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:53.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:22:53.635Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:53.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:22:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:22:53.637Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:22:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:54 smithi130 bash[27579]: cluster 2023-08-27T22:22:53.443688+0000 mgr.y (mgr.14901) 731 : cluster [DBG] pgmap v738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:54.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:54 smithi006 bash[33918]: cluster 2023-08-27T22:22:53.443688+0000 mgr.y (mgr.14901) 731 : cluster [DBG] pgmap v738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:54.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:54 smithi006 bash[27292]: cluster 2023-08-27T22:22:53.443688+0000 mgr.y (mgr.14901) 731 : cluster [DBG] pgmap v738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:55.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:55 smithi130 bash[27579]: audit 2023-08-27T22:22:55.390459+0000 mon.a (mon.0) 1196 : audit [DBG] from='client.? 172.21.15.6:0/1885407441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:55.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:55 smithi006 bash[33918]: audit 2023-08-27T22:22:55.390459+0000 mon.a (mon.0) 1196 : audit [DBG] from='client.? 172.21.15.6:0/1885407441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:55.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:55 smithi006 bash[27292]: audit 2023-08-27T22:22:55.390459+0000 mon.a (mon.0) 1196 : audit [DBG] from='client.? 172.21.15.6:0/1885407441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:56 smithi130 bash[27579]: cluster 2023-08-27T22:22:55.444921+0000 mgr.y (mgr.14901) 732 : cluster [DBG] pgmap v739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:56 smithi130 bash[27579]: audit 2023-08-27T22:22:56.022201+0000 mon.b (mon.1) 196 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:22:56.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:56 smithi006 bash[33918]: cluster 2023-08-27T22:22:55.444921+0000 mgr.y (mgr.14901) 732 : cluster [DBG] pgmap v739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:56.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:56 smithi006 bash[33918]: audit 2023-08-27T22:22:56.022201+0000 mon.b (mon.1) 196 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:22:56.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:56 smithi006 bash[27292]: cluster 2023-08-27T22:22:55.444921+0000 mgr.y (mgr.14901) 732 : cluster [DBG] pgmap v739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:56.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:56 smithi006 bash[27292]: audit 2023-08-27T22:22:56.022201+0000 mon.b (mon.1) 196 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:22:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:58 smithi130 bash[27579]: cluster 2023-08-27T22:22:57.445672+0000 mgr.y (mgr.14901) 733 : cluster [DBG] pgmap v740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:22:58 smithi130 bash[27579]: audit 2023-08-27T22:22:57.807882+0000 mon.c (mon.2) 313 : audit [DBG] from='client.? 172.21.15.6:0/2035090366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:58.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:58 smithi006 bash[33918]: cluster 2023-08-27T22:22:57.445672+0000 mgr.y (mgr.14901) 733 : cluster [DBG] pgmap v740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:58.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:22:58 smithi006 bash[33918]: audit 2023-08-27T22:22:57.807882+0000 mon.c (mon.2) 313 : audit [DBG] from='client.? 172.21.15.6:0/2035090366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:58.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:58 smithi006 bash[27292]: cluster 2023-08-27T22:22:57.445672+0000 mgr.y (mgr.14901) 733 : cluster [DBG] pgmap v740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:22:58.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:22:58 smithi006 bash[27292]: audit 2023-08-27T22:22:57.807882+0000 mon.c (mon.2) 313 : audit [DBG] from='client.? 172.21.15.6:0/2035090366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:00.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:23:00.178Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:00.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:23:00.180Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:00 smithi130 bash[27579]: cluster 2023-08-27T22:22:59.446520+0000 mgr.y (mgr.14901) 734 : cluster [DBG] pgmap v741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:00 smithi130 bash[27579]: audit 2023-08-27T22:23:00.231390+0000 mon.c (mon.2) 314 : audit [DBG] from='client.? 172.21.15.6:0/1809606537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:00.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:23:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:23:00] "GET /metrics HTTP/1.1" 200 193067 "" "Prometheus/2.33.4" 2023-08-27T22:23:00.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:00 smithi006 bash[27292]: cluster 2023-08-27T22:22:59.446520+0000 mgr.y (mgr.14901) 734 : cluster [DBG] pgmap v741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:00.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:00 smithi006 bash[27292]: audit 2023-08-27T22:23:00.231390+0000 mon.c (mon.2) 314 : audit [DBG] from='client.? 172.21.15.6:0/1809606537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:00.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:00 smithi006 bash[33918]: cluster 2023-08-27T22:22:59.446520+0000 mgr.y (mgr.14901) 734 : cluster [DBG] pgmap v741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:00.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:00 smithi006 bash[33918]: audit 2023-08-27T22:23:00.231390+0000 mon.c (mon.2) 314 : audit [DBG] from='client.? 172.21.15.6:0/1809606537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:23:01.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:23:01.744Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:03.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:02 smithi006 bash[33918]: cluster 2023-08-27T22:23:01.447432+0000 mgr.y (mgr.14901) 735 : cluster [DBG] pgmap v742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:03.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:02 smithi006 bash[33918]: audit 2023-08-27T22:23:01.907209+0000 mon.a (mon.0) 1197 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:23:03.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:02 smithi006 bash[33918]: audit 2023-08-27T22:23:02.342977+0000 mon.a (mon.0) 1198 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:23:03.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:02 smithi006 bash[33918]: audit 2023-08-27T22:23:02.637613+0000 mon.a (mon.0) 1199 : audit [DBG] from='client.? 172.21.15.6:0/2805913967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:03.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:02 smithi006 bash[27292]: cluster 2023-08-27T22:23:01.447432+0000 mgr.y (mgr.14901) 735 : cluster [DBG] pgmap v742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:03.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:02 smithi006 bash[27292]: audit 2023-08-27T22:23:01.907209+0000 mon.a (mon.0) 1197 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:23:03.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:02 smithi006 bash[27292]: audit 2023-08-27T22:23:02.342977+0000 mon.a (mon.0) 1198 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:23:03.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:02 smithi006 bash[27292]: audit 2023-08-27T22:23:02.637613+0000 mon.a (mon.0) 1199 : audit [DBG] from='client.? 172.21.15.6:0/2805913967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:03.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:02 smithi130 bash[27579]: cluster 2023-08-27T22:23:01.447432+0000 mgr.y (mgr.14901) 735 : cluster [DBG] pgmap v742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:03.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:02 smithi130 bash[27579]: audit 2023-08-27T22:23:01.907209+0000 mon.a (mon.0) 1197 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:23:03.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:02 smithi130 bash[27579]: audit 2023-08-27T22:23:02.342977+0000 mon.a (mon.0) 1198 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:23:03.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:02 smithi130 bash[27579]: audit 2023-08-27T22:23:02.637613+0000 mon.a (mon.0) 1199 : audit [DBG] from='client.? 172.21.15.6:0/2805913967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:23:03.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:23:03.638Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:05.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:04 smithi006 bash[33918]: cluster 2023-08-27T22:23:03.448141+0000 mgr.y (mgr.14901) 736 : cluster [DBG] pgmap v743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:05.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:04 smithi006 bash[27292]: cluster 2023-08-27T22:23:03.448141+0000 mgr.y (mgr.14901) 736 : cluster [DBG] pgmap v743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:04 smithi130 bash[27579]: cluster 2023-08-27T22:23:03.448141+0000 mgr.y (mgr.14901) 736 : cluster [DBG] pgmap v743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:06.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:05 smithi006 bash[33918]: audit 2023-08-27T22:23:05.056103+0000 mon.c (mon.2) 315 : audit [DBG] from='client.? 172.21.15.6:0/4227022308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:06.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:05 smithi006 bash[27292]: audit 2023-08-27T22:23:05.056103+0000 mon.c (mon.2) 315 : audit [DBG] from='client.? 172.21.15.6:0/4227022308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:05 smithi130 bash[27579]: audit 2023-08-27T22:23:05.056103+0000 mon.c (mon.2) 315 : audit [DBG] from='client.? 172.21.15.6:0/4227022308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:07.200 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:06 smithi130 bash[27579]: cluster 2023-08-27T22:23:05.449211+0000 mgr.y (mgr.14901) 737 : cluster [DBG] pgmap v744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:07.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:06 smithi006 bash[33918]: cluster 2023-08-27T22:23:05.449211+0000 mgr.y (mgr.14901) 737 : cluster [DBG] pgmap v744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:07.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:06 smithi006 bash[27292]: cluster 2023-08-27T22:23:05.449211+0000 mgr.y (mgr.14901) 737 : cluster [DBG] pgmap v744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:08.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:07 smithi006 bash[27292]: audit 2023-08-27T22:23:07.471754+0000 mon.c (mon.2) 316 : audit [DBG] from='client.? 172.21.15.6:0/3106814956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:08.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:07 smithi006 bash[33918]: audit 2023-08-27T22:23:07.471754+0000 mon.c (mon.2) 316 : audit [DBG] from='client.? 172.21.15.6:0/3106814956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:08.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:07 smithi130 bash[27579]: audit 2023-08-27T22:23:07.471754+0000 mon.c (mon.2) 316 : audit [DBG] from='client.? 172.21.15.6:0/3106814956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:09.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:08 smithi006 bash[27292]: cluster 2023-08-27T22:23:07.449935+0000 mgr.y (mgr.14901) 738 : cluster [DBG] pgmap v745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:08 smithi006 bash[33918]: cluster 2023-08-27T22:23:07.449935+0000 mgr.y (mgr.14901) 738 : cluster [DBG] pgmap v745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:09.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:08 smithi130 bash[27579]: cluster 2023-08-27T22:23:07.449935+0000 mgr.y (mgr.14901) 738 : cluster [DBG] pgmap v745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:10.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:09 smithi006 bash[27292]: audit 2023-08-27T22:23:09.884869+0000 mon.a (mon.0) 1200 : audit [DBG] from='client.? 172.21.15.6:0/2446375026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:10.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:09 smithi006 bash[33918]: audit 2023-08-27T22:23:09.884869+0000 mon.a (mon.0) 1200 : audit [DBG] from='client.? 172.21.15.6:0/2446375026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:10.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:23:10.178Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:10.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:23:10.180Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:10.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:09 smithi130 bash[27579]: audit 2023-08-27T22:23:09.884869+0000 mon.a (mon.0) 1200 : audit [DBG] from='client.? 172.21.15.6:0/2446375026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:11.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:23:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:23:10] "GET /metrics HTTP/1.1" 200 193067 "" "Prometheus/2.33.4" 2023-08-27T22:23:11.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:10 smithi006 bash[33918]: cluster 2023-08-27T22:23:09.450790+0000 mgr.y (mgr.14901) 739 : cluster [DBG] pgmap v746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:11.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:10 smithi006 bash[27292]: cluster 2023-08-27T22:23:09.450790+0000 mgr.y (mgr.14901) 739 : cluster [DBG] pgmap v746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:11.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:10 smithi130 bash[27579]: cluster 2023-08-27T22:23:09.450790+0000 mgr.y (mgr.14901) 739 : cluster [DBG] pgmap v746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:23:11.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:12.239 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:23:11.744Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:13.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:12 smithi006 bash[27292]: cluster 2023-08-27T22:23:11.451896+0000 mgr.y (mgr.14901) 740 : cluster [DBG] pgmap v747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:13.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:12 smithi006 bash[27292]: audit 2023-08-27T22:23:12.303480+0000 mon.c (mon.2) 317 : audit [DBG] from='client.? 172.21.15.6:0/3106654444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:13.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:12 smithi006 bash[27292]: audit 2023-08-27T22:23:12.404426+0000 mon.a (mon.0) 1201 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:23:13.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:12 smithi006 bash[27292]: audit 2023-08-27T22:23:12.676771+0000 mon.b (mon.1) 197 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:23:13.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:12 smithi006 bash[27292]: audit 2023-08-27T22:23:12.679090+0000 mon.b (mon.1) 198 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:23:13.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:12 smithi006 bash[27292]: audit 2023-08-27T22:23:12.683413+0000 mon.a (mon.0) 1202 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:23:13.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:12 smithi006 bash[33918]: cluster 2023-08-27T22:23:11.451896+0000 mgr.y (mgr.14901) 740 : cluster [DBG] pgmap v747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:13.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:12 smithi006 bash[33918]: audit 2023-08-27T22:23:12.303480+0000 mon.c (mon.2) 317 : audit [DBG] from='client.? 172.21.15.6:0/3106654444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:13.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:12 smithi006 bash[33918]: audit 2023-08-27T22:23:12.404426+0000 mon.a (mon.0) 1201 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:23:13.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:12 smithi006 bash[33918]: audit 2023-08-27T22:23:12.676771+0000 mon.b (mon.1) 197 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:23:13.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:12 smithi006 bash[33918]: audit 2023-08-27T22:23:12.679090+0000 mon.b (mon.1) 198 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:23:13.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:12 smithi006 bash[33918]: audit 2023-08-27T22:23:12.683413+0000 mon.a (mon.0) 1202 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:23:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:12 smithi130 bash[27579]: cluster 2023-08-27T22:23:11.451896+0000 mgr.y (mgr.14901) 740 : cluster [DBG] pgmap v747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:12 smithi130 bash[27579]: audit 2023-08-27T22:23:12.303480+0000 mon.c (mon.2) 317 : audit [DBG] from='client.? 172.21.15.6:0/3106654444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:12 smithi130 bash[27579]: audit 2023-08-27T22:23:12.404426+0000 mon.a (mon.0) 1201 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:23:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:12 smithi130 bash[27579]: audit 2023-08-27T22:23:12.676771+0000 mon.b (mon.1) 197 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:23:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:12 smithi130 bash[27579]: audit 2023-08-27T22:23:12.679090+0000 mon.b (mon.1) 198 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:23:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:12 smithi130 bash[27579]: audit 2023-08-27T22:23:12.683413+0000 mon.a (mon.0) 1202 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:23:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:23:13.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:23:13.638Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:14 smithi130 bash[27579]: cluster 2023-08-27T22:23:13.452553+0000 mgr.y (mgr.14901) 741 : cluster [DBG] pgmap v748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:14 smithi130 bash[27579]: audit 2023-08-27T22:23:14.715305+0000 mon.a (mon.0) 1203 : audit [DBG] from='client.? 172.21.15.6:0/174527910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:15.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:14 smithi006 bash[33918]: cluster 2023-08-27T22:23:13.452553+0000 mgr.y (mgr.14901) 741 : cluster [DBG] pgmap v748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:15.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:14 smithi006 bash[33918]: audit 2023-08-27T22:23:14.715305+0000 mon.a (mon.0) 1203 : audit [DBG] from='client.? 172.21.15.6:0/174527910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:15.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:14 smithi006 bash[27292]: cluster 2023-08-27T22:23:13.452553+0000 mgr.y (mgr.14901) 741 : cluster [DBG] pgmap v748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:15.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:14 smithi006 bash[27292]: audit 2023-08-27T22:23:14.715305+0000 mon.a (mon.0) 1203 : audit [DBG] from='client.? 172.21.15.6:0/174527910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:17.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:16 smithi130 bash[27579]: cluster 2023-08-27T22:23:15.453615+0000 mgr.y (mgr.14901) 742 : cluster [DBG] pgmap v749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:17.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:16 smithi006 bash[27292]: cluster 2023-08-27T22:23:15.453615+0000 mgr.y (mgr.14901) 742 : cluster [DBG] pgmap v749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:17.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:16 smithi006 bash[33918]: cluster 2023-08-27T22:23:15.453615+0000 mgr.y (mgr.14901) 742 : cluster [DBG] pgmap v749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:17 smithi130 bash[27579]: audit 2023-08-27T22:23:17.128269+0000 mon.c (mon.2) 318 : audit [DBG] from='client.? 172.21.15.6:0/2756765341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:18.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:17 smithi006 bash[27292]: audit 2023-08-27T22:23:17.128269+0000 mon.c (mon.2) 318 : audit [DBG] from='client.? 172.21.15.6:0/2756765341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:18.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:17 smithi006 bash[33918]: audit 2023-08-27T22:23:17.128269+0000 mon.c (mon.2) 318 : audit [DBG] from='client.? 172.21.15.6:0/2756765341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:19.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:18 smithi130 bash[27579]: cluster 2023-08-27T22:23:17.454362+0000 mgr.y (mgr.14901) 743 : cluster [DBG] pgmap v750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:19.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:18 smithi006 bash[27292]: cluster 2023-08-27T22:23:17.454362+0000 mgr.y (mgr.14901) 743 : cluster [DBG] pgmap v750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:19.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:18 smithi006 bash[33918]: cluster 2023-08-27T22:23:17.454362+0000 mgr.y (mgr.14901) 743 : cluster [DBG] pgmap v750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:19 smithi130 bash[27579]: audit 2023-08-27T22:23:19.543192+0000 mon.a (mon.0) 1204 : audit [DBG] from='client.? 172.21.15.6:0/3299017737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:20.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:19 smithi006 bash[33918]: audit 2023-08-27T22:23:19.543192+0000 mon.a (mon.0) 1204 : audit [DBG] from='client.? 172.21.15.6:0/3299017737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:20.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:19 smithi006 bash[27292]: audit 2023-08-27T22:23:19.543192+0000 mon.a (mon.0) 1204 : audit [DBG] from='client.? 172.21.15.6:0/3299017737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:20.473 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:23:20.178Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:20.473 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:23:20.180Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:21.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:23:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:23:20] "GET /metrics HTTP/1.1" 200 193072 "" "Prometheus/2.33.4" 2023-08-27T22:23:21.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:20 smithi006 bash[27292]: cluster 2023-08-27T22:23:19.455287+0000 mgr.y (mgr.14901) 744 : cluster [DBG] pgmap v751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:21.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:20 smithi006 bash[33918]: cluster 2023-08-27T22:23:19.455287+0000 mgr.y (mgr.14901) 744 : cluster [DBG] pgmap v751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:21.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:21 smithi130 bash[27579]: cluster 2023-08-27T22:23:19.455287+0000 mgr.y (mgr.14901) 744 : cluster [DBG] pgmap v751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:22.009 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:23:21.744Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:22.009 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:23:21.745Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:22.010 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:22 smithi006 bash[27292]: audit 2023-08-27T22:23:21.968989+0000 mon.c (mon.2) 319 : audit [DBG] from='client.? 172.21.15.6:0/404980794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:22.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:22 smithi130 bash[27579]: audit 2023-08-27T22:23:21.968989+0000 mon.c (mon.2) 319 : audit [DBG] from='client.? 172.21.15.6:0/404980794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:22.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:22 smithi006 bash[33918]: audit 2023-08-27T22:23:21.968989+0000 mon.c (mon.2) 319 : audit [DBG] from='client.? 172.21.15.6:0/404980794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:23.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:23 smithi130 bash[27579]: cluster 2023-08-27T22:23:21.456434+0000 mgr.y (mgr.14901) 745 : cluster [DBG] pgmap v752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:23.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:23 smithi006 bash[33918]: cluster 2023-08-27T22:23:21.456434+0000 mgr.y (mgr.14901) 745 : cluster [DBG] pgmap v752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:23.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:23 smithi006 bash[27292]: cluster 2023-08-27T22:23:21.456434+0000 mgr.y (mgr.14901) 745 : cluster [DBG] pgmap v752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:23:23.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:23:23.639Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:25 smithi130 bash[27579]: cluster 2023-08-27T22:23:23.457154+0000 mgr.y (mgr.14901) 746 : cluster [DBG] pgmap v753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:25 smithi130 bash[27579]: audit 2023-08-27T22:23:24.390152+0000 mon.c (mon.2) 320 : audit [DBG] from='client.? 172.21.15.6:0/1734815614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:25.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:25 smithi006 bash[27292]: cluster 2023-08-27T22:23:23.457154+0000 mgr.y (mgr.14901) 746 : cluster [DBG] pgmap v753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:25.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:25 smithi006 bash[27292]: audit 2023-08-27T22:23:24.390152+0000 mon.c (mon.2) 320 : audit [DBG] from='client.? 172.21.15.6:0/1734815614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:25.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:25 smithi006 bash[33918]: cluster 2023-08-27T22:23:23.457154+0000 mgr.y (mgr.14901) 746 : cluster [DBG] pgmap v753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:25.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:25 smithi006 bash[33918]: audit 2023-08-27T22:23:24.390152+0000 mon.c (mon.2) 320 : audit [DBG] from='client.? 172.21.15.6:0/1734815614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:27.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:27 smithi130 bash[27579]: cluster 2023-08-27T22:23:25.458215+0000 mgr.y (mgr.14901) 747 : cluster [DBG] pgmap v754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:27.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:27 smithi130 bash[27579]: audit 2023-08-27T22:23:26.807402+0000 mon.c (mon.2) 321 : audit [DBG] from='client.? 172.21.15.6:0/1688500220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:27.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:27 smithi006 bash[27292]: cluster 2023-08-27T22:23:25.458215+0000 mgr.y (mgr.14901) 747 : cluster [DBG] pgmap v754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:27.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:27 smithi006 bash[27292]: audit 2023-08-27T22:23:26.807402+0000 mon.c (mon.2) 321 : audit [DBG] from='client.? 172.21.15.6:0/1688500220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:27.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:27 smithi006 bash[33918]: cluster 2023-08-27T22:23:25.458215+0000 mgr.y (mgr.14901) 747 : cluster [DBG] pgmap v754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:27.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:27 smithi006 bash[33918]: audit 2023-08-27T22:23:26.807402+0000 mon.c (mon.2) 321 : audit [DBG] from='client.? 172.21.15.6:0/1688500220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:29.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:29 smithi130 bash[27579]: cluster 2023-08-27T22:23:27.458977+0000 mgr.y (mgr.14901) 748 : cluster [DBG] pgmap v755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:29.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:29 smithi006 bash[27292]: cluster 2023-08-27T22:23:27.458977+0000 mgr.y (mgr.14901) 748 : cluster [DBG] pgmap v755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:29.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:29 smithi006 bash[33918]: cluster 2023-08-27T22:23:27.458977+0000 mgr.y (mgr.14901) 748 : cluster [DBG] pgmap v755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:30 smithi130 bash[27579]: audit 2023-08-27T22:23:29.227664+0000 mon.c (mon.2) 322 : audit [DBG] from='client.? 172.21.15.6:0/1931572840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:30.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:23:30.179Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:30.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:23:30.182Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:30.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:30 smithi006 bash[27292]: audit 2023-08-27T22:23:29.227664+0000 mon.c (mon.2) 322 : audit [DBG] from='client.? 172.21.15.6:0/1931572840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:30.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:30 smithi006 bash[33918]: audit 2023-08-27T22:23:29.227664+0000 mon.c (mon.2) 322 : audit [DBG] from='client.? 172.21.15.6:0/1931572840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:31.043 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:23:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:23:30] "GET /metrics HTTP/1.1" 200 193068 "" "Prometheus/2.33.4" 2023-08-27T22:23:31.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:31 smithi130 bash[27579]: cluster 2023-08-27T22:23:29.459875+0000 mgr.y (mgr.14901) 749 : cluster [DBG] pgmap v756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:31.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:31 smithi006 bash[33918]: cluster 2023-08-27T22:23:29.459875+0000 mgr.y (mgr.14901) 749 : cluster [DBG] pgmap v756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:31.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:31 smithi006 bash[27292]: cluster 2023-08-27T22:23:29.459875+0000 mgr.y (mgr.14901) 749 : cluster [DBG] pgmap v756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:32.052 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:23:31.744Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:32.052 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:23:31.747Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:32 smithi130 bash[27579]: cluster 2023-08-27T22:23:31.460827+0000 mgr.y (mgr.14901) 750 : cluster [DBG] pgmap v757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:32 smithi130 bash[27579]: audit 2023-08-27T22:23:31.649974+0000 mon.c (mon.2) 323 : audit [DBG] from='client.? 172.21.15.6:0/208700295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:32.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:32 smithi006 bash[33918]: cluster 2023-08-27T22:23:31.460827+0000 mgr.y (mgr.14901) 750 : cluster [DBG] pgmap v757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:32.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:32 smithi006 bash[33918]: audit 2023-08-27T22:23:31.649974+0000 mon.c (mon.2) 323 : audit [DBG] from='client.? 172.21.15.6:0/208700295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:32.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:32 smithi006 bash[27292]: cluster 2023-08-27T22:23:31.460827+0000 mgr.y (mgr.14901) 750 : cluster [DBG] pgmap v757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:32.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:32 smithi006 bash[27292]: audit 2023-08-27T22:23:31.649974+0000 mon.c (mon.2) 323 : audit [DBG] from='client.? 172.21.15.6:0/208700295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:23:33.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:23:33.640Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:34 smithi130 bash[27579]: cluster 2023-08-27T22:23:33.461496+0000 mgr.y (mgr.14901) 751 : cluster [DBG] pgmap v758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:34 smithi130 bash[27579]: audit 2023-08-27T22:23:34.074262+0000 mon.c (mon.2) 324 : audit [DBG] from='client.? 172.21.15.6:0/3160393920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:34.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:34 smithi006 bash[33918]: cluster 2023-08-27T22:23:33.461496+0000 mgr.y (mgr.14901) 751 : cluster [DBG] pgmap v758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:34.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:34 smithi006 bash[33918]: audit 2023-08-27T22:23:34.074262+0000 mon.c (mon.2) 324 : audit [DBG] from='client.? 172.21.15.6:0/3160393920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:34.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:34 smithi006 bash[27292]: cluster 2023-08-27T22:23:33.461496+0000 mgr.y (mgr.14901) 751 : cluster [DBG] pgmap v758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:34.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:34 smithi006 bash[27292]: audit 2023-08-27T22:23:34.074262+0000 mon.c (mon.2) 324 : audit [DBG] from='client.? 172.21.15.6:0/3160393920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:36.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:36 smithi130 bash[27579]: cluster 2023-08-27T22:23:35.462539+0000 mgr.y (mgr.14901) 752 : cluster [DBG] pgmap v759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:36.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:36 smithi130 bash[27579]: audit 2023-08-27T22:23:36.485075+0000 mon.c (mon.2) 325 : audit [DBG] from='client.? 172.21.15.6:0/3132324946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:36.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:36 smithi006 bash[27292]: cluster 2023-08-27T22:23:35.462539+0000 mgr.y (mgr.14901) 752 : cluster [DBG] pgmap v759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:36.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:36 smithi006 bash[27292]: audit 2023-08-27T22:23:36.485075+0000 mon.c (mon.2) 325 : audit [DBG] from='client.? 172.21.15.6:0/3132324946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:36.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:36 smithi006 bash[33918]: cluster 2023-08-27T22:23:35.462539+0000 mgr.y (mgr.14901) 752 : cluster [DBG] pgmap v759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:36.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:36 smithi006 bash[33918]: audit 2023-08-27T22:23:36.485075+0000 mon.c (mon.2) 325 : audit [DBG] from='client.? 172.21.15.6:0/3132324946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:38 smithi130 bash[27579]: cluster 2023-08-27T22:23:37.463291+0000 mgr.y (mgr.14901) 753 : cluster [DBG] pgmap v760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:38.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:38 smithi006 bash[27292]: cluster 2023-08-27T22:23:37.463291+0000 mgr.y (mgr.14901) 753 : cluster [DBG] pgmap v760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:38.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:38 smithi006 bash[33918]: cluster 2023-08-27T22:23:37.463291+0000 mgr.y (mgr.14901) 753 : cluster [DBG] pgmap v760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:39.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:39 smithi130 bash[27579]: audit 2023-08-27T22:23:38.902368+0000 mon.a (mon.0) 1205 : audit [DBG] from='client.? 172.21.15.6:0/2372598839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:39.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:39 smithi006 bash[33918]: audit 2023-08-27T22:23:38.902368+0000 mon.a (mon.0) 1205 : audit [DBG] from='client.? 172.21.15.6:0/2372598839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:39.974 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:39 smithi006 bash[27292]: audit 2023-08-27T22:23:38.902368+0000 mon.a (mon.0) 1205 : audit [DBG] from='client.? 172.21.15.6:0/2372598839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:40.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:23:40.180Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:40.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:23:40.182Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:40 smithi130 bash[27579]: cluster 2023-08-27T22:23:39.464130+0000 mgr.y (mgr.14901) 754 : cluster [DBG] pgmap v761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:40.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:40 smithi006 bash[27292]: cluster 2023-08-27T22:23:39.464130+0000 mgr.y (mgr.14901) 754 : cluster [DBG] pgmap v761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:40.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:23:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:23:40] "GET /metrics HTTP/1.1" 200 193068 "" "Prometheus/2.33.4" 2023-08-27T22:23:40.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:40 smithi006 bash[33918]: cluster 2023-08-27T22:23:39.464130+0000 mgr.y (mgr.14901) 754 : cluster [DBG] pgmap v761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:41.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:41 smithi130 bash[27579]: audit 2023-08-27T22:23:41.321200+0000 mon.c (mon.2) 326 : audit [DBG] from='client.? 172.21.15.6:0/2465029347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:41.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:41 smithi006 bash[27292]: audit 2023-08-27T22:23:41.321200+0000 mon.c (mon.2) 326 : audit [DBG] from='client.? 172.21.15.6:0/2465029347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:41.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:41 smithi006 bash[33918]: audit 2023-08-27T22:23:41.321200+0000 mon.c (mon.2) 326 : audit [DBG] from='client.? 172.21.15.6:0/2465029347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:41.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:23:41.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:41.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:23:41.747Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:42 smithi130 bash[27579]: cluster 2023-08-27T22:23:41.465145+0000 mgr.y (mgr.14901) 755 : cluster [DBG] pgmap v762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:42.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:42 smithi006 bash[27292]: cluster 2023-08-27T22:23:41.465145+0000 mgr.y (mgr.14901) 755 : cluster [DBG] pgmap v762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:42.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:42 smithi006 bash[33918]: cluster 2023-08-27T22:23:41.465145+0000 mgr.y (mgr.14901) 755 : cluster [DBG] pgmap v762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:23:43.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:23:43.641Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:44 smithi130 bash[27579]: cluster 2023-08-27T22:23:43.465808+0000 mgr.y (mgr.14901) 756 : cluster [DBG] pgmap v763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:44.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:44 smithi130 bash[27579]: audit 2023-08-27T22:23:43.739916+0000 mon.c (mon.2) 327 : audit [DBG] from='client.? 172.21.15.6:0/1705187242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:44.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:44 smithi006 bash[27292]: cluster 2023-08-27T22:23:43.465808+0000 mgr.y (mgr.14901) 756 : cluster [DBG] pgmap v763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:44.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:44 smithi006 bash[27292]: audit 2023-08-27T22:23:43.739916+0000 mon.c (mon.2) 327 : audit [DBG] from='client.? 172.21.15.6:0/1705187242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:44.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:44 smithi006 bash[33918]: cluster 2023-08-27T22:23:43.465808+0000 mgr.y (mgr.14901) 756 : cluster [DBG] pgmap v763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:44.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:44 smithi006 bash[33918]: audit 2023-08-27T22:23:43.739916+0000 mon.c (mon.2) 327 : audit [DBG] from='client.? 172.21.15.6:0/1705187242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:46 smithi130 bash[27579]: cluster 2023-08-27T22:23:45.466919+0000 mgr.y (mgr.14901) 757 : cluster [DBG] pgmap v764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:46 smithi130 bash[27579]: audit 2023-08-27T22:23:46.157858+0000 mon.c (mon.2) 328 : audit [DBG] from='client.? 172.21.15.6:0/860647712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:46.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:46 smithi006 bash[27292]: cluster 2023-08-27T22:23:45.466919+0000 mgr.y (mgr.14901) 757 : cluster [DBG] pgmap v764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:46.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:46 smithi006 bash[27292]: audit 2023-08-27T22:23:46.157858+0000 mon.c (mon.2) 328 : audit [DBG] from='client.? 172.21.15.6:0/860647712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:46.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:46 smithi006 bash[33918]: cluster 2023-08-27T22:23:45.466919+0000 mgr.y (mgr.14901) 757 : cluster [DBG] pgmap v764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:46.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:46 smithi006 bash[33918]: audit 2023-08-27T22:23:46.157858+0000 mon.c (mon.2) 328 : audit [DBG] from='client.? 172.21.15.6:0/860647712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:48 smithi130 bash[27579]: cluster 2023-08-27T22:23:47.467632+0000 mgr.y (mgr.14901) 758 : cluster [DBG] pgmap v765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:48.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:48 smithi006 bash[27292]: cluster 2023-08-27T22:23:47.467632+0000 mgr.y (mgr.14901) 758 : cluster [DBG] pgmap v765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:48.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:48 smithi006 bash[33918]: cluster 2023-08-27T22:23:47.467632+0000 mgr.y (mgr.14901) 758 : cluster [DBG] pgmap v765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:49.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:49 smithi130 bash[27579]: audit 2023-08-27T22:23:48.572177+0000 mon.a (mon.0) 1206 : audit [DBG] from='client.? 172.21.15.6:0/3877527970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:49.974 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:49 smithi006 bash[27292]: audit 2023-08-27T22:23:48.572177+0000 mon.a (mon.0) 1206 : audit [DBG] from='client.? 172.21.15.6:0/3877527970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:49.975 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:49 smithi006 bash[33918]: audit 2023-08-27T22:23:48.572177+0000 mon.a (mon.0) 1206 : audit [DBG] from='client.? 172.21.15.6:0/3877527970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:50.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:23:50.180Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:50.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:23:50.182Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:50 smithi130 bash[27579]: cluster 2023-08-27T22:23:49.468491+0000 mgr.y (mgr.14901) 759 : cluster [DBG] pgmap v766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:50.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:23:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:23:50] "GET /metrics HTTP/1.1" 200 193080 "" "Prometheus/2.33.4" 2023-08-27T22:23:50.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:50 smithi006 bash[33918]: cluster 2023-08-27T22:23:49.468491+0000 mgr.y (mgr.14901) 759 : cluster [DBG] pgmap v766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:50.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:50 smithi006 bash[27292]: cluster 2023-08-27T22:23:49.468491+0000 mgr.y (mgr.14901) 759 : cluster [DBG] pgmap v766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:51.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:51 smithi130 bash[27579]: audit 2023-08-27T22:23:50.991933+0000 mon.c (mon.2) 329 : audit [DBG] from='client.? 172.21.15.6:0/4015661858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:51.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:51 smithi006 bash[27292]: audit 2023-08-27T22:23:50.991933+0000 mon.c (mon.2) 329 : audit [DBG] from='client.? 172.21.15.6:0/4015661858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:51.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:51 smithi006 bash[33918]: audit 2023-08-27T22:23:50.991933+0000 mon.c (mon.2) 329 : audit [DBG] from='client.? 172.21.15.6:0/4015661858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:51.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:23:51.746Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:51.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:23:51.747Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:52 smithi130 bash[27579]: cluster 2023-08-27T22:23:51.469512+0000 mgr.y (mgr.14901) 760 : cluster [DBG] pgmap v767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:52.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:52 smithi006 bash[27292]: cluster 2023-08-27T22:23:51.469512+0000 mgr.y (mgr.14901) 760 : cluster [DBG] pgmap v767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:52.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:52 smithi006 bash[33918]: cluster 2023-08-27T22:23:51.469512+0000 mgr.y (mgr.14901) 760 : cluster [DBG] pgmap v767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:53.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:53 smithi130 bash[27579]: audit 2023-08-27T22:23:53.401711+0000 mon.c (mon.2) 330 : audit [DBG] from='client.? 172.21.15.6:0/3434368579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:53.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:53 smithi006 bash[27292]: audit 2023-08-27T22:23:53.401711+0000 mon.c (mon.2) 330 : audit [DBG] from='client.? 172.21.15.6:0/3434368579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:53.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:53 smithi006 bash[33918]: audit 2023-08-27T22:23:53.401711+0000 mon.c (mon.2) 330 : audit [DBG] from='client.? 172.21.15.6:0/3434368579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:53.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:23:53.640Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:53.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:23:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:23:53.642Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:23:54.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:54 smithi130 bash[27579]: cluster 2023-08-27T22:23:53.470208+0000 mgr.y (mgr.14901) 761 : cluster [DBG] pgmap v768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:54.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:54 smithi006 bash[33918]: cluster 2023-08-27T22:23:53.470208+0000 mgr.y (mgr.14901) 761 : cluster [DBG] pgmap v768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:54.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:54 smithi006 bash[27292]: cluster 2023-08-27T22:23:53.470208+0000 mgr.y (mgr.14901) 761 : cluster [DBG] pgmap v768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:56 smithi130 bash[27579]: cluster 2023-08-27T22:23:55.471265+0000 mgr.y (mgr.14901) 762 : cluster [DBG] pgmap v769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:56 smithi130 bash[27579]: audit 2023-08-27T22:23:55.817359+0000 mon.c (mon.2) 331 : audit [DBG] from='client.? 172.21.15.6:0/1590825372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:56.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:56 smithi006 bash[33918]: cluster 2023-08-27T22:23:55.471265+0000 mgr.y (mgr.14901) 762 : cluster [DBG] pgmap v769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:56.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:56 smithi006 bash[33918]: audit 2023-08-27T22:23:55.817359+0000 mon.c (mon.2) 331 : audit [DBG] from='client.? 172.21.15.6:0/1590825372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:56.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:56 smithi006 bash[27292]: cluster 2023-08-27T22:23:55.471265+0000 mgr.y (mgr.14901) 762 : cluster [DBG] pgmap v769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:56.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:56 smithi006 bash[27292]: audit 2023-08-27T22:23:55.817359+0000 mon.c (mon.2) 331 : audit [DBG] from='client.? 172.21.15.6:0/1590825372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:58.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:58 smithi006 bash[33918]: cluster 2023-08-27T22:23:57.472012+0000 mgr.y (mgr.14901) 763 : cluster [DBG] pgmap v770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:58.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:23:58 smithi006 bash[33918]: audit 2023-08-27T22:23:58.236296+0000 mon.c (mon.2) 332 : audit [DBG] from='client.? 172.21.15.6:0/1331718509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:58.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:58 smithi006 bash[27292]: cluster 2023-08-27T22:23:57.472012+0000 mgr.y (mgr.14901) 763 : cluster [DBG] pgmap v770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:58.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:23:58 smithi006 bash[27292]: audit 2023-08-27T22:23:58.236296+0000 mon.c (mon.2) 332 : audit [DBG] from='client.? 172.21.15.6:0/1331718509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:58 smithi130 bash[27579]: cluster 2023-08-27T22:23:57.472012+0000 mgr.y (mgr.14901) 763 : cluster [DBG] pgmap v770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:23:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:23:58 smithi130 bash[27579]: audit 2023-08-27T22:23:58.236296+0000 mon.c (mon.2) 332 : audit [DBG] from='client.? 172.21.15.6:0/1331718509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:00.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:24:00.181Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:00.473 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:24:00.183Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:00.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:24:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:24:00] "GET /metrics HTTP/1.1" 200 193076 "" "Prometheus/2.33.4" 2023-08-27T22:24:00.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:00 smithi006 bash[33918]: cluster 2023-08-27T22:23:59.472819+0000 mgr.y (mgr.14901) 764 : cluster [DBG] pgmap v771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:00.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:00 smithi006 bash[27292]: cluster 2023-08-27T22:23:59.472819+0000 mgr.y (mgr.14901) 764 : cluster [DBG] pgmap v771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:00 smithi130 bash[27579]: cluster 2023-08-27T22:23:59.472819+0000 mgr.y (mgr.14901) 764 : cluster [DBG] pgmap v771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:01.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:01 smithi006 bash[33918]: audit 2023-08-27T22:24:00.673413+0000 mon.a (mon.0) 1207 : audit [DBG] from='client.? 172.21.15.6:0/1892475511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:01.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:24:01.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:01.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:24:01.749Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:01.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:01 smithi006 bash[27292]: audit 2023-08-27T22:24:00.673413+0000 mon.a (mon.0) 1207 : audit [DBG] from='client.? 172.21.15.6:0/1892475511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:02.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:01 smithi130 bash[27579]: audit 2023-08-27T22:24:00.673413+0000 mon.a (mon.0) 1207 : audit [DBG] from='client.? 172.21.15.6:0/1892475511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:02.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:02 smithi006 bash[27292]: cluster 2023-08-27T22:24:01.473727+0000 mgr.y (mgr.14901) 765 : cluster [DBG] pgmap v772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:02.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:02 smithi006 bash[33918]: cluster 2023-08-27T22:24:01.473727+0000 mgr.y (mgr.14901) 765 : cluster [DBG] pgmap v772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:02 smithi130 bash[27579]: cluster 2023-08-27T22:24:01.473727+0000 mgr.y (mgr.14901) 765 : cluster [DBG] pgmap v772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:03.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:03 smithi006 bash[33918]: audit 2023-08-27T22:24:03.089092+0000 mon.a (mon.0) 1208 : audit [DBG] from='client.? 172.21.15.6:0/2896999328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:03.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:03 smithi006 bash[27292]: audit 2023-08-27T22:24:03.089092+0000 mon.a (mon.0) 1208 : audit [DBG] from='client.? 172.21.15.6:0/2896999328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:03.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:24:03.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:03.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:24:03.642Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:04.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:03 smithi130 bash[27579]: audit 2023-08-27T22:24:03.089092+0000 mon.a (mon.0) 1208 : audit [DBG] from='client.? 172.21.15.6:0/2896999328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:04.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:04 smithi006 bash[27292]: cluster 2023-08-27T22:24:03.474378+0000 mgr.y (mgr.14901) 766 : cluster [DBG] pgmap v773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:05.006 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:04 smithi006 bash[33918]: cluster 2023-08-27T22:24:03.474378+0000 mgr.y (mgr.14901) 766 : cluster [DBG] pgmap v773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:04 smithi130 bash[27579]: cluster 2023-08-27T22:24:03.474378+0000 mgr.y (mgr.14901) 766 : cluster [DBG] pgmap v773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:05.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:05 smithi006 bash[33918]: audit 2023-08-27T22:24:05.502532+0000 mon.c (mon.2) 333 : audit [DBG] from='client.? 172.21.15.6:0/3437897896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:06.090 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:05 smithi006 bash[27292]: audit 2023-08-27T22:24:05.502532+0000 mon.c (mon.2) 333 : audit [DBG] from='client.? 172.21.15.6:0/3437897896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:05 smithi130 bash[27579]: audit 2023-08-27T22:24:05.502532+0000 mon.c (mon.2) 333 : audit [DBG] from='client.? 172.21.15.6:0/3437897896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:06.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:06 smithi006 bash[33918]: cluster 2023-08-27T22:24:05.475383+0000 mgr.y (mgr.14901) 767 : cluster [DBG] pgmap v774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:07.507 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:06 smithi006 bash[27292]: cluster 2023-08-27T22:24:05.475383+0000 mgr.y (mgr.14901) 767 : cluster [DBG] pgmap v774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:07.508 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:06 smithi130 bash[27579]: cluster 2023-08-27T22:24:05.475383+0000 mgr.y (mgr.14901) 767 : cluster [DBG] pgmap v774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:08.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:08 smithi006 bash[27292]: cluster 2023-08-27T22:24:07.476245+0000 mgr.y (mgr.14901) 768 : cluster [DBG] pgmap v775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:08.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:08 smithi006 bash[27292]: audit 2023-08-27T22:24:07.916768+0000 mon.c (mon.2) 334 : audit [DBG] from='client.? 172.21.15.6:0/889889206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:08.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:08 smithi006 bash[33918]: cluster 2023-08-27T22:24:07.476245+0000 mgr.y (mgr.14901) 768 : cluster [DBG] pgmap v775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:08.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:08 smithi006 bash[33918]: audit 2023-08-27T22:24:07.916768+0000 mon.c (mon.2) 334 : audit [DBG] from='client.? 172.21.15.6:0/889889206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:08 smithi130 bash[27579]: cluster 2023-08-27T22:24:07.476245+0000 mgr.y (mgr.14901) 768 : cluster [DBG] pgmap v775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:08 smithi130 bash[27579]: audit 2023-08-27T22:24:07.916768+0000 mon.c (mon.2) 334 : audit [DBG] from='client.? 172.21.15.6:0/889889206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:10.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:24:10.182Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:10.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:24:10.184Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:10.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:24:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:24:10] "GET /metrics HTTP/1.1" 200 193076 "" "Prometheus/2.33.4" 2023-08-27T22:24:10.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:10 smithi006 bash[27292]: cluster 2023-08-27T22:24:09.477133+0000 mgr.y (mgr.14901) 769 : cluster [DBG] pgmap v776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:10.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:10 smithi006 bash[27292]: audit 2023-08-27T22:24:10.339855+0000 mon.a (mon.0) 1209 : audit [DBG] from='client.? 172.21.15.6:0/1643957426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:10.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:10 smithi006 bash[33918]: cluster 2023-08-27T22:24:09.477133+0000 mgr.y (mgr.14901) 769 : cluster [DBG] pgmap v776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:10.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:10 smithi006 bash[33918]: audit 2023-08-27T22:24:10.339855+0000 mon.a (mon.0) 1209 : audit [DBG] from='client.? 172.21.15.6:0/1643957426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:10 smithi130 bash[27579]: cluster 2023-08-27T22:24:09.477133+0000 mgr.y (mgr.14901) 769 : cluster [DBG] pgmap v776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:10 smithi130 bash[27579]: audit 2023-08-27T22:24:10.339855+0000 mon.a (mon.0) 1209 : audit [DBG] from='client.? 172.21.15.6:0/1643957426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:24:11.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:12.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:24:11.750Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:12.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:12 smithi006 bash[33918]: cluster 2023-08-27T22:24:11.478034+0000 mgr.y (mgr.14901) 770 : cluster [DBG] pgmap v777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:12.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:12 smithi006 bash[27292]: cluster 2023-08-27T22:24:11.478034+0000 mgr.y (mgr.14901) 770 : cluster [DBG] pgmap v777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:12 smithi130 bash[27579]: cluster 2023-08-27T22:24:11.478034+0000 mgr.y (mgr.14901) 770 : cluster [DBG] pgmap v777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:13.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:13 smithi006 bash[33918]: audit 2023-08-27T22:24:12.701417+0000 mon.b (mon.1) 199 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:24:13.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:13 smithi006 bash[33918]: audit 2023-08-27T22:24:12.762295+0000 mon.c (mon.2) 335 : audit [DBG] from='client.? 172.21.15.6:0/3466582105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:13.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:13 smithi006 bash[27292]: audit 2023-08-27T22:24:12.701417+0000 mon.b (mon.1) 199 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:24:13.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:13 smithi006 bash[27292]: audit 2023-08-27T22:24:12.762295+0000 mon.c (mon.2) 335 : audit [DBG] from='client.? 172.21.15.6:0/3466582105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:13.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:24:13.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:13.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:24:13.643Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:14.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:13 smithi130 bash[27579]: audit 2023-08-27T22:24:12.701417+0000 mon.b (mon.1) 199 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:24:14.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:13 smithi130 bash[27579]: audit 2023-08-27T22:24:12.762295+0000 mon.c (mon.2) 335 : audit [DBG] from='client.? 172.21.15.6:0/3466582105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:14.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:14 smithi006 bash[27292]: cluster 2023-08-27T22:24:13.478604+0000 mgr.y (mgr.14901) 771 : cluster [DBG] pgmap v778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:14.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:14 smithi006 bash[33918]: cluster 2023-08-27T22:24:13.478604+0000 mgr.y (mgr.14901) 771 : cluster [DBG] pgmap v778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:14 smithi130 bash[27579]: cluster 2023-08-27T22:24:13.478604+0000 mgr.y (mgr.14901) 771 : cluster [DBG] pgmap v778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:15.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:15 smithi006 bash[27292]: audit 2023-08-27T22:24:15.191815+0000 mon.c (mon.2) 336 : audit [DBG] from='client.? 172.21.15.6:0/2813929544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:15.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:15 smithi006 bash[33918]: audit 2023-08-27T22:24:15.191815+0000 mon.c (mon.2) 336 : audit [DBG] from='client.? 172.21.15.6:0/2813929544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:16.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:15 smithi130 bash[27579]: audit 2023-08-27T22:24:15.191815+0000 mon.c (mon.2) 336 : audit [DBG] from='client.? 172.21.15.6:0/2813929544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:16.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:16 smithi006 bash[33918]: cluster 2023-08-27T22:24:15.479643+0000 mgr.y (mgr.14901) 772 : cluster [DBG] pgmap v779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:16.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:16 smithi006 bash[27292]: cluster 2023-08-27T22:24:15.479643+0000 mgr.y (mgr.14901) 772 : cluster [DBG] pgmap v779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:17.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:16 smithi130 bash[27579]: cluster 2023-08-27T22:24:15.479643+0000 mgr.y (mgr.14901) 772 : cluster [DBG] pgmap v779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:17.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:17 smithi006 bash[27292]: audit 2023-08-27T22:24:17.616414+0000 mon.a (mon.0) 1210 : audit [DBG] from='client.? 172.21.15.6:0/2789513796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:17.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:17 smithi006 bash[33918]: audit 2023-08-27T22:24:17.616414+0000 mon.a (mon.0) 1210 : audit [DBG] from='client.? 172.21.15.6:0/2789513796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:18.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:17 smithi130 bash[27579]: audit 2023-08-27T22:24:17.616414+0000 mon.a (mon.0) 1210 : audit [DBG] from='client.? 172.21.15.6:0/2789513796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:18.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:18 smithi006 bash[27292]: cluster 2023-08-27T22:24:17.480355+0000 mgr.y (mgr.14901) 773 : cluster [DBG] pgmap v780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:18.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:18 smithi006 bash[27292]: audit 2023-08-27T22:24:18.576764+0000 mon.a (mon.0) 1211 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:24:18.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:18 smithi006 bash[33918]: cluster 2023-08-27T22:24:17.480355+0000 mgr.y (mgr.14901) 773 : cluster [DBG] pgmap v780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:18.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:18 smithi006 bash[33918]: audit 2023-08-27T22:24:18.576764+0000 mon.a (mon.0) 1211 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:24:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:18 smithi130 bash[27579]: cluster 2023-08-27T22:24:17.480355+0000 mgr.y (mgr.14901) 773 : cluster [DBG] pgmap v780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:18 smithi130 bash[27579]: audit 2023-08-27T22:24:18.576764+0000 mon.a (mon.0) 1211 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:24:20.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:24:20.183Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:20.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:24:20.186Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:20.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:20 smithi006 bash[27292]: audit 2023-08-27T22:24:19.110480+0000 mon.a (mon.0) 1212 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:24:20.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:20 smithi006 bash[27292]: audit 2023-08-27T22:24:19.411478+0000 mon.b (mon.1) 200 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:24:20.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:20 smithi006 bash[27292]: audit 2023-08-27T22:24:19.413516+0000 mon.b (mon.1) 201 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:24:20.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:20 smithi006 bash[27292]: audit 2023-08-27T22:24:19.416220+0000 mon.a (mon.0) 1213 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:24:20.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:20 smithi006 bash[27292]: cluster 2023-08-27T22:24:19.481273+0000 mgr.y (mgr.14901) 774 : cluster [DBG] pgmap v781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:20.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:20 smithi006 bash[27292]: audit 2023-08-27T22:24:20.024993+0000 mon.a (mon.0) 1214 : audit [DBG] from='client.? 172.21.15.6:0/797635220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:20.474 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:20 smithi006 bash[33918]: audit 2023-08-27T22:24:19.110480+0000 mon.a (mon.0) 1212 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:24:20.474 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:20 smithi006 bash[33918]: audit 2023-08-27T22:24:19.411478+0000 mon.b (mon.1) 200 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:24:20.474 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:20 smithi006 bash[33918]: audit 2023-08-27T22:24:19.413516+0000 mon.b (mon.1) 201 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:24:20.474 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:20 smithi006 bash[33918]: audit 2023-08-27T22:24:19.416220+0000 mon.a (mon.0) 1213 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:24:20.474 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:20 smithi006 bash[33918]: cluster 2023-08-27T22:24:19.481273+0000 mgr.y (mgr.14901) 774 : cluster [DBG] pgmap v781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:20.474 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:20 smithi006 bash[33918]: audit 2023-08-27T22:24:20.024993+0000 mon.a (mon.0) 1214 : audit [DBG] from='client.? 172.21.15.6:0/797635220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:20 smithi130 bash[27579]: audit 2023-08-27T22:24:19.110480+0000 mon.a (mon.0) 1212 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:24:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:20 smithi130 bash[27579]: audit 2023-08-27T22:24:19.411478+0000 mon.b (mon.1) 200 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:24:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:20 smithi130 bash[27579]: audit 2023-08-27T22:24:19.413516+0000 mon.b (mon.1) 201 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:24:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:20 smithi130 bash[27579]: audit 2023-08-27T22:24:19.416220+0000 mon.a (mon.0) 1213 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:24:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:20 smithi130 bash[27579]: cluster 2023-08-27T22:24:19.481273+0000 mgr.y (mgr.14901) 774 : cluster [DBG] pgmap v781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:20 smithi130 bash[27579]: audit 2023-08-27T22:24:20.024993+0000 mon.a (mon.0) 1214 : audit [DBG] from='client.? 172.21.15.6:0/797635220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:21.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:24:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:24:20] "GET /metrics HTTP/1.1" 200 193080 "" "Prometheus/2.33.4" 2023-08-27T22:24:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:24:21.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:24:21.750Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:22 smithi130 bash[27579]: cluster 2023-08-27T22:24:21.482228+0000 mgr.y (mgr.14901) 775 : cluster [DBG] pgmap v782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:22 smithi130 bash[27579]: audit 2023-08-27T22:24:22.436952+0000 mon.a (mon.0) 1215 : audit [DBG] from='client.? 172.21.15.6:0/3890248451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:22.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:22 smithi006 bash[33918]: cluster 2023-08-27T22:24:21.482228+0000 mgr.y (mgr.14901) 775 : cluster [DBG] pgmap v782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:22.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:22 smithi006 bash[33918]: audit 2023-08-27T22:24:22.436952+0000 mon.a (mon.0) 1215 : audit [DBG] from='client.? 172.21.15.6:0/3890248451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:22.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:22 smithi006 bash[27292]: cluster 2023-08-27T22:24:21.482228+0000 mgr.y (mgr.14901) 775 : cluster [DBG] pgmap v782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:22.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:22 smithi006 bash[27292]: audit 2023-08-27T22:24:22.436952+0000 mon.a (mon.0) 1215 : audit [DBG] from='client.? 172.21.15.6:0/3890248451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:24:23.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:23.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:24:23.644Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:24 smithi130 bash[27579]: cluster 2023-08-27T22:24:23.482816+0000 mgr.y (mgr.14901) 776 : cluster [DBG] pgmap v783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:24.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:24 smithi006 bash[33918]: cluster 2023-08-27T22:24:23.482816+0000 mgr.y (mgr.14901) 776 : cluster [DBG] pgmap v783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:24.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:24 smithi006 bash[27292]: cluster 2023-08-27T22:24:23.482816+0000 mgr.y (mgr.14901) 776 : cluster [DBG] pgmap v783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:25.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:25 smithi130 bash[27579]: audit 2023-08-27T22:24:24.862929+0000 mon.a (mon.0) 1216 : audit [DBG] from='client.? 172.21.15.6:0/1646557854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:25.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:25 smithi006 bash[33918]: audit 2023-08-27T22:24:24.862929+0000 mon.a (mon.0) 1216 : audit [DBG] from='client.? 172.21.15.6:0/1646557854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:25.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:25 smithi006 bash[27292]: audit 2023-08-27T22:24:24.862929+0000 mon.a (mon.0) 1216 : audit [DBG] from='client.? 172.21.15.6:0/1646557854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:26.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:26 smithi130 bash[27579]: cluster 2023-08-27T22:24:25.483966+0000 mgr.y (mgr.14901) 777 : cluster [DBG] pgmap v784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:26.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:26 smithi006 bash[27292]: cluster 2023-08-27T22:24:25.483966+0000 mgr.y (mgr.14901) 777 : cluster [DBG] pgmap v784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:26.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:26 smithi006 bash[33918]: cluster 2023-08-27T22:24:25.483966+0000 mgr.y (mgr.14901) 777 : cluster [DBG] pgmap v784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:27 smithi130 bash[27579]: audit 2023-08-27T22:24:27.282106+0000 mon.a (mon.0) 1217 : audit [DBG] from='client.? 172.21.15.6:0/2054717141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:27.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:27 smithi006 bash[27292]: audit 2023-08-27T22:24:27.282106+0000 mon.a (mon.0) 1217 : audit [DBG] from='client.? 172.21.15.6:0/2054717141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:27.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:27 smithi006 bash[33918]: audit 2023-08-27T22:24:27.282106+0000 mon.a (mon.0) 1217 : audit [DBG] from='client.? 172.21.15.6:0/2054717141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:28 smithi130 bash[27579]: cluster 2023-08-27T22:24:27.484767+0000 mgr.y (mgr.14901) 778 : cluster [DBG] pgmap v785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:28.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:28 smithi006 bash[33918]: cluster 2023-08-27T22:24:27.484767+0000 mgr.y (mgr.14901) 778 : cluster [DBG] pgmap v785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:28.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:28 smithi006 bash[27292]: cluster 2023-08-27T22:24:27.484767+0000 mgr.y (mgr.14901) 778 : cluster [DBG] pgmap v785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:30.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:24:30.184Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:30.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:24:30.187Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:30 smithi130 bash[27579]: cluster 2023-08-27T22:24:29.485536+0000 mgr.y (mgr.14901) 779 : cluster [DBG] pgmap v786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:30 smithi130 bash[27579]: audit 2023-08-27T22:24:29.694904+0000 mon.c (mon.2) 337 : audit [DBG] from='client.? 172.21.15.6:0/1340233279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:30.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:24:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:24:30] "GET /metrics HTTP/1.1" 200 193078 "" "Prometheus/2.33.4" 2023-08-27T22:24:30.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:30 smithi006 bash[27292]: cluster 2023-08-27T22:24:29.485536+0000 mgr.y (mgr.14901) 779 : cluster [DBG] pgmap v786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:30.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:30 smithi006 bash[27292]: audit 2023-08-27T22:24:29.694904+0000 mon.c (mon.2) 337 : audit [DBG] from='client.? 172.21.15.6:0/1340233279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:30.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:30 smithi006 bash[33918]: cluster 2023-08-27T22:24:29.485536+0000 mgr.y (mgr.14901) 779 : cluster [DBG] pgmap v786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:30.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:30 smithi006 bash[33918]: audit 2023-08-27T22:24:29.694904+0000 mon.c (mon.2) 337 : audit [DBG] from='client.? 172.21.15.6:0/1340233279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:24:31.749Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:24:31.751Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:32 smithi130 bash[27579]: cluster 2023-08-27T22:24:31.486488+0000 mgr.y (mgr.14901) 780 : cluster [DBG] pgmap v787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:32.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:32 smithi130 bash[27579]: audit 2023-08-27T22:24:32.117741+0000 mon.a (mon.0) 1218 : audit [DBG] from='client.? 172.21.15.6:0/81102967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:32.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:32 smithi006 bash[33918]: cluster 2023-08-27T22:24:31.486488+0000 mgr.y (mgr.14901) 780 : cluster [DBG] pgmap v787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:32.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:32 smithi006 bash[33918]: audit 2023-08-27T22:24:32.117741+0000 mon.a (mon.0) 1218 : audit [DBG] from='client.? 172.21.15.6:0/81102967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:32.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:32 smithi006 bash[27292]: cluster 2023-08-27T22:24:31.486488+0000 mgr.y (mgr.14901) 780 : cluster [DBG] pgmap v787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:32.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:32 smithi006 bash[27292]: audit 2023-08-27T22:24:32.117741+0000 mon.a (mon.0) 1218 : audit [DBG] from='client.? 172.21.15.6:0/81102967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:24:33.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:24:33.645Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:34 smithi130 bash[27579]: cluster 2023-08-27T22:24:33.487146+0000 mgr.y (mgr.14901) 781 : cluster [DBG] pgmap v788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:34 smithi130 bash[27579]: audit 2023-08-27T22:24:34.534368+0000 mon.a (mon.0) 1219 : audit [DBG] from='client.? 172.21.15.6:0/252916758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:34.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:34 smithi006 bash[33918]: cluster 2023-08-27T22:24:33.487146+0000 mgr.y (mgr.14901) 781 : cluster [DBG] pgmap v788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:34.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:34 smithi006 bash[33918]: audit 2023-08-27T22:24:34.534368+0000 mon.a (mon.0) 1219 : audit [DBG] from='client.? 172.21.15.6:0/252916758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:34.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:34 smithi006 bash[27292]: cluster 2023-08-27T22:24:33.487146+0000 mgr.y (mgr.14901) 781 : cluster [DBG] pgmap v788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:34.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:34 smithi006 bash[27292]: audit 2023-08-27T22:24:34.534368+0000 mon.a (mon.0) 1219 : audit [DBG] from='client.? 172.21.15.6:0/252916758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:36 smithi130 bash[27579]: cluster 2023-08-27T22:24:35.488178+0000 mgr.y (mgr.14901) 782 : cluster [DBG] pgmap v789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:36.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:36 smithi006 bash[27292]: cluster 2023-08-27T22:24:35.488178+0000 mgr.y (mgr.14901) 782 : cluster [DBG] pgmap v789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:36.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:36 smithi006 bash[33918]: cluster 2023-08-27T22:24:35.488178+0000 mgr.y (mgr.14901) 782 : cluster [DBG] pgmap v789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:37.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:37 smithi130 bash[27579]: audit 2023-08-27T22:24:36.946495+0000 mon.a (mon.0) 1220 : audit [DBG] from='client.? 172.21.15.6:0/3234547043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:37.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:37 smithi006 bash[33918]: audit 2023-08-27T22:24:36.946495+0000 mon.a (mon.0) 1220 : audit [DBG] from='client.? 172.21.15.6:0/3234547043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:37.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:37 smithi006 bash[27292]: audit 2023-08-27T22:24:36.946495+0000 mon.a (mon.0) 1220 : audit [DBG] from='client.? 172.21.15.6:0/3234547043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:38.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:38 smithi130 bash[27579]: cluster 2023-08-27T22:24:37.488950+0000 mgr.y (mgr.14901) 783 : cluster [DBG] pgmap v790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:38.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:38 smithi006 bash[27292]: cluster 2023-08-27T22:24:37.488950+0000 mgr.y (mgr.14901) 783 : cluster [DBG] pgmap v790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:38.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:38 smithi006 bash[33918]: cluster 2023-08-27T22:24:37.488950+0000 mgr.y (mgr.14901) 783 : cluster [DBG] pgmap v790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:39.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:39 smithi006 bash[33918]: audit 2023-08-27T22:24:39.366750+0000 mon.a (mon.0) 1221 : audit [DBG] from='client.? 172.21.15.6:0/3291309574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:39.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:39 smithi006 bash[27292]: audit 2023-08-27T22:24:39.366750+0000 mon.a (mon.0) 1221 : audit [DBG] from='client.? 172.21.15.6:0/3291309574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:39 smithi130 bash[27579]: audit 2023-08-27T22:24:39.366750+0000 mon.a (mon.0) 1221 : audit [DBG] from='client.? 172.21.15.6:0/3291309574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:40.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:24:40.185Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:40.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:24:40.187Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:40.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:24:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:24:40] "GET /metrics HTTP/1.1" 200 193078 "" "Prometheus/2.33.4" 2023-08-27T22:24:40.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:40 smithi006 bash[27292]: cluster 2023-08-27T22:24:39.489804+0000 mgr.y (mgr.14901) 784 : cluster [DBG] pgmap v791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:40.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:40 smithi006 bash[33918]: cluster 2023-08-27T22:24:39.489804+0000 mgr.y (mgr.14901) 784 : cluster [DBG] pgmap v791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:40 smithi130 bash[27579]: cluster 2023-08-27T22:24:39.489804+0000 mgr.y (mgr.14901) 784 : cluster [DBG] pgmap v791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:24:41.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:24:41.752Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:42.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:42 smithi006 bash[33918]: cluster 2023-08-27T22:24:41.490741+0000 mgr.y (mgr.14901) 785 : cluster [DBG] pgmap v792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:42.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:42 smithi006 bash[33918]: audit 2023-08-27T22:24:41.783940+0000 mon.a (mon.0) 1222 : audit [DBG] from='client.? 172.21.15.6:0/2605328536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:42.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:42 smithi006 bash[27292]: cluster 2023-08-27T22:24:41.490741+0000 mgr.y (mgr.14901) 785 : cluster [DBG] pgmap v792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:42.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:42 smithi006 bash[27292]: audit 2023-08-27T22:24:41.783940+0000 mon.a (mon.0) 1222 : audit [DBG] from='client.? 172.21.15.6:0/2605328536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:42 smithi130 bash[27579]: cluster 2023-08-27T22:24:41.490741+0000 mgr.y (mgr.14901) 785 : cluster [DBG] pgmap v792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:42 smithi130 bash[27579]: audit 2023-08-27T22:24:41.783940+0000 mon.a (mon.0) 1222 : audit [DBG] from='client.? 172.21.15.6:0/2605328536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:24:43.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:24:43.646Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:44.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:44 smithi006 bash[33918]: cluster 2023-08-27T22:24:43.491317+0000 mgr.y (mgr.14901) 786 : cluster [DBG] pgmap v793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:44.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:44 smithi006 bash[33918]: audit 2023-08-27T22:24:44.196751+0000 mon.a (mon.0) 1223 : audit [DBG] from='client.? 172.21.15.6:0/1100706033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:44.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:44 smithi006 bash[27292]: cluster 2023-08-27T22:24:43.491317+0000 mgr.y (mgr.14901) 786 : cluster [DBG] pgmap v793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:44.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:44 smithi006 bash[27292]: audit 2023-08-27T22:24:44.196751+0000 mon.a (mon.0) 1223 : audit [DBG] from='client.? 172.21.15.6:0/1100706033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:44 smithi130 bash[27579]: cluster 2023-08-27T22:24:43.491317+0000 mgr.y (mgr.14901) 786 : cluster [DBG] pgmap v793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:44 smithi130 bash[27579]: audit 2023-08-27T22:24:44.196751+0000 mon.a (mon.0) 1223 : audit [DBG] from='client.? 172.21.15.6:0/1100706033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:46.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:46 smithi006 bash[33918]: cluster 2023-08-27T22:24:45.492493+0000 mgr.y (mgr.14901) 787 : cluster [DBG] pgmap v794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:46.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:46 smithi006 bash[33918]: audit 2023-08-27T22:24:46.607246+0000 mon.a (mon.0) 1224 : audit [DBG] from='client.? 172.21.15.6:0/2899300698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:46.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:46 smithi006 bash[27292]: cluster 2023-08-27T22:24:45.492493+0000 mgr.y (mgr.14901) 787 : cluster [DBG] pgmap v794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:46.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:46 smithi006 bash[27292]: audit 2023-08-27T22:24:46.607246+0000 mon.a (mon.0) 1224 : audit [DBG] from='client.? 172.21.15.6:0/2899300698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:47.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:46 smithi130 bash[27579]: cluster 2023-08-27T22:24:45.492493+0000 mgr.y (mgr.14901) 787 : cluster [DBG] pgmap v794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:47.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:46 smithi130 bash[27579]: audit 2023-08-27T22:24:46.607246+0000 mon.a (mon.0) 1224 : audit [DBG] from='client.? 172.21.15.6:0/2899300698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:48.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:48 smithi006 bash[27292]: cluster 2023-08-27T22:24:47.493335+0000 mgr.y (mgr.14901) 788 : cluster [DBG] pgmap v795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:48.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:48 smithi006 bash[33918]: cluster 2023-08-27T22:24:47.493335+0000 mgr.y (mgr.14901) 788 : cluster [DBG] pgmap v795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:49.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:48 smithi130 bash[27579]: cluster 2023-08-27T22:24:47.493335+0000 mgr.y (mgr.14901) 788 : cluster [DBG] pgmap v795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:49.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:49 smithi006 bash[33918]: audit 2023-08-27T22:24:49.030451+0000 mon.a (mon.0) 1225 : audit [DBG] from='client.? 172.21.15.6:0/3783341679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:49.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:49 smithi006 bash[27292]: audit 2023-08-27T22:24:49.030451+0000 mon.a (mon.0) 1225 : audit [DBG] from='client.? 172.21.15.6:0/3783341679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:50.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:49 smithi130 bash[27579]: audit 2023-08-27T22:24:49.030451+0000 mon.a (mon.0) 1225 : audit [DBG] from='client.? 172.21.15.6:0/3783341679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:50.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:24:50.186Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:50.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:24:50.189Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:50.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:50 smithi006 bash[27292]: cluster 2023-08-27T22:24:49.494230+0000 mgr.y (mgr.14901) 789 : cluster [DBG] pgmap v796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:50.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:24:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:24:50] "GET /metrics HTTP/1.1" 200 193072 "" "Prometheus/2.33.4" 2023-08-27T22:24:50.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:50 smithi006 bash[33918]: cluster 2023-08-27T22:24:49.494230+0000 mgr.y (mgr.14901) 789 : cluster [DBG] pgmap v796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:50 smithi130 bash[27579]: cluster 2023-08-27T22:24:49.494230+0000 mgr.y (mgr.14901) 789 : cluster [DBG] pgmap v796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:51.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:51 smithi006 bash[27292]: audit 2023-08-27T22:24:51.448349+0000 mon.c (mon.2) 338 : audit [DBG] from='client.? 172.21.15.6:0/4094593250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:51.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:51 smithi006 bash[33918]: audit 2023-08-27T22:24:51.448349+0000 mon.c (mon.2) 338 : audit [DBG] from='client.? 172.21.15.6:0/4094593250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:51.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:24:51.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:51.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:24:51.753Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:52.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:51 smithi130 bash[27579]: audit 2023-08-27T22:24:51.448349+0000 mon.c (mon.2) 338 : audit [DBG] from='client.? 172.21.15.6:0/4094593250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:52.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:52 smithi006 bash[27292]: cluster 2023-08-27T22:24:51.495219+0000 mgr.y (mgr.14901) 790 : cluster [DBG] pgmap v797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:52.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:52 smithi006 bash[33918]: cluster 2023-08-27T22:24:51.495219+0000 mgr.y (mgr.14901) 790 : cluster [DBG] pgmap v797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:52 smithi130 bash[27579]: cluster 2023-08-27T22:24:51.495219+0000 mgr.y (mgr.14901) 790 : cluster [DBG] pgmap v797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:53.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:24:53.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:53.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:24:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:24:53.646Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:24:54.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:54 smithi006 bash[33918]: cluster 2023-08-27T22:24:53.495902+0000 mgr.y (mgr.14901) 791 : cluster [DBG] pgmap v798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:54.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:54 smithi006 bash[33918]: audit 2023-08-27T22:24:53.862767+0000 mon.c (mon.2) 339 : audit [DBG] from='client.? 172.21.15.6:0/2569193590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:54.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:54 smithi006 bash[27292]: cluster 2023-08-27T22:24:53.495902+0000 mgr.y (mgr.14901) 791 : cluster [DBG] pgmap v798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:54.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:54 smithi006 bash[27292]: audit 2023-08-27T22:24:53.862767+0000 mon.c (mon.2) 339 : audit [DBG] from='client.? 172.21.15.6:0/2569193590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:54 smithi130 bash[27579]: cluster 2023-08-27T22:24:53.495902+0000 mgr.y (mgr.14901) 791 : cluster [DBG] pgmap v798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:54 smithi130 bash[27579]: audit 2023-08-27T22:24:53.862767+0000 mon.c (mon.2) 339 : audit [DBG] from='client.? 172.21.15.6:0/2569193590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:56.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:56 smithi006 bash[27292]: cluster 2023-08-27T22:24:55.497009+0000 mgr.y (mgr.14901) 792 : cluster [DBG] pgmap v799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:56.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:56 smithi006 bash[27292]: audit 2023-08-27T22:24:56.277720+0000 mon.a (mon.0) 1226 : audit [DBG] from='client.? 172.21.15.6:0/1840945675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:56.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:56 smithi006 bash[33918]: cluster 2023-08-27T22:24:55.497009+0000 mgr.y (mgr.14901) 792 : cluster [DBG] pgmap v799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:56.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:56 smithi006 bash[33918]: audit 2023-08-27T22:24:56.277720+0000 mon.a (mon.0) 1226 : audit [DBG] from='client.? 172.21.15.6:0/1840945675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:57.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:56 smithi130 bash[27579]: cluster 2023-08-27T22:24:55.497009+0000 mgr.y (mgr.14901) 792 : cluster [DBG] pgmap v799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:57.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:56 smithi130 bash[27579]: audit 2023-08-27T22:24:56.277720+0000 mon.a (mon.0) 1226 : audit [DBG] from='client.? 172.21.15.6:0/1840945675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:58.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:58 smithi006 bash[27292]: cluster 2023-08-27T22:24:57.497744+0000 mgr.y (mgr.14901) 793 : cluster [DBG] pgmap v800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:58.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:58 smithi006 bash[33918]: cluster 2023-08-27T22:24:57.497744+0000 mgr.y (mgr.14901) 793 : cluster [DBG] pgmap v800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:58 smithi130 bash[27579]: cluster 2023-08-27T22:24:57.497744+0000 mgr.y (mgr.14901) 793 : cluster [DBG] pgmap v800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:24:59.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:24:59 smithi006 bash[27292]: audit 2023-08-27T22:24:58.699059+0000 mon.a (mon.0) 1227 : audit [DBG] from='client.? 172.21.15.6:0/811149426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:59.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:24:59 smithi006 bash[33918]: audit 2023-08-27T22:24:58.699059+0000 mon.a (mon.0) 1227 : audit [DBG] from='client.? 172.21.15.6:0/811149426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:24:59 smithi130 bash[27579]: audit 2023-08-27T22:24:58.699059+0000 mon.a (mon.0) 1227 : audit [DBG] from='client.? 172.21.15.6:0/811149426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:00.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:25:00.187Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:00.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:25:00.189Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:00.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:00 smithi006 bash[27292]: cluster 2023-08-27T22:24:59.498590+0000 mgr.y (mgr.14901) 794 : cluster [DBG] pgmap v801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:00.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:00 smithi006 bash[33918]: cluster 2023-08-27T22:24:59.498590+0000 mgr.y (mgr.14901) 794 : cluster [DBG] pgmap v801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:00.973 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:25:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:25:00] "GET /metrics HTTP/1.1" 200 193077 "" "Prometheus/2.33.4" 2023-08-27T22:25:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:00 smithi130 bash[27579]: cluster 2023-08-27T22:24:59.498590+0000 mgr.y (mgr.14901) 794 : cluster [DBG] pgmap v801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:01.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:01 smithi006 bash[27292]: audit 2023-08-27T22:25:01.122074+0000 mon.a (mon.0) 1228 : audit [DBG] from='client.? 172.21.15.6:0/3311897287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:01.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:01 smithi006 bash[33918]: audit 2023-08-27T22:25:01.122074+0000 mon.a (mon.0) 1228 : audit [DBG] from='client.? 172.21.15.6:0/3311897287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:01.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:25:01.752Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:01.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:25:01.754Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:02.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:01 smithi130 bash[27579]: audit 2023-08-27T22:25:01.122074+0000 mon.a (mon.0) 1228 : audit [DBG] from='client.? 172.21.15.6:0/3311897287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:02.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:02 smithi006 bash[33918]: cluster 2023-08-27T22:25:01.499622+0000 mgr.y (mgr.14901) 795 : cluster [DBG] pgmap v802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:02.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:02 smithi006 bash[27292]: cluster 2023-08-27T22:25:01.499622+0000 mgr.y (mgr.14901) 795 : cluster [DBG] pgmap v802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:02 smithi130 bash[27579]: cluster 2023-08-27T22:25:01.499622+0000 mgr.y (mgr.14901) 795 : cluster [DBG] pgmap v802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:25:03.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:25:03.646Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:04 smithi130 bash[27579]: cluster 2023-08-27T22:25:03.500208+0000 mgr.y (mgr.14901) 796 : cluster [DBG] pgmap v803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:04 smithi130 bash[27579]: audit 2023-08-27T22:25:03.536723+0000 mon.c (mon.2) 340 : audit [DBG] from='client.? 172.21.15.6:0/352695736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:04.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:04 smithi006 bash[27292]: cluster 2023-08-27T22:25:03.500208+0000 mgr.y (mgr.14901) 796 : cluster [DBG] pgmap v803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:04.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:04 smithi006 bash[27292]: audit 2023-08-27T22:25:03.536723+0000 mon.c (mon.2) 340 : audit [DBG] from='client.? 172.21.15.6:0/352695736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:04.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:04 smithi006 bash[33918]: cluster 2023-08-27T22:25:03.500208+0000 mgr.y (mgr.14901) 796 : cluster [DBG] pgmap v803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:04.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:04 smithi006 bash[33918]: audit 2023-08-27T22:25:03.536723+0000 mon.c (mon.2) 340 : audit [DBG] from='client.? 172.21.15.6:0/352695736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:06 smithi130 bash[27579]: cluster 2023-08-27T22:25:05.501245+0000 mgr.y (mgr.14901) 797 : cluster [DBG] pgmap v804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:06 smithi130 bash[27579]: audit 2023-08-27T22:25:05.948025+0000 mon.a (mon.0) 1229 : audit [DBG] from='client.? 172.21.15.6:0/4031462204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:06.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:06 smithi006 bash[33918]: cluster 2023-08-27T22:25:05.501245+0000 mgr.y (mgr.14901) 797 : cluster [DBG] pgmap v804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:06.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:06 smithi006 bash[33918]: audit 2023-08-27T22:25:05.948025+0000 mon.a (mon.0) 1229 : audit [DBG] from='client.? 172.21.15.6:0/4031462204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:06.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:06 smithi006 bash[27292]: cluster 2023-08-27T22:25:05.501245+0000 mgr.y (mgr.14901) 797 : cluster [DBG] pgmap v804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:06.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:06 smithi006 bash[27292]: audit 2023-08-27T22:25:05.948025+0000 mon.a (mon.0) 1229 : audit [DBG] from='client.? 172.21.15.6:0/4031462204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:08 smithi130 bash[27579]: cluster 2023-08-27T22:25:07.502019+0000 mgr.y (mgr.14901) 798 : cluster [DBG] pgmap v805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:08 smithi130 bash[27579]: audit 2023-08-27T22:25:08.355027+0000 mon.a (mon.0) 1230 : audit [DBG] from='client.? 172.21.15.6:0/716165028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:08.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:08 smithi006 bash[27292]: cluster 2023-08-27T22:25:07.502019+0000 mgr.y (mgr.14901) 798 : cluster [DBG] pgmap v805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:08.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:08 smithi006 bash[27292]: audit 2023-08-27T22:25:08.355027+0000 mon.a (mon.0) 1230 : audit [DBG] from='client.? 172.21.15.6:0/716165028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:08.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:08 smithi006 bash[33918]: cluster 2023-08-27T22:25:07.502019+0000 mgr.y (mgr.14901) 798 : cluster [DBG] pgmap v805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:08.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:08 smithi006 bash[33918]: audit 2023-08-27T22:25:08.355027+0000 mon.a (mon.0) 1230 : audit [DBG] from='client.? 172.21.15.6:0/716165028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:10.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:25:10.188Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:10.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:25:10.191Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:10 smithi130 bash[27579]: cluster 2023-08-27T22:25:09.502911+0000 mgr.y (mgr.14901) 799 : cluster [DBG] pgmap v806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:10.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:10 smithi006 bash[33918]: cluster 2023-08-27T22:25:09.502911+0000 mgr.y (mgr.14901) 799 : cluster [DBG] pgmap v806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:10.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:10 smithi006 bash[27292]: cluster 2023-08-27T22:25:09.502911+0000 mgr.y (mgr.14901) 799 : cluster [DBG] pgmap v806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:10.973 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:25:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:25:10] "GET /metrics HTTP/1.1" 200 193077 "" "Prometheus/2.33.4" 2023-08-27T22:25:11.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:11 smithi130 bash[27579]: audit 2023-08-27T22:25:10.787131+0000 mon.c (mon.2) 341 : audit [DBG] from='client.? 172.21.15.6:0/2388355196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:11.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:11 smithi006 bash[27292]: audit 2023-08-27T22:25:10.787131+0000 mon.c (mon.2) 341 : audit [DBG] from='client.? 172.21.15.6:0/2388355196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:11.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:25:11.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:11.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:25:11.754Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:11.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:11 smithi006 bash[33918]: audit 2023-08-27T22:25:10.787131+0000 mon.c (mon.2) 341 : audit [DBG] from='client.? 172.21.15.6:0/2388355196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:12 smithi130 bash[27579]: cluster 2023-08-27T22:25:11.503992+0000 mgr.y (mgr.14901) 800 : cluster [DBG] pgmap v807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:12.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:12 smithi006 bash[27292]: cluster 2023-08-27T22:25:11.503992+0000 mgr.y (mgr.14901) 800 : cluster [DBG] pgmap v807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:12.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:12 smithi006 bash[33918]: cluster 2023-08-27T22:25:11.503992+0000 mgr.y (mgr.14901) 800 : cluster [DBG] pgmap v807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:13.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:13 smithi130 bash[27579]: audit 2023-08-27T22:25:13.206208+0000 mon.c (mon.2) 342 : audit [DBG] from='client.? 172.21.15.6:0/3531055810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:13.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:13 smithi006 bash[27292]: audit 2023-08-27T22:25:13.206208+0000 mon.c (mon.2) 342 : audit [DBG] from='client.? 172.21.15.6:0/3531055810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:13.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:13 smithi006 bash[33918]: audit 2023-08-27T22:25:13.206208+0000 mon.c (mon.2) 342 : audit [DBG] from='client.? 172.21.15.6:0/3531055810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:13.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:25:13.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:13.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:25:13.647Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:14 smithi130 bash[27579]: cluster 2023-08-27T22:25:13.504802+0000 mgr.y (mgr.14901) 801 : cluster [DBG] pgmap v808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:14.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:14 smithi006 bash[27292]: cluster 2023-08-27T22:25:13.504802+0000 mgr.y (mgr.14901) 801 : cluster [DBG] pgmap v808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:14.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:14 smithi006 bash[33918]: cluster 2023-08-27T22:25:13.504802+0000 mgr.y (mgr.14901) 801 : cluster [DBG] pgmap v808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:16 smithi130 bash[27579]: cluster 2023-08-27T22:25:15.505911+0000 mgr.y (mgr.14901) 802 : cluster [DBG] pgmap v809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:16 smithi130 bash[27579]: audit 2023-08-27T22:25:15.627535+0000 mon.a (mon.0) 1231 : audit [DBG] from='client.? 172.21.15.6:0/2855567029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:16.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:16 smithi006 bash[33918]: cluster 2023-08-27T22:25:15.505911+0000 mgr.y (mgr.14901) 802 : cluster [DBG] pgmap v809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:16.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:16 smithi006 bash[33918]: audit 2023-08-27T22:25:15.627535+0000 mon.a (mon.0) 1231 : audit [DBG] from='client.? 172.21.15.6:0/2855567029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:16.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:16 smithi006 bash[27292]: cluster 2023-08-27T22:25:15.505911+0000 mgr.y (mgr.14901) 802 : cluster [DBG] pgmap v809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:16.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:16 smithi006 bash[27292]: audit 2023-08-27T22:25:15.627535+0000 mon.a (mon.0) 1231 : audit [DBG] from='client.? 172.21.15.6:0/2855567029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:18 smithi130 bash[27579]: cluster 2023-08-27T22:25:17.506809+0000 mgr.y (mgr.14901) 803 : cluster [DBG] pgmap v810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:18 smithi130 bash[27579]: audit 2023-08-27T22:25:18.044140+0000 mon.a (mon.0) 1232 : audit [DBG] from='client.? 172.21.15.6:0/2989665026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:18.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:18 smithi006 bash[27292]: cluster 2023-08-27T22:25:17.506809+0000 mgr.y (mgr.14901) 803 : cluster [DBG] pgmap v810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:18.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:18 smithi006 bash[27292]: audit 2023-08-27T22:25:18.044140+0000 mon.a (mon.0) 1232 : audit [DBG] from='client.? 172.21.15.6:0/2989665026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:18.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:18 smithi006 bash[33918]: cluster 2023-08-27T22:25:17.506809+0000 mgr.y (mgr.14901) 803 : cluster [DBG] pgmap v810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:18.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:18 smithi006 bash[33918]: audit 2023-08-27T22:25:18.044140+0000 mon.a (mon.0) 1232 : audit [DBG] from='client.? 172.21.15.6:0/2989665026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:19.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:19 smithi130 bash[27579]: audit 2023-08-27T22:25:19.433976+0000 mon.b (mon.1) 202 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:25:19.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:19 smithi006 bash[27292]: audit 2023-08-27T22:25:19.433976+0000 mon.b (mon.1) 202 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:25:19.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:19 smithi006 bash[33918]: audit 2023-08-27T22:25:19.433976+0000 mon.b (mon.1) 202 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:25:20.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:25:20.189Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:20.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:25:20.191Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:20 smithi130 bash[27579]: cluster 2023-08-27T22:25:19.507842+0000 mgr.y (mgr.14901) 804 : cluster [DBG] pgmap v811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:20 smithi130 bash[27579]: audit 2023-08-27T22:25:20.464405+0000 mon.c (mon.2) 343 : audit [DBG] from='client.? 172.21.15.6:0/2897500997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:20.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:25:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:25:20] "GET /metrics HTTP/1.1" 200 193079 "" "Prometheus/2.33.4" 2023-08-27T22:25:20.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:20 smithi006 bash[33918]: cluster 2023-08-27T22:25:19.507842+0000 mgr.y (mgr.14901) 804 : cluster [DBG] pgmap v811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:20.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:20 smithi006 bash[33918]: audit 2023-08-27T22:25:20.464405+0000 mon.c (mon.2) 343 : audit [DBG] from='client.? 172.21.15.6:0/2897500997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:20.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:20 smithi006 bash[27292]: cluster 2023-08-27T22:25:19.507842+0000 mgr.y (mgr.14901) 804 : cluster [DBG] pgmap v811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:20.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:20 smithi006 bash[27292]: audit 2023-08-27T22:25:20.464405+0000 mon.c (mon.2) 343 : audit [DBG] from='client.? 172.21.15.6:0/2897500997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:25:21.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:25:21.756Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:22 smithi130 bash[27579]: cluster 2023-08-27T22:25:21.508971+0000 mgr.y (mgr.14901) 805 : cluster [DBG] pgmap v812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:22.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:22 smithi006 bash[33918]: cluster 2023-08-27T22:25:21.508971+0000 mgr.y (mgr.14901) 805 : cluster [DBG] pgmap v812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:22.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:22 smithi006 bash[27292]: cluster 2023-08-27T22:25:21.508971+0000 mgr.y (mgr.14901) 805 : cluster [DBG] pgmap v812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:23.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:23 smithi130 bash[27579]: audit 2023-08-27T22:25:22.885527+0000 mon.a (mon.0) 1233 : audit [DBG] from='client.? 172.21.15.6:0/4236028904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:23.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:23 smithi006 bash[33918]: audit 2023-08-27T22:25:22.885527+0000 mon.a (mon.0) 1233 : audit [DBG] from='client.? 172.21.15.6:0/4236028904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:23.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:23 smithi006 bash[27292]: audit 2023-08-27T22:25:22.885527+0000 mon.a (mon.0) 1233 : audit [DBG] from='client.? 172.21.15.6:0/4236028904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:23.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:25:23.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:23.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:25:23.648Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:24 smithi130 bash[27579]: cluster 2023-08-27T22:25:23.509750+0000 mgr.y (mgr.14901) 806 : cluster [DBG] pgmap v813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:24.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:24 smithi006 bash[33918]: cluster 2023-08-27T22:25:23.509750+0000 mgr.y (mgr.14901) 806 : cluster [DBG] pgmap v813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:24.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:24 smithi006 bash[27292]: cluster 2023-08-27T22:25:23.509750+0000 mgr.y (mgr.14901) 806 : cluster [DBG] pgmap v813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:26 smithi130 bash[27579]: audit 2023-08-27T22:25:25.297715+0000 mon.a (mon.0) 1234 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:25:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:26 smithi130 bash[27579]: audit 2023-08-27T22:25:25.327251+0000 mon.a (mon.0) 1235 : audit [DBG] from='client.? 172.21.15.6:0/3363947368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:26 smithi130 bash[27579]: cluster 2023-08-27T22:25:25.510891+0000 mgr.y (mgr.14901) 807 : cluster [DBG] pgmap v814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:26 smithi130 bash[27579]: audit 2023-08-27T22:25:25.745176+0000 mon.a (mon.0) 1236 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:25:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:26 smithi130 bash[27579]: audit 2023-08-27T22:25:26.022113+0000 mon.b (mon.1) 203 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:25:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:26 smithi130 bash[27579]: audit 2023-08-27T22:25:26.023689+0000 mon.b (mon.1) 204 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:25:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:26 smithi130 bash[27579]: audit 2023-08-27T22:25:26.028073+0000 mon.a (mon.0) 1237 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:25:26.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:26 smithi006 bash[33918]: audit 2023-08-27T22:25:25.297715+0000 mon.a (mon.0) 1234 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:25:26.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:26 smithi006 bash[33918]: audit 2023-08-27T22:25:25.327251+0000 mon.a (mon.0) 1235 : audit [DBG] from='client.? 172.21.15.6:0/3363947368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:26.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:26 smithi006 bash[33918]: cluster 2023-08-27T22:25:25.510891+0000 mgr.y (mgr.14901) 807 : cluster [DBG] pgmap v814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:26.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:26 smithi006 bash[33918]: audit 2023-08-27T22:25:25.745176+0000 mon.a (mon.0) 1236 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:25:26.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:26 smithi006 bash[33918]: audit 2023-08-27T22:25:26.022113+0000 mon.b (mon.1) 203 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:25:26.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:26 smithi006 bash[33918]: audit 2023-08-27T22:25:26.023689+0000 mon.b (mon.1) 204 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:25:26.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:26 smithi006 bash[33918]: audit 2023-08-27T22:25:26.028073+0000 mon.a (mon.0) 1237 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:25:26.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:26 smithi006 bash[27292]: audit 2023-08-27T22:25:25.297715+0000 mon.a (mon.0) 1234 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:25:26.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:26 smithi006 bash[27292]: audit 2023-08-27T22:25:25.327251+0000 mon.a (mon.0) 1235 : audit [DBG] from='client.? 172.21.15.6:0/3363947368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:26.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:26 smithi006 bash[27292]: cluster 2023-08-27T22:25:25.510891+0000 mgr.y (mgr.14901) 807 : cluster [DBG] pgmap v814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:26.724 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:26 smithi006 bash[27292]: audit 2023-08-27T22:25:25.745176+0000 mon.a (mon.0) 1236 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:25:26.724 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:26 smithi006 bash[27292]: audit 2023-08-27T22:25:26.022113+0000 mon.b (mon.1) 203 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:25:26.724 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:26 smithi006 bash[27292]: audit 2023-08-27T22:25:26.023689+0000 mon.b (mon.1) 204 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:25:26.724 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:26 smithi006 bash[27292]: audit 2023-08-27T22:25:26.028073+0000 mon.a (mon.0) 1237 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:25:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:28 smithi130 bash[27579]: cluster 2023-08-27T22:25:27.511456+0000 mgr.y (mgr.14901) 808 : cluster [DBG] pgmap v815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:28 smithi130 bash[27579]: audit 2023-08-27T22:25:27.748145+0000 mon.c (mon.2) 344 : audit [DBG] from='client.? 172.21.15.6:0/2043857272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:28.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:28 smithi006 bash[33918]: cluster 2023-08-27T22:25:27.511456+0000 mgr.y (mgr.14901) 808 : cluster [DBG] pgmap v815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:28.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:28 smithi006 bash[33918]: audit 2023-08-27T22:25:27.748145+0000 mon.c (mon.2) 344 : audit [DBG] from='client.? 172.21.15.6:0/2043857272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:28.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:28 smithi006 bash[27292]: cluster 2023-08-27T22:25:27.511456+0000 mgr.y (mgr.14901) 808 : cluster [DBG] pgmap v815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:28.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:28 smithi006 bash[27292]: audit 2023-08-27T22:25:27.748145+0000 mon.c (mon.2) 344 : audit [DBG] from='client.? 172.21.15.6:0/2043857272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:30.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:25:30.190Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:30.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:25:30.192Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:30 smithi130 bash[27579]: cluster 2023-08-27T22:25:29.512319+0000 mgr.y (mgr.14901) 809 : cluster [DBG] pgmap v816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:30 smithi130 bash[27579]: audit 2023-08-27T22:25:30.159962+0000 mon.a (mon.0) 1238 : audit [DBG] from='client.? 172.21.15.6:0/893299290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:30.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:30 smithi006 bash[27292]: cluster 2023-08-27T22:25:29.512319+0000 mgr.y (mgr.14901) 809 : cluster [DBG] pgmap v816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:30.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:30 smithi006 bash[27292]: audit 2023-08-27T22:25:30.159962+0000 mon.a (mon.0) 1238 : audit [DBG] from='client.? 172.21.15.6:0/893299290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:30.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:25:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:25:30] "GET /metrics HTTP/1.1" 200 193081 "" "Prometheus/2.33.4" 2023-08-27T22:25:30.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:30 smithi006 bash[33918]: cluster 2023-08-27T22:25:29.512319+0000 mgr.y (mgr.14901) 809 : cluster [DBG] pgmap v816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:30.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:30 smithi006 bash[33918]: audit 2023-08-27T22:25:30.159962+0000 mon.a (mon.0) 1238 : audit [DBG] from='client.? 172.21.15.6:0/893299290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:25:31.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:25:31.756Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:32 smithi130 bash[27579]: cluster 2023-08-27T22:25:31.513364+0000 mgr.y (mgr.14901) 810 : cluster [DBG] pgmap v817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:32 smithi130 bash[27579]: audit 2023-08-27T22:25:32.573476+0000 mon.c (mon.2) 345 : audit [DBG] from='client.? 172.21.15.6:0/953347957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:32.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:32 smithi006 bash[27292]: cluster 2023-08-27T22:25:31.513364+0000 mgr.y (mgr.14901) 810 : cluster [DBG] pgmap v817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:32.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:32 smithi006 bash[27292]: audit 2023-08-27T22:25:32.573476+0000 mon.c (mon.2) 345 : audit [DBG] from='client.? 172.21.15.6:0/953347957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:32.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:32 smithi006 bash[33918]: cluster 2023-08-27T22:25:31.513364+0000 mgr.y (mgr.14901) 810 : cluster [DBG] pgmap v817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:32.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:32 smithi006 bash[33918]: audit 2023-08-27T22:25:32.573476+0000 mon.c (mon.2) 345 : audit [DBG] from='client.? 172.21.15.6:0/953347957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:25:33.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:33.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:25:33.648Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:34 smithi130 bash[27579]: cluster 2023-08-27T22:25:33.514032+0000 mgr.y (mgr.14901) 811 : cluster [DBG] pgmap v818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:34.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:34 smithi006 bash[27292]: cluster 2023-08-27T22:25:33.514032+0000 mgr.y (mgr.14901) 811 : cluster [DBG] pgmap v818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:34.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:34 smithi006 bash[33918]: cluster 2023-08-27T22:25:33.514032+0000 mgr.y (mgr.14901) 811 : cluster [DBG] pgmap v818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:35.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:35 smithi130 bash[27579]: audit 2023-08-27T22:25:34.981327+0000 mon.a (mon.0) 1239 : audit [DBG] from='client.? 172.21.15.6:0/2811975361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:35.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:35 smithi006 bash[33918]: audit 2023-08-27T22:25:34.981327+0000 mon.a (mon.0) 1239 : audit [DBG] from='client.? 172.21.15.6:0/2811975361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:35.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:35 smithi006 bash[27292]: audit 2023-08-27T22:25:34.981327+0000 mon.a (mon.0) 1239 : audit [DBG] from='client.? 172.21.15.6:0/2811975361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:36 smithi130 bash[27579]: cluster 2023-08-27T22:25:35.515075+0000 mgr.y (mgr.14901) 812 : cluster [DBG] pgmap v819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:36.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:36 smithi006 bash[27292]: cluster 2023-08-27T22:25:35.515075+0000 mgr.y (mgr.14901) 812 : cluster [DBG] pgmap v819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:36.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:36 smithi006 bash[33918]: cluster 2023-08-27T22:25:35.515075+0000 mgr.y (mgr.14901) 812 : cluster [DBG] pgmap v819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:37.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:37 smithi006 bash[27292]: audit 2023-08-27T22:25:37.390981+0000 mon.a (mon.0) 1240 : audit [DBG] from='client.? 172.21.15.6:0/779349814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:37.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:37 smithi006 bash[33918]: audit 2023-08-27T22:25:37.390981+0000 mon.a (mon.0) 1240 : audit [DBG] from='client.? 172.21.15.6:0/779349814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:38.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:37 smithi130 bash[27579]: audit 2023-08-27T22:25:37.390981+0000 mon.a (mon.0) 1240 : audit [DBG] from='client.? 172.21.15.6:0/779349814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:38.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:38 smithi006 bash[33918]: cluster 2023-08-27T22:25:37.515867+0000 mgr.y (mgr.14901) 813 : cluster [DBG] pgmap v820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:38.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:38 smithi006 bash[27292]: cluster 2023-08-27T22:25:37.515867+0000 mgr.y (mgr.14901) 813 : cluster [DBG] pgmap v820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:38 smithi130 bash[27579]: cluster 2023-08-27T22:25:37.515867+0000 mgr.y (mgr.14901) 813 : cluster [DBG] pgmap v820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:40.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:25:40.191Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:40.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:25:40.193Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:40.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:25:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:25:40] "GET /metrics HTTP/1.1" 200 193081 "" "Prometheus/2.33.4" 2023-08-27T22:25:40.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:40 smithi006 bash[33918]: cluster 2023-08-27T22:25:39.516703+0000 mgr.y (mgr.14901) 814 : cluster [DBG] pgmap v821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:40.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:40 smithi006 bash[33918]: audit 2023-08-27T22:25:39.810333+0000 mon.a (mon.0) 1241 : audit [DBG] from='client.? 172.21.15.6:0/32936842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:40.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:40 smithi006 bash[27292]: cluster 2023-08-27T22:25:39.516703+0000 mgr.y (mgr.14901) 814 : cluster [DBG] pgmap v821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:40.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:40 smithi006 bash[27292]: audit 2023-08-27T22:25:39.810333+0000 mon.a (mon.0) 1241 : audit [DBG] from='client.? 172.21.15.6:0/32936842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:40 smithi130 bash[27579]: cluster 2023-08-27T22:25:39.516703+0000 mgr.y (mgr.14901) 814 : cluster [DBG] pgmap v821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:40 smithi130 bash[27579]: audit 2023-08-27T22:25:39.810333+0000 mon.a (mon.0) 1241 : audit [DBG] from='client.? 172.21.15.6:0/32936842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:25:41.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:25:41.756Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:42.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:42 smithi006 bash[33918]: cluster 2023-08-27T22:25:41.517653+0000 mgr.y (mgr.14901) 815 : cluster [DBG] pgmap v822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:42.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:42 smithi006 bash[33918]: audit 2023-08-27T22:25:42.229832+0000 mon.c (mon.2) 346 : audit [DBG] from='client.? 172.21.15.6:0/4159266535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:42.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:42 smithi006 bash[27292]: cluster 2023-08-27T22:25:41.517653+0000 mgr.y (mgr.14901) 815 : cluster [DBG] pgmap v822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:42.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:42 smithi006 bash[27292]: audit 2023-08-27T22:25:42.229832+0000 mon.c (mon.2) 346 : audit [DBG] from='client.? 172.21.15.6:0/4159266535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:43.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:42 smithi130 bash[27579]: cluster 2023-08-27T22:25:41.517653+0000 mgr.y (mgr.14901) 815 : cluster [DBG] pgmap v822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:42 smithi130 bash[27579]: audit 2023-08-27T22:25:42.229832+0000 mon.c (mon.2) 346 : audit [DBG] from='client.? 172.21.15.6:0/4159266535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:25:43.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:25:43.649Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:44.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:44 smithi006 bash[33918]: cluster 2023-08-27T22:25:43.518290+0000 mgr.y (mgr.14901) 816 : cluster [DBG] pgmap v823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:44.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:44 smithi006 bash[27292]: cluster 2023-08-27T22:25:43.518290+0000 mgr.y (mgr.14901) 816 : cluster [DBG] pgmap v823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:44 smithi130 bash[27579]: cluster 2023-08-27T22:25:43.518290+0000 mgr.y (mgr.14901) 816 : cluster [DBG] pgmap v823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:45.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:45 smithi006 bash[33918]: audit 2023-08-27T22:25:44.640578+0000 mon.c (mon.2) 347 : audit [DBG] from='client.? 172.21.15.6:0/438918095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:45.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:45 smithi006 bash[27292]: audit 2023-08-27T22:25:44.640578+0000 mon.c (mon.2) 347 : audit [DBG] from='client.? 172.21.15.6:0/438918095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:46.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:45 smithi130 bash[27579]: audit 2023-08-27T22:25:44.640578+0000 mon.c (mon.2) 347 : audit [DBG] from='client.? 172.21.15.6:0/438918095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:46.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:46 smithi006 bash[27292]: cluster 2023-08-27T22:25:45.519348+0000 mgr.y (mgr.14901) 817 : cluster [DBG] pgmap v824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:46.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:46 smithi006 bash[33918]: cluster 2023-08-27T22:25:45.519348+0000 mgr.y (mgr.14901) 817 : cluster [DBG] pgmap v824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:47.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:46 smithi130 bash[27579]: cluster 2023-08-27T22:25:45.519348+0000 mgr.y (mgr.14901) 817 : cluster [DBG] pgmap v824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:47.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:47 smithi006 bash[33918]: audit 2023-08-27T22:25:47.054099+0000 mon.a (mon.0) 1242 : audit [DBG] from='client.? 172.21.15.6:0/2978113974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:47.974 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:47 smithi006 bash[27292]: audit 2023-08-27T22:25:47.054099+0000 mon.a (mon.0) 1242 : audit [DBG] from='client.? 172.21.15.6:0/2978113974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:48.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:47 smithi130 bash[27579]: audit 2023-08-27T22:25:47.054099+0000 mon.a (mon.0) 1242 : audit [DBG] from='client.? 172.21.15.6:0/2978113974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:48.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:48 smithi006 bash[27292]: cluster 2023-08-27T22:25:47.520124+0000 mgr.y (mgr.14901) 818 : cluster [DBG] pgmap v825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:48.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:48 smithi006 bash[33918]: cluster 2023-08-27T22:25:47.520124+0000 mgr.y (mgr.14901) 818 : cluster [DBG] pgmap v825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:48 smithi130 bash[27579]: cluster 2023-08-27T22:25:47.520124+0000 mgr.y (mgr.14901) 818 : cluster [DBG] pgmap v825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:49.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:49 smithi006 bash[33918]: audit 2023-08-27T22:25:49.469388+0000 mon.c (mon.2) 348 : audit [DBG] from='client.? 172.21.15.6:0/2182112892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:49.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:49 smithi006 bash[27292]: audit 2023-08-27T22:25:49.469388+0000 mon.c (mon.2) 348 : audit [DBG] from='client.? 172.21.15.6:0/2182112892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:50.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:49 smithi130 bash[27579]: audit 2023-08-27T22:25:49.469388+0000 mon.c (mon.2) 348 : audit [DBG] from='client.? 172.21.15.6:0/2182112892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:50.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:25:50.192Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:50.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:25:50.194Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:50.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:25:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:25:50] "GET /metrics HTTP/1.1" 200 193081 "" "Prometheus/2.33.4" 2023-08-27T22:25:50.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:50 smithi006 bash[33918]: cluster 2023-08-27T22:25:49.520730+0000 mgr.y (mgr.14901) 819 : cluster [DBG] pgmap v826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:50.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:50 smithi006 bash[27292]: cluster 2023-08-27T22:25:49.520730+0000 mgr.y (mgr.14901) 819 : cluster [DBG] pgmap v826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:50 smithi130 bash[27579]: cluster 2023-08-27T22:25:49.520730+0000 mgr.y (mgr.14901) 819 : cluster [DBG] pgmap v826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:25:51.755Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:25:51.757Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:52.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:52 smithi006 bash[33918]: cluster 2023-08-27T22:25:51.521397+0000 mgr.y (mgr.14901) 820 : cluster [DBG] pgmap v827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:52.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:52 smithi006 bash[33918]: audit 2023-08-27T22:25:51.899336+0000 mon.a (mon.0) 1243 : audit [DBG] from='client.? 172.21.15.6:0/3984238778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:52.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:52 smithi006 bash[27292]: cluster 2023-08-27T22:25:51.521397+0000 mgr.y (mgr.14901) 820 : cluster [DBG] pgmap v827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:52.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:52 smithi006 bash[27292]: audit 2023-08-27T22:25:51.899336+0000 mon.a (mon.0) 1243 : audit [DBG] from='client.? 172.21.15.6:0/3984238778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:52 smithi130 bash[27579]: cluster 2023-08-27T22:25:51.521397+0000 mgr.y (mgr.14901) 820 : cluster [DBG] pgmap v827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:52 smithi130 bash[27579]: audit 2023-08-27T22:25:51.899336+0000 mon.a (mon.0) 1243 : audit [DBG] from='client.? 172.21.15.6:0/3984238778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:53.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:25:53.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:53.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:25:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:25:53.650Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:25:54.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:54 smithi006 bash[33918]: cluster 2023-08-27T22:25:53.522181+0000 mgr.y (mgr.14901) 821 : cluster [DBG] pgmap v828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:54.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:54 smithi006 bash[33918]: audit 2023-08-27T22:25:54.318766+0000 mon.c (mon.2) 349 : audit [DBG] from='client.? 172.21.15.6:0/3145365256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:54.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:54 smithi006 bash[27292]: cluster 2023-08-27T22:25:53.522181+0000 mgr.y (mgr.14901) 821 : cluster [DBG] pgmap v828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:54.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:54 smithi006 bash[27292]: audit 2023-08-27T22:25:54.318766+0000 mon.c (mon.2) 349 : audit [DBG] from='client.? 172.21.15.6:0/3145365256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:54 smithi130 bash[27579]: cluster 2023-08-27T22:25:53.522181+0000 mgr.y (mgr.14901) 821 : cluster [DBG] pgmap v828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:54 smithi130 bash[27579]: audit 2023-08-27T22:25:54.318766+0000 mon.c (mon.2) 349 : audit [DBG] from='client.? 172.21.15.6:0/3145365256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:56.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:56 smithi006 bash[33918]: cluster 2023-08-27T22:25:55.523412+0000 mgr.y (mgr.14901) 822 : cluster [DBG] pgmap v829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:56.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:56 smithi006 bash[27292]: cluster 2023-08-27T22:25:55.523412+0000 mgr.y (mgr.14901) 822 : cluster [DBG] pgmap v829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:56 smithi130 bash[27579]: cluster 2023-08-27T22:25:55.523412+0000 mgr.y (mgr.14901) 822 : cluster [DBG] pgmap v829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:57.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:57 smithi006 bash[33918]: audit 2023-08-27T22:25:56.742636+0000 mon.c (mon.2) 350 : audit [DBG] from='client.? 172.21.15.6:0/2342378253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:57.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:57 smithi006 bash[27292]: audit 2023-08-27T22:25:56.742636+0000 mon.c (mon.2) 350 : audit [DBG] from='client.? 172.21.15.6:0/2342378253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:58.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:57 smithi130 bash[27579]: audit 2023-08-27T22:25:56.742636+0000 mon.c (mon.2) 350 : audit [DBG] from='client.? 172.21.15.6:0/2342378253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:58.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:58 smithi006 bash[27292]: cluster 2023-08-27T22:25:57.524251+0000 mgr.y (mgr.14901) 823 : cluster [DBG] pgmap v830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:58.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:58 smithi006 bash[33918]: cluster 2023-08-27T22:25:57.524251+0000 mgr.y (mgr.14901) 823 : cluster [DBG] pgmap v830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:58 smithi130 bash[27579]: cluster 2023-08-27T22:25:57.524251+0000 mgr.y (mgr.14901) 823 : cluster [DBG] pgmap v830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:25:59.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:25:59 smithi006 bash[33918]: audit 2023-08-27T22:25:59.159794+0000 mon.c (mon.2) 351 : audit [DBG] from='client.? 172.21.15.6:0/2760036603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:59.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:25:59 smithi006 bash[27292]: audit 2023-08-27T22:25:59.159794+0000 mon.c (mon.2) 351 : audit [DBG] from='client.? 172.21.15.6:0/2760036603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:25:59 smithi130 bash[27579]: audit 2023-08-27T22:25:59.159794+0000 mon.c (mon.2) 351 : audit [DBG] from='client.? 172.21.15.6:0/2760036603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:00.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:26:00.192Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:00.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:26:00.194Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:00.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:26:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:26:00] "GET /metrics HTTP/1.1" 200 193074 "" "Prometheus/2.33.4" 2023-08-27T22:26:00.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:00 smithi006 bash[33918]: cluster 2023-08-27T22:25:59.525089+0000 mgr.y (mgr.14901) 824 : cluster [DBG] pgmap v831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:00.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:00 smithi006 bash[27292]: cluster 2023-08-27T22:25:59.525089+0000 mgr.y (mgr.14901) 824 : cluster [DBG] pgmap v831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:00 smithi130 bash[27579]: cluster 2023-08-27T22:25:59.525089+0000 mgr.y (mgr.14901) 824 : cluster [DBG] pgmap v831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:02.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:01 smithi130 bash[27579]: audit 2023-08-27T22:26:01.578663+0000 mon.a (mon.0) 1244 : audit [DBG] from='client.? 172.21.15.6:0/553187133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:02.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:01 smithi006 bash[27292]: audit 2023-08-27T22:26:01.578663+0000 mon.a (mon.0) 1244 : audit [DBG] from='client.? 172.21.15.6:0/553187133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:02.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:01 smithi006 bash[33918]: audit 2023-08-27T22:26:01.578663+0000 mon.a (mon.0) 1244 : audit [DBG] from='client.? 172.21.15.6:0/553187133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:02.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:26:01.755Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:02.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:26:01.757Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:02 smithi130 bash[27579]: cluster 2023-08-27T22:26:01.526036+0000 mgr.y (mgr.14901) 825 : cluster [DBG] pgmap v832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:03.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:02 smithi006 bash[27292]: cluster 2023-08-27T22:26:01.526036+0000 mgr.y (mgr.14901) 825 : cluster [DBG] pgmap v832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:03.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:02 smithi006 bash[33918]: cluster 2023-08-27T22:26:01.526036+0000 mgr.y (mgr.14901) 825 : cluster [DBG] pgmap v832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:26:03.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:26:03.651Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:04 smithi130 bash[27579]: cluster 2023-08-27T22:26:03.526708+0000 mgr.y (mgr.14901) 826 : cluster [DBG] pgmap v833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:04 smithi130 bash[27579]: audit 2023-08-27T22:26:03.985243+0000 mon.a (mon.0) 1245 : audit [DBG] from='client.? 172.21.15.6:0/3413702612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:05.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:04 smithi006 bash[33918]: cluster 2023-08-27T22:26:03.526708+0000 mgr.y (mgr.14901) 826 : cluster [DBG] pgmap v833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:05.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:04 smithi006 bash[33918]: audit 2023-08-27T22:26:03.985243+0000 mon.a (mon.0) 1245 : audit [DBG] from='client.? 172.21.15.6:0/3413702612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:05.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:04 smithi006 bash[27292]: cluster 2023-08-27T22:26:03.526708+0000 mgr.y (mgr.14901) 826 : cluster [DBG] pgmap v833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:05.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:04 smithi006 bash[27292]: audit 2023-08-27T22:26:03.985243+0000 mon.a (mon.0) 1245 : audit [DBG] from='client.? 172.21.15.6:0/3413702612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:06 smithi130 bash[27579]: cluster 2023-08-27T22:26:05.527822+0000 mgr.y (mgr.14901) 827 : cluster [DBG] pgmap v834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:06 smithi130 bash[27579]: audit 2023-08-27T22:26:06.401925+0000 mon.a (mon.0) 1246 : audit [DBG] from='client.? 172.21.15.6:0/2726786590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:07.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:06 smithi006 bash[27292]: cluster 2023-08-27T22:26:05.527822+0000 mgr.y (mgr.14901) 827 : cluster [DBG] pgmap v834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:07.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:06 smithi006 bash[27292]: audit 2023-08-27T22:26:06.401925+0000 mon.a (mon.0) 1246 : audit [DBG] from='client.? 172.21.15.6:0/2726786590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:07.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:06 smithi006 bash[33918]: cluster 2023-08-27T22:26:05.527822+0000 mgr.y (mgr.14901) 827 : cluster [DBG] pgmap v834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:07.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:06 smithi006 bash[33918]: audit 2023-08-27T22:26:06.401925+0000 mon.a (mon.0) 1246 : audit [DBG] from='client.? 172.21.15.6:0/2726786590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:08 smithi130 bash[27579]: cluster 2023-08-27T22:26:07.528730+0000 mgr.y (mgr.14901) 828 : cluster [DBG] pgmap v835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:09.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:08 smithi006 bash[27292]: cluster 2023-08-27T22:26:07.528730+0000 mgr.y (mgr.14901) 828 : cluster [DBG] pgmap v835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:08 smithi006 bash[33918]: cluster 2023-08-27T22:26:07.528730+0000 mgr.y (mgr.14901) 828 : cluster [DBG] pgmap v835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:10.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:09 smithi130 bash[27579]: audit 2023-08-27T22:26:08.817821+0000 mon.a (mon.0) 1247 : audit [DBG] from='client.? 172.21.15.6:0/2813232312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:10.200 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:09 smithi006 bash[27292]: audit 2023-08-27T22:26:08.817821+0000 mon.a (mon.0) 1247 : audit [DBG] from='client.? 172.21.15.6:0/2813232312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:10.201 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:09 smithi006 bash[33918]: audit 2023-08-27T22:26:08.817821+0000 mon.a (mon.0) 1247 : audit [DBG] from='client.? 172.21.15.6:0/2813232312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:10.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:26:10.193Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:10.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:26:10.195Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:10 smithi130 bash[27579]: cluster 2023-08-27T22:26:09.529711+0000 mgr.y (mgr.14901) 829 : cluster [DBG] pgmap v836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:11.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:26:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:26:10] "GET /metrics HTTP/1.1" 200 193074 "" "Prometheus/2.33.4" 2023-08-27T22:26:11.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:10 smithi006 bash[33918]: cluster 2023-08-27T22:26:09.529711+0000 mgr.y (mgr.14901) 829 : cluster [DBG] pgmap v836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:11.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:10 smithi006 bash[27292]: cluster 2023-08-27T22:26:09.529711+0000 mgr.y (mgr.14901) 829 : cluster [DBG] pgmap v836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:12.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:11 smithi130 bash[27579]: audit 2023-08-27T22:26:11.232614+0000 mon.c (mon.2) 352 : audit [DBG] from='client.? 172.21.15.6:0/2402367027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:12.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:11 smithi006 bash[33918]: audit 2023-08-27T22:26:11.232614+0000 mon.c (mon.2) 352 : audit [DBG] from='client.? 172.21.15.6:0/2402367027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:12.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:11 smithi006 bash[27292]: audit 2023-08-27T22:26:11.232614+0000 mon.c (mon.2) 352 : audit [DBG] from='client.? 172.21.15.6:0/2402367027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:26:11.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:12.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:26:11.757Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:12 smithi130 bash[27579]: cluster 2023-08-27T22:26:11.530749+0000 mgr.y (mgr.14901) 830 : cluster [DBG] pgmap v837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:13.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:12 smithi006 bash[33918]: cluster 2023-08-27T22:26:11.530749+0000 mgr.y (mgr.14901) 830 : cluster [DBG] pgmap v837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:13.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:12 smithi006 bash[27292]: cluster 2023-08-27T22:26:11.530749+0000 mgr.y (mgr.14901) 830 : cluster [DBG] pgmap v837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:26:13.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:26:13.651Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:14 smithi130 bash[27579]: cluster 2023-08-27T22:26:13.531425+0000 mgr.y (mgr.14901) 831 : cluster [DBG] pgmap v838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:14 smithi130 bash[27579]: audit 2023-08-27T22:26:13.683669+0000 mon.c (mon.2) 353 : audit [DBG] from='client.? 172.21.15.6:0/1878098149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:14.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:14 smithi006 bash[27292]: cluster 2023-08-27T22:26:13.531425+0000 mgr.y (mgr.14901) 831 : cluster [DBG] pgmap v838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:14.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:14 smithi006 bash[27292]: audit 2023-08-27T22:26:13.683669+0000 mon.c (mon.2) 353 : audit [DBG] from='client.? 172.21.15.6:0/1878098149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:14.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:14 smithi006 bash[33918]: cluster 2023-08-27T22:26:13.531425+0000 mgr.y (mgr.14901) 831 : cluster [DBG] pgmap v838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:14.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:14 smithi006 bash[33918]: audit 2023-08-27T22:26:13.683669+0000 mon.c (mon.2) 353 : audit [DBG] from='client.? 172.21.15.6:0/1878098149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:16 smithi130 bash[27579]: cluster 2023-08-27T22:26:15.532587+0000 mgr.y (mgr.14901) 832 : cluster [DBG] pgmap v839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:16 smithi130 bash[27579]: audit 2023-08-27T22:26:16.100003+0000 mon.a (mon.0) 1248 : audit [DBG] from='client.? 172.21.15.6:0/1106631522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:16.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:16 smithi006 bash[33918]: cluster 2023-08-27T22:26:15.532587+0000 mgr.y (mgr.14901) 832 : cluster [DBG] pgmap v839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:16.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:16 smithi006 bash[33918]: audit 2023-08-27T22:26:16.100003+0000 mon.a (mon.0) 1248 : audit [DBG] from='client.? 172.21.15.6:0/1106631522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:16.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:16 smithi006 bash[27292]: cluster 2023-08-27T22:26:15.532587+0000 mgr.y (mgr.14901) 832 : cluster [DBG] pgmap v839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:16.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:16 smithi006 bash[27292]: audit 2023-08-27T22:26:16.100003+0000 mon.a (mon.0) 1248 : audit [DBG] from='client.? 172.21.15.6:0/1106631522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:18 smithi130 bash[27579]: cluster 2023-08-27T22:26:17.533335+0000 mgr.y (mgr.14901) 833 : cluster [DBG] pgmap v840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:18 smithi130 bash[27579]: audit 2023-08-27T22:26:18.512701+0000 mon.c (mon.2) 354 : audit [DBG] from='client.? 172.21.15.6:0/1374179889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:18.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:18 smithi006 bash[33918]: cluster 2023-08-27T22:26:17.533335+0000 mgr.y (mgr.14901) 833 : cluster [DBG] pgmap v840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:18.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:18 smithi006 bash[33918]: audit 2023-08-27T22:26:18.512701+0000 mon.c (mon.2) 354 : audit [DBG] from='client.? 172.21.15.6:0/1374179889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:18.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:18 smithi006 bash[27292]: cluster 2023-08-27T22:26:17.533335+0000 mgr.y (mgr.14901) 833 : cluster [DBG] pgmap v840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:18.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:18 smithi006 bash[27292]: audit 2023-08-27T22:26:18.512701+0000 mon.c (mon.2) 354 : audit [DBG] from='client.? 172.21.15.6:0/1374179889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:20.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:26:20.193Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:20.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:26:20.196Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:20 smithi130 bash[27579]: cluster 2023-08-27T22:26:19.534125+0000 mgr.y (mgr.14901) 834 : cluster [DBG] pgmap v841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:20.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:20 smithi006 bash[33918]: cluster 2023-08-27T22:26:19.534125+0000 mgr.y (mgr.14901) 834 : cluster [DBG] pgmap v841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:20.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:20 smithi006 bash[27292]: cluster 2023-08-27T22:26:19.534125+0000 mgr.y (mgr.14901) 834 : cluster [DBG] pgmap v841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:20.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:26:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:26:20] "GET /metrics HTTP/1.1" 200 193077 "" "Prometheus/2.33.4" 2023-08-27T22:26:21.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:21 smithi130 bash[27579]: audit 2023-08-27T22:26:20.930525+0000 mon.c (mon.2) 355 : audit [DBG] from='client.? 172.21.15.6:0/104023660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:21.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:21 smithi006 bash[27292]: audit 2023-08-27T22:26:20.930525+0000 mon.c (mon.2) 355 : audit [DBG] from='client.? 172.21.15.6:0/104023660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:21.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:26:21.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:21.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:26:21.758Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:21.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:21 smithi006 bash[33918]: audit 2023-08-27T22:26:20.930525+0000 mon.c (mon.2) 355 : audit [DBG] from='client.? 172.21.15.6:0/104023660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:22 smithi130 bash[27579]: cluster 2023-08-27T22:26:21.535196+0000 mgr.y (mgr.14901) 835 : cluster [DBG] pgmap v842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:22.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:22 smithi006 bash[27292]: cluster 2023-08-27T22:26:21.535196+0000 mgr.y (mgr.14901) 835 : cluster [DBG] pgmap v842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:22.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:22 smithi006 bash[33918]: cluster 2023-08-27T22:26:21.535196+0000 mgr.y (mgr.14901) 835 : cluster [DBG] pgmap v842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:23.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:23 smithi130 bash[27579]: audit 2023-08-27T22:26:23.347070+0000 mon.c (mon.2) 356 : audit [DBG] from='client.? 172.21.15.6:0/1754812636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:23.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:23 smithi006 bash[27292]: audit 2023-08-27T22:26:23.347070+0000 mon.c (mon.2) 356 : audit [DBG] from='client.? 172.21.15.6:0/1754812636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:26:23.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:26:23.653Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:23.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:23 smithi006 bash[33918]: audit 2023-08-27T22:26:23.347070+0000 mon.c (mon.2) 356 : audit [DBG] from='client.? 172.21.15.6:0/1754812636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:24 smithi130 bash[27579]: cluster 2023-08-27T22:26:23.535881+0000 mgr.y (mgr.14901) 836 : cluster [DBG] pgmap v843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:24.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:24 smithi006 bash[27292]: cluster 2023-08-27T22:26:23.535881+0000 mgr.y (mgr.14901) 836 : cluster [DBG] pgmap v843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:24.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:24 smithi006 bash[33918]: cluster 2023-08-27T22:26:23.535881+0000 mgr.y (mgr.14901) 836 : cluster [DBG] pgmap v843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:26 smithi130 bash[27579]: cluster 2023-08-27T22:26:25.536947+0000 mgr.y (mgr.14901) 837 : cluster [DBG] pgmap v844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:26 smithi130 bash[27579]: audit 2023-08-27T22:26:25.758523+0000 mon.a (mon.0) 1249 : audit [DBG] from='client.? 172.21.15.6:0/1217099941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:26 smithi130 bash[27579]: audit 2023-08-27T22:26:26.060010+0000 mon.b (mon.1) 205 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:26:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:26 smithi130 bash[27579]: audit 2023-08-27T22:26:26.325817+0000 mon.a (mon.0) 1250 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:26:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:26 smithi130 bash[27579]: audit 2023-08-27T22:26:26.332787+0000 mon.a (mon.0) 1251 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:26:26.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:26 smithi006 bash[27292]: cluster 2023-08-27T22:26:25.536947+0000 mgr.y (mgr.14901) 837 : cluster [DBG] pgmap v844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:26.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:26 smithi006 bash[27292]: audit 2023-08-27T22:26:25.758523+0000 mon.a (mon.0) 1249 : audit [DBG] from='client.? 172.21.15.6:0/1217099941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:26.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:26 smithi006 bash[27292]: audit 2023-08-27T22:26:26.060010+0000 mon.b (mon.1) 205 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:26:26.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:26 smithi006 bash[27292]: audit 2023-08-27T22:26:26.325817+0000 mon.a (mon.0) 1250 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:26:26.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:26 smithi006 bash[27292]: audit 2023-08-27T22:26:26.332787+0000 mon.a (mon.0) 1251 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:26:26.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:26 smithi006 bash[33918]: cluster 2023-08-27T22:26:25.536947+0000 mgr.y (mgr.14901) 837 : cluster [DBG] pgmap v844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:26.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:26 smithi006 bash[33918]: audit 2023-08-27T22:26:25.758523+0000 mon.a (mon.0) 1249 : audit [DBG] from='client.? 172.21.15.6:0/1217099941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:26.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:26 smithi006 bash[33918]: audit 2023-08-27T22:26:26.060010+0000 mon.b (mon.1) 205 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:26:26.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:26 smithi006 bash[33918]: audit 2023-08-27T22:26:26.325817+0000 mon.a (mon.0) 1250 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:26:26.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:26 smithi006 bash[33918]: audit 2023-08-27T22:26:26.332787+0000 mon.a (mon.0) 1251 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:26:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:28 smithi130 bash[27579]: cluster 2023-08-27T22:26:27.538076+0000 mgr.y (mgr.14901) 838 : cluster [DBG] pgmap v845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:28 smithi130 bash[27579]: audit 2023-08-27T22:26:28.174889+0000 mon.a (mon.0) 1252 : audit [DBG] from='client.? 172.21.15.6:0/1074730485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:28.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:28 smithi006 bash[33918]: cluster 2023-08-27T22:26:27.538076+0000 mgr.y (mgr.14901) 838 : cluster [DBG] pgmap v845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:28.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:28 smithi006 bash[33918]: audit 2023-08-27T22:26:28.174889+0000 mon.a (mon.0) 1252 : audit [DBG] from='client.? 172.21.15.6:0/1074730485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:28.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:28 smithi006 bash[27292]: cluster 2023-08-27T22:26:27.538076+0000 mgr.y (mgr.14901) 838 : cluster [DBG] pgmap v845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:28.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:28 smithi006 bash[27292]: audit 2023-08-27T22:26:28.174889+0000 mon.a (mon.0) 1252 : audit [DBG] from='client.? 172.21.15.6:0/1074730485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:30.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:26:30.195Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:30.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:26:30.197Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:30.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:30 smithi130 bash[27579]: cluster 2023-08-27T22:26:29.538906+0000 mgr.y (mgr.14901) 839 : cluster [DBG] pgmap v846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:30.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:30 smithi006 bash[27292]: cluster 2023-08-27T22:26:29.538906+0000 mgr.y (mgr.14901) 839 : cluster [DBG] pgmap v846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:30.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:26:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:26:30] "GET /metrics HTTP/1.1" 200 193079 "" "Prometheus/2.33.4" 2023-08-27T22:26:30.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:30 smithi006 bash[33918]: cluster 2023-08-27T22:26:29.538906+0000 mgr.y (mgr.14901) 839 : cluster [DBG] pgmap v846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:31.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:31 smithi130 bash[27579]: audit 2023-08-27T22:26:30.599672+0000 mon.c (mon.2) 357 : audit [DBG] from='client.? 172.21.15.6:0/840294216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:31.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:31 smithi006 bash[33918]: audit 2023-08-27T22:26:30.599672+0000 mon.c (mon.2) 357 : audit [DBG] from='client.? 172.21.15.6:0/840294216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:31.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:31 smithi006 bash[27292]: audit 2023-08-27T22:26:30.599672+0000 mon.c (mon.2) 357 : audit [DBG] from='client.? 172.21.15.6:0/840294216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:31.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:26:31.757Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:31.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:26:31.759Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:32 smithi130 bash[27579]: cluster 2023-08-27T22:26:31.539784+0000 mgr.y (mgr.14901) 840 : cluster [DBG] pgmap v847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:32 smithi130 bash[27579]: audit 2023-08-27T22:26:32.206330+0000 mon.a (mon.0) 1253 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:26:32.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:32 smithi006 bash[27292]: cluster 2023-08-27T22:26:31.539784+0000 mgr.y (mgr.14901) 840 : cluster [DBG] pgmap v847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:32.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:32 smithi006 bash[27292]: audit 2023-08-27T22:26:32.206330+0000 mon.a (mon.0) 1253 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:26:32.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:32 smithi006 bash[33918]: cluster 2023-08-27T22:26:31.539784+0000 mgr.y (mgr.14901) 840 : cluster [DBG] pgmap v847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:32.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:32 smithi006 bash[33918]: audit 2023-08-27T22:26:32.206330+0000 mon.a (mon.0) 1253 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:26:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:26:33.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:26:33.654Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:33.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:33 smithi006 bash[33918]: audit 2023-08-27T22:26:32.685245+0000 mon.a (mon.0) 1254 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:26:33.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:33 smithi006 bash[33918]: audit 2023-08-27T22:26:32.955761+0000 mon.b (mon.1) 206 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:26:33.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:33 smithi006 bash[33918]: audit 2023-08-27T22:26:32.957624+0000 mon.b (mon.1) 207 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:26:33.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:33 smithi006 bash[33918]: audit 2023-08-27T22:26:32.960024+0000 mon.a (mon.0) 1255 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:26:33.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:33 smithi006 bash[33918]: audit 2023-08-27T22:26:33.027915+0000 mon.a (mon.0) 1256 : audit [DBG] from='client.? 172.21.15.6:0/776848644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:33.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:33 smithi006 bash[27292]: audit 2023-08-27T22:26:32.685245+0000 mon.a (mon.0) 1254 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:26:33.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:33 smithi006 bash[27292]: audit 2023-08-27T22:26:32.955761+0000 mon.b (mon.1) 206 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:26:33.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:33 smithi006 bash[27292]: audit 2023-08-27T22:26:32.957624+0000 mon.b (mon.1) 207 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:26:33.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:33 smithi006 bash[27292]: audit 2023-08-27T22:26:32.960024+0000 mon.a (mon.0) 1255 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:26:33.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:33 smithi006 bash[27292]: audit 2023-08-27T22:26:33.027915+0000 mon.a (mon.0) 1256 : audit [DBG] from='client.? 172.21.15.6:0/776848644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:33 smithi130 bash[27579]: audit 2023-08-27T22:26:32.685245+0000 mon.a (mon.0) 1254 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:26:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:33 smithi130 bash[27579]: audit 2023-08-27T22:26:32.955761+0000 mon.b (mon.1) 206 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:26:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:33 smithi130 bash[27579]: audit 2023-08-27T22:26:32.957624+0000 mon.b (mon.1) 207 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:26:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:33 smithi130 bash[27579]: audit 2023-08-27T22:26:32.960024+0000 mon.a (mon.0) 1255 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:26:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:33 smithi130 bash[27579]: audit 2023-08-27T22:26:33.027915+0000 mon.a (mon.0) 1256 : audit [DBG] from='client.? 172.21.15.6:0/776848644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:35.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:34 smithi006 bash[27292]: cluster 2023-08-27T22:26:33.540431+0000 mgr.y (mgr.14901) 841 : cluster [DBG] pgmap v848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:35.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:34 smithi006 bash[33918]: cluster 2023-08-27T22:26:33.540431+0000 mgr.y (mgr.14901) 841 : cluster [DBG] pgmap v848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:35.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:34 smithi130 bash[27579]: cluster 2023-08-27T22:26:33.540431+0000 mgr.y (mgr.14901) 841 : cluster [DBG] pgmap v848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:35 smithi130 bash[27579]: audit 2023-08-27T22:26:35.443428+0000 mon.c (mon.2) 358 : audit [DBG] from='client.? 172.21.15.6:0/1518848721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:36.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:35 smithi006 bash[33918]: audit 2023-08-27T22:26:35.443428+0000 mon.c (mon.2) 358 : audit [DBG] from='client.? 172.21.15.6:0/1518848721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:36.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:35 smithi006 bash[27292]: audit 2023-08-27T22:26:35.443428+0000 mon.c (mon.2) 358 : audit [DBG] from='client.? 172.21.15.6:0/1518848721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:37.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:36 smithi130 bash[27579]: cluster 2023-08-27T22:26:35.541534+0000 mgr.y (mgr.14901) 842 : cluster [DBG] pgmap v849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:37.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:36 smithi006 bash[27292]: cluster 2023-08-27T22:26:35.541534+0000 mgr.y (mgr.14901) 842 : cluster [DBG] pgmap v849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:37.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:36 smithi006 bash[33918]: cluster 2023-08-27T22:26:35.541534+0000 mgr.y (mgr.14901) 842 : cluster [DBG] pgmap v849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:38.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:37 smithi130 bash[27579]: audit 2023-08-27T22:26:37.858030+0000 mon.a (mon.0) 1257 : audit [DBG] from='client.? 172.21.15.6:0/2946327584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:38.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:37 smithi006 bash[27292]: audit 2023-08-27T22:26:37.858030+0000 mon.a (mon.0) 1257 : audit [DBG] from='client.? 172.21.15.6:0/2946327584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:38.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:37 smithi006 bash[33918]: audit 2023-08-27T22:26:37.858030+0000 mon.a (mon.0) 1257 : audit [DBG] from='client.? 172.21.15.6:0/2946327584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:38 smithi130 bash[27579]: cluster 2023-08-27T22:26:37.542298+0000 mgr.y (mgr.14901) 843 : cluster [DBG] pgmap v850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:39.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:38 smithi006 bash[33918]: cluster 2023-08-27T22:26:37.542298+0000 mgr.y (mgr.14901) 843 : cluster [DBG] pgmap v850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:39.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:38 smithi006 bash[27292]: cluster 2023-08-27T22:26:37.542298+0000 mgr.y (mgr.14901) 843 : cluster [DBG] pgmap v850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:40.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:26:40.196Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:40.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:26:40.198Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:41.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:26:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:26:40] "GET /metrics HTTP/1.1" 200 193079 "" "Prometheus/2.33.4" 2023-08-27T22:26:41.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:40 smithi006 bash[27292]: cluster 2023-08-27T22:26:39.543118+0000 mgr.y (mgr.14901) 844 : cluster [DBG] pgmap v851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:41.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:40 smithi006 bash[27292]: audit 2023-08-27T22:26:40.272698+0000 mon.a (mon.0) 1258 : audit [DBG] from='client.? 172.21.15.6:0/824054233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:41.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:40 smithi006 bash[33918]: cluster 2023-08-27T22:26:39.543118+0000 mgr.y (mgr.14901) 844 : cluster [DBG] pgmap v851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:41.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:40 smithi006 bash[33918]: audit 2023-08-27T22:26:40.272698+0000 mon.a (mon.0) 1258 : audit [DBG] from='client.? 172.21.15.6:0/824054233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:41.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:40 smithi130 bash[27579]: cluster 2023-08-27T22:26:39.543118+0000 mgr.y (mgr.14901) 844 : cluster [DBG] pgmap v851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:41.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:40 smithi130 bash[27579]: audit 2023-08-27T22:26:40.272698+0000 mon.a (mon.0) 1258 : audit [DBG] from='client.? 172.21.15.6:0/824054233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:26:41.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:26:41.760Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:43.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:43 smithi130 bash[27579]: cluster 2023-08-27T22:26:41.544076+0000 mgr.y (mgr.14901) 845 : cluster [DBG] pgmap v852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:43.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:43 smithi130 bash[27579]: audit 2023-08-27T22:26:42.688330+0000 mon.a (mon.0) 1259 : audit [DBG] from='client.? 172.21.15.6:0/1318932914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:43.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:42 smithi006 bash[27292]: cluster 2023-08-27T22:26:41.544076+0000 mgr.y (mgr.14901) 845 : cluster [DBG] pgmap v852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:43.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:42 smithi006 bash[27292]: audit 2023-08-27T22:26:42.688330+0000 mon.a (mon.0) 1259 : audit [DBG] from='client.? 172.21.15.6:0/1318932914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:43.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:42 smithi006 bash[33918]: cluster 2023-08-27T22:26:41.544076+0000 mgr.y (mgr.14901) 845 : cluster [DBG] pgmap v852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:43.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:42 smithi006 bash[33918]: audit 2023-08-27T22:26:42.688330+0000 mon.a (mon.0) 1259 : audit [DBG] from='client.? 172.21.15.6:0/1318932914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:26:43.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:26:43.655Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:45 smithi130 bash[27579]: cluster 2023-08-27T22:26:43.544759+0000 mgr.y (mgr.14901) 846 : cluster [DBG] pgmap v853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:45.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:44 smithi006 bash[27292]: cluster 2023-08-27T22:26:43.544759+0000 mgr.y (mgr.14901) 846 : cluster [DBG] pgmap v853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:45.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:44 smithi006 bash[33918]: cluster 2023-08-27T22:26:43.544759+0000 mgr.y (mgr.14901) 846 : cluster [DBG] pgmap v853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:46 smithi130 bash[27579]: audit 2023-08-27T22:26:45.103179+0000 mon.c (mon.2) 359 : audit [DBG] from='client.? 172.21.15.6:0/228581208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:46.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:46 smithi006 bash[27292]: audit 2023-08-27T22:26:45.103179+0000 mon.c (mon.2) 359 : audit [DBG] from='client.? 172.21.15.6:0/228581208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:46.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:46 smithi006 bash[33918]: audit 2023-08-27T22:26:45.103179+0000 mon.c (mon.2) 359 : audit [DBG] from='client.? 172.21.15.6:0/228581208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:47.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:47 smithi130 bash[27579]: cluster 2023-08-27T22:26:45.545813+0000 mgr.y (mgr.14901) 847 : cluster [DBG] pgmap v854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:47.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:47 smithi006 bash[27292]: cluster 2023-08-27T22:26:45.545813+0000 mgr.y (mgr.14901) 847 : cluster [DBG] pgmap v854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:47.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:47 smithi006 bash[33918]: cluster 2023-08-27T22:26:45.545813+0000 mgr.y (mgr.14901) 847 : cluster [DBG] pgmap v854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:48 smithi130 bash[27579]: audit 2023-08-27T22:26:47.515820+0000 mon.c (mon.2) 360 : audit [DBG] from='client.? 172.21.15.6:0/2473878736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:48.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:48 smithi006 bash[27292]: audit 2023-08-27T22:26:47.515820+0000 mon.c (mon.2) 360 : audit [DBG] from='client.? 172.21.15.6:0/2473878736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:48.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:48 smithi006 bash[33918]: audit 2023-08-27T22:26:47.515820+0000 mon.c (mon.2) 360 : audit [DBG] from='client.? 172.21.15.6:0/2473878736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:49.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:49 smithi130 bash[27579]: cluster 2023-08-27T22:26:47.546514+0000 mgr.y (mgr.14901) 848 : cluster [DBG] pgmap v855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:49.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:49 smithi006 bash[27292]: cluster 2023-08-27T22:26:47.546514+0000 mgr.y (mgr.14901) 848 : cluster [DBG] pgmap v855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:49.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:49 smithi006 bash[33918]: cluster 2023-08-27T22:26:47.546514+0000 mgr.y (mgr.14901) 848 : cluster [DBG] pgmap v855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:50.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:50 smithi130 bash[27579]: audit 2023-08-27T22:26:49.929483+0000 mon.c (mon.2) 361 : audit [DBG] from='client.? 172.21.15.6:0/1692409474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:50.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:50 smithi006 bash[27292]: audit 2023-08-27T22:26:49.929483+0000 mon.c (mon.2) 361 : audit [DBG] from='client.? 172.21.15.6:0/1692409474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:50.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:50 smithi006 bash[33918]: audit 2023-08-27T22:26:49.929483+0000 mon.c (mon.2) 361 : audit [DBG] from='client.? 172.21.15.6:0/1692409474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:50.473 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:26:50.196Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:50.473 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:26:50.198Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:51.042 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:26:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:26:50] "GET /metrics HTTP/1.1" 200 193074 "" "Prometheus/2.33.4" 2023-08-27T22:26:51.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:51 smithi130 bash[27579]: cluster 2023-08-27T22:26:49.547228+0000 mgr.y (mgr.14901) 849 : cluster [DBG] pgmap v856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:51.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:51 smithi006 bash[27292]: cluster 2023-08-27T22:26:49.547228+0000 mgr.y (mgr.14901) 849 : cluster [DBG] pgmap v856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:51.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:51 smithi006 bash[33918]: cluster 2023-08-27T22:26:49.547228+0000 mgr.y (mgr.14901) 849 : cluster [DBG] pgmap v856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:26:51.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:26:51.761Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:53.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:53 smithi130 bash[27579]: cluster 2023-08-27T22:26:51.548157+0000 mgr.y (mgr.14901) 850 : cluster [DBG] pgmap v857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:53 smithi130 bash[27579]: audit 2023-08-27T22:26:52.348546+0000 mon.c (mon.2) 362 : audit [DBG] from='client.? 172.21.15.6:0/919536365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:53.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:53 smithi006 bash[27292]: cluster 2023-08-27T22:26:51.548157+0000 mgr.y (mgr.14901) 850 : cluster [DBG] pgmap v857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:53.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:53 smithi006 bash[27292]: audit 2023-08-27T22:26:52.348546+0000 mon.c (mon.2) 362 : audit [DBG] from='client.? 172.21.15.6:0/919536365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:53 smithi006 bash[33918]: cluster 2023-08-27T22:26:51.548157+0000 mgr.y (mgr.14901) 850 : cluster [DBG] pgmap v857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:53 smithi006 bash[33918]: audit 2023-08-27T22:26:52.348546+0000 mon.c (mon.2) 362 : audit [DBG] from='client.? 172.21.15.6:0/919536365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:53.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:26:53.653Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:53.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:26:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:26:53.655Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:26:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:54 smithi130 bash[27579]: cluster 2023-08-27T22:26:53.548807+0000 mgr.y (mgr.14901) 851 : cluster [DBG] pgmap v858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:54.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:54 smithi006 bash[27292]: cluster 2023-08-27T22:26:53.548807+0000 mgr.y (mgr.14901) 851 : cluster [DBG] pgmap v858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:54.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:54 smithi006 bash[33918]: cluster 2023-08-27T22:26:53.548807+0000 mgr.y (mgr.14901) 851 : cluster [DBG] pgmap v858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:55.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:55 smithi130 bash[27579]: audit 2023-08-27T22:26:54.768085+0000 mon.c (mon.2) 363 : audit [DBG] from='client.? 172.21.15.6:0/822094086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:55.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:55 smithi006 bash[27292]: audit 2023-08-27T22:26:54.768085+0000 mon.c (mon.2) 363 : audit [DBG] from='client.? 172.21.15.6:0/822094086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:55.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:55 smithi006 bash[33918]: audit 2023-08-27T22:26:54.768085+0000 mon.c (mon.2) 363 : audit [DBG] from='client.? 172.21.15.6:0/822094086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:56 smithi130 bash[27579]: cluster 2023-08-27T22:26:55.549943+0000 mgr.y (mgr.14901) 852 : cluster [DBG] pgmap v859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:56.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:56 smithi006 bash[27292]: cluster 2023-08-27T22:26:55.549943+0000 mgr.y (mgr.14901) 852 : cluster [DBG] pgmap v859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:56.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:56 smithi006 bash[33918]: cluster 2023-08-27T22:26:55.549943+0000 mgr.y (mgr.14901) 852 : cluster [DBG] pgmap v859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:57.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:57 smithi130 bash[27579]: audit 2023-08-27T22:26:57.198270+0000 mon.c (mon.2) 364 : audit [DBG] from='client.? 172.21.15.6:0/1455144501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:57.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:57 smithi006 bash[33918]: audit 2023-08-27T22:26:57.198270+0000 mon.c (mon.2) 364 : audit [DBG] from='client.? 172.21.15.6:0/1455144501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:57.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:57 smithi006 bash[27292]: audit 2023-08-27T22:26:57.198270+0000 mon.c (mon.2) 364 : audit [DBG] from='client.? 172.21.15.6:0/1455144501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:26:58 smithi130 bash[27579]: cluster 2023-08-27T22:26:57.550679+0000 mgr.y (mgr.14901) 853 : cluster [DBG] pgmap v860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:58.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:26:58 smithi006 bash[27292]: cluster 2023-08-27T22:26:57.550679+0000 mgr.y (mgr.14901) 853 : cluster [DBG] pgmap v860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:26:58.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:26:58 smithi006 bash[33918]: cluster 2023-08-27T22:26:57.550679+0000 mgr.y (mgr.14901) 853 : cluster [DBG] pgmap v860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:00.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:27:00.198Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:00.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:27:00.200Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:00 smithi130 bash[27579]: cluster 2023-08-27T22:26:59.551261+0000 mgr.y (mgr.14901) 854 : cluster [DBG] pgmap v861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:00 smithi130 bash[27579]: audit 2023-08-27T22:26:59.609392+0000 mon.a (mon.0) 1260 : audit [DBG] from='client.? 172.21.15.6:0/2114030962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:00.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:27:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:27:00] "GET /metrics HTTP/1.1" 200 193078 "" "Prometheus/2.33.4" 2023-08-27T22:27:00.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:00 smithi006 bash[33918]: cluster 2023-08-27T22:26:59.551261+0000 mgr.y (mgr.14901) 854 : cluster [DBG] pgmap v861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:00.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:00 smithi006 bash[33918]: audit 2023-08-27T22:26:59.609392+0000 mon.a (mon.0) 1260 : audit [DBG] from='client.? 172.21.15.6:0/2114030962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:00.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:00 smithi006 bash[27292]: cluster 2023-08-27T22:26:59.551261+0000 mgr.y (mgr.14901) 854 : cluster [DBG] pgmap v861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:00.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:00 smithi006 bash[27292]: audit 2023-08-27T22:26:59.609392+0000 mon.a (mon.0) 1260 : audit [DBG] from='client.? 172.21.15.6:0/2114030962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:27:01.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:27:01.761Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:02 smithi130 bash[27579]: cluster 2023-08-27T22:27:01.552151+0000 mgr.y (mgr.14901) 855 : cluster [DBG] pgmap v862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:02 smithi130 bash[27579]: audit 2023-08-27T22:27:02.029643+0000 mon.a (mon.0) 1261 : audit [DBG] from='client.? 172.21.15.6:0/3873417842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:02.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:02 smithi006 bash[33918]: cluster 2023-08-27T22:27:01.552151+0000 mgr.y (mgr.14901) 855 : cluster [DBG] pgmap v862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:02.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:02 smithi006 bash[33918]: audit 2023-08-27T22:27:02.029643+0000 mon.a (mon.0) 1261 : audit [DBG] from='client.? 172.21.15.6:0/3873417842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:02.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:02 smithi006 bash[27292]: cluster 2023-08-27T22:27:01.552151+0000 mgr.y (mgr.14901) 855 : cluster [DBG] pgmap v862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:02.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:02 smithi006 bash[27292]: audit 2023-08-27T22:27:02.029643+0000 mon.a (mon.0) 1261 : audit [DBG] from='client.? 172.21.15.6:0/3873417842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:27:03.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:27:03.656Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:04 smithi130 bash[27579]: cluster 2023-08-27T22:27:03.552808+0000 mgr.y (mgr.14901) 856 : cluster [DBG] pgmap v863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:04 smithi130 bash[27579]: audit 2023-08-27T22:27:04.440610+0000 mon.c (mon.2) 365 : audit [DBG] from='client.? 172.21.15.6:0/2354212832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:04.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:04 smithi006 bash[27292]: cluster 2023-08-27T22:27:03.552808+0000 mgr.y (mgr.14901) 856 : cluster [DBG] pgmap v863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:04.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:04 smithi006 bash[27292]: audit 2023-08-27T22:27:04.440610+0000 mon.c (mon.2) 365 : audit [DBG] from='client.? 172.21.15.6:0/2354212832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:04.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:04 smithi006 bash[33918]: cluster 2023-08-27T22:27:03.552808+0000 mgr.y (mgr.14901) 856 : cluster [DBG] pgmap v863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:04.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:04 smithi006 bash[33918]: audit 2023-08-27T22:27:04.440610+0000 mon.c (mon.2) 365 : audit [DBG] from='client.? 172.21.15.6:0/2354212832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:06 smithi130 bash[27579]: cluster 2023-08-27T22:27:05.553835+0000 mgr.y (mgr.14901) 857 : cluster [DBG] pgmap v864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:06.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:06 smithi006 bash[33918]: cluster 2023-08-27T22:27:05.553835+0000 mgr.y (mgr.14901) 857 : cluster [DBG] pgmap v864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:06.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:06 smithi006 bash[27292]: cluster 2023-08-27T22:27:05.553835+0000 mgr.y (mgr.14901) 857 : cluster [DBG] pgmap v864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:07.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:07 smithi130 bash[27579]: audit 2023-08-27T22:27:06.858650+0000 mon.a (mon.0) 1262 : audit [DBG] from='client.? 172.21.15.6:0/2548101797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:07.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:07 smithi006 bash[33918]: audit 2023-08-27T22:27:06.858650+0000 mon.a (mon.0) 1262 : audit [DBG] from='client.? 172.21.15.6:0/2548101797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:07.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:07 smithi006 bash[27292]: audit 2023-08-27T22:27:06.858650+0000 mon.a (mon.0) 1262 : audit [DBG] from='client.? 172.21.15.6:0/2548101797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:08 smithi130 bash[27579]: cluster 2023-08-27T22:27:07.554597+0000 mgr.y (mgr.14901) 858 : cluster [DBG] pgmap v865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:08.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:08 smithi006 bash[27292]: cluster 2023-08-27T22:27:07.554597+0000 mgr.y (mgr.14901) 858 : cluster [DBG] pgmap v865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:08.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:08 smithi006 bash[33918]: cluster 2023-08-27T22:27:07.554597+0000 mgr.y (mgr.14901) 858 : cluster [DBG] pgmap v865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:09 smithi130 bash[27579]: audit 2023-08-27T22:27:09.283454+0000 mon.a (mon.0) 1263 : audit [DBG] from='client.? 172.21.15.6:0/1449101960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:09.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:09 smithi006 bash[33918]: audit 2023-08-27T22:27:09.283454+0000 mon.a (mon.0) 1263 : audit [DBG] from='client.? 172.21.15.6:0/1449101960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:09.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:09 smithi006 bash[27292]: audit 2023-08-27T22:27:09.283454+0000 mon.a (mon.0) 1263 : audit [DBG] from='client.? 172.21.15.6:0/1449101960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:10.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:27:10.199Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:10.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:27:10.201Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:10 smithi130 bash[27579]: cluster 2023-08-27T22:27:09.555440+0000 mgr.y (mgr.14901) 859 : cluster [DBG] pgmap v866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:10.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:27:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:27:10] "GET /metrics HTTP/1.1" 200 193078 "" "Prometheus/2.33.4" 2023-08-27T22:27:10.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:10 smithi006 bash[27292]: cluster 2023-08-27T22:27:09.555440+0000 mgr.y (mgr.14901) 859 : cluster [DBG] pgmap v866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:10.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:10 smithi006 bash[33918]: cluster 2023-08-27T22:27:09.555440+0000 mgr.y (mgr.14901) 859 : cluster [DBG] pgmap v866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:27:11.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:27:11.763Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:12 smithi130 bash[27579]: cluster 2023-08-27T22:27:11.556356+0000 mgr.y (mgr.14901) 860 : cluster [DBG] pgmap v867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:12 smithi130 bash[27579]: audit 2023-08-27T22:27:11.706453+0000 mon.a (mon.0) 1264 : audit [DBG] from='client.? 172.21.15.6:0/3568191149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:12.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:12 smithi006 bash[33918]: cluster 2023-08-27T22:27:11.556356+0000 mgr.y (mgr.14901) 860 : cluster [DBG] pgmap v867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:12.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:12 smithi006 bash[33918]: audit 2023-08-27T22:27:11.706453+0000 mon.a (mon.0) 1264 : audit [DBG] from='client.? 172.21.15.6:0/3568191149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:12.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:12 smithi006 bash[27292]: cluster 2023-08-27T22:27:11.556356+0000 mgr.y (mgr.14901) 860 : cluster [DBG] pgmap v867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:12.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:12 smithi006 bash[27292]: audit 2023-08-27T22:27:11.706453+0000 mon.a (mon.0) 1264 : audit [DBG] from='client.? 172.21.15.6:0/3568191149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:27:13.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:27:13.657Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:14 smithi130 bash[27579]: cluster 2023-08-27T22:27:13.556963+0000 mgr.y (mgr.14901) 861 : cluster [DBG] pgmap v868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:14 smithi130 bash[27579]: audit 2023-08-27T22:27:14.120518+0000 mon.a (mon.0) 1265 : audit [DBG] from='client.? 172.21.15.6:0/3233918565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:14.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:14 smithi006 bash[33918]: cluster 2023-08-27T22:27:13.556963+0000 mgr.y (mgr.14901) 861 : cluster [DBG] pgmap v868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:14.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:14 smithi006 bash[33918]: audit 2023-08-27T22:27:14.120518+0000 mon.a (mon.0) 1265 : audit [DBG] from='client.? 172.21.15.6:0/3233918565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:14.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:14 smithi006 bash[27292]: cluster 2023-08-27T22:27:13.556963+0000 mgr.y (mgr.14901) 861 : cluster [DBG] pgmap v868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:14.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:14 smithi006 bash[27292]: audit 2023-08-27T22:27:14.120518+0000 mon.a (mon.0) 1265 : audit [DBG] from='client.? 172.21.15.6:0/3233918565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:16 smithi130 bash[27579]: cluster 2023-08-27T22:27:15.557941+0000 mgr.y (mgr.14901) 862 : cluster [DBG] pgmap v869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:16 smithi130 bash[27579]: audit 2023-08-27T22:27:16.547401+0000 mon.a (mon.0) 1266 : audit [DBG] from='client.? 172.21.15.6:0/234474645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:16.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:16 smithi006 bash[27292]: cluster 2023-08-27T22:27:15.557941+0000 mgr.y (mgr.14901) 862 : cluster [DBG] pgmap v869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:16.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:16 smithi006 bash[27292]: audit 2023-08-27T22:27:16.547401+0000 mon.a (mon.0) 1266 : audit [DBG] from='client.? 172.21.15.6:0/234474645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:16.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:16 smithi006 bash[33918]: cluster 2023-08-27T22:27:15.557941+0000 mgr.y (mgr.14901) 862 : cluster [DBG] pgmap v869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:16.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:16 smithi006 bash[33918]: audit 2023-08-27T22:27:16.547401+0000 mon.a (mon.0) 1266 : audit [DBG] from='client.? 172.21.15.6:0/234474645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:18 smithi130 bash[27579]: cluster 2023-08-27T22:27:17.558781+0000 mgr.y (mgr.14901) 863 : cluster [DBG] pgmap v870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:18.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:18 smithi006 bash[33918]: cluster 2023-08-27T22:27:17.558781+0000 mgr.y (mgr.14901) 863 : cluster [DBG] pgmap v870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:18.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:18 smithi006 bash[27292]: cluster 2023-08-27T22:27:17.558781+0000 mgr.y (mgr.14901) 863 : cluster [DBG] pgmap v870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:19.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:19 smithi130 bash[27579]: audit 2023-08-27T22:27:18.976547+0000 mon.a (mon.0) 1267 : audit [DBG] from='client.? 172.21.15.6:0/2720234444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:19.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:19 smithi006 bash[27292]: audit 2023-08-27T22:27:18.976547+0000 mon.a (mon.0) 1267 : audit [DBG] from='client.? 172.21.15.6:0/2720234444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:19.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:19 smithi006 bash[33918]: audit 2023-08-27T22:27:18.976547+0000 mon.a (mon.0) 1267 : audit [DBG] from='client.? 172.21.15.6:0/2720234444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:20.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:27:20.199Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:20.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:27:20.201Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:20 smithi130 bash[27579]: cluster 2023-08-27T22:27:19.559660+0000 mgr.y (mgr.14901) 864 : cluster [DBG] pgmap v871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:20.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:27:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:27:20] "GET /metrics HTTP/1.1" 200 193078 "" "Prometheus/2.33.4" 2023-08-27T22:27:20.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:20 smithi006 bash[27292]: cluster 2023-08-27T22:27:19.559660+0000 mgr.y (mgr.14901) 864 : cluster [DBG] pgmap v871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:20.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:20 smithi006 bash[33918]: cluster 2023-08-27T22:27:19.559660+0000 mgr.y (mgr.14901) 864 : cluster [DBG] pgmap v871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:21.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:21 smithi130 bash[27579]: audit 2023-08-27T22:27:21.392790+0000 mon.c (mon.2) 366 : audit [DBG] from='client.? 172.21.15.6:0/1048930069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:21.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:21 smithi006 bash[33918]: audit 2023-08-27T22:27:21.392790+0000 mon.c (mon.2) 366 : audit [DBG] from='client.? 172.21.15.6:0/1048930069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:21.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:21 smithi006 bash[27292]: audit 2023-08-27T22:27:21.392790+0000 mon.c (mon.2) 366 : audit [DBG] from='client.? 172.21.15.6:0/1048930069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:21.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:27:21.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:21.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:27:21.763Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:22 smithi130 bash[27579]: cluster 2023-08-27T22:27:21.560637+0000 mgr.y (mgr.14901) 865 : cluster [DBG] pgmap v872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:22.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:22 smithi006 bash[27292]: cluster 2023-08-27T22:27:21.560637+0000 mgr.y (mgr.14901) 865 : cluster [DBG] pgmap v872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:22.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:22 smithi006 bash[33918]: cluster 2023-08-27T22:27:21.560637+0000 mgr.y (mgr.14901) 865 : cluster [DBG] pgmap v872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:27:23.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:27:23.657Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:24 smithi130 bash[27579]: cluster 2023-08-27T22:27:23.561273+0000 mgr.y (mgr.14901) 866 : cluster [DBG] pgmap v873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:24 smithi130 bash[27579]: audit 2023-08-27T22:27:23.842608+0000 mon.a (mon.0) 1268 : audit [DBG] from='client.? 172.21.15.6:0/506834672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:24.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:24 smithi006 bash[27292]: cluster 2023-08-27T22:27:23.561273+0000 mgr.y (mgr.14901) 866 : cluster [DBG] pgmap v873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:24.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:24 smithi006 bash[27292]: audit 2023-08-27T22:27:23.842608+0000 mon.a (mon.0) 1268 : audit [DBG] from='client.? 172.21.15.6:0/506834672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:24.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:24 smithi006 bash[33918]: cluster 2023-08-27T22:27:23.561273+0000 mgr.y (mgr.14901) 866 : cluster [DBG] pgmap v873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:24.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:24 smithi006 bash[33918]: audit 2023-08-27T22:27:23.842608+0000 mon.a (mon.0) 1268 : audit [DBG] from='client.? 172.21.15.6:0/506834672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:26.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:26 smithi130 bash[27579]: cluster 2023-08-27T22:27:25.562290+0000 mgr.y (mgr.14901) 867 : cluster [DBG] pgmap v874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:26 smithi130 bash[27579]: audit 2023-08-27T22:27:26.252694+0000 mon.c (mon.2) 367 : audit [DBG] from='client.? 172.21.15.6:0/4067574557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:26.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:26 smithi006 bash[33918]: cluster 2023-08-27T22:27:25.562290+0000 mgr.y (mgr.14901) 867 : cluster [DBG] pgmap v874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:26.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:26 smithi006 bash[33918]: audit 2023-08-27T22:27:26.252694+0000 mon.c (mon.2) 367 : audit [DBG] from='client.? 172.21.15.6:0/4067574557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:26.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:26 smithi006 bash[27292]: cluster 2023-08-27T22:27:25.562290+0000 mgr.y (mgr.14901) 867 : cluster [DBG] pgmap v874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:26.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:26 smithi006 bash[27292]: audit 2023-08-27T22:27:26.252694+0000 mon.c (mon.2) 367 : audit [DBG] from='client.? 172.21.15.6:0/4067574557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:28.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:28 smithi006 bash[33918]: cluster 2023-08-27T22:27:27.563070+0000 mgr.y (mgr.14901) 868 : cluster [DBG] pgmap v875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:28.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:28 smithi006 bash[27292]: cluster 2023-08-27T22:27:27.563070+0000 mgr.y (mgr.14901) 868 : cluster [DBG] pgmap v875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:28 smithi130 bash[27579]: cluster 2023-08-27T22:27:27.563070+0000 mgr.y (mgr.14901) 868 : cluster [DBG] pgmap v875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:29.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:29 smithi006 bash[33918]: audit 2023-08-27T22:27:28.664532+0000 mon.a (mon.0) 1269 : audit [DBG] from='client.? 172.21.15.6:0/182578943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:29.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:29 smithi006 bash[27292]: audit 2023-08-27T22:27:28.664532+0000 mon.a (mon.0) 1269 : audit [DBG] from='client.? 172.21.15.6:0/182578943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:30.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:29 smithi130 bash[27579]: audit 2023-08-27T22:27:28.664532+0000 mon.a (mon.0) 1269 : audit [DBG] from='client.? 172.21.15.6:0/182578943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:30.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:27:30.200Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:30.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:27:30.202Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:30.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:30 smithi006 bash[27292]: cluster 2023-08-27T22:27:29.563912+0000 mgr.y (mgr.14901) 869 : cluster [DBG] pgmap v876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:30.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:27:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:27:30] "GET /metrics HTTP/1.1" 200 193074 "" "Prometheus/2.33.4" 2023-08-27T22:27:30.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:30 smithi006 bash[33918]: cluster 2023-08-27T22:27:29.563912+0000 mgr.y (mgr.14901) 869 : cluster [DBG] pgmap v876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:30 smithi130 bash[27579]: cluster 2023-08-27T22:27:29.563912+0000 mgr.y (mgr.14901) 869 : cluster [DBG] pgmap v876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:31.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:31 smithi006 bash[27292]: audit 2023-08-27T22:27:31.076485+0000 mon.c (mon.2) 368 : audit [DBG] from='client.? 172.21.15.6:0/1996732222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:31.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:31 smithi006 bash[33918]: audit 2023-08-27T22:27:31.076485+0000 mon.c (mon.2) 368 : audit [DBG] from='client.? 172.21.15.6:0/1996732222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:31.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:27:31.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:31.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:27:31.765Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:32.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:31 smithi130 bash[27579]: audit 2023-08-27T22:27:31.076485+0000 mon.c (mon.2) 368 : audit [DBG] from='client.? 172.21.15.6:0/1996732222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:32.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:32 smithi006 bash[33918]: cluster 2023-08-27T22:27:31.565042+0000 mgr.y (mgr.14901) 870 : cluster [DBG] pgmap v877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:32.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:32 smithi006 bash[27292]: cluster 2023-08-27T22:27:31.565042+0000 mgr.y (mgr.14901) 870 : cluster [DBG] pgmap v877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:33.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:32 smithi130 bash[27579]: cluster 2023-08-27T22:27:31.565042+0000 mgr.y (mgr.14901) 870 : cluster [DBG] pgmap v877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:33.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:33 smithi006 bash[33918]: audit 2023-08-27T22:27:32.975008+0000 mon.b (mon.1) 208 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:27:33.975 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:33 smithi006 bash[33918]: audit 2023-08-27T22:27:33.488774+0000 mon.a (mon.0) 1270 : audit [DBG] from='client.? 172.21.15.6:0/2034043283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:33.975 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:33 smithi006 bash[27292]: audit 2023-08-27T22:27:32.975008+0000 mon.b (mon.1) 208 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:27:33.975 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:33 smithi006 bash[27292]: audit 2023-08-27T22:27:33.488774+0000 mon.a (mon.0) 1270 : audit [DBG] from='client.? 172.21.15.6:0/2034043283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:33.976 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:27:33.656Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:33.976 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:27:33.658Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:34.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:33 smithi130 bash[27579]: audit 2023-08-27T22:27:32.975008+0000 mon.b (mon.1) 208 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:27:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:33 smithi130 bash[27579]: audit 2023-08-27T22:27:33.488774+0000 mon.a (mon.0) 1270 : audit [DBG] from='client.? 172.21.15.6:0/2034043283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:34.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:34 smithi006 bash[27292]: cluster 2023-08-27T22:27:33.565746+0000 mgr.y (mgr.14901) 871 : cluster [DBG] pgmap v878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:34.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:34 smithi006 bash[33918]: cluster 2023-08-27T22:27:33.565746+0000 mgr.y (mgr.14901) 871 : cluster [DBG] pgmap v878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:34 smithi130 bash[27579]: cluster 2023-08-27T22:27:33.565746+0000 mgr.y (mgr.14901) 871 : cluster [DBG] pgmap v878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:36.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:36 smithi006 bash[33918]: cluster 2023-08-27T22:27:35.566920+0000 mgr.y (mgr.14901) 872 : cluster [DBG] pgmap v879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:36.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:36 smithi006 bash[33918]: audit 2023-08-27T22:27:35.908068+0000 mon.a (mon.0) 1271 : audit [DBG] from='client.? 172.21.15.6:0/3137021301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:36.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:36 smithi006 bash[27292]: cluster 2023-08-27T22:27:35.566920+0000 mgr.y (mgr.14901) 872 : cluster [DBG] pgmap v879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:36.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:36 smithi006 bash[27292]: audit 2023-08-27T22:27:35.908068+0000 mon.a (mon.0) 1271 : audit [DBG] from='client.? 172.21.15.6:0/3137021301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:36 smithi130 bash[27579]: cluster 2023-08-27T22:27:35.566920+0000 mgr.y (mgr.14901) 872 : cluster [DBG] pgmap v879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:37.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:36 smithi130 bash[27579]: audit 2023-08-27T22:27:35.908068+0000 mon.a (mon.0) 1271 : audit [DBG] from='client.? 172.21.15.6:0/3137021301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:38.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:38 smithi006 bash[33918]: cluster 2023-08-27T22:27:37.567751+0000 mgr.y (mgr.14901) 873 : cluster [DBG] pgmap v880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:38.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:38 smithi006 bash[33918]: audit 2023-08-27T22:27:38.364983+0000 mon.a (mon.0) 1272 : audit [DBG] from='client.? 172.21.15.6:0/730177916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:38.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:38 smithi006 bash[27292]: cluster 2023-08-27T22:27:37.567751+0000 mgr.y (mgr.14901) 873 : cluster [DBG] pgmap v880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:38.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:38 smithi006 bash[27292]: audit 2023-08-27T22:27:38.364983+0000 mon.a (mon.0) 1272 : audit [DBG] from='client.? 172.21.15.6:0/730177916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:38 smithi130 bash[27579]: cluster 2023-08-27T22:27:37.567751+0000 mgr.y (mgr.14901) 873 : cluster [DBG] pgmap v880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:38 smithi130 bash[27579]: audit 2023-08-27T22:27:38.364983+0000 mon.a (mon.0) 1272 : audit [DBG] from='client.? 172.21.15.6:0/730177916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:39 smithi130 bash[27579]: audit 2023-08-27T22:27:38.820108+0000 mon.a (mon.0) 1273 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:27:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:39 smithi130 bash[27579]: audit 2023-08-27T22:27:39.261520+0000 mon.a (mon.0) 1274 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:27:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:39 smithi130 bash[27579]: audit 2023-08-27T22:27:39.534180+0000 mon.b (mon.1) 209 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:27:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:39 smithi130 bash[27579]: audit 2023-08-27T22:27:39.536356+0000 mon.b (mon.1) 210 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:27:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:39 smithi130 bash[27579]: audit 2023-08-27T22:27:39.539931+0000 mon.a (mon.0) 1275 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:27:40.208 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:39 smithi006 bash[27292]: audit 2023-08-27T22:27:38.820108+0000 mon.a (mon.0) 1273 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:27:40.209 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:39 smithi006 bash[27292]: audit 2023-08-27T22:27:39.261520+0000 mon.a (mon.0) 1274 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:27:40.209 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:39 smithi006 bash[27292]: audit 2023-08-27T22:27:39.534180+0000 mon.b (mon.1) 209 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:27:40.209 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:39 smithi006 bash[27292]: audit 2023-08-27T22:27:39.536356+0000 mon.b (mon.1) 210 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:27:40.209 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:39 smithi006 bash[27292]: audit 2023-08-27T22:27:39.539931+0000 mon.a (mon.0) 1275 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:27:40.209 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:39 smithi006 bash[33918]: audit 2023-08-27T22:27:38.820108+0000 mon.a (mon.0) 1273 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:27:40.209 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:39 smithi006 bash[33918]: audit 2023-08-27T22:27:39.261520+0000 mon.a (mon.0) 1274 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:27:40.209 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:39 smithi006 bash[33918]: audit 2023-08-27T22:27:39.534180+0000 mon.b (mon.1) 209 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:27:40.209 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:39 smithi006 bash[33918]: audit 2023-08-27T22:27:39.536356+0000 mon.b (mon.1) 210 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:27:40.209 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:39 smithi006 bash[33918]: audit 2023-08-27T22:27:39.539931+0000 mon.a (mon.0) 1275 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:27:40.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:27:40.201Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:40.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:27:40.203Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:40 smithi130 bash[27579]: cluster 2023-08-27T22:27:39.568726+0000 mgr.y (mgr.14901) 874 : cluster [DBG] pgmap v881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:40 smithi130 bash[27579]: audit 2023-08-27T22:27:40.788807+0000 mon.a (mon.0) 1276 : audit [DBG] from='client.? 172.21.15.6:0/241160328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:41.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:27:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:27:40] "GET /metrics HTTP/1.1" 200 193074 "" "Prometheus/2.33.4" 2023-08-27T22:27:41.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:40 smithi006 bash[27292]: cluster 2023-08-27T22:27:39.568726+0000 mgr.y (mgr.14901) 874 : cluster [DBG] pgmap v881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:41.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:40 smithi006 bash[27292]: audit 2023-08-27T22:27:40.788807+0000 mon.a (mon.0) 1276 : audit [DBG] from='client.? 172.21.15.6:0/241160328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:41.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:40 smithi006 bash[33918]: cluster 2023-08-27T22:27:39.568726+0000 mgr.y (mgr.14901) 874 : cluster [DBG] pgmap v881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:41.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:40 smithi006 bash[33918]: audit 2023-08-27T22:27:40.788807+0000 mon.a (mon.0) 1276 : audit [DBG] from='client.? 172.21.15.6:0/241160328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:27:41.763Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:27:41.765Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:43.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:42 smithi130 bash[27579]: cluster 2023-08-27T22:27:41.569656+0000 mgr.y (mgr.14901) 875 : cluster [DBG] pgmap v882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:43.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:42 smithi006 bash[33918]: cluster 2023-08-27T22:27:41.569656+0000 mgr.y (mgr.14901) 875 : cluster [DBG] pgmap v882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:43.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:42 smithi006 bash[27292]: cluster 2023-08-27T22:27:41.569656+0000 mgr.y (mgr.14901) 875 : cluster [DBG] pgmap v882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:43.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:43 smithi006 bash[33918]: audit 2023-08-27T22:27:43.210962+0000 mon.c (mon.2) 369 : audit [DBG] from='client.? 172.21.15.6:0/316339713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:27:43.656Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:27:43.658Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:43.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:43 smithi006 bash[27292]: audit 2023-08-27T22:27:43.210962+0000 mon.c (mon.2) 369 : audit [DBG] from='client.? 172.21.15.6:0/316339713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:44.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:43 smithi130 bash[27579]: audit 2023-08-27T22:27:43.210962+0000 mon.c (mon.2) 369 : audit [DBG] from='client.? 172.21.15.6:0/316339713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:44 smithi130 bash[27579]: cluster 2023-08-27T22:27:43.570349+0000 mgr.y (mgr.14901) 876 : cluster [DBG] pgmap v883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:45.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:44 smithi006 bash[33918]: cluster 2023-08-27T22:27:43.570349+0000 mgr.y (mgr.14901) 876 : cluster [DBG] pgmap v883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:45.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:44 smithi006 bash[27292]: cluster 2023-08-27T22:27:43.570349+0000 mgr.y (mgr.14901) 876 : cluster [DBG] pgmap v883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:46.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:45 smithi130 bash[27579]: audit 2023-08-27T22:27:45.623122+0000 mon.c (mon.2) 370 : audit [DBG] from='client.? 172.21.15.6:0/2061331230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:46.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:45 smithi006 bash[27292]: audit 2023-08-27T22:27:45.623122+0000 mon.c (mon.2) 370 : audit [DBG] from='client.? 172.21.15.6:0/2061331230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:46.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:45 smithi006 bash[33918]: audit 2023-08-27T22:27:45.623122+0000 mon.c (mon.2) 370 : audit [DBG] from='client.? 172.21.15.6:0/2061331230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:46 smithi130 bash[27579]: cluster 2023-08-27T22:27:45.571289+0000 mgr.y (mgr.14901) 877 : cluster [DBG] pgmap v884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:47.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:46 smithi006 bash[27292]: cluster 2023-08-27T22:27:45.571289+0000 mgr.y (mgr.14901) 877 : cluster [DBG] pgmap v884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:47.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:46 smithi006 bash[33918]: cluster 2023-08-27T22:27:45.571289+0000 mgr.y (mgr.14901) 877 : cluster [DBG] pgmap v884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:48 smithi130 bash[27579]: cluster 2023-08-27T22:27:47.572035+0000 mgr.y (mgr.14901) 878 : cluster [DBG] pgmap v885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:48 smithi130 bash[27579]: audit 2023-08-27T22:27:48.034862+0000 mon.c (mon.2) 371 : audit [DBG] from='client.? 172.21.15.6:0/2065959613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:49.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:48 smithi006 bash[27292]: cluster 2023-08-27T22:27:47.572035+0000 mgr.y (mgr.14901) 878 : cluster [DBG] pgmap v885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:49.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:48 smithi006 bash[27292]: audit 2023-08-27T22:27:48.034862+0000 mon.c (mon.2) 371 : audit [DBG] from='client.? 172.21.15.6:0/2065959613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:49.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:48 smithi006 bash[33918]: cluster 2023-08-27T22:27:47.572035+0000 mgr.y (mgr.14901) 878 : cluster [DBG] pgmap v885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:49.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:48 smithi006 bash[33918]: audit 2023-08-27T22:27:48.034862+0000 mon.c (mon.2) 371 : audit [DBG] from='client.? 172.21.15.6:0/2065959613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:50.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:27:50.201Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:50.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:27:50.203Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:50 smithi130 bash[27579]: cluster 2023-08-27T22:27:49.572949+0000 mgr.y (mgr.14901) 879 : cluster [DBG] pgmap v886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:50 smithi130 bash[27579]: audit 2023-08-27T22:27:50.470751+0000 mon.a (mon.0) 1277 : audit [DBG] from='client.? 172.21.15.6:0/2885102211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:51.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:27:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:27:50] "GET /metrics HTTP/1.1" 200 193076 "" "Prometheus/2.33.4" 2023-08-27T22:27:51.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:50 smithi006 bash[27292]: cluster 2023-08-27T22:27:49.572949+0000 mgr.y (mgr.14901) 879 : cluster [DBG] pgmap v886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:51.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:50 smithi006 bash[27292]: audit 2023-08-27T22:27:50.470751+0000 mon.a (mon.0) 1277 : audit [DBG] from='client.? 172.21.15.6:0/2885102211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:51.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:50 smithi006 bash[33918]: cluster 2023-08-27T22:27:49.572949+0000 mgr.y (mgr.14901) 879 : cluster [DBG] pgmap v886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:51.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:50 smithi006 bash[33918]: audit 2023-08-27T22:27:50.470751+0000 mon.a (mon.0) 1277 : audit [DBG] from='client.? 172.21.15.6:0/2885102211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:27:51.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:27:51.766Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:53.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:52 smithi006 bash[27292]: cluster 2023-08-27T22:27:51.573862+0000 mgr.y (mgr.14901) 880 : cluster [DBG] pgmap v887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:53.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:52 smithi006 bash[33918]: cluster 2023-08-27T22:27:51.573862+0000 mgr.y (mgr.14901) 880 : cluster [DBG] pgmap v887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:53.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:52 smithi130 bash[27579]: cluster 2023-08-27T22:27:51.573862+0000 mgr.y (mgr.14901) 880 : cluster [DBG] pgmap v887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:53.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:53 smithi006 bash[27292]: audit 2023-08-27T22:27:52.876687+0000 mon.a (mon.0) 1278 : audit [DBG] from='client.? 172.21.15.6:0/2702674170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:53.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:53 smithi006 bash[33918]: audit 2023-08-27T22:27:52.876687+0000 mon.a (mon.0) 1278 : audit [DBG] from='client.? 172.21.15.6:0/2702674170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:53.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:27:53.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:53.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:27:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:27:53.659Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:27:54.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:53 smithi130 bash[27579]: audit 2023-08-27T22:27:52.876687+0000 mon.a (mon.0) 1278 : audit [DBG] from='client.? 172.21.15.6:0/2702674170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:55.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:54 smithi006 bash[27292]: cluster 2023-08-27T22:27:53.574507+0000 mgr.y (mgr.14901) 881 : cluster [DBG] pgmap v888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:55.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:54 smithi006 bash[33918]: cluster 2023-08-27T22:27:53.574507+0000 mgr.y (mgr.14901) 881 : cluster [DBG] pgmap v888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:55.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:54 smithi130 bash[27579]: cluster 2023-08-27T22:27:53.574507+0000 mgr.y (mgr.14901) 881 : cluster [DBG] pgmap v888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:56.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:55 smithi006 bash[27292]: audit 2023-08-27T22:27:55.280379+0000 mon.a (mon.0) 1279 : audit [DBG] from='client.? 172.21.15.6:0/202607905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:56.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:55 smithi006 bash[33918]: audit 2023-08-27T22:27:55.280379+0000 mon.a (mon.0) 1279 : audit [DBG] from='client.? 172.21.15.6:0/202607905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:56.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:55 smithi130 bash[27579]: audit 2023-08-27T22:27:55.280379+0000 mon.a (mon.0) 1279 : audit [DBG] from='client.? 172.21.15.6:0/202607905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:57.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:56 smithi006 bash[33918]: cluster 2023-08-27T22:27:55.575618+0000 mgr.y (mgr.14901) 882 : cluster [DBG] pgmap v889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:57.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:56 smithi006 bash[27292]: cluster 2023-08-27T22:27:55.575618+0000 mgr.y (mgr.14901) 882 : cluster [DBG] pgmap v889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:57.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:56 smithi130 bash[27579]: cluster 2023-08-27T22:27:55.575618+0000 mgr.y (mgr.14901) 882 : cluster [DBG] pgmap v889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:58.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:57 smithi006 bash[27292]: audit 2023-08-27T22:27:57.694637+0000 mon.a (mon.0) 1280 : audit [DBG] from='client.? 172.21.15.6:0/3923687303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:58.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:57 smithi006 bash[33918]: audit 2023-08-27T22:27:57.694637+0000 mon.a (mon.0) 1280 : audit [DBG] from='client.? 172.21.15.6:0/3923687303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:58.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:57 smithi130 bash[27579]: audit 2023-08-27T22:27:57.694637+0000 mon.a (mon.0) 1280 : audit [DBG] from='client.? 172.21.15.6:0/3923687303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:59.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:27:58 smithi006 bash[33918]: cluster 2023-08-27T22:27:57.576342+0000 mgr.y (mgr.14901) 883 : cluster [DBG] pgmap v890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:59.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:27:58 smithi006 bash[27292]: cluster 2023-08-27T22:27:57.576342+0000 mgr.y (mgr.14901) 883 : cluster [DBG] pgmap v890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:27:59.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:27:58 smithi130 bash[27579]: cluster 2023-08-27T22:27:57.576342+0000 mgr.y (mgr.14901) 883 : cluster [DBG] pgmap v890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:00.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:28:00.202Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:00.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:28:00.204Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:01.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:28:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:28:00] "GET /metrics HTTP/1.1" 200 193077 "" "Prometheus/2.33.4" 2023-08-27T22:28:01.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:00 smithi006 bash[33918]: cluster 2023-08-27T22:27:59.577256+0000 mgr.y (mgr.14901) 884 : cluster [DBG] pgmap v891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:01.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:00 smithi006 bash[33918]: audit 2023-08-27T22:28:00.160689+0000 mon.a (mon.0) 1281 : audit [DBG] from='client.? 172.21.15.6:0/637385453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:01.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:00 smithi006 bash[27292]: cluster 2023-08-27T22:27:59.577256+0000 mgr.y (mgr.14901) 884 : cluster [DBG] pgmap v891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:01.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:00 smithi006 bash[27292]: audit 2023-08-27T22:28:00.160689+0000 mon.a (mon.0) 1281 : audit [DBG] from='client.? 172.21.15.6:0/637385453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:01.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:00 smithi130 bash[27579]: cluster 2023-08-27T22:27:59.577256+0000 mgr.y (mgr.14901) 884 : cluster [DBG] pgmap v891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:01.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:00 smithi130 bash[27579]: audit 2023-08-27T22:28:00.160689+0000 mon.a (mon.0) 1281 : audit [DBG] from='client.? 172.21.15.6:0/637385453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:28:01.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:28:01.767Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:03.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:02 smithi006 bash[27292]: cluster 2023-08-27T22:28:01.578183+0000 mgr.y (mgr.14901) 885 : cluster [DBG] pgmap v892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:03.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:02 smithi006 bash[27292]: audit 2023-08-27T22:28:02.571093+0000 mon.a (mon.0) 1282 : audit [DBG] from='client.? 172.21.15.6:0/839315823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:03.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:02 smithi006 bash[33918]: cluster 2023-08-27T22:28:01.578183+0000 mgr.y (mgr.14901) 885 : cluster [DBG] pgmap v892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:03.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:02 smithi006 bash[33918]: audit 2023-08-27T22:28:02.571093+0000 mon.a (mon.0) 1282 : audit [DBG] from='client.? 172.21.15.6:0/839315823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:03.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:02 smithi130 bash[27579]: cluster 2023-08-27T22:28:01.578183+0000 mgr.y (mgr.14901) 885 : cluster [DBG] pgmap v892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:03.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:02 smithi130 bash[27579]: audit 2023-08-27T22:28:02.571093+0000 mon.a (mon.0) 1282 : audit [DBG] from='client.? 172.21.15.6:0/839315823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:28:03.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:28:03.660Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:05.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:04 smithi006 bash[33918]: cluster 2023-08-27T22:28:03.578825+0000 mgr.y (mgr.14901) 886 : cluster [DBG] pgmap v893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:05.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:04 smithi006 bash[27292]: cluster 2023-08-27T22:28:03.578825+0000 mgr.y (mgr.14901) 886 : cluster [DBG] pgmap v893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:04 smithi130 bash[27579]: cluster 2023-08-27T22:28:03.578825+0000 mgr.y (mgr.14901) 886 : cluster [DBG] pgmap v893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:06.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:05 smithi006 bash[33918]: audit 2023-08-27T22:28:04.986158+0000 mon.c (mon.2) 372 : audit [DBG] from='client.? 172.21.15.6:0/2320109966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:06.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:05 smithi006 bash[27292]: audit 2023-08-27T22:28:04.986158+0000 mon.c (mon.2) 372 : audit [DBG] from='client.? 172.21.15.6:0/2320109966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:06.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:05 smithi130 bash[27579]: audit 2023-08-27T22:28:04.986158+0000 mon.c (mon.2) 372 : audit [DBG] from='client.? 172.21.15.6:0/2320109966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:07.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:06 smithi006 bash[33918]: cluster 2023-08-27T22:28:05.579862+0000 mgr.y (mgr.14901) 887 : cluster [DBG] pgmap v894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:07.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:06 smithi006 bash[27292]: cluster 2023-08-27T22:28:05.579862+0000 mgr.y (mgr.14901) 887 : cluster [DBG] pgmap v894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:07.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:06 smithi130 bash[27579]: cluster 2023-08-27T22:28:05.579862+0000 mgr.y (mgr.14901) 887 : cluster [DBG] pgmap v894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:08.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:07 smithi006 bash[27292]: audit 2023-08-27T22:28:07.393221+0000 mon.c (mon.2) 373 : audit [DBG] from='client.? 172.21.15.6:0/1285726060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:08.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:07 smithi006 bash[33918]: audit 2023-08-27T22:28:07.393221+0000 mon.c (mon.2) 373 : audit [DBG] from='client.? 172.21.15.6:0/1285726060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:08.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:07 smithi130 bash[27579]: audit 2023-08-27T22:28:07.393221+0000 mon.c (mon.2) 373 : audit [DBG] from='client.? 172.21.15.6:0/1285726060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:09.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:08 smithi006 bash[27292]: cluster 2023-08-27T22:28:07.580634+0000 mgr.y (mgr.14901) 888 : cluster [DBG] pgmap v895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:08 smithi006 bash[33918]: cluster 2023-08-27T22:28:07.580634+0000 mgr.y (mgr.14901) 888 : cluster [DBG] pgmap v895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:09.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:08 smithi130 bash[27579]: cluster 2023-08-27T22:28:07.580634+0000 mgr.y (mgr.14901) 888 : cluster [DBG] pgmap v895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:10.210 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:09 smithi006 bash[33918]: audit 2023-08-27T22:28:09.814133+0000 mon.c (mon.2) 374 : audit [DBG] from='client.? 172.21.15.6:0/159249119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:10.210 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:09 smithi006 bash[27292]: audit 2023-08-27T22:28:09.814133+0000 mon.c (mon.2) 374 : audit [DBG] from='client.? 172.21.15.6:0/159249119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:10.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:09 smithi130 bash[27579]: audit 2023-08-27T22:28:09.814133+0000 mon.c (mon.2) 374 : audit [DBG] from='client.? 172.21.15.6:0/159249119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:10.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:28:10.203Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:10.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:28:10.205Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:11.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:28:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:28:10] "GET /metrics HTTP/1.1" 200 193077 "" "Prometheus/2.33.4" 2023-08-27T22:28:11.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:10 smithi006 bash[27292]: cluster 2023-08-27T22:28:09.581492+0000 mgr.y (mgr.14901) 889 : cluster [DBG] pgmap v896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:11.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:10 smithi006 bash[33918]: cluster 2023-08-27T22:28:09.581492+0000 mgr.y (mgr.14901) 889 : cluster [DBG] pgmap v896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:11.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:10 smithi130 bash[27579]: cluster 2023-08-27T22:28:09.581492+0000 mgr.y (mgr.14901) 889 : cluster [DBG] pgmap v896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:28:11.766Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:28:11.768Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:13.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:12 smithi006 bash[33918]: cluster 2023-08-27T22:28:11.582416+0000 mgr.y (mgr.14901) 890 : cluster [DBG] pgmap v897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:13.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:12 smithi006 bash[33918]: audit 2023-08-27T22:28:12.229602+0000 mon.a (mon.0) 1283 : audit [DBG] from='client.? 172.21.15.6:0/3642574654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:13.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:12 smithi006 bash[27292]: cluster 2023-08-27T22:28:11.582416+0000 mgr.y (mgr.14901) 890 : cluster [DBG] pgmap v897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:13.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:12 smithi006 bash[27292]: audit 2023-08-27T22:28:12.229602+0000 mon.a (mon.0) 1283 : audit [DBG] from='client.? 172.21.15.6:0/3642574654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:13.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:12 smithi130 bash[27579]: cluster 2023-08-27T22:28:11.582416+0000 mgr.y (mgr.14901) 890 : cluster [DBG] pgmap v897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:12 smithi130 bash[27579]: audit 2023-08-27T22:28:12.229602+0000 mon.a (mon.0) 1283 : audit [DBG] from='client.? 172.21.15.6:0/3642574654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:28:13.659Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:28:13.661Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:15.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:14 smithi006 bash[33918]: cluster 2023-08-27T22:28:13.583055+0000 mgr.y (mgr.14901) 891 : cluster [DBG] pgmap v898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:15.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:14 smithi006 bash[33918]: audit 2023-08-27T22:28:14.645918+0000 mon.c (mon.2) 375 : audit [DBG] from='client.? 172.21.15.6:0/3519925356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:15.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:14 smithi006 bash[27292]: cluster 2023-08-27T22:28:13.583055+0000 mgr.y (mgr.14901) 891 : cluster [DBG] pgmap v898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:15.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:14 smithi006 bash[27292]: audit 2023-08-27T22:28:14.645918+0000 mon.c (mon.2) 375 : audit [DBG] from='client.? 172.21.15.6:0/3519925356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:15.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:14 smithi130 bash[27579]: cluster 2023-08-27T22:28:13.583055+0000 mgr.y (mgr.14901) 891 : cluster [DBG] pgmap v898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:15.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:14 smithi130 bash[27579]: audit 2023-08-27T22:28:14.645918+0000 mon.c (mon.2) 375 : audit [DBG] from='client.? 172.21.15.6:0/3519925356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:17.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:16 smithi006 bash[27292]: cluster 2023-08-27T22:28:15.584101+0000 mgr.y (mgr.14901) 892 : cluster [DBG] pgmap v899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:17.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:16 smithi006 bash[33918]: cluster 2023-08-27T22:28:15.584101+0000 mgr.y (mgr.14901) 892 : cluster [DBG] pgmap v899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:17.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:16 smithi130 bash[27579]: cluster 2023-08-27T22:28:15.584101+0000 mgr.y (mgr.14901) 892 : cluster [DBG] pgmap v899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:18.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:17 smithi130 bash[27579]: audit 2023-08-27T22:28:17.063410+0000 mon.a (mon.0) 1284 : audit [DBG] from='client.? 172.21.15.6:0/2196861810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:18.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:17 smithi006 bash[27292]: audit 2023-08-27T22:28:17.063410+0000 mon.a (mon.0) 1284 : audit [DBG] from='client.? 172.21.15.6:0/2196861810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:18.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:17 smithi006 bash[33918]: audit 2023-08-27T22:28:17.063410+0000 mon.a (mon.0) 1284 : audit [DBG] from='client.? 172.21.15.6:0/2196861810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:19.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:18 smithi130 bash[27579]: cluster 2023-08-27T22:28:17.584863+0000 mgr.y (mgr.14901) 893 : cluster [DBG] pgmap v900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:19.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:18 smithi006 bash[27292]: cluster 2023-08-27T22:28:17.584863+0000 mgr.y (mgr.14901) 893 : cluster [DBG] pgmap v900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:19.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:18 smithi006 bash[33918]: cluster 2023-08-27T22:28:17.584863+0000 mgr.y (mgr.14901) 893 : cluster [DBG] pgmap v900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:20.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:19 smithi130 bash[27579]: audit 2023-08-27T22:28:19.475808+0000 mon.c (mon.2) 376 : audit [DBG] from='client.? 172.21.15.6:0/2216432795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:20.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:19 smithi006 bash[33918]: audit 2023-08-27T22:28:19.475808+0000 mon.c (mon.2) 376 : audit [DBG] from='client.? 172.21.15.6:0/2216432795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:20.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:19 smithi006 bash[27292]: audit 2023-08-27T22:28:19.475808+0000 mon.c (mon.2) 376 : audit [DBG] from='client.? 172.21.15.6:0/2216432795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:20.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:28:20.203Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:20.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:28:20.205Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:21.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:28:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:28:20] "GET /metrics HTTP/1.1" 200 193079 "" "Prometheus/2.33.4" 2023-08-27T22:28:21.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:20 smithi006 bash[33918]: cluster 2023-08-27T22:28:19.585681+0000 mgr.y (mgr.14901) 894 : cluster [DBG] pgmap v901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:21.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:20 smithi006 bash[27292]: cluster 2023-08-27T22:28:19.585681+0000 mgr.y (mgr.14901) 894 : cluster [DBG] pgmap v901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:21.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:20 smithi130 bash[27579]: cluster 2023-08-27T22:28:19.585681+0000 mgr.y (mgr.14901) 894 : cluster [DBG] pgmap v901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:22.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:21 smithi006 bash[33918]: audit 2023-08-27T22:28:21.890611+0000 mon.a (mon.0) 1285 : audit [DBG] from='client.? 172.21.15.6:0/3022935032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:22.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:21 smithi006 bash[27292]: audit 2023-08-27T22:28:21.890611+0000 mon.a (mon.0) 1285 : audit [DBG] from='client.? 172.21.15.6:0/3022935032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:28:21.766Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:28:21.769Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:22.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:21 smithi130 bash[27579]: audit 2023-08-27T22:28:21.890611+0000 mon.a (mon.0) 1285 : audit [DBG] from='client.? 172.21.15.6:0/3022935032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:23.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:23 smithi130 bash[27579]: cluster 2023-08-27T22:28:21.586598+0000 mgr.y (mgr.14901) 895 : cluster [DBG] pgmap v902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:23.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:22 smithi006 bash[27292]: cluster 2023-08-27T22:28:21.586598+0000 mgr.y (mgr.14901) 895 : cluster [DBG] pgmap v902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:23.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:22 smithi006 bash[33918]: cluster 2023-08-27T22:28:21.586598+0000 mgr.y (mgr.14901) 895 : cluster [DBG] pgmap v902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:28:23.659Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:28:23.661Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:25.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:25 smithi130 bash[27579]: cluster 2023-08-27T22:28:23.587271+0000 mgr.y (mgr.14901) 896 : cluster [DBG] pgmap v903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:25 smithi130 bash[27579]: audit 2023-08-27T22:28:24.301151+0000 mon.a (mon.0) 1286 : audit [DBG] from='client.? 172.21.15.6:0/2892016688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:25.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:24 smithi006 bash[27292]: cluster 2023-08-27T22:28:23.587271+0000 mgr.y (mgr.14901) 896 : cluster [DBG] pgmap v903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:25.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:25 smithi006 bash[27292]: audit 2023-08-27T22:28:24.301151+0000 mon.a (mon.0) 1286 : audit [DBG] from='client.? 172.21.15.6:0/2892016688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:25.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:25 smithi006 bash[33918]: cluster 2023-08-27T22:28:23.587271+0000 mgr.y (mgr.14901) 896 : cluster [DBG] pgmap v903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:25.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:25 smithi006 bash[33918]: audit 2023-08-27T22:28:24.301151+0000 mon.a (mon.0) 1286 : audit [DBG] from='client.? 172.21.15.6:0/2892016688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:27.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:27 smithi130 bash[27579]: cluster 2023-08-27T22:28:25.588316+0000 mgr.y (mgr.14901) 897 : cluster [DBG] pgmap v904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:27.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:27 smithi130 bash[27579]: audit 2023-08-27T22:28:26.714752+0000 mon.a (mon.0) 1287 : audit [DBG] from='client.? 172.21.15.6:0/2768115243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:27.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:27 smithi006 bash[27292]: cluster 2023-08-27T22:28:25.588316+0000 mgr.y (mgr.14901) 897 : cluster [DBG] pgmap v904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:27.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:27 smithi006 bash[27292]: audit 2023-08-27T22:28:26.714752+0000 mon.a (mon.0) 1287 : audit [DBG] from='client.? 172.21.15.6:0/2768115243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:27.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:27 smithi006 bash[33918]: cluster 2023-08-27T22:28:25.588316+0000 mgr.y (mgr.14901) 897 : cluster [DBG] pgmap v904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:27.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:27 smithi006 bash[33918]: audit 2023-08-27T22:28:26.714752+0000 mon.a (mon.0) 1287 : audit [DBG] from='client.? 172.21.15.6:0/2768115243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:29.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:29 smithi130 bash[27579]: cluster 2023-08-27T22:28:27.589068+0000 mgr.y (mgr.14901) 898 : cluster [DBG] pgmap v905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:29.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:29 smithi006 bash[33918]: cluster 2023-08-27T22:28:27.589068+0000 mgr.y (mgr.14901) 898 : cluster [DBG] pgmap v905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:29.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:29 smithi006 bash[27292]: cluster 2023-08-27T22:28:27.589068+0000 mgr.y (mgr.14901) 898 : cluster [DBG] pgmap v905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:30.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:30 smithi130 bash[27579]: audit 2023-08-27T22:28:29.126716+0000 mon.a (mon.0) 1288 : audit [DBG] from='client.? 172.21.15.6:0/871464128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:30.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:30 smithi006 bash[27292]: audit 2023-08-27T22:28:29.126716+0000 mon.a (mon.0) 1288 : audit [DBG] from='client.? 172.21.15.6:0/871464128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:30.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:30 smithi006 bash[33918]: audit 2023-08-27T22:28:29.126716+0000 mon.a (mon.0) 1288 : audit [DBG] from='client.? 172.21.15.6:0/871464128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:30.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:28:30.205Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:30.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:28:30.207Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:31.031 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:28:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:28:30] "GET /metrics HTTP/1.1" 200 193076 "" "Prometheus/2.33.4" 2023-08-27T22:28:31.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:31 smithi130 bash[27579]: cluster 2023-08-27T22:28:29.589932+0000 mgr.y (mgr.14901) 899 : cluster [DBG] pgmap v906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:31.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:31 smithi006 bash[33918]: cluster 2023-08-27T22:28:29.589932+0000 mgr.y (mgr.14901) 899 : cluster [DBG] pgmap v906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:31.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:31 smithi006 bash[27292]: cluster 2023-08-27T22:28:29.589932+0000 mgr.y (mgr.14901) 899 : cluster [DBG] pgmap v906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:32.039 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:28:31.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:32.039 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:28:31.769Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:32.040 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:32 smithi006 bash[27292]: audit 2023-08-27T22:28:31.539276+0000 mon.a (mon.0) 1289 : audit [DBG] from='client.? 172.21.15.6:0/129897637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:32.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:32 smithi130 bash[27579]: audit 2023-08-27T22:28:31.539276+0000 mon.a (mon.0) 1289 : audit [DBG] from='client.? 172.21.15.6:0/129897637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:32.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:32 smithi006 bash[33918]: audit 2023-08-27T22:28:31.539276+0000 mon.a (mon.0) 1289 : audit [DBG] from='client.? 172.21.15.6:0/129897637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:33.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:33 smithi130 bash[27579]: cluster 2023-08-27T22:28:31.590800+0000 mgr.y (mgr.14901) 900 : cluster [DBG] pgmap v907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:33.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:33 smithi006 bash[27292]: cluster 2023-08-27T22:28:31.590800+0000 mgr.y (mgr.14901) 900 : cluster [DBG] pgmap v907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:33.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:33 smithi006 bash[33918]: cluster 2023-08-27T22:28:31.590800+0000 mgr.y (mgr.14901) 900 : cluster [DBG] pgmap v907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:28:33.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:28:33.664Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:34.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:34 smithi130 bash[27579]: cluster 2023-08-27T22:28:33.591444+0000 mgr.y (mgr.14901) 901 : cluster [DBG] pgmap v908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:34 smithi130 bash[27579]: audit 2023-08-27T22:28:33.954153+0000 mon.c (mon.2) 377 : audit [DBG] from='client.? 172.21.15.6:0/3658231116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:34.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:34 smithi006 bash[33918]: cluster 2023-08-27T22:28:33.591444+0000 mgr.y (mgr.14901) 901 : cluster [DBG] pgmap v908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:34.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:34 smithi006 bash[33918]: audit 2023-08-27T22:28:33.954153+0000 mon.c (mon.2) 377 : audit [DBG] from='client.? 172.21.15.6:0/3658231116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:34.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:34 smithi006 bash[27292]: cluster 2023-08-27T22:28:33.591444+0000 mgr.y (mgr.14901) 901 : cluster [DBG] pgmap v908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:34.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:34 smithi006 bash[27292]: audit 2023-08-27T22:28:33.954153+0000 mon.c (mon.2) 377 : audit [DBG] from='client.? 172.21.15.6:0/3658231116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:36.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:36 smithi130 bash[27579]: cluster 2023-08-27T22:28:35.592518+0000 mgr.y (mgr.14901) 902 : cluster [DBG] pgmap v909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:36 smithi130 bash[27579]: audit 2023-08-27T22:28:36.370507+0000 mon.c (mon.2) 378 : audit [DBG] from='client.? 172.21.15.6:0/1370201267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:36.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:36 smithi006 bash[27292]: cluster 2023-08-27T22:28:35.592518+0000 mgr.y (mgr.14901) 902 : cluster [DBG] pgmap v909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:36.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:36 smithi006 bash[27292]: audit 2023-08-27T22:28:36.370507+0000 mon.c (mon.2) 378 : audit [DBG] from='client.? 172.21.15.6:0/1370201267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:36.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:36 smithi006 bash[33918]: cluster 2023-08-27T22:28:35.592518+0000 mgr.y (mgr.14901) 902 : cluster [DBG] pgmap v909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:36.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:36 smithi006 bash[33918]: audit 2023-08-27T22:28:36.370507+0000 mon.c (mon.2) 378 : audit [DBG] from='client.? 172.21.15.6:0/1370201267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:38.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:38 smithi006 bash[27292]: cluster 2023-08-27T22:28:37.593256+0000 mgr.y (mgr.14901) 903 : cluster [DBG] pgmap v910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:38.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:38 smithi006 bash[33918]: cluster 2023-08-27T22:28:37.593256+0000 mgr.y (mgr.14901) 903 : cluster [DBG] pgmap v910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:38 smithi130 bash[27579]: cluster 2023-08-27T22:28:37.593256+0000 mgr.y (mgr.14901) 903 : cluster [DBG] pgmap v910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:39.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:39 smithi006 bash[33918]: audit 2023-08-27T22:28:38.788881+0000 mon.a (mon.0) 1290 : audit [DBG] from='client.? 172.21.15.6:0/3842200629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:39.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:39 smithi006 bash[33918]: audit 2023-08-27T22:28:39.556717+0000 mon.b (mon.1) 211 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:28:39.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:39 smithi006 bash[27292]: audit 2023-08-27T22:28:38.788881+0000 mon.a (mon.0) 1290 : audit [DBG] from='client.? 172.21.15.6:0/3842200629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:39.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:39 smithi006 bash[27292]: audit 2023-08-27T22:28:39.556717+0000 mon.b (mon.1) 211 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:28:40.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:39 smithi130 bash[27579]: audit 2023-08-27T22:28:38.788881+0000 mon.a (mon.0) 1290 : audit [DBG] from='client.? 172.21.15.6:0/3842200629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:39 smithi130 bash[27579]: audit 2023-08-27T22:28:39.556717+0000 mon.b (mon.1) 211 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:28:40.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:28:40.205Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:40.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:28:40.207Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:40.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:28:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:28:40] "GET /metrics HTTP/1.1" 200 193076 "" "Prometheus/2.33.4" 2023-08-27T22:28:40.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:40 smithi006 bash[27292]: cluster 2023-08-27T22:28:39.594105+0000 mgr.y (mgr.14901) 904 : cluster [DBG] pgmap v911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:40.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:40 smithi006 bash[33918]: cluster 2023-08-27T22:28:39.594105+0000 mgr.y (mgr.14901) 904 : cluster [DBG] pgmap v911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:40 smithi130 bash[27579]: cluster 2023-08-27T22:28:39.594105+0000 mgr.y (mgr.14901) 904 : cluster [DBG] pgmap v911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:41.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:41 smithi006 bash[33918]: audit 2023-08-27T22:28:41.204140+0000 mon.c (mon.2) 379 : audit [DBG] from='client.? 172.21.15.6:0/420531985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:41.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:28:41.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:41.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:28:41.770Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:41.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:41 smithi006 bash[27292]: audit 2023-08-27T22:28:41.204140+0000 mon.c (mon.2) 379 : audit [DBG] from='client.? 172.21.15.6:0/420531985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:42.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:41 smithi130 bash[27579]: audit 2023-08-27T22:28:41.204140+0000 mon.c (mon.2) 379 : audit [DBG] from='client.? 172.21.15.6:0/420531985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:42.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:42 smithi006 bash[27292]: cluster 2023-08-27T22:28:41.594994+0000 mgr.y (mgr.14901) 905 : cluster [DBG] pgmap v912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:42.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:42 smithi006 bash[33918]: cluster 2023-08-27T22:28:41.594994+0000 mgr.y (mgr.14901) 905 : cluster [DBG] pgmap v912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:43.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:42 smithi130 bash[27579]: cluster 2023-08-27T22:28:41.594994+0000 mgr.y (mgr.14901) 905 : cluster [DBG] pgmap v912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:43.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:28:43.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:28:43.663Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:44 smithi130 bash[27579]: cluster 2023-08-27T22:28:43.595594+0000 mgr.y (mgr.14901) 906 : cluster [DBG] pgmap v913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:44 smithi130 bash[27579]: audit 2023-08-27T22:28:43.619364+0000 mon.c (mon.2) 380 : audit [DBG] from='client.? 172.21.15.6:0/714278373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:44.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:44 smithi006 bash[27292]: cluster 2023-08-27T22:28:43.595594+0000 mgr.y (mgr.14901) 906 : cluster [DBG] pgmap v913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:44.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:44 smithi006 bash[27292]: audit 2023-08-27T22:28:43.619364+0000 mon.c (mon.2) 380 : audit [DBG] from='client.? 172.21.15.6:0/714278373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:44.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:44 smithi006 bash[33918]: cluster 2023-08-27T22:28:43.595594+0000 mgr.y (mgr.14901) 906 : cluster [DBG] pgmap v913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:44.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:44 smithi006 bash[33918]: audit 2023-08-27T22:28:43.619364+0000 mon.c (mon.2) 380 : audit [DBG] from='client.? 172.21.15.6:0/714278373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:46 smithi130 bash[27579]: audit 2023-08-27T22:28:45.497668+0000 mon.a (mon.0) 1291 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:28:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:46 smithi130 bash[27579]: cluster 2023-08-27T22:28:45.596621+0000 mgr.y (mgr.14901) 907 : cluster [DBG] pgmap v914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:46 smithi130 bash[27579]: audit 2023-08-27T22:28:45.934976+0000 mon.a (mon.0) 1292 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:28:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:46 smithi130 bash[27579]: audit 2023-08-27T22:28:46.037276+0000 mon.c (mon.2) 381 : audit [DBG] from='client.? 172.21.15.6:0/4291734900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:46 smithi130 bash[27579]: audit 2023-08-27T22:28:46.231537+0000 mon.b (mon.1) 212 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:28:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:46 smithi130 bash[27579]: audit 2023-08-27T22:28:46.233802+0000 mon.b (mon.1) 213 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:28:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:46 smithi130 bash[27579]: audit 2023-08-27T22:28:46.236719+0000 mon.a (mon.0) 1293 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:28:46.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:46 smithi006 bash[33918]: audit 2023-08-27T22:28:45.497668+0000 mon.a (mon.0) 1291 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:28:46.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:46 smithi006 bash[33918]: cluster 2023-08-27T22:28:45.596621+0000 mgr.y (mgr.14901) 907 : cluster [DBG] pgmap v914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:46.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:46 smithi006 bash[33918]: audit 2023-08-27T22:28:45.934976+0000 mon.a (mon.0) 1292 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:28:46.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:46 smithi006 bash[33918]: audit 2023-08-27T22:28:46.037276+0000 mon.c (mon.2) 381 : audit [DBG] from='client.? 172.21.15.6:0/4291734900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:46.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:46 smithi006 bash[33918]: audit 2023-08-27T22:28:46.231537+0000 mon.b (mon.1) 212 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:28:46.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:46 smithi006 bash[33918]: audit 2023-08-27T22:28:46.233802+0000 mon.b (mon.1) 213 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:28:46.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:46 smithi006 bash[33918]: audit 2023-08-27T22:28:46.236719+0000 mon.a (mon.0) 1293 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:28:46.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:46 smithi006 bash[27292]: audit 2023-08-27T22:28:45.497668+0000 mon.a (mon.0) 1291 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:28:46.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:46 smithi006 bash[27292]: cluster 2023-08-27T22:28:45.596621+0000 mgr.y (mgr.14901) 907 : cluster [DBG] pgmap v914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:46.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:46 smithi006 bash[27292]: audit 2023-08-27T22:28:45.934976+0000 mon.a (mon.0) 1292 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:28:46.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:46 smithi006 bash[27292]: audit 2023-08-27T22:28:46.037276+0000 mon.c (mon.2) 381 : audit [DBG] from='client.? 172.21.15.6:0/4291734900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:46.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:46 smithi006 bash[27292]: audit 2023-08-27T22:28:46.231537+0000 mon.b (mon.1) 212 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:28:46.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:46 smithi006 bash[27292]: audit 2023-08-27T22:28:46.233802+0000 mon.b (mon.1) 213 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:28:46.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:46 smithi006 bash[27292]: audit 2023-08-27T22:28:46.236719+0000 mon.a (mon.0) 1293 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:28:48.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:48 smithi006 bash[27292]: cluster 2023-08-27T22:28:47.597394+0000 mgr.y (mgr.14901) 908 : cluster [DBG] pgmap v915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:48.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:48 smithi006 bash[27292]: audit 2023-08-27T22:28:48.455663+0000 mon.c (mon.2) 382 : audit [DBG] from='client.? 172.21.15.6:0/1323569230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:48.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:48 smithi006 bash[33918]: cluster 2023-08-27T22:28:47.597394+0000 mgr.y (mgr.14901) 908 : cluster [DBG] pgmap v915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:48.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:48 smithi006 bash[33918]: audit 2023-08-27T22:28:48.455663+0000 mon.c (mon.2) 382 : audit [DBG] from='client.? 172.21.15.6:0/1323569230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:49.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:48 smithi130 bash[27579]: cluster 2023-08-27T22:28:47.597394+0000 mgr.y (mgr.14901) 908 : cluster [DBG] pgmap v915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:48 smithi130 bash[27579]: audit 2023-08-27T22:28:48.455663+0000 mon.c (mon.2) 382 : audit [DBG] from='client.? 172.21.15.6:0/1323569230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:50.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:28:50.207Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:50.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:28:50.209Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:50.971 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:28:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:28:50] "GET /metrics HTTP/1.1" 200 193073 "" "Prometheus/2.33.4" 2023-08-27T22:28:50.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:50 smithi006 bash[33918]: cluster 2023-08-27T22:28:49.598374+0000 mgr.y (mgr.14901) 909 : cluster [DBG] pgmap v916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:50.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:50 smithi006 bash[27292]: cluster 2023-08-27T22:28:49.598374+0000 mgr.y (mgr.14901) 909 : cluster [DBG] pgmap v916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:50 smithi130 bash[27579]: cluster 2023-08-27T22:28:49.598374+0000 mgr.y (mgr.14901) 909 : cluster [DBG] pgmap v916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:51.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:51 smithi006 bash[27292]: audit 2023-08-27T22:28:50.868686+0000 mon.c (mon.2) 383 : audit [DBG] from='client.? 172.21.15.6:0/947756878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:51.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:51 smithi006 bash[33918]: audit 2023-08-27T22:28:50.868686+0000 mon.c (mon.2) 383 : audit [DBG] from='client.? 172.21.15.6:0/947756878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:51.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:28:51.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:51.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:28:51.771Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:52.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:51 smithi130 bash[27579]: audit 2023-08-27T22:28:50.868686+0000 mon.c (mon.2) 383 : audit [DBG] from='client.? 172.21.15.6:0/947756878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:52.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:52 smithi006 bash[33918]: cluster 2023-08-27T22:28:51.599439+0000 mgr.y (mgr.14901) 910 : cluster [DBG] pgmap v917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:52.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:52 smithi006 bash[27292]: cluster 2023-08-27T22:28:51.599439+0000 mgr.y (mgr.14901) 910 : cluster [DBG] pgmap v917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:52 smithi130 bash[27579]: cluster 2023-08-27T22:28:51.599439+0000 mgr.y (mgr.14901) 910 : cluster [DBG] pgmap v917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:53.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:53 smithi006 bash[27292]: audit 2023-08-27T22:28:53.282685+0000 mon.a (mon.0) 1294 : audit [DBG] from='client.? 172.21.15.6:0/1967195328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:53.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:53 smithi006 bash[33918]: audit 2023-08-27T22:28:53.282685+0000 mon.a (mon.0) 1294 : audit [DBG] from='client.? 172.21.15.6:0/1967195328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:53.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:28:53.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:53.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:28:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:28:53.664Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:28:54.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:53 smithi130 bash[27579]: audit 2023-08-27T22:28:53.282685+0000 mon.a (mon.0) 1294 : audit [DBG] from='client.? 172.21.15.6:0/1967195328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:54.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:54 smithi006 bash[27292]: cluster 2023-08-27T22:28:53.600144+0000 mgr.y (mgr.14901) 911 : cluster [DBG] pgmap v918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:54.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:54 smithi006 bash[33918]: cluster 2023-08-27T22:28:53.600144+0000 mgr.y (mgr.14901) 911 : cluster [DBG] pgmap v918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:54 smithi130 bash[27579]: cluster 2023-08-27T22:28:53.600144+0000 mgr.y (mgr.14901) 911 : cluster [DBG] pgmap v918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:56.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:56 smithi006 bash[27292]: cluster 2023-08-27T22:28:55.601197+0000 mgr.y (mgr.14901) 912 : cluster [DBG] pgmap v919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:56.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:56 smithi006 bash[27292]: audit 2023-08-27T22:28:55.696082+0000 mon.a (mon.0) 1295 : audit [DBG] from='client.? 172.21.15.6:0/2378400394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:56.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:56 smithi006 bash[33918]: cluster 2023-08-27T22:28:55.601197+0000 mgr.y (mgr.14901) 912 : cluster [DBG] pgmap v919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:56.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:56 smithi006 bash[33918]: audit 2023-08-27T22:28:55.696082+0000 mon.a (mon.0) 1295 : audit [DBG] from='client.? 172.21.15.6:0/2378400394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:56 smithi130 bash[27579]: cluster 2023-08-27T22:28:55.601197+0000 mgr.y (mgr.14901) 912 : cluster [DBG] pgmap v919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:56 smithi130 bash[27579]: audit 2023-08-27T22:28:55.696082+0000 mon.a (mon.0) 1295 : audit [DBG] from='client.? 172.21.15.6:0/2378400394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:58.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:58 smithi006 bash[27292]: cluster 2023-08-27T22:28:57.601954+0000 mgr.y (mgr.14901) 913 : cluster [DBG] pgmap v920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:58.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:28:58 smithi006 bash[27292]: audit 2023-08-27T22:28:58.101947+0000 mon.a (mon.0) 1296 : audit [DBG] from='client.? 172.21.15.6:0/1098622228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:58.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:58 smithi006 bash[33918]: cluster 2023-08-27T22:28:57.601954+0000 mgr.y (mgr.14901) 913 : cluster [DBG] pgmap v920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:58.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:28:58 smithi006 bash[33918]: audit 2023-08-27T22:28:58.101947+0000 mon.a (mon.0) 1296 : audit [DBG] from='client.? 172.21.15.6:0/1098622228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:58 smithi130 bash[27579]: cluster 2023-08-27T22:28:57.601954+0000 mgr.y (mgr.14901) 913 : cluster [DBG] pgmap v920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:28:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:28:58 smithi130 bash[27579]: audit 2023-08-27T22:28:58.101947+0000 mon.a (mon.0) 1296 : audit [DBG] from='client.? 172.21.15.6:0/1098622228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:00.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:29:00.207Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:00.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:29:00.209Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:00.971 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:29:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:29:00] "GET /metrics HTTP/1.1" 200 193076 "" "Prometheus/2.33.4" 2023-08-27T22:29:00.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:00 smithi006 bash[27292]: cluster 2023-08-27T22:28:59.603076+0000 mgr.y (mgr.14901) 914 : cluster [DBG] pgmap v921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:00.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:00 smithi006 bash[27292]: audit 2023-08-27T22:29:00.516524+0000 mon.a (mon.0) 1297 : audit [DBG] from='client.? 172.21.15.6:0/3905942421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:00.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:00 smithi006 bash[33918]: cluster 2023-08-27T22:28:59.603076+0000 mgr.y (mgr.14901) 914 : cluster [DBG] pgmap v921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:00.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:00 smithi006 bash[33918]: audit 2023-08-27T22:29:00.516524+0000 mon.a (mon.0) 1297 : audit [DBG] from='client.? 172.21.15.6:0/3905942421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:00 smithi130 bash[27579]: cluster 2023-08-27T22:28:59.603076+0000 mgr.y (mgr.14901) 914 : cluster [DBG] pgmap v921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:00 smithi130 bash[27579]: audit 2023-08-27T22:29:00.516524+0000 mon.a (mon.0) 1297 : audit [DBG] from='client.? 172.21.15.6:0/3905942421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:29:01.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:29:01.771Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:02.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:02 smithi006 bash[33918]: cluster 2023-08-27T22:29:01.603731+0000 mgr.y (mgr.14901) 915 : cluster [DBG] pgmap v922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:02.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:02 smithi006 bash[27292]: cluster 2023-08-27T22:29:01.603731+0000 mgr.y (mgr.14901) 915 : cluster [DBG] pgmap v922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:02 smithi130 bash[27579]: cluster 2023-08-27T22:29:01.603731+0000 mgr.y (mgr.14901) 915 : cluster [DBG] pgmap v922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:03.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:03 smithi006 bash[33918]: audit 2023-08-27T22:29:02.933433+0000 mon.a (mon.0) 1298 : audit [DBG] from='client.? 172.21.15.6:0/2952453924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:03.973 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:03 smithi006 bash[27292]: audit 2023-08-27T22:29:02.933433+0000 mon.a (mon.0) 1298 : audit [DBG] from='client.? 172.21.15.6:0/2952453924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:03.974 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:29:03.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:03.974 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:29:03.666Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:04.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:03 smithi130 bash[27579]: audit 2023-08-27T22:29:02.933433+0000 mon.a (mon.0) 1298 : audit [DBG] from='client.? 172.21.15.6:0/2952453924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:04 smithi130 bash[27579]: cluster 2023-08-27T22:29:03.604443+0000 mgr.y (mgr.14901) 916 : cluster [DBG] pgmap v923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:05.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:04 smithi006 bash[33918]: cluster 2023-08-27T22:29:03.604443+0000 mgr.y (mgr.14901) 916 : cluster [DBG] pgmap v923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:05.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:04 smithi006 bash[27292]: cluster 2023-08-27T22:29:03.604443+0000 mgr.y (mgr.14901) 916 : cluster [DBG] pgmap v923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:06.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:05 smithi130 bash[27579]: audit 2023-08-27T22:29:05.350028+0000 mon.a (mon.0) 1299 : audit [DBG] from='client.? 172.21.15.6:0/2184457125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:06.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:05 smithi006 bash[33918]: audit 2023-08-27T22:29:05.350028+0000 mon.a (mon.0) 1299 : audit [DBG] from='client.? 172.21.15.6:0/2184457125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:06.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:05 smithi006 bash[27292]: audit 2023-08-27T22:29:05.350028+0000 mon.a (mon.0) 1299 : audit [DBG] from='client.? 172.21.15.6:0/2184457125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:06 smithi130 bash[27579]: cluster 2023-08-27T22:29:05.605612+0000 mgr.y (mgr.14901) 917 : cluster [DBG] pgmap v924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:07.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:06 smithi006 bash[33918]: cluster 2023-08-27T22:29:05.605612+0000 mgr.y (mgr.14901) 917 : cluster [DBG] pgmap v924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:07.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:06 smithi006 bash[27292]: cluster 2023-08-27T22:29:05.605612+0000 mgr.y (mgr.14901) 917 : cluster [DBG] pgmap v924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:08 smithi130 bash[27579]: cluster 2023-08-27T22:29:07.606087+0000 mgr.y (mgr.14901) 918 : cluster [DBG] pgmap v925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:08 smithi130 bash[27579]: audit 2023-08-27T22:29:07.762820+0000 mon.c (mon.2) 384 : audit [DBG] from='client.? 172.21.15.6:0/2853228173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:08 smithi006 bash[33918]: cluster 2023-08-27T22:29:07.606087+0000 mgr.y (mgr.14901) 918 : cluster [DBG] pgmap v925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:08 smithi006 bash[33918]: audit 2023-08-27T22:29:07.762820+0000 mon.c (mon.2) 384 : audit [DBG] from='client.? 172.21.15.6:0/2853228173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:09.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:08 smithi006 bash[27292]: cluster 2023-08-27T22:29:07.606087+0000 mgr.y (mgr.14901) 918 : cluster [DBG] pgmap v925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:09.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:08 smithi006 bash[27292]: audit 2023-08-27T22:29:07.762820+0000 mon.c (mon.2) 384 : audit [DBG] from='client.? 172.21.15.6:0/2853228173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:10.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:29:10.208Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:10.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:29:10.210Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:10 smithi130 bash[27579]: cluster 2023-08-27T22:29:09.607168+0000 mgr.y (mgr.14901) 919 : cluster [DBG] pgmap v926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:10 smithi130 bash[27579]: audit 2023-08-27T22:29:10.186565+0000 mon.c (mon.2) 385 : audit [DBG] from='client.? 172.21.15.6:0/3963841576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:11.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:29:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:29:10] "GET /metrics HTTP/1.1" 200 193076 "" "Prometheus/2.33.4" 2023-08-27T22:29:11.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:10 smithi006 bash[27292]: cluster 2023-08-27T22:29:09.607168+0000 mgr.y (mgr.14901) 919 : cluster [DBG] pgmap v926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:11.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:10 smithi006 bash[27292]: audit 2023-08-27T22:29:10.186565+0000 mon.c (mon.2) 385 : audit [DBG] from='client.? 172.21.15.6:0/3963841576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:11.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:10 smithi006 bash[33918]: cluster 2023-08-27T22:29:09.607168+0000 mgr.y (mgr.14901) 919 : cluster [DBG] pgmap v926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:11.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:10 smithi006 bash[33918]: audit 2023-08-27T22:29:10.186565+0000 mon.c (mon.2) 385 : audit [DBG] from='client.? 172.21.15.6:0/3963841576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:29:11.770Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:29:11.772Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:13.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:12 smithi130 bash[27579]: cluster 2023-08-27T22:29:11.607835+0000 mgr.y (mgr.14901) 920 : cluster [DBG] pgmap v927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:13.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:12 smithi130 bash[27579]: audit 2023-08-27T22:29:12.600149+0000 mon.a (mon.0) 1300 : audit [DBG] from='client.? 172.21.15.6:0/160244458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:13.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:12 smithi006 bash[27292]: cluster 2023-08-27T22:29:11.607835+0000 mgr.y (mgr.14901) 920 : cluster [DBG] pgmap v927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:13.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:12 smithi006 bash[27292]: audit 2023-08-27T22:29:12.600149+0000 mon.a (mon.0) 1300 : audit [DBG] from='client.? 172.21.15.6:0/160244458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:13.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:12 smithi006 bash[33918]: cluster 2023-08-27T22:29:11.607835+0000 mgr.y (mgr.14901) 920 : cluster [DBG] pgmap v927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:13.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:12 smithi006 bash[33918]: audit 2023-08-27T22:29:12.600149+0000 mon.a (mon.0) 1300 : audit [DBG] from='client.? 172.21.15.6:0/160244458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:29:13.664Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:29:13.666Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:14.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:14 smithi006 bash[33918]: cluster 2023-08-27T22:29:13.608557+0000 mgr.y (mgr.14901) 921 : cluster [DBG] pgmap v928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:14.974 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:14 smithi006 bash[27292]: cluster 2023-08-27T22:29:13.608557+0000 mgr.y (mgr.14901) 921 : cluster [DBG] pgmap v928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:14 smithi130 bash[27579]: cluster 2023-08-27T22:29:13.608557+0000 mgr.y (mgr.14901) 921 : cluster [DBG] pgmap v928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:15.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:15 smithi006 bash[33918]: audit 2023-08-27T22:29:15.015309+0000 mon.c (mon.2) 386 : audit [DBG] from='client.? 172.21.15.6:0/2473298034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:15.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:15 smithi006 bash[27292]: audit 2023-08-27T22:29:15.015309+0000 mon.c (mon.2) 386 : audit [DBG] from='client.? 172.21.15.6:0/2473298034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:16.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:15 smithi130 bash[27579]: audit 2023-08-27T22:29:15.015309+0000 mon.c (mon.2) 386 : audit [DBG] from='client.? 172.21.15.6:0/2473298034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:16.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:16 smithi006 bash[27292]: cluster 2023-08-27T22:29:15.609677+0000 mgr.y (mgr.14901) 922 : cluster [DBG] pgmap v929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:16.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:16 smithi006 bash[33918]: cluster 2023-08-27T22:29:15.609677+0000 mgr.y (mgr.14901) 922 : cluster [DBG] pgmap v929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:16 smithi130 bash[27579]: cluster 2023-08-27T22:29:15.609677+0000 mgr.y (mgr.14901) 922 : cluster [DBG] pgmap v929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:17.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:17 smithi006 bash[27292]: audit 2023-08-27T22:29:17.432238+0000 mon.a (mon.0) 1301 : audit [DBG] from='client.? 172.21.15.6:0/1813548246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:17.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:17 smithi006 bash[33918]: audit 2023-08-27T22:29:17.432238+0000 mon.a (mon.0) 1301 : audit [DBG] from='client.? 172.21.15.6:0/1813548246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:18.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:17 smithi130 bash[27579]: audit 2023-08-27T22:29:17.432238+0000 mon.a (mon.0) 1301 : audit [DBG] from='client.? 172.21.15.6:0/1813548246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:18.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:18 smithi006 bash[27292]: cluster 2023-08-27T22:29:17.610352+0000 mgr.y (mgr.14901) 923 : cluster [DBG] pgmap v930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:18.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:18 smithi006 bash[33918]: cluster 2023-08-27T22:29:17.610352+0000 mgr.y (mgr.14901) 923 : cluster [DBG] pgmap v930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:18 smithi130 bash[27579]: cluster 2023-08-27T22:29:17.610352+0000 mgr.y (mgr.14901) 923 : cluster [DBG] pgmap v930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:20.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:29:20.208Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:20.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:29:20.210Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:20.971 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:29:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:29:20] "GET /metrics HTTP/1.1" 200 193075 "" "Prometheus/2.33.4" 2023-08-27T22:29:20.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:20 smithi006 bash[33918]: cluster 2023-08-27T22:29:19.611530+0000 mgr.y (mgr.14901) 924 : cluster [DBG] pgmap v931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:20.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:20 smithi006 bash[33918]: audit 2023-08-27T22:29:19.850470+0000 mon.c (mon.2) 387 : audit [DBG] from='client.? 172.21.15.6:0/163392782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:20.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:20 smithi006 bash[27292]: cluster 2023-08-27T22:29:19.611530+0000 mgr.y (mgr.14901) 924 : cluster [DBG] pgmap v931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:20.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:20 smithi006 bash[27292]: audit 2023-08-27T22:29:19.850470+0000 mon.c (mon.2) 387 : audit [DBG] from='client.? 172.21.15.6:0/163392782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:20 smithi130 bash[27579]: cluster 2023-08-27T22:29:19.611530+0000 mgr.y (mgr.14901) 924 : cluster [DBG] pgmap v931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:20 smithi130 bash[27579]: audit 2023-08-27T22:29:19.850470+0000 mon.c (mon.2) 387 : audit [DBG] from='client.? 172.21.15.6:0/163392782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:29:21.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:29:21.773Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:22.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:22 smithi006 bash[33918]: cluster 2023-08-27T22:29:21.612186+0000 mgr.y (mgr.14901) 925 : cluster [DBG] pgmap v932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:22.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:22 smithi006 bash[33918]: audit 2023-08-27T22:29:22.264281+0000 mon.c (mon.2) 388 : audit [DBG] from='client.? 172.21.15.6:0/1143018282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:22.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:22 smithi006 bash[27292]: cluster 2023-08-27T22:29:21.612186+0000 mgr.y (mgr.14901) 925 : cluster [DBG] pgmap v932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:22.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:22 smithi006 bash[27292]: audit 2023-08-27T22:29:22.264281+0000 mon.c (mon.2) 388 : audit [DBG] from='client.? 172.21.15.6:0/1143018282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:23.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:22 smithi130 bash[27579]: cluster 2023-08-27T22:29:21.612186+0000 mgr.y (mgr.14901) 925 : cluster [DBG] pgmap v932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:23.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:22 smithi130 bash[27579]: audit 2023-08-27T22:29:22.264281+0000 mon.c (mon.2) 388 : audit [DBG] from='client.? 172.21.15.6:0/1143018282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:29:23.665Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:29:23.667Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:24.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:24 smithi006 bash[33918]: cluster 2023-08-27T22:29:23.612828+0000 mgr.y (mgr.14901) 926 : cluster [DBG] pgmap v933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:24.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:24 smithi006 bash[27292]: cluster 2023-08-27T22:29:23.612828+0000 mgr.y (mgr.14901) 926 : cluster [DBG] pgmap v933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:24 smithi130 bash[27579]: cluster 2023-08-27T22:29:23.612828+0000 mgr.y (mgr.14901) 926 : cluster [DBG] pgmap v933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:25.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:25 smithi006 bash[27292]: audit 2023-08-27T22:29:24.675981+0000 mon.c (mon.2) 389 : audit [DBG] from='client.? 172.21.15.6:0/1305998084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:25.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:25 smithi006 bash[33918]: audit 2023-08-27T22:29:24.675981+0000 mon.c (mon.2) 389 : audit [DBG] from='client.? 172.21.15.6:0/1305998084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:26.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:25 smithi130 bash[27579]: audit 2023-08-27T22:29:24.675981+0000 mon.c (mon.2) 389 : audit [DBG] from='client.? 172.21.15.6:0/1305998084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:26.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:26 smithi006 bash[27292]: cluster 2023-08-27T22:29:25.613942+0000 mgr.y (mgr.14901) 927 : cluster [DBG] pgmap v934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:26.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:26 smithi006 bash[33918]: cluster 2023-08-27T22:29:25.613942+0000 mgr.y (mgr.14901) 927 : cluster [DBG] pgmap v934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:26 smithi130 bash[27579]: cluster 2023-08-27T22:29:25.613942+0000 mgr.y (mgr.14901) 927 : cluster [DBG] pgmap v934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:27.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:27 smithi006 bash[33918]: audit 2023-08-27T22:29:27.090389+0000 mon.c (mon.2) 390 : audit [DBG] from='client.? 172.21.15.6:0/702617969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:27.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:27 smithi006 bash[27292]: audit 2023-08-27T22:29:27.090389+0000 mon.c (mon.2) 390 : audit [DBG] from='client.? 172.21.15.6:0/702617969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:28.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:27 smithi130 bash[27579]: audit 2023-08-27T22:29:27.090389+0000 mon.c (mon.2) 390 : audit [DBG] from='client.? 172.21.15.6:0/702617969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:28.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:28 smithi006 bash[27292]: cluster 2023-08-27T22:29:27.614647+0000 mgr.y (mgr.14901) 928 : cluster [DBG] pgmap v935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:28.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:28 smithi006 bash[33918]: cluster 2023-08-27T22:29:27.614647+0000 mgr.y (mgr.14901) 928 : cluster [DBG] pgmap v935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:28 smithi130 bash[27579]: cluster 2023-08-27T22:29:27.614647+0000 mgr.y (mgr.14901) 928 : cluster [DBG] pgmap v935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:29.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:29 smithi006 bash[27292]: audit 2023-08-27T22:29:29.506971+0000 mon.c (mon.2) 391 : audit [DBG] from='client.? 172.21.15.6:0/1392474226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:29.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:29 smithi006 bash[33918]: audit 2023-08-27T22:29:29.506971+0000 mon.c (mon.2) 391 : audit [DBG] from='client.? 172.21.15.6:0/1392474226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:30.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:29 smithi130 bash[27579]: audit 2023-08-27T22:29:29.506971+0000 mon.c (mon.2) 391 : audit [DBG] from='client.? 172.21.15.6:0/1392474226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:30.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:29:30.209Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:30.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:29:30.211Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:30.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:29:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:29:30] "GET /metrics HTTP/1.1" 200 193074 "" "Prometheus/2.33.4" 2023-08-27T22:29:30.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:30 smithi006 bash[33918]: cluster 2023-08-27T22:29:29.615934+0000 mgr.y (mgr.14901) 929 : cluster [DBG] pgmap v936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:30.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:30 smithi006 bash[27292]: cluster 2023-08-27T22:29:29.615934+0000 mgr.y (mgr.14901) 929 : cluster [DBG] pgmap v936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:30 smithi130 bash[27579]: cluster 2023-08-27T22:29:29.615934+0000 mgr.y (mgr.14901) 929 : cluster [DBG] pgmap v936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:29:31.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:29:31.773Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:32.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:32 smithi006 bash[27292]: cluster 2023-08-27T22:29:31.616650+0000 mgr.y (mgr.14901) 930 : cluster [DBG] pgmap v937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:32.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:32 smithi006 bash[27292]: audit 2023-08-27T22:29:31.928051+0000 mon.c (mon.2) 392 : audit [DBG] from='client.? 172.21.15.6:0/2950462274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:32.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:32 smithi006 bash[33918]: cluster 2023-08-27T22:29:31.616650+0000 mgr.y (mgr.14901) 930 : cluster [DBG] pgmap v937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:32.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:32 smithi006 bash[33918]: audit 2023-08-27T22:29:31.928051+0000 mon.c (mon.2) 392 : audit [DBG] from='client.? 172.21.15.6:0/2950462274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:32 smithi130 bash[27579]: cluster 2023-08-27T22:29:31.616650+0000 mgr.y (mgr.14901) 930 : cluster [DBG] pgmap v937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:32 smithi130 bash[27579]: audit 2023-08-27T22:29:31.928051+0000 mon.c (mon.2) 392 : audit [DBG] from='client.? 172.21.15.6:0/2950462274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:29:33.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:29:33.669Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:34.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:34 smithi006 bash[27292]: cluster 2023-08-27T22:29:33.617313+0000 mgr.y (mgr.14901) 931 : cluster [DBG] pgmap v938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:34.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:34 smithi006 bash[27292]: audit 2023-08-27T22:29:34.348557+0000 mon.a (mon.0) 1302 : audit [DBG] from='client.? 172.21.15.6:0/274997901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:34.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:34 smithi006 bash[33918]: cluster 2023-08-27T22:29:33.617313+0000 mgr.y (mgr.14901) 931 : cluster [DBG] pgmap v938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:34.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:34 smithi006 bash[33918]: audit 2023-08-27T22:29:34.348557+0000 mon.a (mon.0) 1302 : audit [DBG] from='client.? 172.21.15.6:0/274997901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:34 smithi130 bash[27579]: cluster 2023-08-27T22:29:33.617313+0000 mgr.y (mgr.14901) 931 : cluster [DBG] pgmap v938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:34 smithi130 bash[27579]: audit 2023-08-27T22:29:34.348557+0000 mon.a (mon.0) 1302 : audit [DBG] from='client.? 172.21.15.6:0/274997901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:36.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:36 smithi006 bash[27292]: cluster 2023-08-27T22:29:35.618481+0000 mgr.y (mgr.14901) 932 : cluster [DBG] pgmap v939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:36.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:36 smithi006 bash[33918]: cluster 2023-08-27T22:29:35.618481+0000 mgr.y (mgr.14901) 932 : cluster [DBG] pgmap v939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:36 smithi130 bash[27579]: cluster 2023-08-27T22:29:35.618481+0000 mgr.y (mgr.14901) 932 : cluster [DBG] pgmap v939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:37.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:37 smithi006 bash[27292]: audit 2023-08-27T22:29:36.757993+0000 mon.a (mon.0) 1303 : audit [DBG] from='client.? 172.21.15.6:0/3295519626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:37.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:37 smithi006 bash[33918]: audit 2023-08-27T22:29:36.757993+0000 mon.a (mon.0) 1303 : audit [DBG] from='client.? 172.21.15.6:0/3295519626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:38.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:37 smithi130 bash[27579]: audit 2023-08-27T22:29:36.757993+0000 mon.a (mon.0) 1303 : audit [DBG] from='client.? 172.21.15.6:0/3295519626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:38.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:38 smithi006 bash[33918]: cluster 2023-08-27T22:29:37.619175+0000 mgr.y (mgr.14901) 933 : cluster [DBG] pgmap v940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:38.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:38 smithi006 bash[27292]: cluster 2023-08-27T22:29:37.619175+0000 mgr.y (mgr.14901) 933 : cluster [DBG] pgmap v940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:38 smithi130 bash[27579]: cluster 2023-08-27T22:29:37.619175+0000 mgr.y (mgr.14901) 933 : cluster [DBG] pgmap v940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:39.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:39 smithi006 bash[33918]: audit 2023-08-27T22:29:39.173519+0000 mon.c (mon.2) 393 : audit [DBG] from='client.? 172.21.15.6:0/266319536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:39.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:39 smithi006 bash[27292]: audit 2023-08-27T22:29:39.173519+0000 mon.c (mon.2) 393 : audit [DBG] from='client.? 172.21.15.6:0/266319536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:40.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:39 smithi130 bash[27579]: audit 2023-08-27T22:29:39.173519+0000 mon.c (mon.2) 393 : audit [DBG] from='client.? 172.21.15.6:0/266319536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:40.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:29:40.210Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:40.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:29:40.212Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:40.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:40 smithi006 bash[27292]: cluster 2023-08-27T22:29:39.620323+0000 mgr.y (mgr.14901) 934 : cluster [DBG] pgmap v941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:40.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:29:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:29:40] "GET /metrics HTTP/1.1" 200 193074 "" "Prometheus/2.33.4" 2023-08-27T22:29:40.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:40 smithi006 bash[33918]: cluster 2023-08-27T22:29:39.620323+0000 mgr.y (mgr.14901) 934 : cluster [DBG] pgmap v941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:40 smithi130 bash[27579]: cluster 2023-08-27T22:29:39.620323+0000 mgr.y (mgr.14901) 934 : cluster [DBG] pgmap v941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:41.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:41 smithi006 bash[33918]: audit 2023-08-27T22:29:41.587853+0000 mon.c (mon.2) 394 : audit [DBG] from='client.? 172.21.15.6:0/1489260465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:41.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:41 smithi006 bash[27292]: audit 2023-08-27T22:29:41.587853+0000 mon.c (mon.2) 394 : audit [DBG] from='client.? 172.21.15.6:0/1489260465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:41.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:29:41.773Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:41.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:29:41.774Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:42.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:41 smithi130 bash[27579]: audit 2023-08-27T22:29:41.587853+0000 mon.c (mon.2) 394 : audit [DBG] from='client.? 172.21.15.6:0/1489260465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:42.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:42 smithi006 bash[33918]: cluster 2023-08-27T22:29:41.620994+0000 mgr.y (mgr.14901) 935 : cluster [DBG] pgmap v942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:42.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:42 smithi006 bash[27292]: cluster 2023-08-27T22:29:41.620994+0000 mgr.y (mgr.14901) 935 : cluster [DBG] pgmap v942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:43.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:42 smithi130 bash[27579]: cluster 2023-08-27T22:29:41.620994+0000 mgr.y (mgr.14901) 935 : cluster [DBG] pgmap v942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:29:43.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:29:43.668Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:44.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:44 smithi006 bash[27292]: cluster 2023-08-27T22:29:43.621613+0000 mgr.y (mgr.14901) 936 : cluster [DBG] pgmap v943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:44.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:44 smithi006 bash[27292]: audit 2023-08-27T22:29:44.011998+0000 mon.a (mon.0) 1304 : audit [DBG] from='client.? 172.21.15.6:0/3849083944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:44.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:44 smithi006 bash[33918]: cluster 2023-08-27T22:29:43.621613+0000 mgr.y (mgr.14901) 936 : cluster [DBG] pgmap v943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:44.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:44 smithi006 bash[33918]: audit 2023-08-27T22:29:44.011998+0000 mon.a (mon.0) 1304 : audit [DBG] from='client.? 172.21.15.6:0/3849083944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:44 smithi130 bash[27579]: cluster 2023-08-27T22:29:43.621613+0000 mgr.y (mgr.14901) 936 : cluster [DBG] pgmap v943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:44 smithi130 bash[27579]: audit 2023-08-27T22:29:44.011998+0000 mon.a (mon.0) 1304 : audit [DBG] from='client.? 172.21.15.6:0/3849083944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:46.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:46 smithi006 bash[27292]: cluster 2023-08-27T22:29:45.622739+0000 mgr.y (mgr.14901) 937 : cluster [DBG] pgmap v944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:46.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:46 smithi006 bash[27292]: audit 2023-08-27T22:29:46.252895+0000 mon.b (mon.1) 214 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:29:46.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:46 smithi006 bash[27292]: audit 2023-08-27T22:29:46.427582+0000 mon.a (mon.0) 1305 : audit [DBG] from='client.? 172.21.15.6:0/4236951107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:46.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:46 smithi006 bash[33918]: cluster 2023-08-27T22:29:45.622739+0000 mgr.y (mgr.14901) 937 : cluster [DBG] pgmap v944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:46.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:46 smithi006 bash[33918]: audit 2023-08-27T22:29:46.252895+0000 mon.b (mon.1) 214 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:29:46.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:46 smithi006 bash[33918]: audit 2023-08-27T22:29:46.427582+0000 mon.a (mon.0) 1305 : audit [DBG] from='client.? 172.21.15.6:0/4236951107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:46 smithi130 bash[27579]: cluster 2023-08-27T22:29:45.622739+0000 mgr.y (mgr.14901) 937 : cluster [DBG] pgmap v944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:46 smithi130 bash[27579]: audit 2023-08-27T22:29:46.252895+0000 mon.b (mon.1) 214 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:29:47.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:46 smithi130 bash[27579]: audit 2023-08-27T22:29:46.427582+0000 mon.a (mon.0) 1305 : audit [DBG] from='client.? 172.21.15.6:0/4236951107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:48.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:48 smithi006 bash[33918]: cluster 2023-08-27T22:29:47.623426+0000 mgr.y (mgr.14901) 938 : cluster [DBG] pgmap v945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:48.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:48 smithi006 bash[27292]: cluster 2023-08-27T22:29:47.623426+0000 mgr.y (mgr.14901) 938 : cluster [DBG] pgmap v945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:49.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:48 smithi130 bash[27579]: cluster 2023-08-27T22:29:47.623426+0000 mgr.y (mgr.14901) 938 : cluster [DBG] pgmap v945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:49.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:49 smithi006 bash[27292]: audit 2023-08-27T22:29:48.845623+0000 mon.a (mon.0) 1306 : audit [DBG] from='client.? 172.21.15.6:0/1876613520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:49.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:49 smithi006 bash[33918]: audit 2023-08-27T22:29:48.845623+0000 mon.a (mon.0) 1306 : audit [DBG] from='client.? 172.21.15.6:0/1876613520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:50.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:49 smithi130 bash[27579]: audit 2023-08-27T22:29:48.845623+0000 mon.a (mon.0) 1306 : audit [DBG] from='client.? 172.21.15.6:0/1876613520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:50.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:29:50.211Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:50.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:29:50.213Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:50.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:50 smithi006 bash[27292]: cluster 2023-08-27T22:29:49.624450+0000 mgr.y (mgr.14901) 939 : cluster [DBG] pgmap v946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:50.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:29:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:29:50] "GET /metrics HTTP/1.1" 200 193074 "" "Prometheus/2.33.4" 2023-08-27T22:29:50.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:50 smithi006 bash[33918]: cluster 2023-08-27T22:29:49.624450+0000 mgr.y (mgr.14901) 939 : cluster [DBG] pgmap v946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:50 smithi130 bash[27579]: cluster 2023-08-27T22:29:49.624450+0000 mgr.y (mgr.14901) 939 : cluster [DBG] pgmap v946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:51.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:51 smithi006 bash[27292]: audit 2023-08-27T22:29:51.265505+0000 mon.a (mon.0) 1307 : audit [DBG] from='client.? 172.21.15.6:0/1205248749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:51.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:51 smithi006 bash[33918]: audit 2023-08-27T22:29:51.265505+0000 mon.a (mon.0) 1307 : audit [DBG] from='client.? 172.21.15.6:0/1205248749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:51.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:29:51.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:51.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:29:51.776Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:52.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:51 smithi130 bash[27579]: audit 2023-08-27T22:29:51.265505+0000 mon.a (mon.0) 1307 : audit [DBG] from='client.? 172.21.15.6:0/1205248749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:52.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:52 smithi006 bash[33918]: cluster 2023-08-27T22:29:51.625221+0000 mgr.y (mgr.14901) 940 : cluster [DBG] pgmap v947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:52.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:52 smithi006 bash[33918]: audit 2023-08-27T22:29:52.130489+0000 mon.a (mon.0) 1308 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:29:52.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:52 smithi006 bash[33918]: audit 2023-08-27T22:29:52.586501+0000 mon.a (mon.0) 1309 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:29:52.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:52 smithi006 bash[27292]: cluster 2023-08-27T22:29:51.625221+0000 mgr.y (mgr.14901) 940 : cluster [DBG] pgmap v947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:52.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:52 smithi006 bash[27292]: audit 2023-08-27T22:29:52.130489+0000 mon.a (mon.0) 1308 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:29:52.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:52 smithi006 bash[27292]: audit 2023-08-27T22:29:52.586501+0000 mon.a (mon.0) 1309 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:29:53.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:52 smithi130 bash[27579]: cluster 2023-08-27T22:29:51.625221+0000 mgr.y (mgr.14901) 940 : cluster [DBG] pgmap v947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:53.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:52 smithi130 bash[27579]: audit 2023-08-27T22:29:52.130489+0000 mon.a (mon.0) 1308 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:29:53.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:52 smithi130 bash[27579]: audit 2023-08-27T22:29:52.586501+0000 mon.a (mon.0) 1309 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:29:53.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:53 smithi006 bash[27292]: audit 2023-08-27T22:29:52.865704+0000 mon.b (mon.1) 215 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:29:53.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:53 smithi006 bash[27292]: audit 2023-08-27T22:29:52.867888+0000 mon.b (mon.1) 216 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:29:53.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:53 smithi006 bash[27292]: audit 2023-08-27T22:29:52.872487+0000 mon.a (mon.0) 1310 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:29:53.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:53 smithi006 bash[27292]: audit 2023-08-27T22:29:53.682570+0000 mon.a (mon.0) 1311 : audit [DBG] from='client.? 172.21.15.6:0/1923688027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:53.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:29:53.668Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:53.973 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:29:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:29:53.669Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:29:53.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:53 smithi006 bash[33918]: audit 2023-08-27T22:29:52.865704+0000 mon.b (mon.1) 215 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:29:53.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:53 smithi006 bash[33918]: audit 2023-08-27T22:29:52.867888+0000 mon.b (mon.1) 216 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:29:53.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:53 smithi006 bash[33918]: audit 2023-08-27T22:29:52.872487+0000 mon.a (mon.0) 1310 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:29:53.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:53 smithi006 bash[33918]: audit 2023-08-27T22:29:53.682570+0000 mon.a (mon.0) 1311 : audit [DBG] from='client.? 172.21.15.6:0/1923688027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:54.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:53 smithi130 bash[27579]: audit 2023-08-27T22:29:52.865704+0000 mon.b (mon.1) 215 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:29:54.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:53 smithi130 bash[27579]: audit 2023-08-27T22:29:52.867888+0000 mon.b (mon.1) 216 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:29:54.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:53 smithi130 bash[27579]: audit 2023-08-27T22:29:52.872487+0000 mon.a (mon.0) 1310 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:29:54.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:53 smithi130 bash[27579]: audit 2023-08-27T22:29:53.682570+0000 mon.a (mon.0) 1311 : audit [DBG] from='client.? 172.21.15.6:0/1923688027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:54 smithi130 bash[27579]: cluster 2023-08-27T22:29:53.625837+0000 mgr.y (mgr.14901) 941 : cluster [DBG] pgmap v948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:55.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:54 smithi006 bash[27292]: cluster 2023-08-27T22:29:53.625837+0000 mgr.y (mgr.14901) 941 : cluster [DBG] pgmap v948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:55.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:54 smithi006 bash[33918]: cluster 2023-08-27T22:29:53.625837+0000 mgr.y (mgr.14901) 941 : cluster [DBG] pgmap v948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:56 smithi130 bash[27579]: cluster 2023-08-27T22:29:55.626959+0000 mgr.y (mgr.14901) 942 : cluster [DBG] pgmap v949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:56 smithi130 bash[27579]: audit 2023-08-27T22:29:56.100517+0000 mon.c (mon.2) 395 : audit [DBG] from='client.? 172.21.15.6:0/2964000597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:57.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:56 smithi006 bash[27292]: cluster 2023-08-27T22:29:55.626959+0000 mgr.y (mgr.14901) 942 : cluster [DBG] pgmap v949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:57.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:56 smithi006 bash[27292]: audit 2023-08-27T22:29:56.100517+0000 mon.c (mon.2) 395 : audit [DBG] from='client.? 172.21.15.6:0/2964000597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:57.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:56 smithi006 bash[33918]: cluster 2023-08-27T22:29:55.626959+0000 mgr.y (mgr.14901) 942 : cluster [DBG] pgmap v949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:57.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:56 smithi006 bash[33918]: audit 2023-08-27T22:29:56.100517+0000 mon.c (mon.2) 395 : audit [DBG] from='client.? 172.21.15.6:0/2964000597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:58 smithi130 bash[27579]: cluster 2023-08-27T22:29:57.627655+0000 mgr.y (mgr.14901) 943 : cluster [DBG] pgmap v950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:29:58 smithi130 bash[27579]: audit 2023-08-27T22:29:58.513654+0000 mon.a (mon.0) 1312 : audit [DBG] from='client.? 172.21.15.6:0/2146734942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:59.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:58 smithi006 bash[33918]: cluster 2023-08-27T22:29:57.627655+0000 mgr.y (mgr.14901) 943 : cluster [DBG] pgmap v950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:59.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:29:58 smithi006 bash[33918]: audit 2023-08-27T22:29:58.513654+0000 mon.a (mon.0) 1312 : audit [DBG] from='client.? 172.21.15.6:0/2146734942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:59.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:58 smithi006 bash[27292]: cluster 2023-08-27T22:29:57.627655+0000 mgr.y (mgr.14901) 943 : cluster [DBG] pgmap v950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:29:59.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:29:58 smithi006 bash[27292]: audit 2023-08-27T22:29:58.513654+0000 mon.a (mon.0) 1312 : audit [DBG] from='client.? 172.21.15.6:0/2146734942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:00.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:30:00.211Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:00.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:30:00.213Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:00 smithi130 bash[27579]: cluster 2023-08-27T22:29:59.628897+0000 mgr.y (mgr.14901) 944 : cluster [DBG] pgmap v951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:00 smithi130 bash[27579]: cluster 2023-08-27T22:30:00.000153+0000 mon.a (mon.0) 1313 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:30:01.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:30:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:30:00] "GET /metrics HTTP/1.1" 200 193075 "" "Prometheus/2.33.4" 2023-08-27T22:30:01.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:00 smithi006 bash[27292]: cluster 2023-08-27T22:29:59.628897+0000 mgr.y (mgr.14901) 944 : cluster [DBG] pgmap v951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:01.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:00 smithi006 bash[27292]: cluster 2023-08-27T22:30:00.000153+0000 mon.a (mon.0) 1313 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:30:01.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:00 smithi006 bash[33918]: cluster 2023-08-27T22:29:59.628897+0000 mgr.y (mgr.14901) 944 : cluster [DBG] pgmap v951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:01.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:00 smithi006 bash[33918]: cluster 2023-08-27T22:30:00.000153+0000 mon.a (mon.0) 1313 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:30:02.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:01 smithi130 bash[27579]: audit 2023-08-27T22:30:00.931195+0000 mon.a (mon.0) 1314 : audit [DBG] from='client.? 172.21.15.6:0/198791441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:02.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:01 smithi006 bash[27292]: audit 2023-08-27T22:30:00.931195+0000 mon.a (mon.0) 1314 : audit [DBG] from='client.? 172.21.15.6:0/198791441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:02.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:01 smithi006 bash[33918]: audit 2023-08-27T22:30:00.931195+0000 mon.a (mon.0) 1314 : audit [DBG] from='client.? 172.21.15.6:0/198791441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:30:01.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:30:01.776Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:02 smithi130 bash[27579]: cluster 2023-08-27T22:30:01.629569+0000 mgr.y (mgr.14901) 945 : cluster [DBG] pgmap v952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:03.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:02 smithi006 bash[27292]: cluster 2023-08-27T22:30:01.629569+0000 mgr.y (mgr.14901) 945 : cluster [DBG] pgmap v952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:03.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:02 smithi006 bash[33918]: cluster 2023-08-27T22:30:01.629569+0000 mgr.y (mgr.14901) 945 : cluster [DBG] pgmap v952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:03.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:03 smithi006 bash[27292]: audit 2023-08-27T22:30:03.344455+0000 mon.a (mon.0) 1315 : audit [DBG] from='client.? 172.21.15.6:0/1422869474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:03.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:03 smithi006 bash[33918]: audit 2023-08-27T22:30:03.344455+0000 mon.a (mon.0) 1315 : audit [DBG] from='client.? 172.21.15.6:0/1422869474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:30:03.669Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:30:03.671Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:04.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:03 smithi130 bash[27579]: audit 2023-08-27T22:30:03.344455+0000 mon.a (mon.0) 1315 : audit [DBG] from='client.? 172.21.15.6:0/1422869474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:04 smithi130 bash[27579]: cluster 2023-08-27T22:30:03.630251+0000 mgr.y (mgr.14901) 946 : cluster [DBG] pgmap v953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:05.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:04 smithi006 bash[27292]: cluster 2023-08-27T22:30:03.630251+0000 mgr.y (mgr.14901) 946 : cluster [DBG] pgmap v953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:05.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:04 smithi006 bash[33918]: cluster 2023-08-27T22:30:03.630251+0000 mgr.y (mgr.14901) 946 : cluster [DBG] pgmap v953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:06 smithi130 bash[27579]: cluster 2023-08-27T22:30:05.631281+0000 mgr.y (mgr.14901) 947 : cluster [DBG] pgmap v954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:06 smithi130 bash[27579]: audit 2023-08-27T22:30:05.756275+0000 mon.a (mon.0) 1316 : audit [DBG] from='client.? 172.21.15.6:0/2439471989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:07.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:06 smithi006 bash[27292]: cluster 2023-08-27T22:30:05.631281+0000 mgr.y (mgr.14901) 947 : cluster [DBG] pgmap v954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:07.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:06 smithi006 bash[27292]: audit 2023-08-27T22:30:05.756275+0000 mon.a (mon.0) 1316 : audit [DBG] from='client.? 172.21.15.6:0/2439471989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:07.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:06 smithi006 bash[33918]: cluster 2023-08-27T22:30:05.631281+0000 mgr.y (mgr.14901) 947 : cluster [DBG] pgmap v954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:07.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:06 smithi006 bash[33918]: audit 2023-08-27T22:30:05.756275+0000 mon.a (mon.0) 1316 : audit [DBG] from='client.? 172.21.15.6:0/2439471989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:08 smithi130 bash[27579]: cluster 2023-08-27T22:30:07.631963+0000 mgr.y (mgr.14901) 948 : cluster [DBG] pgmap v955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:08 smithi130 bash[27579]: audit 2023-08-27T22:30:08.175073+0000 mon.c (mon.2) 396 : audit [DBG] from='client.? 172.21.15.6:0/2115584113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:08 smithi006 bash[33918]: cluster 2023-08-27T22:30:07.631963+0000 mgr.y (mgr.14901) 948 : cluster [DBG] pgmap v955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:08 smithi006 bash[33918]: audit 2023-08-27T22:30:08.175073+0000 mon.c (mon.2) 396 : audit [DBG] from='client.? 172.21.15.6:0/2115584113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:09.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:08 smithi006 bash[27292]: cluster 2023-08-27T22:30:07.631963+0000 mgr.y (mgr.14901) 948 : cluster [DBG] pgmap v955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:09.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:08 smithi006 bash[27292]: audit 2023-08-27T22:30:08.175073+0000 mon.c (mon.2) 396 : audit [DBG] from='client.? 172.21.15.6:0/2115584113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:10.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:30:10.211Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:10.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:30:10.213Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:10 smithi130 bash[27579]: cluster 2023-08-27T22:30:09.633109+0000 mgr.y (mgr.14901) 949 : cluster [DBG] pgmap v956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:10 smithi130 bash[27579]: audit 2023-08-27T22:30:10.593766+0000 mon.c (mon.2) 397 : audit [DBG] from='client.? 172.21.15.6:0/2422661880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:11.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:30:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:30:10] "GET /metrics HTTP/1.1" 200 193075 "" "Prometheus/2.33.4" 2023-08-27T22:30:11.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:10 smithi006 bash[27292]: cluster 2023-08-27T22:30:09.633109+0000 mgr.y (mgr.14901) 949 : cluster [DBG] pgmap v956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:11.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:10 smithi006 bash[27292]: audit 2023-08-27T22:30:10.593766+0000 mon.c (mon.2) 397 : audit [DBG] from='client.? 172.21.15.6:0/2422661880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:11.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:10 smithi006 bash[33918]: cluster 2023-08-27T22:30:09.633109+0000 mgr.y (mgr.14901) 949 : cluster [DBG] pgmap v956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:11.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:10 smithi006 bash[33918]: audit 2023-08-27T22:30:10.593766+0000 mon.c (mon.2) 397 : audit [DBG] from='client.? 172.21.15.6:0/2422661880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:30:11.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:30:11.776Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:13.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:12 smithi130 bash[27579]: cluster 2023-08-27T22:30:11.633762+0000 mgr.y (mgr.14901) 950 : cluster [DBG] pgmap v957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:13.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:12 smithi006 bash[33918]: cluster 2023-08-27T22:30:11.633762+0000 mgr.y (mgr.14901) 950 : cluster [DBG] pgmap v957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:13.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:12 smithi006 bash[27292]: cluster 2023-08-27T22:30:11.633762+0000 mgr.y (mgr.14901) 950 : cluster [DBG] pgmap v957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:13.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:13 smithi006 bash[33918]: audit 2023-08-27T22:30:13.006687+0000 mon.a (mon.0) 1317 : audit [DBG] from='client.? 172.21.15.6:0/3283403376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:13.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:13 smithi006 bash[27292]: audit 2023-08-27T22:30:13.006687+0000 mon.a (mon.0) 1317 : audit [DBG] from='client.? 172.21.15.6:0/3283403376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:30:13.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:30:13.672Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:14.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:13 smithi130 bash[27579]: audit 2023-08-27T22:30:13.006687+0000 mon.a (mon.0) 1317 : audit [DBG] from='client.? 172.21.15.6:0/3283403376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:14 smithi130 bash[27579]: cluster 2023-08-27T22:30:13.634331+0000 mgr.y (mgr.14901) 951 : cluster [DBG] pgmap v958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:15.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:14 smithi006 bash[27292]: cluster 2023-08-27T22:30:13.634331+0000 mgr.y (mgr.14901) 951 : cluster [DBG] pgmap v958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:15.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:14 smithi006 bash[33918]: cluster 2023-08-27T22:30:13.634331+0000 mgr.y (mgr.14901) 951 : cluster [DBG] pgmap v958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:16.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:15 smithi130 bash[27579]: audit 2023-08-27T22:30:15.424705+0000 mon.a (mon.0) 1318 : audit [DBG] from='client.? 172.21.15.6:0/2132158716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:16.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:15 smithi006 bash[33918]: audit 2023-08-27T22:30:15.424705+0000 mon.a (mon.0) 1318 : audit [DBG] from='client.? 172.21.15.6:0/2132158716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:16.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:15 smithi006 bash[27292]: audit 2023-08-27T22:30:15.424705+0000 mon.a (mon.0) 1318 : audit [DBG] from='client.? 172.21.15.6:0/2132158716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:16 smithi130 bash[27579]: cluster 2023-08-27T22:30:15.635482+0000 mgr.y (mgr.14901) 952 : cluster [DBG] pgmap v959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:17.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:16 smithi006 bash[33918]: cluster 2023-08-27T22:30:15.635482+0000 mgr.y (mgr.14901) 952 : cluster [DBG] pgmap v959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:17.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:16 smithi006 bash[27292]: cluster 2023-08-27T22:30:15.635482+0000 mgr.y (mgr.14901) 952 : cluster [DBG] pgmap v959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:18 smithi130 bash[27579]: cluster 2023-08-27T22:30:17.636190+0000 mgr.y (mgr.14901) 953 : cluster [DBG] pgmap v960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:18 smithi130 bash[27579]: audit 2023-08-27T22:30:17.836706+0000 mon.a (mon.0) 1319 : audit [DBG] from='client.? 172.21.15.6:0/2709807847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:19.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:18 smithi006 bash[33918]: cluster 2023-08-27T22:30:17.636190+0000 mgr.y (mgr.14901) 953 : cluster [DBG] pgmap v960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:19.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:18 smithi006 bash[33918]: audit 2023-08-27T22:30:17.836706+0000 mon.a (mon.0) 1319 : audit [DBG] from='client.? 172.21.15.6:0/2709807847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:19.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:18 smithi006 bash[27292]: cluster 2023-08-27T22:30:17.636190+0000 mgr.y (mgr.14901) 953 : cluster [DBG] pgmap v960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:19.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:18 smithi006 bash[27292]: audit 2023-08-27T22:30:17.836706+0000 mon.a (mon.0) 1319 : audit [DBG] from='client.? 172.21.15.6:0/2709807847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:20.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:30:20.212Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:20.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:30:20.214Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:20 smithi130 bash[27579]: cluster 2023-08-27T22:30:19.637335+0000 mgr.y (mgr.14901) 954 : cluster [DBG] pgmap v961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:20 smithi130 bash[27579]: audit 2023-08-27T22:30:20.248199+0000 mon.a (mon.0) 1320 : audit [DBG] from='client.? 172.21.15.6:0/3864107465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:21.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:30:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:30:20] "GET /metrics HTTP/1.1" 200 193075 "" "Prometheus/2.33.4" 2023-08-27T22:30:21.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:20 smithi006 bash[27292]: cluster 2023-08-27T22:30:19.637335+0000 mgr.y (mgr.14901) 954 : cluster [DBG] pgmap v961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:21.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:20 smithi006 bash[27292]: audit 2023-08-27T22:30:20.248199+0000 mon.a (mon.0) 1320 : audit [DBG] from='client.? 172.21.15.6:0/3864107465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:21.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:20 smithi006 bash[33918]: cluster 2023-08-27T22:30:19.637335+0000 mgr.y (mgr.14901) 954 : cluster [DBG] pgmap v961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:21.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:20 smithi006 bash[33918]: audit 2023-08-27T22:30:20.248199+0000 mon.a (mon.0) 1320 : audit [DBG] from='client.? 172.21.15.6:0/3864107465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:30:21.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:30:21.777Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:23.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:22 smithi130 bash[27579]: cluster 2023-08-27T22:30:21.638019+0000 mgr.y (mgr.14901) 955 : cluster [DBG] pgmap v962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:23.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:22 smithi130 bash[27579]: audit 2023-08-27T22:30:22.662546+0000 mon.a (mon.0) 1321 : audit [DBG] from='client.? 172.21.15.6:0/3975065280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:23.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:22 smithi006 bash[27292]: cluster 2023-08-27T22:30:21.638019+0000 mgr.y (mgr.14901) 955 : cluster [DBG] pgmap v962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:23.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:22 smithi006 bash[27292]: audit 2023-08-27T22:30:22.662546+0000 mon.a (mon.0) 1321 : audit [DBG] from='client.? 172.21.15.6:0/3975065280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:23.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:22 smithi006 bash[33918]: cluster 2023-08-27T22:30:21.638019+0000 mgr.y (mgr.14901) 955 : cluster [DBG] pgmap v962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:23.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:22 smithi006 bash[33918]: audit 2023-08-27T22:30:22.662546+0000 mon.a (mon.0) 1321 : audit [DBG] from='client.? 172.21.15.6:0/3975065280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:30:23.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:30:23.672Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:24.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:24 smithi006 bash[33918]: cluster 2023-08-27T22:30:23.638724+0000 mgr.y (mgr.14901) 956 : cluster [DBG] pgmap v963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:24.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:24 smithi006 bash[27292]: cluster 2023-08-27T22:30:23.638724+0000 mgr.y (mgr.14901) 956 : cluster [DBG] pgmap v963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:24 smithi130 bash[27579]: cluster 2023-08-27T22:30:23.638724+0000 mgr.y (mgr.14901) 956 : cluster [DBG] pgmap v963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:25.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:25 smithi006 bash[27292]: audit 2023-08-27T22:30:25.084446+0000 mon.a (mon.0) 1322 : audit [DBG] from='client.? 172.21.15.6:0/2084507441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:25.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:25 smithi006 bash[33918]: audit 2023-08-27T22:30:25.084446+0000 mon.a (mon.0) 1322 : audit [DBG] from='client.? 172.21.15.6:0/2084507441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:26.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:25 smithi130 bash[27579]: audit 2023-08-27T22:30:25.084446+0000 mon.a (mon.0) 1322 : audit [DBG] from='client.? 172.21.15.6:0/2084507441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:26.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:26 smithi006 bash[27292]: cluster 2023-08-27T22:30:25.640177+0000 mgr.y (mgr.14901) 957 : cluster [DBG] pgmap v964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:26.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:26 smithi006 bash[33918]: cluster 2023-08-27T22:30:25.640177+0000 mgr.y (mgr.14901) 957 : cluster [DBG] pgmap v964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:26 smithi130 bash[27579]: cluster 2023-08-27T22:30:25.640177+0000 mgr.y (mgr.14901) 957 : cluster [DBG] pgmap v964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:27.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:27 smithi006 bash[27292]: audit 2023-08-27T22:30:27.498271+0000 mon.c (mon.2) 398 : audit [DBG] from='client.? 172.21.15.6:0/851649048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:27.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:27 smithi006 bash[33918]: audit 2023-08-27T22:30:27.498271+0000 mon.c (mon.2) 398 : audit [DBG] from='client.? 172.21.15.6:0/851649048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:28.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:27 smithi130 bash[27579]: audit 2023-08-27T22:30:27.498271+0000 mon.c (mon.2) 398 : audit [DBG] from='client.? 172.21.15.6:0/851649048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:28.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:28 smithi006 bash[33918]: cluster 2023-08-27T22:30:27.640906+0000 mgr.y (mgr.14901) 958 : cluster [DBG] pgmap v965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:28.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:28 smithi006 bash[27292]: cluster 2023-08-27T22:30:27.640906+0000 mgr.y (mgr.14901) 958 : cluster [DBG] pgmap v965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:28 smithi130 bash[27579]: cluster 2023-08-27T22:30:27.640906+0000 mgr.y (mgr.14901) 958 : cluster [DBG] pgmap v965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:30.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:30:30.213Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:30.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:30:30.215Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:30.971 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:30:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:30:30] "GET /metrics HTTP/1.1" 200 193071 "" "Prometheus/2.33.4" 2023-08-27T22:30:30.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:30 smithi006 bash[27292]: cluster 2023-08-27T22:30:29.641852+0000 mgr.y (mgr.14901) 959 : cluster [DBG] pgmap v966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:30.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:30 smithi006 bash[27292]: audit 2023-08-27T22:30:29.913689+0000 mon.a (mon.0) 1323 : audit [DBG] from='client.? 172.21.15.6:0/804675007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:30.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:30 smithi006 bash[33918]: cluster 2023-08-27T22:30:29.641852+0000 mgr.y (mgr.14901) 959 : cluster [DBG] pgmap v966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:30.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:30 smithi006 bash[33918]: audit 2023-08-27T22:30:29.913689+0000 mon.a (mon.0) 1323 : audit [DBG] from='client.? 172.21.15.6:0/804675007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:30 smithi130 bash[27579]: cluster 2023-08-27T22:30:29.641852+0000 mgr.y (mgr.14901) 959 : cluster [DBG] pgmap v966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:30 smithi130 bash[27579]: audit 2023-08-27T22:30:29.913689+0000 mon.a (mon.0) 1323 : audit [DBG] from='client.? 172.21.15.6:0/804675007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:30:31.776Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:30:31.777Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:32.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:32 smithi006 bash[33918]: cluster 2023-08-27T22:30:31.642620+0000 mgr.y (mgr.14901) 960 : cluster [DBG] pgmap v967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:32.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:32 smithi006 bash[33918]: audit 2023-08-27T22:30:32.330405+0000 mon.c (mon.2) 399 : audit [DBG] from='client.? 172.21.15.6:0/3755620122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:32.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:32 smithi006 bash[27292]: cluster 2023-08-27T22:30:31.642620+0000 mgr.y (mgr.14901) 960 : cluster [DBG] pgmap v967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:32.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:32 smithi006 bash[27292]: audit 2023-08-27T22:30:32.330405+0000 mon.c (mon.2) 399 : audit [DBG] from='client.? 172.21.15.6:0/3755620122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:32 smithi130 bash[27579]: cluster 2023-08-27T22:30:31.642620+0000 mgr.y (mgr.14901) 960 : cluster [DBG] pgmap v967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:32 smithi130 bash[27579]: audit 2023-08-27T22:30:32.330405+0000 mon.c (mon.2) 399 : audit [DBG] from='client.? 172.21.15.6:0/3755620122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:30:33.671Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:30:33.673Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:34.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:34 smithi006 bash[33918]: cluster 2023-08-27T22:30:33.643293+0000 mgr.y (mgr.14901) 961 : cluster [DBG] pgmap v968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:34.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:34 smithi006 bash[27292]: cluster 2023-08-27T22:30:33.643293+0000 mgr.y (mgr.14901) 961 : cluster [DBG] pgmap v968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:34 smithi130 bash[27579]: cluster 2023-08-27T22:30:33.643293+0000 mgr.y (mgr.14901) 961 : cluster [DBG] pgmap v968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:35.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:35 smithi006 bash[27292]: audit 2023-08-27T22:30:34.750190+0000 mon.c (mon.2) 400 : audit [DBG] from='client.? 172.21.15.6:0/1738534529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:35.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:35 smithi006 bash[33918]: audit 2023-08-27T22:30:34.750190+0000 mon.c (mon.2) 400 : audit [DBG] from='client.? 172.21.15.6:0/1738534529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:36.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:35 smithi130 bash[27579]: audit 2023-08-27T22:30:34.750190+0000 mon.c (mon.2) 400 : audit [DBG] from='client.? 172.21.15.6:0/1738534529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:36.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:36 smithi006 bash[27292]: cluster 2023-08-27T22:30:35.644424+0000 mgr.y (mgr.14901) 962 : cluster [DBG] pgmap v969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:36.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:36 smithi006 bash[33918]: cluster 2023-08-27T22:30:35.644424+0000 mgr.y (mgr.14901) 962 : cluster [DBG] pgmap v969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:36 smithi130 bash[27579]: cluster 2023-08-27T22:30:35.644424+0000 mgr.y (mgr.14901) 962 : cluster [DBG] pgmap v969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:37.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:37 smithi006 bash[27292]: audit 2023-08-27T22:30:37.173920+0000 mon.a (mon.0) 1324 : audit [DBG] from='client.? 172.21.15.6:0/3957995208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:37.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:37 smithi006 bash[33918]: audit 2023-08-27T22:30:37.173920+0000 mon.a (mon.0) 1324 : audit [DBG] from='client.? 172.21.15.6:0/3957995208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:38.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:37 smithi130 bash[27579]: audit 2023-08-27T22:30:37.173920+0000 mon.a (mon.0) 1324 : audit [DBG] from='client.? 172.21.15.6:0/3957995208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:38.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:38 smithi006 bash[27292]: cluster 2023-08-27T22:30:37.645140+0000 mgr.y (mgr.14901) 963 : cluster [DBG] pgmap v970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:38.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:38 smithi006 bash[33918]: cluster 2023-08-27T22:30:37.645140+0000 mgr.y (mgr.14901) 963 : cluster [DBG] pgmap v970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:38 smithi130 bash[27579]: cluster 2023-08-27T22:30:37.645140+0000 mgr.y (mgr.14901) 963 : cluster [DBG] pgmap v970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:39.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:39 smithi006 bash[27292]: audit 2023-08-27T22:30:39.585943+0000 mon.c (mon.2) 401 : audit [DBG] from='client.? 172.21.15.6:0/957076603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:39.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:39 smithi006 bash[33918]: audit 2023-08-27T22:30:39.585943+0000 mon.c (mon.2) 401 : audit [DBG] from='client.? 172.21.15.6:0/957076603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:40.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:39 smithi130 bash[27579]: audit 2023-08-27T22:30:39.585943+0000 mon.c (mon.2) 401 : audit [DBG] from='client.? 172.21.15.6:0/957076603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:40.684 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:30:40.215Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:40.685 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:30:40.216Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:40.971 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:30:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:30:40] "GET /metrics HTTP/1.1" 200 193071 "" "Prometheus/2.33.4" 2023-08-27T22:30:40.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:40 smithi006 bash[33918]: cluster 2023-08-27T22:30:39.646232+0000 mgr.y (mgr.14901) 964 : cluster [DBG] pgmap v971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:40.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:40 smithi006 bash[27292]: cluster 2023-08-27T22:30:39.646232+0000 mgr.y (mgr.14901) 964 : cluster [DBG] pgmap v971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:40 smithi130 bash[27579]: cluster 2023-08-27T22:30:39.646232+0000 mgr.y (mgr.14901) 964 : cluster [DBG] pgmap v971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:30:41.777Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:30:41.779Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:42.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:42 smithi006 bash[33918]: cluster 2023-08-27T22:30:41.646917+0000 mgr.y (mgr.14901) 965 : cluster [DBG] pgmap v972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:42.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:42 smithi006 bash[33918]: audit 2023-08-27T22:30:41.993771+0000 mon.a (mon.0) 1325 : audit [DBG] from='client.? 172.21.15.6:0/102372153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:42.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:42 smithi006 bash[27292]: cluster 2023-08-27T22:30:41.646917+0000 mgr.y (mgr.14901) 965 : cluster [DBG] pgmap v972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:42.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:42 smithi006 bash[27292]: audit 2023-08-27T22:30:41.993771+0000 mon.a (mon.0) 1325 : audit [DBG] from='client.? 172.21.15.6:0/102372153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:43.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:42 smithi130 bash[27579]: cluster 2023-08-27T22:30:41.646917+0000 mgr.y (mgr.14901) 965 : cluster [DBG] pgmap v972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:43.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:42 smithi130 bash[27579]: audit 2023-08-27T22:30:41.993771+0000 mon.a (mon.0) 1325 : audit [DBG] from='client.? 172.21.15.6:0/102372153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:30:43.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:30:43.674Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:44.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:44 smithi006 bash[27292]: cluster 2023-08-27T22:30:43.647472+0000 mgr.y (mgr.14901) 966 : cluster [DBG] pgmap v973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:44.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:44 smithi006 bash[27292]: audit 2023-08-27T22:30:44.411801+0000 mon.a (mon.0) 1326 : audit [DBG] from='client.? 172.21.15.6:0/1826973849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:44.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:44 smithi006 bash[33918]: cluster 2023-08-27T22:30:43.647472+0000 mgr.y (mgr.14901) 966 : cluster [DBG] pgmap v973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:44.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:44 smithi006 bash[33918]: audit 2023-08-27T22:30:44.411801+0000 mon.a (mon.0) 1326 : audit [DBG] from='client.? 172.21.15.6:0/1826973849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:44 smithi130 bash[27579]: cluster 2023-08-27T22:30:43.647472+0000 mgr.y (mgr.14901) 966 : cluster [DBG] pgmap v973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:44 smithi130 bash[27579]: audit 2023-08-27T22:30:44.411801+0000 mon.a (mon.0) 1326 : audit [DBG] from='client.? 172.21.15.6:0/1826973849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:46.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:46 smithi006 bash[33918]: cluster 2023-08-27T22:30:45.648604+0000 mgr.y (mgr.14901) 967 : cluster [DBG] pgmap v974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:46.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:46 smithi006 bash[27292]: cluster 2023-08-27T22:30:45.648604+0000 mgr.y (mgr.14901) 967 : cluster [DBG] pgmap v974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:46 smithi130 bash[27579]: cluster 2023-08-27T22:30:45.648604+0000 mgr.y (mgr.14901) 967 : cluster [DBG] pgmap v974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:47.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:47 smithi006 bash[27292]: audit 2023-08-27T22:30:46.829967+0000 mon.c (mon.2) 402 : audit [DBG] from='client.? 172.21.15.6:0/385367922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:47.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:47 smithi006 bash[33918]: audit 2023-08-27T22:30:46.829967+0000 mon.c (mon.2) 402 : audit [DBG] from='client.? 172.21.15.6:0/385367922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:48.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:47 smithi130 bash[27579]: audit 2023-08-27T22:30:46.829967+0000 mon.c (mon.2) 402 : audit [DBG] from='client.? 172.21.15.6:0/385367922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:49.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:48 smithi130 bash[27579]: cluster 2023-08-27T22:30:47.649285+0000 mgr.y (mgr.14901) 968 : cluster [DBG] pgmap v975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:49.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:48 smithi006 bash[27292]: cluster 2023-08-27T22:30:47.649285+0000 mgr.y (mgr.14901) 968 : cluster [DBG] pgmap v975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:49.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:48 smithi006 bash[33918]: cluster 2023-08-27T22:30:47.649285+0000 mgr.y (mgr.14901) 968 : cluster [DBG] pgmap v975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:50.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:49 smithi130 bash[27579]: audit 2023-08-27T22:30:49.244578+0000 mon.c (mon.2) 403 : audit [DBG] from='client.? 172.21.15.6:0/2821165317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:50.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:49 smithi006 bash[33918]: audit 2023-08-27T22:30:49.244578+0000 mon.c (mon.2) 403 : audit [DBG] from='client.? 172.21.15.6:0/2821165317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:50.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:49 smithi006 bash[27292]: audit 2023-08-27T22:30:49.244578+0000 mon.c (mon.2) 403 : audit [DBG] from='client.? 172.21.15.6:0/2821165317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:50.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:30:50.216Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:50.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:30:50.218Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:50 smithi130 bash[27579]: cluster 2023-08-27T22:30:49.650529+0000 mgr.y (mgr.14901) 969 : cluster [DBG] pgmap v976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:51.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:30:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:30:50] "GET /metrics HTTP/1.1" 200 193078 "" "Prometheus/2.33.4" 2023-08-27T22:30:51.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:50 smithi006 bash[33918]: cluster 2023-08-27T22:30:49.650529+0000 mgr.y (mgr.14901) 969 : cluster [DBG] pgmap v976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:51.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:50 smithi006 bash[27292]: cluster 2023-08-27T22:30:49.650529+0000 mgr.y (mgr.14901) 969 : cluster [DBG] pgmap v976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:52.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:51 smithi130 bash[27579]: audit 2023-08-27T22:30:51.652230+0000 mon.a (mon.0) 1327 : audit [DBG] from='client.? 172.21.15.6:0/2124621980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:52.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:51 smithi006 bash[27292]: audit 2023-08-27T22:30:51.652230+0000 mon.a (mon.0) 1327 : audit [DBG] from='client.? 172.21.15.6:0/2124621980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:30:51.777Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:30:51.779Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:52.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:51 smithi006 bash[33918]: audit 2023-08-27T22:30:51.652230+0000 mon.a (mon.0) 1327 : audit [DBG] from='client.? 172.21.15.6:0/2124621980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:53.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:52 smithi130 bash[27579]: cluster 2023-08-27T22:30:51.651265+0000 mgr.y (mgr.14901) 970 : cluster [DBG] pgmap v977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:53.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:52 smithi006 bash[33918]: cluster 2023-08-27T22:30:51.651265+0000 mgr.y (mgr.14901) 970 : cluster [DBG] pgmap v977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:53.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:52 smithi006 bash[27292]: cluster 2023-08-27T22:30:51.651265+0000 mgr.y (mgr.14901) 970 : cluster [DBG] pgmap v977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:53.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:53 smithi006 bash[27292]: audit 2023-08-27T22:30:52.887933+0000 mon.b (mon.1) 217 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:30:53.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:53 smithi006 bash[33918]: audit 2023-08-27T22:30:52.887933+0000 mon.b (mon.1) 217 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:30:53.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:30:53.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:53.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:30:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:30:53.675Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:30:54.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:53 smithi130 bash[27579]: audit 2023-08-27T22:30:52.887933+0000 mon.b (mon.1) 217 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:30:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:54 smithi130 bash[27579]: cluster 2023-08-27T22:30:53.651959+0000 mgr.y (mgr.14901) 971 : cluster [DBG] pgmap v978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:54 smithi130 bash[27579]: audit 2023-08-27T22:30:54.069964+0000 mon.a (mon.0) 1328 : audit [DBG] from='client.? 172.21.15.6:0/3150018710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:55.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:54 smithi006 bash[33918]: cluster 2023-08-27T22:30:53.651959+0000 mgr.y (mgr.14901) 971 : cluster [DBG] pgmap v978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:55.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:54 smithi006 bash[33918]: audit 2023-08-27T22:30:54.069964+0000 mon.a (mon.0) 1328 : audit [DBG] from='client.? 172.21.15.6:0/3150018710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:55.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:54 smithi006 bash[27292]: cluster 2023-08-27T22:30:53.651959+0000 mgr.y (mgr.14901) 971 : cluster [DBG] pgmap v978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:55.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:54 smithi006 bash[27292]: audit 2023-08-27T22:30:54.069964+0000 mon.a (mon.0) 1328 : audit [DBG] from='client.? 172.21.15.6:0/3150018710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:56 smithi130 bash[27579]: cluster 2023-08-27T22:30:55.653403+0000 mgr.y (mgr.14901) 972 : cluster [DBG] pgmap v979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:56 smithi130 bash[27579]: audit 2023-08-27T22:30:56.491515+0000 mon.a (mon.0) 1329 : audit [DBG] from='client.? 172.21.15.6:0/1840604958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:57.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:56 smithi006 bash[27292]: cluster 2023-08-27T22:30:55.653403+0000 mgr.y (mgr.14901) 972 : cluster [DBG] pgmap v979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:57.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:56 smithi006 bash[27292]: audit 2023-08-27T22:30:56.491515+0000 mon.a (mon.0) 1329 : audit [DBG] from='client.? 172.21.15.6:0/1840604958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:57.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:56 smithi006 bash[33918]: cluster 2023-08-27T22:30:55.653403+0000 mgr.y (mgr.14901) 972 : cluster [DBG] pgmap v979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:57.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:56 smithi006 bash[33918]: audit 2023-08-27T22:30:56.491515+0000 mon.a (mon.0) 1329 : audit [DBG] from='client.? 172.21.15.6:0/1840604958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:58 smithi130 bash[27579]: cluster 2023-08-27T22:30:57.654103+0000 mgr.y (mgr.14901) 973 : cluster [DBG] pgmap v980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:59.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:58 smithi006 bash[33918]: cluster 2023-08-27T22:30:57.654103+0000 mgr.y (mgr.14901) 973 : cluster [DBG] pgmap v980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:30:59.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:58 smithi006 bash[27292]: cluster 2023-08-27T22:30:57.654103+0000 mgr.y (mgr.14901) 973 : cluster [DBG] pgmap v980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:00.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:59 smithi130 bash[27579]: audit 2023-08-27T22:30:58.791137+0000 mon.a (mon.0) 1330 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:31:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:59 smithi130 bash[27579]: audit 2023-08-27T22:30:58.949392+0000 mon.a (mon.0) 1331 : audit [DBG] from='client.? 172.21.15.6:0/3907845851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:59 smithi130 bash[27579]: audit 2023-08-27T22:30:59.219609+0000 mon.a (mon.0) 1332 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:31:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:59 smithi130 bash[27579]: audit 2023-08-27T22:30:59.497222+0000 mon.b (mon.1) 218 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:31:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:59 smithi130 bash[27579]: audit 2023-08-27T22:30:59.499219+0000 mon.b (mon.1) 219 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:31:00.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:30:59 smithi130 bash[27579]: audit 2023-08-27T22:30:59.501873+0000 mon.a (mon.0) 1333 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:31:00.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:59 smithi006 bash[33918]: audit 2023-08-27T22:30:58.791137+0000 mon.a (mon.0) 1330 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:31:00.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:59 smithi006 bash[33918]: audit 2023-08-27T22:30:58.949392+0000 mon.a (mon.0) 1331 : audit [DBG] from='client.? 172.21.15.6:0/3907845851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:00.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:59 smithi006 bash[33918]: audit 2023-08-27T22:30:59.219609+0000 mon.a (mon.0) 1332 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:31:00.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:59 smithi006 bash[33918]: audit 2023-08-27T22:30:59.497222+0000 mon.b (mon.1) 218 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:31:00.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:59 smithi006 bash[33918]: audit 2023-08-27T22:30:59.499219+0000 mon.b (mon.1) 219 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:31:00.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:30:59 smithi006 bash[33918]: audit 2023-08-27T22:30:59.501873+0000 mon.a (mon.0) 1333 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:31:00.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:59 smithi006 bash[27292]: audit 2023-08-27T22:30:58.791137+0000 mon.a (mon.0) 1330 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:31:00.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:59 smithi006 bash[27292]: audit 2023-08-27T22:30:58.949392+0000 mon.a (mon.0) 1331 : audit [DBG] from='client.? 172.21.15.6:0/3907845851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:00.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:59 smithi006 bash[27292]: audit 2023-08-27T22:30:59.219609+0000 mon.a (mon.0) 1332 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:31:00.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:59 smithi006 bash[27292]: audit 2023-08-27T22:30:59.497222+0000 mon.b (mon.1) 218 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:31:00.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:59 smithi006 bash[27292]: audit 2023-08-27T22:30:59.499219+0000 mon.b (mon.1) 219 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:31:00.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:30:59 smithi006 bash[27292]: audit 2023-08-27T22:30:59.501873+0000 mon.a (mon.0) 1333 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:31:00.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:31:00.216Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:00.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:31:00.219Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:00 smithi130 bash[27579]: cluster 2023-08-27T22:30:59.655313+0000 mgr.y (mgr.14901) 974 : cluster [DBG] pgmap v981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:01.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:31:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:31:00] "GET /metrics HTTP/1.1" 200 193077 "" "Prometheus/2.33.4" 2023-08-27T22:31:01.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:00 smithi006 bash[33918]: cluster 2023-08-27T22:30:59.655313+0000 mgr.y (mgr.14901) 974 : cluster [DBG] pgmap v981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:01.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:00 smithi006 bash[27292]: cluster 2023-08-27T22:30:59.655313+0000 mgr.y (mgr.14901) 974 : cluster [DBG] pgmap v981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:02.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:01 smithi130 bash[27579]: audit 2023-08-27T22:31:01.369198+0000 mon.c (mon.2) 404 : audit [DBG] from='client.? 172.21.15.6:0/2900040112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:02.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:01 smithi006 bash[27292]: audit 2023-08-27T22:31:01.369198+0000 mon.c (mon.2) 404 : audit [DBG] from='client.? 172.21.15.6:0/2900040112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:02.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:01 smithi006 bash[33918]: audit 2023-08-27T22:31:01.369198+0000 mon.c (mon.2) 404 : audit [DBG] from='client.? 172.21.15.6:0/2900040112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:31:01.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:31:01.780Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:02 smithi130 bash[27579]: cluster 2023-08-27T22:31:01.656140+0000 mgr.y (mgr.14901) 975 : cluster [DBG] pgmap v982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:03.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:02 smithi006 bash[27292]: cluster 2023-08-27T22:31:01.656140+0000 mgr.y (mgr.14901) 975 : cluster [DBG] pgmap v982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:03.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:02 smithi006 bash[33918]: cluster 2023-08-27T22:31:01.656140+0000 mgr.y (mgr.14901) 975 : cluster [DBG] pgmap v982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:31:03.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:31:03.675Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:04.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:04 smithi006 bash[27292]: cluster 2023-08-27T22:31:03.656768+0000 mgr.y (mgr.14901) 976 : cluster [DBG] pgmap v983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:04.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:04 smithi006 bash[27292]: audit 2023-08-27T22:31:03.787867+0000 mon.a (mon.0) 1334 : audit [DBG] from='client.? 172.21.15.6:0/3432709331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:04.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:04 smithi006 bash[33918]: cluster 2023-08-27T22:31:03.656768+0000 mgr.y (mgr.14901) 976 : cluster [DBG] pgmap v983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:04.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:04 smithi006 bash[33918]: audit 2023-08-27T22:31:03.787867+0000 mon.a (mon.0) 1334 : audit [DBG] from='client.? 172.21.15.6:0/3432709331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:04 smithi130 bash[27579]: cluster 2023-08-27T22:31:03.656768+0000 mgr.y (mgr.14901) 976 : cluster [DBG] pgmap v983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:04 smithi130 bash[27579]: audit 2023-08-27T22:31:03.787867+0000 mon.a (mon.0) 1334 : audit [DBG] from='client.? 172.21.15.6:0/3432709331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:06 smithi130 bash[27579]: cluster 2023-08-27T22:31:05.657953+0000 mgr.y (mgr.14901) 977 : cluster [DBG] pgmap v984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:06 smithi130 bash[27579]: audit 2023-08-27T22:31:06.218708+0000 mon.a (mon.0) 1335 : audit [DBG] from='client.? 172.21.15.6:0/2486939064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:07.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:06 smithi006 bash[27292]: cluster 2023-08-27T22:31:05.657953+0000 mgr.y (mgr.14901) 977 : cluster [DBG] pgmap v984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:07.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:06 smithi006 bash[27292]: audit 2023-08-27T22:31:06.218708+0000 mon.a (mon.0) 1335 : audit [DBG] from='client.? 172.21.15.6:0/2486939064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:07.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:06 smithi006 bash[33918]: cluster 2023-08-27T22:31:05.657953+0000 mgr.y (mgr.14901) 977 : cluster [DBG] pgmap v984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:07.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:06 smithi006 bash[33918]: audit 2023-08-27T22:31:06.218708+0000 mon.a (mon.0) 1335 : audit [DBG] from='client.? 172.21.15.6:0/2486939064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:08 smithi130 bash[27579]: cluster 2023-08-27T22:31:07.658599+0000 mgr.y (mgr.14901) 978 : cluster [DBG] pgmap v985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:08 smithi130 bash[27579]: audit 2023-08-27T22:31:08.631257+0000 mon.c (mon.2) 405 : audit [DBG] from='client.? 172.21.15.6:0/654059698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:09.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:08 smithi006 bash[27292]: cluster 2023-08-27T22:31:07.658599+0000 mgr.y (mgr.14901) 978 : cluster [DBG] pgmap v985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:09.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:08 smithi006 bash[27292]: audit 2023-08-27T22:31:08.631257+0000 mon.c (mon.2) 405 : audit [DBG] from='client.? 172.21.15.6:0/654059698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:08 smithi006 bash[33918]: cluster 2023-08-27T22:31:07.658599+0000 mgr.y (mgr.14901) 978 : cluster [DBG] pgmap v985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:08 smithi006 bash[33918]: audit 2023-08-27T22:31:08.631257+0000 mon.c (mon.2) 405 : audit [DBG] from='client.? 172.21.15.6:0/654059698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:31:10.217Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:10.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:31:10.219Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:10 smithi130 bash[27579]: cluster 2023-08-27T22:31:09.659729+0000 mgr.y (mgr.14901) 979 : cluster [DBG] pgmap v986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:11.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:10 smithi006 bash[27292]: cluster 2023-08-27T22:31:09.659729+0000 mgr.y (mgr.14901) 979 : cluster [DBG] pgmap v986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:11.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:31:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:31:10] "GET /metrics HTTP/1.1" 200 193077 "" "Prometheus/2.33.4" 2023-08-27T22:31:11.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:10 smithi006 bash[33918]: cluster 2023-08-27T22:31:09.659729+0000 mgr.y (mgr.14901) 979 : cluster [DBG] pgmap v986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:12.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:11 smithi130 bash[27579]: audit 2023-08-27T22:31:11.050567+0000 mon.c (mon.2) 406 : audit [DBG] from='client.? 172.21.15.6:0/3564380013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:12.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:11 smithi006 bash[27292]: audit 2023-08-27T22:31:11.050567+0000 mon.c (mon.2) 406 : audit [DBG] from='client.? 172.21.15.6:0/3564380013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:31:11.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:31:11.781Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:12.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:11 smithi006 bash[33918]: audit 2023-08-27T22:31:11.050567+0000 mon.c (mon.2) 406 : audit [DBG] from='client.? 172.21.15.6:0/3564380013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:13.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:12 smithi130 bash[27579]: cluster 2023-08-27T22:31:11.660392+0000 mgr.y (mgr.14901) 980 : cluster [DBG] pgmap v987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:13.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:12 smithi006 bash[27292]: cluster 2023-08-27T22:31:11.660392+0000 mgr.y (mgr.14901) 980 : cluster [DBG] pgmap v987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:13.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:12 smithi006 bash[33918]: cluster 2023-08-27T22:31:11.660392+0000 mgr.y (mgr.14901) 980 : cluster [DBG] pgmap v987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:13.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:13 smithi006 bash[33918]: audit 2023-08-27T22:31:13.471849+0000 mon.a (mon.0) 1336 : audit [DBG] from='client.? 172.21.15.6:0/306752979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:13.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:13 smithi006 bash[27292]: audit 2023-08-27T22:31:13.471849+0000 mon.a (mon.0) 1336 : audit [DBG] from='client.? 172.21.15.6:0/306752979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:31:13.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:31:13.677Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:14.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:13 smithi130 bash[27579]: audit 2023-08-27T22:31:13.471849+0000 mon.a (mon.0) 1336 : audit [DBG] from='client.? 172.21.15.6:0/306752979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:14 smithi130 bash[27579]: cluster 2023-08-27T22:31:13.661186+0000 mgr.y (mgr.14901) 981 : cluster [DBG] pgmap v988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:15.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:14 smithi006 bash[33918]: cluster 2023-08-27T22:31:13.661186+0000 mgr.y (mgr.14901) 981 : cluster [DBG] pgmap v988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:15.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:14 smithi006 bash[27292]: cluster 2023-08-27T22:31:13.661186+0000 mgr.y (mgr.14901) 981 : cluster [DBG] pgmap v988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:16 smithi130 bash[27579]: cluster 2023-08-27T22:31:15.662322+0000 mgr.y (mgr.14901) 982 : cluster [DBG] pgmap v989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:16 smithi130 bash[27579]: audit 2023-08-27T22:31:15.887156+0000 mon.a (mon.0) 1337 : audit [DBG] from='client.? 172.21.15.6:0/1149216935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:17.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:16 smithi006 bash[27292]: cluster 2023-08-27T22:31:15.662322+0000 mgr.y (mgr.14901) 982 : cluster [DBG] pgmap v989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:17.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:16 smithi006 bash[27292]: audit 2023-08-27T22:31:15.887156+0000 mon.a (mon.0) 1337 : audit [DBG] from='client.? 172.21.15.6:0/1149216935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:17.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:16 smithi006 bash[33918]: cluster 2023-08-27T22:31:15.662322+0000 mgr.y (mgr.14901) 982 : cluster [DBG] pgmap v989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:17.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:16 smithi006 bash[33918]: audit 2023-08-27T22:31:15.887156+0000 mon.a (mon.0) 1337 : audit [DBG] from='client.? 172.21.15.6:0/1149216935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:18 smithi130 bash[27579]: cluster 2023-08-27T22:31:17.662943+0000 mgr.y (mgr.14901) 983 : cluster [DBG] pgmap v990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:18 smithi130 bash[27579]: audit 2023-08-27T22:31:18.297256+0000 mon.c (mon.2) 407 : audit [DBG] from='client.? 172.21.15.6:0/4222520965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:19.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:18 smithi006 bash[33918]: cluster 2023-08-27T22:31:17.662943+0000 mgr.y (mgr.14901) 983 : cluster [DBG] pgmap v990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:19.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:18 smithi006 bash[33918]: audit 2023-08-27T22:31:18.297256+0000 mon.c (mon.2) 407 : audit [DBG] from='client.? 172.21.15.6:0/4222520965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:19.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:18 smithi006 bash[27292]: cluster 2023-08-27T22:31:17.662943+0000 mgr.y (mgr.14901) 983 : cluster [DBG] pgmap v990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:19.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:18 smithi006 bash[27292]: audit 2023-08-27T22:31:18.297256+0000 mon.c (mon.2) 407 : audit [DBG] from='client.? 172.21.15.6:0/4222520965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:31:20.218Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:20.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:31:20.221Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:20 smithi130 bash[27579]: cluster 2023-08-27T22:31:19.664043+0000 mgr.y (mgr.14901) 984 : cluster [DBG] pgmap v991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:20 smithi130 bash[27579]: audit 2023-08-27T22:31:20.716365+0000 mon.c (mon.2) 408 : audit [DBG] from='client.? 172.21.15.6:0/1491401705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:21.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:20 smithi006 bash[27292]: cluster 2023-08-27T22:31:19.664043+0000 mgr.y (mgr.14901) 984 : cluster [DBG] pgmap v991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:21.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:20 smithi006 bash[27292]: audit 2023-08-27T22:31:20.716365+0000 mon.c (mon.2) 408 : audit [DBG] from='client.? 172.21.15.6:0/1491401705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:21.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:31:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:31:20] "GET /metrics HTTP/1.1" 200 193083 "" "Prometheus/2.33.4" 2023-08-27T22:31:21.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:20 smithi006 bash[33918]: cluster 2023-08-27T22:31:19.664043+0000 mgr.y (mgr.14901) 984 : cluster [DBG] pgmap v991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:21.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:20 smithi006 bash[33918]: audit 2023-08-27T22:31:20.716365+0000 mon.c (mon.2) 408 : audit [DBG] from='client.? 172.21.15.6:0/1491401705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:31:21.780Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:31:21.782Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:23.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:22 smithi130 bash[27579]: cluster 2023-08-27T22:31:21.664706+0000 mgr.y (mgr.14901) 985 : cluster [DBG] pgmap v992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:23.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:22 smithi006 bash[27292]: cluster 2023-08-27T22:31:21.664706+0000 mgr.y (mgr.14901) 985 : cluster [DBG] pgmap v992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:23.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:22 smithi006 bash[33918]: cluster 2023-08-27T22:31:21.664706+0000 mgr.y (mgr.14901) 985 : cluster [DBG] pgmap v992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:23.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:23 smithi006 bash[33918]: audit 2023-08-27T22:31:23.125089+0000 mon.c (mon.2) 409 : audit [DBG] from='client.? 172.21.15.6:0/3704281787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:23.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:23 smithi006 bash[27292]: audit 2023-08-27T22:31:23.125089+0000 mon.c (mon.2) 409 : audit [DBG] from='client.? 172.21.15.6:0/3704281787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:31:23.676Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:31:23.677Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:24.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:23 smithi130 bash[27579]: audit 2023-08-27T22:31:23.125089+0000 mon.c (mon.2) 409 : audit [DBG] from='client.? 172.21.15.6:0/3704281787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:24 smithi130 bash[27579]: cluster 2023-08-27T22:31:23.665278+0000 mgr.y (mgr.14901) 986 : cluster [DBG] pgmap v993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:25.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:24 smithi006 bash[33918]: cluster 2023-08-27T22:31:23.665278+0000 mgr.y (mgr.14901) 986 : cluster [DBG] pgmap v993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:25.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:24 smithi006 bash[27292]: cluster 2023-08-27T22:31:23.665278+0000 mgr.y (mgr.14901) 986 : cluster [DBG] pgmap v993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:26.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:25 smithi130 bash[27579]: audit 2023-08-27T22:31:25.545616+0000 mon.a (mon.0) 1338 : audit [DBG] from='client.? 172.21.15.6:0/2462780315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:26.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:25 smithi006 bash[33918]: audit 2023-08-27T22:31:25.545616+0000 mon.a (mon.0) 1338 : audit [DBG] from='client.? 172.21.15.6:0/2462780315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:26.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:25 smithi006 bash[27292]: audit 2023-08-27T22:31:25.545616+0000 mon.a (mon.0) 1338 : audit [DBG] from='client.? 172.21.15.6:0/2462780315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:26 smithi130 bash[27579]: cluster 2023-08-27T22:31:25.666395+0000 mgr.y (mgr.14901) 987 : cluster [DBG] pgmap v994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:26 smithi006 bash[27292]: cluster 2023-08-27T22:31:25.666395+0000 mgr.y (mgr.14901) 987 : cluster [DBG] pgmap v994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:27.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:26 smithi006 bash[33918]: cluster 2023-08-27T22:31:25.666395+0000 mgr.y (mgr.14901) 987 : cluster [DBG] pgmap v994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:28 smithi130 bash[27579]: cluster 2023-08-27T22:31:27.667042+0000 mgr.y (mgr.14901) 988 : cluster [DBG] pgmap v995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:28 smithi130 bash[27579]: audit 2023-08-27T22:31:27.959061+0000 mon.c (mon.2) 410 : audit [DBG] from='client.? 172.21.15.6:0/2810593441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:29.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:28 smithi006 bash[33918]: cluster 2023-08-27T22:31:27.667042+0000 mgr.y (mgr.14901) 988 : cluster [DBG] pgmap v995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:29.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:28 smithi006 bash[33918]: audit 2023-08-27T22:31:27.959061+0000 mon.c (mon.2) 410 : audit [DBG] from='client.? 172.21.15.6:0/2810593441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:29.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:28 smithi006 bash[27292]: cluster 2023-08-27T22:31:27.667042+0000 mgr.y (mgr.14901) 988 : cluster [DBG] pgmap v995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:29.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:28 smithi006 bash[27292]: audit 2023-08-27T22:31:27.959061+0000 mon.c (mon.2) 410 : audit [DBG] from='client.? 172.21.15.6:0/2810593441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:31:30.219Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:30.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:31:30.221Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:30 smithi130 bash[27579]: cluster 2023-08-27T22:31:29.668131+0000 mgr.y (mgr.14901) 989 : cluster [DBG] pgmap v996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:30 smithi130 bash[27579]: audit 2023-08-27T22:31:30.394121+0000 mon.c (mon.2) 411 : audit [DBG] from='client.? 172.21.15.6:0/2891560697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:31.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:31:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:31:30] "GET /metrics HTTP/1.1" 200 193076 "" "Prometheus/2.33.4" 2023-08-27T22:31:31.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:30 smithi006 bash[33918]: cluster 2023-08-27T22:31:29.668131+0000 mgr.y (mgr.14901) 989 : cluster [DBG] pgmap v996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:31.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:30 smithi006 bash[33918]: audit 2023-08-27T22:31:30.394121+0000 mon.c (mon.2) 411 : audit [DBG] from='client.? 172.21.15.6:0/2891560697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:31.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:30 smithi006 bash[27292]: cluster 2023-08-27T22:31:29.668131+0000 mgr.y (mgr.14901) 989 : cluster [DBG] pgmap v996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:31.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:30 smithi006 bash[27292]: audit 2023-08-27T22:31:30.394121+0000 mon.c (mon.2) 411 : audit [DBG] from='client.? 172.21.15.6:0/2891560697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:31:31.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:31:31.783Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:33.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:32 smithi130 bash[27579]: cluster 2023-08-27T22:31:31.668738+0000 mgr.y (mgr.14901) 990 : cluster [DBG] pgmap v997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:33.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:32 smithi130 bash[27579]: audit 2023-08-27T22:31:32.816229+0000 mon.c (mon.2) 412 : audit [DBG] from='client.? 172.21.15.6:0/1019451354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:32 smithi006 bash[27292]: cluster 2023-08-27T22:31:31.668738+0000 mgr.y (mgr.14901) 990 : cluster [DBG] pgmap v997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:32 smithi006 bash[27292]: audit 2023-08-27T22:31:32.816229+0000 mon.c (mon.2) 412 : audit [DBG] from='client.? 172.21.15.6:0/1019451354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:33.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:32 smithi006 bash[33918]: cluster 2023-08-27T22:31:31.668738+0000 mgr.y (mgr.14901) 990 : cluster [DBG] pgmap v997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:33.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:32 smithi006 bash[33918]: audit 2023-08-27T22:31:32.816229+0000 mon.c (mon.2) 412 : audit [DBG] from='client.? 172.21.15.6:0/1019451354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:31:33.676Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:31:33.679Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:34.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:34 smithi006 bash[33918]: cluster 2023-08-27T22:31:33.669397+0000 mgr.y (mgr.14901) 991 : cluster [DBG] pgmap v998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:34.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:34 smithi006 bash[27292]: cluster 2023-08-27T22:31:33.669397+0000 mgr.y (mgr.14901) 991 : cluster [DBG] pgmap v998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:34 smithi130 bash[27579]: cluster 2023-08-27T22:31:33.669397+0000 mgr.y (mgr.14901) 991 : cluster [DBG] pgmap v998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:35.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:35 smithi006 bash[33918]: audit 2023-08-27T22:31:35.234946+0000 mon.c (mon.2) 413 : audit [DBG] from='client.? 172.21.15.6:0/1121460410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:35.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:35 smithi006 bash[27292]: audit 2023-08-27T22:31:35.234946+0000 mon.c (mon.2) 413 : audit [DBG] from='client.? 172.21.15.6:0/1121460410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:36.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:35 smithi130 bash[27579]: audit 2023-08-27T22:31:35.234946+0000 mon.c (mon.2) 413 : audit [DBG] from='client.? 172.21.15.6:0/1121460410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:36.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:36 smithi006 bash[27292]: cluster 2023-08-27T22:31:35.670429+0000 mgr.y (mgr.14901) 992 : cluster [DBG] pgmap v999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:36.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:36 smithi006 bash[33918]: cluster 2023-08-27T22:31:35.670429+0000 mgr.y (mgr.14901) 992 : cluster [DBG] pgmap v999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:36 smithi130 bash[27579]: cluster 2023-08-27T22:31:35.670429+0000 mgr.y (mgr.14901) 992 : cluster [DBG] pgmap v999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:37.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:37 smithi006 bash[33918]: audit 2023-08-27T22:31:37.651705+0000 mon.c (mon.2) 414 : audit [DBG] from='client.? 172.21.15.6:0/3765331740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:37.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:37 smithi006 bash[27292]: audit 2023-08-27T22:31:37.651705+0000 mon.c (mon.2) 414 : audit [DBG] from='client.? 172.21.15.6:0/3765331740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:38.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:37 smithi130 bash[27579]: audit 2023-08-27T22:31:37.651705+0000 mon.c (mon.2) 414 : audit [DBG] from='client.? 172.21.15.6:0/3765331740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:38.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:38 smithi006 bash[33918]: cluster 2023-08-27T22:31:37.670950+0000 mgr.y (mgr.14901) 993 : cluster [DBG] pgmap v1000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:38.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:38 smithi006 bash[27292]: cluster 2023-08-27T22:31:37.670950+0000 mgr.y (mgr.14901) 993 : cluster [DBG] pgmap v1000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:38 smithi130 bash[27579]: cluster 2023-08-27T22:31:37.670950+0000 mgr.y (mgr.14901) 993 : cluster [DBG] pgmap v1000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:40.686 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:31:40.220Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:40.686 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:31:40.222Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:40.971 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:31:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:31:40] "GET /metrics HTTP/1.1" 200 193076 "" "Prometheus/2.33.4" 2023-08-27T22:31:40.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:40 smithi006 bash[27292]: cluster 2023-08-27T22:31:39.672016+0000 mgr.y (mgr.14901) 994 : cluster [DBG] pgmap v1001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:40.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:40 smithi006 bash[27292]: audit 2023-08-27T22:31:40.066746+0000 mon.c (mon.2) 415 : audit [DBG] from='client.? 172.21.15.6:0/3438505258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:40.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:40 smithi006 bash[33918]: cluster 2023-08-27T22:31:39.672016+0000 mgr.y (mgr.14901) 994 : cluster [DBG] pgmap v1001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:40.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:40 smithi006 bash[33918]: audit 2023-08-27T22:31:40.066746+0000 mon.c (mon.2) 415 : audit [DBG] from='client.? 172.21.15.6:0/3438505258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:40 smithi130 bash[27579]: cluster 2023-08-27T22:31:39.672016+0000 mgr.y (mgr.14901) 994 : cluster [DBG] pgmap v1001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:40 smithi130 bash[27579]: audit 2023-08-27T22:31:40.066746+0000 mon.c (mon.2) 415 : audit [DBG] from='client.? 172.21.15.6:0/3438505258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:31:41.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:31:41.784Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:42.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:42 smithi006 bash[27292]: cluster 2023-08-27T22:31:41.672656+0000 mgr.y (mgr.14901) 995 : cluster [DBG] pgmap v1002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:42.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:42 smithi006 bash[27292]: audit 2023-08-27T22:31:42.483093+0000 mon.a (mon.0) 1339 : audit [DBG] from='client.? 172.21.15.6:0/2662122572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:42.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:42 smithi006 bash[33918]: cluster 2023-08-27T22:31:41.672656+0000 mgr.y (mgr.14901) 995 : cluster [DBG] pgmap v1002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:42.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:42 smithi006 bash[33918]: audit 2023-08-27T22:31:42.483093+0000 mon.a (mon.0) 1339 : audit [DBG] from='client.? 172.21.15.6:0/2662122572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:42 smithi130 bash[27579]: cluster 2023-08-27T22:31:41.672656+0000 mgr.y (mgr.14901) 995 : cluster [DBG] pgmap v1002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:43.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:42 smithi130 bash[27579]: audit 2023-08-27T22:31:42.483093+0000 mon.a (mon.0) 1339 : audit [DBG] from='client.? 172.21.15.6:0/2662122572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:43.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:31:43.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:43.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:31:43.679Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:44.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:44 smithi006 bash[33918]: cluster 2023-08-27T22:31:43.673311+0000 mgr.y (mgr.14901) 996 : cluster [DBG] pgmap v1003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:44.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:44 smithi006 bash[27292]: cluster 2023-08-27T22:31:43.673311+0000 mgr.y (mgr.14901) 996 : cluster [DBG] pgmap v1003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:44 smithi130 bash[27579]: cluster 2023-08-27T22:31:43.673311+0000 mgr.y (mgr.14901) 996 : cluster [DBG] pgmap v1003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:45.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:45 smithi006 bash[27292]: audit 2023-08-27T22:31:44.900910+0000 mon.a (mon.0) 1340 : audit [DBG] from='client.? 172.21.15.6:0/4109468216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:45.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:45 smithi006 bash[33918]: audit 2023-08-27T22:31:44.900910+0000 mon.a (mon.0) 1340 : audit [DBG] from='client.? 172.21.15.6:0/4109468216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:46.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:45 smithi130 bash[27579]: audit 2023-08-27T22:31:44.900910+0000 mon.a (mon.0) 1340 : audit [DBG] from='client.? 172.21.15.6:0/4109468216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:46.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:46 smithi006 bash[33918]: cluster 2023-08-27T22:31:45.674411+0000 mgr.y (mgr.14901) 997 : cluster [DBG] pgmap v1004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:46.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:46 smithi006 bash[27292]: cluster 2023-08-27T22:31:45.674411+0000 mgr.y (mgr.14901) 997 : cluster [DBG] pgmap v1004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:46 smithi130 bash[27579]: cluster 2023-08-27T22:31:45.674411+0000 mgr.y (mgr.14901) 997 : cluster [DBG] pgmap v1004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:47.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:47 smithi006 bash[33918]: audit 2023-08-27T22:31:47.310489+0000 mon.a (mon.0) 1341 : audit [DBG] from='client.? 172.21.15.6:0/1691195977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:47.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:47 smithi006 bash[27292]: audit 2023-08-27T22:31:47.310489+0000 mon.a (mon.0) 1341 : audit [DBG] from='client.? 172.21.15.6:0/1691195977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:48.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:47 smithi130 bash[27579]: audit 2023-08-27T22:31:47.310489+0000 mon.a (mon.0) 1341 : audit [DBG] from='client.? 172.21.15.6:0/1691195977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:48.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:48 smithi006 bash[33918]: cluster 2023-08-27T22:31:47.675015+0000 mgr.y (mgr.14901) 998 : cluster [DBG] pgmap v1005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:48.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:48 smithi006 bash[27292]: cluster 2023-08-27T22:31:47.675015+0000 mgr.y (mgr.14901) 998 : cluster [DBG] pgmap v1005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:49.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:48 smithi130 bash[27579]: cluster 2023-08-27T22:31:47.675015+0000 mgr.y (mgr.14901) 998 : cluster [DBG] pgmap v1005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:50.693 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:31:50.220Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:50.693 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:31:50.222Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:50.971 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:31:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:31:50] "GET /metrics HTTP/1.1" 200 193081 "" "Prometheus/2.33.4" 2023-08-27T22:31:50.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:50 smithi006 bash[33918]: cluster 2023-08-27T22:31:49.675725+0000 mgr.y (mgr.14901) 999 : cluster [DBG] pgmap v1006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:50.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:50 smithi006 bash[33918]: audit 2023-08-27T22:31:49.719411+0000 mon.c (mon.2) 416 : audit [DBG] from='client.? 172.21.15.6:0/2333241778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:50.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:50 smithi006 bash[27292]: cluster 2023-08-27T22:31:49.675725+0000 mgr.y (mgr.14901) 999 : cluster [DBG] pgmap v1006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:50.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:50 smithi006 bash[27292]: audit 2023-08-27T22:31:49.719411+0000 mon.c (mon.2) 416 : audit [DBG] from='client.? 172.21.15.6:0/2333241778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:50 smithi130 bash[27579]: cluster 2023-08-27T22:31:49.675725+0000 mgr.y (mgr.14901) 999 : cluster [DBG] pgmap v1006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:51.169 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:50 smithi130 bash[27579]: audit 2023-08-27T22:31:49.719411+0000 mon.c (mon.2) 416 : audit [DBG] from='client.? 172.21.15.6:0/2333241778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:31:51.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:52.292 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:31:51.785Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:52.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:52 smithi006 bash[33918]: cluster 2023-08-27T22:31:51.676341+0000 mgr.y (mgr.14901) 1000 : cluster [DBG] pgmap v1007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:53.003 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:52 smithi006 bash[33918]: audit 2023-08-27T22:31:52.140288+0000 mon.c (mon.2) 417 : audit [DBG] from='client.? 172.21.15.6:0/3776336207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:53.003 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:52 smithi006 bash[27292]: cluster 2023-08-27T22:31:51.676341+0000 mgr.y (mgr.14901) 1000 : cluster [DBG] pgmap v1007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:53.003 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:52 smithi006 bash[27292]: audit 2023-08-27T22:31:52.140288+0000 mon.c (mon.2) 417 : audit [DBG] from='client.? 172.21.15.6:0/3776336207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:53.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:52 smithi130 bash[27579]: cluster 2023-08-27T22:31:51.676341+0000 mgr.y (mgr.14901) 1000 : cluster [DBG] pgmap v1007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:53.215 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:52 smithi130 bash[27579]: audit 2023-08-27T22:31:52.140288+0000 mon.c (mon.2) 417 : audit [DBG] from='client.? 172.21.15.6:0/3776336207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:31:53.679Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:54.011 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:31:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:31:53.681Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:31:54.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:54 smithi006 bash[27292]: cluster 2023-08-27T22:31:53.677010+0000 mgr.y (mgr.14901) 1001 : cluster [DBG] pgmap v1008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:55.364 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:54 smithi006 bash[27292]: audit 2023-08-27T22:31:54.547676+0000 mon.c (mon.2) 418 : audit [DBG] from='client.? 172.21.15.6:0/848552514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:55.364 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:54 smithi006 bash[33918]: cluster 2023-08-27T22:31:53.677010+0000 mgr.y (mgr.14901) 1001 : cluster [DBG] pgmap v1008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:55.364 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:54 smithi006 bash[33918]: audit 2023-08-27T22:31:54.547676+0000 mon.c (mon.2) 418 : audit [DBG] from='client.? 172.21.15.6:0/848552514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:55.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:54 smithi130 bash[27579]: cluster 2023-08-27T22:31:53.677010+0000 mgr.y (mgr.14901) 1001 : cluster [DBG] pgmap v1008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:55.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:54 smithi130 bash[27579]: audit 2023-08-27T22:31:54.547676+0000 mon.c (mon.2) 418 : audit [DBG] from='client.? 172.21.15.6:0/848552514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:56 smithi130 bash[27579]: cluster 2023-08-27T22:31:55.678111+0000 mgr.y (mgr.14901) 1002 : cluster [DBG] pgmap v1009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:57.346 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:56 smithi006 bash[27292]: cluster 2023-08-27T22:31:55.678111+0000 mgr.y (mgr.14901) 1002 : cluster [DBG] pgmap v1009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:57.346 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:56 smithi006 bash[33918]: cluster 2023-08-27T22:31:55.678111+0000 mgr.y (mgr.14901) 1002 : cluster [DBG] pgmap v1009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:58.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:57 smithi130 bash[27579]: audit 2023-08-27T22:31:56.961642+0000 mon.c (mon.2) 419 : audit [DBG] from='client.? 172.21.15.6:0/2209720714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:58.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:57 smithi006 bash[33918]: audit 2023-08-27T22:31:56.961642+0000 mon.c (mon.2) 419 : audit [DBG] from='client.? 172.21.15.6:0/2209720714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:58.299 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:57 smithi006 bash[27292]: audit 2023-08-27T22:31:56.961642+0000 mon.c (mon.2) 419 : audit [DBG] from='client.? 172.21.15.6:0/2209720714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:58 smithi130 bash[27579]: cluster 2023-08-27T22:31:57.678758+0000 mgr.y (mgr.14901) 1003 : cluster [DBG] pgmap v1010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:59.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:58 smithi006 bash[33918]: cluster 2023-08-27T22:31:57.678758+0000 mgr.y (mgr.14901) 1003 : cluster [DBG] pgmap v1010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:31:59.574 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:58 smithi006 bash[27292]: cluster 2023-08-27T22:31:57.678758+0000 mgr.y (mgr.14901) 1003 : cluster [DBG] pgmap v1010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:00.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:59 smithi130 bash[27579]: audit 2023-08-27T22:31:59.384195+0000 mon.a (mon.0) 1342 : audit [DBG] from='client.? 172.21.15.6:0/3245953807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:00.333 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:31:59 smithi130 bash[27579]: audit 2023-08-27T22:31:59.520243+0000 mon.b (mon.1) 220 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:32:00.334 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:59 smithi006 bash[27292]: audit 2023-08-27T22:31:59.384195+0000 mon.a (mon.0) 1342 : audit [DBG] from='client.? 172.21.15.6:0/3245953807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:00.336 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:31:59 smithi006 bash[27292]: audit 2023-08-27T22:31:59.520243+0000 mon.b (mon.1) 220 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:32:00.336 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:59 smithi006 bash[33918]: audit 2023-08-27T22:31:59.384195+0000 mon.a (mon.0) 1342 : audit [DBG] from='client.? 172.21.15.6:0/3245953807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:00.336 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:31:59 smithi006 bash[33918]: audit 2023-08-27T22:31:59.520243+0000 mon.b (mon.1) 220 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:32:00.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:32:00.221Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:00.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:32:00.223Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:00 smithi130 bash[27579]: cluster 2023-08-27T22:31:59.679856+0000 mgr.y (mgr.14901) 1004 : cluster [DBG] pgmap v1011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:01.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:32:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:32:00] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:32:01.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:00 smithi006 bash[27292]: cluster 2023-08-27T22:31:59.679856+0000 mgr.y (mgr.14901) 1004 : cluster [DBG] pgmap v1011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:01.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:00 smithi006 bash[33918]: cluster 2023-08-27T22:31:59.679856+0000 mgr.y (mgr.14901) 1004 : cluster [DBG] pgmap v1011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:32:01.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:32:01.785Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:02 smithi130 bash[27579]: cluster 2023-08-27T22:32:01.680482+0000 mgr.y (mgr.14901) 1005 : cluster [DBG] pgmap v1012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:02 smithi130 bash[27579]: audit 2023-08-27T22:32:01.800493+0000 mon.a (mon.0) 1343 : audit [DBG] from='client.? 172.21.15.6:0/551665820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:03.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:02 smithi006 bash[33918]: cluster 2023-08-27T22:32:01.680482+0000 mgr.y (mgr.14901) 1005 : cluster [DBG] pgmap v1012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:03.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:02 smithi006 bash[33918]: audit 2023-08-27T22:32:01.800493+0000 mon.a (mon.0) 1343 : audit [DBG] from='client.? 172.21.15.6:0/551665820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:03.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:02 smithi006 bash[27292]: cluster 2023-08-27T22:32:01.680482+0000 mgr.y (mgr.14901) 1005 : cluster [DBG] pgmap v1012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:03.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:02 smithi006 bash[27292]: audit 2023-08-27T22:32:01.800493+0000 mon.a (mon.0) 1343 : audit [DBG] from='client.? 172.21.15.6:0/551665820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:32:03.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:32:03.682Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:04.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:04 smithi006 bash[27292]: cluster 2023-08-27T22:32:03.681132+0000 mgr.y (mgr.14901) 1006 : cluster [DBG] pgmap v1013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:04.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:04 smithi006 bash[27292]: audit 2023-08-27T22:32:04.212104+0000 mon.a (mon.0) 1344 : audit [DBG] from='client.? 172.21.15.6:0/2500304776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:04.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:04 smithi006 bash[33918]: cluster 2023-08-27T22:32:03.681132+0000 mgr.y (mgr.14901) 1006 : cluster [DBG] pgmap v1013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:04.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:04 smithi006 bash[33918]: audit 2023-08-27T22:32:04.212104+0000 mon.a (mon.0) 1344 : audit [DBG] from='client.? 172.21.15.6:0/2500304776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:04 smithi130 bash[27579]: cluster 2023-08-27T22:32:03.681132+0000 mgr.y (mgr.14901) 1006 : cluster [DBG] pgmap v1013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:04 smithi130 bash[27579]: audit 2023-08-27T22:32:04.212104+0000 mon.a (mon.0) 1344 : audit [DBG] from='client.? 172.21.15.6:0/2500304776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:06.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:06 smithi006 bash[33918]: audit 2023-08-27T22:32:05.442099+0000 mon.a (mon.0) 1345 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:32:06.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:06 smithi006 bash[33918]: cluster 2023-08-27T22:32:05.682202+0000 mgr.y (mgr.14901) 1007 : cluster [DBG] pgmap v1014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:06.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:06 smithi006 bash[33918]: audit 2023-08-27T22:32:05.886814+0000 mon.a (mon.0) 1346 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:32:06.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:06 smithi006 bash[33918]: audit 2023-08-27T22:32:06.191714+0000 mon.b (mon.1) 221 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:32:06.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:06 smithi006 bash[33918]: audit 2023-08-27T22:32:06.194018+0000 mon.b (mon.1) 222 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:32:06.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:06 smithi006 bash[33918]: audit 2023-08-27T22:32:06.199939+0000 mon.a (mon.0) 1347 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:32:06.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:06 smithi006 bash[27292]: audit 2023-08-27T22:32:05.442099+0000 mon.a (mon.0) 1345 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:32:06.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:06 smithi006 bash[27292]: cluster 2023-08-27T22:32:05.682202+0000 mgr.y (mgr.14901) 1007 : cluster [DBG] pgmap v1014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:06.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:06 smithi006 bash[27292]: audit 2023-08-27T22:32:05.886814+0000 mon.a (mon.0) 1346 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:32:06.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:06 smithi006 bash[27292]: audit 2023-08-27T22:32:06.191714+0000 mon.b (mon.1) 221 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:32:06.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:06 smithi006 bash[27292]: audit 2023-08-27T22:32:06.194018+0000 mon.b (mon.1) 222 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:32:06.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:06 smithi006 bash[27292]: audit 2023-08-27T22:32:06.199939+0000 mon.a (mon.0) 1347 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:32:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:06 smithi130 bash[27579]: audit 2023-08-27T22:32:05.442099+0000 mon.a (mon.0) 1345 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:32:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:06 smithi130 bash[27579]: cluster 2023-08-27T22:32:05.682202+0000 mgr.y (mgr.14901) 1007 : cluster [DBG] pgmap v1014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:06 smithi130 bash[27579]: audit 2023-08-27T22:32:05.886814+0000 mon.a (mon.0) 1346 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:32:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:06 smithi130 bash[27579]: audit 2023-08-27T22:32:06.191714+0000 mon.b (mon.1) 221 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:32:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:06 smithi130 bash[27579]: audit 2023-08-27T22:32:06.194018+0000 mon.b (mon.1) 222 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:32:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:06 smithi130 bash[27579]: audit 2023-08-27T22:32:06.199939+0000 mon.a (mon.0) 1347 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:32:07.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:07 smithi006 bash[33918]: audit 2023-08-27T22:32:06.630659+0000 mon.c (mon.2) 420 : audit [DBG] from='client.? 172.21.15.6:0/1798927464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:07.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:07 smithi006 bash[27292]: audit 2023-08-27T22:32:06.630659+0000 mon.c (mon.2) 420 : audit [DBG] from='client.? 172.21.15.6:0/1798927464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:07.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:07 smithi130 bash[27579]: audit 2023-08-27T22:32:06.630659+0000 mon.c (mon.2) 420 : audit [DBG] from='client.? 172.21.15.6:0/1798927464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:08.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:08 smithi006 bash[33918]: cluster 2023-08-27T22:32:07.682841+0000 mgr.y (mgr.14901) 1008 : cluster [DBG] pgmap v1015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:08.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:08 smithi006 bash[27292]: cluster 2023-08-27T22:32:07.682841+0000 mgr.y (mgr.14901) 1008 : cluster [DBG] pgmap v1015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:08 smithi130 bash[27579]: cluster 2023-08-27T22:32:07.682841+0000 mgr.y (mgr.14901) 1008 : cluster [DBG] pgmap v1015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:09.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:09 smithi006 bash[27292]: audit 2023-08-27T22:32:09.044661+0000 mon.c (mon.2) 421 : audit [DBG] from='client.? 172.21.15.6:0/2333591400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:09.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:09 smithi006 bash[33918]: audit 2023-08-27T22:32:09.044661+0000 mon.c (mon.2) 421 : audit [DBG] from='client.? 172.21.15.6:0/2333591400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:10.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:09 smithi130 bash[27579]: audit 2023-08-27T22:32:09.044661+0000 mon.c (mon.2) 421 : audit [DBG] from='client.? 172.21.15.6:0/2333591400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:10.674 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:32:10.222Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:10.674 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:32:10.224Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:10.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:10 smithi006 bash[27292]: cluster 2023-08-27T22:32:09.683974+0000 mgr.y (mgr.14901) 1009 : cluster [DBG] pgmap v1016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:10.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:32:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:32:10] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:32:10.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:10 smithi006 bash[33918]: cluster 2023-08-27T22:32:09.683974+0000 mgr.y (mgr.14901) 1009 : cluster [DBG] pgmap v1016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:10 smithi130 bash[27579]: cluster 2023-08-27T22:32:09.683974+0000 mgr.y (mgr.14901) 1009 : cluster [DBG] pgmap v1016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:11.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:11 smithi006 bash[27292]: audit 2023-08-27T22:32:11.462748+0000 mon.a (mon.0) 1348 : audit [DBG] from='client.? 172.21.15.6:0/2287081982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:11.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:11 smithi006 bash[33918]: audit 2023-08-27T22:32:11.462748+0000 mon.a (mon.0) 1348 : audit [DBG] from='client.? 172.21.15.6:0/2287081982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:11.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:32:11.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:11.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:32:11.786Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:12.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:11 smithi130 bash[27579]: audit 2023-08-27T22:32:11.462748+0000 mon.a (mon.0) 1348 : audit [DBG] from='client.? 172.21.15.6:0/2287081982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:12.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:12 smithi006 bash[27292]: cluster 2023-08-27T22:32:11.684558+0000 mgr.y (mgr.14901) 1010 : cluster [DBG] pgmap v1017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:12.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:12 smithi006 bash[33918]: cluster 2023-08-27T22:32:11.684558+0000 mgr.y (mgr.14901) 1010 : cluster [DBG] pgmap v1017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:13.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:12 smithi130 bash[27579]: cluster 2023-08-27T22:32:11.684558+0000 mgr.y (mgr.14901) 1010 : cluster [DBG] pgmap v1017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:32:13.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:32:13.683Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:14.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:14 smithi006 bash[27292]: cluster 2023-08-27T22:32:13.685166+0000 mgr.y (mgr.14901) 1011 : cluster [DBG] pgmap v1018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:14.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:14 smithi006 bash[27292]: audit 2023-08-27T22:32:13.881197+0000 mon.a (mon.0) 1349 : audit [DBG] from='client.? 172.21.15.6:0/1676428926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:14.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:14 smithi006 bash[33918]: cluster 2023-08-27T22:32:13.685166+0000 mgr.y (mgr.14901) 1011 : cluster [DBG] pgmap v1018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:14.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:14 smithi006 bash[33918]: audit 2023-08-27T22:32:13.881197+0000 mon.a (mon.0) 1349 : audit [DBG] from='client.? 172.21.15.6:0/1676428926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:14 smithi130 bash[27579]: cluster 2023-08-27T22:32:13.685166+0000 mgr.y (mgr.14901) 1011 : cluster [DBG] pgmap v1018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:14 smithi130 bash[27579]: audit 2023-08-27T22:32:13.881197+0000 mon.a (mon.0) 1349 : audit [DBG] from='client.? 172.21.15.6:0/1676428926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:16 smithi130 bash[27579]: cluster 2023-08-27T22:32:15.686232+0000 mgr.y (mgr.14901) 1012 : cluster [DBG] pgmap v1019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:16 smithi130 bash[27579]: audit 2023-08-27T22:32:16.296908+0000 mon.c (mon.2) 422 : audit [DBG] from='client.? 172.21.15.6:0/4070135699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:17.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:16 smithi006 bash[27292]: cluster 2023-08-27T22:32:15.686232+0000 mgr.y (mgr.14901) 1012 : cluster [DBG] pgmap v1019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:17.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:16 smithi006 bash[27292]: audit 2023-08-27T22:32:16.296908+0000 mon.c (mon.2) 422 : audit [DBG] from='client.? 172.21.15.6:0/4070135699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:17.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:16 smithi006 bash[33918]: cluster 2023-08-27T22:32:15.686232+0000 mgr.y (mgr.14901) 1012 : cluster [DBG] pgmap v1019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:17.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:16 smithi006 bash[33918]: audit 2023-08-27T22:32:16.296908+0000 mon.c (mon.2) 422 : audit [DBG] from='client.? 172.21.15.6:0/4070135699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:18 smithi130 bash[27579]: cluster 2023-08-27T22:32:17.686995+0000 mgr.y (mgr.14901) 1013 : cluster [DBG] pgmap v1020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:18 smithi130 bash[27579]: audit 2023-08-27T22:32:18.706516+0000 mon.a (mon.0) 1350 : audit [DBG] from='client.? 172.21.15.6:0/3209830602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:19.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:18 smithi006 bash[33918]: cluster 2023-08-27T22:32:17.686995+0000 mgr.y (mgr.14901) 1013 : cluster [DBG] pgmap v1020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:19.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:18 smithi006 bash[33918]: audit 2023-08-27T22:32:18.706516+0000 mon.a (mon.0) 1350 : audit [DBG] from='client.? 172.21.15.6:0/3209830602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:19.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:18 smithi006 bash[27292]: cluster 2023-08-27T22:32:17.686995+0000 mgr.y (mgr.14901) 1013 : cluster [DBG] pgmap v1020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:19.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:18 smithi006 bash[27292]: audit 2023-08-27T22:32:18.706516+0000 mon.a (mon.0) 1350 : audit [DBG] from='client.? 172.21.15.6:0/3209830602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:32:20.223Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:32:20.225Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:20 smithi130 bash[27579]: cluster 2023-08-27T22:32:19.688213+0000 mgr.y (mgr.14901) 1014 : cluster [DBG] pgmap v1021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:21.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:32:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:32:20] "GET /metrics HTTP/1.1" 200 193080 "" "Prometheus/2.33.4" 2023-08-27T22:32:21.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:20 smithi006 bash[33918]: cluster 2023-08-27T22:32:19.688213+0000 mgr.y (mgr.14901) 1014 : cluster [DBG] pgmap v1021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:21.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:20 smithi006 bash[27292]: cluster 2023-08-27T22:32:19.688213+0000 mgr.y (mgr.14901) 1014 : cluster [DBG] pgmap v1021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:22.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:21 smithi130 bash[27579]: audit 2023-08-27T22:32:21.126729+0000 mon.c (mon.2) 423 : audit [DBG] from='client.? 172.21.15.6:0/81702398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:22.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:21 smithi006 bash[27292]: audit 2023-08-27T22:32:21.126729+0000 mon.c (mon.2) 423 : audit [DBG] from='client.? 172.21.15.6:0/81702398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:22.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:21 smithi006 bash[33918]: audit 2023-08-27T22:32:21.126729+0000 mon.c (mon.2) 423 : audit [DBG] from='client.? 172.21.15.6:0/81702398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:32:21.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:22.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:32:21.787Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:23.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:22 smithi130 bash[27579]: cluster 2023-08-27T22:32:21.688975+0000 mgr.y (mgr.14901) 1015 : cluster [DBG] pgmap v1022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:23.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:22 smithi006 bash[27292]: cluster 2023-08-27T22:32:21.688975+0000 mgr.y (mgr.14901) 1015 : cluster [DBG] pgmap v1022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:23.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:22 smithi006 bash[33918]: cluster 2023-08-27T22:32:21.688975+0000 mgr.y (mgr.14901) 1015 : cluster [DBG] pgmap v1022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:23.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:23 smithi006 bash[27292]: audit 2023-08-27T22:32:23.552372+0000 mon.c (mon.2) 424 : audit [DBG] from='client.? 172.21.15.6:0/448757979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:23.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:23 smithi006 bash[33918]: audit 2023-08-27T22:32:23.552372+0000 mon.c (mon.2) 424 : audit [DBG] from='client.? 172.21.15.6:0/448757979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:32:23.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:32:23.683Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:24.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:23 smithi130 bash[27579]: audit 2023-08-27T22:32:23.552372+0000 mon.c (mon.2) 424 : audit [DBG] from='client.? 172.21.15.6:0/448757979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:24 smithi130 bash[27579]: cluster 2023-08-27T22:32:23.689625+0000 mgr.y (mgr.14901) 1016 : cluster [DBG] pgmap v1023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:25.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:24 smithi006 bash[33918]: cluster 2023-08-27T22:32:23.689625+0000 mgr.y (mgr.14901) 1016 : cluster [DBG] pgmap v1023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:25.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:24 smithi006 bash[27292]: cluster 2023-08-27T22:32:23.689625+0000 mgr.y (mgr.14901) 1016 : cluster [DBG] pgmap v1023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:26 smithi130 bash[27579]: cluster 2023-08-27T22:32:25.690919+0000 mgr.y (mgr.14901) 1017 : cluster [DBG] pgmap v1024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:26 smithi130 bash[27579]: audit 2023-08-27T22:32:25.959425+0000 mon.a (mon.0) 1351 : audit [DBG] from='client.? 172.21.15.6:0/3165430697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:27.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:26 smithi006 bash[33918]: cluster 2023-08-27T22:32:25.690919+0000 mgr.y (mgr.14901) 1017 : cluster [DBG] pgmap v1024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:27.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:26 smithi006 bash[33918]: audit 2023-08-27T22:32:25.959425+0000 mon.a (mon.0) 1351 : audit [DBG] from='client.? 172.21.15.6:0/3165430697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:26 smithi006 bash[27292]: cluster 2023-08-27T22:32:25.690919+0000 mgr.y (mgr.14901) 1017 : cluster [DBG] pgmap v1024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:26 smithi006 bash[27292]: audit 2023-08-27T22:32:25.959425+0000 mon.a (mon.0) 1351 : audit [DBG] from='client.? 172.21.15.6:0/3165430697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:28 smithi130 bash[27579]: cluster 2023-08-27T22:32:27.691717+0000 mgr.y (mgr.14901) 1018 : cluster [DBG] pgmap v1025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:28 smithi130 bash[27579]: audit 2023-08-27T22:32:28.373620+0000 mon.a (mon.0) 1352 : audit [DBG] from='client.? 172.21.15.6:0/403108739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:29.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:28 smithi006 bash[33918]: cluster 2023-08-27T22:32:27.691717+0000 mgr.y (mgr.14901) 1018 : cluster [DBG] pgmap v1025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:29.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:28 smithi006 bash[33918]: audit 2023-08-27T22:32:28.373620+0000 mon.a (mon.0) 1352 : audit [DBG] from='client.? 172.21.15.6:0/403108739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:29.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:28 smithi006 bash[27292]: cluster 2023-08-27T22:32:27.691717+0000 mgr.y (mgr.14901) 1018 : cluster [DBG] pgmap v1025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:29.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:28 smithi006 bash[27292]: audit 2023-08-27T22:32:28.373620+0000 mon.a (mon.0) 1352 : audit [DBG] from='client.? 172.21.15.6:0/403108739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:32:30.224Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:32:30.225Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:30 smithi130 bash[27579]: cluster 2023-08-27T22:32:29.693010+0000 mgr.y (mgr.14901) 1019 : cluster [DBG] pgmap v1026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:31.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:32:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:32:30] "GET /metrics HTTP/1.1" 200 193076 "" "Prometheus/2.33.4" 2023-08-27T22:32:31.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:30 smithi006 bash[33918]: cluster 2023-08-27T22:32:29.693010+0000 mgr.y (mgr.14901) 1019 : cluster [DBG] pgmap v1026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:31.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:30 smithi006 bash[27292]: cluster 2023-08-27T22:32:29.693010+0000 mgr.y (mgr.14901) 1019 : cluster [DBG] pgmap v1026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:32.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:31 smithi130 bash[27579]: audit 2023-08-27T22:32:30.799241+0000 mon.c (mon.2) 425 : audit [DBG] from='client.? 172.21.15.6:0/2232140498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:32.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:31 smithi006 bash[27292]: audit 2023-08-27T22:32:30.799241+0000 mon.c (mon.2) 425 : audit [DBG] from='client.? 172.21.15.6:0/2232140498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:32.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:31 smithi006 bash[33918]: audit 2023-08-27T22:32:30.799241+0000 mon.c (mon.2) 425 : audit [DBG] from='client.? 172.21.15.6:0/2232140498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:32:31.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:32:31.787Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:33.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:32 smithi130 bash[27579]: cluster 2023-08-27T22:32:31.693687+0000 mgr.y (mgr.14901) 1020 : cluster [DBG] pgmap v1027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:32 smithi006 bash[27292]: cluster 2023-08-27T22:32:31.693687+0000 mgr.y (mgr.14901) 1020 : cluster [DBG] pgmap v1027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:33.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:32 smithi006 bash[33918]: cluster 2023-08-27T22:32:31.693687+0000 mgr.y (mgr.14901) 1020 : cluster [DBG] pgmap v1027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:33.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:33 smithi006 bash[33918]: audit 2023-08-27T22:32:33.206518+0000 mon.a (mon.0) 1353 : audit [DBG] from='client.? 172.21.15.6:0/197462519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:32:33.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:32:33.683Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:33.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:33 smithi006 bash[27292]: audit 2023-08-27T22:32:33.206518+0000 mon.a (mon.0) 1353 : audit [DBG] from='client.? 172.21.15.6:0/197462519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:34.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:33 smithi130 bash[27579]: audit 2023-08-27T22:32:33.206518+0000 mon.a (mon.0) 1353 : audit [DBG] from='client.? 172.21.15.6:0/197462519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:34 smithi130 bash[27579]: cluster 2023-08-27T22:32:33.694308+0000 mgr.y (mgr.14901) 1021 : cluster [DBG] pgmap v1028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:35.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:34 smithi006 bash[33918]: cluster 2023-08-27T22:32:33.694308+0000 mgr.y (mgr.14901) 1021 : cluster [DBG] pgmap v1028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:35.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:34 smithi006 bash[27292]: cluster 2023-08-27T22:32:33.694308+0000 mgr.y (mgr.14901) 1021 : cluster [DBG] pgmap v1028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:36.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:35 smithi130 bash[27579]: audit 2023-08-27T22:32:35.617811+0000 mon.a (mon.0) 1354 : audit [DBG] from='client.? 172.21.15.6:0/3966635138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:36.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:35 smithi006 bash[27292]: audit 2023-08-27T22:32:35.617811+0000 mon.a (mon.0) 1354 : audit [DBG] from='client.? 172.21.15.6:0/3966635138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:36.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:35 smithi006 bash[33918]: audit 2023-08-27T22:32:35.617811+0000 mon.a (mon.0) 1354 : audit [DBG] from='client.? 172.21.15.6:0/3966635138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:36 smithi130 bash[27579]: cluster 2023-08-27T22:32:35.695435+0000 mgr.y (mgr.14901) 1022 : cluster [DBG] pgmap v1029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:37.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:36 smithi006 bash[33918]: cluster 2023-08-27T22:32:35.695435+0000 mgr.y (mgr.14901) 1022 : cluster [DBG] pgmap v1029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:37.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:36 smithi006 bash[27292]: cluster 2023-08-27T22:32:35.695435+0000 mgr.y (mgr.14901) 1022 : cluster [DBG] pgmap v1029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:38 smithi130 bash[27579]: cluster 2023-08-27T22:32:37.695968+0000 mgr.y (mgr.14901) 1023 : cluster [DBG] pgmap v1030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:38 smithi130 bash[27579]: audit 2023-08-27T22:32:38.032281+0000 mon.a (mon.0) 1355 : audit [DBG] from='client.? 172.21.15.6:0/1678797021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:39.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:38 smithi006 bash[33918]: cluster 2023-08-27T22:32:37.695968+0000 mgr.y (mgr.14901) 1023 : cluster [DBG] pgmap v1030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:39.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:38 smithi006 bash[33918]: audit 2023-08-27T22:32:38.032281+0000 mon.a (mon.0) 1355 : audit [DBG] from='client.? 172.21.15.6:0/1678797021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:39.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:38 smithi006 bash[27292]: cluster 2023-08-27T22:32:37.695968+0000 mgr.y (mgr.14901) 1023 : cluster [DBG] pgmap v1030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:39.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:38 smithi006 bash[27292]: audit 2023-08-27T22:32:38.032281+0000 mon.a (mon.0) 1355 : audit [DBG] from='client.? 172.21.15.6:0/1678797021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:32:40.224Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:32:40.226Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:40 smithi130 bash[27579]: cluster 2023-08-27T22:32:39.697049+0000 mgr.y (mgr.14901) 1024 : cluster [DBG] pgmap v1031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:40 smithi130 bash[27579]: audit 2023-08-27T22:32:40.455462+0000 mon.c (mon.2) 426 : audit [DBG] from='client.? 172.21.15.6:0/2252104492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:41.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:32:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:32:40] "GET /metrics HTTP/1.1" 200 193076 "" "Prometheus/2.33.4" 2023-08-27T22:32:41.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:40 smithi006 bash[27292]: cluster 2023-08-27T22:32:39.697049+0000 mgr.y (mgr.14901) 1024 : cluster [DBG] pgmap v1031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:41.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:40 smithi006 bash[27292]: audit 2023-08-27T22:32:40.455462+0000 mon.c (mon.2) 426 : audit [DBG] from='client.? 172.21.15.6:0/2252104492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:41.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:40 smithi006 bash[33918]: cluster 2023-08-27T22:32:39.697049+0000 mgr.y (mgr.14901) 1024 : cluster [DBG] pgmap v1031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:41.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:40 smithi006 bash[33918]: audit 2023-08-27T22:32:40.455462+0000 mon.c (mon.2) 426 : audit [DBG] from='client.? 172.21.15.6:0/2252104492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:32:41.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:32:41.788Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:43.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:42 smithi130 bash[27579]: cluster 2023-08-27T22:32:41.697663+0000 mgr.y (mgr.14901) 1025 : cluster [DBG] pgmap v1032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:43.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:42 smithi006 bash[27292]: cluster 2023-08-27T22:32:41.697663+0000 mgr.y (mgr.14901) 1025 : cluster [DBG] pgmap v1032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:43.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:42 smithi006 bash[33918]: cluster 2023-08-27T22:32:41.697663+0000 mgr.y (mgr.14901) 1025 : cluster [DBG] pgmap v1032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:43.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:43 smithi006 bash[27292]: audit 2023-08-27T22:32:42.865851+0000 mon.a (mon.0) 1356 : audit [DBG] from='client.? 172.21.15.6:0/1271718522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:43.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:43 smithi006 bash[33918]: audit 2023-08-27T22:32:42.865851+0000 mon.a (mon.0) 1356 : audit [DBG] from='client.? 172.21.15.6:0/1271718522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:32:43.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:32:43.684Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:44.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:43 smithi130 bash[27579]: audit 2023-08-27T22:32:42.865851+0000 mon.a (mon.0) 1356 : audit [DBG] from='client.? 172.21.15.6:0/1271718522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:44 smithi130 bash[27579]: cluster 2023-08-27T22:32:43.698329+0000 mgr.y (mgr.14901) 1026 : cluster [DBG] pgmap v1033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:45.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:44 smithi006 bash[33918]: cluster 2023-08-27T22:32:43.698329+0000 mgr.y (mgr.14901) 1026 : cluster [DBG] pgmap v1033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:45.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:44 smithi006 bash[27292]: cluster 2023-08-27T22:32:43.698329+0000 mgr.y (mgr.14901) 1026 : cluster [DBG] pgmap v1033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:46.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:45 smithi130 bash[27579]: audit 2023-08-27T22:32:45.273664+0000 mon.a (mon.0) 1357 : audit [DBG] from='client.? 172.21.15.6:0/518490699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:46.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:45 smithi006 bash[33918]: audit 2023-08-27T22:32:45.273664+0000 mon.a (mon.0) 1357 : audit [DBG] from='client.? 172.21.15.6:0/518490699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:46.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:45 smithi006 bash[27292]: audit 2023-08-27T22:32:45.273664+0000 mon.a (mon.0) 1357 : audit [DBG] from='client.? 172.21.15.6:0/518490699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:46 smithi130 bash[27579]: cluster 2023-08-27T22:32:45.699506+0000 mgr.y (mgr.14901) 1027 : cluster [DBG] pgmap v1034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:47.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:46 smithi006 bash[33918]: cluster 2023-08-27T22:32:45.699506+0000 mgr.y (mgr.14901) 1027 : cluster [DBG] pgmap v1034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:47.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:46 smithi006 bash[27292]: cluster 2023-08-27T22:32:45.699506+0000 mgr.y (mgr.14901) 1027 : cluster [DBG] pgmap v1034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:48.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:47 smithi006 bash[33918]: audit 2023-08-27T22:32:47.691619+0000 mon.a (mon.0) 1358 : audit [DBG] from='client.? 172.21.15.6:0/1558304161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:48.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:47 smithi006 bash[27292]: audit 2023-08-27T22:32:47.691619+0000 mon.a (mon.0) 1358 : audit [DBG] from='client.? 172.21.15.6:0/1558304161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:48.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:47 smithi130 bash[27579]: audit 2023-08-27T22:32:47.691619+0000 mon.a (mon.0) 1358 : audit [DBG] from='client.? 172.21.15.6:0/1558304161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:49.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:48 smithi006 bash[33918]: cluster 2023-08-27T22:32:47.699960+0000 mgr.y (mgr.14901) 1028 : cluster [DBG] pgmap v1035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:49.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:48 smithi006 bash[27292]: cluster 2023-08-27T22:32:47.699960+0000 mgr.y (mgr.14901) 1028 : cluster [DBG] pgmap v1035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:49.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:48 smithi130 bash[27579]: cluster 2023-08-27T22:32:47.699960+0000 mgr.y (mgr.14901) 1028 : cluster [DBG] pgmap v1035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:32:50.224Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:32:50.226Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:51.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:32:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:32:50] "GET /metrics HTTP/1.1" 200 193077 "" "Prometheus/2.33.4" 2023-08-27T22:32:51.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:50 smithi006 bash[27292]: cluster 2023-08-27T22:32:49.701164+0000 mgr.y (mgr.14901) 1029 : cluster [DBG] pgmap v1036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:51.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:50 smithi006 bash[27292]: audit 2023-08-27T22:32:50.110539+0000 mon.a (mon.0) 1359 : audit [DBG] from='client.? 172.21.15.6:0/4120984824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:51.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:50 smithi006 bash[33918]: cluster 2023-08-27T22:32:49.701164+0000 mgr.y (mgr.14901) 1029 : cluster [DBG] pgmap v1036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:51.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:50 smithi006 bash[33918]: audit 2023-08-27T22:32:50.110539+0000 mon.a (mon.0) 1359 : audit [DBG] from='client.? 172.21.15.6:0/4120984824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:51.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:50 smithi130 bash[27579]: cluster 2023-08-27T22:32:49.701164+0000 mgr.y (mgr.14901) 1029 : cluster [DBG] pgmap v1036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:51.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:50 smithi130 bash[27579]: audit 2023-08-27T22:32:50.110539+0000 mon.a (mon.0) 1359 : audit [DBG] from='client.? 172.21.15.6:0/4120984824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:32:51.788Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:32:51.790Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:53.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:52 smithi006 bash[33918]: cluster 2023-08-27T22:32:51.701831+0000 mgr.y (mgr.14901) 1030 : cluster [DBG] pgmap v1037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:53.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:52 smithi006 bash[33918]: audit 2023-08-27T22:32:52.523178+0000 mon.c (mon.2) 427 : audit [DBG] from='client.? 172.21.15.6:0/2760745892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:53.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:52 smithi006 bash[27292]: cluster 2023-08-27T22:32:51.701831+0000 mgr.y (mgr.14901) 1030 : cluster [DBG] pgmap v1037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:53.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:52 smithi006 bash[27292]: audit 2023-08-27T22:32:52.523178+0000 mon.c (mon.2) 427 : audit [DBG] from='client.? 172.21.15.6:0/2760745892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:53.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:52 smithi130 bash[27579]: cluster 2023-08-27T22:32:51.701831+0000 mgr.y (mgr.14901) 1030 : cluster [DBG] pgmap v1037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:53.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:52 smithi130 bash[27579]: audit 2023-08-27T22:32:52.523178+0000 mon.c (mon.2) 427 : audit [DBG] from='client.? 172.21.15.6:0/2760745892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:32:53.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:32:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:32:53.685Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:32:54.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:54 smithi006 bash[27292]: cluster 2023-08-27T22:32:53.702610+0000 mgr.y (mgr.14901) 1031 : cluster [DBG] pgmap v1038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:54.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:54 smithi006 bash[33918]: cluster 2023-08-27T22:32:53.702610+0000 mgr.y (mgr.14901) 1031 : cluster [DBG] pgmap v1038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:54 smithi130 bash[27579]: cluster 2023-08-27T22:32:53.702610+0000 mgr.y (mgr.14901) 1031 : cluster [DBG] pgmap v1038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:55.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:55 smithi006 bash[27292]: audit 2023-08-27T22:32:54.940808+0000 mon.a (mon.0) 1360 : audit [DBG] from='client.? 172.21.15.6:0/3308200628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:55.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:55 smithi006 bash[33918]: audit 2023-08-27T22:32:54.940808+0000 mon.a (mon.0) 1360 : audit [DBG] from='client.? 172.21.15.6:0/3308200628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:56.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:55 smithi130 bash[27579]: audit 2023-08-27T22:32:54.940808+0000 mon.a (mon.0) 1360 : audit [DBG] from='client.? 172.21.15.6:0/3308200628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:56.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:56 smithi006 bash[33918]: cluster 2023-08-27T22:32:55.703788+0000 mgr.y (mgr.14901) 1032 : cluster [DBG] pgmap v1039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:56.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:56 smithi006 bash[27292]: cluster 2023-08-27T22:32:55.703788+0000 mgr.y (mgr.14901) 1032 : cluster [DBG] pgmap v1039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:56 smithi130 bash[27579]: cluster 2023-08-27T22:32:55.703788+0000 mgr.y (mgr.14901) 1032 : cluster [DBG] pgmap v1039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:57.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:57 smithi006 bash[27292]: audit 2023-08-27T22:32:57.360520+0000 mon.a (mon.0) 1361 : audit [DBG] from='client.? 172.21.15.6:0/3587538388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:57.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:57 smithi006 bash[33918]: audit 2023-08-27T22:32:57.360520+0000 mon.a (mon.0) 1361 : audit [DBG] from='client.? 172.21.15.6:0/3587538388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:58.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:57 smithi130 bash[27579]: audit 2023-08-27T22:32:57.360520+0000 mon.a (mon.0) 1361 : audit [DBG] from='client.? 172.21.15.6:0/3587538388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:58.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:32:58 smithi006 bash[27292]: cluster 2023-08-27T22:32:57.704427+0000 mgr.y (mgr.14901) 1033 : cluster [DBG] pgmap v1040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:58.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:32:58 smithi006 bash[33918]: cluster 2023-08-27T22:32:57.704427+0000 mgr.y (mgr.14901) 1033 : cluster [DBG] pgmap v1040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:32:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:32:58 smithi130 bash[27579]: cluster 2023-08-27T22:32:57.704427+0000 mgr.y (mgr.14901) 1033 : cluster [DBG] pgmap v1040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:00.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:33:00.225Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:00.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:33:00.227Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:00 smithi130 bash[27579]: cluster 2023-08-27T22:32:59.705147+0000 mgr.y (mgr.14901) 1034 : cluster [DBG] pgmap v1041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:00 smithi130 bash[27579]: audit 2023-08-27T22:32:59.767970+0000 mon.c (mon.2) 428 : audit [DBG] from='client.? 172.21.15.6:0/3748107739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:01.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:00 smithi006 bash[27292]: cluster 2023-08-27T22:32:59.705147+0000 mgr.y (mgr.14901) 1034 : cluster [DBG] pgmap v1041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:01.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:00 smithi006 bash[27292]: audit 2023-08-27T22:32:59.767970+0000 mon.c (mon.2) 428 : audit [DBG] from='client.? 172.21.15.6:0/3748107739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:01.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:33:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:33:00] "GET /metrics HTTP/1.1" 200 193075 "" "Prometheus/2.33.4" 2023-08-27T22:33:01.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:00 smithi006 bash[33918]: cluster 2023-08-27T22:32:59.705147+0000 mgr.y (mgr.14901) 1034 : cluster [DBG] pgmap v1041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:01.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:00 smithi006 bash[33918]: audit 2023-08-27T22:32:59.767970+0000 mon.c (mon.2) 428 : audit [DBG] from='client.? 172.21.15.6:0/3748107739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:33:01.788Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:33:01.790Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:02 smithi130 bash[27579]: cluster 2023-08-27T22:33:01.705723+0000 mgr.y (mgr.14901) 1035 : cluster [DBG] pgmap v1042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:02 smithi130 bash[27579]: audit 2023-08-27T22:33:02.184087+0000 mon.c (mon.2) 429 : audit [DBG] from='client.? 172.21.15.6:0/1921396917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:03.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:02 smithi006 bash[27292]: cluster 2023-08-27T22:33:01.705723+0000 mgr.y (mgr.14901) 1035 : cluster [DBG] pgmap v1042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:03.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:02 smithi006 bash[27292]: audit 2023-08-27T22:33:02.184087+0000 mon.c (mon.2) 429 : audit [DBG] from='client.? 172.21.15.6:0/1921396917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:03.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:02 smithi006 bash[33918]: cluster 2023-08-27T22:33:01.705723+0000 mgr.y (mgr.14901) 1035 : cluster [DBG] pgmap v1042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:03.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:02 smithi006 bash[33918]: audit 2023-08-27T22:33:02.184087+0000 mon.c (mon.2) 429 : audit [DBG] from='client.? 172.21.15.6:0/1921396917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:33:03.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:33:03.685Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:04.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:04 smithi006 bash[33918]: cluster 2023-08-27T22:33:03.706392+0000 mgr.y (mgr.14901) 1036 : cluster [DBG] pgmap v1043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:04.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:04 smithi006 bash[33918]: audit 2023-08-27T22:33:04.592655+0000 mon.c (mon.2) 430 : audit [DBG] from='client.? 172.21.15.6:0/481843949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:04.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:04 smithi006 bash[27292]: cluster 2023-08-27T22:33:03.706392+0000 mgr.y (mgr.14901) 1036 : cluster [DBG] pgmap v1043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:04.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:04 smithi006 bash[27292]: audit 2023-08-27T22:33:04.592655+0000 mon.c (mon.2) 430 : audit [DBG] from='client.? 172.21.15.6:0/481843949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:04 smithi130 bash[27579]: cluster 2023-08-27T22:33:03.706392+0000 mgr.y (mgr.14901) 1036 : cluster [DBG] pgmap v1043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:04 smithi130 bash[27579]: audit 2023-08-27T22:33:04.592655+0000 mon.c (mon.2) 430 : audit [DBG] from='client.? 172.21.15.6:0/481843949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:06 smithi130 bash[27579]: cluster 2023-08-27T22:33:05.707509+0000 mgr.y (mgr.14901) 1037 : cluster [DBG] pgmap v1044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:06 smithi130 bash[27579]: audit 2023-08-27T22:33:06.216793+0000 mon.b (mon.1) 223 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:33:07.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:06 smithi006 bash[33918]: cluster 2023-08-27T22:33:05.707509+0000 mgr.y (mgr.14901) 1037 : cluster [DBG] pgmap v1044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:07.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:06 smithi006 bash[33918]: audit 2023-08-27T22:33:06.216793+0000 mon.b (mon.1) 223 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:33:07.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:06 smithi006 bash[27292]: cluster 2023-08-27T22:33:05.707509+0000 mgr.y (mgr.14901) 1037 : cluster [DBG] pgmap v1044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:07.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:06 smithi006 bash[27292]: audit 2023-08-27T22:33:06.216793+0000 mon.b (mon.1) 223 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:33:08.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:07 smithi130 bash[27579]: audit 2023-08-27T22:33:07.009311+0000 mon.a (mon.0) 1362 : audit [DBG] from='client.? 172.21.15.6:0/1983648474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:08.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:07 smithi006 bash[33918]: audit 2023-08-27T22:33:07.009311+0000 mon.a (mon.0) 1362 : audit [DBG] from='client.? 172.21.15.6:0/1983648474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:08.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:07 smithi006 bash[27292]: audit 2023-08-27T22:33:07.009311+0000 mon.a (mon.0) 1362 : audit [DBG] from='client.? 172.21.15.6:0/1983648474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:08 smithi130 bash[27579]: cluster 2023-08-27T22:33:07.708127+0000 mgr.y (mgr.14901) 1038 : cluster [DBG] pgmap v1045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:09.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:08 smithi006 bash[33918]: cluster 2023-08-27T22:33:07.708127+0000 mgr.y (mgr.14901) 1038 : cluster [DBG] pgmap v1045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:09.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:08 smithi006 bash[27292]: cluster 2023-08-27T22:33:07.708127+0000 mgr.y (mgr.14901) 1038 : cluster [DBG] pgmap v1045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:10.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:09 smithi130 bash[27579]: audit 2023-08-27T22:33:09.427545+0000 mon.a (mon.0) 1363 : audit [DBG] from='client.? 172.21.15.6:0/186500892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:10.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:09 smithi006 bash[27292]: audit 2023-08-27T22:33:09.427545+0000 mon.a (mon.0) 1363 : audit [DBG] from='client.? 172.21.15.6:0/186500892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:10.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:09 smithi006 bash[33918]: audit 2023-08-27T22:33:09.427545+0000 mon.a (mon.0) 1363 : audit [DBG] from='client.? 172.21.15.6:0/186500892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:33:10.227Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:33:10.229Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:10 smithi130 bash[27579]: cluster 2023-08-27T22:33:09.709464+0000 mgr.y (mgr.14901) 1039 : cluster [DBG] pgmap v1046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:11.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:33:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:33:10] "GET /metrics HTTP/1.1" 200 193075 "" "Prometheus/2.33.4" 2023-08-27T22:33:11.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:10 smithi006 bash[33918]: cluster 2023-08-27T22:33:09.709464+0000 mgr.y (mgr.14901) 1039 : cluster [DBG] pgmap v1046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:11.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:10 smithi006 bash[27292]: cluster 2023-08-27T22:33:09.709464+0000 mgr.y (mgr.14901) 1039 : cluster [DBG] pgmap v1046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:33:11.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:33:11.791Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:13.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:12 smithi130 bash[27579]: cluster 2023-08-27T22:33:11.710157+0000 mgr.y (mgr.14901) 1040 : cluster [DBG] pgmap v1047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:13.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:12 smithi130 bash[27579]: audit 2023-08-27T22:33:11.880512+0000 mon.a (mon.0) 1364 : audit [DBG] from='client.? 172.21.15.6:0/2335268232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:13.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:12 smithi130 bash[27579]: audit 2023-08-27T22:33:12.046245+0000 mon.a (mon.0) 1365 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:33:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:12 smithi130 bash[27579]: audit 2023-08-27T22:33:12.520637+0000 mon.a (mon.0) 1366 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:33:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:12 smithi130 bash[27579]: audit 2023-08-27T22:33:12.799895+0000 mon.b (mon.1) 224 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:33:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:12 smithi130 bash[27579]: audit 2023-08-27T22:33:12.802277+0000 mon.b (mon.1) 225 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:33:13.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:12 smithi006 bash[33918]: cluster 2023-08-27T22:33:11.710157+0000 mgr.y (mgr.14901) 1040 : cluster [DBG] pgmap v1047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:13.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:12 smithi006 bash[33918]: audit 2023-08-27T22:33:11.880512+0000 mon.a (mon.0) 1364 : audit [DBG] from='client.? 172.21.15.6:0/2335268232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:13.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:12 smithi006 bash[33918]: audit 2023-08-27T22:33:12.046245+0000 mon.a (mon.0) 1365 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:33:13.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:12 smithi006 bash[33918]: audit 2023-08-27T22:33:12.520637+0000 mon.a (mon.0) 1366 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:33:13.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:12 smithi006 bash[33918]: audit 2023-08-27T22:33:12.799895+0000 mon.b (mon.1) 224 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:33:13.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:12 smithi006 bash[33918]: audit 2023-08-27T22:33:12.802277+0000 mon.b (mon.1) 225 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:33:13.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:12 smithi006 bash[27292]: cluster 2023-08-27T22:33:11.710157+0000 mgr.y (mgr.14901) 1040 : cluster [DBG] pgmap v1047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:13.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:12 smithi006 bash[27292]: audit 2023-08-27T22:33:11.880512+0000 mon.a (mon.0) 1364 : audit [DBG] from='client.? 172.21.15.6:0/2335268232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:13.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:12 smithi006 bash[27292]: audit 2023-08-27T22:33:12.046245+0000 mon.a (mon.0) 1365 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:33:13.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:12 smithi006 bash[27292]: audit 2023-08-27T22:33:12.520637+0000 mon.a (mon.0) 1366 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:33:13.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:12 smithi006 bash[27292]: audit 2023-08-27T22:33:12.799895+0000 mon.b (mon.1) 224 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:33:13.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:12 smithi006 bash[27292]: audit 2023-08-27T22:33:12.802277+0000 mon.b (mon.1) 225 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:33:13.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:13 smithi006 bash[33918]: audit 2023-08-27T22:33:12.810647+0000 mon.a (mon.0) 1367 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:33:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:33:13.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:33:13.685Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:13.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:13 smithi006 bash[27292]: audit 2023-08-27T22:33:12.810647+0000 mon.a (mon.0) 1367 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:33:14.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:13 smithi130 bash[27579]: audit 2023-08-27T22:33:12.810647+0000 mon.a (mon.0) 1367 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:33:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:14 smithi130 bash[27579]: cluster 2023-08-27T22:33:13.710840+0000 mgr.y (mgr.14901) 1041 : cluster [DBG] pgmap v1048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:14 smithi130 bash[27579]: audit 2023-08-27T22:33:14.305104+0000 mon.c (mon.2) 431 : audit [DBG] from='client.? 172.21.15.6:0/514287546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:15.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:14 smithi006 bash[33918]: cluster 2023-08-27T22:33:13.710840+0000 mgr.y (mgr.14901) 1041 : cluster [DBG] pgmap v1048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:15.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:14 smithi006 bash[33918]: audit 2023-08-27T22:33:14.305104+0000 mon.c (mon.2) 431 : audit [DBG] from='client.? 172.21.15.6:0/514287546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:15.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:14 smithi006 bash[27292]: cluster 2023-08-27T22:33:13.710840+0000 mgr.y (mgr.14901) 1041 : cluster [DBG] pgmap v1048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:15.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:14 smithi006 bash[27292]: audit 2023-08-27T22:33:14.305104+0000 mon.c (mon.2) 431 : audit [DBG] from='client.? 172.21.15.6:0/514287546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:16 smithi130 bash[27579]: cluster 2023-08-27T22:33:15.711955+0000 mgr.y (mgr.14901) 1042 : cluster [DBG] pgmap v1049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:16 smithi130 bash[27579]: audit 2023-08-27T22:33:16.717252+0000 mon.c (mon.2) 432 : audit [DBG] from='client.? 172.21.15.6:0/1313958845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:17.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:16 smithi006 bash[33918]: cluster 2023-08-27T22:33:15.711955+0000 mgr.y (mgr.14901) 1042 : cluster [DBG] pgmap v1049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:17.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:16 smithi006 bash[33918]: audit 2023-08-27T22:33:16.717252+0000 mon.c (mon.2) 432 : audit [DBG] from='client.? 172.21.15.6:0/1313958845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:17.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:16 smithi006 bash[27292]: cluster 2023-08-27T22:33:15.711955+0000 mgr.y (mgr.14901) 1042 : cluster [DBG] pgmap v1049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:17.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:16 smithi006 bash[27292]: audit 2023-08-27T22:33:16.717252+0000 mon.c (mon.2) 432 : audit [DBG] from='client.? 172.21.15.6:0/1313958845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:18 smithi130 bash[27579]: cluster 2023-08-27T22:33:17.712551+0000 mgr.y (mgr.14901) 1043 : cluster [DBG] pgmap v1050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:19.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:18 smithi006 bash[27292]: cluster 2023-08-27T22:33:17.712551+0000 mgr.y (mgr.14901) 1043 : cluster [DBG] pgmap v1050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:19.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:18 smithi006 bash[33918]: cluster 2023-08-27T22:33:17.712551+0000 mgr.y (mgr.14901) 1043 : cluster [DBG] pgmap v1050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:20.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:19 smithi130 bash[27579]: audit 2023-08-27T22:33:19.129480+0000 mon.c (mon.2) 433 : audit [DBG] from='client.? 172.21.15.6:0/1975365005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:20.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:19 smithi006 bash[33918]: audit 2023-08-27T22:33:19.129480+0000 mon.c (mon.2) 433 : audit [DBG] from='client.? 172.21.15.6:0/1975365005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:20.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:19 smithi006 bash[27292]: audit 2023-08-27T22:33:19.129480+0000 mon.c (mon.2) 433 : audit [DBG] from='client.? 172.21.15.6:0/1975365005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:33:20.227Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:33:20.230Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:20 smithi130 bash[27579]: cluster 2023-08-27T22:33:19.713842+0000 mgr.y (mgr.14901) 1044 : cluster [DBG] pgmap v1051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:21.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:20 smithi006 bash[27292]: cluster 2023-08-27T22:33:19.713842+0000 mgr.y (mgr.14901) 1044 : cluster [DBG] pgmap v1051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:21.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:33:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:33:20] "GET /metrics HTTP/1.1" 200 193080 "" "Prometheus/2.33.4" 2023-08-27T22:33:21.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:20 smithi006 bash[33918]: cluster 2023-08-27T22:33:19.713842+0000 mgr.y (mgr.14901) 1044 : cluster [DBG] pgmap v1051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:22.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:21 smithi130 bash[27579]: audit 2023-08-27T22:33:21.549408+0000 mon.a (mon.0) 1368 : audit [DBG] from='client.? 172.21.15.6:0/2391618903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:22.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:21 smithi006 bash[33918]: audit 2023-08-27T22:33:21.549408+0000 mon.a (mon.0) 1368 : audit [DBG] from='client.? 172.21.15.6:0/2391618903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:22.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:21 smithi006 bash[27292]: audit 2023-08-27T22:33:21.549408+0000 mon.a (mon.0) 1368 : audit [DBG] from='client.? 172.21.15.6:0/2391618903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:33:21.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:33:21.792Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:23.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:22 smithi130 bash[27579]: cluster 2023-08-27T22:33:21.714593+0000 mgr.y (mgr.14901) 1045 : cluster [DBG] pgmap v1052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:23.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:22 smithi006 bash[27292]: cluster 2023-08-27T22:33:21.714593+0000 mgr.y (mgr.14901) 1045 : cluster [DBG] pgmap v1052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:23.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:22 smithi006 bash[33918]: cluster 2023-08-27T22:33:21.714593+0000 mgr.y (mgr.14901) 1045 : cluster [DBG] pgmap v1052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:33:23.684Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:33:23.686Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:24.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:24 smithi006 bash[27292]: cluster 2023-08-27T22:33:23.715175+0000 mgr.y (mgr.14901) 1046 : cluster [DBG] pgmap v1053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:24.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:24 smithi006 bash[27292]: audit 2023-08-27T22:33:23.958671+0000 mon.a (mon.0) 1369 : audit [DBG] from='client.? 172.21.15.6:0/989883721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:24.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:24 smithi006 bash[33918]: cluster 2023-08-27T22:33:23.715175+0000 mgr.y (mgr.14901) 1046 : cluster [DBG] pgmap v1053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:24.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:24 smithi006 bash[33918]: audit 2023-08-27T22:33:23.958671+0000 mon.a (mon.0) 1369 : audit [DBG] from='client.? 172.21.15.6:0/989883721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:24 smithi130 bash[27579]: cluster 2023-08-27T22:33:23.715175+0000 mgr.y (mgr.14901) 1046 : cluster [DBG] pgmap v1053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:24 smithi130 bash[27579]: audit 2023-08-27T22:33:23.958671+0000 mon.a (mon.0) 1369 : audit [DBG] from='client.? 172.21.15.6:0/989883721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:26 smithi130 bash[27579]: cluster 2023-08-27T22:33:25.716321+0000 mgr.y (mgr.14901) 1047 : cluster [DBG] pgmap v1054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:26 smithi130 bash[27579]: audit 2023-08-27T22:33:26.375610+0000 mon.c (mon.2) 434 : audit [DBG] from='client.? 172.21.15.6:0/4282270256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:26 smithi006 bash[27292]: cluster 2023-08-27T22:33:25.716321+0000 mgr.y (mgr.14901) 1047 : cluster [DBG] pgmap v1054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:26 smithi006 bash[27292]: audit 2023-08-27T22:33:26.375610+0000 mon.c (mon.2) 434 : audit [DBG] from='client.? 172.21.15.6:0/4282270256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:27.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:26 smithi006 bash[33918]: cluster 2023-08-27T22:33:25.716321+0000 mgr.y (mgr.14901) 1047 : cluster [DBG] pgmap v1054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:27.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:26 smithi006 bash[33918]: audit 2023-08-27T22:33:26.375610+0000 mon.c (mon.2) 434 : audit [DBG] from='client.? 172.21.15.6:0/4282270256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:28 smithi130 bash[27579]: cluster 2023-08-27T22:33:27.717038+0000 mgr.y (mgr.14901) 1048 : cluster [DBG] pgmap v1055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:29.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:28 smithi006 bash[27292]: cluster 2023-08-27T22:33:27.717038+0000 mgr.y (mgr.14901) 1048 : cluster [DBG] pgmap v1055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:29.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:28 smithi006 bash[33918]: cluster 2023-08-27T22:33:27.717038+0000 mgr.y (mgr.14901) 1048 : cluster [DBG] pgmap v1055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:30.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:29 smithi130 bash[27579]: audit 2023-08-27T22:33:28.785282+0000 mon.a (mon.0) 1370 : audit [DBG] from='client.? 172.21.15.6:0/2686443648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:30.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:29 smithi006 bash[27292]: audit 2023-08-27T22:33:28.785282+0000 mon.a (mon.0) 1370 : audit [DBG] from='client.? 172.21.15.6:0/2686443648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:30.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:29 smithi006 bash[33918]: audit 2023-08-27T22:33:28.785282+0000 mon.a (mon.0) 1370 : audit [DBG] from='client.? 172.21.15.6:0/2686443648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:33:30.228Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:33:30.230Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:30 smithi130 bash[27579]: cluster 2023-08-27T22:33:29.718149+0000 mgr.y (mgr.14901) 1049 : cluster [DBG] pgmap v1056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:31.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:33:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:33:30] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:33:31.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:30 smithi006 bash[33918]: cluster 2023-08-27T22:33:29.718149+0000 mgr.y (mgr.14901) 1049 : cluster [DBG] pgmap v1056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:31.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:30 smithi006 bash[27292]: cluster 2023-08-27T22:33:29.718149+0000 mgr.y (mgr.14901) 1049 : cluster [DBG] pgmap v1056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:32.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:31 smithi130 bash[27579]: audit 2023-08-27T22:33:31.198875+0000 mon.a (mon.0) 1371 : audit [DBG] from='client.? 172.21.15.6:0/1294431760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:32.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:31 smithi006 bash[27292]: audit 2023-08-27T22:33:31.198875+0000 mon.a (mon.0) 1371 : audit [DBG] from='client.? 172.21.15.6:0/1294431760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:32.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:31 smithi006 bash[33918]: audit 2023-08-27T22:33:31.198875+0000 mon.a (mon.0) 1371 : audit [DBG] from='client.? 172.21.15.6:0/1294431760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:33:31.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:33:31.793Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:33.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:32 smithi130 bash[27579]: cluster 2023-08-27T22:33:31.718830+0000 mgr.y (mgr.14901) 1050 : cluster [DBG] pgmap v1057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:33.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:32 smithi006 bash[33918]: cluster 2023-08-27T22:33:31.718830+0000 mgr.y (mgr.14901) 1050 : cluster [DBG] pgmap v1057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:32 smithi006 bash[27292]: cluster 2023-08-27T22:33:31.718830+0000 mgr.y (mgr.14901) 1050 : cluster [DBG] pgmap v1057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:33.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:33 smithi006 bash[27292]: audit 2023-08-27T22:33:33.614117+0000 mon.c (mon.2) 435 : audit [DBG] from='client.? 172.21.15.6:0/4123761840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:33:33.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:33:33.687Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:33.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:33 smithi006 bash[33918]: audit 2023-08-27T22:33:33.614117+0000 mon.c (mon.2) 435 : audit [DBG] from='client.? 172.21.15.6:0/4123761840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:34.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:33 smithi130 bash[27579]: audit 2023-08-27T22:33:33.614117+0000 mon.c (mon.2) 435 : audit [DBG] from='client.? 172.21.15.6:0/4123761840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:34 smithi130 bash[27579]: cluster 2023-08-27T22:33:33.719422+0000 mgr.y (mgr.14901) 1051 : cluster [DBG] pgmap v1058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:35.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:34 smithi006 bash[27292]: cluster 2023-08-27T22:33:33.719422+0000 mgr.y (mgr.14901) 1051 : cluster [DBG] pgmap v1058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:35.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:34 smithi006 bash[33918]: cluster 2023-08-27T22:33:33.719422+0000 mgr.y (mgr.14901) 1051 : cluster [DBG] pgmap v1058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:36 smithi130 bash[27579]: cluster 2023-08-27T22:33:35.720189+0000 mgr.y (mgr.14901) 1052 : cluster [DBG] pgmap v1059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:36 smithi130 bash[27579]: audit 2023-08-27T22:33:36.029336+0000 mon.c (mon.2) 436 : audit [DBG] from='client.? 172.21.15.6:0/3106820350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:37.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:36 smithi006 bash[33918]: cluster 2023-08-27T22:33:35.720189+0000 mgr.y (mgr.14901) 1052 : cluster [DBG] pgmap v1059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:37.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:36 smithi006 bash[33918]: audit 2023-08-27T22:33:36.029336+0000 mon.c (mon.2) 436 : audit [DBG] from='client.? 172.21.15.6:0/3106820350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:37.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:36 smithi006 bash[27292]: cluster 2023-08-27T22:33:35.720189+0000 mgr.y (mgr.14901) 1052 : cluster [DBG] pgmap v1059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:37.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:36 smithi006 bash[27292]: audit 2023-08-27T22:33:36.029336+0000 mon.c (mon.2) 436 : audit [DBG] from='client.? 172.21.15.6:0/3106820350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:38 smithi130 bash[27579]: cluster 2023-08-27T22:33:37.720869+0000 mgr.y (mgr.14901) 1053 : cluster [DBG] pgmap v1060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:38 smithi130 bash[27579]: audit 2023-08-27T22:33:38.436968+0000 mon.c (mon.2) 437 : audit [DBG] from='client.? 172.21.15.6:0/2062949626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:39.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:38 smithi006 bash[33918]: cluster 2023-08-27T22:33:37.720869+0000 mgr.y (mgr.14901) 1053 : cluster [DBG] pgmap v1060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:39.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:38 smithi006 bash[33918]: audit 2023-08-27T22:33:38.436968+0000 mon.c (mon.2) 437 : audit [DBG] from='client.? 172.21.15.6:0/2062949626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:39.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:38 smithi006 bash[27292]: cluster 2023-08-27T22:33:37.720869+0000 mgr.y (mgr.14901) 1053 : cluster [DBG] pgmap v1060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:39.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:38 smithi006 bash[27292]: audit 2023-08-27T22:33:38.436968+0000 mon.c (mon.2) 437 : audit [DBG] from='client.? 172.21.15.6:0/2062949626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:33:40.229Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:33:40.231Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:40 smithi130 bash[27579]: cluster 2023-08-27T22:33:39.721898+0000 mgr.y (mgr.14901) 1054 : cluster [DBG] pgmap v1061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:41.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:33:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:33:40] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:33:41.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:40 smithi006 bash[33918]: cluster 2023-08-27T22:33:39.721898+0000 mgr.y (mgr.14901) 1054 : cluster [DBG] pgmap v1061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:41.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:40 smithi006 bash[27292]: cluster 2023-08-27T22:33:39.721898+0000 mgr.y (mgr.14901) 1054 : cluster [DBG] pgmap v1061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:42.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:41 smithi130 bash[27579]: audit 2023-08-27T22:33:40.861693+0000 mon.c (mon.2) 438 : audit [DBG] from='client.? 172.21.15.6:0/180000935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:42.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:41 smithi006 bash[33918]: audit 2023-08-27T22:33:40.861693+0000 mon.c (mon.2) 438 : audit [DBG] from='client.? 172.21.15.6:0/180000935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:42.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:41 smithi006 bash[27292]: audit 2023-08-27T22:33:40.861693+0000 mon.c (mon.2) 438 : audit [DBG] from='client.? 172.21.15.6:0/180000935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:33:41.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:33:41.794Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:43.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:42 smithi130 bash[27579]: cluster 2023-08-27T22:33:41.722544+0000 mgr.y (mgr.14901) 1055 : cluster [DBG] pgmap v1062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:43.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:42 smithi006 bash[27292]: cluster 2023-08-27T22:33:41.722544+0000 mgr.y (mgr.14901) 1055 : cluster [DBG] pgmap v1062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:43.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:42 smithi006 bash[33918]: cluster 2023-08-27T22:33:41.722544+0000 mgr.y (mgr.14901) 1055 : cluster [DBG] pgmap v1062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:43.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:43 smithi006 bash[27292]: audit 2023-08-27T22:33:43.282968+0000 mon.c (mon.2) 439 : audit [DBG] from='client.? 172.21.15.6:0/2144795546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:43.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:43 smithi006 bash[33918]: audit 2023-08-27T22:33:43.282968+0000 mon.c (mon.2) 439 : audit [DBG] from='client.? 172.21.15.6:0/2144795546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:33:43.686Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:43.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:33:43.688Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:44.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:43 smithi130 bash[27579]: audit 2023-08-27T22:33:43.282968+0000 mon.c (mon.2) 439 : audit [DBG] from='client.? 172.21.15.6:0/2144795546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:44 smithi130 bash[27579]: cluster 2023-08-27T22:33:43.723215+0000 mgr.y (mgr.14901) 1056 : cluster [DBG] pgmap v1063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:45.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:44 smithi006 bash[33918]: cluster 2023-08-27T22:33:43.723215+0000 mgr.y (mgr.14901) 1056 : cluster [DBG] pgmap v1063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:45.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:44 smithi006 bash[27292]: cluster 2023-08-27T22:33:43.723215+0000 mgr.y (mgr.14901) 1056 : cluster [DBG] pgmap v1063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:46.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:45 smithi130 bash[27579]: audit 2023-08-27T22:33:45.693982+0000 mon.a (mon.0) 1372 : audit [DBG] from='client.? 172.21.15.6:0/226167608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:46.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:45 smithi006 bash[27292]: audit 2023-08-27T22:33:45.693982+0000 mon.a (mon.0) 1372 : audit [DBG] from='client.? 172.21.15.6:0/226167608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:46.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:45 smithi006 bash[33918]: audit 2023-08-27T22:33:45.693982+0000 mon.a (mon.0) 1372 : audit [DBG] from='client.? 172.21.15.6:0/226167608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:46 smithi130 bash[27579]: cluster 2023-08-27T22:33:45.724251+0000 mgr.y (mgr.14901) 1057 : cluster [DBG] pgmap v1064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:47.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:46 smithi006 bash[33918]: cluster 2023-08-27T22:33:45.724251+0000 mgr.y (mgr.14901) 1057 : cluster [DBG] pgmap v1064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:47.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:46 smithi006 bash[27292]: cluster 2023-08-27T22:33:45.724251+0000 mgr.y (mgr.14901) 1057 : cluster [DBG] pgmap v1064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:49.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:48 smithi006 bash[33918]: cluster 2023-08-27T22:33:47.724907+0000 mgr.y (mgr.14901) 1058 : cluster [DBG] pgmap v1065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:49.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:48 smithi006 bash[33918]: audit 2023-08-27T22:33:48.106643+0000 mon.a (mon.0) 1373 : audit [DBG] from='client.? 172.21.15.6:0/3938692016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:49.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:48 smithi006 bash[27292]: cluster 2023-08-27T22:33:47.724907+0000 mgr.y (mgr.14901) 1058 : cluster [DBG] pgmap v1065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:49.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:48 smithi006 bash[27292]: audit 2023-08-27T22:33:48.106643+0000 mon.a (mon.0) 1373 : audit [DBG] from='client.? 172.21.15.6:0/3938692016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:49.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:48 smithi130 bash[27579]: cluster 2023-08-27T22:33:47.724907+0000 mgr.y (mgr.14901) 1058 : cluster [DBG] pgmap v1065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:49.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:48 smithi130 bash[27579]: audit 2023-08-27T22:33:48.106643+0000 mon.a (mon.0) 1373 : audit [DBG] from='client.? 172.21.15.6:0/3938692016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:33:50.229Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:33:50.231Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:51.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:33:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:33:50] "GET /metrics HTTP/1.1" 200 193077 "" "Prometheus/2.33.4" 2023-08-27T22:33:51.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:50 smithi006 bash[33918]: cluster 2023-08-27T22:33:49.725942+0000 mgr.y (mgr.14901) 1059 : cluster [DBG] pgmap v1066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:51.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:50 smithi006 bash[33918]: audit 2023-08-27T22:33:50.530484+0000 mon.a (mon.0) 1374 : audit [DBG] from='client.? 172.21.15.6:0/584578227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:51.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:50 smithi006 bash[27292]: cluster 2023-08-27T22:33:49.725942+0000 mgr.y (mgr.14901) 1059 : cluster [DBG] pgmap v1066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:51.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:50 smithi006 bash[27292]: audit 2023-08-27T22:33:50.530484+0000 mon.a (mon.0) 1374 : audit [DBG] from='client.? 172.21.15.6:0/584578227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:51.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:50 smithi130 bash[27579]: cluster 2023-08-27T22:33:49.725942+0000 mgr.y (mgr.14901) 1059 : cluster [DBG] pgmap v1066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:51.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:50 smithi130 bash[27579]: audit 2023-08-27T22:33:50.530484+0000 mon.a (mon.0) 1374 : audit [DBG] from='client.? 172.21.15.6:0/584578227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:33:51.793Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:33:51.795Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:53.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:52 smithi006 bash[27292]: cluster 2023-08-27T22:33:51.726570+0000 mgr.y (mgr.14901) 1060 : cluster [DBG] pgmap v1067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:53.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:52 smithi006 bash[33918]: cluster 2023-08-27T22:33:51.726570+0000 mgr.y (mgr.14901) 1060 : cluster [DBG] pgmap v1067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:53.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:52 smithi130 bash[27579]: cluster 2023-08-27T22:33:51.726570+0000 mgr.y (mgr.14901) 1060 : cluster [DBG] pgmap v1067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:53.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:53 smithi006 bash[27292]: audit 2023-08-27T22:33:52.946666+0000 mon.a (mon.0) 1375 : audit [DBG] from='client.? 172.21.15.6:0/3658337295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:53.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:53 smithi006 bash[33918]: audit 2023-08-27T22:33:52.946666+0000 mon.a (mon.0) 1375 : audit [DBG] from='client.? 172.21.15.6:0/3658337295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:33:53.687Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:33:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:33:53.689Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:33:54.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:53 smithi130 bash[27579]: audit 2023-08-27T22:33:52.946666+0000 mon.a (mon.0) 1375 : audit [DBG] from='client.? 172.21.15.6:0/3658337295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:55.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:54 smithi006 bash[27292]: cluster 2023-08-27T22:33:53.727306+0000 mgr.y (mgr.14901) 1061 : cluster [DBG] pgmap v1068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:55.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:54 smithi006 bash[33918]: cluster 2023-08-27T22:33:53.727306+0000 mgr.y (mgr.14901) 1061 : cluster [DBG] pgmap v1068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:55.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:54 smithi130 bash[27579]: cluster 2023-08-27T22:33:53.727306+0000 mgr.y (mgr.14901) 1061 : cluster [DBG] pgmap v1068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:56.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:55 smithi006 bash[27292]: audit 2023-08-27T22:33:55.354765+0000 mon.c (mon.2) 440 : audit [DBG] from='client.? 172.21.15.6:0/769337190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:56.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:55 smithi006 bash[33918]: audit 2023-08-27T22:33:55.354765+0000 mon.c (mon.2) 440 : audit [DBG] from='client.? 172.21.15.6:0/769337190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:56.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:55 smithi130 bash[27579]: audit 2023-08-27T22:33:55.354765+0000 mon.c (mon.2) 440 : audit [DBG] from='client.? 172.21.15.6:0/769337190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:57.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:56 smithi006 bash[27292]: cluster 2023-08-27T22:33:55.728419+0000 mgr.y (mgr.14901) 1062 : cluster [DBG] pgmap v1069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:57.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:56 smithi006 bash[33918]: cluster 2023-08-27T22:33:55.728419+0000 mgr.y (mgr.14901) 1062 : cluster [DBG] pgmap v1069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:57.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:56 smithi130 bash[27579]: cluster 2023-08-27T22:33:55.728419+0000 mgr.y (mgr.14901) 1062 : cluster [DBG] pgmap v1069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:58.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:57 smithi006 bash[33918]: audit 2023-08-27T22:33:57.774493+0000 mon.c (mon.2) 441 : audit [DBG] from='client.? 172.21.15.6:0/1245233373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:58.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:57 smithi006 bash[27292]: audit 2023-08-27T22:33:57.774493+0000 mon.c (mon.2) 441 : audit [DBG] from='client.? 172.21.15.6:0/1245233373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:58.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:57 smithi130 bash[27579]: audit 2023-08-27T22:33:57.774493+0000 mon.c (mon.2) 441 : audit [DBG] from='client.? 172.21.15.6:0/1245233373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:59.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:33:58 smithi006 bash[27292]: cluster 2023-08-27T22:33:57.729067+0000 mgr.y (mgr.14901) 1063 : cluster [DBG] pgmap v1070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:59.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:33:58 smithi006 bash[33918]: cluster 2023-08-27T22:33:57.729067+0000 mgr.y (mgr.14901) 1063 : cluster [DBG] pgmap v1070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:33:59.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:33:58 smithi130 bash[27579]: cluster 2023-08-27T22:33:57.729067+0000 mgr.y (mgr.14901) 1063 : cluster [DBG] pgmap v1070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:00.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:34:00.230Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:00.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:34:00.232Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:01.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:34:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:34:00] "GET /metrics HTTP/1.1" 200 193080 "" "Prometheus/2.33.4" 2023-08-27T22:34:01.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:00 smithi006 bash[27292]: cluster 2023-08-27T22:33:59.730169+0000 mgr.y (mgr.14901) 1064 : cluster [DBG] pgmap v1071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:01.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:00 smithi006 bash[27292]: audit 2023-08-27T22:34:00.187407+0000 mon.a (mon.0) 1376 : audit [DBG] from='client.? 172.21.15.6:0/685129286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:01.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:00 smithi006 bash[33918]: cluster 2023-08-27T22:33:59.730169+0000 mgr.y (mgr.14901) 1064 : cluster [DBG] pgmap v1071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:01.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:00 smithi006 bash[33918]: audit 2023-08-27T22:34:00.187407+0000 mon.a (mon.0) 1376 : audit [DBG] from='client.? 172.21.15.6:0/685129286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:01.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:00 smithi130 bash[27579]: cluster 2023-08-27T22:33:59.730169+0000 mgr.y (mgr.14901) 1064 : cluster [DBG] pgmap v1071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:01.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:00 smithi130 bash[27579]: audit 2023-08-27T22:34:00.187407+0000 mon.a (mon.0) 1376 : audit [DBG] from='client.? 172.21.15.6:0/685129286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:34:01.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:34:01.796Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:03.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:02 smithi006 bash[33918]: cluster 2023-08-27T22:34:01.730841+0000 mgr.y (mgr.14901) 1065 : cluster [DBG] pgmap v1072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:03.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:02 smithi006 bash[33918]: audit 2023-08-27T22:34:02.601201+0000 mon.a (mon.0) 1377 : audit [DBG] from='client.? 172.21.15.6:0/1943353263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:03.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:02 smithi006 bash[27292]: cluster 2023-08-27T22:34:01.730841+0000 mgr.y (mgr.14901) 1065 : cluster [DBG] pgmap v1072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:03.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:02 smithi006 bash[27292]: audit 2023-08-27T22:34:02.601201+0000 mon.a (mon.0) 1377 : audit [DBG] from='client.? 172.21.15.6:0/1943353263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:03.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:02 smithi130 bash[27579]: cluster 2023-08-27T22:34:01.730841+0000 mgr.y (mgr.14901) 1065 : cluster [DBG] pgmap v1072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:03.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:02 smithi130 bash[27579]: audit 2023-08-27T22:34:02.601201+0000 mon.a (mon.0) 1377 : audit [DBG] from='client.? 172.21.15.6:0/1943353263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:34:03.688Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:34:03.690Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:04.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:04 smithi006 bash[33918]: cluster 2023-08-27T22:34:03.731495+0000 mgr.y (mgr.14901) 1066 : cluster [DBG] pgmap v1073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:04.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:04 smithi006 bash[27292]: cluster 2023-08-27T22:34:03.731495+0000 mgr.y (mgr.14901) 1066 : cluster [DBG] pgmap v1073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:04 smithi130 bash[27579]: cluster 2023-08-27T22:34:03.731495+0000 mgr.y (mgr.14901) 1066 : cluster [DBG] pgmap v1073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:05.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:05 smithi006 bash[27292]: audit 2023-08-27T22:34:05.024378+0000 mon.a (mon.0) 1378 : audit [DBG] from='client.? 172.21.15.6:0/3673388043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:05.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:05 smithi006 bash[33918]: audit 2023-08-27T22:34:05.024378+0000 mon.a (mon.0) 1378 : audit [DBG] from='client.? 172.21.15.6:0/3673388043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:06.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:05 smithi130 bash[27579]: audit 2023-08-27T22:34:05.024378+0000 mon.a (mon.0) 1378 : audit [DBG] from='client.? 172.21.15.6:0/3673388043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:06 smithi130 bash[27579]: cluster 2023-08-27T22:34:05.732619+0000 mgr.y (mgr.14901) 1067 : cluster [DBG] pgmap v1074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:07.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:06 smithi006 bash[27292]: cluster 2023-08-27T22:34:05.732619+0000 mgr.y (mgr.14901) 1067 : cluster [DBG] pgmap v1074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:07.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:06 smithi006 bash[33918]: cluster 2023-08-27T22:34:05.732619+0000 mgr.y (mgr.14901) 1067 : cluster [DBG] pgmap v1074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:08.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:07 smithi130 bash[27579]: audit 2023-08-27T22:34:07.432008+0000 mon.a (mon.0) 1379 : audit [DBG] from='client.? 172.21.15.6:0/1967074319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:08.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:07 smithi006 bash[33918]: audit 2023-08-27T22:34:07.432008+0000 mon.a (mon.0) 1379 : audit [DBG] from='client.? 172.21.15.6:0/1967074319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:08.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:07 smithi006 bash[27292]: audit 2023-08-27T22:34:07.432008+0000 mon.a (mon.0) 1379 : audit [DBG] from='client.? 172.21.15.6:0/1967074319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:08 smithi130 bash[27579]: cluster 2023-08-27T22:34:07.733237+0000 mgr.y (mgr.14901) 1068 : cluster [DBG] pgmap v1075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:09.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:08 smithi006 bash[27292]: cluster 2023-08-27T22:34:07.733237+0000 mgr.y (mgr.14901) 1068 : cluster [DBG] pgmap v1075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:09.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:08 smithi006 bash[33918]: cluster 2023-08-27T22:34:07.733237+0000 mgr.y (mgr.14901) 1068 : cluster [DBG] pgmap v1075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:34:10.230Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:34:10.232Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:10 smithi130 bash[27579]: cluster 2023-08-27T22:34:09.734264+0000 mgr.y (mgr.14901) 1069 : cluster [DBG] pgmap v1076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:10 smithi130 bash[27579]: audit 2023-08-27T22:34:09.850642+0000 mon.c (mon.2) 442 : audit [DBG] from='client.? 172.21.15.6:0/695510380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:11.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:10 smithi006 bash[27292]: cluster 2023-08-27T22:34:09.734264+0000 mgr.y (mgr.14901) 1069 : cluster [DBG] pgmap v1076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:11.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:10 smithi006 bash[27292]: audit 2023-08-27T22:34:09.850642+0000 mon.c (mon.2) 442 : audit [DBG] from='client.? 172.21.15.6:0/695510380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:11.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:34:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:34:10] "GET /metrics HTTP/1.1" 200 193080 "" "Prometheus/2.33.4" 2023-08-27T22:34:11.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:10 smithi006 bash[33918]: cluster 2023-08-27T22:34:09.734264+0000 mgr.y (mgr.14901) 1069 : cluster [DBG] pgmap v1076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:11.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:10 smithi006 bash[33918]: audit 2023-08-27T22:34:09.850642+0000 mon.c (mon.2) 442 : audit [DBG] from='client.? 172.21.15.6:0/695510380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:34:11.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:34:11.797Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:13.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:12 smithi130 bash[27579]: cluster 2023-08-27T22:34:11.734941+0000 mgr.y (mgr.14901) 1070 : cluster [DBG] pgmap v1077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:13.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:12 smithi130 bash[27579]: audit 2023-08-27T22:34:12.277674+0000 mon.a (mon.0) 1380 : audit [DBG] from='client.? 172.21.15.6:0/2756313540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:13.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:12 smithi006 bash[33918]: cluster 2023-08-27T22:34:11.734941+0000 mgr.y (mgr.14901) 1070 : cluster [DBG] pgmap v1077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:13.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:12 smithi006 bash[33918]: audit 2023-08-27T22:34:12.277674+0000 mon.a (mon.0) 1380 : audit [DBG] from='client.? 172.21.15.6:0/2756313540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:13.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:12 smithi006 bash[27292]: cluster 2023-08-27T22:34:11.734941+0000 mgr.y (mgr.14901) 1070 : cluster [DBG] pgmap v1077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:13.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:12 smithi006 bash[27292]: audit 2023-08-27T22:34:12.277674+0000 mon.a (mon.0) 1380 : audit [DBG] from='client.? 172.21.15.6:0/2756313540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:13.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:13 smithi006 bash[27292]: audit 2023-08-27T22:34:12.826134+0000 mon.b (mon.1) 226 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:34:13.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:13 smithi006 bash[33918]: audit 2023-08-27T22:34:12.826134+0000 mon.b (mon.1) 226 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:34:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:34:13.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:13.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:34:13.691Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:14.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:13 smithi130 bash[27579]: audit 2023-08-27T22:34:12.826134+0000 mon.b (mon.1) 226 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:34:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:14 smithi130 bash[27579]: cluster 2023-08-27T22:34:13.735630+0000 mgr.y (mgr.14901) 1071 : cluster [DBG] pgmap v1078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:14 smithi130 bash[27579]: audit 2023-08-27T22:34:14.703537+0000 mon.c (mon.2) 443 : audit [DBG] from='client.? 172.21.15.6:0/778792990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:15.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:14 smithi006 bash[33918]: cluster 2023-08-27T22:34:13.735630+0000 mgr.y (mgr.14901) 1071 : cluster [DBG] pgmap v1078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:15.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:14 smithi006 bash[33918]: audit 2023-08-27T22:34:14.703537+0000 mon.c (mon.2) 443 : audit [DBG] from='client.? 172.21.15.6:0/778792990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:15.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:14 smithi006 bash[27292]: cluster 2023-08-27T22:34:13.735630+0000 mgr.y (mgr.14901) 1071 : cluster [DBG] pgmap v1078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:15.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:14 smithi006 bash[27292]: audit 2023-08-27T22:34:14.703537+0000 mon.c (mon.2) 443 : audit [DBG] from='client.? 172.21.15.6:0/778792990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:16 smithi130 bash[27579]: cluster 2023-08-27T22:34:15.736802+0000 mgr.y (mgr.14901) 1072 : cluster [DBG] pgmap v1079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:17.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:16 smithi006 bash[27292]: cluster 2023-08-27T22:34:15.736802+0000 mgr.y (mgr.14901) 1072 : cluster [DBG] pgmap v1079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:17.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:16 smithi006 bash[33918]: cluster 2023-08-27T22:34:15.736802+0000 mgr.y (mgr.14901) 1072 : cluster [DBG] pgmap v1079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:18.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:17 smithi130 bash[27579]: audit 2023-08-27T22:34:17.126529+0000 mon.c (mon.2) 444 : audit [DBG] from='client.? 172.21.15.6:0/4702765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:18.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:17 smithi006 bash[33918]: audit 2023-08-27T22:34:17.126529+0000 mon.c (mon.2) 444 : audit [DBG] from='client.? 172.21.15.6:0/4702765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:18.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:17 smithi006 bash[27292]: audit 2023-08-27T22:34:17.126529+0000 mon.c (mon.2) 444 : audit [DBG] from='client.? 172.21.15.6:0/4702765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:18 smithi130 bash[27579]: cluster 2023-08-27T22:34:17.737289+0000 mgr.y (mgr.14901) 1073 : cluster [DBG] pgmap v1080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:18 smithi130 bash[27579]: audit 2023-08-27T22:34:18.722803+0000 mon.a (mon.0) 1381 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:34:19.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:18 smithi006 bash[27292]: cluster 2023-08-27T22:34:17.737289+0000 mgr.y (mgr.14901) 1073 : cluster [DBG] pgmap v1080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:19.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:18 smithi006 bash[27292]: audit 2023-08-27T22:34:18.722803+0000 mon.a (mon.0) 1381 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:34:19.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:18 smithi006 bash[33918]: cluster 2023-08-27T22:34:17.737289+0000 mgr.y (mgr.14901) 1073 : cluster [DBG] pgmap v1080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:19.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:18 smithi006 bash[33918]: audit 2023-08-27T22:34:18.722803+0000 mon.a (mon.0) 1381 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:34:20.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:34:20.231Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:20.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:34:20.233Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:20.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:20 smithi006 bash[33918]: audit 2023-08-27T22:34:19.152945+0000 mon.a (mon.0) 1382 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:34:20.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:20 smithi006 bash[33918]: audit 2023-08-27T22:34:19.425971+0000 mon.b (mon.1) 227 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:34:20.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:20 smithi006 bash[33918]: audit 2023-08-27T22:34:19.427666+0000 mon.b (mon.1) 228 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:34:20.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:20 smithi006 bash[33918]: audit 2023-08-27T22:34:19.430108+0000 mon.a (mon.0) 1383 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:34:20.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:20 smithi006 bash[33918]: audit 2023-08-27T22:34:19.543684+0000 mon.a (mon.0) 1384 : audit [DBG] from='client.? 172.21.15.6:0/3055200771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:20.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:20 smithi006 bash[27292]: audit 2023-08-27T22:34:19.152945+0000 mon.a (mon.0) 1382 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:34:20.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:20 smithi006 bash[27292]: audit 2023-08-27T22:34:19.425971+0000 mon.b (mon.1) 227 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:34:20.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:20 smithi006 bash[27292]: audit 2023-08-27T22:34:19.427666+0000 mon.b (mon.1) 228 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:34:20.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:20 smithi006 bash[27292]: audit 2023-08-27T22:34:19.430108+0000 mon.a (mon.0) 1383 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:34:20.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:20 smithi006 bash[27292]: audit 2023-08-27T22:34:19.543684+0000 mon.a (mon.0) 1384 : audit [DBG] from='client.? 172.21.15.6:0/3055200771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:20.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:20 smithi130 bash[27579]: audit 2023-08-27T22:34:19.152945+0000 mon.a (mon.0) 1382 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:34:20.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:20 smithi130 bash[27579]: audit 2023-08-27T22:34:19.425971+0000 mon.b (mon.1) 227 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:34:20.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:20 smithi130 bash[27579]: audit 2023-08-27T22:34:19.427666+0000 mon.b (mon.1) 228 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:34:20.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:20 smithi130 bash[27579]: audit 2023-08-27T22:34:19.430108+0000 mon.a (mon.0) 1383 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:34:20.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:20 smithi130 bash[27579]: audit 2023-08-27T22:34:19.543684+0000 mon.a (mon.0) 1384 : audit [DBG] from='client.? 172.21.15.6:0/3055200771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:21.161 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:34:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:34:20] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:34:21.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:21 smithi006 bash[27292]: cluster 2023-08-27T22:34:19.738339+0000 mgr.y (mgr.14901) 1074 : cluster [DBG] pgmap v1081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:21.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:21 smithi006 bash[33918]: cluster 2023-08-27T22:34:19.738339+0000 mgr.y (mgr.14901) 1074 : cluster [DBG] pgmap v1081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:21.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:21 smithi130 bash[27579]: cluster 2023-08-27T22:34:19.738339+0000 mgr.y (mgr.14901) 1074 : cluster [DBG] pgmap v1081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:22.170 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:34:21.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:22.170 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:34:21.798Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:22.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:22 smithi006 bash[27292]: audit 2023-08-27T22:34:21.957071+0000 mon.c (mon.2) 445 : audit [DBG] from='client.? 172.21.15.6:0/875855882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:22.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:22 smithi006 bash[33918]: audit 2023-08-27T22:34:21.957071+0000 mon.c (mon.2) 445 : audit [DBG] from='client.? 172.21.15.6:0/875855882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:22.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:22 smithi130 bash[27579]: audit 2023-08-27T22:34:21.957071+0000 mon.c (mon.2) 445 : audit [DBG] from='client.? 172.21.15.6:0/875855882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:23.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:23 smithi006 bash[27292]: cluster 2023-08-27T22:34:21.739000+0000 mgr.y (mgr.14901) 1075 : cluster [DBG] pgmap v1082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:23.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:23 smithi006 bash[33918]: cluster 2023-08-27T22:34:21.739000+0000 mgr.y (mgr.14901) 1075 : cluster [DBG] pgmap v1082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:23.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:23 smithi130 bash[27579]: cluster 2023-08-27T22:34:21.739000+0000 mgr.y (mgr.14901) 1075 : cluster [DBG] pgmap v1082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:34:23.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:34:23.692Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:24.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:24 smithi006 bash[27292]: cluster 2023-08-27T22:34:23.739759+0000 mgr.y (mgr.14901) 1076 : cluster [DBG] pgmap v1083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:24.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:24 smithi006 bash[27292]: audit 2023-08-27T22:34:24.370976+0000 mon.a (mon.0) 1385 : audit [DBG] from='client.? 172.21.15.6:0/2485653869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:24.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:24 smithi006 bash[33918]: cluster 2023-08-27T22:34:23.739759+0000 mgr.y (mgr.14901) 1076 : cluster [DBG] pgmap v1083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:24.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:24 smithi006 bash[33918]: audit 2023-08-27T22:34:24.370976+0000 mon.a (mon.0) 1385 : audit [DBG] from='client.? 172.21.15.6:0/2485653869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:24 smithi130 bash[27579]: cluster 2023-08-27T22:34:23.739759+0000 mgr.y (mgr.14901) 1076 : cluster [DBG] pgmap v1083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:24 smithi130 bash[27579]: audit 2023-08-27T22:34:24.370976+0000 mon.a (mon.0) 1385 : audit [DBG] from='client.? 172.21.15.6:0/2485653869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:26 smithi130 bash[27579]: cluster 2023-08-27T22:34:25.740912+0000 mgr.y (mgr.14901) 1077 : cluster [DBG] pgmap v1084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:26 smithi130 bash[27579]: audit 2023-08-27T22:34:26.780089+0000 mon.a (mon.0) 1386 : audit [DBG] from='client.? 172.21.15.6:0/2711981523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:26 smithi006 bash[27292]: cluster 2023-08-27T22:34:25.740912+0000 mgr.y (mgr.14901) 1077 : cluster [DBG] pgmap v1084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:26 smithi006 bash[27292]: audit 2023-08-27T22:34:26.780089+0000 mon.a (mon.0) 1386 : audit [DBG] from='client.? 172.21.15.6:0/2711981523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:27.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:26 smithi006 bash[33918]: cluster 2023-08-27T22:34:25.740912+0000 mgr.y (mgr.14901) 1077 : cluster [DBG] pgmap v1084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:27.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:26 smithi006 bash[33918]: audit 2023-08-27T22:34:26.780089+0000 mon.a (mon.0) 1386 : audit [DBG] from='client.? 172.21.15.6:0/2711981523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:28 smithi130 bash[27579]: cluster 2023-08-27T22:34:27.741505+0000 mgr.y (mgr.14901) 1078 : cluster [DBG] pgmap v1085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:29.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:28 smithi006 bash[27292]: cluster 2023-08-27T22:34:27.741505+0000 mgr.y (mgr.14901) 1078 : cluster [DBG] pgmap v1085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:29.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:28 smithi006 bash[33918]: cluster 2023-08-27T22:34:27.741505+0000 mgr.y (mgr.14901) 1078 : cluster [DBG] pgmap v1085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:30.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:29 smithi130 bash[27579]: audit 2023-08-27T22:34:29.198871+0000 mon.a (mon.0) 1387 : audit [DBG] from='client.? 172.21.15.6:0/65546681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:30.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:29 smithi006 bash[27292]: audit 2023-08-27T22:34:29.198871+0000 mon.a (mon.0) 1387 : audit [DBG] from='client.? 172.21.15.6:0/65546681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:30.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:29 smithi006 bash[33918]: audit 2023-08-27T22:34:29.198871+0000 mon.a (mon.0) 1387 : audit [DBG] from='client.? 172.21.15.6:0/65546681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:34:30.232Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:34:30.234Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:30 smithi130 bash[27579]: cluster 2023-08-27T22:34:29.742616+0000 mgr.y (mgr.14901) 1079 : cluster [DBG] pgmap v1086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:31.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:34:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:34:30] "GET /metrics HTTP/1.1" 200 193079 "" "Prometheus/2.33.4" 2023-08-27T22:34:31.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:30 smithi006 bash[27292]: cluster 2023-08-27T22:34:29.742616+0000 mgr.y (mgr.14901) 1079 : cluster [DBG] pgmap v1086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:31.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:30 smithi006 bash[33918]: cluster 2023-08-27T22:34:29.742616+0000 mgr.y (mgr.14901) 1079 : cluster [DBG] pgmap v1086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:32.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:31 smithi130 bash[27579]: audit 2023-08-27T22:34:31.619679+0000 mon.c (mon.2) 446 : audit [DBG] from='client.? 172.21.15.6:0/467530017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:32.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:31 smithi006 bash[33918]: audit 2023-08-27T22:34:31.619679+0000 mon.c (mon.2) 446 : audit [DBG] from='client.? 172.21.15.6:0/467530017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:32.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:31 smithi006 bash[27292]: audit 2023-08-27T22:34:31.619679+0000 mon.c (mon.2) 446 : audit [DBG] from='client.? 172.21.15.6:0/467530017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:34:31.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:34:31.799Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:33.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:32 smithi130 bash[27579]: cluster 2023-08-27T22:34:31.743304+0000 mgr.y (mgr.14901) 1080 : cluster [DBG] pgmap v1087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:33.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:32 smithi006 bash[33918]: cluster 2023-08-27T22:34:31.743304+0000 mgr.y (mgr.14901) 1080 : cluster [DBG] pgmap v1087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:32 smithi006 bash[27292]: cluster 2023-08-27T22:34:31.743304+0000 mgr.y (mgr.14901) 1080 : cluster [DBG] pgmap v1087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:34:33.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:34:33.693Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:34.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:34 smithi006 bash[27292]: cluster 2023-08-27T22:34:33.743834+0000 mgr.y (mgr.14901) 1081 : cluster [DBG] pgmap v1088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:34.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:34 smithi006 bash[27292]: audit 2023-08-27T22:34:34.040471+0000 mon.a (mon.0) 1388 : audit [DBG] from='client.? 172.21.15.6:0/136694890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:34.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:34 smithi006 bash[33918]: cluster 2023-08-27T22:34:33.743834+0000 mgr.y (mgr.14901) 1081 : cluster [DBG] pgmap v1088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:34.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:34 smithi006 bash[33918]: audit 2023-08-27T22:34:34.040471+0000 mon.a (mon.0) 1388 : audit [DBG] from='client.? 172.21.15.6:0/136694890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:34 smithi130 bash[27579]: cluster 2023-08-27T22:34:33.743834+0000 mgr.y (mgr.14901) 1081 : cluster [DBG] pgmap v1088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:34 smithi130 bash[27579]: audit 2023-08-27T22:34:34.040471+0000 mon.a (mon.0) 1388 : audit [DBG] from='client.? 172.21.15.6:0/136694890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:36 smithi130 bash[27579]: cluster 2023-08-27T22:34:35.744937+0000 mgr.y (mgr.14901) 1082 : cluster [DBG] pgmap v1089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:36 smithi130 bash[27579]: audit 2023-08-27T22:34:36.454482+0000 mon.c (mon.2) 447 : audit [DBG] from='client.? 172.21.15.6:0/4242194679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:37.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:36 smithi006 bash[27292]: cluster 2023-08-27T22:34:35.744937+0000 mgr.y (mgr.14901) 1082 : cluster [DBG] pgmap v1089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:37.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:36 smithi006 bash[27292]: audit 2023-08-27T22:34:36.454482+0000 mon.c (mon.2) 447 : audit [DBG] from='client.? 172.21.15.6:0/4242194679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:37.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:36 smithi006 bash[33918]: cluster 2023-08-27T22:34:35.744937+0000 mgr.y (mgr.14901) 1082 : cluster [DBG] pgmap v1089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:37.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:36 smithi006 bash[33918]: audit 2023-08-27T22:34:36.454482+0000 mon.c (mon.2) 447 : audit [DBG] from='client.? 172.21.15.6:0/4242194679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:38 smithi130 bash[27579]: cluster 2023-08-27T22:34:37.745615+0000 mgr.y (mgr.14901) 1083 : cluster [DBG] pgmap v1090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:39.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:38 smithi006 bash[33918]: cluster 2023-08-27T22:34:37.745615+0000 mgr.y (mgr.14901) 1083 : cluster [DBG] pgmap v1090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:39.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:38 smithi006 bash[27292]: cluster 2023-08-27T22:34:37.745615+0000 mgr.y (mgr.14901) 1083 : cluster [DBG] pgmap v1090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:40.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:39 smithi130 bash[27579]: audit 2023-08-27T22:34:38.866570+0000 mon.a (mon.0) 1389 : audit [DBG] from='client.? 172.21.15.6:0/2248503227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:40.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:39 smithi006 bash[33918]: audit 2023-08-27T22:34:38.866570+0000 mon.a (mon.0) 1389 : audit [DBG] from='client.? 172.21.15.6:0/2248503227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:40.590 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:39 smithi006 bash[27292]: audit 2023-08-27T22:34:38.866570+0000 mon.a (mon.0) 1389 : audit [DBG] from='client.? 172.21.15.6:0/2248503227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:34:40.233Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:40.748 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:34:40.235Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:40 smithi130 bash[27579]: cluster 2023-08-27T22:34:39.746701+0000 mgr.y (mgr.14901) 1084 : cluster [DBG] pgmap v1091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:41.251 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:40 smithi006 bash[27292]: cluster 2023-08-27T22:34:39.746701+0000 mgr.y (mgr.14901) 1084 : cluster [DBG] pgmap v1091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:41.255 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:34:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:34:40] "GET /metrics HTTP/1.1" 200 193079 "" "Prometheus/2.33.4" 2023-08-27T22:34:41.255 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:40 smithi006 bash[33918]: cluster 2023-08-27T22:34:39.746701+0000 mgr.y (mgr.14901) 1084 : cluster [DBG] pgmap v1091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:42.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:41 smithi130 bash[27579]: audit 2023-08-27T22:34:41.280217+0000 mon.a (mon.0) 1390 : audit [DBG] from='client.? 172.21.15.6:0/595668672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:44.287 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:41 smithi006 bash[33918]: audit 2023-08-27T22:34:41.280217+0000 mon.a (mon.0) 1390 : audit [DBG] from='client.? 172.21.15.6:0/595668672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:44.287 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:41 smithi006 bash[27292]: audit 2023-08-27T22:34:41.280217+0000 mon.a (mon.0) 1390 : audit [DBG] from='client.? 172.21.15.6:0/595668672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:44.287 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:42 smithi130 bash[27579]: cluster 2023-08-27T22:34:41.747320+0000 mgr.y (mgr.14901) 1085 : cluster [DBG] pgmap v1092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:44.288 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:43 smithi130 bash[27579]: audit 2023-08-27T22:34:43.689384+0000 mon.a (mon.0) 1391 : audit [DBG] from='client.? 172.21.15.6:0/3099813038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:44.455 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:34:41.798Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:44.455 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:34:41.800Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:44.456 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:42 smithi006 bash[33918]: cluster 2023-08-27T22:34:41.747320+0000 mgr.y (mgr.14901) 1085 : cluster [DBG] pgmap v1092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:44.456 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:42 smithi006 bash[27292]: cluster 2023-08-27T22:34:41.747320+0000 mgr.y (mgr.14901) 1085 : cluster [DBG] pgmap v1092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:44.456 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:43 smithi006 bash[27292]: audit 2023-08-27T22:34:43.689384+0000 mon.a (mon.0) 1391 : audit [DBG] from='client.? 172.21.15.6:0/3099813038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:44.456 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:43 smithi006 bash[33918]: audit 2023-08-27T22:34:43.689384+0000 mon.a (mon.0) 1391 : audit [DBG] from='client.? 172.21.15.6:0/3099813038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:44.457 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:34:43.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:44.457 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:34:43.693Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:44 smithi130 bash[27579]: cluster 2023-08-27T22:34:43.748082+0000 mgr.y (mgr.14901) 1086 : cluster [DBG] pgmap v1093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:45.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:44 smithi006 bash[27292]: cluster 2023-08-27T22:34:43.748082+0000 mgr.y (mgr.14901) 1086 : cluster [DBG] pgmap v1093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:45.275 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:44 smithi006 bash[33918]: cluster 2023-08-27T22:34:43.748082+0000 mgr.y (mgr.14901) 1086 : cluster [DBG] pgmap v1093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:46 smithi130 bash[27579]: cluster 2023-08-27T22:34:45.748940+0000 mgr.y (mgr.14901) 1087 : cluster [DBG] pgmap v1094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:47.174 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:46 smithi130 bash[27579]: audit 2023-08-27T22:34:46.109179+0000 mon.c (mon.2) 448 : audit [DBG] from='client.? 172.21.15.6:0/1589765468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:47.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:46 smithi006 bash[33918]: cluster 2023-08-27T22:34:45.748940+0000 mgr.y (mgr.14901) 1087 : cluster [DBG] pgmap v1094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:47.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:46 smithi006 bash[33918]: audit 2023-08-27T22:34:46.109179+0000 mon.c (mon.2) 448 : audit [DBG] from='client.? 172.21.15.6:0/1589765468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:47.358 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:46 smithi006 bash[27292]: cluster 2023-08-27T22:34:45.748940+0000 mgr.y (mgr.14901) 1087 : cluster [DBG] pgmap v1094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:47.661 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:46 smithi006 bash[27292]: audit 2023-08-27T22:34:46.109179+0000 mon.c (mon.2) 448 : audit [DBG] from='client.? 172.21.15.6:0/1589765468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:49.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:48 smithi130 bash[27579]: cluster 2023-08-27T22:34:47.749561+0000 mgr.y (mgr.14901) 1088 : cluster [DBG] pgmap v1095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:49.128 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:48 smithi130 bash[27579]: audit 2023-08-27T22:34:48.522405+0000 mon.a (mon.0) 1392 : audit [DBG] from='client.? 172.21.15.6:0/2104544991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:49.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:48 smithi006 bash[33918]: cluster 2023-08-27T22:34:47.749561+0000 mgr.y (mgr.14901) 1088 : cluster [DBG] pgmap v1095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:49.267 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:48 smithi006 bash[33918]: audit 2023-08-27T22:34:48.522405+0000 mon.a (mon.0) 1392 : audit [DBG] from='client.? 172.21.15.6:0/2104544991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:49.267 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:48 smithi006 bash[27292]: cluster 2023-08-27T22:34:47.749561+0000 mgr.y (mgr.14901) 1088 : cluster [DBG] pgmap v1095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:49.268 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:48 smithi006 bash[27292]: audit 2023-08-27T22:34:48.522405+0000 mon.a (mon.0) 1392 : audit [DBG] from='client.? 172.21.15.6:0/2104544991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:34:50.234Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:50.731 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:34:50.236Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:50 smithi130 bash[27579]: cluster 2023-08-27T22:34:49.750680+0000 mgr.y (mgr.14901) 1089 : cluster [DBG] pgmap v1096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:51.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:34:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:34:50] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:34:51.443 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:50 smithi006 bash[33918]: cluster 2023-08-27T22:34:49.750680+0000 mgr.y (mgr.14901) 1089 : cluster [DBG] pgmap v1096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:51.443 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:50 smithi006 bash[27292]: cluster 2023-08-27T22:34:49.750680+0000 mgr.y (mgr.14901) 1089 : cluster [DBG] pgmap v1096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:52.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:51 smithi006 bash[33918]: audit 2023-08-27T22:34:50.942460+0000 mon.c (mon.2) 449 : audit [DBG] from='client.? 172.21.15.6:0/899048489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:52.224 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:34:51.799Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:52.224 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:34:51.801Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:52.224 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:51 smithi006 bash[27292]: audit 2023-08-27T22:34:50.942460+0000 mon.c (mon.2) 449 : audit [DBG] from='client.? 172.21.15.6:0/899048489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:52.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:51 smithi130 bash[27579]: audit 2023-08-27T22:34:50.942460+0000 mon.c (mon.2) 449 : audit [DBG] from='client.? 172.21.15.6:0/899048489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:53.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:52 smithi006 bash[33918]: cluster 2023-08-27T22:34:51.751449+0000 mgr.y (mgr.14901) 1090 : cluster [DBG] pgmap v1097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:53.319 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:52 smithi006 bash[27292]: cluster 2023-08-27T22:34:51.751449+0000 mgr.y (mgr.14901) 1090 : cluster [DBG] pgmap v1097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:53.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:52 smithi130 bash[27579]: cluster 2023-08-27T22:34:51.751449+0000 mgr.y (mgr.14901) 1090 : cluster [DBG] pgmap v1097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:53.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:53 smithi006 bash[27292]: audit 2023-08-27T22:34:53.360592+0000 mon.c (mon.2) 450 : audit [DBG] from='client.? 172.21.15.6:0/329878753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:34:53.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:34:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:34:53.694Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:34:53.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:53 smithi006 bash[33918]: audit 2023-08-27T22:34:53.360592+0000 mon.c (mon.2) 450 : audit [DBG] from='client.? 172.21.15.6:0/329878753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:54.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:53 smithi130 bash[27579]: audit 2023-08-27T22:34:53.360592+0000 mon.c (mon.2) 450 : audit [DBG] from='client.? 172.21.15.6:0/329878753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:55.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:54 smithi006 bash[27292]: cluster 2023-08-27T22:34:53.752085+0000 mgr.y (mgr.14901) 1091 : cluster [DBG] pgmap v1098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:55.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:54 smithi006 bash[33918]: cluster 2023-08-27T22:34:53.752085+0000 mgr.y (mgr.14901) 1091 : cluster [DBG] pgmap v1098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:55.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:54 smithi130 bash[27579]: cluster 2023-08-27T22:34:53.752085+0000 mgr.y (mgr.14901) 1091 : cluster [DBG] pgmap v1098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:56.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:55 smithi006 bash[27292]: audit 2023-08-27T22:34:55.769546+0000 mon.c (mon.2) 451 : audit [DBG] from='client.? 172.21.15.6:0/175095497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:56.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:55 smithi006 bash[33918]: audit 2023-08-27T22:34:55.769546+0000 mon.c (mon.2) 451 : audit [DBG] from='client.? 172.21.15.6:0/175095497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:56.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:55 smithi130 bash[27579]: audit 2023-08-27T22:34:55.769546+0000 mon.c (mon.2) 451 : audit [DBG] from='client.? 172.21.15.6:0/175095497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:57.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:56 smithi006 bash[33918]: cluster 2023-08-27T22:34:55.753214+0000 mgr.y (mgr.14901) 1092 : cluster [DBG] pgmap v1099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:57.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:56 smithi006 bash[27292]: cluster 2023-08-27T22:34:55.753214+0000 mgr.y (mgr.14901) 1092 : cluster [DBG] pgmap v1099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:57.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:56 smithi130 bash[27579]: cluster 2023-08-27T22:34:55.753214+0000 mgr.y (mgr.14901) 1092 : cluster [DBG] pgmap v1099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:59.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:58 smithi006 bash[33918]: cluster 2023-08-27T22:34:57.753904+0000 mgr.y (mgr.14901) 1093 : cluster [DBG] pgmap v1100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:59.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:34:58 smithi006 bash[33918]: audit 2023-08-27T22:34:58.181259+0000 mon.c (mon.2) 452 : audit [DBG] from='client.? 172.21.15.6:0/2885776340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:59.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:58 smithi006 bash[27292]: cluster 2023-08-27T22:34:57.753904+0000 mgr.y (mgr.14901) 1093 : cluster [DBG] pgmap v1100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:59.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:34:58 smithi006 bash[27292]: audit 2023-08-27T22:34:58.181259+0000 mon.c (mon.2) 452 : audit [DBG] from='client.? 172.21.15.6:0/2885776340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:59.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:58 smithi130 bash[27579]: cluster 2023-08-27T22:34:57.753904+0000 mgr.y (mgr.14901) 1093 : cluster [DBG] pgmap v1100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:34:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:34:58 smithi130 bash[27579]: audit 2023-08-27T22:34:58.181259+0000 mon.c (mon.2) 452 : audit [DBG] from='client.? 172.21.15.6:0/2885776340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:00.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:35:00.234Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:00.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:35:00.237Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:01.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:35:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:35:00] "GET /metrics HTTP/1.1" 200 193080 "" "Prometheus/2.33.4" 2023-08-27T22:35:01.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:00 smithi006 bash[27292]: cluster 2023-08-27T22:34:59.754972+0000 mgr.y (mgr.14901) 1094 : cluster [DBG] pgmap v1101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:01.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:00 smithi006 bash[27292]: audit 2023-08-27T22:35:00.604699+0000 mon.c (mon.2) 453 : audit [DBG] from='client.? 172.21.15.6:0/4061357731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:01.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:00 smithi006 bash[33918]: cluster 2023-08-27T22:34:59.754972+0000 mgr.y (mgr.14901) 1094 : cluster [DBG] pgmap v1101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:01.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:00 smithi006 bash[33918]: audit 2023-08-27T22:35:00.604699+0000 mon.c (mon.2) 453 : audit [DBG] from='client.? 172.21.15.6:0/4061357731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:01.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:00 smithi130 bash[27579]: cluster 2023-08-27T22:34:59.754972+0000 mgr.y (mgr.14901) 1094 : cluster [DBG] pgmap v1101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:01.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:00 smithi130 bash[27579]: audit 2023-08-27T22:35:00.604699+0000 mon.c (mon.2) 453 : audit [DBG] from='client.? 172.21.15.6:0/4061357731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:35:01.799Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:35:01.801Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:03.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:02 smithi006 bash[33918]: cluster 2023-08-27T22:35:01.755606+0000 mgr.y (mgr.14901) 1095 : cluster [DBG] pgmap v1102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:03.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:02 smithi006 bash[27292]: cluster 2023-08-27T22:35:01.755606+0000 mgr.y (mgr.14901) 1095 : cluster [DBG] pgmap v1102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:03.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:02 smithi130 bash[27579]: cluster 2023-08-27T22:35:01.755606+0000 mgr.y (mgr.14901) 1095 : cluster [DBG] pgmap v1102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:03.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:03 smithi006 bash[33918]: audit 2023-08-27T22:35:03.033154+0000 mon.a (mon.0) 1393 : audit [DBG] from='client.? 172.21.15.6:0/4083189867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:03.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:03 smithi006 bash[27292]: audit 2023-08-27T22:35:03.033154+0000 mon.a (mon.0) 1393 : audit [DBG] from='client.? 172.21.15.6:0/4083189867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:35:03.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:03.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:35:03.695Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:04.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:03 smithi130 bash[27579]: audit 2023-08-27T22:35:03.033154+0000 mon.a (mon.0) 1393 : audit [DBG] from='client.? 172.21.15.6:0/4083189867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:05.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:04 smithi006 bash[27292]: cluster 2023-08-27T22:35:03.756278+0000 mgr.y (mgr.14901) 1096 : cluster [DBG] pgmap v1103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:05.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:04 smithi006 bash[33918]: cluster 2023-08-27T22:35:03.756278+0000 mgr.y (mgr.14901) 1096 : cluster [DBG] pgmap v1103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:04 smithi130 bash[27579]: cluster 2023-08-27T22:35:03.756278+0000 mgr.y (mgr.14901) 1096 : cluster [DBG] pgmap v1103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:06.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:05 smithi006 bash[27292]: audit 2023-08-27T22:35:05.441657+0000 mon.c (mon.2) 454 : audit [DBG] from='client.? 172.21.15.6:0/3452698373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:06.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:05 smithi006 bash[33918]: audit 2023-08-27T22:35:05.441657+0000 mon.c (mon.2) 454 : audit [DBG] from='client.? 172.21.15.6:0/3452698373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:06.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:05 smithi130 bash[27579]: audit 2023-08-27T22:35:05.441657+0000 mon.c (mon.2) 454 : audit [DBG] from='client.? 172.21.15.6:0/3452698373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:07.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:06 smithi006 bash[33918]: cluster 2023-08-27T22:35:05.757451+0000 mgr.y (mgr.14901) 1097 : cluster [DBG] pgmap v1104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:07.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:06 smithi006 bash[27292]: cluster 2023-08-27T22:35:05.757451+0000 mgr.y (mgr.14901) 1097 : cluster [DBG] pgmap v1104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:07.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:06 smithi130 bash[27579]: cluster 2023-08-27T22:35:05.757451+0000 mgr.y (mgr.14901) 1097 : cluster [DBG] pgmap v1104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:08.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:07 smithi006 bash[33918]: audit 2023-08-27T22:35:07.860285+0000 mon.c (mon.2) 455 : audit [DBG] from='client.? 172.21.15.6:0/2217109951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:08.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:07 smithi006 bash[27292]: audit 2023-08-27T22:35:07.860285+0000 mon.c (mon.2) 455 : audit [DBG] from='client.? 172.21.15.6:0/2217109951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:08.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:07 smithi130 bash[27579]: audit 2023-08-27T22:35:07.860285+0000 mon.c (mon.2) 455 : audit [DBG] from='client.? 172.21.15.6:0/2217109951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:09.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:08 smithi006 bash[27292]: cluster 2023-08-27T22:35:07.758023+0000 mgr.y (mgr.14901) 1098 : cluster [DBG] pgmap v1105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:09.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:08 smithi006 bash[33918]: cluster 2023-08-27T22:35:07.758023+0000 mgr.y (mgr.14901) 1098 : cluster [DBG] pgmap v1105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:09.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:08 smithi130 bash[27579]: cluster 2023-08-27T22:35:07.758023+0000 mgr.y (mgr.14901) 1098 : cluster [DBG] pgmap v1105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:35:10.235Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:35:10.237Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:11.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:35:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:35:10] "GET /metrics HTTP/1.1" 200 193080 "" "Prometheus/2.33.4" 2023-08-27T22:35:11.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:10 smithi006 bash[27292]: cluster 2023-08-27T22:35:09.759040+0000 mgr.y (mgr.14901) 1099 : cluster [DBG] pgmap v1106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:11.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:10 smithi006 bash[27292]: audit 2023-08-27T22:35:10.270336+0000 mon.a (mon.0) 1394 : audit [DBG] from='client.? 172.21.15.6:0/1147578820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:11.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:10 smithi006 bash[33918]: cluster 2023-08-27T22:35:09.759040+0000 mgr.y (mgr.14901) 1099 : cluster [DBG] pgmap v1106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:11.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:10 smithi006 bash[33918]: audit 2023-08-27T22:35:10.270336+0000 mon.a (mon.0) 1394 : audit [DBG] from='client.? 172.21.15.6:0/1147578820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:11.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:10 smithi130 bash[27579]: cluster 2023-08-27T22:35:09.759040+0000 mgr.y (mgr.14901) 1099 : cluster [DBG] pgmap v1106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:11.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:10 smithi130 bash[27579]: audit 2023-08-27T22:35:10.270336+0000 mon.a (mon.0) 1394 : audit [DBG] from='client.? 172.21.15.6:0/1147578820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:35:11.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:35:11.802Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:13.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:12 smithi006 bash[33918]: cluster 2023-08-27T22:35:11.759701+0000 mgr.y (mgr.14901) 1100 : cluster [DBG] pgmap v1107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:13.240 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:12 smithi006 bash[33918]: audit 2023-08-27T22:35:12.685579+0000 mon.c (mon.2) 456 : audit [DBG] from='client.? 172.21.15.6:0/1351560883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:13.241 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:12 smithi006 bash[27292]: cluster 2023-08-27T22:35:11.759701+0000 mgr.y (mgr.14901) 1100 : cluster [DBG] pgmap v1107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:13.241 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:12 smithi006 bash[27292]: audit 2023-08-27T22:35:12.685579+0000 mon.c (mon.2) 456 : audit [DBG] from='client.? 172.21.15.6:0/1351560883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:13.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:12 smithi130 bash[27579]: cluster 2023-08-27T22:35:11.759701+0000 mgr.y (mgr.14901) 1100 : cluster [DBG] pgmap v1107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:13.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:12 smithi130 bash[27579]: audit 2023-08-27T22:35:12.685579+0000 mon.c (mon.2) 456 : audit [DBG] from='client.? 172.21.15.6:0/1351560883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:35:13.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:35:13.695Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:14 smithi130 bash[27579]: cluster 2023-08-27T22:35:13.760344+0000 mgr.y (mgr.14901) 1101 : cluster [DBG] pgmap v1108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:15.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:14 smithi006 bash[27292]: cluster 2023-08-27T22:35:13.760344+0000 mgr.y (mgr.14901) 1101 : cluster [DBG] pgmap v1108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:15.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:14 smithi006 bash[33918]: cluster 2023-08-27T22:35:13.760344+0000 mgr.y (mgr.14901) 1101 : cluster [DBG] pgmap v1108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:16.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:15 smithi130 bash[27579]: audit 2023-08-27T22:35:15.102005+0000 mon.a (mon.0) 1395 : audit [DBG] from='client.? 172.21.15.6:0/3893501748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:16.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:15 smithi006 bash[27292]: audit 2023-08-27T22:35:15.102005+0000 mon.a (mon.0) 1395 : audit [DBG] from='client.? 172.21.15.6:0/3893501748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:16.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:15 smithi006 bash[33918]: audit 2023-08-27T22:35:15.102005+0000 mon.a (mon.0) 1395 : audit [DBG] from='client.? 172.21.15.6:0/3893501748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:16 smithi130 bash[27579]: cluster 2023-08-27T22:35:15.761450+0000 mgr.y (mgr.14901) 1102 : cluster [DBG] pgmap v1109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:17.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:16 smithi006 bash[33918]: cluster 2023-08-27T22:35:15.761450+0000 mgr.y (mgr.14901) 1102 : cluster [DBG] pgmap v1109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:17.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:16 smithi006 bash[27292]: cluster 2023-08-27T22:35:15.761450+0000 mgr.y (mgr.14901) 1102 : cluster [DBG] pgmap v1109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:18.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:17 smithi130 bash[27579]: audit 2023-08-27T22:35:17.511181+0000 mon.c (mon.2) 457 : audit [DBG] from='client.? 172.21.15.6:0/3125821079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:18.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:17 smithi006 bash[33918]: audit 2023-08-27T22:35:17.511181+0000 mon.c (mon.2) 457 : audit [DBG] from='client.? 172.21.15.6:0/3125821079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:18.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:17 smithi006 bash[27292]: audit 2023-08-27T22:35:17.511181+0000 mon.c (mon.2) 457 : audit [DBG] from='client.? 172.21.15.6:0/3125821079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:18 smithi130 bash[27579]: cluster 2023-08-27T22:35:17.762110+0000 mgr.y (mgr.14901) 1103 : cluster [DBG] pgmap v1110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:19.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:18 smithi006 bash[33918]: cluster 2023-08-27T22:35:17.762110+0000 mgr.y (mgr.14901) 1103 : cluster [DBG] pgmap v1110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:19.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:18 smithi006 bash[27292]: cluster 2023-08-27T22:35:17.762110+0000 mgr.y (mgr.14901) 1103 : cluster [DBG] pgmap v1110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:20.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:19 smithi130 bash[27579]: audit 2023-08-27T22:35:19.444609+0000 mon.b (mon.1) 229 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:35:20.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:19 smithi006 bash[33918]: audit 2023-08-27T22:35:19.444609+0000 mon.b (mon.1) 229 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:35:20.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:19 smithi006 bash[27292]: audit 2023-08-27T22:35:19.444609+0000 mon.b (mon.1) 229 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:35:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:35:20.236Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:35:20.238Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:20 smithi130 bash[27579]: cluster 2023-08-27T22:35:19.763320+0000 mgr.y (mgr.14901) 1104 : cluster [DBG] pgmap v1111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:20 smithi130 bash[27579]: audit 2023-08-27T22:35:19.925370+0000 mon.c (mon.2) 458 : audit [DBG] from='client.? 172.21.15.6:0/4084718542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:21.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:35:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:35:20] "GET /metrics HTTP/1.1" 200 193081 "" "Prometheus/2.33.4" 2023-08-27T22:35:21.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:20 smithi006 bash[33918]: cluster 2023-08-27T22:35:19.763320+0000 mgr.y (mgr.14901) 1104 : cluster [DBG] pgmap v1111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:21.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:20 smithi006 bash[33918]: audit 2023-08-27T22:35:19.925370+0000 mon.c (mon.2) 458 : audit [DBG] from='client.? 172.21.15.6:0/4084718542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:21.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:20 smithi006 bash[27292]: cluster 2023-08-27T22:35:19.763320+0000 mgr.y (mgr.14901) 1104 : cluster [DBG] pgmap v1111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:21.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:20 smithi006 bash[27292]: audit 2023-08-27T22:35:19.925370+0000 mon.c (mon.2) 458 : audit [DBG] from='client.? 172.21.15.6:0/4084718542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:35:21.801Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:35:21.804Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:23.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:22 smithi130 bash[27579]: cluster 2023-08-27T22:35:21.763984+0000 mgr.y (mgr.14901) 1105 : cluster [DBG] pgmap v1112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:23.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:22 smithi130 bash[27579]: audit 2023-08-27T22:35:22.341052+0000 mon.a (mon.0) 1396 : audit [DBG] from='client.? 172.21.15.6:0/2792357008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:23.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:22 smithi006 bash[33918]: cluster 2023-08-27T22:35:21.763984+0000 mgr.y (mgr.14901) 1105 : cluster [DBG] pgmap v1112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:23.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:22 smithi006 bash[33918]: audit 2023-08-27T22:35:22.341052+0000 mon.a (mon.0) 1396 : audit [DBG] from='client.? 172.21.15.6:0/2792357008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:23.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:22 smithi006 bash[27292]: cluster 2023-08-27T22:35:21.763984+0000 mgr.y (mgr.14901) 1105 : cluster [DBG] pgmap v1112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:23.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:22 smithi006 bash[27292]: audit 2023-08-27T22:35:22.341052+0000 mon.a (mon.0) 1396 : audit [DBG] from='client.? 172.21.15.6:0/2792357008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:35:23.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:35:23.696Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:24 smithi130 bash[27579]: cluster 2023-08-27T22:35:23.764493+0000 mgr.y (mgr.14901) 1106 : cluster [DBG] pgmap v1113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:25.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:24 smithi006 bash[27292]: cluster 2023-08-27T22:35:23.764493+0000 mgr.y (mgr.14901) 1106 : cluster [DBG] pgmap v1113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:25.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:24 smithi006 bash[33918]: cluster 2023-08-27T22:35:23.764493+0000 mgr.y (mgr.14901) 1106 : cluster [DBG] pgmap v1113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:26.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:25 smithi130 bash[27579]: audit 2023-08-27T22:35:24.806532+0000 mon.a (mon.0) 1397 : audit [DBG] from='client.? 172.21.15.6:0/2077617098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:26.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:25 smithi130 bash[27579]: audit 2023-08-27T22:35:25.340326+0000 mon.a (mon.0) 1398 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:35:26.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:25 smithi006 bash[33918]: audit 2023-08-27T22:35:24.806532+0000 mon.a (mon.0) 1397 : audit [DBG] from='client.? 172.21.15.6:0/2077617098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:26.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:25 smithi006 bash[33918]: audit 2023-08-27T22:35:25.340326+0000 mon.a (mon.0) 1398 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:35:26.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:25 smithi006 bash[27292]: audit 2023-08-27T22:35:24.806532+0000 mon.a (mon.0) 1397 : audit [DBG] from='client.? 172.21.15.6:0/2077617098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:26.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:25 smithi006 bash[27292]: audit 2023-08-27T22:35:25.340326+0000 mon.a (mon.0) 1398 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:35:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:26 smithi130 bash[27579]: audit 2023-08-27T22:35:25.757191+0000 mon.a (mon.0) 1399 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:35:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:26 smithi130 bash[27579]: cluster 2023-08-27T22:35:25.765441+0000 mgr.y (mgr.14901) 1107 : cluster [DBG] pgmap v1114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:26 smithi130 bash[27579]: audit 2023-08-27T22:35:26.040047+0000 mon.b (mon.1) 230 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:35:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:26 smithi130 bash[27579]: audit 2023-08-27T22:35:26.042082+0000 mon.b (mon.1) 231 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:35:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:26 smithi130 bash[27579]: audit 2023-08-27T22:35:26.045504+0000 mon.a (mon.0) 1400 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:35:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:26 smithi006 bash[27292]: audit 2023-08-27T22:35:25.757191+0000 mon.a (mon.0) 1399 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:35:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:26 smithi006 bash[27292]: cluster 2023-08-27T22:35:25.765441+0000 mgr.y (mgr.14901) 1107 : cluster [DBG] pgmap v1114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:26 smithi006 bash[27292]: audit 2023-08-27T22:35:26.040047+0000 mon.b (mon.1) 230 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:35:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:26 smithi006 bash[27292]: audit 2023-08-27T22:35:26.042082+0000 mon.b (mon.1) 231 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:35:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:26 smithi006 bash[27292]: audit 2023-08-27T22:35:26.045504+0000 mon.a (mon.0) 1400 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:35:27.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:26 smithi006 bash[33918]: audit 2023-08-27T22:35:25.757191+0000 mon.a (mon.0) 1399 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:35:27.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:26 smithi006 bash[33918]: cluster 2023-08-27T22:35:25.765441+0000 mgr.y (mgr.14901) 1107 : cluster [DBG] pgmap v1114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:27.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:26 smithi006 bash[33918]: audit 2023-08-27T22:35:26.040047+0000 mon.b (mon.1) 230 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:35:27.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:26 smithi006 bash[33918]: audit 2023-08-27T22:35:26.042082+0000 mon.b (mon.1) 231 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:35:27.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:26 smithi006 bash[33918]: audit 2023-08-27T22:35:26.045504+0000 mon.a (mon.0) 1400 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:35:28.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:27 smithi130 bash[27579]: audit 2023-08-27T22:35:27.221823+0000 mon.a (mon.0) 1401 : audit [DBG] from='client.? 172.21.15.6:0/2144282838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:28.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:27 smithi006 bash[27292]: audit 2023-08-27T22:35:27.221823+0000 mon.a (mon.0) 1401 : audit [DBG] from='client.? 172.21.15.6:0/2144282838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:28.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:27 smithi006 bash[33918]: audit 2023-08-27T22:35:27.221823+0000 mon.a (mon.0) 1401 : audit [DBG] from='client.? 172.21.15.6:0/2144282838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:28 smithi130 bash[27579]: cluster 2023-08-27T22:35:27.766206+0000 mgr.y (mgr.14901) 1108 : cluster [DBG] pgmap v1115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:29.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:28 smithi006 bash[33918]: cluster 2023-08-27T22:35:27.766206+0000 mgr.y (mgr.14901) 1108 : cluster [DBG] pgmap v1115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:29.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:28 smithi006 bash[27292]: cluster 2023-08-27T22:35:27.766206+0000 mgr.y (mgr.14901) 1108 : cluster [DBG] pgmap v1115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:30.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:29 smithi130 bash[27579]: audit 2023-08-27T22:35:29.639105+0000 mon.c (mon.2) 459 : audit [DBG] from='client.? 172.21.15.6:0/3820267005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:30.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:29 smithi006 bash[27292]: audit 2023-08-27T22:35:29.639105+0000 mon.c (mon.2) 459 : audit [DBG] from='client.? 172.21.15.6:0/3820267005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:30.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:29 smithi006 bash[33918]: audit 2023-08-27T22:35:29.639105+0000 mon.c (mon.2) 459 : audit [DBG] from='client.? 172.21.15.6:0/3820267005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:35:30.237Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:35:30.239Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:30 smithi130 bash[27579]: cluster 2023-08-27T22:35:29.767348+0000 mgr.y (mgr.14901) 1109 : cluster [DBG] pgmap v1116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:31.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:35:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:35:30] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T22:35:31.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:30 smithi006 bash[33918]: cluster 2023-08-27T22:35:29.767348+0000 mgr.y (mgr.14901) 1109 : cluster [DBG] pgmap v1116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:31.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:30 smithi006 bash[27292]: cluster 2023-08-27T22:35:29.767348+0000 mgr.y (mgr.14901) 1109 : cluster [DBG] pgmap v1116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:35:31.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:32.484 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:35:31.804Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:33.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:32 smithi130 bash[27579]: cluster 2023-08-27T22:35:31.767960+0000 mgr.y (mgr.14901) 1110 : cluster [DBG] pgmap v1117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:33.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:32 smithi130 bash[27579]: audit 2023-08-27T22:35:32.047211+0000 mon.a (mon.0) 1402 : audit [DBG] from='client.? 172.21.15.6:0/3589194868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:33.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:32 smithi006 bash[33918]: cluster 2023-08-27T22:35:31.767960+0000 mgr.y (mgr.14901) 1110 : cluster [DBG] pgmap v1117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:33.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:32 smithi006 bash[33918]: audit 2023-08-27T22:35:32.047211+0000 mon.a (mon.0) 1402 : audit [DBG] from='client.? 172.21.15.6:0/3589194868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:32 smithi006 bash[27292]: cluster 2023-08-27T22:35:31.767960+0000 mgr.y (mgr.14901) 1110 : cluster [DBG] pgmap v1117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:32 smithi006 bash[27292]: audit 2023-08-27T22:35:32.047211+0000 mon.a (mon.0) 1402 : audit [DBG] from='client.? 172.21.15.6:0/3589194868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:35:33.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:35:33.697Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:34 smithi130 bash[27579]: cluster 2023-08-27T22:35:33.768744+0000 mgr.y (mgr.14901) 1111 : cluster [DBG] pgmap v1118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:34 smithi130 bash[27579]: audit 2023-08-27T22:35:34.453561+0000 mon.a (mon.0) 1403 : audit [DBG] from='client.? 172.21.15.6:0/4248569499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:35.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:34 smithi006 bash[27292]: cluster 2023-08-27T22:35:33.768744+0000 mgr.y (mgr.14901) 1111 : cluster [DBG] pgmap v1118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:35.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:34 smithi006 bash[27292]: audit 2023-08-27T22:35:34.453561+0000 mon.a (mon.0) 1403 : audit [DBG] from='client.? 172.21.15.6:0/4248569499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:35.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:34 smithi006 bash[33918]: cluster 2023-08-27T22:35:33.768744+0000 mgr.y (mgr.14901) 1111 : cluster [DBG] pgmap v1118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:35.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:34 smithi006 bash[33918]: audit 2023-08-27T22:35:34.453561+0000 mon.a (mon.0) 1403 : audit [DBG] from='client.? 172.21.15.6:0/4248569499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:36 smithi130 bash[27579]: cluster 2023-08-27T22:35:35.769883+0000 mgr.y (mgr.14901) 1112 : cluster [DBG] pgmap v1119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:37.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:36 smithi006 bash[27292]: cluster 2023-08-27T22:35:35.769883+0000 mgr.y (mgr.14901) 1112 : cluster [DBG] pgmap v1119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:37.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:36 smithi006 bash[33918]: cluster 2023-08-27T22:35:35.769883+0000 mgr.y (mgr.14901) 1112 : cluster [DBG] pgmap v1119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:38.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:37 smithi130 bash[27579]: audit 2023-08-27T22:35:36.864704+0000 mon.c (mon.2) 460 : audit [DBG] from='client.? 172.21.15.6:0/1320468867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:38.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:37 smithi006 bash[27292]: audit 2023-08-27T22:35:36.864704+0000 mon.c (mon.2) 460 : audit [DBG] from='client.? 172.21.15.6:0/1320468867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:38.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:37 smithi006 bash[33918]: audit 2023-08-27T22:35:36.864704+0000 mon.c (mon.2) 460 : audit [DBG] from='client.? 172.21.15.6:0/1320468867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:38 smithi130 bash[27579]: cluster 2023-08-27T22:35:37.770523+0000 mgr.y (mgr.14901) 1113 : cluster [DBG] pgmap v1120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:39.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:38 smithi006 bash[33918]: cluster 2023-08-27T22:35:37.770523+0000 mgr.y (mgr.14901) 1113 : cluster [DBG] pgmap v1120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:39.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:38 smithi006 bash[27292]: cluster 2023-08-27T22:35:37.770523+0000 mgr.y (mgr.14901) 1113 : cluster [DBG] pgmap v1120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:40.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:39 smithi130 bash[27579]: audit 2023-08-27T22:35:39.284409+0000 mon.a (mon.0) 1404 : audit [DBG] from='client.? 172.21.15.6:0/3180307355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:40.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:39 smithi006 bash[33918]: audit 2023-08-27T22:35:39.284409+0000 mon.a (mon.0) 1404 : audit [DBG] from='client.? 172.21.15.6:0/3180307355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:40.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:39 smithi006 bash[27292]: audit 2023-08-27T22:35:39.284409+0000 mon.a (mon.0) 1404 : audit [DBG] from='client.? 172.21.15.6:0/3180307355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:35:40.237Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:35:40.239Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:40 smithi130 bash[27579]: cluster 2023-08-27T22:35:39.771616+0000 mgr.y (mgr.14901) 1114 : cluster [DBG] pgmap v1121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:41.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:35:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:35:40] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T22:35:41.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:40 smithi006 bash[27292]: cluster 2023-08-27T22:35:39.771616+0000 mgr.y (mgr.14901) 1114 : cluster [DBG] pgmap v1121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:41.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:40 smithi006 bash[33918]: cluster 2023-08-27T22:35:39.771616+0000 mgr.y (mgr.14901) 1114 : cluster [DBG] pgmap v1121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:42.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:41 smithi006 bash[33918]: audit 2023-08-27T22:35:41.708602+0000 mon.c (mon.2) 461 : audit [DBG] from='client.? 172.21.15.6:0/1666501098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:42.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:41 smithi006 bash[27292]: audit 2023-08-27T22:35:41.708602+0000 mon.c (mon.2) 461 : audit [DBG] from='client.? 172.21.15.6:0/1666501098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:35:41.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:35:41.804Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:42.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:41 smithi130 bash[27579]: audit 2023-08-27T22:35:41.708602+0000 mon.c (mon.2) 461 : audit [DBG] from='client.? 172.21.15.6:0/1666501098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:43.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:42 smithi006 bash[27292]: cluster 2023-08-27T22:35:41.772279+0000 mgr.y (mgr.14901) 1115 : cluster [DBG] pgmap v1122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:43.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:42 smithi006 bash[33918]: cluster 2023-08-27T22:35:41.772279+0000 mgr.y (mgr.14901) 1115 : cluster [DBG] pgmap v1122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:43.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:42 smithi130 bash[27579]: cluster 2023-08-27T22:35:41.772279+0000 mgr.y (mgr.14901) 1115 : cluster [DBG] pgmap v1122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:43.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:35:43.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:43.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:35:43.698Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:44 smithi130 bash[27579]: cluster 2023-08-27T22:35:43.772962+0000 mgr.y (mgr.14901) 1116 : cluster [DBG] pgmap v1123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:44 smithi130 bash[27579]: audit 2023-08-27T22:35:44.130804+0000 mon.a (mon.0) 1405 : audit [DBG] from='client.? 172.21.15.6:0/2544414170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:45.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:44 smithi006 bash[27292]: cluster 2023-08-27T22:35:43.772962+0000 mgr.y (mgr.14901) 1116 : cluster [DBG] pgmap v1123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:45.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:44 smithi006 bash[27292]: audit 2023-08-27T22:35:44.130804+0000 mon.a (mon.0) 1405 : audit [DBG] from='client.? 172.21.15.6:0/2544414170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:45.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:44 smithi006 bash[33918]: cluster 2023-08-27T22:35:43.772962+0000 mgr.y (mgr.14901) 1116 : cluster [DBG] pgmap v1123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:45.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:44 smithi006 bash[33918]: audit 2023-08-27T22:35:44.130804+0000 mon.a (mon.0) 1405 : audit [DBG] from='client.? 172.21.15.6:0/2544414170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:46 smithi130 bash[27579]: cluster 2023-08-27T22:35:45.774094+0000 mgr.y (mgr.14901) 1117 : cluster [DBG] pgmap v1124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:46 smithi130 bash[27579]: audit 2023-08-27T22:35:46.545199+0000 mon.a (mon.0) 1406 : audit [DBG] from='client.? 172.21.15.6:0/165642088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:47.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:46 smithi006 bash[27292]: cluster 2023-08-27T22:35:45.774094+0000 mgr.y (mgr.14901) 1117 : cluster [DBG] pgmap v1124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:47.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:46 smithi006 bash[27292]: audit 2023-08-27T22:35:46.545199+0000 mon.a (mon.0) 1406 : audit [DBG] from='client.? 172.21.15.6:0/165642088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:47.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:46 smithi006 bash[33918]: cluster 2023-08-27T22:35:45.774094+0000 mgr.y (mgr.14901) 1117 : cluster [DBG] pgmap v1124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:47.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:46 smithi006 bash[33918]: audit 2023-08-27T22:35:46.545199+0000 mon.a (mon.0) 1406 : audit [DBG] from='client.? 172.21.15.6:0/165642088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:49.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:48 smithi130 bash[27579]: cluster 2023-08-27T22:35:47.774761+0000 mgr.y (mgr.14901) 1118 : cluster [DBG] pgmap v1125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:49.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:48 smithi006 bash[27292]: cluster 2023-08-27T22:35:47.774761+0000 mgr.y (mgr.14901) 1118 : cluster [DBG] pgmap v1125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:49.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:48 smithi006 bash[33918]: cluster 2023-08-27T22:35:47.774761+0000 mgr.y (mgr.14901) 1118 : cluster [DBG] pgmap v1125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:50.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:49 smithi130 bash[27579]: audit 2023-08-27T22:35:48.962382+0000 mon.a (mon.0) 1407 : audit [DBG] from='client.? 172.21.15.6:0/3264184595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:50.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:49 smithi006 bash[33918]: audit 2023-08-27T22:35:48.962382+0000 mon.a (mon.0) 1407 : audit [DBG] from='client.? 172.21.15.6:0/3264184595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:50.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:49 smithi006 bash[27292]: audit 2023-08-27T22:35:48.962382+0000 mon.a (mon.0) 1407 : audit [DBG] from='client.? 172.21.15.6:0/3264184595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:35:50.238Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:35:50.240Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:50 smithi130 bash[27579]: cluster 2023-08-27T22:35:49.775860+0000 mgr.y (mgr.14901) 1119 : cluster [DBG] pgmap v1126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:51.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:50 smithi006 bash[27292]: cluster 2023-08-27T22:35:49.775860+0000 mgr.y (mgr.14901) 1119 : cluster [DBG] pgmap v1126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:51.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:35:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:35:50] "GET /metrics HTTP/1.1" 200 193083 "" "Prometheus/2.33.4" 2023-08-27T22:35:51.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:50 smithi006 bash[33918]: cluster 2023-08-27T22:35:49.775860+0000 mgr.y (mgr.14901) 1119 : cluster [DBG] pgmap v1126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:52.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:51 smithi130 bash[27579]: audit 2023-08-27T22:35:51.380922+0000 mon.c (mon.2) 462 : audit [DBG] from='client.? 172.21.15.6:0/2585584113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:52.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:51 smithi006 bash[33918]: audit 2023-08-27T22:35:51.380922+0000 mon.c (mon.2) 462 : audit [DBG] from='client.? 172.21.15.6:0/2585584113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:52.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:51 smithi006 bash[27292]: audit 2023-08-27T22:35:51.380922+0000 mon.c (mon.2) 462 : audit [DBG] from='client.? 172.21.15.6:0/2585584113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:35:51.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:35:51.806Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:53.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:52 smithi130 bash[27579]: cluster 2023-08-27T22:35:51.776627+0000 mgr.y (mgr.14901) 1120 : cluster [DBG] pgmap v1127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:53.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:52 smithi006 bash[27292]: cluster 2023-08-27T22:35:51.776627+0000 mgr.y (mgr.14901) 1120 : cluster [DBG] pgmap v1127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:53.310 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:52 smithi006 bash[33918]: cluster 2023-08-27T22:35:51.776627+0000 mgr.y (mgr.14901) 1120 : cluster [DBG] pgmap v1127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:35:53.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:35:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:35:53.699Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:35:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:54 smithi130 bash[27579]: cluster 2023-08-27T22:35:53.777122+0000 mgr.y (mgr.14901) 1121 : cluster [DBG] pgmap v1128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:54 smithi130 bash[27579]: audit 2023-08-27T22:35:53.798487+0000 mon.c (mon.2) 463 : audit [DBG] from='client.? 172.21.15.6:0/1526607807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:55.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:54 smithi006 bash[33918]: cluster 2023-08-27T22:35:53.777122+0000 mgr.y (mgr.14901) 1121 : cluster [DBG] pgmap v1128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:55.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:54 smithi006 bash[33918]: audit 2023-08-27T22:35:53.798487+0000 mon.c (mon.2) 463 : audit [DBG] from='client.? 172.21.15.6:0/1526607807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:55.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:54 smithi006 bash[27292]: cluster 2023-08-27T22:35:53.777122+0000 mgr.y (mgr.14901) 1121 : cluster [DBG] pgmap v1128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:55.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:54 smithi006 bash[27292]: audit 2023-08-27T22:35:53.798487+0000 mon.c (mon.2) 463 : audit [DBG] from='client.? 172.21.15.6:0/1526607807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:56 smithi130 bash[27579]: cluster 2023-08-27T22:35:55.778319+0000 mgr.y (mgr.14901) 1122 : cluster [DBG] pgmap v1129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:56 smithi130 bash[27579]: audit 2023-08-27T22:35:56.220394+0000 mon.c (mon.2) 464 : audit [DBG] from='client.? 172.21.15.6:0/3248645188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:57.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:56 smithi006 bash[33918]: cluster 2023-08-27T22:35:55.778319+0000 mgr.y (mgr.14901) 1122 : cluster [DBG] pgmap v1129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:57.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:56 smithi006 bash[33918]: audit 2023-08-27T22:35:56.220394+0000 mon.c (mon.2) 464 : audit [DBG] from='client.? 172.21.15.6:0/3248645188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:57.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:56 smithi006 bash[27292]: cluster 2023-08-27T22:35:55.778319+0000 mgr.y (mgr.14901) 1122 : cluster [DBG] pgmap v1129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:57.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:56 smithi006 bash[27292]: audit 2023-08-27T22:35:56.220394+0000 mon.c (mon.2) 464 : audit [DBG] from='client.? 172.21.15.6:0/3248645188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:58 smithi130 bash[27579]: cluster 2023-08-27T22:35:57.778997+0000 mgr.y (mgr.14901) 1123 : cluster [DBG] pgmap v1130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:35:58 smithi130 bash[27579]: audit 2023-08-27T22:35:58.644697+0000 mon.a (mon.0) 1408 : audit [DBG] from='client.? 172.21.15.6:0/25707397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:59.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:58 smithi006 bash[27292]: cluster 2023-08-27T22:35:57.778997+0000 mgr.y (mgr.14901) 1123 : cluster [DBG] pgmap v1130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:59.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:35:58 smithi006 bash[27292]: audit 2023-08-27T22:35:58.644697+0000 mon.a (mon.0) 1408 : audit [DBG] from='client.? 172.21.15.6:0/25707397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:59.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:58 smithi006 bash[33918]: cluster 2023-08-27T22:35:57.778997+0000 mgr.y (mgr.14901) 1123 : cluster [DBG] pgmap v1130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:35:59.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:35:58 smithi006 bash[33918]: audit 2023-08-27T22:35:58.644697+0000 mon.a (mon.0) 1408 : audit [DBG] from='client.? 172.21.15.6:0/25707397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:00.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:36:00.239Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:00.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:36:00.241Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:00 smithi130 bash[27579]: cluster 2023-08-27T22:35:59.780214+0000 mgr.y (mgr.14901) 1124 : cluster [DBG] pgmap v1131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:01.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:36:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:36:00] "GET /metrics HTTP/1.1" 200 193079 "" "Prometheus/2.33.4" 2023-08-27T22:36:01.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:00 smithi006 bash[33918]: cluster 2023-08-27T22:35:59.780214+0000 mgr.y (mgr.14901) 1124 : cluster [DBG] pgmap v1131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:01.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:00 smithi006 bash[27292]: cluster 2023-08-27T22:35:59.780214+0000 mgr.y (mgr.14901) 1124 : cluster [DBG] pgmap v1131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:02.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:01 smithi130 bash[27579]: audit 2023-08-27T22:36:01.058691+0000 mon.c (mon.2) 465 : audit [DBG] from='client.? 172.21.15.6:0/1363747812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:02.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:01 smithi006 bash[33918]: audit 2023-08-27T22:36:01.058691+0000 mon.c (mon.2) 465 : audit [DBG] from='client.? 172.21.15.6:0/1363747812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:36:01.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:36:01.807Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:02.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:01 smithi006 bash[27292]: audit 2023-08-27T22:36:01.058691+0000 mon.c (mon.2) 465 : audit [DBG] from='client.? 172.21.15.6:0/1363747812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:02 smithi130 bash[27579]: cluster 2023-08-27T22:36:01.780971+0000 mgr.y (mgr.14901) 1125 : cluster [DBG] pgmap v1132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:03.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:02 smithi006 bash[27292]: cluster 2023-08-27T22:36:01.780971+0000 mgr.y (mgr.14901) 1125 : cluster [DBG] pgmap v1132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:03.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:02 smithi006 bash[33918]: cluster 2023-08-27T22:36:01.780971+0000 mgr.y (mgr.14901) 1125 : cluster [DBG] pgmap v1132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:36:03.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:36:03.698Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:03.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:03 smithi006 bash[27292]: audit 2023-08-27T22:36:03.482497+0000 mon.a (mon.0) 1409 : audit [DBG] from='client.? 172.21.15.6:0/3056724559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:03.972 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:03 smithi006 bash[33918]: audit 2023-08-27T22:36:03.482497+0000 mon.a (mon.0) 1409 : audit [DBG] from='client.? 172.21.15.6:0/3056724559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:04.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:03 smithi130 bash[27579]: audit 2023-08-27T22:36:03.482497+0000 mon.a (mon.0) 1409 : audit [DBG] from='client.? 172.21.15.6:0/3056724559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:05.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:04 smithi006 bash[33918]: cluster 2023-08-27T22:36:03.781633+0000 mgr.y (mgr.14901) 1126 : cluster [DBG] pgmap v1133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:05.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:04 smithi006 bash[27292]: cluster 2023-08-27T22:36:03.781633+0000 mgr.y (mgr.14901) 1126 : cluster [DBG] pgmap v1133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:04 smithi130 bash[27579]: cluster 2023-08-27T22:36:03.781633+0000 mgr.y (mgr.14901) 1126 : cluster [DBG] pgmap v1133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:07.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:06 smithi006 bash[33918]: cluster 2023-08-27T22:36:05.782689+0000 mgr.y (mgr.14901) 1127 : cluster [DBG] pgmap v1134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:07.393 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:06 smithi006 bash[33918]: audit 2023-08-27T22:36:05.899048+0000 mon.c (mon.2) 466 : audit [DBG] from='client.? 172.21.15.6:0/1779615088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:07.394 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:06 smithi006 bash[27292]: cluster 2023-08-27T22:36:05.782689+0000 mgr.y (mgr.14901) 1127 : cluster [DBG] pgmap v1134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:07.394 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:06 smithi006 bash[27292]: audit 2023-08-27T22:36:05.899048+0000 mon.c (mon.2) 466 : audit [DBG] from='client.? 172.21.15.6:0/1779615088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:07.395 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:06 smithi130 bash[27579]: cluster 2023-08-27T22:36:05.782689+0000 mgr.y (mgr.14901) 1127 : cluster [DBG] pgmap v1134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:07.396 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:06 smithi130 bash[27579]: audit 2023-08-27T22:36:05.899048+0000 mon.c (mon.2) 466 : audit [DBG] from='client.? 172.21.15.6:0/1779615088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:09.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:08 smithi006 bash[33918]: cluster 2023-08-27T22:36:07.783350+0000 mgr.y (mgr.14901) 1128 : cluster [DBG] pgmap v1135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:09.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:08 smithi006 bash[33918]: audit 2023-08-27T22:36:08.334830+0000 mon.a (mon.0) 1410 : audit [DBG] from='client.? 172.21.15.6:0/1255290000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:09.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:08 smithi006 bash[27292]: cluster 2023-08-27T22:36:07.783350+0000 mgr.y (mgr.14901) 1128 : cluster [DBG] pgmap v1135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:09.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:08 smithi006 bash[27292]: audit 2023-08-27T22:36:08.334830+0000 mon.a (mon.0) 1410 : audit [DBG] from='client.? 172.21.15.6:0/1255290000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:09.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:08 smithi130 bash[27579]: cluster 2023-08-27T22:36:07.783350+0000 mgr.y (mgr.14901) 1128 : cluster [DBG] pgmap v1135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:09.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:08 smithi130 bash[27579]: audit 2023-08-27T22:36:08.334830+0000 mon.a (mon.0) 1410 : audit [DBG] from='client.? 172.21.15.6:0/1255290000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:36:10.240Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:11.001 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:36:10.241Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:11.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:36:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:36:10] "GET /metrics HTTP/1.1" 200 193079 "" "Prometheus/2.33.4" 2023-08-27T22:36:11.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:10 smithi006 bash[27292]: cluster 2023-08-27T22:36:09.784441+0000 mgr.y (mgr.14901) 1129 : cluster [DBG] pgmap v1136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:11.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:10 smithi006 bash[27292]: audit 2023-08-27T22:36:10.753805+0000 mon.a (mon.0) 1411 : audit [DBG] from='client.? 172.21.15.6:0/4041721711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:11.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:10 smithi006 bash[33918]: cluster 2023-08-27T22:36:09.784441+0000 mgr.y (mgr.14901) 1129 : cluster [DBG] pgmap v1136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:11.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:10 smithi006 bash[33918]: audit 2023-08-27T22:36:10.753805+0000 mon.a (mon.0) 1411 : audit [DBG] from='client.? 172.21.15.6:0/4041721711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:11.718 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:10 smithi130 bash[27579]: cluster 2023-08-27T22:36:09.784441+0000 mgr.y (mgr.14901) 1129 : cluster [DBG] pgmap v1136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:11.718 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:10 smithi130 bash[27579]: audit 2023-08-27T22:36:10.753805+0000 mon.a (mon.0) 1411 : audit [DBG] from='client.? 172.21.15.6:0/4041721711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:36:11.806Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:36:11.808Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:13.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:12 smithi006 bash[33918]: cluster 2023-08-27T22:36:11.785115+0000 mgr.y (mgr.14901) 1130 : cluster [DBG] pgmap v1137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:13.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:12 smithi006 bash[27292]: cluster 2023-08-27T22:36:11.785115+0000 mgr.y (mgr.14901) 1130 : cluster [DBG] pgmap v1137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:13.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:12 smithi130 bash[27579]: cluster 2023-08-27T22:36:11.785115+0000 mgr.y (mgr.14901) 1130 : cluster [DBG] pgmap v1137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:13.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:13 smithi006 bash[33918]: audit 2023-08-27T22:36:13.169268+0000 mon.c (mon.2) 467 : audit [DBG] from='client.? 172.21.15.6:0/1744315037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:36:13.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:36:13.698Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:13.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:13 smithi006 bash[27292]: audit 2023-08-27T22:36:13.169268+0000 mon.c (mon.2) 467 : audit [DBG] from='client.? 172.21.15.6:0/1744315037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:14.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:13 smithi130 bash[27579]: audit 2023-08-27T22:36:13.169268+0000 mon.c (mon.2) 467 : audit [DBG] from='client.? 172.21.15.6:0/1744315037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:15.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:14 smithi006 bash[27292]: cluster 2023-08-27T22:36:13.785709+0000 mgr.y (mgr.14901) 1131 : cluster [DBG] pgmap v1138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:15.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:14 smithi006 bash[33918]: cluster 2023-08-27T22:36:13.785709+0000 mgr.y (mgr.14901) 1131 : cluster [DBG] pgmap v1138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:15.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:14 smithi130 bash[27579]: cluster 2023-08-27T22:36:13.785709+0000 mgr.y (mgr.14901) 1131 : cluster [DBG] pgmap v1138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:16.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:15 smithi006 bash[27292]: audit 2023-08-27T22:36:15.584145+0000 mon.a (mon.0) 1412 : audit [DBG] from='client.? 172.21.15.6:0/449985694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:16.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:15 smithi006 bash[33918]: audit 2023-08-27T22:36:15.584145+0000 mon.a (mon.0) 1412 : audit [DBG] from='client.? 172.21.15.6:0/449985694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:16.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:15 smithi130 bash[27579]: audit 2023-08-27T22:36:15.584145+0000 mon.a (mon.0) 1412 : audit [DBG] from='client.? 172.21.15.6:0/449985694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:17.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:16 smithi006 bash[33918]: cluster 2023-08-27T22:36:15.786821+0000 mgr.y (mgr.14901) 1132 : cluster [DBG] pgmap v1139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:17.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:16 smithi006 bash[27292]: cluster 2023-08-27T22:36:15.786821+0000 mgr.y (mgr.14901) 1132 : cluster [DBG] pgmap v1139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:17.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:16 smithi130 bash[27579]: cluster 2023-08-27T22:36:15.786821+0000 mgr.y (mgr.14901) 1132 : cluster [DBG] pgmap v1139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:19.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:18 smithi006 bash[27292]: cluster 2023-08-27T22:36:17.787264+0000 mgr.y (mgr.14901) 1133 : cluster [DBG] pgmap v1140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:19.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:18 smithi006 bash[27292]: audit 2023-08-27T22:36:17.999501+0000 mon.c (mon.2) 468 : audit [DBG] from='client.? 172.21.15.6:0/3383249039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:19.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:18 smithi006 bash[33918]: cluster 2023-08-27T22:36:17.787264+0000 mgr.y (mgr.14901) 1133 : cluster [DBG] pgmap v1140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:19.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:18 smithi006 bash[33918]: audit 2023-08-27T22:36:17.999501+0000 mon.c (mon.2) 468 : audit [DBG] from='client.? 172.21.15.6:0/3383249039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:19.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:18 smithi130 bash[27579]: cluster 2023-08-27T22:36:17.787264+0000 mgr.y (mgr.14901) 1133 : cluster [DBG] pgmap v1140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:19.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:18 smithi130 bash[27579]: audit 2023-08-27T22:36:17.999501+0000 mon.c (mon.2) 468 : audit [DBG] from='client.? 172.21.15.6:0/3383249039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:36:20.240Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:36:20.242Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:21.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:36:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:36:20] "GET /metrics HTTP/1.1" 200 193083 "" "Prometheus/2.33.4" 2023-08-27T22:36:21.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:20 smithi006 bash[33918]: cluster 2023-08-27T22:36:19.788296+0000 mgr.y (mgr.14901) 1134 : cluster [DBG] pgmap v1141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:21.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:20 smithi006 bash[33918]: audit 2023-08-27T22:36:20.426986+0000 mon.a (mon.0) 1413 : audit [DBG] from='client.? 172.21.15.6:0/2845018934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:21.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:20 smithi006 bash[27292]: cluster 2023-08-27T22:36:19.788296+0000 mgr.y (mgr.14901) 1134 : cluster [DBG] pgmap v1141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:21.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:20 smithi006 bash[27292]: audit 2023-08-27T22:36:20.426986+0000 mon.a (mon.0) 1413 : audit [DBG] from='client.? 172.21.15.6:0/2845018934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:21.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:20 smithi130 bash[27579]: cluster 2023-08-27T22:36:19.788296+0000 mgr.y (mgr.14901) 1134 : cluster [DBG] pgmap v1141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:21.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:20 smithi130 bash[27579]: audit 2023-08-27T22:36:20.426986+0000 mon.a (mon.0) 1413 : audit [DBG] from='client.? 172.21.15.6:0/2845018934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:36:21.806Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:36:21.808Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:23.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:22 smithi006 bash[33918]: cluster 2023-08-27T22:36:21.788959+0000 mgr.y (mgr.14901) 1135 : cluster [DBG] pgmap v1142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:23.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:22 smithi006 bash[33918]: audit 2023-08-27T22:36:22.846976+0000 mon.a (mon.0) 1414 : audit [DBG] from='client.? 172.21.15.6:0/1911250994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:23.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:22 smithi006 bash[27292]: cluster 2023-08-27T22:36:21.788959+0000 mgr.y (mgr.14901) 1135 : cluster [DBG] pgmap v1142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:23.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:22 smithi006 bash[27292]: audit 2023-08-27T22:36:22.846976+0000 mon.a (mon.0) 1414 : audit [DBG] from='client.? 172.21.15.6:0/1911250994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:23.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:22 smithi130 bash[27579]: cluster 2023-08-27T22:36:21.788959+0000 mgr.y (mgr.14901) 1135 : cluster [DBG] pgmap v1142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:23.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:22 smithi130 bash[27579]: audit 2023-08-27T22:36:22.846976+0000 mon.a (mon.0) 1414 : audit [DBG] from='client.? 172.21.15.6:0/1911250994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:36:23.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:36:23.700Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:24 smithi130 bash[27579]: cluster 2023-08-27T22:36:23.789542+0000 mgr.y (mgr.14901) 1136 : cluster [DBG] pgmap v1143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:25.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:24 smithi006 bash[33918]: cluster 2023-08-27T22:36:23.789542+0000 mgr.y (mgr.14901) 1136 : cluster [DBG] pgmap v1143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:25.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:24 smithi006 bash[27292]: cluster 2023-08-27T22:36:23.789542+0000 mgr.y (mgr.14901) 1136 : cluster [DBG] pgmap v1143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:26.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:25 smithi130 bash[27579]: audit 2023-08-27T22:36:25.263405+0000 mon.a (mon.0) 1415 : audit [DBG] from='client.? 172.21.15.6:0/2682280037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:26.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:25 smithi006 bash[33918]: audit 2023-08-27T22:36:25.263405+0000 mon.a (mon.0) 1415 : audit [DBG] from='client.? 172.21.15.6:0/2682280037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:26.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:25 smithi006 bash[27292]: audit 2023-08-27T22:36:25.263405+0000 mon.a (mon.0) 1415 : audit [DBG] from='client.? 172.21.15.6:0/2682280037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:26 smithi130 bash[27579]: cluster 2023-08-27T22:36:25.790674+0000 mgr.y (mgr.14901) 1137 : cluster [DBG] pgmap v1144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:26 smithi130 bash[27579]: audit 2023-08-27T22:36:26.060540+0000 mon.b (mon.1) 232 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:36:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:26 smithi006 bash[27292]: cluster 2023-08-27T22:36:25.790674+0000 mgr.y (mgr.14901) 1137 : cluster [DBG] pgmap v1144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:26 smithi006 bash[27292]: audit 2023-08-27T22:36:26.060540+0000 mon.b (mon.1) 232 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:36:27.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:26 smithi006 bash[33918]: cluster 2023-08-27T22:36:25.790674+0000 mgr.y (mgr.14901) 1137 : cluster [DBG] pgmap v1144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:27.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:26 smithi006 bash[33918]: audit 2023-08-27T22:36:26.060540+0000 mon.b (mon.1) 232 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:36:28.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:27 smithi130 bash[27579]: audit 2023-08-27T22:36:27.678027+0000 mon.a (mon.0) 1416 : audit [DBG] from='client.? 172.21.15.6:0/1302740956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:28.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:27 smithi006 bash[33918]: audit 2023-08-27T22:36:27.678027+0000 mon.a (mon.0) 1416 : audit [DBG] from='client.? 172.21.15.6:0/1302740956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:28.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:27 smithi006 bash[27292]: audit 2023-08-27T22:36:27.678027+0000 mon.a (mon.0) 1416 : audit [DBG] from='client.? 172.21.15.6:0/1302740956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:28 smithi130 bash[27579]: cluster 2023-08-27T22:36:27.791330+0000 mgr.y (mgr.14901) 1138 : cluster [DBG] pgmap v1145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:29.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:28 smithi006 bash[33918]: cluster 2023-08-27T22:36:27.791330+0000 mgr.y (mgr.14901) 1138 : cluster [DBG] pgmap v1145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:29.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:28 smithi006 bash[27292]: cluster 2023-08-27T22:36:27.791330+0000 mgr.y (mgr.14901) 1138 : cluster [DBG] pgmap v1145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:36:30.240Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:36:30.242Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:30 smithi130 bash[27579]: cluster 2023-08-27T22:36:29.792713+0000 mgr.y (mgr.14901) 1139 : cluster [DBG] pgmap v1146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:30 smithi130 bash[27579]: audit 2023-08-27T22:36:30.102242+0000 mon.a (mon.0) 1417 : audit [DBG] from='client.? 172.21.15.6:0/3950909410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:31.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:36:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:36:30] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:36:31.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:30 smithi006 bash[27292]: cluster 2023-08-27T22:36:29.792713+0000 mgr.y (mgr.14901) 1139 : cluster [DBG] pgmap v1146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:31.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:30 smithi006 bash[27292]: audit 2023-08-27T22:36:30.102242+0000 mon.a (mon.0) 1417 : audit [DBG] from='client.? 172.21.15.6:0/3950909410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:31.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:30 smithi006 bash[33918]: cluster 2023-08-27T22:36:29.792713+0000 mgr.y (mgr.14901) 1139 : cluster [DBG] pgmap v1146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:31.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:30 smithi006 bash[33918]: audit 2023-08-27T22:36:30.102242+0000 mon.a (mon.0) 1417 : audit [DBG] from='client.? 172.21.15.6:0/3950909410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:36:31.806Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:36:31.809Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:32 smithi006 bash[27292]: cluster 2023-08-27T22:36:31.793400+0000 mgr.y (mgr.14901) 1140 : cluster [DBG] pgmap v1147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:32 smithi006 bash[27292]: audit 2023-08-27T22:36:31.935246+0000 mon.a (mon.0) 1418 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:36:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:32 smithi006 bash[27292]: audit 2023-08-27T22:36:32.332047+0000 mon.a (mon.0) 1419 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:36:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:32 smithi006 bash[27292]: audit 2023-08-27T22:36:32.517746+0000 mon.a (mon.0) 1420 : audit [DBG] from='client.? 172.21.15.6:0/1471072795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:32 smithi006 bash[27292]: audit 2023-08-27T22:36:32.615535+0000 mon.b (mon.1) 233 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:36:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:32 smithi006 bash[27292]: audit 2023-08-27T22:36:32.617320+0000 mon.b (mon.1) 234 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:36:33.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:32 smithi006 bash[27292]: audit 2023-08-27T22:36:32.620266+0000 mon.a (mon.0) 1421 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:36:33.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:32 smithi006 bash[33918]: cluster 2023-08-27T22:36:31.793400+0000 mgr.y (mgr.14901) 1140 : cluster [DBG] pgmap v1147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:33.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:32 smithi006 bash[33918]: audit 2023-08-27T22:36:31.935246+0000 mon.a (mon.0) 1418 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:36:33.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:32 smithi006 bash[33918]: audit 2023-08-27T22:36:32.332047+0000 mon.a (mon.0) 1419 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:36:33.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:32 smithi006 bash[33918]: audit 2023-08-27T22:36:32.517746+0000 mon.a (mon.0) 1420 : audit [DBG] from='client.? 172.21.15.6:0/1471072795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:33.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:32 smithi006 bash[33918]: audit 2023-08-27T22:36:32.615535+0000 mon.b (mon.1) 233 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:36:33.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:32 smithi006 bash[33918]: audit 2023-08-27T22:36:32.617320+0000 mon.b (mon.1) 234 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:36:33.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:32 smithi006 bash[33918]: audit 2023-08-27T22:36:32.620266+0000 mon.a (mon.0) 1421 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:36:33.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:32 smithi130 bash[27579]: cluster 2023-08-27T22:36:31.793400+0000 mgr.y (mgr.14901) 1140 : cluster [DBG] pgmap v1147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:33.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:32 smithi130 bash[27579]: audit 2023-08-27T22:36:31.935246+0000 mon.a (mon.0) 1418 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:36:33.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:32 smithi130 bash[27579]: audit 2023-08-27T22:36:32.332047+0000 mon.a (mon.0) 1419 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:36:33.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:32 smithi130 bash[27579]: audit 2023-08-27T22:36:32.517746+0000 mon.a (mon.0) 1420 : audit [DBG] from='client.? 172.21.15.6:0/1471072795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:32 smithi130 bash[27579]: audit 2023-08-27T22:36:32.615535+0000 mon.b (mon.1) 233 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:36:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:32 smithi130 bash[27579]: audit 2023-08-27T22:36:32.617320+0000 mon.b (mon.1) 234 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:36:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:32 smithi130 bash[27579]: audit 2023-08-27T22:36:32.620266+0000 mon.a (mon.0) 1421 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:36:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:36:33.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:36:33.700Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:34 smithi130 bash[27579]: cluster 2023-08-27T22:36:33.794066+0000 mgr.y (mgr.14901) 1141 : cluster [DBG] pgmap v1148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:35.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:34 smithi006 bash[33918]: cluster 2023-08-27T22:36:33.794066+0000 mgr.y (mgr.14901) 1141 : cluster [DBG] pgmap v1148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:35.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:34 smithi006 bash[27292]: cluster 2023-08-27T22:36:33.794066+0000 mgr.y (mgr.14901) 1141 : cluster [DBG] pgmap v1148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:36.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:35 smithi130 bash[27579]: audit 2023-08-27T22:36:34.933536+0000 mon.c (mon.2) 469 : audit [DBG] from='client.? 172.21.15.6:0/722739935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:36.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:35 smithi006 bash[33918]: audit 2023-08-27T22:36:34.933536+0000 mon.c (mon.2) 469 : audit [DBG] from='client.? 172.21.15.6:0/722739935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:36.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:35 smithi006 bash[27292]: audit 2023-08-27T22:36:34.933536+0000 mon.c (mon.2) 469 : audit [DBG] from='client.? 172.21.15.6:0/722739935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:36 smithi130 bash[27579]: cluster 2023-08-27T22:36:35.794810+0000 mgr.y (mgr.14901) 1142 : cluster [DBG] pgmap v1149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:37.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:36 smithi006 bash[33918]: cluster 2023-08-27T22:36:35.794810+0000 mgr.y (mgr.14901) 1142 : cluster [DBG] pgmap v1149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:37.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:36 smithi006 bash[27292]: cluster 2023-08-27T22:36:35.794810+0000 mgr.y (mgr.14901) 1142 : cluster [DBG] pgmap v1149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:38.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:37 smithi130 bash[27579]: audit 2023-08-27T22:36:37.352815+0000 mon.c (mon.2) 470 : audit [DBG] from='client.? 172.21.15.6:0/4169445716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:38.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:37 smithi006 bash[33918]: audit 2023-08-27T22:36:37.352815+0000 mon.c (mon.2) 470 : audit [DBG] from='client.? 172.21.15.6:0/4169445716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:38.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:37 smithi006 bash[27292]: audit 2023-08-27T22:36:37.352815+0000 mon.c (mon.2) 470 : audit [DBG] from='client.? 172.21.15.6:0/4169445716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:38 smithi130 bash[27579]: cluster 2023-08-27T22:36:37.795431+0000 mgr.y (mgr.14901) 1143 : cluster [DBG] pgmap v1150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:39.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:38 smithi006 bash[27292]: cluster 2023-08-27T22:36:37.795431+0000 mgr.y (mgr.14901) 1143 : cluster [DBG] pgmap v1150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:39.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:38 smithi006 bash[33918]: cluster 2023-08-27T22:36:37.795431+0000 mgr.y (mgr.14901) 1143 : cluster [DBG] pgmap v1150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:40.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:39 smithi130 bash[27579]: audit 2023-08-27T22:36:39.767454+0000 mon.a (mon.0) 1422 : audit [DBG] from='client.? 172.21.15.6:0/685677308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:40.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:39 smithi006 bash[33918]: audit 2023-08-27T22:36:39.767454+0000 mon.a (mon.0) 1422 : audit [DBG] from='client.? 172.21.15.6:0/685677308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:40.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:39 smithi006 bash[27292]: audit 2023-08-27T22:36:39.767454+0000 mon.a (mon.0) 1422 : audit [DBG] from='client.? 172.21.15.6:0/685677308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:36:40.241Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:36:40.243Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:40 smithi130 bash[27579]: cluster 2023-08-27T22:36:39.796539+0000 mgr.y (mgr.14901) 1144 : cluster [DBG] pgmap v1151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:41.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:36:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:36:40] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:36:41.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:40 smithi006 bash[27292]: cluster 2023-08-27T22:36:39.796539+0000 mgr.y (mgr.14901) 1144 : cluster [DBG] pgmap v1151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:41.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:40 smithi006 bash[33918]: cluster 2023-08-27T22:36:39.796539+0000 mgr.y (mgr.14901) 1144 : cluster [DBG] pgmap v1151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:36:41.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:36:41.809Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:43.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:42 smithi130 bash[27579]: cluster 2023-08-27T22:36:41.797179+0000 mgr.y (mgr.14901) 1145 : cluster [DBG] pgmap v1152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:43.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:42 smithi130 bash[27579]: audit 2023-08-27T22:36:42.182044+0000 mon.a (mon.0) 1423 : audit [DBG] from='client.? 172.21.15.6:0/1440588120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:43.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:42 smithi006 bash[33918]: cluster 2023-08-27T22:36:41.797179+0000 mgr.y (mgr.14901) 1145 : cluster [DBG] pgmap v1152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:43.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:42 smithi006 bash[33918]: audit 2023-08-27T22:36:42.182044+0000 mon.a (mon.0) 1423 : audit [DBG] from='client.? 172.21.15.6:0/1440588120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:43.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:42 smithi006 bash[27292]: cluster 2023-08-27T22:36:41.797179+0000 mgr.y (mgr.14901) 1145 : cluster [DBG] pgmap v1152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:43.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:42 smithi006 bash[27292]: audit 2023-08-27T22:36:42.182044+0000 mon.a (mon.0) 1423 : audit [DBG] from='client.? 172.21.15.6:0/1440588120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:43.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:36:43.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:43.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:36:43.701Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:44 smithi130 bash[27579]: cluster 2023-08-27T22:36:43.797864+0000 mgr.y (mgr.14901) 1146 : cluster [DBG] pgmap v1153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:44 smithi130 bash[27579]: audit 2023-08-27T22:36:44.592717+0000 mon.a (mon.0) 1424 : audit [DBG] from='client.? 172.21.15.6:0/482626230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:45.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:44 smithi006 bash[27292]: cluster 2023-08-27T22:36:43.797864+0000 mgr.y (mgr.14901) 1146 : cluster [DBG] pgmap v1153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:45.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:44 smithi006 bash[27292]: audit 2023-08-27T22:36:44.592717+0000 mon.a (mon.0) 1424 : audit [DBG] from='client.? 172.21.15.6:0/482626230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:45.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:44 smithi006 bash[33918]: cluster 2023-08-27T22:36:43.797864+0000 mgr.y (mgr.14901) 1146 : cluster [DBG] pgmap v1153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:45.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:44 smithi006 bash[33918]: audit 2023-08-27T22:36:44.592717+0000 mon.a (mon.0) 1424 : audit [DBG] from='client.? 172.21.15.6:0/482626230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:46 smithi130 bash[27579]: cluster 2023-08-27T22:36:45.798985+0000 mgr.y (mgr.14901) 1147 : cluster [DBG] pgmap v1154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:47.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:46 smithi006 bash[27292]: cluster 2023-08-27T22:36:45.798985+0000 mgr.y (mgr.14901) 1147 : cluster [DBG] pgmap v1154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:47.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:46 smithi006 bash[33918]: cluster 2023-08-27T22:36:45.798985+0000 mgr.y (mgr.14901) 1147 : cluster [DBG] pgmap v1154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:48.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:47 smithi006 bash[33918]: audit 2023-08-27T22:36:47.013925+0000 mon.a (mon.0) 1425 : audit [DBG] from='client.? 172.21.15.6:0/2701803569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:48.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:47 smithi006 bash[27292]: audit 2023-08-27T22:36:47.013925+0000 mon.a (mon.0) 1425 : audit [DBG] from='client.? 172.21.15.6:0/2701803569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:48.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:47 smithi130 bash[27579]: audit 2023-08-27T22:36:47.013925+0000 mon.a (mon.0) 1425 : audit [DBG] from='client.? 172.21.15.6:0/2701803569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:49.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:48 smithi006 bash[27292]: cluster 2023-08-27T22:36:47.799695+0000 mgr.y (mgr.14901) 1148 : cluster [DBG] pgmap v1155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:49.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:48 smithi006 bash[33918]: cluster 2023-08-27T22:36:47.799695+0000 mgr.y (mgr.14901) 1148 : cluster [DBG] pgmap v1155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:49.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:48 smithi130 bash[27579]: cluster 2023-08-27T22:36:47.799695+0000 mgr.y (mgr.14901) 1148 : cluster [DBG] pgmap v1155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:50.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:49 smithi006 bash[33918]: audit 2023-08-27T22:36:49.432914+0000 mon.a (mon.0) 1426 : audit [DBG] from='client.? 172.21.15.6:0/152101124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:50.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:49 smithi006 bash[27292]: audit 2023-08-27T22:36:49.432914+0000 mon.a (mon.0) 1426 : audit [DBG] from='client.? 172.21.15.6:0/152101124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:50.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:49 smithi130 bash[27579]: audit 2023-08-27T22:36:49.432914+0000 mon.a (mon.0) 1426 : audit [DBG] from='client.? 172.21.15.6:0/152101124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:36:50.241Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:36:50.243Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:51.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:50 smithi006 bash[27292]: cluster 2023-08-27T22:36:49.800979+0000 mgr.y (mgr.14901) 1149 : cluster [DBG] pgmap v1156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:51.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:36:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:36:50] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:36:51.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:50 smithi006 bash[33918]: cluster 2023-08-27T22:36:49.800979+0000 mgr.y (mgr.14901) 1149 : cluster [DBG] pgmap v1156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:51.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:50 smithi130 bash[27579]: cluster 2023-08-27T22:36:49.800979+0000 mgr.y (mgr.14901) 1149 : cluster [DBG] pgmap v1156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:52.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:51 smithi006 bash[33918]: audit 2023-08-27T22:36:51.862573+0000 mon.a (mon.0) 1427 : audit [DBG] from='client.? 172.21.15.6:0/3623667835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:52.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:51 smithi006 bash[27292]: audit 2023-08-27T22:36:51.862573+0000 mon.a (mon.0) 1427 : audit [DBG] from='client.? 172.21.15.6:0/3623667835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:36:51.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:52.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:36:51.809Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:52.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:51 smithi130 bash[27579]: audit 2023-08-27T22:36:51.862573+0000 mon.a (mon.0) 1427 : audit [DBG] from='client.? 172.21.15.6:0/3623667835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:53.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:52 smithi006 bash[33918]: cluster 2023-08-27T22:36:51.801629+0000 mgr.y (mgr.14901) 1150 : cluster [DBG] pgmap v1157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:53.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:52 smithi006 bash[27292]: cluster 2023-08-27T22:36:51.801629+0000 mgr.y (mgr.14901) 1150 : cluster [DBG] pgmap v1157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:53.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:52 smithi130 bash[27579]: cluster 2023-08-27T22:36:51.801629+0000 mgr.y (mgr.14901) 1150 : cluster [DBG] pgmap v1157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:36:53.699Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:36:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:36:53.703Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:36:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:54 smithi130 bash[27579]: cluster 2023-08-27T22:36:53.802370+0000 mgr.y (mgr.14901) 1151 : cluster [DBG] pgmap v1158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:54 smithi130 bash[27579]: audit 2023-08-27T22:36:54.280234+0000 mon.c (mon.2) 471 : audit [DBG] from='client.? 172.21.15.6:0/3618734211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:55.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:54 smithi006 bash[27292]: cluster 2023-08-27T22:36:53.802370+0000 mgr.y (mgr.14901) 1151 : cluster [DBG] pgmap v1158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:55.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:54 smithi006 bash[27292]: audit 2023-08-27T22:36:54.280234+0000 mon.c (mon.2) 471 : audit [DBG] from='client.? 172.21.15.6:0/3618734211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:55.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:54 smithi006 bash[33918]: cluster 2023-08-27T22:36:53.802370+0000 mgr.y (mgr.14901) 1151 : cluster [DBG] pgmap v1158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:55.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:54 smithi006 bash[33918]: audit 2023-08-27T22:36:54.280234+0000 mon.c (mon.2) 471 : audit [DBG] from='client.? 172.21.15.6:0/3618734211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:57.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:56 smithi006 bash[33918]: cluster 2023-08-27T22:36:55.803543+0000 mgr.y (mgr.14901) 1152 : cluster [DBG] pgmap v1159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:58.199 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:56 smithi006 bash[33918]: audit 2023-08-27T22:36:56.691312+0000 mon.a (mon.0) 1428 : audit [DBG] from='client.? 172.21.15.6:0/3736336411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:58.199 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:56 smithi006 bash[27292]: cluster 2023-08-27T22:36:55.803543+0000 mgr.y (mgr.14901) 1152 : cluster [DBG] pgmap v1159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:58.200 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:56 smithi006 bash[27292]: audit 2023-08-27T22:36:56.691312+0000 mon.a (mon.0) 1428 : audit [DBG] from='client.? 172.21.15.6:0/3736336411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:58.200 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:56 smithi130 bash[27579]: cluster 2023-08-27T22:36:55.803543+0000 mgr.y (mgr.14901) 1152 : cluster [DBG] pgmap v1159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:36:58.200 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:56 smithi130 bash[27579]: audit 2023-08-27T22:36:56.691312+0000 mon.a (mon.0) 1428 : audit [DBG] from='client.? 172.21.15.6:0/3736336411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:59.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:58 smithi006 bash[27292]: cluster 2023-08-27T22:36:57.804206+0000 mgr.y (mgr.14901) 1153 : cluster [DBG] pgmap v1160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:00.377 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:58 smithi006 bash[33918]: cluster 2023-08-27T22:36:57.804206+0000 mgr.y (mgr.14901) 1153 : cluster [DBG] pgmap v1160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:00.377 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:36:59 smithi006 bash[33918]: audit 2023-08-27T22:36:59.107886+0000 mon.c (mon.2) 472 : audit [DBG] from='client.? 172.21.15.6:0/1734833362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:00.377 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:36:59 smithi006 bash[27292]: audit 2023-08-27T22:36:59.107886+0000 mon.c (mon.2) 472 : audit [DBG] from='client.? 172.21.15.6:0/1734833362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:00.378 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:58 smithi130 bash[27579]: cluster 2023-08-27T22:36:57.804206+0000 mgr.y (mgr.14901) 1153 : cluster [DBG] pgmap v1160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:00.378 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:36:59 smithi130 bash[27579]: audit 2023-08-27T22:36:59.107886+0000 mon.c (mon.2) 472 : audit [DBG] from='client.? 172.21.15.6:0/1734833362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:00.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:37:00.242Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:00.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:37:00.244Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:01.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:00 smithi006 bash[27292]: cluster 2023-08-27T22:36:59.805368+0000 mgr.y (mgr.14901) 1154 : cluster [DBG] pgmap v1161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:01.243 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:37:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:37:00] "GET /metrics HTTP/1.1" 200 193078 "" "Prometheus/2.33.4" 2023-08-27T22:37:01.243 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:00 smithi006 bash[33918]: cluster 2023-08-27T22:36:59.805368+0000 mgr.y (mgr.14901) 1154 : cluster [DBG] pgmap v1161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:01.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:00 smithi130 bash[27579]: cluster 2023-08-27T22:36:59.805368+0000 mgr.y (mgr.14901) 1154 : cluster [DBG] pgmap v1161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:02.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:01 smithi006 bash[33918]: audit 2023-08-27T22:37:01.534537+0000 mon.c (mon.2) 473 : audit [DBG] from='client.? 172.21.15.6:0/4245004678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:02.527 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:01 smithi006 bash[27292]: audit 2023-08-27T22:37:01.534537+0000 mon.c (mon.2) 473 : audit [DBG] from='client.? 172.21.15.6:0/4245004678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:02.527 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:37:01.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:02.527 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:37:01.810Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:02.527 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:01 smithi130 bash[27579]: audit 2023-08-27T22:37:01.534537+0000 mon.c (mon.2) 473 : audit [DBG] from='client.? 172.21.15.6:0/4245004678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:03.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:02 smithi006 bash[33918]: cluster 2023-08-27T22:37:01.806059+0000 mgr.y (mgr.14901) 1155 : cluster [DBG] pgmap v1162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:03.354 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:02 smithi006 bash[27292]: cluster 2023-08-27T22:37:01.806059+0000 mgr.y (mgr.14901) 1155 : cluster [DBG] pgmap v1162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:03.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:02 smithi130 bash[27579]: cluster 2023-08-27T22:37:01.806059+0000 mgr.y (mgr.14901) 1155 : cluster [DBG] pgmap v1162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:37:03.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:03.980 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:37:03.702Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:04 smithi130 bash[27579]: cluster 2023-08-27T22:37:03.806655+0000 mgr.y (mgr.14901) 1156 : cluster [DBG] pgmap v1163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:05.126 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:04 smithi130 bash[27579]: audit 2023-08-27T22:37:03.958517+0000 mon.a (mon.0) 1429 : audit [DBG] from='client.? 172.21.15.6:0/2272862594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:05.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:04 smithi006 bash[27292]: cluster 2023-08-27T22:37:03.806655+0000 mgr.y (mgr.14901) 1156 : cluster [DBG] pgmap v1163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:05.232 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:04 smithi006 bash[27292]: audit 2023-08-27T22:37:03.958517+0000 mon.a (mon.0) 1429 : audit [DBG] from='client.? 172.21.15.6:0/2272862594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:05.232 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:04 smithi006 bash[33918]: cluster 2023-08-27T22:37:03.806655+0000 mgr.y (mgr.14901) 1156 : cluster [DBG] pgmap v1163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:05.232 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:04 smithi006 bash[33918]: audit 2023-08-27T22:37:03.958517+0000 mon.a (mon.0) 1429 : audit [DBG] from='client.? 172.21.15.6:0/2272862594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:07.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:06 smithi006 bash[27292]: cluster 2023-08-27T22:37:05.807752+0000 mgr.y (mgr.14901) 1157 : cluster [DBG] pgmap v1164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:07.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:06 smithi006 bash[27292]: audit 2023-08-27T22:37:06.376041+0000 mon.a (mon.0) 1430 : audit [DBG] from='client.? 172.21.15.6:0/235689033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:07.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:06 smithi006 bash[33918]: cluster 2023-08-27T22:37:05.807752+0000 mgr.y (mgr.14901) 1157 : cluster [DBG] pgmap v1164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:07.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:06 smithi006 bash[33918]: audit 2023-08-27T22:37:06.376041+0000 mon.a (mon.0) 1430 : audit [DBG] from='client.? 172.21.15.6:0/235689033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:07.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:06 smithi130 bash[27579]: cluster 2023-08-27T22:37:05.807752+0000 mgr.y (mgr.14901) 1157 : cluster [DBG] pgmap v1164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:07.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:06 smithi130 bash[27579]: audit 2023-08-27T22:37:06.376041+0000 mon.a (mon.0) 1430 : audit [DBG] from='client.? 172.21.15.6:0/235689033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:09.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:08 smithi006 bash[27292]: cluster 2023-08-27T22:37:07.808420+0000 mgr.y (mgr.14901) 1158 : cluster [DBG] pgmap v1165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:09.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:08 smithi006 bash[27292]: audit 2023-08-27T22:37:08.799409+0000 mon.c (mon.2) 474 : audit [DBG] from='client.? 172.21.15.6:0/2889654157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:08 smithi006 bash[33918]: cluster 2023-08-27T22:37:07.808420+0000 mgr.y (mgr.14901) 1158 : cluster [DBG] pgmap v1165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:08 smithi006 bash[33918]: audit 2023-08-27T22:37:08.799409+0000 mon.c (mon.2) 474 : audit [DBG] from='client.? 172.21.15.6:0/2889654157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:09.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:08 smithi130 bash[27579]: cluster 2023-08-27T22:37:07.808420+0000 mgr.y (mgr.14901) 1158 : cluster [DBG] pgmap v1165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:09.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:08 smithi130 bash[27579]: audit 2023-08-27T22:37:08.799409+0000 mon.c (mon.2) 474 : audit [DBG] from='client.? 172.21.15.6:0/2889654157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:37:10.243Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:37:10.245Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:11.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:37:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:37:10] "GET /metrics HTTP/1.1" 200 193078 "" "Prometheus/2.33.4" 2023-08-27T22:37:11.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:10 smithi006 bash[33918]: cluster 2023-08-27T22:37:09.809547+0000 mgr.y (mgr.14901) 1159 : cluster [DBG] pgmap v1166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:11.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:10 smithi006 bash[27292]: cluster 2023-08-27T22:37:09.809547+0000 mgr.y (mgr.14901) 1159 : cluster [DBG] pgmap v1166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:11.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:10 smithi130 bash[27579]: cluster 2023-08-27T22:37:09.809547+0000 mgr.y (mgr.14901) 1159 : cluster [DBG] pgmap v1166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:12.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:11 smithi006 bash[33918]: audit 2023-08-27T22:37:11.219489+0000 mon.a (mon.0) 1431 : audit [DBG] from='client.? 172.21.15.6:0/3534976550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:12.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:11 smithi006 bash[27292]: audit 2023-08-27T22:37:11.219489+0000 mon.a (mon.0) 1431 : audit [DBG] from='client.? 172.21.15.6:0/3534976550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:37:11.809Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:37:11.811Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:12.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:11 smithi130 bash[27579]: audit 2023-08-27T22:37:11.219489+0000 mon.a (mon.0) 1431 : audit [DBG] from='client.? 172.21.15.6:0/3534976550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:13.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:12 smithi006 bash[33918]: cluster 2023-08-27T22:37:11.810180+0000 mgr.y (mgr.14901) 1160 : cluster [DBG] pgmap v1167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:13.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:12 smithi006 bash[27292]: cluster 2023-08-27T22:37:11.810180+0000 mgr.y (mgr.14901) 1160 : cluster [DBG] pgmap v1167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:13.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:12 smithi130 bash[27579]: cluster 2023-08-27T22:37:11.810180+0000 mgr.y (mgr.14901) 1160 : cluster [DBG] pgmap v1167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:13.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:13 smithi006 bash[33918]: audit 2023-08-27T22:37:13.633628+0000 mon.c (mon.2) 475 : audit [DBG] from='client.? 172.21.15.6:0/1676767482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:13.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:13 smithi006 bash[27292]: audit 2023-08-27T22:37:13.633628+0000 mon.c (mon.2) 475 : audit [DBG] from='client.? 172.21.15.6:0/1676767482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:37:13.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:37:13.704Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:14.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:13 smithi130 bash[27579]: audit 2023-08-27T22:37:13.633628+0000 mon.c (mon.2) 475 : audit [DBG] from='client.? 172.21.15.6:0/1676767482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:15.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:14 smithi006 bash[27292]: cluster 2023-08-27T22:37:13.810839+0000 mgr.y (mgr.14901) 1161 : cluster [DBG] pgmap v1168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:15.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:14 smithi006 bash[33918]: cluster 2023-08-27T22:37:13.810839+0000 mgr.y (mgr.14901) 1161 : cluster [DBG] pgmap v1168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:15.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:14 smithi130 bash[27579]: cluster 2023-08-27T22:37:13.810839+0000 mgr.y (mgr.14901) 1161 : cluster [DBG] pgmap v1168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:17.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:16 smithi006 bash[33918]: cluster 2023-08-27T22:37:15.811916+0000 mgr.y (mgr.14901) 1162 : cluster [DBG] pgmap v1169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:17.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:16 smithi006 bash[33918]: audit 2023-08-27T22:37:16.047557+0000 mon.c (mon.2) 476 : audit [DBG] from='client.? 172.21.15.6:0/1918600437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:17.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:16 smithi006 bash[27292]: cluster 2023-08-27T22:37:15.811916+0000 mgr.y (mgr.14901) 1162 : cluster [DBG] pgmap v1169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:17.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:16 smithi006 bash[27292]: audit 2023-08-27T22:37:16.047557+0000 mon.c (mon.2) 476 : audit [DBG] from='client.? 172.21.15.6:0/1918600437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:17.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:16 smithi130 bash[27579]: cluster 2023-08-27T22:37:15.811916+0000 mgr.y (mgr.14901) 1162 : cluster [DBG] pgmap v1169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:17.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:16 smithi130 bash[27579]: audit 2023-08-27T22:37:16.047557+0000 mon.c (mon.2) 476 : audit [DBG] from='client.? 172.21.15.6:0/1918600437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:19.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:18 smithi006 bash[27292]: cluster 2023-08-27T22:37:17.812574+0000 mgr.y (mgr.14901) 1163 : cluster [DBG] pgmap v1170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:19.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:18 smithi006 bash[27292]: audit 2023-08-27T22:37:18.474306+0000 mon.a (mon.0) 1432 : audit [DBG] from='client.? 172.21.15.6:0/2366307396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:19.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:18 smithi006 bash[33918]: cluster 2023-08-27T22:37:17.812574+0000 mgr.y (mgr.14901) 1163 : cluster [DBG] pgmap v1170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:19.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:18 smithi006 bash[33918]: audit 2023-08-27T22:37:18.474306+0000 mon.a (mon.0) 1432 : audit [DBG] from='client.? 172.21.15.6:0/2366307396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:19.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:18 smithi130 bash[27579]: cluster 2023-08-27T22:37:17.812574+0000 mgr.y (mgr.14901) 1163 : cluster [DBG] pgmap v1170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:19.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:18 smithi130 bash[27579]: audit 2023-08-27T22:37:18.474306+0000 mon.a (mon.0) 1432 : audit [DBG] from='client.? 172.21.15.6:0/2366307396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:37:20.243Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:37:20.246Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:21.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:37:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:37:20] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T22:37:21.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:20 smithi006 bash[27292]: cluster 2023-08-27T22:37:19.813725+0000 mgr.y (mgr.14901) 1164 : cluster [DBG] pgmap v1171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:21.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:20 smithi006 bash[27292]: audit 2023-08-27T22:37:20.893993+0000 mon.c (mon.2) 477 : audit [DBG] from='client.? 172.21.15.6:0/2878830191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:21.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:20 smithi006 bash[33918]: cluster 2023-08-27T22:37:19.813725+0000 mgr.y (mgr.14901) 1164 : cluster [DBG] pgmap v1171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:21.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:20 smithi006 bash[33918]: audit 2023-08-27T22:37:20.893993+0000 mon.c (mon.2) 477 : audit [DBG] from='client.? 172.21.15.6:0/2878830191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:21.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:20 smithi130 bash[27579]: cluster 2023-08-27T22:37:19.813725+0000 mgr.y (mgr.14901) 1164 : cluster [DBG] pgmap v1171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:21.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:20 smithi130 bash[27579]: audit 2023-08-27T22:37:20.893993+0000 mon.c (mon.2) 477 : audit [DBG] from='client.? 172.21.15.6:0/2878830191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:37:21.810Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:37:21.812Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:23.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:22 smithi006 bash[27292]: cluster 2023-08-27T22:37:21.814392+0000 mgr.y (mgr.14901) 1165 : cluster [DBG] pgmap v1172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:23.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:22 smithi006 bash[33918]: cluster 2023-08-27T22:37:21.814392+0000 mgr.y (mgr.14901) 1165 : cluster [DBG] pgmap v1172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:23.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:22 smithi130 bash[27579]: cluster 2023-08-27T22:37:21.814392+0000 mgr.y (mgr.14901) 1165 : cluster [DBG] pgmap v1172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:23.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:23 smithi006 bash[33918]: audit 2023-08-27T22:37:23.306489+0000 mon.a (mon.0) 1433 : audit [DBG] from='client.? 172.21.15.6:0/1113514584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:23.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:23 smithi006 bash[27292]: audit 2023-08-27T22:37:23.306489+0000 mon.a (mon.0) 1433 : audit [DBG] from='client.? 172.21.15.6:0/1113514584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:37:23.702Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:23.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:37:23.704Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:24.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:23 smithi130 bash[27579]: audit 2023-08-27T22:37:23.306489+0000 mon.a (mon.0) 1433 : audit [DBG] from='client.? 172.21.15.6:0/1113514584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:25.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:24 smithi006 bash[33918]: cluster 2023-08-27T22:37:23.815063+0000 mgr.y (mgr.14901) 1166 : cluster [DBG] pgmap v1173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:25.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:24 smithi006 bash[27292]: cluster 2023-08-27T22:37:23.815063+0000 mgr.y (mgr.14901) 1166 : cluster [DBG] pgmap v1173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:25.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:24 smithi130 bash[27579]: cluster 2023-08-27T22:37:23.815063+0000 mgr.y (mgr.14901) 1166 : cluster [DBG] pgmap v1173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:26.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:25 smithi006 bash[33918]: audit 2023-08-27T22:37:25.720030+0000 mon.c (mon.2) 478 : audit [DBG] from='client.? 172.21.15.6:0/1970044127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:26.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:25 smithi006 bash[27292]: audit 2023-08-27T22:37:25.720030+0000 mon.c (mon.2) 478 : audit [DBG] from='client.? 172.21.15.6:0/1970044127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:26.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:25 smithi130 bash[27579]: audit 2023-08-27T22:37:25.720030+0000 mon.c (mon.2) 478 : audit [DBG] from='client.? 172.21.15.6:0/1970044127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:26 smithi006 bash[27292]: cluster 2023-08-27T22:37:25.816182+0000 mgr.y (mgr.14901) 1167 : cluster [DBG] pgmap v1174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:27.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:26 smithi006 bash[33918]: cluster 2023-08-27T22:37:25.816182+0000 mgr.y (mgr.14901) 1167 : cluster [DBG] pgmap v1174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:27.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:26 smithi130 bash[27579]: cluster 2023-08-27T22:37:25.816182+0000 mgr.y (mgr.14901) 1167 : cluster [DBG] pgmap v1174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:29.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:28 smithi006 bash[33918]: cluster 2023-08-27T22:37:27.816817+0000 mgr.y (mgr.14901) 1168 : cluster [DBG] pgmap v1175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:29.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:28 smithi006 bash[33918]: audit 2023-08-27T22:37:28.138944+0000 mon.c (mon.2) 479 : audit [DBG] from='client.? 172.21.15.6:0/1462186549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:29.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:28 smithi006 bash[27292]: cluster 2023-08-27T22:37:27.816817+0000 mgr.y (mgr.14901) 1168 : cluster [DBG] pgmap v1175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:29.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:28 smithi006 bash[27292]: audit 2023-08-27T22:37:28.138944+0000 mon.c (mon.2) 479 : audit [DBG] from='client.? 172.21.15.6:0/1462186549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:29.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:28 smithi130 bash[27579]: cluster 2023-08-27T22:37:27.816817+0000 mgr.y (mgr.14901) 1168 : cluster [DBG] pgmap v1175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:29.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:28 smithi130 bash[27579]: audit 2023-08-27T22:37:28.138944+0000 mon.c (mon.2) 479 : audit [DBG] from='client.? 172.21.15.6:0/1462186549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:37:30.244Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:37:30.246Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:31.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:37:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:37:30] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T22:37:31.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:30 smithi006 bash[27292]: cluster 2023-08-27T22:37:29.817893+0000 mgr.y (mgr.14901) 1169 : cluster [DBG] pgmap v1176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:31.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:30 smithi006 bash[27292]: audit 2023-08-27T22:37:30.579075+0000 mon.a (mon.0) 1434 : audit [DBG] from='client.? 172.21.15.6:0/2686356273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:31.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:30 smithi006 bash[33918]: cluster 2023-08-27T22:37:29.817893+0000 mgr.y (mgr.14901) 1169 : cluster [DBG] pgmap v1176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:31.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:30 smithi006 bash[33918]: audit 2023-08-27T22:37:30.579075+0000 mon.a (mon.0) 1434 : audit [DBG] from='client.? 172.21.15.6:0/2686356273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:31.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:30 smithi130 bash[27579]: cluster 2023-08-27T22:37:29.817893+0000 mgr.y (mgr.14901) 1169 : cluster [DBG] pgmap v1176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:31.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:30 smithi130 bash[27579]: audit 2023-08-27T22:37:30.579075+0000 mon.a (mon.0) 1434 : audit [DBG] from='client.? 172.21.15.6:0/2686356273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:37:31.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:37:31.813Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:32 smithi006 bash[27292]: cluster 2023-08-27T22:37:31.818544+0000 mgr.y (mgr.14901) 1170 : cluster [DBG] pgmap v1177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:32 smithi006 bash[27292]: audit 2023-08-27T22:37:32.637693+0000 mon.b (mon.1) 235 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:37:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:32 smithi006 bash[27292]: audit 2023-08-27T22:37:32.899327+0000 mon.a (mon.0) 1435 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:37:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:32 smithi006 bash[27292]: audit 2023-08-27T22:37:32.908909+0000 mon.a (mon.0) 1436 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:37:33.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:32 smithi006 bash[33918]: cluster 2023-08-27T22:37:31.818544+0000 mgr.y (mgr.14901) 1170 : cluster [DBG] pgmap v1177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:33.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:32 smithi006 bash[33918]: audit 2023-08-27T22:37:32.637693+0000 mon.b (mon.1) 235 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:37:33.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:32 smithi006 bash[33918]: audit 2023-08-27T22:37:32.899327+0000 mon.a (mon.0) 1435 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:37:33.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:32 smithi006 bash[33918]: audit 2023-08-27T22:37:32.908909+0000 mon.a (mon.0) 1436 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:37:33.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:32 smithi130 bash[27579]: cluster 2023-08-27T22:37:31.818544+0000 mgr.y (mgr.14901) 1170 : cluster [DBG] pgmap v1177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:33.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:32 smithi130 bash[27579]: audit 2023-08-27T22:37:32.637693+0000 mon.b (mon.1) 235 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:37:33.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:32 smithi130 bash[27579]: audit 2023-08-27T22:37:32.899327+0000 mon.a (mon.0) 1435 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:37:33.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:32 smithi130 bash[27579]: audit 2023-08-27T22:37:32.908909+0000 mon.a (mon.0) 1436 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:37:33.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:33 smithi006 bash[33918]: audit 2023-08-27T22:37:33.010131+0000 mon.a (mon.0) 1437 : audit [DBG] from='client.? 172.21.15.6:0/1789198819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:33.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:33 smithi006 bash[27292]: audit 2023-08-27T22:37:33.010131+0000 mon.a (mon.0) 1437 : audit [DBG] from='client.? 172.21.15.6:0/1789198819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:37:33.703Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:37:33.705Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:34.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:33 smithi130 bash[27579]: audit 2023-08-27T22:37:33.010131+0000 mon.a (mon.0) 1437 : audit [DBG] from='client.? 172.21.15.6:0/1789198819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:35.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:34 smithi006 bash[27292]: cluster 2023-08-27T22:37:33.819208+0000 mgr.y (mgr.14901) 1171 : cluster [DBG] pgmap v1178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:35.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:34 smithi006 bash[33918]: cluster 2023-08-27T22:37:33.819208+0000 mgr.y (mgr.14901) 1171 : cluster [DBG] pgmap v1178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:35.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:34 smithi130 bash[27579]: cluster 2023-08-27T22:37:33.819208+0000 mgr.y (mgr.14901) 1171 : cluster [DBG] pgmap v1178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:36.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:35 smithi006 bash[33918]: audit 2023-08-27T22:37:35.424061+0000 mon.a (mon.0) 1438 : audit [DBG] from='client.? 172.21.15.6:0/4128420608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:36.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:35 smithi006 bash[27292]: audit 2023-08-27T22:37:35.424061+0000 mon.a (mon.0) 1438 : audit [DBG] from='client.? 172.21.15.6:0/4128420608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:36.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:35 smithi130 bash[27579]: audit 2023-08-27T22:37:35.424061+0000 mon.a (mon.0) 1438 : audit [DBG] from='client.? 172.21.15.6:0/4128420608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:37.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:36 smithi006 bash[33918]: cluster 2023-08-27T22:37:35.820288+0000 mgr.y (mgr.14901) 1172 : cluster [DBG] pgmap v1179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:37.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:36 smithi006 bash[27292]: cluster 2023-08-27T22:37:35.820288+0000 mgr.y (mgr.14901) 1172 : cluster [DBG] pgmap v1179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:37.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:36 smithi130 bash[27579]: cluster 2023-08-27T22:37:35.820288+0000 mgr.y (mgr.14901) 1172 : cluster [DBG] pgmap v1179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:38.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:37 smithi130 bash[27579]: audit 2023-08-27T22:37:37.843620+0000 mon.a (mon.0) 1439 : audit [DBG] from='client.? 172.21.15.6:0/1322736199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:38.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:37 smithi006 bash[33918]: audit 2023-08-27T22:37:37.843620+0000 mon.a (mon.0) 1439 : audit [DBG] from='client.? 172.21.15.6:0/1322736199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:38.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:37 smithi006 bash[27292]: audit 2023-08-27T22:37:37.843620+0000 mon.a (mon.0) 1439 : audit [DBG] from='client.? 172.21.15.6:0/1322736199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:39.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:38 smithi130 bash[27579]: cluster 2023-08-27T22:37:37.820692+0000 mgr.y (mgr.14901) 1173 : cluster [DBG] pgmap v1180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:39.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:38 smithi130 bash[27579]: audit 2023-08-27T22:37:38.747106+0000 mon.a (mon.0) 1440 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:37:39.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:38 smithi006 bash[33918]: cluster 2023-08-27T22:37:37.820692+0000 mgr.y (mgr.14901) 1173 : cluster [DBG] pgmap v1180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:39.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:38 smithi006 bash[33918]: audit 2023-08-27T22:37:38.747106+0000 mon.a (mon.0) 1440 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:37:39.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:38 smithi006 bash[27292]: cluster 2023-08-27T22:37:37.820692+0000 mgr.y (mgr.14901) 1173 : cluster [DBG] pgmap v1180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:39.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:38 smithi006 bash[27292]: audit 2023-08-27T22:37:38.747106+0000 mon.a (mon.0) 1440 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:37:40.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:40 smithi130 bash[27579]: audit 2023-08-27T22:37:39.278997+0000 mon.a (mon.0) 1441 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:37:40.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:40 smithi130 bash[27579]: audit 2023-08-27T22:37:39.579161+0000 mon.b (mon.1) 236 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:37:40.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:40 smithi130 bash[27579]: audit 2023-08-27T22:37:39.581605+0000 mon.b (mon.1) 237 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:37:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:40 smithi130 bash[27579]: audit 2023-08-27T22:37:39.586482+0000 mon.a (mon.0) 1442 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:37:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:40 smithi130 bash[27579]: cluster 2023-08-27T22:37:39.821770+0000 mgr.y (mgr.14901) 1174 : cluster [DBG] pgmap v1181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:40 smithi130 bash[27579]: audit 2023-08-27T22:37:40.273688+0000 mon.c (mon.2) 480 : audit [DBG] from='client.? 172.21.15.6:0/3264928960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:37:40.245Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:37:40.247Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:40.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:40 smithi006 bash[27292]: audit 2023-08-27T22:37:39.278997+0000 mon.a (mon.0) 1441 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:37:40.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:40 smithi006 bash[27292]: audit 2023-08-27T22:37:39.579161+0000 mon.b (mon.1) 236 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:37:40.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:40 smithi006 bash[27292]: audit 2023-08-27T22:37:39.581605+0000 mon.b (mon.1) 237 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:37:40.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:40 smithi006 bash[27292]: audit 2023-08-27T22:37:39.586482+0000 mon.a (mon.0) 1442 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:37:40.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:40 smithi006 bash[27292]: cluster 2023-08-27T22:37:39.821770+0000 mgr.y (mgr.14901) 1174 : cluster [DBG] pgmap v1181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:40.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:40 smithi006 bash[27292]: audit 2023-08-27T22:37:40.273688+0000 mon.c (mon.2) 480 : audit [DBG] from='client.? 172.21.15.6:0/3264928960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:40.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:40 smithi006 bash[33918]: audit 2023-08-27T22:37:39.278997+0000 mon.a (mon.0) 1441 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:37:40.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:40 smithi006 bash[33918]: audit 2023-08-27T22:37:39.579161+0000 mon.b (mon.1) 236 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:37:40.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:40 smithi006 bash[33918]: audit 2023-08-27T22:37:39.581605+0000 mon.b (mon.1) 237 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:37:40.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:40 smithi006 bash[33918]: audit 2023-08-27T22:37:39.586482+0000 mon.a (mon.0) 1442 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:37:40.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:40 smithi006 bash[33918]: cluster 2023-08-27T22:37:39.821770+0000 mgr.y (mgr.14901) 1174 : cluster [DBG] pgmap v1181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:40.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:40 smithi006 bash[33918]: audit 2023-08-27T22:37:40.273688+0000 mon.c (mon.2) 480 : audit [DBG] from='client.? 172.21.15.6:0/3264928960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:41.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:37:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:37:40] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T22:37:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:37:41.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:37:41.813Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:43.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:42 smithi006 bash[27292]: cluster 2023-08-27T22:37:41.822478+0000 mgr.y (mgr.14901) 1175 : cluster [DBG] pgmap v1182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:43.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:42 smithi006 bash[27292]: audit 2023-08-27T22:37:42.696542+0000 mon.c (mon.2) 481 : audit [DBG] from='client.? 172.21.15.6:0/1065383090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:43.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:42 smithi006 bash[33918]: cluster 2023-08-27T22:37:41.822478+0000 mgr.y (mgr.14901) 1175 : cluster [DBG] pgmap v1182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:43.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:42 smithi006 bash[33918]: audit 2023-08-27T22:37:42.696542+0000 mon.c (mon.2) 481 : audit [DBG] from='client.? 172.21.15.6:0/1065383090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:43.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:42 smithi130 bash[27579]: cluster 2023-08-27T22:37:41.822478+0000 mgr.y (mgr.14901) 1175 : cluster [DBG] pgmap v1182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:43.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:42 smithi130 bash[27579]: audit 2023-08-27T22:37:42.696542+0000 mon.c (mon.2) 481 : audit [DBG] from='client.? 172.21.15.6:0/1065383090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:43.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:37:43.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:43.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:37:43.707Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:44 smithi130 bash[27579]: cluster 2023-08-27T22:37:43.823135+0000 mgr.y (mgr.14901) 1176 : cluster [DBG] pgmap v1183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:45.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:44 smithi006 bash[33918]: cluster 2023-08-27T22:37:43.823135+0000 mgr.y (mgr.14901) 1176 : cluster [DBG] pgmap v1183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:45.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:44 smithi006 bash[27292]: cluster 2023-08-27T22:37:43.823135+0000 mgr.y (mgr.14901) 1176 : cluster [DBG] pgmap v1183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:46.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:45 smithi130 bash[27579]: audit 2023-08-27T22:37:45.119627+0000 mon.c (mon.2) 482 : audit [DBG] from='client.? 172.21.15.6:0/2777106337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:46.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:45 smithi006 bash[33918]: audit 2023-08-27T22:37:45.119627+0000 mon.c (mon.2) 482 : audit [DBG] from='client.? 172.21.15.6:0/2777106337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:46.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:45 smithi006 bash[27292]: audit 2023-08-27T22:37:45.119627+0000 mon.c (mon.2) 482 : audit [DBG] from='client.? 172.21.15.6:0/2777106337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:46 smithi130 bash[27579]: cluster 2023-08-27T22:37:45.824247+0000 mgr.y (mgr.14901) 1177 : cluster [DBG] pgmap v1184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:47.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:46 smithi006 bash[33918]: cluster 2023-08-27T22:37:45.824247+0000 mgr.y (mgr.14901) 1177 : cluster [DBG] pgmap v1184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:47.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:46 smithi006 bash[27292]: cluster 2023-08-27T22:37:45.824247+0000 mgr.y (mgr.14901) 1177 : cluster [DBG] pgmap v1184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:48.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:47 smithi130 bash[27579]: audit 2023-08-27T22:37:47.535214+0000 mon.a (mon.0) 1443 : audit [DBG] from='client.? 172.21.15.6:0/2894139063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:48.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:47 smithi006 bash[33918]: audit 2023-08-27T22:37:47.535214+0000 mon.a (mon.0) 1443 : audit [DBG] from='client.? 172.21.15.6:0/2894139063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:48.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:47 smithi006 bash[27292]: audit 2023-08-27T22:37:47.535214+0000 mon.a (mon.0) 1443 : audit [DBG] from='client.? 172.21.15.6:0/2894139063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:49.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:48 smithi130 bash[27579]: cluster 2023-08-27T22:37:47.824895+0000 mgr.y (mgr.14901) 1178 : cluster [DBG] pgmap v1185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:49.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:48 smithi006 bash[33918]: cluster 2023-08-27T22:37:47.824895+0000 mgr.y (mgr.14901) 1178 : cluster [DBG] pgmap v1185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:49.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:48 smithi006 bash[27292]: cluster 2023-08-27T22:37:47.824895+0000 mgr.y (mgr.14901) 1178 : cluster [DBG] pgmap v1185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:37:50.246Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:37:50.248Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:51.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:37:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:37:50] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T22:37:51.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:50 smithi006 bash[27292]: cluster 2023-08-27T22:37:49.825614+0000 mgr.y (mgr.14901) 1179 : cluster [DBG] pgmap v1186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:51.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:50 smithi006 bash[27292]: audit 2023-08-27T22:37:49.944222+0000 mon.a (mon.0) 1444 : audit [DBG] from='client.? 172.21.15.6:0/1016084746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:51.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:50 smithi006 bash[33918]: cluster 2023-08-27T22:37:49.825614+0000 mgr.y (mgr.14901) 1179 : cluster [DBG] pgmap v1186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:51.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:50 smithi006 bash[33918]: audit 2023-08-27T22:37:49.944222+0000 mon.a (mon.0) 1444 : audit [DBG] from='client.? 172.21.15.6:0/1016084746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:51.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:50 smithi130 bash[27579]: cluster 2023-08-27T22:37:49.825614+0000 mgr.y (mgr.14901) 1179 : cluster [DBG] pgmap v1186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:51.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:50 smithi130 bash[27579]: audit 2023-08-27T22:37:49.944222+0000 mon.a (mon.0) 1444 : audit [DBG] from='client.? 172.21.15.6:0/1016084746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:37:51.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:37:51.813Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:53.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:52 smithi006 bash[27292]: cluster 2023-08-27T22:37:51.826229+0000 mgr.y (mgr.14901) 1180 : cluster [DBG] pgmap v1187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:53.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:52 smithi006 bash[27292]: audit 2023-08-27T22:37:52.358433+0000 mon.c (mon.2) 483 : audit [DBG] from='client.? 172.21.15.6:0/3040042254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:53.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:52 smithi006 bash[33918]: cluster 2023-08-27T22:37:51.826229+0000 mgr.y (mgr.14901) 1180 : cluster [DBG] pgmap v1187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:53.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:52 smithi006 bash[33918]: audit 2023-08-27T22:37:52.358433+0000 mon.c (mon.2) 483 : audit [DBG] from='client.? 172.21.15.6:0/3040042254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:53.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:52 smithi130 bash[27579]: cluster 2023-08-27T22:37:51.826229+0000 mgr.y (mgr.14901) 1180 : cluster [DBG] pgmap v1187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:53.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:52 smithi130 bash[27579]: audit 2023-08-27T22:37:52.358433+0000 mon.c (mon.2) 483 : audit [DBG] from='client.? 172.21.15.6:0/3040042254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:37:53.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:37:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:37:53.708Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:37:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:54 smithi130 bash[27579]: cluster 2023-08-27T22:37:53.826905+0000 mgr.y (mgr.14901) 1181 : cluster [DBG] pgmap v1188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:55.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:54 smithi006 bash[27292]: cluster 2023-08-27T22:37:53.826905+0000 mgr.y (mgr.14901) 1181 : cluster [DBG] pgmap v1188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:55.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:54 smithi006 bash[33918]: cluster 2023-08-27T22:37:53.826905+0000 mgr.y (mgr.14901) 1181 : cluster [DBG] pgmap v1188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:56.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:55 smithi130 bash[27579]: audit 2023-08-27T22:37:54.778693+0000 mon.c (mon.2) 484 : audit [DBG] from='client.? 172.21.15.6:0/363279334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:56.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:55 smithi006 bash[27292]: audit 2023-08-27T22:37:54.778693+0000 mon.c (mon.2) 484 : audit [DBG] from='client.? 172.21.15.6:0/363279334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:56.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:55 smithi006 bash[33918]: audit 2023-08-27T22:37:54.778693+0000 mon.c (mon.2) 484 : audit [DBG] from='client.? 172.21.15.6:0/363279334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:56 smithi130 bash[27579]: cluster 2023-08-27T22:37:55.828023+0000 mgr.y (mgr.14901) 1182 : cluster [DBG] pgmap v1189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:57.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:56 smithi006 bash[27292]: cluster 2023-08-27T22:37:55.828023+0000 mgr.y (mgr.14901) 1182 : cluster [DBG] pgmap v1189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:57.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:56 smithi006 bash[33918]: cluster 2023-08-27T22:37:55.828023+0000 mgr.y (mgr.14901) 1182 : cluster [DBG] pgmap v1189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:58.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:57 smithi130 bash[27579]: audit 2023-08-27T22:37:57.189671+0000 mon.a (mon.0) 1445 : audit [DBG] from='client.? 172.21.15.6:0/3966471917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:58.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:57 smithi006 bash[27292]: audit 2023-08-27T22:37:57.189671+0000 mon.a (mon.0) 1445 : audit [DBG] from='client.? 172.21.15.6:0/3966471917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:58.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:57 smithi006 bash[33918]: audit 2023-08-27T22:37:57.189671+0000 mon.a (mon.0) 1445 : audit [DBG] from='client.? 172.21.15.6:0/3966471917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:58 smithi130 bash[27579]: cluster 2023-08-27T22:37:57.828695+0000 mgr.y (mgr.14901) 1183 : cluster [DBG] pgmap v1190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:59.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:58 smithi006 bash[33918]: cluster 2023-08-27T22:37:57.828695+0000 mgr.y (mgr.14901) 1183 : cluster [DBG] pgmap v1190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:37:59.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:58 smithi006 bash[27292]: cluster 2023-08-27T22:37:57.828695+0000 mgr.y (mgr.14901) 1183 : cluster [DBG] pgmap v1190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:00.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:37:59 smithi130 bash[27579]: audit 2023-08-27T22:37:59.600477+0000 mon.c (mon.2) 485 : audit [DBG] from='client.? 172.21.15.6:0/4173141635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:00.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:37:59 smithi006 bash[33918]: audit 2023-08-27T22:37:59.600477+0000 mon.c (mon.2) 485 : audit [DBG] from='client.? 172.21.15.6:0/4173141635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:00.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:37:59 smithi006 bash[27292]: audit 2023-08-27T22:37:59.600477+0000 mon.c (mon.2) 485 : audit [DBG] from='client.? 172.21.15.6:0/4173141635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:00.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:38:00.246Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:00.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:38:00.248Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:00 smithi130 bash[27579]: cluster 2023-08-27T22:37:59.829965+0000 mgr.y (mgr.14901) 1184 : cluster [DBG] pgmap v1191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:01.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:38:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:38:00] "GET /metrics HTTP/1.1" 200 193083 "" "Prometheus/2.33.4" 2023-08-27T22:38:01.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:00 smithi006 bash[27292]: cluster 2023-08-27T22:37:59.829965+0000 mgr.y (mgr.14901) 1184 : cluster [DBG] pgmap v1191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:01.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:00 smithi006 bash[33918]: cluster 2023-08-27T22:37:59.829965+0000 mgr.y (mgr.14901) 1184 : cluster [DBG] pgmap v1191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:38:01.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:38:01.814Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:03.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:02 smithi006 bash[33918]: cluster 2023-08-27T22:38:01.830586+0000 mgr.y (mgr.14901) 1185 : cluster [DBG] pgmap v1192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:03.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:02 smithi006 bash[33918]: audit 2023-08-27T22:38:02.017290+0000 mon.a (mon.0) 1446 : audit [DBG] from='client.? 172.21.15.6:0/267347386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:03.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:02 smithi006 bash[27292]: cluster 2023-08-27T22:38:01.830586+0000 mgr.y (mgr.14901) 1185 : cluster [DBG] pgmap v1192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:03.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:02 smithi006 bash[27292]: audit 2023-08-27T22:38:02.017290+0000 mon.a (mon.0) 1446 : audit [DBG] from='client.? 172.21.15.6:0/267347386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:03.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:02 smithi130 bash[27579]: cluster 2023-08-27T22:38:01.830586+0000 mgr.y (mgr.14901) 1185 : cluster [DBG] pgmap v1192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:03.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:02 smithi130 bash[27579]: audit 2023-08-27T22:38:02.017290+0000 mon.a (mon.0) 1446 : audit [DBG] from='client.? 172.21.15.6:0/267347386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:03.974 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:38:03.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:03.974 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:38:03.709Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:04 smithi130 bash[27579]: cluster 2023-08-27T22:38:03.831232+0000 mgr.y (mgr.14901) 1186 : cluster [DBG] pgmap v1193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:04 smithi130 bash[27579]: audit 2023-08-27T22:38:04.428538+0000 mon.c (mon.2) 486 : audit [DBG] from='client.? 172.21.15.6:0/1886384563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:05.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:04 smithi006 bash[27292]: cluster 2023-08-27T22:38:03.831232+0000 mgr.y (mgr.14901) 1186 : cluster [DBG] pgmap v1193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:05.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:04 smithi006 bash[27292]: audit 2023-08-27T22:38:04.428538+0000 mon.c (mon.2) 486 : audit [DBG] from='client.? 172.21.15.6:0/1886384563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:05.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:04 smithi006 bash[33918]: cluster 2023-08-27T22:38:03.831232+0000 mgr.y (mgr.14901) 1186 : cluster [DBG] pgmap v1193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:05.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:04 smithi006 bash[33918]: audit 2023-08-27T22:38:04.428538+0000 mon.c (mon.2) 486 : audit [DBG] from='client.? 172.21.15.6:0/1886384563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:07.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:06 smithi006 bash[33918]: cluster 2023-08-27T22:38:05.832467+0000 mgr.y (mgr.14901) 1187 : cluster [DBG] pgmap v1194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:07.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:06 smithi006 bash[33918]: audit 2023-08-27T22:38:06.841086+0000 mon.c (mon.2) 487 : audit [DBG] from='client.? 172.21.15.6:0/1792733365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:07.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:06 smithi006 bash[27292]: cluster 2023-08-27T22:38:05.832467+0000 mgr.y (mgr.14901) 1187 : cluster [DBG] pgmap v1194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:07.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:06 smithi006 bash[27292]: audit 2023-08-27T22:38:06.841086+0000 mon.c (mon.2) 487 : audit [DBG] from='client.? 172.21.15.6:0/1792733365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:07.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:06 smithi130 bash[27579]: cluster 2023-08-27T22:38:05.832467+0000 mgr.y (mgr.14901) 1187 : cluster [DBG] pgmap v1194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:07.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:06 smithi130 bash[27579]: audit 2023-08-27T22:38:06.841086+0000 mon.c (mon.2) 487 : audit [DBG] from='client.? 172.21.15.6:0/1792733365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:09.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:08 smithi006 bash[27292]: cluster 2023-08-27T22:38:07.833159+0000 mgr.y (mgr.14901) 1188 : cluster [DBG] pgmap v1195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:09.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:08 smithi006 bash[33918]: cluster 2023-08-27T22:38:07.833159+0000 mgr.y (mgr.14901) 1188 : cluster [DBG] pgmap v1195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:09.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:08 smithi130 bash[27579]: cluster 2023-08-27T22:38:07.833159+0000 mgr.y (mgr.14901) 1188 : cluster [DBG] pgmap v1195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:10.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:09 smithi006 bash[27292]: audit 2023-08-27T22:38:09.267490+0000 mon.a (mon.0) 1447 : audit [DBG] from='client.? 172.21.15.6:0/3127693662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:10.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:09 smithi006 bash[33918]: audit 2023-08-27T22:38:09.267490+0000 mon.a (mon.0) 1447 : audit [DBG] from='client.? 172.21.15.6:0/3127693662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:10.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:09 smithi130 bash[27579]: audit 2023-08-27T22:38:09.267490+0000 mon.a (mon.0) 1447 : audit [DBG] from='client.? 172.21.15.6:0/3127693662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:38:10.247Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:38:10.249Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:11.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:10 smithi006 bash[33918]: cluster 2023-08-27T22:38:09.834263+0000 mgr.y (mgr.14901) 1189 : cluster [DBG] pgmap v1196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:11.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:10 smithi006 bash[27292]: cluster 2023-08-27T22:38:09.834263+0000 mgr.y (mgr.14901) 1189 : cluster [DBG] pgmap v1196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:11.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:38:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:38:10] "GET /metrics HTTP/1.1" 200 193083 "" "Prometheus/2.33.4" 2023-08-27T22:38:11.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:10 smithi130 bash[27579]: cluster 2023-08-27T22:38:09.834263+0000 mgr.y (mgr.14901) 1189 : cluster [DBG] pgmap v1196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:12.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:11 smithi006 bash[27292]: audit 2023-08-27T22:38:11.678599+0000 mon.c (mon.2) 488 : audit [DBG] from='client.? 172.21.15.6:0/1698885625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:12.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:11 smithi006 bash[33918]: audit 2023-08-27T22:38:11.678599+0000 mon.c (mon.2) 488 : audit [DBG] from='client.? 172.21.15.6:0/1698885625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:38:11.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:38:11.815Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:12.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:11 smithi130 bash[27579]: audit 2023-08-27T22:38:11.678599+0000 mon.c (mon.2) 488 : audit [DBG] from='client.? 172.21.15.6:0/1698885625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:13.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:12 smithi006 bash[33918]: cluster 2023-08-27T22:38:11.834931+0000 mgr.y (mgr.14901) 1190 : cluster [DBG] pgmap v1197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:13.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:12 smithi006 bash[27292]: cluster 2023-08-27T22:38:11.834931+0000 mgr.y (mgr.14901) 1190 : cluster [DBG] pgmap v1197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:13.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:12 smithi130 bash[27579]: cluster 2023-08-27T22:38:11.834931+0000 mgr.y (mgr.14901) 1190 : cluster [DBG] pgmap v1197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:38:13.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:38:13.710Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:14 smithi130 bash[27579]: cluster 2023-08-27T22:38:13.835389+0000 mgr.y (mgr.14901) 1191 : cluster [DBG] pgmap v1198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:14 smithi130 bash[27579]: audit 2023-08-27T22:38:14.111577+0000 mon.c (mon.2) 489 : audit [DBG] from='client.? 172.21.15.6:0/2865194995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:15.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:14 smithi006 bash[27292]: cluster 2023-08-27T22:38:13.835389+0000 mgr.y (mgr.14901) 1191 : cluster [DBG] pgmap v1198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:15.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:14 smithi006 bash[27292]: audit 2023-08-27T22:38:14.111577+0000 mon.c (mon.2) 489 : audit [DBG] from='client.? 172.21.15.6:0/2865194995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:15.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:14 smithi006 bash[33918]: cluster 2023-08-27T22:38:13.835389+0000 mgr.y (mgr.14901) 1191 : cluster [DBG] pgmap v1198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:15.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:14 smithi006 bash[33918]: audit 2023-08-27T22:38:14.111577+0000 mon.c (mon.2) 489 : audit [DBG] from='client.? 172.21.15.6:0/2865194995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:17.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:16 smithi006 bash[33918]: cluster 2023-08-27T22:38:15.836580+0000 mgr.y (mgr.14901) 1192 : cluster [DBG] pgmap v1199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:17.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:16 smithi006 bash[33918]: audit 2023-08-27T22:38:16.523091+0000 mon.a (mon.0) 1448 : audit [DBG] from='client.? 172.21.15.6:0/3995382136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:17.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:16 smithi006 bash[27292]: cluster 2023-08-27T22:38:15.836580+0000 mgr.y (mgr.14901) 1192 : cluster [DBG] pgmap v1199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:17.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:16 smithi006 bash[27292]: audit 2023-08-27T22:38:16.523091+0000 mon.a (mon.0) 1448 : audit [DBG] from='client.? 172.21.15.6:0/3995382136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:17.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:16 smithi130 bash[27579]: cluster 2023-08-27T22:38:15.836580+0000 mgr.y (mgr.14901) 1192 : cluster [DBG] pgmap v1199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:17.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:16 smithi130 bash[27579]: audit 2023-08-27T22:38:16.523091+0000 mon.a (mon.0) 1448 : audit [DBG] from='client.? 172.21.15.6:0/3995382136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:19.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:18 smithi006 bash[33918]: cluster 2023-08-27T22:38:17.837301+0000 mgr.y (mgr.14901) 1193 : cluster [DBG] pgmap v1200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:19.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:18 smithi006 bash[27292]: cluster 2023-08-27T22:38:17.837301+0000 mgr.y (mgr.14901) 1193 : cluster [DBG] pgmap v1200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:19.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:18 smithi130 bash[27579]: cluster 2023-08-27T22:38:17.837301+0000 mgr.y (mgr.14901) 1193 : cluster [DBG] pgmap v1200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:20.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:19 smithi006 bash[27292]: audit 2023-08-27T22:38:18.942076+0000 mon.c (mon.2) 490 : audit [DBG] from='client.? 172.21.15.6:0/4157931561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:20.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:19 smithi006 bash[33918]: audit 2023-08-27T22:38:18.942076+0000 mon.c (mon.2) 490 : audit [DBG] from='client.? 172.21.15.6:0/4157931561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:20.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:19 smithi130 bash[27579]: audit 2023-08-27T22:38:18.942076+0000 mon.c (mon.2) 490 : audit [DBG] from='client.? 172.21.15.6:0/4157931561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:38:20.247Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:38:20.249Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:21.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:20 smithi006 bash[27292]: cluster 2023-08-27T22:38:19.838402+0000 mgr.y (mgr.14901) 1194 : cluster [DBG] pgmap v1201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:21.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:38:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:38:20] "GET /metrics HTTP/1.1" 200 193084 "" "Prometheus/2.33.4" 2023-08-27T22:38:21.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:20 smithi006 bash[33918]: cluster 2023-08-27T22:38:19.838402+0000 mgr.y (mgr.14901) 1194 : cluster [DBG] pgmap v1201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:21.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:20 smithi130 bash[27579]: cluster 2023-08-27T22:38:19.838402+0000 mgr.y (mgr.14901) 1194 : cluster [DBG] pgmap v1201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:22.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:21 smithi006 bash[33918]: audit 2023-08-27T22:38:21.357783+0000 mon.c (mon.2) 491 : audit [DBG] from='client.? 172.21.15.6:0/263271045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:22.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:21 smithi006 bash[27292]: audit 2023-08-27T22:38:21.357783+0000 mon.c (mon.2) 491 : audit [DBG] from='client.? 172.21.15.6:0/263271045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:38:21.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:38:21.815Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:22.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:21 smithi130 bash[27579]: audit 2023-08-27T22:38:21.357783+0000 mon.c (mon.2) 491 : audit [DBG] from='client.? 172.21.15.6:0/263271045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:23.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:22 smithi006 bash[33918]: cluster 2023-08-27T22:38:21.839053+0000 mgr.y (mgr.14901) 1195 : cluster [DBG] pgmap v1202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:23.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:22 smithi006 bash[27292]: cluster 2023-08-27T22:38:21.839053+0000 mgr.y (mgr.14901) 1195 : cluster [DBG] pgmap v1202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:23.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:22 smithi130 bash[27579]: cluster 2023-08-27T22:38:21.839053+0000 mgr.y (mgr.14901) 1195 : cluster [DBG] pgmap v1202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:38:23.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:38:23.711Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:24 smithi130 bash[27579]: audit 2023-08-27T22:38:23.775005+0000 mon.c (mon.2) 492 : audit [DBG] from='client.? 172.21.15.6:0/915834479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:24 smithi130 bash[27579]: cluster 2023-08-27T22:38:23.839689+0000 mgr.y (mgr.14901) 1196 : cluster [DBG] pgmap v1203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:25.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:24 smithi006 bash[33918]: audit 2023-08-27T22:38:23.775005+0000 mon.c (mon.2) 492 : audit [DBG] from='client.? 172.21.15.6:0/915834479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:25.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:24 smithi006 bash[33918]: cluster 2023-08-27T22:38:23.839689+0000 mgr.y (mgr.14901) 1196 : cluster [DBG] pgmap v1203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:25.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:24 smithi006 bash[27292]: audit 2023-08-27T22:38:23.775005+0000 mon.c (mon.2) 492 : audit [DBG] from='client.? 172.21.15.6:0/915834479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:25.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:24 smithi006 bash[27292]: cluster 2023-08-27T22:38:23.839689+0000 mgr.y (mgr.14901) 1196 : cluster [DBG] pgmap v1203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:27.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:26 smithi006 bash[33918]: cluster 2023-08-27T22:38:25.840828+0000 mgr.y (mgr.14901) 1197 : cluster [DBG] pgmap v1204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:27.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:26 smithi006 bash[33918]: audit 2023-08-27T22:38:26.186381+0000 mon.a (mon.0) 1449 : audit [DBG] from='client.? 172.21.15.6:0/785164355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:26 smithi006 bash[27292]: cluster 2023-08-27T22:38:25.840828+0000 mgr.y (mgr.14901) 1197 : cluster [DBG] pgmap v1204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:26 smithi006 bash[27292]: audit 2023-08-27T22:38:26.186381+0000 mon.a (mon.0) 1449 : audit [DBG] from='client.? 172.21.15.6:0/785164355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:27.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:26 smithi130 bash[27579]: cluster 2023-08-27T22:38:25.840828+0000 mgr.y (mgr.14901) 1197 : cluster [DBG] pgmap v1204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:27.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:26 smithi130 bash[27579]: audit 2023-08-27T22:38:26.186381+0000 mon.a (mon.0) 1449 : audit [DBG] from='client.? 172.21.15.6:0/785164355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:29.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:28 smithi006 bash[33918]: cluster 2023-08-27T22:38:27.841467+0000 mgr.y (mgr.14901) 1198 : cluster [DBG] pgmap v1205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:29.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:28 smithi006 bash[33918]: audit 2023-08-27T22:38:28.595058+0000 mon.a (mon.0) 1450 : audit [DBG] from='client.? 172.21.15.6:0/3355355445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:29.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:28 smithi006 bash[27292]: cluster 2023-08-27T22:38:27.841467+0000 mgr.y (mgr.14901) 1198 : cluster [DBG] pgmap v1205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:29.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:28 smithi006 bash[27292]: audit 2023-08-27T22:38:28.595058+0000 mon.a (mon.0) 1450 : audit [DBG] from='client.? 172.21.15.6:0/3355355445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:29.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:28 smithi130 bash[27579]: cluster 2023-08-27T22:38:27.841467+0000 mgr.y (mgr.14901) 1198 : cluster [DBG] pgmap v1205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:29.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:28 smithi130 bash[27579]: audit 2023-08-27T22:38:28.595058+0000 mon.a (mon.0) 1450 : audit [DBG] from='client.? 172.21.15.6:0/3355355445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:38:30.248Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:38:30.250Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:31.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:38:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:38:30] "GET /metrics HTTP/1.1" 200 193078 "" "Prometheus/2.33.4" 2023-08-27T22:38:31.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:30 smithi006 bash[33918]: cluster 2023-08-27T22:38:29.842557+0000 mgr.y (mgr.14901) 1199 : cluster [DBG] pgmap v1206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:31.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:30 smithi006 bash[27292]: cluster 2023-08-27T22:38:29.842557+0000 mgr.y (mgr.14901) 1199 : cluster [DBG] pgmap v1206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:31.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:30 smithi130 bash[27579]: cluster 2023-08-27T22:38:29.842557+0000 mgr.y (mgr.14901) 1199 : cluster [DBG] pgmap v1206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:32.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:31 smithi006 bash[33918]: audit 2023-08-27T22:38:31.011589+0000 mon.a (mon.0) 1451 : audit [DBG] from='client.? 172.21.15.6:0/3980183866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:32.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:31 smithi006 bash[27292]: audit 2023-08-27T22:38:31.011589+0000 mon.a (mon.0) 1451 : audit [DBG] from='client.? 172.21.15.6:0/3980183866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:38:31.814Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:38:31.816Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:32.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:31 smithi130 bash[27579]: audit 2023-08-27T22:38:31.011589+0000 mon.a (mon.0) 1451 : audit [DBG] from='client.? 172.21.15.6:0/3980183866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:33.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:32 smithi006 bash[33918]: cluster 2023-08-27T22:38:31.843233+0000 mgr.y (mgr.14901) 1200 : cluster [DBG] pgmap v1207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:33.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:32 smithi006 bash[27292]: cluster 2023-08-27T22:38:31.843233+0000 mgr.y (mgr.14901) 1200 : cluster [DBG] pgmap v1207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:33.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:32 smithi130 bash[27579]: cluster 2023-08-27T22:38:31.843233+0000 mgr.y (mgr.14901) 1200 : cluster [DBG] pgmap v1207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:33.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:33 smithi006 bash[27292]: audit 2023-08-27T22:38:33.431415+0000 mon.c (mon.2) 493 : audit [DBG] from='client.? 172.21.15.6:0/921801327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:33.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:33 smithi006 bash[33918]: audit 2023-08-27T22:38:33.431415+0000 mon.c (mon.2) 493 : audit [DBG] from='client.? 172.21.15.6:0/921801327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:38:33.709Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:33.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:38:33.711Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:34.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:33 smithi130 bash[27579]: audit 2023-08-27T22:38:33.431415+0000 mon.c (mon.2) 493 : audit [DBG] from='client.? 172.21.15.6:0/921801327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:35.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:34 smithi006 bash[33918]: cluster 2023-08-27T22:38:33.843901+0000 mgr.y (mgr.14901) 1201 : cluster [DBG] pgmap v1208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:35.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:34 smithi006 bash[27292]: cluster 2023-08-27T22:38:33.843901+0000 mgr.y (mgr.14901) 1201 : cluster [DBG] pgmap v1208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:35.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:34 smithi130 bash[27579]: cluster 2023-08-27T22:38:33.843901+0000 mgr.y (mgr.14901) 1201 : cluster [DBG] pgmap v1208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:36.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:35 smithi006 bash[33918]: audit 2023-08-27T22:38:35.849194+0000 mon.c (mon.2) 494 : audit [DBG] from='client.? 172.21.15.6:0/3412342314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:36.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:35 smithi006 bash[27292]: audit 2023-08-27T22:38:35.849194+0000 mon.c (mon.2) 494 : audit [DBG] from='client.? 172.21.15.6:0/3412342314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:36.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:35 smithi130 bash[27579]: audit 2023-08-27T22:38:35.849194+0000 mon.c (mon.2) 494 : audit [DBG] from='client.? 172.21.15.6:0/3412342314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:37.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:36 smithi006 bash[27292]: cluster 2023-08-27T22:38:35.844974+0000 mgr.y (mgr.14901) 1202 : cluster [DBG] pgmap v1209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:37.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:36 smithi006 bash[33918]: cluster 2023-08-27T22:38:35.844974+0000 mgr.y (mgr.14901) 1202 : cluster [DBG] pgmap v1209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:37.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:36 smithi130 bash[27579]: cluster 2023-08-27T22:38:35.844974+0000 mgr.y (mgr.14901) 1202 : cluster [DBG] pgmap v1209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:39.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:38 smithi006 bash[33918]: cluster 2023-08-27T22:38:37.845562+0000 mgr.y (mgr.14901) 1203 : cluster [DBG] pgmap v1210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:39.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:38 smithi006 bash[33918]: audit 2023-08-27T22:38:38.268979+0000 mon.a (mon.0) 1452 : audit [DBG] from='client.? 172.21.15.6:0/3927277462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:39.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:38 smithi006 bash[27292]: cluster 2023-08-27T22:38:37.845562+0000 mgr.y (mgr.14901) 1203 : cluster [DBG] pgmap v1210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:39.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:38 smithi006 bash[27292]: audit 2023-08-27T22:38:38.268979+0000 mon.a (mon.0) 1452 : audit [DBG] from='client.? 172.21.15.6:0/3927277462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:39.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:38 smithi130 bash[27579]: cluster 2023-08-27T22:38:37.845562+0000 mgr.y (mgr.14901) 1203 : cluster [DBG] pgmap v1210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:39.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:38 smithi130 bash[27579]: audit 2023-08-27T22:38:38.268979+0000 mon.a (mon.0) 1452 : audit [DBG] from='client.? 172.21.15.6:0/3927277462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:40.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:39 smithi006 bash[27292]: audit 2023-08-27T22:38:39.603252+0000 mon.b (mon.1) 238 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:38:40.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:39 smithi006 bash[33918]: audit 2023-08-27T22:38:39.603252+0000 mon.b (mon.1) 238 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:38:40.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:39 smithi130 bash[27579]: audit 2023-08-27T22:38:39.603252+0000 mon.b (mon.1) 238 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:38:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:38:40.249Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:38:40.251Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:41.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:38:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:38:40] "GET /metrics HTTP/1.1" 200 193078 "" "Prometheus/2.33.4" 2023-08-27T22:38:41.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:40 smithi006 bash[33918]: cluster 2023-08-27T22:38:39.846692+0000 mgr.y (mgr.14901) 1204 : cluster [DBG] pgmap v1211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:41.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:40 smithi006 bash[33918]: audit 2023-08-27T22:38:40.698415+0000 mon.a (mon.0) 1453 : audit [DBG] from='client.? 172.21.15.6:0/4204361137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:41.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:40 smithi006 bash[27292]: cluster 2023-08-27T22:38:39.846692+0000 mgr.y (mgr.14901) 1204 : cluster [DBG] pgmap v1211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:41.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:40 smithi006 bash[27292]: audit 2023-08-27T22:38:40.698415+0000 mon.a (mon.0) 1453 : audit [DBG] from='client.? 172.21.15.6:0/4204361137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:41.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:40 smithi130 bash[27579]: cluster 2023-08-27T22:38:39.846692+0000 mgr.y (mgr.14901) 1204 : cluster [DBG] pgmap v1211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:41.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:40 smithi130 bash[27579]: audit 2023-08-27T22:38:40.698415+0000 mon.a (mon.0) 1453 : audit [DBG] from='client.? 172.21.15.6:0/4204361137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:38:41.815Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:38:41.817Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:43.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:42 smithi006 bash[33918]: cluster 2023-08-27T22:38:41.847370+0000 mgr.y (mgr.14901) 1205 : cluster [DBG] pgmap v1212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:43.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:42 smithi006 bash[27292]: cluster 2023-08-27T22:38:41.847370+0000 mgr.y (mgr.14901) 1205 : cluster [DBG] pgmap v1212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:43.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:42 smithi130 bash[27579]: cluster 2023-08-27T22:38:41.847370+0000 mgr.y (mgr.14901) 1205 : cluster [DBG] pgmap v1212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:43.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:38:43.709Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:43.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:38:43.711Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:44.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:43 smithi006 bash[27292]: audit 2023-08-27T22:38:43.120383+0000 mon.c (mon.2) 495 : audit [DBG] from='client.? 172.21.15.6:0/597787617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:44.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:43 smithi006 bash[33918]: audit 2023-08-27T22:38:43.120383+0000 mon.c (mon.2) 495 : audit [DBG] from='client.? 172.21.15.6:0/597787617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:44.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:43 smithi130 bash[27579]: audit 2023-08-27T22:38:43.120383+0000 mon.c (mon.2) 495 : audit [DBG] from='client.? 172.21.15.6:0/597787617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:45.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:44 smithi006 bash[27292]: cluster 2023-08-27T22:38:43.848135+0000 mgr.y (mgr.14901) 1206 : cluster [DBG] pgmap v1213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:45.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:44 smithi006 bash[33918]: cluster 2023-08-27T22:38:43.848135+0000 mgr.y (mgr.14901) 1206 : cluster [DBG] pgmap v1213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:45.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:44 smithi130 bash[27579]: cluster 2023-08-27T22:38:43.848135+0000 mgr.y (mgr.14901) 1206 : cluster [DBG] pgmap v1213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:46 smithi130 bash[27579]: audit 2023-08-27T22:38:45.478176+0000 mon.a (mon.0) 1454 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:38:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:46 smithi130 bash[27579]: audit 2023-08-27T22:38:45.599240+0000 mon.b (mon.1) 239 : audit [DBG] from='client.? 172.21.15.6:0/1145171036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:46 smithi130 bash[27579]: cluster 2023-08-27T22:38:45.849312+0000 mgr.y (mgr.14901) 1207 : cluster [DBG] pgmap v1214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:46 smithi130 bash[27579]: audit 2023-08-27T22:38:45.952990+0000 mon.a (mon.0) 1455 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:38:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:46 smithi130 bash[27579]: audit 2023-08-27T22:38:46.223498+0000 mon.b (mon.1) 240 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:38:46.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:46 smithi130 bash[27579]: audit 2023-08-27T22:38:46.225779+0000 mon.b (mon.1) 241 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:38:46.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:46 smithi130 bash[27579]: audit 2023-08-27T22:38:46.229246+0000 mon.a (mon.0) 1456 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:38:46.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:46 smithi006 bash[27292]: audit 2023-08-27T22:38:45.478176+0000 mon.a (mon.0) 1454 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:38:46.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:46 smithi006 bash[27292]: audit 2023-08-27T22:38:45.599240+0000 mon.b (mon.1) 239 : audit [DBG] from='client.? 172.21.15.6:0/1145171036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:46.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:46 smithi006 bash[27292]: cluster 2023-08-27T22:38:45.849312+0000 mgr.y (mgr.14901) 1207 : cluster [DBG] pgmap v1214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:46.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:46 smithi006 bash[27292]: audit 2023-08-27T22:38:45.952990+0000 mon.a (mon.0) 1455 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:38:46.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:46 smithi006 bash[27292]: audit 2023-08-27T22:38:46.223498+0000 mon.b (mon.1) 240 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:38:46.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:46 smithi006 bash[27292]: audit 2023-08-27T22:38:46.225779+0000 mon.b (mon.1) 241 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:38:46.972 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:46 smithi006 bash[27292]: audit 2023-08-27T22:38:46.229246+0000 mon.a (mon.0) 1456 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:38:46.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:46 smithi006 bash[33918]: audit 2023-08-27T22:38:45.478176+0000 mon.a (mon.0) 1454 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:38:46.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:46 smithi006 bash[33918]: audit 2023-08-27T22:38:45.599240+0000 mon.b (mon.1) 239 : audit [DBG] from='client.? 172.21.15.6:0/1145171036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:46.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:46 smithi006 bash[33918]: cluster 2023-08-27T22:38:45.849312+0000 mgr.y (mgr.14901) 1207 : cluster [DBG] pgmap v1214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:46.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:46 smithi006 bash[33918]: audit 2023-08-27T22:38:45.952990+0000 mon.a (mon.0) 1455 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:38:46.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:46 smithi006 bash[33918]: audit 2023-08-27T22:38:46.223498+0000 mon.b (mon.1) 240 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:38:46.973 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:46 smithi006 bash[33918]: audit 2023-08-27T22:38:46.225779+0000 mon.b (mon.1) 241 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:38:46.974 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:46 smithi006 bash[33918]: audit 2023-08-27T22:38:46.229246+0000 mon.a (mon.0) 1456 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:38:49.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:48 smithi006 bash[33918]: cluster 2023-08-27T22:38:47.849915+0000 mgr.y (mgr.14901) 1208 : cluster [DBG] pgmap v1215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:49.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:48 smithi006 bash[33918]: audit 2023-08-27T22:38:48.003543+0000 mon.c (mon.2) 496 : audit [DBG] from='client.? 172.21.15.6:0/396025942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:49.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:48 smithi006 bash[27292]: cluster 2023-08-27T22:38:47.849915+0000 mgr.y (mgr.14901) 1208 : cluster [DBG] pgmap v1215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:49.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:48 smithi006 bash[27292]: audit 2023-08-27T22:38:48.003543+0000 mon.c (mon.2) 496 : audit [DBG] from='client.? 172.21.15.6:0/396025942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:49.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:48 smithi130 bash[27579]: cluster 2023-08-27T22:38:47.849915+0000 mgr.y (mgr.14901) 1208 : cluster [DBG] pgmap v1215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:49.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:48 smithi130 bash[27579]: audit 2023-08-27T22:38:48.003543+0000 mon.c (mon.2) 496 : audit [DBG] from='client.? 172.21.15.6:0/396025942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:38:50.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:38:50.252Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:51.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:38:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:38:50] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:38:51.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:50 smithi006 bash[27292]: cluster 2023-08-27T22:38:49.851042+0000 mgr.y (mgr.14901) 1209 : cluster [DBG] pgmap v1216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:51.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:50 smithi006 bash[27292]: audit 2023-08-27T22:38:50.426788+0000 mon.a (mon.0) 1457 : audit [DBG] from='client.? 172.21.15.6:0/38513934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:51.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:50 smithi006 bash[33918]: cluster 2023-08-27T22:38:49.851042+0000 mgr.y (mgr.14901) 1209 : cluster [DBG] pgmap v1216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:51.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:50 smithi006 bash[33918]: audit 2023-08-27T22:38:50.426788+0000 mon.a (mon.0) 1457 : audit [DBG] from='client.? 172.21.15.6:0/38513934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:51.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:50 smithi130 bash[27579]: cluster 2023-08-27T22:38:49.851042+0000 mgr.y (mgr.14901) 1209 : cluster [DBG] pgmap v1216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:51.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:50 smithi130 bash[27579]: audit 2023-08-27T22:38:50.426788+0000 mon.a (mon.0) 1457 : audit [DBG] from='client.? 172.21.15.6:0/38513934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:38:51.815Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:38:51.817Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:53.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:52 smithi006 bash[27292]: cluster 2023-08-27T22:38:51.851683+0000 mgr.y (mgr.14901) 1210 : cluster [DBG] pgmap v1217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:53.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:52 smithi006 bash[27292]: audit 2023-08-27T22:38:52.842233+0000 mon.c (mon.2) 497 : audit [DBG] from='client.? 172.21.15.6:0/4153275295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:53.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:52 smithi006 bash[33918]: cluster 2023-08-27T22:38:51.851683+0000 mgr.y (mgr.14901) 1210 : cluster [DBG] pgmap v1217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:53.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:52 smithi006 bash[33918]: audit 2023-08-27T22:38:52.842233+0000 mon.c (mon.2) 497 : audit [DBG] from='client.? 172.21.15.6:0/4153275295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:53.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:52 smithi130 bash[27579]: cluster 2023-08-27T22:38:51.851683+0000 mgr.y (mgr.14901) 1210 : cluster [DBG] pgmap v1217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:53.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:52 smithi130 bash[27579]: audit 2023-08-27T22:38:52.842233+0000 mon.c (mon.2) 497 : audit [DBG] from='client.? 172.21.15.6:0/4153275295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:38:53.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:53.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:38:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:38:53.712Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:38:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:54 smithi130 bash[27579]: cluster 2023-08-27T22:38:53.852344+0000 mgr.y (mgr.14901) 1211 : cluster [DBG] pgmap v1218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:55.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:54 smithi006 bash[33918]: cluster 2023-08-27T22:38:53.852344+0000 mgr.y (mgr.14901) 1211 : cluster [DBG] pgmap v1218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:55.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:54 smithi006 bash[27292]: cluster 2023-08-27T22:38:53.852344+0000 mgr.y (mgr.14901) 1211 : cluster [DBG] pgmap v1218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:56.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:55 smithi130 bash[27579]: audit 2023-08-27T22:38:55.243583+0000 mon.c (mon.2) 498 : audit [DBG] from='client.? 172.21.15.6:0/1027899312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:56.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:55 smithi006 bash[33918]: audit 2023-08-27T22:38:55.243583+0000 mon.c (mon.2) 498 : audit [DBG] from='client.? 172.21.15.6:0/1027899312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:56.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:55 smithi006 bash[27292]: audit 2023-08-27T22:38:55.243583+0000 mon.c (mon.2) 498 : audit [DBG] from='client.? 172.21.15.6:0/1027899312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:56 smithi130 bash[27579]: cluster 2023-08-27T22:38:55.853518+0000 mgr.y (mgr.14901) 1212 : cluster [DBG] pgmap v1219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:57.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:56 smithi006 bash[33918]: cluster 2023-08-27T22:38:55.853518+0000 mgr.y (mgr.14901) 1212 : cluster [DBG] pgmap v1219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:57.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:56 smithi006 bash[27292]: cluster 2023-08-27T22:38:55.853518+0000 mgr.y (mgr.14901) 1212 : cluster [DBG] pgmap v1219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:58.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:57 smithi130 bash[27579]: audit 2023-08-27T22:38:57.663839+0000 mon.c (mon.2) 499 : audit [DBG] from='client.? 172.21.15.6:0/2708828946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:58.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:57 smithi006 bash[33918]: audit 2023-08-27T22:38:57.663839+0000 mon.c (mon.2) 499 : audit [DBG] from='client.? 172.21.15.6:0/2708828946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:58.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:57 smithi006 bash[27292]: audit 2023-08-27T22:38:57.663839+0000 mon.c (mon.2) 499 : audit [DBG] from='client.? 172.21.15.6:0/2708828946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:38:58 smithi130 bash[27579]: cluster 2023-08-27T22:38:57.854098+0000 mgr.y (mgr.14901) 1213 : cluster [DBG] pgmap v1220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:59.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:38:58 smithi006 bash[33918]: cluster 2023-08-27T22:38:57.854098+0000 mgr.y (mgr.14901) 1213 : cluster [DBG] pgmap v1220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:38:59.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:38:58 smithi006 bash[27292]: cluster 2023-08-27T22:38:57.854098+0000 mgr.y (mgr.14901) 1213 : cluster [DBG] pgmap v1220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:00.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:39:00.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:00.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:39:00.252Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:01.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:39:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:39:00] "GET /metrics HTTP/1.1" 200 193083 "" "Prometheus/2.33.4" 2023-08-27T22:39:01.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:00 smithi006 bash[33918]: cluster 2023-08-27T22:38:59.855146+0000 mgr.y (mgr.14901) 1214 : cluster [DBG] pgmap v1221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:01.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:00 smithi006 bash[33918]: audit 2023-08-27T22:39:00.076421+0000 mon.a (mon.0) 1458 : audit [DBG] from='client.? 172.21.15.6:0/2543510901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:01.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:00 smithi006 bash[27292]: cluster 2023-08-27T22:38:59.855146+0000 mgr.y (mgr.14901) 1214 : cluster [DBG] pgmap v1221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:01.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:00 smithi006 bash[27292]: audit 2023-08-27T22:39:00.076421+0000 mon.a (mon.0) 1458 : audit [DBG] from='client.? 172.21.15.6:0/2543510901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:01.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:00 smithi130 bash[27579]: cluster 2023-08-27T22:38:59.855146+0000 mgr.y (mgr.14901) 1214 : cluster [DBG] pgmap v1221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:01.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:00 smithi130 bash[27579]: audit 2023-08-27T22:39:00.076421+0000 mon.a (mon.0) 1458 : audit [DBG] from='client.? 172.21.15.6:0/2543510901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:39:01.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:39:01.818Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:03.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:02 smithi006 bash[33918]: cluster 2023-08-27T22:39:01.855940+0000 mgr.y (mgr.14901) 1215 : cluster [DBG] pgmap v1222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:03.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:02 smithi006 bash[33918]: audit 2023-08-27T22:39:02.493223+0000 mon.a (mon.0) 1459 : audit [DBG] from='client.? 172.21.15.6:0/321236604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:03.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:02 smithi006 bash[27292]: cluster 2023-08-27T22:39:01.855940+0000 mgr.y (mgr.14901) 1215 : cluster [DBG] pgmap v1222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:03.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:02 smithi006 bash[27292]: audit 2023-08-27T22:39:02.493223+0000 mon.a (mon.0) 1459 : audit [DBG] from='client.? 172.21.15.6:0/321236604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:03.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:02 smithi130 bash[27579]: cluster 2023-08-27T22:39:01.855940+0000 mgr.y (mgr.14901) 1215 : cluster [DBG] pgmap v1222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:03.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:02 smithi130 bash[27579]: audit 2023-08-27T22:39:02.493223+0000 mon.a (mon.0) 1459 : audit [DBG] from='client.? 172.21.15.6:0/321236604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:39:03.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:03.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:39:03.713Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:04 smithi130 bash[27579]: cluster 2023-08-27T22:39:03.856788+0000 mgr.y (mgr.14901) 1216 : cluster [DBG] pgmap v1223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:05.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:04 smithi006 bash[33918]: cluster 2023-08-27T22:39:03.856788+0000 mgr.y (mgr.14901) 1216 : cluster [DBG] pgmap v1223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:05.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:04 smithi006 bash[27292]: cluster 2023-08-27T22:39:03.856788+0000 mgr.y (mgr.14901) 1216 : cluster [DBG] pgmap v1223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:06.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:05 smithi130 bash[27579]: audit 2023-08-27T22:39:04.901715+0000 mon.c (mon.2) 500 : audit [DBG] from='client.? 172.21.15.6:0/2908398773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:06.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:05 smithi006 bash[27292]: audit 2023-08-27T22:39:04.901715+0000 mon.c (mon.2) 500 : audit [DBG] from='client.? 172.21.15.6:0/2908398773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:06.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:05 smithi006 bash[33918]: audit 2023-08-27T22:39:04.901715+0000 mon.c (mon.2) 500 : audit [DBG] from='client.? 172.21.15.6:0/2908398773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:06 smithi130 bash[27579]: cluster 2023-08-27T22:39:05.857927+0000 mgr.y (mgr.14901) 1217 : cluster [DBG] pgmap v1224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:07.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:06 smithi006 bash[27292]: cluster 2023-08-27T22:39:05.857927+0000 mgr.y (mgr.14901) 1217 : cluster [DBG] pgmap v1224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:07.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:06 smithi006 bash[33918]: cluster 2023-08-27T22:39:05.857927+0000 mgr.y (mgr.14901) 1217 : cluster [DBG] pgmap v1224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:08.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:07 smithi130 bash[27579]: audit 2023-08-27T22:39:07.326675+0000 mon.c (mon.2) 501 : audit [DBG] from='client.? 172.21.15.6:0/3037688435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:08.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:07 smithi006 bash[33918]: audit 2023-08-27T22:39:07.326675+0000 mon.c (mon.2) 501 : audit [DBG] from='client.? 172.21.15.6:0/3037688435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:08.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:07 smithi006 bash[27292]: audit 2023-08-27T22:39:07.326675+0000 mon.c (mon.2) 501 : audit [DBG] from='client.? 172.21.15.6:0/3037688435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:08 smithi130 bash[27579]: cluster 2023-08-27T22:39:07.858683+0000 mgr.y (mgr.14901) 1218 : cluster [DBG] pgmap v1225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:09.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:08 smithi006 bash[27292]: cluster 2023-08-27T22:39:07.858683+0000 mgr.y (mgr.14901) 1218 : cluster [DBG] pgmap v1225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:09.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:08 smithi006 bash[33918]: cluster 2023-08-27T22:39:07.858683+0000 mgr.y (mgr.14901) 1218 : cluster [DBG] pgmap v1225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:10.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:09 smithi130 bash[27579]: audit 2023-08-27T22:39:09.745764+0000 mon.c (mon.2) 502 : audit [DBG] from='client.? 172.21.15.6:0/1966290433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:10.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:09 smithi006 bash[33918]: audit 2023-08-27T22:39:09.745764+0000 mon.c (mon.2) 502 : audit [DBG] from='client.? 172.21.15.6:0/1966290433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:10.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:09 smithi006 bash[27292]: audit 2023-08-27T22:39:09.745764+0000 mon.c (mon.2) 502 : audit [DBG] from='client.? 172.21.15.6:0/1966290433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:39:10.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:39:10.252Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:10 smithi130 bash[27579]: cluster 2023-08-27T22:39:09.859869+0000 mgr.y (mgr.14901) 1219 : cluster [DBG] pgmap v1226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:11.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:10 smithi006 bash[27292]: cluster 2023-08-27T22:39:09.859869+0000 mgr.y (mgr.14901) 1219 : cluster [DBG] pgmap v1226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:11.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:39:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:39:10] "GET /metrics HTTP/1.1" 200 193083 "" "Prometheus/2.33.4" 2023-08-27T22:39:11.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:10 smithi006 bash[33918]: cluster 2023-08-27T22:39:09.859869+0000 mgr.y (mgr.14901) 1219 : cluster [DBG] pgmap v1226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:39:11.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:39:11.819Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:13.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:12 smithi006 bash[33918]: cluster 2023-08-27T22:39:11.860388+0000 mgr.y (mgr.14901) 1220 : cluster [DBG] pgmap v1227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:13.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:12 smithi006 bash[33918]: audit 2023-08-27T22:39:12.163143+0000 mon.a (mon.0) 1460 : audit [DBG] from='client.? 172.21.15.6:0/99451697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:13.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:12 smithi006 bash[27292]: cluster 2023-08-27T22:39:11.860388+0000 mgr.y (mgr.14901) 1220 : cluster [DBG] pgmap v1227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:13.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:12 smithi006 bash[27292]: audit 2023-08-27T22:39:12.163143+0000 mon.a (mon.0) 1460 : audit [DBG] from='client.? 172.21.15.6:0/99451697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:13.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:12 smithi130 bash[27579]: cluster 2023-08-27T22:39:11.860388+0000 mgr.y (mgr.14901) 1220 : cluster [DBG] pgmap v1227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:13.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:12 smithi130 bash[27579]: audit 2023-08-27T22:39:12.163143+0000 mon.a (mon.0) 1460 : audit [DBG] from='client.? 172.21.15.6:0/99451697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:39:13.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:13.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:39:13.714Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:14 smithi130 bash[27579]: cluster 2023-08-27T22:39:13.861074+0000 mgr.y (mgr.14901) 1221 : cluster [DBG] pgmap v1228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:14 smithi130 bash[27579]: audit 2023-08-27T22:39:14.588333+0000 mon.c (mon.2) 503 : audit [DBG] from='client.? 172.21.15.6:0/407498250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:15.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:14 smithi006 bash[27292]: cluster 2023-08-27T22:39:13.861074+0000 mgr.y (mgr.14901) 1221 : cluster [DBG] pgmap v1228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:15.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:14 smithi006 bash[27292]: audit 2023-08-27T22:39:14.588333+0000 mon.c (mon.2) 503 : audit [DBG] from='client.? 172.21.15.6:0/407498250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:15.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:14 smithi006 bash[33918]: cluster 2023-08-27T22:39:13.861074+0000 mgr.y (mgr.14901) 1221 : cluster [DBG] pgmap v1228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:15.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:14 smithi006 bash[33918]: audit 2023-08-27T22:39:14.588333+0000 mon.c (mon.2) 503 : audit [DBG] from='client.? 172.21.15.6:0/407498250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:17.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:16 smithi006 bash[33918]: cluster 2023-08-27T22:39:15.862176+0000 mgr.y (mgr.14901) 1222 : cluster [DBG] pgmap v1229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:17.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:16 smithi006 bash[27292]: cluster 2023-08-27T22:39:15.862176+0000 mgr.y (mgr.14901) 1222 : cluster [DBG] pgmap v1229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:17.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:16 smithi130 bash[27579]: cluster 2023-08-27T22:39:15.862176+0000 mgr.y (mgr.14901) 1222 : cluster [DBG] pgmap v1229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:18.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:17 smithi006 bash[27292]: audit 2023-08-27T22:39:16.994980+0000 mon.a (mon.0) 1461 : audit [DBG] from='client.? 172.21.15.6:0/858270836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:18.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:17 smithi006 bash[33918]: audit 2023-08-27T22:39:16.994980+0000 mon.a (mon.0) 1461 : audit [DBG] from='client.? 172.21.15.6:0/858270836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:18.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:17 smithi130 bash[27579]: audit 2023-08-27T22:39:16.994980+0000 mon.a (mon.0) 1461 : audit [DBG] from='client.? 172.21.15.6:0/858270836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:19.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:18 smithi006 bash[27292]: cluster 2023-08-27T22:39:17.862828+0000 mgr.y (mgr.14901) 1223 : cluster [DBG] pgmap v1230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:19.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:18 smithi006 bash[33918]: cluster 2023-08-27T22:39:17.862828+0000 mgr.y (mgr.14901) 1223 : cluster [DBG] pgmap v1230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:19.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:18 smithi130 bash[27579]: cluster 2023-08-27T22:39:17.862828+0000 mgr.y (mgr.14901) 1223 : cluster [DBG] pgmap v1230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:20.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:19 smithi006 bash[33918]: audit 2023-08-27T22:39:19.410212+0000 mon.a (mon.0) 1462 : audit [DBG] from='client.? 172.21.15.6:0/2485521624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:20.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:19 smithi006 bash[27292]: audit 2023-08-27T22:39:19.410212+0000 mon.a (mon.0) 1462 : audit [DBG] from='client.? 172.21.15.6:0/2485521624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:20.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:19 smithi130 bash[27579]: audit 2023-08-27T22:39:19.410212+0000 mon.a (mon.0) 1462 : audit [DBG] from='client.? 172.21.15.6:0/2485521624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:39:20.252Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:39:20.254Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:21.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:39:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:39:20] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T22:39:21.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:20 smithi006 bash[27292]: cluster 2023-08-27T22:39:19.863866+0000 mgr.y (mgr.14901) 1224 : cluster [DBG] pgmap v1231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:21.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:20 smithi006 bash[33918]: cluster 2023-08-27T22:39:19.863866+0000 mgr.y (mgr.14901) 1224 : cluster [DBG] pgmap v1231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:21.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:20 smithi130 bash[27579]: cluster 2023-08-27T22:39:19.863866+0000 mgr.y (mgr.14901) 1224 : cluster [DBG] pgmap v1231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:22.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:21 smithi006 bash[27292]: audit 2023-08-27T22:39:21.834148+0000 mon.a (mon.0) 1463 : audit [DBG] from='client.? 172.21.15.6:0/892608343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:22.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:21 smithi006 bash[33918]: audit 2023-08-27T22:39:21.834148+0000 mon.a (mon.0) 1463 : audit [DBG] from='client.? 172.21.15.6:0/892608343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:39:21.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:39:21.819Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:22.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:21 smithi130 bash[27579]: audit 2023-08-27T22:39:21.834148+0000 mon.a (mon.0) 1463 : audit [DBG] from='client.? 172.21.15.6:0/892608343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:23.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:22 smithi006 bash[33918]: cluster 2023-08-27T22:39:21.864485+0000 mgr.y (mgr.14901) 1225 : cluster [DBG] pgmap v1232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:23.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:22 smithi006 bash[27292]: cluster 2023-08-27T22:39:21.864485+0000 mgr.y (mgr.14901) 1225 : cluster [DBG] pgmap v1232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:23.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:22 smithi130 bash[27579]: cluster 2023-08-27T22:39:21.864485+0000 mgr.y (mgr.14901) 1225 : cluster [DBG] pgmap v1232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:39:23.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:23.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:39:23.714Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:24 smithi130 bash[27579]: cluster 2023-08-27T22:39:23.865272+0000 mgr.y (mgr.14901) 1226 : cluster [DBG] pgmap v1233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:24 smithi130 bash[27579]: audit 2023-08-27T22:39:24.247627+0000 mon.a (mon.0) 1464 : audit [DBG] from='client.? 172.21.15.6:0/1726760583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:25.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:24 smithi006 bash[27292]: cluster 2023-08-27T22:39:23.865272+0000 mgr.y (mgr.14901) 1226 : cluster [DBG] pgmap v1233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:25.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:24 smithi006 bash[27292]: audit 2023-08-27T22:39:24.247627+0000 mon.a (mon.0) 1464 : audit [DBG] from='client.? 172.21.15.6:0/1726760583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:25.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:24 smithi006 bash[33918]: cluster 2023-08-27T22:39:23.865272+0000 mgr.y (mgr.14901) 1226 : cluster [DBG] pgmap v1233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:25.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:24 smithi006 bash[33918]: audit 2023-08-27T22:39:24.247627+0000 mon.a (mon.0) 1464 : audit [DBG] from='client.? 172.21.15.6:0/1726760583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:27.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:26 smithi006 bash[33918]: cluster 2023-08-27T22:39:25.866422+0000 mgr.y (mgr.14901) 1227 : cluster [DBG] pgmap v1234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:27.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:26 smithi006 bash[33918]: audit 2023-08-27T22:39:26.654439+0000 mon.c (mon.2) 504 : audit [DBG] from='client.? 172.21.15.6:0/639088706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:26 smithi006 bash[27292]: cluster 2023-08-27T22:39:25.866422+0000 mgr.y (mgr.14901) 1227 : cluster [DBG] pgmap v1234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:26 smithi006 bash[27292]: audit 2023-08-27T22:39:26.654439+0000 mon.c (mon.2) 504 : audit [DBG] from='client.? 172.21.15.6:0/639088706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:27.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:26 smithi130 bash[27579]: cluster 2023-08-27T22:39:25.866422+0000 mgr.y (mgr.14901) 1227 : cluster [DBG] pgmap v1234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:27.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:26 smithi130 bash[27579]: audit 2023-08-27T22:39:26.654439+0000 mon.c (mon.2) 504 : audit [DBG] from='client.? 172.21.15.6:0/639088706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:29.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:28 smithi006 bash[33918]: cluster 2023-08-27T22:39:27.867068+0000 mgr.y (mgr.14901) 1228 : cluster [DBG] pgmap v1235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:29.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:28 smithi006 bash[27292]: cluster 2023-08-27T22:39:27.867068+0000 mgr.y (mgr.14901) 1228 : cluster [DBG] pgmap v1235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:29.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:28 smithi130 bash[27579]: cluster 2023-08-27T22:39:27.867068+0000 mgr.y (mgr.14901) 1228 : cluster [DBG] pgmap v1235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:30.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:29 smithi006 bash[27292]: audit 2023-08-27T22:39:29.072068+0000 mon.c (mon.2) 505 : audit [DBG] from='client.? 172.21.15.6:0/1460804483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:30.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:29 smithi006 bash[33918]: audit 2023-08-27T22:39:29.072068+0000 mon.c (mon.2) 505 : audit [DBG] from='client.? 172.21.15.6:0/1460804483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:30.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:29 smithi130 bash[27579]: audit 2023-08-27T22:39:29.072068+0000 mon.c (mon.2) 505 : audit [DBG] from='client.? 172.21.15.6:0/1460804483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:39:30.252Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:39:30.254Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:31.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:30 smithi006 bash[27292]: cluster 2023-08-27T22:39:29.868180+0000 mgr.y (mgr.14901) 1229 : cluster [DBG] pgmap v1236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:31.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:39:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:39:30] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:39:31.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:30 smithi006 bash[33918]: cluster 2023-08-27T22:39:29.868180+0000 mgr.y (mgr.14901) 1229 : cluster [DBG] pgmap v1236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:31.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:30 smithi130 bash[27579]: cluster 2023-08-27T22:39:29.868180+0000 mgr.y (mgr.14901) 1229 : cluster [DBG] pgmap v1236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:32.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:31 smithi006 bash[33918]: audit 2023-08-27T22:39:31.489115+0000 mon.c (mon.2) 506 : audit [DBG] from='client.? 172.21.15.6:0/1765977674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:39:31.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:39:31.820Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:32.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:31 smithi006 bash[27292]: audit 2023-08-27T22:39:31.489115+0000 mon.c (mon.2) 506 : audit [DBG] from='client.? 172.21.15.6:0/1765977674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:32.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:31 smithi130 bash[27579]: audit 2023-08-27T22:39:31.489115+0000 mon.c (mon.2) 506 : audit [DBG] from='client.? 172.21.15.6:0/1765977674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:33.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:32 smithi006 bash[27292]: cluster 2023-08-27T22:39:31.868807+0000 mgr.y (mgr.14901) 1230 : cluster [DBG] pgmap v1237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:33.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:32 smithi006 bash[33918]: cluster 2023-08-27T22:39:31.868807+0000 mgr.y (mgr.14901) 1230 : cluster [DBG] pgmap v1237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:33.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:32 smithi130 bash[27579]: cluster 2023-08-27T22:39:31.868807+0000 mgr.y (mgr.14901) 1230 : cluster [DBG] pgmap v1237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:39:33.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:33.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:39:33.716Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:34 smithi130 bash[27579]: cluster 2023-08-27T22:39:33.869417+0000 mgr.y (mgr.14901) 1231 : cluster [DBG] pgmap v1238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:34 smithi130 bash[27579]: audit 2023-08-27T22:39:33.902412+0000 mon.a (mon.0) 1465 : audit [DBG] from='client.? 172.21.15.6:0/2946788847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:35.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:34 smithi006 bash[27292]: cluster 2023-08-27T22:39:33.869417+0000 mgr.y (mgr.14901) 1231 : cluster [DBG] pgmap v1238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:35.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:34 smithi006 bash[27292]: audit 2023-08-27T22:39:33.902412+0000 mon.a (mon.0) 1465 : audit [DBG] from='client.? 172.21.15.6:0/2946788847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:35.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:34 smithi006 bash[33918]: cluster 2023-08-27T22:39:33.869417+0000 mgr.y (mgr.14901) 1231 : cluster [DBG] pgmap v1238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:35.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:34 smithi006 bash[33918]: audit 2023-08-27T22:39:33.902412+0000 mon.a (mon.0) 1465 : audit [DBG] from='client.? 172.21.15.6:0/2946788847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:37.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:36 smithi006 bash[27292]: cluster 2023-08-27T22:39:35.870404+0000 mgr.y (mgr.14901) 1232 : cluster [DBG] pgmap v1239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:37.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:36 smithi006 bash[27292]: audit 2023-08-27T22:39:36.316701+0000 mon.c (mon.2) 507 : audit [DBG] from='client.? 172.21.15.6:0/1121008202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:37.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:36 smithi006 bash[33918]: cluster 2023-08-27T22:39:35.870404+0000 mgr.y (mgr.14901) 1232 : cluster [DBG] pgmap v1239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:37.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:36 smithi006 bash[33918]: audit 2023-08-27T22:39:36.316701+0000 mon.c (mon.2) 507 : audit [DBG] from='client.? 172.21.15.6:0/1121008202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:37.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:36 smithi130 bash[27579]: cluster 2023-08-27T22:39:35.870404+0000 mgr.y (mgr.14901) 1232 : cluster [DBG] pgmap v1239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:37.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:36 smithi130 bash[27579]: audit 2023-08-27T22:39:36.316701+0000 mon.c (mon.2) 507 : audit [DBG] from='client.? 172.21.15.6:0/1121008202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:39.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:38 smithi006 bash[33918]: cluster 2023-08-27T22:39:37.871045+0000 mgr.y (mgr.14901) 1233 : cluster [DBG] pgmap v1240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:39.239 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:38 smithi006 bash[33918]: audit 2023-08-27T22:39:38.731445+0000 mon.c (mon.2) 508 : audit [DBG] from='client.? 172.21.15.6:0/662016459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:39.240 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:38 smithi006 bash[27292]: cluster 2023-08-27T22:39:37.871045+0000 mgr.y (mgr.14901) 1233 : cluster [DBG] pgmap v1240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:39.240 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:38 smithi006 bash[27292]: audit 2023-08-27T22:39:38.731445+0000 mon.c (mon.2) 508 : audit [DBG] from='client.? 172.21.15.6:0/662016459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:39.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:38 smithi130 bash[27579]: cluster 2023-08-27T22:39:37.871045+0000 mgr.y (mgr.14901) 1233 : cluster [DBG] pgmap v1240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:38 smithi130 bash[27579]: audit 2023-08-27T22:39:38.731445+0000 mon.c (mon.2) 508 : audit [DBG] from='client.? 172.21.15.6:0/662016459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:39:40.253Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:39:40.255Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:41.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:40 smithi006 bash[27292]: cluster 2023-08-27T22:39:39.872130+0000 mgr.y (mgr.14901) 1234 : cluster [DBG] pgmap v1241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:41.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:39:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:39:40] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:39:41.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:40 smithi006 bash[33918]: cluster 2023-08-27T22:39:39.872130+0000 mgr.y (mgr.14901) 1234 : cluster [DBG] pgmap v1241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:41.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:40 smithi130 bash[27579]: cluster 2023-08-27T22:39:39.872130+0000 mgr.y (mgr.14901) 1234 : cluster [DBG] pgmap v1241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:42.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:41 smithi006 bash[33918]: audit 2023-08-27T22:39:41.142412+0000 mon.a (mon.0) 1466 : audit [DBG] from='client.? 172.21.15.6:0/1615583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:42.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:41 smithi006 bash[27292]: audit 2023-08-27T22:39:41.142412+0000 mon.a (mon.0) 1466 : audit [DBG] from='client.? 172.21.15.6:0/1615583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:39:41.819Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:39:41.821Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:42.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:41 smithi130 bash[27579]: audit 2023-08-27T22:39:41.142412+0000 mon.a (mon.0) 1466 : audit [DBG] from='client.? 172.21.15.6:0/1615583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:43.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:42 smithi006 bash[33918]: cluster 2023-08-27T22:39:41.872741+0000 mgr.y (mgr.14901) 1235 : cluster [DBG] pgmap v1242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:43.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:42 smithi006 bash[27292]: cluster 2023-08-27T22:39:41.872741+0000 mgr.y (mgr.14901) 1235 : cluster [DBG] pgmap v1242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:43.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:42 smithi130 bash[27579]: cluster 2023-08-27T22:39:41.872741+0000 mgr.y (mgr.14901) 1235 : cluster [DBG] pgmap v1242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:44.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:43 smithi006 bash[27292]: audit 2023-08-27T22:39:43.551119+0000 mon.a (mon.0) 1467 : audit [DBG] from='client.? 172.21.15.6:0/1473905875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:44.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:43 smithi006 bash[33918]: audit 2023-08-27T22:39:43.551119+0000 mon.a (mon.0) 1467 : audit [DBG] from='client.? 172.21.15.6:0/1473905875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:44.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:39:43.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:44.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:39:43.717Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:44.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:43 smithi130 bash[27579]: audit 2023-08-27T22:39:43.551119+0000 mon.a (mon.0) 1467 : audit [DBG] from='client.? 172.21.15.6:0/1473905875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:45.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:44 smithi006 bash[33918]: cluster 2023-08-27T22:39:43.873426+0000 mgr.y (mgr.14901) 1236 : cluster [DBG] pgmap v1243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:45.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:44 smithi006 bash[27292]: cluster 2023-08-27T22:39:43.873426+0000 mgr.y (mgr.14901) 1236 : cluster [DBG] pgmap v1243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:45.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:44 smithi130 bash[27579]: cluster 2023-08-27T22:39:43.873426+0000 mgr.y (mgr.14901) 1236 : cluster [DBG] pgmap v1243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:47.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:46 smithi130 bash[27579]: cluster 2023-08-27T22:39:45.874557+0000 mgr.y (mgr.14901) 1237 : cluster [DBG] pgmap v1244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:47.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:46 smithi130 bash[27579]: audit 2023-08-27T22:39:45.972301+0000 mon.c (mon.2) 509 : audit [DBG] from='client.? 172.21.15.6:0/716208629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:47.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:46 smithi130 bash[27579]: audit 2023-08-27T22:39:46.245821+0000 mon.b (mon.1) 242 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:39:47.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:46 smithi006 bash[27292]: cluster 2023-08-27T22:39:45.874557+0000 mgr.y (mgr.14901) 1237 : cluster [DBG] pgmap v1244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:47.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:46 smithi006 bash[27292]: audit 2023-08-27T22:39:45.972301+0000 mon.c (mon.2) 509 : audit [DBG] from='client.? 172.21.15.6:0/716208629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:47.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:46 smithi006 bash[27292]: audit 2023-08-27T22:39:46.245821+0000 mon.b (mon.1) 242 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:39:47.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:46 smithi006 bash[33918]: cluster 2023-08-27T22:39:45.874557+0000 mgr.y (mgr.14901) 1237 : cluster [DBG] pgmap v1244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:47.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:46 smithi006 bash[33918]: audit 2023-08-27T22:39:45.972301+0000 mon.c (mon.2) 509 : audit [DBG] from='client.? 172.21.15.6:0/716208629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:47.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:46 smithi006 bash[33918]: audit 2023-08-27T22:39:46.245821+0000 mon.b (mon.1) 242 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:39:49.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:48 smithi130 bash[27579]: cluster 2023-08-27T22:39:47.875206+0000 mgr.y (mgr.14901) 1238 : cluster [DBG] pgmap v1245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:49.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:48 smithi130 bash[27579]: audit 2023-08-27T22:39:48.381746+0000 mon.a (mon.0) 1468 : audit [DBG] from='client.? 172.21.15.6:0/3942708299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:49.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:48 smithi006 bash[27292]: cluster 2023-08-27T22:39:47.875206+0000 mgr.y (mgr.14901) 1238 : cluster [DBG] pgmap v1245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:49.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:48 smithi006 bash[27292]: audit 2023-08-27T22:39:48.381746+0000 mon.a (mon.0) 1468 : audit [DBG] from='client.? 172.21.15.6:0/3942708299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:49.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:48 smithi006 bash[33918]: cluster 2023-08-27T22:39:47.875206+0000 mgr.y (mgr.14901) 1238 : cluster [DBG] pgmap v1245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:49.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:48 smithi006 bash[33918]: audit 2023-08-27T22:39:48.381746+0000 mon.a (mon.0) 1468 : audit [DBG] from='client.? 172.21.15.6:0/3942708299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:39:50.254Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:39:50.256Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:51.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:39:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:39:50] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T22:39:51.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:50 smithi006 bash[27292]: cluster 2023-08-27T22:39:49.876494+0000 mgr.y (mgr.14901) 1239 : cluster [DBG] pgmap v1246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:51.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:50 smithi006 bash[27292]: audit 2023-08-27T22:39:50.815427+0000 mon.a (mon.0) 1469 : audit [DBG] from='client.? 172.21.15.6:0/1127447821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:51.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:50 smithi006 bash[33918]: cluster 2023-08-27T22:39:49.876494+0000 mgr.y (mgr.14901) 1239 : cluster [DBG] pgmap v1246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:51.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:50 smithi006 bash[33918]: audit 2023-08-27T22:39:50.815427+0000 mon.a (mon.0) 1469 : audit [DBG] from='client.? 172.21.15.6:0/1127447821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:51.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:50 smithi130 bash[27579]: cluster 2023-08-27T22:39:49.876494+0000 mgr.y (mgr.14901) 1239 : cluster [DBG] pgmap v1246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:51.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:50 smithi130 bash[27579]: audit 2023-08-27T22:39:50.815427+0000 mon.a (mon.0) 1469 : audit [DBG] from='client.? 172.21.15.6:0/1127447821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:39:51.820Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:52.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:39:51.822Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:53.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:53 smithi006 bash[33918]: cluster 2023-08-27T22:39:51.877288+0000 mgr.y (mgr.14901) 1240 : cluster [DBG] pgmap v1247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:53.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:53 smithi006 bash[33918]: audit 2023-08-27T22:39:52.144432+0000 mon.a (mon.0) 1470 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:39:53.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:53 smithi006 bash[33918]: audit 2023-08-27T22:39:52.525626+0000 mon.a (mon.0) 1471 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:39:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:53 smithi006 bash[33918]: audit 2023-08-27T22:39:52.798773+0000 mon.b (mon.1) 243 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:39:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:53 smithi006 bash[33918]: audit 2023-08-27T22:39:52.800895+0000 mon.b (mon.1) 244 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:39:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:53 smithi006 bash[33918]: audit 2023-08-27T22:39:52.804172+0000 mon.a (mon.0) 1472 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:39:53.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:53 smithi006 bash[27292]: cluster 2023-08-27T22:39:51.877288+0000 mgr.y (mgr.14901) 1240 : cluster [DBG] pgmap v1247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:53.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:53 smithi006 bash[27292]: audit 2023-08-27T22:39:52.144432+0000 mon.a (mon.0) 1470 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:39:53.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:53 smithi006 bash[27292]: audit 2023-08-27T22:39:52.525626+0000 mon.a (mon.0) 1471 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:39:53.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:53 smithi006 bash[27292]: audit 2023-08-27T22:39:52.798773+0000 mon.b (mon.1) 243 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:39:53.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:53 smithi006 bash[27292]: audit 2023-08-27T22:39:52.800895+0000 mon.b (mon.1) 244 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:39:53.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:53 smithi006 bash[27292]: audit 2023-08-27T22:39:52.804172+0000 mon.a (mon.0) 1472 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:39:53.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:53 smithi130 bash[27579]: cluster 2023-08-27T22:39:51.877288+0000 mgr.y (mgr.14901) 1240 : cluster [DBG] pgmap v1247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:53.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:53 smithi130 bash[27579]: audit 2023-08-27T22:39:52.144432+0000 mon.a (mon.0) 1470 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:39:53.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:53 smithi130 bash[27579]: audit 2023-08-27T22:39:52.525626+0000 mon.a (mon.0) 1471 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:39:53.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:53 smithi130 bash[27579]: audit 2023-08-27T22:39:52.798773+0000 mon.b (mon.1) 243 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:39:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:53 smithi130 bash[27579]: audit 2023-08-27T22:39:52.800895+0000 mon.b (mon.1) 244 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:39:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:53 smithi130 bash[27579]: audit 2023-08-27T22:39:52.804172+0000 mon.a (mon.0) 1472 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:39:54.153 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:39:53.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:54.154 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:39:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:39:53.718Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:39:54.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:54 smithi006 bash[33918]: audit 2023-08-27T22:39:53.239859+0000 mon.a (mon.0) 1473 : audit [DBG] from='client.? 172.21.15.6:0/4085561940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:54.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:54 smithi006 bash[27292]: audit 2023-08-27T22:39:53.239859+0000 mon.a (mon.0) 1473 : audit [DBG] from='client.? 172.21.15.6:0/4085561940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:54.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:54 smithi130 bash[27579]: audit 2023-08-27T22:39:53.239859+0000 mon.a (mon.0) 1473 : audit [DBG] from='client.? 172.21.15.6:0/4085561940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:55.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:55 smithi006 bash[33918]: cluster 2023-08-27T22:39:53.877933+0000 mgr.y (mgr.14901) 1241 : cluster [DBG] pgmap v1248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:55.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:55 smithi006 bash[27292]: cluster 2023-08-27T22:39:53.877933+0000 mgr.y (mgr.14901) 1241 : cluster [DBG] pgmap v1248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:55.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:55 smithi130 bash[27579]: cluster 2023-08-27T22:39:53.877933+0000 mgr.y (mgr.14901) 1241 : cluster [DBG] pgmap v1248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:56.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:56 smithi006 bash[33918]: audit 2023-08-27T22:39:55.654000+0000 mon.a (mon.0) 1474 : audit [DBG] from='client.? 172.21.15.6:0/1403302596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:56.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:56 smithi006 bash[27292]: audit 2023-08-27T22:39:55.654000+0000 mon.a (mon.0) 1474 : audit [DBG] from='client.? 172.21.15.6:0/1403302596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:56.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:56 smithi130 bash[27579]: audit 2023-08-27T22:39:55.654000+0000 mon.a (mon.0) 1474 : audit [DBG] from='client.? 172.21.15.6:0/1403302596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:57.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:57 smithi006 bash[33918]: cluster 2023-08-27T22:39:55.879214+0000 mgr.y (mgr.14901) 1242 : cluster [DBG] pgmap v1249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:57.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:57 smithi006 bash[27292]: cluster 2023-08-27T22:39:55.879214+0000 mgr.y (mgr.14901) 1242 : cluster [DBG] pgmap v1249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:57.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:57 smithi130 bash[27579]: cluster 2023-08-27T22:39:55.879214+0000 mgr.y (mgr.14901) 1242 : cluster [DBG] pgmap v1249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:58.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:58 smithi006 bash[33918]: audit 2023-08-27T22:39:58.077351+0000 mon.a (mon.0) 1475 : audit [DBG] from='client.? 172.21.15.6:0/891322322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:58.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:58 smithi006 bash[27292]: audit 2023-08-27T22:39:58.077351+0000 mon.a (mon.0) 1475 : audit [DBG] from='client.? 172.21.15.6:0/891322322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:58.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:58 smithi130 bash[27579]: audit 2023-08-27T22:39:58.077351+0000 mon.a (mon.0) 1475 : audit [DBG] from='client.? 172.21.15.6:0/891322322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:59.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:39:59 smithi006 bash[33918]: cluster 2023-08-27T22:39:57.879720+0000 mgr.y (mgr.14901) 1243 : cluster [DBG] pgmap v1250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:59.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:39:59 smithi006 bash[27292]: cluster 2023-08-27T22:39:57.879720+0000 mgr.y (mgr.14901) 1243 : cluster [DBG] pgmap v1250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:39:59.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:39:59 smithi130 bash[27579]: cluster 2023-08-27T22:39:57.879720+0000 mgr.y (mgr.14901) 1243 : cluster [DBG] pgmap v1250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:00.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:00 smithi006 bash[27292]: cluster 2023-08-27T22:40:00.000156+0000 mon.a (mon.0) 1476 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:40:00.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:00 smithi006 bash[33918]: cluster 2023-08-27T22:40:00.000156+0000 mon.a (mon.0) 1476 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:40:00.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:40:00.254Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:00.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:40:00.258Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:00.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:00 smithi130 bash[27579]: cluster 2023-08-27T22:40:00.000156+0000 mon.a (mon.0) 1476 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:40:01.193 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:40:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:40:00] "GET /metrics HTTP/1.1" 200 193083 "" "Prometheus/2.33.4" 2023-08-27T22:40:01.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:01 smithi006 bash[33918]: cluster 2023-08-27T22:39:59.880762+0000 mgr.y (mgr.14901) 1244 : cluster [DBG] pgmap v1251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:01.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:01 smithi006 bash[33918]: audit 2023-08-27T22:40:00.496351+0000 mon.c (mon.2) 510 : audit [DBG] from='client.? 172.21.15.6:0/4242992677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:01.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:01 smithi006 bash[27292]: cluster 2023-08-27T22:39:59.880762+0000 mgr.y (mgr.14901) 1244 : cluster [DBG] pgmap v1251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:01.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:01 smithi006 bash[27292]: audit 2023-08-27T22:40:00.496351+0000 mon.c (mon.2) 510 : audit [DBG] from='client.? 172.21.15.6:0/4242992677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:01.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:01 smithi130 bash[27579]: cluster 2023-08-27T22:39:59.880762+0000 mgr.y (mgr.14901) 1244 : cluster [DBG] pgmap v1251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:01.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:01 smithi130 bash[27579]: audit 2023-08-27T22:40:00.496351+0000 mon.c (mon.2) 510 : audit [DBG] from='client.? 172.21.15.6:0/4242992677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:40:01.820Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:40:01.822Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:03.474 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:03 smithi006 bash[33918]: cluster 2023-08-27T22:40:01.881536+0000 mgr.y (mgr.14901) 1245 : cluster [DBG] pgmap v1252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:03.474 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:03 smithi006 bash[33918]: audit 2023-08-27T22:40:02.914282+0000 mon.a (mon.0) 1477 : audit [DBG] from='client.? 172.21.15.6:0/636882746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:03.474 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:03 smithi006 bash[27292]: cluster 2023-08-27T22:40:01.881536+0000 mgr.y (mgr.14901) 1245 : cluster [DBG] pgmap v1252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:03.474 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:03 smithi006 bash[27292]: audit 2023-08-27T22:40:02.914282+0000 mon.a (mon.0) 1477 : audit [DBG] from='client.? 172.21.15.6:0/636882746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:03.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:03 smithi130 bash[27579]: cluster 2023-08-27T22:40:01.881536+0000 mgr.y (mgr.14901) 1245 : cluster [DBG] pgmap v1252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:03.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:03 smithi130 bash[27579]: audit 2023-08-27T22:40:02.914282+0000 mon.a (mon.0) 1477 : audit [DBG] from='client.? 172.21.15.6:0/636882746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:04.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:40:03.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:04.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:40:03.719Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:04 smithi130 bash[27579]: cluster 2023-08-27T22:40:03.882223+0000 mgr.y (mgr.14901) 1246 : cluster [DBG] pgmap v1253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:05.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:04 smithi006 bash[33918]: cluster 2023-08-27T22:40:03.882223+0000 mgr.y (mgr.14901) 1246 : cluster [DBG] pgmap v1253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:05.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:04 smithi006 bash[27292]: cluster 2023-08-27T22:40:03.882223+0000 mgr.y (mgr.14901) 1246 : cluster [DBG] pgmap v1253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:06.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:05 smithi130 bash[27579]: audit 2023-08-27T22:40:05.330470+0000 mon.a (mon.0) 1478 : audit [DBG] from='client.? 172.21.15.6:0/163070048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:06.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:05 smithi006 bash[33918]: audit 2023-08-27T22:40:05.330470+0000 mon.a (mon.0) 1478 : audit [DBG] from='client.? 172.21.15.6:0/163070048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:06.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:05 smithi006 bash[27292]: audit 2023-08-27T22:40:05.330470+0000 mon.a (mon.0) 1478 : audit [DBG] from='client.? 172.21.15.6:0/163070048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:06 smithi130 bash[27579]: cluster 2023-08-27T22:40:05.883359+0000 mgr.y (mgr.14901) 1247 : cluster [DBG] pgmap v1254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:07.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:06 smithi006 bash[27292]: cluster 2023-08-27T22:40:05.883359+0000 mgr.y (mgr.14901) 1247 : cluster [DBG] pgmap v1254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:07.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:06 smithi006 bash[33918]: cluster 2023-08-27T22:40:05.883359+0000 mgr.y (mgr.14901) 1247 : cluster [DBG] pgmap v1254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:08.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:07 smithi130 bash[27579]: audit 2023-08-27T22:40:07.748625+0000 mon.c (mon.2) 511 : audit [DBG] from='client.? 172.21.15.6:0/1335850471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:08.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:07 smithi006 bash[33918]: audit 2023-08-27T22:40:07.748625+0000 mon.c (mon.2) 511 : audit [DBG] from='client.? 172.21.15.6:0/1335850471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:08.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:07 smithi006 bash[27292]: audit 2023-08-27T22:40:07.748625+0000 mon.c (mon.2) 511 : audit [DBG] from='client.? 172.21.15.6:0/1335850471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:08 smithi130 bash[27579]: cluster 2023-08-27T22:40:07.884042+0000 mgr.y (mgr.14901) 1248 : cluster [DBG] pgmap v1255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:09.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:08 smithi006 bash[33918]: cluster 2023-08-27T22:40:07.884042+0000 mgr.y (mgr.14901) 1248 : cluster [DBG] pgmap v1255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:09.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:08 smithi006 bash[27292]: cluster 2023-08-27T22:40:07.884042+0000 mgr.y (mgr.14901) 1248 : cluster [DBG] pgmap v1255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:40:10.255Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:40:10.257Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:11.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:40:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:40:10] "GET /metrics HTTP/1.1" 200 193083 "" "Prometheus/2.33.4" 2023-08-27T22:40:11.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:10 smithi006 bash[33918]: cluster 2023-08-27T22:40:09.885085+0000 mgr.y (mgr.14901) 1249 : cluster [DBG] pgmap v1256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:11.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:10 smithi006 bash[33918]: audit 2023-08-27T22:40:10.170445+0000 mon.a (mon.0) 1479 : audit [DBG] from='client.? 172.21.15.6:0/2745015154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:11.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:10 smithi006 bash[27292]: cluster 2023-08-27T22:40:09.885085+0000 mgr.y (mgr.14901) 1249 : cluster [DBG] pgmap v1256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:11.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:10 smithi006 bash[27292]: audit 2023-08-27T22:40:10.170445+0000 mon.a (mon.0) 1479 : audit [DBG] from='client.? 172.21.15.6:0/2745015154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:11.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:10 smithi130 bash[27579]: cluster 2023-08-27T22:40:09.885085+0000 mgr.y (mgr.14901) 1249 : cluster [DBG] pgmap v1256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:11.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:10 smithi130 bash[27579]: audit 2023-08-27T22:40:10.170445+0000 mon.a (mon.0) 1479 : audit [DBG] from='client.? 172.21.15.6:0/2745015154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:40:11.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:40:11.823Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:13.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:12 smithi006 bash[33918]: cluster 2023-08-27T22:40:11.885717+0000 mgr.y (mgr.14901) 1250 : cluster [DBG] pgmap v1257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:13.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:12 smithi006 bash[33918]: audit 2023-08-27T22:40:12.576255+0000 mon.c (mon.2) 512 : audit [DBG] from='client.? 172.21.15.6:0/102973824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:13.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:12 smithi006 bash[27292]: cluster 2023-08-27T22:40:11.885717+0000 mgr.y (mgr.14901) 1250 : cluster [DBG] pgmap v1257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:13.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:12 smithi006 bash[27292]: audit 2023-08-27T22:40:12.576255+0000 mon.c (mon.2) 512 : audit [DBG] from='client.? 172.21.15.6:0/102973824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:13.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:12 smithi130 bash[27579]: cluster 2023-08-27T22:40:11.885717+0000 mgr.y (mgr.14901) 1250 : cluster [DBG] pgmap v1257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:13.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:12 smithi130 bash[27579]: audit 2023-08-27T22:40:12.576255+0000 mon.c (mon.2) 512 : audit [DBG] from='client.? 172.21.15.6:0/102973824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:14.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:40:13.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:14.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:40:13.720Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:14 smithi130 bash[27579]: cluster 2023-08-27T22:40:13.886365+0000 mgr.y (mgr.14901) 1251 : cluster [DBG] pgmap v1258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:15.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:14 smithi006 bash[33918]: cluster 2023-08-27T22:40:13.886365+0000 mgr.y (mgr.14901) 1251 : cluster [DBG] pgmap v1258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:15.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:14 smithi006 bash[27292]: cluster 2023-08-27T22:40:13.886365+0000 mgr.y (mgr.14901) 1251 : cluster [DBG] pgmap v1258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:16.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:15 smithi130 bash[27579]: audit 2023-08-27T22:40:14.996831+0000 mon.a (mon.0) 1480 : audit [DBG] from='client.? 172.21.15.6:0/438394636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:16.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:15 smithi006 bash[33918]: audit 2023-08-27T22:40:14.996831+0000 mon.a (mon.0) 1480 : audit [DBG] from='client.? 172.21.15.6:0/438394636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:16.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:15 smithi006 bash[27292]: audit 2023-08-27T22:40:14.996831+0000 mon.a (mon.0) 1480 : audit [DBG] from='client.? 172.21.15.6:0/438394636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:16 smithi130 bash[27579]: cluster 2023-08-27T22:40:15.887076+0000 mgr.y (mgr.14901) 1252 : cluster [DBG] pgmap v1259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:17.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:16 smithi006 bash[27292]: cluster 2023-08-27T22:40:15.887076+0000 mgr.y (mgr.14901) 1252 : cluster [DBG] pgmap v1259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:17.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:16 smithi006 bash[33918]: cluster 2023-08-27T22:40:15.887076+0000 mgr.y (mgr.14901) 1252 : cluster [DBG] pgmap v1259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:18.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:17 smithi130 bash[27579]: audit 2023-08-27T22:40:17.412302+0000 mon.c (mon.2) 513 : audit [DBG] from='client.? 172.21.15.6:0/2170716703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:18.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:17 smithi006 bash[27292]: audit 2023-08-27T22:40:17.412302+0000 mon.c (mon.2) 513 : audit [DBG] from='client.? 172.21.15.6:0/2170716703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:18.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:17 smithi006 bash[33918]: audit 2023-08-27T22:40:17.412302+0000 mon.c (mon.2) 513 : audit [DBG] from='client.? 172.21.15.6:0/2170716703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:18 smithi130 bash[27579]: cluster 2023-08-27T22:40:17.887704+0000 mgr.y (mgr.14901) 1253 : cluster [DBG] pgmap v1260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:19.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:18 smithi006 bash[27292]: cluster 2023-08-27T22:40:17.887704+0000 mgr.y (mgr.14901) 1253 : cluster [DBG] pgmap v1260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:19.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:18 smithi006 bash[33918]: cluster 2023-08-27T22:40:17.887704+0000 mgr.y (mgr.14901) 1253 : cluster [DBG] pgmap v1260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:20.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:19 smithi130 bash[27579]: audit 2023-08-27T22:40:19.830992+0000 mon.a (mon.0) 1481 : audit [DBG] from='client.? 172.21.15.6:0/2742076380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:20.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:19 smithi006 bash[27292]: audit 2023-08-27T22:40:19.830992+0000 mon.a (mon.0) 1481 : audit [DBG] from='client.? 172.21.15.6:0/2742076380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:20.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:19 smithi006 bash[33918]: audit 2023-08-27T22:40:19.830992+0000 mon.a (mon.0) 1481 : audit [DBG] from='client.? 172.21.15.6:0/2742076380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:40:20.256Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:40:20.258Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:21.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:40:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:40:20] "GET /metrics HTTP/1.1" 200 193083 "" "Prometheus/2.33.4" 2023-08-27T22:40:21.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:20 smithi006 bash[27292]: cluster 2023-08-27T22:40:19.888511+0000 mgr.y (mgr.14901) 1254 : cluster [DBG] pgmap v1261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:21.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:20 smithi006 bash[33918]: cluster 2023-08-27T22:40:19.888511+0000 mgr.y (mgr.14901) 1254 : cluster [DBG] pgmap v1261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:21.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:20 smithi130 bash[27579]: cluster 2023-08-27T22:40:19.888511+0000 mgr.y (mgr.14901) 1254 : cluster [DBG] pgmap v1261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:40:21.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:40:21.823Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:23.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:22 smithi006 bash[33918]: cluster 2023-08-27T22:40:21.889187+0000 mgr.y (mgr.14901) 1255 : cluster [DBG] pgmap v1262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:23.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:22 smithi006 bash[33918]: audit 2023-08-27T22:40:22.253877+0000 mon.a (mon.0) 1482 : audit [DBG] from='client.? 172.21.15.6:0/2516476509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:23.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:22 smithi006 bash[27292]: cluster 2023-08-27T22:40:21.889187+0000 mgr.y (mgr.14901) 1255 : cluster [DBG] pgmap v1262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:23.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:22 smithi006 bash[27292]: audit 2023-08-27T22:40:22.253877+0000 mon.a (mon.0) 1482 : audit [DBG] from='client.? 172.21.15.6:0/2516476509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:23.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:22 smithi130 bash[27579]: cluster 2023-08-27T22:40:21.889187+0000 mgr.y (mgr.14901) 1255 : cluster [DBG] pgmap v1262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:23.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:22 smithi130 bash[27579]: audit 2023-08-27T22:40:22.253877+0000 mon.a (mon.0) 1482 : audit [DBG] from='client.? 172.21.15.6:0/2516476509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:24.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:40:23.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:24.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:40:23.721Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:24 smithi130 bash[27579]: cluster 2023-08-27T22:40:23.889878+0000 mgr.y (mgr.14901) 1256 : cluster [DBG] pgmap v1263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:24 smithi130 bash[27579]: audit 2023-08-27T22:40:24.672875+0000 mon.c (mon.2) 514 : audit [DBG] from='client.? 172.21.15.6:0/3215502879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:25.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:24 smithi006 bash[33918]: cluster 2023-08-27T22:40:23.889878+0000 mgr.y (mgr.14901) 1256 : cluster [DBG] pgmap v1263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:25.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:24 smithi006 bash[33918]: audit 2023-08-27T22:40:24.672875+0000 mon.c (mon.2) 514 : audit [DBG] from='client.? 172.21.15.6:0/3215502879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:25.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:24 smithi006 bash[27292]: cluster 2023-08-27T22:40:23.889878+0000 mgr.y (mgr.14901) 1256 : cluster [DBG] pgmap v1263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:25.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:24 smithi006 bash[27292]: audit 2023-08-27T22:40:24.672875+0000 mon.c (mon.2) 514 : audit [DBG] from='client.? 172.21.15.6:0/3215502879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:26 smithi006 bash[27292]: cluster 2023-08-27T22:40:25.891010+0000 mgr.y (mgr.14901) 1257 : cluster [DBG] pgmap v1264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:27.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:26 smithi006 bash[33918]: cluster 2023-08-27T22:40:25.891010+0000 mgr.y (mgr.14901) 1257 : cluster [DBG] pgmap v1264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:27.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:26 smithi130 bash[27579]: cluster 2023-08-27T22:40:25.891010+0000 mgr.y (mgr.14901) 1257 : cluster [DBG] pgmap v1264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:28.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:27 smithi006 bash[27292]: audit 2023-08-27T22:40:27.081250+0000 mon.a (mon.0) 1483 : audit [DBG] from='client.? 172.21.15.6:0/1161457582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:28.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:27 smithi006 bash[33918]: audit 2023-08-27T22:40:27.081250+0000 mon.a (mon.0) 1483 : audit [DBG] from='client.? 172.21.15.6:0/1161457582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:28.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:27 smithi130 bash[27579]: audit 2023-08-27T22:40:27.081250+0000 mon.a (mon.0) 1483 : audit [DBG] from='client.? 172.21.15.6:0/1161457582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:29.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:29 smithi006 bash[33918]: cluster 2023-08-27T22:40:27.891696+0000 mgr.y (mgr.14901) 1258 : cluster [DBG] pgmap v1265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:29.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:28 smithi006 bash[27292]: cluster 2023-08-27T22:40:27.891696+0000 mgr.y (mgr.14901) 1258 : cluster [DBG] pgmap v1265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:29.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:28 smithi130 bash[27579]: cluster 2023-08-27T22:40:27.891696+0000 mgr.y (mgr.14901) 1258 : cluster [DBG] pgmap v1265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:30.264 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:29 smithi006 bash[33918]: audit 2023-08-27T22:40:29.492300+0000 mon.a (mon.0) 1484 : audit [DBG] from='client.? 172.21.15.6:0/1175676146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:30.264 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:29 smithi006 bash[27292]: audit 2023-08-27T22:40:29.492300+0000 mon.a (mon.0) 1484 : audit [DBG] from='client.? 172.21.15.6:0/1175676146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:30.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:29 smithi130 bash[27579]: audit 2023-08-27T22:40:29.492300+0000 mon.a (mon.0) 1484 : audit [DBG] from='client.? 172.21.15.6:0/1175676146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:40:30.257Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:40:30.259Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:31.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:40:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:40:30] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T22:40:31.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:30 smithi006 bash[33918]: cluster 2023-08-27T22:40:29.892798+0000 mgr.y (mgr.14901) 1259 : cluster [DBG] pgmap v1266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:31.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:30 smithi006 bash[27292]: cluster 2023-08-27T22:40:29.892798+0000 mgr.y (mgr.14901) 1259 : cluster [DBG] pgmap v1266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:31.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:30 smithi130 bash[27579]: cluster 2023-08-27T22:40:29.892798+0000 mgr.y (mgr.14901) 1259 : cluster [DBG] pgmap v1266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:32.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:31 smithi006 bash[27292]: audit 2023-08-27T22:40:31.907130+0000 mon.c (mon.2) 515 : audit [DBG] from='client.? 172.21.15.6:0/4174079072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:32.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:31 smithi006 bash[33918]: audit 2023-08-27T22:40:31.907130+0000 mon.c (mon.2) 515 : audit [DBG] from='client.? 172.21.15.6:0/4174079072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:40:31.823Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:40:31.825Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:32.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:31 smithi130 bash[27579]: audit 2023-08-27T22:40:31.907130+0000 mon.c (mon.2) 515 : audit [DBG] from='client.? 172.21.15.6:0/4174079072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:33.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:32 smithi130 bash[27579]: cluster 2023-08-27T22:40:31.893384+0000 mgr.y (mgr.14901) 1260 : cluster [DBG] pgmap v1267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:33.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:32 smithi006 bash[27292]: cluster 2023-08-27T22:40:31.893384+0000 mgr.y (mgr.14901) 1260 : cluster [DBG] pgmap v1267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:33.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:32 smithi006 bash[33918]: cluster 2023-08-27T22:40:31.893384+0000 mgr.y (mgr.14901) 1260 : cluster [DBG] pgmap v1267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:34.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:40:33.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:34.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:40:33.721Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:34 smithi130 bash[27579]: cluster 2023-08-27T22:40:33.894064+0000 mgr.y (mgr.14901) 1261 : cluster [DBG] pgmap v1268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:34 smithi130 bash[27579]: audit 2023-08-27T22:40:34.312056+0000 mon.a (mon.0) 1485 : audit [DBG] from='client.? 172.21.15.6:0/878438457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:35.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:34 smithi006 bash[27292]: cluster 2023-08-27T22:40:33.894064+0000 mgr.y (mgr.14901) 1261 : cluster [DBG] pgmap v1268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:35.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:34 smithi006 bash[27292]: audit 2023-08-27T22:40:34.312056+0000 mon.a (mon.0) 1485 : audit [DBG] from='client.? 172.21.15.6:0/878438457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:35.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:34 smithi006 bash[33918]: cluster 2023-08-27T22:40:33.894064+0000 mgr.y (mgr.14901) 1261 : cluster [DBG] pgmap v1268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:35.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:34 smithi006 bash[33918]: audit 2023-08-27T22:40:34.312056+0000 mon.a (mon.0) 1485 : audit [DBG] from='client.? 172.21.15.6:0/878438457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:37.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:36 smithi006 bash[33918]: cluster 2023-08-27T22:40:35.895186+0000 mgr.y (mgr.14901) 1262 : cluster [DBG] pgmap v1269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:37.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:36 smithi006 bash[33918]: audit 2023-08-27T22:40:36.719707+0000 mon.c (mon.2) 516 : audit [DBG] from='client.? 172.21.15.6:0/3829457470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:37.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:36 smithi006 bash[27292]: cluster 2023-08-27T22:40:35.895186+0000 mgr.y (mgr.14901) 1262 : cluster [DBG] pgmap v1269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:37.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:36 smithi006 bash[27292]: audit 2023-08-27T22:40:36.719707+0000 mon.c (mon.2) 516 : audit [DBG] from='client.? 172.21.15.6:0/3829457470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:37.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:36 smithi130 bash[27579]: cluster 2023-08-27T22:40:35.895186+0000 mgr.y (mgr.14901) 1262 : cluster [DBG] pgmap v1269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:37.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:36 smithi130 bash[27579]: audit 2023-08-27T22:40:36.719707+0000 mon.c (mon.2) 516 : audit [DBG] from='client.? 172.21.15.6:0/3829457470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:39.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:38 smithi006 bash[27292]: cluster 2023-08-27T22:40:37.895790+0000 mgr.y (mgr.14901) 1263 : cluster [DBG] pgmap v1270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:39.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:38 smithi006 bash[33918]: cluster 2023-08-27T22:40:37.895790+0000 mgr.y (mgr.14901) 1263 : cluster [DBG] pgmap v1270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:39.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:38 smithi130 bash[27579]: cluster 2023-08-27T22:40:37.895790+0000 mgr.y (mgr.14901) 1263 : cluster [DBG] pgmap v1270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:40.265 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:39 smithi006 bash[33918]: audit 2023-08-27T22:40:39.137666+0000 mon.c (mon.2) 517 : audit [DBG] from='client.? 172.21.15.6:0/881698418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:40.266 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:39 smithi006 bash[27292]: audit 2023-08-27T22:40:39.137666+0000 mon.c (mon.2) 517 : audit [DBG] from='client.? 172.21.15.6:0/881698418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:40.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:39 smithi130 bash[27579]: audit 2023-08-27T22:40:39.137666+0000 mon.c (mon.2) 517 : audit [DBG] from='client.? 172.21.15.6:0/881698418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:40:40.258Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:40:40.260Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:41.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:40 smithi006 bash[27292]: cluster 2023-08-27T22:40:39.896913+0000 mgr.y (mgr.14901) 1264 : cluster [DBG] pgmap v1271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:41.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:40:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:40:40] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T22:40:41.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:40 smithi006 bash[33918]: cluster 2023-08-27T22:40:39.896913+0000 mgr.y (mgr.14901) 1264 : cluster [DBG] pgmap v1271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:41.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:40 smithi130 bash[27579]: cluster 2023-08-27T22:40:39.896913+0000 mgr.y (mgr.14901) 1264 : cluster [DBG] pgmap v1271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:42.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:41 smithi006 bash[27292]: audit 2023-08-27T22:40:41.554697+0000 mon.a (mon.0) 1486 : audit [DBG] from='client.? 172.21.15.6:0/1889915214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:42.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:41 smithi006 bash[33918]: audit 2023-08-27T22:40:41.554697+0000 mon.a (mon.0) 1486 : audit [DBG] from='client.? 172.21.15.6:0/1889915214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:40:41.823Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:40:41.825Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:42.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:41 smithi130 bash[27579]: audit 2023-08-27T22:40:41.554697+0000 mon.a (mon.0) 1486 : audit [DBG] from='client.? 172.21.15.6:0/1889915214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:43.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:42 smithi130 bash[27579]: cluster 2023-08-27T22:40:41.897529+0000 mgr.y (mgr.14901) 1265 : cluster [DBG] pgmap v1272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:43.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:42 smithi006 bash[33918]: cluster 2023-08-27T22:40:41.897529+0000 mgr.y (mgr.14901) 1265 : cluster [DBG] pgmap v1272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:43.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:42 smithi006 bash[27292]: cluster 2023-08-27T22:40:41.897529+0000 mgr.y (mgr.14901) 1265 : cluster [DBG] pgmap v1272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:44.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:40:43.720Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:44.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:40:43.722Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:44 smithi130 bash[27579]: cluster 2023-08-27T22:40:43.897957+0000 mgr.y (mgr.14901) 1266 : cluster [DBG] pgmap v1273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:44 smithi130 bash[27579]: audit 2023-08-27T22:40:43.969758+0000 mon.a (mon.0) 1487 : audit [DBG] from='client.? 172.21.15.6:0/444764652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:45.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:44 smithi006 bash[33918]: cluster 2023-08-27T22:40:43.897957+0000 mgr.y (mgr.14901) 1266 : cluster [DBG] pgmap v1273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:45.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:44 smithi006 bash[33918]: audit 2023-08-27T22:40:43.969758+0000 mon.a (mon.0) 1487 : audit [DBG] from='client.? 172.21.15.6:0/444764652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:45.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:44 smithi006 bash[27292]: cluster 2023-08-27T22:40:43.897957+0000 mgr.y (mgr.14901) 1266 : cluster [DBG] pgmap v1273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:45.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:44 smithi006 bash[27292]: audit 2023-08-27T22:40:43.969758+0000 mon.a (mon.0) 1487 : audit [DBG] from='client.? 172.21.15.6:0/444764652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:47.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:46 smithi006 bash[33918]: cluster 2023-08-27T22:40:45.899060+0000 mgr.y (mgr.14901) 1267 : cluster [DBG] pgmap v1274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:47.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:46 smithi006 bash[33918]: audit 2023-08-27T22:40:46.380630+0000 mon.c (mon.2) 518 : audit [DBG] from='client.? 172.21.15.6:0/1644935649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:47.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:46 smithi006 bash[27292]: cluster 2023-08-27T22:40:45.899060+0000 mgr.y (mgr.14901) 1267 : cluster [DBG] pgmap v1274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:47.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:46 smithi006 bash[27292]: audit 2023-08-27T22:40:46.380630+0000 mon.c (mon.2) 518 : audit [DBG] from='client.? 172.21.15.6:0/1644935649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:47.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:46 smithi130 bash[27579]: cluster 2023-08-27T22:40:45.899060+0000 mgr.y (mgr.14901) 1267 : cluster [DBG] pgmap v1274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:47.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:46 smithi130 bash[27579]: audit 2023-08-27T22:40:46.380630+0000 mon.c (mon.2) 518 : audit [DBG] from='client.? 172.21.15.6:0/1644935649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:49.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:48 smithi006 bash[33918]: cluster 2023-08-27T22:40:47.899707+0000 mgr.y (mgr.14901) 1268 : cluster [DBG] pgmap v1275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:49.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:48 smithi006 bash[33918]: audit 2023-08-27T22:40:48.795615+0000 mon.a (mon.0) 1488 : audit [DBG] from='client.? 172.21.15.6:0/233101105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:49.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:48 smithi006 bash[27292]: cluster 2023-08-27T22:40:47.899707+0000 mgr.y (mgr.14901) 1268 : cluster [DBG] pgmap v1275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:49.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:48 smithi006 bash[27292]: audit 2023-08-27T22:40:48.795615+0000 mon.a (mon.0) 1488 : audit [DBG] from='client.? 172.21.15.6:0/233101105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:49.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:48 smithi130 bash[27579]: cluster 2023-08-27T22:40:47.899707+0000 mgr.y (mgr.14901) 1268 : cluster [DBG] pgmap v1275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:49.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:48 smithi130 bash[27579]: audit 2023-08-27T22:40:48.795615+0000 mon.a (mon.0) 1488 : audit [DBG] from='client.? 172.21.15.6:0/233101105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:40:50.258Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:40:50.260Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:51.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:40:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:40:50] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:40:51.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:50 smithi006 bash[33918]: cluster 2023-08-27T22:40:49.900824+0000 mgr.y (mgr.14901) 1269 : cluster [DBG] pgmap v1276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:51.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:50 smithi006 bash[27292]: cluster 2023-08-27T22:40:49.900824+0000 mgr.y (mgr.14901) 1269 : cluster [DBG] pgmap v1276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:51.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:50 smithi130 bash[27579]: cluster 2023-08-27T22:40:49.900824+0000 mgr.y (mgr.14901) 1269 : cluster [DBG] pgmap v1276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:52.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:51 smithi006 bash[27292]: audit 2023-08-27T22:40:51.208739+0000 mon.a (mon.0) 1489 : audit [DBG] from='client.? 172.21.15.6:0/3623981064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:52.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:51 smithi006 bash[33918]: audit 2023-08-27T22:40:51.208739+0000 mon.a (mon.0) 1489 : audit [DBG] from='client.? 172.21.15.6:0/3623981064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:40:51.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:40:51.826Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:52.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:51 smithi130 bash[27579]: audit 2023-08-27T22:40:51.208739+0000 mon.a (mon.0) 1489 : audit [DBG] from='client.? 172.21.15.6:0/3623981064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:53.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:52 smithi130 bash[27579]: cluster 2023-08-27T22:40:51.901412+0000 mgr.y (mgr.14901) 1270 : cluster [DBG] pgmap v1277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:53.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:52 smithi130 bash[27579]: audit 2023-08-27T22:40:52.820801+0000 mon.b (mon.1) 245 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:40:53.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:52 smithi006 bash[27292]: cluster 2023-08-27T22:40:51.901412+0000 mgr.y (mgr.14901) 1270 : cluster [DBG] pgmap v1277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:53.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:52 smithi006 bash[27292]: audit 2023-08-27T22:40:52.820801+0000 mon.b (mon.1) 245 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:40:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:52 smithi006 bash[33918]: cluster 2023-08-27T22:40:51.901412+0000 mgr.y (mgr.14901) 1270 : cluster [DBG] pgmap v1277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:52 smithi006 bash[33918]: audit 2023-08-27T22:40:52.820801+0000 mon.b (mon.1) 245 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:40:53.986 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:40:53.720Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:53.986 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:40:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:40:53.723Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:40:54.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:53 smithi130 bash[27579]: audit 2023-08-27T22:40:53.618845+0000 mon.a (mon.0) 1490 : audit [DBG] from='client.? 172.21.15.6:0/293934546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:54.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:53 smithi006 bash[27292]: audit 2023-08-27T22:40:53.618845+0000 mon.a (mon.0) 1490 : audit [DBG] from='client.? 172.21.15.6:0/293934546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:54.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:53 smithi006 bash[33918]: audit 2023-08-27T22:40:53.618845+0000 mon.a (mon.0) 1490 : audit [DBG] from='client.? 172.21.15.6:0/293934546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:55.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:54 smithi130 bash[27579]: cluster 2023-08-27T22:40:53.902034+0000 mgr.y (mgr.14901) 1271 : cluster [DBG] pgmap v1278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:55.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:54 smithi006 bash[33918]: cluster 2023-08-27T22:40:53.902034+0000 mgr.y (mgr.14901) 1271 : cluster [DBG] pgmap v1278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:55.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:54 smithi006 bash[27292]: cluster 2023-08-27T22:40:53.902034+0000 mgr.y (mgr.14901) 1271 : cluster [DBG] pgmap v1278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:57.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:56 smithi130 bash[27579]: cluster 2023-08-27T22:40:55.903128+0000 mgr.y (mgr.14901) 1272 : cluster [DBG] pgmap v1279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:57.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:56 smithi130 bash[27579]: audit 2023-08-27T22:40:56.029076+0000 mon.a (mon.0) 1491 : audit [DBG] from='client.? 172.21.15.6:0/725830226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:57.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:56 smithi006 bash[33918]: cluster 2023-08-27T22:40:55.903128+0000 mgr.y (mgr.14901) 1272 : cluster [DBG] pgmap v1279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:57.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:56 smithi006 bash[33918]: audit 2023-08-27T22:40:56.029076+0000 mon.a (mon.0) 1491 : audit [DBG] from='client.? 172.21.15.6:0/725830226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:57.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:56 smithi006 bash[27292]: cluster 2023-08-27T22:40:55.903128+0000 mgr.y (mgr.14901) 1272 : cluster [DBG] pgmap v1279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:57.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:56 smithi006 bash[27292]: audit 2023-08-27T22:40:56.029076+0000 mon.a (mon.0) 1491 : audit [DBG] from='client.? 172.21.15.6:0/725830226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:59.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:59 smithi130 bash[27579]: cluster 2023-08-27T22:40:57.903720+0000 mgr.y (mgr.14901) 1273 : cluster [DBG] pgmap v1280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:59.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:59 smithi130 bash[27579]: audit 2023-08-27T22:40:58.464368+0000 mon.c (mon.2) 519 : audit [DBG] from='client.? 172.21.15.6:0/936323264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:59.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:40:59 smithi130 bash[27579]: audit 2023-08-27T22:40:58.690890+0000 mon.a (mon.0) 1492 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:40:59.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:58 smithi006 bash[33918]: cluster 2023-08-27T22:40:57.903720+0000 mgr.y (mgr.14901) 1273 : cluster [DBG] pgmap v1280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:59.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:58 smithi006 bash[33918]: audit 2023-08-27T22:40:58.464368+0000 mon.c (mon.2) 519 : audit [DBG] from='client.? 172.21.15.6:0/936323264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:59.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:40:58 smithi006 bash[33918]: audit 2023-08-27T22:40:58.690890+0000 mon.a (mon.0) 1492 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:40:59.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:58 smithi006 bash[27292]: cluster 2023-08-27T22:40:57.903720+0000 mgr.y (mgr.14901) 1273 : cluster [DBG] pgmap v1280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:40:59.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:58 smithi006 bash[27292]: audit 2023-08-27T22:40:58.464368+0000 mon.c (mon.2) 519 : audit [DBG] from='client.? 172.21.15.6:0/936323264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:59.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:40:58 smithi006 bash[27292]: audit 2023-08-27T22:40:58.690890+0000 mon.a (mon.0) 1492 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:41:00.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:41:00.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:00.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:41:00.261Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:00.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:00 smithi006 bash[27292]: audit 2023-08-27T22:40:59.171367+0000 mon.a (mon.0) 1493 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:41:00.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:00 smithi006 bash[27292]: audit 2023-08-27T22:40:59.441482+0000 mon.b (mon.1) 246 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:41:00.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:00 smithi006 bash[27292]: audit 2023-08-27T22:40:59.443592+0000 mon.b (mon.1) 247 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:41:00.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:00 smithi006 bash[27292]: audit 2023-08-27T22:40:59.447167+0000 mon.a (mon.0) 1494 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:41:00.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:00 smithi006 bash[33918]: audit 2023-08-27T22:40:59.171367+0000 mon.a (mon.0) 1493 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:41:00.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:00 smithi006 bash[33918]: audit 2023-08-27T22:40:59.441482+0000 mon.b (mon.1) 246 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:41:00.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:00 smithi006 bash[33918]: audit 2023-08-27T22:40:59.443592+0000 mon.b (mon.1) 247 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:41:00.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:00 smithi006 bash[33918]: audit 2023-08-27T22:40:59.447167+0000 mon.a (mon.0) 1494 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:41:00.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:00 smithi130 bash[27579]: audit 2023-08-27T22:40:59.171367+0000 mon.a (mon.0) 1493 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:41:00.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:00 smithi130 bash[27579]: audit 2023-08-27T22:40:59.441482+0000 mon.b (mon.1) 246 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:41:00.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:00 smithi130 bash[27579]: audit 2023-08-27T22:40:59.443592+0000 mon.b (mon.1) 247 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:41:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:00 smithi130 bash[27579]: audit 2023-08-27T22:40:59.447167+0000 mon.a (mon.0) 1494 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:41:01.181 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:41:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:41:00] "GET /metrics HTTP/1.1" 200 193084 "" "Prometheus/2.33.4" 2023-08-27T22:41:01.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:01 smithi006 bash[33918]: cluster 2023-08-27T22:40:59.904772+0000 mgr.y (mgr.14901) 1274 : cluster [DBG] pgmap v1281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:01.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:01 smithi006 bash[33918]: audit 2023-08-27T22:41:00.880216+0000 mon.c (mon.2) 520 : audit [DBG] from='client.? 172.21.15.6:0/1934453416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:01.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:01 smithi006 bash[27292]: cluster 2023-08-27T22:40:59.904772+0000 mgr.y (mgr.14901) 1274 : cluster [DBG] pgmap v1281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:01.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:01 smithi006 bash[27292]: audit 2023-08-27T22:41:00.880216+0000 mon.c (mon.2) 520 : audit [DBG] from='client.? 172.21.15.6:0/1934453416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:01.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:01 smithi130 bash[27579]: cluster 2023-08-27T22:40:59.904772+0000 mgr.y (mgr.14901) 1274 : cluster [DBG] pgmap v1281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:01.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:01 smithi130 bash[27579]: audit 2023-08-27T22:41:00.880216+0000 mon.c (mon.2) 520 : audit [DBG] from='client.? 172.21.15.6:0/1934453416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:41:01.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:41:01.827Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:03.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:03 smithi006 bash[27292]: cluster 2023-08-27T22:41:01.905545+0000 mgr.y (mgr.14901) 1275 : cluster [DBG] pgmap v1282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:03.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:03 smithi006 bash[33918]: cluster 2023-08-27T22:41:01.905545+0000 mgr.y (mgr.14901) 1275 : cluster [DBG] pgmap v1282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:03.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:03 smithi130 bash[27579]: cluster 2023-08-27T22:41:01.905545+0000 mgr.y (mgr.14901) 1275 : cluster [DBG] pgmap v1282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:04.189 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:41:03.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:04.190 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:41:03.723Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:04.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:04 smithi006 bash[33918]: audit 2023-08-27T22:41:03.297498+0000 mon.c (mon.2) 521 : audit [DBG] from='client.? 172.21.15.6:0/660789172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:04.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:04 smithi006 bash[27292]: audit 2023-08-27T22:41:03.297498+0000 mon.c (mon.2) 521 : audit [DBG] from='client.? 172.21.15.6:0/660789172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:04.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:04 smithi130 bash[27579]: audit 2023-08-27T22:41:03.297498+0000 mon.c (mon.2) 521 : audit [DBG] from='client.? 172.21.15.6:0/660789172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:05.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:05 smithi006 bash[27292]: cluster 2023-08-27T22:41:03.906232+0000 mgr.y (mgr.14901) 1276 : cluster [DBG] pgmap v1283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:05.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:05 smithi006 bash[33918]: cluster 2023-08-27T22:41:03.906232+0000 mgr.y (mgr.14901) 1276 : cluster [DBG] pgmap v1283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:05.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:05 smithi130 bash[27579]: cluster 2023-08-27T22:41:03.906232+0000 mgr.y (mgr.14901) 1276 : cluster [DBG] pgmap v1283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:06.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:06 smithi006 bash[33918]: audit 2023-08-27T22:41:05.710593+0000 mon.c (mon.2) 522 : audit [DBG] from='client.? 172.21.15.6:0/4275767152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:06.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:06 smithi006 bash[27292]: audit 2023-08-27T22:41:05.710593+0000 mon.c (mon.2) 522 : audit [DBG] from='client.? 172.21.15.6:0/4275767152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:06.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:06 smithi130 bash[27579]: audit 2023-08-27T22:41:05.710593+0000 mon.c (mon.2) 522 : audit [DBG] from='client.? 172.21.15.6:0/4275767152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:07.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:07 smithi006 bash[33918]: cluster 2023-08-27T22:41:05.907332+0000 mgr.y (mgr.14901) 1277 : cluster [DBG] pgmap v1284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:07.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:07 smithi006 bash[27292]: cluster 2023-08-27T22:41:05.907332+0000 mgr.y (mgr.14901) 1277 : cluster [DBG] pgmap v1284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:07.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:07 smithi130 bash[27579]: cluster 2023-08-27T22:41:05.907332+0000 mgr.y (mgr.14901) 1277 : cluster [DBG] pgmap v1284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:08.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:08 smithi006 bash[27292]: audit 2023-08-27T22:41:08.129008+0000 mon.c (mon.2) 523 : audit [DBG] from='client.? 172.21.15.6:0/4022608182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:08.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:08 smithi006 bash[33918]: audit 2023-08-27T22:41:08.129008+0000 mon.c (mon.2) 523 : audit [DBG] from='client.? 172.21.15.6:0/4022608182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:09.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:09 smithi006 bash[27292]: cluster 2023-08-27T22:41:07.907835+0000 mgr.y (mgr.14901) 1278 : cluster [DBG] pgmap v1285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:09.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:09 smithi006 bash[33918]: cluster 2023-08-27T22:41:07.907835+0000 mgr.y (mgr.14901) 1278 : cluster [DBG] pgmap v1285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:09.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:08 smithi130 bash[27579]: audit 2023-08-27T22:41:08.129008+0000 mon.c (mon.2) 523 : audit [DBG] from='client.? 172.21.15.6:0/4022608182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:09.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:09 smithi130 bash[27579]: cluster 2023-08-27T22:41:07.907835+0000 mgr.y (mgr.14901) 1278 : cluster [DBG] pgmap v1285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:41:10.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:41:10.261Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:11.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:41:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:41:10] "GET /metrics HTTP/1.1" 200 193084 "" "Prometheus/2.33.4" 2023-08-27T22:41:11.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:11 smithi006 bash[27292]: cluster 2023-08-27T22:41:09.908899+0000 mgr.y (mgr.14901) 1279 : cluster [DBG] pgmap v1286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:11.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:11 smithi006 bash[27292]: audit 2023-08-27T22:41:10.556519+0000 mon.a (mon.0) 1495 : audit [DBG] from='client.? 172.21.15.6:0/2598196616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:11.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:11 smithi006 bash[33918]: cluster 2023-08-27T22:41:09.908899+0000 mgr.y (mgr.14901) 1279 : cluster [DBG] pgmap v1286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:11.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:11 smithi006 bash[33918]: audit 2023-08-27T22:41:10.556519+0000 mon.a (mon.0) 1495 : audit [DBG] from='client.? 172.21.15.6:0/2598196616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:11.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:11 smithi130 bash[27579]: cluster 2023-08-27T22:41:09.908899+0000 mgr.y (mgr.14901) 1279 : cluster [DBG] pgmap v1286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:11.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:11 smithi130 bash[27579]: audit 2023-08-27T22:41:10.556519+0000 mon.a (mon.0) 1495 : audit [DBG] from='client.? 172.21.15.6:0/2598196616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:41:11.826Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:41:11.828Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:13.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:13 smithi130 bash[27579]: cluster 2023-08-27T22:41:11.909643+0000 mgr.y (mgr.14901) 1280 : cluster [DBG] pgmap v1287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:13.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:13 smithi130 bash[27579]: audit 2023-08-27T22:41:12.964451+0000 mon.c (mon.2) 524 : audit [DBG] from='client.? 172.21.15.6:0/2169984285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:13.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:13 smithi006 bash[33918]: cluster 2023-08-27T22:41:11.909643+0000 mgr.y (mgr.14901) 1280 : cluster [DBG] pgmap v1287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:13.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:13 smithi006 bash[33918]: audit 2023-08-27T22:41:12.964451+0000 mon.c (mon.2) 524 : audit [DBG] from='client.? 172.21.15.6:0/2169984285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:13.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:13 smithi006 bash[27292]: cluster 2023-08-27T22:41:11.909643+0000 mgr.y (mgr.14901) 1280 : cluster [DBG] pgmap v1287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:13.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:13 smithi006 bash[27292]: audit 2023-08-27T22:41:12.964451+0000 mon.c (mon.2) 524 : audit [DBG] from='client.? 172.21.15.6:0/2169984285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:14.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:41:13.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:14.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:41:13.724Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:14 smithi130 bash[27579]: cluster 2023-08-27T22:41:13.910357+0000 mgr.y (mgr.14901) 1281 : cluster [DBG] pgmap v1288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:15.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:14 smithi006 bash[33918]: cluster 2023-08-27T22:41:13.910357+0000 mgr.y (mgr.14901) 1281 : cluster [DBG] pgmap v1288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:15.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:14 smithi006 bash[27292]: cluster 2023-08-27T22:41:13.910357+0000 mgr.y (mgr.14901) 1281 : cluster [DBG] pgmap v1288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:16.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:15 smithi130 bash[27579]: audit 2023-08-27T22:41:15.378107+0000 mon.c (mon.2) 525 : audit [DBG] from='client.? 172.21.15.6:0/1422378725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:16.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:15 smithi006 bash[27292]: audit 2023-08-27T22:41:15.378107+0000 mon.c (mon.2) 525 : audit [DBG] from='client.? 172.21.15.6:0/1422378725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:16.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:15 smithi006 bash[33918]: audit 2023-08-27T22:41:15.378107+0000 mon.c (mon.2) 525 : audit [DBG] from='client.? 172.21.15.6:0/1422378725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:16 smithi130 bash[27579]: cluster 2023-08-27T22:41:15.911508+0000 mgr.y (mgr.14901) 1282 : cluster [DBG] pgmap v1289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:17.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:16 smithi006 bash[33918]: cluster 2023-08-27T22:41:15.911508+0000 mgr.y (mgr.14901) 1282 : cluster [DBG] pgmap v1289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:17.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:16 smithi006 bash[27292]: cluster 2023-08-27T22:41:15.911508+0000 mgr.y (mgr.14901) 1282 : cluster [DBG] pgmap v1289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:18.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:17 smithi130 bash[27579]: audit 2023-08-27T22:41:17.787787+0000 mon.c (mon.2) 526 : audit [DBG] from='client.? 172.21.15.6:0/2006361478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:18.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:17 smithi006 bash[27292]: audit 2023-08-27T22:41:17.787787+0000 mon.c (mon.2) 526 : audit [DBG] from='client.? 172.21.15.6:0/2006361478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:18.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:17 smithi006 bash[33918]: audit 2023-08-27T22:41:17.787787+0000 mon.c (mon.2) 526 : audit [DBG] from='client.? 172.21.15.6:0/2006361478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:19.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:18 smithi006 bash[27292]: cluster 2023-08-27T22:41:17.912190+0000 mgr.y (mgr.14901) 1283 : cluster [DBG] pgmap v1290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:19.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:18 smithi006 bash[33918]: cluster 2023-08-27T22:41:17.912190+0000 mgr.y (mgr.14901) 1283 : cluster [DBG] pgmap v1290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:19.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:18 smithi130 bash[27579]: cluster 2023-08-27T22:41:17.912190+0000 mgr.y (mgr.14901) 1283 : cluster [DBG] pgmap v1290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:41:20.260Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:41:20.262Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:21.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:41:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:41:20] "GET /metrics HTTP/1.1" 200 193084 "" "Prometheus/2.33.4" 2023-08-27T22:41:21.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:20 smithi006 bash[27292]: cluster 2023-08-27T22:41:19.913299+0000 mgr.y (mgr.14901) 1284 : cluster [DBG] pgmap v1291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:21.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:20 smithi006 bash[27292]: audit 2023-08-27T22:41:20.206879+0000 mon.c (mon.2) 527 : audit [DBG] from='client.? 172.21.15.6:0/2881483149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:21.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:20 smithi006 bash[33918]: cluster 2023-08-27T22:41:19.913299+0000 mgr.y (mgr.14901) 1284 : cluster [DBG] pgmap v1291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:21.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:20 smithi006 bash[33918]: audit 2023-08-27T22:41:20.206879+0000 mon.c (mon.2) 527 : audit [DBG] from='client.? 172.21.15.6:0/2881483149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:21.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:20 smithi130 bash[27579]: cluster 2023-08-27T22:41:19.913299+0000 mgr.y (mgr.14901) 1284 : cluster [DBG] pgmap v1291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:21.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:20 smithi130 bash[27579]: audit 2023-08-27T22:41:20.206879+0000 mon.c (mon.2) 527 : audit [DBG] from='client.? 172.21.15.6:0/2881483149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:41:21.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:41:21.829Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:23.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:22 smithi130 bash[27579]: cluster 2023-08-27T22:41:21.914062+0000 mgr.y (mgr.14901) 1285 : cluster [DBG] pgmap v1292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:23.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:22 smithi130 bash[27579]: audit 2023-08-27T22:41:22.613621+0000 mon.a (mon.0) 1496 : audit [DBG] from='client.? 172.21.15.6:0/2601600924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:23.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:22 smithi006 bash[27292]: cluster 2023-08-27T22:41:21.914062+0000 mgr.y (mgr.14901) 1285 : cluster [DBG] pgmap v1292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:23.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:22 smithi006 bash[27292]: audit 2023-08-27T22:41:22.613621+0000 mon.a (mon.0) 1496 : audit [DBG] from='client.? 172.21.15.6:0/2601600924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:23.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:22 smithi006 bash[33918]: cluster 2023-08-27T22:41:21.914062+0000 mgr.y (mgr.14901) 1285 : cluster [DBG] pgmap v1292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:23.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:22 smithi006 bash[33918]: audit 2023-08-27T22:41:22.613621+0000 mon.a (mon.0) 1496 : audit [DBG] from='client.? 172.21.15.6:0/2601600924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:24.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:41:23.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:24.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:41:23.725Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:24 smithi130 bash[27579]: cluster 2023-08-27T22:41:23.914770+0000 mgr.y (mgr.14901) 1286 : cluster [DBG] pgmap v1293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:25.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:24 smithi006 bash[33918]: cluster 2023-08-27T22:41:23.914770+0000 mgr.y (mgr.14901) 1286 : cluster [DBG] pgmap v1293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:25.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:24 smithi006 bash[27292]: cluster 2023-08-27T22:41:23.914770+0000 mgr.y (mgr.14901) 1286 : cluster [DBG] pgmap v1293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:26.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:25 smithi130 bash[27579]: audit 2023-08-27T22:41:25.037365+0000 mon.a (mon.0) 1497 : audit [DBG] from='client.? 172.21.15.6:0/4105744626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:26.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:25 smithi006 bash[33918]: audit 2023-08-27T22:41:25.037365+0000 mon.a (mon.0) 1497 : audit [DBG] from='client.? 172.21.15.6:0/4105744626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:26.352 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:25 smithi006 bash[27292]: audit 2023-08-27T22:41:25.037365+0000 mon.a (mon.0) 1497 : audit [DBG] from='client.? 172.21.15.6:0/4105744626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:26 smithi130 bash[27579]: cluster 2023-08-27T22:41:25.915845+0000 mgr.y (mgr.14901) 1287 : cluster [DBG] pgmap v1294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:27.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:26 smithi006 bash[33918]: cluster 2023-08-27T22:41:25.915845+0000 mgr.y (mgr.14901) 1287 : cluster [DBG] pgmap v1294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:27.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:26 smithi006 bash[27292]: cluster 2023-08-27T22:41:25.915845+0000 mgr.y (mgr.14901) 1287 : cluster [DBG] pgmap v1294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:28.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:27 smithi130 bash[27579]: audit 2023-08-27T22:41:27.455855+0000 mon.c (mon.2) 528 : audit [DBG] from='client.? 172.21.15.6:0/590629043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:28.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:27 smithi006 bash[27292]: audit 2023-08-27T22:41:27.455855+0000 mon.c (mon.2) 528 : audit [DBG] from='client.? 172.21.15.6:0/590629043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:28.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:27 smithi006 bash[33918]: audit 2023-08-27T22:41:27.455855+0000 mon.c (mon.2) 528 : audit [DBG] from='client.? 172.21.15.6:0/590629043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:29.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:28 smithi006 bash[27292]: cluster 2023-08-27T22:41:27.916540+0000 mgr.y (mgr.14901) 1288 : cluster [DBG] pgmap v1295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:29.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:28 smithi006 bash[33918]: cluster 2023-08-27T22:41:27.916540+0000 mgr.y (mgr.14901) 1288 : cluster [DBG] pgmap v1295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:29.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:28 smithi130 bash[27579]: cluster 2023-08-27T22:41:27.916540+0000 mgr.y (mgr.14901) 1288 : cluster [DBG] pgmap v1295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:30.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:29 smithi006 bash[33918]: audit 2023-08-27T22:41:29.871999+0000 mon.c (mon.2) 529 : audit [DBG] from='client.? 172.21.15.6:0/3584381158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:30.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:29 smithi006 bash[27292]: audit 2023-08-27T22:41:29.871999+0000 mon.c (mon.2) 529 : audit [DBG] from='client.? 172.21.15.6:0/3584381158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:30.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:29 smithi130 bash[27579]: audit 2023-08-27T22:41:29.871999+0000 mon.c (mon.2) 529 : audit [DBG] from='client.? 172.21.15.6:0/3584381158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:41:30.261Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:41:30.263Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:31.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:41:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:41:30] "GET /metrics HTTP/1.1" 200 193083 "" "Prometheus/2.33.4" 2023-08-27T22:41:31.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:30 smithi006 bash[33918]: cluster 2023-08-27T22:41:29.917731+0000 mgr.y (mgr.14901) 1289 : cluster [DBG] pgmap v1296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:31.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:30 smithi006 bash[27292]: cluster 2023-08-27T22:41:29.917731+0000 mgr.y (mgr.14901) 1289 : cluster [DBG] pgmap v1296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:31.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:30 smithi130 bash[27579]: cluster 2023-08-27T22:41:29.917731+0000 mgr.y (mgr.14901) 1289 : cluster [DBG] pgmap v1296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:41:31.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:41:31.830Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:33.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:32 smithi130 bash[27579]: cluster 2023-08-27T22:41:31.918561+0000 mgr.y (mgr.14901) 1290 : cluster [DBG] pgmap v1297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:33.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:32 smithi130 bash[27579]: audit 2023-08-27T22:41:32.297056+0000 mon.a (mon.0) 1498 : audit [DBG] from='client.? 172.21.15.6:0/1389963138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:33.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:32 smithi006 bash[33918]: cluster 2023-08-27T22:41:31.918561+0000 mgr.y (mgr.14901) 1290 : cluster [DBG] pgmap v1297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:33.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:32 smithi006 bash[33918]: audit 2023-08-27T22:41:32.297056+0000 mon.a (mon.0) 1498 : audit [DBG] from='client.? 172.21.15.6:0/1389963138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:33.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:32 smithi006 bash[27292]: cluster 2023-08-27T22:41:31.918561+0000 mgr.y (mgr.14901) 1290 : cluster [DBG] pgmap v1297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:33.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:32 smithi006 bash[27292]: audit 2023-08-27T22:41:32.297056+0000 mon.a (mon.0) 1498 : audit [DBG] from='client.? 172.21.15.6:0/1389963138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:34.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:41:33.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:34.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:41:33.727Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:34 smithi130 bash[27579]: cluster 2023-08-27T22:41:33.919330+0000 mgr.y (mgr.14901) 1291 : cluster [DBG] pgmap v1298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:34 smithi130 bash[27579]: audit 2023-08-27T22:41:34.707957+0000 mon.a (mon.0) 1499 : audit [DBG] from='client.? 172.21.15.6:0/747447403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:35.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:34 smithi006 bash[27292]: cluster 2023-08-27T22:41:33.919330+0000 mgr.y (mgr.14901) 1291 : cluster [DBG] pgmap v1298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:35.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:34 smithi006 bash[27292]: audit 2023-08-27T22:41:34.707957+0000 mon.a (mon.0) 1499 : audit [DBG] from='client.? 172.21.15.6:0/747447403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:35.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:34 smithi006 bash[33918]: cluster 2023-08-27T22:41:33.919330+0000 mgr.y (mgr.14901) 1291 : cluster [DBG] pgmap v1298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:35.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:34 smithi006 bash[33918]: audit 2023-08-27T22:41:34.707957+0000 mon.a (mon.0) 1499 : audit [DBG] from='client.? 172.21.15.6:0/747447403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:37.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:36 smithi130 bash[27579]: cluster 2023-08-27T22:41:35.920424+0000 mgr.y (mgr.14901) 1292 : cluster [DBG] pgmap v1299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:37.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:36 smithi006 bash[27292]: cluster 2023-08-27T22:41:35.920424+0000 mgr.y (mgr.14901) 1292 : cluster [DBG] pgmap v1299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:37.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:36 smithi006 bash[33918]: cluster 2023-08-27T22:41:35.920424+0000 mgr.y (mgr.14901) 1292 : cluster [DBG] pgmap v1299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:38.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:37 smithi130 bash[27579]: audit 2023-08-27T22:41:37.118812+0000 mon.c (mon.2) 530 : audit [DBG] from='client.? 172.21.15.6:0/4287608161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:38.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:37 smithi006 bash[33918]: audit 2023-08-27T22:41:37.118812+0000 mon.c (mon.2) 530 : audit [DBG] from='client.? 172.21.15.6:0/4287608161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:38.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:37 smithi006 bash[27292]: audit 2023-08-27T22:41:37.118812+0000 mon.c (mon.2) 530 : audit [DBG] from='client.? 172.21.15.6:0/4287608161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:39.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:38 smithi130 bash[27579]: cluster 2023-08-27T22:41:37.921110+0000 mgr.y (mgr.14901) 1293 : cluster [DBG] pgmap v1300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:39.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:38 smithi006 bash[27292]: cluster 2023-08-27T22:41:37.921110+0000 mgr.y (mgr.14901) 1293 : cluster [DBG] pgmap v1300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:39.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:38 smithi006 bash[33918]: cluster 2023-08-27T22:41:37.921110+0000 mgr.y (mgr.14901) 1293 : cluster [DBG] pgmap v1300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:40.268 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:39 smithi006 bash[33918]: audit 2023-08-27T22:41:39.531450+0000 mon.c (mon.2) 531 : audit [DBG] from='client.? 172.21.15.6:0/379960433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:40.269 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:39 smithi006 bash[27292]: audit 2023-08-27T22:41:39.531450+0000 mon.c (mon.2) 531 : audit [DBG] from='client.? 172.21.15.6:0/379960433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:40.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:40 smithi130 bash[27579]: audit 2023-08-27T22:41:39.531450+0000 mon.c (mon.2) 531 : audit [DBG] from='client.? 172.21.15.6:0/379960433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:41:40.261Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:41:40.263Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:41.007 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:41:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:41:40] "GET /metrics HTTP/1.1" 200 193083 "" "Prometheus/2.33.4" 2023-08-27T22:41:41.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:41 smithi130 bash[27579]: cluster 2023-08-27T22:41:39.922239+0000 mgr.y (mgr.14901) 1294 : cluster [DBG] pgmap v1301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:41.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:41 smithi006 bash[33918]: cluster 2023-08-27T22:41:39.922239+0000 mgr.y (mgr.14901) 1294 : cluster [DBG] pgmap v1301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:41.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:41 smithi006 bash[27292]: cluster 2023-08-27T22:41:39.922239+0000 mgr.y (mgr.14901) 1294 : cluster [DBG] pgmap v1301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:42.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:42 smithi006 bash[33918]: audit 2023-08-27T22:41:41.943280+0000 mon.c (mon.2) 532 : audit [DBG] from='client.? 172.21.15.6:0/2274754619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:42.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:42 smithi006 bash[27292]: audit 2023-08-27T22:41:41.943280+0000 mon.c (mon.2) 532 : audit [DBG] from='client.? 172.21.15.6:0/2274754619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:41:41.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:42.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:41:41.830Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:42.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:42 smithi130 bash[27579]: audit 2023-08-27T22:41:41.943280+0000 mon.c (mon.2) 532 : audit [DBG] from='client.? 172.21.15.6:0/2274754619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:43.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:43 smithi130 bash[27579]: cluster 2023-08-27T22:41:41.922850+0000 mgr.y (mgr.14901) 1295 : cluster [DBG] pgmap v1302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:43.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:43 smithi006 bash[33918]: cluster 2023-08-27T22:41:41.922850+0000 mgr.y (mgr.14901) 1295 : cluster [DBG] pgmap v1302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:43.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:43 smithi006 bash[27292]: cluster 2023-08-27T22:41:41.922850+0000 mgr.y (mgr.14901) 1295 : cluster [DBG] pgmap v1302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:44.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:41:43.725Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:44.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:41:43.728Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:44 smithi130 bash[27579]: cluster 2023-08-27T22:41:43.923513+0000 mgr.y (mgr.14901) 1296 : cluster [DBG] pgmap v1303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:44 smithi130 bash[27579]: audit 2023-08-27T22:41:44.359422+0000 mon.a (mon.0) 1500 : audit [DBG] from='client.? 172.21.15.6:0/3112201540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:45.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:44 smithi006 bash[33918]: cluster 2023-08-27T22:41:43.923513+0000 mgr.y (mgr.14901) 1296 : cluster [DBG] pgmap v1303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:45.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:44 smithi006 bash[33918]: audit 2023-08-27T22:41:44.359422+0000 mon.a (mon.0) 1500 : audit [DBG] from='client.? 172.21.15.6:0/3112201540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:45.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:44 smithi006 bash[27292]: cluster 2023-08-27T22:41:43.923513+0000 mgr.y (mgr.14901) 1296 : cluster [DBG] pgmap v1303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:45.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:44 smithi006 bash[27292]: audit 2023-08-27T22:41:44.359422+0000 mon.a (mon.0) 1500 : audit [DBG] from='client.? 172.21.15.6:0/3112201540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:47.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:46 smithi130 bash[27579]: cluster 2023-08-27T22:41:45.924687+0000 mgr.y (mgr.14901) 1297 : cluster [DBG] pgmap v1304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:47.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:46 smithi130 bash[27579]: audit 2023-08-27T22:41:46.772171+0000 mon.c (mon.2) 533 : audit [DBG] from='client.? 172.21.15.6:0/3675510543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:47.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:46 smithi006 bash[33918]: cluster 2023-08-27T22:41:45.924687+0000 mgr.y (mgr.14901) 1297 : cluster [DBG] pgmap v1304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:47.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:46 smithi006 bash[33918]: audit 2023-08-27T22:41:46.772171+0000 mon.c (mon.2) 533 : audit [DBG] from='client.? 172.21.15.6:0/3675510543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:47.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:46 smithi006 bash[27292]: cluster 2023-08-27T22:41:45.924687+0000 mgr.y (mgr.14901) 1297 : cluster [DBG] pgmap v1304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:47.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:46 smithi006 bash[27292]: audit 2023-08-27T22:41:46.772171+0000 mon.c (mon.2) 533 : audit [DBG] from='client.? 172.21.15.6:0/3675510543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:49.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:48 smithi130 bash[27579]: cluster 2023-08-27T22:41:47.925354+0000 mgr.y (mgr.14901) 1298 : cluster [DBG] pgmap v1305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:49.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:48 smithi006 bash[27292]: cluster 2023-08-27T22:41:47.925354+0000 mgr.y (mgr.14901) 1298 : cluster [DBG] pgmap v1305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:49.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:48 smithi006 bash[33918]: cluster 2023-08-27T22:41:47.925354+0000 mgr.y (mgr.14901) 1298 : cluster [DBG] pgmap v1305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:50.270 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:49 smithi006 bash[33918]: audit 2023-08-27T22:41:49.195538+0000 mon.c (mon.2) 534 : audit [DBG] from='client.? 172.21.15.6:0/1715091164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:50.270 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:49 smithi006 bash[27292]: audit 2023-08-27T22:41:49.195538+0000 mon.c (mon.2) 534 : audit [DBG] from='client.? 172.21.15.6:0/1715091164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:50.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:50 smithi130 bash[27579]: audit 2023-08-27T22:41:49.195538+0000 mon.c (mon.2) 534 : audit [DBG] from='client.? 172.21.15.6:0/1715091164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:41:50.262Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:41:50.265Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:51.008 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:41:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:41:50] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T22:41:51.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:51 smithi130 bash[27579]: cluster 2023-08-27T22:41:49.926587+0000 mgr.y (mgr.14901) 1299 : cluster [DBG] pgmap v1306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:51.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:51 smithi006 bash[27292]: cluster 2023-08-27T22:41:49.926587+0000 mgr.y (mgr.14901) 1299 : cluster [DBG] pgmap v1306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:51.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:51 smithi006 bash[33918]: cluster 2023-08-27T22:41:49.926587+0000 mgr.y (mgr.14901) 1299 : cluster [DBG] pgmap v1306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:52.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:52 smithi006 bash[33918]: audit 2023-08-27T22:41:51.607844+0000 mon.a (mon.0) 1501 : audit [DBG] from='client.? 172.21.15.6:0/3703623654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:52.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:52 smithi006 bash[27292]: audit 2023-08-27T22:41:51.607844+0000 mon.a (mon.0) 1501 : audit [DBG] from='client.? 172.21.15.6:0/3703623654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:41:51.829Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:41:51.830Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:52.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:52 smithi130 bash[27579]: audit 2023-08-27T22:41:51.607844+0000 mon.a (mon.0) 1501 : audit [DBG] from='client.? 172.21.15.6:0/3703623654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:53.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:53 smithi130 bash[27579]: cluster 2023-08-27T22:41:51.927291+0000 mgr.y (mgr.14901) 1300 : cluster [DBG] pgmap v1307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:53.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:53 smithi006 bash[33918]: cluster 2023-08-27T22:41:51.927291+0000 mgr.y (mgr.14901) 1300 : cluster [DBG] pgmap v1307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:53.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:53 smithi006 bash[27292]: cluster 2023-08-27T22:41:51.927291+0000 mgr.y (mgr.14901) 1300 : cluster [DBG] pgmap v1307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:54.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:41:53.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:54.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:41:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:41:53.728Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:41:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:54 smithi130 bash[27579]: cluster 2023-08-27T22:41:53.927989+0000 mgr.y (mgr.14901) 1301 : cluster [DBG] pgmap v1308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:54 smithi130 bash[27579]: audit 2023-08-27T22:41:54.022909+0000 mon.c (mon.2) 535 : audit [DBG] from='client.? 172.21.15.6:0/678295578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:55.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:54 smithi006 bash[33918]: cluster 2023-08-27T22:41:53.927989+0000 mgr.y (mgr.14901) 1301 : cluster [DBG] pgmap v1308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:55.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:54 smithi006 bash[33918]: audit 2023-08-27T22:41:54.022909+0000 mon.c (mon.2) 535 : audit [DBG] from='client.? 172.21.15.6:0/678295578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:55.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:54 smithi006 bash[27292]: cluster 2023-08-27T22:41:53.927989+0000 mgr.y (mgr.14901) 1301 : cluster [DBG] pgmap v1308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:55.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:54 smithi006 bash[27292]: audit 2023-08-27T22:41:54.022909+0000 mon.c (mon.2) 535 : audit [DBG] from='client.? 172.21.15.6:0/678295578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:57.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:56 smithi130 bash[27579]: cluster 2023-08-27T22:41:55.929208+0000 mgr.y (mgr.14901) 1302 : cluster [DBG] pgmap v1309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:57.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:56 smithi130 bash[27579]: audit 2023-08-27T22:41:56.433288+0000 mon.a (mon.0) 1502 : audit [DBG] from='client.? 172.21.15.6:0/527322614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:57.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:56 smithi006 bash[27292]: cluster 2023-08-27T22:41:55.929208+0000 mgr.y (mgr.14901) 1302 : cluster [DBG] pgmap v1309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:57.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:56 smithi006 bash[27292]: audit 2023-08-27T22:41:56.433288+0000 mon.a (mon.0) 1502 : audit [DBG] from='client.? 172.21.15.6:0/527322614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:57.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:56 smithi006 bash[33918]: cluster 2023-08-27T22:41:55.929208+0000 mgr.y (mgr.14901) 1302 : cluster [DBG] pgmap v1309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:57.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:56 smithi006 bash[33918]: audit 2023-08-27T22:41:56.433288+0000 mon.a (mon.0) 1502 : audit [DBG] from='client.? 172.21.15.6:0/527322614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:59.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:58 smithi130 bash[27579]: cluster 2023-08-27T22:41:57.929954+0000 mgr.y (mgr.14901) 1303 : cluster [DBG] pgmap v1310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:59.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:41:58 smithi130 bash[27579]: audit 2023-08-27T22:41:58.847666+0000 mon.c (mon.2) 536 : audit [DBG] from='client.? 172.21.15.6:0/1207600810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:59.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:58 smithi006 bash[33918]: cluster 2023-08-27T22:41:57.929954+0000 mgr.y (mgr.14901) 1303 : cluster [DBG] pgmap v1310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:59.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:41:58 smithi006 bash[33918]: audit 2023-08-27T22:41:58.847666+0000 mon.c (mon.2) 536 : audit [DBG] from='client.? 172.21.15.6:0/1207600810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:59.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:58 smithi006 bash[27292]: cluster 2023-08-27T22:41:57.929954+0000 mgr.y (mgr.14901) 1303 : cluster [DBG] pgmap v1310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:41:59.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:41:58 smithi006 bash[27292]: audit 2023-08-27T22:41:58.847666+0000 mon.c (mon.2) 536 : audit [DBG] from='client.? 172.21.15.6:0/1207600810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:00.270 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:00 smithi006 bash[33918]: audit 2023-08-27T22:41:59.463154+0000 mon.b (mon.1) 248 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:42:00.270 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:00 smithi006 bash[27292]: audit 2023-08-27T22:41:59.463154+0000 mon.b (mon.1) 248 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:42:00.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:00 smithi130 bash[27579]: audit 2023-08-27T22:41:59.463154+0000 mon.b (mon.1) 248 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:42:00.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:42:00.263Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:00.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:42:00.265Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:01.013 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:42:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:42:00] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:42:01.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:01 smithi130 bash[27579]: cluster 2023-08-27T22:41:59.931063+0000 mgr.y (mgr.14901) 1304 : cluster [DBG] pgmap v1311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:01.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:01 smithi006 bash[27292]: cluster 2023-08-27T22:41:59.931063+0000 mgr.y (mgr.14901) 1304 : cluster [DBG] pgmap v1311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:01.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:01 smithi006 bash[33918]: cluster 2023-08-27T22:41:59.931063+0000 mgr.y (mgr.14901) 1304 : cluster [DBG] pgmap v1311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:02.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:02 smithi006 bash[33918]: audit 2023-08-27T22:42:01.267701+0000 mon.c (mon.2) 537 : audit [DBG] from='client.? 172.21.15.6:0/1049438676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:02.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:02 smithi006 bash[27292]: audit 2023-08-27T22:42:01.267701+0000 mon.c (mon.2) 537 : audit [DBG] from='client.? 172.21.15.6:0/1049438676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:42:01.829Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:42:01.831Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:02.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:02 smithi130 bash[27579]: audit 2023-08-27T22:42:01.267701+0000 mon.c (mon.2) 537 : audit [DBG] from='client.? 172.21.15.6:0/1049438676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:03.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:03 smithi130 bash[27579]: cluster 2023-08-27T22:42:01.931779+0000 mgr.y (mgr.14901) 1305 : cluster [DBG] pgmap v1312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:03.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:03 smithi006 bash[27292]: cluster 2023-08-27T22:42:01.931779+0000 mgr.y (mgr.14901) 1305 : cluster [DBG] pgmap v1312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:03.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:03 smithi006 bash[33918]: cluster 2023-08-27T22:42:01.931779+0000 mgr.y (mgr.14901) 1305 : cluster [DBG] pgmap v1312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:04.027 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:42:03.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:04.027 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:42:03.728Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:04.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:04 smithi130 bash[27579]: audit 2023-08-27T22:42:03.679962+0000 mon.a (mon.0) 1503 : audit [DBG] from='client.? 172.21.15.6:0/272339569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:04.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:04 smithi006 bash[33918]: audit 2023-08-27T22:42:03.679962+0000 mon.a (mon.0) 1503 : audit [DBG] from='client.? 172.21.15.6:0/272339569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:04.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:04 smithi006 bash[27292]: audit 2023-08-27T22:42:03.679962+0000 mon.a (mon.0) 1503 : audit [DBG] from='client.? 172.21.15.6:0/272339569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:05 smithi130 bash[27579]: cluster 2023-08-27T22:42:03.932411+0000 mgr.y (mgr.14901) 1306 : cluster [DBG] pgmap v1313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:05.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:05 smithi006 bash[27292]: cluster 2023-08-27T22:42:03.932411+0000 mgr.y (mgr.14901) 1306 : cluster [DBG] pgmap v1313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:05.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:05 smithi006 bash[33918]: cluster 2023-08-27T22:42:03.932411+0000 mgr.y (mgr.14901) 1306 : cluster [DBG] pgmap v1313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:06.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:06 smithi130 bash[27579]: audit 2023-08-27T22:42:05.343178+0000 mon.a (mon.0) 1504 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:42:06.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:06 smithi130 bash[27579]: audit 2023-08-27T22:42:05.772876+0000 mon.a (mon.0) 1505 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:42:06.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:06 smithi130 bash[27579]: cluster 2023-08-27T22:42:05.933164+0000 mgr.y (mgr.14901) 1307 : cluster [DBG] pgmap v1314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:06.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:06 smithi130 bash[27579]: audit 2023-08-27T22:42:06.049042+0000 mon.b (mon.1) 249 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:42:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:06 smithi130 bash[27579]: audit 2023-08-27T22:42:06.050643+0000 mon.b (mon.1) 250 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:42:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:06 smithi130 bash[27579]: audit 2023-08-27T22:42:06.052208+0000 mon.a (mon.0) 1506 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:42:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:06 smithi130 bash[27579]: audit 2023-08-27T22:42:06.117867+0000 mon.c (mon.2) 538 : audit [DBG] from='client.? 172.21.15.6:0/2449111061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:06.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:06 smithi006 bash[27292]: audit 2023-08-27T22:42:05.343178+0000 mon.a (mon.0) 1504 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:42:06.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:06 smithi006 bash[27292]: audit 2023-08-27T22:42:05.772876+0000 mon.a (mon.0) 1505 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:42:06.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:06 smithi006 bash[27292]: cluster 2023-08-27T22:42:05.933164+0000 mgr.y (mgr.14901) 1307 : cluster [DBG] pgmap v1314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:06.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:06 smithi006 bash[27292]: audit 2023-08-27T22:42:06.049042+0000 mon.b (mon.1) 249 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:42:06.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:06 smithi006 bash[27292]: audit 2023-08-27T22:42:06.050643+0000 mon.b (mon.1) 250 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:42:06.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:06 smithi006 bash[27292]: audit 2023-08-27T22:42:06.052208+0000 mon.a (mon.0) 1506 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:42:06.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:06 smithi006 bash[27292]: audit 2023-08-27T22:42:06.117867+0000 mon.c (mon.2) 538 : audit [DBG] from='client.? 172.21.15.6:0/2449111061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:06.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:06 smithi006 bash[33918]: audit 2023-08-27T22:42:05.343178+0000 mon.a (mon.0) 1504 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:42:06.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:06 smithi006 bash[33918]: audit 2023-08-27T22:42:05.772876+0000 mon.a (mon.0) 1505 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:42:06.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:06 smithi006 bash[33918]: cluster 2023-08-27T22:42:05.933164+0000 mgr.y (mgr.14901) 1307 : cluster [DBG] pgmap v1314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:06.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:06 smithi006 bash[33918]: audit 2023-08-27T22:42:06.049042+0000 mon.b (mon.1) 249 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:42:06.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:06 smithi006 bash[33918]: audit 2023-08-27T22:42:06.050643+0000 mon.b (mon.1) 250 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:42:06.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:06 smithi006 bash[33918]: audit 2023-08-27T22:42:06.052208+0000 mon.a (mon.0) 1506 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:42:06.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:06 smithi006 bash[33918]: audit 2023-08-27T22:42:06.117867+0000 mon.c (mon.2) 538 : audit [DBG] from='client.? 172.21.15.6:0/2449111061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:09.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:09 smithi130 bash[27579]: cluster 2023-08-27T22:42:07.933773+0000 mgr.y (mgr.14901) 1308 : cluster [DBG] pgmap v1315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:09.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:09 smithi130 bash[27579]: audit 2023-08-27T22:42:08.532938+0000 mon.a (mon.0) 1507 : audit [DBG] from='client.? 172.21.15.6:0/496017218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:09.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:08 smithi006 bash[33918]: cluster 2023-08-27T22:42:07.933773+0000 mgr.y (mgr.14901) 1308 : cluster [DBG] pgmap v1315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:09.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:08 smithi006 bash[33918]: audit 2023-08-27T22:42:08.532938+0000 mon.a (mon.0) 1507 : audit [DBG] from='client.? 172.21.15.6:0/496017218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:09.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:08 smithi006 bash[27292]: cluster 2023-08-27T22:42:07.933773+0000 mgr.y (mgr.14901) 1308 : cluster [DBG] pgmap v1315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:09.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:08 smithi006 bash[27292]: audit 2023-08-27T22:42:08.532938+0000 mon.a (mon.0) 1507 : audit [DBG] from='client.? 172.21.15.6:0/496017218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:42:10.264Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:42:10.266Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:11.005 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:42:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:42:10] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:42:11.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:11 smithi130 bash[27579]: cluster 2023-08-27T22:42:09.934864+0000 mgr.y (mgr.14901) 1309 : cluster [DBG] pgmap v1316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:11.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:11 smithi130 bash[27579]: audit 2023-08-27T22:42:10.954630+0000 mon.a (mon.0) 1508 : audit [DBG] from='client.? 172.21.15.6:0/2292614650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:11.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:10 smithi006 bash[33918]: cluster 2023-08-27T22:42:09.934864+0000 mgr.y (mgr.14901) 1309 : cluster [DBG] pgmap v1316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:11.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:10 smithi006 bash[33918]: audit 2023-08-27T22:42:10.954630+0000 mon.a (mon.0) 1508 : audit [DBG] from='client.? 172.21.15.6:0/2292614650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:11.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:10 smithi006 bash[27292]: cluster 2023-08-27T22:42:09.934864+0000 mgr.y (mgr.14901) 1309 : cluster [DBG] pgmap v1316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:11.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:10 smithi006 bash[27292]: audit 2023-08-27T22:42:10.954630+0000 mon.a (mon.0) 1508 : audit [DBG] from='client.? 172.21.15.6:0/2292614650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:42:11.830Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:42:11.832Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:13.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:13 smithi130 bash[27579]: cluster 2023-08-27T22:42:11.935533+0000 mgr.y (mgr.14901) 1310 : cluster [DBG] pgmap v1317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:13.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:13 smithi006 bash[33918]: cluster 2023-08-27T22:42:11.935533+0000 mgr.y (mgr.14901) 1310 : cluster [DBG] pgmap v1317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:13.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:13 smithi006 bash[27292]: cluster 2023-08-27T22:42:11.935533+0000 mgr.y (mgr.14901) 1310 : cluster [DBG] pgmap v1317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:14.013 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:42:13.727Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:14.013 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:42:13.729Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:14.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:14 smithi130 bash[27579]: audit 2023-08-27T22:42:13.379550+0000 mon.a (mon.0) 1509 : audit [DBG] from='client.? 172.21.15.6:0/836396850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:14.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:14 smithi006 bash[27292]: audit 2023-08-27T22:42:13.379550+0000 mon.a (mon.0) 1509 : audit [DBG] from='client.? 172.21.15.6:0/836396850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:14.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:14 smithi006 bash[33918]: audit 2023-08-27T22:42:13.379550+0000 mon.a (mon.0) 1509 : audit [DBG] from='client.? 172.21.15.6:0/836396850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:15.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:15 smithi130 bash[27579]: cluster 2023-08-27T22:42:13.936214+0000 mgr.y (mgr.14901) 1311 : cluster [DBG] pgmap v1318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:15.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:15 smithi006 bash[27292]: cluster 2023-08-27T22:42:13.936214+0000 mgr.y (mgr.14901) 1311 : cluster [DBG] pgmap v1318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:15.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:15 smithi006 bash[33918]: cluster 2023-08-27T22:42:13.936214+0000 mgr.y (mgr.14901) 1311 : cluster [DBG] pgmap v1318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:16.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:16 smithi130 bash[27579]: audit 2023-08-27T22:42:15.794640+0000 mon.c (mon.2) 539 : audit [DBG] from='client.? 172.21.15.6:0/2937366642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:16.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:16 smithi006 bash[27292]: audit 2023-08-27T22:42:15.794640+0000 mon.c (mon.2) 539 : audit [DBG] from='client.? 172.21.15.6:0/2937366642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:16.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:16 smithi006 bash[33918]: audit 2023-08-27T22:42:15.794640+0000 mon.c (mon.2) 539 : audit [DBG] from='client.? 172.21.15.6:0/2937366642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:17.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:17 smithi130 bash[27579]: cluster 2023-08-27T22:42:15.937349+0000 mgr.y (mgr.14901) 1312 : cluster [DBG] pgmap v1319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:17.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:17 smithi006 bash[27292]: cluster 2023-08-27T22:42:15.937349+0000 mgr.y (mgr.14901) 1312 : cluster [DBG] pgmap v1319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:17.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:17 smithi006 bash[33918]: cluster 2023-08-27T22:42:15.937349+0000 mgr.y (mgr.14901) 1312 : cluster [DBG] pgmap v1319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:19.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:19 smithi130 bash[27579]: cluster 2023-08-27T22:42:17.937904+0000 mgr.y (mgr.14901) 1313 : cluster [DBG] pgmap v1320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:19.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:19 smithi130 bash[27579]: audit 2023-08-27T22:42:18.212787+0000 mon.a (mon.0) 1510 : audit [DBG] from='client.? 172.21.15.6:0/2902489028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:19.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:19 smithi006 bash[27292]: cluster 2023-08-27T22:42:17.937904+0000 mgr.y (mgr.14901) 1313 : cluster [DBG] pgmap v1320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:19.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:19 smithi006 bash[27292]: audit 2023-08-27T22:42:18.212787+0000 mon.a (mon.0) 1510 : audit [DBG] from='client.? 172.21.15.6:0/2902489028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:19.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:19 smithi006 bash[33918]: cluster 2023-08-27T22:42:17.937904+0000 mgr.y (mgr.14901) 1313 : cluster [DBG] pgmap v1320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:19.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:19 smithi006 bash[33918]: audit 2023-08-27T22:42:18.212787+0000 mon.a (mon.0) 1510 : audit [DBG] from='client.? 172.21.15.6:0/2902489028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:42:20.264Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:20.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:42:20.266Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:21.039 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:42:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:42:20] "GET /metrics HTTP/1.1" 200 193084 "" "Prometheus/2.33.4" 2023-08-27T22:42:21.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:21 smithi130 bash[27579]: cluster 2023-08-27T22:42:19.938981+0000 mgr.y (mgr.14901) 1314 : cluster [DBG] pgmap v1321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:21.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:21 smithi130 bash[27579]: audit 2023-08-27T22:42:20.627858+0000 mon.a (mon.0) 1511 : audit [DBG] from='client.? 172.21.15.6:0/3087942006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:21.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:21 smithi006 bash[33918]: cluster 2023-08-27T22:42:19.938981+0000 mgr.y (mgr.14901) 1314 : cluster [DBG] pgmap v1321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:21.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:21 smithi006 bash[33918]: audit 2023-08-27T22:42:20.627858+0000 mon.a (mon.0) 1511 : audit [DBG] from='client.? 172.21.15.6:0/3087942006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:21.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:21 smithi006 bash[27292]: cluster 2023-08-27T22:42:19.938981+0000 mgr.y (mgr.14901) 1314 : cluster [DBG] pgmap v1321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:21.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:21 smithi006 bash[27292]: audit 2023-08-27T22:42:20.627858+0000 mon.a (mon.0) 1511 : audit [DBG] from='client.? 172.21.15.6:0/3087942006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:42:21.830Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:42:21.832Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:23.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:23 smithi130 bash[27579]: cluster 2023-08-27T22:42:21.939629+0000 mgr.y (mgr.14901) 1315 : cluster [DBG] pgmap v1322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:23.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:23 smithi006 bash[27292]: cluster 2023-08-27T22:42:21.939629+0000 mgr.y (mgr.14901) 1315 : cluster [DBG] pgmap v1322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:23.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:23 smithi006 bash[33918]: cluster 2023-08-27T22:42:21.939629+0000 mgr.y (mgr.14901) 1315 : cluster [DBG] pgmap v1322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:24.047 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:42:23.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:24.047 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:42:23.730Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:24.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:24 smithi130 bash[27579]: audit 2023-08-27T22:42:23.036945+0000 mon.c (mon.2) 540 : audit [DBG] from='client.? 172.21.15.6:0/280285077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:24.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:24 smithi006 bash[27292]: audit 2023-08-27T22:42:23.036945+0000 mon.c (mon.2) 540 : audit [DBG] from='client.? 172.21.15.6:0/280285077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:24.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:24 smithi006 bash[33918]: audit 2023-08-27T22:42:23.036945+0000 mon.c (mon.2) 540 : audit [DBG] from='client.? 172.21.15.6:0/280285077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:25.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:25 smithi130 bash[27579]: cluster 2023-08-27T22:42:23.940348+0000 mgr.y (mgr.14901) 1316 : cluster [DBG] pgmap v1323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:25.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:25 smithi006 bash[27292]: cluster 2023-08-27T22:42:23.940348+0000 mgr.y (mgr.14901) 1316 : cluster [DBG] pgmap v1323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:25.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:25 smithi006 bash[33918]: cluster 2023-08-27T22:42:23.940348+0000 mgr.y (mgr.14901) 1316 : cluster [DBG] pgmap v1323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:26.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:26 smithi130 bash[27579]: audit 2023-08-27T22:42:25.441256+0000 mon.a (mon.0) 1512 : audit [DBG] from='client.? 172.21.15.6:0/2201772688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:26.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:26 smithi006 bash[33918]: audit 2023-08-27T22:42:25.441256+0000 mon.a (mon.0) 1512 : audit [DBG] from='client.? 172.21.15.6:0/2201772688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:26.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:26 smithi006 bash[27292]: audit 2023-08-27T22:42:25.441256+0000 mon.a (mon.0) 1512 : audit [DBG] from='client.? 172.21.15.6:0/2201772688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:27.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:27 smithi130 bash[27579]: cluster 2023-08-27T22:42:25.941505+0000 mgr.y (mgr.14901) 1317 : cluster [DBG] pgmap v1324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:27.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:27 smithi006 bash[27292]: cluster 2023-08-27T22:42:25.941505+0000 mgr.y (mgr.14901) 1317 : cluster [DBG] pgmap v1324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:27.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:27 smithi006 bash[33918]: cluster 2023-08-27T22:42:25.941505+0000 mgr.y (mgr.14901) 1317 : cluster [DBG] pgmap v1324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:28.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:28 smithi130 bash[27579]: audit 2023-08-27T22:42:27.862960+0000 mon.c (mon.2) 541 : audit [DBG] from='client.? 172.21.15.6:0/249049156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:28.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:28 smithi006 bash[27292]: audit 2023-08-27T22:42:27.862960+0000 mon.c (mon.2) 541 : audit [DBG] from='client.? 172.21.15.6:0/249049156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:28.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:28 smithi006 bash[33918]: audit 2023-08-27T22:42:27.862960+0000 mon.c (mon.2) 541 : audit [DBG] from='client.? 172.21.15.6:0/249049156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:29.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:29 smithi130 bash[27579]: cluster 2023-08-27T22:42:27.942310+0000 mgr.y (mgr.14901) 1318 : cluster [DBG] pgmap v1325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:29.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:29 smithi006 bash[33918]: cluster 2023-08-27T22:42:27.942310+0000 mgr.y (mgr.14901) 1318 : cluster [DBG] pgmap v1325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:29.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:29 smithi006 bash[27292]: cluster 2023-08-27T22:42:27.942310+0000 mgr.y (mgr.14901) 1318 : cluster [DBG] pgmap v1325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:30.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:42:30.266Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:30.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:42:30.267Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:31.076 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:42:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:42:30] "GET /metrics HTTP/1.1" 200 193081 "" "Prometheus/2.33.4" 2023-08-27T22:42:31.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:31 smithi130 bash[27579]: cluster 2023-08-27T22:42:29.943446+0000 mgr.y (mgr.14901) 1319 : cluster [DBG] pgmap v1326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:31.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:31 smithi130 bash[27579]: audit 2023-08-27T22:42:30.284167+0000 mon.a (mon.0) 1513 : audit [DBG] from='client.? 172.21.15.6:0/1551225023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:31.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:31 smithi006 bash[27292]: cluster 2023-08-27T22:42:29.943446+0000 mgr.y (mgr.14901) 1319 : cluster [DBG] pgmap v1326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:31.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:31 smithi006 bash[27292]: audit 2023-08-27T22:42:30.284167+0000 mon.a (mon.0) 1513 : audit [DBG] from='client.? 172.21.15.6:0/1551225023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:31.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:31 smithi006 bash[33918]: cluster 2023-08-27T22:42:29.943446+0000 mgr.y (mgr.14901) 1319 : cluster [DBG] pgmap v1326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:31.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:31 smithi006 bash[33918]: audit 2023-08-27T22:42:30.284167+0000 mon.a (mon.0) 1513 : audit [DBG] from='client.? 172.21.15.6:0/1551225023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:42:31.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:42:31.833Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:33.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:33 smithi130 bash[27579]: cluster 2023-08-27T22:42:31.944156+0000 mgr.y (mgr.14901) 1320 : cluster [DBG] pgmap v1327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:33.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:33 smithi130 bash[27579]: audit 2023-08-27T22:42:32.704589+0000 mon.a (mon.0) 1514 : audit [DBG] from='client.? 172.21.15.6:0/3700140369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:33.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:33 smithi006 bash[27292]: cluster 2023-08-27T22:42:31.944156+0000 mgr.y (mgr.14901) 1320 : cluster [DBG] pgmap v1327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:33.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:33 smithi006 bash[27292]: audit 2023-08-27T22:42:32.704589+0000 mon.a (mon.0) 1514 : audit [DBG] from='client.? 172.21.15.6:0/3700140369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:33.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:33 smithi006 bash[33918]: cluster 2023-08-27T22:42:31.944156+0000 mgr.y (mgr.14901) 1320 : cluster [DBG] pgmap v1327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:33.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:33 smithi006 bash[33918]: audit 2023-08-27T22:42:32.704589+0000 mon.a (mon.0) 1514 : audit [DBG] from='client.? 172.21.15.6:0/3700140369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:34.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:42:33.730Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:34.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:42:33.732Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:35.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:34 smithi006 bash[33918]: cluster 2023-08-27T22:42:33.944977+0000 mgr.y (mgr.14901) 1321 : cluster [DBG] pgmap v1328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:35.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:34 smithi006 bash[27292]: cluster 2023-08-27T22:42:33.944977+0000 mgr.y (mgr.14901) 1321 : cluster [DBG] pgmap v1328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:35.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:34 smithi130 bash[27579]: cluster 2023-08-27T22:42:33.944977+0000 mgr.y (mgr.14901) 1321 : cluster [DBG] pgmap v1328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:36.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:35 smithi006 bash[33918]: audit 2023-08-27T22:42:35.122482+0000 mon.a (mon.0) 1515 : audit [DBG] from='client.? 172.21.15.6:0/871564616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:36.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:35 smithi006 bash[27292]: audit 2023-08-27T22:42:35.122482+0000 mon.a (mon.0) 1515 : audit [DBG] from='client.? 172.21.15.6:0/871564616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:36.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:35 smithi130 bash[27579]: audit 2023-08-27T22:42:35.122482+0000 mon.a (mon.0) 1515 : audit [DBG] from='client.? 172.21.15.6:0/871564616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:37.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:36 smithi006 bash[27292]: cluster 2023-08-27T22:42:35.946196+0000 mgr.y (mgr.14901) 1322 : cluster [DBG] pgmap v1329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:37.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:36 smithi006 bash[33918]: cluster 2023-08-27T22:42:35.946196+0000 mgr.y (mgr.14901) 1322 : cluster [DBG] pgmap v1329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:37.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:36 smithi130 bash[27579]: cluster 2023-08-27T22:42:35.946196+0000 mgr.y (mgr.14901) 1322 : cluster [DBG] pgmap v1329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:38.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:37 smithi006 bash[33918]: audit 2023-08-27T22:42:37.535516+0000 mon.c (mon.2) 542 : audit [DBG] from='client.? 172.21.15.6:0/29204114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:38.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:37 smithi006 bash[27292]: audit 2023-08-27T22:42:37.535516+0000 mon.c (mon.2) 542 : audit [DBG] from='client.? 172.21.15.6:0/29204114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:38.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:37 smithi130 bash[27579]: audit 2023-08-27T22:42:37.535516+0000 mon.c (mon.2) 542 : audit [DBG] from='client.? 172.21.15.6:0/29204114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:39.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:38 smithi006 bash[33918]: cluster 2023-08-27T22:42:37.947001+0000 mgr.y (mgr.14901) 1323 : cluster [DBG] pgmap v1330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:39.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:38 smithi006 bash[27292]: cluster 2023-08-27T22:42:37.947001+0000 mgr.y (mgr.14901) 1323 : cluster [DBG] pgmap v1330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:39.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:38 smithi130 bash[27579]: cluster 2023-08-27T22:42:37.947001+0000 mgr.y (mgr.14901) 1323 : cluster [DBG] pgmap v1330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:40.274 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:40 smithi006 bash[27292]: audit 2023-08-27T22:42:39.947900+0000 mon.a (mon.0) 1516 : audit [DBG] from='client.? 172.21.15.6:0/3683466079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:40.274 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:40 smithi006 bash[33918]: audit 2023-08-27T22:42:39.947900+0000 mon.a (mon.0) 1516 : audit [DBG] from='client.? 172.21.15.6:0/3683466079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:40.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:40 smithi130 bash[27579]: audit 2023-08-27T22:42:39.947900+0000 mon.a (mon.0) 1516 : audit [DBG] from='client.? 172.21.15.6:0/3683466079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:42:40.266Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:42:40.268Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:41.017 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:42:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:42:40] "GET /metrics HTTP/1.1" 200 193081 "" "Prometheus/2.33.4" 2023-08-27T22:42:41.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:41 smithi130 bash[27579]: cluster 2023-08-27T22:42:39.948335+0000 mgr.y (mgr.14901) 1324 : cluster [DBG] pgmap v1331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:41.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:41 smithi006 bash[27292]: cluster 2023-08-27T22:42:39.948335+0000 mgr.y (mgr.14901) 1324 : cluster [DBG] pgmap v1331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:41.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:41 smithi006 bash[33918]: cluster 2023-08-27T22:42:39.948335+0000 mgr.y (mgr.14901) 1324 : cluster [DBG] pgmap v1331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:42:41.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:42:41.834Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:43.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:43 smithi130 bash[27579]: cluster 2023-08-27T22:42:41.949133+0000 mgr.y (mgr.14901) 1325 : cluster [DBG] pgmap v1332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:43.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:43 smithi130 bash[27579]: audit 2023-08-27T22:42:42.361610+0000 mon.a (mon.0) 1517 : audit [DBG] from='client.? 172.21.15.6:0/3794213522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:43.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:43 smithi006 bash[27292]: cluster 2023-08-27T22:42:41.949133+0000 mgr.y (mgr.14901) 1325 : cluster [DBG] pgmap v1332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:43.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:43 smithi006 bash[27292]: audit 2023-08-27T22:42:42.361610+0000 mon.a (mon.0) 1517 : audit [DBG] from='client.? 172.21.15.6:0/3794213522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:43.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:43 smithi006 bash[33918]: cluster 2023-08-27T22:42:41.949133+0000 mgr.y (mgr.14901) 1325 : cluster [DBG] pgmap v1332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:43.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:43 smithi006 bash[33918]: audit 2023-08-27T22:42:42.361610+0000 mon.a (mon.0) 1517 : audit [DBG] from='client.? 172.21.15.6:0/3794213522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:44.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:42:43.730Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:44.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:42:43.733Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:45.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:44 smithi006 bash[27292]: cluster 2023-08-27T22:42:43.949826+0000 mgr.y (mgr.14901) 1326 : cluster [DBG] pgmap v1333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:45.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:44 smithi006 bash[27292]: audit 2023-08-27T22:42:44.778202+0000 mon.c (mon.2) 543 : audit [DBG] from='client.? 172.21.15.6:0/166959753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:45.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:44 smithi006 bash[33918]: cluster 2023-08-27T22:42:43.949826+0000 mgr.y (mgr.14901) 1326 : cluster [DBG] pgmap v1333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:45.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:44 smithi006 bash[33918]: audit 2023-08-27T22:42:44.778202+0000 mon.c (mon.2) 543 : audit [DBG] from='client.? 172.21.15.6:0/166959753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:45.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:44 smithi130 bash[27579]: cluster 2023-08-27T22:42:43.949826+0000 mgr.y (mgr.14901) 1326 : cluster [DBG] pgmap v1333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:45.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:44 smithi130 bash[27579]: audit 2023-08-27T22:42:44.778202+0000 mon.c (mon.2) 543 : audit [DBG] from='client.? 172.21.15.6:0/166959753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:47.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:47 smithi130 bash[27579]: cluster 2023-08-27T22:42:45.950955+0000 mgr.y (mgr.14901) 1327 : cluster [DBG] pgmap v1334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:47.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:47 smithi006 bash[27292]: cluster 2023-08-27T22:42:45.950955+0000 mgr.y (mgr.14901) 1327 : cluster [DBG] pgmap v1334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:47.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:47 smithi006 bash[33918]: cluster 2023-08-27T22:42:45.950955+0000 mgr.y (mgr.14901) 1327 : cluster [DBG] pgmap v1334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:48.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:48 smithi130 bash[27579]: audit 2023-08-27T22:42:47.199464+0000 mon.c (mon.2) 544 : audit [DBG] from='client.? 172.21.15.6:0/1840749242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:48.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:48 smithi006 bash[27292]: audit 2023-08-27T22:42:47.199464+0000 mon.c (mon.2) 544 : audit [DBG] from='client.? 172.21.15.6:0/1840749242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:48.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:48 smithi006 bash[33918]: audit 2023-08-27T22:42:47.199464+0000 mon.c (mon.2) 544 : audit [DBG] from='client.? 172.21.15.6:0/1840749242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:49.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:49 smithi130 bash[27579]: cluster 2023-08-27T22:42:47.951629+0000 mgr.y (mgr.14901) 1328 : cluster [DBG] pgmap v1335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:49.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:49 smithi006 bash[33918]: cluster 2023-08-27T22:42:47.951629+0000 mgr.y (mgr.14901) 1328 : cluster [DBG] pgmap v1335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:49.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:49 smithi006 bash[27292]: cluster 2023-08-27T22:42:47.951629+0000 mgr.y (mgr.14901) 1328 : cluster [DBG] pgmap v1335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:50.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:50 smithi130 bash[27579]: audit 2023-08-27T22:42:49.615838+0000 mon.a (mon.0) 1518 : audit [DBG] from='client.? 172.21.15.6:0/834618619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:50.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:50 smithi006 bash[27292]: audit 2023-08-27T22:42:49.615838+0000 mon.a (mon.0) 1518 : audit [DBG] from='client.? 172.21.15.6:0/834618619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:50.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:50 smithi006 bash[33918]: audit 2023-08-27T22:42:49.615838+0000 mon.a (mon.0) 1518 : audit [DBG] from='client.? 172.21.15.6:0/834618619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:50.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:42:50.266Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:50.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:42:50.268Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:51.040 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:42:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:42:50] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T22:42:51.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:51 smithi130 bash[27579]: cluster 2023-08-27T22:42:49.952762+0000 mgr.y (mgr.14901) 1329 : cluster [DBG] pgmap v1336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:51.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:51 smithi006 bash[27292]: cluster 2023-08-27T22:42:49.952762+0000 mgr.y (mgr.14901) 1329 : cluster [DBG] pgmap v1336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:51.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:51 smithi006 bash[33918]: cluster 2023-08-27T22:42:49.952762+0000 mgr.y (mgr.14901) 1329 : cluster [DBG] pgmap v1336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:42:51.833Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:42:51.835Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:53.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:53 smithi130 bash[27579]: cluster 2023-08-27T22:42:51.953415+0000 mgr.y (mgr.14901) 1330 : cluster [DBG] pgmap v1337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:53.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:53 smithi130 bash[27579]: audit 2023-08-27T22:42:52.040689+0000 mon.c (mon.2) 545 : audit [DBG] from='client.? 172.21.15.6:0/444662066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:53.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:53 smithi006 bash[27292]: cluster 2023-08-27T22:42:51.953415+0000 mgr.y (mgr.14901) 1330 : cluster [DBG] pgmap v1337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:53.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:53 smithi006 bash[27292]: audit 2023-08-27T22:42:52.040689+0000 mon.c (mon.2) 545 : audit [DBG] from='client.? 172.21.15.6:0/444662066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:53 smithi006 bash[33918]: cluster 2023-08-27T22:42:51.953415+0000 mgr.y (mgr.14901) 1330 : cluster [DBG] pgmap v1337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:53 smithi006 bash[33918]: audit 2023-08-27T22:42:52.040689+0000 mon.c (mon.2) 545 : audit [DBG] from='client.? 172.21.15.6:0/444662066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:54.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:42:53.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:54.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:42:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:42:53.732Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:42:55.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:54 smithi006 bash[27292]: cluster 2023-08-27T22:42:53.954082+0000 mgr.y (mgr.14901) 1331 : cluster [DBG] pgmap v1338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:55.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:54 smithi006 bash[27292]: audit 2023-08-27T22:42:54.458737+0000 mon.a (mon.0) 1519 : audit [DBG] from='client.? 172.21.15.6:0/1104619792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:55.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:54 smithi006 bash[33918]: cluster 2023-08-27T22:42:53.954082+0000 mgr.y (mgr.14901) 1331 : cluster [DBG] pgmap v1338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:55.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:54 smithi006 bash[33918]: audit 2023-08-27T22:42:54.458737+0000 mon.a (mon.0) 1519 : audit [DBG] from='client.? 172.21.15.6:0/1104619792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:55.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:54 smithi130 bash[27579]: cluster 2023-08-27T22:42:53.954082+0000 mgr.y (mgr.14901) 1331 : cluster [DBG] pgmap v1338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:55.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:54 smithi130 bash[27579]: audit 2023-08-27T22:42:54.458737+0000 mon.a (mon.0) 1519 : audit [DBG] from='client.? 172.21.15.6:0/1104619792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:57.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:57 smithi130 bash[27579]: cluster 2023-08-27T22:42:55.955236+0000 mgr.y (mgr.14901) 1332 : cluster [DBG] pgmap v1339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:57.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:57 smithi130 bash[27579]: audit 2023-08-27T22:42:56.869211+0000 mon.a (mon.0) 1520 : audit [DBG] from='client.? 172.21.15.6:0/3081222498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:57.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:57 smithi006 bash[27292]: cluster 2023-08-27T22:42:55.955236+0000 mgr.y (mgr.14901) 1332 : cluster [DBG] pgmap v1339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:57.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:57 smithi006 bash[27292]: audit 2023-08-27T22:42:56.869211+0000 mon.a (mon.0) 1520 : audit [DBG] from='client.? 172.21.15.6:0/3081222498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:57.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:57 smithi006 bash[33918]: cluster 2023-08-27T22:42:55.955236+0000 mgr.y (mgr.14901) 1332 : cluster [DBG] pgmap v1339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:57.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:57 smithi006 bash[33918]: audit 2023-08-27T22:42:56.869211+0000 mon.a (mon.0) 1520 : audit [DBG] from='client.? 172.21.15.6:0/3081222498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:59.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:42:59 smithi130 bash[27579]: cluster 2023-08-27T22:42:57.955921+0000 mgr.y (mgr.14901) 1333 : cluster [DBG] pgmap v1340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:59.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:42:59 smithi006 bash[27292]: cluster 2023-08-27T22:42:57.955921+0000 mgr.y (mgr.14901) 1333 : cluster [DBG] pgmap v1340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:42:59.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:42:59 smithi006 bash[33918]: cluster 2023-08-27T22:42:57.955921+0000 mgr.y (mgr.14901) 1333 : cluster [DBG] pgmap v1340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:00.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:00 smithi130 bash[27579]: audit 2023-08-27T22:42:59.294062+0000 mon.a (mon.0) 1521 : audit [DBG] from='client.? 172.21.15.6:0/921749516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:00.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:00 smithi006 bash[27292]: audit 2023-08-27T22:42:59.294062+0000 mon.a (mon.0) 1521 : audit [DBG] from='client.? 172.21.15.6:0/921749516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:00.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:00 smithi006 bash[33918]: audit 2023-08-27T22:42:59.294062+0000 mon.a (mon.0) 1521 : audit [DBG] from='client.? 172.21.15.6:0/921749516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:00.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:43:00.268Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:00.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:43:00.269Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:01.038 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:43:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:43:00] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T22:43:01.038 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:01 smithi006 bash[27292]: cluster 2023-08-27T22:42:59.957058+0000 mgr.y (mgr.14901) 1334 : cluster [DBG] pgmap v1341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:01.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:01 smithi130 bash[27579]: cluster 2023-08-27T22:42:59.957058+0000 mgr.y (mgr.14901) 1334 : cluster [DBG] pgmap v1341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:01.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:01 smithi006 bash[33918]: cluster 2023-08-27T22:42:59.957058+0000 mgr.y (mgr.14901) 1334 : cluster [DBG] pgmap v1341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:02.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:02 smithi006 bash[27292]: audit 2023-08-27T22:43:01.709800+0000 mon.c (mon.2) 546 : audit [DBG] from='client.? 172.21.15.6:0/3452977949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:02.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:02 smithi006 bash[33918]: audit 2023-08-27T22:43:01.709800+0000 mon.c (mon.2) 546 : audit [DBG] from='client.? 172.21.15.6:0/3452977949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:43:01.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:43:01.836Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:02.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:02 smithi130 bash[27579]: audit 2023-08-27T22:43:01.709800+0000 mon.c (mon.2) 546 : audit [DBG] from='client.? 172.21.15.6:0/3452977949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:03.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:03 smithi130 bash[27579]: cluster 2023-08-27T22:43:01.957655+0000 mgr.y (mgr.14901) 1335 : cluster [DBG] pgmap v1342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:03.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:03 smithi006 bash[33918]: cluster 2023-08-27T22:43:01.957655+0000 mgr.y (mgr.14901) 1335 : cluster [DBG] pgmap v1342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:03.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:03 smithi006 bash[27292]: cluster 2023-08-27T22:43:01.957655+0000 mgr.y (mgr.14901) 1335 : cluster [DBG] pgmap v1342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:04.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:43:03.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:04.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:43:03.734Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:05.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:04 smithi006 bash[27292]: cluster 2023-08-27T22:43:03.958308+0000 mgr.y (mgr.14901) 1336 : cluster [DBG] pgmap v1343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:05.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:04 smithi006 bash[27292]: audit 2023-08-27T22:43:04.126282+0000 mon.c (mon.2) 547 : audit [DBG] from='client.? 172.21.15.6:0/424493790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:05.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:04 smithi006 bash[33918]: cluster 2023-08-27T22:43:03.958308+0000 mgr.y (mgr.14901) 1336 : cluster [DBG] pgmap v1343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:05.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:04 smithi006 bash[33918]: audit 2023-08-27T22:43:04.126282+0000 mon.c (mon.2) 547 : audit [DBG] from='client.? 172.21.15.6:0/424493790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:04 smithi130 bash[27579]: cluster 2023-08-27T22:43:03.958308+0000 mgr.y (mgr.14901) 1336 : cluster [DBG] pgmap v1343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:04 smithi130 bash[27579]: audit 2023-08-27T22:43:04.126282+0000 mon.c (mon.2) 547 : audit [DBG] from='client.? 172.21.15.6:0/424493790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:07.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:07 smithi130 bash[27579]: cluster 2023-08-27T22:43:05.959416+0000 mgr.y (mgr.14901) 1337 : cluster [DBG] pgmap v1344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:07.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:07 smithi130 bash[27579]: audit 2023-08-27T22:43:06.069895+0000 mon.b (mon.1) 251 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:43:07.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:07 smithi130 bash[27579]: audit 2023-08-27T22:43:06.530806+0000 mon.c (mon.2) 548 : audit [DBG] from='client.? 172.21.15.6:0/2716580056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:07.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:07 smithi006 bash[27292]: cluster 2023-08-27T22:43:05.959416+0000 mgr.y (mgr.14901) 1337 : cluster [DBG] pgmap v1344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:07.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:07 smithi006 bash[27292]: audit 2023-08-27T22:43:06.069895+0000 mon.b (mon.1) 251 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:43:07.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:07 smithi006 bash[27292]: audit 2023-08-27T22:43:06.530806+0000 mon.c (mon.2) 548 : audit [DBG] from='client.? 172.21.15.6:0/2716580056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:07.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:07 smithi006 bash[33918]: cluster 2023-08-27T22:43:05.959416+0000 mgr.y (mgr.14901) 1337 : cluster [DBG] pgmap v1344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:07.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:07 smithi006 bash[33918]: audit 2023-08-27T22:43:06.069895+0000 mon.b (mon.1) 251 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:43:07.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:07 smithi006 bash[33918]: audit 2023-08-27T22:43:06.530806+0000 mon.c (mon.2) 548 : audit [DBG] from='client.? 172.21.15.6:0/2716580056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:09.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:09 smithi130 bash[27579]: cluster 2023-08-27T22:43:07.960078+0000 mgr.y (mgr.14901) 1338 : cluster [DBG] pgmap v1345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:09.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:09 smithi130 bash[27579]: audit 2023-08-27T22:43:08.958753+0000 mon.a (mon.0) 1522 : audit [DBG] from='client.? 172.21.15.6:0/2564757429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:09.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:09 smithi006 bash[27292]: cluster 2023-08-27T22:43:07.960078+0000 mgr.y (mgr.14901) 1338 : cluster [DBG] pgmap v1345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:09.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:09 smithi006 bash[27292]: audit 2023-08-27T22:43:08.958753+0000 mon.a (mon.0) 1522 : audit [DBG] from='client.? 172.21.15.6:0/2564757429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:09.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:09 smithi006 bash[33918]: cluster 2023-08-27T22:43:07.960078+0000 mgr.y (mgr.14901) 1338 : cluster [DBG] pgmap v1345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:09.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:09 smithi006 bash[33918]: audit 2023-08-27T22:43:08.958753+0000 mon.a (mon.0) 1522 : audit [DBG] from='client.? 172.21.15.6:0/2564757429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:43:10.268Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:10.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:43:10.270Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:11.036 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:43:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:43:10] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T22:43:11.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:11 smithi130 bash[27579]: cluster 2023-08-27T22:43:09.961216+0000 mgr.y (mgr.14901) 1339 : cluster [DBG] pgmap v1346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:11.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:11 smithi006 bash[27292]: cluster 2023-08-27T22:43:09.961216+0000 mgr.y (mgr.14901) 1339 : cluster [DBG] pgmap v1346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:11.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:11 smithi006 bash[33918]: cluster 2023-08-27T22:43:09.961216+0000 mgr.y (mgr.14901) 1339 : cluster [DBG] pgmap v1346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:12.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:12 smithi006 bash[27292]: audit 2023-08-27T22:43:11.395414+0000 mon.a (mon.0) 1523 : audit [DBG] from='client.? 172.21.15.6:0/3221902275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:12.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:12 smithi006 bash[27292]: audit 2023-08-27T22:43:11.924537+0000 mon.a (mon.0) 1524 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:43:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:43:11.835Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:43:11.837Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:12.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:12 smithi006 bash[33918]: audit 2023-08-27T22:43:11.395414+0000 mon.a (mon.0) 1523 : audit [DBG] from='client.? 172.21.15.6:0/3221902275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:12.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:12 smithi006 bash[33918]: audit 2023-08-27T22:43:11.924537+0000 mon.a (mon.0) 1524 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:43:12.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:12 smithi130 bash[27579]: audit 2023-08-27T22:43:11.395414+0000 mon.a (mon.0) 1523 : audit [DBG] from='client.? 172.21.15.6:0/3221902275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:12.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:12 smithi130 bash[27579]: audit 2023-08-27T22:43:11.924537+0000 mon.a (mon.0) 1524 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:43:13.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:13 smithi130 bash[27579]: cluster 2023-08-27T22:43:11.961706+0000 mgr.y (mgr.14901) 1340 : cluster [DBG] pgmap v1347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:13 smithi130 bash[27579]: audit 2023-08-27T22:43:12.377670+0000 mon.a (mon.0) 1525 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:43:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:13 smithi130 bash[27579]: audit 2023-08-27T22:43:12.649742+0000 mon.b (mon.1) 252 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:43:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:13 smithi130 bash[27579]: audit 2023-08-27T22:43:12.652147+0000 mon.b (mon.1) 253 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:43:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:13 smithi130 bash[27579]: audit 2023-08-27T22:43:12.655381+0000 mon.a (mon.0) 1526 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:43:13.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:13 smithi006 bash[27292]: cluster 2023-08-27T22:43:11.961706+0000 mgr.y (mgr.14901) 1340 : cluster [DBG] pgmap v1347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:13.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:13 smithi006 bash[27292]: audit 2023-08-27T22:43:12.377670+0000 mon.a (mon.0) 1525 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:43:13.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:13 smithi006 bash[27292]: audit 2023-08-27T22:43:12.649742+0000 mon.b (mon.1) 252 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:43:13.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:13 smithi006 bash[27292]: audit 2023-08-27T22:43:12.652147+0000 mon.b (mon.1) 253 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:43:13.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:13 smithi006 bash[27292]: audit 2023-08-27T22:43:12.655381+0000 mon.a (mon.0) 1526 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:43:13.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:13 smithi006 bash[33918]: cluster 2023-08-27T22:43:11.961706+0000 mgr.y (mgr.14901) 1340 : cluster [DBG] pgmap v1347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:13.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:13 smithi006 bash[33918]: audit 2023-08-27T22:43:12.377670+0000 mon.a (mon.0) 1525 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:43:13.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:13 smithi006 bash[33918]: audit 2023-08-27T22:43:12.649742+0000 mon.b (mon.1) 252 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:43:13.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:13 smithi006 bash[33918]: audit 2023-08-27T22:43:12.652147+0000 mon.b (mon.1) 253 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:43:13.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:13 smithi006 bash[33918]: audit 2023-08-27T22:43:12.655381+0000 mon.a (mon.0) 1526 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:43:14.041 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:43:13.732Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:14.042 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:43:13.734Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:14.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:14 smithi130 bash[27579]: audit 2023-08-27T22:43:13.821609+0000 mon.a (mon.0) 1527 : audit [DBG] from='client.? 172.21.15.6:0/3222199919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:14.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:14 smithi006 bash[27292]: audit 2023-08-27T22:43:13.821609+0000 mon.a (mon.0) 1527 : audit [DBG] from='client.? 172.21.15.6:0/3222199919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:14.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:14 smithi006 bash[33918]: audit 2023-08-27T22:43:13.821609+0000 mon.a (mon.0) 1527 : audit [DBG] from='client.? 172.21.15.6:0/3222199919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:15.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:15 smithi130 bash[27579]: cluster 2023-08-27T22:43:13.962319+0000 mgr.y (mgr.14901) 1341 : cluster [DBG] pgmap v1348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:15.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:15 smithi006 bash[33918]: cluster 2023-08-27T22:43:13.962319+0000 mgr.y (mgr.14901) 1341 : cluster [DBG] pgmap v1348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:15.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:15 smithi006 bash[27292]: cluster 2023-08-27T22:43:13.962319+0000 mgr.y (mgr.14901) 1341 : cluster [DBG] pgmap v1348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:17.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:17 smithi130 bash[27579]: cluster 2023-08-27T22:43:15.963437+0000 mgr.y (mgr.14901) 1342 : cluster [DBG] pgmap v1349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:17.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:17 smithi130 bash[27579]: audit 2023-08-27T22:43:16.231900+0000 mon.c (mon.2) 549 : audit [DBG] from='client.? 172.21.15.6:0/558924632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:17.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:17 smithi006 bash[33918]: cluster 2023-08-27T22:43:15.963437+0000 mgr.y (mgr.14901) 1342 : cluster [DBG] pgmap v1349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:17.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:17 smithi006 bash[33918]: audit 2023-08-27T22:43:16.231900+0000 mon.c (mon.2) 549 : audit [DBG] from='client.? 172.21.15.6:0/558924632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:17.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:17 smithi006 bash[27292]: cluster 2023-08-27T22:43:15.963437+0000 mgr.y (mgr.14901) 1342 : cluster [DBG] pgmap v1349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:17.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:17 smithi006 bash[27292]: audit 2023-08-27T22:43:16.231900+0000 mon.c (mon.2) 549 : audit [DBG] from='client.? 172.21.15.6:0/558924632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:19.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:19 smithi130 bash[27579]: cluster 2023-08-27T22:43:17.964127+0000 mgr.y (mgr.14901) 1343 : cluster [DBG] pgmap v1350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:19.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:19 smithi130 bash[27579]: audit 2023-08-27T22:43:18.652618+0000 mon.c (mon.2) 550 : audit [DBG] from='client.? 172.21.15.6:0/1368757994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:19.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:19 smithi006 bash[27292]: cluster 2023-08-27T22:43:17.964127+0000 mgr.y (mgr.14901) 1343 : cluster [DBG] pgmap v1350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:19.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:19 smithi006 bash[27292]: audit 2023-08-27T22:43:18.652618+0000 mon.c (mon.2) 550 : audit [DBG] from='client.? 172.21.15.6:0/1368757994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:19.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:19 smithi006 bash[33918]: cluster 2023-08-27T22:43:17.964127+0000 mgr.y (mgr.14901) 1343 : cluster [DBG] pgmap v1350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:19.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:19 smithi006 bash[33918]: audit 2023-08-27T22:43:18.652618+0000 mon.c (mon.2) 550 : audit [DBG] from='client.? 172.21.15.6:0/1368757994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:20.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:43:20.269Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:20.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:43:20.271Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:21.057 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:43:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:43:20] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T22:43:21.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:21 smithi130 bash[27579]: cluster 2023-08-27T22:43:19.965235+0000 mgr.y (mgr.14901) 1344 : cluster [DBG] pgmap v1351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:21.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:21 smithi006 bash[33918]: cluster 2023-08-27T22:43:19.965235+0000 mgr.y (mgr.14901) 1344 : cluster [DBG] pgmap v1351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:21.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:21 smithi006 bash[27292]: cluster 2023-08-27T22:43:19.965235+0000 mgr.y (mgr.14901) 1344 : cluster [DBG] pgmap v1351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:22.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:22 smithi006 bash[27292]: audit 2023-08-27T22:43:21.067745+0000 mon.c (mon.2) 551 : audit [DBG] from='client.? 172.21.15.6:0/577695309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:22.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:22 smithi006 bash[33918]: audit 2023-08-27T22:43:21.067745+0000 mon.c (mon.2) 551 : audit [DBG] from='client.? 172.21.15.6:0/577695309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:43:21.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:43:21.837Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:22.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:22 smithi130 bash[27579]: audit 2023-08-27T22:43:21.067745+0000 mon.c (mon.2) 551 : audit [DBG] from='client.? 172.21.15.6:0/577695309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:23.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:23 smithi130 bash[27579]: cluster 2023-08-27T22:43:21.965886+0000 mgr.y (mgr.14901) 1345 : cluster [DBG] pgmap v1352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:23.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:23 smithi006 bash[27292]: cluster 2023-08-27T22:43:21.965886+0000 mgr.y (mgr.14901) 1345 : cluster [DBG] pgmap v1352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:23.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:23 smithi006 bash[33918]: cluster 2023-08-27T22:43:21.965886+0000 mgr.y (mgr.14901) 1345 : cluster [DBG] pgmap v1352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:24.073 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:43:23.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:24.073 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:43:23.735Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:24.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:24 smithi130 bash[27579]: audit 2023-08-27T22:43:23.478692+0000 mon.a (mon.0) 1528 : audit [DBG] from='client.? 172.21.15.6:0/3371942527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:24.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:24 smithi006 bash[27292]: audit 2023-08-27T22:43:23.478692+0000 mon.a (mon.0) 1528 : audit [DBG] from='client.? 172.21.15.6:0/3371942527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:24.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:24 smithi006 bash[33918]: audit 2023-08-27T22:43:23.478692+0000 mon.a (mon.0) 1528 : audit [DBG] from='client.? 172.21.15.6:0/3371942527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:25.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:25 smithi130 bash[27579]: cluster 2023-08-27T22:43:23.966555+0000 mgr.y (mgr.14901) 1346 : cluster [DBG] pgmap v1353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:25.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:25 smithi006 bash[27292]: cluster 2023-08-27T22:43:23.966555+0000 mgr.y (mgr.14901) 1346 : cluster [DBG] pgmap v1353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:25.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:25 smithi006 bash[33918]: cluster 2023-08-27T22:43:23.966555+0000 mgr.y (mgr.14901) 1346 : cluster [DBG] pgmap v1353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:26.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:26 smithi130 bash[27579]: audit 2023-08-27T22:43:25.897580+0000 mon.a (mon.0) 1529 : audit [DBG] from='client.? 172.21.15.6:0/1253335225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:26.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:26 smithi006 bash[27292]: audit 2023-08-27T22:43:25.897580+0000 mon.a (mon.0) 1529 : audit [DBG] from='client.? 172.21.15.6:0/1253335225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:26.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:26 smithi006 bash[33918]: audit 2023-08-27T22:43:25.897580+0000 mon.a (mon.0) 1529 : audit [DBG] from='client.? 172.21.15.6:0/1253335225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:27.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:27 smithi130 bash[27579]: cluster 2023-08-27T22:43:25.967614+0000 mgr.y (mgr.14901) 1347 : cluster [DBG] pgmap v1354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:27.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:27 smithi006 bash[27292]: cluster 2023-08-27T22:43:25.967614+0000 mgr.y (mgr.14901) 1347 : cluster [DBG] pgmap v1354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:27.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:27 smithi006 bash[33918]: cluster 2023-08-27T22:43:25.967614+0000 mgr.y (mgr.14901) 1347 : cluster [DBG] pgmap v1354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:29.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:29 smithi130 bash[27579]: cluster 2023-08-27T22:43:27.968218+0000 mgr.y (mgr.14901) 1348 : cluster [DBG] pgmap v1355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:29.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:29 smithi130 bash[27579]: audit 2023-08-27T22:43:28.312335+0000 mon.a (mon.0) 1530 : audit [DBG] from='client.? 172.21.15.6:0/3774207699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:29.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:29 smithi006 bash[33918]: cluster 2023-08-27T22:43:27.968218+0000 mgr.y (mgr.14901) 1348 : cluster [DBG] pgmap v1355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:29.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:29 smithi006 bash[33918]: audit 2023-08-27T22:43:28.312335+0000 mon.a (mon.0) 1530 : audit [DBG] from='client.? 172.21.15.6:0/3774207699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:29.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:29 smithi006 bash[27292]: cluster 2023-08-27T22:43:27.968218+0000 mgr.y (mgr.14901) 1348 : cluster [DBG] pgmap v1355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:29.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:29 smithi006 bash[27292]: audit 2023-08-27T22:43:28.312335+0000 mon.a (mon.0) 1530 : audit [DBG] from='client.? 172.21.15.6:0/3774207699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:30.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:43:30.270Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:30.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:43:30.272Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:31.094 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:43:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:43:30] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T22:43:31.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:31 smithi130 bash[27579]: cluster 2023-08-27T22:43:29.969470+0000 mgr.y (mgr.14901) 1349 : cluster [DBG] pgmap v1356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:31.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:31 smithi130 bash[27579]: audit 2023-08-27T22:43:30.721443+0000 mon.a (mon.0) 1531 : audit [DBG] from='client.? 172.21.15.6:0/3997175641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:31.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:31 smithi006 bash[33918]: cluster 2023-08-27T22:43:29.969470+0000 mgr.y (mgr.14901) 1349 : cluster [DBG] pgmap v1356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:31.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:31 smithi006 bash[33918]: audit 2023-08-27T22:43:30.721443+0000 mon.a (mon.0) 1531 : audit [DBG] from='client.? 172.21.15.6:0/3997175641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:31.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:31 smithi006 bash[27292]: cluster 2023-08-27T22:43:29.969470+0000 mgr.y (mgr.14901) 1349 : cluster [DBG] pgmap v1356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:31.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:31 smithi006 bash[27292]: audit 2023-08-27T22:43:30.721443+0000 mon.a (mon.0) 1531 : audit [DBG] from='client.? 172.21.15.6:0/3997175641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:43:31.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:43:31.839Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:33.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:33 smithi130 bash[27579]: cluster 2023-08-27T22:43:31.970302+0000 mgr.y (mgr.14901) 1350 : cluster [DBG] pgmap v1357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:33.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:33 smithi006 bash[33918]: cluster 2023-08-27T22:43:31.970302+0000 mgr.y (mgr.14901) 1350 : cluster [DBG] pgmap v1357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:33.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:33 smithi006 bash[27292]: cluster 2023-08-27T22:43:31.970302+0000 mgr.y (mgr.14901) 1350 : cluster [DBG] pgmap v1357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:34.104 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:43:33.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:34.104 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:43:33.735Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:34.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:34 smithi130 bash[27579]: audit 2023-08-27T22:43:33.144317+0000 mon.a (mon.0) 1532 : audit [DBG] from='client.? 172.21.15.6:0/2838797012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:34.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:34 smithi006 bash[33918]: audit 2023-08-27T22:43:33.144317+0000 mon.a (mon.0) 1532 : audit [DBG] from='client.? 172.21.15.6:0/2838797012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:34.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:34 smithi006 bash[27292]: audit 2023-08-27T22:43:33.144317+0000 mon.a (mon.0) 1532 : audit [DBG] from='client.? 172.21.15.6:0/2838797012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:35.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:35 smithi130 bash[27579]: cluster 2023-08-27T22:43:33.970987+0000 mgr.y (mgr.14901) 1351 : cluster [DBG] pgmap v1358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:35.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:35 smithi006 bash[27292]: cluster 2023-08-27T22:43:33.970987+0000 mgr.y (mgr.14901) 1351 : cluster [DBG] pgmap v1358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:35.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:35 smithi006 bash[33918]: cluster 2023-08-27T22:43:33.970987+0000 mgr.y (mgr.14901) 1351 : cluster [DBG] pgmap v1358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:36.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:36 smithi130 bash[27579]: audit 2023-08-27T22:43:35.558145+0000 mon.c (mon.2) 552 : audit [DBG] from='client.? 172.21.15.6:0/263751196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:36.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:36 smithi006 bash[27292]: audit 2023-08-27T22:43:35.558145+0000 mon.c (mon.2) 552 : audit [DBG] from='client.? 172.21.15.6:0/263751196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:36.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:36 smithi006 bash[33918]: audit 2023-08-27T22:43:35.558145+0000 mon.c (mon.2) 552 : audit [DBG] from='client.? 172.21.15.6:0/263751196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:37.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:37 smithi006 bash[27292]: cluster 2023-08-27T22:43:35.972112+0000 mgr.y (mgr.14901) 1352 : cluster [DBG] pgmap v1359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:37.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:37 smithi006 bash[33918]: cluster 2023-08-27T22:43:35.972112+0000 mgr.y (mgr.14901) 1352 : cluster [DBG] pgmap v1359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:37.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:37 smithi130 bash[27579]: cluster 2023-08-27T22:43:35.972112+0000 mgr.y (mgr.14901) 1352 : cluster [DBG] pgmap v1359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:38.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:38 smithi006 bash[33918]: audit 2023-08-27T22:43:37.981075+0000 mon.a (mon.0) 1533 : audit [DBG] from='client.? 172.21.15.6:0/2829284937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:38.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:38 smithi006 bash[27292]: audit 2023-08-27T22:43:37.981075+0000 mon.a (mon.0) 1533 : audit [DBG] from='client.? 172.21.15.6:0/2829284937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:38.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:38 smithi130 bash[27579]: audit 2023-08-27T22:43:37.981075+0000 mon.a (mon.0) 1533 : audit [DBG] from='client.? 172.21.15.6:0/2829284937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:39.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:39 smithi006 bash[27292]: cluster 2023-08-27T22:43:37.972737+0000 mgr.y (mgr.14901) 1353 : cluster [DBG] pgmap v1360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:39.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:39 smithi006 bash[33918]: cluster 2023-08-27T22:43:37.972737+0000 mgr.y (mgr.14901) 1353 : cluster [DBG] pgmap v1360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:39.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:39 smithi130 bash[27579]: cluster 2023-08-27T22:43:37.972737+0000 mgr.y (mgr.14901) 1353 : cluster [DBG] pgmap v1360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:40.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:43:40.270Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:40.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:43:40.272Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:41.132 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:43:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:43:40] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T22:43:41.133 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:41 smithi006 bash[27292]: cluster 2023-08-27T22:43:39.973817+0000 mgr.y (mgr.14901) 1354 : cluster [DBG] pgmap v1361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:41.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:41 smithi006 bash[27292]: audit 2023-08-27T22:43:40.408707+0000 mon.c (mon.2) 553 : audit [DBG] from='client.? 172.21.15.6:0/812617153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:41.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:41 smithi006 bash[33918]: cluster 2023-08-27T22:43:39.973817+0000 mgr.y (mgr.14901) 1354 : cluster [DBG] pgmap v1361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:41.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:41 smithi006 bash[33918]: audit 2023-08-27T22:43:40.408707+0000 mon.c (mon.2) 553 : audit [DBG] from='client.? 172.21.15.6:0/812617153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:41.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:41 smithi130 bash[27579]: cluster 2023-08-27T22:43:39.973817+0000 mgr.y (mgr.14901) 1354 : cluster [DBG] pgmap v1361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:41.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:41 smithi130 bash[27579]: audit 2023-08-27T22:43:40.408707+0000 mon.c (mon.2) 553 : audit [DBG] from='client.? 172.21.15.6:0/812617153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:43:41.837Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:43:41.839Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:43.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:43 smithi006 bash[33918]: cluster 2023-08-27T22:43:41.974411+0000 mgr.y (mgr.14901) 1355 : cluster [DBG] pgmap v1362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:43.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:43 smithi006 bash[33918]: audit 2023-08-27T22:43:42.828355+0000 mon.a (mon.0) 1534 : audit [DBG] from='client.? 172.21.15.6:0/2325869358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:43.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:43 smithi006 bash[27292]: cluster 2023-08-27T22:43:41.974411+0000 mgr.y (mgr.14901) 1355 : cluster [DBG] pgmap v1362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:43.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:43 smithi006 bash[27292]: audit 2023-08-27T22:43:42.828355+0000 mon.a (mon.0) 1534 : audit [DBG] from='client.? 172.21.15.6:0/2325869358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:43.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:43 smithi130 bash[27579]: cluster 2023-08-27T22:43:41.974411+0000 mgr.y (mgr.14901) 1355 : cluster [DBG] pgmap v1362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:43.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:43 smithi130 bash[27579]: audit 2023-08-27T22:43:42.828355+0000 mon.a (mon.0) 1534 : audit [DBG] from='client.? 172.21.15.6:0/2325869358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:44.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:43:43.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:44.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:43:43.737Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:45.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:44 smithi006 bash[27292]: cluster 2023-08-27T22:43:43.975063+0000 mgr.y (mgr.14901) 1356 : cluster [DBG] pgmap v1363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:45.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:44 smithi006 bash[33918]: cluster 2023-08-27T22:43:43.975063+0000 mgr.y (mgr.14901) 1356 : cluster [DBG] pgmap v1363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:45.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:44 smithi130 bash[27579]: cluster 2023-08-27T22:43:43.975063+0000 mgr.y (mgr.14901) 1356 : cluster [DBG] pgmap v1363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:46.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:45 smithi006 bash[27292]: audit 2023-08-27T22:43:45.251527+0000 mon.a (mon.0) 1535 : audit [DBG] from='client.? 172.21.15.6:0/852888797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:46.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:45 smithi006 bash[33918]: audit 2023-08-27T22:43:45.251527+0000 mon.a (mon.0) 1535 : audit [DBG] from='client.? 172.21.15.6:0/852888797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:46.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:45 smithi130 bash[27579]: audit 2023-08-27T22:43:45.251527+0000 mon.a (mon.0) 1535 : audit [DBG] from='client.? 172.21.15.6:0/852888797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:47.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:46 smithi006 bash[33918]: cluster 2023-08-27T22:43:45.976215+0000 mgr.y (mgr.14901) 1357 : cluster [DBG] pgmap v1364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:47.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:46 smithi006 bash[27292]: cluster 2023-08-27T22:43:45.976215+0000 mgr.y (mgr.14901) 1357 : cluster [DBG] pgmap v1364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:47.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:46 smithi130 bash[27579]: cluster 2023-08-27T22:43:45.976215+0000 mgr.y (mgr.14901) 1357 : cluster [DBG] pgmap v1364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:48.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:47 smithi006 bash[27292]: audit 2023-08-27T22:43:47.662023+0000 mon.a (mon.0) 1536 : audit [DBG] from='client.? 172.21.15.6:0/860465817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:48.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:47 smithi006 bash[33918]: audit 2023-08-27T22:43:47.662023+0000 mon.a (mon.0) 1536 : audit [DBG] from='client.? 172.21.15.6:0/860465817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:48.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:47 smithi130 bash[27579]: audit 2023-08-27T22:43:47.662023+0000 mon.a (mon.0) 1536 : audit [DBG] from='client.? 172.21.15.6:0/860465817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:49.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:48 smithi006 bash[27292]: cluster 2023-08-27T22:43:47.976942+0000 mgr.y (mgr.14901) 1358 : cluster [DBG] pgmap v1365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:49.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:48 smithi006 bash[33918]: cluster 2023-08-27T22:43:47.976942+0000 mgr.y (mgr.14901) 1358 : cluster [DBG] pgmap v1365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:49.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:48 smithi130 bash[27579]: cluster 2023-08-27T22:43:47.976942+0000 mgr.y (mgr.14901) 1358 : cluster [DBG] pgmap v1365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:50.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:43:50.271Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:50.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:43:50.273Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:51.044 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:43:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:43:50] "GET /metrics HTTP/1.1" 200 193078 "" "Prometheus/2.33.4" 2023-08-27T22:43:51.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:51 smithi130 bash[27579]: cluster 2023-08-27T22:43:49.977861+0000 mgr.y (mgr.14901) 1359 : cluster [DBG] pgmap v1366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:51.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:51 smithi130 bash[27579]: audit 2023-08-27T22:43:50.074842+0000 mon.a (mon.0) 1537 : audit [DBG] from='client.? 172.21.15.6:0/2589017300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:51.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:51 smithi006 bash[27292]: cluster 2023-08-27T22:43:49.977861+0000 mgr.y (mgr.14901) 1359 : cluster [DBG] pgmap v1366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:51.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:51 smithi006 bash[27292]: audit 2023-08-27T22:43:50.074842+0000 mon.a (mon.0) 1537 : audit [DBG] from='client.? 172.21.15.6:0/2589017300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:51.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:51 smithi006 bash[33918]: cluster 2023-08-27T22:43:49.977861+0000 mgr.y (mgr.14901) 1359 : cluster [DBG] pgmap v1366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:51.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:51 smithi006 bash[33918]: audit 2023-08-27T22:43:50.074842+0000 mon.a (mon.0) 1537 : audit [DBG] from='client.? 172.21.15.6:0/2589017300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:43:51.838Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:43:51.840Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:53.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:53 smithi130 bash[27579]: cluster 2023-08-27T22:43:51.978487+0000 mgr.y (mgr.14901) 1360 : cluster [DBG] pgmap v1367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:53.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:53 smithi130 bash[27579]: audit 2023-08-27T22:43:52.487614+0000 mon.c (mon.2) 554 : audit [DBG] from='client.? 172.21.15.6:0/2549738705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:53 smithi006 bash[33918]: cluster 2023-08-27T22:43:51.978487+0000 mgr.y (mgr.14901) 1360 : cluster [DBG] pgmap v1367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:53 smithi006 bash[33918]: audit 2023-08-27T22:43:52.487614+0000 mon.c (mon.2) 554 : audit [DBG] from='client.? 172.21.15.6:0/2549738705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:53.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:53 smithi006 bash[27292]: cluster 2023-08-27T22:43:51.978487+0000 mgr.y (mgr.14901) 1360 : cluster [DBG] pgmap v1367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:53.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:53 smithi006 bash[27292]: audit 2023-08-27T22:43:52.487614+0000 mon.c (mon.2) 554 : audit [DBG] from='client.? 172.21.15.6:0/2549738705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:54.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:43:53.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:54.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:43:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:43:53.737Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:43:55.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:54 smithi006 bash[33918]: cluster 2023-08-27T22:43:53.979137+0000 mgr.y (mgr.14901) 1361 : cluster [DBG] pgmap v1368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:55.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:54 smithi006 bash[27292]: cluster 2023-08-27T22:43:53.979137+0000 mgr.y (mgr.14901) 1361 : cluster [DBG] pgmap v1368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:55.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:54 smithi130 bash[27579]: cluster 2023-08-27T22:43:53.979137+0000 mgr.y (mgr.14901) 1361 : cluster [DBG] pgmap v1368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:56.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:55 smithi006 bash[33918]: audit 2023-08-27T22:43:54.905752+0000 mon.a (mon.0) 1538 : audit [DBG] from='client.? 172.21.15.6:0/2797783600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:56.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:55 smithi006 bash[27292]: audit 2023-08-27T22:43:54.905752+0000 mon.a (mon.0) 1538 : audit [DBG] from='client.? 172.21.15.6:0/2797783600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:56.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:55 smithi130 bash[27579]: audit 2023-08-27T22:43:54.905752+0000 mon.a (mon.0) 1538 : audit [DBG] from='client.? 172.21.15.6:0/2797783600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:57.221 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:56 smithi006 bash[27292]: cluster 2023-08-27T22:43:55.980280+0000 mgr.y (mgr.14901) 1362 : cluster [DBG] pgmap v1369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:57.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:56 smithi006 bash[33918]: cluster 2023-08-27T22:43:55.980280+0000 mgr.y (mgr.14901) 1362 : cluster [DBG] pgmap v1369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:57.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:56 smithi130 bash[27579]: cluster 2023-08-27T22:43:55.980280+0000 mgr.y (mgr.14901) 1362 : cluster [DBG] pgmap v1369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:58.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:57 smithi006 bash[33918]: audit 2023-08-27T22:43:57.318360+0000 mon.c (mon.2) 555 : audit [DBG] from='client.? 172.21.15.6:0/22184233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:58.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:57 smithi006 bash[27292]: audit 2023-08-27T22:43:57.318360+0000 mon.c (mon.2) 555 : audit [DBG] from='client.? 172.21.15.6:0/22184233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:58.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:57 smithi130 bash[27579]: audit 2023-08-27T22:43:57.318360+0000 mon.c (mon.2) 555 : audit [DBG] from='client.? 172.21.15.6:0/22184233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:59.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:58 smithi006 bash[27292]: cluster 2023-08-27T22:43:57.980982+0000 mgr.y (mgr.14901) 1363 : cluster [DBG] pgmap v1370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:59.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:58 smithi006 bash[33918]: cluster 2023-08-27T22:43:57.980982+0000 mgr.y (mgr.14901) 1363 : cluster [DBG] pgmap v1370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:43:59.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:58 smithi130 bash[27579]: cluster 2023-08-27T22:43:57.980982+0000 mgr.y (mgr.14901) 1363 : cluster [DBG] pgmap v1370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:00.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:43:59 smithi006 bash[33918]: audit 2023-08-27T22:43:59.733086+0000 mon.a (mon.0) 1539 : audit [DBG] from='client.? 172.21.15.6:0/3057327288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:00.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:43:59 smithi006 bash[27292]: audit 2023-08-27T22:43:59.733086+0000 mon.a (mon.0) 1539 : audit [DBG] from='client.? 172.21.15.6:0/3057327288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:00.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:43:59 smithi130 bash[27579]: audit 2023-08-27T22:43:59.733086+0000 mon.a (mon.0) 1539 : audit [DBG] from='client.? 172.21.15.6:0/3057327288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:00.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:44:00.272Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:00.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:44:00.274Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:01.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:00 smithi006 bash[27292]: cluster 2023-08-27T22:43:59.982066+0000 mgr.y (mgr.14901) 1364 : cluster [DBG] pgmap v1371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:01.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:44:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:44:00] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T22:44:01.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:00 smithi006 bash[33918]: cluster 2023-08-27T22:43:59.982066+0000 mgr.y (mgr.14901) 1364 : cluster [DBG] pgmap v1371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:01.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:00 smithi130 bash[27579]: cluster 2023-08-27T22:43:59.982066+0000 mgr.y (mgr.14901) 1364 : cluster [DBG] pgmap v1371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:44:01.839Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:44:01.841Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:03.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:03 smithi130 bash[27579]: cluster 2023-08-27T22:44:01.982682+0000 mgr.y (mgr.14901) 1365 : cluster [DBG] pgmap v1372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:03.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:03 smithi130 bash[27579]: audit 2023-08-27T22:44:02.151051+0000 mon.c (mon.2) 556 : audit [DBG] from='client.? 172.21.15.6:0/708306236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:03.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:03 smithi006 bash[27292]: cluster 2023-08-27T22:44:01.982682+0000 mgr.y (mgr.14901) 1365 : cluster [DBG] pgmap v1372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:03.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:03 smithi006 bash[27292]: audit 2023-08-27T22:44:02.151051+0000 mon.c (mon.2) 556 : audit [DBG] from='client.? 172.21.15.6:0/708306236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:03.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:03 smithi006 bash[33918]: cluster 2023-08-27T22:44:01.982682+0000 mgr.y (mgr.14901) 1365 : cluster [DBG] pgmap v1372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:03.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:03 smithi006 bash[33918]: audit 2023-08-27T22:44:02.151051+0000 mon.c (mon.2) 556 : audit [DBG] from='client.? 172.21.15.6:0/708306236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:04.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:44:03.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:04.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:44:03.737Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:05.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:04 smithi006 bash[27292]: cluster 2023-08-27T22:44:03.983411+0000 mgr.y (mgr.14901) 1366 : cluster [DBG] pgmap v1373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:05.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:04 smithi006 bash[27292]: audit 2023-08-27T22:44:04.565413+0000 mon.a (mon.0) 1540 : audit [DBG] from='client.? 172.21.15.6:0/1475108305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:05.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:04 smithi006 bash[33918]: cluster 2023-08-27T22:44:03.983411+0000 mgr.y (mgr.14901) 1366 : cluster [DBG] pgmap v1373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:05.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:04 smithi006 bash[33918]: audit 2023-08-27T22:44:04.565413+0000 mon.a (mon.0) 1540 : audit [DBG] from='client.? 172.21.15.6:0/1475108305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:04 smithi130 bash[27579]: cluster 2023-08-27T22:44:03.983411+0000 mgr.y (mgr.14901) 1366 : cluster [DBG] pgmap v1373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:04 smithi130 bash[27579]: audit 2023-08-27T22:44:04.565413+0000 mon.a (mon.0) 1540 : audit [DBG] from='client.? 172.21.15.6:0/1475108305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:07.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:07 smithi130 bash[27579]: cluster 2023-08-27T22:44:05.984487+0000 mgr.y (mgr.14901) 1367 : cluster [DBG] pgmap v1374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:07.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:07 smithi130 bash[27579]: audit 2023-08-27T22:44:06.985810+0000 mon.a (mon.0) 1541 : audit [DBG] from='client.? 172.21.15.6:0/3050559543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:07.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:07 smithi006 bash[33918]: cluster 2023-08-27T22:44:05.984487+0000 mgr.y (mgr.14901) 1367 : cluster [DBG] pgmap v1374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:07.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:07 smithi006 bash[33918]: audit 2023-08-27T22:44:06.985810+0000 mon.a (mon.0) 1541 : audit [DBG] from='client.? 172.21.15.6:0/3050559543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:07.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:07 smithi006 bash[27292]: cluster 2023-08-27T22:44:05.984487+0000 mgr.y (mgr.14901) 1367 : cluster [DBG] pgmap v1374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:07.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:07 smithi006 bash[27292]: audit 2023-08-27T22:44:06.985810+0000 mon.a (mon.0) 1541 : audit [DBG] from='client.? 172.21.15.6:0/3050559543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:09.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:09 smithi130 bash[27579]: cluster 2023-08-27T22:44:07.985169+0000 mgr.y (mgr.14901) 1368 : cluster [DBG] pgmap v1375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:09.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:09 smithi006 bash[33918]: cluster 2023-08-27T22:44:07.985169+0000 mgr.y (mgr.14901) 1368 : cluster [DBG] pgmap v1375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:09.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:09 smithi006 bash[27292]: cluster 2023-08-27T22:44:07.985169+0000 mgr.y (mgr.14901) 1368 : cluster [DBG] pgmap v1375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:10.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:10 smithi130 bash[27579]: audit 2023-08-27T22:44:09.398551+0000 mon.a (mon.0) 1542 : audit [DBG] from='client.? 172.21.15.6:0/3502701079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:10.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:10 smithi006 bash[27292]: audit 2023-08-27T22:44:09.398551+0000 mon.a (mon.0) 1542 : audit [DBG] from='client.? 172.21.15.6:0/3502701079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:10.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:10 smithi006 bash[33918]: audit 2023-08-27T22:44:09.398551+0000 mon.a (mon.0) 1542 : audit [DBG] from='client.? 172.21.15.6:0/3502701079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:10.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:44:10.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:10.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:44:10.275Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:11.067 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:44:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:44:10] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T22:44:11.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:11 smithi130 bash[27579]: cluster 2023-08-27T22:44:09.986308+0000 mgr.y (mgr.14901) 1369 : cluster [DBG] pgmap v1376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:11.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:11 smithi006 bash[27292]: cluster 2023-08-27T22:44:09.986308+0000 mgr.y (mgr.14901) 1369 : cluster [DBG] pgmap v1376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:11.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:11 smithi006 bash[33918]: cluster 2023-08-27T22:44:09.986308+0000 mgr.y (mgr.14901) 1369 : cluster [DBG] pgmap v1376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:12.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:12 smithi006 bash[27292]: audit 2023-08-27T22:44:11.824949+0000 mon.c (mon.2) 557 : audit [DBG] from='client.? 172.21.15.6:0/1587102184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:12.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:12 smithi006 bash[33918]: audit 2023-08-27T22:44:11.824949+0000 mon.c (mon.2) 557 : audit [DBG] from='client.? 172.21.15.6:0/1587102184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:44:11.840Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:44:11.842Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:12.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:12 smithi130 bash[27579]: audit 2023-08-27T22:44:11.824949+0000 mon.c (mon.2) 557 : audit [DBG] from='client.? 172.21.15.6:0/1587102184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:13.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:13 smithi130 bash[27579]: cluster 2023-08-27T22:44:11.987013+0000 mgr.y (mgr.14901) 1370 : cluster [DBG] pgmap v1377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:13.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:13 smithi130 bash[27579]: audit 2023-08-27T22:44:12.671413+0000 mon.b (mon.1) 254 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:44:13.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:13 smithi006 bash[27292]: cluster 2023-08-27T22:44:11.987013+0000 mgr.y (mgr.14901) 1370 : cluster [DBG] pgmap v1377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:13.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:13 smithi006 bash[27292]: audit 2023-08-27T22:44:12.671413+0000 mon.b (mon.1) 254 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:44:13.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:13 smithi006 bash[33918]: cluster 2023-08-27T22:44:11.987013+0000 mgr.y (mgr.14901) 1370 : cluster [DBG] pgmap v1377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:13.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:13 smithi006 bash[33918]: audit 2023-08-27T22:44:12.671413+0000 mon.b (mon.1) 254 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:44:14.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:44:13.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:14.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:44:13.738Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:15.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:14 smithi006 bash[27292]: cluster 2023-08-27T22:44:13.987551+0000 mgr.y (mgr.14901) 1371 : cluster [DBG] pgmap v1378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:15.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:14 smithi006 bash[27292]: audit 2023-08-27T22:44:14.262452+0000 mon.c (mon.2) 558 : audit [DBG] from='client.? 172.21.15.6:0/2199956800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:15.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:14 smithi006 bash[33918]: cluster 2023-08-27T22:44:13.987551+0000 mgr.y (mgr.14901) 1371 : cluster [DBG] pgmap v1378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:15.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:14 smithi006 bash[33918]: audit 2023-08-27T22:44:14.262452+0000 mon.c (mon.2) 558 : audit [DBG] from='client.? 172.21.15.6:0/2199956800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:15.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:14 smithi130 bash[27579]: cluster 2023-08-27T22:44:13.987551+0000 mgr.y (mgr.14901) 1371 : cluster [DBG] pgmap v1378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:15.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:14 smithi130 bash[27579]: audit 2023-08-27T22:44:14.262452+0000 mon.c (mon.2) 558 : audit [DBG] from='client.? 172.21.15.6:0/2199956800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:17.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:17 smithi130 bash[27579]: cluster 2023-08-27T22:44:15.988585+0000 mgr.y (mgr.14901) 1372 : cluster [DBG] pgmap v1379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:17.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:17 smithi130 bash[27579]: audit 2023-08-27T22:44:16.676369+0000 mon.a (mon.0) 1543 : audit [DBG] from='client.? 172.21.15.6:0/1367720726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:17.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:17 smithi006 bash[33918]: cluster 2023-08-27T22:44:15.988585+0000 mgr.y (mgr.14901) 1372 : cluster [DBG] pgmap v1379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:17.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:17 smithi006 bash[33918]: audit 2023-08-27T22:44:16.676369+0000 mon.a (mon.0) 1543 : audit [DBG] from='client.? 172.21.15.6:0/1367720726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:17.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:17 smithi006 bash[27292]: cluster 2023-08-27T22:44:15.988585+0000 mgr.y (mgr.14901) 1372 : cluster [DBG] pgmap v1379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:17.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:17 smithi006 bash[27292]: audit 2023-08-27T22:44:16.676369+0000 mon.a (mon.0) 1543 : audit [DBG] from='client.? 172.21.15.6:0/1367720726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:19.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:19 smithi130 bash[27579]: cluster 2023-08-27T22:44:17.989202+0000 mgr.y (mgr.14901) 1373 : cluster [DBG] pgmap v1380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:19.433 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:19 smithi130 bash[27579]: audit 2023-08-27T22:44:18.593096+0000 mon.a (mon.0) 1544 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:44:19.434 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:19 smithi130 bash[27579]: audit 2023-08-27T22:44:19.002350+0000 mon.a (mon.0) 1545 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:44:19.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:19 smithi006 bash[33918]: cluster 2023-08-27T22:44:17.989202+0000 mgr.y (mgr.14901) 1373 : cluster [DBG] pgmap v1380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:19.601 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:19 smithi006 bash[33918]: audit 2023-08-27T22:44:18.593096+0000 mon.a (mon.0) 1544 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:44:19.601 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:19 smithi006 bash[33918]: audit 2023-08-27T22:44:19.002350+0000 mon.a (mon.0) 1545 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:44:19.601 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:19 smithi006 bash[27292]: cluster 2023-08-27T22:44:17.989202+0000 mgr.y (mgr.14901) 1373 : cluster [DBG] pgmap v1380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:19.601 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:19 smithi006 bash[27292]: audit 2023-08-27T22:44:18.593096+0000 mon.a (mon.0) 1544 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:44:19.601 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:19 smithi006 bash[27292]: audit 2023-08-27T22:44:19.002350+0000 mon.a (mon.0) 1545 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:44:20.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:20 smithi130 bash[27579]: audit 2023-08-27T22:44:19.116998+0000 mon.a (mon.0) 1546 : audit [DBG] from='client.? 172.21.15.6:0/1905634151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:20.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:20 smithi130 bash[27579]: audit 2023-08-27T22:44:19.279494+0000 mon.b (mon.1) 255 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:44:20.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:20 smithi130 bash[27579]: audit 2023-08-27T22:44:19.281645+0000 mon.b (mon.1) 256 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:44:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:20 smithi130 bash[27579]: audit 2023-08-27T22:44:19.284705+0000 mon.a (mon.0) 1547 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:44:20.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:20 smithi006 bash[33918]: audit 2023-08-27T22:44:19.116998+0000 mon.a (mon.0) 1546 : audit [DBG] from='client.? 172.21.15.6:0/1905634151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:20.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:20 smithi006 bash[33918]: audit 2023-08-27T22:44:19.279494+0000 mon.b (mon.1) 255 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:44:20.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:20 smithi006 bash[33918]: audit 2023-08-27T22:44:19.281645+0000 mon.b (mon.1) 256 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:44:20.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:20 smithi006 bash[33918]: audit 2023-08-27T22:44:19.284705+0000 mon.a (mon.0) 1547 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:44:20.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:44:20.274Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:20.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:44:20.275Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:20.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:20 smithi006 bash[27292]: audit 2023-08-27T22:44:19.116998+0000 mon.a (mon.0) 1546 : audit [DBG] from='client.? 172.21.15.6:0/1905634151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:20.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:20 smithi006 bash[27292]: audit 2023-08-27T22:44:19.279494+0000 mon.b (mon.1) 255 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:44:20.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:20 smithi006 bash[27292]: audit 2023-08-27T22:44:19.281645+0000 mon.b (mon.1) 256 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:44:20.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:20 smithi006 bash[27292]: audit 2023-08-27T22:44:19.284705+0000 mon.a (mon.0) 1547 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:44:21.063 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:44:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:44:20] "GET /metrics HTTP/1.1" 200 193084 "" "Prometheus/2.33.4" 2023-08-27T22:44:21.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:21 smithi130 bash[27579]: cluster 2023-08-27T22:44:19.990291+0000 mgr.y (mgr.14901) 1374 : cluster [DBG] pgmap v1381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:21.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:21 smithi006 bash[33918]: cluster 2023-08-27T22:44:19.990291+0000 mgr.y (mgr.14901) 1374 : cluster [DBG] pgmap v1381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:21.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:21 smithi006 bash[27292]: cluster 2023-08-27T22:44:19.990291+0000 mgr.y (mgr.14901) 1374 : cluster [DBG] pgmap v1381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:22.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:22 smithi006 bash[27292]: audit 2023-08-27T22:44:21.526462+0000 mon.c (mon.2) 559 : audit [DBG] from='client.? 172.21.15.6:0/898954385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:22.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:22 smithi006 bash[33918]: audit 2023-08-27T22:44:21.526462+0000 mon.c (mon.2) 559 : audit [DBG] from='client.? 172.21.15.6:0/898954385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:44:21.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:22.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:44:21.842Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:22.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:22 smithi130 bash[27579]: audit 2023-08-27T22:44:21.526462+0000 mon.c (mon.2) 559 : audit [DBG] from='client.? 172.21.15.6:0/898954385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:23.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:23 smithi130 bash[27579]: cluster 2023-08-27T22:44:21.990945+0000 mgr.y (mgr.14901) 1375 : cluster [DBG] pgmap v1382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:23.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:23 smithi006 bash[33918]: cluster 2023-08-27T22:44:21.990945+0000 mgr.y (mgr.14901) 1375 : cluster [DBG] pgmap v1382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:23.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:23 smithi006 bash[27292]: cluster 2023-08-27T22:44:21.990945+0000 mgr.y (mgr.14901) 1375 : cluster [DBG] pgmap v1382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:24.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:44:23.737Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:24.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:44:23.739Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:25.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:24 smithi006 bash[33918]: audit 2023-08-27T22:44:23.942416+0000 mon.c (mon.2) 560 : audit [DBG] from='client.? 172.21.15.6:0/3132344619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:25.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:24 smithi006 bash[33918]: cluster 2023-08-27T22:44:23.991745+0000 mgr.y (mgr.14901) 1376 : cluster [DBG] pgmap v1383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:25.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:24 smithi006 bash[27292]: audit 2023-08-27T22:44:23.942416+0000 mon.c (mon.2) 560 : audit [DBG] from='client.? 172.21.15.6:0/3132344619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:25.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:24 smithi006 bash[27292]: cluster 2023-08-27T22:44:23.991745+0000 mgr.y (mgr.14901) 1376 : cluster [DBG] pgmap v1383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:25.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:24 smithi130 bash[27579]: audit 2023-08-27T22:44:23.942416+0000 mon.c (mon.2) 560 : audit [DBG] from='client.? 172.21.15.6:0/3132344619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:25.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:24 smithi130 bash[27579]: cluster 2023-08-27T22:44:23.991745+0000 mgr.y (mgr.14901) 1376 : cluster [DBG] pgmap v1383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:27.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:27 smithi130 bash[27579]: cluster 2023-08-27T22:44:25.992815+0000 mgr.y (mgr.14901) 1377 : cluster [DBG] pgmap v1384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:27.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:27 smithi130 bash[27579]: audit 2023-08-27T22:44:26.366020+0000 mon.a (mon.0) 1548 : audit [DBG] from='client.? 172.21.15.6:0/3253883026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:27.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:27 smithi006 bash[33918]: cluster 2023-08-27T22:44:25.992815+0000 mgr.y (mgr.14901) 1377 : cluster [DBG] pgmap v1384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:27.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:27 smithi006 bash[33918]: audit 2023-08-27T22:44:26.366020+0000 mon.a (mon.0) 1548 : audit [DBG] from='client.? 172.21.15.6:0/3253883026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:27.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:27 smithi006 bash[27292]: cluster 2023-08-27T22:44:25.992815+0000 mgr.y (mgr.14901) 1377 : cluster [DBG] pgmap v1384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:27.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:27 smithi006 bash[27292]: audit 2023-08-27T22:44:26.366020+0000 mon.a (mon.0) 1548 : audit [DBG] from='client.? 172.21.15.6:0/3253883026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:29.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:29 smithi130 bash[27579]: cluster 2023-08-27T22:44:27.993495+0000 mgr.y (mgr.14901) 1378 : cluster [DBG] pgmap v1385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:29.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:29 smithi130 bash[27579]: audit 2023-08-27T22:44:28.776231+0000 mon.a (mon.0) 1549 : audit [DBG] from='client.? 172.21.15.6:0/3907504292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:29.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:29 smithi006 bash[27292]: cluster 2023-08-27T22:44:27.993495+0000 mgr.y (mgr.14901) 1378 : cluster [DBG] pgmap v1385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:29.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:29 smithi006 bash[27292]: audit 2023-08-27T22:44:28.776231+0000 mon.a (mon.0) 1549 : audit [DBG] from='client.? 172.21.15.6:0/3907504292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:29.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:29 smithi006 bash[33918]: cluster 2023-08-27T22:44:27.993495+0000 mgr.y (mgr.14901) 1378 : cluster [DBG] pgmap v1385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:29.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:29 smithi006 bash[33918]: audit 2023-08-27T22:44:28.776231+0000 mon.a (mon.0) 1549 : audit [DBG] from='client.? 172.21.15.6:0/3907504292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:30.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:44:30.275Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:30.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:44:30.277Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:31.066 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:44:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:44:30] "GET /metrics HTTP/1.1" 200 193081 "" "Prometheus/2.33.4" 2023-08-27T22:44:31.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:31 smithi130 bash[27579]: cluster 2023-08-27T22:44:29.994629+0000 mgr.y (mgr.14901) 1379 : cluster [DBG] pgmap v1386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:31.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:31 smithi006 bash[27292]: cluster 2023-08-27T22:44:29.994629+0000 mgr.y (mgr.14901) 1379 : cluster [DBG] pgmap v1386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:31.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:31 smithi006 bash[33918]: cluster 2023-08-27T22:44:29.994629+0000 mgr.y (mgr.14901) 1379 : cluster [DBG] pgmap v1386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:32.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:32 smithi006 bash[33918]: audit 2023-08-27T22:44:31.196324+0000 mon.c (mon.2) 561 : audit [DBG] from='client.? 172.21.15.6:0/1053040939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:32.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:32 smithi006 bash[27292]: audit 2023-08-27T22:44:31.196324+0000 mon.c (mon.2) 561 : audit [DBG] from='client.? 172.21.15.6:0/1053040939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:44:31.842Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:44:31.843Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:32.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:32 smithi130 bash[27579]: audit 2023-08-27T22:44:31.196324+0000 mon.c (mon.2) 561 : audit [DBG] from='client.? 172.21.15.6:0/1053040939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:33.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:33 smithi130 bash[27579]: cluster 2023-08-27T22:44:31.995452+0000 mgr.y (mgr.14901) 1380 : cluster [DBG] pgmap v1387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:33.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:33 smithi006 bash[27292]: cluster 2023-08-27T22:44:31.995452+0000 mgr.y (mgr.14901) 1380 : cluster [DBG] pgmap v1387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:33.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:33 smithi006 bash[33918]: cluster 2023-08-27T22:44:31.995452+0000 mgr.y (mgr.14901) 1380 : cluster [DBG] pgmap v1387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:34.081 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:44:33.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:34.082 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:44:33.740Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:34.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:34 smithi130 bash[27579]: audit 2023-08-27T22:44:33.606725+0000 mon.a (mon.0) 1550 : audit [DBG] from='client.? 172.21.15.6:0/3571421684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:34.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:34 smithi006 bash[33918]: audit 2023-08-27T22:44:33.606725+0000 mon.a (mon.0) 1550 : audit [DBG] from='client.? 172.21.15.6:0/3571421684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:34.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:34 smithi006 bash[27292]: audit 2023-08-27T22:44:33.606725+0000 mon.a (mon.0) 1550 : audit [DBG] from='client.? 172.21.15.6:0/3571421684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:35.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:35 smithi130 bash[27579]: cluster 2023-08-27T22:44:33.996435+0000 mgr.y (mgr.14901) 1381 : cluster [DBG] pgmap v1388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:35.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:35 smithi006 bash[33918]: cluster 2023-08-27T22:44:33.996435+0000 mgr.y (mgr.14901) 1381 : cluster [DBG] pgmap v1388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:35.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:35 smithi006 bash[27292]: cluster 2023-08-27T22:44:33.996435+0000 mgr.y (mgr.14901) 1381 : cluster [DBG] pgmap v1388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:36.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:36 smithi130 bash[27579]: audit 2023-08-27T22:44:36.017120+0000 mon.c (mon.2) 562 : audit [DBG] from='client.? 172.21.15.6:0/3518295363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:36.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:36 smithi006 bash[33918]: audit 2023-08-27T22:44:36.017120+0000 mon.c (mon.2) 562 : audit [DBG] from='client.? 172.21.15.6:0/3518295363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:36.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:36 smithi006 bash[27292]: audit 2023-08-27T22:44:36.017120+0000 mon.c (mon.2) 562 : audit [DBG] from='client.? 172.21.15.6:0/3518295363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:37.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:37 smithi130 bash[27579]: cluster 2023-08-27T22:44:35.997489+0000 mgr.y (mgr.14901) 1382 : cluster [DBG] pgmap v1389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:37.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:37 smithi006 bash[27292]: cluster 2023-08-27T22:44:35.997489+0000 mgr.y (mgr.14901) 1382 : cluster [DBG] pgmap v1389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:37.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:37 smithi006 bash[33918]: cluster 2023-08-27T22:44:35.997489+0000 mgr.y (mgr.14901) 1382 : cluster [DBG] pgmap v1389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:39.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:39 smithi130 bash[27579]: cluster 2023-08-27T22:44:37.998226+0000 mgr.y (mgr.14901) 1383 : cluster [DBG] pgmap v1390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:39.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:39 smithi130 bash[27579]: audit 2023-08-27T22:44:38.442887+0000 mon.c (mon.2) 563 : audit [DBG] from='client.? 172.21.15.6:0/1649156359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:39.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:39 smithi006 bash[33918]: cluster 2023-08-27T22:44:37.998226+0000 mgr.y (mgr.14901) 1383 : cluster [DBG] pgmap v1390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:39.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:39 smithi006 bash[33918]: audit 2023-08-27T22:44:38.442887+0000 mon.c (mon.2) 563 : audit [DBG] from='client.? 172.21.15.6:0/1649156359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:39.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:39 smithi006 bash[27292]: cluster 2023-08-27T22:44:37.998226+0000 mgr.y (mgr.14901) 1383 : cluster [DBG] pgmap v1390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:39.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:39 smithi006 bash[27292]: audit 2023-08-27T22:44:38.442887+0000 mon.c (mon.2) 563 : audit [DBG] from='client.? 172.21.15.6:0/1649156359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:40.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:44:40.276Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:40.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:44:40.278Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:41.105 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:44:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:44:40] "GET /metrics HTTP/1.1" 200 193081 "" "Prometheus/2.33.4" 2023-08-27T22:44:41.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:41 smithi130 bash[27579]: cluster 2023-08-27T22:44:39.999529+0000 mgr.y (mgr.14901) 1384 : cluster [DBG] pgmap v1391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:41.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:41 smithi130 bash[27579]: audit 2023-08-27T22:44:40.858280+0000 mon.c (mon.2) 564 : audit [DBG] from='client.? 172.21.15.6:0/1748573787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:41.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:41 smithi006 bash[33918]: cluster 2023-08-27T22:44:39.999529+0000 mgr.y (mgr.14901) 1384 : cluster [DBG] pgmap v1391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:41.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:41 smithi006 bash[33918]: audit 2023-08-27T22:44:40.858280+0000 mon.c (mon.2) 564 : audit [DBG] from='client.? 172.21.15.6:0/1748573787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:41.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:41 smithi006 bash[27292]: cluster 2023-08-27T22:44:39.999529+0000 mgr.y (mgr.14901) 1384 : cluster [DBG] pgmap v1391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:41.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:41 smithi006 bash[27292]: audit 2023-08-27T22:44:40.858280+0000 mon.c (mon.2) 564 : audit [DBG] from='client.? 172.21.15.6:0/1748573787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:44:41.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:44:41.845Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:43.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:43 smithi130 bash[27579]: cluster 2023-08-27T22:44:42.000265+0000 mgr.y (mgr.14901) 1385 : cluster [DBG] pgmap v1392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:43.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:43 smithi006 bash[27292]: cluster 2023-08-27T22:44:42.000265+0000 mgr.y (mgr.14901) 1385 : cluster [DBG] pgmap v1392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:43.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:43 smithi006 bash[33918]: cluster 2023-08-27T22:44:42.000265+0000 mgr.y (mgr.14901) 1385 : cluster [DBG] pgmap v1392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:44.112 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:44:43.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:44.112 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:44:43.740Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:44 smithi130 bash[27579]: audit 2023-08-27T22:44:43.276143+0000 mon.c (mon.2) 565 : audit [DBG] from='client.? 172.21.15.6:0/219413467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:44.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:44 smithi006 bash[27292]: audit 2023-08-27T22:44:43.276143+0000 mon.c (mon.2) 565 : audit [DBG] from='client.? 172.21.15.6:0/219413467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:44.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:44 smithi006 bash[33918]: audit 2023-08-27T22:44:43.276143+0000 mon.c (mon.2) 565 : audit [DBG] from='client.? 172.21.15.6:0/219413467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:45.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:45 smithi130 bash[27579]: cluster 2023-08-27T22:44:44.001297+0000 mgr.y (mgr.14901) 1386 : cluster [DBG] pgmap v1393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:45.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:45 smithi006 bash[27292]: cluster 2023-08-27T22:44:44.001297+0000 mgr.y (mgr.14901) 1386 : cluster [DBG] pgmap v1393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:45.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:45 smithi006 bash[33918]: cluster 2023-08-27T22:44:44.001297+0000 mgr.y (mgr.14901) 1386 : cluster [DBG] pgmap v1393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:46.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:46 smithi006 bash[27292]: audit 2023-08-27T22:44:45.682300+0000 mon.a (mon.0) 1551 : audit [DBG] from='client.? 172.21.15.6:0/238048668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:46.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:46 smithi006 bash[33918]: audit 2023-08-27T22:44:45.682300+0000 mon.a (mon.0) 1551 : audit [DBG] from='client.? 172.21.15.6:0/238048668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:46.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:46 smithi130 bash[27579]: audit 2023-08-27T22:44:45.682300+0000 mon.a (mon.0) 1551 : audit [DBG] from='client.? 172.21.15.6:0/238048668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:47.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:47 smithi006 bash[27292]: cluster 2023-08-27T22:44:46.002260+0000 mgr.y (mgr.14901) 1387 : cluster [DBG] pgmap v1394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:47.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:47 smithi006 bash[33918]: cluster 2023-08-27T22:44:46.002260+0000 mgr.y (mgr.14901) 1387 : cluster [DBG] pgmap v1394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:47.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:47 smithi130 bash[27579]: cluster 2023-08-27T22:44:46.002260+0000 mgr.y (mgr.14901) 1387 : cluster [DBG] pgmap v1394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:48.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:48 smithi006 bash[27292]: audit 2023-08-27T22:44:48.106686+0000 mon.c (mon.2) 566 : audit [DBG] from='client.? 172.21.15.6:0/118355047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:48.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:48 smithi006 bash[33918]: audit 2023-08-27T22:44:48.106686+0000 mon.c (mon.2) 566 : audit [DBG] from='client.? 172.21.15.6:0/118355047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:48.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:48 smithi130 bash[27579]: audit 2023-08-27T22:44:48.106686+0000 mon.c (mon.2) 566 : audit [DBG] from='client.? 172.21.15.6:0/118355047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:49.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:49 smithi006 bash[27292]: cluster 2023-08-27T22:44:48.002927+0000 mgr.y (mgr.14901) 1388 : cluster [DBG] pgmap v1395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:49.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:49 smithi006 bash[33918]: cluster 2023-08-27T22:44:48.002927+0000 mgr.y (mgr.14901) 1388 : cluster [DBG] pgmap v1395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:49.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:49 smithi130 bash[27579]: cluster 2023-08-27T22:44:48.002927+0000 mgr.y (mgr.14901) 1388 : cluster [DBG] pgmap v1395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:50.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:44:50.276Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:50.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:44:50.278Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:51.142 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:44:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:44:50] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T22:44:51.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:51 smithi006 bash[33918]: cluster 2023-08-27T22:44:50.004359+0000 mgr.y (mgr.14901) 1389 : cluster [DBG] pgmap v1396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:51.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:51 smithi006 bash[33918]: audit 2023-08-27T22:44:50.534957+0000 mon.a (mon.0) 1552 : audit [DBG] from='client.? 172.21.15.6:0/219191121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:51.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:51 smithi006 bash[27292]: cluster 2023-08-27T22:44:50.004359+0000 mgr.y (mgr.14901) 1389 : cluster [DBG] pgmap v1396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:51.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:51 smithi006 bash[27292]: audit 2023-08-27T22:44:50.534957+0000 mon.a (mon.0) 1552 : audit [DBG] from='client.? 172.21.15.6:0/219191121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:51.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:51 smithi130 bash[27579]: cluster 2023-08-27T22:44:50.004359+0000 mgr.y (mgr.14901) 1389 : cluster [DBG] pgmap v1396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:51.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:51 smithi130 bash[27579]: audit 2023-08-27T22:44:50.534957+0000 mon.a (mon.0) 1552 : audit [DBG] from='client.? 172.21.15.6:0/219191121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:44:51.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:44:51.845Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:53.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:53 smithi006 bash[27292]: cluster 2023-08-27T22:44:52.005082+0000 mgr.y (mgr.14901) 1390 : cluster [DBG] pgmap v1397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:53.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:53 smithi006 bash[27292]: audit 2023-08-27T22:44:52.950600+0000 mon.a (mon.0) 1553 : audit [DBG] from='client.? 172.21.15.6:0/3722016196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:53 smithi006 bash[33918]: cluster 2023-08-27T22:44:52.005082+0000 mgr.y (mgr.14901) 1390 : cluster [DBG] pgmap v1397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:53 smithi006 bash[33918]: audit 2023-08-27T22:44:52.950600+0000 mon.a (mon.0) 1553 : audit [DBG] from='client.? 172.21.15.6:0/3722016196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:53.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:53 smithi130 bash[27579]: cluster 2023-08-27T22:44:52.005082+0000 mgr.y (mgr.14901) 1390 : cluster [DBG] pgmap v1397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:53.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:53 smithi130 bash[27579]: audit 2023-08-27T22:44:52.950600+0000 mon.a (mon.0) 1553 : audit [DBG] from='client.? 172.21.15.6:0/3722016196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:54.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:44:53.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:54.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:44:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:44:53.741Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:44:55.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:54 smithi006 bash[27292]: cluster 2023-08-27T22:44:54.005848+0000 mgr.y (mgr.14901) 1391 : cluster [DBG] pgmap v1398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:55.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:54 smithi006 bash[33918]: cluster 2023-08-27T22:44:54.005848+0000 mgr.y (mgr.14901) 1391 : cluster [DBG] pgmap v1398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:55.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:54 smithi130 bash[27579]: cluster 2023-08-27T22:44:54.005848+0000 mgr.y (mgr.14901) 1391 : cluster [DBG] pgmap v1398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:56.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:55 smithi006 bash[27292]: audit 2023-08-27T22:44:55.369145+0000 mon.c (mon.2) 567 : audit [DBG] from='client.? 172.21.15.6:0/157429108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:56.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:55 smithi006 bash[33918]: audit 2023-08-27T22:44:55.369145+0000 mon.c (mon.2) 567 : audit [DBG] from='client.? 172.21.15.6:0/157429108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:56.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:55 smithi130 bash[27579]: audit 2023-08-27T22:44:55.369145+0000 mon.c (mon.2) 567 : audit [DBG] from='client.? 172.21.15.6:0/157429108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:57.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:56 smithi006 bash[33918]: cluster 2023-08-27T22:44:56.006715+0000 mgr.y (mgr.14901) 1392 : cluster [DBG] pgmap v1399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:57.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:56 smithi006 bash[27292]: cluster 2023-08-27T22:44:56.006715+0000 mgr.y (mgr.14901) 1392 : cluster [DBG] pgmap v1399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:57.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:56 smithi130 bash[27579]: cluster 2023-08-27T22:44:56.006715+0000 mgr.y (mgr.14901) 1392 : cluster [DBG] pgmap v1399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:58.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:57 smithi006 bash[27292]: audit 2023-08-27T22:44:57.778153+0000 mon.c (mon.2) 568 : audit [DBG] from='client.? 172.21.15.6:0/114522504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:58.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:57 smithi006 bash[33918]: audit 2023-08-27T22:44:57.778153+0000 mon.c (mon.2) 568 : audit [DBG] from='client.? 172.21.15.6:0/114522504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:58.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:57 smithi130 bash[27579]: audit 2023-08-27T22:44:57.778153+0000 mon.c (mon.2) 568 : audit [DBG] from='client.? 172.21.15.6:0/114522504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:59.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:44:58 smithi006 bash[27292]: cluster 2023-08-27T22:44:58.007392+0000 mgr.y (mgr.14901) 1393 : cluster [DBG] pgmap v1400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:59.238 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:44:58 smithi006 bash[33918]: cluster 2023-08-27T22:44:58.007392+0000 mgr.y (mgr.14901) 1393 : cluster [DBG] pgmap v1400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:44:59.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:44:58 smithi130 bash[27579]: cluster 2023-08-27T22:44:58.007392+0000 mgr.y (mgr.14901) 1393 : cluster [DBG] pgmap v1400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:00.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:45:00.276Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:00.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:45:00.278Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:01.073 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:45:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:45:00] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T22:45:01.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:01 smithi130 bash[27579]: cluster 2023-08-27T22:45:00.009005+0000 mgr.y (mgr.14901) 1394 : cluster [DBG] pgmap v1401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:01.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:01 smithi130 bash[27579]: audit 2023-08-27T22:45:00.195554+0000 mon.a (mon.0) 1554 : audit [DBG] from='client.? 172.21.15.6:0/1245641855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:01.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:01 smithi006 bash[33918]: cluster 2023-08-27T22:45:00.009005+0000 mgr.y (mgr.14901) 1394 : cluster [DBG] pgmap v1401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:01.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:01 smithi006 bash[33918]: audit 2023-08-27T22:45:00.195554+0000 mon.a (mon.0) 1554 : audit [DBG] from='client.? 172.21.15.6:0/1245641855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:01.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:01 smithi006 bash[27292]: cluster 2023-08-27T22:45:00.009005+0000 mgr.y (mgr.14901) 1394 : cluster [DBG] pgmap v1401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:01.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:01 smithi006 bash[27292]: audit 2023-08-27T22:45:00.195554+0000 mon.a (mon.0) 1554 : audit [DBG] from='client.? 172.21.15.6:0/1245641855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:45:01.844Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:45:01.846Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:03.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:03 smithi130 bash[27579]: cluster 2023-08-27T22:45:02.009686+0000 mgr.y (mgr.14901) 1395 : cluster [DBG] pgmap v1402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:03.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:03 smithi130 bash[27579]: audit 2023-08-27T22:45:02.616075+0000 mon.a (mon.0) 1555 : audit [DBG] from='client.? 172.21.15.6:0/3274554516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:03.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:03 smithi006 bash[33918]: cluster 2023-08-27T22:45:02.009686+0000 mgr.y (mgr.14901) 1395 : cluster [DBG] pgmap v1402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:03.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:03 smithi006 bash[33918]: audit 2023-08-27T22:45:02.616075+0000 mon.a (mon.0) 1555 : audit [DBG] from='client.? 172.21.15.6:0/3274554516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:03.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:03 smithi006 bash[27292]: cluster 2023-08-27T22:45:02.009686+0000 mgr.y (mgr.14901) 1395 : cluster [DBG] pgmap v1402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:03.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:03 smithi006 bash[27292]: audit 2023-08-27T22:45:02.616075+0000 mon.a (mon.0) 1555 : audit [DBG] from='client.? 172.21.15.6:0/3274554516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:04.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:45:03.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:04.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:45:03.742Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:05.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:04 smithi006 bash[33918]: cluster 2023-08-27T22:45:04.010629+0000 mgr.y (mgr.14901) 1396 : cluster [DBG] pgmap v1403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:05.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:04 smithi006 bash[27292]: cluster 2023-08-27T22:45:04.010629+0000 mgr.y (mgr.14901) 1396 : cluster [DBG] pgmap v1403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:04 smithi130 bash[27579]: cluster 2023-08-27T22:45:04.010629+0000 mgr.y (mgr.14901) 1396 : cluster [DBG] pgmap v1403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:06.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:05 smithi006 bash[33918]: audit 2023-08-27T22:45:05.033669+0000 mon.a (mon.0) 1556 : audit [DBG] from='client.? 172.21.15.6:0/4263197218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:06.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:05 smithi006 bash[27292]: audit 2023-08-27T22:45:05.033669+0000 mon.a (mon.0) 1556 : audit [DBG] from='client.? 172.21.15.6:0/4263197218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:06.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:05 smithi130 bash[27579]: audit 2023-08-27T22:45:05.033669+0000 mon.a (mon.0) 1556 : audit [DBG] from='client.? 172.21.15.6:0/4263197218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:07.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:06 smithi006 bash[33918]: cluster 2023-08-27T22:45:06.011412+0000 mgr.y (mgr.14901) 1397 : cluster [DBG] pgmap v1404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:07.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:06 smithi006 bash[27292]: cluster 2023-08-27T22:45:06.011412+0000 mgr.y (mgr.14901) 1397 : cluster [DBG] pgmap v1404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:07.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:06 smithi130 bash[27579]: cluster 2023-08-27T22:45:06.011412+0000 mgr.y (mgr.14901) 1397 : cluster [DBG] pgmap v1404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:08.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:07 smithi006 bash[27292]: audit 2023-08-27T22:45:07.449181+0000 mon.a (mon.0) 1557 : audit [DBG] from='client.? 172.21.15.6:0/2645913286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:08.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:07 smithi006 bash[33918]: audit 2023-08-27T22:45:07.449181+0000 mon.a (mon.0) 1557 : audit [DBG] from='client.? 172.21.15.6:0/2645913286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:08.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:07 smithi130 bash[27579]: audit 2023-08-27T22:45:07.449181+0000 mon.a (mon.0) 1557 : audit [DBG] from='client.? 172.21.15.6:0/2645913286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:09.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:08 smithi006 bash[27292]: cluster 2023-08-27T22:45:08.012064+0000 mgr.y (mgr.14901) 1398 : cluster [DBG] pgmap v1405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:08 smithi006 bash[33918]: cluster 2023-08-27T22:45:08.012064+0000 mgr.y (mgr.14901) 1398 : cluster [DBG] pgmap v1405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:09.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:08 smithi130 bash[27579]: cluster 2023-08-27T22:45:08.012064+0000 mgr.y (mgr.14901) 1398 : cluster [DBG] pgmap v1405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:10.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:09 smithi006 bash[33918]: audit 2023-08-27T22:45:09.857555+0000 mon.a (mon.0) 1558 : audit [DBG] from='client.? 172.21.15.6:0/243823611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:10.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:09 smithi006 bash[27292]: audit 2023-08-27T22:45:09.857555+0000 mon.a (mon.0) 1558 : audit [DBG] from='client.? 172.21.15.6:0/243823611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:10.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:09 smithi130 bash[27579]: audit 2023-08-27T22:45:09.857555+0000 mon.a (mon.0) 1558 : audit [DBG] from='client.? 172.21.15.6:0/243823611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:10.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:45:10.278Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:10.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:45:10.280Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:11.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:10 smithi006 bash[27292]: cluster 2023-08-27T22:45:10.012759+0000 mgr.y (mgr.14901) 1399 : cluster [DBG] pgmap v1406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:11.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:45:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:45:10] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T22:45:11.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:10 smithi006 bash[33918]: cluster 2023-08-27T22:45:10.012759+0000 mgr.y (mgr.14901) 1399 : cluster [DBG] pgmap v1406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:11.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:10 smithi130 bash[27579]: cluster 2023-08-27T22:45:10.012759+0000 mgr.y (mgr.14901) 1399 : cluster [DBG] pgmap v1406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:45:11.845Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:45:11.847Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:13.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:13 smithi130 bash[27579]: cluster 2023-08-27T22:45:12.013180+0000 mgr.y (mgr.14901) 1400 : cluster [DBG] pgmap v1407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:13.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:13 smithi130 bash[27579]: audit 2023-08-27T22:45:12.277914+0000 mon.a (mon.0) 1559 : audit [DBG] from='client.? 172.21.15.6:0/1845994738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:13.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:13 smithi006 bash[33918]: cluster 2023-08-27T22:45:12.013180+0000 mgr.y (mgr.14901) 1400 : cluster [DBG] pgmap v1407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:13.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:13 smithi006 bash[33918]: audit 2023-08-27T22:45:12.277914+0000 mon.a (mon.0) 1559 : audit [DBG] from='client.? 172.21.15.6:0/1845994738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:13.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:13 smithi006 bash[27292]: cluster 2023-08-27T22:45:12.013180+0000 mgr.y (mgr.14901) 1400 : cluster [DBG] pgmap v1407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:13.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:13 smithi006 bash[27292]: audit 2023-08-27T22:45:12.277914+0000 mon.a (mon.0) 1559 : audit [DBG] from='client.? 172.21.15.6:0/1845994738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:14.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:45:13.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:14.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:45:13.742Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:15.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:14 smithi006 bash[27292]: cluster 2023-08-27T22:45:14.014027+0000 mgr.y (mgr.14901) 1401 : cluster [DBG] pgmap v1408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:15.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:14 smithi006 bash[27292]: audit 2023-08-27T22:45:14.692465+0000 mon.a (mon.0) 1560 : audit [DBG] from='client.? 172.21.15.6:0/4138077532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:15.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:14 smithi006 bash[33918]: cluster 2023-08-27T22:45:14.014027+0000 mgr.y (mgr.14901) 1401 : cluster [DBG] pgmap v1408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:15.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:14 smithi006 bash[33918]: audit 2023-08-27T22:45:14.692465+0000 mon.a (mon.0) 1560 : audit [DBG] from='client.? 172.21.15.6:0/4138077532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:15.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:14 smithi130 bash[27579]: cluster 2023-08-27T22:45:14.014027+0000 mgr.y (mgr.14901) 1401 : cluster [DBG] pgmap v1408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:15.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:14 smithi130 bash[27579]: audit 2023-08-27T22:45:14.692465+0000 mon.a (mon.0) 1560 : audit [DBG] from='client.? 172.21.15.6:0/4138077532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:17.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:17 smithi130 bash[27579]: cluster 2023-08-27T22:45:16.014790+0000 mgr.y (mgr.14901) 1402 : cluster [DBG] pgmap v1409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:17.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:17 smithi006 bash[27292]: cluster 2023-08-27T22:45:16.014790+0000 mgr.y (mgr.14901) 1402 : cluster [DBG] pgmap v1409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:17.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:17 smithi006 bash[33918]: cluster 2023-08-27T22:45:16.014790+0000 mgr.y (mgr.14901) 1402 : cluster [DBG] pgmap v1409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:18.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:18 smithi130 bash[27579]: audit 2023-08-27T22:45:17.111596+0000 mon.c (mon.2) 569 : audit [DBG] from='client.? 172.21.15.6:0/3908451779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:18.691 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:18 smithi006 bash[27292]: audit 2023-08-27T22:45:17.111596+0000 mon.c (mon.2) 569 : audit [DBG] from='client.? 172.21.15.6:0/3908451779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:18.691 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:18 smithi006 bash[33918]: audit 2023-08-27T22:45:17.111596+0000 mon.c (mon.2) 569 : audit [DBG] from='client.? 172.21.15.6:0/3908451779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:19.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:19 smithi130 bash[27579]: cluster 2023-08-27T22:45:18.015678+0000 mgr.y (mgr.14901) 1403 : cluster [DBG] pgmap v1410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:19.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:19 smithi006 bash[33918]: cluster 2023-08-27T22:45:18.015678+0000 mgr.y (mgr.14901) 1403 : cluster [DBG] pgmap v1410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:19.709 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:19 smithi006 bash[27292]: cluster 2023-08-27T22:45:18.015678+0000 mgr.y (mgr.14901) 1403 : cluster [DBG] pgmap v1410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:20.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:20 smithi130 bash[27579]: audit 2023-08-27T22:45:19.301680+0000 mon.b (mon.1) 257 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:45:20.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:20 smithi130 bash[27579]: audit 2023-08-27T22:45:19.537963+0000 mon.c (mon.2) 570 : audit [DBG] from='client.? 172.21.15.6:0/1763635243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:20.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:20 smithi006 bash[27292]: audit 2023-08-27T22:45:19.301680+0000 mon.b (mon.1) 257 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:45:20.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:20 smithi006 bash[27292]: audit 2023-08-27T22:45:19.537963+0000 mon.c (mon.2) 570 : audit [DBG] from='client.? 172.21.15.6:0/1763635243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:20.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:20 smithi006 bash[33918]: audit 2023-08-27T22:45:19.301680+0000 mon.b (mon.1) 257 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:45:20.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:20 smithi006 bash[33918]: audit 2023-08-27T22:45:19.537963+0000 mon.c (mon.2) 570 : audit [DBG] from='client.? 172.21.15.6:0/1763635243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:20.473 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:45:20.278Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:20.473 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:45:20.280Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:21.108 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:45:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:45:20] "GET /metrics HTTP/1.1" 200 193084 "" "Prometheus/2.33.4" 2023-08-27T22:45:21.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:21 smithi130 bash[27579]: cluster 2023-08-27T22:45:20.016603+0000 mgr.y (mgr.14901) 1404 : cluster [DBG] pgmap v1411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:21.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:21 smithi006 bash[33918]: cluster 2023-08-27T22:45:20.016603+0000 mgr.y (mgr.14901) 1404 : cluster [DBG] pgmap v1411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:21.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:21 smithi006 bash[27292]: cluster 2023-08-27T22:45:20.016603+0000 mgr.y (mgr.14901) 1404 : cluster [DBG] pgmap v1411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:22.118 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:45:21.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:22.118 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:45:21.848Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:22.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:22 smithi006 bash[33918]: audit 2023-08-27T22:45:21.952424+0000 mon.c (mon.2) 571 : audit [DBG] from='client.? 172.21.15.6:0/1737243011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:22.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:22 smithi006 bash[27292]: audit 2023-08-27T22:45:21.952424+0000 mon.c (mon.2) 571 : audit [DBG] from='client.? 172.21.15.6:0/1737243011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:22.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:22 smithi130 bash[27579]: audit 2023-08-27T22:45:21.952424+0000 mon.c (mon.2) 571 : audit [DBG] from='client.? 172.21.15.6:0/1737243011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:23.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:23 smithi006 bash[33918]: cluster 2023-08-27T22:45:22.017377+0000 mgr.y (mgr.14901) 1405 : cluster [DBG] pgmap v1412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:23.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:23 smithi006 bash[27292]: cluster 2023-08-27T22:45:22.017377+0000 mgr.y (mgr.14901) 1405 : cluster [DBG] pgmap v1412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:23.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:23 smithi130 bash[27579]: cluster 2023-08-27T22:45:22.017377+0000 mgr.y (mgr.14901) 1405 : cluster [DBG] pgmap v1412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:24.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:45:23.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:24.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:45:23.743Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:25.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:24 smithi006 bash[33918]: cluster 2023-08-27T22:45:24.018404+0000 mgr.y (mgr.14901) 1406 : cluster [DBG] pgmap v1413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:25.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:24 smithi006 bash[33918]: audit 2023-08-27T22:45:24.376652+0000 mon.a (mon.0) 1561 : audit [DBG] from='client.? 172.21.15.6:0/2785259719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:25.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:24 smithi006 bash[27292]: cluster 2023-08-27T22:45:24.018404+0000 mgr.y (mgr.14901) 1406 : cluster [DBG] pgmap v1413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:25.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:24 smithi006 bash[27292]: audit 2023-08-27T22:45:24.376652+0000 mon.a (mon.0) 1561 : audit [DBG] from='client.? 172.21.15.6:0/2785259719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:25.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:24 smithi130 bash[27579]: cluster 2023-08-27T22:45:24.018404+0000 mgr.y (mgr.14901) 1406 : cluster [DBG] pgmap v1413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:25.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:24 smithi130 bash[27579]: audit 2023-08-27T22:45:24.376652+0000 mon.a (mon.0) 1561 : audit [DBG] from='client.? 172.21.15.6:0/2785259719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:26.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:26 smithi006 bash[33918]: audit 2023-08-27T22:45:25.155653+0000 mon.a (mon.0) 1562 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:45:26.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:26 smithi006 bash[33918]: audit 2023-08-27T22:45:25.614942+0000 mon.a (mon.0) 1563 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:45:26.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:26 smithi006 bash[33918]: audit 2023-08-27T22:45:25.918496+0000 mon.b (mon.1) 258 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:45:26.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:26 smithi006 bash[33918]: audit 2023-08-27T22:45:25.920674+0000 mon.b (mon.1) 259 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:45:26.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:26 smithi006 bash[33918]: audit 2023-08-27T22:45:25.925374+0000 mon.a (mon.0) 1564 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:45:26.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:26 smithi006 bash[27292]: audit 2023-08-27T22:45:25.155653+0000 mon.a (mon.0) 1562 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:45:26.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:26 smithi006 bash[27292]: audit 2023-08-27T22:45:25.614942+0000 mon.a (mon.0) 1563 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:45:26.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:26 smithi006 bash[27292]: audit 2023-08-27T22:45:25.918496+0000 mon.b (mon.1) 258 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:45:26.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:26 smithi006 bash[27292]: audit 2023-08-27T22:45:25.920674+0000 mon.b (mon.1) 259 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:45:26.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:26 smithi006 bash[27292]: audit 2023-08-27T22:45:25.925374+0000 mon.a (mon.0) 1564 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:45:26.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:26 smithi130 bash[27579]: audit 2023-08-27T22:45:25.155653+0000 mon.a (mon.0) 1562 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:45:26.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:26 smithi130 bash[27579]: audit 2023-08-27T22:45:25.614942+0000 mon.a (mon.0) 1563 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:45:26.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:26 smithi130 bash[27579]: audit 2023-08-27T22:45:25.918496+0000 mon.b (mon.1) 258 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:45:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:26 smithi130 bash[27579]: audit 2023-08-27T22:45:25.920674+0000 mon.b (mon.1) 259 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:45:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:26 smithi130 bash[27579]: audit 2023-08-27T22:45:25.925374+0000 mon.a (mon.0) 1564 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:45:27.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:27 smithi006 bash[33918]: cluster 2023-08-27T22:45:26.019198+0000 mgr.y (mgr.14901) 1407 : cluster [DBG] pgmap v1414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:27.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:27 smithi006 bash[33918]: audit 2023-08-27T22:45:26.790856+0000 mon.c (mon.2) 572 : audit [DBG] from='client.? 172.21.15.6:0/4084928302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:27.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:27 smithi006 bash[27292]: cluster 2023-08-27T22:45:26.019198+0000 mgr.y (mgr.14901) 1407 : cluster [DBG] pgmap v1414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:27.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:27 smithi006 bash[27292]: audit 2023-08-27T22:45:26.790856+0000 mon.c (mon.2) 572 : audit [DBG] from='client.? 172.21.15.6:0/4084928302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:27.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:27 smithi130 bash[27579]: cluster 2023-08-27T22:45:26.019198+0000 mgr.y (mgr.14901) 1407 : cluster [DBG] pgmap v1414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:27.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:27 smithi130 bash[27579]: audit 2023-08-27T22:45:26.790856+0000 mon.c (mon.2) 572 : audit [DBG] from='client.? 172.21.15.6:0/4084928302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:29.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:29 smithi006 bash[27292]: cluster 2023-08-27T22:45:28.019925+0000 mgr.y (mgr.14901) 1408 : cluster [DBG] pgmap v1415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:29.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:29 smithi006 bash[33918]: cluster 2023-08-27T22:45:28.019925+0000 mgr.y (mgr.14901) 1408 : cluster [DBG] pgmap v1415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:29.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:29 smithi130 bash[27579]: cluster 2023-08-27T22:45:28.019925+0000 mgr.y (mgr.14901) 1408 : cluster [DBG] pgmap v1415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:30.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:30 smithi006 bash[27292]: audit 2023-08-27T22:45:29.204351+0000 mon.c (mon.2) 573 : audit [DBG] from='client.? 172.21.15.6:0/1576019459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:30.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:30 smithi006 bash[33918]: audit 2023-08-27T22:45:29.204351+0000 mon.c (mon.2) 573 : audit [DBG] from='client.? 172.21.15.6:0/1576019459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:30.473 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:45:30.278Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:30.473 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:45:30.280Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:30.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:30 smithi130 bash[27579]: audit 2023-08-27T22:45:29.204351+0000 mon.c (mon.2) 573 : audit [DBG] from='client.? 172.21.15.6:0/1576019459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:31.181 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:45:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:45:30] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T22:45:31.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:31 smithi006 bash[33918]: cluster 2023-08-27T22:45:30.021098+0000 mgr.y (mgr.14901) 1409 : cluster [DBG] pgmap v1416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:31.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:31 smithi006 bash[27292]: cluster 2023-08-27T22:45:30.021098+0000 mgr.y (mgr.14901) 1409 : cluster [DBG] pgmap v1416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:31.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:31 smithi130 bash[27579]: cluster 2023-08-27T22:45:30.021098+0000 mgr.y (mgr.14901) 1409 : cluster [DBG] pgmap v1416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:32.190 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:45:31.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:32.191 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:45:31.848Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:32.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:32 smithi006 bash[27292]: audit 2023-08-27T22:45:31.612058+0000 mon.c (mon.2) 574 : audit [DBG] from='client.? 172.21.15.6:0/3900710042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:32.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:32 smithi006 bash[33918]: audit 2023-08-27T22:45:31.612058+0000 mon.c (mon.2) 574 : audit [DBG] from='client.? 172.21.15.6:0/3900710042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:32.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:32 smithi130 bash[27579]: audit 2023-08-27T22:45:31.612058+0000 mon.c (mon.2) 574 : audit [DBG] from='client.? 172.21.15.6:0/3900710042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:33.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:33 smithi006 bash[33918]: cluster 2023-08-27T22:45:32.021825+0000 mgr.y (mgr.14901) 1410 : cluster [DBG] pgmap v1417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:33.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:33 smithi006 bash[27292]: cluster 2023-08-27T22:45:32.021825+0000 mgr.y (mgr.14901) 1410 : cluster [DBG] pgmap v1417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:33.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:33 smithi130 bash[27579]: cluster 2023-08-27T22:45:32.021825+0000 mgr.y (mgr.14901) 1410 : cluster [DBG] pgmap v1417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:34.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:45:33.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:34.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:45:33.744Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:35.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:34 smithi006 bash[27292]: cluster 2023-08-27T22:45:34.022591+0000 mgr.y (mgr.14901) 1411 : cluster [DBG] pgmap v1418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:35.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:34 smithi006 bash[27292]: audit 2023-08-27T22:45:34.027305+0000 mon.c (mon.2) 575 : audit [DBG] from='client.? 172.21.15.6:0/3627876591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:35.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:34 smithi006 bash[33918]: cluster 2023-08-27T22:45:34.022591+0000 mgr.y (mgr.14901) 1411 : cluster [DBG] pgmap v1418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:35.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:34 smithi006 bash[33918]: audit 2023-08-27T22:45:34.027305+0000 mon.c (mon.2) 575 : audit [DBG] from='client.? 172.21.15.6:0/3627876591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:35.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:34 smithi130 bash[27579]: cluster 2023-08-27T22:45:34.022591+0000 mgr.y (mgr.14901) 1411 : cluster [DBG] pgmap v1418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:35.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:34 smithi130 bash[27579]: audit 2023-08-27T22:45:34.027305+0000 mon.c (mon.2) 575 : audit [DBG] from='client.? 172.21.15.6:0/3627876591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:37.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:37 smithi130 bash[27579]: cluster 2023-08-27T22:45:36.023296+0000 mgr.y (mgr.14901) 1412 : cluster [DBG] pgmap v1419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:37.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:37 smithi130 bash[27579]: audit 2023-08-27T22:45:36.450191+0000 mon.a (mon.0) 1565 : audit [DBG] from='client.? 172.21.15.6:0/2300326492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:37.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:37 smithi006 bash[27292]: cluster 2023-08-27T22:45:36.023296+0000 mgr.y (mgr.14901) 1412 : cluster [DBG] pgmap v1419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:37.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:37 smithi006 bash[27292]: audit 2023-08-27T22:45:36.450191+0000 mon.a (mon.0) 1565 : audit [DBG] from='client.? 172.21.15.6:0/2300326492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:37.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:37 smithi006 bash[33918]: cluster 2023-08-27T22:45:36.023296+0000 mgr.y (mgr.14901) 1412 : cluster [DBG] pgmap v1419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:37.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:37 smithi006 bash[33918]: audit 2023-08-27T22:45:36.450191+0000 mon.a (mon.0) 1565 : audit [DBG] from='client.? 172.21.15.6:0/2300326492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:39.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:39 smithi130 bash[27579]: cluster 2023-08-27T22:45:38.023942+0000 mgr.y (mgr.14901) 1413 : cluster [DBG] pgmap v1420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:39.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:39 smithi130 bash[27579]: audit 2023-08-27T22:45:38.858248+0000 mon.a (mon.0) 1566 : audit [DBG] from='client.? 172.21.15.6:0/2845409301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:39.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:39 smithi006 bash[27292]: cluster 2023-08-27T22:45:38.023942+0000 mgr.y (mgr.14901) 1413 : cluster [DBG] pgmap v1420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:39.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:39 smithi006 bash[27292]: audit 2023-08-27T22:45:38.858248+0000 mon.a (mon.0) 1566 : audit [DBG] from='client.? 172.21.15.6:0/2845409301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:39.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:39 smithi006 bash[33918]: cluster 2023-08-27T22:45:38.023942+0000 mgr.y (mgr.14901) 1413 : cluster [DBG] pgmap v1420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:39.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:39 smithi006 bash[33918]: audit 2023-08-27T22:45:38.858248+0000 mon.a (mon.0) 1566 : audit [DBG] from='client.? 172.21.15.6:0/2845409301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:40.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:45:40.279Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:40.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:45:40.281Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:41.097 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:45:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:45:40] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T22:45:41.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:41 smithi130 bash[27579]: cluster 2023-08-27T22:45:40.025143+0000 mgr.y (mgr.14901) 1414 : cluster [DBG] pgmap v1421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:41.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:41 smithi006 bash[27292]: cluster 2023-08-27T22:45:40.025143+0000 mgr.y (mgr.14901) 1414 : cluster [DBG] pgmap v1421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:41.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:41 smithi006 bash[33918]: cluster 2023-08-27T22:45:40.025143+0000 mgr.y (mgr.14901) 1414 : cluster [DBG] pgmap v1421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:42.105 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:45:41.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:42.106 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:45:41.849Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:42.107 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:42 smithi006 bash[27292]: audit 2023-08-27T22:45:41.268356+0000 mon.a (mon.0) 1567 : audit [DBG] from='client.? 172.21.15.6:0/2547923918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:42.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:42 smithi130 bash[27579]: audit 2023-08-27T22:45:41.268356+0000 mon.a (mon.0) 1567 : audit [DBG] from='client.? 172.21.15.6:0/2547923918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:42.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:42 smithi006 bash[33918]: audit 2023-08-27T22:45:41.268356+0000 mon.a (mon.0) 1567 : audit [DBG] from='client.? 172.21.15.6:0/2547923918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:43.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:43 smithi130 bash[27579]: cluster 2023-08-27T22:45:42.025859+0000 mgr.y (mgr.14901) 1415 : cluster [DBG] pgmap v1422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:43.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:43 smithi006 bash[27292]: cluster 2023-08-27T22:45:42.025859+0000 mgr.y (mgr.14901) 1415 : cluster [DBG] pgmap v1422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:43.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:43 smithi006 bash[33918]: cluster 2023-08-27T22:45:42.025859+0000 mgr.y (mgr.14901) 1415 : cluster [DBG] pgmap v1422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:44.111 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:45:43.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:44.111 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:45:43.744Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:44.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:44 smithi130 bash[27579]: audit 2023-08-27T22:45:43.688785+0000 mon.a (mon.0) 1568 : audit [DBG] from='client.? 172.21.15.6:0/948225228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:44.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:44 smithi006 bash[27292]: audit 2023-08-27T22:45:43.688785+0000 mon.a (mon.0) 1568 : audit [DBG] from='client.? 172.21.15.6:0/948225228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:44.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:44 smithi006 bash[33918]: audit 2023-08-27T22:45:43.688785+0000 mon.a (mon.0) 1568 : audit [DBG] from='client.? 172.21.15.6:0/948225228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:45.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:45 smithi130 bash[27579]: cluster 2023-08-27T22:45:44.026830+0000 mgr.y (mgr.14901) 1416 : cluster [DBG] pgmap v1423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:45.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:45 smithi006 bash[33918]: cluster 2023-08-27T22:45:44.026830+0000 mgr.y (mgr.14901) 1416 : cluster [DBG] pgmap v1423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:45.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:45 smithi006 bash[27292]: cluster 2023-08-27T22:45:44.026830+0000 mgr.y (mgr.14901) 1416 : cluster [DBG] pgmap v1423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:46.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:46 smithi006 bash[33918]: audit 2023-08-27T22:45:46.108210+0000 mon.a (mon.0) 1569 : audit [DBG] from='client.? 172.21.15.6:0/313439698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:46.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:46 smithi006 bash[27292]: audit 2023-08-27T22:45:46.108210+0000 mon.a (mon.0) 1569 : audit [DBG] from='client.? 172.21.15.6:0/313439698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:46.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:46 smithi130 bash[27579]: audit 2023-08-27T22:45:46.108210+0000 mon.a (mon.0) 1569 : audit [DBG] from='client.? 172.21.15.6:0/313439698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:47.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:47 smithi006 bash[33918]: cluster 2023-08-27T22:45:46.027651+0000 mgr.y (mgr.14901) 1417 : cluster [DBG] pgmap v1424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:47.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:47 smithi006 bash[27292]: cluster 2023-08-27T22:45:46.027651+0000 mgr.y (mgr.14901) 1417 : cluster [DBG] pgmap v1424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:47.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:47 smithi130 bash[27579]: cluster 2023-08-27T22:45:46.027651+0000 mgr.y (mgr.14901) 1417 : cluster [DBG] pgmap v1424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:49.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:49 smithi006 bash[33918]: cluster 2023-08-27T22:45:48.028475+0000 mgr.y (mgr.14901) 1418 : cluster [DBG] pgmap v1425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:49.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:49 smithi006 bash[33918]: audit 2023-08-27T22:45:48.526933+0000 mon.a (mon.0) 1570 : audit [DBG] from='client.? 172.21.15.6:0/1993649547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:49.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:49 smithi006 bash[27292]: cluster 2023-08-27T22:45:48.028475+0000 mgr.y (mgr.14901) 1418 : cluster [DBG] pgmap v1425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:49.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:49 smithi006 bash[27292]: audit 2023-08-27T22:45:48.526933+0000 mon.a (mon.0) 1570 : audit [DBG] from='client.? 172.21.15.6:0/1993649547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:49.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:49 smithi130 bash[27579]: cluster 2023-08-27T22:45:48.028475+0000 mgr.y (mgr.14901) 1418 : cluster [DBG] pgmap v1425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:49.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:49 smithi130 bash[27579]: audit 2023-08-27T22:45:48.526933+0000 mon.a (mon.0) 1570 : audit [DBG] from='client.? 172.21.15.6:0/1993649547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:50.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:45:50.279Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:50.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:45:50.281Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:51.182 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:45:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:45:50] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:45:51.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:51 smithi006 bash[27292]: cluster 2023-08-27T22:45:50.029552+0000 mgr.y (mgr.14901) 1419 : cluster [DBG] pgmap v1426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:51.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:51 smithi006 bash[27292]: audit 2023-08-27T22:45:50.951653+0000 mon.a (mon.0) 1571 : audit [DBG] from='client.? 172.21.15.6:0/1503935467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:51.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:51 smithi006 bash[33918]: cluster 2023-08-27T22:45:50.029552+0000 mgr.y (mgr.14901) 1419 : cluster [DBG] pgmap v1426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:51.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:51 smithi006 bash[33918]: audit 2023-08-27T22:45:50.951653+0000 mon.a (mon.0) 1571 : audit [DBG] from='client.? 172.21.15.6:0/1503935467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:51.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:51 smithi130 bash[27579]: cluster 2023-08-27T22:45:50.029552+0000 mgr.y (mgr.14901) 1419 : cluster [DBG] pgmap v1426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:51.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:51 smithi130 bash[27579]: audit 2023-08-27T22:45:50.951653+0000 mon.a (mon.0) 1571 : audit [DBG] from='client.? 172.21.15.6:0/1503935467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:45:51.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:45:51.849Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:53.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:53 smithi006 bash[27292]: cluster 2023-08-27T22:45:52.030186+0000 mgr.y (mgr.14901) 1420 : cluster [DBG] pgmap v1427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:53 smithi006 bash[33918]: cluster 2023-08-27T22:45:52.030186+0000 mgr.y (mgr.14901) 1420 : cluster [DBG] pgmap v1427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:53.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:53 smithi130 bash[27579]: cluster 2023-08-27T22:45:52.030186+0000 mgr.y (mgr.14901) 1420 : cluster [DBG] pgmap v1427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:54.188 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:45:53.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:54.188 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:45:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:45:53.745Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:45:54.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:54 smithi006 bash[27292]: audit 2023-08-27T22:45:53.373414+0000 mon.a (mon.0) 1572 : audit [DBG] from='client.? 172.21.15.6:0/3914277326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:54.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:54 smithi006 bash[33918]: audit 2023-08-27T22:45:53.373414+0000 mon.a (mon.0) 1572 : audit [DBG] from='client.? 172.21.15.6:0/3914277326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:54.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:54 smithi130 bash[27579]: audit 2023-08-27T22:45:53.373414+0000 mon.a (mon.0) 1572 : audit [DBG] from='client.? 172.21.15.6:0/3914277326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:55.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:55 smithi006 bash[33918]: cluster 2023-08-27T22:45:54.030995+0000 mgr.y (mgr.14901) 1421 : cluster [DBG] pgmap v1428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:55.510 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:55 smithi006 bash[27292]: cluster 2023-08-27T22:45:54.030995+0000 mgr.y (mgr.14901) 1421 : cluster [DBG] pgmap v1428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:55.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:55 smithi130 bash[27579]: cluster 2023-08-27T22:45:54.030995+0000 mgr.y (mgr.14901) 1421 : cluster [DBG] pgmap v1428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:56.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:56 smithi006 bash[27292]: audit 2023-08-27T22:45:55.789405+0000 mon.a (mon.0) 1573 : audit [DBG] from='client.? 172.21.15.6:0/3299656918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:56.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:56 smithi006 bash[33918]: audit 2023-08-27T22:45:55.789405+0000 mon.a (mon.0) 1573 : audit [DBG] from='client.? 172.21.15.6:0/3299656918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:56.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:56 smithi130 bash[27579]: audit 2023-08-27T22:45:55.789405+0000 mon.a (mon.0) 1573 : audit [DBG] from='client.? 172.21.15.6:0/3299656918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:57.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:57 smithi006 bash[27292]: cluster 2023-08-27T22:45:56.031744+0000 mgr.y (mgr.14901) 1422 : cluster [DBG] pgmap v1429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:57.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:57 smithi006 bash[33918]: cluster 2023-08-27T22:45:56.031744+0000 mgr.y (mgr.14901) 1422 : cluster [DBG] pgmap v1429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:57.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:57 smithi130 bash[27579]: cluster 2023-08-27T22:45:56.031744+0000 mgr.y (mgr.14901) 1422 : cluster [DBG] pgmap v1429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:59.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:59 smithi006 bash[33918]: cluster 2023-08-27T22:45:58.032281+0000 mgr.y (mgr.14901) 1423 : cluster [DBG] pgmap v1430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:59.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:45:59 smithi006 bash[33918]: audit 2023-08-27T22:45:58.211537+0000 mon.c (mon.2) 576 : audit [DBG] from='client.? 172.21.15.6:0/3037010516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:59.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:59 smithi006 bash[27292]: cluster 2023-08-27T22:45:58.032281+0000 mgr.y (mgr.14901) 1423 : cluster [DBG] pgmap v1430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:59.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:45:59 smithi006 bash[27292]: audit 2023-08-27T22:45:58.211537+0000 mon.c (mon.2) 576 : audit [DBG] from='client.? 172.21.15.6:0/3037010516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:59.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:59 smithi130 bash[27579]: cluster 2023-08-27T22:45:58.032281+0000 mgr.y (mgr.14901) 1423 : cluster [DBG] pgmap v1430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:45:59.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:45:59 smithi130 bash[27579]: audit 2023-08-27T22:45:58.211537+0000 mon.c (mon.2) 576 : audit [DBG] from='client.? 172.21.15.6:0/3037010516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:00.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:46:00.279Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:00.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:46:00.281Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:46:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:46:00] "GET /metrics HTTP/1.1" 200 193084 "" "Prometheus/2.33.4" 2023-08-27T22:46:01.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:01 smithi006 bash[27292]: cluster 2023-08-27T22:46:00.033348+0000 mgr.y (mgr.14901) 1424 : cluster [DBG] pgmap v1431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:01.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:01 smithi006 bash[27292]: audit 2023-08-27T22:46:00.644801+0000 mon.c (mon.2) 577 : audit [DBG] from='client.? 172.21.15.6:0/2086565474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:01.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:01 smithi006 bash[33918]: cluster 2023-08-27T22:46:00.033348+0000 mgr.y (mgr.14901) 1424 : cluster [DBG] pgmap v1431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:01.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:01 smithi006 bash[33918]: audit 2023-08-27T22:46:00.644801+0000 mon.c (mon.2) 577 : audit [DBG] from='client.? 172.21.15.6:0/2086565474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:01.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:01 smithi130 bash[27579]: cluster 2023-08-27T22:46:00.033348+0000 mgr.y (mgr.14901) 1424 : cluster [DBG] pgmap v1431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:01.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:01 smithi130 bash[27579]: audit 2023-08-27T22:46:00.644801+0000 mon.c (mon.2) 577 : audit [DBG] from='client.? 172.21.15.6:0/2086565474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:46:01.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:46:01.850Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:03.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:03 smithi006 bash[27292]: cluster 2023-08-27T22:46:02.033984+0000 mgr.y (mgr.14901) 1425 : cluster [DBG] pgmap v1432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:03.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:03 smithi006 bash[27292]: audit 2023-08-27T22:46:03.053075+0000 mon.c (mon.2) 578 : audit [DBG] from='client.? 172.21.15.6:0/3661979670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:03.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:03 smithi006 bash[33918]: cluster 2023-08-27T22:46:02.033984+0000 mgr.y (mgr.14901) 1425 : cluster [DBG] pgmap v1432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:03.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:03 smithi006 bash[33918]: audit 2023-08-27T22:46:03.053075+0000 mon.c (mon.2) 578 : audit [DBG] from='client.? 172.21.15.6:0/3661979670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:03.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:03 smithi130 bash[27579]: cluster 2023-08-27T22:46:02.033984+0000 mgr.y (mgr.14901) 1425 : cluster [DBG] pgmap v1432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:03.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:03 smithi130 bash[27579]: audit 2023-08-27T22:46:03.053075+0000 mon.c (mon.2) 578 : audit [DBG] from='client.? 172.21.15.6:0/3661979670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:04.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:46:03.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:04.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:46:03.744Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:05.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:04 smithi006 bash[33918]: cluster 2023-08-27T22:46:04.035013+0000 mgr.y (mgr.14901) 1426 : cluster [DBG] pgmap v1433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:05.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:04 smithi006 bash[27292]: cluster 2023-08-27T22:46:04.035013+0000 mgr.y (mgr.14901) 1426 : cluster [DBG] pgmap v1433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:04 smithi130 bash[27579]: cluster 2023-08-27T22:46:04.035013+0000 mgr.y (mgr.14901) 1426 : cluster [DBG] pgmap v1433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:06.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:05 smithi006 bash[27292]: audit 2023-08-27T22:46:05.464391+0000 mon.c (mon.2) 579 : audit [DBG] from='client.? 172.21.15.6:0/2193981386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:06.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:05 smithi006 bash[33918]: audit 2023-08-27T22:46:05.464391+0000 mon.c (mon.2) 579 : audit [DBG] from='client.? 172.21.15.6:0/2193981386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:06.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:05 smithi130 bash[27579]: audit 2023-08-27T22:46:05.464391+0000 mon.c (mon.2) 579 : audit [DBG] from='client.? 172.21.15.6:0/2193981386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:07.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:06 smithi006 bash[33918]: cluster 2023-08-27T22:46:06.035772+0000 mgr.y (mgr.14901) 1427 : cluster [DBG] pgmap v1434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:07.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:06 smithi006 bash[27292]: cluster 2023-08-27T22:46:06.035772+0000 mgr.y (mgr.14901) 1427 : cluster [DBG] pgmap v1434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:07.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:06 smithi130 bash[27579]: cluster 2023-08-27T22:46:06.035772+0000 mgr.y (mgr.14901) 1427 : cluster [DBG] pgmap v1434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:08.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:07 smithi006 bash[33918]: audit 2023-08-27T22:46:07.871997+0000 mon.a (mon.0) 1574 : audit [DBG] from='client.? 172.21.15.6:0/4230953224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:08.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:07 smithi006 bash[27292]: audit 2023-08-27T22:46:07.871997+0000 mon.a (mon.0) 1574 : audit [DBG] from='client.? 172.21.15.6:0/4230953224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:08.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:07 smithi130 bash[27579]: audit 2023-08-27T22:46:07.871997+0000 mon.a (mon.0) 1574 : audit [DBG] from='client.? 172.21.15.6:0/4230953224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:09.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:08 smithi006 bash[33918]: cluster 2023-08-27T22:46:08.036415+0000 mgr.y (mgr.14901) 1428 : cluster [DBG] pgmap v1435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:09.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:08 smithi006 bash[27292]: cluster 2023-08-27T22:46:08.036415+0000 mgr.y (mgr.14901) 1428 : cluster [DBG] pgmap v1435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:09.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:08 smithi130 bash[27579]: cluster 2023-08-27T22:46:08.036415+0000 mgr.y (mgr.14901) 1428 : cluster [DBG] pgmap v1435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:10.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:46:10.280Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:10.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:46:10.282Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:11.102 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:46:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:46:10] "GET /metrics HTTP/1.1" 200 193084 "" "Prometheus/2.33.4" 2023-08-27T22:46:11.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:11 smithi130 bash[27579]: cluster 2023-08-27T22:46:10.037515+0000 mgr.y (mgr.14901) 1429 : cluster [DBG] pgmap v1436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:11.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:11 smithi130 bash[27579]: audit 2023-08-27T22:46:10.284261+0000 mon.a (mon.0) 1575 : audit [DBG] from='client.? 172.21.15.6:0/3554368088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:11.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:11 smithi006 bash[33918]: cluster 2023-08-27T22:46:10.037515+0000 mgr.y (mgr.14901) 1429 : cluster [DBG] pgmap v1436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:11.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:11 smithi006 bash[33918]: audit 2023-08-27T22:46:10.284261+0000 mon.a (mon.0) 1575 : audit [DBG] from='client.? 172.21.15.6:0/3554368088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:11.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:11 smithi006 bash[27292]: cluster 2023-08-27T22:46:10.037515+0000 mgr.y (mgr.14901) 1429 : cluster [DBG] pgmap v1436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:11.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:11 smithi006 bash[27292]: audit 2023-08-27T22:46:10.284261+0000 mon.a (mon.0) 1575 : audit [DBG] from='client.? 172.21.15.6:0/3554368088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:46:11.849Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:46:11.850Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:13.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:13 smithi130 bash[27579]: cluster 2023-08-27T22:46:12.038168+0000 mgr.y (mgr.14901) 1430 : cluster [DBG] pgmap v1437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:13.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:13 smithi130 bash[27579]: audit 2023-08-27T22:46:12.705617+0000 mon.a (mon.0) 1576 : audit [DBG] from='client.? 172.21.15.6:0/861573086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:13.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:13 smithi006 bash[33918]: cluster 2023-08-27T22:46:12.038168+0000 mgr.y (mgr.14901) 1430 : cluster [DBG] pgmap v1437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:13.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:13 smithi006 bash[33918]: audit 2023-08-27T22:46:12.705617+0000 mon.a (mon.0) 1576 : audit [DBG] from='client.? 172.21.15.6:0/861573086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:13.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:13 smithi006 bash[27292]: cluster 2023-08-27T22:46:12.038168+0000 mgr.y (mgr.14901) 1430 : cluster [DBG] pgmap v1437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:13.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:13 smithi006 bash[27292]: audit 2023-08-27T22:46:12.705617+0000 mon.a (mon.0) 1576 : audit [DBG] from='client.? 172.21.15.6:0/861573086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:14.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:46:13.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:14.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:46:13.745Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:15.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:14 smithi006 bash[33918]: cluster 2023-08-27T22:46:14.039203+0000 mgr.y (mgr.14901) 1431 : cluster [DBG] pgmap v1438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:15.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:14 smithi006 bash[27292]: cluster 2023-08-27T22:46:14.039203+0000 mgr.y (mgr.14901) 1431 : cluster [DBG] pgmap v1438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:15.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:14 smithi130 bash[27579]: cluster 2023-08-27T22:46:14.039203+0000 mgr.y (mgr.14901) 1431 : cluster [DBG] pgmap v1438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:16.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:15 smithi006 bash[33918]: audit 2023-08-27T22:46:15.120535+0000 mon.a (mon.0) 1577 : audit [DBG] from='client.? 172.21.15.6:0/3462881108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:16.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:15 smithi006 bash[27292]: audit 2023-08-27T22:46:15.120535+0000 mon.a (mon.0) 1577 : audit [DBG] from='client.? 172.21.15.6:0/3462881108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:16.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:15 smithi130 bash[27579]: audit 2023-08-27T22:46:15.120535+0000 mon.a (mon.0) 1577 : audit [DBG] from='client.? 172.21.15.6:0/3462881108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:17.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:16 smithi006 bash[27292]: cluster 2023-08-27T22:46:16.039959+0000 mgr.y (mgr.14901) 1432 : cluster [DBG] pgmap v1439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:17.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:16 smithi006 bash[33918]: cluster 2023-08-27T22:46:16.039959+0000 mgr.y (mgr.14901) 1432 : cluster [DBG] pgmap v1439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:17.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:16 smithi130 bash[27579]: cluster 2023-08-27T22:46:16.039959+0000 mgr.y (mgr.14901) 1432 : cluster [DBG] pgmap v1439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:18.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:17 smithi006 bash[27292]: audit 2023-08-27T22:46:17.537498+0000 mon.a (mon.0) 1578 : audit [DBG] from='client.? 172.21.15.6:0/230547979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:18.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:17 smithi006 bash[33918]: audit 2023-08-27T22:46:17.537498+0000 mon.a (mon.0) 1578 : audit [DBG] from='client.? 172.21.15.6:0/230547979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:18.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:17 smithi130 bash[27579]: audit 2023-08-27T22:46:17.537498+0000 mon.a (mon.0) 1578 : audit [DBG] from='client.? 172.21.15.6:0/230547979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:19.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:18 smithi130 bash[27579]: cluster 2023-08-27T22:46:18.040694+0000 mgr.y (mgr.14901) 1433 : cluster [DBG] pgmap v1440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:19.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:18 smithi006 bash[33918]: cluster 2023-08-27T22:46:18.040694+0000 mgr.y (mgr.14901) 1433 : cluster [DBG] pgmap v1440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:19.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:18 smithi006 bash[27292]: cluster 2023-08-27T22:46:18.040694+0000 mgr.y (mgr.14901) 1433 : cluster [DBG] pgmap v1440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:20.290 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:19 smithi006 bash[33918]: audit 2023-08-27T22:46:19.957599+0000 mon.c (mon.2) 580 : audit [DBG] from='client.? 172.21.15.6:0/1291265984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:20.290 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:19 smithi006 bash[27292]: audit 2023-08-27T22:46:19.957599+0000 mon.c (mon.2) 580 : audit [DBG] from='client.? 172.21.15.6:0/1291265984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:20.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:19 smithi130 bash[27579]: audit 2023-08-27T22:46:19.957599+0000 mon.c (mon.2) 580 : audit [DBG] from='client.? 172.21.15.6:0/1291265984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:20.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:46:20.282Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:20.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:46:20.284Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:21.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:20 smithi006 bash[27292]: cluster 2023-08-27T22:46:20.041805+0000 mgr.y (mgr.14901) 1434 : cluster [DBG] pgmap v1441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:21.223 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:46:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:46:20] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:46:21.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:20 smithi006 bash[33918]: cluster 2023-08-27T22:46:20.041805+0000 mgr.y (mgr.14901) 1434 : cluster [DBG] pgmap v1441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:21.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:20 smithi130 bash[27579]: cluster 2023-08-27T22:46:20.041805+0000 mgr.y (mgr.14901) 1434 : cluster [DBG] pgmap v1441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:46:21.850Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:46:21.851Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:23.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:23 smithi130 bash[27579]: cluster 2023-08-27T22:46:22.042416+0000 mgr.y (mgr.14901) 1435 : cluster [DBG] pgmap v1442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:23.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:23 smithi130 bash[27579]: audit 2023-08-27T22:46:22.388225+0000 mon.c (mon.2) 581 : audit [DBG] from='client.? 172.21.15.6:0/2884427359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:23.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:23 smithi006 bash[33918]: cluster 2023-08-27T22:46:22.042416+0000 mgr.y (mgr.14901) 1435 : cluster [DBG] pgmap v1442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:23.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:23 smithi006 bash[33918]: audit 2023-08-27T22:46:22.388225+0000 mon.c (mon.2) 581 : audit [DBG] from='client.? 172.21.15.6:0/2884427359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:23.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:23 smithi006 bash[27292]: cluster 2023-08-27T22:46:22.042416+0000 mgr.y (mgr.14901) 1435 : cluster [DBG] pgmap v1442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:23.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:23 smithi006 bash[27292]: audit 2023-08-27T22:46:22.388225+0000 mon.c (mon.2) 581 : audit [DBG] from='client.? 172.21.15.6:0/2884427359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:24.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:46:23.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:24.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:46:23.745Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:25.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:24 smithi006 bash[33918]: cluster 2023-08-27T22:46:24.043437+0000 mgr.y (mgr.14901) 1436 : cluster [DBG] pgmap v1443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:25.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:24 smithi006 bash[33918]: audit 2023-08-27T22:46:24.797333+0000 mon.c (mon.2) 582 : audit [DBG] from='client.? 172.21.15.6:0/4184134554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:25.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:24 smithi006 bash[27292]: cluster 2023-08-27T22:46:24.043437+0000 mgr.y (mgr.14901) 1436 : cluster [DBG] pgmap v1443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:25.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:24 smithi006 bash[27292]: audit 2023-08-27T22:46:24.797333+0000 mon.c (mon.2) 582 : audit [DBG] from='client.? 172.21.15.6:0/4184134554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:25.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:24 smithi130 bash[27579]: cluster 2023-08-27T22:46:24.043437+0000 mgr.y (mgr.14901) 1436 : cluster [DBG] pgmap v1443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:25.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:24 smithi130 bash[27579]: audit 2023-08-27T22:46:24.797333+0000 mon.c (mon.2) 582 : audit [DBG] from='client.? 172.21.15.6:0/4184134554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:26.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:25 smithi006 bash[33918]: audit 2023-08-27T22:46:25.942226+0000 mon.b (mon.1) 260 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:46:26.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:25 smithi006 bash[27292]: audit 2023-08-27T22:46:25.942226+0000 mon.b (mon.1) 260 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:46:26.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:25 smithi130 bash[27579]: audit 2023-08-27T22:46:25.942226+0000 mon.b (mon.1) 260 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:46:27.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:26 smithi006 bash[33918]: cluster 2023-08-27T22:46:26.044290+0000 mgr.y (mgr.14901) 1437 : cluster [DBG] pgmap v1444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:27.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:26 smithi006 bash[27292]: cluster 2023-08-27T22:46:26.044290+0000 mgr.y (mgr.14901) 1437 : cluster [DBG] pgmap v1444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:27.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:26 smithi130 bash[27579]: cluster 2023-08-27T22:46:26.044290+0000 mgr.y (mgr.14901) 1437 : cluster [DBG] pgmap v1444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:28.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:27 smithi006 bash[27292]: audit 2023-08-27T22:46:27.218009+0000 mon.c (mon.2) 583 : audit [DBG] from='client.? 172.21.15.6:0/554286323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:28.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:27 smithi006 bash[33918]: audit 2023-08-27T22:46:27.218009+0000 mon.c (mon.2) 583 : audit [DBG] from='client.? 172.21.15.6:0/554286323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:28.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:27 smithi130 bash[27579]: audit 2023-08-27T22:46:27.218009+0000 mon.c (mon.2) 583 : audit [DBG] from='client.? 172.21.15.6:0/554286323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:29.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:28 smithi130 bash[27579]: cluster 2023-08-27T22:46:28.045086+0000 mgr.y (mgr.14901) 1438 : cluster [DBG] pgmap v1445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:29.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:28 smithi006 bash[33918]: cluster 2023-08-27T22:46:28.045086+0000 mgr.y (mgr.14901) 1438 : cluster [DBG] pgmap v1445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:29.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:28 smithi006 bash[27292]: cluster 2023-08-27T22:46:28.045086+0000 mgr.y (mgr.14901) 1438 : cluster [DBG] pgmap v1445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:30.290 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:29 smithi006 bash[27292]: audit 2023-08-27T22:46:29.637712+0000 mon.c (mon.2) 584 : audit [DBG] from='client.? 172.21.15.6:0/988664915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:30.291 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:29 smithi006 bash[33918]: audit 2023-08-27T22:46:29.637712+0000 mon.c (mon.2) 584 : audit [DBG] from='client.? 172.21.15.6:0/988664915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:30.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:29 smithi130 bash[27579]: audit 2023-08-27T22:46:29.637712+0000 mon.c (mon.2) 584 : audit [DBG] from='client.? 172.21.15.6:0/988664915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:30.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:46:30.282Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:30.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:46:30.285Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:31.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:46:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:46:30] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:46:31.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:30 smithi006 bash[33918]: cluster 2023-08-27T22:46:30.046305+0000 mgr.y (mgr.14901) 1439 : cluster [DBG] pgmap v1446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:31.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:30 smithi006 bash[27292]: cluster 2023-08-27T22:46:30.046305+0000 mgr.y (mgr.14901) 1439 : cluster [DBG] pgmap v1446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:31.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:30 smithi130 bash[27579]: cluster 2023-08-27T22:46:30.046305+0000 mgr.y (mgr.14901) 1439 : cluster [DBG] pgmap v1446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:46:31.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:46:31.853Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:33.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:32 smithi130 bash[27579]: audit 2023-08-27T22:46:31.819152+0000 mon.a (mon.0) 1579 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:46:33.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:32 smithi130 bash[27579]: cluster 2023-08-27T22:46:32.046979+0000 mgr.y (mgr.14901) 1440 : cluster [DBG] pgmap v1447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:32 smithi130 bash[27579]: audit 2023-08-27T22:46:32.095068+0000 mon.c (mon.2) 585 : audit [DBG] from='client.? 172.21.15.6:0/3658686749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:32 smithi130 bash[27579]: audit 2023-08-27T22:46:32.278903+0000 mon.a (mon.0) 1580 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:46:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:32 smithi130 bash[27579]: audit 2023-08-27T22:46:32.557475+0000 mon.b (mon.1) 261 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:46:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:32 smithi130 bash[27579]: audit 2023-08-27T22:46:32.559710+0000 mon.b (mon.1) 262 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:46:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:32 smithi130 bash[27579]: audit 2023-08-27T22:46:32.562732+0000 mon.a (mon.0) 1581 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:46:33.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:32 smithi006 bash[27292]: audit 2023-08-27T22:46:31.819152+0000 mon.a (mon.0) 1579 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:46:33.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:32 smithi006 bash[27292]: cluster 2023-08-27T22:46:32.046979+0000 mgr.y (mgr.14901) 1440 : cluster [DBG] pgmap v1447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:33.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:32 smithi006 bash[27292]: audit 2023-08-27T22:46:32.095068+0000 mon.c (mon.2) 585 : audit [DBG] from='client.? 172.21.15.6:0/3658686749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:33.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:32 smithi006 bash[27292]: audit 2023-08-27T22:46:32.278903+0000 mon.a (mon.0) 1580 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:46:33.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:32 smithi006 bash[27292]: audit 2023-08-27T22:46:32.557475+0000 mon.b (mon.1) 261 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:46:33.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:32 smithi006 bash[27292]: audit 2023-08-27T22:46:32.559710+0000 mon.b (mon.1) 262 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:46:33.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:32 smithi006 bash[27292]: audit 2023-08-27T22:46:32.562732+0000 mon.a (mon.0) 1581 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:46:33.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:32 smithi006 bash[33918]: audit 2023-08-27T22:46:31.819152+0000 mon.a (mon.0) 1579 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:46:33.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:32 smithi006 bash[33918]: cluster 2023-08-27T22:46:32.046979+0000 mgr.y (mgr.14901) 1440 : cluster [DBG] pgmap v1447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:33.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:32 smithi006 bash[33918]: audit 2023-08-27T22:46:32.095068+0000 mon.c (mon.2) 585 : audit [DBG] from='client.? 172.21.15.6:0/3658686749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:33.224 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:32 smithi006 bash[33918]: audit 2023-08-27T22:46:32.278903+0000 mon.a (mon.0) 1580 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:46:33.224 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:32 smithi006 bash[33918]: audit 2023-08-27T22:46:32.557475+0000 mon.b (mon.1) 261 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:46:33.224 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:32 smithi006 bash[33918]: audit 2023-08-27T22:46:32.559710+0000 mon.b (mon.1) 262 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:46:33.224 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:32 smithi006 bash[33918]: audit 2023-08-27T22:46:32.562732+0000 mon.a (mon.0) 1581 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:46:34.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:46:33.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:34.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:46:33.747Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:35.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:34 smithi006 bash[33918]: cluster 2023-08-27T22:46:34.047954+0000 mgr.y (mgr.14901) 1441 : cluster [DBG] pgmap v1448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:35.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:34 smithi006 bash[33918]: audit 2023-08-27T22:46:34.510543+0000 mon.c (mon.2) 586 : audit [DBG] from='client.? 172.21.15.6:0/159237292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:35.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:34 smithi006 bash[27292]: cluster 2023-08-27T22:46:34.047954+0000 mgr.y (mgr.14901) 1441 : cluster [DBG] pgmap v1448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:35.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:34 smithi006 bash[27292]: audit 2023-08-27T22:46:34.510543+0000 mon.c (mon.2) 586 : audit [DBG] from='client.? 172.21.15.6:0/159237292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:35.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:34 smithi130 bash[27579]: cluster 2023-08-27T22:46:34.047954+0000 mgr.y (mgr.14901) 1441 : cluster [DBG] pgmap v1448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:35.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:34 smithi130 bash[27579]: audit 2023-08-27T22:46:34.510543+0000 mon.c (mon.2) 586 : audit [DBG] from='client.? 172.21.15.6:0/159237292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:37.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:37 smithi130 bash[27579]: cluster 2023-08-27T22:46:36.048724+0000 mgr.y (mgr.14901) 1442 : cluster [DBG] pgmap v1449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:38.067 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:37 smithi130 bash[27579]: audit 2023-08-27T22:46:36.927696+0000 mon.c (mon.2) 587 : audit [DBG] from='client.? 172.21.15.6:0/3611475741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:38.068 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:37 smithi006 bash[33918]: cluster 2023-08-27T22:46:36.048724+0000 mgr.y (mgr.14901) 1442 : cluster [DBG] pgmap v1449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:38.068 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:37 smithi006 bash[33918]: audit 2023-08-27T22:46:36.927696+0000 mon.c (mon.2) 587 : audit [DBG] from='client.? 172.21.15.6:0/3611475741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:38.068 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:37 smithi006 bash[27292]: cluster 2023-08-27T22:46:36.048724+0000 mgr.y (mgr.14901) 1442 : cluster [DBG] pgmap v1449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:38.069 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:37 smithi006 bash[27292]: audit 2023-08-27T22:46:36.927696+0000 mon.c (mon.2) 587 : audit [DBG] from='client.? 172.21.15.6:0/3611475741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:39.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:39 smithi006 bash[27292]: cluster 2023-08-27T22:46:38.049252+0000 mgr.y (mgr.14901) 1443 : cluster [DBG] pgmap v1450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:39.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:39 smithi006 bash[33918]: cluster 2023-08-27T22:46:38.049252+0000 mgr.y (mgr.14901) 1443 : cluster [DBG] pgmap v1450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:39.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:39 smithi130 bash[27579]: cluster 2023-08-27T22:46:38.049252+0000 mgr.y (mgr.14901) 1443 : cluster [DBG] pgmap v1450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:40.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:40 smithi006 bash[33918]: audit 2023-08-27T22:46:39.349934+0000 mon.c (mon.2) 588 : audit [DBG] from='client.? 172.21.15.6:0/2610646025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:40.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:40 smithi006 bash[27292]: audit 2023-08-27T22:46:39.349934+0000 mon.c (mon.2) 588 : audit [DBG] from='client.? 172.21.15.6:0/2610646025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:40.473 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:46:40.283Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:40.473 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:46:40.285Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:40.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:40 smithi130 bash[27579]: audit 2023-08-27T22:46:39.349934+0000 mon.c (mon.2) 588 : audit [DBG] from='client.? 172.21.15.6:0/2610646025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:41.129 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:46:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:46:40] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:46:41.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:41 smithi006 bash[27292]: cluster 2023-08-27T22:46:40.050340+0000 mgr.y (mgr.14901) 1444 : cluster [DBG] pgmap v1451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:41.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:41 smithi006 bash[33918]: cluster 2023-08-27T22:46:40.050340+0000 mgr.y (mgr.14901) 1444 : cluster [DBG] pgmap v1451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:41.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:41 smithi130 bash[27579]: cluster 2023-08-27T22:46:40.050340+0000 mgr.y (mgr.14901) 1444 : cluster [DBG] pgmap v1451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:42.137 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:46:41.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:42.137 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:46:41.853Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:42.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:42 smithi006 bash[27292]: audit 2023-08-27T22:46:41.767306+0000 mon.c (mon.2) 589 : audit [DBG] from='client.? 172.21.15.6:0/448742897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:42.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:42 smithi006 bash[33918]: audit 2023-08-27T22:46:41.767306+0000 mon.c (mon.2) 589 : audit [DBG] from='client.? 172.21.15.6:0/448742897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:42.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:42 smithi130 bash[27579]: audit 2023-08-27T22:46:41.767306+0000 mon.c (mon.2) 589 : audit [DBG] from='client.? 172.21.15.6:0/448742897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:43.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:43 smithi006 bash[27292]: cluster 2023-08-27T22:46:42.050975+0000 mgr.y (mgr.14901) 1445 : cluster [DBG] pgmap v1452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:43.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:43 smithi006 bash[33918]: cluster 2023-08-27T22:46:42.050975+0000 mgr.y (mgr.14901) 1445 : cluster [DBG] pgmap v1452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:43.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:43 smithi130 bash[27579]: cluster 2023-08-27T22:46:42.050975+0000 mgr.y (mgr.14901) 1445 : cluster [DBG] pgmap v1452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:44.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:46:43.746Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:44.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:46:43.748Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:45.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:44 smithi006 bash[33918]: cluster 2023-08-27T22:46:44.051992+0000 mgr.y (mgr.14901) 1446 : cluster [DBG] pgmap v1453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:45.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:44 smithi006 bash[33918]: audit 2023-08-27T22:46:44.186309+0000 mon.a (mon.0) 1582 : audit [DBG] from='client.? 172.21.15.6:0/1567635688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:45.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:44 smithi006 bash[27292]: cluster 2023-08-27T22:46:44.051992+0000 mgr.y (mgr.14901) 1446 : cluster [DBG] pgmap v1453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:45.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:44 smithi006 bash[27292]: audit 2023-08-27T22:46:44.186309+0000 mon.a (mon.0) 1582 : audit [DBG] from='client.? 172.21.15.6:0/1567635688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:45.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:44 smithi130 bash[27579]: cluster 2023-08-27T22:46:44.051992+0000 mgr.y (mgr.14901) 1446 : cluster [DBG] pgmap v1453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:45.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:44 smithi130 bash[27579]: audit 2023-08-27T22:46:44.186309+0000 mon.a (mon.0) 1582 : audit [DBG] from='client.? 172.21.15.6:0/1567635688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:47.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:47 smithi006 bash[27292]: cluster 2023-08-27T22:46:46.052726+0000 mgr.y (mgr.14901) 1447 : cluster [DBG] pgmap v1454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:47.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:47 smithi006 bash[27292]: audit 2023-08-27T22:46:46.595674+0000 mon.c (mon.2) 590 : audit [DBG] from='client.? 172.21.15.6:0/3773068924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:47.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:47 smithi006 bash[33918]: cluster 2023-08-27T22:46:46.052726+0000 mgr.y (mgr.14901) 1447 : cluster [DBG] pgmap v1454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:47.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:47 smithi006 bash[33918]: audit 2023-08-27T22:46:46.595674+0000 mon.c (mon.2) 590 : audit [DBG] from='client.? 172.21.15.6:0/3773068924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:47.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:47 smithi130 bash[27579]: cluster 2023-08-27T22:46:46.052726+0000 mgr.y (mgr.14901) 1447 : cluster [DBG] pgmap v1454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:47.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:47 smithi130 bash[27579]: audit 2023-08-27T22:46:46.595674+0000 mon.c (mon.2) 590 : audit [DBG] from='client.? 172.21.15.6:0/3773068924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:49.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:49 smithi006 bash[27292]: cluster 2023-08-27T22:46:48.053371+0000 mgr.y (mgr.14901) 1448 : cluster [DBG] pgmap v1455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:49.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:49 smithi006 bash[27292]: audit 2023-08-27T22:46:49.002384+0000 mon.a (mon.0) 1583 : audit [DBG] from='client.? 172.21.15.6:0/1414549126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:49.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:49 smithi006 bash[33918]: cluster 2023-08-27T22:46:48.053371+0000 mgr.y (mgr.14901) 1448 : cluster [DBG] pgmap v1455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:49.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:49 smithi006 bash[33918]: audit 2023-08-27T22:46:49.002384+0000 mon.a (mon.0) 1583 : audit [DBG] from='client.? 172.21.15.6:0/1414549126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:49.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:49 smithi130 bash[27579]: cluster 2023-08-27T22:46:48.053371+0000 mgr.y (mgr.14901) 1448 : cluster [DBG] pgmap v1455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:49.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:49 smithi130 bash[27579]: audit 2023-08-27T22:46:49.002384+0000 mon.a (mon.0) 1583 : audit [DBG] from='client.? 172.21.15.6:0/1414549126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:50.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:46:50.283Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:50.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:46:50.286Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:51.127 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:46:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:46:50] "GET /metrics HTTP/1.1" 200 193081 "" "Prometheus/2.33.4" 2023-08-27T22:46:51.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:51 smithi006 bash[27292]: cluster 2023-08-27T22:46:50.054365+0000 mgr.y (mgr.14901) 1449 : cluster [DBG] pgmap v1456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:51.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:51 smithi006 bash[33918]: cluster 2023-08-27T22:46:50.054365+0000 mgr.y (mgr.14901) 1449 : cluster [DBG] pgmap v1456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:51.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:51 smithi130 bash[27579]: cluster 2023-08-27T22:46:50.054365+0000 mgr.y (mgr.14901) 1449 : cluster [DBG] pgmap v1456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:52.135 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:46:51.852Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:52.136 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:46:51.854Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:52.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:52 smithi006 bash[27292]: audit 2023-08-27T22:46:51.417551+0000 mon.c (mon.2) 591 : audit [DBG] from='client.? 172.21.15.6:0/1358408702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:52.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:52 smithi006 bash[33918]: audit 2023-08-27T22:46:51.417551+0000 mon.c (mon.2) 591 : audit [DBG] from='client.? 172.21.15.6:0/1358408702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:52.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:52 smithi130 bash[27579]: audit 2023-08-27T22:46:51.417551+0000 mon.c (mon.2) 591 : audit [DBG] from='client.? 172.21.15.6:0/1358408702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:53 smithi006 bash[33918]: cluster 2023-08-27T22:46:52.054995+0000 mgr.y (mgr.14901) 1450 : cluster [DBG] pgmap v1457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:53.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:53 smithi006 bash[27292]: cluster 2023-08-27T22:46:52.054995+0000 mgr.y (mgr.14901) 1450 : cluster [DBG] pgmap v1457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:53.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:53 smithi130 bash[27579]: cluster 2023-08-27T22:46:52.054995+0000 mgr.y (mgr.14901) 1450 : cluster [DBG] pgmap v1457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:54.141 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:46:53.746Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:54.141 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:46:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:46:53.749Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:46:54.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:54 smithi006 bash[33918]: audit 2023-08-27T22:46:53.831327+0000 mon.c (mon.2) 592 : audit [DBG] from='client.? 172.21.15.6:0/3898969617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:54.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:54 smithi006 bash[27292]: audit 2023-08-27T22:46:53.831327+0000 mon.c (mon.2) 592 : audit [DBG] from='client.? 172.21.15.6:0/3898969617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:54.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:54 smithi130 bash[27579]: audit 2023-08-27T22:46:53.831327+0000 mon.c (mon.2) 592 : audit [DBG] from='client.? 172.21.15.6:0/3898969617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:55.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:55 smithi006 bash[33918]: cluster 2023-08-27T22:46:54.055900+0000 mgr.y (mgr.14901) 1451 : cluster [DBG] pgmap v1458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:55.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:55 smithi006 bash[27292]: cluster 2023-08-27T22:46:54.055900+0000 mgr.y (mgr.14901) 1451 : cluster [DBG] pgmap v1458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:55.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:55 smithi130 bash[27579]: cluster 2023-08-27T22:46:54.055900+0000 mgr.y (mgr.14901) 1451 : cluster [DBG] pgmap v1458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:57.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:57 smithi006 bash[33918]: cluster 2023-08-27T22:46:56.056595+0000 mgr.y (mgr.14901) 1452 : cluster [DBG] pgmap v1459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:57.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:57 smithi006 bash[33918]: audit 2023-08-27T22:46:56.246955+0000 mon.c (mon.2) 593 : audit [DBG] from='client.? 172.21.15.6:0/1432973214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:57.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:57 smithi006 bash[27292]: cluster 2023-08-27T22:46:56.056595+0000 mgr.y (mgr.14901) 1452 : cluster [DBG] pgmap v1459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:57.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:57 smithi006 bash[27292]: audit 2023-08-27T22:46:56.246955+0000 mon.c (mon.2) 593 : audit [DBG] from='client.? 172.21.15.6:0/1432973214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:57.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:57 smithi130 bash[27579]: cluster 2023-08-27T22:46:56.056595+0000 mgr.y (mgr.14901) 1452 : cluster [DBG] pgmap v1459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:57.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:57 smithi130 bash[27579]: audit 2023-08-27T22:46:56.246955+0000 mon.c (mon.2) 593 : audit [DBG] from='client.? 172.21.15.6:0/1432973214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:59.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:59 smithi006 bash[27292]: cluster 2023-08-27T22:46:58.057321+0000 mgr.y (mgr.14901) 1453 : cluster [DBG] pgmap v1460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:59.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:46:59 smithi006 bash[27292]: audit 2023-08-27T22:46:58.695248+0000 mon.a (mon.0) 1584 : audit [DBG] from='client.? 172.21.15.6:0/3642284881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:59.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:59 smithi006 bash[33918]: cluster 2023-08-27T22:46:58.057321+0000 mgr.y (mgr.14901) 1453 : cluster [DBG] pgmap v1460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:59.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:46:59 smithi006 bash[33918]: audit 2023-08-27T22:46:58.695248+0000 mon.a (mon.0) 1584 : audit [DBG] from='client.? 172.21.15.6:0/3642284881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:59.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:59 smithi130 bash[27579]: cluster 2023-08-27T22:46:58.057321+0000 mgr.y (mgr.14901) 1453 : cluster [DBG] pgmap v1460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:46:59.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:46:59 smithi130 bash[27579]: audit 2023-08-27T22:46:58.695248+0000 mon.a (mon.0) 1584 : audit [DBG] from='client.? 172.21.15.6:0/3642284881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:00.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:47:00.283Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:00.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:47:00.285Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:01.157 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:47:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:47:00] "GET /metrics HTTP/1.1" 200 193081 "" "Prometheus/2.33.4" 2023-08-27T22:47:01.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:01 smithi006 bash[33918]: cluster 2023-08-27T22:47:00.058399+0000 mgr.y (mgr.14901) 1454 : cluster [DBG] pgmap v1461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:01.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:01 smithi006 bash[33918]: audit 2023-08-27T22:47:01.110224+0000 mon.c (mon.2) 594 : audit [DBG] from='client.? 172.21.15.6:0/2020708883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:01.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:01 smithi006 bash[27292]: cluster 2023-08-27T22:47:00.058399+0000 mgr.y (mgr.14901) 1454 : cluster [DBG] pgmap v1461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:01.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:01 smithi006 bash[27292]: audit 2023-08-27T22:47:01.110224+0000 mon.c (mon.2) 594 : audit [DBG] from='client.? 172.21.15.6:0/2020708883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:01.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:01 smithi130 bash[27579]: cluster 2023-08-27T22:47:00.058399+0000 mgr.y (mgr.14901) 1454 : cluster [DBG] pgmap v1461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:01.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:01 smithi130 bash[27579]: audit 2023-08-27T22:47:01.110224+0000 mon.c (mon.2) 594 : audit [DBG] from='client.? 172.21.15.6:0/2020708883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:47:01.852Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:47:01.854Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:03.474 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:03 smithi006 bash[33918]: cluster 2023-08-27T22:47:02.059042+0000 mgr.y (mgr.14901) 1455 : cluster [DBG] pgmap v1462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:03.474 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:03 smithi006 bash[27292]: cluster 2023-08-27T22:47:02.059042+0000 mgr.y (mgr.14901) 1455 : cluster [DBG] pgmap v1462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:03.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:03 smithi130 bash[27579]: cluster 2023-08-27T22:47:02.059042+0000 mgr.y (mgr.14901) 1455 : cluster [DBG] pgmap v1462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:04.166 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:47:03.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:04.167 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:47:03.749Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:04.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:04 smithi006 bash[27292]: audit 2023-08-27T22:47:03.532657+0000 mon.a (mon.0) 1585 : audit [DBG] from='client.? 172.21.15.6:0/565976156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:04.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:04 smithi006 bash[33918]: audit 2023-08-27T22:47:03.532657+0000 mon.a (mon.0) 1585 : audit [DBG] from='client.? 172.21.15.6:0/565976156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:04.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:04 smithi130 bash[27579]: audit 2023-08-27T22:47:03.532657+0000 mon.a (mon.0) 1585 : audit [DBG] from='client.? 172.21.15.6:0/565976156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:05.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:05 smithi006 bash[27292]: cluster 2023-08-27T22:47:04.059890+0000 mgr.y (mgr.14901) 1456 : cluster [DBG] pgmap v1463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:05.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:05 smithi006 bash[33918]: cluster 2023-08-27T22:47:04.059890+0000 mgr.y (mgr.14901) 1456 : cluster [DBG] pgmap v1463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:05.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:05 smithi130 bash[27579]: cluster 2023-08-27T22:47:04.059890+0000 mgr.y (mgr.14901) 1456 : cluster [DBG] pgmap v1463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:06.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:06 smithi006 bash[33918]: audit 2023-08-27T22:47:05.950816+0000 mon.a (mon.0) 1586 : audit [DBG] from='client.? 172.21.15.6:0/918530467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:06.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:06 smithi006 bash[27292]: audit 2023-08-27T22:47:05.950816+0000 mon.a (mon.0) 1586 : audit [DBG] from='client.? 172.21.15.6:0/918530467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:06.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:06 smithi130 bash[27579]: audit 2023-08-27T22:47:05.950816+0000 mon.a (mon.0) 1586 : audit [DBG] from='client.? 172.21.15.6:0/918530467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:07.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:07 smithi006 bash[33918]: cluster 2023-08-27T22:47:06.060556+0000 mgr.y (mgr.14901) 1457 : cluster [DBG] pgmap v1464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:07.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:07 smithi006 bash[27292]: cluster 2023-08-27T22:47:06.060556+0000 mgr.y (mgr.14901) 1457 : cluster [DBG] pgmap v1464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:07.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:07 smithi130 bash[27579]: cluster 2023-08-27T22:47:06.060556+0000 mgr.y (mgr.14901) 1457 : cluster [DBG] pgmap v1464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:09.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:09 smithi006 bash[33918]: cluster 2023-08-27T22:47:08.060979+0000 mgr.y (mgr.14901) 1458 : cluster [DBG] pgmap v1465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:09.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:09 smithi006 bash[33918]: audit 2023-08-27T22:47:08.368665+0000 mon.a (mon.0) 1587 : audit [DBG] from='client.? 172.21.15.6:0/3679342645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:09.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:09 smithi006 bash[27292]: cluster 2023-08-27T22:47:08.060979+0000 mgr.y (mgr.14901) 1458 : cluster [DBG] pgmap v1465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:09.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:09 smithi006 bash[27292]: audit 2023-08-27T22:47:08.368665+0000 mon.a (mon.0) 1587 : audit [DBG] from='client.? 172.21.15.6:0/3679342645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:09.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:09 smithi130 bash[27579]: cluster 2023-08-27T22:47:08.060979+0000 mgr.y (mgr.14901) 1458 : cluster [DBG] pgmap v1465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:09.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:09 smithi130 bash[27579]: audit 2023-08-27T22:47:08.368665+0000 mon.a (mon.0) 1587 : audit [DBG] from='client.? 172.21.15.6:0/3679342645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:10.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:47:10.284Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:10.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:47:10.286Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:11.191 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:47:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:47:10] "GET /metrics HTTP/1.1" 200 193081 "" "Prometheus/2.33.4" 2023-08-27T22:47:11.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:11 smithi006 bash[33918]: cluster 2023-08-27T22:47:10.061910+0000 mgr.y (mgr.14901) 1459 : cluster [DBG] pgmap v1466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:11.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:11 smithi006 bash[33918]: audit 2023-08-27T22:47:10.794741+0000 mon.a (mon.0) 1588 : audit [DBG] from='client.? 172.21.15.6:0/3472933481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:11.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:11 smithi006 bash[27292]: cluster 2023-08-27T22:47:10.061910+0000 mgr.y (mgr.14901) 1459 : cluster [DBG] pgmap v1466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:11.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:11 smithi006 bash[27292]: audit 2023-08-27T22:47:10.794741+0000 mon.a (mon.0) 1588 : audit [DBG] from='client.? 172.21.15.6:0/3472933481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:11.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:11 smithi130 bash[27579]: cluster 2023-08-27T22:47:10.061910+0000 mgr.y (mgr.14901) 1459 : cluster [DBG] pgmap v1466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:11.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:11 smithi130 bash[27579]: audit 2023-08-27T22:47:10.794741+0000 mon.a (mon.0) 1588 : audit [DBG] from='client.? 172.21.15.6:0/3472933481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:47:11.853Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:12.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:47:11.855Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:13.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:13 smithi006 bash[33918]: cluster 2023-08-27T22:47:12.062548+0000 mgr.y (mgr.14901) 1460 : cluster [DBG] pgmap v1467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:13.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:13 smithi006 bash[27292]: cluster 2023-08-27T22:47:12.062548+0000 mgr.y (mgr.14901) 1460 : cluster [DBG] pgmap v1467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:13.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:13 smithi130 bash[27579]: cluster 2023-08-27T22:47:12.062548+0000 mgr.y (mgr.14901) 1460 : cluster [DBG] pgmap v1467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:14.198 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:47:13.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:14.198 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:47:13.750Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:14.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:14 smithi006 bash[27292]: audit 2023-08-27T22:47:13.198847+0000 mon.c (mon.2) 595 : audit [DBG] from='client.? 172.21.15.6:0/732364303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:14.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:14 smithi006 bash[33918]: audit 2023-08-27T22:47:13.198847+0000 mon.c (mon.2) 595 : audit [DBG] from='client.? 172.21.15.6:0/732364303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:14.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:14 smithi130 bash[27579]: audit 2023-08-27T22:47:13.198847+0000 mon.c (mon.2) 595 : audit [DBG] from='client.? 172.21.15.6:0/732364303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:15.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:15 smithi006 bash[27292]: cluster 2023-08-27T22:47:14.063512+0000 mgr.y (mgr.14901) 1461 : cluster [DBG] pgmap v1468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:15.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:15 smithi006 bash[33918]: cluster 2023-08-27T22:47:14.063512+0000 mgr.y (mgr.14901) 1461 : cluster [DBG] pgmap v1468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:15.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:15 smithi130 bash[27579]: cluster 2023-08-27T22:47:14.063512+0000 mgr.y (mgr.14901) 1461 : cluster [DBG] pgmap v1468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:16.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:16 smithi006 bash[27292]: audit 2023-08-27T22:47:15.606316+0000 mon.c (mon.2) 596 : audit [DBG] from='client.? 172.21.15.6:0/1632519726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:16.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:16 smithi006 bash[33918]: audit 2023-08-27T22:47:15.606316+0000 mon.c (mon.2) 596 : audit [DBG] from='client.? 172.21.15.6:0/1632519726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:16.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:16 smithi130 bash[27579]: audit 2023-08-27T22:47:15.606316+0000 mon.c (mon.2) 596 : audit [DBG] from='client.? 172.21.15.6:0/1632519726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:17.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:17 smithi006 bash[33918]: cluster 2023-08-27T22:47:16.064287+0000 mgr.y (mgr.14901) 1462 : cluster [DBG] pgmap v1469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:17.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:17 smithi006 bash[27292]: cluster 2023-08-27T22:47:16.064287+0000 mgr.y (mgr.14901) 1462 : cluster [DBG] pgmap v1469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:17.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:17 smithi130 bash[27579]: cluster 2023-08-27T22:47:16.064287+0000 mgr.y (mgr.14901) 1462 : cluster [DBG] pgmap v1469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:18.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:18 smithi006 bash[27292]: audit 2023-08-27T22:47:18.028295+0000 mon.c (mon.2) 597 : audit [DBG] from='client.? 172.21.15.6:0/1157192190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:18.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:18 smithi006 bash[33918]: audit 2023-08-27T22:47:18.028295+0000 mon.c (mon.2) 597 : audit [DBG] from='client.? 172.21.15.6:0/1157192190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:18.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:18 smithi130 bash[27579]: audit 2023-08-27T22:47:18.028295+0000 mon.c (mon.2) 597 : audit [DBG] from='client.? 172.21.15.6:0/1157192190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:19.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:19 smithi130 bash[27579]: cluster 2023-08-27T22:47:18.064885+0000 mgr.y (mgr.14901) 1463 : cluster [DBG] pgmap v1470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:19.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:19 smithi006 bash[27292]: cluster 2023-08-27T22:47:18.064885+0000 mgr.y (mgr.14901) 1463 : cluster [DBG] pgmap v1470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:19.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:19 smithi006 bash[33918]: cluster 2023-08-27T22:47:18.064885+0000 mgr.y (mgr.14901) 1463 : cluster [DBG] pgmap v1470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:20.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:47:20.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:20.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:47:20.287Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:21.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:47:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:47:20] "GET /metrics HTTP/1.1" 200 193083 "" "Prometheus/2.33.4" 2023-08-27T22:47:21.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:21 smithi130 bash[27579]: cluster 2023-08-27T22:47:20.065989+0000 mgr.y (mgr.14901) 1464 : cluster [DBG] pgmap v1471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:21.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:21 smithi130 bash[27579]: audit 2023-08-27T22:47:20.461333+0000 mon.a (mon.0) 1589 : audit [DBG] from='client.? 172.21.15.6:0/3209388152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:21.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:21 smithi006 bash[33918]: cluster 2023-08-27T22:47:20.065989+0000 mgr.y (mgr.14901) 1464 : cluster [DBG] pgmap v1471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:21.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:21 smithi006 bash[33918]: audit 2023-08-27T22:47:20.461333+0000 mon.a (mon.0) 1589 : audit [DBG] from='client.? 172.21.15.6:0/3209388152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:21.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:21 smithi006 bash[27292]: cluster 2023-08-27T22:47:20.065989+0000 mgr.y (mgr.14901) 1464 : cluster [DBG] pgmap v1471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:21.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:21 smithi006 bash[27292]: audit 2023-08-27T22:47:20.461333+0000 mon.a (mon.0) 1589 : audit [DBG] from='client.? 172.21.15.6:0/3209388152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:47:21.854Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:22.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:47:21.856Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:23.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:23 smithi130 bash[27579]: cluster 2023-08-27T22:47:22.066661+0000 mgr.y (mgr.14901) 1465 : cluster [DBG] pgmap v1472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:23.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:23 smithi130 bash[27579]: audit 2023-08-27T22:47:22.869323+0000 mon.a (mon.0) 1590 : audit [DBG] from='client.? 172.21.15.6:0/4054646284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:23.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:23 smithi006 bash[33918]: cluster 2023-08-27T22:47:22.066661+0000 mgr.y (mgr.14901) 1465 : cluster [DBG] pgmap v1472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:23.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:23 smithi006 bash[33918]: audit 2023-08-27T22:47:22.869323+0000 mon.a (mon.0) 1590 : audit [DBG] from='client.? 172.21.15.6:0/4054646284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:23.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:23 smithi006 bash[27292]: cluster 2023-08-27T22:47:22.066661+0000 mgr.y (mgr.14901) 1465 : cluster [DBG] pgmap v1472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:23.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:23 smithi006 bash[27292]: audit 2023-08-27T22:47:22.869323+0000 mon.a (mon.0) 1590 : audit [DBG] from='client.? 172.21.15.6:0/4054646284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:24.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:47:23.749Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:24.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:47:23.751Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:25.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:24 smithi006 bash[33918]: cluster 2023-08-27T22:47:24.067671+0000 mgr.y (mgr.14901) 1466 : cluster [DBG] pgmap v1473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:25.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:24 smithi006 bash[27292]: cluster 2023-08-27T22:47:24.067671+0000 mgr.y (mgr.14901) 1466 : cluster [DBG] pgmap v1473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:25.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:24 smithi130 bash[27579]: cluster 2023-08-27T22:47:24.067671+0000 mgr.y (mgr.14901) 1466 : cluster [DBG] pgmap v1473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:26.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:25 smithi006 bash[27292]: audit 2023-08-27T22:47:25.291924+0000 mon.a (mon.0) 1591 : audit [DBG] from='client.? 172.21.15.6:0/1723799195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:26.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:25 smithi006 bash[33918]: audit 2023-08-27T22:47:25.291924+0000 mon.a (mon.0) 1591 : audit [DBG] from='client.? 172.21.15.6:0/1723799195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:26.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:25 smithi130 bash[27579]: audit 2023-08-27T22:47:25.291924+0000 mon.a (mon.0) 1591 : audit [DBG] from='client.? 172.21.15.6:0/1723799195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:27.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:26 smithi130 bash[27579]: cluster 2023-08-27T22:47:26.068720+0000 mgr.y (mgr.14901) 1467 : cluster [DBG] pgmap v1474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:27.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:26 smithi006 bash[33918]: cluster 2023-08-27T22:47:26.068720+0000 mgr.y (mgr.14901) 1467 : cluster [DBG] pgmap v1474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:27.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:26 smithi006 bash[27292]: cluster 2023-08-27T22:47:26.068720+0000 mgr.y (mgr.14901) 1467 : cluster [DBG] pgmap v1474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:28.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:27 smithi130 bash[27579]: audit 2023-08-27T22:47:27.715212+0000 mon.a (mon.0) 1592 : audit [DBG] from='client.? 172.21.15.6:0/3796054838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:28.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:27 smithi006 bash[27292]: audit 2023-08-27T22:47:27.715212+0000 mon.a (mon.0) 1592 : audit [DBG] from='client.? 172.21.15.6:0/3796054838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:28.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:27 smithi006 bash[33918]: audit 2023-08-27T22:47:27.715212+0000 mon.a (mon.0) 1592 : audit [DBG] from='client.? 172.21.15.6:0/3796054838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:29.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:28 smithi130 bash[27579]: cluster 2023-08-27T22:47:28.069419+0000 mgr.y (mgr.14901) 1468 : cluster [DBG] pgmap v1475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:29.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:28 smithi006 bash[27292]: cluster 2023-08-27T22:47:28.069419+0000 mgr.y (mgr.14901) 1468 : cluster [DBG] pgmap v1475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:29.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:28 smithi006 bash[33918]: cluster 2023-08-27T22:47:28.069419+0000 mgr.y (mgr.14901) 1468 : cluster [DBG] pgmap v1475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:30.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:47:30.286Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:30.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:47:30.287Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:31.136 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:47:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:47:30] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T22:47:31.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:31 smithi006 bash[27292]: cluster 2023-08-27T22:47:30.070478+0000 mgr.y (mgr.14901) 1469 : cluster [DBG] pgmap v1476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:31.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:31 smithi006 bash[27292]: audit 2023-08-27T22:47:30.130339+0000 mon.a (mon.0) 1593 : audit [DBG] from='client.? 172.21.15.6:0/4268403299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:31.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:31 smithi006 bash[33918]: cluster 2023-08-27T22:47:30.070478+0000 mgr.y (mgr.14901) 1469 : cluster [DBG] pgmap v1476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:31.474 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:31 smithi006 bash[33918]: audit 2023-08-27T22:47:30.130339+0000 mon.a (mon.0) 1593 : audit [DBG] from='client.? 172.21.15.6:0/4268403299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:31.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:31 smithi130 bash[27579]: cluster 2023-08-27T22:47:30.070478+0000 mgr.y (mgr.14901) 1469 : cluster [DBG] pgmap v1476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:31.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:31 smithi130 bash[27579]: audit 2023-08-27T22:47:30.130339+0000 mon.a (mon.0) 1593 : audit [DBG] from='client.? 172.21.15.6:0/4268403299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:47:31.855Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:47:31.857Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:33.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:33 smithi006 bash[33918]: cluster 2023-08-27T22:47:32.071120+0000 mgr.y (mgr.14901) 1470 : cluster [DBG] pgmap v1477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:33.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:33 smithi006 bash[33918]: audit 2023-08-27T22:47:32.541329+0000 mon.a (mon.0) 1594 : audit [DBG] from='client.? 172.21.15.6:0/218324596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:33.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:33 smithi006 bash[33918]: audit 2023-08-27T22:47:32.579842+0000 mon.b (mon.1) 263 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:47:33.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:33 smithi006 bash[27292]: cluster 2023-08-27T22:47:32.071120+0000 mgr.y (mgr.14901) 1470 : cluster [DBG] pgmap v1477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:33.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:33 smithi006 bash[27292]: audit 2023-08-27T22:47:32.541329+0000 mon.a (mon.0) 1594 : audit [DBG] from='client.? 172.21.15.6:0/218324596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:33.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:33 smithi006 bash[27292]: audit 2023-08-27T22:47:32.579842+0000 mon.b (mon.1) 263 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:47:33.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:33 smithi130 bash[27579]: cluster 2023-08-27T22:47:32.071120+0000 mgr.y (mgr.14901) 1470 : cluster [DBG] pgmap v1477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:33.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:33 smithi130 bash[27579]: audit 2023-08-27T22:47:32.541329+0000 mon.a (mon.0) 1594 : audit [DBG] from='client.? 172.21.15.6:0/218324596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:33.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:33 smithi130 bash[27579]: audit 2023-08-27T22:47:32.579842+0000 mon.b (mon.1) 263 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:47:34.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:47:33.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:34.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:47:33.752Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:35.222 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:34 smithi006 bash[27292]: cluster 2023-08-27T22:47:34.072089+0000 mgr.y (mgr.14901) 1471 : cluster [DBG] pgmap v1478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:35.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:34 smithi006 bash[33918]: cluster 2023-08-27T22:47:34.072089+0000 mgr.y (mgr.14901) 1471 : cluster [DBG] pgmap v1478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:35.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:34 smithi130 bash[27579]: cluster 2023-08-27T22:47:34.072089+0000 mgr.y (mgr.14901) 1471 : cluster [DBG] pgmap v1478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:36.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:35 smithi006 bash[33918]: audit 2023-08-27T22:47:34.963539+0000 mon.a (mon.0) 1595 : audit [DBG] from='client.? 172.21.15.6:0/3616475620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:36.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:35 smithi006 bash[27292]: audit 2023-08-27T22:47:34.963539+0000 mon.a (mon.0) 1595 : audit [DBG] from='client.? 172.21.15.6:0/3616475620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:36.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:35 smithi130 bash[27579]: audit 2023-08-27T22:47:34.963539+0000 mon.a (mon.0) 1595 : audit [DBG] from='client.? 172.21.15.6:0/3616475620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:37.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:36 smithi130 bash[27579]: cluster 2023-08-27T22:47:36.072919+0000 mgr.y (mgr.14901) 1472 : cluster [DBG] pgmap v1479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:37.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:36 smithi006 bash[33918]: cluster 2023-08-27T22:47:36.072919+0000 mgr.y (mgr.14901) 1472 : cluster [DBG] pgmap v1479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:37.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:36 smithi006 bash[27292]: cluster 2023-08-27T22:47:36.072919+0000 mgr.y (mgr.14901) 1472 : cluster [DBG] pgmap v1479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:38.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:37 smithi130 bash[27579]: audit 2023-08-27T22:47:37.384011+0000 mon.a (mon.0) 1596 : audit [DBG] from='client.? 172.21.15.6:0/2485475401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:38.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:37 smithi006 bash[33918]: audit 2023-08-27T22:47:37.384011+0000 mon.a (mon.0) 1596 : audit [DBG] from='client.? 172.21.15.6:0/2485475401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:38.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:37 smithi006 bash[27292]: audit 2023-08-27T22:47:37.384011+0000 mon.a (mon.0) 1596 : audit [DBG] from='client.? 172.21.15.6:0/2485475401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:39.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:38 smithi130 bash[27579]: cluster 2023-08-27T22:47:38.073524+0000 mgr.y (mgr.14901) 1473 : cluster [DBG] pgmap v1480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:39.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:38 smithi130 bash[27579]: audit 2023-08-27T22:47:38.477827+0000 mon.a (mon.0) 1597 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:47:39.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:38 smithi130 bash[27579]: audit 2023-08-27T22:47:38.920168+0000 mon.a (mon.0) 1598 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:47:39.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:38 smithi006 bash[33918]: cluster 2023-08-27T22:47:38.073524+0000 mgr.y (mgr.14901) 1473 : cluster [DBG] pgmap v1480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:39.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:38 smithi006 bash[33918]: audit 2023-08-27T22:47:38.477827+0000 mon.a (mon.0) 1597 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:47:39.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:38 smithi006 bash[33918]: audit 2023-08-27T22:47:38.920168+0000 mon.a (mon.0) 1598 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:47:39.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:38 smithi006 bash[27292]: cluster 2023-08-27T22:47:38.073524+0000 mgr.y (mgr.14901) 1473 : cluster [DBG] pgmap v1480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:39.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:38 smithi006 bash[27292]: audit 2023-08-27T22:47:38.477827+0000 mon.a (mon.0) 1597 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:47:39.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:38 smithi006 bash[27292]: audit 2023-08-27T22:47:38.920168+0000 mon.a (mon.0) 1598 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:47:40.295 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:39 smithi006 bash[27292]: audit 2023-08-27T22:47:39.227136+0000 mon.b (mon.1) 264 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:47:40.295 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:39 smithi006 bash[27292]: audit 2023-08-27T22:47:39.229286+0000 mon.b (mon.1) 265 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:47:40.295 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:39 smithi006 bash[27292]: audit 2023-08-27T22:47:39.231970+0000 mon.a (mon.0) 1599 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:47:40.295 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:39 smithi006 bash[27292]: audit 2023-08-27T22:47:39.802710+0000 mon.c (mon.2) 598 : audit [DBG] from='client.? 172.21.15.6:0/2186225516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:40.295 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:39 smithi006 bash[33918]: audit 2023-08-27T22:47:39.227136+0000 mon.b (mon.1) 264 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:47:40.295 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:39 smithi006 bash[33918]: audit 2023-08-27T22:47:39.229286+0000 mon.b (mon.1) 265 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:47:40.296 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:39 smithi006 bash[33918]: audit 2023-08-27T22:47:39.231970+0000 mon.a (mon.0) 1599 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:47:40.296 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:39 smithi006 bash[33918]: audit 2023-08-27T22:47:39.802710+0000 mon.c (mon.2) 598 : audit [DBG] from='client.? 172.21.15.6:0/2186225516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:40.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:39 smithi130 bash[27579]: audit 2023-08-27T22:47:39.227136+0000 mon.b (mon.1) 264 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:47:40.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:39 smithi130 bash[27579]: audit 2023-08-27T22:47:39.229286+0000 mon.b (mon.1) 265 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:47:40.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:39 smithi130 bash[27579]: audit 2023-08-27T22:47:39.231970+0000 mon.a (mon.0) 1599 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:47:40.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:39 smithi130 bash[27579]: audit 2023-08-27T22:47:39.802710+0000 mon.c (mon.2) 598 : audit [DBG] from='client.? 172.21.15.6:0/2186225516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:40.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:47:40.286Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:40.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:47:40.288Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:41.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:40 smithi006 bash[33918]: cluster 2023-08-27T22:47:40.074664+0000 mgr.y (mgr.14901) 1474 : cluster [DBG] pgmap v1481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:41.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:40 smithi006 bash[27292]: cluster 2023-08-27T22:47:40.074664+0000 mgr.y (mgr.14901) 1474 : cluster [DBG] pgmap v1481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:41.223 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:47:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:47:40] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T22:47:41.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:40 smithi130 bash[27579]: cluster 2023-08-27T22:47:40.074664+0000 mgr.y (mgr.14901) 1474 : cluster [DBG] pgmap v1481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:47:41.856Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:42.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:47:41.858Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:43.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:43 smithi006 bash[27292]: cluster 2023-08-27T22:47:42.075250+0000 mgr.y (mgr.14901) 1475 : cluster [DBG] pgmap v1482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:43.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:43 smithi006 bash[27292]: audit 2023-08-27T22:47:42.224594+0000 mon.c (mon.2) 599 : audit [DBG] from='client.? 172.21.15.6:0/762949129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:43.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:43 smithi006 bash[33918]: cluster 2023-08-27T22:47:42.075250+0000 mgr.y (mgr.14901) 1475 : cluster [DBG] pgmap v1482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:43.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:43 smithi006 bash[33918]: audit 2023-08-27T22:47:42.224594+0000 mon.c (mon.2) 599 : audit [DBG] from='client.? 172.21.15.6:0/762949129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:43.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:43 smithi130 bash[27579]: cluster 2023-08-27T22:47:42.075250+0000 mgr.y (mgr.14901) 1475 : cluster [DBG] pgmap v1482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:43.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:43 smithi130 bash[27579]: audit 2023-08-27T22:47:42.224594+0000 mon.c (mon.2) 599 : audit [DBG] from='client.? 172.21.15.6:0/762949129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:44.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:47:43.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:44.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:47:43.753Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:45.222 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:44 smithi006 bash[33918]: cluster 2023-08-27T22:47:44.076234+0000 mgr.y (mgr.14901) 1476 : cluster [DBG] pgmap v1483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:45.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:44 smithi006 bash[33918]: audit 2023-08-27T22:47:44.634176+0000 mon.c (mon.2) 600 : audit [DBG] from='client.? 172.21.15.6:0/3616387550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:45.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:44 smithi006 bash[27292]: cluster 2023-08-27T22:47:44.076234+0000 mgr.y (mgr.14901) 1476 : cluster [DBG] pgmap v1483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:45.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:44 smithi006 bash[27292]: audit 2023-08-27T22:47:44.634176+0000 mon.c (mon.2) 600 : audit [DBG] from='client.? 172.21.15.6:0/3616387550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:45.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:44 smithi130 bash[27579]: cluster 2023-08-27T22:47:44.076234+0000 mgr.y (mgr.14901) 1476 : cluster [DBG] pgmap v1483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:45.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:44 smithi130 bash[27579]: audit 2023-08-27T22:47:44.634176+0000 mon.c (mon.2) 600 : audit [DBG] from='client.? 172.21.15.6:0/3616387550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:47.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:47 smithi006 bash[27292]: cluster 2023-08-27T22:47:46.077018+0000 mgr.y (mgr.14901) 1477 : cluster [DBG] pgmap v1484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:47.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:47 smithi006 bash[27292]: audit 2023-08-27T22:47:47.051857+0000 mon.a (mon.0) 1600 : audit [DBG] from='client.? 172.21.15.6:0/3450672599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:47.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:47 smithi006 bash[33918]: cluster 2023-08-27T22:47:46.077018+0000 mgr.y (mgr.14901) 1477 : cluster [DBG] pgmap v1484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:47.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:47 smithi006 bash[33918]: audit 2023-08-27T22:47:47.051857+0000 mon.a (mon.0) 1600 : audit [DBG] from='client.? 172.21.15.6:0/3450672599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:47.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:47 smithi130 bash[27579]: cluster 2023-08-27T22:47:46.077018+0000 mgr.y (mgr.14901) 1477 : cluster [DBG] pgmap v1484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:47.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:47 smithi130 bash[27579]: audit 2023-08-27T22:47:47.051857+0000 mon.a (mon.0) 1600 : audit [DBG] from='client.? 172.21.15.6:0/3450672599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:49.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:49 smithi006 bash[33918]: cluster 2023-08-27T22:47:48.077653+0000 mgr.y (mgr.14901) 1478 : cluster [DBG] pgmap v1485: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:49.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:49 smithi006 bash[27292]: cluster 2023-08-27T22:47:48.077653+0000 mgr.y (mgr.14901) 1478 : cluster [DBG] pgmap v1485: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:49.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:49 smithi130 bash[27579]: cluster 2023-08-27T22:47:48.077653+0000 mgr.y (mgr.14901) 1478 : cluster [DBG] pgmap v1485: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:50.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:50 smithi006 bash[27292]: audit 2023-08-27T22:47:49.468085+0000 mon.a (mon.0) 1601 : audit [DBG] from='client.? 172.21.15.6:0/2461711904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:50.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:50 smithi006 bash[33918]: audit 2023-08-27T22:47:49.468085+0000 mon.a (mon.0) 1601 : audit [DBG] from='client.? 172.21.15.6:0/2461711904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:50.473 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:47:50.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:50.473 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:47:50.290Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:50.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:50 smithi130 bash[27579]: audit 2023-08-27T22:47:49.468085+0000 mon.a (mon.0) 1601 : audit [DBG] from='client.? 172.21.15.6:0/2461711904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:51.158 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:47:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:47:50] "GET /metrics HTTP/1.1" 200 193080 "" "Prometheus/2.33.4" 2023-08-27T22:47:51.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:51 smithi006 bash[33918]: cluster 2023-08-27T22:47:50.078641+0000 mgr.y (mgr.14901) 1479 : cluster [DBG] pgmap v1486: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:51.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:51 smithi006 bash[27292]: cluster 2023-08-27T22:47:50.078641+0000 mgr.y (mgr.14901) 1479 : cluster [DBG] pgmap v1486: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:51.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:51 smithi130 bash[27579]: cluster 2023-08-27T22:47:50.078641+0000 mgr.y (mgr.14901) 1479 : cluster [DBG] pgmap v1486: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:52.167 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:47:51.856Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:52.167 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:47:51.858Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:52.167 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:52 smithi006 bash[27292]: audit 2023-08-27T22:47:51.882697+0000 mon.c (mon.2) 601 : audit [DBG] from='client.? 172.21.15.6:0/936373869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:52.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:52 smithi006 bash[33918]: audit 2023-08-27T22:47:51.882697+0000 mon.c (mon.2) 601 : audit [DBG] from='client.? 172.21.15.6:0/936373869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:52.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:52 smithi130 bash[27579]: audit 2023-08-27T22:47:51.882697+0000 mon.c (mon.2) 601 : audit [DBG] from='client.? 172.21.15.6:0/936373869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:53 smithi006 bash[33918]: cluster 2023-08-27T22:47:52.079341+0000 mgr.y (mgr.14901) 1480 : cluster [DBG] pgmap v1487: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:53.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:53 smithi006 bash[27292]: cluster 2023-08-27T22:47:52.079341+0000 mgr.y (mgr.14901) 1480 : cluster [DBG] pgmap v1487: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:53.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:53 smithi130 bash[27579]: cluster 2023-08-27T22:47:52.079341+0000 mgr.y (mgr.14901) 1480 : cluster [DBG] pgmap v1487: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:54.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:47:53.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:54.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:47:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:47:53.753Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:47:55.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:54 smithi130 bash[27579]: cluster 2023-08-27T22:47:54.080399+0000 mgr.y (mgr.14901) 1481 : cluster [DBG] pgmap v1488: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:55.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:54 smithi130 bash[27579]: audit 2023-08-27T22:47:54.293303+0000 mon.a (mon.0) 1602 : audit [DBG] from='client.? 172.21.15.6:0/3501719917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:55.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:54 smithi006 bash[33918]: cluster 2023-08-27T22:47:54.080399+0000 mgr.y (mgr.14901) 1481 : cluster [DBG] pgmap v1488: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:55.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:54 smithi006 bash[33918]: audit 2023-08-27T22:47:54.293303+0000 mon.a (mon.0) 1602 : audit [DBG] from='client.? 172.21.15.6:0/3501719917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:55.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:54 smithi006 bash[27292]: cluster 2023-08-27T22:47:54.080399+0000 mgr.y (mgr.14901) 1481 : cluster [DBG] pgmap v1488: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:55.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:54 smithi006 bash[27292]: audit 2023-08-27T22:47:54.293303+0000 mon.a (mon.0) 1602 : audit [DBG] from='client.? 172.21.15.6:0/3501719917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:57.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:57 smithi006 bash[27292]: cluster 2023-08-27T22:47:56.081170+0000 mgr.y (mgr.14901) 1482 : cluster [DBG] pgmap v1489: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:57.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:57 smithi006 bash[27292]: audit 2023-08-27T22:47:56.707598+0000 mon.c (mon.2) 602 : audit [DBG] from='client.? 172.21.15.6:0/3394411794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:57.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:57 smithi006 bash[33918]: cluster 2023-08-27T22:47:56.081170+0000 mgr.y (mgr.14901) 1482 : cluster [DBG] pgmap v1489: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:57.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:57 smithi006 bash[33918]: audit 2023-08-27T22:47:56.707598+0000 mon.c (mon.2) 602 : audit [DBG] from='client.? 172.21.15.6:0/3394411794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:57.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:57 smithi130 bash[27579]: cluster 2023-08-27T22:47:56.081170+0000 mgr.y (mgr.14901) 1482 : cluster [DBG] pgmap v1489: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:57.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:57 smithi130 bash[27579]: audit 2023-08-27T22:47:56.707598+0000 mon.c (mon.2) 602 : audit [DBG] from='client.? 172.21.15.6:0/3394411794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:59.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:59 smithi006 bash[27292]: cluster 2023-08-27T22:47:58.081798+0000 mgr.y (mgr.14901) 1483 : cluster [DBG] pgmap v1490: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:59.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:47:59 smithi006 bash[27292]: audit 2023-08-27T22:47:59.129870+0000 mon.c (mon.2) 603 : audit [DBG] from='client.? 172.21.15.6:0/2619522010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:59.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:59 smithi006 bash[33918]: cluster 2023-08-27T22:47:58.081798+0000 mgr.y (mgr.14901) 1483 : cluster [DBG] pgmap v1490: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:59.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:47:59 smithi006 bash[33918]: audit 2023-08-27T22:47:59.129870+0000 mon.c (mon.2) 603 : audit [DBG] from='client.? 172.21.15.6:0/2619522010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:59.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:59 smithi130 bash[27579]: cluster 2023-08-27T22:47:58.081798+0000 mgr.y (mgr.14901) 1483 : cluster [DBG] pgmap v1490: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:47:59.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:47:59 smithi130 bash[27579]: audit 2023-08-27T22:47:59.129870+0000 mon.c (mon.2) 603 : audit [DBG] from='client.? 172.21.15.6:0/2619522010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:00.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:48:00.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:00.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:48:00.291Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:01.156 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:48:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:48:00] "GET /metrics HTTP/1.1" 200 193080 "" "Prometheus/2.33.4" 2023-08-27T22:48:01.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:01 smithi006 bash[33918]: cluster 2023-08-27T22:48:00.082486+0000 mgr.y (mgr.14901) 1484 : cluster [DBG] pgmap v1491: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:01.484 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:01 smithi006 bash[27292]: cluster 2023-08-27T22:48:00.082486+0000 mgr.y (mgr.14901) 1484 : cluster [DBG] pgmap v1491: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:01.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:01 smithi130 bash[27579]: cluster 2023-08-27T22:48:00.082486+0000 mgr.y (mgr.14901) 1484 : cluster [DBG] pgmap v1491: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:02.164 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:48:01.857Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:02.168 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:48:01.859Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:02.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:02 smithi006 bash[27292]: audit 2023-08-27T22:48:01.547196+0000 mon.a (mon.0) 1603 : audit [DBG] from='client.? 172.21.15.6:0/1881588794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:02.482 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:02 smithi006 bash[33918]: audit 2023-08-27T22:48:01.547196+0000 mon.a (mon.0) 1603 : audit [DBG] from='client.? 172.21.15.6:0/1881588794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:02.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:02 smithi130 bash[27579]: audit 2023-08-27T22:48:01.547196+0000 mon.a (mon.0) 1603 : audit [DBG] from='client.? 172.21.15.6:0/1881588794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:03.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:03 smithi006 bash[27292]: cluster 2023-08-27T22:48:02.083209+0000 mgr.y (mgr.14901) 1485 : cluster [DBG] pgmap v1492: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:03.649 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:03 smithi006 bash[33918]: cluster 2023-08-27T22:48:02.083209+0000 mgr.y (mgr.14901) 1485 : cluster [DBG] pgmap v1492: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:03.649 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:03 smithi130 bash[27579]: cluster 2023-08-27T22:48:02.083209+0000 mgr.y (mgr.14901) 1485 : cluster [DBG] pgmap v1492: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:04.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:48:03.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:04.224 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:48:03.753Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:04 smithi130 bash[27579]: audit 2023-08-27T22:48:03.959250+0000 mon.c (mon.2) 604 : audit [DBG] from='client.? 172.21.15.6:0/3571801891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:04 smithi130 bash[27579]: cluster 2023-08-27T22:48:04.084239+0000 mgr.y (mgr.14901) 1486 : cluster [DBG] pgmap v1493: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:05.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:04 smithi006 bash[27292]: audit 2023-08-27T22:48:03.959250+0000 mon.c (mon.2) 604 : audit [DBG] from='client.? 172.21.15.6:0/3571801891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:05.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:04 smithi006 bash[27292]: cluster 2023-08-27T22:48:04.084239+0000 mgr.y (mgr.14901) 1486 : cluster [DBG] pgmap v1493: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:05.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:04 smithi006 bash[33918]: audit 2023-08-27T22:48:03.959250+0000 mon.c (mon.2) 604 : audit [DBG] from='client.? 172.21.15.6:0/3571801891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:05.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:04 smithi006 bash[33918]: cluster 2023-08-27T22:48:04.084239+0000 mgr.y (mgr.14901) 1486 : cluster [DBG] pgmap v1493: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:07.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:07 smithi006 bash[27292]: cluster 2023-08-27T22:48:06.084904+0000 mgr.y (mgr.14901) 1487 : cluster [DBG] pgmap v1494: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:07.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:07 smithi006 bash[27292]: audit 2023-08-27T22:48:06.382988+0000 mon.c (mon.2) 605 : audit [DBG] from='client.? 172.21.15.6:0/1017212775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:07.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:07 smithi006 bash[33918]: cluster 2023-08-27T22:48:06.084904+0000 mgr.y (mgr.14901) 1487 : cluster [DBG] pgmap v1494: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:07.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:07 smithi006 bash[33918]: audit 2023-08-27T22:48:06.382988+0000 mon.c (mon.2) 605 : audit [DBG] from='client.? 172.21.15.6:0/1017212775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:07.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:07 smithi130 bash[27579]: cluster 2023-08-27T22:48:06.084904+0000 mgr.y (mgr.14901) 1487 : cluster [DBG] pgmap v1494: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:07.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:07 smithi130 bash[27579]: audit 2023-08-27T22:48:06.382988+0000 mon.c (mon.2) 605 : audit [DBG] from='client.? 172.21.15.6:0/1017212775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:09.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:09 smithi006 bash[33918]: cluster 2023-08-27T22:48:08.085549+0000 mgr.y (mgr.14901) 1488 : cluster [DBG] pgmap v1495: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:09.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:09 smithi006 bash[33918]: audit 2023-08-27T22:48:08.804012+0000 mon.c (mon.2) 606 : audit [DBG] from='client.? 172.21.15.6:0/4274030217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:09.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:09 smithi006 bash[27292]: cluster 2023-08-27T22:48:08.085549+0000 mgr.y (mgr.14901) 1488 : cluster [DBG] pgmap v1495: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:09.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:09 smithi006 bash[27292]: audit 2023-08-27T22:48:08.804012+0000 mon.c (mon.2) 606 : audit [DBG] from='client.? 172.21.15.6:0/4274030217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:09.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:09 smithi130 bash[27579]: cluster 2023-08-27T22:48:08.085549+0000 mgr.y (mgr.14901) 1488 : cluster [DBG] pgmap v1495: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:09.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:09 smithi130 bash[27579]: audit 2023-08-27T22:48:08.804012+0000 mon.c (mon.2) 606 : audit [DBG] from='client.? 172.21.15.6:0/4274030217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:10.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:48:10.289Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:10.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:48:10.291Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:11.160 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:48:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:48:10] "GET /metrics HTTP/1.1" 200 193080 "" "Prometheus/2.33.4" 2023-08-27T22:48:11.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:11 smithi006 bash[27292]: cluster 2023-08-27T22:48:10.086677+0000 mgr.y (mgr.14901) 1489 : cluster [DBG] pgmap v1496: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:11.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:11 smithi006 bash[33918]: cluster 2023-08-27T22:48:10.086677+0000 mgr.y (mgr.14901) 1489 : cluster [DBG] pgmap v1496: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:11.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:11 smithi130 bash[27579]: cluster 2023-08-27T22:48:10.086677+0000 mgr.y (mgr.14901) 1489 : cluster [DBG] pgmap v1496: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:12.167 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:48:11.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:12.168 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:48:11.860Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:12.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:12 smithi006 bash[33918]: audit 2023-08-27T22:48:11.225154+0000 mon.a (mon.0) 1604 : audit [DBG] from='client.? 172.21.15.6:0/4239045554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:12.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:12 smithi006 bash[27292]: audit 2023-08-27T22:48:11.225154+0000 mon.a (mon.0) 1604 : audit [DBG] from='client.? 172.21.15.6:0/4239045554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:12.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:12 smithi130 bash[27579]: audit 2023-08-27T22:48:11.225154+0000 mon.a (mon.0) 1604 : audit [DBG] from='client.? 172.21.15.6:0/4239045554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:13.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:13 smithi006 bash[33918]: cluster 2023-08-27T22:48:12.087344+0000 mgr.y (mgr.14901) 1490 : cluster [DBG] pgmap v1497: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:13.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:13 smithi006 bash[27292]: cluster 2023-08-27T22:48:12.087344+0000 mgr.y (mgr.14901) 1490 : cluster [DBG] pgmap v1497: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:13.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:13 smithi130 bash[27579]: cluster 2023-08-27T22:48:12.087344+0000 mgr.y (mgr.14901) 1490 : cluster [DBG] pgmap v1497: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:14.174 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:48:13.752Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:14.174 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:48:13.754Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:14.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:14 smithi006 bash[27292]: audit 2023-08-27T22:48:13.646556+0000 mon.c (mon.2) 607 : audit [DBG] from='client.? 172.21.15.6:0/2579388676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:14.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:14 smithi006 bash[33918]: audit 2023-08-27T22:48:13.646556+0000 mon.c (mon.2) 607 : audit [DBG] from='client.? 172.21.15.6:0/2579388676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:14.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:14 smithi130 bash[27579]: audit 2023-08-27T22:48:13.646556+0000 mon.c (mon.2) 607 : audit [DBG] from='client.? 172.21.15.6:0/2579388676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:15.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:15 smithi006 bash[27292]: cluster 2023-08-27T22:48:14.088361+0000 mgr.y (mgr.14901) 1491 : cluster [DBG] pgmap v1498: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:15.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:15 smithi006 bash[33918]: cluster 2023-08-27T22:48:14.088361+0000 mgr.y (mgr.14901) 1491 : cluster [DBG] pgmap v1498: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:15.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:15 smithi130 bash[27579]: cluster 2023-08-27T22:48:14.088361+0000 mgr.y (mgr.14901) 1491 : cluster [DBG] pgmap v1498: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:16.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:16 smithi006 bash[27292]: audit 2023-08-27T22:48:16.063749+0000 mon.c (mon.2) 608 : audit [DBG] from='client.? 172.21.15.6:0/787242705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:16.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:16 smithi006 bash[33918]: audit 2023-08-27T22:48:16.063749+0000 mon.c (mon.2) 608 : audit [DBG] from='client.? 172.21.15.6:0/787242705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:16.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:16 smithi130 bash[27579]: audit 2023-08-27T22:48:16.063749+0000 mon.c (mon.2) 608 : audit [DBG] from='client.? 172.21.15.6:0/787242705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:17.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:17 smithi006 bash[33918]: cluster 2023-08-27T22:48:16.089146+0000 mgr.y (mgr.14901) 1492 : cluster [DBG] pgmap v1499: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:17.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:17 smithi006 bash[27292]: cluster 2023-08-27T22:48:16.089146+0000 mgr.y (mgr.14901) 1492 : cluster [DBG] pgmap v1499: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:17.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:17 smithi130 bash[27579]: cluster 2023-08-27T22:48:16.089146+0000 mgr.y (mgr.14901) 1492 : cluster [DBG] pgmap v1499: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:19.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:19 smithi006 bash[33918]: cluster 2023-08-27T22:48:18.089778+0000 mgr.y (mgr.14901) 1493 : cluster [DBG] pgmap v1500: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:19.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:19 smithi006 bash[33918]: audit 2023-08-27T22:48:18.469774+0000 mon.c (mon.2) 609 : audit [DBG] from='client.? 172.21.15.6:0/3759966964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:19.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:19 smithi006 bash[27292]: cluster 2023-08-27T22:48:18.089778+0000 mgr.y (mgr.14901) 1493 : cluster [DBG] pgmap v1500: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:19.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:19 smithi006 bash[27292]: audit 2023-08-27T22:48:18.469774+0000 mon.c (mon.2) 609 : audit [DBG] from='client.? 172.21.15.6:0/3759966964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:19.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:19 smithi130 bash[27579]: cluster 2023-08-27T22:48:18.089778+0000 mgr.y (mgr.14901) 1493 : cluster [DBG] pgmap v1500: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:19 smithi130 bash[27579]: audit 2023-08-27T22:48:18.469774+0000 mon.c (mon.2) 609 : audit [DBG] from='client.? 172.21.15.6:0/3759966964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:20.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:48:20.291Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:20.724 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:48:20.293Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:21.196 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:48:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:48:20] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T22:48:21.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:21 smithi006 bash[27292]: cluster 2023-08-27T22:48:20.090929+0000 mgr.y (mgr.14901) 1494 : cluster [DBG] pgmap v1501: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:21.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:21 smithi006 bash[27292]: audit 2023-08-27T22:48:20.883131+0000 mon.c (mon.2) 610 : audit [DBG] from='client.? 172.21.15.6:0/1512720861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:21.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:21 smithi006 bash[33918]: cluster 2023-08-27T22:48:20.090929+0000 mgr.y (mgr.14901) 1494 : cluster [DBG] pgmap v1501: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:21.474 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:21 smithi006 bash[33918]: audit 2023-08-27T22:48:20.883131+0000 mon.c (mon.2) 610 : audit [DBG] from='client.? 172.21.15.6:0/1512720861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:21.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:21 smithi130 bash[27579]: cluster 2023-08-27T22:48:20.090929+0000 mgr.y (mgr.14901) 1494 : cluster [DBG] pgmap v1501: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:21.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:21 smithi130 bash[27579]: audit 2023-08-27T22:48:20.883131+0000 mon.c (mon.2) 610 : audit [DBG] from='client.? 172.21.15.6:0/1512720861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:48:21.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:22.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:48:21.860Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:23.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:23 smithi006 bash[27292]: cluster 2023-08-27T22:48:22.091630+0000 mgr.y (mgr.14901) 1495 : cluster [DBG] pgmap v1502: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:23.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:23 smithi006 bash[33918]: cluster 2023-08-27T22:48:22.091630+0000 mgr.y (mgr.14901) 1495 : cluster [DBG] pgmap v1502: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:23.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:23 smithi130 bash[27579]: cluster 2023-08-27T22:48:22.091630+0000 mgr.y (mgr.14901) 1495 : cluster [DBG] pgmap v1502: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:24.202 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:48:23.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:24.202 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:48:23.756Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:24.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:24 smithi006 bash[33918]: audit 2023-08-27T22:48:23.306698+0000 mon.a (mon.0) 1605 : audit [DBG] from='client.? 172.21.15.6:0/1349513804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:24.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:24 smithi006 bash[27292]: audit 2023-08-27T22:48:23.306698+0000 mon.a (mon.0) 1605 : audit [DBG] from='client.? 172.21.15.6:0/1349513804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:24.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:24 smithi130 bash[27579]: audit 2023-08-27T22:48:23.306698+0000 mon.a (mon.0) 1605 : audit [DBG] from='client.? 172.21.15.6:0/1349513804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:25.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:25 smithi006 bash[27292]: cluster 2023-08-27T22:48:24.092582+0000 mgr.y (mgr.14901) 1496 : cluster [DBG] pgmap v1503: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:25.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:25 smithi006 bash[33918]: cluster 2023-08-27T22:48:24.092582+0000 mgr.y (mgr.14901) 1496 : cluster [DBG] pgmap v1503: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:25.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:25 smithi130 bash[27579]: cluster 2023-08-27T22:48:24.092582+0000 mgr.y (mgr.14901) 1496 : cluster [DBG] pgmap v1503: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:26.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:26 smithi006 bash[27292]: audit 2023-08-27T22:48:25.715149+0000 mon.c (mon.2) 611 : audit [DBG] from='client.? 172.21.15.6:0/479304641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:26.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:26 smithi006 bash[33918]: audit 2023-08-27T22:48:25.715149+0000 mon.c (mon.2) 611 : audit [DBG] from='client.? 172.21.15.6:0/479304641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:26.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:26 smithi130 bash[27579]: audit 2023-08-27T22:48:25.715149+0000 mon.c (mon.2) 611 : audit [DBG] from='client.? 172.21.15.6:0/479304641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:27.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:27 smithi006 bash[33918]: cluster 2023-08-27T22:48:26.093392+0000 mgr.y (mgr.14901) 1497 : cluster [DBG] pgmap v1504: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:27.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:27 smithi006 bash[27292]: cluster 2023-08-27T22:48:26.093392+0000 mgr.y (mgr.14901) 1497 : cluster [DBG] pgmap v1504: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:27.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:27 smithi130 bash[27579]: cluster 2023-08-27T22:48:26.093392+0000 mgr.y (mgr.14901) 1497 : cluster [DBG] pgmap v1504: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:28.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:28 smithi130 bash[27579]: audit 2023-08-27T22:48:28.132955+0000 mon.c (mon.2) 612 : audit [DBG] from='client.? 172.21.15.6:0/1644588800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:28.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:28 smithi006 bash[27292]: audit 2023-08-27T22:48:28.132955+0000 mon.c (mon.2) 612 : audit [DBG] from='client.? 172.21.15.6:0/1644588800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:28.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:28 smithi006 bash[33918]: audit 2023-08-27T22:48:28.132955+0000 mon.c (mon.2) 612 : audit [DBG] from='client.? 172.21.15.6:0/1644588800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:29.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:29 smithi130 bash[27579]: cluster 2023-08-27T22:48:28.093957+0000 mgr.y (mgr.14901) 1498 : cluster [DBG] pgmap v1505: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:29.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:29 smithi006 bash[27292]: cluster 2023-08-27T22:48:28.093957+0000 mgr.y (mgr.14901) 1498 : cluster [DBG] pgmap v1505: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:29.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:29 smithi006 bash[33918]: cluster 2023-08-27T22:48:28.093957+0000 mgr.y (mgr.14901) 1498 : cluster [DBG] pgmap v1505: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:30.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:48:30.291Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:30.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:48:30.293Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:31.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:48:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:48:30] "GET /metrics HTTP/1.1" 200 193084 "" "Prometheus/2.33.4" 2023-08-27T22:48:31.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:31 smithi130 bash[27579]: cluster 2023-08-27T22:48:30.095050+0000 mgr.y (mgr.14901) 1499 : cluster [DBG] pgmap v1506: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:31.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:31 smithi130 bash[27579]: audit 2023-08-27T22:48:30.564556+0000 mon.a (mon.0) 1606 : audit [DBG] from='client.? 172.21.15.6:0/3802115048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:31.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:31 smithi006 bash[33918]: cluster 2023-08-27T22:48:30.095050+0000 mgr.y (mgr.14901) 1499 : cluster [DBG] pgmap v1506: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:31.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:31 smithi006 bash[33918]: audit 2023-08-27T22:48:30.564556+0000 mon.a (mon.0) 1606 : audit [DBG] from='client.? 172.21.15.6:0/3802115048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:31.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:31 smithi006 bash[27292]: cluster 2023-08-27T22:48:30.095050+0000 mgr.y (mgr.14901) 1499 : cluster [DBG] pgmap v1506: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:31.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:31 smithi006 bash[27292]: audit 2023-08-27T22:48:30.564556+0000 mon.a (mon.0) 1606 : audit [DBG] from='client.? 172.21.15.6:0/3802115048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:48:31.859Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:32.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:48:31.861Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:33.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:33 smithi130 bash[27579]: cluster 2023-08-27T22:48:32.095761+0000 mgr.y (mgr.14901) 1500 : cluster [DBG] pgmap v1507: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:33.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:33 smithi130 bash[27579]: audit 2023-08-27T22:48:32.985107+0000 mon.a (mon.0) 1607 : audit [DBG] from='client.? 172.21.15.6:0/1507276520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:33.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:33 smithi006 bash[27292]: cluster 2023-08-27T22:48:32.095761+0000 mgr.y (mgr.14901) 1500 : cluster [DBG] pgmap v1507: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:33.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:33 smithi006 bash[27292]: audit 2023-08-27T22:48:32.985107+0000 mon.a (mon.0) 1607 : audit [DBG] from='client.? 172.21.15.6:0/1507276520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:33.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:33 smithi006 bash[33918]: cluster 2023-08-27T22:48:32.095761+0000 mgr.y (mgr.14901) 1500 : cluster [DBG] pgmap v1507: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:33.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:33 smithi006 bash[33918]: audit 2023-08-27T22:48:32.985107+0000 mon.a (mon.0) 1607 : audit [DBG] from='client.? 172.21.15.6:0/1507276520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:34.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:48:33.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:34.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:48:33.757Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:35.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:34 smithi130 bash[27579]: cluster 2023-08-27T22:48:34.096750+0000 mgr.y (mgr.14901) 1501 : cluster [DBG] pgmap v1508: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:35.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:34 smithi006 bash[27292]: cluster 2023-08-27T22:48:34.096750+0000 mgr.y (mgr.14901) 1501 : cluster [DBG] pgmap v1508: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:35.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:34 smithi006 bash[33918]: cluster 2023-08-27T22:48:34.096750+0000 mgr.y (mgr.14901) 1501 : cluster [DBG] pgmap v1508: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:36.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:35 smithi130 bash[27579]: audit 2023-08-27T22:48:35.397069+0000 mon.c (mon.2) 613 : audit [DBG] from='client.? 172.21.15.6:0/1954884361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:36.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:35 smithi006 bash[33918]: audit 2023-08-27T22:48:35.397069+0000 mon.c (mon.2) 613 : audit [DBG] from='client.? 172.21.15.6:0/1954884361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:36.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:35 smithi006 bash[27292]: audit 2023-08-27T22:48:35.397069+0000 mon.c (mon.2) 613 : audit [DBG] from='client.? 172.21.15.6:0/1954884361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:37.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:36 smithi130 bash[27579]: cluster 2023-08-27T22:48:36.097488+0000 mgr.y (mgr.14901) 1502 : cluster [DBG] pgmap v1509: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:37.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:36 smithi006 bash[33918]: cluster 2023-08-27T22:48:36.097488+0000 mgr.y (mgr.14901) 1502 : cluster [DBG] pgmap v1509: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:37.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:36 smithi006 bash[27292]: cluster 2023-08-27T22:48:36.097488+0000 mgr.y (mgr.14901) 1502 : cluster [DBG] pgmap v1509: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:38.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:37 smithi130 bash[27579]: audit 2023-08-27T22:48:37.808633+0000 mon.c (mon.2) 614 : audit [DBG] from='client.? 172.21.15.6:0/601919133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:38.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:37 smithi006 bash[33918]: audit 2023-08-27T22:48:37.808633+0000 mon.c (mon.2) 614 : audit [DBG] from='client.? 172.21.15.6:0/601919133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:38.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:37 smithi006 bash[27292]: audit 2023-08-27T22:48:37.808633+0000 mon.c (mon.2) 614 : audit [DBG] from='client.? 172.21.15.6:0/601919133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:39.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:39 smithi130 bash[27579]: cluster 2023-08-27T22:48:38.098120+0000 mgr.y (mgr.14901) 1503 : cluster [DBG] pgmap v1510: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:39.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:38 smithi006 bash[33918]: cluster 2023-08-27T22:48:38.098120+0000 mgr.y (mgr.14901) 1503 : cluster [DBG] pgmap v1510: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:39.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:38 smithi006 bash[27292]: cluster 2023-08-27T22:48:38.098120+0000 mgr.y (mgr.14901) 1503 : cluster [DBG] pgmap v1510: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:40.300 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:40 smithi006 bash[33918]: audit 2023-08-27T22:48:39.249204+0000 mon.b (mon.1) 266 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:48:40.300 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:40 smithi006 bash[33918]: audit 2023-08-27T22:48:39.520758+0000 mon.a (mon.0) 1608 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:48:40.300 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:40 smithi006 bash[33918]: audit 2023-08-27T22:48:39.529122+0000 mon.a (mon.0) 1609 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:48:40.301 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:40 smithi006 bash[27292]: audit 2023-08-27T22:48:39.249204+0000 mon.b (mon.1) 266 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:48:40.301 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:40 smithi006 bash[27292]: audit 2023-08-27T22:48:39.520758+0000 mon.a (mon.0) 1608 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:48:40.301 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:40 smithi006 bash[27292]: audit 2023-08-27T22:48:39.529122+0000 mon.a (mon.0) 1609 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:48:40.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:40 smithi130 bash[27579]: audit 2023-08-27T22:48:39.249204+0000 mon.b (mon.1) 266 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:48:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:40 smithi130 bash[27579]: audit 2023-08-27T22:48:39.520758+0000 mon.a (mon.0) 1608 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:48:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:40 smithi130 bash[27579]: audit 2023-08-27T22:48:39.529122+0000 mon.a (mon.0) 1609 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:48:40.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:48:40.292Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:40.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:48:40.294Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:41.020 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:48:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:48:40] "GET /metrics HTTP/1.1" 200 193084 "" "Prometheus/2.33.4" 2023-08-27T22:48:41.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:41 smithi130 bash[27579]: cluster 2023-08-27T22:48:40.099182+0000 mgr.y (mgr.14901) 1504 : cluster [DBG] pgmap v1511: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:41.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:41 smithi130 bash[27579]: audit 2023-08-27T22:48:40.227765+0000 mon.a (mon.0) 1610 : audit [DBG] from='client.? 172.21.15.6:0/1457381899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:41.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:41 smithi006 bash[27292]: cluster 2023-08-27T22:48:40.099182+0000 mgr.y (mgr.14901) 1504 : cluster [DBG] pgmap v1511: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:41.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:41 smithi006 bash[27292]: audit 2023-08-27T22:48:40.227765+0000 mon.a (mon.0) 1610 : audit [DBG] from='client.? 172.21.15.6:0/1457381899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:41.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:41 smithi006 bash[33918]: cluster 2023-08-27T22:48:40.099182+0000 mgr.y (mgr.14901) 1504 : cluster [DBG] pgmap v1511: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:41.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:41 smithi006 bash[33918]: audit 2023-08-27T22:48:40.227765+0000 mon.a (mon.0) 1610 : audit [DBG] from='client.? 172.21.15.6:0/1457381899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:48:41.859Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:42.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:48:41.861Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:43.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:43 smithi006 bash[27292]: cluster 2023-08-27T22:48:42.099820+0000 mgr.y (mgr.14901) 1505 : cluster [DBG] pgmap v1512: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:43.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:43 smithi006 bash[27292]: audit 2023-08-27T22:48:42.649386+0000 mon.c (mon.2) 615 : audit [DBG] from='client.? 172.21.15.6:0/1256644439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:43.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:43 smithi006 bash[33918]: cluster 2023-08-27T22:48:42.099820+0000 mgr.y (mgr.14901) 1505 : cluster [DBG] pgmap v1512: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:43.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:43 smithi006 bash[33918]: audit 2023-08-27T22:48:42.649386+0000 mon.c (mon.2) 615 : audit [DBG] from='client.? 172.21.15.6:0/1256644439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:43.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:43 smithi130 bash[27579]: cluster 2023-08-27T22:48:42.099820+0000 mgr.y (mgr.14901) 1505 : cluster [DBG] pgmap v1512: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:43.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:43 smithi130 bash[27579]: audit 2023-08-27T22:48:42.649386+0000 mon.c (mon.2) 615 : audit [DBG] from='client.? 172.21.15.6:0/1256644439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:44.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:48:43.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:44.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:48:43.757Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:45.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:44 smithi130 bash[27579]: cluster 2023-08-27T22:48:44.100911+0000 mgr.y (mgr.14901) 1506 : cluster [DBG] pgmap v1513: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:45.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:44 smithi006 bash[27292]: cluster 2023-08-27T22:48:44.100911+0000 mgr.y (mgr.14901) 1506 : cluster [DBG] pgmap v1513: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:45.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:44 smithi006 bash[33918]: cluster 2023-08-27T22:48:44.100911+0000 mgr.y (mgr.14901) 1506 : cluster [DBG] pgmap v1513: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:46.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:45 smithi130 bash[27579]: audit 2023-08-27T22:48:45.110808+0000 mon.a (mon.0) 1611 : audit [DBG] from='client.? 172.21.15.6:0/2728254271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:46.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:45 smithi130 bash[27579]: audit 2023-08-27T22:48:45.383116+0000 mon.a (mon.0) 1612 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:48:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:45 smithi130 bash[27579]: audit 2023-08-27T22:48:45.870414+0000 mon.a (mon.0) 1613 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:48:46.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:45 smithi006 bash[27292]: audit 2023-08-27T22:48:45.110808+0000 mon.a (mon.0) 1611 : audit [DBG] from='client.? 172.21.15.6:0/2728254271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:46.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:45 smithi006 bash[27292]: audit 2023-08-27T22:48:45.383116+0000 mon.a (mon.0) 1612 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:48:46.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:45 smithi006 bash[27292]: audit 2023-08-27T22:48:45.870414+0000 mon.a (mon.0) 1613 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:48:46.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:45 smithi006 bash[33918]: audit 2023-08-27T22:48:45.110808+0000 mon.a (mon.0) 1611 : audit [DBG] from='client.? 172.21.15.6:0/2728254271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:46.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:45 smithi006 bash[33918]: audit 2023-08-27T22:48:45.383116+0000 mon.a (mon.0) 1612 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:48:46.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:45 smithi006 bash[33918]: audit 2023-08-27T22:48:45.870414+0000 mon.a (mon.0) 1613 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:48:47.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:47 smithi130 bash[27579]: cluster 2023-08-27T22:48:46.101628+0000 mgr.y (mgr.14901) 1507 : cluster [DBG] pgmap v1514: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:47.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:47 smithi130 bash[27579]: audit 2023-08-27T22:48:46.174813+0000 mon.b (mon.1) 267 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:48:47.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:47 smithi130 bash[27579]: audit 2023-08-27T22:48:46.177117+0000 mon.b (mon.1) 268 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:48:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:47 smithi130 bash[27579]: audit 2023-08-27T22:48:46.183550+0000 mon.a (mon.0) 1614 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:48:47.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:46 smithi006 bash[33918]: cluster 2023-08-27T22:48:46.101628+0000 mgr.y (mgr.14901) 1507 : cluster [DBG] pgmap v1514: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:47.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:46 smithi006 bash[33918]: audit 2023-08-27T22:48:46.174813+0000 mon.b (mon.1) 267 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:48:47.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:46 smithi006 bash[33918]: audit 2023-08-27T22:48:46.177117+0000 mon.b (mon.1) 268 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:48:47.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:46 smithi006 bash[33918]: audit 2023-08-27T22:48:46.183550+0000 mon.a (mon.0) 1614 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:48:47.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:46 smithi006 bash[27292]: cluster 2023-08-27T22:48:46.101628+0000 mgr.y (mgr.14901) 1507 : cluster [DBG] pgmap v1514: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:47.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:46 smithi006 bash[27292]: audit 2023-08-27T22:48:46.174813+0000 mon.b (mon.1) 267 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:48:47.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:46 smithi006 bash[27292]: audit 2023-08-27T22:48:46.177117+0000 mon.b (mon.1) 268 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:48:47.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:46 smithi006 bash[27292]: audit 2023-08-27T22:48:46.183550+0000 mon.a (mon.0) 1614 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:48:48.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:48 smithi130 bash[27579]: audit 2023-08-27T22:48:47.530542+0000 mon.a (mon.0) 1615 : audit [DBG] from='client.? 172.21.15.6:0/585830269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:48.543 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:47 smithi006 bash[27292]: audit 2023-08-27T22:48:47.530542+0000 mon.a (mon.0) 1615 : audit [DBG] from='client.? 172.21.15.6:0/585830269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:48.544 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:47 smithi006 bash[33918]: audit 2023-08-27T22:48:47.530542+0000 mon.a (mon.0) 1615 : audit [DBG] from='client.? 172.21.15.6:0/585830269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:49.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:49 smithi130 bash[27579]: cluster 2023-08-27T22:48:48.102308+0000 mgr.y (mgr.14901) 1508 : cluster [DBG] pgmap v1515: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:49.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:49 smithi006 bash[33918]: cluster 2023-08-27T22:48:48.102308+0000 mgr.y (mgr.14901) 1508 : cluster [DBG] pgmap v1515: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:49.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:49 smithi006 bash[27292]: cluster 2023-08-27T22:48:48.102308+0000 mgr.y (mgr.14901) 1508 : cluster [DBG] pgmap v1515: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:50.301 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:50 smithi006 bash[27292]: audit 2023-08-27T22:48:49.944864+0000 mon.a (mon.0) 1616 : audit [DBG] from='client.? 172.21.15.6:0/2782239201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:50.301 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:50 smithi006 bash[33918]: audit 2023-08-27T22:48:49.944864+0000 mon.a (mon.0) 1616 : audit [DBG] from='client.? 172.21.15.6:0/2782239201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:50.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:50 smithi130 bash[27579]: audit 2023-08-27T22:48:49.944864+0000 mon.a (mon.0) 1616 : audit [DBG] from='client.? 172.21.15.6:0/2782239201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:50.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:48:50.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:50.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:48:50.295Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:51.025 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:48:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:48:50] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T22:48:51.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:51 smithi130 bash[27579]: cluster 2023-08-27T22:48:50.103478+0000 mgr.y (mgr.14901) 1509 : cluster [DBG] pgmap v1516: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:51.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:51 smithi006 bash[33918]: cluster 2023-08-27T22:48:50.103478+0000 mgr.y (mgr.14901) 1509 : cluster [DBG] pgmap v1516: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:51.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:51 smithi006 bash[27292]: cluster 2023-08-27T22:48:50.103478+0000 mgr.y (mgr.14901) 1509 : cluster [DBG] pgmap v1516: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:48:51.859Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:52.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:48:51.862Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:53 smithi006 bash[33918]: cluster 2023-08-27T22:48:52.104059+0000 mgr.y (mgr.14901) 1510 : cluster [DBG] pgmap v1517: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:53 smithi006 bash[33918]: audit 2023-08-27T22:48:52.369588+0000 mon.a (mon.0) 1617 : audit [DBG] from='client.? 172.21.15.6:0/452161038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:53.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:53 smithi006 bash[27292]: cluster 2023-08-27T22:48:52.104059+0000 mgr.y (mgr.14901) 1510 : cluster [DBG] pgmap v1517: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:53.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:53 smithi006 bash[27292]: audit 2023-08-27T22:48:52.369588+0000 mon.a (mon.0) 1617 : audit [DBG] from='client.? 172.21.15.6:0/452161038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:53.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:53 smithi130 bash[27579]: cluster 2023-08-27T22:48:52.104059+0000 mgr.y (mgr.14901) 1510 : cluster [DBG] pgmap v1517: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:53.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:53 smithi130 bash[27579]: audit 2023-08-27T22:48:52.369588+0000 mon.a (mon.0) 1617 : audit [DBG] from='client.? 172.21.15.6:0/452161038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:54.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:48:53.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:54.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:48:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:48:53.758Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:48:55.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:54 smithi130 bash[27579]: cluster 2023-08-27T22:48:54.105078+0000 mgr.y (mgr.14901) 1511 : cluster [DBG] pgmap v1518: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:55.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:54 smithi130 bash[27579]: audit 2023-08-27T22:48:54.792134+0000 mon.c (mon.2) 616 : audit [DBG] from='client.? 172.21.15.6:0/62687896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:55.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:54 smithi006 bash[27292]: cluster 2023-08-27T22:48:54.105078+0000 mgr.y (mgr.14901) 1511 : cluster [DBG] pgmap v1518: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:55.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:54 smithi006 bash[27292]: audit 2023-08-27T22:48:54.792134+0000 mon.c (mon.2) 616 : audit [DBG] from='client.? 172.21.15.6:0/62687896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:55.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:54 smithi006 bash[33918]: cluster 2023-08-27T22:48:54.105078+0000 mgr.y (mgr.14901) 1511 : cluster [DBG] pgmap v1518: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:55.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:54 smithi006 bash[33918]: audit 2023-08-27T22:48:54.792134+0000 mon.c (mon.2) 616 : audit [DBG] from='client.? 172.21.15.6:0/62687896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:57.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:57 smithi006 bash[33918]: cluster 2023-08-27T22:48:56.105818+0000 mgr.y (mgr.14901) 1512 : cluster [DBG] pgmap v1519: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:57.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:57 smithi006 bash[27292]: cluster 2023-08-27T22:48:56.105818+0000 mgr.y (mgr.14901) 1512 : cluster [DBG] pgmap v1519: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:57.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:57 smithi130 bash[27579]: cluster 2023-08-27T22:48:56.105818+0000 mgr.y (mgr.14901) 1512 : cluster [DBG] pgmap v1519: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:58.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:58 smithi006 bash[27292]: audit 2023-08-27T22:48:57.209004+0000 mon.a (mon.0) 1618 : audit [DBG] from='client.? 172.21.15.6:0/3334662843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:58.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:58 smithi006 bash[33918]: audit 2023-08-27T22:48:57.209004+0000 mon.a (mon.0) 1618 : audit [DBG] from='client.? 172.21.15.6:0/3334662843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:58.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:58 smithi130 bash[27579]: audit 2023-08-27T22:48:57.209004+0000 mon.a (mon.0) 1618 : audit [DBG] from='client.? 172.21.15.6:0/3334662843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:59.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:48:59 smithi006 bash[27292]: cluster 2023-08-27T22:48:58.106460+0000 mgr.y (mgr.14901) 1513 : cluster [DBG] pgmap v1520: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:59.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:48:59 smithi006 bash[33918]: cluster 2023-08-27T22:48:58.106460+0000 mgr.y (mgr.14901) 1513 : cluster [DBG] pgmap v1520: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:48:59.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:48:59 smithi130 bash[27579]: cluster 2023-08-27T22:48:58.106460+0000 mgr.y (mgr.14901) 1513 : cluster [DBG] pgmap v1520: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:00.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:00 smithi006 bash[27292]: audit 2023-08-27T22:48:59.629838+0000 mon.a (mon.0) 1619 : audit [DBG] from='client.? 172.21.15.6:0/966112779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:00.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:00 smithi006 bash[33918]: audit 2023-08-27T22:48:59.629838+0000 mon.a (mon.0) 1619 : audit [DBG] from='client.? 172.21.15.6:0/966112779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:00.473 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:49:00.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:00.473 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:49:00.296Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:00.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:00 smithi130 bash[27579]: audit 2023-08-27T22:48:59.629838+0000 mon.a (mon.0) 1619 : audit [DBG] from='client.? 172.21.15.6:0/966112779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:01.191 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:49:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:49:00] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T22:49:01.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:01 smithi006 bash[27292]: cluster 2023-08-27T22:49:00.107548+0000 mgr.y (mgr.14901) 1514 : cluster [DBG] pgmap v1521: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:01.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:01 smithi006 bash[33918]: cluster 2023-08-27T22:49:00.107548+0000 mgr.y (mgr.14901) 1514 : cluster [DBG] pgmap v1521: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:01.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:01 smithi130 bash[27579]: cluster 2023-08-27T22:49:00.107548+0000 mgr.y (mgr.14901) 1514 : cluster [DBG] pgmap v1521: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:02.194 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:49:01.860Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:02.194 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:49:01.862Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:02.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:02 smithi006 bash[33918]: audit 2023-08-27T22:49:02.048188+0000 mon.c (mon.2) 617 : audit [DBG] from='client.? 172.21.15.6:0/3062622838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:02.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:02 smithi006 bash[27292]: audit 2023-08-27T22:49:02.048188+0000 mon.c (mon.2) 617 : audit [DBG] from='client.? 172.21.15.6:0/3062622838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:02.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:02 smithi130 bash[27579]: audit 2023-08-27T22:49:02.048188+0000 mon.c (mon.2) 617 : audit [DBG] from='client.? 172.21.15.6:0/3062622838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:03.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:03 smithi006 bash[27292]: cluster 2023-08-27T22:49:02.108220+0000 mgr.y (mgr.14901) 1515 : cluster [DBG] pgmap v1522: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:03.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:03 smithi006 bash[33918]: cluster 2023-08-27T22:49:02.108220+0000 mgr.y (mgr.14901) 1515 : cluster [DBG] pgmap v1522: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:03.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:03 smithi130 bash[27579]: cluster 2023-08-27T22:49:02.108220+0000 mgr.y (mgr.14901) 1515 : cluster [DBG] pgmap v1522: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:04.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:49:03.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:04.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:49:03.758Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:04 smithi130 bash[27579]: cluster 2023-08-27T22:49:04.109308+0000 mgr.y (mgr.14901) 1516 : cluster [DBG] pgmap v1523: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:04 smithi130 bash[27579]: audit 2023-08-27T22:49:04.459813+0000 mon.a (mon.0) 1620 : audit [DBG] from='client.? 172.21.15.6:0/2838102086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:05.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:04 smithi006 bash[27292]: cluster 2023-08-27T22:49:04.109308+0000 mgr.y (mgr.14901) 1516 : cluster [DBG] pgmap v1523: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:05.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:04 smithi006 bash[27292]: audit 2023-08-27T22:49:04.459813+0000 mon.a (mon.0) 1620 : audit [DBG] from='client.? 172.21.15.6:0/2838102086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:05.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:04 smithi006 bash[33918]: cluster 2023-08-27T22:49:04.109308+0000 mgr.y (mgr.14901) 1516 : cluster [DBG] pgmap v1523: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:05.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:04 smithi006 bash[33918]: audit 2023-08-27T22:49:04.459813+0000 mon.a (mon.0) 1620 : audit [DBG] from='client.? 172.21.15.6:0/2838102086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:07.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:07 smithi006 bash[27292]: cluster 2023-08-27T22:49:06.110058+0000 mgr.y (mgr.14901) 1517 : cluster [DBG] pgmap v1524: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:07.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:07 smithi006 bash[27292]: audit 2023-08-27T22:49:06.875789+0000 mon.a (mon.0) 1621 : audit [DBG] from='client.? 172.21.15.6:0/888517760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:07.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:07 smithi006 bash[33918]: cluster 2023-08-27T22:49:06.110058+0000 mgr.y (mgr.14901) 1517 : cluster [DBG] pgmap v1524: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:07.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:07 smithi006 bash[33918]: audit 2023-08-27T22:49:06.875789+0000 mon.a (mon.0) 1621 : audit [DBG] from='client.? 172.21.15.6:0/888517760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:07.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:07 smithi130 bash[27579]: cluster 2023-08-27T22:49:06.110058+0000 mgr.y (mgr.14901) 1517 : cluster [DBG] pgmap v1524: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:07.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:07 smithi130 bash[27579]: audit 2023-08-27T22:49:06.875789+0000 mon.a (mon.0) 1621 : audit [DBG] from='client.? 172.21.15.6:0/888517760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:09.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:09 smithi006 bash[33918]: cluster 2023-08-27T22:49:08.110720+0000 mgr.y (mgr.14901) 1518 : cluster [DBG] pgmap v1525: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:09.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:09 smithi006 bash[27292]: cluster 2023-08-27T22:49:08.110720+0000 mgr.y (mgr.14901) 1518 : cluster [DBG] pgmap v1525: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:09.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:09 smithi130 bash[27579]: cluster 2023-08-27T22:49:08.110720+0000 mgr.y (mgr.14901) 1518 : cluster [DBG] pgmap v1525: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:10.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:10 smithi006 bash[33918]: audit 2023-08-27T22:49:09.289875+0000 mon.c (mon.2) 618 : audit [DBG] from='client.? 172.21.15.6:0/1913403070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:11.296 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:10 smithi006 bash[27292]: audit 2023-08-27T22:49:09.289875+0000 mon.c (mon.2) 618 : audit [DBG] from='client.? 172.21.15.6:0/1913403070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:11.296 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:49:10.295Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:11.296 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:49:10.297Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:11.297 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:49:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:49:10] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T22:49:11.297 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:10 smithi130 bash[27579]: audit 2023-08-27T22:49:09.289875+0000 mon.c (mon.2) 618 : audit [DBG] from='client.? 172.21.15.6:0/1913403070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:11.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:11 smithi006 bash[33918]: cluster 2023-08-27T22:49:10.112225+0000 mgr.y (mgr.14901) 1519 : cluster [DBG] pgmap v1526: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:11.536 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:11 smithi006 bash[27292]: cluster 2023-08-27T22:49:10.112225+0000 mgr.y (mgr.14901) 1519 : cluster [DBG] pgmap v1526: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:11.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:11 smithi130 bash[27579]: cluster 2023-08-27T22:49:10.112225+0000 mgr.y (mgr.14901) 1519 : cluster [DBG] pgmap v1526: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:12.203 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:49:11.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:12.203 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:49:11.863Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:12.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:12 smithi006 bash[27292]: audit 2023-08-27T22:49:11.704636+0000 mon.c (mon.2) 619 : audit [DBG] from='client.? 172.21.15.6:0/3985982780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:12.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:12 smithi006 bash[33918]: audit 2023-08-27T22:49:11.704636+0000 mon.c (mon.2) 619 : audit [DBG] from='client.? 172.21.15.6:0/3985982780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:12.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:12 smithi130 bash[27579]: audit 2023-08-27T22:49:11.704636+0000 mon.c (mon.2) 619 : audit [DBG] from='client.? 172.21.15.6:0/3985982780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:13.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:13 smithi006 bash[33918]: cluster 2023-08-27T22:49:12.112974+0000 mgr.y (mgr.14901) 1520 : cluster [DBG] pgmap v1527: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:13.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:13 smithi006 bash[27292]: cluster 2023-08-27T22:49:12.112974+0000 mgr.y (mgr.14901) 1520 : cluster [DBG] pgmap v1527: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:13.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:13 smithi130 bash[27579]: cluster 2023-08-27T22:49:12.112974+0000 mgr.y (mgr.14901) 1520 : cluster [DBG] pgmap v1527: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:14.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:49:13.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:14.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:49:13.760Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:15.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:14 smithi130 bash[27579]: cluster 2023-08-27T22:49:14.113926+0000 mgr.y (mgr.14901) 1521 : cluster [DBG] pgmap v1528: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:15.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:14 smithi130 bash[27579]: audit 2023-08-27T22:49:14.127184+0000 mon.a (mon.0) 1622 : audit [DBG] from='client.? 172.21.15.6:0/1723459409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:15.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:14 smithi006 bash[33918]: cluster 2023-08-27T22:49:14.113926+0000 mgr.y (mgr.14901) 1521 : cluster [DBG] pgmap v1528: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:15.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:14 smithi006 bash[33918]: audit 2023-08-27T22:49:14.127184+0000 mon.a (mon.0) 1622 : audit [DBG] from='client.? 172.21.15.6:0/1723459409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:15.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:14 smithi006 bash[27292]: cluster 2023-08-27T22:49:14.113926+0000 mgr.y (mgr.14901) 1521 : cluster [DBG] pgmap v1528: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:15.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:14 smithi006 bash[27292]: audit 2023-08-27T22:49:14.127184+0000 mon.a (mon.0) 1622 : audit [DBG] from='client.? 172.21.15.6:0/1723459409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:17.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:17 smithi006 bash[33918]: cluster 2023-08-27T22:49:16.114672+0000 mgr.y (mgr.14901) 1522 : cluster [DBG] pgmap v1529: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:17.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:17 smithi006 bash[33918]: audit 2023-08-27T22:49:16.548648+0000 mon.a (mon.0) 1623 : audit [DBG] from='client.? 172.21.15.6:0/2125711548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:17.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:17 smithi006 bash[27292]: cluster 2023-08-27T22:49:16.114672+0000 mgr.y (mgr.14901) 1522 : cluster [DBG] pgmap v1529: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:17.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:17 smithi006 bash[27292]: audit 2023-08-27T22:49:16.548648+0000 mon.a (mon.0) 1623 : audit [DBG] from='client.? 172.21.15.6:0/2125711548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:17.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:17 smithi130 bash[27579]: cluster 2023-08-27T22:49:16.114672+0000 mgr.y (mgr.14901) 1522 : cluster [DBG] pgmap v1529: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:17.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:17 smithi130 bash[27579]: audit 2023-08-27T22:49:16.548648+0000 mon.a (mon.0) 1623 : audit [DBG] from='client.? 172.21.15.6:0/2125711548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:19.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:19 smithi006 bash[27292]: cluster 2023-08-27T22:49:18.115291+0000 mgr.y (mgr.14901) 1523 : cluster [DBG] pgmap v1530: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:19.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:19 smithi006 bash[27292]: audit 2023-08-27T22:49:18.964368+0000 mon.c (mon.2) 620 : audit [DBG] from='client.? 172.21.15.6:0/959761149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:19.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:19 smithi006 bash[33918]: cluster 2023-08-27T22:49:18.115291+0000 mgr.y (mgr.14901) 1523 : cluster [DBG] pgmap v1530: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:19.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:19 smithi006 bash[33918]: audit 2023-08-27T22:49:18.964368+0000 mon.c (mon.2) 620 : audit [DBG] from='client.? 172.21.15.6:0/959761149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:19.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:19 smithi130 bash[27579]: cluster 2023-08-27T22:49:18.115291+0000 mgr.y (mgr.14901) 1523 : cluster [DBG] pgmap v1530: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:19.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:19 smithi130 bash[27579]: audit 2023-08-27T22:49:18.964368+0000 mon.c (mon.2) 620 : audit [DBG] from='client.? 172.21.15.6:0/959761149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:20.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:49:20.296Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:20.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:49:20.298Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:21.189 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:49:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:49:20] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:49:21.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:21 smithi006 bash[33918]: cluster 2023-08-27T22:49:20.116435+0000 mgr.y (mgr.14901) 1524 : cluster [DBG] pgmap v1531: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:21.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:21 smithi006 bash[27292]: cluster 2023-08-27T22:49:20.116435+0000 mgr.y (mgr.14901) 1524 : cluster [DBG] pgmap v1531: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:21 smithi130 bash[27579]: cluster 2023-08-27T22:49:20.116435+0000 mgr.y (mgr.14901) 1524 : cluster [DBG] pgmap v1531: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:22.197 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:49:21.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:22.197 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:49:21.864Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:22.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:22 smithi006 bash[33918]: audit 2023-08-27T22:49:21.379452+0000 mon.a (mon.0) 1624 : audit [DBG] from='client.? 172.21.15.6:0/450926303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:22.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:22 smithi006 bash[27292]: audit 2023-08-27T22:49:21.379452+0000 mon.a (mon.0) 1624 : audit [DBG] from='client.? 172.21.15.6:0/450926303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:22.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:22 smithi130 bash[27579]: audit 2023-08-27T22:49:21.379452+0000 mon.a (mon.0) 1624 : audit [DBG] from='client.? 172.21.15.6:0/450926303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:23.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:23 smithi006 bash[27292]: cluster 2023-08-27T22:49:22.117230+0000 mgr.y (mgr.14901) 1525 : cluster [DBG] pgmap v1532: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:23.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:23 smithi006 bash[33918]: cluster 2023-08-27T22:49:22.117230+0000 mgr.y (mgr.14901) 1525 : cluster [DBG] pgmap v1532: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:23.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:23 smithi130 bash[27579]: cluster 2023-08-27T22:49:22.117230+0000 mgr.y (mgr.14901) 1525 : cluster [DBG] pgmap v1532: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:24.202 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:49:23.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:24.202 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:49:23.761Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:24.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:24 smithi006 bash[33918]: audit 2023-08-27T22:49:23.792947+0000 mon.a (mon.0) 1625 : audit [DBG] from='client.? 172.21.15.6:0/3966356365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:24.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:24 smithi006 bash[27292]: audit 2023-08-27T22:49:23.792947+0000 mon.a (mon.0) 1625 : audit [DBG] from='client.? 172.21.15.6:0/3966356365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:24.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:24 smithi130 bash[27579]: audit 2023-08-27T22:49:23.792947+0000 mon.a (mon.0) 1625 : audit [DBG] from='client.? 172.21.15.6:0/3966356365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:25.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:25 smithi006 bash[33918]: cluster 2023-08-27T22:49:24.118300+0000 mgr.y (mgr.14901) 1526 : cluster [DBG] pgmap v1533: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:25.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:25 smithi006 bash[27292]: cluster 2023-08-27T22:49:24.118300+0000 mgr.y (mgr.14901) 1526 : cluster [DBG] pgmap v1533: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:25.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:25 smithi130 bash[27579]: cluster 2023-08-27T22:49:24.118300+0000 mgr.y (mgr.14901) 1526 : cluster [DBG] pgmap v1533: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:27.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:27 smithi006 bash[27292]: cluster 2023-08-27T22:49:26.118835+0000 mgr.y (mgr.14901) 1527 : cluster [DBG] pgmap v1534: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:27.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:27 smithi006 bash[27292]: audit 2023-08-27T22:49:26.211095+0000 mon.c (mon.2) 621 : audit [DBG] from='client.? 172.21.15.6:0/209899296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:27.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:27 smithi006 bash[33918]: cluster 2023-08-27T22:49:26.118835+0000 mgr.y (mgr.14901) 1527 : cluster [DBG] pgmap v1534: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:27.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:27 smithi006 bash[33918]: audit 2023-08-27T22:49:26.211095+0000 mon.c (mon.2) 621 : audit [DBG] from='client.? 172.21.15.6:0/209899296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:27.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:27 smithi130 bash[27579]: cluster 2023-08-27T22:49:26.118835+0000 mgr.y (mgr.14901) 1527 : cluster [DBG] pgmap v1534: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:27.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:27 smithi130 bash[27579]: audit 2023-08-27T22:49:26.211095+0000 mon.c (mon.2) 621 : audit [DBG] from='client.? 172.21.15.6:0/209899296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:29.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:29 smithi006 bash[27292]: cluster 2023-08-27T22:49:28.119432+0000 mgr.y (mgr.14901) 1528 : cluster [DBG] pgmap v1535: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:29.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:29 smithi006 bash[27292]: audit 2023-08-27T22:49:28.628499+0000 mon.c (mon.2) 622 : audit [DBG] from='client.? 172.21.15.6:0/955718169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:29.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:29 smithi006 bash[33918]: cluster 2023-08-27T22:49:28.119432+0000 mgr.y (mgr.14901) 1528 : cluster [DBG] pgmap v1535: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:29.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:29 smithi006 bash[33918]: audit 2023-08-27T22:49:28.628499+0000 mon.c (mon.2) 622 : audit [DBG] from='client.? 172.21.15.6:0/955718169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:29.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:29 smithi130 bash[27579]: cluster 2023-08-27T22:49:28.119432+0000 mgr.y (mgr.14901) 1528 : cluster [DBG] pgmap v1535: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:29.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:29 smithi130 bash[27579]: audit 2023-08-27T22:49:28.628499+0000 mon.c (mon.2) 622 : audit [DBG] from='client.? 172.21.15.6:0/955718169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:30.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:49:30.297Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:30.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:49:30.299Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:31.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:49:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:49:30] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:49:31.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:31 smithi006 bash[33918]: cluster 2023-08-27T22:49:30.120552+0000 mgr.y (mgr.14901) 1529 : cluster [DBG] pgmap v1536: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:31.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:31 smithi006 bash[33918]: audit 2023-08-27T22:49:31.045436+0000 mon.c (mon.2) 623 : audit [DBG] from='client.? 172.21.15.6:0/496533041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:31.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:31 smithi006 bash[27292]: cluster 2023-08-27T22:49:30.120552+0000 mgr.y (mgr.14901) 1529 : cluster [DBG] pgmap v1536: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:31.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:31 smithi006 bash[27292]: audit 2023-08-27T22:49:31.045436+0000 mon.c (mon.2) 623 : audit [DBG] from='client.? 172.21.15.6:0/496533041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:31.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:31 smithi130 bash[27579]: cluster 2023-08-27T22:49:30.120552+0000 mgr.y (mgr.14901) 1529 : cluster [DBG] pgmap v1536: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:31.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:31 smithi130 bash[27579]: audit 2023-08-27T22:49:31.045436+0000 mon.c (mon.2) 623 : audit [DBG] from='client.? 172.21.15.6:0/496533041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:49:31.863Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:49:31.865Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:33.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:33 smithi130 bash[27579]: cluster 2023-08-27T22:49:32.121241+0000 mgr.y (mgr.14901) 1530 : cluster [DBG] pgmap v1537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:33.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:33 smithi006 bash[27292]: cluster 2023-08-27T22:49:32.121241+0000 mgr.y (mgr.14901) 1530 : cluster [DBG] pgmap v1537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:33.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:33 smithi006 bash[33918]: cluster 2023-08-27T22:49:32.121241+0000 mgr.y (mgr.14901) 1530 : cluster [DBG] pgmap v1537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:34.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:49:33.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:34.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:49:33.764Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:34.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:34 smithi130 bash[27579]: audit 2023-08-27T22:49:33.468334+0000 mon.c (mon.2) 624 : audit [DBG] from='client.? 172.21.15.6:0/1978827850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:34.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:34 smithi006 bash[33918]: audit 2023-08-27T22:49:33.468334+0000 mon.c (mon.2) 624 : audit [DBG] from='client.? 172.21.15.6:0/1978827850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:34.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:34 smithi006 bash[27292]: audit 2023-08-27T22:49:33.468334+0000 mon.c (mon.2) 624 : audit [DBG] from='client.? 172.21.15.6:0/1978827850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:35.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:35 smithi130 bash[27579]: cluster 2023-08-27T22:49:34.122262+0000 mgr.y (mgr.14901) 1531 : cluster [DBG] pgmap v1538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:35.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:35 smithi006 bash[27292]: cluster 2023-08-27T22:49:34.122262+0000 mgr.y (mgr.14901) 1531 : cluster [DBG] pgmap v1538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:35.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:35 smithi006 bash[33918]: cluster 2023-08-27T22:49:34.122262+0000 mgr.y (mgr.14901) 1531 : cluster [DBG] pgmap v1538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:36.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:36 smithi130 bash[27579]: audit 2023-08-27T22:49:35.880501+0000 mon.c (mon.2) 625 : audit [DBG] from='client.? 172.21.15.6:0/3508491892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:36.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:36 smithi006 bash[33918]: audit 2023-08-27T22:49:35.880501+0000 mon.c (mon.2) 625 : audit [DBG] from='client.? 172.21.15.6:0/3508491892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:36.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:36 smithi006 bash[27292]: audit 2023-08-27T22:49:35.880501+0000 mon.c (mon.2) 625 : audit [DBG] from='client.? 172.21.15.6:0/3508491892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:37.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:37 smithi130 bash[27579]: cluster 2023-08-27T22:49:36.123052+0000 mgr.y (mgr.14901) 1532 : cluster [DBG] pgmap v1539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:37.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:37 smithi006 bash[33918]: cluster 2023-08-27T22:49:36.123052+0000 mgr.y (mgr.14901) 1532 : cluster [DBG] pgmap v1539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:37.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:37 smithi006 bash[27292]: cluster 2023-08-27T22:49:36.123052+0000 mgr.y (mgr.14901) 1532 : cluster [DBG] pgmap v1539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:39.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:39 smithi130 bash[27579]: cluster 2023-08-27T22:49:38.123677+0000 mgr.y (mgr.14901) 1533 : cluster [DBG] pgmap v1540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:39.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:39 smithi130 bash[27579]: audit 2023-08-27T22:49:38.288588+0000 mon.a (mon.0) 1626 : audit [DBG] from='client.? 172.21.15.6:0/820038208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:39.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:39 smithi006 bash[27292]: cluster 2023-08-27T22:49:38.123677+0000 mgr.y (mgr.14901) 1533 : cluster [DBG] pgmap v1540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:39.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:39 smithi006 bash[27292]: audit 2023-08-27T22:49:38.288588+0000 mon.a (mon.0) 1626 : audit [DBG] from='client.? 172.21.15.6:0/820038208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:39.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:39 smithi006 bash[33918]: cluster 2023-08-27T22:49:38.123677+0000 mgr.y (mgr.14901) 1533 : cluster [DBG] pgmap v1540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:39.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:39 smithi006 bash[33918]: audit 2023-08-27T22:49:38.288588+0000 mon.a (mon.0) 1626 : audit [DBG] from='client.? 172.21.15.6:0/820038208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:40.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:49:40.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:40.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:49:40.300Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:41.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:49:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:49:40] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:49:41.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:41 smithi130 bash[27579]: cluster 2023-08-27T22:49:40.124889+0000 mgr.y (mgr.14901) 1534 : cluster [DBG] pgmap v1541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:41.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:41 smithi130 bash[27579]: audit 2023-08-27T22:49:40.712445+0000 mon.a (mon.0) 1627 : audit [DBG] from='client.? 172.21.15.6:0/813223117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:41.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:41 smithi006 bash[33918]: cluster 2023-08-27T22:49:40.124889+0000 mgr.y (mgr.14901) 1534 : cluster [DBG] pgmap v1541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:41.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:41 smithi006 bash[33918]: audit 2023-08-27T22:49:40.712445+0000 mon.a (mon.0) 1627 : audit [DBG] from='client.? 172.21.15.6:0/813223117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:41.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:41 smithi006 bash[27292]: cluster 2023-08-27T22:49:40.124889+0000 mgr.y (mgr.14901) 1534 : cluster [DBG] pgmap v1541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:41.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:41 smithi006 bash[27292]: audit 2023-08-27T22:49:40.712445+0000 mon.a (mon.0) 1627 : audit [DBG] from='client.? 172.21.15.6:0/813223117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:49:41.863Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:49:41.865Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:43.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:43 smithi130 bash[27579]: cluster 2023-08-27T22:49:42.125497+0000 mgr.y (mgr.14901) 1535 : cluster [DBG] pgmap v1542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:43 smithi130 bash[27579]: audit 2023-08-27T22:49:43.119382+0000 mon.a (mon.0) 1628 : audit [DBG] from='client.? 172.21.15.6:0/1815751877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:43.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:43 smithi006 bash[33918]: cluster 2023-08-27T22:49:42.125497+0000 mgr.y (mgr.14901) 1535 : cluster [DBG] pgmap v1542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:43.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:43 smithi006 bash[33918]: audit 2023-08-27T22:49:43.119382+0000 mon.a (mon.0) 1628 : audit [DBG] from='client.? 172.21.15.6:0/1815751877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:43.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:43 smithi006 bash[27292]: cluster 2023-08-27T22:49:42.125497+0000 mgr.y (mgr.14901) 1535 : cluster [DBG] pgmap v1542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:43.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:43 smithi006 bash[27292]: audit 2023-08-27T22:49:43.119382+0000 mon.a (mon.0) 1628 : audit [DBG] from='client.? 172.21.15.6:0/1815751877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:44.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:49:43.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:44.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:49:43.763Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:45.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:45 smithi130 bash[27579]: cluster 2023-08-27T22:49:44.126501+0000 mgr.y (mgr.14901) 1536 : cluster [DBG] pgmap v1543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:45.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:44 smithi006 bash[33918]: cluster 2023-08-27T22:49:44.126501+0000 mgr.y (mgr.14901) 1536 : cluster [DBG] pgmap v1543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:45.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:44 smithi006 bash[27292]: cluster 2023-08-27T22:49:44.126501+0000 mgr.y (mgr.14901) 1536 : cluster [DBG] pgmap v1543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:46.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:46 smithi130 bash[27579]: audit 2023-08-27T22:49:45.534954+0000 mon.a (mon.0) 1629 : audit [DBG] from='client.? 172.21.15.6:0/2540861897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:46.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:45 smithi006 bash[27292]: audit 2023-08-27T22:49:45.534954+0000 mon.a (mon.0) 1629 : audit [DBG] from='client.? 172.21.15.6:0/2540861897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:46.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:45 smithi006 bash[33918]: audit 2023-08-27T22:49:45.534954+0000 mon.a (mon.0) 1629 : audit [DBG] from='client.? 172.21.15.6:0/2540861897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:47.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:47 smithi130 bash[27579]: cluster 2023-08-27T22:49:46.127304+0000 mgr.y (mgr.14901) 1537 : cluster [DBG] pgmap v1544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:47.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:47 smithi130 bash[27579]: audit 2023-08-27T22:49:46.202135+0000 mon.b (mon.1) 269 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:49:47.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:47 smithi006 bash[27292]: cluster 2023-08-27T22:49:46.127304+0000 mgr.y (mgr.14901) 1537 : cluster [DBG] pgmap v1544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:47.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:47 smithi006 bash[27292]: audit 2023-08-27T22:49:46.202135+0000 mon.b (mon.1) 269 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:49:47.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:47 smithi006 bash[33918]: cluster 2023-08-27T22:49:46.127304+0000 mgr.y (mgr.14901) 1537 : cluster [DBG] pgmap v1544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:47.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:47 smithi006 bash[33918]: audit 2023-08-27T22:49:46.202135+0000 mon.b (mon.1) 269 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:49:48.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:48 smithi130 bash[27579]: audit 2023-08-27T22:49:47.957472+0000 mon.a (mon.0) 1630 : audit [DBG] from='client.? 172.21.15.6:0/3000790230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:48.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:48 smithi006 bash[33918]: audit 2023-08-27T22:49:47.957472+0000 mon.a (mon.0) 1630 : audit [DBG] from='client.? 172.21.15.6:0/3000790230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:48.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:48 smithi006 bash[27292]: audit 2023-08-27T22:49:47.957472+0000 mon.a (mon.0) 1630 : audit [DBG] from='client.? 172.21.15.6:0/3000790230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:49.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:49 smithi130 bash[27579]: cluster 2023-08-27T22:49:48.127984+0000 mgr.y (mgr.14901) 1538 : cluster [DBG] pgmap v1545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:49.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:49 smithi006 bash[33918]: cluster 2023-08-27T22:49:48.127984+0000 mgr.y (mgr.14901) 1538 : cluster [DBG] pgmap v1545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:49.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:49 smithi006 bash[27292]: cluster 2023-08-27T22:49:48.127984+0000 mgr.y (mgr.14901) 1538 : cluster [DBG] pgmap v1545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:50.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:49:50.299Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:50.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:49:50.301Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:51.193 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:49:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:49:50] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T22:49:51.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:51 smithi006 bash[27292]: cluster 2023-08-27T22:49:50.128936+0000 mgr.y (mgr.14901) 1539 : cluster [DBG] pgmap v1546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:51.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:51 smithi006 bash[27292]: audit 2023-08-27T22:49:50.377643+0000 mon.a (mon.0) 1631 : audit [DBG] from='client.? 172.21.15.6:0/3202246267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:51.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:51 smithi006 bash[33918]: cluster 2023-08-27T22:49:50.128936+0000 mgr.y (mgr.14901) 1539 : cluster [DBG] pgmap v1546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:51.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:51 smithi006 bash[33918]: audit 2023-08-27T22:49:50.377643+0000 mon.a (mon.0) 1631 : audit [DBG] from='client.? 172.21.15.6:0/3202246267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:51.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:51 smithi130 bash[27579]: cluster 2023-08-27T22:49:50.128936+0000 mgr.y (mgr.14901) 1539 : cluster [DBG] pgmap v1546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:51.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:51 smithi130 bash[27579]: audit 2023-08-27T22:49:50.377643+0000 mon.a (mon.0) 1631 : audit [DBG] from='client.? 172.21.15.6:0/3202246267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:49:51.864Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:49:51.867Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:53 smithi006 bash[33918]: audit 2023-08-27T22:49:52.106072+0000 mon.a (mon.0) 1632 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:49:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:53 smithi006 bash[33918]: cluster 2023-08-27T22:49:52.129499+0000 mgr.y (mgr.14901) 1540 : cluster [DBG] pgmap v1547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:53 smithi006 bash[33918]: audit 2023-08-27T22:49:52.496278+0000 mon.a (mon.0) 1633 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:49:53.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:53 smithi006 bash[33918]: audit 2023-08-27T22:49:52.791465+0000 mon.a (mon.0) 1634 : audit [DBG] from='client.? 172.21.15.6:0/561545326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:53.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:53 smithi006 bash[27292]: audit 2023-08-27T22:49:52.106072+0000 mon.a (mon.0) 1632 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:49:53.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:53 smithi006 bash[27292]: cluster 2023-08-27T22:49:52.129499+0000 mgr.y (mgr.14901) 1540 : cluster [DBG] pgmap v1547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:53.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:53 smithi006 bash[27292]: audit 2023-08-27T22:49:52.496278+0000 mon.a (mon.0) 1633 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:49:53.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:53 smithi006 bash[27292]: audit 2023-08-27T22:49:52.791465+0000 mon.a (mon.0) 1634 : audit [DBG] from='client.? 172.21.15.6:0/561545326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:53.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:53 smithi130 bash[27579]: audit 2023-08-27T22:49:52.106072+0000 mon.a (mon.0) 1632 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:49:53.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:53 smithi130 bash[27579]: cluster 2023-08-27T22:49:52.129499+0000 mgr.y (mgr.14901) 1540 : cluster [DBG] pgmap v1547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:53 smithi130 bash[27579]: audit 2023-08-27T22:49:52.496278+0000 mon.a (mon.0) 1633 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:49:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:53 smithi130 bash[27579]: audit 2023-08-27T22:49:52.791465+0000 mon.a (mon.0) 1634 : audit [DBG] from='client.? 172.21.15.6:0/561545326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:54.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:49:53.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:54.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:49:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:49:53.764Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:49:55.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:55 smithi130 bash[27579]: cluster 2023-08-27T22:49:54.130476+0000 mgr.y (mgr.14901) 1541 : cluster [DBG] pgmap v1548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:55.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:55 smithi006 bash[27292]: cluster 2023-08-27T22:49:54.130476+0000 mgr.y (mgr.14901) 1541 : cluster [DBG] pgmap v1548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:55.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:55 smithi006 bash[33918]: cluster 2023-08-27T22:49:54.130476+0000 mgr.y (mgr.14901) 1541 : cluster [DBG] pgmap v1548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:56.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:56 smithi130 bash[27579]: audit 2023-08-27T22:49:55.200948+0000 mon.a (mon.0) 1635 : audit [DBG] from='client.? 172.21.15.6:0/3250575783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:56.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:56 smithi006 bash[33918]: audit 2023-08-27T22:49:55.200948+0000 mon.a (mon.0) 1635 : audit [DBG] from='client.? 172.21.15.6:0/3250575783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:56.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:56 smithi006 bash[27292]: audit 2023-08-27T22:49:55.200948+0000 mon.a (mon.0) 1635 : audit [DBG] from='client.? 172.21.15.6:0/3250575783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:57.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:57 smithi130 bash[27579]: cluster 2023-08-27T22:49:56.131235+0000 mgr.y (mgr.14901) 1542 : cluster [DBG] pgmap v1549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:57.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:57 smithi006 bash[27292]: cluster 2023-08-27T22:49:56.131235+0000 mgr.y (mgr.14901) 1542 : cluster [DBG] pgmap v1549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:57.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:57 smithi006 bash[33918]: cluster 2023-08-27T22:49:56.131235+0000 mgr.y (mgr.14901) 1542 : cluster [DBG] pgmap v1549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:58.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:58 smithi130 bash[27579]: audit 2023-08-27T22:49:57.606856+0000 mon.a (mon.0) 1636 : audit [DBG] from='client.? 172.21.15.6:0/1376064702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:58.427 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:58 smithi006 bash[33918]: audit 2023-08-27T22:49:57.606856+0000 mon.a (mon.0) 1636 : audit [DBG] from='client.? 172.21.15.6:0/1376064702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:58.427 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:58 smithi006 bash[27292]: audit 2023-08-27T22:49:57.606856+0000 mon.a (mon.0) 1636 : audit [DBG] from='client.? 172.21.15.6:0/1376064702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:59.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:49:59 smithi130 bash[27579]: cluster 2023-08-27T22:49:58.132191+0000 mgr.y (mgr.14901) 1543 : cluster [DBG] pgmap v1550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:59.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:49:59 smithi006 bash[33918]: cluster 2023-08-27T22:49:58.132191+0000 mgr.y (mgr.14901) 1543 : cluster [DBG] pgmap v1550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:49:59.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:49:59 smithi006 bash[27292]: cluster 2023-08-27T22:49:58.132191+0000 mgr.y (mgr.14901) 1543 : cluster [DBG] pgmap v1550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:00.307 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:00 smithi006 bash[27292]: cluster 2023-08-27T22:50:00.000125+0000 mon.a (mon.0) 1637 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:50:00.307 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:00 smithi006 bash[27292]: audit 2023-08-27T22:50:00.001165+0000 mon.a (mon.0) 1638 : audit [DBG] from='client.? 172.21.15.6:0/2769013729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:00.308 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:00 smithi006 bash[33918]: cluster 2023-08-27T22:50:00.000125+0000 mon.a (mon.0) 1637 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:50:00.308 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:00 smithi006 bash[33918]: audit 2023-08-27T22:50:00.001165+0000 mon.a (mon.0) 1638 : audit [DBG] from='client.? 172.21.15.6:0/2769013729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:00.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:00 smithi130 bash[27579]: cluster 2023-08-27T22:50:00.000125+0000 mon.a (mon.0) 1637 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:50:00.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:00 smithi130 bash[27579]: audit 2023-08-27T22:50:00.001165+0000 mon.a (mon.0) 1638 : audit [DBG] from='client.? 172.21.15.6:0/2769013729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:00.703 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:50:00.300Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:00.703 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:50:00.301Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:00.972 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:50:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:50:00] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:50:01.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:01 smithi130 bash[27579]: cluster 2023-08-27T22:50:00.133440+0000 mgr.y (mgr.14901) 1544 : cluster [DBG] pgmap v1551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:01.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:01 smithi006 bash[27292]: cluster 2023-08-27T22:50:00.133440+0000 mgr.y (mgr.14901) 1544 : cluster [DBG] pgmap v1551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:01.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:01 smithi006 bash[33918]: cluster 2023-08-27T22:50:00.133440+0000 mgr.y (mgr.14901) 1544 : cluster [DBG] pgmap v1551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:01.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:50:01.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:01.972 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:50:01.868Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:03.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:03 smithi006 bash[27292]: cluster 2023-08-27T22:50:02.133885+0000 mgr.y (mgr.14901) 1545 : cluster [DBG] pgmap v1552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:03.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:03 smithi006 bash[27292]: audit 2023-08-27T22:50:02.398333+0000 mon.a (mon.0) 1639 : audit [DBG] from='client.? 172.21.15.6:0/1455589007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:03.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:03 smithi006 bash[27292]: audit 2023-08-27T22:50:02.606007+0000 mon.a (mon.0) 1640 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:50:03.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:03 smithi006 bash[27292]: audit 2023-08-27T22:50:02.902914+0000 mon.b (mon.1) 270 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:50:03.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:03 smithi006 bash[27292]: audit 2023-08-27T22:50:02.904946+0000 mon.b (mon.1) 271 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:50:03.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:03 smithi006 bash[27292]: audit 2023-08-27T22:50:02.906779+0000 mon.a (mon.0) 1641 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:50:03.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:03 smithi006 bash[33918]: cluster 2023-08-27T22:50:02.133885+0000 mgr.y (mgr.14901) 1545 : cluster [DBG] pgmap v1552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:03.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:03 smithi006 bash[33918]: audit 2023-08-27T22:50:02.398333+0000 mon.a (mon.0) 1639 : audit [DBG] from='client.? 172.21.15.6:0/1455589007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:03.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:03 smithi006 bash[33918]: audit 2023-08-27T22:50:02.606007+0000 mon.a (mon.0) 1640 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:50:03.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:03 smithi006 bash[33918]: audit 2023-08-27T22:50:02.902914+0000 mon.b (mon.1) 270 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:50:03.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:03 smithi006 bash[33918]: audit 2023-08-27T22:50:02.904946+0000 mon.b (mon.1) 271 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:50:03.474 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:03 smithi006 bash[33918]: audit 2023-08-27T22:50:02.906779+0000 mon.a (mon.0) 1641 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:50:03.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:03 smithi130 bash[27579]: cluster 2023-08-27T22:50:02.133885+0000 mgr.y (mgr.14901) 1545 : cluster [DBG] pgmap v1552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:03.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:03 smithi130 bash[27579]: audit 2023-08-27T22:50:02.398333+0000 mon.a (mon.0) 1639 : audit [DBG] from='client.? 172.21.15.6:0/1455589007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:03.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:03 smithi130 bash[27579]: audit 2023-08-27T22:50:02.606007+0000 mon.a (mon.0) 1640 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:50:03.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:03 smithi130 bash[27579]: audit 2023-08-27T22:50:02.902914+0000 mon.b (mon.1) 270 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:50:03.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:03 smithi130 bash[27579]: audit 2023-08-27T22:50:02.904946+0000 mon.b (mon.1) 271 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:50:03.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:03 smithi130 bash[27579]: audit 2023-08-27T22:50:02.906779+0000 mon.a (mon.0) 1641 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:50:04.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:50:03.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:04.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:50:03.764Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:05 smithi130 bash[27579]: cluster 2023-08-27T22:50:04.134860+0000 mgr.y (mgr.14901) 1546 : cluster [DBG] pgmap v1553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:05 smithi130 bash[27579]: audit 2023-08-27T22:50:04.808821+0000 mon.a (mon.0) 1642 : audit [DBG] from='client.? 172.21.15.6:0/4106332036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:05.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:05 smithi006 bash[27292]: cluster 2023-08-27T22:50:04.134860+0000 mgr.y (mgr.14901) 1546 : cluster [DBG] pgmap v1553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:05.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:05 smithi006 bash[27292]: audit 2023-08-27T22:50:04.808821+0000 mon.a (mon.0) 1642 : audit [DBG] from='client.? 172.21.15.6:0/4106332036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:05.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:05 smithi006 bash[33918]: cluster 2023-08-27T22:50:04.134860+0000 mgr.y (mgr.14901) 1546 : cluster [DBG] pgmap v1553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:05.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:05 smithi006 bash[33918]: audit 2023-08-27T22:50:04.808821+0000 mon.a (mon.0) 1642 : audit [DBG] from='client.? 172.21.15.6:0/4106332036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:07.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:07 smithi006 bash[27292]: cluster 2023-08-27T22:50:06.135644+0000 mgr.y (mgr.14901) 1547 : cluster [DBG] pgmap v1554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:07.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:07 smithi006 bash[33918]: cluster 2023-08-27T22:50:06.135644+0000 mgr.y (mgr.14901) 1547 : cluster [DBG] pgmap v1554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:07.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:07 smithi130 bash[27579]: cluster 2023-08-27T22:50:06.135644+0000 mgr.y (mgr.14901) 1547 : cluster [DBG] pgmap v1554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:08.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:08 smithi006 bash[33918]: audit 2023-08-27T22:50:07.229618+0000 mon.c (mon.2) 626 : audit [DBG] from='client.? 172.21.15.6:0/2747938147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:08.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:08 smithi006 bash[27292]: audit 2023-08-27T22:50:07.229618+0000 mon.c (mon.2) 626 : audit [DBG] from='client.? 172.21.15.6:0/2747938147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:08.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:08 smithi130 bash[27579]: audit 2023-08-27T22:50:07.229618+0000 mon.c (mon.2) 626 : audit [DBG] from='client.? 172.21.15.6:0/2747938147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:09.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:09 smithi006 bash[27292]: cluster 2023-08-27T22:50:08.136313+0000 mgr.y (mgr.14901) 1548 : cluster [DBG] pgmap v1555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:09.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:09 smithi006 bash[33918]: cluster 2023-08-27T22:50:08.136313+0000 mgr.y (mgr.14901) 1548 : cluster [DBG] pgmap v1555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:09.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:09 smithi130 bash[27579]: cluster 2023-08-27T22:50:08.136313+0000 mgr.y (mgr.14901) 1548 : cluster [DBG] pgmap v1555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:10.474 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:10 smithi006 bash[33918]: audit 2023-08-27T22:50:09.643310+0000 mon.c (mon.2) 627 : audit [DBG] from='client.? 172.21.15.6:0/2246842472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:10.474 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:10 smithi006 bash[27292]: audit 2023-08-27T22:50:09.643310+0000 mon.c (mon.2) 627 : audit [DBG] from='client.? 172.21.15.6:0/2246842472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:10.474 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:50:10.300Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:10.475 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:50:10.303Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:10.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:10 smithi130 bash[27579]: audit 2023-08-27T22:50:09.643310+0000 mon.c (mon.2) 627 : audit [DBG] from='client.? 172.21.15.6:0/2246842472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:11.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:50:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:50:10] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:50:11.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:11 smithi006 bash[27292]: cluster 2023-08-27T22:50:10.137482+0000 mgr.y (mgr.14901) 1549 : cluster [DBG] pgmap v1556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:11.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:11 smithi006 bash[33918]: cluster 2023-08-27T22:50:10.137482+0000 mgr.y (mgr.14901) 1549 : cluster [DBG] pgmap v1556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:11.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:11 smithi130 bash[27579]: cluster 2023-08-27T22:50:10.137482+0000 mgr.y (mgr.14901) 1549 : cluster [DBG] pgmap v1556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:50:11.866Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:12.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:50:11.868Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:12.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:12 smithi130 bash[27579]: audit 2023-08-27T22:50:12.056792+0000 mon.c (mon.2) 628 : audit [DBG] from='client.? 172.21.15.6:0/2011071673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:12.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:12 smithi006 bash[33918]: audit 2023-08-27T22:50:12.056792+0000 mon.c (mon.2) 628 : audit [DBG] from='client.? 172.21.15.6:0/2011071673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:12.727 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:12 smithi006 bash[27292]: audit 2023-08-27T22:50:12.056792+0000 mon.c (mon.2) 628 : audit [DBG] from='client.? 172.21.15.6:0/2011071673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:13.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:13 smithi130 bash[27579]: cluster 2023-08-27T22:50:12.138122+0000 mgr.y (mgr.14901) 1550 : cluster [DBG] pgmap v1557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:13.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:13 smithi006 bash[33918]: cluster 2023-08-27T22:50:12.138122+0000 mgr.y (mgr.14901) 1550 : cluster [DBG] pgmap v1557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:13.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:13 smithi006 bash[27292]: cluster 2023-08-27T22:50:12.138122+0000 mgr.y (mgr.14901) 1550 : cluster [DBG] pgmap v1557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:14.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:50:13.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:14.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:50:13.764Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:15.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:15 smithi130 bash[27579]: cluster 2023-08-27T22:50:14.138832+0000 mgr.y (mgr.14901) 1551 : cluster [DBG] pgmap v1558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:15.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:15 smithi130 bash[27579]: audit 2023-08-27T22:50:14.478151+0000 mon.a (mon.0) 1643 : audit [DBG] from='client.? 172.21.15.6:0/18925985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:15.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:15 smithi006 bash[27292]: cluster 2023-08-27T22:50:14.138832+0000 mgr.y (mgr.14901) 1551 : cluster [DBG] pgmap v1558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:15.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:15 smithi006 bash[27292]: audit 2023-08-27T22:50:14.478151+0000 mon.a (mon.0) 1643 : audit [DBG] from='client.? 172.21.15.6:0/18925985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:15.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:15 smithi006 bash[33918]: cluster 2023-08-27T22:50:14.138832+0000 mgr.y (mgr.14901) 1551 : cluster [DBG] pgmap v1558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:15.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:15 smithi006 bash[33918]: audit 2023-08-27T22:50:14.478151+0000 mon.a (mon.0) 1643 : audit [DBG] from='client.? 172.21.15.6:0/18925985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:17.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:17 smithi006 bash[33918]: cluster 2023-08-27T22:50:16.139530+0000 mgr.y (mgr.14901) 1552 : cluster [DBG] pgmap v1559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:17.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:17 smithi006 bash[33918]: audit 2023-08-27T22:50:16.893368+0000 mon.a (mon.0) 1644 : audit [DBG] from='client.? 172.21.15.6:0/1931725266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:17.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:17 smithi006 bash[27292]: cluster 2023-08-27T22:50:16.139530+0000 mgr.y (mgr.14901) 1552 : cluster [DBG] pgmap v1559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:17.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:17 smithi006 bash[27292]: audit 2023-08-27T22:50:16.893368+0000 mon.a (mon.0) 1644 : audit [DBG] from='client.? 172.21.15.6:0/1931725266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:17.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:17 smithi130 bash[27579]: cluster 2023-08-27T22:50:16.139530+0000 mgr.y (mgr.14901) 1552 : cluster [DBG] pgmap v1559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:17.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:17 smithi130 bash[27579]: audit 2023-08-27T22:50:16.893368+0000 mon.a (mon.0) 1644 : audit [DBG] from='client.? 172.21.15.6:0/1931725266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:19.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:19 smithi006 bash[33918]: cluster 2023-08-27T22:50:18.140186+0000 mgr.y (mgr.14901) 1553 : cluster [DBG] pgmap v1560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:19.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:19 smithi006 bash[27292]: cluster 2023-08-27T22:50:18.140186+0000 mgr.y (mgr.14901) 1553 : cluster [DBG] pgmap v1560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:19.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:19 smithi130 bash[27579]: cluster 2023-08-27T22:50:18.140186+0000 mgr.y (mgr.14901) 1553 : cluster [DBG] pgmap v1560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:20.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:20 smithi130 bash[27579]: audit 2023-08-27T22:50:19.311641+0000 mon.a (mon.0) 1645 : audit [DBG] from='client.? 172.21.15.6:0/3210761460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:20.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:20 smithi006 bash[27292]: audit 2023-08-27T22:50:19.311641+0000 mon.a (mon.0) 1645 : audit [DBG] from='client.? 172.21.15.6:0/3210761460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:20.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:20 smithi006 bash[33918]: audit 2023-08-27T22:50:19.311641+0000 mon.a (mon.0) 1645 : audit [DBG] from='client.? 172.21.15.6:0/3210761460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:20.723 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:50:20.301Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:20.724 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:50:20.303Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:21.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:50:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:50:20] "GET /metrics HTTP/1.1" 200 193084 "" "Prometheus/2.33.4" 2023-08-27T22:50:21.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:21 smithi130 bash[27579]: cluster 2023-08-27T22:50:20.141348+0000 mgr.y (mgr.14901) 1554 : cluster [DBG] pgmap v1561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:21.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:21 smithi006 bash[33918]: cluster 2023-08-27T22:50:20.141348+0000 mgr.y (mgr.14901) 1554 : cluster [DBG] pgmap v1561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:21.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:21 smithi006 bash[27292]: cluster 2023-08-27T22:50:20.141348+0000 mgr.y (mgr.14901) 1554 : cluster [DBG] pgmap v1561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:50:21.867Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:50:21.869Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:22.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:22 smithi130 bash[27579]: audit 2023-08-27T22:50:21.731047+0000 mon.c (mon.2) 629 : audit [DBG] from='client.? 172.21.15.6:0/479398594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:22.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:22 smithi006 bash[33918]: audit 2023-08-27T22:50:21.731047+0000 mon.c (mon.2) 629 : audit [DBG] from='client.? 172.21.15.6:0/479398594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:22.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:22 smithi006 bash[27292]: audit 2023-08-27T22:50:21.731047+0000 mon.c (mon.2) 629 : audit [DBG] from='client.? 172.21.15.6:0/479398594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:23.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:23 smithi130 bash[27579]: cluster 2023-08-27T22:50:22.142033+0000 mgr.y (mgr.14901) 1555 : cluster [DBG] pgmap v1562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:23.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:23 smithi006 bash[27292]: cluster 2023-08-27T22:50:22.142033+0000 mgr.y (mgr.14901) 1555 : cluster [DBG] pgmap v1562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:23.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:23 smithi006 bash[33918]: cluster 2023-08-27T22:50:22.142033+0000 mgr.y (mgr.14901) 1555 : cluster [DBG] pgmap v1562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:24.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:50:23.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:24.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:50:23.766Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:25.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:25 smithi130 bash[27579]: cluster 2023-08-27T22:50:24.143036+0000 mgr.y (mgr.14901) 1556 : cluster [DBG] pgmap v1563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:25.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:25 smithi130 bash[27579]: audit 2023-08-27T22:50:24.148509+0000 mon.a (mon.0) 1646 : audit [DBG] from='client.? 172.21.15.6:0/349175053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:25.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:25 smithi006 bash[27292]: cluster 2023-08-27T22:50:24.143036+0000 mgr.y (mgr.14901) 1556 : cluster [DBG] pgmap v1563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:25.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:25 smithi006 bash[27292]: audit 2023-08-27T22:50:24.148509+0000 mon.a (mon.0) 1646 : audit [DBG] from='client.? 172.21.15.6:0/349175053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:25.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:25 smithi006 bash[33918]: cluster 2023-08-27T22:50:24.143036+0000 mgr.y (mgr.14901) 1556 : cluster [DBG] pgmap v1563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:25.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:25 smithi006 bash[33918]: audit 2023-08-27T22:50:24.148509+0000 mon.a (mon.0) 1646 : audit [DBG] from='client.? 172.21.15.6:0/349175053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:27.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:27 smithi006 bash[33918]: cluster 2023-08-27T22:50:26.143785+0000 mgr.y (mgr.14901) 1557 : cluster [DBG] pgmap v1564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:27.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:27 smithi006 bash[33918]: audit 2023-08-27T22:50:26.560994+0000 mon.a (mon.0) 1647 : audit [DBG] from='client.? 172.21.15.6:0/3321840962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:27.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:27 smithi006 bash[27292]: cluster 2023-08-27T22:50:26.143785+0000 mgr.y (mgr.14901) 1557 : cluster [DBG] pgmap v1564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:27.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:27 smithi006 bash[27292]: audit 2023-08-27T22:50:26.560994+0000 mon.a (mon.0) 1647 : audit [DBG] from='client.? 172.21.15.6:0/3321840962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:27.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:27 smithi130 bash[27579]: cluster 2023-08-27T22:50:26.143785+0000 mgr.y (mgr.14901) 1557 : cluster [DBG] pgmap v1564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:27.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:27 smithi130 bash[27579]: audit 2023-08-27T22:50:26.560994+0000 mon.a (mon.0) 1647 : audit [DBG] from='client.? 172.21.15.6:0/3321840962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:29.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:29 smithi006 bash[33918]: cluster 2023-08-27T22:50:28.144443+0000 mgr.y (mgr.14901) 1558 : cluster [DBG] pgmap v1565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:29.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:29 smithi006 bash[33918]: audit 2023-08-27T22:50:28.976597+0000 mon.c (mon.2) 630 : audit [DBG] from='client.? 172.21.15.6:0/85811743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:29.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:29 smithi006 bash[27292]: cluster 2023-08-27T22:50:28.144443+0000 mgr.y (mgr.14901) 1558 : cluster [DBG] pgmap v1565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:29.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:29 smithi006 bash[27292]: audit 2023-08-27T22:50:28.976597+0000 mon.c (mon.2) 630 : audit [DBG] from='client.? 172.21.15.6:0/85811743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:29.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:29 smithi130 bash[27579]: cluster 2023-08-27T22:50:28.144443+0000 mgr.y (mgr.14901) 1558 : cluster [DBG] pgmap v1565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:29.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:29 smithi130 bash[27579]: audit 2023-08-27T22:50:28.976597+0000 mon.c (mon.2) 630 : audit [DBG] from='client.? 172.21.15.6:0/85811743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:30.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:50:30.302Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:30.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:50:30.304Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:50:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:50:30] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T22:50:31.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:31 smithi006 bash[27292]: cluster 2023-08-27T22:50:30.145692+0000 mgr.y (mgr.14901) 1559 : cluster [DBG] pgmap v1566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:31.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:31 smithi006 bash[33918]: cluster 2023-08-27T22:50:30.145692+0000 mgr.y (mgr.14901) 1559 : cluster [DBG] pgmap v1566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:31.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:31 smithi130 bash[27579]: cluster 2023-08-27T22:50:30.145692+0000 mgr.y (mgr.14901) 1559 : cluster [DBG] pgmap v1566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:50:31.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:50:31.870Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:32.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:32 smithi130 bash[27579]: audit 2023-08-27T22:50:31.394609+0000 mon.c (mon.2) 631 : audit [DBG] from='client.? 172.21.15.6:0/313119332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:32.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:32 smithi006 bash[33918]: audit 2023-08-27T22:50:31.394609+0000 mon.c (mon.2) 631 : audit [DBG] from='client.? 172.21.15.6:0/313119332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:32.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:32 smithi006 bash[27292]: audit 2023-08-27T22:50:31.394609+0000 mon.c (mon.2) 631 : audit [DBG] from='client.? 172.21.15.6:0/313119332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:33.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:33 smithi130 bash[27579]: cluster 2023-08-27T22:50:32.146408+0000 mgr.y (mgr.14901) 1560 : cluster [DBG] pgmap v1567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:33.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:33 smithi006 bash[33918]: cluster 2023-08-27T22:50:32.146408+0000 mgr.y (mgr.14901) 1560 : cluster [DBG] pgmap v1567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:33.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:33 smithi006 bash[27292]: cluster 2023-08-27T22:50:32.146408+0000 mgr.y (mgr.14901) 1560 : cluster [DBG] pgmap v1567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:34.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:50:33.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:34.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:50:33.766Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:34.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:34 smithi130 bash[27579]: audit 2023-08-27T22:50:33.809132+0000 mon.a (mon.0) 1648 : audit [DBG] from='client.? 172.21.15.6:0/3078144646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:34.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:34 smithi006 bash[33918]: audit 2023-08-27T22:50:33.809132+0000 mon.a (mon.0) 1648 : audit [DBG] from='client.? 172.21.15.6:0/3078144646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:34.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:34 smithi006 bash[27292]: audit 2023-08-27T22:50:33.809132+0000 mon.a (mon.0) 1648 : audit [DBG] from='client.? 172.21.15.6:0/3078144646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:35.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:35 smithi130 bash[27579]: cluster 2023-08-27T22:50:34.147430+0000 mgr.y (mgr.14901) 1561 : cluster [DBG] pgmap v1568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:35.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:35 smithi006 bash[27292]: cluster 2023-08-27T22:50:34.147430+0000 mgr.y (mgr.14901) 1561 : cluster [DBG] pgmap v1568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:35.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:35 smithi006 bash[33918]: cluster 2023-08-27T22:50:34.147430+0000 mgr.y (mgr.14901) 1561 : cluster [DBG] pgmap v1568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:36.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:36 smithi130 bash[27579]: audit 2023-08-27T22:50:36.215483+0000 mon.a (mon.0) 1649 : audit [DBG] from='client.? 172.21.15.6:0/2651524207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:36.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:36 smithi006 bash[27292]: audit 2023-08-27T22:50:36.215483+0000 mon.a (mon.0) 1649 : audit [DBG] from='client.? 172.21.15.6:0/2651524207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:36.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:36 smithi006 bash[33918]: audit 2023-08-27T22:50:36.215483+0000 mon.a (mon.0) 1649 : audit [DBG] from='client.? 172.21.15.6:0/2651524207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:37.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:37 smithi130 bash[27579]: cluster 2023-08-27T22:50:36.148360+0000 mgr.y (mgr.14901) 1562 : cluster [DBG] pgmap v1569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:37.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:37 smithi006 bash[33918]: cluster 2023-08-27T22:50:36.148360+0000 mgr.y (mgr.14901) 1562 : cluster [DBG] pgmap v1569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:37.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:37 smithi006 bash[27292]: cluster 2023-08-27T22:50:36.148360+0000 mgr.y (mgr.14901) 1562 : cluster [DBG] pgmap v1569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:39.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:39 smithi130 bash[27579]: cluster 2023-08-27T22:50:38.148986+0000 mgr.y (mgr.14901) 1563 : cluster [DBG] pgmap v1570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:39.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:39 smithi130 bash[27579]: audit 2023-08-27T22:50:38.629761+0000 mon.a (mon.0) 1650 : audit [DBG] from='client.? 172.21.15.6:0/472118391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:39.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:39 smithi006 bash[27292]: cluster 2023-08-27T22:50:38.148986+0000 mgr.y (mgr.14901) 1563 : cluster [DBG] pgmap v1570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:39.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:39 smithi006 bash[27292]: audit 2023-08-27T22:50:38.629761+0000 mon.a (mon.0) 1650 : audit [DBG] from='client.? 172.21.15.6:0/472118391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:39.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:39 smithi006 bash[33918]: cluster 2023-08-27T22:50:38.148986+0000 mgr.y (mgr.14901) 1563 : cluster [DBG] pgmap v1570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:39.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:39 smithi006 bash[33918]: audit 2023-08-27T22:50:38.629761+0000 mon.a (mon.0) 1650 : audit [DBG] from='client.? 172.21.15.6:0/472118391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:40.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:50:40.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:40.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:50:40.307Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:41.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:50:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:50:40] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T22:50:41.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:41 smithi130 bash[27579]: cluster 2023-08-27T22:50:40.150042+0000 mgr.y (mgr.14901) 1564 : cluster [DBG] pgmap v1571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:41.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:41 smithi130 bash[27579]: audit 2023-08-27T22:50:41.045099+0000 mon.a (mon.0) 1651 : audit [DBG] from='client.? 172.21.15.6:0/1729933130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:41.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:41 smithi006 bash[33918]: cluster 2023-08-27T22:50:40.150042+0000 mgr.y (mgr.14901) 1564 : cluster [DBG] pgmap v1571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:41.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:41 smithi006 bash[33918]: audit 2023-08-27T22:50:41.045099+0000 mon.a (mon.0) 1651 : audit [DBG] from='client.? 172.21.15.6:0/1729933130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:41.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:41 smithi006 bash[27292]: cluster 2023-08-27T22:50:40.150042+0000 mgr.y (mgr.14901) 1564 : cluster [DBG] pgmap v1571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:41.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:41 smithi006 bash[27292]: audit 2023-08-27T22:50:41.045099+0000 mon.a (mon.0) 1651 : audit [DBG] from='client.? 172.21.15.6:0/1729933130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:50:41.869Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:42.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:50:41.871Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:43.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:43 smithi130 bash[27579]: cluster 2023-08-27T22:50:42.150718+0000 mgr.y (mgr.14901) 1565 : cluster [DBG] pgmap v1572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:43.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:43 smithi006 bash[33918]: cluster 2023-08-27T22:50:42.150718+0000 mgr.y (mgr.14901) 1565 : cluster [DBG] pgmap v1572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:43.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:43 smithi006 bash[27292]: cluster 2023-08-27T22:50:42.150718+0000 mgr.y (mgr.14901) 1565 : cluster [DBG] pgmap v1572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:44.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:50:43.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:44.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:50:43.767Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:44.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:44 smithi130 bash[27579]: audit 2023-08-27T22:50:43.463398+0000 mon.a (mon.0) 1652 : audit [DBG] from='client.? 172.21.15.6:0/3418387799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:44.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:44 smithi006 bash[27292]: audit 2023-08-27T22:50:43.463398+0000 mon.a (mon.0) 1652 : audit [DBG] from='client.? 172.21.15.6:0/3418387799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:44.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:44 smithi006 bash[33918]: audit 2023-08-27T22:50:43.463398+0000 mon.a (mon.0) 1652 : audit [DBG] from='client.? 172.21.15.6:0/3418387799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:45.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:45 smithi130 bash[27579]: cluster 2023-08-27T22:50:44.151748+0000 mgr.y (mgr.14901) 1566 : cluster [DBG] pgmap v1573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:45.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:45 smithi006 bash[33918]: cluster 2023-08-27T22:50:44.151748+0000 mgr.y (mgr.14901) 1566 : cluster [DBG] pgmap v1573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:45.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:45 smithi006 bash[27292]: cluster 2023-08-27T22:50:44.151748+0000 mgr.y (mgr.14901) 1566 : cluster [DBG] pgmap v1573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:46.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:46 smithi130 bash[27579]: audit 2023-08-27T22:50:45.882188+0000 mon.a (mon.0) 1653 : audit [DBG] from='client.? 172.21.15.6:0/4093339748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:46.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:46 smithi006 bash[33918]: audit 2023-08-27T22:50:45.882188+0000 mon.a (mon.0) 1653 : audit [DBG] from='client.? 172.21.15.6:0/4093339748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:46.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:46 smithi006 bash[27292]: audit 2023-08-27T22:50:45.882188+0000 mon.a (mon.0) 1653 : audit [DBG] from='client.? 172.21.15.6:0/4093339748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:47.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:47 smithi130 bash[27579]: cluster 2023-08-27T22:50:46.152401+0000 mgr.y (mgr.14901) 1567 : cluster [DBG] pgmap v1574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:47.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:47 smithi006 bash[33918]: cluster 2023-08-27T22:50:46.152401+0000 mgr.y (mgr.14901) 1567 : cluster [DBG] pgmap v1574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:47.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:47 smithi006 bash[27292]: cluster 2023-08-27T22:50:46.152401+0000 mgr.y (mgr.14901) 1567 : cluster [DBG] pgmap v1574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:49.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:49 smithi130 bash[27579]: cluster 2023-08-27T22:50:48.153096+0000 mgr.y (mgr.14901) 1568 : cluster [DBG] pgmap v1575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:49.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:49 smithi130 bash[27579]: audit 2023-08-27T22:50:48.297058+0000 mon.a (mon.0) 1654 : audit [DBG] from='client.? 172.21.15.6:0/3059381495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:49.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:49 smithi006 bash[33918]: cluster 2023-08-27T22:50:48.153096+0000 mgr.y (mgr.14901) 1568 : cluster [DBG] pgmap v1575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:49.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:49 smithi006 bash[33918]: audit 2023-08-27T22:50:48.297058+0000 mon.a (mon.0) 1654 : audit [DBG] from='client.? 172.21.15.6:0/3059381495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:49.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:49 smithi006 bash[27292]: cluster 2023-08-27T22:50:48.153096+0000 mgr.y (mgr.14901) 1568 : cluster [DBG] pgmap v1575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:49.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:49 smithi006 bash[27292]: audit 2023-08-27T22:50:48.297058+0000 mon.a (mon.0) 1654 : audit [DBG] from='client.? 172.21.15.6:0/3059381495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:50.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:50:50.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:50.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:50:50.306Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:51.222 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:50:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:50:50] "GET /metrics HTTP/1.1" 200 193080 "" "Prometheus/2.33.4" 2023-08-27T22:50:51.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:51 smithi130 bash[27579]: cluster 2023-08-27T22:50:50.154209+0000 mgr.y (mgr.14901) 1569 : cluster [DBG] pgmap v1576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:51.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:51 smithi130 bash[27579]: audit 2023-08-27T22:50:50.718158+0000 mon.a (mon.0) 1655 : audit [DBG] from='client.? 172.21.15.6:0/1444688731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:51.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:51 smithi006 bash[27292]: cluster 2023-08-27T22:50:50.154209+0000 mgr.y (mgr.14901) 1569 : cluster [DBG] pgmap v1576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:51.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:51 smithi006 bash[27292]: audit 2023-08-27T22:50:50.718158+0000 mon.a (mon.0) 1655 : audit [DBG] from='client.? 172.21.15.6:0/1444688731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:51.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:51 smithi006 bash[33918]: cluster 2023-08-27T22:50:50.154209+0000 mgr.y (mgr.14901) 1569 : cluster [DBG] pgmap v1576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:51.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:51 smithi006 bash[33918]: audit 2023-08-27T22:50:50.718158+0000 mon.a (mon.0) 1655 : audit [DBG] from='client.? 172.21.15.6:0/1444688731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:50:51.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:52.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:50:51.872Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:53.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:53 smithi130 bash[27579]: cluster 2023-08-27T22:50:52.154883+0000 mgr.y (mgr.14901) 1570 : cluster [DBG] pgmap v1577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:53.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:53 smithi130 bash[27579]: audit 2023-08-27T22:50:53.129615+0000 mon.a (mon.0) 1656 : audit [DBG] from='client.? 172.21.15.6:0/1552872637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:53.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:53 smithi006 bash[33918]: cluster 2023-08-27T22:50:52.154883+0000 mgr.y (mgr.14901) 1570 : cluster [DBG] pgmap v1577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:53.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:53 smithi006 bash[33918]: audit 2023-08-27T22:50:53.129615+0000 mon.a (mon.0) 1656 : audit [DBG] from='client.? 172.21.15.6:0/1552872637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:53.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:53 smithi006 bash[27292]: cluster 2023-08-27T22:50:52.154883+0000 mgr.y (mgr.14901) 1570 : cluster [DBG] pgmap v1577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:53.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:53 smithi006 bash[27292]: audit 2023-08-27T22:50:53.129615+0000 mon.a (mon.0) 1656 : audit [DBG] from='client.? 172.21.15.6:0/1552872637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:54.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:50:53.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:54.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:50:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:50:53.768Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:50:55.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:55 smithi130 bash[27579]: cluster 2023-08-27T22:50:54.155777+0000 mgr.y (mgr.14901) 1571 : cluster [DBG] pgmap v1578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:55.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:55 smithi006 bash[33918]: cluster 2023-08-27T22:50:54.155777+0000 mgr.y (mgr.14901) 1571 : cluster [DBG] pgmap v1578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:55.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:55 smithi006 bash[27292]: cluster 2023-08-27T22:50:54.155777+0000 mgr.y (mgr.14901) 1571 : cluster [DBG] pgmap v1578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:56.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:56 smithi130 bash[27579]: audit 2023-08-27T22:50:55.533000+0000 mon.a (mon.0) 1657 : audit [DBG] from='client.? 172.21.15.6:0/1410066003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:56.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:56 smithi006 bash[33918]: audit 2023-08-27T22:50:55.533000+0000 mon.a (mon.0) 1657 : audit [DBG] from='client.? 172.21.15.6:0/1410066003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:56.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:56 smithi006 bash[27292]: audit 2023-08-27T22:50:55.533000+0000 mon.a (mon.0) 1657 : audit [DBG] from='client.? 172.21.15.6:0/1410066003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:57.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:57 smithi130 bash[27579]: cluster 2023-08-27T22:50:56.156508+0000 mgr.y (mgr.14901) 1572 : cluster [DBG] pgmap v1579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:57.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:57 smithi006 bash[33918]: cluster 2023-08-27T22:50:56.156508+0000 mgr.y (mgr.14901) 1572 : cluster [DBG] pgmap v1579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:57.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:57 smithi006 bash[27292]: cluster 2023-08-27T22:50:56.156508+0000 mgr.y (mgr.14901) 1572 : cluster [DBG] pgmap v1579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:58.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:58 smithi130 bash[27579]: audit 2023-08-27T22:50:57.948056+0000 mon.c (mon.2) 632 : audit [DBG] from='client.? 172.21.15.6:0/2639663819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:58.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:58 smithi006 bash[27292]: audit 2023-08-27T22:50:57.948056+0000 mon.c (mon.2) 632 : audit [DBG] from='client.? 172.21.15.6:0/2639663819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:58.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:58 smithi006 bash[33918]: audit 2023-08-27T22:50:57.948056+0000 mon.c (mon.2) 632 : audit [DBG] from='client.? 172.21.15.6:0/2639663819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:59.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:50:59 smithi130 bash[27579]: cluster 2023-08-27T22:50:58.157165+0000 mgr.y (mgr.14901) 1573 : cluster [DBG] pgmap v1580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:59.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:50:59 smithi006 bash[27292]: cluster 2023-08-27T22:50:58.157165+0000 mgr.y (mgr.14901) 1573 : cluster [DBG] pgmap v1580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:50:59.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:50:59 smithi006 bash[33918]: cluster 2023-08-27T22:50:58.157165+0000 mgr.y (mgr.14901) 1573 : cluster [DBG] pgmap v1580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:00.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:51:00.304Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:00.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:51:00.306Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:01.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:51:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:51:00] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T22:51:01.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:01 smithi130 bash[27579]: cluster 2023-08-27T22:51:00.158220+0000 mgr.y (mgr.14901) 1574 : cluster [DBG] pgmap v1581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:01.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:01 smithi130 bash[27579]: audit 2023-08-27T22:51:00.371957+0000 mon.a (mon.0) 1658 : audit [DBG] from='client.? 172.21.15.6:0/2141110839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:01.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:01 smithi006 bash[33918]: cluster 2023-08-27T22:51:00.158220+0000 mgr.y (mgr.14901) 1574 : cluster [DBG] pgmap v1581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:01.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:01 smithi006 bash[33918]: audit 2023-08-27T22:51:00.371957+0000 mon.a (mon.0) 1658 : audit [DBG] from='client.? 172.21.15.6:0/2141110839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:01.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:01 smithi006 bash[27292]: cluster 2023-08-27T22:51:00.158220+0000 mgr.y (mgr.14901) 1574 : cluster [DBG] pgmap v1581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:01.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:01 smithi006 bash[27292]: audit 2023-08-27T22:51:00.371957+0000 mon.a (mon.0) 1658 : audit [DBG] from='client.? 172.21.15.6:0/2141110839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:51:01.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:02.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:51:01.872Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:03.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:03 smithi130 bash[27579]: cluster 2023-08-27T22:51:02.158842+0000 mgr.y (mgr.14901) 1575 : cluster [DBG] pgmap v1582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:03.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:03 smithi130 bash[27579]: audit 2023-08-27T22:51:02.784559+0000 mon.a (mon.0) 1659 : audit [DBG] from='client.? 172.21.15.6:0/4213963442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:03.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:03 smithi130 bash[27579]: audit 2023-08-27T22:51:02.922432+0000 mon.b (mon.1) 272 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:51:03.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:03 smithi006 bash[33918]: cluster 2023-08-27T22:51:02.158842+0000 mgr.y (mgr.14901) 1575 : cluster [DBG] pgmap v1582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:03.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:03 smithi006 bash[33918]: audit 2023-08-27T22:51:02.784559+0000 mon.a (mon.0) 1659 : audit [DBG] from='client.? 172.21.15.6:0/4213963442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:03.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:03 smithi006 bash[33918]: audit 2023-08-27T22:51:02.922432+0000 mon.b (mon.1) 272 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:51:03.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:03 smithi006 bash[27292]: cluster 2023-08-27T22:51:02.158842+0000 mgr.y (mgr.14901) 1575 : cluster [DBG] pgmap v1582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:03.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:03 smithi006 bash[27292]: audit 2023-08-27T22:51:02.784559+0000 mon.a (mon.0) 1659 : audit [DBG] from='client.? 172.21.15.6:0/4213963442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:03.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:03 smithi006 bash[27292]: audit 2023-08-27T22:51:02.922432+0000 mon.b (mon.1) 272 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:51:04.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:51:03.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:04.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:51:03.769Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:05 smithi130 bash[27579]: cluster 2023-08-27T22:51:04.159758+0000 mgr.y (mgr.14901) 1576 : cluster [DBG] pgmap v1583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:05.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:05 smithi006 bash[27292]: cluster 2023-08-27T22:51:04.159758+0000 mgr.y (mgr.14901) 1576 : cluster [DBG] pgmap v1583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:05.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:05 smithi006 bash[33918]: cluster 2023-08-27T22:51:04.159758+0000 mgr.y (mgr.14901) 1576 : cluster [DBG] pgmap v1583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:06.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:06 smithi130 bash[27579]: audit 2023-08-27T22:51:05.220669+0000 mon.c (mon.2) 633 : audit [DBG] from='client.? 172.21.15.6:0/1858042994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:06.475 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:06 smithi006 bash[27292]: audit 2023-08-27T22:51:05.220669+0000 mon.c (mon.2) 633 : audit [DBG] from='client.? 172.21.15.6:0/1858042994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:06.475 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:06 smithi006 bash[33918]: audit 2023-08-27T22:51:05.220669+0000 mon.c (mon.2) 633 : audit [DBG] from='client.? 172.21.15.6:0/1858042994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:07.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:07 smithi130 bash[27579]: cluster 2023-08-27T22:51:06.160561+0000 mgr.y (mgr.14901) 1577 : cluster [DBG] pgmap v1584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:07.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:07 smithi006 bash[27292]: cluster 2023-08-27T22:51:06.160561+0000 mgr.y (mgr.14901) 1577 : cluster [DBG] pgmap v1584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:07.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:07 smithi006 bash[33918]: cluster 2023-08-27T22:51:06.160561+0000 mgr.y (mgr.14901) 1577 : cluster [DBG] pgmap v1584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:08.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:08 smithi130 bash[27579]: audit 2023-08-27T22:51:07.632696+0000 mon.c (mon.2) 634 : audit [DBG] from='client.? 172.21.15.6:0/2159037108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:08.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:08 smithi006 bash[33918]: audit 2023-08-27T22:51:07.632696+0000 mon.c (mon.2) 634 : audit [DBG] from='client.? 172.21.15.6:0/2159037108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:08.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:08 smithi006 bash[27292]: audit 2023-08-27T22:51:07.632696+0000 mon.c (mon.2) 634 : audit [DBG] from='client.? 172.21.15.6:0/2159037108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:09.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:09 smithi130 bash[27579]: cluster 2023-08-27T22:51:08.161234+0000 mgr.y (mgr.14901) 1578 : cluster [DBG] pgmap v1585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:09.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:09 smithi130 bash[27579]: audit 2023-08-27T22:51:08.806723+0000 mon.a (mon.0) 1660 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:51:09.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:09 smithi006 bash[27292]: cluster 2023-08-27T22:51:08.161234+0000 mgr.y (mgr.14901) 1578 : cluster [DBG] pgmap v1585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:09.474 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:09 smithi006 bash[27292]: audit 2023-08-27T22:51:08.806723+0000 mon.a (mon.0) 1660 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:51:09.474 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:09 smithi006 bash[33918]: cluster 2023-08-27T22:51:08.161234+0000 mgr.y (mgr.14901) 1578 : cluster [DBG] pgmap v1585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:09.474 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:09 smithi006 bash[33918]: audit 2023-08-27T22:51:08.806723+0000 mon.a (mon.0) 1660 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:51:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:10 smithi130 bash[27579]: audit 2023-08-27T22:51:09.290957+0000 mon.a (mon.0) 1661 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:51:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:10 smithi130 bash[27579]: audit 2023-08-27T22:51:09.592215+0000 mon.b (mon.1) 273 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:51:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:10 smithi130 bash[27579]: audit 2023-08-27T22:51:09.594153+0000 mon.b (mon.1) 274 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:51:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:10 smithi130 bash[27579]: audit 2023-08-27T22:51:09.596540+0000 mon.a (mon.0) 1662 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:51:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:10 smithi130 bash[27579]: audit 2023-08-27T22:51:10.052397+0000 mon.c (mon.2) 635 : audit [DBG] from='client.? 172.21.15.6:0/2865533594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:51:10.305Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:51:10.307Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:10.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:10 smithi006 bash[33918]: audit 2023-08-27T22:51:09.290957+0000 mon.a (mon.0) 1661 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:51:10.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:10 smithi006 bash[33918]: audit 2023-08-27T22:51:09.592215+0000 mon.b (mon.1) 273 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:51:10.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:10 smithi006 bash[33918]: audit 2023-08-27T22:51:09.594153+0000 mon.b (mon.1) 274 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:51:10.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:10 smithi006 bash[33918]: audit 2023-08-27T22:51:09.596540+0000 mon.a (mon.0) 1662 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:51:10.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:10 smithi006 bash[33918]: audit 2023-08-27T22:51:10.052397+0000 mon.c (mon.2) 635 : audit [DBG] from='client.? 172.21.15.6:0/2865533594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:10.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:10 smithi006 bash[27292]: audit 2023-08-27T22:51:09.290957+0000 mon.a (mon.0) 1661 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:51:10.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:10 smithi006 bash[27292]: audit 2023-08-27T22:51:09.592215+0000 mon.b (mon.1) 273 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:51:10.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:10 smithi006 bash[27292]: audit 2023-08-27T22:51:09.594153+0000 mon.b (mon.1) 274 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:51:10.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:10 smithi006 bash[27292]: audit 2023-08-27T22:51:09.596540+0000 mon.a (mon.0) 1662 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:51:10.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:10 smithi006 bash[27292]: audit 2023-08-27T22:51:10.052397+0000 mon.c (mon.2) 635 : audit [DBG] from='client.? 172.21.15.6:0/2865533594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:11.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:51:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:51:10] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T22:51:11.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:11 smithi130 bash[27579]: cluster 2023-08-27T22:51:10.162303+0000 mgr.y (mgr.14901) 1579 : cluster [DBG] pgmap v1586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:11.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:11 smithi006 bash[33918]: cluster 2023-08-27T22:51:10.162303+0000 mgr.y (mgr.14901) 1579 : cluster [DBG] pgmap v1586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:11.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:11 smithi006 bash[27292]: cluster 2023-08-27T22:51:10.162303+0000 mgr.y (mgr.14901) 1579 : cluster [DBG] pgmap v1586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:51:11.871Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:51:11.873Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:13.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:13 smithi130 bash[27579]: cluster 2023-08-27T22:51:12.163062+0000 mgr.y (mgr.14901) 1580 : cluster [DBG] pgmap v1587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:13 smithi130 bash[27579]: audit 2023-08-27T22:51:12.469533+0000 mon.a (mon.0) 1663 : audit [DBG] from='client.? 172.21.15.6:0/1854552835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:13.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:13 smithi006 bash[33918]: cluster 2023-08-27T22:51:12.163062+0000 mgr.y (mgr.14901) 1580 : cluster [DBG] pgmap v1587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:13.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:13 smithi006 bash[33918]: audit 2023-08-27T22:51:12.469533+0000 mon.a (mon.0) 1663 : audit [DBG] from='client.? 172.21.15.6:0/1854552835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:13.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:13 smithi006 bash[27292]: cluster 2023-08-27T22:51:12.163062+0000 mgr.y (mgr.14901) 1580 : cluster [DBG] pgmap v1587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:13.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:13 smithi006 bash[27292]: audit 2023-08-27T22:51:12.469533+0000 mon.a (mon.0) 1663 : audit [DBG] from='client.? 172.21.15.6:0/1854552835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:14.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:51:13.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:14.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:51:13.770Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:15.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:15 smithi130 bash[27579]: cluster 2023-08-27T22:51:14.164114+0000 mgr.y (mgr.14901) 1581 : cluster [DBG] pgmap v1588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:15 smithi130 bash[27579]: audit 2023-08-27T22:51:14.882906+0000 mon.a (mon.0) 1664 : audit [DBG] from='client.? 172.21.15.6:0/2137257015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:15.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:15 smithi006 bash[33918]: cluster 2023-08-27T22:51:14.164114+0000 mgr.y (mgr.14901) 1581 : cluster [DBG] pgmap v1588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:15.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:15 smithi006 bash[33918]: audit 2023-08-27T22:51:14.882906+0000 mon.a (mon.0) 1664 : audit [DBG] from='client.? 172.21.15.6:0/2137257015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:15.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:15 smithi006 bash[27292]: cluster 2023-08-27T22:51:14.164114+0000 mgr.y (mgr.14901) 1581 : cluster [DBG] pgmap v1588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:15.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:15 smithi006 bash[27292]: audit 2023-08-27T22:51:14.882906+0000 mon.a (mon.0) 1664 : audit [DBG] from='client.? 172.21.15.6:0/2137257015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:17.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:17 smithi130 bash[27579]: cluster 2023-08-27T22:51:16.164891+0000 mgr.y (mgr.14901) 1582 : cluster [DBG] pgmap v1589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:17.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:17 smithi006 bash[33918]: cluster 2023-08-27T22:51:16.164891+0000 mgr.y (mgr.14901) 1582 : cluster [DBG] pgmap v1589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:17.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:17 smithi006 bash[27292]: cluster 2023-08-27T22:51:16.164891+0000 mgr.y (mgr.14901) 1582 : cluster [DBG] pgmap v1589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:18.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:18 smithi130 bash[27579]: audit 2023-08-27T22:51:17.305349+0000 mon.c (mon.2) 636 : audit [DBG] from='client.? 172.21.15.6:0/3053587701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:18.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:18 smithi006 bash[33918]: audit 2023-08-27T22:51:17.305349+0000 mon.c (mon.2) 636 : audit [DBG] from='client.? 172.21.15.6:0/3053587701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:18.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:18 smithi006 bash[27292]: audit 2023-08-27T22:51:17.305349+0000 mon.c (mon.2) 636 : audit [DBG] from='client.? 172.21.15.6:0/3053587701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:19.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:19 smithi130 bash[27579]: cluster 2023-08-27T22:51:18.165538+0000 mgr.y (mgr.14901) 1583 : cluster [DBG] pgmap v1590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:19.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:19 smithi006 bash[27292]: cluster 2023-08-27T22:51:18.165538+0000 mgr.y (mgr.14901) 1583 : cluster [DBG] pgmap v1590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:19.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:19 smithi006 bash[33918]: cluster 2023-08-27T22:51:18.165538+0000 mgr.y (mgr.14901) 1583 : cluster [DBG] pgmap v1590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:20.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:20 smithi130 bash[27579]: audit 2023-08-27T22:51:19.720579+0000 mon.a (mon.0) 1665 : audit [DBG] from='client.? 172.21.15.6:0/91162474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:20.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:20 smithi006 bash[27292]: audit 2023-08-27T22:51:19.720579+0000 mon.a (mon.0) 1665 : audit [DBG] from='client.? 172.21.15.6:0/91162474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:20.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:20 smithi006 bash[33918]: audit 2023-08-27T22:51:19.720579+0000 mon.a (mon.0) 1665 : audit [DBG] from='client.? 172.21.15.6:0/91162474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:20.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:51:20.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:20.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:51:20.308Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:21.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:51:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:51:20] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T22:51:21.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:21 smithi130 bash[27579]: cluster 2023-08-27T22:51:20.166579+0000 mgr.y (mgr.14901) 1584 : cluster [DBG] pgmap v1591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:21.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:21 smithi006 bash[27292]: cluster 2023-08-27T22:51:20.166579+0000 mgr.y (mgr.14901) 1584 : cluster [DBG] pgmap v1591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:21.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:21 smithi006 bash[33918]: cluster 2023-08-27T22:51:20.166579+0000 mgr.y (mgr.14901) 1584 : cluster [DBG] pgmap v1591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:51:21.872Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:51:21.874Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:22.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:22 smithi130 bash[27579]: audit 2023-08-27T22:51:22.140689+0000 mon.c (mon.2) 637 : audit [DBG] from='client.? 172.21.15.6:0/4275926813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:22.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:22 smithi006 bash[27292]: audit 2023-08-27T22:51:22.140689+0000 mon.c (mon.2) 637 : audit [DBG] from='client.? 172.21.15.6:0/4275926813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:22.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:22 smithi006 bash[33918]: audit 2023-08-27T22:51:22.140689+0000 mon.c (mon.2) 637 : audit [DBG] from='client.? 172.21.15.6:0/4275926813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:23.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:23 smithi130 bash[27579]: cluster 2023-08-27T22:51:22.167232+0000 mgr.y (mgr.14901) 1585 : cluster [DBG] pgmap v1592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:23.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:23 smithi006 bash[33918]: cluster 2023-08-27T22:51:22.167232+0000 mgr.y (mgr.14901) 1585 : cluster [DBG] pgmap v1592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:23.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:23 smithi006 bash[27292]: cluster 2023-08-27T22:51:22.167232+0000 mgr.y (mgr.14901) 1585 : cluster [DBG] pgmap v1592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:24.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:51:23.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:24.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:51:23.770Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:25.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:25 smithi130 bash[27579]: cluster 2023-08-27T22:51:24.168378+0000 mgr.y (mgr.14901) 1586 : cluster [DBG] pgmap v1593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:25.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:25 smithi130 bash[27579]: audit 2023-08-27T22:51:24.557459+0000 mon.a (mon.0) 1666 : audit [DBG] from='client.? 172.21.15.6:0/2290900668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:25.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:25 smithi006 bash[27292]: cluster 2023-08-27T22:51:24.168378+0000 mgr.y (mgr.14901) 1586 : cluster [DBG] pgmap v1593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:25.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:25 smithi006 bash[27292]: audit 2023-08-27T22:51:24.557459+0000 mon.a (mon.0) 1666 : audit [DBG] from='client.? 172.21.15.6:0/2290900668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:25.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:25 smithi006 bash[33918]: cluster 2023-08-27T22:51:24.168378+0000 mgr.y (mgr.14901) 1586 : cluster [DBG] pgmap v1593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:25.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:25 smithi006 bash[33918]: audit 2023-08-27T22:51:24.557459+0000 mon.a (mon.0) 1666 : audit [DBG] from='client.? 172.21.15.6:0/2290900668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:27.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:27 smithi130 bash[27579]: cluster 2023-08-27T22:51:26.169180+0000 mgr.y (mgr.14901) 1587 : cluster [DBG] pgmap v1594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:27.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:27 smithi130 bash[27579]: audit 2023-08-27T22:51:26.972863+0000 mon.a (mon.0) 1667 : audit [DBG] from='client.? 172.21.15.6:0/2975226588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:27.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:27 smithi006 bash[27292]: cluster 2023-08-27T22:51:26.169180+0000 mgr.y (mgr.14901) 1587 : cluster [DBG] pgmap v1594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:27.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:27 smithi006 bash[27292]: audit 2023-08-27T22:51:26.972863+0000 mon.a (mon.0) 1667 : audit [DBG] from='client.? 172.21.15.6:0/2975226588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:27.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:27 smithi006 bash[33918]: cluster 2023-08-27T22:51:26.169180+0000 mgr.y (mgr.14901) 1587 : cluster [DBG] pgmap v1594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:27.723 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:27 smithi006 bash[33918]: audit 2023-08-27T22:51:26.972863+0000 mon.a (mon.0) 1667 : audit [DBG] from='client.? 172.21.15.6:0/2975226588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:29.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:29 smithi130 bash[27579]: cluster 2023-08-27T22:51:28.169951+0000 mgr.y (mgr.14901) 1588 : cluster [DBG] pgmap v1595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:29.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:29 smithi006 bash[27292]: cluster 2023-08-27T22:51:28.169951+0000 mgr.y (mgr.14901) 1588 : cluster [DBG] pgmap v1595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:29.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:29 smithi006 bash[33918]: cluster 2023-08-27T22:51:28.169951+0000 mgr.y (mgr.14901) 1588 : cluster [DBG] pgmap v1595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:30.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:30 smithi130 bash[27579]: audit 2023-08-27T22:51:29.387115+0000 mon.a (mon.0) 1668 : audit [DBG] from='client.? 172.21.15.6:0/4271914447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:30.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:30 smithi006 bash[27292]: audit 2023-08-27T22:51:29.387115+0000 mon.a (mon.0) 1668 : audit [DBG] from='client.? 172.21.15.6:0/4271914447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:30.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:30 smithi006 bash[33918]: audit 2023-08-27T22:51:29.387115+0000 mon.a (mon.0) 1668 : audit [DBG] from='client.? 172.21.15.6:0/4271914447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:30.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:51:30.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:30.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:51:30.309Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:31.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:51:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:51:30] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T22:51:31.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:31 smithi130 bash[27579]: cluster 2023-08-27T22:51:30.171099+0000 mgr.y (mgr.14901) 1589 : cluster [DBG] pgmap v1596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:31.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:31 smithi006 bash[27292]: cluster 2023-08-27T22:51:30.171099+0000 mgr.y (mgr.14901) 1589 : cluster [DBG] pgmap v1596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:31.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:31 smithi006 bash[33918]: cluster 2023-08-27T22:51:30.171099+0000 mgr.y (mgr.14901) 1589 : cluster [DBG] pgmap v1596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:51:31.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:32.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:51:31.875Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:32.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:32 smithi130 bash[27579]: audit 2023-08-27T22:51:31.797294+0000 mon.c (mon.2) 638 : audit [DBG] from='client.? 172.21.15.6:0/278669088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:32.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:32 smithi006 bash[27292]: audit 2023-08-27T22:51:31.797294+0000 mon.c (mon.2) 638 : audit [DBG] from='client.? 172.21.15.6:0/278669088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:32.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:32 smithi006 bash[33918]: audit 2023-08-27T22:51:31.797294+0000 mon.c (mon.2) 638 : audit [DBG] from='client.? 172.21.15.6:0/278669088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:33.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:33 smithi130 bash[27579]: cluster 2023-08-27T22:51:32.171765+0000 mgr.y (mgr.14901) 1590 : cluster [DBG] pgmap v1597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:33.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:33 smithi006 bash[33918]: cluster 2023-08-27T22:51:32.171765+0000 mgr.y (mgr.14901) 1590 : cluster [DBG] pgmap v1597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:33.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:33 smithi006 bash[27292]: cluster 2023-08-27T22:51:32.171765+0000 mgr.y (mgr.14901) 1590 : cluster [DBG] pgmap v1597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:34.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:51:33.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:34.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:51:33.771Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:35.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:35 smithi130 bash[27579]: cluster 2023-08-27T22:51:34.172446+0000 mgr.y (mgr.14901) 1591 : cluster [DBG] pgmap v1598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:35.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:35 smithi130 bash[27579]: audit 2023-08-27T22:51:34.221046+0000 mon.c (mon.2) 639 : audit [DBG] from='client.? 172.21.15.6:0/1507184574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:35.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:35 smithi006 bash[33918]: cluster 2023-08-27T22:51:34.172446+0000 mgr.y (mgr.14901) 1591 : cluster [DBG] pgmap v1598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:35.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:35 smithi006 bash[33918]: audit 2023-08-27T22:51:34.221046+0000 mon.c (mon.2) 639 : audit [DBG] from='client.? 172.21.15.6:0/1507184574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:35.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:35 smithi006 bash[27292]: cluster 2023-08-27T22:51:34.172446+0000 mgr.y (mgr.14901) 1591 : cluster [DBG] pgmap v1598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:35.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:35 smithi006 bash[27292]: audit 2023-08-27T22:51:34.221046+0000 mon.c (mon.2) 639 : audit [DBG] from='client.? 172.21.15.6:0/1507184574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:37.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:37 smithi130 bash[27579]: cluster 2023-08-27T22:51:36.173199+0000 mgr.y (mgr.14901) 1592 : cluster [DBG] pgmap v1599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:37.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:37 smithi130 bash[27579]: audit 2023-08-27T22:51:36.639192+0000 mon.a (mon.0) 1669 : audit [DBG] from='client.? 172.21.15.6:0/444331020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:37.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:37 smithi006 bash[27292]: cluster 2023-08-27T22:51:36.173199+0000 mgr.y (mgr.14901) 1592 : cluster [DBG] pgmap v1599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:37.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:37 smithi006 bash[27292]: audit 2023-08-27T22:51:36.639192+0000 mon.a (mon.0) 1669 : audit [DBG] from='client.? 172.21.15.6:0/444331020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:37.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:37 smithi006 bash[33918]: cluster 2023-08-27T22:51:36.173199+0000 mgr.y (mgr.14901) 1592 : cluster [DBG] pgmap v1599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:37.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:37 smithi006 bash[33918]: audit 2023-08-27T22:51:36.639192+0000 mon.a (mon.0) 1669 : audit [DBG] from='client.? 172.21.15.6:0/444331020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:39.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:39 smithi130 bash[27579]: cluster 2023-08-27T22:51:38.173819+0000 mgr.y (mgr.14901) 1593 : cluster [DBG] pgmap v1600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:39.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:39 smithi130 bash[27579]: audit 2023-08-27T22:51:39.060766+0000 mon.c (mon.2) 640 : audit [DBG] from='client.? 172.21.15.6:0/3436438316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:39.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:39 smithi006 bash[33918]: cluster 2023-08-27T22:51:38.173819+0000 mgr.y (mgr.14901) 1593 : cluster [DBG] pgmap v1600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:39.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:39 smithi006 bash[33918]: audit 2023-08-27T22:51:39.060766+0000 mon.c (mon.2) 640 : audit [DBG] from='client.? 172.21.15.6:0/3436438316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:39.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:39 smithi006 bash[27292]: cluster 2023-08-27T22:51:38.173819+0000 mgr.y (mgr.14901) 1593 : cluster [DBG] pgmap v1600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:39.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:39 smithi006 bash[27292]: audit 2023-08-27T22:51:39.060766+0000 mon.c (mon.2) 640 : audit [DBG] from='client.? 172.21.15.6:0/3436438316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:51:40.308Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:40.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:51:40.310Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:41.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:51:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:51:40] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T22:51:41.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:41 smithi130 bash[27579]: cluster 2023-08-27T22:51:40.174905+0000 mgr.y (mgr.14901) 1594 : cluster [DBG] pgmap v1601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:41.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:41 smithi006 bash[33918]: cluster 2023-08-27T22:51:40.174905+0000 mgr.y (mgr.14901) 1594 : cluster [DBG] pgmap v1601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:41.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:41 smithi006 bash[27292]: cluster 2023-08-27T22:51:40.174905+0000 mgr.y (mgr.14901) 1594 : cluster [DBG] pgmap v1601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:51:41.874Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:51:41.878Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:42.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:42 smithi130 bash[27579]: audit 2023-08-27T22:51:41.479207+0000 mon.c (mon.2) 641 : audit [DBG] from='client.? 172.21.15.6:0/2299292776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:42.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:42 smithi006 bash[33918]: audit 2023-08-27T22:51:41.479207+0000 mon.c (mon.2) 641 : audit [DBG] from='client.? 172.21.15.6:0/2299292776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:42.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:42 smithi006 bash[27292]: audit 2023-08-27T22:51:41.479207+0000 mon.c (mon.2) 641 : audit [DBG] from='client.? 172.21.15.6:0/2299292776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:43.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:43 smithi130 bash[27579]: cluster 2023-08-27T22:51:42.175529+0000 mgr.y (mgr.14901) 1595 : cluster [DBG] pgmap v1602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:43.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:43 smithi006 bash[33918]: cluster 2023-08-27T22:51:42.175529+0000 mgr.y (mgr.14901) 1595 : cluster [DBG] pgmap v1602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:43.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:43 smithi006 bash[27292]: cluster 2023-08-27T22:51:42.175529+0000 mgr.y (mgr.14901) 1595 : cluster [DBG] pgmap v1602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:44.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:51:43.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:44.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:51:43.771Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:44.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:44 smithi130 bash[27579]: audit 2023-08-27T22:51:43.901099+0000 mon.a (mon.0) 1670 : audit [DBG] from='client.? 172.21.15.6:0/2824192054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:44.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:44 smithi006 bash[27292]: audit 2023-08-27T22:51:43.901099+0000 mon.a (mon.0) 1670 : audit [DBG] from='client.? 172.21.15.6:0/2824192054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:44.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:44 smithi006 bash[33918]: audit 2023-08-27T22:51:43.901099+0000 mon.a (mon.0) 1670 : audit [DBG] from='client.? 172.21.15.6:0/2824192054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:45.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:45 smithi130 bash[27579]: cluster 2023-08-27T22:51:44.176500+0000 mgr.y (mgr.14901) 1596 : cluster [DBG] pgmap v1603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:45.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:45 smithi006 bash[33918]: cluster 2023-08-27T22:51:44.176500+0000 mgr.y (mgr.14901) 1596 : cluster [DBG] pgmap v1603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:45.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:45 smithi006 bash[27292]: cluster 2023-08-27T22:51:44.176500+0000 mgr.y (mgr.14901) 1596 : cluster [DBG] pgmap v1603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:47.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:47 smithi130 bash[27579]: cluster 2023-08-27T22:51:46.177057+0000 mgr.y (mgr.14901) 1597 : cluster [DBG] pgmap v1604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:47.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:47 smithi130 bash[27579]: audit 2023-08-27T22:51:46.318626+0000 mon.a (mon.0) 1671 : audit [DBG] from='client.? 172.21.15.6:0/486353839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:47.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:47 smithi006 bash[33918]: cluster 2023-08-27T22:51:46.177057+0000 mgr.y (mgr.14901) 1597 : cluster [DBG] pgmap v1604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:47.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:47 smithi006 bash[33918]: audit 2023-08-27T22:51:46.318626+0000 mon.a (mon.0) 1671 : audit [DBG] from='client.? 172.21.15.6:0/486353839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:47.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:47 smithi006 bash[27292]: cluster 2023-08-27T22:51:46.177057+0000 mgr.y (mgr.14901) 1597 : cluster [DBG] pgmap v1604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:47.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:47 smithi006 bash[27292]: audit 2023-08-27T22:51:46.318626+0000 mon.a (mon.0) 1671 : audit [DBG] from='client.? 172.21.15.6:0/486353839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:49.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:49 smithi130 bash[27579]: cluster 2023-08-27T22:51:48.177661+0000 mgr.y (mgr.14901) 1598 : cluster [DBG] pgmap v1605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:49.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:49 smithi130 bash[27579]: audit 2023-08-27T22:51:48.731563+0000 mon.c (mon.2) 642 : audit [DBG] from='client.? 172.21.15.6:0/450990743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:49.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:49 smithi006 bash[33918]: cluster 2023-08-27T22:51:48.177661+0000 mgr.y (mgr.14901) 1598 : cluster [DBG] pgmap v1605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:49.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:49 smithi006 bash[33918]: audit 2023-08-27T22:51:48.731563+0000 mon.c (mon.2) 642 : audit [DBG] from='client.? 172.21.15.6:0/450990743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:49.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:49 smithi006 bash[27292]: cluster 2023-08-27T22:51:48.177661+0000 mgr.y (mgr.14901) 1598 : cluster [DBG] pgmap v1605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:49.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:49 smithi006 bash[27292]: audit 2023-08-27T22:51:48.731563+0000 mon.c (mon.2) 642 : audit [DBG] from='client.? 172.21.15.6:0/450990743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:51:50.309Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:51:50.311Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:51.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:51:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:51:50] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T22:51:51.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:51 smithi130 bash[27579]: cluster 2023-08-27T22:51:50.178743+0000 mgr.y (mgr.14901) 1599 : cluster [DBG] pgmap v1606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:51.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:51 smithi130 bash[27579]: audit 2023-08-27T22:51:51.140432+0000 mon.c (mon.2) 643 : audit [DBG] from='client.? 172.21.15.6:0/1271190645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:51.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:51 smithi006 bash[33918]: cluster 2023-08-27T22:51:50.178743+0000 mgr.y (mgr.14901) 1599 : cluster [DBG] pgmap v1606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:51.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:51 smithi006 bash[33918]: audit 2023-08-27T22:51:51.140432+0000 mon.c (mon.2) 643 : audit [DBG] from='client.? 172.21.15.6:0/1271190645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:51.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:51 smithi006 bash[27292]: cluster 2023-08-27T22:51:50.178743+0000 mgr.y (mgr.14901) 1599 : cluster [DBG] pgmap v1606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:51.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:51 smithi006 bash[27292]: audit 2023-08-27T22:51:51.140432+0000 mon.c (mon.2) 643 : audit [DBG] from='client.? 172.21.15.6:0/1271190645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:51:51.875Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:51:51.877Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:53.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:53 smithi130 bash[27579]: cluster 2023-08-27T22:51:52.179422+0000 mgr.y (mgr.14901) 1600 : cluster [DBG] pgmap v1607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:53.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:53 smithi006 bash[27292]: cluster 2023-08-27T22:51:52.179422+0000 mgr.y (mgr.14901) 1600 : cluster [DBG] pgmap v1607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:53.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:53 smithi006 bash[33918]: cluster 2023-08-27T22:51:52.179422+0000 mgr.y (mgr.14901) 1600 : cluster [DBG] pgmap v1607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:54.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:51:53.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:54.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:51:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:51:53.771Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:51:54.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:54 smithi130 bash[27579]: audit 2023-08-27T22:51:53.560794+0000 mon.a (mon.0) 1672 : audit [DBG] from='client.? 172.21.15.6:0/3656144073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:54.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:54 smithi006 bash[27292]: audit 2023-08-27T22:51:53.560794+0000 mon.a (mon.0) 1672 : audit [DBG] from='client.? 172.21.15.6:0/3656144073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:54.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:54 smithi006 bash[33918]: audit 2023-08-27T22:51:53.560794+0000 mon.a (mon.0) 1672 : audit [DBG] from='client.? 172.21.15.6:0/3656144073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:55.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:55 smithi130 bash[27579]: cluster 2023-08-27T22:51:54.180492+0000 mgr.y (mgr.14901) 1601 : cluster [DBG] pgmap v1608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:55.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:55 smithi006 bash[33918]: cluster 2023-08-27T22:51:54.180492+0000 mgr.y (mgr.14901) 1601 : cluster [DBG] pgmap v1608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:55.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:55 smithi006 bash[27292]: cluster 2023-08-27T22:51:54.180492+0000 mgr.y (mgr.14901) 1601 : cluster [DBG] pgmap v1608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:56.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:56 smithi130 bash[27579]: audit 2023-08-27T22:51:55.974570+0000 mon.a (mon.0) 1673 : audit [DBG] from='client.? 172.21.15.6:0/638148225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:56.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:56 smithi006 bash[33918]: audit 2023-08-27T22:51:55.974570+0000 mon.a (mon.0) 1673 : audit [DBG] from='client.? 172.21.15.6:0/638148225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:56.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:56 smithi006 bash[27292]: audit 2023-08-27T22:51:55.974570+0000 mon.a (mon.0) 1673 : audit [DBG] from='client.? 172.21.15.6:0/638148225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:57.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:57 smithi130 bash[27579]: cluster 2023-08-27T22:51:56.181154+0000 mgr.y (mgr.14901) 1602 : cluster [DBG] pgmap v1609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:57.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:57 smithi006 bash[27292]: cluster 2023-08-27T22:51:56.181154+0000 mgr.y (mgr.14901) 1602 : cluster [DBG] pgmap v1609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:57.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:57 smithi006 bash[33918]: cluster 2023-08-27T22:51:56.181154+0000 mgr.y (mgr.14901) 1602 : cluster [DBG] pgmap v1609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:59.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:59 smithi130 bash[27579]: cluster 2023-08-27T22:51:58.181775+0000 mgr.y (mgr.14901) 1603 : cluster [DBG] pgmap v1610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:51:59 smithi130 bash[27579]: audit 2023-08-27T22:51:58.394575+0000 mon.a (mon.0) 1674 : audit [DBG] from='client.? 172.21.15.6:0/2954664686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:59.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:59 smithi006 bash[33918]: cluster 2023-08-27T22:51:58.181775+0000 mgr.y (mgr.14901) 1603 : cluster [DBG] pgmap v1610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:59.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:51:59 smithi006 bash[33918]: audit 2023-08-27T22:51:58.394575+0000 mon.a (mon.0) 1674 : audit [DBG] from='client.? 172.21.15.6:0/2954664686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:59.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:59 smithi006 bash[27292]: cluster 2023-08-27T22:51:58.181775+0000 mgr.y (mgr.14901) 1603 : cluster [DBG] pgmap v1610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:51:59.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:51:59 smithi006 bash[27292]: audit 2023-08-27T22:51:58.394575+0000 mon.a (mon.0) 1674 : audit [DBG] from='client.? 172.21.15.6:0/2954664686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:00.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:52:00.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:00.937 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:52:00.312Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:01.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:52:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:52:00] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T22:52:01.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:01 smithi130 bash[27579]: cluster 2023-08-27T22:52:00.182759+0000 mgr.y (mgr.14901) 1604 : cluster [DBG] pgmap v1611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:01.651 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:01 smithi130 bash[27579]: audit 2023-08-27T22:52:00.803959+0000 mon.c (mon.2) 644 : audit [DBG] from='client.? 172.21.15.6:0/2708938892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:01.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:01 smithi006 bash[33918]: cluster 2023-08-27T22:52:00.182759+0000 mgr.y (mgr.14901) 1604 : cluster [DBG] pgmap v1611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:01.858 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:01 smithi006 bash[33918]: audit 2023-08-27T22:52:00.803959+0000 mon.c (mon.2) 644 : audit [DBG] from='client.? 172.21.15.6:0/2708938892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:01.858 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:01 smithi006 bash[27292]: cluster 2023-08-27T22:52:00.182759+0000 mgr.y (mgr.14901) 1604 : cluster [DBG] pgmap v1611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:01.858 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:01 smithi006 bash[27292]: audit 2023-08-27T22:52:00.803959+0000 mon.c (mon.2) 644 : audit [DBG] from='client.? 172.21.15.6:0/2708938892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:52:01.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:02.349 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:52:01.877Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:03.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:03 smithi130 bash[27579]: cluster 2023-08-27T22:52:02.183350+0000 mgr.y (mgr.14901) 1605 : cluster [DBG] pgmap v1612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:03.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:03 smithi130 bash[27579]: audit 2023-08-27T22:52:03.231529+0000 mon.c (mon.2) 645 : audit [DBG] from='client.? 172.21.15.6:0/4293621745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:03.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:03 smithi006 bash[33918]: cluster 2023-08-27T22:52:02.183350+0000 mgr.y (mgr.14901) 1605 : cluster [DBG] pgmap v1612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:03.800 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:03 smithi006 bash[33918]: audit 2023-08-27T22:52:03.231529+0000 mon.c (mon.2) 645 : audit [DBG] from='client.? 172.21.15.6:0/4293621745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:03.800 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:03 smithi006 bash[27292]: cluster 2023-08-27T22:52:02.183350+0000 mgr.y (mgr.14901) 1605 : cluster [DBG] pgmap v1612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:03.800 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:03 smithi006 bash[27292]: audit 2023-08-27T22:52:03.231529+0000 mon.c (mon.2) 645 : audit [DBG] from='client.? 172.21.15.6:0/4293621745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:04.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:52:03.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:04.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:52:03.772Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:05 smithi130 bash[27579]: cluster 2023-08-27T22:52:04.184438+0000 mgr.y (mgr.14901) 1606 : cluster [DBG] pgmap v1613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:05.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:05 smithi006 bash[27292]: cluster 2023-08-27T22:52:04.184438+0000 mgr.y (mgr.14901) 1606 : cluster [DBG] pgmap v1613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:05.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:05 smithi006 bash[33918]: cluster 2023-08-27T22:52:04.184438+0000 mgr.y (mgr.14901) 1606 : cluster [DBG] pgmap v1613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:06.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:06 smithi130 bash[27579]: audit 2023-08-27T22:52:05.651592+0000 mon.c (mon.2) 646 : audit [DBG] from='client.? 172.21.15.6:0/355471379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:06.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:06 smithi006 bash[27292]: audit 2023-08-27T22:52:05.651592+0000 mon.c (mon.2) 646 : audit [DBG] from='client.? 172.21.15.6:0/355471379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:06.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:06 smithi006 bash[33918]: audit 2023-08-27T22:52:05.651592+0000 mon.c (mon.2) 646 : audit [DBG] from='client.? 172.21.15.6:0/355471379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:07.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:07 smithi130 bash[27579]: cluster 2023-08-27T22:52:06.185144+0000 mgr.y (mgr.14901) 1607 : cluster [DBG] pgmap v1614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:07.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:07 smithi006 bash[27292]: cluster 2023-08-27T22:52:06.185144+0000 mgr.y (mgr.14901) 1607 : cluster [DBG] pgmap v1614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:07.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:07 smithi006 bash[33918]: cluster 2023-08-27T22:52:06.185144+0000 mgr.y (mgr.14901) 1607 : cluster [DBG] pgmap v1614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:08.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:08 smithi006 bash[27292]: audit 2023-08-27T22:52:08.059647+0000 mon.a (mon.0) 1675 : audit [DBG] from='client.? 172.21.15.6:0/1619027717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:08.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:08 smithi006 bash[33918]: audit 2023-08-27T22:52:08.059647+0000 mon.a (mon.0) 1675 : audit [DBG] from='client.? 172.21.15.6:0/1619027717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:08.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:08 smithi130 bash[27579]: audit 2023-08-27T22:52:08.059647+0000 mon.a (mon.0) 1675 : audit [DBG] from='client.? 172.21.15.6:0/1619027717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:09.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:09 smithi006 bash[27292]: cluster 2023-08-27T22:52:08.185772+0000 mgr.y (mgr.14901) 1608 : cluster [DBG] pgmap v1615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:09.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:09 smithi006 bash[33918]: cluster 2023-08-27T22:52:08.185772+0000 mgr.y (mgr.14901) 1608 : cluster [DBG] pgmap v1615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:09.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:09 smithi130 bash[27579]: cluster 2023-08-27T22:52:08.185772+0000 mgr.y (mgr.14901) 1608 : cluster [DBG] pgmap v1615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:10.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:10 smithi006 bash[33918]: audit 2023-08-27T22:52:09.614097+0000 mon.b (mon.1) 275 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:52:10.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:52:10.311Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:10.472 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:52:10.313Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:10.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:10 smithi006 bash[27292]: audit 2023-08-27T22:52:09.614097+0000 mon.b (mon.1) 275 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:52:10.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:10 smithi130 bash[27579]: audit 2023-08-27T22:52:09.614097+0000 mon.b (mon.1) 275 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:52:11.137 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:52:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:52:10] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T22:52:11.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:11 smithi006 bash[33918]: cluster 2023-08-27T22:52:10.186849+0000 mgr.y (mgr.14901) 1609 : cluster [DBG] pgmap v1616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:11.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:11 smithi006 bash[33918]: audit 2023-08-27T22:52:10.495376+0000 mon.c (mon.2) 647 : audit [DBG] from='client.? 172.21.15.6:0/3076017682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:11.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:11 smithi006 bash[27292]: cluster 2023-08-27T22:52:10.186849+0000 mgr.y (mgr.14901) 1609 : cluster [DBG] pgmap v1616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:11.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:11 smithi006 bash[27292]: audit 2023-08-27T22:52:10.495376+0000 mon.c (mon.2) 647 : audit [DBG] from='client.? 172.21.15.6:0/3076017682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:11.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:11 smithi130 bash[27579]: cluster 2023-08-27T22:52:10.186849+0000 mgr.y (mgr.14901) 1609 : cluster [DBG] pgmap v1616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:11.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:11 smithi130 bash[27579]: audit 2023-08-27T22:52:10.495376+0000 mon.c (mon.2) 647 : audit [DBG] from='client.? 172.21.15.6:0/3076017682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:52:11.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:52:11.878Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:13.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:13 smithi130 bash[27579]: cluster 2023-08-27T22:52:12.187523+0000 mgr.y (mgr.14901) 1610 : cluster [DBG] pgmap v1617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:13 smithi130 bash[27579]: audit 2023-08-27T22:52:12.915043+0000 mon.c (mon.2) 648 : audit [DBG] from='client.? 172.21.15.6:0/1377883702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:13.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:13 smithi006 bash[27292]: cluster 2023-08-27T22:52:12.187523+0000 mgr.y (mgr.14901) 1610 : cluster [DBG] pgmap v1617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:13.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:13 smithi006 bash[27292]: audit 2023-08-27T22:52:12.915043+0000 mon.c (mon.2) 648 : audit [DBG] from='client.? 172.21.15.6:0/1377883702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:13.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:13 smithi006 bash[33918]: cluster 2023-08-27T22:52:12.187523+0000 mgr.y (mgr.14901) 1610 : cluster [DBG] pgmap v1617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:13.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:13 smithi006 bash[33918]: audit 2023-08-27T22:52:12.915043+0000 mon.c (mon.2) 648 : audit [DBG] from='client.? 172.21.15.6:0/1377883702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:14.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:52:13.770Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:14.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:52:13.773Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:15.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:15 smithi130 bash[27579]: cluster 2023-08-27T22:52:14.188567+0000 mgr.y (mgr.14901) 1611 : cluster [DBG] pgmap v1618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:15.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:15 smithi006 bash[33918]: cluster 2023-08-27T22:52:14.188567+0000 mgr.y (mgr.14901) 1611 : cluster [DBG] pgmap v1618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:15.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:15 smithi006 bash[27292]: cluster 2023-08-27T22:52:14.188567+0000 mgr.y (mgr.14901) 1611 : cluster [DBG] pgmap v1618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:16.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:16 smithi130 bash[27579]: audit 2023-08-27T22:52:15.348911+0000 mon.c (mon.2) 649 : audit [DBG] from='client.? 172.21.15.6:0/2057666599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:16.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:16 smithi130 bash[27579]: audit 2023-08-27T22:52:15.478070+0000 mon.a (mon.0) 1676 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:52:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:16 smithi130 bash[27579]: audit 2023-08-27T22:52:15.917060+0000 mon.a (mon.0) 1677 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:52:16.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:16 smithi006 bash[33918]: audit 2023-08-27T22:52:15.348911+0000 mon.c (mon.2) 649 : audit [DBG] from='client.? 172.21.15.6:0/2057666599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:16.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:16 smithi006 bash[33918]: audit 2023-08-27T22:52:15.478070+0000 mon.a (mon.0) 1676 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:52:16.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:16 smithi006 bash[33918]: audit 2023-08-27T22:52:15.917060+0000 mon.a (mon.0) 1677 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:52:16.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:16 smithi006 bash[27292]: audit 2023-08-27T22:52:15.348911+0000 mon.c (mon.2) 649 : audit [DBG] from='client.? 172.21.15.6:0/2057666599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:16.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:16 smithi006 bash[27292]: audit 2023-08-27T22:52:15.478070+0000 mon.a (mon.0) 1676 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:52:16.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:16 smithi006 bash[27292]: audit 2023-08-27T22:52:15.917060+0000 mon.a (mon.0) 1677 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:52:17.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:17 smithi130 bash[27579]: cluster 2023-08-27T22:52:16.189221+0000 mgr.y (mgr.14901) 1612 : cluster [DBG] pgmap v1619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:17.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:17 smithi130 bash[27579]: audit 2023-08-27T22:52:16.214135+0000 mon.b (mon.1) 276 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:52:17.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:17 smithi130 bash[27579]: audit 2023-08-27T22:52:16.216352+0000 mon.b (mon.1) 277 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:52:17.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:17 smithi130 bash[27579]: audit 2023-08-27T22:52:16.222827+0000 mon.a (mon.0) 1678 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:52:17.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:17 smithi006 bash[33918]: cluster 2023-08-27T22:52:16.189221+0000 mgr.y (mgr.14901) 1612 : cluster [DBG] pgmap v1619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:17.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:17 smithi006 bash[33918]: audit 2023-08-27T22:52:16.214135+0000 mon.b (mon.1) 276 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:52:17.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:17 smithi006 bash[33918]: audit 2023-08-27T22:52:16.216352+0000 mon.b (mon.1) 277 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:52:17.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:17 smithi006 bash[33918]: audit 2023-08-27T22:52:16.222827+0000 mon.a (mon.0) 1678 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:52:17.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:17 smithi006 bash[27292]: cluster 2023-08-27T22:52:16.189221+0000 mgr.y (mgr.14901) 1612 : cluster [DBG] pgmap v1619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:17.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:17 smithi006 bash[27292]: audit 2023-08-27T22:52:16.214135+0000 mon.b (mon.1) 276 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:52:17.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:17 smithi006 bash[27292]: audit 2023-08-27T22:52:16.216352+0000 mon.b (mon.1) 277 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:52:17.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:17 smithi006 bash[27292]: audit 2023-08-27T22:52:16.222827+0000 mon.a (mon.0) 1678 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:52:18.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:18 smithi130 bash[27579]: audit 2023-08-27T22:52:17.772708+0000 mon.c (mon.2) 650 : audit [DBG] from='client.? 172.21.15.6:0/269396137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:18.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:18 smithi006 bash[33918]: audit 2023-08-27T22:52:17.772708+0000 mon.c (mon.2) 650 : audit [DBG] from='client.? 172.21.15.6:0/269396137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:18.472 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:18 smithi006 bash[27292]: audit 2023-08-27T22:52:17.772708+0000 mon.c (mon.2) 650 : audit [DBG] from='client.? 172.21.15.6:0/269396137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:19.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:19 smithi130 bash[27579]: cluster 2023-08-27T22:52:18.189911+0000 mgr.y (mgr.14901) 1613 : cluster [DBG] pgmap v1620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:19.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:19 smithi006 bash[33918]: cluster 2023-08-27T22:52:18.189911+0000 mgr.y (mgr.14901) 1613 : cluster [DBG] pgmap v1620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:19.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:19 smithi006 bash[27292]: cluster 2023-08-27T22:52:18.189911+0000 mgr.y (mgr.14901) 1613 : cluster [DBG] pgmap v1620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:20.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:20 smithi130 bash[27579]: audit 2023-08-27T22:52:20.181295+0000 mon.a (mon.0) 1679 : audit [DBG] from='client.? 172.21.15.6:0/2706554569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:20.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:20 smithi006 bash[27292]: audit 2023-08-27T22:52:20.181295+0000 mon.a (mon.0) 1679 : audit [DBG] from='client.? 172.21.15.6:0/2706554569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:20.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:20 smithi006 bash[33918]: audit 2023-08-27T22:52:20.181295+0000 mon.a (mon.0) 1679 : audit [DBG] from='client.? 172.21.15.6:0/2706554569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:20.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:52:20.312Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:20.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:52:20.313Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:21.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:52:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:52:20] "GET /metrics HTTP/1.1" 200 193017 "" "Prometheus/2.33.4" 2023-08-27T22:52:21.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:21 smithi130 bash[27579]: cluster 2023-08-27T22:52:20.190850+0000 mgr.y (mgr.14901) 1614 : cluster [DBG] pgmap v1621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:21.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:21 smithi006 bash[27292]: cluster 2023-08-27T22:52:20.190850+0000 mgr.y (mgr.14901) 1614 : cluster [DBG] pgmap v1621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:21.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:21 smithi006 bash[33918]: cluster 2023-08-27T22:52:20.190850+0000 mgr.y (mgr.14901) 1614 : cluster [DBG] pgmap v1621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:52:21.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:52:21.879Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:23.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:23 smithi130 bash[27579]: cluster 2023-08-27T22:52:22.191502+0000 mgr.y (mgr.14901) 1615 : cluster [DBG] pgmap v1622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:23.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:23 smithi130 bash[27579]: audit 2023-08-27T22:52:22.593027+0000 mon.a (mon.0) 1680 : audit [DBG] from='client.? 172.21.15.6:0/237536356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:23.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:23 smithi006 bash[27292]: cluster 2023-08-27T22:52:22.191502+0000 mgr.y (mgr.14901) 1615 : cluster [DBG] pgmap v1622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:23.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:23 smithi006 bash[27292]: audit 2023-08-27T22:52:22.593027+0000 mon.a (mon.0) 1680 : audit [DBG] from='client.? 172.21.15.6:0/237536356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:23.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:23 smithi006 bash[33918]: cluster 2023-08-27T22:52:22.191502+0000 mgr.y (mgr.14901) 1615 : cluster [DBG] pgmap v1622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:23.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:23 smithi006 bash[33918]: audit 2023-08-27T22:52:22.593027+0000 mon.a (mon.0) 1680 : audit [DBG] from='client.? 172.21.15.6:0/237536356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:24.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:52:23.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:24.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:52:23.773Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:25.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:25 smithi130 bash[27579]: cluster 2023-08-27T22:52:24.192538+0000 mgr.y (mgr.14901) 1616 : cluster [DBG] pgmap v1623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:25.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:25 smithi130 bash[27579]: audit 2023-08-27T22:52:25.007981+0000 mon.a (mon.0) 1681 : audit [DBG] from='client.? 172.21.15.6:0/4130601240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:25.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:25 smithi006 bash[27292]: cluster 2023-08-27T22:52:24.192538+0000 mgr.y (mgr.14901) 1616 : cluster [DBG] pgmap v1623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:25.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:25 smithi006 bash[27292]: audit 2023-08-27T22:52:25.007981+0000 mon.a (mon.0) 1681 : audit [DBG] from='client.? 172.21.15.6:0/4130601240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:25.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:25 smithi006 bash[33918]: cluster 2023-08-27T22:52:24.192538+0000 mgr.y (mgr.14901) 1616 : cluster [DBG] pgmap v1623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:25.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:25 smithi006 bash[33918]: audit 2023-08-27T22:52:25.007981+0000 mon.a (mon.0) 1681 : audit [DBG] from='client.? 172.21.15.6:0/4130601240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:27.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:27 smithi130 bash[27579]: cluster 2023-08-27T22:52:26.193223+0000 mgr.y (mgr.14901) 1617 : cluster [DBG] pgmap v1624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:27.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:27 smithi006 bash[27292]: cluster 2023-08-27T22:52:26.193223+0000 mgr.y (mgr.14901) 1617 : cluster [DBG] pgmap v1624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:27.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:27 smithi006 bash[33918]: cluster 2023-08-27T22:52:26.193223+0000 mgr.y (mgr.14901) 1617 : cluster [DBG] pgmap v1624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:28.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:28 smithi130 bash[27579]: audit 2023-08-27T22:52:27.417030+0000 mon.c (mon.2) 651 : audit [DBG] from='client.? 172.21.15.6:0/3929235495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:28.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:28 smithi006 bash[27292]: audit 2023-08-27T22:52:27.417030+0000 mon.c (mon.2) 651 : audit [DBG] from='client.? 172.21.15.6:0/3929235495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:28.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:28 smithi006 bash[33918]: audit 2023-08-27T22:52:27.417030+0000 mon.c (mon.2) 651 : audit [DBG] from='client.? 172.21.15.6:0/3929235495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:29.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:29 smithi130 bash[27579]: cluster 2023-08-27T22:52:28.193887+0000 mgr.y (mgr.14901) 1618 : cluster [DBG] pgmap v1625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:29.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:29 smithi006 bash[27292]: cluster 2023-08-27T22:52:28.193887+0000 mgr.y (mgr.14901) 1618 : cluster [DBG] pgmap v1625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:29.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:29 smithi006 bash[33918]: cluster 2023-08-27T22:52:28.193887+0000 mgr.y (mgr.14901) 1618 : cluster [DBG] pgmap v1625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:30.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:30 smithi130 bash[27579]: audit 2023-08-27T22:52:29.833787+0000 mon.a (mon.0) 1682 : audit [DBG] from='client.? 172.21.15.6:0/960601694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:30.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:30 smithi006 bash[33918]: audit 2023-08-27T22:52:29.833787+0000 mon.a (mon.0) 1682 : audit [DBG] from='client.? 172.21.15.6:0/960601694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:30.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:30 smithi006 bash[27292]: audit 2023-08-27T22:52:29.833787+0000 mon.a (mon.0) 1682 : audit [DBG] from='client.? 172.21.15.6:0/960601694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:30.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:52:30.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:30.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:52:30.314Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:31.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:52:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:52:30] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T22:52:31.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:31 smithi130 bash[27579]: cluster 2023-08-27T22:52:30.195061+0000 mgr.y (mgr.14901) 1619 : cluster [DBG] pgmap v1626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:31.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:31 smithi006 bash[33918]: cluster 2023-08-27T22:52:30.195061+0000 mgr.y (mgr.14901) 1619 : cluster [DBG] pgmap v1626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:31.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:31 smithi006 bash[27292]: cluster 2023-08-27T22:52:30.195061+0000 mgr.y (mgr.14901) 1619 : cluster [DBG] pgmap v1626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:52:31.878Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:52:31.879Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:32.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:32 smithi130 bash[27579]: audit 2023-08-27T22:52:32.246804+0000 mon.a (mon.0) 1683 : audit [DBG] from='client.? 172.21.15.6:0/2919163781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:32.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:32 smithi006 bash[27292]: audit 2023-08-27T22:52:32.246804+0000 mon.a (mon.0) 1683 : audit [DBG] from='client.? 172.21.15.6:0/2919163781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:32.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:32 smithi006 bash[33918]: audit 2023-08-27T22:52:32.246804+0000 mon.a (mon.0) 1683 : audit [DBG] from='client.? 172.21.15.6:0/2919163781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:33.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:33 smithi130 bash[27579]: cluster 2023-08-27T22:52:32.195675+0000 mgr.y (mgr.14901) 1620 : cluster [DBG] pgmap v1627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:33.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:33 smithi006 bash[33918]: cluster 2023-08-27T22:52:32.195675+0000 mgr.y (mgr.14901) 1620 : cluster [DBG] pgmap v1627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:33.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:33 smithi006 bash[27292]: cluster 2023-08-27T22:52:32.195675+0000 mgr.y (mgr.14901) 1620 : cluster [DBG] pgmap v1627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:34.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:52:33.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:34.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:52:33.774Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:35.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:35 smithi130 bash[27579]: cluster 2023-08-27T22:52:34.196695+0000 mgr.y (mgr.14901) 1621 : cluster [DBG] pgmap v1628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:35.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:35 smithi130 bash[27579]: audit 2023-08-27T22:52:34.654711+0000 mon.c (mon.2) 652 : audit [DBG] from='client.? 172.21.15.6:0/361423800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:35.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:35 smithi006 bash[27292]: cluster 2023-08-27T22:52:34.196695+0000 mgr.y (mgr.14901) 1621 : cluster [DBG] pgmap v1628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:35.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:35 smithi006 bash[27292]: audit 2023-08-27T22:52:34.654711+0000 mon.c (mon.2) 652 : audit [DBG] from='client.? 172.21.15.6:0/361423800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:35.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:35 smithi006 bash[33918]: cluster 2023-08-27T22:52:34.196695+0000 mgr.y (mgr.14901) 1621 : cluster [DBG] pgmap v1628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:35.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:35 smithi006 bash[33918]: audit 2023-08-27T22:52:34.654711+0000 mon.c (mon.2) 652 : audit [DBG] from='client.? 172.21.15.6:0/361423800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:37.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:37 smithi130 bash[27579]: cluster 2023-08-27T22:52:36.197329+0000 mgr.y (mgr.14901) 1622 : cluster [DBG] pgmap v1629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:37.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:37 smithi130 bash[27579]: audit 2023-08-27T22:52:37.064750+0000 mon.c (mon.2) 653 : audit [DBG] from='client.? 172.21.15.6:0/2459044536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:37.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:37 smithi006 bash[33918]: cluster 2023-08-27T22:52:36.197329+0000 mgr.y (mgr.14901) 1622 : cluster [DBG] pgmap v1629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:37.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:37 smithi006 bash[33918]: audit 2023-08-27T22:52:37.064750+0000 mon.c (mon.2) 653 : audit [DBG] from='client.? 172.21.15.6:0/2459044536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:37.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:37 smithi006 bash[27292]: cluster 2023-08-27T22:52:36.197329+0000 mgr.y (mgr.14901) 1622 : cluster [DBG] pgmap v1629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:37.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:37 smithi006 bash[27292]: audit 2023-08-27T22:52:37.064750+0000 mon.c (mon.2) 653 : audit [DBG] from='client.? 172.21.15.6:0/2459044536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:39.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:39 smithi130 bash[27579]: cluster 2023-08-27T22:52:38.198015+0000 mgr.y (mgr.14901) 1623 : cluster [DBG] pgmap v1630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:39.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:39 smithi006 bash[33918]: cluster 2023-08-27T22:52:38.198015+0000 mgr.y (mgr.14901) 1623 : cluster [DBG] pgmap v1630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:39.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:39 smithi006 bash[27292]: cluster 2023-08-27T22:52:38.198015+0000 mgr.y (mgr.14901) 1623 : cluster [DBG] pgmap v1630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:40.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:40 smithi130 bash[27579]: audit 2023-08-27T22:52:39.491933+0000 mon.a (mon.0) 1684 : audit [DBG] from='client.? 172.21.15.6:0/2954423765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:40.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:40 smithi006 bash[33918]: audit 2023-08-27T22:52:39.491933+0000 mon.a (mon.0) 1684 : audit [DBG] from='client.? 172.21.15.6:0/2954423765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:40.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:40 smithi006 bash[27292]: audit 2023-08-27T22:52:39.491933+0000 mon.a (mon.0) 1684 : audit [DBG] from='client.? 172.21.15.6:0/2954423765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:40.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:52:40.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:40.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:52:40.315Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:41.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:52:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:52:40] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T22:52:41.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:41 smithi130 bash[27579]: cluster 2023-08-27T22:52:40.199081+0000 mgr.y (mgr.14901) 1624 : cluster [DBG] pgmap v1631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:41.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:41 smithi006 bash[33918]: cluster 2023-08-27T22:52:40.199081+0000 mgr.y (mgr.14901) 1624 : cluster [DBG] pgmap v1631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:41.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:41 smithi006 bash[27292]: cluster 2023-08-27T22:52:40.199081+0000 mgr.y (mgr.14901) 1624 : cluster [DBG] pgmap v1631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:52:41.878Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:52:41.880Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:42.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:42 smithi130 bash[27579]: audit 2023-08-27T22:52:41.902792+0000 mon.c (mon.2) 654 : audit [DBG] from='client.? 172.21.15.6:0/457544728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:42.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:42 smithi006 bash[33918]: audit 2023-08-27T22:52:41.902792+0000 mon.c (mon.2) 654 : audit [DBG] from='client.? 172.21.15.6:0/457544728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:42.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:42 smithi006 bash[27292]: audit 2023-08-27T22:52:41.902792+0000 mon.c (mon.2) 654 : audit [DBG] from='client.? 172.21.15.6:0/457544728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:43.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:43 smithi130 bash[27579]: cluster 2023-08-27T22:52:42.199744+0000 mgr.y (mgr.14901) 1625 : cluster [DBG] pgmap v1632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:43.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:43 smithi006 bash[27292]: cluster 2023-08-27T22:52:42.199744+0000 mgr.y (mgr.14901) 1625 : cluster [DBG] pgmap v1632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:43.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:43 smithi006 bash[33918]: cluster 2023-08-27T22:52:42.199744+0000 mgr.y (mgr.14901) 1625 : cluster [DBG] pgmap v1632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:44.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:52:43.773Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:44.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:52:43.775Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:45.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:45 smithi130 bash[27579]: cluster 2023-08-27T22:52:44.200494+0000 mgr.y (mgr.14901) 1626 : cluster [DBG] pgmap v1633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:45 smithi130 bash[27579]: audit 2023-08-27T22:52:44.318293+0000 mon.c (mon.2) 655 : audit [DBG] from='client.? 172.21.15.6:0/1627709172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:45.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:45 smithi006 bash[27292]: cluster 2023-08-27T22:52:44.200494+0000 mgr.y (mgr.14901) 1626 : cluster [DBG] pgmap v1633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:45.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:45 smithi006 bash[27292]: audit 2023-08-27T22:52:44.318293+0000 mon.c (mon.2) 655 : audit [DBG] from='client.? 172.21.15.6:0/1627709172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:45.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:45 smithi006 bash[33918]: cluster 2023-08-27T22:52:44.200494+0000 mgr.y (mgr.14901) 1626 : cluster [DBG] pgmap v1633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:45.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:45 smithi006 bash[33918]: audit 2023-08-27T22:52:44.318293+0000 mon.c (mon.2) 655 : audit [DBG] from='client.? 172.21.15.6:0/1627709172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:47.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:47 smithi130 bash[27579]: cluster 2023-08-27T22:52:46.201112+0000 mgr.y (mgr.14901) 1627 : cluster [DBG] pgmap v1634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:47.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:47 smithi130 bash[27579]: audit 2023-08-27T22:52:46.738537+0000 mon.a (mon.0) 1685 : audit [DBG] from='client.? 172.21.15.6:0/1946545971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:47.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:47 smithi006 bash[33918]: cluster 2023-08-27T22:52:46.201112+0000 mgr.y (mgr.14901) 1627 : cluster [DBG] pgmap v1634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:47.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:47 smithi006 bash[33918]: audit 2023-08-27T22:52:46.738537+0000 mon.a (mon.0) 1685 : audit [DBG] from='client.? 172.21.15.6:0/1946545971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:47.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:47 smithi006 bash[27292]: cluster 2023-08-27T22:52:46.201112+0000 mgr.y (mgr.14901) 1627 : cluster [DBG] pgmap v1634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:47.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:47 smithi006 bash[27292]: audit 2023-08-27T22:52:46.738537+0000 mon.a (mon.0) 1685 : audit [DBG] from='client.? 172.21.15.6:0/1946545971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:49.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:49 smithi130 bash[27579]: cluster 2023-08-27T22:52:48.201787+0000 mgr.y (mgr.14901) 1628 : cluster [DBG] pgmap v1635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:49.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:49 smithi130 bash[27579]: audit 2023-08-27T22:52:49.164926+0000 mon.a (mon.0) 1686 : audit [DBG] from='client.? 172.21.15.6:0/1759722898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:49.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:49 smithi006 bash[27292]: cluster 2023-08-27T22:52:48.201787+0000 mgr.y (mgr.14901) 1628 : cluster [DBG] pgmap v1635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:49.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:49 smithi006 bash[27292]: audit 2023-08-27T22:52:49.164926+0000 mon.a (mon.0) 1686 : audit [DBG] from='client.? 172.21.15.6:0/1759722898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:49.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:49 smithi006 bash[33918]: cluster 2023-08-27T22:52:48.201787+0000 mgr.y (mgr.14901) 1628 : cluster [DBG] pgmap v1635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:49.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:49 smithi006 bash[33918]: audit 2023-08-27T22:52:49.164926+0000 mon.a (mon.0) 1686 : audit [DBG] from='client.? 172.21.15.6:0/1759722898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:52:50.315Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:50.722 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:52:50.317Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:51.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:52:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:52:50] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T22:52:51.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:51 smithi130 bash[27579]: cluster 2023-08-27T22:52:50.202949+0000 mgr.y (mgr.14901) 1629 : cluster [DBG] pgmap v1636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:51.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:51 smithi006 bash[27292]: cluster 2023-08-27T22:52:50.202949+0000 mgr.y (mgr.14901) 1629 : cluster [DBG] pgmap v1636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:51.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:51 smithi006 bash[33918]: cluster 2023-08-27T22:52:50.202949+0000 mgr.y (mgr.14901) 1629 : cluster [DBG] pgmap v1636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:52:51.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:52:51.880Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:52.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:52 smithi130 bash[27579]: audit 2023-08-27T22:52:51.570920+0000 mon.a (mon.0) 1687 : audit [DBG] from='client.? 172.21.15.6:0/2137920966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:52.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:52 smithi006 bash[33918]: audit 2023-08-27T22:52:51.570920+0000 mon.a (mon.0) 1687 : audit [DBG] from='client.? 172.21.15.6:0/2137920966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:52.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:52 smithi006 bash[27292]: audit 2023-08-27T22:52:51.570920+0000 mon.a (mon.0) 1687 : audit [DBG] from='client.? 172.21.15.6:0/2137920966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:53.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:53 smithi130 bash[27579]: cluster 2023-08-27T22:52:52.203613+0000 mgr.y (mgr.14901) 1630 : cluster [DBG] pgmap v1637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:53.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:53 smithi006 bash[33918]: cluster 2023-08-27T22:52:52.203613+0000 mgr.y (mgr.14901) 1630 : cluster [DBG] pgmap v1637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:53.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:53 smithi006 bash[27292]: cluster 2023-08-27T22:52:52.203613+0000 mgr.y (mgr.14901) 1630 : cluster [DBG] pgmap v1637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:54.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:52:53.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:54.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:52:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:52:53.776Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:52:54.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:54 smithi130 bash[27579]: audit 2023-08-27T22:52:53.989985+0000 mon.c (mon.2) 656 : audit [DBG] from='client.? 172.21.15.6:0/1496831707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:54.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:54 smithi006 bash[33918]: audit 2023-08-27T22:52:53.989985+0000 mon.c (mon.2) 656 : audit [DBG] from='client.? 172.21.15.6:0/1496831707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:54.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:54 smithi006 bash[27292]: audit 2023-08-27T22:52:53.989985+0000 mon.c (mon.2) 656 : audit [DBG] from='client.? 172.21.15.6:0/1496831707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:55.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:55 smithi130 bash[27579]: cluster 2023-08-27T22:52:54.204877+0000 mgr.y (mgr.14901) 1631 : cluster [DBG] pgmap v1638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:55.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:55 smithi006 bash[33918]: cluster 2023-08-27T22:52:54.204877+0000 mgr.y (mgr.14901) 1631 : cluster [DBG] pgmap v1638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:55.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:55 smithi006 bash[27292]: cluster 2023-08-27T22:52:54.204877+0000 mgr.y (mgr.14901) 1631 : cluster [DBG] pgmap v1638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:57.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:57 smithi130 bash[27579]: cluster 2023-08-27T22:52:56.205700+0000 mgr.y (mgr.14901) 1632 : cluster [DBG] pgmap v1639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:57.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:57 smithi130 bash[27579]: audit 2023-08-27T22:52:56.403884+0000 mon.a (mon.0) 1688 : audit [DBG] from='client.? 172.21.15.6:0/2003354732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:57.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:57 smithi006 bash[33918]: cluster 2023-08-27T22:52:56.205700+0000 mgr.y (mgr.14901) 1632 : cluster [DBG] pgmap v1639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:57.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:57 smithi006 bash[33918]: audit 2023-08-27T22:52:56.403884+0000 mon.a (mon.0) 1688 : audit [DBG] from='client.? 172.21.15.6:0/2003354732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:57.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:57 smithi006 bash[27292]: cluster 2023-08-27T22:52:56.205700+0000 mgr.y (mgr.14901) 1632 : cluster [DBG] pgmap v1639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:57.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:57 smithi006 bash[27292]: audit 2023-08-27T22:52:56.403884+0000 mon.a (mon.0) 1688 : audit [DBG] from='client.? 172.21.15.6:0/2003354732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:59.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:59 smithi130 bash[27579]: cluster 2023-08-27T22:52:58.206381+0000 mgr.y (mgr.14901) 1633 : cluster [DBG] pgmap v1640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:52:59 smithi130 bash[27579]: audit 2023-08-27T22:52:58.814339+0000 mon.c (mon.2) 657 : audit [DBG] from='client.? 172.21.15.6:0/724285517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:59.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:59 smithi006 bash[27292]: cluster 2023-08-27T22:52:58.206381+0000 mgr.y (mgr.14901) 1633 : cluster [DBG] pgmap v1640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:59.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:52:59 smithi006 bash[27292]: audit 2023-08-27T22:52:58.814339+0000 mon.c (mon.2) 657 : audit [DBG] from='client.? 172.21.15.6:0/724285517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:59.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:59 smithi006 bash[33918]: cluster 2023-08-27T22:52:58.206381+0000 mgr.y (mgr.14901) 1633 : cluster [DBG] pgmap v1640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:52:59.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:52:59 smithi006 bash[33918]: audit 2023-08-27T22:52:58.814339+0000 mon.c (mon.2) 657 : audit [DBG] from='client.? 172.21.15.6:0/724285517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:00.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:53:00.315Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:00.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:53:00.318Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:01.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:53:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:53:00] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T22:53:01.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:01 smithi130 bash[27579]: cluster 2023-08-27T22:53:00.207499+0000 mgr.y (mgr.14901) 1634 : cluster [DBG] pgmap v1641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:01.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:01 smithi130 bash[27579]: audit 2023-08-27T22:53:01.237560+0000 mon.c (mon.2) 658 : audit [DBG] from='client.? 172.21.15.6:0/3099589103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:01.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:01 smithi006 bash[27292]: cluster 2023-08-27T22:53:00.207499+0000 mgr.y (mgr.14901) 1634 : cluster [DBG] pgmap v1641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:01.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:01 smithi006 bash[27292]: audit 2023-08-27T22:53:01.237560+0000 mon.c (mon.2) 658 : audit [DBG] from='client.? 172.21.15.6:0/3099589103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:01.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:01 smithi006 bash[33918]: cluster 2023-08-27T22:53:00.207499+0000 mgr.y (mgr.14901) 1634 : cluster [DBG] pgmap v1641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:01.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:01 smithi006 bash[33918]: audit 2023-08-27T22:53:01.237560+0000 mon.c (mon.2) 658 : audit [DBG] from='client.? 172.21.15.6:0/3099589103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:53:01.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:53:01.881Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:03.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:03 smithi130 bash[27579]: cluster 2023-08-27T22:53:02.208096+0000 mgr.y (mgr.14901) 1635 : cluster [DBG] pgmap v1642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:03.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:03 smithi006 bash[27292]: cluster 2023-08-27T22:53:02.208096+0000 mgr.y (mgr.14901) 1635 : cluster [DBG] pgmap v1642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:03.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:03 smithi006 bash[33918]: cluster 2023-08-27T22:53:02.208096+0000 mgr.y (mgr.14901) 1635 : cluster [DBG] pgmap v1642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:04.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:53:03.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:04.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:53:03.776Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:04.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:04 smithi130 bash[27579]: audit 2023-08-27T22:53:03.658617+0000 mon.a (mon.0) 1689 : audit [DBG] from='client.? 172.21.15.6:0/160919076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:04.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:04 smithi006 bash[27292]: audit 2023-08-27T22:53:03.658617+0000 mon.a (mon.0) 1689 : audit [DBG] from='client.? 172.21.15.6:0/160919076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:04.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:04 smithi006 bash[33918]: audit 2023-08-27T22:53:03.658617+0000 mon.a (mon.0) 1689 : audit [DBG] from='client.? 172.21.15.6:0/160919076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:05.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:05 smithi130 bash[27579]: cluster 2023-08-27T22:53:04.209254+0000 mgr.y (mgr.14901) 1636 : cluster [DBG] pgmap v1643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:05.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:05 smithi006 bash[27292]: cluster 2023-08-27T22:53:04.209254+0000 mgr.y (mgr.14901) 1636 : cluster [DBG] pgmap v1643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:05.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:05 smithi006 bash[33918]: cluster 2023-08-27T22:53:04.209254+0000 mgr.y (mgr.14901) 1636 : cluster [DBG] pgmap v1643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:06.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:06 smithi130 bash[27579]: audit 2023-08-27T22:53:06.071273+0000 mon.a (mon.0) 1690 : audit [DBG] from='client.? 172.21.15.6:0/2399731331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:06.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:06 smithi006 bash[33918]: audit 2023-08-27T22:53:06.071273+0000 mon.a (mon.0) 1690 : audit [DBG] from='client.? 172.21.15.6:0/2399731331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:06.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:06 smithi006 bash[27292]: audit 2023-08-27T22:53:06.071273+0000 mon.a (mon.0) 1690 : audit [DBG] from='client.? 172.21.15.6:0/2399731331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:07.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:07 smithi130 bash[27579]: cluster 2023-08-27T22:53:06.209894+0000 mgr.y (mgr.14901) 1637 : cluster [DBG] pgmap v1644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:07.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:07 smithi006 bash[27292]: cluster 2023-08-27T22:53:06.209894+0000 mgr.y (mgr.14901) 1637 : cluster [DBG] pgmap v1644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:07.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:07 smithi006 bash[33918]: cluster 2023-08-27T22:53:06.209894+0000 mgr.y (mgr.14901) 1637 : cluster [DBG] pgmap v1644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:09.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:09 smithi130 bash[27579]: cluster 2023-08-27T22:53:08.210518+0000 mgr.y (mgr.14901) 1638 : cluster [DBG] pgmap v1645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:09 smithi130 bash[27579]: audit 2023-08-27T22:53:08.485545+0000 mon.c (mon.2) 659 : audit [DBG] from='client.? 172.21.15.6:0/4031061976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:09.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:09 smithi006 bash[27292]: cluster 2023-08-27T22:53:08.210518+0000 mgr.y (mgr.14901) 1638 : cluster [DBG] pgmap v1645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:09.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:09 smithi006 bash[27292]: audit 2023-08-27T22:53:08.485545+0000 mon.c (mon.2) 659 : audit [DBG] from='client.? 172.21.15.6:0/4031061976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:09.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:09 smithi006 bash[33918]: cluster 2023-08-27T22:53:08.210518+0000 mgr.y (mgr.14901) 1638 : cluster [DBG] pgmap v1645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:09.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:09 smithi006 bash[33918]: audit 2023-08-27T22:53:08.485545+0000 mon.c (mon.2) 659 : audit [DBG] from='client.? 172.21.15.6:0/4031061976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:53:10.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:10.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:53:10.319Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:11.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:53:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:53:10] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T22:53:11.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:11 smithi130 bash[27579]: cluster 2023-08-27T22:53:10.211564+0000 mgr.y (mgr.14901) 1639 : cluster [DBG] pgmap v1646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:11.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:11 smithi130 bash[27579]: audit 2023-08-27T22:53:10.899327+0000 mon.a (mon.0) 1691 : audit [DBG] from='client.? 172.21.15.6:0/1562767568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:11.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:11 smithi006 bash[33918]: cluster 2023-08-27T22:53:10.211564+0000 mgr.y (mgr.14901) 1639 : cluster [DBG] pgmap v1646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:11.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:11 smithi006 bash[33918]: audit 2023-08-27T22:53:10.899327+0000 mon.a (mon.0) 1691 : audit [DBG] from='client.? 172.21.15.6:0/1562767568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:11.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:11 smithi006 bash[27292]: cluster 2023-08-27T22:53:10.211564+0000 mgr.y (mgr.14901) 1639 : cluster [DBG] pgmap v1646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:11.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:11 smithi006 bash[27292]: audit 2023-08-27T22:53:10.899327+0000 mon.a (mon.0) 1691 : audit [DBG] from='client.? 172.21.15.6:0/1562767568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:53:11.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:12.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:53:11.882Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:13.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:13 smithi130 bash[27579]: cluster 2023-08-27T22:53:12.212215+0000 mgr.y (mgr.14901) 1640 : cluster [DBG] pgmap v1647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:13.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:13 smithi130 bash[27579]: audit 2023-08-27T22:53:13.321100+0000 mon.a (mon.0) 1692 : audit [DBG] from='client.? 172.21.15.6:0/1380252240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:13.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:13 smithi006 bash[33918]: cluster 2023-08-27T22:53:12.212215+0000 mgr.y (mgr.14901) 1640 : cluster [DBG] pgmap v1647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:13.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:13 smithi006 bash[33918]: audit 2023-08-27T22:53:13.321100+0000 mon.a (mon.0) 1692 : audit [DBG] from='client.? 172.21.15.6:0/1380252240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:13.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:13 smithi006 bash[27292]: cluster 2023-08-27T22:53:12.212215+0000 mgr.y (mgr.14901) 1640 : cluster [DBG] pgmap v1647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:13.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:13 smithi006 bash[27292]: audit 2023-08-27T22:53:13.321100+0000 mon.a (mon.0) 1692 : audit [DBG] from='client.? 172.21.15.6:0/1380252240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:14.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:53:13.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:14.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:53:13.777Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:15.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:15 smithi130 bash[27579]: cluster 2023-08-27T22:53:14.213289+0000 mgr.y (mgr.14901) 1641 : cluster [DBG] pgmap v1648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:15.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:15 smithi006 bash[33918]: cluster 2023-08-27T22:53:14.213289+0000 mgr.y (mgr.14901) 1641 : cluster [DBG] pgmap v1648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:15.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:15 smithi006 bash[27292]: cluster 2023-08-27T22:53:14.213289+0000 mgr.y (mgr.14901) 1641 : cluster [DBG] pgmap v1648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:16.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:16 smithi130 bash[27579]: audit 2023-08-27T22:53:15.740433+0000 mon.a (mon.0) 1693 : audit [DBG] from='client.? 172.21.15.6:0/474997719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:16.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:16 smithi006 bash[33918]: audit 2023-08-27T22:53:15.740433+0000 mon.a (mon.0) 1693 : audit [DBG] from='client.? 172.21.15.6:0/474997719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:16.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:16 smithi006 bash[27292]: audit 2023-08-27T22:53:15.740433+0000 mon.a (mon.0) 1693 : audit [DBG] from='client.? 172.21.15.6:0/474997719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:17.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:17 smithi130 bash[27579]: cluster 2023-08-27T22:53:16.213965+0000 mgr.y (mgr.14901) 1642 : cluster [DBG] pgmap v1649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:17.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:17 smithi130 bash[27579]: audit 2023-08-27T22:53:16.237397+0000 mon.b (mon.1) 278 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:53:17.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:17 smithi006 bash[27292]: cluster 2023-08-27T22:53:16.213965+0000 mgr.y (mgr.14901) 1642 : cluster [DBG] pgmap v1649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:17.473 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:17 smithi006 bash[27292]: audit 2023-08-27T22:53:16.237397+0000 mon.b (mon.1) 278 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:53:17.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:17 smithi006 bash[33918]: cluster 2023-08-27T22:53:16.213965+0000 mgr.y (mgr.14901) 1642 : cluster [DBG] pgmap v1649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:17.473 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:17 smithi006 bash[33918]: audit 2023-08-27T22:53:16.237397+0000 mon.b (mon.1) 278 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:53:18.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:18 smithi130 bash[27579]: audit 2023-08-27T22:53:18.164471+0000 mon.a (mon.0) 1694 : audit [DBG] from='client.? 172.21.15.6:0/2880177856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:18.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:18 smithi006 bash[33918]: audit 2023-08-27T22:53:18.164471+0000 mon.a (mon.0) 1694 : audit [DBG] from='client.? 172.21.15.6:0/2880177856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:18.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:18 smithi006 bash[27292]: audit 2023-08-27T22:53:18.164471+0000 mon.a (mon.0) 1694 : audit [DBG] from='client.? 172.21.15.6:0/2880177856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:19.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:19 smithi130 bash[27579]: cluster 2023-08-27T22:53:18.214757+0000 mgr.y (mgr.14901) 1643 : cluster [DBG] pgmap v1650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:19.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:19 smithi006 bash[33918]: cluster 2023-08-27T22:53:18.214757+0000 mgr.y (mgr.14901) 1643 : cluster [DBG] pgmap v1650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:19.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:19 smithi006 bash[27292]: cluster 2023-08-27T22:53:18.214757+0000 mgr.y (mgr.14901) 1643 : cluster [DBG] pgmap v1650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:53:20.318Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:20.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:53:20.319Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:21.221 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:53:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:53:20] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T22:53:21.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:21 smithi130 bash[27579]: cluster 2023-08-27T22:53:20.215923+0000 mgr.y (mgr.14901) 1644 : cluster [DBG] pgmap v1651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:21.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:21 smithi130 bash[27579]: audit 2023-08-27T22:53:20.590250+0000 mon.a (mon.0) 1695 : audit [DBG] from='client.? 172.21.15.6:0/3389863783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:21.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:21 smithi006 bash[33918]: cluster 2023-08-27T22:53:20.215923+0000 mgr.y (mgr.14901) 1644 : cluster [DBG] pgmap v1651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:21.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:21 smithi006 bash[33918]: audit 2023-08-27T22:53:20.590250+0000 mon.a (mon.0) 1695 : audit [DBG] from='client.? 172.21.15.6:0/3389863783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:21.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:21 smithi006 bash[27292]: cluster 2023-08-27T22:53:20.215923+0000 mgr.y (mgr.14901) 1644 : cluster [DBG] pgmap v1651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:21.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:21 smithi006 bash[27292]: audit 2023-08-27T22:53:20.590250+0000 mon.a (mon.0) 1695 : audit [DBG] from='client.? 172.21.15.6:0/3389863783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:53:21.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:22.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:53:21.882Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:23.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:23 smithi130 bash[27579]: audit 2023-08-27T22:53:22.101370+0000 mon.a (mon.0) 1696 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:53:23.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:23 smithi130 bash[27579]: cluster 2023-08-27T22:53:22.216577+0000 mgr.y (mgr.14901) 1645 : cluster [DBG] pgmap v1652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:23.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:23 smithi130 bash[27579]: audit 2023-08-27T22:53:22.552034+0000 mon.a (mon.0) 1697 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:53:23.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:23 smithi130 bash[27579]: audit 2023-08-27T22:53:23.001046+0000 mon.a (mon.0) 1698 : audit [DBG] from='client.? 172.21.15.6:0/1265839872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:23.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:23 smithi006 bash[27292]: audit 2023-08-27T22:53:22.101370+0000 mon.a (mon.0) 1696 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:53:23.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:23 smithi006 bash[27292]: cluster 2023-08-27T22:53:22.216577+0000 mgr.y (mgr.14901) 1645 : cluster [DBG] pgmap v1652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:23.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:23 smithi006 bash[27292]: audit 2023-08-27T22:53:22.552034+0000 mon.a (mon.0) 1697 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:53:23.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:23 smithi006 bash[27292]: audit 2023-08-27T22:53:23.001046+0000 mon.a (mon.0) 1698 : audit [DBG] from='client.? 172.21.15.6:0/1265839872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:23.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:23 smithi006 bash[33918]: audit 2023-08-27T22:53:22.101370+0000 mon.a (mon.0) 1696 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:53:23.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:23 smithi006 bash[33918]: cluster 2023-08-27T22:53:22.216577+0000 mgr.y (mgr.14901) 1645 : cluster [DBG] pgmap v1652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:23.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:23 smithi006 bash[33918]: audit 2023-08-27T22:53:22.552034+0000 mon.a (mon.0) 1697 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:53:23.472 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:23 smithi006 bash[33918]: audit 2023-08-27T22:53:23.001046+0000 mon.a (mon.0) 1698 : audit [DBG] from='client.? 172.21.15.6:0/1265839872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:24.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:53:23.776Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:24.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:53:23.778Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:25.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:25 smithi130 bash[27579]: cluster 2023-08-27T22:53:24.217702+0000 mgr.y (mgr.14901) 1646 : cluster [DBG] pgmap v1653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:25.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:25 smithi006 bash[33918]: cluster 2023-08-27T22:53:24.217702+0000 mgr.y (mgr.14901) 1646 : cluster [DBG] pgmap v1653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:25.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:25 smithi006 bash[27292]: cluster 2023-08-27T22:53:24.217702+0000 mgr.y (mgr.14901) 1646 : cluster [DBG] pgmap v1653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:26.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:26 smithi130 bash[27579]: audit 2023-08-27T22:53:25.417334+0000 mon.a (mon.0) 1699 : audit [DBG] from='client.? 172.21.15.6:0/4142539992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:26.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:26 smithi006 bash[27292]: audit 2023-08-27T22:53:25.417334+0000 mon.a (mon.0) 1699 : audit [DBG] from='client.? 172.21.15.6:0/4142539992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:26.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:26 smithi006 bash[33918]: audit 2023-08-27T22:53:25.417334+0000 mon.a (mon.0) 1699 : audit [DBG] from='client.? 172.21.15.6:0/4142539992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:27.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:27 smithi130 bash[27579]: cluster 2023-08-27T22:53:26.218398+0000 mgr.y (mgr.14901) 1647 : cluster [DBG] pgmap v1654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:27.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:27 smithi006 bash[27292]: cluster 2023-08-27T22:53:26.218398+0000 mgr.y (mgr.14901) 1647 : cluster [DBG] pgmap v1654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:27.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:27 smithi006 bash[33918]: cluster 2023-08-27T22:53:26.218398+0000 mgr.y (mgr.14901) 1647 : cluster [DBG] pgmap v1654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:28.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:28 smithi130 bash[27579]: audit 2023-08-27T22:53:27.838395+0000 mon.c (mon.2) 660 : audit [DBG] from='client.? 172.21.15.6:0/4287264463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:28.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:28 smithi006 bash[33918]: audit 2023-08-27T22:53:27.838395+0000 mon.c (mon.2) 660 : audit [DBG] from='client.? 172.21.15.6:0/4287264463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:28.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:28 smithi006 bash[27292]: audit 2023-08-27T22:53:27.838395+0000 mon.c (mon.2) 660 : audit [DBG] from='client.? 172.21.15.6:0/4287264463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:29.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:29 smithi130 bash[27579]: cluster 2023-08-27T22:53:28.219077+0000 mgr.y (mgr.14901) 1648 : cluster [DBG] pgmap v1655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:29.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:29 smithi006 bash[33918]: cluster 2023-08-27T22:53:28.219077+0000 mgr.y (mgr.14901) 1648 : cluster [DBG] pgmap v1655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:29.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:29 smithi006 bash[27292]: cluster 2023-08-27T22:53:28.219077+0000 mgr.y (mgr.14901) 1648 : cluster [DBG] pgmap v1655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:30.594 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:30 smithi130 bash[27579]: audit 2023-08-27T22:53:30.245992+0000 mon.a (mon.0) 1700 : audit [DBG] from='client.? 172.21.15.6:0/3950025903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:30.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:30 smithi006 bash[33918]: audit 2023-08-27T22:53:30.245992+0000 mon.a (mon.0) 1700 : audit [DBG] from='client.? 172.21.15.6:0/3950025903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:30.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:30 smithi006 bash[27292]: audit 2023-08-27T22:53:30.245992+0000 mon.a (mon.0) 1700 : audit [DBG] from='client.? 172.21.15.6:0/3950025903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:53:30.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:30.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:53:30.320Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:31.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:53:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:53:30] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T22:53:31.558 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:31 smithi130 bash[27579]: cluster 2023-08-27T22:53:30.220140+0000 mgr.y (mgr.14901) 1649 : cluster [DBG] pgmap v1656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:31.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:31 smithi006 bash[27292]: cluster 2023-08-27T22:53:30.220140+0000 mgr.y (mgr.14901) 1649 : cluster [DBG] pgmap v1656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:31.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:31 smithi006 bash[33918]: cluster 2023-08-27T22:53:30.220140+0000 mgr.y (mgr.14901) 1649 : cluster [DBG] pgmap v1656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:53:31.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:32.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:53:31.883Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:33 smithi130 bash[27579]: cluster 2023-08-27T22:53:32.220773+0000 mgr.y (mgr.14901) 1650 : cluster [DBG] pgmap v1657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:33 smithi130 bash[27579]: audit 2023-08-27T22:53:32.412149+0000 mon.a (mon.0) 1701 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:53:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:33 smithi130 bash[27579]: audit 2023-08-27T22:53:32.665909+0000 mon.c (mon.2) 661 : audit [DBG] from='client.? 172.21.15.6:0/617153879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:33 smithi130 bash[27579]: audit 2023-08-27T22:53:32.681641+0000 mon.b (mon.1) 279 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:53:33.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:33 smithi130 bash[27579]: audit 2023-08-27T22:53:32.683193+0000 mon.b (mon.1) 280 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:53:33.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:33 smithi130 bash[27579]: audit 2023-08-27T22:53:32.686491+0000 mon.a (mon.0) 1702 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:53:33.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:33 smithi006 bash[33918]: cluster 2023-08-27T22:53:32.220773+0000 mgr.y (mgr.14901) 1650 : cluster [DBG] pgmap v1657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:33.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:33 smithi006 bash[33918]: audit 2023-08-27T22:53:32.412149+0000 mon.a (mon.0) 1701 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:53:33.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:33 smithi006 bash[33918]: audit 2023-08-27T22:53:32.665909+0000 mon.c (mon.2) 661 : audit [DBG] from='client.? 172.21.15.6:0/617153879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:33.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:33 smithi006 bash[33918]: audit 2023-08-27T22:53:32.681641+0000 mon.b (mon.1) 279 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:53:33.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:33 smithi006 bash[33918]: audit 2023-08-27T22:53:32.683193+0000 mon.b (mon.1) 280 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:53:33.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:33 smithi006 bash[33918]: audit 2023-08-27T22:53:32.686491+0000 mon.a (mon.0) 1702 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:53:33.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:33 smithi006 bash[27292]: cluster 2023-08-27T22:53:32.220773+0000 mgr.y (mgr.14901) 1650 : cluster [DBG] pgmap v1657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:33.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:33 smithi006 bash[27292]: audit 2023-08-27T22:53:32.412149+0000 mon.a (mon.0) 1701 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:53:33.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:33 smithi006 bash[27292]: audit 2023-08-27T22:53:32.665909+0000 mon.c (mon.2) 661 : audit [DBG] from='client.? 172.21.15.6:0/617153879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:33.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:33 smithi006 bash[27292]: audit 2023-08-27T22:53:32.681641+0000 mon.b (mon.1) 279 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:53:33.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:33 smithi006 bash[27292]: audit 2023-08-27T22:53:32.683193+0000 mon.b (mon.1) 280 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:53:33.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:33 smithi006 bash[27292]: audit 2023-08-27T22:53:32.686491+0000 mon.a (mon.0) 1702 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:53:34.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:53:33.777Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:34.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:53:33.779Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:35.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:35 smithi130 bash[27579]: cluster 2023-08-27T22:53:34.221763+0000 mgr.y (mgr.14901) 1651 : cluster [DBG] pgmap v1658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:35.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:35 smithi006 bash[27292]: cluster 2023-08-27T22:53:34.221763+0000 mgr.y (mgr.14901) 1651 : cluster [DBG] pgmap v1658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:35.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:35 smithi006 bash[33918]: cluster 2023-08-27T22:53:34.221763+0000 mgr.y (mgr.14901) 1651 : cluster [DBG] pgmap v1658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:36.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:36 smithi130 bash[27579]: audit 2023-08-27T22:53:35.088424+0000 mon.a (mon.0) 1703 : audit [DBG] from='client.? 172.21.15.6:0/2452446761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:36.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:36 smithi006 bash[33918]: audit 2023-08-27T22:53:35.088424+0000 mon.a (mon.0) 1703 : audit [DBG] from='client.? 172.21.15.6:0/2452446761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:36.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:36 smithi006 bash[27292]: audit 2023-08-27T22:53:35.088424+0000 mon.a (mon.0) 1703 : audit [DBG] from='client.? 172.21.15.6:0/2452446761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:37.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:37 smithi130 bash[27579]: cluster 2023-08-27T22:53:36.222539+0000 mgr.y (mgr.14901) 1652 : cluster [DBG] pgmap v1659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:37.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:37 smithi006 bash[27292]: cluster 2023-08-27T22:53:36.222539+0000 mgr.y (mgr.14901) 1652 : cluster [DBG] pgmap v1659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:37.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:37 smithi006 bash[33918]: cluster 2023-08-27T22:53:36.222539+0000 mgr.y (mgr.14901) 1652 : cluster [DBG] pgmap v1659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:38.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:38 smithi130 bash[27579]: audit 2023-08-27T22:53:37.498779+0000 mon.c (mon.2) 662 : audit [DBG] from='client.? 172.21.15.6:0/1551119541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:38.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:38 smithi006 bash[33918]: audit 2023-08-27T22:53:37.498779+0000 mon.c (mon.2) 662 : audit [DBG] from='client.? 172.21.15.6:0/1551119541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:38.894 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:38 smithi006 bash[27292]: audit 2023-08-27T22:53:37.498779+0000 mon.c (mon.2) 662 : audit [DBG] from='client.? 172.21.15.6:0/1551119541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:39 smithi130 bash[27579]: cluster 2023-08-27T22:53:38.223253+0000 mgr.y (mgr.14901) 1653 : cluster [DBG] pgmap v1660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:39.669 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:39 smithi006 bash[33918]: cluster 2023-08-27T22:53:38.223253+0000 mgr.y (mgr.14901) 1653 : cluster [DBG] pgmap v1660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:39.669 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:39 smithi006 bash[27292]: cluster 2023-08-27T22:53:38.223253+0000 mgr.y (mgr.14901) 1653 : cluster [DBG] pgmap v1660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:40.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:40 smithi130 bash[27579]: audit 2023-08-27T22:53:39.916183+0000 mon.c (mon.2) 663 : audit [DBG] from='client.? 172.21.15.6:0/3647310230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:40.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:40 smithi006 bash[33918]: audit 2023-08-27T22:53:39.916183+0000 mon.c (mon.2) 663 : audit [DBG] from='client.? 172.21.15.6:0/3647310230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:40.479 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:40 smithi006 bash[27292]: audit 2023-08-27T22:53:39.916183+0000 mon.c (mon.2) 663 : audit [DBG] from='client.? 172.21.15.6:0/3647310230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:40.479 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:53:40.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:40.479 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:53:40.322Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:41.111 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:53:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:53:40] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T22:53:41.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:41 smithi130 bash[27579]: cluster 2023-08-27T22:53:40.224402+0000 mgr.y (mgr.14901) 1654 : cluster [DBG] pgmap v1661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:41.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:41 smithi006 bash[27292]: cluster 2023-08-27T22:53:40.224402+0000 mgr.y (mgr.14901) 1654 : cluster [DBG] pgmap v1661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:41.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:41 smithi006 bash[33918]: cluster 2023-08-27T22:53:40.224402+0000 mgr.y (mgr.14901) 1654 : cluster [DBG] pgmap v1661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:53:41.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:42.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:53:41.884Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:43.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:43 smithi130 bash[27579]: cluster 2023-08-27T22:53:42.225035+0000 mgr.y (mgr.14901) 1655 : cluster [DBG] pgmap v1662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:43 smithi130 bash[27579]: audit 2023-08-27T22:53:42.340264+0000 mon.a (mon.0) 1704 : audit [DBG] from='client.? 172.21.15.6:0/93763938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:43.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:43 smithi006 bash[33918]: cluster 2023-08-27T22:53:42.225035+0000 mgr.y (mgr.14901) 1655 : cluster [DBG] pgmap v1662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:43.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:43 smithi006 bash[33918]: audit 2023-08-27T22:53:42.340264+0000 mon.a (mon.0) 1704 : audit [DBG] from='client.? 172.21.15.6:0/93763938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:43.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:43 smithi006 bash[27292]: cluster 2023-08-27T22:53:42.225035+0000 mgr.y (mgr.14901) 1655 : cluster [DBG] pgmap v1662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:43.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:43 smithi006 bash[27292]: audit 2023-08-27T22:53:42.340264+0000 mon.a (mon.0) 1704 : audit [DBG] from='client.? 172.21.15.6:0/93763938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:44.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:53:43.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:44.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:53:43.780Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:45.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:45 smithi130 bash[27579]: cluster 2023-08-27T22:53:44.226119+0000 mgr.y (mgr.14901) 1656 : cluster [DBG] pgmap v1663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:45 smithi130 bash[27579]: audit 2023-08-27T22:53:44.752464+0000 mon.a (mon.0) 1705 : audit [DBG] from='client.? 172.21.15.6:0/4120366185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:45.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:45 smithi006 bash[27292]: cluster 2023-08-27T22:53:44.226119+0000 mgr.y (mgr.14901) 1656 : cluster [DBG] pgmap v1663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:45.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:45 smithi006 bash[27292]: audit 2023-08-27T22:53:44.752464+0000 mon.a (mon.0) 1705 : audit [DBG] from='client.? 172.21.15.6:0/4120366185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:45.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:45 smithi006 bash[33918]: cluster 2023-08-27T22:53:44.226119+0000 mgr.y (mgr.14901) 1656 : cluster [DBG] pgmap v1663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:45.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:45 smithi006 bash[33918]: audit 2023-08-27T22:53:44.752464+0000 mon.a (mon.0) 1705 : audit [DBG] from='client.? 172.21.15.6:0/4120366185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:47.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:47 smithi130 bash[27579]: cluster 2023-08-27T22:53:46.226797+0000 mgr.y (mgr.14901) 1657 : cluster [DBG] pgmap v1664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:47.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:47 smithi130 bash[27579]: audit 2023-08-27T22:53:47.173111+0000 mon.c (mon.2) 664 : audit [DBG] from='client.? 172.21.15.6:0/682673586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:47.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:47 smithi006 bash[33918]: cluster 2023-08-27T22:53:46.226797+0000 mgr.y (mgr.14901) 1657 : cluster [DBG] pgmap v1664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:47.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:47 smithi006 bash[33918]: audit 2023-08-27T22:53:47.173111+0000 mon.c (mon.2) 664 : audit [DBG] from='client.? 172.21.15.6:0/682673586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:47.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:47 smithi006 bash[27292]: cluster 2023-08-27T22:53:46.226797+0000 mgr.y (mgr.14901) 1657 : cluster [DBG] pgmap v1664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:47.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:47 smithi006 bash[27292]: audit 2023-08-27T22:53:47.173111+0000 mon.c (mon.2) 664 : audit [DBG] from='client.? 172.21.15.6:0/682673586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:49.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:49 smithi130 bash[27579]: cluster 2023-08-27T22:53:48.227512+0000 mgr.y (mgr.14901) 1658 : cluster [DBG] pgmap v1665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:49.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:49 smithi006 bash[33918]: cluster 2023-08-27T22:53:48.227512+0000 mgr.y (mgr.14901) 1658 : cluster [DBG] pgmap v1665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:49.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:49 smithi006 bash[27292]: cluster 2023-08-27T22:53:48.227512+0000 mgr.y (mgr.14901) 1658 : cluster [DBG] pgmap v1665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:50.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:50 smithi130 bash[27579]: audit 2023-08-27T22:53:49.592054+0000 mon.c (mon.2) 665 : audit [DBG] from='client.? 172.21.15.6:0/2331957621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:50.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:50 smithi006 bash[33918]: audit 2023-08-27T22:53:49.592054+0000 mon.c (mon.2) 665 : audit [DBG] from='client.? 172.21.15.6:0/2331957621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:50.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:50 smithi006 bash[27292]: audit 2023-08-27T22:53:49.592054+0000 mon.c (mon.2) 665 : audit [DBG] from='client.? 172.21.15.6:0/2331957621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:53:50.321Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:50.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:53:50.323Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:51.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:53:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:53:50] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T22:53:51.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:51 smithi130 bash[27579]: cluster 2023-08-27T22:53:50.228662+0000 mgr.y (mgr.14901) 1659 : cluster [DBG] pgmap v1666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:51.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:51 smithi006 bash[27292]: cluster 2023-08-27T22:53:50.228662+0000 mgr.y (mgr.14901) 1659 : cluster [DBG] pgmap v1666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:51.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:51 smithi006 bash[33918]: cluster 2023-08-27T22:53:50.228662+0000 mgr.y (mgr.14901) 1659 : cluster [DBG] pgmap v1666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:52.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:53:51.883Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:52.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:53:51.885Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:52.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:52 smithi130 bash[27579]: audit 2023-08-27T22:53:52.010459+0000 mon.a (mon.0) 1706 : audit [DBG] from='client.? 172.21.15.6:0/2117145414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:52.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:52 smithi006 bash[33918]: audit 2023-08-27T22:53:52.010459+0000 mon.a (mon.0) 1706 : audit [DBG] from='client.? 172.21.15.6:0/2117145414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:52.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:52 smithi006 bash[27292]: audit 2023-08-27T22:53:52.010459+0000 mon.a (mon.0) 1706 : audit [DBG] from='client.? 172.21.15.6:0/2117145414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:53.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:53 smithi130 bash[27579]: cluster 2023-08-27T22:53:52.229345+0000 mgr.y (mgr.14901) 1660 : cluster [DBG] pgmap v1667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:53.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:53 smithi006 bash[33918]: cluster 2023-08-27T22:53:52.229345+0000 mgr.y (mgr.14901) 1660 : cluster [DBG] pgmap v1667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:53.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:53 smithi006 bash[27292]: cluster 2023-08-27T22:53:52.229345+0000 mgr.y (mgr.14901) 1660 : cluster [DBG] pgmap v1667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:54.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:53:53.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:54.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:53:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:53:53.780Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:53:55.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:55 smithi130 bash[27579]: cluster 2023-08-27T22:53:54.230531+0000 mgr.y (mgr.14901) 1661 : cluster [DBG] pgmap v1668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:55.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:55 smithi130 bash[27579]: audit 2023-08-27T22:53:54.426519+0000 mon.a (mon.0) 1707 : audit [DBG] from='client.? 172.21.15.6:0/2271222233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:55.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:55 smithi006 bash[33918]: cluster 2023-08-27T22:53:54.230531+0000 mgr.y (mgr.14901) 1661 : cluster [DBG] pgmap v1668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:55.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:55 smithi006 bash[33918]: audit 2023-08-27T22:53:54.426519+0000 mon.a (mon.0) 1707 : audit [DBG] from='client.? 172.21.15.6:0/2271222233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:55.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:55 smithi006 bash[27292]: cluster 2023-08-27T22:53:54.230531+0000 mgr.y (mgr.14901) 1661 : cluster [DBG] pgmap v1668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:55.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:55 smithi006 bash[27292]: audit 2023-08-27T22:53:54.426519+0000 mon.a (mon.0) 1707 : audit [DBG] from='client.? 172.21.15.6:0/2271222233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:57.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:57 smithi130 bash[27579]: cluster 2023-08-27T22:53:56.231319+0000 mgr.y (mgr.14901) 1662 : cluster [DBG] pgmap v1669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:57.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:57 smithi130 bash[27579]: audit 2023-08-27T22:53:56.838223+0000 mon.a (mon.0) 1708 : audit [DBG] from='client.? 172.21.15.6:0/2941804349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:57.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:57 smithi006 bash[33918]: cluster 2023-08-27T22:53:56.231319+0000 mgr.y (mgr.14901) 1662 : cluster [DBG] pgmap v1669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:57.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:57 smithi006 bash[33918]: audit 2023-08-27T22:53:56.838223+0000 mon.a (mon.0) 1708 : audit [DBG] from='client.? 172.21.15.6:0/2941804349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:57.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:57 smithi006 bash[27292]: cluster 2023-08-27T22:53:56.231319+0000 mgr.y (mgr.14901) 1662 : cluster [DBG] pgmap v1669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:57.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:57 smithi006 bash[27292]: audit 2023-08-27T22:53:56.838223+0000 mon.a (mon.0) 1708 : audit [DBG] from='client.? 172.21.15.6:0/2941804349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:59.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:59 smithi130 bash[27579]: cluster 2023-08-27T22:53:58.232130+0000 mgr.y (mgr.14901) 1663 : cluster [DBG] pgmap v1670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:53:59 smithi130 bash[27579]: audit 2023-08-27T22:53:59.252094+0000 mon.a (mon.0) 1709 : audit [DBG] from='client.? 172.21.15.6:0/3829989707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:59.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:59 smithi006 bash[27292]: cluster 2023-08-27T22:53:58.232130+0000 mgr.y (mgr.14901) 1663 : cluster [DBG] pgmap v1670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:59.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:53:59 smithi006 bash[27292]: audit 2023-08-27T22:53:59.252094+0000 mon.a (mon.0) 1709 : audit [DBG] from='client.? 172.21.15.6:0/3829989707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:59.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:59 smithi006 bash[33918]: cluster 2023-08-27T22:53:58.232130+0000 mgr.y (mgr.14901) 1663 : cluster [DBG] pgmap v1670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:53:59.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:53:59 smithi006 bash[33918]: audit 2023-08-27T22:53:59.252094+0000 mon.a (mon.0) 1709 : audit [DBG] from='client.? 172.21.15.6:0/3829989707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:00.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:54:00.321Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:00.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:54:00.324Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:01.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:54:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:54:00] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T22:54:01.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:01 smithi130 bash[27579]: cluster 2023-08-27T22:54:00.233403+0000 mgr.y (mgr.14901) 1664 : cluster [DBG] pgmap v1671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:01.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:01 smithi006 bash[33918]: cluster 2023-08-27T22:54:00.233403+0000 mgr.y (mgr.14901) 1664 : cluster [DBG] pgmap v1671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:01.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:01 smithi006 bash[27292]: cluster 2023-08-27T22:54:00.233403+0000 mgr.y (mgr.14901) 1664 : cluster [DBG] pgmap v1671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:02.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:54:01.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:02.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:54:01.886Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:02.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:02 smithi130 bash[27579]: audit 2023-08-27T22:54:01.666096+0000 mon.a (mon.0) 1710 : audit [DBG] from='client.? 172.21.15.6:0/1201093940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:02.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:02 smithi006 bash[33918]: audit 2023-08-27T22:54:01.666096+0000 mon.a (mon.0) 1710 : audit [DBG] from='client.? 172.21.15.6:0/1201093940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:02.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:02 smithi006 bash[27292]: audit 2023-08-27T22:54:01.666096+0000 mon.a (mon.0) 1710 : audit [DBG] from='client.? 172.21.15.6:0/1201093940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:03.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:03 smithi130 bash[27579]: cluster 2023-08-27T22:54:02.234095+0000 mgr.y (mgr.14901) 1665 : cluster [DBG] pgmap v1672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:03.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:03 smithi006 bash[27292]: cluster 2023-08-27T22:54:02.234095+0000 mgr.y (mgr.14901) 1665 : cluster [DBG] pgmap v1672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:03.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:03 smithi006 bash[33918]: cluster 2023-08-27T22:54:02.234095+0000 mgr.y (mgr.14901) 1665 : cluster [DBG] pgmap v1672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:04.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:54:03.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:04.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:54:03.781Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:05 smithi130 bash[27579]: audit 2023-08-27T22:54:04.086685+0000 mon.a (mon.0) 1711 : audit [DBG] from='client.? 172.21.15.6:0/2836157018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:05 smithi130 bash[27579]: cluster 2023-08-27T22:54:04.235204+0000 mgr.y (mgr.14901) 1666 : cluster [DBG] pgmap v1673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:05.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:05 smithi006 bash[33918]: audit 2023-08-27T22:54:04.086685+0000 mon.a (mon.0) 1711 : audit [DBG] from='client.? 172.21.15.6:0/2836157018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:05.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:05 smithi006 bash[33918]: cluster 2023-08-27T22:54:04.235204+0000 mgr.y (mgr.14901) 1666 : cluster [DBG] pgmap v1673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:05.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:05 smithi006 bash[27292]: audit 2023-08-27T22:54:04.086685+0000 mon.a (mon.0) 1711 : audit [DBG] from='client.? 172.21.15.6:0/2836157018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:05.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:05 smithi006 bash[27292]: cluster 2023-08-27T22:54:04.235204+0000 mgr.y (mgr.14901) 1666 : cluster [DBG] pgmap v1673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:07.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:07 smithi130 bash[27579]: cluster 2023-08-27T22:54:06.235662+0000 mgr.y (mgr.14901) 1667 : cluster [DBG] pgmap v1674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:07.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:07 smithi130 bash[27579]: audit 2023-08-27T22:54:06.501154+0000 mon.a (mon.0) 1712 : audit [DBG] from='client.? 172.21.15.6:0/3923735473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:07.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:07 smithi006 bash[33918]: cluster 2023-08-27T22:54:06.235662+0000 mgr.y (mgr.14901) 1667 : cluster [DBG] pgmap v1674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:08.044 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:07 smithi006 bash[33918]: audit 2023-08-27T22:54:06.501154+0000 mon.a (mon.0) 1712 : audit [DBG] from='client.? 172.21.15.6:0/3923735473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:08.044 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:07 smithi006 bash[27292]: cluster 2023-08-27T22:54:06.235662+0000 mgr.y (mgr.14901) 1667 : cluster [DBG] pgmap v1674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:08.044 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:07 smithi006 bash[27292]: audit 2023-08-27T22:54:06.501154+0000 mon.a (mon.0) 1712 : audit [DBG] from='client.? 172.21.15.6:0/3923735473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:09.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:09 smithi130 bash[27579]: cluster 2023-08-27T22:54:08.236372+0000 mgr.y (mgr.14901) 1668 : cluster [DBG] pgmap v1675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:09 smithi130 bash[27579]: audit 2023-08-27T22:54:08.912351+0000 mon.c (mon.2) 666 : audit [DBG] from='client.? 172.21.15.6:0/2270371708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:09.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:09 smithi006 bash[33918]: cluster 2023-08-27T22:54:08.236372+0000 mgr.y (mgr.14901) 1668 : cluster [DBG] pgmap v1675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:09.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:09 smithi006 bash[33918]: audit 2023-08-27T22:54:08.912351+0000 mon.c (mon.2) 666 : audit [DBG] from='client.? 172.21.15.6:0/2270371708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:09.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:09 smithi006 bash[27292]: cluster 2023-08-27T22:54:08.236372+0000 mgr.y (mgr.14901) 1668 : cluster [DBG] pgmap v1675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:09.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:09 smithi006 bash[27292]: audit 2023-08-27T22:54:08.912351+0000 mon.c (mon.2) 666 : audit [DBG] from='client.? 172.21.15.6:0/2270371708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:10.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:54:10.321Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:10.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:54:10.323Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:11.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:54:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:54:10] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T22:54:11.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:11 smithi130 bash[27579]: cluster 2023-08-27T22:54:10.237556+0000 mgr.y (mgr.14901) 1669 : cluster [DBG] pgmap v1676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:11.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:11 smithi006 bash[33918]: cluster 2023-08-27T22:54:10.237556+0000 mgr.y (mgr.14901) 1669 : cluster [DBG] pgmap v1676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:11.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:11 smithi006 bash[27292]: cluster 2023-08-27T22:54:10.237556+0000 mgr.y (mgr.14901) 1669 : cluster [DBG] pgmap v1676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:12.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:54:11.885Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:12.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:54:11.887Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:12.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:12 smithi130 bash[27579]: audit 2023-08-27T22:54:11.320270+0000 mon.c (mon.2) 667 : audit [DBG] from='client.? 172.21.15.6:0/613252641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:12.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:12 smithi006 bash[33918]: audit 2023-08-27T22:54:11.320270+0000 mon.c (mon.2) 667 : audit [DBG] from='client.? 172.21.15.6:0/613252641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:12.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:12 smithi006 bash[27292]: audit 2023-08-27T22:54:11.320270+0000 mon.c (mon.2) 667 : audit [DBG] from='client.? 172.21.15.6:0/613252641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:13 smithi130 bash[27579]: cluster 2023-08-27T22:54:12.238227+0000 mgr.y (mgr.14901) 1670 : cluster [DBG] pgmap v1677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:13.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:13 smithi006 bash[27292]: cluster 2023-08-27T22:54:12.238227+0000 mgr.y (mgr.14901) 1670 : cluster [DBG] pgmap v1677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:13.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:13 smithi006 bash[33918]: cluster 2023-08-27T22:54:12.238227+0000 mgr.y (mgr.14901) 1670 : cluster [DBG] pgmap v1677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:14.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:54:13.780Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:14.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:54:13.782Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:14.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:14 smithi130 bash[27579]: audit 2023-08-27T22:54:13.733122+0000 mon.a (mon.0) 1713 : audit [DBG] from='client.? 172.21.15.6:0/1749965353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:14.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:14 smithi006 bash[27292]: audit 2023-08-27T22:54:13.733122+0000 mon.a (mon.0) 1713 : audit [DBG] from='client.? 172.21.15.6:0/1749965353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:14.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:14 smithi006 bash[33918]: audit 2023-08-27T22:54:13.733122+0000 mon.a (mon.0) 1713 : audit [DBG] from='client.? 172.21.15.6:0/1749965353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:15.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:15 smithi130 bash[27579]: cluster 2023-08-27T22:54:14.239466+0000 mgr.y (mgr.14901) 1671 : cluster [DBG] pgmap v1678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:15.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:15 smithi006 bash[33918]: cluster 2023-08-27T22:54:14.239466+0000 mgr.y (mgr.14901) 1671 : cluster [DBG] pgmap v1678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:15.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:15 smithi006 bash[27292]: cluster 2023-08-27T22:54:14.239466+0000 mgr.y (mgr.14901) 1671 : cluster [DBG] pgmap v1678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:16 smithi130 bash[27579]: audit 2023-08-27T22:54:16.144682+0000 mon.a (mon.0) 1714 : audit [DBG] from='client.? 172.21.15.6:0/541540393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:16.837 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:16 smithi006 bash[33918]: audit 2023-08-27T22:54:16.144682+0000 mon.a (mon.0) 1714 : audit [DBG] from='client.? 172.21.15.6:0/541540393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:16.837 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:16 smithi006 bash[27292]: audit 2023-08-27T22:54:16.144682+0000 mon.a (mon.0) 1714 : audit [DBG] from='client.? 172.21.15.6:0/541540393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:17.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:17 smithi130 bash[27579]: cluster 2023-08-27T22:54:16.240181+0000 mgr.y (mgr.14901) 1672 : cluster [DBG] pgmap v1679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:20.503 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:17 smithi006 bash[27292]: cluster 2023-08-27T22:54:16.240181+0000 mgr.y (mgr.14901) 1672 : cluster [DBG] pgmap v1679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:20.504 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:19 smithi130 bash[27579]: cluster 2023-08-27T22:54:18.240805+0000 mgr.y (mgr.14901) 1673 : cluster [DBG] pgmap v1680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:20.504 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:19 smithi130 bash[27579]: audit 2023-08-27T22:54:18.554212+0000 mon.a (mon.0) 1715 : audit [DBG] from='client.? 172.21.15.6:0/1797545803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:20.504 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:17 smithi006 bash[33918]: cluster 2023-08-27T22:54:16.240181+0000 mgr.y (mgr.14901) 1672 : cluster [DBG] pgmap v1679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:20.504 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:19 smithi006 bash[33918]: cluster 2023-08-27T22:54:18.240805+0000 mgr.y (mgr.14901) 1673 : cluster [DBG] pgmap v1680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:20.504 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:19 smithi006 bash[33918]: audit 2023-08-27T22:54:18.554212+0000 mon.a (mon.0) 1715 : audit [DBG] from='client.? 172.21.15.6:0/1797545803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:20.505 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:19 smithi006 bash[27292]: cluster 2023-08-27T22:54:18.240805+0000 mgr.y (mgr.14901) 1673 : cluster [DBG] pgmap v1680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:20.505 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:19 smithi006 bash[27292]: audit 2023-08-27T22:54:18.554212+0000 mon.a (mon.0) 1715 : audit [DBG] from='client.? 172.21.15.6:0/1797545803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:20.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:54:20.322Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:21.025 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:54:20.325Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:21.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:54:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:54:20] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T22:54:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:21 smithi130 bash[27579]: cluster 2023-08-27T22:54:20.241888+0000 mgr.y (mgr.14901) 1674 : cluster [DBG] pgmap v1681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:21.995 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:21 smithi130 bash[27579]: audit 2023-08-27T22:54:20.978724+0000 mon.a (mon.0) 1716 : audit [DBG] from='client.? 172.21.15.6:0/1382092776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:21.995 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:21 smithi006 bash[27292]: cluster 2023-08-27T22:54:20.241888+0000 mgr.y (mgr.14901) 1674 : cluster [DBG] pgmap v1681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:21.996 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:21 smithi006 bash[27292]: audit 2023-08-27T22:54:20.978724+0000 mon.a (mon.0) 1716 : audit [DBG] from='client.? 172.21.15.6:0/1382092776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:21.999 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:21 smithi006 bash[33918]: cluster 2023-08-27T22:54:20.241888+0000 mgr.y (mgr.14901) 1674 : cluster [DBG] pgmap v1681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:21.999 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:21 smithi006 bash[33918]: audit 2023-08-27T22:54:20.978724+0000 mon.a (mon.0) 1716 : audit [DBG] from='client.? 172.21.15.6:0/1382092776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:22.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:54:21.885Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:22.266 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:54:21.887Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:23.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:23 smithi130 bash[27579]: cluster 2023-08-27T22:54:22.242563+0000 mgr.y (mgr.14901) 1675 : cluster [DBG] pgmap v1682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:23.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:23 smithi006 bash[33918]: cluster 2023-08-27T22:54:22.242563+0000 mgr.y (mgr.14901) 1675 : cluster [DBG] pgmap v1682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:23.768 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:23 smithi006 bash[27292]: cluster 2023-08-27T22:54:22.242563+0000 mgr.y (mgr.14901) 1675 : cluster [DBG] pgmap v1682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:24.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:54:23.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:24.575 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:54:23.783Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:24.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:24 smithi130 bash[27579]: audit 2023-08-27T22:54:23.390747+0000 mon.a (mon.0) 1717 : audit [DBG] from='client.? 172.21.15.6:0/2790691113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:24.863 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:24 smithi006 bash[33918]: audit 2023-08-27T22:54:23.390747+0000 mon.a (mon.0) 1717 : audit [DBG] from='client.? 172.21.15.6:0/2790691113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:24.863 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:24 smithi006 bash[27292]: audit 2023-08-27T22:54:23.390747+0000 mon.a (mon.0) 1717 : audit [DBG] from='client.? 172.21.15.6:0/2790691113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:25.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:25 smithi006 bash[33918]: cluster 2023-08-27T22:54:24.243643+0000 mgr.y (mgr.14901) 1676 : cluster [DBG] pgmap v1683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:25.926 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:25 smithi006 bash[27292]: cluster 2023-08-27T22:54:24.243643+0000 mgr.y (mgr.14901) 1676 : cluster [DBG] pgmap v1683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:25.927 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:25 smithi130 bash[27579]: cluster 2023-08-27T22:54:24.243643+0000 mgr.y (mgr.14901) 1676 : cluster [DBG] pgmap v1683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:26.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:26 smithi006 bash[33918]: audit 2023-08-27T22:54:25.798454+0000 mon.c (mon.2) 668 : audit [DBG] from='client.? 172.21.15.6:0/456057980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:26.731 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:26 smithi006 bash[27292]: audit 2023-08-27T22:54:25.798454+0000 mon.c (mon.2) 668 : audit [DBG] from='client.? 172.21.15.6:0/456057980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:26.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:26 smithi130 bash[27579]: audit 2023-08-27T22:54:25.798454+0000 mon.c (mon.2) 668 : audit [DBG] from='client.? 172.21.15.6:0/456057980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:27.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:27 smithi006 bash[33918]: cluster 2023-08-27T22:54:26.244301+0000 mgr.y (mgr.14901) 1677 : cluster [DBG] pgmap v1684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:27.827 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:27 smithi006 bash[27292]: cluster 2023-08-27T22:54:26.244301+0000 mgr.y (mgr.14901) 1677 : cluster [DBG] pgmap v1684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:27.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:27 smithi130 bash[27579]: cluster 2023-08-27T22:54:26.244301+0000 mgr.y (mgr.14901) 1677 : cluster [DBG] pgmap v1684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:28.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:28 smithi006 bash[33918]: audit 2023-08-27T22:54:28.210108+0000 mon.a (mon.0) 1718 : audit [DBG] from='client.? 172.21.15.6:0/1422490998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:28.835 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:28 smithi006 bash[27292]: audit 2023-08-27T22:54:28.210108+0000 mon.a (mon.0) 1718 : audit [DBG] from='client.? 172.21.15.6:0/1422490998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:28 smithi130 bash[27579]: audit 2023-08-27T22:54:28.210108+0000 mon.a (mon.0) 1718 : audit [DBG] from='client.? 172.21.15.6:0/1422490998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:29.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:29 smithi006 bash[33918]: cluster 2023-08-27T22:54:28.245009+0000 mgr.y (mgr.14901) 1678 : cluster [DBG] pgmap v1685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:29.906 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:29 smithi006 bash[27292]: cluster 2023-08-27T22:54:28.245009+0000 mgr.y (mgr.14901) 1678 : cluster [DBG] pgmap v1685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:29.907 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:29 smithi130 bash[27579]: cluster 2023-08-27T22:54:28.245009+0000 mgr.y (mgr.14901) 1678 : cluster [DBG] pgmap v1685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:30.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:54:30.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:30.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:54:30.325Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:31.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:54:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:54:30] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T22:54:31.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:31 smithi006 bash[33918]: cluster 2023-08-27T22:54:30.246152+0000 mgr.y (mgr.14901) 1679 : cluster [DBG] pgmap v1686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:31.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:31 smithi006 bash[33918]: audit 2023-08-27T22:54:30.632257+0000 mon.c (mon.2) 669 : audit [DBG] from='client.? 172.21.15.6:0/2284803775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:31.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:31 smithi006 bash[27292]: cluster 2023-08-27T22:54:30.246152+0000 mgr.y (mgr.14901) 1679 : cluster [DBG] pgmap v1686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:31.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:31 smithi006 bash[27292]: audit 2023-08-27T22:54:30.632257+0000 mon.c (mon.2) 669 : audit [DBG] from='client.? 172.21.15.6:0/2284803775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:31.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:31 smithi130 bash[27579]: cluster 2023-08-27T22:54:30.246152+0000 mgr.y (mgr.14901) 1679 : cluster [DBG] pgmap v1686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:31.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:31 smithi130 bash[27579]: audit 2023-08-27T22:54:30.632257+0000 mon.c (mon.2) 669 : audit [DBG] from='client.? 172.21.15.6:0/2284803775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:32.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:54:31.886Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:32.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:54:31.888Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:32.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:32 smithi006 bash[27292]: cluster 2023-08-27T22:54:32.246932+0000 mgr.y (mgr.14901) 1680 : cluster [DBG] pgmap v1687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:32.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:32 smithi006 bash[33918]: cluster 2023-08-27T22:54:32.246932+0000 mgr.y (mgr.14901) 1680 : cluster [DBG] pgmap v1687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:32 smithi130 bash[27579]: cluster 2023-08-27T22:54:32.246932+0000 mgr.y (mgr.14901) 1680 : cluster [DBG] pgmap v1687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:33.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:33 smithi006 bash[27292]: audit 2023-08-27T22:54:32.701294+0000 mon.b (mon.1) 281 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:54:33.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:33 smithi006 bash[27292]: audit 2023-08-27T22:54:33.051927+0000 mon.a (mon.0) 1719 : audit [DBG] from='client.? 172.21.15.6:0/3271148643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:33.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:33 smithi006 bash[33918]: audit 2023-08-27T22:54:32.701294+0000 mon.b (mon.1) 281 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:54:33.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:33 smithi006 bash[33918]: audit 2023-08-27T22:54:33.051927+0000 mon.a (mon.0) 1719 : audit [DBG] from='client.? 172.21.15.6:0/3271148643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:33 smithi130 bash[27579]: audit 2023-08-27T22:54:32.701294+0000 mon.b (mon.1) 281 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:54:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:33 smithi130 bash[27579]: audit 2023-08-27T22:54:33.051927+0000 mon.a (mon.0) 1719 : audit [DBG] from='client.? 172.21.15.6:0/3271148643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:34.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:54:33.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:34.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:54:33.784Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:35.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:35 smithi130 bash[27579]: cluster 2023-08-27T22:54:34.248086+0000 mgr.y (mgr.14901) 1681 : cluster [DBG] pgmap v1688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:35.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:35 smithi006 bash[27292]: cluster 2023-08-27T22:54:34.248086+0000 mgr.y (mgr.14901) 1681 : cluster [DBG] pgmap v1688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:35.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:35 smithi006 bash[33918]: cluster 2023-08-27T22:54:34.248086+0000 mgr.y (mgr.14901) 1681 : cluster [DBG] pgmap v1688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:36.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:36 smithi130 bash[27579]: audit 2023-08-27T22:54:35.477447+0000 mon.c (mon.2) 670 : audit [DBG] from='client.? 172.21.15.6:0/1564300899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:36.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:36 smithi006 bash[27292]: audit 2023-08-27T22:54:35.477447+0000 mon.c (mon.2) 670 : audit [DBG] from='client.? 172.21.15.6:0/1564300899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:36.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:36 smithi006 bash[33918]: audit 2023-08-27T22:54:35.477447+0000 mon.c (mon.2) 670 : audit [DBG] from='client.? 172.21.15.6:0/1564300899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:37.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:37 smithi130 bash[27579]: cluster 2023-08-27T22:54:36.248778+0000 mgr.y (mgr.14901) 1682 : cluster [DBG] pgmap v1689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:37.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:37 smithi006 bash[27292]: cluster 2023-08-27T22:54:36.248778+0000 mgr.y (mgr.14901) 1682 : cluster [DBG] pgmap v1689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:37.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:37 smithi006 bash[33918]: cluster 2023-08-27T22:54:36.248778+0000 mgr.y (mgr.14901) 1682 : cluster [DBG] pgmap v1689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:38.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:38 smithi130 bash[27579]: audit 2023-08-27T22:54:37.908541+0000 mon.c (mon.2) 671 : audit [DBG] from='client.? 172.21.15.6:0/2644778405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:38.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:38 smithi006 bash[27292]: audit 2023-08-27T22:54:37.908541+0000 mon.c (mon.2) 671 : audit [DBG] from='client.? 172.21.15.6:0/2644778405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:38.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:38 smithi006 bash[33918]: audit 2023-08-27T22:54:37.908541+0000 mon.c (mon.2) 671 : audit [DBG] from='client.? 172.21.15.6:0/2644778405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:39.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:39 smithi130 bash[27579]: cluster 2023-08-27T22:54:38.249270+0000 mgr.y (mgr.14901) 1683 : cluster [DBG] pgmap v1690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:39.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:39 smithi130 bash[27579]: audit 2023-08-27T22:54:38.565169+0000 mon.a (mon.0) 1720 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:54:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:39 smithi130 bash[27579]: audit 2023-08-27T22:54:39.038933+0000 mon.a (mon.0) 1721 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:54:39.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:39 smithi006 bash[27292]: cluster 2023-08-27T22:54:38.249270+0000 mgr.y (mgr.14901) 1683 : cluster [DBG] pgmap v1690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:39.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:39 smithi006 bash[27292]: audit 2023-08-27T22:54:38.565169+0000 mon.a (mon.0) 1720 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:54:39.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:39 smithi006 bash[27292]: audit 2023-08-27T22:54:39.038933+0000 mon.a (mon.0) 1721 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:54:39.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:39 smithi006 bash[33918]: cluster 2023-08-27T22:54:38.249270+0000 mgr.y (mgr.14901) 1683 : cluster [DBG] pgmap v1690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:39.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:39 smithi006 bash[33918]: audit 2023-08-27T22:54:38.565169+0000 mon.a (mon.0) 1720 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:54:39.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:39 smithi006 bash[33918]: audit 2023-08-27T22:54:39.038933+0000 mon.a (mon.0) 1721 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:54:40.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:54:40.324Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:40.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:54:40.326Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:40.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:40 smithi006 bash[33918]: audit 2023-08-27T22:54:39.312891+0000 mon.b (mon.1) 282 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:54:40.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:40 smithi006 bash[33918]: audit 2023-08-27T22:54:39.315159+0000 mon.b (mon.1) 283 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:54:40.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:40 smithi006 bash[33918]: audit 2023-08-27T22:54:39.320565+0000 mon.a (mon.0) 1722 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:54:40.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:40 smithi006 bash[27292]: audit 2023-08-27T22:54:39.312891+0000 mon.b (mon.1) 282 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:54:40.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:40 smithi006 bash[27292]: audit 2023-08-27T22:54:39.315159+0000 mon.b (mon.1) 283 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:54:40.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:40 smithi006 bash[27292]: audit 2023-08-27T22:54:39.320565+0000 mon.a (mon.0) 1722 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:54:40.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:40 smithi130 bash[27579]: audit 2023-08-27T22:54:39.312891+0000 mon.b (mon.1) 282 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:54:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:40 smithi130 bash[27579]: audit 2023-08-27T22:54:39.315159+0000 mon.b (mon.1) 283 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:54:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:40 smithi130 bash[27579]: audit 2023-08-27T22:54:39.320565+0000 mon.a (mon.0) 1722 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:54:41.126 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:54:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:54:40] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T22:54:41.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:41 smithi006 bash[33918]: cluster 2023-08-27T22:54:40.250330+0000 mgr.y (mgr.14901) 1684 : cluster [DBG] pgmap v1691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:41.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:41 smithi006 bash[33918]: audit 2023-08-27T22:54:40.328730+0000 mon.a (mon.0) 1723 : audit [DBG] from='client.? 172.21.15.6:0/3799257617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:41.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:41 smithi006 bash[27292]: cluster 2023-08-27T22:54:40.250330+0000 mgr.y (mgr.14901) 1684 : cluster [DBG] pgmap v1691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:41.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:41 smithi006 bash[27292]: audit 2023-08-27T22:54:40.328730+0000 mon.a (mon.0) 1723 : audit [DBG] from='client.? 172.21.15.6:0/3799257617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:41.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:41 smithi130 bash[27579]: cluster 2023-08-27T22:54:40.250330+0000 mgr.y (mgr.14901) 1684 : cluster [DBG] pgmap v1691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:41.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:41 smithi130 bash[27579]: audit 2023-08-27T22:54:40.328730+0000 mon.a (mon.0) 1723 : audit [DBG] from='client.? 172.21.15.6:0/3799257617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:42.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:54:41.886Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:43.028 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:54:41.888Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:43 smithi130 bash[27579]: cluster 2023-08-27T22:54:42.250985+0000 mgr.y (mgr.14901) 1685 : cluster [DBG] pgmap v1692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:43.852 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:43 smithi130 bash[27579]: audit 2023-08-27T22:54:42.739368+0000 mon.a (mon.0) 1724 : audit [DBG] from='client.? 172.21.15.6:0/2131673533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:43.853 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:43 smithi006 bash[27292]: cluster 2023-08-27T22:54:42.250985+0000 mgr.y (mgr.14901) 1685 : cluster [DBG] pgmap v1692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:43.853 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:43 smithi006 bash[27292]: audit 2023-08-27T22:54:42.739368+0000 mon.a (mon.0) 1724 : audit [DBG] from='client.? 172.21.15.6:0/2131673533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:43.853 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:43 smithi006 bash[33918]: cluster 2023-08-27T22:54:42.250985+0000 mgr.y (mgr.14901) 1685 : cluster [DBG] pgmap v1692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:43.853 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:43 smithi006 bash[33918]: audit 2023-08-27T22:54:42.739368+0000 mon.a (mon.0) 1724 : audit [DBG] from='client.? 172.21.15.6:0/2131673533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:44.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:54:43.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:44.259 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:54:43.785Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:45.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:45 smithi130 bash[27579]: cluster 2023-08-27T22:54:44.252062+0000 mgr.y (mgr.14901) 1686 : cluster [DBG] pgmap v1693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:45.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:45 smithi006 bash[27292]: cluster 2023-08-27T22:54:44.252062+0000 mgr.y (mgr.14901) 1686 : cluster [DBG] pgmap v1693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:45.696 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:45 smithi006 bash[33918]: cluster 2023-08-27T22:54:44.252062+0000 mgr.y (mgr.14901) 1686 : cluster [DBG] pgmap v1693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:46.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:46 smithi130 bash[27579]: audit 2023-08-27T22:54:45.155349+0000 mon.a (mon.0) 1725 : audit [DBG] from='client.? 172.21.15.6:0/3400241109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:46.772 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:46 smithi006 bash[27292]: audit 2023-08-27T22:54:45.155349+0000 mon.a (mon.0) 1725 : audit [DBG] from='client.? 172.21.15.6:0/3400241109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:46.772 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:46 smithi006 bash[33918]: audit 2023-08-27T22:54:45.155349+0000 mon.a (mon.0) 1725 : audit [DBG] from='client.? 172.21.15.6:0/3400241109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:47.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:47 smithi130 bash[27579]: cluster 2023-08-27T22:54:46.252713+0000 mgr.y (mgr.14901) 1687 : cluster [DBG] pgmap v1694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:47.610 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:47 smithi006 bash[27292]: cluster 2023-08-27T22:54:46.252713+0000 mgr.y (mgr.14901) 1687 : cluster [DBG] pgmap v1694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:47.610 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:47 smithi006 bash[33918]: cluster 2023-08-27T22:54:46.252713+0000 mgr.y (mgr.14901) 1687 : cluster [DBG] pgmap v1694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:48.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:48 smithi130 bash[27579]: audit 2023-08-27T22:54:47.571561+0000 mon.c (mon.2) 672 : audit [DBG] from='client.? 172.21.15.6:0/251818581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:48.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:48 smithi006 bash[27292]: audit 2023-08-27T22:54:47.571561+0000 mon.c (mon.2) 672 : audit [DBG] from='client.? 172.21.15.6:0/251818581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:48.540 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:48 smithi006 bash[33918]: audit 2023-08-27T22:54:47.571561+0000 mon.c (mon.2) 672 : audit [DBG] from='client.? 172.21.15.6:0/251818581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:49.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:49 smithi006 bash[27292]: cluster 2023-08-27T22:54:48.253399+0000 mgr.y (mgr.14901) 1688 : cluster [DBG] pgmap v1695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:49.528 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:49 smithi006 bash[33918]: cluster 2023-08-27T22:54:48.253399+0000 mgr.y (mgr.14901) 1688 : cluster [DBG] pgmap v1695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:49.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:49 smithi130 bash[27579]: cluster 2023-08-27T22:54:48.253399+0000 mgr.y (mgr.14901) 1688 : cluster [DBG] pgmap v1695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:50.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:50 smithi006 bash[27292]: audit 2023-08-27T22:54:49.986068+0000 mon.a (mon.0) 1726 : audit [DBG] from='client.? 172.21.15.6:0/1590358911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:50.504 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:50 smithi006 bash[33918]: audit 2023-08-27T22:54:49.986068+0000 mon.a (mon.0) 1726 : audit [DBG] from='client.? 172.21.15.6:0/1590358911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:50.504 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:54:50.324Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:50.504 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:54:50.327Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:50 smithi130 bash[27579]: audit 2023-08-27T22:54:49.986068+0000 mon.a (mon.0) 1726 : audit [DBG] from='client.? 172.21.15.6:0/1590358911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:51.130 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:54:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:54:50] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T22:54:51.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:51 smithi006 bash[33918]: cluster 2023-08-27T22:54:50.254575+0000 mgr.y (mgr.14901) 1689 : cluster [DBG] pgmap v1696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:51.476 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:51 smithi006 bash[27292]: cluster 2023-08-27T22:54:50.254575+0000 mgr.y (mgr.14901) 1689 : cluster [DBG] pgmap v1696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:51.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:51 smithi130 bash[27579]: cluster 2023-08-27T22:54:50.254575+0000 mgr.y (mgr.14901) 1689 : cluster [DBG] pgmap v1696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:52.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:54:51.886Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:52.292 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:54:51.889Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:53.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:53 smithi130 bash[27579]: cluster 2023-08-27T22:54:52.255204+0000 mgr.y (mgr.14901) 1690 : cluster [DBG] pgmap v1697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:53 smithi130 bash[27579]: audit 2023-08-27T22:54:52.400133+0000 mon.a (mon.0) 1727 : audit [DBG] from='client.? 172.21.15.6:0/3775591332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:53.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:53 smithi006 bash[33918]: cluster 2023-08-27T22:54:52.255204+0000 mgr.y (mgr.14901) 1690 : cluster [DBG] pgmap v1697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:53.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:53 smithi006 bash[33918]: audit 2023-08-27T22:54:52.400133+0000 mon.a (mon.0) 1727 : audit [DBG] from='client.? 172.21.15.6:0/3775591332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:53.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:53 smithi006 bash[27292]: cluster 2023-08-27T22:54:52.255204+0000 mgr.y (mgr.14901) 1690 : cluster [DBG] pgmap v1697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:53.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:53 smithi006 bash[27292]: audit 2023-08-27T22:54:52.400133+0000 mon.a (mon.0) 1727 : audit [DBG] from='client.? 172.21.15.6:0/3775591332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:54.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:54:53.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:54.225 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:54:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:54:53.785Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:54:55.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:55 smithi130 bash[27579]: cluster 2023-08-27T22:54:54.256427+0000 mgr.y (mgr.14901) 1691 : cluster [DBG] pgmap v1698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:55.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:55 smithi130 bash[27579]: audit 2023-08-27T22:54:54.819090+0000 mon.c (mon.2) 673 : audit [DBG] from='client.? 172.21.15.6:0/4293046685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:55.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:55 smithi006 bash[27292]: cluster 2023-08-27T22:54:54.256427+0000 mgr.y (mgr.14901) 1691 : cluster [DBG] pgmap v1698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:55.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:55 smithi006 bash[27292]: audit 2023-08-27T22:54:54.819090+0000 mon.c (mon.2) 673 : audit [DBG] from='client.? 172.21.15.6:0/4293046685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:55.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:55 smithi006 bash[33918]: cluster 2023-08-27T22:54:54.256427+0000 mgr.y (mgr.14901) 1691 : cluster [DBG] pgmap v1698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:55.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:55 smithi006 bash[33918]: audit 2023-08-27T22:54:54.819090+0000 mon.c (mon.2) 673 : audit [DBG] from='client.? 172.21.15.6:0/4293046685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:57.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:57 smithi130 bash[27579]: cluster 2023-08-27T22:54:56.257245+0000 mgr.y (mgr.14901) 1692 : cluster [DBG] pgmap v1699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:57.998 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:57 smithi130 bash[27579]: audit 2023-08-27T22:54:57.242199+0000 mon.a (mon.0) 1728 : audit [DBG] from='client.? 172.21.15.6:0/866487733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:57.999 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:57 smithi006 bash[27292]: cluster 2023-08-27T22:54:56.257245+0000 mgr.y (mgr.14901) 1692 : cluster [DBG] pgmap v1699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:57.999 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:57 smithi006 bash[27292]: audit 2023-08-27T22:54:57.242199+0000 mon.a (mon.0) 1728 : audit [DBG] from='client.? 172.21.15.6:0/866487733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:57.999 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:57 smithi006 bash[33918]: cluster 2023-08-27T22:54:56.257245+0000 mgr.y (mgr.14901) 1692 : cluster [DBG] pgmap v1699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:57.999 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:57 smithi006 bash[33918]: audit 2023-08-27T22:54:57.242199+0000 mon.a (mon.0) 1728 : audit [DBG] from='client.? 172.21.15.6:0/866487733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:59.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:54:59 smithi130 bash[27579]: cluster 2023-08-27T22:54:58.257921+0000 mgr.y (mgr.14901) 1693 : cluster [DBG] pgmap v1700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:54:59.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:54:59 smithi006 bash[33918]: cluster 2023-08-27T22:54:58.257921+0000 mgr.y (mgr.14901) 1693 : cluster [DBG] pgmap v1700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:00.001 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:54:59 smithi006 bash[27292]: cluster 2023-08-27T22:54:58.257921+0000 mgr.y (mgr.14901) 1693 : cluster [DBG] pgmap v1700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:00.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:00 smithi130 bash[27579]: audit 2023-08-27T22:54:59.654488+0000 mon.a (mon.0) 1729 : audit [DBG] from='client.? 172.21.15.6:0/3742123266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:00.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:00 smithi006 bash[27292]: audit 2023-08-27T22:54:59.654488+0000 mon.a (mon.0) 1729 : audit [DBG] from='client.? 172.21.15.6:0/3742123266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:00.816 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:00 smithi006 bash[33918]: audit 2023-08-27T22:54:59.654488+0000 mon.a (mon.0) 1729 : audit [DBG] from='client.? 172.21.15.6:0/3742123266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:00.816 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:55:00.326Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:00.816 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:55:00.328Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:01.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:55:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:55:00] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T22:55:01.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:01 smithi130 bash[27579]: cluster 2023-08-27T22:55:00.259195+0000 mgr.y (mgr.14901) 1694 : cluster [DBG] pgmap v1701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:01.761 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:01 smithi006 bash[33918]: cluster 2023-08-27T22:55:00.259195+0000 mgr.y (mgr.14901) 1694 : cluster [DBG] pgmap v1701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:01.761 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:01 smithi006 bash[27292]: cluster 2023-08-27T22:55:00.259195+0000 mgr.y (mgr.14901) 1694 : cluster [DBG] pgmap v1701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:02.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:55:01.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:02.297 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:55:01.889Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:02.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:02 smithi130 bash[27579]: audit 2023-08-27T22:55:02.072912+0000 mon.a (mon.0) 1730 : audit [DBG] from='client.? 172.21.15.6:0/1228962168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:02.982 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:02 smithi006 bash[27292]: audit 2023-08-27T22:55:02.072912+0000 mon.a (mon.0) 1730 : audit [DBG] from='client.? 172.21.15.6:0/1228962168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:02.982 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:02 smithi006 bash[33918]: audit 2023-08-27T22:55:02.072912+0000 mon.a (mon.0) 1730 : audit [DBG] from='client.? 172.21.15.6:0/1228962168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:03.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:03 smithi130 bash[27579]: cluster 2023-08-27T22:55:02.259954+0000 mgr.y (mgr.14901) 1695 : cluster [DBG] pgmap v1702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:03.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:03 smithi006 bash[33918]: cluster 2023-08-27T22:55:02.259954+0000 mgr.y (mgr.14901) 1695 : cluster [DBG] pgmap v1702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:03.922 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:03 smithi006 bash[27292]: cluster 2023-08-27T22:55:02.259954+0000 mgr.y (mgr.14901) 1695 : cluster [DBG] pgmap v1702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:04.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:55:03.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:04.337 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:55:03.787Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:05 smithi130 bash[27579]: cluster 2023-08-27T22:55:04.261069+0000 mgr.y (mgr.14901) 1696 : cluster [DBG] pgmap v1703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:06.506 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:05 smithi130 bash[27579]: audit 2023-08-27T22:55:04.484169+0000 mon.c (mon.2) 674 : audit [DBG] from='client.? 172.21.15.6:0/3129139982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:06.506 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:05 smithi006 bash[33918]: cluster 2023-08-27T22:55:04.261069+0000 mgr.y (mgr.14901) 1696 : cluster [DBG] pgmap v1703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:06.506 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:05 smithi006 bash[33918]: audit 2023-08-27T22:55:04.484169+0000 mon.c (mon.2) 674 : audit [DBG] from='client.? 172.21.15.6:0/3129139982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:06.507 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:05 smithi006 bash[27292]: cluster 2023-08-27T22:55:04.261069+0000 mgr.y (mgr.14901) 1696 : cluster [DBG] pgmap v1703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:06.507 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:05 smithi006 bash[27292]: audit 2023-08-27T22:55:04.484169+0000 mon.c (mon.2) 674 : audit [DBG] from='client.? 172.21.15.6:0/3129139982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:07.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:07 smithi130 bash[27579]: cluster 2023-08-27T22:55:06.261682+0000 mgr.y (mgr.14901) 1697 : cluster [DBG] pgmap v1704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:08.124 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:07 smithi130 bash[27579]: audit 2023-08-27T22:55:06.900243+0000 mon.c (mon.2) 675 : audit [DBG] from='client.? 172.21.15.6:0/1644720980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:08.125 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:07 smithi006 bash[27292]: cluster 2023-08-27T22:55:06.261682+0000 mgr.y (mgr.14901) 1697 : cluster [DBG] pgmap v1704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:08.125 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:07 smithi006 bash[27292]: audit 2023-08-27T22:55:06.900243+0000 mon.c (mon.2) 675 : audit [DBG] from='client.? 172.21.15.6:0/1644720980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:08.125 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:07 smithi006 bash[33918]: cluster 2023-08-27T22:55:06.261682+0000 mgr.y (mgr.14901) 1697 : cluster [DBG] pgmap v1704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:08.125 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:07 smithi006 bash[33918]: audit 2023-08-27T22:55:06.900243+0000 mon.c (mon.2) 675 : audit [DBG] from='client.? 172.21.15.6:0/1644720980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:09.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:09 smithi130 bash[27579]: cluster 2023-08-27T22:55:08.262353+0000 mgr.y (mgr.14901) 1698 : cluster [DBG] pgmap v1705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:09.923 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:09 smithi130 bash[27579]: audit 2023-08-27T22:55:09.315281+0000 mon.a (mon.0) 1731 : audit [DBG] from='client.? 172.21.15.6:0/2693368007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:09.924 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:09 smithi006 bash[27292]: cluster 2023-08-27T22:55:08.262353+0000 mgr.y (mgr.14901) 1698 : cluster [DBG] pgmap v1705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:09.924 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:09 smithi006 bash[27292]: audit 2023-08-27T22:55:09.315281+0000 mon.a (mon.0) 1731 : audit [DBG] from='client.? 172.21.15.6:0/2693368007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:09.924 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:09 smithi006 bash[33918]: cluster 2023-08-27T22:55:08.262353+0000 mgr.y (mgr.14901) 1698 : cluster [DBG] pgmap v1705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:09.924 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:09 smithi006 bash[33918]: audit 2023-08-27T22:55:09.315281+0000 mon.a (mon.0) 1731 : audit [DBG] from='client.? 172.21.15.6:0/2693368007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:10.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:55:10.326Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:10.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:55:10.329Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:11.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:55:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:55:10] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T22:55:11.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:11 smithi130 bash[27579]: cluster 2023-08-27T22:55:10.263536+0000 mgr.y (mgr.14901) 1699 : cluster [DBG] pgmap v1706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:11.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:11 smithi006 bash[27292]: cluster 2023-08-27T22:55:10.263536+0000 mgr.y (mgr.14901) 1699 : cluster [DBG] pgmap v1706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:11.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:11 smithi006 bash[33918]: cluster 2023-08-27T22:55:10.263536+0000 mgr.y (mgr.14901) 1699 : cluster [DBG] pgmap v1706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:12.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:55:11.888Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:12.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:55:11.890Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:12.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:12 smithi130 bash[27579]: audit 2023-08-27T22:55:11.733728+0000 mon.c (mon.2) 676 : audit [DBG] from='client.? 172.21.15.6:0/531463018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:12.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:12 smithi006 bash[33918]: audit 2023-08-27T22:55:11.733728+0000 mon.c (mon.2) 676 : audit [DBG] from='client.? 172.21.15.6:0/531463018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:12.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:12 smithi006 bash[27292]: audit 2023-08-27T22:55:11.733728+0000 mon.c (mon.2) 676 : audit [DBG] from='client.? 172.21.15.6:0/531463018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:13.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:13 smithi130 bash[27579]: cluster 2023-08-27T22:55:12.264172+0000 mgr.y (mgr.14901) 1700 : cluster [DBG] pgmap v1707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:13.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:13 smithi006 bash[27292]: cluster 2023-08-27T22:55:12.264172+0000 mgr.y (mgr.14901) 1700 : cluster [DBG] pgmap v1707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:13.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:13 smithi006 bash[33918]: cluster 2023-08-27T22:55:12.264172+0000 mgr.y (mgr.14901) 1700 : cluster [DBG] pgmap v1707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:14.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:55:13.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:14.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:55:13.787Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:15.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:15 smithi130 bash[27579]: audit 2023-08-27T22:55:14.149541+0000 mon.c (mon.2) 677 : audit [DBG] from='client.? 172.21.15.6:0/1686689535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:15 smithi130 bash[27579]: cluster 2023-08-27T22:55:14.265237+0000 mgr.y (mgr.14901) 1701 : cluster [DBG] pgmap v1708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:15.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:15 smithi006 bash[33918]: audit 2023-08-27T22:55:14.149541+0000 mon.c (mon.2) 677 : audit [DBG] from='client.? 172.21.15.6:0/1686689535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:15.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:15 smithi006 bash[33918]: cluster 2023-08-27T22:55:14.265237+0000 mgr.y (mgr.14901) 1701 : cluster [DBG] pgmap v1708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:15.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:15 smithi006 bash[27292]: audit 2023-08-27T22:55:14.149541+0000 mon.c (mon.2) 677 : audit [DBG] from='client.? 172.21.15.6:0/1686689535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:15.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:15 smithi006 bash[27292]: cluster 2023-08-27T22:55:14.265237+0000 mgr.y (mgr.14901) 1701 : cluster [DBG] pgmap v1708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:17.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:17 smithi130 bash[27579]: cluster 2023-08-27T22:55:16.265883+0000 mgr.y (mgr.14901) 1702 : cluster [DBG] pgmap v1709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:17.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:17 smithi130 bash[27579]: audit 2023-08-27T22:55:16.567612+0000 mon.a (mon.0) 1732 : audit [DBG] from='client.? 172.21.15.6:0/2091921068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:17.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:17 smithi006 bash[33918]: cluster 2023-08-27T22:55:16.265883+0000 mgr.y (mgr.14901) 1702 : cluster [DBG] pgmap v1709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:17.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:17 smithi006 bash[33918]: audit 2023-08-27T22:55:16.567612+0000 mon.a (mon.0) 1732 : audit [DBG] from='client.? 172.21.15.6:0/2091921068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:17.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:17 smithi006 bash[27292]: cluster 2023-08-27T22:55:16.265883+0000 mgr.y (mgr.14901) 1702 : cluster [DBG] pgmap v1709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:17.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:17 smithi006 bash[27292]: audit 2023-08-27T22:55:16.567612+0000 mon.a (mon.0) 1732 : audit [DBG] from='client.? 172.21.15.6:0/2091921068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:19.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:19 smithi130 bash[27579]: cluster 2023-08-27T22:55:18.266544+0000 mgr.y (mgr.14901) 1703 : cluster [DBG] pgmap v1710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:19 smithi130 bash[27579]: audit 2023-08-27T22:55:18.983997+0000 mon.c (mon.2) 678 : audit [DBG] from='client.? 172.21.15.6:0/3780797630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:19.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:19 smithi006 bash[27292]: cluster 2023-08-27T22:55:18.266544+0000 mgr.y (mgr.14901) 1703 : cluster [DBG] pgmap v1710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:19.723 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:19 smithi006 bash[27292]: audit 2023-08-27T22:55:18.983997+0000 mon.c (mon.2) 678 : audit [DBG] from='client.? 172.21.15.6:0/3780797630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:19.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:19 smithi006 bash[33918]: cluster 2023-08-27T22:55:18.266544+0000 mgr.y (mgr.14901) 1703 : cluster [DBG] pgmap v1710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:19.724 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:19 smithi006 bash[33918]: audit 2023-08-27T22:55:18.983997+0000 mon.c (mon.2) 678 : audit [DBG] from='client.? 172.21.15.6:0/3780797630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:20.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:55:20.326Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:20.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:55:20.329Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:21.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:55:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:55:20] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T22:55:21.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:21 smithi130 bash[27579]: cluster 2023-08-27T22:55:20.267665+0000 mgr.y (mgr.14901) 1704 : cluster [DBG] pgmap v1711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:21.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:21 smithi006 bash[33918]: cluster 2023-08-27T22:55:20.267665+0000 mgr.y (mgr.14901) 1704 : cluster [DBG] pgmap v1711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:21.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:21 smithi006 bash[27292]: cluster 2023-08-27T22:55:20.267665+0000 mgr.y (mgr.14901) 1704 : cluster [DBG] pgmap v1711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:22.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:55:21.888Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:22.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:55:21.890Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:22.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:22 smithi130 bash[27579]: audit 2023-08-27T22:55:21.403069+0000 mon.a (mon.0) 1733 : audit [DBG] from='client.? 172.21.15.6:0/1140214019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:22.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:22 smithi006 bash[27292]: audit 2023-08-27T22:55:21.403069+0000 mon.a (mon.0) 1733 : audit [DBG] from='client.? 172.21.15.6:0/1140214019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:22.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:22 smithi006 bash[33918]: audit 2023-08-27T22:55:21.403069+0000 mon.a (mon.0) 1733 : audit [DBG] from='client.? 172.21.15.6:0/1140214019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:23.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:23 smithi130 bash[27579]: cluster 2023-08-27T22:55:22.268309+0000 mgr.y (mgr.14901) 1705 : cluster [DBG] pgmap v1712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:23.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:23 smithi006 bash[33918]: cluster 2023-08-27T22:55:22.268309+0000 mgr.y (mgr.14901) 1705 : cluster [DBG] pgmap v1712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:23.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:23 smithi006 bash[27292]: cluster 2023-08-27T22:55:22.268309+0000 mgr.y (mgr.14901) 1705 : cluster [DBG] pgmap v1712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:24.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:55:23.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:24.222 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:55:23.788Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:24.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:24 smithi130 bash[27579]: audit 2023-08-27T22:55:23.821673+0000 mon.c (mon.2) 679 : audit [DBG] from='client.? 172.21.15.6:0/3693153642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:24.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:24 smithi006 bash[27292]: audit 2023-08-27T22:55:23.821673+0000 mon.c (mon.2) 679 : audit [DBG] from='client.? 172.21.15.6:0/3693153642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:24.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:24 smithi006 bash[33918]: audit 2023-08-27T22:55:23.821673+0000 mon.c (mon.2) 679 : audit [DBG] from='client.? 172.21.15.6:0/3693153642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:25.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:25 smithi130 bash[27579]: cluster 2023-08-27T22:55:24.269422+0000 mgr.y (mgr.14901) 1706 : cluster [DBG] pgmap v1713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:25.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:25 smithi006 bash[27292]: cluster 2023-08-27T22:55:24.269422+0000 mgr.y (mgr.14901) 1706 : cluster [DBG] pgmap v1713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:25.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:25 smithi006 bash[33918]: cluster 2023-08-27T22:55:24.269422+0000 mgr.y (mgr.14901) 1706 : cluster [DBG] pgmap v1713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:26.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:26 smithi130 bash[27579]: audit 2023-08-27T22:55:26.248002+0000 mon.b (mon.1) 284 : audit [DBG] from='client.? 172.21.15.6:0/2229895738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:26.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:26 smithi006 bash[33918]: audit 2023-08-27T22:55:26.248002+0000 mon.b (mon.1) 284 : audit [DBG] from='client.? 172.21.15.6:0/2229895738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:26.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:26 smithi006 bash[27292]: audit 2023-08-27T22:55:26.248002+0000 mon.b (mon.1) 284 : audit [DBG] from='client.? 172.21.15.6:0/2229895738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:27.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:27 smithi006 bash[27292]: cluster 2023-08-27T22:55:26.269982+0000 mgr.y (mgr.14901) 1707 : cluster [DBG] pgmap v1714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:27.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:27 smithi006 bash[33918]: cluster 2023-08-27T22:55:26.269982+0000 mgr.y (mgr.14901) 1707 : cluster [DBG] pgmap v1714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:27.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:27 smithi130 bash[27579]: cluster 2023-08-27T22:55:26.269982+0000 mgr.y (mgr.14901) 1707 : cluster [DBG] pgmap v1714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:29.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:29 smithi006 bash[27292]: cluster 2023-08-27T22:55:28.270578+0000 mgr.y (mgr.14901) 1708 : cluster [DBG] pgmap v1715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:29.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:29 smithi006 bash[27292]: audit 2023-08-27T22:55:28.657345+0000 mon.a (mon.0) 1734 : audit [DBG] from='client.? 172.21.15.6:0/3212137540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:29.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:29 smithi006 bash[33918]: cluster 2023-08-27T22:55:28.270578+0000 mgr.y (mgr.14901) 1708 : cluster [DBG] pgmap v1715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:29.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:29 smithi006 bash[33918]: audit 2023-08-27T22:55:28.657345+0000 mon.a (mon.0) 1734 : audit [DBG] from='client.? 172.21.15.6:0/3212137540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:29.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:29 smithi130 bash[27579]: cluster 2023-08-27T22:55:28.270578+0000 mgr.y (mgr.14901) 1708 : cluster [DBG] pgmap v1715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:29.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:29 smithi130 bash[27579]: audit 2023-08-27T22:55:28.657345+0000 mon.a (mon.0) 1734 : audit [DBG] from='client.? 172.21.15.6:0/3212137540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:30.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:55:30.328Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:30.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:55:30.331Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:31.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:55:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:55:30] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T22:55:31.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:31 smithi006 bash[33918]: cluster 2023-08-27T22:55:30.271703+0000 mgr.y (mgr.14901) 1709 : cluster [DBG] pgmap v1716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:31.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:31 smithi006 bash[33918]: audit 2023-08-27T22:55:31.070786+0000 mon.c (mon.2) 680 : audit [DBG] from='client.? 172.21.15.6:0/3052023294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:31.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:31 smithi006 bash[27292]: cluster 2023-08-27T22:55:30.271703+0000 mgr.y (mgr.14901) 1709 : cluster [DBG] pgmap v1716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:31.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:31 smithi006 bash[27292]: audit 2023-08-27T22:55:31.070786+0000 mon.c (mon.2) 680 : audit [DBG] from='client.? 172.21.15.6:0/3052023294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:31.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:31 smithi130 bash[27579]: cluster 2023-08-27T22:55:30.271703+0000 mgr.y (mgr.14901) 1709 : cluster [DBG] pgmap v1716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:31.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:31 smithi130 bash[27579]: audit 2023-08-27T22:55:31.070786+0000 mon.c (mon.2) 680 : audit [DBG] from='client.? 172.21.15.6:0/3052023294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:32.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:55:31.889Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:32.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:55:31.891Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:33.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:33 smithi006 bash[27292]: cluster 2023-08-27T22:55:32.272354+0000 mgr.y (mgr.14901) 1710 : cluster [DBG] pgmap v1717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:33.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:33 smithi006 bash[33918]: cluster 2023-08-27T22:55:32.272354+0000 mgr.y (mgr.14901) 1710 : cluster [DBG] pgmap v1717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:33 smithi130 bash[27579]: cluster 2023-08-27T22:55:32.272354+0000 mgr.y (mgr.14901) 1710 : cluster [DBG] pgmap v1717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:34.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:55:33.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:34.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:55:33.789Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:34.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:34 smithi006 bash[27292]: audit 2023-08-27T22:55:33.492689+0000 mon.a (mon.0) 1735 : audit [DBG] from='client.? 172.21.15.6:0/1999381415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:34.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:34 smithi006 bash[33918]: audit 2023-08-27T22:55:33.492689+0000 mon.a (mon.0) 1735 : audit [DBG] from='client.? 172.21.15.6:0/1999381415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:34.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:34 smithi130 bash[27579]: audit 2023-08-27T22:55:33.492689+0000 mon.a (mon.0) 1735 : audit [DBG] from='client.? 172.21.15.6:0/1999381415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:35.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:35 smithi006 bash[27292]: cluster 2023-08-27T22:55:34.273467+0000 mgr.y (mgr.14901) 1711 : cluster [DBG] pgmap v1718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:35.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:35 smithi006 bash[33918]: cluster 2023-08-27T22:55:34.273467+0000 mgr.y (mgr.14901) 1711 : cluster [DBG] pgmap v1718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:35.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:35 smithi130 bash[27579]: cluster 2023-08-27T22:55:34.273467+0000 mgr.y (mgr.14901) 1711 : cluster [DBG] pgmap v1718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:36.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:36 smithi006 bash[33918]: audit 2023-08-27T22:55:35.906786+0000 mon.c (mon.2) 681 : audit [DBG] from='client.? 172.21.15.6:0/2934304571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:36.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:36 smithi006 bash[27292]: audit 2023-08-27T22:55:35.906786+0000 mon.c (mon.2) 681 : audit [DBG] from='client.? 172.21.15.6:0/2934304571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:36.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:36 smithi130 bash[27579]: audit 2023-08-27T22:55:35.906786+0000 mon.c (mon.2) 681 : audit [DBG] from='client.? 172.21.15.6:0/2934304571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:37.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:37 smithi006 bash[27292]: cluster 2023-08-27T22:55:36.274127+0000 mgr.y (mgr.14901) 1712 : cluster [DBG] pgmap v1719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:37.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:37 smithi006 bash[33918]: cluster 2023-08-27T22:55:36.274127+0000 mgr.y (mgr.14901) 1712 : cluster [DBG] pgmap v1719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:37.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:37 smithi130 bash[27579]: cluster 2023-08-27T22:55:36.274127+0000 mgr.y (mgr.14901) 1712 : cluster [DBG] pgmap v1719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:38.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:38 smithi006 bash[27292]: audit 2023-08-27T22:55:38.326357+0000 mon.a (mon.0) 1736 : audit [DBG] from='client.? 172.21.15.6:0/1728489711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:38.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:38 smithi006 bash[33918]: audit 2023-08-27T22:55:38.326357+0000 mon.a (mon.0) 1736 : audit [DBG] from='client.? 172.21.15.6:0/1728489711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:38 smithi130 bash[27579]: audit 2023-08-27T22:55:38.326357+0000 mon.a (mon.0) 1736 : audit [DBG] from='client.? 172.21.15.6:0/1728489711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:39.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:39 smithi006 bash[27292]: cluster 2023-08-27T22:55:38.274756+0000 mgr.y (mgr.14901) 1713 : cluster [DBG] pgmap v1720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:39.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:39 smithi006 bash[27292]: audit 2023-08-27T22:55:39.336051+0000 mon.b (mon.1) 285 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:55:39.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:39 smithi006 bash[33918]: cluster 2023-08-27T22:55:38.274756+0000 mgr.y (mgr.14901) 1713 : cluster [DBG] pgmap v1720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:39.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:39 smithi006 bash[33918]: audit 2023-08-27T22:55:39.336051+0000 mon.b (mon.1) 285 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:55:39.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:39 smithi130 bash[27579]: cluster 2023-08-27T22:55:38.274756+0000 mgr.y (mgr.14901) 1713 : cluster [DBG] pgmap v1720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:39.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:39 smithi130 bash[27579]: audit 2023-08-27T22:55:39.336051+0000 mon.b (mon.1) 285 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:55:40.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:40 smithi006 bash[33918]: cluster 2023-08-27T22:55:40.275856+0000 mgr.y (mgr.14901) 1714 : cluster [DBG] pgmap v1721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:40.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:55:40.329Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:40.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:55:40.332Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:40.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:40 smithi006 bash[27292]: cluster 2023-08-27T22:55:40.275856+0000 mgr.y (mgr.14901) 1714 : cluster [DBG] pgmap v1721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:40 smithi130 bash[27579]: cluster 2023-08-27T22:55:40.275856+0000 mgr.y (mgr.14901) 1714 : cluster [DBG] pgmap v1721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:41.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:55:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:55:40] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T22:55:41.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:41 smithi006 bash[33918]: audit 2023-08-27T22:55:40.746664+0000 mon.a (mon.0) 1737 : audit [DBG] from='client.? 172.21.15.6:0/1362039990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:41.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:41 smithi006 bash[27292]: audit 2023-08-27T22:55:40.746664+0000 mon.a (mon.0) 1737 : audit [DBG] from='client.? 172.21.15.6:0/1362039990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:41.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:41 smithi130 bash[27579]: audit 2023-08-27T22:55:40.746664+0000 mon.a (mon.0) 1737 : audit [DBG] from='client.? 172.21.15.6:0/1362039990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:42.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:55:41.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:42.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:55:41.892Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:42.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:42 smithi006 bash[27292]: cluster 2023-08-27T22:55:42.276540+0000 mgr.y (mgr.14901) 1715 : cluster [DBG] pgmap v1722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:42.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:42 smithi006 bash[33918]: cluster 2023-08-27T22:55:42.276540+0000 mgr.y (mgr.14901) 1715 : cluster [DBG] pgmap v1722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:42 smithi130 bash[27579]: cluster 2023-08-27T22:55:42.276540+0000 mgr.y (mgr.14901) 1715 : cluster [DBG] pgmap v1722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:43.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:43 smithi006 bash[33918]: audit 2023-08-27T22:55:43.160511+0000 mon.c (mon.2) 682 : audit [DBG] from='client.? 172.21.15.6:0/1758642533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:43.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:43 smithi006 bash[27292]: audit 2023-08-27T22:55:43.160511+0000 mon.c (mon.2) 682 : audit [DBG] from='client.? 172.21.15.6:0/1758642533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:43.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:43 smithi130 bash[27579]: audit 2023-08-27T22:55:43.160511+0000 mon.c (mon.2) 682 : audit [DBG] from='client.? 172.21.15.6:0/1758642533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:44.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:55:43.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:44.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:55:43.790Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:45.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:45 smithi130 bash[27579]: cluster 2023-08-27T22:55:44.277622+0000 mgr.y (mgr.14901) 1716 : cluster [DBG] pgmap v1723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:45.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:45 smithi006 bash[33918]: cluster 2023-08-27T22:55:44.277622+0000 mgr.y (mgr.14901) 1716 : cluster [DBG] pgmap v1723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:45.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:45 smithi006 bash[27292]: cluster 2023-08-27T22:55:44.277622+0000 mgr.y (mgr.14901) 1716 : cluster [DBG] pgmap v1723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:46.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:46 smithi130 bash[27579]: audit 2023-08-27T22:55:45.241398+0000 mon.a (mon.0) 1738 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:55:47.585 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:46 smithi130 bash[27579]: audit 2023-08-27T22:55:45.571090+0000 mon.c (mon.2) 683 : audit [DBG] from='client.? 172.21.15.6:0/4058703500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:47.585 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:46 smithi130 bash[27579]: audit 2023-08-27T22:55:45.666759+0000 mon.a (mon.0) 1739 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:55:47.585 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:46 smithi130 bash[27579]: audit 2023-08-27T22:55:45.964200+0000 mon.b (mon.1) 286 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:55:47.586 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:46 smithi130 bash[27579]: audit 2023-08-27T22:55:45.966346+0000 mon.b (mon.1) 287 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:55:47.586 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:46 smithi130 bash[27579]: audit 2023-08-27T22:55:45.971784+0000 mon.a (mon.0) 1740 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:55:47.586 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:46 smithi006 bash[33918]: audit 2023-08-27T22:55:45.241398+0000 mon.a (mon.0) 1738 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:55:47.586 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:46 smithi006 bash[33918]: audit 2023-08-27T22:55:45.571090+0000 mon.c (mon.2) 683 : audit [DBG] from='client.? 172.21.15.6:0/4058703500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:47.587 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:46 smithi006 bash[33918]: audit 2023-08-27T22:55:45.666759+0000 mon.a (mon.0) 1739 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:55:47.587 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:46 smithi006 bash[33918]: audit 2023-08-27T22:55:45.964200+0000 mon.b (mon.1) 286 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:55:47.587 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:46 smithi006 bash[33918]: audit 2023-08-27T22:55:45.966346+0000 mon.b (mon.1) 287 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:55:47.587 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:46 smithi006 bash[33918]: audit 2023-08-27T22:55:45.971784+0000 mon.a (mon.0) 1740 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:55:47.587 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:46 smithi006 bash[27292]: audit 2023-08-27T22:55:45.241398+0000 mon.a (mon.0) 1738 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:55:47.587 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:46 smithi006 bash[27292]: audit 2023-08-27T22:55:45.571090+0000 mon.c (mon.2) 683 : audit [DBG] from='client.? 172.21.15.6:0/4058703500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:47.587 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:46 smithi006 bash[27292]: audit 2023-08-27T22:55:45.666759+0000 mon.a (mon.0) 1739 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:55:47.587 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:46 smithi006 bash[27292]: audit 2023-08-27T22:55:45.964200+0000 mon.b (mon.1) 286 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:55:47.587 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:46 smithi006 bash[27292]: audit 2023-08-27T22:55:45.966346+0000 mon.b (mon.1) 287 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:55:47.588 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:46 smithi006 bash[27292]: audit 2023-08-27T22:55:45.971784+0000 mon.a (mon.0) 1740 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:55:47.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:47 smithi130 bash[27579]: cluster 2023-08-27T22:55:46.278271+0000 mgr.y (mgr.14901) 1717 : cluster [DBG] pgmap v1724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:47.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:47 smithi006 bash[33918]: cluster 2023-08-27T22:55:46.278271+0000 mgr.y (mgr.14901) 1717 : cluster [DBG] pgmap v1724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:47.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:47 smithi006 bash[27292]: cluster 2023-08-27T22:55:46.278271+0000 mgr.y (mgr.14901) 1717 : cluster [DBG] pgmap v1724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:48.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:48 smithi130 bash[27579]: audit 2023-08-27T22:55:47.990402+0000 mon.a (mon.0) 1741 : audit [DBG] from='client.? 172.21.15.6:0/1796726121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:48.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:48 smithi006 bash[33918]: audit 2023-08-27T22:55:47.990402+0000 mon.a (mon.0) 1741 : audit [DBG] from='client.? 172.21.15.6:0/1796726121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:48.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:48 smithi006 bash[27292]: audit 2023-08-27T22:55:47.990402+0000 mon.a (mon.0) 1741 : audit [DBG] from='client.? 172.21.15.6:0/1796726121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:49.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:49 smithi130 bash[27579]: cluster 2023-08-27T22:55:48.278922+0000 mgr.y (mgr.14901) 1718 : cluster [DBG] pgmap v1725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:49.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:49 smithi006 bash[27292]: cluster 2023-08-27T22:55:48.278922+0000 mgr.y (mgr.14901) 1718 : cluster [DBG] pgmap v1725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:49.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:49 smithi006 bash[33918]: cluster 2023-08-27T22:55:48.278922+0000 mgr.y (mgr.14901) 1718 : cluster [DBG] pgmap v1725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:50.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:55:50.330Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:50.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:55:50.332Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:51.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:55:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:55:50] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T22:55:51.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:51 smithi130 bash[27579]: cluster 2023-08-27T22:55:50.279981+0000 mgr.y (mgr.14901) 1719 : cluster [DBG] pgmap v1726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:51.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:51 smithi130 bash[27579]: audit 2023-08-27T22:55:50.417281+0000 mon.a (mon.0) 1742 : audit [DBG] from='client.? 172.21.15.6:0/2247132622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:51.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:51 smithi006 bash[27292]: cluster 2023-08-27T22:55:50.279981+0000 mgr.y (mgr.14901) 1719 : cluster [DBG] pgmap v1726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:51.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:51 smithi006 bash[27292]: audit 2023-08-27T22:55:50.417281+0000 mon.a (mon.0) 1742 : audit [DBG] from='client.? 172.21.15.6:0/2247132622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:51.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:51 smithi006 bash[33918]: cluster 2023-08-27T22:55:50.279981+0000 mgr.y (mgr.14901) 1719 : cluster [DBG] pgmap v1726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:51.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:51 smithi006 bash[33918]: audit 2023-08-27T22:55:50.417281+0000 mon.a (mon.0) 1742 : audit [DBG] from='client.? 172.21.15.6:0/2247132622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:52.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:55:51.891Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:52.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:55:51.893Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:53 smithi130 bash[27579]: cluster 2023-08-27T22:55:52.280637+0000 mgr.y (mgr.14901) 1720 : cluster [DBG] pgmap v1727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:53 smithi130 bash[27579]: audit 2023-08-27T22:55:52.830663+0000 mon.c (mon.2) 684 : audit [DBG] from='client.? 172.21.15.6:0/1164944059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:53.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:53 smithi006 bash[27292]: cluster 2023-08-27T22:55:52.280637+0000 mgr.y (mgr.14901) 1720 : cluster [DBG] pgmap v1727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:53.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:53 smithi006 bash[27292]: audit 2023-08-27T22:55:52.830663+0000 mon.c (mon.2) 684 : audit [DBG] from='client.? 172.21.15.6:0/1164944059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:53.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:53 smithi006 bash[33918]: cluster 2023-08-27T22:55:52.280637+0000 mgr.y (mgr.14901) 1720 : cluster [DBG] pgmap v1727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:53.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:53 smithi006 bash[33918]: audit 2023-08-27T22:55:52.830663+0000 mon.c (mon.2) 684 : audit [DBG] from='client.? 172.21.15.6:0/1164944059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:54.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:55:53.788Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:54.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:55:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:55:53.790Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:55:55.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:55 smithi006 bash[27292]: cluster 2023-08-27T22:55:54.281758+0000 mgr.y (mgr.14901) 1721 : cluster [DBG] pgmap v1728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:55.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:55 smithi006 bash[33918]: cluster 2023-08-27T22:55:54.281758+0000 mgr.y (mgr.14901) 1721 : cluster [DBG] pgmap v1728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:55.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:55 smithi130 bash[27579]: cluster 2023-08-27T22:55:54.281758+0000 mgr.y (mgr.14901) 1721 : cluster [DBG] pgmap v1728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:56.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:56 smithi006 bash[27292]: audit 2023-08-27T22:55:55.248292+0000 mon.a (mon.0) 1743 : audit [DBG] from='client.? 172.21.15.6:0/1778881850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:56.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:56 smithi006 bash[33918]: audit 2023-08-27T22:55:55.248292+0000 mon.a (mon.0) 1743 : audit [DBG] from='client.? 172.21.15.6:0/1778881850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:56.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:56 smithi130 bash[27579]: audit 2023-08-27T22:55:55.248292+0000 mon.a (mon.0) 1743 : audit [DBG] from='client.? 172.21.15.6:0/1778881850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:57.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:57 smithi006 bash[33918]: cluster 2023-08-27T22:55:56.282395+0000 mgr.y (mgr.14901) 1722 : cluster [DBG] pgmap v1729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:57.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:57 smithi006 bash[27292]: cluster 2023-08-27T22:55:56.282395+0000 mgr.y (mgr.14901) 1722 : cluster [DBG] pgmap v1729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:57.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:57 smithi130 bash[27579]: cluster 2023-08-27T22:55:56.282395+0000 mgr.y (mgr.14901) 1722 : cluster [DBG] pgmap v1729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:58.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:58 smithi006 bash[27292]: audit 2023-08-27T22:55:57.661383+0000 mon.a (mon.0) 1744 : audit [DBG] from='client.? 172.21.15.6:0/987951817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:58.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:58 smithi006 bash[33918]: audit 2023-08-27T22:55:57.661383+0000 mon.a (mon.0) 1744 : audit [DBG] from='client.? 172.21.15.6:0/987951817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:58.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:58 smithi130 bash[27579]: audit 2023-08-27T22:55:57.661383+0000 mon.a (mon.0) 1744 : audit [DBG] from='client.? 172.21.15.6:0/987951817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:59.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:55:59 smithi006 bash[27292]: cluster 2023-08-27T22:55:58.283027+0000 mgr.y (mgr.14901) 1723 : cluster [DBG] pgmap v1730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:59.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:55:59 smithi006 bash[33918]: cluster 2023-08-27T22:55:58.283027+0000 mgr.y (mgr.14901) 1723 : cluster [DBG] pgmap v1730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:55:59.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:55:59 smithi130 bash[27579]: cluster 2023-08-27T22:55:58.283027+0000 mgr.y (mgr.14901) 1723 : cluster [DBG] pgmap v1730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:00.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:00 smithi006 bash[33918]: audit 2023-08-27T22:56:00.072676+0000 mon.a (mon.0) 1745 : audit [DBG] from='client.? 172.21.15.6:0/3903816837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:00.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:00 smithi006 bash[27292]: audit 2023-08-27T22:56:00.072676+0000 mon.a (mon.0) 1745 : audit [DBG] from='client.? 172.21.15.6:0/3903816837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:00.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:56:00.331Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:00.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:56:00.333Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:00.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:00 smithi130 bash[27579]: audit 2023-08-27T22:56:00.072676+0000 mon.a (mon.0) 1745 : audit [DBG] from='client.? 172.21.15.6:0/3903816837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:01.158 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:56:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:56:00] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T22:56:01.158 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:01 smithi006 bash[27292]: cluster 2023-08-27T22:56:00.284141+0000 mgr.y (mgr.14901) 1724 : cluster [DBG] pgmap v1731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:01.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:01 smithi006 bash[33918]: cluster 2023-08-27T22:56:00.284141+0000 mgr.y (mgr.14901) 1724 : cluster [DBG] pgmap v1731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:01.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:01 smithi130 bash[27579]: cluster 2023-08-27T22:56:00.284141+0000 mgr.y (mgr.14901) 1724 : cluster [DBG] pgmap v1731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:02.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:56:01.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:02.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:56:01.893Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:03.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:03 smithi130 bash[27579]: cluster 2023-08-27T22:56:02.284773+0000 mgr.y (mgr.14901) 1725 : cluster [DBG] pgmap v1732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:03.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:03 smithi130 bash[27579]: audit 2023-08-27T22:56:02.490788+0000 mon.a (mon.0) 1746 : audit [DBG] from='client.? 172.21.15.6:0/2066096448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:03.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:03 smithi006 bash[33918]: cluster 2023-08-27T22:56:02.284773+0000 mgr.y (mgr.14901) 1725 : cluster [DBG] pgmap v1732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:03.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:03 smithi006 bash[33918]: audit 2023-08-27T22:56:02.490788+0000 mon.a (mon.0) 1746 : audit [DBG] from='client.? 172.21.15.6:0/2066096448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:03.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:03 smithi006 bash[27292]: cluster 2023-08-27T22:56:02.284773+0000 mgr.y (mgr.14901) 1725 : cluster [DBG] pgmap v1732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:03.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:03 smithi006 bash[27292]: audit 2023-08-27T22:56:02.490788+0000 mon.a (mon.0) 1746 : audit [DBG] from='client.? 172.21.15.6:0/2066096448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:04.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:56:03.789Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:04.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:56:03.791Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:05.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:05 smithi006 bash[33918]: cluster 2023-08-27T22:56:04.285949+0000 mgr.y (mgr.14901) 1726 : cluster [DBG] pgmap v1733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:05.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:05 smithi006 bash[33918]: audit 2023-08-27T22:56:04.906531+0000 mon.a (mon.0) 1747 : audit [DBG] from='client.? 172.21.15.6:0/1887162250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:05.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:05 smithi006 bash[27292]: cluster 2023-08-27T22:56:04.285949+0000 mgr.y (mgr.14901) 1726 : cluster [DBG] pgmap v1733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:05.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:05 smithi006 bash[27292]: audit 2023-08-27T22:56:04.906531+0000 mon.a (mon.0) 1747 : audit [DBG] from='client.? 172.21.15.6:0/1887162250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:05.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:05 smithi130 bash[27579]: cluster 2023-08-27T22:56:04.285949+0000 mgr.y (mgr.14901) 1726 : cluster [DBG] pgmap v1733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:05.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:05 smithi130 bash[27579]: audit 2023-08-27T22:56:04.906531+0000 mon.a (mon.0) 1747 : audit [DBG] from='client.? 172.21.15.6:0/1887162250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:07.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:07 smithi130 bash[27579]: cluster 2023-08-27T22:56:06.286659+0000 mgr.y (mgr.14901) 1727 : cluster [DBG] pgmap v1734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:07.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:07 smithi130 bash[27579]: audit 2023-08-27T22:56:07.320925+0000 mon.a (mon.0) 1748 : audit [DBG] from='client.? 172.21.15.6:0/1273447250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:07.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:07 smithi006 bash[33918]: cluster 2023-08-27T22:56:06.286659+0000 mgr.y (mgr.14901) 1727 : cluster [DBG] pgmap v1734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:07.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:07 smithi006 bash[33918]: audit 2023-08-27T22:56:07.320925+0000 mon.a (mon.0) 1748 : audit [DBG] from='client.? 172.21.15.6:0/1273447250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:07.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:07 smithi006 bash[27292]: cluster 2023-08-27T22:56:06.286659+0000 mgr.y (mgr.14901) 1727 : cluster [DBG] pgmap v1734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:07.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:07 smithi006 bash[27292]: audit 2023-08-27T22:56:07.320925+0000 mon.a (mon.0) 1748 : audit [DBG] from='client.? 172.21.15.6:0/1273447250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:09 smithi130 bash[27579]: cluster 2023-08-27T22:56:08.287341+0000 mgr.y (mgr.14901) 1728 : cluster [DBG] pgmap v1735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:09 smithi006 bash[33918]: cluster 2023-08-27T22:56:08.287341+0000 mgr.y (mgr.14901) 1728 : cluster [DBG] pgmap v1735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:09.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:09 smithi006 bash[27292]: cluster 2023-08-27T22:56:08.287341+0000 mgr.y (mgr.14901) 1728 : cluster [DBG] pgmap v1735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:10.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:10 smithi130 bash[27579]: audit 2023-08-27T22:56:09.734790+0000 mon.c (mon.2) 685 : audit [DBG] from='client.? 172.21.15.6:0/2738872140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:10.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:10 smithi006 bash[27292]: audit 2023-08-27T22:56:09.734790+0000 mon.c (mon.2) 685 : audit [DBG] from='client.? 172.21.15.6:0/2738872140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:10.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:10 smithi006 bash[33918]: audit 2023-08-27T22:56:09.734790+0000 mon.c (mon.2) 685 : audit [DBG] from='client.? 172.21.15.6:0/2738872140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:10.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:56:10.331Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:10.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:56:10.334Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:11.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:56:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:56:10] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T22:56:11.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:11 smithi130 bash[27579]: cluster 2023-08-27T22:56:10.288495+0000 mgr.y (mgr.14901) 1729 : cluster [DBG] pgmap v1736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:11.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:11 smithi006 bash[33918]: cluster 2023-08-27T22:56:10.288495+0000 mgr.y (mgr.14901) 1729 : cluster [DBG] pgmap v1736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:11.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:11 smithi006 bash[27292]: cluster 2023-08-27T22:56:10.288495+0000 mgr.y (mgr.14901) 1729 : cluster [DBG] pgmap v1736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:12.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:56:11.893Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:12.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:56:11.894Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:12.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:12 smithi006 bash[33918]: audit 2023-08-27T22:56:12.154939+0000 mon.a (mon.0) 1749 : audit [DBG] from='client.? 172.21.15.6:0/1367221831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:12.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:12 smithi006 bash[27292]: audit 2023-08-27T22:56:12.154939+0000 mon.a (mon.0) 1749 : audit [DBG] from='client.? 172.21.15.6:0/1367221831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:12 smithi130 bash[27579]: audit 2023-08-27T22:56:12.154939+0000 mon.a (mon.0) 1749 : audit [DBG] from='client.? 172.21.15.6:0/1367221831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:13.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:13 smithi006 bash[33918]: cluster 2023-08-27T22:56:12.289153+0000 mgr.y (mgr.14901) 1730 : cluster [DBG] pgmap v1737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:13.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:13 smithi006 bash[27292]: cluster 2023-08-27T22:56:12.289153+0000 mgr.y (mgr.14901) 1730 : cluster [DBG] pgmap v1737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:13.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:13 smithi130 bash[27579]: cluster 2023-08-27T22:56:12.289153+0000 mgr.y (mgr.14901) 1730 : cluster [DBG] pgmap v1737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:14.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:56:13.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:14.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:56:13.792Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:15.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:15 smithi006 bash[33918]: cluster 2023-08-27T22:56:14.289969+0000 mgr.y (mgr.14901) 1731 : cluster [DBG] pgmap v1738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:15.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:15 smithi006 bash[33918]: audit 2023-08-27T22:56:14.564058+0000 mon.a (mon.0) 1750 : audit [DBG] from='client.? 172.21.15.6:0/3977483899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:15.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:15 smithi006 bash[27292]: cluster 2023-08-27T22:56:14.289969+0000 mgr.y (mgr.14901) 1731 : cluster [DBG] pgmap v1738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:15.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:15 smithi006 bash[27292]: audit 2023-08-27T22:56:14.564058+0000 mon.a (mon.0) 1750 : audit [DBG] from='client.? 172.21.15.6:0/3977483899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:15.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:15 smithi130 bash[27579]: cluster 2023-08-27T22:56:14.289969+0000 mgr.y (mgr.14901) 1731 : cluster [DBG] pgmap v1738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:15.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:15 smithi130 bash[27579]: audit 2023-08-27T22:56:14.564058+0000 mon.a (mon.0) 1750 : audit [DBG] from='client.? 172.21.15.6:0/3977483899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:17.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:17 smithi130 bash[27579]: cluster 2023-08-27T22:56:16.290597+0000 mgr.y (mgr.14901) 1732 : cluster [DBG] pgmap v1739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:17.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:17 smithi130 bash[27579]: audit 2023-08-27T22:56:16.977051+0000 mon.a (mon.0) 1751 : audit [DBG] from='client.? 172.21.15.6:0/3405645064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:17.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:17 smithi006 bash[33918]: cluster 2023-08-27T22:56:16.290597+0000 mgr.y (mgr.14901) 1732 : cluster [DBG] pgmap v1739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:17.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:17 smithi006 bash[33918]: audit 2023-08-27T22:56:16.977051+0000 mon.a (mon.0) 1751 : audit [DBG] from='client.? 172.21.15.6:0/3405645064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:17.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:17 smithi006 bash[27292]: cluster 2023-08-27T22:56:16.290597+0000 mgr.y (mgr.14901) 1732 : cluster [DBG] pgmap v1739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:17.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:17 smithi006 bash[27292]: audit 2023-08-27T22:56:16.977051+0000 mon.a (mon.0) 1751 : audit [DBG] from='client.? 172.21.15.6:0/3405645064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:19.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:19 smithi130 bash[27579]: cluster 2023-08-27T22:56:18.291318+0000 mgr.y (mgr.14901) 1733 : cluster [DBG] pgmap v1740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:19.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:19 smithi006 bash[27292]: cluster 2023-08-27T22:56:18.291318+0000 mgr.y (mgr.14901) 1733 : cluster [DBG] pgmap v1740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:19.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:19 smithi006 bash[33918]: cluster 2023-08-27T22:56:18.291318+0000 mgr.y (mgr.14901) 1733 : cluster [DBG] pgmap v1740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:20.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:20 smithi006 bash[33918]: audit 2023-08-27T22:56:19.386320+0000 mon.c (mon.2) 686 : audit [DBG] from='client.? 172.21.15.6:0/327568982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:20.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:20 smithi006 bash[27292]: audit 2023-08-27T22:56:19.386320+0000 mon.c (mon.2) 686 : audit [DBG] from='client.? 172.21.15.6:0/327568982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:20.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:56:20.332Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:20.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:56:20.334Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:20.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:20 smithi130 bash[27579]: audit 2023-08-27T22:56:19.386320+0000 mon.c (mon.2) 686 : audit [DBG] from='client.? 172.21.15.6:0/327568982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:21.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:56:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:56:20] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T22:56:21.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:21 smithi006 bash[33918]: cluster 2023-08-27T22:56:20.292473+0000 mgr.y (mgr.14901) 1734 : cluster [DBG] pgmap v1741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:21.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:21 smithi006 bash[27292]: cluster 2023-08-27T22:56:20.292473+0000 mgr.y (mgr.14901) 1734 : cluster [DBG] pgmap v1741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:21.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:21 smithi130 bash[27579]: cluster 2023-08-27T22:56:20.292473+0000 mgr.y (mgr.14901) 1734 : cluster [DBG] pgmap v1741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:22.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:56:21.893Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:22.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:56:21.895Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:22.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:22 smithi006 bash[27292]: audit 2023-08-27T22:56:21.799877+0000 mon.a (mon.0) 1752 : audit [DBG] from='client.? 172.21.15.6:0/1250433474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:22.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:22 smithi006 bash[33918]: audit 2023-08-27T22:56:21.799877+0000 mon.a (mon.0) 1752 : audit [DBG] from='client.? 172.21.15.6:0/1250433474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:22 smithi130 bash[27579]: audit 2023-08-27T22:56:21.799877+0000 mon.a (mon.0) 1752 : audit [DBG] from='client.? 172.21.15.6:0/1250433474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:23.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:23 smithi006 bash[33918]: cluster 2023-08-27T22:56:22.293156+0000 mgr.y (mgr.14901) 1735 : cluster [DBG] pgmap v1742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:23.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:23 smithi006 bash[27292]: cluster 2023-08-27T22:56:22.293156+0000 mgr.y (mgr.14901) 1735 : cluster [DBG] pgmap v1742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:23.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:23 smithi130 bash[27579]: cluster 2023-08-27T22:56:22.293156+0000 mgr.y (mgr.14901) 1735 : cluster [DBG] pgmap v1742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:24.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:56:23.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:24.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:56:23.793Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:25.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:25 smithi006 bash[33918]: audit 2023-08-27T22:56:24.214810+0000 mon.c (mon.2) 687 : audit [DBG] from='client.? 172.21.15.6:0/257813376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:25.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:25 smithi006 bash[33918]: cluster 2023-08-27T22:56:24.294278+0000 mgr.y (mgr.14901) 1736 : cluster [DBG] pgmap v1743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:25.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:25 smithi006 bash[27292]: audit 2023-08-27T22:56:24.214810+0000 mon.c (mon.2) 687 : audit [DBG] from='client.? 172.21.15.6:0/257813376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:25.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:25 smithi006 bash[27292]: cluster 2023-08-27T22:56:24.294278+0000 mgr.y (mgr.14901) 1736 : cluster [DBG] pgmap v1743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:25.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:25 smithi130 bash[27579]: audit 2023-08-27T22:56:24.214810+0000 mon.c (mon.2) 687 : audit [DBG] from='client.? 172.21.15.6:0/257813376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:25.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:25 smithi130 bash[27579]: cluster 2023-08-27T22:56:24.294278+0000 mgr.y (mgr.14901) 1736 : cluster [DBG] pgmap v1743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:27.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:27 smithi130 bash[27579]: cluster 2023-08-27T22:56:26.294904+0000 mgr.y (mgr.14901) 1737 : cluster [DBG] pgmap v1744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:27.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:27 smithi130 bash[27579]: audit 2023-08-27T22:56:26.627775+0000 mon.c (mon.2) 688 : audit [DBG] from='client.? 172.21.15.6:0/1639323753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:27.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:27 smithi006 bash[33918]: cluster 2023-08-27T22:56:26.294904+0000 mgr.y (mgr.14901) 1737 : cluster [DBG] pgmap v1744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:27.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:27 smithi006 bash[33918]: audit 2023-08-27T22:56:26.627775+0000 mon.c (mon.2) 688 : audit [DBG] from='client.? 172.21.15.6:0/1639323753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:27.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:27 smithi006 bash[27292]: cluster 2023-08-27T22:56:26.294904+0000 mgr.y (mgr.14901) 1737 : cluster [DBG] pgmap v1744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:27.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:27 smithi006 bash[27292]: audit 2023-08-27T22:56:26.627775+0000 mon.c (mon.2) 688 : audit [DBG] from='client.? 172.21.15.6:0/1639323753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:29.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:29 smithi130 bash[27579]: cluster 2023-08-27T22:56:28.295579+0000 mgr.y (mgr.14901) 1738 : cluster [DBG] pgmap v1745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:29.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:29 smithi130 bash[27579]: audit 2023-08-27T22:56:29.045772+0000 mon.a (mon.0) 1753 : audit [DBG] from='client.? 172.21.15.6:0/99836164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:29.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:29 smithi006 bash[27292]: cluster 2023-08-27T22:56:28.295579+0000 mgr.y (mgr.14901) 1738 : cluster [DBG] pgmap v1745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:29.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:29 smithi006 bash[27292]: audit 2023-08-27T22:56:29.045772+0000 mon.a (mon.0) 1753 : audit [DBG] from='client.? 172.21.15.6:0/99836164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:29.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:29 smithi006 bash[33918]: cluster 2023-08-27T22:56:28.295579+0000 mgr.y (mgr.14901) 1738 : cluster [DBG] pgmap v1745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:29.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:29 smithi006 bash[33918]: audit 2023-08-27T22:56:29.045772+0000 mon.a (mon.0) 1753 : audit [DBG] from='client.? 172.21.15.6:0/99836164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:30.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:56:30.333Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:30.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:56:30.335Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:31.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:56:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:56:30] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T22:56:31.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:31 smithi006 bash[33918]: cluster 2023-08-27T22:56:30.296701+0000 mgr.y (mgr.14901) 1739 : cluster [DBG] pgmap v1746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:31.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:31 smithi006 bash[27292]: cluster 2023-08-27T22:56:30.296701+0000 mgr.y (mgr.14901) 1739 : cluster [DBG] pgmap v1746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:31.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:31 smithi130 bash[27579]: cluster 2023-08-27T22:56:30.296701+0000 mgr.y (mgr.14901) 1739 : cluster [DBG] pgmap v1746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:32.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:56:31.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:32.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:56:31.896Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:32.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:32 smithi006 bash[33918]: audit 2023-08-27T22:56:31.456335+0000 mon.a (mon.0) 1754 : audit [DBG] from='client.? 172.21.15.6:0/69058164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:32.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:32 smithi006 bash[27292]: audit 2023-08-27T22:56:31.456335+0000 mon.a (mon.0) 1754 : audit [DBG] from='client.? 172.21.15.6:0/69058164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:32 smithi130 bash[27579]: audit 2023-08-27T22:56:31.456335+0000 mon.a (mon.0) 1754 : audit [DBG] from='client.? 172.21.15.6:0/69058164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:33.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:33 smithi006 bash[33918]: cluster 2023-08-27T22:56:32.297363+0000 mgr.y (mgr.14901) 1740 : cluster [DBG] pgmap v1747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:33.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:33 smithi006 bash[27292]: cluster 2023-08-27T22:56:32.297363+0000 mgr.y (mgr.14901) 1740 : cluster [DBG] pgmap v1747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:33 smithi130 bash[27579]: cluster 2023-08-27T22:56:32.297363+0000 mgr.y (mgr.14901) 1740 : cluster [DBG] pgmap v1747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:34.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:56:33.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:34.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:56:33.793Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:34.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:34 smithi006 bash[27292]: audit 2023-08-27T22:56:33.866597+0000 mon.c (mon.2) 689 : audit [DBG] from='client.? 172.21.15.6:0/3314137053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:34.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:34 smithi006 bash[33918]: audit 2023-08-27T22:56:33.866597+0000 mon.c (mon.2) 689 : audit [DBG] from='client.? 172.21.15.6:0/3314137053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:34.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:34 smithi130 bash[27579]: audit 2023-08-27T22:56:33.866597+0000 mon.c (mon.2) 689 : audit [DBG] from='client.? 172.21.15.6:0/3314137053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:35.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:35 smithi006 bash[27292]: cluster 2023-08-27T22:56:34.298570+0000 mgr.y (mgr.14901) 1741 : cluster [DBG] pgmap v1748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:35.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:35 smithi006 bash[33918]: cluster 2023-08-27T22:56:34.298570+0000 mgr.y (mgr.14901) 1741 : cluster [DBG] pgmap v1748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:35.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:35 smithi130 bash[27579]: cluster 2023-08-27T22:56:34.298570+0000 mgr.y (mgr.14901) 1741 : cluster [DBG] pgmap v1748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:36.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:36 smithi006 bash[27292]: audit 2023-08-27T22:56:36.279405+0000 mon.a (mon.0) 1755 : audit [DBG] from='client.? 172.21.15.6:0/1406417384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:36.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:36 smithi006 bash[33918]: audit 2023-08-27T22:56:36.279405+0000 mon.a (mon.0) 1755 : audit [DBG] from='client.? 172.21.15.6:0/1406417384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:36 smithi130 bash[27579]: audit 2023-08-27T22:56:36.279405+0000 mon.a (mon.0) 1755 : audit [DBG] from='client.? 172.21.15.6:0/1406417384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:37.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:37 smithi006 bash[27292]: cluster 2023-08-27T22:56:36.299320+0000 mgr.y (mgr.14901) 1742 : cluster [DBG] pgmap v1749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:37.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:37 smithi006 bash[33918]: cluster 2023-08-27T22:56:36.299320+0000 mgr.y (mgr.14901) 1742 : cluster [DBG] pgmap v1749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:37.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:37 smithi130 bash[27579]: cluster 2023-08-27T22:56:36.299320+0000 mgr.y (mgr.14901) 1742 : cluster [DBG] pgmap v1749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:39.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:39 smithi006 bash[33918]: cluster 2023-08-27T22:56:38.300130+0000 mgr.y (mgr.14901) 1743 : cluster [DBG] pgmap v1750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:39.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:39 smithi006 bash[33918]: audit 2023-08-27T22:56:38.698588+0000 mon.a (mon.0) 1756 : audit [DBG] from='client.? 172.21.15.6:0/2381297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:39.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:39 smithi006 bash[27292]: cluster 2023-08-27T22:56:38.300130+0000 mgr.y (mgr.14901) 1743 : cluster [DBG] pgmap v1750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:39.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:39 smithi006 bash[27292]: audit 2023-08-27T22:56:38.698588+0000 mon.a (mon.0) 1756 : audit [DBG] from='client.? 172.21.15.6:0/2381297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:39.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:39 smithi130 bash[27579]: cluster 2023-08-27T22:56:38.300130+0000 mgr.y (mgr.14901) 1743 : cluster [DBG] pgmap v1750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:39.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:39 smithi130 bash[27579]: audit 2023-08-27T22:56:38.698588+0000 mon.a (mon.0) 1756 : audit [DBG] from='client.? 172.21.15.6:0/2381297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:40.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:56:40.334Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:40.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:56:40.336Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:41.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:56:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:56:40] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T22:56:41.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:41 smithi006 bash[33918]: cluster 2023-08-27T22:56:40.301348+0000 mgr.y (mgr.14901) 1744 : cluster [DBG] pgmap v1751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:41.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:41 smithi006 bash[33918]: audit 2023-08-27T22:56:41.107936+0000 mon.a (mon.0) 1757 : audit [DBG] from='client.? 172.21.15.6:0/1899935536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:41.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:41 smithi006 bash[27292]: cluster 2023-08-27T22:56:40.301348+0000 mgr.y (mgr.14901) 1744 : cluster [DBG] pgmap v1751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:41.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:41 smithi006 bash[27292]: audit 2023-08-27T22:56:41.107936+0000 mon.a (mon.0) 1757 : audit [DBG] from='client.? 172.21.15.6:0/1899935536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:41.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:41 smithi130 bash[27579]: cluster 2023-08-27T22:56:40.301348+0000 mgr.y (mgr.14901) 1744 : cluster [DBG] pgmap v1751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:41.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:41 smithi130 bash[27579]: audit 2023-08-27T22:56:41.107936+0000 mon.a (mon.0) 1757 : audit [DBG] from='client.? 172.21.15.6:0/1899935536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:42.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:56:41.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:42.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:56:41.896Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:43.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:43 smithi006 bash[33918]: cluster 2023-08-27T22:56:42.301997+0000 mgr.y (mgr.14901) 1745 : cluster [DBG] pgmap v1752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:43.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:43 smithi006 bash[27292]: cluster 2023-08-27T22:56:42.301997+0000 mgr.y (mgr.14901) 1745 : cluster [DBG] pgmap v1752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:43.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:43 smithi130 bash[27579]: cluster 2023-08-27T22:56:42.301997+0000 mgr.y (mgr.14901) 1745 : cluster [DBG] pgmap v1752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:44.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:56:43.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:44.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:56:43.794Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:44.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:44 smithi006 bash[27292]: audit 2023-08-27T22:56:43.527510+0000 mon.a (mon.0) 1758 : audit [DBG] from='client.? 172.21.15.6:0/3655476101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:44.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:44 smithi006 bash[33918]: audit 2023-08-27T22:56:43.527510+0000 mon.a (mon.0) 1758 : audit [DBG] from='client.? 172.21.15.6:0/3655476101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:44 smithi130 bash[27579]: audit 2023-08-27T22:56:43.527510+0000 mon.a (mon.0) 1758 : audit [DBG] from='client.? 172.21.15.6:0/3655476101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:45.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:45 smithi006 bash[27292]: cluster 2023-08-27T22:56:44.303178+0000 mgr.y (mgr.14901) 1746 : cluster [DBG] pgmap v1753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:45.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:45 smithi006 bash[33918]: cluster 2023-08-27T22:56:44.303178+0000 mgr.y (mgr.14901) 1746 : cluster [DBG] pgmap v1753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:45.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:45 smithi130 bash[27579]: cluster 2023-08-27T22:56:44.303178+0000 mgr.y (mgr.14901) 1746 : cluster [DBG] pgmap v1753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:46.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:46 smithi006 bash[33918]: audit 2023-08-27T22:56:45.944938+0000 mon.c (mon.2) 690 : audit [DBG] from='client.? 172.21.15.6:0/1140361644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:46.914 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:46 smithi006 bash[33918]: audit 2023-08-27T22:56:45.986328+0000 mon.b (mon.1) 288 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:56:46.914 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:46 smithi006 bash[27292]: audit 2023-08-27T22:56:45.944938+0000 mon.c (mon.2) 690 : audit [DBG] from='client.? 172.21.15.6:0/1140361644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:46.914 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:46 smithi006 bash[27292]: audit 2023-08-27T22:56:45.986328+0000 mon.b (mon.1) 288 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:56:46.915 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:46 smithi130 bash[27579]: audit 2023-08-27T22:56:45.944938+0000 mon.c (mon.2) 690 : audit [DBG] from='client.? 172.21.15.6:0/1140361644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:46.915 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:46 smithi130 bash[27579]: audit 2023-08-27T22:56:45.986328+0000 mon.b (mon.1) 288 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:56:47.722 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:47 smithi006 bash[27292]: cluster 2023-08-27T22:56:46.303971+0000 mgr.y (mgr.14901) 1747 : cluster [DBG] pgmap v1754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:47.722 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:47 smithi006 bash[33918]: cluster 2023-08-27T22:56:46.303971+0000 mgr.y (mgr.14901) 1747 : cluster [DBG] pgmap v1754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:47.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:47 smithi130 bash[27579]: cluster 2023-08-27T22:56:46.303971+0000 mgr.y (mgr.14901) 1747 : cluster [DBG] pgmap v1754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:48.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:48 smithi006 bash[33918]: cluster 2023-08-27T22:56:48.304677+0000 mgr.y (mgr.14901) 1748 : cluster [DBG] pgmap v1755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:48.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:48 smithi006 bash[33918]: audit 2023-08-27T22:56:48.357475+0000 mon.c (mon.2) 691 : audit [DBG] from='client.? 172.21.15.6:0/4012201086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:48.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:48 smithi006 bash[27292]: cluster 2023-08-27T22:56:48.304677+0000 mgr.y (mgr.14901) 1748 : cluster [DBG] pgmap v1755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:48.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:48 smithi006 bash[27292]: audit 2023-08-27T22:56:48.357475+0000 mon.c (mon.2) 691 : audit [DBG] from='client.? 172.21.15.6:0/4012201086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:48 smithi130 bash[27579]: cluster 2023-08-27T22:56:48.304677+0000 mgr.y (mgr.14901) 1748 : cluster [DBG] pgmap v1755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:48 smithi130 bash[27579]: audit 2023-08-27T22:56:48.357475+0000 mon.c (mon.2) 691 : audit [DBG] from='client.? 172.21.15.6:0/4012201086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:50.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:56:50.335Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:50.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:56:50.336Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:51.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:56:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:56:50] "GET /metrics HTTP/1.1" 200 193082 "" "Prometheus/2.33.4" 2023-08-27T22:56:51.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:51 smithi006 bash[27292]: cluster 2023-08-27T22:56:50.305591+0000 mgr.y (mgr.14901) 1749 : cluster [DBG] pgmap v1756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:51.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:51 smithi006 bash[27292]: audit 2023-08-27T22:56:50.777257+0000 mon.c (mon.2) 692 : audit [DBG] from='client.? 172.21.15.6:0/884710326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:51.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:51 smithi006 bash[33918]: cluster 2023-08-27T22:56:50.305591+0000 mgr.y (mgr.14901) 1749 : cluster [DBG] pgmap v1756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:51.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:51 smithi006 bash[33918]: audit 2023-08-27T22:56:50.777257+0000 mon.c (mon.2) 692 : audit [DBG] from='client.? 172.21.15.6:0/884710326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:51.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:51 smithi130 bash[27579]: cluster 2023-08-27T22:56:50.305591+0000 mgr.y (mgr.14901) 1749 : cluster [DBG] pgmap v1756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:51.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:51 smithi130 bash[27579]: audit 2023-08-27T22:56:50.777257+0000 mon.c (mon.2) 692 : audit [DBG] from='client.? 172.21.15.6:0/884710326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:52.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:56:51.895Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:52.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:56:51.897Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:53.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:52 smithi006 bash[27292]: audit 2023-08-27T22:56:51.860123+0000 mon.a (mon.0) 1759 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:56:53.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:52 smithi006 bash[27292]: cluster 2023-08-27T22:56:52.306325+0000 mgr.y (mgr.14901) 1750 : cluster [DBG] pgmap v1757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:53.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:52 smithi006 bash[27292]: audit 2023-08-27T22:56:52.316135+0000 mon.a (mon.0) 1760 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:56:53.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:52 smithi006 bash[27292]: audit 2023-08-27T22:56:52.583145+0000 mon.b (mon.1) 289 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:56:53.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:52 smithi006 bash[27292]: audit 2023-08-27T22:56:52.585310+0000 mon.b (mon.1) 290 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:56:53.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:52 smithi006 bash[27292]: audit 2023-08-27T22:56:52.590731+0000 mon.a (mon.0) 1761 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:56:53.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:52 smithi006 bash[33918]: audit 2023-08-27T22:56:51.860123+0000 mon.a (mon.0) 1759 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:56:53.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:52 smithi006 bash[33918]: cluster 2023-08-27T22:56:52.306325+0000 mgr.y (mgr.14901) 1750 : cluster [DBG] pgmap v1757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:53.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:52 smithi006 bash[33918]: audit 2023-08-27T22:56:52.316135+0000 mon.a (mon.0) 1760 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:56:53.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:52 smithi006 bash[33918]: audit 2023-08-27T22:56:52.583145+0000 mon.b (mon.1) 289 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:56:53.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:52 smithi006 bash[33918]: audit 2023-08-27T22:56:52.585310+0000 mon.b (mon.1) 290 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:56:53.221 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:52 smithi006 bash[33918]: audit 2023-08-27T22:56:52.590731+0000 mon.a (mon.0) 1761 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:56:53.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:52 smithi130 bash[27579]: audit 2023-08-27T22:56:51.860123+0000 mon.a (mon.0) 1759 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:56:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:52 smithi130 bash[27579]: cluster 2023-08-27T22:56:52.306325+0000 mgr.y (mgr.14901) 1750 : cluster [DBG] pgmap v1757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:52 smithi130 bash[27579]: audit 2023-08-27T22:56:52.316135+0000 mon.a (mon.0) 1760 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:56:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:52 smithi130 bash[27579]: audit 2023-08-27T22:56:52.583145+0000 mon.b (mon.1) 289 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:56:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:52 smithi130 bash[27579]: audit 2023-08-27T22:56:52.585310+0000 mon.b (mon.1) 290 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:56:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:52 smithi130 bash[27579]: audit 2023-08-27T22:56:52.590731+0000 mon.a (mon.0) 1761 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:56:54.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:53 smithi006 bash[33918]: audit 2023-08-27T22:56:53.210708+0000 mon.a (mon.0) 1762 : audit [DBG] from='client.? 172.21.15.6:0/239444422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:54.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:53 smithi006 bash[27292]: audit 2023-08-27T22:56:53.210708+0000 mon.a (mon.0) 1762 : audit [DBG] from='client.? 172.21.15.6:0/239444422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:54.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:56:53.793Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:54.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:56:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:56:53.795Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:56:54.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:53 smithi130 bash[27579]: audit 2023-08-27T22:56:53.210708+0000 mon.a (mon.0) 1762 : audit [DBG] from='client.? 172.21.15.6:0/239444422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:55.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:55 smithi006 bash[33918]: cluster 2023-08-27T22:56:54.307376+0000 mgr.y (mgr.14901) 1751 : cluster [DBG] pgmap v1758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:55.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:55 smithi006 bash[27292]: cluster 2023-08-27T22:56:54.307376+0000 mgr.y (mgr.14901) 1751 : cluster [DBG] pgmap v1758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:55.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:55 smithi130 bash[27579]: cluster 2023-08-27T22:56:54.307376+0000 mgr.y (mgr.14901) 1751 : cluster [DBG] pgmap v1758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:56.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:56 smithi006 bash[27292]: audit 2023-08-27T22:56:55.630739+0000 mon.a (mon.0) 1763 : audit [DBG] from='client.? 172.21.15.6:0/3184545243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:56.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:56 smithi006 bash[33918]: audit 2023-08-27T22:56:55.630739+0000 mon.a (mon.0) 1763 : audit [DBG] from='client.? 172.21.15.6:0/3184545243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:56.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:56 smithi130 bash[27579]: audit 2023-08-27T22:56:55.630739+0000 mon.a (mon.0) 1763 : audit [DBG] from='client.? 172.21.15.6:0/3184545243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:57.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:57 smithi006 bash[27292]: cluster 2023-08-27T22:56:56.308103+0000 mgr.y (mgr.14901) 1752 : cluster [DBG] pgmap v1759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:57.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:57 smithi006 bash[33918]: cluster 2023-08-27T22:56:56.308103+0000 mgr.y (mgr.14901) 1752 : cluster [DBG] pgmap v1759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:57.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:57 smithi130 bash[27579]: cluster 2023-08-27T22:56:56.308103+0000 mgr.y (mgr.14901) 1752 : cluster [DBG] pgmap v1759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:58.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:58 smithi006 bash[27292]: audit 2023-08-27T22:56:58.046445+0000 mon.c (mon.2) 693 : audit [DBG] from='client.? 172.21.15.6:0/667455977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:58.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:58 smithi006 bash[33918]: audit 2023-08-27T22:56:58.046445+0000 mon.c (mon.2) 693 : audit [DBG] from='client.? 172.21.15.6:0/667455977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:58.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:58 smithi130 bash[27579]: audit 2023-08-27T22:56:58.046445+0000 mon.c (mon.2) 693 : audit [DBG] from='client.? 172.21.15.6:0/667455977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:59.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:56:59 smithi006 bash[27292]: cluster 2023-08-27T22:56:58.308927+0000 mgr.y (mgr.14901) 1753 : cluster [DBG] pgmap v1760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:59.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:56:59 smithi006 bash[33918]: cluster 2023-08-27T22:56:58.308927+0000 mgr.y (mgr.14901) 1753 : cluster [DBG] pgmap v1760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:56:59.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:56:59 smithi130 bash[27579]: cluster 2023-08-27T22:56:58.308927+0000 mgr.y (mgr.14901) 1753 : cluster [DBG] pgmap v1760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:00.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:57:00.335Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:00.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:57:00.337Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:01.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:57:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:57:00] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T22:57:01.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:01 smithi006 bash[33918]: cluster 2023-08-27T22:57:00.309945+0000 mgr.y (mgr.14901) 1754 : cluster [DBG] pgmap v1761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:01.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:01 smithi006 bash[33918]: audit 2023-08-27T22:57:00.464012+0000 mon.a (mon.0) 1764 : audit [DBG] from='client.? 172.21.15.6:0/3135514585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:01.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:01 smithi006 bash[27292]: cluster 2023-08-27T22:57:00.309945+0000 mgr.y (mgr.14901) 1754 : cluster [DBG] pgmap v1761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:01.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:01 smithi006 bash[27292]: audit 2023-08-27T22:57:00.464012+0000 mon.a (mon.0) 1764 : audit [DBG] from='client.? 172.21.15.6:0/3135514585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:01.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:01 smithi130 bash[27579]: cluster 2023-08-27T22:57:00.309945+0000 mgr.y (mgr.14901) 1754 : cluster [DBG] pgmap v1761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:01.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:01 smithi130 bash[27579]: audit 2023-08-27T22:57:00.464012+0000 mon.a (mon.0) 1764 : audit [DBG] from='client.? 172.21.15.6:0/3135514585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:02.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:57:01.896Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:02.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:57:01.898Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:03.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:03 smithi006 bash[27292]: cluster 2023-08-27T22:57:02.310607+0000 mgr.y (mgr.14901) 1755 : cluster [DBG] pgmap v1762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:03.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:03 smithi006 bash[27292]: audit 2023-08-27T22:57:02.875044+0000 mon.a (mon.0) 1765 : audit [DBG] from='client.? 172.21.15.6:0/2569509128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:03.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:03 smithi006 bash[33918]: cluster 2023-08-27T22:57:02.310607+0000 mgr.y (mgr.14901) 1755 : cluster [DBG] pgmap v1762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:03.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:03 smithi006 bash[33918]: audit 2023-08-27T22:57:02.875044+0000 mon.a (mon.0) 1765 : audit [DBG] from='client.? 172.21.15.6:0/2569509128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:03.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:03 smithi130 bash[27579]: cluster 2023-08-27T22:57:02.310607+0000 mgr.y (mgr.14901) 1755 : cluster [DBG] pgmap v1762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:03.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:03 smithi130 bash[27579]: audit 2023-08-27T22:57:02.875044+0000 mon.a (mon.0) 1765 : audit [DBG] from='client.? 172.21.15.6:0/2569509128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:04.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:57:03.793Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:04.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:57:03.795Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:05.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:05 smithi006 bash[27292]: cluster 2023-08-27T22:57:04.311715+0000 mgr.y (mgr.14901) 1756 : cluster [DBG] pgmap v1763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:05.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:05 smithi006 bash[33918]: cluster 2023-08-27T22:57:04.311715+0000 mgr.y (mgr.14901) 1756 : cluster [DBG] pgmap v1763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:05.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:05 smithi130 bash[27579]: cluster 2023-08-27T22:57:04.311715+0000 mgr.y (mgr.14901) 1756 : cluster [DBG] pgmap v1763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:06.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:06 smithi006 bash[27292]: audit 2023-08-27T22:57:05.290483+0000 mon.a (mon.0) 1766 : audit [DBG] from='client.? 172.21.15.6:0/4191732558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:06.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:06 smithi006 bash[33918]: audit 2023-08-27T22:57:05.290483+0000 mon.a (mon.0) 1766 : audit [DBG] from='client.? 172.21.15.6:0/4191732558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:06.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:06 smithi130 bash[27579]: audit 2023-08-27T22:57:05.290483+0000 mon.a (mon.0) 1766 : audit [DBG] from='client.? 172.21.15.6:0/4191732558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:07.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:07 smithi006 bash[27292]: cluster 2023-08-27T22:57:06.312475+0000 mgr.y (mgr.14901) 1757 : cluster [DBG] pgmap v1764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:07.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:07 smithi006 bash[33918]: cluster 2023-08-27T22:57:06.312475+0000 mgr.y (mgr.14901) 1757 : cluster [DBG] pgmap v1764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:07.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:07 smithi130 bash[27579]: cluster 2023-08-27T22:57:06.312475+0000 mgr.y (mgr.14901) 1757 : cluster [DBG] pgmap v1764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:08.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:08 smithi006 bash[27292]: audit 2023-08-27T22:57:07.697480+0000 mon.a (mon.0) 1767 : audit [DBG] from='client.? 172.21.15.6:0/2382552436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:08.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:08 smithi006 bash[33918]: audit 2023-08-27T22:57:07.697480+0000 mon.a (mon.0) 1767 : audit [DBG] from='client.? 172.21.15.6:0/2382552436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:08.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:08 smithi130 bash[27579]: audit 2023-08-27T22:57:07.697480+0000 mon.a (mon.0) 1767 : audit [DBG] from='client.? 172.21.15.6:0/2382552436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:09.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:09 smithi006 bash[27292]: cluster 2023-08-27T22:57:08.313200+0000 mgr.y (mgr.14901) 1758 : cluster [DBG] pgmap v1765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:09.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:09 smithi006 bash[33918]: cluster 2023-08-27T22:57:08.313200+0000 mgr.y (mgr.14901) 1758 : cluster [DBG] pgmap v1765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:09 smithi130 bash[27579]: cluster 2023-08-27T22:57:08.313200+0000 mgr.y (mgr.14901) 1758 : cluster [DBG] pgmap v1765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:10.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:10 smithi006 bash[27292]: audit 2023-08-27T22:57:10.114132+0000 mon.a (mon.0) 1768 : audit [DBG] from='client.? 172.21.15.6:0/1464787676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:10.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:10 smithi006 bash[33918]: audit 2023-08-27T22:57:10.114132+0000 mon.a (mon.0) 1768 : audit [DBG] from='client.? 172.21.15.6:0/1464787676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:10.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:57:10.336Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:10.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:57:10.338Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:10.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:10 smithi130 bash[27579]: audit 2023-08-27T22:57:10.114132+0000 mon.a (mon.0) 1768 : audit [DBG] from='client.? 172.21.15.6:0/1464787676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:11.179 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:57:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:57:10] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T22:57:11.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:11 smithi006 bash[27292]: cluster 2023-08-27T22:57:10.314499+0000 mgr.y (mgr.14901) 1759 : cluster [DBG] pgmap v1766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:11.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:11 smithi006 bash[33918]: cluster 2023-08-27T22:57:10.314499+0000 mgr.y (mgr.14901) 1759 : cluster [DBG] pgmap v1766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:11.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:11 smithi130 bash[27579]: cluster 2023-08-27T22:57:10.314499+0000 mgr.y (mgr.14901) 1759 : cluster [DBG] pgmap v1766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:12.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:57:11.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:12.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:57:11.899Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:13.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:13 smithi006 bash[27292]: cluster 2023-08-27T22:57:12.315196+0000 mgr.y (mgr.14901) 1760 : cluster [DBG] pgmap v1767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:13.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:13 smithi006 bash[27292]: audit 2023-08-27T22:57:12.527454+0000 mon.a (mon.0) 1769 : audit [DBG] from='client.? 172.21.15.6:0/3337381678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:13.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:13 smithi006 bash[33918]: cluster 2023-08-27T22:57:12.315196+0000 mgr.y (mgr.14901) 1760 : cluster [DBG] pgmap v1767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:13.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:13 smithi006 bash[33918]: audit 2023-08-27T22:57:12.527454+0000 mon.a (mon.0) 1769 : audit [DBG] from='client.? 172.21.15.6:0/3337381678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:13.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:13 smithi130 bash[27579]: cluster 2023-08-27T22:57:12.315196+0000 mgr.y (mgr.14901) 1760 : cluster [DBG] pgmap v1767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:13.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:13 smithi130 bash[27579]: audit 2023-08-27T22:57:12.527454+0000 mon.a (mon.0) 1769 : audit [DBG] from='client.? 172.21.15.6:0/3337381678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:14.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:57:13.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:14.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:57:13.796Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:15.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:15 smithi006 bash[27292]: cluster 2023-08-27T22:57:14.316308+0000 mgr.y (mgr.14901) 1761 : cluster [DBG] pgmap v1768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:15.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:15 smithi006 bash[27292]: audit 2023-08-27T22:57:14.949271+0000 mon.c (mon.2) 694 : audit [DBG] from='client.? 172.21.15.6:0/2586021404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:15.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:15 smithi006 bash[33918]: cluster 2023-08-27T22:57:14.316308+0000 mgr.y (mgr.14901) 1761 : cluster [DBG] pgmap v1768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:15.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:15 smithi006 bash[33918]: audit 2023-08-27T22:57:14.949271+0000 mon.c (mon.2) 694 : audit [DBG] from='client.? 172.21.15.6:0/2586021404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:15.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:15 smithi130 bash[27579]: cluster 2023-08-27T22:57:14.316308+0000 mgr.y (mgr.14901) 1761 : cluster [DBG] pgmap v1768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:15.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:15 smithi130 bash[27579]: audit 2023-08-27T22:57:14.949271+0000 mon.c (mon.2) 694 : audit [DBG] from='client.? 172.21.15.6:0/2586021404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:17.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:17 smithi006 bash[27292]: cluster 2023-08-27T22:57:16.316989+0000 mgr.y (mgr.14901) 1762 : cluster [DBG] pgmap v1769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:17.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:17 smithi006 bash[27292]: audit 2023-08-27T22:57:17.364952+0000 mon.c (mon.2) 695 : audit [DBG] from='client.? 172.21.15.6:0/3697912411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:17.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:17 smithi006 bash[33918]: cluster 2023-08-27T22:57:16.316989+0000 mgr.y (mgr.14901) 1762 : cluster [DBG] pgmap v1769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:17.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:17 smithi006 bash[33918]: audit 2023-08-27T22:57:17.364952+0000 mon.c (mon.2) 695 : audit [DBG] from='client.? 172.21.15.6:0/3697912411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:17.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:17 smithi130 bash[27579]: cluster 2023-08-27T22:57:16.316989+0000 mgr.y (mgr.14901) 1762 : cluster [DBG] pgmap v1769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:17.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:17 smithi130 bash[27579]: audit 2023-08-27T22:57:17.364952+0000 mon.c (mon.2) 695 : audit [DBG] from='client.? 172.21.15.6:0/3697912411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:19.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:19 smithi006 bash[33918]: cluster 2023-08-27T22:57:18.317629+0000 mgr.y (mgr.14901) 1763 : cluster [DBG] pgmap v1770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:19.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:19 smithi006 bash[27292]: cluster 2023-08-27T22:57:18.317629+0000 mgr.y (mgr.14901) 1763 : cluster [DBG] pgmap v1770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:19.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:19 smithi130 bash[27579]: cluster 2023-08-27T22:57:18.317629+0000 mgr.y (mgr.14901) 1763 : cluster [DBG] pgmap v1770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:20.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:20 smithi006 bash[33918]: audit 2023-08-27T22:57:19.781797+0000 mon.c (mon.2) 696 : audit [DBG] from='client.? 172.21.15.6:0/3709430730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:20.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:20 smithi006 bash[27292]: audit 2023-08-27T22:57:19.781797+0000 mon.c (mon.2) 696 : audit [DBG] from='client.? 172.21.15.6:0/3709430730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:20.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:57:20.336Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:20.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:57:20.338Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:20.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:20 smithi130 bash[27579]: audit 2023-08-27T22:57:19.781797+0000 mon.c (mon.2) 696 : audit [DBG] from='client.? 172.21.15.6:0/3709430730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:21.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:57:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:57:20] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T22:57:21.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:21 smithi006 bash[33918]: cluster 2023-08-27T22:57:20.318767+0000 mgr.y (mgr.14901) 1764 : cluster [DBG] pgmap v1771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:21.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:21 smithi006 bash[27292]: cluster 2023-08-27T22:57:20.318767+0000 mgr.y (mgr.14901) 1764 : cluster [DBG] pgmap v1771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:21.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:21 smithi130 bash[27579]: cluster 2023-08-27T22:57:20.318767+0000 mgr.y (mgr.14901) 1764 : cluster [DBG] pgmap v1771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:22.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:57:21.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:22.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:57:21.900Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:22.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:22 smithi006 bash[33918]: audit 2023-08-27T22:57:22.202918+0000 mon.c (mon.2) 697 : audit [DBG] from='client.? 172.21.15.6:0/3563134636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:22.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:22 smithi006 bash[27292]: audit 2023-08-27T22:57:22.202918+0000 mon.c (mon.2) 697 : audit [DBG] from='client.? 172.21.15.6:0/3563134636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:22 smithi130 bash[27579]: audit 2023-08-27T22:57:22.202918+0000 mon.c (mon.2) 697 : audit [DBG] from='client.? 172.21.15.6:0/3563134636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:23.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:23 smithi006 bash[27292]: cluster 2023-08-27T22:57:22.319444+0000 mgr.y (mgr.14901) 1765 : cluster [DBG] pgmap v1772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:23.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:23 smithi006 bash[33918]: cluster 2023-08-27T22:57:22.319444+0000 mgr.y (mgr.14901) 1765 : cluster [DBG] pgmap v1772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:23.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:23 smithi130 bash[27579]: cluster 2023-08-27T22:57:22.319444+0000 mgr.y (mgr.14901) 1765 : cluster [DBG] pgmap v1772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:24.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:57:23.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:24.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:57:23.797Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:25.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:25 smithi006 bash[27292]: cluster 2023-08-27T22:57:24.320504+0000 mgr.y (mgr.14901) 1766 : cluster [DBG] pgmap v1773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:25.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:25 smithi006 bash[27292]: audit 2023-08-27T22:57:24.619993+0000 mon.c (mon.2) 698 : audit [DBG] from='client.? 172.21.15.6:0/1212771945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:25.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:25 smithi006 bash[33918]: cluster 2023-08-27T22:57:24.320504+0000 mgr.y (mgr.14901) 1766 : cluster [DBG] pgmap v1773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:25.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:25 smithi006 bash[33918]: audit 2023-08-27T22:57:24.619993+0000 mon.c (mon.2) 698 : audit [DBG] from='client.? 172.21.15.6:0/1212771945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:25.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:25 smithi130 bash[27579]: cluster 2023-08-27T22:57:24.320504+0000 mgr.y (mgr.14901) 1766 : cluster [DBG] pgmap v1773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:25.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:25 smithi130 bash[27579]: audit 2023-08-27T22:57:24.619993+0000 mon.c (mon.2) 698 : audit [DBG] from='client.? 172.21.15.6:0/1212771945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:27.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:27 smithi006 bash[33918]: cluster 2023-08-27T22:57:26.321144+0000 mgr.y (mgr.14901) 1767 : cluster [DBG] pgmap v1774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:27.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:27 smithi006 bash[33918]: audit 2023-08-27T22:57:27.029816+0000 mon.c (mon.2) 699 : audit [DBG] from='client.? 172.21.15.6:0/748317840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:27.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:27 smithi006 bash[27292]: cluster 2023-08-27T22:57:26.321144+0000 mgr.y (mgr.14901) 1767 : cluster [DBG] pgmap v1774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:27.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:27 smithi006 bash[27292]: audit 2023-08-27T22:57:27.029816+0000 mon.c (mon.2) 699 : audit [DBG] from='client.? 172.21.15.6:0/748317840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:27.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:27 smithi130 bash[27579]: cluster 2023-08-27T22:57:26.321144+0000 mgr.y (mgr.14901) 1767 : cluster [DBG] pgmap v1774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:27 smithi130 bash[27579]: audit 2023-08-27T22:57:27.029816+0000 mon.c (mon.2) 699 : audit [DBG] from='client.? 172.21.15.6:0/748317840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:29.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:29 smithi006 bash[27292]: cluster 2023-08-27T22:57:28.321790+0000 mgr.y (mgr.14901) 1768 : cluster [DBG] pgmap v1775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:29.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:29 smithi006 bash[33918]: cluster 2023-08-27T22:57:28.321790+0000 mgr.y (mgr.14901) 1768 : cluster [DBG] pgmap v1775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:29.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:29 smithi130 bash[27579]: cluster 2023-08-27T22:57:28.321790+0000 mgr.y (mgr.14901) 1768 : cluster [DBG] pgmap v1775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:30.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:30 smithi006 bash[27292]: audit 2023-08-27T22:57:29.443222+0000 mon.c (mon.2) 700 : audit [DBG] from='client.? 172.21.15.6:0/4010979233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:30.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:30 smithi006 bash[33918]: audit 2023-08-27T22:57:29.443222+0000 mon.c (mon.2) 700 : audit [DBG] from='client.? 172.21.15.6:0/4010979233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:30.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:57:30.337Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:30.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:57:30.339Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:30.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:30 smithi130 bash[27579]: audit 2023-08-27T22:57:29.443222+0000 mon.c (mon.2) 700 : audit [DBG] from='client.? 172.21.15.6:0/4010979233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:31.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:57:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:57:30] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T22:57:31.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:31 smithi006 bash[27292]: cluster 2023-08-27T22:57:30.322946+0000 mgr.y (mgr.14901) 1769 : cluster [DBG] pgmap v1776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:31.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:31 smithi006 bash[33918]: cluster 2023-08-27T22:57:30.322946+0000 mgr.y (mgr.14901) 1769 : cluster [DBG] pgmap v1776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:31.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:31 smithi130 bash[27579]: cluster 2023-08-27T22:57:30.322946+0000 mgr.y (mgr.14901) 1769 : cluster [DBG] pgmap v1776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:32.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:57:31.899Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:32.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:57:31.901Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:32.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:32 smithi006 bash[27292]: audit 2023-08-27T22:57:31.859606+0000 mon.a (mon.0) 1770 : audit [DBG] from='client.? 172.21.15.6:0/3826003170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:32.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:32 smithi006 bash[33918]: audit 2023-08-27T22:57:31.859606+0000 mon.a (mon.0) 1770 : audit [DBG] from='client.? 172.21.15.6:0/3826003170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:32 smithi130 bash[27579]: audit 2023-08-27T22:57:31.859606+0000 mon.a (mon.0) 1770 : audit [DBG] from='client.? 172.21.15.6:0/3826003170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:33.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:33 smithi006 bash[27292]: cluster 2023-08-27T22:57:32.323584+0000 mgr.y (mgr.14901) 1770 : cluster [DBG] pgmap v1777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:33.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:33 smithi006 bash[33918]: cluster 2023-08-27T22:57:32.323584+0000 mgr.y (mgr.14901) 1770 : cluster [DBG] pgmap v1777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:33 smithi130 bash[27579]: cluster 2023-08-27T22:57:32.323584+0000 mgr.y (mgr.14901) 1770 : cluster [DBG] pgmap v1777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:34.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:57:33.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:34.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:57:33.798Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:35.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:35 smithi006 bash[27292]: audit 2023-08-27T22:57:34.286185+0000 mon.c (mon.2) 701 : audit [DBG] from='client.? 172.21.15.6:0/1835559955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:35.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:35 smithi006 bash[27292]: cluster 2023-08-27T22:57:34.324694+0000 mgr.y (mgr.14901) 1771 : cluster [DBG] pgmap v1778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:35.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:35 smithi006 bash[33918]: audit 2023-08-27T22:57:34.286185+0000 mon.c (mon.2) 701 : audit [DBG] from='client.? 172.21.15.6:0/1835559955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:35.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:35 smithi006 bash[33918]: cluster 2023-08-27T22:57:34.324694+0000 mgr.y (mgr.14901) 1771 : cluster [DBG] pgmap v1778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:35.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:35 smithi130 bash[27579]: audit 2023-08-27T22:57:34.286185+0000 mon.c (mon.2) 701 : audit [DBG] from='client.? 172.21.15.6:0/1835559955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:35.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:35 smithi130 bash[27579]: cluster 2023-08-27T22:57:34.324694+0000 mgr.y (mgr.14901) 1771 : cluster [DBG] pgmap v1778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:37.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:37 smithi006 bash[33918]: cluster 2023-08-27T22:57:36.325250+0000 mgr.y (mgr.14901) 1772 : cluster [DBG] pgmap v1779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:37.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:37 smithi006 bash[33918]: audit 2023-08-27T22:57:36.705183+0000 mon.c (mon.2) 702 : audit [DBG] from='client.? 172.21.15.6:0/656333038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:37.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:37 smithi006 bash[27292]: cluster 2023-08-27T22:57:36.325250+0000 mgr.y (mgr.14901) 1772 : cluster [DBG] pgmap v1779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:37.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:37 smithi006 bash[27292]: audit 2023-08-27T22:57:36.705183+0000 mon.c (mon.2) 702 : audit [DBG] from='client.? 172.21.15.6:0/656333038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:37.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:37 smithi130 bash[27579]: cluster 2023-08-27T22:57:36.325250+0000 mgr.y (mgr.14901) 1772 : cluster [DBG] pgmap v1779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:37.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:37 smithi130 bash[27579]: audit 2023-08-27T22:57:36.705183+0000 mon.c (mon.2) 702 : audit [DBG] from='client.? 172.21.15.6:0/656333038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:39.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:39 smithi006 bash[27292]: cluster 2023-08-27T22:57:38.325919+0000 mgr.y (mgr.14901) 1773 : cluster [DBG] pgmap v1780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:39.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:39 smithi006 bash[27292]: audit 2023-08-27T22:57:39.121310+0000 mon.a (mon.0) 1771 : audit [DBG] from='client.? 172.21.15.6:0/3654538622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:39.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:39 smithi006 bash[33918]: cluster 2023-08-27T22:57:38.325919+0000 mgr.y (mgr.14901) 1773 : cluster [DBG] pgmap v1780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:39.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:39 smithi006 bash[33918]: audit 2023-08-27T22:57:39.121310+0000 mon.a (mon.0) 1771 : audit [DBG] from='client.? 172.21.15.6:0/3654538622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:39.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:39 smithi130 bash[27579]: cluster 2023-08-27T22:57:38.325919+0000 mgr.y (mgr.14901) 1773 : cluster [DBG] pgmap v1780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:39.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:39 smithi130 bash[27579]: audit 2023-08-27T22:57:39.121310+0000 mon.a (mon.0) 1771 : audit [DBG] from='client.? 172.21.15.6:0/3654538622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:40.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:57:40.339Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:40.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:57:40.340Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:41.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:57:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:57:40] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T22:57:41.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:41 smithi006 bash[27292]: cluster 2023-08-27T22:57:40.327221+0000 mgr.y (mgr.14901) 1774 : cluster [DBG] pgmap v1781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:41.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:41 smithi006 bash[33918]: cluster 2023-08-27T22:57:40.327221+0000 mgr.y (mgr.14901) 1774 : cluster [DBG] pgmap v1781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:41.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:41 smithi130 bash[27579]: cluster 2023-08-27T22:57:40.327221+0000 mgr.y (mgr.14901) 1774 : cluster [DBG] pgmap v1781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:42.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:57:41.900Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:42.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:57:41.902Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:42.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:42 smithi006 bash[33918]: audit 2023-08-27T22:57:41.535978+0000 mon.a (mon.0) 1772 : audit [DBG] from='client.? 172.21.15.6:0/3176097585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:42.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:42 smithi006 bash[27292]: audit 2023-08-27T22:57:41.535978+0000 mon.a (mon.0) 1772 : audit [DBG] from='client.? 172.21.15.6:0/3176097585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:42 smithi130 bash[27579]: audit 2023-08-27T22:57:41.535978+0000 mon.a (mon.0) 1772 : audit [DBG] from='client.? 172.21.15.6:0/3176097585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:43.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:43 smithi006 bash[33918]: cluster 2023-08-27T22:57:42.327912+0000 mgr.y (mgr.14901) 1775 : cluster [DBG] pgmap v1782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:43.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:43 smithi006 bash[27292]: cluster 2023-08-27T22:57:42.327912+0000 mgr.y (mgr.14901) 1775 : cluster [DBG] pgmap v1782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:43.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:43 smithi130 bash[27579]: cluster 2023-08-27T22:57:42.327912+0000 mgr.y (mgr.14901) 1775 : cluster [DBG] pgmap v1782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:44.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:57:43.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:44.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:57:43.798Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:44.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:44 smithi006 bash[27292]: audit 2023-08-27T22:57:43.951148+0000 mon.c (mon.2) 703 : audit [DBG] from='client.? 172.21.15.6:0/1024586629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:44.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:44 smithi006 bash[33918]: audit 2023-08-27T22:57:43.951148+0000 mon.c (mon.2) 703 : audit [DBG] from='client.? 172.21.15.6:0/1024586629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:44 smithi130 bash[27579]: audit 2023-08-27T22:57:43.951148+0000 mon.c (mon.2) 703 : audit [DBG] from='client.? 172.21.15.6:0/1024586629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:45.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:45 smithi006 bash[27292]: cluster 2023-08-27T22:57:44.329105+0000 mgr.y (mgr.14901) 1776 : cluster [DBG] pgmap v1783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:45.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:45 smithi006 bash[33918]: cluster 2023-08-27T22:57:44.329105+0000 mgr.y (mgr.14901) 1776 : cluster [DBG] pgmap v1783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:45.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:45 smithi130 bash[27579]: cluster 2023-08-27T22:57:44.329105+0000 mgr.y (mgr.14901) 1776 : cluster [DBG] pgmap v1783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:46.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:46 smithi006 bash[33918]: audit 2023-08-27T22:57:46.367025+0000 mon.c (mon.2) 704 : audit [DBG] from='client.? 172.21.15.6:0/439920332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:46.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:46 smithi006 bash[27292]: audit 2023-08-27T22:57:46.367025+0000 mon.c (mon.2) 704 : audit [DBG] from='client.? 172.21.15.6:0/439920332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:46 smithi130 bash[27579]: audit 2023-08-27T22:57:46.367025+0000 mon.c (mon.2) 704 : audit [DBG] from='client.? 172.21.15.6:0/439920332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:47.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:47 smithi006 bash[27292]: cluster 2023-08-27T22:57:46.329687+0000 mgr.y (mgr.14901) 1777 : cluster [DBG] pgmap v1784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:47.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:47 smithi006 bash[33918]: cluster 2023-08-27T22:57:46.329687+0000 mgr.y (mgr.14901) 1777 : cluster [DBG] pgmap v1784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:47.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:47 smithi130 bash[27579]: cluster 2023-08-27T22:57:46.329687+0000 mgr.y (mgr.14901) 1777 : cluster [DBG] pgmap v1784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:49.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:49 smithi006 bash[33918]: cluster 2023-08-27T22:57:48.330372+0000 mgr.y (mgr.14901) 1778 : cluster [DBG] pgmap v1785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:49.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:49 smithi006 bash[33918]: audit 2023-08-27T22:57:48.779700+0000 mon.a (mon.0) 1773 : audit [DBG] from='client.? 172.21.15.6:0/2326917842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:49.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:49 smithi006 bash[27292]: cluster 2023-08-27T22:57:48.330372+0000 mgr.y (mgr.14901) 1778 : cluster [DBG] pgmap v1785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:49.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:49 smithi006 bash[27292]: audit 2023-08-27T22:57:48.779700+0000 mon.a (mon.0) 1773 : audit [DBG] from='client.? 172.21.15.6:0/2326917842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:49.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:49 smithi130 bash[27579]: cluster 2023-08-27T22:57:48.330372+0000 mgr.y (mgr.14901) 1778 : cluster [DBG] pgmap v1785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:49.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:49 smithi130 bash[27579]: audit 2023-08-27T22:57:48.779700+0000 mon.a (mon.0) 1773 : audit [DBG] from='client.? 172.21.15.6:0/2326917842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:50.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:57:50.339Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:50.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:57:50.341Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:51.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:57:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:57:50] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T22:57:51.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:51 smithi006 bash[33918]: cluster 2023-08-27T22:57:50.331512+0000 mgr.y (mgr.14901) 1779 : cluster [DBG] pgmap v1786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:51.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:51 smithi006 bash[33918]: audit 2023-08-27T22:57:51.198781+0000 mon.a (mon.0) 1774 : audit [DBG] from='client.? 172.21.15.6:0/1790836828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:51.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:51 smithi006 bash[27292]: cluster 2023-08-27T22:57:50.331512+0000 mgr.y (mgr.14901) 1779 : cluster [DBG] pgmap v1786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:51.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:51 smithi006 bash[27292]: audit 2023-08-27T22:57:51.198781+0000 mon.a (mon.0) 1774 : audit [DBG] from='client.? 172.21.15.6:0/1790836828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:51.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:51 smithi130 bash[27579]: cluster 2023-08-27T22:57:50.331512+0000 mgr.y (mgr.14901) 1779 : cluster [DBG] pgmap v1786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:51.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:51 smithi130 bash[27579]: audit 2023-08-27T22:57:51.198781+0000 mon.a (mon.0) 1774 : audit [DBG] from='client.? 172.21.15.6:0/1790836828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:52.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:57:51.900Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:52.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:57:51.902Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:52 smithi130 bash[27579]: cluster 2023-08-27T22:57:52.332172+0000 mgr.y (mgr.14901) 1780 : cluster [DBG] pgmap v1787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:52.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:52 smithi006 bash[27292]: cluster 2023-08-27T22:57:52.332172+0000 mgr.y (mgr.14901) 1780 : cluster [DBG] pgmap v1787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:52.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:52 smithi006 bash[33918]: cluster 2023-08-27T22:57:52.332172+0000 mgr.y (mgr.14901) 1780 : cluster [DBG] pgmap v1787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:53.803 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:53 smithi006 bash[33918]: audit 2023-08-27T22:57:52.604321+0000 mon.b (mon.1) 291 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:57:53.804 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:53 smithi006 bash[27292]: audit 2023-08-27T22:57:52.604321+0000 mon.b (mon.1) 291 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:57:53.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:53 smithi130 bash[27579]: audit 2023-08-27T22:57:52.604321+0000 mon.b (mon.1) 291 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:57:54.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:57:53.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:54.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:57:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:57:53.799Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:57:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:54 smithi130 bash[27579]: audit 2023-08-27T22:57:53.615567+0000 mon.c (mon.2) 705 : audit [DBG] from='client.? 172.21.15.6:0/1538927025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:54 smithi130 bash[27579]: cluster 2023-08-27T22:57:54.333337+0000 mgr.y (mgr.14901) 1781 : cluster [DBG] pgmap v1788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:54.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:54 smithi006 bash[27292]: audit 2023-08-27T22:57:53.615567+0000 mon.c (mon.2) 705 : audit [DBG] from='client.? 172.21.15.6:0/1538927025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:54.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:54 smithi006 bash[27292]: cluster 2023-08-27T22:57:54.333337+0000 mgr.y (mgr.14901) 1781 : cluster [DBG] pgmap v1788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:54.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:54 smithi006 bash[33918]: audit 2023-08-27T22:57:53.615567+0000 mon.c (mon.2) 705 : audit [DBG] from='client.? 172.21.15.6:0/1538927025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:54.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:54 smithi006 bash[33918]: cluster 2023-08-27T22:57:54.333337+0000 mgr.y (mgr.14901) 1781 : cluster [DBG] pgmap v1788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:56.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:56 smithi006 bash[33918]: audit 2023-08-27T22:57:56.027902+0000 mon.c (mon.2) 706 : audit [DBG] from='client.? 172.21.15.6:0/4045449324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:56.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:56 smithi006 bash[27292]: audit 2023-08-27T22:57:56.027902+0000 mon.c (mon.2) 706 : audit [DBG] from='client.? 172.21.15.6:0/4045449324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:56.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:56 smithi130 bash[27579]: audit 2023-08-27T22:57:56.027902+0000 mon.c (mon.2) 706 : audit [DBG] from='client.? 172.21.15.6:0/4045449324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:57.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:57 smithi006 bash[27292]: cluster 2023-08-27T22:57:56.334010+0000 mgr.y (mgr.14901) 1782 : cluster [DBG] pgmap v1789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:57.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:57 smithi006 bash[33918]: cluster 2023-08-27T22:57:56.334010+0000 mgr.y (mgr.14901) 1782 : cluster [DBG] pgmap v1789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:57.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:57 smithi130 bash[27579]: cluster 2023-08-27T22:57:56.334010+0000 mgr.y (mgr.14901) 1782 : cluster [DBG] pgmap v1789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:59.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:59 smithi006 bash[27292]: cluster 2023-08-27T22:57:58.334502+0000 mgr.y (mgr.14901) 1783 : cluster [DBG] pgmap v1790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:59.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:59 smithi006 bash[27292]: audit 2023-08-27T22:57:58.508121+0000 mon.a (mon.0) 1775 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:57:59.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:59 smithi006 bash[27292]: audit 2023-08-27T22:57:58.547173+0000 mon.a (mon.0) 1776 : audit [DBG] from='client.? 172.21.15.6:0/2116024662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:59.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:59 smithi006 bash[27292]: audit 2023-08-27T22:57:58.954332+0000 mon.a (mon.0) 1777 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:57:59.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:59 smithi006 bash[27292]: audit 2023-08-27T22:57:59.232171+0000 mon.b (mon.1) 292 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:57:59.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:59 smithi006 bash[27292]: audit 2023-08-27T22:57:59.234228+0000 mon.b (mon.1) 293 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:57:59.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:57:59 smithi006 bash[27292]: audit 2023-08-27T22:57:59.240289+0000 mon.a (mon.0) 1778 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:57:59.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:59 smithi006 bash[33918]: cluster 2023-08-27T22:57:58.334502+0000 mgr.y (mgr.14901) 1783 : cluster [DBG] pgmap v1790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:59.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:59 smithi006 bash[33918]: audit 2023-08-27T22:57:58.508121+0000 mon.a (mon.0) 1775 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:57:59.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:59 smithi006 bash[33918]: audit 2023-08-27T22:57:58.547173+0000 mon.a (mon.0) 1776 : audit [DBG] from='client.? 172.21.15.6:0/2116024662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:59.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:59 smithi006 bash[33918]: audit 2023-08-27T22:57:58.954332+0000 mon.a (mon.0) 1777 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:57:59.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:59 smithi006 bash[33918]: audit 2023-08-27T22:57:59.232171+0000 mon.b (mon.1) 292 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:57:59.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:59 smithi006 bash[33918]: audit 2023-08-27T22:57:59.234228+0000 mon.b (mon.1) 293 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:57:59.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:57:59 smithi006 bash[33918]: audit 2023-08-27T22:57:59.240289+0000 mon.a (mon.0) 1778 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:57:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:59 smithi130 bash[27579]: cluster 2023-08-27T22:57:58.334502+0000 mgr.y (mgr.14901) 1783 : cluster [DBG] pgmap v1790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:57:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:59 smithi130 bash[27579]: audit 2023-08-27T22:57:58.508121+0000 mon.a (mon.0) 1775 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:57:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:59 smithi130 bash[27579]: audit 2023-08-27T22:57:58.547173+0000 mon.a (mon.0) 1776 : audit [DBG] from='client.? 172.21.15.6:0/2116024662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:59 smithi130 bash[27579]: audit 2023-08-27T22:57:58.954332+0000 mon.a (mon.0) 1777 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:57:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:59 smithi130 bash[27579]: audit 2023-08-27T22:57:59.232171+0000 mon.b (mon.1) 292 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:57:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:59 smithi130 bash[27579]: audit 2023-08-27T22:57:59.234228+0000 mon.b (mon.1) 293 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:57:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:57:59 smithi130 bash[27579]: audit 2023-08-27T22:57:59.240289+0000 mon.a (mon.0) 1778 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:58:00.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:58:00.340Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:00.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:58:00.342Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:01.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:58:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:58:00] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T22:58:01.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:01 smithi006 bash[33918]: cluster 2023-08-27T22:58:00.335348+0000 mgr.y (mgr.14901) 1784 : cluster [DBG] pgmap v1791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:01.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:01 smithi006 bash[33918]: audit 2023-08-27T22:58:00.959091+0000 mon.c (mon.2) 707 : audit [DBG] from='client.? 172.21.15.6:0/1206171628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:01.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:01 smithi006 bash[27292]: cluster 2023-08-27T22:58:00.335348+0000 mgr.y (mgr.14901) 1784 : cluster [DBG] pgmap v1791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:01.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:01 smithi006 bash[27292]: audit 2023-08-27T22:58:00.959091+0000 mon.c (mon.2) 707 : audit [DBG] from='client.? 172.21.15.6:0/1206171628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:01.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:01 smithi130 bash[27579]: cluster 2023-08-27T22:58:00.335348+0000 mgr.y (mgr.14901) 1784 : cluster [DBG] pgmap v1791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:01.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:01 smithi130 bash[27579]: audit 2023-08-27T22:58:00.959091+0000 mon.c (mon.2) 707 : audit [DBG] from='client.? 172.21.15.6:0/1206171628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:02.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:58:01.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:02.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:58:01.903Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:03.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:03 smithi006 bash[33918]: cluster 2023-08-27T22:58:02.335985+0000 mgr.y (mgr.14901) 1785 : cluster [DBG] pgmap v1792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:03.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:03 smithi006 bash[33918]: audit 2023-08-27T22:58:03.371638+0000 mon.c (mon.2) 708 : audit [DBG] from='client.? 172.21.15.6:0/1784594512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:03.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:03 smithi006 bash[27292]: cluster 2023-08-27T22:58:02.335985+0000 mgr.y (mgr.14901) 1785 : cluster [DBG] pgmap v1792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:03.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:03 smithi006 bash[27292]: audit 2023-08-27T22:58:03.371638+0000 mon.c (mon.2) 708 : audit [DBG] from='client.? 172.21.15.6:0/1784594512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:03.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:03 smithi130 bash[27579]: cluster 2023-08-27T22:58:02.335985+0000 mgr.y (mgr.14901) 1785 : cluster [DBG] pgmap v1792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:03.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:03 smithi130 bash[27579]: audit 2023-08-27T22:58:03.371638+0000 mon.c (mon.2) 708 : audit [DBG] from='client.? 172.21.15.6:0/1784594512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:04.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:58:03.798Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:04.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:58:03.800Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:05.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:05 smithi006 bash[33918]: cluster 2023-08-27T22:58:04.337109+0000 mgr.y (mgr.14901) 1786 : cluster [DBG] pgmap v1793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:05.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:05 smithi006 bash[27292]: cluster 2023-08-27T22:58:04.337109+0000 mgr.y (mgr.14901) 1786 : cluster [DBG] pgmap v1793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:05.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:05 smithi130 bash[27579]: cluster 2023-08-27T22:58:04.337109+0000 mgr.y (mgr.14901) 1786 : cluster [DBG] pgmap v1793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:06.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:06 smithi006 bash[33918]: audit 2023-08-27T22:58:05.791033+0000 mon.c (mon.2) 709 : audit [DBG] from='client.? 172.21.15.6:0/3379700828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:06.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:06 smithi006 bash[27292]: audit 2023-08-27T22:58:05.791033+0000 mon.c (mon.2) 709 : audit [DBG] from='client.? 172.21.15.6:0/3379700828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:06.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:06 smithi130 bash[27579]: audit 2023-08-27T22:58:05.791033+0000 mon.c (mon.2) 709 : audit [DBG] from='client.? 172.21.15.6:0/3379700828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:07.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:07 smithi006 bash[27292]: cluster 2023-08-27T22:58:06.337724+0000 mgr.y (mgr.14901) 1787 : cluster [DBG] pgmap v1794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:07.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:07 smithi006 bash[33918]: cluster 2023-08-27T22:58:06.337724+0000 mgr.y (mgr.14901) 1787 : cluster [DBG] pgmap v1794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:07.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:07 smithi130 bash[27579]: cluster 2023-08-27T22:58:06.337724+0000 mgr.y (mgr.14901) 1787 : cluster [DBG] pgmap v1794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:08.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:08 smithi130 bash[27579]: audit 2023-08-27T22:58:08.208109+0000 mon.c (mon.2) 710 : audit [DBG] from='client.? 172.21.15.6:0/3402843422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:08.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:08 smithi006 bash[27292]: audit 2023-08-27T22:58:08.208109+0000 mon.c (mon.2) 710 : audit [DBG] from='client.? 172.21.15.6:0/3402843422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:08.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:08 smithi006 bash[33918]: audit 2023-08-27T22:58:08.208109+0000 mon.c (mon.2) 710 : audit [DBG] from='client.? 172.21.15.6:0/3402843422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:09 smithi130 bash[27579]: cluster 2023-08-27T22:58:08.338423+0000 mgr.y (mgr.14901) 1788 : cluster [DBG] pgmap v1795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:09 smithi006 bash[33918]: cluster 2023-08-27T22:58:08.338423+0000 mgr.y (mgr.14901) 1788 : cluster [DBG] pgmap v1795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:09.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:09 smithi006 bash[27292]: cluster 2023-08-27T22:58:08.338423+0000 mgr.y (mgr.14901) 1788 : cluster [DBG] pgmap v1795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:10.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:58:10.341Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:10.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:58:10.343Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:11.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:58:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:58:10] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T22:58:11.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:11 smithi006 bash[33918]: cluster 2023-08-27T22:58:10.339134+0000 mgr.y (mgr.14901) 1789 : cluster [DBG] pgmap v1796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:11.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:11 smithi006 bash[33918]: audit 2023-08-27T22:58:10.623245+0000 mon.c (mon.2) 711 : audit [DBG] from='client.? 172.21.15.6:0/352237841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:11.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:11 smithi006 bash[27292]: cluster 2023-08-27T22:58:10.339134+0000 mgr.y (mgr.14901) 1789 : cluster [DBG] pgmap v1796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:11.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:11 smithi006 bash[27292]: audit 2023-08-27T22:58:10.623245+0000 mon.c (mon.2) 711 : audit [DBG] from='client.? 172.21.15.6:0/352237841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:11.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:11 smithi130 bash[27579]: cluster 2023-08-27T22:58:10.339134+0000 mgr.y (mgr.14901) 1789 : cluster [DBG] pgmap v1796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:11.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:11 smithi130 bash[27579]: audit 2023-08-27T22:58:10.623245+0000 mon.c (mon.2) 711 : audit [DBG] from='client.? 172.21.15.6:0/352237841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:12.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:58:11.902Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:12.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:58:11.904Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:13.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:13 smithi006 bash[33918]: cluster 2023-08-27T22:58:12.339775+0000 mgr.y (mgr.14901) 1790 : cluster [DBG] pgmap v1797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:13.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:13 smithi006 bash[33918]: audit 2023-08-27T22:58:13.042797+0000 mon.a (mon.0) 1779 : audit [DBG] from='client.? 172.21.15.6:0/1972192309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:13.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:13 smithi006 bash[27292]: cluster 2023-08-27T22:58:12.339775+0000 mgr.y (mgr.14901) 1790 : cluster [DBG] pgmap v1797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:13.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:13 smithi006 bash[27292]: audit 2023-08-27T22:58:13.042797+0000 mon.a (mon.0) 1779 : audit [DBG] from='client.? 172.21.15.6:0/1972192309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:13.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:13 smithi130 bash[27579]: cluster 2023-08-27T22:58:12.339775+0000 mgr.y (mgr.14901) 1790 : cluster [DBG] pgmap v1797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:13.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:13 smithi130 bash[27579]: audit 2023-08-27T22:58:13.042797+0000 mon.a (mon.0) 1779 : audit [DBG] from='client.? 172.21.15.6:0/1972192309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:14.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:58:13.799Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:14.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:58:13.802Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:15.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:15 smithi006 bash[27292]: cluster 2023-08-27T22:58:14.340942+0000 mgr.y (mgr.14901) 1791 : cluster [DBG] pgmap v1798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:15.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:15 smithi006 bash[33918]: cluster 2023-08-27T22:58:14.340942+0000 mgr.y (mgr.14901) 1791 : cluster [DBG] pgmap v1798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:15.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:15 smithi130 bash[27579]: cluster 2023-08-27T22:58:14.340942+0000 mgr.y (mgr.14901) 1791 : cluster [DBG] pgmap v1798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:16.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:16 smithi006 bash[27292]: audit 2023-08-27T22:58:15.448697+0000 mon.a (mon.0) 1780 : audit [DBG] from='client.? 172.21.15.6:0/3926428416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:16.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:16 smithi006 bash[33918]: audit 2023-08-27T22:58:15.448697+0000 mon.a (mon.0) 1780 : audit [DBG] from='client.? 172.21.15.6:0/3926428416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:16.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:16 smithi130 bash[27579]: audit 2023-08-27T22:58:15.448697+0000 mon.a (mon.0) 1780 : audit [DBG] from='client.? 172.21.15.6:0/3926428416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:17.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:17 smithi006 bash[33918]: cluster 2023-08-27T22:58:16.341535+0000 mgr.y (mgr.14901) 1792 : cluster [DBG] pgmap v1799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:17.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:17 smithi006 bash[27292]: cluster 2023-08-27T22:58:16.341535+0000 mgr.y (mgr.14901) 1792 : cluster [DBG] pgmap v1799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:17.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:17 smithi130 bash[27579]: cluster 2023-08-27T22:58:16.341535+0000 mgr.y (mgr.14901) 1792 : cluster [DBG] pgmap v1799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:18.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:18 smithi006 bash[27292]: audit 2023-08-27T22:58:17.865599+0000 mon.a (mon.0) 1781 : audit [DBG] from='client.? 172.21.15.6:0/2201620728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:18.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:18 smithi006 bash[33918]: audit 2023-08-27T22:58:17.865599+0000 mon.a (mon.0) 1781 : audit [DBG] from='client.? 172.21.15.6:0/2201620728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:18.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:18 smithi130 bash[27579]: audit 2023-08-27T22:58:17.865599+0000 mon.a (mon.0) 1781 : audit [DBG] from='client.? 172.21.15.6:0/2201620728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:19.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:19 smithi006 bash[27292]: cluster 2023-08-27T22:58:18.342177+0000 mgr.y (mgr.14901) 1793 : cluster [DBG] pgmap v1800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:19.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:19 smithi006 bash[33918]: cluster 2023-08-27T22:58:18.342177+0000 mgr.y (mgr.14901) 1793 : cluster [DBG] pgmap v1800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:19.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:19 smithi130 bash[27579]: cluster 2023-08-27T22:58:18.342177+0000 mgr.y (mgr.14901) 1793 : cluster [DBG] pgmap v1800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:20.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:20 smithi130 bash[27579]: audit 2023-08-27T22:58:20.279549+0000 mon.c (mon.2) 712 : audit [DBG] from='client.? 172.21.15.6:0/2785751087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:20.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:20 smithi006 bash[27292]: audit 2023-08-27T22:58:20.279549+0000 mon.c (mon.2) 712 : audit [DBG] from='client.? 172.21.15.6:0/2785751087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:20.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:20 smithi006 bash[33918]: audit 2023-08-27T22:58:20.279549+0000 mon.c (mon.2) 712 : audit [DBG] from='client.? 172.21.15.6:0/2785751087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:20.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:58:20.342Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:20.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:58:20.344Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:21.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:58:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:58:20] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T22:58:21.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:21 smithi130 bash[27579]: cluster 2023-08-27T22:58:20.342901+0000 mgr.y (mgr.14901) 1794 : cluster [DBG] pgmap v1801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:21.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:21 smithi006 bash[27292]: cluster 2023-08-27T22:58:20.342901+0000 mgr.y (mgr.14901) 1794 : cluster [DBG] pgmap v1801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:21.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:21 smithi006 bash[33918]: cluster 2023-08-27T22:58:20.342901+0000 mgr.y (mgr.14901) 1794 : cluster [DBG] pgmap v1801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:22.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:58:21.902Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:22.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:58:21.904Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:23.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:23 smithi006 bash[27292]: cluster 2023-08-27T22:58:22.343544+0000 mgr.y (mgr.14901) 1795 : cluster [DBG] pgmap v1802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:23.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:23 smithi006 bash[27292]: audit 2023-08-27T22:58:22.695838+0000 mon.a (mon.0) 1782 : audit [DBG] from='client.? 172.21.15.6:0/4165522839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:23.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:23 smithi006 bash[33918]: cluster 2023-08-27T22:58:22.343544+0000 mgr.y (mgr.14901) 1795 : cluster [DBG] pgmap v1802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:23.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:23 smithi006 bash[33918]: audit 2023-08-27T22:58:22.695838+0000 mon.a (mon.0) 1782 : audit [DBG] from='client.? 172.21.15.6:0/4165522839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:23.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:23 smithi130 bash[27579]: cluster 2023-08-27T22:58:22.343544+0000 mgr.y (mgr.14901) 1795 : cluster [DBG] pgmap v1802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:23.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:23 smithi130 bash[27579]: audit 2023-08-27T22:58:22.695838+0000 mon.a (mon.0) 1782 : audit [DBG] from='client.? 172.21.15.6:0/4165522839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:24.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:58:23.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:24.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:58:23.803Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:25.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:25 smithi006 bash[33918]: cluster 2023-08-27T22:58:24.344718+0000 mgr.y (mgr.14901) 1796 : cluster [DBG] pgmap v1803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:25.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:25 smithi006 bash[33918]: audit 2023-08-27T22:58:25.106855+0000 mon.c (mon.2) 713 : audit [DBG] from='client.? 172.21.15.6:0/3009084724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:25.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:25 smithi006 bash[27292]: cluster 2023-08-27T22:58:24.344718+0000 mgr.y (mgr.14901) 1796 : cluster [DBG] pgmap v1803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:25.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:25 smithi006 bash[27292]: audit 2023-08-27T22:58:25.106855+0000 mon.c (mon.2) 713 : audit [DBG] from='client.? 172.21.15.6:0/3009084724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:25.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:25 smithi130 bash[27579]: cluster 2023-08-27T22:58:24.344718+0000 mgr.y (mgr.14901) 1796 : cluster [DBG] pgmap v1803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:25.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:25 smithi130 bash[27579]: audit 2023-08-27T22:58:25.106855+0000 mon.c (mon.2) 713 : audit [DBG] from='client.? 172.21.15.6:0/3009084724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:27.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:27 smithi006 bash[27292]: cluster 2023-08-27T22:58:26.345340+0000 mgr.y (mgr.14901) 1797 : cluster [DBG] pgmap v1804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:27.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:27 smithi006 bash[33918]: cluster 2023-08-27T22:58:26.345340+0000 mgr.y (mgr.14901) 1797 : cluster [DBG] pgmap v1804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:27.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:27 smithi130 bash[27579]: cluster 2023-08-27T22:58:26.345340+0000 mgr.y (mgr.14901) 1797 : cluster [DBG] pgmap v1804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:28.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:28 smithi006 bash[33918]: audit 2023-08-27T22:58:27.520485+0000 mon.a (mon.0) 1783 : audit [DBG] from='client.? 172.21.15.6:0/2111914448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:28.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:28 smithi006 bash[27292]: audit 2023-08-27T22:58:27.520485+0000 mon.a (mon.0) 1783 : audit [DBG] from='client.? 172.21.15.6:0/2111914448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:28 smithi130 bash[27579]: audit 2023-08-27T22:58:27.520485+0000 mon.a (mon.0) 1783 : audit [DBG] from='client.? 172.21.15.6:0/2111914448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:29.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:29 smithi006 bash[27292]: cluster 2023-08-27T22:58:28.345897+0000 mgr.y (mgr.14901) 1798 : cluster [DBG] pgmap v1805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:29.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:29 smithi006 bash[33918]: cluster 2023-08-27T22:58:28.345897+0000 mgr.y (mgr.14901) 1798 : cluster [DBG] pgmap v1805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:29.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:29 smithi130 bash[27579]: cluster 2023-08-27T22:58:28.345897+0000 mgr.y (mgr.14901) 1798 : cluster [DBG] pgmap v1805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:30.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:30 smithi006 bash[27292]: audit 2023-08-27T22:58:29.941446+0000 mon.c (mon.2) 714 : audit [DBG] from='client.? 172.21.15.6:0/475864619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:30.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:58:30.342Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:30.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:58:30.344Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:30.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:30 smithi006 bash[33918]: audit 2023-08-27T22:58:29.941446+0000 mon.c (mon.2) 714 : audit [DBG] from='client.? 172.21.15.6:0/475864619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:30.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:30 smithi130 bash[27579]: audit 2023-08-27T22:58:29.941446+0000 mon.c (mon.2) 714 : audit [DBG] from='client.? 172.21.15.6:0/475864619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:31.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:58:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:58:30] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T22:58:31.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:31 smithi006 bash[33918]: cluster 2023-08-27T22:58:30.346917+0000 mgr.y (mgr.14901) 1799 : cluster [DBG] pgmap v1806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:31.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:31 smithi006 bash[27292]: cluster 2023-08-27T22:58:30.346917+0000 mgr.y (mgr.14901) 1799 : cluster [DBG] pgmap v1806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:31.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:31 smithi130 bash[27579]: cluster 2023-08-27T22:58:30.346917+0000 mgr.y (mgr.14901) 1799 : cluster [DBG] pgmap v1806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:32.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:58:31.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:32.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:58:31.905Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:32.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:32 smithi006 bash[33918]: audit 2023-08-27T22:58:32.355325+0000 mon.a (mon.0) 1784 : audit [DBG] from='client.? 172.21.15.6:0/3471022307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:32.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:32 smithi006 bash[27292]: audit 2023-08-27T22:58:32.355325+0000 mon.a (mon.0) 1784 : audit [DBG] from='client.? 172.21.15.6:0/3471022307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:32 smithi130 bash[27579]: audit 2023-08-27T22:58:32.355325+0000 mon.a (mon.0) 1784 : audit [DBG] from='client.? 172.21.15.6:0/3471022307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:33.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:33 smithi006 bash[33918]: cluster 2023-08-27T22:58:32.347499+0000 mgr.y (mgr.14901) 1800 : cluster [DBG] pgmap v1807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:33.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:33 smithi006 bash[27292]: cluster 2023-08-27T22:58:32.347499+0000 mgr.y (mgr.14901) 1800 : cluster [DBG] pgmap v1807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:33 smithi130 bash[27579]: cluster 2023-08-27T22:58:32.347499+0000 mgr.y (mgr.14901) 1800 : cluster [DBG] pgmap v1807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:34.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:58:33.801Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:34.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:58:33.803Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:35.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:35 smithi006 bash[27292]: cluster 2023-08-27T22:58:34.348613+0000 mgr.y (mgr.14901) 1801 : cluster [DBG] pgmap v1808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:35.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:35 smithi006 bash[27292]: audit 2023-08-27T22:58:34.768052+0000 mon.c (mon.2) 715 : audit [DBG] from='client.? 172.21.15.6:0/1777227551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:35.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:35 smithi006 bash[33918]: cluster 2023-08-27T22:58:34.348613+0000 mgr.y (mgr.14901) 1801 : cluster [DBG] pgmap v1808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:35.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:35 smithi006 bash[33918]: audit 2023-08-27T22:58:34.768052+0000 mon.c (mon.2) 715 : audit [DBG] from='client.? 172.21.15.6:0/1777227551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:35.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:35 smithi130 bash[27579]: cluster 2023-08-27T22:58:34.348613+0000 mgr.y (mgr.14901) 1801 : cluster [DBG] pgmap v1808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:35.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:35 smithi130 bash[27579]: audit 2023-08-27T22:58:34.768052+0000 mon.c (mon.2) 715 : audit [DBG] from='client.? 172.21.15.6:0/1777227551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:37.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:37 smithi006 bash[27292]: cluster 2023-08-27T22:58:36.349289+0000 mgr.y (mgr.14901) 1802 : cluster [DBG] pgmap v1809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:37.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:37 smithi006 bash[27292]: audit 2023-08-27T22:58:37.176693+0000 mon.a (mon.0) 1785 : audit [DBG] from='client.? 172.21.15.6:0/2114519174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:37.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:37 smithi006 bash[33918]: cluster 2023-08-27T22:58:36.349289+0000 mgr.y (mgr.14901) 1802 : cluster [DBG] pgmap v1809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:37.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:37 smithi006 bash[33918]: audit 2023-08-27T22:58:37.176693+0000 mon.a (mon.0) 1785 : audit [DBG] from='client.? 172.21.15.6:0/2114519174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:37.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:37 smithi130 bash[27579]: cluster 2023-08-27T22:58:36.349289+0000 mgr.y (mgr.14901) 1802 : cluster [DBG] pgmap v1809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:37.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:37 smithi130 bash[27579]: audit 2023-08-27T22:58:37.176693+0000 mon.a (mon.0) 1785 : audit [DBG] from='client.? 172.21.15.6:0/2114519174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:39.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:39 smithi006 bash[33918]: cluster 2023-08-27T22:58:38.349910+0000 mgr.y (mgr.14901) 1803 : cluster [DBG] pgmap v1810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:39.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:39 smithi006 bash[27292]: cluster 2023-08-27T22:58:38.349910+0000 mgr.y (mgr.14901) 1803 : cluster [DBG] pgmap v1810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:39.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:39 smithi130 bash[27579]: cluster 2023-08-27T22:58:38.349910+0000 mgr.y (mgr.14901) 1803 : cluster [DBG] pgmap v1810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:40.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:40 smithi006 bash[33918]: audit 2023-08-27T22:58:39.597569+0000 mon.a (mon.0) 1786 : audit [DBG] from='client.? 172.21.15.6:0/2903746762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:40.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:40 smithi006 bash[27292]: audit 2023-08-27T22:58:39.597569+0000 mon.a (mon.0) 1786 : audit [DBG] from='client.? 172.21.15.6:0/2903746762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:40.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:58:40.343Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:40.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:58:40.344Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:40 smithi130 bash[27579]: audit 2023-08-27T22:58:39.597569+0000 mon.a (mon.0) 1786 : audit [DBG] from='client.? 172.21.15.6:0/2903746762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:41.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:58:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:58:40] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T22:58:41.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:41 smithi006 bash[33918]: cluster 2023-08-27T22:58:40.350956+0000 mgr.y (mgr.14901) 1804 : cluster [DBG] pgmap v1811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:41.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:41 smithi006 bash[27292]: cluster 2023-08-27T22:58:40.350956+0000 mgr.y (mgr.14901) 1804 : cluster [DBG] pgmap v1811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:41.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:41 smithi130 bash[27579]: cluster 2023-08-27T22:58:40.350956+0000 mgr.y (mgr.14901) 1804 : cluster [DBG] pgmap v1811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:42.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:58:41.904Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:42.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:58:41.907Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:42.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:42 smithi006 bash[33918]: audit 2023-08-27T22:58:42.015618+0000 mon.c (mon.2) 716 : audit [DBG] from='client.? 172.21.15.6:0/1794748900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:42.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:42 smithi006 bash[27292]: audit 2023-08-27T22:58:42.015618+0000 mon.c (mon.2) 716 : audit [DBG] from='client.? 172.21.15.6:0/1794748900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:42 smithi130 bash[27579]: audit 2023-08-27T22:58:42.015618+0000 mon.c (mon.2) 716 : audit [DBG] from='client.? 172.21.15.6:0/1794748900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:43.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:43 smithi006 bash[27292]: cluster 2023-08-27T22:58:42.351652+0000 mgr.y (mgr.14901) 1805 : cluster [DBG] pgmap v1812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:43.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:43 smithi006 bash[33918]: cluster 2023-08-27T22:58:42.351652+0000 mgr.y (mgr.14901) 1805 : cluster [DBG] pgmap v1812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:43.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:43 smithi130 bash[27579]: cluster 2023-08-27T22:58:42.351652+0000 mgr.y (mgr.14901) 1805 : cluster [DBG] pgmap v1812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:44.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:58:43.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:44.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:58:43.804Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:45.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:45 smithi006 bash[33918]: cluster 2023-08-27T22:58:44.352689+0000 mgr.y (mgr.14901) 1806 : cluster [DBG] pgmap v1813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:45.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:45 smithi006 bash[33918]: audit 2023-08-27T22:58:44.436656+0000 mon.a (mon.0) 1787 : audit [DBG] from='client.? 172.21.15.6:0/4042770678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:45.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:45 smithi006 bash[27292]: cluster 2023-08-27T22:58:44.352689+0000 mgr.y (mgr.14901) 1806 : cluster [DBG] pgmap v1813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:45.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:45 smithi006 bash[27292]: audit 2023-08-27T22:58:44.436656+0000 mon.a (mon.0) 1787 : audit [DBG] from='client.? 172.21.15.6:0/4042770678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:45.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:45 smithi130 bash[27579]: cluster 2023-08-27T22:58:44.352689+0000 mgr.y (mgr.14901) 1806 : cluster [DBG] pgmap v1813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:45.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:45 smithi130 bash[27579]: audit 2023-08-27T22:58:44.436656+0000 mon.a (mon.0) 1787 : audit [DBG] from='client.? 172.21.15.6:0/4042770678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:47.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:47 smithi006 bash[27292]: cluster 2023-08-27T22:58:46.353363+0000 mgr.y (mgr.14901) 1807 : cluster [DBG] pgmap v1814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:47.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:47 smithi006 bash[27292]: audit 2023-08-27T22:58:46.851107+0000 mon.c (mon.2) 717 : audit [DBG] from='client.? 172.21.15.6:0/631229407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:47.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:47 smithi006 bash[33918]: cluster 2023-08-27T22:58:46.353363+0000 mgr.y (mgr.14901) 1807 : cluster [DBG] pgmap v1814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:47.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:47 smithi006 bash[33918]: audit 2023-08-27T22:58:46.851107+0000 mon.c (mon.2) 717 : audit [DBG] from='client.? 172.21.15.6:0/631229407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:47.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:47 smithi130 bash[27579]: cluster 2023-08-27T22:58:46.353363+0000 mgr.y (mgr.14901) 1807 : cluster [DBG] pgmap v1814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:47.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:47 smithi130 bash[27579]: audit 2023-08-27T22:58:46.851107+0000 mon.c (mon.2) 717 : audit [DBG] from='client.? 172.21.15.6:0/631229407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:49.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:49 smithi006 bash[33918]: cluster 2023-08-27T22:58:48.354009+0000 mgr.y (mgr.14901) 1808 : cluster [DBG] pgmap v1815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:49.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:49 smithi006 bash[33918]: audit 2023-08-27T22:58:49.260548+0000 mon.a (mon.0) 1788 : audit [DBG] from='client.? 172.21.15.6:0/1769981436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:49.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:49 smithi006 bash[27292]: cluster 2023-08-27T22:58:48.354009+0000 mgr.y (mgr.14901) 1808 : cluster [DBG] pgmap v1815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:49.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:49 smithi006 bash[27292]: audit 2023-08-27T22:58:49.260548+0000 mon.a (mon.0) 1788 : audit [DBG] from='client.? 172.21.15.6:0/1769981436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:49.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:49 smithi130 bash[27579]: cluster 2023-08-27T22:58:48.354009+0000 mgr.y (mgr.14901) 1808 : cluster [DBG] pgmap v1815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:49.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:49 smithi130 bash[27579]: audit 2023-08-27T22:58:49.260548+0000 mon.a (mon.0) 1788 : audit [DBG] from='client.? 172.21.15.6:0/1769981436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:50.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:58:50.343Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:50.749 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:58:50.344Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:51.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:58:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:58:50] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T22:58:51.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:51 smithi006 bash[33918]: cluster 2023-08-27T22:58:50.355075+0000 mgr.y (mgr.14901) 1809 : cluster [DBG] pgmap v1816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:51.865 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:51 smithi006 bash[27292]: cluster 2023-08-27T22:58:50.355075+0000 mgr.y (mgr.14901) 1809 : cluster [DBG] pgmap v1816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:51.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:51 smithi130 bash[27579]: cluster 2023-08-27T22:58:50.355075+0000 mgr.y (mgr.14901) 1809 : cluster [DBG] pgmap v1816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:52.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:58:51.905Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:52.514 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:58:51.907Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:52.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:52 smithi006 bash[27292]: audit 2023-08-27T22:58:51.682348+0000 mon.a (mon.0) 1789 : audit [DBG] from='client.? 172.21.15.6:0/392019521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:52.851 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:52 smithi006 bash[33918]: audit 2023-08-27T22:58:51.682348+0000 mon.a (mon.0) 1789 : audit [DBG] from='client.? 172.21.15.6:0/392019521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:52 smithi130 bash[27579]: audit 2023-08-27T22:58:51.682348+0000 mon.a (mon.0) 1789 : audit [DBG] from='client.? 172.21.15.6:0/392019521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:53.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:53 smithi006 bash[33918]: cluster 2023-08-27T22:58:52.355725+0000 mgr.y (mgr.14901) 1810 : cluster [DBG] pgmap v1817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:53.774 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:53 smithi006 bash[27292]: cluster 2023-08-27T22:58:52.355725+0000 mgr.y (mgr.14901) 1810 : cluster [DBG] pgmap v1817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:53.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:53 smithi130 bash[27579]: cluster 2023-08-27T22:58:52.355725+0000 mgr.y (mgr.14901) 1810 : cluster [DBG] pgmap v1817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:54.276 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:58:53.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:54.432 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:58:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:58:53.805Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:58:54.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:54 smithi006 bash[33918]: audit 2023-08-27T22:58:54.099833+0000 mon.a (mon.0) 1790 : audit [DBG] from='client.? 172.21.15.6:0/1347958047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:55.047 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:54 smithi006 bash[27292]: audit 2023-08-27T22:58:54.099833+0000 mon.a (mon.0) 1790 : audit [DBG] from='client.? 172.21.15.6:0/1347958047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:55.051 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:54 smithi130 bash[27579]: audit 2023-08-27T22:58:54.099833+0000 mon.a (mon.0) 1790 : audit [DBG] from='client.? 172.21.15.6:0/1347958047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:55.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:55 smithi006 bash[33918]: cluster 2023-08-27T22:58:54.356860+0000 mgr.y (mgr.14901) 1811 : cluster [DBG] pgmap v1818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:55.809 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:55 smithi006 bash[27292]: cluster 2023-08-27T22:58:54.356860+0000 mgr.y (mgr.14901) 1811 : cluster [DBG] pgmap v1818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:55.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:55 smithi130 bash[27579]: cluster 2023-08-27T22:58:54.356860+0000 mgr.y (mgr.14901) 1811 : cluster [DBG] pgmap v1818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:56.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:56 smithi130 bash[27579]: cluster 2023-08-27T22:58:56.357443+0000 mgr.y (mgr.14901) 1812 : cluster [DBG] pgmap v1819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:56.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:56 smithi006 bash[33918]: cluster 2023-08-27T22:58:56.357443+0000 mgr.y (mgr.14901) 1812 : cluster [DBG] pgmap v1819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:56.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:56 smithi006 bash[27292]: cluster 2023-08-27T22:58:56.357443+0000 mgr.y (mgr.14901) 1812 : cluster [DBG] pgmap v1819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:57.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:57 smithi130 bash[27579]: audit 2023-08-27T22:58:56.516450+0000 mon.a (mon.0) 1791 : audit [DBG] from='client.? 172.21.15.6:0/2744176830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:57.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:57 smithi006 bash[33918]: audit 2023-08-27T22:58:56.516450+0000 mon.a (mon.0) 1791 : audit [DBG] from='client.? 172.21.15.6:0/2744176830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:57.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:57 smithi006 bash[27292]: audit 2023-08-27T22:58:56.516450+0000 mon.a (mon.0) 1791 : audit [DBG] from='client.? 172.21.15.6:0/2744176830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:58 smithi130 bash[27579]: cluster 2023-08-27T22:58:58.358065+0000 mgr.y (mgr.14901) 1813 : cluster [DBG] pgmap v1820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:58.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:58 smithi006 bash[33918]: cluster 2023-08-27T22:58:58.358065+0000 mgr.y (mgr.14901) 1813 : cluster [DBG] pgmap v1820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:58.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:58 smithi006 bash[27292]: cluster 2023-08-27T22:58:58.358065+0000 mgr.y (mgr.14901) 1813 : cluster [DBG] pgmap v1820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:58:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:59 smithi130 bash[27579]: audit 2023-08-27T22:58:58.931101+0000 mon.a (mon.0) 1792 : audit [DBG] from='client.? 172.21.15.6:0/218722942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:58:59 smithi130 bash[27579]: audit 2023-08-27T22:58:59.258797+0000 mon.b (mon.1) 294 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:58:59.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:59 smithi006 bash[33918]: audit 2023-08-27T22:58:58.931101+0000 mon.a (mon.0) 1792 : audit [DBG] from='client.? 172.21.15.6:0/218722942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:59.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:58:59 smithi006 bash[33918]: audit 2023-08-27T22:58:59.258797+0000 mon.b (mon.1) 294 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:58:59.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:59 smithi006 bash[27292]: audit 2023-08-27T22:58:58.931101+0000 mon.a (mon.0) 1792 : audit [DBG] from='client.? 172.21.15.6:0/218722942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:59.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:58:59 smithi006 bash[27292]: audit 2023-08-27T22:58:59.258797+0000 mon.b (mon.1) 294 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:59:00.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:00 smithi006 bash[33918]: audit 2023-08-27T22:58:59.537033+0000 mon.a (mon.0) 1793 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:59:00.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:00 smithi006 bash[33918]: audit 2023-08-27T22:58:59.543649+0000 mon.a (mon.0) 1794 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:59:00.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:00 smithi006 bash[33918]: cluster 2023-08-27T22:59:00.359046+0000 mgr.y (mgr.14901) 1814 : cluster [DBG] pgmap v1821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:00.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:00 smithi006 bash[56298]: level=error ts=2023-08-27T22:59:00.344Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:00.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:00 smithi006 bash[56298]: level=warn ts=2023-08-27T22:59:00.345Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:00.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:00 smithi006 bash[27292]: audit 2023-08-27T22:58:59.537033+0000 mon.a (mon.0) 1793 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:59:00.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:00 smithi006 bash[27292]: audit 2023-08-27T22:58:59.543649+0000 mon.a (mon.0) 1794 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:59:00.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:00 smithi006 bash[27292]: cluster 2023-08-27T22:59:00.359046+0000 mgr.y (mgr.14901) 1814 : cluster [DBG] pgmap v1821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:00 smithi130 bash[27579]: audit 2023-08-27T22:58:59.537033+0000 mon.a (mon.0) 1793 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:59:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:00 smithi130 bash[27579]: audit 2023-08-27T22:58:59.543649+0000 mon.a (mon.0) 1794 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:59:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:00 smithi130 bash[27579]: cluster 2023-08-27T22:59:00.359046+0000 mgr.y (mgr.14901) 1814 : cluster [DBG] pgmap v1821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:01.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:59:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:59:00] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T22:59:01.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:01 smithi130 bash[27579]: audit 2023-08-27T22:59:01.341060+0000 mon.a (mon.0) 1795 : audit [DBG] from='client.? 172.21.15.6:0/1247791622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:01.911 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:01 smithi006 bash[33918]: audit 2023-08-27T22:59:01.341060+0000 mon.a (mon.0) 1795 : audit [DBG] from='client.? 172.21.15.6:0/1247791622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:01.911 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:01 smithi006 bash[27292]: audit 2023-08-27T22:59:01.341060+0000 mon.a (mon.0) 1795 : audit [DBG] from='client.? 172.21.15.6:0/1247791622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:02.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:01 smithi006 bash[56298]: level=error ts=2023-08-27T22:59:01.906Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:02.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:01 smithi006 bash[56298]: level=warn ts=2023-08-27T22:59:01.908Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:02 smithi130 bash[27579]: cluster 2023-08-27T22:59:02.359744+0000 mgr.y (mgr.14901) 1815 : cluster [DBG] pgmap v1822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:02.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:02 smithi006 bash[27292]: cluster 2023-08-27T22:59:02.359744+0000 mgr.y (mgr.14901) 1815 : cluster [DBG] pgmap v1822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:02.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:02 smithi006 bash[33918]: cluster 2023-08-27T22:59:02.359744+0000 mgr.y (mgr.14901) 1815 : cluster [DBG] pgmap v1822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:04.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:03 smithi130 bash[27579]: audit 2023-08-27T22:59:03.757756+0000 mon.c (mon.2) 718 : audit [DBG] from='client.? 172.21.15.6:0/1829385800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:04.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:03 smithi006 bash[33918]: audit 2023-08-27T22:59:03.757756+0000 mon.c (mon.2) 718 : audit [DBG] from='client.? 172.21.15.6:0/1829385800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:04.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:03 smithi006 bash[27292]: audit 2023-08-27T22:59:03.757756+0000 mon.c (mon.2) 718 : audit [DBG] from='client.? 172.21.15.6:0/1829385800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:04.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:03 smithi006 bash[56298]: level=error ts=2023-08-27T22:59:03.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:04.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:03 smithi006 bash[56298]: level=warn ts=2023-08-27T22:59:03.806Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:05.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:05 smithi006 bash[27292]: cluster 2023-08-27T22:59:04.360772+0000 mgr.y (mgr.14901) 1816 : cluster [DBG] pgmap v1823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:05.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:05 smithi006 bash[33918]: cluster 2023-08-27T22:59:04.360772+0000 mgr.y (mgr.14901) 1816 : cluster [DBG] pgmap v1823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:05.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:05 smithi130 bash[27579]: cluster 2023-08-27T22:59:04.360772+0000 mgr.y (mgr.14901) 1816 : cluster [DBG] pgmap v1823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:06.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:06 smithi006 bash[33918]: audit 2023-08-27T22:59:05.425061+0000 mon.a (mon.0) 1796 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:59:06.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:06 smithi006 bash[33918]: audit 2023-08-27T22:59:05.829163+0000 mon.a (mon.0) 1797 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:59:06.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:06 smithi006 bash[33918]: audit 2023-08-27T22:59:06.123589+0000 mon.b (mon.1) 295 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:59:06.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:06 smithi006 bash[33918]: audit 2023-08-27T22:59:06.125350+0000 mon.b (mon.1) 296 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:59:06.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:06 smithi006 bash[33918]: audit 2023-08-27T22:59:06.130499+0000 mon.a (mon.0) 1798 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:59:06.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:06 smithi006 bash[33918]: audit 2023-08-27T22:59:06.178772+0000 mon.c (mon.2) 719 : audit [DBG] from='client.? 172.21.15.6:0/2531913561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:06.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:06 smithi006 bash[27292]: audit 2023-08-27T22:59:05.425061+0000 mon.a (mon.0) 1796 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:59:06.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:06 smithi006 bash[27292]: audit 2023-08-27T22:59:05.829163+0000 mon.a (mon.0) 1797 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:59:06.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:06 smithi006 bash[27292]: audit 2023-08-27T22:59:06.123589+0000 mon.b (mon.1) 295 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:59:06.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:06 smithi006 bash[27292]: audit 2023-08-27T22:59:06.125350+0000 mon.b (mon.1) 296 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:59:06.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:06 smithi006 bash[27292]: audit 2023-08-27T22:59:06.130499+0000 mon.a (mon.0) 1798 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:59:06.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:06 smithi006 bash[27292]: audit 2023-08-27T22:59:06.178772+0000 mon.c (mon.2) 719 : audit [DBG] from='client.? 172.21.15.6:0/2531913561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:06 smithi130 bash[27579]: audit 2023-08-27T22:59:05.425061+0000 mon.a (mon.0) 1796 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:59:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:06 smithi130 bash[27579]: audit 2023-08-27T22:59:05.829163+0000 mon.a (mon.0) 1797 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:59:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:06 smithi130 bash[27579]: audit 2023-08-27T22:59:06.123589+0000 mon.b (mon.1) 295 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:59:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:06 smithi130 bash[27579]: audit 2023-08-27T22:59:06.125350+0000 mon.b (mon.1) 296 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:59:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:06 smithi130 bash[27579]: audit 2023-08-27T22:59:06.130499+0000 mon.a (mon.0) 1798 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T22:59:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:06 smithi130 bash[27579]: audit 2023-08-27T22:59:06.178772+0000 mon.c (mon.2) 719 : audit [DBG] from='client.? 172.21.15.6:0/2531913561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:07.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:07 smithi006 bash[27292]: cluster 2023-08-27T22:59:06.361492+0000 mgr.y (mgr.14901) 1817 : cluster [DBG] pgmap v1824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:07.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:07 smithi006 bash[33918]: cluster 2023-08-27T22:59:06.361492+0000 mgr.y (mgr.14901) 1817 : cluster [DBG] pgmap v1824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:07.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:07 smithi130 bash[27579]: cluster 2023-08-27T22:59:06.361492+0000 mgr.y (mgr.14901) 1817 : cluster [DBG] pgmap v1824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:09 smithi006 bash[33918]: cluster 2023-08-27T22:59:08.362106+0000 mgr.y (mgr.14901) 1818 : cluster [DBG] pgmap v1825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:09 smithi006 bash[33918]: audit 2023-08-27T22:59:08.591961+0000 mon.c (mon.2) 720 : audit [DBG] from='client.? 172.21.15.6:0/2439029494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:09.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:09 smithi006 bash[27292]: cluster 2023-08-27T22:59:08.362106+0000 mgr.y (mgr.14901) 1818 : cluster [DBG] pgmap v1825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:09.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:09 smithi006 bash[27292]: audit 2023-08-27T22:59:08.591961+0000 mon.c (mon.2) 720 : audit [DBG] from='client.? 172.21.15.6:0/2439029494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:09 smithi130 bash[27579]: cluster 2023-08-27T22:59:08.362106+0000 mgr.y (mgr.14901) 1818 : cluster [DBG] pgmap v1825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:09 smithi130 bash[27579]: audit 2023-08-27T22:59:08.591961+0000 mon.c (mon.2) 720 : audit [DBG] from='client.? 172.21.15.6:0/2439029494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:10.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:10 smithi006 bash[56298]: level=error ts=2023-08-27T22:59:10.344Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:10.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:10 smithi006 bash[56298]: level=warn ts=2023-08-27T22:59:10.346Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:11.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:59:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:59:10] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T22:59:11.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:11 smithi006 bash[27292]: cluster 2023-08-27T22:59:10.363216+0000 mgr.y (mgr.14901) 1819 : cluster [DBG] pgmap v1826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:11.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:11 smithi006 bash[27292]: audit 2023-08-27T22:59:11.006418+0000 mon.a (mon.0) 1799 : audit [DBG] from='client.? 172.21.15.6:0/3080906220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:11.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:11 smithi006 bash[33918]: cluster 2023-08-27T22:59:10.363216+0000 mgr.y (mgr.14901) 1819 : cluster [DBG] pgmap v1826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:11.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:11 smithi006 bash[33918]: audit 2023-08-27T22:59:11.006418+0000 mon.a (mon.0) 1799 : audit [DBG] from='client.? 172.21.15.6:0/3080906220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:11.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:11 smithi130 bash[27579]: cluster 2023-08-27T22:59:10.363216+0000 mgr.y (mgr.14901) 1819 : cluster [DBG] pgmap v1826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:11.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:11 smithi130 bash[27579]: audit 2023-08-27T22:59:11.006418+0000 mon.a (mon.0) 1799 : audit [DBG] from='client.? 172.21.15.6:0/3080906220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:12.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:11 smithi006 bash[56298]: level=error ts=2023-08-27T22:59:11.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:12.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:11 smithi006 bash[56298]: level=warn ts=2023-08-27T22:59:11.909Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:13.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:13 smithi006 bash[33918]: cluster 2023-08-27T22:59:12.363860+0000 mgr.y (mgr.14901) 1820 : cluster [DBG] pgmap v1827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:13.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:13 smithi006 bash[33918]: audit 2023-08-27T22:59:13.418396+0000 mon.a (mon.0) 1800 : audit [DBG] from='client.? 172.21.15.6:0/4067210887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:13.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:13 smithi006 bash[27292]: cluster 2023-08-27T22:59:12.363860+0000 mgr.y (mgr.14901) 1820 : cluster [DBG] pgmap v1827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:13.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:13 smithi006 bash[27292]: audit 2023-08-27T22:59:13.418396+0000 mon.a (mon.0) 1800 : audit [DBG] from='client.? 172.21.15.6:0/4067210887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:13.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:13 smithi130 bash[27579]: cluster 2023-08-27T22:59:12.363860+0000 mgr.y (mgr.14901) 1820 : cluster [DBG] pgmap v1827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:13.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:13 smithi130 bash[27579]: audit 2023-08-27T22:59:13.418396+0000 mon.a (mon.0) 1800 : audit [DBG] from='client.? 172.21.15.6:0/4067210887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:14.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:13 smithi006 bash[56298]: level=error ts=2023-08-27T22:59:13.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:14.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:13 smithi006 bash[56298]: level=warn ts=2023-08-27T22:59:13.808Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:15.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:15 smithi006 bash[33918]: cluster 2023-08-27T22:59:14.365001+0000 mgr.y (mgr.14901) 1821 : cluster [DBG] pgmap v1828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:15.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:15 smithi006 bash[27292]: cluster 2023-08-27T22:59:14.365001+0000 mgr.y (mgr.14901) 1821 : cluster [DBG] pgmap v1828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:15.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:15 smithi130 bash[27579]: cluster 2023-08-27T22:59:14.365001+0000 mgr.y (mgr.14901) 1821 : cluster [DBG] pgmap v1828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:16.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:16 smithi006 bash[33918]: audit 2023-08-27T22:59:15.833623+0000 mon.c (mon.2) 721 : audit [DBG] from='client.? 172.21.15.6:0/3462264861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:16.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:16 smithi006 bash[27292]: audit 2023-08-27T22:59:15.833623+0000 mon.c (mon.2) 721 : audit [DBG] from='client.? 172.21.15.6:0/3462264861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:16.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:16 smithi130 bash[27579]: audit 2023-08-27T22:59:15.833623+0000 mon.c (mon.2) 721 : audit [DBG] from='client.? 172.21.15.6:0/3462264861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:17.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:17 smithi006 bash[27292]: cluster 2023-08-27T22:59:16.365633+0000 mgr.y (mgr.14901) 1822 : cluster [DBG] pgmap v1829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:17.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:17 smithi006 bash[33918]: cluster 2023-08-27T22:59:16.365633+0000 mgr.y (mgr.14901) 1822 : cluster [DBG] pgmap v1829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:17.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:17 smithi130 bash[27579]: cluster 2023-08-27T22:59:16.365633+0000 mgr.y (mgr.14901) 1822 : cluster [DBG] pgmap v1829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:18.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:18 smithi130 bash[27579]: audit 2023-08-27T22:59:18.255314+0000 mon.b (mon.1) 297 : audit [DBG] from='client.? 172.21.15.6:0/4060228918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:18.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:18 smithi006 bash[33918]: audit 2023-08-27T22:59:18.255314+0000 mon.b (mon.1) 297 : audit [DBG] from='client.? 172.21.15.6:0/4060228918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:18.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:18 smithi006 bash[27292]: audit 2023-08-27T22:59:18.255314+0000 mon.b (mon.1) 297 : audit [DBG] from='client.? 172.21.15.6:0/4060228918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:19.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:19 smithi130 bash[27579]: cluster 2023-08-27T22:59:18.366357+0000 mgr.y (mgr.14901) 1823 : cluster [DBG] pgmap v1830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:19.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:19 smithi006 bash[33918]: cluster 2023-08-27T22:59:18.366357+0000 mgr.y (mgr.14901) 1823 : cluster [DBG] pgmap v1830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:19.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:19 smithi006 bash[27292]: cluster 2023-08-27T22:59:18.366357+0000 mgr.y (mgr.14901) 1823 : cluster [DBG] pgmap v1830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:20.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:20 smithi006 bash[56298]: level=error ts=2023-08-27T22:59:20.345Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:20.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:20 smithi006 bash[56298]: level=warn ts=2023-08-27T22:59:20.347Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:21.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:59:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:59:20] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T22:59:21.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:21 smithi006 bash[27292]: cluster 2023-08-27T22:59:20.367397+0000 mgr.y (mgr.14901) 1824 : cluster [DBG] pgmap v1831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:21.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:21 smithi006 bash[27292]: audit 2023-08-27T22:59:20.669211+0000 mon.c (mon.2) 722 : audit [DBG] from='client.? 172.21.15.6:0/1203278829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:21.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:21 smithi006 bash[33918]: cluster 2023-08-27T22:59:20.367397+0000 mgr.y (mgr.14901) 1824 : cluster [DBG] pgmap v1831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:21.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:21 smithi006 bash[33918]: audit 2023-08-27T22:59:20.669211+0000 mon.c (mon.2) 722 : audit [DBG] from='client.? 172.21.15.6:0/1203278829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:21.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:21 smithi130 bash[27579]: cluster 2023-08-27T22:59:20.367397+0000 mgr.y (mgr.14901) 1824 : cluster [DBG] pgmap v1831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:21.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:21 smithi130 bash[27579]: audit 2023-08-27T22:59:20.669211+0000 mon.c (mon.2) 722 : audit [DBG] from='client.? 172.21.15.6:0/1203278829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:22.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:21 smithi006 bash[56298]: level=error ts=2023-08-27T22:59:21.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:22.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:21 smithi006 bash[56298]: level=warn ts=2023-08-27T22:59:21.910Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:23.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:23 smithi006 bash[27292]: cluster 2023-08-27T22:59:22.368027+0000 mgr.y (mgr.14901) 1825 : cluster [DBG] pgmap v1832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:23.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:23 smithi006 bash[27292]: audit 2023-08-27T22:59:23.081002+0000 mon.a (mon.0) 1801 : audit [DBG] from='client.? 172.21.15.6:0/3296167075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:23.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:23 smithi006 bash[33918]: cluster 2023-08-27T22:59:22.368027+0000 mgr.y (mgr.14901) 1825 : cluster [DBG] pgmap v1832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:23.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:23 smithi006 bash[33918]: audit 2023-08-27T22:59:23.081002+0000 mon.a (mon.0) 1801 : audit [DBG] from='client.? 172.21.15.6:0/3296167075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:23.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:23 smithi130 bash[27579]: cluster 2023-08-27T22:59:22.368027+0000 mgr.y (mgr.14901) 1825 : cluster [DBG] pgmap v1832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:23.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:23 smithi130 bash[27579]: audit 2023-08-27T22:59:23.081002+0000 mon.a (mon.0) 1801 : audit [DBG] from='client.? 172.21.15.6:0/3296167075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:24.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:23 smithi006 bash[56298]: level=error ts=2023-08-27T22:59:23.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:24.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:23 smithi006 bash[56298]: level=warn ts=2023-08-27T22:59:23.807Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:25.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:25 smithi006 bash[27292]: cluster 2023-08-27T22:59:24.369211+0000 mgr.y (mgr.14901) 1826 : cluster [DBG] pgmap v1833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:25.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:25 smithi006 bash[33918]: cluster 2023-08-27T22:59:24.369211+0000 mgr.y (mgr.14901) 1826 : cluster [DBG] pgmap v1833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:25.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:25 smithi130 bash[27579]: cluster 2023-08-27T22:59:24.369211+0000 mgr.y (mgr.14901) 1826 : cluster [DBG] pgmap v1833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:26.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:26 smithi006 bash[27292]: audit 2023-08-27T22:59:25.494058+0000 mon.c (mon.2) 723 : audit [DBG] from='client.? 172.21.15.6:0/120820502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:26.648 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:26 smithi006 bash[33918]: audit 2023-08-27T22:59:25.494058+0000 mon.c (mon.2) 723 : audit [DBG] from='client.? 172.21.15.6:0/120820502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:26.649 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:26 smithi130 bash[27579]: audit 2023-08-27T22:59:25.494058+0000 mon.c (mon.2) 723 : audit [DBG] from='client.? 172.21.15.6:0/120820502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:27.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:27 smithi006 bash[27292]: cluster 2023-08-27T22:59:26.369859+0000 mgr.y (mgr.14901) 1827 : cluster [DBG] pgmap v1834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:27.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:27 smithi006 bash[33918]: cluster 2023-08-27T22:59:26.369859+0000 mgr.y (mgr.14901) 1827 : cluster [DBG] pgmap v1834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:27.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:27 smithi130 bash[27579]: cluster 2023-08-27T22:59:26.369859+0000 mgr.y (mgr.14901) 1827 : cluster [DBG] pgmap v1834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:28.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:28 smithi006 bash[27292]: audit 2023-08-27T22:59:27.911130+0000 mon.c (mon.2) 724 : audit [DBG] from='client.? 172.21.15.6:0/3742202137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:28.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:28 smithi006 bash[33918]: audit 2023-08-27T22:59:27.911130+0000 mon.c (mon.2) 724 : audit [DBG] from='client.? 172.21.15.6:0/3742202137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:28.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:28 smithi130 bash[27579]: audit 2023-08-27T22:59:27.911130+0000 mon.c (mon.2) 724 : audit [DBG] from='client.? 172.21.15.6:0/3742202137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:29.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:29 smithi006 bash[27292]: cluster 2023-08-27T22:59:28.370547+0000 mgr.y (mgr.14901) 1828 : cluster [DBG] pgmap v1835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:29.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:29 smithi006 bash[33918]: cluster 2023-08-27T22:59:28.370547+0000 mgr.y (mgr.14901) 1828 : cluster [DBG] pgmap v1835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:29.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:29 smithi130 bash[27579]: cluster 2023-08-27T22:59:28.370547+0000 mgr.y (mgr.14901) 1828 : cluster [DBG] pgmap v1835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:30.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:30 smithi006 bash[33918]: audit 2023-08-27T22:59:30.335189+0000 mon.a (mon.0) 1802 : audit [DBG] from='client.? 172.21.15.6:0/966770960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:30.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:30 smithi006 bash[56298]: level=error ts=2023-08-27T22:59:30.346Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:30.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:30 smithi006 bash[56298]: level=warn ts=2023-08-27T22:59:30.348Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:30.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:30 smithi006 bash[27292]: audit 2023-08-27T22:59:30.335189+0000 mon.a (mon.0) 1802 : audit [DBG] from='client.? 172.21.15.6:0/966770960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:30.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:30 smithi130 bash[27579]: audit 2023-08-27T22:59:30.335189+0000 mon.a (mon.0) 1802 : audit [DBG] from='client.? 172.21.15.6:0/966770960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:31.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:59:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:59:30] "GET /metrics HTTP/1.1" 200 193081 "" "Prometheus/2.33.4" 2023-08-27T22:59:31.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:31 smithi006 bash[33918]: cluster 2023-08-27T22:59:30.371646+0000 mgr.y (mgr.14901) 1829 : cluster [DBG] pgmap v1836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:31.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:31 smithi006 bash[27292]: cluster 2023-08-27T22:59:30.371646+0000 mgr.y (mgr.14901) 1829 : cluster [DBG] pgmap v1836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:31.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:31 smithi130 bash[27579]: cluster 2023-08-27T22:59:30.371646+0000 mgr.y (mgr.14901) 1829 : cluster [DBG] pgmap v1836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:32.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:31 smithi006 bash[56298]: level=error ts=2023-08-27T22:59:31.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:32.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:31 smithi006 bash[56298]: level=warn ts=2023-08-27T22:59:31.912Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:33.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:33 smithi006 bash[27292]: cluster 2023-08-27T22:59:32.372297+0000 mgr.y (mgr.14901) 1830 : cluster [DBG] pgmap v1837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:33.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:33 smithi006 bash[27292]: audit 2023-08-27T22:59:32.752037+0000 mon.c (mon.2) 725 : audit [DBG] from='client.? 172.21.15.6:0/1037492479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:33.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:33 smithi006 bash[33918]: cluster 2023-08-27T22:59:32.372297+0000 mgr.y (mgr.14901) 1830 : cluster [DBG] pgmap v1837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:33.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:33 smithi006 bash[33918]: audit 2023-08-27T22:59:32.752037+0000 mon.c (mon.2) 725 : audit [DBG] from='client.? 172.21.15.6:0/1037492479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:33 smithi130 bash[27579]: cluster 2023-08-27T22:59:32.372297+0000 mgr.y (mgr.14901) 1830 : cluster [DBG] pgmap v1837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:33.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:33 smithi130 bash[27579]: audit 2023-08-27T22:59:32.752037+0000 mon.c (mon.2) 725 : audit [DBG] from='client.? 172.21.15.6:0/1037492479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:34.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:33 smithi006 bash[56298]: level=error ts=2023-08-27T22:59:33.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:34.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:33 smithi006 bash[56298]: level=warn ts=2023-08-27T22:59:33.808Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:35.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:35 smithi006 bash[33918]: cluster 2023-08-27T22:59:34.373461+0000 mgr.y (mgr.14901) 1831 : cluster [DBG] pgmap v1838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:35.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:35 smithi006 bash[33918]: audit 2023-08-27T22:59:35.163264+0000 mon.c (mon.2) 726 : audit [DBG] from='client.? 172.21.15.6:0/2562760275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:35.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:35 smithi006 bash[27292]: cluster 2023-08-27T22:59:34.373461+0000 mgr.y (mgr.14901) 1831 : cluster [DBG] pgmap v1838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:35.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:35 smithi006 bash[27292]: audit 2023-08-27T22:59:35.163264+0000 mon.c (mon.2) 726 : audit [DBG] from='client.? 172.21.15.6:0/2562760275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:35.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:35 smithi130 bash[27579]: cluster 2023-08-27T22:59:34.373461+0000 mgr.y (mgr.14901) 1831 : cluster [DBG] pgmap v1838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:35.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:35 smithi130 bash[27579]: audit 2023-08-27T22:59:35.163264+0000 mon.c (mon.2) 726 : audit [DBG] from='client.? 172.21.15.6:0/2562760275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:37.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:37 smithi006 bash[33918]: cluster 2023-08-27T22:59:36.374094+0000 mgr.y (mgr.14901) 1832 : cluster [DBG] pgmap v1839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:37.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:37 smithi006 bash[27292]: cluster 2023-08-27T22:59:36.374094+0000 mgr.y (mgr.14901) 1832 : cluster [DBG] pgmap v1839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:37.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:37 smithi130 bash[27579]: cluster 2023-08-27T22:59:36.374094+0000 mgr.y (mgr.14901) 1832 : cluster [DBG] pgmap v1839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:38.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:38 smithi006 bash[27292]: audit 2023-08-27T22:59:37.579953+0000 mon.a (mon.0) 1803 : audit [DBG] from='client.? 172.21.15.6:0/2088313368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:38.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:38 smithi006 bash[33918]: audit 2023-08-27T22:59:37.579953+0000 mon.a (mon.0) 1803 : audit [DBG] from='client.? 172.21.15.6:0/2088313368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:38.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:38 smithi130 bash[27579]: audit 2023-08-27T22:59:37.579953+0000 mon.a (mon.0) 1803 : audit [DBG] from='client.? 172.21.15.6:0/2088313368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:39.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:39 smithi006 bash[27292]: cluster 2023-08-27T22:59:38.374720+0000 mgr.y (mgr.14901) 1833 : cluster [DBG] pgmap v1840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:39.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:39 smithi006 bash[33918]: cluster 2023-08-27T22:59:38.374720+0000 mgr.y (mgr.14901) 1833 : cluster [DBG] pgmap v1840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:39.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:39 smithi130 bash[27579]: cluster 2023-08-27T22:59:38.374720+0000 mgr.y (mgr.14901) 1833 : cluster [DBG] pgmap v1840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:40.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:40 smithi006 bash[27292]: audit 2023-08-27T22:59:39.991552+0000 mon.c (mon.2) 727 : audit [DBG] from='client.? 172.21.15.6:0/148082997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:40.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:40 smithi006 bash[56298]: level=error ts=2023-08-27T22:59:40.347Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:40.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:40 smithi006 bash[56298]: level=warn ts=2023-08-27T22:59:40.349Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:40.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:40 smithi006 bash[33918]: audit 2023-08-27T22:59:39.991552+0000 mon.c (mon.2) 727 : audit [DBG] from='client.? 172.21.15.6:0/148082997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:40 smithi130 bash[27579]: audit 2023-08-27T22:59:39.991552+0000 mon.c (mon.2) 727 : audit [DBG] from='client.? 172.21.15.6:0/148082997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:41.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:59:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:59:40] "GET /metrics HTTP/1.1" 200 193081 "" "Prometheus/2.33.4" 2023-08-27T22:59:41.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:41 smithi006 bash[33918]: cluster 2023-08-27T22:59:40.375529+0000 mgr.y (mgr.14901) 1834 : cluster [DBG] pgmap v1841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:41.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:41 smithi006 bash[27292]: cluster 2023-08-27T22:59:40.375529+0000 mgr.y (mgr.14901) 1834 : cluster [DBG] pgmap v1841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:41.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:41 smithi130 bash[27579]: cluster 2023-08-27T22:59:40.375529+0000 mgr.y (mgr.14901) 1834 : cluster [DBG] pgmap v1841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:42.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:41 smithi006 bash[56298]: level=error ts=2023-08-27T22:59:41.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:42.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:41 smithi006 bash[56298]: level=warn ts=2023-08-27T22:59:41.912Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:42.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:42 smithi006 bash[33918]: cluster 2023-08-27T22:59:42.376117+0000 mgr.y (mgr.14901) 1835 : cluster [DBG] pgmap v1842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:42.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:42 smithi006 bash[33918]: audit 2023-08-27T22:59:42.410422+0000 mon.c (mon.2) 728 : audit [DBG] from='client.? 172.21.15.6:0/3213871808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:42.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:42 smithi006 bash[27292]: cluster 2023-08-27T22:59:42.376117+0000 mgr.y (mgr.14901) 1835 : cluster [DBG] pgmap v1842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:42.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:42 smithi006 bash[27292]: audit 2023-08-27T22:59:42.410422+0000 mon.c (mon.2) 728 : audit [DBG] from='client.? 172.21.15.6:0/3213871808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:42 smithi130 bash[27579]: cluster 2023-08-27T22:59:42.376117+0000 mgr.y (mgr.14901) 1835 : cluster [DBG] pgmap v1842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:42 smithi130 bash[27579]: audit 2023-08-27T22:59:42.410422+0000 mon.c (mon.2) 728 : audit [DBG] from='client.? 172.21.15.6:0/3213871808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:44.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:43 smithi006 bash[56298]: level=error ts=2023-08-27T22:59:43.806Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:44.223 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:43 smithi006 bash[56298]: level=warn ts=2023-08-27T22:59:43.808Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:45.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:45 smithi006 bash[27292]: cluster 2023-08-27T22:59:44.377280+0000 mgr.y (mgr.14901) 1836 : cluster [DBG] pgmap v1843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:45.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:45 smithi006 bash[27292]: audit 2023-08-27T22:59:44.826045+0000 mon.c (mon.2) 729 : audit [DBG] from='client.? 172.21.15.6:0/2170554967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:45.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:45 smithi006 bash[33918]: cluster 2023-08-27T22:59:44.377280+0000 mgr.y (mgr.14901) 1836 : cluster [DBG] pgmap v1843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:45.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:45 smithi006 bash[33918]: audit 2023-08-27T22:59:44.826045+0000 mon.c (mon.2) 729 : audit [DBG] from='client.? 172.21.15.6:0/2170554967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:45.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:45 smithi130 bash[27579]: cluster 2023-08-27T22:59:44.377280+0000 mgr.y (mgr.14901) 1836 : cluster [DBG] pgmap v1843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:45.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:45 smithi130 bash[27579]: audit 2023-08-27T22:59:44.826045+0000 mon.c (mon.2) 729 : audit [DBG] from='client.? 172.21.15.6:0/2170554967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:47.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:47 smithi006 bash[27292]: cluster 2023-08-27T22:59:46.377942+0000 mgr.y (mgr.14901) 1837 : cluster [DBG] pgmap v1844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:47.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:47 smithi006 bash[27292]: audit 2023-08-27T22:59:47.237959+0000 mon.c (mon.2) 730 : audit [DBG] from='client.? 172.21.15.6:0/1920342893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:47.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:47 smithi006 bash[33918]: cluster 2023-08-27T22:59:46.377942+0000 mgr.y (mgr.14901) 1837 : cluster [DBG] pgmap v1844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:47.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:47 smithi006 bash[33918]: audit 2023-08-27T22:59:47.237959+0000 mon.c (mon.2) 730 : audit [DBG] from='client.? 172.21.15.6:0/1920342893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:47.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:47 smithi130 bash[27579]: cluster 2023-08-27T22:59:46.377942+0000 mgr.y (mgr.14901) 1837 : cluster [DBG] pgmap v1844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:47.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:47 smithi130 bash[27579]: audit 2023-08-27T22:59:47.237959+0000 mon.c (mon.2) 730 : audit [DBG] from='client.? 172.21.15.6:0/1920342893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:49.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:49 smithi006 bash[27292]: cluster 2023-08-27T22:59:48.378581+0000 mgr.y (mgr.14901) 1838 : cluster [DBG] pgmap v1845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:49.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:49 smithi006 bash[33918]: cluster 2023-08-27T22:59:48.378581+0000 mgr.y (mgr.14901) 1838 : cluster [DBG] pgmap v1845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:49.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:49 smithi130 bash[27579]: cluster 2023-08-27T22:59:48.378581+0000 mgr.y (mgr.14901) 1838 : cluster [DBG] pgmap v1845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:50.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:50 smithi006 bash[27292]: audit 2023-08-27T22:59:49.648635+0000 mon.c (mon.2) 731 : audit [DBG] from='client.? 172.21.15.6:0/1663567419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:50.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:50 smithi006 bash[33918]: audit 2023-08-27T22:59:49.648635+0000 mon.c (mon.2) 731 : audit [DBG] from='client.? 172.21.15.6:0/1663567419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:50.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:50 smithi006 bash[56298]: level=error ts=2023-08-27T22:59:50.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:50.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:50 smithi006 bash[56298]: level=warn ts=2023-08-27T22:59:50.350Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:50 smithi130 bash[27579]: audit 2023-08-27T22:59:49.648635+0000 mon.c (mon.2) 731 : audit [DBG] from='client.? 172.21.15.6:0/1663567419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:51.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 22:59:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:22:59:50] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T22:59:51.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:51 smithi006 bash[27292]: cluster 2023-08-27T22:59:50.379623+0000 mgr.y (mgr.14901) 1839 : cluster [DBG] pgmap v1846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:51.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:51 smithi006 bash[33918]: cluster 2023-08-27T22:59:50.379623+0000 mgr.y (mgr.14901) 1839 : cluster [DBG] pgmap v1846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:51.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:51 smithi130 bash[27579]: cluster 2023-08-27T22:59:50.379623+0000 mgr.y (mgr.14901) 1839 : cluster [DBG] pgmap v1846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:52.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:51 smithi006 bash[56298]: level=error ts=2023-08-27T22:59:51.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:52.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:51 smithi006 bash[56298]: level=warn ts=2023-08-27T22:59:51.913Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:52.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:52 smithi006 bash[33918]: audit 2023-08-27T22:59:52.065034+0000 mon.a (mon.0) 1804 : audit [DBG] from='client.? 172.21.15.6:0/1225085666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:52.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:52 smithi006 bash[33918]: cluster 2023-08-27T22:59:52.380333+0000 mgr.y (mgr.14901) 1840 : cluster [DBG] pgmap v1847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:52.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:52 smithi006 bash[27292]: audit 2023-08-27T22:59:52.065034+0000 mon.a (mon.0) 1804 : audit [DBG] from='client.? 172.21.15.6:0/1225085666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:52.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:52 smithi006 bash[27292]: cluster 2023-08-27T22:59:52.380333+0000 mgr.y (mgr.14901) 1840 : cluster [DBG] pgmap v1847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:52 smithi130 bash[27579]: audit 2023-08-27T22:59:52.065034+0000 mon.a (mon.0) 1804 : audit [DBG] from='client.? 172.21.15.6:0/1225085666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:52 smithi130 bash[27579]: cluster 2023-08-27T22:59:52.380333+0000 mgr.y (mgr.14901) 1840 : cluster [DBG] pgmap v1847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:53.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 22:59:53 smithi130 bash[51302]: debug 2023-08-27T22:59:52.995+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:53.002583+0000) 2023-08-27T22:59:54.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:53 smithi006 bash[56298]: level=error ts=2023-08-27T22:59:53.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:54.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 22:59:53 smithi006 bash[56298]: level=warn ts=2023-08-27T22:59:53.809Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T22:59:54.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 22:59:54 smithi130 bash[51302]: debug 2023-08-27T22:59:53.999+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:54.002893+0000) 2023-08-27T22:59:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:55 smithi130 bash[27579]: cluster 2023-08-27T22:59:54.381449+0000 mgr.y (mgr.14901) 1841 : cluster [DBG] pgmap v1848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:55 smithi130 bash[27579]: audit 2023-08-27T22:59:54.470211+0000 mon.a (mon.0) 1805 : audit [DBG] from='client.? 172.21.15.6:0/69984299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 22:59:55 smithi130 bash[51302]: debug 2023-08-27T22:59:54.999+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:55.003198+0000) 2023-08-27T22:59:55.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:55 smithi006 bash[33918]: cluster 2023-08-27T22:59:54.381449+0000 mgr.y (mgr.14901) 1841 : cluster [DBG] pgmap v1848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:55.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:55 smithi006 bash[33918]: audit 2023-08-27T22:59:54.470211+0000 mon.a (mon.0) 1805 : audit [DBG] from='client.? 172.21.15.6:0/69984299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:55.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:55 smithi006 bash[27292]: cluster 2023-08-27T22:59:54.381449+0000 mgr.y (mgr.14901) 1841 : cluster [DBG] pgmap v1848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:55.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:55 smithi006 bash[27292]: audit 2023-08-27T22:59:54.470211+0000 mon.a (mon.0) 1805 : audit [DBG] from='client.? 172.21.15.6:0/69984299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:56.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 22:59:56 smithi130 bash[51302]: debug 2023-08-27T22:59:55.999+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:56.003461+0000) 2023-08-27T22:59:57.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 22:59:57 smithi130 bash[51302]: debug 2023-08-27T22:59:56.999+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:57.003754+0000) 2023-08-27T22:59:57.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:57 smithi006 bash[33918]: cluster 2023-08-27T22:59:56.382063+0000 mgr.y (mgr.14901) 1842 : cluster [DBG] pgmap v1849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:57.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:57 smithi006 bash[33918]: audit 2023-08-27T22:59:56.884553+0000 mon.c (mon.2) 732 : audit [DBG] from='client.? 172.21.15.6:0/4053682996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:57.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:57 smithi006 bash[27292]: cluster 2023-08-27T22:59:56.382063+0000 mgr.y (mgr.14901) 1842 : cluster [DBG] pgmap v1849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:57.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:57 smithi006 bash[27292]: audit 2023-08-27T22:59:56.884553+0000 mon.c (mon.2) 732 : audit [DBG] from='client.? 172.21.15.6:0/4053682996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:57.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:57 smithi130 bash[27579]: cluster 2023-08-27T22:59:56.382063+0000 mgr.y (mgr.14901) 1842 : cluster [DBG] pgmap v1849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:57.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:57 smithi130 bash[27579]: audit 2023-08-27T22:59:56.884553+0000 mon.c (mon.2) 732 : audit [DBG] from='client.? 172.21.15.6:0/4053682996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:58.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 22:59:58 smithi130 bash[51302]: debug 2023-08-27T22:59:57.999+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:58.004019+0000) 2023-08-27T22:59:59.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 22:59:59 smithi130 bash[51302]: debug 2023-08-27T22:59:58.999+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:59.004261+0000) 2023-08-27T22:59:59.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:59 smithi006 bash[27292]: cluster 2023-08-27T22:59:58.382724+0000 mgr.y (mgr.14901) 1843 : cluster [DBG] pgmap v1850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:59.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 22:59:59 smithi006 bash[27292]: audit 2023-08-27T22:59:59.305259+0000 mon.c (mon.2) 733 : audit [DBG] from='client.? 172.21.15.6:0/3511402759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:59.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:59 smithi006 bash[33918]: cluster 2023-08-27T22:59:58.382724+0000 mgr.y (mgr.14901) 1843 : cluster [DBG] pgmap v1850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:59.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 22:59:59 smithi006 bash[33918]: audit 2023-08-27T22:59:59.305259+0000 mon.c (mon.2) 733 : audit [DBG] from='client.? 172.21.15.6:0/3511402759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:59 smithi130 bash[27579]: cluster 2023-08-27T22:59:58.382724+0000 mgr.y (mgr.14901) 1843 : cluster [DBG] pgmap v1850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T22:59:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 22:59:59 smithi130 bash[27579]: audit 2023-08-27T22:59:59.305259+0000 mon.c (mon.2) 733 : audit [DBG] from='client.? 172.21.15.6:0/3511402759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:00.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:00 smithi130 bash[51302]: debug 2023-08-27T22:59:59.999+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:00.004572+0000) 2023-08-27T23:00:00.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:00 smithi006 bash[27292]: cluster 2023-08-27T23:00:00.000157+0000 mon.a (mon.0) 1806 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:00:00.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:00 smithi006 bash[33918]: cluster 2023-08-27T23:00:00.000157+0000 mon.a (mon.0) 1806 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:00:00.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:00:00.349Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:00.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:00:00.351Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:00 smithi130 bash[27579]: cluster 2023-08-27T23:00:00.000157+0000 mon.a (mon.0) 1806 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:00:01.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:00:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:00:00] "GET /metrics HTTP/1.1" 200 193094 "" "Prometheus/2.33.4" 2023-08-27T23:00:01.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:01 smithi130 bash[51302]: debug 2023-08-27T23:00:00.998+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:01.004845+0000) 2023-08-27T23:00:01.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:01 smithi006 bash[27292]: cluster 2023-08-27T23:00:00.383997+0000 mgr.y (mgr.14901) 1844 : cluster [DBG] pgmap v1851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:01.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:01 smithi006 bash[33918]: cluster 2023-08-27T23:00:00.383997+0000 mgr.y (mgr.14901) 1844 : cluster [DBG] pgmap v1851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:01.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:01 smithi130 bash[27579]: cluster 2023-08-27T23:00:00.383997+0000 mgr.y (mgr.14901) 1844 : cluster [DBG] pgmap v1851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:02.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:01 smithi006 bash[56298]: level=error ts=2023-08-27T23:00:01.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:02.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:01 smithi006 bash[56298]: level=warn ts=2023-08-27T23:00:01.914Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:02.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:02 smithi130 bash[51302]: debug 2023-08-27T23:00:01.998+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:02.005121+0000) 2023-08-27T23:00:02.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:02 smithi006 bash[33918]: audit 2023-08-27T23:00:01.725970+0000 mon.a (mon.0) 1807 : audit [DBG] from='client.? 172.21.15.6:0/678084891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:02.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:02 smithi006 bash[33918]: cluster 2023-08-27T23:00:02.384645+0000 mgr.y (mgr.14901) 1845 : cluster [DBG] pgmap v1852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:02.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:02 smithi006 bash[27292]: audit 2023-08-27T23:00:01.725970+0000 mon.a (mon.0) 1807 : audit [DBG] from='client.? 172.21.15.6:0/678084891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:02.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:02 smithi006 bash[27292]: cluster 2023-08-27T23:00:02.384645+0000 mgr.y (mgr.14901) 1845 : cluster [DBG] pgmap v1852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:02 smithi130 bash[27579]: audit 2023-08-27T23:00:01.725970+0000 mon.a (mon.0) 1807 : audit [DBG] from='client.? 172.21.15.6:0/678084891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:02 smithi130 bash[27579]: cluster 2023-08-27T23:00:02.384645+0000 mgr.y (mgr.14901) 1845 : cluster [DBG] pgmap v1852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:03.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:03 smithi130 bash[51302]: debug 2023-08-27T23:00:03.002+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:03.005439+0000) 2023-08-27T23:00:04.197 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:00:03.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:04.197 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:00:03.809Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:04.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:04 smithi130 bash[27579]: audit 2023-08-27T23:00:04.140365+0000 mon.c (mon.2) 734 : audit [DBG] from='client.? 172.21.15.6:0/2818156134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:04 smithi130 bash[51302]: debug 2023-08-27T23:00:03.998+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:04.005727+0000) 2023-08-27T23:00:04.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:04 smithi006 bash[27292]: audit 2023-08-27T23:00:04.140365+0000 mon.c (mon.2) 734 : audit [DBG] from='client.? 172.21.15.6:0/2818156134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:04.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:04 smithi006 bash[33918]: audit 2023-08-27T23:00:04.140365+0000 mon.c (mon.2) 734 : audit [DBG] from='client.? 172.21.15.6:0/2818156134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:05.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:05 smithi130 bash[51302]: debug 2023-08-27T23:00:05.002+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:05.006018+0000) 2023-08-27T23:00:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:05 smithi130 bash[27579]: cluster 2023-08-27T23:00:04.385793+0000 mgr.y (mgr.14901) 1846 : cluster [DBG] pgmap v1853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:05.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:05 smithi006 bash[33918]: cluster 2023-08-27T23:00:04.385793+0000 mgr.y (mgr.14901) 1846 : cluster [DBG] pgmap v1853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:05.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:05 smithi006 bash[27292]: cluster 2023-08-27T23:00:04.385793+0000 mgr.y (mgr.14901) 1846 : cluster [DBG] pgmap v1853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:06.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:06 smithi130 bash[27579]: audit 2023-08-27T23:00:06.142056+0000 mon.b (mon.1) 298 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:00:06.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:06 smithi130 bash[51302]: debug 2023-08-27T23:00:05.998+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:06.006257+0000) 2023-08-27T23:00:06.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:06 smithi006 bash[33918]: audit 2023-08-27T23:00:06.142056+0000 mon.b (mon.1) 298 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:00:06.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:06 smithi006 bash[27292]: audit 2023-08-27T23:00:06.142056+0000 mon.b (mon.1) 298 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:00:07.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:07 smithi130 bash[27579]: cluster 2023-08-27T23:00:06.386507+0000 mgr.y (mgr.14901) 1847 : cluster [DBG] pgmap v1854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:07.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:07 smithi130 bash[27579]: audit 2023-08-27T23:00:06.561917+0000 mon.b (mon.1) 299 : audit [DBG] from='client.? 172.21.15.6:0/3406038266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:07.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:07 smithi130 bash[51302]: debug 2023-08-27T23:00:06.998+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:07.006525+0000) 2023-08-27T23:00:07.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:07 smithi006 bash[27292]: cluster 2023-08-27T23:00:06.386507+0000 mgr.y (mgr.14901) 1847 : cluster [DBG] pgmap v1854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:07.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:07 smithi006 bash[27292]: audit 2023-08-27T23:00:06.561917+0000 mon.b (mon.1) 299 : audit [DBG] from='client.? 172.21.15.6:0/3406038266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:07.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:07 smithi006 bash[33918]: cluster 2023-08-27T23:00:06.386507+0000 mgr.y (mgr.14901) 1847 : cluster [DBG] pgmap v1854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:07.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:07 smithi006 bash[33918]: audit 2023-08-27T23:00:06.561917+0000 mon.b (mon.1) 299 : audit [DBG] from='client.? 172.21.15.6:0/3406038266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:08.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:08 smithi130 bash[51302]: debug 2023-08-27T23:00:08.002+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:08.006826+0000) 2023-08-27T23:00:09.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:09 smithi130 bash[51302]: debug 2023-08-27T23:00:09.006+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:09.007081+0000) 2023-08-27T23:00:09.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:09 smithi006 bash[27292]: cluster 2023-08-27T23:00:08.387349+0000 mgr.y (mgr.14901) 1848 : cluster [DBG] pgmap v1855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:09.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:09 smithi006 bash[27292]: audit 2023-08-27T23:00:08.981255+0000 mon.a (mon.0) 1808 : audit [DBG] from='client.? 172.21.15.6:0/448771733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:09 smithi006 bash[33918]: cluster 2023-08-27T23:00:08.387349+0000 mgr.y (mgr.14901) 1848 : cluster [DBG] pgmap v1855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:09 smithi006 bash[33918]: audit 2023-08-27T23:00:08.981255+0000 mon.a (mon.0) 1808 : audit [DBG] from='client.? 172.21.15.6:0/448771733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:09.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:09 smithi130 bash[27579]: cluster 2023-08-27T23:00:08.387349+0000 mgr.y (mgr.14901) 1848 : cluster [DBG] pgmap v1855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:09 smithi130 bash[27579]: audit 2023-08-27T23:00:08.981255+0000 mon.a (mon.0) 1808 : audit [DBG] from='client.? 172.21.15.6:0/448771733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:10.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:10 smithi130 bash[51302]: debug 2023-08-27T23:00:10.006+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:10.007384+0000) 2023-08-27T23:00:10.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:00:10.350Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:10.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:00:10.351Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:11.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:00:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:00:10] "GET /metrics HTTP/1.1" 200 193094 "" "Prometheus/2.33.4" 2023-08-27T23:00:11.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:11 smithi130 bash[51302]: debug 2023-08-27T23:00:11.006+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:11.007597+0000) 2023-08-27T23:00:11.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:11 smithi006 bash[27292]: cluster 2023-08-27T23:00:10.388637+0000 mgr.y (mgr.14901) 1849 : cluster [DBG] pgmap v1856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:11.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:11 smithi006 bash[27292]: audit 2023-08-27T23:00:11.437892+0000 mon.a (mon.0) 1809 : audit [DBG] from='client.? 172.21.15.6:0/3573766041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:11.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:11 smithi006 bash[33918]: cluster 2023-08-27T23:00:10.388637+0000 mgr.y (mgr.14901) 1849 : cluster [DBG] pgmap v1856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:11.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:11 smithi006 bash[33918]: audit 2023-08-27T23:00:11.437892+0000 mon.a (mon.0) 1809 : audit [DBG] from='client.? 172.21.15.6:0/3573766041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:11.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:11 smithi130 bash[27579]: cluster 2023-08-27T23:00:10.388637+0000 mgr.y (mgr.14901) 1849 : cluster [DBG] pgmap v1856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:11.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:11 smithi130 bash[27579]: audit 2023-08-27T23:00:11.437892+0000 mon.a (mon.0) 1809 : audit [DBG] from='client.? 172.21.15.6:0/3573766041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:12.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:11 smithi006 bash[56298]: level=error ts=2023-08-27T23:00:11.912Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:12.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:11 smithi006 bash[56298]: level=warn ts=2023-08-27T23:00:11.914Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:12.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:12 smithi130 bash[51302]: debug 2023-08-27T23:00:12.006+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:12.007783+0000) 2023-08-27T23:00:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:13 smithi130 bash[51302]: debug 2023-08-27T23:00:13.006+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:13.007982+0000) 2023-08-27T23:00:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:13 smithi130 bash[27579]: audit 2023-08-27T23:00:11.995502+0000 mon.a (mon.0) 1810 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:00:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:13 smithi130 bash[27579]: cluster 2023-08-27T23:00:12.389293+0000 mgr.y (mgr.14901) 1850 : cluster [DBG] pgmap v1857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:13 smithi130 bash[27579]: audit 2023-08-27T23:00:12.474143+0000 mon.a (mon.0) 1811 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:00:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:13 smithi130 bash[27579]: audit 2023-08-27T23:00:12.750700+0000 mon.b (mon.1) 300 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:00:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:13 smithi130 bash[27579]: audit 2023-08-27T23:00:12.752994+0000 mon.b (mon.1) 301 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:00:13.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:13 smithi130 bash[27579]: audit 2023-08-27T23:00:12.761875+0000 mon.a (mon.0) 1812 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:00:13.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:13 smithi006 bash[33918]: audit 2023-08-27T23:00:11.995502+0000 mon.a (mon.0) 1810 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:00:13.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:13 smithi006 bash[33918]: cluster 2023-08-27T23:00:12.389293+0000 mgr.y (mgr.14901) 1850 : cluster [DBG] pgmap v1857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:13.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:13 smithi006 bash[33918]: audit 2023-08-27T23:00:12.474143+0000 mon.a (mon.0) 1811 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:00:13.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:13 smithi006 bash[33918]: audit 2023-08-27T23:00:12.750700+0000 mon.b (mon.1) 300 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:00:13.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:13 smithi006 bash[33918]: audit 2023-08-27T23:00:12.752994+0000 mon.b (mon.1) 301 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:00:13.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:13 smithi006 bash[33918]: audit 2023-08-27T23:00:12.761875+0000 mon.a (mon.0) 1812 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:00:13.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:13 smithi006 bash[27292]: audit 2023-08-27T23:00:11.995502+0000 mon.a (mon.0) 1810 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:00:13.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:13 smithi006 bash[27292]: cluster 2023-08-27T23:00:12.389293+0000 mgr.y (mgr.14901) 1850 : cluster [DBG] pgmap v1857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:13.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:13 smithi006 bash[27292]: audit 2023-08-27T23:00:12.474143+0000 mon.a (mon.0) 1811 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:00:13.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:13 smithi006 bash[27292]: audit 2023-08-27T23:00:12.750700+0000 mon.b (mon.1) 300 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:00:13.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:13 smithi006 bash[27292]: audit 2023-08-27T23:00:12.752994+0000 mon.b (mon.1) 301 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:00:13.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:13 smithi006 bash[27292]: audit 2023-08-27T23:00:12.761875+0000 mon.a (mon.0) 1812 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:00:14.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:14 smithi006 bash[27292]: audit 2023-08-27T23:00:13.863014+0000 mon.a (mon.0) 1813 : audit [DBG] from='client.? 172.21.15.6:0/3027548732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:14.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:14 smithi006 bash[33918]: audit 2023-08-27T23:00:13.863014+0000 mon.a (mon.0) 1813 : audit [DBG] from='client.? 172.21.15.6:0/3027548732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:14.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:00:13.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:14.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:00:13.812Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:14.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:14 smithi130 bash[51302]: debug 2023-08-27T23:00:14.006+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:14.008221+0000) 2023-08-27T23:00:14.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:14 smithi130 bash[27579]: audit 2023-08-27T23:00:13.863014+0000 mon.a (mon.0) 1813 : audit [DBG] from='client.? 172.21.15.6:0/3027548732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:15 smithi130 bash[27579]: cluster 2023-08-27T23:00:14.390386+0000 mgr.y (mgr.14901) 1851 : cluster [DBG] pgmap v1858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:15 smithi130 bash[51302]: debug 2023-08-27T23:00:15.006+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:15.008525+0000) 2023-08-27T23:00:15.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:15 smithi006 bash[27292]: cluster 2023-08-27T23:00:14.390386+0000 mgr.y (mgr.14901) 1851 : cluster [DBG] pgmap v1858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:15.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:15 smithi006 bash[33918]: cluster 2023-08-27T23:00:14.390386+0000 mgr.y (mgr.14901) 1851 : cluster [DBG] pgmap v1858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:16.336 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:16 smithi130 bash[51302]: debug 2023-08-27T23:00:16.006+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:16.008772+0000) 2023-08-27T23:00:16.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:16 smithi130 bash[27579]: audit 2023-08-27T23:00:16.276681+0000 mon.a (mon.0) 1814 : audit [DBG] from='client.? 172.21.15.6:0/1836380017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:16.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:16 smithi006 bash[33918]: audit 2023-08-27T23:00:16.276681+0000 mon.a (mon.0) 1814 : audit [DBG] from='client.? 172.21.15.6:0/1836380017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:16.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:16 smithi006 bash[27292]: audit 2023-08-27T23:00:16.276681+0000 mon.a (mon.0) 1814 : audit [DBG] from='client.? 172.21.15.6:0/1836380017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:17.345 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:17 smithi130 bash[51302]: debug 2023-08-27T23:00:17.006+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:17.009033+0000) 2023-08-27T23:00:17.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:17 smithi130 bash[27579]: cluster 2023-08-27T23:00:16.391170+0000 mgr.y (mgr.14901) 1852 : cluster [DBG] pgmap v1859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:17.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:17 smithi006 bash[27292]: cluster 2023-08-27T23:00:16.391170+0000 mgr.y (mgr.14901) 1852 : cluster [DBG] pgmap v1859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:17.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:17 smithi006 bash[33918]: cluster 2023-08-27T23:00:16.391170+0000 mgr.y (mgr.14901) 1852 : cluster [DBG] pgmap v1859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:18.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:18 smithi130 bash[51302]: debug 2023-08-27T23:00:18.006+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:18.009328+0000) 2023-08-27T23:00:19.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:19 smithi130 bash[51302]: debug 2023-08-27T23:00:19.006+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:19.009606+0000) 2023-08-27T23:00:19.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:19 smithi006 bash[27292]: cluster 2023-08-27T23:00:18.391842+0000 mgr.y (mgr.14901) 1853 : cluster [DBG] pgmap v1860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:19.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:19 smithi006 bash[27292]: audit 2023-08-27T23:00:18.689107+0000 mon.a (mon.0) 1815 : audit [DBG] from='client.? 172.21.15.6:0/1273649623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:19.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:19 smithi006 bash[33918]: cluster 2023-08-27T23:00:18.391842+0000 mgr.y (mgr.14901) 1853 : cluster [DBG] pgmap v1860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:19.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:19 smithi006 bash[33918]: audit 2023-08-27T23:00:18.689107+0000 mon.a (mon.0) 1815 : audit [DBG] from='client.? 172.21.15.6:0/1273649623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:19.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:19 smithi130 bash[27579]: cluster 2023-08-27T23:00:18.391842+0000 mgr.y (mgr.14901) 1853 : cluster [DBG] pgmap v1860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:19.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:19 smithi130 bash[27579]: audit 2023-08-27T23:00:18.689107+0000 mon.a (mon.0) 1815 : audit [DBG] from='client.? 172.21.15.6:0/1273649623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:20.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:20 smithi130 bash[51302]: debug 2023-08-27T23:00:20.006+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:20.009889+0000) 2023-08-27T23:00:20.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:00:20.351Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:20.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:00:20.353Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:21.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:00:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:00:20] "GET /metrics HTTP/1.1" 200 193091 "" "Prometheus/2.33.4" 2023-08-27T23:00:21.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:21 smithi130 bash[51302]: debug 2023-08-27T23:00:21.006+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:21.010166+0000) 2023-08-27T23:00:21.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:21 smithi006 bash[33918]: cluster 2023-08-27T23:00:20.392929+0000 mgr.y (mgr.14901) 1854 : cluster [DBG] pgmap v1861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:21.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:21 smithi006 bash[33918]: audit 2023-08-27T23:00:21.110563+0000 mon.a (mon.0) 1816 : audit [DBG] from='client.? 172.21.15.6:0/1863249036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:21.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:21 smithi006 bash[27292]: cluster 2023-08-27T23:00:20.392929+0000 mgr.y (mgr.14901) 1854 : cluster [DBG] pgmap v1861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:21.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:21 smithi006 bash[27292]: audit 2023-08-27T23:00:21.110563+0000 mon.a (mon.0) 1816 : audit [DBG] from='client.? 172.21.15.6:0/1863249036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:21.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:21 smithi130 bash[27579]: cluster 2023-08-27T23:00:20.392929+0000 mgr.y (mgr.14901) 1854 : cluster [DBG] pgmap v1861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:21.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:21 smithi130 bash[27579]: audit 2023-08-27T23:00:21.110563+0000 mon.a (mon.0) 1816 : audit [DBG] from='client.? 172.21.15.6:0/1863249036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:22.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:21 smithi006 bash[56298]: level=error ts=2023-08-27T23:00:21.912Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:22.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:21 smithi006 bash[56298]: level=warn ts=2023-08-27T23:00:21.914Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:22.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:22 smithi130 bash[51302]: debug 2023-08-27T23:00:22.006+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:22.010483+0000) 2023-08-27T23:00:23.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:23 smithi130 bash[51302]: debug 2023-08-27T23:00:23.006+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:23.010768+0000) 2023-08-27T23:00:23.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:23 smithi006 bash[27292]: cluster 2023-08-27T23:00:22.393669+0000 mgr.y (mgr.14901) 1855 : cluster [DBG] pgmap v1862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:23.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:23 smithi006 bash[33918]: cluster 2023-08-27T23:00:22.393669+0000 mgr.y (mgr.14901) 1855 : cluster [DBG] pgmap v1862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:23.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:23 smithi130 bash[27579]: cluster 2023-08-27T23:00:22.393669+0000 mgr.y (mgr.14901) 1855 : cluster [DBG] pgmap v1862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:24.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:00:23.809Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:24.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:00:23.811Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:24 smithi130 bash[51302]: debug 2023-08-27T23:00:24.010+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:24.011029+0000) 2023-08-27T23:00:24.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:24 smithi006 bash[33918]: audit 2023-08-27T23:00:23.531999+0000 mon.a (mon.0) 1817 : audit [DBG] from='client.? 172.21.15.6:0/866459678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:24.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:24 smithi006 bash[33918]: cluster 2023-08-27T23:00:24.394788+0000 mgr.y (mgr.14901) 1856 : cluster [DBG] pgmap v1863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:24.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:24 smithi006 bash[27292]: audit 2023-08-27T23:00:23.531999+0000 mon.a (mon.0) 1817 : audit [DBG] from='client.? 172.21.15.6:0/866459678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:24.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:24 smithi006 bash[27292]: cluster 2023-08-27T23:00:24.394788+0000 mgr.y (mgr.14901) 1856 : cluster [DBG] pgmap v1863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:24 smithi130 bash[27579]: audit 2023-08-27T23:00:23.531999+0000 mon.a (mon.0) 1817 : audit [DBG] from='client.? 172.21.15.6:0/866459678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:24 smithi130 bash[27579]: cluster 2023-08-27T23:00:24.394788+0000 mgr.y (mgr.14901) 1856 : cluster [DBG] pgmap v1863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:25 smithi130 bash[51302]: debug 2023-08-27T23:00:25.010+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:25.011258+0000) 2023-08-27T23:00:26.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:26 smithi130 bash[51302]: debug 2023-08-27T23:00:26.010+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:26.011482+0000) 2023-08-27T23:00:26.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:26 smithi130 bash[27579]: audit 2023-08-27T23:00:25.950316+0000 mon.a (mon.0) 1818 : audit [DBG] from='client.? 172.21.15.6:0/2689226947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:26.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:26 smithi006 bash[27292]: audit 2023-08-27T23:00:25.950316+0000 mon.a (mon.0) 1818 : audit [DBG] from='client.? 172.21.15.6:0/2689226947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:26.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:26 smithi006 bash[33918]: audit 2023-08-27T23:00:25.950316+0000 mon.a (mon.0) 1818 : audit [DBG] from='client.? 172.21.15.6:0/2689226947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:27.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:27 smithi130 bash[27579]: cluster 2023-08-27T23:00:26.395361+0000 mgr.y (mgr.14901) 1857 : cluster [DBG] pgmap v1864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:27.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:27 smithi130 bash[51302]: debug 2023-08-27T23:00:27.010+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:27.011829+0000) 2023-08-27T23:00:27.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:27 smithi006 bash[33918]: cluster 2023-08-27T23:00:26.395361+0000 mgr.y (mgr.14901) 1857 : cluster [DBG] pgmap v1864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:27.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:27 smithi006 bash[27292]: cluster 2023-08-27T23:00:26.395361+0000 mgr.y (mgr.14901) 1857 : cluster [DBG] pgmap v1864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:28.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:28 smithi130 bash[51302]: debug 2023-08-27T23:00:28.010+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:28.012103+0000) 2023-08-27T23:00:28.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:28 smithi006 bash[33918]: audit 2023-08-27T23:00:28.360488+0000 mon.a (mon.0) 1819 : audit [DBG] from='client.? 172.21.15.6:0/1536253017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:28.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:28 smithi006 bash[27292]: audit 2023-08-27T23:00:28.360488+0000 mon.a (mon.0) 1819 : audit [DBG] from='client.? 172.21.15.6:0/1536253017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:28 smithi130 bash[27579]: audit 2023-08-27T23:00:28.360488+0000 mon.a (mon.0) 1819 : audit [DBG] from='client.? 172.21.15.6:0/1536253017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:29.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:29 smithi130 bash[51302]: debug 2023-08-27T23:00:29.010+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:29.012381+0000) 2023-08-27T23:00:29.763 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:29 smithi006 bash[27292]: cluster 2023-08-27T23:00:28.396003+0000 mgr.y (mgr.14901) 1858 : cluster [DBG] pgmap v1865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:29.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:29 smithi130 bash[27579]: cluster 2023-08-27T23:00:28.396003+0000 mgr.y (mgr.14901) 1858 : cluster [DBG] pgmap v1865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:30.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:29 smithi006 bash[33918]: cluster 2023-08-27T23:00:28.396003+0000 mgr.y (mgr.14901) 1858 : cluster [DBG] pgmap v1865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:30.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:30 smithi130 bash[51302]: debug 2023-08-27T23:00:30.010+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:30.012646+0000) 2023-08-27T23:00:30.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:00:30.351Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:30.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:00:30.353Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:31.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:00:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:00:30] "GET /metrics HTTP/1.1" 200 193091 "" "Prometheus/2.33.4" 2023-08-27T23:00:31.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:31 smithi130 bash[51302]: debug 2023-08-27T23:00:31.010+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:31.012908+0000) 2023-08-27T23:00:31.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:31 smithi006 bash[27292]: cluster 2023-08-27T23:00:30.397064+0000 mgr.y (mgr.14901) 1859 : cluster [DBG] pgmap v1866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:31.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:31 smithi006 bash[27292]: audit 2023-08-27T23:00:30.776667+0000 mon.a (mon.0) 1820 : audit [DBG] from='client.? 172.21.15.6:0/2225162802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:31.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:31 smithi006 bash[33918]: cluster 2023-08-27T23:00:30.397064+0000 mgr.y (mgr.14901) 1859 : cluster [DBG] pgmap v1866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:31.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:31 smithi006 bash[33918]: audit 2023-08-27T23:00:30.776667+0000 mon.a (mon.0) 1820 : audit [DBG] from='client.? 172.21.15.6:0/2225162802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:31.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:31 smithi130 bash[27579]: cluster 2023-08-27T23:00:30.397064+0000 mgr.y (mgr.14901) 1859 : cluster [DBG] pgmap v1866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:31.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:31 smithi130 bash[27579]: audit 2023-08-27T23:00:30.776667+0000 mon.a (mon.0) 1820 : audit [DBG] from='client.? 172.21.15.6:0/2225162802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:32.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:31 smithi006 bash[56298]: level=error ts=2023-08-27T23:00:31.913Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:32.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:31 smithi006 bash[56298]: level=warn ts=2023-08-27T23:00:31.915Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:32.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:32 smithi130 bash[51302]: debug 2023-08-27T23:00:32.010+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:32.013202+0000) 2023-08-27T23:00:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:32 smithi130 bash[27579]: cluster 2023-08-27T23:00:32.397697+0000 mgr.y (mgr.14901) 1860 : cluster [DBG] pgmap v1867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:32.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:32 smithi006 bash[33918]: cluster 2023-08-27T23:00:32.397697+0000 mgr.y (mgr.14901) 1860 : cluster [DBG] pgmap v1867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:32.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:32 smithi006 bash[27292]: cluster 2023-08-27T23:00:32.397697+0000 mgr.y (mgr.14901) 1860 : cluster [DBG] pgmap v1867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:33.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:33 smithi130 bash[51302]: debug 2023-08-27T23:00:33.010+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:33.013471+0000) 2023-08-27T23:00:33.814 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:33 smithi006 bash[27292]: audit 2023-08-27T23:00:33.191823+0000 mon.c (mon.2) 735 : audit [DBG] from='client.? 172.21.15.6:0/83207633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:33.814 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:33 smithi006 bash[33918]: audit 2023-08-27T23:00:33.191823+0000 mon.c (mon.2) 735 : audit [DBG] from='client.? 172.21.15.6:0/83207633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:33 smithi130 bash[27579]: audit 2023-08-27T23:00:33.191823+0000 mon.c (mon.2) 735 : audit [DBG] from='client.? 172.21.15.6:0/83207633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:34.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:00:33.809Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:34.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:00:33.811Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:34.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:34 smithi130 bash[51302]: debug 2023-08-27T23:00:34.010+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:34.013780+0000) 2023-08-27T23:00:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:35 smithi130 bash[27579]: cluster 2023-08-27T23:00:34.398822+0000 mgr.y (mgr.14901) 1861 : cluster [DBG] pgmap v1868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:35.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:35 smithi130 bash[51302]: debug 2023-08-27T23:00:35.010+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:35.014066+0000) 2023-08-27T23:00:35.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:35 smithi006 bash[33918]: cluster 2023-08-27T23:00:34.398822+0000 mgr.y (mgr.14901) 1861 : cluster [DBG] pgmap v1868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:35.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:35 smithi006 bash[27292]: cluster 2023-08-27T23:00:34.398822+0000 mgr.y (mgr.14901) 1861 : cluster [DBG] pgmap v1868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:36 smithi130 bash[27579]: audit 2023-08-27T23:00:35.600576+0000 mon.a (mon.0) 1821 : audit [DBG] from='client.? 172.21.15.6:0/3872458636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:36.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:36 smithi130 bash[51302]: debug 2023-08-27T23:00:36.010+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:36.014363+0000) 2023-08-27T23:00:36.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:36 smithi006 bash[27292]: audit 2023-08-27T23:00:35.600576+0000 mon.a (mon.0) 1821 : audit [DBG] from='client.? 172.21.15.6:0/3872458636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:36.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:36 smithi006 bash[33918]: audit 2023-08-27T23:00:35.600576+0000 mon.a (mon.0) 1821 : audit [DBG] from='client.? 172.21.15.6:0/3872458636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:37.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:37 smithi130 bash[27579]: cluster 2023-08-27T23:00:36.399482+0000 mgr.y (mgr.14901) 1862 : cluster [DBG] pgmap v1869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:37 smithi130 bash[51302]: debug 2023-08-27T23:00:37.010+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:37.014618+0000) 2023-08-27T23:00:37.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:37 smithi006 bash[27292]: cluster 2023-08-27T23:00:36.399482+0000 mgr.y (mgr.14901) 1862 : cluster [DBG] pgmap v1869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:37.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:37 smithi006 bash[33918]: cluster 2023-08-27T23:00:36.399482+0000 mgr.y (mgr.14901) 1862 : cluster [DBG] pgmap v1869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:38.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:38 smithi130 bash[27579]: audit 2023-08-27T23:00:38.015129+0000 mon.c (mon.2) 736 : audit [DBG] from='client.? 172.21.15.6:0/523794819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:38.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:38 smithi130 bash[51302]: debug 2023-08-27T23:00:38.014+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:38.014937+0000) 2023-08-27T23:00:38.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:38 smithi006 bash[27292]: audit 2023-08-27T23:00:38.015129+0000 mon.c (mon.2) 736 : audit [DBG] from='client.? 172.21.15.6:0/523794819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:38.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:38 smithi006 bash[33918]: audit 2023-08-27T23:00:38.015129+0000 mon.c (mon.2) 736 : audit [DBG] from='client.? 172.21.15.6:0/523794819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:39.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:39 smithi130 bash[27579]: cluster 2023-08-27T23:00:38.400135+0000 mgr.y (mgr.14901) 1863 : cluster [DBG] pgmap v1870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:39.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:39 smithi130 bash[51302]: debug 2023-08-27T23:00:39.014+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:39.015196+0000) 2023-08-27T23:00:39.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:39 smithi006 bash[33918]: cluster 2023-08-27T23:00:38.400135+0000 mgr.y (mgr.14901) 1863 : cluster [DBG] pgmap v1870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:39.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:39 smithi006 bash[27292]: cluster 2023-08-27T23:00:38.400135+0000 mgr.y (mgr.14901) 1863 : cluster [DBG] pgmap v1870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:40.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:40 smithi130 bash[51302]: debug 2023-08-27T23:00:40.014+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:40.015423+0000) 2023-08-27T23:00:40.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:40 smithi006 bash[27292]: cluster 2023-08-27T23:00:40.401120+0000 mgr.y (mgr.14901) 1864 : cluster [DBG] pgmap v1871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:40.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:40 smithi006 bash[27292]: audit 2023-08-27T23:00:40.445059+0000 mon.c (mon.2) 737 : audit [DBG] from='client.? 172.21.15.6:0/1471510021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:40.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:40 smithi006 bash[33918]: cluster 2023-08-27T23:00:40.401120+0000 mgr.y (mgr.14901) 1864 : cluster [DBG] pgmap v1871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:40.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:40 smithi006 bash[33918]: audit 2023-08-27T23:00:40.445059+0000 mon.c (mon.2) 737 : audit [DBG] from='client.? 172.21.15.6:0/1471510021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:40.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:00:40.352Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:40.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:00:40.353Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:40 smithi130 bash[27579]: cluster 2023-08-27T23:00:40.401120+0000 mgr.y (mgr.14901) 1864 : cluster [DBG] pgmap v1871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:40 smithi130 bash[27579]: audit 2023-08-27T23:00:40.445059+0000 mon.c (mon.2) 737 : audit [DBG] from='client.? 172.21.15.6:0/1471510021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:41.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:00:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:00:40] "GET /metrics HTTP/1.1" 200 193091 "" "Prometheus/2.33.4" 2023-08-27T23:00:41.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:41 smithi130 bash[51302]: debug 2023-08-27T23:00:41.014+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:41.015731+0000) 2023-08-27T23:00:42.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:41 smithi006 bash[56298]: level=error ts=2023-08-27T23:00:41.914Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:42.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:41 smithi006 bash[56298]: level=warn ts=2023-08-27T23:00:41.915Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:42.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:42 smithi130 bash[51302]: debug 2023-08-27T23:00:42.014+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:42.016003+0000) 2023-08-27T23:00:43.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:43 smithi130 bash[51302]: debug 2023-08-27T23:00:43.014+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:43.016287+0000) 2023-08-27T23:00:43.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:43 smithi006 bash[27292]: cluster 2023-08-27T23:00:42.401785+0000 mgr.y (mgr.14901) 1865 : cluster [DBG] pgmap v1872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:43.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:43 smithi006 bash[27292]: audit 2023-08-27T23:00:42.863534+0000 mon.a (mon.0) 1822 : audit [DBG] from='client.? 172.21.15.6:0/1442272356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:43.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:43 smithi006 bash[33918]: cluster 2023-08-27T23:00:42.401785+0000 mgr.y (mgr.14901) 1865 : cluster [DBG] pgmap v1872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:43.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:43 smithi006 bash[33918]: audit 2023-08-27T23:00:42.863534+0000 mon.a (mon.0) 1822 : audit [DBG] from='client.? 172.21.15.6:0/1442272356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:43.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:43 smithi130 bash[27579]: cluster 2023-08-27T23:00:42.401785+0000 mgr.y (mgr.14901) 1865 : cluster [DBG] pgmap v1872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:43.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:43 smithi130 bash[27579]: audit 2023-08-27T23:00:42.863534+0000 mon.a (mon.0) 1822 : audit [DBG] from='client.? 172.21.15.6:0/1442272356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:44.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:00:43.810Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:44.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:00:43.812Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:44.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:44 smithi130 bash[51302]: debug 2023-08-27T23:00:44.014+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:44.016586+0000) 2023-08-27T23:00:45.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:45 smithi130 bash[27579]: cluster 2023-08-27T23:00:44.402917+0000 mgr.y (mgr.14901) 1866 : cluster [DBG] pgmap v1873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:45 smithi130 bash[51302]: debug 2023-08-27T23:00:45.014+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:45.016890+0000) 2023-08-27T23:00:45.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:45 smithi006 bash[33918]: cluster 2023-08-27T23:00:44.402917+0000 mgr.y (mgr.14901) 1866 : cluster [DBG] pgmap v1873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:45.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:45 smithi006 bash[27292]: cluster 2023-08-27T23:00:44.402917+0000 mgr.y (mgr.14901) 1866 : cluster [DBG] pgmap v1873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:46 smithi130 bash[27579]: audit 2023-08-27T23:00:45.277311+0000 mon.c (mon.2) 738 : audit [DBG] from='client.? 172.21.15.6:0/3385733345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:46 smithi130 bash[51302]: debug 2023-08-27T23:00:46.014+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:46.017108+0000) 2023-08-27T23:00:46.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:46 smithi006 bash[27292]: audit 2023-08-27T23:00:45.277311+0000 mon.c (mon.2) 738 : audit [DBG] from='client.? 172.21.15.6:0/3385733345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:46.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:46 smithi006 bash[33918]: audit 2023-08-27T23:00:45.277311+0000 mon.c (mon.2) 738 : audit [DBG] from='client.? 172.21.15.6:0/3385733345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:47.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:47 smithi130 bash[27579]: cluster 2023-08-27T23:00:46.403543+0000 mgr.y (mgr.14901) 1867 : cluster [DBG] pgmap v1874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:47 smithi130 bash[51302]: debug 2023-08-27T23:00:47.014+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:47.017373+0000) 2023-08-27T23:00:47.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:47 smithi006 bash[33918]: cluster 2023-08-27T23:00:46.403543+0000 mgr.y (mgr.14901) 1867 : cluster [DBG] pgmap v1874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:47.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:47 smithi006 bash[27292]: cluster 2023-08-27T23:00:46.403543+0000 mgr.y (mgr.14901) 1867 : cluster [DBG] pgmap v1874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:48.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:48 smithi130 bash[27579]: audit 2023-08-27T23:00:47.689043+0000 mon.c (mon.2) 739 : audit [DBG] from='client.? 172.21.15.6:0/2171644405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:48.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:48 smithi130 bash[51302]: debug 2023-08-27T23:00:48.014+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:48.017661+0000) 2023-08-27T23:00:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:48 smithi006 bash[33918]: audit 2023-08-27T23:00:47.689043+0000 mon.c (mon.2) 739 : audit [DBG] from='client.? 172.21.15.6:0/2171644405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:48.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:48 smithi006 bash[27292]: audit 2023-08-27T23:00:47.689043+0000 mon.c (mon.2) 739 : audit [DBG] from='client.? 172.21.15.6:0/2171644405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:49.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:49 smithi130 bash[27579]: cluster 2023-08-27T23:00:48.404195+0000 mgr.y (mgr.14901) 1868 : cluster [DBG] pgmap v1875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:49 smithi130 bash[51302]: debug 2023-08-27T23:00:49.014+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:49.017979+0000) 2023-08-27T23:00:49.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:49 smithi006 bash[27292]: cluster 2023-08-27T23:00:48.404195+0000 mgr.y (mgr.14901) 1868 : cluster [DBG] pgmap v1875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:49.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:49 smithi006 bash[33918]: cluster 2023-08-27T23:00:48.404195+0000 mgr.y (mgr.14901) 1868 : cluster [DBG] pgmap v1875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:50.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:50 smithi130 bash[27579]: audit 2023-08-27T23:00:50.106034+0000 mon.a (mon.0) 1823 : audit [DBG] from='client.? 172.21.15.6:0/417957406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:50 smithi130 bash[51302]: debug 2023-08-27T23:00:50.014+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:50.018281+0000) 2023-08-27T23:00:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:50 smithi006 bash[27292]: audit 2023-08-27T23:00:50.106034+0000 mon.a (mon.0) 1823 : audit [DBG] from='client.? 172.21.15.6:0/417957406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:50.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:50 smithi006 bash[33918]: audit 2023-08-27T23:00:50.106034+0000 mon.a (mon.0) 1823 : audit [DBG] from='client.? 172.21.15.6:0/417957406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:50.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:00:50.352Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:50.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:00:50.354Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:51.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:00:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:00:50] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:00:51.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:51 smithi130 bash[27579]: cluster 2023-08-27T23:00:50.405384+0000 mgr.y (mgr.14901) 1869 : cluster [DBG] pgmap v1876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:51.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:51 smithi130 bash[51302]: debug 2023-08-27T23:00:51.014+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:51.018585+0000) 2023-08-27T23:00:51.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:51 smithi006 bash[27292]: cluster 2023-08-27T23:00:50.405384+0000 mgr.y (mgr.14901) 1869 : cluster [DBG] pgmap v1876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:51.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:51 smithi006 bash[33918]: cluster 2023-08-27T23:00:50.405384+0000 mgr.y (mgr.14901) 1869 : cluster [DBG] pgmap v1876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:52.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:51 smithi006 bash[56298]: level=error ts=2023-08-27T23:00:51.914Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:52.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:51 smithi006 bash[56298]: level=warn ts=2023-08-27T23:00:51.916Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:52.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:52 smithi130 bash[51302]: debug 2023-08-27T23:00:52.018+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:52.018908+0000) 2023-08-27T23:00:53.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:53 smithi130 bash[51302]: debug 2023-08-27T23:00:53.018+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:53.019155+0000) 2023-08-27T23:00:53.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:53 smithi006 bash[27292]: cluster 2023-08-27T23:00:52.405881+0000 mgr.y (mgr.14901) 1870 : cluster [DBG] pgmap v1877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:53.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:53 smithi006 bash[27292]: audit 2023-08-27T23:00:52.524427+0000 mon.a (mon.0) 1824 : audit [DBG] from='client.? 172.21.15.6:0/2963354053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:53.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:53 smithi006 bash[33918]: cluster 2023-08-27T23:00:52.405881+0000 mgr.y (mgr.14901) 1870 : cluster [DBG] pgmap v1877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:53.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:53 smithi006 bash[33918]: audit 2023-08-27T23:00:52.524427+0000 mon.a (mon.0) 1824 : audit [DBG] from='client.? 172.21.15.6:0/2963354053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:53.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:53 smithi130 bash[27579]: cluster 2023-08-27T23:00:52.405881+0000 mgr.y (mgr.14901) 1870 : cluster [DBG] pgmap v1877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:53.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:53 smithi130 bash[27579]: audit 2023-08-27T23:00:52.524427+0000 mon.a (mon.0) 1824 : audit [DBG] from='client.? 172.21.15.6:0/2963354053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:54.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:00:53.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:54.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:00:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:00:53.813Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:00:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:54 smithi130 bash[51302]: debug 2023-08-27T23:00:54.018+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:54.019394+0000) 2023-08-27T23:00:55.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:55 smithi130 bash[51302]: debug 2023-08-27T23:00:55.018+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:55.019700+0000) 2023-08-27T23:00:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:55 smithi130 bash[27579]: cluster 2023-08-27T23:00:54.406929+0000 mgr.y (mgr.14901) 1871 : cluster [DBG] pgmap v1878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:55 smithi130 bash[27579]: audit 2023-08-27T23:00:54.937867+0000 mon.c (mon.2) 740 : audit [DBG] from='client.? 172.21.15.6:0/306892861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:55.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:55 smithi006 bash[27292]: cluster 2023-08-27T23:00:54.406929+0000 mgr.y (mgr.14901) 1871 : cluster [DBG] pgmap v1878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:55.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:55 smithi006 bash[27292]: audit 2023-08-27T23:00:54.937867+0000 mon.c (mon.2) 740 : audit [DBG] from='client.? 172.21.15.6:0/306892861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:55.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:55 smithi006 bash[33918]: cluster 2023-08-27T23:00:54.406929+0000 mgr.y (mgr.14901) 1871 : cluster [DBG] pgmap v1878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:55.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:55 smithi006 bash[33918]: audit 2023-08-27T23:00:54.937867+0000 mon.c (mon.2) 740 : audit [DBG] from='client.? 172.21.15.6:0/306892861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:56.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:56 smithi130 bash[51302]: debug 2023-08-27T23:00:56.017+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:56.019942+0000) 2023-08-27T23:00:57.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:57 smithi130 bash[51302]: debug 2023-08-27T23:00:57.017+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:57.020245+0000) 2023-08-27T23:00:57.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:57 smithi130 bash[27579]: cluster 2023-08-27T23:00:56.407610+0000 mgr.y (mgr.14901) 1872 : cluster [DBG] pgmap v1879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:57.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:57 smithi130 bash[27579]: audit 2023-08-27T23:00:57.351573+0000 mon.a (mon.0) 1825 : audit [DBG] from='client.? 172.21.15.6:0/3307781596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:57.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:57 smithi006 bash[33918]: cluster 2023-08-27T23:00:56.407610+0000 mgr.y (mgr.14901) 1872 : cluster [DBG] pgmap v1879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:57.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:57 smithi006 bash[33918]: audit 2023-08-27T23:00:57.351573+0000 mon.a (mon.0) 1825 : audit [DBG] from='client.? 172.21.15.6:0/3307781596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:57.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:57 smithi006 bash[27292]: cluster 2023-08-27T23:00:56.407610+0000 mgr.y (mgr.14901) 1872 : cluster [DBG] pgmap v1879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:57.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:57 smithi006 bash[27292]: audit 2023-08-27T23:00:57.351573+0000 mon.a (mon.0) 1825 : audit [DBG] from='client.? 172.21.15.6:0/3307781596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:58.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:58 smithi130 bash[51302]: debug 2023-08-27T23:00:58.017+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:58.020541+0000) 2023-08-27T23:00:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:00:58 smithi130 bash[27579]: cluster 2023-08-27T23:00:58.408234+0000 mgr.y (mgr.14901) 1873 : cluster [DBG] pgmap v1880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:58.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:00:58 smithi006 bash[33918]: cluster 2023-08-27T23:00:58.408234+0000 mgr.y (mgr.14901) 1873 : cluster [DBG] pgmap v1880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:58.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:00:58 smithi006 bash[27292]: cluster 2023-08-27T23:00:58.408234+0000 mgr.y (mgr.14901) 1873 : cluster [DBG] pgmap v1880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:00:59.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:00:59 smithi130 bash[51302]: debug 2023-08-27T23:00:59.017+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:59.020836+0000) 2023-08-27T23:01:00.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:00 smithi130 bash[27579]: audit 2023-08-27T23:00:59.763881+0000 mon.a (mon.0) 1826 : audit [DBG] from='client.? 172.21.15.6:0/4021343063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:00.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:00 smithi130 bash[51302]: debug 2023-08-27T23:01:00.017+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:00.021100+0000) 2023-08-27T23:01:00.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:00 smithi006 bash[33918]: audit 2023-08-27T23:00:59.763881+0000 mon.a (mon.0) 1826 : audit [DBG] from='client.? 172.21.15.6:0/4021343063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:00.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:00 smithi006 bash[27292]: audit 2023-08-27T23:00:59.763881+0000 mon.a (mon.0) 1826 : audit [DBG] from='client.? 172.21.15.6:0/4021343063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:00.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:01:00.353Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:00.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:01:00.354Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:01.212 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:01:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:01:00] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:01:01.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:01 smithi130 bash[27579]: cluster 2023-08-27T23:01:00.409387+0000 mgr.y (mgr.14901) 1874 : cluster [DBG] pgmap v1881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:01.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:01 smithi130 bash[51302]: debug 2023-08-27T23:01:01.017+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:01.021381+0000) 2023-08-27T23:01:01.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:01 smithi006 bash[27292]: cluster 2023-08-27T23:01:00.409387+0000 mgr.y (mgr.14901) 1874 : cluster [DBG] pgmap v1881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:01.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:01 smithi006 bash[33918]: cluster 2023-08-27T23:01:00.409387+0000 mgr.y (mgr.14901) 1874 : cluster [DBG] pgmap v1881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:02.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:01 smithi006 bash[56298]: level=error ts=2023-08-27T23:01:01.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:02.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:01 smithi006 bash[56298]: level=warn ts=2023-08-27T23:01:01.916Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:02.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:02 smithi130 bash[27579]: audit 2023-08-27T23:01:02.191922+0000 mon.c (mon.2) 741 : audit [DBG] from='client.? 172.21.15.6:0/4160804687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:02.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:02 smithi130 bash[51302]: debug 2023-08-27T23:01:02.017+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:02.021647+0000) 2023-08-27T23:01:02.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:02 smithi006 bash[27292]: audit 2023-08-27T23:01:02.191922+0000 mon.c (mon.2) 741 : audit [DBG] from='client.? 172.21.15.6:0/4160804687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:02.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:02 smithi006 bash[33918]: audit 2023-08-27T23:01:02.191922+0000 mon.c (mon.2) 741 : audit [DBG] from='client.? 172.21.15.6:0/4160804687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:03.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:03 smithi130 bash[27579]: cluster 2023-08-27T23:01:02.410067+0000 mgr.y (mgr.14901) 1875 : cluster [DBG] pgmap v1882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:03 smithi130 bash[51302]: debug 2023-08-27T23:01:03.017+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:03.021918+0000) 2023-08-27T23:01:03.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:03 smithi006 bash[27292]: cluster 2023-08-27T23:01:02.410067+0000 mgr.y (mgr.14901) 1875 : cluster [DBG] pgmap v1882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:03.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:03 smithi006 bash[33918]: cluster 2023-08-27T23:01:02.410067+0000 mgr.y (mgr.14901) 1875 : cluster [DBG] pgmap v1882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:04.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:01:03.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:04.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:01:03.813Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:04.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:04 smithi130 bash[51302]: debug 2023-08-27T23:01:04.021+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:04.022189+0000) 2023-08-27T23:01:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:05 smithi130 bash[27579]: cluster 2023-08-27T23:01:04.411246+0000 mgr.y (mgr.14901) 1876 : cluster [DBG] pgmap v1883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:05 smithi130 bash[27579]: audit 2023-08-27T23:01:04.608281+0000 mon.a (mon.0) 1827 : audit [DBG] from='client.? 172.21.15.6:0/2692968164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:05 smithi130 bash[51302]: debug 2023-08-27T23:01:05.017+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:05.022481+0000) 2023-08-27T23:01:05.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:05 smithi006 bash[33918]: cluster 2023-08-27T23:01:04.411246+0000 mgr.y (mgr.14901) 1876 : cluster [DBG] pgmap v1883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:05.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:05 smithi006 bash[33918]: audit 2023-08-27T23:01:04.608281+0000 mon.a (mon.0) 1827 : audit [DBG] from='client.? 172.21.15.6:0/2692968164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:05.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:05 smithi006 bash[27292]: cluster 2023-08-27T23:01:04.411246+0000 mgr.y (mgr.14901) 1876 : cluster [DBG] pgmap v1883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:05.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:05 smithi006 bash[27292]: audit 2023-08-27T23:01:04.608281+0000 mon.a (mon.0) 1827 : audit [DBG] from='client.? 172.21.15.6:0/2692968164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:06.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:06 smithi130 bash[51302]: debug 2023-08-27T23:01:06.017+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:06.022746+0000) 2023-08-27T23:01:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:06 smithi130 bash[27579]: cluster 2023-08-27T23:01:06.411934+0000 mgr.y (mgr.14901) 1877 : cluster [DBG] pgmap v1884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:06.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:06 smithi006 bash[33918]: cluster 2023-08-27T23:01:06.411934+0000 mgr.y (mgr.14901) 1877 : cluster [DBG] pgmap v1884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:06.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:06 smithi006 bash[27292]: cluster 2023-08-27T23:01:06.411934+0000 mgr.y (mgr.14901) 1877 : cluster [DBG] pgmap v1884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:07.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:07 smithi130 bash[51302]: debug 2023-08-27T23:01:07.021+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:07.023004+0000) 2023-08-27T23:01:07.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:07 smithi130 bash[27579]: audit 2023-08-27T23:01:07.017630+0000 mon.c (mon.2) 742 : audit [DBG] from='client.? 172.21.15.6:0/414459525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:07.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:07 smithi006 bash[27292]: audit 2023-08-27T23:01:07.017630+0000 mon.c (mon.2) 742 : audit [DBG] from='client.? 172.21.15.6:0/414459525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:07.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:07 smithi006 bash[33918]: audit 2023-08-27T23:01:07.017630+0000 mon.c (mon.2) 742 : audit [DBG] from='client.? 172.21.15.6:0/414459525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:08.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:08 smithi130 bash[51302]: debug 2023-08-27T23:01:08.021+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:08.023272+0000) 2023-08-27T23:01:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:08 smithi130 bash[27579]: cluster 2023-08-27T23:01:08.412573+0000 mgr.y (mgr.14901) 1878 : cluster [DBG] pgmap v1885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:08.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:08 smithi006 bash[27292]: cluster 2023-08-27T23:01:08.412573+0000 mgr.y (mgr.14901) 1878 : cluster [DBG] pgmap v1885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:08.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:08 smithi006 bash[33918]: cluster 2023-08-27T23:01:08.412573+0000 mgr.y (mgr.14901) 1878 : cluster [DBG] pgmap v1885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:09.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:09 smithi130 bash[51302]: debug 2023-08-27T23:01:09.021+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:09.023569+0000) 2023-08-27T23:01:10.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:10 smithi130 bash[27579]: audit 2023-08-27T23:01:09.452418+0000 mon.c (mon.2) 743 : audit [DBG] from='client.? 172.21.15.6:0/1739864869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:10.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:10 smithi130 bash[51302]: debug 2023-08-27T23:01:10.021+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:10.023822+0000) 2023-08-27T23:01:10.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:10 smithi006 bash[27292]: audit 2023-08-27T23:01:09.452418+0000 mon.c (mon.2) 743 : audit [DBG] from='client.? 172.21.15.6:0/1739864869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:10.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:10 smithi006 bash[33918]: audit 2023-08-27T23:01:09.452418+0000 mon.c (mon.2) 743 : audit [DBG] from='client.? 172.21.15.6:0/1739864869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:10.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:01:10.353Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:10.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:01:10.355Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:11.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:01:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:01:10] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:01:11.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:11 smithi130 bash[27579]: cluster 2023-08-27T23:01:10.413741+0000 mgr.y (mgr.14901) 1879 : cluster [DBG] pgmap v1886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:11.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:11 smithi130 bash[51302]: debug 2023-08-27T23:01:11.021+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:11.024073+0000) 2023-08-27T23:01:11.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:11 smithi006 bash[33918]: cluster 2023-08-27T23:01:10.413741+0000 mgr.y (mgr.14901) 1879 : cluster [DBG] pgmap v1886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:11.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:11 smithi006 bash[27292]: cluster 2023-08-27T23:01:10.413741+0000 mgr.y (mgr.14901) 1879 : cluster [DBG] pgmap v1886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:12.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:11 smithi006 bash[56298]: level=error ts=2023-08-27T23:01:11.916Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:12.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:11 smithi006 bash[56298]: level=warn ts=2023-08-27T23:01:11.918Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:12.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:12 smithi130 bash[27579]: audit 2023-08-27T23:01:11.859313+0000 mon.c (mon.2) 744 : audit [DBG] from='client.? 172.21.15.6:0/827819459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:12 smithi130 bash[51302]: debug 2023-08-27T23:01:12.021+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:12.024375+0000) 2023-08-27T23:01:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:12 smithi006 bash[33918]: audit 2023-08-27T23:01:11.859313+0000 mon.c (mon.2) 744 : audit [DBG] from='client.? 172.21.15.6:0/827819459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:12 smithi006 bash[27292]: audit 2023-08-27T23:01:11.859313+0000 mon.c (mon.2) 744 : audit [DBG] from='client.? 172.21.15.6:0/827819459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:13.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:13 smithi130 bash[27579]: cluster 2023-08-27T23:01:12.414463+0000 mgr.y (mgr.14901) 1880 : cluster [DBG] pgmap v1887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:13 smithi130 bash[27579]: audit 2023-08-27T23:01:12.775871+0000 mon.b (mon.1) 302 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:01:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:13 smithi130 bash[51302]: debug 2023-08-27T23:01:13.021+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:13.024663+0000) 2023-08-27T23:01:13.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:13 smithi006 bash[33918]: cluster 2023-08-27T23:01:12.414463+0000 mgr.y (mgr.14901) 1880 : cluster [DBG] pgmap v1887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:13.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:13 smithi006 bash[33918]: audit 2023-08-27T23:01:12.775871+0000 mon.b (mon.1) 302 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:01:13.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:13 smithi006 bash[27292]: cluster 2023-08-27T23:01:12.414463+0000 mgr.y (mgr.14901) 1880 : cluster [DBG] pgmap v1887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:13.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:13 smithi006 bash[27292]: audit 2023-08-27T23:01:12.775871+0000 mon.b (mon.1) 302 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:01:14.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:01:13.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:14.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:01:13.815Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:14.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:14 smithi130 bash[51302]: debug 2023-08-27T23:01:14.021+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:14.024923+0000) 2023-08-27T23:01:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:15 smithi130 bash[27579]: audit 2023-08-27T23:01:14.286048+0000 mon.c (mon.2) 745 : audit [DBG] from='client.? 172.21.15.6:0/1869107482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:15 smithi130 bash[27579]: cluster 2023-08-27T23:01:14.415613+0000 mgr.y (mgr.14901) 1881 : cluster [DBG] pgmap v1888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:15 smithi130 bash[51302]: debug 2023-08-27T23:01:15.021+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:15.025205+0000) 2023-08-27T23:01:15.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:15 smithi006 bash[33918]: audit 2023-08-27T23:01:14.286048+0000 mon.c (mon.2) 745 : audit [DBG] from='client.? 172.21.15.6:0/1869107482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:15.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:15 smithi006 bash[33918]: cluster 2023-08-27T23:01:14.415613+0000 mgr.y (mgr.14901) 1881 : cluster [DBG] pgmap v1888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:15.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:15 smithi006 bash[27292]: audit 2023-08-27T23:01:14.286048+0000 mon.c (mon.2) 745 : audit [DBG] from='client.? 172.21.15.6:0/1869107482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:15.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:15 smithi006 bash[27292]: cluster 2023-08-27T23:01:14.415613+0000 mgr.y (mgr.14901) 1881 : cluster [DBG] pgmap v1888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:16.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:16 smithi130 bash[51302]: debug 2023-08-27T23:01:16.021+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:16.025514+0000) 2023-08-27T23:01:16.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:16 smithi130 bash[27579]: cluster 2023-08-27T23:01:16.416010+0000 mgr.y (mgr.14901) 1882 : cluster [DBG] pgmap v1889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:16.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:16 smithi006 bash[27292]: cluster 2023-08-27T23:01:16.416010+0000 mgr.y (mgr.14901) 1882 : cluster [DBG] pgmap v1889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:16.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:16 smithi006 bash[33918]: cluster 2023-08-27T23:01:16.416010+0000 mgr.y (mgr.14901) 1882 : cluster [DBG] pgmap v1889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:17.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:17 smithi130 bash[51302]: debug 2023-08-27T23:01:17.021+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:17.025776+0000) 2023-08-27T23:01:17.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:17 smithi130 bash[27579]: audit 2023-08-27T23:01:16.706744+0000 mon.c (mon.2) 746 : audit [DBG] from='client.? 172.21.15.6:0/1407141908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:17.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:17 smithi006 bash[27292]: audit 2023-08-27T23:01:16.706744+0000 mon.c (mon.2) 746 : audit [DBG] from='client.? 172.21.15.6:0/1407141908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:17.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:17 smithi006 bash[33918]: audit 2023-08-27T23:01:16.706744+0000 mon.c (mon.2) 746 : audit [DBG] from='client.? 172.21.15.6:0/1407141908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:18.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:18 smithi130 bash[51302]: debug 2023-08-27T23:01:18.021+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:18.026000+0000) 2023-08-27T23:01:18.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:18 smithi130 bash[27579]: cluster 2023-08-27T23:01:18.416480+0000 mgr.y (mgr.14901) 1883 : cluster [DBG] pgmap v1890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:18.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:18 smithi006 bash[27292]: cluster 2023-08-27T23:01:18.416480+0000 mgr.y (mgr.14901) 1883 : cluster [DBG] pgmap v1890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:18.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:18 smithi006 bash[33918]: cluster 2023-08-27T23:01:18.416480+0000 mgr.y (mgr.14901) 1883 : cluster [DBG] pgmap v1890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:19.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:19 smithi130 bash[51302]: debug 2023-08-27T23:01:19.021+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:19.026251+0000) 2023-08-27T23:01:19.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:19 smithi006 bash[27292]: audit 2023-08-27T23:01:18.666669+0000 mon.a (mon.0) 1828 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:01:19.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:19 smithi006 bash[27292]: audit 2023-08-27T23:01:19.135750+0000 mon.a (mon.0) 1829 : audit [DBG] from='client.? 172.21.15.6:0/545602393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:19.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:19 smithi006 bash[27292]: audit 2023-08-27T23:01:19.173666+0000 mon.a (mon.0) 1830 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:01:19.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:19 smithi006 bash[27292]: audit 2023-08-27T23:01:19.473814+0000 mon.b (mon.1) 303 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:01:19.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:19 smithi006 bash[27292]: audit 2023-08-27T23:01:19.475903+0000 mon.b (mon.1) 304 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:01:19.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:19 smithi006 bash[27292]: audit 2023-08-27T23:01:19.481744+0000 mon.a (mon.0) 1831 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:01:19.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:19 smithi006 bash[33918]: audit 2023-08-27T23:01:18.666669+0000 mon.a (mon.0) 1828 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:01:19.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:19 smithi006 bash[33918]: audit 2023-08-27T23:01:19.135750+0000 mon.a (mon.0) 1829 : audit [DBG] from='client.? 172.21.15.6:0/545602393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:19.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:19 smithi006 bash[33918]: audit 2023-08-27T23:01:19.173666+0000 mon.a (mon.0) 1830 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:01:19.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:19 smithi006 bash[33918]: audit 2023-08-27T23:01:19.473814+0000 mon.b (mon.1) 303 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:01:19.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:19 smithi006 bash[33918]: audit 2023-08-27T23:01:19.475903+0000 mon.b (mon.1) 304 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:01:19.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:19 smithi006 bash[33918]: audit 2023-08-27T23:01:19.481744+0000 mon.a (mon.0) 1831 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:01:20.027 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:19 smithi130 bash[27579]: audit 2023-08-27T23:01:18.666669+0000 mon.a (mon.0) 1828 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:01:20.027 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:19 smithi130 bash[27579]: audit 2023-08-27T23:01:19.135750+0000 mon.a (mon.0) 1829 : audit [DBG] from='client.? 172.21.15.6:0/545602393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:20.028 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:19 smithi130 bash[27579]: audit 2023-08-27T23:01:19.173666+0000 mon.a (mon.0) 1830 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:01:20.028 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:19 smithi130 bash[27579]: audit 2023-08-27T23:01:19.473814+0000 mon.b (mon.1) 303 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:01:20.028 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:19 smithi130 bash[27579]: audit 2023-08-27T23:01:19.475903+0000 mon.b (mon.1) 304 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:01:20.028 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:19 smithi130 bash[27579]: audit 2023-08-27T23:01:19.481744+0000 mon.a (mon.0) 1831 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:01:20.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:20 smithi130 bash[51302]: debug 2023-08-27T23:01:20.021+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:20.026543+0000) 2023-08-27T23:01:20.677 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:01:20.355Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:20.677 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:01:20.357Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:20.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:01:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:01:20] "GET /metrics HTTP/1.1" 200 193091 "" "Prometheus/2.33.4" 2023-08-27T23:01:20.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:20 smithi006 bash[33918]: cluster 2023-08-27T23:01:20.417584+0000 mgr.y (mgr.14901) 1884 : cluster [DBG] pgmap v1891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:20.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:20 smithi006 bash[27292]: cluster 2023-08-27T23:01:20.417584+0000 mgr.y (mgr.14901) 1884 : cluster [DBG] pgmap v1891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:21.027 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:20 smithi130 bash[27579]: cluster 2023-08-27T23:01:20.417584+0000 mgr.y (mgr.14901) 1884 : cluster [DBG] pgmap v1891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:21.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:21 smithi130 bash[51302]: debug 2023-08-27T23:01:21.025+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:21.026883+0000) 2023-08-27T23:01:21.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:21 smithi006 bash[27292]: audit 2023-08-27T23:01:21.553375+0000 mon.a (mon.0) 1832 : audit [DBG] from='client.? 172.21.15.6:0/710639066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:21.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:21 smithi006 bash[33918]: audit 2023-08-27T23:01:21.553375+0000 mon.a (mon.0) 1832 : audit [DBG] from='client.? 172.21.15.6:0/710639066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:21.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:21 smithi006 bash[56298]: level=error ts=2023-08-27T23:01:21.917Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:21.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:21 smithi006 bash[56298]: level=warn ts=2023-08-27T23:01:21.919Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:22.028 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:21 smithi130 bash[27579]: audit 2023-08-27T23:01:21.553375+0000 mon.a (mon.0) 1832 : audit [DBG] from='client.? 172.21.15.6:0/710639066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:22.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:22 smithi130 bash[51302]: debug 2023-08-27T23:01:22.025+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:22.027195+0000) 2023-08-27T23:01:22.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:22 smithi006 bash[27292]: cluster 2023-08-27T23:01:22.418221+0000 mgr.y (mgr.14901) 1885 : cluster [DBG] pgmap v1892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:22.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:22 smithi006 bash[33918]: cluster 2023-08-27T23:01:22.418221+0000 mgr.y (mgr.14901) 1885 : cluster [DBG] pgmap v1892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:23.028 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:22 smithi130 bash[27579]: cluster 2023-08-27T23:01:22.418221+0000 mgr.y (mgr.14901) 1885 : cluster [DBG] pgmap v1892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:23.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:23 smithi130 bash[51302]: debug 2023-08-27T23:01:23.025+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:23.027511+0000) 2023-08-27T23:01:24.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:24 smithi006 bash[33918]: audit 2023-08-27T23:01:23.970960+0000 mon.c (mon.2) 747 : audit [DBG] from='client.? 172.21.15.6:0/680225560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:24.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:24 smithi006 bash[27292]: audit 2023-08-27T23:01:23.970960+0000 mon.c (mon.2) 747 : audit [DBG] from='client.? 172.21.15.6:0/680225560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:24.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:01:23.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:24.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:01:23.815Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:24.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:24 smithi130 bash[51302]: debug 2023-08-27T23:01:24.025+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:24.027870+0000) 2023-08-27T23:01:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:24 smithi130 bash[27579]: audit 2023-08-27T23:01:23.970960+0000 mon.c (mon.2) 747 : audit [DBG] from='client.? 172.21.15.6:0/680225560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:25.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:25 smithi130 bash[27579]: cluster 2023-08-27T23:01:24.419343+0000 mgr.y (mgr.14901) 1886 : cluster [DBG] pgmap v1893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:25 smithi130 bash[51302]: debug 2023-08-27T23:01:25.025+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:25.028155+0000) 2023-08-27T23:01:25.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:25 smithi006 bash[27292]: cluster 2023-08-27T23:01:24.419343+0000 mgr.y (mgr.14901) 1886 : cluster [DBG] pgmap v1893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:25.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:25 smithi006 bash[33918]: cluster 2023-08-27T23:01:24.419343+0000 mgr.y (mgr.14901) 1886 : cluster [DBG] pgmap v1893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:26.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:26 smithi130 bash[51302]: debug 2023-08-27T23:01:26.025+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:26.028511+0000) 2023-08-27T23:01:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:26 smithi006 bash[27292]: audit 2023-08-27T23:01:26.385758+0000 mon.a (mon.0) 1833 : audit [DBG] from='client.? 172.21.15.6:0/1908234600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:26 smithi006 bash[33918]: audit 2023-08-27T23:01:26.385758+0000 mon.a (mon.0) 1833 : audit [DBG] from='client.? 172.21.15.6:0/1908234600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:26 smithi130 bash[27579]: audit 2023-08-27T23:01:26.385758+0000 mon.a (mon.0) 1833 : audit [DBG] from='client.? 172.21.15.6:0/1908234600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:27.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:27 smithi130 bash[51302]: debug 2023-08-27T23:01:27.025+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:27.028770+0000) 2023-08-27T23:01:27.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:27 smithi006 bash[33918]: cluster 2023-08-27T23:01:26.420003+0000 mgr.y (mgr.14901) 1887 : cluster [DBG] pgmap v1894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:27.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:27 smithi006 bash[27292]: cluster 2023-08-27T23:01:26.420003+0000 mgr.y (mgr.14901) 1887 : cluster [DBG] pgmap v1894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:27 smithi130 bash[27579]: cluster 2023-08-27T23:01:26.420003+0000 mgr.y (mgr.14901) 1887 : cluster [DBG] pgmap v1894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:28.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:28 smithi130 bash[51302]: debug 2023-08-27T23:01:28.025+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:28.029074+0000) 2023-08-27T23:01:29.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:29 smithi130 bash[51302]: debug 2023-08-27T23:01:29.025+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:29.029345+0000) 2023-08-27T23:01:29.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:29 smithi006 bash[27292]: cluster 2023-08-27T23:01:28.420667+0000 mgr.y (mgr.14901) 1888 : cluster [DBG] pgmap v1895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:29.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:29 smithi006 bash[27292]: audit 2023-08-27T23:01:28.804567+0000 mon.c (mon.2) 748 : audit [DBG] from='client.? 172.21.15.6:0/270861474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:29.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:29 smithi006 bash[33918]: cluster 2023-08-27T23:01:28.420667+0000 mgr.y (mgr.14901) 1888 : cluster [DBG] pgmap v1895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:29.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:29 smithi006 bash[33918]: audit 2023-08-27T23:01:28.804567+0000 mon.c (mon.2) 748 : audit [DBG] from='client.? 172.21.15.6:0/270861474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:29.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:29 smithi130 bash[27579]: cluster 2023-08-27T23:01:28.420667+0000 mgr.y (mgr.14901) 1888 : cluster [DBG] pgmap v1895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:29.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:29 smithi130 bash[27579]: audit 2023-08-27T23:01:28.804567+0000 mon.c (mon.2) 748 : audit [DBG] from='client.? 172.21.15.6:0/270861474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:30 smithi130 bash[51302]: debug 2023-08-27T23:01:30.025+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:30.029606+0000) 2023-08-27T23:01:30.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:01:30.356Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:30.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:01:30.358Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:31.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:01:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:01:30] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T23:01:31.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:31 smithi130 bash[51302]: debug 2023-08-27T23:01:31.029+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:31.029915+0000) 2023-08-27T23:01:31.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:31 smithi006 bash[27292]: cluster 2023-08-27T23:01:30.421811+0000 mgr.y (mgr.14901) 1889 : cluster [DBG] pgmap v1896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:31.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:31 smithi006 bash[27292]: audit 2023-08-27T23:01:31.215263+0000 mon.c (mon.2) 749 : audit [DBG] from='client.? 172.21.15.6:0/3613097271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:31.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:31 smithi006 bash[33918]: cluster 2023-08-27T23:01:30.421811+0000 mgr.y (mgr.14901) 1889 : cluster [DBG] pgmap v1896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:31.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:31 smithi006 bash[33918]: audit 2023-08-27T23:01:31.215263+0000 mon.c (mon.2) 749 : audit [DBG] from='client.? 172.21.15.6:0/3613097271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:31.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:31 smithi130 bash[27579]: cluster 2023-08-27T23:01:30.421811+0000 mgr.y (mgr.14901) 1889 : cluster [DBG] pgmap v1896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:31.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:31 smithi130 bash[27579]: audit 2023-08-27T23:01:31.215263+0000 mon.c (mon.2) 749 : audit [DBG] from='client.? 172.21.15.6:0/3613097271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:32.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:31 smithi006 bash[56298]: level=error ts=2023-08-27T23:01:31.918Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:32.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:31 smithi006 bash[56298]: level=warn ts=2023-08-27T23:01:31.920Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:32.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:32 smithi130 bash[51302]: debug 2023-08-27T23:01:32.029+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:32.030231+0000) 2023-08-27T23:01:33.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:33 smithi130 bash[51302]: debug 2023-08-27T23:01:33.025+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:33.030493+0000) 2023-08-27T23:01:33.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:33 smithi006 bash[33918]: cluster 2023-08-27T23:01:32.422592+0000 mgr.y (mgr.14901) 1890 : cluster [DBG] pgmap v1897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:33.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:33 smithi006 bash[27292]: cluster 2023-08-27T23:01:32.422592+0000 mgr.y (mgr.14901) 1890 : cluster [DBG] pgmap v1897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:33.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:33 smithi130 bash[27579]: cluster 2023-08-27T23:01:32.422592+0000 mgr.y (mgr.14901) 1890 : cluster [DBG] pgmap v1897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:34.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:01:33.814Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:34.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:01:33.816Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:34.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:34 smithi130 bash[51302]: debug 2023-08-27T23:01:34.025+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:34.030771+0000) 2023-08-27T23:01:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:34 smithi006 bash[33918]: audit 2023-08-27T23:01:33.621204+0000 mon.a (mon.0) 1834 : audit [DBG] from='client.? 172.21.15.6:0/2536241342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:34.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:34 smithi006 bash[27292]: audit 2023-08-27T23:01:33.621204+0000 mon.a (mon.0) 1834 : audit [DBG] from='client.? 172.21.15.6:0/2536241342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:34.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:34 smithi130 bash[27579]: audit 2023-08-27T23:01:33.621204+0000 mon.a (mon.0) 1834 : audit [DBG] from='client.? 172.21.15.6:0/2536241342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:35.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:35 smithi130 bash[51302]: debug 2023-08-27T23:01:35.029+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:35.031072+0000) 2023-08-27T23:01:35.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:35 smithi130 bash[27579]: cluster 2023-08-27T23:01:34.423764+0000 mgr.y (mgr.14901) 1891 : cluster [DBG] pgmap v1898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:35.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:35 smithi006 bash[27292]: cluster 2023-08-27T23:01:34.423764+0000 mgr.y (mgr.14901) 1891 : cluster [DBG] pgmap v1898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:35.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:35 smithi006 bash[33918]: cluster 2023-08-27T23:01:34.423764+0000 mgr.y (mgr.14901) 1891 : cluster [DBG] pgmap v1898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:36.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:36 smithi130 bash[51302]: debug 2023-08-27T23:01:36.029+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:36.031337+0000) 2023-08-27T23:01:36.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:36 smithi130 bash[27579]: audit 2023-08-27T23:01:36.032237+0000 mon.a (mon.0) 1835 : audit [DBG] from='client.? 172.21.15.6:0/3665382473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:36 smithi006 bash[33918]: audit 2023-08-27T23:01:36.032237+0000 mon.a (mon.0) 1835 : audit [DBG] from='client.? 172.21.15.6:0/3665382473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:36.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:36 smithi006 bash[27292]: audit 2023-08-27T23:01:36.032237+0000 mon.a (mon.0) 1835 : audit [DBG] from='client.? 172.21.15.6:0/3665382473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:37.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:37 smithi130 bash[51302]: debug 2023-08-27T23:01:37.029+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:37.031560+0000) 2023-08-27T23:01:37.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:37 smithi130 bash[27579]: cluster 2023-08-27T23:01:36.424432+0000 mgr.y (mgr.14901) 1892 : cluster [DBG] pgmap v1899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:37.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:37 smithi006 bash[33918]: cluster 2023-08-27T23:01:36.424432+0000 mgr.y (mgr.14901) 1892 : cluster [DBG] pgmap v1899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:37.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:37 smithi006 bash[27292]: cluster 2023-08-27T23:01:36.424432+0000 mgr.y (mgr.14901) 1892 : cluster [DBG] pgmap v1899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:38.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:38 smithi130 bash[51302]: debug 2023-08-27T23:01:38.029+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:38.031851+0000) 2023-08-27T23:01:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:38 smithi130 bash[27579]: cluster 2023-08-27T23:01:38.425002+0000 mgr.y (mgr.14901) 1893 : cluster [DBG] pgmap v1900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:38 smithi130 bash[27579]: audit 2023-08-27T23:01:38.447585+0000 mon.a (mon.0) 1836 : audit [DBG] from='client.? 172.21.15.6:0/3216043197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:38.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:38 smithi006 bash[33918]: cluster 2023-08-27T23:01:38.425002+0000 mgr.y (mgr.14901) 1893 : cluster [DBG] pgmap v1900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:38.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:38 smithi006 bash[33918]: audit 2023-08-27T23:01:38.447585+0000 mon.a (mon.0) 1836 : audit [DBG] from='client.? 172.21.15.6:0/3216043197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:38.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:38 smithi006 bash[27292]: cluster 2023-08-27T23:01:38.425002+0000 mgr.y (mgr.14901) 1893 : cluster [DBG] pgmap v1900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:38.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:38 smithi006 bash[27292]: audit 2023-08-27T23:01:38.447585+0000 mon.a (mon.0) 1836 : audit [DBG] from='client.? 172.21.15.6:0/3216043197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:39.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:39 smithi130 bash[51302]: debug 2023-08-27T23:01:39.029+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:39.032171+0000) 2023-08-27T23:01:40.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:40 smithi130 bash[51302]: debug 2023-08-27T23:01:40.029+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:40.032474+0000) 2023-08-27T23:01:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:40 smithi006 bash[27292]: cluster 2023-08-27T23:01:40.426059+0000 mgr.y (mgr.14901) 1894 : cluster [DBG] pgmap v1901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:40 smithi006 bash[33918]: cluster 2023-08-27T23:01:40.426059+0000 mgr.y (mgr.14901) 1894 : cluster [DBG] pgmap v1901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:40.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:01:40.356Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:40.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:01:40.358Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:40 smithi130 bash[27579]: cluster 2023-08-27T23:01:40.426059+0000 mgr.y (mgr.14901) 1894 : cluster [DBG] pgmap v1901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:41.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:01:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:01:40] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T23:01:41.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:41 smithi130 bash[51302]: debug 2023-08-27T23:01:41.029+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:41.032745+0000) 2023-08-27T23:01:41.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:41 smithi130 bash[27579]: audit 2023-08-27T23:01:40.858305+0000 mon.c (mon.2) 750 : audit [DBG] from='client.? 172.21.15.6:0/778936237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:41.923 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:41 smithi006 bash[33918]: audit 2023-08-27T23:01:40.858305+0000 mon.c (mon.2) 750 : audit [DBG] from='client.? 172.21.15.6:0/778936237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:41.923 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:41 smithi006 bash[27292]: audit 2023-08-27T23:01:40.858305+0000 mon.c (mon.2) 750 : audit [DBG] from='client.? 172.21.15.6:0/778936237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:42.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:41 smithi006 bash[56298]: level=error ts=2023-08-27T23:01:41.919Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:42.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:41 smithi006 bash[56298]: level=warn ts=2023-08-27T23:01:41.921Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:42.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:42 smithi130 bash[51302]: debug 2023-08-27T23:01:42.029+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:42.033013+0000) 2023-08-27T23:01:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:42 smithi130 bash[27579]: cluster 2023-08-27T23:01:42.426756+0000 mgr.y (mgr.14901) 1895 : cluster [DBG] pgmap v1902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:42.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:42 smithi006 bash[33918]: cluster 2023-08-27T23:01:42.426756+0000 mgr.y (mgr.14901) 1895 : cluster [DBG] pgmap v1902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:42.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:42 smithi006 bash[27292]: cluster 2023-08-27T23:01:42.426756+0000 mgr.y (mgr.14901) 1895 : cluster [DBG] pgmap v1902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:43.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:43 smithi130 bash[51302]: debug 2023-08-27T23:01:43.029+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:43.033314+0000) 2023-08-27T23:01:43.819 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:43 smithi006 bash[27292]: audit 2023-08-27T23:01:43.281900+0000 mon.c (mon.2) 751 : audit [DBG] from='client.? 172.21.15.6:0/2386461709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:43.819 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:43 smithi006 bash[33918]: audit 2023-08-27T23:01:43.281900+0000 mon.c (mon.2) 751 : audit [DBG] from='client.? 172.21.15.6:0/2386461709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:43.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:43 smithi130 bash[27579]: audit 2023-08-27T23:01:43.281900+0000 mon.c (mon.2) 751 : audit [DBG] from='client.? 172.21.15.6:0/2386461709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:44.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:01:43.814Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:44.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:01:43.817Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:44.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:44 smithi130 bash[51302]: debug 2023-08-27T23:01:44.029+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:44.033618+0000) 2023-08-27T23:01:45.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:45 smithi130 bash[27579]: cluster 2023-08-27T23:01:44.427817+0000 mgr.y (mgr.14901) 1896 : cluster [DBG] pgmap v1903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:45 smithi130 bash[51302]: debug 2023-08-27T23:01:45.029+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:45.033877+0000) 2023-08-27T23:01:45.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:45 smithi006 bash[27292]: cluster 2023-08-27T23:01:44.427817+0000 mgr.y (mgr.14901) 1896 : cluster [DBG] pgmap v1903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:45.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:45 smithi006 bash[33918]: cluster 2023-08-27T23:01:44.427817+0000 mgr.y (mgr.14901) 1896 : cluster [DBG] pgmap v1903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:46 smithi130 bash[27579]: audit 2023-08-27T23:01:45.702392+0000 mon.c (mon.2) 752 : audit [DBG] from='client.? 172.21.15.6:0/4021130276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:46 smithi130 bash[51302]: debug 2023-08-27T23:01:46.029+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:46.034167+0000) 2023-08-27T23:01:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:46 smithi006 bash[27292]: audit 2023-08-27T23:01:45.702392+0000 mon.c (mon.2) 752 : audit [DBG] from='client.? 172.21.15.6:0/4021130276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:46.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:46 smithi006 bash[33918]: audit 2023-08-27T23:01:45.702392+0000 mon.c (mon.2) 752 : audit [DBG] from='client.? 172.21.15.6:0/4021130276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:47.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:47 smithi130 bash[27579]: cluster 2023-08-27T23:01:46.428471+0000 mgr.y (mgr.14901) 1897 : cluster [DBG] pgmap v1904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:47 smithi130 bash[51302]: debug 2023-08-27T23:01:47.029+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:47.034396+0000) 2023-08-27T23:01:47.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:47 smithi006 bash[33918]: cluster 2023-08-27T23:01:46.428471+0000 mgr.y (mgr.14901) 1897 : cluster [DBG] pgmap v1904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:47.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:47 smithi006 bash[27292]: cluster 2023-08-27T23:01:46.428471+0000 mgr.y (mgr.14901) 1897 : cluster [DBG] pgmap v1904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:48.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:48 smithi130 bash[51302]: debug 2023-08-27T23:01:48.033+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:48.034737+0000) 2023-08-27T23:01:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:48 smithi130 bash[27579]: audit 2023-08-27T23:01:48.127874+0000 mon.c (mon.2) 753 : audit [DBG] from='client.? 172.21.15.6:0/1375519263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:48 smithi006 bash[33918]: audit 2023-08-27T23:01:48.127874+0000 mon.c (mon.2) 753 : audit [DBG] from='client.? 172.21.15.6:0/1375519263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:48.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:48 smithi006 bash[27292]: audit 2023-08-27T23:01:48.127874+0000 mon.c (mon.2) 753 : audit [DBG] from='client.? 172.21.15.6:0/1375519263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:49.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:49 smithi130 bash[27579]: cluster 2023-08-27T23:01:48.429118+0000 mgr.y (mgr.14901) 1898 : cluster [DBG] pgmap v1905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:49 smithi130 bash[51302]: debug 2023-08-27T23:01:49.033+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:49.035026+0000) 2023-08-27T23:01:49.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:49 smithi006 bash[27292]: cluster 2023-08-27T23:01:48.429118+0000 mgr.y (mgr.14901) 1898 : cluster [DBG] pgmap v1905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:49.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:49 smithi006 bash[33918]: cluster 2023-08-27T23:01:48.429118+0000 mgr.y (mgr.14901) 1898 : cluster [DBG] pgmap v1905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:50 smithi130 bash[51302]: debug 2023-08-27T23:01:50.033+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:50.035249+0000) 2023-08-27T23:01:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:50 smithi006 bash[27292]: cluster 2023-08-27T23:01:50.430339+0000 mgr.y (mgr.14901) 1899 : cluster [DBG] pgmap v1906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:50 smithi006 bash[33918]: cluster 2023-08-27T23:01:50.430339+0000 mgr.y (mgr.14901) 1899 : cluster [DBG] pgmap v1906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:50.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:01:50.357Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:50.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:01:50.359Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:50 smithi130 bash[27579]: cluster 2023-08-27T23:01:50.430339+0000 mgr.y (mgr.14901) 1899 : cluster [DBG] pgmap v1906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:51.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:01:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:01:50] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:01:51.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:51 smithi130 bash[51302]: debug 2023-08-27T23:01:51.032+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:51.035522+0000) 2023-08-27T23:01:51.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:51 smithi130 bash[27579]: audit 2023-08-27T23:01:50.560811+0000 mon.a (mon.0) 1837 : audit [DBG] from='client.? 172.21.15.6:0/717624463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:51.923 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:51 smithi006 bash[27292]: audit 2023-08-27T23:01:50.560811+0000 mon.a (mon.0) 1837 : audit [DBG] from='client.? 172.21.15.6:0/717624463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:51.923 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:51 smithi006 bash[33918]: audit 2023-08-27T23:01:50.560811+0000 mon.a (mon.0) 1837 : audit [DBG] from='client.? 172.21.15.6:0/717624463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:52.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:51 smithi006 bash[56298]: level=error ts=2023-08-27T23:01:51.919Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:52.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:51 smithi006 bash[56298]: level=warn ts=2023-08-27T23:01:51.921Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:52.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:52 smithi130 bash[51302]: debug 2023-08-27T23:01:52.032+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:52.035812+0000) 2023-08-27T23:01:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:52 smithi130 bash[27579]: cluster 2023-08-27T23:01:52.431058+0000 mgr.y (mgr.14901) 1900 : cluster [DBG] pgmap v1907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:52.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:52 smithi006 bash[33918]: cluster 2023-08-27T23:01:52.431058+0000 mgr.y (mgr.14901) 1900 : cluster [DBG] pgmap v1907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:52.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:52 smithi006 bash[27292]: cluster 2023-08-27T23:01:52.431058+0000 mgr.y (mgr.14901) 1900 : cluster [DBG] pgmap v1907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:53.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:53 smithi130 bash[51302]: debug 2023-08-27T23:01:53.032+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:53.036104+0000) 2023-08-27T23:01:53.820 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:53 smithi006 bash[27292]: audit 2023-08-27T23:01:52.980828+0000 mon.c (mon.2) 754 : audit [DBG] from='client.? 172.21.15.6:0/2492728562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:53.820 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:53 smithi006 bash[33918]: audit 2023-08-27T23:01:52.980828+0000 mon.c (mon.2) 754 : audit [DBG] from='client.? 172.21.15.6:0/2492728562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:53.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:53 smithi130 bash[27579]: audit 2023-08-27T23:01:52.980828+0000 mon.c (mon.2) 754 : audit [DBG] from='client.? 172.21.15.6:0/2492728562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:54.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:01:53.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:54.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:01:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:01:53.818Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:01:54.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:54 smithi130 bash[51302]: debug 2023-08-27T23:01:54.032+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:54.036389+0000) 2023-08-27T23:01:55.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:55 smithi130 bash[27579]: cluster 2023-08-27T23:01:54.432182+0000 mgr.y (mgr.14901) 1901 : cluster [DBG] pgmap v1908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:55 smithi130 bash[51302]: debug 2023-08-27T23:01:55.032+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:55.036681+0000) 2023-08-27T23:01:55.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:55 smithi006 bash[27292]: cluster 2023-08-27T23:01:54.432182+0000 mgr.y (mgr.14901) 1901 : cluster [DBG] pgmap v1908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:55.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:55 smithi006 bash[33918]: cluster 2023-08-27T23:01:54.432182+0000 mgr.y (mgr.14901) 1901 : cluster [DBG] pgmap v1908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:56.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:56 smithi130 bash[51302]: debug 2023-08-27T23:01:56.032+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:56.036987+0000) 2023-08-27T23:01:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:56 smithi130 bash[27579]: audit 2023-08-27T23:01:55.398732+0000 mon.a (mon.0) 1838 : audit [DBG] from='client.? 172.21.15.6:0/1221992791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:56 smithi006 bash[27292]: audit 2023-08-27T23:01:55.398732+0000 mon.a (mon.0) 1838 : audit [DBG] from='client.? 172.21.15.6:0/1221992791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:56.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:56 smithi006 bash[33918]: audit 2023-08-27T23:01:55.398732+0000 mon.a (mon.0) 1838 : audit [DBG] from='client.? 172.21.15.6:0/1221992791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:57.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:57 smithi130 bash[27579]: cluster 2023-08-27T23:01:56.432778+0000 mgr.y (mgr.14901) 1902 : cluster [DBG] pgmap v1909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:57.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:57 smithi130 bash[51302]: debug 2023-08-27T23:01:57.032+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:57.037287+0000) 2023-08-27T23:01:57.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:57 smithi006 bash[33918]: cluster 2023-08-27T23:01:56.432778+0000 mgr.y (mgr.14901) 1902 : cluster [DBG] pgmap v1909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:57.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:57 smithi006 bash[27292]: cluster 2023-08-27T23:01:56.432778+0000 mgr.y (mgr.14901) 1902 : cluster [DBG] pgmap v1909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:58.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:58 smithi130 bash[27579]: audit 2023-08-27T23:01:57.815025+0000 mon.c (mon.2) 755 : audit [DBG] from='client.? 172.21.15.6:0/873864189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:58.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:58 smithi130 bash[51302]: debug 2023-08-27T23:01:58.032+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:58.037599+0000) 2023-08-27T23:01:58.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:58 smithi006 bash[27292]: audit 2023-08-27T23:01:57.815025+0000 mon.c (mon.2) 755 : audit [DBG] from='client.? 172.21.15.6:0/873864189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:58.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:58 smithi006 bash[33918]: audit 2023-08-27T23:01:57.815025+0000 mon.c (mon.2) 755 : audit [DBG] from='client.? 172.21.15.6:0/873864189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:59.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:01:59 smithi130 bash[27579]: cluster 2023-08-27T23:01:58.433524+0000 mgr.y (mgr.14901) 1903 : cluster [DBG] pgmap v1910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:01:59 smithi130 bash[51302]: debug 2023-08-27T23:01:59.032+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:59.037871+0000) 2023-08-27T23:01:59.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:01:59 smithi006 bash[27292]: cluster 2023-08-27T23:01:58.433524+0000 mgr.y (mgr.14901) 1903 : cluster [DBG] pgmap v1910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:01:59.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:01:59 smithi006 bash[33918]: cluster 2023-08-27T23:01:58.433524+0000 mgr.y (mgr.14901) 1903 : cluster [DBG] pgmap v1910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:00.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:00 smithi130 bash[27579]: audit 2023-08-27T23:02:00.235022+0000 mon.c (mon.2) 756 : audit [DBG] from='client.? 172.21.15.6:0/4006021962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:00.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:00 smithi130 bash[51302]: debug 2023-08-27T23:02:00.032+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:00.038141+0000) 2023-08-27T23:02:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:00 smithi006 bash[33918]: audit 2023-08-27T23:02:00.235022+0000 mon.c (mon.2) 756 : audit [DBG] from='client.? 172.21.15.6:0/4006021962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:00.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:00 smithi006 bash[27292]: audit 2023-08-27T23:02:00.235022+0000 mon.c (mon.2) 756 : audit [DBG] from='client.? 172.21.15.6:0/4006021962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:00.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:02:00.357Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:00.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:02:00.359Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:01.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:02:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:02:00] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:02:01.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:01 smithi130 bash[27579]: cluster 2023-08-27T23:02:00.434672+0000 mgr.y (mgr.14901) 1904 : cluster [DBG] pgmap v1911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:01.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:01 smithi130 bash[51302]: debug 2023-08-27T23:02:01.032+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:01.038406+0000) 2023-08-27T23:02:01.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:01 smithi006 bash[33918]: cluster 2023-08-27T23:02:00.434672+0000 mgr.y (mgr.14901) 1904 : cluster [DBG] pgmap v1911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:01.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:01 smithi006 bash[27292]: cluster 2023-08-27T23:02:00.434672+0000 mgr.y (mgr.14901) 1904 : cluster [DBG] pgmap v1911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:02.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:01 smithi006 bash[56298]: level=error ts=2023-08-27T23:02:01.920Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:02.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:01 smithi006 bash[56298]: level=warn ts=2023-08-27T23:02:01.922Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:02.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:02 smithi130 bash[51302]: debug 2023-08-27T23:02:02.032+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:02.038659+0000) 2023-08-27T23:02:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:02 smithi130 bash[27579]: cluster 2023-08-27T23:02:02.435307+0000 mgr.y (mgr.14901) 1905 : cluster [DBG] pgmap v1912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:02.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:02 smithi006 bash[27292]: cluster 2023-08-27T23:02:02.435307+0000 mgr.y (mgr.14901) 1905 : cluster [DBG] pgmap v1912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:02.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:02 smithi006 bash[33918]: cluster 2023-08-27T23:02:02.435307+0000 mgr.y (mgr.14901) 1905 : cluster [DBG] pgmap v1912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:03.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:03 smithi130 bash[51302]: debug 2023-08-27T23:02:03.036+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:03.038965+0000) 2023-08-27T23:02:03.820 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:03 smithi006 bash[33918]: audit 2023-08-27T23:02:02.655463+0000 mon.a (mon.0) 1839 : audit [DBG] from='client.? 172.21.15.6:0/3978582265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:03.895 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:03 smithi006 bash[27292]: audit 2023-08-27T23:02:02.655463+0000 mon.a (mon.0) 1839 : audit [DBG] from='client.? 172.21.15.6:0/3978582265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:03.895 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:03 smithi130 bash[27579]: audit 2023-08-27T23:02:02.655463+0000 mon.a (mon.0) 1839 : audit [DBG] from='client.? 172.21.15.6:0/3978582265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:04.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:02:03.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:04.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:02:03.819Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:04.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:04 smithi130 bash[51302]: debug 2023-08-27T23:02:04.036+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:04.039260+0000) 2023-08-27T23:02:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:05 smithi130 bash[27579]: cluster 2023-08-27T23:02:04.436426+0000 mgr.y (mgr.14901) 1906 : cluster [DBG] pgmap v1913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:05 smithi130 bash[27579]: audit 2023-08-27T23:02:05.065448+0000 mon.a (mon.0) 1840 : audit [DBG] from='client.? 172.21.15.6:0/2528627685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:05 smithi130 bash[51302]: debug 2023-08-27T23:02:05.036+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:05.039552+0000) 2023-08-27T23:02:05.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:05 smithi006 bash[33918]: cluster 2023-08-27T23:02:04.436426+0000 mgr.y (mgr.14901) 1906 : cluster [DBG] pgmap v1913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:05.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:05 smithi006 bash[33918]: audit 2023-08-27T23:02:05.065448+0000 mon.a (mon.0) 1840 : audit [DBG] from='client.? 172.21.15.6:0/2528627685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:05.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:05 smithi006 bash[27292]: cluster 2023-08-27T23:02:04.436426+0000 mgr.y (mgr.14901) 1906 : cluster [DBG] pgmap v1913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:05.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:05 smithi006 bash[27292]: audit 2023-08-27T23:02:05.065448+0000 mon.a (mon.0) 1840 : audit [DBG] from='client.? 172.21.15.6:0/2528627685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:06.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:06 smithi130 bash[51302]: debug 2023-08-27T23:02:06.036+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:06.039878+0000) 2023-08-27T23:02:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:06 smithi130 bash[27579]: cluster 2023-08-27T23:02:06.436986+0000 mgr.y (mgr.14901) 1907 : cluster [DBG] pgmap v1914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:07.041 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:06 smithi006 bash[27292]: cluster 2023-08-27T23:02:06.436986+0000 mgr.y (mgr.14901) 1907 : cluster [DBG] pgmap v1914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:07.041 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:06 smithi006 bash[33918]: cluster 2023-08-27T23:02:06.436986+0000 mgr.y (mgr.14901) 1907 : cluster [DBG] pgmap v1914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:07.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:07 smithi130 bash[51302]: debug 2023-08-27T23:02:07.036+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:07.040176+0000) 2023-08-27T23:02:07.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:07 smithi130 bash[27579]: audit 2023-08-27T23:02:07.486627+0000 mon.c (mon.2) 757 : audit [DBG] from='client.? 172.21.15.6:0/1873097785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:07.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:07 smithi006 bash[27292]: audit 2023-08-27T23:02:07.486627+0000 mon.c (mon.2) 757 : audit [DBG] from='client.? 172.21.15.6:0/1873097785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:07.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:07 smithi006 bash[33918]: audit 2023-08-27T23:02:07.486627+0000 mon.c (mon.2) 757 : audit [DBG] from='client.? 172.21.15.6:0/1873097785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:08.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:08 smithi130 bash[51302]: debug 2023-08-27T23:02:08.036+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:08.040465+0000) 2023-08-27T23:02:09.201 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:08 smithi006 bash[27292]: cluster 2023-08-27T23:02:08.437626+0000 mgr.y (mgr.14901) 1908 : cluster [DBG] pgmap v1915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:09.201 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:08 smithi130 bash[27579]: cluster 2023-08-27T23:02:08.437626+0000 mgr.y (mgr.14901) 1908 : cluster [DBG] pgmap v1915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:09.202 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:08 smithi006 bash[33918]: cluster 2023-08-27T23:02:08.437626+0000 mgr.y (mgr.14901) 1908 : cluster [DBG] pgmap v1915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:09.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:09 smithi130 bash[51302]: debug 2023-08-27T23:02:09.036+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:09.040750+0000) 2023-08-27T23:02:10.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:10 smithi130 bash[27579]: audit 2023-08-27T23:02:09.900744+0000 mon.c (mon.2) 758 : audit [DBG] from='client.? 172.21.15.6:0/3432977139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:10.425 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:10 smithi130 bash[51302]: debug 2023-08-27T23:02:10.036+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:10.041072+0000) 2023-08-27T23:02:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:10 smithi006 bash[33918]: audit 2023-08-27T23:02:09.900744+0000 mon.c (mon.2) 758 : audit [DBG] from='client.? 172.21.15.6:0/3432977139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:11.242 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:10 smithi006 bash[27292]: audit 2023-08-27T23:02:09.900744+0000 mon.c (mon.2) 758 : audit [DBG] from='client.? 172.21.15.6:0/3432977139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:11.242 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:02:10.358Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:11.242 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:02:10.360Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:11.243 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:02:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:02:10] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:02:11.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:11 smithi130 bash[27579]: cluster 2023-08-27T23:02:10.438776+0000 mgr.y (mgr.14901) 1909 : cluster [DBG] pgmap v1916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:11.408 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:11 smithi130 bash[51302]: debug 2023-08-27T23:02:11.036+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:11.041349+0000) 2023-08-27T23:02:11.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:11 smithi006 bash[33918]: cluster 2023-08-27T23:02:10.438776+0000 mgr.y (mgr.14901) 1909 : cluster [DBG] pgmap v1916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:11.753 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:11 smithi006 bash[27292]: cluster 2023-08-27T23:02:10.438776+0000 mgr.y (mgr.14901) 1909 : cluster [DBG] pgmap v1916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:12.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:11 smithi006 bash[56298]: level=error ts=2023-08-27T23:02:11.920Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:12.287 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:11 smithi006 bash[56298]: level=warn ts=2023-08-27T23:02:11.922Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:12.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:12 smithi130 bash[51302]: debug 2023-08-27T23:02:12.036+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:12.041652+0000) 2023-08-27T23:02:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:12 smithi006 bash[33918]: audit 2023-08-27T23:02:12.318690+0000 mon.a (mon.0) 1841 : audit [DBG] from='client.? 172.21.15.6:0/1833787584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:12.784 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:12 smithi006 bash[27292]: audit 2023-08-27T23:02:12.318690+0000 mon.a (mon.0) 1841 : audit [DBG] from='client.? 172.21.15.6:0/1833787584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:12 smithi130 bash[27579]: audit 2023-08-27T23:02:12.318690+0000 mon.a (mon.0) 1841 : audit [DBG] from='client.? 172.21.15.6:0/1833787584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:13.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:13 smithi130 bash[51302]: debug 2023-08-27T23:02:13.036+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:13.041965+0000) 2023-08-27T23:02:13.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:13 smithi006 bash[27292]: cluster 2023-08-27T23:02:12.439410+0000 mgr.y (mgr.14901) 1910 : cluster [DBG] pgmap v1917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:13.761 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:13 smithi006 bash[33918]: cluster 2023-08-27T23:02:12.439410+0000 mgr.y (mgr.14901) 1910 : cluster [DBG] pgmap v1917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:13.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:13 smithi130 bash[27579]: cluster 2023-08-27T23:02:12.439410+0000 mgr.y (mgr.14901) 1910 : cluster [DBG] pgmap v1917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:14.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:02:13.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:14.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:02:13.819Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:14.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:14 smithi130 bash[51302]: debug 2023-08-27T23:02:14.036+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:14.042306+0000) 2023-08-27T23:02:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:15 smithi130 bash[27579]: cluster 2023-08-27T23:02:14.440674+0000 mgr.y (mgr.14901) 1911 : cluster [DBG] pgmap v1918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:15 smithi130 bash[27579]: audit 2023-08-27T23:02:14.735576+0000 mon.a (mon.0) 1842 : audit [DBG] from='client.? 172.21.15.6:0/3701091046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:15 smithi130 bash[51302]: debug 2023-08-27T23:02:15.036+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:15.042583+0000) 2023-08-27T23:02:15.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:15 smithi006 bash[27292]: cluster 2023-08-27T23:02:14.440674+0000 mgr.y (mgr.14901) 1911 : cluster [DBG] pgmap v1918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:15.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:15 smithi006 bash[27292]: audit 2023-08-27T23:02:14.735576+0000 mon.a (mon.0) 1842 : audit [DBG] from='client.? 172.21.15.6:0/3701091046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:15.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:15 smithi006 bash[33918]: cluster 2023-08-27T23:02:14.440674+0000 mgr.y (mgr.14901) 1911 : cluster [DBG] pgmap v1918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:15.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:15 smithi006 bash[33918]: audit 2023-08-27T23:02:14.735576+0000 mon.a (mon.0) 1842 : audit [DBG] from='client.? 172.21.15.6:0/3701091046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:16.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:16 smithi130 bash[51302]: debug 2023-08-27T23:02:16.040+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:16.042909+0000) 2023-08-27T23:02:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:16 smithi130 bash[27579]: cluster 2023-08-27T23:02:16.441357+0000 mgr.y (mgr.14901) 1912 : cluster [DBG] pgmap v1919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:16.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:16 smithi006 bash[27292]: cluster 2023-08-27T23:02:16.441357+0000 mgr.y (mgr.14901) 1912 : cluster [DBG] pgmap v1919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:16.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:16 smithi006 bash[33918]: cluster 2023-08-27T23:02:16.441357+0000 mgr.y (mgr.14901) 1912 : cluster [DBG] pgmap v1919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:17.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:17 smithi130 bash[51302]: debug 2023-08-27T23:02:17.040+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:17.043206+0000) 2023-08-27T23:02:17.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:17 smithi130 bash[27579]: audit 2023-08-27T23:02:17.146253+0000 mon.c (mon.2) 759 : audit [DBG] from='client.? 172.21.15.6:0/119412223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:17.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:17 smithi006 bash[27292]: audit 2023-08-27T23:02:17.146253+0000 mon.c (mon.2) 759 : audit [DBG] from='client.? 172.21.15.6:0/119412223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:17.991 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:17 smithi006 bash[33918]: audit 2023-08-27T23:02:17.146253+0000 mon.c (mon.2) 759 : audit [DBG] from='client.? 172.21.15.6:0/119412223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:18.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:18 smithi130 bash[51302]: debug 2023-08-27T23:02:18.040+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:18.043447+0000) 2023-08-27T23:02:18.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:18 smithi130 bash[27579]: cluster 2023-08-27T23:02:18.442022+0000 mgr.y (mgr.14901) 1913 : cluster [DBG] pgmap v1920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:18.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:18 smithi006 bash[33918]: cluster 2023-08-27T23:02:18.442022+0000 mgr.y (mgr.14901) 1913 : cluster [DBG] pgmap v1920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:18.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:18 smithi006 bash[27292]: cluster 2023-08-27T23:02:18.442022+0000 mgr.y (mgr.14901) 1913 : cluster [DBG] pgmap v1920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:19.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:19 smithi130 bash[51302]: debug 2023-08-27T23:02:19.040+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:19.043737+0000) 2023-08-27T23:02:20.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:20 smithi130 bash[51302]: debug 2023-08-27T23:02:20.040+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:20.043986+0000) 2023-08-27T23:02:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:20 smithi130 bash[27579]: audit 2023-08-27T23:02:19.493447+0000 mon.b (mon.1) 305 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:02:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:20 smithi130 bash[27579]: audit 2023-08-27T23:02:19.560428+0000 mon.a (mon.0) 1843 : audit [DBG] from='client.? 172.21.15.6:0/3515762655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:20.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:20 smithi006 bash[33918]: audit 2023-08-27T23:02:19.493447+0000 mon.b (mon.1) 305 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:02:20.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:20 smithi006 bash[33918]: audit 2023-08-27T23:02:19.560428+0000 mon.a (mon.0) 1843 : audit [DBG] from='client.? 172.21.15.6:0/3515762655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:20.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:02:20.359Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:20.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:02:20.361Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:20.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:20 smithi006 bash[27292]: audit 2023-08-27T23:02:19.493447+0000 mon.b (mon.1) 305 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:02:20.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:20 smithi006 bash[27292]: audit 2023-08-27T23:02:19.560428+0000 mon.a (mon.0) 1843 : audit [DBG] from='client.? 172.21.15.6:0/3515762655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:21.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:02:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:02:20] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:02:21.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:21 smithi130 bash[27579]: cluster 2023-08-27T23:02:20.443160+0000 mgr.y (mgr.14901) 1914 : cluster [DBG] pgmap v1921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:21.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:21 smithi130 bash[51302]: debug 2023-08-27T23:02:21.040+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:21.044264+0000) 2023-08-27T23:02:21.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:21 smithi006 bash[27292]: cluster 2023-08-27T23:02:20.443160+0000 mgr.y (mgr.14901) 1914 : cluster [DBG] pgmap v1921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:21.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:21 smithi006 bash[33918]: cluster 2023-08-27T23:02:20.443160+0000 mgr.y (mgr.14901) 1914 : cluster [DBG] pgmap v1921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:22.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:21 smithi006 bash[56298]: level=error ts=2023-08-27T23:02:21.922Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:22.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:21 smithi006 bash[56298]: level=warn ts=2023-08-27T23:02:21.923Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:22.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:22 smithi130 bash[27579]: audit 2023-08-27T23:02:21.979706+0000 mon.c (mon.2) 760 : audit [DBG] from='client.? 172.21.15.6:0/139137329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:22.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:22 smithi130 bash[51302]: debug 2023-08-27T23:02:22.040+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:22.044546+0000) 2023-08-27T23:02:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:22 smithi006 bash[27292]: audit 2023-08-27T23:02:21.979706+0000 mon.c (mon.2) 760 : audit [DBG] from='client.? 172.21.15.6:0/139137329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:22 smithi006 bash[33918]: audit 2023-08-27T23:02:21.979706+0000 mon.c (mon.2) 760 : audit [DBG] from='client.? 172.21.15.6:0/139137329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:23.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:23 smithi130 bash[27579]: cluster 2023-08-27T23:02:22.443826+0000 mgr.y (mgr.14901) 1915 : cluster [DBG] pgmap v1922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:23.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:23 smithi130 bash[51302]: debug 2023-08-27T23:02:23.044+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:23.044853+0000) 2023-08-27T23:02:23.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:23 smithi006 bash[27292]: cluster 2023-08-27T23:02:22.443826+0000 mgr.y (mgr.14901) 1915 : cluster [DBG] pgmap v1922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:23.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:23 smithi006 bash[33918]: cluster 2023-08-27T23:02:22.443826+0000 mgr.y (mgr.14901) 1915 : cluster [DBG] pgmap v1922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:24.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:02:23.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:24.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:02:23.820Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:24 smithi130 bash[51302]: debug 2023-08-27T23:02:24.040+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:24.045110+0000) 2023-08-27T23:02:25.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:25 smithi130 bash[27579]: audit 2023-08-27T23:02:24.396023+0000 mon.a (mon.0) 1844 : audit [DBG] from='client.? 172.21.15.6:0/3932874866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:25 smithi130 bash[27579]: cluster 2023-08-27T23:02:24.444875+0000 mgr.y (mgr.14901) 1916 : cluster [DBG] pgmap v1923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:25 smithi130 bash[51302]: debug 2023-08-27T23:02:25.040+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:25.045367+0000) 2023-08-27T23:02:25.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:25 smithi006 bash[27292]: audit 2023-08-27T23:02:24.396023+0000 mon.a (mon.0) 1844 : audit [DBG] from='client.? 172.21.15.6:0/3932874866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:25.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:25 smithi006 bash[27292]: cluster 2023-08-27T23:02:24.444875+0000 mgr.y (mgr.14901) 1916 : cluster [DBG] pgmap v1923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:25.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:25 smithi006 bash[33918]: audit 2023-08-27T23:02:24.396023+0000 mon.a (mon.0) 1844 : audit [DBG] from='client.? 172.21.15.6:0/3932874866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:25.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:25 smithi006 bash[33918]: cluster 2023-08-27T23:02:24.444875+0000 mgr.y (mgr.14901) 1916 : cluster [DBG] pgmap v1923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:26.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:26 smithi130 bash[51302]: debug 2023-08-27T23:02:26.040+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:26.045639+0000) 2023-08-27T23:02:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:26 smithi006 bash[27292]: audit 2023-08-27T23:02:25.377591+0000 mon.a (mon.0) 1845 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:02:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:26 smithi006 bash[27292]: audit 2023-08-27T23:02:25.860634+0000 mon.a (mon.0) 1846 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:02:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:26 smithi006 bash[27292]: audit 2023-08-27T23:02:26.156733+0000 mon.b (mon.1) 306 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:02:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:26 smithi006 bash[27292]: audit 2023-08-27T23:02:26.159027+0000 mon.b (mon.1) 307 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:02:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:26 smithi006 bash[27292]: audit 2023-08-27T23:02:26.167639+0000 mon.a (mon.0) 1847 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:02:26.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:26 smithi006 bash[33918]: audit 2023-08-27T23:02:25.377591+0000 mon.a (mon.0) 1845 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:02:26.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:26 smithi006 bash[33918]: audit 2023-08-27T23:02:25.860634+0000 mon.a (mon.0) 1846 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:02:26.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:26 smithi006 bash[33918]: audit 2023-08-27T23:02:26.156733+0000 mon.b (mon.1) 306 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:02:26.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:26 smithi006 bash[33918]: audit 2023-08-27T23:02:26.159027+0000 mon.b (mon.1) 307 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:02:26.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:26 smithi006 bash[33918]: audit 2023-08-27T23:02:26.167639+0000 mon.a (mon.0) 1847 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:02:26.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:26 smithi130 bash[27579]: audit 2023-08-27T23:02:25.377591+0000 mon.a (mon.0) 1845 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:02:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:26 smithi130 bash[27579]: audit 2023-08-27T23:02:25.860634+0000 mon.a (mon.0) 1846 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:02:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:26 smithi130 bash[27579]: audit 2023-08-27T23:02:26.156733+0000 mon.b (mon.1) 306 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:02:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:26 smithi130 bash[27579]: audit 2023-08-27T23:02:26.159027+0000 mon.b (mon.1) 307 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:02:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:26 smithi130 bash[27579]: audit 2023-08-27T23:02:26.167639+0000 mon.a (mon.0) 1847 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:02:27.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:27 smithi130 bash[51302]: debug 2023-08-27T23:02:27.040+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:27.045927+0000) 2023-08-27T23:02:27.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:27 smithi006 bash[33918]: cluster 2023-08-27T23:02:26.445442+0000 mgr.y (mgr.14901) 1917 : cluster [DBG] pgmap v1924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:27.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:27 smithi006 bash[33918]: audit 2023-08-27T23:02:26.808947+0000 mon.a (mon.0) 1848 : audit [DBG] from='client.? 172.21.15.6:0/3130772341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:27.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:27 smithi006 bash[27292]: cluster 2023-08-27T23:02:26.445442+0000 mgr.y (mgr.14901) 1917 : cluster [DBG] pgmap v1924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:27.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:27 smithi006 bash[27292]: audit 2023-08-27T23:02:26.808947+0000 mon.a (mon.0) 1848 : audit [DBG] from='client.? 172.21.15.6:0/3130772341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:27.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:27 smithi130 bash[27579]: cluster 2023-08-27T23:02:26.445442+0000 mgr.y (mgr.14901) 1917 : cluster [DBG] pgmap v1924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:27 smithi130 bash[27579]: audit 2023-08-27T23:02:26.808947+0000 mon.a (mon.0) 1848 : audit [DBG] from='client.? 172.21.15.6:0/3130772341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:28.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:28 smithi130 bash[51302]: debug 2023-08-27T23:02:28.040+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:28.046223+0000) 2023-08-27T23:02:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:28 smithi130 bash[27579]: cluster 2023-08-27T23:02:28.449000+0000 mgr.y (mgr.14901) 1918 : cluster [DBG] pgmap v1925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:28.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:28 smithi006 bash[27292]: cluster 2023-08-27T23:02:28.449000+0000 mgr.y (mgr.14901) 1918 : cluster [DBG] pgmap v1925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:28.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:28 smithi006 bash[33918]: cluster 2023-08-27T23:02:28.449000+0000 mgr.y (mgr.14901) 1918 : cluster [DBG] pgmap v1925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:29.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:29 smithi130 bash[51302]: debug 2023-08-27T23:02:29.040+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:29.046518+0000) 2023-08-27T23:02:29.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:29 smithi130 bash[27579]: audit 2023-08-27T23:02:29.227966+0000 mon.c (mon.2) 761 : audit [DBG] from='client.? 172.21.15.6:0/3091260337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:29.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:29 smithi006 bash[33918]: audit 2023-08-27T23:02:29.227966+0000 mon.c (mon.2) 761 : audit [DBG] from='client.? 172.21.15.6:0/3091260337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:29.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:29 smithi006 bash[27292]: audit 2023-08-27T23:02:29.227966+0000 mon.c (mon.2) 761 : audit [DBG] from='client.? 172.21.15.6:0/3091260337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:30.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:30 smithi130 bash[51302]: debug 2023-08-27T23:02:30.040+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:30.046775+0000) 2023-08-27T23:02:30.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:30 smithi006 bash[27292]: cluster 2023-08-27T23:02:30.450135+0000 mgr.y (mgr.14901) 1919 : cluster [DBG] pgmap v1926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:30.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:30 smithi006 bash[33918]: cluster 2023-08-27T23:02:30.450135+0000 mgr.y (mgr.14901) 1919 : cluster [DBG] pgmap v1926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:30.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:02:30.360Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:30.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:02:30.362Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:30.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:30 smithi130 bash[27579]: cluster 2023-08-27T23:02:30.450135+0000 mgr.y (mgr.14901) 1919 : cluster [DBG] pgmap v1926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:31.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:02:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:02:30] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T23:02:31.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:31 smithi130 bash[51302]: debug 2023-08-27T23:02:31.044+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:31.047070+0000) 2023-08-27T23:02:31.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:31 smithi006 bash[33918]: audit 2023-08-27T23:02:31.644532+0000 mon.c (mon.2) 762 : audit [DBG] from='client.? 172.21.15.6:0/4075062504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:31.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:31 smithi006 bash[27292]: audit 2023-08-27T23:02:31.644532+0000 mon.c (mon.2) 762 : audit [DBG] from='client.? 172.21.15.6:0/4075062504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:31.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:31 smithi006 bash[56298]: level=error ts=2023-08-27T23:02:31.922Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:31.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:31 smithi006 bash[56298]: level=warn ts=2023-08-27T23:02:31.924Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:32.048 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:31 smithi130 bash[27579]: audit 2023-08-27T23:02:31.644532+0000 mon.c (mon.2) 762 : audit [DBG] from='client.? 172.21.15.6:0/4075062504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:32.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:32 smithi130 bash[51302]: debug 2023-08-27T23:02:32.044+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:32.047372+0000) 2023-08-27T23:02:32.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:32 smithi006 bash[33918]: cluster 2023-08-27T23:02:32.450811+0000 mgr.y (mgr.14901) 1920 : cluster [DBG] pgmap v1927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:32.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:32 smithi006 bash[27292]: cluster 2023-08-27T23:02:32.450811+0000 mgr.y (mgr.14901) 1920 : cluster [DBG] pgmap v1927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:33.048 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:32 smithi130 bash[27579]: cluster 2023-08-27T23:02:32.450811+0000 mgr.y (mgr.14901) 1920 : cluster [DBG] pgmap v1927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:33.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:33 smithi130 bash[51302]: debug 2023-08-27T23:02:33.044+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:33.047691+0000) 2023-08-27T23:02:34.118 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:02:33.819Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:34.118 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:02:33.821Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:34.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:34 smithi130 bash[27579]: audit 2023-08-27T23:02:34.061677+0000 mon.c (mon.2) 763 : audit [DBG] from='client.? 172.21.15.6:0/3426044850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:34.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:34 smithi130 bash[51302]: debug 2023-08-27T23:02:34.044+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:34.048013+0000) 2023-08-27T23:02:34.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:34 smithi006 bash[27292]: audit 2023-08-27T23:02:34.061677+0000 mon.c (mon.2) 763 : audit [DBG] from='client.? 172.21.15.6:0/3426044850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:34.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:34 smithi006 bash[33918]: audit 2023-08-27T23:02:34.061677+0000 mon.c (mon.2) 763 : audit [DBG] from='client.? 172.21.15.6:0/3426044850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:35 smithi130 bash[27579]: cluster 2023-08-27T23:02:34.451932+0000 mgr.y (mgr.14901) 1921 : cluster [DBG] pgmap v1928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:35.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:35 smithi130 bash[51302]: debug 2023-08-27T23:02:35.044+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:35.048365+0000) 2023-08-27T23:02:35.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:35 smithi006 bash[27292]: cluster 2023-08-27T23:02:34.451932+0000 mgr.y (mgr.14901) 1921 : cluster [DBG] pgmap v1928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:35.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:35 smithi006 bash[33918]: cluster 2023-08-27T23:02:34.451932+0000 mgr.y (mgr.14901) 1921 : cluster [DBG] pgmap v1928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:36.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:36 smithi130 bash[51302]: debug 2023-08-27T23:02:36.048+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:36.048654+0000) 2023-08-27T23:02:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:36 smithi130 bash[27579]: cluster 2023-08-27T23:02:36.452672+0000 mgr.y (mgr.14901) 1922 : cluster [DBG] pgmap v1929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:36 smithi130 bash[27579]: audit 2023-08-27T23:02:36.489521+0000 mon.c (mon.2) 764 : audit [DBG] from='client.? 172.21.15.6:0/4289186174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:36.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:36 smithi006 bash[27292]: cluster 2023-08-27T23:02:36.452672+0000 mgr.y (mgr.14901) 1922 : cluster [DBG] pgmap v1929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:36.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:36 smithi006 bash[27292]: audit 2023-08-27T23:02:36.489521+0000 mon.c (mon.2) 764 : audit [DBG] from='client.? 172.21.15.6:0/4289186174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:36 smithi006 bash[33918]: cluster 2023-08-27T23:02:36.452672+0000 mgr.y (mgr.14901) 1922 : cluster [DBG] pgmap v1929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:36 smithi006 bash[33918]: audit 2023-08-27T23:02:36.489521+0000 mon.c (mon.2) 764 : audit [DBG] from='client.? 172.21.15.6:0/4289186174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:37.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:37 smithi130 bash[51302]: debug 2023-08-27T23:02:37.044+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:37.048922+0000) 2023-08-27T23:02:38.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:38 smithi130 bash[51302]: debug 2023-08-27T23:02:38.044+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:38.049182+0000) 2023-08-27T23:02:38.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:38 smithi130 bash[27579]: cluster 2023-08-27T23:02:38.453465+0000 mgr.y (mgr.14901) 1923 : cluster [DBG] pgmap v1930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:38.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:38 smithi006 bash[27292]: cluster 2023-08-27T23:02:38.453465+0000 mgr.y (mgr.14901) 1923 : cluster [DBG] pgmap v1930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:38.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:38 smithi006 bash[33918]: cluster 2023-08-27T23:02:38.453465+0000 mgr.y (mgr.14901) 1923 : cluster [DBG] pgmap v1930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:39.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:39 smithi130 bash[51302]: debug 2023-08-27T23:02:39.044+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:39.049456+0000) 2023-08-27T23:02:39.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:39 smithi130 bash[27579]: audit 2023-08-27T23:02:38.906674+0000 mon.a (mon.0) 1849 : audit [DBG] from='client.? 172.21.15.6:0/3457814547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:39.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:39 smithi006 bash[27292]: audit 2023-08-27T23:02:38.906674+0000 mon.a (mon.0) 1849 : audit [DBG] from='client.? 172.21.15.6:0/3457814547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:39.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:39 smithi006 bash[33918]: audit 2023-08-27T23:02:38.906674+0000 mon.a (mon.0) 1849 : audit [DBG] from='client.? 172.21.15.6:0/3457814547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:40.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:40 smithi130 bash[51302]: debug 2023-08-27T23:02:40.048+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:40.049783+0000) 2023-08-27T23:02:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:40 smithi006 bash[33918]: cluster 2023-08-27T23:02:40.454655+0000 mgr.y (mgr.14901) 1924 : cluster [DBG] pgmap v1931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:40.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:02:40.361Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:40.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:02:40.363Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:40.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:40 smithi006 bash[27292]: cluster 2023-08-27T23:02:40.454655+0000 mgr.y (mgr.14901) 1924 : cluster [DBG] pgmap v1931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:40 smithi130 bash[27579]: cluster 2023-08-27T23:02:40.454655+0000 mgr.y (mgr.14901) 1924 : cluster [DBG] pgmap v1931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:41.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:02:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:02:40] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T23:02:41.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:41 smithi130 bash[51302]: debug 2023-08-27T23:02:41.044+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:41.050059+0000) 2023-08-27T23:02:41.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:41 smithi130 bash[27579]: audit 2023-08-27T23:02:41.320240+0000 mon.a (mon.0) 1850 : audit [DBG] from='client.? 172.21.15.6:0/449375498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:41.926 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:41 smithi006 bash[33918]: audit 2023-08-27T23:02:41.320240+0000 mon.a (mon.0) 1850 : audit [DBG] from='client.? 172.21.15.6:0/449375498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:41.926 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:41 smithi006 bash[27292]: audit 2023-08-27T23:02:41.320240+0000 mon.a (mon.0) 1850 : audit [DBG] from='client.? 172.21.15.6:0/449375498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:42.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:41 smithi006 bash[56298]: level=error ts=2023-08-27T23:02:41.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:42.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:41 smithi006 bash[56298]: level=warn ts=2023-08-27T23:02:41.924Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:42.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:42 smithi130 bash[51302]: debug 2023-08-27T23:02:42.044+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:42.050401+0000) 2023-08-27T23:02:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:42 smithi130 bash[27579]: cluster 2023-08-27T23:02:42.455301+0000 mgr.y (mgr.14901) 1925 : cluster [DBG] pgmap v1932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:42.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:42 smithi006 bash[27292]: cluster 2023-08-27T23:02:42.455301+0000 mgr.y (mgr.14901) 1925 : cluster [DBG] pgmap v1932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:42.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:42 smithi006 bash[33918]: cluster 2023-08-27T23:02:42.455301+0000 mgr.y (mgr.14901) 1925 : cluster [DBG] pgmap v1932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:43.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:43 smithi130 bash[51302]: debug 2023-08-27T23:02:43.044+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:43.050732+0000) 2023-08-27T23:02:44.052 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:43 smithi130 bash[27579]: audit 2023-08-27T23:02:43.731906+0000 mon.c (mon.2) 765 : audit [DBG] from='client.? 172.21.15.6:0/4196334835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:44.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:43 smithi006 bash[33918]: audit 2023-08-27T23:02:43.731906+0000 mon.c (mon.2) 765 : audit [DBG] from='client.? 172.21.15.6:0/4196334835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:44.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:43 smithi006 bash[27292]: audit 2023-08-27T23:02:43.731906+0000 mon.c (mon.2) 765 : audit [DBG] from='client.? 172.21.15.6:0/4196334835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:44.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:02:43.820Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:44.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:02:43.823Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:44.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:44 smithi130 bash[51302]: debug 2023-08-27T23:02:44.048+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:44.051012+0000) 2023-08-27T23:02:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:45 smithi130 bash[27579]: cluster 2023-08-27T23:02:44.456422+0000 mgr.y (mgr.14901) 1926 : cluster [DBG] pgmap v1933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:45 smithi130 bash[51302]: debug 2023-08-27T23:02:45.048+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:45.051319+0000) 2023-08-27T23:02:45.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:45 smithi006 bash[27292]: cluster 2023-08-27T23:02:44.456422+0000 mgr.y (mgr.14901) 1926 : cluster [DBG] pgmap v1933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:45.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:45 smithi006 bash[33918]: cluster 2023-08-27T23:02:44.456422+0000 mgr.y (mgr.14901) 1926 : cluster [DBG] pgmap v1933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:46 smithi130 bash[27579]: audit 2023-08-27T23:02:46.149378+0000 mon.a (mon.0) 1851 : audit [DBG] from='client.? 172.21.15.6:0/1212894606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:46 smithi130 bash[51302]: debug 2023-08-27T23:02:46.047+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:46.051590+0000) 2023-08-27T23:02:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:46 smithi006 bash[27292]: audit 2023-08-27T23:02:46.149378+0000 mon.a (mon.0) 1851 : audit [DBG] from='client.? 172.21.15.6:0/1212894606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:46.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:46 smithi006 bash[33918]: audit 2023-08-27T23:02:46.149378+0000 mon.a (mon.0) 1851 : audit [DBG] from='client.? 172.21.15.6:0/1212894606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:47 smithi130 bash[27579]: cluster 2023-08-27T23:02:46.457089+0000 mgr.y (mgr.14901) 1927 : cluster [DBG] pgmap v1934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:47 smithi130 bash[51302]: debug 2023-08-27T23:02:47.047+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:47.051874+0000) 2023-08-27T23:02:47.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:47 smithi006 bash[27292]: cluster 2023-08-27T23:02:46.457089+0000 mgr.y (mgr.14901) 1927 : cluster [DBG] pgmap v1934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:47.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:47 smithi006 bash[33918]: cluster 2023-08-27T23:02:46.457089+0000 mgr.y (mgr.14901) 1927 : cluster [DBG] pgmap v1934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:48.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:48 smithi130 bash[51302]: debug 2023-08-27T23:02:48.047+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:48.052115+0000) 2023-08-27T23:02:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:48 smithi130 bash[27579]: cluster 2023-08-27T23:02:48.457695+0000 mgr.y (mgr.14901) 1928 : cluster [DBG] pgmap v1935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:48.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:48 smithi006 bash[33918]: cluster 2023-08-27T23:02:48.457695+0000 mgr.y (mgr.14901) 1928 : cluster [DBG] pgmap v1935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:48.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:48 smithi006 bash[27292]: cluster 2023-08-27T23:02:48.457695+0000 mgr.y (mgr.14901) 1928 : cluster [DBG] pgmap v1935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:49.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:49 smithi130 bash[51302]: debug 2023-08-27T23:02:49.051+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:49.052416+0000) 2023-08-27T23:02:49.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:49 smithi130 bash[27579]: audit 2023-08-27T23:02:48.566921+0000 mon.a (mon.0) 1852 : audit [DBG] from='client.? 172.21.15.6:0/3054154255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:49.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:49 smithi006 bash[27292]: audit 2023-08-27T23:02:48.566921+0000 mon.a (mon.0) 1852 : audit [DBG] from='client.? 172.21.15.6:0/3054154255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:49.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:49 smithi006 bash[33918]: audit 2023-08-27T23:02:48.566921+0000 mon.a (mon.0) 1852 : audit [DBG] from='client.? 172.21.15.6:0/3054154255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:50.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:50 smithi130 bash[51302]: debug 2023-08-27T23:02:50.051+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:50.052726+0000) 2023-08-27T23:02:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:02:50.362Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:50.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:02:50.363Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:50 smithi006 bash[33918]: cluster 2023-08-27T23:02:50.458816+0000 mgr.y (mgr.14901) 1929 : cluster [DBG] pgmap v1936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:50 smithi006 bash[27292]: cluster 2023-08-27T23:02:50.458816+0000 mgr.y (mgr.14901) 1929 : cluster [DBG] pgmap v1936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:50 smithi130 bash[27579]: cluster 2023-08-27T23:02:50.458816+0000 mgr.y (mgr.14901) 1929 : cluster [DBG] pgmap v1936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:51.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:02:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:02:50] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:02:51.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:51 smithi130 bash[51302]: debug 2023-08-27T23:02:51.051+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:51.053078+0000) 2023-08-27T23:02:51.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:51 smithi130 bash[27579]: audit 2023-08-27T23:02:50.992298+0000 mon.a (mon.0) 1853 : audit [DBG] from='client.? 172.21.15.6:0/1566355003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:51.927 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:51 smithi006 bash[33918]: audit 2023-08-27T23:02:50.992298+0000 mon.a (mon.0) 1853 : audit [DBG] from='client.? 172.21.15.6:0/1566355003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:51.927 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:51 smithi006 bash[27292]: audit 2023-08-27T23:02:50.992298+0000 mon.a (mon.0) 1853 : audit [DBG] from='client.? 172.21.15.6:0/1566355003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:52.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:51 smithi006 bash[56298]: level=error ts=2023-08-27T23:02:51.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:52.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:51 smithi006 bash[56298]: level=warn ts=2023-08-27T23:02:51.925Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:52.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:52 smithi130 bash[51302]: debug 2023-08-27T23:02:52.051+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:52.053380+0000) 2023-08-27T23:02:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:52 smithi130 bash[27579]: cluster 2023-08-27T23:02:52.459449+0000 mgr.y (mgr.14901) 1930 : cluster [DBG] pgmap v1937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:52.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:52 smithi006 bash[33918]: cluster 2023-08-27T23:02:52.459449+0000 mgr.y (mgr.14901) 1930 : cluster [DBG] pgmap v1937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:52.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:52 smithi006 bash[27292]: cluster 2023-08-27T23:02:52.459449+0000 mgr.y (mgr.14901) 1930 : cluster [DBG] pgmap v1937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:53.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:53 smithi130 bash[51302]: debug 2023-08-27T23:02:53.051+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:53.053661+0000) 2023-08-27T23:02:53.825 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:53 smithi006 bash[33918]: audit 2023-08-27T23:02:53.402344+0000 mon.a (mon.0) 1854 : audit [DBG] from='client.? 172.21.15.6:0/3492398667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:53.825 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:53 smithi006 bash[27292]: audit 2023-08-27T23:02:53.402344+0000 mon.a (mon.0) 1854 : audit [DBG] from='client.? 172.21.15.6:0/3492398667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:53.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:53 smithi130 bash[27579]: audit 2023-08-27T23:02:53.402344+0000 mon.a (mon.0) 1854 : audit [DBG] from='client.? 172.21.15.6:0/3492398667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:54.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:02:53.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:54.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:02:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:02:53.824Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:02:54.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:54 smithi130 bash[51302]: debug 2023-08-27T23:02:54.047+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:54.053937+0000) 2023-08-27T23:02:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:55 smithi130 bash[27579]: cluster 2023-08-27T23:02:54.460589+0000 mgr.y (mgr.14901) 1931 : cluster [DBG] pgmap v1938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:55 smithi130 bash[51302]: debug 2023-08-27T23:02:55.047+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:55.054251+0000) 2023-08-27T23:02:55.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:55 smithi006 bash[27292]: cluster 2023-08-27T23:02:54.460589+0000 mgr.y (mgr.14901) 1931 : cluster [DBG] pgmap v1938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:55.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:55 smithi006 bash[33918]: cluster 2023-08-27T23:02:54.460589+0000 mgr.y (mgr.14901) 1931 : cluster [DBG] pgmap v1938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:56.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:56 smithi130 bash[27579]: audit 2023-08-27T23:02:55.818763+0000 mon.c (mon.2) 766 : audit [DBG] from='client.? 172.21.15.6:0/1559363915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:56.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:56 smithi130 bash[51302]: debug 2023-08-27T23:02:56.047+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:56.054604+0000) 2023-08-27T23:02:56.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:56 smithi006 bash[33918]: audit 2023-08-27T23:02:55.818763+0000 mon.c (mon.2) 766 : audit [DBG] from='client.? 172.21.15.6:0/1559363915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:56 smithi006 bash[27292]: audit 2023-08-27T23:02:55.818763+0000 mon.c (mon.2) 766 : audit [DBG] from='client.? 172.21.15.6:0/1559363915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:57.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:57 smithi130 bash[27579]: cluster 2023-08-27T23:02:56.461294+0000 mgr.y (mgr.14901) 1932 : cluster [DBG] pgmap v1939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:57.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:57 smithi130 bash[51302]: debug 2023-08-27T23:02:57.051+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:57.054899+0000) 2023-08-27T23:02:57.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:57 smithi006 bash[33918]: cluster 2023-08-27T23:02:56.461294+0000 mgr.y (mgr.14901) 1932 : cluster [DBG] pgmap v1939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:57.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:57 smithi006 bash[27292]: cluster 2023-08-27T23:02:56.461294+0000 mgr.y (mgr.14901) 1932 : cluster [DBG] pgmap v1939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:58.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:58 smithi130 bash[27579]: audit 2023-08-27T23:02:58.233861+0000 mon.a (mon.0) 1855 : audit [DBG] from='client.? 172.21.15.6:0/3127262532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:58.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:58 smithi130 bash[51302]: debug 2023-08-27T23:02:58.051+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:58.055122+0000) 2023-08-27T23:02:58.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:58 smithi006 bash[33918]: audit 2023-08-27T23:02:58.233861+0000 mon.a (mon.0) 1855 : audit [DBG] from='client.? 172.21.15.6:0/3127262532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:58.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:58 smithi006 bash[27292]: audit 2023-08-27T23:02:58.233861+0000 mon.a (mon.0) 1855 : audit [DBG] from='client.? 172.21.15.6:0/3127262532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:02:59 smithi130 bash[27579]: cluster 2023-08-27T23:02:58.461735+0000 mgr.y (mgr.14901) 1933 : cluster [DBG] pgmap v1940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:02:59 smithi130 bash[51302]: debug 2023-08-27T23:02:59.051+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:59.055387+0000) 2023-08-27T23:02:59.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:02:59 smithi006 bash[27292]: cluster 2023-08-27T23:02:58.461735+0000 mgr.y (mgr.14901) 1933 : cluster [DBG] pgmap v1940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:02:59.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:02:59 smithi006 bash[33918]: cluster 2023-08-27T23:02:58.461735+0000 mgr.y (mgr.14901) 1933 : cluster [DBG] pgmap v1940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:00.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:00 smithi130 bash[51302]: debug 2023-08-27T23:03:00.051+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:00.055582+0000) 2023-08-27T23:03:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:00 smithi006 bash[33918]: cluster 2023-08-27T23:03:00.462695+0000 mgr.y (mgr.14901) 1934 : cluster [DBG] pgmap v1941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:00.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:00 smithi006 bash[27292]: cluster 2023-08-27T23:03:00.462695+0000 mgr.y (mgr.14901) 1934 : cluster [DBG] pgmap v1941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:00.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:03:00.362Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:00.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:03:00.364Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:00 smithi130 bash[27579]: cluster 2023-08-27T23:03:00.462695+0000 mgr.y (mgr.14901) 1934 : cluster [DBG] pgmap v1941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:01.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:03:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:03:00] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T23:03:01.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:01 smithi130 bash[51302]: debug 2023-08-27T23:03:01.051+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:01.055883+0000) 2023-08-27T23:03:01.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:01 smithi130 bash[27579]: audit 2023-08-27T23:03:00.654223+0000 mon.c (mon.2) 767 : audit [DBG] from='client.? 172.21.15.6:0/3581331718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:01.927 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:01 smithi006 bash[33918]: audit 2023-08-27T23:03:00.654223+0000 mon.c (mon.2) 767 : audit [DBG] from='client.? 172.21.15.6:0/3581331718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:01.927 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:01 smithi006 bash[27292]: audit 2023-08-27T23:03:00.654223+0000 mon.c (mon.2) 767 : audit [DBG] from='client.? 172.21.15.6:0/3581331718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:02.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:01 smithi006 bash[56298]: level=error ts=2023-08-27T23:03:01.924Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:02.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:01 smithi006 bash[56298]: level=warn ts=2023-08-27T23:03:01.925Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:02.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:02 smithi130 bash[51302]: debug 2023-08-27T23:03:02.051+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:02.056149+0000) 2023-08-27T23:03:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:02 smithi130 bash[27579]: cluster 2023-08-27T23:03:02.463329+0000 mgr.y (mgr.14901) 1935 : cluster [DBG] pgmap v1942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:02.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:02 smithi006 bash[27292]: cluster 2023-08-27T23:03:02.463329+0000 mgr.y (mgr.14901) 1935 : cluster [DBG] pgmap v1942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:02.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:02 smithi006 bash[33918]: cluster 2023-08-27T23:03:02.463329+0000 mgr.y (mgr.14901) 1935 : cluster [DBG] pgmap v1942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:03.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:03 smithi130 bash[51302]: debug 2023-08-27T23:03:03.051+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:03.056459+0000) 2023-08-27T23:03:03.826 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:03 smithi006 bash[33918]: audit 2023-08-27T23:03:03.061752+0000 mon.c (mon.2) 768 : audit [DBG] from='client.? 172.21.15.6:0/1425472233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:03.826 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:03 smithi006 bash[27292]: audit 2023-08-27T23:03:03.061752+0000 mon.c (mon.2) 768 : audit [DBG] from='client.? 172.21.15.6:0/1425472233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:03.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:03 smithi130 bash[27579]: audit 2023-08-27T23:03:03.061752+0000 mon.c (mon.2) 768 : audit [DBG] from='client.? 172.21.15.6:0/1425472233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:04.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:03:03.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:04.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:03:03.824Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:04.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:04 smithi130 bash[51302]: debug 2023-08-27T23:03:04.051+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:04.056738+0000) 2023-08-27T23:03:05.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:05 smithi130 bash[27579]: cluster 2023-08-27T23:03:04.464449+0000 mgr.y (mgr.14901) 1936 : cluster [DBG] pgmap v1943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:05 smithi130 bash[51302]: debug 2023-08-27T23:03:05.051+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:05.057008+0000) 2023-08-27T23:03:05.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:05 smithi006 bash[27292]: cluster 2023-08-27T23:03:04.464449+0000 mgr.y (mgr.14901) 1936 : cluster [DBG] pgmap v1943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:05.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:05 smithi006 bash[33918]: cluster 2023-08-27T23:03:04.464449+0000 mgr.y (mgr.14901) 1936 : cluster [DBG] pgmap v1943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:06 smithi130 bash[27579]: audit 2023-08-27T23:03:05.481647+0000 mon.a (mon.0) 1856 : audit [DBG] from='client.? 172.21.15.6:0/971434165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:06.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:06 smithi130 bash[51302]: debug 2023-08-27T23:03:06.051+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:06.057227+0000) 2023-08-27T23:03:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:06 smithi006 bash[27292]: audit 2023-08-27T23:03:05.481647+0000 mon.a (mon.0) 1856 : audit [DBG] from='client.? 172.21.15.6:0/971434165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:06 smithi006 bash[33918]: audit 2023-08-27T23:03:05.481647+0000 mon.a (mon.0) 1856 : audit [DBG] from='client.? 172.21.15.6:0/971434165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:07.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:07 smithi130 bash[27579]: cluster 2023-08-27T23:03:06.465131+0000 mgr.y (mgr.14901) 1937 : cluster [DBG] pgmap v1944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:07.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:07 smithi130 bash[51302]: debug 2023-08-27T23:03:07.051+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:07.057499+0000) 2023-08-27T23:03:07.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:07 smithi006 bash[27292]: cluster 2023-08-27T23:03:06.465131+0000 mgr.y (mgr.14901) 1937 : cluster [DBG] pgmap v1944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:07.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:07 smithi006 bash[33918]: cluster 2023-08-27T23:03:06.465131+0000 mgr.y (mgr.14901) 1937 : cluster [DBG] pgmap v1944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:08.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:08 smithi130 bash[27579]: audit 2023-08-27T23:03:07.899855+0000 mon.a (mon.0) 1857 : audit [DBG] from='client.? 172.21.15.6:0/2990729098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:08.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:08 smithi130 bash[51302]: debug 2023-08-27T23:03:08.051+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:08.057810+0000) 2023-08-27T23:03:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:08 smithi006 bash[33918]: audit 2023-08-27T23:03:07.899855+0000 mon.a (mon.0) 1857 : audit [DBG] from='client.? 172.21.15.6:0/2990729098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:08.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:08 smithi006 bash[27292]: audit 2023-08-27T23:03:07.899855+0000 mon.a (mon.0) 1857 : audit [DBG] from='client.? 172.21.15.6:0/2990729098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:09.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:09 smithi130 bash[27579]: cluster 2023-08-27T23:03:08.465813+0000 mgr.y (mgr.14901) 1938 : cluster [DBG] pgmap v1945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:09.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:09 smithi130 bash[51302]: debug 2023-08-27T23:03:09.051+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:09.058085+0000) 2023-08-27T23:03:09.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:09 smithi006 bash[27292]: cluster 2023-08-27T23:03:08.465813+0000 mgr.y (mgr.14901) 1938 : cluster [DBG] pgmap v1945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:09.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:09 smithi006 bash[33918]: cluster 2023-08-27T23:03:08.465813+0000 mgr.y (mgr.14901) 1938 : cluster [DBG] pgmap v1945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:10.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:10 smithi130 bash[51302]: debug 2023-08-27T23:03:10.051+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:10.058362+0000) 2023-08-27T23:03:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:10 smithi006 bash[27292]: audit 2023-08-27T23:03:10.320745+0000 mon.a (mon.0) 1858 : audit [DBG] from='client.? 172.21.15.6:0/2715689397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:10 smithi006 bash[33918]: audit 2023-08-27T23:03:10.320745+0000 mon.a (mon.0) 1858 : audit [DBG] from='client.? 172.21.15.6:0/2715689397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:10.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:03:10.363Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:10.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:03:10.366Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:10 smithi130 bash[27579]: audit 2023-08-27T23:03:10.320745+0000 mon.a (mon.0) 1858 : audit [DBG] from='client.? 172.21.15.6:0/2715689397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:11.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:03:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:03:10] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T23:03:11.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:11 smithi130 bash[51302]: debug 2023-08-27T23:03:11.051+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:11.058616+0000) 2023-08-27T23:03:11.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:11 smithi006 bash[33918]: cluster 2023-08-27T23:03:10.466946+0000 mgr.y (mgr.14901) 1939 : cluster [DBG] pgmap v1946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:11.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:11 smithi006 bash[27292]: cluster 2023-08-27T23:03:10.466946+0000 mgr.y (mgr.14901) 1939 : cluster [DBG] pgmap v1946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:11.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:11 smithi130 bash[27579]: cluster 2023-08-27T23:03:10.466946+0000 mgr.y (mgr.14901) 1939 : cluster [DBG] pgmap v1946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:12.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:11 smithi006 bash[56298]: level=error ts=2023-08-27T23:03:11.924Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:12.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:11 smithi006 bash[56298]: level=warn ts=2023-08-27T23:03:11.926Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:12.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:12 smithi130 bash[51302]: debug 2023-08-27T23:03:12.055+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:12.058913+0000) 2023-08-27T23:03:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:12 smithi130 bash[27579]: cluster 2023-08-27T23:03:12.467415+0000 mgr.y (mgr.14901) 1940 : cluster [DBG] pgmap v1947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:12.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:12 smithi006 bash[27292]: cluster 2023-08-27T23:03:12.467415+0000 mgr.y (mgr.14901) 1940 : cluster [DBG] pgmap v1947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:12.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:12 smithi006 bash[33918]: cluster 2023-08-27T23:03:12.467415+0000 mgr.y (mgr.14901) 1940 : cluster [DBG] pgmap v1947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:13 smithi130 bash[51302]: debug 2023-08-27T23:03:13.055+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:13.059212+0000) 2023-08-27T23:03:13.827 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:13 smithi006 bash[27292]: audit 2023-08-27T23:03:12.731056+0000 mon.c (mon.2) 769 : audit [DBG] from='client.? 172.21.15.6:0/1688957292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:13.827 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:13 smithi006 bash[33918]: audit 2023-08-27T23:03:12.731056+0000 mon.c (mon.2) 769 : audit [DBG] from='client.? 172.21.15.6:0/1688957292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:13.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:13 smithi130 bash[27579]: audit 2023-08-27T23:03:12.731056+0000 mon.c (mon.2) 769 : audit [DBG] from='client.? 172.21.15.6:0/1688957292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:14.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:03:13.823Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:14.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:03:13.825Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:14.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:14 smithi130 bash[51302]: debug 2023-08-27T23:03:14.055+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:14.059534+0000) 2023-08-27T23:03:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:15 smithi130 bash[27579]: cluster 2023-08-27T23:03:14.468523+0000 mgr.y (mgr.14901) 1941 : cluster [DBG] pgmap v1948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:15 smithi130 bash[27579]: audit 2023-08-27T23:03:15.144648+0000 mon.c (mon.2) 770 : audit [DBG] from='client.? 172.21.15.6:0/3483556658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:15 smithi130 bash[51302]: debug 2023-08-27T23:03:15.055+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:15.059794+0000) 2023-08-27T23:03:15.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:15 smithi006 bash[27292]: cluster 2023-08-27T23:03:14.468523+0000 mgr.y (mgr.14901) 1941 : cluster [DBG] pgmap v1948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:15.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:15 smithi006 bash[27292]: audit 2023-08-27T23:03:15.144648+0000 mon.c (mon.2) 770 : audit [DBG] from='client.? 172.21.15.6:0/3483556658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:15.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:15 smithi006 bash[33918]: cluster 2023-08-27T23:03:14.468523+0000 mgr.y (mgr.14901) 1941 : cluster [DBG] pgmap v1948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:15.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:15 smithi006 bash[33918]: audit 2023-08-27T23:03:15.144648+0000 mon.c (mon.2) 770 : audit [DBG] from='client.? 172.21.15.6:0/3483556658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:16.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:16 smithi130 bash[51302]: debug 2023-08-27T23:03:16.055+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:16.060060+0000) 2023-08-27T23:03:16.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:16 smithi130 bash[27579]: cluster 2023-08-27T23:03:16.469170+0000 mgr.y (mgr.14901) 1942 : cluster [DBG] pgmap v1949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:16.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:16 smithi006 bash[27292]: cluster 2023-08-27T23:03:16.469170+0000 mgr.y (mgr.14901) 1942 : cluster [DBG] pgmap v1949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:16.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:16 smithi006 bash[33918]: cluster 2023-08-27T23:03:16.469170+0000 mgr.y (mgr.14901) 1942 : cluster [DBG] pgmap v1949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:17.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:17 smithi130 bash[51302]: debug 2023-08-27T23:03:17.059+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:17.060346+0000) 2023-08-27T23:03:17.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:17 smithi130 bash[27579]: audit 2023-08-27T23:03:17.556598+0000 mon.c (mon.2) 771 : audit [DBG] from='client.? 172.21.15.6:0/2387171342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:17.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:17 smithi006 bash[27292]: audit 2023-08-27T23:03:17.556598+0000 mon.c (mon.2) 771 : audit [DBG] from='client.? 172.21.15.6:0/2387171342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:17.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:17 smithi006 bash[33918]: audit 2023-08-27T23:03:17.556598+0000 mon.c (mon.2) 771 : audit [DBG] from='client.? 172.21.15.6:0/2387171342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:18.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:18 smithi130 bash[51302]: debug 2023-08-27T23:03:18.055+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:18.060665+0000) 2023-08-27T23:03:18.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:18 smithi006 bash[27292]: cluster 2023-08-27T23:03:18.469834+0000 mgr.y (mgr.14901) 1943 : cluster [DBG] pgmap v1950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:18.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:18 smithi006 bash[33918]: cluster 2023-08-27T23:03:18.469834+0000 mgr.y (mgr.14901) 1943 : cluster [DBG] pgmap v1950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:19.062 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:18 smithi130 bash[27579]: cluster 2023-08-27T23:03:18.469834+0000 mgr.y (mgr.14901) 1943 : cluster [DBG] pgmap v1950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:19.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:19 smithi130 bash[51302]: debug 2023-08-27T23:03:19.055+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:19.060958+0000) 2023-08-27T23:03:20.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:20 smithi130 bash[51302]: debug 2023-08-27T23:03:20.059+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:20.061242+0000) 2023-08-27T23:03:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:20 smithi130 bash[27579]: audit 2023-08-27T23:03:19.966668+0000 mon.c (mon.2) 772 : audit [DBG] from='client.? 172.21.15.6:0/2490954184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:20.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:20 smithi006 bash[33918]: audit 2023-08-27T23:03:19.966668+0000 mon.c (mon.2) 772 : audit [DBG] from='client.? 172.21.15.6:0/2490954184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:20 smithi006 bash[27292]: audit 2023-08-27T23:03:19.966668+0000 mon.c (mon.2) 772 : audit [DBG] from='client.? 172.21.15.6:0/2490954184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:20.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:03:20.363Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:20.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:03:20.365Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:21.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:03:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:03:20] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:03:21.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:21 smithi130 bash[27579]: cluster 2023-08-27T23:03:20.470982+0000 mgr.y (mgr.14901) 1944 : cluster [DBG] pgmap v1951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:21.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:21 smithi130 bash[51302]: debug 2023-08-27T23:03:21.055+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:21.061558+0000) 2023-08-27T23:03:21.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:21 smithi006 bash[33918]: cluster 2023-08-27T23:03:20.470982+0000 mgr.y (mgr.14901) 1944 : cluster [DBG] pgmap v1951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:21.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:21 smithi006 bash[27292]: cluster 2023-08-27T23:03:20.470982+0000 mgr.y (mgr.14901) 1944 : cluster [DBG] pgmap v1951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:22.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:21 smithi006 bash[56298]: level=error ts=2023-08-27T23:03:21.925Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:22.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:21 smithi006 bash[56298]: level=warn ts=2023-08-27T23:03:21.927Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:22.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:22 smithi130 bash[51302]: debug 2023-08-27T23:03:22.059+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:22.061821+0000) 2023-08-27T23:03:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:22 smithi006 bash[33918]: audit 2023-08-27T23:03:22.380971+0000 mon.c (mon.2) 773 : audit [DBG] from='client.? 172.21.15.6:0/4067622999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:22 smithi006 bash[27292]: audit 2023-08-27T23:03:22.380971+0000 mon.c (mon.2) 773 : audit [DBG] from='client.? 172.21.15.6:0/4067622999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:22 smithi130 bash[27579]: audit 2023-08-27T23:03:22.380971+0000 mon.c (mon.2) 773 : audit [DBG] from='client.? 172.21.15.6:0/4067622999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:23.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:23 smithi130 bash[51302]: debug 2023-08-27T23:03:23.059+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:23.062149+0000) 2023-08-27T23:03:23.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:23 smithi006 bash[33918]: cluster 2023-08-27T23:03:22.471638+0000 mgr.y (mgr.14901) 1945 : cluster [DBG] pgmap v1952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:23.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:23 smithi006 bash[27292]: cluster 2023-08-27T23:03:22.471638+0000 mgr.y (mgr.14901) 1945 : cluster [DBG] pgmap v1952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:23.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:23 smithi130 bash[27579]: cluster 2023-08-27T23:03:22.471638+0000 mgr.y (mgr.14901) 1945 : cluster [DBG] pgmap v1952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:24.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:03:23.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:24.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:03:23.827Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:24 smithi130 bash[51302]: debug 2023-08-27T23:03:24.059+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:24.062502+0000) 2023-08-27T23:03:25.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:25 smithi130 bash[27579]: cluster 2023-08-27T23:03:24.472728+0000 mgr.y (mgr.14901) 1946 : cluster [DBG] pgmap v1953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:25 smithi130 bash[27579]: audit 2023-08-27T23:03:24.795326+0000 mon.a (mon.0) 1859 : audit [DBG] from='client.? 172.21.15.6:0/63867098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:25 smithi130 bash[51302]: debug 2023-08-27T23:03:25.059+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:25.062798+0000) 2023-08-27T23:03:25.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:25 smithi006 bash[27292]: cluster 2023-08-27T23:03:24.472728+0000 mgr.y (mgr.14901) 1946 : cluster [DBG] pgmap v1953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:25.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:25 smithi006 bash[27292]: audit 2023-08-27T23:03:24.795326+0000 mon.a (mon.0) 1859 : audit [DBG] from='client.? 172.21.15.6:0/63867098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:25.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:25 smithi006 bash[33918]: cluster 2023-08-27T23:03:24.472728+0000 mgr.y (mgr.14901) 1946 : cluster [DBG] pgmap v1953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:25.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:25 smithi006 bash[33918]: audit 2023-08-27T23:03:24.795326+0000 mon.a (mon.0) 1859 : audit [DBG] from='client.? 172.21.15.6:0/63867098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:26.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:26 smithi130 bash[27579]: audit 2023-08-27T23:03:26.178216+0000 mon.b (mon.1) 308 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:03:26.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:26 smithi130 bash[51302]: debug 2023-08-27T23:03:26.059+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:26.063082+0000) 2023-08-27T23:03:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:26 smithi006 bash[33918]: audit 2023-08-27T23:03:26.178216+0000 mon.b (mon.1) 308 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:03:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:26 smithi006 bash[27292]: audit 2023-08-27T23:03:26.178216+0000 mon.b (mon.1) 308 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:03:27.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:27 smithi130 bash[51302]: debug 2023-08-27T23:03:27.059+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:27.063371+0000) 2023-08-27T23:03:27.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:27 smithi130 bash[27579]: cluster 2023-08-27T23:03:26.473407+0000 mgr.y (mgr.14901) 1947 : cluster [DBG] pgmap v1954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:27.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:27 smithi130 bash[27579]: audit 2023-08-27T23:03:27.213296+0000 mon.a (mon.0) 1860 : audit [DBG] from='client.? 172.21.15.6:0/217024173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:27.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:27 smithi006 bash[33918]: cluster 2023-08-27T23:03:26.473407+0000 mgr.y (mgr.14901) 1947 : cluster [DBG] pgmap v1954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:27.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:27 smithi006 bash[33918]: audit 2023-08-27T23:03:27.213296+0000 mon.a (mon.0) 1860 : audit [DBG] from='client.? 172.21.15.6:0/217024173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:27.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:27 smithi006 bash[27292]: cluster 2023-08-27T23:03:26.473407+0000 mgr.y (mgr.14901) 1947 : cluster [DBG] pgmap v1954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:27.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:27 smithi006 bash[27292]: audit 2023-08-27T23:03:27.213296+0000 mon.a (mon.0) 1860 : audit [DBG] from='client.? 172.21.15.6:0/217024173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:28.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:28 smithi130 bash[51302]: debug 2023-08-27T23:03:28.059+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:28.063604+0000) 2023-08-27T23:03:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:28 smithi130 bash[27579]: cluster 2023-08-27T23:03:28.474085+0000 mgr.y (mgr.14901) 1948 : cluster [DBG] pgmap v1955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:28 smithi006 bash[27292]: cluster 2023-08-27T23:03:28.474085+0000 mgr.y (mgr.14901) 1948 : cluster [DBG] pgmap v1955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:28.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:28 smithi006 bash[33918]: cluster 2023-08-27T23:03:28.474085+0000 mgr.y (mgr.14901) 1948 : cluster [DBG] pgmap v1955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:29.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:29 smithi130 bash[51302]: debug 2023-08-27T23:03:29.059+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:29.063899+0000) 2023-08-27T23:03:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:30 smithi130 bash[27579]: audit 2023-08-27T23:03:29.638609+0000 mon.a (mon.0) 1861 : audit [DBG] from='client.? 172.21.15.6:0/2917636795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:30 smithi130 bash[51302]: debug 2023-08-27T23:03:30.059+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:30.064128+0000) 2023-08-27T23:03:30.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:30 smithi006 bash[33918]: audit 2023-08-27T23:03:29.638609+0000 mon.a (mon.0) 1861 : audit [DBG] from='client.? 172.21.15.6:0/2917636795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:30.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:30 smithi006 bash[27292]: audit 2023-08-27T23:03:29.638609+0000 mon.a (mon.0) 1861 : audit [DBG] from='client.? 172.21.15.6:0/2917636795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:30.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:03:30.364Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:30.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:03:30.366Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:31.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:03:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:03:30] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:03:31.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:31 smithi130 bash[27579]: cluster 2023-08-27T23:03:30.475188+0000 mgr.y (mgr.14901) 1949 : cluster [DBG] pgmap v1956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:31.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:31 smithi130 bash[51302]: debug 2023-08-27T23:03:31.059+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:31.064354+0000) 2023-08-27T23:03:31.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:31 smithi006 bash[33918]: cluster 2023-08-27T23:03:30.475188+0000 mgr.y (mgr.14901) 1949 : cluster [DBG] pgmap v1956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:31.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:31 smithi006 bash[27292]: cluster 2023-08-27T23:03:30.475188+0000 mgr.y (mgr.14901) 1949 : cluster [DBG] pgmap v1956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:32.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:31 smithi006 bash[56298]: level=error ts=2023-08-27T23:03:31.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:32.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:31 smithi006 bash[56298]: level=warn ts=2023-08-27T23:03:31.930Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:32.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:32 smithi130 bash[51302]: debug 2023-08-27T23:03:32.059+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:32.064539+0000) 2023-08-27T23:03:33.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:33 smithi130 bash[51302]: debug 2023-08-27T23:03:33.059+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:33.064787+0000) 2023-08-27T23:03:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:33 smithi130 bash[27579]: audit 2023-08-27T23:03:32.059822+0000 mon.a (mon.0) 1862 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:03:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:33 smithi130 bash[27579]: audit 2023-08-27T23:03:32.076492+0000 mon.a (mon.0) 1863 : audit [DBG] from='client.? 172.21.15.6:0/742908171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:33 smithi130 bash[27579]: cluster 2023-08-27T23:03:32.475876+0000 mgr.y (mgr.14901) 1950 : cluster [DBG] pgmap v1957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:33 smithi130 bash[27579]: audit 2023-08-27T23:03:32.501846+0000 mon.a (mon.0) 1864 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:03:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:33 smithi130 bash[27579]: audit 2023-08-27T23:03:32.770562+0000 mon.b (mon.1) 309 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:03:33.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:33 smithi130 bash[27579]: audit 2023-08-27T23:03:32.772789+0000 mon.b (mon.1) 310 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:03:33.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:33 smithi130 bash[27579]: audit 2023-08-27T23:03:32.782551+0000 mon.a (mon.0) 1865 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:03:33.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:33 smithi006 bash[33918]: audit 2023-08-27T23:03:32.059822+0000 mon.a (mon.0) 1862 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:03:33.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:33 smithi006 bash[33918]: audit 2023-08-27T23:03:32.076492+0000 mon.a (mon.0) 1863 : audit [DBG] from='client.? 172.21.15.6:0/742908171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:33.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:33 smithi006 bash[33918]: cluster 2023-08-27T23:03:32.475876+0000 mgr.y (mgr.14901) 1950 : cluster [DBG] pgmap v1957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:33.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:33 smithi006 bash[33918]: audit 2023-08-27T23:03:32.501846+0000 mon.a (mon.0) 1864 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:03:33.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:33 smithi006 bash[33918]: audit 2023-08-27T23:03:32.770562+0000 mon.b (mon.1) 309 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:03:33.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:33 smithi006 bash[33918]: audit 2023-08-27T23:03:32.772789+0000 mon.b (mon.1) 310 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:03:33.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:33 smithi006 bash[33918]: audit 2023-08-27T23:03:32.782551+0000 mon.a (mon.0) 1865 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:03:33.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:33 smithi006 bash[27292]: audit 2023-08-27T23:03:32.059822+0000 mon.a (mon.0) 1862 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:03:33.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:33 smithi006 bash[27292]: audit 2023-08-27T23:03:32.076492+0000 mon.a (mon.0) 1863 : audit [DBG] from='client.? 172.21.15.6:0/742908171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:33.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:33 smithi006 bash[27292]: cluster 2023-08-27T23:03:32.475876+0000 mgr.y (mgr.14901) 1950 : cluster [DBG] pgmap v1957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:33.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:33 smithi006 bash[27292]: audit 2023-08-27T23:03:32.501846+0000 mon.a (mon.0) 1864 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:03:33.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:33 smithi006 bash[27292]: audit 2023-08-27T23:03:32.770562+0000 mon.b (mon.1) 309 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:03:33.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:33 smithi006 bash[27292]: audit 2023-08-27T23:03:32.772789+0000 mon.b (mon.1) 310 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:03:33.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:33 smithi006 bash[27292]: audit 2023-08-27T23:03:32.782551+0000 mon.a (mon.0) 1865 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:03:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:03:33.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:34.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:03:33.827Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:34.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:34 smithi130 bash[51302]: debug 2023-08-27T23:03:34.059+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:34.065086+0000) 2023-08-27T23:03:35.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:35 smithi130 bash[51302]: debug 2023-08-27T23:03:35.059+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:35.065294+0000) 2023-08-27T23:03:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:35 smithi130 bash[27579]: cluster 2023-08-27T23:03:34.477078+0000 mgr.y (mgr.14901) 1951 : cluster [DBG] pgmap v1958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:35 smithi130 bash[27579]: audit 2023-08-27T23:03:34.488750+0000 mon.a (mon.0) 1866 : audit [DBG] from='client.? 172.21.15.6:0/3127946605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:35.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:35 smithi006 bash[27292]: cluster 2023-08-27T23:03:34.477078+0000 mgr.y (mgr.14901) 1951 : cluster [DBG] pgmap v1958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:35.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:35 smithi006 bash[27292]: audit 2023-08-27T23:03:34.488750+0000 mon.a (mon.0) 1866 : audit [DBG] from='client.? 172.21.15.6:0/3127946605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:35.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:35 smithi006 bash[33918]: cluster 2023-08-27T23:03:34.477078+0000 mgr.y (mgr.14901) 1951 : cluster [DBG] pgmap v1958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:35.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:35 smithi006 bash[33918]: audit 2023-08-27T23:03:34.488750+0000 mon.a (mon.0) 1866 : audit [DBG] from='client.? 172.21.15.6:0/3127946605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:36.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:36 smithi130 bash[51302]: debug 2023-08-27T23:03:36.059+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:36.065575+0000) 2023-08-27T23:03:36.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:36 smithi130 bash[27579]: cluster 2023-08-27T23:03:36.477717+0000 mgr.y (mgr.14901) 1952 : cluster [DBG] pgmap v1959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:36 smithi006 bash[33918]: cluster 2023-08-27T23:03:36.477717+0000 mgr.y (mgr.14901) 1952 : cluster [DBG] pgmap v1959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:36.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:36 smithi006 bash[27292]: cluster 2023-08-27T23:03:36.477717+0000 mgr.y (mgr.14901) 1952 : cluster [DBG] pgmap v1959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:37 smithi130 bash[51302]: debug 2023-08-27T23:03:37.063+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:37.065850+0000) 2023-08-27T23:03:37.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:37 smithi130 bash[27579]: audit 2023-08-27T23:03:36.900242+0000 mon.c (mon.2) 774 : audit [DBG] from='client.? 172.21.15.6:0/3260913480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:37.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:37 smithi006 bash[33918]: audit 2023-08-27T23:03:36.900242+0000 mon.c (mon.2) 774 : audit [DBG] from='client.? 172.21.15.6:0/3260913480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:37.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:37 smithi006 bash[27292]: audit 2023-08-27T23:03:36.900242+0000 mon.c (mon.2) 774 : audit [DBG] from='client.? 172.21.15.6:0/3260913480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:38.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:38 smithi130 bash[51302]: debug 2023-08-27T23:03:38.063+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:38.066157+0000) 2023-08-27T23:03:38.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:38 smithi130 bash[27579]: cluster 2023-08-27T23:03:38.478368+0000 mgr.y (mgr.14901) 1953 : cluster [DBG] pgmap v1960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:38.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:38 smithi006 bash[33918]: cluster 2023-08-27T23:03:38.478368+0000 mgr.y (mgr.14901) 1953 : cluster [DBG] pgmap v1960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:38.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:38 smithi006 bash[27292]: cluster 2023-08-27T23:03:38.478368+0000 mgr.y (mgr.14901) 1953 : cluster [DBG] pgmap v1960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:39.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:39 smithi130 bash[51302]: debug 2023-08-27T23:03:39.059+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:39.066494+0000) 2023-08-27T23:03:40.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:40 smithi130 bash[27579]: audit 2023-08-27T23:03:39.316230+0000 mon.a (mon.0) 1867 : audit [DBG] from='client.? 172.21.15.6:0/321638883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:40.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:40 smithi130 bash[51302]: debug 2023-08-27T23:03:40.059+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:40.066774+0000) 2023-08-27T23:03:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:40 smithi006 bash[33918]: audit 2023-08-27T23:03:39.316230+0000 mon.a (mon.0) 1867 : audit [DBG] from='client.? 172.21.15.6:0/321638883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:40 smithi006 bash[27292]: audit 2023-08-27T23:03:39.316230+0000 mon.a (mon.0) 1867 : audit [DBG] from='client.? 172.21.15.6:0/321638883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:40.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:03:40.365Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:40.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:03:40.367Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:41.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:03:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:03:40] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:03:41.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:41 smithi130 bash[51302]: debug 2023-08-27T23:03:41.062+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:41.067110+0000) 2023-08-27T23:03:41.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:41 smithi130 bash[27579]: cluster 2023-08-27T23:03:40.479500+0000 mgr.y (mgr.14901) 1954 : cluster [DBG] pgmap v1961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:41.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:41 smithi006 bash[27292]: cluster 2023-08-27T23:03:40.479500+0000 mgr.y (mgr.14901) 1954 : cluster [DBG] pgmap v1961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:41.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:41 smithi006 bash[33918]: cluster 2023-08-27T23:03:40.479500+0000 mgr.y (mgr.14901) 1954 : cluster [DBG] pgmap v1961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:42.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:41 smithi006 bash[56298]: level=error ts=2023-08-27T23:03:41.927Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:42.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:41 smithi006 bash[56298]: level=warn ts=2023-08-27T23:03:41.929Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:42.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:42 smithi130 bash[27579]: audit 2023-08-27T23:03:41.736452+0000 mon.a (mon.0) 1868 : audit [DBG] from='client.? 172.21.15.6:0/1247495945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:42.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:42 smithi130 bash[51302]: debug 2023-08-27T23:03:42.062+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:42.067423+0000) 2023-08-27T23:03:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:42 smithi006 bash[27292]: audit 2023-08-27T23:03:41.736452+0000 mon.a (mon.0) 1868 : audit [DBG] from='client.? 172.21.15.6:0/1247495945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:42.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:42 smithi006 bash[33918]: audit 2023-08-27T23:03:41.736452+0000 mon.a (mon.0) 1868 : audit [DBG] from='client.? 172.21.15.6:0/1247495945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:43.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:43 smithi130 bash[27579]: cluster 2023-08-27T23:03:42.480145+0000 mgr.y (mgr.14901) 1955 : cluster [DBG] pgmap v1962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:43.371 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:43 smithi130 bash[51302]: debug 2023-08-27T23:03:43.066+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:43.067691+0000) 2023-08-27T23:03:43.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:43 smithi006 bash[27292]: cluster 2023-08-27T23:03:42.480145+0000 mgr.y (mgr.14901) 1955 : cluster [DBG] pgmap v1962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:43.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:43 smithi006 bash[33918]: cluster 2023-08-27T23:03:42.480145+0000 mgr.y (mgr.14901) 1955 : cluster [DBG] pgmap v1962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:03:43.826Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:44.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:03:43.828Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:44.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:44 smithi130 bash[27579]: audit 2023-08-27T23:03:44.151336+0000 mon.c (mon.2) 775 : audit [DBG] from='client.? 172.21.15.6:0/1790330402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:44.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:44 smithi130 bash[51302]: debug 2023-08-27T23:03:44.066+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:44.068053+0000) 2023-08-27T23:03:44.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:44 smithi006 bash[27292]: audit 2023-08-27T23:03:44.151336+0000 mon.c (mon.2) 775 : audit [DBG] from='client.? 172.21.15.6:0/1790330402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:44.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:44 smithi006 bash[33918]: audit 2023-08-27T23:03:44.151336+0000 mon.c (mon.2) 775 : audit [DBG] from='client.? 172.21.15.6:0/1790330402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:45.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:45 smithi130 bash[51302]: debug 2023-08-27T23:03:45.066+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:45.068387+0000) 2023-08-27T23:03:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:45 smithi130 bash[27579]: cluster 2023-08-27T23:03:44.481176+0000 mgr.y (mgr.14901) 1956 : cluster [DBG] pgmap v1963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:45.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:45 smithi006 bash[33918]: cluster 2023-08-27T23:03:44.481176+0000 mgr.y (mgr.14901) 1956 : cluster [DBG] pgmap v1963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:45.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:45 smithi006 bash[27292]: cluster 2023-08-27T23:03:44.481176+0000 mgr.y (mgr.14901) 1956 : cluster [DBG] pgmap v1963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:46 smithi130 bash[51302]: debug 2023-08-27T23:03:46.062+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:46.068718+0000) 2023-08-27T23:03:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:46 smithi130 bash[27579]: cluster 2023-08-27T23:03:46.481731+0000 mgr.y (mgr.14901) 1957 : cluster [DBG] pgmap v1964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:46 smithi006 bash[27292]: cluster 2023-08-27T23:03:46.481731+0000 mgr.y (mgr.14901) 1957 : cluster [DBG] pgmap v1964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:46.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:46 smithi006 bash[33918]: cluster 2023-08-27T23:03:46.481731+0000 mgr.y (mgr.14901) 1957 : cluster [DBG] pgmap v1964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:47 smithi130 bash[51302]: debug 2023-08-27T23:03:47.066+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:47.069073+0000) 2023-08-27T23:03:47.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:47 smithi130 bash[27579]: audit 2023-08-27T23:03:46.573240+0000 mon.a (mon.0) 1869 : audit [DBG] from='client.? 172.21.15.6:0/2691114509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:47.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:47 smithi006 bash[27292]: audit 2023-08-27T23:03:46.573240+0000 mon.a (mon.0) 1869 : audit [DBG] from='client.? 172.21.15.6:0/2691114509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:47.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:47 smithi006 bash[33918]: audit 2023-08-27T23:03:46.573240+0000 mon.a (mon.0) 1869 : audit [DBG] from='client.? 172.21.15.6:0/2691114509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:48.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:48 smithi130 bash[51302]: debug 2023-08-27T23:03:48.066+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:48.069431+0000) 2023-08-27T23:03:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:48 smithi130 bash[27579]: cluster 2023-08-27T23:03:48.482333+0000 mgr.y (mgr.14901) 1958 : cluster [DBG] pgmap v1965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:48 smithi006 bash[27292]: cluster 2023-08-27T23:03:48.482333+0000 mgr.y (mgr.14901) 1958 : cluster [DBG] pgmap v1965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:48.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:48 smithi006 bash[33918]: cluster 2023-08-27T23:03:48.482333+0000 mgr.y (mgr.14901) 1958 : cluster [DBG] pgmap v1965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:49.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:49 smithi130 bash[51302]: debug 2023-08-27T23:03:49.066+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:49.069785+0000) 2023-08-27T23:03:49.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:49 smithi130 bash[27579]: audit 2023-08-27T23:03:48.995475+0000 mon.c (mon.2) 776 : audit [DBG] from='client.? 172.21.15.6:0/698164077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:49.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:49 smithi006 bash[33918]: audit 2023-08-27T23:03:48.995475+0000 mon.c (mon.2) 776 : audit [DBG] from='client.? 172.21.15.6:0/698164077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:49.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:49 smithi006 bash[27292]: audit 2023-08-27T23:03:48.995475+0000 mon.c (mon.2) 776 : audit [DBG] from='client.? 172.21.15.6:0/698164077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:50.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:50 smithi130 bash[51302]: debug 2023-08-27T23:03:50.066+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:50.070053+0000) 2023-08-27T23:03:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:50 smithi006 bash[33918]: cluster 2023-08-27T23:03:50.483380+0000 mgr.y (mgr.14901) 1959 : cluster [DBG] pgmap v1966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:50.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:03:50.366Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:50.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:03:50.367Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:50 smithi006 bash[27292]: cluster 2023-08-27T23:03:50.483380+0000 mgr.y (mgr.14901) 1959 : cluster [DBG] pgmap v1966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:50 smithi130 bash[27579]: cluster 2023-08-27T23:03:50.483380+0000 mgr.y (mgr.14901) 1959 : cluster [DBG] pgmap v1966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:03:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:03:50] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:03:51.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:51 smithi130 bash[51302]: debug 2023-08-27T23:03:51.066+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:51.070390+0000) 2023-08-27T23:03:51.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:51 smithi130 bash[27579]: audit 2023-08-27T23:03:51.408297+0000 mon.a (mon.0) 1870 : audit [DBG] from='client.? 172.21.15.6:0/4190755496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:51.931 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:51 smithi006 bash[33918]: audit 2023-08-27T23:03:51.408297+0000 mon.a (mon.0) 1870 : audit [DBG] from='client.? 172.21.15.6:0/4190755496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:51.931 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:51 smithi006 bash[27292]: audit 2023-08-27T23:03:51.408297+0000 mon.a (mon.0) 1870 : audit [DBG] from='client.? 172.21.15.6:0/4190755496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:52.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:51 smithi006 bash[56298]: level=error ts=2023-08-27T23:03:51.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:52.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:51 smithi006 bash[56298]: level=warn ts=2023-08-27T23:03:51.929Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:52.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:52 smithi130 bash[51302]: debug 2023-08-27T23:03:52.066+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:52.070743+0000) 2023-08-27T23:03:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:52 smithi130 bash[27579]: cluster 2023-08-27T23:03:52.484012+0000 mgr.y (mgr.14901) 1960 : cluster [DBG] pgmap v1967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:52 smithi006 bash[27292]: cluster 2023-08-27T23:03:52.484012+0000 mgr.y (mgr.14901) 1960 : cluster [DBG] pgmap v1967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:52.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:52 smithi006 bash[33918]: cluster 2023-08-27T23:03:52.484012+0000 mgr.y (mgr.14901) 1960 : cluster [DBG] pgmap v1967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:53.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:53 smithi130 bash[51302]: debug 2023-08-27T23:03:53.070+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:53.071120+0000) 2023-08-27T23:03:54.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:53 smithi006 bash[27292]: audit 2023-08-27T23:03:53.821411+0000 mon.a (mon.0) 1871 : audit [DBG] from='client.? 172.21.15.6:0/745013130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:54.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:53 smithi006 bash[33918]: audit 2023-08-27T23:03:53.821411+0000 mon.a (mon.0) 1871 : audit [DBG] from='client.? 172.21.15.6:0/745013130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:54.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:03:53.826Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:54.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:03:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:03:53.828Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:03:54.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:53 smithi130 bash[27579]: audit 2023-08-27T23:03:53.821411+0000 mon.a (mon.0) 1871 : audit [DBG] from='client.? 172.21.15.6:0/745013130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:54 smithi130 bash[51302]: debug 2023-08-27T23:03:54.070+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:54.071391+0000) 2023-08-27T23:03:55.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:55 smithi130 bash[27579]: cluster 2023-08-27T23:03:54.485153+0000 mgr.y (mgr.14901) 1961 : cluster [DBG] pgmap v1968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:55 smithi130 bash[51302]: debug 2023-08-27T23:03:55.070+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:55.071693+0000) 2023-08-27T23:03:55.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:55 smithi006 bash[33918]: cluster 2023-08-27T23:03:54.485153+0000 mgr.y (mgr.14901) 1961 : cluster [DBG] pgmap v1968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:55.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:55 smithi006 bash[27292]: cluster 2023-08-27T23:03:54.485153+0000 mgr.y (mgr.14901) 1961 : cluster [DBG] pgmap v1968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:56.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:56 smithi130 bash[27579]: audit 2023-08-27T23:03:56.240632+0000 mon.a (mon.0) 1872 : audit [DBG] from='client.? 172.21.15.6:0/2362214489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:56.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:56 smithi130 bash[51302]: debug 2023-08-27T23:03:56.070+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:56.072035+0000) 2023-08-27T23:03:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:56 smithi006 bash[33918]: audit 2023-08-27T23:03:56.240632+0000 mon.a (mon.0) 1872 : audit [DBG] from='client.? 172.21.15.6:0/2362214489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:56 smithi006 bash[27292]: audit 2023-08-27T23:03:56.240632+0000 mon.a (mon.0) 1872 : audit [DBG] from='client.? 172.21.15.6:0/2362214489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:57.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:57 smithi130 bash[27579]: cluster 2023-08-27T23:03:56.485731+0000 mgr.y (mgr.14901) 1962 : cluster [DBG] pgmap v1969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:57.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:57 smithi130 bash[51302]: debug 2023-08-27T23:03:57.070+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:57.072381+0000) 2023-08-27T23:03:57.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:57 smithi006 bash[33918]: cluster 2023-08-27T23:03:56.485731+0000 mgr.y (mgr.14901) 1962 : cluster [DBG] pgmap v1969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:57.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:57 smithi006 bash[27292]: cluster 2023-08-27T23:03:56.485731+0000 mgr.y (mgr.14901) 1962 : cluster [DBG] pgmap v1969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:58.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:58 smithi130 bash[51302]: debug 2023-08-27T23:03:58.070+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:58.072708+0000) 2023-08-27T23:03:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:58 smithi130 bash[27579]: cluster 2023-08-27T23:03:58.486356+0000 mgr.y (mgr.14901) 1963 : cluster [DBG] pgmap v1970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:58 smithi006 bash[33918]: cluster 2023-08-27T23:03:58.486356+0000 mgr.y (mgr.14901) 1963 : cluster [DBG] pgmap v1970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:58.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:58 smithi006 bash[27292]: cluster 2023-08-27T23:03:58.486356+0000 mgr.y (mgr.14901) 1963 : cluster [DBG] pgmap v1970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:03:59.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:03:59 smithi130 bash[51302]: debug 2023-08-27T23:03:59.070+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:59.073089+0000) 2023-08-27T23:03:59.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:03:59 smithi130 bash[27579]: audit 2023-08-27T23:03:58.659270+0000 mon.a (mon.0) 1873 : audit [DBG] from='client.? 172.21.15.6:0/1614224585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:03:59 smithi006 bash[33918]: audit 2023-08-27T23:03:58.659270+0000 mon.a (mon.0) 1873 : audit [DBG] from='client.? 172.21.15.6:0/1614224585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:59.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:03:59 smithi006 bash[27292]: audit 2023-08-27T23:03:58.659270+0000 mon.a (mon.0) 1873 : audit [DBG] from='client.? 172.21.15.6:0/1614224585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:00.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:00 smithi130 bash[51302]: debug 2023-08-27T23:04:00.070+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:00.073406+0000) 2023-08-27T23:04:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:00 smithi006 bash[33918]: cluster 2023-08-27T23:04:00.487500+0000 mgr.y (mgr.14901) 1964 : cluster [DBG] pgmap v1971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:00 smithi006 bash[27292]: cluster 2023-08-27T23:04:00.487500+0000 mgr.y (mgr.14901) 1964 : cluster [DBG] pgmap v1971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:00.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:04:00.366Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:00.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:04:00.368Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:00 smithi130 bash[27579]: cluster 2023-08-27T23:04:00.487500+0000 mgr.y (mgr.14901) 1964 : cluster [DBG] pgmap v1971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:04:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:04:00] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:04:01.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:01 smithi130 bash[51302]: debug 2023-08-27T23:04:01.070+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:01.073731+0000) 2023-08-27T23:04:01.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:01 smithi130 bash[27579]: audit 2023-08-27T23:04:01.076795+0000 mon.a (mon.0) 1874 : audit [DBG] from='client.? 172.21.15.6:0/2020454464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:01.932 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:01 smithi006 bash[33918]: audit 2023-08-27T23:04:01.076795+0000 mon.a (mon.0) 1874 : audit [DBG] from='client.? 172.21.15.6:0/2020454464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:01.932 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:01 smithi006 bash[27292]: audit 2023-08-27T23:04:01.076795+0000 mon.a (mon.0) 1874 : audit [DBG] from='client.? 172.21.15.6:0/2020454464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:02.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:01 smithi006 bash[56298]: level=error ts=2023-08-27T23:04:01.929Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:02.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:01 smithi006 bash[56298]: level=warn ts=2023-08-27T23:04:01.931Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:02.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:02 smithi130 bash[51302]: debug 2023-08-27T23:04:02.070+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:02.074050+0000) 2023-08-27T23:04:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:02 smithi130 bash[27579]: cluster 2023-08-27T23:04:02.488189+0000 mgr.y (mgr.14901) 1965 : cluster [DBG] pgmap v1972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:02 smithi006 bash[27292]: cluster 2023-08-27T23:04:02.488189+0000 mgr.y (mgr.14901) 1965 : cluster [DBG] pgmap v1972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:02.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:02 smithi006 bash[33918]: cluster 2023-08-27T23:04:02.488189+0000 mgr.y (mgr.14901) 1965 : cluster [DBG] pgmap v1972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:03.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:03 smithi130 bash[51302]: debug 2023-08-27T23:04:03.070+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:03.074388+0000) 2023-08-27T23:04:03.830 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:03 smithi006 bash[27292]: audit 2023-08-27T23:04:03.485881+0000 mon.c (mon.2) 777 : audit [DBG] from='client.? 172.21.15.6:0/2058181003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:03.831 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:03 smithi006 bash[33918]: audit 2023-08-27T23:04:03.485881+0000 mon.c (mon.2) 777 : audit [DBG] from='client.? 172.21.15.6:0/2058181003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:03.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:03 smithi130 bash[27579]: audit 2023-08-27T23:04:03.485881+0000 mon.c (mon.2) 777 : audit [DBG] from='client.? 172.21.15.6:0/2058181003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:04:03.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:04:03.830Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:04.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:04 smithi130 bash[51302]: debug 2023-08-27T23:04:04.074+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:04.074801+0000) 2023-08-27T23:04:05.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:05 smithi130 bash[51302]: debug 2023-08-27T23:04:05.074+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:05.075132+0000) 2023-08-27T23:04:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:05 smithi130 bash[27579]: cluster 2023-08-27T23:04:04.489498+0000 mgr.y (mgr.14901) 1966 : cluster [DBG] pgmap v1973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:05.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:05 smithi006 bash[27292]: cluster 2023-08-27T23:04:04.489498+0000 mgr.y (mgr.14901) 1966 : cluster [DBG] pgmap v1973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:05.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:05 smithi006 bash[33918]: cluster 2023-08-27T23:04:04.489498+0000 mgr.y (mgr.14901) 1966 : cluster [DBG] pgmap v1973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:06 smithi130 bash[27579]: audit 2023-08-27T23:04:05.896106+0000 mon.a (mon.0) 1875 : audit [DBG] from='client.? 172.21.15.6:0/2238013856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:06.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:06 smithi130 bash[51302]: debug 2023-08-27T23:04:06.074+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:06.075410+0000) 2023-08-27T23:04:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:06 smithi006 bash[33918]: audit 2023-08-27T23:04:05.896106+0000 mon.a (mon.0) 1875 : audit [DBG] from='client.? 172.21.15.6:0/2238013856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:06 smithi006 bash[27292]: audit 2023-08-27T23:04:05.896106+0000 mon.a (mon.0) 1875 : audit [DBG] from='client.? 172.21.15.6:0/2238013856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:07.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:07 smithi130 bash[27579]: cluster 2023-08-27T23:04:06.490199+0000 mgr.y (mgr.14901) 1967 : cluster [DBG] pgmap v1974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:07.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:07 smithi130 bash[51302]: debug 2023-08-27T23:04:07.074+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:07.075669+0000) 2023-08-27T23:04:07.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:07 smithi006 bash[27292]: cluster 2023-08-27T23:04:06.490199+0000 mgr.y (mgr.14901) 1967 : cluster [DBG] pgmap v1974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:07.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:07 smithi006 bash[33918]: cluster 2023-08-27T23:04:06.490199+0000 mgr.y (mgr.14901) 1967 : cluster [DBG] pgmap v1974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:08.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:08 smithi130 bash[51302]: debug 2023-08-27T23:04:08.074+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:08.076042+0000) 2023-08-27T23:04:08.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:08 smithi130 bash[27579]: audit 2023-08-27T23:04:08.309131+0000 mon.a (mon.0) 1876 : audit [DBG] from='client.? 172.21.15.6:0/2371657829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:08 smithi006 bash[27292]: audit 2023-08-27T23:04:08.309131+0000 mon.a (mon.0) 1876 : audit [DBG] from='client.? 172.21.15.6:0/2371657829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:08 smithi006 bash[33918]: audit 2023-08-27T23:04:08.309131+0000 mon.a (mon.0) 1876 : audit [DBG] from='client.? 172.21.15.6:0/2371657829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:09.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:09 smithi130 bash[51302]: debug 2023-08-27T23:04:09.074+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:09.076383+0000) 2023-08-27T23:04:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:09 smithi006 bash[27292]: cluster 2023-08-27T23:04:08.490892+0000 mgr.y (mgr.14901) 1968 : cluster [DBG] pgmap v1975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:09.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:09 smithi006 bash[33918]: cluster 2023-08-27T23:04:08.490892+0000 mgr.y (mgr.14901) 1968 : cluster [DBG] pgmap v1975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:09.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:09 smithi130 bash[27579]: cluster 2023-08-27T23:04:08.490892+0000 mgr.y (mgr.14901) 1968 : cluster [DBG] pgmap v1975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:10.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:10 smithi130 bash[51302]: debug 2023-08-27T23:04:10.074+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:10.076741+0000) 2023-08-27T23:04:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:10 smithi006 bash[33918]: cluster 2023-08-27T23:04:10.491927+0000 mgr.y (mgr.14901) 1969 : cluster [DBG] pgmap v1976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:10 smithi006 bash[27292]: cluster 2023-08-27T23:04:10.491927+0000 mgr.y (mgr.14901) 1969 : cluster [DBG] pgmap v1976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:10.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:04:10.367Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:10.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:04:10.369Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:10 smithi130 bash[27579]: cluster 2023-08-27T23:04:10.491927+0000 mgr.y (mgr.14901) 1969 : cluster [DBG] pgmap v1976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:04:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:04:10] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:04:11.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:11 smithi130 bash[51302]: debug 2023-08-27T23:04:11.074+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:11.077113+0000) 2023-08-27T23:04:11.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:11 smithi130 bash[27579]: audit 2023-08-27T23:04:10.727057+0000 mon.c (mon.2) 778 : audit [DBG] from='client.? 172.21.15.6:0/865677970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:11.932 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:11 smithi006 bash[27292]: audit 2023-08-27T23:04:10.727057+0000 mon.c (mon.2) 778 : audit [DBG] from='client.? 172.21.15.6:0/865677970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:11.933 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:11 smithi006 bash[33918]: audit 2023-08-27T23:04:10.727057+0000 mon.c (mon.2) 778 : audit [DBG] from='client.? 172.21.15.6:0/865677970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:12.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:11 smithi006 bash[56298]: level=error ts=2023-08-27T23:04:11.929Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:12.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:11 smithi006 bash[56298]: level=warn ts=2023-08-27T23:04:11.932Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:12 smithi130 bash[51302]: debug 2023-08-27T23:04:12.074+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:12.077426+0000) 2023-08-27T23:04:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:12 smithi130 bash[27579]: cluster 2023-08-27T23:04:12.492617+0000 mgr.y (mgr.14901) 1970 : cluster [DBG] pgmap v1977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:12 smithi006 bash[33918]: cluster 2023-08-27T23:04:12.492617+0000 mgr.y (mgr.14901) 1970 : cluster [DBG] pgmap v1977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:12 smithi006 bash[27292]: cluster 2023-08-27T23:04:12.492617+0000 mgr.y (mgr.14901) 1970 : cluster [DBG] pgmap v1977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:13.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:13 smithi130 bash[51302]: debug 2023-08-27T23:04:13.074+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:13.077793+0000) 2023-08-27T23:04:13.831 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:13 smithi006 bash[27292]: audit 2023-08-27T23:04:13.136419+0000 mon.c (mon.2) 779 : audit [DBG] from='client.? 172.21.15.6:0/796945817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:13.832 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:13 smithi006 bash[33918]: audit 2023-08-27T23:04:13.136419+0000 mon.c (mon.2) 779 : audit [DBG] from='client.? 172.21.15.6:0/796945817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:13.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:13 smithi130 bash[27579]: audit 2023-08-27T23:04:13.136419+0000 mon.c (mon.2) 779 : audit [DBG] from='client.? 172.21.15.6:0/796945817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:04:13.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:14.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:04:13.831Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:14.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:14 smithi130 bash[51302]: debug 2023-08-27T23:04:14.074+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:14.078125+0000) 2023-08-27T23:04:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:15 smithi130 bash[27579]: cluster 2023-08-27T23:04:14.493766+0000 mgr.y (mgr.14901) 1971 : cluster [DBG] pgmap v1978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:15 smithi130 bash[51302]: debug 2023-08-27T23:04:15.074+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:15.078490+0000) 2023-08-27T23:04:15.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:15 smithi006 bash[27292]: cluster 2023-08-27T23:04:14.493766+0000 mgr.y (mgr.14901) 1971 : cluster [DBG] pgmap v1978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:15.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:15 smithi006 bash[33918]: cluster 2023-08-27T23:04:14.493766+0000 mgr.y (mgr.14901) 1971 : cluster [DBG] pgmap v1978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:16.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:16 smithi130 bash[51302]: debug 2023-08-27T23:04:16.078+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:16.078823+0000) 2023-08-27T23:04:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:16 smithi130 bash[27579]: audit 2023-08-27T23:04:15.544159+0000 mon.c (mon.2) 780 : audit [DBG] from='client.? 172.21.15.6:0/4257052596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:16 smithi006 bash[27292]: audit 2023-08-27T23:04:15.544159+0000 mon.c (mon.2) 780 : audit [DBG] from='client.? 172.21.15.6:0/4257052596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:16 smithi006 bash[33918]: audit 2023-08-27T23:04:15.544159+0000 mon.c (mon.2) 780 : audit [DBG] from='client.? 172.21.15.6:0/4257052596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:17.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:17 smithi130 bash[27579]: cluster 2023-08-27T23:04:16.494434+0000 mgr.y (mgr.14901) 1972 : cluster [DBG] pgmap v1979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:17.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:17 smithi130 bash[51302]: debug 2023-08-27T23:04:17.078+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:17.079200+0000) 2023-08-27T23:04:17.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:17 smithi006 bash[27292]: cluster 2023-08-27T23:04:16.494434+0000 mgr.y (mgr.14901) 1972 : cluster [DBG] pgmap v1979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:17.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:17 smithi006 bash[33918]: cluster 2023-08-27T23:04:16.494434+0000 mgr.y (mgr.14901) 1972 : cluster [DBG] pgmap v1979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:18.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:18 smithi130 bash[51302]: debug 2023-08-27T23:04:18.078+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:18.079509+0000) 2023-08-27T23:04:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:18 smithi130 bash[27579]: audit 2023-08-27T23:04:17.956955+0000 mon.c (mon.2) 781 : audit [DBG] from='client.? 172.21.15.6:0/2769089976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:18 smithi006 bash[33918]: audit 2023-08-27T23:04:17.956955+0000 mon.c (mon.2) 781 : audit [DBG] from='client.? 172.21.15.6:0/2769089976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:18 smithi006 bash[27292]: audit 2023-08-27T23:04:17.956955+0000 mon.c (mon.2) 781 : audit [DBG] from='client.? 172.21.15.6:0/2769089976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:19.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:19 smithi130 bash[27579]: cluster 2023-08-27T23:04:18.495102+0000 mgr.y (mgr.14901) 1973 : cluster [DBG] pgmap v1980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:19.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:19 smithi130 bash[51302]: debug 2023-08-27T23:04:19.078+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:19.079873+0000) 2023-08-27T23:04:19.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:19 smithi006 bash[27292]: cluster 2023-08-27T23:04:18.495102+0000 mgr.y (mgr.14901) 1973 : cluster [DBG] pgmap v1980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:19.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:19 smithi006 bash[33918]: cluster 2023-08-27T23:04:18.495102+0000 mgr.y (mgr.14901) 1973 : cluster [DBG] pgmap v1980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:20.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:20 smithi130 bash[51302]: debug 2023-08-27T23:04:20.078+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:20.080192+0000) 2023-08-27T23:04:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:20 smithi006 bash[27292]: audit 2023-08-27T23:04:20.368810+0000 mon.a (mon.0) 1877 : audit [DBG] from='client.? 172.21.15.6:0/510041890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:20.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:20 smithi006 bash[33918]: audit 2023-08-27T23:04:20.368810+0000 mon.a (mon.0) 1877 : audit [DBG] from='client.? 172.21.15.6:0/510041890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:20.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:04:20.368Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:20.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:04:20.370Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:20.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:20 smithi130 bash[27579]: audit 2023-08-27T23:04:20.368810+0000 mon.a (mon.0) 1877 : audit [DBG] from='client.? 172.21.15.6:0/510041890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:04:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:04:20] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:04:21.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:21 smithi130 bash[51302]: debug 2023-08-27T23:04:21.078+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:21.080556+0000) 2023-08-27T23:04:21.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:21 smithi006 bash[33918]: cluster 2023-08-27T23:04:20.496165+0000 mgr.y (mgr.14901) 1974 : cluster [DBG] pgmap v1981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:21.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:21 smithi006 bash[27292]: cluster 2023-08-27T23:04:20.496165+0000 mgr.y (mgr.14901) 1974 : cluster [DBG] pgmap v1981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:21.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:21 smithi130 bash[27579]: cluster 2023-08-27T23:04:20.496165+0000 mgr.y (mgr.14901) 1974 : cluster [DBG] pgmap v1981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:22.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:21 smithi006 bash[56298]: level=error ts=2023-08-27T23:04:21.930Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:22.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:21 smithi006 bash[56298]: level=warn ts=2023-08-27T23:04:21.931Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:22.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:22 smithi130 bash[51302]: debug 2023-08-27T23:04:22.078+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:22.080893+0000) 2023-08-27T23:04:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:22 smithi130 bash[27579]: cluster 2023-08-27T23:04:22.496770+0000 mgr.y (mgr.14901) 1975 : cluster [DBG] pgmap v1982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:22 smithi006 bash[33918]: cluster 2023-08-27T23:04:22.496770+0000 mgr.y (mgr.14901) 1975 : cluster [DBG] pgmap v1982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:22 smithi006 bash[27292]: cluster 2023-08-27T23:04:22.496770+0000 mgr.y (mgr.14901) 1975 : cluster [DBG] pgmap v1982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:23.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:23 smithi130 bash[51302]: debug 2023-08-27T23:04:23.078+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:23.081238+0000) 2023-08-27T23:04:23.832 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:23 smithi006 bash[33918]: audit 2023-08-27T23:04:22.783773+0000 mon.c (mon.2) 782 : audit [DBG] from='client.? 172.21.15.6:0/2242248552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:23.833 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:23 smithi006 bash[27292]: audit 2023-08-27T23:04:22.783773+0000 mon.c (mon.2) 782 : audit [DBG] from='client.? 172.21.15.6:0/2242248552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:23.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:23 smithi130 bash[27579]: audit 2023-08-27T23:04:22.783773+0000 mon.c (mon.2) 782 : audit [DBG] from='client.? 172.21.15.6:0/2242248552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:04:23.829Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:24.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:04:23.832Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:24.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:24 smithi130 bash[51302]: debug 2023-08-27T23:04:24.078+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:24.081575+0000) 2023-08-27T23:04:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:25 smithi130 bash[51302]: debug 2023-08-27T23:04:25.078+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:25.081909+0000) 2023-08-27T23:04:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:25 smithi130 bash[27579]: cluster 2023-08-27T23:04:24.497892+0000 mgr.y (mgr.14901) 1976 : cluster [DBG] pgmap v1983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:25 smithi130 bash[27579]: audit 2023-08-27T23:04:25.195615+0000 mon.c (mon.2) 783 : audit [DBG] from='client.? 172.21.15.6:0/792690684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:25.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:25 smithi006 bash[33918]: cluster 2023-08-27T23:04:24.497892+0000 mgr.y (mgr.14901) 1976 : cluster [DBG] pgmap v1983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:25.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:25 smithi006 bash[33918]: audit 2023-08-27T23:04:25.195615+0000 mon.c (mon.2) 783 : audit [DBG] from='client.? 172.21.15.6:0/792690684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:25.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:25 smithi006 bash[27292]: cluster 2023-08-27T23:04:24.497892+0000 mgr.y (mgr.14901) 1976 : cluster [DBG] pgmap v1983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:25.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:25 smithi006 bash[27292]: audit 2023-08-27T23:04:25.195615+0000 mon.c (mon.2) 783 : audit [DBG] from='client.? 172.21.15.6:0/792690684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:26.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:26 smithi130 bash[51302]: debug 2023-08-27T23:04:26.078+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:26.082229+0000) 2023-08-27T23:04:26.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:26 smithi130 bash[27579]: cluster 2023-08-27T23:04:26.498525+0000 mgr.y (mgr.14901) 1977 : cluster [DBG] pgmap v1984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:26 smithi006 bash[33918]: cluster 2023-08-27T23:04:26.498525+0000 mgr.y (mgr.14901) 1977 : cluster [DBG] pgmap v1984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:26.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:26 smithi006 bash[27292]: cluster 2023-08-27T23:04:26.498525+0000 mgr.y (mgr.14901) 1977 : cluster [DBG] pgmap v1984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:27.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:27 smithi130 bash[51302]: debug 2023-08-27T23:04:27.078+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:27.082586+0000) 2023-08-27T23:04:27.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:27 smithi006 bash[27292]: audit 2023-08-27T23:04:27.606062+0000 mon.a (mon.0) 1878 : audit [DBG] from='client.? 172.21.15.6:0/1617800258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:27.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:27 smithi006 bash[33918]: audit 2023-08-27T23:04:27.606062+0000 mon.a (mon.0) 1878 : audit [DBG] from='client.? 172.21.15.6:0/1617800258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:28.084 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:27 smithi130 bash[27579]: audit 2023-08-27T23:04:27.606062+0000 mon.a (mon.0) 1878 : audit [DBG] from='client.? 172.21.15.6:0/1617800258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:28.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:28 smithi130 bash[51302]: debug 2023-08-27T23:04:28.082+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:28.082934+0000) 2023-08-27T23:04:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:28 smithi006 bash[27292]: cluster 2023-08-27T23:04:28.499156+0000 mgr.y (mgr.14901) 1978 : cluster [DBG] pgmap v1985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:28 smithi006 bash[33918]: cluster 2023-08-27T23:04:28.499156+0000 mgr.y (mgr.14901) 1978 : cluster [DBG] pgmap v1985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:29.084 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:28 smithi130 bash[27579]: cluster 2023-08-27T23:04:28.499156+0000 mgr.y (mgr.14901) 1978 : cluster [DBG] pgmap v1985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:29.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:29 smithi130 bash[51302]: debug 2023-08-27T23:04:29.082+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:29.083260+0000) 2023-08-27T23:04:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:30 smithi130 bash[27579]: audit 2023-08-27T23:04:30.026777+0000 mon.c (mon.2) 784 : audit [DBG] from='client.? 172.21.15.6:0/2973187233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:30 smithi130 bash[51302]: debug 2023-08-27T23:04:30.082+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:30.083545+0000) 2023-08-27T23:04:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:30 smithi006 bash[33918]: audit 2023-08-27T23:04:30.026777+0000 mon.c (mon.2) 784 : audit [DBG] from='client.? 172.21.15.6:0/2973187233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:30 smithi006 bash[27292]: audit 2023-08-27T23:04:30.026777+0000 mon.c (mon.2) 784 : audit [DBG] from='client.? 172.21.15.6:0/2973187233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:30.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:04:30.369Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:30.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:04:30.371Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:04:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:04:30] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:04:31.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:31 smithi130 bash[27579]: cluster 2023-08-27T23:04:30.500339+0000 mgr.y (mgr.14901) 1979 : cluster [DBG] pgmap v1986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:31.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:31 smithi130 bash[51302]: debug 2023-08-27T23:04:31.082+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:31.083930+0000) 2023-08-27T23:04:31.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:31 smithi006 bash[33918]: cluster 2023-08-27T23:04:30.500339+0000 mgr.y (mgr.14901) 1979 : cluster [DBG] pgmap v1986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:31.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:31 smithi006 bash[27292]: cluster 2023-08-27T23:04:30.500339+0000 mgr.y (mgr.14901) 1979 : cluster [DBG] pgmap v1986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:32.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:31 smithi006 bash[56298]: level=error ts=2023-08-27T23:04:31.931Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:32.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:31 smithi006 bash[56298]: level=warn ts=2023-08-27T23:04:31.933Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:32.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:32 smithi130 bash[51302]: debug 2023-08-27T23:04:32.082+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:32.084256+0000) 2023-08-27T23:04:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:32 smithi130 bash[27579]: audit 2023-08-27T23:04:32.439353+0000 mon.a (mon.0) 1879 : audit [DBG] from='client.? 172.21.15.6:0/3790160282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:32.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:32 smithi006 bash[27292]: audit 2023-08-27T23:04:32.439353+0000 mon.a (mon.0) 1879 : audit [DBG] from='client.? 172.21.15.6:0/3790160282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:32.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:32 smithi006 bash[33918]: audit 2023-08-27T23:04:32.439353+0000 mon.a (mon.0) 1879 : audit [DBG] from='client.? 172.21.15.6:0/3790160282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:33.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:33 smithi130 bash[51302]: debug 2023-08-27T23:04:33.082+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:33.084586+0000) 2023-08-27T23:04:33.833 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:33 smithi006 bash[33918]: cluster 2023-08-27T23:04:32.500883+0000 mgr.y (mgr.14901) 1980 : cluster [DBG] pgmap v1987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:33.833 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:33 smithi006 bash[33918]: audit 2023-08-27T23:04:32.794082+0000 mon.b (mon.1) 311 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:04:33.834 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:33 smithi006 bash[27292]: cluster 2023-08-27T23:04:32.500883+0000 mgr.y (mgr.14901) 1980 : cluster [DBG] pgmap v1987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:33.834 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:33 smithi006 bash[27292]: audit 2023-08-27T23:04:32.794082+0000 mon.b (mon.1) 311 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:04:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:33 smithi130 bash[27579]: cluster 2023-08-27T23:04:32.500883+0000 mgr.y (mgr.14901) 1980 : cluster [DBG] pgmap v1987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:33.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:33 smithi130 bash[27579]: audit 2023-08-27T23:04:32.794082+0000 mon.b (mon.1) 311 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:04:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:04:33.830Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:04:33.833Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:34.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:34 smithi130 bash[51302]: debug 2023-08-27T23:04:34.082+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:34.084947+0000) 2023-08-27T23:04:35.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:35 smithi130 bash[51302]: debug 2023-08-27T23:04:35.082+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:35.085257+0000) 2023-08-27T23:04:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:35 smithi130 bash[27579]: cluster 2023-08-27T23:04:34.501935+0000 mgr.y (mgr.14901) 1981 : cluster [DBG] pgmap v1988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:35 smithi130 bash[27579]: audit 2023-08-27T23:04:34.854883+0000 mon.c (mon.2) 785 : audit [DBG] from='client.? 172.21.15.6:0/2473573524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:35.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:35 smithi006 bash[33918]: cluster 2023-08-27T23:04:34.501935+0000 mgr.y (mgr.14901) 1981 : cluster [DBG] pgmap v1988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:35.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:35 smithi006 bash[33918]: audit 2023-08-27T23:04:34.854883+0000 mon.c (mon.2) 785 : audit [DBG] from='client.? 172.21.15.6:0/2473573524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:35.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:35 smithi006 bash[27292]: cluster 2023-08-27T23:04:34.501935+0000 mgr.y (mgr.14901) 1981 : cluster [DBG] pgmap v1988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:35.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:35 smithi006 bash[27292]: audit 2023-08-27T23:04:34.854883+0000 mon.c (mon.2) 785 : audit [DBG] from='client.? 172.21.15.6:0/2473573524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:36.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:36 smithi130 bash[51302]: debug 2023-08-27T23:04:36.081+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:36.085494+0000) 2023-08-27T23:04:36.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:36 smithi130 bash[27579]: cluster 2023-08-27T23:04:36.502567+0000 mgr.y (mgr.14901) 1982 : cluster [DBG] pgmap v1989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:36 smithi006 bash[27292]: cluster 2023-08-27T23:04:36.502567+0000 mgr.y (mgr.14901) 1982 : cluster [DBG] pgmap v1989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:36 smithi006 bash[33918]: cluster 2023-08-27T23:04:36.502567+0000 mgr.y (mgr.14901) 1982 : cluster [DBG] pgmap v1989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:37.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:37 smithi130 bash[51302]: debug 2023-08-27T23:04:37.081+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:37.085751+0000) 2023-08-27T23:04:37.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:37 smithi130 bash[27579]: audit 2023-08-27T23:04:37.287168+0000 mon.c (mon.2) 786 : audit [DBG] from='client.? 172.21.15.6:0/423296647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:37.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:37 smithi006 bash[33918]: audit 2023-08-27T23:04:37.287168+0000 mon.c (mon.2) 786 : audit [DBG] from='client.? 172.21.15.6:0/423296647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:37.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:37 smithi006 bash[27292]: audit 2023-08-27T23:04:37.287168+0000 mon.c (mon.2) 786 : audit [DBG] from='client.? 172.21.15.6:0/423296647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:38.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:38 smithi130 bash[51302]: debug 2023-08-27T23:04:38.081+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:38.086044+0000) 2023-08-27T23:04:39.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:39 smithi130 bash[51302]: debug 2023-08-27T23:04:39.081+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:39.086305+0000) 2023-08-27T23:04:40.087 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:39 smithi130 bash[27579]: cluster 2023-08-27T23:04:38.503335+0000 mgr.y (mgr.14901) 1983 : cluster [DBG] pgmap v1990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:40.088 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:39 smithi130 bash[27579]: audit 2023-08-27T23:04:38.729037+0000 mon.a (mon.0) 1880 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:04:40.088 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:39 smithi130 bash[27579]: audit 2023-08-27T23:04:39.107910+0000 mon.a (mon.0) 1881 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:04:40.088 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:39 smithi130 bash[27579]: audit 2023-08-27T23:04:39.378562+0000 mon.b (mon.1) 312 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:04:40.088 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:39 smithi130 bash[27579]: audit 2023-08-27T23:04:39.380705+0000 mon.b (mon.1) 313 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:04:40.088 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:39 smithi130 bash[27579]: audit 2023-08-27T23:04:39.389263+0000 mon.a (mon.0) 1882 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:04:40.088 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:39 smithi130 bash[27579]: audit 2023-08-27T23:04:39.699195+0000 mon.c (mon.2) 787 : audit [DBG] from='client.? 172.21.15.6:0/3842743359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:40.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:39 smithi006 bash[33918]: cluster 2023-08-27T23:04:38.503335+0000 mgr.y (mgr.14901) 1983 : cluster [DBG] pgmap v1990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:40.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:39 smithi006 bash[33918]: audit 2023-08-27T23:04:38.729037+0000 mon.a (mon.0) 1880 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:04:40.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:39 smithi006 bash[33918]: audit 2023-08-27T23:04:39.107910+0000 mon.a (mon.0) 1881 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:04:40.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:39 smithi006 bash[33918]: audit 2023-08-27T23:04:39.378562+0000 mon.b (mon.1) 312 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:04:40.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:39 smithi006 bash[33918]: audit 2023-08-27T23:04:39.380705+0000 mon.b (mon.1) 313 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:04:40.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:39 smithi006 bash[33918]: audit 2023-08-27T23:04:39.389263+0000 mon.a (mon.0) 1882 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:04:40.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:39 smithi006 bash[33918]: audit 2023-08-27T23:04:39.699195+0000 mon.c (mon.2) 787 : audit [DBG] from='client.? 172.21.15.6:0/3842743359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:40.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:39 smithi006 bash[27292]: cluster 2023-08-27T23:04:38.503335+0000 mgr.y (mgr.14901) 1983 : cluster [DBG] pgmap v1990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:40.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:39 smithi006 bash[27292]: audit 2023-08-27T23:04:38.729037+0000 mon.a (mon.0) 1880 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:04:40.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:39 smithi006 bash[27292]: audit 2023-08-27T23:04:39.107910+0000 mon.a (mon.0) 1881 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:04:40.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:39 smithi006 bash[27292]: audit 2023-08-27T23:04:39.378562+0000 mon.b (mon.1) 312 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:04:40.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:39 smithi006 bash[27292]: audit 2023-08-27T23:04:39.380705+0000 mon.b (mon.1) 313 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:04:40.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:39 smithi006 bash[27292]: audit 2023-08-27T23:04:39.389263+0000 mon.a (mon.0) 1882 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:04:40.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:39 smithi006 bash[27292]: audit 2023-08-27T23:04:39.699195+0000 mon.c (mon.2) 787 : audit [DBG] from='client.? 172.21.15.6:0/3842743359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:40.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:40 smithi130 bash[51302]: debug 2023-08-27T23:04:40.081+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:40.086573+0000) 2023-08-27T23:04:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:04:40.370Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:04:40.372Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:04:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:04:40] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:04:41.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:41 smithi130 bash[51302]: debug 2023-08-27T23:04:41.085+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:41.086895+0000) 2023-08-27T23:04:42.088 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:41 smithi130 bash[27579]: cluster 2023-08-27T23:04:40.504517+0000 mgr.y (mgr.14901) 1984 : cluster [DBG] pgmap v1991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:42.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:41 smithi006 bash[33918]: cluster 2023-08-27T23:04:40.504517+0000 mgr.y (mgr.14901) 1984 : cluster [DBG] pgmap v1991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:42.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:41 smithi006 bash[27292]: cluster 2023-08-27T23:04:40.504517+0000 mgr.y (mgr.14901) 1984 : cluster [DBG] pgmap v1991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:42.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:41 smithi006 bash[56298]: level=error ts=2023-08-27T23:04:41.931Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:42.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:41 smithi006 bash[56298]: level=warn ts=2023-08-27T23:04:41.933Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:42.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:42 smithi130 bash[51302]: debug 2023-08-27T23:04:42.085+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:42.087101+0000) 2023-08-27T23:04:43.088 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:42 smithi130 bash[27579]: audit 2023-08-27T23:04:42.121325+0000 mon.a (mon.0) 1883 : audit [DBG] from='client.? 172.21.15.6:0/1075889425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:42 smithi006 bash[33918]: audit 2023-08-27T23:04:42.121325+0000 mon.a (mon.0) 1883 : audit [DBG] from='client.? 172.21.15.6:0/1075889425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:42 smithi006 bash[27292]: audit 2023-08-27T23:04:42.121325+0000 mon.a (mon.0) 1883 : audit [DBG] from='client.? 172.21.15.6:0/1075889425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:43.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:43 smithi130 bash[51302]: debug 2023-08-27T23:04:43.085+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:43.087434+0000) 2023-08-27T23:04:44.088 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:43 smithi130 bash[27579]: cluster 2023-08-27T23:04:42.505210+0000 mgr.y (mgr.14901) 1985 : cluster [DBG] pgmap v1992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:04:43.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:04:43.833Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:44.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:43 smithi006 bash[27292]: cluster 2023-08-27T23:04:42.505210+0000 mgr.y (mgr.14901) 1985 : cluster [DBG] pgmap v1992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:44.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:43 smithi006 bash[33918]: cluster 2023-08-27T23:04:42.505210+0000 mgr.y (mgr.14901) 1985 : cluster [DBG] pgmap v1992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:44.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:44 smithi130 bash[51302]: debug 2023-08-27T23:04:44.085+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:44.087773+0000) 2023-08-27T23:04:45.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:45 smithi130 bash[27579]: audit 2023-08-27T23:04:44.529078+0000 mon.a (mon.0) 1884 : audit [DBG] from='client.? 172.21.15.6:0/2994559800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:45 smithi130 bash[51302]: debug 2023-08-27T23:04:45.085+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:45.088051+0000) 2023-08-27T23:04:45.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:45 smithi006 bash[33918]: audit 2023-08-27T23:04:44.529078+0000 mon.a (mon.0) 1884 : audit [DBG] from='client.? 172.21.15.6:0/2994559800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:45.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:45 smithi006 bash[27292]: audit 2023-08-27T23:04:44.529078+0000 mon.a (mon.0) 1884 : audit [DBG] from='client.? 172.21.15.6:0/2994559800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:46.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:46 smithi130 bash[27579]: cluster 2023-08-27T23:04:44.506296+0000 mgr.y (mgr.14901) 1986 : cluster [DBG] pgmap v1993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:46 smithi130 bash[51302]: debug 2023-08-27T23:04:46.085+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:46.088349+0000) 2023-08-27T23:04:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:46 smithi006 bash[33918]: cluster 2023-08-27T23:04:44.506296+0000 mgr.y (mgr.14901) 1986 : cluster [DBG] pgmap v1993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:46 smithi006 bash[27292]: cluster 2023-08-27T23:04:44.506296+0000 mgr.y (mgr.14901) 1986 : cluster [DBG] pgmap v1993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:47.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:47 smithi130 bash[27579]: audit 2023-08-27T23:04:46.943683+0000 mon.a (mon.0) 1885 : audit [DBG] from='client.? 172.21.15.6:0/2623712846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:47 smithi130 bash[51302]: debug 2023-08-27T23:04:47.085+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:47.088630+0000) 2023-08-27T23:04:47.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:47 smithi006 bash[27292]: audit 2023-08-27T23:04:46.943683+0000 mon.a (mon.0) 1885 : audit [DBG] from='client.? 172.21.15.6:0/2623712846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:47.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:47 smithi006 bash[33918]: audit 2023-08-27T23:04:46.943683+0000 mon.a (mon.0) 1885 : audit [DBG] from='client.? 172.21.15.6:0/2623712846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:48.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:48 smithi130 bash[27579]: cluster 2023-08-27T23:04:46.506957+0000 mgr.y (mgr.14901) 1987 : cluster [DBG] pgmap v1994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:48.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:48 smithi130 bash[51302]: debug 2023-08-27T23:04:48.085+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:48.088937+0000) 2023-08-27T23:04:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:48 smithi006 bash[27292]: cluster 2023-08-27T23:04:46.506957+0000 mgr.y (mgr.14901) 1987 : cluster [DBG] pgmap v1994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:48 smithi006 bash[33918]: cluster 2023-08-27T23:04:46.506957+0000 mgr.y (mgr.14901) 1987 : cluster [DBG] pgmap v1994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:49.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:49 smithi130 bash[51302]: debug 2023-08-27T23:04:49.085+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:49.089208+0000) 2023-08-27T23:04:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:50 smithi130 bash[51302]: debug 2023-08-27T23:04:50.085+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:50.089483+0000) 2023-08-27T23:04:50.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:50 smithi130 bash[27579]: cluster 2023-08-27T23:04:48.507610+0000 mgr.y (mgr.14901) 1988 : cluster [DBG] pgmap v1995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:50.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:50 smithi130 bash[27579]: audit 2023-08-27T23:04:49.363792+0000 mon.c (mon.2) 788 : audit [DBG] from='client.? 172.21.15.6:0/1120975178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:50 smithi006 bash[27292]: cluster 2023-08-27T23:04:48.507610+0000 mgr.y (mgr.14901) 1988 : cluster [DBG] pgmap v1995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:50 smithi006 bash[27292]: audit 2023-08-27T23:04:49.363792+0000 mon.c (mon.2) 788 : audit [DBG] from='client.? 172.21.15.6:0/1120975178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:50 smithi006 bash[33918]: cluster 2023-08-27T23:04:48.507610+0000 mgr.y (mgr.14901) 1988 : cluster [DBG] pgmap v1995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:50 smithi006 bash[33918]: audit 2023-08-27T23:04:49.363792+0000 mon.c (mon.2) 788 : audit [DBG] from='client.? 172.21.15.6:0/1120975178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:50.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:04:50.370Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:50.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:04:50.372Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:04:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:04:50] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T23:04:51.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:51 smithi130 bash[51302]: debug 2023-08-27T23:04:51.085+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:51.089794+0000) 2023-08-27T23:04:52.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:51 smithi006 bash[56298]: level=error ts=2023-08-27T23:04:51.933Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:52.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:51 smithi006 bash[56298]: level=warn ts=2023-08-27T23:04:51.935Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:52.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:52 smithi130 bash[51302]: debug 2023-08-27T23:04:52.085+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:52.090227+0000) 2023-08-27T23:04:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:52 smithi130 bash[27579]: cluster 2023-08-27T23:04:50.508683+0000 mgr.y (mgr.14901) 1989 : cluster [DBG] pgmap v1996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:52 smithi130 bash[27579]: audit 2023-08-27T23:04:51.775526+0000 mon.c (mon.2) 789 : audit [DBG] from='client.? 172.21.15.6:0/75957922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:52.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:52 smithi006 bash[27292]: cluster 2023-08-27T23:04:50.508683+0000 mgr.y (mgr.14901) 1989 : cluster [DBG] pgmap v1996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:52 smithi006 bash[27292]: audit 2023-08-27T23:04:51.775526+0000 mon.c (mon.2) 789 : audit [DBG] from='client.? 172.21.15.6:0/75957922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:52 smithi006 bash[33918]: cluster 2023-08-27T23:04:50.508683+0000 mgr.y (mgr.14901) 1989 : cluster [DBG] pgmap v1996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:52 smithi006 bash[33918]: audit 2023-08-27T23:04:51.775526+0000 mon.c (mon.2) 789 : audit [DBG] from='client.? 172.21.15.6:0/75957922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:53.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:53 smithi130 bash[51302]: debug 2023-08-27T23:04:53.085+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:53.090607+0000) 2023-08-27T23:04:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:04:53.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:04:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:04:53.833Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:04:54.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:54 smithi130 bash[51302]: debug 2023-08-27T23:04:54.089+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:54.090956+0000) 2023-08-27T23:04:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:54 smithi130 bash[27579]: cluster 2023-08-27T23:04:52.509460+0000 mgr.y (mgr.14901) 1990 : cluster [DBG] pgmap v1997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:54 smithi130 bash[27579]: audit 2023-08-27T23:04:54.190127+0000 mon.a (mon.0) 1886 : audit [DBG] from='client.? 172.21.15.6:0/3238256878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:54.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:54 smithi006 bash[33918]: cluster 2023-08-27T23:04:52.509460+0000 mgr.y (mgr.14901) 1990 : cluster [DBG] pgmap v1997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:54.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:54 smithi006 bash[33918]: audit 2023-08-27T23:04:54.190127+0000 mon.a (mon.0) 1886 : audit [DBG] from='client.? 172.21.15.6:0/3238256878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:54.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:54 smithi006 bash[27292]: cluster 2023-08-27T23:04:52.509460+0000 mgr.y (mgr.14901) 1990 : cluster [DBG] pgmap v1997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:54.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:54 smithi006 bash[27292]: audit 2023-08-27T23:04:54.190127+0000 mon.a (mon.0) 1886 : audit [DBG] from='client.? 172.21.15.6:0/3238256878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:55.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:55 smithi130 bash[51302]: debug 2023-08-27T23:04:55.089+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:55.091265+0000) 2023-08-27T23:04:56.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:56 smithi130 bash[27579]: cluster 2023-08-27T23:04:54.510643+0000 mgr.y (mgr.14901) 1991 : cluster [DBG] pgmap v1998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:56.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:56 smithi130 bash[51302]: debug 2023-08-27T23:04:56.089+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:56.091483+0000) 2023-08-27T23:04:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:56 smithi006 bash[33918]: cluster 2023-08-27T23:04:54.510643+0000 mgr.y (mgr.14901) 1991 : cluster [DBG] pgmap v1998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:56 smithi006 bash[27292]: cluster 2023-08-27T23:04:54.510643+0000 mgr.y (mgr.14901) 1991 : cluster [DBG] pgmap v1998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:57.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:57 smithi130 bash[27579]: audit 2023-08-27T23:04:56.604782+0000 mon.c (mon.2) 790 : audit [DBG] from='client.? 172.21.15.6:0/179291391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:57.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:57 smithi130 bash[51302]: debug 2023-08-27T23:04:57.089+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:57.091750+0000) 2023-08-27T23:04:57.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:57 smithi006 bash[33918]: audit 2023-08-27T23:04:56.604782+0000 mon.c (mon.2) 790 : audit [DBG] from='client.? 172.21.15.6:0/179291391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:57.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:57 smithi006 bash[27292]: audit 2023-08-27T23:04:56.604782+0000 mon.c (mon.2) 790 : audit [DBG] from='client.? 172.21.15.6:0/179291391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:58.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:58 smithi130 bash[27579]: cluster 2023-08-27T23:04:56.511258+0000 mgr.y (mgr.14901) 1992 : cluster [DBG] pgmap v1999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:58.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:58 smithi130 bash[51302]: debug 2023-08-27T23:04:58.089+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:58.092079+0000) 2023-08-27T23:04:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:58 smithi006 bash[27292]: cluster 2023-08-27T23:04:56.511258+0000 mgr.y (mgr.14901) 1992 : cluster [DBG] pgmap v1999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:58 smithi006 bash[33918]: cluster 2023-08-27T23:04:56.511258+0000 mgr.y (mgr.14901) 1992 : cluster [DBG] pgmap v1999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:04:59.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:04:59 smithi130 bash[27579]: audit 2023-08-27T23:04:59.016518+0000 mon.c (mon.2) 791 : audit [DBG] from='client.? 172.21.15.6:0/1253478026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:04:59 smithi130 bash[51302]: debug 2023-08-27T23:04:59.089+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:59.092387+0000) 2023-08-27T23:04:59.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:04:59 smithi006 bash[27292]: audit 2023-08-27T23:04:59.016518+0000 mon.c (mon.2) 791 : audit [DBG] from='client.? 172.21.15.6:0/1253478026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:59.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:04:59 smithi006 bash[33918]: audit 2023-08-27T23:04:59.016518+0000 mon.c (mon.2) 791 : audit [DBG] from='client.? 172.21.15.6:0/1253478026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:00.344 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:00 smithi130 bash[51302]: debug 2023-08-27T23:05:00.089+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:00.092591+0000) 2023-08-27T23:05:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:00 smithi130 bash[27579]: cluster 2023-08-27T23:04:58.511876+0000 mgr.y (mgr.14901) 1993 : cluster [DBG] pgmap v2000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:00 smithi006 bash[27292]: cluster 2023-08-27T23:04:58.511876+0000 mgr.y (mgr.14901) 1993 : cluster [DBG] pgmap v2000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:00 smithi006 bash[33918]: cluster 2023-08-27T23:04:58.511876+0000 mgr.y (mgr.14901) 1993 : cluster [DBG] pgmap v2000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:00.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:05:00.371Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:00.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:05:00.372Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:05:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:05:00] "GET /metrics HTTP/1.1" 200 193091 "" "Prometheus/2.33.4" 2023-08-27T23:05:01.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:01 smithi130 bash[51302]: debug 2023-08-27T23:05:01.089+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:01.092873+0000) 2023-08-27T23:05:02.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:01 smithi006 bash[56298]: level=error ts=2023-08-27T23:05:01.933Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:02.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:01 smithi006 bash[56298]: level=warn ts=2023-08-27T23:05:01.935Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:02.351 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:02 smithi130 bash[51302]: debug 2023-08-27T23:05:02.089+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:02.093124+0000) 2023-08-27T23:05:02.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:02 smithi130 bash[27579]: cluster 2023-08-27T23:05:00.512979+0000 mgr.y (mgr.14901) 1994 : cluster [DBG] pgmap v2001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:02 smithi130 bash[27579]: audit 2023-08-27T23:05:01.432498+0000 mon.a (mon.0) 1887 : audit [DBG] from='client.? 172.21.15.6:0/1214108698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:02 smithi006 bash[27292]: cluster 2023-08-27T23:05:00.512979+0000 mgr.y (mgr.14901) 1994 : cluster [DBG] pgmap v2001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:02 smithi006 bash[27292]: audit 2023-08-27T23:05:01.432498+0000 mon.a (mon.0) 1887 : audit [DBG] from='client.? 172.21.15.6:0/1214108698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:02 smithi006 bash[33918]: cluster 2023-08-27T23:05:00.512979+0000 mgr.y (mgr.14901) 1994 : cluster [DBG] pgmap v2001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:02 smithi006 bash[33918]: audit 2023-08-27T23:05:01.432498+0000 mon.a (mon.0) 1887 : audit [DBG] from='client.? 172.21.15.6:0/1214108698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:03.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:03 smithi130 bash[51302]: debug 2023-08-27T23:05:03.089+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:03.093380+0000) 2023-08-27T23:05:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:05:03.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:05:03.834Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:04.357 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:04 smithi130 bash[51302]: debug 2023-08-27T23:05:04.093+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:04.093662+0000) 2023-08-27T23:05:04.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:04 smithi130 bash[27579]: cluster 2023-08-27T23:05:02.513651+0000 mgr.y (mgr.14901) 1995 : cluster [DBG] pgmap v2002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:04 smithi130 bash[27579]: audit 2023-08-27T23:05:03.844672+0000 mon.a (mon.0) 1888 : audit [DBG] from='client.? 172.21.15.6:0/1398581487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:04.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:04 smithi006 bash[27292]: cluster 2023-08-27T23:05:02.513651+0000 mgr.y (mgr.14901) 1995 : cluster [DBG] pgmap v2002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:04.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:04 smithi006 bash[27292]: audit 2023-08-27T23:05:03.844672+0000 mon.a (mon.0) 1888 : audit [DBG] from='client.? 172.21.15.6:0/1398581487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:04.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:04 smithi006 bash[33918]: cluster 2023-08-27T23:05:02.513651+0000 mgr.y (mgr.14901) 1995 : cluster [DBG] pgmap v2002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:04.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:04 smithi006 bash[33918]: audit 2023-08-27T23:05:03.844672+0000 mon.a (mon.0) 1888 : audit [DBG] from='client.? 172.21.15.6:0/1398581487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:05.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:05 smithi130 bash[51302]: debug 2023-08-27T23:05:05.089+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:05.093972+0000) 2023-08-27T23:05:06.362 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:06 smithi130 bash[51302]: debug 2023-08-27T23:05:06.089+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:06.094283+0000) 2023-08-27T23:05:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:06 smithi130 bash[27579]: cluster 2023-08-27T23:05:04.514858+0000 mgr.y (mgr.14901) 1996 : cluster [DBG] pgmap v2003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:06 smithi130 bash[27579]: audit 2023-08-27T23:05:06.261460+0000 mon.a (mon.0) 1889 : audit [DBG] from='client.? 172.21.15.6:0/4221460001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:06 smithi006 bash[27292]: cluster 2023-08-27T23:05:04.514858+0000 mgr.y (mgr.14901) 1996 : cluster [DBG] pgmap v2003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:06 smithi006 bash[27292]: audit 2023-08-27T23:05:06.261460+0000 mon.a (mon.0) 1889 : audit [DBG] from='client.? 172.21.15.6:0/4221460001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:06 smithi006 bash[33918]: cluster 2023-08-27T23:05:04.514858+0000 mgr.y (mgr.14901) 1996 : cluster [DBG] pgmap v2003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:06 smithi006 bash[33918]: audit 2023-08-27T23:05:06.261460+0000 mon.a (mon.0) 1889 : audit [DBG] from='client.? 172.21.15.6:0/4221460001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:07.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:07 smithi130 bash[51302]: debug 2023-08-27T23:05:07.089+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:07.094562+0000) 2023-08-27T23:05:08.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:08 smithi130 bash[51302]: debug 2023-08-27T23:05:08.093+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:08.094887+0000) 2023-08-27T23:05:08.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:08 smithi130 bash[27579]: cluster 2023-08-27T23:05:06.515496+0000 mgr.y (mgr.14901) 1997 : cluster [DBG] pgmap v2004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:08 smithi006 bash[27292]: cluster 2023-08-27T23:05:06.515496+0000 mgr.y (mgr.14901) 1997 : cluster [DBG] pgmap v2004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:08 smithi006 bash[33918]: cluster 2023-08-27T23:05:06.515496+0000 mgr.y (mgr.14901) 1997 : cluster [DBG] pgmap v2004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:09.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:09 smithi130 bash[51302]: debug 2023-08-27T23:05:09.093+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:09.095162+0000) 2023-08-27T23:05:09.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:09 smithi006 bash[33918]: audit 2023-08-27T23:05:08.671211+0000 mon.a (mon.0) 1890 : audit [DBG] from='client.? 172.21.15.6:0/2802340754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:09 smithi006 bash[27292]: audit 2023-08-27T23:05:08.671211+0000 mon.a (mon.0) 1890 : audit [DBG] from='client.? 172.21.15.6:0/2802340754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:09.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:09 smithi130 bash[27579]: audit 2023-08-27T23:05:08.671211+0000 mon.a (mon.0) 1890 : audit [DBG] from='client.? 172.21.15.6:0/2802340754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:10.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:10 smithi130 bash[51302]: debug 2023-08-27T23:05:10.093+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:10.095486+0000) 2023-08-27T23:05:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:05:10.371Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:05:10.373Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:10 smithi006 bash[27292]: cluster 2023-08-27T23:05:08.516265+0000 mgr.y (mgr.14901) 1998 : cluster [DBG] pgmap v2005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:10 smithi006 bash[33918]: cluster 2023-08-27T23:05:08.516265+0000 mgr.y (mgr.14901) 1998 : cluster [DBG] pgmap v2005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:10 smithi130 bash[27579]: cluster 2023-08-27T23:05:08.516265+0000 mgr.y (mgr.14901) 1998 : cluster [DBG] pgmap v2005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:05:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:05:10] "GET /metrics HTTP/1.1" 200 193091 "" "Prometheus/2.33.4" 2023-08-27T23:05:11.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:11 smithi130 bash[51302]: debug 2023-08-27T23:05:11.093+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:11.095762+0000) 2023-08-27T23:05:11.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:11 smithi006 bash[27292]: audit 2023-08-27T23:05:11.094658+0000 mon.a (mon.0) 1891 : audit [DBG] from='client.? 172.21.15.6:0/697306445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:11.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:11 smithi006 bash[33918]: audit 2023-08-27T23:05:11.094658+0000 mon.a (mon.0) 1891 : audit [DBG] from='client.? 172.21.15.6:0/697306445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:11.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:11 smithi130 bash[27579]: audit 2023-08-27T23:05:11.094658+0000 mon.a (mon.0) 1891 : audit [DBG] from='client.? 172.21.15.6:0/697306445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:12.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:11 smithi006 bash[56298]: level=error ts=2023-08-27T23:05:11.934Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:12.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:11 smithi006 bash[56298]: level=warn ts=2023-08-27T23:05:11.935Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:12.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:12 smithi130 bash[51302]: debug 2023-08-27T23:05:12.093+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:12.096014+0000) 2023-08-27T23:05:12.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:12 smithi006 bash[27292]: cluster 2023-08-27T23:05:10.517479+0000 mgr.y (mgr.14901) 1999 : cluster [DBG] pgmap v2006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:12.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:12 smithi006 bash[33918]: cluster 2023-08-27T23:05:10.517479+0000 mgr.y (mgr.14901) 1999 : cluster [DBG] pgmap v2006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:12 smithi130 bash[27579]: cluster 2023-08-27T23:05:10.517479+0000 mgr.y (mgr.14901) 1999 : cluster [DBG] pgmap v2006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:13 smithi130 bash[51302]: debug 2023-08-27T23:05:13.093+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:13.096281+0000) 2023-08-27T23:05:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:05:13.833Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:05:13.836Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:14.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:14 smithi130 bash[51302]: debug 2023-08-27T23:05:14.093+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:14.096607+0000) 2023-08-27T23:05:14.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:14 smithi006 bash[27292]: cluster 2023-08-27T23:05:12.518163+0000 mgr.y (mgr.14901) 2000 : cluster [DBG] pgmap v2007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:14.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:14 smithi006 bash[27292]: audit 2023-08-27T23:05:13.512598+0000 mon.c (mon.2) 792 : audit [DBG] from='client.? 172.21.15.6:0/735805380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:14.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:14 smithi006 bash[33918]: cluster 2023-08-27T23:05:12.518163+0000 mgr.y (mgr.14901) 2000 : cluster [DBG] pgmap v2007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:14.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:14 smithi006 bash[33918]: audit 2023-08-27T23:05:13.512598+0000 mon.c (mon.2) 792 : audit [DBG] from='client.? 172.21.15.6:0/735805380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:14 smithi130 bash[27579]: cluster 2023-08-27T23:05:12.518163+0000 mgr.y (mgr.14901) 2000 : cluster [DBG] pgmap v2007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:14 smithi130 bash[27579]: audit 2023-08-27T23:05:13.512598+0000 mon.c (mon.2) 792 : audit [DBG] from='client.? 172.21.15.6:0/735805380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:15.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:15 smithi130 bash[51302]: debug 2023-08-27T23:05:15.093+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:15.096902+0000) 2023-08-27T23:05:16.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:16 smithi130 bash[51302]: debug 2023-08-27T23:05:16.093+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:16.097174+0000) 2023-08-27T23:05:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:16 smithi006 bash[27292]: cluster 2023-08-27T23:05:14.519302+0000 mgr.y (mgr.14901) 2001 : cluster [DBG] pgmap v2008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:16 smithi006 bash[27292]: audit 2023-08-27T23:05:15.926307+0000 mon.c (mon.2) 793 : audit [DBG] from='client.? 172.21.15.6:0/4284138694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:16 smithi006 bash[33918]: cluster 2023-08-27T23:05:14.519302+0000 mgr.y (mgr.14901) 2001 : cluster [DBG] pgmap v2008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:16 smithi006 bash[33918]: audit 2023-08-27T23:05:15.926307+0000 mon.c (mon.2) 793 : audit [DBG] from='client.? 172.21.15.6:0/4284138694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:16.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:16 smithi130 bash[27579]: cluster 2023-08-27T23:05:14.519302+0000 mgr.y (mgr.14901) 2001 : cluster [DBG] pgmap v2008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:16.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:16 smithi130 bash[27579]: audit 2023-08-27T23:05:15.926307+0000 mon.c (mon.2) 793 : audit [DBG] from='client.? 172.21.15.6:0/4284138694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:17.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:17 smithi130 bash[51302]: debug 2023-08-27T23:05:17.093+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:17.097461+0000) 2023-08-27T23:05:18.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:18 smithi130 bash[51302]: debug 2023-08-27T23:05:18.093+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:18.097752+0000) 2023-08-27T23:05:18.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:18 smithi006 bash[27292]: cluster 2023-08-27T23:05:16.519959+0000 mgr.y (mgr.14901) 2002 : cluster [DBG] pgmap v2009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:18.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:18 smithi006 bash[27292]: audit 2023-08-27T23:05:18.342588+0000 mon.c (mon.2) 794 : audit [DBG] from='client.? 172.21.15.6:0/172785599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:18.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:18 smithi006 bash[33918]: cluster 2023-08-27T23:05:16.519959+0000 mgr.y (mgr.14901) 2002 : cluster [DBG] pgmap v2009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:18.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:18 smithi006 bash[33918]: audit 2023-08-27T23:05:18.342588+0000 mon.c (mon.2) 794 : audit [DBG] from='client.? 172.21.15.6:0/172785599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:18.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:18 smithi130 bash[27579]: cluster 2023-08-27T23:05:16.519959+0000 mgr.y (mgr.14901) 2002 : cluster [DBG] pgmap v2009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:18 smithi130 bash[27579]: audit 2023-08-27T23:05:18.342588+0000 mon.c (mon.2) 794 : audit [DBG] from='client.? 172.21.15.6:0/172785599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:19.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:19 smithi130 bash[51302]: debug 2023-08-27T23:05:19.093+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:19.098052+0000) 2023-08-27T23:05:20.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:20 smithi130 bash[51302]: debug 2023-08-27T23:05:20.097+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:20.098350+0000) 2023-08-27T23:05:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:20 smithi006 bash[27292]: cluster 2023-08-27T23:05:18.520657+0000 mgr.y (mgr.14901) 2003 : cluster [DBG] pgmap v2010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:20 smithi006 bash[33918]: cluster 2023-08-27T23:05:18.520657+0000 mgr.y (mgr.14901) 2003 : cluster [DBG] pgmap v2010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:20.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:05:20.371Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:20.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:05:20.373Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:20.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:20 smithi130 bash[27579]: cluster 2023-08-27T23:05:18.520657+0000 mgr.y (mgr.14901) 2003 : cluster [DBG] pgmap v2010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:05:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:05:20] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:05:21.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:21 smithi130 bash[51302]: debug 2023-08-27T23:05:21.093+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:21.098699+0000) 2023-08-27T23:05:21.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:21 smithi006 bash[27292]: audit 2023-08-27T23:05:20.761494+0000 mon.a (mon.0) 1892 : audit [DBG] from='client.? 172.21.15.6:0/3274121265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:21.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:21 smithi006 bash[33918]: audit 2023-08-27T23:05:20.761494+0000 mon.a (mon.0) 1892 : audit [DBG] from='client.? 172.21.15.6:0/3274121265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:21.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:21 smithi130 bash[27579]: audit 2023-08-27T23:05:20.761494+0000 mon.a (mon.0) 1892 : audit [DBG] from='client.? 172.21.15.6:0/3274121265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:22.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:21 smithi006 bash[56298]: level=error ts=2023-08-27T23:05:21.935Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:22.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:21 smithi006 bash[56298]: level=warn ts=2023-08-27T23:05:21.936Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:22.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:22 smithi130 bash[51302]: debug 2023-08-27T23:05:22.097+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:22.098945+0000) 2023-08-27T23:05:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:22 smithi006 bash[27292]: cluster 2023-08-27T23:05:20.521755+0000 mgr.y (mgr.14901) 2004 : cluster [DBG] pgmap v2011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:22 smithi006 bash[33918]: cluster 2023-08-27T23:05:20.521755+0000 mgr.y (mgr.14901) 2004 : cluster [DBG] pgmap v2011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:22 smithi130 bash[27579]: cluster 2023-08-27T23:05:20.521755+0000 mgr.y (mgr.14901) 2004 : cluster [DBG] pgmap v2011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:23.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:23 smithi130 bash[51302]: debug 2023-08-27T23:05:23.097+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:23.099158+0000) 2023-08-27T23:05:23.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:23 smithi006 bash[27292]: audit 2023-08-27T23:05:23.184223+0000 mon.c (mon.2) 795 : audit [DBG] from='client.? 172.21.15.6:0/409333665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:23.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:23 smithi006 bash[33918]: audit 2023-08-27T23:05:23.184223+0000 mon.c (mon.2) 795 : audit [DBG] from='client.? 172.21.15.6:0/409333665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:23.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:23 smithi130 bash[27579]: audit 2023-08-27T23:05:23.184223+0000 mon.c (mon.2) 795 : audit [DBG] from='client.? 172.21.15.6:0/409333665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:05:23.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:05:23.836Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:24.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:24 smithi130 bash[51302]: debug 2023-08-27T23:05:24.097+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:24.099444+0000) 2023-08-27T23:05:24.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:24 smithi006 bash[27292]: cluster 2023-08-27T23:05:22.522431+0000 mgr.y (mgr.14901) 2005 : cluster [DBG] pgmap v2012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:24.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:24 smithi006 bash[33918]: cluster 2023-08-27T23:05:22.522431+0000 mgr.y (mgr.14901) 2005 : cluster [DBG] pgmap v2012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:24 smithi130 bash[27579]: cluster 2023-08-27T23:05:22.522431+0000 mgr.y (mgr.14901) 2005 : cluster [DBG] pgmap v2012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:25.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:25 smithi130 bash[51302]: debug 2023-08-27T23:05:25.097+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:25.099716+0000) 2023-08-27T23:05:26.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:26 smithi130 bash[51302]: debug 2023-08-27T23:05:26.097+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:26.099929+0000) 2023-08-27T23:05:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:26 smithi006 bash[27292]: cluster 2023-08-27T23:05:24.523560+0000 mgr.y (mgr.14901) 2006 : cluster [DBG] pgmap v2013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:26 smithi006 bash[27292]: audit 2023-08-27T23:05:25.598234+0000 mon.c (mon.2) 796 : audit [DBG] from='client.? 172.21.15.6:0/2630417504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:26 smithi006 bash[33918]: cluster 2023-08-27T23:05:24.523560+0000 mgr.y (mgr.14901) 2006 : cluster [DBG] pgmap v2013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:26 smithi006 bash[33918]: audit 2023-08-27T23:05:25.598234+0000 mon.c (mon.2) 796 : audit [DBG] from='client.? 172.21.15.6:0/2630417504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:26.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:26 smithi130 bash[27579]: cluster 2023-08-27T23:05:24.523560+0000 mgr.y (mgr.14901) 2006 : cluster [DBG] pgmap v2013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:26 smithi130 bash[27579]: audit 2023-08-27T23:05:25.598234+0000 mon.c (mon.2) 796 : audit [DBG] from='client.? 172.21.15.6:0/2630417504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:27.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:27 smithi130 bash[51302]: debug 2023-08-27T23:05:27.097+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:27.100186+0000) 2023-08-27T23:05:28.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:28 smithi130 bash[51302]: debug 2023-08-27T23:05:28.097+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:28.100408+0000) 2023-08-27T23:05:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:28 smithi006 bash[33918]: cluster 2023-08-27T23:05:26.524212+0000 mgr.y (mgr.14901) 2007 : cluster [DBG] pgmap v2014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:28 smithi006 bash[33918]: audit 2023-08-27T23:05:28.011430+0000 mon.a (mon.0) 1893 : audit [DBG] from='client.? 172.21.15.6:0/367280211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:28.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:28 smithi006 bash[27292]: cluster 2023-08-27T23:05:26.524212+0000 mgr.y (mgr.14901) 2007 : cluster [DBG] pgmap v2014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:28.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:28 smithi006 bash[27292]: audit 2023-08-27T23:05:28.011430+0000 mon.a (mon.0) 1893 : audit [DBG] from='client.? 172.21.15.6:0/367280211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:28 smithi130 bash[27579]: cluster 2023-08-27T23:05:26.524212+0000 mgr.y (mgr.14901) 2007 : cluster [DBG] pgmap v2014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:28 smithi130 bash[27579]: audit 2023-08-27T23:05:28.011430+0000 mon.a (mon.0) 1893 : audit [DBG] from='client.? 172.21.15.6:0/367280211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:29.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:29 smithi130 bash[51302]: debug 2023-08-27T23:05:29.097+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:29.100753+0000) 2023-08-27T23:05:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:30 smithi130 bash[51302]: debug 2023-08-27T23:05:30.096+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:30.101065+0000) 2023-08-27T23:05:30.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:30 smithi006 bash[33918]: cluster 2023-08-27T23:05:28.524828+0000 mgr.y (mgr.14901) 2008 : cluster [DBG] pgmap v2015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:30.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:30 smithi006 bash[33918]: audit 2023-08-27T23:05:30.432495+0000 mon.c (mon.2) 797 : audit [DBG] from='client.? 172.21.15.6:0/946162726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:30.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:30 smithi006 bash[27292]: cluster 2023-08-27T23:05:28.524828+0000 mgr.y (mgr.14901) 2008 : cluster [DBG] pgmap v2015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:30.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:30 smithi006 bash[27292]: audit 2023-08-27T23:05:30.432495+0000 mon.c (mon.2) 797 : audit [DBG] from='client.? 172.21.15.6:0/946162726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:30.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:05:30.372Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:30.720 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:05:30.374Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:30.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:30 smithi130 bash[27579]: cluster 2023-08-27T23:05:28.524828+0000 mgr.y (mgr.14901) 2008 : cluster [DBG] pgmap v2015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:30 smithi130 bash[27579]: audit 2023-08-27T23:05:30.432495+0000 mon.c (mon.2) 797 : audit [DBG] from='client.? 172.21.15.6:0/946162726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:05:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:05:30] "GET /metrics HTTP/1.1" 200 193093 "" "Prometheus/2.33.4" 2023-08-27T23:05:31.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:31 smithi130 bash[51302]: debug 2023-08-27T23:05:31.100+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:31.101360+0000) 2023-08-27T23:05:32.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:31 smithi006 bash[56298]: level=error ts=2023-08-27T23:05:31.935Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:32.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:31 smithi006 bash[56298]: level=warn ts=2023-08-27T23:05:31.937Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:32.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:32 smithi130 bash[51302]: debug 2023-08-27T23:05:32.096+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:32.101552+0000) 2023-08-27T23:05:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:32 smithi006 bash[27292]: cluster 2023-08-27T23:05:30.525885+0000 mgr.y (mgr.14901) 2009 : cluster [DBG] pgmap v2016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:32.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:32 smithi006 bash[33918]: cluster 2023-08-27T23:05:30.525885+0000 mgr.y (mgr.14901) 2009 : cluster [DBG] pgmap v2016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:32 smithi130 bash[27579]: cluster 2023-08-27T23:05:30.525885+0000 mgr.y (mgr.14901) 2009 : cluster [DBG] pgmap v2016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:33.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:33 smithi130 bash[51302]: debug 2023-08-27T23:05:33.096+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:33.101802+0000) 2023-08-27T23:05:33.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:33 smithi006 bash[33918]: audit 2023-08-27T23:05:32.857832+0000 mon.a (mon.0) 1894 : audit [DBG] from='client.? 172.21.15.6:0/1053615910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:33.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:33 smithi006 bash[27292]: audit 2023-08-27T23:05:32.857832+0000 mon.a (mon.0) 1894 : audit [DBG] from='client.? 172.21.15.6:0/1053615910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:33 smithi130 bash[27579]: audit 2023-08-27T23:05:32.857832+0000 mon.a (mon.0) 1894 : audit [DBG] from='client.? 172.21.15.6:0/1053615910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:34.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:05:33.835Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:34.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:05:33.837Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:34.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:34 smithi130 bash[51302]: debug 2023-08-27T23:05:34.096+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:34.102079+0000) 2023-08-27T23:05:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:34 smithi006 bash[33918]: cluster 2023-08-27T23:05:32.526497+0000 mgr.y (mgr.14901) 2010 : cluster [DBG] pgmap v2017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:34 smithi006 bash[27292]: cluster 2023-08-27T23:05:32.526497+0000 mgr.y (mgr.14901) 2010 : cluster [DBG] pgmap v2017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:34.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:34 smithi130 bash[27579]: cluster 2023-08-27T23:05:32.526497+0000 mgr.y (mgr.14901) 2010 : cluster [DBG] pgmap v2017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:35.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:35 smithi130 bash[51302]: debug 2023-08-27T23:05:35.096+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:35.102336+0000) 2023-08-27T23:05:35.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:35 smithi130 bash[27579]: audit 2023-08-27T23:05:35.270142+0000 mon.c (mon.2) 798 : audit [DBG] from='client.? 172.21.15.6:0/3656397669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:35.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:35 smithi006 bash[33918]: audit 2023-08-27T23:05:35.270142+0000 mon.c (mon.2) 798 : audit [DBG] from='client.? 172.21.15.6:0/3656397669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:35.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:35 smithi006 bash[27292]: audit 2023-08-27T23:05:35.270142+0000 mon.c (mon.2) 798 : audit [DBG] from='client.? 172.21.15.6:0/3656397669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:36.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:36 smithi130 bash[51302]: debug 2023-08-27T23:05:36.096+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:36.102620+0000) 2023-08-27T23:05:36.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:36 smithi130 bash[27579]: cluster 2023-08-27T23:05:34.527713+0000 mgr.y (mgr.14901) 2011 : cluster [DBG] pgmap v2018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:36 smithi006 bash[27292]: cluster 2023-08-27T23:05:34.527713+0000 mgr.y (mgr.14901) 2011 : cluster [DBG] pgmap v2018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:36 smithi006 bash[33918]: cluster 2023-08-27T23:05:34.527713+0000 mgr.y (mgr.14901) 2011 : cluster [DBG] pgmap v2018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:37 smithi130 bash[51302]: debug 2023-08-27T23:05:37.100+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:37.102915+0000) 2023-08-27T23:05:38.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:38 smithi130 bash[51302]: debug 2023-08-27T23:05:38.100+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:38.103205+0000) 2023-08-27T23:05:38.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:38 smithi130 bash[27579]: cluster 2023-08-27T23:05:36.528515+0000 mgr.y (mgr.14901) 2012 : cluster [DBG] pgmap v2019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:38 smithi130 bash[27579]: audit 2023-08-27T23:05:37.686219+0000 mon.a (mon.0) 1895 : audit [DBG] from='client.? 172.21.15.6:0/566653184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:38 smithi006 bash[27292]: cluster 2023-08-27T23:05:36.528515+0000 mgr.y (mgr.14901) 2012 : cluster [DBG] pgmap v2019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:38 smithi006 bash[27292]: audit 2023-08-27T23:05:37.686219+0000 mon.a (mon.0) 1895 : audit [DBG] from='client.? 172.21.15.6:0/566653184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:38 smithi006 bash[33918]: cluster 2023-08-27T23:05:36.528515+0000 mgr.y (mgr.14901) 2012 : cluster [DBG] pgmap v2019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:38.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:38 smithi006 bash[33918]: audit 2023-08-27T23:05:37.686219+0000 mon.a (mon.0) 1895 : audit [DBG] from='client.? 172.21.15.6:0/566653184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:39.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:39 smithi130 bash[51302]: debug 2023-08-27T23:05:39.100+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:39.103428+0000) 2023-08-27T23:05:40.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:40 smithi130 bash[51302]: debug 2023-08-27T23:05:40.100+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:40.103722+0000) 2023-08-27T23:05:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:40 smithi130 bash[27579]: cluster 2023-08-27T23:05:38.529244+0000 mgr.y (mgr.14901) 2013 : cluster [DBG] pgmap v2020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:40.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:40 smithi130 bash[27579]: audit 2023-08-27T23:05:39.401761+0000 mon.b (mon.1) 314 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:05:40.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:40 smithi130 bash[27579]: audit 2023-08-27T23:05:40.094055+0000 mon.c (mon.2) 799 : audit [DBG] from='client.? 172.21.15.6:0/96431580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:05:40.373Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:05:40.375Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:40 smithi006 bash[27292]: cluster 2023-08-27T23:05:38.529244+0000 mgr.y (mgr.14901) 2013 : cluster [DBG] pgmap v2020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:40 smithi006 bash[27292]: audit 2023-08-27T23:05:39.401761+0000 mon.b (mon.1) 314 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:05:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:40 smithi006 bash[27292]: audit 2023-08-27T23:05:40.094055+0000 mon.c (mon.2) 799 : audit [DBG] from='client.? 172.21.15.6:0/96431580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:40 smithi006 bash[33918]: cluster 2023-08-27T23:05:38.529244+0000 mgr.y (mgr.14901) 2013 : cluster [DBG] pgmap v2020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:40 smithi006 bash[33918]: audit 2023-08-27T23:05:39.401761+0000 mon.b (mon.1) 314 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:05:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:40 smithi006 bash[33918]: audit 2023-08-27T23:05:40.094055+0000 mon.c (mon.2) 799 : audit [DBG] from='client.? 172.21.15.6:0/96431580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:05:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:05:40] "GET /metrics HTTP/1.1" 200 193093 "" "Prometheus/2.33.4" 2023-08-27T23:05:41.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:41 smithi130 bash[51302]: debug 2023-08-27T23:05:41.100+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:41.104059+0000) 2023-08-27T23:05:42.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:41 smithi006 bash[56298]: level=error ts=2023-08-27T23:05:41.936Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:42.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:41 smithi006 bash[56298]: level=warn ts=2023-08-27T23:05:41.938Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:42.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:42 smithi130 bash[27579]: cluster 2023-08-27T23:05:40.530416+0000 mgr.y (mgr.14901) 2014 : cluster [DBG] pgmap v2021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:42.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:42 smithi130 bash[51302]: debug 2023-08-27T23:05:42.100+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:42.104293+0000) 2023-08-27T23:05:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:42 smithi006 bash[27292]: cluster 2023-08-27T23:05:40.530416+0000 mgr.y (mgr.14901) 2014 : cluster [DBG] pgmap v2021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:42.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:42 smithi006 bash[33918]: cluster 2023-08-27T23:05:40.530416+0000 mgr.y (mgr.14901) 2014 : cluster [DBG] pgmap v2021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:43.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:43 smithi130 bash[27579]: audit 2023-08-27T23:05:42.514444+0000 mon.a (mon.0) 1896 : audit [DBG] from='client.? 172.21.15.6:0/3399159560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:43.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:43 smithi130 bash[51302]: debug 2023-08-27T23:05:43.100+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:43.104550+0000) 2023-08-27T23:05:43.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:43 smithi006 bash[27292]: audit 2023-08-27T23:05:42.514444+0000 mon.a (mon.0) 1896 : audit [DBG] from='client.? 172.21.15.6:0/3399159560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:43.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:43 smithi006 bash[33918]: audit 2023-08-27T23:05:42.514444+0000 mon.a (mon.0) 1896 : audit [DBG] from='client.? 172.21.15.6:0/3399159560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:05:43.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:05:43.838Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:44.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:44 smithi130 bash[27579]: cluster 2023-08-27T23:05:42.531033+0000 mgr.y (mgr.14901) 2015 : cluster [DBG] pgmap v2022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:44.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:44 smithi130 bash[51302]: debug 2023-08-27T23:05:44.100+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:44.104830+0000) 2023-08-27T23:05:44.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:44 smithi006 bash[33918]: cluster 2023-08-27T23:05:42.531033+0000 mgr.y (mgr.14901) 2015 : cluster [DBG] pgmap v2022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:44.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:44 smithi006 bash[27292]: cluster 2023-08-27T23:05:42.531033+0000 mgr.y (mgr.14901) 2015 : cluster [DBG] pgmap v2022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:45.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:45 smithi130 bash[27579]: audit 2023-08-27T23:05:44.957649+0000 mon.c (mon.2) 800 : audit [DBG] from='client.? 172.21.15.6:0/153573717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:45.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:45 smithi130 bash[27579]: audit 2023-08-27T23:05:45.292237+0000 mon.a (mon.0) 1897 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:05:45.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:45 smithi130 bash[51302]: debug 2023-08-27T23:05:45.100+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:45.105048+0000) 2023-08-27T23:05:45.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:45 smithi006 bash[33918]: audit 2023-08-27T23:05:44.957649+0000 mon.c (mon.2) 800 : audit [DBG] from='client.? 172.21.15.6:0/153573717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:45.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:45 smithi006 bash[33918]: audit 2023-08-27T23:05:45.292237+0000 mon.a (mon.0) 1897 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:05:45.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:45 smithi006 bash[27292]: audit 2023-08-27T23:05:44.957649+0000 mon.c (mon.2) 800 : audit [DBG] from='client.? 172.21.15.6:0/153573717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:45.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:45 smithi006 bash[27292]: audit 2023-08-27T23:05:45.292237+0000 mon.a (mon.0) 1897 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:05:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:46 smithi130 bash[51302]: debug 2023-08-27T23:05:46.100+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:46.105301+0000) 2023-08-27T23:05:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:46 smithi130 bash[27579]: cluster 2023-08-27T23:05:44.532377+0000 mgr.y (mgr.14901) 2016 : cluster [DBG] pgmap v2023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:46 smithi130 bash[27579]: audit 2023-08-27T23:05:45.768770+0000 mon.a (mon.0) 1898 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:05:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:46 smithi130 bash[27579]: audit 2023-08-27T23:05:46.070459+0000 mon.b (mon.1) 315 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:05:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:46 smithi130 bash[27579]: audit 2023-08-27T23:05:46.072597+0000 mon.b (mon.1) 316 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:05:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:46 smithi130 bash[27579]: audit 2023-08-27T23:05:46.082857+0000 mon.a (mon.0) 1899 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:05:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:46 smithi006 bash[33918]: cluster 2023-08-27T23:05:44.532377+0000 mgr.y (mgr.14901) 2016 : cluster [DBG] pgmap v2023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:46 smithi006 bash[33918]: audit 2023-08-27T23:05:45.768770+0000 mon.a (mon.0) 1898 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:05:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:46 smithi006 bash[33918]: audit 2023-08-27T23:05:46.070459+0000 mon.b (mon.1) 315 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:05:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:46 smithi006 bash[33918]: audit 2023-08-27T23:05:46.072597+0000 mon.b (mon.1) 316 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:05:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:46 smithi006 bash[33918]: audit 2023-08-27T23:05:46.082857+0000 mon.a (mon.0) 1899 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:05:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:46 smithi006 bash[27292]: cluster 2023-08-27T23:05:44.532377+0000 mgr.y (mgr.14901) 2016 : cluster [DBG] pgmap v2023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:46 smithi006 bash[27292]: audit 2023-08-27T23:05:45.768770+0000 mon.a (mon.0) 1898 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:05:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:46 smithi006 bash[27292]: audit 2023-08-27T23:05:46.070459+0000 mon.b (mon.1) 315 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:05:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:46 smithi006 bash[27292]: audit 2023-08-27T23:05:46.072597+0000 mon.b (mon.1) 316 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:05:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:46 smithi006 bash[27292]: audit 2023-08-27T23:05:46.082857+0000 mon.a (mon.0) 1899 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:05:47.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:47 smithi130 bash[51302]: debug 2023-08-27T23:05:47.104+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:47.105612+0000) 2023-08-27T23:05:48.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:48 smithi130 bash[51302]: debug 2023-08-27T23:05:48.100+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:48.105908+0000) 2023-08-27T23:05:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:48 smithi130 bash[27579]: cluster 2023-08-27T23:05:46.533003+0000 mgr.y (mgr.14901) 2017 : cluster [DBG] pgmap v2024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:48 smithi130 bash[27579]: audit 2023-08-27T23:05:47.383935+0000 mon.a (mon.0) 1900 : audit [DBG] from='client.? 172.21.15.6:0/2464525501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:48 smithi006 bash[33918]: cluster 2023-08-27T23:05:46.533003+0000 mgr.y (mgr.14901) 2017 : cluster [DBG] pgmap v2024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:48 smithi006 bash[33918]: audit 2023-08-27T23:05:47.383935+0000 mon.a (mon.0) 1900 : audit [DBG] from='client.? 172.21.15.6:0/2464525501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:48 smithi006 bash[27292]: cluster 2023-08-27T23:05:46.533003+0000 mgr.y (mgr.14901) 2017 : cluster [DBG] pgmap v2024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:48 smithi006 bash[27292]: audit 2023-08-27T23:05:47.383935+0000 mon.a (mon.0) 1900 : audit [DBG] from='client.? 172.21.15.6:0/2464525501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:49.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:49 smithi130 bash[51302]: debug 2023-08-27T23:05:49.104+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:49.106174+0000) 2023-08-27T23:05:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:50 smithi130 bash[51302]: debug 2023-08-27T23:05:50.100+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:50.106475+0000) 2023-08-27T23:05:50.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:50 smithi130 bash[27579]: cluster 2023-08-27T23:05:48.533652+0000 mgr.y (mgr.14901) 2018 : cluster [DBG] pgmap v2025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:50.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:50 smithi130 bash[27579]: audit 2023-08-27T23:05:49.798094+0000 mon.a (mon.0) 1901 : audit [DBG] from='client.? 172.21.15.6:0/633061648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:50 smithi006 bash[27292]: cluster 2023-08-27T23:05:48.533652+0000 mgr.y (mgr.14901) 2018 : cluster [DBG] pgmap v2025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:50 smithi006 bash[27292]: audit 2023-08-27T23:05:49.798094+0000 mon.a (mon.0) 1901 : audit [DBG] from='client.? 172.21.15.6:0/633061648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:50 smithi006 bash[33918]: cluster 2023-08-27T23:05:48.533652+0000 mgr.y (mgr.14901) 2018 : cluster [DBG] pgmap v2025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:50 smithi006 bash[33918]: audit 2023-08-27T23:05:49.798094+0000 mon.a (mon.0) 1901 : audit [DBG] from='client.? 172.21.15.6:0/633061648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:50.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:05:50.374Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:50.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:05:50.375Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:05:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:05:50] "GET /metrics HTTP/1.1" 200 193094 "" "Prometheus/2.33.4" 2023-08-27T23:05:51.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:51 smithi130 bash[51302]: debug 2023-08-27T23:05:51.100+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:51.106761+0000) 2023-08-27T23:05:52.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:51 smithi006 bash[56298]: level=error ts=2023-08-27T23:05:51.936Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:52.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:51 smithi006 bash[56298]: level=warn ts=2023-08-27T23:05:51.938Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:52.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:52 smithi130 bash[27579]: cluster 2023-08-27T23:05:50.534763+0000 mgr.y (mgr.14901) 2019 : cluster [DBG] pgmap v2026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:52 smithi130 bash[27579]: audit 2023-08-27T23:05:52.202089+0000 mon.c (mon.2) 801 : audit [DBG] from='client.? 172.21.15.6:0/1673561826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:52.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:52 smithi130 bash[51302]: debug 2023-08-27T23:05:52.104+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:52.106952+0000) 2023-08-27T23:05:52.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:52 smithi006 bash[27292]: cluster 2023-08-27T23:05:50.534763+0000 mgr.y (mgr.14901) 2019 : cluster [DBG] pgmap v2026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:52.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:52 smithi006 bash[27292]: audit 2023-08-27T23:05:52.202089+0000 mon.c (mon.2) 801 : audit [DBG] from='client.? 172.21.15.6:0/1673561826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:52 smithi006 bash[33918]: cluster 2023-08-27T23:05:50.534763+0000 mgr.y (mgr.14901) 2019 : cluster [DBG] pgmap v2026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:52 smithi006 bash[33918]: audit 2023-08-27T23:05:52.202089+0000 mon.c (mon.2) 801 : audit [DBG] from='client.? 172.21.15.6:0/1673561826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:53.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:53 smithi130 bash[51302]: debug 2023-08-27T23:05:53.104+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:53.107220+0000) 2023-08-27T23:05:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:05:53.837Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:05:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:05:53.839Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:05:54.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:54 smithi130 bash[27579]: cluster 2023-08-27T23:05:52.535442+0000 mgr.y (mgr.14901) 2020 : cluster [DBG] pgmap v2027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:54 smithi130 bash[51302]: debug 2023-08-27T23:05:54.104+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:54.107472+0000) 2023-08-27T23:05:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:54 smithi006 bash[27292]: cluster 2023-08-27T23:05:52.535442+0000 mgr.y (mgr.14901) 2020 : cluster [DBG] pgmap v2027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:54.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:54 smithi006 bash[33918]: cluster 2023-08-27T23:05:52.535442+0000 mgr.y (mgr.14901) 2020 : cluster [DBG] pgmap v2027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:55.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:55 smithi130 bash[27579]: audit 2023-08-27T23:05:54.621874+0000 mon.c (mon.2) 802 : audit [DBG] from='client.? 172.21.15.6:0/2977371173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:55 smithi130 bash[51302]: debug 2023-08-27T23:05:55.104+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:55.107747+0000) 2023-08-27T23:05:55.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:55 smithi006 bash[27292]: audit 2023-08-27T23:05:54.621874+0000 mon.c (mon.2) 802 : audit [DBG] from='client.? 172.21.15.6:0/2977371173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:55.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:55 smithi006 bash[33918]: audit 2023-08-27T23:05:54.621874+0000 mon.c (mon.2) 802 : audit [DBG] from='client.? 172.21.15.6:0/2977371173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:56.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:56 smithi130 bash[51302]: debug 2023-08-27T23:05:56.104+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:56.108001+0000) 2023-08-27T23:05:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:56 smithi130 bash[27579]: cluster 2023-08-27T23:05:54.536581+0000 mgr.y (mgr.14901) 2021 : cluster [DBG] pgmap v2028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:56 smithi006 bash[27292]: cluster 2023-08-27T23:05:54.536581+0000 mgr.y (mgr.14901) 2021 : cluster [DBG] pgmap v2028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:56 smithi006 bash[33918]: cluster 2023-08-27T23:05:54.536581+0000 mgr.y (mgr.14901) 2021 : cluster [DBG] pgmap v2028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:57.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:57 smithi130 bash[51302]: debug 2023-08-27T23:05:57.104+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:57.108307+0000) 2023-08-27T23:05:57.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:57 smithi130 bash[27579]: audit 2023-08-27T23:05:57.035876+0000 mon.a (mon.0) 1902 : audit [DBG] from='client.? 172.21.15.6:0/2203783020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:57.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:57 smithi006 bash[27292]: audit 2023-08-27T23:05:57.035876+0000 mon.a (mon.0) 1902 : audit [DBG] from='client.? 172.21.15.6:0/2203783020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:57.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:57 smithi006 bash[33918]: audit 2023-08-27T23:05:57.035876+0000 mon.a (mon.0) 1902 : audit [DBG] from='client.? 172.21.15.6:0/2203783020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:58.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:05:58 smithi130 bash[27579]: cluster 2023-08-27T23:05:56.537200+0000 mgr.y (mgr.14901) 2022 : cluster [DBG] pgmap v2029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:58.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:58 smithi130 bash[51302]: debug 2023-08-27T23:05:58.104+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:58.108576+0000) 2023-08-27T23:05:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:05:58 smithi006 bash[33918]: cluster 2023-08-27T23:05:56.537200+0000 mgr.y (mgr.14901) 2022 : cluster [DBG] pgmap v2029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:05:58 smithi006 bash[27292]: cluster 2023-08-27T23:05:56.537200+0000 mgr.y (mgr.14901) 2022 : cluster [DBG] pgmap v2029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:05:59.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:05:59 smithi130 bash[51302]: debug 2023-08-27T23:05:59.104+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:59.108829+0000) 2023-08-27T23:06:00.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:00 smithi130 bash[51302]: debug 2023-08-27T23:06:00.104+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:00.109094+0000) 2023-08-27T23:06:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:00 smithi130 bash[27579]: cluster 2023-08-27T23:05:58.537804+0000 mgr.y (mgr.14901) 2023 : cluster [DBG] pgmap v2030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:00 smithi130 bash[27579]: audit 2023-08-27T23:05:59.468255+0000 mon.a (mon.0) 1903 : audit [DBG] from='client.? 172.21.15.6:0/2898909569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:00 smithi006 bash[33918]: cluster 2023-08-27T23:05:58.537804+0000 mgr.y (mgr.14901) 2023 : cluster [DBG] pgmap v2030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:00 smithi006 bash[33918]: audit 2023-08-27T23:05:59.468255+0000 mon.a (mon.0) 1903 : audit [DBG] from='client.? 172.21.15.6:0/2898909569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:06:00.374Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:00.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:06:00.376Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:00 smithi006 bash[27292]: cluster 2023-08-27T23:05:58.537804+0000 mgr.y (mgr.14901) 2023 : cluster [DBG] pgmap v2030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:00 smithi006 bash[27292]: audit 2023-08-27T23:05:59.468255+0000 mon.a (mon.0) 1903 : audit [DBG] from='client.? 172.21.15.6:0/2898909569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:06:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:06:00] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:06:01.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:01 smithi130 bash[51302]: debug 2023-08-27T23:06:01.108+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:01.109398+0000) 2023-08-27T23:06:02.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:01 smithi006 bash[56298]: level=error ts=2023-08-27T23:06:01.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:02.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:01 smithi006 bash[56298]: level=warn ts=2023-08-27T23:06:01.939Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:02.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:02 smithi130 bash[51302]: debug 2023-08-27T23:06:02.104+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:02.109693+0000) 2023-08-27T23:06:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:02 smithi006 bash[33918]: cluster 2023-08-27T23:06:00.538757+0000 mgr.y (mgr.14901) 2024 : cluster [DBG] pgmap v2031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:02 smithi006 bash[33918]: audit 2023-08-27T23:06:01.889044+0000 mon.a (mon.0) 1904 : audit [DBG] from='client.? 172.21.15.6:0/2413889156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:02 smithi006 bash[27292]: cluster 2023-08-27T23:06:00.538757+0000 mgr.y (mgr.14901) 2024 : cluster [DBG] pgmap v2031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:02 smithi006 bash[27292]: audit 2023-08-27T23:06:01.889044+0000 mon.a (mon.0) 1904 : audit [DBG] from='client.? 172.21.15.6:0/2413889156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:02 smithi130 bash[27579]: cluster 2023-08-27T23:06:00.538757+0000 mgr.y (mgr.14901) 2024 : cluster [DBG] pgmap v2031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:02 smithi130 bash[27579]: audit 2023-08-27T23:06:01.889044+0000 mon.a (mon.0) 1904 : audit [DBG] from='client.? 172.21.15.6:0/2413889156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:03 smithi130 bash[51302]: debug 2023-08-27T23:06:03.104+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:03.109995+0000) 2023-08-27T23:06:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:06:03.838Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:06:03.840Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:04.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:04 smithi130 bash[51302]: debug 2023-08-27T23:06:04.108+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:04.110324+0000) 2023-08-27T23:06:04.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:04 smithi006 bash[33918]: cluster 2023-08-27T23:06:02.539426+0000 mgr.y (mgr.14901) 2025 : cluster [DBG] pgmap v2032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:04.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:04 smithi006 bash[33918]: audit 2023-08-27T23:06:04.306819+0000 mon.c (mon.2) 803 : audit [DBG] from='client.? 172.21.15.6:0/3738666723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:04.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:04 smithi006 bash[27292]: cluster 2023-08-27T23:06:02.539426+0000 mgr.y (mgr.14901) 2025 : cluster [DBG] pgmap v2032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:04.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:04 smithi006 bash[27292]: audit 2023-08-27T23:06:04.306819+0000 mon.c (mon.2) 803 : audit [DBG] from='client.? 172.21.15.6:0/3738666723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:04 smithi130 bash[27579]: cluster 2023-08-27T23:06:02.539426+0000 mgr.y (mgr.14901) 2025 : cluster [DBG] pgmap v2032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:04 smithi130 bash[27579]: audit 2023-08-27T23:06:04.306819+0000 mon.c (mon.2) 803 : audit [DBG] from='client.? 172.21.15.6:0/3738666723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:05.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:05 smithi130 bash[51302]: debug 2023-08-27T23:06:05.108+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:05.110658+0000) 2023-08-27T23:06:06.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:06 smithi130 bash[51302]: debug 2023-08-27T23:06:06.108+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:06.110970+0000) 2023-08-27T23:06:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:06 smithi006 bash[33918]: cluster 2023-08-27T23:06:04.540595+0000 mgr.y (mgr.14901) 2026 : cluster [DBG] pgmap v2033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:06 smithi006 bash[27292]: cluster 2023-08-27T23:06:04.540595+0000 mgr.y (mgr.14901) 2026 : cluster [DBG] pgmap v2033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:06 smithi130 bash[27579]: cluster 2023-08-27T23:06:04.540595+0000 mgr.y (mgr.14901) 2026 : cluster [DBG] pgmap v2033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:07.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:07 smithi130 bash[51302]: debug 2023-08-27T23:06:07.108+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:07.111242+0000) 2023-08-27T23:06:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:07 smithi006 bash[33918]: audit 2023-08-27T23:06:06.726198+0000 mon.c (mon.2) 804 : audit [DBG] from='client.? 172.21.15.6:0/387095316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:07.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:07 smithi006 bash[27292]: audit 2023-08-27T23:06:06.726198+0000 mon.c (mon.2) 804 : audit [DBG] from='client.? 172.21.15.6:0/387095316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:07.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:07 smithi130 bash[27579]: audit 2023-08-27T23:06:06.726198+0000 mon.c (mon.2) 804 : audit [DBG] from='client.? 172.21.15.6:0/387095316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:08.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:08 smithi130 bash[51302]: debug 2023-08-27T23:06:08.108+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:08.111477+0000) 2023-08-27T23:06:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:08 smithi006 bash[33918]: cluster 2023-08-27T23:06:06.541409+0000 mgr.y (mgr.14901) 2027 : cluster [DBG] pgmap v2034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:08 smithi006 bash[27292]: cluster 2023-08-27T23:06:06.541409+0000 mgr.y (mgr.14901) 2027 : cluster [DBG] pgmap v2034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:08 smithi130 bash[27579]: cluster 2023-08-27T23:06:06.541409+0000 mgr.y (mgr.14901) 2027 : cluster [DBG] pgmap v2034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:09.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:09 smithi130 bash[51302]: debug 2023-08-27T23:06:09.108+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:09.111782+0000) 2023-08-27T23:06:09.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:09 smithi006 bash[33918]: audit 2023-08-27T23:06:09.146197+0000 mon.c (mon.2) 805 : audit [DBG] from='client.? 172.21.15.6:0/3173852738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:09 smithi006 bash[27292]: audit 2023-08-27T23:06:09.146197+0000 mon.c (mon.2) 805 : audit [DBG] from='client.? 172.21.15.6:0/3173852738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:09 smithi130 bash[27579]: audit 2023-08-27T23:06:09.146197+0000 mon.c (mon.2) 805 : audit [DBG] from='client.? 172.21.15.6:0/3173852738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:10.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:10 smithi130 bash[51302]: debug 2023-08-27T23:06:10.108+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:10.112041+0000) 2023-08-27T23:06:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:10 smithi006 bash[27292]: cluster 2023-08-27T23:06:08.542253+0000 mgr.y (mgr.14901) 2028 : cluster [DBG] pgmap v2035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:10 smithi006 bash[33918]: cluster 2023-08-27T23:06:08.542253+0000 mgr.y (mgr.14901) 2028 : cluster [DBG] pgmap v2035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:06:10.375Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:10.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:06:10.377Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:10 smithi130 bash[27579]: cluster 2023-08-27T23:06:08.542253+0000 mgr.y (mgr.14901) 2028 : cluster [DBG] pgmap v2035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:06:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:06:10] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:06:11.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:11 smithi130 bash[51302]: debug 2023-08-27T23:06:11.108+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:11.112301+0000) 2023-08-27T23:06:12.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:11 smithi006 bash[56298]: level=error ts=2023-08-27T23:06:11.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:12.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:11 smithi006 bash[56298]: level=warn ts=2023-08-27T23:06:11.939Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:12 smithi130 bash[51302]: debug 2023-08-27T23:06:12.108+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:12.112517+0000) 2023-08-27T23:06:12.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:12 smithi006 bash[27292]: cluster 2023-08-27T23:06:10.543446+0000 mgr.y (mgr.14901) 2029 : cluster [DBG] pgmap v2036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:12.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:12 smithi006 bash[27292]: audit 2023-08-27T23:06:11.566440+0000 mon.c (mon.2) 806 : audit [DBG] from='client.? 172.21.15.6:0/1677577642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:12.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:12 smithi006 bash[33918]: cluster 2023-08-27T23:06:10.543446+0000 mgr.y (mgr.14901) 2029 : cluster [DBG] pgmap v2036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:12.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:12 smithi006 bash[33918]: audit 2023-08-27T23:06:11.566440+0000 mon.c (mon.2) 806 : audit [DBG] from='client.? 172.21.15.6:0/1677577642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:12 smithi130 bash[27579]: cluster 2023-08-27T23:06:10.543446+0000 mgr.y (mgr.14901) 2029 : cluster [DBG] pgmap v2036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:12 smithi130 bash[27579]: audit 2023-08-27T23:06:11.566440+0000 mon.c (mon.2) 806 : audit [DBG] from='client.? 172.21.15.6:0/1677577642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:13 smithi130 bash[51302]: debug 2023-08-27T23:06:13.112+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:13.112775+0000) 2023-08-27T23:06:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:06:13.839Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:06:13.841Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:14.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:14 smithi130 bash[51302]: debug 2023-08-27T23:06:14.108+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:14.113061+0000) 2023-08-27T23:06:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:14 smithi006 bash[33918]: cluster 2023-08-27T23:06:12.544106+0000 mgr.y (mgr.14901) 2030 : cluster [DBG] pgmap v2037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:14 smithi006 bash[33918]: audit 2023-08-27T23:06:13.981319+0000 mon.a (mon.0) 1905 : audit [DBG] from='client.? 172.21.15.6:0/3867244768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:14.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:14 smithi006 bash[27292]: cluster 2023-08-27T23:06:12.544106+0000 mgr.y (mgr.14901) 2030 : cluster [DBG] pgmap v2037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:14.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:14 smithi006 bash[27292]: audit 2023-08-27T23:06:13.981319+0000 mon.a (mon.0) 1905 : audit [DBG] from='client.? 172.21.15.6:0/3867244768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:14 smithi130 bash[27579]: cluster 2023-08-27T23:06:12.544106+0000 mgr.y (mgr.14901) 2030 : cluster [DBG] pgmap v2037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:14 smithi130 bash[27579]: audit 2023-08-27T23:06:13.981319+0000 mon.a (mon.0) 1905 : audit [DBG] from='client.? 172.21.15.6:0/3867244768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:15 smithi130 bash[51302]: debug 2023-08-27T23:06:15.108+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:15.113361+0000) 2023-08-27T23:06:16.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:16 smithi130 bash[51302]: debug 2023-08-27T23:06:16.108+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:16.113637+0000) 2023-08-27T23:06:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:16 smithi006 bash[27292]: cluster 2023-08-27T23:06:14.545259+0000 mgr.y (mgr.14901) 2031 : cluster [DBG] pgmap v2038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:16 smithi006 bash[27292]: audit 2023-08-27T23:06:16.399015+0000 mon.c (mon.2) 807 : audit [DBG] from='client.? 172.21.15.6:0/1287286745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:16 smithi006 bash[33918]: cluster 2023-08-27T23:06:14.545259+0000 mgr.y (mgr.14901) 2031 : cluster [DBG] pgmap v2038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:16 smithi006 bash[33918]: audit 2023-08-27T23:06:16.399015+0000 mon.c (mon.2) 807 : audit [DBG] from='client.? 172.21.15.6:0/1287286745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:16 smithi130 bash[27579]: cluster 2023-08-27T23:06:14.545259+0000 mgr.y (mgr.14901) 2031 : cluster [DBG] pgmap v2038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:16 smithi130 bash[27579]: audit 2023-08-27T23:06:16.399015+0000 mon.c (mon.2) 807 : audit [DBG] from='client.? 172.21.15.6:0/1287286745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:17.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:17 smithi130 bash[51302]: debug 2023-08-27T23:06:17.112+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:17.113947+0000) 2023-08-27T23:06:18.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:18 smithi130 bash[51302]: debug 2023-08-27T23:06:18.108+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:18.114240+0000) 2023-08-27T23:06:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:18 smithi006 bash[33918]: cluster 2023-08-27T23:06:16.545936+0000 mgr.y (mgr.14901) 2032 : cluster [DBG] pgmap v2039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:18.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:18 smithi006 bash[27292]: cluster 2023-08-27T23:06:16.545936+0000 mgr.y (mgr.14901) 2032 : cluster [DBG] pgmap v2039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:18.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:18 smithi130 bash[27579]: cluster 2023-08-27T23:06:16.545936+0000 mgr.y (mgr.14901) 2032 : cluster [DBG] pgmap v2039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:19.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:19 smithi130 bash[51302]: debug 2023-08-27T23:06:19.108+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:19.114584+0000) 2023-08-27T23:06:19.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:19 smithi006 bash[33918]: audit 2023-08-27T23:06:18.815407+0000 mon.a (mon.0) 1906 : audit [DBG] from='client.? 172.21.15.6:0/1807014061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:19.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:19 smithi006 bash[27292]: audit 2023-08-27T23:06:18.815407+0000 mon.a (mon.0) 1906 : audit [DBG] from='client.? 172.21.15.6:0/1807014061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:19.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:19 smithi130 bash[27579]: audit 2023-08-27T23:06:18.815407+0000 mon.a (mon.0) 1906 : audit [DBG] from='client.? 172.21.15.6:0/1807014061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:20.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:20 smithi130 bash[51302]: debug 2023-08-27T23:06:20.112+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:20.114888+0000) 2023-08-27T23:06:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:20 smithi006 bash[33918]: cluster 2023-08-27T23:06:18.546735+0000 mgr.y (mgr.14901) 2033 : cluster [DBG] pgmap v2040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:20 smithi006 bash[27292]: cluster 2023-08-27T23:06:18.546735+0000 mgr.y (mgr.14901) 2033 : cluster [DBG] pgmap v2040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:20.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:06:20.376Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:20.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:06:20.378Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:20 smithi130 bash[27579]: cluster 2023-08-27T23:06:18.546735+0000 mgr.y (mgr.14901) 2033 : cluster [DBG] pgmap v2040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:06:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:06:20] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:06:21.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:21 smithi130 bash[51302]: debug 2023-08-27T23:06:21.112+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:21.115161+0000) 2023-08-27T23:06:21.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:21 smithi006 bash[33918]: audit 2023-08-27T23:06:21.227646+0000 mon.c (mon.2) 808 : audit [DBG] from='client.? 172.21.15.6:0/1960799588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:21.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:21 smithi006 bash[27292]: audit 2023-08-27T23:06:21.227646+0000 mon.c (mon.2) 808 : audit [DBG] from='client.? 172.21.15.6:0/1960799588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:21.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:21 smithi130 bash[27579]: audit 2023-08-27T23:06:21.227646+0000 mon.c (mon.2) 808 : audit [DBG] from='client.? 172.21.15.6:0/1960799588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:22.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:21 smithi006 bash[56298]: level=error ts=2023-08-27T23:06:21.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:22.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:21 smithi006 bash[56298]: level=warn ts=2023-08-27T23:06:21.940Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:22.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:22 smithi130 bash[51302]: debug 2023-08-27T23:06:22.112+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:22.115395+0000) 2023-08-27T23:06:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:22 smithi006 bash[27292]: cluster 2023-08-27T23:06:20.547885+0000 mgr.y (mgr.14901) 2034 : cluster [DBG] pgmap v2041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:22 smithi006 bash[33918]: cluster 2023-08-27T23:06:20.547885+0000 mgr.y (mgr.14901) 2034 : cluster [DBG] pgmap v2041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:22 smithi130 bash[27579]: cluster 2023-08-27T23:06:20.547885+0000 mgr.y (mgr.14901) 2034 : cluster [DBG] pgmap v2041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:23.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:23 smithi130 bash[51302]: debug 2023-08-27T23:06:23.112+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:23.115708+0000) 2023-08-27T23:06:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:06:23.840Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:06:23.842Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:24.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:24 smithi130 bash[51302]: debug 2023-08-27T23:06:24.112+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:24.116021+0000) 2023-08-27T23:06:24.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:24 smithi006 bash[33918]: cluster 2023-08-27T23:06:22.548547+0000 mgr.y (mgr.14901) 2035 : cluster [DBG] pgmap v2042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:24.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:24 smithi006 bash[33918]: audit 2023-08-27T23:06:23.652429+0000 mon.a (mon.0) 1907 : audit [DBG] from='client.? 172.21.15.6:0/3467239926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:24.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:24 smithi006 bash[27292]: cluster 2023-08-27T23:06:22.548547+0000 mgr.y (mgr.14901) 2035 : cluster [DBG] pgmap v2042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:24.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:24 smithi006 bash[27292]: audit 2023-08-27T23:06:23.652429+0000 mon.a (mon.0) 1907 : audit [DBG] from='client.? 172.21.15.6:0/3467239926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:24 smithi130 bash[27579]: cluster 2023-08-27T23:06:22.548547+0000 mgr.y (mgr.14901) 2035 : cluster [DBG] pgmap v2042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:24 smithi130 bash[27579]: audit 2023-08-27T23:06:23.652429+0000 mon.a (mon.0) 1907 : audit [DBG] from='client.? 172.21.15.6:0/3467239926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:25.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:25 smithi130 bash[51302]: debug 2023-08-27T23:06:25.115+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:25.116354+0000) 2023-08-27T23:06:26.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:26 smithi130 bash[51302]: debug 2023-08-27T23:06:26.115+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:26.116646+0000) 2023-08-27T23:06:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:26 smithi006 bash[33918]: cluster 2023-08-27T23:06:24.549760+0000 mgr.y (mgr.14901) 2036 : cluster [DBG] pgmap v2043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:26 smithi006 bash[33918]: audit 2023-08-27T23:06:26.073502+0000 mon.a (mon.0) 1908 : audit [DBG] from='client.? 172.21.15.6:0/1985892789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:26 smithi006 bash[27292]: cluster 2023-08-27T23:06:24.549760+0000 mgr.y (mgr.14901) 2036 : cluster [DBG] pgmap v2043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:26 smithi006 bash[27292]: audit 2023-08-27T23:06:26.073502+0000 mon.a (mon.0) 1908 : audit [DBG] from='client.? 172.21.15.6:0/1985892789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:26 smithi130 bash[27579]: cluster 2023-08-27T23:06:24.549760+0000 mgr.y (mgr.14901) 2036 : cluster [DBG] pgmap v2043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:26 smithi130 bash[27579]: audit 2023-08-27T23:06:26.073502+0000 mon.a (mon.0) 1908 : audit [DBG] from='client.? 172.21.15.6:0/1985892789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:27.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:27 smithi130 bash[51302]: debug 2023-08-27T23:06:27.111+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:27.116936+0000) 2023-08-27T23:06:28.460 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:28 smithi130 bash[51302]: debug 2023-08-27T23:06:28.111+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:28.117230+0000) 2023-08-27T23:06:28.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:28 smithi006 bash[27292]: cluster 2023-08-27T23:06:26.550413+0000 mgr.y (mgr.14901) 2037 : cluster [DBG] pgmap v2044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:28 smithi006 bash[33918]: cluster 2023-08-27T23:06:26.550413+0000 mgr.y (mgr.14901) 2037 : cluster [DBG] pgmap v2044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:28 smithi130 bash[27579]: cluster 2023-08-27T23:06:26.550413+0000 mgr.y (mgr.14901) 2037 : cluster [DBG] pgmap v2044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:29.469 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:29 smithi130 bash[51302]: debug 2023-08-27T23:06:29.115+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:29.117580+0000) 2023-08-27T23:06:29.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:29 smithi130 bash[27579]: audit 2023-08-27T23:06:28.481287+0000 mon.c (mon.2) 809 : audit [DBG] from='client.? 172.21.15.6:0/1327600470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:29.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:29 smithi006 bash[27292]: audit 2023-08-27T23:06:28.481287+0000 mon.c (mon.2) 809 : audit [DBG] from='client.? 172.21.15.6:0/1327600470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:29.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:29 smithi006 bash[33918]: audit 2023-08-27T23:06:28.481287+0000 mon.c (mon.2) 809 : audit [DBG] from='client.? 172.21.15.6:0/1327600470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:30.471 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:30 smithi130 bash[51302]: debug 2023-08-27T23:06:30.111+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:30.117854+0000) 2023-08-27T23:06:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:30 smithi006 bash[27292]: cluster 2023-08-27T23:06:28.551167+0000 mgr.y (mgr.14901) 2038 : cluster [DBG] pgmap v2045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:06:30.377Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:06:30.378Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:30 smithi006 bash[33918]: cluster 2023-08-27T23:06:28.551167+0000 mgr.y (mgr.14901) 2038 : cluster [DBG] pgmap v2045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:30.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:30 smithi130 bash[27579]: cluster 2023-08-27T23:06:28.551167+0000 mgr.y (mgr.14901) 2038 : cluster [DBG] pgmap v2045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:06:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:06:30] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:06:31.481 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:31 smithi130 bash[51302]: debug 2023-08-27T23:06:31.115+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:31.118161+0000) 2023-08-27T23:06:31.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:31 smithi130 bash[27579]: audit 2023-08-27T23:06:30.892454+0000 mon.a (mon.0) 1909 : audit [DBG] from='client.? 172.21.15.6:0/3741560756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:31.941 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:31 smithi006 bash[33918]: audit 2023-08-27T23:06:30.892454+0000 mon.a (mon.0) 1909 : audit [DBG] from='client.? 172.21.15.6:0/3741560756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:31.941 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:31 smithi006 bash[27292]: audit 2023-08-27T23:06:30.892454+0000 mon.a (mon.0) 1909 : audit [DBG] from='client.? 172.21.15.6:0/3741560756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:32.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:31 smithi006 bash[56298]: level=error ts=2023-08-27T23:06:31.938Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:32.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:31 smithi006 bash[56298]: level=warn ts=2023-08-27T23:06:31.940Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:32.484 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:32 smithi130 bash[51302]: debug 2023-08-27T23:06:32.111+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:32.118377+0000) 2023-08-27T23:06:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:32 smithi130 bash[27579]: cluster 2023-08-27T23:06:30.552362+0000 mgr.y (mgr.14901) 2039 : cluster [DBG] pgmap v2046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:32 smithi006 bash[33918]: cluster 2023-08-27T23:06:30.552362+0000 mgr.y (mgr.14901) 2039 : cluster [DBG] pgmap v2046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:32.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:32 smithi006 bash[27292]: cluster 2023-08-27T23:06:30.552362+0000 mgr.y (mgr.14901) 2039 : cluster [DBG] pgmap v2046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:33.493 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:33 smithi130 bash[51302]: debug 2023-08-27T23:06:33.115+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:33.118645+0000) 2023-08-27T23:06:33.843 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:33 smithi006 bash[27292]: audit 2023-08-27T23:06:33.305837+0000 mon.c (mon.2) 810 : audit [DBG] from='client.? 172.21.15.6:0/1180881577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:33.843 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:33 smithi006 bash[33918]: audit 2023-08-27T23:06:33.305837+0000 mon.c (mon.2) 810 : audit [DBG] from='client.? 172.21.15.6:0/1180881577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:33 smithi130 bash[27579]: audit 2023-08-27T23:06:33.305837+0000 mon.c (mon.2) 810 : audit [DBG] from='client.? 172.21.15.6:0/1180881577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:06:33.840Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:06:33.842Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:34.498 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:34 smithi130 bash[51302]: debug 2023-08-27T23:06:34.115+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:34.118950+0000) 2023-08-27T23:06:34.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:34 smithi130 bash[27579]: cluster 2023-08-27T23:06:32.553100+0000 mgr.y (mgr.14901) 2040 : cluster [DBG] pgmap v2047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:34.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:34 smithi006 bash[33918]: cluster 2023-08-27T23:06:32.553100+0000 mgr.y (mgr.14901) 2040 : cluster [DBG] pgmap v2047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:34.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:34 smithi006 bash[27292]: cluster 2023-08-27T23:06:32.553100+0000 mgr.y (mgr.14901) 2040 : cluster [DBG] pgmap v2047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:35.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:35 smithi130 bash[51302]: debug 2023-08-27T23:06:35.115+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:35.119178+0000) 2023-08-27T23:06:36.503 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:36 smithi130 bash[51302]: debug 2023-08-27T23:06:36.115+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:36.119484+0000) 2023-08-27T23:06:36.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:36 smithi130 bash[27579]: cluster 2023-08-27T23:06:34.554241+0000 mgr.y (mgr.14901) 2041 : cluster [DBG] pgmap v2048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:36 smithi130 bash[27579]: audit 2023-08-27T23:06:35.712424+0000 mon.c (mon.2) 811 : audit [DBG] from='client.? 172.21.15.6:0/1118945685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:36 smithi006 bash[33918]: cluster 2023-08-27T23:06:34.554241+0000 mgr.y (mgr.14901) 2041 : cluster [DBG] pgmap v2048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:36 smithi006 bash[33918]: audit 2023-08-27T23:06:35.712424+0000 mon.c (mon.2) 811 : audit [DBG] from='client.? 172.21.15.6:0/1118945685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:36 smithi006 bash[27292]: cluster 2023-08-27T23:06:34.554241+0000 mgr.y (mgr.14901) 2041 : cluster [DBG] pgmap v2048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:36 smithi006 bash[27292]: audit 2023-08-27T23:06:35.712424+0000 mon.c (mon.2) 811 : audit [DBG] from='client.? 172.21.15.6:0/1118945685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:37.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:37 smithi130 bash[51302]: debug 2023-08-27T23:06:37.115+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:37.119770+0000) 2023-08-27T23:06:38.508 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:38 smithi130 bash[51302]: debug 2023-08-27T23:06:38.115+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:38.120098+0000) 2023-08-27T23:06:38.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:38 smithi130 bash[27579]: cluster 2023-08-27T23:06:36.554859+0000 mgr.y (mgr.14901) 2042 : cluster [DBG] pgmap v2049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:38 smithi130 bash[27579]: audit 2023-08-27T23:06:38.131354+0000 mon.c (mon.2) 812 : audit [DBG] from='client.? 172.21.15.6:0/2585231848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:38 smithi006 bash[33918]: cluster 2023-08-27T23:06:36.554859+0000 mgr.y (mgr.14901) 2042 : cluster [DBG] pgmap v2049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:38 smithi006 bash[33918]: audit 2023-08-27T23:06:38.131354+0000 mon.c (mon.2) 812 : audit [DBG] from='client.? 172.21.15.6:0/2585231848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:38 smithi006 bash[27292]: cluster 2023-08-27T23:06:36.554859+0000 mgr.y (mgr.14901) 2042 : cluster [DBG] pgmap v2049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:38.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:38 smithi006 bash[27292]: audit 2023-08-27T23:06:38.131354+0000 mon.c (mon.2) 812 : audit [DBG] from='client.? 172.21.15.6:0/2585231848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:39.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:39 smithi130 bash[51302]: debug 2023-08-27T23:06:39.115+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:39.120365+0000) 2023-08-27T23:06:40.513 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:40 smithi130 bash[51302]: debug 2023-08-27T23:06:40.115+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:40.120672+0000) 2023-08-27T23:06:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:40 smithi006 bash[33918]: cluster 2023-08-27T23:06:38.555411+0000 mgr.y (mgr.14901) 2043 : cluster [DBG] pgmap v2050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:40 smithi006 bash[27292]: cluster 2023-08-27T23:06:38.555411+0000 mgr.y (mgr.14901) 2043 : cluster [DBG] pgmap v2050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:06:40.377Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:40.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:06:40.379Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:40 smithi130 bash[27579]: cluster 2023-08-27T23:06:38.555411+0000 mgr.y (mgr.14901) 2043 : cluster [DBG] pgmap v2050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:06:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:06:40] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:06:41.521 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:41 smithi130 bash[51302]: debug 2023-08-27T23:06:41.115+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:41.120988+0000) 2023-08-27T23:06:41.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:41 smithi130 bash[27579]: audit 2023-08-27T23:06:40.552141+0000 mon.a (mon.0) 1910 : audit [DBG] from='client.? 172.21.15.6:0/2419595387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:41.942 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:41 smithi006 bash[27292]: audit 2023-08-27T23:06:40.552141+0000 mon.a (mon.0) 1910 : audit [DBG] from='client.? 172.21.15.6:0/2419595387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:41.942 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:41 smithi006 bash[33918]: audit 2023-08-27T23:06:40.552141+0000 mon.a (mon.0) 1910 : audit [DBG] from='client.? 172.21.15.6:0/2419595387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:42.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:41 smithi006 bash[56298]: level=error ts=2023-08-27T23:06:41.939Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:42.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:41 smithi006 bash[56298]: level=warn ts=2023-08-27T23:06:41.941Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:42.524 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:42 smithi130 bash[51302]: debug 2023-08-27T23:06:42.115+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:42.121258+0000) 2023-08-27T23:06:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:42 smithi130 bash[27579]: cluster 2023-08-27T23:06:40.556419+0000 mgr.y (mgr.14901) 2044 : cluster [DBG] pgmap v2051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:42 smithi006 bash[27292]: cluster 2023-08-27T23:06:40.556419+0000 mgr.y (mgr.14901) 2044 : cluster [DBG] pgmap v2051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:42 smithi006 bash[33918]: cluster 2023-08-27T23:06:40.556419+0000 mgr.y (mgr.14901) 2044 : cluster [DBG] pgmap v2051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:43.532 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:43 smithi130 bash[51302]: debug 2023-08-27T23:06:43.115+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:43.121508+0000) 2023-08-27T23:06:43.844 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:43 smithi006 bash[33918]: audit 2023-08-27T23:06:42.967160+0000 mon.c (mon.2) 813 : audit [DBG] from='client.? 172.21.15.6:0/1213105785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:43.844 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:43 smithi006 bash[27292]: audit 2023-08-27T23:06:42.967160+0000 mon.c (mon.2) 813 : audit [DBG] from='client.? 172.21.15.6:0/1213105785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:43.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:43 smithi130 bash[27579]: audit 2023-08-27T23:06:42.967160+0000 mon.c (mon.2) 813 : audit [DBG] from='client.? 172.21.15.6:0/1213105785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:06:43.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:06:43.843Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:44.535 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:44 smithi130 bash[51302]: debug 2023-08-27T23:06:44.115+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:44.121824+0000) 2023-08-27T23:06:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:44 smithi130 bash[27579]: cluster 2023-08-27T23:06:42.557265+0000 mgr.y (mgr.14901) 2045 : cluster [DBG] pgmap v2052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:44.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:44 smithi006 bash[27292]: cluster 2023-08-27T23:06:42.557265+0000 mgr.y (mgr.14901) 2045 : cluster [DBG] pgmap v2052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:44 smithi006 bash[33918]: cluster 2023-08-27T23:06:42.557265+0000 mgr.y (mgr.14901) 2045 : cluster [DBG] pgmap v2052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:45.545 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:45 smithi130 bash[51302]: debug 2023-08-27T23:06:45.115+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:45.122088+0000) 2023-08-27T23:06:45.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:45 smithi130 bash[27579]: cluster 2023-08-27T23:06:44.558413+0000 mgr.y (mgr.14901) 2046 : cluster [DBG] pgmap v2053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:45.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:45 smithi130 bash[27579]: audit 2023-08-27T23:06:45.385501+0000 mon.a (mon.0) 1911 : audit [DBG] from='client.? 172.21.15.6:0/1704076015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:45.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:45 smithi006 bash[33918]: cluster 2023-08-27T23:06:44.558413+0000 mgr.y (mgr.14901) 2046 : cluster [DBG] pgmap v2053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:45.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:45 smithi006 bash[33918]: audit 2023-08-27T23:06:45.385501+0000 mon.a (mon.0) 1911 : audit [DBG] from='client.? 172.21.15.6:0/1704076015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:45.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:45 smithi006 bash[27292]: cluster 2023-08-27T23:06:44.558413+0000 mgr.y (mgr.14901) 2046 : cluster [DBG] pgmap v2053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:45.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:45 smithi006 bash[27292]: audit 2023-08-27T23:06:45.385501+0000 mon.a (mon.0) 1911 : audit [DBG] from='client.? 172.21.15.6:0/1704076015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:46.547 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:46 smithi130 bash[51302]: debug 2023-08-27T23:06:46.115+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:46.122302+0000) 2023-08-27T23:06:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:46 smithi130 bash[27579]: audit 2023-08-27T23:06:46.093770+0000 mon.b (mon.1) 317 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:06:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:46 smithi006 bash[33918]: audit 2023-08-27T23:06:46.093770+0000 mon.b (mon.1) 317 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:06:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:46 smithi006 bash[27292]: audit 2023-08-27T23:06:46.093770+0000 mon.b (mon.1) 317 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:06:47.554 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:47 smithi130 bash[51302]: debug 2023-08-27T23:06:47.115+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:47.122566+0000) 2023-08-27T23:06:47.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:47 smithi130 bash[27579]: cluster 2023-08-27T23:06:46.559098+0000 mgr.y (mgr.14901) 2047 : cluster [DBG] pgmap v2054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:47.976 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:47 smithi006 bash[33918]: cluster 2023-08-27T23:06:46.559098+0000 mgr.y (mgr.14901) 2047 : cluster [DBG] pgmap v2054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:47.976 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:47 smithi006 bash[27292]: cluster 2023-08-27T23:06:46.559098+0000 mgr.y (mgr.14901) 2047 : cluster [DBG] pgmap v2054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:48.558 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:48 smithi130 bash[51302]: debug 2023-08-27T23:06:48.119+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:48.122877+0000) 2023-08-27T23:06:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:48 smithi130 bash[27579]: audit 2023-08-27T23:06:47.812412+0000 mon.a (mon.0) 1912 : audit [DBG] from='client.? 172.21.15.6:0/3131168630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:48 smithi006 bash[33918]: audit 2023-08-27T23:06:47.812412+0000 mon.a (mon.0) 1912 : audit [DBG] from='client.? 172.21.15.6:0/3131168630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:48 smithi006 bash[27292]: audit 2023-08-27T23:06:47.812412+0000 mon.a (mon.0) 1912 : audit [DBG] from='client.? 172.21.15.6:0/3131168630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:49.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:49 smithi130 bash[51302]: debug 2023-08-27T23:06:49.119+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:49.123151+0000) 2023-08-27T23:06:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:50 smithi130 bash[27579]: cluster 2023-08-27T23:06:48.559697+0000 mgr.y (mgr.14901) 2048 : cluster [DBG] pgmap v2055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:50 smithi130 bash[27579]: audit 2023-08-27T23:06:50.232669+0000 mon.c (mon.2) 814 : audit [DBG] from='client.? 172.21.15.6:0/3980056381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:50.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:50 smithi130 bash[51302]: debug 2023-08-27T23:06:50.119+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:50.123373+0000) 2023-08-27T23:06:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:50 smithi006 bash[27292]: cluster 2023-08-27T23:06:48.559697+0000 mgr.y (mgr.14901) 2048 : cluster [DBG] pgmap v2055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:50 smithi006 bash[27292]: audit 2023-08-27T23:06:50.232669+0000 mon.c (mon.2) 814 : audit [DBG] from='client.? 172.21.15.6:0/3980056381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:06:50.378Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:06:50.380Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:50 smithi006 bash[33918]: cluster 2023-08-27T23:06:48.559697+0000 mgr.y (mgr.14901) 2048 : cluster [DBG] pgmap v2055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:50 smithi006 bash[33918]: audit 2023-08-27T23:06:50.232669+0000 mon.c (mon.2) 814 : audit [DBG] from='client.? 172.21.15.6:0/3980056381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:06:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:06:50] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:06:51.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:51 smithi130 bash[51302]: debug 2023-08-27T23:06:51.119+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:51.123645+0000) 2023-08-27T23:06:52.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:51 smithi006 bash[56298]: level=error ts=2023-08-27T23:06:51.940Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:52.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:51 smithi006 bash[56298]: level=warn ts=2023-08-27T23:06:51.941Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:52.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:52 smithi130 bash[27579]: cluster 2023-08-27T23:06:50.560794+0000 mgr.y (mgr.14901) 2049 : cluster [DBG] pgmap v2056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:52 smithi130 bash[27579]: audit 2023-08-27T23:06:52.022597+0000 mon.a (mon.0) 1913 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:06:52.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:52 smithi130 bash[51302]: debug 2023-08-27T23:06:52.119+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:52.123831+0000) 2023-08-27T23:06:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:52 smithi006 bash[33918]: cluster 2023-08-27T23:06:50.560794+0000 mgr.y (mgr.14901) 2049 : cluster [DBG] pgmap v2056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:52 smithi006 bash[33918]: audit 2023-08-27T23:06:52.022597+0000 mon.a (mon.0) 1913 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:06:52.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:52 smithi006 bash[27292]: cluster 2023-08-27T23:06:50.560794+0000 mgr.y (mgr.14901) 2049 : cluster [DBG] pgmap v2056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:52.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:52 smithi006 bash[27292]: audit 2023-08-27T23:06:52.022597+0000 mon.a (mon.0) 1913 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:06:53.397 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:53 smithi130 bash[51302]: debug 2023-08-27T23:06:53.123+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:53.124077+0000) 2023-08-27T23:06:53.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:53 smithi006 bash[27292]: audit 2023-08-27T23:06:52.386583+0000 mon.a (mon.0) 1914 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:06:53.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:53 smithi006 bash[27292]: audit 2023-08-27T23:06:52.636532+0000 mon.a (mon.0) 1915 : audit [DBG] from='client.? 172.21.15.6:0/4279404109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:53.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:53 smithi006 bash[27292]: audit 2023-08-27T23:06:52.654207+0000 mon.b (mon.1) 318 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:06:53.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:53 smithi006 bash[27292]: audit 2023-08-27T23:06:52.656309+0000 mon.b (mon.1) 319 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:06:53.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:53 smithi006 bash[27292]: audit 2023-08-27T23:06:52.662440+0000 mon.a (mon.0) 1916 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:06:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:53 smithi006 bash[33918]: audit 2023-08-27T23:06:52.386583+0000 mon.a (mon.0) 1914 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:06:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:53 smithi006 bash[33918]: audit 2023-08-27T23:06:52.636532+0000 mon.a (mon.0) 1915 : audit [DBG] from='client.? 172.21.15.6:0/4279404109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:53 smithi006 bash[33918]: audit 2023-08-27T23:06:52.654207+0000 mon.b (mon.1) 318 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:06:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:53 smithi006 bash[33918]: audit 2023-08-27T23:06:52.656309+0000 mon.b (mon.1) 319 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:06:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:53 smithi006 bash[33918]: audit 2023-08-27T23:06:52.662440+0000 mon.a (mon.0) 1916 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:06:53.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:53 smithi130 bash[27579]: audit 2023-08-27T23:06:52.386583+0000 mon.a (mon.0) 1914 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:06:53.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:53 smithi130 bash[27579]: audit 2023-08-27T23:06:52.636532+0000 mon.a (mon.0) 1915 : audit [DBG] from='client.? 172.21.15.6:0/4279404109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:53.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:53 smithi130 bash[27579]: audit 2023-08-27T23:06:52.654207+0000 mon.b (mon.1) 318 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:06:53.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:53 smithi130 bash[27579]: audit 2023-08-27T23:06:52.656309+0000 mon.b (mon.1) 319 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:06:53.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:53 smithi130 bash[27579]: audit 2023-08-27T23:06:52.662440+0000 mon.a (mon.0) 1916 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:06:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:06:53.842Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:06:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:06:53.844Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:06:54.400 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:54 smithi130 bash[51302]: debug 2023-08-27T23:06:54.119+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:54.124400+0000) 2023-08-27T23:06:54.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:54 smithi006 bash[27292]: cluster 2023-08-27T23:06:52.561404+0000 mgr.y (mgr.14901) 2050 : cluster [DBG] pgmap v2057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:54.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:54 smithi006 bash[33918]: cluster 2023-08-27T23:06:52.561404+0000 mgr.y (mgr.14901) 2050 : cluster [DBG] pgmap v2057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:54 smithi130 bash[27579]: cluster 2023-08-27T23:06:52.561404+0000 mgr.y (mgr.14901) 2050 : cluster [DBG] pgmap v2057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:55.410 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:55 smithi130 bash[51302]: debug 2023-08-27T23:06:55.123+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:55.124669+0000) 2023-08-27T23:06:55.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:55 smithi006 bash[33918]: audit 2023-08-27T23:06:55.057154+0000 mon.c (mon.2) 815 : audit [DBG] from='client.? 172.21.15.6:0/449305722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:55.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:55 smithi006 bash[27292]: audit 2023-08-27T23:06:55.057154+0000 mon.c (mon.2) 815 : audit [DBG] from='client.? 172.21.15.6:0/449305722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:55.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:55 smithi130 bash[27579]: audit 2023-08-27T23:06:55.057154+0000 mon.c (mon.2) 815 : audit [DBG] from='client.? 172.21.15.6:0/449305722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:56.414 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:56 smithi130 bash[51302]: debug 2023-08-27T23:06:56.119+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:56.124963+0000) 2023-08-27T23:06:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:56 smithi006 bash[33918]: cluster 2023-08-27T23:06:54.562728+0000 mgr.y (mgr.14901) 2051 : cluster [DBG] pgmap v2058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:56 smithi006 bash[27292]: cluster 2023-08-27T23:06:54.562728+0000 mgr.y (mgr.14901) 2051 : cluster [DBG] pgmap v2058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:56.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:56 smithi130 bash[27579]: cluster 2023-08-27T23:06:54.562728+0000 mgr.y (mgr.14901) 2051 : cluster [DBG] pgmap v2058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:57.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:57 smithi130 bash[51302]: debug 2023-08-27T23:06:57.119+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:57.125280+0000) 2023-08-27T23:06:58.421 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:58 smithi130 bash[51302]: debug 2023-08-27T23:06:58.119+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:58.125554+0000) 2023-08-27T23:06:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:58 smithi006 bash[33918]: cluster 2023-08-27T23:06:56.563420+0000 mgr.y (mgr.14901) 2052 : cluster [DBG] pgmap v2059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:06:58 smithi006 bash[33918]: audit 2023-08-27T23:06:57.471528+0000 mon.a (mon.0) 1917 : audit [DBG] from='client.? 172.21.15.6:0/3335459719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:58 smithi006 bash[27292]: cluster 2023-08-27T23:06:56.563420+0000 mgr.y (mgr.14901) 2052 : cluster [DBG] pgmap v2059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:06:58 smithi006 bash[27292]: audit 2023-08-27T23:06:57.471528+0000 mon.a (mon.0) 1917 : audit [DBG] from='client.? 172.21.15.6:0/3335459719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:58 smithi130 bash[27579]: cluster 2023-08-27T23:06:56.563420+0000 mgr.y (mgr.14901) 2052 : cluster [DBG] pgmap v2059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:06:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:06:58 smithi130 bash[27579]: audit 2023-08-27T23:06:57.471528+0000 mon.a (mon.0) 1917 : audit [DBG] from='client.? 172.21.15.6:0/3335459719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:59.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:06:59 smithi130 bash[51302]: debug 2023-08-27T23:06:59.119+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:59.125826+0000) 2023-08-27T23:07:00.426 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:00 smithi130 bash[51302]: debug 2023-08-27T23:07:00.119+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:00.126121+0000) 2023-08-27T23:07:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:00 smithi006 bash[33918]: cluster 2023-08-27T23:06:58.564194+0000 mgr.y (mgr.14901) 2053 : cluster [DBG] pgmap v2060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:00 smithi006 bash[33918]: audit 2023-08-27T23:06:59.886376+0000 mon.a (mon.0) 1918 : audit [DBG] from='client.? 172.21.15.6:0/3237772245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:00 smithi006 bash[27292]: cluster 2023-08-27T23:06:58.564194+0000 mgr.y (mgr.14901) 2053 : cluster [DBG] pgmap v2060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:00 smithi006 bash[27292]: audit 2023-08-27T23:06:59.886376+0000 mon.a (mon.0) 1918 : audit [DBG] from='client.? 172.21.15.6:0/3237772245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:00.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:07:00.378Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:00.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:07:00.380Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:00 smithi130 bash[27579]: cluster 2023-08-27T23:06:58.564194+0000 mgr.y (mgr.14901) 2053 : cluster [DBG] pgmap v2060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:00 smithi130 bash[27579]: audit 2023-08-27T23:06:59.886376+0000 mon.a (mon.0) 1918 : audit [DBG] from='client.? 172.21.15.6:0/3237772245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:07:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:07:00] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:07:01.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:01 smithi130 bash[51302]: debug 2023-08-27T23:07:01.119+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:01.126431+0000) 2023-08-27T23:07:02.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:01 smithi006 bash[56298]: level=error ts=2023-08-27T23:07:01.941Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:02.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:01 smithi006 bash[56298]: level=warn ts=2023-08-27T23:07:01.943Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:02.431 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:02 smithi130 bash[51302]: debug 2023-08-27T23:07:02.119+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:02.126676+0000) 2023-08-27T23:07:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:02 smithi006 bash[27292]: cluster 2023-08-27T23:07:00.565255+0000 mgr.y (mgr.14901) 2054 : cluster [DBG] pgmap v2061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:02 smithi006 bash[27292]: audit 2023-08-27T23:07:02.302194+0000 mon.c (mon.2) 816 : audit [DBG] from='client.? 172.21.15.6:0/1638289466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:02 smithi006 bash[33918]: cluster 2023-08-27T23:07:00.565255+0000 mgr.y (mgr.14901) 2054 : cluster [DBG] pgmap v2061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:02 smithi006 bash[33918]: audit 2023-08-27T23:07:02.302194+0000 mon.c (mon.2) 816 : audit [DBG] from='client.? 172.21.15.6:0/1638289466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:02 smithi130 bash[27579]: cluster 2023-08-27T23:07:00.565255+0000 mgr.y (mgr.14901) 2054 : cluster [DBG] pgmap v2061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:02 smithi130 bash[27579]: audit 2023-08-27T23:07:02.302194+0000 mon.c (mon.2) 816 : audit [DBG] from='client.? 172.21.15.6:0/1638289466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:03.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:03 smithi130 bash[51302]: debug 2023-08-27T23:07:03.123+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:03.127068+0000) 2023-08-27T23:07:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:07:03.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:07:03.845Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:04.436 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:04 smithi130 bash[51302]: debug 2023-08-27T23:07:04.123+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:04.127325+0000) 2023-08-27T23:07:04.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:04 smithi006 bash[27292]: cluster 2023-08-27T23:07:02.565891+0000 mgr.y (mgr.14901) 2055 : cluster [DBG] pgmap v2062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:04.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:04 smithi006 bash[33918]: cluster 2023-08-27T23:07:02.565891+0000 mgr.y (mgr.14901) 2055 : cluster [DBG] pgmap v2062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:04 smithi130 bash[27579]: cluster 2023-08-27T23:07:02.565891+0000 mgr.y (mgr.14901) 2055 : cluster [DBG] pgmap v2062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:05.444 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:05 smithi130 bash[51302]: debug 2023-08-27T23:07:05.123+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:05.127575+0000) 2023-08-27T23:07:05.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:05 smithi006 bash[27292]: audit 2023-08-27T23:07:04.715423+0000 mon.c (mon.2) 817 : audit [DBG] from='client.? 172.21.15.6:0/675879142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:05.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:05 smithi006 bash[33918]: audit 2023-08-27T23:07:04.715423+0000 mon.c (mon.2) 817 : audit [DBG] from='client.? 172.21.15.6:0/675879142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:05.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:05 smithi130 bash[27579]: audit 2023-08-27T23:07:04.715423+0000 mon.c (mon.2) 817 : audit [DBG] from='client.? 172.21.15.6:0/675879142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:06.447 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:06 smithi130 bash[51302]: debug 2023-08-27T23:07:06.123+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:06.127828+0000) 2023-08-27T23:07:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:06 smithi006 bash[33918]: cluster 2023-08-27T23:07:04.566934+0000 mgr.y (mgr.14901) 2056 : cluster [DBG] pgmap v2063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:06 smithi006 bash[27292]: cluster 2023-08-27T23:07:04.566934+0000 mgr.y (mgr.14901) 2056 : cluster [DBG] pgmap v2063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:06 smithi130 bash[27579]: cluster 2023-08-27T23:07:04.566934+0000 mgr.y (mgr.14901) 2056 : cluster [DBG] pgmap v2063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:07.455 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:07 smithi130 bash[51302]: debug 2023-08-27T23:07:07.123+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:07.128050+0000) 2023-08-27T23:07:07.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:07 smithi006 bash[27292]: audit 2023-08-27T23:07:07.132527+0000 mon.c (mon.2) 818 : audit [DBG] from='client.? 172.21.15.6:0/1303506288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:07 smithi006 bash[33918]: audit 2023-08-27T23:07:07.132527+0000 mon.c (mon.2) 818 : audit [DBG] from='client.? 172.21.15.6:0/1303506288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:07.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:07 smithi130 bash[27579]: audit 2023-08-27T23:07:07.132527+0000 mon.c (mon.2) 818 : audit [DBG] from='client.? 172.21.15.6:0/1303506288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:08.458 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:08 smithi130 bash[51302]: debug 2023-08-27T23:07:08.127+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:08.128273+0000) 2023-08-27T23:07:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:08 smithi006 bash[33918]: cluster 2023-08-27T23:07:06.567571+0000 mgr.y (mgr.14901) 2057 : cluster [DBG] pgmap v2064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:08 smithi006 bash[27292]: cluster 2023-08-27T23:07:06.567571+0000 mgr.y (mgr.14901) 2057 : cluster [DBG] pgmap v2064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:08 smithi130 bash[27579]: cluster 2023-08-27T23:07:06.567571+0000 mgr.y (mgr.14901) 2057 : cluster [DBG] pgmap v2064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:09.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:09 smithi130 bash[51302]: debug 2023-08-27T23:07:09.127+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:09.128619+0000) 2023-08-27T23:07:10.463 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:10 smithi130 bash[51302]: debug 2023-08-27T23:07:10.123+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:10.128891+0000) 2023-08-27T23:07:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:07:10.379Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:07:10.381Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:10 smithi006 bash[27292]: cluster 2023-08-27T23:07:08.568231+0000 mgr.y (mgr.14901) 2058 : cluster [DBG] pgmap v2065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:10 smithi006 bash[27292]: audit 2023-08-27T23:07:09.548906+0000 mon.c (mon.2) 819 : audit [DBG] from='client.? 172.21.15.6:0/2916295603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:10 smithi006 bash[33918]: cluster 2023-08-27T23:07:08.568231+0000 mgr.y (mgr.14901) 2058 : cluster [DBG] pgmap v2065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:10 smithi006 bash[33918]: audit 2023-08-27T23:07:09.548906+0000 mon.c (mon.2) 819 : audit [DBG] from='client.? 172.21.15.6:0/2916295603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:10.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:10 smithi130 bash[27579]: cluster 2023-08-27T23:07:08.568231+0000 mgr.y (mgr.14901) 2058 : cluster [DBG] pgmap v2065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:10.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:10 smithi130 bash[27579]: audit 2023-08-27T23:07:09.548906+0000 mon.c (mon.2) 819 : audit [DBG] from='client.? 172.21.15.6:0/2916295603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:07:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:07:10] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:07:11.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:11 smithi130 bash[51302]: debug 2023-08-27T23:07:11.123+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:11.129145+0000) 2023-08-27T23:07:12.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:11 smithi006 bash[56298]: level=error ts=2023-08-27T23:07:11.942Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:12.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:11 smithi006 bash[56298]: level=warn ts=2023-08-27T23:07:11.943Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:12.468 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:12 smithi130 bash[51302]: debug 2023-08-27T23:07:12.123+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:12.129362+0000) 2023-08-27T23:07:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:12 smithi130 bash[27579]: cluster 2023-08-27T23:07:10.569390+0000 mgr.y (mgr.14901) 2059 : cluster [DBG] pgmap v2066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:12 smithi130 bash[27579]: audit 2023-08-27T23:07:11.966507+0000 mon.c (mon.2) 820 : audit [DBG] from='client.? 172.21.15.6:0/1147826197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:12 smithi006 bash[33918]: cluster 2023-08-27T23:07:10.569390+0000 mgr.y (mgr.14901) 2059 : cluster [DBG] pgmap v2066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:12 smithi006 bash[33918]: audit 2023-08-27T23:07:11.966507+0000 mon.c (mon.2) 820 : audit [DBG] from='client.? 172.21.15.6:0/1147826197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:12 smithi006 bash[27292]: cluster 2023-08-27T23:07:10.569390+0000 mgr.y (mgr.14901) 2059 : cluster [DBG] pgmap v2066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:12 smithi006 bash[27292]: audit 2023-08-27T23:07:11.966507+0000 mon.c (mon.2) 820 : audit [DBG] from='client.? 172.21.15.6:0/1147826197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:13.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:13 smithi130 bash[51302]: debug 2023-08-27T23:07:13.127+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:13.129640+0000) 2023-08-27T23:07:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:07:13.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:07:13.845Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:14.474 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:14 smithi130 bash[51302]: debug 2023-08-27T23:07:14.123+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:14.129933+0000) 2023-08-27T23:07:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:14 smithi130 bash[27579]: cluster 2023-08-27T23:07:12.570052+0000 mgr.y (mgr.14901) 2060 : cluster [DBG] pgmap v2067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:14 smithi130 bash[27579]: audit 2023-08-27T23:07:14.386764+0000 mon.c (mon.2) 821 : audit [DBG] from='client.? 172.21.15.6:0/1569413709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:14 smithi006 bash[27292]: cluster 2023-08-27T23:07:12.570052+0000 mgr.y (mgr.14901) 2060 : cluster [DBG] pgmap v2067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:14 smithi006 bash[27292]: audit 2023-08-27T23:07:14.386764+0000 mon.c (mon.2) 821 : audit [DBG] from='client.? 172.21.15.6:0/1569413709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:14 smithi006 bash[33918]: cluster 2023-08-27T23:07:12.570052+0000 mgr.y (mgr.14901) 2060 : cluster [DBG] pgmap v2067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:14 smithi006 bash[33918]: audit 2023-08-27T23:07:14.386764+0000 mon.c (mon.2) 821 : audit [DBG] from='client.? 172.21.15.6:0/1569413709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:15.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:15 smithi130 bash[51302]: debug 2023-08-27T23:07:15.127+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:15.130213+0000) 2023-08-27T23:07:16.479 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:16 smithi130 bash[51302]: debug 2023-08-27T23:07:16.127+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:16.130577+0000) 2023-08-27T23:07:16.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:16 smithi130 bash[27579]: cluster 2023-08-27T23:07:14.571165+0000 mgr.y (mgr.14901) 2061 : cluster [DBG] pgmap v2068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:16 smithi006 bash[33918]: cluster 2023-08-27T23:07:14.571165+0000 mgr.y (mgr.14901) 2061 : cluster [DBG] pgmap v2068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:16 smithi006 bash[27292]: cluster 2023-08-27T23:07:14.571165+0000 mgr.y (mgr.14901) 2061 : cluster [DBG] pgmap v2068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:17.488 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:17 smithi130 bash[51302]: debug 2023-08-27T23:07:17.127+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:17.130831+0000) 2023-08-27T23:07:17.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:17 smithi130 bash[27579]: audit 2023-08-27T23:07:16.802009+0000 mon.c (mon.2) 822 : audit [DBG] from='client.? 172.21.15.6:0/1762482544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:17.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:17 smithi006 bash[27292]: audit 2023-08-27T23:07:16.802009+0000 mon.c (mon.2) 822 : audit [DBG] from='client.? 172.21.15.6:0/1762482544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:17.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:17 smithi006 bash[33918]: audit 2023-08-27T23:07:16.802009+0000 mon.c (mon.2) 822 : audit [DBG] from='client.? 172.21.15.6:0/1762482544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:18.491 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:18 smithi130 bash[51302]: debug 2023-08-27T23:07:18.127+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:18.131134+0000) 2023-08-27T23:07:18.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:18 smithi130 bash[27579]: cluster 2023-08-27T23:07:16.571916+0000 mgr.y (mgr.14901) 2062 : cluster [DBG] pgmap v2069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:18 smithi006 bash[27292]: cluster 2023-08-27T23:07:16.571916+0000 mgr.y (mgr.14901) 2062 : cluster [DBG] pgmap v2069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:18 smithi006 bash[33918]: cluster 2023-08-27T23:07:16.571916+0000 mgr.y (mgr.14901) 2062 : cluster [DBG] pgmap v2069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:19.499 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:19 smithi130 bash[51302]: debug 2023-08-27T23:07:19.127+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:19.131355+0000) 2023-08-27T23:07:19.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:19 smithi130 bash[27579]: audit 2023-08-27T23:07:19.220443+0000 mon.a (mon.0) 1919 : audit [DBG] from='client.? 172.21.15.6:0/300374969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:19 smithi006 bash[33918]: audit 2023-08-27T23:07:19.220443+0000 mon.a (mon.0) 1919 : audit [DBG] from='client.? 172.21.15.6:0/300374969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:19.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:19 smithi006 bash[27292]: audit 2023-08-27T23:07:19.220443+0000 mon.a (mon.0) 1919 : audit [DBG] from='client.? 172.21.15.6:0/300374969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:20.502 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:20 smithi130 bash[51302]: debug 2023-08-27T23:07:20.126+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:20.131570+0000) 2023-08-27T23:07:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:20 smithi006 bash[27292]: cluster 2023-08-27T23:07:18.572618+0000 mgr.y (mgr.14901) 2063 : cluster [DBG] pgmap v2070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:20 smithi006 bash[33918]: cluster 2023-08-27T23:07:18.572618+0000 mgr.y (mgr.14901) 2063 : cluster [DBG] pgmap v2070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:07:20.379Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:07:20.381Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:20.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:20 smithi130 bash[27579]: cluster 2023-08-27T23:07:18.572618+0000 mgr.y (mgr.14901) 2063 : cluster [DBG] pgmap v2070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:07:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:07:20] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:07:21.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:21 smithi130 bash[51302]: debug 2023-08-27T23:07:21.130+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:21.131853+0000) 2023-08-27T23:07:22.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:21 smithi006 bash[56298]: level=error ts=2023-08-27T23:07:21.943Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:22.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:21 smithi006 bash[56298]: level=warn ts=2023-08-27T23:07:21.944Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:22.507 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:22 smithi130 bash[51302]: debug 2023-08-27T23:07:22.126+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:22.132053+0000) 2023-08-27T23:07:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:22 smithi130 bash[27579]: cluster 2023-08-27T23:07:20.573644+0000 mgr.y (mgr.14901) 2064 : cluster [DBG] pgmap v2071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:22 smithi130 bash[27579]: audit 2023-08-27T23:07:21.635959+0000 mon.a (mon.0) 1920 : audit [DBG] from='client.? 172.21.15.6:0/1370131939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:22.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:22 smithi006 bash[33918]: cluster 2023-08-27T23:07:20.573644+0000 mgr.y (mgr.14901) 2064 : cluster [DBG] pgmap v2071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:22 smithi006 bash[33918]: audit 2023-08-27T23:07:21.635959+0000 mon.a (mon.0) 1920 : audit [DBG] from='client.? 172.21.15.6:0/1370131939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:22 smithi006 bash[27292]: cluster 2023-08-27T23:07:20.573644+0000 mgr.y (mgr.14901) 2064 : cluster [DBG] pgmap v2071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:22 smithi006 bash[27292]: audit 2023-08-27T23:07:21.635959+0000 mon.a (mon.0) 1920 : audit [DBG] from='client.? 172.21.15.6:0/1370131939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:23.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:23 smithi130 bash[51302]: debug 2023-08-27T23:07:23.126+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:23.132315+0000) 2023-08-27T23:07:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:07:23.844Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:07:23.846Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:24.512 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:24 smithi130 bash[51302]: debug 2023-08-27T23:07:24.130+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:24.132584+0000) 2023-08-27T23:07:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:24 smithi130 bash[27579]: cluster 2023-08-27T23:07:22.574269+0000 mgr.y (mgr.14901) 2065 : cluster [DBG] pgmap v2072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:24 smithi130 bash[27579]: audit 2023-08-27T23:07:24.045828+0000 mon.a (mon.0) 1921 : audit [DBG] from='client.? 172.21.15.6:0/838834826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:24.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:24 smithi006 bash[33918]: cluster 2023-08-27T23:07:22.574269+0000 mgr.y (mgr.14901) 2065 : cluster [DBG] pgmap v2072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:24.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:24 smithi006 bash[33918]: audit 2023-08-27T23:07:24.045828+0000 mon.a (mon.0) 1921 : audit [DBG] from='client.? 172.21.15.6:0/838834826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:24 smithi006 bash[27292]: cluster 2023-08-27T23:07:22.574269+0000 mgr.y (mgr.14901) 2065 : cluster [DBG] pgmap v2072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:24 smithi006 bash[27292]: audit 2023-08-27T23:07:24.045828+0000 mon.a (mon.0) 1921 : audit [DBG] from='client.? 172.21.15.6:0/838834826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:25.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:25 smithi130 bash[51302]: debug 2023-08-27T23:07:25.126+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:25.132846+0000) 2023-08-27T23:07:26.517 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:26 smithi130 bash[51302]: debug 2023-08-27T23:07:26.130+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:26.133063+0000) 2023-08-27T23:07:26.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:26 smithi130 bash[27579]: cluster 2023-08-27T23:07:24.575502+0000 mgr.y (mgr.14901) 2066 : cluster [DBG] pgmap v2073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:26 smithi130 bash[27579]: audit 2023-08-27T23:07:26.460444+0000 mon.a (mon.0) 1922 : audit [DBG] from='client.? 172.21.15.6:0/1187730662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:26 smithi006 bash[33918]: cluster 2023-08-27T23:07:24.575502+0000 mgr.y (mgr.14901) 2066 : cluster [DBG] pgmap v2073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:26 smithi006 bash[33918]: audit 2023-08-27T23:07:26.460444+0000 mon.a (mon.0) 1922 : audit [DBG] from='client.? 172.21.15.6:0/1187730662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:26 smithi006 bash[27292]: cluster 2023-08-27T23:07:24.575502+0000 mgr.y (mgr.14901) 2066 : cluster [DBG] pgmap v2073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:26 smithi006 bash[27292]: audit 2023-08-27T23:07:26.460444+0000 mon.a (mon.0) 1922 : audit [DBG] from='client.? 172.21.15.6:0/1187730662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:27.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:27 smithi130 bash[51302]: debug 2023-08-27T23:07:27.126+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:27.133313+0000) 2023-08-27T23:07:28.522 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:28 smithi130 bash[51302]: debug 2023-08-27T23:07:28.130+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:28.133595+0000) 2023-08-27T23:07:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:28 smithi130 bash[27579]: cluster 2023-08-27T23:07:26.576150+0000 mgr.y (mgr.14901) 2067 : cluster [DBG] pgmap v2074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:28.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:28 smithi006 bash[27292]: cluster 2023-08-27T23:07:26.576150+0000 mgr.y (mgr.14901) 2067 : cluster [DBG] pgmap v2074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:28 smithi006 bash[33918]: cluster 2023-08-27T23:07:26.576150+0000 mgr.y (mgr.14901) 2067 : cluster [DBG] pgmap v2074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:29.529 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:29 smithi130 bash[51302]: debug 2023-08-27T23:07:29.130+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:29.133890+0000) 2023-08-27T23:07:29.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:29 smithi130 bash[27579]: audit 2023-08-27T23:07:28.875662+0000 mon.a (mon.0) 1923 : audit [DBG] from='client.? 172.21.15.6:0/4282517695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:29.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:29 smithi006 bash[33918]: audit 2023-08-27T23:07:28.875662+0000 mon.a (mon.0) 1923 : audit [DBG] from='client.? 172.21.15.6:0/4282517695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:29.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:29 smithi006 bash[27292]: audit 2023-08-27T23:07:28.875662+0000 mon.a (mon.0) 1923 : audit [DBG] from='client.? 172.21.15.6:0/4282517695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:30.532 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:30 smithi130 bash[51302]: debug 2023-08-27T23:07:30.130+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:30.134185+0000) 2023-08-27T23:07:30.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:30 smithi006 bash[33918]: cluster 2023-08-27T23:07:28.576717+0000 mgr.y (mgr.14901) 2068 : cluster [DBG] pgmap v2075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:30 smithi006 bash[27292]: cluster 2023-08-27T23:07:28.576717+0000 mgr.y (mgr.14901) 2068 : cluster [DBG] pgmap v2075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:07:30.379Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:07:30.382Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:30 smithi130 bash[27579]: cluster 2023-08-27T23:07:28.576717+0000 mgr.y (mgr.14901) 2068 : cluster [DBG] pgmap v2075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:07:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:07:30] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:07:31.540 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:31 smithi130 bash[51302]: debug 2023-08-27T23:07:31.130+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:31.134456+0000) 2023-08-27T23:07:31.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:31 smithi130 bash[27579]: audit 2023-08-27T23:07:31.291043+0000 mon.a (mon.0) 1924 : audit [DBG] from='client.? 172.21.15.6:0/1888213419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:31.946 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:31 smithi006 bash[33918]: audit 2023-08-27T23:07:31.291043+0000 mon.a (mon.0) 1924 : audit [DBG] from='client.? 172.21.15.6:0/1888213419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:31.946 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:31 smithi006 bash[27292]: audit 2023-08-27T23:07:31.291043+0000 mon.a (mon.0) 1924 : audit [DBG] from='client.? 172.21.15.6:0/1888213419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:32.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:31 smithi006 bash[56298]: level=error ts=2023-08-27T23:07:31.943Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:32.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:31 smithi006 bash[56298]: level=warn ts=2023-08-27T23:07:31.945Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:32.543 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:32 smithi130 bash[51302]: debug 2023-08-27T23:07:32.130+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:32.134706+0000) 2023-08-27T23:07:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:32 smithi130 bash[27579]: cluster 2023-08-27T23:07:30.577683+0000 mgr.y (mgr.14901) 2069 : cluster [DBG] pgmap v2076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:32 smithi006 bash[27292]: cluster 2023-08-27T23:07:30.577683+0000 mgr.y (mgr.14901) 2069 : cluster [DBG] pgmap v2076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:32 smithi006 bash[33918]: cluster 2023-08-27T23:07:30.577683+0000 mgr.y (mgr.14901) 2069 : cluster [DBG] pgmap v2076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:33.551 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:33 smithi130 bash[51302]: debug 2023-08-27T23:07:33.134+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:33.134991+0000) 2023-08-27T23:07:33.847 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:33 smithi006 bash[27292]: cluster 2023-08-27T23:07:32.578313+0000 mgr.y (mgr.14901) 2070 : cluster [DBG] pgmap v2077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:33.847 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:33 smithi006 bash[33918]: cluster 2023-08-27T23:07:32.578313+0000 mgr.y (mgr.14901) 2070 : cluster [DBG] pgmap v2077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:33 smithi130 bash[27579]: cluster 2023-08-27T23:07:32.578313+0000 mgr.y (mgr.14901) 2070 : cluster [DBG] pgmap v2077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:07:33.845Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:07:33.847Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:34.553 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:34 smithi130 bash[51302]: debug 2023-08-27T23:07:34.134+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:34.135299+0000) 2023-08-27T23:07:34.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:34 smithi130 bash[27579]: audit 2023-08-27T23:07:33.716388+0000 mon.a (mon.0) 1925 : audit [DBG] from='client.? 172.21.15.6:0/2943236570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:34.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:34 smithi006 bash[33918]: audit 2023-08-27T23:07:33.716388+0000 mon.a (mon.0) 1925 : audit [DBG] from='client.? 172.21.15.6:0/2943236570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:34.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:34 smithi006 bash[27292]: audit 2023-08-27T23:07:33.716388+0000 mon.a (mon.0) 1925 : audit [DBG] from='client.? 172.21.15.6:0/2943236570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:35.561 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:35 smithi130 bash[51302]: debug 2023-08-27T23:07:35.134+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:35.135517+0000) 2023-08-27T23:07:35.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:35 smithi130 bash[27579]: cluster 2023-08-27T23:07:34.579413+0000 mgr.y (mgr.14901) 2071 : cluster [DBG] pgmap v2078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:35.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:35 smithi006 bash[27292]: cluster 2023-08-27T23:07:34.579413+0000 mgr.y (mgr.14901) 2071 : cluster [DBG] pgmap v2078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:35.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:35 smithi006 bash[33918]: cluster 2023-08-27T23:07:34.579413+0000 mgr.y (mgr.14901) 2071 : cluster [DBG] pgmap v2078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:36.564 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:36 smithi130 bash[51302]: debug 2023-08-27T23:07:36.134+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:36.135799+0000) 2023-08-27T23:07:36.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:36 smithi130 bash[27579]: audit 2023-08-27T23:07:36.131410+0000 mon.a (mon.0) 1926 : audit [DBG] from='client.? 172.21.15.6:0/2531079792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:36.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:36 smithi006 bash[33918]: audit 2023-08-27T23:07:36.131410+0000 mon.a (mon.0) 1926 : audit [DBG] from='client.? 172.21.15.6:0/2531079792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:36 smithi006 bash[27292]: audit 2023-08-27T23:07:36.131410+0000 mon.a (mon.0) 1926 : audit [DBG] from='client.? 172.21.15.6:0/2531079792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:37.571 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:37 smithi130 bash[51302]: debug 2023-08-27T23:07:37.134+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:37.136059+0000) 2023-08-27T23:07:37.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:37 smithi130 bash[27579]: cluster 2023-08-27T23:07:36.580116+0000 mgr.y (mgr.14901) 2072 : cluster [DBG] pgmap v2079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:37.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:37 smithi006 bash[33918]: cluster 2023-08-27T23:07:36.580116+0000 mgr.y (mgr.14901) 2072 : cluster [DBG] pgmap v2079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:37.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:37 smithi006 bash[27292]: cluster 2023-08-27T23:07:36.580116+0000 mgr.y (mgr.14901) 2072 : cluster [DBG] pgmap v2079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:38.574 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:38 smithi130 bash[51302]: debug 2023-08-27T23:07:38.134+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:38.136326+0000) 2023-08-27T23:07:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:38 smithi130 bash[27579]: audit 2023-08-27T23:07:38.552821+0000 mon.a (mon.0) 1927 : audit [DBG] from='client.? 172.21.15.6:0/3869390346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:38.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:38 smithi006 bash[27292]: audit 2023-08-27T23:07:38.552821+0000 mon.a (mon.0) 1927 : audit [DBG] from='client.? 172.21.15.6:0/3869390346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:38 smithi006 bash[33918]: audit 2023-08-27T23:07:38.552821+0000 mon.a (mon.0) 1927 : audit [DBG] from='client.? 172.21.15.6:0/3869390346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:39.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:39 smithi130 bash[51302]: debug 2023-08-27T23:07:39.134+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:39.136631+0000) 2023-08-27T23:07:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:40 smithi130 bash[27579]: cluster 2023-08-27T23:07:38.580647+0000 mgr.y (mgr.14901) 2073 : cluster [DBG] pgmap v2080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:40.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:40 smithi130 bash[51302]: debug 2023-08-27T23:07:40.134+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:40.136921+0000) 2023-08-27T23:07:40.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:40 smithi006 bash[27292]: cluster 2023-08-27T23:07:38.580647+0000 mgr.y (mgr.14901) 2073 : cluster [DBG] pgmap v2080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:07:40.380Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:07:40.382Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:40 smithi006 bash[33918]: cluster 2023-08-27T23:07:38.580647+0000 mgr.y (mgr.14901) 2073 : cluster [DBG] pgmap v2080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:07:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:07:40] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:07:41.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:41 smithi130 bash[27579]: audit 2023-08-27T23:07:40.973037+0000 mon.c (mon.2) 823 : audit [DBG] from='client.? 172.21.15.6:0/2850743272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:41.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:41 smithi130 bash[51302]: debug 2023-08-27T23:07:41.134+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:41.137214+0000) 2023-08-27T23:07:41.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:41 smithi006 bash[33918]: audit 2023-08-27T23:07:40.973037+0000 mon.c (mon.2) 823 : audit [DBG] from='client.? 172.21.15.6:0/2850743272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:41.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:41 smithi006 bash[27292]: audit 2023-08-27T23:07:40.973037+0000 mon.c (mon.2) 823 : audit [DBG] from='client.? 172.21.15.6:0/2850743272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:42.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:41 smithi006 bash[56298]: level=error ts=2023-08-27T23:07:41.944Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:42.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:41 smithi006 bash[56298]: level=warn ts=2023-08-27T23:07:41.946Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:42 smithi130 bash[27579]: cluster 2023-08-27T23:07:40.581586+0000 mgr.y (mgr.14901) 2074 : cluster [DBG] pgmap v2081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:42.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:42 smithi130 bash[51302]: debug 2023-08-27T23:07:42.134+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:42.137485+0000) 2023-08-27T23:07:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:42 smithi006 bash[27292]: cluster 2023-08-27T23:07:40.581586+0000 mgr.y (mgr.14901) 2074 : cluster [DBG] pgmap v2081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:42.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:42 smithi006 bash[33918]: cluster 2023-08-27T23:07:40.581586+0000 mgr.y (mgr.14901) 2074 : cluster [DBG] pgmap v2081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:43.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:43 smithi130 bash[51302]: debug 2023-08-27T23:07:43.134+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:43.137716+0000) 2023-08-27T23:07:44.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:07:43.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:07:43.850Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:44 smithi130 bash[27579]: cluster 2023-08-27T23:07:42.582333+0000 mgr.y (mgr.14901) 2075 : cluster [DBG] pgmap v2082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:44 smithi130 bash[27579]: audit 2023-08-27T23:07:43.400902+0000 mon.a (mon.0) 1928 : audit [DBG] from='client.? 172.21.15.6:0/2493323084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:44.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:44 smithi130 bash[51302]: debug 2023-08-27T23:07:44.134+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:44.137943+0000) 2023-08-27T23:07:44.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:44 smithi006 bash[27292]: cluster 2023-08-27T23:07:42.582333+0000 mgr.y (mgr.14901) 2075 : cluster [DBG] pgmap v2082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:44.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:44 smithi006 bash[27292]: audit 2023-08-27T23:07:43.400902+0000 mon.a (mon.0) 1928 : audit [DBG] from='client.? 172.21.15.6:0/2493323084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:44.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:44 smithi006 bash[33918]: cluster 2023-08-27T23:07:42.582333+0000 mgr.y (mgr.14901) 2075 : cluster [DBG] pgmap v2082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:44.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:44 smithi006 bash[33918]: audit 2023-08-27T23:07:43.400902+0000 mon.a (mon.0) 1928 : audit [DBG] from='client.? 172.21.15.6:0/2493323084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:45.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:45 smithi130 bash[51302]: debug 2023-08-27T23:07:45.134+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:45.138295+0000) 2023-08-27T23:07:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:46 smithi130 bash[27579]: cluster 2023-08-27T23:07:44.583359+0000 mgr.y (mgr.14901) 2076 : cluster [DBG] pgmap v2083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:46 smithi130 bash[27579]: audit 2023-08-27T23:07:45.816514+0000 mon.c (mon.2) 824 : audit [DBG] from='client.? 172.21.15.6:0/2171798062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:46.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:46 smithi130 bash[51302]: debug 2023-08-27T23:07:46.134+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:46.138602+0000) 2023-08-27T23:07:46.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:46 smithi006 bash[33918]: cluster 2023-08-27T23:07:44.583359+0000 mgr.y (mgr.14901) 2076 : cluster [DBG] pgmap v2083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:46 smithi006 bash[33918]: audit 2023-08-27T23:07:45.816514+0000 mon.c (mon.2) 824 : audit [DBG] from='client.? 172.21.15.6:0/2171798062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:46 smithi006 bash[27292]: cluster 2023-08-27T23:07:44.583359+0000 mgr.y (mgr.14901) 2076 : cluster [DBG] pgmap v2083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:46 smithi006 bash[27292]: audit 2023-08-27T23:07:45.816514+0000 mon.c (mon.2) 824 : audit [DBG] from='client.? 172.21.15.6:0/2171798062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:47.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:47 smithi130 bash[51302]: debug 2023-08-27T23:07:47.138+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:47.138887+0000) 2023-08-27T23:07:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:48 smithi130 bash[27579]: cluster 2023-08-27T23:07:46.584026+0000 mgr.y (mgr.14901) 2077 : cluster [DBG] pgmap v2084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:48 smithi130 bash[27579]: audit 2023-08-27T23:07:48.237533+0000 mon.a (mon.0) 1929 : audit [DBG] from='client.? 172.21.15.6:0/3298370835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:48.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:48 smithi130 bash[51302]: debug 2023-08-27T23:07:48.138+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:48.139122+0000) 2023-08-27T23:07:48.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:48 smithi006 bash[33918]: cluster 2023-08-27T23:07:46.584026+0000 mgr.y (mgr.14901) 2077 : cluster [DBG] pgmap v2084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:48 smithi006 bash[33918]: audit 2023-08-27T23:07:48.237533+0000 mon.a (mon.0) 1929 : audit [DBG] from='client.? 172.21.15.6:0/3298370835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:48 smithi006 bash[27292]: cluster 2023-08-27T23:07:46.584026+0000 mgr.y (mgr.14901) 2077 : cluster [DBG] pgmap v2084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:48 smithi006 bash[27292]: audit 2023-08-27T23:07:48.237533+0000 mon.a (mon.0) 1929 : audit [DBG] from='client.? 172.21.15.6:0/3298370835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:49.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:49 smithi130 bash[51302]: debug 2023-08-27T23:07:49.138+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:49.139395+0000) 2023-08-27T23:07:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:50 smithi130 bash[27579]: cluster 2023-08-27T23:07:48.584646+0000 mgr.y (mgr.14901) 2078 : cluster [DBG] pgmap v2085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:50.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:50 smithi130 bash[51302]: debug 2023-08-27T23:07:50.138+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:50.139644+0000) 2023-08-27T23:07:50.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:50 smithi006 bash[33918]: cluster 2023-08-27T23:07:48.584646+0000 mgr.y (mgr.14901) 2078 : cluster [DBG] pgmap v2085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:50 smithi006 bash[27292]: cluster 2023-08-27T23:07:48.584646+0000 mgr.y (mgr.14901) 2078 : cluster [DBG] pgmap v2085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:07:50.381Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:07:50.383Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:07:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:07:50] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:07:51.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:51 smithi130 bash[27579]: audit 2023-08-27T23:07:50.662665+0000 mon.a (mon.0) 1930 : audit [DBG] from='client.? 172.21.15.6:0/1389270519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:51.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:51 smithi130 bash[51302]: debug 2023-08-27T23:07:51.138+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:51.140009+0000) 2023-08-27T23:07:51.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:51 smithi006 bash[33918]: audit 2023-08-27T23:07:50.662665+0000 mon.a (mon.0) 1930 : audit [DBG] from='client.? 172.21.15.6:0/1389270519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:51.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:51 smithi006 bash[27292]: audit 2023-08-27T23:07:50.662665+0000 mon.a (mon.0) 1930 : audit [DBG] from='client.? 172.21.15.6:0/1389270519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:52.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:51 smithi006 bash[56298]: level=error ts=2023-08-27T23:07:51.945Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:52.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:51 smithi006 bash[56298]: level=warn ts=2023-08-27T23:07:51.946Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:52 smithi130 bash[27579]: cluster 2023-08-27T23:07:50.585363+0000 mgr.y (mgr.14901) 2079 : cluster [DBG] pgmap v2086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:52.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:52 smithi130 bash[51302]: debug 2023-08-27T23:07:52.138+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:52.140244+0000) 2023-08-27T23:07:52.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:52 smithi006 bash[27292]: cluster 2023-08-27T23:07:50.585363+0000 mgr.y (mgr.14901) 2079 : cluster [DBG] pgmap v2086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:52 smithi006 bash[33918]: cluster 2023-08-27T23:07:50.585363+0000 mgr.y (mgr.14901) 2079 : cluster [DBG] pgmap v2086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:53 smithi130 bash[27579]: audit 2023-08-27T23:07:52.675874+0000 mon.b (mon.1) 320 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:07:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:53 smithi130 bash[27579]: audit 2023-08-27T23:07:53.117523+0000 mon.a (mon.0) 1931 : audit [DBG] from='client.? 172.21.15.6:0/2408912826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:53.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:53 smithi130 bash[51302]: debug 2023-08-27T23:07:53.138+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:53.140516+0000) 2023-08-27T23:07:53.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:53 smithi006 bash[33918]: audit 2023-08-27T23:07:52.675874+0000 mon.b (mon.1) 320 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:07:53.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:53 smithi006 bash[33918]: audit 2023-08-27T23:07:53.117523+0000 mon.a (mon.0) 1931 : audit [DBG] from='client.? 172.21.15.6:0/2408912826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:53.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:53 smithi006 bash[27292]: audit 2023-08-27T23:07:52.675874+0000 mon.b (mon.1) 320 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:07:53.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:53 smithi006 bash[27292]: audit 2023-08-27T23:07:53.117523+0000 mon.a (mon.0) 1931 : audit [DBG] from='client.? 172.21.15.6:0/2408912826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:07:53.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:07:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:07:53.848Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:07:54.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:54 smithi130 bash[27579]: cluster 2023-08-27T23:07:52.586013+0000 mgr.y (mgr.14901) 2080 : cluster [DBG] pgmap v2087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:54.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:54 smithi130 bash[51302]: debug 2023-08-27T23:07:54.138+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:54.140795+0000) 2023-08-27T23:07:54.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:54 smithi006 bash[33918]: cluster 2023-08-27T23:07:52.586013+0000 mgr.y (mgr.14901) 2080 : cluster [DBG] pgmap v2087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:54 smithi006 bash[27292]: cluster 2023-08-27T23:07:52.586013+0000 mgr.y (mgr.14901) 2080 : cluster [DBG] pgmap v2087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:55.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:55 smithi130 bash[51302]: debug 2023-08-27T23:07:55.138+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:55.141108+0000) 2023-08-27T23:07:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:56 smithi130 bash[27579]: cluster 2023-08-27T23:07:54.587101+0000 mgr.y (mgr.14901) 2081 : cluster [DBG] pgmap v2088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:56 smithi130 bash[27579]: audit 2023-08-27T23:07:55.532619+0000 mon.c (mon.2) 825 : audit [DBG] from='client.? 172.21.15.6:0/4239096181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:56.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:56 smithi130 bash[51302]: debug 2023-08-27T23:07:56.138+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:56.141542+0000) 2023-08-27T23:07:56.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:56 smithi006 bash[33918]: cluster 2023-08-27T23:07:54.587101+0000 mgr.y (mgr.14901) 2081 : cluster [DBG] pgmap v2088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:56 smithi006 bash[33918]: audit 2023-08-27T23:07:55.532619+0000 mon.c (mon.2) 825 : audit [DBG] from='client.? 172.21.15.6:0/4239096181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:56 smithi006 bash[27292]: cluster 2023-08-27T23:07:54.587101+0000 mgr.y (mgr.14901) 2081 : cluster [DBG] pgmap v2088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:56 smithi006 bash[27292]: audit 2023-08-27T23:07:55.532619+0000 mon.c (mon.2) 825 : audit [DBG] from='client.? 172.21.15.6:0/4239096181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:57.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:57 smithi130 bash[51302]: debug 2023-08-27T23:07:57.138+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:57.141814+0000) 2023-08-27T23:07:58.396 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:58 smithi130 bash[51302]: debug 2023-08-27T23:07:58.138+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:58.142072+0000) 2023-08-27T23:07:58.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:58 smithi006 bash[33918]: cluster 2023-08-27T23:07:56.587738+0000 mgr.y (mgr.14901) 2082 : cluster [DBG] pgmap v2089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:58 smithi006 bash[33918]: audit 2023-08-27T23:07:57.956906+0000 mon.a (mon.0) 1932 : audit [DBG] from='client.? 172.21.15.6:0/1298224890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:58 smithi006 bash[27292]: cluster 2023-08-27T23:07:56.587738+0000 mgr.y (mgr.14901) 2082 : cluster [DBG] pgmap v2089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:58 smithi006 bash[27292]: audit 2023-08-27T23:07:57.956906+0000 mon.a (mon.0) 1932 : audit [DBG] from='client.? 172.21.15.6:0/1298224890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:58 smithi130 bash[27579]: cluster 2023-08-27T23:07:56.587738+0000 mgr.y (mgr.14901) 2082 : cluster [DBG] pgmap v2089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:58 smithi130 bash[27579]: audit 2023-08-27T23:07:57.956906+0000 mon.a (mon.0) 1932 : audit [DBG] from='client.? 172.21.15.6:0/1298224890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:59.595 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:07:59 smithi130 bash[51302]: debug 2023-08-27T23:07:59.138+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:59.142256+0000) 2023-08-27T23:07:59.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:59 smithi130 bash[27579]: audit 2023-08-27T23:07:58.587876+0000 mon.a (mon.0) 1933 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:07:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:59 smithi130 bash[27579]: cluster 2023-08-27T23:07:58.588270+0000 mgr.y (mgr.14901) 2083 : cluster [DBG] pgmap v2090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:59 smithi130 bash[27579]: audit 2023-08-27T23:07:59.013011+0000 mon.a (mon.0) 1934 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:07:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:59 smithi130 bash[27579]: audit 2023-08-27T23:07:59.287234+0000 mon.b (mon.1) 321 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:07:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:59 smithi130 bash[27579]: audit 2023-08-27T23:07:59.289177+0000 mon.b (mon.1) 322 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:07:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:07:59 smithi130 bash[27579]: audit 2023-08-27T23:07:59.298971+0000 mon.a (mon.0) 1935 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:07:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:59 smithi006 bash[33918]: audit 2023-08-27T23:07:58.587876+0000 mon.a (mon.0) 1933 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:07:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:59 smithi006 bash[33918]: cluster 2023-08-27T23:07:58.588270+0000 mgr.y (mgr.14901) 2083 : cluster [DBG] pgmap v2090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:59 smithi006 bash[33918]: audit 2023-08-27T23:07:59.013011+0000 mon.a (mon.0) 1934 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:07:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:59 smithi006 bash[33918]: audit 2023-08-27T23:07:59.287234+0000 mon.b (mon.1) 321 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:07:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:59 smithi006 bash[33918]: audit 2023-08-27T23:07:59.289177+0000 mon.b (mon.1) 322 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:07:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:07:59 smithi006 bash[33918]: audit 2023-08-27T23:07:59.298971+0000 mon.a (mon.0) 1935 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:07:59.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:59 smithi006 bash[27292]: audit 2023-08-27T23:07:58.587876+0000 mon.a (mon.0) 1933 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:07:59.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:59 smithi006 bash[27292]: cluster 2023-08-27T23:07:58.588270+0000 mgr.y (mgr.14901) 2083 : cluster [DBG] pgmap v2090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:07:59.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:59 smithi006 bash[27292]: audit 2023-08-27T23:07:59.013011+0000 mon.a (mon.0) 1934 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:07:59.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:59 smithi006 bash[27292]: audit 2023-08-27T23:07:59.287234+0000 mon.b (mon.1) 321 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:07:59.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:59 smithi006 bash[27292]: audit 2023-08-27T23:07:59.289177+0000 mon.b (mon.1) 322 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:07:59.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:07:59 smithi006 bash[27292]: audit 2023-08-27T23:07:59.298971+0000 mon.a (mon.0) 1935 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:08:00.602 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:00 smithi130 bash[51302]: debug 2023-08-27T23:08:00.138+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:00.142544+0000) 2023-08-27T23:08:00.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:00 smithi006 bash[33918]: audit 2023-08-27T23:08:00.385192+0000 mon.a (mon.0) 1936 : audit [DBG] from='client.? 172.21.15.6:0/723498942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:00 smithi006 bash[27292]: audit 2023-08-27T23:08:00.385192+0000 mon.a (mon.0) 1936 : audit [DBG] from='client.? 172.21.15.6:0/723498942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:08:00.382Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:08:00.384Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:00 smithi130 bash[27579]: audit 2023-08-27T23:08:00.385192+0000 mon.a (mon.0) 1936 : audit [DBG] from='client.? 172.21.15.6:0/723498942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:08:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:08:00] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:08:01.605 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:01 smithi130 bash[51302]: debug 2023-08-27T23:08:01.142+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:01.142842+0000) 2023-08-27T23:08:01.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:01 smithi130 bash[27579]: cluster 2023-08-27T23:08:00.589279+0000 mgr.y (mgr.14901) 2084 : cluster [DBG] pgmap v2091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:01.947 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:01 smithi006 bash[33918]: cluster 2023-08-27T23:08:00.589279+0000 mgr.y (mgr.14901) 2084 : cluster [DBG] pgmap v2091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:01.948 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:01 smithi006 bash[27292]: cluster 2023-08-27T23:08:00.589279+0000 mgr.y (mgr.14901) 2084 : cluster [DBG] pgmap v2091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:02.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:01 smithi006 bash[56298]: level=error ts=2023-08-27T23:08:01.945Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:02.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:01 smithi006 bash[56298]: level=warn ts=2023-08-27T23:08:01.947Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:02.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:02 smithi130 bash[51302]: debug 2023-08-27T23:08:02.142+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:02.143212+0000) 2023-08-27T23:08:03.610 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:03 smithi130 bash[51302]: debug 2023-08-27T23:08:03.142+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:03.143500+0000) 2023-08-27T23:08:03.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:03 smithi130 bash[27579]: cluster 2023-08-27T23:08:02.589752+0000 mgr.y (mgr.14901) 2085 : cluster [DBG] pgmap v2092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:03.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:03 smithi130 bash[27579]: audit 2023-08-27T23:08:02.802113+0000 mon.c (mon.2) 826 : audit [DBG] from='client.? 172.21.15.6:0/1975930245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:03.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:03 smithi006 bash[33918]: cluster 2023-08-27T23:08:02.589752+0000 mgr.y (mgr.14901) 2085 : cluster [DBG] pgmap v2092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:03.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:03 smithi006 bash[33918]: audit 2023-08-27T23:08:02.802113+0000 mon.c (mon.2) 826 : audit [DBG] from='client.? 172.21.15.6:0/1975930245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:03.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:03 smithi006 bash[27292]: cluster 2023-08-27T23:08:02.589752+0000 mgr.y (mgr.14901) 2085 : cluster [DBG] pgmap v2092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:03.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:03 smithi006 bash[27292]: audit 2023-08-27T23:08:02.802113+0000 mon.c (mon.2) 826 : audit [DBG] from='client.? 172.21.15.6:0/1975930245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:03.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:08:03.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:03.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:08:03.849Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:04.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:04 smithi130 bash[51302]: debug 2023-08-27T23:08:04.142+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:04.143792+0000) 2023-08-27T23:08:05.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:05 smithi130 bash[27579]: audit 2023-08-27T23:08:05.222560+0000 mon.c (mon.2) 827 : audit [DBG] from='client.? 172.21.15.6:0/3746050737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:05.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:05 smithi130 bash[51302]: debug 2023-08-27T23:08:05.142+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:05.144065+0000) 2023-08-27T23:08:05.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:05 smithi006 bash[33918]: audit 2023-08-27T23:08:05.222560+0000 mon.c (mon.2) 827 : audit [DBG] from='client.? 172.21.15.6:0/3746050737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:05.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:05 smithi006 bash[27292]: audit 2023-08-27T23:08:05.222560+0000 mon.c (mon.2) 827 : audit [DBG] from='client.? 172.21.15.6:0/3746050737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:06.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:06 smithi130 bash[51302]: debug 2023-08-27T23:08:06.142+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:06.144363+0000) 2023-08-27T23:08:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:06 smithi130 bash[27579]: cluster 2023-08-27T23:08:04.590820+0000 mgr.y (mgr.14901) 2086 : cluster [DBG] pgmap v2093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:06.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:06 smithi006 bash[33918]: cluster 2023-08-27T23:08:04.590820+0000 mgr.y (mgr.14901) 2086 : cluster [DBG] pgmap v2093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:06 smithi006 bash[27292]: cluster 2023-08-27T23:08:04.590820+0000 mgr.y (mgr.14901) 2086 : cluster [DBG] pgmap v2093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:07.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:07 smithi130 bash[51302]: debug 2023-08-27T23:08:07.142+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:07.144613+0000) 2023-08-27T23:08:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:08 smithi130 bash[27579]: cluster 2023-08-27T23:08:06.591499+0000 mgr.y (mgr.14901) 2087 : cluster [DBG] pgmap v2094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:08 smithi130 bash[27579]: audit 2023-08-27T23:08:07.640798+0000 mon.c (mon.2) 828 : audit [DBG] from='client.? 172.21.15.6:0/3948067149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:08.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:08 smithi130 bash[51302]: debug 2023-08-27T23:08:08.142+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:08.144930+0000) 2023-08-27T23:08:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:08 smithi006 bash[27292]: cluster 2023-08-27T23:08:06.591499+0000 mgr.y (mgr.14901) 2087 : cluster [DBG] pgmap v2094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:08 smithi006 bash[27292]: audit 2023-08-27T23:08:07.640798+0000 mon.c (mon.2) 828 : audit [DBG] from='client.? 172.21.15.6:0/3948067149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:08 smithi006 bash[33918]: cluster 2023-08-27T23:08:06.591499+0000 mgr.y (mgr.14901) 2087 : cluster [DBG] pgmap v2094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:08 smithi006 bash[33918]: audit 2023-08-27T23:08:07.640798+0000 mon.c (mon.2) 828 : audit [DBG] from='client.? 172.21.15.6:0/3948067149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:09.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:09 smithi130 bash[51302]: debug 2023-08-27T23:08:09.142+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:09.145151+0000) 2023-08-27T23:08:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:10 smithi130 bash[27579]: cluster 2023-08-27T23:08:08.592280+0000 mgr.y (mgr.14901) 2088 : cluster [DBG] pgmap v2095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:10 smithi130 bash[27579]: audit 2023-08-27T23:08:10.058425+0000 mon.c (mon.2) 829 : audit [DBG] from='client.? 172.21.15.6:0/2397268246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:10.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:10 smithi130 bash[51302]: debug 2023-08-27T23:08:10.142+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:10.145398+0000) 2023-08-27T23:08:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:10 smithi006 bash[27292]: cluster 2023-08-27T23:08:08.592280+0000 mgr.y (mgr.14901) 2088 : cluster [DBG] pgmap v2095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:10 smithi006 bash[27292]: audit 2023-08-27T23:08:10.058425+0000 mon.c (mon.2) 829 : audit [DBG] from='client.? 172.21.15.6:0/2397268246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:10 smithi006 bash[33918]: cluster 2023-08-27T23:08:08.592280+0000 mgr.y (mgr.14901) 2088 : cluster [DBG] pgmap v2095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:10 smithi006 bash[33918]: audit 2023-08-27T23:08:10.058425+0000 mon.c (mon.2) 829 : audit [DBG] from='client.? 172.21.15.6:0/2397268246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:08:10.383Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:10.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:08:10.385Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:08:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:08:10] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:08:11.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:11 smithi130 bash[51302]: debug 2023-08-27T23:08:11.142+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:11.145711+0000) 2023-08-27T23:08:12.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:11 smithi006 bash[56298]: level=error ts=2023-08-27T23:08:11.946Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:12.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:11 smithi006 bash[56298]: level=warn ts=2023-08-27T23:08:11.948Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:12 smithi130 bash[27579]: cluster 2023-08-27T23:08:10.593461+0000 mgr.y (mgr.14901) 2089 : cluster [DBG] pgmap v2096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:12.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:12 smithi130 bash[51302]: debug 2023-08-27T23:08:12.142+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:12.145932+0000) 2023-08-27T23:08:12.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:12 smithi006 bash[33918]: cluster 2023-08-27T23:08:10.593461+0000 mgr.y (mgr.14901) 2089 : cluster [DBG] pgmap v2096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:12.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:12 smithi006 bash[27292]: cluster 2023-08-27T23:08:10.593461+0000 mgr.y (mgr.14901) 2089 : cluster [DBG] pgmap v2096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:13 smithi130 bash[27579]: audit 2023-08-27T23:08:12.479550+0000 mon.c (mon.2) 830 : audit [DBG] from='client.? 172.21.15.6:0/789472655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:13.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:13 smithi130 bash[51302]: debug 2023-08-27T23:08:13.142+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:13.146197+0000) 2023-08-27T23:08:13.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:13 smithi006 bash[27292]: audit 2023-08-27T23:08:12.479550+0000 mon.c (mon.2) 830 : audit [DBG] from='client.? 172.21.15.6:0/789472655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:13.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:13 smithi006 bash[33918]: audit 2023-08-27T23:08:12.479550+0000 mon.c (mon.2) 830 : audit [DBG] from='client.? 172.21.15.6:0/789472655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:08:13.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:08:13.850Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:14 smithi130 bash[27579]: cluster 2023-08-27T23:08:12.594126+0000 mgr.y (mgr.14901) 2090 : cluster [DBG] pgmap v2097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:14.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:14 smithi130 bash[51302]: debug 2023-08-27T23:08:14.142+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:14.146483+0000) 2023-08-27T23:08:14.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:14 smithi006 bash[27292]: cluster 2023-08-27T23:08:12.594126+0000 mgr.y (mgr.14901) 2090 : cluster [DBG] pgmap v2097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:14 smithi006 bash[33918]: cluster 2023-08-27T23:08:12.594126+0000 mgr.y (mgr.14901) 2090 : cluster [DBG] pgmap v2097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:15.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:15 smithi130 bash[51302]: debug 2023-08-27T23:08:15.142+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:15.146749+0000) 2023-08-27T23:08:15.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:15 smithi130 bash[27579]: audit 2023-08-27T23:08:14.897458+0000 mon.a (mon.0) 1937 : audit [DBG] from='client.? 172.21.15.6:0/579854687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:15.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:15 smithi006 bash[27292]: audit 2023-08-27T23:08:14.897458+0000 mon.a (mon.0) 1937 : audit [DBG] from='client.? 172.21.15.6:0/579854687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:15.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:15 smithi006 bash[33918]: audit 2023-08-27T23:08:14.897458+0000 mon.a (mon.0) 1937 : audit [DBG] from='client.? 172.21.15.6:0/579854687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:16.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:16 smithi130 bash[27579]: cluster 2023-08-27T23:08:14.594917+0000 mgr.y (mgr.14901) 2091 : cluster [DBG] pgmap v2098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:16.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:16 smithi130 bash[51302]: debug 2023-08-27T23:08:16.146+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:16.147042+0000) 2023-08-27T23:08:16.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:16 smithi006 bash[33918]: cluster 2023-08-27T23:08:14.594917+0000 mgr.y (mgr.14901) 2091 : cluster [DBG] pgmap v2098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:16 smithi006 bash[27292]: cluster 2023-08-27T23:08:14.594917+0000 mgr.y (mgr.14901) 2091 : cluster [DBG] pgmap v2098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:17.402 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:17 smithi130 bash[51302]: debug 2023-08-27T23:08:17.145+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:17.147325+0000) 2023-08-27T23:08:17.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:17 smithi006 bash[27292]: audit 2023-08-27T23:08:17.319031+0000 mon.c (mon.2) 831 : audit [DBG] from='client.? 172.21.15.6:0/377116476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:17.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:17 smithi006 bash[33918]: audit 2023-08-27T23:08:17.319031+0000 mon.c (mon.2) 831 : audit [DBG] from='client.? 172.21.15.6:0/377116476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:17.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:17 smithi130 bash[27579]: audit 2023-08-27T23:08:17.319031+0000 mon.c (mon.2) 831 : audit [DBG] from='client.? 172.21.15.6:0/377116476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:18.405 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:18 smithi130 bash[51302]: debug 2023-08-27T23:08:18.145+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:18.147614+0000) 2023-08-27T23:08:18.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:18 smithi006 bash[27292]: cluster 2023-08-27T23:08:16.595559+0000 mgr.y (mgr.14901) 2092 : cluster [DBG] pgmap v2099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:18 smithi006 bash[33918]: cluster 2023-08-27T23:08:16.595559+0000 mgr.y (mgr.14901) 2092 : cluster [DBG] pgmap v2099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:18 smithi130 bash[27579]: cluster 2023-08-27T23:08:16.595559+0000 mgr.y (mgr.14901) 2092 : cluster [DBG] pgmap v2099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:19.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:19 smithi130 bash[51302]: debug 2023-08-27T23:08:19.145+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:19.147892+0000) 2023-08-27T23:08:20.410 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:20 smithi130 bash[51302]: debug 2023-08-27T23:08:20.145+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:20.148150+0000) 2023-08-27T23:08:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:20 smithi006 bash[27292]: cluster 2023-08-27T23:08:18.596237+0000 mgr.y (mgr.14901) 2093 : cluster [DBG] pgmap v2100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:20 smithi006 bash[27292]: audit 2023-08-27T23:08:19.731667+0000 mon.a (mon.0) 1938 : audit [DBG] from='client.? 172.21.15.6:0/3289156359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:08:20.384Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:08:20.386Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:20.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:20 smithi006 bash[33918]: cluster 2023-08-27T23:08:18.596237+0000 mgr.y (mgr.14901) 2093 : cluster [DBG] pgmap v2100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:20.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:20 smithi006 bash[33918]: audit 2023-08-27T23:08:19.731667+0000 mon.a (mon.0) 1938 : audit [DBG] from='client.? 172.21.15.6:0/3289156359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:20 smithi130 bash[27579]: cluster 2023-08-27T23:08:18.596237+0000 mgr.y (mgr.14901) 2093 : cluster [DBG] pgmap v2100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:20 smithi130 bash[27579]: audit 2023-08-27T23:08:19.731667+0000 mon.a (mon.0) 1938 : audit [DBG] from='client.? 172.21.15.6:0/3289156359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:21.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:08:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:08:20] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:08:21.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:21 smithi130 bash[51302]: debug 2023-08-27T23:08:21.145+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:21.148437+0000) 2023-08-27T23:08:22.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:21 smithi006 bash[56298]: level=error ts=2023-08-27T23:08:21.946Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:22.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:21 smithi006 bash[56298]: level=warn ts=2023-08-27T23:08:21.948Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:22.415 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:22 smithi130 bash[51302]: debug 2023-08-27T23:08:22.145+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:22.148744+0000) 2023-08-27T23:08:22.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:22 smithi006 bash[33918]: cluster 2023-08-27T23:08:20.597340+0000 mgr.y (mgr.14901) 2094 : cluster [DBG] pgmap v2101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:22 smithi006 bash[33918]: audit 2023-08-27T23:08:22.148461+0000 mon.c (mon.2) 832 : audit [DBG] from='client.? 172.21.15.6:0/970274676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:22 smithi006 bash[27292]: cluster 2023-08-27T23:08:20.597340+0000 mgr.y (mgr.14901) 2094 : cluster [DBG] pgmap v2101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:22 smithi006 bash[27292]: audit 2023-08-27T23:08:22.148461+0000 mon.c (mon.2) 832 : audit [DBG] from='client.? 172.21.15.6:0/970274676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:22 smithi130 bash[27579]: cluster 2023-08-27T23:08:20.597340+0000 mgr.y (mgr.14901) 2094 : cluster [DBG] pgmap v2101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:22 smithi130 bash[27579]: audit 2023-08-27T23:08:22.148461+0000 mon.c (mon.2) 832 : audit [DBG] from='client.? 172.21.15.6:0/970274676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:23.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:23 smithi130 bash[51302]: debug 2023-08-27T23:08:23.145+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:23.149048+0000) 2023-08-27T23:08:24.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:08:23.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:08:23.850Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:24.420 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:24 smithi130 bash[51302]: debug 2023-08-27T23:08:24.145+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:24.149342+0000) 2023-08-27T23:08:24.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:24 smithi006 bash[33918]: cluster 2023-08-27T23:08:22.597989+0000 mgr.y (mgr.14901) 2095 : cluster [DBG] pgmap v2102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:24.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:24 smithi006 bash[27292]: cluster 2023-08-27T23:08:22.597989+0000 mgr.y (mgr.14901) 2095 : cluster [DBG] pgmap v2102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:24 smithi130 bash[27579]: cluster 2023-08-27T23:08:22.597989+0000 mgr.y (mgr.14901) 2095 : cluster [DBG] pgmap v2102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:25.430 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:25 smithi130 bash[51302]: debug 2023-08-27T23:08:25.145+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:25.149612+0000) 2023-08-27T23:08:25.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:25 smithi006 bash[27292]: audit 2023-08-27T23:08:24.565996+0000 mon.c (mon.2) 833 : audit [DBG] from='client.? 172.21.15.6:0/235800312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:25.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:25 smithi006 bash[33918]: audit 2023-08-27T23:08:24.565996+0000 mon.c (mon.2) 833 : audit [DBG] from='client.? 172.21.15.6:0/235800312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:25.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:25 smithi130 bash[27579]: audit 2023-08-27T23:08:24.565996+0000 mon.c (mon.2) 833 : audit [DBG] from='client.? 172.21.15.6:0/235800312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:26.434 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:26 smithi130 bash[51302]: debug 2023-08-27T23:08:26.145+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:26.149887+0000) 2023-08-27T23:08:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:26 smithi006 bash[27292]: cluster 2023-08-27T23:08:24.599156+0000 mgr.y (mgr.14901) 2096 : cluster [DBG] pgmap v2103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:26 smithi006 bash[33918]: cluster 2023-08-27T23:08:24.599156+0000 mgr.y (mgr.14901) 2096 : cluster [DBG] pgmap v2103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:26.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:26 smithi130 bash[27579]: cluster 2023-08-27T23:08:24.599156+0000 mgr.y (mgr.14901) 2096 : cluster [DBG] pgmap v2103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:27.442 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:27 smithi130 bash[51302]: debug 2023-08-27T23:08:27.149+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:27.150203+0000) 2023-08-27T23:08:27.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:27 smithi006 bash[27292]: audit 2023-08-27T23:08:26.989087+0000 mon.c (mon.2) 834 : audit [DBG] from='client.? 172.21.15.6:0/3548289797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:27.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:27 smithi006 bash[33918]: audit 2023-08-27T23:08:26.989087+0000 mon.c (mon.2) 834 : audit [DBG] from='client.? 172.21.15.6:0/3548289797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:27.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:27 smithi130 bash[27579]: audit 2023-08-27T23:08:26.989087+0000 mon.c (mon.2) 834 : audit [DBG] from='client.? 172.21.15.6:0/3548289797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:28.444 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:28 smithi130 bash[51302]: debug 2023-08-27T23:08:28.145+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:28.150467+0000) 2023-08-27T23:08:28.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:28 smithi006 bash[27292]: cluster 2023-08-27T23:08:26.599610+0000 mgr.y (mgr.14901) 2097 : cluster [DBG] pgmap v2104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:28 smithi006 bash[33918]: cluster 2023-08-27T23:08:26.599610+0000 mgr.y (mgr.14901) 2097 : cluster [DBG] pgmap v2104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:28 smithi130 bash[27579]: cluster 2023-08-27T23:08:26.599610+0000 mgr.y (mgr.14901) 2097 : cluster [DBG] pgmap v2104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:29.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:29 smithi130 bash[51302]: debug 2023-08-27T23:08:29.145+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:29.150742+0000) 2023-08-27T23:08:30.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:30 smithi130 bash[51302]: debug 2023-08-27T23:08:30.149+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:30.151042+0000) 2023-08-27T23:08:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:30 smithi130 bash[27579]: cluster 2023-08-27T23:08:28.600220+0000 mgr.y (mgr.14901) 2098 : cluster [DBG] pgmap v2105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:30 smithi130 bash[27579]: audit 2023-08-27T23:08:29.410735+0000 mon.c (mon.2) 835 : audit [DBG] from='client.? 172.21.15.6:0/4019043364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:30.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:30 smithi006 bash[33918]: cluster 2023-08-27T23:08:28.600220+0000 mgr.y (mgr.14901) 2098 : cluster [DBG] pgmap v2105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:30 smithi006 bash[33918]: audit 2023-08-27T23:08:29.410735+0000 mon.c (mon.2) 835 : audit [DBG] from='client.? 172.21.15.6:0/4019043364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:30 smithi006 bash[27292]: cluster 2023-08-27T23:08:28.600220+0000 mgr.y (mgr.14901) 2098 : cluster [DBG] pgmap v2105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:30 smithi006 bash[27292]: audit 2023-08-27T23:08:29.410735+0000 mon.c (mon.2) 835 : audit [DBG] from='client.? 172.21.15.6:0/4019043364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:08:30.384Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:08:30.386Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:08:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:08:30] "GET /metrics HTTP/1.1" 200 193091 "" "Prometheus/2.33.4" 2023-08-27T23:08:31.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:31 smithi130 bash[51302]: debug 2023-08-27T23:08:31.149+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:31.151290+0000) 2023-08-27T23:08:32.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:31 smithi006 bash[56298]: level=error ts=2023-08-27T23:08:31.947Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:32.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:31 smithi006 bash[56298]: level=warn ts=2023-08-27T23:08:31.949Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:32 smithi130 bash[27579]: cluster 2023-08-27T23:08:30.601248+0000 mgr.y (mgr.14901) 2099 : cluster [DBG] pgmap v2106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:32 smithi130 bash[27579]: audit 2023-08-27T23:08:31.823887+0000 mon.a (mon.0) 1939 : audit [DBG] from='client.? 172.21.15.6:0/2099698174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:32.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:32 smithi130 bash[51302]: debug 2023-08-27T23:08:32.149+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:32.151689+0000) 2023-08-27T23:08:32.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:32 smithi006 bash[27292]: cluster 2023-08-27T23:08:30.601248+0000 mgr.y (mgr.14901) 2099 : cluster [DBG] pgmap v2106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:32 smithi006 bash[27292]: audit 2023-08-27T23:08:31.823887+0000 mon.a (mon.0) 1939 : audit [DBG] from='client.? 172.21.15.6:0/2099698174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:32.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:32 smithi006 bash[33918]: cluster 2023-08-27T23:08:30.601248+0000 mgr.y (mgr.14901) 2099 : cluster [DBG] pgmap v2106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:32.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:32 smithi006 bash[33918]: audit 2023-08-27T23:08:31.823887+0000 mon.a (mon.0) 1939 : audit [DBG] from='client.? 172.21.15.6:0/2099698174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:33.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:33 smithi130 bash[51302]: debug 2023-08-27T23:08:33.149+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:33.151971+0000) 2023-08-27T23:08:34.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:08:33.849Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:08:33.852Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:34 smithi130 bash[27579]: cluster 2023-08-27T23:08:32.601944+0000 mgr.y (mgr.14901) 2100 : cluster [DBG] pgmap v2107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:34 smithi130 bash[27579]: audit 2023-08-27T23:08:34.241932+0000 mon.a (mon.0) 1940 : audit [DBG] from='client.? 172.21.15.6:0/842579087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:34.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:34 smithi130 bash[51302]: debug 2023-08-27T23:08:34.149+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:34.152241+0000) 2023-08-27T23:08:34.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:34 smithi006 bash[27292]: cluster 2023-08-27T23:08:32.601944+0000 mgr.y (mgr.14901) 2100 : cluster [DBG] pgmap v2107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:34 smithi006 bash[27292]: audit 2023-08-27T23:08:34.241932+0000 mon.a (mon.0) 1940 : audit [DBG] from='client.? 172.21.15.6:0/842579087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:34 smithi006 bash[33918]: cluster 2023-08-27T23:08:32.601944+0000 mgr.y (mgr.14901) 2100 : cluster [DBG] pgmap v2107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:34 smithi006 bash[33918]: audit 2023-08-27T23:08:34.241932+0000 mon.a (mon.0) 1940 : audit [DBG] from='client.? 172.21.15.6:0/842579087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:35.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:35 smithi130 bash[51302]: debug 2023-08-27T23:08:35.149+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:35.152530+0000) 2023-08-27T23:08:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:36 smithi130 bash[27579]: cluster 2023-08-27T23:08:34.603080+0000 mgr.y (mgr.14901) 2101 : cluster [DBG] pgmap v2108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:36.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:36 smithi130 bash[51302]: debug 2023-08-27T23:08:36.149+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:36.152858+0000) 2023-08-27T23:08:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:36 smithi006 bash[33918]: cluster 2023-08-27T23:08:34.603080+0000 mgr.y (mgr.14901) 2101 : cluster [DBG] pgmap v2108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:36 smithi006 bash[27292]: cluster 2023-08-27T23:08:34.603080+0000 mgr.y (mgr.14901) 2101 : cluster [DBG] pgmap v2108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:37.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:37 smithi130 bash[27579]: audit 2023-08-27T23:08:36.653927+0000 mon.a (mon.0) 1941 : audit [DBG] from='client.? 172.21.15.6:0/832303647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:37.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:37 smithi130 bash[51302]: debug 2023-08-27T23:08:37.149+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:37.153141+0000) 2023-08-27T23:08:37.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:37 smithi006 bash[27292]: audit 2023-08-27T23:08:36.653927+0000 mon.a (mon.0) 1941 : audit [DBG] from='client.? 172.21.15.6:0/832303647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:37.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:37 smithi006 bash[33918]: audit 2023-08-27T23:08:36.653927+0000 mon.a (mon.0) 1941 : audit [DBG] from='client.? 172.21.15.6:0/832303647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:38 smithi130 bash[27579]: cluster 2023-08-27T23:08:36.603677+0000 mgr.y (mgr.14901) 2102 : cluster [DBG] pgmap v2109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:38.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:38 smithi130 bash[51302]: debug 2023-08-27T23:08:38.149+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:38.153450+0000) 2023-08-27T23:08:38.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:38 smithi006 bash[27292]: cluster 2023-08-27T23:08:36.603677+0000 mgr.y (mgr.14901) 2102 : cluster [DBG] pgmap v2109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:38.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:38 smithi006 bash[33918]: cluster 2023-08-27T23:08:36.603677+0000 mgr.y (mgr.14901) 2102 : cluster [DBG] pgmap v2109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:39.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:39 smithi130 bash[27579]: audit 2023-08-27T23:08:39.072681+0000 mon.a (mon.0) 1942 : audit [DBG] from='client.? 172.21.15.6:0/2963655585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:39.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:39 smithi130 bash[51302]: debug 2023-08-27T23:08:39.149+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:39.153784+0000) 2023-08-27T23:08:39.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:39 smithi006 bash[27292]: audit 2023-08-27T23:08:39.072681+0000 mon.a (mon.0) 1942 : audit [DBG] from='client.? 172.21.15.6:0/2963655585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:39.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:39 smithi006 bash[33918]: audit 2023-08-27T23:08:39.072681+0000 mon.a (mon.0) 1942 : audit [DBG] from='client.? 172.21.15.6:0/2963655585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:40 smithi130 bash[27579]: cluster 2023-08-27T23:08:38.604300+0000 mgr.y (mgr.14901) 2103 : cluster [DBG] pgmap v2110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:40.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:40 smithi130 bash[51302]: debug 2023-08-27T23:08:40.149+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:40.154058+0000) 2023-08-27T23:08:40.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:40 smithi006 bash[33918]: cluster 2023-08-27T23:08:38.604300+0000 mgr.y (mgr.14901) 2103 : cluster [DBG] pgmap v2110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:40 smithi006 bash[27292]: cluster 2023-08-27T23:08:38.604300+0000 mgr.y (mgr.14901) 2103 : cluster [DBG] pgmap v2110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:08:40.385Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:08:40.388Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:08:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:08:40] "GET /metrics HTTP/1.1" 200 193091 "" "Prometheus/2.33.4" 2023-08-27T23:08:41.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:41 smithi130 bash[51302]: debug 2023-08-27T23:08:41.153+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:41.154337+0000) 2023-08-27T23:08:42.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:41 smithi006 bash[56298]: level=error ts=2023-08-27T23:08:41.948Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:42.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:41 smithi006 bash[56298]: level=warn ts=2023-08-27T23:08:41.950Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:42 smithi130 bash[27579]: cluster 2023-08-27T23:08:40.605428+0000 mgr.y (mgr.14901) 2104 : cluster [DBG] pgmap v2111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:42 smithi130 bash[27579]: audit 2023-08-27T23:08:41.490764+0000 mon.c (mon.2) 836 : audit [DBG] from='client.? 172.21.15.6:0/1364294350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:42.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:42 smithi130 bash[51302]: debug 2023-08-27T23:08:42.149+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:42.154620+0000) 2023-08-27T23:08:42.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:42 smithi006 bash[27292]: cluster 2023-08-27T23:08:40.605428+0000 mgr.y (mgr.14901) 2104 : cluster [DBG] pgmap v2111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:42 smithi006 bash[27292]: audit 2023-08-27T23:08:41.490764+0000 mon.c (mon.2) 836 : audit [DBG] from='client.? 172.21.15.6:0/1364294350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:42.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:42 smithi006 bash[33918]: cluster 2023-08-27T23:08:40.605428+0000 mgr.y (mgr.14901) 2104 : cluster [DBG] pgmap v2111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:42.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:42 smithi006 bash[33918]: audit 2023-08-27T23:08:41.490764+0000 mon.c (mon.2) 836 : audit [DBG] from='client.? 172.21.15.6:0/1364294350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:43.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:43 smithi130 bash[51302]: debug 2023-08-27T23:08:43.153+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:43.154943+0000) 2023-08-27T23:08:44.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:08:43.850Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:08:43.852Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:44 smithi130 bash[27579]: cluster 2023-08-27T23:08:42.606118+0000 mgr.y (mgr.14901) 2105 : cluster [DBG] pgmap v2112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:44 smithi130 bash[27579]: audit 2023-08-27T23:08:43.908086+0000 mon.c (mon.2) 837 : audit [DBG] from='client.? 172.21.15.6:0/386972084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:44.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:44 smithi130 bash[51302]: debug 2023-08-27T23:08:44.153+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:44.155183+0000) 2023-08-27T23:08:44.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:44 smithi006 bash[33918]: cluster 2023-08-27T23:08:42.606118+0000 mgr.y (mgr.14901) 2105 : cluster [DBG] pgmap v2112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:44.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:44 smithi006 bash[33918]: audit 2023-08-27T23:08:43.908086+0000 mon.c (mon.2) 837 : audit [DBG] from='client.? 172.21.15.6:0/386972084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:44.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:44 smithi006 bash[27292]: cluster 2023-08-27T23:08:42.606118+0000 mgr.y (mgr.14901) 2105 : cluster [DBG] pgmap v2112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:44.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:44 smithi006 bash[27292]: audit 2023-08-27T23:08:43.908086+0000 mon.c (mon.2) 837 : audit [DBG] from='client.? 172.21.15.6:0/386972084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:45.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:45 smithi130 bash[51302]: debug 2023-08-27T23:08:45.153+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:45.155493+0000) 2023-08-27T23:08:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:46 smithi130 bash[27579]: cluster 2023-08-27T23:08:44.607387+0000 mgr.y (mgr.14901) 2106 : cluster [DBG] pgmap v2113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:46 smithi130 bash[27579]: audit 2023-08-27T23:08:46.320454+0000 mon.c (mon.2) 838 : audit [DBG] from='client.? 172.21.15.6:0/3803099597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:46.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:46 smithi130 bash[51302]: debug 2023-08-27T23:08:46.153+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:46.155707+0000) 2023-08-27T23:08:46.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:46 smithi006 bash[33918]: cluster 2023-08-27T23:08:44.607387+0000 mgr.y (mgr.14901) 2106 : cluster [DBG] pgmap v2113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:46 smithi006 bash[33918]: audit 2023-08-27T23:08:46.320454+0000 mon.c (mon.2) 838 : audit [DBG] from='client.? 172.21.15.6:0/3803099597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:46 smithi006 bash[27292]: cluster 2023-08-27T23:08:44.607387+0000 mgr.y (mgr.14901) 2106 : cluster [DBG] pgmap v2113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:46 smithi006 bash[27292]: audit 2023-08-27T23:08:46.320454+0000 mon.c (mon.2) 838 : audit [DBG] from='client.? 172.21.15.6:0/3803099597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:47.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:47 smithi130 bash[51302]: debug 2023-08-27T23:08:47.153+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:47.155988+0000) 2023-08-27T23:08:48.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:48 smithi130 bash[27579]: cluster 2023-08-27T23:08:46.608102+0000 mgr.y (mgr.14901) 2107 : cluster [DBG] pgmap v2114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:48.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:48 smithi130 bash[51302]: debug 2023-08-27T23:08:48.153+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:48.156264+0000) 2023-08-27T23:08:48.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:48 smithi006 bash[27292]: cluster 2023-08-27T23:08:46.608102+0000 mgr.y (mgr.14901) 2107 : cluster [DBG] pgmap v2114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:48 smithi006 bash[33918]: cluster 2023-08-27T23:08:46.608102+0000 mgr.y (mgr.14901) 2107 : cluster [DBG] pgmap v2114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:49.413 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:49 smithi130 bash[51302]: debug 2023-08-27T23:08:49.153+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:49.156539+0000) 2023-08-27T23:08:49.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:49 smithi006 bash[33918]: audit 2023-08-27T23:08:48.740430+0000 mon.a (mon.0) 1943 : audit [DBG] from='client.? 172.21.15.6:0/660188834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:49.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:49 smithi006 bash[27292]: audit 2023-08-27T23:08:48.740430+0000 mon.a (mon.0) 1943 : audit [DBG] from='client.? 172.21.15.6:0/660188834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:49.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:49 smithi130 bash[27579]: audit 2023-08-27T23:08:48.740430+0000 mon.a (mon.0) 1943 : audit [DBG] from='client.? 172.21.15.6:0/660188834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:50.415 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:50 smithi130 bash[51302]: debug 2023-08-27T23:08:50.153+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:50.156822+0000) 2023-08-27T23:08:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:50 smithi006 bash[33918]: cluster 2023-08-27T23:08:48.608732+0000 mgr.y (mgr.14901) 2108 : cluster [DBG] pgmap v2115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:50 smithi006 bash[27292]: cluster 2023-08-27T23:08:48.608732+0000 mgr.y (mgr.14901) 2108 : cluster [DBG] pgmap v2115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:08:50.386Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:08:50.389Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:50 smithi130 bash[27579]: cluster 2023-08-27T23:08:48.608732+0000 mgr.y (mgr.14901) 2108 : cluster [DBG] pgmap v2115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:08:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:08:50] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:08:51.425 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:51 smithi130 bash[51302]: debug 2023-08-27T23:08:51.153+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:51.157106+0000) 2023-08-27T23:08:51.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:51 smithi006 bash[33918]: audit 2023-08-27T23:08:51.167956+0000 mon.c (mon.2) 839 : audit [DBG] from='client.? 172.21.15.6:0/1465232438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:51.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:51 smithi006 bash[27292]: audit 2023-08-27T23:08:51.167956+0000 mon.c (mon.2) 839 : audit [DBG] from='client.? 172.21.15.6:0/1465232438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:51.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:51 smithi130 bash[27579]: audit 2023-08-27T23:08:51.167956+0000 mon.c (mon.2) 839 : audit [DBG] from='client.? 172.21.15.6:0/1465232438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:52.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:51 smithi006 bash[56298]: level=error ts=2023-08-27T23:08:51.950Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:52.230 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:51 smithi006 bash[56298]: level=warn ts=2023-08-27T23:08:51.951Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:52.429 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:52 smithi130 bash[51302]: debug 2023-08-27T23:08:52.153+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:52.157324+0000) 2023-08-27T23:08:52.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:52 smithi006 bash[33918]: cluster 2023-08-27T23:08:50.609731+0000 mgr.y (mgr.14901) 2109 : cluster [DBG] pgmap v2116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:52.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:52 smithi006 bash[27292]: cluster 2023-08-27T23:08:50.609731+0000 mgr.y (mgr.14901) 2109 : cluster [DBG] pgmap v2116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:52 smithi130 bash[27579]: cluster 2023-08-27T23:08:50.609731+0000 mgr.y (mgr.14901) 2109 : cluster [DBG] pgmap v2116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:53.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:53 smithi130 bash[51302]: debug 2023-08-27T23:08:53.157+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:53.157600+0000) 2023-08-27T23:08:54.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:08:53.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:08:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:08:53.853Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:08:54.434 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:54 smithi130 bash[51302]: debug 2023-08-27T23:08:54.153+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:54.157903+0000) 2023-08-27T23:08:54.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:54 smithi006 bash[27292]: cluster 2023-08-27T23:08:52.610378+0000 mgr.y (mgr.14901) 2110 : cluster [DBG] pgmap v2117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:54 smithi006 bash[27292]: audit 2023-08-27T23:08:53.589386+0000 mon.a (mon.0) 1944 : audit [DBG] from='client.? 172.21.15.6:0/4293357592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:54.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:54 smithi006 bash[33918]: cluster 2023-08-27T23:08:52.610378+0000 mgr.y (mgr.14901) 2110 : cluster [DBG] pgmap v2117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:54.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:54 smithi006 bash[33918]: audit 2023-08-27T23:08:53.589386+0000 mon.a (mon.0) 1944 : audit [DBG] from='client.? 172.21.15.6:0/4293357592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:54 smithi130 bash[27579]: cluster 2023-08-27T23:08:52.610378+0000 mgr.y (mgr.14901) 2110 : cluster [DBG] pgmap v2117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:54 smithi130 bash[27579]: audit 2023-08-27T23:08:53.589386+0000 mon.a (mon.0) 1944 : audit [DBG] from='client.? 172.21.15.6:0/4293357592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:55.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:55 smithi130 bash[51302]: debug 2023-08-27T23:08:55.153+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:55.158166+0000) 2023-08-27T23:08:56.439 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:56 smithi130 bash[51302]: debug 2023-08-27T23:08:56.157+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:56.158423+0000) 2023-08-27T23:08:56.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:56 smithi006 bash[33918]: cluster 2023-08-27T23:08:54.611495+0000 mgr.y (mgr.14901) 2111 : cluster [DBG] pgmap v2118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:56 smithi006 bash[33918]: audit 2023-08-27T23:08:56.001644+0000 mon.c (mon.2) 840 : audit [DBG] from='client.? 172.21.15.6:0/3549054556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:56 smithi006 bash[27292]: cluster 2023-08-27T23:08:54.611495+0000 mgr.y (mgr.14901) 2111 : cluster [DBG] pgmap v2118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:56 smithi006 bash[27292]: audit 2023-08-27T23:08:56.001644+0000 mon.c (mon.2) 840 : audit [DBG] from='client.? 172.21.15.6:0/3549054556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:56.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:56 smithi130 bash[27579]: cluster 2023-08-27T23:08:54.611495+0000 mgr.y (mgr.14901) 2111 : cluster [DBG] pgmap v2118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:56 smithi130 bash[27579]: audit 2023-08-27T23:08:56.001644+0000 mon.c (mon.2) 840 : audit [DBG] from='client.? 172.21.15.6:0/3549054556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:57.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:57 smithi130 bash[51302]: debug 2023-08-27T23:08:57.157+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:57.158722+0000) 2023-08-27T23:08:58.444 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:58 smithi130 bash[51302]: debug 2023-08-27T23:08:58.157+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:58.159038+0000) 2023-08-27T23:08:58.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:58 smithi006 bash[27292]: cluster 2023-08-27T23:08:56.612143+0000 mgr.y (mgr.14901) 2112 : cluster [DBG] pgmap v2119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:58 smithi006 bash[27292]: audit 2023-08-27T23:08:58.417313+0000 mon.c (mon.2) 841 : audit [DBG] from='client.? 172.21.15.6:0/1626654388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:58 smithi006 bash[33918]: cluster 2023-08-27T23:08:56.612143+0000 mgr.y (mgr.14901) 2112 : cluster [DBG] pgmap v2119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:58 smithi006 bash[33918]: audit 2023-08-27T23:08:58.417313+0000 mon.c (mon.2) 841 : audit [DBG] from='client.? 172.21.15.6:0/1626654388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:58 smithi130 bash[27579]: cluster 2023-08-27T23:08:56.612143+0000 mgr.y (mgr.14901) 2112 : cluster [DBG] pgmap v2119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:08:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:58 smithi130 bash[27579]: audit 2023-08-27T23:08:58.417313+0000 mon.c (mon.2) 841 : audit [DBG] from='client.? 172.21.15.6:0/1626654388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:59.452 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:08:59 smithi130 bash[51302]: debug 2023-08-27T23:08:59.157+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:59.159312+0000) 2023-08-27T23:08:59.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:08:59 smithi006 bash[27292]: audit 2023-08-27T23:08:59.310316+0000 mon.b (mon.1) 323 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:08:59.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:08:59 smithi006 bash[33918]: audit 2023-08-27T23:08:59.310316+0000 mon.b (mon.1) 323 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:08:59.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:08:59 smithi130 bash[27579]: audit 2023-08-27T23:08:59.310316+0000 mon.b (mon.1) 323 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:09:00.457 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:00 smithi130 bash[51302]: debug 2023-08-27T23:09:00.157+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:00.159589+0000) 2023-08-27T23:09:00.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:00 smithi006 bash[27292]: cluster 2023-08-27T23:08:58.612764+0000 mgr.y (mgr.14901) 2113 : cluster [DBG] pgmap v2120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:00 smithi006 bash[33918]: cluster 2023-08-27T23:08:58.612764+0000 mgr.y (mgr.14901) 2113 : cluster [DBG] pgmap v2120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:09:00.387Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:09:00.389Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:00 smithi130 bash[27579]: cluster 2023-08-27T23:08:58.612764+0000 mgr.y (mgr.14901) 2113 : cluster [DBG] pgmap v2120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:09:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:09:00] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:09:01.464 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:01 smithi130 bash[51302]: debug 2023-08-27T23:09:01.157+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:01.159900+0000) 2023-08-27T23:09:01.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:01 smithi006 bash[27292]: audit 2023-08-27T23:09:00.839891+0000 mon.c (mon.2) 842 : audit [DBG] from='client.? 172.21.15.6:0/1676674757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:01.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:01 smithi006 bash[33918]: audit 2023-08-27T23:09:00.839891+0000 mon.c (mon.2) 842 : audit [DBG] from='client.? 172.21.15.6:0/1676674757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:01.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:01 smithi130 bash[27579]: audit 2023-08-27T23:09:00.839891+0000 mon.c (mon.2) 842 : audit [DBG] from='client.? 172.21.15.6:0/1676674757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:02.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:01 smithi006 bash[56298]: level=error ts=2023-08-27T23:09:01.950Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:02.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:01 smithi006 bash[56298]: level=warn ts=2023-08-27T23:09:01.952Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:02.468 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:02 smithi130 bash[51302]: debug 2023-08-27T23:09:02.157+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:02.160196+0000) 2023-08-27T23:09:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:02 smithi130 bash[27579]: cluster 2023-08-27T23:09:00.613844+0000 mgr.y (mgr.14901) 2114 : cluster [DBG] pgmap v2121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:02 smithi006 bash[27292]: cluster 2023-08-27T23:09:00.613844+0000 mgr.y (mgr.14901) 2114 : cluster [DBG] pgmap v2121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:02 smithi006 bash[33918]: cluster 2023-08-27T23:09:00.613844+0000 mgr.y (mgr.14901) 2114 : cluster [DBG] pgmap v2121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:03.476 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:03 smithi130 bash[51302]: debug 2023-08-27T23:09:03.157+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:03.160478+0000) 2023-08-27T23:09:03.854 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:03 smithi006 bash[27292]: audit 2023-08-27T23:09:03.260380+0000 mon.c (mon.2) 843 : audit [DBG] from='client.? 172.21.15.6:0/4116745942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:03.855 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:03 smithi006 bash[33918]: audit 2023-08-27T23:09:03.260380+0000 mon.c (mon.2) 843 : audit [DBG] from='client.? 172.21.15.6:0/4116745942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:03.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:03 smithi130 bash[27579]: audit 2023-08-27T23:09:03.260380+0000 mon.c (mon.2) 843 : audit [DBG] from='client.? 172.21.15.6:0/4116745942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:04.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:09:03.852Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:09:03.854Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:04.480 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:04 smithi130 bash[51302]: debug 2023-08-27T23:09:04.157+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:04.160740+0000) 2023-08-27T23:09:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:04 smithi130 bash[27579]: cluster 2023-08-27T23:09:02.614410+0000 mgr.y (mgr.14901) 2115 : cluster [DBG] pgmap v2122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:04.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:04 smithi006 bash[33918]: cluster 2023-08-27T23:09:02.614410+0000 mgr.y (mgr.14901) 2115 : cluster [DBG] pgmap v2122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:04.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:04 smithi006 bash[27292]: cluster 2023-08-27T23:09:02.614410+0000 mgr.y (mgr.14901) 2115 : cluster [DBG] pgmap v2122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:05.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:05 smithi130 bash[51302]: debug 2023-08-27T23:09:05.157+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:05.161022+0000) 2023-08-27T23:09:06.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:06 smithi130 bash[51302]: debug 2023-08-27T23:09:06.157+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:06.161318+0000) 2023-08-27T23:09:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:06 smithi130 bash[27579]: cluster 2023-08-27T23:09:04.615774+0000 mgr.y (mgr.14901) 2116 : cluster [DBG] pgmap v2123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:06 smithi130 bash[27579]: audit 2023-08-27T23:09:05.221427+0000 mon.a (mon.0) 1945 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:09:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:06 smithi130 bash[27579]: audit 2023-08-27T23:09:05.610757+0000 mon.a (mon.0) 1946 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:09:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:06 smithi130 bash[27579]: audit 2023-08-27T23:09:05.689684+0000 mon.a (mon.0) 1947 : audit [DBG] from='client.? 172.21.15.6:0/4151264074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:06 smithi130 bash[27579]: audit 2023-08-27T23:09:05.898561+0000 mon.b (mon.1) 324 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:09:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:06 smithi130 bash[27579]: audit 2023-08-27T23:09:05.900560+0000 mon.b (mon.1) 325 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:09:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:06 smithi130 bash[27579]: audit 2023-08-27T23:09:05.910801+0000 mon.a (mon.0) 1948 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:09:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:06 smithi006 bash[27292]: cluster 2023-08-27T23:09:04.615774+0000 mgr.y (mgr.14901) 2116 : cluster [DBG] pgmap v2123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:06 smithi006 bash[27292]: audit 2023-08-27T23:09:05.221427+0000 mon.a (mon.0) 1945 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:09:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:06 smithi006 bash[27292]: audit 2023-08-27T23:09:05.610757+0000 mon.a (mon.0) 1946 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:09:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:06 smithi006 bash[27292]: audit 2023-08-27T23:09:05.689684+0000 mon.a (mon.0) 1947 : audit [DBG] from='client.? 172.21.15.6:0/4151264074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:06 smithi006 bash[27292]: audit 2023-08-27T23:09:05.898561+0000 mon.b (mon.1) 324 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:09:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:06 smithi006 bash[27292]: audit 2023-08-27T23:09:05.900560+0000 mon.b (mon.1) 325 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:09:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:06 smithi006 bash[27292]: audit 2023-08-27T23:09:05.910801+0000 mon.a (mon.0) 1948 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:09:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:06 smithi006 bash[33918]: cluster 2023-08-27T23:09:04.615774+0000 mgr.y (mgr.14901) 2116 : cluster [DBG] pgmap v2123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:06 smithi006 bash[33918]: audit 2023-08-27T23:09:05.221427+0000 mon.a (mon.0) 1945 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:09:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:06 smithi006 bash[33918]: audit 2023-08-27T23:09:05.610757+0000 mon.a (mon.0) 1946 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:09:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:06 smithi006 bash[33918]: audit 2023-08-27T23:09:05.689684+0000 mon.a (mon.0) 1947 : audit [DBG] from='client.? 172.21.15.6:0/4151264074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:06 smithi006 bash[33918]: audit 2023-08-27T23:09:05.898561+0000 mon.b (mon.1) 324 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:09:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:06 smithi006 bash[33918]: audit 2023-08-27T23:09:05.900560+0000 mon.b (mon.1) 325 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:09:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:06 smithi006 bash[33918]: audit 2023-08-27T23:09:05.910801+0000 mon.a (mon.0) 1948 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:09:07.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:07 smithi130 bash[51302]: debug 2023-08-27T23:09:07.161+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:07.161615+0000) 2023-08-27T23:09:08.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:08 smithi130 bash[51302]: debug 2023-08-27T23:09:08.157+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:08.161917+0000) 2023-08-27T23:09:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:08 smithi130 bash[27579]: cluster 2023-08-27T23:09:06.616464+0000 mgr.y (mgr.14901) 2117 : cluster [DBG] pgmap v2124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:08 smithi130 bash[27579]: audit 2023-08-27T23:09:08.111519+0000 mon.a (mon.0) 1949 : audit [DBG] from='client.? 172.21.15.6:0/1499900991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:08 smithi006 bash[33918]: cluster 2023-08-27T23:09:06.616464+0000 mgr.y (mgr.14901) 2117 : cluster [DBG] pgmap v2124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:08 smithi006 bash[33918]: audit 2023-08-27T23:09:08.111519+0000 mon.a (mon.0) 1949 : audit [DBG] from='client.? 172.21.15.6:0/1499900991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:08 smithi006 bash[27292]: cluster 2023-08-27T23:09:06.616464+0000 mgr.y (mgr.14901) 2117 : cluster [DBG] pgmap v2124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:08 smithi006 bash[27292]: audit 2023-08-27T23:09:08.111519+0000 mon.a (mon.0) 1949 : audit [DBG] from='client.? 172.21.15.6:0/1499900991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:09.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:09 smithi130 bash[51302]: debug 2023-08-27T23:09:09.157+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:09.162264+0000) 2023-08-27T23:09:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:10 smithi130 bash[27579]: cluster 2023-08-27T23:09:08.617183+0000 mgr.y (mgr.14901) 2118 : cluster [DBG] pgmap v2125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:10.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:10 smithi130 bash[51302]: debug 2023-08-27T23:09:10.157+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:10.162569+0000) 2023-08-27T23:09:10.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:10 smithi006 bash[27292]: cluster 2023-08-27T23:09:08.617183+0000 mgr.y (mgr.14901) 2118 : cluster [DBG] pgmap v2125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:10 smithi006 bash[33918]: cluster 2023-08-27T23:09:08.617183+0000 mgr.y (mgr.14901) 2118 : cluster [DBG] pgmap v2125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:09:10.388Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:09:10.390Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:09:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:09:10] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:09:11.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:11 smithi130 bash[27579]: audit 2023-08-27T23:09:10.540879+0000 mon.a (mon.0) 1950 : audit [DBG] from='client.? 172.21.15.6:0/1148247131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:11.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:11 smithi130 bash[51302]: debug 2023-08-27T23:09:11.161+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:11.162872+0000) 2023-08-27T23:09:11.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:11 smithi006 bash[27292]: audit 2023-08-27T23:09:10.540879+0000 mon.a (mon.0) 1950 : audit [DBG] from='client.? 172.21.15.6:0/1148247131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:11.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:11 smithi006 bash[33918]: audit 2023-08-27T23:09:10.540879+0000 mon.a (mon.0) 1950 : audit [DBG] from='client.? 172.21.15.6:0/1148247131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:12.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:11 smithi006 bash[56298]: level=error ts=2023-08-27T23:09:11.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:12.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:11 smithi006 bash[56298]: level=warn ts=2023-08-27T23:09:11.953Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:12 smithi130 bash[27579]: cluster 2023-08-27T23:09:10.618215+0000 mgr.y (mgr.14901) 2119 : cluster [DBG] pgmap v2126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:12.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:12 smithi130 bash[51302]: debug 2023-08-27T23:09:12.161+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:12.163138+0000) 2023-08-27T23:09:12.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:12 smithi006 bash[27292]: cluster 2023-08-27T23:09:10.618215+0000 mgr.y (mgr.14901) 2119 : cluster [DBG] pgmap v2126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:12.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:12 smithi006 bash[33918]: cluster 2023-08-27T23:09:10.618215+0000 mgr.y (mgr.14901) 2119 : cluster [DBG] pgmap v2126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:13.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:13 smithi130 bash[27579]: audit 2023-08-27T23:09:12.952656+0000 mon.c (mon.2) 844 : audit [DBG] from='client.? 172.21.15.6:0/3685558282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:13.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:13 smithi130 bash[51302]: debug 2023-08-27T23:09:13.161+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:13.163411+0000) 2023-08-27T23:09:13.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:13 smithi006 bash[27292]: audit 2023-08-27T23:09:12.952656+0000 mon.c (mon.2) 844 : audit [DBG] from='client.? 172.21.15.6:0/3685558282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:13.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:13 smithi006 bash[33918]: audit 2023-08-27T23:09:12.952656+0000 mon.c (mon.2) 844 : audit [DBG] from='client.? 172.21.15.6:0/3685558282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:09:13.853Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:09:13.855Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:14 smithi130 bash[27579]: cluster 2023-08-27T23:09:12.618720+0000 mgr.y (mgr.14901) 2120 : cluster [DBG] pgmap v2127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:14.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:14 smithi130 bash[51302]: debug 2023-08-27T23:09:14.161+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:14.163731+0000) 2023-08-27T23:09:14.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:14 smithi006 bash[27292]: cluster 2023-08-27T23:09:12.618720+0000 mgr.y (mgr.14901) 2120 : cluster [DBG] pgmap v2127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:14 smithi006 bash[33918]: cluster 2023-08-27T23:09:12.618720+0000 mgr.y (mgr.14901) 2120 : cluster [DBG] pgmap v2127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:15.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:15 smithi130 bash[27579]: audit 2023-08-27T23:09:15.361334+0000 mon.a (mon.0) 1951 : audit [DBG] from='client.? 172.21.15.6:0/77623282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:15.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:15 smithi130 bash[51302]: debug 2023-08-27T23:09:15.160+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:15.164052+0000) 2023-08-27T23:09:15.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:15 smithi006 bash[27292]: audit 2023-08-27T23:09:15.361334+0000 mon.a (mon.0) 1951 : audit [DBG] from='client.? 172.21.15.6:0/77623282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:15.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:15 smithi006 bash[33918]: audit 2023-08-27T23:09:15.361334+0000 mon.a (mon.0) 1951 : audit [DBG] from='client.? 172.21.15.6:0/77623282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:16 smithi130 bash[27579]: cluster 2023-08-27T23:09:14.619821+0000 mgr.y (mgr.14901) 2121 : cluster [DBG] pgmap v2128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:16.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:16 smithi130 bash[51302]: debug 2023-08-27T23:09:16.160+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:16.164382+0000) 2023-08-27T23:09:16.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:16 smithi006 bash[27292]: cluster 2023-08-27T23:09:14.619821+0000 mgr.y (mgr.14901) 2121 : cluster [DBG] pgmap v2128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:16 smithi006 bash[33918]: cluster 2023-08-27T23:09:14.619821+0000 mgr.y (mgr.14901) 2121 : cluster [DBG] pgmap v2128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:17.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:17 smithi130 bash[51302]: debug 2023-08-27T23:09:17.160+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:17.164653+0000) 2023-08-27T23:09:18.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:18 smithi130 bash[27579]: cluster 2023-08-27T23:09:16.620451+0000 mgr.y (mgr.14901) 2122 : cluster [DBG] pgmap v2129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:18 smithi130 bash[27579]: audit 2023-08-27T23:09:17.783695+0000 mon.a (mon.0) 1952 : audit [DBG] from='client.? 172.21.15.6:0/489522387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:18.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:18 smithi130 bash[51302]: debug 2023-08-27T23:09:18.160+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:18.165004+0000) 2023-08-27T23:09:18.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:18 smithi006 bash[33918]: cluster 2023-08-27T23:09:16.620451+0000 mgr.y (mgr.14901) 2122 : cluster [DBG] pgmap v2129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:18.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:18 smithi006 bash[33918]: audit 2023-08-27T23:09:17.783695+0000 mon.a (mon.0) 1952 : audit [DBG] from='client.? 172.21.15.6:0/489522387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:18 smithi006 bash[27292]: cluster 2023-08-27T23:09:16.620451+0000 mgr.y (mgr.14901) 2122 : cluster [DBG] pgmap v2129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:18 smithi006 bash[27292]: audit 2023-08-27T23:09:17.783695+0000 mon.a (mon.0) 1952 : audit [DBG] from='client.? 172.21.15.6:0/489522387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:19.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:19 smithi130 bash[51302]: debug 2023-08-27T23:09:19.160+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:19.165268+0000) 2023-08-27T23:09:20.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:20 smithi130 bash[51302]: debug 2023-08-27T23:09:20.164+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:20.165613+0000) 2023-08-27T23:09:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:20 smithi130 bash[27579]: cluster 2023-08-27T23:09:18.621139+0000 mgr.y (mgr.14901) 2123 : cluster [DBG] pgmap v2130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:20 smithi130 bash[27579]: audit 2023-08-27T23:09:20.199693+0000 mon.c (mon.2) 845 : audit [DBG] from='client.? 172.21.15.6:0/1214563293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:20.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:09:20.389Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:20.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:09:20.391Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:20 smithi006 bash[27292]: cluster 2023-08-27T23:09:18.621139+0000 mgr.y (mgr.14901) 2123 : cluster [DBG] pgmap v2130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:20 smithi006 bash[27292]: audit 2023-08-27T23:09:20.199693+0000 mon.c (mon.2) 845 : audit [DBG] from='client.? 172.21.15.6:0/1214563293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:20 smithi006 bash[33918]: cluster 2023-08-27T23:09:18.621139+0000 mgr.y (mgr.14901) 2123 : cluster [DBG] pgmap v2130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:20 smithi006 bash[33918]: audit 2023-08-27T23:09:20.199693+0000 mon.c (mon.2) 845 : audit [DBG] from='client.? 172.21.15.6:0/1214563293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:21.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:09:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:09:20] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:09:21.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:21 smithi130 bash[51302]: debug 2023-08-27T23:09:21.160+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:21.165890+0000) 2023-08-27T23:09:22.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:21 smithi006 bash[56298]: level=error ts=2023-08-27T23:09:21.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:22.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:21 smithi006 bash[56298]: level=warn ts=2023-08-27T23:09:21.954Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:22.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:22 smithi130 bash[27579]: cluster 2023-08-27T23:09:20.622192+0000 mgr.y (mgr.14901) 2124 : cluster [DBG] pgmap v2131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:22.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:22 smithi130 bash[51302]: debug 2023-08-27T23:09:22.160+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:22.166182+0000) 2023-08-27T23:09:22.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:22 smithi006 bash[33918]: cluster 2023-08-27T23:09:20.622192+0000 mgr.y (mgr.14901) 2124 : cluster [DBG] pgmap v2131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:22 smithi006 bash[27292]: cluster 2023-08-27T23:09:20.622192+0000 mgr.y (mgr.14901) 2124 : cluster [DBG] pgmap v2131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:23.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:23 smithi130 bash[27579]: audit 2023-08-27T23:09:22.625005+0000 mon.a (mon.0) 1953 : audit [DBG] from='client.? 172.21.15.6:0/1270638814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:23.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:23 smithi130 bash[51302]: debug 2023-08-27T23:09:23.164+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:23.166462+0000) 2023-08-27T23:09:23.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:23 smithi006 bash[27292]: audit 2023-08-27T23:09:22.625005+0000 mon.a (mon.0) 1953 : audit [DBG] from='client.? 172.21.15.6:0/1270638814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:23.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:23 smithi006 bash[33918]: audit 2023-08-27T23:09:22.625005+0000 mon.a (mon.0) 1953 : audit [DBG] from='client.? 172.21.15.6:0/1270638814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:24.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:09:23.854Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:24.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:09:23.856Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:24.425 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:24 smithi130 bash[51302]: debug 2023-08-27T23:09:24.164+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:24.166742+0000) 2023-08-27T23:09:24.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:24 smithi006 bash[33918]: cluster 2023-08-27T23:09:22.622791+0000 mgr.y (mgr.14901) 2125 : cluster [DBG] pgmap v2132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:24.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:24 smithi006 bash[27292]: cluster 2023-08-27T23:09:22.622791+0000 mgr.y (mgr.14901) 2125 : cluster [DBG] pgmap v2132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:24 smithi130 bash[27579]: cluster 2023-08-27T23:09:22.622791+0000 mgr.y (mgr.14901) 2125 : cluster [DBG] pgmap v2132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:25.433 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:25 smithi130 bash[51302]: debug 2023-08-27T23:09:25.164+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:25.167051+0000) 2023-08-27T23:09:25.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:25 smithi006 bash[33918]: audit 2023-08-27T23:09:25.052054+0000 mon.a (mon.0) 1954 : audit [DBG] from='client.? 172.21.15.6:0/908588064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:25.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:25 smithi006 bash[27292]: audit 2023-08-27T23:09:25.052054+0000 mon.a (mon.0) 1954 : audit [DBG] from='client.? 172.21.15.6:0/908588064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:25.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:25 smithi130 bash[27579]: audit 2023-08-27T23:09:25.052054+0000 mon.a (mon.0) 1954 : audit [DBG] from='client.? 172.21.15.6:0/908588064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:26.436 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:26 smithi130 bash[51302]: debug 2023-08-27T23:09:26.164+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:26.167324+0000) 2023-08-27T23:09:26.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:26 smithi006 bash[33918]: cluster 2023-08-27T23:09:24.623971+0000 mgr.y (mgr.14901) 2126 : cluster [DBG] pgmap v2133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:26 smithi006 bash[27292]: cluster 2023-08-27T23:09:24.623971+0000 mgr.y (mgr.14901) 2126 : cluster [DBG] pgmap v2133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:26.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:26 smithi130 bash[27579]: cluster 2023-08-27T23:09:24.623971+0000 mgr.y (mgr.14901) 2126 : cluster [DBG] pgmap v2133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:27.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:27 smithi130 bash[51302]: debug 2023-08-27T23:09:27.164+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:27.167601+0000) 2023-08-27T23:09:28.441 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:28 smithi130 bash[51302]: debug 2023-08-27T23:09:28.164+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:28.167915+0000) 2023-08-27T23:09:28.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:28 smithi006 bash[33918]: cluster 2023-08-27T23:09:26.624616+0000 mgr.y (mgr.14901) 2127 : cluster [DBG] pgmap v2134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:28 smithi006 bash[33918]: audit 2023-08-27T23:09:27.468546+0000 mon.c (mon.2) 846 : audit [DBG] from='client.? 172.21.15.6:0/2685086564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:28.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:28 smithi006 bash[27292]: cluster 2023-08-27T23:09:26.624616+0000 mgr.y (mgr.14901) 2127 : cluster [DBG] pgmap v2134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:28.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:28 smithi006 bash[27292]: audit 2023-08-27T23:09:27.468546+0000 mon.c (mon.2) 846 : audit [DBG] from='client.? 172.21.15.6:0/2685086564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:28 smithi130 bash[27579]: cluster 2023-08-27T23:09:26.624616+0000 mgr.y (mgr.14901) 2127 : cluster [DBG] pgmap v2134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:28 smithi130 bash[27579]: audit 2023-08-27T23:09:27.468546+0000 mon.c (mon.2) 846 : audit [DBG] from='client.? 172.21.15.6:0/2685086564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:29.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:29 smithi130 bash[51302]: debug 2023-08-27T23:09:29.164+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:29.168169+0000) 2023-08-27T23:09:30.446 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:30 smithi130 bash[51302]: debug 2023-08-27T23:09:30.164+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:30.168433+0000) 2023-08-27T23:09:30.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:30 smithi006 bash[27292]: cluster 2023-08-27T23:09:28.625373+0000 mgr.y (mgr.14901) 2128 : cluster [DBG] pgmap v2135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:30 smithi006 bash[27292]: audit 2023-08-27T23:09:29.874710+0000 mon.a (mon.0) 1955 : audit [DBG] from='client.? 172.21.15.6:0/3158554976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:09:30.390Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:09:30.392Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:30 smithi006 bash[33918]: cluster 2023-08-27T23:09:28.625373+0000 mgr.y (mgr.14901) 2128 : cluster [DBG] pgmap v2135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:30 smithi006 bash[33918]: audit 2023-08-27T23:09:29.874710+0000 mon.a (mon.0) 1955 : audit [DBG] from='client.? 172.21.15.6:0/3158554976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:30.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:30 smithi130 bash[27579]: cluster 2023-08-27T23:09:28.625373+0000 mgr.y (mgr.14901) 2128 : cluster [DBG] pgmap v2135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:30 smithi130 bash[27579]: audit 2023-08-27T23:09:29.874710+0000 mon.a (mon.0) 1955 : audit [DBG] from='client.? 172.21.15.6:0/3158554976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:09:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:09:30] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:09:31.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:31 smithi130 bash[51302]: debug 2023-08-27T23:09:31.164+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:31.168705+0000) 2023-08-27T23:09:32.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:31 smithi006 bash[56298]: level=error ts=2023-08-27T23:09:31.954Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:32.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:31 smithi006 bash[56298]: level=warn ts=2023-08-27T23:09:31.956Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:32.451 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:32 smithi130 bash[51302]: debug 2023-08-27T23:09:32.164+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:32.168991+0000) 2023-08-27T23:09:32.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:32 smithi006 bash[33918]: cluster 2023-08-27T23:09:30.626479+0000 mgr.y (mgr.14901) 2129 : cluster [DBG] pgmap v2136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:32.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:32 smithi006 bash[33918]: audit 2023-08-27T23:09:32.281184+0000 mon.a (mon.0) 1956 : audit [DBG] from='client.? 172.21.15.6:0/3267868558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:32 smithi006 bash[27292]: cluster 2023-08-27T23:09:30.626479+0000 mgr.y (mgr.14901) 2129 : cluster [DBG] pgmap v2136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:32 smithi006 bash[27292]: audit 2023-08-27T23:09:32.281184+0000 mon.a (mon.0) 1956 : audit [DBG] from='client.? 172.21.15.6:0/3267868558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:32 smithi130 bash[27579]: cluster 2023-08-27T23:09:30.626479+0000 mgr.y (mgr.14901) 2129 : cluster [DBG] pgmap v2136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:32 smithi130 bash[27579]: audit 2023-08-27T23:09:32.281184+0000 mon.a (mon.0) 1956 : audit [DBG] from='client.? 172.21.15.6:0/3267868558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:33.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:33 smithi130 bash[51302]: debug 2023-08-27T23:09:33.168+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:33.169257+0000) 2023-08-27T23:09:34.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:09:33.855Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:09:33.857Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:34.468 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:34 smithi130 bash[51302]: debug 2023-08-27T23:09:34.164+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:34.169543+0000) 2023-08-27T23:09:34.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:34 smithi130 bash[27579]: cluster 2023-08-27T23:09:32.627165+0000 mgr.y (mgr.14901) 2130 : cluster [DBG] pgmap v2137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:34.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:34 smithi006 bash[27292]: cluster 2023-08-27T23:09:32.627165+0000 mgr.y (mgr.14901) 2130 : cluster [DBG] pgmap v2137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:34.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:34 smithi006 bash[33918]: cluster 2023-08-27T23:09:32.627165+0000 mgr.y (mgr.14901) 2130 : cluster [DBG] pgmap v2137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:35.478 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:35 smithi130 bash[51302]: debug 2023-08-27T23:09:35.164+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:35.169839+0000) 2023-08-27T23:09:35.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:35 smithi130 bash[27579]: audit 2023-08-27T23:09:34.697304+0000 mon.a (mon.0) 1957 : audit [DBG] from='client.? 172.21.15.6:0/3170659905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:35.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:35 smithi006 bash[27292]: audit 2023-08-27T23:09:34.697304+0000 mon.a (mon.0) 1957 : audit [DBG] from='client.? 172.21.15.6:0/3170659905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:35.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:35 smithi006 bash[33918]: audit 2023-08-27T23:09:34.697304+0000 mon.a (mon.0) 1957 : audit [DBG] from='client.? 172.21.15.6:0/3170659905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:36.481 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:36 smithi130 bash[51302]: debug 2023-08-27T23:09:36.164+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:36.170140+0000) 2023-08-27T23:09:36.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:36 smithi130 bash[27579]: cluster 2023-08-27T23:09:34.628225+0000 mgr.y (mgr.14901) 2131 : cluster [DBG] pgmap v2138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:36.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:36 smithi006 bash[27292]: cluster 2023-08-27T23:09:34.628225+0000 mgr.y (mgr.14901) 2131 : cluster [DBG] pgmap v2138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:36 smithi006 bash[33918]: cluster 2023-08-27T23:09:34.628225+0000 mgr.y (mgr.14901) 2131 : cluster [DBG] pgmap v2138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:37.490 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:37 smithi130 bash[51302]: debug 2023-08-27T23:09:37.168+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:37.170414+0000) 2023-08-27T23:09:37.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:37 smithi130 bash[27579]: audit 2023-08-27T23:09:37.114743+0000 mon.a (mon.0) 1958 : audit [DBG] from='client.? 172.21.15.6:0/297439663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:37.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:37 smithi006 bash[33918]: audit 2023-08-27T23:09:37.114743+0000 mon.a (mon.0) 1958 : audit [DBG] from='client.? 172.21.15.6:0/297439663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:37.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:37 smithi006 bash[27292]: audit 2023-08-27T23:09:37.114743+0000 mon.a (mon.0) 1958 : audit [DBG] from='client.? 172.21.15.6:0/297439663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:38.491 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:38 smithi130 bash[51302]: debug 2023-08-27T23:09:38.168+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:38.170727+0000) 2023-08-27T23:09:38.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:38 smithi130 bash[27579]: cluster 2023-08-27T23:09:36.628877+0000 mgr.y (mgr.14901) 2132 : cluster [DBG] pgmap v2139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:38.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:38 smithi006 bash[27292]: cluster 2023-08-27T23:09:36.628877+0000 mgr.y (mgr.14901) 2132 : cluster [DBG] pgmap v2139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:38.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:38 smithi006 bash[33918]: cluster 2023-08-27T23:09:36.628877+0000 mgr.y (mgr.14901) 2132 : cluster [DBG] pgmap v2139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:39.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:39 smithi130 bash[51302]: debug 2023-08-27T23:09:39.168+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:39.171002+0000) 2023-08-27T23:09:40.497 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:40 smithi130 bash[51302]: debug 2023-08-27T23:09:40.168+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:40.171299+0000) 2023-08-27T23:09:40.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:40 smithi006 bash[27292]: cluster 2023-08-27T23:09:38.629535+0000 mgr.y (mgr.14901) 2133 : cluster [DBG] pgmap v2140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:40 smithi006 bash[27292]: audit 2023-08-27T23:09:39.528118+0000 mon.a (mon.0) 1959 : audit [DBG] from='client.? 172.21.15.6:0/2637141390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:40 smithi006 bash[33918]: cluster 2023-08-27T23:09:38.629535+0000 mgr.y (mgr.14901) 2133 : cluster [DBG] pgmap v2140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:40 smithi006 bash[33918]: audit 2023-08-27T23:09:39.528118+0000 mon.a (mon.0) 1959 : audit [DBG] from='client.? 172.21.15.6:0/2637141390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:09:40.390Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:09:40.393Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:40 smithi130 bash[27579]: cluster 2023-08-27T23:09:38.629535+0000 mgr.y (mgr.14901) 2133 : cluster [DBG] pgmap v2140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:40 smithi130 bash[27579]: audit 2023-08-27T23:09:39.528118+0000 mon.a (mon.0) 1959 : audit [DBG] from='client.? 172.21.15.6:0/2637141390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:09:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:09:40] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:09:41.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:41 smithi130 bash[51302]: debug 2023-08-27T23:09:41.168+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:41.171623+0000) 2023-08-27T23:09:42.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:41 smithi006 bash[56298]: level=error ts=2023-08-27T23:09:41.955Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:42.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:41 smithi006 bash[56298]: level=warn ts=2023-08-27T23:09:41.956Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:42.502 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:42 smithi130 bash[51302]: debug 2023-08-27T23:09:42.168+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:42.171850+0000) 2023-08-27T23:09:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:42 smithi130 bash[27579]: cluster 2023-08-27T23:09:40.630680+0000 mgr.y (mgr.14901) 2134 : cluster [DBG] pgmap v2141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:42 smithi130 bash[27579]: audit 2023-08-27T23:09:41.938278+0000 mon.c (mon.2) 847 : audit [DBG] from='client.? 172.21.15.6:0/3760546124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:42.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:42 smithi006 bash[33918]: cluster 2023-08-27T23:09:40.630680+0000 mgr.y (mgr.14901) 2134 : cluster [DBG] pgmap v2141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:42.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:42 smithi006 bash[33918]: audit 2023-08-27T23:09:41.938278+0000 mon.c (mon.2) 847 : audit [DBG] from='client.? 172.21.15.6:0/3760546124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:42 smithi006 bash[27292]: cluster 2023-08-27T23:09:40.630680+0000 mgr.y (mgr.14901) 2134 : cluster [DBG] pgmap v2141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:42 smithi006 bash[27292]: audit 2023-08-27T23:09:41.938278+0000 mon.c (mon.2) 847 : audit [DBG] from='client.? 172.21.15.6:0/3760546124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:43.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:43 smithi130 bash[51302]: debug 2023-08-27T23:09:43.168+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:43.172119+0000) 2023-08-27T23:09:44.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:09:43.855Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:09:43.857Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:44.510 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:44 smithi130 bash[51302]: debug 2023-08-27T23:09:44.168+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:44.172394+0000) 2023-08-27T23:09:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:44 smithi130 bash[27579]: cluster 2023-08-27T23:09:42.631288+0000 mgr.y (mgr.14901) 2135 : cluster [DBG] pgmap v2142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:44 smithi130 bash[27579]: audit 2023-08-27T23:09:44.362201+0000 mon.c (mon.2) 848 : audit [DBG] from='client.? 172.21.15.6:0/1946545398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:44.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:44 smithi006 bash[27292]: cluster 2023-08-27T23:09:42.631288+0000 mgr.y (mgr.14901) 2135 : cluster [DBG] pgmap v2142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:44.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:44 smithi006 bash[27292]: audit 2023-08-27T23:09:44.362201+0000 mon.c (mon.2) 848 : audit [DBG] from='client.? 172.21.15.6:0/1946545398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:44 smithi006 bash[33918]: cluster 2023-08-27T23:09:42.631288+0000 mgr.y (mgr.14901) 2135 : cluster [DBG] pgmap v2142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:44 smithi006 bash[33918]: audit 2023-08-27T23:09:44.362201+0000 mon.c (mon.2) 848 : audit [DBG] from='client.? 172.21.15.6:0/1946545398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:45.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:45 smithi130 bash[51302]: debug 2023-08-27T23:09:45.168+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:45.172658+0000) 2023-08-27T23:09:46.520 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:46 smithi130 bash[51302]: debug 2023-08-27T23:09:46.168+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:46.172973+0000) 2023-08-27T23:09:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:46 smithi130 bash[27579]: cluster 2023-08-27T23:09:44.632389+0000 mgr.y (mgr.14901) 2136 : cluster [DBG] pgmap v2143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:46 smithi006 bash[33918]: cluster 2023-08-27T23:09:44.632389+0000 mgr.y (mgr.14901) 2136 : cluster [DBG] pgmap v2143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:46 smithi006 bash[27292]: cluster 2023-08-27T23:09:44.632389+0000 mgr.y (mgr.14901) 2136 : cluster [DBG] pgmap v2143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:47.529 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:47 smithi130 bash[51302]: debug 2023-08-27T23:09:47.172+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:47.173328+0000) 2023-08-27T23:09:47.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:47 smithi130 bash[27579]: audit 2023-08-27T23:09:46.783529+0000 mon.a (mon.0) 1960 : audit [DBG] from='client.? 172.21.15.6:0/468636802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:47.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:47 smithi006 bash[27292]: audit 2023-08-27T23:09:46.783529+0000 mon.a (mon.0) 1960 : audit [DBG] from='client.? 172.21.15.6:0/468636802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:47.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:47 smithi006 bash[33918]: audit 2023-08-27T23:09:46.783529+0000 mon.a (mon.0) 1960 : audit [DBG] from='client.? 172.21.15.6:0/468636802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:48.532 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:48 smithi130 bash[51302]: debug 2023-08-27T23:09:48.172+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:48.173644+0000) 2023-08-27T23:09:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:48 smithi130 bash[27579]: cluster 2023-08-27T23:09:46.633133+0000 mgr.y (mgr.14901) 2137 : cluster [DBG] pgmap v2144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:48.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:48 smithi006 bash[33918]: cluster 2023-08-27T23:09:46.633133+0000 mgr.y (mgr.14901) 2137 : cluster [DBG] pgmap v2144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:48.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:48 smithi006 bash[27292]: cluster 2023-08-27T23:09:46.633133+0000 mgr.y (mgr.14901) 2137 : cluster [DBG] pgmap v2144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:49.539 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:49 smithi130 bash[51302]: debug 2023-08-27T23:09:49.168+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:49.173935+0000) 2023-08-27T23:09:49.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:49 smithi130 bash[27579]: audit 2023-08-27T23:09:49.193986+0000 mon.c (mon.2) 849 : audit [DBG] from='client.? 172.21.15.6:0/2996642824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:49.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:49 smithi006 bash[27292]: audit 2023-08-27T23:09:49.193986+0000 mon.c (mon.2) 849 : audit [DBG] from='client.? 172.21.15.6:0/2996642824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:49.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:49 smithi006 bash[33918]: audit 2023-08-27T23:09:49.193986+0000 mon.c (mon.2) 849 : audit [DBG] from='client.? 172.21.15.6:0/2996642824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:50.543 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:50 smithi130 bash[51302]: debug 2023-08-27T23:09:50.168+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:50.174196+0000) 2023-08-27T23:09:50.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:50 smithi006 bash[27292]: cluster 2023-08-27T23:09:48.633955+0000 mgr.y (mgr.14901) 2138 : cluster [DBG] pgmap v2145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:50 smithi006 bash[33918]: cluster 2023-08-27T23:09:48.633955+0000 mgr.y (mgr.14901) 2138 : cluster [DBG] pgmap v2145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:09:50.391Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:09:50.393Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:50 smithi130 bash[27579]: cluster 2023-08-27T23:09:48.633955+0000 mgr.y (mgr.14901) 2138 : cluster [DBG] pgmap v2145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:09:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:09:50] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:09:51.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:51 smithi130 bash[51302]: debug 2023-08-27T23:09:51.168+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:51.174478+0000) 2023-08-27T23:09:52.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:51 smithi006 bash[56298]: level=error ts=2023-08-27T23:09:51.956Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:52.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:51 smithi006 bash[56298]: level=warn ts=2023-08-27T23:09:51.958Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:52.547 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:52 smithi130 bash[51302]: debug 2023-08-27T23:09:52.168+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:52.174775+0000) 2023-08-27T23:09:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:52 smithi130 bash[27579]: cluster 2023-08-27T23:09:50.635083+0000 mgr.y (mgr.14901) 2139 : cluster [DBG] pgmap v2146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:52 smithi130 bash[27579]: audit 2023-08-27T23:09:51.605510+0000 mon.a (mon.0) 1961 : audit [DBG] from='client.? 172.21.15.6:0/2088612404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:52 smithi006 bash[33918]: cluster 2023-08-27T23:09:50.635083+0000 mgr.y (mgr.14901) 2139 : cluster [DBG] pgmap v2146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:52 smithi006 bash[33918]: audit 2023-08-27T23:09:51.605510+0000 mon.a (mon.0) 1961 : audit [DBG] from='client.? 172.21.15.6:0/2088612404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:52 smithi006 bash[27292]: cluster 2023-08-27T23:09:50.635083+0000 mgr.y (mgr.14901) 2139 : cluster [DBG] pgmap v2146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:52 smithi006 bash[27292]: audit 2023-08-27T23:09:51.605510+0000 mon.a (mon.0) 1961 : audit [DBG] from='client.? 172.21.15.6:0/2088612404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:53.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:53 smithi130 bash[51302]: debug 2023-08-27T23:09:53.172+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:53.175075+0000) 2023-08-27T23:09:54.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:09:53.855Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:54.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:09:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:09:53.857Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:09:54.551 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:54 smithi130 bash[51302]: debug 2023-08-27T23:09:54.172+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:54.175390+0000) 2023-08-27T23:09:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:54 smithi130 bash[27579]: cluster 2023-08-27T23:09:52.635727+0000 mgr.y (mgr.14901) 2140 : cluster [DBG] pgmap v2147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:54 smithi130 bash[27579]: audit 2023-08-27T23:09:54.022398+0000 mon.a (mon.0) 1962 : audit [DBG] from='client.? 172.21.15.6:0/1487749666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:54 smithi006 bash[27292]: cluster 2023-08-27T23:09:52.635727+0000 mgr.y (mgr.14901) 2140 : cluster [DBG] pgmap v2147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:54 smithi006 bash[27292]: audit 2023-08-27T23:09:54.022398+0000 mon.a (mon.0) 1962 : audit [DBG] from='client.? 172.21.15.6:0/1487749666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:54 smithi006 bash[33918]: cluster 2023-08-27T23:09:52.635727+0000 mgr.y (mgr.14901) 2140 : cluster [DBG] pgmap v2147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:54 smithi006 bash[33918]: audit 2023-08-27T23:09:54.022398+0000 mon.a (mon.0) 1962 : audit [DBG] from='client.? 172.21.15.6:0/1487749666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:55.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:55 smithi130 bash[51302]: debug 2023-08-27T23:09:55.172+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:55.175714+0000) 2023-08-27T23:09:56.556 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:56 smithi130 bash[51302]: debug 2023-08-27T23:09:56.172+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:56.175979+0000) 2023-08-27T23:09:56.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:56 smithi130 bash[27579]: cluster 2023-08-27T23:09:54.636880+0000 mgr.y (mgr.14901) 2141 : cluster [DBG] pgmap v2148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:56.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:56 smithi130 bash[27579]: audit 2023-08-27T23:09:56.437257+0000 mon.c (mon.2) 850 : audit [DBG] from='client.? 172.21.15.6:0/3611381639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:56.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:56 smithi006 bash[27292]: cluster 2023-08-27T23:09:54.636880+0000 mgr.y (mgr.14901) 2141 : cluster [DBG] pgmap v2148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:56.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:56 smithi006 bash[27292]: audit 2023-08-27T23:09:56.437257+0000 mon.c (mon.2) 850 : audit [DBG] from='client.? 172.21.15.6:0/3611381639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:56 smithi006 bash[33918]: cluster 2023-08-27T23:09:54.636880+0000 mgr.y (mgr.14901) 2141 : cluster [DBG] pgmap v2148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:56 smithi006 bash[33918]: audit 2023-08-27T23:09:56.437257+0000 mon.c (mon.2) 850 : audit [DBG] from='client.? 172.21.15.6:0/3611381639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:57.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:57 smithi130 bash[51302]: debug 2023-08-27T23:09:57.172+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:57.176276+0000) 2023-08-27T23:09:58.560 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:58 smithi130 bash[51302]: debug 2023-08-27T23:09:58.172+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:58.176624+0000) 2023-08-27T23:09:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:58 smithi130 bash[27579]: cluster 2023-08-27T23:09:56.637508+0000 mgr.y (mgr.14901) 2142 : cluster [DBG] pgmap v2149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:58.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:58 smithi006 bash[27292]: cluster 2023-08-27T23:09:56.637508+0000 mgr.y (mgr.14901) 2142 : cluster [DBG] pgmap v2149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:58.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:58 smithi006 bash[33918]: cluster 2023-08-27T23:09:56.637508+0000 mgr.y (mgr.14901) 2142 : cluster [DBG] pgmap v2149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:09:59.571 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:09:59 smithi130 bash[51302]: debug 2023-08-27T23:09:59.172+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:59.176855+0000) 2023-08-27T23:09:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:09:59 smithi130 bash[27579]: audit 2023-08-27T23:09:58.846961+0000 mon.c (mon.2) 851 : audit [DBG] from='client.? 172.21.15.6:0/1976490663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:59.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:09:59 smithi006 bash[27292]: audit 2023-08-27T23:09:58.846961+0000 mon.c (mon.2) 851 : audit [DBG] from='client.? 172.21.15.6:0/1976490663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:09:59 smithi006 bash[33918]: audit 2023-08-27T23:09:58.846961+0000 mon.c (mon.2) 851 : audit [DBG] from='client.? 172.21.15.6:0/1976490663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:00.573 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:00 smithi130 bash[51302]: debug 2023-08-27T23:10:00.176+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:00.177100+0000) 2023-08-27T23:10:00.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:00 smithi006 bash[33918]: cluster 2023-08-27T23:09:58.638283+0000 mgr.y (mgr.14901) 2143 : cluster [DBG] pgmap v2150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:00.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:00 smithi006 bash[33918]: cluster 2023-08-27T23:10:00.000142+0000 mon.a (mon.0) 1963 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:10:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:00 smithi006 bash[27292]: cluster 2023-08-27T23:09:58.638283+0000 mgr.y (mgr.14901) 2143 : cluster [DBG] pgmap v2150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:00 smithi006 bash[27292]: cluster 2023-08-27T23:10:00.000142+0000 mon.a (mon.0) 1963 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:10:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:10:00.391Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:10:00.393Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:00 smithi130 bash[27579]: cluster 2023-08-27T23:09:58.638283+0000 mgr.y (mgr.14901) 2143 : cluster [DBG] pgmap v2150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:00 smithi130 bash[27579]: cluster 2023-08-27T23:10:00.000142+0000 mon.a (mon.0) 1963 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:10:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:10:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:10:00] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:10:01.580 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:01 smithi130 bash[51302]: debug 2023-08-27T23:10:01.172+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:01.177410+0000) 2023-08-27T23:10:01.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:01 smithi130 bash[27579]: cluster 2023-08-27T23:10:00.639326+0000 mgr.y (mgr.14901) 2144 : cluster [DBG] pgmap v2151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:01.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:01 smithi130 bash[27579]: audit 2023-08-27T23:10:01.258180+0000 mon.c (mon.2) 852 : audit [DBG] from='client.? 172.21.15.6:0/1498641584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:01.959 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:01 smithi006 bash[27292]: cluster 2023-08-27T23:10:00.639326+0000 mgr.y (mgr.14901) 2144 : cluster [DBG] pgmap v2151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:01.959 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:01 smithi006 bash[27292]: audit 2023-08-27T23:10:01.258180+0000 mon.c (mon.2) 852 : audit [DBG] from='client.? 172.21.15.6:0/1498641584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:01.959 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:01 smithi006 bash[33918]: cluster 2023-08-27T23:10:00.639326+0000 mgr.y (mgr.14901) 2144 : cluster [DBG] pgmap v2151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:01.960 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:01 smithi006 bash[33918]: audit 2023-08-27T23:10:01.258180+0000 mon.c (mon.2) 852 : audit [DBG] from='client.? 172.21.15.6:0/1498641584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:02.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:01 smithi006 bash[56298]: level=error ts=2023-08-27T23:10:01.957Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:02.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:01 smithi006 bash[56298]: level=warn ts=2023-08-27T23:10:01.959Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:02.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:02 smithi130 bash[51302]: debug 2023-08-27T23:10:02.172+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:02.177745+0000) 2023-08-27T23:10:03.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:03 smithi130 bash[51302]: debug 2023-08-27T23:10:03.176+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:03.178018+0000) 2023-08-27T23:10:03.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:03 smithi006 bash[33918]: cluster 2023-08-27T23:10:02.639993+0000 mgr.y (mgr.14901) 2145 : cluster [DBG] pgmap v2152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:03.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:03 smithi006 bash[33918]: audit 2023-08-27T23:10:03.674780+0000 mon.a (mon.0) 1964 : audit [DBG] from='client.? 172.21.15.6:0/110412839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:03.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:03 smithi006 bash[27292]: cluster 2023-08-27T23:10:02.639993+0000 mgr.y (mgr.14901) 2145 : cluster [DBG] pgmap v2152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:03.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:03 smithi006 bash[27292]: audit 2023-08-27T23:10:03.674780+0000 mon.a (mon.0) 1964 : audit [DBG] from='client.? 172.21.15.6:0/110412839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:03.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:10:03.856Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:03.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:10:03.858Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:04.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:03 smithi130 bash[27579]: cluster 2023-08-27T23:10:02.639993+0000 mgr.y (mgr.14901) 2145 : cluster [DBG] pgmap v2152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:04.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:03 smithi130 bash[27579]: audit 2023-08-27T23:10:03.674780+0000 mon.a (mon.0) 1964 : audit [DBG] from='client.? 172.21.15.6:0/110412839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:04.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:04 smithi130 bash[51302]: debug 2023-08-27T23:10:04.176+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:04.178296+0000) 2023-08-27T23:10:05.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:05 smithi130 bash[51302]: debug 2023-08-27T23:10:05.172+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:05.178553+0000) 2023-08-27T23:10:06.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:06 smithi130 bash[51302]: debug 2023-08-27T23:10:06.172+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:06.178751+0000) 2023-08-27T23:10:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:06 smithi130 bash[27579]: cluster 2023-08-27T23:10:04.641138+0000 mgr.y (mgr.14901) 2146 : cluster [DBG] pgmap v2153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:06 smithi130 bash[27579]: audit 2023-08-27T23:10:05.920952+0000 mon.b (mon.1) 326 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:10:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:06 smithi130 bash[27579]: audit 2023-08-27T23:10:06.101549+0000 mon.c (mon.2) 853 : audit [DBG] from='client.? 172.21.15.6:0/2722287214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:06 smithi130 bash[27579]: audit 2023-08-27T23:10:06.186967+0000 mon.a (mon.0) 1965 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:10:06.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:06 smithi130 bash[27579]: audit 2023-08-27T23:10:06.196936+0000 mon.a (mon.0) 1966 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:10:06.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:06 smithi006 bash[33918]: cluster 2023-08-27T23:10:04.641138+0000 mgr.y (mgr.14901) 2146 : cluster [DBG] pgmap v2153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:06 smithi006 bash[33918]: audit 2023-08-27T23:10:05.920952+0000 mon.b (mon.1) 326 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:10:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:06 smithi006 bash[33918]: audit 2023-08-27T23:10:06.101549+0000 mon.c (mon.2) 853 : audit [DBG] from='client.? 172.21.15.6:0/2722287214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:06 smithi006 bash[33918]: audit 2023-08-27T23:10:06.186967+0000 mon.a (mon.0) 1965 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:10:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:06 smithi006 bash[33918]: audit 2023-08-27T23:10:06.196936+0000 mon.a (mon.0) 1966 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:10:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:06 smithi006 bash[27292]: cluster 2023-08-27T23:10:04.641138+0000 mgr.y (mgr.14901) 2146 : cluster [DBG] pgmap v2153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:06 smithi006 bash[27292]: audit 2023-08-27T23:10:05.920952+0000 mon.b (mon.1) 326 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:10:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:06 smithi006 bash[27292]: audit 2023-08-27T23:10:06.101549+0000 mon.c (mon.2) 853 : audit [DBG] from='client.? 172.21.15.6:0/2722287214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:06 smithi006 bash[27292]: audit 2023-08-27T23:10:06.186967+0000 mon.a (mon.0) 1965 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:10:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:06 smithi006 bash[27292]: audit 2023-08-27T23:10:06.196936+0000 mon.a (mon.0) 1966 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:10:07.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:07 smithi130 bash[51302]: debug 2023-08-27T23:10:07.176+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:07.179049+0000) 2023-08-27T23:10:08.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:08 smithi130 bash[27579]: cluster 2023-08-27T23:10:06.641709+0000 mgr.y (mgr.14901) 2147 : cluster [DBG] pgmap v2154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:08.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:08 smithi130 bash[51302]: debug 2023-08-27T23:10:08.176+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:08.179327+0000) 2023-08-27T23:10:08.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:08 smithi006 bash[27292]: cluster 2023-08-27T23:10:06.641709+0000 mgr.y (mgr.14901) 2147 : cluster [DBG] pgmap v2154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:08 smithi006 bash[33918]: cluster 2023-08-27T23:10:06.641709+0000 mgr.y (mgr.14901) 2147 : cluster [DBG] pgmap v2154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:09.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:09 smithi130 bash[27579]: audit 2023-08-27T23:10:08.521614+0000 mon.a (mon.0) 1967 : audit [DBG] from='client.? 172.21.15.6:0/3964728301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:09.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:09 smithi130 bash[51302]: debug 2023-08-27T23:10:09.176+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:09.179616+0000) 2023-08-27T23:10:09.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:09 smithi006 bash[33918]: audit 2023-08-27T23:10:08.521614+0000 mon.a (mon.0) 1967 : audit [DBG] from='client.? 172.21.15.6:0/3964728301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:09 smithi006 bash[27292]: audit 2023-08-27T23:10:08.521614+0000 mon.a (mon.0) 1967 : audit [DBG] from='client.? 172.21.15.6:0/3964728301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:10.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:10 smithi130 bash[27579]: cluster 2023-08-27T23:10:08.642348+0000 mgr.y (mgr.14901) 2148 : cluster [DBG] pgmap v2155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:10.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:10 smithi130 bash[51302]: debug 2023-08-27T23:10:10.176+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:10.179900+0000) 2023-08-27T23:10:10.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:10 smithi006 bash[27292]: cluster 2023-08-27T23:10:08.642348+0000 mgr.y (mgr.14901) 2148 : cluster [DBG] pgmap v2155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:10:10.391Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:10:10.393Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:10 smithi006 bash[33918]: cluster 2023-08-27T23:10:08.642348+0000 mgr.y (mgr.14901) 2148 : cluster [DBG] pgmap v2155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:10:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:10:10] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:10:11.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:11 smithi130 bash[27579]: audit 2023-08-27T23:10:10.946337+0000 mon.a (mon.0) 1968 : audit [DBG] from='client.? 172.21.15.6:0/3773606327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:11.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:11 smithi130 bash[51302]: debug 2023-08-27T23:10:11.176+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:11.180170+0000) 2023-08-27T23:10:11.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:11 smithi006 bash[27292]: audit 2023-08-27T23:10:10.946337+0000 mon.a (mon.0) 1968 : audit [DBG] from='client.? 172.21.15.6:0/3773606327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:11.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:11 smithi006 bash[33918]: audit 2023-08-27T23:10:10.946337+0000 mon.a (mon.0) 1968 : audit [DBG] from='client.? 172.21.15.6:0/3773606327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:12.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:11 smithi006 bash[56298]: level=error ts=2023-08-27T23:10:11.958Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:12.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:11 smithi006 bash[56298]: level=warn ts=2023-08-27T23:10:11.960Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:12.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:12 smithi130 bash[51302]: debug 2023-08-27T23:10:12.175+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:12.180446+0000) 2023-08-27T23:10:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:12 smithi130 bash[27579]: cluster 2023-08-27T23:10:10.643361+0000 mgr.y (mgr.14901) 2149 : cluster [DBG] pgmap v2156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:12 smithi130 bash[27579]: audit 2023-08-27T23:10:12.081829+0000 mon.a (mon.0) 1969 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:10:12.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:12 smithi006 bash[27292]: cluster 2023-08-27T23:10:10.643361+0000 mgr.y (mgr.14901) 2149 : cluster [DBG] pgmap v2156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:12.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:12 smithi006 bash[27292]: audit 2023-08-27T23:10:12.081829+0000 mon.a (mon.0) 1969 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:10:12.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:12 smithi006 bash[33918]: cluster 2023-08-27T23:10:10.643361+0000 mgr.y (mgr.14901) 2149 : cluster [DBG] pgmap v2156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:12.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:12 smithi006 bash[33918]: audit 2023-08-27T23:10:12.081829+0000 mon.a (mon.0) 1969 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:10:13.504 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:13 smithi130 bash[51302]: debug 2023-08-27T23:10:13.175+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:13.180749+0000) 2023-08-27T23:10:13.858 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:13 smithi006 bash[33918]: audit 2023-08-27T23:10:12.492431+0000 mon.a (mon.0) 1970 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:10:13.858 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:13 smithi006 bash[33918]: audit 2023-08-27T23:10:12.766843+0000 mon.b (mon.1) 327 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:10:13.859 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:13 smithi006 bash[33918]: audit 2023-08-27T23:10:12.769114+0000 mon.b (mon.1) 328 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:10:13.859 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:13 smithi006 bash[33918]: audit 2023-08-27T23:10:12.780124+0000 mon.a (mon.0) 1971 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:10:13.859 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:13 smithi006 bash[33918]: audit 2023-08-27T23:10:13.366865+0000 mon.c (mon.2) 854 : audit [DBG] from='client.? 172.21.15.6:0/2230205587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:13.859 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:13 smithi006 bash[27292]: audit 2023-08-27T23:10:12.492431+0000 mon.a (mon.0) 1970 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:10:13.859 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:13 smithi006 bash[27292]: audit 2023-08-27T23:10:12.766843+0000 mon.b (mon.1) 327 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:10:13.859 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:13 smithi006 bash[27292]: audit 2023-08-27T23:10:12.769114+0000 mon.b (mon.1) 328 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:10:13.859 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:13 smithi006 bash[27292]: audit 2023-08-27T23:10:12.780124+0000 mon.a (mon.0) 1971 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:10:13.859 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:13 smithi006 bash[27292]: audit 2023-08-27T23:10:13.366865+0000 mon.c (mon.2) 854 : audit [DBG] from='client.? 172.21.15.6:0/2230205587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:13.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:13 smithi130 bash[27579]: audit 2023-08-27T23:10:12.492431+0000 mon.a (mon.0) 1970 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:10:13.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:13 smithi130 bash[27579]: audit 2023-08-27T23:10:12.766843+0000 mon.b (mon.1) 327 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:10:13.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:13 smithi130 bash[27579]: audit 2023-08-27T23:10:12.769114+0000 mon.b (mon.1) 328 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:10:13.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:13 smithi130 bash[27579]: audit 2023-08-27T23:10:12.780124+0000 mon.a (mon.0) 1971 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:10:13.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:13 smithi130 bash[27579]: audit 2023-08-27T23:10:13.366865+0000 mon.c (mon.2) 854 : audit [DBG] from='client.? 172.21.15.6:0/2230205587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:10:13.856Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:10:13.858Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:14.509 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:14 smithi130 bash[51302]: debug 2023-08-27T23:10:14.175+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:14.181051+0000) 2023-08-27T23:10:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:14 smithi130 bash[27579]: cluster 2023-08-27T23:10:12.643987+0000 mgr.y (mgr.14901) 2150 : cluster [DBG] pgmap v2157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:14.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:14 smithi006 bash[33918]: cluster 2023-08-27T23:10:12.643987+0000 mgr.y (mgr.14901) 2150 : cluster [DBG] pgmap v2157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:14 smithi006 bash[27292]: cluster 2023-08-27T23:10:12.643987+0000 mgr.y (mgr.14901) 2150 : cluster [DBG] pgmap v2157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:15.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:15 smithi130 bash[51302]: debug 2023-08-27T23:10:15.175+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:15.181409+0000) 2023-08-27T23:10:16.512 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:16 smithi130 bash[51302]: debug 2023-08-27T23:10:16.179+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:16.181751+0000) 2023-08-27T23:10:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:16 smithi130 bash[27579]: cluster 2023-08-27T23:10:14.645093+0000 mgr.y (mgr.14901) 2151 : cluster [DBG] pgmap v2158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:16 smithi130 bash[27579]: audit 2023-08-27T23:10:15.780443+0000 mon.c (mon.2) 855 : audit [DBG] from='client.? 172.21.15.6:0/2267071021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:16.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:16 smithi006 bash[33918]: cluster 2023-08-27T23:10:14.645093+0000 mgr.y (mgr.14901) 2151 : cluster [DBG] pgmap v2158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:16 smithi006 bash[33918]: audit 2023-08-27T23:10:15.780443+0000 mon.c (mon.2) 855 : audit [DBG] from='client.? 172.21.15.6:0/2267071021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:16 smithi006 bash[27292]: cluster 2023-08-27T23:10:14.645093+0000 mgr.y (mgr.14901) 2151 : cluster [DBG] pgmap v2158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:16 smithi006 bash[27292]: audit 2023-08-27T23:10:15.780443+0000 mon.c (mon.2) 855 : audit [DBG] from='client.? 172.21.15.6:0/2267071021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:17.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:17 smithi130 bash[51302]: debug 2023-08-27T23:10:17.179+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:17.182098+0000) 2023-08-27T23:10:18.519 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:18 smithi130 bash[51302]: debug 2023-08-27T23:10:18.179+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:18.182470+0000) 2023-08-27T23:10:18.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:18 smithi130 bash[27579]: cluster 2023-08-27T23:10:16.645700+0000 mgr.y (mgr.14901) 2152 : cluster [DBG] pgmap v2159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:18 smithi130 bash[27579]: audit 2023-08-27T23:10:18.194271+0000 mon.c (mon.2) 856 : audit [DBG] from='client.? 172.21.15.6:0/3820701757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:18.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:18 smithi006 bash[33918]: cluster 2023-08-27T23:10:16.645700+0000 mgr.y (mgr.14901) 2152 : cluster [DBG] pgmap v2159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:18 smithi006 bash[33918]: audit 2023-08-27T23:10:18.194271+0000 mon.c (mon.2) 856 : audit [DBG] from='client.? 172.21.15.6:0/3820701757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:18 smithi006 bash[27292]: cluster 2023-08-27T23:10:16.645700+0000 mgr.y (mgr.14901) 2152 : cluster [DBG] pgmap v2159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:18 smithi006 bash[27292]: audit 2023-08-27T23:10:18.194271+0000 mon.c (mon.2) 856 : audit [DBG] from='client.? 172.21.15.6:0/3820701757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:19.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:19 smithi130 bash[51302]: debug 2023-08-27T23:10:19.179+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:19.182793+0000) 2023-08-27T23:10:20.522 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:20 smithi130 bash[51302]: debug 2023-08-27T23:10:20.179+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:20.183108+0000) 2023-08-27T23:10:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:20 smithi006 bash[27292]: cluster 2023-08-27T23:10:18.646365+0000 mgr.y (mgr.14901) 2153 : cluster [DBG] pgmap v2160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:20 smithi006 bash[33918]: cluster 2023-08-27T23:10:18.646365+0000 mgr.y (mgr.14901) 2153 : cluster [DBG] pgmap v2160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:10:20.392Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:10:20.394Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:20.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:20 smithi130 bash[27579]: cluster 2023-08-27T23:10:18.646365+0000 mgr.y (mgr.14901) 2153 : cluster [DBG] pgmap v2160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:21.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:10:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:10:20] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:10:21.532 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:21 smithi130 bash[51302]: debug 2023-08-27T23:10:21.179+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:21.183386+0000) 2023-08-27T23:10:21.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:21 smithi130 bash[27579]: audit 2023-08-27T23:10:20.618294+0000 mon.c (mon.2) 857 : audit [DBG] from='client.? 172.21.15.6:0/1094241006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:21.960 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:21 smithi006 bash[33918]: audit 2023-08-27T23:10:20.618294+0000 mon.c (mon.2) 857 : audit [DBG] from='client.? 172.21.15.6:0/1094241006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:21.960 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:21 smithi006 bash[27292]: audit 2023-08-27T23:10:20.618294+0000 mon.c (mon.2) 857 : audit [DBG] from='client.? 172.21.15.6:0/1094241006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:22.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:21 smithi006 bash[56298]: level=error ts=2023-08-27T23:10:21.958Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:22.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:21 smithi006 bash[56298]: level=warn ts=2023-08-27T23:10:21.960Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:22.535 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:22 smithi130 bash[51302]: debug 2023-08-27T23:10:22.179+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:22.183727+0000) 2023-08-27T23:10:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:22 smithi130 bash[27579]: cluster 2023-08-27T23:10:20.647460+0000 mgr.y (mgr.14901) 2154 : cluster [DBG] pgmap v2161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:22.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:22 smithi006 bash[27292]: cluster 2023-08-27T23:10:20.647460+0000 mgr.y (mgr.14901) 2154 : cluster [DBG] pgmap v2161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:22.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:22 smithi006 bash[33918]: cluster 2023-08-27T23:10:20.647460+0000 mgr.y (mgr.14901) 2154 : cluster [DBG] pgmap v2161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:23.546 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:23 smithi130 bash[51302]: debug 2023-08-27T23:10:23.179+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:23.184045+0000) 2023-08-27T23:10:23.859 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:23 smithi006 bash[27292]: audit 2023-08-27T23:10:23.034057+0000 mon.a (mon.0) 1972 : audit [DBG] from='client.? 172.21.15.6:0/3854347698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:23.860 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:23 smithi006 bash[33918]: audit 2023-08-27T23:10:23.034057+0000 mon.a (mon.0) 1972 : audit [DBG] from='client.? 172.21.15.6:0/3854347698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:23.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:23 smithi130 bash[27579]: audit 2023-08-27T23:10:23.034057+0000 mon.a (mon.0) 1972 : audit [DBG] from='client.? 172.21.15.6:0/3854347698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:24.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:10:23.857Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:10:23.859Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:24.548 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:24 smithi130 bash[51302]: debug 2023-08-27T23:10:24.179+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:24.184371+0000) 2023-08-27T23:10:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:24 smithi130 bash[27579]: cluster 2023-08-27T23:10:22.648086+0000 mgr.y (mgr.14901) 2155 : cluster [DBG] pgmap v2162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:24.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:24 smithi006 bash[27292]: cluster 2023-08-27T23:10:22.648086+0000 mgr.y (mgr.14901) 2155 : cluster [DBG] pgmap v2162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:24.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:24 smithi006 bash[33918]: cluster 2023-08-27T23:10:22.648086+0000 mgr.y (mgr.14901) 2155 : cluster [DBG] pgmap v2162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:25.551 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:25 smithi130 bash[51302]: debug 2023-08-27T23:10:25.179+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:25.184716+0000) 2023-08-27T23:10:25.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:25 smithi130 bash[27579]: audit 2023-08-27T23:10:25.439592+0000 mon.a (mon.0) 1973 : audit [DBG] from='client.? 172.21.15.6:0/2557578437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:25.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:25 smithi006 bash[27292]: audit 2023-08-27T23:10:25.439592+0000 mon.a (mon.0) 1973 : audit [DBG] from='client.? 172.21.15.6:0/2557578437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:25.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:25 smithi006 bash[33918]: audit 2023-08-27T23:10:25.439592+0000 mon.a (mon.0) 1973 : audit [DBG] from='client.? 172.21.15.6:0/2557578437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:26.561 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:26 smithi130 bash[51302]: debug 2023-08-27T23:10:26.179+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:26.185028+0000) 2023-08-27T23:10:26.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:26 smithi130 bash[27579]: cluster 2023-08-27T23:10:24.649212+0000 mgr.y (mgr.14901) 2156 : cluster [DBG] pgmap v2163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:26.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:26 smithi006 bash[27292]: cluster 2023-08-27T23:10:24.649212+0000 mgr.y (mgr.14901) 2156 : cluster [DBG] pgmap v2163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:26 smithi006 bash[33918]: cluster 2023-08-27T23:10:24.649212+0000 mgr.y (mgr.14901) 2156 : cluster [DBG] pgmap v2163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:27.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:27 smithi130 bash[51302]: debug 2023-08-27T23:10:27.179+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:27.185340+0000) 2023-08-27T23:10:28.566 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:28 smithi130 bash[51302]: debug 2023-08-27T23:10:28.179+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:28.185699+0000) 2023-08-27T23:10:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:28 smithi130 bash[27579]: cluster 2023-08-27T23:10:26.649871+0000 mgr.y (mgr.14901) 2157 : cluster [DBG] pgmap v2164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:28 smithi130 bash[27579]: audit 2023-08-27T23:10:27.858778+0000 mon.c (mon.2) 858 : audit [DBG] from='client.? 172.21.15.6:0/2066563569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:28.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:28 smithi006 bash[27292]: cluster 2023-08-27T23:10:26.649871+0000 mgr.y (mgr.14901) 2157 : cluster [DBG] pgmap v2164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:28.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:28 smithi006 bash[27292]: audit 2023-08-27T23:10:27.858778+0000 mon.c (mon.2) 858 : audit [DBG] from='client.? 172.21.15.6:0/2066563569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:28 smithi006 bash[33918]: cluster 2023-08-27T23:10:26.649871+0000 mgr.y (mgr.14901) 2157 : cluster [DBG] pgmap v2164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:28.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:28 smithi006 bash[33918]: audit 2023-08-27T23:10:27.858778+0000 mon.c (mon.2) 858 : audit [DBG] from='client.? 172.21.15.6:0/2066563569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:29.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:29 smithi130 bash[51302]: debug 2023-08-27T23:10:29.179+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:29.186006+0000) 2023-08-27T23:10:30.573 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:30 smithi130 bash[51302]: debug 2023-08-27T23:10:30.179+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:30.186353+0000) 2023-08-27T23:10:30.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:10:30.392Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:30.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:10:30.395Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:30 smithi006 bash[27292]: cluster 2023-08-27T23:10:28.650515+0000 mgr.y (mgr.14901) 2158 : cluster [DBG] pgmap v2165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:30 smithi006 bash[27292]: audit 2023-08-27T23:10:30.280603+0000 mon.a (mon.0) 1974 : audit [DBG] from='client.? 172.21.15.6:0/4137811392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:30 smithi006 bash[33918]: cluster 2023-08-27T23:10:28.650515+0000 mgr.y (mgr.14901) 2158 : cluster [DBG] pgmap v2165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:30 smithi006 bash[33918]: audit 2023-08-27T23:10:30.280603+0000 mon.a (mon.0) 1974 : audit [DBG] from='client.? 172.21.15.6:0/4137811392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:30.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:30 smithi130 bash[27579]: cluster 2023-08-27T23:10:28.650515+0000 mgr.y (mgr.14901) 2158 : cluster [DBG] pgmap v2165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:30 smithi130 bash[27579]: audit 2023-08-27T23:10:30.280603+0000 mon.a (mon.0) 1974 : audit [DBG] from='client.? 172.21.15.6:0/4137811392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:10:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:10:30] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:10:31.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:31 smithi130 bash[51302]: debug 2023-08-27T23:10:31.183+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:31.186734+0000) 2023-08-27T23:10:32.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:31 smithi006 bash[56298]: level=error ts=2023-08-27T23:10:31.959Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:32.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:31 smithi006 bash[56298]: level=warn ts=2023-08-27T23:10:31.960Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:32.580 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:32 smithi130 bash[51302]: debug 2023-08-27T23:10:32.183+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:32.187017+0000) 2023-08-27T23:10:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:32 smithi130 bash[27579]: cluster 2023-08-27T23:10:30.651602+0000 mgr.y (mgr.14901) 2159 : cluster [DBG] pgmap v2166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:32 smithi006 bash[33918]: cluster 2023-08-27T23:10:30.651602+0000 mgr.y (mgr.14901) 2159 : cluster [DBG] pgmap v2166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:32.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:32 smithi006 bash[27292]: cluster 2023-08-27T23:10:30.651602+0000 mgr.y (mgr.14901) 2159 : cluster [DBG] pgmap v2166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:33.587 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:33 smithi130 bash[51302]: debug 2023-08-27T23:10:33.183+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:33.187293+0000) 2023-08-27T23:10:33.861 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:33 smithi006 bash[33918]: cluster 2023-08-27T23:10:32.652034+0000 mgr.y (mgr.14901) 2160 : cluster [DBG] pgmap v2167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:33.861 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:33 smithi006 bash[33918]: audit 2023-08-27T23:10:32.693181+0000 mon.a (mon.0) 1975 : audit [DBG] from='client.? 172.21.15.6:0/3688498328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:33.861 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:33 smithi006 bash[27292]: cluster 2023-08-27T23:10:32.652034+0000 mgr.y (mgr.14901) 2160 : cluster [DBG] pgmap v2167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:33.861 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:33 smithi006 bash[27292]: audit 2023-08-27T23:10:32.693181+0000 mon.a (mon.0) 1975 : audit [DBG] from='client.? 172.21.15.6:0/3688498328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:33 smithi130 bash[27579]: cluster 2023-08-27T23:10:32.652034+0000 mgr.y (mgr.14901) 2160 : cluster [DBG] pgmap v2167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:33 smithi130 bash[27579]: audit 2023-08-27T23:10:32.693181+0000 mon.a (mon.0) 1975 : audit [DBG] from='client.? 172.21.15.6:0/3688498328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:34.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:10:33.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:34.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:10:33.861Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:34.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:34 smithi130 bash[51302]: debug 2023-08-27T23:10:34.183+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:34.187558+0000) 2023-08-27T23:10:35.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:35 smithi130 bash[27579]: audit 2023-08-27T23:10:35.105077+0000 mon.a (mon.0) 1976 : audit [DBG] from='client.? 172.21.15.6:0/3255120619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:35.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:35 smithi130 bash[51302]: debug 2023-08-27T23:10:35.187+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:35.187881+0000) 2023-08-27T23:10:35.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:35 smithi006 bash[27292]: audit 2023-08-27T23:10:35.105077+0000 mon.a (mon.0) 1976 : audit [DBG] from='client.? 172.21.15.6:0/3255120619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:35.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:35 smithi006 bash[33918]: audit 2023-08-27T23:10:35.105077+0000 mon.a (mon.0) 1976 : audit [DBG] from='client.? 172.21.15.6:0/3255120619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:36.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:36 smithi130 bash[27579]: cluster 2023-08-27T23:10:34.653150+0000 mgr.y (mgr.14901) 2161 : cluster [DBG] pgmap v2168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:36.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:36 smithi130 bash[51302]: debug 2023-08-27T23:10:36.187+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:36.188259+0000) 2023-08-27T23:10:36.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:36 smithi006 bash[33918]: cluster 2023-08-27T23:10:34.653150+0000 mgr.y (mgr.14901) 2161 : cluster [DBG] pgmap v2168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:36 smithi006 bash[27292]: cluster 2023-08-27T23:10:34.653150+0000 mgr.y (mgr.14901) 2161 : cluster [DBG] pgmap v2168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:37.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:37 smithi130 bash[51302]: debug 2023-08-27T23:10:37.183+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:37.188625+0000) 2023-08-27T23:10:38.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:38 smithi130 bash[27579]: cluster 2023-08-27T23:10:36.653771+0000 mgr.y (mgr.14901) 2162 : cluster [DBG] pgmap v2169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:38 smithi130 bash[27579]: audit 2023-08-27T23:10:37.522225+0000 mon.c (mon.2) 859 : audit [DBG] from='client.? 172.21.15.6:0/2190941062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:38.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:38 smithi130 bash[51302]: debug 2023-08-27T23:10:38.183+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:38.188980+0000) 2023-08-27T23:10:38.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:38 smithi006 bash[33918]: cluster 2023-08-27T23:10:36.653771+0000 mgr.y (mgr.14901) 2162 : cluster [DBG] pgmap v2169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:38.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:38 smithi006 bash[33918]: audit 2023-08-27T23:10:37.522225+0000 mon.c (mon.2) 859 : audit [DBG] from='client.? 172.21.15.6:0/2190941062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:38.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:38 smithi006 bash[27292]: cluster 2023-08-27T23:10:36.653771+0000 mgr.y (mgr.14901) 2162 : cluster [DBG] pgmap v2169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:38.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:38 smithi006 bash[27292]: audit 2023-08-27T23:10:37.522225+0000 mon.c (mon.2) 859 : audit [DBG] from='client.? 172.21.15.6:0/2190941062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:39.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:39 smithi130 bash[51302]: debug 2023-08-27T23:10:39.183+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:39.189214+0000) 2023-08-27T23:10:40.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:40 smithi130 bash[51302]: debug 2023-08-27T23:10:40.187+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:40.189512+0000) 2023-08-27T23:10:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:40 smithi130 bash[27579]: cluster 2023-08-27T23:10:38.654435+0000 mgr.y (mgr.14901) 2163 : cluster [DBG] pgmap v2170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:40 smithi130 bash[27579]: audit 2023-08-27T23:10:39.935967+0000 mon.a (mon.0) 1977 : audit [DBG] from='client.? 172.21.15.6:0/3851506877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:40.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:40 smithi006 bash[33918]: cluster 2023-08-27T23:10:38.654435+0000 mgr.y (mgr.14901) 2163 : cluster [DBG] pgmap v2170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:40.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:40 smithi006 bash[33918]: audit 2023-08-27T23:10:39.935967+0000 mon.a (mon.0) 1977 : audit [DBG] from='client.? 172.21.15.6:0/3851506877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:10:40.393Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:10:40.395Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:40 smithi006 bash[27292]: cluster 2023-08-27T23:10:38.654435+0000 mgr.y (mgr.14901) 2163 : cluster [DBG] pgmap v2170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:40 smithi006 bash[27292]: audit 2023-08-27T23:10:39.935967+0000 mon.a (mon.0) 1977 : audit [DBG] from='client.? 172.21.15.6:0/3851506877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:10:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:10:40] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:10:41.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:41 smithi130 bash[51302]: debug 2023-08-27T23:10:41.187+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:41.189797+0000) 2023-08-27T23:10:42.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:41 smithi006 bash[56298]: level=error ts=2023-08-27T23:10:41.959Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:42.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:41 smithi006 bash[56298]: level=warn ts=2023-08-27T23:10:41.961Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:42.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:42 smithi130 bash[51302]: debug 2023-08-27T23:10:42.183+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:42.190122+0000) 2023-08-27T23:10:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:42 smithi130 bash[27579]: cluster 2023-08-27T23:10:40.655601+0000 mgr.y (mgr.14901) 2164 : cluster [DBG] pgmap v2171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:42 smithi130 bash[27579]: audit 2023-08-27T23:10:42.354246+0000 mon.c (mon.2) 860 : audit [DBG] from='client.? 172.21.15.6:0/4088235018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:42.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:42 smithi006 bash[33918]: cluster 2023-08-27T23:10:40.655601+0000 mgr.y (mgr.14901) 2164 : cluster [DBG] pgmap v2171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:42.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:42 smithi006 bash[33918]: audit 2023-08-27T23:10:42.354246+0000 mon.c (mon.2) 860 : audit [DBG] from='client.? 172.21.15.6:0/4088235018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:42 smithi006 bash[27292]: cluster 2023-08-27T23:10:40.655601+0000 mgr.y (mgr.14901) 2164 : cluster [DBG] pgmap v2171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:42 smithi006 bash[27292]: audit 2023-08-27T23:10:42.354246+0000 mon.c (mon.2) 860 : audit [DBG] from='client.? 172.21.15.6:0/4088235018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:43.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:43 smithi130 bash[51302]: debug 2023-08-27T23:10:43.187+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:43.190396+0000) 2023-08-27T23:10:44.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:10:43.859Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:44.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:10:43.862Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:44.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:44 smithi130 bash[27579]: cluster 2023-08-27T23:10:42.656285+0000 mgr.y (mgr.14901) 2165 : cluster [DBG] pgmap v2172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:44.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:44 smithi130 bash[51302]: debug 2023-08-27T23:10:44.187+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:44.190607+0000) 2023-08-27T23:10:44.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:44 smithi006 bash[27292]: cluster 2023-08-27T23:10:42.656285+0000 mgr.y (mgr.14901) 2165 : cluster [DBG] pgmap v2172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:44.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:44 smithi006 bash[33918]: cluster 2023-08-27T23:10:42.656285+0000 mgr.y (mgr.14901) 2165 : cluster [DBG] pgmap v2172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:45.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:45 smithi130 bash[27579]: audit 2023-08-27T23:10:44.775025+0000 mon.c (mon.2) 861 : audit [DBG] from='client.? 172.21.15.6:0/4148922067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:45.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:45 smithi130 bash[51302]: debug 2023-08-27T23:10:45.187+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:45.190913+0000) 2023-08-27T23:10:45.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:45 smithi006 bash[27292]: audit 2023-08-27T23:10:44.775025+0000 mon.c (mon.2) 861 : audit [DBG] from='client.? 172.21.15.6:0/4148922067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:45.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:45 smithi006 bash[33918]: audit 2023-08-27T23:10:44.775025+0000 mon.c (mon.2) 861 : audit [DBG] from='client.? 172.21.15.6:0/4148922067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:46 smithi130 bash[27579]: cluster 2023-08-27T23:10:44.657038+0000 mgr.y (mgr.14901) 2166 : cluster [DBG] pgmap v2173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:46.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:46 smithi130 bash[51302]: debug 2023-08-27T23:10:46.187+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:46.191160+0000) 2023-08-27T23:10:46.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:46 smithi006 bash[27292]: cluster 2023-08-27T23:10:44.657038+0000 mgr.y (mgr.14901) 2166 : cluster [DBG] pgmap v2173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:46 smithi006 bash[33918]: cluster 2023-08-27T23:10:44.657038+0000 mgr.y (mgr.14901) 2166 : cluster [DBG] pgmap v2173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:47.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:47 smithi130 bash[27579]: audit 2023-08-27T23:10:47.192707+0000 mon.a (mon.0) 1978 : audit [DBG] from='client.? 172.21.15.6:0/4240243246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:47.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:47 smithi130 bash[51302]: debug 2023-08-27T23:10:47.187+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:47.191430+0000) 2023-08-27T23:10:47.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:47 smithi006 bash[27292]: audit 2023-08-27T23:10:47.192707+0000 mon.a (mon.0) 1978 : audit [DBG] from='client.? 172.21.15.6:0/4240243246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:47.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:47 smithi006 bash[33918]: audit 2023-08-27T23:10:47.192707+0000 mon.a (mon.0) 1978 : audit [DBG] from='client.? 172.21.15.6:0/4240243246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:48.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:48 smithi130 bash[27579]: cluster 2023-08-27T23:10:46.657615+0000 mgr.y (mgr.14901) 2167 : cluster [DBG] pgmap v2174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:48.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:48 smithi130 bash[51302]: debug 2023-08-27T23:10:48.187+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:48.191729+0000) 2023-08-27T23:10:48.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:48 smithi006 bash[27292]: cluster 2023-08-27T23:10:46.657615+0000 mgr.y (mgr.14901) 2167 : cluster [DBG] pgmap v2174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:48.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:48 smithi006 bash[33918]: cluster 2023-08-27T23:10:46.657615+0000 mgr.y (mgr.14901) 2167 : cluster [DBG] pgmap v2174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:49.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:49 smithi130 bash[51302]: debug 2023-08-27T23:10:49.187+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:49.192009+0000) 2023-08-27T23:10:50.446 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:50 smithi130 bash[51302]: debug 2023-08-27T23:10:50.191+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:50.192278+0000) 2023-08-27T23:10:50.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:50 smithi006 bash[27292]: cluster 2023-08-27T23:10:48.658293+0000 mgr.y (mgr.14901) 2168 : cluster [DBG] pgmap v2175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:50.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:50 smithi006 bash[27292]: audit 2023-08-27T23:10:49.620980+0000 mon.c (mon.2) 862 : audit [DBG] from='client.? 172.21.15.6:0/3297016420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:50 smithi006 bash[33918]: cluster 2023-08-27T23:10:48.658293+0000 mgr.y (mgr.14901) 2168 : cluster [DBG] pgmap v2175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:50 smithi006 bash[33918]: audit 2023-08-27T23:10:49.620980+0000 mon.c (mon.2) 862 : audit [DBG] from='client.? 172.21.15.6:0/3297016420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:10:50.394Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:10:50.396Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:50 smithi130 bash[27579]: cluster 2023-08-27T23:10:48.658293+0000 mgr.y (mgr.14901) 2168 : cluster [DBG] pgmap v2175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:50 smithi130 bash[27579]: audit 2023-08-27T23:10:49.620980+0000 mon.c (mon.2) 862 : audit [DBG] from='client.? 172.21.15.6:0/3297016420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:10:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:10:50] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:10:51.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:51 smithi130 bash[51302]: debug 2023-08-27T23:10:51.191+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:51.192568+0000) 2023-08-27T23:10:52.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:51 smithi006 bash[56298]: level=error ts=2023-08-27T23:10:51.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:52.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:51 smithi006 bash[56298]: level=warn ts=2023-08-27T23:10:51.962Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:52.451 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:52 smithi130 bash[51302]: debug 2023-08-27T23:10:52.187+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:52.192874+0000) 2023-08-27T23:10:52.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:52 smithi006 bash[27292]: cluster 2023-08-27T23:10:50.659349+0000 mgr.y (mgr.14901) 2169 : cluster [DBG] pgmap v2176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:52.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:52 smithi006 bash[27292]: audit 2023-08-27T23:10:52.036468+0000 mon.a (mon.0) 1979 : audit [DBG] from='client.? 172.21.15.6:0/1400705360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:52 smithi006 bash[33918]: cluster 2023-08-27T23:10:50.659349+0000 mgr.y (mgr.14901) 2169 : cluster [DBG] pgmap v2176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:52 smithi006 bash[33918]: audit 2023-08-27T23:10:52.036468+0000 mon.a (mon.0) 1979 : audit [DBG] from='client.? 172.21.15.6:0/1400705360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:52 smithi130 bash[27579]: cluster 2023-08-27T23:10:50.659349+0000 mgr.y (mgr.14901) 2169 : cluster [DBG] pgmap v2176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:52 smithi130 bash[27579]: audit 2023-08-27T23:10:52.036468+0000 mon.a (mon.0) 1979 : audit [DBG] from='client.? 172.21.15.6:0/1400705360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:53.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:53 smithi130 bash[51302]: debug 2023-08-27T23:10:53.187+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:53.193135+0000) 2023-08-27T23:10:54.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:10:53.859Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:54.485 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:10:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:10:53.862Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:10:54.486 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:54 smithi130 bash[51302]: debug 2023-08-27T23:10:54.187+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:54.193467+0000) 2023-08-27T23:10:54.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:54 smithi006 bash[33918]: cluster 2023-08-27T23:10:52.659944+0000 mgr.y (mgr.14901) 2170 : cluster [DBG] pgmap v2177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:54 smithi006 bash[27292]: cluster 2023-08-27T23:10:52.659944+0000 mgr.y (mgr.14901) 2170 : cluster [DBG] pgmap v2177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:54 smithi130 bash[27579]: cluster 2023-08-27T23:10:52.659944+0000 mgr.y (mgr.14901) 2170 : cluster [DBG] pgmap v2177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:55.458 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:55 smithi130 bash[51302]: debug 2023-08-27T23:10:55.187+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:55.193674+0000) 2023-08-27T23:10:55.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:55 smithi006 bash[33918]: audit 2023-08-27T23:10:54.462430+0000 mon.a (mon.0) 1980 : audit [DBG] from='client.? 172.21.15.6:0/2045915139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:55.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:55 smithi006 bash[27292]: audit 2023-08-27T23:10:54.462430+0000 mon.a (mon.0) 1980 : audit [DBG] from='client.? 172.21.15.6:0/2045915139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:55.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:55 smithi130 bash[27579]: audit 2023-08-27T23:10:54.462430+0000 mon.a (mon.0) 1980 : audit [DBG] from='client.? 172.21.15.6:0/2045915139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:56.467 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:56 smithi130 bash[51302]: debug 2023-08-27T23:10:56.191+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:56.193935+0000) 2023-08-27T23:10:56.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:56 smithi130 bash[27579]: cluster 2023-08-27T23:10:54.661096+0000 mgr.y (mgr.14901) 2171 : cluster [DBG] pgmap v2178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:56.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:56 smithi006 bash[33918]: cluster 2023-08-27T23:10:54.661096+0000 mgr.y (mgr.14901) 2171 : cluster [DBG] pgmap v2178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:56.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:56 smithi006 bash[27292]: cluster 2023-08-27T23:10:54.661096+0000 mgr.y (mgr.14901) 2171 : cluster [DBG] pgmap v2178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:57.476 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:57 smithi130 bash[51302]: debug 2023-08-27T23:10:57.187+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:57.194275+0000) 2023-08-27T23:10:57.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:57 smithi130 bash[27579]: audit 2023-08-27T23:10:56.875930+0000 mon.c (mon.2) 863 : audit [DBG] from='client.? 172.21.15.6:0/177310234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:57.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:57 smithi006 bash[27292]: audit 2023-08-27T23:10:56.875930+0000 mon.c (mon.2) 863 : audit [DBG] from='client.? 172.21.15.6:0/177310234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:57.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:57 smithi006 bash[33918]: audit 2023-08-27T23:10:56.875930+0000 mon.c (mon.2) 863 : audit [DBG] from='client.? 172.21.15.6:0/177310234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:58.478 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:58 smithi130 bash[51302]: debug 2023-08-27T23:10:58.187+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:58.194555+0000) 2023-08-27T23:10:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:58 smithi130 bash[27579]: cluster 2023-08-27T23:10:56.661830+0000 mgr.y (mgr.14901) 2172 : cluster [DBG] pgmap v2179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:58.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:58 smithi006 bash[27292]: cluster 2023-08-27T23:10:56.661830+0000 mgr.y (mgr.14901) 2172 : cluster [DBG] pgmap v2179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:58.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:58 smithi006 bash[33918]: cluster 2023-08-27T23:10:56.661830+0000 mgr.y (mgr.14901) 2172 : cluster [DBG] pgmap v2179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:10:59.487 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:10:59 smithi130 bash[51302]: debug 2023-08-27T23:10:59.191+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:59.194849+0000) 2023-08-27T23:10:59.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:10:59 smithi130 bash[27579]: audit 2023-08-27T23:10:59.290431+0000 mon.a (mon.0) 1981 : audit [DBG] from='client.? 172.21.15.6:0/3871229996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:59.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:10:59 smithi006 bash[33918]: audit 2023-08-27T23:10:59.290431+0000 mon.a (mon.0) 1981 : audit [DBG] from='client.? 172.21.15.6:0/3871229996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:59.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:10:59 smithi006 bash[27292]: audit 2023-08-27T23:10:59.290431+0000 mon.a (mon.0) 1981 : audit [DBG] from='client.? 172.21.15.6:0/3871229996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:00.490 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:00 smithi130 bash[51302]: debug 2023-08-27T23:11:00.191+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:00.195152+0000) 2023-08-27T23:11:00.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:00 smithi006 bash[27292]: cluster 2023-08-27T23:10:58.662543+0000 mgr.y (mgr.14901) 2173 : cluster [DBG] pgmap v2180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:00 smithi006 bash[33918]: cluster 2023-08-27T23:10:58.662543+0000 mgr.y (mgr.14901) 2173 : cluster [DBG] pgmap v2180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:11:00.395Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:11:00.397Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:00 smithi130 bash[27579]: cluster 2023-08-27T23:10:58.662543+0000 mgr.y (mgr.14901) 2173 : cluster [DBG] pgmap v2180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:11:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:11:00] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:11:01.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:01 smithi130 bash[51302]: debug 2023-08-27T23:11:01.191+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:01.195480+0000) 2023-08-27T23:11:02.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:01 smithi006 bash[56298]: level=error ts=2023-08-27T23:11:01.961Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:02.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:01 smithi006 bash[56298]: level=warn ts=2023-08-27T23:11:01.963Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:02.495 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:02 smithi130 bash[51302]: debug 2023-08-27T23:11:02.191+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:02.195778+0000) 2023-08-27T23:11:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:02 smithi130 bash[27579]: cluster 2023-08-27T23:11:00.663713+0000 mgr.y (mgr.14901) 2174 : cluster [DBG] pgmap v2181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:02 smithi130 bash[27579]: audit 2023-08-27T23:11:01.708095+0000 mon.c (mon.2) 864 : audit [DBG] from='client.? 172.21.15.6:0/4026926417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:02 smithi006 bash[33918]: cluster 2023-08-27T23:11:00.663713+0000 mgr.y (mgr.14901) 2174 : cluster [DBG] pgmap v2181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:02 smithi006 bash[33918]: audit 2023-08-27T23:11:01.708095+0000 mon.c (mon.2) 864 : audit [DBG] from='client.? 172.21.15.6:0/4026926417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:02 smithi006 bash[27292]: cluster 2023-08-27T23:11:00.663713+0000 mgr.y (mgr.14901) 2174 : cluster [DBG] pgmap v2181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:02 smithi006 bash[27292]: audit 2023-08-27T23:11:01.708095+0000 mon.c (mon.2) 864 : audit [DBG] from='client.? 172.21.15.6:0/4026926417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:03.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:03 smithi130 bash[51302]: debug 2023-08-27T23:11:03.191+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:03.195984+0000) 2023-08-27T23:11:04.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:11:03.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:04.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:11:03.862Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:04.500 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:04 smithi130 bash[51302]: debug 2023-08-27T23:11:04.191+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:04.196180+0000) 2023-08-27T23:11:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:04 smithi130 bash[27579]: cluster 2023-08-27T23:11:02.664359+0000 mgr.y (mgr.14901) 2175 : cluster [DBG] pgmap v2182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:04 smithi130 bash[27579]: audit 2023-08-27T23:11:04.120771+0000 mon.a (mon.0) 1982 : audit [DBG] from='client.? 172.21.15.6:0/2473949421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:04.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:04 smithi006 bash[27292]: cluster 2023-08-27T23:11:02.664359+0000 mgr.y (mgr.14901) 2175 : cluster [DBG] pgmap v2182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:04.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:04 smithi006 bash[27292]: audit 2023-08-27T23:11:04.120771+0000 mon.a (mon.0) 1982 : audit [DBG] from='client.? 172.21.15.6:0/2473949421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:04 smithi006 bash[33918]: cluster 2023-08-27T23:11:02.664359+0000 mgr.y (mgr.14901) 2175 : cluster [DBG] pgmap v2182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:04 smithi006 bash[33918]: audit 2023-08-27T23:11:04.120771+0000 mon.a (mon.0) 1982 : audit [DBG] from='client.? 172.21.15.6:0/2473949421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:05.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:05 smithi130 bash[51302]: debug 2023-08-27T23:11:05.195+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:05.196477+0000) 2023-08-27T23:11:06.506 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:06 smithi130 bash[51302]: debug 2023-08-27T23:11:06.195+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:06.196774+0000) 2023-08-27T23:11:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:06 smithi130 bash[27579]: cluster 2023-08-27T23:11:04.665542+0000 mgr.y (mgr.14901) 2176 : cluster [DBG] pgmap v2183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:06.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:06 smithi006 bash[33918]: cluster 2023-08-27T23:11:04.665542+0000 mgr.y (mgr.14901) 2176 : cluster [DBG] pgmap v2183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:06.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:06 smithi006 bash[27292]: cluster 2023-08-27T23:11:04.665542+0000 mgr.y (mgr.14901) 2176 : cluster [DBG] pgmap v2183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:07.513 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:07 smithi130 bash[51302]: debug 2023-08-27T23:11:07.191+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:07.197086+0000) 2023-08-27T23:11:07.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:07 smithi130 bash[27579]: audit 2023-08-27T23:11:06.541946+0000 mon.a (mon.0) 1983 : audit [DBG] from='client.? 172.21.15.6:0/756054897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:07.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:07 smithi006 bash[27292]: audit 2023-08-27T23:11:06.541946+0000 mon.a (mon.0) 1983 : audit [DBG] from='client.? 172.21.15.6:0/756054897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:07.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:07 smithi006 bash[33918]: audit 2023-08-27T23:11:06.541946+0000 mon.a (mon.0) 1983 : audit [DBG] from='client.? 172.21.15.6:0/756054897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:08.516 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:08 smithi130 bash[51302]: debug 2023-08-27T23:11:08.191+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:08.197382+0000) 2023-08-27T23:11:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:08 smithi130 bash[27579]: cluster 2023-08-27T23:11:06.666170+0000 mgr.y (mgr.14901) 2177 : cluster [DBG] pgmap v2184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:08.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:08 smithi006 bash[27292]: cluster 2023-08-27T23:11:06.666170+0000 mgr.y (mgr.14901) 2177 : cluster [DBG] pgmap v2184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:08.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:08 smithi006 bash[33918]: cluster 2023-08-27T23:11:06.666170+0000 mgr.y (mgr.14901) 2177 : cluster [DBG] pgmap v2184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:09.524 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:09 smithi130 bash[51302]: debug 2023-08-27T23:11:09.191+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:09.197721+0000) 2023-08-27T23:11:09.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:09 smithi130 bash[27579]: audit 2023-08-27T23:11:08.959965+0000 mon.c (mon.2) 865 : audit [DBG] from='client.? 172.21.15.6:0/2650493075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:09.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:09 smithi006 bash[27292]: audit 2023-08-27T23:11:08.959965+0000 mon.c (mon.2) 865 : audit [DBG] from='client.? 172.21.15.6:0/2650493075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:09.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:09 smithi006 bash[33918]: audit 2023-08-27T23:11:08.959965+0000 mon.c (mon.2) 865 : audit [DBG] from='client.? 172.21.15.6:0/2650493075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:10.528 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:10 smithi130 bash[51302]: debug 2023-08-27T23:11:10.194+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:10.198014+0000) 2023-08-27T23:11:10.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:10 smithi006 bash[27292]: cluster 2023-08-27T23:11:08.666681+0000 mgr.y (mgr.14901) 2178 : cluster [DBG] pgmap v2185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:10 smithi006 bash[33918]: cluster 2023-08-27T23:11:08.666681+0000 mgr.y (mgr.14901) 2178 : cluster [DBG] pgmap v2185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:11:10.396Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:11:10.398Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:10 smithi130 bash[27579]: cluster 2023-08-27T23:11:08.666681+0000 mgr.y (mgr.14901) 2178 : cluster [DBG] pgmap v2185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:11:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:11:10] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:11:11.535 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:11 smithi130 bash[51302]: debug 2023-08-27T23:11:11.190+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:11.198289+0000) 2023-08-27T23:11:11.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:11 smithi130 bash[27579]: audit 2023-08-27T23:11:11.379622+0000 mon.a (mon.0) 1984 : audit [DBG] from='client.? 172.21.15.6:0/3888387965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:11.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:11 smithi006 bash[33918]: audit 2023-08-27T23:11:11.379622+0000 mon.a (mon.0) 1984 : audit [DBG] from='client.? 172.21.15.6:0/3888387965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:11.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:11 smithi006 bash[27292]: audit 2023-08-27T23:11:11.379622+0000 mon.a (mon.0) 1984 : audit [DBG] from='client.? 172.21.15.6:0/3888387965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:12.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:11 smithi006 bash[56298]: level=error ts=2023-08-27T23:11:11.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:12.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:11 smithi006 bash[56298]: level=warn ts=2023-08-27T23:11:11.964Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:12.538 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:12 smithi130 bash[51302]: debug 2023-08-27T23:11:12.190+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:12.198503+0000) 2023-08-27T23:11:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:12 smithi130 bash[27579]: cluster 2023-08-27T23:11:10.667914+0000 mgr.y (mgr.14901) 2179 : cluster [DBG] pgmap v2186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:12 smithi006 bash[33918]: cluster 2023-08-27T23:11:10.667914+0000 mgr.y (mgr.14901) 2179 : cluster [DBG] pgmap v2186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:12 smithi006 bash[27292]: cluster 2023-08-27T23:11:10.667914+0000 mgr.y (mgr.14901) 2179 : cluster [DBG] pgmap v2186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:13.548 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:13 smithi130 bash[51302]: debug 2023-08-27T23:11:13.194+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:13.198782+0000) 2023-08-27T23:11:13.864 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:13 smithi006 bash[33918]: audit 2023-08-27T23:11:12.791163+0000 mon.b (mon.1) 329 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:11:13.864 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:13 smithi006 bash[27292]: audit 2023-08-27T23:11:12.791163+0000 mon.b (mon.1) 329 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:11:13.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:13 smithi130 bash[27579]: audit 2023-08-27T23:11:12.791163+0000 mon.b (mon.1) 329 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:11:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:11:13.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:11:13.864Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:14.551 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:14 smithi130 bash[51302]: debug 2023-08-27T23:11:14.194+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:14.199050+0000) 2023-08-27T23:11:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:14 smithi130 bash[27579]: cluster 2023-08-27T23:11:12.668627+0000 mgr.y (mgr.14901) 2180 : cluster [DBG] pgmap v2187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:14 smithi130 bash[27579]: audit 2023-08-27T23:11:13.791019+0000 mon.c (mon.2) 866 : audit [DBG] from='client.? 172.21.15.6:0/2580633952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:14.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:14 smithi006 bash[33918]: cluster 2023-08-27T23:11:12.668627+0000 mgr.y (mgr.14901) 2180 : cluster [DBG] pgmap v2187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:14.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:14 smithi006 bash[33918]: audit 2023-08-27T23:11:13.791019+0000 mon.c (mon.2) 866 : audit [DBG] from='client.? 172.21.15.6:0/2580633952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:14 smithi006 bash[27292]: cluster 2023-08-27T23:11:12.668627+0000 mgr.y (mgr.14901) 2180 : cluster [DBG] pgmap v2187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:14 smithi006 bash[27292]: audit 2023-08-27T23:11:13.791019+0000 mon.c (mon.2) 866 : audit [DBG] from='client.? 172.21.15.6:0/2580633952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:15.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:15 smithi130 bash[51302]: debug 2023-08-27T23:11:15.198+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:15.199332+0000) 2023-08-27T23:11:16.556 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:16 smithi130 bash[51302]: debug 2023-08-27T23:11:16.198+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:16.199558+0000) 2023-08-27T23:11:16.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:16 smithi130 bash[27579]: cluster 2023-08-27T23:11:14.669749+0000 mgr.y (mgr.14901) 2181 : cluster [DBG] pgmap v2188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:16 smithi130 bash[27579]: audit 2023-08-27T23:11:16.212645+0000 mon.c (mon.2) 867 : audit [DBG] from='client.? 172.21.15.6:0/470903140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:16.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:16 smithi006 bash[33918]: cluster 2023-08-27T23:11:14.669749+0000 mgr.y (mgr.14901) 2181 : cluster [DBG] pgmap v2188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:16.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:16 smithi006 bash[33918]: audit 2023-08-27T23:11:16.212645+0000 mon.c (mon.2) 867 : audit [DBG] from='client.? 172.21.15.6:0/470903140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:16 smithi006 bash[27292]: cluster 2023-08-27T23:11:14.669749+0000 mgr.y (mgr.14901) 2181 : cluster [DBG] pgmap v2188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:16 smithi006 bash[27292]: audit 2023-08-27T23:11:16.212645+0000 mon.c (mon.2) 867 : audit [DBG] from='client.? 172.21.15.6:0/470903140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:17.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:17 smithi130 bash[51302]: debug 2023-08-27T23:11:17.198+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:17.199813+0000) 2023-08-27T23:11:18.561 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:18 smithi130 bash[51302]: debug 2023-08-27T23:11:18.198+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:18.200049+0000) 2023-08-27T23:11:18.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:18 smithi130 bash[27579]: cluster 2023-08-27T23:11:16.670381+0000 mgr.y (mgr.14901) 2182 : cluster [DBG] pgmap v2189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:18.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:18 smithi006 bash[27292]: cluster 2023-08-27T23:11:16.670381+0000 mgr.y (mgr.14901) 2182 : cluster [DBG] pgmap v2189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:18.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:18 smithi006 bash[33918]: cluster 2023-08-27T23:11:16.670381+0000 mgr.y (mgr.14901) 2182 : cluster [DBG] pgmap v2189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:19.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:19 smithi130 bash[51302]: debug 2023-08-27T23:11:19.198+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:19.200241+0000) 2023-08-27T23:11:19.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:19 smithi006 bash[33918]: cluster 2023-08-27T23:11:18.670848+0000 mgr.y (mgr.14901) 2183 : cluster [DBG] pgmap v2190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:19 smithi006 bash[33918]: audit 2023-08-27T23:11:18.687863+0000 mon.a (mon.0) 1985 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:11:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:19 smithi006 bash[33918]: audit 2023-08-27T23:11:18.695795+0000 mon.a (mon.0) 1986 : audit [DBG] from='client.? 172.21.15.6:0/3986442199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:19 smithi006 bash[33918]: audit 2023-08-27T23:11:19.138308+0000 mon.a (mon.0) 1987 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:11:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:19 smithi006 bash[33918]: audit 2023-08-27T23:11:19.432488+0000 mon.b (mon.1) 330 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:11:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:19 smithi006 bash[33918]: audit 2023-08-27T23:11:19.435058+0000 mon.b (mon.1) 331 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:11:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:19 smithi006 bash[33918]: audit 2023-08-27T23:11:19.444483+0000 mon.a (mon.0) 1988 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:11:19.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:19 smithi006 bash[27292]: cluster 2023-08-27T23:11:18.670848+0000 mgr.y (mgr.14901) 2183 : cluster [DBG] pgmap v2190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:19.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:19 smithi006 bash[27292]: audit 2023-08-27T23:11:18.687863+0000 mon.a (mon.0) 1985 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:11:19.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:19 smithi006 bash[27292]: audit 2023-08-27T23:11:18.695795+0000 mon.a (mon.0) 1986 : audit [DBG] from='client.? 172.21.15.6:0/3986442199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:19.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:19 smithi006 bash[27292]: audit 2023-08-27T23:11:19.138308+0000 mon.a (mon.0) 1987 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:11:19.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:19 smithi006 bash[27292]: audit 2023-08-27T23:11:19.432488+0000 mon.b (mon.1) 330 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:11:19.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:19 smithi006 bash[27292]: audit 2023-08-27T23:11:19.435058+0000 mon.b (mon.1) 331 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:11:19.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:19 smithi006 bash[27292]: audit 2023-08-27T23:11:19.444483+0000 mon.a (mon.0) 1988 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:11:20.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:19 smithi130 bash[27579]: cluster 2023-08-27T23:11:18.670848+0000 mgr.y (mgr.14901) 2183 : cluster [DBG] pgmap v2190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:20.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:19 smithi130 bash[27579]: audit 2023-08-27T23:11:18.687863+0000 mon.a (mon.0) 1985 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:11:20.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:19 smithi130 bash[27579]: audit 2023-08-27T23:11:18.695795+0000 mon.a (mon.0) 1986 : audit [DBG] from='client.? 172.21.15.6:0/3986442199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:20.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:19 smithi130 bash[27579]: audit 2023-08-27T23:11:19.138308+0000 mon.a (mon.0) 1987 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:11:20.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:19 smithi130 bash[27579]: audit 2023-08-27T23:11:19.432488+0000 mon.b (mon.1) 330 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:11:20.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:19 smithi130 bash[27579]: audit 2023-08-27T23:11:19.435058+0000 mon.b (mon.1) 331 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:11:20.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:19 smithi130 bash[27579]: audit 2023-08-27T23:11:19.444483+0000 mon.a (mon.0) 1988 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:11:20.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:20 smithi130 bash[51302]: debug 2023-08-27T23:11:20.198+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:20.200536+0000) 2023-08-27T23:11:20.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:11:20.397Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:20.728 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:11:20.399Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:21.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:11:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:11:20] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:11:21.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:21 smithi130 bash[51302]: debug 2023-08-27T23:11:21.198+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:21.200802+0000) 2023-08-27T23:11:21.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:21 smithi006 bash[33918]: cluster 2023-08-27T23:11:20.671864+0000 mgr.y (mgr.14901) 2184 : cluster [DBG] pgmap v2191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:21.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:21 smithi006 bash[33918]: audit 2023-08-27T23:11:21.113528+0000 mon.a (mon.0) 1989 : audit [DBG] from='client.? 172.21.15.6:0/2822818771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:21.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:21 smithi006 bash[27292]: cluster 2023-08-27T23:11:20.671864+0000 mgr.y (mgr.14901) 2184 : cluster [DBG] pgmap v2191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:21.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:21 smithi006 bash[27292]: audit 2023-08-27T23:11:21.113528+0000 mon.a (mon.0) 1989 : audit [DBG] from='client.? 172.21.15.6:0/2822818771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:22.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:21 smithi130 bash[27579]: cluster 2023-08-27T23:11:20.671864+0000 mgr.y (mgr.14901) 2184 : cluster [DBG] pgmap v2191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:22.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:21 smithi130 bash[27579]: audit 2023-08-27T23:11:21.113528+0000 mon.a (mon.0) 1989 : audit [DBG] from='client.? 172.21.15.6:0/2822818771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:22.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:21 smithi006 bash[56298]: level=error ts=2023-08-27T23:11:21.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:22.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:21 smithi006 bash[56298]: level=warn ts=2023-08-27T23:11:21.964Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:22.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:22 smithi130 bash[51302]: debug 2023-08-27T23:11:22.198+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:22.201172+0000) 2023-08-27T23:11:23.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:23 smithi130 bash[51302]: debug 2023-08-27T23:11:23.198+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:23.201482+0000) 2023-08-27T23:11:23.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:23 smithi006 bash[27292]: cluster 2023-08-27T23:11:22.672490+0000 mgr.y (mgr.14901) 2185 : cluster [DBG] pgmap v2192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:23.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:23 smithi006 bash[27292]: audit 2023-08-27T23:11:23.520737+0000 mon.a (mon.0) 1990 : audit [DBG] from='client.? 172.21.15.6:0/3468424690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:23.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:23 smithi006 bash[33918]: cluster 2023-08-27T23:11:22.672490+0000 mgr.y (mgr.14901) 2185 : cluster [DBG] pgmap v2192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:23.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:23 smithi006 bash[33918]: audit 2023-08-27T23:11:23.520737+0000 mon.a (mon.0) 1990 : audit [DBG] from='client.? 172.21.15.6:0/3468424690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:23.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:11:23.863Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:23.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:11:23.865Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:24.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:23 smithi130 bash[27579]: cluster 2023-08-27T23:11:22.672490+0000 mgr.y (mgr.14901) 2185 : cluster [DBG] pgmap v2192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:24.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:23 smithi130 bash[27579]: audit 2023-08-27T23:11:23.520737+0000 mon.a (mon.0) 1990 : audit [DBG] from='client.? 172.21.15.6:0/3468424690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:24.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:24 smithi130 bash[51302]: debug 2023-08-27T23:11:24.198+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:24.201781+0000) 2023-08-27T23:11:25.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:25 smithi130 bash[51302]: debug 2023-08-27T23:11:25.198+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:25.202049+0000) 2023-08-27T23:11:26.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:26 smithi130 bash[27579]: cluster 2023-08-27T23:11:24.673631+0000 mgr.y (mgr.14901) 2186 : cluster [DBG] pgmap v2193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:26 smithi130 bash[27579]: audit 2023-08-27T23:11:25.926083+0000 mon.a (mon.0) 1991 : audit [DBG] from='client.? 172.21.15.6:0/4291149769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:26.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:26 smithi130 bash[51302]: debug 2023-08-27T23:11:26.198+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:26.202361+0000) 2023-08-27T23:11:26.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:26 smithi006 bash[27292]: cluster 2023-08-27T23:11:24.673631+0000 mgr.y (mgr.14901) 2186 : cluster [DBG] pgmap v2193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:26.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:26 smithi006 bash[27292]: audit 2023-08-27T23:11:25.926083+0000 mon.a (mon.0) 1991 : audit [DBG] from='client.? 172.21.15.6:0/4291149769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:26.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:26 smithi006 bash[33918]: cluster 2023-08-27T23:11:24.673631+0000 mgr.y (mgr.14901) 2186 : cluster [DBG] pgmap v2193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:26 smithi006 bash[33918]: audit 2023-08-27T23:11:25.926083+0000 mon.a (mon.0) 1991 : audit [DBG] from='client.? 172.21.15.6:0/4291149769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:27.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:27 smithi130 bash[51302]: debug 2023-08-27T23:11:27.198+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:27.202613+0000) 2023-08-27T23:11:28.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:28 smithi130 bash[51302]: debug 2023-08-27T23:11:28.202+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:28.202893+0000) 2023-08-27T23:11:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:28 smithi130 bash[27579]: cluster 2023-08-27T23:11:26.674259+0000 mgr.y (mgr.14901) 2187 : cluster [DBG] pgmap v2194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:28 smithi130 bash[27579]: audit 2023-08-27T23:11:28.335821+0000 mon.a (mon.0) 1992 : audit [DBG] from='client.? 172.21.15.6:0/3620346118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:28.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:28 smithi006 bash[27292]: cluster 2023-08-27T23:11:26.674259+0000 mgr.y (mgr.14901) 2187 : cluster [DBG] pgmap v2194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:28.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:28 smithi006 bash[27292]: audit 2023-08-27T23:11:28.335821+0000 mon.a (mon.0) 1992 : audit [DBG] from='client.? 172.21.15.6:0/3620346118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:28 smithi006 bash[33918]: cluster 2023-08-27T23:11:26.674259+0000 mgr.y (mgr.14901) 2187 : cluster [DBG] pgmap v2194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:28 smithi006 bash[33918]: audit 2023-08-27T23:11:28.335821+0000 mon.a (mon.0) 1992 : audit [DBG] from='client.? 172.21.15.6:0/3620346118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:29.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:29 smithi130 bash[51302]: debug 2023-08-27T23:11:29.202+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:29.203144+0000) 2023-08-27T23:11:30.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:30 smithi130 bash[27579]: cluster 2023-08-27T23:11:28.674872+0000 mgr.y (mgr.14901) 2188 : cluster [DBG] pgmap v2195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:30.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:30 smithi130 bash[51302]: debug 2023-08-27T23:11:30.202+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:30.203388+0000) 2023-08-27T23:11:30.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:30 smithi006 bash[27292]: cluster 2023-08-27T23:11:28.674872+0000 mgr.y (mgr.14901) 2188 : cluster [DBG] pgmap v2195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:30 smithi006 bash[33918]: cluster 2023-08-27T23:11:28.674872+0000 mgr.y (mgr.14901) 2188 : cluster [DBG] pgmap v2195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:11:30.398Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:11:30.400Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:11:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:11:30] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:11:31.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:31 smithi130 bash[27579]: audit 2023-08-27T23:11:30.761999+0000 mon.c (mon.2) 868 : audit [DBG] from='client.? 172.21.15.6:0/1276903619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:31.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:31 smithi130 bash[51302]: debug 2023-08-27T23:11:31.202+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:31.203688+0000) 2023-08-27T23:11:31.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:31 smithi006 bash[27292]: audit 2023-08-27T23:11:30.761999+0000 mon.c (mon.2) 868 : audit [DBG] from='client.? 172.21.15.6:0/1276903619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:31.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:31 smithi006 bash[33918]: audit 2023-08-27T23:11:30.761999+0000 mon.c (mon.2) 868 : audit [DBG] from='client.? 172.21.15.6:0/1276903619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:32.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:31 smithi006 bash[56298]: level=error ts=2023-08-27T23:11:31.963Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:32.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:31 smithi006 bash[56298]: level=warn ts=2023-08-27T23:11:31.965Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:32.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:32 smithi130 bash[27579]: cluster 2023-08-27T23:11:30.676106+0000 mgr.y (mgr.14901) 2189 : cluster [DBG] pgmap v2196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:32.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:32 smithi130 bash[51302]: debug 2023-08-27T23:11:32.202+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:32.203949+0000) 2023-08-27T23:11:32.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:32 smithi006 bash[27292]: cluster 2023-08-27T23:11:30.676106+0000 mgr.y (mgr.14901) 2189 : cluster [DBG] pgmap v2196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:32.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:32 smithi006 bash[33918]: cluster 2023-08-27T23:11:30.676106+0000 mgr.y (mgr.14901) 2189 : cluster [DBG] pgmap v2196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:33 smithi130 bash[27579]: audit 2023-08-27T23:11:33.171503+0000 mon.c (mon.2) 869 : audit [DBG] from='client.? 172.21.15.6:0/1041871791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:33.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:33 smithi130 bash[51302]: debug 2023-08-27T23:11:33.202+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:33.204277+0000) 2023-08-27T23:11:33.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:33 smithi006 bash[27292]: audit 2023-08-27T23:11:33.171503+0000 mon.c (mon.2) 869 : audit [DBG] from='client.? 172.21.15.6:0/1041871791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:33.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:33 smithi006 bash[33918]: audit 2023-08-27T23:11:33.171503+0000 mon.c (mon.2) 869 : audit [DBG] from='client.? 172.21.15.6:0/1041871791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:34.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:11:33.864Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:34.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:11:33.866Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:34.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:34 smithi130 bash[27579]: cluster 2023-08-27T23:11:32.676925+0000 mgr.y (mgr.14901) 2190 : cluster [DBG] pgmap v2197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:34.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:34 smithi130 bash[51302]: debug 2023-08-27T23:11:34.202+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:34.204557+0000) 2023-08-27T23:11:34.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:34 smithi006 bash[27292]: cluster 2023-08-27T23:11:32.676925+0000 mgr.y (mgr.14901) 2190 : cluster [DBG] pgmap v2197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:34.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:34 smithi006 bash[33918]: cluster 2023-08-27T23:11:32.676925+0000 mgr.y (mgr.14901) 2190 : cluster [DBG] pgmap v2197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:35.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:35 smithi130 bash[51302]: debug 2023-08-27T23:11:35.202+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:35.204845+0000) 2023-08-27T23:11:36.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:36 smithi130 bash[51302]: debug 2023-08-27T23:11:36.202+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:36.205150+0000) 2023-08-27T23:11:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:36 smithi130 bash[27579]: cluster 2023-08-27T23:11:34.677933+0000 mgr.y (mgr.14901) 2191 : cluster [DBG] pgmap v2198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:36 smithi130 bash[27579]: audit 2023-08-27T23:11:35.579119+0000 mon.c (mon.2) 870 : audit [DBG] from='client.? 172.21.15.6:0/2604305205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:36.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:36 smithi006 bash[27292]: cluster 2023-08-27T23:11:34.677933+0000 mgr.y (mgr.14901) 2191 : cluster [DBG] pgmap v2198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:36.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:36 smithi006 bash[27292]: audit 2023-08-27T23:11:35.579119+0000 mon.c (mon.2) 870 : audit [DBG] from='client.? 172.21.15.6:0/2604305205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:36.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:36 smithi006 bash[33918]: cluster 2023-08-27T23:11:34.677933+0000 mgr.y (mgr.14901) 2191 : cluster [DBG] pgmap v2198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:36 smithi006 bash[33918]: audit 2023-08-27T23:11:35.579119+0000 mon.c (mon.2) 870 : audit [DBG] from='client.? 172.21.15.6:0/2604305205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:37.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:37 smithi130 bash[51302]: debug 2023-08-27T23:11:37.202+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:37.205395+0000) 2023-08-27T23:11:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:38 smithi130 bash[27579]: cluster 2023-08-27T23:11:36.678528+0000 mgr.y (mgr.14901) 2192 : cluster [DBG] pgmap v2199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:38 smithi130 bash[27579]: audit 2023-08-27T23:11:38.003340+0000 mon.a (mon.0) 1993 : audit [DBG] from='client.? 172.21.15.6:0/2284605486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:38.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:38 smithi130 bash[51302]: debug 2023-08-27T23:11:38.202+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:38.205675+0000) 2023-08-27T23:11:38.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:38 smithi006 bash[33918]: cluster 2023-08-27T23:11:36.678528+0000 mgr.y (mgr.14901) 2192 : cluster [DBG] pgmap v2199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:38.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:38 smithi006 bash[33918]: audit 2023-08-27T23:11:38.003340+0000 mon.a (mon.0) 1993 : audit [DBG] from='client.? 172.21.15.6:0/2284605486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:38.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:38 smithi006 bash[27292]: cluster 2023-08-27T23:11:36.678528+0000 mgr.y (mgr.14901) 2192 : cluster [DBG] pgmap v2199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:38.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:38 smithi006 bash[27292]: audit 2023-08-27T23:11:38.003340+0000 mon.a (mon.0) 1993 : audit [DBG] from='client.? 172.21.15.6:0/2284605486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:39.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:39 smithi130 bash[51302]: debug 2023-08-27T23:11:39.202+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:39.205966+0000) 2023-08-27T23:11:40.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:40 smithi130 bash[27579]: cluster 2023-08-27T23:11:38.679317+0000 mgr.y (mgr.14901) 2193 : cluster [DBG] pgmap v2200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:40 smithi130 bash[27579]: audit 2023-08-27T23:11:40.436403+0000 mon.c (mon.2) 871 : audit [DBG] from='client.? 172.21.15.6:0/1378055675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:40.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:40 smithi130 bash[51302]: debug 2023-08-27T23:11:40.202+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:40.206250+0000) 2023-08-27T23:11:40.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:40 smithi006 bash[33918]: cluster 2023-08-27T23:11:38.679317+0000 mgr.y (mgr.14901) 2193 : cluster [DBG] pgmap v2200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:40.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:40 smithi006 bash[33918]: audit 2023-08-27T23:11:40.436403+0000 mon.c (mon.2) 871 : audit [DBG] from='client.? 172.21.15.6:0/1378055675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:40 smithi006 bash[27292]: cluster 2023-08-27T23:11:38.679317+0000 mgr.y (mgr.14901) 2193 : cluster [DBG] pgmap v2200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:40 smithi006 bash[27292]: audit 2023-08-27T23:11:40.436403+0000 mon.c (mon.2) 871 : audit [DBG] from='client.? 172.21.15.6:0/1378055675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:11:40.399Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:11:40.400Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:11:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:11:40] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:11:41.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:41 smithi130 bash[51302]: debug 2023-08-27T23:11:41.202+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:41.206532+0000) 2023-08-27T23:11:42.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:41 smithi006 bash[56298]: level=error ts=2023-08-27T23:11:41.964Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:42.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:41 smithi006 bash[56298]: level=warn ts=2023-08-27T23:11:41.965Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:42.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:42 smithi130 bash[27579]: cluster 2023-08-27T23:11:40.680605+0000 mgr.y (mgr.14901) 2194 : cluster [DBG] pgmap v2201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:42.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:42 smithi130 bash[51302]: debug 2023-08-27T23:11:42.202+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:42.206805+0000) 2023-08-27T23:11:42.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:42 smithi006 bash[27292]: cluster 2023-08-27T23:11:40.680605+0000 mgr.y (mgr.14901) 2194 : cluster [DBG] pgmap v2201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:42.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:42 smithi006 bash[33918]: cluster 2023-08-27T23:11:40.680605+0000 mgr.y (mgr.14901) 2194 : cluster [DBG] pgmap v2201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:43.463 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:43 smithi130 bash[51302]: debug 2023-08-27T23:11:43.206+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:43.207099+0000) 2023-08-27T23:11:43.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:43 smithi006 bash[27292]: audit 2023-08-27T23:11:42.851778+0000 mon.c (mon.2) 872 : audit [DBG] from='client.? 172.21.15.6:0/1743817706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:43.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:43 smithi006 bash[33918]: audit 2023-08-27T23:11:42.851778+0000 mon.c (mon.2) 872 : audit [DBG] from='client.? 172.21.15.6:0/1743817706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:43.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:43 smithi130 bash[27579]: audit 2023-08-27T23:11:42.851778+0000 mon.c (mon.2) 872 : audit [DBG] from='client.? 172.21.15.6:0/1743817706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:44.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:11:43.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:44.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:11:43.867Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:44.467 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:44 smithi130 bash[51302]: debug 2023-08-27T23:11:44.206+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:44.207361+0000) 2023-08-27T23:11:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:44 smithi130 bash[27579]: cluster 2023-08-27T23:11:42.681336+0000 mgr.y (mgr.14901) 2195 : cluster [DBG] pgmap v2202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:44.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:44 smithi006 bash[27292]: cluster 2023-08-27T23:11:42.681336+0000 mgr.y (mgr.14901) 2195 : cluster [DBG] pgmap v2202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:44.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:44 smithi006 bash[33918]: cluster 2023-08-27T23:11:42.681336+0000 mgr.y (mgr.14901) 2195 : cluster [DBG] pgmap v2202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:45.476 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:45 smithi130 bash[51302]: debug 2023-08-27T23:11:45.206+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:45.207703+0000) 2023-08-27T23:11:45.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:45 smithi130 bash[27579]: audit 2023-08-27T23:11:45.271233+0000 mon.a (mon.0) 1994 : audit [DBG] from='client.? 172.21.15.6:0/1404527877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:45.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:45 smithi006 bash[27292]: audit 2023-08-27T23:11:45.271233+0000 mon.a (mon.0) 1994 : audit [DBG] from='client.? 172.21.15.6:0/1404527877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:45.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:45 smithi006 bash[33918]: audit 2023-08-27T23:11:45.271233+0000 mon.a (mon.0) 1994 : audit [DBG] from='client.? 172.21.15.6:0/1404527877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:46.479 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:46 smithi130 bash[51302]: debug 2023-08-27T23:11:46.206+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:46.208018+0000) 2023-08-27T23:11:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:46 smithi130 bash[27579]: cluster 2023-08-27T23:11:44.682611+0000 mgr.y (mgr.14901) 2196 : cluster [DBG] pgmap v2203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:46.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:46 smithi006 bash[27292]: cluster 2023-08-27T23:11:44.682611+0000 mgr.y (mgr.14901) 2196 : cluster [DBG] pgmap v2203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:46 smithi006 bash[33918]: cluster 2023-08-27T23:11:44.682611+0000 mgr.y (mgr.14901) 2196 : cluster [DBG] pgmap v2203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:47.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:47 smithi130 bash[51302]: debug 2023-08-27T23:11:47.206+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:47.208290+0000) 2023-08-27T23:11:48.483 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:48 smithi130 bash[51302]: debug 2023-08-27T23:11:48.206+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:48.208572+0000) 2023-08-27T23:11:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:48 smithi130 bash[27579]: cluster 2023-08-27T23:11:46.683276+0000 mgr.y (mgr.14901) 2197 : cluster [DBG] pgmap v2204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:48 smithi130 bash[27579]: audit 2023-08-27T23:11:47.682723+0000 mon.a (mon.0) 1995 : audit [DBG] from='client.? 172.21.15.6:0/2273382096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:48.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:48 smithi006 bash[27292]: cluster 2023-08-27T23:11:46.683276+0000 mgr.y (mgr.14901) 2197 : cluster [DBG] pgmap v2204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:48 smithi006 bash[27292]: audit 2023-08-27T23:11:47.682723+0000 mon.a (mon.0) 1995 : audit [DBG] from='client.? 172.21.15.6:0/2273382096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:48 smithi006 bash[33918]: cluster 2023-08-27T23:11:46.683276+0000 mgr.y (mgr.14901) 2197 : cluster [DBG] pgmap v2204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:48 smithi006 bash[33918]: audit 2023-08-27T23:11:47.682723+0000 mon.a (mon.0) 1995 : audit [DBG] from='client.? 172.21.15.6:0/2273382096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:49.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:49 smithi130 bash[51302]: debug 2023-08-27T23:11:49.206+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:49.208833+0000) 2023-08-27T23:11:50.489 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:50 smithi130 bash[51302]: debug 2023-08-27T23:11:50.206+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:50.209129+0000) 2023-08-27T23:11:50.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:50 smithi006 bash[33918]: cluster 2023-08-27T23:11:48.684238+0000 mgr.y (mgr.14901) 2198 : cluster [DBG] pgmap v2205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:50.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:50 smithi006 bash[33918]: audit 2023-08-27T23:11:50.100581+0000 mon.c (mon.2) 873 : audit [DBG] from='client.? 172.21.15.6:0/1590601314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:50 smithi006 bash[27292]: cluster 2023-08-27T23:11:48.684238+0000 mgr.y (mgr.14901) 2198 : cluster [DBG] pgmap v2205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:50 smithi006 bash[27292]: audit 2023-08-27T23:11:50.100581+0000 mon.c (mon.2) 873 : audit [DBG] from='client.? 172.21.15.6:0/1590601314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:11:50.399Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:11:50.402Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:50 smithi130 bash[27579]: cluster 2023-08-27T23:11:48.684238+0000 mgr.y (mgr.14901) 2198 : cluster [DBG] pgmap v2205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:50 smithi130 bash[27579]: audit 2023-08-27T23:11:50.100581+0000 mon.c (mon.2) 873 : audit [DBG] from='client.? 172.21.15.6:0/1590601314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:11:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:11:50] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:11:51.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:51 smithi130 bash[51302]: debug 2023-08-27T23:11:51.206+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:51.209451+0000) 2023-08-27T23:11:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:51 smithi006 bash[56298]: level=error ts=2023-08-27T23:11:51.964Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:51 smithi006 bash[56298]: level=warn ts=2023-08-27T23:11:51.966Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:52.493 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:52 smithi130 bash[51302]: debug 2023-08-27T23:11:52.206+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:52.209722+0000) 2023-08-27T23:11:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:52 smithi130 bash[27579]: cluster 2023-08-27T23:11:50.685390+0000 mgr.y (mgr.14901) 2199 : cluster [DBG] pgmap v2206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:52 smithi006 bash[27292]: cluster 2023-08-27T23:11:50.685390+0000 mgr.y (mgr.14901) 2199 : cluster [DBG] pgmap v2206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:52 smithi006 bash[33918]: cluster 2023-08-27T23:11:50.685390+0000 mgr.y (mgr.14901) 2199 : cluster [DBG] pgmap v2206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:53.501 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:53 smithi130 bash[51302]: debug 2023-08-27T23:11:53.206+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:53.210047+0000) 2023-08-27T23:11:53.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:53 smithi130 bash[27579]: audit 2023-08-27T23:11:52.510419+0000 mon.c (mon.2) 874 : audit [DBG] from='client.? 172.21.15.6:0/7653261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:53.868 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:53 smithi006 bash[27292]: audit 2023-08-27T23:11:52.510419+0000 mon.c (mon.2) 874 : audit [DBG] from='client.? 172.21.15.6:0/7653261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:53.868 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:53 smithi006 bash[33918]: audit 2023-08-27T23:11:52.510419+0000 mon.c (mon.2) 874 : audit [DBG] from='client.? 172.21.15.6:0/7653261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:54.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:11:53.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:54.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:11:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:11:53.869Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:11:54.505 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:54 smithi130 bash[51302]: debug 2023-08-27T23:11:54.206+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:54.210348+0000) 2023-08-27T23:11:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:54 smithi130 bash[27579]: cluster 2023-08-27T23:11:52.686035+0000 mgr.y (mgr.14901) 2200 : cluster [DBG] pgmap v2207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:54.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:54 smithi006 bash[27292]: cluster 2023-08-27T23:11:52.686035+0000 mgr.y (mgr.14901) 2200 : cluster [DBG] pgmap v2207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:54.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:54 smithi006 bash[33918]: cluster 2023-08-27T23:11:52.686035+0000 mgr.y (mgr.14901) 2200 : cluster [DBG] pgmap v2207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:55.511 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:55 smithi130 bash[51302]: debug 2023-08-27T23:11:55.206+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:55.210615+0000) 2023-08-27T23:11:55.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:55 smithi130 bash[27579]: audit 2023-08-27T23:11:54.932782+0000 mon.a (mon.0) 1996 : audit [DBG] from='client.? 172.21.15.6:0/3567915976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:55.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:55 smithi006 bash[27292]: audit 2023-08-27T23:11:54.932782+0000 mon.a (mon.0) 1996 : audit [DBG] from='client.? 172.21.15.6:0/3567915976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:55.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:55 smithi006 bash[33918]: audit 2023-08-27T23:11:54.932782+0000 mon.a (mon.0) 1996 : audit [DBG] from='client.? 172.21.15.6:0/3567915976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:56.514 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:56 smithi130 bash[51302]: debug 2023-08-27T23:11:56.210+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:56.210902+0000) 2023-08-27T23:11:56.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:56 smithi130 bash[27579]: cluster 2023-08-27T23:11:54.687106+0000 mgr.y (mgr.14901) 2201 : cluster [DBG] pgmap v2208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:56.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:56 smithi006 bash[27292]: cluster 2023-08-27T23:11:54.687106+0000 mgr.y (mgr.14901) 2201 : cluster [DBG] pgmap v2208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:56 smithi006 bash[33918]: cluster 2023-08-27T23:11:54.687106+0000 mgr.y (mgr.14901) 2201 : cluster [DBG] pgmap v2208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:57.523 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:57 smithi130 bash[51302]: debug 2023-08-27T23:11:57.210+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:57.211197+0000) 2023-08-27T23:11:57.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:57 smithi130 bash[27579]: audit 2023-08-27T23:11:57.351392+0000 mon.a (mon.0) 1997 : audit [DBG] from='client.? 172.21.15.6:0/1939035296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:57.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:57 smithi006 bash[27292]: audit 2023-08-27T23:11:57.351392+0000 mon.a (mon.0) 1997 : audit [DBG] from='client.? 172.21.15.6:0/1939035296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:57.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:57 smithi006 bash[33918]: audit 2023-08-27T23:11:57.351392+0000 mon.a (mon.0) 1997 : audit [DBG] from='client.? 172.21.15.6:0/1939035296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:58.525 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:58 smithi130 bash[51302]: debug 2023-08-27T23:11:58.210+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:58.211447+0000) 2023-08-27T23:11:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:11:58 smithi130 bash[27579]: cluster 2023-08-27T23:11:56.687711+0000 mgr.y (mgr.14901) 2202 : cluster [DBG] pgmap v2209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:58.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:11:58 smithi006 bash[27292]: cluster 2023-08-27T23:11:56.687711+0000 mgr.y (mgr.14901) 2202 : cluster [DBG] pgmap v2209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:58.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:11:58 smithi006 bash[33918]: cluster 2023-08-27T23:11:56.687711+0000 mgr.y (mgr.14901) 2202 : cluster [DBG] pgmap v2209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:11:59.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:11:59 smithi130 bash[51302]: debug 2023-08-27T23:11:59.210+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:59.211740+0000) 2023-08-27T23:12:00.531 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:00 smithi130 bash[51302]: debug 2023-08-27T23:12:00.210+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:00.212037+0000) 2023-08-27T23:12:00.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:00 smithi006 bash[27292]: cluster 2023-08-27T23:11:58.688230+0000 mgr.y (mgr.14901) 2203 : cluster [DBG] pgmap v2210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:00.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:00 smithi006 bash[27292]: audit 2023-08-27T23:11:59.766202+0000 mon.a (mon.0) 1998 : audit [DBG] from='client.? 172.21.15.6:0/3055630655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:00 smithi006 bash[33918]: cluster 2023-08-27T23:11:58.688230+0000 mgr.y (mgr.14901) 2203 : cluster [DBG] pgmap v2210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:00 smithi006 bash[33918]: audit 2023-08-27T23:11:59.766202+0000 mon.a (mon.0) 1998 : audit [DBG] from='client.? 172.21.15.6:0/3055630655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:12:00.400Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:12:00.402Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:00 smithi130 bash[27579]: cluster 2023-08-27T23:11:58.688230+0000 mgr.y (mgr.14901) 2203 : cluster [DBG] pgmap v2210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:00 smithi130 bash[27579]: audit 2023-08-27T23:11:59.766202+0000 mon.a (mon.0) 1998 : audit [DBG] from='client.? 172.21.15.6:0/3055630655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:12:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:12:00] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:12:01.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:01 smithi130 bash[51302]: debug 2023-08-27T23:12:01.210+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:01.212399+0000) 2023-08-27T23:12:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:01 smithi006 bash[56298]: level=error ts=2023-08-27T23:12:01.965Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:01 smithi006 bash[56298]: level=warn ts=2023-08-27T23:12:01.967Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:02.536 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:02 smithi130 bash[51302]: debug 2023-08-27T23:12:02.210+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:02.212759+0000) 2023-08-27T23:12:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:02 smithi130 bash[27579]: cluster 2023-08-27T23:12:00.689370+0000 mgr.y (mgr.14901) 2204 : cluster [DBG] pgmap v2211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:02 smithi130 bash[27579]: audit 2023-08-27T23:12:02.184173+0000 mon.c (mon.2) 875 : audit [DBG] from='client.? 172.21.15.6:0/2971292794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:02 smithi006 bash[33918]: cluster 2023-08-27T23:12:00.689370+0000 mgr.y (mgr.14901) 2204 : cluster [DBG] pgmap v2211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:02 smithi006 bash[33918]: audit 2023-08-27T23:12:02.184173+0000 mon.c (mon.2) 875 : audit [DBG] from='client.? 172.21.15.6:0/2971292794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:02 smithi006 bash[27292]: cluster 2023-08-27T23:12:00.689370+0000 mgr.y (mgr.14901) 2204 : cluster [DBG] pgmap v2211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:02 smithi006 bash[27292]: audit 2023-08-27T23:12:02.184173+0000 mon.c (mon.2) 875 : audit [DBG] from='client.? 172.21.15.6:0/2971292794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:03.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:03 smithi130 bash[51302]: debug 2023-08-27T23:12:03.210+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:03.213124+0000) 2023-08-27T23:12:04.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:12:03.866Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:04.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:12:03.868Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:04.541 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:04 smithi130 bash[51302]: debug 2023-08-27T23:12:04.210+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:04.213465+0000) 2023-08-27T23:12:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:04 smithi130 bash[27579]: cluster 2023-08-27T23:12:02.690015+0000 mgr.y (mgr.14901) 2205 : cluster [DBG] pgmap v2212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:04.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:04 smithi006 bash[33918]: cluster 2023-08-27T23:12:02.690015+0000 mgr.y (mgr.14901) 2205 : cluster [DBG] pgmap v2212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:04.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:04 smithi006 bash[27292]: cluster 2023-08-27T23:12:02.690015+0000 mgr.y (mgr.14901) 2205 : cluster [DBG] pgmap v2212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:05.543 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:05 smithi130 bash[51302]: debug 2023-08-27T23:12:05.210+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:05.213848+0000) 2023-08-27T23:12:05.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:05 smithi130 bash[27579]: audit 2023-08-27T23:12:04.604741+0000 mon.a (mon.0) 1999 : audit [DBG] from='client.? 172.21.15.6:0/2720010890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:05.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:05 smithi006 bash[33918]: audit 2023-08-27T23:12:04.604741+0000 mon.a (mon.0) 1999 : audit [DBG] from='client.? 172.21.15.6:0/2720010890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:05.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:05 smithi006 bash[27292]: audit 2023-08-27T23:12:04.604741+0000 mon.a (mon.0) 1999 : audit [DBG] from='client.? 172.21.15.6:0/2720010890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:06.556 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:06 smithi130 bash[51302]: debug 2023-08-27T23:12:06.209+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:06.214169+0000) 2023-08-27T23:12:07.669 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:06 smithi006 bash[33918]: cluster 2023-08-27T23:12:04.691150+0000 mgr.y (mgr.14901) 2206 : cluster [DBG] pgmap v2213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:07.672 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:06 smithi130 bash[27579]: cluster 2023-08-27T23:12:04.691150+0000 mgr.y (mgr.14901) 2206 : cluster [DBG] pgmap v2213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:07.672 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:06 smithi006 bash[27292]: cluster 2023-08-27T23:12:04.691150+0000 mgr.y (mgr.14901) 2206 : cluster [DBG] pgmap v2213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:07.672 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:07 smithi130 bash[51302]: debug 2023-08-27T23:12:07.209+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:07.214485+0000) 2023-08-27T23:12:07.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:07 smithi130 bash[27579]: audit 2023-08-27T23:12:07.027157+0000 mon.a (mon.0) 2000 : audit [DBG] from='client.? 172.21.15.6:0/38866413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:08.133 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:07 smithi006 bash[27292]: audit 2023-08-27T23:12:07.027157+0000 mon.a (mon.0) 2000 : audit [DBG] from='client.? 172.21.15.6:0/38866413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:08.133 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:07 smithi006 bash[33918]: audit 2023-08-27T23:12:07.027157+0000 mon.a (mon.0) 2000 : audit [DBG] from='client.? 172.21.15.6:0/38866413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:08.565 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:08 smithi130 bash[51302]: debug 2023-08-27T23:12:08.209+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:08.214793+0000) 2023-08-27T23:12:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:08 smithi130 bash[27579]: cluster 2023-08-27T23:12:06.691804+0000 mgr.y (mgr.14901) 2207 : cluster [DBG] pgmap v2214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:09.062 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:08 smithi006 bash[33918]: cluster 2023-08-27T23:12:06.691804+0000 mgr.y (mgr.14901) 2207 : cluster [DBG] pgmap v2214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:09.062 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:08 smithi006 bash[27292]: cluster 2023-08-27T23:12:06.691804+0000 mgr.y (mgr.14901) 2207 : cluster [DBG] pgmap v2214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:09.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:09 smithi130 bash[51302]: debug 2023-08-27T23:12:09.213+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:09.215021+0000) 2023-08-27T23:12:10.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:10 smithi130 bash[51302]: debug 2023-08-27T23:12:10.213+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:10.215245+0000) 2023-08-27T23:12:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:10 smithi130 bash[27579]: cluster 2023-08-27T23:12:08.692429+0000 mgr.y (mgr.14901) 2208 : cluster [DBG] pgmap v2215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:10 smithi130 bash[27579]: audit 2023-08-27T23:12:09.447414+0000 mon.c (mon.2) 876 : audit [DBG] from='client.? 172.21.15.6:0/1990854294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:10.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:10 smithi006 bash[27292]: cluster 2023-08-27T23:12:08.692429+0000 mgr.y (mgr.14901) 2208 : cluster [DBG] pgmap v2215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:10.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:10 smithi006 bash[27292]: audit 2023-08-27T23:12:09.447414+0000 mon.c (mon.2) 876 : audit [DBG] from='client.? 172.21.15.6:0/1990854294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:12:10.401Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:12:10.404Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:10 smithi006 bash[33918]: cluster 2023-08-27T23:12:08.692429+0000 mgr.y (mgr.14901) 2208 : cluster [DBG] pgmap v2215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:10 smithi006 bash[33918]: audit 2023-08-27T23:12:09.447414+0000 mon.c (mon.2) 876 : audit [DBG] from='client.? 172.21.15.6:0/1990854294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:12:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:12:10] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:12:11.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:11 smithi130 bash[51302]: debug 2023-08-27T23:12:11.213+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:11.215519+0000) 2023-08-27T23:12:12.412 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:11 smithi006 bash[56298]: level=error ts=2023-08-27T23:12:11.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:12.412 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:11 smithi006 bash[56298]: level=warn ts=2023-08-27T23:12:11.968Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:12.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:12 smithi130 bash[27579]: cluster 2023-08-27T23:12:10.693466+0000 mgr.y (mgr.14901) 2209 : cluster [DBG] pgmap v2216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:12.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:12 smithi130 bash[27579]: audit 2023-08-27T23:12:11.861949+0000 mon.c (mon.2) 877 : audit [DBG] from='client.? 172.21.15.6:0/575831677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:12.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:12 smithi130 bash[51302]: debug 2023-08-27T23:12:12.213+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:12.215863+0000) 2023-08-27T23:12:12.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:12 smithi006 bash[27292]: cluster 2023-08-27T23:12:10.693466+0000 mgr.y (mgr.14901) 2209 : cluster [DBG] pgmap v2216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:12.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:12 smithi006 bash[27292]: audit 2023-08-27T23:12:11.861949+0000 mon.c (mon.2) 877 : audit [DBG] from='client.? 172.21.15.6:0/575831677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:12.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:12 smithi006 bash[33918]: cluster 2023-08-27T23:12:10.693466+0000 mgr.y (mgr.14901) 2209 : cluster [DBG] pgmap v2216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:12.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:12 smithi006 bash[33918]: audit 2023-08-27T23:12:11.861949+0000 mon.c (mon.2) 877 : audit [DBG] from='client.? 172.21.15.6:0/575831677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:13.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:13 smithi130 bash[51302]: debug 2023-08-27T23:12:13.213+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:13.216130+0000) 2023-08-27T23:12:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:12:13.867Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:12:13.870Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:14.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:14 smithi130 bash[51302]: debug 2023-08-27T23:12:14.213+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:14.216391+0000) 2023-08-27T23:12:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:14 smithi130 bash[27579]: cluster 2023-08-27T23:12:12.694073+0000 mgr.y (mgr.14901) 2210 : cluster [DBG] pgmap v2217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:14 smithi130 bash[27579]: audit 2023-08-27T23:12:14.273313+0000 mon.a (mon.0) 2001 : audit [DBG] from='client.? 172.21.15.6:0/3798161704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:14.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:14 smithi006 bash[27292]: cluster 2023-08-27T23:12:12.694073+0000 mgr.y (mgr.14901) 2210 : cluster [DBG] pgmap v2217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:14.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:14 smithi006 bash[27292]: audit 2023-08-27T23:12:14.273313+0000 mon.a (mon.0) 2001 : audit [DBG] from='client.? 172.21.15.6:0/3798161704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:14 smithi006 bash[33918]: cluster 2023-08-27T23:12:12.694073+0000 mgr.y (mgr.14901) 2210 : cluster [DBG] pgmap v2217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:14 smithi006 bash[33918]: audit 2023-08-27T23:12:14.273313+0000 mon.a (mon.0) 2001 : audit [DBG] from='client.? 172.21.15.6:0/3798161704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:15.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:15 smithi130 bash[51302]: debug 2023-08-27T23:12:15.213+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:15.216666+0000) 2023-08-27T23:12:16.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:16 smithi130 bash[27579]: cluster 2023-08-27T23:12:14.695220+0000 mgr.y (mgr.14901) 2211 : cluster [DBG] pgmap v2218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:16.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:16 smithi130 bash[51302]: debug 2023-08-27T23:12:16.213+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:16.216938+0000) 2023-08-27T23:12:16.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:16 smithi006 bash[27292]: cluster 2023-08-27T23:12:14.695220+0000 mgr.y (mgr.14901) 2211 : cluster [DBG] pgmap v2218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:16.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:16 smithi006 bash[33918]: cluster 2023-08-27T23:12:14.695220+0000 mgr.y (mgr.14901) 2211 : cluster [DBG] pgmap v2218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:17.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:17 smithi130 bash[27579]: audit 2023-08-27T23:12:16.698349+0000 mon.c (mon.2) 878 : audit [DBG] from='client.? 172.21.15.6:0/1647574136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:17.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:17 smithi130 bash[51302]: debug 2023-08-27T23:12:17.213+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:17.217213+0000) 2023-08-27T23:12:17.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:17 smithi006 bash[27292]: audit 2023-08-27T23:12:16.698349+0000 mon.c (mon.2) 878 : audit [DBG] from='client.? 172.21.15.6:0/1647574136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:17.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:17 smithi006 bash[33918]: audit 2023-08-27T23:12:16.698349+0000 mon.c (mon.2) 878 : audit [DBG] from='client.? 172.21.15.6:0/1647574136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:18.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:18 smithi130 bash[27579]: cluster 2023-08-27T23:12:16.695649+0000 mgr.y (mgr.14901) 2212 : cluster [DBG] pgmap v2219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:18.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:18 smithi130 bash[51302]: debug 2023-08-27T23:12:18.213+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:18.217503+0000) 2023-08-27T23:12:18.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:18 smithi006 bash[27292]: cluster 2023-08-27T23:12:16.695649+0000 mgr.y (mgr.14901) 2212 : cluster [DBG] pgmap v2219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:18 smithi006 bash[33918]: cluster 2023-08-27T23:12:16.695649+0000 mgr.y (mgr.14901) 2212 : cluster [DBG] pgmap v2219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:19.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:19 smithi130 bash[51302]: debug 2023-08-27T23:12:19.213+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:19.217782+0000) 2023-08-27T23:12:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:19 smithi130 bash[27579]: audit 2023-08-27T23:12:19.119057+0000 mon.a (mon.0) 2002 : audit [DBG] from='client.? 172.21.15.6:0/2546609402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:19.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:19 smithi006 bash[27292]: audit 2023-08-27T23:12:19.119057+0000 mon.a (mon.0) 2002 : audit [DBG] from='client.? 172.21.15.6:0/2546609402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:19.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:19 smithi006 bash[33918]: audit 2023-08-27T23:12:19.119057+0000 mon.a (mon.0) 2002 : audit [DBG] from='client.? 172.21.15.6:0/2546609402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:20.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:20 smithi130 bash[51302]: debug 2023-08-27T23:12:20.213+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:20.218084+0000) 2023-08-27T23:12:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:20 smithi130 bash[27579]: cluster 2023-08-27T23:12:18.696355+0000 mgr.y (mgr.14901) 2213 : cluster [DBG] pgmap v2220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:20 smithi130 bash[27579]: audit 2023-08-27T23:12:19.455677+0000 mon.b (mon.1) 332 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:12:20.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:20 smithi006 bash[33918]: cluster 2023-08-27T23:12:18.696355+0000 mgr.y (mgr.14901) 2213 : cluster [DBG] pgmap v2220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:20.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:20 smithi006 bash[33918]: audit 2023-08-27T23:12:19.455677+0000 mon.b (mon.1) 332 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:12:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:12:20.401Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:12:20.403Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:20 smithi006 bash[27292]: cluster 2023-08-27T23:12:18.696355+0000 mgr.y (mgr.14901) 2213 : cluster [DBG] pgmap v2220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:20 smithi006 bash[27292]: audit 2023-08-27T23:12:19.455677+0000 mon.b (mon.1) 332 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:12:21.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:12:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:12:20] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:12:21.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:21 smithi130 bash[51302]: debug 2023-08-27T23:12:21.217+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:21.218366+0000) 2023-08-27T23:12:22.451 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:21 smithi006 bash[56298]: level=error ts=2023-08-27T23:12:21.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:22.452 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:21 smithi006 bash[56298]: level=warn ts=2023-08-27T23:12:21.968Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:22.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:22 smithi130 bash[27579]: cluster 2023-08-27T23:12:20.697458+0000 mgr.y (mgr.14901) 2214 : cluster [DBG] pgmap v2221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:22.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:22 smithi130 bash[27579]: audit 2023-08-27T23:12:21.530400+0000 mon.c (mon.2) 879 : audit [DBG] from='client.? 172.21.15.6:0/3244363779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:22.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:22 smithi130 bash[51302]: debug 2023-08-27T23:12:22.213+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:22.218626+0000) 2023-08-27T23:12:22.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:22 smithi006 bash[27292]: cluster 2023-08-27T23:12:20.697458+0000 mgr.y (mgr.14901) 2214 : cluster [DBG] pgmap v2221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:22.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:22 smithi006 bash[27292]: audit 2023-08-27T23:12:21.530400+0000 mon.c (mon.2) 879 : audit [DBG] from='client.? 172.21.15.6:0/3244363779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:22 smithi006 bash[33918]: cluster 2023-08-27T23:12:20.697458+0000 mgr.y (mgr.14901) 2214 : cluster [DBG] pgmap v2221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:22 smithi006 bash[33918]: audit 2023-08-27T23:12:21.530400+0000 mon.c (mon.2) 879 : audit [DBG] from='client.? 172.21.15.6:0/3244363779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:23.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:23 smithi130 bash[51302]: debug 2023-08-27T23:12:23.217+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:23.218918+0000) 2023-08-27T23:12:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:12:23.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:12:23.870Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:24.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:24 smithi130 bash[27579]: cluster 2023-08-27T23:12:22.698056+0000 mgr.y (mgr.14901) 2215 : cluster [DBG] pgmap v2222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:24 smithi130 bash[27579]: audit 2023-08-27T23:12:23.963575+0000 mon.a (mon.0) 2003 : audit [DBG] from='client.? 172.21.15.6:0/2569313772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:24.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:24 smithi130 bash[51302]: debug 2023-08-27T23:12:24.217+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:24.219188+0000) 2023-08-27T23:12:24.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:24 smithi006 bash[27292]: cluster 2023-08-27T23:12:22.698056+0000 mgr.y (mgr.14901) 2215 : cluster [DBG] pgmap v2222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:24.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:24 smithi006 bash[27292]: audit 2023-08-27T23:12:23.963575+0000 mon.a (mon.0) 2003 : audit [DBG] from='client.? 172.21.15.6:0/2569313772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:24.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:24 smithi006 bash[33918]: cluster 2023-08-27T23:12:22.698056+0000 mgr.y (mgr.14901) 2215 : cluster [DBG] pgmap v2222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:24.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:24 smithi006 bash[33918]: audit 2023-08-27T23:12:23.963575+0000 mon.a (mon.0) 2003 : audit [DBG] from='client.? 172.21.15.6:0/2569313772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:25.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:25 smithi130 bash[51302]: debug 2023-08-27T23:12:25.217+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:25.219400+0000) 2023-08-27T23:12:26.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:26 smithi130 bash[51302]: debug 2023-08-27T23:12:26.217+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:26.219599+0000) 2023-08-27T23:12:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:26 smithi130 bash[27579]: cluster 2023-08-27T23:12:24.698990+0000 mgr.y (mgr.14901) 2216 : cluster [DBG] pgmap v2223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:26 smithi130 bash[27579]: audit 2023-08-27T23:12:25.354198+0000 mon.a (mon.0) 2004 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:12:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:26 smithi130 bash[27579]: audit 2023-08-27T23:12:25.806230+0000 mon.a (mon.0) 2005 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:12:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:26 smithi130 bash[27579]: audit 2023-08-27T23:12:26.100725+0000 mon.b (mon.1) 333 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:12:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:26 smithi130 bash[27579]: audit 2023-08-27T23:12:26.102721+0000 mon.b (mon.1) 334 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:12:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:26 smithi130 bash[27579]: audit 2023-08-27T23:12:26.110664+0000 mon.a (mon.0) 2006 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:12:26.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:26 smithi006 bash[27292]: cluster 2023-08-27T23:12:24.698990+0000 mgr.y (mgr.14901) 2216 : cluster [DBG] pgmap v2223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:26.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:26 smithi006 bash[27292]: audit 2023-08-27T23:12:25.354198+0000 mon.a (mon.0) 2004 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:12:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:26 smithi006 bash[27292]: audit 2023-08-27T23:12:25.806230+0000 mon.a (mon.0) 2005 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:12:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:26 smithi006 bash[27292]: audit 2023-08-27T23:12:26.100725+0000 mon.b (mon.1) 333 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:12:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:26 smithi006 bash[27292]: audit 2023-08-27T23:12:26.102721+0000 mon.b (mon.1) 334 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:12:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:26 smithi006 bash[27292]: audit 2023-08-27T23:12:26.110664+0000 mon.a (mon.0) 2006 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:12:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:26 smithi006 bash[33918]: cluster 2023-08-27T23:12:24.698990+0000 mgr.y (mgr.14901) 2216 : cluster [DBG] pgmap v2223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:26 smithi006 bash[33918]: audit 2023-08-27T23:12:25.354198+0000 mon.a (mon.0) 2004 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:12:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:26 smithi006 bash[33918]: audit 2023-08-27T23:12:25.806230+0000 mon.a (mon.0) 2005 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:12:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:26 smithi006 bash[33918]: audit 2023-08-27T23:12:26.100725+0000 mon.b (mon.1) 333 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:12:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:26 smithi006 bash[33918]: audit 2023-08-27T23:12:26.102721+0000 mon.b (mon.1) 334 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:12:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:26 smithi006 bash[33918]: audit 2023-08-27T23:12:26.110664+0000 mon.a (mon.0) 2006 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:12:27.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:27 smithi130 bash[27579]: audit 2023-08-27T23:12:26.381397+0000 mon.a (mon.0) 2007 : audit [DBG] from='client.? 172.21.15.6:0/1624283921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:27.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:27 smithi130 bash[51302]: debug 2023-08-27T23:12:27.217+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:27.219890+0000) 2023-08-27T23:12:27.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:27 smithi006 bash[27292]: audit 2023-08-27T23:12:26.381397+0000 mon.a (mon.0) 2007 : audit [DBG] from='client.? 172.21.15.6:0/1624283921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:27.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:27 smithi006 bash[33918]: audit 2023-08-27T23:12:26.381397+0000 mon.a (mon.0) 2007 : audit [DBG] from='client.? 172.21.15.6:0/1624283921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:28.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:28 smithi130 bash[27579]: cluster 2023-08-27T23:12:26.699609+0000 mgr.y (mgr.14901) 2217 : cluster [DBG] pgmap v2224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:28.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:28 smithi130 bash[51302]: debug 2023-08-27T23:12:28.217+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:28.220200+0000) 2023-08-27T23:12:28.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:28 smithi006 bash[27292]: cluster 2023-08-27T23:12:26.699609+0000 mgr.y (mgr.14901) 2217 : cluster [DBG] pgmap v2224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:28.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:28 smithi006 bash[33918]: cluster 2023-08-27T23:12:26.699609+0000 mgr.y (mgr.14901) 2217 : cluster [DBG] pgmap v2224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:29.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:29 smithi130 bash[27579]: audit 2023-08-27T23:12:28.789478+0000 mon.a (mon.0) 2008 : audit [DBG] from='client.? 172.21.15.6:0/2340774557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:29.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:29 smithi130 bash[51302]: debug 2023-08-27T23:12:29.217+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:29.220499+0000) 2023-08-27T23:12:29.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:29 smithi006 bash[27292]: audit 2023-08-27T23:12:28.789478+0000 mon.a (mon.0) 2008 : audit [DBG] from='client.? 172.21.15.6:0/2340774557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:29.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:29 smithi006 bash[33918]: audit 2023-08-27T23:12:28.789478+0000 mon.a (mon.0) 2008 : audit [DBG] from='client.? 172.21.15.6:0/2340774557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:30.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:30 smithi130 bash[27579]: cluster 2023-08-27T23:12:28.700246+0000 mgr.y (mgr.14901) 2218 : cluster [DBG] pgmap v2225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:30.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:30 smithi130 bash[51302]: debug 2023-08-27T23:12:30.217+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:30.220782+0000) 2023-08-27T23:12:30.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:30 smithi006 bash[33918]: cluster 2023-08-27T23:12:28.700246+0000 mgr.y (mgr.14901) 2218 : cluster [DBG] pgmap v2225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:30.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:30 smithi006 bash[27292]: cluster 2023-08-27T23:12:28.700246+0000 mgr.y (mgr.14901) 2218 : cluster [DBG] pgmap v2225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:12:30.402Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:12:30.405Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:12:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:12:30] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:12:31.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:31 smithi130 bash[27579]: audit 2023-08-27T23:12:31.204087+0000 mon.a (mon.0) 2009 : audit [DBG] from='client.? 172.21.15.6:0/2397291602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:31.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:31 smithi130 bash[51302]: debug 2023-08-27T23:12:31.217+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:31.221062+0000) 2023-08-27T23:12:31.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:31 smithi006 bash[27292]: audit 2023-08-27T23:12:31.204087+0000 mon.a (mon.0) 2009 : audit [DBG] from='client.? 172.21.15.6:0/2397291602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:31.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:31 smithi006 bash[33918]: audit 2023-08-27T23:12:31.204087+0000 mon.a (mon.0) 2009 : audit [DBG] from='client.? 172.21.15.6:0/2397291602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:32.422 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:31 smithi006 bash[56298]: level=error ts=2023-08-27T23:12:31.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:32.423 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:31 smithi006 bash[56298]: level=warn ts=2023-08-27T23:12:31.968Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:32.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:32 smithi130 bash[27579]: cluster 2023-08-27T23:12:30.701382+0000 mgr.y (mgr.14901) 2219 : cluster [DBG] pgmap v2226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:32.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:32 smithi130 bash[51302]: debug 2023-08-27T23:12:32.217+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:32.221365+0000) 2023-08-27T23:12:32.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:32 smithi006 bash[33918]: cluster 2023-08-27T23:12:30.701382+0000 mgr.y (mgr.14901) 2219 : cluster [DBG] pgmap v2226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:32.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:32 smithi006 bash[27292]: cluster 2023-08-27T23:12:30.701382+0000 mgr.y (mgr.14901) 2219 : cluster [DBG] pgmap v2226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:33.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:33 smithi130 bash[51302]: debug 2023-08-27T23:12:33.217+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:33.221639+0000) 2023-08-27T23:12:34.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:12:33.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:34.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:12:33.870Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:34.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:34 smithi130 bash[27579]: cluster 2023-08-27T23:12:32.702042+0000 mgr.y (mgr.14901) 2220 : cluster [DBG] pgmap v2227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:34 smithi130 bash[27579]: audit 2023-08-27T23:12:33.627370+0000 mon.c (mon.2) 880 : audit [DBG] from='client.? 172.21.15.6:0/4266630252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:34.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:34 smithi130 bash[51302]: debug 2023-08-27T23:12:34.217+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:34.221944+0000) 2023-08-27T23:12:34.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:34 smithi006 bash[27292]: cluster 2023-08-27T23:12:32.702042+0000 mgr.y (mgr.14901) 2220 : cluster [DBG] pgmap v2227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:34.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:34 smithi006 bash[27292]: audit 2023-08-27T23:12:33.627370+0000 mon.c (mon.2) 880 : audit [DBG] from='client.? 172.21.15.6:0/4266630252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:34.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:34 smithi006 bash[33918]: cluster 2023-08-27T23:12:32.702042+0000 mgr.y (mgr.14901) 2220 : cluster [DBG] pgmap v2227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:34 smithi006 bash[33918]: audit 2023-08-27T23:12:33.627370+0000 mon.c (mon.2) 880 : audit [DBG] from='client.? 172.21.15.6:0/4266630252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:35.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:35 smithi130 bash[51302]: debug 2023-08-27T23:12:35.217+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:35.222244+0000) 2023-08-27T23:12:36.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:36 smithi130 bash[27579]: cluster 2023-08-27T23:12:34.703157+0000 mgr.y (mgr.14901) 2221 : cluster [DBG] pgmap v2228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:36 smithi130 bash[27579]: audit 2023-08-27T23:12:36.040190+0000 mon.a (mon.0) 2010 : audit [DBG] from='client.? 172.21.15.6:0/2167244397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:36.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:36 smithi130 bash[51302]: debug 2023-08-27T23:12:36.221+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:36.222583+0000) 2023-08-27T23:12:36.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:36 smithi006 bash[33918]: cluster 2023-08-27T23:12:34.703157+0000 mgr.y (mgr.14901) 2221 : cluster [DBG] pgmap v2228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:36.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:36 smithi006 bash[33918]: audit 2023-08-27T23:12:36.040190+0000 mon.a (mon.0) 2010 : audit [DBG] from='client.? 172.21.15.6:0/2167244397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:36 smithi006 bash[27292]: cluster 2023-08-27T23:12:34.703157+0000 mgr.y (mgr.14901) 2221 : cluster [DBG] pgmap v2228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:36 smithi006 bash[27292]: audit 2023-08-27T23:12:36.040190+0000 mon.a (mon.0) 2010 : audit [DBG] from='client.? 172.21.15.6:0/2167244397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:37.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:37 smithi130 bash[51302]: debug 2023-08-27T23:12:37.221+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:37.222895+0000) 2023-08-27T23:12:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:38 smithi130 bash[27579]: cluster 2023-08-27T23:12:36.703789+0000 mgr.y (mgr.14901) 2222 : cluster [DBG] pgmap v2229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:38.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:38 smithi130 bash[51302]: debug 2023-08-27T23:12:38.221+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:38.223137+0000) 2023-08-27T23:12:38.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:38 smithi006 bash[33918]: cluster 2023-08-27T23:12:36.703789+0000 mgr.y (mgr.14901) 2222 : cluster [DBG] pgmap v2229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:38.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:38 smithi006 bash[27292]: cluster 2023-08-27T23:12:36.703789+0000 mgr.y (mgr.14901) 2222 : cluster [DBG] pgmap v2229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:39.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:39 smithi130 bash[51302]: debug 2023-08-27T23:12:39.221+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:39.223400+0000) 2023-08-27T23:12:39.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:39 smithi130 bash[27579]: audit 2023-08-27T23:12:38.461440+0000 mon.c (mon.2) 881 : audit [DBG] from='client.? 172.21.15.6:0/1939584331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:39.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:39 smithi006 bash[27292]: audit 2023-08-27T23:12:38.461440+0000 mon.c (mon.2) 881 : audit [DBG] from='client.? 172.21.15.6:0/1939584331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:39.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:39 smithi006 bash[33918]: audit 2023-08-27T23:12:38.461440+0000 mon.c (mon.2) 881 : audit [DBG] from='client.? 172.21.15.6:0/1939584331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:40.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:40 smithi130 bash[27579]: cluster 2023-08-27T23:12:38.704432+0000 mgr.y (mgr.14901) 2223 : cluster [DBG] pgmap v2230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:40.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:40 smithi130 bash[51302]: debug 2023-08-27T23:12:40.221+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:40.223692+0000) 2023-08-27T23:12:40.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:40 smithi006 bash[27292]: cluster 2023-08-27T23:12:38.704432+0000 mgr.y (mgr.14901) 2223 : cluster [DBG] pgmap v2230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:40.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:40 smithi006 bash[33918]: cluster 2023-08-27T23:12:38.704432+0000 mgr.y (mgr.14901) 2223 : cluster [DBG] pgmap v2230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:12:40.403Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:12:40.405Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:12:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:12:40] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:12:41.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:41 smithi130 bash[27579]: audit 2023-08-27T23:12:40.875872+0000 mon.c (mon.2) 882 : audit [DBG] from='client.? 172.21.15.6:0/971948316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:41.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:41 smithi130 bash[51302]: debug 2023-08-27T23:12:41.221+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:41.223994+0000) 2023-08-27T23:12:41.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:41 smithi006 bash[33918]: audit 2023-08-27T23:12:40.875872+0000 mon.c (mon.2) 882 : audit [DBG] from='client.? 172.21.15.6:0/971948316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:41.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:41 smithi006 bash[27292]: audit 2023-08-27T23:12:40.875872+0000 mon.c (mon.2) 882 : audit [DBG] from='client.? 172.21.15.6:0/971948316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:42.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:41 smithi006 bash[56298]: level=error ts=2023-08-27T23:12:41.967Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:42.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:41 smithi006 bash[56298]: level=warn ts=2023-08-27T23:12:41.969Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:42.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:42 smithi130 bash[27579]: cluster 2023-08-27T23:12:40.705449+0000 mgr.y (mgr.14901) 2224 : cluster [DBG] pgmap v2231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:42.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:42 smithi130 bash[51302]: debug 2023-08-27T23:12:42.221+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:42.224258+0000) 2023-08-27T23:12:42.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:42 smithi006 bash[27292]: cluster 2023-08-27T23:12:40.705449+0000 mgr.y (mgr.14901) 2224 : cluster [DBG] pgmap v2231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:42.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:42 smithi006 bash[33918]: cluster 2023-08-27T23:12:40.705449+0000 mgr.y (mgr.14901) 2224 : cluster [DBG] pgmap v2231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:43.476 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:43 smithi130 bash[51302]: debug 2023-08-27T23:12:43.221+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:43.224536+0000) 2023-08-27T23:12:43.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:43 smithi130 bash[27579]: audit 2023-08-27T23:12:43.288380+0000 mon.a (mon.0) 2011 : audit [DBG] from='client.? 172.21.15.6:0/2472514761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:43.871 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:43 smithi006 bash[33918]: audit 2023-08-27T23:12:43.288380+0000 mon.a (mon.0) 2011 : audit [DBG] from='client.? 172.21.15.6:0/2472514761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:43.871 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:43 smithi006 bash[27292]: audit 2023-08-27T23:12:43.288380+0000 mon.a (mon.0) 2011 : audit [DBG] from='client.? 172.21.15.6:0/2472514761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:44.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:12:43.869Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:44.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:12:43.872Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:44.479 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:44 smithi130 bash[51302]: debug 2023-08-27T23:12:44.221+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:44.224858+0000) 2023-08-27T23:12:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:44 smithi130 bash[27579]: cluster 2023-08-27T23:12:42.706048+0000 mgr.y (mgr.14901) 2225 : cluster [DBG] pgmap v2232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:44.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:44 smithi006 bash[27292]: cluster 2023-08-27T23:12:42.706048+0000 mgr.y (mgr.14901) 2225 : cluster [DBG] pgmap v2232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:44.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:44 smithi006 bash[33918]: cluster 2023-08-27T23:12:42.706048+0000 mgr.y (mgr.14901) 2225 : cluster [DBG] pgmap v2232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:45.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:45 smithi130 bash[51302]: debug 2023-08-27T23:12:45.221+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:45.225124+0000) 2023-08-27T23:12:46.484 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:46 smithi130 bash[51302]: debug 2023-08-27T23:12:46.221+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:46.225392+0000) 2023-08-27T23:12:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:46 smithi130 bash[27579]: cluster 2023-08-27T23:12:44.707136+0000 mgr.y (mgr.14901) 2226 : cluster [DBG] pgmap v2233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:46 smithi130 bash[27579]: audit 2023-08-27T23:12:45.704860+0000 mon.a (mon.0) 2012 : audit [DBG] from='client.? 172.21.15.6:0/1461677584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:46 smithi006 bash[33918]: cluster 2023-08-27T23:12:44.707136+0000 mgr.y (mgr.14901) 2226 : cluster [DBG] pgmap v2233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:46 smithi006 bash[33918]: audit 2023-08-27T23:12:45.704860+0000 mon.a (mon.0) 2012 : audit [DBG] from='client.? 172.21.15.6:0/1461677584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:46 smithi006 bash[27292]: cluster 2023-08-27T23:12:44.707136+0000 mgr.y (mgr.14901) 2226 : cluster [DBG] pgmap v2233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:46 smithi006 bash[27292]: audit 2023-08-27T23:12:45.704860+0000 mon.a (mon.0) 2012 : audit [DBG] from='client.? 172.21.15.6:0/1461677584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:47.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:47 smithi130 bash[51302]: debug 2023-08-27T23:12:47.221+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:47.225648+0000) 2023-08-27T23:12:48.489 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:48 smithi130 bash[51302]: debug 2023-08-27T23:12:48.221+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:48.225929+0000) 2023-08-27T23:12:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:48 smithi130 bash[27579]: cluster 2023-08-27T23:12:46.707751+0000 mgr.y (mgr.14901) 2227 : cluster [DBG] pgmap v2234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:48 smithi130 bash[27579]: audit 2023-08-27T23:12:48.119431+0000 mon.a (mon.0) 2013 : audit [DBG] from='client.? 172.21.15.6:0/423046542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:48.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:48 smithi006 bash[27292]: cluster 2023-08-27T23:12:46.707751+0000 mgr.y (mgr.14901) 2227 : cluster [DBG] pgmap v2234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:48.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:48 smithi006 bash[27292]: audit 2023-08-27T23:12:48.119431+0000 mon.a (mon.0) 2013 : audit [DBG] from='client.? 172.21.15.6:0/423046542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:48 smithi006 bash[33918]: cluster 2023-08-27T23:12:46.707751+0000 mgr.y (mgr.14901) 2227 : cluster [DBG] pgmap v2234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:48 smithi006 bash[33918]: audit 2023-08-27T23:12:48.119431+0000 mon.a (mon.0) 2013 : audit [DBG] from='client.? 172.21.15.6:0/423046542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:49.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:49 smithi130 bash[51302]: debug 2023-08-27T23:12:49.221+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:49.226216+0000) 2023-08-27T23:12:50.494 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:50 smithi130 bash[51302]: debug 2023-08-27T23:12:50.221+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:50.226526+0000) 2023-08-27T23:12:50.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:50 smithi006 bash[33918]: cluster 2023-08-27T23:12:48.708413+0000 mgr.y (mgr.14901) 2228 : cluster [DBG] pgmap v2235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:50.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:50 smithi006 bash[27292]: cluster 2023-08-27T23:12:48.708413+0000 mgr.y (mgr.14901) 2228 : cluster [DBG] pgmap v2235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:12:50.403Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:12:50.405Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:50 smithi130 bash[27579]: cluster 2023-08-27T23:12:48.708413+0000 mgr.y (mgr.14901) 2228 : cluster [DBG] pgmap v2235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:12:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:12:50] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:12:51.497 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:51 smithi130 bash[51302]: debug 2023-08-27T23:12:51.225+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:51.226817+0000) 2023-08-27T23:12:51.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:51 smithi130 bash[27579]: audit 2023-08-27T23:12:50.550391+0000 mon.a (mon.0) 2014 : audit [DBG] from='client.? 172.21.15.6:0/85781808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:51.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:51 smithi006 bash[33918]: audit 2023-08-27T23:12:50.550391+0000 mon.a (mon.0) 2014 : audit [DBG] from='client.? 172.21.15.6:0/85781808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:51.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:51 smithi006 bash[27292]: audit 2023-08-27T23:12:50.550391+0000 mon.a (mon.0) 2014 : audit [DBG] from='client.? 172.21.15.6:0/85781808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:51 smithi006 bash[56298]: level=error ts=2023-08-27T23:12:51.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:51 smithi006 bash[56298]: level=warn ts=2023-08-27T23:12:51.969Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:52.506 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:52 smithi130 bash[51302]: debug 2023-08-27T23:12:52.225+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:52.227035+0000) 2023-08-27T23:12:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:52 smithi130 bash[27579]: cluster 2023-08-27T23:12:50.709572+0000 mgr.y (mgr.14901) 2229 : cluster [DBG] pgmap v2236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:52 smithi006 bash[27292]: cluster 2023-08-27T23:12:50.709572+0000 mgr.y (mgr.14901) 2229 : cluster [DBG] pgmap v2236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:52 smithi006 bash[33918]: cluster 2023-08-27T23:12:50.709572+0000 mgr.y (mgr.14901) 2229 : cluster [DBG] pgmap v2236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:53.514 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:53 smithi130 bash[51302]: debug 2023-08-27T23:12:53.225+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:53.227314+0000) 2023-08-27T23:12:53.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:53 smithi130 bash[27579]: audit 2023-08-27T23:12:52.959406+0000 mon.a (mon.0) 2015 : audit [DBG] from='client.? 172.21.15.6:0/977233375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:53.871 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:53 smithi006 bash[33918]: audit 2023-08-27T23:12:52.959406+0000 mon.a (mon.0) 2015 : audit [DBG] from='client.? 172.21.15.6:0/977233375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:53.872 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:53 smithi006 bash[27292]: audit 2023-08-27T23:12:52.959406+0000 mon.a (mon.0) 2015 : audit [DBG] from='client.? 172.21.15.6:0/977233375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:54.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:12:53.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:54.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:12:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:12:53.872Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:12:54.517 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:54 smithi130 bash[51302]: debug 2023-08-27T23:12:54.225+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:54.227541+0000) 2023-08-27T23:12:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:54 smithi130 bash[27579]: cluster 2023-08-27T23:12:52.710243+0000 mgr.y (mgr.14901) 2230 : cluster [DBG] pgmap v2237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:54.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:54 smithi006 bash[27292]: cluster 2023-08-27T23:12:52.710243+0000 mgr.y (mgr.14901) 2230 : cluster [DBG] pgmap v2237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:54.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:54 smithi006 bash[33918]: cluster 2023-08-27T23:12:52.710243+0000 mgr.y (mgr.14901) 2230 : cluster [DBG] pgmap v2237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:55.528 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:55 smithi130 bash[51302]: debug 2023-08-27T23:12:55.225+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:55.227843+0000) 2023-08-27T23:12:55.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:55 smithi130 bash[27579]: audit 2023-08-27T23:12:55.383060+0000 mon.a (mon.0) 2016 : audit [DBG] from='client.? 172.21.15.6:0/1134995578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:55.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:55 smithi006 bash[33918]: audit 2023-08-27T23:12:55.383060+0000 mon.a (mon.0) 2016 : audit [DBG] from='client.? 172.21.15.6:0/1134995578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:55.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:55 smithi006 bash[27292]: audit 2023-08-27T23:12:55.383060+0000 mon.a (mon.0) 2016 : audit [DBG] from='client.? 172.21.15.6:0/1134995578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:56.532 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:56 smithi130 bash[51302]: debug 2023-08-27T23:12:56.225+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:56.228121+0000) 2023-08-27T23:12:56.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:56 smithi130 bash[27579]: cluster 2023-08-27T23:12:54.711269+0000 mgr.y (mgr.14901) 2231 : cluster [DBG] pgmap v2238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:56.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:56 smithi006 bash[27292]: cluster 2023-08-27T23:12:54.711269+0000 mgr.y (mgr.14901) 2231 : cluster [DBG] pgmap v2238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:56 smithi006 bash[33918]: cluster 2023-08-27T23:12:54.711269+0000 mgr.y (mgr.14901) 2231 : cluster [DBG] pgmap v2238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:57.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:57 smithi130 bash[51302]: debug 2023-08-27T23:12:57.225+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:57.228415+0000) 2023-08-27T23:12:58.536 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:58 smithi130 bash[51302]: debug 2023-08-27T23:12:58.225+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:58.228678+0000) 2023-08-27T23:12:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:58 smithi130 bash[27579]: cluster 2023-08-27T23:12:56.711914+0000 mgr.y (mgr.14901) 2232 : cluster [DBG] pgmap v2239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:12:58 smithi130 bash[27579]: audit 2023-08-27T23:12:57.805452+0000 mon.a (mon.0) 2017 : audit [DBG] from='client.? 172.21.15.6:0/3341137134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:58.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:58 smithi006 bash[33918]: cluster 2023-08-27T23:12:56.711914+0000 mgr.y (mgr.14901) 2232 : cluster [DBG] pgmap v2239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:58.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:12:58 smithi006 bash[33918]: audit 2023-08-27T23:12:57.805452+0000 mon.a (mon.0) 2017 : audit [DBG] from='client.? 172.21.15.6:0/3341137134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:58 smithi006 bash[27292]: cluster 2023-08-27T23:12:56.711914+0000 mgr.y (mgr.14901) 2232 : cluster [DBG] pgmap v2239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:12:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:12:58 smithi006 bash[27292]: audit 2023-08-27T23:12:57.805452+0000 mon.a (mon.0) 2017 : audit [DBG] from='client.? 172.21.15.6:0/3341137134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:59.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:12:59 smithi130 bash[51302]: debug 2023-08-27T23:12:59.225+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:59.228952+0000) 2023-08-27T23:13:00.542 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:00 smithi130 bash[51302]: debug 2023-08-27T23:13:00.225+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:00.229254+0000) 2023-08-27T23:13:00.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:13:00.404Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:00.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:13:00.406Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:00 smithi006 bash[27292]: cluster 2023-08-27T23:12:58.712555+0000 mgr.y (mgr.14901) 2233 : cluster [DBG] pgmap v2240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:00 smithi006 bash[27292]: audit 2023-08-27T23:13:00.245350+0000 mon.c (mon.2) 883 : audit [DBG] from='client.? 172.21.15.6:0/278030624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:00 smithi006 bash[33918]: cluster 2023-08-27T23:12:58.712555+0000 mgr.y (mgr.14901) 2233 : cluster [DBG] pgmap v2240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:00 smithi006 bash[33918]: audit 2023-08-27T23:13:00.245350+0000 mon.c (mon.2) 883 : audit [DBG] from='client.? 172.21.15.6:0/278030624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:00 smithi130 bash[27579]: cluster 2023-08-27T23:12:58.712555+0000 mgr.y (mgr.14901) 2233 : cluster [DBG] pgmap v2240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:00 smithi130 bash[27579]: audit 2023-08-27T23:13:00.245350+0000 mon.c (mon.2) 883 : audit [DBG] from='client.? 172.21.15.6:0/278030624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:13:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:13:00] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:13:01.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:01 smithi130 bash[51302]: debug 2023-08-27T23:13:01.225+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:01.229566+0000) 2023-08-27T23:13:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:01 smithi006 bash[56298]: level=error ts=2023-08-27T23:13:01.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:01 smithi006 bash[56298]: level=warn ts=2023-08-27T23:13:01.970Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:02.546 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:02 smithi130 bash[51302]: debug 2023-08-27T23:13:02.225+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:02.229830+0000) 2023-08-27T23:13:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:02 smithi130 bash[27579]: cluster 2023-08-27T23:13:00.713666+0000 mgr.y (mgr.14901) 2234 : cluster [DBG] pgmap v2241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:02 smithi006 bash[27292]: cluster 2023-08-27T23:13:00.713666+0000 mgr.y (mgr.14901) 2234 : cluster [DBG] pgmap v2241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:02 smithi006 bash[33918]: cluster 2023-08-27T23:13:00.713666+0000 mgr.y (mgr.14901) 2234 : cluster [DBG] pgmap v2241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:03.554 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:03 smithi130 bash[51302]: debug 2023-08-27T23:13:03.224+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:03.230119+0000) 2023-08-27T23:13:03.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:03 smithi130 bash[27579]: audit 2023-08-27T23:13:02.662242+0000 mon.c (mon.2) 884 : audit [DBG] from='client.? 172.21.15.6:0/2975667517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:03.872 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:03 smithi006 bash[33918]: audit 2023-08-27T23:13:02.662242+0000 mon.c (mon.2) 884 : audit [DBG] from='client.? 172.21.15.6:0/2975667517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:03.873 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:03 smithi006 bash[27292]: audit 2023-08-27T23:13:02.662242+0000 mon.c (mon.2) 884 : audit [DBG] from='client.? 172.21.15.6:0/2975667517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:04.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:13:03.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:04.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:13:03.873Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:04.557 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:04 smithi130 bash[51302]: debug 2023-08-27T23:13:04.224+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:04.230398+0000) 2023-08-27T23:13:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:04 smithi130 bash[27579]: cluster 2023-08-27T23:13:02.714341+0000 mgr.y (mgr.14901) 2235 : cluster [DBG] pgmap v2242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:04.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:04 smithi006 bash[33918]: cluster 2023-08-27T23:13:02.714341+0000 mgr.y (mgr.14901) 2235 : cluster [DBG] pgmap v2242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:04.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:04 smithi006 bash[27292]: cluster 2023-08-27T23:13:02.714341+0000 mgr.y (mgr.14901) 2235 : cluster [DBG] pgmap v2242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:05.565 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:05 smithi130 bash[51302]: debug 2023-08-27T23:13:05.224+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:05.230761+0000) 2023-08-27T23:13:05.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:05 smithi130 bash[27579]: audit 2023-08-27T23:13:05.075145+0000 mon.a (mon.0) 2018 : audit [DBG] from='client.? 172.21.15.6:0/705311233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:05.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:05 smithi006 bash[33918]: audit 2023-08-27T23:13:05.075145+0000 mon.a (mon.0) 2018 : audit [DBG] from='client.? 172.21.15.6:0/705311233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:05.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:05 smithi006 bash[27292]: audit 2023-08-27T23:13:05.075145+0000 mon.a (mon.0) 2018 : audit [DBG] from='client.? 172.21.15.6:0/705311233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:06.568 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:06 smithi130 bash[51302]: debug 2023-08-27T23:13:06.228+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:06.231032+0000) 2023-08-27T23:13:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:06 smithi130 bash[27579]: cluster 2023-08-27T23:13:04.715503+0000 mgr.y (mgr.14901) 2236 : cluster [DBG] pgmap v2243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:06.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:06 smithi006 bash[33918]: cluster 2023-08-27T23:13:04.715503+0000 mgr.y (mgr.14901) 2236 : cluster [DBG] pgmap v2243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:06.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:06 smithi006 bash[27292]: cluster 2023-08-27T23:13:04.715503+0000 mgr.y (mgr.14901) 2236 : cluster [DBG] pgmap v2243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:07.577 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:07 smithi130 bash[51302]: debug 2023-08-27T23:13:07.228+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:07.231294+0000) 2023-08-27T23:13:07.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:07 smithi130 bash[27579]: audit 2023-08-27T23:13:07.485974+0000 mon.a (mon.0) 2019 : audit [DBG] from='client.? 172.21.15.6:0/4079115134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:07.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:07 smithi006 bash[33918]: audit 2023-08-27T23:13:07.485974+0000 mon.a (mon.0) 2019 : audit [DBG] from='client.? 172.21.15.6:0/4079115134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:07.985 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:07 smithi006 bash[27292]: audit 2023-08-27T23:13:07.485974+0000 mon.a (mon.0) 2019 : audit [DBG] from='client.? 172.21.15.6:0/4079115134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:08.580 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:08 smithi130 bash[51302]: debug 2023-08-27T23:13:08.228+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:08.231644+0000) 2023-08-27T23:13:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:08 smithi130 bash[27579]: cluster 2023-08-27T23:13:06.716280+0000 mgr.y (mgr.14901) 2237 : cluster [DBG] pgmap v2244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:08.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:08 smithi006 bash[33918]: cluster 2023-08-27T23:13:06.716280+0000 mgr.y (mgr.14901) 2237 : cluster [DBG] pgmap v2244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:08.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:08 smithi006 bash[27292]: cluster 2023-08-27T23:13:06.716280+0000 mgr.y (mgr.14901) 2237 : cluster [DBG] pgmap v2244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:09.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:09 smithi130 bash[51302]: debug 2023-08-27T23:13:09.228+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:09.231949+0000) 2023-08-27T23:13:10.585 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:10 smithi130 bash[51302]: debug 2023-08-27T23:13:10.228+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:10.232255+0000) 2023-08-27T23:13:10.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:10 smithi006 bash[33918]: cluster 2023-08-27T23:13:08.717090+0000 mgr.y (mgr.14901) 2238 : cluster [DBG] pgmap v2245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:10.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:10 smithi006 bash[33918]: audit 2023-08-27T23:13:09.897473+0000 mon.a (mon.0) 2020 : audit [DBG] from='client.? 172.21.15.6:0/1119180428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:13:10.404Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:13:10.406Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:10 smithi006 bash[27292]: cluster 2023-08-27T23:13:08.717090+0000 mgr.y (mgr.14901) 2238 : cluster [DBG] pgmap v2245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:10 smithi006 bash[27292]: audit 2023-08-27T23:13:09.897473+0000 mon.a (mon.0) 2020 : audit [DBG] from='client.? 172.21.15.6:0/1119180428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:10 smithi130 bash[27579]: cluster 2023-08-27T23:13:08.717090+0000 mgr.y (mgr.14901) 2238 : cluster [DBG] pgmap v2245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:10 smithi130 bash[27579]: audit 2023-08-27T23:13:09.897473+0000 mon.a (mon.0) 2020 : audit [DBG] from='client.? 172.21.15.6:0/1119180428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:13:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:13:10] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:13:11.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:11 smithi130 bash[51302]: debug 2023-08-27T23:13:11.228+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:11.232532+0000) 2023-08-27T23:13:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:11 smithi006 bash[56298]: level=error ts=2023-08-27T23:13:11.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:11 smithi006 bash[56298]: level=warn ts=2023-08-27T23:13:11.971Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:12.590 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:12 smithi130 bash[51302]: debug 2023-08-27T23:13:12.228+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:12.232870+0000) 2023-08-27T23:13:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:12 smithi130 bash[27579]: cluster 2023-08-27T23:13:10.718161+0000 mgr.y (mgr.14901) 2239 : cluster [DBG] pgmap v2246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:12 smithi130 bash[27579]: audit 2023-08-27T23:13:12.313128+0000 mon.c (mon.2) 885 : audit [DBG] from='client.? 172.21.15.6:0/2214510134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:12 smithi006 bash[33918]: cluster 2023-08-27T23:13:10.718161+0000 mgr.y (mgr.14901) 2239 : cluster [DBG] pgmap v2246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:12 smithi006 bash[33918]: audit 2023-08-27T23:13:12.313128+0000 mon.c (mon.2) 885 : audit [DBG] from='client.? 172.21.15.6:0/2214510134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:12 smithi006 bash[27292]: cluster 2023-08-27T23:13:10.718161+0000 mgr.y (mgr.14901) 2239 : cluster [DBG] pgmap v2246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:12 smithi006 bash[27292]: audit 2023-08-27T23:13:12.313128+0000 mon.c (mon.2) 885 : audit [DBG] from='client.? 172.21.15.6:0/2214510134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:13.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:13 smithi130 bash[51302]: debug 2023-08-27T23:13:13.232+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:13.233144+0000) 2023-08-27T23:13:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:13:13.871Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:13:13.874Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:14.596 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:14 smithi130 bash[51302]: debug 2023-08-27T23:13:14.228+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:14.233413+0000) 2023-08-27T23:13:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:14 smithi130 bash[27579]: cluster 2023-08-27T23:13:12.718811+0000 mgr.y (mgr.14901) 2240 : cluster [DBG] pgmap v2247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:14.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:14 smithi006 bash[27292]: cluster 2023-08-27T23:13:12.718811+0000 mgr.y (mgr.14901) 2240 : cluster [DBG] pgmap v2247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:14.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:14 smithi006 bash[33918]: cluster 2023-08-27T23:13:12.718811+0000 mgr.y (mgr.14901) 2240 : cluster [DBG] pgmap v2247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:15.607 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:15 smithi130 bash[51302]: debug 2023-08-27T23:13:15.228+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:15.233676+0000) 2023-08-27T23:13:15.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:15 smithi130 bash[27579]: audit 2023-08-27T23:13:14.732069+0000 mon.a (mon.0) 2021 : audit [DBG] from='client.? 172.21.15.6:0/2930425991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:15.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:15 smithi006 bash[27292]: audit 2023-08-27T23:13:14.732069+0000 mon.a (mon.0) 2021 : audit [DBG] from='client.? 172.21.15.6:0/2930425991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:15.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:15 smithi006 bash[33918]: audit 2023-08-27T23:13:14.732069+0000 mon.a (mon.0) 2021 : audit [DBG] from='client.? 172.21.15.6:0/2930425991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:16.610 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:16 smithi130 bash[51302]: debug 2023-08-27T23:13:16.232+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:16.233972+0000) 2023-08-27T23:13:16.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:16 smithi130 bash[27579]: cluster 2023-08-27T23:13:14.719872+0000 mgr.y (mgr.14901) 2241 : cluster [DBG] pgmap v2248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:16.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:16 smithi006 bash[33918]: cluster 2023-08-27T23:13:14.719872+0000 mgr.y (mgr.14901) 2241 : cluster [DBG] pgmap v2248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:16.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:16 smithi006 bash[27292]: cluster 2023-08-27T23:13:14.719872+0000 mgr.y (mgr.14901) 2241 : cluster [DBG] pgmap v2248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:17.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:17 smithi130 bash[51302]: debug 2023-08-27T23:13:17.232+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:17.234273+0000) 2023-08-27T23:13:17.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:17 smithi006 bash[33918]: audit 2023-08-27T23:13:17.142503+0000 mon.c (mon.2) 886 : audit [DBG] from='client.? 172.21.15.6:0/3723885354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:17.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:17 smithi006 bash[27292]: audit 2023-08-27T23:13:17.142503+0000 mon.c (mon.2) 886 : audit [DBG] from='client.? 172.21.15.6:0/3723885354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:18.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:17 smithi130 bash[27579]: audit 2023-08-27T23:13:17.142503+0000 mon.c (mon.2) 886 : audit [DBG] from='client.? 172.21.15.6:0/3723885354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:18.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:18 smithi130 bash[51302]: debug 2023-08-27T23:13:18.232+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:18.234589+0000) 2023-08-27T23:13:18.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:18 smithi006 bash[33918]: cluster 2023-08-27T23:13:16.720574+0000 mgr.y (mgr.14901) 2242 : cluster [DBG] pgmap v2249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:18.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:18 smithi006 bash[27292]: cluster 2023-08-27T23:13:16.720574+0000 mgr.y (mgr.14901) 2242 : cluster [DBG] pgmap v2249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:18 smithi130 bash[27579]: cluster 2023-08-27T23:13:16.720574+0000 mgr.y (mgr.14901) 2242 : cluster [DBG] pgmap v2249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:19.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:19 smithi130 bash[51302]: debug 2023-08-27T23:13:19.232+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:19.234912+0000) 2023-08-27T23:13:20.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:20 smithi130 bash[51302]: debug 2023-08-27T23:13:20.232+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:20.235212+0000) 2023-08-27T23:13:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:20 smithi130 bash[27579]: cluster 2023-08-27T23:13:18.721276+0000 mgr.y (mgr.14901) 2243 : cluster [DBG] pgmap v2250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:20 smithi130 bash[27579]: audit 2023-08-27T23:13:19.553042+0000 mon.c (mon.2) 887 : audit [DBG] from='client.? 172.21.15.6:0/744225817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:20 smithi006 bash[27292]: cluster 2023-08-27T23:13:18.721276+0000 mgr.y (mgr.14901) 2243 : cluster [DBG] pgmap v2250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:20 smithi006 bash[27292]: audit 2023-08-27T23:13:19.553042+0000 mon.c (mon.2) 887 : audit [DBG] from='client.? 172.21.15.6:0/744225817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:20 smithi006 bash[33918]: cluster 2023-08-27T23:13:18.721276+0000 mgr.y (mgr.14901) 2243 : cluster [DBG] pgmap v2250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:20 smithi006 bash[33918]: audit 2023-08-27T23:13:19.553042+0000 mon.c (mon.2) 887 : audit [DBG] from='client.? 172.21.15.6:0/744225817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:13:20.406Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:13:20.408Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:21.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:13:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:13:20] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:13:21.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:21 smithi130 bash[51302]: debug 2023-08-27T23:13:21.232+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:21.235531+0000) 2023-08-27T23:13:22.429 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:21 smithi006 bash[56298]: level=error ts=2023-08-27T23:13:21.971Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:22.429 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:21 smithi006 bash[56298]: level=warn ts=2023-08-27T23:13:21.973Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:22.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:22 smithi130 bash[27579]: cluster 2023-08-27T23:13:20.722384+0000 mgr.y (mgr.14901) 2244 : cluster [DBG] pgmap v2251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:22.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:22 smithi130 bash[27579]: audit 2023-08-27T23:13:21.971070+0000 mon.c (mon.2) 888 : audit [DBG] from='client.? 172.21.15.6:0/1063917544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:22.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:22 smithi130 bash[51302]: debug 2023-08-27T23:13:22.232+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:22.235825+0000) 2023-08-27T23:13:22.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:22 smithi006 bash[33918]: cluster 2023-08-27T23:13:20.722384+0000 mgr.y (mgr.14901) 2244 : cluster [DBG] pgmap v2251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:22.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:22 smithi006 bash[33918]: audit 2023-08-27T23:13:21.971070+0000 mon.c (mon.2) 888 : audit [DBG] from='client.? 172.21.15.6:0/1063917544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:22 smithi006 bash[27292]: cluster 2023-08-27T23:13:20.722384+0000 mgr.y (mgr.14901) 2244 : cluster [DBG] pgmap v2251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:22 smithi006 bash[27292]: audit 2023-08-27T23:13:21.971070+0000 mon.c (mon.2) 888 : audit [DBG] from='client.? 172.21.15.6:0/1063917544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:23.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:23 smithi130 bash[51302]: debug 2023-08-27T23:13:23.232+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:23.236092+0000) 2023-08-27T23:13:24.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:13:23.872Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:24.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:13:23.874Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:24.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:24 smithi130 bash[27579]: cluster 2023-08-27T23:13:22.723063+0000 mgr.y (mgr.14901) 2245 : cluster [DBG] pgmap v2252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:24 smithi130 bash[27579]: audit 2023-08-27T23:13:24.386733+0000 mon.a (mon.0) 2022 : audit [DBG] from='client.? 172.21.15.6:0/1431646109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:24.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:24 smithi130 bash[51302]: debug 2023-08-27T23:13:24.232+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:24.236379+0000) 2023-08-27T23:13:24.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:24 smithi006 bash[33918]: cluster 2023-08-27T23:13:22.723063+0000 mgr.y (mgr.14901) 2245 : cluster [DBG] pgmap v2252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:24.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:24 smithi006 bash[33918]: audit 2023-08-27T23:13:24.386733+0000 mon.a (mon.0) 2022 : audit [DBG] from='client.? 172.21.15.6:0/1431646109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:24.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:24 smithi006 bash[27292]: cluster 2023-08-27T23:13:22.723063+0000 mgr.y (mgr.14901) 2245 : cluster [DBG] pgmap v2252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:24.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:24 smithi006 bash[27292]: audit 2023-08-27T23:13:24.386733+0000 mon.a (mon.0) 2022 : audit [DBG] from='client.? 172.21.15.6:0/1431646109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:25.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:25 smithi130 bash[51302]: debug 2023-08-27T23:13:25.232+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:25.236673+0000) 2023-08-27T23:13:26.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:26 smithi130 bash[27579]: cluster 2023-08-27T23:13:24.724180+0000 mgr.y (mgr.14901) 2246 : cluster [DBG] pgmap v2253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:26 smithi130 bash[27579]: audit 2023-08-27T23:13:26.121756+0000 mon.b (mon.1) 335 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:13:26.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:26 smithi130 bash[51302]: debug 2023-08-27T23:13:26.232+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:26.236954+0000) 2023-08-27T23:13:26.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:26 smithi006 bash[27292]: cluster 2023-08-27T23:13:24.724180+0000 mgr.y (mgr.14901) 2246 : cluster [DBG] pgmap v2253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:26.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:26 smithi006 bash[27292]: audit 2023-08-27T23:13:26.121756+0000 mon.b (mon.1) 335 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:13:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:26 smithi006 bash[33918]: cluster 2023-08-27T23:13:24.724180+0000 mgr.y (mgr.14901) 2246 : cluster [DBG] pgmap v2253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:26 smithi006 bash[33918]: audit 2023-08-27T23:13:26.121756+0000 mon.b (mon.1) 335 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:13:27.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:27 smithi130 bash[27579]: audit 2023-08-27T23:13:26.800669+0000 mon.c (mon.2) 889 : audit [DBG] from='client.? 172.21.15.6:0/2413589822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:27.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:27 smithi130 bash[51302]: debug 2023-08-27T23:13:27.236+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:27.237262+0000) 2023-08-27T23:13:27.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:27 smithi006 bash[27292]: audit 2023-08-27T23:13:26.800669+0000 mon.c (mon.2) 889 : audit [DBG] from='client.? 172.21.15.6:0/2413589822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:27.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:27 smithi006 bash[33918]: audit 2023-08-27T23:13:26.800669+0000 mon.c (mon.2) 889 : audit [DBG] from='client.? 172.21.15.6:0/2413589822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:28.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:28 smithi130 bash[27579]: cluster 2023-08-27T23:13:26.724618+0000 mgr.y (mgr.14901) 2247 : cluster [DBG] pgmap v2254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:28.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:28 smithi130 bash[51302]: debug 2023-08-27T23:13:28.232+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:28.237582+0000) 2023-08-27T23:13:28.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:28 smithi006 bash[33918]: cluster 2023-08-27T23:13:26.724618+0000 mgr.y (mgr.14901) 2247 : cluster [DBG] pgmap v2254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:28.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:28 smithi006 bash[27292]: cluster 2023-08-27T23:13:26.724618+0000 mgr.y (mgr.14901) 2247 : cluster [DBG] pgmap v2254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:29.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:29 smithi130 bash[27579]: audit 2023-08-27T23:13:29.218863+0000 mon.c (mon.2) 890 : audit [DBG] from='client.? 172.21.15.6:0/387209173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:29.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:29 smithi130 bash[51302]: debug 2023-08-27T23:13:29.232+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:29.237894+0000) 2023-08-27T23:13:29.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:29 smithi006 bash[27292]: audit 2023-08-27T23:13:29.218863+0000 mon.c (mon.2) 890 : audit [DBG] from='client.? 172.21.15.6:0/387209173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:29.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:29 smithi006 bash[33918]: audit 2023-08-27T23:13:29.218863+0000 mon.c (mon.2) 890 : audit [DBG] from='client.? 172.21.15.6:0/387209173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:30.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:30 smithi130 bash[27579]: cluster 2023-08-27T23:13:28.725364+0000 mgr.y (mgr.14901) 2248 : cluster [DBG] pgmap v2255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:30.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:30 smithi130 bash[51302]: debug 2023-08-27T23:13:30.232+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:30.238202+0000) 2023-08-27T23:13:30.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:30 smithi006 bash[27292]: cluster 2023-08-27T23:13:28.725364+0000 mgr.y (mgr.14901) 2248 : cluster [DBG] pgmap v2255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:30 smithi006 bash[33918]: cluster 2023-08-27T23:13:28.725364+0000 mgr.y (mgr.14901) 2248 : cluster [DBG] pgmap v2255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:13:30.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:13:30.408Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:13:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:13:30] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:13:31.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:31 smithi130 bash[51302]: debug 2023-08-27T23:13:31.232+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:31.238422+0000) 2023-08-27T23:13:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:31 smithi006 bash[56298]: level=error ts=2023-08-27T23:13:31.971Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:31 smithi006 bash[56298]: level=warn ts=2023-08-27T23:13:31.973Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:32.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:32 smithi130 bash[51302]: debug 2023-08-27T23:13:32.232+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:32.238692+0000) 2023-08-27T23:13:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:32 smithi130 bash[27579]: cluster 2023-08-27T23:13:30.726541+0000 mgr.y (mgr.14901) 2249 : cluster [DBG] pgmap v2256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:32 smithi130 bash[27579]: audit 2023-08-27T23:13:31.666807+0000 mon.c (mon.2) 891 : audit [DBG] from='client.? 172.21.15.6:0/3861244771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:32 smithi130 bash[27579]: audit 2023-08-27T23:13:32.037544+0000 mon.a (mon.0) 2023 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:13:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:32 smithi130 bash[27579]: audit 2023-08-27T23:13:32.439554+0000 mon.a (mon.0) 2024 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:13:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:32 smithi006 bash[27292]: cluster 2023-08-27T23:13:30.726541+0000 mgr.y (mgr.14901) 2249 : cluster [DBG] pgmap v2256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:32 smithi006 bash[27292]: audit 2023-08-27T23:13:31.666807+0000 mon.c (mon.2) 891 : audit [DBG] from='client.? 172.21.15.6:0/3861244771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:32 smithi006 bash[27292]: audit 2023-08-27T23:13:32.037544+0000 mon.a (mon.0) 2023 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:13:32.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:32 smithi006 bash[27292]: audit 2023-08-27T23:13:32.439554+0000 mon.a (mon.0) 2024 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:13:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:32 smithi006 bash[33918]: cluster 2023-08-27T23:13:30.726541+0000 mgr.y (mgr.14901) 2249 : cluster [DBG] pgmap v2256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:32 smithi006 bash[33918]: audit 2023-08-27T23:13:31.666807+0000 mon.c (mon.2) 891 : audit [DBG] from='client.? 172.21.15.6:0/3861244771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:32 smithi006 bash[33918]: audit 2023-08-27T23:13:32.037544+0000 mon.a (mon.0) 2023 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:13:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:32 smithi006 bash[33918]: audit 2023-08-27T23:13:32.439554+0000 mon.a (mon.0) 2024 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:13:33.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:33 smithi130 bash[27579]: audit 2023-08-27T23:13:32.711574+0000 mon.b (mon.1) 336 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:13:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:33 smithi130 bash[27579]: audit 2023-08-27T23:13:32.714171+0000 mon.b (mon.1) 337 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:13:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:33 smithi130 bash[27579]: audit 2023-08-27T23:13:32.722984+0000 mon.a (mon.0) 2025 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:13:33.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:33 smithi130 bash[51302]: debug 2023-08-27T23:13:33.236+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:33.238985+0000) 2023-08-27T23:13:33.875 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:33 smithi006 bash[33918]: audit 2023-08-27T23:13:32.711574+0000 mon.b (mon.1) 336 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:13:33.875 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:33 smithi006 bash[33918]: audit 2023-08-27T23:13:32.714171+0000 mon.b (mon.1) 337 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:13:33.875 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:33 smithi006 bash[33918]: audit 2023-08-27T23:13:32.722984+0000 mon.a (mon.0) 2025 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:13:33.875 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:33 smithi006 bash[27292]: audit 2023-08-27T23:13:32.711574+0000 mon.b (mon.1) 336 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:13:33.875 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:33 smithi006 bash[27292]: audit 2023-08-27T23:13:32.714171+0000 mon.b (mon.1) 337 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:13:33.876 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:33 smithi006 bash[27292]: audit 2023-08-27T23:13:32.722984+0000 mon.a (mon.0) 2025 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:13:34.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:13:33.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:34.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:13:33.875Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:34.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:34 smithi130 bash[27579]: cluster 2023-08-27T23:13:32.727250+0000 mgr.y (mgr.14901) 2250 : cluster [DBG] pgmap v2257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:34 smithi130 bash[27579]: audit 2023-08-27T23:13:34.093575+0000 mon.a (mon.0) 2026 : audit [DBG] from='client.? 172.21.15.6:0/638307548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:34.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:34 smithi130 bash[51302]: debug 2023-08-27T23:13:34.236+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:34.239245+0000) 2023-08-27T23:13:34.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:34 smithi006 bash[27292]: cluster 2023-08-27T23:13:32.727250+0000 mgr.y (mgr.14901) 2250 : cluster [DBG] pgmap v2257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:34.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:34 smithi006 bash[27292]: audit 2023-08-27T23:13:34.093575+0000 mon.a (mon.0) 2026 : audit [DBG] from='client.? 172.21.15.6:0/638307548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:34.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:34 smithi006 bash[33918]: cluster 2023-08-27T23:13:32.727250+0000 mgr.y (mgr.14901) 2250 : cluster [DBG] pgmap v2257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:34.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:34 smithi006 bash[33918]: audit 2023-08-27T23:13:34.093575+0000 mon.a (mon.0) 2026 : audit [DBG] from='client.? 172.21.15.6:0/638307548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:35.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:35 smithi130 bash[51302]: debug 2023-08-27T23:13:35.236+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:35.239565+0000) 2023-08-27T23:13:36.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:36 smithi130 bash[27579]: cluster 2023-08-27T23:13:34.728352+0000 mgr.y (mgr.14901) 2251 : cluster [DBG] pgmap v2258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:36.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:36 smithi130 bash[51302]: debug 2023-08-27T23:13:36.236+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:36.239854+0000) 2023-08-27T23:13:36.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:36 smithi006 bash[27292]: cluster 2023-08-27T23:13:34.728352+0000 mgr.y (mgr.14901) 2251 : cluster [DBG] pgmap v2258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:36.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:36 smithi006 bash[33918]: cluster 2023-08-27T23:13:34.728352+0000 mgr.y (mgr.14901) 2251 : cluster [DBG] pgmap v2258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:37.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:37 smithi130 bash[27579]: audit 2023-08-27T23:13:36.509279+0000 mon.a (mon.0) 2027 : audit [DBG] from='client.? 172.21.15.6:0/4278016477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:37.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:37 smithi130 bash[51302]: debug 2023-08-27T23:13:37.236+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:37.240177+0000) 2023-08-27T23:13:37.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:37 smithi006 bash[27292]: audit 2023-08-27T23:13:36.509279+0000 mon.a (mon.0) 2027 : audit [DBG] from='client.? 172.21.15.6:0/4278016477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:37.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:37 smithi006 bash[33918]: audit 2023-08-27T23:13:36.509279+0000 mon.a (mon.0) 2027 : audit [DBG] from='client.? 172.21.15.6:0/4278016477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:38.499 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:38 smithi130 bash[51302]: debug 2023-08-27T23:13:38.236+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:38.240457+0000) 2023-08-27T23:13:38.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:38 smithi130 bash[27579]: cluster 2023-08-27T23:13:36.729007+0000 mgr.y (mgr.14901) 2252 : cluster [DBG] pgmap v2259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:38.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:38 smithi006 bash[27292]: cluster 2023-08-27T23:13:36.729007+0000 mgr.y (mgr.14901) 2252 : cluster [DBG] pgmap v2259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:38.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:38 smithi006 bash[33918]: cluster 2023-08-27T23:13:36.729007+0000 mgr.y (mgr.14901) 2252 : cluster [DBG] pgmap v2259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:39.508 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:39 smithi130 bash[51302]: debug 2023-08-27T23:13:39.236+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:39.240777+0000) 2023-08-27T23:13:39.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:39 smithi130 bash[27579]: audit 2023-08-27T23:13:38.917534+0000 mon.a (mon.0) 2028 : audit [DBG] from='client.? 172.21.15.6:0/1851712584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:39.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:39 smithi006 bash[27292]: audit 2023-08-27T23:13:38.917534+0000 mon.a (mon.0) 2028 : audit [DBG] from='client.? 172.21.15.6:0/1851712584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:39.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:39 smithi006 bash[33918]: audit 2023-08-27T23:13:38.917534+0000 mon.a (mon.0) 2028 : audit [DBG] from='client.? 172.21.15.6:0/1851712584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:40.512 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:40 smithi130 bash[51302]: debug 2023-08-27T23:13:40.236+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:40.241064+0000) 2023-08-27T23:13:40.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:40 smithi006 bash[33918]: cluster 2023-08-27T23:13:38.729609+0000 mgr.y (mgr.14901) 2253 : cluster [DBG] pgmap v2260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:40.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:40 smithi006 bash[27292]: cluster 2023-08-27T23:13:38.729609+0000 mgr.y (mgr.14901) 2253 : cluster [DBG] pgmap v2260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:13:40.408Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:13:40.410Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:40 smithi130 bash[27579]: cluster 2023-08-27T23:13:38.729609+0000 mgr.y (mgr.14901) 2253 : cluster [DBG] pgmap v2260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:13:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:13:40] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:13:41.522 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:41 smithi130 bash[51302]: debug 2023-08-27T23:13:41.240+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:41.241365+0000) 2023-08-27T23:13:41.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:41 smithi130 bash[27579]: audit 2023-08-27T23:13:41.328706+0000 mon.a (mon.0) 2029 : audit [DBG] from='client.? 172.21.15.6:0/1279513612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:41.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:41 smithi006 bash[27292]: audit 2023-08-27T23:13:41.328706+0000 mon.a (mon.0) 2029 : audit [DBG] from='client.? 172.21.15.6:0/1279513612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:41.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:41 smithi006 bash[33918]: audit 2023-08-27T23:13:41.328706+0000 mon.a (mon.0) 2029 : audit [DBG] from='client.? 172.21.15.6:0/1279513612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:42.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:41 smithi006 bash[56298]: level=error ts=2023-08-27T23:13:41.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:42.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:41 smithi006 bash[56298]: level=warn ts=2023-08-27T23:13:41.974Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:42.528 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:42 smithi130 bash[51302]: debug 2023-08-27T23:13:42.236+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:42.241597+0000) 2023-08-27T23:13:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:42 smithi130 bash[27579]: cluster 2023-08-27T23:13:40.730738+0000 mgr.y (mgr.14901) 2254 : cluster [DBG] pgmap v2261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:42 smithi006 bash[27292]: cluster 2023-08-27T23:13:40.730738+0000 mgr.y (mgr.14901) 2254 : cluster [DBG] pgmap v2261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:42.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:42 smithi006 bash[33918]: cluster 2023-08-27T23:13:40.730738+0000 mgr.y (mgr.14901) 2254 : cluster [DBG] pgmap v2261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:43.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:43 smithi130 bash[51302]: debug 2023-08-27T23:13:43.236+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:43.241928+0000) 2023-08-27T23:13:44.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:13:43.874Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:44.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:13:43.876Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:44.533 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:44 smithi130 bash[51302]: debug 2023-08-27T23:13:44.236+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:44.242251+0000) 2023-08-27T23:13:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:44 smithi130 bash[27579]: cluster 2023-08-27T23:13:42.731422+0000 mgr.y (mgr.14901) 2255 : cluster [DBG] pgmap v2262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:44 smithi130 bash[27579]: audit 2023-08-27T23:13:43.739569+0000 mon.c (mon.2) 892 : audit [DBG] from='client.? 172.21.15.6:0/1356332616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:44.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:44 smithi006 bash[33918]: cluster 2023-08-27T23:13:42.731422+0000 mgr.y (mgr.14901) 2255 : cluster [DBG] pgmap v2262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:44.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:44 smithi006 bash[33918]: audit 2023-08-27T23:13:43.739569+0000 mon.c (mon.2) 892 : audit [DBG] from='client.? 172.21.15.6:0/1356332616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:44.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:44 smithi006 bash[27292]: cluster 2023-08-27T23:13:42.731422+0000 mgr.y (mgr.14901) 2255 : cluster [DBG] pgmap v2262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:44.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:44 smithi006 bash[27292]: audit 2023-08-27T23:13:43.739569+0000 mon.c (mon.2) 892 : audit [DBG] from='client.? 172.21.15.6:0/1356332616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:45.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:45 smithi130 bash[51302]: debug 2023-08-27T23:13:45.236+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:45.242561+0000) 2023-08-27T23:13:46.538 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:46 smithi130 bash[51302]: debug 2023-08-27T23:13:46.240+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:46.242831+0000) 2023-08-27T23:13:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:46 smithi130 bash[27579]: cluster 2023-08-27T23:13:44.732502+0000 mgr.y (mgr.14901) 2256 : cluster [DBG] pgmap v2263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:46 smithi130 bash[27579]: audit 2023-08-27T23:13:46.153343+0000 mon.c (mon.2) 893 : audit [DBG] from='client.? 172.21.15.6:0/2051992040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:46.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:46 smithi006 bash[27292]: cluster 2023-08-27T23:13:44.732502+0000 mgr.y (mgr.14901) 2256 : cluster [DBG] pgmap v2263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:46.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:46 smithi006 bash[27292]: audit 2023-08-27T23:13:46.153343+0000 mon.c (mon.2) 893 : audit [DBG] from='client.? 172.21.15.6:0/2051992040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:46 smithi006 bash[33918]: cluster 2023-08-27T23:13:44.732502+0000 mgr.y (mgr.14901) 2256 : cluster [DBG] pgmap v2263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:46 smithi006 bash[33918]: audit 2023-08-27T23:13:46.153343+0000 mon.c (mon.2) 893 : audit [DBG] from='client.? 172.21.15.6:0/2051992040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:47.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:47 smithi130 bash[51302]: debug 2023-08-27T23:13:47.240+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:47.243157+0000) 2023-08-27T23:13:48.543 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:48 smithi130 bash[51302]: debug 2023-08-27T23:13:48.240+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:48.243480+0000) 2023-08-27T23:13:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:48 smithi130 bash[27579]: cluster 2023-08-27T23:13:46.733081+0000 mgr.y (mgr.14901) 2257 : cluster [DBG] pgmap v2264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:48.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:48 smithi006 bash[33918]: cluster 2023-08-27T23:13:46.733081+0000 mgr.y (mgr.14901) 2257 : cluster [DBG] pgmap v2264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:48.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:48 smithi006 bash[27292]: cluster 2023-08-27T23:13:46.733081+0000 mgr.y (mgr.14901) 2257 : cluster [DBG] pgmap v2264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:49.552 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:49 smithi130 bash[51302]: debug 2023-08-27T23:13:49.240+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:49.243757+0000) 2023-08-27T23:13:49.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:49 smithi130 bash[27579]: audit 2023-08-27T23:13:48.570902+0000 mon.a (mon.0) 2030 : audit [DBG] from='client.? 172.21.15.6:0/2247957574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:49.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:49 smithi006 bash[27292]: audit 2023-08-27T23:13:48.570902+0000 mon.a (mon.0) 2030 : audit [DBG] from='client.? 172.21.15.6:0/2247957574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:49.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:49 smithi006 bash[33918]: audit 2023-08-27T23:13:48.570902+0000 mon.a (mon.0) 2030 : audit [DBG] from='client.? 172.21.15.6:0/2247957574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:50.555 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:50 smithi130 bash[51302]: debug 2023-08-27T23:13:50.240+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:50.243998+0000) 2023-08-27T23:13:50.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:13:50.409Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:50.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:13:50.410Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:50 smithi006 bash[27292]: cluster 2023-08-27T23:13:48.733712+0000 mgr.y (mgr.14901) 2258 : cluster [DBG] pgmap v2265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:50 smithi006 bash[33918]: cluster 2023-08-27T23:13:48.733712+0000 mgr.y (mgr.14901) 2258 : cluster [DBG] pgmap v2265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:50 smithi130 bash[27579]: cluster 2023-08-27T23:13:48.733712+0000 mgr.y (mgr.14901) 2258 : cluster [DBG] pgmap v2265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:13:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:13:50] "GET /metrics HTTP/1.1" 200 193091 "" "Prometheus/2.33.4" 2023-08-27T23:13:51.564 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:51 smithi130 bash[51302]: debug 2023-08-27T23:13:51.240+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:51.244286+0000) 2023-08-27T23:13:51.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:51 smithi130 bash[27579]: audit 2023-08-27T23:13:50.986715+0000 mon.a (mon.0) 2031 : audit [DBG] from='client.? 172.21.15.6:0/1318085581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:51.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:51 smithi006 bash[27292]: audit 2023-08-27T23:13:50.986715+0000 mon.a (mon.0) 2031 : audit [DBG] from='client.? 172.21.15.6:0/1318085581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:51.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:51 smithi006 bash[33918]: audit 2023-08-27T23:13:50.986715+0000 mon.a (mon.0) 2031 : audit [DBG] from='client.? 172.21.15.6:0/1318085581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:51 smithi006 bash[56298]: level=error ts=2023-08-27T23:13:51.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:51 smithi006 bash[56298]: level=warn ts=2023-08-27T23:13:51.974Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:52.567 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:52 smithi130 bash[51302]: debug 2023-08-27T23:13:52.240+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:52.244581+0000) 2023-08-27T23:13:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:52 smithi130 bash[27579]: cluster 2023-08-27T23:13:50.734723+0000 mgr.y (mgr.14901) 2259 : cluster [DBG] pgmap v2266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:52 smithi006 bash[27292]: cluster 2023-08-27T23:13:50.734723+0000 mgr.y (mgr.14901) 2259 : cluster [DBG] pgmap v2266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:52 smithi006 bash[33918]: cluster 2023-08-27T23:13:50.734723+0000 mgr.y (mgr.14901) 2259 : cluster [DBG] pgmap v2266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:53.575 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:53 smithi130 bash[51302]: debug 2023-08-27T23:13:53.244+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:53.244922+0000) 2023-08-27T23:13:53.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:53 smithi130 bash[27579]: audit 2023-08-27T23:13:53.398127+0000 mon.a (mon.0) 2032 : audit [DBG] from='client.? 172.21.15.6:0/1981025810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:53.877 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:53 smithi006 bash[27292]: audit 2023-08-27T23:13:53.398127+0000 mon.a (mon.0) 2032 : audit [DBG] from='client.? 172.21.15.6:0/1981025810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:53.877 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:53 smithi006 bash[33918]: audit 2023-08-27T23:13:53.398127+0000 mon.a (mon.0) 2032 : audit [DBG] from='client.? 172.21.15.6:0/1981025810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:54.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:13:53.875Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:54.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:13:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:13:53.877Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:13:54.578 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:54 smithi130 bash[51302]: debug 2023-08-27T23:13:54.240+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:54.245209+0000) 2023-08-27T23:13:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:54 smithi130 bash[27579]: cluster 2023-08-27T23:13:52.735385+0000 mgr.y (mgr.14901) 2260 : cluster [DBG] pgmap v2267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:54.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:54 smithi006 bash[33918]: cluster 2023-08-27T23:13:52.735385+0000 mgr.y (mgr.14901) 2260 : cluster [DBG] pgmap v2267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:54.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:54 smithi006 bash[27292]: cluster 2023-08-27T23:13:52.735385+0000 mgr.y (mgr.14901) 2260 : cluster [DBG] pgmap v2267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:55.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:55 smithi130 bash[51302]: debug 2023-08-27T23:13:55.240+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:55.245521+0000) 2023-08-27T23:13:56.583 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:56 smithi130 bash[51302]: debug 2023-08-27T23:13:56.240+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:56.245792+0000) 2023-08-27T23:13:56.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:56 smithi130 bash[27579]: cluster 2023-08-27T23:13:54.736532+0000 mgr.y (mgr.14901) 2261 : cluster [DBG] pgmap v2268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:56 smithi130 bash[27579]: audit 2023-08-27T23:13:55.816403+0000 mon.c (mon.2) 894 : audit [DBG] from='client.? 172.21.15.6:0/824245059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:56.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:56 smithi006 bash[27292]: cluster 2023-08-27T23:13:54.736532+0000 mgr.y (mgr.14901) 2261 : cluster [DBG] pgmap v2268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:56.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:56 smithi006 bash[27292]: audit 2023-08-27T23:13:55.816403+0000 mon.c (mon.2) 894 : audit [DBG] from='client.? 172.21.15.6:0/824245059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:56.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:56 smithi006 bash[33918]: cluster 2023-08-27T23:13:54.736532+0000 mgr.y (mgr.14901) 2261 : cluster [DBG] pgmap v2268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:56.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:56 smithi006 bash[33918]: audit 2023-08-27T23:13:55.816403+0000 mon.c (mon.2) 894 : audit [DBG] from='client.? 172.21.15.6:0/824245059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:57.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:57 smithi130 bash[51302]: debug 2023-08-27T23:13:57.244+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:57.246062+0000) 2023-08-27T23:13:58.589 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:58 smithi130 bash[51302]: debug 2023-08-27T23:13:58.240+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:58.246354+0000) 2023-08-27T23:13:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:58 smithi130 bash[27579]: cluster 2023-08-27T23:13:56.737192+0000 mgr.y (mgr.14901) 2262 : cluster [DBG] pgmap v2269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:13:58 smithi130 bash[27579]: audit 2023-08-27T23:13:58.232812+0000 mon.c (mon.2) 895 : audit [DBG] from='client.? 172.21.15.6:0/2418128865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:58.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:58 smithi006 bash[27292]: cluster 2023-08-27T23:13:56.737192+0000 mgr.y (mgr.14901) 2262 : cluster [DBG] pgmap v2269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:58.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:13:58 smithi006 bash[27292]: audit 2023-08-27T23:13:58.232812+0000 mon.c (mon.2) 895 : audit [DBG] from='client.? 172.21.15.6:0/2418128865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:58 smithi006 bash[33918]: cluster 2023-08-27T23:13:56.737192+0000 mgr.y (mgr.14901) 2262 : cluster [DBG] pgmap v2269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:13:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:13:58 smithi006 bash[33918]: audit 2023-08-27T23:13:58.232812+0000 mon.c (mon.2) 895 : audit [DBG] from='client.? 172.21.15.6:0/2418128865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:59.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:13:59 smithi130 bash[51302]: debug 2023-08-27T23:13:59.239+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:59.246637+0000) 2023-08-27T23:14:00.592 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:00 smithi130 bash[51302]: debug 2023-08-27T23:14:00.243+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:00.246926+0000) 2023-08-27T23:14:00.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:00 smithi006 bash[33918]: cluster 2023-08-27T23:13:58.737817+0000 mgr.y (mgr.14901) 2263 : cluster [DBG] pgmap v2270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:00 smithi006 bash[27292]: cluster 2023-08-27T23:13:58.737817+0000 mgr.y (mgr.14901) 2263 : cluster [DBG] pgmap v2270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:14:00.409Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:14:00.411Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:00 smithi130 bash[27579]: cluster 2023-08-27T23:13:58.737817+0000 mgr.y (mgr.14901) 2263 : cluster [DBG] pgmap v2270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:14:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:14:00] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T23:14:01.600 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:01 smithi130 bash[51302]: debug 2023-08-27T23:14:01.243+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:01.247218+0000) 2023-08-27T23:14:01.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:01 smithi130 bash[27579]: audit 2023-08-27T23:14:00.653943+0000 mon.c (mon.2) 896 : audit [DBG] from='client.? 172.21.15.6:0/3807705743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:01.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:01 smithi006 bash[33918]: audit 2023-08-27T23:14:00.653943+0000 mon.c (mon.2) 896 : audit [DBG] from='client.? 172.21.15.6:0/3807705743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:01.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:01 smithi006 bash[27292]: audit 2023-08-27T23:14:00.653943+0000 mon.c (mon.2) 896 : audit [DBG] from='client.? 172.21.15.6:0/3807705743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:01 smithi006 bash[56298]: level=error ts=2023-08-27T23:14:01.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:01 smithi006 bash[56298]: level=warn ts=2023-08-27T23:14:01.975Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:02.603 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:02 smithi130 bash[51302]: debug 2023-08-27T23:14:02.243+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:02.247448+0000) 2023-08-27T23:14:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:02 smithi130 bash[27579]: cluster 2023-08-27T23:14:00.738934+0000 mgr.y (mgr.14901) 2264 : cluster [DBG] pgmap v2271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:02 smithi006 bash[33918]: cluster 2023-08-27T23:14:00.738934+0000 mgr.y (mgr.14901) 2264 : cluster [DBG] pgmap v2271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:02 smithi006 bash[27292]: cluster 2023-08-27T23:14:00.738934+0000 mgr.y (mgr.14901) 2264 : cluster [DBG] pgmap v2271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:03.612 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:03 smithi130 bash[51302]: debug 2023-08-27T23:14:03.243+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:03.247723+0000) 2023-08-27T23:14:03.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:03 smithi130 bash[27579]: audit 2023-08-27T23:14:03.075398+0000 mon.c (mon.2) 897 : audit [DBG] from='client.? 172.21.15.6:0/2450832656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:03.878 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:03 smithi006 bash[27292]: audit 2023-08-27T23:14:03.075398+0000 mon.c (mon.2) 897 : audit [DBG] from='client.? 172.21.15.6:0/2450832656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:03.878 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:03 smithi006 bash[33918]: audit 2023-08-27T23:14:03.075398+0000 mon.c (mon.2) 897 : audit [DBG] from='client.? 172.21.15.6:0/2450832656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:04.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:14:03.875Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:04.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:14:03.877Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:04.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:04 smithi130 bash[51302]: debug 2023-08-27T23:14:04.243+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:04.248016+0000) 2023-08-27T23:14:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:04 smithi130 bash[27579]: cluster 2023-08-27T23:14:02.739581+0000 mgr.y (mgr.14901) 2265 : cluster [DBG] pgmap v2272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:04.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:04 smithi006 bash[27292]: cluster 2023-08-27T23:14:02.739581+0000 mgr.y (mgr.14901) 2265 : cluster [DBG] pgmap v2272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:04.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:04 smithi006 bash[33918]: cluster 2023-08-27T23:14:02.739581+0000 mgr.y (mgr.14901) 2265 : cluster [DBG] pgmap v2272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:05.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:05 smithi130 bash[51302]: debug 2023-08-27T23:14:05.243+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:05.248293+0000) 2023-08-27T23:14:05.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:05 smithi006 bash[27292]: audit 2023-08-27T23:14:05.496606+0000 mon.a (mon.0) 2033 : audit [DBG] from='client.? 172.21.15.6:0/4049425686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:05.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:05 smithi006 bash[33918]: audit 2023-08-27T23:14:05.496606+0000 mon.a (mon.0) 2033 : audit [DBG] from='client.? 172.21.15.6:0/4049425686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:06.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:05 smithi130 bash[27579]: audit 2023-08-27T23:14:05.496606+0000 mon.a (mon.0) 2033 : audit [DBG] from='client.? 172.21.15.6:0/4049425686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:06.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:06 smithi130 bash[51302]: debug 2023-08-27T23:14:06.243+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:06.248563+0000) 2023-08-27T23:14:06.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:06 smithi006 bash[33918]: cluster 2023-08-27T23:14:04.740776+0000 mgr.y (mgr.14901) 2266 : cluster [DBG] pgmap v2273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:06 smithi006 bash[27292]: cluster 2023-08-27T23:14:04.740776+0000 mgr.y (mgr.14901) 2266 : cluster [DBG] pgmap v2273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:06 smithi130 bash[27579]: cluster 2023-08-27T23:14:04.740776+0000 mgr.y (mgr.14901) 2266 : cluster [DBG] pgmap v2273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:07.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:07 smithi130 bash[51302]: debug 2023-08-27T23:14:07.243+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:07.248821+0000) 2023-08-27T23:14:07.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:07 smithi006 bash[27292]: cluster 2023-08-27T23:14:06.741384+0000 mgr.y (mgr.14901) 2267 : cluster [DBG] pgmap v2274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:07.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:07 smithi006 bash[33918]: cluster 2023-08-27T23:14:06.741384+0000 mgr.y (mgr.14901) 2267 : cluster [DBG] pgmap v2274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:08.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:07 smithi130 bash[27579]: cluster 2023-08-27T23:14:06.741384+0000 mgr.y (mgr.14901) 2267 : cluster [DBG] pgmap v2274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:08.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:08 smithi130 bash[51302]: debug 2023-08-27T23:14:08.243+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:08.249089+0000) 2023-08-27T23:14:08.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:08 smithi006 bash[33918]: audit 2023-08-27T23:14:07.905033+0000 mon.c (mon.2) 898 : audit [DBG] from='client.? 172.21.15.6:0/1984001419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:08 smithi006 bash[27292]: audit 2023-08-27T23:14:07.905033+0000 mon.c (mon.2) 898 : audit [DBG] from='client.? 172.21.15.6:0/1984001419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:08 smithi130 bash[27579]: audit 2023-08-27T23:14:07.905033+0000 mon.c (mon.2) 898 : audit [DBG] from='client.? 172.21.15.6:0/1984001419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:09.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:09 smithi130 bash[51302]: debug 2023-08-27T23:14:09.247+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:09.249332+0000) 2023-08-27T23:14:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:10 smithi130 bash[27579]: cluster 2023-08-27T23:14:08.742082+0000 mgr.y (mgr.14901) 2268 : cluster [DBG] pgmap v2275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:10 smithi130 bash[27579]: audit 2023-08-27T23:14:10.324049+0000 mon.a (mon.0) 2034 : audit [DBG] from='client.? 172.21.15.6:0/2034901171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:10.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:10 smithi130 bash[51302]: debug 2023-08-27T23:14:10.243+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:10.249624+0000) 2023-08-27T23:14:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:10 smithi006 bash[33918]: cluster 2023-08-27T23:14:08.742082+0000 mgr.y (mgr.14901) 2268 : cluster [DBG] pgmap v2275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:10 smithi006 bash[33918]: audit 2023-08-27T23:14:10.324049+0000 mon.a (mon.0) 2034 : audit [DBG] from='client.? 172.21.15.6:0/2034901171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:10.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:10 smithi006 bash[27292]: cluster 2023-08-27T23:14:08.742082+0000 mgr.y (mgr.14901) 2268 : cluster [DBG] pgmap v2275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:10.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:10 smithi006 bash[27292]: audit 2023-08-27T23:14:10.324049+0000 mon.a (mon.0) 2034 : audit [DBG] from='client.? 172.21.15.6:0/2034901171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:10.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:14:10.409Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:10.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:14:10.411Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:14:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:14:10] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T23:14:11.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:11 smithi130 bash[51302]: debug 2023-08-27T23:14:11.247+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:11.249894+0000) 2023-08-27T23:14:12.455 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:11 smithi006 bash[56298]: level=error ts=2023-08-27T23:14:11.974Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:12.455 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:11 smithi006 bash[56298]: level=warn ts=2023-08-27T23:14:11.976Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:12.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:12 smithi130 bash[27579]: cluster 2023-08-27T23:14:10.743361+0000 mgr.y (mgr.14901) 2269 : cluster [DBG] pgmap v2276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:12.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:12 smithi130 bash[51302]: debug 2023-08-27T23:14:12.243+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:12.250188+0000) 2023-08-27T23:14:12.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:12 smithi006 bash[27292]: cluster 2023-08-27T23:14:10.743361+0000 mgr.y (mgr.14901) 2269 : cluster [DBG] pgmap v2276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:12.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:12 smithi006 bash[33918]: cluster 2023-08-27T23:14:10.743361+0000 mgr.y (mgr.14901) 2269 : cluster [DBG] pgmap v2276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:13.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:13 smithi130 bash[27579]: audit 2023-08-27T23:14:12.744495+0000 mon.a (mon.0) 2035 : audit [DBG] from='client.? 172.21.15.6:0/1978907373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:13.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:13 smithi130 bash[51302]: debug 2023-08-27T23:14:13.247+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:13.250454+0000) 2023-08-27T23:14:13.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:13 smithi006 bash[33918]: audit 2023-08-27T23:14:12.744495+0000 mon.a (mon.0) 2035 : audit [DBG] from='client.? 172.21.15.6:0/1978907373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:13.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:13 smithi006 bash[27292]: audit 2023-08-27T23:14:12.744495+0000 mon.a (mon.0) 2035 : audit [DBG] from='client.? 172.21.15.6:0/1978907373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:14:13.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:14:13.878Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:14.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:14 smithi130 bash[27579]: cluster 2023-08-27T23:14:12.744058+0000 mgr.y (mgr.14901) 2270 : cluster [DBG] pgmap v2277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:14.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:14 smithi130 bash[51302]: debug 2023-08-27T23:14:14.243+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:14.250742+0000) 2023-08-27T23:14:14.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:14 smithi006 bash[27292]: cluster 2023-08-27T23:14:12.744058+0000 mgr.y (mgr.14901) 2270 : cluster [DBG] pgmap v2277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:14.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:14 smithi006 bash[33918]: cluster 2023-08-27T23:14:12.744058+0000 mgr.y (mgr.14901) 2270 : cluster [DBG] pgmap v2277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:15.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:15 smithi130 bash[27579]: audit 2023-08-27T23:14:15.166117+0000 mon.a (mon.0) 2036 : audit [DBG] from='client.? 172.21.15.6:0/915934269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:15.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:15 smithi130 bash[51302]: debug 2023-08-27T23:14:15.247+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:15.251056+0000) 2023-08-27T23:14:15.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:15 smithi006 bash[27292]: audit 2023-08-27T23:14:15.166117+0000 mon.a (mon.0) 2036 : audit [DBG] from='client.? 172.21.15.6:0/915934269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:15.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:15 smithi006 bash[33918]: audit 2023-08-27T23:14:15.166117+0000 mon.a (mon.0) 2036 : audit [DBG] from='client.? 172.21.15.6:0/915934269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:16.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:16 smithi130 bash[27579]: cluster 2023-08-27T23:14:14.745247+0000 mgr.y (mgr.14901) 2271 : cluster [DBG] pgmap v2278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:16.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:16 smithi130 bash[51302]: debug 2023-08-27T23:14:16.247+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:16.251329+0000) 2023-08-27T23:14:16.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:16 smithi006 bash[27292]: cluster 2023-08-27T23:14:14.745247+0000 mgr.y (mgr.14901) 2271 : cluster [DBG] pgmap v2278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:16.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:16 smithi006 bash[33918]: cluster 2023-08-27T23:14:14.745247+0000 mgr.y (mgr.14901) 2271 : cluster [DBG] pgmap v2278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:17.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:17 smithi130 bash[51302]: debug 2023-08-27T23:14:17.247+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:17.251634+0000) 2023-08-27T23:14:18.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:18 smithi130 bash[51302]: debug 2023-08-27T23:14:18.247+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:18.251942+0000) 2023-08-27T23:14:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:18 smithi130 bash[27579]: cluster 2023-08-27T23:14:16.745914+0000 mgr.y (mgr.14901) 2272 : cluster [DBG] pgmap v2279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:18 smithi130 bash[27579]: audit 2023-08-27T23:14:17.580148+0000 mon.c (mon.2) 899 : audit [DBG] from='client.? 172.21.15.6:0/1617678082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:18.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:18 smithi006 bash[33918]: cluster 2023-08-27T23:14:16.745914+0000 mgr.y (mgr.14901) 2272 : cluster [DBG] pgmap v2279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:18.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:18 smithi006 bash[33918]: audit 2023-08-27T23:14:17.580148+0000 mon.c (mon.2) 899 : audit [DBG] from='client.? 172.21.15.6:0/1617678082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:18 smithi006 bash[27292]: cluster 2023-08-27T23:14:16.745914+0000 mgr.y (mgr.14901) 2272 : cluster [DBG] pgmap v2279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:18 smithi006 bash[27292]: audit 2023-08-27T23:14:17.580148+0000 mon.c (mon.2) 899 : audit [DBG] from='client.? 172.21.15.6:0/1617678082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:19.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:19 smithi130 bash[51302]: debug 2023-08-27T23:14:19.251+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:19.252222+0000) 2023-08-27T23:14:20.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:20 smithi130 bash[27579]: cluster 2023-08-27T23:14:18.746573+0000 mgr.y (mgr.14901) 2273 : cluster [DBG] pgmap v2280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:20 smithi130 bash[27579]: audit 2023-08-27T23:14:19.995568+0000 mon.c (mon.2) 900 : audit [DBG] from='client.? 172.21.15.6:0/2454125157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:20.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:20 smithi130 bash[51302]: debug 2023-08-27T23:14:20.251+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:20.252521+0000) 2023-08-27T23:14:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:20 smithi006 bash[27292]: cluster 2023-08-27T23:14:18.746573+0000 mgr.y (mgr.14901) 2273 : cluster [DBG] pgmap v2280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:20 smithi006 bash[27292]: audit 2023-08-27T23:14:19.995568+0000 mon.c (mon.2) 900 : audit [DBG] from='client.? 172.21.15.6:0/2454125157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:20 smithi006 bash[33918]: cluster 2023-08-27T23:14:18.746573+0000 mgr.y (mgr.14901) 2273 : cluster [DBG] pgmap v2280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:20 smithi006 bash[33918]: audit 2023-08-27T23:14:19.995568+0000 mon.c (mon.2) 900 : audit [DBG] from='client.? 172.21.15.6:0/2454125157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:14:20.410Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:14:20.412Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:21.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:14:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:14:20] "GET /metrics HTTP/1.1" 200 193093 "" "Prometheus/2.33.4" 2023-08-27T23:14:21.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:21 smithi130 bash[51302]: debug 2023-08-27T23:14:21.251+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:21.252848+0000) 2023-08-27T23:14:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:21 smithi006 bash[56298]: level=error ts=2023-08-27T23:14:21.974Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:21 smithi006 bash[56298]: level=warn ts=2023-08-27T23:14:21.976Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:22.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:22 smithi130 bash[51302]: debug 2023-08-27T23:14:22.247+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:22.253073+0000) 2023-08-27T23:14:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:22 smithi130 bash[27579]: cluster 2023-08-27T23:14:20.747725+0000 mgr.y (mgr.14901) 2274 : cluster [DBG] pgmap v2281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:22 smithi130 bash[27579]: audit 2023-08-27T23:14:22.409237+0000 mon.a (mon.0) 2037 : audit [DBG] from='client.? 172.21.15.6:0/2596275138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:22.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:22 smithi006 bash[27292]: cluster 2023-08-27T23:14:20.747725+0000 mgr.y (mgr.14901) 2274 : cluster [DBG] pgmap v2281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:22.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:22 smithi006 bash[27292]: audit 2023-08-27T23:14:22.409237+0000 mon.a (mon.0) 2037 : audit [DBG] from='client.? 172.21.15.6:0/2596275138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:22 smithi006 bash[33918]: cluster 2023-08-27T23:14:20.747725+0000 mgr.y (mgr.14901) 2274 : cluster [DBG] pgmap v2281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:22 smithi006 bash[33918]: audit 2023-08-27T23:14:22.409237+0000 mon.a (mon.0) 2037 : audit [DBG] from='client.? 172.21.15.6:0/2596275138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:23.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:23 smithi130 bash[51302]: debug 2023-08-27T23:14:23.251+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:23.253390+0000) 2023-08-27T23:14:24.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:14:23.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:24.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:14:23.878Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:24.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:24 smithi130 bash[27579]: cluster 2023-08-27T23:14:22.748438+0000 mgr.y (mgr.14901) 2275 : cluster [DBG] pgmap v2282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:24.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:24 smithi130 bash[51302]: debug 2023-08-27T23:14:24.251+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:24.253712+0000) 2023-08-27T23:14:24.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:24 smithi006 bash[27292]: cluster 2023-08-27T23:14:22.748438+0000 mgr.y (mgr.14901) 2275 : cluster [DBG] pgmap v2282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:24.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:24 smithi006 bash[33918]: cluster 2023-08-27T23:14:22.748438+0000 mgr.y (mgr.14901) 2275 : cluster [DBG] pgmap v2282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:25.509 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:25 smithi130 bash[51302]: debug 2023-08-27T23:14:25.251+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:25.254020+0000) 2023-08-27T23:14:25.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:25 smithi130 bash[27579]: audit 2023-08-27T23:14:24.821031+0000 mon.a (mon.0) 2038 : audit [DBG] from='client.? 172.21.15.6:0/1857554338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:25.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:25 smithi006 bash[27292]: audit 2023-08-27T23:14:24.821031+0000 mon.a (mon.0) 2038 : audit [DBG] from='client.? 172.21.15.6:0/1857554338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:25.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:25 smithi006 bash[33918]: audit 2023-08-27T23:14:24.821031+0000 mon.a (mon.0) 2038 : audit [DBG] from='client.? 172.21.15.6:0/1857554338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:26.512 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:26 smithi130 bash[51302]: debug 2023-08-27T23:14:26.251+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:26.254353+0000) 2023-08-27T23:14:26.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:26 smithi130 bash[27579]: cluster 2023-08-27T23:14:24.749562+0000 mgr.y (mgr.14901) 2276 : cluster [DBG] pgmap v2283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:26.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:26 smithi006 bash[27292]: cluster 2023-08-27T23:14:24.749562+0000 mgr.y (mgr.14901) 2276 : cluster [DBG] pgmap v2283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:26 smithi006 bash[33918]: cluster 2023-08-27T23:14:24.749562+0000 mgr.y (mgr.14901) 2276 : cluster [DBG] pgmap v2283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:27.520 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:27 smithi130 bash[51302]: debug 2023-08-27T23:14:27.251+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:27.254640+0000) 2023-08-27T23:14:27.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:27 smithi130 bash[27579]: audit 2023-08-27T23:14:27.230564+0000 mon.c (mon.2) 901 : audit [DBG] from='client.? 172.21.15.6:0/1397832908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:27.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:27 smithi006 bash[27292]: audit 2023-08-27T23:14:27.230564+0000 mon.c (mon.2) 901 : audit [DBG] from='client.? 172.21.15.6:0/1397832908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:27.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:27 smithi006 bash[33918]: audit 2023-08-27T23:14:27.230564+0000 mon.c (mon.2) 901 : audit [DBG] from='client.? 172.21.15.6:0/1397832908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:28.523 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:28 smithi130 bash[51302]: debug 2023-08-27T23:14:28.251+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:28.254981+0000) 2023-08-27T23:14:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:28 smithi130 bash[27579]: cluster 2023-08-27T23:14:26.750219+0000 mgr.y (mgr.14901) 2277 : cluster [DBG] pgmap v2284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:28.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:28 smithi006 bash[27292]: cluster 2023-08-27T23:14:26.750219+0000 mgr.y (mgr.14901) 2277 : cluster [DBG] pgmap v2284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:28.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:28 smithi006 bash[33918]: cluster 2023-08-27T23:14:26.750219+0000 mgr.y (mgr.14901) 2277 : cluster [DBG] pgmap v2284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:29.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:29 smithi130 bash[51302]: debug 2023-08-27T23:14:29.251+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:29.255240+0000) 2023-08-27T23:14:30.528 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:30 smithi130 bash[51302]: debug 2023-08-27T23:14:30.251+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:30.255533+0000) 2023-08-27T23:14:30.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:30 smithi006 bash[33918]: cluster 2023-08-27T23:14:28.750902+0000 mgr.y (mgr.14901) 2278 : cluster [DBG] pgmap v2285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:30.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:30 smithi006 bash[33918]: audit 2023-08-27T23:14:29.644299+0000 mon.a (mon.0) 2039 : audit [DBG] from='client.? 172.21.15.6:0/939630638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:14:30.410Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:14:30.412Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:30 smithi006 bash[27292]: cluster 2023-08-27T23:14:28.750902+0000 mgr.y (mgr.14901) 2278 : cluster [DBG] pgmap v2285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:30 smithi006 bash[27292]: audit 2023-08-27T23:14:29.644299+0000 mon.a (mon.0) 2039 : audit [DBG] from='client.? 172.21.15.6:0/939630638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:30.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:30 smithi130 bash[27579]: cluster 2023-08-27T23:14:28.750902+0000 mgr.y (mgr.14901) 2278 : cluster [DBG] pgmap v2285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:30 smithi130 bash[27579]: audit 2023-08-27T23:14:29.644299+0000 mon.a (mon.0) 2039 : audit [DBG] from='client.? 172.21.15.6:0/939630638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:14:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:14:30] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:14:31.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:31 smithi130 bash[51302]: debug 2023-08-27T23:14:31.251+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:31.255916+0000) 2023-08-27T23:14:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:31 smithi006 bash[56298]: level=error ts=2023-08-27T23:14:31.975Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:31 smithi006 bash[56298]: level=warn ts=2023-08-27T23:14:31.978Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:32.532 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:32 smithi130 bash[51302]: debug 2023-08-27T23:14:32.251+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:32.256265+0000) 2023-08-27T23:14:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:32 smithi130 bash[27579]: cluster 2023-08-27T23:14:30.752050+0000 mgr.y (mgr.14901) 2279 : cluster [DBG] pgmap v2286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:32 smithi130 bash[27579]: audit 2023-08-27T23:14:32.059192+0000 mon.a (mon.0) 2040 : audit [DBG] from='client.? 172.21.15.6:0/1992942118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:32 smithi006 bash[33918]: cluster 2023-08-27T23:14:30.752050+0000 mgr.y (mgr.14901) 2279 : cluster [DBG] pgmap v2286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:32 smithi006 bash[33918]: audit 2023-08-27T23:14:32.059192+0000 mon.a (mon.0) 2040 : audit [DBG] from='client.? 172.21.15.6:0/1992942118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:32.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:32 smithi006 bash[27292]: cluster 2023-08-27T23:14:30.752050+0000 mgr.y (mgr.14901) 2279 : cluster [DBG] pgmap v2286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:32.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:32 smithi006 bash[27292]: audit 2023-08-27T23:14:32.059192+0000 mon.a (mon.0) 2040 : audit [DBG] from='client.? 172.21.15.6:0/1992942118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:33.542 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:33 smithi130 bash[51302]: debug 2023-08-27T23:14:33.251+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:33.256555+0000) 2023-08-27T23:14:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:33 smithi130 bash[27579]: audit 2023-08-27T23:14:32.735853+0000 mon.b (mon.1) 338 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:14:33.879 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:33 smithi006 bash[27292]: audit 2023-08-27T23:14:32.735853+0000 mon.b (mon.1) 338 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:14:33.879 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:33 smithi006 bash[33918]: audit 2023-08-27T23:14:32.735853+0000 mon.b (mon.1) 338 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:14:34.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:14:33.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:34.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:14:33.879Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:34.548 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:34 smithi130 bash[51302]: debug 2023-08-27T23:14:34.251+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:34.256870+0000) 2023-08-27T23:14:34.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:34 smithi130 bash[27579]: cluster 2023-08-27T23:14:32.752679+0000 mgr.y (mgr.14901) 2280 : cluster [DBG] pgmap v2287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:34.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:34 smithi130 bash[27579]: audit 2023-08-27T23:14:34.482786+0000 mon.a (mon.0) 2041 : audit [DBG] from='client.? 172.21.15.6:0/2068992827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:34.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:34 smithi006 bash[27292]: cluster 2023-08-27T23:14:32.752679+0000 mgr.y (mgr.14901) 2280 : cluster [DBG] pgmap v2287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:34.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:34 smithi006 bash[27292]: audit 2023-08-27T23:14:34.482786+0000 mon.a (mon.0) 2041 : audit [DBG] from='client.? 172.21.15.6:0/2068992827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:34.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:34 smithi006 bash[33918]: cluster 2023-08-27T23:14:32.752679+0000 mgr.y (mgr.14901) 2280 : cluster [DBG] pgmap v2287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:34.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:34 smithi006 bash[33918]: audit 2023-08-27T23:14:34.482786+0000 mon.a (mon.0) 2041 : audit [DBG] from='client.? 172.21.15.6:0/2068992827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:35.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:35 smithi130 bash[51302]: debug 2023-08-27T23:14:35.255+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:35.257205+0000) 2023-08-27T23:14:36.553 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:36 smithi130 bash[51302]: debug 2023-08-27T23:14:36.255+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:36.257504+0000) 2023-08-27T23:14:36.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:36 smithi130 bash[27579]: cluster 2023-08-27T23:14:34.753838+0000 mgr.y (mgr.14901) 2281 : cluster [DBG] pgmap v2288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:36.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:36 smithi006 bash[33918]: cluster 2023-08-27T23:14:34.753838+0000 mgr.y (mgr.14901) 2281 : cluster [DBG] pgmap v2288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:36.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:36 smithi006 bash[27292]: cluster 2023-08-27T23:14:34.753838+0000 mgr.y (mgr.14901) 2281 : cluster [DBG] pgmap v2288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:37.563 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:37 smithi130 bash[51302]: debug 2023-08-27T23:14:37.251+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:37.257781+0000) 2023-08-27T23:14:37.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:37 smithi130 bash[27579]: audit 2023-08-27T23:14:36.897224+0000 mon.c (mon.2) 902 : audit [DBG] from='client.? 172.21.15.6:0/3898813061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:37.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:37 smithi006 bash[33918]: audit 2023-08-27T23:14:36.897224+0000 mon.c (mon.2) 902 : audit [DBG] from='client.? 172.21.15.6:0/3898813061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:37.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:37 smithi006 bash[27292]: audit 2023-08-27T23:14:36.897224+0000 mon.c (mon.2) 902 : audit [DBG] from='client.? 172.21.15.6:0/3898813061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:38.567 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:38 smithi130 bash[51302]: debug 2023-08-27T23:14:38.251+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:38.258006+0000) 2023-08-27T23:14:38.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:38 smithi130 bash[27579]: cluster 2023-08-27T23:14:36.754317+0000 mgr.y (mgr.14901) 2282 : cluster [DBG] pgmap v2289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:38.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:38 smithi006 bash[27292]: cluster 2023-08-27T23:14:36.754317+0000 mgr.y (mgr.14901) 2282 : cluster [DBG] pgmap v2289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:38.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:38 smithi006 bash[33918]: cluster 2023-08-27T23:14:36.754317+0000 mgr.y (mgr.14901) 2282 : cluster [DBG] pgmap v2289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:39.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:39 smithi130 bash[51302]: debug 2023-08-27T23:14:39.251+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:39.258263+0000) 2023-08-27T23:14:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:39 smithi006 bash[33918]: audit 2023-08-27T23:14:38.649126+0000 mon.a (mon.0) 2042 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:14:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:39 smithi006 bash[33918]: cluster 2023-08-27T23:14:38.754961+0000 mgr.y (mgr.14901) 2283 : cluster [DBG] pgmap v2290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:39 smithi006 bash[33918]: audit 2023-08-27T23:14:39.050079+0000 mon.a (mon.0) 2043 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:14:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:39 smithi006 bash[33918]: audit 2023-08-27T23:14:39.307826+0000 mon.c (mon.2) 903 : audit [DBG] from='client.? 172.21.15.6:0/1373929745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:39 smithi006 bash[33918]: audit 2023-08-27T23:14:39.349910+0000 mon.b (mon.1) 339 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:14:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:39 smithi006 bash[33918]: audit 2023-08-27T23:14:39.351884+0000 mon.b (mon.1) 340 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:14:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:39 smithi006 bash[33918]: audit 2023-08-27T23:14:39.363521+0000 mon.a (mon.0) 2044 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:14:39.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:39 smithi006 bash[27292]: audit 2023-08-27T23:14:38.649126+0000 mon.a (mon.0) 2042 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:14:39.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:39 smithi006 bash[27292]: cluster 2023-08-27T23:14:38.754961+0000 mgr.y (mgr.14901) 2283 : cluster [DBG] pgmap v2290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:39.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:39 smithi006 bash[27292]: audit 2023-08-27T23:14:39.050079+0000 mon.a (mon.0) 2043 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:14:39.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:39 smithi006 bash[27292]: audit 2023-08-27T23:14:39.307826+0000 mon.c (mon.2) 903 : audit [DBG] from='client.? 172.21.15.6:0/1373929745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:39.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:39 smithi006 bash[27292]: audit 2023-08-27T23:14:39.349910+0000 mon.b (mon.1) 339 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:14:39.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:39 smithi006 bash[27292]: audit 2023-08-27T23:14:39.351884+0000 mon.b (mon.1) 340 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:14:39.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:39 smithi006 bash[27292]: audit 2023-08-27T23:14:39.363521+0000 mon.a (mon.0) 2044 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:14:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:39 smithi130 bash[27579]: audit 2023-08-27T23:14:38.649126+0000 mon.a (mon.0) 2042 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:14:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:39 smithi130 bash[27579]: cluster 2023-08-27T23:14:38.754961+0000 mgr.y (mgr.14901) 2283 : cluster [DBG] pgmap v2290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:39 smithi130 bash[27579]: audit 2023-08-27T23:14:39.050079+0000 mon.a (mon.0) 2043 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:14:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:39 smithi130 bash[27579]: audit 2023-08-27T23:14:39.307826+0000 mon.c (mon.2) 903 : audit [DBG] from='client.? 172.21.15.6:0/1373929745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:39 smithi130 bash[27579]: audit 2023-08-27T23:14:39.349910+0000 mon.b (mon.1) 339 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:14:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:39 smithi130 bash[27579]: audit 2023-08-27T23:14:39.351884+0000 mon.b (mon.1) 340 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:14:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:39 smithi130 bash[27579]: audit 2023-08-27T23:14:39.363521+0000 mon.a (mon.0) 2044 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:14:40.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:40 smithi130 bash[51302]: debug 2023-08-27T23:14:40.251+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:40.258537+0000) 2023-08-27T23:14:40.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:14:40.411Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:40.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:14:40.413Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:14:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:14:40] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:14:41.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:41 smithi130 bash[51302]: debug 2023-08-27T23:14:41.255+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:41.258841+0000) 2023-08-27T23:14:42.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:41 smithi130 bash[27579]: cluster 2023-08-27T23:14:40.755987+0000 mgr.y (mgr.14901) 2284 : cluster [DBG] pgmap v2291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:42.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:41 smithi130 bash[27579]: audit 2023-08-27T23:14:41.721977+0000 mon.a (mon.0) 2045 : audit [DBG] from='client.? 172.21.15.6:0/3465064938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:42.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:41 smithi006 bash[33918]: cluster 2023-08-27T23:14:40.755987+0000 mgr.y (mgr.14901) 2284 : cluster [DBG] pgmap v2291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:42.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:41 smithi006 bash[33918]: audit 2023-08-27T23:14:41.721977+0000 mon.a (mon.0) 2045 : audit [DBG] from='client.? 172.21.15.6:0/3465064938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:42.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:41 smithi006 bash[27292]: cluster 2023-08-27T23:14:40.755987+0000 mgr.y (mgr.14901) 2284 : cluster [DBG] pgmap v2291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:42.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:41 smithi006 bash[27292]: audit 2023-08-27T23:14:41.721977+0000 mon.a (mon.0) 2045 : audit [DBG] from='client.? 172.21.15.6:0/3465064938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:42.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:41 smithi006 bash[56298]: level=error ts=2023-08-27T23:14:41.976Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:42.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:41 smithi006 bash[56298]: level=warn ts=2023-08-27T23:14:41.978Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:42.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:42 smithi130 bash[51302]: debug 2023-08-27T23:14:42.255+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:42.259172+0000) 2023-08-27T23:14:43.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:43 smithi130 bash[51302]: debug 2023-08-27T23:14:43.259+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:43.259439+0000) 2023-08-27T23:14:44.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:43 smithi130 bash[27579]: cluster 2023-08-27T23:14:42.756680+0000 mgr.y (mgr.14901) 2285 : cluster [DBG] pgmap v2292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:44.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:43 smithi006 bash[33918]: cluster 2023-08-27T23:14:42.756680+0000 mgr.y (mgr.14901) 2285 : cluster [DBG] pgmap v2292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:44.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:43 smithi006 bash[27292]: cluster 2023-08-27T23:14:42.756680+0000 mgr.y (mgr.14901) 2285 : cluster [DBG] pgmap v2292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:14:43.878Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:14:43.880Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:44.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:44 smithi130 bash[51302]: debug 2023-08-27T23:14:44.259+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:44.259706+0000) 2023-08-27T23:14:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:44 smithi130 bash[27579]: audit 2023-08-27T23:14:44.142029+0000 mon.a (mon.0) 2046 : audit [DBG] from='client.? 172.21.15.6:0/304701501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:45.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:44 smithi006 bash[33918]: audit 2023-08-27T23:14:44.142029+0000 mon.a (mon.0) 2046 : audit [DBG] from='client.? 172.21.15.6:0/304701501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:45.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:44 smithi006 bash[27292]: audit 2023-08-27T23:14:44.142029+0000 mon.a (mon.0) 2046 : audit [DBG] from='client.? 172.21.15.6:0/304701501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:45.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:45 smithi130 bash[51302]: debug 2023-08-27T23:14:45.259+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:45.260016+0000) 2023-08-27T23:14:46.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:46 smithi130 bash[27579]: cluster 2023-08-27T23:14:44.757798+0000 mgr.y (mgr.14901) 2286 : cluster [DBG] pgmap v2293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:46.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:46 smithi130 bash[51302]: debug 2023-08-27T23:14:46.255+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:46.260330+0000) 2023-08-27T23:14:46.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:46 smithi006 bash[27292]: cluster 2023-08-27T23:14:44.757798+0000 mgr.y (mgr.14901) 2286 : cluster [DBG] pgmap v2293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:46 smithi006 bash[33918]: cluster 2023-08-27T23:14:44.757798+0000 mgr.y (mgr.14901) 2286 : cluster [DBG] pgmap v2293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:47.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:47 smithi130 bash[27579]: audit 2023-08-27T23:14:46.566423+0000 mon.c (mon.2) 904 : audit [DBG] from='client.? 172.21.15.6:0/664707313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:47.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:47 smithi130 bash[51302]: debug 2023-08-27T23:14:47.255+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:47.260581+0000) 2023-08-27T23:14:47.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:47 smithi006 bash[33918]: audit 2023-08-27T23:14:46.566423+0000 mon.c (mon.2) 904 : audit [DBG] from='client.? 172.21.15.6:0/664707313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:47.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:47 smithi006 bash[27292]: audit 2023-08-27T23:14:46.566423+0000 mon.c (mon.2) 904 : audit [DBG] from='client.? 172.21.15.6:0/664707313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:48.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:48 smithi130 bash[51302]: debug 2023-08-27T23:14:48.255+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:48.260851+0000) 2023-08-27T23:14:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:48 smithi130 bash[27579]: cluster 2023-08-27T23:14:46.758484+0000 mgr.y (mgr.14901) 2287 : cluster [DBG] pgmap v2294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:48.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:48 smithi006 bash[33918]: cluster 2023-08-27T23:14:46.758484+0000 mgr.y (mgr.14901) 2287 : cluster [DBG] pgmap v2294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:48 smithi006 bash[27292]: cluster 2023-08-27T23:14:46.758484+0000 mgr.y (mgr.14901) 2287 : cluster [DBG] pgmap v2294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:49.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:49 smithi130 bash[27579]: audit 2023-08-27T23:14:48.992355+0000 mon.a (mon.0) 2047 : audit [DBG] from='client.? 172.21.15.6:0/511062027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:49.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:49 smithi130 bash[51302]: debug 2023-08-27T23:14:49.259+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:49.261142+0000) 2023-08-27T23:14:49.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:49 smithi006 bash[27292]: audit 2023-08-27T23:14:48.992355+0000 mon.a (mon.0) 2047 : audit [DBG] from='client.? 172.21.15.6:0/511062027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:49.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:49 smithi006 bash[33918]: audit 2023-08-27T23:14:48.992355+0000 mon.a (mon.0) 2047 : audit [DBG] from='client.? 172.21.15.6:0/511062027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:50.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:50 smithi130 bash[27579]: cluster 2023-08-27T23:14:48.759003+0000 mgr.y (mgr.14901) 2288 : cluster [DBG] pgmap v2295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:50.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:50 smithi130 bash[51302]: debug 2023-08-27T23:14:50.255+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:50.261425+0000) 2023-08-27T23:14:50.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:50 smithi006 bash[33918]: cluster 2023-08-27T23:14:48.759003+0000 mgr.y (mgr.14901) 2288 : cluster [DBG] pgmap v2295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:50.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:50 smithi006 bash[27292]: cluster 2023-08-27T23:14:48.759003+0000 mgr.y (mgr.14901) 2288 : cluster [DBG] pgmap v2295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:14:50.411Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:14:50.414Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:14:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:14:50] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:14:51.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:51 smithi130 bash[27579]: audit 2023-08-27T23:14:51.415873+0000 mon.a (mon.0) 2048 : audit [DBG] from='client.? 172.21.15.6:0/1507082329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:51.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:51 smithi130 bash[51302]: debug 2023-08-27T23:14:51.259+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:51.261707+0000) 2023-08-27T23:14:51.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:51 smithi006 bash[27292]: audit 2023-08-27T23:14:51.415873+0000 mon.a (mon.0) 2048 : audit [DBG] from='client.? 172.21.15.6:0/1507082329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:51.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:51 smithi006 bash[33918]: audit 2023-08-27T23:14:51.415873+0000 mon.a (mon.0) 2048 : audit [DBG] from='client.? 172.21.15.6:0/1507082329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:51 smithi006 bash[56298]: level=error ts=2023-08-27T23:14:51.976Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:51 smithi006 bash[56298]: level=warn ts=2023-08-27T23:14:51.978Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:52.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:52 smithi130 bash[27579]: cluster 2023-08-27T23:14:50.760112+0000 mgr.y (mgr.14901) 2289 : cluster [DBG] pgmap v2296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:52.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:52 smithi130 bash[51302]: debug 2023-08-27T23:14:52.255+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:52.262016+0000) 2023-08-27T23:14:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:52 smithi006 bash[33918]: cluster 2023-08-27T23:14:50.760112+0000 mgr.y (mgr.14901) 2289 : cluster [DBG] pgmap v2296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:52 smithi006 bash[27292]: cluster 2023-08-27T23:14:50.760112+0000 mgr.y (mgr.14901) 2289 : cluster [DBG] pgmap v2296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:53.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:53 smithi130 bash[51302]: debug 2023-08-27T23:14:53.259+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:53.262303+0000) 2023-08-27T23:14:54.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:14:53.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:54.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:14:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:14:53.881Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:14:54.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:54 smithi130 bash[51302]: debug 2023-08-27T23:14:54.255+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:54.262618+0000) 2023-08-27T23:14:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:54 smithi130 bash[27579]: cluster 2023-08-27T23:14:52.760830+0000 mgr.y (mgr.14901) 2290 : cluster [DBG] pgmap v2297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:54 smithi130 bash[27579]: audit 2023-08-27T23:14:53.828628+0000 mon.a (mon.0) 2049 : audit [DBG] from='client.? 172.21.15.6:0/2115487686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:54.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:54 smithi006 bash[27292]: cluster 2023-08-27T23:14:52.760830+0000 mgr.y (mgr.14901) 2290 : cluster [DBG] pgmap v2297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:54.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:54 smithi006 bash[27292]: audit 2023-08-27T23:14:53.828628+0000 mon.a (mon.0) 2049 : audit [DBG] from='client.? 172.21.15.6:0/2115487686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:54 smithi006 bash[33918]: cluster 2023-08-27T23:14:52.760830+0000 mgr.y (mgr.14901) 2290 : cluster [DBG] pgmap v2297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:54 smithi006 bash[33918]: audit 2023-08-27T23:14:53.828628+0000 mon.a (mon.0) 2049 : audit [DBG] from='client.? 172.21.15.6:0/2115487686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:55.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:55 smithi130 bash[51302]: debug 2023-08-27T23:14:55.258+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:55.262921+0000) 2023-08-27T23:14:56.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:56 smithi130 bash[51302]: debug 2023-08-27T23:14:56.258+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:56.263210+0000) 2023-08-27T23:14:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:56 smithi130 bash[27579]: cluster 2023-08-27T23:14:54.761955+0000 mgr.y (mgr.14901) 2291 : cluster [DBG] pgmap v2298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:56 smithi130 bash[27579]: audit 2023-08-27T23:14:56.239489+0000 mon.a (mon.0) 2050 : audit [DBG] from='client.? 172.21.15.6:0/1383831845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:56.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:56 smithi006 bash[33918]: cluster 2023-08-27T23:14:54.761955+0000 mgr.y (mgr.14901) 2291 : cluster [DBG] pgmap v2298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:56.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:56 smithi006 bash[33918]: audit 2023-08-27T23:14:56.239489+0000 mon.a (mon.0) 2050 : audit [DBG] from='client.? 172.21.15.6:0/1383831845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:56 smithi006 bash[27292]: cluster 2023-08-27T23:14:54.761955+0000 mgr.y (mgr.14901) 2291 : cluster [DBG] pgmap v2298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:56 smithi006 bash[27292]: audit 2023-08-27T23:14:56.239489+0000 mon.a (mon.0) 2050 : audit [DBG] from='client.? 172.21.15.6:0/1383831845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:57.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:57 smithi130 bash[51302]: debug 2023-08-27T23:14:57.262+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:57.263498+0000) 2023-08-27T23:14:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:58 smithi130 bash[27579]: cluster 2023-08-27T23:14:56.762634+0000 mgr.y (mgr.14901) 2292 : cluster [DBG] pgmap v2299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:58.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:58 smithi130 bash[51302]: debug 2023-08-27T23:14:58.258+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:58.263735+0000) 2023-08-27T23:14:58.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:58 smithi006 bash[33918]: cluster 2023-08-27T23:14:56.762634+0000 mgr.y (mgr.14901) 2292 : cluster [DBG] pgmap v2299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:58 smithi006 bash[27292]: cluster 2023-08-27T23:14:56.762634+0000 mgr.y (mgr.14901) 2292 : cluster [DBG] pgmap v2299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:14:59.520 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:14:59 smithi130 bash[51302]: debug 2023-08-27T23:14:59.262+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:59.264008+0000) 2023-08-27T23:14:59.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:14:59 smithi130 bash[27579]: audit 2023-08-27T23:14:58.649609+0000 mon.a (mon.0) 2051 : audit [DBG] from='client.? 172.21.15.6:0/2203703090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:59.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:14:59 smithi006 bash[33918]: audit 2023-08-27T23:14:58.649609+0000 mon.a (mon.0) 2051 : audit [DBG] from='client.? 172.21.15.6:0/2203703090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:59.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:14:59 smithi006 bash[27292]: audit 2023-08-27T23:14:58.649609+0000 mon.a (mon.0) 2051 : audit [DBG] from='client.? 172.21.15.6:0/2203703090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:00.523 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:00 smithi130 bash[51302]: debug 2023-08-27T23:15:00.262+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:00.264277+0000) 2023-08-27T23:15:00.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:00 smithi006 bash[27292]: cluster 2023-08-27T23:14:58.763290+0000 mgr.y (mgr.14901) 2293 : cluster [DBG] pgmap v2300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:00 smithi006 bash[33918]: cluster 2023-08-27T23:14:58.763290+0000 mgr.y (mgr.14901) 2293 : cluster [DBG] pgmap v2300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:15:00.412Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:15:00.414Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:00 smithi130 bash[27579]: cluster 2023-08-27T23:14:58.763290+0000 mgr.y (mgr.14901) 2293 : cluster [DBG] pgmap v2300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:15:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:15:00] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:15:01.531 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:01 smithi130 bash[51302]: debug 2023-08-27T23:15:01.262+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:01.264572+0000) 2023-08-27T23:15:01.824 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:01 smithi130 bash[27579]: audit 2023-08-27T23:15:01.064974+0000 mon.c (mon.2) 905 : audit [DBG] from='client.? 172.21.15.6:0/11144263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:01.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:01 smithi006 bash[27292]: audit 2023-08-27T23:15:01.064974+0000 mon.c (mon.2) 905 : audit [DBG] from='client.? 172.21.15.6:0/11144263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:01.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:01 smithi006 bash[33918]: audit 2023-08-27T23:15:01.064974+0000 mon.c (mon.2) 905 : audit [DBG] from='client.? 172.21.15.6:0/11144263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:01 smithi006 bash[56298]: level=error ts=2023-08-27T23:15:01.977Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:01 smithi006 bash[56298]: level=warn ts=2023-08-27T23:15:01.979Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:02.534 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:02 smithi130 bash[51302]: debug 2023-08-27T23:15:02.262+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:02.264882+0000) 2023-08-27T23:15:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:02 smithi130 bash[27579]: cluster 2023-08-27T23:15:00.764393+0000 mgr.y (mgr.14901) 2294 : cluster [DBG] pgmap v2301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:02 smithi006 bash[33918]: cluster 2023-08-27T23:15:00.764393+0000 mgr.y (mgr.14901) 2294 : cluster [DBG] pgmap v2301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:02 smithi006 bash[27292]: cluster 2023-08-27T23:15:00.764393+0000 mgr.y (mgr.14901) 2294 : cluster [DBG] pgmap v2301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:03.543 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:03 smithi130 bash[51302]: debug 2023-08-27T23:15:03.262+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:03.265189+0000) 2023-08-27T23:15:03.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:03 smithi130 bash[27579]: audit 2023-08-27T23:15:03.474649+0000 mon.c (mon.2) 906 : audit [DBG] from='client.? 172.21.15.6:0/1022371513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:03.881 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:03 smithi006 bash[33918]: audit 2023-08-27T23:15:03.474649+0000 mon.c (mon.2) 906 : audit [DBG] from='client.? 172.21.15.6:0/1022371513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:03.881 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:03 smithi006 bash[27292]: audit 2023-08-27T23:15:03.474649+0000 mon.c (mon.2) 906 : audit [DBG] from='client.? 172.21.15.6:0/1022371513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:04.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:15:03.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:04.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:15:03.881Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:04.549 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:04 smithi130 bash[51302]: debug 2023-08-27T23:15:04.262+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:04.265446+0000) 2023-08-27T23:15:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:04 smithi130 bash[27579]: cluster 2023-08-27T23:15:02.765065+0000 mgr.y (mgr.14901) 2295 : cluster [DBG] pgmap v2302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:04.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:04 smithi006 bash[27292]: cluster 2023-08-27T23:15:02.765065+0000 mgr.y (mgr.14901) 2295 : cluster [DBG] pgmap v2302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:04 smithi006 bash[33918]: cluster 2023-08-27T23:15:02.765065+0000 mgr.y (mgr.14901) 2295 : cluster [DBG] pgmap v2302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:05.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:05 smithi130 bash[51302]: debug 2023-08-27T23:15:05.262+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:05.265709+0000) 2023-08-27T23:15:06.555 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:06 smithi130 bash[51302]: debug 2023-08-27T23:15:06.262+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:06.266016+0000) 2023-08-27T23:15:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:06 smithi130 bash[27579]: cluster 2023-08-27T23:15:04.766139+0000 mgr.y (mgr.14901) 2296 : cluster [DBG] pgmap v2303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:06 smithi130 bash[27579]: audit 2023-08-27T23:15:05.884035+0000 mon.a (mon.0) 2052 : audit [DBG] from='client.? 172.21.15.6:0/889973024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:06.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:06 smithi006 bash[33918]: cluster 2023-08-27T23:15:04.766139+0000 mgr.y (mgr.14901) 2296 : cluster [DBG] pgmap v2303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:06 smithi006 bash[33918]: audit 2023-08-27T23:15:05.884035+0000 mon.a (mon.0) 2052 : audit [DBG] from='client.? 172.21.15.6:0/889973024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:06 smithi006 bash[27292]: cluster 2023-08-27T23:15:04.766139+0000 mgr.y (mgr.14901) 2296 : cluster [DBG] pgmap v2303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:06 smithi006 bash[27292]: audit 2023-08-27T23:15:05.884035+0000 mon.a (mon.0) 2052 : audit [DBG] from='client.? 172.21.15.6:0/889973024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:07.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:07 smithi130 bash[51302]: debug 2023-08-27T23:15:07.262+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:07.266295+0000) 2023-08-27T23:15:08.559 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:08 smithi130 bash[51302]: debug 2023-08-27T23:15:08.262+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:08.266585+0000) 2023-08-27T23:15:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:08 smithi130 bash[27579]: cluster 2023-08-27T23:15:06.766792+0000 mgr.y (mgr.14901) 2297 : cluster [DBG] pgmap v2304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:08 smithi130 bash[27579]: audit 2023-08-27T23:15:08.295572+0000 mon.a (mon.0) 2053 : audit [DBG] from='client.? 172.21.15.6:0/1370892470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:08.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:08 smithi006 bash[33918]: cluster 2023-08-27T23:15:06.766792+0000 mgr.y (mgr.14901) 2297 : cluster [DBG] pgmap v2304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:08.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:08 smithi006 bash[33918]: audit 2023-08-27T23:15:08.295572+0000 mon.a (mon.0) 2053 : audit [DBG] from='client.? 172.21.15.6:0/1370892470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:08 smithi006 bash[27292]: cluster 2023-08-27T23:15:06.766792+0000 mgr.y (mgr.14901) 2297 : cluster [DBG] pgmap v2304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:08 smithi006 bash[27292]: audit 2023-08-27T23:15:08.295572+0000 mon.a (mon.0) 2053 : audit [DBG] from='client.? 172.21.15.6:0/1370892470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:09.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:09 smithi130 bash[51302]: debug 2023-08-27T23:15:09.266+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:09.266879+0000) 2023-08-27T23:15:10.564 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:10 smithi130 bash[51302]: debug 2023-08-27T23:15:10.266+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:10.267102+0000) 2023-08-27T23:15:10.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:15:10.412Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:10.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:15:10.414Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:10 smithi006 bash[27292]: cluster 2023-08-27T23:15:08.767393+0000 mgr.y (mgr.14901) 2298 : cluster [DBG] pgmap v2305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:10 smithi006 bash[33918]: cluster 2023-08-27T23:15:08.767393+0000 mgr.y (mgr.14901) 2298 : cluster [DBG] pgmap v2305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:10 smithi130 bash[27579]: cluster 2023-08-27T23:15:08.767393+0000 mgr.y (mgr.14901) 2298 : cluster [DBG] pgmap v2305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:15:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:15:10] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:15:11.567 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:11 smithi130 bash[51302]: debug 2023-08-27T23:15:11.266+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:11.267336+0000) 2023-08-27T23:15:11.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:11 smithi130 bash[27579]: audit 2023-08-27T23:15:10.715314+0000 mon.a (mon.0) 2054 : audit [DBG] from='client.? 172.21.15.6:0/1991239242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:11.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:11 smithi006 bash[27292]: audit 2023-08-27T23:15:10.715314+0000 mon.a (mon.0) 2054 : audit [DBG] from='client.? 172.21.15.6:0/1991239242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:11.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:11 smithi006 bash[33918]: audit 2023-08-27T23:15:10.715314+0000 mon.a (mon.0) 2054 : audit [DBG] from='client.? 172.21.15.6:0/1991239242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:11 smithi006 bash[56298]: level=error ts=2023-08-27T23:15:11.978Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:11 smithi006 bash[56298]: level=warn ts=2023-08-27T23:15:11.979Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:12.576 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:12 smithi130 bash[51302]: debug 2023-08-27T23:15:12.266+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:12.267620+0000) 2023-08-27T23:15:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:12 smithi130 bash[27579]: cluster 2023-08-27T23:15:10.768535+0000 mgr.y (mgr.14901) 2299 : cluster [DBG] pgmap v2306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:12 smithi006 bash[27292]: cluster 2023-08-27T23:15:10.768535+0000 mgr.y (mgr.14901) 2299 : cluster [DBG] pgmap v2306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:12 smithi006 bash[33918]: cluster 2023-08-27T23:15:10.768535+0000 mgr.y (mgr.14901) 2299 : cluster [DBG] pgmap v2306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:13.584 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:13 smithi130 bash[51302]: debug 2023-08-27T23:15:13.266+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:13.267966+0000) 2023-08-27T23:15:13.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:13 smithi130 bash[27579]: audit 2023-08-27T23:15:13.128281+0000 mon.a (mon.0) 2055 : audit [DBG] from='client.? 172.21.15.6:0/3633275572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:13.882 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:13 smithi006 bash[33918]: audit 2023-08-27T23:15:13.128281+0000 mon.a (mon.0) 2055 : audit [DBG] from='client.? 172.21.15.6:0/3633275572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:13.882 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:13 smithi006 bash[27292]: audit 2023-08-27T23:15:13.128281+0000 mon.a (mon.0) 2055 : audit [DBG] from='client.? 172.21.15.6:0/3633275572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:15:13.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:15:13.882Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:14.587 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:14 smithi130 bash[51302]: debug 2023-08-27T23:15:14.266+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:14.268241+0000) 2023-08-27T23:15:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:14 smithi130 bash[27579]: cluster 2023-08-27T23:15:12.769343+0000 mgr.y (mgr.14901) 2300 : cluster [DBG] pgmap v2307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:14.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:14 smithi006 bash[33918]: cluster 2023-08-27T23:15:12.769343+0000 mgr.y (mgr.14901) 2300 : cluster [DBG] pgmap v2307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:14.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:14 smithi006 bash[27292]: cluster 2023-08-27T23:15:12.769343+0000 mgr.y (mgr.14901) 2300 : cluster [DBG] pgmap v2307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:15.593 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:15 smithi130 bash[51302]: debug 2023-08-27T23:15:15.266+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:15.268522+0000) 2023-08-27T23:15:15.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:15 smithi130 bash[27579]: audit 2023-08-27T23:15:15.542650+0000 mon.a (mon.0) 2056 : audit [DBG] from='client.? 172.21.15.6:0/3351355766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:15.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:15 smithi006 bash[33918]: audit 2023-08-27T23:15:15.542650+0000 mon.a (mon.0) 2056 : audit [DBG] from='client.? 172.21.15.6:0/3351355766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:15.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:15 smithi006 bash[27292]: audit 2023-08-27T23:15:15.542650+0000 mon.a (mon.0) 2056 : audit [DBG] from='client.? 172.21.15.6:0/3351355766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:16.596 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:16 smithi130 bash[51302]: debug 2023-08-27T23:15:16.266+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:16.268879+0000) 2023-08-27T23:15:16.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:16 smithi130 bash[27579]: cluster 2023-08-27T23:15:14.770517+0000 mgr.y (mgr.14901) 2301 : cluster [DBG] pgmap v2308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:16.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:16 smithi006 bash[33918]: cluster 2023-08-27T23:15:14.770517+0000 mgr.y (mgr.14901) 2301 : cluster [DBG] pgmap v2308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:16.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:16 smithi006 bash[27292]: cluster 2023-08-27T23:15:14.770517+0000 mgr.y (mgr.14901) 2301 : cluster [DBG] pgmap v2308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:17.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:17 smithi130 bash[51302]: debug 2023-08-27T23:15:17.266+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:17.269147+0000) 2023-08-27T23:15:18.602 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:18 smithi130 bash[51302]: debug 2023-08-27T23:15:18.266+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:18.269438+0000) 2023-08-27T23:15:18.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:18 smithi130 bash[27579]: cluster 2023-08-27T23:15:16.771181+0000 mgr.y (mgr.14901) 2302 : cluster [DBG] pgmap v2309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:18 smithi130 bash[27579]: audit 2023-08-27T23:15:17.955538+0000 mon.a (mon.0) 2057 : audit [DBG] from='client.? 172.21.15.6:0/1998852180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:18.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:18 smithi006 bash[33918]: cluster 2023-08-27T23:15:16.771181+0000 mgr.y (mgr.14901) 2302 : cluster [DBG] pgmap v2309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:18.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:18 smithi006 bash[33918]: audit 2023-08-27T23:15:17.955538+0000 mon.a (mon.0) 2057 : audit [DBG] from='client.? 172.21.15.6:0/1998852180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:18 smithi006 bash[27292]: cluster 2023-08-27T23:15:16.771181+0000 mgr.y (mgr.14901) 2302 : cluster [DBG] pgmap v2309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:18 smithi006 bash[27292]: audit 2023-08-27T23:15:17.955538+0000 mon.a (mon.0) 2057 : audit [DBG] from='client.? 172.21.15.6:0/1998852180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:19.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:19 smithi130 bash[51302]: debug 2023-08-27T23:15:19.266+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:19.269719+0000) 2023-08-27T23:15:20.607 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:20 smithi130 bash[51302]: debug 2023-08-27T23:15:20.266+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:20.270024+0000) 2023-08-27T23:15:20.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:20 smithi006 bash[33918]: cluster 2023-08-27T23:15:18.771924+0000 mgr.y (mgr.14901) 2303 : cluster [DBG] pgmap v2310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:20.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:20 smithi006 bash[33918]: audit 2023-08-27T23:15:20.376882+0000 mon.c (mon.2) 907 : audit [DBG] from='client.? 172.21.15.6:0/3924524733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:15:20.414Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:15:20.416Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:20 smithi006 bash[27292]: cluster 2023-08-27T23:15:18.771924+0000 mgr.y (mgr.14901) 2303 : cluster [DBG] pgmap v2310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:20 smithi006 bash[27292]: audit 2023-08-27T23:15:20.376882+0000 mon.c (mon.2) 907 : audit [DBG] from='client.? 172.21.15.6:0/3924524733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:20.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:20 smithi130 bash[27579]: cluster 2023-08-27T23:15:18.771924+0000 mgr.y (mgr.14901) 2303 : cluster [DBG] pgmap v2310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:20.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:20 smithi130 bash[27579]: audit 2023-08-27T23:15:20.376882+0000 mon.c (mon.2) 907 : audit [DBG] from='client.? 172.21.15.6:0/3924524733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:21.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:15:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:15:20] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:15:21.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:21 smithi130 bash[51302]: debug 2023-08-27T23:15:21.266+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:21.270319+0000) 2023-08-27T23:15:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:21 smithi006 bash[56298]: level=error ts=2023-08-27T23:15:21.979Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:21 smithi006 bash[56298]: level=warn ts=2023-08-27T23:15:21.981Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:22.611 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:22 smithi130 bash[51302]: debug 2023-08-27T23:15:22.266+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:22.270573+0000) 2023-08-27T23:15:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:22 smithi130 bash[27579]: cluster 2023-08-27T23:15:20.773128+0000 mgr.y (mgr.14901) 2304 : cluster [DBG] pgmap v2311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:22.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:22 smithi006 bash[33918]: cluster 2023-08-27T23:15:20.773128+0000 mgr.y (mgr.14901) 2304 : cluster [DBG] pgmap v2311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:22 smithi006 bash[27292]: cluster 2023-08-27T23:15:20.773128+0000 mgr.y (mgr.14901) 2304 : cluster [DBG] pgmap v2311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:23.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:23 smithi130 bash[51302]: debug 2023-08-27T23:15:23.270+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:23.270902+0000) 2023-08-27T23:15:23.883 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:23 smithi006 bash[33918]: audit 2023-08-27T23:15:22.797081+0000 mon.c (mon.2) 908 : audit [DBG] from='client.? 172.21.15.6:0/1049437723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:23.884 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:23 smithi006 bash[27292]: audit 2023-08-27T23:15:22.797081+0000 mon.c (mon.2) 908 : audit [DBG] from='client.? 172.21.15.6:0/1049437723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:24.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:23 smithi130 bash[27579]: audit 2023-08-27T23:15:22.797081+0000 mon.c (mon.2) 908 : audit [DBG] from='client.? 172.21.15.6:0/1049437723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:24.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:15:23.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:24.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:15:23.884Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:24.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:24 smithi130 bash[51302]: debug 2023-08-27T23:15:24.270+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:24.271176+0000) 2023-08-27T23:15:24.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:24 smithi006 bash[33918]: cluster 2023-08-27T23:15:22.773619+0000 mgr.y (mgr.14901) 2305 : cluster [DBG] pgmap v2312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:24 smithi006 bash[27292]: cluster 2023-08-27T23:15:22.773619+0000 mgr.y (mgr.14901) 2305 : cluster [DBG] pgmap v2312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:24 smithi130 bash[27579]: cluster 2023-08-27T23:15:22.773619+0000 mgr.y (mgr.14901) 2305 : cluster [DBG] pgmap v2312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:25.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:25 smithi130 bash[51302]: debug 2023-08-27T23:15:25.270+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:25.271452+0000) 2023-08-27T23:15:25.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:25 smithi006 bash[33918]: audit 2023-08-27T23:15:25.221485+0000 mon.a (mon.0) 2058 : audit [DBG] from='client.? 172.21.15.6:0/241102019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:25.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:25 smithi006 bash[27292]: audit 2023-08-27T23:15:25.221485+0000 mon.a (mon.0) 2058 : audit [DBG] from='client.? 172.21.15.6:0/241102019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:26.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:25 smithi130 bash[27579]: audit 2023-08-27T23:15:25.221485+0000 mon.a (mon.0) 2058 : audit [DBG] from='client.? 172.21.15.6:0/241102019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:26.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:26 smithi130 bash[51302]: debug 2023-08-27T23:15:26.270+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:26.271725+0000) 2023-08-27T23:15:26.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:26 smithi006 bash[33918]: cluster 2023-08-27T23:15:24.774674+0000 mgr.y (mgr.14901) 2306 : cluster [DBG] pgmap v2313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:26 smithi006 bash[27292]: cluster 2023-08-27T23:15:24.774674+0000 mgr.y (mgr.14901) 2306 : cluster [DBG] pgmap v2313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:26 smithi130 bash[27579]: cluster 2023-08-27T23:15:24.774674+0000 mgr.y (mgr.14901) 2306 : cluster [DBG] pgmap v2313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:27.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:27 smithi130 bash[51302]: debug 2023-08-27T23:15:27.270+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:27.272020+0000) 2023-08-27T23:15:27.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:27 smithi006 bash[27292]: cluster 2023-08-27T23:15:26.775332+0000 mgr.y (mgr.14901) 2307 : cluster [DBG] pgmap v2314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:27.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:27 smithi006 bash[27292]: audit 2023-08-27T23:15:27.639355+0000 mon.c (mon.2) 909 : audit [DBG] from='client.? 172.21.15.6:0/788153081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:27.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:27 smithi006 bash[33918]: cluster 2023-08-27T23:15:26.775332+0000 mgr.y (mgr.14901) 2307 : cluster [DBG] pgmap v2314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:27.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:27 smithi006 bash[33918]: audit 2023-08-27T23:15:27.639355+0000 mon.c (mon.2) 909 : audit [DBG] from='client.? 172.21.15.6:0/788153081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:28.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:27 smithi130 bash[27579]: cluster 2023-08-27T23:15:26.775332+0000 mgr.y (mgr.14901) 2307 : cluster [DBG] pgmap v2314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:28.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:27 smithi130 bash[27579]: audit 2023-08-27T23:15:27.639355+0000 mon.c (mon.2) 909 : audit [DBG] from='client.? 172.21.15.6:0/788153081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:28.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:28 smithi130 bash[51302]: debug 2023-08-27T23:15:28.270+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:28.272294+0000) 2023-08-27T23:15:29.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:29 smithi130 bash[51302]: debug 2023-08-27T23:15:29.270+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:29.272511+0000) 2023-08-27T23:15:30.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:30 smithi130 bash[51302]: debug 2023-08-27T23:15:30.270+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:30.272796+0000) 2023-08-27T23:15:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:30 smithi130 bash[27579]: cluster 2023-08-27T23:15:28.775975+0000 mgr.y (mgr.14901) 2308 : cluster [DBG] pgmap v2315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:30 smithi130 bash[27579]: audit 2023-08-27T23:15:30.045897+0000 mon.a (mon.0) 2059 : audit [DBG] from='client.? 172.21.15.6:0/202822256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:30.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:30 smithi006 bash[33918]: cluster 2023-08-27T23:15:28.775975+0000 mgr.y (mgr.14901) 2308 : cluster [DBG] pgmap v2315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:30.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:30 smithi006 bash[33918]: audit 2023-08-27T23:15:30.045897+0000 mon.a (mon.0) 2059 : audit [DBG] from='client.? 172.21.15.6:0/202822256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:30 smithi006 bash[27292]: cluster 2023-08-27T23:15:28.775975+0000 mgr.y (mgr.14901) 2308 : cluster [DBG] pgmap v2315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:30 smithi006 bash[27292]: audit 2023-08-27T23:15:30.045897+0000 mon.a (mon.0) 2059 : audit [DBG] from='client.? 172.21.15.6:0/202822256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:15:30.415Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:15:30.416Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:15:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:15:30] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:15:31.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:31 smithi130 bash[51302]: debug 2023-08-27T23:15:31.270+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:31.273067+0000) 2023-08-27T23:15:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:31 smithi006 bash[56298]: level=error ts=2023-08-27T23:15:31.980Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:31 smithi006 bash[56298]: level=warn ts=2023-08-27T23:15:31.982Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:32.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:32 smithi130 bash[27579]: cluster 2023-08-27T23:15:30.777129+0000 mgr.y (mgr.14901) 2309 : cluster [DBG] pgmap v2316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:32.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:32 smithi130 bash[27579]: audit 2023-08-27T23:15:32.459007+0000 mon.a (mon.0) 2060 : audit [DBG] from='client.? 172.21.15.6:0/2780033070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:32.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:32 smithi130 bash[51302]: debug 2023-08-27T23:15:32.270+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:32.273374+0000) 2023-08-27T23:15:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:32 smithi006 bash[33918]: cluster 2023-08-27T23:15:30.777129+0000 mgr.y (mgr.14901) 2309 : cluster [DBG] pgmap v2316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:32 smithi006 bash[33918]: audit 2023-08-27T23:15:32.459007+0000 mon.a (mon.0) 2060 : audit [DBG] from='client.? 172.21.15.6:0/2780033070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:32.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:32 smithi006 bash[27292]: cluster 2023-08-27T23:15:30.777129+0000 mgr.y (mgr.14901) 2309 : cluster [DBG] pgmap v2316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:32.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:32 smithi006 bash[27292]: audit 2023-08-27T23:15:32.459007+0000 mon.a (mon.0) 2060 : audit [DBG] from='client.? 172.21.15.6:0/2780033070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:33.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:33 smithi130 bash[51302]: debug 2023-08-27T23:15:33.270+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:33.273656+0000) 2023-08-27T23:15:34.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:15:33.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:34.232 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:15:33.884Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:34 smithi130 bash[27579]: cluster 2023-08-27T23:15:32.777736+0000 mgr.y (mgr.14901) 2310 : cluster [DBG] pgmap v2317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:34.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:34 smithi130 bash[51302]: debug 2023-08-27T23:15:34.270+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:34.273967+0000) 2023-08-27T23:15:34.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:34 smithi006 bash[27292]: cluster 2023-08-27T23:15:32.777736+0000 mgr.y (mgr.14901) 2310 : cluster [DBG] pgmap v2317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:34.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:34 smithi006 bash[33918]: cluster 2023-08-27T23:15:32.777736+0000 mgr.y (mgr.14901) 2310 : cluster [DBG] pgmap v2317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:35.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:35 smithi130 bash[27579]: audit 2023-08-27T23:15:34.871006+0000 mon.a (mon.0) 2061 : audit [DBG] from='client.? 172.21.15.6:0/1367660409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:35.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:35 smithi130 bash[51302]: debug 2023-08-27T23:15:35.270+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:35.274228+0000) 2023-08-27T23:15:35.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:35 smithi006 bash[27292]: audit 2023-08-27T23:15:34.871006+0000 mon.a (mon.0) 2061 : audit [DBG] from='client.? 172.21.15.6:0/1367660409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:35.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:35 smithi006 bash[33918]: audit 2023-08-27T23:15:34.871006+0000 mon.a (mon.0) 2061 : audit [DBG] from='client.? 172.21.15.6:0/1367660409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:36 smithi130 bash[27579]: cluster 2023-08-27T23:15:34.778817+0000 mgr.y (mgr.14901) 2311 : cluster [DBG] pgmap v2318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:36.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:36 smithi130 bash[51302]: debug 2023-08-27T23:15:36.274+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:36.274496+0000) 2023-08-27T23:15:36.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:36 smithi006 bash[27292]: cluster 2023-08-27T23:15:34.778817+0000 mgr.y (mgr.14901) 2311 : cluster [DBG] pgmap v2318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:36.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:36 smithi006 bash[33918]: cluster 2023-08-27T23:15:34.778817+0000 mgr.y (mgr.14901) 2311 : cluster [DBG] pgmap v2318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:37.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:37 smithi130 bash[27579]: audit 2023-08-27T23:15:37.283986+0000 mon.a (mon.0) 2062 : audit [DBG] from='client.? 172.21.15.6:0/3197775306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:37.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:37 smithi130 bash[51302]: debug 2023-08-27T23:15:37.270+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:37.274767+0000) 2023-08-27T23:15:37.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:37 smithi006 bash[33918]: audit 2023-08-27T23:15:37.283986+0000 mon.a (mon.0) 2062 : audit [DBG] from='client.? 172.21.15.6:0/3197775306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:37.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:37 smithi006 bash[27292]: audit 2023-08-27T23:15:37.283986+0000 mon.a (mon.0) 2062 : audit [DBG] from='client.? 172.21.15.6:0/3197775306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:38.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:38 smithi130 bash[27579]: cluster 2023-08-27T23:15:36.779412+0000 mgr.y (mgr.14901) 2312 : cluster [DBG] pgmap v2319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:39.371 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:38 smithi130 bash[51302]: debug 2023-08-27T23:15:38.274+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:38.275093+0000) 2023-08-27T23:15:39.372 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:38 smithi006 bash[33918]: cluster 2023-08-27T23:15:36.779412+0000 mgr.y (mgr.14901) 2312 : cluster [DBG] pgmap v2319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:39.372 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:38 smithi006 bash[27292]: cluster 2023-08-27T23:15:36.779412+0000 mgr.y (mgr.14901) 2312 : cluster [DBG] pgmap v2319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:39.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:39 smithi130 bash[27579]: audit 2023-08-27T23:15:39.374667+0000 mon.b (mon.1) 341 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:15:39.625 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:39 smithi130 bash[51302]: debug 2023-08-27T23:15:39.274+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:39.275322+0000) 2023-08-27T23:15:39.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:39 smithi006 bash[33918]: audit 2023-08-27T23:15:39.374667+0000 mon.b (mon.1) 341 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:15:39.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:39 smithi006 bash[27292]: audit 2023-08-27T23:15:39.374667+0000 mon.b (mon.1) 341 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:15:40.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:40 smithi130 bash[27579]: cluster 2023-08-27T23:15:38.780055+0000 mgr.y (mgr.14901) 2313 : cluster [DBG] pgmap v2320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:40.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:40 smithi130 bash[27579]: audit 2023-08-27T23:15:39.700215+0000 mon.c (mon.2) 910 : audit [DBG] from='client.? 172.21.15.6:0/3343689898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:40.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:40 smithi130 bash[51302]: debug 2023-08-27T23:15:40.274+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:40.275630+0000) 2023-08-27T23:15:40.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:40 smithi006 bash[33918]: cluster 2023-08-27T23:15:38.780055+0000 mgr.y (mgr.14901) 2313 : cluster [DBG] pgmap v2320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:40 smithi006 bash[33918]: audit 2023-08-27T23:15:39.700215+0000 mon.c (mon.2) 910 : audit [DBG] from='client.? 172.21.15.6:0/3343689898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:40 smithi006 bash[27292]: cluster 2023-08-27T23:15:38.780055+0000 mgr.y (mgr.14901) 2313 : cluster [DBG] pgmap v2320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:40 smithi006 bash[27292]: audit 2023-08-27T23:15:39.700215+0000 mon.c (mon.2) 910 : audit [DBG] from='client.? 172.21.15.6:0/3343689898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:15:40.415Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:15:40.417Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:15:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:15:40] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:15:41.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:41 smithi130 bash[51302]: debug 2023-08-27T23:15:41.274+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:41.275915+0000) 2023-08-27T23:15:42.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:41 smithi006 bash[56298]: level=error ts=2023-08-27T23:15:41.980Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:42.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:41 smithi006 bash[56298]: level=warn ts=2023-08-27T23:15:41.982Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:42 smithi130 bash[27579]: cluster 2023-08-27T23:15:40.781188+0000 mgr.y (mgr.14901) 2314 : cluster [DBG] pgmap v2321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:42 smithi130 bash[27579]: audit 2023-08-27T23:15:42.127094+0000 mon.a (mon.0) 2063 : audit [DBG] from='client.? 172.21.15.6:0/3106995966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:42.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:42 smithi130 bash[51302]: debug 2023-08-27T23:15:42.274+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:42.276160+0000) 2023-08-27T23:15:42.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:42 smithi006 bash[33918]: cluster 2023-08-27T23:15:40.781188+0000 mgr.y (mgr.14901) 2314 : cluster [DBG] pgmap v2321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:42.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:42 smithi006 bash[33918]: audit 2023-08-27T23:15:42.127094+0000 mon.a (mon.0) 2063 : audit [DBG] from='client.? 172.21.15.6:0/3106995966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:42 smithi006 bash[27292]: cluster 2023-08-27T23:15:40.781188+0000 mgr.y (mgr.14901) 2314 : cluster [DBG] pgmap v2321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:42 smithi006 bash[27292]: audit 2023-08-27T23:15:42.127094+0000 mon.a (mon.0) 2063 : audit [DBG] from='client.? 172.21.15.6:0/3106995966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:43.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:43 smithi130 bash[51302]: debug 2023-08-27T23:15:43.274+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:43.276382+0000) 2023-08-27T23:15:44.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:15:43.883Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:44.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:15:43.885Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:44.529 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:44 smithi130 bash[51302]: debug 2023-08-27T23:15:44.274+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:44.276566+0000) 2023-08-27T23:15:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:44 smithi130 bash[27579]: cluster 2023-08-27T23:15:42.781815+0000 mgr.y (mgr.14901) 2315 : cluster [DBG] pgmap v2322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:44.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:44 smithi006 bash[33918]: cluster 2023-08-27T23:15:42.781815+0000 mgr.y (mgr.14901) 2315 : cluster [DBG] pgmap v2322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:44.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:44 smithi006 bash[27292]: cluster 2023-08-27T23:15:42.781815+0000 mgr.y (mgr.14901) 2315 : cluster [DBG] pgmap v2322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:45.535 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:45 smithi130 bash[51302]: debug 2023-08-27T23:15:45.274+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:45.276800+0000) 2023-08-27T23:15:45.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:45 smithi130 bash[27579]: audit 2023-08-27T23:15:44.550853+0000 mon.c (mon.2) 911 : audit [DBG] from='client.? 172.21.15.6:0/2927160889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:45.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:45 smithi130 bash[27579]: audit 2023-08-27T23:15:45.249337+0000 mon.a (mon.0) 2064 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:15:45.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:45 smithi006 bash[27292]: audit 2023-08-27T23:15:44.550853+0000 mon.c (mon.2) 911 : audit [DBG] from='client.? 172.21.15.6:0/2927160889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:45.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:45 smithi006 bash[27292]: audit 2023-08-27T23:15:45.249337+0000 mon.a (mon.0) 2064 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:15:45.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:45 smithi006 bash[33918]: audit 2023-08-27T23:15:44.550853+0000 mon.c (mon.2) 911 : audit [DBG] from='client.? 172.21.15.6:0/2927160889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:45.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:45 smithi006 bash[33918]: audit 2023-08-27T23:15:45.249337+0000 mon.a (mon.0) 2064 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:15:46.539 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:46 smithi130 bash[51302]: debug 2023-08-27T23:15:46.274+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:46.277091+0000) 2023-08-27T23:15:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:46 smithi130 bash[27579]: cluster 2023-08-27T23:15:44.782940+0000 mgr.y (mgr.14901) 2316 : cluster [DBG] pgmap v2323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:46 smithi130 bash[27579]: audit 2023-08-27T23:15:45.722648+0000 mon.a (mon.0) 2065 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:15:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:46 smithi130 bash[27579]: audit 2023-08-27T23:15:46.014791+0000 mon.b (mon.1) 342 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:15:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:46 smithi130 bash[27579]: audit 2023-08-27T23:15:46.017004+0000 mon.b (mon.1) 343 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:15:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:46 smithi130 bash[27579]: audit 2023-08-27T23:15:46.025039+0000 mon.a (mon.0) 2066 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:15:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:46 smithi006 bash[33918]: cluster 2023-08-27T23:15:44.782940+0000 mgr.y (mgr.14901) 2316 : cluster [DBG] pgmap v2323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:46 smithi006 bash[33918]: audit 2023-08-27T23:15:45.722648+0000 mon.a (mon.0) 2065 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:15:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:46 smithi006 bash[33918]: audit 2023-08-27T23:15:46.014791+0000 mon.b (mon.1) 342 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:15:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:46 smithi006 bash[33918]: audit 2023-08-27T23:15:46.017004+0000 mon.b (mon.1) 343 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:15:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:46 smithi006 bash[33918]: audit 2023-08-27T23:15:46.025039+0000 mon.a (mon.0) 2066 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:15:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:46 smithi006 bash[27292]: cluster 2023-08-27T23:15:44.782940+0000 mgr.y (mgr.14901) 2316 : cluster [DBG] pgmap v2323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:46 smithi006 bash[27292]: audit 2023-08-27T23:15:45.722648+0000 mon.a (mon.0) 2065 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:15:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:46 smithi006 bash[27292]: audit 2023-08-27T23:15:46.014791+0000 mon.b (mon.1) 342 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:15:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:46 smithi006 bash[27292]: audit 2023-08-27T23:15:46.017004+0000 mon.b (mon.1) 343 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:15:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:46 smithi006 bash[27292]: audit 2023-08-27T23:15:46.025039+0000 mon.a (mon.0) 2066 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:15:47.547 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:47 smithi130 bash[51302]: debug 2023-08-27T23:15:47.274+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:47.277392+0000) 2023-08-27T23:15:47.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:47 smithi130 bash[27579]: audit 2023-08-27T23:15:46.962246+0000 mon.c (mon.2) 912 : audit [DBG] from='client.? 172.21.15.6:0/1686679932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:47.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:47 smithi006 bash[33918]: audit 2023-08-27T23:15:46.962246+0000 mon.c (mon.2) 912 : audit [DBG] from='client.? 172.21.15.6:0/1686679932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:47.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:47 smithi006 bash[27292]: audit 2023-08-27T23:15:46.962246+0000 mon.c (mon.2) 912 : audit [DBG] from='client.? 172.21.15.6:0/1686679932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:48.550 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:48 smithi130 bash[51302]: debug 2023-08-27T23:15:48.274+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:48.277679+0000) 2023-08-27T23:15:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:48 smithi130 bash[27579]: cluster 2023-08-27T23:15:46.783688+0000 mgr.y (mgr.14901) 2317 : cluster [DBG] pgmap v2324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:48.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:48 smithi006 bash[33918]: cluster 2023-08-27T23:15:46.783688+0000 mgr.y (mgr.14901) 2317 : cluster [DBG] pgmap v2324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:48 smithi006 bash[27292]: cluster 2023-08-27T23:15:46.783688+0000 mgr.y (mgr.14901) 2317 : cluster [DBG] pgmap v2324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:49.561 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:49 smithi130 bash[51302]: debug 2023-08-27T23:15:49.274+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:49.277933+0000) 2023-08-27T23:15:49.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:49 smithi130 bash[27579]: audit 2023-08-27T23:15:49.381199+0000 mon.a (mon.0) 2067 : audit [DBG] from='client.? 172.21.15.6:0/2956096383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:49.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:49 smithi006 bash[27292]: audit 2023-08-27T23:15:49.381199+0000 mon.a (mon.0) 2067 : audit [DBG] from='client.? 172.21.15.6:0/2956096383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:49.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:49 smithi006 bash[33918]: audit 2023-08-27T23:15:49.381199+0000 mon.a (mon.0) 2067 : audit [DBG] from='client.? 172.21.15.6:0/2956096383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:50.564 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:50 smithi130 bash[51302]: debug 2023-08-27T23:15:50.274+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:50.278217+0000) 2023-08-27T23:15:50.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:50 smithi006 bash[33918]: cluster 2023-08-27T23:15:48.784371+0000 mgr.y (mgr.14901) 2318 : cluster [DBG] pgmap v2325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:50 smithi006 bash[27292]: cluster 2023-08-27T23:15:48.784371+0000 mgr.y (mgr.14901) 2318 : cluster [DBG] pgmap v2325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:15:50.415Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:15:50.417Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:50 smithi130 bash[27579]: cluster 2023-08-27T23:15:48.784371+0000 mgr.y (mgr.14901) 2318 : cluster [DBG] pgmap v2325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:15:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:15:50] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:15:51.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:51 smithi130 bash[51302]: debug 2023-08-27T23:15:51.277+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:51.278526+0000) 2023-08-27T23:15:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:51 smithi006 bash[56298]: level=error ts=2023-08-27T23:15:51.981Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:51 smithi006 bash[56298]: level=warn ts=2023-08-27T23:15:51.983Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:52.569 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:52 smithi130 bash[51302]: debug 2023-08-27T23:15:52.273+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:52.278798+0000) 2023-08-27T23:15:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:52 smithi130 bash[27579]: cluster 2023-08-27T23:15:50.785440+0000 mgr.y (mgr.14901) 2319 : cluster [DBG] pgmap v2326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:52 smithi130 bash[27579]: audit 2023-08-27T23:15:51.803055+0000 mon.a (mon.0) 2068 : audit [DBG] from='client.? 172.21.15.6:0/458013304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:52 smithi006 bash[33918]: cluster 2023-08-27T23:15:50.785440+0000 mgr.y (mgr.14901) 2319 : cluster [DBG] pgmap v2326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:52 smithi006 bash[33918]: audit 2023-08-27T23:15:51.803055+0000 mon.a (mon.0) 2068 : audit [DBG] from='client.? 172.21.15.6:0/458013304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:52 smithi006 bash[27292]: cluster 2023-08-27T23:15:50.785440+0000 mgr.y (mgr.14901) 2319 : cluster [DBG] pgmap v2326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:52 smithi006 bash[27292]: audit 2023-08-27T23:15:51.803055+0000 mon.a (mon.0) 2068 : audit [DBG] from='client.? 172.21.15.6:0/458013304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:53.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:53 smithi130 bash[51302]: debug 2023-08-27T23:15:53.277+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:53.279065+0000) 2023-08-27T23:15:54.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:15:53.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:54.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:15:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:15:53.886Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:15:54.574 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:54 smithi130 bash[51302]: debug 2023-08-27T23:15:54.277+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:54.279292+0000) 2023-08-27T23:15:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:54 smithi130 bash[27579]: cluster 2023-08-27T23:15:52.786062+0000 mgr.y (mgr.14901) 2320 : cluster [DBG] pgmap v2327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:54 smithi130 bash[27579]: audit 2023-08-27T23:15:54.217256+0000 mon.c (mon.2) 913 : audit [DBG] from='client.? 172.21.15.6:0/997107432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:54.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:54 smithi006 bash[33918]: cluster 2023-08-27T23:15:52.786062+0000 mgr.y (mgr.14901) 2320 : cluster [DBG] pgmap v2327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:54 smithi006 bash[33918]: audit 2023-08-27T23:15:54.217256+0000 mon.c (mon.2) 913 : audit [DBG] from='client.? 172.21.15.6:0/997107432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:54 smithi006 bash[27292]: cluster 2023-08-27T23:15:52.786062+0000 mgr.y (mgr.14901) 2320 : cluster [DBG] pgmap v2327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:54 smithi006 bash[27292]: audit 2023-08-27T23:15:54.217256+0000 mon.c (mon.2) 913 : audit [DBG] from='client.? 172.21.15.6:0/997107432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:55.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:55 smithi130 bash[51302]: debug 2023-08-27T23:15:55.277+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:55.279553+0000) 2023-08-27T23:15:56.579 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:56 smithi130 bash[51302]: debug 2023-08-27T23:15:56.277+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:56.279833+0000) 2023-08-27T23:15:56.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:56 smithi130 bash[27579]: cluster 2023-08-27T23:15:54.787202+0000 mgr.y (mgr.14901) 2321 : cluster [DBG] pgmap v2328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:56.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:56 smithi006 bash[33918]: cluster 2023-08-27T23:15:54.787202+0000 mgr.y (mgr.14901) 2321 : cluster [DBG] pgmap v2328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:56.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:56 smithi006 bash[27292]: cluster 2023-08-27T23:15:54.787202+0000 mgr.y (mgr.14901) 2321 : cluster [DBG] pgmap v2328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:57.586 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:57 smithi130 bash[51302]: debug 2023-08-27T23:15:57.277+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:57.280108+0000) 2023-08-27T23:15:57.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:57 smithi130 bash[27579]: audit 2023-08-27T23:15:56.628991+0000 mon.c (mon.2) 914 : audit [DBG] from='client.? 172.21.15.6:0/3391634016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:57.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:57 smithi006 bash[27292]: audit 2023-08-27T23:15:56.628991+0000 mon.c (mon.2) 914 : audit [DBG] from='client.? 172.21.15.6:0/3391634016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:57.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:57 smithi006 bash[33918]: audit 2023-08-27T23:15:56.628991+0000 mon.c (mon.2) 914 : audit [DBG] from='client.? 172.21.15.6:0/3391634016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:58.589 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:58 smithi130 bash[51302]: debug 2023-08-27T23:15:58.277+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:58.280398+0000) 2023-08-27T23:15:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:58 smithi130 bash[27579]: cluster 2023-08-27T23:15:56.787882+0000 mgr.y (mgr.14901) 2322 : cluster [DBG] pgmap v2329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:58.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:58 smithi006 bash[33918]: cluster 2023-08-27T23:15:56.787882+0000 mgr.y (mgr.14901) 2322 : cluster [DBG] pgmap v2329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:58 smithi006 bash[27292]: cluster 2023-08-27T23:15:56.787882+0000 mgr.y (mgr.14901) 2322 : cluster [DBG] pgmap v2329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:15:59.597 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:15:59 smithi130 bash[51302]: debug 2023-08-27T23:15:59.277+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:59.280672+0000) 2023-08-27T23:15:59.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:15:59 smithi130 bash[27579]: audit 2023-08-27T23:15:59.043986+0000 mon.c (mon.2) 915 : audit [DBG] from='client.? 172.21.15.6:0/1495665786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:59.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:15:59 smithi006 bash[33918]: audit 2023-08-27T23:15:59.043986+0000 mon.c (mon.2) 915 : audit [DBG] from='client.? 172.21.15.6:0/1495665786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:59.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:15:59 smithi006 bash[27292]: audit 2023-08-27T23:15:59.043986+0000 mon.c (mon.2) 915 : audit [DBG] from='client.? 172.21.15.6:0/1495665786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:00.600 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:00 smithi130 bash[51302]: debug 2023-08-27T23:16:00.277+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:00.280989+0000) 2023-08-27T23:16:00.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:00 smithi006 bash[27292]: cluster 2023-08-27T23:15:58.788385+0000 mgr.y (mgr.14901) 2323 : cluster [DBG] pgmap v2330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:00 smithi006 bash[33918]: cluster 2023-08-27T23:15:58.788385+0000 mgr.y (mgr.14901) 2323 : cluster [DBG] pgmap v2330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:16:00.416Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:16:00.418Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:00 smithi130 bash[27579]: cluster 2023-08-27T23:15:58.788385+0000 mgr.y (mgr.14901) 2323 : cluster [DBG] pgmap v2330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:16:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:16:00] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:16:01.609 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:01 smithi130 bash[51302]: debug 2023-08-27T23:16:01.277+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:01.281307+0000) 2023-08-27T23:16:01.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:01 smithi130 bash[27579]: audit 2023-08-27T23:16:01.460701+0000 mon.a (mon.0) 2069 : audit [DBG] from='client.? 172.21.15.6:0/2394874424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:01.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:01 smithi006 bash[27292]: audit 2023-08-27T23:16:01.460701+0000 mon.a (mon.0) 2069 : audit [DBG] from='client.? 172.21.15.6:0/2394874424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:01.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:01 smithi006 bash[33918]: audit 2023-08-27T23:16:01.460701+0000 mon.a (mon.0) 2069 : audit [DBG] from='client.? 172.21.15.6:0/2394874424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:01 smithi006 bash[56298]: level=error ts=2023-08-27T23:16:01.981Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:01 smithi006 bash[56298]: level=warn ts=2023-08-27T23:16:01.983Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:02.612 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:02 smithi130 bash[51302]: debug 2023-08-27T23:16:02.277+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:02.281615+0000) 2023-08-27T23:16:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:02 smithi130 bash[27579]: cluster 2023-08-27T23:16:00.789485+0000 mgr.y (mgr.14901) 2324 : cluster [DBG] pgmap v2331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:02 smithi006 bash[33918]: cluster 2023-08-27T23:16:00.789485+0000 mgr.y (mgr.14901) 2324 : cluster [DBG] pgmap v2331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:02 smithi006 bash[27292]: cluster 2023-08-27T23:16:00.789485+0000 mgr.y (mgr.14901) 2324 : cluster [DBG] pgmap v2331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:03.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:03 smithi130 bash[51302]: debug 2023-08-27T23:16:03.277+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:03.281916+0000) 2023-08-27T23:16:04.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:16:03.885Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:04.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:16:03.887Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:04.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:04 smithi130 bash[51302]: debug 2023-08-27T23:16:04.281+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:04.282196+0000) 2023-08-27T23:16:04.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:04 smithi006 bash[33918]: cluster 2023-08-27T23:16:02.790135+0000 mgr.y (mgr.14901) 2325 : cluster [DBG] pgmap v2332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:04.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:04 smithi006 bash[33918]: audit 2023-08-27T23:16:03.875181+0000 mon.a (mon.0) 2070 : audit [DBG] from='client.? 172.21.15.6:0/2520544771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:04.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:04 smithi006 bash[27292]: cluster 2023-08-27T23:16:02.790135+0000 mgr.y (mgr.14901) 2325 : cluster [DBG] pgmap v2332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:04.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:04 smithi006 bash[27292]: audit 2023-08-27T23:16:03.875181+0000 mon.a (mon.0) 2070 : audit [DBG] from='client.? 172.21.15.6:0/2520544771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:04 smithi130 bash[27579]: cluster 2023-08-27T23:16:02.790135+0000 mgr.y (mgr.14901) 2325 : cluster [DBG] pgmap v2332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:04 smithi130 bash[27579]: audit 2023-08-27T23:16:03.875181+0000 mon.a (mon.0) 2070 : audit [DBG] from='client.? 172.21.15.6:0/2520544771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:05.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:05 smithi130 bash[51302]: debug 2023-08-27T23:16:05.277+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:05.282474+0000) 2023-08-27T23:16:06.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:06 smithi130 bash[51302]: debug 2023-08-27T23:16:06.277+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:06.282790+0000) 2023-08-27T23:16:06.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:06 smithi006 bash[27292]: cluster 2023-08-27T23:16:04.791268+0000 mgr.y (mgr.14901) 2326 : cluster [DBG] pgmap v2333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:06 smithi006 bash[27292]: audit 2023-08-27T23:16:06.295144+0000 mon.c (mon.2) 916 : audit [DBG] from='client.? 172.21.15.6:0/2849005282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:06 smithi006 bash[33918]: cluster 2023-08-27T23:16:04.791268+0000 mgr.y (mgr.14901) 2326 : cluster [DBG] pgmap v2333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:06 smithi006 bash[33918]: audit 2023-08-27T23:16:06.295144+0000 mon.c (mon.2) 916 : audit [DBG] from='client.? 172.21.15.6:0/2849005282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:06 smithi130 bash[27579]: cluster 2023-08-27T23:16:04.791268+0000 mgr.y (mgr.14901) 2326 : cluster [DBG] pgmap v2333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:06 smithi130 bash[27579]: audit 2023-08-27T23:16:06.295144+0000 mon.c (mon.2) 916 : audit [DBG] from='client.? 172.21.15.6:0/2849005282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:07.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:07 smithi130 bash[51302]: debug 2023-08-27T23:16:07.281+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:07.283065+0000) 2023-08-27T23:16:08.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:08 smithi130 bash[51302]: debug 2023-08-27T23:16:08.281+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:08.283306+0000) 2023-08-27T23:16:08.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:08 smithi006 bash[33918]: cluster 2023-08-27T23:16:06.791930+0000 mgr.y (mgr.14901) 2327 : cluster [DBG] pgmap v2334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:08 smithi006 bash[27292]: cluster 2023-08-27T23:16:06.791930+0000 mgr.y (mgr.14901) 2327 : cluster [DBG] pgmap v2334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:08 smithi130 bash[27579]: cluster 2023-08-27T23:16:06.791930+0000 mgr.y (mgr.14901) 2327 : cluster [DBG] pgmap v2334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:09.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:09 smithi130 bash[51302]: debug 2023-08-27T23:16:09.281+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:09.283524+0000) 2023-08-27T23:16:09.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:09 smithi006 bash[33918]: audit 2023-08-27T23:16:08.710217+0000 mon.c (mon.2) 917 : audit [DBG] from='client.? 172.21.15.6:0/4035038930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:09.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:09 smithi006 bash[27292]: audit 2023-08-27T23:16:08.710217+0000 mon.c (mon.2) 917 : audit [DBG] from='client.? 172.21.15.6:0/4035038930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:10.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:09 smithi130 bash[27579]: audit 2023-08-27T23:16:08.710217+0000 mon.c (mon.2) 917 : audit [DBG] from='client.? 172.21.15.6:0/4035038930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:10.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:10 smithi130 bash[51302]: debug 2023-08-27T23:16:10.281+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:10.283808+0000) 2023-08-27T23:16:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:10 smithi006 bash[33918]: cluster 2023-08-27T23:16:08.792543+0000 mgr.y (mgr.14901) 2328 : cluster [DBG] pgmap v2335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:10 smithi006 bash[27292]: cluster 2023-08-27T23:16:08.792543+0000 mgr.y (mgr.14901) 2328 : cluster [DBG] pgmap v2335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:16:10.418Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:16:10.419Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:10 smithi130 bash[27579]: cluster 2023-08-27T23:16:08.792543+0000 mgr.y (mgr.14901) 2328 : cluster [DBG] pgmap v2335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:16:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:16:10] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:16:11.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:11 smithi130 bash[51302]: debug 2023-08-27T23:16:11.281+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:11.284084+0000) 2023-08-27T23:16:11.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:11 smithi006 bash[27292]: cluster 2023-08-27T23:16:10.793686+0000 mgr.y (mgr.14901) 2329 : cluster [DBG] pgmap v2336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:11.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:11 smithi006 bash[27292]: audit 2023-08-27T23:16:11.139554+0000 mon.c (mon.2) 918 : audit [DBG] from='client.? 172.21.15.6:0/4005915653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:11.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:11 smithi006 bash[33918]: cluster 2023-08-27T23:16:10.793686+0000 mgr.y (mgr.14901) 2329 : cluster [DBG] pgmap v2336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:11.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:11 smithi006 bash[33918]: audit 2023-08-27T23:16:11.139554+0000 mon.c (mon.2) 918 : audit [DBG] from='client.? 172.21.15.6:0/4005915653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:12.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:11 smithi130 bash[27579]: cluster 2023-08-27T23:16:10.793686+0000 mgr.y (mgr.14901) 2329 : cluster [DBG] pgmap v2336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:12.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:11 smithi130 bash[27579]: audit 2023-08-27T23:16:11.139554+0000 mon.c (mon.2) 918 : audit [DBG] from='client.? 172.21.15.6:0/4005915653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:11 smithi006 bash[56298]: level=error ts=2023-08-27T23:16:11.982Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:11 smithi006 bash[56298]: level=warn ts=2023-08-27T23:16:11.984Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:12.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:12 smithi130 bash[51302]: debug 2023-08-27T23:16:12.281+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:12.284434+0000) 2023-08-27T23:16:13.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:13 smithi130 bash[51302]: debug 2023-08-27T23:16:13.281+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:13.284647+0000) 2023-08-27T23:16:14.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:13 smithi130 bash[27579]: cluster 2023-08-27T23:16:12.794386+0000 mgr.y (mgr.14901) 2330 : cluster [DBG] pgmap v2337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:14.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:13 smithi130 bash[27579]: audit 2023-08-27T23:16:13.554941+0000 mon.a (mon.0) 2071 : audit [DBG] from='client.? 172.21.15.6:0/487911415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:14.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:13 smithi006 bash[33918]: cluster 2023-08-27T23:16:12.794386+0000 mgr.y (mgr.14901) 2330 : cluster [DBG] pgmap v2337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:14.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:13 smithi006 bash[33918]: audit 2023-08-27T23:16:13.554941+0000 mon.a (mon.0) 2071 : audit [DBG] from='client.? 172.21.15.6:0/487911415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:16:13.886Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:16:13.888Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:14.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:13 smithi006 bash[27292]: cluster 2023-08-27T23:16:12.794386+0000 mgr.y (mgr.14901) 2330 : cluster [DBG] pgmap v2337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:14.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:13 smithi006 bash[27292]: audit 2023-08-27T23:16:13.554941+0000 mon.a (mon.0) 2071 : audit [DBG] from='client.? 172.21.15.6:0/487911415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:14.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:14 smithi130 bash[51302]: debug 2023-08-27T23:16:14.281+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:14.284958+0000) 2023-08-27T23:16:15.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:15 smithi130 bash[51302]: debug 2023-08-27T23:16:15.281+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:15.285228+0000) 2023-08-27T23:16:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:16 smithi130 bash[27579]: cluster 2023-08-27T23:16:14.795555+0000 mgr.y (mgr.14901) 2331 : cluster [DBG] pgmap v2338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:16 smithi130 bash[27579]: audit 2023-08-27T23:16:15.966603+0000 mon.c (mon.2) 919 : audit [DBG] from='client.? 172.21.15.6:0/361332511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:16.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:16 smithi130 bash[51302]: debug 2023-08-27T23:16:16.281+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:16.285497+0000) 2023-08-27T23:16:16.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:16 smithi006 bash[27292]: cluster 2023-08-27T23:16:14.795555+0000 mgr.y (mgr.14901) 2331 : cluster [DBG] pgmap v2338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:16.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:16 smithi006 bash[27292]: audit 2023-08-27T23:16:15.966603+0000 mon.c (mon.2) 919 : audit [DBG] from='client.? 172.21.15.6:0/361332511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:16 smithi006 bash[33918]: cluster 2023-08-27T23:16:14.795555+0000 mgr.y (mgr.14901) 2331 : cluster [DBG] pgmap v2338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:16 smithi006 bash[33918]: audit 2023-08-27T23:16:15.966603+0000 mon.c (mon.2) 919 : audit [DBG] from='client.? 172.21.15.6:0/361332511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:17.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:17 smithi130 bash[51302]: debug 2023-08-27T23:16:17.285+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:17.285798+0000) 2023-08-27T23:16:18.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:18 smithi130 bash[27579]: cluster 2023-08-27T23:16:16.796218+0000 mgr.y (mgr.14901) 2332 : cluster [DBG] pgmap v2339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:18 smithi130 bash[27579]: audit 2023-08-27T23:16:18.376701+0000 mon.a (mon.0) 2072 : audit [DBG] from='client.? 172.21.15.6:0/2158302390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:18.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:18 smithi130 bash[51302]: debug 2023-08-27T23:16:18.285+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:18.286085+0000) 2023-08-27T23:16:18.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:18 smithi006 bash[33918]: cluster 2023-08-27T23:16:16.796218+0000 mgr.y (mgr.14901) 2332 : cluster [DBG] pgmap v2339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:18.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:18 smithi006 bash[33918]: audit 2023-08-27T23:16:18.376701+0000 mon.a (mon.0) 2072 : audit [DBG] from='client.? 172.21.15.6:0/2158302390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:18 smithi006 bash[27292]: cluster 2023-08-27T23:16:16.796218+0000 mgr.y (mgr.14901) 2332 : cluster [DBG] pgmap v2339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:18 smithi006 bash[27292]: audit 2023-08-27T23:16:18.376701+0000 mon.a (mon.0) 2072 : audit [DBG] from='client.? 172.21.15.6:0/2158302390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:19.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:19 smithi130 bash[51302]: debug 2023-08-27T23:16:19.281+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:19.286360+0000) 2023-08-27T23:16:20.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:20 smithi130 bash[27579]: cluster 2023-08-27T23:16:18.796939+0000 mgr.y (mgr.14901) 2333 : cluster [DBG] pgmap v2340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:20.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:20 smithi130 bash[51302]: debug 2023-08-27T23:16:20.285+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:20.286667+0000) 2023-08-27T23:16:20.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:20 smithi006 bash[33918]: cluster 2023-08-27T23:16:18.796939+0000 mgr.y (mgr.14901) 2333 : cluster [DBG] pgmap v2340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:20 smithi006 bash[27292]: cluster 2023-08-27T23:16:18.796939+0000 mgr.y (mgr.14901) 2333 : cluster [DBG] pgmap v2340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:16:20.418Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:16:20.420Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:21.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:16:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:16:20] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:16:21.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:21 smithi130 bash[27579]: audit 2023-08-27T23:16:20.802526+0000 mon.a (mon.0) 2073 : audit [DBG] from='client.? 172.21.15.6:0/1743083834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:21.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:21 smithi130 bash[51302]: debug 2023-08-27T23:16:21.285+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:21.286987+0000) 2023-08-27T23:16:21.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:21 smithi006 bash[27292]: audit 2023-08-27T23:16:20.802526+0000 mon.a (mon.0) 2073 : audit [DBG] from='client.? 172.21.15.6:0/1743083834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:21.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:21 smithi006 bash[33918]: audit 2023-08-27T23:16:20.802526+0000 mon.a (mon.0) 2073 : audit [DBG] from='client.? 172.21.15.6:0/1743083834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:21 smithi006 bash[56298]: level=error ts=2023-08-27T23:16:21.983Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:21 smithi006 bash[56298]: level=warn ts=2023-08-27T23:16:21.985Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:22.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:22 smithi130 bash[27579]: cluster 2023-08-27T23:16:20.797970+0000 mgr.y (mgr.14901) 2334 : cluster [DBG] pgmap v2341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:22.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:22 smithi130 bash[51302]: debug 2023-08-27T23:16:22.285+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:22.287234+0000) 2023-08-27T23:16:22.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:22 smithi006 bash[27292]: cluster 2023-08-27T23:16:20.797970+0000 mgr.y (mgr.14901) 2334 : cluster [DBG] pgmap v2341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:22.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:22 smithi006 bash[33918]: cluster 2023-08-27T23:16:20.797970+0000 mgr.y (mgr.14901) 2334 : cluster [DBG] pgmap v2341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:23.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:23 smithi130 bash[27579]: audit 2023-08-27T23:16:23.211545+0000 mon.a (mon.0) 2074 : audit [DBG] from='client.? 172.21.15.6:0/2073628594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:23.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:23 smithi130 bash[51302]: debug 2023-08-27T23:16:23.285+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:23.287453+0000) 2023-08-27T23:16:23.889 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:23 smithi006 bash[33918]: audit 2023-08-27T23:16:23.211545+0000 mon.a (mon.0) 2074 : audit [DBG] from='client.? 172.21.15.6:0/2073628594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:23.890 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:23 smithi006 bash[27292]: audit 2023-08-27T23:16:23.211545+0000 mon.a (mon.0) 2074 : audit [DBG] from='client.? 172.21.15.6:0/2073628594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:24.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:16:23.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:24.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:16:23.889Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:24.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:24 smithi130 bash[27579]: cluster 2023-08-27T23:16:22.798616+0000 mgr.y (mgr.14901) 2335 : cluster [DBG] pgmap v2342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:24.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:24 smithi130 bash[51302]: debug 2023-08-27T23:16:24.285+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:24.287759+0000) 2023-08-27T23:16:24.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:24 smithi006 bash[33918]: cluster 2023-08-27T23:16:22.798616+0000 mgr.y (mgr.14901) 2335 : cluster [DBG] pgmap v2342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:24.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:24 smithi006 bash[27292]: cluster 2023-08-27T23:16:22.798616+0000 mgr.y (mgr.14901) 2335 : cluster [DBG] pgmap v2342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:25.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:25 smithi130 bash[51302]: debug 2023-08-27T23:16:25.285+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:25.288042+0000) 2023-08-27T23:16:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:26 smithi130 bash[27579]: cluster 2023-08-27T23:16:24.799753+0000 mgr.y (mgr.14901) 2336 : cluster [DBG] pgmap v2343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:26 smithi130 bash[27579]: audit 2023-08-27T23:16:25.628959+0000 mon.c (mon.2) 920 : audit [DBG] from='client.? 172.21.15.6:0/4192771305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:26.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:26 smithi130 bash[51302]: debug 2023-08-27T23:16:26.285+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:26.288315+0000) 2023-08-27T23:16:26.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:26 smithi006 bash[27292]: cluster 2023-08-27T23:16:24.799753+0000 mgr.y (mgr.14901) 2336 : cluster [DBG] pgmap v2343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:26 smithi006 bash[27292]: audit 2023-08-27T23:16:25.628959+0000 mon.c (mon.2) 920 : audit [DBG] from='client.? 172.21.15.6:0/4192771305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:26 smithi006 bash[33918]: cluster 2023-08-27T23:16:24.799753+0000 mgr.y (mgr.14901) 2336 : cluster [DBG] pgmap v2343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:26 smithi006 bash[33918]: audit 2023-08-27T23:16:25.628959+0000 mon.c (mon.2) 920 : audit [DBG] from='client.? 172.21.15.6:0/4192771305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:27.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:27 smithi130 bash[51302]: debug 2023-08-27T23:16:27.285+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:27.288576+0000) 2023-08-27T23:16:28.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:28 smithi130 bash[27579]: cluster 2023-08-27T23:16:26.800349+0000 mgr.y (mgr.14901) 2337 : cluster [DBG] pgmap v2344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:28 smithi130 bash[27579]: audit 2023-08-27T23:16:28.048942+0000 mon.a (mon.0) 2075 : audit [DBG] from='client.? 172.21.15.6:0/3537278598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:28.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:28 smithi130 bash[51302]: debug 2023-08-27T23:16:28.285+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:28.288866+0000) 2023-08-27T23:16:28.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:28 smithi006 bash[27292]: cluster 2023-08-27T23:16:26.800349+0000 mgr.y (mgr.14901) 2337 : cluster [DBG] pgmap v2344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:28 smithi006 bash[27292]: audit 2023-08-27T23:16:28.048942+0000 mon.a (mon.0) 2075 : audit [DBG] from='client.? 172.21.15.6:0/3537278598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:28 smithi006 bash[33918]: cluster 2023-08-27T23:16:26.800349+0000 mgr.y (mgr.14901) 2337 : cluster [DBG] pgmap v2344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:28 smithi006 bash[33918]: audit 2023-08-27T23:16:28.048942+0000 mon.a (mon.0) 2075 : audit [DBG] from='client.? 172.21.15.6:0/3537278598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:29.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:29 smithi130 bash[51302]: debug 2023-08-27T23:16:29.285+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:29.289152+0000) 2023-08-27T23:16:30.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:30 smithi130 bash[27579]: cluster 2023-08-27T23:16:28.801016+0000 mgr.y (mgr.14901) 2338 : cluster [DBG] pgmap v2345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:30 smithi130 bash[27579]: audit 2023-08-27T23:16:30.467454+0000 mon.a (mon.0) 2076 : audit [DBG] from='client.? 172.21.15.6:0/3943987016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:30.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:30 smithi130 bash[51302]: debug 2023-08-27T23:16:30.285+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:30.289439+0000) 2023-08-27T23:16:30.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:30 smithi006 bash[27292]: cluster 2023-08-27T23:16:28.801016+0000 mgr.y (mgr.14901) 2338 : cluster [DBG] pgmap v2345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:30.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:30 smithi006 bash[27292]: audit 2023-08-27T23:16:30.467454+0000 mon.a (mon.0) 2076 : audit [DBG] from='client.? 172.21.15.6:0/3943987016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:30 smithi006 bash[33918]: cluster 2023-08-27T23:16:28.801016+0000 mgr.y (mgr.14901) 2338 : cluster [DBG] pgmap v2345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:30 smithi006 bash[33918]: audit 2023-08-27T23:16:30.467454+0000 mon.a (mon.0) 2076 : audit [DBG] from='client.? 172.21.15.6:0/3943987016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:16:30.419Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:16:30.421Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:16:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:16:30] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:16:31.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:31 smithi130 bash[51302]: debug 2023-08-27T23:16:31.289+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:31.289716+0000) 2023-08-27T23:16:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:31 smithi006 bash[56298]: level=error ts=2023-08-27T23:16:31.984Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:31 smithi006 bash[56298]: level=warn ts=2023-08-27T23:16:31.986Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:32.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:32 smithi130 bash[27579]: cluster 2023-08-27T23:16:30.802116+0000 mgr.y (mgr.14901) 2339 : cluster [DBG] pgmap v2346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:32.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:32 smithi130 bash[51302]: debug 2023-08-27T23:16:32.285+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:32.290030+0000) 2023-08-27T23:16:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:32 smithi006 bash[33918]: cluster 2023-08-27T23:16:30.802116+0000 mgr.y (mgr.14901) 2339 : cluster [DBG] pgmap v2346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:32.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:32 smithi006 bash[27292]: cluster 2023-08-27T23:16:30.802116+0000 mgr.y (mgr.14901) 2339 : cluster [DBG] pgmap v2346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:33.549 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:33 smithi130 bash[51302]: debug 2023-08-27T23:16:33.285+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:33.290295+0000) 2023-08-27T23:16:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:33 smithi130 bash[27579]: audit 2023-08-27T23:16:32.886376+0000 mon.a (mon.0) 2077 : audit [DBG] from='client.? 172.21.15.6:0/622022039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:33.890 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:33 smithi006 bash[27292]: audit 2023-08-27T23:16:32.886376+0000 mon.a (mon.0) 2077 : audit [DBG] from='client.? 172.21.15.6:0/622022039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:33.890 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:33 smithi006 bash[33918]: audit 2023-08-27T23:16:32.886376+0000 mon.a (mon.0) 2077 : audit [DBG] from='client.? 172.21.15.6:0/622022039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:34.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:16:33.888Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:16:33.890Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:34.552 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:34 smithi130 bash[51302]: debug 2023-08-27T23:16:34.285+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:34.290572+0000) 2023-08-27T23:16:34.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:34 smithi130 bash[27579]: cluster 2023-08-27T23:16:32.802778+0000 mgr.y (mgr.14901) 2340 : cluster [DBG] pgmap v2347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:34.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:34 smithi006 bash[27292]: cluster 2023-08-27T23:16:32.802778+0000 mgr.y (mgr.14901) 2340 : cluster [DBG] pgmap v2347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:34.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:34 smithi006 bash[33918]: cluster 2023-08-27T23:16:32.802778+0000 mgr.y (mgr.14901) 2340 : cluster [DBG] pgmap v2347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:35.563 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:35 smithi130 bash[51302]: debug 2023-08-27T23:16:35.289+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:35.290894+0000) 2023-08-27T23:16:35.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:35 smithi130 bash[27579]: audit 2023-08-27T23:16:35.302141+0000 mon.a (mon.0) 2078 : audit [DBG] from='client.? 172.21.15.6:0/889499324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:35.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:35 smithi006 bash[27292]: audit 2023-08-27T23:16:35.302141+0000 mon.a (mon.0) 2078 : audit [DBG] from='client.? 172.21.15.6:0/889499324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:35.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:35 smithi006 bash[33918]: audit 2023-08-27T23:16:35.302141+0000 mon.a (mon.0) 2078 : audit [DBG] from='client.? 172.21.15.6:0/889499324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:36.565 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:36 smithi130 bash[51302]: debug 2023-08-27T23:16:36.289+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:36.291174+0000) 2023-08-27T23:16:36.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:36 smithi130 bash[27579]: cluster 2023-08-27T23:16:34.803883+0000 mgr.y (mgr.14901) 2341 : cluster [DBG] pgmap v2348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:36.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:36 smithi006 bash[27292]: cluster 2023-08-27T23:16:34.803883+0000 mgr.y (mgr.14901) 2341 : cluster [DBG] pgmap v2348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:36 smithi006 bash[33918]: cluster 2023-08-27T23:16:34.803883+0000 mgr.y (mgr.14901) 2341 : cluster [DBG] pgmap v2348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:37.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:37 smithi130 bash[51302]: debug 2023-08-27T23:16:37.289+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:37.291484+0000) 2023-08-27T23:16:38.570 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:38 smithi130 bash[51302]: debug 2023-08-27T23:16:38.289+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:38.291805+0000) 2023-08-27T23:16:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:38 smithi130 bash[27579]: cluster 2023-08-27T23:16:36.804594+0000 mgr.y (mgr.14901) 2342 : cluster [DBG] pgmap v2349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:38 smithi130 bash[27579]: audit 2023-08-27T23:16:37.730433+0000 mon.a (mon.0) 2079 : audit [DBG] from='client.? 172.21.15.6:0/307948304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:38 smithi006 bash[27292]: cluster 2023-08-27T23:16:36.804594+0000 mgr.y (mgr.14901) 2342 : cluster [DBG] pgmap v2349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:38 smithi006 bash[27292]: audit 2023-08-27T23:16:37.730433+0000 mon.a (mon.0) 2079 : audit [DBG] from='client.? 172.21.15.6:0/307948304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:38.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:38 smithi006 bash[33918]: cluster 2023-08-27T23:16:36.804594+0000 mgr.y (mgr.14901) 2342 : cluster [DBG] pgmap v2349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:38.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:38 smithi006 bash[33918]: audit 2023-08-27T23:16:37.730433+0000 mon.a (mon.0) 2079 : audit [DBG] from='client.? 172.21.15.6:0/307948304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:39.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:39 smithi130 bash[51302]: debug 2023-08-27T23:16:39.289+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:39.292086+0000) 2023-08-27T23:16:40.575 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:40 smithi130 bash[51302]: debug 2023-08-27T23:16:40.289+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:40.292369+0000) 2023-08-27T23:16:40.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:40 smithi006 bash[27292]: cluster 2023-08-27T23:16:38.805271+0000 mgr.y (mgr.14901) 2343 : cluster [DBG] pgmap v2350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:40.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:40 smithi006 bash[27292]: audit 2023-08-27T23:16:40.142373+0000 mon.a (mon.0) 2080 : audit [DBG] from='client.? 172.21.15.6:0/2884656797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:40 smithi006 bash[33918]: cluster 2023-08-27T23:16:38.805271+0000 mgr.y (mgr.14901) 2343 : cluster [DBG] pgmap v2350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:40 smithi006 bash[33918]: audit 2023-08-27T23:16:40.142373+0000 mon.a (mon.0) 2080 : audit [DBG] from='client.? 172.21.15.6:0/2884656797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:16:40.420Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:16:40.422Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:40 smithi130 bash[27579]: cluster 2023-08-27T23:16:38.805271+0000 mgr.y (mgr.14901) 2343 : cluster [DBG] pgmap v2350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:40 smithi130 bash[27579]: audit 2023-08-27T23:16:40.142373+0000 mon.a (mon.0) 2080 : audit [DBG] from='client.? 172.21.15.6:0/2884656797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:16:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:16:40] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:16:41.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:41 smithi130 bash[51302]: debug 2023-08-27T23:16:41.289+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:41.292694+0000) 2023-08-27T23:16:42.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:41 smithi006 bash[56298]: level=error ts=2023-08-27T23:16:41.984Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:42.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:41 smithi006 bash[56298]: level=warn ts=2023-08-27T23:16:41.986Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:42.580 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:42 smithi130 bash[51302]: debug 2023-08-27T23:16:42.289+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:42.293017+0000) 2023-08-27T23:16:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:42 smithi130 bash[27579]: cluster 2023-08-27T23:16:40.806411+0000 mgr.y (mgr.14901) 2344 : cluster [DBG] pgmap v2351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:42 smithi130 bash[27579]: audit 2023-08-27T23:16:42.555746+0000 mon.c (mon.2) 921 : audit [DBG] from='client.? 172.21.15.6:0/2618634670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:42 smithi006 bash[27292]: cluster 2023-08-27T23:16:40.806411+0000 mgr.y (mgr.14901) 2344 : cluster [DBG] pgmap v2351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:42 smithi006 bash[27292]: audit 2023-08-27T23:16:42.555746+0000 mon.c (mon.2) 921 : audit [DBG] from='client.? 172.21.15.6:0/2618634670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:42 smithi006 bash[33918]: cluster 2023-08-27T23:16:40.806411+0000 mgr.y (mgr.14901) 2344 : cluster [DBG] pgmap v2351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:42 smithi006 bash[33918]: audit 2023-08-27T23:16:42.555746+0000 mon.c (mon.2) 921 : audit [DBG] from='client.? 172.21.15.6:0/2618634670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:43.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:43 smithi130 bash[51302]: debug 2023-08-27T23:16:43.293+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:43.293311+0000) 2023-08-27T23:16:44.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:16:43.888Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:16:43.890Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:44.585 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:44 smithi130 bash[51302]: debug 2023-08-27T23:16:44.293+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:44.293679+0000) 2023-08-27T23:16:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:44 smithi130 bash[27579]: cluster 2023-08-27T23:16:42.807109+0000 mgr.y (mgr.14901) 2345 : cluster [DBG] pgmap v2352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:44.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:44 smithi006 bash[27292]: cluster 2023-08-27T23:16:42.807109+0000 mgr.y (mgr.14901) 2345 : cluster [DBG] pgmap v2352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:44 smithi006 bash[33918]: cluster 2023-08-27T23:16:42.807109+0000 mgr.y (mgr.14901) 2345 : cluster [DBG] pgmap v2352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:45.593 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:45 smithi130 bash[51302]: debug 2023-08-27T23:16:45.289+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:45.293988+0000) 2023-08-27T23:16:45.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:45 smithi130 bash[27579]: audit 2023-08-27T23:16:44.973253+0000 mon.a (mon.0) 2081 : audit [DBG] from='client.? 172.21.15.6:0/3053226402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:45.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:45 smithi006 bash[33918]: audit 2023-08-27T23:16:44.973253+0000 mon.a (mon.0) 2081 : audit [DBG] from='client.? 172.21.15.6:0/3053226402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:45.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:45 smithi006 bash[27292]: audit 2023-08-27T23:16:44.973253+0000 mon.a (mon.0) 2081 : audit [DBG] from='client.? 172.21.15.6:0/3053226402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:46.597 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:46 smithi130 bash[51302]: debug 2023-08-27T23:16:46.289+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:46.294214+0000) 2023-08-27T23:16:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:46 smithi130 bash[27579]: cluster 2023-08-27T23:16:44.808178+0000 mgr.y (mgr.14901) 2346 : cluster [DBG] pgmap v2353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:46 smithi130 bash[27579]: audit 2023-08-27T23:16:46.036192+0000 mon.b (mon.1) 344 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:16:46.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:46 smithi006 bash[27292]: cluster 2023-08-27T23:16:44.808178+0000 mgr.y (mgr.14901) 2346 : cluster [DBG] pgmap v2353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:46.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:46 smithi006 bash[27292]: audit 2023-08-27T23:16:46.036192+0000 mon.b (mon.1) 344 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:16:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:46 smithi006 bash[33918]: cluster 2023-08-27T23:16:44.808178+0000 mgr.y (mgr.14901) 2346 : cluster [DBG] pgmap v2353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:46 smithi006 bash[33918]: audit 2023-08-27T23:16:46.036192+0000 mon.b (mon.1) 344 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:16:47.605 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:47 smithi130 bash[51302]: debug 2023-08-27T23:16:47.288+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:47.294439+0000) 2023-08-27T23:16:47.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:47 smithi130 bash[27579]: audit 2023-08-27T23:16:47.401003+0000 mon.c (mon.2) 922 : audit [DBG] from='client.? 172.21.15.6:0/1488023356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:47.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:47 smithi006 bash[27292]: audit 2023-08-27T23:16:47.401003+0000 mon.c (mon.2) 922 : audit [DBG] from='client.? 172.21.15.6:0/1488023356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:47.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:47 smithi006 bash[33918]: audit 2023-08-27T23:16:47.401003+0000 mon.c (mon.2) 922 : audit [DBG] from='client.? 172.21.15.6:0/1488023356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:48.608 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:48 smithi130 bash[51302]: debug 2023-08-27T23:16:48.288+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:48.294647+0000) 2023-08-27T23:16:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:48 smithi130 bash[27579]: cluster 2023-08-27T23:16:46.808829+0000 mgr.y (mgr.14901) 2347 : cluster [DBG] pgmap v2354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:48.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:48 smithi006 bash[27292]: cluster 2023-08-27T23:16:46.808829+0000 mgr.y (mgr.14901) 2347 : cluster [DBG] pgmap v2354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:48 smithi006 bash[33918]: cluster 2023-08-27T23:16:46.808829+0000 mgr.y (mgr.14901) 2347 : cluster [DBG] pgmap v2354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:49.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:49 smithi130 bash[51302]: debug 2023-08-27T23:16:49.292+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:49.294979+0000) 2023-08-27T23:16:50.613 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:50 smithi130 bash[51302]: debug 2023-08-27T23:16:50.292+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:50.295281+0000) 2023-08-27T23:16:50.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:50 smithi006 bash[33918]: cluster 2023-08-27T23:16:48.809494+0000 mgr.y (mgr.14901) 2348 : cluster [DBG] pgmap v2355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:50.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:50 smithi006 bash[33918]: audit 2023-08-27T23:16:49.816954+0000 mon.a (mon.0) 2082 : audit [DBG] from='client.? 172.21.15.6:0/1268020468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:16:50.421Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:16:50.423Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:50 smithi006 bash[27292]: cluster 2023-08-27T23:16:48.809494+0000 mgr.y (mgr.14901) 2348 : cluster [DBG] pgmap v2355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:50 smithi006 bash[27292]: audit 2023-08-27T23:16:49.816954+0000 mon.a (mon.0) 2082 : audit [DBG] from='client.? 172.21.15.6:0/1268020468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:50 smithi130 bash[27579]: cluster 2023-08-27T23:16:48.809494+0000 mgr.y (mgr.14901) 2348 : cluster [DBG] pgmap v2355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:50 smithi130 bash[27579]: audit 2023-08-27T23:16:49.816954+0000 mon.a (mon.0) 2082 : audit [DBG] from='client.? 172.21.15.6:0/1268020468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:16:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:16:50] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:16:51.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:51 smithi130 bash[51302]: debug 2023-08-27T23:16:51.292+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:51.295504+0000) 2023-08-27T23:16:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:51 smithi006 bash[56298]: level=error ts=2023-08-27T23:16:51.985Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:51 smithi006 bash[56298]: level=warn ts=2023-08-27T23:16:51.986Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:52.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:52 smithi130 bash[51302]: debug 2023-08-27T23:16:52.292+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:52.295800+0000) 2023-08-27T23:16:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:52 smithi006 bash[27292]: cluster 2023-08-27T23:16:50.810665+0000 mgr.y (mgr.14901) 2349 : cluster [DBG] pgmap v2356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:52 smithi006 bash[27292]: audit 2023-08-27T23:16:51.954588+0000 mon.a (mon.0) 2083 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:16:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:52 smithi006 bash[27292]: audit 2023-08-27T23:16:52.235838+0000 mon.a (mon.0) 2084 : audit [DBG] from='client.? 172.21.15.6:0/1305990426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:52 smithi006 bash[27292]: audit 2023-08-27T23:16:52.369173+0000 mon.a (mon.0) 2085 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:16:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:52 smithi006 bash[33918]: cluster 2023-08-27T23:16:50.810665+0000 mgr.y (mgr.14901) 2349 : cluster [DBG] pgmap v2356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:52 smithi006 bash[33918]: audit 2023-08-27T23:16:51.954588+0000 mon.a (mon.0) 2083 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:16:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:52 smithi006 bash[33918]: audit 2023-08-27T23:16:52.235838+0000 mon.a (mon.0) 2084 : audit [DBG] from='client.? 172.21.15.6:0/1305990426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:52 smithi006 bash[33918]: audit 2023-08-27T23:16:52.369173+0000 mon.a (mon.0) 2085 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:16:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:52 smithi130 bash[27579]: cluster 2023-08-27T23:16:50.810665+0000 mgr.y (mgr.14901) 2349 : cluster [DBG] pgmap v2356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:52 smithi130 bash[27579]: audit 2023-08-27T23:16:51.954588+0000 mon.a (mon.0) 2083 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:16:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:52 smithi130 bash[27579]: audit 2023-08-27T23:16:52.235838+0000 mon.a (mon.0) 2084 : audit [DBG] from='client.? 172.21.15.6:0/1305990426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:52 smithi130 bash[27579]: audit 2023-08-27T23:16:52.369173+0000 mon.a (mon.0) 2085 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:16:53.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:53 smithi130 bash[51302]: debug 2023-08-27T23:16:53.292+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:53.296081+0000) 2023-08-27T23:16:53.892 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:53 smithi006 bash[27292]: audit 2023-08-27T23:16:52.643960+0000 mon.b (mon.1) 345 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:16:53.892 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:53 smithi006 bash[27292]: audit 2023-08-27T23:16:52.646250+0000 mon.b (mon.1) 346 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:16:53.892 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:53 smithi006 bash[27292]: audit 2023-08-27T23:16:52.653840+0000 mon.a (mon.0) 2086 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:16:53.893 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:53 smithi006 bash[33918]: audit 2023-08-27T23:16:52.643960+0000 mon.b (mon.1) 345 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:16:53.893 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:53 smithi006 bash[33918]: audit 2023-08-27T23:16:52.646250+0000 mon.b (mon.1) 346 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:16:53.893 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:53 smithi006 bash[33918]: audit 2023-08-27T23:16:52.653840+0000 mon.a (mon.0) 2086 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:16:54.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:53 smithi130 bash[27579]: audit 2023-08-27T23:16:52.643960+0000 mon.b (mon.1) 345 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:16:54.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:53 smithi130 bash[27579]: audit 2023-08-27T23:16:52.646250+0000 mon.b (mon.1) 346 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:16:54.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:53 smithi130 bash[27579]: audit 2023-08-27T23:16:52.653840+0000 mon.a (mon.0) 2086 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:16:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:16:53.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:16:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:16:53.892Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:16:54.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:54 smithi130 bash[51302]: debug 2023-08-27T23:16:54.292+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:54.296408+0000) 2023-08-27T23:16:54.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:54 smithi006 bash[33918]: cluster 2023-08-27T23:16:52.811326+0000 mgr.y (mgr.14901) 2350 : cluster [DBG] pgmap v2357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:54 smithi006 bash[27292]: cluster 2023-08-27T23:16:52.811326+0000 mgr.y (mgr.14901) 2350 : cluster [DBG] pgmap v2357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:54 smithi130 bash[27579]: cluster 2023-08-27T23:16:52.811326+0000 mgr.y (mgr.14901) 2350 : cluster [DBG] pgmap v2357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:55.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:55 smithi130 bash[51302]: debug 2023-08-27T23:16:55.292+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:55.296699+0000) 2023-08-27T23:16:55.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:55 smithi006 bash[27292]: audit 2023-08-27T23:16:54.659559+0000 mon.a (mon.0) 2087 : audit [DBG] from='client.? 172.21.15.6:0/3231829663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:55.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:55 smithi006 bash[33918]: audit 2023-08-27T23:16:54.659559+0000 mon.a (mon.0) 2087 : audit [DBG] from='client.? 172.21.15.6:0/3231829663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:56.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:55 smithi130 bash[27579]: audit 2023-08-27T23:16:54.659559+0000 mon.a (mon.0) 2087 : audit [DBG] from='client.? 172.21.15.6:0/3231829663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:56.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:56 smithi130 bash[51302]: debug 2023-08-27T23:16:56.296+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:56.296989+0000) 2023-08-27T23:16:56.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:56 smithi006 bash[33918]: cluster 2023-08-27T23:16:54.812465+0000 mgr.y (mgr.14901) 2351 : cluster [DBG] pgmap v2358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:56 smithi006 bash[27292]: cluster 2023-08-27T23:16:54.812465+0000 mgr.y (mgr.14901) 2351 : cluster [DBG] pgmap v2358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:56 smithi130 bash[27579]: cluster 2023-08-27T23:16:54.812465+0000 mgr.y (mgr.14901) 2351 : cluster [DBG] pgmap v2358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:57.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:57 smithi130 bash[51302]: debug 2023-08-27T23:16:57.296+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:57.297293+0000) 2023-08-27T23:16:57.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:57 smithi006 bash[27292]: audit 2023-08-27T23:16:57.078384+0000 mon.c (mon.2) 923 : audit [DBG] from='client.? 172.21.15.6:0/3150796527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:57.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:57 smithi006 bash[33918]: audit 2023-08-27T23:16:57.078384+0000 mon.c (mon.2) 923 : audit [DBG] from='client.? 172.21.15.6:0/3150796527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:58.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:57 smithi130 bash[27579]: audit 2023-08-27T23:16:57.078384+0000 mon.c (mon.2) 923 : audit [DBG] from='client.? 172.21.15.6:0/3150796527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:58.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:58 smithi130 bash[51302]: debug 2023-08-27T23:16:58.296+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:58.297611+0000) 2023-08-27T23:16:58.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:16:58 smithi006 bash[33918]: cluster 2023-08-27T23:16:56.813071+0000 mgr.y (mgr.14901) 2352 : cluster [DBG] pgmap v2359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:16:58 smithi006 bash[27292]: cluster 2023-08-27T23:16:56.813071+0000 mgr.y (mgr.14901) 2352 : cluster [DBG] pgmap v2359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:16:58 smithi130 bash[27579]: cluster 2023-08-27T23:16:56.813071+0000 mgr.y (mgr.14901) 2352 : cluster [DBG] pgmap v2359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:16:59.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:16:59 smithi130 bash[51302]: debug 2023-08-27T23:16:59.292+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:59.297880+0000) 2023-08-27T23:17:00.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:00 smithi130 bash[51302]: debug 2023-08-27T23:17:00.292+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:00.298098+0000) 2023-08-27T23:17:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:00 smithi130 bash[27579]: cluster 2023-08-27T23:16:58.813706+0000 mgr.y (mgr.14901) 2353 : cluster [DBG] pgmap v2360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:00 smithi130 bash[27579]: audit 2023-08-27T23:16:59.503536+0000 mon.a (mon.0) 2088 : audit [DBG] from='client.? 172.21.15.6:0/3786934569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:00.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:17:00.422Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:00.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:17:00.424Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:00 smithi006 bash[33918]: cluster 2023-08-27T23:16:58.813706+0000 mgr.y (mgr.14901) 2353 : cluster [DBG] pgmap v2360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:00 smithi006 bash[33918]: audit 2023-08-27T23:16:59.503536+0000 mon.a (mon.0) 2088 : audit [DBG] from='client.? 172.21.15.6:0/3786934569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:00 smithi006 bash[27292]: cluster 2023-08-27T23:16:58.813706+0000 mgr.y (mgr.14901) 2353 : cluster [DBG] pgmap v2360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:00 smithi006 bash[27292]: audit 2023-08-27T23:16:59.503536+0000 mon.a (mon.0) 2088 : audit [DBG] from='client.? 172.21.15.6:0/3786934569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:01.044 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:17:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:17:00] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T23:17:01.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:01 smithi130 bash[51302]: debug 2023-08-27T23:17:01.296+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:01.298404+0000) 2023-08-27T23:17:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:01 smithi006 bash[56298]: level=error ts=2023-08-27T23:17:01.985Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:01 smithi006 bash[56298]: level=warn ts=2023-08-27T23:17:01.987Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:02.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:02 smithi130 bash[27579]: cluster 2023-08-27T23:17:00.814869+0000 mgr.y (mgr.14901) 2354 : cluster [DBG] pgmap v2361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:02 smithi130 bash[27579]: audit 2023-08-27T23:17:01.912015+0000 mon.a (mon.0) 2089 : audit [DBG] from='client.? 172.21.15.6:0/386235329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:02.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:02 smithi130 bash[51302]: debug 2023-08-27T23:17:02.292+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:02.298711+0000) 2023-08-27T23:17:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:02 smithi006 bash[27292]: cluster 2023-08-27T23:17:00.814869+0000 mgr.y (mgr.14901) 2354 : cluster [DBG] pgmap v2361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:02 smithi006 bash[27292]: audit 2023-08-27T23:17:01.912015+0000 mon.a (mon.0) 2089 : audit [DBG] from='client.? 172.21.15.6:0/386235329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:02 smithi006 bash[33918]: cluster 2023-08-27T23:17:00.814869+0000 mgr.y (mgr.14901) 2354 : cluster [DBG] pgmap v2361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:02 smithi006 bash[33918]: audit 2023-08-27T23:17:01.912015+0000 mon.a (mon.0) 2089 : audit [DBG] from='client.? 172.21.15.6:0/386235329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:03.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:03 smithi130 bash[51302]: debug 2023-08-27T23:17:03.296+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:03.299004+0000) 2023-08-27T23:17:04.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:17:03.891Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:17:03.893Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:04.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:04 smithi130 bash[51302]: debug 2023-08-27T23:17:04.296+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:04.299280+0000) 2023-08-27T23:17:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:04 smithi130 bash[27579]: cluster 2023-08-27T23:17:02.815530+0000 mgr.y (mgr.14901) 2355 : cluster [DBG] pgmap v2362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:04 smithi130 bash[27579]: audit 2023-08-27T23:17:04.334040+0000 mon.a (mon.0) 2090 : audit [DBG] from='client.? 172.21.15.6:0/3780321750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:04.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:04 smithi006 bash[33918]: cluster 2023-08-27T23:17:02.815530+0000 mgr.y (mgr.14901) 2355 : cluster [DBG] pgmap v2362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:04 smithi006 bash[33918]: audit 2023-08-27T23:17:04.334040+0000 mon.a (mon.0) 2090 : audit [DBG] from='client.? 172.21.15.6:0/3780321750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:04.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:04 smithi006 bash[27292]: cluster 2023-08-27T23:17:02.815530+0000 mgr.y (mgr.14901) 2355 : cluster [DBG] pgmap v2362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:04.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:04 smithi006 bash[27292]: audit 2023-08-27T23:17:04.334040+0000 mon.a (mon.0) 2090 : audit [DBG] from='client.? 172.21.15.6:0/3780321750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:05.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:05 smithi130 bash[51302]: debug 2023-08-27T23:17:05.296+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:05.299580+0000) 2023-08-27T23:17:06.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:06 smithi130 bash[27579]: cluster 2023-08-27T23:17:04.816659+0000 mgr.y (mgr.14901) 2356 : cluster [DBG] pgmap v2363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:06.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:06 smithi130 bash[51302]: debug 2023-08-27T23:17:06.296+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:06.299908+0000) 2023-08-27T23:17:06.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:06 smithi006 bash[33918]: cluster 2023-08-27T23:17:04.816659+0000 mgr.y (mgr.14901) 2356 : cluster [DBG] pgmap v2363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:06 smithi006 bash[27292]: cluster 2023-08-27T23:17:04.816659+0000 mgr.y (mgr.14901) 2356 : cluster [DBG] pgmap v2363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:07.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:07 smithi130 bash[27579]: audit 2023-08-27T23:17:06.752266+0000 mon.a (mon.0) 2091 : audit [DBG] from='client.? 172.21.15.6:0/3911138775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:07.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:07 smithi130 bash[51302]: debug 2023-08-27T23:17:07.296+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:07.300168+0000) 2023-08-27T23:17:07.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:07 smithi006 bash[33918]: audit 2023-08-27T23:17:06.752266+0000 mon.a (mon.0) 2091 : audit [DBG] from='client.? 172.21.15.6:0/3911138775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:07.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:07 smithi006 bash[27292]: audit 2023-08-27T23:17:06.752266+0000 mon.a (mon.0) 2091 : audit [DBG] from='client.? 172.21.15.6:0/3911138775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:08.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:08 smithi130 bash[27579]: cluster 2023-08-27T23:17:06.817289+0000 mgr.y (mgr.14901) 2357 : cluster [DBG] pgmap v2364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:08.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:08 smithi130 bash[51302]: debug 2023-08-27T23:17:08.296+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:08.300435+0000) 2023-08-27T23:17:08.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:08 smithi006 bash[27292]: cluster 2023-08-27T23:17:06.817289+0000 mgr.y (mgr.14901) 2357 : cluster [DBG] pgmap v2364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:08 smithi006 bash[33918]: cluster 2023-08-27T23:17:06.817289+0000 mgr.y (mgr.14901) 2357 : cluster [DBG] pgmap v2364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:09.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:09 smithi130 bash[51302]: debug 2023-08-27T23:17:09.296+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:09.300703+0000) 2023-08-27T23:17:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:09 smithi130 bash[27579]: audit 2023-08-27T23:17:09.167620+0000 mon.c (mon.2) 924 : audit [DBG] from='client.? 172.21.15.6:0/4117149404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:09.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:09 smithi006 bash[27292]: audit 2023-08-27T23:17:09.167620+0000 mon.c (mon.2) 924 : audit [DBG] from='client.? 172.21.15.6:0/4117149404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:09.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:09 smithi006 bash[33918]: audit 2023-08-27T23:17:09.167620+0000 mon.c (mon.2) 924 : audit [DBG] from='client.? 172.21.15.6:0/4117149404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:10.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:10 smithi130 bash[27579]: cluster 2023-08-27T23:17:08.817936+0000 mgr.y (mgr.14901) 2358 : cluster [DBG] pgmap v2365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:10.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:10 smithi130 bash[51302]: debug 2023-08-27T23:17:10.300+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:10.301023+0000) 2023-08-27T23:17:10.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:10 smithi006 bash[27292]: cluster 2023-08-27T23:17:08.817936+0000 mgr.y (mgr.14901) 2358 : cluster [DBG] pgmap v2365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:10 smithi006 bash[33918]: cluster 2023-08-27T23:17:08.817936+0000 mgr.y (mgr.14901) 2358 : cluster [DBG] pgmap v2365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:17:10.422Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:17:10.425Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:17:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:17:10] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T23:17:11.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:11 smithi130 bash[51302]: debug 2023-08-27T23:17:11.296+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:11.301316+0000) 2023-08-27T23:17:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:11 smithi006 bash[56298]: level=error ts=2023-08-27T23:17:11.985Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:11 smithi006 bash[56298]: level=warn ts=2023-08-27T23:17:11.987Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:12.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:12 smithi130 bash[27579]: cluster 2023-08-27T23:17:10.819082+0000 mgr.y (mgr.14901) 2359 : cluster [DBG] pgmap v2366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:12 smithi130 bash[27579]: audit 2023-08-27T23:17:11.588773+0000 mon.c (mon.2) 925 : audit [DBG] from='client.? 172.21.15.6:0/3270101852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:12.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:12 smithi130 bash[51302]: debug 2023-08-27T23:17:12.296+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:12.301539+0000) 2023-08-27T23:17:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:12 smithi006 bash[33918]: cluster 2023-08-27T23:17:10.819082+0000 mgr.y (mgr.14901) 2359 : cluster [DBG] pgmap v2366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:12 smithi006 bash[33918]: audit 2023-08-27T23:17:11.588773+0000 mon.c (mon.2) 925 : audit [DBG] from='client.? 172.21.15.6:0/3270101852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:12 smithi006 bash[27292]: cluster 2023-08-27T23:17:10.819082+0000 mgr.y (mgr.14901) 2359 : cluster [DBG] pgmap v2366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:12 smithi006 bash[27292]: audit 2023-08-27T23:17:11.588773+0000 mon.c (mon.2) 925 : audit [DBG] from='client.? 172.21.15.6:0/3270101852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:13.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:13 smithi130 bash[51302]: debug 2023-08-27T23:17:13.300+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:13.301850+0000) 2023-08-27T23:17:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:17:13.891Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:17:13.892Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:14.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:14 smithi130 bash[51302]: debug 2023-08-27T23:17:14.296+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:14.302056+0000) 2023-08-27T23:17:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:14 smithi130 bash[27579]: cluster 2023-08-27T23:17:12.819753+0000 mgr.y (mgr.14901) 2360 : cluster [DBG] pgmap v2367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:14 smithi130 bash[27579]: audit 2023-08-27T23:17:13.997893+0000 mon.a (mon.0) 2092 : audit [DBG] from='client.? 172.21.15.6:0/2835355493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:14 smithi006 bash[33918]: cluster 2023-08-27T23:17:12.819753+0000 mgr.y (mgr.14901) 2360 : cluster [DBG] pgmap v2367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:14 smithi006 bash[33918]: audit 2023-08-27T23:17:13.997893+0000 mon.a (mon.0) 2092 : audit [DBG] from='client.? 172.21.15.6:0/2835355493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:14 smithi006 bash[27292]: cluster 2023-08-27T23:17:12.819753+0000 mgr.y (mgr.14901) 2360 : cluster [DBG] pgmap v2367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:14 smithi006 bash[27292]: audit 2023-08-27T23:17:13.997893+0000 mon.a (mon.0) 2092 : audit [DBG] from='client.? 172.21.15.6:0/2835355493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:15.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:15 smithi130 bash[51302]: debug 2023-08-27T23:17:15.300+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:15.302401+0000) 2023-08-27T23:17:16.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:16 smithi130 bash[51302]: debug 2023-08-27T23:17:16.300+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:16.302720+0000) 2023-08-27T23:17:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:16 smithi130 bash[27579]: cluster 2023-08-27T23:17:14.820929+0000 mgr.y (mgr.14901) 2361 : cluster [DBG] pgmap v2368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:16 smithi130 bash[27579]: audit 2023-08-27T23:17:16.409441+0000 mon.c (mon.2) 926 : audit [DBG] from='client.? 172.21.15.6:0/3341391721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:16.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:16 smithi006 bash[33918]: cluster 2023-08-27T23:17:14.820929+0000 mgr.y (mgr.14901) 2361 : cluster [DBG] pgmap v2368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:16 smithi006 bash[33918]: audit 2023-08-27T23:17:16.409441+0000 mon.c (mon.2) 926 : audit [DBG] from='client.? 172.21.15.6:0/3341391721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:16 smithi006 bash[27292]: cluster 2023-08-27T23:17:14.820929+0000 mgr.y (mgr.14901) 2361 : cluster [DBG] pgmap v2368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:16 smithi006 bash[27292]: audit 2023-08-27T23:17:16.409441+0000 mon.c (mon.2) 926 : audit [DBG] from='client.? 172.21.15.6:0/3341391721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:17.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:17 smithi130 bash[51302]: debug 2023-08-27T23:17:17.300+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:17.303023+0000) 2023-08-27T23:17:18.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:18 smithi130 bash[27579]: cluster 2023-08-27T23:17:16.821618+0000 mgr.y (mgr.14901) 2362 : cluster [DBG] pgmap v2369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:18.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:18 smithi130 bash[51302]: debug 2023-08-27T23:17:18.300+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:18.303325+0000) 2023-08-27T23:17:18.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:18 smithi006 bash[33918]: cluster 2023-08-27T23:17:16.821618+0000 mgr.y (mgr.14901) 2362 : cluster [DBG] pgmap v2369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:18 smithi006 bash[27292]: cluster 2023-08-27T23:17:16.821618+0000 mgr.y (mgr.14901) 2362 : cluster [DBG] pgmap v2369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:19.562 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:19 smithi130 bash[51302]: debug 2023-08-27T23:17:19.300+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:19.303617+0000) 2023-08-27T23:17:19.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:19 smithi130 bash[27579]: audit 2023-08-27T23:17:18.831568+0000 mon.a (mon.0) 2093 : audit [DBG] from='client.? 172.21.15.6:0/3647713312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:19.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:19 smithi006 bash[27292]: audit 2023-08-27T23:17:18.831568+0000 mon.a (mon.0) 2093 : audit [DBG] from='client.? 172.21.15.6:0/3647713312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:19 smithi006 bash[33918]: audit 2023-08-27T23:17:18.831568+0000 mon.a (mon.0) 2093 : audit [DBG] from='client.? 172.21.15.6:0/3647713312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:20.566 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:20 smithi130 bash[51302]: debug 2023-08-27T23:17:20.300+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:20.303925+0000) 2023-08-27T23:17:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:20 smithi006 bash[27292]: cluster 2023-08-27T23:17:18.822078+0000 mgr.y (mgr.14901) 2363 : cluster [DBG] pgmap v2370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:20 smithi006 bash[33918]: cluster 2023-08-27T23:17:18.822078+0000 mgr.y (mgr.14901) 2363 : cluster [DBG] pgmap v2370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:17:20.423Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:17:20.425Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:20.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:20 smithi130 bash[27579]: cluster 2023-08-27T23:17:18.822078+0000 mgr.y (mgr.14901) 2363 : cluster [DBG] pgmap v2370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:21.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:17:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:17:20] "GET /metrics HTTP/1.1" 200 193084 "" "Prometheus/2.33.4" 2023-08-27T23:17:21.574 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:21 smithi130 bash[51302]: debug 2023-08-27T23:17:21.300+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:21.304212+0000) 2023-08-27T23:17:21.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:21 smithi130 bash[27579]: audit 2023-08-27T23:17:21.246619+0000 mon.a (mon.0) 2094 : audit [DBG] from='client.? 172.21.15.6:0/1506915217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:21.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:21 smithi006 bash[27292]: audit 2023-08-27T23:17:21.246619+0000 mon.a (mon.0) 2094 : audit [DBG] from='client.? 172.21.15.6:0/1506915217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:21.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:21 smithi006 bash[33918]: audit 2023-08-27T23:17:21.246619+0000 mon.a (mon.0) 2094 : audit [DBG] from='client.? 172.21.15.6:0/1506915217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:21 smithi006 bash[56298]: level=error ts=2023-08-27T23:17:21.985Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:21 smithi006 bash[56298]: level=warn ts=2023-08-27T23:17:21.988Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:22.576 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:22 smithi130 bash[51302]: debug 2023-08-27T23:17:22.300+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:22.304510+0000) 2023-08-27T23:17:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:22 smithi130 bash[27579]: cluster 2023-08-27T23:17:20.823148+0000 mgr.y (mgr.14901) 2364 : cluster [DBG] pgmap v2371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:22.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:22 smithi006 bash[27292]: cluster 2023-08-27T23:17:20.823148+0000 mgr.y (mgr.14901) 2364 : cluster [DBG] pgmap v2371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:22.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:22 smithi006 bash[33918]: cluster 2023-08-27T23:17:20.823148+0000 mgr.y (mgr.14901) 2364 : cluster [DBG] pgmap v2371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:23.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:23 smithi130 bash[51302]: debug 2023-08-27T23:17:23.300+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:23.304783+0000) 2023-08-27T23:17:24.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:17:23.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:24.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:17:23.894Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:24.584 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:24 smithi130 bash[51302]: debug 2023-08-27T23:17:24.304+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:24.305088+0000) 2023-08-27T23:17:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:24 smithi130 bash[27579]: cluster 2023-08-27T23:17:22.823808+0000 mgr.y (mgr.14901) 2365 : cluster [DBG] pgmap v2372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:24 smithi130 bash[27579]: audit 2023-08-27T23:17:23.664912+0000 mon.a (mon.0) 2095 : audit [DBG] from='client.? 172.21.15.6:0/2037047703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:24.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:24 smithi006 bash[27292]: cluster 2023-08-27T23:17:22.823808+0000 mgr.y (mgr.14901) 2365 : cluster [DBG] pgmap v2372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:24.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:24 smithi006 bash[27292]: audit 2023-08-27T23:17:23.664912+0000 mon.a (mon.0) 2095 : audit [DBG] from='client.? 172.21.15.6:0/2037047703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:24.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:24 smithi006 bash[33918]: cluster 2023-08-27T23:17:22.823808+0000 mgr.y (mgr.14901) 2365 : cluster [DBG] pgmap v2372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:24.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:24 smithi006 bash[33918]: audit 2023-08-27T23:17:23.664912+0000 mon.a (mon.0) 2095 : audit [DBG] from='client.? 172.21.15.6:0/2037047703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:25.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:25 smithi130 bash[51302]: debug 2023-08-27T23:17:25.300+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:25.305391+0000) 2023-08-27T23:17:26.589 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:26 smithi130 bash[51302]: debug 2023-08-27T23:17:26.304+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:26.305675+0000) 2023-08-27T23:17:26.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:26 smithi130 bash[27579]: cluster 2023-08-27T23:17:24.824968+0000 mgr.y (mgr.14901) 2366 : cluster [DBG] pgmap v2373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:26 smithi130 bash[27579]: audit 2023-08-27T23:17:26.086856+0000 mon.c (mon.2) 927 : audit [DBG] from='client.? 172.21.15.6:0/809208357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:26.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:26 smithi006 bash[33918]: cluster 2023-08-27T23:17:24.824968+0000 mgr.y (mgr.14901) 2366 : cluster [DBG] pgmap v2373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:26 smithi006 bash[33918]: audit 2023-08-27T23:17:26.086856+0000 mon.c (mon.2) 927 : audit [DBG] from='client.? 172.21.15.6:0/809208357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:26 smithi006 bash[27292]: cluster 2023-08-27T23:17:24.824968+0000 mgr.y (mgr.14901) 2366 : cluster [DBG] pgmap v2373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:26 smithi006 bash[27292]: audit 2023-08-27T23:17:26.086856+0000 mon.c (mon.2) 927 : audit [DBG] from='client.? 172.21.15.6:0/809208357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:27.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:27 smithi130 bash[51302]: debug 2023-08-27T23:17:27.300+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:27.305990+0000) 2023-08-27T23:17:28.594 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:28 smithi130 bash[51302]: debug 2023-08-27T23:17:28.300+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:28.306256+0000) 2023-08-27T23:17:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:28 smithi130 bash[27579]: cluster 2023-08-27T23:17:26.825662+0000 mgr.y (mgr.14901) 2367 : cluster [DBG] pgmap v2374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:28 smithi130 bash[27579]: audit 2023-08-27T23:17:28.503394+0000 mon.a (mon.0) 2096 : audit [DBG] from='client.? 172.21.15.6:0/3538424248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:28.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:28 smithi006 bash[27292]: cluster 2023-08-27T23:17:26.825662+0000 mgr.y (mgr.14901) 2367 : cluster [DBG] pgmap v2374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:28.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:28 smithi006 bash[27292]: audit 2023-08-27T23:17:28.503394+0000 mon.a (mon.0) 2096 : audit [DBG] from='client.? 172.21.15.6:0/3538424248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:28 smithi006 bash[33918]: cluster 2023-08-27T23:17:26.825662+0000 mgr.y (mgr.14901) 2367 : cluster [DBG] pgmap v2374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:28 smithi006 bash[33918]: audit 2023-08-27T23:17:28.503394+0000 mon.a (mon.0) 2096 : audit [DBG] from='client.? 172.21.15.6:0/3538424248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:29.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:29 smithi130 bash[51302]: debug 2023-08-27T23:17:29.300+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:29.306493+0000) 2023-08-27T23:17:30.598 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:30 smithi130 bash[51302]: debug 2023-08-27T23:17:30.304+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:30.306758+0000) 2023-08-27T23:17:30.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:30 smithi006 bash[27292]: cluster 2023-08-27T23:17:28.826354+0000 mgr.y (mgr.14901) 2368 : cluster [DBG] pgmap v2375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:30.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:30 smithi006 bash[33918]: cluster 2023-08-27T23:17:28.826354+0000 mgr.y (mgr.14901) 2368 : cluster [DBG] pgmap v2375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:17:30.424Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:17:30.426Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:30.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:30 smithi130 bash[27579]: cluster 2023-08-27T23:17:28.826354+0000 mgr.y (mgr.14901) 2368 : cluster [DBG] pgmap v2375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:17:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:17:30] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:17:31.608 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:31 smithi130 bash[51302]: debug 2023-08-27T23:17:31.304+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:31.307037+0000) 2023-08-27T23:17:31.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:31 smithi130 bash[27579]: audit 2023-08-27T23:17:30.926457+0000 mon.c (mon.2) 928 : audit [DBG] from='client.? 172.21.15.6:0/3436453019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:31.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:31 smithi006 bash[33918]: audit 2023-08-27T23:17:30.926457+0000 mon.c (mon.2) 928 : audit [DBG] from='client.? 172.21.15.6:0/3436453019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:31.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:31 smithi006 bash[27292]: audit 2023-08-27T23:17:30.926457+0000 mon.c (mon.2) 928 : audit [DBG] from='client.? 172.21.15.6:0/3436453019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:31 smithi006 bash[56298]: level=error ts=2023-08-27T23:17:31.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:31 smithi006 bash[56298]: level=warn ts=2023-08-27T23:17:31.988Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:32.610 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:32 smithi130 bash[51302]: debug 2023-08-27T23:17:32.304+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:32.307303+0000) 2023-08-27T23:17:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:32 smithi130 bash[27579]: cluster 2023-08-27T23:17:30.827470+0000 mgr.y (mgr.14901) 2369 : cluster [DBG] pgmap v2376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:32 smithi006 bash[27292]: cluster 2023-08-27T23:17:30.827470+0000 mgr.y (mgr.14901) 2369 : cluster [DBG] pgmap v2376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:32 smithi006 bash[33918]: cluster 2023-08-27T23:17:30.827470+0000 mgr.y (mgr.14901) 2369 : cluster [DBG] pgmap v2376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:33.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:33 smithi130 bash[51302]: debug 2023-08-27T23:17:33.304+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:33.307563+0000) 2023-08-27T23:17:33.895 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:33 smithi006 bash[33918]: audit 2023-08-27T23:17:33.344456+0000 mon.c (mon.2) 929 : audit [DBG] from='client.? 172.21.15.6:0/414549736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:33.895 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:33 smithi006 bash[27292]: audit 2023-08-27T23:17:33.344456+0000 mon.c (mon.2) 929 : audit [DBG] from='client.? 172.21.15.6:0/414549736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:34.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:33 smithi130 bash[27579]: audit 2023-08-27T23:17:33.344456+0000 mon.c (mon.2) 929 : audit [DBG] from='client.? 172.21.15.6:0/414549736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:34.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:17:33.893Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:17:33.895Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:34.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:34 smithi130 bash[51302]: debug 2023-08-27T23:17:34.304+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:34.307893+0000) 2023-08-27T23:17:34.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:34 smithi006 bash[33918]: cluster 2023-08-27T23:17:32.828157+0000 mgr.y (mgr.14901) 2370 : cluster [DBG] pgmap v2377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:34.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:34 smithi006 bash[27292]: cluster 2023-08-27T23:17:32.828157+0000 mgr.y (mgr.14901) 2370 : cluster [DBG] pgmap v2377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:34 smithi130 bash[27579]: cluster 2023-08-27T23:17:32.828157+0000 mgr.y (mgr.14901) 2370 : cluster [DBG] pgmap v2377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:35.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:35 smithi130 bash[51302]: debug 2023-08-27T23:17:35.304+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:35.308175+0000) 2023-08-27T23:17:36.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:36 smithi130 bash[51302]: debug 2023-08-27T23:17:36.308+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:36.308447+0000) 2023-08-27T23:17:36.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:36 smithi006 bash[33918]: cluster 2023-08-27T23:17:34.829303+0000 mgr.y (mgr.14901) 2371 : cluster [DBG] pgmap v2378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:36 smithi006 bash[33918]: audit 2023-08-27T23:17:35.759948+0000 mon.a (mon.0) 2097 : audit [DBG] from='client.? 172.21.15.6:0/138233745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:36 smithi006 bash[27292]: cluster 2023-08-27T23:17:34.829303+0000 mgr.y (mgr.14901) 2371 : cluster [DBG] pgmap v2378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:36 smithi006 bash[27292]: audit 2023-08-27T23:17:35.759948+0000 mon.a (mon.0) 2097 : audit [DBG] from='client.? 172.21.15.6:0/138233745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:36 smithi130 bash[27579]: cluster 2023-08-27T23:17:34.829303+0000 mgr.y (mgr.14901) 2371 : cluster [DBG] pgmap v2378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:37.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:36 smithi130 bash[27579]: audit 2023-08-27T23:17:35.759948+0000 mon.a (mon.0) 2097 : audit [DBG] from='client.? 172.21.15.6:0/138233745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:37.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:37 smithi130 bash[51302]: debug 2023-08-27T23:17:37.308+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:37.308712+0000) 2023-08-27T23:17:38.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:38 smithi130 bash[51302]: debug 2023-08-27T23:17:38.308+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:38.308987+0000) 2023-08-27T23:17:38.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:38 smithi006 bash[33918]: cluster 2023-08-27T23:17:36.829951+0000 mgr.y (mgr.14901) 2372 : cluster [DBG] pgmap v2379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:38 smithi006 bash[33918]: audit 2023-08-27T23:17:38.174947+0000 mon.a (mon.0) 2098 : audit [DBG] from='client.? 172.21.15.6:0/4271240508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:38 smithi006 bash[27292]: cluster 2023-08-27T23:17:36.829951+0000 mgr.y (mgr.14901) 2372 : cluster [DBG] pgmap v2379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:38 smithi006 bash[27292]: audit 2023-08-27T23:17:38.174947+0000 mon.a (mon.0) 2098 : audit [DBG] from='client.? 172.21.15.6:0/4271240508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:38 smithi130 bash[27579]: cluster 2023-08-27T23:17:36.829951+0000 mgr.y (mgr.14901) 2372 : cluster [DBG] pgmap v2379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:38 smithi130 bash[27579]: audit 2023-08-27T23:17:38.174947+0000 mon.a (mon.0) 2098 : audit [DBG] from='client.? 172.21.15.6:0/4271240508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:39.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:39 smithi130 bash[51302]: debug 2023-08-27T23:17:39.308+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:39.309265+0000) 2023-08-27T23:17:40.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:40 smithi130 bash[51302]: debug 2023-08-27T23:17:40.308+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:40.309540+0000) 2023-08-27T23:17:40.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:40 smithi006 bash[33918]: cluster 2023-08-27T23:17:38.830658+0000 mgr.y (mgr.14901) 2373 : cluster [DBG] pgmap v2380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:40 smithi006 bash[33918]: audit 2023-08-27T23:17:40.597274+0000 mon.c (mon.2) 930 : audit [DBG] from='client.? 172.21.15.6:0/1160771075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:40 smithi006 bash[27292]: cluster 2023-08-27T23:17:38.830658+0000 mgr.y (mgr.14901) 2373 : cluster [DBG] pgmap v2380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:40 smithi006 bash[27292]: audit 2023-08-27T23:17:40.597274+0000 mon.c (mon.2) 930 : audit [DBG] from='client.? 172.21.15.6:0/1160771075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:17:40.424Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:17:40.427Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:40 smithi130 bash[27579]: cluster 2023-08-27T23:17:38.830658+0000 mgr.y (mgr.14901) 2373 : cluster [DBG] pgmap v2380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:40 smithi130 bash[27579]: audit 2023-08-27T23:17:40.597274+0000 mon.c (mon.2) 930 : audit [DBG] from='client.? 172.21.15.6:0/1160771075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:17:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:17:40] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:17:41.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:41 smithi130 bash[51302]: debug 2023-08-27T23:17:41.304+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:41.309888+0000) 2023-08-27T23:17:42.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:41 smithi006 bash[56298]: level=error ts=2023-08-27T23:17:41.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:41 smithi006 bash[56298]: level=warn ts=2023-08-27T23:17:41.988Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:42.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:42 smithi130 bash[51302]: debug 2023-08-27T23:17:42.304+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:42.310224+0000) 2023-08-27T23:17:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:42 smithi006 bash[27292]: cluster 2023-08-27T23:17:40.831798+0000 mgr.y (mgr.14901) 2374 : cluster [DBG] pgmap v2381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:42.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:42 smithi006 bash[33918]: cluster 2023-08-27T23:17:40.831798+0000 mgr.y (mgr.14901) 2374 : cluster [DBG] pgmap v2381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:43.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:42 smithi130 bash[27579]: cluster 2023-08-27T23:17:40.831798+0000 mgr.y (mgr.14901) 2374 : cluster [DBG] pgmap v2381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:43.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:43 smithi130 bash[51302]: debug 2023-08-27T23:17:43.303+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:43.310527+0000) 2023-08-27T23:17:43.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:43 smithi006 bash[27292]: audit 2023-08-27T23:17:43.016974+0000 mon.c (mon.2) 931 : audit [DBG] from='client.? 172.21.15.6:0/825663239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:43.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:17:43.893Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:43.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:17:43.895Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:43.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:43 smithi006 bash[33918]: audit 2023-08-27T23:17:43.016974+0000 mon.c (mon.2) 931 : audit [DBG] from='client.? 172.21.15.6:0/825663239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:44.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:43 smithi130 bash[27579]: audit 2023-08-27T23:17:43.016974+0000 mon.c (mon.2) 931 : audit [DBG] from='client.? 172.21.15.6:0/825663239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:44.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:44 smithi130 bash[51302]: debug 2023-08-27T23:17:44.307+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:44.310880+0000) 2023-08-27T23:17:44.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:44 smithi006 bash[33918]: cluster 2023-08-27T23:17:42.832432+0000 mgr.y (mgr.14901) 2375 : cluster [DBG] pgmap v2382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:44.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:44 smithi006 bash[27292]: cluster 2023-08-27T23:17:42.832432+0000 mgr.y (mgr.14901) 2375 : cluster [DBG] pgmap v2382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:44 smithi130 bash[27579]: cluster 2023-08-27T23:17:42.832432+0000 mgr.y (mgr.14901) 2375 : cluster [DBG] pgmap v2382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:45.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:45 smithi130 bash[51302]: debug 2023-08-27T23:17:45.307+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:45.311106+0000) 2023-08-27T23:17:45.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:45 smithi006 bash[33918]: audit 2023-08-27T23:17:45.440025+0000 mon.a (mon.0) 2099 : audit [DBG] from='client.? 172.21.15.6:0/1918995651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:45.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:45 smithi006 bash[27292]: audit 2023-08-27T23:17:45.440025+0000 mon.a (mon.0) 2099 : audit [DBG] from='client.? 172.21.15.6:0/1918995651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:46.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:45 smithi130 bash[27579]: audit 2023-08-27T23:17:45.440025+0000 mon.a (mon.0) 2099 : audit [DBG] from='client.? 172.21.15.6:0/1918995651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:46.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:46 smithi130 bash[51302]: debug 2023-08-27T23:17:46.307+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:46.311330+0000) 2023-08-27T23:17:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:46 smithi006 bash[33918]: cluster 2023-08-27T23:17:44.833652+0000 mgr.y (mgr.14901) 2376 : cluster [DBG] pgmap v2383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:46 smithi006 bash[27292]: cluster 2023-08-27T23:17:44.833652+0000 mgr.y (mgr.14901) 2376 : cluster [DBG] pgmap v2383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:46 smithi130 bash[27579]: cluster 2023-08-27T23:17:44.833652+0000 mgr.y (mgr.14901) 2376 : cluster [DBG] pgmap v2383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:47.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:47 smithi130 bash[51302]: debug 2023-08-27T23:17:47.307+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:47.311632+0000) 2023-08-27T23:17:47.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:47 smithi006 bash[33918]: cluster 2023-08-27T23:17:46.834359+0000 mgr.y (mgr.14901) 2377 : cluster [DBG] pgmap v2384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:47.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:47 smithi006 bash[27292]: cluster 2023-08-27T23:17:46.834359+0000 mgr.y (mgr.14901) 2377 : cluster [DBG] pgmap v2384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:48.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:47 smithi130 bash[27579]: cluster 2023-08-27T23:17:46.834359+0000 mgr.y (mgr.14901) 2377 : cluster [DBG] pgmap v2384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:48.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:48 smithi130 bash[51302]: debug 2023-08-27T23:17:48.307+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:48.311917+0000) 2023-08-27T23:17:48.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:48 smithi006 bash[27292]: audit 2023-08-27T23:17:47.861320+0000 mon.c (mon.2) 932 : audit [DBG] from='client.? 172.21.15.6:0/3420927827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:48 smithi006 bash[33918]: audit 2023-08-27T23:17:47.861320+0000 mon.c (mon.2) 932 : audit [DBG] from='client.? 172.21.15.6:0/3420927827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:49.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:48 smithi130 bash[27579]: audit 2023-08-27T23:17:47.861320+0000 mon.c (mon.2) 932 : audit [DBG] from='client.? 172.21.15.6:0/3420927827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:49.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:49 smithi130 bash[51302]: debug 2023-08-27T23:17:49.307+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:49.312201+0000) 2023-08-27T23:17:50.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:50 smithi130 bash[51302]: debug 2023-08-27T23:17:50.307+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:50.312476+0000) 2023-08-27T23:17:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:50 smithi130 bash[27579]: cluster 2023-08-27T23:17:48.835028+0000 mgr.y (mgr.14901) 2378 : cluster [DBG] pgmap v2385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:50 smithi130 bash[27579]: audit 2023-08-27T23:17:50.279231+0000 mon.c (mon.2) 933 : audit [DBG] from='client.? 172.21.15.6:0/3083994459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:50.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:17:50.425Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:17:50.427Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:50 smithi006 bash[27292]: cluster 2023-08-27T23:17:48.835028+0000 mgr.y (mgr.14901) 2378 : cluster [DBG] pgmap v2385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:50 smithi006 bash[27292]: audit 2023-08-27T23:17:50.279231+0000 mon.c (mon.2) 933 : audit [DBG] from='client.? 172.21.15.6:0/3083994459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:50 smithi006 bash[33918]: cluster 2023-08-27T23:17:48.835028+0000 mgr.y (mgr.14901) 2378 : cluster [DBG] pgmap v2385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:50 smithi006 bash[33918]: audit 2023-08-27T23:17:50.279231+0000 mon.c (mon.2) 933 : audit [DBG] from='client.? 172.21.15.6:0/3083994459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:17:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:17:50] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:17:51.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:51 smithi130 bash[51302]: debug 2023-08-27T23:17:51.307+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:51.312797+0000) 2023-08-27T23:17:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:51 smithi006 bash[56298]: level=error ts=2023-08-27T23:17:51.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:51 smithi006 bash[56298]: level=warn ts=2023-08-27T23:17:51.988Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:52.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:52 smithi130 bash[27579]: cluster 2023-08-27T23:17:50.836180+0000 mgr.y (mgr.14901) 2379 : cluster [DBG] pgmap v2386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:52.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:52 smithi130 bash[51302]: debug 2023-08-27T23:17:52.307+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:52.313071+0000) 2023-08-27T23:17:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:52 smithi006 bash[27292]: cluster 2023-08-27T23:17:50.836180+0000 mgr.y (mgr.14901) 2379 : cluster [DBG] pgmap v2386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:52 smithi006 bash[33918]: cluster 2023-08-27T23:17:50.836180+0000 mgr.y (mgr.14901) 2379 : cluster [DBG] pgmap v2386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:53 smithi130 bash[27579]: audit 2023-08-27T23:17:52.664912+0000 mon.b (mon.1) 347 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:17:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:53 smithi130 bash[27579]: audit 2023-08-27T23:17:52.698454+0000 mon.a (mon.0) 2100 : audit [DBG] from='client.? 172.21.15.6:0/757026923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:53.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:53 smithi130 bash[51302]: debug 2023-08-27T23:17:53.311+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:53.313401+0000) 2023-08-27T23:17:53.896 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:53 smithi006 bash[33918]: audit 2023-08-27T23:17:52.664912+0000 mon.b (mon.1) 347 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:17:53.896 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:53 smithi006 bash[33918]: audit 2023-08-27T23:17:52.698454+0000 mon.a (mon.0) 2100 : audit [DBG] from='client.? 172.21.15.6:0/757026923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:53.896 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:53 smithi006 bash[27292]: audit 2023-08-27T23:17:52.664912+0000 mon.b (mon.1) 347 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:17:53.896 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:53 smithi006 bash[27292]: audit 2023-08-27T23:17:52.698454+0000 mon.a (mon.0) 2100 : audit [DBG] from='client.? 172.21.15.6:0/757026923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:54.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:17:53.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:17:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:17:53.895Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:17:54.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:54 smithi130 bash[27579]: cluster 2023-08-27T23:17:52.836833+0000 mgr.y (mgr.14901) 2380 : cluster [DBG] pgmap v2387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:54.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:54 smithi130 bash[51302]: debug 2023-08-27T23:17:54.311+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:54.313715+0000) 2023-08-27T23:17:54.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:54 smithi006 bash[33918]: cluster 2023-08-27T23:17:52.836833+0000 mgr.y (mgr.14901) 2380 : cluster [DBG] pgmap v2387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:54 smithi006 bash[27292]: cluster 2023-08-27T23:17:52.836833+0000 mgr.y (mgr.14901) 2380 : cluster [DBG] pgmap v2387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:55.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:55 smithi130 bash[27579]: audit 2023-08-27T23:17:55.123624+0000 mon.c (mon.2) 934 : audit [DBG] from='client.? 172.21.15.6:0/4182589440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:55.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:55 smithi130 bash[51302]: debug 2023-08-27T23:17:55.311+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:55.314003+0000) 2023-08-27T23:17:55.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:55 smithi006 bash[27292]: audit 2023-08-27T23:17:55.123624+0000 mon.c (mon.2) 934 : audit [DBG] from='client.? 172.21.15.6:0/4182589440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:55.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:55 smithi006 bash[33918]: audit 2023-08-27T23:17:55.123624+0000 mon.c (mon.2) 934 : audit [DBG] from='client.? 172.21.15.6:0/4182589440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:56.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:56 smithi130 bash[27579]: cluster 2023-08-27T23:17:54.837547+0000 mgr.y (mgr.14901) 2381 : cluster [DBG] pgmap v2388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:56.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:56 smithi130 bash[51302]: debug 2023-08-27T23:17:56.311+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:56.314280+0000) 2023-08-27T23:17:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:56 smithi006 bash[27292]: cluster 2023-08-27T23:17:54.837547+0000 mgr.y (mgr.14901) 2381 : cluster [DBG] pgmap v2388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:56 smithi006 bash[33918]: cluster 2023-08-27T23:17:54.837547+0000 mgr.y (mgr.14901) 2381 : cluster [DBG] pgmap v2388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:57.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:57 smithi130 bash[51302]: debug 2023-08-27T23:17:57.307+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:57.314593+0000) 2023-08-27T23:17:58.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:58 smithi130 bash[51302]: debug 2023-08-27T23:17:58.311+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:58.314920+0000) 2023-08-27T23:17:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:58 smithi130 bash[27579]: cluster 2023-08-27T23:17:56.838268+0000 mgr.y (mgr.14901) 2382 : cluster [DBG] pgmap v2389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:58 smithi130 bash[27579]: audit 2023-08-27T23:17:57.544832+0000 mon.c (mon.2) 935 : audit [DBG] from='client.? 172.21.15.6:0/2924394515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:58.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:58 smithi006 bash[27292]: cluster 2023-08-27T23:17:56.838268+0000 mgr.y (mgr.14901) 2382 : cluster [DBG] pgmap v2389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:58 smithi006 bash[27292]: audit 2023-08-27T23:17:57.544832+0000 mon.c (mon.2) 935 : audit [DBG] from='client.? 172.21.15.6:0/2924394515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:58 smithi006 bash[33918]: cluster 2023-08-27T23:17:56.838268+0000 mgr.y (mgr.14901) 2382 : cluster [DBG] pgmap v2389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:17:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:58 smithi006 bash[33918]: audit 2023-08-27T23:17:57.544832+0000 mon.c (mon.2) 935 : audit [DBG] from='client.? 172.21.15.6:0/2924394515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:59.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:17:59 smithi130 bash[51302]: debug 2023-08-27T23:17:59.311+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:59.315146+0000) 2023-08-27T23:17:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:59 smithi130 bash[27579]: audit 2023-08-27T23:17:58.555234+0000 mon.a (mon.0) 2101 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:17:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:59 smithi130 bash[27579]: audit 2023-08-27T23:17:59.007296+0000 mon.a (mon.0) 2102 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:17:59.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:59 smithi130 bash[27579]: audit 2023-08-27T23:17:59.287233+0000 mon.b (mon.1) 348 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:17:59.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:59 smithi130 bash[27579]: audit 2023-08-27T23:17:59.289346+0000 mon.b (mon.1) 349 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:17:59.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:17:59 smithi130 bash[27579]: audit 2023-08-27T23:17:59.299980+0000 mon.a (mon.0) 2103 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:17:59.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:59 smithi006 bash[27292]: audit 2023-08-27T23:17:58.555234+0000 mon.a (mon.0) 2101 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:17:59.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:59 smithi006 bash[27292]: audit 2023-08-27T23:17:59.007296+0000 mon.a (mon.0) 2102 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:17:59.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:59 smithi006 bash[27292]: audit 2023-08-27T23:17:59.287233+0000 mon.b (mon.1) 348 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:17:59.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:59 smithi006 bash[27292]: audit 2023-08-27T23:17:59.289346+0000 mon.b (mon.1) 349 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:17:59.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:17:59 smithi006 bash[27292]: audit 2023-08-27T23:17:59.299980+0000 mon.a (mon.0) 2103 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:17:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:59 smithi006 bash[33918]: audit 2023-08-27T23:17:58.555234+0000 mon.a (mon.0) 2101 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:17:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:59 smithi006 bash[33918]: audit 2023-08-27T23:17:59.007296+0000 mon.a (mon.0) 2102 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:17:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:59 smithi006 bash[33918]: audit 2023-08-27T23:17:59.287233+0000 mon.b (mon.1) 348 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:17:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:59 smithi006 bash[33918]: audit 2023-08-27T23:17:59.289346+0000 mon.b (mon.1) 349 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:17:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:17:59 smithi006 bash[33918]: audit 2023-08-27T23:17:59.299980+0000 mon.a (mon.0) 2103 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:18:00.571 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:00 smithi130 bash[51302]: debug 2023-08-27T23:18:00.311+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:00.315448+0000) 2023-08-27T23:18:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:00 smithi006 bash[33918]: cluster 2023-08-27T23:17:58.838873+0000 mgr.y (mgr.14901) 2383 : cluster [DBG] pgmap v2390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:00 smithi006 bash[33918]: audit 2023-08-27T23:17:59.958111+0000 mon.a (mon.0) 2104 : audit [DBG] from='client.? 172.21.15.6:0/1807033842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:18:00.427Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:18:00.429Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:00 smithi006 bash[27292]: cluster 2023-08-27T23:17:58.838873+0000 mgr.y (mgr.14901) 2383 : cluster [DBG] pgmap v2390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:00 smithi006 bash[27292]: audit 2023-08-27T23:17:59.958111+0000 mon.a (mon.0) 2104 : audit [DBG] from='client.? 172.21.15.6:0/1807033842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:00 smithi130 bash[27579]: cluster 2023-08-27T23:17:58.838873+0000 mgr.y (mgr.14901) 2383 : cluster [DBG] pgmap v2390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:00 smithi130 bash[27579]: audit 2023-08-27T23:17:59.958111+0000 mon.a (mon.0) 2104 : audit [DBG] from='client.? 172.21.15.6:0/1807033842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:18:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:18:00] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T23:18:01.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:01 smithi130 bash[51302]: debug 2023-08-27T23:18:01.311+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:01.315762+0000) 2023-08-27T23:18:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:01 smithi006 bash[56298]: level=error ts=2023-08-27T23:18:01.988Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:01 smithi006 bash[56298]: level=warn ts=2023-08-27T23:18:01.989Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:02.576 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:02 smithi130 bash[51302]: debug 2023-08-27T23:18:02.311+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:02.316061+0000) 2023-08-27T23:18:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:02 smithi130 bash[27579]: cluster 2023-08-27T23:18:00.839949+0000 mgr.y (mgr.14901) 2384 : cluster [DBG] pgmap v2391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:02 smithi130 bash[27579]: audit 2023-08-27T23:18:02.372552+0000 mon.a (mon.0) 2105 : audit [DBG] from='client.? 172.21.15.6:0/1845771694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:02 smithi006 bash[33918]: cluster 2023-08-27T23:18:00.839949+0000 mgr.y (mgr.14901) 2384 : cluster [DBG] pgmap v2391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:02 smithi006 bash[33918]: audit 2023-08-27T23:18:02.372552+0000 mon.a (mon.0) 2105 : audit [DBG] from='client.? 172.21.15.6:0/1845771694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:02 smithi006 bash[27292]: cluster 2023-08-27T23:18:00.839949+0000 mgr.y (mgr.14901) 2384 : cluster [DBG] pgmap v2391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:02 smithi006 bash[27292]: audit 2023-08-27T23:18:02.372552+0000 mon.a (mon.0) 2105 : audit [DBG] from='client.? 172.21.15.6:0/1845771694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:03.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:03 smithi130 bash[51302]: debug 2023-08-27T23:18:03.315+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:03.316405+0000) 2023-08-27T23:18:04.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:18:03.895Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:18:03.897Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:04.582 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:04 smithi130 bash[51302]: debug 2023-08-27T23:18:04.315+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:04.316835+0000) 2023-08-27T23:18:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:04 smithi130 bash[27579]: cluster 2023-08-27T23:18:02.840623+0000 mgr.y (mgr.14901) 2385 : cluster [DBG] pgmap v2392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:04.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:04 smithi006 bash[33918]: cluster 2023-08-27T23:18:02.840623+0000 mgr.y (mgr.14901) 2385 : cluster [DBG] pgmap v2392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:04.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:04 smithi006 bash[27292]: cluster 2023-08-27T23:18:02.840623+0000 mgr.y (mgr.14901) 2385 : cluster [DBG] pgmap v2392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:05.590 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:05 smithi130 bash[51302]: debug 2023-08-27T23:18:05.315+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:05.317111+0000) 2023-08-27T23:18:05.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:05 smithi130 bash[27579]: audit 2023-08-27T23:18:04.793564+0000 mon.c (mon.2) 936 : audit [DBG] from='client.? 172.21.15.6:0/2120925871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:05.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:05 smithi006 bash[33918]: audit 2023-08-27T23:18:04.793564+0000 mon.c (mon.2) 936 : audit [DBG] from='client.? 172.21.15.6:0/2120925871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:05.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:05 smithi006 bash[27292]: audit 2023-08-27T23:18:04.793564+0000 mon.c (mon.2) 936 : audit [DBG] from='client.? 172.21.15.6:0/2120925871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:06.593 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:06 smithi130 bash[51302]: debug 2023-08-27T23:18:06.311+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:06.317409+0000) 2023-08-27T23:18:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:06 smithi130 bash[27579]: cluster 2023-08-27T23:18:04.841743+0000 mgr.y (mgr.14901) 2386 : cluster [DBG] pgmap v2393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:06.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:06 smithi006 bash[27292]: cluster 2023-08-27T23:18:04.841743+0000 mgr.y (mgr.14901) 2386 : cluster [DBG] pgmap v2393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:06.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:06 smithi006 bash[33918]: cluster 2023-08-27T23:18:04.841743+0000 mgr.y (mgr.14901) 2386 : cluster [DBG] pgmap v2393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:07.602 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:07 smithi130 bash[51302]: debug 2023-08-27T23:18:07.315+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:07.317720+0000) 2023-08-27T23:18:07.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:07 smithi130 bash[27579]: audit 2023-08-27T23:18:07.209269+0000 mon.c (mon.2) 937 : audit [DBG] from='client.? 172.21.15.6:0/3307054491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:07.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:07 smithi006 bash[27292]: audit 2023-08-27T23:18:07.209269+0000 mon.c (mon.2) 937 : audit [DBG] from='client.? 172.21.15.6:0/3307054491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:07.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:07 smithi006 bash[33918]: audit 2023-08-27T23:18:07.209269+0000 mon.c (mon.2) 937 : audit [DBG] from='client.? 172.21.15.6:0/3307054491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:08.605 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:08 smithi130 bash[51302]: debug 2023-08-27T23:18:08.315+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:08.318069+0000) 2023-08-27T23:18:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:08 smithi130 bash[27579]: cluster 2023-08-27T23:18:06.842391+0000 mgr.y (mgr.14901) 2387 : cluster [DBG] pgmap v2394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:08 smithi006 bash[27292]: cluster 2023-08-27T23:18:06.842391+0000 mgr.y (mgr.14901) 2387 : cluster [DBG] pgmap v2394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:08 smithi006 bash[33918]: cluster 2023-08-27T23:18:06.842391+0000 mgr.y (mgr.14901) 2387 : cluster [DBG] pgmap v2394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:09.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:09 smithi130 bash[51302]: debug 2023-08-27T23:18:09.315+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:09.318377+0000) 2023-08-27T23:18:10.610 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:10 smithi130 bash[51302]: debug 2023-08-27T23:18:10.315+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:10.318661+0000) 2023-08-27T23:18:10.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:10 smithi006 bash[33918]: cluster 2023-08-27T23:18:08.843054+0000 mgr.y (mgr.14901) 2388 : cluster [DBG] pgmap v2395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:10.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:10 smithi006 bash[33918]: audit 2023-08-27T23:18:09.627380+0000 mon.c (mon.2) 938 : audit [DBG] from='client.? 172.21.15.6:0/2724650592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:10 smithi006 bash[27292]: cluster 2023-08-27T23:18:08.843054+0000 mgr.y (mgr.14901) 2388 : cluster [DBG] pgmap v2395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:10 smithi006 bash[27292]: audit 2023-08-27T23:18:09.627380+0000 mon.c (mon.2) 938 : audit [DBG] from='client.? 172.21.15.6:0/2724650592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:18:10.427Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:18:10.429Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:10 smithi130 bash[27579]: cluster 2023-08-27T23:18:08.843054+0000 mgr.y (mgr.14901) 2388 : cluster [DBG] pgmap v2395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:10 smithi130 bash[27579]: audit 2023-08-27T23:18:09.627380+0000 mon.c (mon.2) 938 : audit [DBG] from='client.? 172.21.15.6:0/2724650592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:18:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:18:10] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T23:18:11.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:11 smithi130 bash[51302]: debug 2023-08-27T23:18:11.315+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:11.318970+0000) 2023-08-27T23:18:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:11 smithi006 bash[56298]: level=error ts=2023-08-27T23:18:11.988Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:11 smithi006 bash[56298]: level=warn ts=2023-08-27T23:18:11.990Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:12.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:12 smithi130 bash[51302]: debug 2023-08-27T23:18:12.315+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:12.319214+0000) 2023-08-27T23:18:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:12 smithi130 bash[27579]: cluster 2023-08-27T23:18:10.844172+0000 mgr.y (mgr.14901) 2389 : cluster [DBG] pgmap v2396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:12 smithi130 bash[27579]: audit 2023-08-27T23:18:12.045862+0000 mon.c (mon.2) 939 : audit [DBG] from='client.? 172.21.15.6:0/642897343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:12 smithi006 bash[27292]: cluster 2023-08-27T23:18:10.844172+0000 mgr.y (mgr.14901) 2389 : cluster [DBG] pgmap v2396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:12 smithi006 bash[27292]: audit 2023-08-27T23:18:12.045862+0000 mon.c (mon.2) 939 : audit [DBG] from='client.? 172.21.15.6:0/642897343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:12 smithi006 bash[33918]: cluster 2023-08-27T23:18:10.844172+0000 mgr.y (mgr.14901) 2389 : cluster [DBG] pgmap v2396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:12 smithi006 bash[33918]: audit 2023-08-27T23:18:12.045862+0000 mon.c (mon.2) 939 : audit [DBG] from='client.? 172.21.15.6:0/642897343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:13.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:13 smithi130 bash[51302]: debug 2023-08-27T23:18:13.315+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:13.319478+0000) 2023-08-27T23:18:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:18:13.896Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:18:13.898Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:14.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:14 smithi130 bash[51302]: debug 2023-08-27T23:18:14.315+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:14.319757+0000) 2023-08-27T23:18:14.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:14 smithi006 bash[27292]: cluster 2023-08-27T23:18:12.844883+0000 mgr.y (mgr.14901) 2390 : cluster [DBG] pgmap v2397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:14 smithi006 bash[27292]: audit 2023-08-27T23:18:14.473722+0000 mon.a (mon.0) 2106 : audit [DBG] from='client.? 172.21.15.6:0/680084029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:14 smithi006 bash[33918]: cluster 2023-08-27T23:18:12.844883+0000 mgr.y (mgr.14901) 2390 : cluster [DBG] pgmap v2397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:14 smithi006 bash[33918]: audit 2023-08-27T23:18:14.473722+0000 mon.a (mon.0) 2106 : audit [DBG] from='client.? 172.21.15.6:0/680084029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:14 smithi130 bash[27579]: cluster 2023-08-27T23:18:12.844883+0000 mgr.y (mgr.14901) 2390 : cluster [DBG] pgmap v2397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:14 smithi130 bash[27579]: audit 2023-08-27T23:18:14.473722+0000 mon.a (mon.0) 2106 : audit [DBG] from='client.? 172.21.15.6:0/680084029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:15.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:15 smithi130 bash[51302]: debug 2023-08-27T23:18:15.315+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:15.320052+0000) 2023-08-27T23:18:16.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:16 smithi130 bash[51302]: debug 2023-08-27T23:18:16.319+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:16.320351+0000) 2023-08-27T23:18:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:16 smithi006 bash[27292]: cluster 2023-08-27T23:18:14.845926+0000 mgr.y (mgr.14901) 2391 : cluster [DBG] pgmap v2398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:16 smithi006 bash[33918]: cluster 2023-08-27T23:18:14.845926+0000 mgr.y (mgr.14901) 2391 : cluster [DBG] pgmap v2398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:16 smithi130 bash[27579]: cluster 2023-08-27T23:18:14.845926+0000 mgr.y (mgr.14901) 2391 : cluster [DBG] pgmap v2398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:17.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:17 smithi130 bash[51302]: debug 2023-08-27T23:18:17.315+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:17.320617+0000) 2023-08-27T23:18:17.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:17 smithi006 bash[27292]: audit 2023-08-27T23:18:16.892803+0000 mon.a (mon.0) 2107 : audit [DBG] from='client.? 172.21.15.6:0/2747690620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:17.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:17 smithi006 bash[33918]: audit 2023-08-27T23:18:16.892803+0000 mon.a (mon.0) 2107 : audit [DBG] from='client.? 172.21.15.6:0/2747690620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:18.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:17 smithi130 bash[27579]: audit 2023-08-27T23:18:16.892803+0000 mon.a (mon.0) 2107 : audit [DBG] from='client.? 172.21.15.6:0/2747690620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:18.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:18 smithi130 bash[51302]: debug 2023-08-27T23:18:18.319+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:18.320958+0000) 2023-08-27T23:18:18.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:18 smithi006 bash[27292]: cluster 2023-08-27T23:18:16.846371+0000 mgr.y (mgr.14901) 2392 : cluster [DBG] pgmap v2399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:18 smithi006 bash[33918]: cluster 2023-08-27T23:18:16.846371+0000 mgr.y (mgr.14901) 2392 : cluster [DBG] pgmap v2399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:18 smithi130 bash[27579]: cluster 2023-08-27T23:18:16.846371+0000 mgr.y (mgr.14901) 2392 : cluster [DBG] pgmap v2399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:19.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:19 smithi130 bash[51302]: debug 2023-08-27T23:18:19.315+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:19.321246+0000) 2023-08-27T23:18:19.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:19 smithi006 bash[27292]: audit 2023-08-27T23:18:19.308055+0000 mon.a (mon.0) 2108 : audit [DBG] from='client.? 172.21.15.6:0/2349171491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:19 smithi006 bash[33918]: audit 2023-08-27T23:18:19.308055+0000 mon.a (mon.0) 2108 : audit [DBG] from='client.? 172.21.15.6:0/2349171491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:20.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:19 smithi130 bash[27579]: audit 2023-08-27T23:18:19.308055+0000 mon.a (mon.0) 2108 : audit [DBG] from='client.? 172.21.15.6:0/2349171491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:20.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:20 smithi130 bash[51302]: debug 2023-08-27T23:18:20.315+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:20.321502+0000) 2023-08-27T23:18:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:20 smithi006 bash[27292]: cluster 2023-08-27T23:18:18.847024+0000 mgr.y (mgr.14901) 2393 : cluster [DBG] pgmap v2400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:20 smithi006 bash[33918]: cluster 2023-08-27T23:18:18.847024+0000 mgr.y (mgr.14901) 2393 : cluster [DBG] pgmap v2400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:20.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:18:20.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:20.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:18:20.430Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:20 smithi130 bash[27579]: cluster 2023-08-27T23:18:18.847024+0000 mgr.y (mgr.14901) 2393 : cluster [DBG] pgmap v2400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:21.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:18:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:18:20] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T23:18:21.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:21 smithi130 bash[51302]: debug 2023-08-27T23:18:21.315+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:21.321782+0000) 2023-08-27T23:18:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:21 smithi006 bash[56298]: level=error ts=2023-08-27T23:18:21.989Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:21 smithi006 bash[56298]: level=warn ts=2023-08-27T23:18:21.991Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:22.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:22 smithi130 bash[51302]: debug 2023-08-27T23:18:22.315+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:22.322054+0000) 2023-08-27T23:18:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:22 smithi006 bash[27292]: cluster 2023-08-27T23:18:20.848136+0000 mgr.y (mgr.14901) 2394 : cluster [DBG] pgmap v2401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:22 smithi006 bash[27292]: audit 2023-08-27T23:18:21.727598+0000 mon.c (mon.2) 940 : audit [DBG] from='client.? 172.21.15.6:0/1366731480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:22 smithi006 bash[33918]: cluster 2023-08-27T23:18:20.848136+0000 mgr.y (mgr.14901) 2394 : cluster [DBG] pgmap v2401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:22 smithi006 bash[33918]: audit 2023-08-27T23:18:21.727598+0000 mon.c (mon.2) 940 : audit [DBG] from='client.? 172.21.15.6:0/1366731480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:23.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:22 smithi130 bash[27579]: cluster 2023-08-27T23:18:20.848136+0000 mgr.y (mgr.14901) 2394 : cluster [DBG] pgmap v2401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:22 smithi130 bash[27579]: audit 2023-08-27T23:18:21.727598+0000 mon.c (mon.2) 940 : audit [DBG] from='client.? 172.21.15.6:0/1366731480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:23.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:23 smithi130 bash[51302]: debug 2023-08-27T23:18:23.315+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:23.322350+0000) 2023-08-27T23:18:24.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:18:23.896Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:18:23.899Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:24.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:24 smithi130 bash[51302]: debug 2023-08-27T23:18:24.319+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:24.322631+0000) 2023-08-27T23:18:24.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:24 smithi006 bash[27292]: cluster 2023-08-27T23:18:22.848791+0000 mgr.y (mgr.14901) 2395 : cluster [DBG] pgmap v2402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:24 smithi006 bash[27292]: audit 2023-08-27T23:18:24.147552+0000 mon.a (mon.0) 2109 : audit [DBG] from='client.? 172.21.15.6:0/1547196722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:24.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:24 smithi006 bash[33918]: cluster 2023-08-27T23:18:22.848791+0000 mgr.y (mgr.14901) 2395 : cluster [DBG] pgmap v2402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:24.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:24 smithi006 bash[33918]: audit 2023-08-27T23:18:24.147552+0000 mon.a (mon.0) 2109 : audit [DBG] from='client.? 172.21.15.6:0/1547196722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:24 smithi130 bash[27579]: cluster 2023-08-27T23:18:22.848791+0000 mgr.y (mgr.14901) 2395 : cluster [DBG] pgmap v2402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:24 smithi130 bash[27579]: audit 2023-08-27T23:18:24.147552+0000 mon.a (mon.0) 2109 : audit [DBG] from='client.? 172.21.15.6:0/1547196722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:25.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:25 smithi130 bash[51302]: debug 2023-08-27T23:18:25.319+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:25.322944+0000) 2023-08-27T23:18:26.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:26 smithi130 bash[51302]: debug 2023-08-27T23:18:26.319+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:26.323203+0000) 2023-08-27T23:18:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:26 smithi006 bash[33918]: cluster 2023-08-27T23:18:24.849822+0000 mgr.y (mgr.14901) 2396 : cluster [DBG] pgmap v2403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:26 smithi006 bash[33918]: audit 2023-08-27T23:18:26.569139+0000 mon.c (mon.2) 941 : audit [DBG] from='client.? 172.21.15.6:0/245707386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:26 smithi006 bash[27292]: cluster 2023-08-27T23:18:24.849822+0000 mgr.y (mgr.14901) 2396 : cluster [DBG] pgmap v2403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:26 smithi006 bash[27292]: audit 2023-08-27T23:18:26.569139+0000 mon.c (mon.2) 941 : audit [DBG] from='client.? 172.21.15.6:0/245707386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:26 smithi130 bash[27579]: cluster 2023-08-27T23:18:24.849822+0000 mgr.y (mgr.14901) 2396 : cluster [DBG] pgmap v2403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:26 smithi130 bash[27579]: audit 2023-08-27T23:18:26.569139+0000 mon.c (mon.2) 941 : audit [DBG] from='client.? 172.21.15.6:0/245707386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:27.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:27 smithi130 bash[51302]: debug 2023-08-27T23:18:27.319+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:27.323494+0000) 2023-08-27T23:18:28.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:28 smithi130 bash[51302]: debug 2023-08-27T23:18:28.319+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:28.323788+0000) 2023-08-27T23:18:28.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:28 smithi006 bash[27292]: cluster 2023-08-27T23:18:26.850423+0000 mgr.y (mgr.14901) 2397 : cluster [DBG] pgmap v2404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:28 smithi006 bash[33918]: cluster 2023-08-27T23:18:26.850423+0000 mgr.y (mgr.14901) 2397 : cluster [DBG] pgmap v2404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:28 smithi130 bash[27579]: cluster 2023-08-27T23:18:26.850423+0000 mgr.y (mgr.14901) 2397 : cluster [DBG] pgmap v2404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:29.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:29 smithi130 bash[51302]: debug 2023-08-27T23:18:29.319+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:29.324049+0000) 2023-08-27T23:18:29.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:29 smithi006 bash[33918]: audit 2023-08-27T23:18:28.995057+0000 mon.a (mon.0) 2110 : audit [DBG] from='client.? 172.21.15.6:0/2072029552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:29.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:29 smithi006 bash[27292]: audit 2023-08-27T23:18:28.995057+0000 mon.a (mon.0) 2110 : audit [DBG] from='client.? 172.21.15.6:0/2072029552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:30.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:29 smithi130 bash[27579]: audit 2023-08-27T23:18:28.995057+0000 mon.a (mon.0) 2110 : audit [DBG] from='client.? 172.21.15.6:0/2072029552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:30.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:30 smithi130 bash[51302]: debug 2023-08-27T23:18:30.319+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:30.324343+0000) 2023-08-27T23:18:30.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:30 smithi006 bash[27292]: cluster 2023-08-27T23:18:28.851008+0000 mgr.y (mgr.14901) 2398 : cluster [DBG] pgmap v2405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:30 smithi006 bash[33918]: cluster 2023-08-27T23:18:28.851008+0000 mgr.y (mgr.14901) 2398 : cluster [DBG] pgmap v2405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:18:30.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:18:30.430Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:30 smithi130 bash[27579]: cluster 2023-08-27T23:18:28.851008+0000 mgr.y (mgr.14901) 2398 : cluster [DBG] pgmap v2405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:18:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:18:30] "GET /metrics HTTP/1.1" 200 193091 "" "Prometheus/2.33.4" 2023-08-27T23:18:31.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:31 smithi130 bash[51302]: debug 2023-08-27T23:18:31.319+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:31.324641+0000) 2023-08-27T23:18:31.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:31 smithi006 bash[33918]: cluster 2023-08-27T23:18:30.852132+0000 mgr.y (mgr.14901) 2399 : cluster [DBG] pgmap v2406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:31.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:31 smithi006 bash[33918]: audit 2023-08-27T23:18:31.418756+0000 mon.c (mon.2) 942 : audit [DBG] from='client.? 172.21.15.6:0/221845286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:31.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:31 smithi006 bash[27292]: cluster 2023-08-27T23:18:30.852132+0000 mgr.y (mgr.14901) 2399 : cluster [DBG] pgmap v2406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:31.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:31 smithi006 bash[27292]: audit 2023-08-27T23:18:31.418756+0000 mon.c (mon.2) 942 : audit [DBG] from='client.? 172.21.15.6:0/221845286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:32.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:31 smithi130 bash[27579]: cluster 2023-08-27T23:18:30.852132+0000 mgr.y (mgr.14901) 2399 : cluster [DBG] pgmap v2406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:32.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:31 smithi130 bash[27579]: audit 2023-08-27T23:18:31.418756+0000 mon.c (mon.2) 942 : audit [DBG] from='client.? 172.21.15.6:0/221845286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:31 smithi006 bash[56298]: level=error ts=2023-08-27T23:18:31.990Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:31 smithi006 bash[56298]: level=warn ts=2023-08-27T23:18:31.991Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:32.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:32 smithi130 bash[51302]: debug 2023-08-27T23:18:32.319+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:32.324895+0000) 2023-08-27T23:18:33.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:33 smithi130 bash[51302]: debug 2023-08-27T23:18:33.323+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:33.325112+0000) 2023-08-27T23:18:34.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:33 smithi006 bash[27292]: cluster 2023-08-27T23:18:32.852800+0000 mgr.y (mgr.14901) 2400 : cluster [DBG] pgmap v2407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:34.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:33 smithi006 bash[27292]: audit 2023-08-27T23:18:33.833926+0000 mon.a (mon.0) 2111 : audit [DBG] from='client.? 172.21.15.6:0/2675783635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:34.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:33 smithi006 bash[33918]: cluster 2023-08-27T23:18:32.852800+0000 mgr.y (mgr.14901) 2400 : cluster [DBG] pgmap v2407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:34.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:33 smithi006 bash[33918]: audit 2023-08-27T23:18:33.833926+0000 mon.a (mon.0) 2111 : audit [DBG] from='client.? 172.21.15.6:0/2675783635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:18:33.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:18:33.900Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:34.326 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:33 smithi130 bash[27579]: cluster 2023-08-27T23:18:32.852800+0000 mgr.y (mgr.14901) 2400 : cluster [DBG] pgmap v2407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:34.326 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:33 smithi130 bash[27579]: audit 2023-08-27T23:18:33.833926+0000 mon.a (mon.0) 2111 : audit [DBG] from='client.? 172.21.15.6:0/2675783635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:34.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:34 smithi130 bash[51302]: debug 2023-08-27T23:18:34.319+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:34.325383+0000) 2023-08-27T23:18:35.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:35 smithi130 bash[51302]: debug 2023-08-27T23:18:35.323+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:35.325701+0000) 2023-08-27T23:18:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:36 smithi130 bash[27579]: cluster 2023-08-27T23:18:34.853953+0000 mgr.y (mgr.14901) 2401 : cluster [DBG] pgmap v2408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:36 smithi130 bash[27579]: audit 2023-08-27T23:18:36.258056+0000 mon.c (mon.2) 943 : audit [DBG] from='client.? 172.21.15.6:0/3695891964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:36.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:36 smithi130 bash[51302]: debug 2023-08-27T23:18:36.323+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:36.325992+0000) 2023-08-27T23:18:36.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:36 smithi006 bash[33918]: cluster 2023-08-27T23:18:34.853953+0000 mgr.y (mgr.14901) 2401 : cluster [DBG] pgmap v2408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:36 smithi006 bash[33918]: audit 2023-08-27T23:18:36.258056+0000 mon.c (mon.2) 943 : audit [DBG] from='client.? 172.21.15.6:0/3695891964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:36 smithi006 bash[27292]: cluster 2023-08-27T23:18:34.853953+0000 mgr.y (mgr.14901) 2401 : cluster [DBG] pgmap v2408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:36 smithi006 bash[27292]: audit 2023-08-27T23:18:36.258056+0000 mon.c (mon.2) 943 : audit [DBG] from='client.? 172.21.15.6:0/3695891964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:37.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:37 smithi130 bash[51302]: debug 2023-08-27T23:18:37.323+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:37.326283+0000) 2023-08-27T23:18:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:38 smithi130 bash[27579]: cluster 2023-08-27T23:18:36.854585+0000 mgr.y (mgr.14901) 2402 : cluster [DBG] pgmap v2409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:38.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:38 smithi130 bash[51302]: debug 2023-08-27T23:18:38.318+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:38.326653+0000) 2023-08-27T23:18:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:38 smithi006 bash[33918]: cluster 2023-08-27T23:18:36.854585+0000 mgr.y (mgr.14901) 2402 : cluster [DBG] pgmap v2409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:38 smithi006 bash[27292]: cluster 2023-08-27T23:18:36.854585+0000 mgr.y (mgr.14901) 2402 : cluster [DBG] pgmap v2409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:39.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:39 smithi130 bash[27579]: audit 2023-08-27T23:18:38.681635+0000 mon.a (mon.0) 2112 : audit [DBG] from='client.? 172.21.15.6:0/2758485392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:39.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:39 smithi130 bash[51302]: debug 2023-08-27T23:18:39.322+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:39.326953+0000) 2023-08-27T23:18:39.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:39 smithi006 bash[27292]: audit 2023-08-27T23:18:38.681635+0000 mon.a (mon.0) 2112 : audit [DBG] from='client.? 172.21.15.6:0/2758485392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:39 smithi006 bash[33918]: audit 2023-08-27T23:18:38.681635+0000 mon.a (mon.0) 2112 : audit [DBG] from='client.? 172.21.15.6:0/2758485392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:40.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:40 smithi130 bash[27579]: cluster 2023-08-27T23:18:38.855207+0000 mgr.y (mgr.14901) 2403 : cluster [DBG] pgmap v2410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:40.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:40 smithi130 bash[51302]: debug 2023-08-27T23:18:40.322+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:40.327237+0000) 2023-08-27T23:18:40.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:40 smithi006 bash[27292]: cluster 2023-08-27T23:18:38.855207+0000 mgr.y (mgr.14901) 2403 : cluster [DBG] pgmap v2410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:40 smithi006 bash[33918]: cluster 2023-08-27T23:18:38.855207+0000 mgr.y (mgr.14901) 2403 : cluster [DBG] pgmap v2410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:18:40.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:18:40.430Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:18:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:18:40] "GET /metrics HTTP/1.1" 200 193091 "" "Prometheus/2.33.4" 2023-08-27T23:18:41.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:41 smithi130 bash[27579]: audit 2023-08-27T23:18:41.104174+0000 mon.a (mon.0) 2113 : audit [DBG] from='client.? 172.21.15.6:0/1764216965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:41.880 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:41 smithi130 bash[51302]: debug 2023-08-27T23:18:41.326+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:41.327572+0000) 2023-08-27T23:18:41.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:41 smithi006 bash[33918]: audit 2023-08-27T23:18:41.104174+0000 mon.a (mon.0) 2113 : audit [DBG] from='client.? 172.21.15.6:0/1764216965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:41.983 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:41 smithi006 bash[27292]: audit 2023-08-27T23:18:41.104174+0000 mon.a (mon.0) 2113 : audit [DBG] from='client.? 172.21.15.6:0/1764216965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:41 smithi006 bash[56298]: level=error ts=2023-08-27T23:18:41.990Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:41 smithi006 bash[56298]: level=warn ts=2023-08-27T23:18:41.993Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:42.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:42 smithi130 bash[27579]: cluster 2023-08-27T23:18:40.856173+0000 mgr.y (mgr.14901) 2404 : cluster [DBG] pgmap v2411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:42.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:42 smithi130 bash[51302]: debug 2023-08-27T23:18:42.322+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:42.327834+0000) 2023-08-27T23:18:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:42 smithi006 bash[27292]: cluster 2023-08-27T23:18:40.856173+0000 mgr.y (mgr.14901) 2404 : cluster [DBG] pgmap v2411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:42 smithi006 bash[33918]: cluster 2023-08-27T23:18:40.856173+0000 mgr.y (mgr.14901) 2404 : cluster [DBG] pgmap v2411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:43.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:43 smithi130 bash[51302]: debug 2023-08-27T23:18:43.326+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:43.328177+0000) 2023-08-27T23:18:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:43 smithi130 bash[27579]: audit 2023-08-27T23:18:43.521843+0000 mon.a (mon.0) 2114 : audit [DBG] from='client.? 172.21.15.6:0/1955726617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:43.901 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:43 smithi006 bash[33918]: audit 2023-08-27T23:18:43.521843+0000 mon.a (mon.0) 2114 : audit [DBG] from='client.? 172.21.15.6:0/1955726617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:43.901 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:43 smithi006 bash[27292]: audit 2023-08-27T23:18:43.521843+0000 mon.a (mon.0) 2114 : audit [DBG] from='client.? 172.21.15.6:0/1955726617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:44.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:18:43.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:18:43.900Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:44 smithi130 bash[27579]: cluster 2023-08-27T23:18:42.856817+0000 mgr.y (mgr.14901) 2405 : cluster [DBG] pgmap v2412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:44.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:44 smithi130 bash[51302]: debug 2023-08-27T23:18:44.322+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:44.328470+0000) 2023-08-27T23:18:44.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:44 smithi006 bash[27292]: cluster 2023-08-27T23:18:42.856817+0000 mgr.y (mgr.14901) 2405 : cluster [DBG] pgmap v2412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:44 smithi006 bash[33918]: cluster 2023-08-27T23:18:42.856817+0000 mgr.y (mgr.14901) 2405 : cluster [DBG] pgmap v2412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:45.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:45 smithi130 bash[51302]: debug 2023-08-27T23:18:45.326+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:45.328744+0000) 2023-08-27T23:18:46.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:46 smithi130 bash[51302]: debug 2023-08-27T23:18:46.326+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:46.329030+0000) 2023-08-27T23:18:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:46 smithi130 bash[27579]: cluster 2023-08-27T23:18:44.858027+0000 mgr.y (mgr.14901) 2406 : cluster [DBG] pgmap v2413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:46 smithi130 bash[27579]: audit 2023-08-27T23:18:45.948103+0000 mon.c (mon.2) 944 : audit [DBG] from='client.? 172.21.15.6:0/701073870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:46.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:46 smithi006 bash[27292]: cluster 2023-08-27T23:18:44.858027+0000 mgr.y (mgr.14901) 2406 : cluster [DBG] pgmap v2413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:46 smithi006 bash[27292]: audit 2023-08-27T23:18:45.948103+0000 mon.c (mon.2) 944 : audit [DBG] from='client.? 172.21.15.6:0/701073870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:46 smithi006 bash[33918]: cluster 2023-08-27T23:18:44.858027+0000 mgr.y (mgr.14901) 2406 : cluster [DBG] pgmap v2413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:46 smithi006 bash[33918]: audit 2023-08-27T23:18:45.948103+0000 mon.c (mon.2) 944 : audit [DBG] from='client.? 172.21.15.6:0/701073870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:47.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:47 smithi130 bash[51302]: debug 2023-08-27T23:18:47.326+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:47.329374+0000) 2023-08-27T23:18:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:48 smithi130 bash[27579]: cluster 2023-08-27T23:18:46.858691+0000 mgr.y (mgr.14901) 2407 : cluster [DBG] pgmap v2414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:48 smithi130 bash[27579]: audit 2023-08-27T23:18:48.369250+0000 mon.a (mon.0) 2115 : audit [DBG] from='client.? 172.21.15.6:0/3673407445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:48.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:48 smithi130 bash[51302]: debug 2023-08-27T23:18:48.326+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:48.329712+0000) 2023-08-27T23:18:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:48 smithi006 bash[27292]: cluster 2023-08-27T23:18:46.858691+0000 mgr.y (mgr.14901) 2407 : cluster [DBG] pgmap v2414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:48 smithi006 bash[27292]: audit 2023-08-27T23:18:48.369250+0000 mon.a (mon.0) 2115 : audit [DBG] from='client.? 172.21.15.6:0/3673407445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:48 smithi006 bash[33918]: cluster 2023-08-27T23:18:46.858691+0000 mgr.y (mgr.14901) 2407 : cluster [DBG] pgmap v2414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:48 smithi006 bash[33918]: audit 2023-08-27T23:18:48.369250+0000 mon.a (mon.0) 2115 : audit [DBG] from='client.? 172.21.15.6:0/3673407445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:49.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:49 smithi130 bash[51302]: debug 2023-08-27T23:18:49.326+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:49.329974+0000) 2023-08-27T23:18:50.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:50 smithi130 bash[27579]: cluster 2023-08-27T23:18:48.859362+0000 mgr.y (mgr.14901) 2408 : cluster [DBG] pgmap v2415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:50.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:50 smithi130 bash[51302]: debug 2023-08-27T23:18:50.326+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:50.330262+0000) 2023-08-27T23:18:50.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:18:50.429Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:50.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:18:50.431Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:50 smithi006 bash[27292]: cluster 2023-08-27T23:18:48.859362+0000 mgr.y (mgr.14901) 2408 : cluster [DBG] pgmap v2415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:50 smithi006 bash[33918]: cluster 2023-08-27T23:18:48.859362+0000 mgr.y (mgr.14901) 2408 : cluster [DBG] pgmap v2415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:18:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:18:50] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:18:51.586 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:51 smithi130 bash[51302]: debug 2023-08-27T23:18:51.326+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:51.330541+0000) 2023-08-27T23:18:51.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:51 smithi130 bash[27579]: audit 2023-08-27T23:18:50.787002+0000 mon.a (mon.0) 2116 : audit [DBG] from='client.? 172.21.15.6:0/444198121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:51.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:51 smithi006 bash[27292]: audit 2023-08-27T23:18:50.787002+0000 mon.a (mon.0) 2116 : audit [DBG] from='client.? 172.21.15.6:0/444198121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:51.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:51 smithi006 bash[33918]: audit 2023-08-27T23:18:50.787002+0000 mon.a (mon.0) 2116 : audit [DBG] from='client.? 172.21.15.6:0/444198121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:51 smithi006 bash[56298]: level=error ts=2023-08-27T23:18:51.992Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:51 smithi006 bash[56298]: level=warn ts=2023-08-27T23:18:51.994Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:52.588 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:52 smithi130 bash[51302]: debug 2023-08-27T23:18:52.330+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:52.330878+0000) 2023-08-27T23:18:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:52 smithi130 bash[27579]: cluster 2023-08-27T23:18:50.860399+0000 mgr.y (mgr.14901) 2409 : cluster [DBG] pgmap v2416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:52 smithi006 bash[27292]: cluster 2023-08-27T23:18:50.860399+0000 mgr.y (mgr.14901) 2409 : cluster [DBG] pgmap v2416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:52 smithi006 bash[33918]: cluster 2023-08-27T23:18:50.860399+0000 mgr.y (mgr.14901) 2409 : cluster [DBG] pgmap v2416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:53.596 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:53 smithi130 bash[51302]: debug 2023-08-27T23:18:53.330+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:53.331140+0000) 2023-08-27T23:18:53.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:53 smithi130 bash[27579]: audit 2023-08-27T23:18:53.208103+0000 mon.a (mon.0) 2117 : audit [DBG] from='client.? 172.21.15.6:0/2188065437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:53.902 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:53 smithi006 bash[33918]: audit 2023-08-27T23:18:53.208103+0000 mon.a (mon.0) 2117 : audit [DBG] from='client.? 172.21.15.6:0/2188065437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:53.902 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:53 smithi006 bash[27292]: audit 2023-08-27T23:18:53.208103+0000 mon.a (mon.0) 2117 : audit [DBG] from='client.? 172.21.15.6:0/2188065437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:54.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:18:53.899Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:18:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:18:53.901Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:18:54.598 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:54 smithi130 bash[51302]: debug 2023-08-27T23:18:54.330+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:54.331564+0000) 2023-08-27T23:18:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:54 smithi130 bash[27579]: cluster 2023-08-27T23:18:52.861066+0000 mgr.y (mgr.14901) 2410 : cluster [DBG] pgmap v2417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:54.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:54 smithi006 bash[33918]: cluster 2023-08-27T23:18:52.861066+0000 mgr.y (mgr.14901) 2410 : cluster [DBG] pgmap v2417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:54 smithi006 bash[27292]: cluster 2023-08-27T23:18:52.861066+0000 mgr.y (mgr.14901) 2410 : cluster [DBG] pgmap v2417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:55.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:55 smithi130 bash[51302]: debug 2023-08-27T23:18:55.330+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:55.331862+0000) 2023-08-27T23:18:56.604 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:56 smithi130 bash[51302]: debug 2023-08-27T23:18:56.330+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:56.332129+0000) 2023-08-27T23:18:56.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:56 smithi130 bash[27579]: cluster 2023-08-27T23:18:54.862213+0000 mgr.y (mgr.14901) 2411 : cluster [DBG] pgmap v2418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:56 smithi130 bash[27579]: audit 2023-08-27T23:18:55.621452+0000 mon.c (mon.2) 945 : audit [DBG] from='client.? 172.21.15.6:0/4030699727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:56 smithi006 bash[33918]: cluster 2023-08-27T23:18:54.862213+0000 mgr.y (mgr.14901) 2411 : cluster [DBG] pgmap v2418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:56 smithi006 bash[33918]: audit 2023-08-27T23:18:55.621452+0000 mon.c (mon.2) 945 : audit [DBG] from='client.? 172.21.15.6:0/4030699727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:56 smithi006 bash[27292]: cluster 2023-08-27T23:18:54.862213+0000 mgr.y (mgr.14901) 2411 : cluster [DBG] pgmap v2418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:56 smithi006 bash[27292]: audit 2023-08-27T23:18:55.621452+0000 mon.c (mon.2) 945 : audit [DBG] from='client.? 172.21.15.6:0/4030699727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:57.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:57 smithi130 bash[51302]: debug 2023-08-27T23:18:57.330+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:57.332442+0000) 2023-08-27T23:18:58.609 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:58 smithi130 bash[51302]: debug 2023-08-27T23:18:58.330+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:58.332730+0000) 2023-08-27T23:18:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:58 smithi130 bash[27579]: cluster 2023-08-27T23:18:56.862874+0000 mgr.y (mgr.14901) 2412 : cluster [DBG] pgmap v2419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:58 smithi130 bash[27579]: audit 2023-08-27T23:18:58.034862+0000 mon.a (mon.0) 2118 : audit [DBG] from='client.? 172.21.15.6:0/1327009779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:58 smithi006 bash[33918]: cluster 2023-08-27T23:18:56.862874+0000 mgr.y (mgr.14901) 2412 : cluster [DBG] pgmap v2419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:58 smithi006 bash[33918]: audit 2023-08-27T23:18:58.034862+0000 mon.a (mon.0) 2118 : audit [DBG] from='client.? 172.21.15.6:0/1327009779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:58 smithi006 bash[27292]: cluster 2023-08-27T23:18:56.862874+0000 mgr.y (mgr.14901) 2412 : cluster [DBG] pgmap v2419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:18:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:58 smithi006 bash[27292]: audit 2023-08-27T23:18:58.034862+0000 mon.a (mon.0) 2118 : audit [DBG] from='client.? 172.21.15.6:0/1327009779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:59.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:18:59 smithi130 bash[51302]: debug 2023-08-27T23:18:59.330+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:59.333048+0000) 2023-08-27T23:18:59.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:18:59 smithi006 bash[27292]: audit 2023-08-27T23:18:59.312421+0000 mon.b (mon.1) 350 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:18:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:18:59 smithi006 bash[33918]: audit 2023-08-27T23:18:59.312421+0000 mon.b (mon.1) 350 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:19:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:18:59 smithi130 bash[27579]: audit 2023-08-27T23:18:59.312421+0000 mon.b (mon.1) 350 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:19:00.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:00 smithi130 bash[51302]: debug 2023-08-27T23:19:00.330+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:00.333340+0000) 2023-08-27T23:19:00.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:00 smithi006 bash[33918]: cluster 2023-08-27T23:18:58.863533+0000 mgr.y (mgr.14901) 2413 : cluster [DBG] pgmap v2420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:00 smithi006 bash[33918]: audit 2023-08-27T23:19:00.456029+0000 mon.c (mon.2) 946 : audit [DBG] from='client.? 172.21.15.6:0/4245825676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:19:00.430Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:19:00.432Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:00 smithi006 bash[27292]: cluster 2023-08-27T23:18:58.863533+0000 mgr.y (mgr.14901) 2413 : cluster [DBG] pgmap v2420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:00 smithi006 bash[27292]: audit 2023-08-27T23:19:00.456029+0000 mon.c (mon.2) 946 : audit [DBG] from='client.? 172.21.15.6:0/4245825676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:00 smithi130 bash[27579]: cluster 2023-08-27T23:18:58.863533+0000 mgr.y (mgr.14901) 2413 : cluster [DBG] pgmap v2420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:00 smithi130 bash[27579]: audit 2023-08-27T23:19:00.456029+0000 mon.c (mon.2) 946 : audit [DBG] from='client.? 172.21.15.6:0/4245825676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:19:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:19:00] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:19:01.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:01 smithi130 bash[51302]: debug 2023-08-27T23:19:01.330+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:01.333616+0000) 2023-08-27T23:19:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:01 smithi006 bash[56298]: level=error ts=2023-08-27T23:19:01.992Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:01 smithi006 bash[56298]: level=warn ts=2023-08-27T23:19:01.995Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:02.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:02 smithi130 bash[51302]: debug 2023-08-27T23:19:02.330+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:02.333900+0000) 2023-08-27T23:19:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:02 smithi006 bash[27292]: cluster 2023-08-27T23:19:00.864692+0000 mgr.y (mgr.14901) 2414 : cluster [DBG] pgmap v2421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:02 smithi006 bash[33918]: cluster 2023-08-27T23:19:00.864692+0000 mgr.y (mgr.14901) 2414 : cluster [DBG] pgmap v2421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:02 smithi130 bash[27579]: cluster 2023-08-27T23:19:00.864692+0000 mgr.y (mgr.14901) 2414 : cluster [DBG] pgmap v2421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:03.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:03 smithi130 bash[51302]: debug 2023-08-27T23:19:03.330+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:03.334176+0000) 2023-08-27T23:19:03.902 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:03 smithi006 bash[33918]: audit 2023-08-27T23:19:02.874308+0000 mon.c (mon.2) 947 : audit [DBG] from='client.? 172.21.15.6:0/676217591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:03.903 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:03 smithi006 bash[27292]: audit 2023-08-27T23:19:02.874308+0000 mon.c (mon.2) 947 : audit [DBG] from='client.? 172.21.15.6:0/676217591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:04.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:03 smithi130 bash[27579]: audit 2023-08-27T23:19:02.874308+0000 mon.c (mon.2) 947 : audit [DBG] from='client.? 172.21.15.6:0/676217591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:19:03.900Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:19:03.902Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:04.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:04 smithi130 bash[51302]: debug 2023-08-27T23:19:04.330+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:04.334476+0000) 2023-08-27T23:19:04.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:04 smithi006 bash[27292]: cluster 2023-08-27T23:19:02.865315+0000 mgr.y (mgr.14901) 2415 : cluster [DBG] pgmap v2422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:04 smithi006 bash[33918]: cluster 2023-08-27T23:19:02.865315+0000 mgr.y (mgr.14901) 2415 : cluster [DBG] pgmap v2422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:04 smithi130 bash[27579]: cluster 2023-08-27T23:19:02.865315+0000 mgr.y (mgr.14901) 2415 : cluster [DBG] pgmap v2422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:05.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:05 smithi130 bash[51302]: debug 2023-08-27T23:19:05.330+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:05.334696+0000) 2023-08-27T23:19:06.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:06 smithi130 bash[51302]: debug 2023-08-27T23:19:06.334+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:06.334982+0000) 2023-08-27T23:19:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:06 smithi130 bash[27579]: cluster 2023-08-27T23:19:04.866425+0000 mgr.y (mgr.14901) 2416 : cluster [DBG] pgmap v2423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:06 smithi130 bash[27579]: audit 2023-08-27T23:19:05.242275+0000 mon.a (mon.0) 2119 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:19:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:06 smithi130 bash[27579]: audit 2023-08-27T23:19:05.326960+0000 mon.a (mon.0) 2120 : audit [DBG] from='client.? 172.21.15.6:0/2155148587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:06 smithi130 bash[27579]: audit 2023-08-27T23:19:05.593537+0000 mon.a (mon.0) 2121 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:19:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:06 smithi130 bash[27579]: audit 2023-08-27T23:19:05.890239+0000 mon.b (mon.1) 351 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:19:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:06 smithi130 bash[27579]: audit 2023-08-27T23:19:05.892805+0000 mon.b (mon.1) 352 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:19:06.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:06 smithi130 bash[27579]: audit 2023-08-27T23:19:05.901068+0000 mon.a (mon.0) 2122 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:19:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:06 smithi006 bash[33918]: cluster 2023-08-27T23:19:04.866425+0000 mgr.y (mgr.14901) 2416 : cluster [DBG] pgmap v2423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:06 smithi006 bash[33918]: audit 2023-08-27T23:19:05.242275+0000 mon.a (mon.0) 2119 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:19:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:06 smithi006 bash[33918]: audit 2023-08-27T23:19:05.326960+0000 mon.a (mon.0) 2120 : audit [DBG] from='client.? 172.21.15.6:0/2155148587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:06 smithi006 bash[33918]: audit 2023-08-27T23:19:05.593537+0000 mon.a (mon.0) 2121 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:19:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:06 smithi006 bash[33918]: audit 2023-08-27T23:19:05.890239+0000 mon.b (mon.1) 351 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:19:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:06 smithi006 bash[33918]: audit 2023-08-27T23:19:05.892805+0000 mon.b (mon.1) 352 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:19:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:06 smithi006 bash[33918]: audit 2023-08-27T23:19:05.901068+0000 mon.a (mon.0) 2122 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:19:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:06 smithi006 bash[27292]: cluster 2023-08-27T23:19:04.866425+0000 mgr.y (mgr.14901) 2416 : cluster [DBG] pgmap v2423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:06 smithi006 bash[27292]: audit 2023-08-27T23:19:05.242275+0000 mon.a (mon.0) 2119 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:19:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:06 smithi006 bash[27292]: audit 2023-08-27T23:19:05.326960+0000 mon.a (mon.0) 2120 : audit [DBG] from='client.? 172.21.15.6:0/2155148587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:06 smithi006 bash[27292]: audit 2023-08-27T23:19:05.593537+0000 mon.a (mon.0) 2121 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:19:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:06 smithi006 bash[27292]: audit 2023-08-27T23:19:05.890239+0000 mon.b (mon.1) 351 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:19:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:06 smithi006 bash[27292]: audit 2023-08-27T23:19:05.892805+0000 mon.b (mon.1) 352 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:19:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:06 smithi006 bash[27292]: audit 2023-08-27T23:19:05.901068+0000 mon.a (mon.0) 2122 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:19:07.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:07 smithi130 bash[51302]: debug 2023-08-27T23:19:07.334+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:07.335202+0000) 2023-08-27T23:19:08.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:08 smithi130 bash[51302]: debug 2023-08-27T23:19:08.334+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:08.335434+0000) 2023-08-27T23:19:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:08 smithi130 bash[27579]: cluster 2023-08-27T23:19:06.867084+0000 mgr.y (mgr.14901) 2417 : cluster [DBG] pgmap v2424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:08 smithi130 bash[27579]: audit 2023-08-27T23:19:07.745684+0000 mon.a (mon.0) 2123 : audit [DBG] from='client.? 172.21.15.6:0/1035944085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:08 smithi006 bash[27292]: cluster 2023-08-27T23:19:06.867084+0000 mgr.y (mgr.14901) 2417 : cluster [DBG] pgmap v2424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:08 smithi006 bash[27292]: audit 2023-08-27T23:19:07.745684+0000 mon.a (mon.0) 2123 : audit [DBG] from='client.? 172.21.15.6:0/1035944085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:08 smithi006 bash[33918]: cluster 2023-08-27T23:19:06.867084+0000 mgr.y (mgr.14901) 2417 : cluster [DBG] pgmap v2424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:08 smithi006 bash[33918]: audit 2023-08-27T23:19:07.745684+0000 mon.a (mon.0) 2123 : audit [DBG] from='client.? 172.21.15.6:0/1035944085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:09.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:09 smithi130 bash[51302]: debug 2023-08-27T23:19:09.334+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:09.335729+0000) 2023-08-27T23:19:10.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:10 smithi130 bash[27579]: cluster 2023-08-27T23:19:08.867734+0000 mgr.y (mgr.14901) 2418 : cluster [DBG] pgmap v2425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:10 smithi130 bash[27579]: audit 2023-08-27T23:19:10.171298+0000 mon.b (mon.1) 353 : audit [DBG] from='client.? 172.21.15.6:0/4116146091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:10.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:10 smithi130 bash[51302]: debug 2023-08-27T23:19:10.334+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:10.336029+0000) 2023-08-27T23:19:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:10 smithi006 bash[27292]: cluster 2023-08-27T23:19:08.867734+0000 mgr.y (mgr.14901) 2418 : cluster [DBG] pgmap v2425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:10 smithi006 bash[27292]: audit 2023-08-27T23:19:10.171298+0000 mon.b (mon.1) 353 : audit [DBG] from='client.? 172.21.15.6:0/4116146091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:10 smithi006 bash[33918]: cluster 2023-08-27T23:19:08.867734+0000 mgr.y (mgr.14901) 2418 : cluster [DBG] pgmap v2425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:10 smithi006 bash[33918]: audit 2023-08-27T23:19:10.171298+0000 mon.b (mon.1) 353 : audit [DBG] from='client.? 172.21.15.6:0/4116146091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:19:10.431Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:10.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:19:10.433Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:19:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:19:10] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:19:11.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:11 smithi130 bash[51302]: debug 2023-08-27T23:19:11.334+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:11.336325+0000) 2023-08-27T23:19:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:11 smithi006 bash[56298]: level=error ts=2023-08-27T23:19:11.994Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:11 smithi006 bash[56298]: level=warn ts=2023-08-27T23:19:11.996Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:12.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:12 smithi130 bash[27579]: cluster 2023-08-27T23:19:10.868880+0000 mgr.y (mgr.14901) 2419 : cluster [DBG] pgmap v2426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:12.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:12 smithi130 bash[51302]: debug 2023-08-27T23:19:12.334+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:12.336568+0000) 2023-08-27T23:19:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:12 smithi006 bash[27292]: cluster 2023-08-27T23:19:10.868880+0000 mgr.y (mgr.14901) 2419 : cluster [DBG] pgmap v2426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:12 smithi006 bash[33918]: cluster 2023-08-27T23:19:10.868880+0000 mgr.y (mgr.14901) 2419 : cluster [DBG] pgmap v2426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:13 smithi130 bash[27579]: audit 2023-08-27T23:19:12.586104+0000 mon.a (mon.0) 2124 : audit [DBG] from='client.? 172.21.15.6:0/1329921517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:13.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:13 smithi130 bash[51302]: debug 2023-08-27T23:19:13.334+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:13.336865+0000) 2023-08-27T23:19:13.903 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:13 smithi006 bash[33918]: audit 2023-08-27T23:19:12.586104+0000 mon.a (mon.0) 2124 : audit [DBG] from='client.? 172.21.15.6:0/1329921517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:13.904 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:13 smithi006 bash[27292]: audit 2023-08-27T23:19:12.586104+0000 mon.a (mon.0) 2124 : audit [DBG] from='client.? 172.21.15.6:0/1329921517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:19:13.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:19:13.903Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:14.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:14 smithi130 bash[27579]: cluster 2023-08-27T23:19:12.869550+0000 mgr.y (mgr.14901) 2420 : cluster [DBG] pgmap v2427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:14.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:14 smithi130 bash[51302]: debug 2023-08-27T23:19:14.334+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:14.337265+0000) 2023-08-27T23:19:14.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:14 smithi006 bash[27292]: cluster 2023-08-27T23:19:12.869550+0000 mgr.y (mgr.14901) 2420 : cluster [DBG] pgmap v2427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:14 smithi006 bash[33918]: cluster 2023-08-27T23:19:12.869550+0000 mgr.y (mgr.14901) 2420 : cluster [DBG] pgmap v2427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:15.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:15 smithi130 bash[27579]: audit 2023-08-27T23:19:15.008094+0000 mon.c (mon.2) 948 : audit [DBG] from='client.? 172.21.15.6:0/2870617970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:15.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:15 smithi130 bash[51302]: debug 2023-08-27T23:19:15.334+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:15.337586+0000) 2023-08-27T23:19:15.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:15 smithi006 bash[33918]: audit 2023-08-27T23:19:15.008094+0000 mon.c (mon.2) 948 : audit [DBG] from='client.? 172.21.15.6:0/2870617970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:15.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:15 smithi006 bash[27292]: audit 2023-08-27T23:19:15.008094+0000 mon.c (mon.2) 948 : audit [DBG] from='client.? 172.21.15.6:0/2870617970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:16.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:16 smithi130 bash[27579]: cluster 2023-08-27T23:19:14.870273+0000 mgr.y (mgr.14901) 2421 : cluster [DBG] pgmap v2428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:16.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:16 smithi130 bash[51302]: debug 2023-08-27T23:19:16.334+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:16.337858+0000) 2023-08-27T23:19:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:16 smithi006 bash[27292]: cluster 2023-08-27T23:19:14.870273+0000 mgr.y (mgr.14901) 2421 : cluster [DBG] pgmap v2428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:16 smithi006 bash[33918]: cluster 2023-08-27T23:19:14.870273+0000 mgr.y (mgr.14901) 2421 : cluster [DBG] pgmap v2428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:17.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:17 smithi130 bash[27579]: audit 2023-08-27T23:19:17.414781+0000 mon.a (mon.0) 2125 : audit [DBG] from='client.? 172.21.15.6:0/2334642822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:17.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:17 smithi130 bash[51302]: debug 2023-08-27T23:19:17.334+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:17.338160+0000) 2023-08-27T23:19:17.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:17 smithi006 bash[33918]: audit 2023-08-27T23:19:17.414781+0000 mon.a (mon.0) 2125 : audit [DBG] from='client.? 172.21.15.6:0/2334642822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:17.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:17 smithi006 bash[27292]: audit 2023-08-27T23:19:17.414781+0000 mon.a (mon.0) 2125 : audit [DBG] from='client.? 172.21.15.6:0/2334642822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:18.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:18 smithi130 bash[27579]: cluster 2023-08-27T23:19:16.870879+0000 mgr.y (mgr.14901) 2422 : cluster [DBG] pgmap v2429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:18.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:18 smithi130 bash[51302]: debug 2023-08-27T23:19:18.334+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:18.338471+0000) 2023-08-27T23:19:18.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:18 smithi006 bash[33918]: cluster 2023-08-27T23:19:16.870879+0000 mgr.y (mgr.14901) 2422 : cluster [DBG] pgmap v2429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:19.011 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:18 smithi006 bash[27292]: cluster 2023-08-27T23:19:16.870879+0000 mgr.y (mgr.14901) 2422 : cluster [DBG] pgmap v2429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:19.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:19 smithi130 bash[51302]: debug 2023-08-27T23:19:19.334+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:19.338739+0000) 2023-08-27T23:19:20.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:20 smithi130 bash[51302]: debug 2023-08-27T23:19:20.338+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:20.339029+0000) 2023-08-27T23:19:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:20 smithi130 bash[27579]: cluster 2023-08-27T23:19:18.871527+0000 mgr.y (mgr.14901) 2423 : cluster [DBG] pgmap v2430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:20 smithi130 bash[27579]: audit 2023-08-27T23:19:19.832755+0000 mon.a (mon.0) 2126 : audit [DBG] from='client.? 172.21.15.6:0/995865676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:20 smithi006 bash[27292]: cluster 2023-08-27T23:19:18.871527+0000 mgr.y (mgr.14901) 2423 : cluster [DBG] pgmap v2430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:20 smithi006 bash[27292]: audit 2023-08-27T23:19:19.832755+0000 mon.a (mon.0) 2126 : audit [DBG] from='client.? 172.21.15.6:0/995865676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:19:20.431Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:19:20.433Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:20 smithi006 bash[33918]: cluster 2023-08-27T23:19:18.871527+0000 mgr.y (mgr.14901) 2423 : cluster [DBG] pgmap v2430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:20.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:20 smithi006 bash[33918]: audit 2023-08-27T23:19:19.832755+0000 mon.a (mon.0) 2126 : audit [DBG] from='client.? 172.21.15.6:0/995865676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:19:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:19:20] "GET /metrics HTTP/1.1" 200 193083 "" "Prometheus/2.33.4" 2023-08-27T23:19:21.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:21 smithi130 bash[51302]: debug 2023-08-27T23:19:21.338+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:21.339313+0000) 2023-08-27T23:19:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:21 smithi006 bash[56298]: level=error ts=2023-08-27T23:19:21.994Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:21 smithi006 bash[56298]: level=warn ts=2023-08-27T23:19:21.997Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:22.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:22 smithi130 bash[51302]: debug 2023-08-27T23:19:22.338+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:22.339602+0000) 2023-08-27T23:19:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:22 smithi130 bash[27579]: cluster 2023-08-27T23:19:20.872575+0000 mgr.y (mgr.14901) 2424 : cluster [DBG] pgmap v2431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:22 smithi130 bash[27579]: audit 2023-08-27T23:19:22.245359+0000 mon.a (mon.0) 2127 : audit [DBG] from='client.? 172.21.15.6:0/410397527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:22 smithi006 bash[27292]: cluster 2023-08-27T23:19:20.872575+0000 mgr.y (mgr.14901) 2424 : cluster [DBG] pgmap v2431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:22 smithi006 bash[27292]: audit 2023-08-27T23:19:22.245359+0000 mon.a (mon.0) 2127 : audit [DBG] from='client.? 172.21.15.6:0/410397527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:22 smithi006 bash[33918]: cluster 2023-08-27T23:19:20.872575+0000 mgr.y (mgr.14901) 2424 : cluster [DBG] pgmap v2431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:22 smithi006 bash[33918]: audit 2023-08-27T23:19:22.245359+0000 mon.a (mon.0) 2127 : audit [DBG] from='client.? 172.21.15.6:0/410397527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:23.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:23 smithi130 bash[51302]: debug 2023-08-27T23:19:23.338+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:23.339884+0000) 2023-08-27T23:19:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:19:23.902Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:19:23.903Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:24.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:24 smithi130 bash[51302]: debug 2023-08-27T23:19:24.338+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:24.340178+0000) 2023-08-27T23:19:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:24 smithi130 bash[27579]: cluster 2023-08-27T23:19:22.873265+0000 mgr.y (mgr.14901) 2425 : cluster [DBG] pgmap v2432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:24 smithi006 bash[27292]: cluster 2023-08-27T23:19:22.873265+0000 mgr.y (mgr.14901) 2425 : cluster [DBG] pgmap v2432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:24.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:24 smithi006 bash[33918]: cluster 2023-08-27T23:19:22.873265+0000 mgr.y (mgr.14901) 2425 : cluster [DBG] pgmap v2432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:25.592 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:25 smithi130 bash[51302]: debug 2023-08-27T23:19:25.338+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:25.340493+0000) 2023-08-27T23:19:25.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:25 smithi130 bash[27579]: audit 2023-08-27T23:19:24.661602+0000 mon.c (mon.2) 949 : audit [DBG] from='client.? 172.21.15.6:0/3480999643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:25.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:25 smithi006 bash[27292]: audit 2023-08-27T23:19:24.661602+0000 mon.c (mon.2) 949 : audit [DBG] from='client.? 172.21.15.6:0/3480999643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:25.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:25 smithi006 bash[33918]: audit 2023-08-27T23:19:24.661602+0000 mon.c (mon.2) 949 : audit [DBG] from='client.? 172.21.15.6:0/3480999643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:26.601 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:26 smithi130 bash[51302]: debug 2023-08-27T23:19:26.338+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:26.340765+0000) 2023-08-27T23:19:26.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:26 smithi130 bash[27579]: cluster 2023-08-27T23:19:24.874393+0000 mgr.y (mgr.14901) 2426 : cluster [DBG] pgmap v2433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:26 smithi006 bash[27292]: cluster 2023-08-27T23:19:24.874393+0000 mgr.y (mgr.14901) 2426 : cluster [DBG] pgmap v2433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:26 smithi006 bash[33918]: cluster 2023-08-27T23:19:24.874393+0000 mgr.y (mgr.14901) 2426 : cluster [DBG] pgmap v2433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:27.604 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:27 smithi130 bash[51302]: debug 2023-08-27T23:19:27.338+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:27.341078+0000) 2023-08-27T23:19:27.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:27 smithi130 bash[27579]: audit 2023-08-27T23:19:27.080194+0000 mon.c (mon.2) 950 : audit [DBG] from='client.? 172.21.15.6:0/951824971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:27.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:27 smithi006 bash[27292]: audit 2023-08-27T23:19:27.080194+0000 mon.c (mon.2) 950 : audit [DBG] from='client.? 172.21.15.6:0/951824971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:27.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:27 smithi006 bash[33918]: audit 2023-08-27T23:19:27.080194+0000 mon.c (mon.2) 950 : audit [DBG] from='client.? 172.21.15.6:0/951824971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:28.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:28 smithi130 bash[51302]: debug 2023-08-27T23:19:28.338+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:28.341371+0000) 2023-08-27T23:19:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:28 smithi130 bash[27579]: cluster 2023-08-27T23:19:26.874862+0000 mgr.y (mgr.14901) 2427 : cluster [DBG] pgmap v2434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:28 smithi006 bash[27292]: cluster 2023-08-27T23:19:26.874862+0000 mgr.y (mgr.14901) 2427 : cluster [DBG] pgmap v2434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:28 smithi006 bash[33918]: cluster 2023-08-27T23:19:26.874862+0000 mgr.y (mgr.14901) 2427 : cluster [DBG] pgmap v2434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:29.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:29 smithi130 bash[51302]: debug 2023-08-27T23:19:29.338+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:29.341651+0000) 2023-08-27T23:19:29.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:29 smithi006 bash[27292]: audit 2023-08-27T23:19:29.487788+0000 mon.c (mon.2) 951 : audit [DBG] from='client.? 172.21.15.6:0/1209982151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:29.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:29 smithi006 bash[33918]: audit 2023-08-27T23:19:29.487788+0000 mon.c (mon.2) 951 : audit [DBG] from='client.? 172.21.15.6:0/1209982151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:30.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:29 smithi130 bash[27579]: audit 2023-08-27T23:19:29.487788+0000 mon.c (mon.2) 951 : audit [DBG] from='client.? 172.21.15.6:0/1209982151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:30.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:30 smithi130 bash[51302]: debug 2023-08-27T23:19:30.338+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:30.341946+0000) 2023-08-27T23:19:30.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:30 smithi006 bash[27292]: cluster 2023-08-27T23:19:28.875475+0000 mgr.y (mgr.14901) 2428 : cluster [DBG] pgmap v2435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:30 smithi006 bash[33918]: cluster 2023-08-27T23:19:28.875475+0000 mgr.y (mgr.14901) 2428 : cluster [DBG] pgmap v2435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:19:30.432Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:19:30.434Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:30 smithi130 bash[27579]: cluster 2023-08-27T23:19:28.875475+0000 mgr.y (mgr.14901) 2428 : cluster [DBG] pgmap v2435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:19:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:19:30] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:19:31.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:31 smithi130 bash[51302]: debug 2023-08-27T23:19:31.338+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:31.342212+0000) 2023-08-27T23:19:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:31 smithi006 bash[56298]: level=error ts=2023-08-27T23:19:31.995Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:31 smithi006 bash[56298]: level=warn ts=2023-08-27T23:19:31.997Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:32.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:32 smithi130 bash[51302]: debug 2023-08-27T23:19:32.338+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:32.342502+0000) 2023-08-27T23:19:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:32 smithi006 bash[33918]: cluster 2023-08-27T23:19:30.876571+0000 mgr.y (mgr.14901) 2429 : cluster [DBG] pgmap v2436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:32 smithi006 bash[33918]: audit 2023-08-27T23:19:31.901986+0000 mon.c (mon.2) 952 : audit [DBG] from='client.? 172.21.15.6:0/2511457250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:32.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:32 smithi006 bash[27292]: cluster 2023-08-27T23:19:30.876571+0000 mgr.y (mgr.14901) 2429 : cluster [DBG] pgmap v2436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:32.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:32 smithi006 bash[27292]: audit 2023-08-27T23:19:31.901986+0000 mon.c (mon.2) 952 : audit [DBG] from='client.? 172.21.15.6:0/2511457250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:33.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:32 smithi130 bash[27579]: cluster 2023-08-27T23:19:30.876571+0000 mgr.y (mgr.14901) 2429 : cluster [DBG] pgmap v2436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:33.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:32 smithi130 bash[27579]: audit 2023-08-27T23:19:31.901986+0000 mon.c (mon.2) 952 : audit [DBG] from='client.? 172.21.15.6:0/2511457250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:33.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:33 smithi130 bash[51302]: debug 2023-08-27T23:19:33.338+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:33.342816+0000) 2023-08-27T23:19:34.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:19:33.902Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:19:33.904Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:34.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:34 smithi130 bash[51302]: debug 2023-08-27T23:19:34.341+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:34.343115+0000) 2023-08-27T23:19:34.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:34 smithi006 bash[33918]: cluster 2023-08-27T23:19:32.877349+0000 mgr.y (mgr.14901) 2430 : cluster [DBG] pgmap v2437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:34.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:34 smithi006 bash[33918]: audit 2023-08-27T23:19:34.317935+0000 mon.a (mon.0) 2128 : audit [DBG] from='client.? 172.21.15.6:0/899256279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:34.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:34 smithi006 bash[27292]: cluster 2023-08-27T23:19:32.877349+0000 mgr.y (mgr.14901) 2430 : cluster [DBG] pgmap v2437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:34.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:34 smithi006 bash[27292]: audit 2023-08-27T23:19:34.317935+0000 mon.a (mon.0) 2128 : audit [DBG] from='client.? 172.21.15.6:0/899256279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:35.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:34 smithi130 bash[27579]: cluster 2023-08-27T23:19:32.877349+0000 mgr.y (mgr.14901) 2430 : cluster [DBG] pgmap v2437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:35.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:34 smithi130 bash[27579]: audit 2023-08-27T23:19:34.317935+0000 mon.a (mon.0) 2128 : audit [DBG] from='client.? 172.21.15.6:0/899256279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:35.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:35 smithi130 bash[51302]: debug 2023-08-27T23:19:35.341+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:35.343474+0000) 2023-08-27T23:19:36.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:36 smithi130 bash[51302]: debug 2023-08-27T23:19:36.341+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:36.343759+0000) 2023-08-27T23:19:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:36 smithi006 bash[27292]: cluster 2023-08-27T23:19:34.878417+0000 mgr.y (mgr.14901) 2431 : cluster [DBG] pgmap v2438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:36 smithi006 bash[33918]: cluster 2023-08-27T23:19:34.878417+0000 mgr.y (mgr.14901) 2431 : cluster [DBG] pgmap v2438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:37.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:36 smithi130 bash[27579]: cluster 2023-08-27T23:19:34.878417+0000 mgr.y (mgr.14901) 2431 : cluster [DBG] pgmap v2438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:37.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:37 smithi130 bash[51302]: debug 2023-08-27T23:19:37.341+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:37.344037+0000) 2023-08-27T23:19:37.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:37 smithi006 bash[27292]: audit 2023-08-27T23:19:36.732544+0000 mon.c (mon.2) 953 : audit [DBG] from='client.? 172.21.15.6:0/383831292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:37.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:37 smithi006 bash[33918]: audit 2023-08-27T23:19:36.732544+0000 mon.c (mon.2) 953 : audit [DBG] from='client.? 172.21.15.6:0/383831292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:38.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:37 smithi130 bash[27579]: audit 2023-08-27T23:19:36.732544+0000 mon.c (mon.2) 953 : audit [DBG] from='client.? 172.21.15.6:0/383831292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:38.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:38 smithi130 bash[51302]: debug 2023-08-27T23:19:38.341+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:38.344343+0000) 2023-08-27T23:19:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:38 smithi006 bash[27292]: cluster 2023-08-27T23:19:36.879045+0000 mgr.y (mgr.14901) 2432 : cluster [DBG] pgmap v2439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:38 smithi006 bash[33918]: cluster 2023-08-27T23:19:36.879045+0000 mgr.y (mgr.14901) 2432 : cluster [DBG] pgmap v2439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:38 smithi130 bash[27579]: cluster 2023-08-27T23:19:36.879045+0000 mgr.y (mgr.14901) 2432 : cluster [DBG] pgmap v2439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:39.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:39 smithi130 bash[51302]: debug 2023-08-27T23:19:39.341+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:39.344623+0000) 2023-08-27T23:19:39.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:39 smithi006 bash[27292]: audit 2023-08-27T23:19:39.142681+0000 mon.a (mon.0) 2129 : audit [DBG] from='client.? 172.21.15.6:0/2602263926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:39 smithi006 bash[33918]: audit 2023-08-27T23:19:39.142681+0000 mon.a (mon.0) 2129 : audit [DBG] from='client.? 172.21.15.6:0/2602263926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:40.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:39 smithi130 bash[27579]: audit 2023-08-27T23:19:39.142681+0000 mon.a (mon.0) 2129 : audit [DBG] from='client.? 172.21.15.6:0/2602263926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:40.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:40 smithi130 bash[51302]: debug 2023-08-27T23:19:40.341+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:40.344958+0000) 2023-08-27T23:19:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:40 smithi006 bash[27292]: cluster 2023-08-27T23:19:38.879545+0000 mgr.y (mgr.14901) 2433 : cluster [DBG] pgmap v2440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:19:40.433Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:19:40.435Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:40 smithi006 bash[33918]: cluster 2023-08-27T23:19:38.879545+0000 mgr.y (mgr.14901) 2433 : cluster [DBG] pgmap v2440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:40 smithi130 bash[27579]: cluster 2023-08-27T23:19:38.879545+0000 mgr.y (mgr.14901) 2433 : cluster [DBG] pgmap v2440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:19:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:19:40] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:19:41.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:41 smithi130 bash[51302]: debug 2023-08-27T23:19:41.341+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:41.345248+0000) 2023-08-27T23:19:41.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:41 smithi006 bash[33918]: audit 2023-08-27T23:19:41.558137+0000 mon.c (mon.2) 954 : audit [DBG] from='client.? 172.21.15.6:0/1576713286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:41.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:41 smithi006 bash[27292]: audit 2023-08-27T23:19:41.558137+0000 mon.c (mon.2) 954 : audit [DBG] from='client.? 172.21.15.6:0/1576713286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:42.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:41 smithi130 bash[27579]: audit 2023-08-27T23:19:41.558137+0000 mon.c (mon.2) 954 : audit [DBG] from='client.? 172.21.15.6:0/1576713286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:41 smithi006 bash[56298]: level=error ts=2023-08-27T23:19:41.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:41 smithi006 bash[56298]: level=warn ts=2023-08-27T23:19:41.998Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:42.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:42 smithi130 bash[51302]: debug 2023-08-27T23:19:42.341+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:42.345570+0000) 2023-08-27T23:19:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:42 smithi006 bash[27292]: cluster 2023-08-27T23:19:40.880602+0000 mgr.y (mgr.14901) 2434 : cluster [DBG] pgmap v2441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:42 smithi006 bash[33918]: cluster 2023-08-27T23:19:40.880602+0000 mgr.y (mgr.14901) 2434 : cluster [DBG] pgmap v2441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:43.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:42 smithi130 bash[27579]: cluster 2023-08-27T23:19:40.880602+0000 mgr.y (mgr.14901) 2434 : cluster [DBG] pgmap v2441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:43.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:43 smithi130 bash[51302]: debug 2023-08-27T23:19:43.341+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:43.345841+0000) 2023-08-27T23:19:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:19:43.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:19:43.905Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:44.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:44 smithi130 bash[51302]: debug 2023-08-27T23:19:44.341+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:44.346132+0000) 2023-08-27T23:19:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:44 smithi006 bash[33918]: cluster 2023-08-27T23:19:42.881254+0000 mgr.y (mgr.14901) 2435 : cluster [DBG] pgmap v2442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:44 smithi006 bash[33918]: audit 2023-08-27T23:19:43.959748+0000 mon.c (mon.2) 955 : audit [DBG] from='client.? 172.21.15.6:0/2906819895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:44.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:44 smithi006 bash[27292]: cluster 2023-08-27T23:19:42.881254+0000 mgr.y (mgr.14901) 2435 : cluster [DBG] pgmap v2442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:44.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:44 smithi006 bash[27292]: audit 2023-08-27T23:19:43.959748+0000 mon.c (mon.2) 955 : audit [DBG] from='client.? 172.21.15.6:0/2906819895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:44 smithi130 bash[27579]: cluster 2023-08-27T23:19:42.881254+0000 mgr.y (mgr.14901) 2435 : cluster [DBG] pgmap v2442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:44 smithi130 bash[27579]: audit 2023-08-27T23:19:43.959748+0000 mon.c (mon.2) 955 : audit [DBG] from='client.? 172.21.15.6:0/2906819895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:45.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:45 smithi130 bash[51302]: debug 2023-08-27T23:19:45.341+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:45.346433+0000) 2023-08-27T23:19:46.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:46 smithi130 bash[51302]: debug 2023-08-27T23:19:46.345+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:46.346723+0000) 2023-08-27T23:19:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:46 smithi006 bash[33918]: cluster 2023-08-27T23:19:44.882288+0000 mgr.y (mgr.14901) 2436 : cluster [DBG] pgmap v2443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:46 smithi006 bash[33918]: audit 2023-08-27T23:19:46.371607+0000 mon.c (mon.2) 956 : audit [DBG] from='client.? 172.21.15.6:0/3413094829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:46 smithi006 bash[27292]: cluster 2023-08-27T23:19:44.882288+0000 mgr.y (mgr.14901) 2436 : cluster [DBG] pgmap v2443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:46.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:46 smithi006 bash[27292]: audit 2023-08-27T23:19:46.371607+0000 mon.c (mon.2) 956 : audit [DBG] from='client.? 172.21.15.6:0/3413094829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:46 smithi130 bash[27579]: cluster 2023-08-27T23:19:44.882288+0000 mgr.y (mgr.14901) 2436 : cluster [DBG] pgmap v2443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:47.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:46 smithi130 bash[27579]: audit 2023-08-27T23:19:46.371607+0000 mon.c (mon.2) 956 : audit [DBG] from='client.? 172.21.15.6:0/3413094829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:47.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:47 smithi130 bash[51302]: debug 2023-08-27T23:19:47.345+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:47.346995+0000) 2023-08-27T23:19:48.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:48 smithi130 bash[51302]: debug 2023-08-27T23:19:48.345+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:48.347307+0000) 2023-08-27T23:19:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:48 smithi006 bash[33918]: cluster 2023-08-27T23:19:46.883019+0000 mgr.y (mgr.14901) 2437 : cluster [DBG] pgmap v2444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:48 smithi006 bash[27292]: cluster 2023-08-27T23:19:46.883019+0000 mgr.y (mgr.14901) 2437 : cluster [DBG] pgmap v2444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:49.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:48 smithi130 bash[27579]: cluster 2023-08-27T23:19:46.883019+0000 mgr.y (mgr.14901) 2437 : cluster [DBG] pgmap v2444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:49.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:49 smithi130 bash[51302]: debug 2023-08-27T23:19:49.345+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:49.347621+0000) 2023-08-27T23:19:49.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:49 smithi006 bash[33918]: audit 2023-08-27T23:19:48.785405+0000 mon.c (mon.2) 957 : audit [DBG] from='client.? 172.21.15.6:0/3782855897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:49.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:49 smithi006 bash[33918]: cluster 2023-08-27T23:19:48.883749+0000 mgr.y (mgr.14901) 2438 : cluster [DBG] pgmap v2445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:49.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:49 smithi006 bash[27292]: audit 2023-08-27T23:19:48.785405+0000 mon.c (mon.2) 957 : audit [DBG] from='client.? 172.21.15.6:0/3782855897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:49.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:49 smithi006 bash[27292]: cluster 2023-08-27T23:19:48.883749+0000 mgr.y (mgr.14901) 2438 : cluster [DBG] pgmap v2445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:50.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:49 smithi130 bash[27579]: audit 2023-08-27T23:19:48.785405+0000 mon.c (mon.2) 957 : audit [DBG] from='client.? 172.21.15.6:0/3782855897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:50.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:49 smithi130 bash[27579]: cluster 2023-08-27T23:19:48.883749+0000 mgr.y (mgr.14901) 2438 : cluster [DBG] pgmap v2445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:50.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:50 smithi130 bash[51302]: debug 2023-08-27T23:19:50.345+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:50.347945+0000) 2023-08-27T23:19:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:19:50.433Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:19:50.435Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:19:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:19:50] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:19:51.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:51 smithi130 bash[51302]: debug 2023-08-27T23:19:51.345+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:51.348275+0000) 2023-08-27T23:19:52.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:51 smithi006 bash[27292]: cluster 2023-08-27T23:19:50.884525+0000 mgr.y (mgr.14901) 2439 : cluster [DBG] pgmap v2446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:52.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:51 smithi006 bash[27292]: audit 2023-08-27T23:19:51.202008+0000 mon.a (mon.0) 2130 : audit [DBG] from='client.? 172.21.15.6:0/157285776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:52.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:51 smithi006 bash[33918]: cluster 2023-08-27T23:19:50.884525+0000 mgr.y (mgr.14901) 2439 : cluster [DBG] pgmap v2446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:52.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:51 smithi006 bash[33918]: audit 2023-08-27T23:19:51.202008+0000 mon.a (mon.0) 2130 : audit [DBG] from='client.? 172.21.15.6:0/157285776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:52.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:51 smithi006 bash[56298]: level=error ts=2023-08-27T23:19:51.997Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:52.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:51 smithi006 bash[56298]: level=warn ts=2023-08-27T23:19:51.998Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:52.349 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:51 smithi130 bash[27579]: cluster 2023-08-27T23:19:50.884525+0000 mgr.y (mgr.14901) 2439 : cluster [DBG] pgmap v2446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:52.349 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:51 smithi130 bash[27579]: audit 2023-08-27T23:19:51.202008+0000 mon.a (mon.0) 2130 : audit [DBG] from='client.? 172.21.15.6:0/157285776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:52.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:52 smithi130 bash[51302]: debug 2023-08-27T23:19:52.345+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:52.348536+0000) 2023-08-27T23:19:53.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:53 smithi130 bash[51302]: debug 2023-08-27T23:19:53.345+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:53.348744+0000) 2023-08-27T23:19:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:19:53.904Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:19:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:19:53.906Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:19:54.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:53 smithi006 bash[27292]: cluster 2023-08-27T23:19:52.885166+0000 mgr.y (mgr.14901) 2440 : cluster [DBG] pgmap v2447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:54.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:53 smithi006 bash[27292]: audit 2023-08-27T23:19:53.617677+0000 mon.a (mon.0) 2131 : audit [DBG] from='client.? 172.21.15.6:0/442618564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:54.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:53 smithi006 bash[33918]: cluster 2023-08-27T23:19:52.885166+0000 mgr.y (mgr.14901) 2440 : cluster [DBG] pgmap v2447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:54.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:53 smithi006 bash[33918]: audit 2023-08-27T23:19:53.617677+0000 mon.a (mon.0) 2131 : audit [DBG] from='client.? 172.21.15.6:0/442618564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:54.350 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:53 smithi130 bash[27579]: cluster 2023-08-27T23:19:52.885166+0000 mgr.y (mgr.14901) 2440 : cluster [DBG] pgmap v2447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:54.350 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:53 smithi130 bash[27579]: audit 2023-08-27T23:19:53.617677+0000 mon.a (mon.0) 2131 : audit [DBG] from='client.? 172.21.15.6:0/442618564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:54.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:54 smithi130 bash[51302]: debug 2023-08-27T23:19:54.345+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:54.349059+0000) 2023-08-27T23:19:55.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:55 smithi130 bash[51302]: debug 2023-08-27T23:19:55.345+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:55.349334+0000) 2023-08-27T23:19:56.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:56 smithi130 bash[51302]: debug 2023-08-27T23:19:56.345+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:56.349592+0000) 2023-08-27T23:19:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:56 smithi130 bash[27579]: cluster 2023-08-27T23:19:54.886261+0000 mgr.y (mgr.14901) 2441 : cluster [DBG] pgmap v2448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:56 smithi130 bash[27579]: audit 2023-08-27T23:19:56.026686+0000 mon.a (mon.0) 2132 : audit [DBG] from='client.? 172.21.15.6:0/540524363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:56 smithi006 bash[33918]: cluster 2023-08-27T23:19:54.886261+0000 mgr.y (mgr.14901) 2441 : cluster [DBG] pgmap v2448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:56 smithi006 bash[33918]: audit 2023-08-27T23:19:56.026686+0000 mon.a (mon.0) 2132 : audit [DBG] from='client.? 172.21.15.6:0/540524363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:56.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:56 smithi006 bash[27292]: cluster 2023-08-27T23:19:54.886261+0000 mgr.y (mgr.14901) 2441 : cluster [DBG] pgmap v2448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:56.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:56 smithi006 bash[27292]: audit 2023-08-27T23:19:56.026686+0000 mon.a (mon.0) 2132 : audit [DBG] from='client.? 172.21.15.6:0/540524363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:57.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:57 smithi130 bash[51302]: debug 2023-08-27T23:19:57.345+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:57.349854+0000) 2023-08-27T23:19:58.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:58 smithi130 bash[51302]: debug 2023-08-27T23:19:58.345+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:58.350131+0000) 2023-08-27T23:19:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:58 smithi130 bash[27579]: cluster 2023-08-27T23:19:56.886917+0000 mgr.y (mgr.14901) 2442 : cluster [DBG] pgmap v2449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:19:58 smithi130 bash[27579]: audit 2023-08-27T23:19:58.439810+0000 mon.a (mon.0) 2133 : audit [DBG] from='client.? 172.21.15.6:0/4146898258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:58 smithi006 bash[27292]: cluster 2023-08-27T23:19:56.886917+0000 mgr.y (mgr.14901) 2442 : cluster [DBG] pgmap v2449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:19:58 smithi006 bash[27292]: audit 2023-08-27T23:19:58.439810+0000 mon.a (mon.0) 2133 : audit [DBG] from='client.? 172.21.15.6:0/4146898258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:58 smithi006 bash[33918]: cluster 2023-08-27T23:19:56.886917+0000 mgr.y (mgr.14901) 2442 : cluster [DBG] pgmap v2449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:19:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:19:58 smithi006 bash[33918]: audit 2023-08-27T23:19:58.439810+0000 mon.a (mon.0) 2133 : audit [DBG] from='client.? 172.21.15.6:0/4146898258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:59.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:19:59 smithi130 bash[51302]: debug 2023-08-27T23:19:59.345+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:59.350389+0000) 2023-08-27T23:20:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:00 smithi130 bash[27579]: cluster 2023-08-27T23:19:58.887547+0000 mgr.y (mgr.14901) 2443 : cluster [DBG] pgmap v2450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:00 smithi130 bash[27579]: cluster 2023-08-27T23:20:00.000120+0000 mon.a (mon.0) 2134 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:20:00.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:00 smithi130 bash[51302]: debug 2023-08-27T23:20:00.349+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:00.350723+0000) 2023-08-27T23:20:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:00 smithi006 bash[33918]: cluster 2023-08-27T23:19:58.887547+0000 mgr.y (mgr.14901) 2443 : cluster [DBG] pgmap v2450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:00 smithi006 bash[33918]: cluster 2023-08-27T23:20:00.000120+0000 mon.a (mon.0) 2134 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:20:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:00 smithi006 bash[27292]: cluster 2023-08-27T23:19:58.887547+0000 mgr.y (mgr.14901) 2443 : cluster [DBG] pgmap v2450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:00 smithi006 bash[27292]: cluster 2023-08-27T23:20:00.000120+0000 mon.a (mon.0) 2134 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:20:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:20:00.434Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:00.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:20:00.436Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:20:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:20:00] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:20:01.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:01 smithi130 bash[27579]: audit 2023-08-27T23:20:00.860217+0000 mon.c (mon.2) 958 : audit [DBG] from='client.? 172.21.15.6:0/3330997450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:01.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:01 smithi130 bash[51302]: debug 2023-08-27T23:20:01.349+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:01.351040+0000) 2023-08-27T23:20:01.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:01 smithi006 bash[27292]: audit 2023-08-27T23:20:00.860217+0000 mon.c (mon.2) 958 : audit [DBG] from='client.? 172.21.15.6:0/3330997450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:01.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:01 smithi006 bash[33918]: audit 2023-08-27T23:20:00.860217+0000 mon.c (mon.2) 958 : audit [DBG] from='client.? 172.21.15.6:0/3330997450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:01 smithi006 bash[56298]: level=error ts=2023-08-27T23:20:01.997Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:20:02.000Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:02.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:02 smithi130 bash[51302]: debug 2023-08-27T23:20:02.349+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:02.351369+0000) 2023-08-27T23:20:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:02 smithi130 bash[27579]: cluster 2023-08-27T23:20:00.888666+0000 mgr.y (mgr.14901) 2444 : cluster [DBG] pgmap v2451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:02 smithi006 bash[33918]: cluster 2023-08-27T23:20:00.888666+0000 mgr.y (mgr.14901) 2444 : cluster [DBG] pgmap v2451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:02 smithi006 bash[27292]: cluster 2023-08-27T23:20:00.888666+0000 mgr.y (mgr.14901) 2444 : cluster [DBG] pgmap v2451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:03.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:03 smithi130 bash[27579]: audit 2023-08-27T23:20:03.276250+0000 mon.a (mon.0) 2135 : audit [DBG] from='client.? 172.21.15.6:0/3561740232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:03.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:03 smithi130 bash[51302]: debug 2023-08-27T23:20:03.349+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:03.351593+0000) 2023-08-27T23:20:03.907 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:03 smithi006 bash[33918]: audit 2023-08-27T23:20:03.276250+0000 mon.a (mon.0) 2135 : audit [DBG] from='client.? 172.21.15.6:0/3561740232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:03.907 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:03 smithi006 bash[27292]: audit 2023-08-27T23:20:03.276250+0000 mon.a (mon.0) 2135 : audit [DBG] from='client.? 172.21.15.6:0/3561740232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:20:03.905Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:20:03.907Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:04.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:04 smithi130 bash[27579]: cluster 2023-08-27T23:20:02.889251+0000 mgr.y (mgr.14901) 2445 : cluster [DBG] pgmap v2452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:04.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:04 smithi130 bash[51302]: debug 2023-08-27T23:20:04.349+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:04.351878+0000) 2023-08-27T23:20:04.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:04 smithi006 bash[27292]: cluster 2023-08-27T23:20:02.889251+0000 mgr.y (mgr.14901) 2445 : cluster [DBG] pgmap v2452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:04 smithi006 bash[33918]: cluster 2023-08-27T23:20:02.889251+0000 mgr.y (mgr.14901) 2445 : cluster [DBG] pgmap v2452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:05.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:05 smithi130 bash[51302]: debug 2023-08-27T23:20:05.349+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:05.352153+0000) 2023-08-27T23:20:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:06 smithi130 bash[27579]: cluster 2023-08-27T23:20:04.890359+0000 mgr.y (mgr.14901) 2446 : cluster [DBG] pgmap v2453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:06 smithi130 bash[27579]: audit 2023-08-27T23:20:05.688137+0000 mon.c (mon.2) 959 : audit [DBG] from='client.? 172.21.15.6:0/156951831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:06 smithi130 bash[27579]: audit 2023-08-27T23:20:05.913336+0000 mon.b (mon.1) 354 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:20:06.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:06 smithi130 bash[51302]: debug 2023-08-27T23:20:06.349+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:06.352514+0000) 2023-08-27T23:20:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:06 smithi006 bash[27292]: cluster 2023-08-27T23:20:04.890359+0000 mgr.y (mgr.14901) 2446 : cluster [DBG] pgmap v2453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:06 smithi006 bash[27292]: audit 2023-08-27T23:20:05.688137+0000 mon.c (mon.2) 959 : audit [DBG] from='client.? 172.21.15.6:0/156951831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:06 smithi006 bash[27292]: audit 2023-08-27T23:20:05.913336+0000 mon.b (mon.1) 354 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:20:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:06 smithi006 bash[33918]: cluster 2023-08-27T23:20:04.890359+0000 mgr.y (mgr.14901) 2446 : cluster [DBG] pgmap v2453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:06 smithi006 bash[33918]: audit 2023-08-27T23:20:05.688137+0000 mon.c (mon.2) 959 : audit [DBG] from='client.? 172.21.15.6:0/156951831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:06.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:06 smithi006 bash[33918]: audit 2023-08-27T23:20:05.913336+0000 mon.b (mon.1) 354 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:20:07.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:07 smithi130 bash[51302]: debug 2023-08-27T23:20:07.349+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:07.352840+0000) 2023-08-27T23:20:08.608 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:08 smithi130 bash[51302]: debug 2023-08-27T23:20:08.349+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:08.353164+0000) 2023-08-27T23:20:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:08 smithi130 bash[27579]: cluster 2023-08-27T23:20:06.891137+0000 mgr.y (mgr.14901) 2447 : cluster [DBG] pgmap v2454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:08 smithi130 bash[27579]: audit 2023-08-27T23:20:08.099304+0000 mon.c (mon.2) 960 : audit [DBG] from='client.? 172.21.15.6:0/3573666524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:08 smithi006 bash[27292]: cluster 2023-08-27T23:20:06.891137+0000 mgr.y (mgr.14901) 2447 : cluster [DBG] pgmap v2454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:08 smithi006 bash[27292]: audit 2023-08-27T23:20:08.099304+0000 mon.c (mon.2) 960 : audit [DBG] from='client.? 172.21.15.6:0/3573666524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:08 smithi006 bash[33918]: cluster 2023-08-27T23:20:06.891137+0000 mgr.y (mgr.14901) 2447 : cluster [DBG] pgmap v2454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:08 smithi006 bash[33918]: audit 2023-08-27T23:20:08.099304+0000 mon.c (mon.2) 960 : audit [DBG] from='client.? 172.21.15.6:0/3573666524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:09.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:09 smithi130 bash[51302]: debug 2023-08-27T23:20:09.349+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:09.353466+0000) 2023-08-27T23:20:10.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:10 smithi130 bash[51302]: debug 2023-08-27T23:20:10.349+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:10.353791+0000) 2023-08-27T23:20:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:10 smithi006 bash[33918]: cluster 2023-08-27T23:20:08.891821+0000 mgr.y (mgr.14901) 2448 : cluster [DBG] pgmap v2455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:10 smithi006 bash[33918]: audit 2023-08-27T23:20:10.538752+0000 mon.c (mon.2) 961 : audit [DBG] from='client.? 172.21.15.6:0/824185040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:20:10.435Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:20:10.437Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:10 smithi006 bash[27292]: cluster 2023-08-27T23:20:08.891821+0000 mgr.y (mgr.14901) 2448 : cluster [DBG] pgmap v2455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:10 smithi006 bash[27292]: audit 2023-08-27T23:20:10.538752+0000 mon.c (mon.2) 961 : audit [DBG] from='client.? 172.21.15.6:0/824185040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:10 smithi130 bash[27579]: cluster 2023-08-27T23:20:08.891821+0000 mgr.y (mgr.14901) 2448 : cluster [DBG] pgmap v2455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:10 smithi130 bash[27579]: audit 2023-08-27T23:20:10.538752+0000 mon.c (mon.2) 961 : audit [DBG] from='client.? 172.21.15.6:0/824185040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:20:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:20:10] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:20:11.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:11 smithi130 bash[51302]: debug 2023-08-27T23:20:11.349+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:11.354008+0000) 2023-08-27T23:20:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:11 smithi006 bash[56298]: level=error ts=2023-08-27T23:20:11.999Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:20:12.001Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:12.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:12 smithi130 bash[51302]: debug 2023-08-27T23:20:12.349+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:12.354208+0000) 2023-08-27T23:20:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:12 smithi006 bash[33918]: cluster 2023-08-27T23:20:10.893005+0000 mgr.y (mgr.14901) 2449 : cluster [DBG] pgmap v2456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:12 smithi006 bash[33918]: audit 2023-08-27T23:20:11.775390+0000 mon.a (mon.0) 2136 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:20:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:12 smithi006 bash[33918]: audit 2023-08-27T23:20:12.305458+0000 mon.a (mon.0) 2137 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:20:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:12 smithi006 bash[27292]: cluster 2023-08-27T23:20:10.893005+0000 mgr.y (mgr.14901) 2449 : cluster [DBG] pgmap v2456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:12 smithi006 bash[27292]: audit 2023-08-27T23:20:11.775390+0000 mon.a (mon.0) 2136 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:20:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:12 smithi006 bash[27292]: audit 2023-08-27T23:20:12.305458+0000 mon.a (mon.0) 2137 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:20:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:12 smithi130 bash[27579]: cluster 2023-08-27T23:20:10.893005+0000 mgr.y (mgr.14901) 2449 : cluster [DBG] pgmap v2456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:12 smithi130 bash[27579]: audit 2023-08-27T23:20:11.775390+0000 mon.a (mon.0) 2136 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:20:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:12 smithi130 bash[27579]: audit 2023-08-27T23:20:12.305458+0000 mon.a (mon.0) 2137 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:20:13.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:13 smithi130 bash[51302]: debug 2023-08-27T23:20:13.349+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:13.354481+0000) 2023-08-27T23:20:13.909 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:13 smithi006 bash[33918]: audit 2023-08-27T23:20:12.958482+0000 mon.c (mon.2) 962 : audit [DBG] from='client.? 172.21.15.6:0/4275697928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:13.909 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:13 smithi006 bash[27292]: audit 2023-08-27T23:20:12.958482+0000 mon.c (mon.2) 962 : audit [DBG] from='client.? 172.21.15.6:0/4275697928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:14.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:13 smithi130 bash[27579]: audit 2023-08-27T23:20:12.958482+0000 mon.c (mon.2) 962 : audit [DBG] from='client.? 172.21.15.6:0/4275697928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:20:13.906Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:20:13.908Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:14.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:14 smithi130 bash[51302]: debug 2023-08-27T23:20:14.349+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:14.354786+0000) 2023-08-27T23:20:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:14 smithi006 bash[33918]: cluster 2023-08-27T23:20:12.893485+0000 mgr.y (mgr.14901) 2450 : cluster [DBG] pgmap v2457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:14 smithi006 bash[27292]: cluster 2023-08-27T23:20:12.893485+0000 mgr.y (mgr.14901) 2450 : cluster [DBG] pgmap v2457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:14 smithi130 bash[27579]: cluster 2023-08-27T23:20:12.893485+0000 mgr.y (mgr.14901) 2450 : cluster [DBG] pgmap v2457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:15.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:15 smithi130 bash[51302]: debug 2023-08-27T23:20:15.353+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:15.355060+0000) 2023-08-27T23:20:15.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:15 smithi006 bash[27292]: audit 2023-08-27T23:20:15.375072+0000 mon.c (mon.2) 963 : audit [DBG] from='client.? 172.21.15.6:0/2994084685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:15.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:15 smithi006 bash[33918]: audit 2023-08-27T23:20:15.375072+0000 mon.c (mon.2) 963 : audit [DBG] from='client.? 172.21.15.6:0/2994084685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:16.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:15 smithi130 bash[27579]: audit 2023-08-27T23:20:15.375072+0000 mon.c (mon.2) 963 : audit [DBG] from='client.? 172.21.15.6:0/2994084685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:16.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:16 smithi130 bash[51302]: debug 2023-08-27T23:20:16.353+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:16.355295+0000) 2023-08-27T23:20:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:16 smithi006 bash[33918]: cluster 2023-08-27T23:20:14.894687+0000 mgr.y (mgr.14901) 2451 : cluster [DBG] pgmap v2458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:16 smithi006 bash[27292]: cluster 2023-08-27T23:20:14.894687+0000 mgr.y (mgr.14901) 2451 : cluster [DBG] pgmap v2458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:16 smithi130 bash[27579]: cluster 2023-08-27T23:20:14.894687+0000 mgr.y (mgr.14901) 2451 : cluster [DBG] pgmap v2458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:17.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:17 smithi130 bash[51302]: debug 2023-08-27T23:20:17.353+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:17.355564+0000) 2023-08-27T23:20:18.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:18 smithi130 bash[51302]: debug 2023-08-27T23:20:18.353+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:18.355881+0000) 2023-08-27T23:20:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:18 smithi006 bash[27292]: cluster 2023-08-27T23:20:16.895151+0000 mgr.y (mgr.14901) 2452 : cluster [DBG] pgmap v2459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:18 smithi006 bash[27292]: audit 2023-08-27T23:20:17.781910+0000 mon.c (mon.2) 964 : audit [DBG] from='client.? 172.21.15.6:0/440223178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:18 smithi006 bash[33918]: cluster 2023-08-27T23:20:16.895151+0000 mgr.y (mgr.14901) 2452 : cluster [DBG] pgmap v2459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:18 smithi006 bash[33918]: audit 2023-08-27T23:20:17.781910+0000 mon.c (mon.2) 964 : audit [DBG] from='client.? 172.21.15.6:0/440223178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:18 smithi130 bash[27579]: cluster 2023-08-27T23:20:16.895151+0000 mgr.y (mgr.14901) 2452 : cluster [DBG] pgmap v2459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:18 smithi130 bash[27579]: audit 2023-08-27T23:20:17.781910+0000 mon.c (mon.2) 964 : audit [DBG] from='client.? 172.21.15.6:0/440223178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:19.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:19 smithi130 bash[51302]: debug 2023-08-27T23:20:19.353+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:19.356151+0000) 2023-08-27T23:20:20.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:20 smithi130 bash[51302]: debug 2023-08-27T23:20:20.353+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:20.356389+0000) 2023-08-27T23:20:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:20 smithi006 bash[27292]: cluster 2023-08-27T23:20:18.895597+0000 mgr.y (mgr.14901) 2453 : cluster [DBG] pgmap v2460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:20 smithi006 bash[27292]: audit 2023-08-27T23:20:20.192889+0000 mon.a (mon.0) 2138 : audit [DBG] from='client.? 172.21.15.6:0/4284903306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:20 smithi006 bash[33918]: cluster 2023-08-27T23:20:18.895597+0000 mgr.y (mgr.14901) 2453 : cluster [DBG] pgmap v2460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:20 smithi006 bash[33918]: audit 2023-08-27T23:20:20.192889+0000 mon.a (mon.0) 2138 : audit [DBG] from='client.? 172.21.15.6:0/4284903306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:20:20.435Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:20:20.437Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:20 smithi130 bash[27579]: cluster 2023-08-27T23:20:18.895597+0000 mgr.y (mgr.14901) 2453 : cluster [DBG] pgmap v2460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:20 smithi130 bash[27579]: audit 2023-08-27T23:20:20.192889+0000 mon.a (mon.0) 2138 : audit [DBG] from='client.? 172.21.15.6:0/4284903306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:20:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:20:20] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:20:21.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:21 smithi130 bash[51302]: debug 2023-08-27T23:20:21.353+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:21.356723+0000) 2023-08-27T23:20:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:20:21.999Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:20:22.002Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:22.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:22 smithi130 bash[51302]: debug 2023-08-27T23:20:22.353+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:22.356971+0000) 2023-08-27T23:20:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:22 smithi006 bash[27292]: cluster 2023-08-27T23:20:20.896527+0000 mgr.y (mgr.14901) 2454 : cluster [DBG] pgmap v2461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:22 smithi006 bash[27292]: audit 2023-08-27T23:20:21.807851+0000 mon.a (mon.0) 2139 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:20:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:22 smithi006 bash[27292]: audit 2023-08-27T23:20:22.107975+0000 mon.b (mon.1) 355 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:20:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:22 smithi006 bash[27292]: audit 2023-08-27T23:20:22.109851+0000 mon.b (mon.1) 356 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:20:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:22 smithi006 bash[27292]: audit 2023-08-27T23:20:22.118981+0000 mon.a (mon.0) 2140 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:20:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:22 smithi006 bash[27292]: audit 2023-08-27T23:20:22.609639+0000 mon.c (mon.2) 965 : audit [DBG] from='client.? 172.21.15.6:0/1416694072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:22.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:22 smithi006 bash[33918]: cluster 2023-08-27T23:20:20.896527+0000 mgr.y (mgr.14901) 2454 : cluster [DBG] pgmap v2461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:22.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:22 smithi006 bash[33918]: audit 2023-08-27T23:20:21.807851+0000 mon.a (mon.0) 2139 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:20:22.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:22 smithi006 bash[33918]: audit 2023-08-27T23:20:22.107975+0000 mon.b (mon.1) 355 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:20:22.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:22 smithi006 bash[33918]: audit 2023-08-27T23:20:22.109851+0000 mon.b (mon.1) 356 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:20:22.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:22 smithi006 bash[33918]: audit 2023-08-27T23:20:22.118981+0000 mon.a (mon.0) 2140 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:20:22.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:22 smithi006 bash[33918]: audit 2023-08-27T23:20:22.609639+0000 mon.c (mon.2) 965 : audit [DBG] from='client.? 172.21.15.6:0/1416694072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:22 smithi130 bash[27579]: cluster 2023-08-27T23:20:20.896527+0000 mgr.y (mgr.14901) 2454 : cluster [DBG] pgmap v2461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:22 smithi130 bash[27579]: audit 2023-08-27T23:20:21.807851+0000 mon.a (mon.0) 2139 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:20:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:22 smithi130 bash[27579]: audit 2023-08-27T23:20:22.107975+0000 mon.b (mon.1) 355 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:20:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:22 smithi130 bash[27579]: audit 2023-08-27T23:20:22.109851+0000 mon.b (mon.1) 356 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:20:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:22 smithi130 bash[27579]: audit 2023-08-27T23:20:22.118981+0000 mon.a (mon.0) 2140 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:20:23.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:22 smithi130 bash[27579]: audit 2023-08-27T23:20:22.609639+0000 mon.c (mon.2) 965 : audit [DBG] from='client.? 172.21.15.6:0/1416694072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:23.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:23 smithi130 bash[51302]: debug 2023-08-27T23:20:23.353+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:23.357242+0000) 2023-08-27T23:20:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:20:23.906Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:20:23.908Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:24.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:24 smithi130 bash[51302]: debug 2023-08-27T23:20:24.353+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:24.357513+0000) 2023-08-27T23:20:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:24 smithi006 bash[27292]: cluster 2023-08-27T23:20:22.897168+0000 mgr.y (mgr.14901) 2455 : cluster [DBG] pgmap v2462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:24.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:24 smithi006 bash[33918]: cluster 2023-08-27T23:20:22.897168+0000 mgr.y (mgr.14901) 2455 : cluster [DBG] pgmap v2462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:24 smithi130 bash[27579]: cluster 2023-08-27T23:20:22.897168+0000 mgr.y (mgr.14901) 2455 : cluster [DBG] pgmap v2462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:25.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:25 smithi130 bash[51302]: debug 2023-08-27T23:20:25.353+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:25.357819+0000) 2023-08-27T23:20:25.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:25 smithi006 bash[33918]: audit 2023-08-27T23:20:25.033661+0000 mon.a (mon.0) 2141 : audit [DBG] from='client.? 172.21.15.6:0/3056193590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:25.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:25 smithi006 bash[27292]: audit 2023-08-27T23:20:25.033661+0000 mon.a (mon.0) 2141 : audit [DBG] from='client.? 172.21.15.6:0/3056193590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:26.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:25 smithi130 bash[27579]: audit 2023-08-27T23:20:25.033661+0000 mon.a (mon.0) 2141 : audit [DBG] from='client.? 172.21.15.6:0/3056193590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:26.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:26 smithi130 bash[51302]: debug 2023-08-27T23:20:26.353+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:26.358131+0000) 2023-08-27T23:20:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:26 smithi006 bash[27292]: cluster 2023-08-27T23:20:24.898286+0000 mgr.y (mgr.14901) 2456 : cluster [DBG] pgmap v2463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:26 smithi006 bash[33918]: cluster 2023-08-27T23:20:24.898286+0000 mgr.y (mgr.14901) 2456 : cluster [DBG] pgmap v2463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:26 smithi130 bash[27579]: cluster 2023-08-27T23:20:24.898286+0000 mgr.y (mgr.14901) 2456 : cluster [DBG] pgmap v2463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:27.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:27 smithi130 bash[51302]: debug 2023-08-27T23:20:27.353+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:27.358365+0000) 2023-08-27T23:20:27.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:27 smithi006 bash[27292]: audit 2023-08-27T23:20:27.441682+0000 mon.a (mon.0) 2142 : audit [DBG] from='client.? 172.21.15.6:0/2980699891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:27.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:27 smithi006 bash[33918]: audit 2023-08-27T23:20:27.441682+0000 mon.a (mon.0) 2142 : audit [DBG] from='client.? 172.21.15.6:0/2980699891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:28.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:27 smithi130 bash[27579]: audit 2023-08-27T23:20:27.441682+0000 mon.a (mon.0) 2142 : audit [DBG] from='client.? 172.21.15.6:0/2980699891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:28.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:28 smithi130 bash[51302]: debug 2023-08-27T23:20:28.357+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:28.358617+0000) 2023-08-27T23:20:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:28 smithi006 bash[27292]: cluster 2023-08-27T23:20:26.898938+0000 mgr.y (mgr.14901) 2457 : cluster [DBG] pgmap v2464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:28 smithi006 bash[33918]: cluster 2023-08-27T23:20:26.898938+0000 mgr.y (mgr.14901) 2457 : cluster [DBG] pgmap v2464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:28 smithi130 bash[27579]: cluster 2023-08-27T23:20:26.898938+0000 mgr.y (mgr.14901) 2457 : cluster [DBG] pgmap v2464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:29.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:29 smithi130 bash[51302]: debug 2023-08-27T23:20:29.356+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:29.358926+0000) 2023-08-27T23:20:30.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:30 smithi130 bash[51302]: debug 2023-08-27T23:20:30.356+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:30.359248+0000) 2023-08-27T23:20:30.694 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:20:30.436Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:30.694 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:20:30.438Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:20:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:20:30] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:20:30.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:30 smithi006 bash[27292]: cluster 2023-08-27T23:20:28.899586+0000 mgr.y (mgr.14901) 2458 : cluster [DBG] pgmap v2465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:30.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:30 smithi006 bash[27292]: audit 2023-08-27T23:20:29.853204+0000 mon.a (mon.0) 2143 : audit [DBG] from='client.? 172.21.15.6:0/3606109148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:30.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:30 smithi006 bash[33918]: cluster 2023-08-27T23:20:28.899586+0000 mgr.y (mgr.14901) 2458 : cluster [DBG] pgmap v2465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:30.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:30 smithi006 bash[33918]: audit 2023-08-27T23:20:29.853204+0000 mon.a (mon.0) 2143 : audit [DBG] from='client.? 172.21.15.6:0/3606109148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:30 smithi130 bash[27579]: cluster 2023-08-27T23:20:28.899586+0000 mgr.y (mgr.14901) 2458 : cluster [DBG] pgmap v2465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:31.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:30 smithi130 bash[27579]: audit 2023-08-27T23:20:29.853204+0000 mon.a (mon.0) 2143 : audit [DBG] from='client.? 172.21.15.6:0/3606109148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:31.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:31 smithi130 bash[51302]: debug 2023-08-27T23:20:31.356+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:31.359558+0000) 2023-08-27T23:20:32.003 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:20:32.000Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:32.003 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:31 smithi006 bash[27292]: cluster 2023-08-27T23:20:30.900872+0000 mgr.y (mgr.14901) 2459 : cluster [DBG] pgmap v2466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:32.004 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:31 smithi006 bash[33918]: cluster 2023-08-27T23:20:30.900872+0000 mgr.y (mgr.14901) 2459 : cluster [DBG] pgmap v2466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:32.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:31 smithi130 bash[27579]: cluster 2023-08-27T23:20:30.900872+0000 mgr.y (mgr.14901) 2459 : cluster [DBG] pgmap v2466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:20:32.002Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:32.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:32 smithi130 bash[51302]: debug 2023-08-27T23:20:32.356+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:32.359785+0000) 2023-08-27T23:20:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:32 smithi130 bash[27579]: audit 2023-08-27T23:20:32.273860+0000 mon.c (mon.2) 966 : audit [DBG] from='client.? 172.21.15.6:0/2391386699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:32 smithi006 bash[33918]: audit 2023-08-27T23:20:32.273860+0000 mon.c (mon.2) 966 : audit [DBG] from='client.? 172.21.15.6:0/2391386699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:32 smithi006 bash[27292]: audit 2023-08-27T23:20:32.273860+0000 mon.c (mon.2) 966 : audit [DBG] from='client.? 172.21.15.6:0/2391386699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:33.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:33 smithi130 bash[51302]: debug 2023-08-27T23:20:33.356+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:33.360091+0000) 2023-08-27T23:20:34.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:33 smithi130 bash[27579]: cluster 2023-08-27T23:20:32.901554+0000 mgr.y (mgr.14901) 2460 : cluster [DBG] pgmap v2467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:34.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:33 smithi006 bash[33918]: cluster 2023-08-27T23:20:32.901554+0000 mgr.y (mgr.14901) 2460 : cluster [DBG] pgmap v2467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:34.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:33 smithi006 bash[27292]: cluster 2023-08-27T23:20:32.901554+0000 mgr.y (mgr.14901) 2460 : cluster [DBG] pgmap v2467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:20:33.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:34.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:20:33.908Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:34.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:34 smithi130 bash[51302]: debug 2023-08-27T23:20:34.356+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:34.360394+0000) 2023-08-27T23:20:35.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:35 smithi130 bash[27579]: audit 2023-08-27T23:20:34.692357+0000 mon.a (mon.0) 2144 : audit [DBG] from='client.? 172.21.15.6:0/2242959196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:35.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:35 smithi130 bash[51302]: debug 2023-08-27T23:20:35.356+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:35.360677+0000) 2023-08-27T23:20:35.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:35 smithi006 bash[27292]: audit 2023-08-27T23:20:34.692357+0000 mon.a (mon.0) 2144 : audit [DBG] from='client.? 172.21.15.6:0/2242959196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:35.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:35 smithi006 bash[33918]: audit 2023-08-27T23:20:34.692357+0000 mon.a (mon.0) 2144 : audit [DBG] from='client.? 172.21.15.6:0/2242959196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:36.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:36 smithi130 bash[27579]: cluster 2023-08-27T23:20:34.902662+0000 mgr.y (mgr.14901) 2461 : cluster [DBG] pgmap v2468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:36.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:36 smithi130 bash[51302]: debug 2023-08-27T23:20:36.356+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:36.360944+0000) 2023-08-27T23:20:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:36 smithi006 bash[33918]: cluster 2023-08-27T23:20:34.902662+0000 mgr.y (mgr.14901) 2461 : cluster [DBG] pgmap v2468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:36 smithi006 bash[27292]: cluster 2023-08-27T23:20:34.902662+0000 mgr.y (mgr.14901) 2461 : cluster [DBG] pgmap v2468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:37.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:37 smithi130 bash[27579]: audit 2023-08-27T23:20:37.112374+0000 mon.c (mon.2) 967 : audit [DBG] from='client.? 172.21.15.6:0/3158940812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:37.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:37 smithi130 bash[51302]: debug 2023-08-27T23:20:37.356+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:37.361229+0000) 2023-08-27T23:20:37.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:37 smithi006 bash[27292]: audit 2023-08-27T23:20:37.112374+0000 mon.c (mon.2) 967 : audit [DBG] from='client.? 172.21.15.6:0/3158940812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:37.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:37 smithi006 bash[33918]: audit 2023-08-27T23:20:37.112374+0000 mon.c (mon.2) 967 : audit [DBG] from='client.? 172.21.15.6:0/3158940812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:38 smithi130 bash[27579]: cluster 2023-08-27T23:20:36.903154+0000 mgr.y (mgr.14901) 2462 : cluster [DBG] pgmap v2469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:38.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:38 smithi130 bash[51302]: debug 2023-08-27T23:20:38.356+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:38.361589+0000) 2023-08-27T23:20:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:38 smithi006 bash[33918]: cluster 2023-08-27T23:20:36.903154+0000 mgr.y (mgr.14901) 2462 : cluster [DBG] pgmap v2469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:38 smithi006 bash[27292]: cluster 2023-08-27T23:20:36.903154+0000 mgr.y (mgr.14901) 2462 : cluster [DBG] pgmap v2469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:39.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:39 smithi130 bash[27579]: audit 2023-08-27T23:20:39.532884+0000 mon.a (mon.0) 2145 : audit [DBG] from='client.? 172.21.15.6:0/3015485434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:39.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:39 smithi130 bash[51302]: debug 2023-08-27T23:20:39.356+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:39.361948+0000) 2023-08-27T23:20:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:39 smithi006 bash[33918]: audit 2023-08-27T23:20:39.532884+0000 mon.a (mon.0) 2145 : audit [DBG] from='client.? 172.21.15.6:0/3015485434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:39.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:39 smithi006 bash[27292]: audit 2023-08-27T23:20:39.532884+0000 mon.a (mon.0) 2145 : audit [DBG] from='client.? 172.21.15.6:0/3015485434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:40.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:40 smithi130 bash[27579]: cluster 2023-08-27T23:20:38.903784+0000 mgr.y (mgr.14901) 2463 : cluster [DBG] pgmap v2470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:40.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:40 smithi130 bash[51302]: debug 2023-08-27T23:20:40.360+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:40.362219+0000) 2023-08-27T23:20:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:40 smithi006 bash[27292]: cluster 2023-08-27T23:20:38.903784+0000 mgr.y (mgr.14901) 2463 : cluster [DBG] pgmap v2470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:40 smithi006 bash[33918]: cluster 2023-08-27T23:20:38.903784+0000 mgr.y (mgr.14901) 2463 : cluster [DBG] pgmap v2470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:20:40.437Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:40.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:20:40.439Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:20:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:20:40] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:20:41.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:41 smithi130 bash[51302]: debug 2023-08-27T23:20:41.356+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:41.362450+0000) 2023-08-27T23:20:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:20:42.001Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:20:42.003Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:42.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:42 smithi130 bash[27579]: cluster 2023-08-27T23:20:40.904930+0000 mgr.y (mgr.14901) 2464 : cluster [DBG] pgmap v2471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:42.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:42 smithi130 bash[27579]: audit 2023-08-27T23:20:41.949402+0000 mon.a (mon.0) 2146 : audit [DBG] from='client.? 172.21.15.6:0/983552612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:42.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:42 smithi130 bash[51302]: debug 2023-08-27T23:20:42.356+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:42.362684+0000) 2023-08-27T23:20:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:42 smithi006 bash[33918]: cluster 2023-08-27T23:20:40.904930+0000 mgr.y (mgr.14901) 2464 : cluster [DBG] pgmap v2471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:42 smithi006 bash[33918]: audit 2023-08-27T23:20:41.949402+0000 mon.a (mon.0) 2146 : audit [DBG] from='client.? 172.21.15.6:0/983552612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:42 smithi006 bash[27292]: cluster 2023-08-27T23:20:40.904930+0000 mgr.y (mgr.14901) 2464 : cluster [DBG] pgmap v2471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:42 smithi006 bash[27292]: audit 2023-08-27T23:20:41.949402+0000 mon.a (mon.0) 2146 : audit [DBG] from='client.? 172.21.15.6:0/983552612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:43.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:43 smithi130 bash[51302]: debug 2023-08-27T23:20:43.360+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:43.363035+0000) 2023-08-27T23:20:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:20:43.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:44.316 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:20:43.909Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:44.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:44 smithi130 bash[27579]: cluster 2023-08-27T23:20:42.905531+0000 mgr.y (mgr.14901) 2465 : cluster [DBG] pgmap v2472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:44.840 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:44 smithi130 bash[27579]: audit 2023-08-27T23:20:44.362198+0000 mon.c (mon.2) 968 : audit [DBG] from='client.? 172.21.15.6:0/428300759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:44.842 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:44 smithi130 bash[51302]: debug 2023-08-27T23:20:44.360+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:44.363268+0000) 2023-08-27T23:20:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:44 smithi006 bash[33918]: cluster 2023-08-27T23:20:42.905531+0000 mgr.y (mgr.14901) 2465 : cluster [DBG] pgmap v2472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:44 smithi006 bash[33918]: audit 2023-08-27T23:20:44.362198+0000 mon.c (mon.2) 968 : audit [DBG] from='client.? 172.21.15.6:0/428300759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:44.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:44 smithi006 bash[27292]: cluster 2023-08-27T23:20:42.905531+0000 mgr.y (mgr.14901) 2465 : cluster [DBG] pgmap v2472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:44.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:44 smithi006 bash[27292]: audit 2023-08-27T23:20:44.362198+0000 mon.c (mon.2) 968 : audit [DBG] from='client.? 172.21.15.6:0/428300759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:45.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:45 smithi130 bash[51302]: debug 2023-08-27T23:20:45.360+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:45.363502+0000) 2023-08-27T23:20:46.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:46 smithi130 bash[51302]: debug 2023-08-27T23:20:46.360+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:46.363773+0000) 2023-08-27T23:20:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:46 smithi130 bash[27579]: cluster 2023-08-27T23:20:44.906592+0000 mgr.y (mgr.14901) 2466 : cluster [DBG] pgmap v2473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:46 smithi006 bash[33918]: cluster 2023-08-27T23:20:44.906592+0000 mgr.y (mgr.14901) 2466 : cluster [DBG] pgmap v2473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:46 smithi006 bash[27292]: cluster 2023-08-27T23:20:44.906592+0000 mgr.y (mgr.14901) 2466 : cluster [DBG] pgmap v2473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:47.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:47 smithi130 bash[51302]: debug 2023-08-27T23:20:47.360+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:47.364080+0000) 2023-08-27T23:20:47.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:47 smithi006 bash[27292]: audit 2023-08-27T23:20:46.770664+0000 mon.c (mon.2) 969 : audit [DBG] from='client.? 172.21.15.6:0/3737400254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:47.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:47 smithi006 bash[33918]: audit 2023-08-27T23:20:46.770664+0000 mon.c (mon.2) 969 : audit [DBG] from='client.? 172.21.15.6:0/3737400254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:48.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:47 smithi130 bash[27579]: audit 2023-08-27T23:20:46.770664+0000 mon.c (mon.2) 969 : audit [DBG] from='client.? 172.21.15.6:0/3737400254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:48.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:48 smithi130 bash[51302]: debug 2023-08-27T23:20:48.360+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:48.364369+0000) 2023-08-27T23:20:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:48 smithi006 bash[27292]: cluster 2023-08-27T23:20:46.907247+0000 mgr.y (mgr.14901) 2467 : cluster [DBG] pgmap v2474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:48 smithi006 bash[33918]: cluster 2023-08-27T23:20:46.907247+0000 mgr.y (mgr.14901) 2467 : cluster [DBG] pgmap v2474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:48 smithi130 bash[27579]: cluster 2023-08-27T23:20:46.907247+0000 mgr.y (mgr.14901) 2467 : cluster [DBG] pgmap v2474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:49.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:49 smithi130 bash[51302]: debug 2023-08-27T23:20:49.360+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:49.364648+0000) 2023-08-27T23:20:49.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:49 smithi006 bash[33918]: audit 2023-08-27T23:20:49.190170+0000 mon.c (mon.2) 970 : audit [DBG] from='client.? 172.21.15.6:0/993064540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:49.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:49 smithi006 bash[27292]: audit 2023-08-27T23:20:49.190170+0000 mon.c (mon.2) 970 : audit [DBG] from='client.? 172.21.15.6:0/993064540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:50.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:49 smithi130 bash[27579]: audit 2023-08-27T23:20:49.190170+0000 mon.c (mon.2) 970 : audit [DBG] from='client.? 172.21.15.6:0/993064540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:50.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:50 smithi130 bash[51302]: debug 2023-08-27T23:20:50.360+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:50.364855+0000) 2023-08-27T23:20:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:50 smithi006 bash[27292]: cluster 2023-08-27T23:20:48.907847+0000 mgr.y (mgr.14901) 2468 : cluster [DBG] pgmap v2475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:50 smithi006 bash[33918]: cluster 2023-08-27T23:20:48.907847+0000 mgr.y (mgr.14901) 2468 : cluster [DBG] pgmap v2475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:20:50.438Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:20:50.439Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:50 smithi130 bash[27579]: cluster 2023-08-27T23:20:48.907847+0000 mgr.y (mgr.14901) 2468 : cluster [DBG] pgmap v2475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:20:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:20:50] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:20:51.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:51 smithi130 bash[51302]: debug 2023-08-27T23:20:51.360+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:51.365104+0000) 2023-08-27T23:20:51.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:51 smithi006 bash[27292]: audit 2023-08-27T23:20:51.609575+0000 mon.a (mon.0) 2147 : audit [DBG] from='client.? 172.21.15.6:0/2413339892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:51.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:51 smithi006 bash[33918]: audit 2023-08-27T23:20:51.609575+0000 mon.a (mon.0) 2147 : audit [DBG] from='client.? 172.21.15.6:0/2413339892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:52.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:51 smithi130 bash[27579]: audit 2023-08-27T23:20:51.609575+0000 mon.a (mon.0) 2147 : audit [DBG] from='client.? 172.21.15.6:0/2413339892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:20:52.001Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:20:52.003Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:52.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:52 smithi130 bash[51302]: debug 2023-08-27T23:20:52.360+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:52.365395+0000) 2023-08-27T23:20:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:52 smithi006 bash[33918]: cluster 2023-08-27T23:20:50.909035+0000 mgr.y (mgr.14901) 2469 : cluster [DBG] pgmap v2476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:52 smithi006 bash[27292]: cluster 2023-08-27T23:20:50.909035+0000 mgr.y (mgr.14901) 2469 : cluster [DBG] pgmap v2476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:53.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:52 smithi130 bash[27579]: cluster 2023-08-27T23:20:50.909035+0000 mgr.y (mgr.14901) 2469 : cluster [DBG] pgmap v2476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:53.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:53 smithi130 bash[51302]: debug 2023-08-27T23:20:53.360+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:53.365653+0000) 2023-08-27T23:20:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:20:53.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:20:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:20:53.909Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:20:54.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:54 smithi130 bash[51302]: debug 2023-08-27T23:20:54.364+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:54.365937+0000) 2023-08-27T23:20:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:54 smithi006 bash[33918]: cluster 2023-08-27T23:20:52.909810+0000 mgr.y (mgr.14901) 2470 : cluster [DBG] pgmap v2477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:54 smithi006 bash[33918]: audit 2023-08-27T23:20:54.026084+0000 mon.c (mon.2) 971 : audit [DBG] from='client.? 172.21.15.6:0/1073341268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:54 smithi006 bash[27292]: cluster 2023-08-27T23:20:52.909810+0000 mgr.y (mgr.14901) 2470 : cluster [DBG] pgmap v2477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:54.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:54 smithi006 bash[27292]: audit 2023-08-27T23:20:54.026084+0000 mon.c (mon.2) 971 : audit [DBG] from='client.? 172.21.15.6:0/1073341268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:54 smithi130 bash[27579]: cluster 2023-08-27T23:20:52.909810+0000 mgr.y (mgr.14901) 2470 : cluster [DBG] pgmap v2477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:54 smithi130 bash[27579]: audit 2023-08-27T23:20:54.026084+0000 mon.c (mon.2) 971 : audit [DBG] from='client.? 172.21.15.6:0/1073341268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:55.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:55 smithi130 bash[51302]: debug 2023-08-27T23:20:55.364+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:55.366237+0000) 2023-08-27T23:20:56.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:56 smithi130 bash[51302]: debug 2023-08-27T23:20:56.360+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:56.366502+0000) 2023-08-27T23:20:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:56 smithi006 bash[27292]: cluster 2023-08-27T23:20:54.910944+0000 mgr.y (mgr.14901) 2471 : cluster [DBG] pgmap v2478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:56 smithi006 bash[27292]: audit 2023-08-27T23:20:56.444694+0000 mon.c (mon.2) 972 : audit [DBG] from='client.? 172.21.15.6:0/3085038754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:56 smithi006 bash[33918]: cluster 2023-08-27T23:20:54.910944+0000 mgr.y (mgr.14901) 2471 : cluster [DBG] pgmap v2478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:56 smithi006 bash[33918]: audit 2023-08-27T23:20:56.444694+0000 mon.c (mon.2) 972 : audit [DBG] from='client.? 172.21.15.6:0/3085038754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:56 smithi130 bash[27579]: cluster 2023-08-27T23:20:54.910944+0000 mgr.y (mgr.14901) 2471 : cluster [DBG] pgmap v2478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:56 smithi130 bash[27579]: audit 2023-08-27T23:20:56.444694+0000 mon.c (mon.2) 972 : audit [DBG] from='client.? 172.21.15.6:0/3085038754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:57.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:57 smithi130 bash[51302]: debug 2023-08-27T23:20:57.364+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:57.366775+0000) 2023-08-27T23:20:58.665 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:58 smithi130 bash[51302]: debug 2023-08-27T23:20:58.364+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:58.367055+0000) 2023-08-27T23:20:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:58 smithi006 bash[33918]: cluster 2023-08-27T23:20:56.911563+0000 mgr.y (mgr.14901) 2472 : cluster [DBG] pgmap v2479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:58 smithi006 bash[27292]: cluster 2023-08-27T23:20:56.911563+0000 mgr.y (mgr.14901) 2472 : cluster [DBG] pgmap v2479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:58 smithi130 bash[27579]: cluster 2023-08-27T23:20:56.911563+0000 mgr.y (mgr.14901) 2472 : cluster [DBG] pgmap v2479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:20:59.674 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:20:59 smithi130 bash[51302]: debug 2023-08-27T23:20:59.364+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:59.367267+0000) 2023-08-27T23:20:59.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:20:59 smithi006 bash[27292]: audit 2023-08-27T23:20:58.849809+0000 mon.c (mon.2) 973 : audit [DBG] from='client.? 172.21.15.6:0/2025913022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:20:59 smithi006 bash[33918]: audit 2023-08-27T23:20:58.849809+0000 mon.c (mon.2) 973 : audit [DBG] from='client.? 172.21.15.6:0/2025913022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:00.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:20:59 smithi130 bash[27579]: audit 2023-08-27T23:20:58.849809+0000 mon.c (mon.2) 973 : audit [DBG] from='client.? 172.21.15.6:0/2025913022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:00.677 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:00 smithi130 bash[51302]: debug 2023-08-27T23:21:00.364+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:00.367517+0000) 2023-08-27T23:21:00.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:00 smithi006 bash[33918]: cluster 2023-08-27T23:20:58.912205+0000 mgr.y (mgr.14901) 2473 : cluster [DBG] pgmap v2480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:00 smithi006 bash[27292]: cluster 2023-08-27T23:20:58.912205+0000 mgr.y (mgr.14901) 2473 : cluster [DBG] pgmap v2480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:21:00.438Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:21:00.440Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:00 smithi130 bash[27579]: cluster 2023-08-27T23:20:58.912205+0000 mgr.y (mgr.14901) 2473 : cluster [DBG] pgmap v2480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:21:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:21:00] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:21:01.686 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:01 smithi130 bash[51302]: debug 2023-08-27T23:21:01.364+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:01.367805+0000) 2023-08-27T23:21:01.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:01 smithi006 bash[27292]: audit 2023-08-27T23:21:01.275545+0000 mon.b (mon.1) 357 : audit [DBG] from='client.? 172.21.15.6:0/2131608817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:01.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:01 smithi006 bash[33918]: audit 2023-08-27T23:21:01.275545+0000 mon.b (mon.1) 357 : audit [DBG] from='client.? 172.21.15.6:0/2131608817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:02.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:01 smithi130 bash[27579]: audit 2023-08-27T23:21:01.275545+0000 mon.b (mon.1) 357 : audit [DBG] from='client.? 172.21.15.6:0/2131608817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:21:02.002Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:21:02.004Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:02.688 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:02 smithi130 bash[51302]: debug 2023-08-27T23:21:02.364+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:02.368029+0000) 2023-08-27T23:21:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:02 smithi006 bash[33918]: cluster 2023-08-27T23:21:00.913032+0000 mgr.y (mgr.14901) 2474 : cluster [DBG] pgmap v2481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:02 smithi006 bash[27292]: cluster 2023-08-27T23:21:00.913032+0000 mgr.y (mgr.14901) 2474 : cluster [DBG] pgmap v2481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:02 smithi130 bash[27579]: cluster 2023-08-27T23:21:00.913032+0000 mgr.y (mgr.14901) 2474 : cluster [DBG] pgmap v2481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:03.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:03 smithi130 bash[51302]: debug 2023-08-27T23:21:03.364+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:03.368300+0000) 2023-08-27T23:21:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:21:03.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:21:03.911Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:04.693 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:04 smithi130 bash[51302]: debug 2023-08-27T23:21:04.368+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:04.368603+0000) 2023-08-27T23:21:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:04 smithi006 bash[33918]: cluster 2023-08-27T23:21:02.913618+0000 mgr.y (mgr.14901) 2475 : cluster [DBG] pgmap v2482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:04 smithi006 bash[33918]: audit 2023-08-27T23:21:03.691359+0000 mon.c (mon.2) 974 : audit [DBG] from='client.? 172.21.15.6:0/3883600959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:04.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:04 smithi006 bash[27292]: cluster 2023-08-27T23:21:02.913618+0000 mgr.y (mgr.14901) 2475 : cluster [DBG] pgmap v2482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:04.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:04 smithi006 bash[27292]: audit 2023-08-27T23:21:03.691359+0000 mon.c (mon.2) 974 : audit [DBG] from='client.? 172.21.15.6:0/3883600959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:04 smithi130 bash[27579]: cluster 2023-08-27T23:21:02.913618+0000 mgr.y (mgr.14901) 2475 : cluster [DBG] pgmap v2482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:04 smithi130 bash[27579]: audit 2023-08-27T23:21:03.691359+0000 mon.c (mon.2) 974 : audit [DBG] from='client.? 172.21.15.6:0/3883600959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:05.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:05 smithi130 bash[51302]: debug 2023-08-27T23:21:05.364+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:05.368879+0000) 2023-08-27T23:21:06.698 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:06 smithi130 bash[51302]: debug 2023-08-27T23:21:06.368+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:06.369150+0000) 2023-08-27T23:21:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:06 smithi006 bash[27292]: cluster 2023-08-27T23:21:04.914843+0000 mgr.y (mgr.14901) 2476 : cluster [DBG] pgmap v2483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:06 smithi006 bash[27292]: audit 2023-08-27T23:21:06.107569+0000 mon.a (mon.0) 2148 : audit [DBG] from='client.? 172.21.15.6:0/3673362280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:06 smithi006 bash[33918]: cluster 2023-08-27T23:21:04.914843+0000 mgr.y (mgr.14901) 2476 : cluster [DBG] pgmap v2483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:06 smithi006 bash[33918]: audit 2023-08-27T23:21:06.107569+0000 mon.a (mon.0) 2148 : audit [DBG] from='client.? 172.21.15.6:0/3673362280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:06 smithi130 bash[27579]: cluster 2023-08-27T23:21:04.914843+0000 mgr.y (mgr.14901) 2476 : cluster [DBG] pgmap v2483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:06 smithi130 bash[27579]: audit 2023-08-27T23:21:06.107569+0000 mon.a (mon.0) 2148 : audit [DBG] from='client.? 172.21.15.6:0/3673362280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:07.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:07 smithi130 bash[51302]: debug 2023-08-27T23:21:07.364+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:07.369403+0000) 2023-08-27T23:21:08.703 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:08 smithi130 bash[51302]: debug 2023-08-27T23:21:08.364+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:08.369709+0000) 2023-08-27T23:21:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:08 smithi006 bash[27292]: cluster 2023-08-27T23:21:06.915697+0000 mgr.y (mgr.14901) 2477 : cluster [DBG] pgmap v2484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:08 smithi006 bash[27292]: audit 2023-08-27T23:21:08.515891+0000 mon.c (mon.2) 975 : audit [DBG] from='client.? 172.21.15.6:0/2232302231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:08 smithi006 bash[33918]: cluster 2023-08-27T23:21:06.915697+0000 mgr.y (mgr.14901) 2477 : cluster [DBG] pgmap v2484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:08 smithi006 bash[33918]: audit 2023-08-27T23:21:08.515891+0000 mon.c (mon.2) 975 : audit [DBG] from='client.? 172.21.15.6:0/2232302231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:08 smithi130 bash[27579]: cluster 2023-08-27T23:21:06.915697+0000 mgr.y (mgr.14901) 2477 : cluster [DBG] pgmap v2484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:08 smithi130 bash[27579]: audit 2023-08-27T23:21:08.515891+0000 mon.c (mon.2) 975 : audit [DBG] from='client.? 172.21.15.6:0/2232302231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:09.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:09 smithi130 bash[51302]: debug 2023-08-27T23:21:09.368+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:09.369984+0000) 2023-08-27T23:21:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:10 smithi006 bash[33918]: cluster 2023-08-27T23:21:08.916407+0000 mgr.y (mgr.14901) 2478 : cluster [DBG] pgmap v2485: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:21:10.439Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:21:10.441Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:10 smithi006 bash[27292]: cluster 2023-08-27T23:21:08.916407+0000 mgr.y (mgr.14901) 2478 : cluster [DBG] pgmap v2485: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:10 smithi130 bash[27579]: cluster 2023-08-27T23:21:08.916407+0000 mgr.y (mgr.14901) 2478 : cluster [DBG] pgmap v2485: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:10 smithi130 bash[51302]: debug 2023-08-27T23:21:10.368+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:10.370298+0000) 2023-08-27T23:21:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:21:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:21:10] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:21:11.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:11 smithi130 bash[27579]: audit 2023-08-27T23:21:10.944958+0000 mon.c (mon.2) 976 : audit [DBG] from='client.? 172.21.15.6:0/998192853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:11.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:11 smithi130 bash[51302]: debug 2023-08-27T23:21:11.364+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:11.370643+0000) 2023-08-27T23:21:11.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:11 smithi006 bash[27292]: audit 2023-08-27T23:21:10.944958+0000 mon.c (mon.2) 976 : audit [DBG] from='client.? 172.21.15.6:0/998192853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:11.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:11 smithi006 bash[33918]: audit 2023-08-27T23:21:10.944958+0000 mon.c (mon.2) 976 : audit [DBG] from='client.? 172.21.15.6:0/998192853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:21:12.003Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:21:12.004Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:12 smithi130 bash[27579]: cluster 2023-08-27T23:21:10.917630+0000 mgr.y (mgr.14901) 2479 : cluster [DBG] pgmap v2486: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:12 smithi130 bash[51302]: debug 2023-08-27T23:21:12.368+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:12.370962+0000) 2023-08-27T23:21:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:12 smithi006 bash[33918]: cluster 2023-08-27T23:21:10.917630+0000 mgr.y (mgr.14901) 2479 : cluster [DBG] pgmap v2486: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:12 smithi006 bash[27292]: cluster 2023-08-27T23:21:10.917630+0000 mgr.y (mgr.14901) 2479 : cluster [DBG] pgmap v2486: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:13.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:13 smithi130 bash[27579]: audit 2023-08-27T23:21:13.356891+0000 mon.c (mon.2) 977 : audit [DBG] from='client.? 172.21.15.6:0/3913014557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:13.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:13 smithi130 bash[51302]: debug 2023-08-27T23:21:13.368+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:13.371250+0000) 2023-08-27T23:21:13.911 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:13 smithi006 bash[27292]: audit 2023-08-27T23:21:13.356891+0000 mon.c (mon.2) 977 : audit [DBG] from='client.? 172.21.15.6:0/3913014557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:13.911 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:13 smithi006 bash[33918]: audit 2023-08-27T23:21:13.356891+0000 mon.c (mon.2) 977 : audit [DBG] from='client.? 172.21.15.6:0/3913014557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:21:13.909Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:21:13.911Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:14 smithi130 bash[27579]: cluster 2023-08-27T23:21:12.918377+0000 mgr.y (mgr.14901) 2480 : cluster [DBG] pgmap v2487: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:14 smithi130 bash[51302]: debug 2023-08-27T23:21:14.368+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:14.371549+0000) 2023-08-27T23:21:14.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:14 smithi006 bash[27292]: cluster 2023-08-27T23:21:12.918377+0000 mgr.y (mgr.14901) 2480 : cluster [DBG] pgmap v2487: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:14 smithi006 bash[33918]: cluster 2023-08-27T23:21:12.918377+0000 mgr.y (mgr.14901) 2480 : cluster [DBG] pgmap v2487: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:15.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:15 smithi130 bash[51302]: debug 2023-08-27T23:21:15.368+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:15.371836+0000) 2023-08-27T23:21:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:16 smithi130 bash[27579]: cluster 2023-08-27T23:21:14.919398+0000 mgr.y (mgr.14901) 2481 : cluster [DBG] pgmap v2488: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:16 smithi130 bash[27579]: audit 2023-08-27T23:21:15.771793+0000 mon.a (mon.0) 2149 : audit [DBG] from='client.? 172.21.15.6:0/1470126450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:16.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:16 smithi130 bash[51302]: debug 2023-08-27T23:21:16.368+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:16.372152+0000) 2023-08-27T23:21:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:16 smithi006 bash[33918]: cluster 2023-08-27T23:21:14.919398+0000 mgr.y (mgr.14901) 2481 : cluster [DBG] pgmap v2488: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:16 smithi006 bash[33918]: audit 2023-08-27T23:21:15.771793+0000 mon.a (mon.0) 2149 : audit [DBG] from='client.? 172.21.15.6:0/1470126450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:16 smithi006 bash[27292]: cluster 2023-08-27T23:21:14.919398+0000 mgr.y (mgr.14901) 2481 : cluster [DBG] pgmap v2488: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:16 smithi006 bash[27292]: audit 2023-08-27T23:21:15.771793+0000 mon.a (mon.0) 2149 : audit [DBG] from='client.? 172.21.15.6:0/1470126450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:17.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:17 smithi130 bash[51302]: debug 2023-08-27T23:21:17.368+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:17.372426+0000) 2023-08-27T23:21:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:18 smithi130 bash[27579]: cluster 2023-08-27T23:21:16.920054+0000 mgr.y (mgr.14901) 2482 : cluster [DBG] pgmap v2489: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:18 smithi130 bash[27579]: audit 2023-08-27T23:21:18.195685+0000 mon.c (mon.2) 978 : audit [DBG] from='client.? 172.21.15.6:0/2224748934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:18 smithi130 bash[51302]: debug 2023-08-27T23:21:18.368+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:18.372709+0000) 2023-08-27T23:21:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:18 smithi006 bash[33918]: cluster 2023-08-27T23:21:16.920054+0000 mgr.y (mgr.14901) 2482 : cluster [DBG] pgmap v2489: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:18 smithi006 bash[33918]: audit 2023-08-27T23:21:18.195685+0000 mon.c (mon.2) 978 : audit [DBG] from='client.? 172.21.15.6:0/2224748934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:18 smithi006 bash[27292]: cluster 2023-08-27T23:21:16.920054+0000 mgr.y (mgr.14901) 2482 : cluster [DBG] pgmap v2489: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:18 smithi006 bash[27292]: audit 2023-08-27T23:21:18.195685+0000 mon.c (mon.2) 978 : audit [DBG] from='client.? 172.21.15.6:0/2224748934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:19.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:19 smithi130 bash[51302]: debug 2023-08-27T23:21:19.368+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:19.372974+0000) 2023-08-27T23:21:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:21:20.440Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:21:20.442Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:20 smithi006 bash[27292]: cluster 2023-08-27T23:21:18.920703+0000 mgr.y (mgr.14901) 2483 : cluster [DBG] pgmap v2490: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:20 smithi006 bash[27292]: audit 2023-08-27T23:21:20.606988+0000 mon.a (mon.0) 2150 : audit [DBG] from='client.? 172.21.15.6:0/1133809602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:20.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:20 smithi006 bash[33918]: cluster 2023-08-27T23:21:18.920703+0000 mgr.y (mgr.14901) 2483 : cluster [DBG] pgmap v2490: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:20.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:20 smithi006 bash[33918]: audit 2023-08-27T23:21:20.606988+0000 mon.a (mon.0) 2150 : audit [DBG] from='client.? 172.21.15.6:0/1133809602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:20 smithi130 bash[51302]: debug 2023-08-27T23:21:20.368+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:20.373238+0000) 2023-08-27T23:21:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:20 smithi130 bash[27579]: cluster 2023-08-27T23:21:18.920703+0000 mgr.y (mgr.14901) 2483 : cluster [DBG] pgmap v2490: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:20 smithi130 bash[27579]: audit 2023-08-27T23:21:20.606988+0000 mon.a (mon.0) 2150 : audit [DBG] from='client.? 172.21.15.6:0/1133809602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:21.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:21:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:21:20] "GET /metrics HTTP/1.1" 200 193083 "" "Prometheus/2.33.4" 2023-08-27T23:21:21.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:21 smithi130 bash[51302]: debug 2023-08-27T23:21:21.368+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:21.373514+0000) 2023-08-27T23:21:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:21:22.003Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:21:22.005Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:22.626 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:22 smithi130 bash[51302]: debug 2023-08-27T23:21:22.368+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:22.373760+0000) 2023-08-27T23:21:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:22 smithi006 bash[33918]: cluster 2023-08-27T23:21:20.921845+0000 mgr.y (mgr.14901) 2484 : cluster [DBG] pgmap v2491: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:22 smithi006 bash[33918]: audit 2023-08-27T23:21:22.133927+0000 mon.b (mon.1) 358 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:21:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:22 smithi006 bash[33918]: audit 2023-08-27T23:21:22.413901+0000 mon.a (mon.0) 2151 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:21:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:22 smithi006 bash[33918]: audit 2023-08-27T23:21:22.420550+0000 mon.a (mon.0) 2152 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:21:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:22 smithi006 bash[27292]: cluster 2023-08-27T23:21:20.921845+0000 mgr.y (mgr.14901) 2484 : cluster [DBG] pgmap v2491: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:22 smithi006 bash[27292]: audit 2023-08-27T23:21:22.133927+0000 mon.b (mon.1) 358 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:21:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:22 smithi006 bash[27292]: audit 2023-08-27T23:21:22.413901+0000 mon.a (mon.0) 2151 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:21:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:22 smithi006 bash[27292]: audit 2023-08-27T23:21:22.420550+0000 mon.a (mon.0) 2152 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:21:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:22 smithi130 bash[27579]: cluster 2023-08-27T23:21:20.921845+0000 mgr.y (mgr.14901) 2484 : cluster [DBG] pgmap v2491: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:22 smithi130 bash[27579]: audit 2023-08-27T23:21:22.133927+0000 mon.b (mon.1) 358 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:21:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:22 smithi130 bash[27579]: audit 2023-08-27T23:21:22.413901+0000 mon.a (mon.0) 2151 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:21:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:22 smithi130 bash[27579]: audit 2023-08-27T23:21:22.420550+0000 mon.a (mon.0) 2152 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:21:23.637 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:23 smithi130 bash[51302]: debug 2023-08-27T23:21:23.368+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:23.374038+0000) 2023-08-27T23:21:23.912 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:23 smithi006 bash[27292]: audit 2023-08-27T23:21:23.015021+0000 mon.a (mon.0) 2153 : audit [DBG] from='client.? 172.21.15.6:0/1962697012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:23.912 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:23 smithi006 bash[33918]: audit 2023-08-27T23:21:23.015021+0000 mon.a (mon.0) 2153 : audit [DBG] from='client.? 172.21.15.6:0/1962697012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:24.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:23 smithi130 bash[27579]: audit 2023-08-27T23:21:23.015021+0000 mon.a (mon.0) 2153 : audit [DBG] from='client.? 172.21.15.6:0/1962697012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:21:23.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:21:23.912Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:24.638 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:24 smithi130 bash[51302]: debug 2023-08-27T23:21:24.371+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:24.374314+0000) 2023-08-27T23:21:24.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:24 smithi006 bash[33918]: cluster 2023-08-27T23:21:22.922478+0000 mgr.y (mgr.14901) 2485 : cluster [DBG] pgmap v2492: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:24 smithi006 bash[27292]: cluster 2023-08-27T23:21:22.922478+0000 mgr.y (mgr.14901) 2485 : cluster [DBG] pgmap v2492: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:24 smithi130 bash[27579]: cluster 2023-08-27T23:21:22.922478+0000 mgr.y (mgr.14901) 2485 : cluster [DBG] pgmap v2492: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:25.650 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:25 smithi130 bash[51302]: debug 2023-08-27T23:21:25.367+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:25.374619+0000) 2023-08-27T23:21:25.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:25 smithi006 bash[27292]: audit 2023-08-27T23:21:25.436109+0000 mon.a (mon.0) 2154 : audit [DBG] from='client.? 172.21.15.6:0/652458181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:25.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:25 smithi006 bash[33918]: audit 2023-08-27T23:21:25.436109+0000 mon.a (mon.0) 2154 : audit [DBG] from='client.? 172.21.15.6:0/652458181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:26.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:25 smithi130 bash[27579]: audit 2023-08-27T23:21:25.436109+0000 mon.a (mon.0) 2154 : audit [DBG] from='client.? 172.21.15.6:0/652458181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:26.653 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:26 smithi130 bash[51302]: debug 2023-08-27T23:21:26.371+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:26.375046+0000) 2023-08-27T23:21:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:26 smithi006 bash[33918]: cluster 2023-08-27T23:21:24.923616+0000 mgr.y (mgr.14901) 2486 : cluster [DBG] pgmap v2493: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:26 smithi006 bash[27292]: cluster 2023-08-27T23:21:24.923616+0000 mgr.y (mgr.14901) 2486 : cluster [DBG] pgmap v2493: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:26 smithi130 bash[27579]: cluster 2023-08-27T23:21:24.923616+0000 mgr.y (mgr.14901) 2486 : cluster [DBG] pgmap v2493: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:27.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:27 smithi130 bash[51302]: debug 2023-08-27T23:21:27.371+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:27.375248+0000) 2023-08-27T23:21:28.658 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:28 smithi130 bash[51302]: debug 2023-08-27T23:21:28.371+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:28.375453+0000) 2023-08-27T23:21:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:28 smithi006 bash[27292]: cluster 2023-08-27T23:21:26.924454+0000 mgr.y (mgr.14901) 2487 : cluster [DBG] pgmap v2494: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:28 smithi006 bash[27292]: audit 2023-08-27T23:21:27.872165+0000 mon.c (mon.2) 979 : audit [DBG] from='client.? 172.21.15.6:0/1801210474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:28 smithi006 bash[27292]: audit 2023-08-27T23:21:28.293150+0000 mon.a (mon.0) 2155 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:21:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:28 smithi006 bash[33918]: cluster 2023-08-27T23:21:26.924454+0000 mgr.y (mgr.14901) 2487 : cluster [DBG] pgmap v2494: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:28 smithi006 bash[33918]: audit 2023-08-27T23:21:27.872165+0000 mon.c (mon.2) 979 : audit [DBG] from='client.? 172.21.15.6:0/1801210474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:28 smithi006 bash[33918]: audit 2023-08-27T23:21:28.293150+0000 mon.a (mon.0) 2155 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:21:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:28 smithi130 bash[27579]: cluster 2023-08-27T23:21:26.924454+0000 mgr.y (mgr.14901) 2487 : cluster [DBG] pgmap v2494: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:28 smithi130 bash[27579]: audit 2023-08-27T23:21:27.872165+0000 mon.c (mon.2) 979 : audit [DBG] from='client.? 172.21.15.6:0/1801210474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:28 smithi130 bash[27579]: audit 2023-08-27T23:21:28.293150+0000 mon.a (mon.0) 2155 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:21:29.746 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:29 smithi130 bash[51302]: debug 2023-08-27T23:21:29.371+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:29.375746+0000) 2023-08-27T23:21:30.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:29 smithi130 bash[27579]: audit 2023-08-27T23:21:28.737468+0000 mon.a (mon.0) 2156 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:21:30.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:29 smithi130 bash[27579]: cluster 2023-08-27T23:21:28.925196+0000 mgr.y (mgr.14901) 2488 : cluster [DBG] pgmap v2495: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:30.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:29 smithi130 bash[27579]: audit 2023-08-27T23:21:29.035498+0000 mon.b (mon.1) 359 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:21:30.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:29 smithi130 bash[27579]: audit 2023-08-27T23:21:29.037962+0000 mon.b (mon.1) 360 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:21:30.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:29 smithi130 bash[27579]: audit 2023-08-27T23:21:29.046127+0000 mon.a (mon.0) 2157 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:21:30.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:29 smithi006 bash[27292]: audit 2023-08-27T23:21:28.737468+0000 mon.a (mon.0) 2156 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:21:30.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:29 smithi006 bash[27292]: cluster 2023-08-27T23:21:28.925196+0000 mgr.y (mgr.14901) 2488 : cluster [DBG] pgmap v2495: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:30.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:29 smithi006 bash[27292]: audit 2023-08-27T23:21:29.035498+0000 mon.b (mon.1) 359 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:21:30.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:29 smithi006 bash[27292]: audit 2023-08-27T23:21:29.037962+0000 mon.b (mon.1) 360 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:21:30.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:29 smithi006 bash[27292]: audit 2023-08-27T23:21:29.046127+0000 mon.a (mon.0) 2157 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:21:30.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:29 smithi006 bash[33918]: audit 2023-08-27T23:21:28.737468+0000 mon.a (mon.0) 2156 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:21:30.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:29 smithi006 bash[33918]: cluster 2023-08-27T23:21:28.925196+0000 mgr.y (mgr.14901) 2488 : cluster [DBG] pgmap v2495: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:30.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:29 smithi006 bash[33918]: audit 2023-08-27T23:21:29.035498+0000 mon.b (mon.1) 359 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:21:30.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:29 smithi006 bash[33918]: audit 2023-08-27T23:21:29.037962+0000 mon.b (mon.1) 360 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:21:30.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:29 smithi006 bash[33918]: audit 2023-08-27T23:21:29.046127+0000 mon.a (mon.0) 2157 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:21:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:21:30.440Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:21:30.442Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:30.748 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:30 smithi130 bash[51302]: debug 2023-08-27T23:21:30.371+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:30.376042+0000) 2023-08-27T23:21:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:30 smithi130 bash[27579]: audit 2023-08-27T23:21:30.297266+0000 mon.a (mon.0) 2158 : audit [DBG] from='client.? 172.21.15.6:0/3625381750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:21:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:21:30] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:21:31.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:30 smithi006 bash[27292]: audit 2023-08-27T23:21:30.297266+0000 mon.a (mon.0) 2158 : audit [DBG] from='client.? 172.21.15.6:0/3625381750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:31.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:30 smithi006 bash[33918]: audit 2023-08-27T23:21:30.297266+0000 mon.a (mon.0) 2158 : audit [DBG] from='client.? 172.21.15.6:0/3625381750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:31.755 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:31 smithi130 bash[51302]: debug 2023-08-27T23:21:31.371+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:31.376307+0000) 2023-08-27T23:21:32.006 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:31 smithi006 bash[33918]: cluster 2023-08-27T23:21:30.926303+0000 mgr.y (mgr.14901) 2489 : cluster [DBG] pgmap v2496: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:32.006 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:31 smithi006 bash[27292]: cluster 2023-08-27T23:21:30.926303+0000 mgr.y (mgr.14901) 2489 : cluster [DBG] pgmap v2496: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:32.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:31 smithi130 bash[27579]: cluster 2023-08-27T23:21:30.926303+0000 mgr.y (mgr.14901) 2489 : cluster [DBG] pgmap v2496: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:21:32.004Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:21:32.005Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:32.759 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:32 smithi130 bash[51302]: debug 2023-08-27T23:21:32.371+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:32.376636+0000) 2023-08-27T23:21:33.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:32 smithi130 bash[27579]: audit 2023-08-27T23:21:32.709932+0000 mon.a (mon.0) 2159 : audit [DBG] from='client.? 172.21.15.6:0/1672909298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:32 smithi006 bash[33918]: audit 2023-08-27T23:21:32.709932+0000 mon.a (mon.0) 2159 : audit [DBG] from='client.? 172.21.15.6:0/1672909298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:32 smithi006 bash[27292]: audit 2023-08-27T23:21:32.709932+0000 mon.a (mon.0) 2159 : audit [DBG] from='client.? 172.21.15.6:0/1672909298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:33.766 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:33 smithi130 bash[51302]: debug 2023-08-27T23:21:33.371+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:33.376910+0000) 2023-08-27T23:21:34.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:33 smithi130 bash[27579]: cluster 2023-08-27T23:21:32.927011+0000 mgr.y (mgr.14901) 2490 : cluster [DBG] pgmap v2497: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:34.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:33 smithi006 bash[33918]: cluster 2023-08-27T23:21:32.927011+0000 mgr.y (mgr.14901) 2490 : cluster [DBG] pgmap v2497: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:34.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:33 smithi006 bash[27292]: cluster 2023-08-27T23:21:32.927011+0000 mgr.y (mgr.14901) 2490 : cluster [DBG] pgmap v2497: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:21:33.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:21:33.912Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:34.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:34 smithi130 bash[51302]: debug 2023-08-27T23:21:34.371+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:34.377200+0000) 2023-08-27T23:21:35.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:35 smithi130 bash[27579]: audit 2023-08-27T23:21:35.127598+0000 mon.a (mon.0) 2160 : audit [DBG] from='client.? 172.21.15.6:0/1822458121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:35.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:35 smithi130 bash[51302]: debug 2023-08-27T23:21:35.371+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:35.377505+0000) 2023-08-27T23:21:35.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:35 smithi006 bash[33918]: audit 2023-08-27T23:21:35.127598+0000 mon.a (mon.0) 2160 : audit [DBG] from='client.? 172.21.15.6:0/1822458121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:35.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:35 smithi006 bash[27292]: audit 2023-08-27T23:21:35.127598+0000 mon.a (mon.0) 2160 : audit [DBG] from='client.? 172.21.15.6:0/1822458121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:36 smithi130 bash[27579]: cluster 2023-08-27T23:21:34.928009+0000 mgr.y (mgr.14901) 2491 : cluster [DBG] pgmap v2498: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:36 smithi130 bash[51302]: debug 2023-08-27T23:21:36.375+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:36.377826+0000) 2023-08-27T23:21:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:36 smithi006 bash[33918]: cluster 2023-08-27T23:21:34.928009+0000 mgr.y (mgr.14901) 2491 : cluster [DBG] pgmap v2498: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:36 smithi006 bash[27292]: cluster 2023-08-27T23:21:34.928009+0000 mgr.y (mgr.14901) 2491 : cluster [DBG] pgmap v2498: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:37.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:37 smithi130 bash[27579]: audit 2023-08-27T23:21:37.551615+0000 mon.c (mon.2) 980 : audit [DBG] from='client.? 172.21.15.6:0/2603182658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:37 smithi130 bash[51302]: debug 2023-08-27T23:21:37.375+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:37.378142+0000) 2023-08-27T23:21:37.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:37 smithi006 bash[33918]: audit 2023-08-27T23:21:37.551615+0000 mon.c (mon.2) 980 : audit [DBG] from='client.? 172.21.15.6:0/2603182658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:37.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:37 smithi006 bash[27292]: audit 2023-08-27T23:21:37.551615+0000 mon.c (mon.2) 980 : audit [DBG] from='client.? 172.21.15.6:0/2603182658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:38 smithi130 bash[27579]: cluster 2023-08-27T23:21:36.928685+0000 mgr.y (mgr.14901) 2492 : cluster [DBG] pgmap v2499: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:38 smithi130 bash[51302]: debug 2023-08-27T23:21:38.371+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:38.378418+0000) 2023-08-27T23:21:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:38 smithi006 bash[33918]: cluster 2023-08-27T23:21:36.928685+0000 mgr.y (mgr.14901) 2492 : cluster [DBG] pgmap v2499: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:38 smithi006 bash[27292]: cluster 2023-08-27T23:21:36.928685+0000 mgr.y (mgr.14901) 2492 : cluster [DBG] pgmap v2499: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:39 smithi130 bash[51302]: debug 2023-08-27T23:21:39.371+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:39.378697+0000) 2023-08-27T23:21:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:40 smithi006 bash[33918]: cluster 2023-08-27T23:21:38.929334+0000 mgr.y (mgr.14901) 2493 : cluster [DBG] pgmap v2500: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:40 smithi006 bash[33918]: audit 2023-08-27T23:21:39.973314+0000 mon.c (mon.2) 981 : audit [DBG] from='client.? 172.21.15.6:0/1032015301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:40 smithi006 bash[27292]: cluster 2023-08-27T23:21:38.929334+0000 mgr.y (mgr.14901) 2493 : cluster [DBG] pgmap v2500: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:40 smithi006 bash[27292]: audit 2023-08-27T23:21:39.973314+0000 mon.c (mon.2) 981 : audit [DBG] from='client.? 172.21.15.6:0/1032015301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:40.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:21:40.442Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:40.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:21:40.445Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:40 smithi130 bash[27579]: cluster 2023-08-27T23:21:38.929334+0000 mgr.y (mgr.14901) 2493 : cluster [DBG] pgmap v2500: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:40 smithi130 bash[27579]: audit 2023-08-27T23:21:39.973314+0000 mon.c (mon.2) 981 : audit [DBG] from='client.? 172.21.15.6:0/1032015301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:40 smithi130 bash[51302]: debug 2023-08-27T23:21:40.375+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:40.378999+0000) 2023-08-27T23:21:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:21:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:21:40] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:21:41.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:41 smithi130 bash[51302]: debug 2023-08-27T23:21:41.375+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:41.379230+0000) 2023-08-27T23:21:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:21:42.004Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:21:42.006Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:42.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:42 smithi130 bash[51302]: debug 2023-08-27T23:21:42.375+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:42.379459+0000) 2023-08-27T23:21:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:42 smithi130 bash[27579]: cluster 2023-08-27T23:21:40.930483+0000 mgr.y (mgr.14901) 2494 : cluster [DBG] pgmap v2501: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:42 smithi130 bash[27579]: audit 2023-08-27T23:21:42.400414+0000 mon.a (mon.0) 2161 : audit [DBG] from='client.? 172.21.15.6:0/1997791791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:42 smithi006 bash[33918]: cluster 2023-08-27T23:21:40.930483+0000 mgr.y (mgr.14901) 2494 : cluster [DBG] pgmap v2501: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:42 smithi006 bash[33918]: audit 2023-08-27T23:21:42.400414+0000 mon.a (mon.0) 2161 : audit [DBG] from='client.? 172.21.15.6:0/1997791791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:42 smithi006 bash[27292]: cluster 2023-08-27T23:21:40.930483+0000 mgr.y (mgr.14901) 2494 : cluster [DBG] pgmap v2501: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:42 smithi006 bash[27292]: audit 2023-08-27T23:21:42.400414+0000 mon.a (mon.0) 2161 : audit [DBG] from='client.? 172.21.15.6:0/1997791791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:43.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:43 smithi130 bash[51302]: debug 2023-08-27T23:21:43.375+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:43.379747+0000) 2023-08-27T23:21:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:21:43.912Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:21:43.914Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:44 smithi130 bash[27579]: cluster 2023-08-27T23:21:42.931158+0000 mgr.y (mgr.14901) 2495 : cluster [DBG] pgmap v2502: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:44 smithi130 bash[51302]: debug 2023-08-27T23:21:44.375+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:44.379972+0000) 2023-08-27T23:21:44.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:44 smithi006 bash[27292]: cluster 2023-08-27T23:21:42.931158+0000 mgr.y (mgr.14901) 2495 : cluster [DBG] pgmap v2502: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:44 smithi006 bash[33918]: cluster 2023-08-27T23:21:42.931158+0000 mgr.y (mgr.14901) 2495 : cluster [DBG] pgmap v2502: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:45.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:45 smithi130 bash[51302]: debug 2023-08-27T23:21:45.379+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:45.380259+0000) 2023-08-27T23:21:45.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:45 smithi130 bash[27579]: audit 2023-08-27T23:21:44.817903+0000 mon.a (mon.0) 2162 : audit [DBG] from='client.? 172.21.15.6:0/1556592105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:45.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:45 smithi006 bash[33918]: audit 2023-08-27T23:21:44.817903+0000 mon.a (mon.0) 2162 : audit [DBG] from='client.? 172.21.15.6:0/1556592105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:45.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:45 smithi006 bash[27292]: audit 2023-08-27T23:21:44.817903+0000 mon.a (mon.0) 2162 : audit [DBG] from='client.? 172.21.15.6:0/1556592105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:46 smithi130 bash[27579]: cluster 2023-08-27T23:21:44.932446+0000 mgr.y (mgr.14901) 2496 : cluster [DBG] pgmap v2503: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:46.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:46 smithi130 bash[51302]: debug 2023-08-27T23:21:46.375+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:46.380531+0000) 2023-08-27T23:21:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:46 smithi006 bash[27292]: cluster 2023-08-27T23:21:44.932446+0000 mgr.y (mgr.14901) 2496 : cluster [DBG] pgmap v2503: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:46 smithi006 bash[33918]: cluster 2023-08-27T23:21:44.932446+0000 mgr.y (mgr.14901) 2496 : cluster [DBG] pgmap v2503: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:47.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:47 smithi130 bash[27579]: audit 2023-08-27T23:21:47.236879+0000 mon.a (mon.0) 2163 : audit [DBG] from='client.? 172.21.15.6:0/900867444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:47.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:47 smithi130 bash[51302]: debug 2023-08-27T23:21:47.375+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:47.380818+0000) 2023-08-27T23:21:47.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:47 smithi006 bash[33918]: audit 2023-08-27T23:21:47.236879+0000 mon.a (mon.0) 2163 : audit [DBG] from='client.? 172.21.15.6:0/900867444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:47.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:47 smithi006 bash[27292]: audit 2023-08-27T23:21:47.236879+0000 mon.a (mon.0) 2163 : audit [DBG] from='client.? 172.21.15.6:0/900867444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:48.642 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:48 smithi130 bash[51302]: debug 2023-08-27T23:21:48.375+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:48.381118+0000) 2023-08-27T23:21:48.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:48 smithi006 bash[27292]: cluster 2023-08-27T23:21:46.933252+0000 mgr.y (mgr.14901) 2497 : cluster [DBG] pgmap v2504: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:48 smithi006 bash[33918]: cluster 2023-08-27T23:21:46.933252+0000 mgr.y (mgr.14901) 2497 : cluster [DBG] pgmap v2504: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:49.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:48 smithi130 bash[27579]: cluster 2023-08-27T23:21:46.933252+0000 mgr.y (mgr.14901) 2497 : cluster [DBG] pgmap v2504: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:49.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:49 smithi130 bash[51302]: debug 2023-08-27T23:21:49.375+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:49.381371+0000) 2023-08-27T23:21:50.647 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:50 smithi130 bash[51302]: debug 2023-08-27T23:21:50.375+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:50.381645+0000) 2023-08-27T23:21:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:50 smithi006 bash[33918]: cluster 2023-08-27T23:21:48.933997+0000 mgr.y (mgr.14901) 2498 : cluster [DBG] pgmap v2505: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:50 smithi006 bash[33918]: audit 2023-08-27T23:21:49.651824+0000 mon.c (mon.2) 982 : audit [DBG] from='client.? 172.21.15.6:0/400632872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:21:50.442Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:21:50.444Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:50 smithi006 bash[27292]: cluster 2023-08-27T23:21:48.933997+0000 mgr.y (mgr.14901) 2498 : cluster [DBG] pgmap v2505: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:50 smithi006 bash[27292]: audit 2023-08-27T23:21:49.651824+0000 mon.c (mon.2) 982 : audit [DBG] from='client.? 172.21.15.6:0/400632872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:50 smithi130 bash[27579]: cluster 2023-08-27T23:21:48.933997+0000 mgr.y (mgr.14901) 2498 : cluster [DBG] pgmap v2505: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:50 smithi130 bash[27579]: audit 2023-08-27T23:21:49.651824+0000 mon.c (mon.2) 982 : audit [DBG] from='client.? 172.21.15.6:0/400632872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:21:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:21:50] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T23:21:51.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:51 smithi130 bash[51302]: debug 2023-08-27T23:21:51.375+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:51.381949+0000) 2023-08-27T23:21:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:21:52.005Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:21:52.006Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:52.660 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:52 smithi130 bash[51302]: debug 2023-08-27T23:21:52.375+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:52.382256+0000) 2023-08-27T23:21:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:52 smithi006 bash[33918]: cluster 2023-08-27T23:21:50.935140+0000 mgr.y (mgr.14901) 2499 : cluster [DBG] pgmap v2506: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:52 smithi006 bash[33918]: audit 2023-08-27T23:21:52.061968+0000 mon.c (mon.2) 983 : audit [DBG] from='client.? 172.21.15.6:0/134262630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:52 smithi006 bash[27292]: cluster 2023-08-27T23:21:50.935140+0000 mgr.y (mgr.14901) 2499 : cluster [DBG] pgmap v2506: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:52 smithi006 bash[27292]: audit 2023-08-27T23:21:52.061968+0000 mon.c (mon.2) 983 : audit [DBG] from='client.? 172.21.15.6:0/134262630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:53.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:52 smithi130 bash[27579]: cluster 2023-08-27T23:21:50.935140+0000 mgr.y (mgr.14901) 2499 : cluster [DBG] pgmap v2506: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:52 smithi130 bash[27579]: audit 2023-08-27T23:21:52.061968+0000 mon.c (mon.2) 983 : audit [DBG] from='client.? 172.21.15.6:0/134262630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:53.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:53 smithi130 bash[51302]: debug 2023-08-27T23:21:53.375+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:53.382553+0000) 2023-08-27T23:21:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:21:53.913Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:21:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:21:53.915Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:21:54.665 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:54 smithi130 bash[51302]: debug 2023-08-27T23:21:54.375+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:54.382828+0000) 2023-08-27T23:21:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:54 smithi006 bash[27292]: cluster 2023-08-27T23:21:52.935907+0000 mgr.y (mgr.14901) 2500 : cluster [DBG] pgmap v2507: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:54 smithi006 bash[27292]: audit 2023-08-27T23:21:54.475137+0000 mon.c (mon.2) 984 : audit [DBG] from='client.? 172.21.15.6:0/4013311852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:54 smithi006 bash[33918]: cluster 2023-08-27T23:21:52.935907+0000 mgr.y (mgr.14901) 2500 : cluster [DBG] pgmap v2507: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:54 smithi006 bash[33918]: audit 2023-08-27T23:21:54.475137+0000 mon.c (mon.2) 984 : audit [DBG] from='client.? 172.21.15.6:0/4013311852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:54 smithi130 bash[27579]: cluster 2023-08-27T23:21:52.935907+0000 mgr.y (mgr.14901) 2500 : cluster [DBG] pgmap v2507: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:54 smithi130 bash[27579]: audit 2023-08-27T23:21:54.475137+0000 mon.c (mon.2) 984 : audit [DBG] from='client.? 172.21.15.6:0/4013311852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:55.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:55 smithi130 bash[51302]: debug 2023-08-27T23:21:55.379+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:55.383167+0000) 2023-08-27T23:21:56.670 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:56 smithi130 bash[51302]: debug 2023-08-27T23:21:56.379+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:56.383493+0000) 2023-08-27T23:21:56.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:56 smithi006 bash[27292]: cluster 2023-08-27T23:21:54.937101+0000 mgr.y (mgr.14901) 2501 : cluster [DBG] pgmap v2508: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:56 smithi006 bash[33918]: cluster 2023-08-27T23:21:54.937101+0000 mgr.y (mgr.14901) 2501 : cluster [DBG] pgmap v2508: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:56 smithi130 bash[27579]: cluster 2023-08-27T23:21:54.937101+0000 mgr.y (mgr.14901) 2501 : cluster [DBG] pgmap v2508: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:57.672 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:57 smithi130 bash[51302]: debug 2023-08-27T23:21:57.379+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:57.383724+0000) 2023-08-27T23:21:57.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:57 smithi006 bash[27292]: audit 2023-08-27T23:21:56.895566+0000 mon.a (mon.0) 2164 : audit [DBG] from='client.? 172.21.15.6:0/1618703335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:57.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:57 smithi006 bash[33918]: audit 2023-08-27T23:21:56.895566+0000 mon.a (mon.0) 2164 : audit [DBG] from='client.? 172.21.15.6:0/1618703335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:58.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:57 smithi130 bash[27579]: audit 2023-08-27T23:21:56.895566+0000 mon.a (mon.0) 2164 : audit [DBG] from='client.? 172.21.15.6:0/1618703335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:58.681 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:58 smithi130 bash[51302]: debug 2023-08-27T23:21:58.379+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:58.383992+0000) 2023-08-27T23:21:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:58 smithi006 bash[33918]: cluster 2023-08-27T23:21:56.937766+0000 mgr.y (mgr.14901) 2502 : cluster [DBG] pgmap v2509: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:58 smithi006 bash[27292]: cluster 2023-08-27T23:21:56.937766+0000 mgr.y (mgr.14901) 2502 : cluster [DBG] pgmap v2509: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:58 smithi130 bash[27579]: cluster 2023-08-27T23:21:56.937766+0000 mgr.y (mgr.14901) 2502 : cluster [DBG] pgmap v2509: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:21:59.690 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:21:59 smithi130 bash[51302]: debug 2023-08-27T23:21:59.379+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:59.384252+0000) 2023-08-27T23:21:59.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:21:59 smithi006 bash[33918]: audit 2023-08-27T23:21:59.308877+0000 mon.c (mon.2) 985 : audit [DBG] from='client.? 172.21.15.6:0/3558467409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:59.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:21:59 smithi006 bash[27292]: audit 2023-08-27T23:21:59.308877+0000 mon.c (mon.2) 985 : audit [DBG] from='client.? 172.21.15.6:0/3558467409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:00.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:21:59 smithi130 bash[27579]: audit 2023-08-27T23:21:59.308877+0000 mon.c (mon.2) 985 : audit [DBG] from='client.? 172.21.15.6:0/3558467409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:00.693 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:00 smithi130 bash[51302]: debug 2023-08-27T23:22:00.383+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:00.384562+0000) 2023-08-27T23:22:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:00 smithi006 bash[27292]: cluster 2023-08-27T23:21:58.938284+0000 mgr.y (mgr.14901) 2503 : cluster [DBG] pgmap v2510: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:00 smithi006 bash[33918]: cluster 2023-08-27T23:21:58.938284+0000 mgr.y (mgr.14901) 2503 : cluster [DBG] pgmap v2510: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:22:00.442Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:22:00.444Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:00 smithi130 bash[27579]: cluster 2023-08-27T23:21:58.938284+0000 mgr.y (mgr.14901) 2503 : cluster [DBG] pgmap v2510: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:22:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:22:00] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:22:01.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:01 smithi130 bash[51302]: debug 2023-08-27T23:22:01.379+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:01.384833+0000) 2023-08-27T23:22:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:22:02.005Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:22:02.007Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:02.697 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:02 smithi130 bash[51302]: debug 2023-08-27T23:22:02.379+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:02.385110+0000) 2023-08-27T23:22:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:02 smithi006 bash[27292]: cluster 2023-08-27T23:22:00.939334+0000 mgr.y (mgr.14901) 2504 : cluster [DBG] pgmap v2511: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:02 smithi006 bash[27292]: audit 2023-08-27T23:22:01.721888+0000 mon.a (mon.0) 2165 : audit [DBG] from='client.? 172.21.15.6:0/1384736489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:02 smithi006 bash[33918]: cluster 2023-08-27T23:22:00.939334+0000 mgr.y (mgr.14901) 2504 : cluster [DBG] pgmap v2511: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:02 smithi006 bash[33918]: audit 2023-08-27T23:22:01.721888+0000 mon.a (mon.0) 2165 : audit [DBG] from='client.? 172.21.15.6:0/1384736489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:02 smithi130 bash[27579]: cluster 2023-08-27T23:22:00.939334+0000 mgr.y (mgr.14901) 2504 : cluster [DBG] pgmap v2511: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:02 smithi130 bash[27579]: audit 2023-08-27T23:22:01.721888+0000 mon.a (mon.0) 2165 : audit [DBG] from='client.? 172.21.15.6:0/1384736489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:03.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:03 smithi130 bash[51302]: debug 2023-08-27T23:22:03.383+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:03.385434+0000) 2023-08-27T23:22:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:22:03.913Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:04.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:22:03.915Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:04.702 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:04 smithi130 bash[51302]: debug 2023-08-27T23:22:04.383+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:04.385713+0000) 2023-08-27T23:22:04.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:04 smithi006 bash[27292]: cluster 2023-08-27T23:22:02.940040+0000 mgr.y (mgr.14901) 2505 : cluster [DBG] pgmap v2512: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:04.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:04 smithi006 bash[27292]: audit 2023-08-27T23:22:04.135021+0000 mon.a (mon.0) 2166 : audit [DBG] from='client.? 172.21.15.6:0/3124054990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:04 smithi006 bash[33918]: cluster 2023-08-27T23:22:02.940040+0000 mgr.y (mgr.14901) 2505 : cluster [DBG] pgmap v2512: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:04 smithi006 bash[33918]: audit 2023-08-27T23:22:04.135021+0000 mon.a (mon.0) 2166 : audit [DBG] from='client.? 172.21.15.6:0/3124054990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:04 smithi130 bash[27579]: cluster 2023-08-27T23:22:02.940040+0000 mgr.y (mgr.14901) 2505 : cluster [DBG] pgmap v2512: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:04 smithi130 bash[27579]: audit 2023-08-27T23:22:04.135021+0000 mon.a (mon.0) 2166 : audit [DBG] from='client.? 172.21.15.6:0/3124054990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:05.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:05 smithi130 bash[51302]: debug 2023-08-27T23:22:05.379+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:05.385963+0000) 2023-08-27T23:22:06.706 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:06 smithi130 bash[51302]: debug 2023-08-27T23:22:06.379+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:06.386223+0000) 2023-08-27T23:22:06.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:06 smithi006 bash[27292]: cluster 2023-08-27T23:22:04.941212+0000 mgr.y (mgr.14901) 2506 : cluster [DBG] pgmap v2513: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:06 smithi006 bash[27292]: audit 2023-08-27T23:22:06.545649+0000 mon.a (mon.0) 2167 : audit [DBG] from='client.? 172.21.15.6:0/2402607823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:06 smithi006 bash[33918]: cluster 2023-08-27T23:22:04.941212+0000 mgr.y (mgr.14901) 2506 : cluster [DBG] pgmap v2513: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:06 smithi006 bash[33918]: audit 2023-08-27T23:22:06.545649+0000 mon.a (mon.0) 2167 : audit [DBG] from='client.? 172.21.15.6:0/2402607823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:06 smithi130 bash[27579]: cluster 2023-08-27T23:22:04.941212+0000 mgr.y (mgr.14901) 2506 : cluster [DBG] pgmap v2513: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:06 smithi130 bash[27579]: audit 2023-08-27T23:22:06.545649+0000 mon.a (mon.0) 2167 : audit [DBG] from='client.? 172.21.15.6:0/2402607823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:07.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:07 smithi130 bash[51302]: debug 2023-08-27T23:22:07.379+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:07.386495+0000) 2023-08-27T23:22:08.711 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:08 smithi130 bash[51302]: debug 2023-08-27T23:22:08.379+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:08.386832+0000) 2023-08-27T23:22:08.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:08 smithi006 bash[27292]: cluster 2023-08-27T23:22:06.941835+0000 mgr.y (mgr.14901) 2507 : cluster [DBG] pgmap v2514: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:08 smithi006 bash[33918]: cluster 2023-08-27T23:22:06.941835+0000 mgr.y (mgr.14901) 2507 : cluster [DBG] pgmap v2514: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:08 smithi130 bash[27579]: cluster 2023-08-27T23:22:06.941835+0000 mgr.y (mgr.14901) 2507 : cluster [DBG] pgmap v2514: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:09.715 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:09 smithi130 bash[51302]: debug 2023-08-27T23:22:09.383+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:09.387172+0000) 2023-08-27T23:22:09.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:09 smithi006 bash[33918]: audit 2023-08-27T23:22:08.967158+0000 mon.c (mon.2) 986 : audit [DBG] from='client.? 172.21.15.6:0/730675225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:09.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:09 smithi006 bash[27292]: audit 2023-08-27T23:22:08.967158+0000 mon.c (mon.2) 986 : audit [DBG] from='client.? 172.21.15.6:0/730675225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:10.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:09 smithi130 bash[27579]: audit 2023-08-27T23:22:08.967158+0000 mon.c (mon.2) 986 : audit [DBG] from='client.? 172.21.15.6:0/730675225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:22:10.443Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:22:10.445Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:10.718 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:10 smithi130 bash[51302]: debug 2023-08-27T23:22:10.383+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:10.387429+0000) 2023-08-27T23:22:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:10 smithi130 bash[27579]: cluster 2023-08-27T23:22:08.942471+0000 mgr.y (mgr.14901) 2508 : cluster [DBG] pgmap v2515: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:22:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:22:10] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:22:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:10 smithi006 bash[33918]: cluster 2023-08-27T23:22:08.942471+0000 mgr.y (mgr.14901) 2508 : cluster [DBG] pgmap v2515: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:11.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:10 smithi006 bash[27292]: cluster 2023-08-27T23:22:08.942471+0000 mgr.y (mgr.14901) 2508 : cluster [DBG] pgmap v2515: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:11.725 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:11 smithi130 bash[51302]: debug 2023-08-27T23:22:11.383+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:11.387732+0000) 2023-08-27T23:22:12.009 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:11 smithi006 bash[33918]: cluster 2023-08-27T23:22:10.943570+0000 mgr.y (mgr.14901) 2509 : cluster [DBG] pgmap v2516: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:12.009 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:11 smithi006 bash[33918]: audit 2023-08-27T23:22:11.389134+0000 mon.a (mon.0) 2168 : audit [DBG] from='client.? 172.21.15.6:0/1878587229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:12.009 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:11 smithi006 bash[27292]: cluster 2023-08-27T23:22:10.943570+0000 mgr.y (mgr.14901) 2509 : cluster [DBG] pgmap v2516: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:12.010 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:11 smithi006 bash[27292]: audit 2023-08-27T23:22:11.389134+0000 mon.a (mon.0) 2168 : audit [DBG] from='client.? 172.21.15.6:0/1878587229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:12.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:11 smithi130 bash[27579]: cluster 2023-08-27T23:22:10.943570+0000 mgr.y (mgr.14901) 2509 : cluster [DBG] pgmap v2516: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:12.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:11 smithi130 bash[27579]: audit 2023-08-27T23:22:11.389134+0000 mon.a (mon.0) 2168 : audit [DBG] from='client.? 172.21.15.6:0/1878587229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:22:12.006Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:22:12.008Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:12.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:12 smithi130 bash[51302]: debug 2023-08-27T23:22:12.383+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:12.388017+0000) 2023-08-27T23:22:13.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:13 smithi130 bash[51302]: debug 2023-08-27T23:22:13.383+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:13.388255+0000) 2023-08-27T23:22:14.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:14 smithi006 bash[33918]: cluster 2023-08-27T23:22:12.944275+0000 mgr.y (mgr.14901) 2510 : cluster [DBG] pgmap v2517: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:14.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:14 smithi006 bash[33918]: audit 2023-08-27T23:22:13.807791+0000 mon.c (mon.2) 987 : audit [DBG] from='client.? 172.21.15.6:0/1792634186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:14.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:14 smithi006 bash[27292]: cluster 2023-08-27T23:22:12.944275+0000 mgr.y (mgr.14901) 2510 : cluster [DBG] pgmap v2517: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:14.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:14 smithi006 bash[27292]: audit 2023-08-27T23:22:13.807791+0000 mon.c (mon.2) 987 : audit [DBG] from='client.? 172.21.15.6:0/1792634186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:22:13.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:22:13.917Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:14.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:14 smithi130 bash[27579]: cluster 2023-08-27T23:22:12.944275+0000 mgr.y (mgr.14901) 2510 : cluster [DBG] pgmap v2517: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:14.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:14 smithi130 bash[27579]: audit 2023-08-27T23:22:13.807791+0000 mon.c (mon.2) 987 : audit [DBG] from='client.? 172.21.15.6:0/1792634186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:14.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:14 smithi130 bash[51302]: debug 2023-08-27T23:22:14.383+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:14.388534+0000) 2023-08-27T23:22:15.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:15 smithi130 bash[51302]: debug 2023-08-27T23:22:15.383+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:15.388836+0000) 2023-08-27T23:22:16.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:16 smithi130 bash[51302]: debug 2023-08-27T23:22:16.383+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:16.389101+0000) 2023-08-27T23:22:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:16 smithi130 bash[27579]: cluster 2023-08-27T23:22:14.945395+0000 mgr.y (mgr.14901) 2511 : cluster [DBG] pgmap v2518: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:16 smithi130 bash[27579]: audit 2023-08-27T23:22:16.222396+0000 mon.c (mon.2) 988 : audit [DBG] from='client.? 172.21.15.6:0/1444511972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:16.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:16 smithi006 bash[33918]: cluster 2023-08-27T23:22:14.945395+0000 mgr.y (mgr.14901) 2511 : cluster [DBG] pgmap v2518: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:16 smithi006 bash[33918]: audit 2023-08-27T23:22:16.222396+0000 mon.c (mon.2) 988 : audit [DBG] from='client.? 172.21.15.6:0/1444511972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:16 smithi006 bash[27292]: cluster 2023-08-27T23:22:14.945395+0000 mgr.y (mgr.14901) 2511 : cluster [DBG] pgmap v2518: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:16 smithi006 bash[27292]: audit 2023-08-27T23:22:16.222396+0000 mon.c (mon.2) 988 : audit [DBG] from='client.? 172.21.15.6:0/1444511972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:17.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:17 smithi130 bash[51302]: debug 2023-08-27T23:22:17.387+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:17.389428+0000) 2023-08-27T23:22:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:18 smithi130 bash[27579]: cluster 2023-08-27T23:22:16.946166+0000 mgr.y (mgr.14901) 2512 : cluster [DBG] pgmap v2519: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:18 smithi130 bash[51302]: debug 2023-08-27T23:22:18.383+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:18.389823+0000) 2023-08-27T23:22:18.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:18 smithi006 bash[27292]: cluster 2023-08-27T23:22:16.946166+0000 mgr.y (mgr.14901) 2512 : cluster [DBG] pgmap v2519: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:18 smithi006 bash[33918]: cluster 2023-08-27T23:22:16.946166+0000 mgr.y (mgr.14901) 2512 : cluster [DBG] pgmap v2519: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:19.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:19 smithi130 bash[27579]: audit 2023-08-27T23:22:18.642408+0000 mon.a (mon.0) 2169 : audit [DBG] from='client.? 172.21.15.6:0/3374778487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:19 smithi130 bash[51302]: debug 2023-08-27T23:22:19.386+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:19.390109+0000) 2023-08-27T23:22:19.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:19 smithi006 bash[27292]: audit 2023-08-27T23:22:18.642408+0000 mon.a (mon.0) 2169 : audit [DBG] from='client.? 172.21.15.6:0/3374778487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:19 smithi006 bash[33918]: audit 2023-08-27T23:22:18.642408+0000 mon.a (mon.0) 2169 : audit [DBG] from='client.? 172.21.15.6:0/3374778487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:20 smithi006 bash[27292]: cluster 2023-08-27T23:22:18.947001+0000 mgr.y (mgr.14901) 2513 : cluster [DBG] pgmap v2520: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:20.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:20 smithi006 bash[33918]: cluster 2023-08-27T23:22:18.947001+0000 mgr.y (mgr.14901) 2513 : cluster [DBG] pgmap v2520: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:20.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:22:20.444Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:20.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:22:20.446Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:20.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:20 smithi130 bash[27579]: cluster 2023-08-27T23:22:18.947001+0000 mgr.y (mgr.14901) 2513 : cluster [DBG] pgmap v2520: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:20 smithi130 bash[51302]: debug 2023-08-27T23:22:20.382+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:20.390406+0000) 2023-08-27T23:22:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:22:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:22:20] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:22:21.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:21 smithi130 bash[27579]: audit 2023-08-27T23:22:21.054011+0000 mon.c (mon.2) 989 : audit [DBG] from='client.? 172.21.15.6:0/2965518082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:21.928 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:21 smithi130 bash[51302]: debug 2023-08-27T23:22:21.382+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:21.390689+0000) 2023-08-27T23:22:21.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:21 smithi006 bash[33918]: audit 2023-08-27T23:22:21.054011+0000 mon.c (mon.2) 989 : audit [DBG] from='client.? 172.21.15.6:0/2965518082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:21.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:21 smithi006 bash[27292]: audit 2023-08-27T23:22:21.054011+0000 mon.c (mon.2) 989 : audit [DBG] from='client.? 172.21.15.6:0/2965518082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:22:22.007Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:22:22.009Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:22 smithi130 bash[27579]: cluster 2023-08-27T23:22:20.948111+0000 mgr.y (mgr.14901) 2514 : cluster [DBG] pgmap v2521: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:22 smithi130 bash[51302]: debug 2023-08-27T23:22:22.386+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:22.390978+0000) 2023-08-27T23:22:22.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:22 smithi006 bash[33918]: cluster 2023-08-27T23:22:20.948111+0000 mgr.y (mgr.14901) 2514 : cluster [DBG] pgmap v2521: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:22 smithi006 bash[27292]: cluster 2023-08-27T23:22:20.948111+0000 mgr.y (mgr.14901) 2514 : cluster [DBG] pgmap v2521: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:23.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:23 smithi130 bash[27579]: audit 2023-08-27T23:22:23.465314+0000 mon.a (mon.0) 2170 : audit [DBG] from='client.? 172.21.15.6:0/2854935402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:23.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:23 smithi130 bash[51302]: debug 2023-08-27T23:22:23.386+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:23.391218+0000) 2023-08-27T23:22:23.918 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:23 smithi006 bash[27292]: audit 2023-08-27T23:22:23.465314+0000 mon.a (mon.0) 2170 : audit [DBG] from='client.? 172.21.15.6:0/2854935402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:23.918 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:23 smithi006 bash[33918]: audit 2023-08-27T23:22:23.465314+0000 mon.a (mon.0) 2170 : audit [DBG] from='client.? 172.21.15.6:0/2854935402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:22:23.916Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:22:23.918Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:24 smithi130 bash[27579]: cluster 2023-08-27T23:22:22.948884+0000 mgr.y (mgr.14901) 2515 : cluster [DBG] pgmap v2522: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:24 smithi130 bash[51302]: debug 2023-08-27T23:22:24.386+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:24.391522+0000) 2023-08-27T23:22:24.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:24 smithi006 bash[33918]: cluster 2023-08-27T23:22:22.948884+0000 mgr.y (mgr.14901) 2515 : cluster [DBG] pgmap v2522: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:24 smithi006 bash[27292]: cluster 2023-08-27T23:22:22.948884+0000 mgr.y (mgr.14901) 2515 : cluster [DBG] pgmap v2522: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:25.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:25 smithi130 bash[51302]: debug 2023-08-27T23:22:25.386+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:25.391807+0000) 2023-08-27T23:22:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:26 smithi130 bash[27579]: cluster 2023-08-27T23:22:24.950032+0000 mgr.y (mgr.14901) 2516 : cluster [DBG] pgmap v2523: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:26 smithi130 bash[27579]: audit 2023-08-27T23:22:25.882536+0000 mon.a (mon.0) 2171 : audit [DBG] from='client.? 172.21.15.6:0/3763123559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:26.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:26 smithi130 bash[51302]: debug 2023-08-27T23:22:26.390+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:26.392073+0000) 2023-08-27T23:22:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:26 smithi006 bash[33918]: cluster 2023-08-27T23:22:24.950032+0000 mgr.y (mgr.14901) 2516 : cluster [DBG] pgmap v2523: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:26.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:26 smithi006 bash[33918]: audit 2023-08-27T23:22:25.882536+0000 mon.a (mon.0) 2171 : audit [DBG] from='client.? 172.21.15.6:0/3763123559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:26.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:26 smithi006 bash[27292]: cluster 2023-08-27T23:22:24.950032+0000 mgr.y (mgr.14901) 2516 : cluster [DBG] pgmap v2523: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:26.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:26 smithi006 bash[27292]: audit 2023-08-27T23:22:25.882536+0000 mon.a (mon.0) 2171 : audit [DBG] from='client.? 172.21.15.6:0/3763123559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:27.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:27 smithi130 bash[51302]: debug 2023-08-27T23:22:27.390+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:27.392335+0000) 2023-08-27T23:22:28.647 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:28 smithi130 bash[51302]: debug 2023-08-27T23:22:28.390+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:28.392607+0000) 2023-08-27T23:22:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:28 smithi006 bash[27292]: cluster 2023-08-27T23:22:26.950725+0000 mgr.y (mgr.14901) 2517 : cluster [DBG] pgmap v2524: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:28 smithi006 bash[27292]: audit 2023-08-27T23:22:28.294366+0000 mon.c (mon.2) 990 : audit [DBG] from='client.? 172.21.15.6:0/2846311915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:28 smithi006 bash[33918]: cluster 2023-08-27T23:22:26.950725+0000 mgr.y (mgr.14901) 2517 : cluster [DBG] pgmap v2524: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:28 smithi006 bash[33918]: audit 2023-08-27T23:22:28.294366+0000 mon.c (mon.2) 990 : audit [DBG] from='client.? 172.21.15.6:0/2846311915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:28 smithi130 bash[27579]: cluster 2023-08-27T23:22:26.950725+0000 mgr.y (mgr.14901) 2517 : cluster [DBG] pgmap v2524: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:28 smithi130 bash[27579]: audit 2023-08-27T23:22:28.294366+0000 mon.c (mon.2) 990 : audit [DBG] from='client.? 172.21.15.6:0/2846311915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:29.654 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:29 smithi130 bash[51302]: debug 2023-08-27T23:22:29.390+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:29.392879+0000) 2023-08-27T23:22:29.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:29 smithi006 bash[27292]: audit 2023-08-27T23:22:29.058730+0000 mon.b (mon.1) 361 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:22:29.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:29 smithi006 bash[33918]: audit 2023-08-27T23:22:29.058730+0000 mon.b (mon.1) 361 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:22:30.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:29 smithi130 bash[27579]: audit 2023-08-27T23:22:29.058730+0000 mon.b (mon.1) 361 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:22:30.656 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:30 smithi130 bash[51302]: debug 2023-08-27T23:22:30.390+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:30.393165+0000) 2023-08-27T23:22:30.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:22:30.444Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:22:30.446Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:30 smithi006 bash[27292]: cluster 2023-08-27T23:22:28.951408+0000 mgr.y (mgr.14901) 2518 : cluster [DBG] pgmap v2525: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:30 smithi006 bash[33918]: cluster 2023-08-27T23:22:28.951408+0000 mgr.y (mgr.14901) 2518 : cluster [DBG] pgmap v2525: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:30 smithi130 bash[27579]: cluster 2023-08-27T23:22:28.951408+0000 mgr.y (mgr.14901) 2518 : cluster [DBG] pgmap v2525: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:22:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:22:30] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:22:31.664 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:31 smithi130 bash[51302]: debug 2023-08-27T23:22:31.390+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:31.393440+0000) 2023-08-27T23:22:31.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:31 smithi006 bash[27292]: audit 2023-08-27T23:22:30.719862+0000 mon.a (mon.0) 2172 : audit [DBG] from='client.? 172.21.15.6:0/3565995681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:31.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:31 smithi006 bash[33918]: audit 2023-08-27T23:22:30.719862+0000 mon.a (mon.0) 2172 : audit [DBG] from='client.? 172.21.15.6:0/3565995681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:32.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:31 smithi130 bash[27579]: audit 2023-08-27T23:22:30.719862+0000 mon.a (mon.0) 2172 : audit [DBG] from='client.? 172.21.15.6:0/3565995681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:22:32.009Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:22:32.011Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:32.667 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:32 smithi130 bash[51302]: debug 2023-08-27T23:22:32.390+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:32.393719+0000) 2023-08-27T23:22:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:32 smithi006 bash[27292]: cluster 2023-08-27T23:22:30.952571+0000 mgr.y (mgr.14901) 2519 : cluster [DBG] pgmap v2526: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:32 smithi006 bash[33918]: cluster 2023-08-27T23:22:30.952571+0000 mgr.y (mgr.14901) 2519 : cluster [DBG] pgmap v2526: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:33.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:32 smithi130 bash[27579]: cluster 2023-08-27T23:22:30.952571+0000 mgr.y (mgr.14901) 2519 : cluster [DBG] pgmap v2526: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:33.675 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:33 smithi130 bash[51302]: debug 2023-08-27T23:22:33.390+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:33.393946+0000) 2023-08-27T23:22:33.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:33 smithi006 bash[27292]: audit 2023-08-27T23:22:33.133995+0000 mon.c (mon.2) 991 : audit [DBG] from='client.? 172.21.15.6:0/3575212484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:33.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:33 smithi006 bash[33918]: audit 2023-08-27T23:22:33.133995+0000 mon.c (mon.2) 991 : audit [DBG] from='client.? 172.21.15.6:0/3575212484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:33.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:22:33.917Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:33.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:22:33.919Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:34.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:33 smithi130 bash[27579]: audit 2023-08-27T23:22:33.133995+0000 mon.c (mon.2) 991 : audit [DBG] from='client.? 172.21.15.6:0/3575212484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:34.678 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:34 smithi130 bash[51302]: debug 2023-08-27T23:22:34.390+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:34.394126+0000) 2023-08-27T23:22:34.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:34 smithi006 bash[27292]: cluster 2023-08-27T23:22:32.953217+0000 mgr.y (mgr.14901) 2520 : cluster [DBG] pgmap v2527: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:34.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:34 smithi006 bash[33918]: cluster 2023-08-27T23:22:32.953217+0000 mgr.y (mgr.14901) 2520 : cluster [DBG] pgmap v2527: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:34 smithi130 bash[27579]: cluster 2023-08-27T23:22:32.953217+0000 mgr.y (mgr.14901) 2520 : cluster [DBG] pgmap v2527: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:35.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:35 smithi130 bash[51302]: debug 2023-08-27T23:22:35.390+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:35.394335+0000) 2023-08-27T23:22:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:36 smithi130 bash[27579]: cluster 2023-08-27T23:22:34.954283+0000 mgr.y (mgr.14901) 2521 : cluster [DBG] pgmap v2528: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:36 smithi130 bash[27579]: audit 2023-08-27T23:22:35.017176+0000 mon.a (mon.0) 2173 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:22:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:36 smithi130 bash[27579]: audit 2023-08-27T23:22:35.343821+0000 mon.a (mon.0) 2174 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:22:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:36 smithi130 bash[27579]: audit 2023-08-27T23:22:35.534522+0000 mon.a (mon.0) 2175 : audit [DBG] from='client.? 172.21.15.6:0/239495280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:36 smithi130 bash[27579]: audit 2023-08-27T23:22:35.614078+0000 mon.b (mon.1) 362 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:22:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:36 smithi130 bash[27579]: audit 2023-08-27T23:22:35.616163+0000 mon.b (mon.1) 363 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:22:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:36 smithi130 bash[27579]: audit 2023-08-27T23:22:35.624479+0000 mon.a (mon.0) 2176 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:22:36.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:36 smithi006 bash[33918]: cluster 2023-08-27T23:22:34.954283+0000 mgr.y (mgr.14901) 2521 : cluster [DBG] pgmap v2528: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:36.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:36 smithi006 bash[33918]: audit 2023-08-27T23:22:35.017176+0000 mon.a (mon.0) 2173 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:22:36.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:36 smithi006 bash[33918]: audit 2023-08-27T23:22:35.343821+0000 mon.a (mon.0) 2174 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:22:36.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:36 smithi006 bash[33918]: audit 2023-08-27T23:22:35.534522+0000 mon.a (mon.0) 2175 : audit [DBG] from='client.? 172.21.15.6:0/239495280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:36.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:36 smithi006 bash[33918]: audit 2023-08-27T23:22:35.614078+0000 mon.b (mon.1) 362 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:22:36.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:36 smithi006 bash[33918]: audit 2023-08-27T23:22:35.616163+0000 mon.b (mon.1) 363 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:22:36.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:36 smithi006 bash[33918]: audit 2023-08-27T23:22:35.624479+0000 mon.a (mon.0) 2176 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:22:36.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:36 smithi006 bash[27292]: cluster 2023-08-27T23:22:34.954283+0000 mgr.y (mgr.14901) 2521 : cluster [DBG] pgmap v2528: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:36.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:36 smithi006 bash[27292]: audit 2023-08-27T23:22:35.017176+0000 mon.a (mon.0) 2173 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:22:36.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:36 smithi006 bash[27292]: audit 2023-08-27T23:22:35.343821+0000 mon.a (mon.0) 2174 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:22:36.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:36 smithi006 bash[27292]: audit 2023-08-27T23:22:35.534522+0000 mon.a (mon.0) 2175 : audit [DBG] from='client.? 172.21.15.6:0/239495280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:36.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:36 smithi006 bash[27292]: audit 2023-08-27T23:22:35.614078+0000 mon.b (mon.1) 362 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:22:36.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:36 smithi006 bash[27292]: audit 2023-08-27T23:22:35.616163+0000 mon.b (mon.1) 363 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:22:36.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:36 smithi006 bash[27292]: audit 2023-08-27T23:22:35.624479+0000 mon.a (mon.0) 2176 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:22:36.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:36 smithi130 bash[51302]: debug 2023-08-27T23:22:36.390+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:36.394590+0000) 2023-08-27T23:22:37.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:37 smithi130 bash[51302]: debug 2023-08-27T23:22:37.394+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:37.394931+0000) 2023-08-27T23:22:38.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:38 smithi130 bash[27579]: cluster 2023-08-27T23:22:36.954977+0000 mgr.y (mgr.14901) 2522 : cluster [DBG] pgmap v2529: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:38.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:38 smithi130 bash[27579]: audit 2023-08-27T23:22:37.952126+0000 mon.a (mon.0) 2177 : audit [DBG] from='client.? 172.21.15.6:0/3583674492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:38.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:38 smithi006 bash[27292]: cluster 2023-08-27T23:22:36.954977+0000 mgr.y (mgr.14901) 2522 : cluster [DBG] pgmap v2529: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:38.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:38 smithi006 bash[27292]: audit 2023-08-27T23:22:37.952126+0000 mon.a (mon.0) 2177 : audit [DBG] from='client.? 172.21.15.6:0/3583674492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:38.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:38 smithi006 bash[33918]: cluster 2023-08-27T23:22:36.954977+0000 mgr.y (mgr.14901) 2522 : cluster [DBG] pgmap v2529: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:38.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:38 smithi006 bash[33918]: audit 2023-08-27T23:22:37.952126+0000 mon.a (mon.0) 2177 : audit [DBG] from='client.? 172.21.15.6:0/3583674492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:38.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:38 smithi130 bash[51302]: debug 2023-08-27T23:22:38.394+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:38.395186+0000) 2023-08-27T23:22:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:39 smithi130 bash[51302]: debug 2023-08-27T23:22:39.394+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:39.395498+0000) 2023-08-27T23:22:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:40 smithi006 bash[33918]: cluster 2023-08-27T23:22:38.955612+0000 mgr.y (mgr.14901) 2523 : cluster [DBG] pgmap v2530: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:40 smithi006 bash[33918]: audit 2023-08-27T23:22:40.369323+0000 mon.a (mon.0) 2178 : audit [DBG] from='client.? 172.21.15.6:0/798261622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:22:40.446Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:22:40.448Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:40 smithi006 bash[27292]: cluster 2023-08-27T23:22:38.955612+0000 mgr.y (mgr.14901) 2523 : cluster [DBG] pgmap v2530: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:40 smithi006 bash[27292]: audit 2023-08-27T23:22:40.369323+0000 mon.a (mon.0) 2178 : audit [DBG] from='client.? 172.21.15.6:0/798261622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:40 smithi130 bash[27579]: cluster 2023-08-27T23:22:38.955612+0000 mgr.y (mgr.14901) 2523 : cluster [DBG] pgmap v2530: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:40 smithi130 bash[27579]: audit 2023-08-27T23:22:40.369323+0000 mon.a (mon.0) 2178 : audit [DBG] from='client.? 172.21.15.6:0/798261622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:40 smithi130 bash[51302]: debug 2023-08-27T23:22:40.394+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:40.395780+0000) 2023-08-27T23:22:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:22:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:22:40] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:22:41.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:41 smithi130 bash[51302]: debug 2023-08-27T23:22:41.394+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:41.396113+0000) 2023-08-27T23:22:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:22:42.010Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:22:42.012Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:42 smithi130 bash[27579]: cluster 2023-08-27T23:22:40.956776+0000 mgr.y (mgr.14901) 2524 : cluster [DBG] pgmap v2531: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:42 smithi130 bash[51302]: debug 2023-08-27T23:22:42.394+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:42.396420+0000) 2023-08-27T23:22:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:42 smithi006 bash[27292]: cluster 2023-08-27T23:22:40.956776+0000 mgr.y (mgr.14901) 2524 : cluster [DBG] pgmap v2531: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:42 smithi006 bash[33918]: cluster 2023-08-27T23:22:40.956776+0000 mgr.y (mgr.14901) 2524 : cluster [DBG] pgmap v2531: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:43.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:43 smithi130 bash[27579]: audit 2023-08-27T23:22:42.788533+0000 mon.a (mon.0) 2179 : audit [DBG] from='client.? 172.21.15.6:0/2051531796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:43.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:43 smithi130 bash[51302]: debug 2023-08-27T23:22:43.394+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:43.396698+0000) 2023-08-27T23:22:43.920 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:43 smithi006 bash[33918]: audit 2023-08-27T23:22:42.788533+0000 mon.a (mon.0) 2179 : audit [DBG] from='client.? 172.21.15.6:0/2051531796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:43.921 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:43 smithi006 bash[27292]: audit 2023-08-27T23:22:42.788533+0000 mon.a (mon.0) 2179 : audit [DBG] from='client.? 172.21.15.6:0/2051531796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:44.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:22:43.918Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:22:43.921Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:44 smithi130 bash[51302]: debug 2023-08-27T23:22:44.394+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:44.396969+0000) 2023-08-27T23:22:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:44 smithi130 bash[27579]: cluster 2023-08-27T23:22:42.957430+0000 mgr.y (mgr.14901) 2525 : cluster [DBG] pgmap v2532: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:44.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:44 smithi006 bash[27292]: cluster 2023-08-27T23:22:42.957430+0000 mgr.y (mgr.14901) 2525 : cluster [DBG] pgmap v2532: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:44 smithi006 bash[33918]: cluster 2023-08-27T23:22:42.957430+0000 mgr.y (mgr.14901) 2525 : cluster [DBG] pgmap v2532: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:45.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:45 smithi130 bash[51302]: debug 2023-08-27T23:22:45.394+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:45.397287+0000) 2023-08-27T23:22:45.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:45 smithi130 bash[27579]: audit 2023-08-27T23:22:45.201648+0000 mon.c (mon.2) 992 : audit [DBG] from='client.? 172.21.15.6:0/2294181111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:45.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:45 smithi006 bash[27292]: audit 2023-08-27T23:22:45.201648+0000 mon.c (mon.2) 992 : audit [DBG] from='client.? 172.21.15.6:0/2294181111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:45.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:45 smithi006 bash[33918]: audit 2023-08-27T23:22:45.201648+0000 mon.c (mon.2) 992 : audit [DBG] from='client.? 172.21.15.6:0/2294181111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:46 smithi130 bash[27579]: cluster 2023-08-27T23:22:44.958490+0000 mgr.y (mgr.14901) 2526 : cluster [DBG] pgmap v2533: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:46.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:46 smithi130 bash[51302]: debug 2023-08-27T23:22:46.394+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:46.397506+0000) 2023-08-27T23:22:46.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:46 smithi006 bash[27292]: cluster 2023-08-27T23:22:44.958490+0000 mgr.y (mgr.14901) 2526 : cluster [DBG] pgmap v2533: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:46 smithi006 bash[33918]: cluster 2023-08-27T23:22:44.958490+0000 mgr.y (mgr.14901) 2526 : cluster [DBG] pgmap v2533: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:47.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:47 smithi130 bash[27579]: audit 2023-08-27T23:22:47.611657+0000 mon.a (mon.0) 2180 : audit [DBG] from='client.? 172.21.15.6:0/2316095882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:47.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:47 smithi130 bash[51302]: debug 2023-08-27T23:22:47.394+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:47.397769+0000) 2023-08-27T23:22:47.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:47 smithi006 bash[27292]: audit 2023-08-27T23:22:47.611657+0000 mon.a (mon.0) 2180 : audit [DBG] from='client.? 172.21.15.6:0/2316095882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:47.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:47 smithi006 bash[33918]: audit 2023-08-27T23:22:47.611657+0000 mon.a (mon.0) 2180 : audit [DBG] from='client.? 172.21.15.6:0/2316095882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:48 smithi130 bash[27579]: cluster 2023-08-27T23:22:46.959157+0000 mgr.y (mgr.14901) 2527 : cluster [DBG] pgmap v2534: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:48 smithi130 bash[51302]: debug 2023-08-27T23:22:48.394+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:48.398079+0000) 2023-08-27T23:22:48.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:48 smithi006 bash[27292]: cluster 2023-08-27T23:22:46.959157+0000 mgr.y (mgr.14901) 2527 : cluster [DBG] pgmap v2534: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:48 smithi006 bash[33918]: cluster 2023-08-27T23:22:46.959157+0000 mgr.y (mgr.14901) 2527 : cluster [DBG] pgmap v2534: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:49.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:49 smithi130 bash[51302]: debug 2023-08-27T23:22:49.394+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:49.398421+0000) 2023-08-27T23:22:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:50 smithi006 bash[33918]: cluster 2023-08-27T23:22:48.959817+0000 mgr.y (mgr.14901) 2528 : cluster [DBG] pgmap v2535: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:50 smithi006 bash[33918]: audit 2023-08-27T23:22:50.022247+0000 mon.a (mon.0) 2181 : audit [DBG] from='client.? 172.21.15.6:0/1465058599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:22:50.447Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:22:50.449Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:50 smithi006 bash[27292]: cluster 2023-08-27T23:22:48.959817+0000 mgr.y (mgr.14901) 2528 : cluster [DBG] pgmap v2535: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:50 smithi006 bash[27292]: audit 2023-08-27T23:22:50.022247+0000 mon.a (mon.0) 2181 : audit [DBG] from='client.? 172.21.15.6:0/1465058599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:50 smithi130 bash[51302]: debug 2023-08-27T23:22:50.394+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:50.398694+0000) 2023-08-27T23:22:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:50 smithi130 bash[27579]: cluster 2023-08-27T23:22:48.959817+0000 mgr.y (mgr.14901) 2528 : cluster [DBG] pgmap v2535: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:50 smithi130 bash[27579]: audit 2023-08-27T23:22:50.022247+0000 mon.a (mon.0) 2181 : audit [DBG] from='client.? 172.21.15.6:0/1465058599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:22:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:22:50] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:22:51.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:51 smithi130 bash[51302]: debug 2023-08-27T23:22:51.398+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:51.399024+0000) 2023-08-27T23:22:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:22:52.011Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:22:52.013Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:52 smithi130 bash[27579]: cluster 2023-08-27T23:22:50.961025+0000 mgr.y (mgr.14901) 2529 : cluster [DBG] pgmap v2536: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:52 smithi130 bash[27579]: audit 2023-08-27T23:22:52.428416+0000 mon.c (mon.2) 993 : audit [DBG] from='client.? 172.21.15.6:0/1846937061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:52.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:52 smithi130 bash[51302]: debug 2023-08-27T23:22:52.398+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:52.399287+0000) 2023-08-27T23:22:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:52 smithi006 bash[27292]: cluster 2023-08-27T23:22:50.961025+0000 mgr.y (mgr.14901) 2529 : cluster [DBG] pgmap v2536: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:52 smithi006 bash[27292]: audit 2023-08-27T23:22:52.428416+0000 mon.c (mon.2) 993 : audit [DBG] from='client.? 172.21.15.6:0/1846937061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:52 smithi006 bash[33918]: cluster 2023-08-27T23:22:50.961025+0000 mgr.y (mgr.14901) 2529 : cluster [DBG] pgmap v2536: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:52 smithi006 bash[33918]: audit 2023-08-27T23:22:52.428416+0000 mon.c (mon.2) 993 : audit [DBG] from='client.? 172.21.15.6:0/1846937061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:53.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:53 smithi130 bash[51302]: debug 2023-08-27T23:22:53.398+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:53.399591+0000) 2023-08-27T23:22:54.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:22:53.918Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:22:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:22:53.921Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:22:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:54 smithi130 bash[27579]: cluster 2023-08-27T23:22:52.961730+0000 mgr.y (mgr.14901) 2530 : cluster [DBG] pgmap v2537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:54.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:54 smithi130 bash[51302]: debug 2023-08-27T23:22:54.398+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:54.399911+0000) 2023-08-27T23:22:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:54 smithi006 bash[33918]: cluster 2023-08-27T23:22:52.961730+0000 mgr.y (mgr.14901) 2530 : cluster [DBG] pgmap v2537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:54 smithi006 bash[27292]: cluster 2023-08-27T23:22:52.961730+0000 mgr.y (mgr.14901) 2530 : cluster [DBG] pgmap v2537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:55.658 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:55 smithi130 bash[51302]: debug 2023-08-27T23:22:55.398+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:55.400193+0000) 2023-08-27T23:22:55.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:55 smithi006 bash[27292]: audit 2023-08-27T23:22:54.850481+0000 mon.a (mon.0) 2182 : audit [DBG] from='client.? 172.21.15.6:0/76671398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:55.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:55 smithi006 bash[33918]: audit 2023-08-27T23:22:54.850481+0000 mon.a (mon.0) 2182 : audit [DBG] from='client.? 172.21.15.6:0/76671398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:56.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:55 smithi130 bash[27579]: audit 2023-08-27T23:22:54.850481+0000 mon.a (mon.0) 2182 : audit [DBG] from='client.? 172.21.15.6:0/76671398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:56.661 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:56 smithi130 bash[51302]: debug 2023-08-27T23:22:56.398+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:56.400471+0000) 2023-08-27T23:22:56.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:56 smithi006 bash[27292]: cluster 2023-08-27T23:22:54.962825+0000 mgr.y (mgr.14901) 2531 : cluster [DBG] pgmap v2538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:56 smithi006 bash[33918]: cluster 2023-08-27T23:22:54.962825+0000 mgr.y (mgr.14901) 2531 : cluster [DBG] pgmap v2538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:56 smithi130 bash[27579]: cluster 2023-08-27T23:22:54.962825+0000 mgr.y (mgr.14901) 2531 : cluster [DBG] pgmap v2538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:57.669 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:57 smithi130 bash[51302]: debug 2023-08-27T23:22:57.398+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:57.400769+0000) 2023-08-27T23:22:57.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:57 smithi006 bash[27292]: audit 2023-08-27T23:22:57.263332+0000 mon.a (mon.0) 2183 : audit [DBG] from='client.? 172.21.15.6:0/2643075130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:57.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:57 smithi006 bash[33918]: audit 2023-08-27T23:22:57.263332+0000 mon.a (mon.0) 2183 : audit [DBG] from='client.? 172.21.15.6:0/2643075130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:58.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:57 smithi130 bash[27579]: audit 2023-08-27T23:22:57.263332+0000 mon.a (mon.0) 2183 : audit [DBG] from='client.? 172.21.15.6:0/2643075130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:58.671 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:58 smithi130 bash[51302]: debug 2023-08-27T23:22:58.398+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:58.401099+0000) 2023-08-27T23:22:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:22:58 smithi006 bash[27292]: cluster 2023-08-27T23:22:56.963352+0000 mgr.y (mgr.14901) 2532 : cluster [DBG] pgmap v2539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:22:58 smithi006 bash[33918]: cluster 2023-08-27T23:22:56.963352+0000 mgr.y (mgr.14901) 2532 : cluster [DBG] pgmap v2539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:22:58 smithi130 bash[27579]: cluster 2023-08-27T23:22:56.963352+0000 mgr.y (mgr.14901) 2532 : cluster [DBG] pgmap v2539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:22:59.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:22:59 smithi130 bash[51302]: debug 2023-08-27T23:22:59.398+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:59.401364+0000) 2023-08-27T23:23:00.676 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:00 smithi130 bash[51302]: debug 2023-08-27T23:23:00.398+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:00.401633+0000) 2023-08-27T23:23:00.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:00 smithi006 bash[27292]: cluster 2023-08-27T23:22:58.963986+0000 mgr.y (mgr.14901) 2533 : cluster [DBG] pgmap v2540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:00 smithi006 bash[27292]: audit 2023-08-27T23:22:59.679327+0000 mon.c (mon.2) 994 : audit [DBG] from='client.? 172.21.15.6:0/3910363478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:00 smithi006 bash[33918]: cluster 2023-08-27T23:22:58.963986+0000 mgr.y (mgr.14901) 2533 : cluster [DBG] pgmap v2540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:00 smithi006 bash[33918]: audit 2023-08-27T23:22:59.679327+0000 mon.c (mon.2) 994 : audit [DBG] from='client.? 172.21.15.6:0/3910363478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:23:00.448Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:23:00.450Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:00 smithi130 bash[27579]: cluster 2023-08-27T23:22:58.963986+0000 mgr.y (mgr.14901) 2533 : cluster [DBG] pgmap v2540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:00 smithi130 bash[27579]: audit 2023-08-27T23:22:59.679327+0000 mon.c (mon.2) 994 : audit [DBG] from='client.? 172.21.15.6:0/3910363478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:23:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:23:00] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:23:01.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:01 smithi130 bash[51302]: debug 2023-08-27T23:23:01.398+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:01.401917+0000) 2023-08-27T23:23:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:23:02.012Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:23:02.014Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:02.681 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:02 smithi130 bash[51302]: debug 2023-08-27T23:23:02.398+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:02.402241+0000) 2023-08-27T23:23:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:02 smithi006 bash[33918]: cluster 2023-08-27T23:23:00.965116+0000 mgr.y (mgr.14901) 2534 : cluster [DBG] pgmap v2541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:02 smithi006 bash[33918]: audit 2023-08-27T23:23:02.089900+0000 mon.a (mon.0) 2184 : audit [DBG] from='client.? 172.21.15.6:0/391070658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:02 smithi006 bash[27292]: cluster 2023-08-27T23:23:00.965116+0000 mgr.y (mgr.14901) 2534 : cluster [DBG] pgmap v2541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:02 smithi006 bash[27292]: audit 2023-08-27T23:23:02.089900+0000 mon.a (mon.0) 2184 : audit [DBG] from='client.? 172.21.15.6:0/391070658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:02 smithi130 bash[27579]: cluster 2023-08-27T23:23:00.965116+0000 mgr.y (mgr.14901) 2534 : cluster [DBG] pgmap v2541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:02 smithi130 bash[27579]: audit 2023-08-27T23:23:02.089900+0000 mon.a (mon.0) 2184 : audit [DBG] from='client.? 172.21.15.6:0/391070658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:03.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:03 smithi130 bash[51302]: debug 2023-08-27T23:23:03.402+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:03.402513+0000) 2023-08-27T23:23:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:23:03.919Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:23:03.921Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:04.686 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:04 smithi130 bash[51302]: debug 2023-08-27T23:23:04.402+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:04.402809+0000) 2023-08-27T23:23:04.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:04 smithi006 bash[27292]: cluster 2023-08-27T23:23:02.965718+0000 mgr.y (mgr.14901) 2535 : cluster [DBG] pgmap v2542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:04.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:04 smithi006 bash[27292]: audit 2023-08-27T23:23:04.502999+0000 mon.a (mon.0) 2185 : audit [DBG] from='client.? 172.21.15.6:0/2401610009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:04 smithi006 bash[33918]: cluster 2023-08-27T23:23:02.965718+0000 mgr.y (mgr.14901) 2535 : cluster [DBG] pgmap v2542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:04 smithi006 bash[33918]: audit 2023-08-27T23:23:04.502999+0000 mon.a (mon.0) 2185 : audit [DBG] from='client.? 172.21.15.6:0/2401610009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:04 smithi130 bash[27579]: cluster 2023-08-27T23:23:02.965718+0000 mgr.y (mgr.14901) 2535 : cluster [DBG] pgmap v2542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:04 smithi130 bash[27579]: audit 2023-08-27T23:23:04.502999+0000 mon.a (mon.0) 2185 : audit [DBG] from='client.? 172.21.15.6:0/2401610009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:05.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:05 smithi130 bash[51302]: debug 2023-08-27T23:23:05.402+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:05.403138+0000) 2023-08-27T23:23:06.691 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:06 smithi130 bash[51302]: debug 2023-08-27T23:23:06.402+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:06.403407+0000) 2023-08-27T23:23:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:06 smithi006 bash[27292]: cluster 2023-08-27T23:23:04.966931+0000 mgr.y (mgr.14901) 2536 : cluster [DBG] pgmap v2543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:06 smithi006 bash[33918]: cluster 2023-08-27T23:23:04.966931+0000 mgr.y (mgr.14901) 2536 : cluster [DBG] pgmap v2543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:06 smithi130 bash[27579]: cluster 2023-08-27T23:23:04.966931+0000 mgr.y (mgr.14901) 2536 : cluster [DBG] pgmap v2543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:07.700 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:07 smithi130 bash[51302]: debug 2023-08-27T23:23:07.402+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:07.403710+0000) 2023-08-27T23:23:07.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:07 smithi006 bash[33918]: audit 2023-08-27T23:23:06.915397+0000 mon.a (mon.0) 2186 : audit [DBG] from='client.? 172.21.15.6:0/1492572828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:07.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:07 smithi006 bash[27292]: audit 2023-08-27T23:23:06.915397+0000 mon.a (mon.0) 2186 : audit [DBG] from='client.? 172.21.15.6:0/1492572828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:08.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:07 smithi130 bash[27579]: audit 2023-08-27T23:23:06.915397+0000 mon.a (mon.0) 2186 : audit [DBG] from='client.? 172.21.15.6:0/1492572828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:08.703 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:08 smithi130 bash[51302]: debug 2023-08-27T23:23:08.402+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:08.404042+0000) 2023-08-27T23:23:08.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:08 smithi006 bash[27292]: cluster 2023-08-27T23:23:06.967737+0000 mgr.y (mgr.14901) 2537 : cluster [DBG] pgmap v2544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:08 smithi006 bash[33918]: cluster 2023-08-27T23:23:06.967737+0000 mgr.y (mgr.14901) 2537 : cluster [DBG] pgmap v2544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:08 smithi130 bash[27579]: cluster 2023-08-27T23:23:06.967737+0000 mgr.y (mgr.14901) 2537 : cluster [DBG] pgmap v2544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:09.717 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:09 smithi130 bash[51302]: debug 2023-08-27T23:23:09.402+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:09.404363+0000) 2023-08-27T23:23:10.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:09 smithi130 bash[27579]: audit 2023-08-27T23:23:09.332015+0000 mon.a (mon.0) 2187 : audit [DBG] from='client.? 172.21.15.6:0/3948506852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:10.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:09 smithi006 bash[27292]: audit 2023-08-27T23:23:09.332015+0000 mon.a (mon.0) 2187 : audit [DBG] from='client.? 172.21.15.6:0/3948506852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:10.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:09 smithi006 bash[33918]: audit 2023-08-27T23:23:09.332015+0000 mon.a (mon.0) 2187 : audit [DBG] from='client.? 172.21.15.6:0/3948506852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:23:10.449Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:23:10.451Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:10.720 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:10 smithi130 bash[51302]: debug 2023-08-27T23:23:10.402+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:10.404694+0000) 2023-08-27T23:23:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:10 smithi130 bash[27579]: cluster 2023-08-27T23:23:08.968284+0000 mgr.y (mgr.14901) 2538 : cluster [DBG] pgmap v2545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:10 smithi006 bash[33918]: cluster 2023-08-27T23:23:08.968284+0000 mgr.y (mgr.14901) 2538 : cluster [DBG] pgmap v2545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:11.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:10 smithi006 bash[27292]: cluster 2023-08-27T23:23:08.968284+0000 mgr.y (mgr.14901) 2538 : cluster [DBG] pgmap v2545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:23:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:23:10] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:23:11.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:11 smithi130 bash[51302]: debug 2023-08-27T23:23:11.402+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:11.404963+0000) 2023-08-27T23:23:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:23:12.012Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:23:12.014Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:12.725 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:12 smithi130 bash[51302]: debug 2023-08-27T23:23:12.402+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:12.405241+0000) 2023-08-27T23:23:13.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:12 smithi130 bash[27579]: cluster 2023-08-27T23:23:10.969394+0000 mgr.y (mgr.14901) 2539 : cluster [DBG] pgmap v2546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:12 smithi130 bash[27579]: audit 2023-08-27T23:23:11.743410+0000 mon.a (mon.0) 2188 : audit [DBG] from='client.? 172.21.15.6:0/188277174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:13.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:12 smithi006 bash[27292]: cluster 2023-08-27T23:23:10.969394+0000 mgr.y (mgr.14901) 2539 : cluster [DBG] pgmap v2546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:12 smithi006 bash[27292]: audit 2023-08-27T23:23:11.743410+0000 mon.a (mon.0) 2188 : audit [DBG] from='client.? 172.21.15.6:0/188277174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:12 smithi006 bash[33918]: cluster 2023-08-27T23:23:10.969394+0000 mgr.y (mgr.14901) 2539 : cluster [DBG] pgmap v2546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:12 smithi006 bash[33918]: audit 2023-08-27T23:23:11.743410+0000 mon.a (mon.0) 2188 : audit [DBG] from='client.? 172.21.15.6:0/188277174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:13.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:13 smithi130 bash[51302]: debug 2023-08-27T23:23:13.402+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:13.405512+0000) 2023-08-27T23:23:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:23:13.920Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:23:13.922Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:14.729 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:14 smithi130 bash[51302]: debug 2023-08-27T23:23:14.402+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:14.405818+0000) 2023-08-27T23:23:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:14 smithi130 bash[27579]: cluster 2023-08-27T23:23:12.970046+0000 mgr.y (mgr.14901) 2540 : cluster [DBG] pgmap v2547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:14 smithi130 bash[27579]: audit 2023-08-27T23:23:14.163037+0000 mon.a (mon.0) 2189 : audit [DBG] from='client.? 172.21.15.6:0/1441257214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:14 smithi006 bash[27292]: cluster 2023-08-27T23:23:12.970046+0000 mgr.y (mgr.14901) 2540 : cluster [DBG] pgmap v2547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:14 smithi006 bash[27292]: audit 2023-08-27T23:23:14.163037+0000 mon.a (mon.0) 2189 : audit [DBG] from='client.? 172.21.15.6:0/1441257214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:14 smithi006 bash[33918]: cluster 2023-08-27T23:23:12.970046+0000 mgr.y (mgr.14901) 2540 : cluster [DBG] pgmap v2547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:14 smithi006 bash[33918]: audit 2023-08-27T23:23:14.163037+0000 mon.a (mon.0) 2189 : audit [DBG] from='client.? 172.21.15.6:0/1441257214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:15.738 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:15 smithi130 bash[51302]: debug 2023-08-27T23:23:15.401+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:15.406093+0000) 2023-08-27T23:23:16.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:15 smithi130 bash[27579]: cluster 2023-08-27T23:23:14.971175+0000 mgr.y (mgr.14901) 2541 : cluster [DBG] pgmap v2548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:16.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:15 smithi006 bash[27292]: cluster 2023-08-27T23:23:14.971175+0000 mgr.y (mgr.14901) 2541 : cluster [DBG] pgmap v2548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:16.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:15 smithi006 bash[33918]: cluster 2023-08-27T23:23:14.971175+0000 mgr.y (mgr.14901) 2541 : cluster [DBG] pgmap v2548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:16.740 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:16 smithi130 bash[51302]: debug 2023-08-27T23:23:16.401+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:16.406374+0000) 2023-08-27T23:23:17.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:16 smithi130 bash[27579]: audit 2023-08-27T23:23:16.582511+0000 mon.a (mon.0) 2190 : audit [DBG] from='client.? 172.21.15.6:0/273448044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:17.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:16 smithi006 bash[27292]: audit 2023-08-27T23:23:16.582511+0000 mon.a (mon.0) 2190 : audit [DBG] from='client.? 172.21.15.6:0/273448044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:16 smithi006 bash[33918]: audit 2023-08-27T23:23:16.582511+0000 mon.a (mon.0) 2190 : audit [DBG] from='client.? 172.21.15.6:0/273448044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:17.748 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:17 smithi130 bash[51302]: debug 2023-08-27T23:23:17.401+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:17.406647+0000) 2023-08-27T23:23:18.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:17 smithi130 bash[27579]: cluster 2023-08-27T23:23:16.971855+0000 mgr.y (mgr.14901) 2542 : cluster [DBG] pgmap v2549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:18.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:17 smithi006 bash[33918]: cluster 2023-08-27T23:23:16.971855+0000 mgr.y (mgr.14901) 2542 : cluster [DBG] pgmap v2549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:18.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:17 smithi006 bash[27292]: cluster 2023-08-27T23:23:16.971855+0000 mgr.y (mgr.14901) 2542 : cluster [DBG] pgmap v2549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:18 smithi130 bash[51302]: debug 2023-08-27T23:23:18.405+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:18.406974+0000) 2023-08-27T23:23:19.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:19 smithi130 bash[27579]: audit 2023-08-27T23:23:18.995501+0000 mon.c (mon.2) 995 : audit [DBG] from='client.? 172.21.15.6:0/3733170613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:19.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:19 smithi006 bash[33918]: audit 2023-08-27T23:23:18.995501+0000 mon.c (mon.2) 995 : audit [DBG] from='client.? 172.21.15.6:0/3733170613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:19.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:19 smithi006 bash[27292]: audit 2023-08-27T23:23:18.995501+0000 mon.c (mon.2) 995 : audit [DBG] from='client.? 172.21.15.6:0/3733170613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:19 smithi130 bash[51302]: debug 2023-08-27T23:23:19.405+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:19.407221+0000) 2023-08-27T23:23:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:20 smithi006 bash[33918]: cluster 2023-08-27T23:23:18.972491+0000 mgr.y (mgr.14901) 2543 : cluster [DBG] pgmap v2550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:23:20.450Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:23:20.452Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:20 smithi006 bash[27292]: cluster 2023-08-27T23:23:18.972491+0000 mgr.y (mgr.14901) 2543 : cluster [DBG] pgmap v2550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:20 smithi130 bash[27579]: cluster 2023-08-27T23:23:18.972491+0000 mgr.y (mgr.14901) 2543 : cluster [DBG] pgmap v2550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:20 smithi130 bash[51302]: debug 2023-08-27T23:23:20.405+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:20.407522+0000) 2023-08-27T23:23:21.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:23:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:23:20] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:23:21.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:21 smithi130 bash[27579]: audit 2023-08-27T23:23:21.413027+0000 mon.a (mon.0) 2191 : audit [DBG] from='client.? 172.21.15.6:0/769819670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:21.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:21 smithi130 bash[51302]: debug 2023-08-27T23:23:21.405+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:21.407791+0000) 2023-08-27T23:23:21.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:21 smithi006 bash[27292]: audit 2023-08-27T23:23:21.413027+0000 mon.a (mon.0) 2191 : audit [DBG] from='client.? 172.21.15.6:0/769819670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:21.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:21 smithi006 bash[33918]: audit 2023-08-27T23:23:21.413027+0000 mon.a (mon.0) 2191 : audit [DBG] from='client.? 172.21.15.6:0/769819670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:23:22.013Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:23:22.015Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:22 smithi130 bash[27579]: cluster 2023-08-27T23:23:20.973655+0000 mgr.y (mgr.14901) 2544 : cluster [DBG] pgmap v2551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:22 smithi130 bash[51302]: debug 2023-08-27T23:23:22.405+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:22.408107+0000) 2023-08-27T23:23:22.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:22 smithi006 bash[33918]: cluster 2023-08-27T23:23:20.973655+0000 mgr.y (mgr.14901) 2544 : cluster [DBG] pgmap v2551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:22 smithi006 bash[27292]: cluster 2023-08-27T23:23:20.973655+0000 mgr.y (mgr.14901) 2544 : cluster [DBG] pgmap v2551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:23.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:23 smithi130 bash[51302]: debug 2023-08-27T23:23:23.405+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:23.408433+0000) 2023-08-27T23:23:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:23:23.920Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:23:23.922Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:24 smithi130 bash[51302]: debug 2023-08-27T23:23:24.405+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:24.408673+0000) 2023-08-27T23:23:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:24 smithi130 bash[27579]: cluster 2023-08-27T23:23:22.974341+0000 mgr.y (mgr.14901) 2545 : cluster [DBG] pgmap v2552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:24 smithi130 bash[27579]: audit 2023-08-27T23:23:23.830465+0000 mon.c (mon.2) 996 : audit [DBG] from='client.? 172.21.15.6:0/1970283694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:24.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:24 smithi006 bash[33918]: cluster 2023-08-27T23:23:22.974341+0000 mgr.y (mgr.14901) 2545 : cluster [DBG] pgmap v2552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:24.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:24 smithi006 bash[33918]: audit 2023-08-27T23:23:23.830465+0000 mon.c (mon.2) 996 : audit [DBG] from='client.? 172.21.15.6:0/1970283694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:24 smithi006 bash[27292]: cluster 2023-08-27T23:23:22.974341+0000 mgr.y (mgr.14901) 2545 : cluster [DBG] pgmap v2552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:24 smithi006 bash[27292]: audit 2023-08-27T23:23:23.830465+0000 mon.c (mon.2) 996 : audit [DBG] from='client.? 172.21.15.6:0/1970283694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:25.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:25 smithi130 bash[51302]: debug 2023-08-27T23:23:25.405+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:25.408993+0000) 2023-08-27T23:23:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:26 smithi130 bash[27579]: cluster 2023-08-27T23:23:24.975386+0000 mgr.y (mgr.14901) 2546 : cluster [DBG] pgmap v2553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:26 smithi130 bash[27579]: audit 2023-08-27T23:23:26.247193+0000 mon.a (mon.0) 2192 : audit [DBG] from='client.? 172.21.15.6:0/2948382636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:26.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:26 smithi130 bash[51302]: debug 2023-08-27T23:23:26.405+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:26.409278+0000) 2023-08-27T23:23:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:26 smithi006 bash[33918]: cluster 2023-08-27T23:23:24.975386+0000 mgr.y (mgr.14901) 2546 : cluster [DBG] pgmap v2553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:26 smithi006 bash[33918]: audit 2023-08-27T23:23:26.247193+0000 mon.a (mon.0) 2192 : audit [DBG] from='client.? 172.21.15.6:0/2948382636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:26 smithi006 bash[27292]: cluster 2023-08-27T23:23:24.975386+0000 mgr.y (mgr.14901) 2546 : cluster [DBG] pgmap v2553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:26 smithi006 bash[27292]: audit 2023-08-27T23:23:26.247193+0000 mon.a (mon.0) 2192 : audit [DBG] from='client.? 172.21.15.6:0/2948382636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:27.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:27 smithi130 bash[51302]: debug 2023-08-27T23:23:27.405+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:27.409547+0000) 2023-08-27T23:23:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:28 smithi130 bash[27579]: cluster 2023-08-27T23:23:26.976111+0000 mgr.y (mgr.14901) 2547 : cluster [DBG] pgmap v2554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:28 smithi130 bash[51302]: debug 2023-08-27T23:23:28.405+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:28.409861+0000) 2023-08-27T23:23:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:28 smithi006 bash[27292]: cluster 2023-08-27T23:23:26.976111+0000 mgr.y (mgr.14901) 2547 : cluster [DBG] pgmap v2554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:28 smithi006 bash[33918]: cluster 2023-08-27T23:23:26.976111+0000 mgr.y (mgr.14901) 2547 : cluster [DBG] pgmap v2554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:29.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:29 smithi130 bash[27579]: audit 2023-08-27T23:23:28.665686+0000 mon.c (mon.2) 997 : audit [DBG] from='client.? 172.21.15.6:0/4125129713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:29.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:29 smithi130 bash[51302]: debug 2023-08-27T23:23:29.405+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:29.410169+0000) 2023-08-27T23:23:29.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:29 smithi006 bash[33918]: audit 2023-08-27T23:23:28.665686+0000 mon.c (mon.2) 997 : audit [DBG] from='client.? 172.21.15.6:0/4125129713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:29.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:29 smithi006 bash[27292]: audit 2023-08-27T23:23:28.665686+0000 mon.c (mon.2) 997 : audit [DBG] from='client.? 172.21.15.6:0/4125129713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:30 smithi006 bash[27292]: cluster 2023-08-27T23:23:28.976806+0000 mgr.y (mgr.14901) 2548 : cluster [DBG] pgmap v2555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:30 smithi006 bash[33918]: cluster 2023-08-27T23:23:28.976806+0000 mgr.y (mgr.14901) 2548 : cluster [DBG] pgmap v2555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:23:30.451Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:23:30.453Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:30 smithi130 bash[27579]: cluster 2023-08-27T23:23:28.976806+0000 mgr.y (mgr.14901) 2548 : cluster [DBG] pgmap v2555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:30 smithi130 bash[51302]: debug 2023-08-27T23:23:30.405+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:30.410433+0000) 2023-08-27T23:23:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:23:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:23:30] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:23:31.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:31 smithi130 bash[27579]: audit 2023-08-27T23:23:31.082844+0000 mon.a (mon.0) 2193 : audit [DBG] from='client.? 172.21.15.6:0/3659373845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:31.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:31 smithi130 bash[51302]: debug 2023-08-27T23:23:31.405+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:31.410661+0000) 2023-08-27T23:23:31.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:31 smithi006 bash[27292]: audit 2023-08-27T23:23:31.082844+0000 mon.a (mon.0) 2193 : audit [DBG] from='client.? 172.21.15.6:0/3659373845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:31.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:31 smithi006 bash[33918]: audit 2023-08-27T23:23:31.082844+0000 mon.a (mon.0) 2193 : audit [DBG] from='client.? 172.21.15.6:0/3659373845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:23:32.013Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:23:32.015Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:32 smithi130 bash[27579]: cluster 2023-08-27T23:23:30.977579+0000 mgr.y (mgr.14901) 2549 : cluster [DBG] pgmap v2556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:32 smithi130 bash[51302]: debug 2023-08-27T23:23:32.409+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:32.410922+0000) 2023-08-27T23:23:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:32 smithi006 bash[33918]: cluster 2023-08-27T23:23:30.977579+0000 mgr.y (mgr.14901) 2549 : cluster [DBG] pgmap v2556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:32.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:32 smithi006 bash[27292]: cluster 2023-08-27T23:23:30.977579+0000 mgr.y (mgr.14901) 2549 : cluster [DBG] pgmap v2556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:33.669 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:33 smithi130 bash[51302]: debug 2023-08-27T23:23:33.409+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:33.411209+0000) 2023-08-27T23:23:33.923 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:33 smithi006 bash[33918]: audit 2023-08-27T23:23:33.495030+0000 mon.a (mon.0) 2194 : audit [DBG] from='client.? 172.21.15.6:0/3381015577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:33.923 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:33 smithi006 bash[27292]: audit 2023-08-27T23:23:33.495030+0000 mon.a (mon.0) 2194 : audit [DBG] from='client.? 172.21.15.6:0/3381015577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:34.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:33 smithi130 bash[27579]: audit 2023-08-27T23:23:33.495030+0000 mon.a (mon.0) 2194 : audit [DBG] from='client.? 172.21.15.6:0/3381015577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:34.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:23:33.921Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:34.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:23:33.923Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:34.674 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:34 smithi130 bash[51302]: debug 2023-08-27T23:23:34.409+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:34.411485+0000) 2023-08-27T23:23:34.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:34 smithi006 bash[27292]: cluster 2023-08-27T23:23:32.978165+0000 mgr.y (mgr.14901) 2550 : cluster [DBG] pgmap v2557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:34.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:34 smithi006 bash[33918]: cluster 2023-08-27T23:23:32.978165+0000 mgr.y (mgr.14901) 2550 : cluster [DBG] pgmap v2557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:34 smithi130 bash[27579]: cluster 2023-08-27T23:23:32.978165+0000 mgr.y (mgr.14901) 2550 : cluster [DBG] pgmap v2557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:35.681 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:35 smithi130 bash[51302]: debug 2023-08-27T23:23:35.409+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:35.411835+0000) 2023-08-27T23:23:35.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:35 smithi006 bash[33918]: audit 2023-08-27T23:23:35.634761+0000 mon.b (mon.1) 364 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:23:35.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:35 smithi006 bash[27292]: audit 2023-08-27T23:23:35.634761+0000 mon.b (mon.1) 364 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:23:36.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:35 smithi130 bash[27579]: audit 2023-08-27T23:23:35.634761+0000 mon.b (mon.1) 364 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:23:36.683 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:36 smithi130 bash[51302]: debug 2023-08-27T23:23:36.409+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:36.412114+0000) 2023-08-27T23:23:36.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:36 smithi006 bash[27292]: cluster 2023-08-27T23:23:34.979252+0000 mgr.y (mgr.14901) 2551 : cluster [DBG] pgmap v2558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:36 smithi006 bash[27292]: audit 2023-08-27T23:23:35.915145+0000 mon.a (mon.0) 2195 : audit [DBG] from='client.? 172.21.15.6:0/3796752091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:36 smithi006 bash[33918]: cluster 2023-08-27T23:23:34.979252+0000 mgr.y (mgr.14901) 2551 : cluster [DBG] pgmap v2558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:36 smithi006 bash[33918]: audit 2023-08-27T23:23:35.915145+0000 mon.a (mon.0) 2195 : audit [DBG] from='client.? 172.21.15.6:0/3796752091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:36 smithi130 bash[27579]: cluster 2023-08-27T23:23:34.979252+0000 mgr.y (mgr.14901) 2551 : cluster [DBG] pgmap v2558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:37.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:36 smithi130 bash[27579]: audit 2023-08-27T23:23:35.915145+0000 mon.a (mon.0) 2195 : audit [DBG] from='client.? 172.21.15.6:0/3796752091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:37 smithi130 bash[51302]: debug 2023-08-27T23:23:37.409+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:37.412375+0000) 2023-08-27T23:23:38.688 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:38 smithi130 bash[51302]: debug 2023-08-27T23:23:38.409+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:38.412700+0000) 2023-08-27T23:23:38.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:38 smithi006 bash[27292]: cluster 2023-08-27T23:23:36.979901+0000 mgr.y (mgr.14901) 2552 : cluster [DBG] pgmap v2559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:38 smithi006 bash[27292]: audit 2023-08-27T23:23:38.339595+0000 mon.c (mon.2) 998 : audit [DBG] from='client.? 172.21.15.6:0/2720317694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:38 smithi006 bash[33918]: cluster 2023-08-27T23:23:36.979901+0000 mgr.y (mgr.14901) 2552 : cluster [DBG] pgmap v2559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:38 smithi006 bash[33918]: audit 2023-08-27T23:23:38.339595+0000 mon.c (mon.2) 998 : audit [DBG] from='client.? 172.21.15.6:0/2720317694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:38 smithi130 bash[27579]: cluster 2023-08-27T23:23:36.979901+0000 mgr.y (mgr.14901) 2552 : cluster [DBG] pgmap v2559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:38 smithi130 bash[27579]: audit 2023-08-27T23:23:38.339595+0000 mon.c (mon.2) 998 : audit [DBG] from='client.? 172.21.15.6:0/2720317694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:39 smithi130 bash[51302]: debug 2023-08-27T23:23:39.409+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:39.412976+0000) 2023-08-27T23:23:40.693 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:40 smithi130 bash[51302]: debug 2023-08-27T23:23:40.409+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:40.413281+0000) 2023-08-27T23:23:40.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:40 smithi006 bash[33918]: cluster 2023-08-27T23:23:38.980500+0000 mgr.y (mgr.14901) 2553 : cluster [DBG] pgmap v2560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:40 smithi006 bash[27292]: cluster 2023-08-27T23:23:38.980500+0000 mgr.y (mgr.14901) 2553 : cluster [DBG] pgmap v2560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:23:40.452Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:23:40.454Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:40 smithi130 bash[27579]: cluster 2023-08-27T23:23:38.980500+0000 mgr.y (mgr.14901) 2553 : cluster [DBG] pgmap v2560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:23:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:23:40] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:23:41.701 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:41 smithi130 bash[51302]: debug 2023-08-27T23:23:41.409+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:41.413481+0000) 2023-08-27T23:23:41.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:41 smithi006 bash[33918]: audit 2023-08-27T23:23:40.787004+0000 mon.c (mon.2) 999 : audit [DBG] from='client.? 172.21.15.6:0/474310602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:41.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:41 smithi006 bash[33918]: audit 2023-08-27T23:23:41.466530+0000 mon.a (mon.0) 2196 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:23:41.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:41 smithi006 bash[27292]: audit 2023-08-27T23:23:40.787004+0000 mon.c (mon.2) 999 : audit [DBG] from='client.? 172.21.15.6:0/474310602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:41.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:41 smithi006 bash[27292]: audit 2023-08-27T23:23:41.466530+0000 mon.a (mon.0) 2196 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:23:42.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:41 smithi130 bash[27579]: audit 2023-08-27T23:23:40.787004+0000 mon.c (mon.2) 999 : audit [DBG] from='client.? 172.21.15.6:0/474310602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:42.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:41 smithi130 bash[27579]: audit 2023-08-27T23:23:41.466530+0000 mon.a (mon.0) 2196 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:23:42.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:23:42.015Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:23:42.016Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:42.703 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:42 smithi130 bash[51302]: debug 2023-08-27T23:23:42.409+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:42.413772+0000) 2023-08-27T23:23:42.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:42 smithi006 bash[33918]: cluster 2023-08-27T23:23:40.981762+0000 mgr.y (mgr.14901) 2554 : cluster [DBG] pgmap v2561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:42 smithi006 bash[33918]: audit 2023-08-27T23:23:41.948114+0000 mon.a (mon.0) 2197 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:23:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:42 smithi006 bash[27292]: cluster 2023-08-27T23:23:40.981762+0000 mgr.y (mgr.14901) 2554 : cluster [DBG] pgmap v2561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:42 smithi006 bash[27292]: audit 2023-08-27T23:23:41.948114+0000 mon.a (mon.0) 2197 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:23:43.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:42 smithi130 bash[27579]: cluster 2023-08-27T23:23:40.981762+0000 mgr.y (mgr.14901) 2554 : cluster [DBG] pgmap v2561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:42 smithi130 bash[27579]: audit 2023-08-27T23:23:41.948114+0000 mon.a (mon.0) 2197 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:23:43.712 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:43 smithi130 bash[51302]: debug 2023-08-27T23:23:43.409+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:43.414093+0000) 2023-08-27T23:23:43.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:43 smithi006 bash[33918]: audit 2023-08-27T23:23:43.203359+0000 mon.a (mon.0) 2198 : audit [DBG] from='client.? 172.21.15.6:0/1933286671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:43.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:43 smithi006 bash[27292]: audit 2023-08-27T23:23:43.203359+0000 mon.a (mon.0) 2198 : audit [DBG] from='client.? 172.21.15.6:0/1933286671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:43.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:23:43.921Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:43.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:23:43.924Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:44.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:43 smithi130 bash[27579]: audit 2023-08-27T23:23:43.203359+0000 mon.a (mon.0) 2198 : audit [DBG] from='client.? 172.21.15.6:0/1933286671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:44.713 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:44 smithi130 bash[51302]: debug 2023-08-27T23:23:44.413+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:44.414407+0000) 2023-08-27T23:23:44.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:44 smithi006 bash[27292]: cluster 2023-08-27T23:23:42.982398+0000 mgr.y (mgr.14901) 2555 : cluster [DBG] pgmap v2562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:44 smithi006 bash[33918]: cluster 2023-08-27T23:23:42.982398+0000 mgr.y (mgr.14901) 2555 : cluster [DBG] pgmap v2562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:44 smithi130 bash[27579]: cluster 2023-08-27T23:23:42.982398+0000 mgr.y (mgr.14901) 2555 : cluster [DBG] pgmap v2562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:45.723 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:45 smithi130 bash[51302]: debug 2023-08-27T23:23:45.413+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:45.414766+0000) 2023-08-27T23:23:46.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:45 smithi130 bash[27579]: audit 2023-08-27T23:23:45.624477+0000 mon.c (mon.2) 1000 : audit [DBG] from='client.? 172.21.15.6:0/1677793434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:46.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:45 smithi006 bash[33918]: audit 2023-08-27T23:23:45.624477+0000 mon.c (mon.2) 1000 : audit [DBG] from='client.? 172.21.15.6:0/1677793434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:46.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:45 smithi006 bash[27292]: audit 2023-08-27T23:23:45.624477+0000 mon.c (mon.2) 1000 : audit [DBG] from='client.? 172.21.15.6:0/1677793434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:46.725 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:46 smithi130 bash[51302]: debug 2023-08-27T23:23:46.413+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:46.415175+0000) 2023-08-27T23:23:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:46 smithi130 bash[27579]: cluster 2023-08-27T23:23:44.983417+0000 mgr.y (mgr.14901) 2556 : cluster [DBG] pgmap v2563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:47.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:46 smithi006 bash[33918]: cluster 2023-08-27T23:23:44.983417+0000 mgr.y (mgr.14901) 2556 : cluster [DBG] pgmap v2563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:47.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:46 smithi006 bash[27292]: cluster 2023-08-27T23:23:44.983417+0000 mgr.y (mgr.14901) 2556 : cluster [DBG] pgmap v2563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:47.850 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:47 smithi130 bash[51302]: debug 2023-08-27T23:23:47.413+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:47.415475+0000) 2023-08-27T23:23:48.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:48 smithi130 bash[51302]: debug 2023-08-27T23:23:48.413+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:48.415776+0000) 2023-08-27T23:23:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:48 smithi130 bash[27579]: cluster 2023-08-27T23:23:46.984023+0000 mgr.y (mgr.14901) 2557 : cluster [DBG] pgmap v2564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:48 smithi130 bash[27579]: audit 2023-08-27T23:23:48.036976+0000 mon.c (mon.2) 1001 : audit [DBG] from='client.? 172.21.15.6:0/2605353881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:49.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:48 smithi006 bash[33918]: cluster 2023-08-27T23:23:46.984023+0000 mgr.y (mgr.14901) 2557 : cluster [DBG] pgmap v2564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:49.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:48 smithi006 bash[33918]: audit 2023-08-27T23:23:48.036976+0000 mon.c (mon.2) 1001 : audit [DBG] from='client.? 172.21.15.6:0/2605353881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:49.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:48 smithi006 bash[27292]: cluster 2023-08-27T23:23:46.984023+0000 mgr.y (mgr.14901) 2557 : cluster [DBG] pgmap v2564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:49.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:48 smithi006 bash[27292]: audit 2023-08-27T23:23:48.036976+0000 mon.c (mon.2) 1001 : audit [DBG] from='client.? 172.21.15.6:0/2605353881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:49.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:49 smithi130 bash[51302]: debug 2023-08-27T23:23:49.413+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:49.416043+0000) 2023-08-27T23:23:50.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:23:50.453Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:23:50.455Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:50.735 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:50 smithi130 bash[51302]: debug 2023-08-27T23:23:50.413+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:50.416279+0000) 2023-08-27T23:23:51.007 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:50 smithi130 bash[27579]: cluster 2023-08-27T23:23:48.984674+0000 mgr.y (mgr.14901) 2558 : cluster [DBG] pgmap v2565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:51.007 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:50 smithi130 bash[27579]: audit 2023-08-27T23:23:50.467087+0000 mon.a (mon.0) 2199 : audit [DBG] from='client.? 172.21.15.6:0/3442481748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:23:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:23:50] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:23:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:50 smithi006 bash[27292]: cluster 2023-08-27T23:23:48.984674+0000 mgr.y (mgr.14901) 2558 : cluster [DBG] pgmap v2565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:50 smithi006 bash[27292]: audit 2023-08-27T23:23:50.467087+0000 mon.a (mon.0) 2199 : audit [DBG] from='client.? 172.21.15.6:0/3442481748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:50 smithi006 bash[33918]: cluster 2023-08-27T23:23:48.984674+0000 mgr.y (mgr.14901) 2558 : cluster [DBG] pgmap v2565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:50 smithi006 bash[33918]: audit 2023-08-27T23:23:50.467087+0000 mon.a (mon.0) 2199 : audit [DBG] from='client.? 172.21.15.6:0/3442481748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:51.790 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:51 smithi130 bash[51302]: debug 2023-08-27T23:23:51.413+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:51.416549+0000) 2023-08-27T23:23:52.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:51 smithi130 bash[27579]: cluster 2023-08-27T23:23:50.985777+0000 mgr.y (mgr.14901) 2559 : cluster [DBG] pgmap v2566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:52.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:51 smithi006 bash[33918]: cluster 2023-08-27T23:23:50.985777+0000 mgr.y (mgr.14901) 2559 : cluster [DBG] pgmap v2566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:52.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:51 smithi006 bash[27292]: cluster 2023-08-27T23:23:50.985777+0000 mgr.y (mgr.14901) 2559 : cluster [DBG] pgmap v2566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:52.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:23:52.016Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:52.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:23:52.018Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:52.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:52 smithi130 bash[51302]: debug 2023-08-27T23:23:52.413+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:52.416852+0000) 2023-08-27T23:23:53.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:52 smithi130 bash[27579]: audit 2023-08-27T23:23:51.973794+0000 mon.a (mon.0) 2200 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:23:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:52 smithi130 bash[27579]: audit 2023-08-27T23:23:52.249499+0000 mon.b (mon.1) 365 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:23:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:52 smithi130 bash[27579]: audit 2023-08-27T23:23:52.251728+0000 mon.b (mon.1) 366 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:23:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:52 smithi130 bash[27579]: audit 2023-08-27T23:23:52.262060+0000 mon.a (mon.0) 2201 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:23:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:52 smithi130 bash[27579]: audit 2023-08-27T23:23:52.877989+0000 mon.c (mon.2) 1002 : audit [DBG] from='client.? 172.21.15.6:0/3875833326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:52 smithi006 bash[27292]: audit 2023-08-27T23:23:51.973794+0000 mon.a (mon.0) 2200 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:23:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:52 smithi006 bash[27292]: audit 2023-08-27T23:23:52.249499+0000 mon.b (mon.1) 365 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:23:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:52 smithi006 bash[27292]: audit 2023-08-27T23:23:52.251728+0000 mon.b (mon.1) 366 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:23:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:52 smithi006 bash[27292]: audit 2023-08-27T23:23:52.262060+0000 mon.a (mon.0) 2201 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:23:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:52 smithi006 bash[27292]: audit 2023-08-27T23:23:52.877989+0000 mon.c (mon.2) 1002 : audit [DBG] from='client.? 172.21.15.6:0/3875833326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:53.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:52 smithi006 bash[33918]: audit 2023-08-27T23:23:51.973794+0000 mon.a (mon.0) 2200 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:23:53.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:52 smithi006 bash[33918]: audit 2023-08-27T23:23:52.249499+0000 mon.b (mon.1) 365 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:23:53.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:52 smithi006 bash[33918]: audit 2023-08-27T23:23:52.251728+0000 mon.b (mon.1) 366 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:23:53.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:52 smithi006 bash[33918]: audit 2023-08-27T23:23:52.262060+0000 mon.a (mon.0) 2201 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:23:53.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:52 smithi006 bash[33918]: audit 2023-08-27T23:23:52.877989+0000 mon.c (mon.2) 1002 : audit [DBG] from='client.? 172.21.15.6:0/3875833326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:53.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:53 smithi130 bash[51302]: debug 2023-08-27T23:23:53.413+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:53.417267+0000) 2023-08-27T23:23:54.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:53 smithi006 bash[33918]: cluster 2023-08-27T23:23:52.986399+0000 mgr.y (mgr.14901) 2560 : cluster [DBG] pgmap v2567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:54.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:53 smithi006 bash[27292]: cluster 2023-08-27T23:23:52.986399+0000 mgr.y (mgr.14901) 2560 : cluster [DBG] pgmap v2567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:23:53.922Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:23:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:23:53.924Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:23:54.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:53 smithi130 bash[27579]: cluster 2023-08-27T23:23:52.986399+0000 mgr.y (mgr.14901) 2560 : cluster [DBG] pgmap v2567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:54.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:54 smithi130 bash[51302]: debug 2023-08-27T23:23:54.417+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:54.417553+0000) 2023-08-27T23:23:55.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:55 smithi130 bash[27579]: audit 2023-08-27T23:23:55.301540+0000 mon.c (mon.2) 1003 : audit [DBG] from='client.? 172.21.15.6:0/3718852639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:55.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:55 smithi130 bash[51302]: debug 2023-08-27T23:23:55.417+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:55.417879+0000) 2023-08-27T23:23:55.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:55 smithi006 bash[33918]: audit 2023-08-27T23:23:55.301540+0000 mon.c (mon.2) 1003 : audit [DBG] from='client.? 172.21.15.6:0/3718852639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:55.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:55 smithi006 bash[27292]: audit 2023-08-27T23:23:55.301540+0000 mon.c (mon.2) 1003 : audit [DBG] from='client.? 172.21.15.6:0/3718852639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:56 smithi130 bash[27579]: cluster 2023-08-27T23:23:54.987163+0000 mgr.y (mgr.14901) 2561 : cluster [DBG] pgmap v2568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:56.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:56 smithi130 bash[51302]: debug 2023-08-27T23:23:56.413+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:56.418154+0000) 2023-08-27T23:23:56.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:56 smithi006 bash[33918]: cluster 2023-08-27T23:23:54.987163+0000 mgr.y (mgr.14901) 2561 : cluster [DBG] pgmap v2568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:56 smithi006 bash[27292]: cluster 2023-08-27T23:23:54.987163+0000 mgr.y (mgr.14901) 2561 : cluster [DBG] pgmap v2568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:57.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:57 smithi130 bash[51302]: debug 2023-08-27T23:23:57.413+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:57.418423+0000) 2023-08-27T23:23:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:58 smithi130 bash[27579]: cluster 2023-08-27T23:23:56.987805+0000 mgr.y (mgr.14901) 2562 : cluster [DBG] pgmap v2569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:23:58 smithi130 bash[27579]: audit 2023-08-27T23:23:57.725428+0000 mon.c (mon.2) 1004 : audit [DBG] from='client.? 172.21.15.6:0/3141339647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:58.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:58 smithi130 bash[51302]: debug 2023-08-27T23:23:58.417+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:58.418766+0000) 2023-08-27T23:23:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:58 smithi006 bash[27292]: cluster 2023-08-27T23:23:56.987805+0000 mgr.y (mgr.14901) 2562 : cluster [DBG] pgmap v2569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:23:58 smithi006 bash[27292]: audit 2023-08-27T23:23:57.725428+0000 mon.c (mon.2) 1004 : audit [DBG] from='client.? 172.21.15.6:0/3141339647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:58 smithi006 bash[33918]: cluster 2023-08-27T23:23:56.987805+0000 mgr.y (mgr.14901) 2562 : cluster [DBG] pgmap v2569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:23:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:23:58 smithi006 bash[33918]: audit 2023-08-27T23:23:57.725428+0000 mon.c (mon.2) 1004 : audit [DBG] from='client.? 172.21.15.6:0/3141339647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:59.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:23:59 smithi130 bash[51302]: debug 2023-08-27T23:23:59.417+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:59.419081+0000) 2023-08-27T23:24:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:00 smithi006 bash[33918]: cluster 2023-08-27T23:23:58.988498+0000 mgr.y (mgr.14901) 2563 : cluster [DBG] pgmap v2570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:00 smithi006 bash[33918]: audit 2023-08-27T23:24:00.137210+0000 mon.c (mon.2) 1005 : audit [DBG] from='client.? 172.21.15.6:0/3874096691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:00 smithi006 bash[27292]: cluster 2023-08-27T23:23:58.988498+0000 mgr.y (mgr.14901) 2563 : cluster [DBG] pgmap v2570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:00 smithi006 bash[27292]: audit 2023-08-27T23:24:00.137210+0000 mon.c (mon.2) 1005 : audit [DBG] from='client.? 172.21.15.6:0/3874096691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:24:00.454Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:00.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:24:00.456Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:00 smithi130 bash[27579]: cluster 2023-08-27T23:23:58.988498+0000 mgr.y (mgr.14901) 2563 : cluster [DBG] pgmap v2570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:00 smithi130 bash[27579]: audit 2023-08-27T23:24:00.137210+0000 mon.c (mon.2) 1005 : audit [DBG] from='client.? 172.21.15.6:0/3874096691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:00.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:00 smithi130 bash[51302]: debug 2023-08-27T23:24:00.417+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:00.419351+0000) 2023-08-27T23:24:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:24:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:24:00] "GET /metrics HTTP/1.1" 200 193084 "" "Prometheus/2.33.4" 2023-08-27T23:24:01.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:01 smithi130 bash[51302]: debug 2023-08-27T23:24:01.417+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:01.419647+0000) 2023-08-27T23:24:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:24:02.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:24:02.018Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:02 smithi130 bash[27579]: cluster 2023-08-27T23:24:00.989696+0000 mgr.y (mgr.14901) 2564 : cluster [DBG] pgmap v2571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:02 smithi130 bash[27579]: audit 2023-08-27T23:24:02.551531+0000 mon.c (mon.2) 1006 : audit [DBG] from='client.? 172.21.15.6:0/734095466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:02.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:02 smithi130 bash[51302]: debug 2023-08-27T23:24:02.417+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:02.419952+0000) 2023-08-27T23:24:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:02 smithi006 bash[27292]: cluster 2023-08-27T23:24:00.989696+0000 mgr.y (mgr.14901) 2564 : cluster [DBG] pgmap v2571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:02 smithi006 bash[27292]: audit 2023-08-27T23:24:02.551531+0000 mon.c (mon.2) 1006 : audit [DBG] from='client.? 172.21.15.6:0/734095466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:02 smithi006 bash[33918]: cluster 2023-08-27T23:24:00.989696+0000 mgr.y (mgr.14901) 2564 : cluster [DBG] pgmap v2571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:02 smithi006 bash[33918]: audit 2023-08-27T23:24:02.551531+0000 mon.c (mon.2) 1006 : audit [DBG] from='client.? 172.21.15.6:0/734095466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:03.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:03 smithi130 bash[51302]: debug 2023-08-27T23:24:03.417+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:03.420223+0000) 2023-08-27T23:24:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:24:03.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:24:03.925Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:04 smithi130 bash[27579]: cluster 2023-08-27T23:24:02.990341+0000 mgr.y (mgr.14901) 2565 : cluster [DBG] pgmap v2572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:04.925 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:04 smithi130 bash[51302]: debug 2023-08-27T23:24:04.417+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:04.420492+0000) 2023-08-27T23:24:04.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:04 smithi006 bash[27292]: cluster 2023-08-27T23:24:02.990341+0000 mgr.y (mgr.14901) 2565 : cluster [DBG] pgmap v2572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:04 smithi006 bash[33918]: cluster 2023-08-27T23:24:02.990341+0000 mgr.y (mgr.14901) 2565 : cluster [DBG] pgmap v2572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:05.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:05 smithi130 bash[27579]: audit 2023-08-27T23:24:04.969559+0000 mon.a (mon.0) 2202 : audit [DBG] from='client.? 172.21.15.6:0/2661179062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:05.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:05 smithi130 bash[51302]: debug 2023-08-27T23:24:05.417+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:05.420760+0000) 2023-08-27T23:24:05.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:05 smithi006 bash[27292]: audit 2023-08-27T23:24:04.969559+0000 mon.a (mon.0) 2202 : audit [DBG] from='client.? 172.21.15.6:0/2661179062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:05.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:05 smithi006 bash[33918]: audit 2023-08-27T23:24:04.969559+0000 mon.a (mon.0) 2202 : audit [DBG] from='client.? 172.21.15.6:0/2661179062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:06 smithi130 bash[27579]: cluster 2023-08-27T23:24:04.991694+0000 mgr.y (mgr.14901) 2566 : cluster [DBG] pgmap v2573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:06.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:06 smithi130 bash[51302]: debug 2023-08-27T23:24:06.417+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:06.421116+0000) 2023-08-27T23:24:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:06 smithi006 bash[27292]: cluster 2023-08-27T23:24:04.991694+0000 mgr.y (mgr.14901) 2566 : cluster [DBG] pgmap v2573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:06 smithi006 bash[33918]: cluster 2023-08-27T23:24:04.991694+0000 mgr.y (mgr.14901) 2566 : cluster [DBG] pgmap v2573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:07.674 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:07 smithi130 bash[51302]: debug 2023-08-27T23:24:07.417+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:07.421425+0000) 2023-08-27T23:24:07.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:07 smithi006 bash[27292]: audit 2023-08-27T23:24:07.389444+0000 mon.a (mon.0) 2203 : audit [DBG] from='client.? 172.21.15.6:0/171016190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:07.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:07 smithi006 bash[33918]: audit 2023-08-27T23:24:07.389444+0000 mon.a (mon.0) 2203 : audit [DBG] from='client.? 172.21.15.6:0/171016190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:08.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:07 smithi130 bash[27579]: audit 2023-08-27T23:24:07.389444+0000 mon.a (mon.0) 2203 : audit [DBG] from='client.? 172.21.15.6:0/171016190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:08.677 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:08 smithi130 bash[51302]: debug 2023-08-27T23:24:08.417+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:08.421720+0000) 2023-08-27T23:24:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:08 smithi006 bash[27292]: cluster 2023-08-27T23:24:06.992398+0000 mgr.y (mgr.14901) 2567 : cluster [DBG] pgmap v2574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:08 smithi006 bash[33918]: cluster 2023-08-27T23:24:06.992398+0000 mgr.y (mgr.14901) 2567 : cluster [DBG] pgmap v2574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:08 smithi130 bash[27579]: cluster 2023-08-27T23:24:06.992398+0000 mgr.y (mgr.14901) 2567 : cluster [DBG] pgmap v2574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:09.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:09 smithi130 bash[51302]: debug 2023-08-27T23:24:09.421+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:09.421997+0000) 2023-08-27T23:24:10.686 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:10 smithi130 bash[51302]: debug 2023-08-27T23:24:10.416+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:10.422308+0000) 2023-08-27T23:24:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:10 smithi006 bash[27292]: cluster 2023-08-27T23:24:08.993063+0000 mgr.y (mgr.14901) 2568 : cluster [DBG] pgmap v2575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:10 smithi006 bash[27292]: audit 2023-08-27T23:24:09.796089+0000 mon.c (mon.2) 1007 : audit [DBG] from='client.? 172.21.15.6:0/671698881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:24:10.455Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:24:10.457Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:10 smithi006 bash[33918]: cluster 2023-08-27T23:24:08.993063+0000 mgr.y (mgr.14901) 2568 : cluster [DBG] pgmap v2575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:10 smithi006 bash[33918]: audit 2023-08-27T23:24:09.796089+0000 mon.c (mon.2) 1007 : audit [DBG] from='client.? 172.21.15.6:0/671698881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:10 smithi130 bash[27579]: cluster 2023-08-27T23:24:08.993063+0000 mgr.y (mgr.14901) 2568 : cluster [DBG] pgmap v2575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:10 smithi130 bash[27579]: audit 2023-08-27T23:24:09.796089+0000 mon.c (mon.2) 1007 : audit [DBG] from='client.? 172.21.15.6:0/671698881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:24:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:24:10] "GET /metrics HTTP/1.1" 200 193084 "" "Prometheus/2.33.4" 2023-08-27T23:24:11.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:11 smithi130 bash[51302]: debug 2023-08-27T23:24:11.416+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:11.422619+0000) 2023-08-27T23:24:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:24:12.018Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:24:12.020Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:12.691 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:12 smithi130 bash[51302]: debug 2023-08-27T23:24:12.420+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:12.422988+0000) 2023-08-27T23:24:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:12 smithi006 bash[33918]: cluster 2023-08-27T23:24:10.994182+0000 mgr.y (mgr.14901) 2569 : cluster [DBG] pgmap v2576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:12 smithi006 bash[33918]: audit 2023-08-27T23:24:12.210173+0000 mon.a (mon.0) 2204 : audit [DBG] from='client.? 172.21.15.6:0/3353252685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:12 smithi006 bash[27292]: cluster 2023-08-27T23:24:10.994182+0000 mgr.y (mgr.14901) 2569 : cluster [DBG] pgmap v2576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:12 smithi006 bash[27292]: audit 2023-08-27T23:24:12.210173+0000 mon.a (mon.0) 2204 : audit [DBG] from='client.? 172.21.15.6:0/3353252685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:12 smithi130 bash[27579]: cluster 2023-08-27T23:24:10.994182+0000 mgr.y (mgr.14901) 2569 : cluster [DBG] pgmap v2576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:12 smithi130 bash[27579]: audit 2023-08-27T23:24:12.210173+0000 mon.a (mon.0) 2204 : audit [DBG] from='client.? 172.21.15.6:0/3353252685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:13.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:13 smithi130 bash[51302]: debug 2023-08-27T23:24:13.420+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:13.423223+0000) 2023-08-27T23:24:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:24:13.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:24:13.925Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:14.695 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:14 smithi130 bash[51302]: debug 2023-08-27T23:24:14.420+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:14.423544+0000) 2023-08-27T23:24:14.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:14 smithi006 bash[27292]: cluster 2023-08-27T23:24:12.994846+0000 mgr.y (mgr.14901) 2570 : cluster [DBG] pgmap v2577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:14 smithi006 bash[27292]: audit 2023-08-27T23:24:14.628124+0000 mon.a (mon.0) 2205 : audit [DBG] from='client.? 172.21.15.6:0/969681785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:14 smithi006 bash[33918]: cluster 2023-08-27T23:24:12.994846+0000 mgr.y (mgr.14901) 2570 : cluster [DBG] pgmap v2577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:14 smithi006 bash[33918]: audit 2023-08-27T23:24:14.628124+0000 mon.a (mon.0) 2205 : audit [DBG] from='client.? 172.21.15.6:0/969681785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:14 smithi130 bash[27579]: cluster 2023-08-27T23:24:12.994846+0000 mgr.y (mgr.14901) 2570 : cluster [DBG] pgmap v2577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:14 smithi130 bash[27579]: audit 2023-08-27T23:24:14.628124+0000 mon.a (mon.0) 2205 : audit [DBG] from='client.? 172.21.15.6:0/969681785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:15.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:15 smithi130 bash[51302]: debug 2023-08-27T23:24:15.420+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:15.423845+0000) 2023-08-27T23:24:16.701 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:16 smithi130 bash[51302]: debug 2023-08-27T23:24:16.420+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:16.424122+0000) 2023-08-27T23:24:16.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:16 smithi006 bash[27292]: cluster 2023-08-27T23:24:14.996373+0000 mgr.y (mgr.14901) 2571 : cluster [DBG] pgmap v2578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:16 smithi006 bash[33918]: cluster 2023-08-27T23:24:14.996373+0000 mgr.y (mgr.14901) 2571 : cluster [DBG] pgmap v2578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:16 smithi130 bash[27579]: cluster 2023-08-27T23:24:14.996373+0000 mgr.y (mgr.14901) 2571 : cluster [DBG] pgmap v2578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:17.710 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:17 smithi130 bash[51302]: debug 2023-08-27T23:24:17.420+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:17.424406+0000) 2023-08-27T23:24:17.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:17 smithi006 bash[33918]: audit 2023-08-27T23:24:17.035886+0000 mon.c (mon.2) 1008 : audit [DBG] from='client.? 172.21.15.6:0/3243622056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:17.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:17 smithi006 bash[27292]: audit 2023-08-27T23:24:17.035886+0000 mon.c (mon.2) 1008 : audit [DBG] from='client.? 172.21.15.6:0/3243622056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:18.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:17 smithi130 bash[27579]: audit 2023-08-27T23:24:17.035886+0000 mon.c (mon.2) 1008 : audit [DBG] from='client.? 172.21.15.6:0/3243622056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:18.712 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:18 smithi130 bash[51302]: debug 2023-08-27T23:24:18.420+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:18.424688+0000) 2023-08-27T23:24:18.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:18 smithi006 bash[27292]: cluster 2023-08-27T23:24:16.997034+0000 mgr.y (mgr.14901) 2572 : cluster [DBG] pgmap v2579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:18 smithi006 bash[33918]: cluster 2023-08-27T23:24:16.997034+0000 mgr.y (mgr.14901) 2572 : cluster [DBG] pgmap v2579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:18 smithi130 bash[27579]: cluster 2023-08-27T23:24:16.997034+0000 mgr.y (mgr.14901) 2572 : cluster [DBG] pgmap v2579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:19.719 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:19 smithi130 bash[51302]: debug 2023-08-27T23:24:19.420+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:19.424979+0000) 2023-08-27T23:24:20.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:19 smithi130 bash[27579]: audit 2023-08-27T23:24:19.455114+0000 mon.a (mon.0) 2206 : audit [DBG] from='client.? 172.21.15.6:0/1025297379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:19 smithi006 bash[27292]: audit 2023-08-27T23:24:19.455114+0000 mon.a (mon.0) 2206 : audit [DBG] from='client.? 172.21.15.6:0/1025297379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:20.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:19 smithi006 bash[33918]: audit 2023-08-27T23:24:19.455114+0000 mon.a (mon.0) 2206 : audit [DBG] from='client.? 172.21.15.6:0/1025297379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:24:20.456Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:24:20.458Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:20.730 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:20 smithi130 bash[51302]: debug 2023-08-27T23:24:20.420+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:20.425278+0000) 2023-08-27T23:24:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:20 smithi130 bash[27579]: cluster 2023-08-27T23:24:18.997689+0000 mgr.y (mgr.14901) 2573 : cluster [DBG] pgmap v2580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:21.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:20 smithi006 bash[33918]: cluster 2023-08-27T23:24:18.997689+0000 mgr.y (mgr.14901) 2573 : cluster [DBG] pgmap v2580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:20 smithi006 bash[27292]: cluster 2023-08-27T23:24:18.997689+0000 mgr.y (mgr.14901) 2573 : cluster [DBG] pgmap v2580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:24:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:24:20] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:24:21.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:21 smithi130 bash[51302]: debug 2023-08-27T23:24:21.420+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:21.425540+0000) 2023-08-27T23:24:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:24:22.018Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:24:22.020Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:22.735 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:22 smithi130 bash[51302]: debug 2023-08-27T23:24:22.420+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:22.425762+0000) 2023-08-27T23:24:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:22 smithi130 bash[27579]: cluster 2023-08-27T23:24:20.998948+0000 mgr.y (mgr.14901) 2574 : cluster [DBG] pgmap v2581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:22 smithi130 bash[27579]: audit 2023-08-27T23:24:21.874383+0000 mon.c (mon.2) 1009 : audit [DBG] from='client.? 172.21.15.6:0/4134197093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:22 smithi006 bash[27292]: cluster 2023-08-27T23:24:20.998948+0000 mgr.y (mgr.14901) 2574 : cluster [DBG] pgmap v2581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:22 smithi006 bash[27292]: audit 2023-08-27T23:24:21.874383+0000 mon.c (mon.2) 1009 : audit [DBG] from='client.? 172.21.15.6:0/4134197093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:22 smithi006 bash[33918]: cluster 2023-08-27T23:24:20.998948+0000 mgr.y (mgr.14901) 2574 : cluster [DBG] pgmap v2581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:22 smithi006 bash[33918]: audit 2023-08-27T23:24:21.874383+0000 mon.c (mon.2) 1009 : audit [DBG] from='client.? 172.21.15.6:0/4134197093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:23.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:23 smithi130 bash[51302]: debug 2023-08-27T23:24:23.424+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:23.426050+0000) 2023-08-27T23:24:24.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:24:23.924Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:24:23.927Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:24.740 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:24 smithi130 bash[51302]: debug 2023-08-27T23:24:24.420+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:24.426368+0000) 2023-08-27T23:24:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:24 smithi130 bash[27579]: cluster 2023-08-27T23:24:22.999685+0000 mgr.y (mgr.14901) 2575 : cluster [DBG] pgmap v2582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:25.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:24 smithi130 bash[27579]: audit 2023-08-27T23:24:24.295787+0000 mon.a (mon.0) 2207 : audit [DBG] from='client.? 172.21.15.6:0/2383489448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:24 smithi006 bash[33918]: cluster 2023-08-27T23:24:22.999685+0000 mgr.y (mgr.14901) 2575 : cluster [DBG] pgmap v2582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:24 smithi006 bash[33918]: audit 2023-08-27T23:24:24.295787+0000 mon.a (mon.0) 2207 : audit [DBG] from='client.? 172.21.15.6:0/2383489448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:24 smithi006 bash[27292]: cluster 2023-08-27T23:24:22.999685+0000 mgr.y (mgr.14901) 2575 : cluster [DBG] pgmap v2582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:24 smithi006 bash[27292]: audit 2023-08-27T23:24:24.295787+0000 mon.a (mon.0) 2207 : audit [DBG] from='client.? 172.21.15.6:0/2383489448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:25.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:25 smithi130 bash[51302]: debug 2023-08-27T23:24:25.424+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:25.426642+0000) 2023-08-27T23:24:26.745 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:26 smithi130 bash[51302]: debug 2023-08-27T23:24:26.424+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:26.426950+0000) 2023-08-27T23:24:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:26 smithi130 bash[27579]: cluster 2023-08-27T23:24:25.000818+0000 mgr.y (mgr.14901) 2576 : cluster [DBG] pgmap v2583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:26 smithi130 bash[27579]: audit 2023-08-27T23:24:26.702144+0000 mon.a (mon.0) 2208 : audit [DBG] from='client.? 172.21.15.6:0/3229786688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:26 smithi006 bash[27292]: cluster 2023-08-27T23:24:25.000818+0000 mgr.y (mgr.14901) 2576 : cluster [DBG] pgmap v2583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:26 smithi006 bash[27292]: audit 2023-08-27T23:24:26.702144+0000 mon.a (mon.0) 2208 : audit [DBG] from='client.? 172.21.15.6:0/3229786688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:26 smithi006 bash[33918]: cluster 2023-08-27T23:24:25.000818+0000 mgr.y (mgr.14901) 2576 : cluster [DBG] pgmap v2583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:27.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:26 smithi006 bash[33918]: audit 2023-08-27T23:24:26.702144+0000 mon.a (mon.0) 2208 : audit [DBG] from='client.? 172.21.15.6:0/3229786688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:27.753 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:27 smithi130 bash[51302]: debug 2023-08-27T23:24:27.424+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:27.427182+0000) 2023-08-27T23:24:28.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:27 smithi130 bash[27579]: cluster 2023-08-27T23:24:27.001471+0000 mgr.y (mgr.14901) 2577 : cluster [DBG] pgmap v2584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:28.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:27 smithi006 bash[27292]: cluster 2023-08-27T23:24:27.001471+0000 mgr.y (mgr.14901) 2577 : cluster [DBG] pgmap v2584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:28.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:27 smithi006 bash[33918]: cluster 2023-08-27T23:24:27.001471+0000 mgr.y (mgr.14901) 2577 : cluster [DBG] pgmap v2584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:28 smithi130 bash[51302]: debug 2023-08-27T23:24:28.424+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:28.427505+0000) 2023-08-27T23:24:29.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:29 smithi130 bash[51302]: debug 2023-08-27T23:24:29.424+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:29.427837+0000) 2023-08-27T23:24:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:30 smithi130 bash[27579]: cluster 2023-08-27T23:24:29.002087+0000 mgr.y (mgr.14901) 2578 : cluster [DBG] pgmap v2585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:30 smithi130 bash[27579]: audit 2023-08-27T23:24:29.114371+0000 mon.c (mon.2) 1010 : audit [DBG] from='client.? 172.21.15.6:0/2964776316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:30.459 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:30 smithi006 bash[33918]: cluster 2023-08-27T23:24:29.002087+0000 mgr.y (mgr.14901) 2578 : cluster [DBG] pgmap v2585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:30.459 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:30 smithi006 bash[33918]: audit 2023-08-27T23:24:29.114371+0000 mon.c (mon.2) 1010 : audit [DBG] from='client.? 172.21.15.6:0/2964776316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:30.460 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:30 smithi006 bash[27292]: cluster 2023-08-27T23:24:29.002087+0000 mgr.y (mgr.14901) 2578 : cluster [DBG] pgmap v2585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:30.460 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:30 smithi006 bash[27292]: audit 2023-08-27T23:24:29.114371+0000 mon.c (mon.2) 1010 : audit [DBG] from='client.? 172.21.15.6:0/2964776316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:30.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:24:30.457Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:24:30.459Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:30 smithi130 bash[51302]: debug 2023-08-27T23:24:30.424+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:30.428111+0000) 2023-08-27T23:24:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:24:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:24:30] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T23:24:31.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:31 smithi130 bash[51302]: debug 2023-08-27T23:24:31.424+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:31.428411+0000) 2023-08-27T23:24:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:32 smithi130 bash[27579]: cluster 2023-08-27T23:24:31.003237+0000 mgr.y (mgr.14901) 2579 : cluster [DBG] pgmap v2586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:32 smithi130 bash[27579]: audit 2023-08-27T23:24:31.529203+0000 mon.a (mon.0) 2209 : audit [DBG] from='client.? 172.21.15.6:0/1638216926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:32.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:32 smithi006 bash[27292]: cluster 2023-08-27T23:24:31.003237+0000 mgr.y (mgr.14901) 2579 : cluster [DBG] pgmap v2586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:32.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:32 smithi006 bash[27292]: audit 2023-08-27T23:24:31.529203+0000 mon.a (mon.0) 2209 : audit [DBG] from='client.? 172.21.15.6:0/1638216926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:24:32.019Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:24:32.021Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:32.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:32 smithi006 bash[33918]: cluster 2023-08-27T23:24:31.003237+0000 mgr.y (mgr.14901) 2579 : cluster [DBG] pgmap v2586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:32.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:32 smithi006 bash[33918]: audit 2023-08-27T23:24:31.529203+0000 mon.a (mon.0) 2209 : audit [DBG] from='client.? 172.21.15.6:0/1638216926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:32 smithi130 bash[51302]: debug 2023-08-27T23:24:32.424+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:32.428697+0000) 2023-08-27T23:24:33.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:33 smithi130 bash[51302]: debug 2023-08-27T23:24:33.428+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:33.428956+0000) 2023-08-27T23:24:34.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:34 smithi006 bash[27292]: cluster 2023-08-27T23:24:33.003889+0000 mgr.y (mgr.14901) 2580 : cluster [DBG] pgmap v2587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:34.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:34 smithi006 bash[27292]: audit 2023-08-27T23:24:33.947090+0000 mon.a (mon.0) 2210 : audit [DBG] from='client.? 172.21.15.6:0/2271877729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:24:33.925Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:24:33.927Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:34.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:34 smithi006 bash[33918]: cluster 2023-08-27T23:24:33.003889+0000 mgr.y (mgr.14901) 2580 : cluster [DBG] pgmap v2587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:34.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:34 smithi006 bash[33918]: audit 2023-08-27T23:24:33.947090+0000 mon.a (mon.0) 2210 : audit [DBG] from='client.? 172.21.15.6:0/2271877729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:34.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:34 smithi130 bash[27579]: cluster 2023-08-27T23:24:33.003889+0000 mgr.y (mgr.14901) 2580 : cluster [DBG] pgmap v2587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:34.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:34 smithi130 bash[27579]: audit 2023-08-27T23:24:33.947090+0000 mon.a (mon.0) 2210 : audit [DBG] from='client.? 172.21.15.6:0/2271877729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:34.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:34 smithi130 bash[51302]: debug 2023-08-27T23:24:34.424+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:34.429269+0000) 2023-08-27T23:24:35.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:35 smithi130 bash[51302]: debug 2023-08-27T23:24:35.428+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:35.429552+0000) 2023-08-27T23:24:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:36 smithi130 bash[27579]: cluster 2023-08-27T23:24:35.005082+0000 mgr.y (mgr.14901) 2581 : cluster [DBG] pgmap v2588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:36 smithi130 bash[27579]: audit 2023-08-27T23:24:36.352145+0000 mon.c (mon.2) 1011 : audit [DBG] from='client.? 172.21.15.6:0/179006107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:36 smithi130 bash[51302]: debug 2023-08-27T23:24:36.428+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:36.429817+0000) 2023-08-27T23:24:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:36 smithi006 bash[27292]: cluster 2023-08-27T23:24:35.005082+0000 mgr.y (mgr.14901) 2581 : cluster [DBG] pgmap v2588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:36 smithi006 bash[27292]: audit 2023-08-27T23:24:36.352145+0000 mon.c (mon.2) 1011 : audit [DBG] from='client.? 172.21.15.6:0/179006107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:36 smithi006 bash[33918]: cluster 2023-08-27T23:24:35.005082+0000 mgr.y (mgr.14901) 2581 : cluster [DBG] pgmap v2588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:36 smithi006 bash[33918]: audit 2023-08-27T23:24:36.352145+0000 mon.c (mon.2) 1011 : audit [DBG] from='client.? 172.21.15.6:0/179006107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:37.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:37 smithi130 bash[51302]: debug 2023-08-27T23:24:37.424+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:37.430112+0000) 2023-08-27T23:24:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:38 smithi130 bash[27579]: cluster 2023-08-27T23:24:37.005762+0000 mgr.y (mgr.14901) 2582 : cluster [DBG] pgmap v2589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:38 smithi130 bash[51302]: debug 2023-08-27T23:24:38.428+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:38.430406+0000) 2023-08-27T23:24:38.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:38 smithi006 bash[27292]: cluster 2023-08-27T23:24:37.005762+0000 mgr.y (mgr.14901) 2582 : cluster [DBG] pgmap v2589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:38 smithi006 bash[33918]: cluster 2023-08-27T23:24:37.005762+0000 mgr.y (mgr.14901) 2582 : cluster [DBG] pgmap v2589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:39.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:39 smithi130 bash[27579]: audit 2023-08-27T23:24:38.765576+0000 mon.c (mon.2) 1012 : audit [DBG] from='client.? 172.21.15.6:0/2069841131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:39 smithi130 bash[51302]: debug 2023-08-27T23:24:39.428+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:39.430718+0000) 2023-08-27T23:24:39.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:39 smithi006 bash[27292]: audit 2023-08-27T23:24:38.765576+0000 mon.c (mon.2) 1012 : audit [DBG] from='client.? 172.21.15.6:0/2069841131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:39 smithi006 bash[33918]: audit 2023-08-27T23:24:38.765576+0000 mon.c (mon.2) 1012 : audit [DBG] from='client.? 172.21.15.6:0/2069841131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:40 smithi006 bash[27292]: cluster 2023-08-27T23:24:39.006519+0000 mgr.y (mgr.14901) 2583 : cluster [DBG] pgmap v2590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:40 smithi006 bash[33918]: cluster 2023-08-27T23:24:39.006519+0000 mgr.y (mgr.14901) 2583 : cluster [DBG] pgmap v2590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:24:40.457Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:24:40.459Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:40 smithi130 bash[27579]: cluster 2023-08-27T23:24:39.006519+0000 mgr.y (mgr.14901) 2583 : cluster [DBG] pgmap v2590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:40 smithi130 bash[51302]: debug 2023-08-27T23:24:40.428+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:40.431023+0000) 2023-08-27T23:24:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:24:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:24:40] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T23:24:41.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:41 smithi130 bash[27579]: audit 2023-08-27T23:24:41.176333+0000 mon.c (mon.2) 1013 : audit [DBG] from='client.? 172.21.15.6:0/1680741348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:41.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:41 smithi130 bash[51302]: debug 2023-08-27T23:24:41.428+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:41.431278+0000) 2023-08-27T23:24:41.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:41 smithi006 bash[27292]: audit 2023-08-27T23:24:41.176333+0000 mon.c (mon.2) 1013 : audit [DBG] from='client.? 172.21.15.6:0/1680741348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:41.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:41 smithi006 bash[33918]: audit 2023-08-27T23:24:41.176333+0000 mon.c (mon.2) 1013 : audit [DBG] from='client.? 172.21.15.6:0/1680741348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:42.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:24:42.019Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:24:42.023Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:42 smithi130 bash[27579]: cluster 2023-08-27T23:24:41.007655+0000 mgr.y (mgr.14901) 2584 : cluster [DBG] pgmap v2591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:42 smithi130 bash[51302]: debug 2023-08-27T23:24:42.428+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:42.431536+0000) 2023-08-27T23:24:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:42 smithi006 bash[27292]: cluster 2023-08-27T23:24:41.007655+0000 mgr.y (mgr.14901) 2584 : cluster [DBG] pgmap v2591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:42 smithi006 bash[33918]: cluster 2023-08-27T23:24:41.007655+0000 mgr.y (mgr.14901) 2584 : cluster [DBG] pgmap v2591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:43.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:43 smithi130 bash[27579]: audit 2023-08-27T23:24:43.610236+0000 mon.a (mon.0) 2211 : audit [DBG] from='client.? 172.21.15.6:0/2639563178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:43.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:43 smithi130 bash[51302]: debug 2023-08-27T23:24:43.428+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:43.431826+0000) 2023-08-27T23:24:43.928 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:43 smithi006 bash[27292]: audit 2023-08-27T23:24:43.610236+0000 mon.a (mon.0) 2211 : audit [DBG] from='client.? 172.21.15.6:0/2639563178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:43.929 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:43 smithi006 bash[33918]: audit 2023-08-27T23:24:43.610236+0000 mon.a (mon.0) 2211 : audit [DBG] from='client.? 172.21.15.6:0/2639563178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:24:43.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:24:43.928Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:44 smithi130 bash[27579]: cluster 2023-08-27T23:24:43.008478+0000 mgr.y (mgr.14901) 2585 : cluster [DBG] pgmap v2592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:44 smithi130 bash[51302]: debug 2023-08-27T23:24:44.428+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:44.432105+0000) 2023-08-27T23:24:44.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:44 smithi006 bash[27292]: cluster 2023-08-27T23:24:43.008478+0000 mgr.y (mgr.14901) 2585 : cluster [DBG] pgmap v2592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:44 smithi006 bash[33918]: cluster 2023-08-27T23:24:43.008478+0000 mgr.y (mgr.14901) 2585 : cluster [DBG] pgmap v2592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:45.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:45 smithi130 bash[51302]: debug 2023-08-27T23:24:45.428+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:45.432366+0000) 2023-08-27T23:24:46.685 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:46 smithi130 bash[51302]: debug 2023-08-27T23:24:46.428+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:46.432654+0000) 2023-08-27T23:24:46.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:46 smithi006 bash[27292]: cluster 2023-08-27T23:24:45.009771+0000 mgr.y (mgr.14901) 2586 : cluster [DBG] pgmap v2593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:46 smithi006 bash[27292]: audit 2023-08-27T23:24:46.035383+0000 mon.a (mon.0) 2212 : audit [DBG] from='client.? 172.21.15.6:0/3614818762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:46 smithi006 bash[33918]: cluster 2023-08-27T23:24:45.009771+0000 mgr.y (mgr.14901) 2586 : cluster [DBG] pgmap v2593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:46 smithi006 bash[33918]: audit 2023-08-27T23:24:46.035383+0000 mon.a (mon.0) 2212 : audit [DBG] from='client.? 172.21.15.6:0/3614818762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:47.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:46 smithi130 bash[27579]: cluster 2023-08-27T23:24:45.009771+0000 mgr.y (mgr.14901) 2586 : cluster [DBG] pgmap v2593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:47.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:46 smithi130 bash[27579]: audit 2023-08-27T23:24:46.035383+0000 mon.a (mon.0) 2212 : audit [DBG] from='client.? 172.21.15.6:0/3614818762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:47.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:47 smithi130 bash[51302]: debug 2023-08-27T23:24:47.432+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:47.432932+0000) 2023-08-27T23:24:48.690 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:48 smithi130 bash[51302]: debug 2023-08-27T23:24:48.432+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:48.433218+0000) 2023-08-27T23:24:48.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:48 smithi006 bash[27292]: cluster 2023-08-27T23:24:47.010451+0000 mgr.y (mgr.14901) 2587 : cluster [DBG] pgmap v2594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:48 smithi006 bash[27292]: audit 2023-08-27T23:24:48.450413+0000 mon.a (mon.0) 2213 : audit [DBG] from='client.? 172.21.15.6:0/403285920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:48 smithi006 bash[33918]: cluster 2023-08-27T23:24:47.010451+0000 mgr.y (mgr.14901) 2587 : cluster [DBG] pgmap v2594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:48 smithi006 bash[33918]: audit 2023-08-27T23:24:48.450413+0000 mon.a (mon.0) 2213 : audit [DBG] from='client.? 172.21.15.6:0/403285920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:48 smithi130 bash[27579]: cluster 2023-08-27T23:24:47.010451+0000 mgr.y (mgr.14901) 2587 : cluster [DBG] pgmap v2594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:48 smithi130 bash[27579]: audit 2023-08-27T23:24:48.450413+0000 mon.a (mon.0) 2213 : audit [DBG] from='client.? 172.21.15.6:0/403285920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:49.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:49 smithi130 bash[51302]: debug 2023-08-27T23:24:49.428+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:49.433552+0000) 2023-08-27T23:24:50.695 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:50 smithi130 bash[51302]: debug 2023-08-27T23:24:50.428+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:50.433816+0000) 2023-08-27T23:24:50.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:50 smithi006 bash[27292]: cluster 2023-08-27T23:24:49.011133+0000 mgr.y (mgr.14901) 2588 : cluster [DBG] pgmap v2595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:50 smithi006 bash[33918]: cluster 2023-08-27T23:24:49.011133+0000 mgr.y (mgr.14901) 2588 : cluster [DBG] pgmap v2595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:24:50.459Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:24:50.461Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:50 smithi130 bash[27579]: cluster 2023-08-27T23:24:49.011133+0000 mgr.y (mgr.14901) 2588 : cluster [DBG] pgmap v2595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:24:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:24:50] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:24:51.698 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:51 smithi130 bash[51302]: debug 2023-08-27T23:24:51.428+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:51.434117+0000) 2023-08-27T23:24:51.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:51 smithi006 bash[33918]: audit 2023-08-27T23:24:50.867913+0000 mon.c (mon.2) 1014 : audit [DBG] from='client.? 172.21.15.6:0/3135433816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:51.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:51 smithi006 bash[27292]: audit 2023-08-27T23:24:50.867913+0000 mon.c (mon.2) 1014 : audit [DBG] from='client.? 172.21.15.6:0/3135433816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:52.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:51 smithi130 bash[27579]: audit 2023-08-27T23:24:50.867913+0000 mon.c (mon.2) 1014 : audit [DBG] from='client.? 172.21.15.6:0/3135433816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:24:52.020Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:24:52.022Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:52.708 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:52 smithi130 bash[51302]: debug 2023-08-27T23:24:52.428+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:52.434401+0000) 2023-08-27T23:24:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:52 smithi006 bash[27292]: cluster 2023-08-27T23:24:51.012403+0000 mgr.y (mgr.14901) 2589 : cluster [DBG] pgmap v2596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:52 smithi006 bash[27292]: audit 2023-08-27T23:24:52.274576+0000 mon.b (mon.1) 367 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:24:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:52 smithi006 bash[33918]: cluster 2023-08-27T23:24:51.012403+0000 mgr.y (mgr.14901) 2589 : cluster [DBG] pgmap v2596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:52 smithi006 bash[33918]: audit 2023-08-27T23:24:52.274576+0000 mon.b (mon.1) 367 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:24:53.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:52 smithi130 bash[27579]: cluster 2023-08-27T23:24:51.012403+0000 mgr.y (mgr.14901) 2589 : cluster [DBG] pgmap v2596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:52 smithi130 bash[27579]: audit 2023-08-27T23:24:52.274576+0000 mon.b (mon.1) 367 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:24:53.717 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:53 smithi130 bash[51302]: debug 2023-08-27T23:24:53.428+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:53.434737+0000) 2023-08-27T23:24:53.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:53 smithi006 bash[33918]: audit 2023-08-27T23:24:53.280139+0000 mon.c (mon.2) 1015 : audit [DBG] from='client.? 172.21.15.6:0/2121330289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:53.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:53 smithi006 bash[27292]: audit 2023-08-27T23:24:53.280139+0000 mon.c (mon.2) 1015 : audit [DBG] from='client.? 172.21.15.6:0/2121330289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:53.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:24:53.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 9 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:53.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:24:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:24:53.929Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:24:54.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:53 smithi130 bash[27579]: audit 2023-08-27T23:24:53.280139+0000 mon.c (mon.2) 1015 : audit [DBG] from='client.? 172.21.15.6:0/2121330289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:54.718 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:54 smithi130 bash[51302]: debug 2023-08-27T23:24:54.432+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:54.435080+0000) 2023-08-27T23:24:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:54 smithi130 bash[27579]: cluster 2023-08-27T23:24:53.013188+0000 mgr.y (mgr.14901) 2590 : cluster [DBG] pgmap v2597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:54 smithi006 bash[33918]: cluster 2023-08-27T23:24:53.013188+0000 mgr.y (mgr.14901) 2590 : cluster [DBG] pgmap v2597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:54 smithi006 bash[27292]: cluster 2023-08-27T23:24:53.013188+0000 mgr.y (mgr.14901) 2590 : cluster [DBG] pgmap v2597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:55.725 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:55 smithi130 bash[51302]: debug 2023-08-27T23:24:55.432+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:55.435342+0000) 2023-08-27T23:24:56.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:55 smithi130 bash[27579]: audit 2023-08-27T23:24:55.703275+0000 mon.a (mon.0) 2214 : audit [DBG] from='client.? 172.21.15.6:0/1174406945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:56.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:55 smithi006 bash[33918]: audit 2023-08-27T23:24:55.703275+0000 mon.a (mon.0) 2214 : audit [DBG] from='client.? 172.21.15.6:0/1174406945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:56.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:55 smithi006 bash[27292]: audit 2023-08-27T23:24:55.703275+0000 mon.a (mon.0) 2214 : audit [DBG] from='client.? 172.21.15.6:0/1174406945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:56.727 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:56 smithi130 bash[51302]: debug 2023-08-27T23:24:56.432+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:56.435612+0000) 2023-08-27T23:24:57.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:56 smithi130 bash[27579]: cluster 2023-08-27T23:24:55.014309+0000 mgr.y (mgr.14901) 2591 : cluster [DBG] pgmap v2598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:57.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:56 smithi006 bash[33918]: cluster 2023-08-27T23:24:55.014309+0000 mgr.y (mgr.14901) 2591 : cluster [DBG] pgmap v2598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:57.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:56 smithi006 bash[27292]: cluster 2023-08-27T23:24:55.014309+0000 mgr.y (mgr.14901) 2591 : cluster [DBG] pgmap v2598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:57.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:57 smithi130 bash[51302]: debug 2023-08-27T23:24:57.432+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:57.435898+0000) 2023-08-27T23:24:58.732 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:58 smithi130 bash[51302]: debug 2023-08-27T23:24:58.432+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:58.436266+0000) 2023-08-27T23:24:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:58 smithi130 bash[27579]: cluster 2023-08-27T23:24:57.015112+0000 mgr.y (mgr.14901) 2592 : cluster [DBG] pgmap v2599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:58 smithi130 bash[27579]: audit 2023-08-27T23:24:58.164963+0000 mon.c (mon.2) 1016 : audit [DBG] from='client.? 172.21.15.6:0/1155726217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:58 smithi130 bash[27579]: audit 2023-08-27T23:24:58.250158+0000 mon.a (mon.0) 2215 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:24:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:58 smithi130 bash[27579]: audit 2023-08-27T23:24:58.612263+0000 mon.a (mon.0) 2216 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:24:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:58 smithi006 bash[27292]: cluster 2023-08-27T23:24:57.015112+0000 mgr.y (mgr.14901) 2592 : cluster [DBG] pgmap v2599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:58 smithi006 bash[27292]: audit 2023-08-27T23:24:58.164963+0000 mon.c (mon.2) 1016 : audit [DBG] from='client.? 172.21.15.6:0/1155726217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:58 smithi006 bash[27292]: audit 2023-08-27T23:24:58.250158+0000 mon.a (mon.0) 2215 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:24:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:58 smithi006 bash[27292]: audit 2023-08-27T23:24:58.612263+0000 mon.a (mon.0) 2216 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:24:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:58 smithi006 bash[33918]: cluster 2023-08-27T23:24:57.015112+0000 mgr.y (mgr.14901) 2592 : cluster [DBG] pgmap v2599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:24:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:58 smithi006 bash[33918]: audit 2023-08-27T23:24:58.164963+0000 mon.c (mon.2) 1016 : audit [DBG] from='client.? 172.21.15.6:0/1155726217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:58 smithi006 bash[33918]: audit 2023-08-27T23:24:58.250158+0000 mon.a (mon.0) 2215 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:24:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:58 smithi006 bash[33918]: audit 2023-08-27T23:24:58.612263+0000 mon.a (mon.0) 2216 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:24:59.737 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:24:59 smithi130 bash[51302]: debug 2023-08-27T23:24:59.436+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:59.436628+0000) 2023-08-27T23:25:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:59 smithi130 bash[27579]: audit 2023-08-27T23:24:58.883916+0000 mon.b (mon.1) 368 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:25:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:59 smithi130 bash[27579]: audit 2023-08-27T23:24:58.886095+0000 mon.b (mon.1) 369 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:25:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:24:59 smithi130 bash[27579]: audit 2023-08-27T23:24:58.897477+0000 mon.a (mon.0) 2217 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:25:00.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:59 smithi006 bash[33918]: audit 2023-08-27T23:24:58.883916+0000 mon.b (mon.1) 368 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:25:00.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:59 smithi006 bash[33918]: audit 2023-08-27T23:24:58.886095+0000 mon.b (mon.1) 369 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:25:00.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:24:59 smithi006 bash[33918]: audit 2023-08-27T23:24:58.897477+0000 mon.a (mon.0) 2217 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:25:00.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:59 smithi006 bash[27292]: audit 2023-08-27T23:24:58.883916+0000 mon.b (mon.1) 368 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:25:00.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:59 smithi006 bash[27292]: audit 2023-08-27T23:24:58.886095+0000 mon.b (mon.1) 369 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:25:00.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:24:59 smithi006 bash[27292]: audit 2023-08-27T23:24:58.897477+0000 mon.a (mon.0) 2217 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:25:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:25:00.460Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:00.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:25:00.462Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:00.739 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:00 smithi130 bash[51302]: debug 2023-08-27T23:25:00.432+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:00.436980+0000) 2023-08-27T23:25:01.054 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:25:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:25:00] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T23:25:01.055 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:00 smithi006 bash[33918]: cluster 2023-08-27T23:24:59.015778+0000 mgr.y (mgr.14901) 2593 : cluster [DBG] pgmap v2600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:01.055 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:00 smithi006 bash[33918]: audit 2023-08-27T23:25:00.582351+0000 mon.c (mon.2) 1017 : audit [DBG] from='client.? 172.21.15.6:0/1801676578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:01.055 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:00 smithi006 bash[27292]: cluster 2023-08-27T23:24:59.015778+0000 mgr.y (mgr.14901) 2593 : cluster [DBG] pgmap v2600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:01.055 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:00 smithi006 bash[27292]: audit 2023-08-27T23:25:00.582351+0000 mon.c (mon.2) 1017 : audit [DBG] from='client.? 172.21.15.6:0/1801676578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:00 smithi130 bash[27579]: cluster 2023-08-27T23:24:59.015778+0000 mgr.y (mgr.14901) 2593 : cluster [DBG] pgmap v2600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:00 smithi130 bash[27579]: audit 2023-08-27T23:25:00.582351+0000 mon.c (mon.2) 1017 : audit [DBG] from='client.? 172.21.15.6:0/1801676578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:01.844 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:01 smithi130 bash[51302]: debug 2023-08-27T23:25:01.432+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:01.437327+0000) 2023-08-27T23:25:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:25:02.020Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:25:02.022Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:02.747 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:02 smithi130 bash[51302]: debug 2023-08-27T23:25:02.432+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:02.437650+0000) 2023-08-27T23:25:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:02 smithi130 bash[27579]: cluster 2023-08-27T23:25:01.016938+0000 mgr.y (mgr.14901) 2594 : cluster [DBG] pgmap v2601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:02 smithi006 bash[27292]: cluster 2023-08-27T23:25:01.016938+0000 mgr.y (mgr.14901) 2594 : cluster [DBG] pgmap v2601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:02 smithi006 bash[33918]: cluster 2023-08-27T23:25:01.016938+0000 mgr.y (mgr.14901) 2594 : cluster [DBG] pgmap v2601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:03.756 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:03 smithi130 bash[51302]: debug 2023-08-27T23:25:03.436+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:03.437961+0000) 2023-08-27T23:25:04.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:03 smithi130 bash[27579]: audit 2023-08-27T23:25:03.002521+0000 mon.c (mon.2) 1018 : audit [DBG] from='client.? 172.21.15.6:0/1037687091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:04.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:03 smithi006 bash[27292]: audit 2023-08-27T23:25:03.002521+0000 mon.c (mon.2) 1018 : audit [DBG] from='client.? 172.21.15.6:0/1037687091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:04.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:03 smithi006 bash[33918]: audit 2023-08-27T23:25:03.002521+0000 mon.c (mon.2) 1018 : audit [DBG] from='client.? 172.21.15.6:0/1037687091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:04.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:25:03.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:04.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:25:03.930Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:04.759 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:04 smithi130 bash[51302]: debug 2023-08-27T23:25:04.436+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:04.438296+0000) 2023-08-27T23:25:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:04 smithi130 bash[27579]: cluster 2023-08-27T23:25:03.017551+0000 mgr.y (mgr.14901) 2595 : cluster [DBG] pgmap v2602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:05.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:04 smithi006 bash[27292]: cluster 2023-08-27T23:25:03.017551+0000 mgr.y (mgr.14901) 2595 : cluster [DBG] pgmap v2602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:04 smithi006 bash[33918]: cluster 2023-08-27T23:25:03.017551+0000 mgr.y (mgr.14901) 2595 : cluster [DBG] pgmap v2602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:05.770 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:05 smithi130 bash[51302]: debug 2023-08-27T23:25:05.435+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:05.438624+0000) 2023-08-27T23:25:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:05 smithi130 bash[27579]: cluster 2023-08-27T23:25:05.018700+0000 mgr.y (mgr.14901) 2596 : cluster [DBG] pgmap v2603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:05 smithi130 bash[27579]: audit 2023-08-27T23:25:05.419840+0000 mon.a (mon.0) 2218 : audit [DBG] from='client.? 172.21.15.6:0/2177442261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:05 smithi006 bash[33918]: cluster 2023-08-27T23:25:05.018700+0000 mgr.y (mgr.14901) 2596 : cluster [DBG] pgmap v2603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:05 smithi006 bash[33918]: audit 2023-08-27T23:25:05.419840+0000 mon.a (mon.0) 2218 : audit [DBG] from='client.? 172.21.15.6:0/2177442261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:06.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:05 smithi006 bash[27292]: cluster 2023-08-27T23:25:05.018700+0000 mgr.y (mgr.14901) 2596 : cluster [DBG] pgmap v2603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:06.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:05 smithi006 bash[27292]: audit 2023-08-27T23:25:05.419840+0000 mon.a (mon.0) 2218 : audit [DBG] from='client.? 172.21.15.6:0/2177442261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:06.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:06 smithi130 bash[51302]: debug 2023-08-27T23:25:06.435+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:06.438975+0000) 2023-08-27T23:25:07.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:07 smithi130 bash[51302]: debug 2023-08-27T23:25:07.435+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:07.439265+0000) 2023-08-27T23:25:08.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:08 smithi130 bash[27579]: cluster 2023-08-27T23:25:07.019311+0000 mgr.y (mgr.14901) 2597 : cluster [DBG] pgmap v2604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:08.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:08 smithi130 bash[27579]: audit 2023-08-27T23:25:07.847852+0000 mon.a (mon.0) 2219 : audit [DBG] from='client.? 172.21.15.6:0/485790798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:08.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:08 smithi006 bash[27292]: cluster 2023-08-27T23:25:07.019311+0000 mgr.y (mgr.14901) 2597 : cluster [DBG] pgmap v2604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:08.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:08 smithi006 bash[27292]: audit 2023-08-27T23:25:07.847852+0000 mon.a (mon.0) 2219 : audit [DBG] from='client.? 172.21.15.6:0/485790798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:08.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:08 smithi006 bash[33918]: cluster 2023-08-27T23:25:07.019311+0000 mgr.y (mgr.14901) 2597 : cluster [DBG] pgmap v2604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:08.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:08 smithi006 bash[33918]: audit 2023-08-27T23:25:07.847852+0000 mon.a (mon.0) 2219 : audit [DBG] from='client.? 172.21.15.6:0/485790798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:08.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:08 smithi130 bash[51302]: debug 2023-08-27T23:25:08.435+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:08.439564+0000) 2023-08-27T23:25:09.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:09 smithi130 bash[51302]: debug 2023-08-27T23:25:09.435+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:09.439786+0000) 2023-08-27T23:25:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:10 smithi006 bash[27292]: cluster 2023-08-27T23:25:09.019938+0000 mgr.y (mgr.14901) 2598 : cluster [DBG] pgmap v2605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:10 smithi006 bash[27292]: audit 2023-08-27T23:25:10.273118+0000 mon.c (mon.2) 1019 : audit [DBG] from='client.? 172.21.15.6:0/3858163105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:25:10.461Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:25:10.462Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:10 smithi006 bash[33918]: cluster 2023-08-27T23:25:09.019938+0000 mgr.y (mgr.14901) 2598 : cluster [DBG] pgmap v2605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:10 smithi006 bash[33918]: audit 2023-08-27T23:25:10.273118+0000 mon.c (mon.2) 1019 : audit [DBG] from='client.? 172.21.15.6:0/3858163105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:10 smithi130 bash[27579]: cluster 2023-08-27T23:25:09.019938+0000 mgr.y (mgr.14901) 2598 : cluster [DBG] pgmap v2605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:10 smithi130 bash[27579]: audit 2023-08-27T23:25:10.273118+0000 mon.c (mon.2) 1019 : audit [DBG] from='client.? 172.21.15.6:0/3858163105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:10 smithi130 bash[51302]: debug 2023-08-27T23:25:10.435+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:10.440053+0000) 2023-08-27T23:25:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:25:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:25:10] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T23:25:11.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:11 smithi130 bash[51302]: debug 2023-08-27T23:25:11.435+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:11.440313+0000) 2023-08-27T23:25:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:25:12.021Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:25:12.024Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:12 smithi130 bash[27579]: cluster 2023-08-27T23:25:11.021226+0000 mgr.y (mgr.14901) 2599 : cluster [DBG] pgmap v2606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:12 smithi130 bash[51302]: debug 2023-08-27T23:25:12.435+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:12.440549+0000) 2023-08-27T23:25:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:12 smithi006 bash[33918]: cluster 2023-08-27T23:25:11.021226+0000 mgr.y (mgr.14901) 2599 : cluster [DBG] pgmap v2606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:12 smithi006 bash[27292]: cluster 2023-08-27T23:25:11.021226+0000 mgr.y (mgr.14901) 2599 : cluster [DBG] pgmap v2606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:13.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:13 smithi130 bash[27579]: audit 2023-08-27T23:25:12.691987+0000 mon.a (mon.0) 2220 : audit [DBG] from='client.? 172.21.15.6:0/2490341766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:13.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:13 smithi130 bash[51302]: debug 2023-08-27T23:25:13.439+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:13.440800+0000) 2023-08-27T23:25:13.931 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:13 smithi006 bash[27292]: audit 2023-08-27T23:25:12.691987+0000 mon.a (mon.0) 2220 : audit [DBG] from='client.? 172.21.15.6:0/2490341766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:13.931 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:13 smithi006 bash[33918]: audit 2023-08-27T23:25:12.691987+0000 mon.a (mon.0) 2220 : audit [DBG] from='client.? 172.21.15.6:0/2490341766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:25:13.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:25:13.931Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:14 smithi130 bash[27579]: cluster 2023-08-27T23:25:13.021915+0000 mgr.y (mgr.14901) 2600 : cluster [DBG] pgmap v2607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:14 smithi130 bash[51302]: debug 2023-08-27T23:25:14.435+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:14.441110+0000) 2023-08-27T23:25:14.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:14 smithi006 bash[33918]: cluster 2023-08-27T23:25:13.021915+0000 mgr.y (mgr.14901) 2600 : cluster [DBG] pgmap v2607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:14 smithi006 bash[27292]: cluster 2023-08-27T23:25:13.021915+0000 mgr.y (mgr.14901) 2600 : cluster [DBG] pgmap v2607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:15.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:15 smithi130 bash[27579]: audit 2023-08-27T23:25:15.109583+0000 mon.c (mon.2) 1020 : audit [DBG] from='client.? 172.21.15.6:0/4058483037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:15.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:15 smithi130 bash[51302]: debug 2023-08-27T23:25:15.435+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:15.441420+0000) 2023-08-27T23:25:15.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:15 smithi006 bash[27292]: audit 2023-08-27T23:25:15.109583+0000 mon.c (mon.2) 1020 : audit [DBG] from='client.? 172.21.15.6:0/4058483037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:15.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:15 smithi006 bash[33918]: audit 2023-08-27T23:25:15.109583+0000 mon.c (mon.2) 1020 : audit [DBG] from='client.? 172.21.15.6:0/4058483037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:16 smithi130 bash[27579]: cluster 2023-08-27T23:25:15.022985+0000 mgr.y (mgr.14901) 2601 : cluster [DBG] pgmap v2608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:16.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:16 smithi130 bash[51302]: debug 2023-08-27T23:25:16.439+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:16.441727+0000) 2023-08-27T23:25:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:16 smithi006 bash[27292]: cluster 2023-08-27T23:25:15.022985+0000 mgr.y (mgr.14901) 2601 : cluster [DBG] pgmap v2608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:16.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:16 smithi006 bash[33918]: cluster 2023-08-27T23:25:15.022985+0000 mgr.y (mgr.14901) 2601 : cluster [DBG] pgmap v2608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:17.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:17 smithi130 bash[27579]: audit 2023-08-27T23:25:17.520409+0000 mon.a (mon.0) 2221 : audit [DBG] from='client.? 172.21.15.6:0/3549447461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:17.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:17 smithi130 bash[51302]: debug 2023-08-27T23:25:17.439+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:17.442010+0000) 2023-08-27T23:25:17.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:17 smithi006 bash[33918]: audit 2023-08-27T23:25:17.520409+0000 mon.a (mon.0) 2221 : audit [DBG] from='client.? 172.21.15.6:0/3549447461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:17.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:17 smithi006 bash[27292]: audit 2023-08-27T23:25:17.520409+0000 mon.a (mon.0) 2221 : audit [DBG] from='client.? 172.21.15.6:0/3549447461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:18 smithi130 bash[27579]: cluster 2023-08-27T23:25:17.023637+0000 mgr.y (mgr.14901) 2602 : cluster [DBG] pgmap v2609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:18 smithi130 bash[51302]: debug 2023-08-27T23:25:18.435+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:18.442356+0000) 2023-08-27T23:25:18.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:18 smithi006 bash[27292]: cluster 2023-08-27T23:25:17.023637+0000 mgr.y (mgr.14901) 2602 : cluster [DBG] pgmap v2609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:18 smithi006 bash[33918]: cluster 2023-08-27T23:25:17.023637+0000 mgr.y (mgr.14901) 2602 : cluster [DBG] pgmap v2609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:19 smithi130 bash[51302]: debug 2023-08-27T23:25:19.439+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:19.442640+0000) 2023-08-27T23:25:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:20 smithi006 bash[27292]: cluster 2023-08-27T23:25:19.024248+0000 mgr.y (mgr.14901) 2603 : cluster [DBG] pgmap v2610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:20 smithi006 bash[27292]: audit 2023-08-27T23:25:19.932330+0000 mon.a (mon.0) 2222 : audit [DBG] from='client.? 172.21.15.6:0/3987137799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:25:20.461Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:25:20.463Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:20.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:20 smithi006 bash[33918]: cluster 2023-08-27T23:25:19.024248+0000 mgr.y (mgr.14901) 2603 : cluster [DBG] pgmap v2610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:20.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:20 smithi006 bash[33918]: audit 2023-08-27T23:25:19.932330+0000 mon.a (mon.0) 2222 : audit [DBG] from='client.? 172.21.15.6:0/3987137799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:20 smithi130 bash[51302]: debug 2023-08-27T23:25:20.439+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:20.442967+0000) 2023-08-27T23:25:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:20 smithi130 bash[27579]: cluster 2023-08-27T23:25:19.024248+0000 mgr.y (mgr.14901) 2603 : cluster [DBG] pgmap v2610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:20 smithi130 bash[27579]: audit 2023-08-27T23:25:19.932330+0000 mon.a (mon.0) 2222 : audit [DBG] from='client.? 172.21.15.6:0/3987137799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:21.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:25:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:25:20] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:25:21.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:21 smithi130 bash[51302]: debug 2023-08-27T23:25:21.439+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:21.443202+0000) 2023-08-27T23:25:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:25:22.022Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:25:22.024Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:22 smithi130 bash[51302]: debug 2023-08-27T23:25:22.439+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:22.443423+0000) 2023-08-27T23:25:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:22 smithi130 bash[27579]: cluster 2023-08-27T23:25:21.025416+0000 mgr.y (mgr.14901) 2604 : cluster [DBG] pgmap v2611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:22 smithi130 bash[27579]: audit 2023-08-27T23:25:22.350434+0000 mon.a (mon.0) 2223 : audit [DBG] from='client.? 172.21.15.6:0/819576227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:22 smithi006 bash[33918]: cluster 2023-08-27T23:25:21.025416+0000 mgr.y (mgr.14901) 2604 : cluster [DBG] pgmap v2611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:22 smithi006 bash[33918]: audit 2023-08-27T23:25:22.350434+0000 mon.a (mon.0) 2223 : audit [DBG] from='client.? 172.21.15.6:0/819576227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:22 smithi006 bash[27292]: cluster 2023-08-27T23:25:21.025416+0000 mgr.y (mgr.14901) 2604 : cluster [DBG] pgmap v2611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:22 smithi006 bash[27292]: audit 2023-08-27T23:25:22.350434+0000 mon.a (mon.0) 2223 : audit [DBG] from='client.? 172.21.15.6:0/819576227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:23.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:23 smithi130 bash[51302]: debug 2023-08-27T23:25:23.439+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:23.443710+0000) 2023-08-27T23:25:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:25:23.929Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:25:23.931Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:24.697 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:24 smithi130 bash[51302]: debug 2023-08-27T23:25:24.443+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:24.443979+0000) 2023-08-27T23:25:24.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:24 smithi006 bash[33918]: cluster 2023-08-27T23:25:23.026100+0000 mgr.y (mgr.14901) 2605 : cluster [DBG] pgmap v2612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:24 smithi006 bash[27292]: cluster 2023-08-27T23:25:23.026100+0000 mgr.y (mgr.14901) 2605 : cluster [DBG] pgmap v2612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:24 smithi130 bash[27579]: cluster 2023-08-27T23:25:23.026100+0000 mgr.y (mgr.14901) 2605 : cluster [DBG] pgmap v2612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:25.709 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:25 smithi130 bash[51302]: debug 2023-08-27T23:25:25.443+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:25.444283+0000) 2023-08-27T23:25:25.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:25 smithi006 bash[27292]: audit 2023-08-27T23:25:24.770742+0000 mon.a (mon.0) 2224 : audit [DBG] from='client.? 172.21.15.6:0/1758473520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:25.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:25 smithi006 bash[33918]: audit 2023-08-27T23:25:24.770742+0000 mon.a (mon.0) 2224 : audit [DBG] from='client.? 172.21.15.6:0/1758473520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:26.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:25 smithi130 bash[27579]: audit 2023-08-27T23:25:24.770742+0000 mon.a (mon.0) 2224 : audit [DBG] from='client.? 172.21.15.6:0/1758473520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:26.712 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:26 smithi130 bash[51302]: debug 2023-08-27T23:25:26.443+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:26.444618+0000) 2023-08-27T23:25:26.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:26 smithi006 bash[33918]: cluster 2023-08-27T23:25:25.027262+0000 mgr.y (mgr.14901) 2606 : cluster [DBG] pgmap v2613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:26 smithi006 bash[27292]: cluster 2023-08-27T23:25:25.027262+0000 mgr.y (mgr.14901) 2606 : cluster [DBG] pgmap v2613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:26 smithi130 bash[27579]: cluster 2023-08-27T23:25:25.027262+0000 mgr.y (mgr.14901) 2606 : cluster [DBG] pgmap v2613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:27.720 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:27 smithi130 bash[51302]: debug 2023-08-27T23:25:27.443+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:27.444922+0000) 2023-08-27T23:25:28.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:27 smithi130 bash[27579]: audit 2023-08-27T23:25:27.182266+0000 mon.c (mon.2) 1021 : audit [DBG] from='client.? 172.21.15.6:0/3501423100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:28.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:27 smithi006 bash[33918]: audit 2023-08-27T23:25:27.182266+0000 mon.c (mon.2) 1021 : audit [DBG] from='client.? 172.21.15.6:0/3501423100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:28.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:27 smithi006 bash[27292]: audit 2023-08-27T23:25:27.182266+0000 mon.c (mon.2) 1021 : audit [DBG] from='client.? 172.21.15.6:0/3501423100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:28.722 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:28 smithi130 bash[51302]: debug 2023-08-27T23:25:28.439+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:28.445296+0000) 2023-08-27T23:25:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:28 smithi130 bash[27579]: cluster 2023-08-27T23:25:27.027796+0000 mgr.y (mgr.14901) 2607 : cluster [DBG] pgmap v2614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:28 smithi006 bash[33918]: cluster 2023-08-27T23:25:27.027796+0000 mgr.y (mgr.14901) 2607 : cluster [DBG] pgmap v2614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:28 smithi006 bash[27292]: cluster 2023-08-27T23:25:27.027796+0000 mgr.y (mgr.14901) 2607 : cluster [DBG] pgmap v2614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:29.729 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:29 smithi130 bash[51302]: debug 2023-08-27T23:25:29.439+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:29.445630+0000) 2023-08-27T23:25:30.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:29 smithi130 bash[27579]: audit 2023-08-27T23:25:29.595111+0000 mon.a (mon.0) 2225 : audit [DBG] from='client.? 172.21.15.6:0/3057513119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:30.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:29 smithi006 bash[33918]: audit 2023-08-27T23:25:29.595111+0000 mon.a (mon.0) 2225 : audit [DBG] from='client.? 172.21.15.6:0/3057513119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:30.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:29 smithi006 bash[27292]: audit 2023-08-27T23:25:29.595111+0000 mon.a (mon.0) 2225 : audit [DBG] from='client.? 172.21.15.6:0/3057513119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:25:30.462Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:25:30.465Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:30.731 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:30 smithi130 bash[51302]: debug 2023-08-27T23:25:30.439+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:30.445907+0000) 2023-08-27T23:25:31.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:30 smithi130 bash[27579]: cluster 2023-08-27T23:25:29.028445+0000 mgr.y (mgr.14901) 2608 : cluster [DBG] pgmap v2615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:31.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:30 smithi006 bash[27292]: cluster 2023-08-27T23:25:29.028445+0000 mgr.y (mgr.14901) 2608 : cluster [DBG] pgmap v2615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:25:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:25:30] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:25:31.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:30 smithi006 bash[33918]: cluster 2023-08-27T23:25:29.028445+0000 mgr.y (mgr.14901) 2608 : cluster [DBG] pgmap v2615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:31.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:31 smithi130 bash[51302]: debug 2023-08-27T23:25:31.443+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:31.446199+0000) 2023-08-27T23:25:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:25:32.022Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:25:32.024Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:32.737 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:32 smithi130 bash[51302]: debug 2023-08-27T23:25:32.439+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:32.446484+0000) 2023-08-27T23:25:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:32 smithi130 bash[27579]: cluster 2023-08-27T23:25:31.029609+0000 mgr.y (mgr.14901) 2609 : cluster [DBG] pgmap v2616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:32 smithi130 bash[27579]: audit 2023-08-27T23:25:32.006695+0000 mon.a (mon.0) 2226 : audit [DBG] from='client.? 172.21.15.6:0/754800207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:33.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:32 smithi006 bash[33918]: cluster 2023-08-27T23:25:31.029609+0000 mgr.y (mgr.14901) 2609 : cluster [DBG] pgmap v2616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:32 smithi006 bash[33918]: audit 2023-08-27T23:25:32.006695+0000 mon.a (mon.0) 2226 : audit [DBG] from='client.? 172.21.15.6:0/754800207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:32 smithi006 bash[27292]: cluster 2023-08-27T23:25:31.029609+0000 mgr.y (mgr.14901) 2609 : cluster [DBG] pgmap v2616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:32 smithi006 bash[27292]: audit 2023-08-27T23:25:32.006695+0000 mon.a (mon.0) 2226 : audit [DBG] from='client.? 172.21.15.6:0/754800207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:33.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:33 smithi130 bash[51302]: debug 2023-08-27T23:25:33.443+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:33.446763+0000) 2023-08-27T23:25:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:25:33.930Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:25:33.933Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:34.750 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:34 smithi130 bash[51302]: debug 2023-08-27T23:25:34.443+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:34.447122+0000) 2023-08-27T23:25:35.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:34 smithi130 bash[27579]: cluster 2023-08-27T23:25:33.030273+0000 mgr.y (mgr.14901) 2610 : cluster [DBG] pgmap v2617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:35.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:34 smithi130 bash[27579]: audit 2023-08-27T23:25:34.415709+0000 mon.c (mon.2) 1022 : audit [DBG] from='client.? 172.21.15.6:0/778199930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:34 smithi006 bash[33918]: cluster 2023-08-27T23:25:33.030273+0000 mgr.y (mgr.14901) 2610 : cluster [DBG] pgmap v2617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:34 smithi006 bash[33918]: audit 2023-08-27T23:25:34.415709+0000 mon.c (mon.2) 1022 : audit [DBG] from='client.? 172.21.15.6:0/778199930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:34 smithi006 bash[27292]: cluster 2023-08-27T23:25:33.030273+0000 mgr.y (mgr.14901) 2610 : cluster [DBG] pgmap v2617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:34 smithi006 bash[27292]: audit 2023-08-27T23:25:34.415709+0000 mon.c (mon.2) 1022 : audit [DBG] from='client.? 172.21.15.6:0/778199930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:35.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:35 smithi130 bash[51302]: debug 2023-08-27T23:25:35.443+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:35.447401+0000) 2023-08-27T23:25:36.755 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:36 smithi130 bash[51302]: debug 2023-08-27T23:25:36.443+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:36.447687+0000) 2023-08-27T23:25:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:36 smithi130 bash[27579]: cluster 2023-08-27T23:25:35.031430+0000 mgr.y (mgr.14901) 2611 : cluster [DBG] pgmap v2618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:37.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:36 smithi006 bash[33918]: cluster 2023-08-27T23:25:35.031430+0000 mgr.y (mgr.14901) 2611 : cluster [DBG] pgmap v2618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:37.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:36 smithi006 bash[27292]: cluster 2023-08-27T23:25:35.031430+0000 mgr.y (mgr.14901) 2611 : cluster [DBG] pgmap v2618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:37.762 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:37 smithi130 bash[51302]: debug 2023-08-27T23:25:37.443+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:37.447967+0000) 2023-08-27T23:25:38.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:37 smithi130 bash[27579]: audit 2023-08-27T23:25:36.831009+0000 mon.c (mon.2) 1023 : audit [DBG] from='client.? 172.21.15.6:0/1036881221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:38.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:37 smithi006 bash[33918]: audit 2023-08-27T23:25:36.831009+0000 mon.c (mon.2) 1023 : audit [DBG] from='client.? 172.21.15.6:0/1036881221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:38.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:37 smithi006 bash[27292]: audit 2023-08-27T23:25:36.831009+0000 mon.c (mon.2) 1023 : audit [DBG] from='client.? 172.21.15.6:0/1036881221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:38.767 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:38 smithi130 bash[51302]: debug 2023-08-27T23:25:38.447+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:38.448260+0000) 2023-08-27T23:25:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:38 smithi130 bash[27579]: cluster 2023-08-27T23:25:37.032104+0000 mgr.y (mgr.14901) 2612 : cluster [DBG] pgmap v2619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:39.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:38 smithi006 bash[33918]: cluster 2023-08-27T23:25:37.032104+0000 mgr.y (mgr.14901) 2612 : cluster [DBG] pgmap v2619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:39.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:38 smithi006 bash[27292]: cluster 2023-08-27T23:25:37.032104+0000 mgr.y (mgr.14901) 2612 : cluster [DBG] pgmap v2619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:39.777 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:39 smithi130 bash[51302]: debug 2023-08-27T23:25:39.447+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:39.448575+0000) 2023-08-27T23:25:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:39 smithi130 bash[27579]: cluster 2023-08-27T23:25:39.032749+0000 mgr.y (mgr.14901) 2613 : cluster [DBG] pgmap v2620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:39 smithi130 bash[27579]: audit 2023-08-27T23:25:39.247123+0000 mon.a (mon.0) 2227 : audit [DBG] from='client.? 172.21.15.6:0/2144441419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:40.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:39 smithi006 bash[27292]: cluster 2023-08-27T23:25:39.032749+0000 mgr.y (mgr.14901) 2613 : cluster [DBG] pgmap v2620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:40.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:39 smithi006 bash[27292]: audit 2023-08-27T23:25:39.247123+0000 mon.a (mon.0) 2227 : audit [DBG] from='client.? 172.21.15.6:0/2144441419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:40.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:39 smithi006 bash[33918]: cluster 2023-08-27T23:25:39.032749+0000 mgr.y (mgr.14901) 2613 : cluster [DBG] pgmap v2620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:40.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:39 smithi006 bash[33918]: audit 2023-08-27T23:25:39.247123+0000 mon.a (mon.0) 2227 : audit [DBG] from='client.? 172.21.15.6:0/2144441419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:25:40.463Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:25:40.465Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:40 smithi130 bash[51302]: debug 2023-08-27T23:25:40.443+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:40.448833+0000) 2023-08-27T23:25:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:25:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:25:40] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:25:41.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:41 smithi130 bash[51302]: debug 2023-08-27T23:25:41.447+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:41.449116+0000) 2023-08-27T23:25:42.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:42 smithi130 bash[27579]: cluster 2023-08-27T23:25:41.033906+0000 mgr.y (mgr.14901) 2614 : cluster [DBG] pgmap v2621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:42 smithi130 bash[27579]: audit 2023-08-27T23:25:41.656689+0000 mon.a (mon.0) 2228 : audit [DBG] from='client.? 172.21.15.6:0/2837784345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:42.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:42 smithi006 bash[33918]: cluster 2023-08-27T23:25:41.033906+0000 mgr.y (mgr.14901) 2614 : cluster [DBG] pgmap v2621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:42.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:42 smithi006 bash[33918]: audit 2023-08-27T23:25:41.656689+0000 mon.a (mon.0) 2228 : audit [DBG] from='client.? 172.21.15.6:0/2837784345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:42.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:42 smithi006 bash[27292]: cluster 2023-08-27T23:25:41.033906+0000 mgr.y (mgr.14901) 2614 : cluster [DBG] pgmap v2621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:42.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:42 smithi006 bash[27292]: audit 2023-08-27T23:25:41.656689+0000 mon.a (mon.0) 2228 : audit [DBG] from='client.? 172.21.15.6:0/2837784345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:25:42.023Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:25:42.026Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:42 smithi130 bash[51302]: debug 2023-08-27T23:25:42.447+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:42.449423+0000) 2023-08-27T23:25:43.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:43 smithi130 bash[51302]: debug 2023-08-27T23:25:43.443+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:43.449774+0000) 2023-08-27T23:25:44.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:44 smithi006 bash[33918]: cluster 2023-08-27T23:25:43.034641+0000 mgr.y (mgr.14901) 2615 : cluster [DBG] pgmap v2622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:44.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:44 smithi006 bash[33918]: audit 2023-08-27T23:25:44.074698+0000 mon.c (mon.2) 1024 : audit [DBG] from='client.? 172.21.15.6:0/1582745124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:44.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:44 smithi006 bash[27292]: cluster 2023-08-27T23:25:43.034641+0000 mgr.y (mgr.14901) 2615 : cluster [DBG] pgmap v2622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:44.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:44 smithi006 bash[27292]: audit 2023-08-27T23:25:44.074698+0000 mon.c (mon.2) 1024 : audit [DBG] from='client.? 172.21.15.6:0/1582745124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:25:43.931Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:44.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:25:43.933Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:44 smithi130 bash[27579]: cluster 2023-08-27T23:25:43.034641+0000 mgr.y (mgr.14901) 2615 : cluster [DBG] pgmap v2622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:44 smithi130 bash[27579]: audit 2023-08-27T23:25:44.074698+0000 mon.c (mon.2) 1024 : audit [DBG] from='client.? 172.21.15.6:0/1582745124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:44.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:44 smithi130 bash[51302]: debug 2023-08-27T23:25:44.447+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:44.450090+0000) 2023-08-27T23:25:45.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:45 smithi130 bash[51302]: debug 2023-08-27T23:25:45.447+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:45.450374+0000) 2023-08-27T23:25:46.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:46 smithi130 bash[51302]: debug 2023-08-27T23:25:46.447+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:46.450637+0000) 2023-08-27T23:25:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:46 smithi130 bash[27579]: cluster 2023-08-27T23:25:45.035777+0000 mgr.y (mgr.14901) 2616 : cluster [DBG] pgmap v2623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:46 smithi130 bash[27579]: audit 2023-08-27T23:25:46.493261+0000 mon.a (mon.0) 2229 : audit [DBG] from='client.? 172.21.15.6:0/2826129105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:46 smithi006 bash[33918]: cluster 2023-08-27T23:25:45.035777+0000 mgr.y (mgr.14901) 2616 : cluster [DBG] pgmap v2623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:46 smithi006 bash[33918]: audit 2023-08-27T23:25:46.493261+0000 mon.a (mon.0) 2229 : audit [DBG] from='client.? 172.21.15.6:0/2826129105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:46 smithi006 bash[27292]: cluster 2023-08-27T23:25:45.035777+0000 mgr.y (mgr.14901) 2616 : cluster [DBG] pgmap v2623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:46 smithi006 bash[27292]: audit 2023-08-27T23:25:46.493261+0000 mon.a (mon.0) 2229 : audit [DBG] from='client.? 172.21.15.6:0/2826129105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:47.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:47 smithi130 bash[51302]: debug 2023-08-27T23:25:47.447+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:47.450981+0000) 2023-08-27T23:25:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:48 smithi130 bash[27579]: cluster 2023-08-27T23:25:47.036464+0000 mgr.y (mgr.14901) 2617 : cluster [DBG] pgmap v2624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:48 smithi130 bash[51302]: debug 2023-08-27T23:25:48.447+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:48.451277+0000) 2023-08-27T23:25:48.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:48 smithi006 bash[27292]: cluster 2023-08-27T23:25:47.036464+0000 mgr.y (mgr.14901) 2617 : cluster [DBG] pgmap v2624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:48 smithi006 bash[33918]: cluster 2023-08-27T23:25:47.036464+0000 mgr.y (mgr.14901) 2617 : cluster [DBG] pgmap v2624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:49.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:49 smithi130 bash[27579]: audit 2023-08-27T23:25:48.912170+0000 mon.c (mon.2) 1025 : audit [DBG] from='client.? 172.21.15.6:0/2753865875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:49.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:49 smithi130 bash[51302]: debug 2023-08-27T23:25:49.447+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:49.451568+0000) 2023-08-27T23:25:49.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:49 smithi006 bash[27292]: audit 2023-08-27T23:25:48.912170+0000 mon.c (mon.2) 1025 : audit [DBG] from='client.? 172.21.15.6:0/2753865875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:49.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:49 smithi006 bash[33918]: audit 2023-08-27T23:25:48.912170+0000 mon.c (mon.2) 1025 : audit [DBG] from='client.? 172.21.15.6:0/2753865875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:50.754 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:50 smithi006 bash[33918]: cluster 2023-08-27T23:25:49.037120+0000 mgr.y (mgr.14901) 2618 : cluster [DBG] pgmap v2625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:50.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:50 smithi006 bash[27292]: cluster 2023-08-27T23:25:49.037120+0000 mgr.y (mgr.14901) 2618 : cluster [DBG] pgmap v2625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:50.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:25:50.464Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:50.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:25:50.466Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:50 smithi130 bash[27579]: cluster 2023-08-27T23:25:49.037120+0000 mgr.y (mgr.14901) 2618 : cluster [DBG] pgmap v2625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:50 smithi130 bash[51302]: debug 2023-08-27T23:25:50.447+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:50.451867+0000) 2023-08-27T23:25:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:25:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:25:50] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T23:25:51.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:51 smithi130 bash[27579]: audit 2023-08-27T23:25:51.333046+0000 mon.a (mon.0) 2230 : audit [DBG] from='client.? 172.21.15.6:0/2504484272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:51.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:51 smithi130 bash[51302]: debug 2023-08-27T23:25:51.447+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:51.452130+0000) 2023-08-27T23:25:51.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:51 smithi006 bash[33918]: audit 2023-08-27T23:25:51.333046+0000 mon.a (mon.0) 2230 : audit [DBG] from='client.? 172.21.15.6:0/2504484272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:51.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:51 smithi006 bash[27292]: audit 2023-08-27T23:25:51.333046+0000 mon.a (mon.0) 2230 : audit [DBG] from='client.? 172.21.15.6:0/2504484272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:25:52.025Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:25:52.027Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:52 smithi130 bash[27579]: cluster 2023-08-27T23:25:51.038315+0000 mgr.y (mgr.14901) 2619 : cluster [DBG] pgmap v2626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:52.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:52 smithi130 bash[51302]: debug 2023-08-27T23:25:52.447+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:52.452407+0000) 2023-08-27T23:25:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:52 smithi006 bash[33918]: cluster 2023-08-27T23:25:51.038315+0000 mgr.y (mgr.14901) 2619 : cluster [DBG] pgmap v2626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:52 smithi006 bash[27292]: cluster 2023-08-27T23:25:51.038315+0000 mgr.y (mgr.14901) 2619 : cluster [DBG] pgmap v2626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:53.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:53 smithi130 bash[51302]: debug 2023-08-27T23:25:53.451+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:53.452712+0000) 2023-08-27T23:25:54.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:25:53.932Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:25:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:25:53.933Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:25:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:54 smithi130 bash[27579]: cluster 2023-08-27T23:25:53.038965+0000 mgr.y (mgr.14901) 2620 : cluster [DBG] pgmap v2627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:54 smithi130 bash[27579]: audit 2023-08-27T23:25:53.748785+0000 mon.a (mon.0) 2231 : audit [DBG] from='client.? 172.21.15.6:0/2710812641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:54.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:54 smithi130 bash[51302]: debug 2023-08-27T23:25:54.451+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:54.453013+0000) 2023-08-27T23:25:54.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:54 smithi006 bash[27292]: cluster 2023-08-27T23:25:53.038965+0000 mgr.y (mgr.14901) 2620 : cluster [DBG] pgmap v2627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:54 smithi006 bash[27292]: audit 2023-08-27T23:25:53.748785+0000 mon.a (mon.0) 2231 : audit [DBG] from='client.? 172.21.15.6:0/2710812641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:54 smithi006 bash[33918]: cluster 2023-08-27T23:25:53.038965+0000 mgr.y (mgr.14901) 2620 : cluster [DBG] pgmap v2627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:54.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:54 smithi006 bash[33918]: audit 2023-08-27T23:25:53.748785+0000 mon.a (mon.0) 2231 : audit [DBG] from='client.? 172.21.15.6:0/2710812641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:55.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:55 smithi130 bash[51302]: debug 2023-08-27T23:25:55.451+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:55.453291+0000) 2023-08-27T23:25:56.705 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:56 smithi130 bash[51302]: debug 2023-08-27T23:25:56.447+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:56.453634+0000) 2023-08-27T23:25:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:56 smithi006 bash[27292]: cluster 2023-08-27T23:25:55.040109+0000 mgr.y (mgr.14901) 2621 : cluster [DBG] pgmap v2628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:56 smithi006 bash[27292]: audit 2023-08-27T23:25:56.163358+0000 mon.a (mon.0) 2232 : audit [DBG] from='client.? 172.21.15.6:0/1112757947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:56 smithi006 bash[33918]: cluster 2023-08-27T23:25:55.040109+0000 mgr.y (mgr.14901) 2621 : cluster [DBG] pgmap v2628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:56 smithi006 bash[33918]: audit 2023-08-27T23:25:56.163358+0000 mon.a (mon.0) 2232 : audit [DBG] from='client.? 172.21.15.6:0/1112757947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:56 smithi130 bash[27579]: cluster 2023-08-27T23:25:55.040109+0000 mgr.y (mgr.14901) 2621 : cluster [DBG] pgmap v2628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:57.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:56 smithi130 bash[27579]: audit 2023-08-27T23:25:56.163358+0000 mon.a (mon.0) 2232 : audit [DBG] from='client.? 172.21.15.6:0/1112757947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:57.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:57 smithi130 bash[51302]: debug 2023-08-27T23:25:57.451+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:57.453918+0000) 2023-08-27T23:25:58.710 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:58 smithi130 bash[51302]: debug 2023-08-27T23:25:58.451+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:58.454230+0000) 2023-08-27T23:25:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:58 smithi006 bash[27292]: cluster 2023-08-27T23:25:57.040751+0000 mgr.y (mgr.14901) 2622 : cluster [DBG] pgmap v2629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:58 smithi006 bash[27292]: audit 2023-08-27T23:25:58.577426+0000 mon.a (mon.0) 2233 : audit [DBG] from='client.? 172.21.15.6:0/1355903451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:58 smithi006 bash[33918]: cluster 2023-08-27T23:25:57.040751+0000 mgr.y (mgr.14901) 2622 : cluster [DBG] pgmap v2629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:58 smithi006 bash[33918]: audit 2023-08-27T23:25:58.577426+0000 mon.a (mon.0) 2233 : audit [DBG] from='client.? 172.21.15.6:0/1355903451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:58 smithi130 bash[27579]: cluster 2023-08-27T23:25:57.040751+0000 mgr.y (mgr.14901) 2622 : cluster [DBG] pgmap v2629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:25:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:58 smithi130 bash[27579]: audit 2023-08-27T23:25:58.577426+0000 mon.a (mon.0) 2233 : audit [DBG] from='client.? 172.21.15.6:0/1355903451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:59.722 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:25:59 smithi130 bash[51302]: debug 2023-08-27T23:25:59.447+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:59.454491+0000) 2023-08-27T23:26:00.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:25:59 smithi130 bash[27579]: audit 2023-08-27T23:25:58.909664+0000 mon.b (mon.1) 370 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:26:00.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:25:59 smithi006 bash[27292]: audit 2023-08-27T23:25:58.909664+0000 mon.b (mon.1) 370 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:26:00.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:25:59 smithi006 bash[33918]: audit 2023-08-27T23:25:58.909664+0000 mon.b (mon.1) 370 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:26:00.725 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:26:00.466Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:00.725 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:26:00.468Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:00.725 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:00 smithi006 bash[33918]: cluster 2023-08-27T23:25:59.041459+0000 mgr.y (mgr.14901) 2623 : cluster [DBG] pgmap v2630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:00.726 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:00 smithi006 bash[27292]: cluster 2023-08-27T23:25:59.041459+0000 mgr.y (mgr.14901) 2623 : cluster [DBG] pgmap v2630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:00.726 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:00 smithi130 bash[51302]: debug 2023-08-27T23:26:00.451+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:00.454770+0000) 2023-08-27T23:26:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:00 smithi130 bash[27579]: cluster 2023-08-27T23:25:59.041459+0000 mgr.y (mgr.14901) 2623 : cluster [DBG] pgmap v2630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:26:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:26:00] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:26:01.731 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:01 smithi130 bash[51302]: debug 2023-08-27T23:26:01.451+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:01.455093+0000) 2023-08-27T23:26:02.028 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:01 smithi006 bash[33918]: audit 2023-08-27T23:26:00.993369+0000 mon.a (mon.0) 2234 : audit [DBG] from='client.? 172.21.15.6:0/3408147353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:02.029 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:01 smithi006 bash[27292]: audit 2023-08-27T23:26:00.993369+0000 mon.a (mon.0) 2234 : audit [DBG] from='client.? 172.21.15.6:0/3408147353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:02.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:01 smithi130 bash[27579]: audit 2023-08-27T23:26:00.993369+0000 mon.a (mon.0) 2234 : audit [DBG] from='client.? 172.21.15.6:0/3408147353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:26:02.026Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:26:02.028Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:02.735 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:02 smithi130 bash[51302]: debug 2023-08-27T23:26:02.450+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:02.455424+0000) 2023-08-27T23:26:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:02 smithi130 bash[27579]: cluster 2023-08-27T23:26:01.042532+0000 mgr.y (mgr.14901) 2624 : cluster [DBG] pgmap v2631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:03.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:02 smithi006 bash[27292]: cluster 2023-08-27T23:26:01.042532+0000 mgr.y (mgr.14901) 2624 : cluster [DBG] pgmap v2631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:02 smithi006 bash[33918]: cluster 2023-08-27T23:26:01.042532+0000 mgr.y (mgr.14901) 2624 : cluster [DBG] pgmap v2631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:03.744 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:03 smithi130 bash[51302]: debug 2023-08-27T23:26:03.454+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:03.455765+0000) 2023-08-27T23:26:04.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:03 smithi130 bash[27579]: audit 2023-08-27T23:26:03.418116+0000 mon.c (mon.2) 1026 : audit [DBG] from='client.? 172.21.15.6:0/2134776475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:04.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:03 smithi006 bash[33918]: audit 2023-08-27T23:26:03.418116+0000 mon.c (mon.2) 1026 : audit [DBG] from='client.? 172.21.15.6:0/2134776475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:04.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:03 smithi006 bash[27292]: audit 2023-08-27T23:26:03.418116+0000 mon.c (mon.2) 1026 : audit [DBG] from='client.? 172.21.15.6:0/2134776475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:26:03.933Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:26:03.935Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:04.746 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:04 smithi130 bash[51302]: debug 2023-08-27T23:26:04.450+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:04.456025+0000) 2023-08-27T23:26:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:04 smithi130 bash[27579]: cluster 2023-08-27T23:26:03.043219+0000 mgr.y (mgr.14901) 2625 : cluster [DBG] pgmap v2632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:05.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:04 smithi006 bash[33918]: cluster 2023-08-27T23:26:03.043219+0000 mgr.y (mgr.14901) 2625 : cluster [DBG] pgmap v2632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:04 smithi006 bash[27292]: cluster 2023-08-27T23:26:03.043219+0000 mgr.y (mgr.14901) 2625 : cluster [DBG] pgmap v2632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:05.846 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:05 smithi130 bash[51302]: debug 2023-08-27T23:26:05.450+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:05.456279+0000) 2023-08-27T23:26:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:05 smithi130 bash[27579]: audit 2023-08-27T23:26:04.837968+0000 mon.a (mon.0) 2235 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:26:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:05 smithi130 bash[27579]: cluster 2023-08-27T23:26:05.044020+0000 mgr.y (mgr.14901) 2626 : cluster [DBG] pgmap v2633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:05 smithi130 bash[27579]: audit 2023-08-27T23:26:05.246798+0000 mon.a (mon.0) 2236 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:26:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:05 smithi130 bash[27579]: audit 2023-08-27T23:26:05.515865+0000 mon.b (mon.1) 371 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:26:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:05 smithi130 bash[27579]: audit 2023-08-27T23:26:05.517976+0000 mon.b (mon.1) 372 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:26:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:05 smithi130 bash[27579]: audit 2023-08-27T23:26:05.525657+0000 mon.a (mon.0) 2237 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:26:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:05 smithi006 bash[33918]: audit 2023-08-27T23:26:04.837968+0000 mon.a (mon.0) 2235 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:26:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:05 smithi006 bash[33918]: cluster 2023-08-27T23:26:05.044020+0000 mgr.y (mgr.14901) 2626 : cluster [DBG] pgmap v2633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:05 smithi006 bash[33918]: audit 2023-08-27T23:26:05.246798+0000 mon.a (mon.0) 2236 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:26:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:05 smithi006 bash[33918]: audit 2023-08-27T23:26:05.515865+0000 mon.b (mon.1) 371 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:26:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:05 smithi006 bash[33918]: audit 2023-08-27T23:26:05.517976+0000 mon.b (mon.1) 372 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:26:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:05 smithi006 bash[33918]: audit 2023-08-27T23:26:05.525657+0000 mon.a (mon.0) 2237 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:26:06.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:05 smithi006 bash[27292]: audit 2023-08-27T23:26:04.837968+0000 mon.a (mon.0) 2235 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:26:06.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:05 smithi006 bash[27292]: cluster 2023-08-27T23:26:05.044020+0000 mgr.y (mgr.14901) 2626 : cluster [DBG] pgmap v2633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:06.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:05 smithi006 bash[27292]: audit 2023-08-27T23:26:05.246798+0000 mon.a (mon.0) 2236 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:26:06.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:05 smithi006 bash[27292]: audit 2023-08-27T23:26:05.515865+0000 mon.b (mon.1) 371 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:26:06.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:05 smithi006 bash[27292]: audit 2023-08-27T23:26:05.517976+0000 mon.b (mon.1) 372 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:26:06.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:05 smithi006 bash[27292]: audit 2023-08-27T23:26:05.525657+0000 mon.a (mon.0) 2237 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:26:06.849 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:06 smithi130 bash[51302]: debug 2023-08-27T23:26:06.454+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:06.456560+0000) 2023-08-27T23:26:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:06 smithi130 bash[27579]: audit 2023-08-27T23:26:05.837698+0000 mon.a (mon.0) 2238 : audit [DBG] from='client.? 172.21.15.6:0/505248855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:07.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:06 smithi006 bash[33918]: audit 2023-08-27T23:26:05.837698+0000 mon.a (mon.0) 2238 : audit [DBG] from='client.? 172.21.15.6:0/505248855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:07.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:06 smithi006 bash[27292]: audit 2023-08-27T23:26:05.837698+0000 mon.a (mon.0) 2238 : audit [DBG] from='client.? 172.21.15.6:0/505248855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:07.857 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:07 smithi130 bash[51302]: debug 2023-08-27T23:26:07.450+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:07.456846+0000) 2023-08-27T23:26:08.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:07 smithi130 bash[27579]: cluster 2023-08-27T23:26:07.044688+0000 mgr.y (mgr.14901) 2627 : cluster [DBG] pgmap v2634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:08.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:07 smithi006 bash[33918]: cluster 2023-08-27T23:26:07.044688+0000 mgr.y (mgr.14901) 2627 : cluster [DBG] pgmap v2634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:08.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:07 smithi006 bash[27292]: cluster 2023-08-27T23:26:07.044688+0000 mgr.y (mgr.14901) 2627 : cluster [DBG] pgmap v2634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:08.860 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:08 smithi130 bash[51302]: debug 2023-08-27T23:26:08.454+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:08.457136+0000) 2023-08-27T23:26:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:08 smithi130 bash[27579]: audit 2023-08-27T23:26:08.257547+0000 mon.a (mon.0) 2239 : audit [DBG] from='client.? 172.21.15.6:0/2992371020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:08 smithi006 bash[33918]: audit 2023-08-27T23:26:08.257547+0000 mon.a (mon.0) 2239 : audit [DBG] from='client.? 172.21.15.6:0/2992371020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:08 smithi006 bash[27292]: audit 2023-08-27T23:26:08.257547+0000 mon.a (mon.0) 2239 : audit [DBG] from='client.? 172.21.15.6:0/2992371020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:09.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:09 smithi130 bash[51302]: debug 2023-08-27T23:26:09.454+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:09.457447+0000) 2023-08-27T23:26:10.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:10 smithi006 bash[27292]: cluster 2023-08-27T23:26:09.045376+0000 mgr.y (mgr.14901) 2628 : cluster [DBG] pgmap v2635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:10.756 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:10 smithi006 bash[33918]: cluster 2023-08-27T23:26:09.045376+0000 mgr.y (mgr.14901) 2628 : cluster [DBG] pgmap v2635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:10.756 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:26:10.467Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:10.756 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:26:10.468Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:10 smithi130 bash[27579]: cluster 2023-08-27T23:26:09.045376+0000 mgr.y (mgr.14901) 2628 : cluster [DBG] pgmap v2635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:10 smithi130 bash[51302]: debug 2023-08-27T23:26:10.454+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:10.457748+0000) 2023-08-27T23:26:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:26:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:26:10] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:26:11.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:11 smithi130 bash[27579]: audit 2023-08-27T23:26:10.687660+0000 mon.a (mon.0) 2240 : audit [DBG] from='client.? 172.21.15.6:0/4065961995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:11.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:11 smithi130 bash[51302]: debug 2023-08-27T23:26:11.454+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:11.458038+0000) 2023-08-27T23:26:11.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:11 smithi006 bash[27292]: audit 2023-08-27T23:26:10.687660+0000 mon.a (mon.0) 2240 : audit [DBG] from='client.? 172.21.15.6:0/4065961995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:11.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:11 smithi006 bash[33918]: audit 2023-08-27T23:26:10.687660+0000 mon.a (mon.0) 2240 : audit [DBG] from='client.? 172.21.15.6:0/4065961995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:26:12.027Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:26:12.029Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:12 smithi130 bash[27579]: cluster 2023-08-27T23:26:11.046518+0000 mgr.y (mgr.14901) 2629 : cluster [DBG] pgmap v2636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:12 smithi130 bash[51302]: debug 2023-08-27T23:26:12.454+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:12.458318+0000) 2023-08-27T23:26:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:12 smithi006 bash[33918]: cluster 2023-08-27T23:26:11.046518+0000 mgr.y (mgr.14901) 2629 : cluster [DBG] pgmap v2636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:12 smithi006 bash[27292]: cluster 2023-08-27T23:26:11.046518+0000 mgr.y (mgr.14901) 2629 : cluster [DBG] pgmap v2636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:13.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:13 smithi130 bash[27579]: audit 2023-08-27T23:26:13.103678+0000 mon.a (mon.0) 2241 : audit [DBG] from='client.? 172.21.15.6:0/2634303404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:13.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:13 smithi130 bash[51302]: debug 2023-08-27T23:26:13.454+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:13.458597+0000) 2023-08-27T23:26:13.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:13 smithi006 bash[33918]: audit 2023-08-27T23:26:13.103678+0000 mon.a (mon.0) 2241 : audit [DBG] from='client.? 172.21.15.6:0/2634303404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:13.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:13 smithi006 bash[27292]: audit 2023-08-27T23:26:13.103678+0000 mon.a (mon.0) 2241 : audit [DBG] from='client.? 172.21.15.6:0/2634303404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:13.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:26:13.933Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:13.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:26:13.936Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:14 smithi130 bash[27579]: cluster 2023-08-27T23:26:13.047009+0000 mgr.y (mgr.14901) 2630 : cluster [DBG] pgmap v2637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:14 smithi130 bash[51302]: debug 2023-08-27T23:26:14.458+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:14.458881+0000) 2023-08-27T23:26:14.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:14 smithi006 bash[27292]: cluster 2023-08-27T23:26:13.047009+0000 mgr.y (mgr.14901) 2630 : cluster [DBG] pgmap v2637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:14 smithi006 bash[33918]: cluster 2023-08-27T23:26:13.047009+0000 mgr.y (mgr.14901) 2630 : cluster [DBG] pgmap v2637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:15.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:15 smithi130 bash[27579]: audit 2023-08-27T23:26:15.524714+0000 mon.a (mon.0) 2242 : audit [DBG] from='client.? 172.21.15.6:0/2112175173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:15.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:15 smithi130 bash[51302]: debug 2023-08-27T23:26:15.458+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:15.459146+0000) 2023-08-27T23:26:15.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:15 smithi006 bash[33918]: audit 2023-08-27T23:26:15.524714+0000 mon.a (mon.0) 2242 : audit [DBG] from='client.? 172.21.15.6:0/2112175173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:15.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:15 smithi006 bash[27292]: audit 2023-08-27T23:26:15.524714+0000 mon.a (mon.0) 2242 : audit [DBG] from='client.? 172.21.15.6:0/2112175173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:16.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:16 smithi130 bash[51302]: debug 2023-08-27T23:26:16.458+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:16.459407+0000) 2023-08-27T23:26:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:16 smithi130 bash[27579]: cluster 2023-08-27T23:26:15.048131+0000 mgr.y (mgr.14901) 2631 : cluster [DBG] pgmap v2638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:16 smithi006 bash[27292]: cluster 2023-08-27T23:26:15.048131+0000 mgr.y (mgr.14901) 2631 : cluster [DBG] pgmap v2638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:16 smithi006 bash[33918]: cluster 2023-08-27T23:26:15.048131+0000 mgr.y (mgr.14901) 2631 : cluster [DBG] pgmap v2638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:17.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:17 smithi130 bash[51302]: debug 2023-08-27T23:26:17.458+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:17.459686+0000) 2023-08-27T23:26:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:18 smithi130 bash[51302]: debug 2023-08-27T23:26:18.458+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:18.459973+0000) 2023-08-27T23:26:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:18 smithi130 bash[27579]: cluster 2023-08-27T23:26:17.048796+0000 mgr.y (mgr.14901) 2632 : cluster [DBG] pgmap v2639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:18 smithi130 bash[27579]: audit 2023-08-27T23:26:17.932451+0000 mon.c (mon.2) 1027 : audit [DBG] from='client.? 172.21.15.6:0/1744692204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:18 smithi006 bash[27292]: cluster 2023-08-27T23:26:17.048796+0000 mgr.y (mgr.14901) 2632 : cluster [DBG] pgmap v2639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:18 smithi006 bash[27292]: audit 2023-08-27T23:26:17.932451+0000 mon.c (mon.2) 1027 : audit [DBG] from='client.? 172.21.15.6:0/1744692204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:18 smithi006 bash[33918]: cluster 2023-08-27T23:26:17.048796+0000 mgr.y (mgr.14901) 2632 : cluster [DBG] pgmap v2639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:18 smithi006 bash[33918]: audit 2023-08-27T23:26:17.932451+0000 mon.c (mon.2) 1027 : audit [DBG] from='client.? 172.21.15.6:0/1744692204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:19.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:19 smithi130 bash[51302]: debug 2023-08-27T23:26:19.458+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:19.460247+0000) 2023-08-27T23:26:20.713 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:20 smithi130 bash[51302]: debug 2023-08-27T23:26:20.458+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:20.460538+0000) 2023-08-27T23:26:20.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:26:20.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:20.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:26:20.470Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:20.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:20 smithi006 bash[27292]: cluster 2023-08-27T23:26:19.049489+0000 mgr.y (mgr.14901) 2633 : cluster [DBG] pgmap v2640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:20.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:20 smithi006 bash[27292]: audit 2023-08-27T23:26:20.348658+0000 mon.a (mon.0) 2243 : audit [DBG] from='client.? 172.21.15.6:0/4191845128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:20.756 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:20 smithi006 bash[33918]: cluster 2023-08-27T23:26:19.049489+0000 mgr.y (mgr.14901) 2633 : cluster [DBG] pgmap v2640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:20.756 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:20 smithi006 bash[33918]: audit 2023-08-27T23:26:20.348658+0000 mon.a (mon.0) 2243 : audit [DBG] from='client.? 172.21.15.6:0/4191845128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:20 smithi130 bash[27579]: cluster 2023-08-27T23:26:19.049489+0000 mgr.y (mgr.14901) 2633 : cluster [DBG] pgmap v2640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:20 smithi130 bash[27579]: audit 2023-08-27T23:26:20.348658+0000 mon.a (mon.0) 2243 : audit [DBG] from='client.? 172.21.15.6:0/4191845128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:21.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:26:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:26:20] "GET /metrics HTTP/1.1" 200 193084 "" "Prometheus/2.33.4" 2023-08-27T23:26:21.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:21 smithi130 bash[51302]: debug 2023-08-27T23:26:21.458+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:21.460823+0000) 2023-08-27T23:26:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:26:22.028Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:26:22.030Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:22.717 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:22 smithi130 bash[51302]: debug 2023-08-27T23:26:22.458+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:22.461128+0000) 2023-08-27T23:26:23.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:22 smithi130 bash[27579]: cluster 2023-08-27T23:26:21.050597+0000 mgr.y (mgr.14901) 2634 : cluster [DBG] pgmap v2641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:23.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:22 smithi006 bash[27292]: cluster 2023-08-27T23:26:21.050597+0000 mgr.y (mgr.14901) 2634 : cluster [DBG] pgmap v2641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:22 smithi006 bash[33918]: cluster 2023-08-27T23:26:21.050597+0000 mgr.y (mgr.14901) 2634 : cluster [DBG] pgmap v2641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:23.727 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:23 smithi130 bash[51302]: debug 2023-08-27T23:26:23.458+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:23.461433+0000) 2023-08-27T23:26:24.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:23 smithi130 bash[27579]: audit 2023-08-27T23:26:22.765346+0000 mon.a (mon.0) 2244 : audit [DBG] from='client.? 172.21.15.6:0/4156097415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:24.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:23 smithi006 bash[27292]: audit 2023-08-27T23:26:22.765346+0000 mon.a (mon.0) 2244 : audit [DBG] from='client.? 172.21.15.6:0/4156097415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:24.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:23 smithi006 bash[33918]: audit 2023-08-27T23:26:22.765346+0000 mon.a (mon.0) 2244 : audit [DBG] from='client.? 172.21.15.6:0/4156097415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:26:23.934Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:26:23.937Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:24.729 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:24 smithi130 bash[51302]: debug 2023-08-27T23:26:24.458+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:24.461720+0000) 2023-08-27T23:26:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:24 smithi130 bash[27579]: cluster 2023-08-27T23:26:23.051320+0000 mgr.y (mgr.14901) 2635 : cluster [DBG] pgmap v2642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:24 smithi006 bash[27292]: cluster 2023-08-27T23:26:23.051320+0000 mgr.y (mgr.14901) 2635 : cluster [DBG] pgmap v2642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:24 smithi006 bash[33918]: cluster 2023-08-27T23:26:23.051320+0000 mgr.y (mgr.14901) 2635 : cluster [DBG] pgmap v2642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:25.740 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:25 smithi130 bash[51302]: debug 2023-08-27T23:26:25.458+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:25.462020+0000) 2023-08-27T23:26:26.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:25 smithi130 bash[27579]: audit 2023-08-27T23:26:25.187251+0000 mon.c (mon.2) 1028 : audit [DBG] from='client.? 172.21.15.6:0/1169328091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:26.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:25 smithi006 bash[33918]: audit 2023-08-27T23:26:25.187251+0000 mon.c (mon.2) 1028 : audit [DBG] from='client.? 172.21.15.6:0/1169328091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:25 smithi006 bash[27292]: audit 2023-08-27T23:26:25.187251+0000 mon.c (mon.2) 1028 : audit [DBG] from='client.? 172.21.15.6:0/1169328091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:26.743 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:26 smithi130 bash[51302]: debug 2023-08-27T23:26:26.458+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:26.462331+0000) 2023-08-27T23:26:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:26 smithi130 bash[27579]: cluster 2023-08-27T23:26:25.052067+0000 mgr.y (mgr.14901) 2636 : cluster [DBG] pgmap v2643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:27.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:26 smithi006 bash[33918]: cluster 2023-08-27T23:26:25.052067+0000 mgr.y (mgr.14901) 2636 : cluster [DBG] pgmap v2643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:26 smithi006 bash[27292]: cluster 2023-08-27T23:26:25.052067+0000 mgr.y (mgr.14901) 2636 : cluster [DBG] pgmap v2643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:27.746 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:27 smithi130 bash[51302]: debug 2023-08-27T23:26:27.458+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:27.462641+0000) 2023-08-27T23:26:28.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:27 smithi130 bash[27579]: audit 2023-08-27T23:26:27.600311+0000 mon.c (mon.2) 1029 : audit [DBG] from='client.? 172.21.15.6:0/2501768568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:28.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:27 smithi006 bash[27292]: audit 2023-08-27T23:26:27.600311+0000 mon.c (mon.2) 1029 : audit [DBG] from='client.? 172.21.15.6:0/2501768568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:28.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:27 smithi006 bash[33918]: audit 2023-08-27T23:26:27.600311+0000 mon.c (mon.2) 1029 : audit [DBG] from='client.? 172.21.15.6:0/2501768568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:28.757 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:28 smithi130 bash[51302]: debug 2023-08-27T23:26:28.462+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:28.462996+0000) 2023-08-27T23:26:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:28 smithi130 bash[27579]: cluster 2023-08-27T23:26:27.052701+0000 mgr.y (mgr.14901) 2637 : cluster [DBG] pgmap v2644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:29.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:28 smithi006 bash[27292]: cluster 2023-08-27T23:26:27.052701+0000 mgr.y (mgr.14901) 2637 : cluster [DBG] pgmap v2644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:28 smithi006 bash[33918]: cluster 2023-08-27T23:26:27.052701+0000 mgr.y (mgr.14901) 2637 : cluster [DBG] pgmap v2644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:29.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:29 smithi130 bash[51302]: debug 2023-08-27T23:26:29.462+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:29.463325+0000) 2023-08-27T23:26:30.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:26:30.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:30.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:26:30.470Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:30.762 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:30 smithi130 bash[51302]: debug 2023-08-27T23:26:30.462+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:30.463545+0000) 2023-08-27T23:26:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:30 smithi130 bash[27579]: cluster 2023-08-27T23:26:29.053363+0000 mgr.y (mgr.14901) 2638 : cluster [DBG] pgmap v2645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:31.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:30 smithi130 bash[27579]: audit 2023-08-27T23:26:30.023508+0000 mon.c (mon.2) 1030 : audit [DBG] from='client.? 172.21.15.6:0/3895606667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:26:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:26:30] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T23:26:31.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:30 smithi006 bash[33918]: cluster 2023-08-27T23:26:29.053363+0000 mgr.y (mgr.14901) 2638 : cluster [DBG] pgmap v2645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:31.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:30 smithi006 bash[33918]: audit 2023-08-27T23:26:30.023508+0000 mon.c (mon.2) 1030 : audit [DBG] from='client.? 172.21.15.6:0/3895606667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:31.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:30 smithi006 bash[27292]: cluster 2023-08-27T23:26:29.053363+0000 mgr.y (mgr.14901) 2638 : cluster [DBG] pgmap v2645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:31.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:30 smithi006 bash[27292]: audit 2023-08-27T23:26:30.023508+0000 mon.c (mon.2) 1030 : audit [DBG] from='client.? 172.21.15.6:0/3895606667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:31.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:31 smithi130 bash[51302]: debug 2023-08-27T23:26:31.462+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:31.463840+0000) 2023-08-27T23:26:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:26:32.028Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:26:32.030Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:32.768 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:32 smithi130 bash[51302]: debug 2023-08-27T23:26:32.462+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:32.464131+0000) 2023-08-27T23:26:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:32 smithi130 bash[27579]: cluster 2023-08-27T23:26:31.054487+0000 mgr.y (mgr.14901) 2639 : cluster [DBG] pgmap v2646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:32 smithi130 bash[27579]: audit 2023-08-27T23:26:32.438084+0000 mon.c (mon.2) 1031 : audit [DBG] from='client.? 172.21.15.6:0/750107492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:33.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:32 smithi006 bash[27292]: cluster 2023-08-27T23:26:31.054487+0000 mgr.y (mgr.14901) 2639 : cluster [DBG] pgmap v2646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:32 smithi006 bash[27292]: audit 2023-08-27T23:26:32.438084+0000 mon.c (mon.2) 1031 : audit [DBG] from='client.? 172.21.15.6:0/750107492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:32 smithi006 bash[33918]: cluster 2023-08-27T23:26:31.054487+0000 mgr.y (mgr.14901) 2639 : cluster [DBG] pgmap v2646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:32 smithi006 bash[33918]: audit 2023-08-27T23:26:32.438084+0000 mon.c (mon.2) 1031 : audit [DBG] from='client.? 172.21.15.6:0/750107492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:33.821 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:33 smithi130 bash[51302]: debug 2023-08-27T23:26:33.462+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:33.464473+0000) 2023-08-27T23:26:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:33 smithi130 bash[27579]: cluster 2023-08-27T23:26:33.055176+0000 mgr.y (mgr.14901) 2640 : cluster [DBG] pgmap v2647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:34.824 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:33 smithi006 bash[27292]: cluster 2023-08-27T23:26:33.055176+0000 mgr.y (mgr.14901) 2640 : cluster [DBG] pgmap v2647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:34.825 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:26:33.935Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:34.825 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:26:33.937Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:34.825 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:33 smithi006 bash[33918]: cluster 2023-08-27T23:26:33.055176+0000 mgr.y (mgr.14901) 2640 : cluster [DBG] pgmap v2647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:34 smithi130 bash[51302]: debug 2023-08-27T23:26:34.462+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:34.464784+0000) 2023-08-27T23:26:35.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:35 smithi130 bash[27579]: audit 2023-08-27T23:26:34.858804+0000 mon.a (mon.0) 2245 : audit [DBG] from='client.? 172.21.15.6:0/1305753524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:35.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:35 smithi130 bash[51302]: debug 2023-08-27T23:26:35.462+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:35.465065+0000) 2023-08-27T23:26:35.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:35 smithi006 bash[27292]: audit 2023-08-27T23:26:34.858804+0000 mon.a (mon.0) 2245 : audit [DBG] from='client.? 172.21.15.6:0/1305753524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:35.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:35 smithi006 bash[33918]: audit 2023-08-27T23:26:34.858804+0000 mon.a (mon.0) 2245 : audit [DBG] from='client.? 172.21.15.6:0/1305753524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:36.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:36 smithi130 bash[27579]: cluster 2023-08-27T23:26:35.056257+0000 mgr.y (mgr.14901) 2641 : cluster [DBG] pgmap v2648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:36 smithi130 bash[51302]: debug 2023-08-27T23:26:36.462+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:36.465338+0000) 2023-08-27T23:26:36.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:36 smithi006 bash[33918]: cluster 2023-08-27T23:26:35.056257+0000 mgr.y (mgr.14901) 2641 : cluster [DBG] pgmap v2648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:36.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:36 smithi006 bash[27292]: cluster 2023-08-27T23:26:35.056257+0000 mgr.y (mgr.14901) 2641 : cluster [DBG] pgmap v2648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:37.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:37 smithi130 bash[27579]: audit 2023-08-27T23:26:37.268930+0000 mon.c (mon.2) 1032 : audit [DBG] from='client.? 172.21.15.6:0/1879756830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:37 smithi130 bash[51302]: debug 2023-08-27T23:26:37.462+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:37.465609+0000) 2023-08-27T23:26:37.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:37 smithi006 bash[33918]: audit 2023-08-27T23:26:37.268930+0000 mon.c (mon.2) 1032 : audit [DBG] from='client.? 172.21.15.6:0/1879756830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:37.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:37 smithi006 bash[27292]: audit 2023-08-27T23:26:37.268930+0000 mon.c (mon.2) 1032 : audit [DBG] from='client.? 172.21.15.6:0/1879756830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:38 smithi130 bash[27579]: cluster 2023-08-27T23:26:37.056885+0000 mgr.y (mgr.14901) 2642 : cluster [DBG] pgmap v2649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:38 smithi130 bash[51302]: debug 2023-08-27T23:26:38.462+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:38.465909+0000) 2023-08-27T23:26:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:38 smithi006 bash[33918]: cluster 2023-08-27T23:26:37.056885+0000 mgr.y (mgr.14901) 2642 : cluster [DBG] pgmap v2649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:38 smithi006 bash[27292]: cluster 2023-08-27T23:26:37.056885+0000 mgr.y (mgr.14901) 2642 : cluster [DBG] pgmap v2649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:39 smithi130 bash[51302]: debug 2023-08-27T23:26:39.462+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:39.466219+0000) 2023-08-27T23:26:40.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:40 smithi006 bash[33918]: cluster 2023-08-27T23:26:39.057526+0000 mgr.y (mgr.14901) 2643 : cluster [DBG] pgmap v2650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:40.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:40 smithi006 bash[33918]: audit 2023-08-27T23:26:39.687970+0000 mon.a (mon.0) 2246 : audit [DBG] from='client.? 172.21.15.6:0/1833070382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:40.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:40 smithi006 bash[27292]: cluster 2023-08-27T23:26:39.057526+0000 mgr.y (mgr.14901) 2643 : cluster [DBG] pgmap v2650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:40.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:40 smithi006 bash[27292]: audit 2023-08-27T23:26:39.687970+0000 mon.a (mon.0) 2246 : audit [DBG] from='client.? 172.21.15.6:0/1833070382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:40.756 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:26:40.469Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:40.756 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:26:40.471Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:40 smithi130 bash[27579]: cluster 2023-08-27T23:26:39.057526+0000 mgr.y (mgr.14901) 2643 : cluster [DBG] pgmap v2650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:40 smithi130 bash[27579]: audit 2023-08-27T23:26:39.687970+0000 mon.a (mon.0) 2246 : audit [DBG] from='client.? 172.21.15.6:0/1833070382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:40 smithi130 bash[51302]: debug 2023-08-27T23:26:40.462+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:40.466448+0000) 2023-08-27T23:26:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:26:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:26:40] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T23:26:41.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:41 smithi130 bash[51302]: debug 2023-08-27T23:26:41.462+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:41.466732+0000) 2023-08-27T23:26:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:26:42.029Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:26:42.031Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:42 smithi130 bash[27579]: cluster 2023-08-27T23:26:41.058691+0000 mgr.y (mgr.14901) 2644 : cluster [DBG] pgmap v2651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:42 smithi130 bash[27579]: audit 2023-08-27T23:26:42.094582+0000 mon.c (mon.2) 1033 : audit [DBG] from='client.? 172.21.15.6:0/1439215236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:42 smithi130 bash[51302]: debug 2023-08-27T23:26:42.466+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:42.466988+0000) 2023-08-27T23:26:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:42 smithi006 bash[27292]: cluster 2023-08-27T23:26:41.058691+0000 mgr.y (mgr.14901) 2644 : cluster [DBG] pgmap v2651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:42 smithi006 bash[27292]: audit 2023-08-27T23:26:42.094582+0000 mon.c (mon.2) 1033 : audit [DBG] from='client.? 172.21.15.6:0/1439215236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:42 smithi006 bash[33918]: cluster 2023-08-27T23:26:41.058691+0000 mgr.y (mgr.14901) 2644 : cluster [DBG] pgmap v2651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:42 smithi006 bash[33918]: audit 2023-08-27T23:26:42.094582+0000 mon.c (mon.2) 1033 : audit [DBG] from='client.? 172.21.15.6:0/1439215236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:43.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:43 smithi130 bash[51302]: debug 2023-08-27T23:26:43.466+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:43.467257+0000) 2023-08-27T23:26:44.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:26:43.936Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:26:43.938Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:44 smithi130 bash[51302]: debug 2023-08-27T23:26:44.466+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:44.467539+0000) 2023-08-27T23:26:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:44 smithi130 bash[27579]: cluster 2023-08-27T23:26:43.059477+0000 mgr.y (mgr.14901) 2645 : cluster [DBG] pgmap v2652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:44 smithi130 bash[27579]: audit 2023-08-27T23:26:44.511253+0000 mon.a (mon.0) 2247 : audit [DBG] from='client.? 172.21.15.6:0/805422802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:44.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:44 smithi006 bash[27292]: cluster 2023-08-27T23:26:43.059477+0000 mgr.y (mgr.14901) 2645 : cluster [DBG] pgmap v2652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:44.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:44 smithi006 bash[27292]: audit 2023-08-27T23:26:44.511253+0000 mon.a (mon.0) 2247 : audit [DBG] from='client.? 172.21.15.6:0/805422802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:44 smithi006 bash[33918]: cluster 2023-08-27T23:26:43.059477+0000 mgr.y (mgr.14901) 2645 : cluster [DBG] pgmap v2652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:44 smithi006 bash[33918]: audit 2023-08-27T23:26:44.511253+0000 mon.a (mon.0) 2247 : audit [DBG] from='client.? 172.21.15.6:0/805422802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:45.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:45 smithi130 bash[51302]: debug 2023-08-27T23:26:45.466+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:45.467826+0000) 2023-08-27T23:26:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:46 smithi130 bash[27579]: cluster 2023-08-27T23:26:45.060621+0000 mgr.y (mgr.14901) 2646 : cluster [DBG] pgmap v2653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:46.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:46 smithi130 bash[51302]: debug 2023-08-27T23:26:46.466+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:46.468173+0000) 2023-08-27T23:26:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:46 smithi006 bash[27292]: cluster 2023-08-27T23:26:45.060621+0000 mgr.y (mgr.14901) 2646 : cluster [DBG] pgmap v2653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:46 smithi006 bash[33918]: cluster 2023-08-27T23:26:45.060621+0000 mgr.y (mgr.14901) 2646 : cluster [DBG] pgmap v2653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:47.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:47 smithi130 bash[27579]: audit 2023-08-27T23:26:46.933114+0000 mon.a (mon.0) 2248 : audit [DBG] from='client.? 172.21.15.6:0/888571179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:47.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:47 smithi130 bash[51302]: debug 2023-08-27T23:26:47.466+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:47.468449+0000) 2023-08-27T23:26:47.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:47 smithi006 bash[27292]: audit 2023-08-27T23:26:46.933114+0000 mon.a (mon.0) 2248 : audit [DBG] from='client.? 172.21.15.6:0/888571179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:47.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:47 smithi006 bash[33918]: audit 2023-08-27T23:26:46.933114+0000 mon.a (mon.0) 2248 : audit [DBG] from='client.? 172.21.15.6:0/888571179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:48 smithi130 bash[27579]: cluster 2023-08-27T23:26:47.061295+0000 mgr.y (mgr.14901) 2647 : cluster [DBG] pgmap v2654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:48 smithi130 bash[51302]: debug 2023-08-27T23:26:48.466+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:48.468730+0000) 2023-08-27T23:26:48.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:48 smithi006 bash[33918]: cluster 2023-08-27T23:26:47.061295+0000 mgr.y (mgr.14901) 2647 : cluster [DBG] pgmap v2654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:48 smithi006 bash[27292]: cluster 2023-08-27T23:26:47.061295+0000 mgr.y (mgr.14901) 2647 : cluster [DBG] pgmap v2654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:49.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:49 smithi130 bash[51302]: debug 2023-08-27T23:26:49.466+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:49.469017+0000) 2023-08-27T23:26:49.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:49 smithi130 bash[27579]: audit 2023-08-27T23:26:49.344316+0000 mon.c (mon.2) 1034 : audit [DBG] from='client.? 172.21.15.6:0/3040768594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:49.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:49 smithi006 bash[27292]: audit 2023-08-27T23:26:49.344316+0000 mon.c (mon.2) 1034 : audit [DBG] from='client.? 172.21.15.6:0/3040768594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:49.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:49 smithi006 bash[33918]: audit 2023-08-27T23:26:49.344316+0000 mon.c (mon.2) 1034 : audit [DBG] from='client.? 172.21.15.6:0/3040768594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:50.754 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:50 smithi006 bash[33918]: cluster 2023-08-27T23:26:49.061983+0000 mgr.y (mgr.14901) 2648 : cluster [DBG] pgmap v2655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:50.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:50 smithi006 bash[27292]: cluster 2023-08-27T23:26:49.061983+0000 mgr.y (mgr.14901) 2648 : cluster [DBG] pgmap v2655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:50.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:26:50.470Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:50.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:26:50.473Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:50 smithi130 bash[27579]: cluster 2023-08-27T23:26:49.061983+0000 mgr.y (mgr.14901) 2648 : cluster [DBG] pgmap v2655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:50 smithi130 bash[51302]: debug 2023-08-27T23:26:50.466+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:50.469325+0000) 2023-08-27T23:26:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:26:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:26:50] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:26:51.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:51 smithi130 bash[51302]: debug 2023-08-27T23:26:51.466+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:51.469641+0000) 2023-08-27T23:26:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:26:52.030Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:26:52.033Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:52 smithi130 bash[27579]: cluster 2023-08-27T23:26:51.063152+0000 mgr.y (mgr.14901) 2649 : cluster [DBG] pgmap v2656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:52 smithi130 bash[27579]: audit 2023-08-27T23:26:51.769169+0000 mon.c (mon.2) 1035 : audit [DBG] from='client.? 172.21.15.6:0/801099249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:52.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:52 smithi130 bash[51302]: debug 2023-08-27T23:26:52.466+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:52.469951+0000) 2023-08-27T23:26:53.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:52 smithi006 bash[33918]: cluster 2023-08-27T23:26:51.063152+0000 mgr.y (mgr.14901) 2649 : cluster [DBG] pgmap v2656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:53.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:52 smithi006 bash[33918]: audit 2023-08-27T23:26:51.769169+0000 mon.c (mon.2) 1035 : audit [DBG] from='client.? 172.21.15.6:0/801099249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:52 smithi006 bash[27292]: cluster 2023-08-27T23:26:51.063152+0000 mgr.y (mgr.14901) 2649 : cluster [DBG] pgmap v2656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:52 smithi006 bash[27292]: audit 2023-08-27T23:26:51.769169+0000 mon.c (mon.2) 1035 : audit [DBG] from='client.? 172.21.15.6:0/801099249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:53.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:53 smithi130 bash[51302]: debug 2023-08-27T23:26:53.466+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:53.470266+0000) 2023-08-27T23:26:54.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:26:53.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:26:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:26:53.939Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:26:54.726 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:54 smithi130 bash[51302]: debug 2023-08-27T23:26:54.466+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:54.470546+0000) 2023-08-27T23:26:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:54 smithi130 bash[27579]: cluster 2023-08-27T23:26:53.063818+0000 mgr.y (mgr.14901) 2650 : cluster [DBG] pgmap v2657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:55.135 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:54 smithi130 bash[27579]: audit 2023-08-27T23:26:54.191189+0000 mon.a (mon.0) 2249 : audit [DBG] from='client.? 172.21.15.6:0/1120828698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:54 smithi006 bash[33918]: cluster 2023-08-27T23:26:53.063818+0000 mgr.y (mgr.14901) 2650 : cluster [DBG] pgmap v2657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:54 smithi006 bash[33918]: audit 2023-08-27T23:26:54.191189+0000 mon.a (mon.0) 2249 : audit [DBG] from='client.? 172.21.15.6:0/1120828698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:54 smithi006 bash[27292]: cluster 2023-08-27T23:26:53.063818+0000 mgr.y (mgr.14901) 2650 : cluster [DBG] pgmap v2657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:54 smithi006 bash[27292]: audit 2023-08-27T23:26:54.191189+0000 mon.a (mon.0) 2249 : audit [DBG] from='client.? 172.21.15.6:0/1120828698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:55.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:55 smithi130 bash[51302]: debug 2023-08-27T23:26:55.466+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:55.470818+0000) 2023-08-27T23:26:56.731 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:56 smithi130 bash[51302]: debug 2023-08-27T23:26:56.470+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:56.471132+0000) 2023-08-27T23:26:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:56 smithi130 bash[27579]: cluster 2023-08-27T23:26:55.064974+0000 mgr.y (mgr.14901) 2651 : cluster [DBG] pgmap v2658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:57.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:56 smithi130 bash[27579]: audit 2023-08-27T23:26:56.600288+0000 mon.a (mon.0) 2250 : audit [DBG] from='client.? 172.21.15.6:0/381550123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:57.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:56 smithi006 bash[33918]: cluster 2023-08-27T23:26:55.064974+0000 mgr.y (mgr.14901) 2651 : cluster [DBG] pgmap v2658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:57.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:56 smithi006 bash[33918]: audit 2023-08-27T23:26:56.600288+0000 mon.a (mon.0) 2250 : audit [DBG] from='client.? 172.21.15.6:0/381550123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:57.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:56 smithi006 bash[27292]: cluster 2023-08-27T23:26:55.064974+0000 mgr.y (mgr.14901) 2651 : cluster [DBG] pgmap v2658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:57.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:56 smithi006 bash[27292]: audit 2023-08-27T23:26:56.600288+0000 mon.a (mon.0) 2250 : audit [DBG] from='client.? 172.21.15.6:0/381550123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:57.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:57 smithi130 bash[51302]: debug 2023-08-27T23:26:57.470+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:57.471411+0000) 2023-08-27T23:26:58.736 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:58 smithi130 bash[51302]: debug 2023-08-27T23:26:58.470+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:58.471697+0000) 2023-08-27T23:26:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:58 smithi130 bash[27579]: cluster 2023-08-27T23:26:57.065569+0000 mgr.y (mgr.14901) 2652 : cluster [DBG] pgmap v2659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:59.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:58 smithi006 bash[33918]: cluster 2023-08-27T23:26:57.065569+0000 mgr.y (mgr.14901) 2652 : cluster [DBG] pgmap v2659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:58 smithi006 bash[27292]: cluster 2023-08-27T23:26:57.065569+0000 mgr.y (mgr.14901) 2652 : cluster [DBG] pgmap v2659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:26:59.738 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:26:59 smithi130 bash[51302]: debug 2023-08-27T23:26:59.469+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:59.471956+0000) 2023-08-27T23:27:00.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:26:59 smithi130 bash[27579]: audit 2023-08-27T23:26:59.010662+0000 mon.a (mon.0) 2251 : audit [DBG] from='client.? 172.21.15.6:0/3999031004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:00.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:26:59 smithi006 bash[33918]: audit 2023-08-27T23:26:59.010662+0000 mon.a (mon.0) 2251 : audit [DBG] from='client.? 172.21.15.6:0/3999031004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:00.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:26:59 smithi006 bash[27292]: audit 2023-08-27T23:26:59.010662+0000 mon.a (mon.0) 2251 : audit [DBG] from='client.? 172.21.15.6:0/3999031004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:00.747 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:27:00.471Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:00.747 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:27:00.472Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:00.748 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:00 smithi130 bash[51302]: debug 2023-08-27T23:27:00.469+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:00.472233+0000) 2023-08-27T23:27:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:00 smithi130 bash[27579]: cluster 2023-08-27T23:26:59.066166+0000 mgr.y (mgr.14901) 2653 : cluster [DBG] pgmap v2660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:27:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:27:00] "GET /metrics HTTP/1.1" 200 193091 "" "Prometheus/2.33.4" 2023-08-27T23:27:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:00 smithi006 bash[33918]: cluster 2023-08-27T23:26:59.066166+0000 mgr.y (mgr.14901) 2653 : cluster [DBG] pgmap v2660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:00 smithi006 bash[27292]: cluster 2023-08-27T23:26:59.066166+0000 mgr.y (mgr.14901) 2653 : cluster [DBG] pgmap v2660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:01.751 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:01 smithi130 bash[51302]: debug 2023-08-27T23:27:01.469+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:01.472603+0000) 2023-08-27T23:27:02.034 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:01 smithi006 bash[33918]: audit 2023-08-27T23:27:01.424939+0000 mon.a (mon.0) 2252 : audit [DBG] from='client.? 172.21.15.6:0/289982092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:02.034 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:01 smithi006 bash[27292]: audit 2023-08-27T23:27:01.424939+0000 mon.a (mon.0) 2252 : audit [DBG] from='client.? 172.21.15.6:0/289982092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:02.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:01 smithi130 bash[27579]: audit 2023-08-27T23:27:01.424939+0000 mon.a (mon.0) 2252 : audit [DBG] from='client.? 172.21.15.6:0/289982092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:27:02.031Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:27:02.033Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:02.759 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:02 smithi130 bash[51302]: debug 2023-08-27T23:27:02.469+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:02.472903+0000) 2023-08-27T23:27:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:02 smithi130 bash[27579]: cluster 2023-08-27T23:27:01.067421+0000 mgr.y (mgr.14901) 2654 : cluster [DBG] pgmap v2661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:03.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:02 smithi006 bash[33918]: cluster 2023-08-27T23:27:01.067421+0000 mgr.y (mgr.14901) 2654 : cluster [DBG] pgmap v2661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:02 smithi006 bash[27292]: cluster 2023-08-27T23:27:01.067421+0000 mgr.y (mgr.14901) 2654 : cluster [DBG] pgmap v2661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:03.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:03 smithi130 bash[51302]: debug 2023-08-27T23:27:03.469+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:03.473238+0000) 2023-08-27T23:27:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:27:03.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:27:03.939Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:04.764 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:04 smithi130 bash[51302]: debug 2023-08-27T23:27:04.469+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:04.473513+0000) 2023-08-27T23:27:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:04 smithi130 bash[27579]: cluster 2023-08-27T23:27:03.068110+0000 mgr.y (mgr.14901) 2655 : cluster [DBG] pgmap v2662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:04 smithi130 bash[27579]: audit 2023-08-27T23:27:03.845278+0000 mon.a (mon.0) 2253 : audit [DBG] from='client.? 172.21.15.6:0/3310029948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:04 smithi006 bash[33918]: cluster 2023-08-27T23:27:03.068110+0000 mgr.y (mgr.14901) 2655 : cluster [DBG] pgmap v2662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:04 smithi006 bash[33918]: audit 2023-08-27T23:27:03.845278+0000 mon.a (mon.0) 2253 : audit [DBG] from='client.? 172.21.15.6:0/3310029948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:04 smithi006 bash[27292]: cluster 2023-08-27T23:27:03.068110+0000 mgr.y (mgr.14901) 2655 : cluster [DBG] pgmap v2662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:04 smithi006 bash[27292]: audit 2023-08-27T23:27:03.845278+0000 mon.a (mon.0) 2253 : audit [DBG] from='client.? 172.21.15.6:0/3310029948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:05.768 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:05 smithi130 bash[51302]: debug 2023-08-27T23:27:05.469+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:05.473832+0000) 2023-08-27T23:27:06.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:05 smithi130 bash[27579]: audit 2023-08-27T23:27:05.536372+0000 mon.b (mon.1) 373 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:27:06.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:05 smithi006 bash[27292]: audit 2023-08-27T23:27:05.536372+0000 mon.b (mon.1) 373 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:27:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:05 smithi006 bash[33918]: audit 2023-08-27T23:27:05.536372+0000 mon.b (mon.1) 373 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:27:06.772 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:06 smithi130 bash[51302]: debug 2023-08-27T23:27:06.469+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:06.474169+0000) 2023-08-27T23:27:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:06 smithi130 bash[27579]: cluster 2023-08-27T23:27:05.069238+0000 mgr.y (mgr.14901) 2656 : cluster [DBG] pgmap v2663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:06 smithi130 bash[27579]: audit 2023-08-27T23:27:06.256220+0000 mon.a (mon.0) 2254 : audit [DBG] from='client.? 172.21.15.6:0/2173048260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:06 smithi006 bash[27292]: cluster 2023-08-27T23:27:05.069238+0000 mgr.y (mgr.14901) 2656 : cluster [DBG] pgmap v2663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:07.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:06 smithi006 bash[27292]: audit 2023-08-27T23:27:06.256220+0000 mon.a (mon.0) 2254 : audit [DBG] from='client.? 172.21.15.6:0/2173048260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:07.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:06 smithi006 bash[33918]: cluster 2023-08-27T23:27:05.069238+0000 mgr.y (mgr.14901) 2656 : cluster [DBG] pgmap v2663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:07.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:06 smithi006 bash[33918]: audit 2023-08-27T23:27:06.256220+0000 mon.a (mon.0) 2254 : audit [DBG] from='client.? 172.21.15.6:0/2173048260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:07.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:07 smithi130 bash[51302]: debug 2023-08-27T23:27:07.473+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:07.474450+0000) 2023-08-27T23:27:08.778 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:08 smithi130 bash[51302]: debug 2023-08-27T23:27:08.469+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:08.474747+0000) 2023-08-27T23:27:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:08 smithi130 bash[27579]: cluster 2023-08-27T23:27:07.069905+0000 mgr.y (mgr.14901) 2657 : cluster [DBG] pgmap v2664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:08 smithi130 bash[27579]: audit 2023-08-27T23:27:08.672272+0000 mon.a (mon.0) 2255 : audit [DBG] from='client.? 172.21.15.6:0/728204991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:08 smithi006 bash[27292]: cluster 2023-08-27T23:27:07.069905+0000 mgr.y (mgr.14901) 2657 : cluster [DBG] pgmap v2664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:08 smithi006 bash[27292]: audit 2023-08-27T23:27:08.672272+0000 mon.a (mon.0) 2255 : audit [DBG] from='client.? 172.21.15.6:0/728204991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:08 smithi006 bash[33918]: cluster 2023-08-27T23:27:07.069905+0000 mgr.y (mgr.14901) 2657 : cluster [DBG] pgmap v2664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:08 smithi006 bash[33918]: audit 2023-08-27T23:27:08.672272+0000 mon.a (mon.0) 2255 : audit [DBG] from='client.? 172.21.15.6:0/728204991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:09.870 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:09 smithi130 bash[51302]: debug 2023-08-27T23:27:09.473+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:09.475040+0000) 2023-08-27T23:27:10.753 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:10 smithi006 bash[27292]: cluster 2023-08-27T23:27:09.070546+0000 mgr.y (mgr.14901) 2658 : cluster [DBG] pgmap v2665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:10.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:27:10.472Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:10.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:27:10.474Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:10.754 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:10 smithi006 bash[33918]: cluster 2023-08-27T23:27:09.070546+0000 mgr.y (mgr.14901) 2658 : cluster [DBG] pgmap v2665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:10 smithi130 bash[27579]: cluster 2023-08-27T23:27:09.070546+0000 mgr.y (mgr.14901) 2658 : cluster [DBG] pgmap v2665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:10 smithi130 bash[51302]: debug 2023-08-27T23:27:10.473+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:10.475292+0000) 2023-08-27T23:27:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:27:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:27:10] "GET /metrics HTTP/1.1" 200 193091 "" "Prometheus/2.33.4" 2023-08-27T23:27:11.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:11 smithi130 bash[27579]: audit 2023-08-27T23:27:11.109412+0000 mon.c (mon.2) 1036 : audit [DBG] from='client.? 172.21.15.6:0/235032652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:11.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:11 smithi130 bash[27579]: audit 2023-08-27T23:27:11.355347+0000 mon.a (mon.0) 2256 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:27:11.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:11 smithi130 bash[51302]: debug 2023-08-27T23:27:11.473+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:11.475544+0000) 2023-08-27T23:27:11.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:11 smithi006 bash[27292]: audit 2023-08-27T23:27:11.109412+0000 mon.c (mon.2) 1036 : audit [DBG] from='client.? 172.21.15.6:0/235032652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:11.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:11 smithi006 bash[27292]: audit 2023-08-27T23:27:11.355347+0000 mon.a (mon.0) 2256 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:27:11.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:11 smithi006 bash[33918]: audit 2023-08-27T23:27:11.109412+0000 mon.c (mon.2) 1036 : audit [DBG] from='client.? 172.21.15.6:0/235032652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:11.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:11 smithi006 bash[33918]: audit 2023-08-27T23:27:11.355347+0000 mon.a (mon.0) 2256 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:27:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:27:12.032Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:27:12.034Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:12 smithi130 bash[51302]: debug 2023-08-27T23:27:12.473+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:12.475770+0000) 2023-08-27T23:27:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:12 smithi130 bash[27579]: cluster 2023-08-27T23:27:11.071292+0000 mgr.y (mgr.14901) 2659 : cluster [DBG] pgmap v2666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:12 smithi130 bash[27579]: audit 2023-08-27T23:27:11.904017+0000 mon.a (mon.0) 2257 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:27:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:12 smithi130 bash[27579]: audit 2023-08-27T23:27:12.180721+0000 mon.b (mon.1) 374 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:27:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:12 smithi130 bash[27579]: audit 2023-08-27T23:27:12.182822+0000 mon.b (mon.1) 375 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:27:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:12 smithi130 bash[27579]: audit 2023-08-27T23:27:12.193564+0000 mon.a (mon.0) 2258 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:27:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:12 smithi006 bash[33918]: cluster 2023-08-27T23:27:11.071292+0000 mgr.y (mgr.14901) 2659 : cluster [DBG] pgmap v2666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:12 smithi006 bash[33918]: audit 2023-08-27T23:27:11.904017+0000 mon.a (mon.0) 2257 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:27:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:12 smithi006 bash[33918]: audit 2023-08-27T23:27:12.180721+0000 mon.b (mon.1) 374 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:27:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:12 smithi006 bash[33918]: audit 2023-08-27T23:27:12.182822+0000 mon.b (mon.1) 375 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:27:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:12 smithi006 bash[33918]: audit 2023-08-27T23:27:12.193564+0000 mon.a (mon.0) 2258 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:27:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:12 smithi006 bash[27292]: cluster 2023-08-27T23:27:11.071292+0000 mgr.y (mgr.14901) 2659 : cluster [DBG] pgmap v2666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:12 smithi006 bash[27292]: audit 2023-08-27T23:27:11.904017+0000 mon.a (mon.0) 2257 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:27:12.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:12 smithi006 bash[27292]: audit 2023-08-27T23:27:12.180721+0000 mon.b (mon.1) 374 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:27:12.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:12 smithi006 bash[27292]: audit 2023-08-27T23:27:12.182822+0000 mon.b (mon.1) 375 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:27:12.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:12 smithi006 bash[27292]: audit 2023-08-27T23:27:12.193564+0000 mon.a (mon.0) 2258 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:27:13.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:13 smithi130 bash[27579]: audit 2023-08-27T23:27:13.526196+0000 mon.c (mon.2) 1037 : audit [DBG] from='client.? 172.21.15.6:0/1421671337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:13.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:13 smithi130 bash[51302]: debug 2023-08-27T23:27:13.473+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:13.476065+0000) 2023-08-27T23:27:13.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:13 smithi006 bash[27292]: audit 2023-08-27T23:27:13.526196+0000 mon.c (mon.2) 1037 : audit [DBG] from='client.? 172.21.15.6:0/1421671337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:13.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:13 smithi006 bash[33918]: audit 2023-08-27T23:27:13.526196+0000 mon.c (mon.2) 1037 : audit [DBG] from='client.? 172.21.15.6:0/1421671337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:13.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:27:13.938Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:13.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:27:13.940Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:14 smithi130 bash[27579]: cluster 2023-08-27T23:27:13.071985+0000 mgr.y (mgr.14901) 2660 : cluster [DBG] pgmap v2667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:14 smithi130 bash[51302]: debug 2023-08-27T23:27:14.473+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:14.476363+0000) 2023-08-27T23:27:14.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:14 smithi006 bash[33918]: cluster 2023-08-27T23:27:13.071985+0000 mgr.y (mgr.14901) 2660 : cluster [DBG] pgmap v2667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:14 smithi006 bash[27292]: cluster 2023-08-27T23:27:13.071985+0000 mgr.y (mgr.14901) 2660 : cluster [DBG] pgmap v2667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:15.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:15 smithi130 bash[51302]: debug 2023-08-27T23:27:15.473+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:15.476697+0000) 2023-08-27T23:27:16.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:16 smithi130 bash[51302]: debug 2023-08-27T23:27:16.473+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:16.477006+0000) 2023-08-27T23:27:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:16 smithi130 bash[27579]: cluster 2023-08-27T23:27:15.073248+0000 mgr.y (mgr.14901) 2661 : cluster [DBG] pgmap v2668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:16 smithi130 bash[27579]: audit 2023-08-27T23:27:15.936483+0000 mon.c (mon.2) 1038 : audit [DBG] from='client.? 172.21.15.6:0/625461803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:17.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:16 smithi006 bash[27292]: cluster 2023-08-27T23:27:15.073248+0000 mgr.y (mgr.14901) 2661 : cluster [DBG] pgmap v2668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:17.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:16 smithi006 bash[27292]: audit 2023-08-27T23:27:15.936483+0000 mon.c (mon.2) 1038 : audit [DBG] from='client.? 172.21.15.6:0/625461803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:16 smithi006 bash[33918]: cluster 2023-08-27T23:27:15.073248+0000 mgr.y (mgr.14901) 2661 : cluster [DBG] pgmap v2668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:16 smithi006 bash[33918]: audit 2023-08-27T23:27:15.936483+0000 mon.c (mon.2) 1038 : audit [DBG] from='client.? 172.21.15.6:0/625461803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:17.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:17 smithi130 bash[51302]: debug 2023-08-27T23:27:17.473+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:17.477339+0000) 2023-08-27T23:27:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:18 smithi130 bash[27579]: cluster 2023-08-27T23:27:17.073998+0000 mgr.y (mgr.14901) 2662 : cluster [DBG] pgmap v2669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:18 smithi130 bash[27579]: audit 2023-08-27T23:27:18.352059+0000 mon.c (mon.2) 1039 : audit [DBG] from='client.? 172.21.15.6:0/3135284645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:18 smithi130 bash[51302]: debug 2023-08-27T23:27:18.473+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:18.477637+0000) 2023-08-27T23:27:19.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:18 smithi006 bash[27292]: cluster 2023-08-27T23:27:17.073998+0000 mgr.y (mgr.14901) 2662 : cluster [DBG] pgmap v2669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:18 smithi006 bash[27292]: audit 2023-08-27T23:27:18.352059+0000 mon.c (mon.2) 1039 : audit [DBG] from='client.? 172.21.15.6:0/3135284645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:18 smithi006 bash[33918]: cluster 2023-08-27T23:27:17.073998+0000 mgr.y (mgr.14901) 2662 : cluster [DBG] pgmap v2669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:18 smithi006 bash[33918]: audit 2023-08-27T23:27:18.352059+0000 mon.c (mon.2) 1039 : audit [DBG] from='client.? 172.21.15.6:0/3135284645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:19.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:19 smithi130 bash[51302]: debug 2023-08-27T23:27:19.477+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:19.477936+0000) 2023-08-27T23:27:20.727 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:27:20.473Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:20.728 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:27:20.474Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:20 smithi130 bash[27579]: cluster 2023-08-27T23:27:19.074750+0000 mgr.y (mgr.14901) 2663 : cluster [DBG] pgmap v2670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:20 smithi130 bash[51302]: debug 2023-08-27T23:27:20.473+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:20.478230+0000) 2023-08-27T23:27:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:20 smithi006 bash[27292]: cluster 2023-08-27T23:27:19.074750+0000 mgr.y (mgr.14901) 2663 : cluster [DBG] pgmap v2670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:27:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:27:20] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:27:21.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:20 smithi006 bash[33918]: cluster 2023-08-27T23:27:19.074750+0000 mgr.y (mgr.14901) 2663 : cluster [DBG] pgmap v2670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:21.731 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:21 smithi130 bash[51302]: debug 2023-08-27T23:27:21.477+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:21.478538+0000) 2023-08-27T23:27:22.035 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:21 smithi006 bash[27292]: audit 2023-08-27T23:27:20.766143+0000 mon.a (mon.0) 2259 : audit [DBG] from='client.? 172.21.15.6:0/261636569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:22.035 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:21 smithi006 bash[33918]: audit 2023-08-27T23:27:20.766143+0000 mon.a (mon.0) 2259 : audit [DBG] from='client.? 172.21.15.6:0/261636569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:22.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:21 smithi130 bash[27579]: audit 2023-08-27T23:27:20.766143+0000 mon.a (mon.0) 2259 : audit [DBG] from='client.? 172.21.15.6:0/261636569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:27:22.032Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:27:22.035Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:22.741 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:22 smithi130 bash[51302]: debug 2023-08-27T23:27:22.473+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:22.478763+0000) 2023-08-27T23:27:23.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:22 smithi130 bash[27579]: cluster 2023-08-27T23:27:21.075909+0000 mgr.y (mgr.14901) 2664 : cluster [DBG] pgmap v2671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:23.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:22 smithi006 bash[27292]: cluster 2023-08-27T23:27:21.075909+0000 mgr.y (mgr.14901) 2664 : cluster [DBG] pgmap v2671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:22 smithi006 bash[33918]: cluster 2023-08-27T23:27:21.075909+0000 mgr.y (mgr.14901) 2664 : cluster [DBG] pgmap v2671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:23.749 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:23 smithi130 bash[51302]: debug 2023-08-27T23:27:23.477+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:23.479038+0000) 2023-08-27T23:27:24.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:23 smithi130 bash[27579]: audit 2023-08-27T23:27:23.186739+0000 mon.c (mon.2) 1040 : audit [DBG] from='client.? 172.21.15.6:0/1601652241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:24.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:23 smithi006 bash[33918]: audit 2023-08-27T23:27:23.186739+0000 mon.c (mon.2) 1040 : audit [DBG] from='client.? 172.21.15.6:0/1601652241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:27:23.938Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:27:23.941Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:24.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:23 smithi006 bash[27292]: audit 2023-08-27T23:27:23.186739+0000 mon.c (mon.2) 1040 : audit [DBG] from='client.? 172.21.15.6:0/1601652241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:24.753 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:24 smithi130 bash[51302]: debug 2023-08-27T23:27:24.477+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:24.479354+0000) 2023-08-27T23:27:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:24 smithi130 bash[27579]: cluster 2023-08-27T23:27:23.076524+0000 mgr.y (mgr.14901) 2665 : cluster [DBG] pgmap v2672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:25.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:24 smithi006 bash[27292]: cluster 2023-08-27T23:27:23.076524+0000 mgr.y (mgr.14901) 2665 : cluster [DBG] pgmap v2672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:24 smithi006 bash[33918]: cluster 2023-08-27T23:27:23.076524+0000 mgr.y (mgr.14901) 2665 : cluster [DBG] pgmap v2672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:25.762 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:25 smithi130 bash[51302]: debug 2023-08-27T23:27:25.477+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:25.479655+0000) 2023-08-27T23:27:26.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:25 smithi130 bash[27579]: audit 2023-08-27T23:27:25.603065+0000 mon.a (mon.0) 2260 : audit [DBG] from='client.? 172.21.15.6:0/3875707029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:26.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:25 smithi006 bash[27292]: audit 2023-08-27T23:27:25.603065+0000 mon.a (mon.0) 2260 : audit [DBG] from='client.? 172.21.15.6:0/3875707029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:26.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:25 smithi006 bash[33918]: audit 2023-08-27T23:27:25.603065+0000 mon.a (mon.0) 2260 : audit [DBG] from='client.? 172.21.15.6:0/3875707029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:26.766 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:26 smithi130 bash[51302]: debug 2023-08-27T23:27:26.477+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:26.479972+0000) 2023-08-27T23:27:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:26 smithi130 bash[27579]: cluster 2023-08-27T23:27:25.077566+0000 mgr.y (mgr.14901) 2666 : cluster [DBG] pgmap v2673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:26 smithi006 bash[33918]: cluster 2023-08-27T23:27:25.077566+0000 mgr.y (mgr.14901) 2666 : cluster [DBG] pgmap v2673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:26 smithi006 bash[27292]: cluster 2023-08-27T23:27:25.077566+0000 mgr.y (mgr.14901) 2666 : cluster [DBG] pgmap v2673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:27.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:27 smithi130 bash[51302]: debug 2023-08-27T23:27:27.477+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:27.480245+0000) 2023-08-27T23:27:28.770 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:28 smithi130 bash[51302]: debug 2023-08-27T23:27:28.477+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:28.480559+0000) 2023-08-27T23:27:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:28 smithi130 bash[27579]: cluster 2023-08-27T23:27:27.078186+0000 mgr.y (mgr.14901) 2667 : cluster [DBG] pgmap v2674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:28 smithi130 bash[27579]: audit 2023-08-27T23:27:28.021193+0000 mon.a (mon.0) 2261 : audit [DBG] from='client.? 172.21.15.6:0/4155559783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:28 smithi006 bash[33918]: cluster 2023-08-27T23:27:27.078186+0000 mgr.y (mgr.14901) 2667 : cluster [DBG] pgmap v2674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:28 smithi006 bash[33918]: audit 2023-08-27T23:27:28.021193+0000 mon.a (mon.0) 2261 : audit [DBG] from='client.? 172.21.15.6:0/4155559783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:28 smithi006 bash[27292]: cluster 2023-08-27T23:27:27.078186+0000 mgr.y (mgr.14901) 2667 : cluster [DBG] pgmap v2674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:28 smithi006 bash[27292]: audit 2023-08-27T23:27:28.021193+0000 mon.a (mon.0) 2261 : audit [DBG] from='client.? 172.21.15.6:0/4155559783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:29.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:29 smithi130 bash[51302]: debug 2023-08-27T23:27:29.477+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:29.480854+0000) 2023-08-27T23:27:30.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:27:30.473Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:30.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:27:30.475Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:30.775 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:30 smithi130 bash[51302]: debug 2023-08-27T23:27:30.477+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:30.481114+0000) 2023-08-27T23:27:31.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:30 smithi130 bash[27579]: cluster 2023-08-27T23:27:29.078930+0000 mgr.y (mgr.14901) 2668 : cluster [DBG] pgmap v2675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:31.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:30 smithi130 bash[27579]: audit 2023-08-27T23:27:30.447279+0000 mon.a (mon.0) 2262 : audit [DBG] from='client.? 172.21.15.6:0/3167336095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:27:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:27:30] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:27:31.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:30 smithi006 bash[27292]: cluster 2023-08-27T23:27:29.078930+0000 mgr.y (mgr.14901) 2668 : cluster [DBG] pgmap v2675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:31.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:30 smithi006 bash[27292]: audit 2023-08-27T23:27:30.447279+0000 mon.a (mon.0) 2262 : audit [DBG] from='client.? 172.21.15.6:0/3167336095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:31.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:30 smithi006 bash[33918]: cluster 2023-08-27T23:27:29.078930+0000 mgr.y (mgr.14901) 2668 : cluster [DBG] pgmap v2675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:31.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:30 smithi006 bash[33918]: audit 2023-08-27T23:27:30.447279+0000 mon.a (mon.0) 2262 : audit [DBG] from='client.? 172.21.15.6:0/3167336095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:31.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:31 smithi130 bash[51302]: debug 2023-08-27T23:27:31.477+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:31.481418+0000) 2023-08-27T23:27:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:27:32.033Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:27:32.035Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:32.786 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:32 smithi130 bash[51302]: debug 2023-08-27T23:27:32.477+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:32.481688+0000) 2023-08-27T23:27:33.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:32 smithi130 bash[27579]: cluster 2023-08-27T23:27:31.080025+0000 mgr.y (mgr.14901) 2669 : cluster [DBG] pgmap v2676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:32 smithi006 bash[27292]: cluster 2023-08-27T23:27:31.080025+0000 mgr.y (mgr.14901) 2669 : cluster [DBG] pgmap v2676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:32 smithi006 bash[33918]: cluster 2023-08-27T23:27:31.080025+0000 mgr.y (mgr.14901) 2669 : cluster [DBG] pgmap v2676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:33.794 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:33 smithi130 bash[51302]: debug 2023-08-27T23:27:33.481+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:33.482026+0000) 2023-08-27T23:27:34.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:33 smithi130 bash[27579]: audit 2023-08-27T23:27:32.867546+0000 mon.a (mon.0) 2263 : audit [DBG] from='client.? 172.21.15.6:0/710153202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:33 smithi130 bash[27579]: cluster 2023-08-27T23:27:33.080648+0000 mgr.y (mgr.14901) 2670 : cluster [DBG] pgmap v2677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:34.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:33 smithi006 bash[33918]: audit 2023-08-27T23:27:32.867546+0000 mon.a (mon.0) 2263 : audit [DBG] from='client.? 172.21.15.6:0/710153202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:34.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:33 smithi006 bash[33918]: cluster 2023-08-27T23:27:33.080648+0000 mgr.y (mgr.14901) 2670 : cluster [DBG] pgmap v2677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:27:33.938Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:27:33.941Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:34.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:33 smithi006 bash[27292]: audit 2023-08-27T23:27:32.867546+0000 mon.a (mon.0) 2263 : audit [DBG] from='client.? 172.21.15.6:0/710153202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:34.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:33 smithi006 bash[27292]: cluster 2023-08-27T23:27:33.080648+0000 mgr.y (mgr.14901) 2670 : cluster [DBG] pgmap v2677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:34.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:34 smithi130 bash[51302]: debug 2023-08-27T23:27:34.481+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:34.482302+0000) 2023-08-27T23:27:35.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:35 smithi130 bash[27579]: audit 2023-08-27T23:27:35.288668+0000 mon.a (mon.0) 2264 : audit [DBG] from='client.? 172.21.15.6:0/1092177768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:35.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:35 smithi130 bash[51302]: debug 2023-08-27T23:27:35.481+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:35.482625+0000) 2023-08-27T23:27:35.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:35 smithi006 bash[33918]: audit 2023-08-27T23:27:35.288668+0000 mon.a (mon.0) 2264 : audit [DBG] from='client.? 172.21.15.6:0/1092177768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:35.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:35 smithi006 bash[27292]: audit 2023-08-27T23:27:35.288668+0000 mon.a (mon.0) 2264 : audit [DBG] from='client.? 172.21.15.6:0/1092177768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:36 smithi130 bash[27579]: cluster 2023-08-27T23:27:35.081398+0000 mgr.y (mgr.14901) 2671 : cluster [DBG] pgmap v2678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:36 smithi130 bash[51302]: debug 2023-08-27T23:27:36.481+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:36.482944+0000) 2023-08-27T23:27:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:36 smithi006 bash[27292]: cluster 2023-08-27T23:27:35.081398+0000 mgr.y (mgr.14901) 2671 : cluster [DBG] pgmap v2678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:36 smithi006 bash[33918]: cluster 2023-08-27T23:27:35.081398+0000 mgr.y (mgr.14901) 2671 : cluster [DBG] pgmap v2678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:37 smithi130 bash[51302]: debug 2023-08-27T23:27:37.481+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:37.483177+0000) 2023-08-27T23:27:38.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:38 smithi130 bash[27579]: cluster 2023-08-27T23:27:37.082002+0000 mgr.y (mgr.14901) 2672 : cluster [DBG] pgmap v2679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:38 smithi130 bash[27579]: audit 2023-08-27T23:27:37.705902+0000 mon.a (mon.0) 2265 : audit [DBG] from='client.? 172.21.15.6:0/1376271841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:38 smithi130 bash[51302]: debug 2023-08-27T23:27:38.481+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:38.483513+0000) 2023-08-27T23:27:38.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:38 smithi006 bash[27292]: cluster 2023-08-27T23:27:37.082002+0000 mgr.y (mgr.14901) 2672 : cluster [DBG] pgmap v2679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:38 smithi006 bash[27292]: audit 2023-08-27T23:27:37.705902+0000 mon.a (mon.0) 2265 : audit [DBG] from='client.? 172.21.15.6:0/1376271841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:38 smithi006 bash[33918]: cluster 2023-08-27T23:27:37.082002+0000 mgr.y (mgr.14901) 2672 : cluster [DBG] pgmap v2679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:38 smithi006 bash[33918]: audit 2023-08-27T23:27:37.705902+0000 mon.a (mon.0) 2265 : audit [DBG] from='client.? 172.21.15.6:0/1376271841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:39 smithi130 bash[51302]: debug 2023-08-27T23:27:39.481+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:39.483816+0000) 2023-08-27T23:27:40.754 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:40 smithi006 bash[27292]: cluster 2023-08-27T23:27:39.082681+0000 mgr.y (mgr.14901) 2673 : cluster [DBG] pgmap v2680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:40.754 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:40 smithi006 bash[27292]: audit 2023-08-27T23:27:40.117502+0000 mon.c (mon.2) 1041 : audit [DBG] from='client.? 172.21.15.6:0/4025058614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:40.754 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:40 smithi006 bash[33918]: cluster 2023-08-27T23:27:39.082681+0000 mgr.y (mgr.14901) 2673 : cluster [DBG] pgmap v2680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:40.754 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:40 smithi006 bash[33918]: audit 2023-08-27T23:27:40.117502+0000 mon.c (mon.2) 1041 : audit [DBG] from='client.? 172.21.15.6:0/4025058614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:40.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:27:40.474Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:40.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:27:40.476Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:40 smithi130 bash[27579]: cluster 2023-08-27T23:27:39.082681+0000 mgr.y (mgr.14901) 2673 : cluster [DBG] pgmap v2680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:40 smithi130 bash[27579]: audit 2023-08-27T23:27:40.117502+0000 mon.c (mon.2) 1041 : audit [DBG] from='client.? 172.21.15.6:0/4025058614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:40 smithi130 bash[51302]: debug 2023-08-27T23:27:40.481+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:40.484113+0000) 2023-08-27T23:27:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:27:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:27:40] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:27:41.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:41 smithi130 bash[51302]: debug 2023-08-27T23:27:41.481+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:41.484393+0000) 2023-08-27T23:27:42.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:27:42.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:27:42.036Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:42 smithi130 bash[27579]: cluster 2023-08-27T23:27:41.083758+0000 mgr.y (mgr.14901) 2674 : cluster [DBG] pgmap v2681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:42 smithi130 bash[27579]: audit 2023-08-27T23:27:42.535861+0000 mon.a (mon.0) 2266 : audit [DBG] from='client.? 172.21.15.6:0/2327549534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:42 smithi130 bash[51302]: debug 2023-08-27T23:27:42.481+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:42.484652+0000) 2023-08-27T23:27:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:42 smithi006 bash[33918]: cluster 2023-08-27T23:27:41.083758+0000 mgr.y (mgr.14901) 2674 : cluster [DBG] pgmap v2681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:42 smithi006 bash[33918]: audit 2023-08-27T23:27:42.535861+0000 mon.a (mon.0) 2266 : audit [DBG] from='client.? 172.21.15.6:0/2327549534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:42 smithi006 bash[27292]: cluster 2023-08-27T23:27:41.083758+0000 mgr.y (mgr.14901) 2674 : cluster [DBG] pgmap v2681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:42 smithi006 bash[27292]: audit 2023-08-27T23:27:42.535861+0000 mon.a (mon.0) 2266 : audit [DBG] from='client.? 172.21.15.6:0/2327549534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:43.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:43 smithi130 bash[51302]: debug 2023-08-27T23:27:43.481+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:43.484960+0000) 2023-08-27T23:27:44.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:27:43.939Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:27:43.942Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:44 smithi130 bash[27579]: cluster 2023-08-27T23:27:43.084327+0000 mgr.y (mgr.14901) 2675 : cluster [DBG] pgmap v2682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:44 smithi130 bash[51302]: debug 2023-08-27T23:27:44.481+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:44.485265+0000) 2023-08-27T23:27:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:44 smithi006 bash[33918]: cluster 2023-08-27T23:27:43.084327+0000 mgr.y (mgr.14901) 2675 : cluster [DBG] pgmap v2682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:45.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:44 smithi006 bash[27292]: cluster 2023-08-27T23:27:43.084327+0000 mgr.y (mgr.14901) 2675 : cluster [DBG] pgmap v2682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:45.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:45 smithi130 bash[27579]: audit 2023-08-27T23:27:44.959212+0000 mon.a (mon.0) 2267 : audit [DBG] from='client.? 172.21.15.6:0/2074130283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:45.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:45 smithi130 bash[51302]: debug 2023-08-27T23:27:45.485+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:45.485540+0000) 2023-08-27T23:27:46.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:45 smithi006 bash[33918]: audit 2023-08-27T23:27:44.959212+0000 mon.a (mon.0) 2267 : audit [DBG] from='client.? 172.21.15.6:0/2074130283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:46.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:45 smithi006 bash[27292]: audit 2023-08-27T23:27:44.959212+0000 mon.a (mon.0) 2267 : audit [DBG] from='client.? 172.21.15.6:0/2074130283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:46 smithi130 bash[27579]: cluster 2023-08-27T23:27:45.085480+0000 mgr.y (mgr.14901) 2676 : cluster [DBG] pgmap v2683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:46.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:46 smithi130 bash[51302]: debug 2023-08-27T23:27:46.481+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:46.485820+0000) 2023-08-27T23:27:47.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:46 smithi006 bash[27292]: cluster 2023-08-27T23:27:45.085480+0000 mgr.y (mgr.14901) 2676 : cluster [DBG] pgmap v2683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:47.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:46 smithi006 bash[33918]: cluster 2023-08-27T23:27:45.085480+0000 mgr.y (mgr.14901) 2676 : cluster [DBG] pgmap v2683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:47.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:47 smithi130 bash[27579]: audit 2023-08-27T23:27:47.371731+0000 mon.c (mon.2) 1042 : audit [DBG] from='client.? 172.21.15.6:0/3261679958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:47.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:47 smithi130 bash[51302]: debug 2023-08-27T23:27:47.485+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:47.486139+0000) 2023-08-27T23:27:48.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:47 smithi006 bash[33918]: audit 2023-08-27T23:27:47.371731+0000 mon.c (mon.2) 1042 : audit [DBG] from='client.? 172.21.15.6:0/3261679958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:48.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:47 smithi006 bash[27292]: audit 2023-08-27T23:27:47.371731+0000 mon.c (mon.2) 1042 : audit [DBG] from='client.? 172.21.15.6:0/3261679958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:48.746 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:48 smithi130 bash[51302]: debug 2023-08-27T23:27:48.485+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:48.486427+0000) 2023-08-27T23:27:49.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:48 smithi130 bash[27579]: cluster 2023-08-27T23:27:47.086242+0000 mgr.y (mgr.14901) 2677 : cluster [DBG] pgmap v2684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:49.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:48 smithi006 bash[33918]: cluster 2023-08-27T23:27:47.086242+0000 mgr.y (mgr.14901) 2677 : cluster [DBG] pgmap v2684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:49.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:48 smithi006 bash[27292]: cluster 2023-08-27T23:27:47.086242+0000 mgr.y (mgr.14901) 2677 : cluster [DBG] pgmap v2684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:49.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:49 smithi130 bash[51302]: debug 2023-08-27T23:27:49.485+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:49.486709+0000) 2023-08-27T23:27:50.750 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:27:50.476Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:50.750 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:27:50.478Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:50.751 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:50 smithi130 bash[51302]: debug 2023-08-27T23:27:50.485+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:50.486971+0000) 2023-08-27T23:27:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:50 smithi130 bash[27579]: cluster 2023-08-27T23:27:49.087017+0000 mgr.y (mgr.14901) 2678 : cluster [DBG] pgmap v2685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:50 smithi130 bash[27579]: audit 2023-08-27T23:27:49.792991+0000 mon.c (mon.2) 1043 : audit [DBG] from='client.? 172.21.15.6:0/3149355793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:27:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:27:50] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:27:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:50 smithi006 bash[33918]: cluster 2023-08-27T23:27:49.087017+0000 mgr.y (mgr.14901) 2678 : cluster [DBG] pgmap v2685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:50 smithi006 bash[33918]: audit 2023-08-27T23:27:49.792991+0000 mon.c (mon.2) 1043 : audit [DBG] from='client.? 172.21.15.6:0/3149355793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:50 smithi006 bash[27292]: cluster 2023-08-27T23:27:49.087017+0000 mgr.y (mgr.14901) 2678 : cluster [DBG] pgmap v2685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:50 smithi006 bash[27292]: audit 2023-08-27T23:27:49.792991+0000 mon.c (mon.2) 1043 : audit [DBG] from='client.? 172.21.15.6:0/3149355793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:51.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:51 smithi130 bash[51302]: debug 2023-08-27T23:27:51.485+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:51.487173+0000) 2023-08-27T23:27:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:27:52.035Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:27:52.038Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:52.755 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:52 smithi130 bash[51302]: debug 2023-08-27T23:27:52.485+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:52.487387+0000) 2023-08-27T23:27:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:52 smithi130 bash[27579]: cluster 2023-08-27T23:27:51.088078+0000 mgr.y (mgr.14901) 2679 : cluster [DBG] pgmap v2686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:52 smithi130 bash[27579]: audit 2023-08-27T23:27:52.207020+0000 mon.c (mon.2) 1044 : audit [DBG] from='client.? 172.21.15.6:0/653317530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:53.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:52 smithi006 bash[33918]: cluster 2023-08-27T23:27:51.088078+0000 mgr.y (mgr.14901) 2679 : cluster [DBG] pgmap v2686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:53.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:52 smithi006 bash[33918]: audit 2023-08-27T23:27:52.207020+0000 mon.c (mon.2) 1044 : audit [DBG] from='client.? 172.21.15.6:0/653317530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:52 smithi006 bash[27292]: cluster 2023-08-27T23:27:51.088078+0000 mgr.y (mgr.14901) 2679 : cluster [DBG] pgmap v2686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:52 smithi006 bash[27292]: audit 2023-08-27T23:27:52.207020+0000 mon.c (mon.2) 1044 : audit [DBG] from='client.? 172.21.15.6:0/653317530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:53.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:53 smithi130 bash[51302]: debug 2023-08-27T23:27:53.485+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:53.487603+0000) 2023-08-27T23:27:54.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:27:53.941Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:27:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:27:53.942Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:27:54.760 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:54 smithi130 bash[51302]: debug 2023-08-27T23:27:54.485+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:54.487816+0000) 2023-08-27T23:27:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:54 smithi130 bash[27579]: cluster 2023-08-27T23:27:53.088678+0000 mgr.y (mgr.14901) 2680 : cluster [DBG] pgmap v2687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:54 smithi130 bash[27579]: audit 2023-08-27T23:27:54.623232+0000 mon.a (mon.0) 2268 : audit [DBG] from='client.? 172.21.15.6:0/2842039434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:55.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:54 smithi006 bash[27292]: cluster 2023-08-27T23:27:53.088678+0000 mgr.y (mgr.14901) 2680 : cluster [DBG] pgmap v2687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:54 smithi006 bash[27292]: audit 2023-08-27T23:27:54.623232+0000 mon.a (mon.0) 2268 : audit [DBG] from='client.? 172.21.15.6:0/2842039434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:54 smithi006 bash[33918]: cluster 2023-08-27T23:27:53.088678+0000 mgr.y (mgr.14901) 2680 : cluster [DBG] pgmap v2687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:54 smithi006 bash[33918]: audit 2023-08-27T23:27:54.623232+0000 mon.a (mon.0) 2268 : audit [DBG] from='client.? 172.21.15.6:0/2842039434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:55.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:55 smithi130 bash[51302]: debug 2023-08-27T23:27:55.484+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:55.488064+0000) 2023-08-27T23:27:56.765 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:56 smithi130 bash[51302]: debug 2023-08-27T23:27:56.484+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:56.488265+0000) 2023-08-27T23:27:57.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:56 smithi130 bash[27579]: cluster 2023-08-27T23:27:55.089836+0000 mgr.y (mgr.14901) 2681 : cluster [DBG] pgmap v2688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:57.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:56 smithi006 bash[33918]: cluster 2023-08-27T23:27:55.089836+0000 mgr.y (mgr.14901) 2681 : cluster [DBG] pgmap v2688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:57.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:56 smithi006 bash[27292]: cluster 2023-08-27T23:27:55.089836+0000 mgr.y (mgr.14901) 2681 : cluster [DBG] pgmap v2688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:57.774 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:57 smithi130 bash[51302]: debug 2023-08-27T23:27:57.484+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:57.488496+0000) 2023-08-27T23:27:58.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:57 smithi130 bash[27579]: audit 2023-08-27T23:27:57.036057+0000 mon.c (mon.2) 1045 : audit [DBG] from='client.? 172.21.15.6:0/3941397634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:58.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:57 smithi006 bash[27292]: audit 2023-08-27T23:27:57.036057+0000 mon.c (mon.2) 1045 : audit [DBG] from='client.? 172.21.15.6:0/3941397634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:58.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:57 smithi006 bash[33918]: audit 2023-08-27T23:27:57.036057+0000 mon.c (mon.2) 1045 : audit [DBG] from='client.? 172.21.15.6:0/3941397634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:58.777 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:58 smithi130 bash[51302]: debug 2023-08-27T23:27:58.484+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:58.488748+0000) 2023-08-27T23:27:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:58 smithi130 bash[27579]: cluster 2023-08-27T23:27:57.090481+0000 mgr.y (mgr.14901) 2682 : cluster [DBG] pgmap v2689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:59.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:58 smithi006 bash[33918]: cluster 2023-08-27T23:27:57.090481+0000 mgr.y (mgr.14901) 2682 : cluster [DBG] pgmap v2689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:58 smithi006 bash[27292]: cluster 2023-08-27T23:27:57.090481+0000 mgr.y (mgr.14901) 2682 : cluster [DBG] pgmap v2689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:27:59.784 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:27:59 smithi130 bash[51302]: debug 2023-08-27T23:27:59.484+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:59.488997+0000) 2023-08-27T23:28:00.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:27:59 smithi130 bash[27579]: audit 2023-08-27T23:27:59.452775+0000 mon.a (mon.0) 2269 : audit [DBG] from='client.? 172.21.15.6:0/1000290684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:00.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:27:59 smithi006 bash[27292]: audit 2023-08-27T23:27:59.452775+0000 mon.a (mon.0) 2269 : audit [DBG] from='client.? 172.21.15.6:0/1000290684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:00.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:27:59 smithi006 bash[33918]: audit 2023-08-27T23:27:59.452775+0000 mon.a (mon.0) 2269 : audit [DBG] from='client.? 172.21.15.6:0/1000290684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:00.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:28:00.476Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:00.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:28:00.478Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:00.787 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:00 smithi130 bash[51302]: debug 2023-08-27T23:28:00.484+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:00.489286+0000) 2023-08-27T23:28:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:00 smithi130 bash[27579]: cluster 2023-08-27T23:27:59.091262+0000 mgr.y (mgr.14901) 2683 : cluster [DBG] pgmap v2690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:28:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:28:00] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:28:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:00 smithi006 bash[33918]: cluster 2023-08-27T23:27:59.091262+0000 mgr.y (mgr.14901) 2683 : cluster [DBG] pgmap v2690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:01.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:00 smithi006 bash[27292]: cluster 2023-08-27T23:27:59.091262+0000 mgr.y (mgr.14901) 2683 : cluster [DBG] pgmap v2690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:01.793 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:01 smithi130 bash[51302]: debug 2023-08-27T23:28:01.484+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:01.489542+0000) 2023-08-27T23:28:02.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:01 smithi130 bash[27579]: cluster 2023-08-27T23:28:01.092284+0000 mgr.y (mgr.14901) 2684 : cluster [DBG] pgmap v2691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:02.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:01 smithi006 bash[33918]: cluster 2023-08-27T23:28:01.092284+0000 mgr.y (mgr.14901) 2684 : cluster [DBG] pgmap v2691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:02.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:01 smithi006 bash[27292]: cluster 2023-08-27T23:28:01.092284+0000 mgr.y (mgr.14901) 2684 : cluster [DBG] pgmap v2691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:02.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:28:02.036Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:02.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:28:02.039Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:02.796 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:02 smithi130 bash[51302]: debug 2023-08-27T23:28:02.484+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:02.489835+0000) 2023-08-27T23:28:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:02 smithi130 bash[27579]: audit 2023-08-27T23:28:01.864633+0000 mon.a (mon.0) 2270 : audit [DBG] from='client.? 172.21.15.6:0/2954703277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:03.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:02 smithi006 bash[33918]: audit 2023-08-27T23:28:01.864633+0000 mon.a (mon.0) 2270 : audit [DBG] from='client.? 172.21.15.6:0/2954703277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:02 smithi006 bash[27292]: audit 2023-08-27T23:28:01.864633+0000 mon.a (mon.0) 2270 : audit [DBG] from='client.? 172.21.15.6:0/2954703277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:03.804 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:03 smithi130 bash[51302]: debug 2023-08-27T23:28:03.484+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:03.490095+0000) 2023-08-27T23:28:04.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:03 smithi130 bash[27579]: cluster 2023-08-27T23:28:03.093001+0000 mgr.y (mgr.14901) 2685 : cluster [DBG] pgmap v2692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:04.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:03 smithi006 bash[27292]: cluster 2023-08-27T23:28:03.093001+0000 mgr.y (mgr.14901) 2685 : cluster [DBG] pgmap v2692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:28:03.941Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:28:03.943Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:04.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:03 smithi006 bash[33918]: cluster 2023-08-27T23:28:03.093001+0000 mgr.y (mgr.14901) 2685 : cluster [DBG] pgmap v2692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:04.808 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:04 smithi130 bash[51302]: debug 2023-08-27T23:28:04.484+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:04.490407+0000) 2023-08-27T23:28:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:04 smithi130 bash[27579]: audit 2023-08-27T23:28:04.282348+0000 mon.c (mon.2) 1046 : audit [DBG] from='client.? 172.21.15.6:0/1526966861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:04 smithi006 bash[27292]: audit 2023-08-27T23:28:04.282348+0000 mon.c (mon.2) 1046 : audit [DBG] from='client.? 172.21.15.6:0/1526966861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:04 smithi006 bash[33918]: audit 2023-08-27T23:28:04.282348+0000 mon.c (mon.2) 1046 : audit [DBG] from='client.? 172.21.15.6:0/1526966861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:05.814 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:05 smithi130 bash[51302]: debug 2023-08-27T23:28:05.488+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:05.490679+0000) 2023-08-27T23:28:06.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:05 smithi130 bash[27579]: cluster 2023-08-27T23:28:05.094146+0000 mgr.y (mgr.14901) 2686 : cluster [DBG] pgmap v2693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:06.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:05 smithi006 bash[33918]: cluster 2023-08-27T23:28:05.094146+0000 mgr.y (mgr.14901) 2686 : cluster [DBG] pgmap v2693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:06.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:05 smithi006 bash[27292]: cluster 2023-08-27T23:28:05.094146+0000 mgr.y (mgr.14901) 2686 : cluster [DBG] pgmap v2693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:06.817 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:06 smithi130 bash[51302]: debug 2023-08-27T23:28:06.488+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:06.490967+0000) 2023-08-27T23:28:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:06 smithi130 bash[27579]: audit 2023-08-27T23:28:06.697579+0000 mon.c (mon.2) 1047 : audit [DBG] from='client.? 172.21.15.6:0/1771878009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:07.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:06 smithi006 bash[33918]: audit 2023-08-27T23:28:06.697579+0000 mon.c (mon.2) 1047 : audit [DBG] from='client.? 172.21.15.6:0/1771878009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:07.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:06 smithi006 bash[27292]: audit 2023-08-27T23:28:06.697579+0000 mon.c (mon.2) 1047 : audit [DBG] from='client.? 172.21.15.6:0/1771878009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:07.826 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:07 smithi130 bash[51302]: debug 2023-08-27T23:28:07.488+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:07.491187+0000) 2023-08-27T23:28:08.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:07 smithi130 bash[27579]: cluster 2023-08-27T23:28:07.094930+0000 mgr.y (mgr.14901) 2687 : cluster [DBG] pgmap v2694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:08.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:07 smithi006 bash[33918]: cluster 2023-08-27T23:28:07.094930+0000 mgr.y (mgr.14901) 2687 : cluster [DBG] pgmap v2694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:08.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:07 smithi006 bash[27292]: cluster 2023-08-27T23:28:07.094930+0000 mgr.y (mgr.14901) 2687 : cluster [DBG] pgmap v2694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:08.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:08 smithi130 bash[51302]: debug 2023-08-27T23:28:08.488+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:08.491496+0000) 2023-08-27T23:28:09.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:09 smithi006 bash[27292]: audit 2023-08-27T23:28:09.107556+0000 mon.a (mon.0) 2271 : audit [DBG] from='client.? 172.21.15.6:0/2152335352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:09.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:09 smithi006 bash[33918]: audit 2023-08-27T23:28:09.107556+0000 mon.a (mon.0) 2271 : audit [DBG] from='client.? 172.21.15.6:0/2152335352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:09.492 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:09 smithi130 bash[27579]: audit 2023-08-27T23:28:09.107556+0000 mon.a (mon.0) 2271 : audit [DBG] from='client.? 172.21.15.6:0/2152335352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:09.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:09 smithi130 bash[51302]: debug 2023-08-27T23:28:09.488+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:09.491790+0000) 2023-08-27T23:28:10.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:10 smithi006 bash[27292]: cluster 2023-08-27T23:28:09.095504+0000 mgr.y (mgr.14901) 2688 : cluster [DBG] pgmap v2695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:10.756 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:28:10.477Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:10.756 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:28:10.479Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:10.756 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:10 smithi006 bash[33918]: cluster 2023-08-27T23:28:09.095504+0000 mgr.y (mgr.14901) 2688 : cluster [DBG] pgmap v2695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:10 smithi130 bash[27579]: cluster 2023-08-27T23:28:09.095504+0000 mgr.y (mgr.14901) 2688 : cluster [DBG] pgmap v2695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:10 smithi130 bash[51302]: debug 2023-08-27T23:28:10.488+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:10.492085+0000) 2023-08-27T23:28:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:28:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:28:10] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:28:11.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:11 smithi130 bash[27579]: audit 2023-08-27T23:28:11.525314+0000 mon.a (mon.0) 2272 : audit [DBG] from='client.? 172.21.15.6:0/2572727984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:11.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:11 smithi130 bash[51302]: debug 2023-08-27T23:28:11.488+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:11.492403+0000) 2023-08-27T23:28:11.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:11 smithi006 bash[27292]: audit 2023-08-27T23:28:11.525314+0000 mon.a (mon.0) 2272 : audit [DBG] from='client.? 172.21.15.6:0/2572727984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:11.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:11 smithi006 bash[33918]: audit 2023-08-27T23:28:11.525314+0000 mon.a (mon.0) 2272 : audit [DBG] from='client.? 172.21.15.6:0/2572727984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:28:12.037Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:28:12.039Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:12 smithi130 bash[27579]: cluster 2023-08-27T23:28:11.096561+0000 mgr.y (mgr.14901) 2689 : cluster [DBG] pgmap v2696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:12 smithi130 bash[27579]: audit 2023-08-27T23:28:12.204666+0000 mon.b (mon.1) 376 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:28:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:12 smithi130 bash[51302]: debug 2023-08-27T23:28:12.488+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:12.492728+0000) 2023-08-27T23:28:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:12 smithi006 bash[27292]: cluster 2023-08-27T23:28:11.096561+0000 mgr.y (mgr.14901) 2689 : cluster [DBG] pgmap v2696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:12 smithi006 bash[27292]: audit 2023-08-27T23:28:12.204666+0000 mon.b (mon.1) 376 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:28:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:12 smithi006 bash[33918]: cluster 2023-08-27T23:28:11.096561+0000 mgr.y (mgr.14901) 2689 : cluster [DBG] pgmap v2696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:12 smithi006 bash[33918]: audit 2023-08-27T23:28:12.204666+0000 mon.b (mon.1) 376 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:28:13.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:13 smithi130 bash[51302]: debug 2023-08-27T23:28:13.488+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:13.493027+0000) 2023-08-27T23:28:14.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:28:13.942Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:28:13.943Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:14.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:14 smithi130 bash[27579]: cluster 2023-08-27T23:28:13.097255+0000 mgr.y (mgr.14901) 2690 : cluster [DBG] pgmap v2697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:14.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:14 smithi130 bash[27579]: audit 2023-08-27T23:28:13.940574+0000 mon.a (mon.0) 2273 : audit [DBG] from='client.? 172.21.15.6:0/2864504683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:14.870 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:14 smithi130 bash[51302]: debug 2023-08-27T23:28:14.488+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:14.493324+0000) 2023-08-27T23:28:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:14 smithi006 bash[27292]: cluster 2023-08-27T23:28:13.097255+0000 mgr.y (mgr.14901) 2690 : cluster [DBG] pgmap v2697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:14 smithi006 bash[27292]: audit 2023-08-27T23:28:13.940574+0000 mon.a (mon.0) 2273 : audit [DBG] from='client.? 172.21.15.6:0/2864504683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:15.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:14 smithi006 bash[33918]: cluster 2023-08-27T23:28:13.097255+0000 mgr.y (mgr.14901) 2690 : cluster [DBG] pgmap v2697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:15.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:14 smithi006 bash[33918]: audit 2023-08-27T23:28:13.940574+0000 mon.a (mon.0) 2273 : audit [DBG] from='client.? 172.21.15.6:0/2864504683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:15.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:15 smithi130 bash[51302]: debug 2023-08-27T23:28:15.488+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:15.493533+0000) 2023-08-27T23:28:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:16 smithi130 bash[27579]: cluster 2023-08-27T23:28:15.098357+0000 mgr.y (mgr.14901) 2691 : cluster [DBG] pgmap v2698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:16 smithi130 bash[27579]: audit 2023-08-27T23:28:16.355525+0000 mon.c (mon.2) 1048 : audit [DBG] from='client.? 172.21.15.6:0/475720974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:16.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:16 smithi130 bash[51302]: debug 2023-08-27T23:28:16.488+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:16.493857+0000) 2023-08-27T23:28:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:16 smithi006 bash[33918]: cluster 2023-08-27T23:28:15.098357+0000 mgr.y (mgr.14901) 2691 : cluster [DBG] pgmap v2698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:16 smithi006 bash[33918]: audit 2023-08-27T23:28:16.355525+0000 mon.c (mon.2) 1048 : audit [DBG] from='client.? 172.21.15.6:0/475720974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:17.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:16 smithi006 bash[27292]: cluster 2023-08-27T23:28:15.098357+0000 mgr.y (mgr.14901) 2691 : cluster [DBG] pgmap v2698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:17.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:16 smithi006 bash[27292]: audit 2023-08-27T23:28:16.355525+0000 mon.c (mon.2) 1048 : audit [DBG] from='client.? 172.21.15.6:0/475720974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:17.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:17 smithi130 bash[51302]: debug 2023-08-27T23:28:17.488+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:17.494144+0000) 2023-08-27T23:28:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:18 smithi130 bash[51302]: debug 2023-08-27T23:28:18.488+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:18.494391+0000) 2023-08-27T23:28:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:18 smithi130 bash[27579]: cluster 2023-08-27T23:28:17.099043+0000 mgr.y (mgr.14901) 2692 : cluster [DBG] pgmap v2699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:18.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:18 smithi130 bash[27579]: audit 2023-08-27T23:28:18.130058+0000 mon.a (mon.0) 2274 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:28:18.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:18 smithi130 bash[27579]: audit 2023-08-27T23:28:18.533148+0000 mon.a (mon.0) 2275 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:28:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:18 smithi006 bash[33918]: cluster 2023-08-27T23:28:17.099043+0000 mgr.y (mgr.14901) 2692 : cluster [DBG] pgmap v2699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:18 smithi006 bash[33918]: audit 2023-08-27T23:28:18.130058+0000 mon.a (mon.0) 2274 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:28:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:18 smithi006 bash[33918]: audit 2023-08-27T23:28:18.533148+0000 mon.a (mon.0) 2275 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:28:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:18 smithi006 bash[27292]: cluster 2023-08-27T23:28:17.099043+0000 mgr.y (mgr.14901) 2692 : cluster [DBG] pgmap v2699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:18 smithi006 bash[27292]: audit 2023-08-27T23:28:18.130058+0000 mon.a (mon.0) 2274 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:28:19.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:18 smithi006 bash[27292]: audit 2023-08-27T23:28:18.533148+0000 mon.a (mon.0) 2275 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:28:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:19 smithi130 bash[51302]: debug 2023-08-27T23:28:19.488+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:19.494648+0000) 2023-08-27T23:28:19.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:19 smithi130 bash[27579]: audit 2023-08-27T23:28:18.764305+0000 mon.a (mon.0) 2276 : audit [DBG] from='client.? 172.21.15.6:0/2221510972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:19.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:19 smithi130 bash[27579]: audit 2023-08-27T23:28:18.796461+0000 mon.b (mon.1) 377 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:28:19.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:19 smithi130 bash[27579]: audit 2023-08-27T23:28:18.798690+0000 mon.b (mon.1) 378 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:28:19.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:19 smithi130 bash[27579]: audit 2023-08-27T23:28:18.808547+0000 mon.a (mon.0) 2277 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:28:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:19 smithi006 bash[27292]: audit 2023-08-27T23:28:18.764305+0000 mon.a (mon.0) 2276 : audit [DBG] from='client.? 172.21.15.6:0/2221510972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:19 smithi006 bash[27292]: audit 2023-08-27T23:28:18.796461+0000 mon.b (mon.1) 377 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:28:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:19 smithi006 bash[27292]: audit 2023-08-27T23:28:18.798690+0000 mon.b (mon.1) 378 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:28:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:19 smithi006 bash[27292]: audit 2023-08-27T23:28:18.808547+0000 mon.a (mon.0) 2277 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:28:20.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:19 smithi006 bash[33918]: audit 2023-08-27T23:28:18.764305+0000 mon.a (mon.0) 2276 : audit [DBG] from='client.? 172.21.15.6:0/2221510972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:20.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:19 smithi006 bash[33918]: audit 2023-08-27T23:28:18.796461+0000 mon.b (mon.1) 377 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:28:20.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:19 smithi006 bash[33918]: audit 2023-08-27T23:28:18.798690+0000 mon.b (mon.1) 378 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:28:20.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:19 smithi006 bash[33918]: audit 2023-08-27T23:28:18.808547+0000 mon.a (mon.0) 2277 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:28:20.739 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:28:20.477Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:20.739 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:28:20.479Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:20 smithi130 bash[27579]: cluster 2023-08-27T23:28:19.100128+0000 mgr.y (mgr.14901) 2693 : cluster [DBG] pgmap v2700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:20 smithi130 bash[51302]: debug 2023-08-27T23:28:20.492+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:20.494985+0000) 2023-08-27T23:28:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:28:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:28:20] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T23:28:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:20 smithi006 bash[27292]: cluster 2023-08-27T23:28:19.100128+0000 mgr.y (mgr.14901) 2693 : cluster [DBG] pgmap v2700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:21.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:20 smithi006 bash[33918]: cluster 2023-08-27T23:28:19.100128+0000 mgr.y (mgr.14901) 2693 : cluster [DBG] pgmap v2700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:21.747 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:21 smithi130 bash[51302]: debug 2023-08-27T23:28:21.492+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:21.495232+0000) 2023-08-27T23:28:22.040 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:21 smithi006 bash[27292]: audit 2023-08-27T23:28:21.181798+0000 mon.a (mon.0) 2278 : audit [DBG] from='client.? 172.21.15.6:0/1570086040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:22.041 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:21 smithi006 bash[33918]: audit 2023-08-27T23:28:21.181798+0000 mon.a (mon.0) 2278 : audit [DBG] from='client.? 172.21.15.6:0/1570086040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:22.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:21 smithi130 bash[27579]: audit 2023-08-27T23:28:21.181798+0000 mon.a (mon.0) 2278 : audit [DBG] from='client.? 172.21.15.6:0/1570086040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:28:22.038Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:28:22.040Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:22.752 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:22 smithi130 bash[51302]: debug 2023-08-27T23:28:22.492+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:22.495486+0000) 2023-08-27T23:28:23.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:22 smithi130 bash[27579]: cluster 2023-08-27T23:28:21.100901+0000 mgr.y (mgr.14901) 2694 : cluster [DBG] pgmap v2701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:23.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:22 smithi006 bash[27292]: cluster 2023-08-27T23:28:21.100901+0000 mgr.y (mgr.14901) 2694 : cluster [DBG] pgmap v2701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:22 smithi006 bash[33918]: cluster 2023-08-27T23:28:21.100901+0000 mgr.y (mgr.14901) 2694 : cluster [DBG] pgmap v2701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:23.761 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:23 smithi130 bash[51302]: debug 2023-08-27T23:28:23.492+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:23.495734+0000) 2023-08-27T23:28:24.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:23 smithi130 bash[27579]: audit 2023-08-27T23:28:23.594057+0000 mon.c (mon.2) 1049 : audit [DBG] from='client.? 172.21.15.6:0/1076467972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:24.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:23 smithi006 bash[33918]: audit 2023-08-27T23:28:23.594057+0000 mon.c (mon.2) 1049 : audit [DBG] from='client.? 172.21.15.6:0/1076467972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:24.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:23 smithi006 bash[27292]: audit 2023-08-27T23:28:23.594057+0000 mon.c (mon.2) 1049 : audit [DBG] from='client.? 172.21.15.6:0/1076467972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:28:23.942Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:28:23.944Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:24.764 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:24 smithi130 bash[51302]: debug 2023-08-27T23:28:24.492+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:24.496022+0000) 2023-08-27T23:28:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:24 smithi130 bash[27579]: cluster 2023-08-27T23:28:23.101560+0000 mgr.y (mgr.14901) 2695 : cluster [DBG] pgmap v2702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:25.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:24 smithi006 bash[33918]: cluster 2023-08-27T23:28:23.101560+0000 mgr.y (mgr.14901) 2695 : cluster [DBG] pgmap v2702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:24 smithi006 bash[27292]: cluster 2023-08-27T23:28:23.101560+0000 mgr.y (mgr.14901) 2695 : cluster [DBG] pgmap v2702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:25.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:25 smithi130 bash[51302]: debug 2023-08-27T23:28:25.492+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:25.496308+0000) 2023-08-27T23:28:26.770 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:26 smithi130 bash[51302]: debug 2023-08-27T23:28:26.492+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:26.496566+0000) 2023-08-27T23:28:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:26 smithi130 bash[27579]: cluster 2023-08-27T23:28:25.102826+0000 mgr.y (mgr.14901) 2696 : cluster [DBG] pgmap v2703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:26 smithi130 bash[27579]: audit 2023-08-27T23:28:26.015186+0000 mon.a (mon.0) 2279 : audit [DBG] from='client.? 172.21.15.6:0/2322444785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:26 smithi006 bash[33918]: cluster 2023-08-27T23:28:25.102826+0000 mgr.y (mgr.14901) 2696 : cluster [DBG] pgmap v2703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:26 smithi006 bash[33918]: audit 2023-08-27T23:28:26.015186+0000 mon.a (mon.0) 2279 : audit [DBG] from='client.? 172.21.15.6:0/2322444785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:26 smithi006 bash[27292]: cluster 2023-08-27T23:28:25.102826+0000 mgr.y (mgr.14901) 2696 : cluster [DBG] pgmap v2703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:26 smithi006 bash[27292]: audit 2023-08-27T23:28:26.015186+0000 mon.a (mon.0) 2279 : audit [DBG] from='client.? 172.21.15.6:0/2322444785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:27.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:27 smithi130 bash[51302]: debug 2023-08-27T23:28:27.492+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:27.496855+0000) 2023-08-27T23:28:28.775 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:28 smithi130 bash[51302]: debug 2023-08-27T23:28:28.492+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:28.497123+0000) 2023-08-27T23:28:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:28 smithi130 bash[27579]: cluster 2023-08-27T23:28:27.103651+0000 mgr.y (mgr.14901) 2697 : cluster [DBG] pgmap v2704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:28 smithi130 bash[27579]: audit 2023-08-27T23:28:28.428457+0000 mon.c (mon.2) 1050 : audit [DBG] from='client.? 172.21.15.6:0/3162950474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:28 smithi006 bash[33918]: cluster 2023-08-27T23:28:27.103651+0000 mgr.y (mgr.14901) 2697 : cluster [DBG] pgmap v2704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:28 smithi006 bash[33918]: audit 2023-08-27T23:28:28.428457+0000 mon.c (mon.2) 1050 : audit [DBG] from='client.? 172.21.15.6:0/3162950474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:28 smithi006 bash[27292]: cluster 2023-08-27T23:28:27.103651+0000 mgr.y (mgr.14901) 2697 : cluster [DBG] pgmap v2704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:28 smithi006 bash[27292]: audit 2023-08-27T23:28:28.428457+0000 mon.c (mon.2) 1050 : audit [DBG] from='client.? 172.21.15.6:0/3162950474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:29.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:29 smithi130 bash[51302]: debug 2023-08-27T23:28:29.496+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:29.497418+0000) 2023-08-27T23:28:30.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:28:30.478Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:30.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:28:30.479Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:30.780 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:30 smithi130 bash[51302]: debug 2023-08-27T23:28:30.492+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:30.497680+0000) 2023-08-27T23:28:31.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:30 smithi130 bash[27579]: cluster 2023-08-27T23:28:29.104665+0000 mgr.y (mgr.14901) 2698 : cluster [DBG] pgmap v2705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:28:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:28:30] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:28:31.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:30 smithi006 bash[33918]: cluster 2023-08-27T23:28:29.104665+0000 mgr.y (mgr.14901) 2698 : cluster [DBG] pgmap v2705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:31.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:30 smithi006 bash[27292]: cluster 2023-08-27T23:28:29.104665+0000 mgr.y (mgr.14901) 2698 : cluster [DBG] pgmap v2705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:31.787 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:31 smithi130 bash[51302]: debug 2023-08-27T23:28:31.492+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:31.497946+0000) 2023-08-27T23:28:32.041 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:31 smithi006 bash[33918]: audit 2023-08-27T23:28:30.853514+0000 mon.a (mon.0) 2280 : audit [DBG] from='client.? 172.21.15.6:0/105178323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:32.041 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:31 smithi006 bash[27292]: audit 2023-08-27T23:28:30.853514+0000 mon.a (mon.0) 2280 : audit [DBG] from='client.? 172.21.15.6:0/105178323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:32.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:31 smithi130 bash[27579]: audit 2023-08-27T23:28:30.853514+0000 mon.a (mon.0) 2280 : audit [DBG] from='client.? 172.21.15.6:0/105178323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:28:32.039Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:28:32.041Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:32.790 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:32 smithi130 bash[51302]: debug 2023-08-27T23:28:32.492+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:32.498261+0000) 2023-08-27T23:28:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:32 smithi130 bash[27579]: cluster 2023-08-27T23:28:31.105532+0000 mgr.y (mgr.14901) 2699 : cluster [DBG] pgmap v2706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:33.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:32 smithi006 bash[33918]: cluster 2023-08-27T23:28:31.105532+0000 mgr.y (mgr.14901) 2699 : cluster [DBG] pgmap v2706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:32 smithi006 bash[27292]: cluster 2023-08-27T23:28:31.105532+0000 mgr.y (mgr.14901) 2699 : cluster [DBG] pgmap v2706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:33.793 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:33 smithi130 bash[51302]: debug 2023-08-27T23:28:33.492+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:33.498541+0000) 2023-08-27T23:28:34.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:33 smithi130 bash[27579]: audit 2023-08-27T23:28:33.268536+0000 mon.a (mon.0) 2281 : audit [DBG] from='client.? 172.21.15.6:0/3262042484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:34.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:33 smithi006 bash[33918]: audit 2023-08-27T23:28:33.268536+0000 mon.a (mon.0) 2281 : audit [DBG] from='client.? 172.21.15.6:0/3262042484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:34.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:33 smithi006 bash[27292]: audit 2023-08-27T23:28:33.268536+0000 mon.a (mon.0) 2281 : audit [DBG] from='client.? 172.21.15.6:0/3262042484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:28:33.943Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:28:33.946Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:34.795 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:34 smithi130 bash[51302]: debug 2023-08-27T23:28:34.492+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:34.498810+0000) 2023-08-27T23:28:35.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:34 smithi130 bash[27579]: cluster 2023-08-27T23:28:33.106180+0000 mgr.y (mgr.14901) 2700 : cluster [DBG] pgmap v2707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:35.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:34 smithi006 bash[27292]: cluster 2023-08-27T23:28:33.106180+0000 mgr.y (mgr.14901) 2700 : cluster [DBG] pgmap v2707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:34 smithi006 bash[33918]: cluster 2023-08-27T23:28:33.106180+0000 mgr.y (mgr.14901) 2700 : cluster [DBG] pgmap v2707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:35.803 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:35 smithi130 bash[51302]: debug 2023-08-27T23:28:35.496+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:35.499105+0000) 2023-08-27T23:28:36.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:35 smithi130 bash[27579]: cluster 2023-08-27T23:28:35.107430+0000 mgr.y (mgr.14901) 2701 : cluster [DBG] pgmap v2708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:36.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:35 smithi130 bash[27579]: audit 2023-08-27T23:28:35.672831+0000 mon.a (mon.0) 2282 : audit [DBG] from='client.? 172.21.15.6:0/873924675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:36.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:35 smithi006 bash[33918]: cluster 2023-08-27T23:28:35.107430+0000 mgr.y (mgr.14901) 2701 : cluster [DBG] pgmap v2708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:36.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:35 smithi006 bash[33918]: audit 2023-08-27T23:28:35.672831+0000 mon.a (mon.0) 2282 : audit [DBG] from='client.? 172.21.15.6:0/873924675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:36.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:35 smithi006 bash[27292]: cluster 2023-08-27T23:28:35.107430+0000 mgr.y (mgr.14901) 2701 : cluster [DBG] pgmap v2708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:36.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:35 smithi006 bash[27292]: audit 2023-08-27T23:28:35.672831+0000 mon.a (mon.0) 2282 : audit [DBG] from='client.? 172.21.15.6:0/873924675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:36.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:36 smithi130 bash[51302]: debug 2023-08-27T23:28:36.496+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:36.499340+0000) 2023-08-27T23:28:37.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:37 smithi130 bash[51302]: debug 2023-08-27T23:28:37.496+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:37.499617+0000) 2023-08-27T23:28:38.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:38 smithi006 bash[27292]: cluster 2023-08-27T23:28:37.108135+0000 mgr.y (mgr.14901) 2702 : cluster [DBG] pgmap v2709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:38.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:38 smithi006 bash[27292]: audit 2023-08-27T23:28:38.085339+0000 mon.a (mon.0) 2283 : audit [DBG] from='client.? 172.21.15.6:0/4100820673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:38.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:38 smithi006 bash[33918]: cluster 2023-08-27T23:28:37.108135+0000 mgr.y (mgr.14901) 2702 : cluster [DBG] pgmap v2709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:38.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:38 smithi006 bash[33918]: audit 2023-08-27T23:28:38.085339+0000 mon.a (mon.0) 2283 : audit [DBG] from='client.? 172.21.15.6:0/4100820673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:38.501 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:38 smithi130 bash[27579]: cluster 2023-08-27T23:28:37.108135+0000 mgr.y (mgr.14901) 2702 : cluster [DBG] pgmap v2709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:38.501 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:38 smithi130 bash[27579]: audit 2023-08-27T23:28:38.085339+0000 mon.a (mon.0) 2283 : audit [DBG] from='client.? 172.21.15.6:0/4100820673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:38.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:38 smithi130 bash[51302]: debug 2023-08-27T23:28:38.496+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:38.499941+0000) 2023-08-27T23:28:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:39 smithi130 bash[51302]: debug 2023-08-27T23:28:39.496+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:39.500212+0000) 2023-08-27T23:28:40.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:28:40.478Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:40.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:28:40.479Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:40.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:40 smithi006 bash[27292]: cluster 2023-08-27T23:28:39.109121+0000 mgr.y (mgr.14901) 2703 : cluster [DBG] pgmap v2710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:40.756 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:40 smithi006 bash[27292]: audit 2023-08-27T23:28:40.501756+0000 mon.c (mon.2) 1051 : audit [DBG] from='client.? 172.21.15.6:0/2746868189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:40.756 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:40 smithi006 bash[33918]: cluster 2023-08-27T23:28:39.109121+0000 mgr.y (mgr.14901) 2703 : cluster [DBG] pgmap v2710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:40.756 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:40 smithi006 bash[33918]: audit 2023-08-27T23:28:40.501756+0000 mon.c (mon.2) 1051 : audit [DBG] from='client.? 172.21.15.6:0/2746868189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:40 smithi130 bash[51302]: debug 2023-08-27T23:28:40.496+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:40.500473+0000) 2023-08-27T23:28:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:40 smithi130 bash[27579]: cluster 2023-08-27T23:28:39.109121+0000 mgr.y (mgr.14901) 2703 : cluster [DBG] pgmap v2710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:40 smithi130 bash[27579]: audit 2023-08-27T23:28:40.501756+0000 mon.c (mon.2) 1051 : audit [DBG] from='client.? 172.21.15.6:0/2746868189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:28:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:28:40] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:28:41.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:41 smithi130 bash[51302]: debug 2023-08-27T23:28:41.496+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:41.500683+0000) 2023-08-27T23:28:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:28:42.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:28:42.041Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:42 smithi130 bash[27579]: cluster 2023-08-27T23:28:41.109932+0000 mgr.y (mgr.14901) 2704 : cluster [DBG] pgmap v2711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:42 smithi130 bash[51302]: debug 2023-08-27T23:28:42.496+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:42.500945+0000) 2023-08-27T23:28:43.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:42 smithi006 bash[33918]: cluster 2023-08-27T23:28:41.109932+0000 mgr.y (mgr.14901) 2704 : cluster [DBG] pgmap v2711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:42 smithi006 bash[27292]: cluster 2023-08-27T23:28:41.109932+0000 mgr.y (mgr.14901) 2704 : cluster [DBG] pgmap v2711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:43.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:43 smithi130 bash[27579]: audit 2023-08-27T23:28:42.928252+0000 mon.a (mon.0) 2284 : audit [DBG] from='client.? 172.21.15.6:0/2432841060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:43.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:43 smithi130 bash[51302]: debug 2023-08-27T23:28:43.496+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:43.501223+0000) 2023-08-27T23:28:44.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:43 smithi006 bash[33918]: audit 2023-08-27T23:28:42.928252+0000 mon.a (mon.0) 2284 : audit [DBG] from='client.? 172.21.15.6:0/2432841060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:44.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:43 smithi006 bash[27292]: audit 2023-08-27T23:28:42.928252+0000 mon.a (mon.0) 2284 : audit [DBG] from='client.? 172.21.15.6:0/2432841060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:28:43.944Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:28:43.946Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:44 smithi130 bash[27579]: cluster 2023-08-27T23:28:43.110659+0000 mgr.y (mgr.14901) 2705 : cluster [DBG] pgmap v2712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:44 smithi130 bash[51302]: debug 2023-08-27T23:28:44.496+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:44.501499+0000) 2023-08-27T23:28:45.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:44 smithi006 bash[27292]: cluster 2023-08-27T23:28:43.110659+0000 mgr.y (mgr.14901) 2705 : cluster [DBG] pgmap v2712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:44 smithi006 bash[33918]: cluster 2023-08-27T23:28:43.110659+0000 mgr.y (mgr.14901) 2705 : cluster [DBG] pgmap v2712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:45.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:45 smithi130 bash[51302]: debug 2023-08-27T23:28:45.496+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:45.501767+0000) 2023-08-27T23:28:45.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:45 smithi130 bash[27579]: audit 2023-08-27T23:28:45.341664+0000 mon.c (mon.2) 1052 : audit [DBG] from='client.? 172.21.15.6:0/2325818375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:46.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:45 smithi006 bash[27292]: audit 2023-08-27T23:28:45.341664+0000 mon.c (mon.2) 1052 : audit [DBG] from='client.? 172.21.15.6:0/2325818375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:46.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:45 smithi006 bash[33918]: audit 2023-08-27T23:28:45.341664+0000 mon.c (mon.2) 1052 : audit [DBG] from='client.? 172.21.15.6:0/2325818375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:46.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:46 smithi130 bash[51302]: debug 2023-08-27T23:28:46.496+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:46.502048+0000) 2023-08-27T23:28:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:46 smithi130 bash[27579]: cluster 2023-08-27T23:28:45.111772+0000 mgr.y (mgr.14901) 2706 : cluster [DBG] pgmap v2713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:47.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:46 smithi006 bash[33918]: cluster 2023-08-27T23:28:45.111772+0000 mgr.y (mgr.14901) 2706 : cluster [DBG] pgmap v2713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:47.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:46 smithi006 bash[27292]: cluster 2023-08-27T23:28:45.111772+0000 mgr.y (mgr.14901) 2706 : cluster [DBG] pgmap v2713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:47.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:47 smithi130 bash[51302]: debug 2023-08-27T23:28:47.496+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:47.502361+0000) 2023-08-27T23:28:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:48 smithi130 bash[27579]: cluster 2023-08-27T23:28:47.112602+0000 mgr.y (mgr.14901) 2707 : cluster [DBG] pgmap v2714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:48 smithi130 bash[27579]: audit 2023-08-27T23:28:47.751359+0000 mon.a (mon.0) 2285 : audit [DBG] from='client.? 172.21.15.6:0/350991975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:48 smithi130 bash[51302]: debug 2023-08-27T23:28:48.496+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:48.502644+0000) 2023-08-27T23:28:49.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:48 smithi006 bash[27292]: cluster 2023-08-27T23:28:47.112602+0000 mgr.y (mgr.14901) 2707 : cluster [DBG] pgmap v2714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:49.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:48 smithi006 bash[27292]: audit 2023-08-27T23:28:47.751359+0000 mon.a (mon.0) 2285 : audit [DBG] from='client.? 172.21.15.6:0/350991975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:49.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:48 smithi006 bash[33918]: cluster 2023-08-27T23:28:47.112602+0000 mgr.y (mgr.14901) 2707 : cluster [DBG] pgmap v2714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:49.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:48 smithi006 bash[33918]: audit 2023-08-27T23:28:47.751359+0000 mon.a (mon.0) 2285 : audit [DBG] from='client.? 172.21.15.6:0/350991975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:49.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:49 smithi130 bash[51302]: debug 2023-08-27T23:28:49.500+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:49.502936+0000) 2023-08-27T23:28:50.750 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:28:50.479Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:50.750 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:28:50.481Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:50 smithi130 bash[27579]: cluster 2023-08-27T23:28:49.113655+0000 mgr.y (mgr.14901) 2708 : cluster [DBG] pgmap v2715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:50 smithi130 bash[27579]: audit 2023-08-27T23:28:50.163893+0000 mon.c (mon.2) 1053 : audit [DBG] from='client.? 172.21.15.6:0/1417407002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:50 smithi130 bash[51302]: debug 2023-08-27T23:28:50.500+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:50.503187+0000) 2023-08-27T23:28:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:28:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:28:50] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:28:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:50 smithi006 bash[33918]: cluster 2023-08-27T23:28:49.113655+0000 mgr.y (mgr.14901) 2708 : cluster [DBG] pgmap v2715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:50 smithi006 bash[33918]: audit 2023-08-27T23:28:50.163893+0000 mon.c (mon.2) 1053 : audit [DBG] from='client.? 172.21.15.6:0/1417407002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:50 smithi006 bash[27292]: cluster 2023-08-27T23:28:49.113655+0000 mgr.y (mgr.14901) 2708 : cluster [DBG] pgmap v2715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:50 smithi006 bash[27292]: audit 2023-08-27T23:28:50.163893+0000 mon.c (mon.2) 1053 : audit [DBG] from='client.? 172.21.15.6:0/1417407002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:51.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:51 smithi130 bash[51302]: debug 2023-08-27T23:28:51.500+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:51.503434+0000) 2023-08-27T23:28:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:28:52.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:28:52.042Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:52.755 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:52 smithi130 bash[51302]: debug 2023-08-27T23:28:52.499+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:52.503710+0000) 2023-08-27T23:28:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:52 smithi130 bash[27579]: cluster 2023-08-27T23:28:51.114485+0000 mgr.y (mgr.14901) 2709 : cluster [DBG] pgmap v2716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:52 smithi130 bash[27579]: audit 2023-08-27T23:28:52.579266+0000 mon.c (mon.2) 1054 : audit [DBG] from='client.? 172.21.15.6:0/2417710891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:53.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:52 smithi006 bash[33918]: cluster 2023-08-27T23:28:51.114485+0000 mgr.y (mgr.14901) 2709 : cluster [DBG] pgmap v2716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:53.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:52 smithi006 bash[33918]: audit 2023-08-27T23:28:52.579266+0000 mon.c (mon.2) 1054 : audit [DBG] from='client.? 172.21.15.6:0/2417710891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:52 smithi006 bash[27292]: cluster 2023-08-27T23:28:51.114485+0000 mgr.y (mgr.14901) 2709 : cluster [DBG] pgmap v2716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:52 smithi006 bash[27292]: audit 2023-08-27T23:28:52.579266+0000 mon.c (mon.2) 1054 : audit [DBG] from='client.? 172.21.15.6:0/2417710891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:53.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:53 smithi130 bash[51302]: debug 2023-08-27T23:28:53.499+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:53.503993+0000) 2023-08-27T23:28:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:28:53.946Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:28:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:28:53.948Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:28:54.766 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:54 smithi130 bash[51302]: debug 2023-08-27T23:28:54.499+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:54.504279+0000) 2023-08-27T23:28:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:54 smithi130 bash[27579]: cluster 2023-08-27T23:28:53.115028+0000 mgr.y (mgr.14901) 2710 : cluster [DBG] pgmap v2717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:55.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:54 smithi006 bash[33918]: cluster 2023-08-27T23:28:53.115028+0000 mgr.y (mgr.14901) 2710 : cluster [DBG] pgmap v2717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:54 smithi006 bash[27292]: cluster 2023-08-27T23:28:53.115028+0000 mgr.y (mgr.14901) 2710 : cluster [DBG] pgmap v2717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:55.776 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:55 smithi130 bash[51302]: debug 2023-08-27T23:28:55.499+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:55.504570+0000) 2023-08-27T23:28:56.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:55 smithi130 bash[27579]: audit 2023-08-27T23:28:54.992353+0000 mon.a (mon.0) 2286 : audit [DBG] from='client.? 172.21.15.6:0/3638511933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:56.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:55 smithi006 bash[33918]: audit 2023-08-27T23:28:54.992353+0000 mon.a (mon.0) 2286 : audit [DBG] from='client.? 172.21.15.6:0/3638511933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:56.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:55 smithi006 bash[27292]: audit 2023-08-27T23:28:54.992353+0000 mon.a (mon.0) 2286 : audit [DBG] from='client.? 172.21.15.6:0/3638511933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:56.780 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:56 smithi130 bash[51302]: debug 2023-08-27T23:28:56.503+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:56.504842+0000) 2023-08-27T23:28:57.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:56 smithi130 bash[27579]: cluster 2023-08-27T23:28:55.116168+0000 mgr.y (mgr.14901) 2711 : cluster [DBG] pgmap v2718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:57.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:56 smithi006 bash[27292]: cluster 2023-08-27T23:28:55.116168+0000 mgr.y (mgr.14901) 2711 : cluster [DBG] pgmap v2718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:57.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:56 smithi006 bash[33918]: cluster 2023-08-27T23:28:55.116168+0000 mgr.y (mgr.14901) 2711 : cluster [DBG] pgmap v2718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:57.790 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:57 smithi130 bash[51302]: debug 2023-08-27T23:28:57.499+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:57.505135+0000) 2023-08-27T23:28:58.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:57 smithi130 bash[27579]: audit 2023-08-27T23:28:57.401784+0000 mon.c (mon.2) 1055 : audit [DBG] from='client.? 172.21.15.6:0/1593346556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:58.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:57 smithi006 bash[27292]: audit 2023-08-27T23:28:57.401784+0000 mon.c (mon.2) 1055 : audit [DBG] from='client.? 172.21.15.6:0/1593346556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:58.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:57 smithi006 bash[33918]: audit 2023-08-27T23:28:57.401784+0000 mon.c (mon.2) 1055 : audit [DBG] from='client.? 172.21.15.6:0/1593346556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:58.793 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:58 smithi130 bash[51302]: debug 2023-08-27T23:28:58.499+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:58.505482+0000) 2023-08-27T23:28:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:28:58 smithi130 bash[27579]: cluster 2023-08-27T23:28:57.116794+0000 mgr.y (mgr.14901) 2712 : cluster [DBG] pgmap v2719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:59.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:28:58 smithi006 bash[27292]: cluster 2023-08-27T23:28:57.116794+0000 mgr.y (mgr.14901) 2712 : cluster [DBG] pgmap v2719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:28:58 smithi006 bash[33918]: cluster 2023-08-27T23:28:57.116794+0000 mgr.y (mgr.14901) 2712 : cluster [DBG] pgmap v2719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:28:59.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:28:59 smithi130 bash[51302]: debug 2023-08-27T23:28:59.503+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:59.505781+0000) 2023-08-27T23:29:00.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:29:00.480Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:00.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:29:00.481Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:00.798 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:00 smithi130 bash[51302]: debug 2023-08-27T23:29:00.503+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:00.506065+0000) 2023-08-27T23:29:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:00 smithi130 bash[27579]: cluster 2023-08-27T23:28:59.117830+0000 mgr.y (mgr.14901) 2713 : cluster [DBG] pgmap v2720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:00 smithi130 bash[27579]: audit 2023-08-27T23:28:59.821200+0000 mon.a (mon.0) 2287 : audit [DBG] from='client.? 172.21.15.6:0/2410368407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:29:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:29:00] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T23:29:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:00 smithi006 bash[27292]: cluster 2023-08-27T23:28:59.117830+0000 mgr.y (mgr.14901) 2713 : cluster [DBG] pgmap v2720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:00 smithi006 bash[27292]: audit 2023-08-27T23:28:59.821200+0000 mon.a (mon.0) 2287 : audit [DBG] from='client.? 172.21.15.6:0/2410368407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:01.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:00 smithi006 bash[33918]: cluster 2023-08-27T23:28:59.117830+0000 mgr.y (mgr.14901) 2713 : cluster [DBG] pgmap v2720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:01.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:00 smithi006 bash[33918]: audit 2023-08-27T23:28:59.821200+0000 mon.a (mon.0) 2287 : audit [DBG] from='client.? 172.21.15.6:0/2410368407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:01.801 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:01 smithi130 bash[51302]: debug 2023-08-27T23:29:01.499+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:01.506321+0000) 2023-08-27T23:29:02.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:01 smithi130 bash[27579]: cluster 2023-08-27T23:29:01.118508+0000 mgr.y (mgr.14901) 2714 : cluster [DBG] pgmap v2721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:02.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:29:02.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:02.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:29:02.042Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:02.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:01 smithi006 bash[27292]: cluster 2023-08-27T23:29:01.118508+0000 mgr.y (mgr.14901) 2714 : cluster [DBG] pgmap v2721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:02.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:01 smithi006 bash[33918]: cluster 2023-08-27T23:29:01.118508+0000 mgr.y (mgr.14901) 2714 : cluster [DBG] pgmap v2721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:02.810 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:02 smithi130 bash[51302]: debug 2023-08-27T23:29:02.499+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:02.506614+0000) 2023-08-27T23:29:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:02 smithi130 bash[27579]: audit 2023-08-27T23:29:02.239645+0000 mon.a (mon.0) 2288 : audit [DBG] from='client.? 172.21.15.6:0/2851386875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:02 smithi006 bash[27292]: audit 2023-08-27T23:29:02.239645+0000 mon.a (mon.0) 2288 : audit [DBG] from='client.? 172.21.15.6:0/2851386875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:02 smithi006 bash[33918]: audit 2023-08-27T23:29:02.239645+0000 mon.a (mon.0) 2288 : audit [DBG] from='client.? 172.21.15.6:0/2851386875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:03.820 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:03 smithi130 bash[51302]: debug 2023-08-27T23:29:03.503+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:03.506932+0000) 2023-08-27T23:29:04.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:03 smithi130 bash[27579]: cluster 2023-08-27T23:29:03.119194+0000 mgr.y (mgr.14901) 2715 : cluster [DBG] pgmap v2722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:04.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:03 smithi006 bash[33918]: cluster 2023-08-27T23:29:03.119194+0000 mgr.y (mgr.14901) 2715 : cluster [DBG] pgmap v2722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:04.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:03 smithi006 bash[27292]: cluster 2023-08-27T23:29:03.119194+0000 mgr.y (mgr.14901) 2715 : cluster [DBG] pgmap v2722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:04.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:29:03.946Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:04.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:29:03.948Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:04.823 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:04 smithi130 bash[51302]: debug 2023-08-27T23:29:04.503+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:04.507220+0000) 2023-08-27T23:29:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:04 smithi130 bash[27579]: audit 2023-08-27T23:29:04.650858+0000 mon.c (mon.2) 1056 : audit [DBG] from='client.? 172.21.15.6:0/1745201425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:04 smithi006 bash[27292]: audit 2023-08-27T23:29:04.650858+0000 mon.c (mon.2) 1056 : audit [DBG] from='client.? 172.21.15.6:0/1745201425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:04 smithi006 bash[33918]: audit 2023-08-27T23:29:04.650858+0000 mon.c (mon.2) 1056 : audit [DBG] from='client.? 172.21.15.6:0/1745201425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:05.831 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:05 smithi130 bash[51302]: debug 2023-08-27T23:29:05.503+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:05.507562+0000) 2023-08-27T23:29:06.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:05 smithi130 bash[27579]: cluster 2023-08-27T23:29:05.120346+0000 mgr.y (mgr.14901) 2716 : cluster [DBG] pgmap v2723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:06.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:05 smithi006 bash[27292]: cluster 2023-08-27T23:29:05.120346+0000 mgr.y (mgr.14901) 2716 : cluster [DBG] pgmap v2723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:05 smithi006 bash[33918]: cluster 2023-08-27T23:29:05.120346+0000 mgr.y (mgr.14901) 2716 : cluster [DBG] pgmap v2723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:06.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:06 smithi130 bash[51302]: debug 2023-08-27T23:29:06.503+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:06.507875+0000) 2023-08-27T23:29:07.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:07 smithi130 bash[27579]: audit 2023-08-27T23:29:07.057404+0000 mon.a (mon.0) 2289 : audit [DBG] from='client.? 172.21.15.6:0/3653663112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:07.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:07 smithi006 bash[33918]: audit 2023-08-27T23:29:07.057404+0000 mon.a (mon.0) 2289 : audit [DBG] from='client.? 172.21.15.6:0/3653663112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:07.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:07 smithi006 bash[27292]: audit 2023-08-27T23:29:07.057404+0000 mon.a (mon.0) 2289 : audit [DBG] from='client.? 172.21.15.6:0/3653663112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:07.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:07 smithi130 bash[51302]: debug 2023-08-27T23:29:07.503+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:07.508247+0000) 2023-08-27T23:29:08.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:08 smithi006 bash[27292]: cluster 2023-08-27T23:29:07.120932+0000 mgr.y (mgr.14901) 2717 : cluster [DBG] pgmap v2724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:08.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:08 smithi006 bash[33918]: cluster 2023-08-27T23:29:07.120932+0000 mgr.y (mgr.14901) 2717 : cluster [DBG] pgmap v2724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:08.509 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:08 smithi130 bash[27579]: cluster 2023-08-27T23:29:07.120932+0000 mgr.y (mgr.14901) 2717 : cluster [DBG] pgmap v2724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:08.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:08 smithi130 bash[51302]: debug 2023-08-27T23:29:08.503+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:08.508578+0000) 2023-08-27T23:29:09.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:09 smithi130 bash[51302]: debug 2023-08-27T23:29:09.503+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:09.508899+0000) 2023-08-27T23:29:10.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:10 smithi006 bash[33918]: cluster 2023-08-27T23:29:09.121943+0000 mgr.y (mgr.14901) 2718 : cluster [DBG] pgmap v2725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:10.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:10 smithi006 bash[33918]: audit 2023-08-27T23:29:09.461509+0000 mon.a (mon.0) 2290 : audit [DBG] from='client.? 172.21.15.6:0/3493443975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:10.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:10 smithi006 bash[27292]: cluster 2023-08-27T23:29:09.121943+0000 mgr.y (mgr.14901) 2718 : cluster [DBG] pgmap v2725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:10.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:10 smithi006 bash[27292]: audit 2023-08-27T23:29:09.461509+0000 mon.a (mon.0) 2290 : audit [DBG] from='client.? 172.21.15.6:0/3493443975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:10.510 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:10 smithi130 bash[27579]: cluster 2023-08-27T23:29:09.121943+0000 mgr.y (mgr.14901) 2718 : cluster [DBG] pgmap v2725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:10.510 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:10 smithi130 bash[27579]: audit 2023-08-27T23:29:09.461509+0000 mon.a (mon.0) 2290 : audit [DBG] from='client.? 172.21.15.6:0/3493443975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:10.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:29:10.480Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:10.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:29:10.482Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:10.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:10 smithi130 bash[51302]: debug 2023-08-27T23:29:10.507+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:10.509222+0000) 2023-08-27T23:29:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:29:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:29:10] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T23:29:11.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:11 smithi130 bash[51302]: debug 2023-08-27T23:29:11.503+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:11.509670+0000) 2023-08-27T23:29:12.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:12 smithi006 bash[33918]: cluster 2023-08-27T23:29:11.122652+0000 mgr.y (mgr.14901) 2719 : cluster [DBG] pgmap v2726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:12.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:12 smithi006 bash[33918]: audit 2023-08-27T23:29:11.875096+0000 mon.a (mon.0) 2291 : audit [DBG] from='client.? 172.21.15.6:0/718780330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:29:12.042Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:29:12.044Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:12.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:12 smithi006 bash[27292]: cluster 2023-08-27T23:29:11.122652+0000 mgr.y (mgr.14901) 2719 : cluster [DBG] pgmap v2726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:12.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:12 smithi006 bash[27292]: audit 2023-08-27T23:29:11.875096+0000 mon.a (mon.0) 2291 : audit [DBG] from='client.? 172.21.15.6:0/718780330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:12.511 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:12 smithi130 bash[27579]: cluster 2023-08-27T23:29:11.122652+0000 mgr.y (mgr.14901) 2719 : cluster [DBG] pgmap v2726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:12.511 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:12 smithi130 bash[27579]: audit 2023-08-27T23:29:11.875096+0000 mon.a (mon.0) 2291 : audit [DBG] from='client.? 172.21.15.6:0/718780330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:12.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:12 smithi130 bash[51302]: debug 2023-08-27T23:29:12.503+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:12.509934+0000) 2023-08-27T23:29:13.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:13 smithi130 bash[51302]: debug 2023-08-27T23:29:13.503+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:13.510259+0000) 2023-08-27T23:29:14.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:14 smithi006 bash[33918]: cluster 2023-08-27T23:29:13.123338+0000 mgr.y (mgr.14901) 2720 : cluster [DBG] pgmap v2727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:14.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:14 smithi006 bash[27292]: cluster 2023-08-27T23:29:13.123338+0000 mgr.y (mgr.14901) 2720 : cluster [DBG] pgmap v2727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:29:13.947Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:29:13.949Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:14.511 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:14 smithi130 bash[27579]: cluster 2023-08-27T23:29:13.123338+0000 mgr.y (mgr.14901) 2720 : cluster [DBG] pgmap v2727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:14.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:14 smithi130 bash[51302]: debug 2023-08-27T23:29:14.507+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:14.510595+0000) 2023-08-27T23:29:15.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:15 smithi006 bash[27292]: audit 2023-08-27T23:29:14.300337+0000 mon.c (mon.2) 1057 : audit [DBG] from='client.? 172.21.15.6:0/2955741137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:15.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:15 smithi006 bash[33918]: audit 2023-08-27T23:29:14.300337+0000 mon.c (mon.2) 1057 : audit [DBG] from='client.? 172.21.15.6:0/2955741137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:15.512 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:15 smithi130 bash[27579]: audit 2023-08-27T23:29:14.300337+0000 mon.c (mon.2) 1057 : audit [DBG] from='client.? 172.21.15.6:0/2955741137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:15.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:15 smithi130 bash[51302]: debug 2023-08-27T23:29:15.507+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:15.510953+0000) 2023-08-27T23:29:16.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:16 smithi130 bash[51302]: debug 2023-08-27T23:29:16.507+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:16.511251+0000) 2023-08-27T23:29:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:16 smithi130 bash[27579]: cluster 2023-08-27T23:29:15.124561+0000 mgr.y (mgr.14901) 2721 : cluster [DBG] pgmap v2728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:16 smithi130 bash[27579]: audit 2023-08-27T23:29:16.712443+0000 mon.a (mon.0) 2292 : audit [DBG] from='client.? 172.21.15.6:0/2666626238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:17.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:16 smithi006 bash[27292]: cluster 2023-08-27T23:29:15.124561+0000 mgr.y (mgr.14901) 2721 : cluster [DBG] pgmap v2728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:17.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:16 smithi006 bash[27292]: audit 2023-08-27T23:29:16.712443+0000 mon.a (mon.0) 2292 : audit [DBG] from='client.? 172.21.15.6:0/2666626238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:16 smithi006 bash[33918]: cluster 2023-08-27T23:29:15.124561+0000 mgr.y (mgr.14901) 2721 : cluster [DBG] pgmap v2728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:16 smithi006 bash[33918]: audit 2023-08-27T23:29:16.712443+0000 mon.a (mon.0) 2292 : audit [DBG] from='client.? 172.21.15.6:0/2666626238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:17.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:17 smithi130 bash[51302]: debug 2023-08-27T23:29:17.507+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:17.511645+0000) 2023-08-27T23:29:18.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:18 smithi130 bash[27579]: cluster 2023-08-27T23:29:17.125277+0000 mgr.y (mgr.14901) 2722 : cluster [DBG] pgmap v2729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:18 smithi130 bash[51302]: debug 2023-08-27T23:29:18.511+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:18.511995+0000) 2023-08-27T23:29:19.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:18 smithi006 bash[27292]: cluster 2023-08-27T23:29:17.125277+0000 mgr.y (mgr.14901) 2722 : cluster [DBG] pgmap v2729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:18 smithi006 bash[33918]: cluster 2023-08-27T23:29:17.125277+0000 mgr.y (mgr.14901) 2722 : cluster [DBG] pgmap v2729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:19.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:19 smithi130 bash[27579]: audit 2023-08-27T23:29:18.820033+0000 mon.b (mon.1) 379 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:29:19.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:19 smithi130 bash[27579]: audit 2023-08-27T23:29:19.134180+0000 mon.c (mon.2) 1058 : audit [DBG] from='client.? 172.21.15.6:0/2554592695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:19 smithi130 bash[51302]: debug 2023-08-27T23:29:19.511+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:19.512300+0000) 2023-08-27T23:29:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:19 smithi006 bash[27292]: audit 2023-08-27T23:29:18.820033+0000 mon.b (mon.1) 379 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:29:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:19 smithi006 bash[27292]: audit 2023-08-27T23:29:19.134180+0000 mon.c (mon.2) 1058 : audit [DBG] from='client.? 172.21.15.6:0/2554592695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:20.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:19 smithi006 bash[33918]: audit 2023-08-27T23:29:18.820033+0000 mon.b (mon.1) 379 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:29:20.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:19 smithi006 bash[33918]: audit 2023-08-27T23:29:19.134180+0000 mon.c (mon.2) 1058 : audit [DBG] from='client.? 172.21.15.6:0/2554592695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:20.739 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:29:20.481Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:20.740 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:29:20.483Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:20.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:20 smithi130 bash[27579]: cluster 2023-08-27T23:29:19.126248+0000 mgr.y (mgr.14901) 2723 : cluster [DBG] pgmap v2730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:20 smithi130 bash[51302]: debug 2023-08-27T23:29:20.507+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:20.512631+0000) 2023-08-27T23:29:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:20 smithi006 bash[27292]: cluster 2023-08-27T23:29:19.126248+0000 mgr.y (mgr.14901) 2723 : cluster [DBG] pgmap v2730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:21.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:20 smithi006 bash[33918]: cluster 2023-08-27T23:29:19.126248+0000 mgr.y (mgr.14901) 2723 : cluster [DBG] pgmap v2730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:29:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:29:20] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:29:21.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:21 smithi130 bash[27579]: audit 2023-08-27T23:29:21.544301+0000 mon.a (mon.0) 2293 : audit [DBG] from='client.? 172.21.15.6:0/3205039320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:21.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:21 smithi130 bash[51302]: debug 2023-08-27T23:29:21.507+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:21.512947+0000) 2023-08-27T23:29:22.045 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:21 smithi006 bash[33918]: audit 2023-08-27T23:29:21.544301+0000 mon.a (mon.0) 2293 : audit [DBG] from='client.? 172.21.15.6:0/3205039320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:22.045 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:21 smithi006 bash[27292]: audit 2023-08-27T23:29:21.544301+0000 mon.a (mon.0) 2293 : audit [DBG] from='client.? 172.21.15.6:0/3205039320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:29:22.042Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:29:22.044Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:22 smithi130 bash[27579]: cluster 2023-08-27T23:29:21.126919+0000 mgr.y (mgr.14901) 2724 : cluster [DBG] pgmap v2731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:22 smithi130 bash[51302]: debug 2023-08-27T23:29:22.507+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:22.513266+0000) 2023-08-27T23:29:23.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:22 smithi006 bash[27292]: cluster 2023-08-27T23:29:21.126919+0000 mgr.y (mgr.14901) 2724 : cluster [DBG] pgmap v2731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:22 smithi006 bash[33918]: cluster 2023-08-27T23:29:21.126919+0000 mgr.y (mgr.14901) 2724 : cluster [DBG] pgmap v2731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:23.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:23 smithi130 bash[51302]: debug 2023-08-27T23:29:23.507+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:23.513549+0000) 2023-08-27T23:29:24.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:29:23.948Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:29:23.949Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:24.776 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:24 smithi130 bash[27579]: cluster 2023-08-27T23:29:23.127742+0000 mgr.y (mgr.14901) 2725 : cluster [DBG] pgmap v2732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:24.776 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:24 smithi130 bash[51302]: debug 2023-08-27T23:29:24.507+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:24.513754+0000) 2023-08-27T23:29:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:24 smithi130 bash[27579]: audit 2023-08-27T23:29:23.968554+0000 mon.c (mon.2) 1059 : audit [DBG] from='client.? 172.21.15.6:0/2603103155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:24 smithi130 bash[27579]: audit 2023-08-27T23:29:24.715422+0000 mon.a (mon.0) 2294 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:29:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:24 smithi006 bash[33918]: cluster 2023-08-27T23:29:23.127742+0000 mgr.y (mgr.14901) 2725 : cluster [DBG] pgmap v2732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:24 smithi006 bash[33918]: audit 2023-08-27T23:29:23.968554+0000 mon.c (mon.2) 1059 : audit [DBG] from='client.? 172.21.15.6:0/2603103155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:24 smithi006 bash[33918]: audit 2023-08-27T23:29:24.715422+0000 mon.a (mon.0) 2294 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:29:25.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:24 smithi006 bash[27292]: cluster 2023-08-27T23:29:23.127742+0000 mgr.y (mgr.14901) 2725 : cluster [DBG] pgmap v2732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:25.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:24 smithi006 bash[27292]: audit 2023-08-27T23:29:23.968554+0000 mon.c (mon.2) 1059 : audit [DBG] from='client.? 172.21.15.6:0/2603103155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:25.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:24 smithi006 bash[27292]: audit 2023-08-27T23:29:24.715422+0000 mon.a (mon.0) 2294 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:29:25.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:25 smithi130 bash[51302]: debug 2023-08-27T23:29:25.507+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:25.514003+0000) 2023-08-27T23:29:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:26 smithi006 bash[27292]: audit 2023-08-27T23:29:25.120056+0000 mon.a (mon.0) 2295 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:29:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:26 smithi006 bash[27292]: cluster 2023-08-27T23:29:25.128784+0000 mgr.y (mgr.14901) 2726 : cluster [DBG] pgmap v2733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:26 smithi006 bash[27292]: audit 2023-08-27T23:29:25.413205+0000 mon.b (mon.1) 380 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:29:26.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:26 smithi006 bash[27292]: audit 2023-08-27T23:29:25.415379+0000 mon.b (mon.1) 381 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:29:26.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:26 smithi006 bash[27292]: audit 2023-08-27T23:29:25.423854+0000 mon.a (mon.0) 2296 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:29:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:26 smithi006 bash[33918]: audit 2023-08-27T23:29:25.120056+0000 mon.a (mon.0) 2295 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:29:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:26 smithi006 bash[33918]: cluster 2023-08-27T23:29:25.128784+0000 mgr.y (mgr.14901) 2726 : cluster [DBG] pgmap v2733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:26 smithi006 bash[33918]: audit 2023-08-27T23:29:25.413205+0000 mon.b (mon.1) 380 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:29:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:26 smithi006 bash[33918]: audit 2023-08-27T23:29:25.415379+0000 mon.b (mon.1) 381 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:29:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:26 smithi006 bash[33918]: audit 2023-08-27T23:29:25.423854+0000 mon.a (mon.0) 2296 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:29:26.515 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:26 smithi130 bash[27579]: audit 2023-08-27T23:29:25.120056+0000 mon.a (mon.0) 2295 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:29:26.515 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:26 smithi130 bash[27579]: cluster 2023-08-27T23:29:25.128784+0000 mgr.y (mgr.14901) 2726 : cluster [DBG] pgmap v2733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:26.516 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:26 smithi130 bash[27579]: audit 2023-08-27T23:29:25.413205+0000 mon.b (mon.1) 380 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:29:26.516 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:26 smithi130 bash[27579]: audit 2023-08-27T23:29:25.415379+0000 mon.b (mon.1) 381 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:29:26.516 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:26 smithi130 bash[27579]: audit 2023-08-27T23:29:25.423854+0000 mon.a (mon.0) 2296 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:29:26.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:26 smithi130 bash[51302]: debug 2023-08-27T23:29:26.507+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:26.514290+0000) 2023-08-27T23:29:27.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:27 smithi006 bash[33918]: audit 2023-08-27T23:29:26.382326+0000 mon.c (mon.2) 1060 : audit [DBG] from='client.? 172.21.15.6:0/3951267557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:27.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:27 smithi006 bash[27292]: audit 2023-08-27T23:29:26.382326+0000 mon.c (mon.2) 1060 : audit [DBG] from='client.? 172.21.15.6:0/3951267557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:27.515 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:27 smithi130 bash[27579]: audit 2023-08-27T23:29:26.382326+0000 mon.c (mon.2) 1060 : audit [DBG] from='client.? 172.21.15.6:0/3951267557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:27.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:27 smithi130 bash[51302]: debug 2023-08-27T23:29:27.507+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:27.514512+0000) 2023-08-27T23:29:28.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:28 smithi006 bash[27292]: cluster 2023-08-27T23:29:27.129418+0000 mgr.y (mgr.14901) 2727 : cluster [DBG] pgmap v2734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:28.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:28 smithi006 bash[33918]: cluster 2023-08-27T23:29:27.129418+0000 mgr.y (mgr.14901) 2727 : cluster [DBG] pgmap v2734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:28.515 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:28 smithi130 bash[27579]: cluster 2023-08-27T23:29:27.129418+0000 mgr.y (mgr.14901) 2727 : cluster [DBG] pgmap v2734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:28.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:28 smithi130 bash[51302]: debug 2023-08-27T23:29:28.511+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:28.514779+0000) 2023-08-27T23:29:29.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:29 smithi006 bash[27292]: audit 2023-08-27T23:29:28.802217+0000 mon.a (mon.0) 2297 : audit [DBG] from='client.? 172.21.15.6:0/727205248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:29.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:29 smithi006 bash[33918]: audit 2023-08-27T23:29:28.802217+0000 mon.a (mon.0) 2297 : audit [DBG] from='client.? 172.21.15.6:0/727205248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:29.516 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:29 smithi130 bash[27579]: audit 2023-08-27T23:29:28.802217+0000 mon.a (mon.0) 2297 : audit [DBG] from='client.? 172.21.15.6:0/727205248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:29.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:29 smithi130 bash[51302]: debug 2023-08-27T23:29:29.511+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:29.515032+0000) 2023-08-27T23:29:30.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:30 smithi006 bash[33918]: cluster 2023-08-27T23:29:29.130519+0000 mgr.y (mgr.14901) 2728 : cluster [DBG] pgmap v2735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:30.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:30 smithi006 bash[27292]: cluster 2023-08-27T23:29:29.130519+0000 mgr.y (mgr.14901) 2728 : cluster [DBG] pgmap v2735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:30.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:29:30.481Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:30.756 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:29:30.483Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:30 smithi130 bash[27579]: cluster 2023-08-27T23:29:29.130519+0000 mgr.y (mgr.14901) 2728 : cluster [DBG] pgmap v2735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:30 smithi130 bash[51302]: debug 2023-08-27T23:29:30.511+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:30.515262+0000) 2023-08-27T23:29:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:29:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:29:30] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:29:31.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:31 smithi130 bash[27579]: audit 2023-08-27T23:29:31.221973+0000 mon.c (mon.2) 1061 : audit [DBG] from='client.? 172.21.15.6:0/1544722043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:31.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:31 smithi130 bash[51302]: debug 2023-08-27T23:29:31.511+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:31.515556+0000) 2023-08-27T23:29:32.046 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:31 smithi006 bash[27292]: audit 2023-08-27T23:29:31.221973+0000 mon.c (mon.2) 1061 : audit [DBG] from='client.? 172.21.15.6:0/1544722043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:32.046 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:31 smithi006 bash[33918]: audit 2023-08-27T23:29:31.221973+0000 mon.c (mon.2) 1061 : audit [DBG] from='client.? 172.21.15.6:0/1544722043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:29:32.043Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:29:32.046Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:32 smithi130 bash[27579]: cluster 2023-08-27T23:29:31.131153+0000 mgr.y (mgr.14901) 2729 : cluster [DBG] pgmap v2736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:32 smithi130 bash[51302]: debug 2023-08-27T23:29:32.511+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:32.515820+0000) 2023-08-27T23:29:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:32 smithi006 bash[27292]: cluster 2023-08-27T23:29:31.131153+0000 mgr.y (mgr.14901) 2729 : cluster [DBG] pgmap v2736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:32 smithi006 bash[33918]: cluster 2023-08-27T23:29:31.131153+0000 mgr.y (mgr.14901) 2729 : cluster [DBG] pgmap v2736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:33.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:33 smithi130 bash[27579]: audit 2023-08-27T23:29:33.633282+0000 mon.a (mon.0) 2298 : audit [DBG] from='client.? 172.21.15.6:0/875083269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:33.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:33 smithi130 bash[51302]: debug 2023-08-27T23:29:33.511+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:33.516041+0000) 2023-08-27T23:29:34.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:33 smithi006 bash[27292]: audit 2023-08-27T23:29:33.633282+0000 mon.a (mon.0) 2298 : audit [DBG] from='client.? 172.21.15.6:0/875083269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:34.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:33 smithi006 bash[33918]: audit 2023-08-27T23:29:33.633282+0000 mon.a (mon.0) 2298 : audit [DBG] from='client.? 172.21.15.6:0/875083269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:29:33.948Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:29:33.950Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:34 smithi130 bash[27579]: cluster 2023-08-27T23:29:33.131948+0000 mgr.y (mgr.14901) 2730 : cluster [DBG] pgmap v2737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:34 smithi130 bash[51302]: debug 2023-08-27T23:29:34.515+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:34.516241+0000) 2023-08-27T23:29:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:34 smithi006 bash[27292]: cluster 2023-08-27T23:29:33.131948+0000 mgr.y (mgr.14901) 2730 : cluster [DBG] pgmap v2737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:34 smithi006 bash[33918]: cluster 2023-08-27T23:29:33.131948+0000 mgr.y (mgr.14901) 2730 : cluster [DBG] pgmap v2737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:35.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:35 smithi130 bash[51302]: debug 2023-08-27T23:29:35.515+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:35.516466+0000) 2023-08-27T23:29:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:36 smithi130 bash[51302]: debug 2023-08-27T23:29:36.511+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:36.516667+0000) 2023-08-27T23:29:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:36 smithi130 bash[27579]: cluster 2023-08-27T23:29:35.133162+0000 mgr.y (mgr.14901) 2731 : cluster [DBG] pgmap v2738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:36 smithi130 bash[27579]: audit 2023-08-27T23:29:36.047250+0000 mon.a (mon.0) 2299 : audit [DBG] from='client.? 172.21.15.6:0/473063086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:37.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:36 smithi006 bash[33918]: cluster 2023-08-27T23:29:35.133162+0000 mgr.y (mgr.14901) 2731 : cluster [DBG] pgmap v2738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:37.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:36 smithi006 bash[33918]: audit 2023-08-27T23:29:36.047250+0000 mon.a (mon.0) 2299 : audit [DBG] from='client.? 172.21.15.6:0/473063086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:37.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:36 smithi006 bash[27292]: cluster 2023-08-27T23:29:35.133162+0000 mgr.y (mgr.14901) 2731 : cluster [DBG] pgmap v2738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:37.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:36 smithi006 bash[27292]: audit 2023-08-27T23:29:36.047250+0000 mon.a (mon.0) 2299 : audit [DBG] from='client.? 172.21.15.6:0/473063086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:37.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:37 smithi130 bash[51302]: debug 2023-08-27T23:29:37.511+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:37.516962+0000) 2023-08-27T23:29:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:38 smithi130 bash[51302]: debug 2023-08-27T23:29:38.515+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:38.517260+0000) 2023-08-27T23:29:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:38 smithi130 bash[27579]: cluster 2023-08-27T23:29:37.133826+0000 mgr.y (mgr.14901) 2732 : cluster [DBG] pgmap v2739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:38 smithi130 bash[27579]: audit 2023-08-27T23:29:38.467659+0000 mon.c (mon.2) 1062 : audit [DBG] from='client.? 172.21.15.6:0/2480031226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:39.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:38 smithi006 bash[33918]: cluster 2023-08-27T23:29:37.133826+0000 mgr.y (mgr.14901) 2732 : cluster [DBG] pgmap v2739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:39.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:38 smithi006 bash[33918]: audit 2023-08-27T23:29:38.467659+0000 mon.c (mon.2) 1062 : audit [DBG] from='client.? 172.21.15.6:0/2480031226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:39.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:38 smithi006 bash[27292]: cluster 2023-08-27T23:29:37.133826+0000 mgr.y (mgr.14901) 2732 : cluster [DBG] pgmap v2739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:39.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:38 smithi006 bash[27292]: audit 2023-08-27T23:29:38.467659+0000 mon.c (mon.2) 1062 : audit [DBG] from='client.? 172.21.15.6:0/2480031226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:39 smithi130 bash[51302]: debug 2023-08-27T23:29:39.515+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:39.517565+0000) 2023-08-27T23:29:40.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:29:40.482Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:40.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:29:40.485Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:40 smithi130 bash[27579]: cluster 2023-08-27T23:29:39.135043+0000 mgr.y (mgr.14901) 2733 : cluster [DBG] pgmap v2740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:40 smithi130 bash[51302]: debug 2023-08-27T23:29:40.515+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:40.517840+0000) 2023-08-27T23:29:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:29:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:29:40] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:29:41.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:40 smithi006 bash[33918]: cluster 2023-08-27T23:29:39.135043+0000 mgr.y (mgr.14901) 2733 : cluster [DBG] pgmap v2740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:40 smithi006 bash[27292]: cluster 2023-08-27T23:29:39.135043+0000 mgr.y (mgr.14901) 2733 : cluster [DBG] pgmap v2740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:41.775 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:41 smithi130 bash[51302]: debug 2023-08-27T23:29:41.515+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:41.518139+0000) 2023-08-27T23:29:42.047 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:41 smithi006 bash[33918]: audit 2023-08-27T23:29:40.882148+0000 mon.c (mon.2) 1063 : audit [DBG] from='client.? 172.21.15.6:0/2408421672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:42.047 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:41 smithi006 bash[27292]: audit 2023-08-27T23:29:40.882148+0000 mon.c (mon.2) 1063 : audit [DBG] from='client.? 172.21.15.6:0/2408421672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:42.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:41 smithi130 bash[27579]: audit 2023-08-27T23:29:40.882148+0000 mon.c (mon.2) 1063 : audit [DBG] from='client.? 172.21.15.6:0/2408421672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:42.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:29:42.044Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:29:42.046Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:42.777 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:42 smithi130 bash[51302]: debug 2023-08-27T23:29:42.511+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:42.518404+0000) 2023-08-27T23:29:43.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:42 smithi130 bash[27579]: cluster 2023-08-27T23:29:41.135781+0000 mgr.y (mgr.14901) 2734 : cluster [DBG] pgmap v2741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:43.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:42 smithi006 bash[27292]: cluster 2023-08-27T23:29:41.135781+0000 mgr.y (mgr.14901) 2734 : cluster [DBG] pgmap v2741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:42 smithi006 bash[33918]: cluster 2023-08-27T23:29:41.135781+0000 mgr.y (mgr.14901) 2734 : cluster [DBG] pgmap v2741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:43.786 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:43 smithi130 bash[51302]: debug 2023-08-27T23:29:43.511+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:43.518687+0000) 2023-08-27T23:29:44.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:43 smithi130 bash[27579]: audit 2023-08-27T23:29:43.289150+0000 mon.c (mon.2) 1064 : audit [DBG] from='client.? 172.21.15.6:0/4211894125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:44.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:43 smithi006 bash[33918]: audit 2023-08-27T23:29:43.289150+0000 mon.c (mon.2) 1064 : audit [DBG] from='client.? 172.21.15.6:0/4211894125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:44.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:43 smithi006 bash[27292]: audit 2023-08-27T23:29:43.289150+0000 mon.c (mon.2) 1064 : audit [DBG] from='client.? 172.21.15.6:0/4211894125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:29:43.949Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:29:43.951Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:44.789 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:44 smithi130 bash[51302]: debug 2023-08-27T23:29:44.515+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:44.519051+0000) 2023-08-27T23:29:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:44 smithi130 bash[27579]: cluster 2023-08-27T23:29:43.136444+0000 mgr.y (mgr.14901) 2735 : cluster [DBG] pgmap v2742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:44 smithi006 bash[27292]: cluster 2023-08-27T23:29:43.136444+0000 mgr.y (mgr.14901) 2735 : cluster [DBG] pgmap v2742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:44 smithi006 bash[33918]: cluster 2023-08-27T23:29:43.136444+0000 mgr.y (mgr.14901) 2735 : cluster [DBG] pgmap v2742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:45.795 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:45 smithi130 bash[51302]: debug 2023-08-27T23:29:45.519+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:45.519404+0000) 2023-08-27T23:29:46.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:45 smithi130 bash[27579]: audit 2023-08-27T23:29:45.703455+0000 mon.c (mon.2) 1065 : audit [DBG] from='client.? 172.21.15.6:0/2868149595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:46.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:45 smithi006 bash[27292]: audit 2023-08-27T23:29:45.703455+0000 mon.c (mon.2) 1065 : audit [DBG] from='client.? 172.21.15.6:0/2868149595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:46.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:45 smithi006 bash[33918]: audit 2023-08-27T23:29:45.703455+0000 mon.c (mon.2) 1065 : audit [DBG] from='client.? 172.21.15.6:0/2868149595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:46.798 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:46 smithi130 bash[51302]: debug 2023-08-27T23:29:46.519+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:46.519732+0000) 2023-08-27T23:29:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:46 smithi130 bash[27579]: cluster 2023-08-27T23:29:45.137641+0000 mgr.y (mgr.14901) 2736 : cluster [DBG] pgmap v2743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:47.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:46 smithi006 bash[27292]: cluster 2023-08-27T23:29:45.137641+0000 mgr.y (mgr.14901) 2736 : cluster [DBG] pgmap v2743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:47.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:46 smithi006 bash[33918]: cluster 2023-08-27T23:29:45.137641+0000 mgr.y (mgr.14901) 2736 : cluster [DBG] pgmap v2743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:47.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:47 smithi130 bash[51302]: debug 2023-08-27T23:29:47.519+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:47.520068+0000) 2023-08-27T23:29:48.804 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:48 smithi130 bash[51302]: debug 2023-08-27T23:29:48.514+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:48.520436+0000) 2023-08-27T23:29:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:48 smithi130 bash[27579]: cluster 2023-08-27T23:29:47.138362+0000 mgr.y (mgr.14901) 2737 : cluster [DBG] pgmap v2744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:48 smithi130 bash[27579]: audit 2023-08-27T23:29:48.112482+0000 mon.c (mon.2) 1066 : audit [DBG] from='client.? 172.21.15.6:0/2248572657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:49.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:48 smithi006 bash[27292]: cluster 2023-08-27T23:29:47.138362+0000 mgr.y (mgr.14901) 2737 : cluster [DBG] pgmap v2744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:49.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:48 smithi006 bash[27292]: audit 2023-08-27T23:29:48.112482+0000 mon.c (mon.2) 1066 : audit [DBG] from='client.? 172.21.15.6:0/2248572657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:49.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:48 smithi006 bash[33918]: cluster 2023-08-27T23:29:47.138362+0000 mgr.y (mgr.14901) 2737 : cluster [DBG] pgmap v2744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:49.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:48 smithi006 bash[33918]: audit 2023-08-27T23:29:48.112482+0000 mon.c (mon.2) 1066 : audit [DBG] from='client.? 172.21.15.6:0/2248572657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:49.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:49 smithi130 bash[51302]: debug 2023-08-27T23:29:49.518+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:49.520784+0000) 2023-08-27T23:29:50.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:29:50.483Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:50.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:29:50.485Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:50.809 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:50 smithi130 bash[51302]: debug 2023-08-27T23:29:50.514+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:50.521108+0000) 2023-08-27T23:29:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:50 smithi130 bash[27579]: cluster 2023-08-27T23:29:49.139428+0000 mgr.y (mgr.14901) 2738 : cluster [DBG] pgmap v2745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:50 smithi130 bash[27579]: audit 2023-08-27T23:29:50.541345+0000 mon.c (mon.2) 1067 : audit [DBG] from='client.? 172.21.15.6:0/1286976258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:29:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:29:50] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:29:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:50 smithi006 bash[27292]: cluster 2023-08-27T23:29:49.139428+0000 mgr.y (mgr.14901) 2738 : cluster [DBG] pgmap v2745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:50 smithi006 bash[27292]: audit 2023-08-27T23:29:50.541345+0000 mon.c (mon.2) 1067 : audit [DBG] from='client.? 172.21.15.6:0/1286976258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:51.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:50 smithi006 bash[33918]: cluster 2023-08-27T23:29:49.139428+0000 mgr.y (mgr.14901) 2738 : cluster [DBG] pgmap v2745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:51.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:50 smithi006 bash[33918]: audit 2023-08-27T23:29:50.541345+0000 mon.c (mon.2) 1067 : audit [DBG] from='client.? 172.21.15.6:0/1286976258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:51.818 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:51 smithi130 bash[51302]: debug 2023-08-27T23:29:51.514+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:51.521409+0000) 2023-08-27T23:29:52.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:51 smithi130 bash[27579]: cluster 2023-08-27T23:29:51.140227+0000 mgr.y (mgr.14901) 2739 : cluster [DBG] pgmap v2746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:52.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:51 smithi006 bash[27292]: cluster 2023-08-27T23:29:51.140227+0000 mgr.y (mgr.14901) 2739 : cluster [DBG] pgmap v2746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:52.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:29:52.044Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:52.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:29:52.046Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:52.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:51 smithi006 bash[33918]: cluster 2023-08-27T23:29:51.140227+0000 mgr.y (mgr.14901) 2739 : cluster [DBG] pgmap v2746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:52.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:52 smithi130 bash[51302]: debug 2023-08-27T23:29:52.514+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:52.521796+0000) 2023-08-27T23:29:53.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:53 smithi130 bash[27579]: audit 2023-08-27T23:29:52.953637+0000 mon.a (mon.0) 2300 : audit [DBG] from='client.? 172.21.15.6:0/1731170764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:53.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:53 smithi006 bash[33918]: audit 2023-08-27T23:29:52.953637+0000 mon.a (mon.0) 2300 : audit [DBG] from='client.? 172.21.15.6:0/1731170764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:53 smithi006 bash[27292]: audit 2023-08-27T23:29:52.953637+0000 mon.a (mon.0) 2300 : audit [DBG] from='client.? 172.21.15.6:0/1731170764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:53.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:53 smithi130 bash[51302]: debug 2023-08-27T23:29:53.514+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:53.522165+0000) 2023-08-27T23:29:54.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:54 smithi006 bash[33918]: cluster 2023-08-27T23:29:53.141095+0000 mgr.y (mgr.14901) 2740 : cluster [DBG] pgmap v2747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:54.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:54 smithi006 bash[27292]: cluster 2023-08-27T23:29:53.141095+0000 mgr.y (mgr.14901) 2740 : cluster [DBG] pgmap v2747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:29:53.950Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:29:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:29:53.952Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:29:54.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:54 smithi130 bash[27579]: cluster 2023-08-27T23:29:53.141095+0000 mgr.y (mgr.14901) 2740 : cluster [DBG] pgmap v2747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:54.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:54 smithi130 bash[51302]: debug 2023-08-27T23:29:54.518+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:54.522500+0000) 2023-08-27T23:29:55.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:55 smithi130 bash[27579]: audit 2023-08-27T23:29:55.367732+0000 mon.a (mon.0) 2301 : audit [DBG] from='client.? 172.21.15.6:0/2689209220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:55.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:55 smithi130 bash[51302]: debug 2023-08-27T23:29:55.518+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:55.522818+0000) 2023-08-27T23:29:56.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:55 smithi006 bash[33918]: audit 2023-08-27T23:29:55.367732+0000 mon.a (mon.0) 2301 : audit [DBG] from='client.? 172.21.15.6:0/2689209220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:56.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:55 smithi006 bash[27292]: audit 2023-08-27T23:29:55.367732+0000 mon.a (mon.0) 2301 : audit [DBG] from='client.? 172.21.15.6:0/2689209220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:56.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:56 smithi130 bash[27579]: cluster 2023-08-27T23:29:55.142264+0000 mgr.y (mgr.14901) 2741 : cluster [DBG] pgmap v2748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:56.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:56 smithi130 bash[51302]: debug 2023-08-27T23:29:56.522+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:56.523216+0000) 2023-08-27T23:29:57.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:56 smithi006 bash[33918]: cluster 2023-08-27T23:29:55.142264+0000 mgr.y (mgr.14901) 2741 : cluster [DBG] pgmap v2748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:57.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:56 smithi006 bash[27292]: cluster 2023-08-27T23:29:55.142264+0000 mgr.y (mgr.14901) 2741 : cluster [DBG] pgmap v2748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:57.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:57 smithi130 bash[51302]: debug 2023-08-27T23:29:57.522+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:57.523495+0000) 2023-08-27T23:29:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:58 smithi130 bash[27579]: cluster 2023-08-27T23:29:57.142915+0000 mgr.y (mgr.14901) 2742 : cluster [DBG] pgmap v2749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:29:58 smithi130 bash[27579]: audit 2023-08-27T23:29:57.790135+0000 mon.a (mon.0) 2302 : audit [DBG] from='client.? 172.21.15.6:0/3639926305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:58.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:58 smithi130 bash[51302]: debug 2023-08-27T23:29:58.522+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:58.523886+0000) 2023-08-27T23:29:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:58 smithi006 bash[33918]: cluster 2023-08-27T23:29:57.142915+0000 mgr.y (mgr.14901) 2742 : cluster [DBG] pgmap v2749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:29:58 smithi006 bash[33918]: audit 2023-08-27T23:29:57.790135+0000 mon.a (mon.0) 2302 : audit [DBG] from='client.? 172.21.15.6:0/3639926305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:58 smithi006 bash[27292]: cluster 2023-08-27T23:29:57.142915+0000 mgr.y (mgr.14901) 2742 : cluster [DBG] pgmap v2749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:29:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:29:58 smithi006 bash[27292]: audit 2023-08-27T23:29:57.790135+0000 mon.a (mon.0) 2302 : audit [DBG] from='client.? 172.21.15.6:0/3639926305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:59.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:29:59 smithi130 bash[51302]: debug 2023-08-27T23:29:59.522+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:59.524235+0000) 2023-08-27T23:30:00.749 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:30:00.484Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:00.749 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:30:00.486Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:00 smithi130 bash[27579]: cluster 2023-08-27T23:29:59.144106+0000 mgr.y (mgr.14901) 2743 : cluster [DBG] pgmap v2750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:00 smithi130 bash[27579]: cluster 2023-08-27T23:30:00.000186+0000 mon.a (mon.0) 2303 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:30:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:00 smithi130 bash[27579]: audit 2023-08-27T23:30:00.207026+0000 mon.a (mon.0) 2304 : audit [DBG] from='client.? 172.21.15.6:0/2823605403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:00.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:00 smithi130 bash[51302]: debug 2023-08-27T23:30:00.522+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:00.524600+0000) 2023-08-27T23:30:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:30:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:30:00] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:30:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:00 smithi006 bash[27292]: cluster 2023-08-27T23:29:59.144106+0000 mgr.y (mgr.14901) 2743 : cluster [DBG] pgmap v2750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:00 smithi006 bash[27292]: cluster 2023-08-27T23:30:00.000186+0000 mon.a (mon.0) 2303 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:30:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:00 smithi006 bash[27292]: audit 2023-08-27T23:30:00.207026+0000 mon.a (mon.0) 2304 : audit [DBG] from='client.? 172.21.15.6:0/2823605403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:00 smithi006 bash[33918]: cluster 2023-08-27T23:29:59.144106+0000 mgr.y (mgr.14901) 2743 : cluster [DBG] pgmap v2750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:01.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:00 smithi006 bash[33918]: cluster 2023-08-27T23:30:00.000186+0000 mon.a (mon.0) 2303 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:30:01.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:00 smithi006 bash[33918]: audit 2023-08-27T23:30:00.207026+0000 mon.a (mon.0) 2304 : audit [DBG] from='client.? 172.21.15.6:0/2823605403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:01.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:01 smithi130 bash[51302]: debug 2023-08-27T23:30:01.522+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:01.524951+0000) 2023-08-27T23:30:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:30:02.045Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:30:02.047Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:02 smithi130 bash[27579]: cluster 2023-08-27T23:30:01.144786+0000 mgr.y (mgr.14901) 2744 : cluster [DBG] pgmap v2751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:02 smithi130 bash[27579]: audit 2023-08-27T23:30:02.631217+0000 mon.c (mon.2) 1068 : audit [DBG] from='client.? 172.21.15.6:0/2779074420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:02.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:02 smithi130 bash[51302]: debug 2023-08-27T23:30:02.522+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:02.525295+0000) 2023-08-27T23:30:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:02 smithi006 bash[33918]: cluster 2023-08-27T23:30:01.144786+0000 mgr.y (mgr.14901) 2744 : cluster [DBG] pgmap v2751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:02 smithi006 bash[33918]: audit 2023-08-27T23:30:02.631217+0000 mon.c (mon.2) 1068 : audit [DBG] from='client.? 172.21.15.6:0/2779074420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:02 smithi006 bash[27292]: cluster 2023-08-27T23:30:01.144786+0000 mgr.y (mgr.14901) 2744 : cluster [DBG] pgmap v2751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:02 smithi006 bash[27292]: audit 2023-08-27T23:30:02.631217+0000 mon.c (mon.2) 1068 : audit [DBG] from='client.? 172.21.15.6:0/2779074420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:03.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:03 smithi130 bash[51302]: debug 2023-08-27T23:30:03.522+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:03.525632+0000) 2023-08-27T23:30:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:30:03.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:30:03.953Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:04.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:04 smithi130 bash[51302]: debug 2023-08-27T23:30:04.522+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:04.525987+0000) 2023-08-27T23:30:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:04 smithi130 bash[27579]: cluster 2023-08-27T23:30:03.145515+0000 mgr.y (mgr.14901) 2745 : cluster [DBG] pgmap v2752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:04 smithi006 bash[33918]: cluster 2023-08-27T23:30:03.145515+0000 mgr.y (mgr.14901) 2745 : cluster [DBG] pgmap v2752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:04 smithi006 bash[27292]: cluster 2023-08-27T23:30:03.145515+0000 mgr.y (mgr.14901) 2745 : cluster [DBG] pgmap v2752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:05.870 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:05 smithi130 bash[51302]: debug 2023-08-27T23:30:05.522+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:05.526360+0000) 2023-08-27T23:30:05.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:05 smithi130 bash[27579]: audit 2023-08-27T23:30:05.048166+0000 mon.c (mon.2) 1069 : audit [DBG] from='client.? 172.21.15.6:0/3964148715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:05 smithi006 bash[33918]: audit 2023-08-27T23:30:05.048166+0000 mon.c (mon.2) 1069 : audit [DBG] from='client.? 172.21.15.6:0/3964148715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:06.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:05 smithi006 bash[27292]: audit 2023-08-27T23:30:05.048166+0000 mon.c (mon.2) 1069 : audit [DBG] from='client.? 172.21.15.6:0/3964148715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:06.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:06 smithi130 bash[51302]: debug 2023-08-27T23:30:06.522+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:06.526705+0000) 2023-08-27T23:30:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:06 smithi130 bash[27579]: cluster 2023-08-27T23:30:05.146687+0000 mgr.y (mgr.14901) 2746 : cluster [DBG] pgmap v2753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:07.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:06 smithi006 bash[33918]: cluster 2023-08-27T23:30:05.146687+0000 mgr.y (mgr.14901) 2746 : cluster [DBG] pgmap v2753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:07.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:06 smithi006 bash[27292]: cluster 2023-08-27T23:30:05.146687+0000 mgr.y (mgr.14901) 2746 : cluster [DBG] pgmap v2753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:07.785 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:07 smithi130 bash[51302]: debug 2023-08-27T23:30:07.526+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:07.527037+0000) 2023-08-27T23:30:08.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:07 smithi130 bash[27579]: audit 2023-08-27T23:30:07.460739+0000 mon.a (mon.0) 2305 : audit [DBG] from='client.? 172.21.15.6:0/3745535910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:08.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:07 smithi006 bash[27292]: audit 2023-08-27T23:30:07.460739+0000 mon.a (mon.0) 2305 : audit [DBG] from='client.? 172.21.15.6:0/3745535910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:08.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:07 smithi006 bash[33918]: audit 2023-08-27T23:30:07.460739+0000 mon.a (mon.0) 2305 : audit [DBG] from='client.? 172.21.15.6:0/3745535910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:08.788 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:08 smithi130 bash[51302]: debug 2023-08-27T23:30:08.526+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:08.527391+0000) 2023-08-27T23:30:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:08 smithi130 bash[27579]: cluster 2023-08-27T23:30:07.147288+0000 mgr.y (mgr.14901) 2747 : cluster [DBG] pgmap v2754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:08 smithi006 bash[33918]: cluster 2023-08-27T23:30:07.147288+0000 mgr.y (mgr.14901) 2747 : cluster [DBG] pgmap v2754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:08 smithi006 bash[27292]: cluster 2023-08-27T23:30:07.147288+0000 mgr.y (mgr.14901) 2747 : cluster [DBG] pgmap v2754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:09.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:09 smithi130 bash[51302]: debug 2023-08-27T23:30:09.526+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:09.527771+0000) 2023-08-27T23:30:10.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:30:10.485Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:10.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:30:10.487Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:10.793 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:10 smithi130 bash[51302]: debug 2023-08-27T23:30:10.526+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:10.528098+0000) 2023-08-27T23:30:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:10 smithi130 bash[27579]: cluster 2023-08-27T23:30:09.148410+0000 mgr.y (mgr.14901) 2748 : cluster [DBG] pgmap v2755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:10 smithi130 bash[27579]: audit 2023-08-27T23:30:09.869114+0000 mon.a (mon.0) 2306 : audit [DBG] from='client.? 172.21.15.6:0/1959633524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:30:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:30:10] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:30:11.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:10 smithi006 bash[27292]: cluster 2023-08-27T23:30:09.148410+0000 mgr.y (mgr.14901) 2748 : cluster [DBG] pgmap v2755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:11.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:10 smithi006 bash[27292]: audit 2023-08-27T23:30:09.869114+0000 mon.a (mon.0) 2306 : audit [DBG] from='client.? 172.21.15.6:0/1959633524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:10 smithi006 bash[33918]: cluster 2023-08-27T23:30:09.148410+0000 mgr.y (mgr.14901) 2748 : cluster [DBG] pgmap v2755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:11.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:10 smithi006 bash[33918]: audit 2023-08-27T23:30:09.869114+0000 mon.a (mon.0) 2306 : audit [DBG] from='client.? 172.21.15.6:0/1959633524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:11.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:11 smithi130 bash[51302]: debug 2023-08-27T23:30:11.526+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:11.528453+0000) 2023-08-27T23:30:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:30:12.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:30:12.048Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:12.799 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:12 smithi130 bash[51302]: debug 2023-08-27T23:30:12.526+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:12.528810+0000) 2023-08-27T23:30:13.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:12 smithi130 bash[27579]: cluster 2023-08-27T23:30:11.149127+0000 mgr.y (mgr.14901) 2749 : cluster [DBG] pgmap v2756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:12 smithi130 bash[27579]: audit 2023-08-27T23:30:12.285084+0000 mon.c (mon.2) 1070 : audit [DBG] from='client.? 172.21.15.6:0/1851508239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:12 smithi006 bash[27292]: cluster 2023-08-27T23:30:11.149127+0000 mgr.y (mgr.14901) 2749 : cluster [DBG] pgmap v2756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:12 smithi006 bash[27292]: audit 2023-08-27T23:30:12.285084+0000 mon.c (mon.2) 1070 : audit [DBG] from='client.? 172.21.15.6:0/1851508239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:12 smithi006 bash[33918]: cluster 2023-08-27T23:30:11.149127+0000 mgr.y (mgr.14901) 2749 : cluster [DBG] pgmap v2756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:12 smithi006 bash[33918]: audit 2023-08-27T23:30:12.285084+0000 mon.c (mon.2) 1070 : audit [DBG] from='client.? 172.21.15.6:0/1851508239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:13.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:13 smithi130 bash[51302]: debug 2023-08-27T23:30:13.526+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:13.529141+0000) 2023-08-27T23:30:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:30:13.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:30:13.953Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:14.804 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:14 smithi130 bash[51302]: debug 2023-08-27T23:30:14.526+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:14.529542+0000) 2023-08-27T23:30:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:14 smithi130 bash[27579]: cluster 2023-08-27T23:30:13.149795+0000 mgr.y (mgr.14901) 2750 : cluster [DBG] pgmap v2757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:14 smithi130 bash[27579]: audit 2023-08-27T23:30:14.704639+0000 mon.c (mon.2) 1071 : audit [DBG] from='client.? 172.21.15.6:0/1776109932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:15.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:14 smithi006 bash[27292]: cluster 2023-08-27T23:30:13.149795+0000 mgr.y (mgr.14901) 2750 : cluster [DBG] pgmap v2757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:14 smithi006 bash[27292]: audit 2023-08-27T23:30:14.704639+0000 mon.c (mon.2) 1071 : audit [DBG] from='client.? 172.21.15.6:0/1776109932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:14 smithi006 bash[33918]: cluster 2023-08-27T23:30:13.149795+0000 mgr.y (mgr.14901) 2750 : cluster [DBG] pgmap v2757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:14 smithi006 bash[33918]: audit 2023-08-27T23:30:14.704639+0000 mon.c (mon.2) 1071 : audit [DBG] from='client.? 172.21.15.6:0/1776109932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:15.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:15 smithi130 bash[51302]: debug 2023-08-27T23:30:15.526+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:15.529856+0000) 2023-08-27T23:30:16.809 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:16 smithi130 bash[51302]: debug 2023-08-27T23:30:16.526+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:16.530144+0000) 2023-08-27T23:30:17.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:16 smithi130 bash[27579]: cluster 2023-08-27T23:30:15.150926+0000 mgr.y (mgr.14901) 2751 : cluster [DBG] pgmap v2758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:17.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:16 smithi006 bash[27292]: cluster 2023-08-27T23:30:15.150926+0000 mgr.y (mgr.14901) 2751 : cluster [DBG] pgmap v2758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:16 smithi006 bash[33918]: cluster 2023-08-27T23:30:15.150926+0000 mgr.y (mgr.14901) 2751 : cluster [DBG] pgmap v2758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:17.818 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:17 smithi130 bash[51302]: debug 2023-08-27T23:30:17.526+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:17.530489+0000) 2023-08-27T23:30:18.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:17 smithi130 bash[27579]: audit 2023-08-27T23:30:17.138845+0000 mon.c (mon.2) 1072 : audit [DBG] from='client.? 172.21.15.6:0/1626121469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:18.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:17 smithi006 bash[33918]: audit 2023-08-27T23:30:17.138845+0000 mon.c (mon.2) 1072 : audit [DBG] from='client.? 172.21.15.6:0/1626121469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:18.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:17 smithi006 bash[27292]: audit 2023-08-27T23:30:17.138845+0000 mon.c (mon.2) 1072 : audit [DBG] from='client.? 172.21.15.6:0/1626121469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:18.821 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:18 smithi130 bash[51302]: debug 2023-08-27T23:30:18.530+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:18.530911+0000) 2023-08-27T23:30:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:18 smithi130 bash[27579]: cluster 2023-08-27T23:30:17.151499+0000 mgr.y (mgr.14901) 2752 : cluster [DBG] pgmap v2759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:18 smithi006 bash[27292]: cluster 2023-08-27T23:30:17.151499+0000 mgr.y (mgr.14901) 2752 : cluster [DBG] pgmap v2759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:18 smithi006 bash[33918]: cluster 2023-08-27T23:30:17.151499+0000 mgr.y (mgr.14901) 2752 : cluster [DBG] pgmap v2759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:19.829 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:19 smithi130 bash[51302]: debug 2023-08-27T23:30:19.530+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:19.531263+0000) 2023-08-27T23:30:20.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:19 smithi130 bash[27579]: cluster 2023-08-27T23:30:19.152539+0000 mgr.y (mgr.14901) 2753 : cluster [DBG] pgmap v2760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:20.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:19 smithi130 bash[27579]: audit 2023-08-27T23:30:19.550340+0000 mon.a (mon.0) 2307 : audit [DBG] from='client.? 172.21.15.6:0/1552070281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:19 smithi006 bash[27292]: cluster 2023-08-27T23:30:19.152539+0000 mgr.y (mgr.14901) 2753 : cluster [DBG] pgmap v2760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:19 smithi006 bash[27292]: audit 2023-08-27T23:30:19.550340+0000 mon.a (mon.0) 2307 : audit [DBG] from='client.? 172.21.15.6:0/1552070281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:20.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:19 smithi006 bash[33918]: cluster 2023-08-27T23:30:19.152539+0000 mgr.y (mgr.14901) 2753 : cluster [DBG] pgmap v2760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:20.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:19 smithi006 bash[33918]: audit 2023-08-27T23:30:19.550340+0000 mon.a (mon.0) 2307 : audit [DBG] from='client.? 172.21.15.6:0/1552070281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:20.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:30:20.486Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:20.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:30:20.488Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:20 smithi130 bash[51302]: debug 2023-08-27T23:30:20.530+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:20.531601+0000) 2023-08-27T23:30:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:30:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:30:20] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T23:30:21.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:21 smithi130 bash[51302]: debug 2023-08-27T23:30:21.530+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:21.531945+0000) 2023-08-27T23:30:22.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:22 smithi006 bash[33918]: cluster 2023-08-27T23:30:21.153292+0000 mgr.y (mgr.14901) 2754 : cluster [DBG] pgmap v2761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:22.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:22 smithi006 bash[33918]: audit 2023-08-27T23:30:21.963237+0000 mon.c (mon.2) 1073 : audit [DBG] from='client.? 172.21.15.6:0/4093080294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:22.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:22 smithi006 bash[27292]: cluster 2023-08-27T23:30:21.153292+0000 mgr.y (mgr.14901) 2754 : cluster [DBG] pgmap v2761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:22.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:22 smithi006 bash[27292]: audit 2023-08-27T23:30:21.963237+0000 mon.c (mon.2) 1073 : audit [DBG] from='client.? 172.21.15.6:0/4093080294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:22.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:30:22.047Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:22.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:30:22.049Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:22.533 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:22 smithi130 bash[27579]: cluster 2023-08-27T23:30:21.153292+0000 mgr.y (mgr.14901) 2754 : cluster [DBG] pgmap v2761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:22.533 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:22 smithi130 bash[27579]: audit 2023-08-27T23:30:21.963237+0000 mon.c (mon.2) 1073 : audit [DBG] from='client.? 172.21.15.6:0/4093080294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:22.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:22 smithi130 bash[51302]: debug 2023-08-27T23:30:22.530+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:22.532312+0000) 2023-08-27T23:30:23.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:23 smithi130 bash[51302]: debug 2023-08-27T23:30:23.530+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:23.532593+0000) 2023-08-27T23:30:24.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:30:23.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:30:23.959Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:24.533 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:24 smithi130 bash[27579]: cluster 2023-08-27T23:30:23.154098+0000 mgr.y (mgr.14901) 2755 : cluster [DBG] pgmap v2762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:24.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:24 smithi006 bash[27292]: cluster 2023-08-27T23:30:23.154098+0000 mgr.y (mgr.14901) 2755 : cluster [DBG] pgmap v2762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:24.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:24 smithi006 bash[33918]: cluster 2023-08-27T23:30:23.154098+0000 mgr.y (mgr.14901) 2755 : cluster [DBG] pgmap v2762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:24 smithi130 bash[51302]: debug 2023-08-27T23:30:24.530+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:24.532880+0000) 2023-08-27T23:30:25.534 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:25 smithi130 bash[27579]: audit 2023-08-27T23:30:24.382891+0000 mon.a (mon.0) 2308 : audit [DBG] from='client.? 172.21.15.6:0/834998790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:25.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:25 smithi006 bash[33918]: audit 2023-08-27T23:30:24.382891+0000 mon.a (mon.0) 2308 : audit [DBG] from='client.? 172.21.15.6:0/834998790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:25.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:25 smithi006 bash[27292]: audit 2023-08-27T23:30:24.382891+0000 mon.a (mon.0) 2308 : audit [DBG] from='client.? 172.21.15.6:0/834998790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:25.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:25 smithi130 bash[51302]: debug 2023-08-27T23:30:25.530+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:25.533142+0000) 2023-08-27T23:30:26.534 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:26 smithi130 bash[27579]: cluster 2023-08-27T23:30:25.155294+0000 mgr.y (mgr.14901) 2756 : cluster [DBG] pgmap v2763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:26.534 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:26 smithi130 bash[27579]: audit 2023-08-27T23:30:25.434969+0000 mon.b (mon.1) 382 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:30:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:26 smithi006 bash[33918]: cluster 2023-08-27T23:30:25.155294+0000 mgr.y (mgr.14901) 2756 : cluster [DBG] pgmap v2763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:26 smithi006 bash[33918]: audit 2023-08-27T23:30:25.434969+0000 mon.b (mon.1) 382 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:30:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:26 smithi006 bash[27292]: cluster 2023-08-27T23:30:25.155294+0000 mgr.y (mgr.14901) 2756 : cluster [DBG] pgmap v2763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:26 smithi006 bash[27292]: audit 2023-08-27T23:30:25.434969+0000 mon.b (mon.1) 382 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:30:26.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:26 smithi130 bash[51302]: debug 2023-08-27T23:30:26.530+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:26.533379+0000) 2023-08-27T23:30:27.534 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:27 smithi130 bash[27579]: audit 2023-08-27T23:30:26.813699+0000 mon.a (mon.0) 2309 : audit [DBG] from='client.? 172.21.15.6:0/881598881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:27.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:27 smithi006 bash[27292]: audit 2023-08-27T23:30:26.813699+0000 mon.a (mon.0) 2309 : audit [DBG] from='client.? 172.21.15.6:0/881598881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:27.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:27 smithi006 bash[33918]: audit 2023-08-27T23:30:26.813699+0000 mon.a (mon.0) 2309 : audit [DBG] from='client.? 172.21.15.6:0/881598881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:27.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:27 smithi130 bash[51302]: debug 2023-08-27T23:30:27.530+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:27.533681+0000) 2023-08-27T23:30:28.535 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:28 smithi130 bash[27579]: cluster 2023-08-27T23:30:27.156029+0000 mgr.y (mgr.14901) 2757 : cluster [DBG] pgmap v2764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:28 smithi006 bash[33918]: cluster 2023-08-27T23:30:27.156029+0000 mgr.y (mgr.14901) 2757 : cluster [DBG] pgmap v2764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:28.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:28 smithi006 bash[27292]: cluster 2023-08-27T23:30:27.156029+0000 mgr.y (mgr.14901) 2757 : cluster [DBG] pgmap v2764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:28.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:28 smithi130 bash[51302]: debug 2023-08-27T23:30:28.530+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:28.533978+0000) 2023-08-27T23:30:29.535 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:29 smithi130 bash[27579]: audit 2023-08-27T23:30:29.226752+0000 mon.c (mon.2) 1074 : audit [DBG] from='client.? 172.21.15.6:0/1182959204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:29.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:29 smithi006 bash[27292]: audit 2023-08-27T23:30:29.226752+0000 mon.c (mon.2) 1074 : audit [DBG] from='client.? 172.21.15.6:0/1182959204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:29.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:29 smithi006 bash[33918]: audit 2023-08-27T23:30:29.226752+0000 mon.c (mon.2) 1074 : audit [DBG] from='client.? 172.21.15.6:0/1182959204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:29.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:29 smithi130 bash[51302]: debug 2023-08-27T23:30:29.530+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:29.534256+0000) 2023-08-27T23:30:30.744 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:30:30.486Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:30.744 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:30:30.488Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:30.744 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:30 smithi006 bash[27292]: cluster 2023-08-27T23:30:29.157102+0000 mgr.y (mgr.14901) 2758 : cluster [DBG] pgmap v2765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:30.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:30 smithi130 bash[27579]: cluster 2023-08-27T23:30:29.157102+0000 mgr.y (mgr.14901) 2758 : cluster [DBG] pgmap v2765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:30 smithi130 bash[51302]: debug 2023-08-27T23:30:30.530+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:30.534494+0000) 2023-08-27T23:30:31.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:30 smithi006 bash[33918]: cluster 2023-08-27T23:30:29.157102+0000 mgr.y (mgr.14901) 2758 : cluster [DBG] pgmap v2765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:30:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:30:30] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:30:31.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:31 smithi130 bash[51302]: debug 2023-08-27T23:30:31.530+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:31.534711+0000) 2023-08-27T23:30:32.321 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:30:32.048Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:32.321 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:30:32.050Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:32.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:32 smithi130 bash[51302]: debug 2023-08-27T23:30:32.534+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:32.535030+0000) 2023-08-27T23:30:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:32 smithi130 bash[27579]: cluster 2023-08-27T23:30:31.157727+0000 mgr.y (mgr.14901) 2759 : cluster [DBG] pgmap v2766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:32 smithi130 bash[27579]: audit 2023-08-27T23:30:31.317643+0000 mon.a (mon.0) 2310 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:30:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:32 smithi130 bash[27579]: audit 2023-08-27T23:30:31.658530+0000 mon.a (mon.0) 2311 : audit [DBG] from='client.? 172.21.15.6:0/4276186098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:32 smithi130 bash[27579]: audit 2023-08-27T23:30:31.760386+0000 mon.a (mon.0) 2312 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:30:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:32 smithi130 bash[27579]: audit 2023-08-27T23:30:32.057813+0000 mon.b (mon.1) 383 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:30:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:32 smithi130 bash[27579]: audit 2023-08-27T23:30:32.059702+0000 mon.b (mon.1) 384 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:30:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:32 smithi130 bash[27579]: audit 2023-08-27T23:30:32.069935+0000 mon.a (mon.0) 2313 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:30:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:32 smithi006 bash[27292]: cluster 2023-08-27T23:30:31.157727+0000 mgr.y (mgr.14901) 2759 : cluster [DBG] pgmap v2766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:32 smithi006 bash[27292]: audit 2023-08-27T23:30:31.317643+0000 mon.a (mon.0) 2310 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:30:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:32 smithi006 bash[27292]: audit 2023-08-27T23:30:31.658530+0000 mon.a (mon.0) 2311 : audit [DBG] from='client.? 172.21.15.6:0/4276186098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:32 smithi006 bash[27292]: audit 2023-08-27T23:30:31.760386+0000 mon.a (mon.0) 2312 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:30:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:32 smithi006 bash[27292]: audit 2023-08-27T23:30:32.057813+0000 mon.b (mon.1) 383 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:30:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:32 smithi006 bash[27292]: audit 2023-08-27T23:30:32.059702+0000 mon.b (mon.1) 384 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:30:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:32 smithi006 bash[27292]: audit 2023-08-27T23:30:32.069935+0000 mon.a (mon.0) 2313 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:30:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:32 smithi006 bash[33918]: cluster 2023-08-27T23:30:31.157727+0000 mgr.y (mgr.14901) 2759 : cluster [DBG] pgmap v2766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:32 smithi006 bash[33918]: audit 2023-08-27T23:30:31.317643+0000 mon.a (mon.0) 2310 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:30:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:32 smithi006 bash[33918]: audit 2023-08-27T23:30:31.658530+0000 mon.a (mon.0) 2311 : audit [DBG] from='client.? 172.21.15.6:0/4276186098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:32 smithi006 bash[33918]: audit 2023-08-27T23:30:31.760386+0000 mon.a (mon.0) 2312 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:30:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:32 smithi006 bash[33918]: audit 2023-08-27T23:30:32.057813+0000 mon.b (mon.1) 383 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:30:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:32 smithi006 bash[33918]: audit 2023-08-27T23:30:32.059702+0000 mon.b (mon.1) 384 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:30:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:32 smithi006 bash[33918]: audit 2023-08-27T23:30:32.069935+0000 mon.a (mon.0) 2313 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:30:33.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:33 smithi130 bash[51302]: debug 2023-08-27T23:30:33.534+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:33.535316+0000) 2023-08-27T23:30:34.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:30:33.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:30:33.954Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:34.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:34 smithi130 bash[51302]: debug 2023-08-27T23:30:34.534+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:34.535648+0000) 2023-08-27T23:30:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:34 smithi130 bash[27579]: cluster 2023-08-27T23:30:33.158422+0000 mgr.y (mgr.14901) 2760 : cluster [DBG] pgmap v2767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:34 smithi130 bash[27579]: audit 2023-08-27T23:30:34.068119+0000 mon.a (mon.0) 2314 : audit [DBG] from='client.? 172.21.15.6:0/2551376503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:34 smithi006 bash[33918]: cluster 2023-08-27T23:30:33.158422+0000 mgr.y (mgr.14901) 2760 : cluster [DBG] pgmap v2767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:34 smithi006 bash[33918]: audit 2023-08-27T23:30:34.068119+0000 mon.a (mon.0) 2314 : audit [DBG] from='client.? 172.21.15.6:0/2551376503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:34 smithi006 bash[27292]: cluster 2023-08-27T23:30:33.158422+0000 mgr.y (mgr.14901) 2760 : cluster [DBG] pgmap v2767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:34 smithi006 bash[27292]: audit 2023-08-27T23:30:34.068119+0000 mon.a (mon.0) 2314 : audit [DBG] from='client.? 172.21.15.6:0/2551376503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:35.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:35 smithi130 bash[51302]: debug 2023-08-27T23:30:35.534+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:35.535961+0000) 2023-08-27T23:30:36.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:36 smithi130 bash[51302]: debug 2023-08-27T23:30:36.534+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:36.536373+0000) 2023-08-27T23:30:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:36 smithi130 bash[27579]: cluster 2023-08-27T23:30:35.159553+0000 mgr.y (mgr.14901) 2761 : cluster [DBG] pgmap v2768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:36 smithi130 bash[27579]: audit 2023-08-27T23:30:36.482869+0000 mon.a (mon.0) 2315 : audit [DBG] from='client.? 172.21.15.6:0/2248509720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:37.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:36 smithi006 bash[33918]: cluster 2023-08-27T23:30:35.159553+0000 mgr.y (mgr.14901) 2761 : cluster [DBG] pgmap v2768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:37.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:36 smithi006 bash[33918]: audit 2023-08-27T23:30:36.482869+0000 mon.a (mon.0) 2315 : audit [DBG] from='client.? 172.21.15.6:0/2248509720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:37.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:36 smithi006 bash[27292]: cluster 2023-08-27T23:30:35.159553+0000 mgr.y (mgr.14901) 2761 : cluster [DBG] pgmap v2768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:37.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:36 smithi006 bash[27292]: audit 2023-08-27T23:30:36.482869+0000 mon.a (mon.0) 2315 : audit [DBG] from='client.? 172.21.15.6:0/2248509720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:37.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:37 smithi130 bash[51302]: debug 2023-08-27T23:30:37.534+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:37.536657+0000) 2023-08-27T23:30:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:38 smithi130 bash[27579]: cluster 2023-08-27T23:30:37.160236+0000 mgr.y (mgr.14901) 2762 : cluster [DBG] pgmap v2769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:38 smithi130 bash[51302]: debug 2023-08-27T23:30:38.534+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:38.536964+0000) 2023-08-27T23:30:39.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:38 smithi006 bash[33918]: cluster 2023-08-27T23:30:37.160236+0000 mgr.y (mgr.14901) 2762 : cluster [DBG] pgmap v2769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:39.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:38 smithi006 bash[27292]: cluster 2023-08-27T23:30:37.160236+0000 mgr.y (mgr.14901) 2762 : cluster [DBG] pgmap v2769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:39.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:39 smithi130 bash[27579]: audit 2023-08-27T23:30:38.895388+0000 mon.c (mon.2) 1075 : audit [DBG] from='client.? 172.21.15.6:0/2238948223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:39 smithi130 bash[51302]: debug 2023-08-27T23:30:39.534+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:39.537276+0000) 2023-08-27T23:30:40.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:39 smithi006 bash[27292]: audit 2023-08-27T23:30:38.895388+0000 mon.c (mon.2) 1075 : audit [DBG] from='client.? 172.21.15.6:0/2238948223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:40.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:39 smithi006 bash[33918]: audit 2023-08-27T23:30:38.895388+0000 mon.c (mon.2) 1075 : audit [DBG] from='client.? 172.21.15.6:0/2238948223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:40.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:30:40.487Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:40.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:30:40.489Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:40.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:40 smithi130 bash[51302]: debug 2023-08-27T23:30:40.534+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:40.537558+0000) 2023-08-27T23:30:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:40 smithi130 bash[27579]: cluster 2023-08-27T23:30:39.161347+0000 mgr.y (mgr.14901) 2763 : cluster [DBG] pgmap v2770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:40 smithi006 bash[27292]: cluster 2023-08-27T23:30:39.161347+0000 mgr.y (mgr.14901) 2763 : cluster [DBG] pgmap v2770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:30:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:30:40] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:30:41.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:40 smithi006 bash[33918]: cluster 2023-08-27T23:30:39.161347+0000 mgr.y (mgr.14901) 2763 : cluster [DBG] pgmap v2770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:41.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:41 smithi130 bash[27579]: audit 2023-08-27T23:30:41.313962+0000 mon.c (mon.2) 1076 : audit [DBG] from='client.? 172.21.15.6:0/1499901580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:41.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:41 smithi130 bash[51302]: debug 2023-08-27T23:30:41.534+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:41.537828+0000) 2023-08-27T23:30:42.051 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:41 smithi006 bash[33918]: audit 2023-08-27T23:30:41.313962+0000 mon.c (mon.2) 1076 : audit [DBG] from='client.? 172.21.15.6:0/1499901580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:42.051 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:41 smithi006 bash[27292]: audit 2023-08-27T23:30:41.313962+0000 mon.c (mon.2) 1076 : audit [DBG] from='client.? 172.21.15.6:0/1499901580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:42.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:30:42.049Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:30:42.051Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:42 smithi130 bash[27579]: cluster 2023-08-27T23:30:41.161958+0000 mgr.y (mgr.14901) 2764 : cluster [DBG] pgmap v2771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:42 smithi130 bash[51302]: debug 2023-08-27T23:30:42.534+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:42.538123+0000) 2023-08-27T23:30:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:42 smithi006 bash[27292]: cluster 2023-08-27T23:30:41.161958+0000 mgr.y (mgr.14901) 2764 : cluster [DBG] pgmap v2771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:42 smithi006 bash[33918]: cluster 2023-08-27T23:30:41.161958+0000 mgr.y (mgr.14901) 2764 : cluster [DBG] pgmap v2771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:43.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:43 smithi130 bash[27579]: audit 2023-08-27T23:30:43.731647+0000 mon.c (mon.2) 1077 : audit [DBG] from='client.? 172.21.15.6:0/2402339399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:43.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:43 smithi130 bash[51302]: debug 2023-08-27T23:30:43.534+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:43.538425+0000) 2023-08-27T23:30:44.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:43 smithi006 bash[27292]: audit 2023-08-27T23:30:43.731647+0000 mon.c (mon.2) 1077 : audit [DBG] from='client.? 172.21.15.6:0/2402339399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:44.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:43 smithi006 bash[33918]: audit 2023-08-27T23:30:43.731647+0000 mon.c (mon.2) 1077 : audit [DBG] from='client.? 172.21.15.6:0/2402339399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:30:43.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:30:43.955Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:44 smithi130 bash[27579]: cluster 2023-08-27T23:30:43.162600+0000 mgr.y (mgr.14901) 2765 : cluster [DBG] pgmap v2772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:44 smithi130 bash[51302]: debug 2023-08-27T23:30:44.537+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:44.538697+0000) 2023-08-27T23:30:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:44 smithi006 bash[33918]: cluster 2023-08-27T23:30:43.162600+0000 mgr.y (mgr.14901) 2765 : cluster [DBG] pgmap v2772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:44 smithi006 bash[27292]: cluster 2023-08-27T23:30:43.162600+0000 mgr.y (mgr.14901) 2765 : cluster [DBG] pgmap v2772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:45.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:45 smithi130 bash[51302]: debug 2023-08-27T23:30:45.537+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:45.539027+0000) 2023-08-27T23:30:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:46 smithi130 bash[27579]: cluster 2023-08-27T23:30:45.163741+0000 mgr.y (mgr.14901) 2766 : cluster [DBG] pgmap v2773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:46 smithi130 bash[27579]: audit 2023-08-27T23:30:46.150982+0000 mon.a (mon.0) 2316 : audit [DBG] from='client.? 172.21.15.6:0/654545734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:46.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:46 smithi130 bash[51302]: debug 2023-08-27T23:30:46.537+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:46.539256+0000) 2023-08-27T23:30:47.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:46 smithi006 bash[27292]: cluster 2023-08-27T23:30:45.163741+0000 mgr.y (mgr.14901) 2766 : cluster [DBG] pgmap v2773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:47.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:46 smithi006 bash[27292]: audit 2023-08-27T23:30:46.150982+0000 mon.a (mon.0) 2316 : audit [DBG] from='client.? 172.21.15.6:0/654545734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:47.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:46 smithi006 bash[33918]: cluster 2023-08-27T23:30:45.163741+0000 mgr.y (mgr.14901) 2766 : cluster [DBG] pgmap v2773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:47.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:46 smithi006 bash[33918]: audit 2023-08-27T23:30:46.150982+0000 mon.a (mon.0) 2316 : audit [DBG] from='client.? 172.21.15.6:0/654545734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:47.870 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:47 smithi130 bash[51302]: debug 2023-08-27T23:30:47.537+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:47.539558+0000) 2023-08-27T23:30:48.791 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:48 smithi130 bash[51302]: debug 2023-08-27T23:30:48.537+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:48.539903+0000) 2023-08-27T23:30:49.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:48 smithi130 bash[27579]: cluster 2023-08-27T23:30:47.164420+0000 mgr.y (mgr.14901) 2767 : cluster [DBG] pgmap v2774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:49.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:48 smithi130 bash[27579]: audit 2023-08-27T23:30:48.562978+0000 mon.c (mon.2) 1078 : audit [DBG] from='client.? 172.21.15.6:0/445211209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:49.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:48 smithi006 bash[33918]: cluster 2023-08-27T23:30:47.164420+0000 mgr.y (mgr.14901) 2767 : cluster [DBG] pgmap v2774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:49.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:48 smithi006 bash[33918]: audit 2023-08-27T23:30:48.562978+0000 mon.c (mon.2) 1078 : audit [DBG] from='client.? 172.21.15.6:0/445211209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:49.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:48 smithi006 bash[27292]: cluster 2023-08-27T23:30:47.164420+0000 mgr.y (mgr.14901) 2767 : cluster [DBG] pgmap v2774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:49.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:48 smithi006 bash[27292]: audit 2023-08-27T23:30:48.562978+0000 mon.c (mon.2) 1078 : audit [DBG] from='client.? 172.21.15.6:0/445211209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:49.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:49 smithi130 bash[51302]: debug 2023-08-27T23:30:49.537+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:49.540194+0000) 2023-08-27T23:30:50.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:30:50.488Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:50.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:30:50.490Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:50.796 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:50 smithi130 bash[51302]: debug 2023-08-27T23:30:50.537+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:50.540463+0000) 2023-08-27T23:30:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:50 smithi130 bash[27579]: cluster 2023-08-27T23:30:49.165528+0000 mgr.y (mgr.14901) 2768 : cluster [DBG] pgmap v2775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:30:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:30:50] "GET /metrics HTTP/1.1" 200 193087 "" "Prometheus/2.33.4" 2023-08-27T23:30:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:50 smithi006 bash[33918]: cluster 2023-08-27T23:30:49.165528+0000 mgr.y (mgr.14901) 2768 : cluster [DBG] pgmap v2775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:50 smithi006 bash[27292]: cluster 2023-08-27T23:30:49.165528+0000 mgr.y (mgr.14901) 2768 : cluster [DBG] pgmap v2775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:51.805 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:51 smithi130 bash[51302]: debug 2023-08-27T23:30:51.537+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:51.540669+0000) 2023-08-27T23:30:52.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:51 smithi130 bash[27579]: audit 2023-08-27T23:30:50.981450+0000 mon.a (mon.0) 2317 : audit [DBG] from='client.? 172.21.15.6:0/403090649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:52.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:51 smithi006 bash[27292]: audit 2023-08-27T23:30:50.981450+0000 mon.a (mon.0) 2317 : audit [DBG] from='client.? 172.21.15.6:0/403090649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:52.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:51 smithi006 bash[33918]: audit 2023-08-27T23:30:50.981450+0000 mon.a (mon.0) 2317 : audit [DBG] from='client.? 172.21.15.6:0/403090649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:52.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:30:52.050Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:52.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:30:52.052Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:52.809 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:52 smithi130 bash[51302]: debug 2023-08-27T23:30:52.537+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:52.540908+0000) 2023-08-27T23:30:53.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:52 smithi130 bash[27579]: cluster 2023-08-27T23:30:51.166187+0000 mgr.y (mgr.14901) 2769 : cluster [DBG] pgmap v2776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:53.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:52 smithi006 bash[33918]: cluster 2023-08-27T23:30:51.166187+0000 mgr.y (mgr.14901) 2769 : cluster [DBG] pgmap v2776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:52 smithi006 bash[27292]: cluster 2023-08-27T23:30:51.166187+0000 mgr.y (mgr.14901) 2769 : cluster [DBG] pgmap v2776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:53.816 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:53 smithi130 bash[51302]: debug 2023-08-27T23:30:53.537+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:53.541179+0000) 2023-08-27T23:30:54.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:53 smithi130 bash[27579]: audit 2023-08-27T23:30:53.399242+0000 mon.a (mon.0) 2318 : audit [DBG] from='client.? 172.21.15.6:0/4044902764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:54.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:53 smithi006 bash[33918]: audit 2023-08-27T23:30:53.399242+0000 mon.a (mon.0) 2318 : audit [DBG] from='client.? 172.21.15.6:0/4044902764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:54.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:53 smithi006 bash[27292]: audit 2023-08-27T23:30:53.399242+0000 mon.a (mon.0) 2318 : audit [DBG] from='client.? 172.21.15.6:0/4044902764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:30:53.953Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:30:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:30:53.956Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:30:54.819 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:54 smithi130 bash[51302]: debug 2023-08-27T23:30:54.537+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:54.541450+0000) 2023-08-27T23:30:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:54 smithi130 bash[27579]: cluster 2023-08-27T23:30:53.166827+0000 mgr.y (mgr.14901) 2770 : cluster [DBG] pgmap v2777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:54 smithi006 bash[33918]: cluster 2023-08-27T23:30:53.166827+0000 mgr.y (mgr.14901) 2770 : cluster [DBG] pgmap v2777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:54 smithi006 bash[27292]: cluster 2023-08-27T23:30:53.166827+0000 mgr.y (mgr.14901) 2770 : cluster [DBG] pgmap v2777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:55.824 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:55 smithi130 bash[51302]: debug 2023-08-27T23:30:55.537+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:55.541754+0000) 2023-08-27T23:30:56.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:55 smithi130 bash[27579]: audit 2023-08-27T23:30:55.814095+0000 mon.a (mon.0) 2319 : audit [DBG] from='client.? 172.21.15.6:0/1627903847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:56.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:55 smithi006 bash[33918]: audit 2023-08-27T23:30:55.814095+0000 mon.a (mon.0) 2319 : audit [DBG] from='client.? 172.21.15.6:0/1627903847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:56.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:55 smithi006 bash[27292]: audit 2023-08-27T23:30:55.814095+0000 mon.a (mon.0) 2319 : audit [DBG] from='client.? 172.21.15.6:0/1627903847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:56.830 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:56 smithi130 bash[51302]: debug 2023-08-27T23:30:56.537+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:56.542006+0000) 2023-08-27T23:30:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:56 smithi130 bash[27579]: cluster 2023-08-27T23:30:55.167959+0000 mgr.y (mgr.14901) 2771 : cluster [DBG] pgmap v2778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:57.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:56 smithi006 bash[33918]: cluster 2023-08-27T23:30:55.167959+0000 mgr.y (mgr.14901) 2771 : cluster [DBG] pgmap v2778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:57.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:56 smithi006 bash[27292]: cluster 2023-08-27T23:30:55.167959+0000 mgr.y (mgr.14901) 2771 : cluster [DBG] pgmap v2778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:57.838 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:57 smithi130 bash[51302]: debug 2023-08-27T23:30:57.537+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:57.542313+0000) 2023-08-27T23:30:58.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:57 smithi130 bash[27579]: cluster 2023-08-27T23:30:57.168606+0000 mgr.y (mgr.14901) 2772 : cluster [DBG] pgmap v2779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:58.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:57 smithi006 bash[27292]: cluster 2023-08-27T23:30:57.168606+0000 mgr.y (mgr.14901) 2772 : cluster [DBG] pgmap v2779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:58.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:57 smithi006 bash[33918]: cluster 2023-08-27T23:30:57.168606+0000 mgr.y (mgr.14901) 2772 : cluster [DBG] pgmap v2779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:30:58.841 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:58 smithi130 bash[51302]: debug 2023-08-27T23:30:58.537+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:58.542582+0000) 2023-08-27T23:30:59.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:30:58 smithi130 bash[27579]: audit 2023-08-27T23:30:58.227911+0000 mon.c (mon.2) 1079 : audit [DBG] from='client.? 172.21.15.6:0/4001371344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:30:58 smithi006 bash[27292]: audit 2023-08-27T23:30:58.227911+0000 mon.c (mon.2) 1079 : audit [DBG] from='client.? 172.21.15.6:0/4001371344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:30:58 smithi006 bash[33918]: audit 2023-08-27T23:30:58.227911+0000 mon.c (mon.2) 1079 : audit [DBG] from='client.? 172.21.15.6:0/4001371344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:59.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:30:59 smithi130 bash[51302]: debug 2023-08-27T23:30:59.541+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:59.542873+0000) 2023-08-27T23:31:00.749 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:31:00.488Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:00.749 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:31:00.490Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:00.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:00 smithi130 bash[51302]: debug 2023-08-27T23:31:00.541+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:00.543164+0000) 2023-08-27T23:31:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:00 smithi130 bash[27579]: cluster 2023-08-27T23:30:59.169708+0000 mgr.y (mgr.14901) 2773 : cluster [DBG] pgmap v2780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:00 smithi130 bash[27579]: audit 2023-08-27T23:31:00.654850+0000 mon.a (mon.0) 2320 : audit [DBG] from='client.? 172.21.15.6:0/1505614730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:31:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:31:00] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:31:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:00 smithi006 bash[27292]: cluster 2023-08-27T23:30:59.169708+0000 mgr.y (mgr.14901) 2773 : cluster [DBG] pgmap v2780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:00 smithi006 bash[27292]: audit 2023-08-27T23:31:00.654850+0000 mon.a (mon.0) 2320 : audit [DBG] from='client.? 172.21.15.6:0/1505614730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:00 smithi006 bash[33918]: cluster 2023-08-27T23:30:59.169708+0000 mgr.y (mgr.14901) 2773 : cluster [DBG] pgmap v2780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:01.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:00 smithi006 bash[33918]: audit 2023-08-27T23:31:00.654850+0000 mon.a (mon.0) 2320 : audit [DBG] from='client.? 172.21.15.6:0/1505614730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:01.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:01 smithi130 bash[51302]: debug 2023-08-27T23:31:01.541+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:01.543444+0000) 2023-08-27T23:31:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:31:02.051Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:31:02.053Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:02.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:02 smithi130 bash[51302]: debug 2023-08-27T23:31:02.541+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:02.543788+0000) 2023-08-27T23:31:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:02 smithi130 bash[27579]: cluster 2023-08-27T23:31:01.170360+0000 mgr.y (mgr.14901) 2774 : cluster [DBG] pgmap v2781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:02 smithi006 bash[33918]: cluster 2023-08-27T23:31:01.170360+0000 mgr.y (mgr.14901) 2774 : cluster [DBG] pgmap v2781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:02 smithi006 bash[27292]: cluster 2023-08-27T23:31:01.170360+0000 mgr.y (mgr.14901) 2774 : cluster [DBG] pgmap v2781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:03.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:03 smithi130 bash[27579]: audit 2023-08-27T23:31:03.067321+0000 mon.c (mon.2) 1080 : audit [DBG] from='client.? 172.21.15.6:0/3652169764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:03.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:03 smithi130 bash[51302]: debug 2023-08-27T23:31:03.541+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:03.544098+0000) 2023-08-27T23:31:04.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:03 smithi006 bash[33918]: audit 2023-08-27T23:31:03.067321+0000 mon.c (mon.2) 1080 : audit [DBG] from='client.? 172.21.15.6:0/3652169764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:04.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:03 smithi006 bash[27292]: audit 2023-08-27T23:31:03.067321+0000 mon.c (mon.2) 1080 : audit [DBG] from='client.? 172.21.15.6:0/3652169764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:31:03.954Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:31:03.956Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:04 smithi130 bash[27579]: cluster 2023-08-27T23:31:03.171045+0000 mgr.y (mgr.14901) 2775 : cluster [DBG] pgmap v2782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:04.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:04 smithi130 bash[51302]: debug 2023-08-27T23:31:04.541+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:04.544418+0000) 2023-08-27T23:31:05.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:04 smithi006 bash[33918]: cluster 2023-08-27T23:31:03.171045+0000 mgr.y (mgr.14901) 2775 : cluster [DBG] pgmap v2782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:04 smithi006 bash[27292]: cluster 2023-08-27T23:31:03.171045+0000 mgr.y (mgr.14901) 2775 : cluster [DBG] pgmap v2782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:05.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:05 smithi130 bash[27579]: audit 2023-08-27T23:31:05.483235+0000 mon.a (mon.0) 2321 : audit [DBG] from='client.? 172.21.15.6:0/3901342703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:05.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:05 smithi130 bash[51302]: debug 2023-08-27T23:31:05.541+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:05.544743+0000) 2023-08-27T23:31:06.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:05 smithi006 bash[27292]: audit 2023-08-27T23:31:05.483235+0000 mon.a (mon.0) 2321 : audit [DBG] from='client.? 172.21.15.6:0/3901342703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:05 smithi006 bash[33918]: audit 2023-08-27T23:31:05.483235+0000 mon.a (mon.0) 2321 : audit [DBG] from='client.? 172.21.15.6:0/3901342703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:06 smithi130 bash[27579]: cluster 2023-08-27T23:31:05.171774+0000 mgr.y (mgr.14901) 2776 : cluster [DBG] pgmap v2783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:06.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:06 smithi130 bash[51302]: debug 2023-08-27T23:31:06.541+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:06.545014+0000) 2023-08-27T23:31:07.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:06 smithi006 bash[33918]: cluster 2023-08-27T23:31:05.171774+0000 mgr.y (mgr.14901) 2776 : cluster [DBG] pgmap v2783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:07.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:06 smithi006 bash[27292]: cluster 2023-08-27T23:31:05.171774+0000 mgr.y (mgr.14901) 2776 : cluster [DBG] pgmap v2783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:07.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:07 smithi130 bash[51302]: debug 2023-08-27T23:31:07.541+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:07.545342+0000) 2023-08-27T23:31:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:08 smithi130 bash[27579]: cluster 2023-08-27T23:31:07.172513+0000 mgr.y (mgr.14901) 2777 : cluster [DBG] pgmap v2784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:08 smithi130 bash[27579]: audit 2023-08-27T23:31:07.897219+0000 mon.a (mon.0) 2322 : audit [DBG] from='client.? 172.21.15.6:0/2266156371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:08.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:08 smithi130 bash[51302]: debug 2023-08-27T23:31:08.541+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:08.545642+0000) 2023-08-27T23:31:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:08 smithi006 bash[33918]: cluster 2023-08-27T23:31:07.172513+0000 mgr.y (mgr.14901) 2777 : cluster [DBG] pgmap v2784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:08 smithi006 bash[33918]: audit 2023-08-27T23:31:07.897219+0000 mon.a (mon.0) 2322 : audit [DBG] from='client.? 172.21.15.6:0/2266156371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:08 smithi006 bash[27292]: cluster 2023-08-27T23:31:07.172513+0000 mgr.y (mgr.14901) 2777 : cluster [DBG] pgmap v2784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:08 smithi006 bash[27292]: audit 2023-08-27T23:31:07.897219+0000 mon.a (mon.0) 2322 : audit [DBG] from='client.? 172.21.15.6:0/2266156371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:09.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:09 smithi130 bash[51302]: debug 2023-08-27T23:31:09.541+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:09.546076+0000) 2023-08-27T23:31:10.756 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:31:10.490Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:10.757 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:31:10.492Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:10.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:10 smithi130 bash[51302]: debug 2023-08-27T23:31:10.541+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:10.546357+0000) 2023-08-27T23:31:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:10 smithi130 bash[27579]: cluster 2023-08-27T23:31:09.173658+0000 mgr.y (mgr.14901) 2778 : cluster [DBG] pgmap v2785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:10 smithi130 bash[27579]: audit 2023-08-27T23:31:10.314467+0000 mon.c (mon.2) 1081 : audit [DBG] from='client.? 172.21.15.6:0/2105773448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:31:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:31:10] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:31:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:10 smithi006 bash[33918]: cluster 2023-08-27T23:31:09.173658+0000 mgr.y (mgr.14901) 2778 : cluster [DBG] pgmap v2785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:10 smithi006 bash[33918]: audit 2023-08-27T23:31:10.314467+0000 mon.c (mon.2) 1081 : audit [DBG] from='client.? 172.21.15.6:0/2105773448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:11.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:10 smithi006 bash[27292]: cluster 2023-08-27T23:31:09.173658+0000 mgr.y (mgr.14901) 2778 : cluster [DBG] pgmap v2785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:11.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:10 smithi006 bash[27292]: audit 2023-08-27T23:31:10.314467+0000 mon.c (mon.2) 1081 : audit [DBG] from='client.? 172.21.15.6:0/2105773448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:11.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:11 smithi130 bash[51302]: debug 2023-08-27T23:31:11.541+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:11.546625+0000) 2023-08-27T23:31:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:31:12.052Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:31:12.054Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:12 smithi130 bash[27579]: cluster 2023-08-27T23:31:11.174241+0000 mgr.y (mgr.14901) 2779 : cluster [DBG] pgmap v2786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:12 smithi130 bash[27579]: audit 2023-08-27T23:31:12.738390+0000 mon.a (mon.0) 2323 : audit [DBG] from='client.? 172.21.15.6:0/4004229262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:12 smithi130 bash[51302]: debug 2023-08-27T23:31:12.545+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:12.546951+0000) 2023-08-27T23:31:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:12 smithi006 bash[33918]: cluster 2023-08-27T23:31:11.174241+0000 mgr.y (mgr.14901) 2779 : cluster [DBG] pgmap v2786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:12 smithi006 bash[33918]: audit 2023-08-27T23:31:12.738390+0000 mon.a (mon.0) 2323 : audit [DBG] from='client.? 172.21.15.6:0/4004229262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:12 smithi006 bash[27292]: cluster 2023-08-27T23:31:11.174241+0000 mgr.y (mgr.14901) 2779 : cluster [DBG] pgmap v2786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:12 smithi006 bash[27292]: audit 2023-08-27T23:31:12.738390+0000 mon.a (mon.0) 2323 : audit [DBG] from='client.? 172.21.15.6:0/4004229262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:13.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:13 smithi130 bash[51302]: debug 2023-08-27T23:31:13.545+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:13.547237+0000) 2023-08-27T23:31:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:31:13.955Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:31:13.957Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:14.803 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:14 smithi130 bash[51302]: debug 2023-08-27T23:31:14.545+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:14.547498+0000) 2023-08-27T23:31:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:14 smithi130 bash[27579]: cluster 2023-08-27T23:31:13.174948+0000 mgr.y (mgr.14901) 2780 : cluster [DBG] pgmap v2787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:14 smithi006 bash[27292]: cluster 2023-08-27T23:31:13.174948+0000 mgr.y (mgr.14901) 2780 : cluster [DBG] pgmap v2787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:14 smithi006 bash[33918]: cluster 2023-08-27T23:31:13.174948+0000 mgr.y (mgr.14901) 2780 : cluster [DBG] pgmap v2787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:15.810 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:15 smithi130 bash[51302]: debug 2023-08-27T23:31:15.545+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:15.547774+0000) 2023-08-27T23:31:16.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:15 smithi130 bash[27579]: audit 2023-08-27T23:31:15.154193+0000 mon.a (mon.0) 2324 : audit [DBG] from='client.? 172.21.15.6:0/3488922767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:16.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:15 smithi006 bash[33918]: audit 2023-08-27T23:31:15.154193+0000 mon.a (mon.0) 2324 : audit [DBG] from='client.? 172.21.15.6:0/3488922767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:16.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:15 smithi006 bash[27292]: audit 2023-08-27T23:31:15.154193+0000 mon.a (mon.0) 2324 : audit [DBG] from='client.? 172.21.15.6:0/3488922767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:16.814 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:16 smithi130 bash[51302]: debug 2023-08-27T23:31:16.545+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:16.548034+0000) 2023-08-27T23:31:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:16 smithi130 bash[27579]: cluster 2023-08-27T23:31:15.176178+0000 mgr.y (mgr.14901) 2781 : cluster [DBG] pgmap v2788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:17.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:16 smithi006 bash[27292]: cluster 2023-08-27T23:31:15.176178+0000 mgr.y (mgr.14901) 2781 : cluster [DBG] pgmap v2788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:17.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:16 smithi006 bash[33918]: cluster 2023-08-27T23:31:15.176178+0000 mgr.y (mgr.14901) 2781 : cluster [DBG] pgmap v2788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:17.823 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:17 smithi130 bash[51302]: debug 2023-08-27T23:31:17.545+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:17.548307+0000) 2023-08-27T23:31:18.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:17 smithi130 bash[27579]: audit 2023-08-27T23:31:17.574213+0000 mon.a (mon.0) 2325 : audit [DBG] from='client.? 172.21.15.6:0/2209701653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:18.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:17 smithi006 bash[33918]: audit 2023-08-27T23:31:17.574213+0000 mon.a (mon.0) 2325 : audit [DBG] from='client.? 172.21.15.6:0/2209701653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:18.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:17 smithi006 bash[27292]: audit 2023-08-27T23:31:17.574213+0000 mon.a (mon.0) 2325 : audit [DBG] from='client.? 172.21.15.6:0/2209701653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:18.827 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:18 smithi130 bash[51302]: debug 2023-08-27T23:31:18.545+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:18.548583+0000) 2023-08-27T23:31:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:18 smithi130 bash[27579]: cluster 2023-08-27T23:31:17.176875+0000 mgr.y (mgr.14901) 2782 : cluster [DBG] pgmap v2789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:18 smithi006 bash[33918]: cluster 2023-08-27T23:31:17.176875+0000 mgr.y (mgr.14901) 2782 : cluster [DBG] pgmap v2789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:18 smithi006 bash[27292]: cluster 2023-08-27T23:31:17.176875+0000 mgr.y (mgr.14901) 2782 : cluster [DBG] pgmap v2789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:19 smithi130 bash[51302]: debug 2023-08-27T23:31:19.545+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:19.548859+0000) 2023-08-27T23:31:20.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:31:20.491Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:20.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:31:20.493Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:20.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:20 smithi130 bash[27579]: cluster 2023-08-27T23:31:19.177899+0000 mgr.y (mgr.14901) 2783 : cluster [DBG] pgmap v2790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:20 smithi130 bash[27579]: audit 2023-08-27T23:31:20.020021+0000 mon.c (mon.2) 1082 : audit [DBG] from='client.? 172.21.15.6:0/2607236787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:20 smithi130 bash[51302]: debug 2023-08-27T23:31:20.545+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:20.549137+0000) 2023-08-27T23:31:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:31:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:31:20] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:31:21.706 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:20 smithi006 bash[27292]: cluster 2023-08-27T23:31:19.177899+0000 mgr.y (mgr.14901) 2783 : cluster [DBG] pgmap v2790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:21.707 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:20 smithi006 bash[27292]: audit 2023-08-27T23:31:20.020021+0000 mon.c (mon.2) 1082 : audit [DBG] from='client.? 172.21.15.6:0/2607236787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:21.707 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:20 smithi006 bash[33918]: cluster 2023-08-27T23:31:19.177899+0000 mgr.y (mgr.14901) 2783 : cluster [DBG] pgmap v2790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:21.708 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:20 smithi006 bash[33918]: audit 2023-08-27T23:31:20.020021+0000 mon.c (mon.2) 1082 : audit [DBG] from='client.? 172.21.15.6:0/2607236787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:21.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:21 smithi130 bash[51302]: debug 2023-08-27T23:31:21.545+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:21.549393+0000) 2023-08-27T23:31:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:31:22.053Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:31:22.055Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:22 smithi130 bash[51302]: debug 2023-08-27T23:31:22.545+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:22.549713+0000) 2023-08-27T23:31:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:22 smithi130 bash[27579]: cluster 2023-08-27T23:31:21.178650+0000 mgr.y (mgr.14901) 2784 : cluster [DBG] pgmap v2791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:22 smithi130 bash[27579]: audit 2023-08-27T23:31:22.437923+0000 mon.a (mon.0) 2326 : audit [DBG] from='client.? 172.21.15.6:0/902517172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:22 smithi006 bash[27292]: cluster 2023-08-27T23:31:21.178650+0000 mgr.y (mgr.14901) 2784 : cluster [DBG] pgmap v2791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:22 smithi006 bash[27292]: audit 2023-08-27T23:31:22.437923+0000 mon.a (mon.0) 2326 : audit [DBG] from='client.? 172.21.15.6:0/902517172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:22 smithi006 bash[33918]: cluster 2023-08-27T23:31:21.178650+0000 mgr.y (mgr.14901) 2784 : cluster [DBG] pgmap v2791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:22 smithi006 bash[33918]: audit 2023-08-27T23:31:22.437923+0000 mon.a (mon.0) 2326 : audit [DBG] from='client.? 172.21.15.6:0/902517172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:23.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:23 smithi130 bash[51302]: debug 2023-08-27T23:31:23.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:23.549998+0000) 2023-08-27T23:31:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:31:23.956Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:31:23.959Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:24 smithi130 bash[27579]: cluster 2023-08-27T23:31:23.179427+0000 mgr.y (mgr.14901) 2785 : cluster [DBG] pgmap v2792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:24 smithi130 bash[51302]: debug 2023-08-27T23:31:24.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:24.550252+0000) 2023-08-27T23:31:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:24 smithi006 bash[33918]: cluster 2023-08-27T23:31:23.179427+0000 mgr.y (mgr.14901) 2785 : cluster [DBG] pgmap v2792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:24 smithi006 bash[27292]: cluster 2023-08-27T23:31:23.179427+0000 mgr.y (mgr.14901) 2785 : cluster [DBG] pgmap v2792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:25.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:25 smithi130 bash[27579]: audit 2023-08-27T23:31:24.853580+0000 mon.c (mon.2) 1083 : audit [DBG] from='client.? 172.21.15.6:0/3238749366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:25.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:25 smithi130 bash[51302]: debug 2023-08-27T23:31:25.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:25.550535+0000) 2023-08-27T23:31:26.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:25 smithi006 bash[33918]: audit 2023-08-27T23:31:24.853580+0000 mon.c (mon.2) 1083 : audit [DBG] from='client.? 172.21.15.6:0/3238749366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:25 smithi006 bash[27292]: audit 2023-08-27T23:31:24.853580+0000 mon.c (mon.2) 1083 : audit [DBG] from='client.? 172.21.15.6:0/3238749366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:26.803 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:26 smithi130 bash[51302]: debug 2023-08-27T23:31:26.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:26.550809+0000) 2023-08-27T23:31:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:26 smithi130 bash[27579]: cluster 2023-08-27T23:31:25.180559+0000 mgr.y (mgr.14901) 2786 : cluster [DBG] pgmap v2793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:26 smithi006 bash[27292]: cluster 2023-08-27T23:31:25.180559+0000 mgr.y (mgr.14901) 2786 : cluster [DBG] pgmap v2793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:26 smithi006 bash[33918]: cluster 2023-08-27T23:31:25.180559+0000 mgr.y (mgr.14901) 2786 : cluster [DBG] pgmap v2793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:27.813 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:27 smithi130 bash[51302]: debug 2023-08-27T23:31:27.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:27.551110+0000) 2023-08-27T23:31:28.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:27 smithi130 bash[27579]: audit 2023-08-27T23:31:27.274017+0000 mon.c (mon.2) 1084 : audit [DBG] from='client.? 172.21.15.6:0/1495042238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:28.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:27 smithi006 bash[27292]: audit 2023-08-27T23:31:27.274017+0000 mon.c (mon.2) 1084 : audit [DBG] from='client.? 172.21.15.6:0/1495042238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:28.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:27 smithi006 bash[33918]: audit 2023-08-27T23:31:27.274017+0000 mon.c (mon.2) 1084 : audit [DBG] from='client.? 172.21.15.6:0/1495042238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:28.816 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:28 smithi130 bash[51302]: debug 2023-08-27T23:31:28.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:28.551405+0000) 2023-08-27T23:31:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:28 smithi130 bash[27579]: cluster 2023-08-27T23:31:27.181178+0000 mgr.y (mgr.14901) 2787 : cluster [DBG] pgmap v2794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:28 smithi006 bash[27292]: cluster 2023-08-27T23:31:27.181178+0000 mgr.y (mgr.14901) 2787 : cluster [DBG] pgmap v2794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:28 smithi006 bash[33918]: cluster 2023-08-27T23:31:27.181178+0000 mgr.y (mgr.14901) 2787 : cluster [DBG] pgmap v2794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:29.822 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:29 smithi130 bash[51302]: debug 2023-08-27T23:31:29.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:29.551690+0000) 2023-08-27T23:31:30.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:29 smithi130 bash[27579]: audit 2023-08-27T23:31:29.690650+0000 mon.c (mon.2) 1085 : audit [DBG] from='client.? 172.21.15.6:0/3292219974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:30.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:29 smithi006 bash[27292]: audit 2023-08-27T23:31:29.690650+0000 mon.c (mon.2) 1085 : audit [DBG] from='client.? 172.21.15.6:0/3292219974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:30.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:29 smithi006 bash[33918]: audit 2023-08-27T23:31:29.690650+0000 mon.c (mon.2) 1085 : audit [DBG] from='client.? 172.21.15.6:0/3292219974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:30.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:31:30.492Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:30.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:31:30.494Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:30.825 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:30 smithi130 bash[51302]: debug 2023-08-27T23:31:30.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:30.551981+0000) 2023-08-27T23:31:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:30 smithi130 bash[27579]: cluster 2023-08-27T23:31:29.182238+0000 mgr.y (mgr.14901) 2788 : cluster [DBG] pgmap v2795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:31.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:30 smithi006 bash[33918]: cluster 2023-08-27T23:31:29.182238+0000 mgr.y (mgr.14901) 2788 : cluster [DBG] pgmap v2795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:31.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:30 smithi006 bash[27292]: cluster 2023-08-27T23:31:29.182238+0000 mgr.y (mgr.14901) 2788 : cluster [DBG] pgmap v2795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:31:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:31:30] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:31:31.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:31 smithi130 bash[51302]: debug 2023-08-27T23:31:31.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:31.552289+0000) 2023-08-27T23:31:32.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:31 smithi006 bash[27292]: cluster 2023-08-27T23:31:31.182861+0000 mgr.y (mgr.14901) 2789 : cluster [DBG] pgmap v2796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:32.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:31 smithi006 bash[33918]: cluster 2023-08-27T23:31:31.182861+0000 mgr.y (mgr.14901) 2789 : cluster [DBG] pgmap v2796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:32.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:31:32.054Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:32.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:31:32.056Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:32.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:31 smithi130 bash[27579]: cluster 2023-08-27T23:31:31.182861+0000 mgr.y (mgr.14901) 2789 : cluster [DBG] pgmap v2796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:32 smithi130 bash[51302]: debug 2023-08-27T23:31:32.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:32.552493+0000) 2023-08-27T23:31:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:32 smithi006 bash[27292]: audit 2023-08-27T23:31:32.080367+0000 mon.b (mon.1) 385 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:31:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:32 smithi006 bash[27292]: audit 2023-08-27T23:31:32.109835+0000 mon.a (mon.0) 2327 : audit [DBG] from='client.? 172.21.15.6:0/1569728317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:32 smithi006 bash[27292]: audit 2023-08-27T23:31:32.353470+0000 mon.a (mon.0) 2328 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:31:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:32 smithi006 bash[27292]: audit 2023-08-27T23:31:32.364877+0000 mon.a (mon.0) 2329 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:31:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:32 smithi006 bash[33918]: audit 2023-08-27T23:31:32.080367+0000 mon.b (mon.1) 385 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:31:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:32 smithi006 bash[33918]: audit 2023-08-27T23:31:32.109835+0000 mon.a (mon.0) 2327 : audit [DBG] from='client.? 172.21.15.6:0/1569728317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:32 smithi006 bash[33918]: audit 2023-08-27T23:31:32.353470+0000 mon.a (mon.0) 2328 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:31:33.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:32 smithi006 bash[33918]: audit 2023-08-27T23:31:32.364877+0000 mon.a (mon.0) 2329 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:31:33.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:32 smithi130 bash[27579]: audit 2023-08-27T23:31:32.080367+0000 mon.b (mon.1) 385 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:31:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:32 smithi130 bash[27579]: audit 2023-08-27T23:31:32.109835+0000 mon.a (mon.0) 2327 : audit [DBG] from='client.? 172.21.15.6:0/1569728317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:32 smithi130 bash[27579]: audit 2023-08-27T23:31:32.353470+0000 mon.a (mon.0) 2328 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:31:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:32 smithi130 bash[27579]: audit 2023-08-27T23:31:32.364877+0000 mon.a (mon.0) 2329 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:31:33.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:33 smithi130 bash[51302]: debug 2023-08-27T23:31:33.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:33.552792+0000) 2023-08-27T23:31:34.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:33 smithi006 bash[33918]: cluster 2023-08-27T23:31:33.183485+0000 mgr.y (mgr.14901) 2790 : cluster [DBG] pgmap v2797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:34.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:33 smithi006 bash[27292]: cluster 2023-08-27T23:31:33.183485+0000 mgr.y (mgr.14901) 2790 : cluster [DBG] pgmap v2797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:31:33.957Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:31:33.959Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:34.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:33 smithi130 bash[27579]: cluster 2023-08-27T23:31:33.183485+0000 mgr.y (mgr.14901) 2790 : cluster [DBG] pgmap v2797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:34.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:34 smithi130 bash[51302]: debug 2023-08-27T23:31:34.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:34.553080+0000) 2023-08-27T23:31:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:34 smithi006 bash[27292]: audit 2023-08-27T23:31:34.534023+0000 mon.a (mon.0) 2330 : audit [DBG] from='client.? 172.21.15.6:0/868498886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:34 smithi006 bash[33918]: audit 2023-08-27T23:31:34.534023+0000 mon.a (mon.0) 2330 : audit [DBG] from='client.? 172.21.15.6:0/868498886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:35.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:34 smithi130 bash[27579]: audit 2023-08-27T23:31:34.534023+0000 mon.a (mon.0) 2330 : audit [DBG] from='client.? 172.21.15.6:0/868498886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:35.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:35 smithi130 bash[51302]: debug 2023-08-27T23:31:35.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:35.553395+0000) 2023-08-27T23:31:36.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:35 smithi006 bash[27292]: cluster 2023-08-27T23:31:35.184405+0000 mgr.y (mgr.14901) 2791 : cluster [DBG] pgmap v2798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:36.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:35 smithi006 bash[33918]: cluster 2023-08-27T23:31:35.184405+0000 mgr.y (mgr.14901) 2791 : cluster [DBG] pgmap v2798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:35 smithi130 bash[27579]: cluster 2023-08-27T23:31:35.184405+0000 mgr.y (mgr.14901) 2791 : cluster [DBG] pgmap v2798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:36.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:36 smithi130 bash[51302]: debug 2023-08-27T23:31:36.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:36.553694+0000) 2023-08-27T23:31:37.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:36 smithi130 bash[27579]: audit 2023-08-27T23:31:36.941989+0000 mon.c (mon.2) 1086 : audit [DBG] from='client.? 172.21.15.6:0/402205491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:37.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:36 smithi006 bash[33918]: audit 2023-08-27T23:31:36.941989+0000 mon.c (mon.2) 1086 : audit [DBG] from='client.? 172.21.15.6:0/402205491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:37.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:36 smithi006 bash[27292]: audit 2023-08-27T23:31:36.941989+0000 mon.c (mon.2) 1086 : audit [DBG] from='client.? 172.21.15.6:0/402205491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:37.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:37 smithi130 bash[51302]: debug 2023-08-27T23:31:37.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:37.553977+0000) 2023-08-27T23:31:38.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:38 smithi130 bash[27579]: cluster 2023-08-27T23:31:37.184977+0000 mgr.y (mgr.14901) 2792 : cluster [DBG] pgmap v2799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:38.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:38 smithi006 bash[27292]: cluster 2023-08-27T23:31:37.184977+0000 mgr.y (mgr.14901) 2792 : cluster [DBG] pgmap v2799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:38.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:38 smithi006 bash[33918]: cluster 2023-08-27T23:31:37.184977+0000 mgr.y (mgr.14901) 2792 : cluster [DBG] pgmap v2799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:38 smithi130 bash[51302]: debug 2023-08-27T23:31:38.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:38.554241+0000) 2023-08-27T23:31:39.555 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:39 smithi130 bash[27579]: audit 2023-08-27T23:31:38.235024+0000 mon.a (mon.0) 2331 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:31:39.556 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:39 smithi130 bash[27579]: audit 2023-08-27T23:31:38.636041+0000 mon.a (mon.0) 2332 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:31:39.556 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:39 smithi130 bash[27579]: audit 2023-08-27T23:31:38.906726+0000 mon.b (mon.1) 386 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:31:39.556 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:39 smithi130 bash[27579]: audit 2023-08-27T23:31:38.909037+0000 mon.b (mon.1) 387 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:31:39.556 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:39 smithi130 bash[27579]: audit 2023-08-27T23:31:38.918271+0000 mon.a (mon.0) 2333 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:31:39.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:39 smithi006 bash[33918]: audit 2023-08-27T23:31:38.235024+0000 mon.a (mon.0) 2331 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:31:39.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:39 smithi006 bash[33918]: audit 2023-08-27T23:31:38.636041+0000 mon.a (mon.0) 2332 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:31:39.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:39 smithi006 bash[33918]: audit 2023-08-27T23:31:38.906726+0000 mon.b (mon.1) 386 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:31:39.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:39 smithi006 bash[33918]: audit 2023-08-27T23:31:38.909037+0000 mon.b (mon.1) 387 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:31:39.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:39 smithi006 bash[33918]: audit 2023-08-27T23:31:38.918271+0000 mon.a (mon.0) 2333 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:31:39.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:39 smithi006 bash[27292]: audit 2023-08-27T23:31:38.235024+0000 mon.a (mon.0) 2331 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:31:39.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:39 smithi006 bash[27292]: audit 2023-08-27T23:31:38.636041+0000 mon.a (mon.0) 2332 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:31:39.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:39 smithi006 bash[27292]: audit 2023-08-27T23:31:38.906726+0000 mon.b (mon.1) 386 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:31:39.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:39 smithi006 bash[27292]: audit 2023-08-27T23:31:38.909037+0000 mon.b (mon.1) 387 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:31:39.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:39 smithi006 bash[27292]: audit 2023-08-27T23:31:38.918271+0000 mon.a (mon.0) 2333 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:31:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:39 smithi130 bash[51302]: debug 2023-08-27T23:31:39.553+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:39.554552+0000) 2023-08-27T23:31:40.495 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:40 smithi006 bash[33918]: cluster 2023-08-27T23:31:39.186002+0000 mgr.y (mgr.14901) 2793 : cluster [DBG] pgmap v2800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:40.502 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:40 smithi006 bash[33918]: audit 2023-08-27T23:31:39.367243+0000 mon.a (mon.0) 2334 : audit [DBG] from='client.? 172.21.15.6:0/2583613282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:40.502 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:40 smithi006 bash[27292]: cluster 2023-08-27T23:31:39.186002+0000 mgr.y (mgr.14901) 2793 : cluster [DBG] pgmap v2800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:40.503 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:40 smithi006 bash[27292]: audit 2023-08-27T23:31:39.367243+0000 mon.a (mon.0) 2334 : audit [DBG] from='client.? 172.21.15.6:0/2583613282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:40.555 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:40 smithi130 bash[27579]: cluster 2023-08-27T23:31:39.186002+0000 mgr.y (mgr.14901) 2793 : cluster [DBG] pgmap v2800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:40.556 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:40 smithi130 bash[27579]: audit 2023-08-27T23:31:39.367243+0000 mon.a (mon.0) 2334 : audit [DBG] from='client.? 172.21.15.6:0/2583613282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:40.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:31:40.492Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:40.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:31:40.495Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:40.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:40 smithi130 bash[51302]: debug 2023-08-27T23:31:40.552+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:40.554929+0000) 2023-08-27T23:31:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:31:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:31:40] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:31:41.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:41 smithi130 bash[51302]: debug 2023-08-27T23:31:41.552+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:41.555235+0000) 2023-08-27T23:31:42.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:42 smithi006 bash[33918]: cluster 2023-08-27T23:31:41.186667+0000 mgr.y (mgr.14901) 2794 : cluster [DBG] pgmap v2801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:42.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:42 smithi006 bash[33918]: audit 2023-08-27T23:31:41.784789+0000 mon.c (mon.2) 1087 : audit [DBG] from='client.? 172.21.15.6:0/3223365541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:42.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:42 smithi006 bash[27292]: cluster 2023-08-27T23:31:41.186667+0000 mgr.y (mgr.14901) 2794 : cluster [DBG] pgmap v2801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:42.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:42 smithi006 bash[27292]: audit 2023-08-27T23:31:41.784789+0000 mon.c (mon.2) 1087 : audit [DBG] from='client.? 172.21.15.6:0/3223365541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:42.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:31:42.055Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:42.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:31:42.057Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:42.556 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:42 smithi130 bash[27579]: cluster 2023-08-27T23:31:41.186667+0000 mgr.y (mgr.14901) 2794 : cluster [DBG] pgmap v2801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:42.556 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:42 smithi130 bash[27579]: audit 2023-08-27T23:31:41.784789+0000 mon.c (mon.2) 1087 : audit [DBG] from='client.? 172.21.15.6:0/3223365541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:42.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:42 smithi130 bash[51302]: debug 2023-08-27T23:31:42.552+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:42.555601+0000) 2023-08-27T23:31:43.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:43 smithi130 bash[51302]: debug 2023-08-27T23:31:43.552+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:43.555950+0000) 2023-08-27T23:31:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:31:43.958Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:31:43.960Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:44.557 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:44 smithi130 bash[27579]: cluster 2023-08-27T23:31:43.187313+0000 mgr.y (mgr.14901) 2795 : cluster [DBG] pgmap v2802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:44.557 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:44 smithi130 bash[27579]: audit 2023-08-27T23:31:44.200257+0000 mon.a (mon.0) 2335 : audit [DBG] from='client.? 172.21.15.6:0/2218324486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:44.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:44 smithi006 bash[33918]: cluster 2023-08-27T23:31:43.187313+0000 mgr.y (mgr.14901) 2795 : cluster [DBG] pgmap v2802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:44.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:44 smithi006 bash[33918]: audit 2023-08-27T23:31:44.200257+0000 mon.a (mon.0) 2335 : audit [DBG] from='client.? 172.21.15.6:0/2218324486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:44.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:44 smithi006 bash[27292]: cluster 2023-08-27T23:31:43.187313+0000 mgr.y (mgr.14901) 2795 : cluster [DBG] pgmap v2802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:44.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:44 smithi006 bash[27292]: audit 2023-08-27T23:31:44.200257+0000 mon.a (mon.0) 2335 : audit [DBG] from='client.? 172.21.15.6:0/2218324486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:44.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:44 smithi130 bash[51302]: debug 2023-08-27T23:31:44.552+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:44.556317+0000) 2023-08-27T23:31:45.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:45 smithi130 bash[51302]: debug 2023-08-27T23:31:45.552+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:45.556680+0000) 2023-08-27T23:31:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:46 smithi130 bash[27579]: cluster 2023-08-27T23:31:45.188282+0000 mgr.y (mgr.14901) 2796 : cluster [DBG] pgmap v2803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:46 smithi130 bash[27579]: audit 2023-08-27T23:31:46.611872+0000 mon.a (mon.0) 2336 : audit [DBG] from='client.? 172.21.15.6:0/946617532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:46.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:46 smithi130 bash[51302]: debug 2023-08-27T23:31:46.552+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:46.557023+0000) 2023-08-27T23:31:47.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:46 smithi006 bash[27292]: cluster 2023-08-27T23:31:45.188282+0000 mgr.y (mgr.14901) 2796 : cluster [DBG] pgmap v2803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:47.449 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:46 smithi006 bash[27292]: audit 2023-08-27T23:31:46.611872+0000 mon.a (mon.0) 2336 : audit [DBG] from='client.? 172.21.15.6:0/946617532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:47.449 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:46 smithi006 bash[33918]: cluster 2023-08-27T23:31:45.188282+0000 mgr.y (mgr.14901) 2796 : cluster [DBG] pgmap v2803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:47.450 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:46 smithi006 bash[33918]: audit 2023-08-27T23:31:46.611872+0000 mon.a (mon.0) 2336 : audit [DBG] from='client.? 172.21.15.6:0/946617532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:47.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:47 smithi130 bash[51302]: debug 2023-08-27T23:31:47.556+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:47.557417+0000) 2023-08-27T23:31:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:48 smithi130 bash[27579]: cluster 2023-08-27T23:31:47.188984+0000 mgr.y (mgr.14901) 2797 : cluster [DBG] pgmap v2804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:48 smithi130 bash[51302]: debug 2023-08-27T23:31:48.556+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:48.557749+0000) 2023-08-27T23:31:49.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:48 smithi006 bash[33918]: cluster 2023-08-27T23:31:47.188984+0000 mgr.y (mgr.14901) 2797 : cluster [DBG] pgmap v2804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:49.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:48 smithi006 bash[27292]: cluster 2023-08-27T23:31:47.188984+0000 mgr.y (mgr.14901) 2797 : cluster [DBG] pgmap v2804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:49.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:49 smithi130 bash[27579]: audit 2023-08-27T23:31:49.028054+0000 mon.c (mon.2) 1088 : audit [DBG] from='client.? 172.21.15.6:0/3573046384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:49.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:49 smithi130 bash[51302]: debug 2023-08-27T23:31:49.552+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:49.558113+0000) 2023-08-27T23:31:50.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:49 smithi006 bash[27292]: audit 2023-08-27T23:31:49.028054+0000 mon.c (mon.2) 1088 : audit [DBG] from='client.? 172.21.15.6:0/3573046384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:50.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:49 smithi006 bash[33918]: audit 2023-08-27T23:31:49.028054+0000 mon.c (mon.2) 1088 : audit [DBG] from='client.? 172.21.15.6:0/3573046384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:50.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:31:50.492Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:50.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:31:50.495Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:50 smithi130 bash[27579]: cluster 2023-08-27T23:31:49.190038+0000 mgr.y (mgr.14901) 2798 : cluster [DBG] pgmap v2805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:50 smithi130 bash[51302]: debug 2023-08-27T23:31:50.556+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:50.558432+0000) 2023-08-27T23:31:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:50 smithi006 bash[33918]: cluster 2023-08-27T23:31:49.190038+0000 mgr.y (mgr.14901) 2798 : cluster [DBG] pgmap v2805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:50 smithi006 bash[27292]: cluster 2023-08-27T23:31:49.190038+0000 mgr.y (mgr.14901) 2798 : cluster [DBG] pgmap v2805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:31:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:31:50] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:31:51.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:51 smithi130 bash[27579]: audit 2023-08-27T23:31:51.450512+0000 mon.a (mon.0) 2337 : audit [DBG] from='client.? 172.21.15.6:0/814679757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:51.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:51 smithi130 bash[51302]: debug 2023-08-27T23:31:51.556+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:51.558754+0000) 2023-08-27T23:31:52.058 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:51 smithi006 bash[33918]: audit 2023-08-27T23:31:51.450512+0000 mon.a (mon.0) 2337 : audit [DBG] from='client.? 172.21.15.6:0/814679757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:52.059 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:51 smithi006 bash[27292]: audit 2023-08-27T23:31:51.450512+0000 mon.a (mon.0) 2337 : audit [DBG] from='client.? 172.21.15.6:0/814679757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:31:52.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:31:52.057Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:52 smithi130 bash[27579]: cluster 2023-08-27T23:31:51.190601+0000 mgr.y (mgr.14901) 2799 : cluster [DBG] pgmap v2806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:52.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:52 smithi130 bash[51302]: debug 2023-08-27T23:31:52.556+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:52.559081+0000) 2023-08-27T23:31:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:52 smithi006 bash[27292]: cluster 2023-08-27T23:31:51.190601+0000 mgr.y (mgr.14901) 2799 : cluster [DBG] pgmap v2806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:53.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:52 smithi006 bash[33918]: cluster 2023-08-27T23:31:51.190601+0000 mgr.y (mgr.14901) 2799 : cluster [DBG] pgmap v2806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:53.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:53 smithi130 bash[51302]: debug 2023-08-27T23:31:53.556+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:53.559385+0000) 2023-08-27T23:31:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:31:53.959Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:31:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:31:53.961Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:31:54.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:54 smithi130 bash[51302]: debug 2023-08-27T23:31:54.556+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:54.559734+0000) 2023-08-27T23:31:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:54 smithi130 bash[27579]: cluster 2023-08-27T23:31:53.191293+0000 mgr.y (mgr.14901) 2800 : cluster [DBG] pgmap v2807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:54 smithi130 bash[27579]: audit 2023-08-27T23:31:53.871852+0000 mon.c (mon.2) 1089 : audit [DBG] from='client.? 172.21.15.6:0/2725560166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:54 smithi006 bash[33918]: cluster 2023-08-27T23:31:53.191293+0000 mgr.y (mgr.14901) 2800 : cluster [DBG] pgmap v2807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:54 smithi006 bash[33918]: audit 2023-08-27T23:31:53.871852+0000 mon.c (mon.2) 1089 : audit [DBG] from='client.? 172.21.15.6:0/2725560166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:54 smithi006 bash[27292]: cluster 2023-08-27T23:31:53.191293+0000 mgr.y (mgr.14901) 2800 : cluster [DBG] pgmap v2807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:54 smithi006 bash[27292]: audit 2023-08-27T23:31:53.871852+0000 mon.c (mon.2) 1089 : audit [DBG] from='client.? 172.21.15.6:0/2725560166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:55.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:55 smithi130 bash[51302]: debug 2023-08-27T23:31:55.556+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:55.560077+0000) 2023-08-27T23:31:56.815 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:56 smithi130 bash[51302]: debug 2023-08-27T23:31:56.556+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:56.560398+0000) 2023-08-27T23:31:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:56 smithi130 bash[27579]: cluster 2023-08-27T23:31:55.192394+0000 mgr.y (mgr.14901) 2801 : cluster [DBG] pgmap v2808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:57.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:56 smithi130 bash[27579]: audit 2023-08-27T23:31:56.285630+0000 mon.c (mon.2) 1090 : audit [DBG] from='client.? 172.21.15.6:0/4252642476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:57.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:56 smithi006 bash[33918]: cluster 2023-08-27T23:31:55.192394+0000 mgr.y (mgr.14901) 2801 : cluster [DBG] pgmap v2808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:57.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:56 smithi006 bash[33918]: audit 2023-08-27T23:31:56.285630+0000 mon.c (mon.2) 1090 : audit [DBG] from='client.? 172.21.15.6:0/4252642476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:57.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:56 smithi006 bash[27292]: cluster 2023-08-27T23:31:55.192394+0000 mgr.y (mgr.14901) 2801 : cluster [DBG] pgmap v2808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:57.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:56 smithi006 bash[27292]: audit 2023-08-27T23:31:56.285630+0000 mon.c (mon.2) 1090 : audit [DBG] from='client.? 172.21.15.6:0/4252642476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:57.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:57 smithi130 bash[51302]: debug 2023-08-27T23:31:57.556+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:57.560744+0000) 2023-08-27T23:31:58.820 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:58 smithi130 bash[51302]: debug 2023-08-27T23:31:58.556+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:58.561117+0000) 2023-08-27T23:31:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:58 smithi130 bash[27579]: cluster 2023-08-27T23:31:57.193046+0000 mgr.y (mgr.14901) 2802 : cluster [DBG] pgmap v2809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:31:58 smithi130 bash[27579]: audit 2023-08-27T23:31:58.703027+0000 mon.c (mon.2) 1091 : audit [DBG] from='client.? 172.21.15.6:0/2348747232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:58 smithi006 bash[27292]: cluster 2023-08-27T23:31:57.193046+0000 mgr.y (mgr.14901) 2802 : cluster [DBG] pgmap v2809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:31:58 smithi006 bash[27292]: audit 2023-08-27T23:31:58.703027+0000 mon.c (mon.2) 1091 : audit [DBG] from='client.? 172.21.15.6:0/2348747232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:58 smithi006 bash[33918]: cluster 2023-08-27T23:31:57.193046+0000 mgr.y (mgr.14901) 2802 : cluster [DBG] pgmap v2809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:31:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:31:58 smithi006 bash[33918]: audit 2023-08-27T23:31:58.703027+0000 mon.c (mon.2) 1091 : audit [DBG] from='client.? 172.21.15.6:0/2348747232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:59.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:31:59 smithi130 bash[51302]: debug 2023-08-27T23:31:59.560+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:59.561456+0000) 2023-08-27T23:32:00.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:32:00.493Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:00.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:32:00.495Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:00.824 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:00 smithi130 bash[51302]: debug 2023-08-27T23:32:00.556+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:00.561802+0000) 2023-08-27T23:32:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:00 smithi130 bash[27579]: cluster 2023-08-27T23:31:59.194184+0000 mgr.y (mgr.14901) 2803 : cluster [DBG] pgmap v2810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:00 smithi006 bash[27292]: cluster 2023-08-27T23:31:59.194184+0000 mgr.y (mgr.14901) 2803 : cluster [DBG] pgmap v2810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:32:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:32:00] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T23:32:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:00 smithi006 bash[33918]: cluster 2023-08-27T23:31:59.194184+0000 mgr.y (mgr.14901) 2803 : cluster [DBG] pgmap v2810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:01.831 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:01 smithi130 bash[51302]: debug 2023-08-27T23:32:01.560+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:01.562129+0000) 2023-08-27T23:32:02.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:01 smithi130 bash[27579]: audit 2023-08-27T23:32:01.128153+0000 mon.c (mon.2) 1092 : audit [DBG] from='client.? 172.21.15.6:0/448937020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:02.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:01 smithi006 bash[33918]: audit 2023-08-27T23:32:01.128153+0000 mon.c (mon.2) 1092 : audit [DBG] from='client.? 172.21.15.6:0/448937020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:02.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:01 smithi006 bash[27292]: audit 2023-08-27T23:32:01.128153+0000 mon.c (mon.2) 1092 : audit [DBG] from='client.? 172.21.15.6:0/448937020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:02.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:32:02.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:02.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:32:02.059Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:02.834 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:02 smithi130 bash[51302]: debug 2023-08-27T23:32:02.556+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:02.562442+0000) 2023-08-27T23:32:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:02 smithi130 bash[27579]: cluster 2023-08-27T23:32:01.194862+0000 mgr.y (mgr.14901) 2804 : cluster [DBG] pgmap v2811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:02 smithi006 bash[33918]: cluster 2023-08-27T23:32:01.194862+0000 mgr.y (mgr.14901) 2804 : cluster [DBG] pgmap v2811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:02 smithi006 bash[27292]: cluster 2023-08-27T23:32:01.194862+0000 mgr.y (mgr.14901) 2804 : cluster [DBG] pgmap v2811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:03.842 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:03 smithi130 bash[51302]: debug 2023-08-27T23:32:03.556+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:03.562765+0000) 2023-08-27T23:32:04.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:03 smithi130 bash[27579]: cluster 2023-08-27T23:32:03.195504+0000 mgr.y (mgr.14901) 2805 : cluster [DBG] pgmap v2812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:04.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:03 smithi130 bash[27579]: audit 2023-08-27T23:32:03.544537+0000 mon.a (mon.0) 2338 : audit [DBG] from='client.? 172.21.15.6:0/2105392771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:04.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:03 smithi006 bash[33918]: cluster 2023-08-27T23:32:03.195504+0000 mgr.y (mgr.14901) 2805 : cluster [DBG] pgmap v2812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:04.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:03 smithi006 bash[33918]: audit 2023-08-27T23:32:03.544537+0000 mon.a (mon.0) 2338 : audit [DBG] from='client.? 172.21.15.6:0/2105392771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:32:03.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:32:03.961Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:04.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:03 smithi006 bash[27292]: cluster 2023-08-27T23:32:03.195504+0000 mgr.y (mgr.14901) 2805 : cluster [DBG] pgmap v2812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:04.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:03 smithi006 bash[27292]: audit 2023-08-27T23:32:03.544537+0000 mon.a (mon.0) 2338 : audit [DBG] from='client.? 172.21.15.6:0/2105392771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:04.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:04 smithi130 bash[51302]: debug 2023-08-27T23:32:04.560+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:04.563102+0000) 2023-08-27T23:32:05.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:05 smithi130 bash[51302]: debug 2023-08-27T23:32:05.560+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:05.563445+0000) 2023-08-27T23:32:06.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:06 smithi130 bash[51302]: debug 2023-08-27T23:32:06.560+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:06.563782+0000) 2023-08-27T23:32:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:06 smithi130 bash[27579]: cluster 2023-08-27T23:32:05.196789+0000 mgr.y (mgr.14901) 2806 : cluster [DBG] pgmap v2813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:06 smithi130 bash[27579]: audit 2023-08-27T23:32:05.962417+0000 mon.c (mon.2) 1093 : audit [DBG] from='client.? 172.21.15.6:0/2022247588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:07.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:06 smithi006 bash[27292]: cluster 2023-08-27T23:32:05.196789+0000 mgr.y (mgr.14901) 2806 : cluster [DBG] pgmap v2813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:07.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:06 smithi006 bash[27292]: audit 2023-08-27T23:32:05.962417+0000 mon.c (mon.2) 1093 : audit [DBG] from='client.? 172.21.15.6:0/2022247588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:07.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:06 smithi006 bash[33918]: cluster 2023-08-27T23:32:05.196789+0000 mgr.y (mgr.14901) 2806 : cluster [DBG] pgmap v2813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:07.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:06 smithi006 bash[33918]: audit 2023-08-27T23:32:05.962417+0000 mon.c (mon.2) 1093 : audit [DBG] from='client.? 172.21.15.6:0/2022247588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:07.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:07 smithi130 bash[51302]: debug 2023-08-27T23:32:07.560+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:07.564108+0000) 2023-08-27T23:32:08.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:08 smithi130 bash[51302]: debug 2023-08-27T23:32:08.560+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:08.564466+0000) 2023-08-27T23:32:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:08 smithi130 bash[27579]: cluster 2023-08-27T23:32:07.197461+0000 mgr.y (mgr.14901) 2807 : cluster [DBG] pgmap v2814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:08 smithi130 bash[27579]: audit 2023-08-27T23:32:08.381568+0000 mon.a (mon.0) 2339 : audit [DBG] from='client.? 172.21.15.6:0/548015300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:08 smithi006 bash[27292]: cluster 2023-08-27T23:32:07.197461+0000 mgr.y (mgr.14901) 2807 : cluster [DBG] pgmap v2814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:08 smithi006 bash[27292]: audit 2023-08-27T23:32:08.381568+0000 mon.a (mon.0) 2339 : audit [DBG] from='client.? 172.21.15.6:0/548015300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:08 smithi006 bash[33918]: cluster 2023-08-27T23:32:07.197461+0000 mgr.y (mgr.14901) 2807 : cluster [DBG] pgmap v2814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:08 smithi006 bash[33918]: audit 2023-08-27T23:32:08.381568+0000 mon.a (mon.0) 2339 : audit [DBG] from='client.? 172.21.15.6:0/548015300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:09.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:09 smithi130 bash[51302]: debug 2023-08-27T23:32:09.564+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:09.564785+0000) 2023-08-27T23:32:10.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:32:10.493Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:10.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:32:10.495Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:10 smithi130 bash[27579]: cluster 2023-08-27T23:32:09.198531+0000 mgr.y (mgr.14901) 2808 : cluster [DBG] pgmap v2815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:10 smithi130 bash[51302]: debug 2023-08-27T23:32:10.564+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:10.565171+0000) 2023-08-27T23:32:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:32:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:32:10] "GET /metrics HTTP/1.1" 200 193092 "" "Prometheus/2.33.4" 2023-08-27T23:32:11.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:10 smithi006 bash[27292]: cluster 2023-08-27T23:32:09.198531+0000 mgr.y (mgr.14901) 2808 : cluster [DBG] pgmap v2815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:10 smithi006 bash[33918]: cluster 2023-08-27T23:32:09.198531+0000 mgr.y (mgr.14901) 2808 : cluster [DBG] pgmap v2815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:11.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:11 smithi130 bash[27579]: audit 2023-08-27T23:32:10.800569+0000 mon.a (mon.0) 2340 : audit [DBG] from='client.? 172.21.15.6:0/112411176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:11.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:11 smithi130 bash[51302]: debug 2023-08-27T23:32:11.560+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:11.565529+0000) 2023-08-27T23:32:12.060 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:11 smithi006 bash[27292]: audit 2023-08-27T23:32:10.800569+0000 mon.a (mon.0) 2340 : audit [DBG] from='client.? 172.21.15.6:0/112411176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:12.060 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:11 smithi006 bash[33918]: audit 2023-08-27T23:32:10.800569+0000 mon.a (mon.0) 2340 : audit [DBG] from='client.? 172.21.15.6:0/112411176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:32:12.057Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:32:12.059Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:12 smithi130 bash[27579]: cluster 2023-08-27T23:32:11.199210+0000 mgr.y (mgr.14901) 2809 : cluster [DBG] pgmap v2816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:12 smithi130 bash[51302]: debug 2023-08-27T23:32:12.560+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:12.565837+0000) 2023-08-27T23:32:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:12 smithi006 bash[33918]: cluster 2023-08-27T23:32:11.199210+0000 mgr.y (mgr.14901) 2809 : cluster [DBG] pgmap v2816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:12 smithi006 bash[27292]: cluster 2023-08-27T23:32:11.199210+0000 mgr.y (mgr.14901) 2809 : cluster [DBG] pgmap v2816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:13.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:13 smithi130 bash[27579]: audit 2023-08-27T23:32:13.212543+0000 mon.a (mon.0) 2341 : audit [DBG] from='client.? 172.21.15.6:0/269010594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:13.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:13 smithi130 bash[51302]: debug 2023-08-27T23:32:13.560+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:13.566116+0000) 2023-08-27T23:32:14.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:13 smithi006 bash[27292]: audit 2023-08-27T23:32:13.212543+0000 mon.a (mon.0) 2341 : audit [DBG] from='client.? 172.21.15.6:0/269010594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:14.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:13 smithi006 bash[33918]: audit 2023-08-27T23:32:13.212543+0000 mon.a (mon.0) 2341 : audit [DBG] from='client.? 172.21.15.6:0/269010594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:14.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:32:13.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:14.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:32:13.963Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:14 smithi130 bash[27579]: cluster 2023-08-27T23:32:13.199820+0000 mgr.y (mgr.14901) 2810 : cluster [DBG] pgmap v2817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:14 smithi130 bash[51302]: debug 2023-08-27T23:32:14.564+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:14.566431+0000) 2023-08-27T23:32:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:14 smithi006 bash[27292]: cluster 2023-08-27T23:32:13.199820+0000 mgr.y (mgr.14901) 2810 : cluster [DBG] pgmap v2817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:14 smithi006 bash[33918]: cluster 2023-08-27T23:32:13.199820+0000 mgr.y (mgr.14901) 2810 : cluster [DBG] pgmap v2817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:15.825 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:15 smithi130 bash[51302]: debug 2023-08-27T23:32:15.560+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:15.566714+0000) 2023-08-27T23:32:16.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:15 smithi130 bash[27579]: audit 2023-08-27T23:32:15.626471+0000 mon.c (mon.2) 1094 : audit [DBG] from='client.? 172.21.15.6:0/4143642252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:16.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:15 smithi006 bash[33918]: audit 2023-08-27T23:32:15.626471+0000 mon.c (mon.2) 1094 : audit [DBG] from='client.? 172.21.15.6:0/4143642252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:16.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:15 smithi006 bash[27292]: audit 2023-08-27T23:32:15.626471+0000 mon.c (mon.2) 1094 : audit [DBG] from='client.? 172.21.15.6:0/4143642252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:16.829 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:16 smithi130 bash[51302]: debug 2023-08-27T23:32:16.564+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:16.566982+0000) 2023-08-27T23:32:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:16 smithi130 bash[27579]: cluster 2023-08-27T23:32:15.201027+0000 mgr.y (mgr.14901) 2811 : cluster [DBG] pgmap v2818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:16 smithi006 bash[33918]: cluster 2023-08-27T23:32:15.201027+0000 mgr.y (mgr.14901) 2811 : cluster [DBG] pgmap v2818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:17.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:16 smithi006 bash[27292]: cluster 2023-08-27T23:32:15.201027+0000 mgr.y (mgr.14901) 2811 : cluster [DBG] pgmap v2818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:17.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:17 smithi130 bash[51302]: debug 2023-08-27T23:32:17.564+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:17.567363+0000) 2023-08-27T23:32:18.833 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:18 smithi130 bash[51302]: debug 2023-08-27T23:32:18.564+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:18.567679+0000) 2023-08-27T23:32:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:18 smithi130 bash[27579]: cluster 2023-08-27T23:32:17.201785+0000 mgr.y (mgr.14901) 2812 : cluster [DBG] pgmap v2819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:18 smithi130 bash[27579]: audit 2023-08-27T23:32:18.039609+0000 mon.c (mon.2) 1095 : audit [DBG] from='client.? 172.21.15.6:0/1927929197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:18 smithi006 bash[33918]: cluster 2023-08-27T23:32:17.201785+0000 mgr.y (mgr.14901) 2812 : cluster [DBG] pgmap v2819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:18 smithi006 bash[33918]: audit 2023-08-27T23:32:18.039609+0000 mon.c (mon.2) 1095 : audit [DBG] from='client.? 172.21.15.6:0/1927929197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:18 smithi006 bash[27292]: cluster 2023-08-27T23:32:17.201785+0000 mgr.y (mgr.14901) 2812 : cluster [DBG] pgmap v2819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:18 smithi006 bash[27292]: audit 2023-08-27T23:32:18.039609+0000 mon.c (mon.2) 1095 : audit [DBG] from='client.? 172.21.15.6:0/1927929197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:19.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:19 smithi130 bash[51302]: debug 2023-08-27T23:32:19.564+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:19.567978+0000) 2023-08-27T23:32:20.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:32:20.494Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:20.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:32:20.496Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:20.838 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:20 smithi130 bash[51302]: debug 2023-08-27T23:32:20.564+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:20.568258+0000) 2023-08-27T23:32:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:20 smithi130 bash[27579]: cluster 2023-08-27T23:32:19.202815+0000 mgr.y (mgr.14901) 2813 : cluster [DBG] pgmap v2820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:20 smithi130 bash[27579]: audit 2023-08-27T23:32:20.464366+0000 mon.c (mon.2) 1096 : audit [DBG] from='client.? 172.21.15.6:0/2422320998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:20 smithi006 bash[27292]: cluster 2023-08-27T23:32:19.202815+0000 mgr.y (mgr.14901) 2813 : cluster [DBG] pgmap v2820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:20 smithi006 bash[27292]: audit 2023-08-27T23:32:20.464366+0000 mon.c (mon.2) 1096 : audit [DBG] from='client.? 172.21.15.6:0/2422320998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:32:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:32:20] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:32:21.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:20 smithi006 bash[33918]: cluster 2023-08-27T23:32:19.202815+0000 mgr.y (mgr.14901) 2813 : cluster [DBG] pgmap v2820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:21.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:20 smithi006 bash[33918]: audit 2023-08-27T23:32:20.464366+0000 mon.c (mon.2) 1096 : audit [DBG] from='client.? 172.21.15.6:0/2422320998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:21.848 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:21 smithi130 bash[51302]: debug 2023-08-27T23:32:21.564+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:21.568528+0000) 2023-08-27T23:32:22.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:21 smithi130 bash[27579]: cluster 2023-08-27T23:32:21.203521+0000 mgr.y (mgr.14901) 2814 : cluster [DBG] pgmap v2821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:22.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:21 smithi006 bash[33918]: cluster 2023-08-27T23:32:21.203521+0000 mgr.y (mgr.14901) 2814 : cluster [DBG] pgmap v2821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:22.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:21 smithi006 bash[27292]: cluster 2023-08-27T23:32:21.203521+0000 mgr.y (mgr.14901) 2814 : cluster [DBG] pgmap v2821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:22.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:32:22.058Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:22.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:32:22.060Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:22.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:22 smithi130 bash[51302]: debug 2023-08-27T23:32:22.564+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:22.568811+0000) 2023-08-27T23:32:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:22 smithi006 bash[33918]: audit 2023-08-27T23:32:22.878209+0000 mon.c (mon.2) 1097 : audit [DBG] from='client.? 172.21.15.6:0/3628506738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:22 smithi006 bash[27292]: audit 2023-08-27T23:32:22.878209+0000 mon.c (mon.2) 1097 : audit [DBG] from='client.? 172.21.15.6:0/3628506738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:23.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:22 smithi130 bash[27579]: audit 2023-08-27T23:32:22.878209+0000 mon.c (mon.2) 1097 : audit [DBG] from='client.? 172.21.15.6:0/3628506738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:23.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:23 smithi130 bash[51302]: debug 2023-08-27T23:32:23.568+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:23.569079+0000) 2023-08-27T23:32:24.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:23 smithi006 bash[27292]: cluster 2023-08-27T23:32:23.204212+0000 mgr.y (mgr.14901) 2815 : cluster [DBG] pgmap v2822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:24.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:23 smithi006 bash[33918]: cluster 2023-08-27T23:32:23.204212+0000 mgr.y (mgr.14901) 2815 : cluster [DBG] pgmap v2822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:32:23.961Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:32:23.963Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:23 smithi130 bash[27579]: cluster 2023-08-27T23:32:23.204212+0000 mgr.y (mgr.14901) 2815 : cluster [DBG] pgmap v2822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:24.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:24 smithi130 bash[51302]: debug 2023-08-27T23:32:24.568+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:24.569361+0000) 2023-08-27T23:32:25.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:25 smithi130 bash[27579]: audit 2023-08-27T23:32:25.300032+0000 mon.c (mon.2) 1098 : audit [DBG] from='client.? 172.21.15.6:0/3768057387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:25.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:25 smithi130 bash[51302]: debug 2023-08-27T23:32:25.568+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:25.569660+0000) 2023-08-27T23:32:26.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:25 smithi006 bash[33918]: audit 2023-08-27T23:32:25.300032+0000 mon.c (mon.2) 1098 : audit [DBG] from='client.? 172.21.15.6:0/3768057387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:25 smithi006 bash[27292]: audit 2023-08-27T23:32:25.300032+0000 mon.c (mon.2) 1098 : audit [DBG] from='client.? 172.21.15.6:0/3768057387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:26 smithi130 bash[27579]: cluster 2023-08-27T23:32:25.205298+0000 mgr.y (mgr.14901) 2816 : cluster [DBG] pgmap v2823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:26.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:26 smithi130 bash[51302]: debug 2023-08-27T23:32:26.568+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:26.569969+0000) 2023-08-27T23:32:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:26 smithi006 bash[27292]: cluster 2023-08-27T23:32:25.205298+0000 mgr.y (mgr.14901) 2816 : cluster [DBG] pgmap v2823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:26 smithi006 bash[33918]: cluster 2023-08-27T23:32:25.205298+0000 mgr.y (mgr.14901) 2816 : cluster [DBG] pgmap v2823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:27.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:27 smithi130 bash[27579]: audit 2023-08-27T23:32:27.709934+0000 mon.a (mon.0) 2342 : audit [DBG] from='client.? 172.21.15.6:0/2187894689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:27.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:27 smithi130 bash[51302]: debug 2023-08-27T23:32:27.564+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:27.570248+0000) 2023-08-27T23:32:28.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:27 smithi006 bash[27292]: audit 2023-08-27T23:32:27.709934+0000 mon.a (mon.0) 2342 : audit [DBG] from='client.? 172.21.15.6:0/2187894689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:28.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:27 smithi006 bash[33918]: audit 2023-08-27T23:32:27.709934+0000 mon.a (mon.0) 2342 : audit [DBG] from='client.? 172.21.15.6:0/2187894689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:28 smithi130 bash[27579]: cluster 2023-08-27T23:32:27.205942+0000 mgr.y (mgr.14901) 2817 : cluster [DBG] pgmap v2824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:28 smithi130 bash[51302]: debug 2023-08-27T23:32:28.568+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:28.570530+0000) 2023-08-27T23:32:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:28 smithi006 bash[27292]: cluster 2023-08-27T23:32:27.205942+0000 mgr.y (mgr.14901) 2817 : cluster [DBG] pgmap v2824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:28 smithi006 bash[33918]: cluster 2023-08-27T23:32:27.205942+0000 mgr.y (mgr.14901) 2817 : cluster [DBG] pgmap v2824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:29.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:29 smithi130 bash[51302]: debug 2023-08-27T23:32:29.568+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:29.570841+0000) 2023-08-27T23:32:30.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:32:30.494Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:30.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:32:30.496Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:30 smithi130 bash[27579]: cluster 2023-08-27T23:32:29.206946+0000 mgr.y (mgr.14901) 2818 : cluster [DBG] pgmap v2825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:30 smithi130 bash[27579]: audit 2023-08-27T23:32:30.117074+0000 mon.a (mon.0) 2343 : audit [DBG] from='client.? 172.21.15.6:0/1174095513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:30 smithi130 bash[51302]: debug 2023-08-27T23:32:30.568+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:30.571167+0000) 2023-08-27T23:32:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:32:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:32:30] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:32:31.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:30 smithi006 bash[27292]: cluster 2023-08-27T23:32:29.206946+0000 mgr.y (mgr.14901) 2818 : cluster [DBG] pgmap v2825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:31.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:30 smithi006 bash[27292]: audit 2023-08-27T23:32:30.117074+0000 mon.a (mon.0) 2343 : audit [DBG] from='client.? 172.21.15.6:0/1174095513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:31.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:30 smithi006 bash[33918]: cluster 2023-08-27T23:32:29.206946+0000 mgr.y (mgr.14901) 2818 : cluster [DBG] pgmap v2825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:31.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:30 smithi006 bash[33918]: audit 2023-08-27T23:32:30.117074+0000 mon.a (mon.0) 2343 : audit [DBG] from='client.? 172.21.15.6:0/1174095513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:31.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:31 smithi130 bash[51302]: debug 2023-08-27T23:32:31.568+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:31.571480+0000) 2023-08-27T23:32:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:32:32.059Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:32:32.061Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:32 smithi130 bash[27579]: cluster 2023-08-27T23:32:31.207604+0000 mgr.y (mgr.14901) 2819 : cluster [DBG] pgmap v2826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:32 smithi130 bash[27579]: audit 2023-08-27T23:32:32.536601+0000 mon.a (mon.0) 2344 : audit [DBG] from='client.? 172.21.15.6:0/3260072666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:32 smithi130 bash[51302]: debug 2023-08-27T23:32:32.568+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:32.571795+0000) 2023-08-27T23:32:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:32 smithi006 bash[33918]: cluster 2023-08-27T23:32:31.207604+0000 mgr.y (mgr.14901) 2819 : cluster [DBG] pgmap v2826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:32 smithi006 bash[33918]: audit 2023-08-27T23:32:32.536601+0000 mon.a (mon.0) 2344 : audit [DBG] from='client.? 172.21.15.6:0/3260072666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:32 smithi006 bash[27292]: cluster 2023-08-27T23:32:31.207604+0000 mgr.y (mgr.14901) 2819 : cluster [DBG] pgmap v2826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:32 smithi006 bash[27292]: audit 2023-08-27T23:32:32.536601+0000 mon.a (mon.0) 2344 : audit [DBG] from='client.? 172.21.15.6:0/3260072666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:33.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:33 smithi130 bash[51302]: debug 2023-08-27T23:32:33.568+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:33.572099+0000) 2023-08-27T23:32:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:32:33.961Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:32:33.963Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:34.824 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:34 smithi130 bash[51302]: debug 2023-08-27T23:32:34.572+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:34.572423+0000) 2023-08-27T23:32:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:34 smithi130 bash[27579]: cluster 2023-08-27T23:32:33.208242+0000 mgr.y (mgr.14901) 2820 : cluster [DBG] pgmap v2827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:34 smithi006 bash[27292]: cluster 2023-08-27T23:32:33.208242+0000 mgr.y (mgr.14901) 2820 : cluster [DBG] pgmap v2827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:34 smithi006 bash[33918]: cluster 2023-08-27T23:32:33.208242+0000 mgr.y (mgr.14901) 2820 : cluster [DBG] pgmap v2827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:35.827 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:35 smithi130 bash[51302]: debug 2023-08-27T23:32:35.572+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:35.572739+0000) 2023-08-27T23:32:36.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:35 smithi130 bash[27579]: audit 2023-08-27T23:32:34.949749+0000 mon.a (mon.0) 2345 : audit [DBG] from='client.? 172.21.15.6:0/1189996790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:36.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:35 smithi006 bash[27292]: audit 2023-08-27T23:32:34.949749+0000 mon.a (mon.0) 2345 : audit [DBG] from='client.? 172.21.15.6:0/1189996790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:36.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:35 smithi006 bash[33918]: audit 2023-08-27T23:32:34.949749+0000 mon.a (mon.0) 2345 : audit [DBG] from='client.? 172.21.15.6:0/1189996790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:36.837 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:36 smithi130 bash[51302]: debug 2023-08-27T23:32:36.567+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:36.573012+0000) 2023-08-27T23:32:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:36 smithi130 bash[27579]: cluster 2023-08-27T23:32:35.209461+0000 mgr.y (mgr.14901) 2821 : cluster [DBG] pgmap v2828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:37.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:36 smithi006 bash[27292]: cluster 2023-08-27T23:32:35.209461+0000 mgr.y (mgr.14901) 2821 : cluster [DBG] pgmap v2828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:37.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:36 smithi006 bash[33918]: cluster 2023-08-27T23:32:35.209461+0000 mgr.y (mgr.14901) 2821 : cluster [DBG] pgmap v2828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:37.846 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:37 smithi130 bash[51302]: debug 2023-08-27T23:32:37.571+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:37.573282+0000) 2023-08-27T23:32:38.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:37 smithi130 bash[27579]: audit 2023-08-27T23:32:37.363867+0000 mon.a (mon.0) 2346 : audit [DBG] from='client.? 172.21.15.6:0/1440278655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:38.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:37 smithi006 bash[33918]: audit 2023-08-27T23:32:37.363867+0000 mon.a (mon.0) 2346 : audit [DBG] from='client.? 172.21.15.6:0/1440278655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:38.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:37 smithi006 bash[27292]: audit 2023-08-27T23:32:37.363867+0000 mon.a (mon.0) 2346 : audit [DBG] from='client.? 172.21.15.6:0/1440278655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:38.850 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:38 smithi130 bash[51302]: debug 2023-08-27T23:32:38.567+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:38.573623+0000) 2023-08-27T23:32:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:38 smithi130 bash[27579]: cluster 2023-08-27T23:32:37.210191+0000 mgr.y (mgr.14901) 2822 : cluster [DBG] pgmap v2829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:39.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:38 smithi006 bash[33918]: cluster 2023-08-27T23:32:37.210191+0000 mgr.y (mgr.14901) 2822 : cluster [DBG] pgmap v2829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:39.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:38 smithi006 bash[27292]: cluster 2023-08-27T23:32:37.210191+0000 mgr.y (mgr.14901) 2822 : cluster [DBG] pgmap v2829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:39.857 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:39 smithi130 bash[51302]: debug 2023-08-27T23:32:39.571+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:39.573911+0000) 2023-08-27T23:32:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:39 smithi130 bash[27579]: audit 2023-08-27T23:32:38.930470+0000 mon.b (mon.1) 388 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:32:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:39 smithi130 bash[27579]: cluster 2023-08-27T23:32:39.211326+0000 mgr.y (mgr.14901) 2823 : cluster [DBG] pgmap v2830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:39 smithi130 bash[27579]: audit 2023-08-27T23:32:39.781246+0000 mon.a (mon.0) 2347 : audit [DBG] from='client.? 172.21.15.6:0/1368802504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:40.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:39 smithi006 bash[33918]: audit 2023-08-27T23:32:38.930470+0000 mon.b (mon.1) 388 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:32:40.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:39 smithi006 bash[33918]: cluster 2023-08-27T23:32:39.211326+0000 mgr.y (mgr.14901) 2823 : cluster [DBG] pgmap v2830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:40.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:39 smithi006 bash[33918]: audit 2023-08-27T23:32:39.781246+0000 mon.a (mon.0) 2347 : audit [DBG] from='client.? 172.21.15.6:0/1368802504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:40.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:39 smithi006 bash[27292]: audit 2023-08-27T23:32:38.930470+0000 mon.b (mon.1) 388 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:32:40.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:39 smithi006 bash[27292]: cluster 2023-08-27T23:32:39.211326+0000 mgr.y (mgr.14901) 2823 : cluster [DBG] pgmap v2830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:40.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:39 smithi006 bash[27292]: audit 2023-08-27T23:32:39.781246+0000 mon.a (mon.0) 2347 : audit [DBG] from='client.? 172.21.15.6:0/1368802504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:40.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:32:40.494Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:40.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:32:40.496Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:40.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:40 smithi130 bash[51302]: debug 2023-08-27T23:32:40.571+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:40.574192+0000) 2023-08-27T23:32:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:32:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:32:40] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:32:41.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:41 smithi130 bash[51302]: debug 2023-08-27T23:32:41.571+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:41.574465+0000) 2023-08-27T23:32:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:32:42.059Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:32:42.061Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:42.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:42 smithi006 bash[27292]: cluster 2023-08-27T23:32:41.211958+0000 mgr.y (mgr.14901) 2824 : cluster [DBG] pgmap v2831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:42.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:42 smithi006 bash[27292]: audit 2023-08-27T23:32:42.194075+0000 mon.a (mon.0) 2348 : audit [DBG] from='client.? 172.21.15.6:0/1562305798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:42.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:42 smithi006 bash[33918]: cluster 2023-08-27T23:32:41.211958+0000 mgr.y (mgr.14901) 2824 : cluster [DBG] pgmap v2831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:42.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:42 smithi006 bash[33918]: audit 2023-08-27T23:32:42.194075+0000 mon.a (mon.0) 2348 : audit [DBG] from='client.? 172.21.15.6:0/1562305798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:42.575 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:42 smithi130 bash[27579]: cluster 2023-08-27T23:32:41.211958+0000 mgr.y (mgr.14901) 2824 : cluster [DBG] pgmap v2831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:42.576 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:42 smithi130 bash[27579]: audit 2023-08-27T23:32:42.194075+0000 mon.a (mon.0) 2348 : audit [DBG] from='client.? 172.21.15.6:0/1562305798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:42.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:42 smithi130 bash[51302]: debug 2023-08-27T23:32:42.567+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:42.574680+0000) 2023-08-27T23:32:43.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:43 smithi130 bash[51302]: debug 2023-08-27T23:32:43.571+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:43.574989+0000) 2023-08-27T23:32:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:32:43.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:32:43.964Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:44.576 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:44 smithi130 bash[27579]: cluster 2023-08-27T23:32:43.212609+0000 mgr.y (mgr.14901) 2825 : cluster [DBG] pgmap v2832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:44.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:44 smithi006 bash[27292]: cluster 2023-08-27T23:32:43.212609+0000 mgr.y (mgr.14901) 2825 : cluster [DBG] pgmap v2832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:44.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:44 smithi006 bash[33918]: cluster 2023-08-27T23:32:43.212609+0000 mgr.y (mgr.14901) 2825 : cluster [DBG] pgmap v2832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:44 smithi130 bash[51302]: debug 2023-08-27T23:32:44.571+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:44.575213+0000) 2023-08-27T23:32:45.576 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:45 smithi130 bash[27579]: audit 2023-08-27T23:32:44.634626+0000 mon.c (mon.2) 1099 : audit [DBG] from='client.? 172.21.15.6:0/3520783162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:45.576 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:45 smithi130 bash[27579]: audit 2023-08-27T23:32:44.839781+0000 mon.a (mon.0) 2349 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:32:45.577 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:45 smithi130 bash[27579]: audit 2023-08-27T23:32:45.245544+0000 mon.a (mon.0) 2350 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:32:45.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:45 smithi006 bash[33918]: audit 2023-08-27T23:32:44.634626+0000 mon.c (mon.2) 1099 : audit [DBG] from='client.? 172.21.15.6:0/3520783162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:45.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:45 smithi006 bash[33918]: audit 2023-08-27T23:32:44.839781+0000 mon.a (mon.0) 2349 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:32:45.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:45 smithi006 bash[33918]: audit 2023-08-27T23:32:45.245544+0000 mon.a (mon.0) 2350 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:32:45.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:45 smithi006 bash[27292]: audit 2023-08-27T23:32:44.634626+0000 mon.c (mon.2) 1099 : audit [DBG] from='client.? 172.21.15.6:0/3520783162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:45.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:45 smithi006 bash[27292]: audit 2023-08-27T23:32:44.839781+0000 mon.a (mon.0) 2349 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:32:45.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:45 smithi006 bash[27292]: audit 2023-08-27T23:32:45.245544+0000 mon.a (mon.0) 2350 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:32:45.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:45 smithi130 bash[51302]: debug 2023-08-27T23:32:45.571+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:45.575419+0000) 2023-08-27T23:32:46.577 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:46 smithi130 bash[27579]: cluster 2023-08-27T23:32:45.213726+0000 mgr.y (mgr.14901) 2826 : cluster [DBG] pgmap v2833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:46.577 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:46 smithi130 bash[27579]: audit 2023-08-27T23:32:45.540447+0000 mon.b (mon.1) 389 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:32:46.577 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:46 smithi130 bash[27579]: audit 2023-08-27T23:32:45.542905+0000 mon.b (mon.1) 390 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:32:46.577 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:46 smithi130 bash[27579]: audit 2023-08-27T23:32:45.552437+0000 mon.a (mon.0) 2351 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:32:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:46 smithi006 bash[27292]: cluster 2023-08-27T23:32:45.213726+0000 mgr.y (mgr.14901) 2826 : cluster [DBG] pgmap v2833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:46 smithi006 bash[27292]: audit 2023-08-27T23:32:45.540447+0000 mon.b (mon.1) 389 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:32:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:46 smithi006 bash[27292]: audit 2023-08-27T23:32:45.542905+0000 mon.b (mon.1) 390 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:32:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:46 smithi006 bash[27292]: audit 2023-08-27T23:32:45.552437+0000 mon.a (mon.0) 2351 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:32:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:46 smithi006 bash[33918]: cluster 2023-08-27T23:32:45.213726+0000 mgr.y (mgr.14901) 2826 : cluster [DBG] pgmap v2833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:46 smithi006 bash[33918]: audit 2023-08-27T23:32:45.540447+0000 mon.b (mon.1) 389 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:32:46.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:46 smithi006 bash[33918]: audit 2023-08-27T23:32:45.542905+0000 mon.b (mon.1) 390 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:32:46.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:46 smithi006 bash[33918]: audit 2023-08-27T23:32:45.552437+0000 mon.a (mon.0) 2351 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:32:46.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:46 smithi130 bash[51302]: debug 2023-08-27T23:32:46.571+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:46.575700+0000) 2023-08-27T23:32:47.577 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:47 smithi130 bash[27579]: audit 2023-08-27T23:32:47.046476+0000 mon.a (mon.0) 2352 : audit [DBG] from='client.? 172.21.15.6:0/1762381507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:47.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:47 smithi006 bash[27292]: audit 2023-08-27T23:32:47.046476+0000 mon.a (mon.0) 2352 : audit [DBG] from='client.? 172.21.15.6:0/1762381507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:47.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:47 smithi006 bash[33918]: audit 2023-08-27T23:32:47.046476+0000 mon.a (mon.0) 2352 : audit [DBG] from='client.? 172.21.15.6:0/1762381507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:47.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:47 smithi130 bash[51302]: debug 2023-08-27T23:32:47.575+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:47.576025+0000) 2023-08-27T23:32:48.577 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:48 smithi130 bash[27579]: cluster 2023-08-27T23:32:47.214537+0000 mgr.y (mgr.14901) 2827 : cluster [DBG] pgmap v2834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:48 smithi006 bash[27292]: cluster 2023-08-27T23:32:47.214537+0000 mgr.y (mgr.14901) 2827 : cluster [DBG] pgmap v2834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:48 smithi006 bash[33918]: cluster 2023-08-27T23:32:47.214537+0000 mgr.y (mgr.14901) 2827 : cluster [DBG] pgmap v2834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:48.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:48 smithi130 bash[51302]: debug 2023-08-27T23:32:48.575+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:48.576314+0000) 2023-08-27T23:32:49.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:49 smithi130 bash[51302]: debug 2023-08-27T23:32:49.575+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:49.576592+0000) 2023-08-27T23:32:50.578 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:50 smithi130 bash[27579]: cluster 2023-08-27T23:32:49.215647+0000 mgr.y (mgr.14901) 2828 : cluster [DBG] pgmap v2835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:50.578 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:50 smithi130 bash[27579]: audit 2023-08-27T23:32:49.457581+0000 mon.a (mon.0) 2353 : audit [DBG] from='client.? 172.21.15.6:0/1678949936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:32:50.495Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:32:50.497Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:50 smithi006 bash[27292]: cluster 2023-08-27T23:32:49.215647+0000 mgr.y (mgr.14901) 2828 : cluster [DBG] pgmap v2835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:50 smithi006 bash[27292]: audit 2023-08-27T23:32:49.457581+0000 mon.a (mon.0) 2353 : audit [DBG] from='client.? 172.21.15.6:0/1678949936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:50 smithi006 bash[33918]: cluster 2023-08-27T23:32:49.215647+0000 mgr.y (mgr.14901) 2828 : cluster [DBG] pgmap v2835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:50 smithi006 bash[33918]: audit 2023-08-27T23:32:49.457581+0000 mon.a (mon.0) 2353 : audit [DBG] from='client.? 172.21.15.6:0/1678949936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:50 smithi130 bash[51302]: debug 2023-08-27T23:32:50.575+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:50.576890+0000) 2023-08-27T23:32:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:32:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:32:50] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:32:51.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:51 smithi130 bash[51302]: debug 2023-08-27T23:32:51.571+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:51.577144+0000) 2023-08-27T23:32:52.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:52 smithi006 bash[33918]: cluster 2023-08-27T23:32:51.216285+0000 mgr.y (mgr.14901) 2829 : cluster [DBG] pgmap v2836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:52.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:52 smithi006 bash[33918]: audit 2023-08-27T23:32:51.867757+0000 mon.c (mon.2) 1100 : audit [DBG] from='client.? 172.21.15.6:0/3580019891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:52.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:52 smithi006 bash[27292]: cluster 2023-08-27T23:32:51.216285+0000 mgr.y (mgr.14901) 2829 : cluster [DBG] pgmap v2836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:52.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:52 smithi006 bash[27292]: audit 2023-08-27T23:32:51.867757+0000 mon.c (mon.2) 1100 : audit [DBG] from='client.? 172.21.15.6:0/3580019891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:52.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:32:52.060Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:52.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:32:52.062Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:52.578 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:52 smithi130 bash[27579]: cluster 2023-08-27T23:32:51.216285+0000 mgr.y (mgr.14901) 2829 : cluster [DBG] pgmap v2836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:52.578 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:52 smithi130 bash[27579]: audit 2023-08-27T23:32:51.867757+0000 mon.c (mon.2) 1100 : audit [DBG] from='client.? 172.21.15.6:0/3580019891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:52.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:52 smithi130 bash[51302]: debug 2023-08-27T23:32:52.571+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:52.577432+0000) 2023-08-27T23:32:53.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:53 smithi130 bash[51302]: debug 2023-08-27T23:32:53.571+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:53.577730+0000) 2023-08-27T23:32:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:32:53.963Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:32:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:32:53.965Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:32:54.579 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:54 smithi130 bash[27579]: cluster 2023-08-27T23:32:53.216958+0000 mgr.y (mgr.14901) 2830 : cluster [DBG] pgmap v2837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:54.579 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:54 smithi130 bash[27579]: audit 2023-08-27T23:32:54.293803+0000 mon.a (mon.0) 2354 : audit [DBG] from='client.? 172.21.15.6:0/4272384040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:54 smithi006 bash[27292]: cluster 2023-08-27T23:32:53.216958+0000 mgr.y (mgr.14901) 2830 : cluster [DBG] pgmap v2837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:54 smithi006 bash[27292]: audit 2023-08-27T23:32:54.293803+0000 mon.a (mon.0) 2354 : audit [DBG] from='client.? 172.21.15.6:0/4272384040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:54.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:54 smithi006 bash[33918]: cluster 2023-08-27T23:32:53.216958+0000 mgr.y (mgr.14901) 2830 : cluster [DBG] pgmap v2837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:54.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:54 smithi006 bash[33918]: audit 2023-08-27T23:32:54.293803+0000 mon.a (mon.0) 2354 : audit [DBG] from='client.? 172.21.15.6:0/4272384040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:54.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:54 smithi130 bash[51302]: debug 2023-08-27T23:32:54.571+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:54.578029+0000) 2023-08-27T23:32:55.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:55 smithi130 bash[51302]: debug 2023-08-27T23:32:55.575+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:55.578335+0000) 2023-08-27T23:32:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:56 smithi130 bash[27579]: cluster 2023-08-27T23:32:55.218073+0000 mgr.y (mgr.14901) 2831 : cluster [DBG] pgmap v2838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:56 smithi130 bash[27579]: audit 2023-08-27T23:32:56.711508+0000 mon.c (mon.2) 1101 : audit [DBG] from='client.? 172.21.15.6:0/1378804443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:56.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:56 smithi130 bash[51302]: debug 2023-08-27T23:32:56.575+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:56.578605+0000) 2023-08-27T23:32:57.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:56 smithi006 bash[33918]: cluster 2023-08-27T23:32:55.218073+0000 mgr.y (mgr.14901) 2831 : cluster [DBG] pgmap v2838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:57.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:56 smithi006 bash[33918]: audit 2023-08-27T23:32:56.711508+0000 mon.c (mon.2) 1101 : audit [DBG] from='client.? 172.21.15.6:0/1378804443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:57.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:56 smithi006 bash[27292]: cluster 2023-08-27T23:32:55.218073+0000 mgr.y (mgr.14901) 2831 : cluster [DBG] pgmap v2838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:57.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:56 smithi006 bash[27292]: audit 2023-08-27T23:32:56.711508+0000 mon.c (mon.2) 1101 : audit [DBG] from='client.? 172.21.15.6:0/1378804443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:57.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:57 smithi130 bash[51302]: debug 2023-08-27T23:32:57.575+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:57.578886+0000) 2023-08-27T23:32:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:58 smithi130 bash[27579]: cluster 2023-08-27T23:32:57.218739+0000 mgr.y (mgr.14901) 2832 : cluster [DBG] pgmap v2839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:58.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:58 smithi130 bash[51302]: debug 2023-08-27T23:32:58.575+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:58.579114+0000) 2023-08-27T23:32:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:58 smithi006 bash[27292]: cluster 2023-08-27T23:32:57.218739+0000 mgr.y (mgr.14901) 2832 : cluster [DBG] pgmap v2839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:58 smithi006 bash[33918]: cluster 2023-08-27T23:32:57.218739+0000 mgr.y (mgr.14901) 2832 : cluster [DBG] pgmap v2839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:32:59.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:32:59 smithi130 bash[51302]: debug 2023-08-27T23:32:59.575+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:59.579380+0000) 2023-08-27T23:32:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:32:59 smithi130 bash[27579]: audit 2023-08-27T23:32:59.127743+0000 mon.c (mon.2) 1102 : audit [DBG] from='client.? 172.21.15.6:0/3662240332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:00.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:32:59 smithi006 bash[27292]: audit 2023-08-27T23:32:59.127743+0000 mon.c (mon.2) 1102 : audit [DBG] from='client.? 172.21.15.6:0/3662240332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:00.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:32:59 smithi006 bash[33918]: audit 2023-08-27T23:32:59.127743+0000 mon.c (mon.2) 1102 : audit [DBG] from='client.? 172.21.15.6:0/3662240332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:00.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:33:00.495Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:00.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:33:00.497Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:00 smithi130 bash[27579]: cluster 2023-08-27T23:32:59.219867+0000 mgr.y (mgr.14901) 2833 : cluster [DBG] pgmap v2840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:00.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:00 smithi130 bash[51302]: debug 2023-08-27T23:33:00.575+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:00.579692+0000) 2023-08-27T23:33:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:00 smithi006 bash[33918]: cluster 2023-08-27T23:32:59.219867+0000 mgr.y (mgr.14901) 2833 : cluster [DBG] pgmap v2840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:00 smithi006 bash[27292]: cluster 2023-08-27T23:32:59.219867+0000 mgr.y (mgr.14901) 2833 : cluster [DBG] pgmap v2840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:33:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:33:00] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:33:01.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:01 smithi130 bash[51302]: debug 2023-08-27T23:33:01.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:01.579978+0000) 2023-08-27T23:33:01.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:01 smithi130 bash[27579]: audit 2023-08-27T23:33:01.547115+0000 mon.c (mon.2) 1103 : audit [DBG] from='client.? 172.21.15.6:0/3436177169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:02.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:01 smithi006 bash[33918]: audit 2023-08-27T23:33:01.547115+0000 mon.c (mon.2) 1103 : audit [DBG] from='client.? 172.21.15.6:0/3436177169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:02.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:01 smithi006 bash[27292]: audit 2023-08-27T23:33:01.547115+0000 mon.c (mon.2) 1103 : audit [DBG] from='client.? 172.21.15.6:0/3436177169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:02.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:33:02.061Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:02.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:33:02.063Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:02 smithi130 bash[27579]: cluster 2023-08-27T23:33:01.220476+0000 mgr.y (mgr.14901) 2834 : cluster [DBG] pgmap v2841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:02.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:02 smithi130 bash[51302]: debug 2023-08-27T23:33:02.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:02.580284+0000) 2023-08-27T23:33:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:02 smithi006 bash[27292]: cluster 2023-08-27T23:33:01.220476+0000 mgr.y (mgr.14901) 2834 : cluster [DBG] pgmap v2841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:02 smithi006 bash[33918]: cluster 2023-08-27T23:33:01.220476+0000 mgr.y (mgr.14901) 2834 : cluster [DBG] pgmap v2841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:03.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:03 smithi130 bash[51302]: debug 2023-08-27T23:33:03.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:03.580608+0000) 2023-08-27T23:33:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:33:03.963Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:33:03.966Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:04.832 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:04 smithi130 bash[51302]: debug 2023-08-27T23:33:04.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:04.580887+0000) 2023-08-27T23:33:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:04 smithi130 bash[27579]: cluster 2023-08-27T23:33:03.221154+0000 mgr.y (mgr.14901) 2835 : cluster [DBG] pgmap v2842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:04 smithi130 bash[27579]: audit 2023-08-27T23:33:03.962310+0000 mon.a (mon.0) 2355 : audit [DBG] from='client.? 172.21.15.6:0/1650612767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:04 smithi006 bash[33918]: cluster 2023-08-27T23:33:03.221154+0000 mgr.y (mgr.14901) 2835 : cluster [DBG] pgmap v2842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:04 smithi006 bash[33918]: audit 2023-08-27T23:33:03.962310+0000 mon.a (mon.0) 2355 : audit [DBG] from='client.? 172.21.15.6:0/1650612767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:04 smithi006 bash[27292]: cluster 2023-08-27T23:33:03.221154+0000 mgr.y (mgr.14901) 2835 : cluster [DBG] pgmap v2842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:04 smithi006 bash[27292]: audit 2023-08-27T23:33:03.962310+0000 mon.a (mon.0) 2355 : audit [DBG] from='client.? 172.21.15.6:0/1650612767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:05.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:05 smithi130 bash[51302]: debug 2023-08-27T23:33:05.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:05.581207+0000) 2023-08-27T23:33:06.838 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:06 smithi130 bash[51302]: debug 2023-08-27T23:33:06.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:06.581540+0000) 2023-08-27T23:33:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:06 smithi130 bash[27579]: cluster 2023-08-27T23:33:05.222227+0000 mgr.y (mgr.14901) 2836 : cluster [DBG] pgmap v2843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:06 smithi130 bash[27579]: audit 2023-08-27T23:33:06.377750+0000 mon.a (mon.0) 2356 : audit [DBG] from='client.? 172.21.15.6:0/3866642273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:07.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:06 smithi006 bash[33918]: cluster 2023-08-27T23:33:05.222227+0000 mgr.y (mgr.14901) 2836 : cluster [DBG] pgmap v2843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:07.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:06 smithi006 bash[33918]: audit 2023-08-27T23:33:06.377750+0000 mon.a (mon.0) 2356 : audit [DBG] from='client.? 172.21.15.6:0/3866642273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:07.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:06 smithi006 bash[27292]: cluster 2023-08-27T23:33:05.222227+0000 mgr.y (mgr.14901) 2836 : cluster [DBG] pgmap v2843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:07.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:06 smithi006 bash[27292]: audit 2023-08-27T23:33:06.377750+0000 mon.a (mon.0) 2356 : audit [DBG] from='client.? 172.21.15.6:0/3866642273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:07.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:07 smithi130 bash[51302]: debug 2023-08-27T23:33:07.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:07.581850+0000) 2023-08-27T23:33:08.842 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:08 smithi130 bash[51302]: debug 2023-08-27T23:33:08.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:08.582150+0000) 2023-08-27T23:33:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:08 smithi130 bash[27579]: cluster 2023-08-27T23:33:07.222885+0000 mgr.y (mgr.14901) 2837 : cluster [DBG] pgmap v2844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:08 smithi130 bash[27579]: audit 2023-08-27T23:33:08.788964+0000 mon.a (mon.0) 2357 : audit [DBG] from='client.? 172.21.15.6:0/2719041851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:08 smithi006 bash[27292]: cluster 2023-08-27T23:33:07.222885+0000 mgr.y (mgr.14901) 2837 : cluster [DBG] pgmap v2844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:08 smithi006 bash[27292]: audit 2023-08-27T23:33:08.788964+0000 mon.a (mon.0) 2357 : audit [DBG] from='client.? 172.21.15.6:0/2719041851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:08 smithi006 bash[33918]: cluster 2023-08-27T23:33:07.222885+0000 mgr.y (mgr.14901) 2837 : cluster [DBG] pgmap v2844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:08 smithi006 bash[33918]: audit 2023-08-27T23:33:08.788964+0000 mon.a (mon.0) 2357 : audit [DBG] from='client.? 172.21.15.6:0/2719041851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:09.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:09 smithi130 bash[51302]: debug 2023-08-27T23:33:09.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:09.582613+0000) 2023-08-27T23:33:10.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:33:10.496Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:10.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:33:10.498Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:10.849 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:10 smithi130 bash[51302]: debug 2023-08-27T23:33:10.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:10.582915+0000) 2023-08-27T23:33:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:10 smithi130 bash[27579]: cluster 2023-08-27T23:33:09.224003+0000 mgr.y (mgr.14901) 2838 : cluster [DBG] pgmap v2845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:10 smithi006 bash[33918]: cluster 2023-08-27T23:33:09.224003+0000 mgr.y (mgr.14901) 2838 : cluster [DBG] pgmap v2845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:11.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:10 smithi006 bash[27292]: cluster 2023-08-27T23:33:09.224003+0000 mgr.y (mgr.14901) 2838 : cluster [DBG] pgmap v2845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:33:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:33:10] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:33:11.864 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:11 smithi130 bash[51302]: debug 2023-08-27T23:33:11.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:11.583120+0000) 2023-08-27T23:33:12.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:11 smithi130 bash[27579]: audit 2023-08-27T23:33:11.200347+0000 mon.c (mon.2) 1104 : audit [DBG] from='client.? 172.21.15.6:0/1698265782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:12.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:11 smithi130 bash[27579]: cluster 2023-08-27T23:33:11.224607+0000 mgr.y (mgr.14901) 2839 : cluster [DBG] pgmap v2846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:12.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:11 smithi006 bash[27292]: audit 2023-08-27T23:33:11.200347+0000 mon.c (mon.2) 1104 : audit [DBG] from='client.? 172.21.15.6:0/1698265782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:12.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:11 smithi006 bash[27292]: cluster 2023-08-27T23:33:11.224607+0000 mgr.y (mgr.14901) 2839 : cluster [DBG] pgmap v2846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:12.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:11 smithi006 bash[33918]: audit 2023-08-27T23:33:11.200347+0000 mon.c (mon.2) 1104 : audit [DBG] from='client.? 172.21.15.6:0/1698265782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:12.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:11 smithi006 bash[33918]: cluster 2023-08-27T23:33:11.224607+0000 mgr.y (mgr.14901) 2839 : cluster [DBG] pgmap v2846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:12.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:33:12.062Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:12.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:33:12.064Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:12 smithi130 bash[51302]: debug 2023-08-27T23:33:12.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:12.583352+0000) 2023-08-27T23:33:13.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:13 smithi130 bash[51302]: debug 2023-08-27T23:33:13.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:13.583625+0000) 2023-08-27T23:33:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:33:13.963Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:33:13.965Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:14.585 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:14 smithi130 bash[27579]: cluster 2023-08-27T23:33:13.225424+0000 mgr.y (mgr.14901) 2840 : cluster [DBG] pgmap v2847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:14.585 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:14 smithi130 bash[27579]: audit 2023-08-27T23:33:13.611384+0000 mon.c (mon.2) 1105 : audit [DBG] from='client.? 172.21.15.6:0/1055612469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:14 smithi006 bash[33918]: cluster 2023-08-27T23:33:13.225424+0000 mgr.y (mgr.14901) 2840 : cluster [DBG] pgmap v2847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:14 smithi006 bash[33918]: audit 2023-08-27T23:33:13.611384+0000 mon.c (mon.2) 1105 : audit [DBG] from='client.? 172.21.15.6:0/1055612469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:14.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:14 smithi006 bash[27292]: cluster 2023-08-27T23:33:13.225424+0000 mgr.y (mgr.14901) 2840 : cluster [DBG] pgmap v2847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:14.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:14 smithi006 bash[27292]: audit 2023-08-27T23:33:13.611384+0000 mon.c (mon.2) 1105 : audit [DBG] from='client.? 172.21.15.6:0/1055612469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:14 smithi130 bash[51302]: debug 2023-08-27T23:33:14.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:14.583939+0000) 2023-08-27T23:33:15.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:15 smithi130 bash[51302]: debug 2023-08-27T23:33:15.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:15.584247+0000) 2023-08-27T23:33:16.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:16 smithi130 bash[51302]: debug 2023-08-27T23:33:16.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:16.584541+0000) 2023-08-27T23:33:16.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:16 smithi130 bash[27579]: cluster 2023-08-27T23:33:15.226555+0000 mgr.y (mgr.14901) 2841 : cluster [DBG] pgmap v2848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:16.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:16 smithi130 bash[27579]: audit 2023-08-27T23:33:16.027811+0000 mon.c (mon.2) 1106 : audit [DBG] from='client.? 172.21.15.6:0/2267807582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:16 smithi006 bash[33918]: cluster 2023-08-27T23:33:15.226555+0000 mgr.y (mgr.14901) 2841 : cluster [DBG] pgmap v2848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:16 smithi006 bash[33918]: audit 2023-08-27T23:33:16.027811+0000 mon.c (mon.2) 1106 : audit [DBG] from='client.? 172.21.15.6:0/2267807582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:17.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:16 smithi006 bash[27292]: cluster 2023-08-27T23:33:15.226555+0000 mgr.y (mgr.14901) 2841 : cluster [DBG] pgmap v2848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:17.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:16 smithi006 bash[27292]: audit 2023-08-27T23:33:16.027811+0000 mon.c (mon.2) 1106 : audit [DBG] from='client.? 172.21.15.6:0/2267807582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:17.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:17 smithi130 bash[51302]: debug 2023-08-27T23:33:17.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:17.584837+0000) 2023-08-27T23:33:18.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:18 smithi130 bash[51302]: debug 2023-08-27T23:33:18.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:18.585118+0000) 2023-08-27T23:33:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:18 smithi130 bash[27579]: cluster 2023-08-27T23:33:17.227225+0000 mgr.y (mgr.14901) 2842 : cluster [DBG] pgmap v2849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:18 smithi130 bash[27579]: audit 2023-08-27T23:33:18.438745+0000 mon.c (mon.2) 1107 : audit [DBG] from='client.? 172.21.15.6:0/2323096595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:18 smithi006 bash[33918]: cluster 2023-08-27T23:33:17.227225+0000 mgr.y (mgr.14901) 2842 : cluster [DBG] pgmap v2849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:18 smithi006 bash[33918]: audit 2023-08-27T23:33:18.438745+0000 mon.c (mon.2) 1107 : audit [DBG] from='client.? 172.21.15.6:0/2323096595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:18 smithi006 bash[27292]: cluster 2023-08-27T23:33:17.227225+0000 mgr.y (mgr.14901) 2842 : cluster [DBG] pgmap v2849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:19.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:18 smithi006 bash[27292]: audit 2023-08-27T23:33:18.438745+0000 mon.c (mon.2) 1107 : audit [DBG] from='client.? 172.21.15.6:0/2323096595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:19 smithi130 bash[51302]: debug 2023-08-27T23:33:19.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:19.585392+0000) 2023-08-27T23:33:20.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:33:20.497Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:20.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:33:20.499Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:20 smithi130 bash[27579]: cluster 2023-08-27T23:33:19.228401+0000 mgr.y (mgr.14901) 2843 : cluster [DBG] pgmap v2850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:20 smithi130 bash[51302]: debug 2023-08-27T23:33:20.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:20.585672+0000) 2023-08-27T23:33:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:20 smithi006 bash[27292]: cluster 2023-08-27T23:33:19.228401+0000 mgr.y (mgr.14901) 2843 : cluster [DBG] pgmap v2850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:33:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:33:20] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:33:21.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:20 smithi006 bash[33918]: cluster 2023-08-27T23:33:19.228401+0000 mgr.y (mgr.14901) 2843 : cluster [DBG] pgmap v2850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:21.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:21 smithi130 bash[27579]: audit 2023-08-27T23:33:20.852957+0000 mon.a (mon.0) 2358 : audit [DBG] from='client.? 172.21.15.6:0/910637725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:21.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:21 smithi130 bash[51302]: debug 2023-08-27T23:33:21.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:21.585973+0000) 2023-08-27T23:33:22.067 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:21 smithi006 bash[27292]: audit 2023-08-27T23:33:20.852957+0000 mon.a (mon.0) 2358 : audit [DBG] from='client.? 172.21.15.6:0/910637725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:22.068 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:21 smithi006 bash[33918]: audit 2023-08-27T23:33:20.852957+0000 mon.a (mon.0) 2358 : audit [DBG] from='client.? 172.21.15.6:0/910637725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:22.068 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:33:22.063Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:33:22.065Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:22 smithi130 bash[51302]: debug 2023-08-27T23:33:22.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:22.586250+0000) 2023-08-27T23:33:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:22 smithi130 bash[27579]: cluster 2023-08-27T23:33:21.229140+0000 mgr.y (mgr.14901) 2844 : cluster [DBG] pgmap v2851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:22 smithi006 bash[27292]: cluster 2023-08-27T23:33:21.229140+0000 mgr.y (mgr.14901) 2844 : cluster [DBG] pgmap v2851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:22 smithi006 bash[33918]: cluster 2023-08-27T23:33:21.229140+0000 mgr.y (mgr.14901) 2844 : cluster [DBG] pgmap v2851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:23.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:23 smithi130 bash[27579]: audit 2023-08-27T23:33:23.269995+0000 mon.a (mon.0) 2359 : audit [DBG] from='client.? 172.21.15.6:0/260338679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:23.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:23 smithi130 bash[51302]: debug 2023-08-27T23:33:23.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:23.586524+0000) 2023-08-27T23:33:24.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:23 smithi006 bash[27292]: audit 2023-08-27T23:33:23.269995+0000 mon.a (mon.0) 2359 : audit [DBG] from='client.? 172.21.15.6:0/260338679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:24.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:23 smithi006 bash[33918]: audit 2023-08-27T23:33:23.269995+0000 mon.a (mon.0) 2359 : audit [DBG] from='client.? 172.21.15.6:0/260338679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:24.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:33:23.964Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:24.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:33:23.967Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:24 smithi130 bash[27579]: cluster 2023-08-27T23:33:23.229729+0000 mgr.y (mgr.14901) 2845 : cluster [DBG] pgmap v2852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:24 smithi130 bash[51302]: debug 2023-08-27T23:33:24.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:24.586795+0000) 2023-08-27T23:33:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:24 smithi006 bash[33918]: cluster 2023-08-27T23:33:23.229729+0000 mgr.y (mgr.14901) 2845 : cluster [DBG] pgmap v2852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:24 smithi006 bash[27292]: cluster 2023-08-27T23:33:23.229729+0000 mgr.y (mgr.14901) 2845 : cluster [DBG] pgmap v2852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:25.840 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:25 smithi130 bash[51302]: debug 2023-08-27T23:33:25.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:25.587075+0000) 2023-08-27T23:33:26.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:25 smithi130 bash[27579]: audit 2023-08-27T23:33:25.683353+0000 mon.a (mon.0) 2360 : audit [DBG] from='client.? 172.21.15.6:0/3939801171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:26.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:25 smithi006 bash[33918]: audit 2023-08-27T23:33:25.683353+0000 mon.a (mon.0) 2360 : audit [DBG] from='client.? 172.21.15.6:0/3939801171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:25 smithi006 bash[27292]: audit 2023-08-27T23:33:25.683353+0000 mon.a (mon.0) 2360 : audit [DBG] from='client.? 172.21.15.6:0/3939801171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:26.844 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:26 smithi130 bash[51302]: debug 2023-08-27T23:33:26.587+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:26.587380+0000) 2023-08-27T23:33:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:26 smithi130 bash[27579]: cluster 2023-08-27T23:33:25.230905+0000 mgr.y (mgr.14901) 2846 : cluster [DBG] pgmap v2853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:26 smithi006 bash[33918]: cluster 2023-08-27T23:33:25.230905+0000 mgr.y (mgr.14901) 2846 : cluster [DBG] pgmap v2853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:26 smithi006 bash[27292]: cluster 2023-08-27T23:33:25.230905+0000 mgr.y (mgr.14901) 2846 : cluster [DBG] pgmap v2853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:27.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:27 smithi130 bash[51302]: debug 2023-08-27T23:33:27.587+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:27.587657+0000) 2023-08-27T23:33:28.848 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:28 smithi130 bash[51302]: debug 2023-08-27T23:33:28.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:28.587951+0000) 2023-08-27T23:33:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:28 smithi130 bash[27579]: cluster 2023-08-27T23:33:27.231594+0000 mgr.y (mgr.14901) 2847 : cluster [DBG] pgmap v2854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:28 smithi130 bash[27579]: audit 2023-08-27T23:33:28.096135+0000 mon.a (mon.0) 2361 : audit [DBG] from='client.? 172.21.15.6:0/1749224517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:28 smithi006 bash[27292]: cluster 2023-08-27T23:33:27.231594+0000 mgr.y (mgr.14901) 2847 : cluster [DBG] pgmap v2854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:28 smithi006 bash[27292]: audit 2023-08-27T23:33:28.096135+0000 mon.a (mon.0) 2361 : audit [DBG] from='client.? 172.21.15.6:0/1749224517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:28 smithi006 bash[33918]: cluster 2023-08-27T23:33:27.231594+0000 mgr.y (mgr.14901) 2847 : cluster [DBG] pgmap v2854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:29.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:28 smithi006 bash[33918]: audit 2023-08-27T23:33:28.096135+0000 mon.a (mon.0) 2361 : audit [DBG] from='client.? 172.21.15.6:0/1749224517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:29.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:29 smithi130 bash[51302]: debug 2023-08-27T23:33:29.587+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:29.588257+0000) 2023-08-27T23:33:30.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:33:30.498Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:30.758 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:33:30.500Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:30 smithi130 bash[51302]: debug 2023-08-27T23:33:30.587+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:30.588542+0000) 2023-08-27T23:33:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:30 smithi130 bash[27579]: cluster 2023-08-27T23:33:29.232745+0000 mgr.y (mgr.14901) 2848 : cluster [DBG] pgmap v2855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:30 smithi130 bash[27579]: audit 2023-08-27T23:33:30.521380+0000 mon.a (mon.0) 2362 : audit [DBG] from='client.? 172.21.15.6:0/2235148662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:33:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:33:30] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:33:31.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:30 smithi006 bash[33918]: cluster 2023-08-27T23:33:29.232745+0000 mgr.y (mgr.14901) 2848 : cluster [DBG] pgmap v2855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:31.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:30 smithi006 bash[33918]: audit 2023-08-27T23:33:30.521380+0000 mon.a (mon.0) 2362 : audit [DBG] from='client.? 172.21.15.6:0/2235148662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:31.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:30 smithi006 bash[27292]: cluster 2023-08-27T23:33:29.232745+0000 mgr.y (mgr.14901) 2848 : cluster [DBG] pgmap v2855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:31.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:30 smithi006 bash[27292]: audit 2023-08-27T23:33:30.521380+0000 mon.a (mon.0) 2362 : audit [DBG] from='client.? 172.21.15.6:0/2235148662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:31.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:31 smithi130 bash[51302]: debug 2023-08-27T23:33:31.587+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:31.588812+0000) 2023-08-27T23:33:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:33:32.064Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:33:32.066Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:32 smithi130 bash[27579]: cluster 2023-08-27T23:33:31.233376+0000 mgr.y (mgr.14901) 2849 : cluster [DBG] pgmap v2856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:32 smithi130 bash[51302]: debug 2023-08-27T23:33:32.582+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:32.589120+0000) 2023-08-27T23:33:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:32 smithi006 bash[27292]: cluster 2023-08-27T23:33:31.233376+0000 mgr.y (mgr.14901) 2849 : cluster [DBG] pgmap v2856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:32 smithi006 bash[33918]: cluster 2023-08-27T23:33:31.233376+0000 mgr.y (mgr.14901) 2849 : cluster [DBG] pgmap v2856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:33 smithi130 bash[27579]: audit 2023-08-27T23:33:32.934299+0000 mon.c (mon.2) 1108 : audit [DBG] from='client.? 172.21.15.6:0/2331286643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:33.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:33 smithi130 bash[51302]: debug 2023-08-27T23:33:33.582+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:33.589405+0000) 2023-08-27T23:33:34.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:33 smithi006 bash[33918]: audit 2023-08-27T23:33:32.934299+0000 mon.c (mon.2) 1108 : audit [DBG] from='client.? 172.21.15.6:0/2331286643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:34.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:33 smithi006 bash[27292]: audit 2023-08-27T23:33:32.934299+0000 mon.c (mon.2) 1108 : audit [DBG] from='client.? 172.21.15.6:0/2331286643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:33:33.964Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:34.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:33:33.967Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:34 smithi130 bash[27579]: cluster 2023-08-27T23:33:33.234011+0000 mgr.y (mgr.14901) 2850 : cluster [DBG] pgmap v2857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:34 smithi130 bash[51302]: debug 2023-08-27T23:33:34.582+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:34.589682+0000) 2023-08-27T23:33:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:34 smithi006 bash[33918]: cluster 2023-08-27T23:33:33.234011+0000 mgr.y (mgr.14901) 2850 : cluster [DBG] pgmap v2857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:34 smithi006 bash[27292]: cluster 2023-08-27T23:33:33.234011+0000 mgr.y (mgr.14901) 2850 : cluster [DBG] pgmap v2857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:35.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:35 smithi130 bash[27579]: audit 2023-08-27T23:33:35.352039+0000 mon.c (mon.2) 1109 : audit [DBG] from='client.? 172.21.15.6:0/4033877265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:35.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:35 smithi130 bash[51302]: debug 2023-08-27T23:33:35.586+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:35.589991+0000) 2023-08-27T23:33:36.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:35 smithi006 bash[33918]: audit 2023-08-27T23:33:35.352039+0000 mon.c (mon.2) 1109 : audit [DBG] from='client.? 172.21.15.6:0/4033877265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:36.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:35 smithi006 bash[27292]: audit 2023-08-27T23:33:35.352039+0000 mon.c (mon.2) 1109 : audit [DBG] from='client.? 172.21.15.6:0/4033877265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:36 smithi130 bash[27579]: cluster 2023-08-27T23:33:35.234987+0000 mgr.y (mgr.14901) 2851 : cluster [DBG] pgmap v2858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:36 smithi130 bash[51302]: debug 2023-08-27T23:33:36.586+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:36.590274+0000) 2023-08-27T23:33:37.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:36 smithi006 bash[33918]: cluster 2023-08-27T23:33:35.234987+0000 mgr.y (mgr.14901) 2851 : cluster [DBG] pgmap v2858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:37.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:36 smithi006 bash[27292]: cluster 2023-08-27T23:33:35.234987+0000 mgr.y (mgr.14901) 2851 : cluster [DBG] pgmap v2858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:37.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:37 smithi130 bash[27579]: audit 2023-08-27T23:33:37.760101+0000 mon.c (mon.2) 1110 : audit [DBG] from='client.? 172.21.15.6:0/269059738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:37 smithi130 bash[51302]: debug 2023-08-27T23:33:37.586+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:37.590583+0000) 2023-08-27T23:33:38.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:37 smithi006 bash[33918]: audit 2023-08-27T23:33:37.760101+0000 mon.c (mon.2) 1110 : audit [DBG] from='client.? 172.21.15.6:0/269059738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:38.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:37 smithi006 bash[27292]: audit 2023-08-27T23:33:37.760101+0000 mon.c (mon.2) 1110 : audit [DBG] from='client.? 172.21.15.6:0/269059738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:38.843 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:38 smithi130 bash[51302]: debug 2023-08-27T23:33:38.586+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:38.590917+0000) 2023-08-27T23:33:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:38 smithi130 bash[27579]: cluster 2023-08-27T23:33:37.235656+0000 mgr.y (mgr.14901) 2852 : cluster [DBG] pgmap v2859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:39.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:38 smithi006 bash[27292]: cluster 2023-08-27T23:33:37.235656+0000 mgr.y (mgr.14901) 2852 : cluster [DBG] pgmap v2859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:39.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:38 smithi006 bash[33918]: cluster 2023-08-27T23:33:37.235656+0000 mgr.y (mgr.14901) 2852 : cluster [DBG] pgmap v2859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:39 smithi130 bash[51302]: debug 2023-08-27T23:33:39.590+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:39.591186+0000) 2023-08-27T23:33:40.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:33:40.498Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:40.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:33:40.500Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:40.849 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:40 smithi130 bash[51302]: debug 2023-08-27T23:33:40.590+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:40.591461+0000) 2023-08-27T23:33:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:40 smithi130 bash[27579]: cluster 2023-08-27T23:33:39.236742+0000 mgr.y (mgr.14901) 2853 : cluster [DBG] pgmap v2860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:40 smithi130 bash[27579]: audit 2023-08-27T23:33:40.175397+0000 mon.a (mon.0) 2363 : audit [DBG] from='client.? 172.21.15.6:0/990511518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:33:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:33:40] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:33:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:40 smithi006 bash[27292]: cluster 2023-08-27T23:33:39.236742+0000 mgr.y (mgr.14901) 2853 : cluster [DBG] pgmap v2860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:40 smithi006 bash[27292]: audit 2023-08-27T23:33:40.175397+0000 mon.a (mon.0) 2363 : audit [DBG] from='client.? 172.21.15.6:0/990511518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:41.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:40 smithi006 bash[33918]: cluster 2023-08-27T23:33:39.236742+0000 mgr.y (mgr.14901) 2853 : cluster [DBG] pgmap v2860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:41.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:40 smithi006 bash[33918]: audit 2023-08-27T23:33:40.175397+0000 mon.a (mon.0) 2363 : audit [DBG] from='client.? 172.21.15.6:0/990511518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:41.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:41 smithi130 bash[51302]: debug 2023-08-27T23:33:41.590+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:41.591748+0000) 2023-08-27T23:33:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:33:42.064Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:42.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:33:42.066Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:42.854 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:42 smithi130 bash[51302]: debug 2023-08-27T23:33:42.590+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:42.592064+0000) 2023-08-27T23:33:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:42 smithi130 bash[27579]: cluster 2023-08-27T23:33:41.237391+0000 mgr.y (mgr.14901) 2854 : cluster [DBG] pgmap v2861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:42 smithi130 bash[27579]: audit 2023-08-27T23:33:42.587750+0000 mon.a (mon.0) 2364 : audit [DBG] from='client.? 172.21.15.6:0/2124387067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:42 smithi006 bash[33918]: cluster 2023-08-27T23:33:41.237391+0000 mgr.y (mgr.14901) 2854 : cluster [DBG] pgmap v2861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:42 smithi006 bash[33918]: audit 2023-08-27T23:33:42.587750+0000 mon.a (mon.0) 2364 : audit [DBG] from='client.? 172.21.15.6:0/2124387067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:42 smithi006 bash[27292]: cluster 2023-08-27T23:33:41.237391+0000 mgr.y (mgr.14901) 2854 : cluster [DBG] pgmap v2861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:42 smithi006 bash[27292]: audit 2023-08-27T23:33:42.587750+0000 mon.a (mon.0) 2364 : audit [DBG] from='client.? 172.21.15.6:0/2124387067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:43.862 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:43 smithi130 bash[51302]: debug 2023-08-27T23:33:43.590+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:43.592350+0000) 2023-08-27T23:33:44.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:43 smithi130 bash[27579]: cluster 2023-08-27T23:33:43.238036+0000 mgr.y (mgr.14901) 2855 : cluster [DBG] pgmap v2862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:44.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:43 smithi006 bash[33918]: cluster 2023-08-27T23:33:43.238036+0000 mgr.y (mgr.14901) 2855 : cluster [DBG] pgmap v2862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:33:43.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:33:43.968Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:44.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:43 smithi006 bash[27292]: cluster 2023-08-27T23:33:43.238036+0000 mgr.y (mgr.14901) 2855 : cluster [DBG] pgmap v2862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:44 smithi130 bash[51302]: debug 2023-08-27T23:33:44.590+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:44.592647+0000) 2023-08-27T23:33:45.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:45 smithi130 bash[27579]: audit 2023-08-27T23:33:45.004108+0000 mon.a (mon.0) 2365 : audit [DBG] from='client.? 172.21.15.6:0/3738446282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:45.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:45 smithi130 bash[27579]: audit 2023-08-27T23:33:45.565028+0000 mon.b (mon.1) 391 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:33:45.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:45 smithi130 bash[51302]: debug 2023-08-27T23:33:45.590+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:45.592918+0000) 2023-08-27T23:33:46.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:45 smithi006 bash[33918]: audit 2023-08-27T23:33:45.004108+0000 mon.a (mon.0) 2365 : audit [DBG] from='client.? 172.21.15.6:0/3738446282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:46.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:45 smithi006 bash[33918]: audit 2023-08-27T23:33:45.565028+0000 mon.b (mon.1) 391 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:33:46.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:45 smithi006 bash[27292]: audit 2023-08-27T23:33:45.004108+0000 mon.a (mon.0) 2365 : audit [DBG] from='client.? 172.21.15.6:0/3738446282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:46.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:45 smithi006 bash[27292]: audit 2023-08-27T23:33:45.565028+0000 mon.b (mon.1) 391 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:33:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:46 smithi130 bash[27579]: cluster 2023-08-27T23:33:45.239169+0000 mgr.y (mgr.14901) 2856 : cluster [DBG] pgmap v2863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:46.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:46 smithi130 bash[51302]: debug 2023-08-27T23:33:46.590+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:46.593187+0000) 2023-08-27T23:33:47.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:46 smithi006 bash[27292]: cluster 2023-08-27T23:33:45.239169+0000 mgr.y (mgr.14901) 2856 : cluster [DBG] pgmap v2863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:47.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:46 smithi006 bash[33918]: cluster 2023-08-27T23:33:45.239169+0000 mgr.y (mgr.14901) 2856 : cluster [DBG] pgmap v2863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:47.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:47 smithi130 bash[27579]: audit 2023-08-27T23:33:47.417192+0000 mon.a (mon.0) 2366 : audit [DBG] from='client.? 172.21.15.6:0/3846978434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:47.870 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:47 smithi130 bash[51302]: debug 2023-08-27T23:33:47.590+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:47.593460+0000) 2023-08-27T23:33:48.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:47 smithi006 bash[33918]: audit 2023-08-27T23:33:47.417192+0000 mon.a (mon.0) 2366 : audit [DBG] from='client.? 172.21.15.6:0/3846978434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:48.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:47 smithi006 bash[27292]: audit 2023-08-27T23:33:47.417192+0000 mon.a (mon.0) 2366 : audit [DBG] from='client.? 172.21.15.6:0/3846978434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:48 smithi130 bash[27579]: cluster 2023-08-27T23:33:47.239899+0000 mgr.y (mgr.14901) 2857 : cluster [DBG] pgmap v2864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:48 smithi130 bash[51302]: debug 2023-08-27T23:33:48.590+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:48.593773+0000) 2023-08-27T23:33:49.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:48 smithi006 bash[27292]: cluster 2023-08-27T23:33:47.239899+0000 mgr.y (mgr.14901) 2857 : cluster [DBG] pgmap v2864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:49.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:48 smithi006 bash[33918]: cluster 2023-08-27T23:33:47.239899+0000 mgr.y (mgr.14901) 2857 : cluster [DBG] pgmap v2864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:49.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:49 smithi130 bash[51302]: debug 2023-08-27T23:33:49.590+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:49.594083+0000) 2023-08-27T23:33:50.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:33:50.499Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:50.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:33:50.501Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:50 smithi130 bash[27579]: cluster 2023-08-27T23:33:49.241163+0000 mgr.y (mgr.14901) 2858 : cluster [DBG] pgmap v2865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:50 smithi130 bash[27579]: audit 2023-08-27T23:33:49.836932+0000 mon.a (mon.0) 2367 : audit [DBG] from='client.? 172.21.15.6:0/1516602719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:50 smithi130 bash[51302]: debug 2023-08-27T23:33:50.590+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:50.594302+0000) 2023-08-27T23:33:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:50 smithi006 bash[27292]: cluster 2023-08-27T23:33:49.241163+0000 mgr.y (mgr.14901) 2858 : cluster [DBG] pgmap v2865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:50 smithi006 bash[27292]: audit 2023-08-27T23:33:49.836932+0000 mon.a (mon.0) 2367 : audit [DBG] from='client.? 172.21.15.6:0/1516602719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:33:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:33:50] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:33:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:50 smithi006 bash[33918]: cluster 2023-08-27T23:33:49.241163+0000 mgr.y (mgr.14901) 2858 : cluster [DBG] pgmap v2865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:51.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:50 smithi006 bash[33918]: audit 2023-08-27T23:33:49.836932+0000 mon.a (mon.0) 2367 : audit [DBG] from='client.? 172.21.15.6:0/1516602719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:51.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:51 smithi130 bash[51302]: debug 2023-08-27T23:33:51.590+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:51.594503+0000) 2023-08-27T23:33:52.417 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:33:52.065Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:52.418 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:33:52.067Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:52 smithi006 bash[33918]: cluster 2023-08-27T23:33:51.241774+0000 mgr.y (mgr.14901) 2859 : cluster [DBG] pgmap v2866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:52 smithi006 bash[33918]: audit 2023-08-27T23:33:51.414325+0000 mon.a (mon.0) 2368 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:33:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:52 smithi006 bash[33918]: audit 2023-08-27T23:33:51.907141+0000 mon.a (mon.0) 2369 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:33:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:52 smithi006 bash[33918]: audit 2023-08-27T23:33:52.204647+0000 mon.b (mon.1) 392 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:33:52.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:52 smithi006 bash[33918]: audit 2023-08-27T23:33:52.206326+0000 mon.b (mon.1) 393 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:33:52.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:52 smithi006 bash[33918]: audit 2023-08-27T23:33:52.212064+0000 mon.a (mon.0) 2370 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:33:52.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:52 smithi006 bash[33918]: audit 2023-08-27T23:33:52.261398+0000 mon.a (mon.0) 2371 : audit [DBG] from='client.? 172.21.15.6:0/826896982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:52.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:52 smithi006 bash[27292]: cluster 2023-08-27T23:33:51.241774+0000 mgr.y (mgr.14901) 2859 : cluster [DBG] pgmap v2866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:52.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:52 smithi006 bash[27292]: audit 2023-08-27T23:33:51.414325+0000 mon.a (mon.0) 2368 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:33:52.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:52 smithi006 bash[27292]: audit 2023-08-27T23:33:51.907141+0000 mon.a (mon.0) 2369 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:33:52.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:52 smithi006 bash[27292]: audit 2023-08-27T23:33:52.204647+0000 mon.b (mon.1) 392 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:33:52.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:52 smithi006 bash[27292]: audit 2023-08-27T23:33:52.206326+0000 mon.b (mon.1) 393 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:33:52.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:52 smithi006 bash[27292]: audit 2023-08-27T23:33:52.212064+0000 mon.a (mon.0) 2370 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:33:52.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:52 smithi006 bash[27292]: audit 2023-08-27T23:33:52.261398+0000 mon.a (mon.0) 2371 : audit [DBG] from='client.? 172.21.15.6:0/826896982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:52.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:52 smithi130 bash[51302]: debug 2023-08-27T23:33:52.590+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:52.594763+0000) 2023-08-27T23:33:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:52 smithi130 bash[27579]: cluster 2023-08-27T23:33:51.241774+0000 mgr.y (mgr.14901) 2859 : cluster [DBG] pgmap v2866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:52 smithi130 bash[27579]: audit 2023-08-27T23:33:51.414325+0000 mon.a (mon.0) 2368 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:33:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:52 smithi130 bash[27579]: audit 2023-08-27T23:33:51.907141+0000 mon.a (mon.0) 2369 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:33:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:52 smithi130 bash[27579]: audit 2023-08-27T23:33:52.204647+0000 mon.b (mon.1) 392 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:33:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:52 smithi130 bash[27579]: audit 2023-08-27T23:33:52.206326+0000 mon.b (mon.1) 393 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:33:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:52 smithi130 bash[27579]: audit 2023-08-27T23:33:52.212064+0000 mon.a (mon.0) 2370 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:33:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:52 smithi130 bash[27579]: audit 2023-08-27T23:33:52.261398+0000 mon.a (mon.0) 2371 : audit [DBG] from='client.? 172.21.15.6:0/826896982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:53.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:53 smithi130 bash[51302]: debug 2023-08-27T23:33:53.594+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:53.595061+0000) 2023-08-27T23:33:54.423 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:33:53.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:54.423 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:33:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:33:53.968Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:33:54.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:54 smithi006 bash[33918]: cluster 2023-08-27T23:33:53.242423+0000 mgr.y (mgr.14901) 2860 : cluster [DBG] pgmap v2867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:54 smithi006 bash[27292]: cluster 2023-08-27T23:33:53.242423+0000 mgr.y (mgr.14901) 2860 : cluster [DBG] pgmap v2867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:54 smithi130 bash[27579]: cluster 2023-08-27T23:33:53.242423+0000 mgr.y (mgr.14901) 2860 : cluster [DBG] pgmap v2867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:54.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:54 smithi130 bash[51302]: debug 2023-08-27T23:33:54.594+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:54.595327+0000) 2023-08-27T23:33:55.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:55 smithi006 bash[27292]: audit 2023-08-27T23:33:54.679439+0000 mon.a (mon.0) 2372 : audit [DBG] from='client.? 172.21.15.6:0/471030246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:55.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:55 smithi006 bash[33918]: audit 2023-08-27T23:33:54.679439+0000 mon.a (mon.0) 2372 : audit [DBG] from='client.? 172.21.15.6:0/471030246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:55.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:55 smithi130 bash[27579]: audit 2023-08-27T23:33:54.679439+0000 mon.a (mon.0) 2372 : audit [DBG] from='client.? 172.21.15.6:0/471030246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:55.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:55 smithi130 bash[51302]: debug 2023-08-27T23:33:55.594+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:55.595634+0000) 2023-08-27T23:33:56.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:56 smithi130 bash[27579]: cluster 2023-08-27T23:33:55.243520+0000 mgr.y (mgr.14901) 2861 : cluster [DBG] pgmap v2868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:56.870 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:56 smithi130 bash[51302]: debug 2023-08-27T23:33:56.594+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:56.595909+0000) 2023-08-27T23:33:57.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:56 smithi006 bash[27292]: cluster 2023-08-27T23:33:55.243520+0000 mgr.y (mgr.14901) 2861 : cluster [DBG] pgmap v2868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:57.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:56 smithi006 bash[33918]: cluster 2023-08-27T23:33:55.243520+0000 mgr.y (mgr.14901) 2861 : cluster [DBG] pgmap v2868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:57.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:57 smithi130 bash[51302]: debug 2023-08-27T23:33:57.594+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:57.596211+0000) 2023-08-27T23:33:57.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:57 smithi130 bash[27579]: audit 2023-08-27T23:33:57.092573+0000 mon.a (mon.0) 2373 : audit [DBG] from='client.? 172.21.15.6:0/705728868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:58.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:57 smithi006 bash[27292]: audit 2023-08-27T23:33:57.092573+0000 mon.a (mon.0) 2373 : audit [DBG] from='client.? 172.21.15.6:0/705728868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:58.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:57 smithi006 bash[33918]: audit 2023-08-27T23:33:57.092573+0000 mon.a (mon.0) 2373 : audit [DBG] from='client.? 172.21.15.6:0/705728868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:58 smithi130 bash[27579]: cluster 2023-08-27T23:33:57.244202+0000 mgr.y (mgr.14901) 2862 : cluster [DBG] pgmap v2869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:58.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:58 smithi130 bash[51302]: debug 2023-08-27T23:33:58.594+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:58.596503+0000) 2023-08-27T23:33:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:58 smithi006 bash[27292]: cluster 2023-08-27T23:33:57.244202+0000 mgr.y (mgr.14901) 2862 : cluster [DBG] pgmap v2869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:58 smithi006 bash[33918]: cluster 2023-08-27T23:33:57.244202+0000 mgr.y (mgr.14901) 2862 : cluster [DBG] pgmap v2869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:33:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:33:59 smithi130 bash[27579]: audit 2023-08-27T23:33:59.503922+0000 mon.a (mon.0) 2374 : audit [DBG] from='client.? 172.21.15.6:0/1962136301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:59.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:33:59 smithi130 bash[51302]: debug 2023-08-27T23:33:59.594+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:59.596797+0000) 2023-08-27T23:34:00.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:33:59 smithi006 bash[27292]: audit 2023-08-27T23:33:59.503922+0000 mon.a (mon.0) 2374 : audit [DBG] from='client.? 172.21.15.6:0/1962136301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:00.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:33:59 smithi006 bash[33918]: audit 2023-08-27T23:33:59.503922+0000 mon.a (mon.0) 2374 : audit [DBG] from='client.? 172.21.15.6:0/1962136301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:00.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:34:00.500Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:00.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:34:00.502Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:00 smithi130 bash[27579]: cluster 2023-08-27T23:33:59.245118+0000 mgr.y (mgr.14901) 2863 : cluster [DBG] pgmap v2870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:00.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:00 smithi130 bash[51302]: debug 2023-08-27T23:34:00.594+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:00.597068+0000) 2023-08-27T23:34:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:34:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:34:00] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:34:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:00 smithi006 bash[33918]: cluster 2023-08-27T23:33:59.245118+0000 mgr.y (mgr.14901) 2863 : cluster [DBG] pgmap v2870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:01.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:00 smithi006 bash[27292]: cluster 2023-08-27T23:33:59.245118+0000 mgr.y (mgr.14901) 2863 : cluster [DBG] pgmap v2870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:01.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:01 smithi130 bash[51302]: debug 2023-08-27T23:34:01.594+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:01.597324+0000) 2023-08-27T23:34:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:34:02.066Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:34:02.068Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:02.848 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:02 smithi130 bash[51302]: debug 2023-08-27T23:34:02.594+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:02.597640+0000) 2023-08-27T23:34:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:02 smithi130 bash[27579]: cluster 2023-08-27T23:34:01.245742+0000 mgr.y (mgr.14901) 2864 : cluster [DBG] pgmap v2871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:02 smithi130 bash[27579]: audit 2023-08-27T23:34:01.922879+0000 mon.c (mon.2) 1111 : audit [DBG] from='client.? 172.21.15.6:0/2894230220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:02 smithi006 bash[33918]: cluster 2023-08-27T23:34:01.245742+0000 mgr.y (mgr.14901) 2864 : cluster [DBG] pgmap v2871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:02 smithi006 bash[33918]: audit 2023-08-27T23:34:01.922879+0000 mon.c (mon.2) 1111 : audit [DBG] from='client.? 172.21.15.6:0/2894230220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:02 smithi006 bash[27292]: cluster 2023-08-27T23:34:01.245742+0000 mgr.y (mgr.14901) 2864 : cluster [DBG] pgmap v2871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:02 smithi006 bash[27292]: audit 2023-08-27T23:34:01.922879+0000 mon.c (mon.2) 1111 : audit [DBG] from='client.? 172.21.15.6:0/2894230220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:03.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:03 smithi130 bash[51302]: debug 2023-08-27T23:34:03.594+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:03.597923+0000) 2023-08-27T23:34:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:34:03.967Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:34:03.969Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:04.853 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:04 smithi130 bash[51302]: debug 2023-08-27T23:34:04.594+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:04.598198+0000) 2023-08-27T23:34:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:04 smithi130 bash[27579]: cluster 2023-08-27T23:34:03.246407+0000 mgr.y (mgr.14901) 2865 : cluster [DBG] pgmap v2872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:04 smithi130 bash[27579]: audit 2023-08-27T23:34:04.341175+0000 mon.c (mon.2) 1112 : audit [DBG] from='client.? 172.21.15.6:0/1399242496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:04 smithi006 bash[33918]: cluster 2023-08-27T23:34:03.246407+0000 mgr.y (mgr.14901) 2865 : cluster [DBG] pgmap v2872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:04 smithi006 bash[33918]: audit 2023-08-27T23:34:04.341175+0000 mon.c (mon.2) 1112 : audit [DBG] from='client.? 172.21.15.6:0/1399242496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:04 smithi006 bash[27292]: cluster 2023-08-27T23:34:03.246407+0000 mgr.y (mgr.14901) 2865 : cluster [DBG] pgmap v2872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:04 smithi006 bash[27292]: audit 2023-08-27T23:34:04.341175+0000 mon.c (mon.2) 1112 : audit [DBG] from='client.? 172.21.15.6:0/1399242496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:05.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:05 smithi130 bash[51302]: debug 2023-08-27T23:34:05.594+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:05.598487+0000) 2023-08-27T23:34:06.858 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:06 smithi130 bash[51302]: debug 2023-08-27T23:34:06.594+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:06.598750+0000) 2023-08-27T23:34:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:06 smithi130 bash[27579]: cluster 2023-08-27T23:34:05.247537+0000 mgr.y (mgr.14901) 2866 : cluster [DBG] pgmap v2873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:06 smithi130 bash[27579]: audit 2023-08-27T23:34:06.761000+0000 mon.a (mon.0) 2375 : audit [DBG] from='client.? 172.21.15.6:0/252100524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:07.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:06 smithi006 bash[33918]: cluster 2023-08-27T23:34:05.247537+0000 mgr.y (mgr.14901) 2866 : cluster [DBG] pgmap v2873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:07.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:06 smithi006 bash[33918]: audit 2023-08-27T23:34:06.761000+0000 mon.a (mon.0) 2375 : audit [DBG] from='client.? 172.21.15.6:0/252100524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:07.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:06 smithi006 bash[27292]: cluster 2023-08-27T23:34:05.247537+0000 mgr.y (mgr.14901) 2866 : cluster [DBG] pgmap v2873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:07.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:06 smithi006 bash[27292]: audit 2023-08-27T23:34:06.761000+0000 mon.a (mon.0) 2375 : audit [DBG] from='client.? 172.21.15.6:0/252100524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:07.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:07 smithi130 bash[51302]: debug 2023-08-27T23:34:07.598+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:07.599039+0000) 2023-08-27T23:34:08.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:07 smithi130 bash[27579]: cluster 2023-08-27T23:34:07.248342+0000 mgr.y (mgr.14901) 2867 : cluster [DBG] pgmap v2874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:08.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:07 smithi006 bash[33918]: cluster 2023-08-27T23:34:07.248342+0000 mgr.y (mgr.14901) 2867 : cluster [DBG] pgmap v2874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:08.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:07 smithi006 bash[27292]: cluster 2023-08-27T23:34:07.248342+0000 mgr.y (mgr.14901) 2867 : cluster [DBG] pgmap v2874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:08.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:08 smithi130 bash[51302]: debug 2023-08-27T23:34:08.598+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:08.599279+0000) 2023-08-27T23:34:09.600 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:09 smithi130 bash[27579]: audit 2023-08-27T23:34:09.185197+0000 mon.c (mon.2) 1113 : audit [DBG] from='client.? 172.21.15.6:0/3750018168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:09.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:09 smithi006 bash[33918]: audit 2023-08-27T23:34:09.185197+0000 mon.c (mon.2) 1113 : audit [DBG] from='client.? 172.21.15.6:0/3750018168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:09 smithi006 bash[27292]: audit 2023-08-27T23:34:09.185197+0000 mon.c (mon.2) 1113 : audit [DBG] from='client.? 172.21.15.6:0/3750018168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:09.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:09 smithi130 bash[51302]: debug 2023-08-27T23:34:09.598+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:09.599509+0000) 2023-08-27T23:34:10.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:34:10.500Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:10.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:34:10.502Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:10 smithi130 bash[27579]: cluster 2023-08-27T23:34:09.249254+0000 mgr.y (mgr.14901) 2868 : cluster [DBG] pgmap v2875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:10 smithi130 bash[51302]: debug 2023-08-27T23:34:10.598+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:10.599794+0000) 2023-08-27T23:34:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:34:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:34:10] "GET /metrics HTTP/1.1" 200 193090 "" "Prometheus/2.33.4" 2023-08-27T23:34:11.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:10 smithi006 bash[27292]: cluster 2023-08-27T23:34:09.249254+0000 mgr.y (mgr.14901) 2868 : cluster [DBG] pgmap v2875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:10 smithi006 bash[33918]: cluster 2023-08-27T23:34:09.249254+0000 mgr.y (mgr.14901) 2868 : cluster [DBG] pgmap v2875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:11.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:11 smithi130 bash[51302]: debug 2023-08-27T23:34:11.598+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:11.600065+0000) 2023-08-27T23:34:11.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:11 smithi130 bash[27579]: audit 2023-08-27T23:34:11.599486+0000 mon.a (mon.0) 2376 : audit [DBG] from='client.? 172.21.15.6:0/1477647849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:12.071 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:11 smithi006 bash[33918]: audit 2023-08-27T23:34:11.599486+0000 mon.a (mon.0) 2376 : audit [DBG] from='client.? 172.21.15.6:0/1477647849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:12.072 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:34:12.067Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:12.072 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:11 smithi006 bash[27292]: audit 2023-08-27T23:34:11.599486+0000 mon.a (mon.0) 2376 : audit [DBG] from='client.? 172.21.15.6:0/1477647849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:12 smithi130 bash[27579]: cluster 2023-08-27T23:34:11.249838+0000 mgr.y (mgr.14901) 2869 : cluster [DBG] pgmap v2876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:12 smithi130 bash[51302]: debug 2023-08-27T23:34:12.598+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:12.600353+0000) 2023-08-27T23:34:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:12 smithi006 bash[27292]: cluster 2023-08-27T23:34:11.249838+0000 mgr.y (mgr.14901) 2869 : cluster [DBG] pgmap v2876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:13.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:34:12.069Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:12 smithi006 bash[33918]: cluster 2023-08-27T23:34:11.249838+0000 mgr.y (mgr.14901) 2869 : cluster [DBG] pgmap v2876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:13.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:13 smithi130 bash[51302]: debug 2023-08-27T23:34:13.598+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:13.600632+0000) 2023-08-27T23:34:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:34:13.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:34:13.970Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:14.870 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:14 smithi130 bash[51302]: debug 2023-08-27T23:34:14.598+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:14.600920+0000) 2023-08-27T23:34:14.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:14 smithi130 bash[27579]: cluster 2023-08-27T23:34:13.250565+0000 mgr.y (mgr.14901) 2870 : cluster [DBG] pgmap v2877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:14.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:14 smithi130 bash[27579]: audit 2023-08-27T23:34:14.009866+0000 mon.a (mon.0) 2377 : audit [DBG] from='client.? 172.21.15.6:0/2902696460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:14 smithi006 bash[27292]: cluster 2023-08-27T23:34:13.250565+0000 mgr.y (mgr.14901) 2870 : cluster [DBG] pgmap v2877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:14 smithi006 bash[27292]: audit 2023-08-27T23:34:14.009866+0000 mon.a (mon.0) 2377 : audit [DBG] from='client.? 172.21.15.6:0/2902696460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:14 smithi006 bash[33918]: cluster 2023-08-27T23:34:13.250565+0000 mgr.y (mgr.14901) 2870 : cluster [DBG] pgmap v2877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:14 smithi006 bash[33918]: audit 2023-08-27T23:34:14.009866+0000 mon.a (mon.0) 2377 : audit [DBG] from='client.? 172.21.15.6:0/2902696460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:15.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:15 smithi130 bash[51302]: debug 2023-08-27T23:34:15.598+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:15.601226+0000) 2023-08-27T23:34:16.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:16 smithi130 bash[27579]: cluster 2023-08-27T23:34:15.251751+0000 mgr.y (mgr.14901) 2871 : cluster [DBG] pgmap v2878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:16 smithi130 bash[27579]: audit 2023-08-27T23:34:16.420225+0000 mon.a (mon.0) 2378 : audit [DBG] from='client.? 172.21.15.6:0/1144003829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:16.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:16 smithi130 bash[51302]: debug 2023-08-27T23:34:16.598+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:16.601492+0000) 2023-08-27T23:34:17.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:16 smithi006 bash[27292]: cluster 2023-08-27T23:34:15.251751+0000 mgr.y (mgr.14901) 2871 : cluster [DBG] pgmap v2878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:17.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:16 smithi006 bash[27292]: audit 2023-08-27T23:34:16.420225+0000 mon.a (mon.0) 2378 : audit [DBG] from='client.? 172.21.15.6:0/1144003829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:16 smithi006 bash[33918]: cluster 2023-08-27T23:34:15.251751+0000 mgr.y (mgr.14901) 2871 : cluster [DBG] pgmap v2878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:16 smithi006 bash[33918]: audit 2023-08-27T23:34:16.420225+0000 mon.a (mon.0) 2378 : audit [DBG] from='client.? 172.21.15.6:0/1144003829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:17.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:17 smithi130 bash[51302]: debug 2023-08-27T23:34:17.598+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:17.601817+0000) 2023-08-27T23:34:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:18 smithi130 bash[27579]: cluster 2023-08-27T23:34:17.252420+0000 mgr.y (mgr.14901) 2872 : cluster [DBG] pgmap v2879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:18 smithi130 bash[51302]: debug 2023-08-27T23:34:18.598+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:18.602124+0000) 2023-08-27T23:34:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:18 smithi006 bash[27292]: cluster 2023-08-27T23:34:17.252420+0000 mgr.y (mgr.14901) 2872 : cluster [DBG] pgmap v2879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:18 smithi006 bash[33918]: cluster 2023-08-27T23:34:17.252420+0000 mgr.y (mgr.14901) 2872 : cluster [DBG] pgmap v2879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:19.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:19 smithi130 bash[27579]: audit 2023-08-27T23:34:18.841227+0000 mon.a (mon.0) 2379 : audit [DBG] from='client.? 172.21.15.6:0/3296952234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:19 smithi130 bash[51302]: debug 2023-08-27T23:34:19.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:19.602420+0000) 2023-08-27T23:34:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:19 smithi006 bash[27292]: audit 2023-08-27T23:34:18.841227+0000 mon.a (mon.0) 2379 : audit [DBG] from='client.? 172.21.15.6:0/3296952234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:20.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:19 smithi006 bash[33918]: audit 2023-08-27T23:34:18.841227+0000 mon.a (mon.0) 2379 : audit [DBG] from='client.? 172.21.15.6:0/3296952234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:20.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:34:20.501Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:20.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:34:20.503Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:20.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:20 smithi130 bash[51302]: debug 2023-08-27T23:34:20.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:20.602699+0000) 2023-08-27T23:34:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:20 smithi130 bash[27579]: cluster 2023-08-27T23:34:19.253534+0000 mgr.y (mgr.14901) 2873 : cluster [DBG] pgmap v2880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:34:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:34:20] "GET /metrics HTTP/1.1" 200 193084 "" "Prometheus/2.33.4" 2023-08-27T23:34:21.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:20 smithi006 bash[33918]: cluster 2023-08-27T23:34:19.253534+0000 mgr.y (mgr.14901) 2873 : cluster [DBG] pgmap v2880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:21.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:20 smithi006 bash[27292]: cluster 2023-08-27T23:34:19.253534+0000 mgr.y (mgr.14901) 2873 : cluster [DBG] pgmap v2880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:21.858 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:21 smithi130 bash[51302]: debug 2023-08-27T23:34:21.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:21.602996+0000) 2023-08-27T23:34:22.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:21 smithi130 bash[27579]: audit 2023-08-27T23:34:21.259546+0000 mon.a (mon.0) 2380 : audit [DBG] from='client.? 172.21.15.6:0/624763748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:22.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:21 smithi006 bash[33918]: audit 2023-08-27T23:34:21.259546+0000 mon.a (mon.0) 2380 : audit [DBG] from='client.? 172.21.15.6:0/624763748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:22.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:21 smithi006 bash[27292]: audit 2023-08-27T23:34:21.259546+0000 mon.a (mon.0) 2380 : audit [DBG] from='client.? 172.21.15.6:0/624763748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:22.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:34:22.068Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:22.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:34:22.070Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:22.861 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:22 smithi130 bash[51302]: debug 2023-08-27T23:34:22.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:22.603220+0000) 2023-08-27T23:34:23.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:22 smithi130 bash[27579]: cluster 2023-08-27T23:34:21.254161+0000 mgr.y (mgr.14901) 2874 : cluster [DBG] pgmap v2881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:22 smithi006 bash[33918]: cluster 2023-08-27T23:34:21.254161+0000 mgr.y (mgr.14901) 2874 : cluster [DBG] pgmap v2881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:22 smithi006 bash[27292]: cluster 2023-08-27T23:34:21.254161+0000 mgr.y (mgr.14901) 2874 : cluster [DBG] pgmap v2881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:23.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:23 smithi130 bash[51302]: debug 2023-08-27T23:34:23.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:23.603453+0000) 2023-08-27T23:34:24.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:23 smithi006 bash[27292]: cluster 2023-08-27T23:34:23.254813+0000 mgr.y (mgr.14901) 2875 : cluster [DBG] pgmap v2882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:24.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:23 smithi006 bash[27292]: audit 2023-08-27T23:34:23.670920+0000 mon.c (mon.2) 1114 : audit [DBG] from='client.? 172.21.15.6:0/3319078732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:24.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:23 smithi006 bash[33918]: cluster 2023-08-27T23:34:23.254813+0000 mgr.y (mgr.14901) 2875 : cluster [DBG] pgmap v2882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:24.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:23 smithi006 bash[33918]: audit 2023-08-27T23:34:23.670920+0000 mon.c (mon.2) 1114 : audit [DBG] from='client.? 172.21.15.6:0/3319078732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:24.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:34:23.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:24.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:34:23.971Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:24.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:23 smithi130 bash[27579]: cluster 2023-08-27T23:34:23.254813+0000 mgr.y (mgr.14901) 2875 : cluster [DBG] pgmap v2882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:23 smithi130 bash[27579]: audit 2023-08-27T23:34:23.670920+0000 mon.c (mon.2) 1114 : audit [DBG] from='client.? 172.21.15.6:0/3319078732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:24.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:24 smithi130 bash[51302]: debug 2023-08-27T23:34:24.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:24.603731+0000) 2023-08-27T23:34:25.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:25 smithi130 bash[51302]: debug 2023-08-27T23:34:25.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:25.603996+0000) 2023-08-27T23:34:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:26 smithi130 bash[27579]: cluster 2023-08-27T23:34:25.255934+0000 mgr.y (mgr.14901) 2876 : cluster [DBG] pgmap v2883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:26 smithi130 bash[27579]: audit 2023-08-27T23:34:26.085641+0000 mon.a (mon.0) 2381 : audit [DBG] from='client.? 172.21.15.6:0/4180500703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:26.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:26 smithi130 bash[51302]: debug 2023-08-27T23:34:26.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:26.604263+0000) 2023-08-27T23:34:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:26 smithi006 bash[33918]: cluster 2023-08-27T23:34:25.255934+0000 mgr.y (mgr.14901) 2876 : cluster [DBG] pgmap v2883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:26 smithi006 bash[33918]: audit 2023-08-27T23:34:26.085641+0000 mon.a (mon.0) 2381 : audit [DBG] from='client.? 172.21.15.6:0/4180500703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:26 smithi006 bash[27292]: cluster 2023-08-27T23:34:25.255934+0000 mgr.y (mgr.14901) 2876 : cluster [DBG] pgmap v2883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:26 smithi006 bash[27292]: audit 2023-08-27T23:34:26.085641+0000 mon.a (mon.0) 2381 : audit [DBG] from='client.? 172.21.15.6:0/4180500703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:27.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:27 smithi130 bash[51302]: debug 2023-08-27T23:34:27.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:27.604529+0000) 2023-08-27T23:34:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:28 smithi130 bash[27579]: cluster 2023-08-27T23:34:27.256669+0000 mgr.y (mgr.14901) 2877 : cluster [DBG] pgmap v2884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:28 smithi130 bash[27579]: audit 2023-08-27T23:34:28.496535+0000 mon.a (mon.0) 2382 : audit [DBG] from='client.? 172.21.15.6:0/614633567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:28 smithi130 bash[51302]: debug 2023-08-27T23:34:28.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:28.604798+0000) 2023-08-27T23:34:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:28 smithi006 bash[33918]: cluster 2023-08-27T23:34:27.256669+0000 mgr.y (mgr.14901) 2877 : cluster [DBG] pgmap v2884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:28 smithi006 bash[33918]: audit 2023-08-27T23:34:28.496535+0000 mon.a (mon.0) 2382 : audit [DBG] from='client.? 172.21.15.6:0/614633567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:28 smithi006 bash[27292]: cluster 2023-08-27T23:34:27.256669+0000 mgr.y (mgr.14901) 2877 : cluster [DBG] pgmap v2884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:28 smithi006 bash[27292]: audit 2023-08-27T23:34:28.496535+0000 mon.a (mon.0) 2382 : audit [DBG] from='client.? 172.21.15.6:0/614633567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:29.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:29 smithi130 bash[51302]: debug 2023-08-27T23:34:29.601+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:29.605088+0000) 2023-08-27T23:34:30.827 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:34:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:34:30] "GET /metrics HTTP/1.1" 200 193081 "" "Prometheus/2.33.4" 2023-08-27T23:34:30.828 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:34:30.502Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:30.828 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:34:30.504Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:30 smithi130 bash[27579]: cluster 2023-08-27T23:34:29.257862+0000 mgr.y (mgr.14901) 2878 : cluster [DBG] pgmap v2885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:30 smithi130 bash[51302]: debug 2023-08-27T23:34:30.601+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:30.605378+0000) 2023-08-27T23:34:31.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:30 smithi006 bash[33918]: cluster 2023-08-27T23:34:29.257862+0000 mgr.y (mgr.14901) 2878 : cluster [DBG] pgmap v2885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:31.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:30 smithi006 bash[27292]: cluster 2023-08-27T23:34:29.257862+0000 mgr.y (mgr.14901) 2878 : cluster [DBG] pgmap v2885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:31.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:31 smithi130 bash[27579]: audit 2023-08-27T23:34:30.921899+0000 mon.a (mon.0) 2383 : audit [DBG] from='client.? 172.21.15.6:0/3647401352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:31.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:31 smithi130 bash[51302]: debug 2023-08-27T23:34:31.601+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:31.605664+0000) 2023-08-27T23:34:32.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:31 smithi006 bash[33918]: audit 2023-08-27T23:34:30.921899+0000 mon.a (mon.0) 2383 : audit [DBG] from='client.? 172.21.15.6:0/3647401352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:32.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:31 smithi006 bash[27292]: audit 2023-08-27T23:34:30.921899+0000 mon.a (mon.0) 2383 : audit [DBG] from='client.? 172.21.15.6:0/3647401352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:32.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:34:32.069Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:32.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:34:32.070Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:32 smithi130 bash[27579]: cluster 2023-08-27T23:34:31.258569+0000 mgr.y (mgr.14901) 2879 : cluster [DBG] pgmap v2886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:32 smithi130 bash[51302]: debug 2023-08-27T23:34:32.601+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:32.605946+0000) 2023-08-27T23:34:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:32 smithi006 bash[33918]: cluster 2023-08-27T23:34:31.258569+0000 mgr.y (mgr.14901) 2879 : cluster [DBG] pgmap v2886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:32 smithi006 bash[27292]: cluster 2023-08-27T23:34:31.258569+0000 mgr.y (mgr.14901) 2879 : cluster [DBG] pgmap v2886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:33 smithi130 bash[27579]: audit 2023-08-27T23:34:33.339514+0000 mon.c (mon.2) 1115 : audit [DBG] from='client.? 172.21.15.6:0/4099468947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:33.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:33 smithi130 bash[51302]: debug 2023-08-27T23:34:33.601+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:33.606156+0000) 2023-08-27T23:34:34.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:33 smithi006 bash[27292]: audit 2023-08-27T23:34:33.339514+0000 mon.c (mon.2) 1115 : audit [DBG] from='client.? 172.21.15.6:0/4099468947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:34.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:33 smithi006 bash[33918]: audit 2023-08-27T23:34:33.339514+0000 mon.c (mon.2) 1115 : audit [DBG] from='client.? 172.21.15.6:0/4099468947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:34:33.969Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:34.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:34:33.971Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:34.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:34 smithi130 bash[27579]: cluster 2023-08-27T23:34:33.259279+0000 mgr.y (mgr.14901) 2880 : cluster [DBG] pgmap v2887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:34 smithi130 bash[51302]: debug 2023-08-27T23:34:34.605+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:34.606436+0000) 2023-08-27T23:34:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:34 smithi006 bash[27292]: cluster 2023-08-27T23:34:33.259279+0000 mgr.y (mgr.14901) 2880 : cluster [DBG] pgmap v2887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:34 smithi006 bash[33918]: cluster 2023-08-27T23:34:33.259279+0000 mgr.y (mgr.14901) 2880 : cluster [DBG] pgmap v2887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:35.859 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:35 smithi130 bash[51302]: debug 2023-08-27T23:34:35.601+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:35.606709+0000) 2023-08-27T23:34:36.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:35 smithi130 bash[27579]: audit 2023-08-27T23:34:35.748941+0000 mon.c (mon.2) 1116 : audit [DBG] from='client.? 172.21.15.6:0/3246600455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:36.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:35 smithi006 bash[33918]: audit 2023-08-27T23:34:35.748941+0000 mon.c (mon.2) 1116 : audit [DBG] from='client.? 172.21.15.6:0/3246600455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:36.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:35 smithi006 bash[27292]: audit 2023-08-27T23:34:35.748941+0000 mon.c (mon.2) 1116 : audit [DBG] from='client.? 172.21.15.6:0/3246600455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:36.860 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:36 smithi130 bash[51302]: debug 2023-08-27T23:34:36.605+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:36.606943+0000) 2023-08-27T23:34:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:36 smithi130 bash[27579]: cluster 2023-08-27T23:34:35.260468+0000 mgr.y (mgr.14901) 2881 : cluster [DBG] pgmap v2888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:37.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:36 smithi006 bash[33918]: cluster 2023-08-27T23:34:35.260468+0000 mgr.y (mgr.14901) 2881 : cluster [DBG] pgmap v2888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:37.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:36 smithi006 bash[27292]: cluster 2023-08-27T23:34:35.260468+0000 mgr.y (mgr.14901) 2881 : cluster [DBG] pgmap v2888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:37.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:37 smithi130 bash[51302]: debug 2023-08-27T23:34:37.605+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:37.607154+0000) 2023-08-27T23:34:38.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:38 smithi130 bash[51302]: debug 2023-08-27T23:34:38.605+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:38.607453+0000) 2023-08-27T23:34:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:38 smithi130 bash[27579]: cluster 2023-08-27T23:34:37.261272+0000 mgr.y (mgr.14901) 2882 : cluster [DBG] pgmap v2889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:38 smithi130 bash[27579]: audit 2023-08-27T23:34:38.170327+0000 mon.a (mon.0) 2384 : audit [DBG] from='client.? 172.21.15.6:0/2232378988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:39.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:38 smithi006 bash[27292]: cluster 2023-08-27T23:34:37.261272+0000 mgr.y (mgr.14901) 2882 : cluster [DBG] pgmap v2889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:39.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:38 smithi006 bash[27292]: audit 2023-08-27T23:34:38.170327+0000 mon.a (mon.0) 2384 : audit [DBG] from='client.? 172.21.15.6:0/2232378988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:39.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:38 smithi006 bash[33918]: cluster 2023-08-27T23:34:37.261272+0000 mgr.y (mgr.14901) 2882 : cluster [DBG] pgmap v2889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:39.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:38 smithi006 bash[33918]: audit 2023-08-27T23:34:38.170327+0000 mon.a (mon.0) 2384 : audit [DBG] from='client.? 172.21.15.6:0/2232378988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:39.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:39 smithi130 bash[51302]: debug 2023-08-27T23:34:39.605+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:39.607758+0000) 2023-08-27T23:34:40.826 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:34:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:34:40] "GET /metrics HTTP/1.1" 200 193081 "" "Prometheus/2.33.4" 2023-08-27T23:34:40.826 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:34:40.503Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:40.826 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:34:40.505Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:40 smithi130 bash[51302]: debug 2023-08-27T23:34:40.605+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:40.608038+0000) 2023-08-27T23:34:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:40 smithi130 bash[27579]: cluster 2023-08-27T23:34:39.262497+0000 mgr.y (mgr.14901) 2883 : cluster [DBG] pgmap v2890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:40 smithi130 bash[27579]: audit 2023-08-27T23:34:40.592586+0000 mon.c (mon.2) 1117 : audit [DBG] from='client.? 172.21.15.6:0/2141808729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:40 smithi006 bash[27292]: cluster 2023-08-27T23:34:39.262497+0000 mgr.y (mgr.14901) 2883 : cluster [DBG] pgmap v2890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:40 smithi006 bash[27292]: audit 2023-08-27T23:34:40.592586+0000 mon.c (mon.2) 1117 : audit [DBG] from='client.? 172.21.15.6:0/2141808729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:41.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:40 smithi006 bash[33918]: cluster 2023-08-27T23:34:39.262497+0000 mgr.y (mgr.14901) 2883 : cluster [DBG] pgmap v2890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:41.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:40 smithi006 bash[33918]: audit 2023-08-27T23:34:40.592586+0000 mon.c (mon.2) 1117 : audit [DBG] from='client.? 172.21.15.6:0/2141808729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:41.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:41 smithi130 bash[51302]: debug 2023-08-27T23:34:41.605+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:41.608313+0000) 2023-08-27T23:34:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:34:42.070Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:34:42.071Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:42 smithi130 bash[27579]: cluster 2023-08-27T23:34:41.262936+0000 mgr.y (mgr.14901) 2884 : cluster [DBG] pgmap v2891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:42 smithi130 bash[51302]: debug 2023-08-27T23:34:42.605+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:42.608593+0000) 2023-08-27T23:34:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:42 smithi006 bash[33918]: cluster 2023-08-27T23:34:41.262936+0000 mgr.y (mgr.14901) 2884 : cluster [DBG] pgmap v2891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:42 smithi006 bash[27292]: cluster 2023-08-27T23:34:41.262936+0000 mgr.y (mgr.14901) 2884 : cluster [DBG] pgmap v2891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:43.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:43 smithi130 bash[27579]: audit 2023-08-27T23:34:43.010813+0000 mon.c (mon.2) 1118 : audit [DBG] from='client.? 172.21.15.6:0/2196658735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:43.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:43 smithi130 bash[51302]: debug 2023-08-27T23:34:43.605+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:43.608863+0000) 2023-08-27T23:34:44.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:43 smithi006 bash[33918]: audit 2023-08-27T23:34:43.010813+0000 mon.c (mon.2) 1118 : audit [DBG] from='client.? 172.21.15.6:0/2196658735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:44.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:43 smithi006 bash[27292]: audit 2023-08-27T23:34:43.010813+0000 mon.c (mon.2) 1118 : audit [DBG] from='client.? 172.21.15.6:0/2196658735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:44.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:34:43.969Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:44.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:34:43.971Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:44 smithi130 bash[27579]: cluster 2023-08-27T23:34:43.263541+0000 mgr.y (mgr.14901) 2885 : cluster [DBG] pgmap v2892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:44 smithi130 bash[51302]: debug 2023-08-27T23:34:44.605+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:44.609169+0000) 2023-08-27T23:34:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:44 smithi006 bash[27292]: cluster 2023-08-27T23:34:43.263541+0000 mgr.y (mgr.14901) 2885 : cluster [DBG] pgmap v2892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:44 smithi006 bash[33918]: cluster 2023-08-27T23:34:43.263541+0000 mgr.y (mgr.14901) 2885 : cluster [DBG] pgmap v2892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:45.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:45 smithi130 bash[27579]: audit 2023-08-27T23:34:45.426315+0000 mon.a (mon.0) 2385 : audit [DBG] from='client.? 172.21.15.6:0/2950283913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:45.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:45 smithi130 bash[51302]: debug 2023-08-27T23:34:45.605+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:45.609457+0000) 2023-08-27T23:34:46.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:45 smithi006 bash[27292]: audit 2023-08-27T23:34:45.426315+0000 mon.a (mon.0) 2385 : audit [DBG] from='client.? 172.21.15.6:0/2950283913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:46.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:45 smithi006 bash[33918]: audit 2023-08-27T23:34:45.426315+0000 mon.a (mon.0) 2385 : audit [DBG] from='client.? 172.21.15.6:0/2950283913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:46.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:46 smithi130 bash[51302]: debug 2023-08-27T23:34:46.605+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:46.609751+0000) 2023-08-27T23:34:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:46 smithi130 bash[27579]: cluster 2023-08-27T23:34:45.264636+0000 mgr.y (mgr.14901) 2886 : cluster [DBG] pgmap v2893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:47.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:46 smithi006 bash[27292]: cluster 2023-08-27T23:34:45.264636+0000 mgr.y (mgr.14901) 2886 : cluster [DBG] pgmap v2893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:47.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:46 smithi006 bash[33918]: cluster 2023-08-27T23:34:45.264636+0000 mgr.y (mgr.14901) 2886 : cluster [DBG] pgmap v2893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:47.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:47 smithi130 bash[51302]: debug 2023-08-27T23:34:47.609+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:47.610026+0000) 2023-08-27T23:34:47.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:47 smithi130 bash[27579]: audit 2023-08-27T23:34:47.833303+0000 mon.c (mon.2) 1119 : audit [DBG] from='client.? 172.21.15.6:0/2943280491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:48.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:47 smithi006 bash[27292]: audit 2023-08-27T23:34:47.833303+0000 mon.c (mon.2) 1119 : audit [DBG] from='client.? 172.21.15.6:0/2943280491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:48.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:47 smithi006 bash[33918]: audit 2023-08-27T23:34:47.833303+0000 mon.c (mon.2) 1119 : audit [DBG] from='client.? 172.21.15.6:0/2943280491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:48.863 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:48 smithi130 bash[51302]: debug 2023-08-27T23:34:48.605+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:48.610309+0000) 2023-08-27T23:34:49.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:48 smithi130 bash[27579]: cluster 2023-08-27T23:34:47.265242+0000 mgr.y (mgr.14901) 2887 : cluster [DBG] pgmap v2894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:49.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:48 smithi006 bash[33918]: cluster 2023-08-27T23:34:47.265242+0000 mgr.y (mgr.14901) 2887 : cluster [DBG] pgmap v2894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:49.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:48 smithi006 bash[27292]: cluster 2023-08-27T23:34:47.265242+0000 mgr.y (mgr.14901) 2887 : cluster [DBG] pgmap v2894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:49.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:49 smithi130 bash[51302]: debug 2023-08-27T23:34:49.605+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:49.610577+0000) 2023-08-27T23:34:50.830 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:34:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:34:50] "GET /metrics HTTP/1.1" 200 193083 "" "Prometheus/2.33.4" 2023-08-27T23:34:50.830 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:34:50.503Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:50.831 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:34:50.505Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:50 smithi130 bash[27579]: cluster 2023-08-27T23:34:49.266266+0000 mgr.y (mgr.14901) 2888 : cluster [DBG] pgmap v2895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:50 smithi130 bash[27579]: audit 2023-08-27T23:34:50.251204+0000 mon.c (mon.2) 1120 : audit [DBG] from='client.? 172.21.15.6:0/2138248227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:50 smithi130 bash[51302]: debug 2023-08-27T23:34:50.609+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:50.610933+0000) 2023-08-27T23:34:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:50 smithi006 bash[27292]: cluster 2023-08-27T23:34:49.266266+0000 mgr.y (mgr.14901) 2888 : cluster [DBG] pgmap v2895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:50 smithi006 bash[27292]: audit 2023-08-27T23:34:50.251204+0000 mon.c (mon.2) 1120 : audit [DBG] from='client.? 172.21.15.6:0/2138248227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:50 smithi006 bash[33918]: cluster 2023-08-27T23:34:49.266266+0000 mgr.y (mgr.14901) 2888 : cluster [DBG] pgmap v2895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:51.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:50 smithi006 bash[33918]: audit 2023-08-27T23:34:50.251204+0000 mon.c (mon.2) 1120 : audit [DBG] from='client.? 172.21.15.6:0/2138248227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:51.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:51 smithi130 bash[51302]: debug 2023-08-27T23:34:51.609+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:51.611209+0000) 2023-08-27T23:34:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:34:52.071Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:34:52.073Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:52.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:52 smithi130 bash[51302]: debug 2023-08-27T23:34:52.609+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:52.611488+0000) 2023-08-27T23:34:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:52 smithi130 bash[27579]: cluster 2023-08-27T23:34:51.266922+0000 mgr.y (mgr.14901) 2889 : cluster [DBG] pgmap v2896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:52 smithi130 bash[27579]: audit 2023-08-27T23:34:52.223336+0000 mon.b (mon.1) 394 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:34:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:52 smithi130 bash[27579]: audit 2023-08-27T23:34:52.667730+0000 mon.c (mon.2) 1121 : audit [DBG] from='client.? 172.21.15.6:0/2407207750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:53.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:52 smithi006 bash[27292]: cluster 2023-08-27T23:34:51.266922+0000 mgr.y (mgr.14901) 2889 : cluster [DBG] pgmap v2896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:53.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:52 smithi006 bash[27292]: audit 2023-08-27T23:34:52.223336+0000 mon.b (mon.1) 394 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:34:53.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:52 smithi006 bash[27292]: audit 2023-08-27T23:34:52.667730+0000 mon.c (mon.2) 1121 : audit [DBG] from='client.? 172.21.15.6:0/2407207750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:53.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:52 smithi006 bash[33918]: cluster 2023-08-27T23:34:51.266922+0000 mgr.y (mgr.14901) 2889 : cluster [DBG] pgmap v2896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:53.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:52 smithi006 bash[33918]: audit 2023-08-27T23:34:52.223336+0000 mon.b (mon.1) 394 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:34:53.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:52 smithi006 bash[33918]: audit 2023-08-27T23:34:52.667730+0000 mon.c (mon.2) 1121 : audit [DBG] from='client.? 172.21.15.6:0/2407207750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:53.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:53 smithi130 bash[51302]: debug 2023-08-27T23:34:53.609+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:53.611712+0000) 2023-08-27T23:34:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:34:53.969Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:34:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:34:53.971Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:34:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:54 smithi130 bash[27579]: cluster 2023-08-27T23:34:53.267510+0000 mgr.y (mgr.14901) 2890 : cluster [DBG] pgmap v2897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:54.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:54 smithi130 bash[51302]: debug 2023-08-27T23:34:54.609+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:54.612021+0000) 2023-08-27T23:34:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:54 smithi006 bash[27292]: cluster 2023-08-27T23:34:53.267510+0000 mgr.y (mgr.14901) 2890 : cluster [DBG] pgmap v2897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:54 smithi006 bash[33918]: cluster 2023-08-27T23:34:53.267510+0000 mgr.y (mgr.14901) 2890 : cluster [DBG] pgmap v2897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:55.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:55 smithi130 bash[27579]: audit 2023-08-27T23:34:55.089139+0000 mon.a (mon.0) 2386 : audit [DBG] from='client.? 172.21.15.6:0/3768250223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:55.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:55 smithi130 bash[51302]: debug 2023-08-27T23:34:55.609+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:55.612304+0000) 2023-08-27T23:34:56.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:55 smithi006 bash[27292]: audit 2023-08-27T23:34:55.089139+0000 mon.a (mon.0) 2386 : audit [DBG] from='client.? 172.21.15.6:0/3768250223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:56.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:55 smithi006 bash[33918]: audit 2023-08-27T23:34:55.089139+0000 mon.a (mon.0) 2386 : audit [DBG] from='client.? 172.21.15.6:0/3768250223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:56.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:56 smithi130 bash[27579]: cluster 2023-08-27T23:34:55.268640+0000 mgr.y (mgr.14901) 2891 : cluster [DBG] pgmap v2898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:56.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:56 smithi130 bash[51302]: debug 2023-08-27T23:34:56.609+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:56.612533+0000) 2023-08-27T23:34:57.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:56 smithi006 bash[33918]: cluster 2023-08-27T23:34:55.268640+0000 mgr.y (mgr.14901) 2891 : cluster [DBG] pgmap v2898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:57.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:56 smithi006 bash[27292]: cluster 2023-08-27T23:34:55.268640+0000 mgr.y (mgr.14901) 2891 : cluster [DBG] pgmap v2898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:57.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:57 smithi130 bash[27579]: audit 2023-08-27T23:34:57.573055+0000 mon.a (mon.0) 2387 : audit [DBG] from='client.? 172.21.15.6:0/288025417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:57.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:57 smithi130 bash[51302]: debug 2023-08-27T23:34:57.609+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:57.612714+0000) 2023-08-27T23:34:58.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:57 smithi006 bash[33918]: audit 2023-08-27T23:34:57.573055+0000 mon.a (mon.0) 2387 : audit [DBG] from='client.? 172.21.15.6:0/288025417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:58.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:57 smithi006 bash[27292]: audit 2023-08-27T23:34:57.573055+0000 mon.a (mon.0) 2387 : audit [DBG] from='client.? 172.21.15.6:0/288025417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:58.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:58 smithi130 bash[51302]: debug 2023-08-27T23:34:58.609+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:58.612985+0000) 2023-08-27T23:34:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:58 smithi130 bash[27579]: cluster 2023-08-27T23:34:57.269123+0000 mgr.y (mgr.14901) 2892 : cluster [DBG] pgmap v2899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:58 smithi130 bash[27579]: audit 2023-08-27T23:34:58.087817+0000 mon.a (mon.0) 2388 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:34:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:58 smithi130 bash[27579]: audit 2023-08-27T23:34:58.549411+0000 mon.a (mon.0) 2389 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:34:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:58 smithi130 bash[27579]: audit 2023-08-27T23:34:58.821906+0000 mon.b (mon.1) 395 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:34:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:58 smithi130 bash[27579]: audit 2023-08-27T23:34:58.824030+0000 mon.b (mon.1) 396 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:34:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:34:58 smithi130 bash[27579]: audit 2023-08-27T23:34:58.830782+0000 mon.a (mon.0) 2390 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:34:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:58 smithi006 bash[33918]: cluster 2023-08-27T23:34:57.269123+0000 mgr.y (mgr.14901) 2892 : cluster [DBG] pgmap v2899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:59.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:58 smithi006 bash[33918]: audit 2023-08-27T23:34:58.087817+0000 mon.a (mon.0) 2388 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:34:59.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:58 smithi006 bash[33918]: audit 2023-08-27T23:34:58.549411+0000 mon.a (mon.0) 2389 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:34:59.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:58 smithi006 bash[33918]: audit 2023-08-27T23:34:58.821906+0000 mon.b (mon.1) 395 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:34:59.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:58 smithi006 bash[33918]: audit 2023-08-27T23:34:58.824030+0000 mon.b (mon.1) 396 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:34:59.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:34:58 smithi006 bash[33918]: audit 2023-08-27T23:34:58.830782+0000 mon.a (mon.0) 2390 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:34:59.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:58 smithi006 bash[27292]: cluster 2023-08-27T23:34:57.269123+0000 mgr.y (mgr.14901) 2892 : cluster [DBG] pgmap v2899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:34:59.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:58 smithi006 bash[27292]: audit 2023-08-27T23:34:58.087817+0000 mon.a (mon.0) 2388 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:34:59.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:58 smithi006 bash[27292]: audit 2023-08-27T23:34:58.549411+0000 mon.a (mon.0) 2389 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:34:59.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:58 smithi006 bash[27292]: audit 2023-08-27T23:34:58.821906+0000 mon.b (mon.1) 395 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:34:59.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:58 smithi006 bash[27292]: audit 2023-08-27T23:34:58.824030+0000 mon.b (mon.1) 396 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:34:59.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:34:58 smithi006 bash[27292]: audit 2023-08-27T23:34:58.830782+0000 mon.a (mon.0) 2390 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:34:59.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:34:59 smithi130 bash[51302]: debug 2023-08-27T23:34:59.609+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:59.613260+0000) 2023-08-27T23:35:00.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:00 smithi130 bash[51302]: debug 2023-08-27T23:35:00.609+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:00.613558+0000) 2023-08-27T23:35:00.868 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:35:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:35:00] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:35:00.868 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:35:00.504Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:00.868 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:35:00.506Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:00 smithi006 bash[33918]: cluster 2023-08-27T23:34:59.270166+0000 mgr.y (mgr.14901) 2893 : cluster [DBG] pgmap v2900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:00 smithi006 bash[33918]: audit 2023-08-27T23:34:59.987578+0000 mon.c (mon.2) 1122 : audit [DBG] from='client.? 172.21.15.6:0/384728572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:00 smithi006 bash[27292]: cluster 2023-08-27T23:34:59.270166+0000 mgr.y (mgr.14901) 2893 : cluster [DBG] pgmap v2900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:00 smithi006 bash[27292]: audit 2023-08-27T23:34:59.987578+0000 mon.c (mon.2) 1122 : audit [DBG] from='client.? 172.21.15.6:0/384728572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:01.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:00 smithi130 bash[27579]: cluster 2023-08-27T23:34:59.270166+0000 mgr.y (mgr.14901) 2893 : cluster [DBG] pgmap v2900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:01.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:00 smithi130 bash[27579]: audit 2023-08-27T23:34:59.987578+0000 mon.c (mon.2) 1122 : audit [DBG] from='client.? 172.21.15.6:0/384728572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:01.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:01 smithi130 bash[51302]: debug 2023-08-27T23:35:01.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:01.613845+0000) 2023-08-27T23:35:02.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:01 smithi006 bash[33918]: cluster 2023-08-27T23:35:01.270843+0000 mgr.y (mgr.14901) 2894 : cluster [DBG] pgmap v2901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:02.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:35:02.071Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:02.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:35:02.073Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:02.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:01 smithi006 bash[27292]: cluster 2023-08-27T23:35:01.270843+0000 mgr.y (mgr.14901) 2894 : cluster [DBG] pgmap v2901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:02.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:01 smithi130 bash[27579]: cluster 2023-08-27T23:35:01.270843+0000 mgr.y (mgr.14901) 2894 : cluster [DBG] pgmap v2901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:02.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:02 smithi130 bash[51302]: debug 2023-08-27T23:35:02.609+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:02.614114+0000) 2023-08-27T23:35:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:02 smithi006 bash[33918]: audit 2023-08-27T23:35:02.405575+0000 mon.c (mon.2) 1123 : audit [DBG] from='client.? 172.21.15.6:0/239149828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:02 smithi006 bash[27292]: audit 2023-08-27T23:35:02.405575+0000 mon.c (mon.2) 1123 : audit [DBG] from='client.? 172.21.15.6:0/239149828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:03.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:02 smithi130 bash[27579]: audit 2023-08-27T23:35:02.405575+0000 mon.c (mon.2) 1123 : audit [DBG] from='client.? 172.21.15.6:0/239149828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:03.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:03 smithi130 bash[51302]: debug 2023-08-27T23:35:03.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:03.614383+0000) 2023-08-27T23:35:04.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:03 smithi006 bash[33918]: cluster 2023-08-27T23:35:03.271432+0000 mgr.y (mgr.14901) 2895 : cluster [DBG] pgmap v2902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:04.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:03 smithi006 bash[27292]: cluster 2023-08-27T23:35:03.271432+0000 mgr.y (mgr.14901) 2895 : cluster [DBG] pgmap v2902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:04.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:35:03.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:04.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:35:03.972Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:04.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:03 smithi130 bash[27579]: cluster 2023-08-27T23:35:03.271432+0000 mgr.y (mgr.14901) 2895 : cluster [DBG] pgmap v2902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:04.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:04 smithi130 bash[51302]: debug 2023-08-27T23:35:04.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:04.614659+0000) 2023-08-27T23:35:05.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:05 smithi130 bash[27579]: audit 2023-08-27T23:35:04.829933+0000 mon.a (mon.0) 2391 : audit [DBG] from='client.? 172.21.15.6:0/795476924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:05.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:05 smithi130 bash[51302]: debug 2023-08-27T23:35:05.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:05.614968+0000) 2023-08-27T23:35:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:05 smithi006 bash[33918]: audit 2023-08-27T23:35:04.829933+0000 mon.a (mon.0) 2391 : audit [DBG] from='client.? 172.21.15.6:0/795476924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:06.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:05 smithi006 bash[27292]: audit 2023-08-27T23:35:04.829933+0000 mon.a (mon.0) 2391 : audit [DBG] from='client.? 172.21.15.6:0/795476924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:06 smithi130 bash[27579]: cluster 2023-08-27T23:35:05.272561+0000 mgr.y (mgr.14901) 2896 : cluster [DBG] pgmap v2903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:06.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:06 smithi130 bash[51302]: debug 2023-08-27T23:35:06.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:06.615281+0000) 2023-08-27T23:35:07.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:06 smithi006 bash[27292]: cluster 2023-08-27T23:35:05.272561+0000 mgr.y (mgr.14901) 2896 : cluster [DBG] pgmap v2903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:07.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:06 smithi006 bash[33918]: cluster 2023-08-27T23:35:05.272561+0000 mgr.y (mgr.14901) 2896 : cluster [DBG] pgmap v2903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:07.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:07 smithi130 bash[27579]: audit 2023-08-27T23:35:07.242038+0000 mon.c (mon.2) 1124 : audit [DBG] from='client.? 172.21.15.6:0/2894152579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:07.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:07 smithi130 bash[51302]: debug 2023-08-27T23:35:07.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:07.615550+0000) 2023-08-27T23:35:08.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:07 smithi006 bash[27292]: audit 2023-08-27T23:35:07.242038+0000 mon.c (mon.2) 1124 : audit [DBG] from='client.? 172.21.15.6:0/2894152579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:08.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:07 smithi006 bash[33918]: audit 2023-08-27T23:35:07.242038+0000 mon.c (mon.2) 1124 : audit [DBG] from='client.? 172.21.15.6:0/2894152579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:08 smithi130 bash[27579]: cluster 2023-08-27T23:35:07.273375+0000 mgr.y (mgr.14901) 2897 : cluster [DBG] pgmap v2904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:08.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:08 smithi130 bash[51302]: debug 2023-08-27T23:35:08.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:08.615858+0000) 2023-08-27T23:35:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:08 smithi006 bash[33918]: cluster 2023-08-27T23:35:07.273375+0000 mgr.y (mgr.14901) 2897 : cluster [DBG] pgmap v2904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:08 smithi006 bash[27292]: cluster 2023-08-27T23:35:07.273375+0000 mgr.y (mgr.14901) 2897 : cluster [DBG] pgmap v2904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:09.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:09 smithi130 bash[27579]: audit 2023-08-27T23:35:09.660315+0000 mon.a (mon.0) 2392 : audit [DBG] from='client.? 172.21.15.6:0/1032883895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:09.870 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:09 smithi130 bash[51302]: debug 2023-08-27T23:35:09.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:09.616138+0000) 2023-08-27T23:35:10.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:09 smithi006 bash[27292]: audit 2023-08-27T23:35:09.660315+0000 mon.a (mon.0) 2392 : audit [DBG] from='client.? 172.21.15.6:0/1032883895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:10.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:09 smithi006 bash[33918]: audit 2023-08-27T23:35:09.660315+0000 mon.a (mon.0) 2392 : audit [DBG] from='client.? 172.21.15.6:0/1032883895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:10.866 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:35:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:35:10] "GET /metrics HTTP/1.1" 200 193086 "" "Prometheus/2.33.4" 2023-08-27T23:35:10.866 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:35:10.505Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:10.866 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:35:10.507Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:10 smithi130 bash[27579]: cluster 2023-08-27T23:35:09.274516+0000 mgr.y (mgr.14901) 2898 : cluster [DBG] pgmap v2905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:10.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:10 smithi130 bash[51302]: debug 2023-08-27T23:35:10.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:10.616441+0000) 2023-08-27T23:35:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:10 smithi006 bash[33918]: cluster 2023-08-27T23:35:09.274516+0000 mgr.y (mgr.14901) 2898 : cluster [DBG] pgmap v2905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:11.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:10 smithi006 bash[27292]: cluster 2023-08-27T23:35:09.274516+0000 mgr.y (mgr.14901) 2898 : cluster [DBG] pgmap v2905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:12.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:11 smithi130 bash[51302]: debug 2023-08-27T23:35:11.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:11.616717+0000) 2023-08-27T23:35:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:35:12.072Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:35:12.073Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:12.871 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:12 smithi130 bash[51302]: debug 2023-08-27T23:35:12.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:12.617008+0000) 2023-08-27T23:35:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:12 smithi006 bash[33918]: cluster 2023-08-27T23:35:11.275177+0000 mgr.y (mgr.14901) 2899 : cluster [DBG] pgmap v2906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:12 smithi006 bash[33918]: audit 2023-08-27T23:35:12.073033+0000 mon.a (mon.0) 2393 : audit [DBG] from='client.? 172.21.15.6:0/1311645435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:12 smithi006 bash[27292]: cluster 2023-08-27T23:35:11.275177+0000 mgr.y (mgr.14901) 2899 : cluster [DBG] pgmap v2906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:13.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:12 smithi006 bash[27292]: audit 2023-08-27T23:35:12.073033+0000 mon.a (mon.0) 2393 : audit [DBG] from='client.? 172.21.15.6:0/1311645435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:13.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:12 smithi130 bash[27579]: cluster 2023-08-27T23:35:11.275177+0000 mgr.y (mgr.14901) 2899 : cluster [DBG] pgmap v2906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:12 smithi130 bash[27579]: audit 2023-08-27T23:35:12.073033+0000 mon.a (mon.0) 2393 : audit [DBG] from='client.? 172.21.15.6:0/1311645435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:13.878 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:13 smithi130 bash[51302]: debug 2023-08-27T23:35:13.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:13.617306+0000) 2023-08-27T23:35:14.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:13 smithi006 bash[33918]: cluster 2023-08-27T23:35:13.275775+0000 mgr.y (mgr.14901) 2900 : cluster [DBG] pgmap v2907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:35:13.971Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:35:13.974Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:14.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:13 smithi006 bash[27292]: cluster 2023-08-27T23:35:13.275775+0000 mgr.y (mgr.14901) 2900 : cluster [DBG] pgmap v2907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:14.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:13 smithi130 bash[27579]: cluster 2023-08-27T23:35:13.275775+0000 mgr.y (mgr.14901) 2900 : cluster [DBG] pgmap v2907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:14.881 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:14 smithi130 bash[51302]: debug 2023-08-27T23:35:14.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:14.617568+0000) 2023-08-27T23:35:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:14 smithi006 bash[33918]: audit 2023-08-27T23:35:14.487825+0000 mon.c (mon.2) 1125 : audit [DBG] from='client.? 172.21.15.6:0/448931755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:14 smithi006 bash[27292]: audit 2023-08-27T23:35:14.487825+0000 mon.c (mon.2) 1125 : audit [DBG] from='client.? 172.21.15.6:0/448931755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:15.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:14 smithi130 bash[27579]: audit 2023-08-27T23:35:14.487825+0000 mon.c (mon.2) 1125 : audit [DBG] from='client.? 172.21.15.6:0/448931755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:15.889 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:15 smithi130 bash[51302]: debug 2023-08-27T23:35:15.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:15.617844+0000) 2023-08-27T23:35:16.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:15 smithi006 bash[33918]: cluster 2023-08-27T23:35:15.276985+0000 mgr.y (mgr.14901) 2901 : cluster [DBG] pgmap v2908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:16.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:15 smithi006 bash[27292]: cluster 2023-08-27T23:35:15.276985+0000 mgr.y (mgr.14901) 2901 : cluster [DBG] pgmap v2908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:16.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:15 smithi130 bash[27579]: cluster 2023-08-27T23:35:15.276985+0000 mgr.y (mgr.14901) 2901 : cluster [DBG] pgmap v2908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:16.978 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:16 smithi130 bash[51302]: debug 2023-08-27T23:35:16.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:16.618120+0000) 2023-08-27T23:35:17.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:16 smithi130 bash[27579]: audit 2023-08-27T23:35:16.921843+0000 mon.c (mon.2) 1126 : audit [DBG] from='client.? 172.21.15.6:0/1443363128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:17.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:16 smithi006 bash[27292]: audit 2023-08-27T23:35:16.921843+0000 mon.c (mon.2) 1126 : audit [DBG] from='client.? 172.21.15.6:0/1443363128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:17.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:16 smithi006 bash[33918]: audit 2023-08-27T23:35:16.921843+0000 mon.c (mon.2) 1126 : audit [DBG] from='client.? 172.21.15.6:0/1443363128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:17.986 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:17 smithi130 bash[51302]: debug 2023-08-27T23:35:17.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:17.618337+0000) 2023-08-27T23:35:18.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:17 smithi130 bash[27579]: cluster 2023-08-27T23:35:17.277574+0000 mgr.y (mgr.14901) 2902 : cluster [DBG] pgmap v2909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:18.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:17 smithi006 bash[33918]: cluster 2023-08-27T23:35:17.277574+0000 mgr.y (mgr.14901) 2902 : cluster [DBG] pgmap v2909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:18.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:17 smithi006 bash[27292]: cluster 2023-08-27T23:35:17.277574+0000 mgr.y (mgr.14901) 2902 : cluster [DBG] pgmap v2909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:19.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:18 smithi130 bash[51302]: debug 2023-08-27T23:35:18.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:18.618632+0000) 2023-08-27T23:35:19.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:19 smithi006 bash[27292]: audit 2023-08-27T23:35:19.330767+0000 mon.a (mon.0) 2394 : audit [DBG] from='client.? 172.21.15.6:0/785211366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:19.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:19 smithi006 bash[33918]: audit 2023-08-27T23:35:19.330767+0000 mon.a (mon.0) 2394 : audit [DBG] from='client.? 172.21.15.6:0/785211366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:19.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:19 smithi130 bash[27579]: audit 2023-08-27T23:35:19.330767+0000 mon.a (mon.0) 2394 : audit [DBG] from='client.? 172.21.15.6:0/785211366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:19 smithi130 bash[51302]: debug 2023-08-27T23:35:19.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:19.618941+0000) 2023-08-27T23:35:20.836 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:35:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:35:20] "GET /metrics HTTP/1.1" 200 193088 "" "Prometheus/2.33.4" 2023-08-27T23:35:20.836 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:35:20.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:20.837 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:35:20.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:20 smithi130 bash[27579]: cluster 2023-08-27T23:35:19.278499+0000 mgr.y (mgr.14901) 2903 : cluster [DBG] pgmap v2910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:21.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:20 smithi130 bash[51302]: debug 2023-08-27T23:35:20.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:20.619248+0000) 2023-08-27T23:35:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:20 smithi006 bash[27292]: cluster 2023-08-27T23:35:19.278499+0000 mgr.y (mgr.14901) 2903 : cluster [DBG] pgmap v2910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:21.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:20 smithi006 bash[33918]: cluster 2023-08-27T23:35:19.278499+0000 mgr.y (mgr.14901) 2903 : cluster [DBG] pgmap v2910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:22.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:21 smithi130 bash[27579]: audit 2023-08-27T23:35:21.743553+0000 mon.a (mon.0) 2395 : audit [DBG] from='client.? 172.21.15.6:0/3507834263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:22.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:21 smithi130 bash[51302]: debug 2023-08-27T23:35:21.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:21.619536+0000) 2023-08-27T23:35:22.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:35:22.072Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:22.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:35:22.074Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:22.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:21 smithi006 bash[27292]: audit 2023-08-27T23:35:21.743553+0000 mon.a (mon.0) 2395 : audit [DBG] from='client.? 172.21.15.6:0/3507834263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:22.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:21 smithi006 bash[33918]: audit 2023-08-27T23:35:21.743553+0000 mon.a (mon.0) 2395 : audit [DBG] from='client.? 172.21.15.6:0/3507834263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:23.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:22 smithi130 bash[27579]: cluster 2023-08-27T23:35:21.279186+0000 mgr.y (mgr.14901) 2904 : cluster [DBG] pgmap v2911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:23.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:22 smithi130 bash[51302]: debug 2023-08-27T23:35:22.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:22.619781+0000) 2023-08-27T23:35:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:22 smithi006 bash[33918]: cluster 2023-08-27T23:35:21.279186+0000 mgr.y (mgr.14901) 2904 : cluster [DBG] pgmap v2911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:23.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:22 smithi006 bash[27292]: cluster 2023-08-27T23:35:21.279186+0000 mgr.y (mgr.14901) 2904 : cluster [DBG] pgmap v2911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:24.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:23 smithi130 bash[51302]: debug 2023-08-27T23:35:23.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:23.620074+0000) 2023-08-27T23:35:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:35:23.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:35:23.974Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:24 smithi130 bash[27579]: cluster 2023-08-27T23:35:23.279810+0000 mgr.y (mgr.14901) 2905 : cluster [DBG] pgmap v2912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:24 smithi130 bash[27579]: audit 2023-08-27T23:35:24.167177+0000 mon.c (mon.2) 1127 : audit [DBG] from='client.? 172.21.15.6:0/1524042769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:24 smithi130 bash[51302]: debug 2023-08-27T23:35:24.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:24.620366+0000) 2023-08-27T23:35:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:24 smithi006 bash[33918]: cluster 2023-08-27T23:35:23.279810+0000 mgr.y (mgr.14901) 2905 : cluster [DBG] pgmap v2912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:24 smithi006 bash[33918]: audit 2023-08-27T23:35:24.167177+0000 mon.c (mon.2) 1127 : audit [DBG] from='client.? 172.21.15.6:0/1524042769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:24 smithi006 bash[27292]: cluster 2023-08-27T23:35:23.279810+0000 mgr.y (mgr.14901) 2905 : cluster [DBG] pgmap v2912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:24 smithi006 bash[27292]: audit 2023-08-27T23:35:24.167177+0000 mon.c (mon.2) 1127 : audit [DBG] from='client.? 172.21.15.6:0/1524042769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:26.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:25 smithi130 bash[51302]: debug 2023-08-27T23:35:25.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:25.620719+0000) 2023-08-27T23:35:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:26 smithi130 bash[27579]: cluster 2023-08-27T23:35:25.280945+0000 mgr.y (mgr.14901) 2906 : cluster [DBG] pgmap v2913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:26 smithi130 bash[27579]: audit 2023-08-27T23:35:26.580096+0000 mon.a (mon.0) 2396 : audit [DBG] from='client.? 172.21.15.6:0/1887919987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:27.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:26 smithi130 bash[51302]: debug 2023-08-27T23:35:26.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:26.620985+0000) 2023-08-27T23:35:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:26 smithi006 bash[27292]: cluster 2023-08-27T23:35:25.280945+0000 mgr.y (mgr.14901) 2906 : cluster [DBG] pgmap v2913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:26 smithi006 bash[27292]: audit 2023-08-27T23:35:26.580096+0000 mon.a (mon.0) 2396 : audit [DBG] from='client.? 172.21.15.6:0/1887919987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:26 smithi006 bash[33918]: cluster 2023-08-27T23:35:25.280945+0000 mgr.y (mgr.14901) 2906 : cluster [DBG] pgmap v2913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:26 smithi006 bash[33918]: audit 2023-08-27T23:35:26.580096+0000 mon.a (mon.0) 2396 : audit [DBG] from='client.? 172.21.15.6:0/1887919987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:28.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:27 smithi130 bash[51302]: debug 2023-08-27T23:35:27.621+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:27.621279+0000) 2023-08-27T23:35:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:28 smithi130 bash[27579]: cluster 2023-08-27T23:35:27.281514+0000 mgr.y (mgr.14901) 2907 : cluster [DBG] pgmap v2914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:29.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:28 smithi130 bash[51302]: debug 2023-08-27T23:35:28.620+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:28.621602+0000) 2023-08-27T23:35:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:28 smithi006 bash[33918]: cluster 2023-08-27T23:35:27.281514+0000 mgr.y (mgr.14901) 2907 : cluster [DBG] pgmap v2914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:29.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:28 smithi006 bash[27292]: cluster 2023-08-27T23:35:27.281514+0000 mgr.y (mgr.14901) 2907 : cluster [DBG] pgmap v2914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:29.875 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:29 smithi130 bash[51302]: debug 2023-08-27T23:35:29.616+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:29.621888+0000) 2023-08-27T23:35:30.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:29 smithi006 bash[27292]: audit 2023-08-27T23:35:28.999687+0000 mon.c (mon.2) 1128 : audit [DBG] from='client.? 172.21.15.6:0/1565744868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:30.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:29 smithi006 bash[33918]: audit 2023-08-27T23:35:28.999687+0000 mon.c (mon.2) 1128 : audit [DBG] from='client.? 172.21.15.6:0/1565744868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:30.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:29 smithi130 bash[27579]: audit 2023-08-27T23:35:28.999687+0000 mon.c (mon.2) 1128 : audit [DBG] from='client.? 172.21.15.6:0/1565744868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:30.875 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:35:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:35:30] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T23:35:30.876 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:35:30.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:30.876 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:35:30.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:30.876 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:30 smithi130 bash[51302]: debug 2023-08-27T23:35:30.620+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:30.622206+0000) 2023-08-27T23:35:31.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:30 smithi006 bash[33918]: cluster 2023-08-27T23:35:29.282623+0000 mgr.y (mgr.14901) 2908 : cluster [DBG] pgmap v2915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:31.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:30 smithi006 bash[27292]: cluster 2023-08-27T23:35:29.282623+0000 mgr.y (mgr.14901) 2908 : cluster [DBG] pgmap v2915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:31.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:30 smithi130 bash[27579]: cluster 2023-08-27T23:35:29.282623+0000 mgr.y (mgr.14901) 2908 : cluster [DBG] pgmap v2915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:31.885 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:31 smithi130 bash[51302]: debug 2023-08-27T23:35:31.620+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:31.622507+0000) 2023-08-27T23:35:32.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:31 smithi006 bash[27292]: cluster 2023-08-27T23:35:31.283210+0000 mgr.y (mgr.14901) 2909 : cluster [DBG] pgmap v2916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:32.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:31 smithi006 bash[27292]: audit 2023-08-27T23:35:31.415983+0000 mon.a (mon.0) 2397 : audit [DBG] from='client.? 172.21.15.6:0/1512159953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:32.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:31 smithi006 bash[33918]: cluster 2023-08-27T23:35:31.283210+0000 mgr.y (mgr.14901) 2909 : cluster [DBG] pgmap v2916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:32.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:31 smithi006 bash[33918]: audit 2023-08-27T23:35:31.415983+0000 mon.a (mon.0) 2397 : audit [DBG] from='client.? 172.21.15.6:0/1512159953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:32.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:35:32.073Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:32.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:35:32.075Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:32.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:31 smithi130 bash[27579]: cluster 2023-08-27T23:35:31.283210+0000 mgr.y (mgr.14901) 2909 : cluster [DBG] pgmap v2916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:31 smithi130 bash[27579]: audit 2023-08-27T23:35:31.415983+0000 mon.a (mon.0) 2397 : audit [DBG] from='client.? 172.21.15.6:0/1512159953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:33.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:32 smithi130 bash[51302]: debug 2023-08-27T23:35:32.620+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:32.622773+0000) 2023-08-27T23:35:34.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:33 smithi130 bash[51302]: debug 2023-08-27T23:35:33.620+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:33.623039+0000) 2023-08-27T23:35:34.343 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:35:33.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:34.344 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:35:33.975Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:34.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:34 smithi130 bash[27579]: cluster 2023-08-27T23:35:33.283849+0000 mgr.y (mgr.14901) 2910 : cluster [DBG] pgmap v2917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:34 smithi130 bash[27579]: audit 2023-08-27T23:35:33.838082+0000 mon.a (mon.0) 2398 : audit [DBG] from='client.? 172.21.15.6:0/344036010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:34 smithi006 bash[27292]: cluster 2023-08-27T23:35:33.283849+0000 mgr.y (mgr.14901) 2910 : cluster [DBG] pgmap v2917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:34 smithi006 bash[27292]: audit 2023-08-27T23:35:33.838082+0000 mon.a (mon.0) 2398 : audit [DBG] from='client.? 172.21.15.6:0/344036010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:34 smithi006 bash[33918]: cluster 2023-08-27T23:35:33.283849+0000 mgr.y (mgr.14901) 2910 : cluster [DBG] pgmap v2917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:34 smithi006 bash[33918]: audit 2023-08-27T23:35:33.838082+0000 mon.a (mon.0) 2398 : audit [DBG] from='client.? 172.21.15.6:0/344036010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:35.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:34 smithi130 bash[51302]: debug 2023-08-27T23:35:34.620+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:34.623290+0000) 2023-08-27T23:35:36.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:35 smithi130 bash[51302]: debug 2023-08-27T23:35:35.620+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:35.623593+0000) 2023-08-27T23:35:37.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:36 smithi130 bash[51302]: debug 2023-08-27T23:35:36.620+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:36.623864+0000) 2023-08-27T23:35:37.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:36 smithi130 bash[27579]: cluster 2023-08-27T23:35:35.284959+0000 mgr.y (mgr.14901) 2911 : cluster [DBG] pgmap v2918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:37.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:36 smithi130 bash[27579]: audit 2023-08-27T23:35:36.249243+0000 mon.a (mon.0) 2399 : audit [DBG] from='client.? 172.21.15.6:0/4276045244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:37.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:36 smithi006 bash[33918]: cluster 2023-08-27T23:35:35.284959+0000 mgr.y (mgr.14901) 2911 : cluster [DBG] pgmap v2918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:37.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:36 smithi006 bash[33918]: audit 2023-08-27T23:35:36.249243+0000 mon.a (mon.0) 2399 : audit [DBG] from='client.? 172.21.15.6:0/4276045244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:37.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:36 smithi006 bash[27292]: cluster 2023-08-27T23:35:35.284959+0000 mgr.y (mgr.14901) 2911 : cluster [DBG] pgmap v2918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:37.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:36 smithi006 bash[27292]: audit 2023-08-27T23:35:36.249243+0000 mon.a (mon.0) 2399 : audit [DBG] from='client.? 172.21.15.6:0/4276045244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:38.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:37 smithi130 bash[51302]: debug 2023-08-27T23:35:37.620+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:37.624125+0000) 2023-08-27T23:35:39.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:38 smithi130 bash[51302]: debug 2023-08-27T23:35:38.620+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:38.624434+0000) 2023-08-27T23:35:39.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:38 smithi130 bash[27579]: cluster 2023-08-27T23:35:37.285578+0000 mgr.y (mgr.14901) 2912 : cluster [DBG] pgmap v2919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:39.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:38 smithi130 bash[27579]: audit 2023-08-27T23:35:38.664322+0000 mon.c (mon.2) 1129 : audit [DBG] from='client.? 172.21.15.6:0/1045966096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:39.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:38 smithi006 bash[33918]: cluster 2023-08-27T23:35:37.285578+0000 mgr.y (mgr.14901) 2912 : cluster [DBG] pgmap v2919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:39.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:38 smithi006 bash[33918]: audit 2023-08-27T23:35:38.664322+0000 mon.c (mon.2) 1129 : audit [DBG] from='client.? 172.21.15.6:0/1045966096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:39.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:38 smithi006 bash[27292]: cluster 2023-08-27T23:35:37.285578+0000 mgr.y (mgr.14901) 2912 : cluster [DBG] pgmap v2919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:39.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:38 smithi006 bash[27292]: audit 2023-08-27T23:35:38.664322+0000 mon.c (mon.2) 1129 : audit [DBG] from='client.? 172.21.15.6:0/1045966096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:40.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:39 smithi130 bash[51302]: debug 2023-08-27T23:35:39.620+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:39.624728+0000) 2023-08-27T23:35:40.851 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:35:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:35:40] "GET /metrics HTTP/1.1" 200 193085 "" "Prometheus/2.33.4" 2023-08-27T23:35:40.851 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:35:40.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:40.851 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:35:40.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:40 smithi130 bash[27579]: cluster 2023-08-27T23:35:39.286651+0000 mgr.y (mgr.14901) 2913 : cluster [DBG] pgmap v2920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:41.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:40 smithi130 bash[51302]: debug 2023-08-27T23:35:40.624+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:40.625037+0000) 2023-08-27T23:35:41.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:40 smithi006 bash[33918]: cluster 2023-08-27T23:35:39.286651+0000 mgr.y (mgr.14901) 2913 : cluster [DBG] pgmap v2920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:40 smithi006 bash[27292]: cluster 2023-08-27T23:35:39.286651+0000 mgr.y (mgr.14901) 2913 : cluster [DBG] pgmap v2920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:42.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:41 smithi130 bash[27579]: audit 2023-08-27T23:35:41.078897+0000 mon.a (mon.0) 2400 : audit [DBG] from='client.? 172.21.15.6:0/3370464246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:42.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:41 smithi130 bash[51302]: debug 2023-08-27T23:35:41.620+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:41.625300+0000) 2023-08-27T23:35:42.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:41 smithi006 bash[33918]: audit 2023-08-27T23:35:41.078897+0000 mon.a (mon.0) 2400 : audit [DBG] from='client.? 172.21.15.6:0/3370464246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:42.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:41 smithi006 bash[27292]: audit 2023-08-27T23:35:41.078897+0000 mon.a (mon.0) 2400 : audit [DBG] from='client.? 172.21.15.6:0/3370464246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:42.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:35:42.073Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:42.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:35:42.075Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:43.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:42 smithi130 bash[27579]: cluster 2023-08-27T23:35:41.287331+0000 mgr.y (mgr.14901) 2914 : cluster [DBG] pgmap v2921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:43.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:42 smithi130 bash[51302]: debug 2023-08-27T23:35:42.620+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:42.625571+0000) 2023-08-27T23:35:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:42 smithi006 bash[27292]: cluster 2023-08-27T23:35:41.287331+0000 mgr.y (mgr.14901) 2914 : cluster [DBG] pgmap v2921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:42 smithi006 bash[33918]: cluster 2023-08-27T23:35:41.287331+0000 mgr.y (mgr.14901) 2914 : cluster [DBG] pgmap v2921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:44.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:43 smithi130 bash[27579]: audit 2023-08-27T23:35:43.496302+0000 mon.a (mon.0) 2401 : audit [DBG] from='client.? 172.21.15.6:0/718974584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:44.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:43 smithi130 bash[51302]: debug 2023-08-27T23:35:43.620+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:43.625861+0000) 2023-08-27T23:35:44.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:43 smithi006 bash[33918]: audit 2023-08-27T23:35:43.496302+0000 mon.a (mon.0) 2401 : audit [DBG] from='client.? 172.21.15.6:0/718974584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:44.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:43 smithi006 bash[27292]: audit 2023-08-27T23:35:43.496302+0000 mon.a (mon.0) 2401 : audit [DBG] from='client.? 172.21.15.6:0/718974584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:44.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:35:43.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:44.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:35:43.975Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:44 smithi130 bash[27579]: cluster 2023-08-27T23:35:43.287983+0000 mgr.y (mgr.14901) 2915 : cluster [DBG] pgmap v2922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:44 smithi130 bash[51302]: debug 2023-08-27T23:35:44.620+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:44.626143+0000) 2023-08-27T23:35:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:44 smithi006 bash[33918]: cluster 2023-08-27T23:35:43.287983+0000 mgr.y (mgr.14901) 2915 : cluster [DBG] pgmap v2922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:44 smithi006 bash[27292]: cluster 2023-08-27T23:35:43.287983+0000 mgr.y (mgr.14901) 2915 : cluster [DBG] pgmap v2922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:45.883 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:45 smithi130 bash[51302]: debug 2023-08-27T23:35:45.620+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:45.626463+0000) 2023-08-27T23:35:46.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:45 smithi006 bash[27292]: cluster 2023-08-27T23:35:45.289175+0000 mgr.y (mgr.14901) 2916 : cluster [DBG] pgmap v2923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:46.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:45 smithi006 bash[33918]: cluster 2023-08-27T23:35:45.289175+0000 mgr.y (mgr.14901) 2916 : cluster [DBG] pgmap v2923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:46.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:45 smithi130 bash[27579]: cluster 2023-08-27T23:35:45.289175+0000 mgr.y (mgr.14901) 2916 : cluster [DBG] pgmap v2923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:46.887 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:46 smithi130 bash[51302]: debug 2023-08-27T23:35:46.620+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:46.626763+0000) 2023-08-27T23:35:47.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:46 smithi006 bash[27292]: audit 2023-08-27T23:35:45.910557+0000 mon.c (mon.2) 1130 : audit [DBG] from='client.? 172.21.15.6:0/3638925150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:47.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:46 smithi006 bash[33918]: audit 2023-08-27T23:35:45.910557+0000 mon.c (mon.2) 1130 : audit [DBG] from='client.? 172.21.15.6:0/3638925150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:47.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:46 smithi130 bash[27579]: audit 2023-08-27T23:35:45.910557+0000 mon.c (mon.2) 1130 : audit [DBG] from='client.? 172.21.15.6:0/3638925150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:47.894 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:47 smithi130 bash[51302]: debug 2023-08-27T23:35:47.624+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:47.627033+0000) 2023-08-27T23:35:48.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:47 smithi006 bash[27292]: cluster 2023-08-27T23:35:47.289839+0000 mgr.y (mgr.14901) 2917 : cluster [DBG] pgmap v2924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:48.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:47 smithi006 bash[33918]: cluster 2023-08-27T23:35:47.289839+0000 mgr.y (mgr.14901) 2917 : cluster [DBG] pgmap v2924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:48.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:47 smithi130 bash[27579]: cluster 2023-08-27T23:35:47.289839+0000 mgr.y (mgr.14901) 2917 : cluster [DBG] pgmap v2924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:48.897 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:48 smithi130 bash[51302]: debug 2023-08-27T23:35:48.624+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:48.627320+0000) 2023-08-27T23:35:49.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:48 smithi006 bash[33918]: audit 2023-08-27T23:35:48.324710+0000 mon.c (mon.2) 1131 : audit [DBG] from='client.? 172.21.15.6:0/373002161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:49.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:48 smithi006 bash[27292]: audit 2023-08-27T23:35:48.324710+0000 mon.c (mon.2) 1131 : audit [DBG] from='client.? 172.21.15.6:0/373002161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:49.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:48 smithi130 bash[27579]: audit 2023-08-27T23:35:48.324710+0000 mon.c (mon.2) 1131 : audit [DBG] from='client.? 172.21.15.6:0/373002161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:50.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:49 smithi130 bash[51302]: debug 2023-08-27T23:35:49.624+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:49.627551+0000) 2023-08-27T23:35:50.843 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:35:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:35:50] "GET /metrics HTTP/1.1" 200 193089 "" "Prometheus/2.33.4" 2023-08-27T23:35:50.844 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:35:50.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:50.844 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:35:50.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:50 smithi130 bash[27579]: cluster 2023-08-27T23:35:49.290902+0000 mgr.y (mgr.14901) 2918 : cluster [DBG] pgmap v2925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:50 smithi130 bash[27579]: audit 2023-08-27T23:35:50.737166+0000 mon.a (mon.0) 2402 : audit [DBG] from='client.? 172.21.15.6:0/2271020791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:51.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:50 smithi130 bash[51302]: debug 2023-08-27T23:35:50.624+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:50.627852+0000) 2023-08-27T23:35:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:50 smithi006 bash[33918]: cluster 2023-08-27T23:35:49.290902+0000 mgr.y (mgr.14901) 2918 : cluster [DBG] pgmap v2925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:50 smithi006 bash[33918]: audit 2023-08-27T23:35:50.737166+0000 mon.a (mon.0) 2402 : audit [DBG] from='client.? 172.21.15.6:0/2271020791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:50 smithi006 bash[27292]: cluster 2023-08-27T23:35:49.290902+0000 mgr.y (mgr.14901) 2918 : cluster [DBG] pgmap v2925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:50 smithi006 bash[27292]: audit 2023-08-27T23:35:50.737166+0000 mon.a (mon.0) 2402 : audit [DBG] from='client.? 172.21.15.6:0/2271020791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:52.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:51 smithi130 bash[51302]: debug 2023-08-27T23:35:51.624+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:51.628108+0000) 2023-08-27T23:35:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:35:52.074Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:35:52.076Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:53.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:52 smithi130 bash[27579]: cluster 2023-08-27T23:35:51.291591+0000 mgr.y (mgr.14901) 2919 : cluster [DBG] pgmap v2926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:52 smithi130 bash[51302]: debug 2023-08-27T23:35:52.624+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:52.628471+0000) 2023-08-27T23:35:53.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:52 smithi006 bash[33918]: cluster 2023-08-27T23:35:51.291591+0000 mgr.y (mgr.14901) 2919 : cluster [DBG] pgmap v2926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:52 smithi006 bash[27292]: cluster 2023-08-27T23:35:51.291591+0000 mgr.y (mgr.14901) 2919 : cluster [DBG] pgmap v2926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:54.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:53 smithi130 bash[51302]: debug 2023-08-27T23:35:53.624+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:53.628737+0000) 2023-08-27T23:35:54.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:53 smithi130 bash[27579]: audit 2023-08-27T23:35:53.153622+0000 mon.c (mon.2) 1132 : audit [DBG] from='client.? 172.21.15.6:0/2284767180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:54.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:53 smithi006 bash[33918]: audit 2023-08-27T23:35:53.153622+0000 mon.c (mon.2) 1132 : audit [DBG] from='client.? 172.21.15.6:0/2284767180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:54.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:53 smithi006 bash[27292]: audit 2023-08-27T23:35:53.153622+0000 mon.c (mon.2) 1132 : audit [DBG] from='client.? 172.21.15.6:0/2284767180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:54.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:35:53.974Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:54.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:35:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:35:53.976Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:35:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:54 smithi130 bash[51302]: debug 2023-08-27T23:35:54.624+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:54.629014+0000) 2023-08-27T23:35:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:54 smithi130 bash[27579]: cluster 2023-08-27T23:35:53.292281+0000 mgr.y (mgr.14901) 2920 : cluster [DBG] pgmap v2927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:54 smithi006 bash[27292]: cluster 2023-08-27T23:35:53.292281+0000 mgr.y (mgr.14901) 2920 : cluster [DBG] pgmap v2927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:55.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:54 smithi006 bash[33918]: cluster 2023-08-27T23:35:53.292281+0000 mgr.y (mgr.14901) 2920 : cluster [DBG] pgmap v2927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:56.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:55 smithi130 bash[27579]: audit 2023-08-27T23:35:55.567446+0000 mon.c (mon.2) 1133 : audit [DBG] from='client.? 172.21.15.6:0/944920607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:56.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:55 smithi130 bash[51302]: debug 2023-08-27T23:35:55.624+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:55.629303+0000) 2023-08-27T23:35:56.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:55 smithi006 bash[27292]: audit 2023-08-27T23:35:55.567446+0000 mon.c (mon.2) 1133 : audit [DBG] from='client.? 172.21.15.6:0/944920607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:56.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:55 smithi006 bash[33918]: audit 2023-08-27T23:35:55.567446+0000 mon.c (mon.2) 1133 : audit [DBG] from='client.? 172.21.15.6:0/944920607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:56 smithi130 bash[27579]: cluster 2023-08-27T23:35:55.293436+0000 mgr.y (mgr.14901) 2921 : cluster [DBG] pgmap v2928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:57.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:56 smithi130 bash[51302]: debug 2023-08-27T23:35:56.624+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:56.629587+0000) 2023-08-27T23:35:57.223 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:56 smithi006 bash[27292]: cluster 2023-08-27T23:35:55.293436+0000 mgr.y (mgr.14901) 2921 : cluster [DBG] pgmap v2928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:57.223 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:56 smithi006 bash[33918]: cluster 2023-08-27T23:35:55.293436+0000 mgr.y (mgr.14901) 2921 : cluster [DBG] pgmap v2928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:58.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:57 smithi130 bash[51302]: debug 2023-08-27T23:35:57.624+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:57.629850+0000) 2023-08-27T23:35:59.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:58 smithi130 bash[51302]: debug 2023-08-27T23:35:58.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:58.630125+0000) 2023-08-27T23:35:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:58 smithi130 bash[27579]: cluster 2023-08-27T23:35:57.294107+0000 mgr.y (mgr.14901) 2922 : cluster [DBG] pgmap v2929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:58 smithi130 bash[27579]: audit 2023-08-27T23:35:57.982627+0000 mon.a (mon.0) 2403 : audit [DBG] from='client.? 172.21.15.6:0/2628924432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:35:58 smithi130 bash[27579]: audit 2023-08-27T23:35:58.842922+0000 mon.b (mon.1) 397 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:35:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:58 smithi006 bash[33918]: cluster 2023-08-27T23:35:57.294107+0000 mgr.y (mgr.14901) 2922 : cluster [DBG] pgmap v2929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:58 smithi006 bash[33918]: audit 2023-08-27T23:35:57.982627+0000 mon.a (mon.0) 2403 : audit [DBG] from='client.? 172.21.15.6:0/2628924432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:35:58 smithi006 bash[33918]: audit 2023-08-27T23:35:58.842922+0000 mon.b (mon.1) 397 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:35:59.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:58 smithi006 bash[27292]: cluster 2023-08-27T23:35:57.294107+0000 mgr.y (mgr.14901) 2922 : cluster [DBG] pgmap v2929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:35:59.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:58 smithi006 bash[27292]: audit 2023-08-27T23:35:57.982627+0000 mon.a (mon.0) 2403 : audit [DBG] from='client.? 172.21.15.6:0/2628924432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:59.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:35:58 smithi006 bash[27292]: audit 2023-08-27T23:35:58.842922+0000 mon.b (mon.1) 397 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:36:00.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:35:59 smithi130 bash[51302]: debug 2023-08-27T23:35:59.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:59.630399+0000) 2023-08-27T23:36:00.846 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:36:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:36:00] "GET /metrics HTTP/1.1" 200 193097 "" "Prometheus/2.33.4" 2023-08-27T23:36:00.847 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:36:00.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:00.847 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:36:00.526Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:00.849 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:00 smithi006 bash[27292]: cluster 2023-08-27T23:35:59.295262+0000 mgr.y (mgr.14901) 2923 : cluster [DBG] pgmap v2930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:00.849 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:00 smithi006 bash[27292]: audit 2023-08-27T23:36:00.408612+0000 mon.c (mon.2) 1134 : audit [DBG] from='client.? 172.21.15.6:0/1617213775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:01.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:00 smithi130 bash[51302]: debug 2023-08-27T23:36:00.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:00.630711+0000) 2023-08-27T23:36:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:00 smithi130 bash[27579]: cluster 2023-08-27T23:35:59.295262+0000 mgr.y (mgr.14901) 2923 : cluster [DBG] pgmap v2930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:00 smithi130 bash[27579]: audit 2023-08-27T23:36:00.408612+0000 mon.c (mon.2) 1134 : audit [DBG] from='client.? 172.21.15.6:0/1617213775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:00 smithi006 bash[33918]: cluster 2023-08-27T23:35:59.295262+0000 mgr.y (mgr.14901) 2923 : cluster [DBG] pgmap v2930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:00 smithi006 bash[33918]: audit 2023-08-27T23:36:00.408612+0000 mon.c (mon.2) 1134 : audit [DBG] from='client.? 172.21.15.6:0/1617213775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:02.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:01 smithi130 bash[51302]: debug 2023-08-27T23:36:01.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:01.630963+0000) 2023-08-27T23:36:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:36:02.075Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:36:02.078Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:02 smithi130 bash[51302]: debug 2023-08-27T23:36:02.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:02.631229+0000) 2023-08-27T23:36:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:02 smithi130 bash[27579]: cluster 2023-08-27T23:36:01.295892+0000 mgr.y (mgr.14901) 2924 : cluster [DBG] pgmap v2931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:02 smithi130 bash[27579]: audit 2023-08-27T23:36:02.828227+0000 mon.a (mon.0) 2404 : audit [DBG] from='client.? 172.21.15.6:0/3332057738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:02 smithi006 bash[27292]: cluster 2023-08-27T23:36:01.295892+0000 mgr.y (mgr.14901) 2924 : cluster [DBG] pgmap v2931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:02 smithi006 bash[27292]: audit 2023-08-27T23:36:02.828227+0000 mon.a (mon.0) 2404 : audit [DBG] from='client.? 172.21.15.6:0/3332057738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:02 smithi006 bash[33918]: cluster 2023-08-27T23:36:01.295892+0000 mgr.y (mgr.14901) 2924 : cluster [DBG] pgmap v2931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:02 smithi006 bash[33918]: audit 2023-08-27T23:36:02.828227+0000 mon.a (mon.0) 2404 : audit [DBG] from='client.? 172.21.15.6:0/3332057738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:04.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:03 smithi130 bash[51302]: debug 2023-08-27T23:36:03.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:03.631539+0000) 2023-08-27T23:36:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:36:03.974Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:36:03.976Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:04 smithi130 bash[27579]: cluster 2023-08-27T23:36:03.296612+0000 mgr.y (mgr.14901) 2925 : cluster [DBG] pgmap v2932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:04 smithi130 bash[27579]: audit 2023-08-27T23:36:04.770330+0000 mon.a (mon.0) 2405 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:36:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:04 smithi130 bash[51302]: debug 2023-08-27T23:36:04.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:04.631738+0000) 2023-08-27T23:36:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:04 smithi006 bash[27292]: cluster 2023-08-27T23:36:03.296612+0000 mgr.y (mgr.14901) 2925 : cluster [DBG] pgmap v2932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:04 smithi006 bash[27292]: audit 2023-08-27T23:36:04.770330+0000 mon.a (mon.0) 2405 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:36:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:04 smithi006 bash[33918]: cluster 2023-08-27T23:36:03.296612+0000 mgr.y (mgr.14901) 2925 : cluster [DBG] pgmap v2932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:05.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:04 smithi006 bash[33918]: audit 2023-08-27T23:36:04.770330+0000 mon.a (mon.0) 2405 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:36:06.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:05 smithi130 bash[51302]: debug 2023-08-27T23:36:05.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:05.632000+0000) 2023-08-27T23:36:06.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:06 smithi006 bash[27292]: audit 2023-08-27T23:36:05.142299+0000 mon.a (mon.0) 2406 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:36:06.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:06 smithi006 bash[27292]: audit 2023-08-27T23:36:05.247604+0000 mon.a (mon.0) 2407 : audit [DBG] from='client.? 172.21.15.6:0/1618438021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:06.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:06 smithi006 bash[27292]: cluster 2023-08-27T23:36:05.297822+0000 mgr.y (mgr.14901) 2926 : cluster [DBG] pgmap v2933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:06.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:06 smithi006 bash[27292]: audit 2023-08-27T23:36:05.439244+0000 mon.b (mon.1) 398 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:36:06.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:06 smithi006 bash[27292]: audit 2023-08-27T23:36:05.441503+0000 mon.b (mon.1) 399 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:36:06.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:06 smithi006 bash[27292]: audit 2023-08-27T23:36:05.449684+0000 mon.a (mon.0) 2408 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:36:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:06 smithi006 bash[33918]: audit 2023-08-27T23:36:05.142299+0000 mon.a (mon.0) 2406 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:36:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:06 smithi006 bash[33918]: audit 2023-08-27T23:36:05.247604+0000 mon.a (mon.0) 2407 : audit [DBG] from='client.? 172.21.15.6:0/1618438021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:06 smithi006 bash[33918]: cluster 2023-08-27T23:36:05.297822+0000 mgr.y (mgr.14901) 2926 : cluster [DBG] pgmap v2933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:06 smithi006 bash[33918]: audit 2023-08-27T23:36:05.439244+0000 mon.b (mon.1) 398 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:36:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:06 smithi006 bash[33918]: audit 2023-08-27T23:36:05.441503+0000 mon.b (mon.1) 399 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:36:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:06 smithi006 bash[33918]: audit 2023-08-27T23:36:05.449684+0000 mon.a (mon.0) 2408 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:36:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:06 smithi130 bash[27579]: audit 2023-08-27T23:36:05.142299+0000 mon.a (mon.0) 2406 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:36:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:06 smithi130 bash[27579]: audit 2023-08-27T23:36:05.247604+0000 mon.a (mon.0) 2407 : audit [DBG] from='client.? 172.21.15.6:0/1618438021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:06 smithi130 bash[27579]: cluster 2023-08-27T23:36:05.297822+0000 mgr.y (mgr.14901) 2926 : cluster [DBG] pgmap v2933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:06 smithi130 bash[27579]: audit 2023-08-27T23:36:05.439244+0000 mon.b (mon.1) 398 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:36:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:06 smithi130 bash[27579]: audit 2023-08-27T23:36:05.441503+0000 mon.b (mon.1) 399 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:36:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:06 smithi130 bash[27579]: audit 2023-08-27T23:36:05.449684+0000 mon.a (mon.0) 2408 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:36:07.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:06 smithi130 bash[51302]: debug 2023-08-27T23:36:06.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:06.632243+0000) 2023-08-27T23:36:08.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:07 smithi130 bash[51302]: debug 2023-08-27T23:36:07.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:07.632545+0000) 2023-08-27T23:36:08.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:08 smithi130 bash[27579]: cluster 2023-08-27T23:36:07.298466+0000 mgr.y (mgr.14901) 2927 : cluster [DBG] pgmap v2934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:08.626 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:08 smithi130 bash[27579]: audit 2023-08-27T23:36:07.659187+0000 mon.a (mon.0) 2409 : audit [DBG] from='client.? 172.21.15.6:0/3413224387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:08 smithi006 bash[33918]: cluster 2023-08-27T23:36:07.298466+0000 mgr.y (mgr.14901) 2927 : cluster [DBG] pgmap v2934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:08 smithi006 bash[33918]: audit 2023-08-27T23:36:07.659187+0000 mon.a (mon.0) 2409 : audit [DBG] from='client.? 172.21.15.6:0/3413224387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:08 smithi006 bash[27292]: cluster 2023-08-27T23:36:07.298466+0000 mgr.y (mgr.14901) 2927 : cluster [DBG] pgmap v2934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:08 smithi006 bash[27292]: audit 2023-08-27T23:36:07.659187+0000 mon.a (mon.0) 2409 : audit [DBG] from='client.? 172.21.15.6:0/3413224387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:09.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:08 smithi130 bash[51302]: debug 2023-08-27T23:36:08.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:08.632839+0000) 2023-08-27T23:36:10.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:09 smithi130 bash[51302]: debug 2023-08-27T23:36:09.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:09.633136+0000) 2023-08-27T23:36:10.849 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:36:10.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:10.865 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:36:10.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:10.866 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:36:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:36:10] "GET /metrics HTTP/1.1" 200 193097 "" "Prometheus/2.33.4" 2023-08-27T23:36:11.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:10 smithi130 bash[51302]: debug 2023-08-27T23:36:10.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:10.633465+0000) 2023-08-27T23:36:11.270 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:10 smithi130 bash[27579]: cluster 2023-08-27T23:36:09.299547+0000 mgr.y (mgr.14901) 2928 : cluster [DBG] pgmap v2935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:11.271 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:10 smithi130 bash[27579]: audit 2023-08-27T23:36:10.069889+0000 mon.c (mon.2) 1135 : audit [DBG] from='client.? 172.21.15.6:0/3335677537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:11.271 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:10 smithi006 bash[33918]: cluster 2023-08-27T23:36:09.299547+0000 mgr.y (mgr.14901) 2928 : cluster [DBG] pgmap v2935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:11.271 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:10 smithi006 bash[33918]: audit 2023-08-27T23:36:10.069889+0000 mon.c (mon.2) 1135 : audit [DBG] from='client.? 172.21.15.6:0/3335677537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:11.272 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:10 smithi006 bash[27292]: cluster 2023-08-27T23:36:09.299547+0000 mgr.y (mgr.14901) 2928 : cluster [DBG] pgmap v2935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:11.272 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:10 smithi006 bash[27292]: audit 2023-08-27T23:36:10.069889+0000 mon.c (mon.2) 1135 : audit [DBG] from='client.? 172.21.15.6:0/3335677537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:12.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:11 smithi130 bash[51302]: debug 2023-08-27T23:36:11.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:11.633777+0000) 2023-08-27T23:36:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:36:12.075Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:36:12.078Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:12 smithi130 bash[27579]: cluster 2023-08-27T23:36:11.300220+0000 mgr.y (mgr.14901) 2929 : cluster [DBG] pgmap v2936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:12 smithi130 bash[27579]: audit 2023-08-27T23:36:12.484868+0000 mon.a (mon.0) 2410 : audit [DBG] from='client.? 172.21.15.6:0/4087138494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:13.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:12 smithi130 bash[51302]: debug 2023-08-27T23:36:12.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:12.634069+0000) 2023-08-27T23:36:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:12 smithi006 bash[27292]: cluster 2023-08-27T23:36:11.300220+0000 mgr.y (mgr.14901) 2929 : cluster [DBG] pgmap v2936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:12 smithi006 bash[27292]: audit 2023-08-27T23:36:12.484868+0000 mon.a (mon.0) 2410 : audit [DBG] from='client.? 172.21.15.6:0/4087138494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:13.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:12 smithi006 bash[33918]: cluster 2023-08-27T23:36:11.300220+0000 mgr.y (mgr.14901) 2929 : cluster [DBG] pgmap v2936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:13.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:12 smithi006 bash[33918]: audit 2023-08-27T23:36:12.484868+0000 mon.a (mon.0) 2410 : audit [DBG] from='client.? 172.21.15.6:0/4087138494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:14.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:13 smithi130 bash[51302]: debug 2023-08-27T23:36:13.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:13.634332+0000) 2023-08-27T23:36:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:36:13.976Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:36:13.978Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:14 smithi130 bash[27579]: cluster 2023-08-27T23:36:13.300904+0000 mgr.y (mgr.14901) 2930 : cluster [DBG] pgmap v2937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:15.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:14 smithi130 bash[51302]: debug 2023-08-27T23:36:14.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:14.634562+0000) 2023-08-27T23:36:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:14 smithi006 bash[33918]: cluster 2023-08-27T23:36:13.300904+0000 mgr.y (mgr.14901) 2930 : cluster [DBG] pgmap v2937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:14 smithi006 bash[27292]: cluster 2023-08-27T23:36:13.300904+0000 mgr.y (mgr.14901) 2930 : cluster [DBG] pgmap v2937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:16.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:15 smithi130 bash[27579]: audit 2023-08-27T23:36:14.895645+0000 mon.a (mon.0) 2411 : audit [DBG] from='client.? 172.21.15.6:0/2855617803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:16.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:15 smithi130 bash[51302]: debug 2023-08-27T23:36:15.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:15.634822+0000) 2023-08-27T23:36:16.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:15 smithi006 bash[33918]: audit 2023-08-27T23:36:14.895645+0000 mon.a (mon.0) 2411 : audit [DBG] from='client.? 172.21.15.6:0/2855617803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:16.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:15 smithi006 bash[27292]: audit 2023-08-27T23:36:14.895645+0000 mon.a (mon.0) 2411 : audit [DBG] from='client.? 172.21.15.6:0/2855617803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:16 smithi130 bash[27579]: cluster 2023-08-27T23:36:15.302079+0000 mgr.y (mgr.14901) 2931 : cluster [DBG] pgmap v2938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:17.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:16 smithi130 bash[51302]: debug 2023-08-27T23:36:16.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:16.635057+0000) 2023-08-27T23:36:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:16 smithi006 bash[33918]: cluster 2023-08-27T23:36:15.302079+0000 mgr.y (mgr.14901) 2931 : cluster [DBG] pgmap v2938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:17.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:16 smithi006 bash[27292]: cluster 2023-08-27T23:36:15.302079+0000 mgr.y (mgr.14901) 2931 : cluster [DBG] pgmap v2938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:18.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:17 smithi130 bash[27579]: audit 2023-08-27T23:36:17.303134+0000 mon.c (mon.2) 1136 : audit [DBG] from='client.? 172.21.15.6:0/4157134579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:18.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:17 smithi130 bash[51302]: debug 2023-08-27T23:36:17.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:17.635354+0000) 2023-08-27T23:36:18.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:17 smithi006 bash[27292]: audit 2023-08-27T23:36:17.303134+0000 mon.c (mon.2) 1136 : audit [DBG] from='client.? 172.21.15.6:0/4157134579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:18.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:17 smithi006 bash[33918]: audit 2023-08-27T23:36:17.303134+0000 mon.c (mon.2) 1136 : audit [DBG] from='client.? 172.21.15.6:0/4157134579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:18 smithi130 bash[27579]: cluster 2023-08-27T23:36:17.302921+0000 mgr.y (mgr.14901) 2932 : cluster [DBG] pgmap v2939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:19.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:18 smithi130 bash[51302]: debug 2023-08-27T23:36:18.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:18.635675+0000) 2023-08-27T23:36:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:18 smithi006 bash[27292]: cluster 2023-08-27T23:36:17.302921+0000 mgr.y (mgr.14901) 2932 : cluster [DBG] pgmap v2939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:18 smithi006 bash[33918]: cluster 2023-08-27T23:36:17.302921+0000 mgr.y (mgr.14901) 2932 : cluster [DBG] pgmap v2939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:19.887 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:19 smithi130 bash[51302]: debug 2023-08-27T23:36:19.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:19.635951+0000) 2023-08-27T23:36:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:19 smithi006 bash[27292]: audit 2023-08-27T23:36:19.711048+0000 mon.a (mon.0) 2412 : audit [DBG] from='client.? 172.21.15.6:0/2800140124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:20.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:19 smithi006 bash[33918]: audit 2023-08-27T23:36:19.711048+0000 mon.a (mon.0) 2412 : audit [DBG] from='client.? 172.21.15.6:0/2800140124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:20.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:19 smithi130 bash[27579]: audit 2023-08-27T23:36:19.711048+0000 mon.a (mon.0) 2412 : audit [DBG] from='client.? 172.21.15.6:0/2800140124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:20.888 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:36:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:36:20] "GET /metrics HTTP/1.1" 200 193096 "" "Prometheus/2.33.4" 2023-08-27T23:36:20.888 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:36:20.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:20.889 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:36:20.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:20.889 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:20 smithi130 bash[51302]: debug 2023-08-27T23:36:20.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:20.636231+0000) 2023-08-27T23:36:21.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:20 smithi006 bash[33918]: cluster 2023-08-27T23:36:19.304062+0000 mgr.y (mgr.14901) 2933 : cluster [DBG] pgmap v2940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:20 smithi006 bash[27292]: cluster 2023-08-27T23:36:19.304062+0000 mgr.y (mgr.14901) 2933 : cluster [DBG] pgmap v2940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:21.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:20 smithi130 bash[27579]: cluster 2023-08-27T23:36:19.304062+0000 mgr.y (mgr.14901) 2933 : cluster [DBG] pgmap v2940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:21.897 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:21 smithi130 bash[51302]: debug 2023-08-27T23:36:21.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:21.636488+0000) 2023-08-27T23:36:22.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:21 smithi006 bash[27292]: cluster 2023-08-27T23:36:21.304745+0000 mgr.y (mgr.14901) 2934 : cluster [DBG] pgmap v2941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:22.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:21 smithi006 bash[33918]: cluster 2023-08-27T23:36:21.304745+0000 mgr.y (mgr.14901) 2934 : cluster [DBG] pgmap v2941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:22.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:36:22.077Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:22.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:36:22.078Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:22.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:21 smithi130 bash[27579]: cluster 2023-08-27T23:36:21.304745+0000 mgr.y (mgr.14901) 2934 : cluster [DBG] pgmap v2941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:22.900 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:22 smithi130 bash[51302]: debug 2023-08-27T23:36:22.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:22.636731+0000) 2023-08-27T23:36:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:22 smithi006 bash[27292]: audit 2023-08-27T23:36:22.123934+0000 mon.a (mon.0) 2413 : audit [DBG] from='client.? 172.21.15.6:0/1254350282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:22 smithi006 bash[33918]: audit 2023-08-27T23:36:22.123934+0000 mon.a (mon.0) 2413 : audit [DBG] from='client.? 172.21.15.6:0/1254350282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:23.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:22 smithi130 bash[27579]: audit 2023-08-27T23:36:22.123934+0000 mon.a (mon.0) 2413 : audit [DBG] from='client.? 172.21.15.6:0/1254350282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:23.910 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:23 smithi130 bash[51302]: debug 2023-08-27T23:36:23.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:23.637027+0000) 2023-08-27T23:36:24.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:23 smithi006 bash[33918]: cluster 2023-08-27T23:36:23.305445+0000 mgr.y (mgr.14901) 2935 : cluster [DBG] pgmap v2942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:24.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:23 smithi006 bash[27292]: cluster 2023-08-27T23:36:23.305445+0000 mgr.y (mgr.14901) 2935 : cluster [DBG] pgmap v2942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:24.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:36:23.977Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:24.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:36:23.979Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:24.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:23 smithi130 bash[27579]: cluster 2023-08-27T23:36:23.305445+0000 mgr.y (mgr.14901) 2935 : cluster [DBG] pgmap v2942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:24.913 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:24 smithi130 bash[51302]: debug 2023-08-27T23:36:24.636+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:24.637301+0000) 2023-08-27T23:36:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:24 smithi006 bash[33918]: audit 2023-08-27T23:36:24.538863+0000 mon.c (mon.2) 1137 : audit [DBG] from='client.? 172.21.15.6:0/2918225668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:25.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:24 smithi006 bash[27292]: audit 2023-08-27T23:36:24.538863+0000 mon.c (mon.2) 1137 : audit [DBG] from='client.? 172.21.15.6:0/2918225668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:24 smithi130 bash[27579]: audit 2023-08-27T23:36:24.538863+0000 mon.c (mon.2) 1137 : audit [DBG] from='client.? 172.21.15.6:0/2918225668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:25.923 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:25 smithi130 bash[51302]: debug 2023-08-27T23:36:25.636+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:25.637578+0000) 2023-08-27T23:36:26.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:25 smithi006 bash[33918]: cluster 2023-08-27T23:36:25.306608+0000 mgr.y (mgr.14901) 2936 : cluster [DBG] pgmap v2943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:26.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:25 smithi006 bash[27292]: cluster 2023-08-27T23:36:25.306608+0000 mgr.y (mgr.14901) 2936 : cluster [DBG] pgmap v2943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:26.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:25 smithi130 bash[27579]: cluster 2023-08-27T23:36:25.306608+0000 mgr.y (mgr.14901) 2936 : cluster [DBG] pgmap v2943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:27.011 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:26 smithi130 bash[51302]: debug 2023-08-27T23:36:26.631+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:26.637840+0000) 2023-08-27T23:36:27.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:27 smithi130 bash[27579]: audit 2023-08-27T23:36:26.954019+0000 mon.c (mon.2) 1138 : audit [DBG] from='client.? 172.21.15.6:0/333936249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:27.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:27 smithi006 bash[33918]: audit 2023-08-27T23:36:26.954019+0000 mon.c (mon.2) 1138 : audit [DBG] from='client.? 172.21.15.6:0/333936249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:27.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:27 smithi006 bash[27292]: audit 2023-08-27T23:36:26.954019+0000 mon.c (mon.2) 1138 : audit [DBG] from='client.? 172.21.15.6:0/333936249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:28.020 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:27 smithi130 bash[51302]: debug 2023-08-27T23:36:27.635+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:27.638114+0000) 2023-08-27T23:36:28.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:28 smithi130 bash[27579]: cluster 2023-08-27T23:36:27.307412+0000 mgr.y (mgr.14901) 2937 : cluster [DBG] pgmap v2944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:28.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:28 smithi006 bash[33918]: cluster 2023-08-27T23:36:27.307412+0000 mgr.y (mgr.14901) 2937 : cluster [DBG] pgmap v2944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:28.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:28 smithi006 bash[27292]: cluster 2023-08-27T23:36:27.307412+0000 mgr.y (mgr.14901) 2937 : cluster [DBG] pgmap v2944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:29.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:28 smithi130 bash[51302]: debug 2023-08-27T23:36:28.635+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:28.638397+0000) 2023-08-27T23:36:30.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:29 smithi130 bash[51302]: debug 2023-08-27T23:36:29.631+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:29.638668+0000) 2023-08-27T23:36:30.642 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:30 smithi130 bash[27579]: cluster 2023-08-27T23:36:29.308525+0000 mgr.y (mgr.14901) 2938 : cluster [DBG] pgmap v2945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:30.642 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:30 smithi130 bash[27579]: audit 2023-08-27T23:36:29.370339+0000 mon.c (mon.2) 1139 : audit [DBG] from='client.? 172.21.15.6:0/4168482278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:30 smithi006 bash[33918]: cluster 2023-08-27T23:36:29.308525+0000 mgr.y (mgr.14901) 2938 : cluster [DBG] pgmap v2945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:30 smithi006 bash[33918]: audit 2023-08-27T23:36:29.370339+0000 mon.c (mon.2) 1139 : audit [DBG] from='client.? 172.21.15.6:0/4168482278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:30 smithi006 bash[27292]: cluster 2023-08-27T23:36:29.308525+0000 mgr.y (mgr.14901) 2938 : cluster [DBG] pgmap v2945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:30.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:30 smithi006 bash[27292]: audit 2023-08-27T23:36:29.370339+0000 mon.c (mon.2) 1139 : audit [DBG] from='client.? 172.21.15.6:0/4168482278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:30.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:36:30.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:30.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:36:30.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:31.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:30 smithi130 bash[51302]: debug 2023-08-27T23:36:30.635+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:30.639027+0000) 2023-08-27T23:36:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:36:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:36:30] "GET /metrics HTTP/1.1" 200 193096 "" "Prometheus/2.33.4" 2023-08-27T23:36:32.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:31 smithi130 bash[51302]: debug 2023-08-27T23:36:31.635+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:31.639290+0000) 2023-08-27T23:36:32.372 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:36:32.077Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:32.373 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:36:32.079Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:32.640 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:32 smithi130 bash[27579]: cluster 2023-08-27T23:36:31.309143+0000 mgr.y (mgr.14901) 2939 : cluster [DBG] pgmap v2946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:32.640 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:32 smithi130 bash[27579]: audit 2023-08-27T23:36:31.779741+0000 mon.c (mon.2) 1140 : audit [DBG] from='client.? 172.21.15.6:0/2320958821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:32 smithi006 bash[33918]: cluster 2023-08-27T23:36:31.309143+0000 mgr.y (mgr.14901) 2939 : cluster [DBG] pgmap v2946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:32 smithi006 bash[33918]: audit 2023-08-27T23:36:31.779741+0000 mon.c (mon.2) 1140 : audit [DBG] from='client.? 172.21.15.6:0/2320958821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:32.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:32 smithi006 bash[27292]: cluster 2023-08-27T23:36:31.309143+0000 mgr.y (mgr.14901) 2939 : cluster [DBG] pgmap v2946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:32.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:32 smithi006 bash[27292]: audit 2023-08-27T23:36:31.779741+0000 mon.c (mon.2) 1140 : audit [DBG] from='client.? 172.21.15.6:0/2320958821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:33.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:32 smithi130 bash[51302]: debug 2023-08-27T23:36:32.635+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:32.639569+0000) 2023-08-27T23:36:34.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:33 smithi130 bash[51302]: debug 2023-08-27T23:36:33.635+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:33.639854+0000) 2023-08-27T23:36:34.377 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:36:33.977Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:34.378 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:36:33.980Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:34.641 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:34 smithi130 bash[27579]: cluster 2023-08-27T23:36:33.309725+0000 mgr.y (mgr.14901) 2940 : cluster [DBG] pgmap v2947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:34.641 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:34 smithi130 bash[27579]: audit 2023-08-27T23:36:34.207211+0000 mon.a (mon.0) 2414 : audit [DBG] from='client.? 172.21.15.6:0/1180763524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:34 smithi006 bash[33918]: cluster 2023-08-27T23:36:33.309725+0000 mgr.y (mgr.14901) 2940 : cluster [DBG] pgmap v2947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:34 smithi006 bash[33918]: audit 2023-08-27T23:36:34.207211+0000 mon.a (mon.0) 2414 : audit [DBG] from='client.? 172.21.15.6:0/1180763524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:34 smithi006 bash[27292]: cluster 2023-08-27T23:36:33.309725+0000 mgr.y (mgr.14901) 2940 : cluster [DBG] pgmap v2947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:34 smithi006 bash[27292]: audit 2023-08-27T23:36:34.207211+0000 mon.a (mon.0) 2414 : audit [DBG] from='client.? 172.21.15.6:0/1180763524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:35.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:34 smithi130 bash[51302]: debug 2023-08-27T23:36:34.639+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:34.640170+0000) 2023-08-27T23:36:36.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:35 smithi130 bash[51302]: debug 2023-08-27T23:36:35.635+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:35.640393+0000) 2023-08-27T23:36:37.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:36 smithi130 bash[27579]: cluster 2023-08-27T23:36:35.310969+0000 mgr.y (mgr.14901) 2941 : cluster [DBG] pgmap v2948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:37.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:36 smithi130 bash[27579]: audit 2023-08-27T23:36:36.612685+0000 mon.a (mon.0) 2415 : audit [DBG] from='client.? 172.21.15.6:0/2534853967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:37.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:36 smithi130 bash[51302]: debug 2023-08-27T23:36:36.639+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:36.640688+0000) 2023-08-27T23:36:37.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:36 smithi006 bash[33918]: cluster 2023-08-27T23:36:35.310969+0000 mgr.y (mgr.14901) 2941 : cluster [DBG] pgmap v2948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:37.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:36 smithi006 bash[33918]: audit 2023-08-27T23:36:36.612685+0000 mon.a (mon.0) 2415 : audit [DBG] from='client.? 172.21.15.6:0/2534853967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:37.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:36 smithi006 bash[27292]: cluster 2023-08-27T23:36:35.310969+0000 mgr.y (mgr.14901) 2941 : cluster [DBG] pgmap v2948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:37.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:36 smithi006 bash[27292]: audit 2023-08-27T23:36:36.612685+0000 mon.a (mon.0) 2415 : audit [DBG] from='client.? 172.21.15.6:0/2534853967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:38.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:37 smithi130 bash[51302]: debug 2023-08-27T23:36:37.635+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:37.640979+0000) 2023-08-27T23:36:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:38 smithi130 bash[27579]: cluster 2023-08-27T23:36:37.311786+0000 mgr.y (mgr.14901) 2942 : cluster [DBG] pgmap v2949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:39.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:38 smithi130 bash[51302]: debug 2023-08-27T23:36:38.639+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:38.641325+0000) 2023-08-27T23:36:39.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:38 smithi006 bash[27292]: cluster 2023-08-27T23:36:37.311786+0000 mgr.y (mgr.14901) 2942 : cluster [DBG] pgmap v2949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:39.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:38 smithi006 bash[33918]: cluster 2023-08-27T23:36:37.311786+0000 mgr.y (mgr.14901) 2942 : cluster [DBG] pgmap v2949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:40.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:39 smithi130 bash[27579]: audit 2023-08-27T23:36:39.033784+0000 mon.c (mon.2) 1141 : audit [DBG] from='client.? 172.21.15.6:0/3813458114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:40.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:39 smithi130 bash[51302]: debug 2023-08-27T23:36:39.639+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:39.641640+0000) 2023-08-27T23:36:40.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:39 smithi006 bash[33918]: audit 2023-08-27T23:36:39.033784+0000 mon.c (mon.2) 1141 : audit [DBG] from='client.? 172.21.15.6:0/3813458114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:40.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:39 smithi006 bash[27292]: audit 2023-08-27T23:36:39.033784+0000 mon.c (mon.2) 1141 : audit [DBG] from='client.? 172.21.15.6:0/3813458114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:40.870 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:36:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:36:40] "GET /metrics HTTP/1.1" 200 193096 "" "Prometheus/2.33.4" 2023-08-27T23:36:40.873 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:36:40.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:40.873 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:36:40.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:40 smithi130 bash[27579]: cluster 2023-08-27T23:36:39.312980+0000 mgr.y (mgr.14901) 2943 : cluster [DBG] pgmap v2950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:41.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:40 smithi130 bash[51302]: debug 2023-08-27T23:36:40.635+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:40.641925+0000) 2023-08-27T23:36:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:40 smithi006 bash[27292]: cluster 2023-08-27T23:36:39.312980+0000 mgr.y (mgr.14901) 2943 : cluster [DBG] pgmap v2950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:41.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:40 smithi006 bash[33918]: cluster 2023-08-27T23:36:39.312980+0000 mgr.y (mgr.14901) 2943 : cluster [DBG] pgmap v2950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:42.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:41 smithi130 bash[27579]: audit 2023-08-27T23:36:41.453269+0000 mon.a (mon.0) 2416 : audit [DBG] from='client.? 172.21.15.6:0/2813196050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:42.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:41 smithi130 bash[51302]: debug 2023-08-27T23:36:41.635+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:41.642185+0000) 2023-08-27T23:36:42.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:41 smithi006 bash[33918]: audit 2023-08-27T23:36:41.453269+0000 mon.a (mon.0) 2416 : audit [DBG] from='client.? 172.21.15.6:0/2813196050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:42.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:41 smithi006 bash[27292]: audit 2023-08-27T23:36:41.453269+0000 mon.a (mon.0) 2416 : audit [DBG] from='client.? 172.21.15.6:0/2813196050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:42.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:36:42.078Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:42.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:36:42.079Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:43.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:42 smithi130 bash[27579]: cluster 2023-08-27T23:36:41.313726+0000 mgr.y (mgr.14901) 2944 : cluster [DBG] pgmap v2951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:43.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:42 smithi130 bash[51302]: debug 2023-08-27T23:36:42.635+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:42.642464+0000) 2023-08-27T23:36:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:42 smithi006 bash[33918]: cluster 2023-08-27T23:36:41.313726+0000 mgr.y (mgr.14901) 2944 : cluster [DBG] pgmap v2951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:42 smithi006 bash[27292]: cluster 2023-08-27T23:36:41.313726+0000 mgr.y (mgr.14901) 2944 : cluster [DBG] pgmap v2951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:44.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:43 smithi130 bash[27579]: audit 2023-08-27T23:36:43.873940+0000 mon.c (mon.2) 1142 : audit [DBG] from='client.? 172.21.15.6:0/1881808073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:44.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:43 smithi130 bash[51302]: debug 2023-08-27T23:36:43.635+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:43.642740+0000) 2023-08-27T23:36:44.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:43 smithi006 bash[33918]: audit 2023-08-27T23:36:43.873940+0000 mon.c (mon.2) 1142 : audit [DBG] from='client.? 172.21.15.6:0/1881808073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:44.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:43 smithi006 bash[27292]: audit 2023-08-27T23:36:43.873940+0000 mon.c (mon.2) 1142 : audit [DBG] from='client.? 172.21.15.6:0/1881808073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:44.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:36:43.978Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:44.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:36:43.979Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:44 smithi130 bash[27579]: cluster 2023-08-27T23:36:43.314451+0000 mgr.y (mgr.14901) 2945 : cluster [DBG] pgmap v2952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:44 smithi130 bash[51302]: debug 2023-08-27T23:36:44.639+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:44.643043+0000) 2023-08-27T23:36:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:44 smithi006 bash[33918]: cluster 2023-08-27T23:36:43.314451+0000 mgr.y (mgr.14901) 2945 : cluster [DBG] pgmap v2952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:45.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:44 smithi006 bash[27292]: cluster 2023-08-27T23:36:43.314451+0000 mgr.y (mgr.14901) 2945 : cluster [DBG] pgmap v2952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:45.901 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:45 smithi130 bash[51302]: debug 2023-08-27T23:36:45.639+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:45.643278+0000) 2023-08-27T23:36:46.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:45 smithi006 bash[33918]: cluster 2023-08-27T23:36:45.315590+0000 mgr.y (mgr.14901) 2946 : cluster [DBG] pgmap v2953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:46.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:45 smithi006 bash[27292]: cluster 2023-08-27T23:36:45.315590+0000 mgr.y (mgr.14901) 2946 : cluster [DBG] pgmap v2953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:46.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:45 smithi130 bash[27579]: cluster 2023-08-27T23:36:45.315590+0000 mgr.y (mgr.14901) 2946 : cluster [DBG] pgmap v2953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:46.902 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:46 smithi130 bash[51302]: debug 2023-08-27T23:36:46.639+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:46.643572+0000) 2023-08-27T23:36:47.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:46 smithi006 bash[27292]: audit 2023-08-27T23:36:46.297562+0000 mon.c (mon.2) 1143 : audit [DBG] from='client.? 172.21.15.6:0/2938316026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:47.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:46 smithi006 bash[33918]: audit 2023-08-27T23:36:46.297562+0000 mon.c (mon.2) 1143 : audit [DBG] from='client.? 172.21.15.6:0/2938316026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:47.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:46 smithi130 bash[27579]: audit 2023-08-27T23:36:46.297562+0000 mon.c (mon.2) 1143 : audit [DBG] from='client.? 172.21.15.6:0/2938316026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:47.911 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:47 smithi130 bash[51302]: debug 2023-08-27T23:36:47.639+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:47.643848+0000) 2023-08-27T23:36:48.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:47 smithi006 bash[27292]: cluster 2023-08-27T23:36:47.316257+0000 mgr.y (mgr.14901) 2947 : cluster [DBG] pgmap v2954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:48.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:47 smithi006 bash[33918]: cluster 2023-08-27T23:36:47.316257+0000 mgr.y (mgr.14901) 2947 : cluster [DBG] pgmap v2954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:48.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:47 smithi130 bash[27579]: cluster 2023-08-27T23:36:47.316257+0000 mgr.y (mgr.14901) 2947 : cluster [DBG] pgmap v2954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:48.915 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:48 smithi130 bash[51302]: debug 2023-08-27T23:36:48.639+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:48.644155+0000) 2023-08-27T23:36:49.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:48 smithi006 bash[27292]: audit 2023-08-27T23:36:48.718432+0000 mon.a (mon.0) 2417 : audit [DBG] from='client.? 172.21.15.6:0/4024606154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:49.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:48 smithi006 bash[33918]: audit 2023-08-27T23:36:48.718432+0000 mon.a (mon.0) 2417 : audit [DBG] from='client.? 172.21.15.6:0/4024606154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:49.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:48 smithi130 bash[27579]: audit 2023-08-27T23:36:48.718432+0000 mon.a (mon.0) 2417 : audit [DBG] from='client.? 172.21.15.6:0/4024606154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:50.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:49 smithi130 bash[51302]: debug 2023-08-27T23:36:49.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:49.644470+0000) 2023-08-27T23:36:50.857 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:36:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:36:50] "GET /metrics HTTP/1.1" 200 193095 "" "Prometheus/2.33.4" 2023-08-27T23:36:50.858 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:36:50.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:50.858 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:36:50.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:50 smithi130 bash[27579]: cluster 2023-08-27T23:36:49.317360+0000 mgr.y (mgr.14901) 2948 : cluster [DBG] pgmap v2955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:51.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:50 smithi130 bash[51302]: debug 2023-08-27T23:36:50.639+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:50.644776+0000) 2023-08-27T23:36:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:50 smithi006 bash[27292]: cluster 2023-08-27T23:36:49.317360+0000 mgr.y (mgr.14901) 2948 : cluster [DBG] pgmap v2955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:50 smithi006 bash[33918]: cluster 2023-08-27T23:36:49.317360+0000 mgr.y (mgr.14901) 2948 : cluster [DBG] pgmap v2955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:52.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:51 smithi130 bash[51302]: debug 2023-08-27T23:36:51.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:51.645048+0000) 2023-08-27T23:36:52.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:51 smithi130 bash[27579]: audit 2023-08-27T23:36:51.130890+0000 mon.c (mon.2) 1144 : audit [DBG] from='client.? 172.21.15.6:0/2483867326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:52.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:51 smithi006 bash[27292]: audit 2023-08-27T23:36:51.130890+0000 mon.c (mon.2) 1144 : audit [DBG] from='client.? 172.21.15.6:0/2483867326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:52.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:51 smithi006 bash[33918]: audit 2023-08-27T23:36:51.130890+0000 mon.c (mon.2) 1144 : audit [DBG] from='client.? 172.21.15.6:0/2483867326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:52.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:36:52.079Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:52.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:36:52.081Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:52 smithi130 bash[27579]: cluster 2023-08-27T23:36:51.318024+0000 mgr.y (mgr.14901) 2949 : cluster [DBG] pgmap v2956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:52 smithi130 bash[51302]: debug 2023-08-27T23:36:52.639+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:52.645317+0000) 2023-08-27T23:36:53.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:52 smithi006 bash[33918]: cluster 2023-08-27T23:36:51.318024+0000 mgr.y (mgr.14901) 2949 : cluster [DBG] pgmap v2956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:52 smithi006 bash[27292]: cluster 2023-08-27T23:36:51.318024+0000 mgr.y (mgr.14901) 2949 : cluster [DBG] pgmap v2956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:54.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:53 smithi130 bash[51302]: debug 2023-08-27T23:36:53.639+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:53.645594+0000) 2023-08-27T23:36:54.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:53 smithi130 bash[27579]: audit 2023-08-27T23:36:53.547535+0000 mon.a (mon.0) 2418 : audit [DBG] from='client.? 172.21.15.6:0/392213643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:54.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:53 smithi006 bash[27292]: audit 2023-08-27T23:36:53.547535+0000 mon.a (mon.0) 2418 : audit [DBG] from='client.? 172.21.15.6:0/392213643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:36:53.979Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:36:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:36:53.981Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:36:54.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:53 smithi006 bash[33918]: audit 2023-08-27T23:36:53.547535+0000 mon.a (mon.0) 2418 : audit [DBG] from='client.? 172.21.15.6:0/392213643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:54 smithi130 bash[27579]: cluster 2023-08-27T23:36:53.318650+0000 mgr.y (mgr.14901) 2950 : cluster [DBG] pgmap v2957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:54 smithi130 bash[51302]: debug 2023-08-27T23:36:54.639+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:54.645912+0000) 2023-08-27T23:36:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:54 smithi006 bash[33918]: cluster 2023-08-27T23:36:53.318650+0000 mgr.y (mgr.14901) 2950 : cluster [DBG] pgmap v2957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:54 smithi006 bash[27292]: cluster 2023-08-27T23:36:53.318650+0000 mgr.y (mgr.14901) 2950 : cluster [DBG] pgmap v2957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:56.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:55 smithi130 bash[51302]: debug 2023-08-27T23:36:55.639+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:55.646136+0000) 2023-08-27T23:36:57.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:56 smithi130 bash[51302]: debug 2023-08-27T23:36:56.639+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:56.646396+0000) 2023-08-27T23:36:57.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:56 smithi130 bash[27579]: cluster 2023-08-27T23:36:55.319789+0000 mgr.y (mgr.14901) 2951 : cluster [DBG] pgmap v2958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:57.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:56 smithi130 bash[27579]: audit 2023-08-27T23:36:55.955383+0000 mon.a (mon.0) 2419 : audit [DBG] from='client.? 172.21.15.6:0/856320736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:57.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:56 smithi006 bash[27292]: cluster 2023-08-27T23:36:55.319789+0000 mgr.y (mgr.14901) 2951 : cluster [DBG] pgmap v2958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:57.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:56 smithi006 bash[27292]: audit 2023-08-27T23:36:55.955383+0000 mon.a (mon.0) 2419 : audit [DBG] from='client.? 172.21.15.6:0/856320736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:57.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:56 smithi006 bash[33918]: cluster 2023-08-27T23:36:55.319789+0000 mgr.y (mgr.14901) 2951 : cluster [DBG] pgmap v2958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:57.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:56 smithi006 bash[33918]: audit 2023-08-27T23:36:55.955383+0000 mon.a (mon.0) 2419 : audit [DBG] from='client.? 172.21.15.6:0/856320736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:57.942 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:57 smithi130 bash[51302]: debug 2023-08-27T23:36:57.639+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:57.646660+0000) 2023-08-27T23:36:58.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:57 smithi006 bash[33918]: cluster 2023-08-27T23:36:57.320468+0000 mgr.y (mgr.14901) 2952 : cluster [DBG] pgmap v2959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:58.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:57 smithi006 bash[27292]: cluster 2023-08-27T23:36:57.320468+0000 mgr.y (mgr.14901) 2952 : cluster [DBG] pgmap v2959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:58.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:57 smithi130 bash[27579]: cluster 2023-08-27T23:36:57.320468+0000 mgr.y (mgr.14901) 2952 : cluster [DBG] pgmap v2959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:36:58.944 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:58 smithi130 bash[51302]: debug 2023-08-27T23:36:58.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:58.646973+0000) 2023-08-27T23:36:59.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:36:58 smithi006 bash[33918]: audit 2023-08-27T23:36:58.368288+0000 mon.a (mon.0) 2420 : audit [DBG] from='client.? 172.21.15.6:0/3259716465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:59.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:36:58 smithi006 bash[27292]: audit 2023-08-27T23:36:58.368288+0000 mon.a (mon.0) 2420 : audit [DBG] from='client.? 172.21.15.6:0/3259716465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:59.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:36:58 smithi130 bash[27579]: audit 2023-08-27T23:36:58.368288+0000 mon.a (mon.0) 2420 : audit [DBG] from='client.? 172.21.15.6:0/3259716465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:00.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:36:59 smithi130 bash[51302]: debug 2023-08-27T23:36:59.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:59.647256+0000) 2023-08-27T23:37:00.860 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:37:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:37:00] "GET /metrics HTTP/1.1" 200 193091 "" "Prometheus/2.33.4" 2023-08-27T23:37:00.860 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:37:00.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:00.860 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:37:00.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:01.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:00 smithi130 bash[51302]: debug 2023-08-27T23:37:00.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:00.647550+0000) 2023-08-27T23:37:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:00 smithi130 bash[27579]: cluster 2023-08-27T23:36:59.321594+0000 mgr.y (mgr.14901) 2953 : cluster [DBG] pgmap v2960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:00 smithi130 bash[27579]: audit 2023-08-27T23:37:00.783903+0000 mon.a (mon.0) 2421 : audit [DBG] from='client.? 172.21.15.6:0/789124982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:00 smithi006 bash[33918]: cluster 2023-08-27T23:36:59.321594+0000 mgr.y (mgr.14901) 2953 : cluster [DBG] pgmap v2960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:00 smithi006 bash[33918]: audit 2023-08-27T23:37:00.783903+0000 mon.a (mon.0) 2421 : audit [DBG] from='client.? 172.21.15.6:0/789124982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:01.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:00 smithi006 bash[27292]: cluster 2023-08-27T23:36:59.321594+0000 mgr.y (mgr.14901) 2953 : cluster [DBG] pgmap v2960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:01.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:00 smithi006 bash[27292]: audit 2023-08-27T23:37:00.783903+0000 mon.a (mon.0) 2421 : audit [DBG] from='client.? 172.21.15.6:0/789124982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:02.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:01 smithi130 bash[51302]: debug 2023-08-27T23:37:01.647+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:01.647838+0000) 2023-08-27T23:37:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:37:02.080Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:37:02.082Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:02 smithi130 bash[27579]: cluster 2023-08-27T23:37:01.322244+0000 mgr.y (mgr.14901) 2954 : cluster [DBG] pgmap v2961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:02 smithi130 bash[51302]: debug 2023-08-27T23:37:02.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:02.648103+0000) 2023-08-27T23:37:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:02 smithi006 bash[33918]: cluster 2023-08-27T23:37:01.322244+0000 mgr.y (mgr.14901) 2954 : cluster [DBG] pgmap v2961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:02 smithi006 bash[27292]: cluster 2023-08-27T23:37:01.322244+0000 mgr.y (mgr.14901) 2954 : cluster [DBG] pgmap v2961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:04.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:03 smithi130 bash[27579]: audit 2023-08-27T23:37:03.201418+0000 mon.a (mon.0) 2422 : audit [DBG] from='client.? 172.21.15.6:0/69077246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:04.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:03 smithi130 bash[51302]: debug 2023-08-27T23:37:03.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:03.648397+0000) 2023-08-27T23:37:04.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:03 smithi006 bash[33918]: audit 2023-08-27T23:37:03.201418+0000 mon.a (mon.0) 2422 : audit [DBG] from='client.? 172.21.15.6:0/69077246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:04.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:03 smithi006 bash[27292]: audit 2023-08-27T23:37:03.201418+0000 mon.a (mon.0) 2422 : audit [DBG] from='client.? 172.21.15.6:0/69077246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:04.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:37:03.979Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:04.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:37:03.981Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:04 smithi130 bash[27579]: cluster 2023-08-27T23:37:03.322937+0000 mgr.y (mgr.14901) 2955 : cluster [DBG] pgmap v2962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:04 smithi130 bash[51302]: debug 2023-08-27T23:37:04.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:04.648672+0000) 2023-08-27T23:37:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:04 smithi006 bash[33918]: cluster 2023-08-27T23:37:03.322937+0000 mgr.y (mgr.14901) 2955 : cluster [DBG] pgmap v2962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:04 smithi006 bash[27292]: cluster 2023-08-27T23:37:03.322937+0000 mgr.y (mgr.14901) 2955 : cluster [DBG] pgmap v2962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:06.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:05 smithi130 bash[27579]: audit 2023-08-27T23:37:05.461401+0000 mon.b (mon.1) 400 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:37:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:05 smithi130 bash[27579]: audit 2023-08-27T23:37:05.619134+0000 mon.c (mon.2) 1145 : audit [DBG] from='client.? 172.21.15.6:0/1215915045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:06.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:05 smithi130 bash[51302]: debug 2023-08-27T23:37:05.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:05.648911+0000) 2023-08-27T23:37:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:05 smithi006 bash[33918]: audit 2023-08-27T23:37:05.461401+0000 mon.b (mon.1) 400 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:37:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:05 smithi006 bash[33918]: audit 2023-08-27T23:37:05.619134+0000 mon.c (mon.2) 1145 : audit [DBG] from='client.? 172.21.15.6:0/1215915045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:06.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:05 smithi006 bash[27292]: audit 2023-08-27T23:37:05.461401+0000 mon.b (mon.1) 400 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:37:06.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:05 smithi006 bash[27292]: audit 2023-08-27T23:37:05.619134+0000 mon.c (mon.2) 1145 : audit [DBG] from='client.? 172.21.15.6:0/1215915045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:06 smithi130 bash[27579]: cluster 2023-08-27T23:37:05.324081+0000 mgr.y (mgr.14901) 2956 : cluster [DBG] pgmap v2963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:07.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:06 smithi130 bash[51302]: debug 2023-08-27T23:37:06.647+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:06.649181+0000) 2023-08-27T23:37:07.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:06 smithi006 bash[27292]: cluster 2023-08-27T23:37:05.324081+0000 mgr.y (mgr.14901) 2956 : cluster [DBG] pgmap v2963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:07.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:06 smithi006 bash[33918]: cluster 2023-08-27T23:37:05.324081+0000 mgr.y (mgr.14901) 2956 : cluster [DBG] pgmap v2963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:08.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:07 smithi130 bash[51302]: debug 2023-08-27T23:37:07.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:07.649450+0000) 2023-08-27T23:37:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:08 smithi130 bash[27579]: cluster 2023-08-27T23:37:07.324727+0000 mgr.y (mgr.14901) 2957 : cluster [DBG] pgmap v2964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:08 smithi130 bash[27579]: audit 2023-08-27T23:37:08.042135+0000 mon.a (mon.0) 2423 : audit [DBG] from='client.? 172.21.15.6:0/1682035297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:09.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:08 smithi130 bash[51302]: debug 2023-08-27T23:37:08.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:08.649800+0000) 2023-08-27T23:37:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:08 smithi006 bash[33918]: cluster 2023-08-27T23:37:07.324727+0000 mgr.y (mgr.14901) 2957 : cluster [DBG] pgmap v2964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:08 smithi006 bash[33918]: audit 2023-08-27T23:37:08.042135+0000 mon.a (mon.0) 2423 : audit [DBG] from='client.? 172.21.15.6:0/1682035297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:08 smithi006 bash[27292]: cluster 2023-08-27T23:37:07.324727+0000 mgr.y (mgr.14901) 2957 : cluster [DBG] pgmap v2964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:09.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:08 smithi006 bash[27292]: audit 2023-08-27T23:37:08.042135+0000 mon.a (mon.0) 2423 : audit [DBG] from='client.? 172.21.15.6:0/1682035297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:10.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:09 smithi130 bash[51302]: debug 2023-08-27T23:37:09.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:09.650083+0000) 2023-08-27T23:37:10.888 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:37:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:37:10] "GET /metrics HTTP/1.1" 200 193091 "" "Prometheus/2.33.4" 2023-08-27T23:37:10.888 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:37:10.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:10.889 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:37:10.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:10 smithi130 bash[27579]: cluster 2023-08-27T23:37:09.325786+0000 mgr.y (mgr.14901) 2958 : cluster [DBG] pgmap v2965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:10 smithi130 bash[27579]: audit 2023-08-27T23:37:10.480427+0000 mon.c (mon.2) 1146 : audit [DBG] from='client.? 172.21.15.6:0/2824949438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:11.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:10 smithi130 bash[51302]: debug 2023-08-27T23:37:10.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:10.650328+0000) 2023-08-27T23:37:11.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:10 smithi006 bash[27292]: cluster 2023-08-27T23:37:09.325786+0000 mgr.y (mgr.14901) 2958 : cluster [DBG] pgmap v2965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:11.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:10 smithi006 bash[27292]: audit 2023-08-27T23:37:10.480427+0000 mon.c (mon.2) 1146 : audit [DBG] from='client.? 172.21.15.6:0/2824949438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:10 smithi006 bash[33918]: cluster 2023-08-27T23:37:09.325786+0000 mgr.y (mgr.14901) 2958 : cluster [DBG] pgmap v2965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:11.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:10 smithi006 bash[33918]: audit 2023-08-27T23:37:10.480427+0000 mon.c (mon.2) 1146 : audit [DBG] from='client.? 172.21.15.6:0/2824949438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:12.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:11 smithi130 bash[51302]: debug 2023-08-27T23:37:11.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:11.650543+0000) 2023-08-27T23:37:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:37:12.081Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:37:12.083Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:12.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:12 smithi006 bash[27292]: audit 2023-08-27T23:37:11.292513+0000 mon.a (mon.0) 2424 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:37:12.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:12 smithi006 bash[27292]: cluster 2023-08-27T23:37:11.326408+0000 mgr.y (mgr.14901) 2959 : cluster [DBG] pgmap v2966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:12.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:12 smithi006 bash[27292]: audit 2023-08-27T23:37:11.774505+0000 mon.a (mon.0) 2425 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:37:12.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:12 smithi006 bash[27292]: audit 2023-08-27T23:37:12.045761+0000 mon.b (mon.1) 401 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:37:12.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:12 smithi006 bash[27292]: audit 2023-08-27T23:37:12.047928+0000 mon.b (mon.1) 402 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:37:12.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:12 smithi006 bash[27292]: audit 2023-08-27T23:37:12.055251+0000 mon.a (mon.0) 2426 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:37:12.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:12 smithi006 bash[33918]: audit 2023-08-27T23:37:11.292513+0000 mon.a (mon.0) 2424 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:37:12.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:12 smithi006 bash[33918]: cluster 2023-08-27T23:37:11.326408+0000 mgr.y (mgr.14901) 2959 : cluster [DBG] pgmap v2966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:12.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:12 smithi006 bash[33918]: audit 2023-08-27T23:37:11.774505+0000 mon.a (mon.0) 2425 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:37:12.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:12 smithi006 bash[33918]: audit 2023-08-27T23:37:12.045761+0000 mon.b (mon.1) 401 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:37:12.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:12 smithi006 bash[33918]: audit 2023-08-27T23:37:12.047928+0000 mon.b (mon.1) 402 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:37:12.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:12 smithi006 bash[33918]: audit 2023-08-27T23:37:12.055251+0000 mon.a (mon.0) 2426 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:37:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:12 smithi130 bash[27579]: audit 2023-08-27T23:37:11.292513+0000 mon.a (mon.0) 2424 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:37:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:12 smithi130 bash[27579]: cluster 2023-08-27T23:37:11.326408+0000 mgr.y (mgr.14901) 2959 : cluster [DBG] pgmap v2966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:12 smithi130 bash[27579]: audit 2023-08-27T23:37:11.774505+0000 mon.a (mon.0) 2425 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:37:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:12 smithi130 bash[27579]: audit 2023-08-27T23:37:12.045761+0000 mon.b (mon.1) 401 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:37:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:12 smithi130 bash[27579]: audit 2023-08-27T23:37:12.047928+0000 mon.b (mon.1) 402 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:37:12.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:12 smithi130 bash[27579]: audit 2023-08-27T23:37:12.055251+0000 mon.a (mon.0) 2426 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:37:13.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:12 smithi130 bash[51302]: debug 2023-08-27T23:37:12.647+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:12.650809+0000) 2023-08-27T23:37:13.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:13 smithi130 bash[27579]: audit 2023-08-27T23:37:12.890884+0000 mon.a (mon.0) 2427 : audit [DBG] from='client.? 172.21.15.6:0/3088209056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:13.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:13 smithi006 bash[27292]: audit 2023-08-27T23:37:12.890884+0000 mon.a (mon.0) 2427 : audit [DBG] from='client.? 172.21.15.6:0/3088209056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:13.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:13 smithi006 bash[33918]: audit 2023-08-27T23:37:12.890884+0000 mon.a (mon.0) 2427 : audit [DBG] from='client.? 172.21.15.6:0/3088209056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:14.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:13 smithi130 bash[51302]: debug 2023-08-27T23:37:13.647+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:13.651121+0000) 2023-08-27T23:37:14.311 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:37:13.980Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:14.311 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:37:13.982Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:14.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:14 smithi130 bash[27579]: cluster 2023-08-27T23:37:13.327039+0000 mgr.y (mgr.14901) 2960 : cluster [DBG] pgmap v2967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:14 smithi006 bash[33918]: cluster 2023-08-27T23:37:13.327039+0000 mgr.y (mgr.14901) 2960 : cluster [DBG] pgmap v2967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:14.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:14 smithi006 bash[27292]: cluster 2023-08-27T23:37:13.327039+0000 mgr.y (mgr.14901) 2960 : cluster [DBG] pgmap v2967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:15.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:14 smithi130 bash[51302]: debug 2023-08-27T23:37:14.647+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:14.651343+0000) 2023-08-27T23:37:16.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:15 smithi130 bash[51302]: debug 2023-08-27T23:37:15.647+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:15.651563+0000) 2023-08-27T23:37:16.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:15 smithi130 bash[27579]: audit 2023-08-27T23:37:15.304508+0000 mon.c (mon.2) 1147 : audit [DBG] from='client.? 172.21.15.6:0/2959848726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:16.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:15 smithi006 bash[27292]: audit 2023-08-27T23:37:15.304508+0000 mon.c (mon.2) 1147 : audit [DBG] from='client.? 172.21.15.6:0/2959848726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:16.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:15 smithi006 bash[33918]: audit 2023-08-27T23:37:15.304508+0000 mon.c (mon.2) 1147 : audit [DBG] from='client.? 172.21.15.6:0/2959848726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:17.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:16 smithi130 bash[27579]: cluster 2023-08-27T23:37:15.328090+0000 mgr.y (mgr.14901) 2961 : cluster [DBG] pgmap v2968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:17.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:16 smithi130 bash[51302]: debug 2023-08-27T23:37:16.647+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:16.651838+0000) 2023-08-27T23:37:17.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:16 smithi006 bash[27292]: cluster 2023-08-27T23:37:15.328090+0000 mgr.y (mgr.14901) 2961 : cluster [DBG] pgmap v2968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:17.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:16 smithi006 bash[33918]: cluster 2023-08-27T23:37:15.328090+0000 mgr.y (mgr.14901) 2961 : cluster [DBG] pgmap v2968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:18.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:17 smithi130 bash[27579]: audit 2023-08-27T23:37:17.720659+0000 mon.a (mon.0) 2428 : audit [DBG] from='client.? 172.21.15.6:0/879082100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:18.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:17 smithi130 bash[51302]: debug 2023-08-27T23:37:17.647+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:17.652114+0000) 2023-08-27T23:37:18.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:17 smithi006 bash[27292]: audit 2023-08-27T23:37:17.720659+0000 mon.a (mon.0) 2428 : audit [DBG] from='client.? 172.21.15.6:0/879082100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:18.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:17 smithi006 bash[33918]: audit 2023-08-27T23:37:17.720659+0000 mon.a (mon.0) 2428 : audit [DBG] from='client.? 172.21.15.6:0/879082100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:18 smithi130 bash[27579]: cluster 2023-08-27T23:37:17.328802+0000 mgr.y (mgr.14901) 2962 : cluster [DBG] pgmap v2969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:19.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:18 smithi130 bash[51302]: debug 2023-08-27T23:37:18.651+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:18.652460+0000) 2023-08-27T23:37:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:18 smithi006 bash[27292]: cluster 2023-08-27T23:37:17.328802+0000 mgr.y (mgr.14901) 2962 : cluster [DBG] pgmap v2969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:18 smithi006 bash[33918]: cluster 2023-08-27T23:37:17.328802+0000 mgr.y (mgr.14901) 2962 : cluster [DBG] pgmap v2969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:20.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:19 smithi130 bash[51302]: debug 2023-08-27T23:37:19.651+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:19.652739+0000) 2023-08-27T23:37:20.878 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:37:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:37:20] "GET /metrics HTTP/1.1" 200 193094 "" "Prometheus/2.33.4" 2023-08-27T23:37:20.879 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:37:20.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:20.879 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:37:20.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:21.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:20 smithi130 bash[51302]: debug 2023-08-27T23:37:20.651+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:20.653055+0000) 2023-08-27T23:37:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:20 smithi130 bash[27579]: cluster 2023-08-27T23:37:19.329943+0000 mgr.y (mgr.14901) 2963 : cluster [DBG] pgmap v2970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:20 smithi130 bash[27579]: audit 2023-08-27T23:37:20.132584+0000 mon.a (mon.0) 2429 : audit [DBG] from='client.? 172.21.15.6:0/2887380013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:20 smithi006 bash[27292]: cluster 2023-08-27T23:37:19.329943+0000 mgr.y (mgr.14901) 2963 : cluster [DBG] pgmap v2970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:20 smithi006 bash[27292]: audit 2023-08-27T23:37:20.132584+0000 mon.a (mon.0) 2429 : audit [DBG] from='client.? 172.21.15.6:0/2887380013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:21.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:20 smithi006 bash[33918]: cluster 2023-08-27T23:37:19.329943+0000 mgr.y (mgr.14901) 2963 : cluster [DBG] pgmap v2970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:21.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:20 smithi006 bash[33918]: audit 2023-08-27T23:37:20.132584+0000 mon.a (mon.0) 2429 : audit [DBG] from='client.? 172.21.15.6:0/2887380013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:22.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:21 smithi130 bash[51302]: debug 2023-08-27T23:37:21.651+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:21.653346+0000) 2023-08-27T23:37:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:37:22.081Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:37:22.083Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:22 smithi130 bash[27579]: cluster 2023-08-27T23:37:21.330623+0000 mgr.y (mgr.14901) 2964 : cluster [DBG] pgmap v2971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:22 smithi130 bash[27579]: audit 2023-08-27T23:37:22.546317+0000 mon.a (mon.0) 2430 : audit [DBG] from='client.? 172.21.15.6:0/4147282742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:23.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:22 smithi130 bash[51302]: debug 2023-08-27T23:37:22.647+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:22.653592+0000) 2023-08-27T23:37:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:22 smithi006 bash[27292]: cluster 2023-08-27T23:37:21.330623+0000 mgr.y (mgr.14901) 2964 : cluster [DBG] pgmap v2971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:22 smithi006 bash[27292]: audit 2023-08-27T23:37:22.546317+0000 mon.a (mon.0) 2430 : audit [DBG] from='client.? 172.21.15.6:0/4147282742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:23.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:22 smithi006 bash[33918]: cluster 2023-08-27T23:37:21.330623+0000 mgr.y (mgr.14901) 2964 : cluster [DBG] pgmap v2971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:23.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:22 smithi006 bash[33918]: audit 2023-08-27T23:37:22.546317+0000 mon.a (mon.0) 2430 : audit [DBG] from='client.? 172.21.15.6:0/4147282742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:24.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:23 smithi130 bash[51302]: debug 2023-08-27T23:37:23.646+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:23.653850+0000) 2023-08-27T23:37:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:37:23.981Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:37:23.983Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:24 smithi130 bash[27579]: cluster 2023-08-27T23:37:23.331296+0000 mgr.y (mgr.14901) 2965 : cluster [DBG] pgmap v2972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:24 smithi130 bash[51302]: debug 2023-08-27T23:37:24.650+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:24.654143+0000) 2023-08-27T23:37:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:24 smithi006 bash[27292]: cluster 2023-08-27T23:37:23.331296+0000 mgr.y (mgr.14901) 2965 : cluster [DBG] pgmap v2972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:24 smithi006 bash[33918]: cluster 2023-08-27T23:37:23.331296+0000 mgr.y (mgr.14901) 2965 : cluster [DBG] pgmap v2972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:25 smithi130 bash[27579]: audit 2023-08-27T23:37:24.955887+0000 mon.c (mon.2) 1148 : audit [DBG] from='client.? 172.21.15.6:0/878292694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:26.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:25 smithi130 bash[51302]: debug 2023-08-27T23:37:25.646+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:25.654450+0000) 2023-08-27T23:37:26.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:25 smithi006 bash[33918]: audit 2023-08-27T23:37:24.955887+0000 mon.c (mon.2) 1148 : audit [DBG] from='client.? 172.21.15.6:0/878292694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:25 smithi006 bash[27292]: audit 2023-08-27T23:37:24.955887+0000 mon.c (mon.2) 1148 : audit [DBG] from='client.? 172.21.15.6:0/878292694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:26 smithi130 bash[27579]: cluster 2023-08-27T23:37:25.332449+0000 mgr.y (mgr.14901) 2966 : cluster [DBG] pgmap v2973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:27.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:26 smithi130 bash[51302]: debug 2023-08-27T23:37:26.646+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:26.654768+0000) 2023-08-27T23:37:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:26 smithi006 bash[33918]: cluster 2023-08-27T23:37:25.332449+0000 mgr.y (mgr.14901) 2966 : cluster [DBG] pgmap v2973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:26 smithi006 bash[27292]: cluster 2023-08-27T23:37:25.332449+0000 mgr.y (mgr.14901) 2966 : cluster [DBG] pgmap v2973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:27.909 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:27 smithi130 bash[51302]: debug 2023-08-27T23:37:27.654+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:27.655079+0000) 2023-08-27T23:37:28.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:27 smithi006 bash[27292]: cluster 2023-08-27T23:37:27.333069+0000 mgr.y (mgr.14901) 2967 : cluster [DBG] pgmap v2974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:28.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:27 smithi006 bash[27292]: audit 2023-08-27T23:37:27.373122+0000 mon.a (mon.0) 2431 : audit [DBG] from='client.? 172.21.15.6:0/1837297841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:28.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:27 smithi006 bash[33918]: cluster 2023-08-27T23:37:27.333069+0000 mgr.y (mgr.14901) 2967 : cluster [DBG] pgmap v2974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:28.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:27 smithi006 bash[33918]: audit 2023-08-27T23:37:27.373122+0000 mon.a (mon.0) 2431 : audit [DBG] from='client.? 172.21.15.6:0/1837297841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:28.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:27 smithi130 bash[27579]: cluster 2023-08-27T23:37:27.333069+0000 mgr.y (mgr.14901) 2967 : cluster [DBG] pgmap v2974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:28.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:27 smithi130 bash[27579]: audit 2023-08-27T23:37:27.373122+0000 mon.a (mon.0) 2431 : audit [DBG] from='client.? 172.21.15.6:0/1837297841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:29.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:28 smithi130 bash[51302]: debug 2023-08-27T23:37:28.654+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:28.655381+0000) 2023-08-27T23:37:30.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:29 smithi130 bash[51302]: debug 2023-08-27T23:37:29.654+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:29.655662+0000) 2023-08-27T23:37:30.657 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:30 smithi130 bash[27579]: cluster 2023-08-27T23:37:29.334138+0000 mgr.y (mgr.14901) 2968 : cluster [DBG] pgmap v2975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:30.657 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:30 smithi130 bash[27579]: audit 2023-08-27T23:37:29.784523+0000 mon.a (mon.0) 2432 : audit [DBG] from='client.? 172.21.15.6:0/3297429759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:30 smithi006 bash[33918]: cluster 2023-08-27T23:37:29.334138+0000 mgr.y (mgr.14901) 2968 : cluster [DBG] pgmap v2975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:30 smithi006 bash[33918]: audit 2023-08-27T23:37:29.784523+0000 mon.a (mon.0) 2432 : audit [DBG] from='client.? 172.21.15.6:0/3297429759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:30.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:30 smithi006 bash[27292]: cluster 2023-08-27T23:37:29.334138+0000 mgr.y (mgr.14901) 2968 : cluster [DBG] pgmap v2975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:30.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:30 smithi006 bash[27292]: audit 2023-08-27T23:37:29.784523+0000 mon.a (mon.0) 2432 : audit [DBG] from='client.? 172.21.15.6:0/3297429759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:30.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:37:30.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:30.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:37:30.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:31.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:30 smithi130 bash[51302]: debug 2023-08-27T23:37:30.654+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:30.655957+0000) 2023-08-27T23:37:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:37:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:37:30] "GET /metrics HTTP/1.1" 200 193110 "" "Prometheus/2.33.4" 2023-08-27T23:37:32.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:31 smithi130 bash[51302]: debug 2023-08-27T23:37:31.654+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:31.656215+0000) 2023-08-27T23:37:32.399 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:37:32.082Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:32.399 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:37:32.083Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:32.657 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:32 smithi130 bash[27579]: cluster 2023-08-27T23:37:31.334847+0000 mgr.y (mgr.14901) 2969 : cluster [DBG] pgmap v2976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:32.657 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:32 smithi130 bash[27579]: audit 2023-08-27T23:37:32.204398+0000 mon.c (mon.2) 1149 : audit [DBG] from='client.? 172.21.15.6:0/3784853070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:32 smithi006 bash[27292]: cluster 2023-08-27T23:37:31.334847+0000 mgr.y (mgr.14901) 2969 : cluster [DBG] pgmap v2976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:32 smithi006 bash[27292]: audit 2023-08-27T23:37:32.204398+0000 mon.c (mon.2) 1149 : audit [DBG] from='client.? 172.21.15.6:0/3784853070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:32.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:32 smithi006 bash[33918]: cluster 2023-08-27T23:37:31.334847+0000 mgr.y (mgr.14901) 2969 : cluster [DBG] pgmap v2976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:32 smithi006 bash[33918]: audit 2023-08-27T23:37:32.204398+0000 mon.c (mon.2) 1149 : audit [DBG] from='client.? 172.21.15.6:0/3784853070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:33.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:32 smithi130 bash[51302]: debug 2023-08-27T23:37:32.654+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:32.656538+0000) 2023-08-27T23:37:34.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:33 smithi130 bash[51302]: debug 2023-08-27T23:37:33.654+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:33.656800+0000) 2023-08-27T23:37:34.404 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:37:33.981Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:34.404 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:37:33.983Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:34.658 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:34 smithi130 bash[27579]: cluster 2023-08-27T23:37:33.335497+0000 mgr.y (mgr.14901) 2970 : cluster [DBG] pgmap v2977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:34 smithi006 bash[27292]: cluster 2023-08-27T23:37:33.335497+0000 mgr.y (mgr.14901) 2970 : cluster [DBG] pgmap v2977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:34 smithi006 bash[33918]: cluster 2023-08-27T23:37:33.335497+0000 mgr.y (mgr.14901) 2970 : cluster [DBG] pgmap v2977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:35.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:34 smithi130 bash[51302]: debug 2023-08-27T23:37:34.654+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:34.657115+0000) 2023-08-27T23:37:35.658 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:35 smithi130 bash[27579]: audit 2023-08-27T23:37:34.627011+0000 mon.a (mon.0) 2433 : audit [DBG] from='client.? 172.21.15.6:0/3194546358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:35.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:35 smithi006 bash[33918]: audit 2023-08-27T23:37:34.627011+0000 mon.a (mon.0) 2433 : audit [DBG] from='client.? 172.21.15.6:0/3194546358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:35.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:35 smithi006 bash[27292]: audit 2023-08-27T23:37:34.627011+0000 mon.a (mon.0) 2433 : audit [DBG] from='client.? 172.21.15.6:0/3194546358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:36.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:35 smithi130 bash[51302]: debug 2023-08-27T23:37:35.654+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:35.657426+0000) 2023-08-27T23:37:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:36 smithi130 bash[27579]: cluster 2023-08-27T23:37:35.336565+0000 mgr.y (mgr.14901) 2971 : cluster [DBG] pgmap v2978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:37.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:36 smithi130 bash[51302]: debug 2023-08-27T23:37:36.654+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:36.657716+0000) 2023-08-27T23:37:37.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:36 smithi006 bash[27292]: cluster 2023-08-27T23:37:35.336565+0000 mgr.y (mgr.14901) 2971 : cluster [DBG] pgmap v2978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:37.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:36 smithi006 bash[33918]: cluster 2023-08-27T23:37:35.336565+0000 mgr.y (mgr.14901) 2971 : cluster [DBG] pgmap v2978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:38.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:37 smithi130 bash[27579]: audit 2023-08-27T23:37:37.044812+0000 mon.c (mon.2) 1150 : audit [DBG] from='client.? 172.21.15.6:0/589687247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:38.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:37 smithi130 bash[51302]: debug 2023-08-27T23:37:37.654+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:37.658016+0000) 2023-08-27T23:37:38.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:37 smithi006 bash[33918]: audit 2023-08-27T23:37:37.044812+0000 mon.c (mon.2) 1150 : audit [DBG] from='client.? 172.21.15.6:0/589687247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:38.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:37 smithi006 bash[27292]: audit 2023-08-27T23:37:37.044812+0000 mon.c (mon.2) 1150 : audit [DBG] from='client.? 172.21.15.6:0/589687247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:38 smithi130 bash[27579]: cluster 2023-08-27T23:37:37.337261+0000 mgr.y (mgr.14901) 2972 : cluster [DBG] pgmap v2979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:39.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:38 smithi130 bash[51302]: debug 2023-08-27T23:37:38.654+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:38.658336+0000) 2023-08-27T23:37:39.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:38 smithi006 bash[33918]: cluster 2023-08-27T23:37:37.337261+0000 mgr.y (mgr.14901) 2972 : cluster [DBG] pgmap v2979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:39.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:38 smithi006 bash[27292]: cluster 2023-08-27T23:37:37.337261+0000 mgr.y (mgr.14901) 2972 : cluster [DBG] pgmap v2979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:40.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:39 smithi130 bash[27579]: audit 2023-08-27T23:37:39.454683+0000 mon.a (mon.0) 2434 : audit [DBG] from='client.? 172.21.15.6:0/4070363463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:40.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:39 smithi130 bash[51302]: debug 2023-08-27T23:37:39.654+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:39.658653+0000) 2023-08-27T23:37:40.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:39 smithi006 bash[33918]: audit 2023-08-27T23:37:39.454683+0000 mon.a (mon.0) 2434 : audit [DBG] from='client.? 172.21.15.6:0/4070363463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:40.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:39 smithi006 bash[27292]: audit 2023-08-27T23:37:39.454683+0000 mon.a (mon.0) 2434 : audit [DBG] from='client.? 172.21.15.6:0/4070363463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:40.908 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:37:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:37:40] "GET /metrics HTTP/1.1" 200 193110 "" "Prometheus/2.33.4" 2023-08-27T23:37:40.908 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:37:40.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:40.909 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:37:40.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:40 smithi130 bash[27579]: cluster 2023-08-27T23:37:39.338340+0000 mgr.y (mgr.14901) 2973 : cluster [DBG] pgmap v2980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:41.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:40 smithi130 bash[51302]: debug 2023-08-27T23:37:40.658+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:40.658953+0000) 2023-08-27T23:37:41.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:40 smithi006 bash[33918]: cluster 2023-08-27T23:37:39.338340+0000 mgr.y (mgr.14901) 2973 : cluster [DBG] pgmap v2980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:41.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:40 smithi006 bash[27292]: cluster 2023-08-27T23:37:39.338340+0000 mgr.y (mgr.14901) 2973 : cluster [DBG] pgmap v2980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:41.916 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:41 smithi130 bash[51302]: debug 2023-08-27T23:37:41.658+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:41.659207+0000) 2023-08-27T23:37:42.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:41 smithi006 bash[27292]: cluster 2023-08-27T23:37:41.339013+0000 mgr.y (mgr.14901) 2974 : cluster [DBG] pgmap v2981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:42.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:41 smithi006 bash[27292]: audit 2023-08-27T23:37:41.871951+0000 mon.c (mon.2) 1151 : audit [DBG] from='client.? 172.21.15.6:0/1517704786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:42.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:37:42.083Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:42.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:37:42.084Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:42.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:41 smithi006 bash[33918]: cluster 2023-08-27T23:37:41.339013+0000 mgr.y (mgr.14901) 2974 : cluster [DBG] pgmap v2981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:42.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:41 smithi006 bash[33918]: audit 2023-08-27T23:37:41.871951+0000 mon.c (mon.2) 1151 : audit [DBG] from='client.? 172.21.15.6:0/1517704786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:42.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:41 smithi130 bash[27579]: cluster 2023-08-27T23:37:41.339013+0000 mgr.y (mgr.14901) 2974 : cluster [DBG] pgmap v2981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:41 smithi130 bash[27579]: audit 2023-08-27T23:37:41.871951+0000 mon.c (mon.2) 1151 : audit [DBG] from='client.? 172.21.15.6:0/1517704786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:43.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:42 smithi130 bash[51302]: debug 2023-08-27T23:37:42.658+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:42.659550+0000) 2023-08-27T23:37:44.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:43 smithi130 bash[51302]: debug 2023-08-27T23:37:43.658+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:43.659848+0000) 2023-08-27T23:37:44.399 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:37:43.982Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:44.399 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:37:43.984Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:44.661 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:44 smithi130 bash[27579]: cluster 2023-08-27T23:37:43.339651+0000 mgr.y (mgr.14901) 2975 : cluster [DBG] pgmap v2982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:44.661 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:44 smithi130 bash[27579]: audit 2023-08-27T23:37:44.289592+0000 mon.c (mon.2) 1152 : audit [DBG] from='client.? 172.21.15.6:0/2657425262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:44.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:44 smithi006 bash[27292]: cluster 2023-08-27T23:37:43.339651+0000 mgr.y (mgr.14901) 2975 : cluster [DBG] pgmap v2982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:44.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:44 smithi006 bash[27292]: audit 2023-08-27T23:37:44.289592+0000 mon.c (mon.2) 1152 : audit [DBG] from='client.? 172.21.15.6:0/2657425262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:44.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:44 smithi006 bash[33918]: cluster 2023-08-27T23:37:43.339651+0000 mgr.y (mgr.14901) 2975 : cluster [DBG] pgmap v2982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:44.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:44 smithi006 bash[33918]: audit 2023-08-27T23:37:44.289592+0000 mon.c (mon.2) 1152 : audit [DBG] from='client.? 172.21.15.6:0/2657425262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:45.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:44 smithi130 bash[51302]: debug 2023-08-27T23:37:44.658+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:44.660114+0000) 2023-08-27T23:37:46.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:45 smithi130 bash[51302]: debug 2023-08-27T23:37:45.658+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:45.660380+0000) 2023-08-27T23:37:47.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:46 smithi130 bash[51302]: debug 2023-08-27T23:37:46.658+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:46.660631+0000) 2023-08-27T23:37:47.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:46 smithi130 bash[27579]: cluster 2023-08-27T23:37:45.340784+0000 mgr.y (mgr.14901) 2976 : cluster [DBG] pgmap v2983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:47.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:46 smithi130 bash[27579]: audit 2023-08-27T23:37:46.706259+0000 mon.a (mon.0) 2435 : audit [DBG] from='client.? 172.21.15.6:0/1243800047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:47.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:46 smithi006 bash[27292]: cluster 2023-08-27T23:37:45.340784+0000 mgr.y (mgr.14901) 2976 : cluster [DBG] pgmap v2983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:47.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:46 smithi006 bash[27292]: audit 2023-08-27T23:37:46.706259+0000 mon.a (mon.0) 2435 : audit [DBG] from='client.? 172.21.15.6:0/1243800047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:47.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:46 smithi006 bash[33918]: cluster 2023-08-27T23:37:45.340784+0000 mgr.y (mgr.14901) 2976 : cluster [DBG] pgmap v2983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:47.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:46 smithi006 bash[33918]: audit 2023-08-27T23:37:46.706259+0000 mon.a (mon.0) 2435 : audit [DBG] from='client.? 172.21.15.6:0/1243800047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:48.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:47 smithi130 bash[51302]: debug 2023-08-27T23:37:47.658+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:47.660950+0000) 2023-08-27T23:37:49.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:48 smithi130 bash[27579]: cluster 2023-08-27T23:37:47.341514+0000 mgr.y (mgr.14901) 2977 : cluster [DBG] pgmap v2984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:49.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:48 smithi130 bash[51302]: debug 2023-08-27T23:37:48.658+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:48.661242+0000) 2023-08-27T23:37:49.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:48 smithi006 bash[33918]: cluster 2023-08-27T23:37:47.341514+0000 mgr.y (mgr.14901) 2977 : cluster [DBG] pgmap v2984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:49.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:48 smithi006 bash[27292]: cluster 2023-08-27T23:37:47.341514+0000 mgr.y (mgr.14901) 2977 : cluster [DBG] pgmap v2984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:50.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:49 smithi130 bash[27579]: audit 2023-08-27T23:37:49.117101+0000 mon.a (mon.0) 2436 : audit [DBG] from='client.? 172.21.15.6:0/626747972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:50.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:49 smithi130 bash[51302]: debug 2023-08-27T23:37:49.658+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:49.661537+0000) 2023-08-27T23:37:50.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:49 smithi006 bash[33918]: audit 2023-08-27T23:37:49.117101+0000 mon.a (mon.0) 2436 : audit [DBG] from='client.? 172.21.15.6:0/626747972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:50.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:49 smithi006 bash[27292]: audit 2023-08-27T23:37:49.117101+0000 mon.a (mon.0) 2436 : audit [DBG] from='client.? 172.21.15.6:0/626747972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:50.900 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:37:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:37:50] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-27T23:37:50.900 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:37:50.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:50.900 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:37:50.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:50 smithi130 bash[27579]: cluster 2023-08-27T23:37:49.342732+0000 mgr.y (mgr.14901) 2978 : cluster [DBG] pgmap v2985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:51.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:50 smithi130 bash[51302]: debug 2023-08-27T23:37:50.658+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:50.661842+0000) 2023-08-27T23:37:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:50 smithi006 bash[27292]: cluster 2023-08-27T23:37:49.342732+0000 mgr.y (mgr.14901) 2978 : cluster [DBG] pgmap v2985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:51.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:50 smithi006 bash[33918]: cluster 2023-08-27T23:37:49.342732+0000 mgr.y (mgr.14901) 2978 : cluster [DBG] pgmap v2985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:52.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:51 smithi130 bash[51302]: debug 2023-08-27T23:37:51.658+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:51.662111+0000) 2023-08-27T23:37:52.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:51 smithi130 bash[27579]: cluster 2023-08-27T23:37:51.343345+0000 mgr.y (mgr.14901) 2979 : cluster [DBG] pgmap v2986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:52.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:51 smithi130 bash[27579]: audit 2023-08-27T23:37:51.536466+0000 mon.a (mon.0) 2437 : audit [DBG] from='client.? 172.21.15.6:0/1450639944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:52.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:51 smithi006 bash[33918]: cluster 2023-08-27T23:37:51.343345+0000 mgr.y (mgr.14901) 2979 : cluster [DBG] pgmap v2986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:52.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:51 smithi006 bash[33918]: audit 2023-08-27T23:37:51.536466+0000 mon.a (mon.0) 2437 : audit [DBG] from='client.? 172.21.15.6:0/1450639944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:52.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:37:52.083Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:52.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:37:52.085Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:52.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:51 smithi006 bash[27292]: cluster 2023-08-27T23:37:51.343345+0000 mgr.y (mgr.14901) 2979 : cluster [DBG] pgmap v2986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:52.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:51 smithi006 bash[27292]: audit 2023-08-27T23:37:51.536466+0000 mon.a (mon.0) 2437 : audit [DBG] from='client.? 172.21.15.6:0/1450639944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:52 smithi130 bash[51302]: debug 2023-08-27T23:37:52.658+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:52.662473+0000) 2023-08-27T23:37:54.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:53 smithi130 bash[51302]: debug 2023-08-27T23:37:53.658+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:53.662753+0000) 2023-08-27T23:37:54.404 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:37:53.983Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:54.404 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:37:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:37:53.985Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:37:54.664 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:54 smithi130 bash[27579]: cluster 2023-08-27T23:37:53.344015+0000 mgr.y (mgr.14901) 2980 : cluster [DBG] pgmap v2987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:54.664 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:54 smithi130 bash[27579]: audit 2023-08-27T23:37:53.955698+0000 mon.a (mon.0) 2438 : audit [DBG] from='client.? 172.21.15.6:0/2388273662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:54.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:54 smithi006 bash[33918]: cluster 2023-08-27T23:37:53.344015+0000 mgr.y (mgr.14901) 2980 : cluster [DBG] pgmap v2987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:54.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:54 smithi006 bash[33918]: audit 2023-08-27T23:37:53.955698+0000 mon.a (mon.0) 2438 : audit [DBG] from='client.? 172.21.15.6:0/2388273662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:54 smithi006 bash[27292]: cluster 2023-08-27T23:37:53.344015+0000 mgr.y (mgr.14901) 2980 : cluster [DBG] pgmap v2987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:54 smithi006 bash[27292]: audit 2023-08-27T23:37:53.955698+0000 mon.a (mon.0) 2438 : audit [DBG] from='client.? 172.21.15.6:0/2388273662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:55.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:54 smithi130 bash[51302]: debug 2023-08-27T23:37:54.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:54.663056+0000) 2023-08-27T23:37:56.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:55 smithi130 bash[51302]: debug 2023-08-27T23:37:55.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:55.663357+0000) 2023-08-27T23:37:57.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:56 smithi130 bash[27579]: cluster 2023-08-27T23:37:55.345249+0000 mgr.y (mgr.14901) 2981 : cluster [DBG] pgmap v2988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:57.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:56 smithi130 bash[27579]: audit 2023-08-27T23:37:56.369357+0000 mon.a (mon.0) 2439 : audit [DBG] from='client.? 172.21.15.6:0/915487987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:57.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:56 smithi130 bash[51302]: debug 2023-08-27T23:37:56.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:56.663592+0000) 2023-08-27T23:37:57.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:56 smithi006 bash[33918]: cluster 2023-08-27T23:37:55.345249+0000 mgr.y (mgr.14901) 2981 : cluster [DBG] pgmap v2988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:57.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:56 smithi006 bash[33918]: audit 2023-08-27T23:37:56.369357+0000 mon.a (mon.0) 2439 : audit [DBG] from='client.? 172.21.15.6:0/915487987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:57.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:56 smithi006 bash[27292]: cluster 2023-08-27T23:37:55.345249+0000 mgr.y (mgr.14901) 2981 : cluster [DBG] pgmap v2988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:57.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:56 smithi006 bash[27292]: audit 2023-08-27T23:37:56.369357+0000 mon.a (mon.0) 2439 : audit [DBG] from='client.? 172.21.15.6:0/915487987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:58.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:57 smithi130 bash[51302]: debug 2023-08-27T23:37:57.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:57.663895+0000) 2023-08-27T23:37:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:58 smithi130 bash[27579]: cluster 2023-08-27T23:37:57.345944+0000 mgr.y (mgr.14901) 2982 : cluster [DBG] pgmap v2989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:37:58 smithi130 bash[27579]: audit 2023-08-27T23:37:58.784014+0000 mon.a (mon.0) 2440 : audit [DBG] from='client.? 172.21.15.6:0/2265002511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:59.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:58 smithi130 bash[51302]: debug 2023-08-27T23:37:58.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:58.664233+0000) 2023-08-27T23:37:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:58 smithi006 bash[33918]: cluster 2023-08-27T23:37:57.345944+0000 mgr.y (mgr.14901) 2982 : cluster [DBG] pgmap v2989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:37:58 smithi006 bash[33918]: audit 2023-08-27T23:37:58.784014+0000 mon.a (mon.0) 2440 : audit [DBG] from='client.? 172.21.15.6:0/2265002511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:59.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:58 smithi006 bash[27292]: cluster 2023-08-27T23:37:57.345944+0000 mgr.y (mgr.14901) 2982 : cluster [DBG] pgmap v2989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:37:59.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:37:58 smithi006 bash[27292]: audit 2023-08-27T23:37:58.784014+0000 mon.a (mon.0) 2440 : audit [DBG] from='client.? 172.21.15.6:0/2265002511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:00.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:37:59 smithi130 bash[51302]: debug 2023-08-27T23:37:59.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:59.664532+0000) 2023-08-27T23:38:00.902 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:38:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:38:00] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-27T23:38:00.903 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:38:00.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:00.903 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:38:00.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:00 smithi130 bash[27579]: cluster 2023-08-27T23:37:59.346998+0000 mgr.y (mgr.14901) 2983 : cluster [DBG] pgmap v2990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:01.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:00 smithi130 bash[51302]: debug 2023-08-27T23:38:00.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:00.664853+0000) 2023-08-27T23:38:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:00 smithi006 bash[33918]: cluster 2023-08-27T23:37:59.346998+0000 mgr.y (mgr.14901) 2983 : cluster [DBG] pgmap v2990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:00 smithi006 bash[27292]: cluster 2023-08-27T23:37:59.346998+0000 mgr.y (mgr.14901) 2983 : cluster [DBG] pgmap v2990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:02.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:01 smithi130 bash[27579]: audit 2023-08-27T23:38:01.197245+0000 mon.a (mon.0) 2441 : audit [DBG] from='client.? 172.21.15.6:0/3560748639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:02.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:01 smithi130 bash[51302]: debug 2023-08-27T23:38:01.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:01.665252+0000) 2023-08-27T23:38:02.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:01 smithi006 bash[27292]: audit 2023-08-27T23:38:01.197245+0000 mon.a (mon.0) 2441 : audit [DBG] from='client.? 172.21.15.6:0/3560748639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:02.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:01 smithi006 bash[33918]: audit 2023-08-27T23:38:01.197245+0000 mon.a (mon.0) 2441 : audit [DBG] from='client.? 172.21.15.6:0/3560748639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:02.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:38:02.084Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:02.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:38:02.086Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:02 smithi130 bash[27579]: cluster 2023-08-27T23:38:01.347673+0000 mgr.y (mgr.14901) 2984 : cluster [DBG] pgmap v2991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:02 smithi130 bash[51302]: debug 2023-08-27T23:38:02.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:02.665482+0000) 2023-08-27T23:38:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:02 smithi006 bash[33918]: cluster 2023-08-27T23:38:01.347673+0000 mgr.y (mgr.14901) 2984 : cluster [DBG] pgmap v2991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:03.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:02 smithi006 bash[27292]: cluster 2023-08-27T23:38:01.347673+0000 mgr.y (mgr.14901) 2984 : cluster [DBG] pgmap v2991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:03.918 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:03 smithi130 bash[51302]: debug 2023-08-27T23:38:03.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:03.665767+0000) 2023-08-27T23:38:04.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:03 smithi006 bash[33918]: cluster 2023-08-27T23:38:03.348095+0000 mgr.y (mgr.14901) 2985 : cluster [DBG] pgmap v2992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:04.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:03 smithi006 bash[33918]: audit 2023-08-27T23:38:03.611130+0000 mon.a (mon.0) 2442 : audit [DBG] from='client.? 172.21.15.6:0/1803578487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:04.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:38:03.984Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:04.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:38:03.986Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:04.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:03 smithi006 bash[27292]: cluster 2023-08-27T23:38:03.348095+0000 mgr.y (mgr.14901) 2985 : cluster [DBG] pgmap v2992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:04.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:03 smithi006 bash[27292]: audit 2023-08-27T23:38:03.611130+0000 mon.a (mon.0) 2442 : audit [DBG] from='client.? 172.21.15.6:0/1803578487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:04.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:03 smithi130 bash[27579]: cluster 2023-08-27T23:38:03.348095+0000 mgr.y (mgr.14901) 2985 : cluster [DBG] pgmap v2992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:04.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:03 smithi130 bash[27579]: audit 2023-08-27T23:38:03.611130+0000 mon.a (mon.0) 2442 : audit [DBG] from='client.? 172.21.15.6:0/1803578487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:05.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:04 smithi130 bash[51302]: debug 2023-08-27T23:38:04.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:04.666054+0000) 2023-08-27T23:38:06.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:05 smithi130 bash[51302]: debug 2023-08-27T23:38:05.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:05.666325+0000) 2023-08-27T23:38:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:06 smithi130 bash[27579]: cluster 2023-08-27T23:38:05.349195+0000 mgr.y (mgr.14901) 2986 : cluster [DBG] pgmap v2993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:06 smithi130 bash[27579]: audit 2023-08-27T23:38:06.027350+0000 mon.a (mon.0) 2443 : audit [DBG] from='client.? 172.21.15.6:0/1180264038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:07.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:06 smithi130 bash[51302]: debug 2023-08-27T23:38:06.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:06.666570+0000) 2023-08-27T23:38:07.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:06 smithi006 bash[33918]: cluster 2023-08-27T23:38:05.349195+0000 mgr.y (mgr.14901) 2986 : cluster [DBG] pgmap v2993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:07.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:06 smithi006 bash[33918]: audit 2023-08-27T23:38:06.027350+0000 mon.a (mon.0) 2443 : audit [DBG] from='client.? 172.21.15.6:0/1180264038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:07.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:06 smithi006 bash[27292]: cluster 2023-08-27T23:38:05.349195+0000 mgr.y (mgr.14901) 2986 : cluster [DBG] pgmap v2993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:07.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:06 smithi006 bash[27292]: audit 2023-08-27T23:38:06.027350+0000 mon.a (mon.0) 2443 : audit [DBG] from='client.? 172.21.15.6:0/1180264038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:08.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:07 smithi130 bash[51302]: debug 2023-08-27T23:38:07.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:07.666884+0000) 2023-08-27T23:38:09.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:08 smithi130 bash[51302]: debug 2023-08-27T23:38:08.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:08.667282+0000) 2023-08-27T23:38:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:08 smithi130 bash[27579]: cluster 2023-08-27T23:38:07.349852+0000 mgr.y (mgr.14901) 2987 : cluster [DBG] pgmap v2994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:08 smithi130 bash[27579]: audit 2023-08-27T23:38:08.434054+0000 mon.a (mon.0) 2444 : audit [DBG] from='client.? 172.21.15.6:0/1847552399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:08 smithi006 bash[33918]: cluster 2023-08-27T23:38:07.349852+0000 mgr.y (mgr.14901) 2987 : cluster [DBG] pgmap v2994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:08 smithi006 bash[33918]: audit 2023-08-27T23:38:08.434054+0000 mon.a (mon.0) 2444 : audit [DBG] from='client.? 172.21.15.6:0/1847552399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:09.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:08 smithi006 bash[27292]: cluster 2023-08-27T23:38:07.349852+0000 mgr.y (mgr.14901) 2987 : cluster [DBG] pgmap v2994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:09.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:08 smithi006 bash[27292]: audit 2023-08-27T23:38:08.434054+0000 mon.a (mon.0) 2444 : audit [DBG] from='client.? 172.21.15.6:0/1847552399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:10.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:09 smithi130 bash[51302]: debug 2023-08-27T23:38:09.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:09.667542+0000) 2023-08-27T23:38:10.901 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:38:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:38:10] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-27T23:38:10.901 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:38:10.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:10.902 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:38:10.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:10 smithi130 bash[27579]: cluster 2023-08-27T23:38:09.350930+0000 mgr.y (mgr.14901) 2988 : cluster [DBG] pgmap v2995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:10 smithi130 bash[27579]: audit 2023-08-27T23:38:10.856648+0000 mon.a (mon.0) 2445 : audit [DBG] from='client.? 172.21.15.6:0/3026520666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:11.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:10 smithi130 bash[51302]: debug 2023-08-27T23:38:10.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:10.667819+0000) 2023-08-27T23:38:11.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:10 smithi006 bash[27292]: cluster 2023-08-27T23:38:09.350930+0000 mgr.y (mgr.14901) 2988 : cluster [DBG] pgmap v2995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:11.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:10 smithi006 bash[27292]: audit 2023-08-27T23:38:10.856648+0000 mon.a (mon.0) 2445 : audit [DBG] from='client.? 172.21.15.6:0/3026520666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:11.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:10 smithi006 bash[33918]: cluster 2023-08-27T23:38:09.350930+0000 mgr.y (mgr.14901) 2988 : cluster [DBG] pgmap v2995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:11.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:10 smithi006 bash[33918]: audit 2023-08-27T23:38:10.856648+0000 mon.a (mon.0) 2445 : audit [DBG] from='client.? 172.21.15.6:0/3026520666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:11.959 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:11 smithi130 bash[51302]: debug 2023-08-27T23:38:11.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:11.668106+0000) 2023-08-27T23:38:12.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:11 smithi006 bash[27292]: cluster 2023-08-27T23:38:11.351607+0000 mgr.y (mgr.14901) 2989 : cluster [DBG] pgmap v2996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:12.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:11 smithi006 bash[33918]: cluster 2023-08-27T23:38:11.351607+0000 mgr.y (mgr.14901) 2989 : cluster [DBG] pgmap v2996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:12.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:38:12.085Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:12.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:38:12.086Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:12.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:11 smithi130 bash[27579]: cluster 2023-08-27T23:38:11.351607+0000 mgr.y (mgr.14901) 2989 : cluster [DBG] pgmap v2996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:12.961 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:12 smithi130 bash[51302]: debug 2023-08-27T23:38:12.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:12.668373+0000) 2023-08-27T23:38:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:12 smithi006 bash[33918]: audit 2023-08-27T23:38:12.068549+0000 mon.b (mon.1) 403 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:38:13.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:12 smithi006 bash[27292]: audit 2023-08-27T23:38:12.068549+0000 mon.b (mon.1) 403 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:38:13.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:12 smithi130 bash[27579]: audit 2023-08-27T23:38:12.068549+0000 mon.b (mon.1) 403 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:38:13.969 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:13 smithi130 bash[51302]: debug 2023-08-27T23:38:13.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:13.668677+0000) 2023-08-27T23:38:14.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:13 smithi130 bash[27579]: audit 2023-08-27T23:38:13.272615+0000 mon.c (mon.2) 1153 : audit [DBG] from='client.? 172.21.15.6:0/2544259647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:14.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:13 smithi130 bash[27579]: cluster 2023-08-27T23:38:13.352216+0000 mgr.y (mgr.14901) 2990 : cluster [DBG] pgmap v2997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:14.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:13 smithi006 bash[33918]: audit 2023-08-27T23:38:13.272615+0000 mon.c (mon.2) 1153 : audit [DBG] from='client.? 172.21.15.6:0/2544259647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:14.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:13 smithi006 bash[33918]: cluster 2023-08-27T23:38:13.352216+0000 mgr.y (mgr.14901) 2990 : cluster [DBG] pgmap v2997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:38:13.984Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:38:13.986Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:14.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:13 smithi006 bash[27292]: audit 2023-08-27T23:38:13.272615+0000 mon.c (mon.2) 1153 : audit [DBG] from='client.? 172.21.15.6:0/2544259647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:14.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:13 smithi006 bash[27292]: cluster 2023-08-27T23:38:13.352216+0000 mgr.y (mgr.14901) 2990 : cluster [DBG] pgmap v2997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:15.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:14 smithi130 bash[51302]: debug 2023-08-27T23:38:14.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:14.668951+0000) 2023-08-27T23:38:16.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:15 smithi130 bash[27579]: audit 2023-08-27T23:38:15.700666+0000 mon.a (mon.0) 2446 : audit [DBG] from='client.? 172.21.15.6:0/3074042197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:16.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:15 smithi130 bash[51302]: debug 2023-08-27T23:38:15.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:15.669250+0000) 2023-08-27T23:38:16.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:15 smithi006 bash[33918]: audit 2023-08-27T23:38:15.700666+0000 mon.a (mon.0) 2446 : audit [DBG] from='client.? 172.21.15.6:0/3074042197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:16.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:15 smithi006 bash[27292]: audit 2023-08-27T23:38:15.700666+0000 mon.a (mon.0) 2446 : audit [DBG] from='client.? 172.21.15.6:0/3074042197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:16 smithi130 bash[27579]: cluster 2023-08-27T23:38:15.353046+0000 mgr.y (mgr.14901) 2991 : cluster [DBG] pgmap v2998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:17.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:16 smithi130 bash[51302]: debug 2023-08-27T23:38:16.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:16.669512+0000) 2023-08-27T23:38:17.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:16 smithi006 bash[33918]: cluster 2023-08-27T23:38:15.353046+0000 mgr.y (mgr.14901) 2991 : cluster [DBG] pgmap v2998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:17.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:16 smithi006 bash[27292]: cluster 2023-08-27T23:38:15.353046+0000 mgr.y (mgr.14901) 2991 : cluster [DBG] pgmap v2998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:18.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:17 smithi130 bash[51302]: debug 2023-08-27T23:38:17.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:17.669791+0000) 2023-08-27T23:38:19.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:18 smithi130 bash[51302]: debug 2023-08-27T23:38:18.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:18.670062+0000) 2023-08-27T23:38:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:18 smithi130 bash[27579]: cluster 2023-08-27T23:38:17.353578+0000 mgr.y (mgr.14901) 2992 : cluster [DBG] pgmap v2999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:18 smithi130 bash[27579]: audit 2023-08-27T23:38:17.952569+0000 mon.a (mon.0) 2447 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:38:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:18 smithi130 bash[27579]: audit 2023-08-27T23:38:18.149045+0000 mon.c (mon.2) 1154 : audit [DBG] from='client.? 172.21.15.6:0/797309270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:18 smithi130 bash[27579]: audit 2023-08-27T23:38:18.426266+0000 mon.a (mon.0) 2448 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:38:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:18 smithi130 bash[27579]: audit 2023-08-27T23:38:18.724717+0000 mon.b (mon.1) 404 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:38:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:18 smithi130 bash[27579]: audit 2023-08-27T23:38:18.727103+0000 mon.b (mon.1) 405 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:38:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:18 smithi130 bash[27579]: audit 2023-08-27T23:38:18.734417+0000 mon.a (mon.0) 2449 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:38:19.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:18 smithi006 bash[33918]: cluster 2023-08-27T23:38:17.353578+0000 mgr.y (mgr.14901) 2992 : cluster [DBG] pgmap v2999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:19.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:18 smithi006 bash[33918]: audit 2023-08-27T23:38:17.952569+0000 mon.a (mon.0) 2447 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:38:19.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:18 smithi006 bash[33918]: audit 2023-08-27T23:38:18.149045+0000 mon.c (mon.2) 1154 : audit [DBG] from='client.? 172.21.15.6:0/797309270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:19.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:18 smithi006 bash[33918]: audit 2023-08-27T23:38:18.426266+0000 mon.a (mon.0) 2448 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:38:19.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:18 smithi006 bash[33918]: audit 2023-08-27T23:38:18.724717+0000 mon.b (mon.1) 404 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:38:19.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:18 smithi006 bash[33918]: audit 2023-08-27T23:38:18.727103+0000 mon.b (mon.1) 405 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:38:19.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:18 smithi006 bash[33918]: audit 2023-08-27T23:38:18.734417+0000 mon.a (mon.0) 2449 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:38:19.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:18 smithi006 bash[27292]: cluster 2023-08-27T23:38:17.353578+0000 mgr.y (mgr.14901) 2992 : cluster [DBG] pgmap v2999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:19.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:18 smithi006 bash[27292]: audit 2023-08-27T23:38:17.952569+0000 mon.a (mon.0) 2447 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:38:19.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:18 smithi006 bash[27292]: audit 2023-08-27T23:38:18.149045+0000 mon.c (mon.2) 1154 : audit [DBG] from='client.? 172.21.15.6:0/797309270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:19.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:18 smithi006 bash[27292]: audit 2023-08-27T23:38:18.426266+0000 mon.a (mon.0) 2448 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:38:19.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:18 smithi006 bash[27292]: audit 2023-08-27T23:38:18.724717+0000 mon.b (mon.1) 404 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:38:19.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:18 smithi006 bash[27292]: audit 2023-08-27T23:38:18.727103+0000 mon.b (mon.1) 405 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:38:19.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:18 smithi006 bash[27292]: audit 2023-08-27T23:38:18.734417+0000 mon.a (mon.0) 2449 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:38:20.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:19 smithi130 bash[51302]: debug 2023-08-27T23:38:19.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:19.670331+0000) 2023-08-27T23:38:20.893 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:38:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:38:20] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-27T23:38:20.894 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:38:20.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:20.894 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:38:20.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:21.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:20 smithi130 bash[51302]: debug 2023-08-27T23:38:20.670+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:20.670604+0000) 2023-08-27T23:38:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:20 smithi130 bash[27579]: cluster 2023-08-27T23:38:19.354638+0000 mgr.y (mgr.14901) 2993 : cluster [DBG] pgmap v3000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:20 smithi130 bash[27579]: audit 2023-08-27T23:38:20.578283+0000 mon.c (mon.2) 1155 : audit [DBG] from='client.? 172.21.15.6:0/1760962753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:21.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:20 smithi006 bash[33918]: cluster 2023-08-27T23:38:19.354638+0000 mgr.y (mgr.14901) 2993 : cluster [DBG] pgmap v3000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:21.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:20 smithi006 bash[33918]: audit 2023-08-27T23:38:20.578283+0000 mon.c (mon.2) 1155 : audit [DBG] from='client.? 172.21.15.6:0/1760962753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:21.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:20 smithi006 bash[27292]: cluster 2023-08-27T23:38:19.354638+0000 mgr.y (mgr.14901) 2993 : cluster [DBG] pgmap v3000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:21.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:20 smithi006 bash[27292]: audit 2023-08-27T23:38:20.578283+0000 mon.c (mon.2) 1155 : audit [DBG] from='client.? 172.21.15.6:0/1760962753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:22.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:21 smithi130 bash[51302]: debug 2023-08-27T23:38:21.669+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:21.670925+0000) 2023-08-27T23:38:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:38:22.086Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:38:22.087Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:23.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:22 smithi130 bash[27579]: cluster 2023-08-27T23:38:21.355295+0000 mgr.y (mgr.14901) 2994 : cluster [DBG] pgmap v3001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:23.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:22 smithi130 bash[51302]: debug 2023-08-27T23:38:22.669+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:22.671261+0000) 2023-08-27T23:38:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:22 smithi006 bash[27292]: cluster 2023-08-27T23:38:21.355295+0000 mgr.y (mgr.14901) 2994 : cluster [DBG] pgmap v3001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:23.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:22 smithi006 bash[33918]: cluster 2023-08-27T23:38:21.355295+0000 mgr.y (mgr.14901) 2994 : cluster [DBG] pgmap v3001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:24.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:23 smithi130 bash[27579]: audit 2023-08-27T23:38:23.004636+0000 mon.a (mon.0) 2450 : audit [DBG] from='client.? 172.21.15.6:0/880284067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:24.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:23 smithi130 bash[51302]: debug 2023-08-27T23:38:23.669+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:23.671569+0000) 2023-08-27T23:38:24.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:23 smithi006 bash[27292]: audit 2023-08-27T23:38:23.004636+0000 mon.a (mon.0) 2450 : audit [DBG] from='client.? 172.21.15.6:0/880284067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:24.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:23 smithi006 bash[33918]: audit 2023-08-27T23:38:23.004636+0000 mon.a (mon.0) 2450 : audit [DBG] from='client.? 172.21.15.6:0/880284067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:24.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:38:23.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:24.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:38:23.988Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:24 smithi130 bash[27579]: cluster 2023-08-27T23:38:23.355950+0000 mgr.y (mgr.14901) 2995 : cluster [DBG] pgmap v3002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:24 smithi130 bash[51302]: debug 2023-08-27T23:38:24.669+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:24.671865+0000) 2023-08-27T23:38:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:24 smithi006 bash[27292]: cluster 2023-08-27T23:38:23.355950+0000 mgr.y (mgr.14901) 2995 : cluster [DBG] pgmap v3002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:24 smithi006 bash[33918]: cluster 2023-08-27T23:38:23.355950+0000 mgr.y (mgr.14901) 2995 : cluster [DBG] pgmap v3002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:25.923 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:25 smithi130 bash[51302]: debug 2023-08-27T23:38:25.669+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:25.672149+0000) 2023-08-27T23:38:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:25 smithi006 bash[27292]: cluster 2023-08-27T23:38:25.357063+0000 mgr.y (mgr.14901) 2996 : cluster [DBG] pgmap v3003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:25 smithi006 bash[27292]: audit 2023-08-27T23:38:25.421352+0000 mon.c (mon.2) 1156 : audit [DBG] from='client.? 172.21.15.6:0/3533542333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:26.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:25 smithi006 bash[33918]: cluster 2023-08-27T23:38:25.357063+0000 mgr.y (mgr.14901) 2996 : cluster [DBG] pgmap v3003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:26.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:25 smithi006 bash[33918]: audit 2023-08-27T23:38:25.421352+0000 mon.c (mon.2) 1156 : audit [DBG] from='client.? 172.21.15.6:0/3533542333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:26.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:25 smithi130 bash[27579]: cluster 2023-08-27T23:38:25.357063+0000 mgr.y (mgr.14901) 2996 : cluster [DBG] pgmap v3003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:25 smithi130 bash[27579]: audit 2023-08-27T23:38:25.421352+0000 mon.c (mon.2) 1156 : audit [DBG] from='client.? 172.21.15.6:0/3533542333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:27.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:26 smithi130 bash[51302]: debug 2023-08-27T23:38:26.669+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:26.672446+0000) 2023-08-27T23:38:28.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:27 smithi130 bash[51302]: debug 2023-08-27T23:38:27.669+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:27.672710+0000) 2023-08-27T23:38:28.674 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:28 smithi130 bash[27579]: cluster 2023-08-27T23:38:27.357686+0000 mgr.y (mgr.14901) 2997 : cluster [DBG] pgmap v3004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:28.674 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:28 smithi130 bash[27579]: audit 2023-08-27T23:38:27.829345+0000 mon.c (mon.2) 1157 : audit [DBG] from='client.? 172.21.15.6:0/4076986730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:28 smithi006 bash[33918]: cluster 2023-08-27T23:38:27.357686+0000 mgr.y (mgr.14901) 2997 : cluster [DBG] pgmap v3004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:28 smithi006 bash[33918]: audit 2023-08-27T23:38:27.829345+0000 mon.c (mon.2) 1157 : audit [DBG] from='client.? 172.21.15.6:0/4076986730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:28.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:28 smithi006 bash[27292]: cluster 2023-08-27T23:38:27.357686+0000 mgr.y (mgr.14901) 2997 : cluster [DBG] pgmap v3004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:28.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:28 smithi006 bash[27292]: audit 2023-08-27T23:38:27.829345+0000 mon.c (mon.2) 1157 : audit [DBG] from='client.? 172.21.15.6:0/4076986730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:29.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:28 smithi130 bash[51302]: debug 2023-08-27T23:38:28.669+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:28.672987+0000) 2023-08-27T23:38:30.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:29 smithi130 bash[51302]: debug 2023-08-27T23:38:29.669+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:29.673277+0000) 2023-08-27T23:38:30.898 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:38:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:38:30] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-27T23:38:30.898 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:38:30.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:30.898 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:38:30.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:30 smithi130 bash[27579]: cluster 2023-08-27T23:38:29.358768+0000 mgr.y (mgr.14901) 2998 : cluster [DBG] pgmap v3005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:31.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:30 smithi130 bash[27579]: audit 2023-08-27T23:38:30.257557+0000 mon.c (mon.2) 1158 : audit [DBG] from='client.? 172.21.15.6:0/792706403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:31.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:30 smithi130 bash[51302]: debug 2023-08-27T23:38:30.669+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:30.673565+0000) 2023-08-27T23:38:31.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:30 smithi006 bash[27292]: cluster 2023-08-27T23:38:29.358768+0000 mgr.y (mgr.14901) 2998 : cluster [DBG] pgmap v3005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:31.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:30 smithi006 bash[27292]: audit 2023-08-27T23:38:30.257557+0000 mon.c (mon.2) 1158 : audit [DBG] from='client.? 172.21.15.6:0/792706403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:31.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:30 smithi006 bash[33918]: cluster 2023-08-27T23:38:29.358768+0000 mgr.y (mgr.14901) 2998 : cluster [DBG] pgmap v3005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:31.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:30 smithi006 bash[33918]: audit 2023-08-27T23:38:30.257557+0000 mon.c (mon.2) 1158 : audit [DBG] from='client.? 172.21.15.6:0/792706403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:32.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:31 smithi130 bash[51302]: debug 2023-08-27T23:38:31.669+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:31.673828+0000) 2023-08-27T23:38:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:38:32.087Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:38:32.088Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:32 smithi130 bash[27579]: cluster 2023-08-27T23:38:31.359411+0000 mgr.y (mgr.14901) 2999 : cluster [DBG] pgmap v3006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:32 smithi130 bash[27579]: audit 2023-08-27T23:38:32.673531+0000 mon.a (mon.0) 2451 : audit [DBG] from='client.? 172.21.15.6:0/2297481204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:33.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:32 smithi130 bash[51302]: debug 2023-08-27T23:38:32.669+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:32.674133+0000) 2023-08-27T23:38:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:32 smithi006 bash[33918]: cluster 2023-08-27T23:38:31.359411+0000 mgr.y (mgr.14901) 2999 : cluster [DBG] pgmap v3006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:32 smithi006 bash[33918]: audit 2023-08-27T23:38:32.673531+0000 mon.a (mon.0) 2451 : audit [DBG] from='client.? 172.21.15.6:0/2297481204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:33.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:32 smithi006 bash[27292]: cluster 2023-08-27T23:38:31.359411+0000 mgr.y (mgr.14901) 2999 : cluster [DBG] pgmap v3006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:33.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:32 smithi006 bash[27292]: audit 2023-08-27T23:38:32.673531+0000 mon.a (mon.0) 2451 : audit [DBG] from='client.? 172.21.15.6:0/2297481204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:34.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:33 smithi130 bash[51302]: debug 2023-08-27T23:38:33.673+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:33.674407+0000) 2023-08-27T23:38:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:38:33.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:38:33.988Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:35.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:34 smithi130 bash[27579]: cluster 2023-08-27T23:38:33.360063+0000 mgr.y (mgr.14901) 3000 : cluster [DBG] pgmap v3007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:35.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:34 smithi130 bash[51302]: debug 2023-08-27T23:38:34.673+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:34.674680+0000) 2023-08-27T23:38:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:34 smithi006 bash[33918]: cluster 2023-08-27T23:38:33.360063+0000 mgr.y (mgr.14901) 3000 : cluster [DBG] pgmap v3007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:34 smithi006 bash[27292]: cluster 2023-08-27T23:38:33.360063+0000 mgr.y (mgr.14901) 3000 : cluster [DBG] pgmap v3007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:36.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:35 smithi130 bash[27579]: audit 2023-08-27T23:38:35.094707+0000 mon.a (mon.0) 2452 : audit [DBG] from='client.? 172.21.15.6:0/632361982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:36.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:35 smithi130 bash[27579]: cluster 2023-08-27T23:38:35.361253+0000 mgr.y (mgr.14901) 3001 : cluster [DBG] pgmap v3008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:36.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:35 smithi130 bash[51302]: debug 2023-08-27T23:38:35.673+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:35.674998+0000) 2023-08-27T23:38:36.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:35 smithi006 bash[33918]: audit 2023-08-27T23:38:35.094707+0000 mon.a (mon.0) 2452 : audit [DBG] from='client.? 172.21.15.6:0/632361982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:36.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:35 smithi006 bash[33918]: cluster 2023-08-27T23:38:35.361253+0000 mgr.y (mgr.14901) 3001 : cluster [DBG] pgmap v3008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:36.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:35 smithi006 bash[27292]: audit 2023-08-27T23:38:35.094707+0000 mon.a (mon.0) 2452 : audit [DBG] from='client.? 172.21.15.6:0/632361982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:36.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:35 smithi006 bash[27292]: cluster 2023-08-27T23:38:35.361253+0000 mgr.y (mgr.14901) 3001 : cluster [DBG] pgmap v3008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:37.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:36 smithi130 bash[51302]: debug 2023-08-27T23:38:36.673+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:36.675263+0000) 2023-08-27T23:38:38.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:37 smithi130 bash[51302]: debug 2023-08-27T23:38:37.673+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:37.675525+0000) 2023-08-27T23:38:38.676 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:38 smithi130 bash[27579]: cluster 2023-08-27T23:38:37.361864+0000 mgr.y (mgr.14901) 3002 : cluster [DBG] pgmap v3009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:38.676 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:38 smithi130 bash[27579]: audit 2023-08-27T23:38:37.516613+0000 mon.c (mon.2) 1159 : audit [DBG] from='client.? 172.21.15.6:0/2106742554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:38.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:38 smithi006 bash[33918]: cluster 2023-08-27T23:38:37.361864+0000 mgr.y (mgr.14901) 3002 : cluster [DBG] pgmap v3009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:38.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:38 smithi006 bash[33918]: audit 2023-08-27T23:38:37.516613+0000 mon.c (mon.2) 1159 : audit [DBG] from='client.? 172.21.15.6:0/2106742554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:38.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:38 smithi006 bash[27292]: cluster 2023-08-27T23:38:37.361864+0000 mgr.y (mgr.14901) 3002 : cluster [DBG] pgmap v3009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:38.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:38 smithi006 bash[27292]: audit 2023-08-27T23:38:37.516613+0000 mon.c (mon.2) 1159 : audit [DBG] from='client.? 172.21.15.6:0/2106742554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:39.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:38 smithi130 bash[51302]: debug 2023-08-27T23:38:38.673+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:38.675762+0000) 2023-08-27T23:38:40.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:39 smithi130 bash[51302]: debug 2023-08-27T23:38:39.673+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:39.676073+0000) 2023-08-27T23:38:40.899 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:38:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:38:40] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-27T23:38:40.899 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:38:40.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:40.899 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:38:40.523Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:41.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:40 smithi130 bash[51302]: debug 2023-08-27T23:38:40.673+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:40.676344+0000) 2023-08-27T23:38:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:40 smithi130 bash[27579]: cluster 2023-08-27T23:38:39.363106+0000 mgr.y (mgr.14901) 3003 : cluster [DBG] pgmap v3010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:40 smithi130 bash[27579]: audit 2023-08-27T23:38:39.937224+0000 mon.c (mon.2) 1160 : audit [DBG] from='client.? 172.21.15.6:0/4291280768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:41.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:40 smithi006 bash[33918]: cluster 2023-08-27T23:38:39.363106+0000 mgr.y (mgr.14901) 3003 : cluster [DBG] pgmap v3010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:41.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:40 smithi006 bash[33918]: audit 2023-08-27T23:38:39.937224+0000 mon.c (mon.2) 1160 : audit [DBG] from='client.? 172.21.15.6:0/4291280768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:41.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:40 smithi006 bash[27292]: cluster 2023-08-27T23:38:39.363106+0000 mgr.y (mgr.14901) 3003 : cluster [DBG] pgmap v3010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:41.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:40 smithi006 bash[27292]: audit 2023-08-27T23:38:39.937224+0000 mon.c (mon.2) 1160 : audit [DBG] from='client.? 172.21.15.6:0/4291280768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:42.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:41 smithi130 bash[51302]: debug 2023-08-27T23:38:41.673+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:41.676664+0000) 2023-08-27T23:38:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:38:42.087Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:38:42.088Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:43.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:42 smithi130 bash[51302]: debug 2023-08-27T23:38:42.673+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:42.676923+0000) 2023-08-27T23:38:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:42 smithi130 bash[27579]: cluster 2023-08-27T23:38:41.363804+0000 mgr.y (mgr.14901) 3004 : cluster [DBG] pgmap v3011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:42 smithi130 bash[27579]: audit 2023-08-27T23:38:42.357638+0000 mon.c (mon.2) 1161 : audit [DBG] from='client.? 172.21.15.6:0/2158990585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:42 smithi006 bash[33918]: cluster 2023-08-27T23:38:41.363804+0000 mgr.y (mgr.14901) 3004 : cluster [DBG] pgmap v3011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:43.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:42 smithi006 bash[33918]: audit 2023-08-27T23:38:42.357638+0000 mon.c (mon.2) 1161 : audit [DBG] from='client.? 172.21.15.6:0/2158990585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:43.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:42 smithi006 bash[27292]: cluster 2023-08-27T23:38:41.363804+0000 mgr.y (mgr.14901) 3004 : cluster [DBG] pgmap v3011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:43.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:42 smithi006 bash[27292]: audit 2023-08-27T23:38:42.357638+0000 mon.c (mon.2) 1161 : audit [DBG] from='client.? 172.21.15.6:0/2158990585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:43.960 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:43 smithi130 bash[51302]: debug 2023-08-27T23:38:43.673+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:43.677193+0000) 2023-08-27T23:38:44.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:43 smithi006 bash[27292]: cluster 2023-08-27T23:38:43.364462+0000 mgr.y (mgr.14901) 3005 : cluster [DBG] pgmap v3012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:44.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:43 smithi006 bash[33918]: cluster 2023-08-27T23:38:43.364462+0000 mgr.y (mgr.14901) 3005 : cluster [DBG] pgmap v3012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:44.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:38:43.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:44.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:38:43.988Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:44.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:43 smithi130 bash[27579]: cluster 2023-08-27T23:38:43.364462+0000 mgr.y (mgr.14901) 3005 : cluster [DBG] pgmap v3012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:44.963 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:44 smithi130 bash[51302]: debug 2023-08-27T23:38:44.673+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:44.677474+0000) 2023-08-27T23:38:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:44 smithi006 bash[27292]: audit 2023-08-27T23:38:44.774336+0000 mon.c (mon.2) 1162 : audit [DBG] from='client.? 172.21.15.6:0/526221239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:45.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:44 smithi006 bash[33918]: audit 2023-08-27T23:38:44.774336+0000 mon.c (mon.2) 1162 : audit [DBG] from='client.? 172.21.15.6:0/526221239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:45.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:44 smithi130 bash[27579]: audit 2023-08-27T23:38:44.774336+0000 mon.c (mon.2) 1162 : audit [DBG] from='client.? 172.21.15.6:0/526221239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:45.972 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:45 smithi130 bash[51302]: debug 2023-08-27T23:38:45.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:45.677755+0000) 2023-08-27T23:38:46.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:45 smithi130 bash[27579]: cluster 2023-08-27T23:38:45.365691+0000 mgr.y (mgr.14901) 3006 : cluster [DBG] pgmap v3013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:45 smithi006 bash[33918]: cluster 2023-08-27T23:38:45.365691+0000 mgr.y (mgr.14901) 3006 : cluster [DBG] pgmap v3013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:46.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:45 smithi006 bash[27292]: cluster 2023-08-27T23:38:45.365691+0000 mgr.y (mgr.14901) 3006 : cluster [DBG] pgmap v3013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:47.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:46 smithi130 bash[51302]: debug 2023-08-27T23:38:46.673+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:46.678042+0000) 2023-08-27T23:38:47.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:47 smithi130 bash[27579]: audit 2023-08-27T23:38:47.186716+0000 mon.a (mon.0) 2453 : audit [DBG] from='client.? 172.21.15.6:0/2977772604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:47.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:47 smithi006 bash[33918]: audit 2023-08-27T23:38:47.186716+0000 mon.a (mon.0) 2453 : audit [DBG] from='client.? 172.21.15.6:0/2977772604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:47.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:47 smithi006 bash[27292]: audit 2023-08-27T23:38:47.186716+0000 mon.a (mon.0) 2453 : audit [DBG] from='client.? 172.21.15.6:0/2977772604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:48.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:47 smithi130 bash[51302]: debug 2023-08-27T23:38:47.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:47.678314+0000) 2023-08-27T23:38:48.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:48 smithi130 bash[27579]: cluster 2023-08-27T23:38:47.366435+0000 mgr.y (mgr.14901) 3007 : cluster [DBG] pgmap v3014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:48 smithi006 bash[27292]: cluster 2023-08-27T23:38:47.366435+0000 mgr.y (mgr.14901) 3007 : cluster [DBG] pgmap v3014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:48 smithi006 bash[33918]: cluster 2023-08-27T23:38:47.366435+0000 mgr.y (mgr.14901) 3007 : cluster [DBG] pgmap v3014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:49.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:48 smithi130 bash[51302]: debug 2023-08-27T23:38:48.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:48.678640+0000) 2023-08-27T23:38:50.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:49 smithi130 bash[51302]: debug 2023-08-27T23:38:49.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:49.678988+0000) 2023-08-27T23:38:50.680 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:50 smithi130 bash[27579]: cluster 2023-08-27T23:38:49.367194+0000 mgr.y (mgr.14901) 3008 : cluster [DBG] pgmap v3015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:50.680 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:50 smithi130 bash[27579]: audit 2023-08-27T23:38:49.604825+0000 mon.c (mon.2) 1163 : audit [DBG] from='client.? 172.21.15.6:0/4264192576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:38:50.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:50.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:38:50.523Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:50 smithi006 bash[27292]: cluster 2023-08-27T23:38:49.367194+0000 mgr.y (mgr.14901) 3008 : cluster [DBG] pgmap v3015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:50 smithi006 bash[27292]: audit 2023-08-27T23:38:49.604825+0000 mon.c (mon.2) 1163 : audit [DBG] from='client.? 172.21.15.6:0/4264192576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:50 smithi006 bash[33918]: cluster 2023-08-27T23:38:49.367194+0000 mgr.y (mgr.14901) 3008 : cluster [DBG] pgmap v3015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:50 smithi006 bash[33918]: audit 2023-08-27T23:38:49.604825+0000 mon.c (mon.2) 1163 : audit [DBG] from='client.? 172.21.15.6:0/4264192576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:51.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:50 smithi130 bash[51302]: debug 2023-08-27T23:38:50.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:50.679272+0000) 2023-08-27T23:38:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:38:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:38:50] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:38:52.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:51 smithi130 bash[51302]: debug 2023-08-27T23:38:51.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:51.679540+0000) 2023-08-27T23:38:52.432 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:38:52.088Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:52.433 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:38:52.090Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:52.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:52 smithi006 bash[27292]: cluster 2023-08-27T23:38:51.367835+0000 mgr.y (mgr.14901) 3009 : cluster [DBG] pgmap v3016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:52 smithi006 bash[27292]: audit 2023-08-27T23:38:52.018339+0000 mon.a (mon.0) 2454 : audit [DBG] from='client.? 172.21.15.6:0/3172044722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:52 smithi006 bash[33918]: cluster 2023-08-27T23:38:51.367835+0000 mgr.y (mgr.14901) 3009 : cluster [DBG] pgmap v3016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:52 smithi006 bash[33918]: audit 2023-08-27T23:38:52.018339+0000 mon.a (mon.0) 2454 : audit [DBG] from='client.? 172.21.15.6:0/3172044722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:52 smithi130 bash[27579]: cluster 2023-08-27T23:38:51.367835+0000 mgr.y (mgr.14901) 3009 : cluster [DBG] pgmap v3016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:52 smithi130 bash[27579]: audit 2023-08-27T23:38:52.018339+0000 mon.a (mon.0) 2454 : audit [DBG] from='client.? 172.21.15.6:0/3172044722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:52.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:52 smithi130 bash[51302]: debug 2023-08-27T23:38:52.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:52.679859+0000) 2023-08-27T23:38:54.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:53 smithi130 bash[51302]: debug 2023-08-27T23:38:53.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:53.680147+0000) 2023-08-27T23:38:54.437 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:38:53.987Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:54.437 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:38:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:38:53.989Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:38:54.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:54 smithi006 bash[33918]: cluster 2023-08-27T23:38:53.368469+0000 mgr.y (mgr.14901) 3010 : cluster [DBG] pgmap v3017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:54 smithi006 bash[27292]: cluster 2023-08-27T23:38:53.368469+0000 mgr.y (mgr.14901) 3010 : cluster [DBG] pgmap v3017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:54.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:54 smithi130 bash[51302]: debug 2023-08-27T23:38:54.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:54.680433+0000) 2023-08-27T23:38:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:54 smithi130 bash[27579]: cluster 2023-08-27T23:38:53.368469+0000 mgr.y (mgr.14901) 3010 : cluster [DBG] pgmap v3017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:55.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:55 smithi006 bash[33918]: audit 2023-08-27T23:38:54.441348+0000 mon.c (mon.2) 1164 : audit [DBG] from='client.? 172.21.15.6:0/2506043181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:55.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:55 smithi006 bash[27292]: audit 2023-08-27T23:38:54.441348+0000 mon.c (mon.2) 1164 : audit [DBG] from='client.? 172.21.15.6:0/2506043181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:55.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:55 smithi130 bash[27579]: audit 2023-08-27T23:38:54.441348+0000 mon.c (mon.2) 1164 : audit [DBG] from='client.? 172.21.15.6:0/2506043181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:55.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:55 smithi130 bash[51302]: debug 2023-08-27T23:38:55.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:55.680710+0000) 2023-08-27T23:38:57.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:56 smithi130 bash[27579]: cluster 2023-08-27T23:38:55.369581+0000 mgr.y (mgr.14901) 3011 : cluster [DBG] pgmap v3018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:57.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:56 smithi130 bash[27579]: audit 2023-08-27T23:38:56.847306+0000 mon.c (mon.2) 1165 : audit [DBG] from='client.? 172.21.15.6:0/1236809642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:57.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:56 smithi130 bash[51302]: debug 2023-08-27T23:38:56.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:56.680984+0000) 2023-08-27T23:38:57.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:56 smithi006 bash[27292]: cluster 2023-08-27T23:38:55.369581+0000 mgr.y (mgr.14901) 3011 : cluster [DBG] pgmap v3018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:57.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:56 smithi006 bash[27292]: audit 2023-08-27T23:38:56.847306+0000 mon.c (mon.2) 1165 : audit [DBG] from='client.? 172.21.15.6:0/1236809642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:57.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:56 smithi006 bash[33918]: cluster 2023-08-27T23:38:55.369581+0000 mgr.y (mgr.14901) 3011 : cluster [DBG] pgmap v3018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:57.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:56 smithi006 bash[33918]: audit 2023-08-27T23:38:56.847306+0000 mon.c (mon.2) 1165 : audit [DBG] from='client.? 172.21.15.6:0/1236809642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:58.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:57 smithi130 bash[51302]: debug 2023-08-27T23:38:57.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:57.681272+0000) 2023-08-27T23:38:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:58 smithi130 bash[27579]: cluster 2023-08-27T23:38:57.370261+0000 mgr.y (mgr.14901) 3012 : cluster [DBG] pgmap v3019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:59.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:58 smithi130 bash[51302]: debug 2023-08-27T23:38:58.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:58.681563+0000) 2023-08-27T23:38:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:58 smithi006 bash[27292]: cluster 2023-08-27T23:38:57.370261+0000 mgr.y (mgr.14901) 3012 : cluster [DBG] pgmap v3019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:38:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:58 smithi006 bash[33918]: cluster 2023-08-27T23:38:57.370261+0000 mgr.y (mgr.14901) 3012 : cluster [DBG] pgmap v3019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:00.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:38:59 smithi130 bash[51302]: debug 2023-08-27T23:38:59.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:59.681877+0000) 2023-08-27T23:39:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:59 smithi130 bash[27579]: audit 2023-08-27T23:38:59.259096+0000 mon.a (mon.0) 2455 : audit [DBG] from='client.? 172.21.15.6:0/466681011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:38:59 smithi130 bash[27579]: cluster 2023-08-27T23:38:59.371352+0000 mgr.y (mgr.14901) 3013 : cluster [DBG] pgmap v3020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:00.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:59 smithi006 bash[33918]: audit 2023-08-27T23:38:59.259096+0000 mon.a (mon.0) 2455 : audit [DBG] from='client.? 172.21.15.6:0/466681011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:00.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:38:59 smithi006 bash[33918]: cluster 2023-08-27T23:38:59.371352+0000 mgr.y (mgr.14901) 3013 : cluster [DBG] pgmap v3020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:00.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:59 smithi006 bash[27292]: audit 2023-08-27T23:38:59.259096+0000 mon.a (mon.0) 2455 : audit [DBG] from='client.? 172.21.15.6:0/466681011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:00.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:38:59 smithi006 bash[27292]: cluster 2023-08-27T23:38:59.371352+0000 mgr.y (mgr.14901) 3013 : cluster [DBG] pgmap v3020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:39:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:39:00] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-27T23:39:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:39:00.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:39:00.524Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:01.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:00 smithi130 bash[51302]: debug 2023-08-27T23:39:00.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:00.682157+0000) 2023-08-27T23:39:02.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:01 smithi130 bash[51302]: debug 2023-08-27T23:39:01.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:01.682420+0000) 2023-08-27T23:39:02.432 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:39:02.089Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:02.432 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:39:02.090Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:02.683 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:02 smithi130 bash[27579]: cluster 2023-08-27T23:39:01.371975+0000 mgr.y (mgr.14901) 3014 : cluster [DBG] pgmap v3021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:02.683 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:02 smithi130 bash[27579]: audit 2023-08-27T23:39:01.670686+0000 mon.a (mon.0) 2456 : audit [DBG] from='client.? 172.21.15.6:0/197596307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:02 smithi006 bash[33918]: cluster 2023-08-27T23:39:01.371975+0000 mgr.y (mgr.14901) 3014 : cluster [DBG] pgmap v3021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:02 smithi006 bash[33918]: audit 2023-08-27T23:39:01.670686+0000 mon.a (mon.0) 2456 : audit [DBG] from='client.? 172.21.15.6:0/197596307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:02.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:02 smithi006 bash[27292]: cluster 2023-08-27T23:39:01.371975+0000 mgr.y (mgr.14901) 3014 : cluster [DBG] pgmap v3021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:02.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:02 smithi006 bash[27292]: audit 2023-08-27T23:39:01.670686+0000 mon.a (mon.0) 2456 : audit [DBG] from='client.? 172.21.15.6:0/197596307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:02 smithi130 bash[51302]: debug 2023-08-27T23:39:02.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:02.682731+0000) 2023-08-27T23:39:04.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:03 smithi130 bash[51302]: debug 2023-08-27T23:39:03.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:03.683047+0000) 2023-08-27T23:39:04.437 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:39:03.988Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:04.437 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:39:03.990Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:04.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:04 smithi006 bash[33918]: cluster 2023-08-27T23:39:03.372625+0000 mgr.y (mgr.14901) 3015 : cluster [DBG] pgmap v3022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:04.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:04 smithi006 bash[33918]: audit 2023-08-27T23:39:04.085238+0000 mon.a (mon.0) 2457 : audit [DBG] from='client.? 172.21.15.6:0/3174230202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:04.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:04 smithi006 bash[27292]: cluster 2023-08-27T23:39:03.372625+0000 mgr.y (mgr.14901) 3015 : cluster [DBG] pgmap v3022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:04.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:04 smithi006 bash[27292]: audit 2023-08-27T23:39:04.085238+0000 mon.a (mon.0) 2457 : audit [DBG] from='client.? 172.21.15.6:0/3174230202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:04 smithi130 bash[27579]: cluster 2023-08-27T23:39:03.372625+0000 mgr.y (mgr.14901) 3015 : cluster [DBG] pgmap v3022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:04 smithi130 bash[27579]: audit 2023-08-27T23:39:04.085238+0000 mon.a (mon.0) 2457 : audit [DBG] from='client.? 172.21.15.6:0/3174230202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:04.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:04 smithi130 bash[51302]: debug 2023-08-27T23:39:04.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:04.683289+0000) 2023-08-27T23:39:06.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:05 smithi130 bash[51302]: debug 2023-08-27T23:39:05.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:05.683602+0000) 2023-08-27T23:39:07.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:06 smithi130 bash[51302]: debug 2023-08-27T23:39:06.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:06.683869+0000) 2023-08-27T23:39:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:06 smithi130 bash[27579]: cluster 2023-08-27T23:39:05.373748+0000 mgr.y (mgr.14901) 3016 : cluster [DBG] pgmap v3023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:06 smithi130 bash[27579]: audit 2023-08-27T23:39:06.501921+0000 mon.c (mon.2) 1166 : audit [DBG] from='client.? 172.21.15.6:0/3129065936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:07.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:06 smithi006 bash[27292]: cluster 2023-08-27T23:39:05.373748+0000 mgr.y (mgr.14901) 3016 : cluster [DBG] pgmap v3023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:07.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:06 smithi006 bash[27292]: audit 2023-08-27T23:39:06.501921+0000 mon.c (mon.2) 1166 : audit [DBG] from='client.? 172.21.15.6:0/3129065936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:07.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:06 smithi006 bash[33918]: cluster 2023-08-27T23:39:05.373748+0000 mgr.y (mgr.14901) 3016 : cluster [DBG] pgmap v3023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:07.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:06 smithi006 bash[33918]: audit 2023-08-27T23:39:06.501921+0000 mon.c (mon.2) 1166 : audit [DBG] from='client.? 172.21.15.6:0/3129065936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:08.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:07 smithi130 bash[51302]: debug 2023-08-27T23:39:07.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:07.684191+0000) 2023-08-27T23:39:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:08 smithi130 bash[27579]: cluster 2023-08-27T23:39:07.374500+0000 mgr.y (mgr.14901) 3017 : cluster [DBG] pgmap v3024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:09.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:08 smithi130 bash[51302]: debug 2023-08-27T23:39:08.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:08.684476+0000) 2023-08-27T23:39:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:08 smithi006 bash[33918]: cluster 2023-08-27T23:39:07.374500+0000 mgr.y (mgr.14901) 3017 : cluster [DBG] pgmap v3024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:09.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:08 smithi006 bash[27292]: cluster 2023-08-27T23:39:07.374500+0000 mgr.y (mgr.14901) 3017 : cluster [DBG] pgmap v3024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:10.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:09 smithi130 bash[51302]: debug 2023-08-27T23:39:09.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:09.684751+0000) 2023-08-27T23:39:10.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:09 smithi130 bash[27579]: audit 2023-08-27T23:39:08.916438+0000 mon.a (mon.0) 2458 : audit [DBG] from='client.? 172.21.15.6:0/3473985517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:10.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:09 smithi130 bash[27579]: cluster 2023-08-27T23:39:09.375746+0000 mgr.y (mgr.14901) 3018 : cluster [DBG] pgmap v3025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:10.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:09 smithi006 bash[27292]: audit 2023-08-27T23:39:08.916438+0000 mon.a (mon.0) 2458 : audit [DBG] from='client.? 172.21.15.6:0/3473985517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:10.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:09 smithi006 bash[27292]: cluster 2023-08-27T23:39:09.375746+0000 mgr.y (mgr.14901) 3018 : cluster [DBG] pgmap v3025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:10.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:09 smithi006 bash[33918]: audit 2023-08-27T23:39:08.916438+0000 mon.a (mon.0) 2458 : audit [DBG] from='client.? 172.21.15.6:0/3473985517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:10.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:09 smithi006 bash[33918]: cluster 2023-08-27T23:39:09.375746+0000 mgr.y (mgr.14901) 3018 : cluster [DBG] pgmap v3025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:39:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:39:10] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-27T23:39:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:39:10.523Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:39:10.525Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:11.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:10 smithi130 bash[51302]: debug 2023-08-27T23:39:10.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:10.685017+0000) 2023-08-27T23:39:11.686 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:11 smithi130 bash[27579]: audit 2023-08-27T23:39:11.329326+0000 mon.a (mon.0) 2459 : audit [DBG] from='client.? 172.21.15.6:0/780183750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:11.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:11 smithi006 bash[27292]: audit 2023-08-27T23:39:11.329326+0000 mon.a (mon.0) 2459 : audit [DBG] from='client.? 172.21.15.6:0/780183750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:11.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:11 smithi006 bash[33918]: audit 2023-08-27T23:39:11.329326+0000 mon.a (mon.0) 2459 : audit [DBG] from='client.? 172.21.15.6:0/780183750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:12.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:11 smithi130 bash[51302]: debug 2023-08-27T23:39:11.685+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:11.685295+0000) 2023-08-27T23:39:12.436 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:39:12.090Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:12.436 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:39:12.092Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:12.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:12 smithi006 bash[27292]: cluster 2023-08-27T23:39:11.376528+0000 mgr.y (mgr.14901) 3019 : cluster [DBG] pgmap v3026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:12.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:12 smithi006 bash[33918]: cluster 2023-08-27T23:39:11.376528+0000 mgr.y (mgr.14901) 3019 : cluster [DBG] pgmap v3026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:12 smithi130 bash[27579]: cluster 2023-08-27T23:39:11.376528+0000 mgr.y (mgr.14901) 3019 : cluster [DBG] pgmap v3026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:12 smithi130 bash[51302]: debug 2023-08-27T23:39:12.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:12.685565+0000) 2023-08-27T23:39:14.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:13 smithi130 bash[51302]: debug 2023-08-27T23:39:13.685+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:13.685877+0000) 2023-08-27T23:39:14.441 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:39:13.989Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:14.441 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:39:13.991Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:14 smithi006 bash[33918]: cluster 2023-08-27T23:39:13.377089+0000 mgr.y (mgr.14901) 3020 : cluster [DBG] pgmap v3027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:14 smithi006 bash[33918]: audit 2023-08-27T23:39:13.742934+0000 mon.c (mon.2) 1167 : audit [DBG] from='client.? 172.21.15.6:0/3408411631' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:14.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:14 smithi006 bash[27292]: cluster 2023-08-27T23:39:13.377089+0000 mgr.y (mgr.14901) 3020 : cluster [DBG] pgmap v3027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:14.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:14 smithi006 bash[27292]: audit 2023-08-27T23:39:13.742934+0000 mon.c (mon.2) 1167 : audit [DBG] from='client.? 172.21.15.6:0/3408411631' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:14.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:14 smithi130 bash[27579]: cluster 2023-08-27T23:39:13.377089+0000 mgr.y (mgr.14901) 3020 : cluster [DBG] pgmap v3027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:14.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:14 smithi130 bash[27579]: audit 2023-08-27T23:39:13.742934+0000 mon.c (mon.2) 1167 : audit [DBG] from='client.? 172.21.15.6:0/3408411631' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:14.872 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:14 smithi130 bash[51302]: debug 2023-08-27T23:39:14.685+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:14.686315+0000) 2023-08-27T23:39:16.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:15 smithi130 bash[51302]: debug 2023-08-27T23:39:15.685+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:15.686587+0000) 2023-08-27T23:39:17.123 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:16 smithi130 bash[27579]: cluster 2023-08-27T23:39:15.378188+0000 mgr.y (mgr.14901) 3021 : cluster [DBG] pgmap v3028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:17.123 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:16 smithi130 bash[27579]: audit 2023-08-27T23:39:16.157840+0000 mon.a (mon.0) 2460 : audit [DBG] from='client.? 172.21.15.6:0/3666210876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:17.124 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:16 smithi130 bash[51302]: debug 2023-08-27T23:39:16.685+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:16.687005+0000) 2023-08-27T23:39:17.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:16 smithi006 bash[27292]: cluster 2023-08-27T23:39:15.378188+0000 mgr.y (mgr.14901) 3021 : cluster [DBG] pgmap v3028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:17.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:16 smithi006 bash[27292]: audit 2023-08-27T23:39:16.157840+0000 mon.a (mon.0) 2460 : audit [DBG] from='client.? 172.21.15.6:0/3666210876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:17.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:16 smithi006 bash[33918]: cluster 2023-08-27T23:39:15.378188+0000 mgr.y (mgr.14901) 3021 : cluster [DBG] pgmap v3028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:17.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:16 smithi006 bash[33918]: audit 2023-08-27T23:39:16.157840+0000 mon.a (mon.0) 2460 : audit [DBG] from='client.? 172.21.15.6:0/3666210876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:17.968 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:17 smithi130 bash[51302]: debug 2023-08-27T23:39:17.685+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:17.687367+0000) 2023-08-27T23:39:18.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:17 smithi006 bash[27292]: cluster 2023-08-27T23:39:17.378859+0000 mgr.y (mgr.14901) 3022 : cluster [DBG] pgmap v3029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:18.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:17 smithi006 bash[33918]: cluster 2023-08-27T23:39:17.378859+0000 mgr.y (mgr.14901) 3022 : cluster [DBG] pgmap v3029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:18.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:17 smithi130 bash[27579]: cluster 2023-08-27T23:39:17.378859+0000 mgr.y (mgr.14901) 3022 : cluster [DBG] pgmap v3029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:18.969 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:18 smithi130 bash[51302]: debug 2023-08-27T23:39:18.684+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:18.687741+0000) 2023-08-27T23:39:19.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:18 smithi130 bash[27579]: audit 2023-08-27T23:39:18.576361+0000 mon.c (mon.2) 1168 : audit [DBG] from='client.? 172.21.15.6:0/523386932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:19.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:18 smithi130 bash[27579]: audit 2023-08-27T23:39:18.747016+0000 mon.b (mon.1) 406 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:39:19.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:18 smithi006 bash[27292]: audit 2023-08-27T23:39:18.576361+0000 mon.c (mon.2) 1168 : audit [DBG] from='client.? 172.21.15.6:0/523386932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:19.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:18 smithi006 bash[27292]: audit 2023-08-27T23:39:18.747016+0000 mon.b (mon.1) 406 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:39:19.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:18 smithi006 bash[33918]: audit 2023-08-27T23:39:18.576361+0000 mon.c (mon.2) 1168 : audit [DBG] from='client.? 172.21.15.6:0/523386932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:19.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:18 smithi006 bash[33918]: audit 2023-08-27T23:39:18.747016+0000 mon.b (mon.1) 406 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:39:20.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:19 smithi130 bash[51302]: debug 2023-08-27T23:39:19.684+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:19.688078+0000) 2023-08-27T23:39:20.910 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:39:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:39:20] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-27T23:39:20.910 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:39:20.523Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:20.910 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:39:20.525Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:20 smithi130 bash[27579]: cluster 2023-08-27T23:39:19.379969+0000 mgr.y (mgr.14901) 3023 : cluster [DBG] pgmap v3030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:21.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:20 smithi130 bash[51302]: debug 2023-08-27T23:39:20.684+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:20.688411+0000) 2023-08-27T23:39:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:20 smithi006 bash[27292]: cluster 2023-08-27T23:39:19.379969+0000 mgr.y (mgr.14901) 3023 : cluster [DBG] pgmap v3030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:21.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:20 smithi006 bash[33918]: cluster 2023-08-27T23:39:19.379969+0000 mgr.y (mgr.14901) 3023 : cluster [DBG] pgmap v3030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:22.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:21 smithi130 bash[27579]: audit 2023-08-27T23:39:20.998635+0000 mon.c (mon.2) 1169 : audit [DBG] from='client.? 172.21.15.6:0/2776209160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:22.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:21 smithi130 bash[51302]: debug 2023-08-27T23:39:21.684+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:21.688721+0000) 2023-08-27T23:39:22.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:21 smithi006 bash[33918]: audit 2023-08-27T23:39:20.998635+0000 mon.c (mon.2) 1169 : audit [DBG] from='client.? 172.21.15.6:0/2776209160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:22.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:21 smithi006 bash[27292]: audit 2023-08-27T23:39:20.998635+0000 mon.c (mon.2) 1169 : audit [DBG] from='client.? 172.21.15.6:0/2776209160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:22.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:39:22.090Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:22.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:39:22.092Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:23.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:22 smithi130 bash[27579]: cluster 2023-08-27T23:39:21.380629+0000 mgr.y (mgr.14901) 3024 : cluster [DBG] pgmap v3031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:23.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:22 smithi130 bash[51302]: debug 2023-08-27T23:39:22.684+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:22.689029+0000) 2023-08-27T23:39:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:22 smithi006 bash[33918]: cluster 2023-08-27T23:39:21.380629+0000 mgr.y (mgr.14901) 3024 : cluster [DBG] pgmap v3031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:23.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:22 smithi006 bash[27292]: cluster 2023-08-27T23:39:21.380629+0000 mgr.y (mgr.14901) 3024 : cluster [DBG] pgmap v3031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:24.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:23 smithi130 bash[27579]: cluster 2023-08-27T23:39:23.381085+0000 mgr.y (mgr.14901) 3025 : cluster [DBG] pgmap v3032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:24.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:23 smithi130 bash[27579]: audit 2023-08-27T23:39:23.417260+0000 mon.c (mon.2) 1170 : audit [DBG] from='client.? 172.21.15.6:0/2871433781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:24.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:23 smithi130 bash[51302]: debug 2023-08-27T23:39:23.684+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:23.689322+0000) 2023-08-27T23:39:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:39:23.989Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:39:23.991Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:24.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:23 smithi006 bash[27292]: cluster 2023-08-27T23:39:23.381085+0000 mgr.y (mgr.14901) 3025 : cluster [DBG] pgmap v3032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:24.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:23 smithi006 bash[27292]: audit 2023-08-27T23:39:23.417260+0000 mon.c (mon.2) 1170 : audit [DBG] from='client.? 172.21.15.6:0/2871433781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:24.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:23 smithi006 bash[33918]: cluster 2023-08-27T23:39:23.381085+0000 mgr.y (mgr.14901) 3025 : cluster [DBG] pgmap v3032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:24.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:23 smithi006 bash[33918]: audit 2023-08-27T23:39:23.417260+0000 mon.c (mon.2) 1170 : audit [DBG] from='client.? 172.21.15.6:0/2871433781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:25.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:24 smithi130 bash[51302]: debug 2023-08-27T23:39:24.684+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:24.689543+0000) 2023-08-27T23:39:25.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:25 smithi006 bash[33918]: audit 2023-08-27T23:39:24.665075+0000 mon.a (mon.0) 2461 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:39:25.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:25 smithi006 bash[33918]: audit 2023-08-27T23:39:25.122212+0000 mon.a (mon.0) 2462 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:39:25.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:25 smithi006 bash[33918]: audit 2023-08-27T23:39:25.417619+0000 mon.b (mon.1) 407 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:39:25.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:25 smithi006 bash[33918]: audit 2023-08-27T23:39:25.419711+0000 mon.b (mon.1) 408 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:39:25.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:25 smithi006 bash[33918]: audit 2023-08-27T23:39:25.429984+0000 mon.a (mon.0) 2463 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:39:25.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:25 smithi006 bash[27292]: audit 2023-08-27T23:39:24.665075+0000 mon.a (mon.0) 2461 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:39:25.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:25 smithi006 bash[27292]: audit 2023-08-27T23:39:25.122212+0000 mon.a (mon.0) 2462 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:39:25.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:25 smithi006 bash[27292]: audit 2023-08-27T23:39:25.417619+0000 mon.b (mon.1) 407 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:39:25.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:25 smithi006 bash[27292]: audit 2023-08-27T23:39:25.419711+0000 mon.b (mon.1) 408 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:39:25.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:25 smithi006 bash[27292]: audit 2023-08-27T23:39:25.429984+0000 mon.a (mon.0) 2463 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:39:26.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:25 smithi130 bash[51302]: debug 2023-08-27T23:39:25.684+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:25.689811+0000) 2023-08-27T23:39:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:25 smithi130 bash[27579]: audit 2023-08-27T23:39:24.665075+0000 mon.a (mon.0) 2461 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:39:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:25 smithi130 bash[27579]: audit 2023-08-27T23:39:25.122212+0000 mon.a (mon.0) 2462 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:39:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:25 smithi130 bash[27579]: audit 2023-08-27T23:39:25.417619+0000 mon.b (mon.1) 407 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:39:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:25 smithi130 bash[27579]: audit 2023-08-27T23:39:25.419711+0000 mon.b (mon.1) 408 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:39:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:25 smithi130 bash[27579]: audit 2023-08-27T23:39:25.429984+0000 mon.a (mon.0) 2463 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:39:26.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:26 smithi006 bash[27292]: cluster 2023-08-27T23:39:25.382183+0000 mgr.y (mgr.14901) 3026 : cluster [DBG] pgmap v3033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:26.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:26 smithi006 bash[27292]: audit 2023-08-27T23:39:25.827033+0000 mon.c (mon.2) 1171 : audit [DBG] from='client.? 172.21.15.6:0/3386388095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:26.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:26 smithi006 bash[33918]: cluster 2023-08-27T23:39:25.382183+0000 mgr.y (mgr.14901) 3026 : cluster [DBG] pgmap v3033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:26.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:26 smithi006 bash[33918]: audit 2023-08-27T23:39:25.827033+0000 mon.c (mon.2) 1171 : audit [DBG] from='client.? 172.21.15.6:0/3386388095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:27.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:26 smithi130 bash[51302]: debug 2023-08-27T23:39:26.684+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:26.690105+0000) 2023-08-27T23:39:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:26 smithi130 bash[27579]: cluster 2023-08-27T23:39:25.382183+0000 mgr.y (mgr.14901) 3026 : cluster [DBG] pgmap v3033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:26 smithi130 bash[27579]: audit 2023-08-27T23:39:25.827033+0000 mon.c (mon.2) 1171 : audit [DBG] from='client.? 172.21.15.6:0/3386388095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:28.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:27 smithi130 bash[51302]: debug 2023-08-27T23:39:27.684+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:27.690376+0000) 2023-08-27T23:39:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:28 smithi006 bash[27292]: cluster 2023-08-27T23:39:27.382817+0000 mgr.y (mgr.14901) 3027 : cluster [DBG] pgmap v3034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:28.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:28 smithi006 bash[27292]: audit 2023-08-27T23:39:28.245048+0000 mon.a (mon.0) 2464 : audit [DBG] from='client.? 172.21.15.6:0/2335700442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:28.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:28 smithi006 bash[33918]: cluster 2023-08-27T23:39:27.382817+0000 mgr.y (mgr.14901) 3027 : cluster [DBG] pgmap v3034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:28.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:28 smithi006 bash[33918]: audit 2023-08-27T23:39:28.245048+0000 mon.a (mon.0) 2464 : audit [DBG] from='client.? 172.21.15.6:0/2335700442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:29.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:28 smithi130 bash[51302]: debug 2023-08-27T23:39:28.684+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:28.690677+0000) 2023-08-27T23:39:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:28 smithi130 bash[27579]: cluster 2023-08-27T23:39:27.382817+0000 mgr.y (mgr.14901) 3027 : cluster [DBG] pgmap v3034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:28 smithi130 bash[27579]: audit 2023-08-27T23:39:28.245048+0000 mon.a (mon.0) 2464 : audit [DBG] from='client.? 172.21.15.6:0/2335700442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:30.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:29 smithi130 bash[51302]: debug 2023-08-27T23:39:29.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:29.690960+0000) 2023-08-27T23:39:30.914 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:39:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:39:30] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-27T23:39:30.915 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:39:30.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:30.915 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:39:30.526Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:30 smithi130 bash[27579]: cluster 2023-08-27T23:39:29.383936+0000 mgr.y (mgr.14901) 3028 : cluster [DBG] pgmap v3035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:31.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:30 smithi130 bash[27579]: audit 2023-08-27T23:39:30.664268+0000 mon.a (mon.0) 2465 : audit [DBG] from='client.? 172.21.15.6:0/4227227157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:31.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:30 smithi130 bash[51302]: debug 2023-08-27T23:39:30.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:30.691261+0000) 2023-08-27T23:39:31.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:30 smithi006 bash[33918]: cluster 2023-08-27T23:39:29.383936+0000 mgr.y (mgr.14901) 3028 : cluster [DBG] pgmap v3035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:31.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:30 smithi006 bash[33918]: audit 2023-08-27T23:39:30.664268+0000 mon.a (mon.0) 2465 : audit [DBG] from='client.? 172.21.15.6:0/4227227157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:31.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:30 smithi006 bash[27292]: cluster 2023-08-27T23:39:29.383936+0000 mgr.y (mgr.14901) 3028 : cluster [DBG] pgmap v3035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:31.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:30 smithi006 bash[27292]: audit 2023-08-27T23:39:30.664268+0000 mon.a (mon.0) 2465 : audit [DBG] from='client.? 172.21.15.6:0/4227227157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:32.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:31 smithi130 bash[51302]: debug 2023-08-27T23:39:31.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:31.691537+0000) 2023-08-27T23:39:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:39:32.091Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:32.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:39:32.093Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:33.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:32 smithi130 bash[27579]: cluster 2023-08-27T23:39:31.384671+0000 mgr.y (mgr.14901) 3029 : cluster [DBG] pgmap v3036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:33.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:32 smithi130 bash[51302]: debug 2023-08-27T23:39:32.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:32.691838+0000) 2023-08-27T23:39:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:32 smithi006 bash[27292]: cluster 2023-08-27T23:39:31.384671+0000 mgr.y (mgr.14901) 3029 : cluster [DBG] pgmap v3036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:33.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:32 smithi006 bash[33918]: cluster 2023-08-27T23:39:31.384671+0000 mgr.y (mgr.14901) 3029 : cluster [DBG] pgmap v3036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:34.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:33 smithi130 bash[51302]: debug 2023-08-27T23:39:33.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:33.692145+0000) 2023-08-27T23:39:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:33 smithi130 bash[27579]: audit 2023-08-27T23:39:33.077823+0000 mon.a (mon.0) 2466 : audit [DBG] from='client.? 172.21.15.6:0/414359565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:33 smithi130 bash[27579]: cluster 2023-08-27T23:39:33.385351+0000 mgr.y (mgr.14901) 3030 : cluster [DBG] pgmap v3037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:34.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:33 smithi006 bash[33918]: audit 2023-08-27T23:39:33.077823+0000 mon.a (mon.0) 2466 : audit [DBG] from='client.? 172.21.15.6:0/414359565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:34.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:33 smithi006 bash[33918]: cluster 2023-08-27T23:39:33.385351+0000 mgr.y (mgr.14901) 3030 : cluster [DBG] pgmap v3037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:34.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:39:33.990Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:34.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:39:33.992Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:34.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:33 smithi006 bash[27292]: audit 2023-08-27T23:39:33.077823+0000 mon.a (mon.0) 2466 : audit [DBG] from='client.? 172.21.15.6:0/414359565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:34.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:33 smithi006 bash[27292]: cluster 2023-08-27T23:39:33.385351+0000 mgr.y (mgr.14901) 3030 : cluster [DBG] pgmap v3037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:35.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:34 smithi130 bash[51302]: debug 2023-08-27T23:39:34.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:34.692463+0000) 2023-08-27T23:39:36.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:35 smithi130 bash[27579]: audit 2023-08-27T23:39:35.482966+0000 mon.c (mon.2) 1172 : audit [DBG] from='client.? 172.21.15.6:0/2851360196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:36.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:35 smithi130 bash[51302]: debug 2023-08-27T23:39:35.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:35.692769+0000) 2023-08-27T23:39:36.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:35 smithi006 bash[33918]: audit 2023-08-27T23:39:35.482966+0000 mon.c (mon.2) 1172 : audit [DBG] from='client.? 172.21.15.6:0/2851360196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:36.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:35 smithi006 bash[27292]: audit 2023-08-27T23:39:35.482966+0000 mon.c (mon.2) 1172 : audit [DBG] from='client.? 172.21.15.6:0/2851360196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:36 smithi130 bash[27579]: cluster 2023-08-27T23:39:35.386416+0000 mgr.y (mgr.14901) 3031 : cluster [DBG] pgmap v3038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:37.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:36 smithi130 bash[51302]: debug 2023-08-27T23:39:36.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:36.693049+0000) 2023-08-27T23:39:37.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:36 smithi006 bash[33918]: cluster 2023-08-27T23:39:35.386416+0000 mgr.y (mgr.14901) 3031 : cluster [DBG] pgmap v3038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:37.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:36 smithi006 bash[27292]: cluster 2023-08-27T23:39:35.386416+0000 mgr.y (mgr.14901) 3031 : cluster [DBG] pgmap v3038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:38.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:37 smithi130 bash[27579]: cluster 2023-08-27T23:39:37.387017+0000 mgr.y (mgr.14901) 3032 : cluster [DBG] pgmap v3039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:38.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:37 smithi130 bash[27579]: audit 2023-08-27T23:39:37.903487+0000 mon.a (mon.0) 2467 : audit [DBG] from='client.? 172.21.15.6:0/1931161636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:38.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:37 smithi130 bash[51302]: debug 2023-08-27T23:39:37.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:37.693333+0000) 2023-08-27T23:39:38.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:37 smithi006 bash[33918]: cluster 2023-08-27T23:39:37.387017+0000 mgr.y (mgr.14901) 3032 : cluster [DBG] pgmap v3039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:38.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:37 smithi006 bash[33918]: audit 2023-08-27T23:39:37.903487+0000 mon.a (mon.0) 2467 : audit [DBG] from='client.? 172.21.15.6:0/1931161636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:38.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:37 smithi006 bash[27292]: cluster 2023-08-27T23:39:37.387017+0000 mgr.y (mgr.14901) 3032 : cluster [DBG] pgmap v3039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:38.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:37 smithi006 bash[27292]: audit 2023-08-27T23:39:37.903487+0000 mon.a (mon.0) 2467 : audit [DBG] from='client.? 172.21.15.6:0/1931161636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:39.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:38 smithi130 bash[51302]: debug 2023-08-27T23:39:38.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:38.693646+0000) 2023-08-27T23:39:40.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:39 smithi130 bash[51302]: debug 2023-08-27T23:39:39.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:39.693907+0000) 2023-08-27T23:39:40.915 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:39:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:39:40] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-27T23:39:40.915 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:39:40.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:40.916 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:39:40.526Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:40 smithi130 bash[27579]: cluster 2023-08-27T23:39:39.388059+0000 mgr.y (mgr.14901) 3033 : cluster [DBG] pgmap v3040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:40 smithi130 bash[27579]: audit 2023-08-27T23:39:40.319246+0000 mon.a (mon.0) 2468 : audit [DBG] from='client.? 172.21.15.6:0/2348705828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:41.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:40 smithi130 bash[51302]: debug 2023-08-27T23:39:40.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:40.694181+0000) 2023-08-27T23:39:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:40 smithi006 bash[27292]: cluster 2023-08-27T23:39:39.388059+0000 mgr.y (mgr.14901) 3033 : cluster [DBG] pgmap v3040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:41.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:40 smithi006 bash[27292]: audit 2023-08-27T23:39:40.319246+0000 mon.a (mon.0) 2468 : audit [DBG] from='client.? 172.21.15.6:0/2348705828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:41.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:40 smithi006 bash[33918]: cluster 2023-08-27T23:39:39.388059+0000 mgr.y (mgr.14901) 3033 : cluster [DBG] pgmap v3040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:41.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:40 smithi006 bash[33918]: audit 2023-08-27T23:39:40.319246+0000 mon.a (mon.0) 2468 : audit [DBG] from='client.? 172.21.15.6:0/2348705828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:42.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:41 smithi130 bash[51302]: debug 2023-08-27T23:39:41.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:41.694455+0000) 2023-08-27T23:39:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:39:42.092Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:39:42.094Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:42 smithi130 bash[27579]: cluster 2023-08-27T23:39:41.388885+0000 mgr.y (mgr.14901) 3034 : cluster [DBG] pgmap v3041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:42 smithi130 bash[27579]: audit 2023-08-27T23:39:42.750745+0000 mon.c (mon.2) 1173 : audit [DBG] from='client.? 172.21.15.6:0/3232616253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:43.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:42 smithi130 bash[51302]: debug 2023-08-27T23:39:42.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:42.694742+0000) 2023-08-27T23:39:43.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:42 smithi006 bash[33918]: cluster 2023-08-27T23:39:41.388885+0000 mgr.y (mgr.14901) 3034 : cluster [DBG] pgmap v3041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:43.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:42 smithi006 bash[33918]: audit 2023-08-27T23:39:42.750745+0000 mon.c (mon.2) 1173 : audit [DBG] from='client.? 172.21.15.6:0/3232616253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:43.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:42 smithi006 bash[27292]: cluster 2023-08-27T23:39:41.388885+0000 mgr.y (mgr.14901) 3034 : cluster [DBG] pgmap v3041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:43.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:42 smithi006 bash[27292]: audit 2023-08-27T23:39:42.750745+0000 mon.c (mon.2) 1173 : audit [DBG] from='client.? 172.21.15.6:0/3232616253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:44.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:43 smithi130 bash[27579]: cluster 2023-08-27T23:39:43.389544+0000 mgr.y (mgr.14901) 3035 : cluster [DBG] pgmap v3042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:44.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:43 smithi130 bash[51302]: debug 2023-08-27T23:39:43.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:43.695047+0000) 2023-08-27T23:39:44.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:43 smithi006 bash[33918]: cluster 2023-08-27T23:39:43.389544+0000 mgr.y (mgr.14901) 3035 : cluster [DBG] pgmap v3042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:44.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:43 smithi006 bash[27292]: cluster 2023-08-27T23:39:43.389544+0000 mgr.y (mgr.14901) 3035 : cluster [DBG] pgmap v3042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:44.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:39:43.991Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:44.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:39:43.993Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:44 smithi130 bash[51302]: debug 2023-08-27T23:39:44.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:44.695329+0000) 2023-08-27T23:39:46.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:45 smithi130 bash[27579]: audit 2023-08-27T23:39:45.171142+0000 mon.a (mon.0) 2469 : audit [DBG] from='client.? 172.21.15.6:0/1378941047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:46.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:45 smithi130 bash[51302]: debug 2023-08-27T23:39:45.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:45.695604+0000) 2023-08-27T23:39:46.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:45 smithi006 bash[27292]: audit 2023-08-27T23:39:45.171142+0000 mon.a (mon.0) 2469 : audit [DBG] from='client.? 172.21.15.6:0/1378941047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:46.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:45 smithi006 bash[33918]: audit 2023-08-27T23:39:45.171142+0000 mon.a (mon.0) 2469 : audit [DBG] from='client.? 172.21.15.6:0/1378941047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:46 smithi130 bash[27579]: cluster 2023-08-27T23:39:45.390694+0000 mgr.y (mgr.14901) 3036 : cluster [DBG] pgmap v3043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:47.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:46 smithi130 bash[51302]: debug 2023-08-27T23:39:46.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:46.695870+0000) 2023-08-27T23:39:47.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:46 smithi006 bash[27292]: cluster 2023-08-27T23:39:45.390694+0000 mgr.y (mgr.14901) 3036 : cluster [DBG] pgmap v3043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:47.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:46 smithi006 bash[33918]: cluster 2023-08-27T23:39:45.390694+0000 mgr.y (mgr.14901) 3036 : cluster [DBG] pgmap v3043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:48.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:47 smithi130 bash[27579]: cluster 2023-08-27T23:39:47.391345+0000 mgr.y (mgr.14901) 3037 : cluster [DBG] pgmap v3044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:48.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:47 smithi130 bash[27579]: audit 2023-08-27T23:39:47.601003+0000 mon.c (mon.2) 1174 : audit [DBG] from='client.? 172.21.15.6:0/2467788658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:48.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:47 smithi130 bash[51302]: debug 2023-08-27T23:39:47.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:47.696190+0000) 2023-08-27T23:39:48.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:47 smithi006 bash[27292]: cluster 2023-08-27T23:39:47.391345+0000 mgr.y (mgr.14901) 3037 : cluster [DBG] pgmap v3044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:48.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:47 smithi006 bash[27292]: audit 2023-08-27T23:39:47.601003+0000 mon.c (mon.2) 1174 : audit [DBG] from='client.? 172.21.15.6:0/2467788658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:48.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:47 smithi006 bash[33918]: cluster 2023-08-27T23:39:47.391345+0000 mgr.y (mgr.14901) 3037 : cluster [DBG] pgmap v3044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:48.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:47 smithi006 bash[33918]: audit 2023-08-27T23:39:47.601003+0000 mon.c (mon.2) 1174 : audit [DBG] from='client.? 172.21.15.6:0/2467788658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:49.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:48 smithi130 bash[51302]: debug 2023-08-27T23:39:48.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:48.696476+0000) 2023-08-27T23:39:50.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:49 smithi130 bash[51302]: debug 2023-08-27T23:39:49.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:49.696787+0000) 2023-08-27T23:39:50.921 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:39:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:39:50] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-27T23:39:50.921 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:39:50.525Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:50.921 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:39:50.526Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:50 smithi130 bash[27579]: cluster 2023-08-27T23:39:49.392431+0000 mgr.y (mgr.14901) 3038 : cluster [DBG] pgmap v3045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:50 smithi130 bash[27579]: audit 2023-08-27T23:39:50.006532+0000 mon.c (mon.2) 1175 : audit [DBG] from='client.? 172.21.15.6:0/3457626449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:51.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:50 smithi130 bash[51302]: debug 2023-08-27T23:39:50.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:50.697106+0000) 2023-08-27T23:39:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:50 smithi006 bash[33918]: cluster 2023-08-27T23:39:49.392431+0000 mgr.y (mgr.14901) 3038 : cluster [DBG] pgmap v3045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:51.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:50 smithi006 bash[33918]: audit 2023-08-27T23:39:50.006532+0000 mon.c (mon.2) 1175 : audit [DBG] from='client.? 172.21.15.6:0/3457626449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:50 smithi006 bash[27292]: cluster 2023-08-27T23:39:49.392431+0000 mgr.y (mgr.14901) 3038 : cluster [DBG] pgmap v3045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:50 smithi006 bash[27292]: audit 2023-08-27T23:39:50.006532+0000 mon.c (mon.2) 1175 : audit [DBG] from='client.? 172.21.15.6:0/3457626449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:52.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:51 smithi130 bash[27579]: cluster 2023-08-27T23:39:51.393120+0000 mgr.y (mgr.14901) 3039 : cluster [DBG] pgmap v3046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:52.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:51 smithi130 bash[51302]: debug 2023-08-27T23:39:51.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:51.697373+0000) 2023-08-27T23:39:52.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:51 smithi006 bash[33918]: cluster 2023-08-27T23:39:51.393120+0000 mgr.y (mgr.14901) 3039 : cluster [DBG] pgmap v3046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:52.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:51 smithi006 bash[27292]: cluster 2023-08-27T23:39:51.393120+0000 mgr.y (mgr.14901) 3039 : cluster [DBG] pgmap v3046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:52.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:39:52.092Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:52.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:39:52.094Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:52 smithi130 bash[27579]: audit 2023-08-27T23:39:52.416269+0000 mon.a (mon.0) 2470 : audit [DBG] from='client.? 172.21.15.6:0/1382963590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:52 smithi130 bash[51302]: debug 2023-08-27T23:39:52.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:52.697670+0000) 2023-08-27T23:39:53.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:52 smithi006 bash[33918]: audit 2023-08-27T23:39:52.416269+0000 mon.a (mon.0) 2470 : audit [DBG] from='client.? 172.21.15.6:0/1382963590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:53.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:52 smithi006 bash[27292]: audit 2023-08-27T23:39:52.416269+0000 mon.a (mon.0) 2470 : audit [DBG] from='client.? 172.21.15.6:0/1382963590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:54.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:53 smithi130 bash[27579]: cluster 2023-08-27T23:39:53.393845+0000 mgr.y (mgr.14901) 3040 : cluster [DBG] pgmap v3047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:54.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:53 smithi130 bash[51302]: debug 2023-08-27T23:39:53.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:53.697969+0000) 2023-08-27T23:39:54.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:53 smithi006 bash[33918]: cluster 2023-08-27T23:39:53.393845+0000 mgr.y (mgr.14901) 3040 : cluster [DBG] pgmap v3047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:54.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:53 smithi006 bash[27292]: cluster 2023-08-27T23:39:53.393845+0000 mgr.y (mgr.14901) 3040 : cluster [DBG] pgmap v3047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:54.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:39:53.992Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:54.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:39:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:39:53.994Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:39:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:54 smithi130 bash[27579]: audit 2023-08-27T23:39:54.829956+0000 mon.c (mon.2) 1176 : audit [DBG] from='client.? 172.21.15.6:0/3521499222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:54 smithi130 bash[51302]: debug 2023-08-27T23:39:54.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:54.698248+0000) 2023-08-27T23:39:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:54 smithi006 bash[27292]: audit 2023-08-27T23:39:54.829956+0000 mon.c (mon.2) 1176 : audit [DBG] from='client.? 172.21.15.6:0/3521499222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:55.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:54 smithi006 bash[33918]: audit 2023-08-27T23:39:54.829956+0000 mon.c (mon.2) 1176 : audit [DBG] from='client.? 172.21.15.6:0/3521499222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:55.951 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:55 smithi130 bash[51302]: debug 2023-08-27T23:39:55.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:55.698556+0000) 2023-08-27T23:39:56.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:55 smithi006 bash[27292]: cluster 2023-08-27T23:39:55.395011+0000 mgr.y (mgr.14901) 3041 : cluster [DBG] pgmap v3048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:56.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:55 smithi006 bash[33918]: cluster 2023-08-27T23:39:55.395011+0000 mgr.y (mgr.14901) 3041 : cluster [DBG] pgmap v3048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:56.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:55 smithi130 bash[27579]: cluster 2023-08-27T23:39:55.395011+0000 mgr.y (mgr.14901) 3041 : cluster [DBG] pgmap v3048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:57.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:56 smithi130 bash[51302]: debug 2023-08-27T23:39:56.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:56.698809+0000) 2023-08-27T23:39:57.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:57 smithi130 bash[27579]: audit 2023-08-27T23:39:57.238508+0000 mon.a (mon.0) 2471 : audit [DBG] from='client.? 172.21.15.6:0/1858153596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:57.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:57 smithi006 bash[33918]: audit 2023-08-27T23:39:57.238508+0000 mon.a (mon.0) 2471 : audit [DBG] from='client.? 172.21.15.6:0/1858153596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:57.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:57 smithi006 bash[27292]: audit 2023-08-27T23:39:57.238508+0000 mon.a (mon.0) 2471 : audit [DBG] from='client.? 172.21.15.6:0/1858153596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:58.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:57 smithi130 bash[51302]: debug 2023-08-27T23:39:57.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:57.699114+0000) 2023-08-27T23:39:58.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:39:58 smithi130 bash[27579]: cluster 2023-08-27T23:39:57.395690+0000 mgr.y (mgr.14901) 3042 : cluster [DBG] pgmap v3049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:39:58 smithi006 bash[27292]: cluster 2023-08-27T23:39:57.395690+0000 mgr.y (mgr.14901) 3042 : cluster [DBG] pgmap v3049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:58.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:39:58 smithi006 bash[33918]: cluster 2023-08-27T23:39:57.395690+0000 mgr.y (mgr.14901) 3042 : cluster [DBG] pgmap v3049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:39:59.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:58 smithi130 bash[51302]: debug 2023-08-27T23:39:58.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:58.699488+0000) 2023-08-27T23:40:00.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:39:59 smithi130 bash[51302]: debug 2023-08-27T23:39:59.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:59.699761+0000) 2023-08-27T23:40:00.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:40:00.526Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:00.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:40:00.528Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:00 smithi006 bash[27292]: cluster 2023-08-27T23:39:59.396403+0000 mgr.y (mgr.14901) 3043 : cluster [DBG] pgmap v3050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:00 smithi006 bash[27292]: audit 2023-08-27T23:39:59.651537+0000 mon.a (mon.0) 2472 : audit [DBG] from='client.? 172.21.15.6:0/1226734423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:00 smithi006 bash[27292]: cluster 2023-08-27T23:40:00.000114+0000 mon.a (mon.0) 2473 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:40:00.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:00 smithi006 bash[33918]: cluster 2023-08-27T23:39:59.396403+0000 mgr.y (mgr.14901) 3043 : cluster [DBG] pgmap v3050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:00.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:00 smithi006 bash[33918]: audit 2023-08-27T23:39:59.651537+0000 mon.a (mon.0) 2472 : audit [DBG] from='client.? 172.21.15.6:0/1226734423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:00.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:00 smithi006 bash[33918]: cluster 2023-08-27T23:40:00.000114+0000 mon.a (mon.0) 2473 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:40:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:00 smithi130 bash[27579]: cluster 2023-08-27T23:39:59.396403+0000 mgr.y (mgr.14901) 3043 : cluster [DBG] pgmap v3050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:00 smithi130 bash[27579]: audit 2023-08-27T23:39:59.651537+0000 mon.a (mon.0) 2472 : audit [DBG] from='client.? 172.21.15.6:0/1226734423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:00 smithi130 bash[27579]: cluster 2023-08-27T23:40:00.000114+0000 mon.a (mon.0) 2473 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:40:00.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:00 smithi130 bash[51302]: debug 2023-08-27T23:40:00.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:00.700030+0000) 2023-08-27T23:40:01.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:40:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:40:00] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-27T23:40:02.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:01 smithi130 bash[51302]: debug 2023-08-27T23:40:01.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:01.700291+0000) 2023-08-27T23:40:02.461 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:40:02.093Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:02.461 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:40:02.094Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:02 smithi006 bash[33918]: cluster 2023-08-27T23:40:01.397010+0000 mgr.y (mgr.14901) 3044 : cluster [DBG] pgmap v3051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:02 smithi006 bash[33918]: audit 2023-08-27T23:40:02.070716+0000 mon.a (mon.0) 2474 : audit [DBG] from='client.? 172.21.15.6:0/269942000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:02.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:02 smithi006 bash[27292]: cluster 2023-08-27T23:40:01.397010+0000 mgr.y (mgr.14901) 3044 : cluster [DBG] pgmap v3051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:02.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:02 smithi006 bash[27292]: audit 2023-08-27T23:40:02.070716+0000 mon.a (mon.0) 2474 : audit [DBG] from='client.? 172.21.15.6:0/269942000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:02 smithi130 bash[27579]: cluster 2023-08-27T23:40:01.397010+0000 mgr.y (mgr.14901) 3044 : cluster [DBG] pgmap v3051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:02 smithi130 bash[27579]: audit 2023-08-27T23:40:02.070716+0000 mon.a (mon.0) 2474 : audit [DBG] from='client.? 172.21.15.6:0/269942000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:02.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:02 smithi130 bash[51302]: debug 2023-08-27T23:40:02.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:02.700589+0000) 2023-08-27T23:40:04.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:03 smithi130 bash[51302]: debug 2023-08-27T23:40:03.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:03.700880+0000) 2023-08-27T23:40:04.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:40:03.993Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:04.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:40:03.994Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:04.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:04 smithi006 bash[33918]: cluster 2023-08-27T23:40:03.397600+0000 mgr.y (mgr.14901) 3045 : cluster [DBG] pgmap v3052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:04.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:04 smithi006 bash[27292]: cluster 2023-08-27T23:40:03.397600+0000 mgr.y (mgr.14901) 3045 : cluster [DBG] pgmap v3052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:04 smithi130 bash[27579]: cluster 2023-08-27T23:40:03.397600+0000 mgr.y (mgr.14901) 3045 : cluster [DBG] pgmap v3052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:04.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:04 smithi130 bash[51302]: debug 2023-08-27T23:40:04.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:04.701169+0000) 2023-08-27T23:40:05.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:05 smithi130 bash[27579]: audit 2023-08-27T23:40:04.487438+0000 mon.a (mon.0) 2475 : audit [DBG] from='client.? 172.21.15.6:0/2230262773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:05.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:05 smithi130 bash[51302]: debug 2023-08-27T23:40:05.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:05.701467+0000) 2023-08-27T23:40:05.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:05 smithi006 bash[27292]: audit 2023-08-27T23:40:04.487438+0000 mon.a (mon.0) 2475 : audit [DBG] from='client.? 172.21.15.6:0/2230262773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:05.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:05 smithi006 bash[33918]: audit 2023-08-27T23:40:04.487438+0000 mon.a (mon.0) 2475 : audit [DBG] from='client.? 172.21.15.6:0/2230262773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:06 smithi130 bash[27579]: cluster 2023-08-27T23:40:05.398788+0000 mgr.y (mgr.14901) 3046 : cluster [DBG] pgmap v3053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:06.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:06 smithi130 bash[51302]: debug 2023-08-27T23:40:06.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:06.701761+0000) 2023-08-27T23:40:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:06 smithi006 bash[33918]: cluster 2023-08-27T23:40:05.398788+0000 mgr.y (mgr.14901) 3046 : cluster [DBG] pgmap v3053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:06 smithi006 bash[27292]: cluster 2023-08-27T23:40:05.398788+0000 mgr.y (mgr.14901) 3046 : cluster [DBG] pgmap v3053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:07.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:07 smithi130 bash[27579]: audit 2023-08-27T23:40:06.915890+0000 mon.a (mon.0) 2476 : audit [DBG] from='client.? 172.21.15.6:0/2098835655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:07.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:07 smithi130 bash[51302]: debug 2023-08-27T23:40:07.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:07.702073+0000) 2023-08-27T23:40:07.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:07 smithi006 bash[27292]: audit 2023-08-27T23:40:06.915890+0000 mon.a (mon.0) 2476 : audit [DBG] from='client.? 172.21.15.6:0/2098835655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:07.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:07 smithi006 bash[33918]: audit 2023-08-27T23:40:06.915890+0000 mon.a (mon.0) 2476 : audit [DBG] from='client.? 172.21.15.6:0/2098835655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:08 smithi130 bash[27579]: cluster 2023-08-27T23:40:07.399450+0000 mgr.y (mgr.14901) 3047 : cluster [DBG] pgmap v3054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:08.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:08 smithi130 bash[51302]: debug 2023-08-27T23:40:08.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:08.702374+0000) 2023-08-27T23:40:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:08 smithi006 bash[27292]: cluster 2023-08-27T23:40:07.399450+0000 mgr.y (mgr.14901) 3047 : cluster [DBG] pgmap v3054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:08.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:08 smithi006 bash[33918]: cluster 2023-08-27T23:40:07.399450+0000 mgr.y (mgr.14901) 3047 : cluster [DBG] pgmap v3054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:09.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:09 smithi130 bash[27579]: audit 2023-08-27T23:40:09.323958+0000 mon.c (mon.2) 1177 : audit [DBG] from='client.? 172.21.15.6:0/3972629232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:09.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:09 smithi130 bash[51302]: debug 2023-08-27T23:40:09.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:09.702686+0000) 2023-08-27T23:40:09.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:09 smithi006 bash[33918]: audit 2023-08-27T23:40:09.323958+0000 mon.c (mon.2) 1177 : audit [DBG] from='client.? 172.21.15.6:0/3972629232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:09.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:09 smithi006 bash[27292]: audit 2023-08-27T23:40:09.323958+0000 mon.c (mon.2) 1177 : audit [DBG] from='client.? 172.21.15.6:0/3972629232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:10.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:10 smithi006 bash[27292]: cluster 2023-08-27T23:40:09.400563+0000 mgr.y (mgr.14901) 3048 : cluster [DBG] pgmap v3055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:10.756 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:10 smithi006 bash[33918]: cluster 2023-08-27T23:40:09.400563+0000 mgr.y (mgr.14901) 3048 : cluster [DBG] pgmap v3055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:10.756 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:40:10.527Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:10.756 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:40:10.529Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:10 smithi130 bash[27579]: cluster 2023-08-27T23:40:09.400563+0000 mgr.y (mgr.14901) 3048 : cluster [DBG] pgmap v3055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:10 smithi130 bash[51302]: debug 2023-08-27T23:40:10.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:10.702993+0000) 2023-08-27T23:40:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:40:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:40:10] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-27T23:40:12.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:11 smithi130 bash[51302]: debug 2023-08-27T23:40:11.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:11.703204+0000) 2023-08-27T23:40:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:40:12.093Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:12.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:40:12.095Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:12.901 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:12 smithi130 bash[51302]: debug 2023-08-27T23:40:12.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:12.703431+0000) 2023-08-27T23:40:12.901 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:12 smithi130 bash[27579]: cluster 2023-08-27T23:40:11.401225+0000 mgr.y (mgr.14901) 3049 : cluster [DBG] pgmap v3056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:12.901 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:12 smithi130 bash[27579]: audit 2023-08-27T23:40:11.738383+0000 mon.c (mon.2) 1178 : audit [DBG] from='client.? 172.21.15.6:0/500653811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:12.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:12 smithi006 bash[27292]: cluster 2023-08-27T23:40:11.401225+0000 mgr.y (mgr.14901) 3049 : cluster [DBG] pgmap v3056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:12.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:12 smithi006 bash[27292]: audit 2023-08-27T23:40:11.738383+0000 mon.c (mon.2) 1178 : audit [DBG] from='client.? 172.21.15.6:0/500653811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:12.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:12 smithi006 bash[33918]: cluster 2023-08-27T23:40:11.401225+0000 mgr.y (mgr.14901) 3049 : cluster [DBG] pgmap v3056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:12.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:12 smithi006 bash[33918]: audit 2023-08-27T23:40:11.738383+0000 mon.c (mon.2) 1178 : audit [DBG] from='client.? 172.21.15.6:0/500653811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:14.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:13 smithi130 bash[51302]: debug 2023-08-27T23:40:13.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:13.703727+0000) 2023-08-27T23:40:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:40:13.993Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:40:13.995Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:14 smithi130 bash[27579]: cluster 2023-08-27T23:40:13.401822+0000 mgr.y (mgr.14901) 3050 : cluster [DBG] pgmap v3057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:14 smithi130 bash[27579]: audit 2023-08-27T23:40:14.155967+0000 mon.a (mon.0) 2477 : audit [DBG] from='client.? 172.21.15.6:0/3150432437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:14 smithi130 bash[51302]: debug 2023-08-27T23:40:14.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:14.703998+0000) 2023-08-27T23:40:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:14 smithi006 bash[33918]: cluster 2023-08-27T23:40:13.401822+0000 mgr.y (mgr.14901) 3050 : cluster [DBG] pgmap v3057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:14.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:14 smithi006 bash[33918]: audit 2023-08-27T23:40:14.155967+0000 mon.a (mon.0) 2477 : audit [DBG] from='client.? 172.21.15.6:0/3150432437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:14.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:14 smithi006 bash[27292]: cluster 2023-08-27T23:40:13.401822+0000 mgr.y (mgr.14901) 3050 : cluster [DBG] pgmap v3057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:14.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:14 smithi006 bash[27292]: audit 2023-08-27T23:40:14.155967+0000 mon.a (mon.0) 2477 : audit [DBG] from='client.? 172.21.15.6:0/3150432437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:16.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:15 smithi130 bash[27579]: cluster 2023-08-27T23:40:15.402942+0000 mgr.y (mgr.14901) 3051 : cluster [DBG] pgmap v3058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:16.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:15 smithi130 bash[51302]: debug 2023-08-27T23:40:15.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:15.704263+0000) 2023-08-27T23:40:16.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:15 smithi006 bash[27292]: cluster 2023-08-27T23:40:15.402942+0000 mgr.y (mgr.14901) 3051 : cluster [DBG] pgmap v3058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:16.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:15 smithi006 bash[33918]: cluster 2023-08-27T23:40:15.402942+0000 mgr.y (mgr.14901) 3051 : cluster [DBG] pgmap v3058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:17.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:16 smithi130 bash[27579]: audit 2023-08-27T23:40:16.575101+0000 mon.a (mon.0) 2478 : audit [DBG] from='client.? 172.21.15.6:0/180224196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:17.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:16 smithi130 bash[51302]: debug 2023-08-27T23:40:16.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:16.704547+0000) 2023-08-27T23:40:17.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:16 smithi006 bash[27292]: audit 2023-08-27T23:40:16.575101+0000 mon.a (mon.0) 2478 : audit [DBG] from='client.? 172.21.15.6:0/180224196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:17.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:16 smithi006 bash[33918]: audit 2023-08-27T23:40:16.575101+0000 mon.a (mon.0) 2478 : audit [DBG] from='client.? 172.21.15.6:0/180224196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:18.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:17 smithi130 bash[27579]: cluster 2023-08-27T23:40:17.403596+0000 mgr.y (mgr.14901) 3052 : cluster [DBG] pgmap v3059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:18.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:17 smithi130 bash[51302]: debug 2023-08-27T23:40:17.699+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:17.704881+0000) 2023-08-27T23:40:18.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:17 smithi006 bash[33918]: cluster 2023-08-27T23:40:17.403596+0000 mgr.y (mgr.14901) 3052 : cluster [DBG] pgmap v3059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:18.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:17 smithi006 bash[27292]: cluster 2023-08-27T23:40:17.403596+0000 mgr.y (mgr.14901) 3052 : cluster [DBG] pgmap v3059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:19.041 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:18 smithi130 bash[51302]: debug 2023-08-27T23:40:18.699+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:18.705192+0000) 2023-08-27T23:40:19.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:19 smithi130 bash[27579]: audit 2023-08-27T23:40:18.984392+0000 mon.c (mon.2) 1179 : audit [DBG] from='client.? 172.21.15.6:0/60940175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:19.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:19 smithi006 bash[33918]: audit 2023-08-27T23:40:18.984392+0000 mon.c (mon.2) 1179 : audit [DBG] from='client.? 172.21.15.6:0/60940175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:19.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:19 smithi006 bash[27292]: audit 2023-08-27T23:40:18.984392+0000 mon.c (mon.2) 1179 : audit [DBG] from='client.? 172.21.15.6:0/60940175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:20.052 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:19 smithi130 bash[51302]: debug 2023-08-27T23:40:19.699+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:19.705457+0000) 2023-08-27T23:40:20.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:20 smithi130 bash[27579]: cluster 2023-08-27T23:40:19.404676+0000 mgr.y (mgr.14901) 3053 : cluster [DBG] pgmap v3060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:20.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:20 smithi006 bash[33918]: cluster 2023-08-27T23:40:19.404676+0000 mgr.y (mgr.14901) 3053 : cluster [DBG] pgmap v3060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:20.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:20 smithi006 bash[27292]: cluster 2023-08-27T23:40:19.404676+0000 mgr.y (mgr.14901) 3053 : cluster [DBG] pgmap v3060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:40:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:40:20] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-27T23:40:20.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:40:20.528Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:20.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:40:20.530Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:21.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:20 smithi130 bash[51302]: debug 2023-08-27T23:40:20.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:20.705763+0000) 2023-08-27T23:40:21.707 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:21 smithi130 bash[27579]: audit 2023-08-27T23:40:21.396590+0000 mon.c (mon.2) 1180 : audit [DBG] from='client.? 172.21.15.6:0/4244359660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:21.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:21 smithi006 bash[33918]: audit 2023-08-27T23:40:21.396590+0000 mon.c (mon.2) 1180 : audit [DBG] from='client.? 172.21.15.6:0/4244359660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:21.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:21 smithi006 bash[27292]: audit 2023-08-27T23:40:21.396590+0000 mon.c (mon.2) 1180 : audit [DBG] from='client.? 172.21.15.6:0/4244359660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:22.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:21 smithi130 bash[51302]: debug 2023-08-27T23:40:21.699+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:21.706028+0000) 2023-08-27T23:40:22.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:40:22.094Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:22.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:40:22.096Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:22 smithi006 bash[27292]: cluster 2023-08-27T23:40:21.405419+0000 mgr.y (mgr.14901) 3054 : cluster [DBG] pgmap v3061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:22 smithi006 bash[33918]: cluster 2023-08-27T23:40:21.405419+0000 mgr.y (mgr.14901) 3054 : cluster [DBG] pgmap v3061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:22 smithi130 bash[27579]: cluster 2023-08-27T23:40:21.405419+0000 mgr.y (mgr.14901) 3054 : cluster [DBG] pgmap v3061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:22 smithi130 bash[51302]: debug 2023-08-27T23:40:22.699+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:22.706315+0000) 2023-08-27T23:40:24.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:23 smithi130 bash[51302]: debug 2023-08-27T23:40:23.699+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:23.706590+0000) 2023-08-27T23:40:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:40:23.994Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:23 smithi006 bash[56298]: level=warn ts=2023-08-27T23:40:23.996Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:24 smithi130 bash[27579]: cluster 2023-08-27T23:40:23.406104+0000 mgr.y (mgr.14901) 3055 : cluster [DBG] pgmap v3062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:24 smithi130 bash[27579]: audit 2023-08-27T23:40:23.820537+0000 mon.a (mon.0) 2479 : audit [DBG] from='client.? 172.21.15.6:0/2781103071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:24 smithi130 bash[51302]: debug 2023-08-27T23:40:24.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:24.706921+0000) 2023-08-27T23:40:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:24 smithi006 bash[27292]: cluster 2023-08-27T23:40:23.406104+0000 mgr.y (mgr.14901) 3055 : cluster [DBG] pgmap v3062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:24.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:24 smithi006 bash[27292]: audit 2023-08-27T23:40:23.820537+0000 mon.a (mon.0) 2479 : audit [DBG] from='client.? 172.21.15.6:0/2781103071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:24.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:24 smithi006 bash[33918]: cluster 2023-08-27T23:40:23.406104+0000 mgr.y (mgr.14901) 3055 : cluster [DBG] pgmap v3062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:24.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:24 smithi006 bash[33918]: audit 2023-08-27T23:40:23.820537+0000 mon.a (mon.0) 2479 : audit [DBG] from='client.? 172.21.15.6:0/2781103071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:26.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:25 smithi130 bash[27579]: cluster 2023-08-27T23:40:25.407231+0000 mgr.y (mgr.14901) 3056 : cluster [DBG] pgmap v3063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:25 smithi130 bash[27579]: audit 2023-08-27T23:40:25.441970+0000 mon.b (mon.1) 409 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:40:26.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:25 smithi130 bash[51302]: debug 2023-08-27T23:40:25.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:25.707161+0000) 2023-08-27T23:40:26.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:25 smithi006 bash[33918]: cluster 2023-08-27T23:40:25.407231+0000 mgr.y (mgr.14901) 3056 : cluster [DBG] pgmap v3063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:26.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:25 smithi006 bash[33918]: audit 2023-08-27T23:40:25.441970+0000 mon.b (mon.1) 409 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:40:26.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:25 smithi006 bash[27292]: cluster 2023-08-27T23:40:25.407231+0000 mgr.y (mgr.14901) 3056 : cluster [DBG] pgmap v3063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:26.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:25 smithi006 bash[27292]: audit 2023-08-27T23:40:25.441970+0000 mon.b (mon.1) 409 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:40:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:26 smithi130 bash[27579]: audit 2023-08-27T23:40:26.233719+0000 mon.c (mon.2) 1181 : audit [DBG] from='client.? 172.21.15.6:0/1796269643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:27.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:26 smithi130 bash[51302]: debug 2023-08-27T23:40:26.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:26.707378+0000) 2023-08-27T23:40:27.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:26 smithi006 bash[33918]: audit 2023-08-27T23:40:26.233719+0000 mon.c (mon.2) 1181 : audit [DBG] from='client.? 172.21.15.6:0/1796269643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:27.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:26 smithi006 bash[27292]: audit 2023-08-27T23:40:26.233719+0000 mon.c (mon.2) 1181 : audit [DBG] from='client.? 172.21.15.6:0/1796269643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:28.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:27 smithi130 bash[27579]: cluster 2023-08-27T23:40:27.407900+0000 mgr.y (mgr.14901) 3057 : cluster [DBG] pgmap v3064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:28.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:27 smithi130 bash[51302]: debug 2023-08-27T23:40:27.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:27.707633+0000) 2023-08-27T23:40:28.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:27 smithi006 bash[33918]: cluster 2023-08-27T23:40:27.407900+0000 mgr.y (mgr.14901) 3057 : cluster [DBG] pgmap v3064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:28.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:27 smithi006 bash[27292]: cluster 2023-08-27T23:40:27.407900+0000 mgr.y (mgr.14901) 3057 : cluster [DBG] pgmap v3064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:28 smithi130 bash[27579]: audit 2023-08-27T23:40:28.657216+0000 mon.c (mon.2) 1182 : audit [DBG] from='client.? 172.21.15.6:0/1377112870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:29.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:28 smithi130 bash[51302]: debug 2023-08-27T23:40:28.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:28.707860+0000) 2023-08-27T23:40:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:28 smithi006 bash[33918]: audit 2023-08-27T23:40:28.657216+0000 mon.c (mon.2) 1182 : audit [DBG] from='client.? 172.21.15.6:0/1377112870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:29.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:28 smithi006 bash[27292]: audit 2023-08-27T23:40:28.657216+0000 mon.c (mon.2) 1182 : audit [DBG] from='client.? 172.21.15.6:0/1377112870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:29.967 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:29 smithi130 bash[51302]: debug 2023-08-27T23:40:29.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:29.708117+0000) 2023-08-27T23:40:30.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:29 smithi006 bash[33918]: cluster 2023-08-27T23:40:29.409009+0000 mgr.y (mgr.14901) 3058 : cluster [DBG] pgmap v3065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:30.220 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:29 smithi006 bash[27292]: cluster 2023-08-27T23:40:29.409009+0000 mgr.y (mgr.14901) 3058 : cluster [DBG] pgmap v3065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:30.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:29 smithi130 bash[27579]: cluster 2023-08-27T23:40:29.409009+0000 mgr.y (mgr.14901) 3058 : cluster [DBG] pgmap v3065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:40:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:40:30] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-27T23:40:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:40:30.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:40:30.531Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:31.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:30 smithi130 bash[51302]: debug 2023-08-27T23:40:30.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:30.708385+0000) 2023-08-27T23:40:31.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:31 smithi006 bash[33918]: audit 2023-08-27T23:40:31.115290+0000 mon.a (mon.0) 2480 : audit [DBG] from='client.? 172.21.15.6:0/2345678686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:31.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:31 smithi006 bash[27292]: audit 2023-08-27T23:40:31.115290+0000 mon.a (mon.0) 2480 : audit [DBG] from='client.? 172.21.15.6:0/2345678686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:31.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:31 smithi130 bash[27579]: audit 2023-08-27T23:40:31.115290+0000 mon.a (mon.0) 2480 : audit [DBG] from='client.? 172.21.15.6:0/2345678686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:32.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:31 smithi130 bash[51302]: debug 2023-08-27T23:40:31.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:31.708648+0000) 2023-08-27T23:40:32.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:40:32.095Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:32.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:40:32.097Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:32.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:32 smithi006 bash[33918]: audit 2023-08-27T23:40:31.232040+0000 mon.a (mon.0) 2481 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:40:32.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:32 smithi006 bash[33918]: cluster 2023-08-27T23:40:31.409463+0000 mgr.y (mgr.14901) 3059 : cluster [DBG] pgmap v3066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:32.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:32 smithi006 bash[33918]: audit 2023-08-27T23:40:31.846359+0000 mon.a (mon.0) 2482 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:40:32.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:32 smithi006 bash[33918]: audit 2023-08-27T23:40:32.121622+0000 mon.b (mon.1) 410 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:40:32.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:32 smithi006 bash[33918]: audit 2023-08-27T23:40:32.123217+0000 mon.b (mon.1) 411 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:40:32.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:32 smithi006 bash[33918]: audit 2023-08-27T23:40:32.130101+0000 mon.a (mon.0) 2483 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:40:32.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:32 smithi006 bash[27292]: audit 2023-08-27T23:40:31.232040+0000 mon.a (mon.0) 2481 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:40:32.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:32 smithi006 bash[27292]: cluster 2023-08-27T23:40:31.409463+0000 mgr.y (mgr.14901) 3059 : cluster [DBG] pgmap v3066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:32.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:32 smithi006 bash[27292]: audit 2023-08-27T23:40:31.846359+0000 mon.a (mon.0) 2482 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:40:32.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:32 smithi006 bash[27292]: audit 2023-08-27T23:40:32.121622+0000 mon.b (mon.1) 410 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:40:32.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:32 smithi006 bash[27292]: audit 2023-08-27T23:40:32.123217+0000 mon.b (mon.1) 411 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:40:32.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:32 smithi006 bash[27292]: audit 2023-08-27T23:40:32.130101+0000 mon.a (mon.0) 2483 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:40:32.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:32 smithi130 bash[27579]: audit 2023-08-27T23:40:31.232040+0000 mon.a (mon.0) 2481 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:40:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:32 smithi130 bash[27579]: cluster 2023-08-27T23:40:31.409463+0000 mgr.y (mgr.14901) 3059 : cluster [DBG] pgmap v3066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:32 smithi130 bash[27579]: audit 2023-08-27T23:40:31.846359+0000 mon.a (mon.0) 2482 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:40:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:32 smithi130 bash[27579]: audit 2023-08-27T23:40:32.121622+0000 mon.b (mon.1) 410 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:40:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:32 smithi130 bash[27579]: audit 2023-08-27T23:40:32.123217+0000 mon.b (mon.1) 411 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:40:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:32 smithi130 bash[27579]: audit 2023-08-27T23:40:32.130101+0000 mon.a (mon.0) 2483 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:40:33.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:32 smithi130 bash[51302]: debug 2023-08-27T23:40:32.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:32.708981+0000) 2023-08-27T23:40:34.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:33 smithi130 bash[51302]: debug 2023-08-27T23:40:33.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:33.709268+0000) 2023-08-27T23:40:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:40:33.995Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:33 smithi006 bash[56298]: level=warn ts=2023-08-27T23:40:33.997Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:34 smithi130 bash[51302]: debug 2023-08-27T23:40:34.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:34.709600+0000) 2023-08-27T23:40:34.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:34 smithi130 bash[27579]: cluster 2023-08-27T23:40:33.410006+0000 mgr.y (mgr.14901) 3060 : cluster [DBG] pgmap v3067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:34.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:34 smithi130 bash[27579]: audit 2023-08-27T23:40:33.531841+0000 mon.a (mon.0) 2484 : audit [DBG] from='client.? 172.21.15.6:0/888406947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:34.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:34 smithi006 bash[27292]: cluster 2023-08-27T23:40:33.410006+0000 mgr.y (mgr.14901) 3060 : cluster [DBG] pgmap v3067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:34.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:34 smithi006 bash[27292]: audit 2023-08-27T23:40:33.531841+0000 mon.a (mon.0) 2484 : audit [DBG] from='client.? 172.21.15.6:0/888406947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:34.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:34 smithi006 bash[33918]: cluster 2023-08-27T23:40:33.410006+0000 mgr.y (mgr.14901) 3060 : cluster [DBG] pgmap v3067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:34.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:34 smithi006 bash[33918]: audit 2023-08-27T23:40:33.531841+0000 mon.a (mon.0) 2484 : audit [DBG] from='client.? 172.21.15.6:0/888406947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:36.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:35 smithi130 bash[27579]: cluster 2023-08-27T23:40:35.411097+0000 mgr.y (mgr.14901) 3061 : cluster [DBG] pgmap v3068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:36.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:35 smithi130 bash[51302]: debug 2023-08-27T23:40:35.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:35.709897+0000) 2023-08-27T23:40:36.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:35 smithi006 bash[27292]: cluster 2023-08-27T23:40:35.411097+0000 mgr.y (mgr.14901) 3061 : cluster [DBG] pgmap v3068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:36.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:35 smithi006 bash[33918]: cluster 2023-08-27T23:40:35.411097+0000 mgr.y (mgr.14901) 3061 : cluster [DBG] pgmap v3068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:36 smithi130 bash[27579]: audit 2023-08-27T23:40:35.950958+0000 mon.a (mon.0) 2485 : audit [DBG] from='client.? 172.21.15.6:0/33757161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:37.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:36 smithi130 bash[51302]: debug 2023-08-27T23:40:36.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:36.710313+0000) 2023-08-27T23:40:37.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:36 smithi006 bash[33918]: audit 2023-08-27T23:40:35.950958+0000 mon.a (mon.0) 2485 : audit [DBG] from='client.? 172.21.15.6:0/33757161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:37.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:36 smithi006 bash[27292]: audit 2023-08-27T23:40:35.950958+0000 mon.a (mon.0) 2485 : audit [DBG] from='client.? 172.21.15.6:0/33757161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:38.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:37 smithi130 bash[27579]: cluster 2023-08-27T23:40:37.411767+0000 mgr.y (mgr.14901) 3062 : cluster [DBG] pgmap v3069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:38.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:37 smithi130 bash[51302]: debug 2023-08-27T23:40:37.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:37.710568+0000) 2023-08-27T23:40:38.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:37 smithi006 bash[27292]: cluster 2023-08-27T23:40:37.411767+0000 mgr.y (mgr.14901) 3062 : cluster [DBG] pgmap v3069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:38.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:37 smithi006 bash[33918]: cluster 2023-08-27T23:40:37.411767+0000 mgr.y (mgr.14901) 3062 : cluster [DBG] pgmap v3069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:38 smithi130 bash[27579]: audit 2023-08-27T23:40:38.370108+0000 mon.c (mon.2) 1183 : audit [DBG] from='client.? 172.21.15.6:0/1326914175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:39.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:38 smithi130 bash[51302]: debug 2023-08-27T23:40:38.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:38.710886+0000) 2023-08-27T23:40:39.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:38 smithi006 bash[33918]: audit 2023-08-27T23:40:38.370108+0000 mon.c (mon.2) 1183 : audit [DBG] from='client.? 172.21.15.6:0/1326914175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:39.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:38 smithi006 bash[27292]: audit 2023-08-27T23:40:38.370108+0000 mon.c (mon.2) 1183 : audit [DBG] from='client.? 172.21.15.6:0/1326914175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:39.964 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:39 smithi130 bash[51302]: debug 2023-08-27T23:40:39.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:39.711102+0000) 2023-08-27T23:40:40.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:39 smithi006 bash[33918]: cluster 2023-08-27T23:40:39.412892+0000 mgr.y (mgr.14901) 3063 : cluster [DBG] pgmap v3070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:40.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:39 smithi006 bash[27292]: cluster 2023-08-27T23:40:39.412892+0000 mgr.y (mgr.14901) 3063 : cluster [DBG] pgmap v3070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:40.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:39 smithi130 bash[27579]: cluster 2023-08-27T23:40:39.412892+0000 mgr.y (mgr.14901) 3063 : cluster [DBG] pgmap v3070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:40:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:40:40] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-27T23:40:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:40:40.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:40:40.531Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:40.967 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:40 smithi130 bash[51302]: debug 2023-08-27T23:40:40.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:40.711410+0000) 2023-08-27T23:40:41.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:40 smithi006 bash[33918]: audit 2023-08-27T23:40:40.790402+0000 mon.a (mon.0) 2486 : audit [DBG] from='client.? 172.21.15.6:0/2505686351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:41.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:40 smithi006 bash[27292]: audit 2023-08-27T23:40:40.790402+0000 mon.a (mon.0) 2486 : audit [DBG] from='client.? 172.21.15.6:0/2505686351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:41.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:40 smithi130 bash[27579]: audit 2023-08-27T23:40:40.790402+0000 mon.a (mon.0) 2486 : audit [DBG] from='client.? 172.21.15.6:0/2505686351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:41.975 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:41 smithi130 bash[51302]: debug 2023-08-27T23:40:41.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:41.711686+0000) 2023-08-27T23:40:42.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:41 smithi130 bash[27579]: cluster 2023-08-27T23:40:41.413506+0000 mgr.y (mgr.14901) 3064 : cluster [DBG] pgmap v3071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:42.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:41 smithi006 bash[27292]: cluster 2023-08-27T23:40:41.413506+0000 mgr.y (mgr.14901) 3064 : cluster [DBG] pgmap v3071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:42.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:41 smithi006 bash[33918]: cluster 2023-08-27T23:40:41.413506+0000 mgr.y (mgr.14901) 3064 : cluster [DBG] pgmap v3071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:42.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:40:42.095Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:42.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:40:42.097Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:43.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:42 smithi130 bash[51302]: debug 2023-08-27T23:40:42.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:42.711955+0000) 2023-08-27T23:40:43.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:43 smithi130 bash[27579]: audit 2023-08-27T23:40:43.203813+0000 mon.a (mon.0) 2487 : audit [DBG] from='client.? 172.21.15.6:0/4059014343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:43.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:43 smithi006 bash[27292]: audit 2023-08-27T23:40:43.203813+0000 mon.a (mon.0) 2487 : audit [DBG] from='client.? 172.21.15.6:0/4059014343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:43.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:43 smithi006 bash[33918]: audit 2023-08-27T23:40:43.203813+0000 mon.a (mon.0) 2487 : audit [DBG] from='client.? 172.21.15.6:0/4059014343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:44.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:43 smithi130 bash[51302]: debug 2023-08-27T23:40:43.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:43.712291+0000) 2023-08-27T23:40:44.268 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:43 smithi006 bash[56298]: level=error ts=2023-08-27T23:40:43.995Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:44.268 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:43 smithi006 bash[56298]: level=warn ts=2023-08-27T23:40:43.998Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:44.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:44 smithi130 bash[27579]: cluster 2023-08-27T23:40:43.414191+0000 mgr.y (mgr.14901) 3065 : cluster [DBG] pgmap v3072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:44.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:44 smithi006 bash[33918]: cluster 2023-08-27T23:40:43.414191+0000 mgr.y (mgr.14901) 3065 : cluster [DBG] pgmap v3072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:44.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:44 smithi006 bash[27292]: cluster 2023-08-27T23:40:43.414191+0000 mgr.y (mgr.14901) 3065 : cluster [DBG] pgmap v3072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:44 smithi130 bash[51302]: debug 2023-08-27T23:40:44.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:44.712613+0000) 2023-08-27T23:40:46.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:45 smithi130 bash[27579]: cluster 2023-08-27T23:40:45.415275+0000 mgr.y (mgr.14901) 3066 : cluster [DBG] pgmap v3073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:46.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:45 smithi130 bash[27579]: audit 2023-08-27T23:40:45.618769+0000 mon.c (mon.2) 1184 : audit [DBG] from='client.? 172.21.15.6:0/405691778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:46.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:45 smithi130 bash[51302]: debug 2023-08-27T23:40:45.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:45.712904+0000) 2023-08-27T23:40:46.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:45 smithi006 bash[27292]: cluster 2023-08-27T23:40:45.415275+0000 mgr.y (mgr.14901) 3066 : cluster [DBG] pgmap v3073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:46.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:45 smithi006 bash[27292]: audit 2023-08-27T23:40:45.618769+0000 mon.c (mon.2) 1184 : audit [DBG] from='client.? 172.21.15.6:0/405691778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:46.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:45 smithi006 bash[33918]: cluster 2023-08-27T23:40:45.415275+0000 mgr.y (mgr.14901) 3066 : cluster [DBG] pgmap v3073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:46.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:45 smithi006 bash[33918]: audit 2023-08-27T23:40:45.618769+0000 mon.c (mon.2) 1184 : audit [DBG] from='client.? 172.21.15.6:0/405691778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:47.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:46 smithi130 bash[51302]: debug 2023-08-27T23:40:46.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:46.713156+0000) 2023-08-27T23:40:48.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:47 smithi130 bash[51302]: debug 2023-08-27T23:40:47.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:47.713457+0000) 2023-08-27T23:40:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:48 smithi130 bash[27579]: cluster 2023-08-27T23:40:47.415943+0000 mgr.y (mgr.14901) 3067 : cluster [DBG] pgmap v3074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:48 smithi130 bash[27579]: audit 2023-08-27T23:40:48.030900+0000 mon.c (mon.2) 1185 : audit [DBG] from='client.? 172.21.15.6:0/537722743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:48 smithi130 bash[51302]: debug 2023-08-27T23:40:48.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:48.713731+0000) 2023-08-27T23:40:48.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:48 smithi006 bash[33918]: cluster 2023-08-27T23:40:47.415943+0000 mgr.y (mgr.14901) 3067 : cluster [DBG] pgmap v3074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:48.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:48 smithi006 bash[33918]: audit 2023-08-27T23:40:48.030900+0000 mon.c (mon.2) 1185 : audit [DBG] from='client.? 172.21.15.6:0/537722743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:48.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:48 smithi006 bash[27292]: cluster 2023-08-27T23:40:47.415943+0000 mgr.y (mgr.14901) 3067 : cluster [DBG] pgmap v3074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:48.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:48 smithi006 bash[27292]: audit 2023-08-27T23:40:48.030900+0000 mon.c (mon.2) 1185 : audit [DBG] from='client.? 172.21.15.6:0/537722743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:50.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:49 smithi130 bash[51302]: debug 2023-08-27T23:40:49.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:49.714022+0000) 2023-08-27T23:40:50.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:40:50.530Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:50.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:40:50.533Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:50.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:50 smithi006 bash[33918]: cluster 2023-08-27T23:40:49.417050+0000 mgr.y (mgr.14901) 3068 : cluster [DBG] pgmap v3075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:50.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:50 smithi006 bash[33918]: audit 2023-08-27T23:40:50.446676+0000 mon.a (mon.0) 2488 : audit [DBG] from='client.? 172.21.15.6:0/4279991076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:50.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:50 smithi006 bash[27292]: cluster 2023-08-27T23:40:49.417050+0000 mgr.y (mgr.14901) 3068 : cluster [DBG] pgmap v3075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:50.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:50 smithi006 bash[27292]: audit 2023-08-27T23:40:50.446676+0000 mon.a (mon.0) 2488 : audit [DBG] from='client.? 172.21.15.6:0/4279991076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:50 smithi130 bash[27579]: cluster 2023-08-27T23:40:49.417050+0000 mgr.y (mgr.14901) 3068 : cluster [DBG] pgmap v3075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:50 smithi130 bash[27579]: audit 2023-08-27T23:40:50.446676+0000 mon.a (mon.0) 2488 : audit [DBG] from='client.? 172.21.15.6:0/4279991076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:50 smithi130 bash[51302]: debug 2023-08-27T23:40:50.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:50.714309+0000) 2023-08-27T23:40:51.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:40:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:40:50] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-27T23:40:52.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:51 smithi130 bash[51302]: debug 2023-08-27T23:40:51.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:51.714609+0000) 2023-08-27T23:40:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:40:52.096Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:52.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:40:52.098Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:52 smithi130 bash[27579]: cluster 2023-08-27T23:40:51.417666+0000 mgr.y (mgr.14901) 3069 : cluster [DBG] pgmap v3076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:52.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:52 smithi130 bash[51302]: debug 2023-08-27T23:40:52.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:52.714929+0000) 2023-08-27T23:40:52.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:52 smithi006 bash[33918]: cluster 2023-08-27T23:40:51.417666+0000 mgr.y (mgr.14901) 3069 : cluster [DBG] pgmap v3076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:52.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:52 smithi006 bash[27292]: cluster 2023-08-27T23:40:51.417666+0000 mgr.y (mgr.14901) 3069 : cluster [DBG] pgmap v3076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:53.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:53 smithi130 bash[27579]: audit 2023-08-27T23:40:52.869291+0000 mon.c (mon.2) 1186 : audit [DBG] from='client.? 172.21.15.6:0/1935783776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:53.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:53 smithi130 bash[51302]: debug 2023-08-27T23:40:53.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:53.715147+0000) 2023-08-27T23:40:53.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:53 smithi006 bash[33918]: audit 2023-08-27T23:40:52.869291+0000 mon.c (mon.2) 1186 : audit [DBG] from='client.? 172.21.15.6:0/1935783776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:53.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:53 smithi006 bash[27292]: audit 2023-08-27T23:40:52.869291+0000 mon.c (mon.2) 1186 : audit [DBG] from='client.? 172.21.15.6:0/1935783776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:53 smithi006 bash[56298]: level=error ts=2023-08-27T23:40:53.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:40:53 smithi006 bash[56298]: level=warn ts=2023-08-27T23:40:53.998Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:40:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:54 smithi130 bash[27579]: cluster 2023-08-27T23:40:53.418328+0000 mgr.y (mgr.14901) 3070 : cluster [DBG] pgmap v3077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:54.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:54 smithi130 bash[51302]: debug 2023-08-27T23:40:54.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:54.715382+0000) 2023-08-27T23:40:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:54 smithi006 bash[33918]: cluster 2023-08-27T23:40:53.418328+0000 mgr.y (mgr.14901) 3070 : cluster [DBG] pgmap v3077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:54.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:54 smithi006 bash[27292]: cluster 2023-08-27T23:40:53.418328+0000 mgr.y (mgr.14901) 3070 : cluster [DBG] pgmap v3077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:56.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:55 smithi130 bash[51302]: debug 2023-08-27T23:40:55.715+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:55.715653+0000) 2023-08-27T23:40:56.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:55 smithi130 bash[27579]: audit 2023-08-27T23:40:55.290173+0000 mon.a (mon.0) 2489 : audit [DBG] from='client.? 172.21.15.6:0/2655095584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:56.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:55 smithi130 bash[27579]: cluster 2023-08-27T23:40:55.419449+0000 mgr.y (mgr.14901) 3071 : cluster [DBG] pgmap v3078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:56.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:55 smithi006 bash[27292]: audit 2023-08-27T23:40:55.290173+0000 mon.a (mon.0) 2489 : audit [DBG] from='client.? 172.21.15.6:0/2655095584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:56.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:55 smithi006 bash[27292]: cluster 2023-08-27T23:40:55.419449+0000 mgr.y (mgr.14901) 3071 : cluster [DBG] pgmap v3078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:56.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:55 smithi006 bash[33918]: audit 2023-08-27T23:40:55.290173+0000 mon.a (mon.0) 2489 : audit [DBG] from='client.? 172.21.15.6:0/2655095584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:56.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:55 smithi006 bash[33918]: cluster 2023-08-27T23:40:55.419449+0000 mgr.y (mgr.14901) 3071 : cluster [DBG] pgmap v3078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:57.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:56 smithi130 bash[51302]: debug 2023-08-27T23:40:56.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:56.715880+0000) 2023-08-27T23:40:58.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:57 smithi130 bash[51302]: debug 2023-08-27T23:40:57.715+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:57.716165+0000) 2023-08-27T23:40:58.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:58 smithi130 bash[51302]: debug 2023-08-27T23:40:58.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:58.716477+0000) 2023-08-27T23:40:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:58 smithi130 bash[27579]: cluster 2023-08-27T23:40:57.420081+0000 mgr.y (mgr.14901) 3072 : cluster [DBG] pgmap v3079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:40:58 smithi130 bash[27579]: audit 2023-08-27T23:40:57.706353+0000 mon.a (mon.0) 2490 : audit [DBG] from='client.? 172.21.15.6:0/2717870917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:58 smithi006 bash[33918]: cluster 2023-08-27T23:40:57.420081+0000 mgr.y (mgr.14901) 3072 : cluster [DBG] pgmap v3079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:58.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:40:58 smithi006 bash[33918]: audit 2023-08-27T23:40:57.706353+0000 mon.a (mon.0) 2490 : audit [DBG] from='client.? 172.21.15.6:0/2717870917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:58.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:58 smithi006 bash[27292]: cluster 2023-08-27T23:40:57.420081+0000 mgr.y (mgr.14901) 3072 : cluster [DBG] pgmap v3079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:40:58.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:40:58 smithi006 bash[27292]: audit 2023-08-27T23:40:57.706353+0000 mon.a (mon.0) 2490 : audit [DBG] from='client.? 172.21.15.6:0/2717870917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:00.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:40:59 smithi130 bash[51302]: debug 2023-08-27T23:40:59.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:59.716742+0000) 2023-08-27T23:41:00.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:00 smithi006 bash[33918]: cluster 2023-08-27T23:40:59.421187+0000 mgr.y (mgr.14901) 3073 : cluster [DBG] pgmap v3080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:00.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:00 smithi006 bash[33918]: audit 2023-08-27T23:41:00.203790+0000 mon.c (mon.2) 1187 : audit [DBG] from='client.? 172.21.15.6:0/2650455988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:00.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:00 smithi006 bash[27292]: cluster 2023-08-27T23:40:59.421187+0000 mgr.y (mgr.14901) 3073 : cluster [DBG] pgmap v3080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:00.758 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:00 smithi006 bash[27292]: audit 2023-08-27T23:41:00.203790+0000 mon.c (mon.2) 1187 : audit [DBG] from='client.? 172.21.15.6:0/2650455988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:00.759 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:41:00.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:00.759 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:41:00.533Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:00.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:00 smithi130 bash[51302]: debug 2023-08-27T23:41:00.715+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:00.717017+0000) 2023-08-27T23:41:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:00 smithi130 bash[27579]: cluster 2023-08-27T23:40:59.421187+0000 mgr.y (mgr.14901) 3073 : cluster [DBG] pgmap v3080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:00 smithi130 bash[27579]: audit 2023-08-27T23:41:00.203790+0000 mon.c (mon.2) 1187 : audit [DBG] from='client.? 172.21.15.6:0/2650455988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:01.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:41:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:41:00] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-27T23:41:02.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:01 smithi130 bash[51302]: debug 2023-08-27T23:41:01.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:01.717279+0000) 2023-08-27T23:41:02.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:41:02.098Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:02.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:41:02.099Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:02.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:02 smithi130 bash[51302]: debug 2023-08-27T23:41:02.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:02.717564+0000) 2023-08-27T23:41:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:02 smithi130 bash[27579]: cluster 2023-08-27T23:41:01.421840+0000 mgr.y (mgr.14901) 3074 : cluster [DBG] pgmap v3081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:02 smithi006 bash[33918]: cluster 2023-08-27T23:41:01.421840+0000 mgr.y (mgr.14901) 3074 : cluster [DBG] pgmap v3081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:02.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:02 smithi006 bash[27292]: cluster 2023-08-27T23:41:01.421840+0000 mgr.y (mgr.14901) 3074 : cluster [DBG] pgmap v3081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:03.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:03 smithi130 bash[27579]: audit 2023-08-27T23:41:02.619816+0000 mon.a (mon.0) 2491 : audit [DBG] from='client.? 172.21.15.6:0/3169953536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:03.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:03 smithi130 bash[51302]: debug 2023-08-27T23:41:03.715+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:03.717886+0000) 2023-08-27T23:41:03.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:03 smithi006 bash[27292]: audit 2023-08-27T23:41:02.619816+0000 mon.a (mon.0) 2491 : audit [DBG] from='client.? 172.21.15.6:0/3169953536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:03.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:03 smithi006 bash[33918]: audit 2023-08-27T23:41:02.619816+0000 mon.a (mon.0) 2491 : audit [DBG] from='client.? 172.21.15.6:0/3169953536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:03 smithi006 bash[56298]: level=error ts=2023-08-27T23:41:03.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:03 smithi006 bash[56298]: level=warn ts=2023-08-27T23:41:03.998Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:04 smithi130 bash[27579]: cluster 2023-08-27T23:41:03.422586+0000 mgr.y (mgr.14901) 3075 : cluster [DBG] pgmap v3082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:04.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:04 smithi130 bash[51302]: debug 2023-08-27T23:41:04.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:04.718166+0000) 2023-08-27T23:41:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:04 smithi006 bash[33918]: cluster 2023-08-27T23:41:03.422586+0000 mgr.y (mgr.14901) 3075 : cluster [DBG] pgmap v3082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:04.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:04 smithi006 bash[27292]: cluster 2023-08-27T23:41:03.422586+0000 mgr.y (mgr.14901) 3075 : cluster [DBG] pgmap v3082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:06.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:05 smithi130 bash[51302]: debug 2023-08-27T23:41:05.715+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:05.718446+0000) 2023-08-27T23:41:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:05 smithi130 bash[27579]: audit 2023-08-27T23:41:05.033942+0000 mon.a (mon.0) 2492 : audit [DBG] from='client.? 172.21.15.6:0/2864497849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:05 smithi130 bash[27579]: cluster 2023-08-27T23:41:05.423754+0000 mgr.y (mgr.14901) 3076 : cluster [DBG] pgmap v3083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:06.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:05 smithi006 bash[27292]: audit 2023-08-27T23:41:05.033942+0000 mon.a (mon.0) 2492 : audit [DBG] from='client.? 172.21.15.6:0/2864497849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:06.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:05 smithi006 bash[27292]: cluster 2023-08-27T23:41:05.423754+0000 mgr.y (mgr.14901) 3076 : cluster [DBG] pgmap v3083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:06.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:05 smithi006 bash[33918]: audit 2023-08-27T23:41:05.033942+0000 mon.a (mon.0) 2492 : audit [DBG] from='client.? 172.21.15.6:0/2864497849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:06.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:05 smithi006 bash[33918]: cluster 2023-08-27T23:41:05.423754+0000 mgr.y (mgr.14901) 3076 : cluster [DBG] pgmap v3083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:07.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:06 smithi130 bash[51302]: debug 2023-08-27T23:41:06.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:06.718706+0000) 2023-08-27T23:41:07.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:07 smithi130 bash[27579]: audit 2023-08-27T23:41:07.452733+0000 mon.a (mon.0) 2493 : audit [DBG] from='client.? 172.21.15.6:0/1104001578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:07.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:07 smithi130 bash[51302]: debug 2023-08-27T23:41:07.715+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:07.719003+0000) 2023-08-27T23:41:07.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:07 smithi006 bash[27292]: audit 2023-08-27T23:41:07.452733+0000 mon.a (mon.0) 2493 : audit [DBG] from='client.? 172.21.15.6:0/1104001578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:07.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:07 smithi006 bash[33918]: audit 2023-08-27T23:41:07.452733+0000 mon.a (mon.0) 2493 : audit [DBG] from='client.? 172.21.15.6:0/1104001578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:08 smithi130 bash[27579]: cluster 2023-08-27T23:41:07.424367+0000 mgr.y (mgr.14901) 3077 : cluster [DBG] pgmap v3084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:08.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:08 smithi130 bash[51302]: debug 2023-08-27T23:41:08.715+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:08.719243+0000) 2023-08-27T23:41:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:08 smithi006 bash[27292]: cluster 2023-08-27T23:41:07.424367+0000 mgr.y (mgr.14901) 3077 : cluster [DBG] pgmap v3084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:08.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:08 smithi006 bash[33918]: cluster 2023-08-27T23:41:07.424367+0000 mgr.y (mgr.14901) 3077 : cluster [DBG] pgmap v3084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:10.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:09 smithi130 bash[51302]: debug 2023-08-27T23:41:09.715+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:09.719455+0000) 2023-08-27T23:41:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:10 smithi130 bash[27579]: cluster 2023-08-27T23:41:09.425441+0000 mgr.y (mgr.14901) 3078 : cluster [DBG] pgmap v3085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:10 smithi130 bash[27579]: audit 2023-08-27T23:41:09.863594+0000 mon.a (mon.0) 2494 : audit [DBG] from='client.? 172.21.15.6:0/4092477521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:10 smithi130 bash[51302]: debug 2023-08-27T23:41:10.715+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:10.719767+0000) 2023-08-27T23:41:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:41:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:41:10] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-27T23:41:10.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:10 smithi006 bash[27292]: cluster 2023-08-27T23:41:09.425441+0000 mgr.y (mgr.14901) 3078 : cluster [DBG] pgmap v3085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:10.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:10 smithi006 bash[27292]: audit 2023-08-27T23:41:09.863594+0000 mon.a (mon.0) 2494 : audit [DBG] from='client.? 172.21.15.6:0/4092477521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:10.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:10 smithi006 bash[33918]: cluster 2023-08-27T23:41:09.425441+0000 mgr.y (mgr.14901) 3078 : cluster [DBG] pgmap v3085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:10.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:10 smithi006 bash[33918]: audit 2023-08-27T23:41:09.863594+0000 mon.a (mon.0) 2494 : audit [DBG] from='client.? 172.21.15.6:0/4092477521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:10.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:41:10.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:10.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:41:10.534Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:12.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:11 smithi130 bash[51302]: debug 2023-08-27T23:41:11.714+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:11.720090+0000) 2023-08-27T23:41:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:41:12.098Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:12.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:41:12.100Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:12.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:12 smithi130 bash[51302]: debug 2023-08-27T23:41:12.718+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:12.720398+0000) 2023-08-27T23:41:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:12 smithi130 bash[27579]: cluster 2023-08-27T23:41:11.426085+0000 mgr.y (mgr.14901) 3079 : cluster [DBG] pgmap v3086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:12 smithi130 bash[27579]: audit 2023-08-27T23:41:12.280130+0000 mon.a (mon.0) 2495 : audit [DBG] from='client.? 172.21.15.6:0/1672549079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:12 smithi006 bash[33918]: cluster 2023-08-27T23:41:11.426085+0000 mgr.y (mgr.14901) 3079 : cluster [DBG] pgmap v3086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:12.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:12 smithi006 bash[33918]: audit 2023-08-27T23:41:12.280130+0000 mon.a (mon.0) 2495 : audit [DBG] from='client.? 172.21.15.6:0/1672549079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:12.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:12 smithi006 bash[27292]: cluster 2023-08-27T23:41:11.426085+0000 mgr.y (mgr.14901) 3079 : cluster [DBG] pgmap v3086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:12.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:12 smithi006 bash[27292]: audit 2023-08-27T23:41:12.280130+0000 mon.a (mon.0) 2495 : audit [DBG] from='client.? 172.21.15.6:0/1672549079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:14.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:13 smithi130 bash[51302]: debug 2023-08-27T23:41:13.718+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:13.720711+0000) 2023-08-27T23:41:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:13 smithi006 bash[56298]: level=error ts=2023-08-27T23:41:13.997Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:13 smithi006 bash[56298]: level=warn ts=2023-08-27T23:41:13.999Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:14.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:14 smithi130 bash[51302]: debug 2023-08-27T23:41:14.718+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:14.721071+0000) 2023-08-27T23:41:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:14 smithi130 bash[27579]: cluster 2023-08-27T23:41:13.426724+0000 mgr.y (mgr.14901) 3080 : cluster [DBG] pgmap v3087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:14 smithi006 bash[27292]: cluster 2023-08-27T23:41:13.426724+0000 mgr.y (mgr.14901) 3080 : cluster [DBG] pgmap v3087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:14.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:14 smithi006 bash[33918]: cluster 2023-08-27T23:41:13.426724+0000 mgr.y (mgr.14901) 3080 : cluster [DBG] pgmap v3087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:15.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:15 smithi130 bash[27579]: audit 2023-08-27T23:41:14.697385+0000 mon.c (mon.2) 1188 : audit [DBG] from='client.? 172.21.15.6:0/1696063187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:15.870 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:15 smithi130 bash[51302]: debug 2023-08-27T23:41:15.718+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:15.721341+0000) 2023-08-27T23:41:15.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:15 smithi006 bash[27292]: audit 2023-08-27T23:41:14.697385+0000 mon.c (mon.2) 1188 : audit [DBG] from='client.? 172.21.15.6:0/1696063187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:15.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:15 smithi006 bash[33918]: audit 2023-08-27T23:41:14.697385+0000 mon.c (mon.2) 1188 : audit [DBG] from='client.? 172.21.15.6:0/1696063187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:16.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:16 smithi130 bash[27579]: cluster 2023-08-27T23:41:15.427924+0000 mgr.y (mgr.14901) 3081 : cluster [DBG] pgmap v3088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:16.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:16 smithi130 bash[51302]: debug 2023-08-27T23:41:16.714+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:16.721602+0000) 2023-08-27T23:41:16.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:16 smithi006 bash[27292]: cluster 2023-08-27T23:41:15.427924+0000 mgr.y (mgr.14901) 3081 : cluster [DBG] pgmap v3088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:16.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:16 smithi006 bash[33918]: cluster 2023-08-27T23:41:15.427924+0000 mgr.y (mgr.14901) 3081 : cluster [DBG] pgmap v3088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:17.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:17 smithi130 bash[51302]: debug 2023-08-27T23:41:17.714+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:17.721924+0000) 2023-08-27T23:41:17.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:17 smithi130 bash[27579]: audit 2023-08-27T23:41:17.113841+0000 mon.a (mon.0) 2496 : audit [DBG] from='client.? 172.21.15.6:0/2726804384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:17.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:17 smithi006 bash[33918]: audit 2023-08-27T23:41:17.113841+0000 mon.a (mon.0) 2496 : audit [DBG] from='client.? 172.21.15.6:0/2726804384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:17.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:17 smithi006 bash[27292]: audit 2023-08-27T23:41:17.113841+0000 mon.a (mon.0) 2496 : audit [DBG] from='client.? 172.21.15.6:0/2726804384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:18 smithi130 bash[27579]: cluster 2023-08-27T23:41:17.428618+0000 mgr.y (mgr.14901) 3082 : cluster [DBG] pgmap v3089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:18 smithi130 bash[51302]: debug 2023-08-27T23:41:18.714+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:18.722253+0000) 2023-08-27T23:41:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:18 smithi006 bash[33918]: cluster 2023-08-27T23:41:17.428618+0000 mgr.y (mgr.14901) 3082 : cluster [DBG] pgmap v3089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:18.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:18 smithi006 bash[27292]: cluster 2023-08-27T23:41:17.428618+0000 mgr.y (mgr.14901) 3082 : cluster [DBG] pgmap v3089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:19.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:19 smithi130 bash[27579]: audit 2023-08-27T23:41:19.533654+0000 mon.c (mon.2) 1189 : audit [DBG] from='client.? 172.21.15.6:0/571019745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:19 smithi130 bash[51302]: debug 2023-08-27T23:41:19.714+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:19.722524+0000) 2023-08-27T23:41:19.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:19 smithi006 bash[27292]: audit 2023-08-27T23:41:19.533654+0000 mon.c (mon.2) 1189 : audit [DBG] from='client.? 172.21.15.6:0/571019745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:19.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:19 smithi006 bash[33918]: audit 2023-08-27T23:41:19.533654+0000 mon.c (mon.2) 1189 : audit [DBG] from='client.? 172.21.15.6:0/571019745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:20.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:20 smithi130 bash[27579]: cluster 2023-08-27T23:41:19.429668+0000 mgr.y (mgr.14901) 3083 : cluster [DBG] pgmap v3090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:20 smithi130 bash[51302]: debug 2023-08-27T23:41:20.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:20.722831+0000) 2023-08-27T23:41:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:41:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:41:20] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-27T23:41:20.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:20 smithi006 bash[27292]: cluster 2023-08-27T23:41:19.429668+0000 mgr.y (mgr.14901) 3083 : cluster [DBG] pgmap v3090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:20.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:20 smithi006 bash[33918]: cluster 2023-08-27T23:41:19.429668+0000 mgr.y (mgr.14901) 3083 : cluster [DBG] pgmap v3090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:20.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:41:20.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:20.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:41:20.535Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:22.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:21 smithi130 bash[51302]: debug 2023-08-27T23:41:21.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:21.723082+0000) 2023-08-27T23:41:22.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:41:22.099Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:22.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:41:22.101Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:22 smithi130 bash[27579]: cluster 2023-08-27T23:41:21.430265+0000 mgr.y (mgr.14901) 3084 : cluster [DBG] pgmap v3091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:22 smithi130 bash[27579]: audit 2023-08-27T23:41:21.952642+0000 mon.c (mon.2) 1190 : audit [DBG] from='client.? 172.21.15.6:0/3281265139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:22 smithi130 bash[51302]: debug 2023-08-27T23:41:22.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:22.723382+0000) 2023-08-27T23:41:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:22 smithi006 bash[33918]: cluster 2023-08-27T23:41:21.430265+0000 mgr.y (mgr.14901) 3084 : cluster [DBG] pgmap v3091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:22.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:22 smithi006 bash[33918]: audit 2023-08-27T23:41:21.952642+0000 mon.c (mon.2) 1190 : audit [DBG] from='client.? 172.21.15.6:0/3281265139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:22.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:22 smithi006 bash[27292]: cluster 2023-08-27T23:41:21.430265+0000 mgr.y (mgr.14901) 3084 : cluster [DBG] pgmap v3091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:22.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:22 smithi006 bash[27292]: audit 2023-08-27T23:41:21.952642+0000 mon.c (mon.2) 1190 : audit [DBG] from='client.? 172.21.15.6:0/3281265139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:24.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:23 smithi130 bash[51302]: debug 2023-08-27T23:41:23.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:23.723664+0000) 2023-08-27T23:41:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:23 smithi006 bash[56298]: level=error ts=2023-08-27T23:41:23.998Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:24 smithi006 bash[56298]: level=warn ts=2023-08-27T23:41:24.000Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:24 smithi130 bash[27579]: cluster 2023-08-27T23:41:23.430913+0000 mgr.y (mgr.14901) 3085 : cluster [DBG] pgmap v3092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:24 smithi130 bash[27579]: audit 2023-08-27T23:41:24.373557+0000 mon.c (mon.2) 1191 : audit [DBG] from='client.? 172.21.15.6:0/1626166046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:24 smithi130 bash[51302]: debug 2023-08-27T23:41:24.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:24.723940+0000) 2023-08-27T23:41:24.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:24 smithi006 bash[27292]: cluster 2023-08-27T23:41:23.430913+0000 mgr.y (mgr.14901) 3085 : cluster [DBG] pgmap v3092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:24.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:24 smithi006 bash[27292]: audit 2023-08-27T23:41:24.373557+0000 mon.c (mon.2) 1191 : audit [DBG] from='client.? 172.21.15.6:0/1626166046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:24.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:24 smithi006 bash[33918]: cluster 2023-08-27T23:41:23.430913+0000 mgr.y (mgr.14901) 3085 : cluster [DBG] pgmap v3092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:24.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:24 smithi006 bash[33918]: audit 2023-08-27T23:41:24.373557+0000 mon.c (mon.2) 1191 : audit [DBG] from='client.? 172.21.15.6:0/1626166046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:25 smithi130 bash[27579]: cluster 2023-08-27T23:41:25.432042+0000 mgr.y (mgr.14901) 3086 : cluster [DBG] pgmap v3093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:26.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:25 smithi130 bash[51302]: debug 2023-08-27T23:41:25.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:25.724200+0000) 2023-08-27T23:41:26.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:25 smithi006 bash[33918]: cluster 2023-08-27T23:41:25.432042+0000 mgr.y (mgr.14901) 3086 : cluster [DBG] pgmap v3093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:26.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:25 smithi006 bash[27292]: cluster 2023-08-27T23:41:25.432042+0000 mgr.y (mgr.14901) 3086 : cluster [DBG] pgmap v3093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:26 smithi130 bash[27579]: audit 2023-08-27T23:41:26.797891+0000 mon.a (mon.0) 2497 : audit [DBG] from='client.? 172.21.15.6:0/2711768357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:27.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:26 smithi130 bash[51302]: debug 2023-08-27T23:41:26.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:26.724468+0000) 2023-08-27T23:41:27.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:26 smithi006 bash[33918]: audit 2023-08-27T23:41:26.797891+0000 mon.a (mon.0) 2497 : audit [DBG] from='client.? 172.21.15.6:0/2711768357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:27.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:26 smithi006 bash[27292]: audit 2023-08-27T23:41:26.797891+0000 mon.a (mon.0) 2497 : audit [DBG] from='client.? 172.21.15.6:0/2711768357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:28.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:27 smithi130 bash[27579]: cluster 2023-08-27T23:41:27.432713+0000 mgr.y (mgr.14901) 3087 : cluster [DBG] pgmap v3094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:28.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:27 smithi130 bash[51302]: debug 2023-08-27T23:41:27.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:27.724745+0000) 2023-08-27T23:41:28.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:27 smithi006 bash[27292]: cluster 2023-08-27T23:41:27.432713+0000 mgr.y (mgr.14901) 3087 : cluster [DBG] pgmap v3094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:28.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:27 smithi006 bash[33918]: cluster 2023-08-27T23:41:27.432713+0000 mgr.y (mgr.14901) 3087 : cluster [DBG] pgmap v3094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:29.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:28 smithi130 bash[51302]: debug 2023-08-27T23:41:28.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:28.725023+0000) 2023-08-27T23:41:29.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:29 smithi130 bash[27579]: audit 2023-08-27T23:41:29.205668+0000 mon.a (mon.0) 2498 : audit [DBG] from='client.? 172.21.15.6:0/170025104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:29.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:29 smithi006 bash[27292]: audit 2023-08-27T23:41:29.205668+0000 mon.a (mon.0) 2498 : audit [DBG] from='client.? 172.21.15.6:0/170025104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:29.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:29 smithi006 bash[33918]: audit 2023-08-27T23:41:29.205668+0000 mon.a (mon.0) 2498 : audit [DBG] from='client.? 172.21.15.6:0/170025104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:30.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:29 smithi130 bash[51302]: debug 2023-08-27T23:41:29.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:29.725280+0000) 2023-08-27T23:41:30.537 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:30 smithi006 bash[33918]: cluster 2023-08-27T23:41:29.433635+0000 mgr.y (mgr.14901) 3088 : cluster [DBG] pgmap v3095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:30.537 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:30 smithi006 bash[27292]: cluster 2023-08-27T23:41:29.433635+0000 mgr.y (mgr.14901) 3088 : cluster [DBG] pgmap v3095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:30 smithi130 bash[27579]: cluster 2023-08-27T23:41:29.433635+0000 mgr.y (mgr.14901) 3088 : cluster [DBG] pgmap v3095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:41:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:41:30] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:41:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:41:30.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:30.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:41:30.536Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:31.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:30 smithi130 bash[51302]: debug 2023-08-27T23:41:30.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:30.725541+0000) 2023-08-27T23:41:32.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:31 smithi130 bash[51302]: debug 2023-08-27T23:41:31.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:31.725760+0000) 2023-08-27T23:41:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:41:32.100Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:32.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:41:32.101Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:32.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:32 smithi130 bash[51302]: debug 2023-08-27T23:41:32.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:32.725983+0000) 2023-08-27T23:41:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:32 smithi130 bash[27579]: cluster 2023-08-27T23:41:31.434264+0000 mgr.y (mgr.14901) 3089 : cluster [DBG] pgmap v3096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:32 smithi130 bash[27579]: audit 2023-08-27T23:41:31.619515+0000 mon.a (mon.0) 2499 : audit [DBG] from='client.? 172.21.15.6:0/4097399295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:32 smithi130 bash[27579]: audit 2023-08-27T23:41:32.141531+0000 mon.b (mon.1) 412 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:41:32.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:32 smithi006 bash[27292]: cluster 2023-08-27T23:41:31.434264+0000 mgr.y (mgr.14901) 3089 : cluster [DBG] pgmap v3096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:32.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:32 smithi006 bash[27292]: audit 2023-08-27T23:41:31.619515+0000 mon.a (mon.0) 2499 : audit [DBG] from='client.? 172.21.15.6:0/4097399295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:32.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:32 smithi006 bash[27292]: audit 2023-08-27T23:41:32.141531+0000 mon.b (mon.1) 412 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:41:32.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:32 smithi006 bash[33918]: cluster 2023-08-27T23:41:31.434264+0000 mgr.y (mgr.14901) 3089 : cluster [DBG] pgmap v3096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:32.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:32 smithi006 bash[33918]: audit 2023-08-27T23:41:31.619515+0000 mon.a (mon.0) 2499 : audit [DBG] from='client.? 172.21.15.6:0/4097399295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:32.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:32 smithi006 bash[33918]: audit 2023-08-27T23:41:32.141531+0000 mon.b (mon.1) 412 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:41:34.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:33 smithi130 bash[51302]: debug 2023-08-27T23:41:33.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:33.726315+0000) 2023-08-27T23:41:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:33 smithi006 bash[56298]: level=error ts=2023-08-27T23:41:33.999Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:34 smithi006 bash[56298]: level=warn ts=2023-08-27T23:41:34.000Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:34.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:34 smithi130 bash[27579]: cluster 2023-08-27T23:41:33.434885+0000 mgr.y (mgr.14901) 3090 : cluster [DBG] pgmap v3097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:34 smithi130 bash[27579]: audit 2023-08-27T23:41:34.030903+0000 mon.a (mon.0) 2500 : audit [DBG] from='client.? 172.21.15.6:0/1206555504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:34 smithi130 bash[51302]: debug 2023-08-27T23:41:34.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:34.726596+0000) 2023-08-27T23:41:34.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:34 smithi006 bash[27292]: cluster 2023-08-27T23:41:33.434885+0000 mgr.y (mgr.14901) 3090 : cluster [DBG] pgmap v3097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:34.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:34 smithi006 bash[27292]: audit 2023-08-27T23:41:34.030903+0000 mon.a (mon.0) 2500 : audit [DBG] from='client.? 172.21.15.6:0/1206555504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:34.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:34 smithi006 bash[33918]: cluster 2023-08-27T23:41:33.434885+0000 mgr.y (mgr.14901) 3090 : cluster [DBG] pgmap v3097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:34.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:34 smithi006 bash[33918]: audit 2023-08-27T23:41:34.030903+0000 mon.a (mon.0) 2500 : audit [DBG] from='client.? 172.21.15.6:0/1206555504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:36.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:35 smithi130 bash[27579]: cluster 2023-08-27T23:41:35.435780+0000 mgr.y (mgr.14901) 3091 : cluster [DBG] pgmap v3098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:36.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:35 smithi130 bash[51302]: debug 2023-08-27T23:41:35.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:35.726938+0000) 2023-08-27T23:41:36.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:35 smithi006 bash[33918]: cluster 2023-08-27T23:41:35.435780+0000 mgr.y (mgr.14901) 3091 : cluster [DBG] pgmap v3098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:36.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:35 smithi006 bash[27292]: cluster 2023-08-27T23:41:35.435780+0000 mgr.y (mgr.14901) 3091 : cluster [DBG] pgmap v3098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:36 smithi130 bash[27579]: audit 2023-08-27T23:41:36.444801+0000 mon.a (mon.0) 2501 : audit [DBG] from='client.? 172.21.15.6:0/1629753075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:37.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:36 smithi130 bash[51302]: debug 2023-08-27T23:41:36.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:36.727170+0000) 2023-08-27T23:41:37.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:36 smithi006 bash[27292]: audit 2023-08-27T23:41:36.444801+0000 mon.a (mon.0) 2501 : audit [DBG] from='client.? 172.21.15.6:0/1629753075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:37.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:36 smithi006 bash[33918]: audit 2023-08-27T23:41:36.444801+0000 mon.a (mon.0) 2501 : audit [DBG] from='client.? 172.21.15.6:0/1629753075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:38.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:37 smithi130 bash[27579]: cluster 2023-08-27T23:41:37.436524+0000 mgr.y (mgr.14901) 3092 : cluster [DBG] pgmap v3099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:38.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:37 smithi130 bash[51302]: debug 2023-08-27T23:41:37.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:37.727387+0000) 2023-08-27T23:41:38.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:37 smithi006 bash[27292]: cluster 2023-08-27T23:41:37.436524+0000 mgr.y (mgr.14901) 3092 : cluster [DBG] pgmap v3099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:38.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:37 smithi006 bash[33918]: cluster 2023-08-27T23:41:37.436524+0000 mgr.y (mgr.14901) 3092 : cluster [DBG] pgmap v3099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:39.101 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:38 smithi130 bash[51302]: debug 2023-08-27T23:41:38.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:38.727613+0000) 2023-08-27T23:41:39.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:39 smithi130 bash[27579]: audit 2023-08-27T23:41:38.098086+0000 mon.a (mon.0) 2502 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:41:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:39 smithi130 bash[27579]: audit 2023-08-27T23:41:38.470052+0000 mon.a (mon.0) 2503 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:41:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:39 smithi130 bash[27579]: audit 2023-08-27T23:41:38.742729+0000 mon.b (mon.1) 413 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:41:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:39 smithi130 bash[27579]: audit 2023-08-27T23:41:38.744772+0000 mon.b (mon.1) 414 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:41:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:39 smithi130 bash[27579]: audit 2023-08-27T23:41:38.750473+0000 mon.a (mon.0) 2504 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:41:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:39 smithi130 bash[27579]: audit 2023-08-27T23:41:38.874736+0000 mon.a (mon.0) 2505 : audit [DBG] from='client.? 172.21.15.6:0/2871584810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:39.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:39 smithi006 bash[27292]: audit 2023-08-27T23:41:38.098086+0000 mon.a (mon.0) 2502 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:41:39.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:39 smithi006 bash[27292]: audit 2023-08-27T23:41:38.470052+0000 mon.a (mon.0) 2503 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:41:39.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:39 smithi006 bash[27292]: audit 2023-08-27T23:41:38.742729+0000 mon.b (mon.1) 413 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:41:39.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:39 smithi006 bash[27292]: audit 2023-08-27T23:41:38.744772+0000 mon.b (mon.1) 414 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:41:39.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:39 smithi006 bash[27292]: audit 2023-08-27T23:41:38.750473+0000 mon.a (mon.0) 2504 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:41:39.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:39 smithi006 bash[27292]: audit 2023-08-27T23:41:38.874736+0000 mon.a (mon.0) 2505 : audit [DBG] from='client.? 172.21.15.6:0/2871584810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:39.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:39 smithi006 bash[33918]: audit 2023-08-27T23:41:38.098086+0000 mon.a (mon.0) 2502 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:41:39.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:39 smithi006 bash[33918]: audit 2023-08-27T23:41:38.470052+0000 mon.a (mon.0) 2503 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:41:39.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:39 smithi006 bash[33918]: audit 2023-08-27T23:41:38.742729+0000 mon.b (mon.1) 413 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:41:39.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:39 smithi006 bash[33918]: audit 2023-08-27T23:41:38.744772+0000 mon.b (mon.1) 414 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:41:39.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:39 smithi006 bash[33918]: audit 2023-08-27T23:41:38.750473+0000 mon.a (mon.0) 2504 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:41:39.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:39 smithi006 bash[33918]: audit 2023-08-27T23:41:38.874736+0000 mon.a (mon.0) 2505 : audit [DBG] from='client.? 172.21.15.6:0/2871584810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:40.111 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:39 smithi130 bash[51302]: debug 2023-08-27T23:41:39.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:39.727899+0000) 2023-08-27T23:41:40.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:40 smithi130 bash[27579]: cluster 2023-08-27T23:41:39.437667+0000 mgr.y (mgr.14901) 3093 : cluster [DBG] pgmap v3100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:40.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:40 smithi006 bash[27292]: cluster 2023-08-27T23:41:39.437667+0000 mgr.y (mgr.14901) 3093 : cluster [DBG] pgmap v3100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:40 smithi006 bash[33918]: cluster 2023-08-27T23:41:39.437667+0000 mgr.y (mgr.14901) 3093 : cluster [DBG] pgmap v3100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:41:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:41:40] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:41:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:41:40.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:41:40.536Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:41.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:40 smithi130 bash[51302]: debug 2023-08-27T23:41:40.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:40.728256+0000) 2023-08-27T23:41:41.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:41 smithi130 bash[27579]: audit 2023-08-27T23:41:41.293155+0000 mon.c (mon.2) 1192 : audit [DBG] from='client.? 172.21.15.6:0/1266576160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:41.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:41 smithi006 bash[33918]: audit 2023-08-27T23:41:41.293155+0000 mon.c (mon.2) 1192 : audit [DBG] from='client.? 172.21.15.6:0/1266576160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:41.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:41 smithi006 bash[27292]: audit 2023-08-27T23:41:41.293155+0000 mon.c (mon.2) 1192 : audit [DBG] from='client.? 172.21.15.6:0/1266576160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:42.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:41 smithi130 bash[51302]: debug 2023-08-27T23:41:41.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:41.728520+0000) 2023-08-27T23:41:42.357 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:41:42.100Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:42.358 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:41:42.102Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:42 smithi130 bash[27579]: cluster 2023-08-27T23:41:41.438356+0000 mgr.y (mgr.14901) 3094 : cluster [DBG] pgmap v3101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:42.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:42 smithi006 bash[27292]: cluster 2023-08-27T23:41:41.438356+0000 mgr.y (mgr.14901) 3094 : cluster [DBG] pgmap v3101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:42.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:42 smithi006 bash[33918]: cluster 2023-08-27T23:41:41.438356+0000 mgr.y (mgr.14901) 3094 : cluster [DBG] pgmap v3101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:43.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:42 smithi130 bash[51302]: debug 2023-08-27T23:41:42.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:42.728771+0000) 2023-08-27T23:41:44.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:43 smithi130 bash[51302]: debug 2023-08-27T23:41:43.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:43.729099+0000) 2023-08-27T23:41:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:44 smithi006 bash[56298]: level=error ts=2023-08-27T23:41:44.000Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:44 smithi006 bash[56298]: level=warn ts=2023-08-27T23:41:44.002Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:44 smithi130 bash[27579]: cluster 2023-08-27T23:41:43.439067+0000 mgr.y (mgr.14901) 3095 : cluster [DBG] pgmap v3102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:44 smithi130 bash[27579]: audit 2023-08-27T23:41:43.711068+0000 mon.c (mon.2) 1193 : audit [DBG] from='client.? 172.21.15.6:0/2933629828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:44 smithi130 bash[51302]: debug 2023-08-27T23:41:44.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:44.729390+0000) 2023-08-27T23:41:44.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:44 smithi006 bash[27292]: cluster 2023-08-27T23:41:43.439067+0000 mgr.y (mgr.14901) 3095 : cluster [DBG] pgmap v3102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:44.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:44 smithi006 bash[27292]: audit 2023-08-27T23:41:43.711068+0000 mon.c (mon.2) 1193 : audit [DBG] from='client.? 172.21.15.6:0/2933629828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:44.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:44 smithi006 bash[33918]: cluster 2023-08-27T23:41:43.439067+0000 mgr.y (mgr.14901) 3095 : cluster [DBG] pgmap v3102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:44.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:44 smithi006 bash[33918]: audit 2023-08-27T23:41:43.711068+0000 mon.c (mon.2) 1193 : audit [DBG] from='client.? 172.21.15.6:0/2933629828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:46.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:45 smithi130 bash[27579]: cluster 2023-08-27T23:41:45.440217+0000 mgr.y (mgr.14901) 3096 : cluster [DBG] pgmap v3103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:46.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:45 smithi130 bash[51302]: debug 2023-08-27T23:41:45.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:45.729710+0000) 2023-08-27T23:41:46.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:45 smithi006 bash[27292]: cluster 2023-08-27T23:41:45.440217+0000 mgr.y (mgr.14901) 3096 : cluster [DBG] pgmap v3103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:46.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:45 smithi006 bash[33918]: cluster 2023-08-27T23:41:45.440217+0000 mgr.y (mgr.14901) 3096 : cluster [DBG] pgmap v3103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:46 smithi130 bash[27579]: audit 2023-08-27T23:41:46.127416+0000 mon.c (mon.2) 1194 : audit [DBG] from='client.? 172.21.15.6:0/212247275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:47.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:46 smithi130 bash[51302]: debug 2023-08-27T23:41:46.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:46.729998+0000) 2023-08-27T23:41:47.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:46 smithi006 bash[33918]: audit 2023-08-27T23:41:46.127416+0000 mon.c (mon.2) 1194 : audit [DBG] from='client.? 172.21.15.6:0/212247275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:47.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:46 smithi006 bash[27292]: audit 2023-08-27T23:41:46.127416+0000 mon.c (mon.2) 1194 : audit [DBG] from='client.? 172.21.15.6:0/212247275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:48.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:47 smithi130 bash[27579]: cluster 2023-08-27T23:41:47.440896+0000 mgr.y (mgr.14901) 3097 : cluster [DBG] pgmap v3104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:48.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:47 smithi130 bash[51302]: debug 2023-08-27T23:41:47.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:47.730351+0000) 2023-08-27T23:41:48.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:47 smithi006 bash[27292]: cluster 2023-08-27T23:41:47.440896+0000 mgr.y (mgr.14901) 3097 : cluster [DBG] pgmap v3104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:48.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:47 smithi006 bash[33918]: cluster 2023-08-27T23:41:47.440896+0000 mgr.y (mgr.14901) 3097 : cluster [DBG] pgmap v3104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:48 smithi130 bash[27579]: audit 2023-08-27T23:41:48.539079+0000 mon.c (mon.2) 1195 : audit [DBG] from='client.? 172.21.15.6:0/2705980845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:49.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:48 smithi130 bash[51302]: debug 2023-08-27T23:41:48.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:48.730640+0000) 2023-08-27T23:41:49.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:48 smithi006 bash[27292]: audit 2023-08-27T23:41:48.539079+0000 mon.c (mon.2) 1195 : audit [DBG] from='client.? 172.21.15.6:0/2705980845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:49.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:48 smithi006 bash[33918]: audit 2023-08-27T23:41:48.539079+0000 mon.c (mon.2) 1195 : audit [DBG] from='client.? 172.21.15.6:0/2705980845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:50.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:49 smithi130 bash[27579]: cluster 2023-08-27T23:41:49.441897+0000 mgr.y (mgr.14901) 3098 : cluster [DBG] pgmap v3105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:50.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:49 smithi130 bash[51302]: debug 2023-08-27T23:41:49.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:49.730936+0000) 2023-08-27T23:41:50.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:49 smithi006 bash[27292]: cluster 2023-08-27T23:41:49.441897+0000 mgr.y (mgr.14901) 3098 : cluster [DBG] pgmap v3105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:50.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:49 smithi006 bash[33918]: cluster 2023-08-27T23:41:49.441897+0000 mgr.y (mgr.14901) 3098 : cluster [DBG] pgmap v3105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:41:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:41:50] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-27T23:41:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:41:50.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:41:50.537Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:50 smithi130 bash[27579]: audit 2023-08-27T23:41:50.965450+0000 mon.a (mon.0) 2506 : audit [DBG] from='client.? 172.21.15.6:0/3815781341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:51.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:50 smithi130 bash[51302]: debug 2023-08-27T23:41:50.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:50.731234+0000) 2023-08-27T23:41:51.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:50 smithi006 bash[27292]: audit 2023-08-27T23:41:50.965450+0000 mon.a (mon.0) 2506 : audit [DBG] from='client.? 172.21.15.6:0/3815781341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:51.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:50 smithi006 bash[33918]: audit 2023-08-27T23:41:50.965450+0000 mon.a (mon.0) 2506 : audit [DBG] from='client.? 172.21.15.6:0/3815781341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:51.989 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:51 smithi130 bash[51302]: debug 2023-08-27T23:41:51.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:51.731546+0000) 2023-08-27T23:41:52.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:51 smithi130 bash[27579]: cluster 2023-08-27T23:41:51.442679+0000 mgr.y (mgr.14901) 3099 : cluster [DBG] pgmap v3106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:52.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:51 smithi006 bash[33918]: cluster 2023-08-27T23:41:51.442679+0000 mgr.y (mgr.14901) 3099 : cluster [DBG] pgmap v3106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:52.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:51 smithi006 bash[27292]: cluster 2023-08-27T23:41:51.442679+0000 mgr.y (mgr.14901) 3099 : cluster [DBG] pgmap v3106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:52.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:41:52.102Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:52.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:41:52.104Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:52 smithi130 bash[51302]: debug 2023-08-27T23:41:52.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:52.731807+0000) 2023-08-27T23:41:53.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:53 smithi006 bash[27292]: audit 2023-08-27T23:41:53.384170+0000 mon.a (mon.0) 2507 : audit [DBG] from='client.? 172.21.15.6:0/1297852866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:53 smithi006 bash[33918]: audit 2023-08-27T23:41:53.384170+0000 mon.a (mon.0) 2507 : audit [DBG] from='client.? 172.21.15.6:0/1297852866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:53.733 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:53 smithi130 bash[27579]: audit 2023-08-27T23:41:53.384170+0000 mon.a (mon.0) 2507 : audit [DBG] from='client.? 172.21.15.6:0/1297852866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:54.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:53 smithi130 bash[51302]: debug 2023-08-27T23:41:53.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:53.732180+0000) 2023-08-27T23:41:54.452 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:54 smithi006 bash[56298]: level=error ts=2023-08-27T23:41:54.001Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:54.452 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:41:54 smithi006 bash[56298]: level=warn ts=2023-08-27T23:41:54.003Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:41:54.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:54 smithi006 bash[27292]: cluster 2023-08-27T23:41:53.443347+0000 mgr.y (mgr.14901) 3100 : cluster [DBG] pgmap v3107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:54.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:54 smithi006 bash[33918]: cluster 2023-08-27T23:41:53.443347+0000 mgr.y (mgr.14901) 3100 : cluster [DBG] pgmap v3107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:54.733 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:54 smithi130 bash[27579]: cluster 2023-08-27T23:41:53.443347+0000 mgr.y (mgr.14901) 3100 : cluster [DBG] pgmap v3107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:55.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:54 smithi130 bash[51302]: debug 2023-08-27T23:41:54.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:54.732523+0000) 2023-08-27T23:41:56.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:55 smithi130 bash[51302]: debug 2023-08-27T23:41:55.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:55.732797+0000) 2023-08-27T23:41:56.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:55 smithi130 bash[27579]: cluster 2023-08-27T23:41:55.444428+0000 mgr.y (mgr.14901) 3101 : cluster [DBG] pgmap v3108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:56.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:55 smithi130 bash[27579]: audit 2023-08-27T23:41:55.797166+0000 mon.a (mon.0) 2508 : audit [DBG] from='client.? 172.21.15.6:0/214202114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:56.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:55 smithi006 bash[33918]: cluster 2023-08-27T23:41:55.444428+0000 mgr.y (mgr.14901) 3101 : cluster [DBG] pgmap v3108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:56.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:55 smithi006 bash[33918]: audit 2023-08-27T23:41:55.797166+0000 mon.a (mon.0) 2508 : audit [DBG] from='client.? 172.21.15.6:0/214202114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:56.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:55 smithi006 bash[27292]: cluster 2023-08-27T23:41:55.444428+0000 mgr.y (mgr.14901) 3101 : cluster [DBG] pgmap v3108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:56.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:55 smithi006 bash[27292]: audit 2023-08-27T23:41:55.797166+0000 mon.a (mon.0) 2508 : audit [DBG] from='client.? 172.21.15.6:0/214202114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:57.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:56 smithi130 bash[51302]: debug 2023-08-27T23:41:56.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:56.733061+0000) 2023-08-27T23:41:58.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:57 smithi130 bash[51302]: debug 2023-08-27T23:41:57.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:57.733317+0000) 2023-08-27T23:41:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:58 smithi130 bash[27579]: cluster 2023-08-27T23:41:57.445147+0000 mgr.y (mgr.14901) 3102 : cluster [DBG] pgmap v3109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:41:58 smithi130 bash[27579]: audit 2023-08-27T23:41:58.216869+0000 mon.c (mon.2) 1196 : audit [DBG] from='client.? 172.21.15.6:0/1238915213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:58.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:58 smithi130 bash[51302]: debug 2023-08-27T23:41:58.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:58.733614+0000) 2023-08-27T23:41:58.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:58 smithi006 bash[27292]: cluster 2023-08-27T23:41:57.445147+0000 mgr.y (mgr.14901) 3102 : cluster [DBG] pgmap v3109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:58.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:41:58 smithi006 bash[27292]: audit 2023-08-27T23:41:58.216869+0000 mon.c (mon.2) 1196 : audit [DBG] from='client.? 172.21.15.6:0/1238915213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:58.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:58 smithi006 bash[33918]: cluster 2023-08-27T23:41:57.445147+0000 mgr.y (mgr.14901) 3102 : cluster [DBG] pgmap v3109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:41:58.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:41:58 smithi006 bash[33918]: audit 2023-08-27T23:41:58.216869+0000 mon.c (mon.2) 1196 : audit [DBG] from='client.? 172.21.15.6:0/1238915213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:00.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:41:59 smithi130 bash[51302]: debug 2023-08-27T23:41:59.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:59.733904+0000) 2023-08-27T23:42:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:00 smithi130 bash[27579]: cluster 2023-08-27T23:41:59.446218+0000 mgr.y (mgr.14901) 3103 : cluster [DBG] pgmap v3110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:00.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:00 smithi130 bash[51302]: debug 2023-08-27T23:42:00.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:00.734153+0000) 2023-08-27T23:42:00.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:42:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:42:00] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-27T23:42:00.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:00 smithi006 bash[33918]: cluster 2023-08-27T23:41:59.446218+0000 mgr.y (mgr.14901) 3103 : cluster [DBG] pgmap v3110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:00.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:00 smithi006 bash[27292]: cluster 2023-08-27T23:41:59.446218+0000 mgr.y (mgr.14901) 3103 : cluster [DBG] pgmap v3110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:00.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:42:00.536Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:00.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:42:00.537Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:01.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:01 smithi130 bash[27579]: audit 2023-08-27T23:42:00.648532+0000 mon.a (mon.0) 2509 : audit [DBG] from='client.? 172.21.15.6:0/2863387614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:01.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:01 smithi130 bash[51302]: debug 2023-08-27T23:42:01.734+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:01.734422+0000) 2023-08-27T23:42:01.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:01 smithi006 bash[27292]: audit 2023-08-27T23:42:00.648532+0000 mon.a (mon.0) 2509 : audit [DBG] from='client.? 172.21.15.6:0/2863387614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:01.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:01 smithi006 bash[33918]: audit 2023-08-27T23:42:00.648532+0000 mon.a (mon.0) 2509 : audit [DBG] from='client.? 172.21.15.6:0/2863387614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:42:02.103Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:42:02.104Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:02 smithi130 bash[27579]: cluster 2023-08-27T23:42:01.446892+0000 mgr.y (mgr.14901) 3104 : cluster [DBG] pgmap v3111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:02.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:02 smithi130 bash[51302]: debug 2023-08-27T23:42:02.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:02.734699+0000) 2023-08-27T23:42:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:02 smithi006 bash[27292]: cluster 2023-08-27T23:42:01.446892+0000 mgr.y (mgr.14901) 3104 : cluster [DBG] pgmap v3111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:02.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:02 smithi006 bash[33918]: cluster 2023-08-27T23:42:01.446892+0000 mgr.y (mgr.14901) 3104 : cluster [DBG] pgmap v3111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:03.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:03 smithi130 bash[27579]: audit 2023-08-27T23:42:03.071052+0000 mon.a (mon.0) 2510 : audit [DBG] from='client.? 172.21.15.6:0/3485137677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:03.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:03 smithi130 bash[51302]: debug 2023-08-27T23:42:03.734+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:03.734991+0000) 2023-08-27T23:42:03.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:03 smithi006 bash[27292]: audit 2023-08-27T23:42:03.071052+0000 mon.a (mon.0) 2510 : audit [DBG] from='client.? 172.21.15.6:0/3485137677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:03.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:03 smithi006 bash[33918]: audit 2023-08-27T23:42:03.071052+0000 mon.a (mon.0) 2510 : audit [DBG] from='client.? 172.21.15.6:0/3485137677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:04 smithi006 bash[56298]: level=error ts=2023-08-27T23:42:04.002Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:04 smithi006 bash[56298]: level=warn ts=2023-08-27T23:42:04.003Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:04 smithi130 bash[27579]: cluster 2023-08-27T23:42:03.447672+0000 mgr.y (mgr.14901) 3105 : cluster [DBG] pgmap v3112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:04.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:04 smithi130 bash[51302]: debug 2023-08-27T23:42:04.734+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:04.735218+0000) 2023-08-27T23:42:04.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:04 smithi006 bash[27292]: cluster 2023-08-27T23:42:03.447672+0000 mgr.y (mgr.14901) 3105 : cluster [DBG] pgmap v3112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:04.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:04 smithi006 bash[33918]: cluster 2023-08-27T23:42:03.447672+0000 mgr.y (mgr.14901) 3105 : cluster [DBG] pgmap v3112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:06.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:05 smithi130 bash[27579]: cluster 2023-08-27T23:42:05.448721+0000 mgr.y (mgr.14901) 3106 : cluster [DBG] pgmap v3113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:05 smithi130 bash[27579]: audit 2023-08-27T23:42:05.491880+0000 mon.a (mon.0) 2511 : audit [DBG] from='client.? 172.21.15.6:0/4177221301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:06.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:05 smithi130 bash[51302]: debug 2023-08-27T23:42:05.734+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:05.735503+0000) 2023-08-27T23:42:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:05 smithi006 bash[33918]: cluster 2023-08-27T23:42:05.448721+0000 mgr.y (mgr.14901) 3106 : cluster [DBG] pgmap v3113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:06.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:05 smithi006 bash[33918]: audit 2023-08-27T23:42:05.491880+0000 mon.a (mon.0) 2511 : audit [DBG] from='client.? 172.21.15.6:0/4177221301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:06.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:05 smithi006 bash[27292]: cluster 2023-08-27T23:42:05.448721+0000 mgr.y (mgr.14901) 3106 : cluster [DBG] pgmap v3113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:06.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:05 smithi006 bash[27292]: audit 2023-08-27T23:42:05.491880+0000 mon.a (mon.0) 2511 : audit [DBG] from='client.? 172.21.15.6:0/4177221301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:07.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:06 smithi130 bash[51302]: debug 2023-08-27T23:42:06.734+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:06.735728+0000) 2023-08-27T23:42:08.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:07 smithi130 bash[51302]: debug 2023-08-27T23:42:07.734+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:07.736021+0000) 2023-08-27T23:42:08.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:08 smithi130 bash[51302]: debug 2023-08-27T23:42:08.733+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:08.736319+0000) 2023-08-27T23:42:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:08 smithi130 bash[27579]: cluster 2023-08-27T23:42:07.449370+0000 mgr.y (mgr.14901) 3107 : cluster [DBG] pgmap v3114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:08 smithi130 bash[27579]: audit 2023-08-27T23:42:07.902646+0000 mon.c (mon.2) 1197 : audit [DBG] from='client.? 172.21.15.6:0/256181692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:08.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:08 smithi006 bash[27292]: cluster 2023-08-27T23:42:07.449370+0000 mgr.y (mgr.14901) 3107 : cluster [DBG] pgmap v3114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:08.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:08 smithi006 bash[27292]: audit 2023-08-27T23:42:07.902646+0000 mon.c (mon.2) 1197 : audit [DBG] from='client.? 172.21.15.6:0/256181692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:08.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:08 smithi006 bash[33918]: cluster 2023-08-27T23:42:07.449370+0000 mgr.y (mgr.14901) 3107 : cluster [DBG] pgmap v3114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:08.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:08 smithi006 bash[33918]: audit 2023-08-27T23:42:07.902646+0000 mon.c (mon.2) 1197 : audit [DBG] from='client.? 172.21.15.6:0/256181692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:10.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:09 smithi130 bash[51302]: debug 2023-08-27T23:42:09.733+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:09.736609+0000) 2023-08-27T23:42:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:10 smithi130 bash[27579]: cluster 2023-08-27T23:42:09.450438+0000 mgr.y (mgr.14901) 3108 : cluster [DBG] pgmap v3115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:10 smithi130 bash[27579]: audit 2023-08-27T23:42:10.315097+0000 mon.a (mon.0) 2512 : audit [DBG] from='client.? 172.21.15.6:0/3170712665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:10 smithi130 bash[51302]: debug 2023-08-27T23:42:10.733+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:10.736914+0000) 2023-08-27T23:42:10.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:42:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:42:10] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-27T23:42:10.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:10 smithi006 bash[27292]: cluster 2023-08-27T23:42:09.450438+0000 mgr.y (mgr.14901) 3108 : cluster [DBG] pgmap v3115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:10.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:10 smithi006 bash[27292]: audit 2023-08-27T23:42:10.315097+0000 mon.a (mon.0) 2512 : audit [DBG] from='client.? 172.21.15.6:0/3170712665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:10.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:10 smithi006 bash[33918]: cluster 2023-08-27T23:42:09.450438+0000 mgr.y (mgr.14901) 3108 : cluster [DBG] pgmap v3115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:10.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:10 smithi006 bash[33918]: audit 2023-08-27T23:42:10.315097+0000 mon.a (mon.0) 2512 : audit [DBG] from='client.? 172.21.15.6:0/3170712665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:10.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:42:10.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:10.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:42:10.539Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:12.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:11 smithi130 bash[51302]: debug 2023-08-27T23:42:11.733+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:11.737169+0000) 2023-08-27T23:42:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:42:12.103Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:12.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:42:12.105Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:12 smithi130 bash[27579]: cluster 2023-08-27T23:42:11.451206+0000 mgr.y (mgr.14901) 3109 : cluster [DBG] pgmap v3116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:12 smithi130 bash[51302]: debug 2023-08-27T23:42:12.733+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:12.737421+0000) 2023-08-27T23:42:12.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:12 smithi006 bash[27292]: cluster 2023-08-27T23:42:11.451206+0000 mgr.y (mgr.14901) 3109 : cluster [DBG] pgmap v3116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:12.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:12 smithi006 bash[33918]: cluster 2023-08-27T23:42:11.451206+0000 mgr.y (mgr.14901) 3109 : cluster [DBG] pgmap v3116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:13.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:13 smithi130 bash[27579]: audit 2023-08-27T23:42:12.723988+0000 mon.a (mon.0) 2513 : audit [DBG] from='client.? 172.21.15.6:0/431689240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:13.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:13 smithi130 bash[51302]: debug 2023-08-27T23:42:13.733+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:13.737692+0000) 2023-08-27T23:42:13.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:13 smithi006 bash[27292]: audit 2023-08-27T23:42:12.723988+0000 mon.a (mon.0) 2513 : audit [DBG] from='client.? 172.21.15.6:0/431689240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:13.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:13 smithi006 bash[33918]: audit 2023-08-27T23:42:12.723988+0000 mon.a (mon.0) 2513 : audit [DBG] from='client.? 172.21.15.6:0/431689240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:14 smithi006 bash[56298]: level=error ts=2023-08-27T23:42:14.002Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:14 smithi006 bash[56298]: level=warn ts=2023-08-27T23:42:14.005Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:14 smithi130 bash[27579]: cluster 2023-08-27T23:42:13.451951+0000 mgr.y (mgr.14901) 3110 : cluster [DBG] pgmap v3117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:14 smithi130 bash[51302]: debug 2023-08-27T23:42:14.733+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:14.738034+0000) 2023-08-27T23:42:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:14 smithi006 bash[33918]: cluster 2023-08-27T23:42:13.451951+0000 mgr.y (mgr.14901) 3110 : cluster [DBG] pgmap v3117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:14.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:14 smithi006 bash[27292]: cluster 2023-08-27T23:42:13.451951+0000 mgr.y (mgr.14901) 3110 : cluster [DBG] pgmap v3117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:16.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:15 smithi130 bash[27579]: audit 2023-08-27T23:42:15.145401+0000 mon.a (mon.0) 2514 : audit [DBG] from='client.? 172.21.15.6:0/1572020778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:16.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:15 smithi130 bash[27579]: cluster 2023-08-27T23:42:15.453157+0000 mgr.y (mgr.14901) 3111 : cluster [DBG] pgmap v3118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:16.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:15 smithi130 bash[51302]: debug 2023-08-27T23:42:15.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:15.738309+0000) 2023-08-27T23:42:16.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:15 smithi006 bash[27292]: audit 2023-08-27T23:42:15.145401+0000 mon.a (mon.0) 2514 : audit [DBG] from='client.? 172.21.15.6:0/1572020778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:16.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:15 smithi006 bash[27292]: cluster 2023-08-27T23:42:15.453157+0000 mgr.y (mgr.14901) 3111 : cluster [DBG] pgmap v3118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:16.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:15 smithi006 bash[33918]: audit 2023-08-27T23:42:15.145401+0000 mon.a (mon.0) 2514 : audit [DBG] from='client.? 172.21.15.6:0/1572020778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:16.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:15 smithi006 bash[33918]: cluster 2023-08-27T23:42:15.453157+0000 mgr.y (mgr.14901) 3111 : cluster [DBG] pgmap v3118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:17.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:16 smithi130 bash[51302]: debug 2023-08-27T23:42:16.733+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:16.738604+0000) 2023-08-27T23:42:18.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:17 smithi130 bash[51302]: debug 2023-08-27T23:42:17.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:17.738923+0000) 2023-08-27T23:42:18.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:18 smithi130 bash[51302]: debug 2023-08-27T23:42:18.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:18.739269+0000) 2023-08-27T23:42:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:18 smithi130 bash[27579]: cluster 2023-08-27T23:42:17.453753+0000 mgr.y (mgr.14901) 3112 : cluster [DBG] pgmap v3119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:18 smithi130 bash[27579]: audit 2023-08-27T23:42:17.567158+0000 mon.c (mon.2) 1198 : audit [DBG] from='client.? 172.21.15.6:0/3098806718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:18.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:18 smithi006 bash[27292]: cluster 2023-08-27T23:42:17.453753+0000 mgr.y (mgr.14901) 3112 : cluster [DBG] pgmap v3119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:18.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:18 smithi006 bash[27292]: audit 2023-08-27T23:42:17.567158+0000 mon.c (mon.2) 1198 : audit [DBG] from='client.? 172.21.15.6:0/3098806718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:18.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:18 smithi006 bash[33918]: cluster 2023-08-27T23:42:17.453753+0000 mgr.y (mgr.14901) 3112 : cluster [DBG] pgmap v3119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:18.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:18 smithi006 bash[33918]: audit 2023-08-27T23:42:17.567158+0000 mon.c (mon.2) 1198 : audit [DBG] from='client.? 172.21.15.6:0/3098806718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:20.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:19 smithi130 bash[51302]: debug 2023-08-27T23:42:19.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:19.739630+0000) 2023-08-27T23:42:20.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:20 smithi130 bash[51302]: debug 2023-08-27T23:42:20.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:20.739904+0000) 2023-08-27T23:42:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:20 smithi130 bash[27579]: cluster 2023-08-27T23:42:19.454866+0000 mgr.y (mgr.14901) 3113 : cluster [DBG] pgmap v3120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:20 smithi130 bash[27579]: audit 2023-08-27T23:42:19.984594+0000 mon.a (mon.0) 2515 : audit [DBG] from='client.? 172.21.15.6:0/2864784762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:20.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:42:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:42:20] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-27T23:42:20.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:20 smithi006 bash[33918]: cluster 2023-08-27T23:42:19.454866+0000 mgr.y (mgr.14901) 3113 : cluster [DBG] pgmap v3120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:20.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:20 smithi006 bash[33918]: audit 2023-08-27T23:42:19.984594+0000 mon.a (mon.0) 2515 : audit [DBG] from='client.? 172.21.15.6:0/2864784762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:20.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:20 smithi006 bash[27292]: cluster 2023-08-27T23:42:19.454866+0000 mgr.y (mgr.14901) 3113 : cluster [DBG] pgmap v3120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:20.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:20 smithi006 bash[27292]: audit 2023-08-27T23:42:19.984594+0000 mon.a (mon.0) 2515 : audit [DBG] from='client.? 172.21.15.6:0/2864784762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:20.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:42:20.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:20.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:42:20.539Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:22.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:21 smithi130 bash[51302]: debug 2023-08-27T23:42:21.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:21.740174+0000) 2023-08-27T23:42:22.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:42:22.104Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:22.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:42:22.105Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:22 smithi130 bash[27579]: cluster 2023-08-27T23:42:21.455519+0000 mgr.y (mgr.14901) 3114 : cluster [DBG] pgmap v3121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:22 smithi130 bash[27579]: audit 2023-08-27T23:42:22.405682+0000 mon.a (mon.0) 2516 : audit [DBG] from='client.? 172.21.15.6:0/970678563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:22 smithi130 bash[51302]: debug 2023-08-27T23:42:22.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:22.740504+0000) 2023-08-27T23:42:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:22 smithi006 bash[33918]: cluster 2023-08-27T23:42:21.455519+0000 mgr.y (mgr.14901) 3114 : cluster [DBG] pgmap v3121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:22.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:22 smithi006 bash[33918]: audit 2023-08-27T23:42:22.405682+0000 mon.a (mon.0) 2516 : audit [DBG] from='client.? 172.21.15.6:0/970678563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:22.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:22 smithi006 bash[27292]: cluster 2023-08-27T23:42:21.455519+0000 mgr.y (mgr.14901) 3114 : cluster [DBG] pgmap v3121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:22.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:22 smithi006 bash[27292]: audit 2023-08-27T23:42:22.405682+0000 mon.a (mon.0) 2516 : audit [DBG] from='client.? 172.21.15.6:0/970678563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:24.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:23 smithi130 bash[51302]: debug 2023-08-27T23:42:23.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:23.740767+0000) 2023-08-27T23:42:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:24 smithi006 bash[56298]: level=error ts=2023-08-27T23:42:24.003Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:24 smithi006 bash[56298]: level=warn ts=2023-08-27T23:42:24.006Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:24 smithi130 bash[27579]: cluster 2023-08-27T23:42:23.456123+0000 mgr.y (mgr.14901) 3115 : cluster [DBG] pgmap v3122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:24 smithi130 bash[51302]: debug 2023-08-27T23:42:24.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:24.741042+0000) 2023-08-27T23:42:24.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:24 smithi006 bash[33918]: cluster 2023-08-27T23:42:23.456123+0000 mgr.y (mgr.14901) 3115 : cluster [DBG] pgmap v3122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:24.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:24 smithi006 bash[27292]: cluster 2023-08-27T23:42:23.456123+0000 mgr.y (mgr.14901) 3115 : cluster [DBG] pgmap v3122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:25.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:25 smithi130 bash[27579]: audit 2023-08-27T23:42:24.816048+0000 mon.c (mon.2) 1199 : audit [DBG] from='client.? 172.21.15.6:0/3250435496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:25.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:25 smithi130 bash[51302]: debug 2023-08-27T23:42:25.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:25.741332+0000) 2023-08-27T23:42:25.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:25 smithi006 bash[27292]: audit 2023-08-27T23:42:24.816048+0000 mon.c (mon.2) 1199 : audit [DBG] from='client.? 172.21.15.6:0/3250435496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:25.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:25 smithi006 bash[33918]: audit 2023-08-27T23:42:24.816048+0000 mon.c (mon.2) 1199 : audit [DBG] from='client.? 172.21.15.6:0/3250435496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:26 smithi130 bash[27579]: cluster 2023-08-27T23:42:25.457278+0000 mgr.y (mgr.14901) 3116 : cluster [DBG] pgmap v3123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:26.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:26 smithi130 bash[51302]: debug 2023-08-27T23:42:26.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:26.741630+0000) 2023-08-27T23:42:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:26 smithi006 bash[27292]: cluster 2023-08-27T23:42:25.457278+0000 mgr.y (mgr.14901) 3116 : cluster [DBG] pgmap v3123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:26.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:26 smithi006 bash[33918]: cluster 2023-08-27T23:42:25.457278+0000 mgr.y (mgr.14901) 3116 : cluster [DBG] pgmap v3123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:27.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:27 smithi130 bash[27579]: audit 2023-08-27T23:42:27.224216+0000 mon.a (mon.0) 2517 : audit [DBG] from='client.? 172.21.15.6:0/3513390138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:27.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:27 smithi130 bash[51302]: debug 2023-08-27T23:42:27.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:27.741953+0000) 2023-08-27T23:42:27.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:27 smithi006 bash[27292]: audit 2023-08-27T23:42:27.224216+0000 mon.a (mon.0) 2517 : audit [DBG] from='client.? 172.21.15.6:0/3513390138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:27.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:27 smithi006 bash[33918]: audit 2023-08-27T23:42:27.224216+0000 mon.a (mon.0) 2517 : audit [DBG] from='client.? 172.21.15.6:0/3513390138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:28 smithi130 bash[27579]: cluster 2023-08-27T23:42:27.457956+0000 mgr.y (mgr.14901) 3117 : cluster [DBG] pgmap v3124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:28 smithi130 bash[51302]: debug 2023-08-27T23:42:28.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:28.742240+0000) 2023-08-27T23:42:28.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:28 smithi006 bash[27292]: cluster 2023-08-27T23:42:27.457956+0000 mgr.y (mgr.14901) 3117 : cluster [DBG] pgmap v3124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:28.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:28 smithi006 bash[33918]: cluster 2023-08-27T23:42:27.457956+0000 mgr.y (mgr.14901) 3117 : cluster [DBG] pgmap v3124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:30.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:29 smithi130 bash[51302]: debug 2023-08-27T23:42:29.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:29.742539+0000) 2023-08-27T23:42:30.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:30 smithi130 bash[27579]: cluster 2023-08-27T23:42:29.459138+0000 mgr.y (mgr.14901) 3118 : cluster [DBG] pgmap v3125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:30 smithi130 bash[27579]: audit 2023-08-27T23:42:29.640464+0000 mon.a (mon.0) 2518 : audit [DBG] from='client.? 172.21.15.6:0/3379294342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:30 smithi130 bash[51302]: debug 2023-08-27T23:42:30.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:30.742887+0000) 2023-08-27T23:42:30.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:42:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:42:30] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-27T23:42:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:42:30.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:42:30.539Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:30.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:30 smithi006 bash[27292]: cluster 2023-08-27T23:42:29.459138+0000 mgr.y (mgr.14901) 3118 : cluster [DBG] pgmap v3125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:30.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:30 smithi006 bash[27292]: audit 2023-08-27T23:42:29.640464+0000 mon.a (mon.0) 2518 : audit [DBG] from='client.? 172.21.15.6:0/3379294342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:30.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:30 smithi006 bash[33918]: cluster 2023-08-27T23:42:29.459138+0000 mgr.y (mgr.14901) 3118 : cluster [DBG] pgmap v3125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:30.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:30 smithi006 bash[33918]: audit 2023-08-27T23:42:29.640464+0000 mon.a (mon.0) 2518 : audit [DBG] from='client.? 172.21.15.6:0/3379294342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:32.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:31 smithi130 bash[51302]: debug 2023-08-27T23:42:31.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:31.743150+0000) 2023-08-27T23:42:32.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:42:32.105Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:32.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:42:32.106Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:32.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:32 smithi130 bash[51302]: debug 2023-08-27T23:42:32.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:32.743429+0000) 2023-08-27T23:42:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:32 smithi130 bash[27579]: cluster 2023-08-27T23:42:31.459781+0000 mgr.y (mgr.14901) 3119 : cluster [DBG] pgmap v3126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:32 smithi130 bash[27579]: audit 2023-08-27T23:42:32.061126+0000 mon.a (mon.0) 2519 : audit [DBG] from='client.? 172.21.15.6:0/1720205582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:32.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:32 smithi006 bash[27292]: cluster 2023-08-27T23:42:31.459781+0000 mgr.y (mgr.14901) 3119 : cluster [DBG] pgmap v3126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:32.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:32 smithi006 bash[27292]: audit 2023-08-27T23:42:32.061126+0000 mon.a (mon.0) 2519 : audit [DBG] from='client.? 172.21.15.6:0/1720205582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:32.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:32 smithi006 bash[33918]: cluster 2023-08-27T23:42:31.459781+0000 mgr.y (mgr.14901) 3119 : cluster [DBG] pgmap v3126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:32.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:32 smithi006 bash[33918]: audit 2023-08-27T23:42:32.061126+0000 mon.a (mon.0) 2519 : audit [DBG] from='client.? 172.21.15.6:0/1720205582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:34.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:33 smithi130 bash[51302]: debug 2023-08-27T23:42:33.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:33.743700+0000) 2023-08-27T23:42:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:34 smithi006 bash[56298]: level=error ts=2023-08-27T23:42:34.004Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:34 smithi006 bash[56298]: level=warn ts=2023-08-27T23:42:34.006Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:34.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:34 smithi130 bash[51302]: debug 2023-08-27T23:42:34.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:34.744021+0000) 2023-08-27T23:42:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:34 smithi130 bash[27579]: cluster 2023-08-27T23:42:33.460435+0000 mgr.y (mgr.14901) 3120 : cluster [DBG] pgmap v3127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:34 smithi130 bash[27579]: audit 2023-08-27T23:42:34.481160+0000 mon.c (mon.2) 1200 : audit [DBG] from='client.? 172.21.15.6:0/3857396846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:34.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:34 smithi006 bash[27292]: cluster 2023-08-27T23:42:33.460435+0000 mgr.y (mgr.14901) 3120 : cluster [DBG] pgmap v3127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:34.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:34 smithi006 bash[27292]: audit 2023-08-27T23:42:34.481160+0000 mon.c (mon.2) 1200 : audit [DBG] from='client.? 172.21.15.6:0/3857396846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:34.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:34 smithi006 bash[33918]: cluster 2023-08-27T23:42:33.460435+0000 mgr.y (mgr.14901) 3120 : cluster [DBG] pgmap v3127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:34.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:34 smithi006 bash[33918]: audit 2023-08-27T23:42:34.481160+0000 mon.c (mon.2) 1200 : audit [DBG] from='client.? 172.21.15.6:0/3857396846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:36.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:35 smithi130 bash[27579]: cluster 2023-08-27T23:42:35.461552+0000 mgr.y (mgr.14901) 3121 : cluster [DBG] pgmap v3128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:36.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:35 smithi130 bash[51302]: debug 2023-08-27T23:42:35.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:35.744333+0000) 2023-08-27T23:42:36.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:35 smithi006 bash[27292]: cluster 2023-08-27T23:42:35.461552+0000 mgr.y (mgr.14901) 3121 : cluster [DBG] pgmap v3128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:36.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:35 smithi006 bash[33918]: cluster 2023-08-27T23:42:35.461552+0000 mgr.y (mgr.14901) 3121 : cluster [DBG] pgmap v3128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:36 smithi130 bash[27579]: audit 2023-08-27T23:42:36.897928+0000 mon.a (mon.0) 2520 : audit [DBG] from='client.? 172.21.15.6:0/1722107045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:37.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:36 smithi130 bash[51302]: debug 2023-08-27T23:42:36.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:36.744623+0000) 2023-08-27T23:42:37.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:36 smithi006 bash[27292]: audit 2023-08-27T23:42:36.897928+0000 mon.a (mon.0) 2520 : audit [DBG] from='client.? 172.21.15.6:0/1722107045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:37.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:36 smithi006 bash[33918]: audit 2023-08-27T23:42:36.897928+0000 mon.a (mon.0) 2520 : audit [DBG] from='client.? 172.21.15.6:0/1722107045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:38.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:37 smithi130 bash[51302]: debug 2023-08-27T23:42:37.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:37.744870+0000) 2023-08-27T23:42:38.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:37 smithi130 bash[27579]: cluster 2023-08-27T23:42:37.462185+0000 mgr.y (mgr.14901) 3122 : cluster [DBG] pgmap v3129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:38.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:37 smithi006 bash[33918]: cluster 2023-08-27T23:42:37.462185+0000 mgr.y (mgr.14901) 3122 : cluster [DBG] pgmap v3129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:38.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:37 smithi006 bash[27292]: cluster 2023-08-27T23:42:37.462185+0000 mgr.y (mgr.14901) 3122 : cluster [DBG] pgmap v3129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:38 smithi130 bash[27579]: audit 2023-08-27T23:42:38.762424+0000 mon.b (mon.1) 415 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:42:39.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:38 smithi130 bash[51302]: debug 2023-08-27T23:42:38.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:38.745178+0000) 2023-08-27T23:42:39.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:38 smithi006 bash[27292]: audit 2023-08-27T23:42:38.762424+0000 mon.b (mon.1) 415 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:42:39.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:38 smithi006 bash[33918]: audit 2023-08-27T23:42:38.762424+0000 mon.b (mon.1) 415 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:42:40.039 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:39 smithi130 bash[51302]: debug 2023-08-27T23:42:39.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:39.745488+0000) 2023-08-27T23:42:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:40 smithi130 bash[27579]: audit 2023-08-27T23:42:39.035214+0000 mon.a (mon.0) 2521 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:42:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:40 smithi130 bash[27579]: audit 2023-08-27T23:42:39.043019+0000 mon.a (mon.0) 2522 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:42:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:40 smithi130 bash[27579]: audit 2023-08-27T23:42:39.312518+0000 mon.a (mon.0) 2523 : audit [DBG] from='client.? 172.21.15.6:0/1578633366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:40 smithi130 bash[27579]: cluster 2023-08-27T23:42:39.463274+0000 mgr.y (mgr.14901) 3123 : cluster [DBG] pgmap v3130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:40 smithi006 bash[33918]: audit 2023-08-27T23:42:39.035214+0000 mon.a (mon.0) 2521 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:42:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:40 smithi006 bash[33918]: audit 2023-08-27T23:42:39.043019+0000 mon.a (mon.0) 2522 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:42:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:40 smithi006 bash[33918]: audit 2023-08-27T23:42:39.312518+0000 mon.a (mon.0) 2523 : audit [DBG] from='client.? 172.21.15.6:0/1578633366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:40 smithi006 bash[33918]: cluster 2023-08-27T23:42:39.463274+0000 mgr.y (mgr.14901) 3123 : cluster [DBG] pgmap v3130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:40.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:40 smithi006 bash[27292]: audit 2023-08-27T23:42:39.035214+0000 mon.a (mon.0) 2521 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:42:40.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:40 smithi006 bash[27292]: audit 2023-08-27T23:42:39.043019+0000 mon.a (mon.0) 2522 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:42:40.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:40 smithi006 bash[27292]: audit 2023-08-27T23:42:39.312518+0000 mon.a (mon.0) 2523 : audit [DBG] from='client.? 172.21.15.6:0/1578633366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:40.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:40 smithi006 bash[27292]: cluster 2023-08-27T23:42:39.463274+0000 mgr.y (mgr.14901) 3123 : cluster [DBG] pgmap v3130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:40.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:42:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:42:40] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-27T23:42:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:42:40.538Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:42:40.540Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:41.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:40 smithi130 bash[51302]: debug 2023-08-27T23:42:40.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:40.745772+0000) 2023-08-27T23:42:42.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:41 smithi130 bash[51302]: debug 2023-08-27T23:42:41.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:41.746056+0000) 2023-08-27T23:42:42.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:42:42.106Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:42.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:42:42.107Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:42 smithi130 bash[27579]: cluster 2023-08-27T23:42:41.463763+0000 mgr.y (mgr.14901) 3124 : cluster [DBG] pgmap v3131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:42 smithi130 bash[27579]: audit 2023-08-27T23:42:41.717403+0000 mon.c (mon.2) 1201 : audit [DBG] from='client.? 172.21.15.6:0/3519017095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:42 smithi130 bash[51302]: debug 2023-08-27T23:42:42.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:42.746338+0000) 2023-08-27T23:42:42.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:42 smithi006 bash[27292]: cluster 2023-08-27T23:42:41.463763+0000 mgr.y (mgr.14901) 3124 : cluster [DBG] pgmap v3131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:42.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:42 smithi006 bash[27292]: audit 2023-08-27T23:42:41.717403+0000 mon.c (mon.2) 1201 : audit [DBG] from='client.? 172.21.15.6:0/3519017095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:42.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:42 smithi006 bash[33918]: cluster 2023-08-27T23:42:41.463763+0000 mgr.y (mgr.14901) 3124 : cluster [DBG] pgmap v3131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:42.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:42 smithi006 bash[33918]: audit 2023-08-27T23:42:41.717403+0000 mon.c (mon.2) 1201 : audit [DBG] from='client.? 172.21.15.6:0/3519017095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:44.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:43 smithi130 bash[51302]: debug 2023-08-27T23:42:43.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:43.746631+0000) 2023-08-27T23:42:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:44 smithi006 bash[56298]: level=error ts=2023-08-27T23:42:44.004Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:44 smithi006 bash[56298]: level=warn ts=2023-08-27T23:42:44.005Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:44 smithi130 bash[27579]: cluster 2023-08-27T23:42:43.464423+0000 mgr.y (mgr.14901) 3125 : cluster [DBG] pgmap v3132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:44 smithi130 bash[27579]: audit 2023-08-27T23:42:44.145348+0000 mon.c (mon.2) 1202 : audit [DBG] from='client.? 172.21.15.6:0/3677725579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:44 smithi130 bash[51302]: debug 2023-08-27T23:42:44.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:44.746887+0000) 2023-08-27T23:42:44.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:44 smithi006 bash[27292]: cluster 2023-08-27T23:42:43.464423+0000 mgr.y (mgr.14901) 3125 : cluster [DBG] pgmap v3132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:44.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:44 smithi006 bash[27292]: audit 2023-08-27T23:42:44.145348+0000 mon.c (mon.2) 1202 : audit [DBG] from='client.? 172.21.15.6:0/3677725579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:44.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:44 smithi006 bash[33918]: cluster 2023-08-27T23:42:43.464423+0000 mgr.y (mgr.14901) 3125 : cluster [DBG] pgmap v3132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:44.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:44 smithi006 bash[33918]: audit 2023-08-27T23:42:44.145348+0000 mon.c (mon.2) 1202 : audit [DBG] from='client.? 172.21.15.6:0/3677725579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:46.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:45 smithi130 bash[51302]: debug 2023-08-27T23:42:45.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:45.747130+0000) 2023-08-27T23:42:46.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:45 smithi130 bash[27579]: audit 2023-08-27T23:42:44.935917+0000 mon.a (mon.0) 2524 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:42:46.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:45 smithi130 bash[27579]: audit 2023-08-27T23:42:45.371215+0000 mon.a (mon.0) 2525 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:42:46.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:45 smithi130 bash[27579]: audit 2023-08-27T23:42:45.641050+0000 mon.b (mon.1) 416 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:42:46.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:45 smithi130 bash[27579]: audit 2023-08-27T23:42:45.643181+0000 mon.b (mon.1) 417 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:42:46.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:45 smithi130 bash[27579]: audit 2023-08-27T23:42:45.650209+0000 mon.a (mon.0) 2526 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:42:46.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:45 smithi006 bash[27292]: audit 2023-08-27T23:42:44.935917+0000 mon.a (mon.0) 2524 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:42:46.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:45 smithi006 bash[27292]: audit 2023-08-27T23:42:45.371215+0000 mon.a (mon.0) 2525 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:42:46.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:45 smithi006 bash[27292]: audit 2023-08-27T23:42:45.641050+0000 mon.b (mon.1) 416 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:42:46.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:45 smithi006 bash[27292]: audit 2023-08-27T23:42:45.643181+0000 mon.b (mon.1) 417 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:42:46.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:45 smithi006 bash[27292]: audit 2023-08-27T23:42:45.650209+0000 mon.a (mon.0) 2526 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:42:46.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:45 smithi006 bash[33918]: audit 2023-08-27T23:42:44.935917+0000 mon.a (mon.0) 2524 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:42:46.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:45 smithi006 bash[33918]: audit 2023-08-27T23:42:45.371215+0000 mon.a (mon.0) 2525 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:42:46.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:45 smithi006 bash[33918]: audit 2023-08-27T23:42:45.641050+0000 mon.b (mon.1) 416 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:42:46.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:45 smithi006 bash[33918]: audit 2023-08-27T23:42:45.643181+0000 mon.b (mon.1) 417 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:42:46.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:45 smithi006 bash[33918]: audit 2023-08-27T23:42:45.650209+0000 mon.a (mon.0) 2526 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:42:47.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:46 smithi130 bash[27579]: cluster 2023-08-27T23:42:45.465552+0000 mgr.y (mgr.14901) 3126 : cluster [DBG] pgmap v3133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:47.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:46 smithi130 bash[27579]: audit 2023-08-27T23:42:46.561895+0000 mon.a (mon.0) 2527 : audit [DBG] from='client.? 172.21.15.6:0/856674798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:47.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:46 smithi130 bash[51302]: debug 2023-08-27T23:42:46.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:46.747428+0000) 2023-08-27T23:42:47.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:46 smithi006 bash[27292]: cluster 2023-08-27T23:42:45.465552+0000 mgr.y (mgr.14901) 3126 : cluster [DBG] pgmap v3133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:47.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:46 smithi006 bash[27292]: audit 2023-08-27T23:42:46.561895+0000 mon.a (mon.0) 2527 : audit [DBG] from='client.? 172.21.15.6:0/856674798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:47.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:46 smithi006 bash[33918]: cluster 2023-08-27T23:42:45.465552+0000 mgr.y (mgr.14901) 3126 : cluster [DBG] pgmap v3133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:47.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:46 smithi006 bash[33918]: audit 2023-08-27T23:42:46.561895+0000 mon.a (mon.0) 2527 : audit [DBG] from='client.? 172.21.15.6:0/856674798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:48.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:47 smithi130 bash[51302]: debug 2023-08-27T23:42:47.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:47.747741+0000) 2023-08-27T23:42:49.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:48 smithi130 bash[27579]: cluster 2023-08-27T23:42:47.466205+0000 mgr.y (mgr.14901) 3127 : cluster [DBG] pgmap v3134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:49.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:48 smithi130 bash[51302]: debug 2023-08-27T23:42:48.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:48.748028+0000) 2023-08-27T23:42:49.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:48 smithi006 bash[27292]: cluster 2023-08-27T23:42:47.466205+0000 mgr.y (mgr.14901) 3127 : cluster [DBG] pgmap v3134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:49.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:48 smithi006 bash[33918]: cluster 2023-08-27T23:42:47.466205+0000 mgr.y (mgr.14901) 3127 : cluster [DBG] pgmap v3134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:50.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:49 smithi130 bash[27579]: audit 2023-08-27T23:42:48.976626+0000 mon.a (mon.0) 2528 : audit [DBG] from='client.? 172.21.15.6:0/3145847440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:50.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:49 smithi130 bash[51302]: debug 2023-08-27T23:42:49.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:49.748354+0000) 2023-08-27T23:42:50.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:49 smithi006 bash[33918]: audit 2023-08-27T23:42:48.976626+0000 mon.a (mon.0) 2528 : audit [DBG] from='client.? 172.21.15.6:0/3145847440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:50.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:49 smithi006 bash[27292]: audit 2023-08-27T23:42:48.976626+0000 mon.a (mon.0) 2528 : audit [DBG] from='client.? 172.21.15.6:0/3145847440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:50.957 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:42:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:42:50] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-27T23:42:50.958 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:42:50.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:50.958 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:42:50.541Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:50 smithi130 bash[27579]: cluster 2023-08-27T23:42:49.467355+0000 mgr.y (mgr.14901) 3128 : cluster [DBG] pgmap v3135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:51.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:50 smithi130 bash[51302]: debug 2023-08-27T23:42:50.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:50.748632+0000) 2023-08-27T23:42:51.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:50 smithi006 bash[33918]: cluster 2023-08-27T23:42:49.467355+0000 mgr.y (mgr.14901) 3128 : cluster [DBG] pgmap v3135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:50 smithi006 bash[27292]: cluster 2023-08-27T23:42:49.467355+0000 mgr.y (mgr.14901) 3128 : cluster [DBG] pgmap v3135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:52.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:51 smithi130 bash[51302]: debug 2023-08-27T23:42:51.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:51.748844+0000) 2023-08-27T23:42:52.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:51 smithi130 bash[27579]: audit 2023-08-27T23:42:51.402681+0000 mon.c (mon.2) 1203 : audit [DBG] from='client.? 172.21.15.6:0/2309537986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:52.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:51 smithi130 bash[27579]: cluster 2023-08-27T23:42:51.468048+0000 mgr.y (mgr.14901) 3129 : cluster [DBG] pgmap v3136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:52.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:51 smithi006 bash[27292]: audit 2023-08-27T23:42:51.402681+0000 mon.c (mon.2) 1203 : audit [DBG] from='client.? 172.21.15.6:0/2309537986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:52.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:51 smithi006 bash[27292]: cluster 2023-08-27T23:42:51.468048+0000 mgr.y (mgr.14901) 3129 : cluster [DBG] pgmap v3136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:52.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:51 smithi006 bash[33918]: audit 2023-08-27T23:42:51.402681+0000 mon.c (mon.2) 1203 : audit [DBG] from='client.? 172.21.15.6:0/2309537986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:52.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:51 smithi006 bash[33918]: cluster 2023-08-27T23:42:51.468048+0000 mgr.y (mgr.14901) 3129 : cluster [DBG] pgmap v3136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:52.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:42:52.107Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:52.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:42:52.109Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:53.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:52 smithi130 bash[51302]: debug 2023-08-27T23:42:52.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:52.749138+0000) 2023-08-27T23:42:54.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:53 smithi130 bash[51302]: debug 2023-08-27T23:42:53.749+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:53.749418+0000) 2023-08-27T23:42:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:54 smithi006 bash[56298]: level=error ts=2023-08-27T23:42:54.004Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:42:54 smithi006 bash[56298]: level=warn ts=2023-08-27T23:42:54.006Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:42:54.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:54 smithi130 bash[51302]: debug 2023-08-27T23:42:54.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:54.749686+0000) 2023-08-27T23:42:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:54 smithi130 bash[27579]: cluster 2023-08-27T23:42:53.468791+0000 mgr.y (mgr.14901) 3130 : cluster [DBG] pgmap v3137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:54 smithi130 bash[27579]: audit 2023-08-27T23:42:53.819392+0000 mon.a (mon.0) 2529 : audit [DBG] from='client.? 172.21.15.6:0/2037629806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:54.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:54 smithi006 bash[27292]: cluster 2023-08-27T23:42:53.468791+0000 mgr.y (mgr.14901) 3130 : cluster [DBG] pgmap v3137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:54.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:54 smithi006 bash[27292]: audit 2023-08-27T23:42:53.819392+0000 mon.a (mon.0) 2529 : audit [DBG] from='client.? 172.21.15.6:0/2037629806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:54.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:54 smithi006 bash[33918]: cluster 2023-08-27T23:42:53.468791+0000 mgr.y (mgr.14901) 3130 : cluster [DBG] pgmap v3137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:54.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:54 smithi006 bash[33918]: audit 2023-08-27T23:42:53.819392+0000 mon.a (mon.0) 2529 : audit [DBG] from='client.? 172.21.15.6:0/2037629806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:56.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:55 smithi130 bash[27579]: cluster 2023-08-27T23:42:55.469926+0000 mgr.y (mgr.14901) 3131 : cluster [DBG] pgmap v3138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:56.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:55 smithi130 bash[51302]: debug 2023-08-27T23:42:55.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:55.749969+0000) 2023-08-27T23:42:56.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:55 smithi006 bash[27292]: cluster 2023-08-27T23:42:55.469926+0000 mgr.y (mgr.14901) 3131 : cluster [DBG] pgmap v3138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:56.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:55 smithi006 bash[33918]: cluster 2023-08-27T23:42:55.469926+0000 mgr.y (mgr.14901) 3131 : cluster [DBG] pgmap v3138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:56 smithi130 bash[27579]: audit 2023-08-27T23:42:56.231874+0000 mon.a (mon.0) 2530 : audit [DBG] from='client.? 172.21.15.6:0/3894340008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:57.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:56 smithi130 bash[51302]: debug 2023-08-27T23:42:56.749+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:56.750229+0000) 2023-08-27T23:42:57.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:56 smithi006 bash[33918]: audit 2023-08-27T23:42:56.231874+0000 mon.a (mon.0) 2530 : audit [DBG] from='client.? 172.21.15.6:0/3894340008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:57.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:56 smithi006 bash[27292]: audit 2023-08-27T23:42:56.231874+0000 mon.a (mon.0) 2530 : audit [DBG] from='client.? 172.21.15.6:0/3894340008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:58.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:57 smithi130 bash[27579]: cluster 2023-08-27T23:42:57.470568+0000 mgr.y (mgr.14901) 3132 : cluster [DBG] pgmap v3139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:58.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:57 smithi130 bash[51302]: debug 2023-08-27T23:42:57.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:57.750542+0000) 2023-08-27T23:42:58.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:57 smithi006 bash[27292]: cluster 2023-08-27T23:42:57.470568+0000 mgr.y (mgr.14901) 3132 : cluster [DBG] pgmap v3139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:58.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:57 smithi006 bash[33918]: cluster 2023-08-27T23:42:57.470568+0000 mgr.y (mgr.14901) 3132 : cluster [DBG] pgmap v3139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:42:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:42:59 smithi130 bash[27579]: audit 2023-08-27T23:42:58.645288+0000 mon.a (mon.0) 2531 : audit [DBG] from='client.? 172.21.15.6:0/1925372306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:59.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:58 smithi130 bash[51302]: debug 2023-08-27T23:42:58.749+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:58.750830+0000) 2023-08-27T23:42:59.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:42:58 smithi006 bash[27292]: audit 2023-08-27T23:42:58.645288+0000 mon.a (mon.0) 2531 : audit [DBG] from='client.? 172.21.15.6:0/1925372306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:59.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:42:58 smithi006 bash[33918]: audit 2023-08-27T23:42:58.645288+0000 mon.a (mon.0) 2531 : audit [DBG] from='client.? 172.21.15.6:0/1925372306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:00.008 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:42:59 smithi130 bash[51302]: debug 2023-08-27T23:42:59.749+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:59.751129+0000) 2023-08-27T23:43:00.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:00 smithi130 bash[27579]: cluster 2023-08-27T23:42:59.471686+0000 mgr.y (mgr.14901) 3133 : cluster [DBG] pgmap v3140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:00.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:00 smithi006 bash[27292]: cluster 2023-08-27T23:42:59.471686+0000 mgr.y (mgr.14901) 3133 : cluster [DBG] pgmap v3140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:00.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:00 smithi006 bash[33918]: cluster 2023-08-27T23:42:59.471686+0000 mgr.y (mgr.14901) 3133 : cluster [DBG] pgmap v3140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:00.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:43:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:43:00] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-27T23:43:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:43:00.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:00.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:43:00.541Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:01.114 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:00 smithi130 bash[51302]: debug 2023-08-27T23:43:00.749+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:00.751406+0000) 2023-08-27T23:43:01.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:01 smithi130 bash[27579]: audit 2023-08-27T23:43:01.056681+0000 mon.c (mon.2) 1204 : audit [DBG] from='client.? 172.21.15.6:0/2062342222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:01.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:01 smithi006 bash[27292]: audit 2023-08-27T23:43:01.056681+0000 mon.c (mon.2) 1204 : audit [DBG] from='client.? 172.21.15.6:0/2062342222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:01.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:01 smithi006 bash[33918]: audit 2023-08-27T23:43:01.056681+0000 mon.c (mon.2) 1204 : audit [DBG] from='client.? 172.21.15.6:0/2062342222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:02.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:01 smithi130 bash[51302]: debug 2023-08-27T23:43:01.749+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:01.751699+0000) 2023-08-27T23:43:02.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:02 smithi006 bash[27292]: cluster 2023-08-27T23:43:01.472375+0000 mgr.y (mgr.14901) 3134 : cluster [DBG] pgmap v3141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:02.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:02 smithi006 bash[33918]: cluster 2023-08-27T23:43:01.472375+0000 mgr.y (mgr.14901) 3134 : cluster [DBG] pgmap v3141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:43:02.107Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:43:02.109Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:02.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:02 smithi130 bash[27579]: cluster 2023-08-27T23:43:01.472375+0000 mgr.y (mgr.14901) 3134 : cluster [DBG] pgmap v3141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:03.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:02 smithi130 bash[51302]: debug 2023-08-27T23:43:02.749+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:02.752043+0000) 2023-08-27T23:43:03.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:03 smithi130 bash[27579]: audit 2023-08-27T23:43:03.472917+0000 mon.a (mon.0) 2532 : audit [DBG] from='client.? 172.21.15.6:0/482462758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:03.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:03 smithi130 bash[51302]: debug 2023-08-27T23:43:03.749+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:03.752306+0000) 2023-08-27T23:43:03.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:03 smithi006 bash[27292]: audit 2023-08-27T23:43:03.472917+0000 mon.a (mon.0) 2532 : audit [DBG] from='client.? 172.21.15.6:0/482462758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:03.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:03 smithi006 bash[33918]: audit 2023-08-27T23:43:03.472917+0000 mon.a (mon.0) 2532 : audit [DBG] from='client.? 172.21.15.6:0/482462758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:04 smithi006 bash[56298]: level=error ts=2023-08-27T23:43:04.005Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:04 smithi006 bash[56298]: level=warn ts=2023-08-27T23:43:04.007Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:04 smithi130 bash[27579]: cluster 2023-08-27T23:43:03.472751+0000 mgr.y (mgr.14901) 3135 : cluster [DBG] pgmap v3142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:04.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:04 smithi130 bash[51302]: debug 2023-08-27T23:43:04.748+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:04.752610+0000) 2023-08-27T23:43:04.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:04 smithi006 bash[33918]: cluster 2023-08-27T23:43:03.472751+0000 mgr.y (mgr.14901) 3135 : cluster [DBG] pgmap v3142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:04.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:04 smithi006 bash[27292]: cluster 2023-08-27T23:43:03.472751+0000 mgr.y (mgr.14901) 3135 : cluster [DBG] pgmap v3142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:06.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:05 smithi130 bash[51302]: debug 2023-08-27T23:43:05.748+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:05.752887+0000) 2023-08-27T23:43:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:05 smithi130 bash[27579]: cluster 2023-08-27T23:43:05.473827+0000 mgr.y (mgr.14901) 3136 : cluster [DBG] pgmap v3143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:05 smithi130 bash[27579]: audit 2023-08-27T23:43:05.894178+0000 mon.a (mon.0) 2533 : audit [DBG] from='client.? 172.21.15.6:0/1344868878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:06.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:05 smithi006 bash[27292]: cluster 2023-08-27T23:43:05.473827+0000 mgr.y (mgr.14901) 3136 : cluster [DBG] pgmap v3143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:06.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:05 smithi006 bash[27292]: audit 2023-08-27T23:43:05.894178+0000 mon.a (mon.0) 2533 : audit [DBG] from='client.? 172.21.15.6:0/1344868878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:05 smithi006 bash[33918]: cluster 2023-08-27T23:43:05.473827+0000 mgr.y (mgr.14901) 3136 : cluster [DBG] pgmap v3143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:05 smithi006 bash[33918]: audit 2023-08-27T23:43:05.894178+0000 mon.a (mon.0) 2533 : audit [DBG] from='client.? 172.21.15.6:0/1344868878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:07.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:06 smithi130 bash[51302]: debug 2023-08-27T23:43:06.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:06.753211+0000) 2023-08-27T23:43:08.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:07 smithi130 bash[51302]: debug 2023-08-27T23:43:07.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:07.753479+0000) 2023-08-27T23:43:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:08 smithi130 bash[27579]: cluster 2023-08-27T23:43:07.474613+0000 mgr.y (mgr.14901) 3137 : cluster [DBG] pgmap v3144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:08 smithi130 bash[27579]: audit 2023-08-27T23:43:08.301170+0000 mon.a (mon.0) 2534 : audit [DBG] from='client.? 172.21.15.6:0/2531733210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:08.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:08 smithi130 bash[51302]: debug 2023-08-27T23:43:08.748+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:08.753775+0000) 2023-08-27T23:43:08.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:08 smithi006 bash[33918]: cluster 2023-08-27T23:43:07.474613+0000 mgr.y (mgr.14901) 3137 : cluster [DBG] pgmap v3144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:08.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:08 smithi006 bash[33918]: audit 2023-08-27T23:43:08.301170+0000 mon.a (mon.0) 2534 : audit [DBG] from='client.? 172.21.15.6:0/2531733210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:08.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:08 smithi006 bash[27292]: cluster 2023-08-27T23:43:07.474613+0000 mgr.y (mgr.14901) 3137 : cluster [DBG] pgmap v3144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:08.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:08 smithi006 bash[27292]: audit 2023-08-27T23:43:08.301170+0000 mon.a (mon.0) 2534 : audit [DBG] from='client.? 172.21.15.6:0/2531733210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:10.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:09 smithi130 bash[51302]: debug 2023-08-27T23:43:09.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:09.754077+0000) 2023-08-27T23:43:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:10 smithi130 bash[27579]: cluster 2023-08-27T23:43:09.475743+0000 mgr.y (mgr.14901) 3138 : cluster [DBG] pgmap v3145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:10 smithi130 bash[51302]: debug 2023-08-27T23:43:10.748+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:10.754324+0000) 2023-08-27T23:43:10.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:10 smithi006 bash[27292]: cluster 2023-08-27T23:43:09.475743+0000 mgr.y (mgr.14901) 3138 : cluster [DBG] pgmap v3145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:10.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:43:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:43:10] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-27T23:43:10.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:10 smithi006 bash[33918]: cluster 2023-08-27T23:43:09.475743+0000 mgr.y (mgr.14901) 3138 : cluster [DBG] pgmap v3145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:10.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:43:10.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:10.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:43:10.541Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:11.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:11 smithi130 bash[27579]: audit 2023-08-27T23:43:10.731568+0000 mon.a (mon.0) 2535 : audit [DBG] from='client.? 172.21.15.6:0/1555845875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:11.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:11 smithi130 bash[51302]: debug 2023-08-27T23:43:11.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:11.754589+0000) 2023-08-27T23:43:11.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:11 smithi006 bash[27292]: audit 2023-08-27T23:43:10.731568+0000 mon.a (mon.0) 2535 : audit [DBG] from='client.? 172.21.15.6:0/1555845875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:11.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:11 smithi006 bash[33918]: audit 2023-08-27T23:43:10.731568+0000 mon.a (mon.0) 2535 : audit [DBG] from='client.? 172.21.15.6:0/1555845875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:43:12.108Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:12.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:43:12.110Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:12 smithi130 bash[27579]: cluster 2023-08-27T23:43:11.476370+0000 mgr.y (mgr.14901) 3139 : cluster [DBG] pgmap v3146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:12 smithi130 bash[51302]: debug 2023-08-27T23:43:12.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:12.754906+0000) 2023-08-27T23:43:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:12 smithi006 bash[27292]: cluster 2023-08-27T23:43:11.476370+0000 mgr.y (mgr.14901) 3139 : cluster [DBG] pgmap v3146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:12.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:12 smithi006 bash[33918]: cluster 2023-08-27T23:43:11.476370+0000 mgr.y (mgr.14901) 3139 : cluster [DBG] pgmap v3146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:13.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:13 smithi130 bash[27579]: audit 2023-08-27T23:43:13.147676+0000 mon.a (mon.0) 2536 : audit [DBG] from='client.? 172.21.15.6:0/3842596876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:13.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:13 smithi130 bash[51302]: debug 2023-08-27T23:43:13.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:13.755223+0000) 2023-08-27T23:43:13.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:13 smithi006 bash[27292]: audit 2023-08-27T23:43:13.147676+0000 mon.a (mon.0) 2536 : audit [DBG] from='client.? 172.21.15.6:0/3842596876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:13.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:13 smithi006 bash[33918]: audit 2023-08-27T23:43:13.147676+0000 mon.a (mon.0) 2536 : audit [DBG] from='client.? 172.21.15.6:0/3842596876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:14 smithi006 bash[56298]: level=error ts=2023-08-27T23:43:14.006Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:14 smithi006 bash[56298]: level=warn ts=2023-08-27T23:43:14.009Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:14 smithi130 bash[27579]: cluster 2023-08-27T23:43:13.477155+0000 mgr.y (mgr.14901) 3140 : cluster [DBG] pgmap v3147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:14 smithi130 bash[51302]: debug 2023-08-27T23:43:14.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:14.755496+0000) 2023-08-27T23:43:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:14 smithi006 bash[27292]: cluster 2023-08-27T23:43:13.477155+0000 mgr.y (mgr.14901) 3140 : cluster [DBG] pgmap v3147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:14.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:14 smithi006 bash[33918]: cluster 2023-08-27T23:43:13.477155+0000 mgr.y (mgr.14901) 3140 : cluster [DBG] pgmap v3147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:16.014 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:15 smithi130 bash[51302]: debug 2023-08-27T23:43:15.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:15.755799+0000) 2023-08-27T23:43:16.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:16 smithi130 bash[27579]: cluster 2023-08-27T23:43:15.478289+0000 mgr.y (mgr.14901) 3141 : cluster [DBG] pgmap v3148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:16.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:16 smithi130 bash[27579]: audit 2023-08-27T23:43:15.567127+0000 mon.a (mon.0) 2537 : audit [DBG] from='client.? 172.21.15.6:0/996564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:16.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:16 smithi006 bash[27292]: cluster 2023-08-27T23:43:15.478289+0000 mgr.y (mgr.14901) 3141 : cluster [DBG] pgmap v3148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:16.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:16 smithi006 bash[27292]: audit 2023-08-27T23:43:15.567127+0000 mon.a (mon.0) 2537 : audit [DBG] from='client.? 172.21.15.6:0/996564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:16.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:16 smithi006 bash[33918]: cluster 2023-08-27T23:43:15.478289+0000 mgr.y (mgr.14901) 3141 : cluster [DBG] pgmap v3148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:16.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:16 smithi006 bash[33918]: audit 2023-08-27T23:43:15.567127+0000 mon.a (mon.0) 2537 : audit [DBG] from='client.? 172.21.15.6:0/996564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:17.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:16 smithi130 bash[51302]: debug 2023-08-27T23:43:16.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:16.756057+0000) 2023-08-27T23:43:18.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:17 smithi130 bash[51302]: debug 2023-08-27T23:43:17.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:17.756325+0000) 2023-08-27T23:43:18.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:18 smithi130 bash[27579]: cluster 2023-08-27T23:43:17.479076+0000 mgr.y (mgr.14901) 3142 : cluster [DBG] pgmap v3149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:18 smithi130 bash[27579]: audit 2023-08-27T23:43:17.979844+0000 mon.c (mon.2) 1205 : audit [DBG] from='client.? 172.21.15.6:0/2375724379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:18 smithi130 bash[51302]: debug 2023-08-27T23:43:18.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:18.756578+0000) 2023-08-27T23:43:18.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:18 smithi006 bash[33918]: cluster 2023-08-27T23:43:17.479076+0000 mgr.y (mgr.14901) 3142 : cluster [DBG] pgmap v3149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:18.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:18 smithi006 bash[33918]: audit 2023-08-27T23:43:17.979844+0000 mon.c (mon.2) 1205 : audit [DBG] from='client.? 172.21.15.6:0/2375724379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:18.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:18 smithi006 bash[27292]: cluster 2023-08-27T23:43:17.479076+0000 mgr.y (mgr.14901) 3142 : cluster [DBG] pgmap v3149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:18.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:18 smithi006 bash[27292]: audit 2023-08-27T23:43:17.979844+0000 mon.c (mon.2) 1205 : audit [DBG] from='client.? 172.21.15.6:0/2375724379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:20.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:19 smithi130 bash[51302]: debug 2023-08-27T23:43:19.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:19.756877+0000) 2023-08-27T23:43:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:20 smithi130 bash[27579]: cluster 2023-08-27T23:43:19.480210+0000 mgr.y (mgr.14901) 3143 : cluster [DBG] pgmap v3150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:20 smithi130 bash[27579]: audit 2023-08-27T23:43:20.400285+0000 mon.c (mon.2) 1206 : audit [DBG] from='client.? 172.21.15.6:0/3290932558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:20.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:20 smithi130 bash[51302]: debug 2023-08-27T23:43:20.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:20.757102+0000) 2023-08-27T23:43:20.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:43:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:43:20] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-27T23:43:20.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:20 smithi006 bash[33918]: cluster 2023-08-27T23:43:19.480210+0000 mgr.y (mgr.14901) 3143 : cluster [DBG] pgmap v3150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:20.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:20 smithi006 bash[33918]: audit 2023-08-27T23:43:20.400285+0000 mon.c (mon.2) 1206 : audit [DBG] from='client.? 172.21.15.6:0/3290932558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:20.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:43:20.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:20.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:43:20.543Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:20.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:20 smithi006 bash[27292]: cluster 2023-08-27T23:43:19.480210+0000 mgr.y (mgr.14901) 3143 : cluster [DBG] pgmap v3150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:20.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:20 smithi006 bash[27292]: audit 2023-08-27T23:43:20.400285+0000 mon.c (mon.2) 1206 : audit [DBG] from='client.? 172.21.15.6:0/3290932558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:22.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:21 smithi130 bash[51302]: debug 2023-08-27T23:43:21.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:21.757399+0000) 2023-08-27T23:43:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:43:22.109Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:22.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:43:22.110Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:22 smithi130 bash[27579]: cluster 2023-08-27T23:43:21.480920+0000 mgr.y (mgr.14901) 3144 : cluster [DBG] pgmap v3151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:22 smithi130 bash[51302]: debug 2023-08-27T23:43:22.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:22.757693+0000) 2023-08-27T23:43:22.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:22 smithi006 bash[27292]: cluster 2023-08-27T23:43:21.480920+0000 mgr.y (mgr.14901) 3144 : cluster [DBG] pgmap v3151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:22.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:22 smithi006 bash[33918]: cluster 2023-08-27T23:43:21.480920+0000 mgr.y (mgr.14901) 3144 : cluster [DBG] pgmap v3151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:23.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:23 smithi130 bash[27579]: audit 2023-08-27T23:43:22.810737+0000 mon.c (mon.2) 1207 : audit [DBG] from='client.? 172.21.15.6:0/119001231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:23.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:23 smithi130 bash[51302]: debug 2023-08-27T23:43:23.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:23.757992+0000) 2023-08-27T23:43:23.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:23 smithi006 bash[27292]: audit 2023-08-27T23:43:22.810737+0000 mon.c (mon.2) 1207 : audit [DBG] from='client.? 172.21.15.6:0/119001231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:23.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:23 smithi006 bash[33918]: audit 2023-08-27T23:43:22.810737+0000 mon.c (mon.2) 1207 : audit [DBG] from='client.? 172.21.15.6:0/119001231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:24 smithi006 bash[56298]: level=error ts=2023-08-27T23:43:24.007Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:24 smithi006 bash[56298]: level=warn ts=2023-08-27T23:43:24.009Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:24 smithi130 bash[27579]: cluster 2023-08-27T23:43:23.481495+0000 mgr.y (mgr.14901) 3145 : cluster [DBG] pgmap v3152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:24 smithi130 bash[51302]: debug 2023-08-27T23:43:24.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:24.758308+0000) 2023-08-27T23:43:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:24 smithi006 bash[27292]: cluster 2023-08-27T23:43:23.481495+0000 mgr.y (mgr.14901) 3145 : cluster [DBG] pgmap v3152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:24.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:24 smithi006 bash[33918]: cluster 2023-08-27T23:43:23.481495+0000 mgr.y (mgr.14901) 3145 : cluster [DBG] pgmap v3152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:26.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:26 smithi130 bash[27579]: audit 2023-08-27T23:43:25.224329+0000 mon.a (mon.0) 2538 : audit [DBG] from='client.? 172.21.15.6:0/3886454265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:26 smithi130 bash[27579]: cluster 2023-08-27T23:43:25.482585+0000 mgr.y (mgr.14901) 3146 : cluster [DBG] pgmap v3153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:26.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:25 smithi130 bash[51302]: debug 2023-08-27T23:43:25.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:25.758545+0000) 2023-08-27T23:43:26.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:25 smithi006 bash[27292]: audit 2023-08-27T23:43:25.224329+0000 mon.a (mon.0) 2538 : audit [DBG] from='client.? 172.21.15.6:0/3886454265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:26.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:26 smithi006 bash[27292]: cluster 2023-08-27T23:43:25.482585+0000 mgr.y (mgr.14901) 3146 : cluster [DBG] pgmap v3153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:26 smithi006 bash[33918]: audit 2023-08-27T23:43:25.224329+0000 mon.a (mon.0) 2538 : audit [DBG] from='client.? 172.21.15.6:0/3886454265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:26 smithi006 bash[33918]: cluster 2023-08-27T23:43:25.482585+0000 mgr.y (mgr.14901) 3146 : cluster [DBG] pgmap v3153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:27.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:26 smithi130 bash[51302]: debug 2023-08-27T23:43:26.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:26.758820+0000) 2023-08-27T23:43:28.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:27 smithi130 bash[51302]: debug 2023-08-27T23:43:27.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:27.759096+0000) 2023-08-27T23:43:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:28 smithi130 bash[27579]: cluster 2023-08-27T23:43:27.483016+0000 mgr.y (mgr.14901) 3147 : cluster [DBG] pgmap v3154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:28 smithi130 bash[27579]: audit 2023-08-27T23:43:27.642992+0000 mon.a (mon.0) 2539 : audit [DBG] from='client.? 172.21.15.6:0/4240775612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:28 smithi130 bash[51302]: debug 2023-08-27T23:43:28.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:28.759377+0000) 2023-08-27T23:43:28.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:28 smithi006 bash[33918]: cluster 2023-08-27T23:43:27.483016+0000 mgr.y (mgr.14901) 3147 : cluster [DBG] pgmap v3154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:28.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:28 smithi006 bash[33918]: audit 2023-08-27T23:43:27.642992+0000 mon.a (mon.0) 2539 : audit [DBG] from='client.? 172.21.15.6:0/4240775612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:28.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:28 smithi006 bash[27292]: cluster 2023-08-27T23:43:27.483016+0000 mgr.y (mgr.14901) 3147 : cluster [DBG] pgmap v3154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:28.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:28 smithi006 bash[27292]: audit 2023-08-27T23:43:27.642992+0000 mon.a (mon.0) 2539 : audit [DBG] from='client.? 172.21.15.6:0/4240775612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:30.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:29 smithi130 bash[51302]: debug 2023-08-27T23:43:29.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:29.759655+0000) 2023-08-27T23:43:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:30 smithi130 bash[51302]: debug 2023-08-27T23:43:30.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:30.759892+0000) 2023-08-27T23:43:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:30 smithi130 bash[27579]: cluster 2023-08-27T23:43:29.484097+0000 mgr.y (mgr.14901) 3148 : cluster [DBG] pgmap v3155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:30 smithi130 bash[27579]: audit 2023-08-27T23:43:30.062493+0000 mon.c (mon.2) 1208 : audit [DBG] from='client.? 172.21.15.6:0/2817079177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:43:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:43:30] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:43:30.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:30 smithi006 bash[27292]: cluster 2023-08-27T23:43:29.484097+0000 mgr.y (mgr.14901) 3148 : cluster [DBG] pgmap v3155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:30.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:30 smithi006 bash[27292]: audit 2023-08-27T23:43:30.062493+0000 mon.c (mon.2) 1208 : audit [DBG] from='client.? 172.21.15.6:0/2817079177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:30.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:30 smithi006 bash[33918]: cluster 2023-08-27T23:43:29.484097+0000 mgr.y (mgr.14901) 3148 : cluster [DBG] pgmap v3155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:30.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:30 smithi006 bash[33918]: audit 2023-08-27T23:43:30.062493+0000 mon.c (mon.2) 1208 : audit [DBG] from='client.? 172.21.15.6:0/2817079177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:30.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:43:30.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:30.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:43:30.543Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:32.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:31 smithi130 bash[51302]: debug 2023-08-27T23:43:31.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:31.760146+0000) 2023-08-27T23:43:32.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:43:32.109Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:32.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:43:32.111Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:32 smithi130 bash[51302]: debug 2023-08-27T23:43:32.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:32.760422+0000) 2023-08-27T23:43:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:32 smithi130 bash[27579]: cluster 2023-08-27T23:43:31.484792+0000 mgr.y (mgr.14901) 3149 : cluster [DBG] pgmap v3156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:32 smithi130 bash[27579]: audit 2023-08-27T23:43:32.472779+0000 mon.c (mon.2) 1209 : audit [DBG] from='client.? 172.21.15.6:0/2449821263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:32.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:32 smithi006 bash[27292]: cluster 2023-08-27T23:43:31.484792+0000 mgr.y (mgr.14901) 3149 : cluster [DBG] pgmap v3156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:32.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:32 smithi006 bash[27292]: audit 2023-08-27T23:43:32.472779+0000 mon.c (mon.2) 1209 : audit [DBG] from='client.? 172.21.15.6:0/2449821263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:32.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:32 smithi006 bash[33918]: cluster 2023-08-27T23:43:31.484792+0000 mgr.y (mgr.14901) 3149 : cluster [DBG] pgmap v3156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:32.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:32 smithi006 bash[33918]: audit 2023-08-27T23:43:32.472779+0000 mon.c (mon.2) 1209 : audit [DBG] from='client.? 172.21.15.6:0/2449821263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:34.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:33 smithi130 bash[51302]: debug 2023-08-27T23:43:33.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:33.760697+0000) 2023-08-27T23:43:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:34 smithi006 bash[56298]: level=error ts=2023-08-27T23:43:34.008Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:34 smithi006 bash[56298]: level=warn ts=2023-08-27T23:43:34.010Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:34.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:34 smithi130 bash[27579]: cluster 2023-08-27T23:43:33.485408+0000 mgr.y (mgr.14901) 3150 : cluster [DBG] pgmap v3157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:34 smithi130 bash[51302]: debug 2023-08-27T23:43:34.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:34.761010+0000) 2023-08-27T23:43:34.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:34 smithi006 bash[27292]: cluster 2023-08-27T23:43:33.485408+0000 mgr.y (mgr.14901) 3150 : cluster [DBG] pgmap v3157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:34.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:34 smithi006 bash[33918]: cluster 2023-08-27T23:43:33.485408+0000 mgr.y (mgr.14901) 3150 : cluster [DBG] pgmap v3157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:35.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:35 smithi130 bash[27579]: audit 2023-08-27T23:43:34.890803+0000 mon.a (mon.0) 2540 : audit [DBG] from='client.? 172.21.15.6:0/162646952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:35.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:35 smithi130 bash[51302]: debug 2023-08-27T23:43:35.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:35.761310+0000) 2023-08-27T23:43:35.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:35 smithi006 bash[33918]: audit 2023-08-27T23:43:34.890803+0000 mon.a (mon.0) 2540 : audit [DBG] from='client.? 172.21.15.6:0/162646952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:35.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:35 smithi006 bash[27292]: audit 2023-08-27T23:43:34.890803+0000 mon.a (mon.0) 2540 : audit [DBG] from='client.? 172.21.15.6:0/162646952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:36.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:36 smithi130 bash[27579]: cluster 2023-08-27T23:43:35.486538+0000 mgr.y (mgr.14901) 3151 : cluster [DBG] pgmap v3158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:36 smithi130 bash[51302]: debug 2023-08-27T23:43:36.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:36.761509+0000) 2023-08-27T23:43:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:36 smithi006 bash[33918]: cluster 2023-08-27T23:43:35.486538+0000 mgr.y (mgr.14901) 3151 : cluster [DBG] pgmap v3158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:36.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:36 smithi006 bash[27292]: cluster 2023-08-27T23:43:35.486538+0000 mgr.y (mgr.14901) 3151 : cluster [DBG] pgmap v3158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:37.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:37 smithi130 bash[27579]: audit 2023-08-27T23:43:37.295671+0000 mon.a (mon.0) 2541 : audit [DBG] from='client.? 172.21.15.6:0/662987577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:37 smithi130 bash[51302]: debug 2023-08-27T23:43:37.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:37.761778+0000) 2023-08-27T23:43:37.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:37 smithi006 bash[33918]: audit 2023-08-27T23:43:37.295671+0000 mon.a (mon.0) 2541 : audit [DBG] from='client.? 172.21.15.6:0/662987577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:37.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:37 smithi006 bash[27292]: audit 2023-08-27T23:43:37.295671+0000 mon.a (mon.0) 2541 : audit [DBG] from='client.? 172.21.15.6:0/662987577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:38.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:38 smithi130 bash[27579]: cluster 2023-08-27T23:43:37.487296+0000 mgr.y (mgr.14901) 3152 : cluster [DBG] pgmap v3159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:38 smithi130 bash[51302]: debug 2023-08-27T23:43:38.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:38.762113+0000) 2023-08-27T23:43:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:38 smithi006 bash[33918]: cluster 2023-08-27T23:43:37.487296+0000 mgr.y (mgr.14901) 3152 : cluster [DBG] pgmap v3159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:38.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:38 smithi006 bash[27292]: cluster 2023-08-27T23:43:37.487296+0000 mgr.y (mgr.14901) 3152 : cluster [DBG] pgmap v3159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:40.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:39 smithi130 bash[51302]: debug 2023-08-27T23:43:39.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:39.762423+0000) 2023-08-27T23:43:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:40 smithi130 bash[27579]: cluster 2023-08-27T23:43:39.488365+0000 mgr.y (mgr.14901) 3153 : cluster [DBG] pgmap v3160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:40 smithi130 bash[27579]: audit 2023-08-27T23:43:39.704158+0000 mon.c (mon.2) 1210 : audit [DBG] from='client.? 172.21.15.6:0/263503313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:40 smithi130 bash[51302]: debug 2023-08-27T23:43:40.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:40.762843+0000) 2023-08-27T23:43:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:43:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:43:40] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:43:40.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:40 smithi006 bash[33918]: cluster 2023-08-27T23:43:39.488365+0000 mgr.y (mgr.14901) 3153 : cluster [DBG] pgmap v3160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:40.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:40 smithi006 bash[33918]: audit 2023-08-27T23:43:39.704158+0000 mon.c (mon.2) 1210 : audit [DBG] from='client.? 172.21.15.6:0/263503313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:40.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:40 smithi006 bash[27292]: cluster 2023-08-27T23:43:39.488365+0000 mgr.y (mgr.14901) 3153 : cluster [DBG] pgmap v3160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:40.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:40 smithi006 bash[27292]: audit 2023-08-27T23:43:39.704158+0000 mon.c (mon.2) 1210 : audit [DBG] from='client.? 172.21.15.6:0/263503313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:40.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:43:40.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:40.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:43:40.544Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:42.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:41 smithi130 bash[51302]: debug 2023-08-27T23:43:41.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:41.763162+0000) 2023-08-27T23:43:42.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:43:42.110Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:42.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:43:42.111Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:42 smithi130 bash[27579]: cluster 2023-08-27T23:43:41.489059+0000 mgr.y (mgr.14901) 3154 : cluster [DBG] pgmap v3161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:42 smithi130 bash[27579]: audit 2023-08-27T23:43:42.125225+0000 mon.c (mon.2) 1211 : audit [DBG] from='client.? 172.21.15.6:0/494437748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:42 smithi130 bash[51302]: debug 2023-08-27T23:43:42.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:42.763477+0000) 2023-08-27T23:43:42.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:42 smithi006 bash[27292]: cluster 2023-08-27T23:43:41.489059+0000 mgr.y (mgr.14901) 3154 : cluster [DBG] pgmap v3161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:42.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:42 smithi006 bash[27292]: audit 2023-08-27T23:43:42.125225+0000 mon.c (mon.2) 1211 : audit [DBG] from='client.? 172.21.15.6:0/494437748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:42.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:42 smithi006 bash[33918]: cluster 2023-08-27T23:43:41.489059+0000 mgr.y (mgr.14901) 3154 : cluster [DBG] pgmap v3161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:42.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:42 smithi006 bash[33918]: audit 2023-08-27T23:43:42.125225+0000 mon.c (mon.2) 1211 : audit [DBG] from='client.? 172.21.15.6:0/494437748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:44.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:43 smithi130 bash[51302]: debug 2023-08-27T23:43:43.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:43.763751+0000) 2023-08-27T23:43:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:44 smithi006 bash[56298]: level=error ts=2023-08-27T23:43:44.008Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:44 smithi006 bash[56298]: level=warn ts=2023-08-27T23:43:44.010Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:44.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:44 smithi130 bash[51302]: debug 2023-08-27T23:43:44.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:44.764041+0000) 2023-08-27T23:43:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:44 smithi130 bash[27579]: cluster 2023-08-27T23:43:43.489638+0000 mgr.y (mgr.14901) 3155 : cluster [DBG] pgmap v3162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:44 smithi130 bash[27579]: audit 2023-08-27T23:43:44.535874+0000 mon.a (mon.0) 2542 : audit [DBG] from='client.? 172.21.15.6:0/4290838632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:44.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:44 smithi006 bash[27292]: cluster 2023-08-27T23:43:43.489638+0000 mgr.y (mgr.14901) 3155 : cluster [DBG] pgmap v3162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:44.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:44 smithi006 bash[27292]: audit 2023-08-27T23:43:44.535874+0000 mon.a (mon.0) 2542 : audit [DBG] from='client.? 172.21.15.6:0/4290838632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:44.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:44 smithi006 bash[33918]: cluster 2023-08-27T23:43:43.489638+0000 mgr.y (mgr.14901) 3155 : cluster [DBG] pgmap v3162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:44.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:44 smithi006 bash[33918]: audit 2023-08-27T23:43:44.535874+0000 mon.a (mon.0) 2542 : audit [DBG] from='client.? 172.21.15.6:0/4290838632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:46.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:46 smithi130 bash[27579]: cluster 2023-08-27T23:43:45.490759+0000 mgr.y (mgr.14901) 3156 : cluster [DBG] pgmap v3163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:46.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:46 smithi130 bash[27579]: audit 2023-08-27T23:43:45.663406+0000 mon.b (mon.1) 418 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:43:46.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:45 smithi130 bash[51302]: debug 2023-08-27T23:43:45.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:45.764292+0000) 2023-08-27T23:43:46.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:46 smithi006 bash[33918]: cluster 2023-08-27T23:43:45.490759+0000 mgr.y (mgr.14901) 3156 : cluster [DBG] pgmap v3163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:46.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:46 smithi006 bash[33918]: audit 2023-08-27T23:43:45.663406+0000 mon.b (mon.1) 418 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:43:46.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:46 smithi006 bash[27292]: cluster 2023-08-27T23:43:45.490759+0000 mgr.y (mgr.14901) 3156 : cluster [DBG] pgmap v3163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:46.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:46 smithi006 bash[27292]: audit 2023-08-27T23:43:45.663406+0000 mon.b (mon.1) 418 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:43:47.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:47 smithi130 bash[27579]: audit 2023-08-27T23:43:46.953628+0000 mon.a (mon.0) 2543 : audit [DBG] from='client.? 172.21.15.6:0/3604836072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:47.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:46 smithi130 bash[51302]: debug 2023-08-27T23:43:46.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:46.764564+0000) 2023-08-27T23:43:47.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:47 smithi006 bash[27292]: audit 2023-08-27T23:43:46.953628+0000 mon.a (mon.0) 2543 : audit [DBG] from='client.? 172.21.15.6:0/3604836072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:47.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:47 smithi006 bash[33918]: audit 2023-08-27T23:43:46.953628+0000 mon.a (mon.0) 2543 : audit [DBG] from='client.? 172.21.15.6:0/3604836072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:48.020 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:47 smithi130 bash[51302]: debug 2023-08-27T23:43:47.764+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:47.764870+0000) 2023-08-27T23:43:48.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:48 smithi130 bash[27579]: cluster 2023-08-27T23:43:47.491453+0000 mgr.y (mgr.14901) 3157 : cluster [DBG] pgmap v3164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:48.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:48 smithi006 bash[27292]: cluster 2023-08-27T23:43:47.491453+0000 mgr.y (mgr.14901) 3157 : cluster [DBG] pgmap v3164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:48.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:48 smithi006 bash[33918]: cluster 2023-08-27T23:43:47.491453+0000 mgr.y (mgr.14901) 3157 : cluster [DBG] pgmap v3164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:49.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:48 smithi130 bash[51302]: debug 2023-08-27T23:43:48.764+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:48.765172+0000) 2023-08-27T23:43:49.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:49 smithi006 bash[33918]: audit 2023-08-27T23:43:49.360065+0000 mon.c (mon.2) 1212 : audit [DBG] from='client.? 172.21.15.6:0/4199614005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:49.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:49 smithi006 bash[27292]: audit 2023-08-27T23:43:49.360065+0000 mon.c (mon.2) 1212 : audit [DBG] from='client.? 172.21.15.6:0/4199614005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:49.766 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:49 smithi130 bash[27579]: audit 2023-08-27T23:43:49.360065+0000 mon.c (mon.2) 1212 : audit [DBG] from='client.? 172.21.15.6:0/4199614005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:50.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:49 smithi130 bash[51302]: debug 2023-08-27T23:43:49.764+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:49.765501+0000) 2023-08-27T23:43:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:50 smithi006 bash[27292]: cluster 2023-08-27T23:43:49.492632+0000 mgr.y (mgr.14901) 3158 : cluster [DBG] pgmap v3165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:50 smithi006 bash[33918]: cluster 2023-08-27T23:43:49.492632+0000 mgr.y (mgr.14901) 3158 : cluster [DBG] pgmap v3165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:50.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:43:50.543Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:50.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:43:50.544Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:50.767 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:50 smithi130 bash[27579]: cluster 2023-08-27T23:43:49.492632+0000 mgr.y (mgr.14901) 3158 : cluster [DBG] pgmap v3165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:51.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:50 smithi130 bash[51302]: debug 2023-08-27T23:43:50.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:50.765728+0000) 2023-08-27T23:43:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:43:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:43:50] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-27T23:43:52.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:51 smithi130 bash[51302]: debug 2023-08-27T23:43:51.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:51.766048+0000) 2023-08-27T23:43:52.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:43:52.111Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:53.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:43:52.112Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:53.466 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:52 smithi130 bash[51302]: debug 2023-08-27T23:43:52.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:52.766275+0000) 2023-08-27T23:43:53.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:52 smithi006 bash[27292]: cluster 2023-08-27T23:43:51.493197+0000 mgr.y (mgr.14901) 3159 : cluster [DBG] pgmap v3166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:53.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:52 smithi006 bash[27292]: audit 2023-08-27T23:43:51.561186+0000 mon.a (mon.0) 2544 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:43:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:52 smithi006 bash[27292]: audit 2023-08-27T23:43:51.801293+0000 mon.a (mon.0) 2545 : audit [DBG] from='client.? 172.21.15.6:0/1950969545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:52 smithi006 bash[27292]: audit 2023-08-27T23:43:52.010380+0000 mon.a (mon.0) 2546 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:43:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:52 smithi006 bash[27292]: audit 2023-08-27T23:43:52.286957+0000 mon.b (mon.1) 419 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:43:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:52 smithi006 bash[27292]: audit 2023-08-27T23:43:52.289343+0000 mon.b (mon.1) 420 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:43:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:52 smithi006 bash[27292]: audit 2023-08-27T23:43:52.297388+0000 mon.a (mon.0) 2547 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:43:53.467 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:52 smithi130 bash[27579]: cluster 2023-08-27T23:43:51.493197+0000 mgr.y (mgr.14901) 3159 : cluster [DBG] pgmap v3166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:53.467 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:52 smithi130 bash[27579]: audit 2023-08-27T23:43:51.561186+0000 mon.a (mon.0) 2544 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:43:53.468 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:52 smithi130 bash[27579]: audit 2023-08-27T23:43:51.801293+0000 mon.a (mon.0) 2545 : audit [DBG] from='client.? 172.21.15.6:0/1950969545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:53.468 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:52 smithi130 bash[27579]: audit 2023-08-27T23:43:52.010380+0000 mon.a (mon.0) 2546 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:43:53.468 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:52 smithi130 bash[27579]: audit 2023-08-27T23:43:52.286957+0000 mon.b (mon.1) 419 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:43:53.468 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:52 smithi130 bash[27579]: audit 2023-08-27T23:43:52.289343+0000 mon.b (mon.1) 420 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:43:53.468 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:52 smithi130 bash[27579]: audit 2023-08-27T23:43:52.297388+0000 mon.a (mon.0) 2547 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:43:53.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:52 smithi006 bash[33918]: cluster 2023-08-27T23:43:51.493197+0000 mgr.y (mgr.14901) 3159 : cluster [DBG] pgmap v3166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:53.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:52 smithi006 bash[33918]: audit 2023-08-27T23:43:51.561186+0000 mon.a (mon.0) 2544 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:43:53.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:52 smithi006 bash[33918]: audit 2023-08-27T23:43:51.801293+0000 mon.a (mon.0) 2545 : audit [DBG] from='client.? 172.21.15.6:0/1950969545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:53.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:52 smithi006 bash[33918]: audit 2023-08-27T23:43:52.010380+0000 mon.a (mon.0) 2546 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:43:53.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:52 smithi006 bash[33918]: audit 2023-08-27T23:43:52.286957+0000 mon.b (mon.1) 419 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:43:53.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:52 smithi006 bash[33918]: audit 2023-08-27T23:43:52.289343+0000 mon.b (mon.1) 420 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:43:53.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:52 smithi006 bash[33918]: audit 2023-08-27T23:43:52.297388+0000 mon.a (mon.0) 2547 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:43:54.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:53 smithi130 bash[51302]: debug 2023-08-27T23:43:53.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:53.766576+0000) 2023-08-27T23:43:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:54 smithi006 bash[56298]: level=error ts=2023-08-27T23:43:54.009Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:43:54 smithi006 bash[56298]: level=warn ts=2023-08-27T23:43:54.011Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:43:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:54 smithi130 bash[27579]: cluster 2023-08-27T23:43:53.493968+0000 mgr.y (mgr.14901) 3160 : cluster [DBG] pgmap v3167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:54 smithi130 bash[27579]: audit 2023-08-27T23:43:54.216180+0000 mon.c (mon.2) 1213 : audit [DBG] from='client.? 172.21.15.6:0/1341509419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:54.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:54 smithi130 bash[51302]: debug 2023-08-27T23:43:54.764+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:54.766914+0000) 2023-08-27T23:43:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:54 smithi006 bash[27292]: cluster 2023-08-27T23:43:53.493968+0000 mgr.y (mgr.14901) 3160 : cluster [DBG] pgmap v3167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:54.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:54 smithi006 bash[27292]: audit 2023-08-27T23:43:54.216180+0000 mon.c (mon.2) 1213 : audit [DBG] from='client.? 172.21.15.6:0/1341509419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:54.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:54 smithi006 bash[33918]: cluster 2023-08-27T23:43:53.493968+0000 mgr.y (mgr.14901) 3160 : cluster [DBG] pgmap v3167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:54.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:54 smithi006 bash[33918]: audit 2023-08-27T23:43:54.216180+0000 mon.c (mon.2) 1213 : audit [DBG] from='client.? 172.21.15.6:0/1341509419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:56.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:56 smithi130 bash[27579]: cluster 2023-08-27T23:43:55.495305+0000 mgr.y (mgr.14901) 3161 : cluster [DBG] pgmap v3168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:56.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:55 smithi130 bash[51302]: debug 2023-08-27T23:43:55.764+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:55.767278+0000) 2023-08-27T23:43:56.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:56 smithi006 bash[33918]: cluster 2023-08-27T23:43:55.495305+0000 mgr.y (mgr.14901) 3161 : cluster [DBG] pgmap v3168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:56.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:56 smithi006 bash[27292]: cluster 2023-08-27T23:43:55.495305+0000 mgr.y (mgr.14901) 3161 : cluster [DBG] pgmap v3168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:57.019 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:56 smithi130 bash[51302]: debug 2023-08-27T23:43:56.764+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:56.767540+0000) 2023-08-27T23:43:57.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:57 smithi130 bash[27579]: audit 2023-08-27T23:43:56.624511+0000 mon.a (mon.0) 2548 : audit [DBG] from='client.? 172.21.15.6:0/1517144454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:57.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:57 smithi006 bash[27292]: audit 2023-08-27T23:43:56.624511+0000 mon.a (mon.0) 2548 : audit [DBG] from='client.? 172.21.15.6:0/1517144454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:57.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:57 smithi006 bash[33918]: audit 2023-08-27T23:43:56.624511+0000 mon.a (mon.0) 2548 : audit [DBG] from='client.? 172.21.15.6:0/1517144454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:58.029 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:57 smithi130 bash[51302]: debug 2023-08-27T23:43:57.764+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:57.767873+0000) 2023-08-27T23:43:58.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:58 smithi130 bash[27579]: cluster 2023-08-27T23:43:57.495967+0000 mgr.y (mgr.14901) 3162 : cluster [DBG] pgmap v3169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:58.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:58 smithi006 bash[27292]: cluster 2023-08-27T23:43:57.495967+0000 mgr.y (mgr.14901) 3162 : cluster [DBG] pgmap v3169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:58.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:58 smithi006 bash[33918]: cluster 2023-08-27T23:43:57.495967+0000 mgr.y (mgr.14901) 3162 : cluster [DBG] pgmap v3169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:43:59.109 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:58 smithi130 bash[51302]: debug 2023-08-27T23:43:58.764+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:58.768135+0000) 2023-08-27T23:43:59.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:43:59 smithi130 bash[27579]: audit 2023-08-27T23:43:59.052358+0000 mon.a (mon.0) 2549 : audit [DBG] from='client.? 172.21.15.6:0/4028522750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:59.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:43:59 smithi006 bash[27292]: audit 2023-08-27T23:43:59.052358+0000 mon.a (mon.0) 2549 : audit [DBG] from='client.? 172.21.15.6:0/4028522750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:59.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:43:59 smithi006 bash[33918]: audit 2023-08-27T23:43:59.052358+0000 mon.a (mon.0) 2549 : audit [DBG] from='client.? 172.21.15.6:0/4028522750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:00.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:43:59 smithi130 bash[51302]: debug 2023-08-27T23:43:59.764+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:59.768411+0000) 2023-08-27T23:44:00.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:00 smithi006 bash[33918]: cluster 2023-08-27T23:43:59.497147+0000 mgr.y (mgr.14901) 3163 : cluster [DBG] pgmap v3170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:00.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:00 smithi006 bash[27292]: cluster 2023-08-27T23:43:59.497147+0000 mgr.y (mgr.14901) 3163 : cluster [DBG] pgmap v3170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:00.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:00 smithi130 bash[27579]: cluster 2023-08-27T23:43:59.497147+0000 mgr.y (mgr.14901) 3163 : cluster [DBG] pgmap v3170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:00.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:44:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:44:00] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-27T23:44:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:44:00.544Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:44:00.546Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:01.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:00 smithi130 bash[51302]: debug 2023-08-27T23:44:00.763+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:00.768690+0000) 2023-08-27T23:44:01.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:01 smithi130 bash[27579]: audit 2023-08-27T23:44:01.471425+0000 mon.a (mon.0) 2550 : audit [DBG] from='client.? 172.21.15.6:0/556287835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:01.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:01 smithi130 bash[51302]: debug 2023-08-27T23:44:01.763+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:01.769018+0000) 2023-08-27T23:44:01.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:01 smithi006 bash[33918]: audit 2023-08-27T23:44:01.471425+0000 mon.a (mon.0) 2550 : audit [DBG] from='client.? 172.21.15.6:0/556287835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:01.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:01 smithi006 bash[27292]: audit 2023-08-27T23:44:01.471425+0000 mon.a (mon.0) 2550 : audit [DBG] from='client.? 172.21.15.6:0/556287835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:44:02.111Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:44:02.113Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:02.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:02 smithi130 bash[51302]: debug 2023-08-27T23:44:02.763+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:02.769240+0000) 2023-08-27T23:44:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:02 smithi130 bash[27579]: cluster 2023-08-27T23:44:01.497760+0000 mgr.y (mgr.14901) 3164 : cluster [DBG] pgmap v3171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:02 smithi006 bash[27292]: cluster 2023-08-27T23:44:01.497760+0000 mgr.y (mgr.14901) 3164 : cluster [DBG] pgmap v3171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:02.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:02 smithi006 bash[33918]: cluster 2023-08-27T23:44:01.497760+0000 mgr.y (mgr.14901) 3164 : cluster [DBG] pgmap v3171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:04.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:03 smithi130 bash[51302]: debug 2023-08-27T23:44:03.763+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:03.769514+0000) 2023-08-27T23:44:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:04 smithi006 bash[56298]: level=error ts=2023-08-27T23:44:04.010Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:04 smithi006 bash[56298]: level=warn ts=2023-08-27T23:44:04.013Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:04.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:04 smithi130 bash[51302]: debug 2023-08-27T23:44:04.763+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:04.769775+0000) 2023-08-27T23:44:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:04 smithi130 bash[27579]: cluster 2023-08-27T23:44:03.498390+0000 mgr.y (mgr.14901) 3165 : cluster [DBG] pgmap v3172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:04 smithi130 bash[27579]: audit 2023-08-27T23:44:03.889861+0000 mon.c (mon.2) 1214 : audit [DBG] from='client.? 172.21.15.6:0/3763164620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:04.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:04 smithi006 bash[27292]: cluster 2023-08-27T23:44:03.498390+0000 mgr.y (mgr.14901) 3165 : cluster [DBG] pgmap v3172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:04.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:04 smithi006 bash[27292]: audit 2023-08-27T23:44:03.889861+0000 mon.c (mon.2) 1214 : audit [DBG] from='client.? 172.21.15.6:0/3763164620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:04.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:04 smithi006 bash[33918]: cluster 2023-08-27T23:44:03.498390+0000 mgr.y (mgr.14901) 3165 : cluster [DBG] pgmap v3172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:04.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:04 smithi006 bash[33918]: audit 2023-08-27T23:44:03.889861+0000 mon.c (mon.2) 1214 : audit [DBG] from='client.? 172.21.15.6:0/3763164620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:06.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:06 smithi130 bash[27579]: cluster 2023-08-27T23:44:05.499532+0000 mgr.y (mgr.14901) 3166 : cluster [DBG] pgmap v3173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:06.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:05 smithi130 bash[51302]: debug 2023-08-27T23:44:05.763+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:05.770038+0000) 2023-08-27T23:44:06.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:06 smithi006 bash[27292]: cluster 2023-08-27T23:44:05.499532+0000 mgr.y (mgr.14901) 3166 : cluster [DBG] pgmap v3173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:06 smithi006 bash[33918]: cluster 2023-08-27T23:44:05.499532+0000 mgr.y (mgr.14901) 3166 : cluster [DBG] pgmap v3173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:07 smithi130 bash[27579]: audit 2023-08-27T23:44:06.305380+0000 mon.c (mon.2) 1215 : audit [DBG] from='client.? 172.21.15.6:0/3520736638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:07.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:06 smithi130 bash[51302]: debug 2023-08-27T23:44:06.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:06.770336+0000) 2023-08-27T23:44:07.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:07 smithi006 bash[27292]: audit 2023-08-27T23:44:06.305380+0000 mon.c (mon.2) 1215 : audit [DBG] from='client.? 172.21.15.6:0/3520736638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:07.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:07 smithi006 bash[33918]: audit 2023-08-27T23:44:06.305380+0000 mon.c (mon.2) 1215 : audit [DBG] from='client.? 172.21.15.6:0/3520736638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:08.025 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:07 smithi130 bash[51302]: debug 2023-08-27T23:44:07.763+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:07.770655+0000) 2023-08-27T23:44:08.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:08 smithi130 bash[27579]: cluster 2023-08-27T23:44:07.500201+0000 mgr.y (mgr.14901) 3167 : cluster [DBG] pgmap v3174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:08.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:08 smithi006 bash[33918]: cluster 2023-08-27T23:44:07.500201+0000 mgr.y (mgr.14901) 3167 : cluster [DBG] pgmap v3174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:08.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:08 smithi006 bash[27292]: cluster 2023-08-27T23:44:07.500201+0000 mgr.y (mgr.14901) 3167 : cluster [DBG] pgmap v3174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:09.029 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:08 smithi130 bash[51302]: debug 2023-08-27T23:44:08.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:08.770987+0000) 2023-08-27T23:44:09.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:09 smithi130 bash[27579]: audit 2023-08-27T23:44:08.727491+0000 mon.a (mon.0) 2551 : audit [DBG] from='client.? 172.21.15.6:0/3142099396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:09.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:09 smithi006 bash[27292]: audit 2023-08-27T23:44:08.727491+0000 mon.a (mon.0) 2551 : audit [DBG] from='client.? 172.21.15.6:0/3142099396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:09.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:09 smithi006 bash[33918]: audit 2023-08-27T23:44:08.727491+0000 mon.a (mon.0) 2551 : audit [DBG] from='client.? 172.21.15.6:0/3142099396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:10.035 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:09 smithi130 bash[51302]: debug 2023-08-27T23:44:09.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:09.771280+0000) 2023-08-27T23:44:10.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:10 smithi130 bash[27579]: cluster 2023-08-27T23:44:09.501385+0000 mgr.y (mgr.14901) 3168 : cluster [DBG] pgmap v3175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:10.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:10 smithi006 bash[33918]: cluster 2023-08-27T23:44:09.501385+0000 mgr.y (mgr.14901) 3168 : cluster [DBG] pgmap v3175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:10.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:10 smithi006 bash[27292]: cluster 2023-08-27T23:44:09.501385+0000 mgr.y (mgr.14901) 3168 : cluster [DBG] pgmap v3175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:44:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:44:10] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-27T23:44:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:44:10.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:44:10.547Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:11.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:10 smithi130 bash[51302]: debug 2023-08-27T23:44:10.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:10.771575+0000) 2023-08-27T23:44:11.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:11 smithi006 bash[27292]: audit 2023-08-27T23:44:11.158653+0000 mon.a (mon.0) 2552 : audit [DBG] from='client.? 172.21.15.6:0/3863096684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:11.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:11 smithi006 bash[33918]: audit 2023-08-27T23:44:11.158653+0000 mon.a (mon.0) 2552 : audit [DBG] from='client.? 172.21.15.6:0/3863096684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:11.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:11 smithi130 bash[27579]: audit 2023-08-27T23:44:11.158653+0000 mon.a (mon.0) 2552 : audit [DBG] from='client.? 172.21.15.6:0/3863096684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:12.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:11 smithi130 bash[51302]: debug 2023-08-27T23:44:11.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:11.771847+0000) 2023-08-27T23:44:12.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:12 smithi006 bash[27292]: cluster 2023-08-27T23:44:11.502163+0000 mgr.y (mgr.14901) 3169 : cluster [DBG] pgmap v3176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:12.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:12 smithi006 bash[33918]: cluster 2023-08-27T23:44:11.502163+0000 mgr.y (mgr.14901) 3169 : cluster [DBG] pgmap v3176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:12.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:44:12.112Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:12.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:44:12.114Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:12.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:12 smithi130 bash[27579]: cluster 2023-08-27T23:44:11.502163+0000 mgr.y (mgr.14901) 3169 : cluster [DBG] pgmap v3176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:13.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:12 smithi130 bash[51302]: debug 2023-08-27T23:44:12.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:12.772127+0000) 2023-08-27T23:44:14.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:13 smithi130 bash[51302]: debug 2023-08-27T23:44:13.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:13.772436+0000) 2023-08-27T23:44:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:14 smithi006 bash[56298]: level=error ts=2023-08-27T23:44:14.011Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:14 smithi006 bash[56298]: level=warn ts=2023-08-27T23:44:14.013Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:14 smithi130 bash[27579]: cluster 2023-08-27T23:44:13.502783+0000 mgr.y (mgr.14901) 3170 : cluster [DBG] pgmap v3177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:14 smithi130 bash[27579]: audit 2023-08-27T23:44:13.574935+0000 mon.c (mon.2) 1216 : audit [DBG] from='client.? 172.21.15.6:0/1607410999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:14 smithi130 bash[51302]: debug 2023-08-27T23:44:14.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:14.772721+0000) 2023-08-27T23:44:14.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:14 smithi006 bash[33918]: cluster 2023-08-27T23:44:13.502783+0000 mgr.y (mgr.14901) 3170 : cluster [DBG] pgmap v3177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:14.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:14 smithi006 bash[33918]: audit 2023-08-27T23:44:13.574935+0000 mon.c (mon.2) 1216 : audit [DBG] from='client.? 172.21.15.6:0/1607410999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:14.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:14 smithi006 bash[27292]: cluster 2023-08-27T23:44:13.502783+0000 mgr.y (mgr.14901) 3170 : cluster [DBG] pgmap v3177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:14.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:14 smithi006 bash[27292]: audit 2023-08-27T23:44:13.574935+0000 mon.c (mon.2) 1216 : audit [DBG] from='client.? 172.21.15.6:0/1607410999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:16.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:16 smithi130 bash[27579]: cluster 2023-08-27T23:44:15.503872+0000 mgr.y (mgr.14901) 3171 : cluster [DBG] pgmap v3178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:16.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:16 smithi130 bash[27579]: audit 2023-08-27T23:44:15.994574+0000 mon.a (mon.0) 2553 : audit [DBG] from='client.? 172.21.15.6:0/557712276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:16.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:15 smithi130 bash[51302]: debug 2023-08-27T23:44:15.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:15.773047+0000) 2023-08-27T23:44:16.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:16 smithi006 bash[27292]: cluster 2023-08-27T23:44:15.503872+0000 mgr.y (mgr.14901) 3171 : cluster [DBG] pgmap v3178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:16.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:16 smithi006 bash[27292]: audit 2023-08-27T23:44:15.994574+0000 mon.a (mon.0) 2553 : audit [DBG] from='client.? 172.21.15.6:0/557712276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:16.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:16 smithi006 bash[33918]: cluster 2023-08-27T23:44:15.503872+0000 mgr.y (mgr.14901) 3171 : cluster [DBG] pgmap v3178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:16.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:16 smithi006 bash[33918]: audit 2023-08-27T23:44:15.994574+0000 mon.a (mon.0) 2553 : audit [DBG] from='client.? 172.21.15.6:0/557712276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:17.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:16 smithi130 bash[51302]: debug 2023-08-27T23:44:16.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:16.773351+0000) 2023-08-27T23:44:18.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:17 smithi130 bash[51302]: debug 2023-08-27T23:44:17.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:17.773664+0000) 2023-08-27T23:44:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:18 smithi130 bash[27579]: cluster 2023-08-27T23:44:17.504496+0000 mgr.y (mgr.14901) 3172 : cluster [DBG] pgmap v3179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:18 smithi130 bash[27579]: audit 2023-08-27T23:44:18.409085+0000 mon.c (mon.2) 1217 : audit [DBG] from='client.? 172.21.15.6:0/2419055046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:18 smithi130 bash[51302]: debug 2023-08-27T23:44:18.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:18.773895+0000) 2023-08-27T23:44:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:18 smithi006 bash[27292]: cluster 2023-08-27T23:44:17.504496+0000 mgr.y (mgr.14901) 3172 : cluster [DBG] pgmap v3179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:18.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:18 smithi006 bash[27292]: audit 2023-08-27T23:44:18.409085+0000 mon.c (mon.2) 1217 : audit [DBG] from='client.? 172.21.15.6:0/2419055046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:18.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:18 smithi006 bash[33918]: cluster 2023-08-27T23:44:17.504496+0000 mgr.y (mgr.14901) 3172 : cluster [DBG] pgmap v3179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:18.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:18 smithi006 bash[33918]: audit 2023-08-27T23:44:18.409085+0000 mon.c (mon.2) 1217 : audit [DBG] from='client.? 172.21.15.6:0/2419055046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:20.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:19 smithi130 bash[51302]: debug 2023-08-27T23:44:19.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:19.774184+0000) 2023-08-27T23:44:20.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:20 smithi130 bash[27579]: cluster 2023-08-27T23:44:19.505752+0000 mgr.y (mgr.14901) 3173 : cluster [DBG] pgmap v3180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:20 smithi130 bash[51302]: debug 2023-08-27T23:44:20.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:20.774513+0000) 2023-08-27T23:44:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:44:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:44:20] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-27T23:44:20.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:20 smithi006 bash[33918]: cluster 2023-08-27T23:44:19.505752+0000 mgr.y (mgr.14901) 3173 : cluster [DBG] pgmap v3180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:20.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:20 smithi006 bash[27292]: cluster 2023-08-27T23:44:19.505752+0000 mgr.y (mgr.14901) 3173 : cluster [DBG] pgmap v3180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:20.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:44:20.546Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:20.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:44:20.548Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:21.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:21 smithi130 bash[27579]: audit 2023-08-27T23:44:20.836699+0000 mon.a (mon.0) 2554 : audit [DBG] from='client.? 172.21.15.6:0/543572729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:21.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:21 smithi130 bash[51302]: debug 2023-08-27T23:44:21.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:21.774783+0000) 2023-08-27T23:44:21.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:21 smithi006 bash[33918]: audit 2023-08-27T23:44:20.836699+0000 mon.a (mon.0) 2554 : audit [DBG] from='client.? 172.21.15.6:0/543572729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:21.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:21 smithi006 bash[27292]: audit 2023-08-27T23:44:20.836699+0000 mon.a (mon.0) 2554 : audit [DBG] from='client.? 172.21.15.6:0/543572729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:44:22.112Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:22.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:44:22.114Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:22 smithi130 bash[27579]: cluster 2023-08-27T23:44:21.506448+0000 mgr.y (mgr.14901) 3174 : cluster [DBG] pgmap v3181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:22 smithi130 bash[51302]: debug 2023-08-27T23:44:22.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:22.775072+0000) 2023-08-27T23:44:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:22 smithi006 bash[33918]: cluster 2023-08-27T23:44:21.506448+0000 mgr.y (mgr.14901) 3174 : cluster [DBG] pgmap v3181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:22.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:22 smithi006 bash[27292]: cluster 2023-08-27T23:44:21.506448+0000 mgr.y (mgr.14901) 3174 : cluster [DBG] pgmap v3181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:23.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:23 smithi130 bash[51302]: debug 2023-08-27T23:44:23.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:23.775388+0000) 2023-08-27T23:44:23.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:23 smithi130 bash[27579]: audit 2023-08-27T23:44:23.250690+0000 mon.c (mon.2) 1218 : audit [DBG] from='client.? 172.21.15.6:0/3584763189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:23.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:23 smithi006 bash[33918]: audit 2023-08-27T23:44:23.250690+0000 mon.c (mon.2) 1218 : audit [DBG] from='client.? 172.21.15.6:0/3584763189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:23.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:23 smithi006 bash[27292]: audit 2023-08-27T23:44:23.250690+0000 mon.c (mon.2) 1218 : audit [DBG] from='client.? 172.21.15.6:0/3584763189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:24 smithi006 bash[56298]: level=error ts=2023-08-27T23:44:24.012Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:24 smithi006 bash[56298]: level=warn ts=2023-08-27T23:44:24.014Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:24 smithi130 bash[27579]: cluster 2023-08-27T23:44:23.507136+0000 mgr.y (mgr.14901) 3175 : cluster [DBG] pgmap v3182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:24 smithi130 bash[51302]: debug 2023-08-27T23:44:24.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:24.775608+0000) 2023-08-27T23:44:24.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:24 smithi006 bash[27292]: cluster 2023-08-27T23:44:23.507136+0000 mgr.y (mgr.14901) 3175 : cluster [DBG] pgmap v3182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:24.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:24 smithi006 bash[33918]: cluster 2023-08-27T23:44:23.507136+0000 mgr.y (mgr.14901) 3175 : cluster [DBG] pgmap v3182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:26.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:25 smithi130 bash[51302]: debug 2023-08-27T23:44:25.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:25.775879+0000) 2023-08-27T23:44:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:26 smithi130 bash[27579]: cluster 2023-08-27T23:44:25.508215+0000 mgr.y (mgr.14901) 3176 : cluster [DBG] pgmap v3183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:26 smithi130 bash[27579]: audit 2023-08-27T23:44:25.665410+0000 mon.a (mon.0) 2555 : audit [DBG] from='client.? 172.21.15.6:0/916795247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:26 smithi006 bash[27292]: cluster 2023-08-27T23:44:25.508215+0000 mgr.y (mgr.14901) 3176 : cluster [DBG] pgmap v3183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:26.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:26 smithi006 bash[27292]: audit 2023-08-27T23:44:25.665410+0000 mon.a (mon.0) 2555 : audit [DBG] from='client.? 172.21.15.6:0/916795247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:26 smithi006 bash[33918]: cluster 2023-08-27T23:44:25.508215+0000 mgr.y (mgr.14901) 3176 : cluster [DBG] pgmap v3183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:26 smithi006 bash[33918]: audit 2023-08-27T23:44:25.665410+0000 mon.a (mon.0) 2555 : audit [DBG] from='client.? 172.21.15.6:0/916795247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:27.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:26 smithi130 bash[51302]: debug 2023-08-27T23:44:26.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:26.776140+0000) 2023-08-27T23:44:28.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:27 smithi130 bash[51302]: debug 2023-08-27T23:44:27.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:27.776408+0000) 2023-08-27T23:44:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:28 smithi130 bash[27579]: cluster 2023-08-27T23:44:27.508947+0000 mgr.y (mgr.14901) 3177 : cluster [DBG] pgmap v3184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:28 smithi130 bash[27579]: audit 2023-08-27T23:44:28.081333+0000 mon.c (mon.2) 1219 : audit [DBG] from='client.? 172.21.15.6:0/1988412277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:28 smithi130 bash[51302]: debug 2023-08-27T23:44:28.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:28.776750+0000) 2023-08-27T23:44:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:28 smithi006 bash[27292]: cluster 2023-08-27T23:44:27.508947+0000 mgr.y (mgr.14901) 3177 : cluster [DBG] pgmap v3184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:28.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:28 smithi006 bash[27292]: audit 2023-08-27T23:44:28.081333+0000 mon.c (mon.2) 1219 : audit [DBG] from='client.? 172.21.15.6:0/1988412277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:28.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:28 smithi006 bash[33918]: cluster 2023-08-27T23:44:27.508947+0000 mgr.y (mgr.14901) 3177 : cluster [DBG] pgmap v3184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:28.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:28 smithi006 bash[33918]: audit 2023-08-27T23:44:28.081333+0000 mon.c (mon.2) 1219 : audit [DBG] from='client.? 172.21.15.6:0/1988412277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:30.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:29 smithi130 bash[51302]: debug 2023-08-27T23:44:29.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:29.777028+0000) 2023-08-27T23:44:30.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:30 smithi130 bash[27579]: cluster 2023-08-27T23:44:29.510065+0000 mgr.y (mgr.14901) 3178 : cluster [DBG] pgmap v3185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:30 smithi130 bash[27579]: audit 2023-08-27T23:44:30.504351+0000 mon.c (mon.2) 1220 : audit [DBG] from='client.? 172.21.15.6:0/957052423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:30 smithi130 bash[51302]: debug 2023-08-27T23:44:30.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:30.777309+0000) 2023-08-27T23:44:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:44:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:44:30] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-27T23:44:30.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:30 smithi006 bash[27292]: cluster 2023-08-27T23:44:29.510065+0000 mgr.y (mgr.14901) 3178 : cluster [DBG] pgmap v3185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:30.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:30 smithi006 bash[27292]: audit 2023-08-27T23:44:30.504351+0000 mon.c (mon.2) 1220 : audit [DBG] from='client.? 172.21.15.6:0/957052423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:30.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:30 smithi006 bash[33918]: cluster 2023-08-27T23:44:29.510065+0000 mgr.y (mgr.14901) 3178 : cluster [DBG] pgmap v3185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:30.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:30 smithi006 bash[33918]: audit 2023-08-27T23:44:30.504351+0000 mon.c (mon.2) 1220 : audit [DBG] from='client.? 172.21.15.6:0/957052423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:30.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:44:30.546Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:30.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:44:30.548Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:32.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:31 smithi130 bash[51302]: debug 2023-08-27T23:44:31.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:31.777594+0000) 2023-08-27T23:44:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:44:32.113Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:32.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:44:32.114Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:32 smithi130 bash[27579]: cluster 2023-08-27T23:44:31.510823+0000 mgr.y (mgr.14901) 3179 : cluster [DBG] pgmap v3186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:32 smithi130 bash[51302]: debug 2023-08-27T23:44:32.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:32.777934+0000) 2023-08-27T23:44:32.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:32 smithi006 bash[27292]: cluster 2023-08-27T23:44:31.510823+0000 mgr.y (mgr.14901) 3179 : cluster [DBG] pgmap v3186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:32.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:32 smithi006 bash[33918]: cluster 2023-08-27T23:44:31.510823+0000 mgr.y (mgr.14901) 3179 : cluster [DBG] pgmap v3186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:33 smithi130 bash[27579]: audit 2023-08-27T23:44:32.921892+0000 mon.a (mon.0) 2556 : audit [DBG] from='client.? 172.21.15.6:0/803520235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:33.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:33 smithi130 bash[51302]: debug 2023-08-27T23:44:33.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:33.778233+0000) 2023-08-27T23:44:33.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:33 smithi006 bash[33918]: audit 2023-08-27T23:44:32.921892+0000 mon.a (mon.0) 2556 : audit [DBG] from='client.? 172.21.15.6:0/803520235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:33.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:33 smithi006 bash[27292]: audit 2023-08-27T23:44:32.921892+0000 mon.a (mon.0) 2556 : audit [DBG] from='client.? 172.21.15.6:0/803520235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:34 smithi006 bash[56298]: level=error ts=2023-08-27T23:44:34.012Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:34 smithi006 bash[56298]: level=warn ts=2023-08-27T23:44:34.014Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:34 smithi130 bash[27579]: cluster 2023-08-27T23:44:33.511527+0000 mgr.y (mgr.14901) 3180 : cluster [DBG] pgmap v3187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:34 smithi130 bash[51302]: debug 2023-08-27T23:44:34.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:34.778525+0000) 2023-08-27T23:44:34.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:34 smithi006 bash[33918]: cluster 2023-08-27T23:44:33.511527+0000 mgr.y (mgr.14901) 3180 : cluster [DBG] pgmap v3187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:34.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:34 smithi006 bash[27292]: cluster 2023-08-27T23:44:33.511527+0000 mgr.y (mgr.14901) 3180 : cluster [DBG] pgmap v3187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:36.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:36 smithi130 bash[27579]: audit 2023-08-27T23:44:35.332447+0000 mon.a (mon.0) 2557 : audit [DBG] from='client.? 172.21.15.6:0/424938233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:36.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:36 smithi130 bash[27579]: cluster 2023-08-27T23:44:35.512634+0000 mgr.y (mgr.14901) 3181 : cluster [DBG] pgmap v3188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:36.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:35 smithi130 bash[51302]: debug 2023-08-27T23:44:35.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:35.778786+0000) 2023-08-27T23:44:36.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:36 smithi006 bash[27292]: audit 2023-08-27T23:44:35.332447+0000 mon.a (mon.0) 2557 : audit [DBG] from='client.? 172.21.15.6:0/424938233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:36.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:36 smithi006 bash[27292]: cluster 2023-08-27T23:44:35.512634+0000 mgr.y (mgr.14901) 3181 : cluster [DBG] pgmap v3188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:36.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:36 smithi006 bash[33918]: audit 2023-08-27T23:44:35.332447+0000 mon.a (mon.0) 2557 : audit [DBG] from='client.? 172.21.15.6:0/424938233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:36.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:36 smithi006 bash[33918]: cluster 2023-08-27T23:44:35.512634+0000 mgr.y (mgr.14901) 3181 : cluster [DBG] pgmap v3188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:37.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:36 smithi130 bash[51302]: debug 2023-08-27T23:44:36.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:36.779087+0000) 2023-08-27T23:44:38.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:37 smithi130 bash[51302]: debug 2023-08-27T23:44:37.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:37.779302+0000) 2023-08-27T23:44:38.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:38 smithi130 bash[27579]: cluster 2023-08-27T23:44:37.513354+0000 mgr.y (mgr.14901) 3182 : cluster [DBG] pgmap v3189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:38 smithi130 bash[27579]: audit 2023-08-27T23:44:37.748028+0000 mon.a (mon.0) 2558 : audit [DBG] from='client.? 172.21.15.6:0/1384294631' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:38 smithi130 bash[51302]: debug 2023-08-27T23:44:38.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:38.779616+0000) 2023-08-27T23:44:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:38 smithi006 bash[27292]: cluster 2023-08-27T23:44:37.513354+0000 mgr.y (mgr.14901) 3182 : cluster [DBG] pgmap v3189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:38.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:38 smithi006 bash[27292]: audit 2023-08-27T23:44:37.748028+0000 mon.a (mon.0) 2558 : audit [DBG] from='client.? 172.21.15.6:0/1384294631' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:38.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:38 smithi006 bash[33918]: cluster 2023-08-27T23:44:37.513354+0000 mgr.y (mgr.14901) 3182 : cluster [DBG] pgmap v3189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:38.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:38 smithi006 bash[33918]: audit 2023-08-27T23:44:37.748028+0000 mon.a (mon.0) 2558 : audit [DBG] from='client.? 172.21.15.6:0/1384294631' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:40.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:39 smithi130 bash[51302]: debug 2023-08-27T23:44:39.779+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:39.779880+0000) 2023-08-27T23:44:40.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:40 smithi130 bash[51302]: debug 2023-08-27T23:44:40.779+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:40.780152+0000) 2023-08-27T23:44:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:40 smithi130 bash[27579]: cluster 2023-08-27T23:44:39.514486+0000 mgr.y (mgr.14901) 3183 : cluster [DBG] pgmap v3190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:40 smithi130 bash[27579]: audit 2023-08-27T23:44:40.164246+0000 mon.a (mon.0) 2559 : audit [DBG] from='client.? 172.21.15.6:0/1649122089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:44:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:44:40] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-27T23:44:40.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:40 smithi006 bash[27292]: cluster 2023-08-27T23:44:39.514486+0000 mgr.y (mgr.14901) 3183 : cluster [DBG] pgmap v3190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:40.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:40 smithi006 bash[27292]: audit 2023-08-27T23:44:40.164246+0000 mon.a (mon.0) 2559 : audit [DBG] from='client.? 172.21.15.6:0/1649122089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:44:40.548Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:44:40.550Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:40.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:40 smithi006 bash[33918]: cluster 2023-08-27T23:44:39.514486+0000 mgr.y (mgr.14901) 3183 : cluster [DBG] pgmap v3190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:40.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:40 smithi006 bash[33918]: audit 2023-08-27T23:44:40.164246+0000 mon.a (mon.0) 2559 : audit [DBG] from='client.? 172.21.15.6:0/1649122089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:42.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:41 smithi130 bash[51302]: debug 2023-08-27T23:44:41.779+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:41.780428+0000) 2023-08-27T23:44:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:44:42.113Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:44:42.115Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:42.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:42 smithi130 bash[51302]: debug 2023-08-27T23:44:42.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:42.780714+0000) 2023-08-27T23:44:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:42 smithi130 bash[27579]: cluster 2023-08-27T23:44:41.515197+0000 mgr.y (mgr.14901) 3184 : cluster [DBG] pgmap v3191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:42 smithi130 bash[27579]: audit 2023-08-27T23:44:42.569282+0000 mon.a (mon.0) 2560 : audit [DBG] from='client.? 172.21.15.6:0/1050064488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:42 smithi006 bash[33918]: cluster 2023-08-27T23:44:41.515197+0000 mgr.y (mgr.14901) 3184 : cluster [DBG] pgmap v3191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:42.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:42 smithi006 bash[33918]: audit 2023-08-27T23:44:42.569282+0000 mon.a (mon.0) 2560 : audit [DBG] from='client.? 172.21.15.6:0/1050064488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:42.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:42 smithi006 bash[27292]: cluster 2023-08-27T23:44:41.515197+0000 mgr.y (mgr.14901) 3184 : cluster [DBG] pgmap v3191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:42.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:42 smithi006 bash[27292]: audit 2023-08-27T23:44:42.569282+0000 mon.a (mon.0) 2560 : audit [DBG] from='client.? 172.21.15.6:0/1050064488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:44.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:43 smithi130 bash[51302]: debug 2023-08-27T23:44:43.779+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:43.780987+0000) 2023-08-27T23:44:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:44 smithi006 bash[56298]: level=error ts=2023-08-27T23:44:44.012Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:44 smithi006 bash[56298]: level=warn ts=2023-08-27T23:44:44.014Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:44.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:44 smithi130 bash[51302]: debug 2023-08-27T23:44:44.779+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:44.781301+0000) 2023-08-27T23:44:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:44 smithi130 bash[27579]: cluster 2023-08-27T23:44:43.515853+0000 mgr.y (mgr.14901) 3185 : cluster [DBG] pgmap v3192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:44 smithi006 bash[33918]: cluster 2023-08-27T23:44:43.515853+0000 mgr.y (mgr.14901) 3185 : cluster [DBG] pgmap v3192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:44.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:44 smithi006 bash[27292]: cluster 2023-08-27T23:44:43.515853+0000 mgr.y (mgr.14901) 3185 : cluster [DBG] pgmap v3192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:45.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:45 smithi130 bash[51302]: debug 2023-08-27T23:44:45.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:45.781587+0000) 2023-08-27T23:44:45.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:45 smithi130 bash[27579]: audit 2023-08-27T23:44:44.979437+0000 mon.a (mon.0) 2561 : audit [DBG] from='client.? 172.21.15.6:0/1397433126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:45.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:45 smithi006 bash[27292]: audit 2023-08-27T23:44:44.979437+0000 mon.a (mon.0) 2561 : audit [DBG] from='client.? 172.21.15.6:0/1397433126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:45.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:45 smithi006 bash[33918]: audit 2023-08-27T23:44:44.979437+0000 mon.a (mon.0) 2561 : audit [DBG] from='client.? 172.21.15.6:0/1397433126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:46.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:46 smithi130 bash[51302]: debug 2023-08-27T23:44:46.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:46.781885+0000) 2023-08-27T23:44:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:46 smithi130 bash[27579]: cluster 2023-08-27T23:44:45.517022+0000 mgr.y (mgr.14901) 3186 : cluster [DBG] pgmap v3193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:46 smithi006 bash[27292]: cluster 2023-08-27T23:44:45.517022+0000 mgr.y (mgr.14901) 3186 : cluster [DBG] pgmap v3193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:46.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:46 smithi006 bash[33918]: cluster 2023-08-27T23:44:45.517022+0000 mgr.y (mgr.14901) 3186 : cluster [DBG] pgmap v3193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:47.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:47 smithi130 bash[51302]: debug 2023-08-27T23:44:47.779+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:47.782170+0000) 2023-08-27T23:44:47.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:47 smithi130 bash[27579]: audit 2023-08-27T23:44:47.386740+0000 mon.c (mon.2) 1221 : audit [DBG] from='client.? 172.21.15.6:0/1096231771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:47.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:47 smithi006 bash[33918]: audit 2023-08-27T23:44:47.386740+0000 mon.c (mon.2) 1221 : audit [DBG] from='client.? 172.21.15.6:0/1096231771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:47.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:47 smithi006 bash[27292]: audit 2023-08-27T23:44:47.386740+0000 mon.c (mon.2) 1221 : audit [DBG] from='client.? 172.21.15.6:0/1096231771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:48 smithi130 bash[27579]: cluster 2023-08-27T23:44:47.517708+0000 mgr.y (mgr.14901) 3187 : cluster [DBG] pgmap v3194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:48 smithi130 bash[51302]: debug 2023-08-27T23:44:48.779+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:48.782468+0000) 2023-08-27T23:44:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:48 smithi006 bash[27292]: cluster 2023-08-27T23:44:47.517708+0000 mgr.y (mgr.14901) 3187 : cluster [DBG] pgmap v3194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:48.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:48 smithi006 bash[33918]: cluster 2023-08-27T23:44:47.517708+0000 mgr.y (mgr.14901) 3187 : cluster [DBG] pgmap v3194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:50.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:49 smithi130 bash[51302]: debug 2023-08-27T23:44:49.779+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:49.782738+0000) 2023-08-27T23:44:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:44:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:44:50] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-27T23:44:50.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:50 smithi006 bash[27292]: cluster 2023-08-27T23:44:49.518839+0000 mgr.y (mgr.14901) 3188 : cluster [DBG] pgmap v3195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:50.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:50 smithi006 bash[27292]: audit 2023-08-27T23:44:49.797778+0000 mon.a (mon.0) 2562 : audit [DBG] from='client.? 172.21.15.6:0/1599841790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:50.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:50 smithi006 bash[33918]: cluster 2023-08-27T23:44:49.518839+0000 mgr.y (mgr.14901) 3188 : cluster [DBG] pgmap v3195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:50.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:50 smithi006 bash[33918]: audit 2023-08-27T23:44:49.797778+0000 mon.a (mon.0) 2562 : audit [DBG] from='client.? 172.21.15.6:0/1599841790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:50.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:44:50.548Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:50.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:44:50.550Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:50 smithi130 bash[27579]: cluster 2023-08-27T23:44:49.518839+0000 mgr.y (mgr.14901) 3188 : cluster [DBG] pgmap v3195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:50 smithi130 bash[27579]: audit 2023-08-27T23:44:49.797778+0000 mon.a (mon.0) 2562 : audit [DBG] from='client.? 172.21.15.6:0/1599841790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:51.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:50 smithi130 bash[51302]: debug 2023-08-27T23:44:50.779+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:50.783090+0000) 2023-08-27T23:44:52.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:51 smithi130 bash[51302]: debug 2023-08-27T23:44:51.779+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:51.783324+0000) 2023-08-27T23:44:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:44:52.114Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:44:52.116Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:52.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:52 smithi006 bash[27292]: cluster 2023-08-27T23:44:51.519577+0000 mgr.y (mgr.14901) 3189 : cluster [DBG] pgmap v3196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:52.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:52 smithi006 bash[27292]: audit 2023-08-27T23:44:52.220322+0000 mon.a (mon.0) 2563 : audit [DBG] from='client.? 172.21.15.6:0/3693550948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:52.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:52 smithi006 bash[27292]: audit 2023-08-27T23:44:52.309577+0000 mon.b (mon.1) 421 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:44:52.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:52 smithi006 bash[33918]: cluster 2023-08-27T23:44:51.519577+0000 mgr.y (mgr.14901) 3189 : cluster [DBG] pgmap v3196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:52.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:52 smithi006 bash[33918]: audit 2023-08-27T23:44:52.220322+0000 mon.a (mon.0) 2563 : audit [DBG] from='client.? 172.21.15.6:0/3693550948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:52.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:52 smithi006 bash[33918]: audit 2023-08-27T23:44:52.309577+0000 mon.b (mon.1) 421 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:44:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:52 smithi130 bash[27579]: cluster 2023-08-27T23:44:51.519577+0000 mgr.y (mgr.14901) 3189 : cluster [DBG] pgmap v3196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:52 smithi130 bash[27579]: audit 2023-08-27T23:44:52.220322+0000 mon.a (mon.0) 2563 : audit [DBG] from='client.? 172.21.15.6:0/3693550948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:52 smithi130 bash[27579]: audit 2023-08-27T23:44:52.309577+0000 mon.b (mon.1) 421 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:44:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:52 smithi130 bash[51302]: debug 2023-08-27T23:44:52.779+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:52.783573+0000) 2023-08-27T23:44:54.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:53 smithi130 bash[51302]: debug 2023-08-27T23:44:53.783+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:53.783882+0000) 2023-08-27T23:44:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:54 smithi006 bash[56298]: level=error ts=2023-08-27T23:44:54.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:44:54 smithi006 bash[56298]: level=warn ts=2023-08-27T23:44:54.016Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:44:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:54 smithi006 bash[27292]: cluster 2023-08-27T23:44:53.520265+0000 mgr.y (mgr.14901) 3190 : cluster [DBG] pgmap v3197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:54.988 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:54 smithi006 bash[33918]: cluster 2023-08-27T23:44:53.520265+0000 mgr.y (mgr.14901) 3190 : cluster [DBG] pgmap v3197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:54 smithi130 bash[27579]: cluster 2023-08-27T23:44:53.520265+0000 mgr.y (mgr.14901) 3190 : cluster [DBG] pgmap v3197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:55.802 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:54 smithi130 bash[51302]: debug 2023-08-27T23:44:54.783+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:54.784171+0000) 2023-08-27T23:44:55.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:55 smithi006 bash[33918]: audit 2023-08-27T23:44:54.644626+0000 mon.a (mon.0) 2564 : audit [DBG] from='client.? 172.21.15.6:0/2874880254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:55.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:55 smithi006 bash[27292]: audit 2023-08-27T23:44:54.644626+0000 mon.a (mon.0) 2564 : audit [DBG] from='client.? 172.21.15.6:0/2874880254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:56.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:55 smithi130 bash[27579]: audit 2023-08-27T23:44:54.644626+0000 mon.a (mon.0) 2564 : audit [DBG] from='client.? 172.21.15.6:0/2874880254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:56.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:55 smithi130 bash[51302]: debug 2023-08-27T23:44:55.779+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:55.784524+0000) 2023-08-27T23:44:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:56 smithi006 bash[27292]: cluster 2023-08-27T23:44:55.521486+0000 mgr.y (mgr.14901) 3191 : cluster [DBG] pgmap v3198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:56.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:56 smithi006 bash[33918]: cluster 2023-08-27T23:44:55.521486+0000 mgr.y (mgr.14901) 3191 : cluster [DBG] pgmap v3198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:56 smithi130 bash[27579]: cluster 2023-08-27T23:44:55.521486+0000 mgr.y (mgr.14901) 3191 : cluster [DBG] pgmap v3198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:57.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:56 smithi130 bash[51302]: debug 2023-08-27T23:44:56.778+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:56.784811+0000) 2023-08-27T23:44:57.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:57 smithi006 bash[27292]: audit 2023-08-27T23:44:57.056417+0000 mon.c (mon.2) 1222 : audit [DBG] from='client.? 172.21.15.6:0/4158268892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:57.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:57 smithi006 bash[33918]: audit 2023-08-27T23:44:57.056417+0000 mon.c (mon.2) 1222 : audit [DBG] from='client.? 172.21.15.6:0/4158268892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:58.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:57 smithi130 bash[27579]: audit 2023-08-27T23:44:57.056417+0000 mon.c (mon.2) 1222 : audit [DBG] from='client.? 172.21.15.6:0/4158268892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:58.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:57 smithi130 bash[51302]: debug 2023-08-27T23:44:57.778+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:57.785019+0000) 2023-08-27T23:44:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:58 smithi006 bash[33918]: cluster 2023-08-27T23:44:57.522157+0000 mgr.y (mgr.14901) 3192 : cluster [DBG] pgmap v3199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:58.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:58 smithi006 bash[33918]: audit 2023-08-27T23:44:58.194292+0000 mon.a (mon.0) 2565 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:44:58.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:58 smithi006 bash[27292]: cluster 2023-08-27T23:44:57.522157+0000 mgr.y (mgr.14901) 3192 : cluster [DBG] pgmap v3199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:58.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:58 smithi006 bash[27292]: audit 2023-08-27T23:44:58.194292+0000 mon.a (mon.0) 2565 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:44:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:58 smithi130 bash[27579]: cluster 2023-08-27T23:44:57.522157+0000 mgr.y (mgr.14901) 3192 : cluster [DBG] pgmap v3199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:44:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:58 smithi130 bash[27579]: audit 2023-08-27T23:44:58.194292+0000 mon.a (mon.0) 2565 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:44:59.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:58 smithi130 bash[51302]: debug 2023-08-27T23:44:58.778+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:58.785315+0000) 2023-08-27T23:44:59.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:59 smithi006 bash[27292]: audit 2023-08-27T23:44:58.654163+0000 mon.a (mon.0) 2566 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:44:59.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:59 smithi006 bash[27292]: audit 2023-08-27T23:44:58.946229+0000 mon.b (mon.1) 422 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:44:59.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:59 smithi006 bash[27292]: audit 2023-08-27T23:44:58.948306+0000 mon.b (mon.1) 423 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:44:59.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:59 smithi006 bash[27292]: audit 2023-08-27T23:44:58.955693+0000 mon.a (mon.0) 2567 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:44:59.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:44:59 smithi006 bash[27292]: audit 2023-08-27T23:44:59.462991+0000 mon.a (mon.0) 2568 : audit [DBG] from='client.? 172.21.15.6:0/1665139824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:59.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:59 smithi006 bash[33918]: audit 2023-08-27T23:44:58.654163+0000 mon.a (mon.0) 2566 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:44:59.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:59 smithi006 bash[33918]: audit 2023-08-27T23:44:58.946229+0000 mon.b (mon.1) 422 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:44:59.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:59 smithi006 bash[33918]: audit 2023-08-27T23:44:58.948306+0000 mon.b (mon.1) 423 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:44:59.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:59 smithi006 bash[33918]: audit 2023-08-27T23:44:58.955693+0000 mon.a (mon.0) 2567 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:44:59.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:44:59 smithi006 bash[33918]: audit 2023-08-27T23:44:59.462991+0000 mon.a (mon.0) 2568 : audit [DBG] from='client.? 172.21.15.6:0/1665139824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:00.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:44:59 smithi130 bash[51302]: debug 2023-08-27T23:44:59.778+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:59.785633+0000) 2023-08-27T23:45:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:59 smithi130 bash[27579]: audit 2023-08-27T23:44:58.654163+0000 mon.a (mon.0) 2566 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:45:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:59 smithi130 bash[27579]: audit 2023-08-27T23:44:58.946229+0000 mon.b (mon.1) 422 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:45:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:59 smithi130 bash[27579]: audit 2023-08-27T23:44:58.948306+0000 mon.b (mon.1) 423 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:45:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:59 smithi130 bash[27579]: audit 2023-08-27T23:44:58.955693+0000 mon.a (mon.0) 2567 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:45:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:44:59 smithi130 bash[27579]: audit 2023-08-27T23:44:59.462991+0000 mon.a (mon.0) 2568 : audit [DBG] from='client.? 172.21.15.6:0/1665139824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:00.961 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:45:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:45:00] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-27T23:45:00.962 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:45:00.549Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:00.962 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:45:00.551Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:00 smithi130 bash[27579]: cluster 2023-08-27T23:44:59.523389+0000 mgr.y (mgr.14901) 3193 : cluster [DBG] pgmap v3200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:01.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:00 smithi130 bash[51302]: debug 2023-08-27T23:45:00.782+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:00.785987+0000) 2023-08-27T23:45:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:00 smithi006 bash[33918]: cluster 2023-08-27T23:44:59.523389+0000 mgr.y (mgr.14901) 3193 : cluster [DBG] pgmap v3200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:01.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:00 smithi006 bash[27292]: cluster 2023-08-27T23:44:59.523389+0000 mgr.y (mgr.14901) 3193 : cluster [DBG] pgmap v3200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:02.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:01 smithi130 bash[51302]: debug 2023-08-27T23:45:01.782+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:01.786303+0000) 2023-08-27T23:45:02.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:01 smithi130 bash[27579]: audit 2023-08-27T23:45:01.876011+0000 mon.a (mon.0) 2569 : audit [DBG] from='client.? 172.21.15.6:0/2983404449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:02.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:01 smithi006 bash[27292]: audit 2023-08-27T23:45:01.876011+0000 mon.a (mon.0) 2569 : audit [DBG] from='client.? 172.21.15.6:0/2983404449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:02.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:01 smithi006 bash[33918]: audit 2023-08-27T23:45:01.876011+0000 mon.a (mon.0) 2569 : audit [DBG] from='client.? 172.21.15.6:0/2983404449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:45:02.114Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:45:02.116Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:02 smithi130 bash[27579]: cluster 2023-08-27T23:45:01.524038+0000 mgr.y (mgr.14901) 3194 : cluster [DBG] pgmap v3201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:02 smithi130 bash[51302]: debug 2023-08-27T23:45:02.782+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:02.786654+0000) 2023-08-27T23:45:03.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:02 smithi006 bash[27292]: cluster 2023-08-27T23:45:01.524038+0000 mgr.y (mgr.14901) 3194 : cluster [DBG] pgmap v3201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:03.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:02 smithi006 bash[33918]: cluster 2023-08-27T23:45:01.524038+0000 mgr.y (mgr.14901) 3194 : cluster [DBG] pgmap v3201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:04.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:03 smithi130 bash[51302]: debug 2023-08-27T23:45:03.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:03.787001+0000) 2023-08-27T23:45:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:04 smithi006 bash[56298]: level=error ts=2023-08-27T23:45:04.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:04 smithi006 bash[56298]: level=warn ts=2023-08-27T23:45:04.016Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:04 smithi130 bash[27579]: cluster 2023-08-27T23:45:03.524691+0000 mgr.y (mgr.14901) 3195 : cluster [DBG] pgmap v3202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:04 smithi130 bash[27579]: audit 2023-08-27T23:45:04.298548+0000 mon.a (mon.0) 2570 : audit [DBG] from='client.? 172.21.15.6:0/656752706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:04 smithi130 bash[51302]: debug 2023-08-27T23:45:04.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:04.787330+0000) 2023-08-27T23:45:05.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:04 smithi006 bash[27292]: cluster 2023-08-27T23:45:03.524691+0000 mgr.y (mgr.14901) 3195 : cluster [DBG] pgmap v3202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:05.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:04 smithi006 bash[27292]: audit 2023-08-27T23:45:04.298548+0000 mon.a (mon.0) 2570 : audit [DBG] from='client.? 172.21.15.6:0/656752706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:05.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:04 smithi006 bash[33918]: cluster 2023-08-27T23:45:03.524691+0000 mgr.y (mgr.14901) 3195 : cluster [DBG] pgmap v3202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:05.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:04 smithi006 bash[33918]: audit 2023-08-27T23:45:04.298548+0000 mon.a (mon.0) 2570 : audit [DBG] from='client.? 172.21.15.6:0/656752706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:06.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:06 smithi130 bash[27579]: cluster 2023-08-27T23:45:05.525855+0000 mgr.y (mgr.14901) 3196 : cluster [DBG] pgmap v3203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:06.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:05 smithi130 bash[51302]: debug 2023-08-27T23:45:05.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:05.787697+0000) 2023-08-27T23:45:06.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:06 smithi006 bash[27292]: cluster 2023-08-27T23:45:05.525855+0000 mgr.y (mgr.14901) 3196 : cluster [DBG] pgmap v3203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:06 smithi006 bash[33918]: cluster 2023-08-27T23:45:05.525855+0000 mgr.y (mgr.14901) 3196 : cluster [DBG] pgmap v3203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:07 smithi130 bash[27579]: audit 2023-08-27T23:45:06.712369+0000 mon.c (mon.2) 1223 : audit [DBG] from='client.? 172.21.15.6:0/4024934727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:07.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:06 smithi130 bash[51302]: debug 2023-08-27T23:45:06.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:06.788010+0000) 2023-08-27T23:45:07.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:07 smithi006 bash[27292]: audit 2023-08-27T23:45:06.712369+0000 mon.c (mon.2) 1223 : audit [DBG] from='client.? 172.21.15.6:0/4024934727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:07.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:07 smithi006 bash[33918]: audit 2023-08-27T23:45:06.712369+0000 mon.c (mon.2) 1223 : audit [DBG] from='client.? 172.21.15.6:0/4024934727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:08.042 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:07 smithi130 bash[51302]: debug 2023-08-27T23:45:07.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:07.788335+0000) 2023-08-27T23:45:08.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:08 smithi130 bash[27579]: cluster 2023-08-27T23:45:07.526542+0000 mgr.y (mgr.14901) 3197 : cluster [DBG] pgmap v3204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:08.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:08 smithi006 bash[33918]: cluster 2023-08-27T23:45:07.526542+0000 mgr.y (mgr.14901) 3197 : cluster [DBG] pgmap v3204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:08.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:08 smithi006 bash[27292]: cluster 2023-08-27T23:45:07.526542+0000 mgr.y (mgr.14901) 3197 : cluster [DBG] pgmap v3204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:09.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:08 smithi130 bash[51302]: debug 2023-08-27T23:45:08.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:08.788696+0000) 2023-08-27T23:45:09.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:09 smithi006 bash[33918]: audit 2023-08-27T23:45:09.125788+0000 mon.a (mon.0) 2571 : audit [DBG] from='client.? 172.21.15.6:0/1829071427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:09.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:09 smithi006 bash[27292]: audit 2023-08-27T23:45:09.125788+0000 mon.a (mon.0) 2571 : audit [DBG] from='client.? 172.21.15.6:0/1829071427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:09.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:09 smithi130 bash[27579]: audit 2023-08-27T23:45:09.125788+0000 mon.a (mon.0) 2571 : audit [DBG] from='client.? 172.21.15.6:0/1829071427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:10.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:09 smithi130 bash[51302]: debug 2023-08-27T23:45:09.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:09.788969+0000) 2023-08-27T23:45:10.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:10 smithi006 bash[27292]: cluster 2023-08-27T23:45:09.527665+0000 mgr.y (mgr.14901) 3198 : cluster [DBG] pgmap v3205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:10.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:10 smithi006 bash[33918]: cluster 2023-08-27T23:45:09.527665+0000 mgr.y (mgr.14901) 3198 : cluster [DBG] pgmap v3205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:10.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:10 smithi130 bash[27579]: cluster 2023-08-27T23:45:09.527665+0000 mgr.y (mgr.14901) 3198 : cluster [DBG] pgmap v3205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:45:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:45:10] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-27T23:45:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:45:10.550Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:45:10.552Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:11.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:10 smithi130 bash[51302]: debug 2023-08-27T23:45:10.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:10.789322+0000) 2023-08-27T23:45:11.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:11 smithi130 bash[51302]: debug 2023-08-27T23:45:11.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:11.789643+0000) 2023-08-27T23:45:11.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:11 smithi130 bash[27579]: audit 2023-08-27T23:45:11.540717+0000 mon.c (mon.2) 1224 : audit [DBG] from='client.? 172.21.15.6:0/1883848444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:11.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:11 smithi006 bash[27292]: audit 2023-08-27T23:45:11.540717+0000 mon.c (mon.2) 1224 : audit [DBG] from='client.? 172.21.15.6:0/1883848444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:11.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:11 smithi006 bash[33918]: audit 2023-08-27T23:45:11.540717+0000 mon.c (mon.2) 1224 : audit [DBG] from='client.? 172.21.15.6:0/1883848444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:45:12.115Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:45:12.118Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:12.962 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:12 smithi130 bash[51302]: debug 2023-08-27T23:45:12.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:12.790046+0000) 2023-08-27T23:45:12.962 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:12 smithi130 bash[27579]: cluster 2023-08-27T23:45:11.528271+0000 mgr.y (mgr.14901) 3199 : cluster [DBG] pgmap v3206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:12 smithi006 bash[33918]: cluster 2023-08-27T23:45:11.528271+0000 mgr.y (mgr.14901) 3199 : cluster [DBG] pgmap v3206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:12.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:12 smithi006 bash[27292]: cluster 2023-08-27T23:45:11.528271+0000 mgr.y (mgr.14901) 3199 : cluster [DBG] pgmap v3206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:14.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:13 smithi130 bash[51302]: debug 2023-08-27T23:45:13.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:13.790353+0000) 2023-08-27T23:45:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:14 smithi006 bash[56298]: level=error ts=2023-08-27T23:45:14.015Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:14 smithi006 bash[56298]: level=warn ts=2023-08-27T23:45:14.017Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:14 smithi130 bash[27579]: cluster 2023-08-27T23:45:13.529031+0000 mgr.y (mgr.14901) 3200 : cluster [DBG] pgmap v3207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:14 smithi130 bash[27579]: audit 2023-08-27T23:45:13.953400+0000 mon.a (mon.0) 2572 : audit [DBG] from='client.? 172.21.15.6:0/3495114774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:14 smithi130 bash[51302]: debug 2023-08-27T23:45:14.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:14.790689+0000) 2023-08-27T23:45:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:14 smithi006 bash[27292]: cluster 2023-08-27T23:45:13.529031+0000 mgr.y (mgr.14901) 3200 : cluster [DBG] pgmap v3207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:14.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:14 smithi006 bash[27292]: audit 2023-08-27T23:45:13.953400+0000 mon.a (mon.0) 2572 : audit [DBG] from='client.? 172.21.15.6:0/3495114774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:14.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:14 smithi006 bash[33918]: cluster 2023-08-27T23:45:13.529031+0000 mgr.y (mgr.14901) 3200 : cluster [DBG] pgmap v3207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:14.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:14 smithi006 bash[33918]: audit 2023-08-27T23:45:13.953400+0000 mon.a (mon.0) 2572 : audit [DBG] from='client.? 172.21.15.6:0/3495114774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:16.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:16 smithi130 bash[27579]: cluster 2023-08-27T23:45:15.530174+0000 mgr.y (mgr.14901) 3201 : cluster [DBG] pgmap v3208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:16.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:15 smithi130 bash[51302]: debug 2023-08-27T23:45:15.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:15.791086+0000) 2023-08-27T23:45:16.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:16 smithi006 bash[27292]: cluster 2023-08-27T23:45:15.530174+0000 mgr.y (mgr.14901) 3201 : cluster [DBG] pgmap v3208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:16.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:16 smithi006 bash[33918]: cluster 2023-08-27T23:45:15.530174+0000 mgr.y (mgr.14901) 3201 : cluster [DBG] pgmap v3208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:17 smithi130 bash[27579]: audit 2023-08-27T23:45:16.373335+0000 mon.a (mon.0) 2573 : audit [DBG] from='client.? 172.21.15.6:0/1902261492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:17.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:16 smithi130 bash[51302]: debug 2023-08-27T23:45:16.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:16.791415+0000) 2023-08-27T23:45:17.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:17 smithi006 bash[27292]: audit 2023-08-27T23:45:16.373335+0000 mon.a (mon.0) 2573 : audit [DBG] from='client.? 172.21.15.6:0/1902261492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:17.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:17 smithi006 bash[33918]: audit 2023-08-27T23:45:16.373335+0000 mon.a (mon.0) 2573 : audit [DBG] from='client.? 172.21.15.6:0/1902261492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:18.047 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:17 smithi130 bash[51302]: debug 2023-08-27T23:45:17.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:17.791724+0000) 2023-08-27T23:45:18.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:18 smithi130 bash[27579]: cluster 2023-08-27T23:45:17.530836+0000 mgr.y (mgr.14901) 3202 : cluster [DBG] pgmap v3209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:18.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:18 smithi006 bash[27292]: cluster 2023-08-27T23:45:17.530836+0000 mgr.y (mgr.14901) 3202 : cluster [DBG] pgmap v3209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:18.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:18 smithi006 bash[33918]: cluster 2023-08-27T23:45:17.530836+0000 mgr.y (mgr.14901) 3202 : cluster [DBG] pgmap v3209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:19.050 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:18 smithi130 bash[51302]: debug 2023-08-27T23:45:18.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:18.791999+0000) 2023-08-27T23:45:19.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:19 smithi130 bash[27579]: audit 2023-08-27T23:45:18.797954+0000 mon.c (mon.2) 1225 : audit [DBG] from='client.? 172.21.15.6:0/2202727050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:19.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:19 smithi006 bash[27292]: audit 2023-08-27T23:45:18.797954+0000 mon.c (mon.2) 1225 : audit [DBG] from='client.? 172.21.15.6:0/2202727050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:19.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:19 smithi006 bash[33918]: audit 2023-08-27T23:45:18.797954+0000 mon.c (mon.2) 1225 : audit [DBG] from='client.? 172.21.15.6:0/2202727050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:20.056 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:19 smithi130 bash[51302]: debug 2023-08-27T23:45:19.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:19.792218+0000) 2023-08-27T23:45:20.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:20 smithi130 bash[27579]: cluster 2023-08-27T23:45:19.531978+0000 mgr.y (mgr.14901) 3203 : cluster [DBG] pgmap v3210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:20.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:20 smithi006 bash[33918]: cluster 2023-08-27T23:45:19.531978+0000 mgr.y (mgr.14901) 3203 : cluster [DBG] pgmap v3210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:20.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:20 smithi006 bash[27292]: cluster 2023-08-27T23:45:19.531978+0000 mgr.y (mgr.14901) 3203 : cluster [DBG] pgmap v3210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:45:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:45:20] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:45:20.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:45:20.551Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:20.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:45:20.553Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:21.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:20 smithi130 bash[51302]: debug 2023-08-27T23:45:20.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:20.792527+0000) 2023-08-27T23:45:21.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:21 smithi130 bash[27579]: audit 2023-08-27T23:45:21.215454+0000 mon.a (mon.0) 2574 : audit [DBG] from='client.? 172.21.15.6:0/1433029448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:21.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:21 smithi006 bash[33918]: audit 2023-08-27T23:45:21.215454+0000 mon.a (mon.0) 2574 : audit [DBG] from='client.? 172.21.15.6:0/1433029448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:21.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:21 smithi006 bash[27292]: audit 2023-08-27T23:45:21.215454+0000 mon.a (mon.0) 2574 : audit [DBG] from='client.? 172.21.15.6:0/1433029448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:22.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:21 smithi130 bash[51302]: debug 2023-08-27T23:45:21.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:21.792819+0000) 2023-08-27T23:45:22.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:22 smithi006 bash[27292]: cluster 2023-08-27T23:45:21.532770+0000 mgr.y (mgr.14901) 3204 : cluster [DBG] pgmap v3211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:22.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:22 smithi006 bash[33918]: cluster 2023-08-27T23:45:21.532770+0000 mgr.y (mgr.14901) 3204 : cluster [DBG] pgmap v3211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:22.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:45:22.116Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:22.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:45:22.117Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:22.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:22 smithi130 bash[27579]: cluster 2023-08-27T23:45:21.532770+0000 mgr.y (mgr.14901) 3204 : cluster [DBG] pgmap v3211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:23.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:22 smithi130 bash[51302]: debug 2023-08-27T23:45:22.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:22.793159+0000) 2023-08-27T23:45:24.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:23 smithi130 bash[51302]: debug 2023-08-27T23:45:23.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:23.793421+0000) 2023-08-27T23:45:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:24 smithi006 bash[56298]: level=error ts=2023-08-27T23:45:24.015Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:24 smithi006 bash[56298]: level=warn ts=2023-08-27T23:45:24.017Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:24.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:24 smithi130 bash[51302]: debug 2023-08-27T23:45:24.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:24.793679+0000) 2023-08-27T23:45:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:24 smithi130 bash[27579]: cluster 2023-08-27T23:45:23.533419+0000 mgr.y (mgr.14901) 3205 : cluster [DBG] pgmap v3212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:24 smithi130 bash[27579]: audit 2023-08-27T23:45:23.632257+0000 mon.c (mon.2) 1226 : audit [DBG] from='client.? 172.21.15.6:0/3538155845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:24 smithi006 bash[27292]: cluster 2023-08-27T23:45:23.533419+0000 mgr.y (mgr.14901) 3205 : cluster [DBG] pgmap v3212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:24.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:24 smithi006 bash[27292]: audit 2023-08-27T23:45:23.632257+0000 mon.c (mon.2) 1226 : audit [DBG] from='client.? 172.21.15.6:0/3538155845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:24.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:24 smithi006 bash[33918]: cluster 2023-08-27T23:45:23.533419+0000 mgr.y (mgr.14901) 3205 : cluster [DBG] pgmap v3212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:24.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:24 smithi006 bash[33918]: audit 2023-08-27T23:45:23.632257+0000 mon.c (mon.2) 1226 : audit [DBG] from='client.? 172.21.15.6:0/3538155845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:26.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:26 smithi130 bash[27579]: cluster 2023-08-27T23:45:25.534528+0000 mgr.y (mgr.14901) 3206 : cluster [DBG] pgmap v3213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:26.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:25 smithi130 bash[51302]: debug 2023-08-27T23:45:25.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:25.793943+0000) 2023-08-27T23:45:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:26 smithi006 bash[27292]: cluster 2023-08-27T23:45:25.534528+0000 mgr.y (mgr.14901) 3206 : cluster [DBG] pgmap v3213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:26 smithi006 bash[33918]: cluster 2023-08-27T23:45:25.534528+0000 mgr.y (mgr.14901) 3206 : cluster [DBG] pgmap v3213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:27 smithi130 bash[27579]: audit 2023-08-27T23:45:26.049580+0000 mon.c (mon.2) 1227 : audit [DBG] from='client.? 172.21.15.6:0/1612230820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:27.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:26 smithi130 bash[51302]: debug 2023-08-27T23:45:26.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:26.794238+0000) 2023-08-27T23:45:27.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:27 smithi006 bash[27292]: audit 2023-08-27T23:45:26.049580+0000 mon.c (mon.2) 1227 : audit [DBG] from='client.? 172.21.15.6:0/1612230820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:27.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:27 smithi006 bash[33918]: audit 2023-08-27T23:45:26.049580+0000 mon.c (mon.2) 1227 : audit [DBG] from='client.? 172.21.15.6:0/1612230820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:28.047 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:27 smithi130 bash[51302]: debug 2023-08-27T23:45:27.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:27.794523+0000) 2023-08-27T23:45:28.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:28 smithi130 bash[27579]: cluster 2023-08-27T23:45:27.535209+0000 mgr.y (mgr.14901) 3207 : cluster [DBG] pgmap v3214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:28.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:28 smithi006 bash[27292]: cluster 2023-08-27T23:45:27.535209+0000 mgr.y (mgr.14901) 3207 : cluster [DBG] pgmap v3214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:28.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:28 smithi006 bash[33918]: cluster 2023-08-27T23:45:27.535209+0000 mgr.y (mgr.14901) 3207 : cluster [DBG] pgmap v3214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:29.049 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:28 smithi130 bash[51302]: debug 2023-08-27T23:45:28.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:28.794817+0000) 2023-08-27T23:45:29.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:29 smithi130 bash[27579]: audit 2023-08-27T23:45:28.462984+0000 mon.a (mon.0) 2575 : audit [DBG] from='client.? 172.21.15.6:0/1347376198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:29.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:29 smithi006 bash[27292]: audit 2023-08-27T23:45:28.462984+0000 mon.a (mon.0) 2575 : audit [DBG] from='client.? 172.21.15.6:0/1347376198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:29.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:29 smithi006 bash[33918]: audit 2023-08-27T23:45:28.462984+0000 mon.a (mon.0) 2575 : audit [DBG] from='client.? 172.21.15.6:0/1347376198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:30.060 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:29 smithi130 bash[51302]: debug 2023-08-27T23:45:29.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:29.795128+0000) 2023-08-27T23:45:30.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:30 smithi130 bash[27579]: cluster 2023-08-27T23:45:29.536312+0000 mgr.y (mgr.14901) 3208 : cluster [DBG] pgmap v3215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:30.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:30 smithi006 bash[27292]: cluster 2023-08-27T23:45:29.536312+0000 mgr.y (mgr.14901) 3208 : cluster [DBG] pgmap v3215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:30.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:30 smithi006 bash[33918]: cluster 2023-08-27T23:45:29.536312+0000 mgr.y (mgr.14901) 3208 : cluster [DBG] pgmap v3215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:45:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:45:30] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-27T23:45:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:45:30.552Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:45:30.554Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:31.063 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:30 smithi130 bash[51302]: debug 2023-08-27T23:45:30.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:30.795438+0000) 2023-08-27T23:45:31.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:31 smithi130 bash[27579]: audit 2023-08-27T23:45:30.870959+0000 mon.a (mon.0) 2576 : audit [DBG] from='client.? 172.21.15.6:0/3833251891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:31.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:31 smithi006 bash[27292]: audit 2023-08-27T23:45:30.870959+0000 mon.a (mon.0) 2576 : audit [DBG] from='client.? 172.21.15.6:0/3833251891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:31.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:31 smithi006 bash[33918]: audit 2023-08-27T23:45:30.870959+0000 mon.a (mon.0) 2576 : audit [DBG] from='client.? 172.21.15.6:0/3833251891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:32.074 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:31 smithi130 bash[51302]: debug 2023-08-27T23:45:31.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:31.795744+0000) 2023-08-27T23:45:32.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:32 smithi130 bash[27579]: cluster 2023-08-27T23:45:31.537011+0000 mgr.y (mgr.14901) 3209 : cluster [DBG] pgmap v3216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:32.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:32 smithi006 bash[27292]: cluster 2023-08-27T23:45:31.537011+0000 mgr.y (mgr.14901) 3209 : cluster [DBG] pgmap v3216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:32.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:32 smithi006 bash[33918]: cluster 2023-08-27T23:45:31.537011+0000 mgr.y (mgr.14901) 3209 : cluster [DBG] pgmap v3216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:32.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:45:32.117Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:32.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:45:32.118Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:33.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:32 smithi130 bash[51302]: debug 2023-08-27T23:45:32.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:32.796143+0000) 2023-08-27T23:45:33.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:33 smithi130 bash[27579]: audit 2023-08-27T23:45:33.284495+0000 mon.a (mon.0) 2577 : audit [DBG] from='client.? 172.21.15.6:0/2268261229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:33.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:33 smithi006 bash[33918]: audit 2023-08-27T23:45:33.284495+0000 mon.a (mon.0) 2577 : audit [DBG] from='client.? 172.21.15.6:0/2268261229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:33.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:33 smithi006 bash[27292]: audit 2023-08-27T23:45:33.284495+0000 mon.a (mon.0) 2577 : audit [DBG] from='client.? 172.21.15.6:0/2268261229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:34.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:33 smithi130 bash[51302]: debug 2023-08-27T23:45:33.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:33.796423+0000) 2023-08-27T23:45:34.348 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:34 smithi006 bash[56298]: level=error ts=2023-08-27T23:45:34.016Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:34.348 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:34 smithi006 bash[56298]: level=warn ts=2023-08-27T23:45:34.018Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:34.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:34 smithi130 bash[27579]: cluster 2023-08-27T23:45:33.537619+0000 mgr.y (mgr.14901) 3210 : cluster [DBG] pgmap v3217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:34 smithi006 bash[33918]: cluster 2023-08-27T23:45:33.537619+0000 mgr.y (mgr.14901) 3210 : cluster [DBG] pgmap v3217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:34 smithi006 bash[27292]: cluster 2023-08-27T23:45:33.537619+0000 mgr.y (mgr.14901) 3210 : cluster [DBG] pgmap v3217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:35.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:34 smithi130 bash[51302]: debug 2023-08-27T23:45:34.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:34.796699+0000) 2023-08-27T23:45:36.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:35 smithi130 bash[51302]: debug 2023-08-27T23:45:35.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:35.796975+0000) 2023-08-27T23:45:36.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:36 smithi130 bash[27579]: cluster 2023-08-27T23:45:35.538643+0000 mgr.y (mgr.14901) 3211 : cluster [DBG] pgmap v3218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:36.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:36 smithi130 bash[27579]: audit 2023-08-27T23:45:35.701199+0000 mon.a (mon.0) 2578 : audit [DBG] from='client.? 172.21.15.6:0/1324852881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:36.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:36 smithi006 bash[27292]: cluster 2023-08-27T23:45:35.538643+0000 mgr.y (mgr.14901) 3211 : cluster [DBG] pgmap v3218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:36.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:36 smithi006 bash[27292]: audit 2023-08-27T23:45:35.701199+0000 mon.a (mon.0) 2578 : audit [DBG] from='client.? 172.21.15.6:0/1324852881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:36.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:36 smithi006 bash[33918]: cluster 2023-08-27T23:45:35.538643+0000 mgr.y (mgr.14901) 3211 : cluster [DBG] pgmap v3218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:36.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:36 smithi006 bash[33918]: audit 2023-08-27T23:45:35.701199+0000 mon.a (mon.0) 2578 : audit [DBG] from='client.? 172.21.15.6:0/1324852881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:37.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:36 smithi130 bash[51302]: debug 2023-08-27T23:45:36.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:36.797275+0000) 2023-08-27T23:45:38.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:37 smithi130 bash[51302]: debug 2023-08-27T23:45:37.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:37.797555+0000) 2023-08-27T23:45:38.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:38 smithi130 bash[27579]: cluster 2023-08-27T23:45:37.539291+0000 mgr.y (mgr.14901) 3212 : cluster [DBG] pgmap v3219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:38 smithi130 bash[27579]: audit 2023-08-27T23:45:38.119054+0000 mon.c (mon.2) 1228 : audit [DBG] from='client.? 172.21.15.6:0/3142227896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:38 smithi130 bash[51302]: debug 2023-08-27T23:45:38.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:38.797863+0000) 2023-08-27T23:45:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:38 smithi006 bash[27292]: cluster 2023-08-27T23:45:37.539291+0000 mgr.y (mgr.14901) 3212 : cluster [DBG] pgmap v3219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:38.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:38 smithi006 bash[27292]: audit 2023-08-27T23:45:38.119054+0000 mon.c (mon.2) 1228 : audit [DBG] from='client.? 172.21.15.6:0/3142227896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:38.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:38 smithi006 bash[33918]: cluster 2023-08-27T23:45:37.539291+0000 mgr.y (mgr.14901) 3212 : cluster [DBG] pgmap v3219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:38.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:38 smithi006 bash[33918]: audit 2023-08-27T23:45:38.119054+0000 mon.c (mon.2) 1228 : audit [DBG] from='client.? 172.21.15.6:0/3142227896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:40.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:39 smithi130 bash[51302]: debug 2023-08-27T23:45:39.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:39.798149+0000) 2023-08-27T23:45:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:40 smithi130 bash[27579]: cluster 2023-08-27T23:45:39.540391+0000 mgr.y (mgr.14901) 3213 : cluster [DBG] pgmap v3220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:40 smithi130 bash[27579]: audit 2023-08-27T23:45:40.549530+0000 mon.a (mon.0) 2579 : audit [DBG] from='client.? 172.21.15.6:0/1075929620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:40 smithi130 bash[51302]: debug 2023-08-27T23:45:40.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:40.798497+0000) 2023-08-27T23:45:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:45:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:45:40] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-27T23:45:40.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:40 smithi006 bash[27292]: cluster 2023-08-27T23:45:39.540391+0000 mgr.y (mgr.14901) 3213 : cluster [DBG] pgmap v3220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:40.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:40 smithi006 bash[27292]: audit 2023-08-27T23:45:40.549530+0000 mon.a (mon.0) 2579 : audit [DBG] from='client.? 172.21.15.6:0/1075929620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:40.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:40 smithi006 bash[33918]: cluster 2023-08-27T23:45:39.540391+0000 mgr.y (mgr.14901) 3213 : cluster [DBG] pgmap v3220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:40.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:40 smithi006 bash[33918]: audit 2023-08-27T23:45:40.549530+0000 mon.a (mon.0) 2579 : audit [DBG] from='client.? 172.21.15.6:0/1075929620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:45:40.552Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:40.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:45:40.554Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:42.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:41 smithi130 bash[51302]: debug 2023-08-27T23:45:41.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:41.798768+0000) 2023-08-27T23:45:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:45:42.118Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:45:42.119Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:42 smithi130 bash[27579]: cluster 2023-08-27T23:45:41.541205+0000 mgr.y (mgr.14901) 3214 : cluster [DBG] pgmap v3221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:42 smithi130 bash[51302]: debug 2023-08-27T23:45:42.798+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:42.799040+0000) 2023-08-27T23:45:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:42 smithi006 bash[27292]: cluster 2023-08-27T23:45:41.541205+0000 mgr.y (mgr.14901) 3214 : cluster [DBG] pgmap v3221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:42 smithi006 bash[33918]: cluster 2023-08-27T23:45:41.541205+0000 mgr.y (mgr.14901) 3214 : cluster [DBG] pgmap v3221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:43.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:43 smithi006 bash[27292]: audit 2023-08-27T23:45:42.962009+0000 mon.a (mon.0) 2580 : audit [DBG] from='client.? 172.21.15.6:0/2558760282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:43.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:43 smithi006 bash[33918]: audit 2023-08-27T23:45:42.962009+0000 mon.a (mon.0) 2580 : audit [DBG] from='client.? 172.21.15.6:0/2558760282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:44.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:43 smithi130 bash[51302]: debug 2023-08-27T23:45:43.798+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:43.799290+0000) 2023-08-27T23:45:44.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:43 smithi130 bash[27579]: audit 2023-08-27T23:45:42.962009+0000 mon.a (mon.0) 2580 : audit [DBG] from='client.? 172.21.15.6:0/2558760282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:44 smithi006 bash[56298]: level=error ts=2023-08-27T23:45:44.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:44 smithi006 bash[56298]: level=warn ts=2023-08-27T23:45:44.019Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:44 smithi006 bash[33918]: cluster 2023-08-27T23:45:43.541855+0000 mgr.y (mgr.14901) 3215 : cluster [DBG] pgmap v3222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:44.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:44 smithi006 bash[27292]: cluster 2023-08-27T23:45:43.541855+0000 mgr.y (mgr.14901) 3215 : cluster [DBG] pgmap v3222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:44 smithi130 bash[27579]: cluster 2023-08-27T23:45:43.541855+0000 mgr.y (mgr.14901) 3215 : cluster [DBG] pgmap v3222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:44 smithi130 bash[51302]: debug 2023-08-27T23:45:44.798+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:44.799601+0000) 2023-08-27T23:45:46.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:46 smithi130 bash[27579]: audit 2023-08-27T23:45:45.388680+0000 mon.a (mon.0) 2581 : audit [DBG] from='client.? 172.21.15.6:0/2469974157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:46.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:46 smithi130 bash[27579]: cluster 2023-08-27T23:45:45.542894+0000 mgr.y (mgr.14901) 3216 : cluster [DBG] pgmap v3223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:46.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:45 smithi130 bash[51302]: debug 2023-08-27T23:45:45.798+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:45.799872+0000) 2023-08-27T23:45:46.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:46 smithi006 bash[27292]: audit 2023-08-27T23:45:45.388680+0000 mon.a (mon.0) 2581 : audit [DBG] from='client.? 172.21.15.6:0/2469974157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:46.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:46 smithi006 bash[27292]: cluster 2023-08-27T23:45:45.542894+0000 mgr.y (mgr.14901) 3216 : cluster [DBG] pgmap v3223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:46.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:46 smithi006 bash[33918]: audit 2023-08-27T23:45:45.388680+0000 mon.a (mon.0) 2581 : audit [DBG] from='client.? 172.21.15.6:0/2469974157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:46.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:46 smithi006 bash[33918]: cluster 2023-08-27T23:45:45.542894+0000 mgr.y (mgr.14901) 3216 : cluster [DBG] pgmap v3223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:47.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:46 smithi130 bash[51302]: debug 2023-08-27T23:45:46.798+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:46.800133+0000) 2023-08-27T23:45:48.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:47 smithi130 bash[51302]: debug 2023-08-27T23:45:47.798+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:47.800446+0000) 2023-08-27T23:45:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:48 smithi130 bash[27579]: cluster 2023-08-27T23:45:47.543441+0000 mgr.y (mgr.14901) 3217 : cluster [DBG] pgmap v3224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:48 smithi130 bash[27579]: audit 2023-08-27T23:45:47.807466+0000 mon.a (mon.0) 2582 : audit [DBG] from='client.? 172.21.15.6:0/2775417701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:48 smithi130 bash[51302]: debug 2023-08-27T23:45:48.798+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:48.800751+0000) 2023-08-27T23:45:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:48 smithi006 bash[33918]: cluster 2023-08-27T23:45:47.543441+0000 mgr.y (mgr.14901) 3217 : cluster [DBG] pgmap v3224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:48.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:48 smithi006 bash[33918]: audit 2023-08-27T23:45:47.807466+0000 mon.a (mon.0) 2582 : audit [DBG] from='client.? 172.21.15.6:0/2775417701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:48.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:48 smithi006 bash[27292]: cluster 2023-08-27T23:45:47.543441+0000 mgr.y (mgr.14901) 3217 : cluster [DBG] pgmap v3224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:48.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:48 smithi006 bash[27292]: audit 2023-08-27T23:45:47.807466+0000 mon.a (mon.0) 2582 : audit [DBG] from='client.? 172.21.15.6:0/2775417701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:50.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:49 smithi130 bash[51302]: debug 2023-08-27T23:45:49.798+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:49.801020+0000) 2023-08-27T23:45:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:50 smithi130 bash[27579]: cluster 2023-08-27T23:45:49.544525+0000 mgr.y (mgr.14901) 3218 : cluster [DBG] pgmap v3225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:50 smithi130 bash[27579]: audit 2023-08-27T23:45:50.224084+0000 mon.c (mon.2) 1229 : audit [DBG] from='client.? 172.21.15.6:0/2822348330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:50 smithi130 bash[51302]: debug 2023-08-27T23:45:50.798+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:50.801341+0000) 2023-08-27T23:45:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:45:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:45:50] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-27T23:45:50.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:50 smithi006 bash[27292]: cluster 2023-08-27T23:45:49.544525+0000 mgr.y (mgr.14901) 3218 : cluster [DBG] pgmap v3225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:50.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:50 smithi006 bash[27292]: audit 2023-08-27T23:45:50.224084+0000 mon.c (mon.2) 1229 : audit [DBG] from='client.? 172.21.15.6:0/2822348330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:50.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:50 smithi006 bash[33918]: cluster 2023-08-27T23:45:49.544525+0000 mgr.y (mgr.14901) 3218 : cluster [DBG] pgmap v3225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:50.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:50 smithi006 bash[33918]: audit 2023-08-27T23:45:50.224084+0000 mon.c (mon.2) 1229 : audit [DBG] from='client.? 172.21.15.6:0/2822348330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:45:50.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:45:50.556Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:52.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:51 smithi130 bash[51302]: debug 2023-08-27T23:45:51.798+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:51.801592+0000) 2023-08-27T23:45:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:45:52.119Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:45:52.121Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:52 smithi130 bash[27579]: cluster 2023-08-27T23:45:51.545226+0000 mgr.y (mgr.14901) 3219 : cluster [DBG] pgmap v3226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:52.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:52 smithi130 bash[51302]: debug 2023-08-27T23:45:52.797+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:52.801906+0000) 2023-08-27T23:45:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:52 smithi006 bash[33918]: cluster 2023-08-27T23:45:51.545226+0000 mgr.y (mgr.14901) 3219 : cluster [DBG] pgmap v3226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:52.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:52 smithi006 bash[27292]: cluster 2023-08-27T23:45:51.545226+0000 mgr.y (mgr.14901) 3219 : cluster [DBG] pgmap v3226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:53.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:53 smithi006 bash[33918]: audit 2023-08-27T23:45:52.639346+0000 mon.a (mon.0) 2583 : audit [DBG] from='client.? 172.21.15.6:0/3910067402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:53.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:53 smithi006 bash[27292]: audit 2023-08-27T23:45:52.639346+0000 mon.a (mon.0) 2583 : audit [DBG] from='client.? 172.21.15.6:0/3910067402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:54.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:53 smithi130 bash[27579]: audit 2023-08-27T23:45:52.639346+0000 mon.a (mon.0) 2583 : audit [DBG] from='client.? 172.21.15.6:0/3910067402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:54.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:53 smithi130 bash[51302]: debug 2023-08-27T23:45:53.797+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:53.802198+0000) 2023-08-27T23:45:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:54 smithi006 bash[56298]: level=error ts=2023-08-27T23:45:54.018Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:45:54 smithi006 bash[56298]: level=warn ts=2023-08-27T23:45:54.023Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:45:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:54 smithi006 bash[27292]: cluster 2023-08-27T23:45:53.545972+0000 mgr.y (mgr.14901) 3220 : cluster [DBG] pgmap v3227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:54.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:54 smithi006 bash[33918]: cluster 2023-08-27T23:45:53.545972+0000 mgr.y (mgr.14901) 3220 : cluster [DBG] pgmap v3227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:54 smithi130 bash[27579]: cluster 2023-08-27T23:45:53.545972+0000 mgr.y (mgr.14901) 3220 : cluster [DBG] pgmap v3227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:54 smithi130 bash[51302]: debug 2023-08-27T23:45:54.797+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:54.802483+0000) 2023-08-27T23:45:56.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:56 smithi130 bash[27579]: audit 2023-08-27T23:45:55.050613+0000 mon.a (mon.0) 2584 : audit [DBG] from='client.? 172.21.15.6:0/3834511804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:56.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:56 smithi130 bash[27579]: cluster 2023-08-27T23:45:55.547115+0000 mgr.y (mgr.14901) 3221 : cluster [DBG] pgmap v3228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:56.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:55 smithi130 bash[51302]: debug 2023-08-27T23:45:55.797+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:55.802744+0000) 2023-08-27T23:45:56.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:56 smithi006 bash[33918]: audit 2023-08-27T23:45:55.050613+0000 mon.a (mon.0) 2584 : audit [DBG] from='client.? 172.21.15.6:0/3834511804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:56.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:56 smithi006 bash[33918]: cluster 2023-08-27T23:45:55.547115+0000 mgr.y (mgr.14901) 3221 : cluster [DBG] pgmap v3228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:56.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:56 smithi006 bash[27292]: audit 2023-08-27T23:45:55.050613+0000 mon.a (mon.0) 2584 : audit [DBG] from='client.? 172.21.15.6:0/3834511804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:56.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:56 smithi006 bash[27292]: cluster 2023-08-27T23:45:55.547115+0000 mgr.y (mgr.14901) 3221 : cluster [DBG] pgmap v3228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:57.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:56 smithi130 bash[51302]: debug 2023-08-27T23:45:56.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:56.803015+0000) 2023-08-27T23:45:57.804 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:57 smithi130 bash[27579]: audit 2023-08-27T23:45:57.469442+0000 mon.a (mon.0) 2585 : audit [DBG] from='client.? 172.21.15.6:0/651649587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:57.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:57 smithi006 bash[27292]: audit 2023-08-27T23:45:57.469442+0000 mon.a (mon.0) 2585 : audit [DBG] from='client.? 172.21.15.6:0/651649587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:57.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:57 smithi006 bash[33918]: audit 2023-08-27T23:45:57.469442+0000 mon.a (mon.0) 2585 : audit [DBG] from='client.? 172.21.15.6:0/651649587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:58.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:57 smithi130 bash[51302]: debug 2023-08-27T23:45:57.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:57.803237+0000) 2023-08-27T23:45:58.806 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:58 smithi130 bash[27579]: cluster 2023-08-27T23:45:57.547937+0000 mgr.y (mgr.14901) 3222 : cluster [DBG] pgmap v3229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:58 smithi006 bash[27292]: cluster 2023-08-27T23:45:57.547937+0000 mgr.y (mgr.14901) 3222 : cluster [DBG] pgmap v3229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:58.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:58 smithi006 bash[33918]: cluster 2023-08-27T23:45:57.547937+0000 mgr.y (mgr.14901) 3222 : cluster [DBG] pgmap v3229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:45:59.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:58 smithi130 bash[51302]: debug 2023-08-27T23:45:58.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:58.803536+0000) 2023-08-27T23:45:59.804 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:45:59 smithi130 bash[27579]: audit 2023-08-27T23:45:58.967750+0000 mon.b (mon.1) 424 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:45:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:45:59 smithi006 bash[33918]: audit 2023-08-27T23:45:58.967750+0000 mon.b (mon.1) 424 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:45:59.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:45:59 smithi006 bash[27292]: audit 2023-08-27T23:45:58.967750+0000 mon.b (mon.1) 424 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:46:00.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:45:59 smithi130 bash[51302]: debug 2023-08-27T23:45:59.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:59.803763+0000) 2023-08-27T23:46:00.805 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:00 smithi130 bash[27579]: cluster 2023-08-27T23:45:59.548995+0000 mgr.y (mgr.14901) 3223 : cluster [DBG] pgmap v3230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:00.805 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:00 smithi130 bash[27579]: audit 2023-08-27T23:45:59.886474+0000 mon.c (mon.2) 1230 : audit [DBG] from='client.? 172.21.15.6:0/1691117066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:46:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:46:00] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-27T23:46:00.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:00 smithi006 bash[27292]: cluster 2023-08-27T23:45:59.548995+0000 mgr.y (mgr.14901) 3223 : cluster [DBG] pgmap v3230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:00.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:00 smithi006 bash[27292]: audit 2023-08-27T23:45:59.886474+0000 mon.c (mon.2) 1230 : audit [DBG] from='client.? 172.21.15.6:0/1691117066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:00.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:00 smithi006 bash[33918]: cluster 2023-08-27T23:45:59.548995+0000 mgr.y (mgr.14901) 3223 : cluster [DBG] pgmap v3230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:00.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:00 smithi006 bash[33918]: audit 2023-08-27T23:45:59.886474+0000 mon.c (mon.2) 1230 : audit [DBG] from='client.? 172.21.15.6:0/1691117066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:46:00.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:46:00.556Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:01.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:00 smithi130 bash[51302]: debug 2023-08-27T23:46:00.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:00.804069+0000) 2023-08-27T23:46:02.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:01 smithi130 bash[51302]: debug 2023-08-27T23:46:01.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:01.804339+0000) 2023-08-27T23:46:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:46:02.120Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:46:02.122Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:02.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:02 smithi130 bash[51302]: debug 2023-08-27T23:46:02.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:02.804558+0000) 2023-08-27T23:46:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:02 smithi130 bash[27579]: cluster 2023-08-27T23:46:01.549541+0000 mgr.y (mgr.14901) 3224 : cluster [DBG] pgmap v3231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:02 smithi130 bash[27579]: audit 2023-08-27T23:46:02.352117+0000 mon.c (mon.2) 1231 : audit [DBG] from='client.? 172.21.15.6:0/1668126094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:02 smithi006 bash[27292]: cluster 2023-08-27T23:46:01.549541+0000 mgr.y (mgr.14901) 3224 : cluster [DBG] pgmap v3231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:02 smithi006 bash[27292]: audit 2023-08-27T23:46:02.352117+0000 mon.c (mon.2) 1231 : audit [DBG] from='client.? 172.21.15.6:0/1668126094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:02.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:02 smithi006 bash[33918]: cluster 2023-08-27T23:46:01.549541+0000 mgr.y (mgr.14901) 3224 : cluster [DBG] pgmap v3231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:02.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:02 smithi006 bash[33918]: audit 2023-08-27T23:46:02.352117+0000 mon.c (mon.2) 1231 : audit [DBG] from='client.? 172.21.15.6:0/1668126094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:04.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:03 smithi130 bash[51302]: debug 2023-08-27T23:46:03.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:03.804809+0000) 2023-08-27T23:46:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:04 smithi006 bash[56298]: level=error ts=2023-08-27T23:46:04.018Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:04 smithi006 bash[56298]: level=warn ts=2023-08-27T23:46:04.020Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:04 smithi130 bash[27579]: cluster 2023-08-27T23:46:03.550183+0000 mgr.y (mgr.14901) 3225 : cluster [DBG] pgmap v3232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:04.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:04 smithi130 bash[51302]: debug 2023-08-27T23:46:04.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:04.805009+0000) 2023-08-27T23:46:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:04 smithi006 bash[33918]: cluster 2023-08-27T23:46:03.550183+0000 mgr.y (mgr.14901) 3225 : cluster [DBG] pgmap v3232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:04.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:04 smithi006 bash[27292]: cluster 2023-08-27T23:46:03.550183+0000 mgr.y (mgr.14901) 3225 : cluster [DBG] pgmap v3232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:05.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:05 smithi006 bash[33918]: audit 2023-08-27T23:46:04.812124+0000 mon.a (mon.0) 2586 : audit [DBG] from='client.? 172.21.15.6:0/1843108325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:05.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:05 smithi006 bash[33918]: audit 2023-08-27T23:46:04.855880+0000 mon.a (mon.0) 2587 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:46:05.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:05 smithi006 bash[33918]: audit 2023-08-27T23:46:05.277501+0000 mon.a (mon.0) 2588 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:46:05.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:05 smithi006 bash[33918]: audit 2023-08-27T23:46:05.580020+0000 mon.b (mon.1) 425 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:46:05.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:05 smithi006 bash[33918]: audit 2023-08-27T23:46:05.581936+0000 mon.b (mon.1) 426 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:46:05.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:05 smithi006 bash[33918]: audit 2023-08-27T23:46:05.590323+0000 mon.a (mon.0) 2589 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:46:05.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:05 smithi006 bash[27292]: audit 2023-08-27T23:46:04.812124+0000 mon.a (mon.0) 2586 : audit [DBG] from='client.? 172.21.15.6:0/1843108325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:05.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:05 smithi006 bash[27292]: audit 2023-08-27T23:46:04.855880+0000 mon.a (mon.0) 2587 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:46:05.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:05 smithi006 bash[27292]: audit 2023-08-27T23:46:05.277501+0000 mon.a (mon.0) 2588 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:46:05.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:05 smithi006 bash[27292]: audit 2023-08-27T23:46:05.580020+0000 mon.b (mon.1) 425 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:46:05.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:05 smithi006 bash[27292]: audit 2023-08-27T23:46:05.581936+0000 mon.b (mon.1) 426 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:46:05.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:05 smithi006 bash[27292]: audit 2023-08-27T23:46:05.590323+0000 mon.a (mon.0) 2589 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:46:06.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:05 smithi130 bash[51302]: debug 2023-08-27T23:46:05.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:05.805265+0000) 2023-08-27T23:46:06.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:05 smithi130 bash[27579]: audit 2023-08-27T23:46:04.812124+0000 mon.a (mon.0) 2586 : audit [DBG] from='client.? 172.21.15.6:0/1843108325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:06.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:05 smithi130 bash[27579]: audit 2023-08-27T23:46:04.855880+0000 mon.a (mon.0) 2587 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:46:06.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:05 smithi130 bash[27579]: audit 2023-08-27T23:46:05.277501+0000 mon.a (mon.0) 2588 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:46:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:05 smithi130 bash[27579]: audit 2023-08-27T23:46:05.580020+0000 mon.b (mon.1) 425 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:46:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:05 smithi130 bash[27579]: audit 2023-08-27T23:46:05.581936+0000 mon.b (mon.1) 426 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:46:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:05 smithi130 bash[27579]: audit 2023-08-27T23:46:05.590323+0000 mon.a (mon.0) 2589 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:46:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:06 smithi006 bash[27292]: cluster 2023-08-27T23:46:05.551324+0000 mgr.y (mgr.14901) 3226 : cluster [DBG] pgmap v3233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:06 smithi006 bash[33918]: cluster 2023-08-27T23:46:05.551324+0000 mgr.y (mgr.14901) 3226 : cluster [DBG] pgmap v3233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:06 smithi130 bash[27579]: cluster 2023-08-27T23:46:05.551324+0000 mgr.y (mgr.14901) 3226 : cluster [DBG] pgmap v3233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:07.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:06 smithi130 bash[51302]: debug 2023-08-27T23:46:06.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:06.805563+0000) 2023-08-27T23:46:07.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:07 smithi006 bash[27292]: audit 2023-08-27T23:46:07.223714+0000 mon.c (mon.2) 1232 : audit [DBG] from='client.? 172.21.15.6:0/2129277508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:07.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:07 smithi006 bash[33918]: audit 2023-08-27T23:46:07.223714+0000 mon.c (mon.2) 1232 : audit [DBG] from='client.? 172.21.15.6:0/2129277508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:08.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:07 smithi130 bash[27579]: audit 2023-08-27T23:46:07.223714+0000 mon.c (mon.2) 1232 : audit [DBG] from='client.? 172.21.15.6:0/2129277508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:08.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:07 smithi130 bash[51302]: debug 2023-08-27T23:46:07.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:07.805842+0000) 2023-08-27T23:46:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:08 smithi006 bash[33918]: cluster 2023-08-27T23:46:07.551996+0000 mgr.y (mgr.14901) 3227 : cluster [DBG] pgmap v3234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:08 smithi006 bash[27292]: cluster 2023-08-27T23:46:07.551996+0000 mgr.y (mgr.14901) 3227 : cluster [DBG] pgmap v3234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:08 smithi130 bash[27579]: cluster 2023-08-27T23:46:07.551996+0000 mgr.y (mgr.14901) 3227 : cluster [DBG] pgmap v3234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:09.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:08 smithi130 bash[51302]: debug 2023-08-27T23:46:08.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:08.806137+0000) 2023-08-27T23:46:10.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:09 smithi130 bash[51302]: debug 2023-08-27T23:46:09.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:09.806405+0000) 2023-08-27T23:46:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:46:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:46:10] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-27T23:46:10.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:10 smithi006 bash[33918]: cluster 2023-08-27T23:46:09.552721+0000 mgr.y (mgr.14901) 3228 : cluster [DBG] pgmap v3235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:10.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:10 smithi006 bash[33918]: audit 2023-08-27T23:46:09.640194+0000 mon.c (mon.2) 1233 : audit [DBG] from='client.? 172.21.15.6:0/3711747713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:46:10.555Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:46:10.557Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:10.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:10 smithi006 bash[27292]: cluster 2023-08-27T23:46:09.552721+0000 mgr.y (mgr.14901) 3228 : cluster [DBG] pgmap v3235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:10.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:10 smithi006 bash[27292]: audit 2023-08-27T23:46:09.640194+0000 mon.c (mon.2) 1233 : audit [DBG] from='client.? 172.21.15.6:0/3711747713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:10 smithi130 bash[27579]: cluster 2023-08-27T23:46:09.552721+0000 mgr.y (mgr.14901) 3228 : cluster [DBG] pgmap v3235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:10 smithi130 bash[27579]: audit 2023-08-27T23:46:09.640194+0000 mon.c (mon.2) 1233 : audit [DBG] from='client.? 172.21.15.6:0/3711747713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:11.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:10 smithi130 bash[51302]: debug 2023-08-27T23:46:10.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:10.806674+0000) 2023-08-27T23:46:12.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:11 smithi130 bash[51302]: debug 2023-08-27T23:46:11.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:11.806945+0000) 2023-08-27T23:46:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:46:12.120Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:46:12.122Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:12 smithi006 bash[27292]: cluster 2023-08-27T23:46:11.553335+0000 mgr.y (mgr.14901) 3229 : cluster [DBG] pgmap v3236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:12 smithi006 bash[27292]: audit 2023-08-27T23:46:12.053820+0000 mon.a (mon.0) 2590 : audit [DBG] from='client.? 172.21.15.6:0/4273734184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:12.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:12 smithi006 bash[33918]: cluster 2023-08-27T23:46:11.553335+0000 mgr.y (mgr.14901) 3229 : cluster [DBG] pgmap v3236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:12.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:12 smithi006 bash[33918]: audit 2023-08-27T23:46:12.053820+0000 mon.a (mon.0) 2590 : audit [DBG] from='client.? 172.21.15.6:0/4273734184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:13.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:12 smithi130 bash[27579]: cluster 2023-08-27T23:46:11.553335+0000 mgr.y (mgr.14901) 3229 : cluster [DBG] pgmap v3236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:12 smithi130 bash[27579]: audit 2023-08-27T23:46:12.053820+0000 mon.a (mon.0) 2590 : audit [DBG] from='client.? 172.21.15.6:0/4273734184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:13.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:12 smithi130 bash[51302]: debug 2023-08-27T23:46:12.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:12.807236+0000) 2023-08-27T23:46:14.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:13 smithi130 bash[51302]: debug 2023-08-27T23:46:13.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:13.807524+0000) 2023-08-27T23:46:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:14 smithi006 bash[56298]: level=error ts=2023-08-27T23:46:14.019Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:14 smithi006 bash[56298]: level=warn ts=2023-08-27T23:46:14.021Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:14 smithi006 bash[33918]: cluster 2023-08-27T23:46:13.553927+0000 mgr.y (mgr.14901) 3230 : cluster [DBG] pgmap v3237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:14 smithi006 bash[33918]: audit 2023-08-27T23:46:14.473478+0000 mon.a (mon.0) 2591 : audit [DBG] from='client.? 172.21.15.6:0/4278057734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:14.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:14 smithi006 bash[27292]: cluster 2023-08-27T23:46:13.553927+0000 mgr.y (mgr.14901) 3230 : cluster [DBG] pgmap v3237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:14.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:14 smithi006 bash[27292]: audit 2023-08-27T23:46:14.473478+0000 mon.a (mon.0) 2591 : audit [DBG] from='client.? 172.21.15.6:0/4278057734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:15.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:14 smithi130 bash[51302]: debug 2023-08-27T23:46:14.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:14.807833+0000) 2023-08-27T23:46:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:14 smithi130 bash[27579]: cluster 2023-08-27T23:46:13.553927+0000 mgr.y (mgr.14901) 3230 : cluster [DBG] pgmap v3237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:14 smithi130 bash[27579]: audit 2023-08-27T23:46:14.473478+0000 mon.a (mon.0) 2591 : audit [DBG] from='client.? 172.21.15.6:0/4278057734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:16.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:16 smithi130 bash[27579]: cluster 2023-08-27T23:46:15.555031+0000 mgr.y (mgr.14901) 3231 : cluster [DBG] pgmap v3238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:16.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:15 smithi130 bash[51302]: debug 2023-08-27T23:46:15.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:15.808200+0000) 2023-08-27T23:46:16.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:16 smithi006 bash[27292]: cluster 2023-08-27T23:46:15.555031+0000 mgr.y (mgr.14901) 3231 : cluster [DBG] pgmap v3238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:16.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:16 smithi006 bash[33918]: cluster 2023-08-27T23:46:15.555031+0000 mgr.y (mgr.14901) 3231 : cluster [DBG] pgmap v3238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:17 smithi130 bash[27579]: audit 2023-08-27T23:46:16.882574+0000 mon.a (mon.0) 2592 : audit [DBG] from='client.? 172.21.15.6:0/3445852018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:17.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:16 smithi130 bash[51302]: debug 2023-08-27T23:46:16.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:16.808464+0000) 2023-08-27T23:46:17.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:17 smithi006 bash[33918]: audit 2023-08-27T23:46:16.882574+0000 mon.a (mon.0) 2592 : audit [DBG] from='client.? 172.21.15.6:0/3445852018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:17.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:17 smithi006 bash[27292]: audit 2023-08-27T23:46:16.882574+0000 mon.a (mon.0) 2592 : audit [DBG] from='client.? 172.21.15.6:0/3445852018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:18.062 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:17 smithi130 bash[51302]: debug 2023-08-27T23:46:17.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:17.808760+0000) 2023-08-27T23:46:18.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:18 smithi130 bash[27579]: cluster 2023-08-27T23:46:17.555730+0000 mgr.y (mgr.14901) 3232 : cluster [DBG] pgmap v3239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:18.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:18 smithi006 bash[27292]: cluster 2023-08-27T23:46:17.555730+0000 mgr.y (mgr.14901) 3232 : cluster [DBG] pgmap v3239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:18.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:18 smithi006 bash[33918]: cluster 2023-08-27T23:46:17.555730+0000 mgr.y (mgr.14901) 3232 : cluster [DBG] pgmap v3239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:19.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:18 smithi130 bash[51302]: debug 2023-08-27T23:46:18.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:18.809044+0000) 2023-08-27T23:46:19.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:19 smithi130 bash[27579]: audit 2023-08-27T23:46:19.295207+0000 mon.a (mon.0) 2593 : audit [DBG] from='client.? 172.21.15.6:0/1496600387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:19.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:19 smithi006 bash[27292]: audit 2023-08-27T23:46:19.295207+0000 mon.a (mon.0) 2593 : audit [DBG] from='client.? 172.21.15.6:0/1496600387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:19.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:19 smithi006 bash[33918]: audit 2023-08-27T23:46:19.295207+0000 mon.a (mon.0) 2593 : audit [DBG] from='client.? 172.21.15.6:0/1496600387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:20.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:19 smithi130 bash[51302]: debug 2023-08-27T23:46:19.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:19.809315+0000) 2023-08-27T23:46:20.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:20 smithi130 bash[27579]: cluster 2023-08-27T23:46:19.556813+0000 mgr.y (mgr.14901) 3233 : cluster [DBG] pgmap v3240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:20 smithi006 bash[33918]: cluster 2023-08-27T23:46:19.556813+0000 mgr.y (mgr.14901) 3233 : cluster [DBG] pgmap v3240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:20 smithi006 bash[27292]: cluster 2023-08-27T23:46:19.556813+0000 mgr.y (mgr.14901) 3233 : cluster [DBG] pgmap v3240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:20.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:46:20.555Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:20.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:46:20.557Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:21.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:20 smithi130 bash[51302]: debug 2023-08-27T23:46:20.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:20.809596+0000) 2023-08-27T23:46:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:46:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:46:20] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-27T23:46:22.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:21 smithi130 bash[51302]: debug 2023-08-27T23:46:21.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:21.809882+0000) 2023-08-27T23:46:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:46:22.121Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:46:22.122Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:22 smithi006 bash[27292]: cluster 2023-08-27T23:46:21.557453+0000 mgr.y (mgr.14901) 3234 : cluster [DBG] pgmap v3241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:22 smithi006 bash[27292]: audit 2023-08-27T23:46:21.705521+0000 mon.a (mon.0) 2594 : audit [DBG] from='client.? 172.21.15.6:0/2514489829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:22.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:22 smithi006 bash[33918]: cluster 2023-08-27T23:46:21.557453+0000 mgr.y (mgr.14901) 3234 : cluster [DBG] pgmap v3241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:22.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:22 smithi006 bash[33918]: audit 2023-08-27T23:46:21.705521+0000 mon.a (mon.0) 2594 : audit [DBG] from='client.? 172.21.15.6:0/2514489829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:23.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:22 smithi130 bash[27579]: cluster 2023-08-27T23:46:21.557453+0000 mgr.y (mgr.14901) 3234 : cluster [DBG] pgmap v3241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:22 smithi130 bash[27579]: audit 2023-08-27T23:46:21.705521+0000 mon.a (mon.0) 2594 : audit [DBG] from='client.? 172.21.15.6:0/2514489829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:23.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:22 smithi130 bash[51302]: debug 2023-08-27T23:46:22.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:22.810179+0000) 2023-08-27T23:46:24.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:23 smithi130 bash[51302]: debug 2023-08-27T23:46:23.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:23.810456+0000) 2023-08-27T23:46:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:24 smithi006 bash[56298]: level=error ts=2023-08-27T23:46:24.020Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:24 smithi006 bash[56298]: level=warn ts=2023-08-27T23:46:24.022Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:24 smithi006 bash[27292]: cluster 2023-08-27T23:46:23.558098+0000 mgr.y (mgr.14901) 3235 : cluster [DBG] pgmap v3242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:24 smithi006 bash[27292]: audit 2023-08-27T23:46:24.127892+0000 mon.c (mon.2) 1234 : audit [DBG] from='client.? 172.21.15.6:0/739449724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:24.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:24 smithi006 bash[33918]: cluster 2023-08-27T23:46:23.558098+0000 mgr.y (mgr.14901) 3235 : cluster [DBG] pgmap v3242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:24.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:24 smithi006 bash[33918]: audit 2023-08-27T23:46:24.127892+0000 mon.c (mon.2) 1234 : audit [DBG] from='client.? 172.21.15.6:0/739449724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:24 smithi130 bash[27579]: cluster 2023-08-27T23:46:23.558098+0000 mgr.y (mgr.14901) 3235 : cluster [DBG] pgmap v3242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:24 smithi130 bash[27579]: audit 2023-08-27T23:46:24.127892+0000 mon.c (mon.2) 1234 : audit [DBG] from='client.? 172.21.15.6:0/739449724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:24 smithi130 bash[51302]: debug 2023-08-27T23:46:24.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:24.810721+0000) 2023-08-27T23:46:26.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:26 smithi130 bash[27579]: cluster 2023-08-27T23:46:25.559218+0000 mgr.y (mgr.14901) 3236 : cluster [DBG] pgmap v3243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:26.164 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:25 smithi130 bash[51302]: debug 2023-08-27T23:46:25.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:25.810973+0000) 2023-08-27T23:46:26.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:26 smithi006 bash[33918]: cluster 2023-08-27T23:46:25.559218+0000 mgr.y (mgr.14901) 3236 : cluster [DBG] pgmap v3243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:26 smithi006 bash[27292]: cluster 2023-08-27T23:46:25.559218+0000 mgr.y (mgr.14901) 3236 : cluster [DBG] pgmap v3243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:27 smithi130 bash[27579]: audit 2023-08-27T23:46:26.542216+0000 mon.a (mon.0) 2595 : audit [DBG] from='client.? 172.21.15.6:0/1966234939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:27.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:26 smithi130 bash[51302]: debug 2023-08-27T23:46:26.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:26.811226+0000) 2023-08-27T23:46:27.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:27 smithi006 bash[27292]: audit 2023-08-27T23:46:26.542216+0000 mon.a (mon.0) 2595 : audit [DBG] from='client.? 172.21.15.6:0/1966234939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:27.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:27 smithi006 bash[33918]: audit 2023-08-27T23:46:26.542216+0000 mon.a (mon.0) 2595 : audit [DBG] from='client.? 172.21.15.6:0/1966234939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:28.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:28 smithi130 bash[27579]: cluster 2023-08-27T23:46:27.559872+0000 mgr.y (mgr.14901) 3237 : cluster [DBG] pgmap v3244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:28.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:27 smithi130 bash[51302]: debug 2023-08-27T23:46:27.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:27.811485+0000) 2023-08-27T23:46:28.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:28 smithi006 bash[27292]: cluster 2023-08-27T23:46:27.559872+0000 mgr.y (mgr.14901) 3237 : cluster [DBG] pgmap v3244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:28.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:28 smithi006 bash[33918]: cluster 2023-08-27T23:46:27.559872+0000 mgr.y (mgr.14901) 3237 : cluster [DBG] pgmap v3244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:29.065 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:28 smithi130 bash[51302]: debug 2023-08-27T23:46:28.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:28.811782+0000) 2023-08-27T23:46:29.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:29 smithi130 bash[27579]: audit 2023-08-27T23:46:28.957009+0000 mon.c (mon.2) 1235 : audit [DBG] from='client.? 172.21.15.6:0/4082184568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:29.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:29 smithi006 bash[27292]: audit 2023-08-27T23:46:28.957009+0000 mon.c (mon.2) 1235 : audit [DBG] from='client.? 172.21.15.6:0/4082184568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:29.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:29 smithi006 bash[33918]: audit 2023-08-27T23:46:28.957009+0000 mon.c (mon.2) 1235 : audit [DBG] from='client.? 172.21.15.6:0/4082184568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:30.075 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:29 smithi130 bash[51302]: debug 2023-08-27T23:46:29.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:29.812049+0000) 2023-08-27T23:46:30.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:30 smithi130 bash[27579]: cluster 2023-08-27T23:46:29.561024+0000 mgr.y (mgr.14901) 3238 : cluster [DBG] pgmap v3245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:30.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:30 smithi006 bash[33918]: cluster 2023-08-27T23:46:29.561024+0000 mgr.y (mgr.14901) 3238 : cluster [DBG] pgmap v3245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:30.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:30 smithi006 bash[27292]: cluster 2023-08-27T23:46:29.561024+0000 mgr.y (mgr.14901) 3238 : cluster [DBG] pgmap v3245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:46:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:46:30] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-27T23:46:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:46:30.555Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:46:30.557Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:31.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:30 smithi130 bash[51302]: debug 2023-08-27T23:46:30.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:30.812333+0000) 2023-08-27T23:46:31.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:31 smithi006 bash[27292]: audit 2023-08-27T23:46:31.373418+0000 mon.c (mon.2) 1236 : audit [DBG] from='client.? 172.21.15.6:0/1879076583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:31.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:31 smithi006 bash[33918]: audit 2023-08-27T23:46:31.373418+0000 mon.c (mon.2) 1236 : audit [DBG] from='client.? 172.21.15.6:0/1879076583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:31.813 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:31 smithi130 bash[27579]: audit 2023-08-27T23:46:31.373418+0000 mon.c (mon.2) 1236 : audit [DBG] from='client.? 172.21.15.6:0/1879076583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:32.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:31 smithi130 bash[51302]: debug 2023-08-27T23:46:31.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:31.812625+0000) 2023-08-27T23:46:32.438 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:46:32.122Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:32.438 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:46:32.124Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:32.439 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:32 smithi006 bash[27292]: cluster 2023-08-27T23:46:31.561791+0000 mgr.y (mgr.14901) 3239 : cluster [DBG] pgmap v3246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:32.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:32 smithi006 bash[33918]: cluster 2023-08-27T23:46:31.561791+0000 mgr.y (mgr.14901) 3239 : cluster [DBG] pgmap v3246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:32.814 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:32 smithi130 bash[27579]: cluster 2023-08-27T23:46:31.561791+0000 mgr.y (mgr.14901) 3239 : cluster [DBG] pgmap v3246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:33.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:32 smithi130 bash[51302]: debug 2023-08-27T23:46:32.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:32.812932+0000) 2023-08-27T23:46:34.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:33 smithi130 bash[51302]: debug 2023-08-27T23:46:33.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:33.813241+0000) 2023-08-27T23:46:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:34 smithi006 bash[56298]: level=error ts=2023-08-27T23:46:34.021Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:34 smithi006 bash[56298]: level=warn ts=2023-08-27T23:46:34.023Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:34.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:34 smithi006 bash[27292]: cluster 2023-08-27T23:46:33.562481+0000 mgr.y (mgr.14901) 3240 : cluster [DBG] pgmap v3247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:34.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:34 smithi006 bash[27292]: audit 2023-08-27T23:46:33.794828+0000 mon.a (mon.0) 2596 : audit [DBG] from='client.? 172.21.15.6:0/919123825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:34.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:34 smithi006 bash[33918]: cluster 2023-08-27T23:46:33.562481+0000 mgr.y (mgr.14901) 3240 : cluster [DBG] pgmap v3247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:34.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:34 smithi006 bash[33918]: audit 2023-08-27T23:46:33.794828+0000 mon.a (mon.0) 2596 : audit [DBG] from='client.? 172.21.15.6:0/919123825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:34 smithi130 bash[27579]: cluster 2023-08-27T23:46:33.562481+0000 mgr.y (mgr.14901) 3240 : cluster [DBG] pgmap v3247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:35.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:34 smithi130 bash[27579]: audit 2023-08-27T23:46:33.794828+0000 mon.a (mon.0) 2596 : audit [DBG] from='client.? 172.21.15.6:0/919123825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:35.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:34 smithi130 bash[51302]: debug 2023-08-27T23:46:34.813+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:34.813523+0000) 2023-08-27T23:46:36.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:36 smithi130 bash[27579]: cluster 2023-08-27T23:46:35.563612+0000 mgr.y (mgr.14901) 3241 : cluster [DBG] pgmap v3248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:36.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:35 smithi130 bash[51302]: debug 2023-08-27T23:46:35.813+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:35.813804+0000) 2023-08-27T23:46:36.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:36 smithi006 bash[33918]: cluster 2023-08-27T23:46:35.563612+0000 mgr.y (mgr.14901) 3241 : cluster [DBG] pgmap v3248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:36.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:36 smithi006 bash[27292]: cluster 2023-08-27T23:46:35.563612+0000 mgr.y (mgr.14901) 3241 : cluster [DBG] pgmap v3248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:37 smithi130 bash[27579]: audit 2023-08-27T23:46:36.208262+0000 mon.a (mon.0) 2597 : audit [DBG] from='client.? 172.21.15.6:0/3616124075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:37.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:36 smithi130 bash[51302]: debug 2023-08-27T23:46:36.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:36.814091+0000) 2023-08-27T23:46:37.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:37 smithi006 bash[33918]: audit 2023-08-27T23:46:36.208262+0000 mon.a (mon.0) 2597 : audit [DBG] from='client.? 172.21.15.6:0/3616124075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:37.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:37 smithi006 bash[27292]: audit 2023-08-27T23:46:36.208262+0000 mon.a (mon.0) 2597 : audit [DBG] from='client.? 172.21.15.6:0/3616124075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:38.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:37 smithi130 bash[51302]: debug 2023-08-27T23:46:37.813+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:37.814401+0000) 2023-08-27T23:46:38.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:38 smithi130 bash[27579]: cluster 2023-08-27T23:46:37.564287+0000 mgr.y (mgr.14901) 3242 : cluster [DBG] pgmap v3249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:38.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:38 smithi006 bash[27292]: cluster 2023-08-27T23:46:37.564287+0000 mgr.y (mgr.14901) 3242 : cluster [DBG] pgmap v3249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:38.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:38 smithi006 bash[33918]: cluster 2023-08-27T23:46:37.564287+0000 mgr.y (mgr.14901) 3242 : cluster [DBG] pgmap v3249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:39 smithi130 bash[27579]: audit 2023-08-27T23:46:38.616496+0000 mon.a (mon.0) 2598 : audit [DBG] from='client.? 172.21.15.6:0/2236523407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:39.197 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:38 smithi130 bash[51302]: debug 2023-08-27T23:46:38.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:38.814760+0000) 2023-08-27T23:46:39.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:39 smithi006 bash[33918]: audit 2023-08-27T23:46:38.616496+0000 mon.a (mon.0) 2598 : audit [DBG] from='client.? 172.21.15.6:0/2236523407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:39 smithi006 bash[27292]: audit 2023-08-27T23:46:38.616496+0000 mon.a (mon.0) 2598 : audit [DBG] from='client.? 172.21.15.6:0/2236523407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:40.071 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:39 smithi130 bash[51302]: debug 2023-08-27T23:46:39.813+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:39.815048+0000) 2023-08-27T23:46:40.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:40 smithi130 bash[27579]: cluster 2023-08-27T23:46:39.565435+0000 mgr.y (mgr.14901) 3243 : cluster [DBG] pgmap v3250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:40.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:40 smithi006 bash[33918]: cluster 2023-08-27T23:46:39.565435+0000 mgr.y (mgr.14901) 3243 : cluster [DBG] pgmap v3250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:40.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:40 smithi006 bash[27292]: cluster 2023-08-27T23:46:39.565435+0000 mgr.y (mgr.14901) 3243 : cluster [DBG] pgmap v3250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:46:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:46:40] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-27T23:46:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:46:40.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:46:40.559Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:41.074 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:40 smithi130 bash[51302]: debug 2023-08-27T23:46:40.813+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:40.815302+0000) 2023-08-27T23:46:41.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:41 smithi130 bash[27579]: audit 2023-08-27T23:46:41.029351+0000 mon.c (mon.2) 1237 : audit [DBG] from='client.? 172.21.15.6:0/1206525376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:41.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:41 smithi006 bash[33918]: audit 2023-08-27T23:46:41.029351+0000 mon.c (mon.2) 1237 : audit [DBG] from='client.? 172.21.15.6:0/1206525376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:41.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:41 smithi006 bash[27292]: audit 2023-08-27T23:46:41.029351+0000 mon.c (mon.2) 1237 : audit [DBG] from='client.? 172.21.15.6:0/1206525376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:42.081 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:41 smithi130 bash[51302]: debug 2023-08-27T23:46:41.813+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:41.815589+0000) 2023-08-27T23:46:42.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:42 smithi130 bash[27579]: cluster 2023-08-27T23:46:41.566075+0000 mgr.y (mgr.14901) 3244 : cluster [DBG] pgmap v3251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:42.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:42 smithi006 bash[27292]: cluster 2023-08-27T23:46:41.566075+0000 mgr.y (mgr.14901) 3244 : cluster [DBG] pgmap v3251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:42.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:42 smithi006 bash[33918]: cluster 2023-08-27T23:46:41.566075+0000 mgr.y (mgr.14901) 3244 : cluster [DBG] pgmap v3251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:42.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:46:42.123Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:42.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:46:42.124Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:43.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:42 smithi130 bash[51302]: debug 2023-08-27T23:46:42.813+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:42.815887+0000) 2023-08-27T23:46:43.817 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:43 smithi130 bash[27579]: audit 2023-08-27T23:46:43.443253+0000 mon.c (mon.2) 1238 : audit [DBG] from='client.? 172.21.15.6:0/93933038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:43.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:43 smithi006 bash[33918]: audit 2023-08-27T23:46:43.443253+0000 mon.c (mon.2) 1238 : audit [DBG] from='client.? 172.21.15.6:0/93933038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:43.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:43 smithi006 bash[27292]: audit 2023-08-27T23:46:43.443253+0000 mon.c (mon.2) 1238 : audit [DBG] from='client.? 172.21.15.6:0/93933038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:44.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:43 smithi130 bash[51302]: debug 2023-08-27T23:46:43.813+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:43.816108+0000) 2023-08-27T23:46:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:44 smithi006 bash[56298]: level=error ts=2023-08-27T23:46:44.022Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:44 smithi006 bash[56298]: level=warn ts=2023-08-27T23:46:44.024Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:44.817 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:44 smithi130 bash[27579]: cluster 2023-08-27T23:46:43.566723+0000 mgr.y (mgr.14901) 3245 : cluster [DBG] pgmap v3252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:44 smithi006 bash[33918]: cluster 2023-08-27T23:46:43.566723+0000 mgr.y (mgr.14901) 3245 : cluster [DBG] pgmap v3252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:44.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:44 smithi006 bash[27292]: cluster 2023-08-27T23:46:43.566723+0000 mgr.y (mgr.14901) 3245 : cluster [DBG] pgmap v3252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:45.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:44 smithi130 bash[51302]: debug 2023-08-27T23:46:44.813+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:44.816372+0000) 2023-08-27T23:46:46.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:45 smithi130 bash[51302]: debug 2023-08-27T23:46:45.813+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:45.816605+0000) 2023-08-27T23:46:46.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:46 smithi130 bash[27579]: cluster 2023-08-27T23:46:45.567342+0000 mgr.y (mgr.14901) 3246 : cluster [DBG] pgmap v3253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:46.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:46 smithi130 bash[27579]: audit 2023-08-27T23:46:45.862543+0000 mon.c (mon.2) 1239 : audit [DBG] from='client.? 172.21.15.6:0/1537986287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:46.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:46 smithi006 bash[27292]: cluster 2023-08-27T23:46:45.567342+0000 mgr.y (mgr.14901) 3246 : cluster [DBG] pgmap v3253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:46.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:46 smithi006 bash[27292]: audit 2023-08-27T23:46:45.862543+0000 mon.c (mon.2) 1239 : audit [DBG] from='client.? 172.21.15.6:0/1537986287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:46.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:46 smithi006 bash[33918]: cluster 2023-08-27T23:46:45.567342+0000 mgr.y (mgr.14901) 3246 : cluster [DBG] pgmap v3253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:46.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:46 smithi006 bash[33918]: audit 2023-08-27T23:46:45.862543+0000 mon.c (mon.2) 1239 : audit [DBG] from='client.? 172.21.15.6:0/1537986287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:47.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:46 smithi130 bash[51302]: debug 2023-08-27T23:46:46.813+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:46.816884+0000) 2023-08-27T23:46:48.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:47 smithi130 bash[51302]: debug 2023-08-27T23:46:47.813+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:47.817164+0000) 2023-08-27T23:46:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:48 smithi006 bash[27292]: cluster 2023-08-27T23:46:47.567956+0000 mgr.y (mgr.14901) 3247 : cluster [DBG] pgmap v3254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:48.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:48 smithi006 bash[27292]: audit 2023-08-27T23:46:48.275003+0000 mon.c (mon.2) 1240 : audit [DBG] from='client.? 172.21.15.6:0/2096015502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:48.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:48 smithi006 bash[33918]: cluster 2023-08-27T23:46:47.567956+0000 mgr.y (mgr.14901) 3247 : cluster [DBG] pgmap v3254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:48.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:48 smithi006 bash[33918]: audit 2023-08-27T23:46:48.275003+0000 mon.c (mon.2) 1240 : audit [DBG] from='client.? 172.21.15.6:0/2096015502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:49.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:48 smithi130 bash[51302]: debug 2023-08-27T23:46:48.812+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:48.817446+0000) 2023-08-27T23:46:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:48 smithi130 bash[27579]: cluster 2023-08-27T23:46:47.567956+0000 mgr.y (mgr.14901) 3247 : cluster [DBG] pgmap v3254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:48 smithi130 bash[27579]: audit 2023-08-27T23:46:48.275003+0000 mon.c (mon.2) 1240 : audit [DBG] from='client.? 172.21.15.6:0/2096015502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:50.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:49 smithi130 bash[51302]: debug 2023-08-27T23:46:49.812+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:49.817736+0000) 2023-08-27T23:46:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:46:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:46:50] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-27T23:46:50.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:50 smithi006 bash[33918]: cluster 2023-08-27T23:46:49.569079+0000 mgr.y (mgr.14901) 3248 : cluster [DBG] pgmap v3255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:50.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:50 smithi006 bash[27292]: cluster 2023-08-27T23:46:49.569079+0000 mgr.y (mgr.14901) 3248 : cluster [DBG] pgmap v3255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:46:50.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:46:50.559Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:51.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:50 smithi130 bash[51302]: debug 2023-08-27T23:46:50.812+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:50.818039+0000) 2023-08-27T23:46:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:50 smithi130 bash[27579]: cluster 2023-08-27T23:46:49.569079+0000 mgr.y (mgr.14901) 3248 : cluster [DBG] pgmap v3255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:51.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:51 smithi006 bash[33918]: audit 2023-08-27T23:46:50.696934+0000 mon.c (mon.2) 1241 : audit [DBG] from='client.? 172.21.15.6:0/3031571062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:51.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:51 smithi006 bash[27292]: audit 2023-08-27T23:46:50.696934+0000 mon.c (mon.2) 1241 : audit [DBG] from='client.? 172.21.15.6:0/3031571062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:52.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:51 smithi130 bash[27579]: audit 2023-08-27T23:46:50.696934+0000 mon.c (mon.2) 1241 : audit [DBG] from='client.? 172.21.15.6:0/3031571062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:52.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:51 smithi130 bash[51302]: debug 2023-08-27T23:46:51.812+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:51.818314+0000) 2023-08-27T23:46:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:46:52.124Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:46:52.125Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:52 smithi006 bash[33918]: cluster 2023-08-27T23:46:51.569711+0000 mgr.y (mgr.14901) 3249 : cluster [DBG] pgmap v3256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:52 smithi006 bash[27292]: cluster 2023-08-27T23:46:51.569711+0000 mgr.y (mgr.14901) 3249 : cluster [DBG] pgmap v3256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:53.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:52 smithi130 bash[27579]: cluster 2023-08-27T23:46:51.569711+0000 mgr.y (mgr.14901) 3249 : cluster [DBG] pgmap v3256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:52 smithi130 bash[51302]: debug 2023-08-27T23:46:52.812+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:52.818596+0000) 2023-08-27T23:46:53.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:53 smithi006 bash[33918]: audit 2023-08-27T23:46:53.110970+0000 mon.a (mon.0) 2599 : audit [DBG] from='client.? 172.21.15.6:0/1148408149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:53.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:53 smithi006 bash[27292]: audit 2023-08-27T23:46:53.110970+0000 mon.a (mon.0) 2599 : audit [DBG] from='client.? 172.21.15.6:0/1148408149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:54.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:53 smithi130 bash[27579]: audit 2023-08-27T23:46:53.110970+0000 mon.a (mon.0) 2599 : audit [DBG] from='client.? 172.21.15.6:0/1148408149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:54.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:53 smithi130 bash[51302]: debug 2023-08-27T23:46:53.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:53.818889+0000) 2023-08-27T23:46:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:54 smithi006 bash[56298]: level=error ts=2023-08-27T23:46:54.022Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:46:54 smithi006 bash[56298]: level=warn ts=2023-08-27T23:46:54.024Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:46:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:54 smithi006 bash[33918]: cluster 2023-08-27T23:46:53.570463+0000 mgr.y (mgr.14901) 3250 : cluster [DBG] pgmap v3257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:54 smithi006 bash[27292]: cluster 2023-08-27T23:46:53.570463+0000 mgr.y (mgr.14901) 3250 : cluster [DBG] pgmap v3257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:54 smithi130 bash[27579]: cluster 2023-08-27T23:46:53.570463+0000 mgr.y (mgr.14901) 3250 : cluster [DBG] pgmap v3257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:54 smithi130 bash[51302]: debug 2023-08-27T23:46:54.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:54.819135+0000) 2023-08-27T23:46:56.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:55 smithi130 bash[51302]: debug 2023-08-27T23:46:55.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:55.819371+0000) 2023-08-27T23:46:56.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:56 smithi130 bash[27579]: audit 2023-08-27T23:46:55.533859+0000 mon.a (mon.0) 2600 : audit [DBG] from='client.? 172.21.15.6:0/3287210173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:56.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:56 smithi130 bash[27579]: cluster 2023-08-27T23:46:55.571602+0000 mgr.y (mgr.14901) 3251 : cluster [DBG] pgmap v3258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:56.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:56 smithi006 bash[27292]: audit 2023-08-27T23:46:55.533859+0000 mon.a (mon.0) 2600 : audit [DBG] from='client.? 172.21.15.6:0/3287210173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:56.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:56 smithi006 bash[27292]: cluster 2023-08-27T23:46:55.571602+0000 mgr.y (mgr.14901) 3251 : cluster [DBG] pgmap v3258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:56.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:56 smithi006 bash[33918]: audit 2023-08-27T23:46:55.533859+0000 mon.a (mon.0) 2600 : audit [DBG] from='client.? 172.21.15.6:0/3287210173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:56.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:56 smithi006 bash[33918]: cluster 2023-08-27T23:46:55.571602+0000 mgr.y (mgr.14901) 3251 : cluster [DBG] pgmap v3258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:57.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:56 smithi130 bash[51302]: debug 2023-08-27T23:46:56.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:56.819584+0000) 2023-08-27T23:46:58.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:57 smithi130 bash[51302]: debug 2023-08-27T23:46:57.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:57.819856+0000) 2023-08-27T23:46:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:58 smithi006 bash[33918]: cluster 2023-08-27T23:46:57.572295+0000 mgr.y (mgr.14901) 3252 : cluster [DBG] pgmap v3259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:46:58 smithi006 bash[33918]: audit 2023-08-27T23:46:57.950802+0000 mon.a (mon.0) 2601 : audit [DBG] from='client.? 172.21.15.6:0/261429644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:58.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:58 smithi006 bash[27292]: cluster 2023-08-27T23:46:57.572295+0000 mgr.y (mgr.14901) 3252 : cluster [DBG] pgmap v3259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:58.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:46:58 smithi006 bash[27292]: audit 2023-08-27T23:46:57.950802+0000 mon.a (mon.0) 2601 : audit [DBG] from='client.? 172.21.15.6:0/261429644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:58 smithi130 bash[27579]: cluster 2023-08-27T23:46:57.572295+0000 mgr.y (mgr.14901) 3252 : cluster [DBG] pgmap v3259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:46:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:46:58 smithi130 bash[27579]: audit 2023-08-27T23:46:57.950802+0000 mon.a (mon.0) 2601 : audit [DBG] from='client.? 172.21.15.6:0/261429644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:59.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:58 smithi130 bash[51302]: debug 2023-08-27T23:46:58.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:58.820127+0000) 2023-08-27T23:47:00.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:46:59 smithi130 bash[51302]: debug 2023-08-27T23:46:59.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:59.820384+0000) 2023-08-27T23:47:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:47:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:47:00] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:47:00.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:00 smithi006 bash[27292]: cluster 2023-08-27T23:46:59.573474+0000 mgr.y (mgr.14901) 3253 : cluster [DBG] pgmap v3260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:00.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:00 smithi006 bash[27292]: audit 2023-08-27T23:47:00.374585+0000 mon.c (mon.2) 1242 : audit [DBG] from='client.? 172.21.15.6:0/3005207477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:47:00.559Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:47:00.561Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:00.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:00 smithi006 bash[33918]: cluster 2023-08-27T23:46:59.573474+0000 mgr.y (mgr.14901) 3253 : cluster [DBG] pgmap v3260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:00.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:00 smithi006 bash[33918]: audit 2023-08-27T23:47:00.374585+0000 mon.c (mon.2) 1242 : audit [DBG] from='client.? 172.21.15.6:0/3005207477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:01.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:00 smithi130 bash[51302]: debug 2023-08-27T23:47:00.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:00.820654+0000) 2023-08-27T23:47:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:00 smithi130 bash[27579]: cluster 2023-08-27T23:46:59.573474+0000 mgr.y (mgr.14901) 3253 : cluster [DBG] pgmap v3260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:00 smithi130 bash[27579]: audit 2023-08-27T23:47:00.374585+0000 mon.c (mon.2) 1242 : audit [DBG] from='client.? 172.21.15.6:0/3005207477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:02.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:01 smithi130 bash[51302]: debug 2023-08-27T23:47:01.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:01.820948+0000) 2023-08-27T23:47:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:47:02.124Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:47:02.126Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:02 smithi006 bash[27292]: cluster 2023-08-27T23:47:01.574186+0000 mgr.y (mgr.14901) 3254 : cluster [DBG] pgmap v3261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:02 smithi006 bash[33918]: cluster 2023-08-27T23:47:01.574186+0000 mgr.y (mgr.14901) 3254 : cluster [DBG] pgmap v3261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:02 smithi130 bash[27579]: cluster 2023-08-27T23:47:01.574186+0000 mgr.y (mgr.14901) 3254 : cluster [DBG] pgmap v3261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:02 smithi130 bash[51302]: debug 2023-08-27T23:47:02.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:02.821282+0000) 2023-08-27T23:47:03.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:03 smithi006 bash[27292]: audit 2023-08-27T23:47:02.795898+0000 mon.a (mon.0) 2602 : audit [DBG] from='client.? 172.21.15.6:0/1761184550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:03.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:03 smithi006 bash[33918]: audit 2023-08-27T23:47:02.795898+0000 mon.a (mon.0) 2602 : audit [DBG] from='client.? 172.21.15.6:0/1761184550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:04.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:03 smithi130 bash[27579]: audit 2023-08-27T23:47:02.795898+0000 mon.a (mon.0) 2602 : audit [DBG] from='client.? 172.21.15.6:0/1761184550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:04.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:03 smithi130 bash[51302]: debug 2023-08-27T23:47:03.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:03.821586+0000) 2023-08-27T23:47:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:04 smithi006 bash[56298]: level=error ts=2023-08-27T23:47:04.024Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:04 smithi006 bash[56298]: level=warn ts=2023-08-27T23:47:04.026Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:04.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:04 smithi006 bash[27292]: cluster 2023-08-27T23:47:03.574890+0000 mgr.y (mgr.14901) 3255 : cluster [DBG] pgmap v3262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:04 smithi006 bash[33918]: cluster 2023-08-27T23:47:03.574890+0000 mgr.y (mgr.14901) 3255 : cluster [DBG] pgmap v3262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:05.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:04 smithi130 bash[51302]: debug 2023-08-27T23:47:04.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:04.821850+0000) 2023-08-27T23:47:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:04 smithi130 bash[27579]: cluster 2023-08-27T23:47:03.574890+0000 mgr.y (mgr.14901) 3255 : cluster [DBG] pgmap v3262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:06.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:06 smithi130 bash[27579]: audit 2023-08-27T23:47:05.212732+0000 mon.c (mon.2) 1243 : audit [DBG] from='client.? 172.21.15.6:0/901529281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:06 smithi130 bash[27579]: cluster 2023-08-27T23:47:05.575998+0000 mgr.y (mgr.14901) 3256 : cluster [DBG] pgmap v3263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:06 smithi130 bash[27579]: audit 2023-08-27T23:47:05.602112+0000 mon.b (mon.1) 427 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:47:06.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:05 smithi130 bash[51302]: debug 2023-08-27T23:47:05.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:05.822050+0000) 2023-08-27T23:47:06.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:06 smithi006 bash[27292]: audit 2023-08-27T23:47:05.212732+0000 mon.c (mon.2) 1243 : audit [DBG] from='client.? 172.21.15.6:0/901529281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:06.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:06 smithi006 bash[27292]: cluster 2023-08-27T23:47:05.575998+0000 mgr.y (mgr.14901) 3256 : cluster [DBG] pgmap v3263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:06.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:06 smithi006 bash[27292]: audit 2023-08-27T23:47:05.602112+0000 mon.b (mon.1) 427 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:47:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:06 smithi006 bash[33918]: audit 2023-08-27T23:47:05.212732+0000 mon.c (mon.2) 1243 : audit [DBG] from='client.? 172.21.15.6:0/901529281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:06 smithi006 bash[33918]: cluster 2023-08-27T23:47:05.575998+0000 mgr.y (mgr.14901) 3256 : cluster [DBG] pgmap v3263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:06 smithi006 bash[33918]: audit 2023-08-27T23:47:05.602112+0000 mon.b (mon.1) 427 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:47:07.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:06 smithi130 bash[51302]: debug 2023-08-27T23:47:06.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:06.822273+0000) 2023-08-27T23:47:07.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:07 smithi006 bash[27292]: audit 2023-08-27T23:47:07.622555+0000 mon.a (mon.0) 2603 : audit [DBG] from='client.? 172.21.15.6:0/1958850928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:07.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:07 smithi006 bash[33918]: audit 2023-08-27T23:47:07.622555+0000 mon.a (mon.0) 2603 : audit [DBG] from='client.? 172.21.15.6:0/1958850928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:08.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:07 smithi130 bash[27579]: audit 2023-08-27T23:47:07.622555+0000 mon.a (mon.0) 2603 : audit [DBG] from='client.? 172.21.15.6:0/1958850928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:08.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:07 smithi130 bash[51302]: debug 2023-08-27T23:47:07.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:07.822578+0000) 2023-08-27T23:47:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:08 smithi006 bash[27292]: cluster 2023-08-27T23:47:07.576610+0000 mgr.y (mgr.14901) 3257 : cluster [DBG] pgmap v3264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:08.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:08 smithi006 bash[33918]: cluster 2023-08-27T23:47:07.576610+0000 mgr.y (mgr.14901) 3257 : cluster [DBG] pgmap v3264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:08 smithi130 bash[27579]: cluster 2023-08-27T23:47:07.576610+0000 mgr.y (mgr.14901) 3257 : cluster [DBG] pgmap v3264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:09.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:08 smithi130 bash[51302]: debug 2023-08-27T23:47:08.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:08.822927+0000) 2023-08-27T23:47:10.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:09 smithi130 bash[51302]: debug 2023-08-27T23:47:09.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:09.823187+0000) 2023-08-27T23:47:10.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:10 smithi006 bash[27292]: cluster 2023-08-27T23:47:09.577684+0000 mgr.y (mgr.14901) 3258 : cluster [DBG] pgmap v3265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:10.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:10 smithi006 bash[27292]: audit 2023-08-27T23:47:10.056107+0000 mon.c (mon.2) 1244 : audit [DBG] from='client.? 172.21.15.6:0/4206887476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:10.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:10 smithi006 bash[33918]: cluster 2023-08-27T23:47:09.577684+0000 mgr.y (mgr.14901) 3258 : cluster [DBG] pgmap v3265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:10.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:10 smithi006 bash[33918]: audit 2023-08-27T23:47:10.056107+0000 mon.c (mon.2) 1244 : audit [DBG] from='client.? 172.21.15.6:0/4206887476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:10.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:47:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:47:10] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:47:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:47:10.560Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:47:10.563Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:11.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:10 smithi130 bash[51302]: debug 2023-08-27T23:47:10.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:10.823442+0000) 2023-08-27T23:47:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:10 smithi130 bash[27579]: cluster 2023-08-27T23:47:09.577684+0000 mgr.y (mgr.14901) 3258 : cluster [DBG] pgmap v3265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:10 smithi130 bash[27579]: audit 2023-08-27T23:47:10.056107+0000 mon.c (mon.2) 1244 : audit [DBG] from='client.? 172.21.15.6:0/4206887476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:12.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:11 smithi130 bash[51302]: debug 2023-08-27T23:47:11.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:11.823672+0000) 2023-08-27T23:47:12.429 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:47:12.125Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:12.429 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:47:12.127Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:12.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:12 smithi006 bash[33918]: audit 2023-08-27T23:47:11.420637+0000 mon.a (mon.0) 2604 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:47:12.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:12 smithi006 bash[33918]: cluster 2023-08-27T23:47:11.578305+0000 mgr.y (mgr.14901) 3259 : cluster [DBG] pgmap v3266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:12 smithi006 bash[33918]: audit 2023-08-27T23:47:11.920817+0000 mon.a (mon.0) 2605 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:47:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:12 smithi006 bash[33918]: audit 2023-08-27T23:47:12.217084+0000 mon.b (mon.1) 428 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:47:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:12 smithi006 bash[33918]: audit 2023-08-27T23:47:12.219058+0000 mon.b (mon.1) 429 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:47:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:12 smithi006 bash[33918]: audit 2023-08-27T23:47:12.226043+0000 mon.a (mon.0) 2606 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:47:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:12 smithi006 bash[27292]: audit 2023-08-27T23:47:11.420637+0000 mon.a (mon.0) 2604 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:47:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:12 smithi006 bash[27292]: cluster 2023-08-27T23:47:11.578305+0000 mgr.y (mgr.14901) 3259 : cluster [DBG] pgmap v3266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:12 smithi006 bash[27292]: audit 2023-08-27T23:47:11.920817+0000 mon.a (mon.0) 2605 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:47:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:12 smithi006 bash[27292]: audit 2023-08-27T23:47:12.217084+0000 mon.b (mon.1) 428 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:47:12.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:12 smithi006 bash[27292]: audit 2023-08-27T23:47:12.219058+0000 mon.b (mon.1) 429 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:47:12.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:12 smithi006 bash[27292]: audit 2023-08-27T23:47:12.226043+0000 mon.a (mon.0) 2606 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:47:12.825 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:12 smithi130 bash[27579]: audit 2023-08-27T23:47:11.420637+0000 mon.a (mon.0) 2604 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:47:12.825 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:12 smithi130 bash[27579]: cluster 2023-08-27T23:47:11.578305+0000 mgr.y (mgr.14901) 3259 : cluster [DBG] pgmap v3266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:12.825 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:12 smithi130 bash[27579]: audit 2023-08-27T23:47:11.920817+0000 mon.a (mon.0) 2605 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:47:12.825 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:12 smithi130 bash[27579]: audit 2023-08-27T23:47:12.217084+0000 mon.b (mon.1) 428 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:47:12.825 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:12 smithi130 bash[27579]: audit 2023-08-27T23:47:12.219058+0000 mon.b (mon.1) 429 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:47:12.825 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:12 smithi130 bash[27579]: audit 2023-08-27T23:47:12.226043+0000 mon.a (mon.0) 2606 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:47:13.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:12 smithi130 bash[51302]: debug 2023-08-27T23:47:12.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:12.823970+0000) 2023-08-27T23:47:13.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:13 smithi006 bash[27292]: audit 2023-08-27T23:47:12.478589+0000 mon.a (mon.0) 2607 : audit [DBG] from='client.? 172.21.15.6:0/3557992060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:13.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:13 smithi006 bash[33918]: audit 2023-08-27T23:47:12.478589+0000 mon.a (mon.0) 2607 : audit [DBG] from='client.? 172.21.15.6:0/3557992060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:13.825 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:13 smithi130 bash[27579]: audit 2023-08-27T23:47:12.478589+0000 mon.a (mon.0) 2607 : audit [DBG] from='client.? 172.21.15.6:0/3557992060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:14.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:13 smithi130 bash[51302]: debug 2023-08-27T23:47:13.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:13.824249+0000) 2023-08-27T23:47:14.442 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:14 smithi006 bash[56298]: level=error ts=2023-08-27T23:47:14.024Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:14.442 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:14 smithi006 bash[56298]: level=warn ts=2023-08-27T23:47:14.027Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:14.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:14 smithi006 bash[27292]: cluster 2023-08-27T23:47:13.578952+0000 mgr.y (mgr.14901) 3260 : cluster [DBG] pgmap v3267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:14 smithi006 bash[33918]: cluster 2023-08-27T23:47:13.578952+0000 mgr.y (mgr.14901) 3260 : cluster [DBG] pgmap v3267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:14.828 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:14 smithi130 bash[27579]: cluster 2023-08-27T23:47:13.578952+0000 mgr.y (mgr.14901) 3260 : cluster [DBG] pgmap v3267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:15.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:14 smithi130 bash[51302]: debug 2023-08-27T23:47:14.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:14.824571+0000) 2023-08-27T23:47:15.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:15 smithi006 bash[27292]: audit 2023-08-27T23:47:14.894641+0000 mon.c (mon.2) 1245 : audit [DBG] from='client.? 172.21.15.6:0/3690396703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:15.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:15 smithi006 bash[33918]: audit 2023-08-27T23:47:14.894641+0000 mon.c (mon.2) 1245 : audit [DBG] from='client.? 172.21.15.6:0/3690396703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:15.825 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:15 smithi130 bash[27579]: audit 2023-08-27T23:47:14.894641+0000 mon.c (mon.2) 1245 : audit [DBG] from='client.? 172.21.15.6:0/3690396703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:16.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:15 smithi130 bash[51302]: debug 2023-08-27T23:47:15.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:15.824867+0000) 2023-08-27T23:47:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:16 smithi006 bash[27292]: cluster 2023-08-27T23:47:15.580050+0000 mgr.y (mgr.14901) 3261 : cluster [DBG] pgmap v3268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:16 smithi006 bash[33918]: cluster 2023-08-27T23:47:15.580050+0000 mgr.y (mgr.14901) 3261 : cluster [DBG] pgmap v3268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:16.826 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:16 smithi130 bash[27579]: cluster 2023-08-27T23:47:15.580050+0000 mgr.y (mgr.14901) 3261 : cluster [DBG] pgmap v3268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:17.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:16 smithi130 bash[51302]: debug 2023-08-27T23:47:16.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:16.825133+0000) 2023-08-27T23:47:17.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:17 smithi006 bash[27292]: audit 2023-08-27T23:47:17.310575+0000 mon.c (mon.2) 1246 : audit [DBG] from='client.? 172.21.15.6:0/2011728118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:17.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:17 smithi006 bash[33918]: audit 2023-08-27T23:47:17.310575+0000 mon.c (mon.2) 1246 : audit [DBG] from='client.? 172.21.15.6:0/2011728118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:17.826 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:17 smithi130 bash[27579]: audit 2023-08-27T23:47:17.310575+0000 mon.c (mon.2) 1246 : audit [DBG] from='client.? 172.21.15.6:0/2011728118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:18.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:17 smithi130 bash[51302]: debug 2023-08-27T23:47:17.824+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:17.825454+0000) 2023-08-27T23:47:18.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:18 smithi006 bash[27292]: cluster 2023-08-27T23:47:17.580707+0000 mgr.y (mgr.14901) 3262 : cluster [DBG] pgmap v3269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:18 smithi006 bash[33918]: cluster 2023-08-27T23:47:17.580707+0000 mgr.y (mgr.14901) 3262 : cluster [DBG] pgmap v3269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:18.826 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:18 smithi130 bash[27579]: cluster 2023-08-27T23:47:17.580707+0000 mgr.y (mgr.14901) 3262 : cluster [DBG] pgmap v3269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:19.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:18 smithi130 bash[51302]: debug 2023-08-27T23:47:18.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:18.825726+0000) 2023-08-27T23:47:20.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:19 smithi130 bash[51302]: debug 2023-08-27T23:47:19.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:19.825980+0000) 2023-08-27T23:47:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:47:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:47:20] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-27T23:47:20.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:47:20.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:20.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:47:20.563Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:20.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:20 smithi006 bash[27292]: cluster 2023-08-27T23:47:19.581773+0000 mgr.y (mgr.14901) 3263 : cluster [DBG] pgmap v3270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:20.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:20 smithi006 bash[27292]: audit 2023-08-27T23:47:19.720891+0000 mon.a (mon.0) 2608 : audit [DBG] from='client.? 172.21.15.6:0/3302310362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:20.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:20 smithi006 bash[33918]: cluster 2023-08-27T23:47:19.581773+0000 mgr.y (mgr.14901) 3263 : cluster [DBG] pgmap v3270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:20.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:20 smithi006 bash[33918]: audit 2023-08-27T23:47:19.720891+0000 mon.a (mon.0) 2608 : audit [DBG] from='client.? 172.21.15.6:0/3302310362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:20 smithi130 bash[27579]: cluster 2023-08-27T23:47:19.581773+0000 mgr.y (mgr.14901) 3263 : cluster [DBG] pgmap v3270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:20 smithi130 bash[27579]: audit 2023-08-27T23:47:19.720891+0000 mon.a (mon.0) 2608 : audit [DBG] from='client.? 172.21.15.6:0/3302310362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:21.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:20 smithi130 bash[51302]: debug 2023-08-27T23:47:20.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:20.826202+0000) 2023-08-27T23:47:22.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:21 smithi130 bash[51302]: debug 2023-08-27T23:47:21.824+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:21.826459+0000) 2023-08-27T23:47:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:47:22.125Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:47:22.127Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:22.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:22 smithi006 bash[27292]: cluster 2023-08-27T23:47:21.582382+0000 mgr.y (mgr.14901) 3264 : cluster [DBG] pgmap v3271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:22.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:22 smithi006 bash[27292]: audit 2023-08-27T23:47:22.132894+0000 mon.c (mon.2) 1247 : audit [DBG] from='client.? 172.21.15.6:0/4259010669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:22.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:22 smithi006 bash[33918]: cluster 2023-08-27T23:47:21.582382+0000 mgr.y (mgr.14901) 3264 : cluster [DBG] pgmap v3271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:22.971 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:22 smithi006 bash[33918]: audit 2023-08-27T23:47:22.132894+0000 mon.c (mon.2) 1247 : audit [DBG] from='client.? 172.21.15.6:0/4259010669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:22 smithi130 bash[27579]: cluster 2023-08-27T23:47:21.582382+0000 mgr.y (mgr.14901) 3264 : cluster [DBG] pgmap v3271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:22 smithi130 bash[27579]: audit 2023-08-27T23:47:22.132894+0000 mon.c (mon.2) 1247 : audit [DBG] from='client.? 172.21.15.6:0/4259010669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:23.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:22 smithi130 bash[51302]: debug 2023-08-27T23:47:22.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:22.826679+0000) 2023-08-27T23:47:24.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:23 smithi130 bash[51302]: debug 2023-08-27T23:47:23.824+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:23.826984+0000) 2023-08-27T23:47:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:24 smithi006 bash[56298]: level=error ts=2023-08-27T23:47:24.026Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:24 smithi006 bash[56298]: level=warn ts=2023-08-27T23:47:24.027Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:24 smithi006 bash[27292]: cluster 2023-08-27T23:47:23.582989+0000 mgr.y (mgr.14901) 3265 : cluster [DBG] pgmap v3272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:24 smithi006 bash[27292]: audit 2023-08-27T23:47:24.550851+0000 mon.c (mon.2) 1248 : audit [DBG] from='client.? 172.21.15.6:0/1457990124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:24.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:24 smithi006 bash[33918]: cluster 2023-08-27T23:47:23.582989+0000 mgr.y (mgr.14901) 3265 : cluster [DBG] pgmap v3272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:24.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:24 smithi006 bash[33918]: audit 2023-08-27T23:47:24.550851+0000 mon.c (mon.2) 1248 : audit [DBG] from='client.? 172.21.15.6:0/1457990124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:24 smithi130 bash[27579]: cluster 2023-08-27T23:47:23.582989+0000 mgr.y (mgr.14901) 3265 : cluster [DBG] pgmap v3272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:24 smithi130 bash[27579]: audit 2023-08-27T23:47:24.550851+0000 mon.c (mon.2) 1248 : audit [DBG] from='client.? 172.21.15.6:0/1457990124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:24 smithi130 bash[51302]: debug 2023-08-27T23:47:24.824+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:24.827277+0000) 2023-08-27T23:47:26.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:26 smithi130 bash[27579]: cluster 2023-08-27T23:47:25.584113+0000 mgr.y (mgr.14901) 3266 : cluster [DBG] pgmap v3273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:26.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:25 smithi130 bash[51302]: debug 2023-08-27T23:47:25.824+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:25.827496+0000) 2023-08-27T23:47:26.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:26 smithi006 bash[33918]: cluster 2023-08-27T23:47:25.584113+0000 mgr.y (mgr.14901) 3266 : cluster [DBG] pgmap v3273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:26 smithi006 bash[27292]: cluster 2023-08-27T23:47:25.584113+0000 mgr.y (mgr.14901) 3266 : cluster [DBG] pgmap v3273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:27 smithi130 bash[27579]: audit 2023-08-27T23:47:26.967181+0000 mon.a (mon.0) 2609 : audit [DBG] from='client.? 172.21.15.6:0/3678583232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:27.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:26 smithi130 bash[51302]: debug 2023-08-27T23:47:26.824+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:26.827765+0000) 2023-08-27T23:47:27.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:27 smithi006 bash[27292]: audit 2023-08-27T23:47:26.967181+0000 mon.a (mon.0) 2609 : audit [DBG] from='client.? 172.21.15.6:0/3678583232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:27.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:27 smithi006 bash[33918]: audit 2023-08-27T23:47:26.967181+0000 mon.a (mon.0) 2609 : audit [DBG] from='client.? 172.21.15.6:0/3678583232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:28.084 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:27 smithi130 bash[51302]: debug 2023-08-27T23:47:27.824+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:27.828041+0000) 2023-08-27T23:47:28.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:28 smithi130 bash[27579]: cluster 2023-08-27T23:47:27.584761+0000 mgr.y (mgr.14901) 3267 : cluster [DBG] pgmap v3274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:28.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:28 smithi006 bash[27292]: cluster 2023-08-27T23:47:27.584761+0000 mgr.y (mgr.14901) 3267 : cluster [DBG] pgmap v3274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:28.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:28 smithi006 bash[33918]: cluster 2023-08-27T23:47:27.584761+0000 mgr.y (mgr.14901) 3267 : cluster [DBG] pgmap v3274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:29.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:28 smithi130 bash[51302]: debug 2023-08-27T23:47:28.824+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:28.828321+0000) 2023-08-27T23:47:29.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:29 smithi006 bash[27292]: audit 2023-08-27T23:47:29.378747+0000 mon.c (mon.2) 1249 : audit [DBG] from='client.? 172.21.15.6:0/709548675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:29.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:29 smithi006 bash[33918]: audit 2023-08-27T23:47:29.378747+0000 mon.c (mon.2) 1249 : audit [DBG] from='client.? 172.21.15.6:0/709548675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:29.829 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:29 smithi130 bash[27579]: audit 2023-08-27T23:47:29.378747+0000 mon.c (mon.2) 1249 : audit [DBG] from='client.? 172.21.15.6:0/709548675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:30.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:29 smithi130 bash[51302]: debug 2023-08-27T23:47:29.828+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:29.828636+0000) 2023-08-27T23:47:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:30 smithi006 bash[27292]: cluster 2023-08-27T23:47:29.585868+0000 mgr.y (mgr.14901) 3268 : cluster [DBG] pgmap v3275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:30 smithi006 bash[33918]: cluster 2023-08-27T23:47:29.585868+0000 mgr.y (mgr.14901) 3268 : cluster [DBG] pgmap v3275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:30.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:47:30.562Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:30.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:47:30.563Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:30.829 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:30 smithi130 bash[27579]: cluster 2023-08-27T23:47:29.585868+0000 mgr.y (mgr.14901) 3268 : cluster [DBG] pgmap v3275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:31.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:30 smithi130 bash[51302]: debug 2023-08-27T23:47:30.824+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:30.828939+0000) 2023-08-27T23:47:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:47:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:47:30] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-27T23:47:32.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:31 smithi130 bash[51302]: debug 2023-08-27T23:47:31.828+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:31.829253+0000) 2023-08-27T23:47:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:47:32.126Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:47:32.128Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:32.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:32 smithi006 bash[27292]: cluster 2023-08-27T23:47:31.586380+0000 mgr.y (mgr.14901) 3269 : cluster [DBG] pgmap v3276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:32.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:32 smithi006 bash[27292]: audit 2023-08-27T23:47:31.788366+0000 mon.a (mon.0) 2610 : audit [DBG] from='client.? 172.21.15.6:0/3905577588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:32.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:32 smithi006 bash[33918]: cluster 2023-08-27T23:47:31.586380+0000 mgr.y (mgr.14901) 3269 : cluster [DBG] pgmap v3276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:32.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:32 smithi006 bash[33918]: audit 2023-08-27T23:47:31.788366+0000 mon.a (mon.0) 2610 : audit [DBG] from='client.? 172.21.15.6:0/3905577588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:33.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:32 smithi130 bash[51302]: debug 2023-08-27T23:47:32.828+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:32.829536+0000) 2023-08-27T23:47:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:32 smithi130 bash[27579]: cluster 2023-08-27T23:47:31.586380+0000 mgr.y (mgr.14901) 3269 : cluster [DBG] pgmap v3276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:32 smithi130 bash[27579]: audit 2023-08-27T23:47:31.788366+0000 mon.a (mon.0) 2610 : audit [DBG] from='client.? 172.21.15.6:0/3905577588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:34.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:33 smithi130 bash[51302]: debug 2023-08-27T23:47:33.828+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:33.829817+0000) 2023-08-27T23:47:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:34 smithi006 bash[56298]: level=error ts=2023-08-27T23:47:34.026Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:34 smithi006 bash[56298]: level=warn ts=2023-08-27T23:47:34.028Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:34.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:34 smithi006 bash[33918]: cluster 2023-08-27T23:47:33.586940+0000 mgr.y (mgr.14901) 3270 : cluster [DBG] pgmap v3277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:34.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:34 smithi006 bash[33918]: audit 2023-08-27T23:47:34.209026+0000 mon.c (mon.2) 1250 : audit [DBG] from='client.? 172.21.15.6:0/487913678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:34.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:34 smithi006 bash[27292]: cluster 2023-08-27T23:47:33.586940+0000 mgr.y (mgr.14901) 3270 : cluster [DBG] pgmap v3277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:34.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:34 smithi006 bash[27292]: audit 2023-08-27T23:47:34.209026+0000 mon.c (mon.2) 1250 : audit [DBG] from='client.? 172.21.15.6:0/487913678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:34 smithi130 bash[27579]: cluster 2023-08-27T23:47:33.586940+0000 mgr.y (mgr.14901) 3270 : cluster [DBG] pgmap v3277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:35.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:34 smithi130 bash[27579]: audit 2023-08-27T23:47:34.209026+0000 mon.c (mon.2) 1250 : audit [DBG] from='client.? 172.21.15.6:0/487913678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:35.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:34 smithi130 bash[51302]: debug 2023-08-27T23:47:34.824+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:34.830131+0000) 2023-08-27T23:47:36.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:36 smithi130 bash[27579]: cluster 2023-08-27T23:47:35.588090+0000 mgr.y (mgr.14901) 3271 : cluster [DBG] pgmap v3278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:36.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:35 smithi130 bash[51302]: debug 2023-08-27T23:47:35.824+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:35.830432+0000) 2023-08-27T23:47:36.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:36 smithi006 bash[27292]: cluster 2023-08-27T23:47:35.588090+0000 mgr.y (mgr.14901) 3271 : cluster [DBG] pgmap v3278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:36.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:36 smithi006 bash[33918]: cluster 2023-08-27T23:47:35.588090+0000 mgr.y (mgr.14901) 3271 : cluster [DBG] pgmap v3278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:37.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:36 smithi130 bash[51302]: debug 2023-08-27T23:47:36.824+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:36.830701+0000) 2023-08-27T23:47:37.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:37 smithi130 bash[27579]: audit 2023-08-27T23:47:36.622799+0000 mon.c (mon.2) 1251 : audit [DBG] from='client.? 172.21.15.6:0/323228023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:37.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:37 smithi006 bash[27292]: audit 2023-08-27T23:47:36.622799+0000 mon.c (mon.2) 1251 : audit [DBG] from='client.? 172.21.15.6:0/323228023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:37.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:37 smithi006 bash[33918]: audit 2023-08-27T23:47:36.622799+0000 mon.c (mon.2) 1251 : audit [DBG] from='client.? 172.21.15.6:0/323228023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:38.083 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:37 smithi130 bash[51302]: debug 2023-08-27T23:47:37.828+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:37.830972+0000) 2023-08-27T23:47:38.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:38 smithi130 bash[27579]: cluster 2023-08-27T23:47:37.588884+0000 mgr.y (mgr.14901) 3272 : cluster [DBG] pgmap v3279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:38.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:38 smithi006 bash[33918]: cluster 2023-08-27T23:47:37.588884+0000 mgr.y (mgr.14901) 3272 : cluster [DBG] pgmap v3279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:38.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:38 smithi006 bash[27292]: cluster 2023-08-27T23:47:37.588884+0000 mgr.y (mgr.14901) 3272 : cluster [DBG] pgmap v3279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:39.085 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:38 smithi130 bash[51302]: debug 2023-08-27T23:47:38.828+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:38.831228+0000) 2023-08-27T23:47:39.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:39 smithi130 bash[27579]: audit 2023-08-27T23:47:39.047308+0000 mon.c (mon.2) 1252 : audit [DBG] from='client.? 172.21.15.6:0/3317006395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:39 smithi006 bash[27292]: audit 2023-08-27T23:47:39.047308+0000 mon.c (mon.2) 1252 : audit [DBG] from='client.? 172.21.15.6:0/3317006395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:39.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:39 smithi006 bash[33918]: audit 2023-08-27T23:47:39.047308+0000 mon.c (mon.2) 1252 : audit [DBG] from='client.? 172.21.15.6:0/3317006395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:40.092 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:39 smithi130 bash[51302]: debug 2023-08-27T23:47:39.828+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:39.831449+0000) 2023-08-27T23:47:40.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:40 smithi130 bash[27579]: cluster 2023-08-27T23:47:39.590032+0000 mgr.y (mgr.14901) 3273 : cluster [DBG] pgmap v3280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:40.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:40 smithi006 bash[33918]: cluster 2023-08-27T23:47:39.590032+0000 mgr.y (mgr.14901) 3273 : cluster [DBG] pgmap v3280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:40.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:40 smithi006 bash[27292]: cluster 2023-08-27T23:47:39.590032+0000 mgr.y (mgr.14901) 3273 : cluster [DBG] pgmap v3280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:47:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:47:40] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-27T23:47:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:47:40.563Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:47:40.565Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:41.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:40 smithi130 bash[51302]: debug 2023-08-27T23:47:40.828+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:40.831718+0000) 2023-08-27T23:47:41.833 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:41 smithi130 bash[27579]: audit 2023-08-27T23:47:41.464300+0000 mon.a (mon.0) 2611 : audit [DBG] from='client.? 172.21.15.6:0/2882261028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:41.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:41 smithi006 bash[27292]: audit 2023-08-27T23:47:41.464300+0000 mon.a (mon.0) 2611 : audit [DBG] from='client.? 172.21.15.6:0/2882261028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:41.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:41 smithi006 bash[33918]: audit 2023-08-27T23:47:41.464300+0000 mon.a (mon.0) 2611 : audit [DBG] from='client.? 172.21.15.6:0/2882261028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:42.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:41 smithi130 bash[51302]: debug 2023-08-27T23:47:41.828+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:41.831978+0000) 2023-08-27T23:47:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:47:42.127Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:47:42.129Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:42.833 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:42 smithi130 bash[27579]: cluster 2023-08-27T23:47:41.590678+0000 mgr.y (mgr.14901) 3274 : cluster [DBG] pgmap v3281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:42 smithi006 bash[27292]: cluster 2023-08-27T23:47:41.590678+0000 mgr.y (mgr.14901) 3274 : cluster [DBG] pgmap v3281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:42 smithi006 bash[33918]: cluster 2023-08-27T23:47:41.590678+0000 mgr.y (mgr.14901) 3274 : cluster [DBG] pgmap v3281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:43.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:42 smithi130 bash[51302]: debug 2023-08-27T23:47:42.832+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:42.832292+0000) 2023-08-27T23:47:44.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:43 smithi130 bash[51302]: debug 2023-08-27T23:47:43.828+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:43.832602+0000) 2023-08-27T23:47:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:44 smithi006 bash[56298]: level=error ts=2023-08-27T23:47:44.027Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:44 smithi006 bash[56298]: level=warn ts=2023-08-27T23:47:44.029Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:44 smithi006 bash[33918]: cluster 2023-08-27T23:47:43.591065+0000 mgr.y (mgr.14901) 3275 : cluster [DBG] pgmap v3282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:44 smithi006 bash[33918]: audit 2023-08-27T23:47:43.882904+0000 mon.c (mon.2) 1253 : audit [DBG] from='client.? 172.21.15.6:0/1406641195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:44.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:44 smithi006 bash[27292]: cluster 2023-08-27T23:47:43.591065+0000 mgr.y (mgr.14901) 3275 : cluster [DBG] pgmap v3282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:44.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:44 smithi006 bash[27292]: audit 2023-08-27T23:47:43.882904+0000 mon.c (mon.2) 1253 : audit [DBG] from='client.? 172.21.15.6:0/1406641195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:45.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:44 smithi130 bash[51302]: debug 2023-08-27T23:47:44.827+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:44.832879+0000) 2023-08-27T23:47:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:44 smithi130 bash[27579]: cluster 2023-08-27T23:47:43.591065+0000 mgr.y (mgr.14901) 3275 : cluster [DBG] pgmap v3282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:44 smithi130 bash[27579]: audit 2023-08-27T23:47:43.882904+0000 mon.c (mon.2) 1253 : audit [DBG] from='client.? 172.21.15.6:0/1406641195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:46.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:46 smithi130 bash[27579]: cluster 2023-08-27T23:47:45.592156+0000 mgr.y (mgr.14901) 3276 : cluster [DBG] pgmap v3283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:46.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:45 smithi130 bash[51302]: debug 2023-08-27T23:47:45.827+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:45.833150+0000) 2023-08-27T23:47:46.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:46 smithi006 bash[27292]: cluster 2023-08-27T23:47:45.592156+0000 mgr.y (mgr.14901) 3276 : cluster [DBG] pgmap v3283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:46.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:46 smithi006 bash[33918]: cluster 2023-08-27T23:47:45.592156+0000 mgr.y (mgr.14901) 3276 : cluster [DBG] pgmap v3283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:47 smithi130 bash[27579]: audit 2023-08-27T23:47:46.299359+0000 mon.a (mon.0) 2612 : audit [DBG] from='client.? 172.21.15.6:0/1424097207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:47.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:46 smithi130 bash[51302]: debug 2023-08-27T23:47:46.831+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:46.833416+0000) 2023-08-27T23:47:47.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:47 smithi006 bash[27292]: audit 2023-08-27T23:47:46.299359+0000 mon.a (mon.0) 2612 : audit [DBG] from='client.? 172.21.15.6:0/1424097207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:47.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:47 smithi006 bash[33918]: audit 2023-08-27T23:47:46.299359+0000 mon.a (mon.0) 2612 : audit [DBG] from='client.? 172.21.15.6:0/1424097207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:48.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:48 smithi130 bash[27579]: cluster 2023-08-27T23:47:47.592864+0000 mgr.y (mgr.14901) 3277 : cluster [DBG] pgmap v3284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:48.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:47 smithi130 bash[51302]: debug 2023-08-27T23:47:47.827+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:47.833703+0000) 2023-08-27T23:47:48.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:48 smithi006 bash[27292]: cluster 2023-08-27T23:47:47.592864+0000 mgr.y (mgr.14901) 3277 : cluster [DBG] pgmap v3284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:48.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:48 smithi006 bash[33918]: cluster 2023-08-27T23:47:47.592864+0000 mgr.y (mgr.14901) 3277 : cluster [DBG] pgmap v3284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:49.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:49 smithi130 bash[27579]: audit 2023-08-27T23:47:48.713869+0000 mon.c (mon.2) 1254 : audit [DBG] from='client.? 172.21.15.6:0/104868770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:49.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:48 smithi130 bash[51302]: debug 2023-08-27T23:47:48.831+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:48.833970+0000) 2023-08-27T23:47:49.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:49 smithi006 bash[33918]: audit 2023-08-27T23:47:48.713869+0000 mon.c (mon.2) 1254 : audit [DBG] from='client.? 172.21.15.6:0/104868770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:49.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:49 smithi006 bash[27292]: audit 2023-08-27T23:47:48.713869+0000 mon.c (mon.2) 1254 : audit [DBG] from='client.? 172.21.15.6:0/104868770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:50.090 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:49 smithi130 bash[51302]: debug 2023-08-27T23:47:49.827+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:49.834256+0000) 2023-08-27T23:47:50.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:50 smithi130 bash[27579]: cluster 2023-08-27T23:47:49.594003+0000 mgr.y (mgr.14901) 3278 : cluster [DBG] pgmap v3285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:50.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:50 smithi006 bash[33918]: cluster 2023-08-27T23:47:49.594003+0000 mgr.y (mgr.14901) 3278 : cluster [DBG] pgmap v3285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:50.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:50 smithi006 bash[27292]: cluster 2023-08-27T23:47:49.594003+0000 mgr.y (mgr.14901) 3278 : cluster [DBG] pgmap v3285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:47:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:47:50] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-27T23:47:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:47:50.563Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:47:50.565Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:51.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:50 smithi130 bash[51302]: debug 2023-08-27T23:47:50.827+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:50.834537+0000) 2023-08-27T23:47:51.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:51 smithi006 bash[27292]: audit 2023-08-27T23:47:51.126653+0000 mon.c (mon.2) 1255 : audit [DBG] from='client.? 172.21.15.6:0/3661010438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:51.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:51 smithi006 bash[33918]: audit 2023-08-27T23:47:51.126653+0000 mon.c (mon.2) 1255 : audit [DBG] from='client.? 172.21.15.6:0/3661010438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:51.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:51 smithi130 bash[27579]: audit 2023-08-27T23:47:51.126653+0000 mon.c (mon.2) 1255 : audit [DBG] from='client.? 172.21.15.6:0/3661010438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:52.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:51 smithi130 bash[51302]: debug 2023-08-27T23:47:51.831+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:51.834835+0000) 2023-08-27T23:47:52.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:52 smithi006 bash[27292]: cluster 2023-08-27T23:47:51.594652+0000 mgr.y (mgr.14901) 3279 : cluster [DBG] pgmap v3286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:52.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:52 smithi006 bash[33918]: cluster 2023-08-27T23:47:51.594652+0000 mgr.y (mgr.14901) 3279 : cluster [DBG] pgmap v3286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:52.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:47:52.128Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:52.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:47:52.130Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:52 smithi130 bash[27579]: cluster 2023-08-27T23:47:51.594652+0000 mgr.y (mgr.14901) 3279 : cluster [DBG] pgmap v3286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:53.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:52 smithi130 bash[51302]: debug 2023-08-27T23:47:52.831+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:52.835107+0000) 2023-08-27T23:47:53.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:53 smithi130 bash[27579]: audit 2023-08-27T23:47:53.549440+0000 mon.c (mon.2) 1256 : audit [DBG] from='client.? 172.21.15.6:0/2703714825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:53.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:53 smithi130 bash[51302]: debug 2023-08-27T23:47:53.831+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:53.835421+0000) 2023-08-27T23:47:53.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:53 smithi006 bash[27292]: audit 2023-08-27T23:47:53.549440+0000 mon.c (mon.2) 1256 : audit [DBG] from='client.? 172.21.15.6:0/2703714825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:53.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:53 smithi006 bash[33918]: audit 2023-08-27T23:47:53.549440+0000 mon.c (mon.2) 1256 : audit [DBG] from='client.? 172.21.15.6:0/2703714825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:54 smithi006 bash[56298]: level=error ts=2023-08-27T23:47:54.028Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:47:54 smithi006 bash[56298]: level=warn ts=2023-08-27T23:47:54.030Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:47:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:54 smithi006 bash[33918]: cluster 2023-08-27T23:47:53.595316+0000 mgr.y (mgr.14901) 3280 : cluster [DBG] pgmap v3287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:54 smithi006 bash[27292]: cluster 2023-08-27T23:47:53.595316+0000 mgr.y (mgr.14901) 3280 : cluster [DBG] pgmap v3287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:54 smithi130 bash[27579]: cluster 2023-08-27T23:47:53.595316+0000 mgr.y (mgr.14901) 3280 : cluster [DBG] pgmap v3287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:54 smithi130 bash[51302]: debug 2023-08-27T23:47:54.831+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:54.835731+0000) 2023-08-27T23:47:56.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:56 smithi130 bash[27579]: cluster 2023-08-27T23:47:55.596335+0000 mgr.y (mgr.14901) 3281 : cluster [DBG] pgmap v3288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:56.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:56 smithi130 bash[27579]: audit 2023-08-27T23:47:55.970642+0000 mon.a (mon.0) 2613 : audit [DBG] from='client.? 172.21.15.6:0/3214469205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:56.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:55 smithi130 bash[51302]: debug 2023-08-27T23:47:55.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:55.836014+0000) 2023-08-27T23:47:56.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:56 smithi006 bash[27292]: cluster 2023-08-27T23:47:55.596335+0000 mgr.y (mgr.14901) 3281 : cluster [DBG] pgmap v3288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:56.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:56 smithi006 bash[27292]: audit 2023-08-27T23:47:55.970642+0000 mon.a (mon.0) 2613 : audit [DBG] from='client.? 172.21.15.6:0/3214469205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:56.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:56 smithi006 bash[33918]: cluster 2023-08-27T23:47:55.596335+0000 mgr.y (mgr.14901) 3281 : cluster [DBG] pgmap v3288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:56.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:56 smithi006 bash[33918]: audit 2023-08-27T23:47:55.970642+0000 mon.a (mon.0) 2613 : audit [DBG] from='client.? 172.21.15.6:0/3214469205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:57.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:56 smithi130 bash[51302]: debug 2023-08-27T23:47:56.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:56.836287+0000) 2023-08-27T23:47:58.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:57 smithi130 bash[51302]: debug 2023-08-27T23:47:57.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:57.836541+0000) 2023-08-27T23:47:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:58 smithi006 bash[27292]: cluster 2023-08-27T23:47:57.597016+0000 mgr.y (mgr.14901) 3282 : cluster [DBG] pgmap v3289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:47:58 smithi006 bash[27292]: audit 2023-08-27T23:47:58.388248+0000 mon.c (mon.2) 1257 : audit [DBG] from='client.? 172.21.15.6:0/72830649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:58.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:58 smithi006 bash[33918]: cluster 2023-08-27T23:47:57.597016+0000 mgr.y (mgr.14901) 3282 : cluster [DBG] pgmap v3289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:58.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:47:58 smithi006 bash[33918]: audit 2023-08-27T23:47:58.388248+0000 mon.c (mon.2) 1257 : audit [DBG] from='client.? 172.21.15.6:0/72830649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:58 smithi130 bash[27579]: cluster 2023-08-27T23:47:57.597016+0000 mgr.y (mgr.14901) 3282 : cluster [DBG] pgmap v3289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:47:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:47:58 smithi130 bash[27579]: audit 2023-08-27T23:47:58.388248+0000 mon.c (mon.2) 1257 : audit [DBG] from='client.? 172.21.15.6:0/72830649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:59.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:58 smithi130 bash[51302]: debug 2023-08-27T23:47:58.831+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:58.836836+0000) 2023-08-27T23:48:00.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:47:59 smithi130 bash[51302]: debug 2023-08-27T23:47:59.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:59.837141+0000) 2023-08-27T23:48:00.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:00 smithi006 bash[27292]: cluster 2023-08-27T23:47:59.598085+0000 mgr.y (mgr.14901) 3283 : cluster [DBG] pgmap v3290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:48:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:48:00] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-27T23:48:00.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:00 smithi006 bash[33918]: cluster 2023-08-27T23:47:59.598085+0000 mgr.y (mgr.14901) 3283 : cluster [DBG] pgmap v3290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:48:00.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:48:00.566Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:00 smithi130 bash[27579]: cluster 2023-08-27T23:47:59.598085+0000 mgr.y (mgr.14901) 3283 : cluster [DBG] pgmap v3290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:01.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:00 smithi130 bash[51302]: debug 2023-08-27T23:48:00.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:00.837426+0000) 2023-08-27T23:48:01.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:01 smithi006 bash[27292]: audit 2023-08-27T23:48:00.815620+0000 mon.c (mon.2) 1258 : audit [DBG] from='client.? 172.21.15.6:0/4143309853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:01.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:01 smithi006 bash[33918]: audit 2023-08-27T23:48:00.815620+0000 mon.c (mon.2) 1258 : audit [DBG] from='client.? 172.21.15.6:0/4143309853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:02.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:01 smithi130 bash[27579]: audit 2023-08-27T23:48:00.815620+0000 mon.c (mon.2) 1258 : audit [DBG] from='client.? 172.21.15.6:0/4143309853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:02.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:01 smithi130 bash[51302]: debug 2023-08-27T23:48:01.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:01.837718+0000) 2023-08-27T23:48:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:48:02.129Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:48:02.131Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:02 smithi006 bash[27292]: cluster 2023-08-27T23:48:01.598748+0000 mgr.y (mgr.14901) 3284 : cluster [DBG] pgmap v3291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:02 smithi006 bash[33918]: cluster 2023-08-27T23:48:01.598748+0000 mgr.y (mgr.14901) 3284 : cluster [DBG] pgmap v3291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:02 smithi130 bash[27579]: cluster 2023-08-27T23:48:01.598748+0000 mgr.y (mgr.14901) 3284 : cluster [DBG] pgmap v3291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:02 smithi130 bash[51302]: debug 2023-08-27T23:48:02.831+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:02.838062+0000) 2023-08-27T23:48:03.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:03 smithi006 bash[33918]: audit 2023-08-27T23:48:03.233330+0000 mon.a (mon.0) 2614 : audit [DBG] from='client.? 172.21.15.6:0/4100915642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:03.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:03 smithi006 bash[27292]: audit 2023-08-27T23:48:03.233330+0000 mon.a (mon.0) 2614 : audit [DBG] from='client.? 172.21.15.6:0/4100915642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:04.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:03 smithi130 bash[27579]: audit 2023-08-27T23:48:03.233330+0000 mon.a (mon.0) 2614 : audit [DBG] from='client.? 172.21.15.6:0/4100915642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:04.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:03 smithi130 bash[51302]: debug 2023-08-27T23:48:03.831+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:03.838380+0000) 2023-08-27T23:48:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:04 smithi006 bash[56298]: level=error ts=2023-08-27T23:48:04.029Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:04 smithi006 bash[56298]: level=warn ts=2023-08-27T23:48:04.032Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:04 smithi006 bash[33918]: cluster 2023-08-27T23:48:03.599407+0000 mgr.y (mgr.14901) 3285 : cluster [DBG] pgmap v3292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:04.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:04 smithi006 bash[27292]: cluster 2023-08-27T23:48:03.599407+0000 mgr.y (mgr.14901) 3285 : cluster [DBG] pgmap v3292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:04 smithi130 bash[27579]: cluster 2023-08-27T23:48:03.599407+0000 mgr.y (mgr.14901) 3285 : cluster [DBG] pgmap v3292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:04 smithi130 bash[51302]: debug 2023-08-27T23:48:04.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:04.838655+0000) 2023-08-27T23:48:06.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:06 smithi130 bash[27579]: cluster 2023-08-27T23:48:05.600627+0000 mgr.y (mgr.14901) 3286 : cluster [DBG] pgmap v3293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:06.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:06 smithi130 bash[27579]: audit 2023-08-27T23:48:05.650746+0000 mon.c (mon.2) 1259 : audit [DBG] from='client.? 172.21.15.6:0/1361136126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:06.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:05 smithi130 bash[51302]: debug 2023-08-27T23:48:05.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:05.838962+0000) 2023-08-27T23:48:06.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:06 smithi006 bash[27292]: cluster 2023-08-27T23:48:05.600627+0000 mgr.y (mgr.14901) 3286 : cluster [DBG] pgmap v3293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:06.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:06 smithi006 bash[27292]: audit 2023-08-27T23:48:05.650746+0000 mon.c (mon.2) 1259 : audit [DBG] from='client.? 172.21.15.6:0/1361136126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:06.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:06 smithi006 bash[33918]: cluster 2023-08-27T23:48:05.600627+0000 mgr.y (mgr.14901) 3286 : cluster [DBG] pgmap v3293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:06 smithi006 bash[33918]: audit 2023-08-27T23:48:05.650746+0000 mon.c (mon.2) 1259 : audit [DBG] from='client.? 172.21.15.6:0/1361136126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:07.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:06 smithi130 bash[51302]: debug 2023-08-27T23:48:06.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:06.839197+0000) 2023-08-27T23:48:08.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:07 smithi130 bash[51302]: debug 2023-08-27T23:48:07.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:07.839477+0000) 2023-08-27T23:48:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:08 smithi006 bash[33918]: cluster 2023-08-27T23:48:07.601348+0000 mgr.y (mgr.14901) 3287 : cluster [DBG] pgmap v3294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:08 smithi006 bash[33918]: audit 2023-08-27T23:48:08.064277+0000 mon.c (mon.2) 1260 : audit [DBG] from='client.? 172.21.15.6:0/2662695576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:08.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:08 smithi006 bash[27292]: cluster 2023-08-27T23:48:07.601348+0000 mgr.y (mgr.14901) 3287 : cluster [DBG] pgmap v3294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:08.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:08 smithi006 bash[27292]: audit 2023-08-27T23:48:08.064277+0000 mon.c (mon.2) 1260 : audit [DBG] from='client.? 172.21.15.6:0/2662695576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:08 smithi130 bash[27579]: cluster 2023-08-27T23:48:07.601348+0000 mgr.y (mgr.14901) 3287 : cluster [DBG] pgmap v3294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:08 smithi130 bash[27579]: audit 2023-08-27T23:48:08.064277+0000 mon.c (mon.2) 1260 : audit [DBG] from='client.? 172.21.15.6:0/2662695576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:09.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:08 smithi130 bash[51302]: debug 2023-08-27T23:48:08.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:08.839786+0000) 2023-08-27T23:48:10.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:09 smithi130 bash[51302]: debug 2023-08-27T23:48:09.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:09.840074+0000) 2023-08-27T23:48:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:48:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:48:10] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-27T23:48:10.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:10 smithi006 bash[27292]: cluster 2023-08-27T23:48:09.602556+0000 mgr.y (mgr.14901) 3288 : cluster [DBG] pgmap v3295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:10.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:10 smithi006 bash[27292]: audit 2023-08-27T23:48:10.494274+0000 mon.c (mon.2) 1261 : audit [DBG] from='client.? 172.21.15.6:0/1751504520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:10.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:10 smithi006 bash[33918]: cluster 2023-08-27T23:48:09.602556+0000 mgr.y (mgr.14901) 3288 : cluster [DBG] pgmap v3295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:10.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:10 smithi006 bash[33918]: audit 2023-08-27T23:48:10.494274+0000 mon.c (mon.2) 1261 : audit [DBG] from='client.? 172.21.15.6:0/1751504520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:48:10.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:48:10.567Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:11.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:10 smithi130 bash[51302]: debug 2023-08-27T23:48:10.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:10.840349+0000) 2023-08-27T23:48:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:10 smithi130 bash[27579]: cluster 2023-08-27T23:48:09.602556+0000 mgr.y (mgr.14901) 3288 : cluster [DBG] pgmap v3295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:10 smithi130 bash[27579]: audit 2023-08-27T23:48:10.494274+0000 mon.c (mon.2) 1261 : audit [DBG] from='client.? 172.21.15.6:0/1751504520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:12.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:11 smithi130 bash[51302]: debug 2023-08-27T23:48:11.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:11.840626+0000) 2023-08-27T23:48:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:48:12.130Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:48:12.132Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:12 smithi006 bash[27292]: cluster 2023-08-27T23:48:11.603236+0000 mgr.y (mgr.14901) 3289 : cluster [DBG] pgmap v3296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:12 smithi006 bash[27292]: audit 2023-08-27T23:48:12.239754+0000 mon.b (mon.1) 430 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:48:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:12 smithi006 bash[33918]: cluster 2023-08-27T23:48:11.603236+0000 mgr.y (mgr.14901) 3289 : cluster [DBG] pgmap v3296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:12.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:12 smithi006 bash[33918]: audit 2023-08-27T23:48:12.239754+0000 mon.b (mon.1) 430 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:48:13.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:12 smithi130 bash[27579]: cluster 2023-08-27T23:48:11.603236+0000 mgr.y (mgr.14901) 3289 : cluster [DBG] pgmap v3296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:12 smithi130 bash[27579]: audit 2023-08-27T23:48:12.239754+0000 mon.b (mon.1) 430 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:48:13.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:12 smithi130 bash[51302]: debug 2023-08-27T23:48:12.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:12.840860+0000) 2023-08-27T23:48:13.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:13 smithi006 bash[33918]: audit 2023-08-27T23:48:12.922148+0000 mon.a (mon.0) 2615 : audit [DBG] from='client.? 172.21.15.6:0/3496080347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:13.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:13 smithi006 bash[27292]: audit 2023-08-27T23:48:12.922148+0000 mon.a (mon.0) 2615 : audit [DBG] from='client.? 172.21.15.6:0/3496080347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:14.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:13 smithi130 bash[27579]: audit 2023-08-27T23:48:12.922148+0000 mon.a (mon.0) 2615 : audit [DBG] from='client.? 172.21.15.6:0/3496080347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:14.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:13 smithi130 bash[51302]: debug 2023-08-27T23:48:13.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:13.841181+0000) 2023-08-27T23:48:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:14 smithi006 bash[56298]: level=error ts=2023-08-27T23:48:14.030Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:14 smithi006 bash[56298]: level=warn ts=2023-08-27T23:48:14.033Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:14 smithi006 bash[33918]: cluster 2023-08-27T23:48:13.603883+0000 mgr.y (mgr.14901) 3290 : cluster [DBG] pgmap v3297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:14 smithi006 bash[27292]: cluster 2023-08-27T23:48:13.603883+0000 mgr.y (mgr.14901) 3290 : cluster [DBG] pgmap v3297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:14 smithi130 bash[27579]: cluster 2023-08-27T23:48:13.603883+0000 mgr.y (mgr.14901) 3290 : cluster [DBG] pgmap v3297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:15.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:14 smithi130 bash[51302]: debug 2023-08-27T23:48:14.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:14.841496+0000) 2023-08-27T23:48:16.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:15 smithi130 bash[51302]: debug 2023-08-27T23:48:15.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:15.841865+0000) 2023-08-27T23:48:16.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:16 smithi130 bash[27579]: audit 2023-08-27T23:48:15.337834+0000 mon.a (mon.0) 2616 : audit [DBG] from='client.? 172.21.15.6:0/260253889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:16.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:16 smithi130 bash[27579]: cluster 2023-08-27T23:48:15.605060+0000 mgr.y (mgr.14901) 3291 : cluster [DBG] pgmap v3298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:16.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:16 smithi006 bash[27292]: audit 2023-08-27T23:48:15.337834+0000 mon.a (mon.0) 2616 : audit [DBG] from='client.? 172.21.15.6:0/260253889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:16.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:16 smithi006 bash[27292]: cluster 2023-08-27T23:48:15.605060+0000 mgr.y (mgr.14901) 3291 : cluster [DBG] pgmap v3298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:16.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:16 smithi006 bash[33918]: audit 2023-08-27T23:48:15.337834+0000 mon.a (mon.0) 2616 : audit [DBG] from='client.? 172.21.15.6:0/260253889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:16.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:16 smithi006 bash[33918]: cluster 2023-08-27T23:48:15.605060+0000 mgr.y (mgr.14901) 3291 : cluster [DBG] pgmap v3298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:17.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:16 smithi130 bash[51302]: debug 2023-08-27T23:48:16.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:16.842284+0000) 2023-08-27T23:48:18.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:17 smithi130 bash[51302]: debug 2023-08-27T23:48:17.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:17.842527+0000) 2023-08-27T23:48:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:18 smithi006 bash[27292]: cluster 2023-08-27T23:48:17.605575+0000 mgr.y (mgr.14901) 3292 : cluster [DBG] pgmap v3299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:18 smithi006 bash[27292]: audit 2023-08-27T23:48:17.748153+0000 mon.a (mon.0) 2617 : audit [DBG] from='client.? 172.21.15.6:0/580944019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:18 smithi006 bash[27292]: audit 2023-08-27T23:48:18.126587+0000 mon.a (mon.0) 2618 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:48:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:18 smithi006 bash[27292]: audit 2023-08-27T23:48:18.537518+0000 mon.a (mon.0) 2619 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:48:18.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:18 smithi006 bash[33918]: cluster 2023-08-27T23:48:17.605575+0000 mgr.y (mgr.14901) 3292 : cluster [DBG] pgmap v3299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:18.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:18 smithi006 bash[33918]: audit 2023-08-27T23:48:17.748153+0000 mon.a (mon.0) 2617 : audit [DBG] from='client.? 172.21.15.6:0/580944019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:18.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:18 smithi006 bash[33918]: audit 2023-08-27T23:48:18.126587+0000 mon.a (mon.0) 2618 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:48:18.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:18 smithi006 bash[33918]: audit 2023-08-27T23:48:18.537518+0000 mon.a (mon.0) 2619 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:48:19.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:18 smithi130 bash[51302]: debug 2023-08-27T23:48:18.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:18.842720+0000) 2023-08-27T23:48:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:18 smithi130 bash[27579]: cluster 2023-08-27T23:48:17.605575+0000 mgr.y (mgr.14901) 3292 : cluster [DBG] pgmap v3299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:18 smithi130 bash[27579]: audit 2023-08-27T23:48:17.748153+0000 mon.a (mon.0) 2617 : audit [DBG] from='client.? 172.21.15.6:0/580944019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:18 smithi130 bash[27579]: audit 2023-08-27T23:48:18.126587+0000 mon.a (mon.0) 2618 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:48:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:18 smithi130 bash[27579]: audit 2023-08-27T23:48:18.537518+0000 mon.a (mon.0) 2619 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:48:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:19 smithi006 bash[33918]: audit 2023-08-27T23:48:18.840439+0000 mon.b (mon.1) 431 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:48:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:19 smithi006 bash[33918]: audit 2023-08-27T23:48:18.843017+0000 mon.b (mon.1) 432 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:48:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:19 smithi006 bash[33918]: audit 2023-08-27T23:48:18.853200+0000 mon.a (mon.0) 2620 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:48:19.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:19 smithi006 bash[27292]: audit 2023-08-27T23:48:18.840439+0000 mon.b (mon.1) 431 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:48:19.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:19 smithi006 bash[27292]: audit 2023-08-27T23:48:18.843017+0000 mon.b (mon.1) 432 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:48:19.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:19 smithi006 bash[27292]: audit 2023-08-27T23:48:18.853200+0000 mon.a (mon.0) 2620 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:48:20.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:19 smithi130 bash[27579]: audit 2023-08-27T23:48:18.840439+0000 mon.b (mon.1) 431 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:48:20.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:19 smithi130 bash[27579]: audit 2023-08-27T23:48:18.843017+0000 mon.b (mon.1) 432 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:48:20.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:19 smithi130 bash[27579]: audit 2023-08-27T23:48:18.853200+0000 mon.a (mon.0) 2620 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:48:20.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:19 smithi130 bash[51302]: debug 2023-08-27T23:48:19.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:19.843014+0000) 2023-08-27T23:48:20.857 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:48:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:48:20] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-27T23:48:20.857 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:48:20.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:20.857 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:48:20.567Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:20.858 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:20 smithi006 bash[27292]: cluster 2023-08-27T23:48:19.606656+0000 mgr.y (mgr.14901) 3293 : cluster [DBG] pgmap v3300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:20.858 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:20 smithi006 bash[27292]: audit 2023-08-27T23:48:20.160295+0000 mon.a (mon.0) 2621 : audit [DBG] from='client.? 172.21.15.6:0/1866781105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:20.859 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:20 smithi006 bash[33918]: cluster 2023-08-27T23:48:19.606656+0000 mgr.y (mgr.14901) 3293 : cluster [DBG] pgmap v3300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:20.859 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:20 smithi006 bash[33918]: audit 2023-08-27T23:48:20.160295+0000 mon.a (mon.0) 2621 : audit [DBG] from='client.? 172.21.15.6:0/1866781105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:21.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:20 smithi130 bash[51302]: debug 2023-08-27T23:48:20.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:20.843236+0000) 2023-08-27T23:48:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:20 smithi130 bash[27579]: cluster 2023-08-27T23:48:19.606656+0000 mgr.y (mgr.14901) 3293 : cluster [DBG] pgmap v3300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:20 smithi130 bash[27579]: audit 2023-08-27T23:48:20.160295+0000 mon.a (mon.0) 2621 : audit [DBG] from='client.? 172.21.15.6:0/1866781105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:22.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:21 smithi130 bash[51302]: debug 2023-08-27T23:48:21.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:21.843485+0000) 2023-08-27T23:48:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:48:22.131Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:48:22.133Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:23.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:22 smithi130 bash[51302]: debug 2023-08-27T23:48:22.843+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:22.843766+0000) 2023-08-27T23:48:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:22 smithi130 bash[27579]: cluster 2023-08-27T23:48:21.607315+0000 mgr.y (mgr.14901) 3294 : cluster [DBG] pgmap v3301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:22 smithi130 bash[27579]: audit 2023-08-27T23:48:22.573107+0000 mon.c (mon.2) 1262 : audit [DBG] from='client.? 172.21.15.6:0/2719769630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:22 smithi006 bash[33918]: cluster 2023-08-27T23:48:21.607315+0000 mgr.y (mgr.14901) 3294 : cluster [DBG] pgmap v3301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:22 smithi006 bash[33918]: audit 2023-08-27T23:48:22.573107+0000 mon.c (mon.2) 1262 : audit [DBG] from='client.? 172.21.15.6:0/2719769630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:22 smithi006 bash[27292]: cluster 2023-08-27T23:48:21.607315+0000 mgr.y (mgr.14901) 3294 : cluster [DBG] pgmap v3301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:23.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:22 smithi006 bash[27292]: audit 2023-08-27T23:48:22.573107+0000 mon.c (mon.2) 1262 : audit [DBG] from='client.? 172.21.15.6:0/2719769630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:24.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:23 smithi130 bash[51302]: debug 2023-08-27T23:48:23.843+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:23.844072+0000) 2023-08-27T23:48:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:24 smithi006 bash[56298]: level=error ts=2023-08-27T23:48:24.031Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:24 smithi006 bash[56298]: level=warn ts=2023-08-27T23:48:24.034Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:24 smithi130 bash[27579]: cluster 2023-08-27T23:48:23.608012+0000 mgr.y (mgr.14901) 3295 : cluster [DBG] pgmap v3302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:25.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:24 smithi130 bash[51302]: debug 2023-08-27T23:48:24.843+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:24.844349+0000) 2023-08-27T23:48:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:24 smithi006 bash[33918]: cluster 2023-08-27T23:48:23.608012+0000 mgr.y (mgr.14901) 3295 : cluster [DBG] pgmap v3302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:24 smithi006 bash[27292]: cluster 2023-08-27T23:48:23.608012+0000 mgr.y (mgr.14901) 3295 : cluster [DBG] pgmap v3302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:26.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:25 smithi130 bash[27579]: audit 2023-08-27T23:48:24.998137+0000 mon.a (mon.0) 2622 : audit [DBG] from='client.? 172.21.15.6:0/2551925455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:26.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:25 smithi130 bash[51302]: debug 2023-08-27T23:48:25.843+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:25.844621+0000) 2023-08-27T23:48:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:25 smithi006 bash[27292]: audit 2023-08-27T23:48:24.998137+0000 mon.a (mon.0) 2622 : audit [DBG] from='client.? 172.21.15.6:0/2551925455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:26.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:25 smithi006 bash[33918]: audit 2023-08-27T23:48:24.998137+0000 mon.a (mon.0) 2622 : audit [DBG] from='client.? 172.21.15.6:0/2551925455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:26 smithi130 bash[27579]: cluster 2023-08-27T23:48:25.609237+0000 mgr.y (mgr.14901) 3296 : cluster [DBG] pgmap v3303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:27.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:26 smithi130 bash[51302]: debug 2023-08-27T23:48:26.843+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:26.844894+0000) 2023-08-27T23:48:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:26 smithi006 bash[33918]: cluster 2023-08-27T23:48:25.609237+0000 mgr.y (mgr.14901) 3296 : cluster [DBG] pgmap v3303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:27.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:26 smithi006 bash[27292]: cluster 2023-08-27T23:48:25.609237+0000 mgr.y (mgr.14901) 3296 : cluster [DBG] pgmap v3303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:28.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:27 smithi130 bash[27579]: audit 2023-08-27T23:48:27.414707+0000 mon.c (mon.2) 1263 : audit [DBG] from='client.? 172.21.15.6:0/2502119520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:28.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:27 smithi130 bash[51302]: debug 2023-08-27T23:48:27.843+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:27.845167+0000) 2023-08-27T23:48:28.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:27 smithi006 bash[33918]: audit 2023-08-27T23:48:27.414707+0000 mon.c (mon.2) 1263 : audit [DBG] from='client.? 172.21.15.6:0/2502119520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:28.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:27 smithi006 bash[27292]: audit 2023-08-27T23:48:27.414707+0000 mon.c (mon.2) 1263 : audit [DBG] from='client.? 172.21.15.6:0/2502119520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:28 smithi130 bash[27579]: cluster 2023-08-27T23:48:27.609888+0000 mgr.y (mgr.14901) 3297 : cluster [DBG] pgmap v3304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:29.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:28 smithi130 bash[51302]: debug 2023-08-27T23:48:28.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:28.845459+0000) 2023-08-27T23:48:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:28 smithi006 bash[27292]: cluster 2023-08-27T23:48:27.609888+0000 mgr.y (mgr.14901) 3297 : cluster [DBG] pgmap v3304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:28 smithi006 bash[33918]: cluster 2023-08-27T23:48:27.609888+0000 mgr.y (mgr.14901) 3297 : cluster [DBG] pgmap v3304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:30.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:29 smithi130 bash[27579]: audit 2023-08-27T23:48:29.829989+0000 mon.a (mon.0) 2623 : audit [DBG] from='client.? 172.21.15.6:0/3651574925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:30.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:29 smithi130 bash[51302]: debug 2023-08-27T23:48:29.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:29.845737+0000) 2023-08-27T23:48:30.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:29 smithi006 bash[33918]: audit 2023-08-27T23:48:29.829989+0000 mon.a (mon.0) 2623 : audit [DBG] from='client.? 172.21.15.6:0/3651574925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:30.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:29 smithi006 bash[27292]: audit 2023-08-27T23:48:29.829989+0000 mon.a (mon.0) 2623 : audit [DBG] from='client.? 172.21.15.6:0/3651574925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:30.893 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:48:30.567Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:30.893 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:48:30.569Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:30.894 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:48:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:48:30] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-27T23:48:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:30 smithi130 bash[27579]: cluster 2023-08-27T23:48:29.610964+0000 mgr.y (mgr.14901) 3298 : cluster [DBG] pgmap v3305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:31.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:30 smithi130 bash[51302]: debug 2023-08-27T23:48:30.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:30.845999+0000) 2023-08-27T23:48:31.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:30 smithi006 bash[33918]: cluster 2023-08-27T23:48:29.610964+0000 mgr.y (mgr.14901) 3298 : cluster [DBG] pgmap v3305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:31.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:30 smithi006 bash[27292]: cluster 2023-08-27T23:48:29.610964+0000 mgr.y (mgr.14901) 3298 : cluster [DBG] pgmap v3305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:32.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:31 smithi130 bash[51302]: debug 2023-08-27T23:48:31.843+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:31.846276+0000) 2023-08-27T23:48:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:48:32.132Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:48:32.134Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:33.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:32 smithi130 bash[27579]: cluster 2023-08-27T23:48:31.611566+0000 mgr.y (mgr.14901) 3299 : cluster [DBG] pgmap v3306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:32 smithi130 bash[27579]: audit 2023-08-27T23:48:32.248488+0000 mon.c (mon.2) 1264 : audit [DBG] from='client.? 172.21.15.6:0/994575594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:33.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:32 smithi130 bash[51302]: debug 2023-08-27T23:48:32.843+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:32.846505+0000) 2023-08-27T23:48:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:32 smithi006 bash[33918]: cluster 2023-08-27T23:48:31.611566+0000 mgr.y (mgr.14901) 3299 : cluster [DBG] pgmap v3306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:32 smithi006 bash[33918]: audit 2023-08-27T23:48:32.248488+0000 mon.c (mon.2) 1264 : audit [DBG] from='client.? 172.21.15.6:0/994575594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:32 smithi006 bash[27292]: cluster 2023-08-27T23:48:31.611566+0000 mgr.y (mgr.14901) 3299 : cluster [DBG] pgmap v3306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:33.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:32 smithi006 bash[27292]: audit 2023-08-27T23:48:32.248488+0000 mon.c (mon.2) 1264 : audit [DBG] from='client.? 172.21.15.6:0/994575594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:34.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:33 smithi130 bash[51302]: debug 2023-08-27T23:48:33.843+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:33.846776+0000) 2023-08-27T23:48:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:34 smithi006 bash[56298]: level=error ts=2023-08-27T23:48:34.032Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:34 smithi006 bash[56298]: level=warn ts=2023-08-27T23:48:34.034Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:35.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:34 smithi130 bash[51302]: debug 2023-08-27T23:48:34.843+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:34.847052+0000) 2023-08-27T23:48:35.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:34 smithi130 bash[27579]: cluster 2023-08-27T23:48:33.612217+0000 mgr.y (mgr.14901) 3300 : cluster [DBG] pgmap v3307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:35.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:34 smithi130 bash[27579]: audit 2023-08-27T23:48:34.661691+0000 mon.c (mon.2) 1265 : audit [DBG] from='client.? 172.21.15.6:0/3031790004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:34 smithi006 bash[27292]: cluster 2023-08-27T23:48:33.612217+0000 mgr.y (mgr.14901) 3300 : cluster [DBG] pgmap v3307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:34 smithi006 bash[27292]: audit 2023-08-27T23:48:34.661691+0000 mon.c (mon.2) 1265 : audit [DBG] from='client.? 172.21.15.6:0/3031790004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:34 smithi006 bash[33918]: cluster 2023-08-27T23:48:33.612217+0000 mgr.y (mgr.14901) 3300 : cluster [DBG] pgmap v3307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:35.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:34 smithi006 bash[33918]: audit 2023-08-27T23:48:34.661691+0000 mon.c (mon.2) 1265 : audit [DBG] from='client.? 172.21.15.6:0/3031790004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:36.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:36 smithi130 bash[27579]: cluster 2023-08-27T23:48:35.613328+0000 mgr.y (mgr.14901) 3301 : cluster [DBG] pgmap v3308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:36.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:35 smithi130 bash[51302]: debug 2023-08-27T23:48:35.843+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:35.847377+0000) 2023-08-27T23:48:36.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:36 smithi006 bash[33918]: cluster 2023-08-27T23:48:35.613328+0000 mgr.y (mgr.14901) 3301 : cluster [DBG] pgmap v3308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:36.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:36 smithi006 bash[27292]: cluster 2023-08-27T23:48:35.613328+0000 mgr.y (mgr.14901) 3301 : cluster [DBG] pgmap v3308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:37.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:36 smithi130 bash[51302]: debug 2023-08-27T23:48:36.847+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:36.847667+0000) 2023-08-27T23:48:37.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:37 smithi006 bash[27292]: audit 2023-08-27T23:48:37.079028+0000 mon.a (mon.0) 2624 : audit [DBG] from='client.? 172.21.15.6:0/830436094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:37.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:37 smithi006 bash[33918]: audit 2023-08-27T23:48:37.079028+0000 mon.a (mon.0) 2624 : audit [DBG] from='client.? 172.21.15.6:0/830436094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:37.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:37 smithi130 bash[27579]: audit 2023-08-27T23:48:37.079028+0000 mon.a (mon.0) 2624 : audit [DBG] from='client.? 172.21.15.6:0/830436094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:38.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:37 smithi130 bash[51302]: debug 2023-08-27T23:48:37.843+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:37.847947+0000) 2023-08-27T23:48:38.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:38 smithi006 bash[27292]: cluster 2023-08-27T23:48:37.614008+0000 mgr.y (mgr.14901) 3302 : cluster [DBG] pgmap v3309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:38.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:38 smithi006 bash[33918]: cluster 2023-08-27T23:48:37.614008+0000 mgr.y (mgr.14901) 3302 : cluster [DBG] pgmap v3309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:38.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:38 smithi130 bash[27579]: cluster 2023-08-27T23:48:37.614008+0000 mgr.y (mgr.14901) 3302 : cluster [DBG] pgmap v3309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:39.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:38 smithi130 bash[51302]: debug 2023-08-27T23:48:38.843+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:38.848256+0000) 2023-08-27T23:48:39.849 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:39 smithi130 bash[27579]: audit 2023-08-27T23:48:39.498158+0000 mon.a (mon.0) 2625 : audit [DBG] from='client.? 172.21.15.6:0/3892257596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:39.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:39 smithi006 bash[27292]: audit 2023-08-27T23:48:39.498158+0000 mon.a (mon.0) 2625 : audit [DBG] from='client.? 172.21.15.6:0/3892257596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:39 smithi006 bash[33918]: audit 2023-08-27T23:48:39.498158+0000 mon.a (mon.0) 2625 : audit [DBG] from='client.? 172.21.15.6:0/3892257596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:40.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:39 smithi130 bash[51302]: debug 2023-08-27T23:48:39.843+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:39.848577+0000) 2023-08-27T23:48:40.849 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:40 smithi130 bash[27579]: cluster 2023-08-27T23:48:39.615152+0000 mgr.y (mgr.14901) 3303 : cluster [DBG] pgmap v3310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:48:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:48:40] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-27T23:48:40.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:40 smithi006 bash[33918]: cluster 2023-08-27T23:48:39.615152+0000 mgr.y (mgr.14901) 3303 : cluster [DBG] pgmap v3310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:40.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:40 smithi006 bash[27292]: cluster 2023-08-27T23:48:39.615152+0000 mgr.y (mgr.14901) 3303 : cluster [DBG] pgmap v3310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:48:40.568Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:48:40.569Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:41.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:40 smithi130 bash[51302]: debug 2023-08-27T23:48:40.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:40.848860+0000) 2023-08-27T23:48:42.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:41 smithi130 bash[51302]: debug 2023-08-27T23:48:41.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:41.849139+0000) 2023-08-27T23:48:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:48:42.133Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:48:42.135Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:42 smithi006 bash[27292]: cluster 2023-08-27T23:48:41.615645+0000 mgr.y (mgr.14901) 3304 : cluster [DBG] pgmap v3311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:43.016 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:42 smithi006 bash[27292]: audit 2023-08-27T23:48:41.911271+0000 mon.c (mon.2) 1266 : audit [DBG] from='client.? 172.21.15.6:0/2756483972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:43.017 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:42 smithi006 bash[33918]: cluster 2023-08-27T23:48:41.615645+0000 mgr.y (mgr.14901) 3304 : cluster [DBG] pgmap v3311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:43.017 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:42 smithi006 bash[33918]: audit 2023-08-27T23:48:41.911271+0000 mon.c (mon.2) 1266 : audit [DBG] from='client.? 172.21.15.6:0/2756483972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:43.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:42 smithi130 bash[51302]: debug 2023-08-27T23:48:42.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:42.849433+0000) 2023-08-27T23:48:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:42 smithi130 bash[27579]: cluster 2023-08-27T23:48:41.615645+0000 mgr.y (mgr.14901) 3304 : cluster [DBG] pgmap v3311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:42 smithi130 bash[27579]: audit 2023-08-27T23:48:41.911271+0000 mon.c (mon.2) 1266 : audit [DBG] from='client.? 172.21.15.6:0/2756483972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:44.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:43 smithi130 bash[51302]: debug 2023-08-27T23:48:43.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:43.849755+0000) 2023-08-27T23:48:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:44 smithi006 bash[56298]: level=error ts=2023-08-27T23:48:44.033Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:44 smithi006 bash[56298]: level=warn ts=2023-08-27T23:48:44.035Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:44 smithi006 bash[33918]: cluster 2023-08-27T23:48:43.616274+0000 mgr.y (mgr.14901) 3305 : cluster [DBG] pgmap v3312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:44 smithi006 bash[33918]: audit 2023-08-27T23:48:44.324343+0000 mon.a (mon.0) 2626 : audit [DBG] from='client.? 172.21.15.6:0/3272554908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:44.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:44 smithi006 bash[27292]: cluster 2023-08-27T23:48:43.616274+0000 mgr.y (mgr.14901) 3305 : cluster [DBG] pgmap v3312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:44.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:44 smithi006 bash[27292]: audit 2023-08-27T23:48:44.324343+0000 mon.a (mon.0) 2626 : audit [DBG] from='client.? 172.21.15.6:0/3272554908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:44 smithi130 bash[27579]: cluster 2023-08-27T23:48:43.616274+0000 mgr.y (mgr.14901) 3305 : cluster [DBG] pgmap v3312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:44 smithi130 bash[27579]: audit 2023-08-27T23:48:44.324343+0000 mon.a (mon.0) 2626 : audit [DBG] from='client.? 172.21.15.6:0/3272554908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:44 smithi130 bash[51302]: debug 2023-08-27T23:48:44.846+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:44.850026+0000) 2023-08-27T23:48:46.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:45 smithi130 bash[51302]: debug 2023-08-27T23:48:45.846+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:45.850303+0000) 2023-08-27T23:48:46.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:46 smithi130 bash[27579]: cluster 2023-08-27T23:48:45.617432+0000 mgr.y (mgr.14901) 3306 : cluster [DBG] pgmap v3313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:46.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:46 smithi006 bash[27292]: cluster 2023-08-27T23:48:45.617432+0000 mgr.y (mgr.14901) 3306 : cluster [DBG] pgmap v3313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:46 smithi006 bash[33918]: cluster 2023-08-27T23:48:45.617432+0000 mgr.y (mgr.14901) 3306 : cluster [DBG] pgmap v3313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:47.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:47 smithi130 bash[27579]: audit 2023-08-27T23:48:46.741197+0000 mon.c (mon.2) 1267 : audit [DBG] from='client.? 172.21.15.6:0/2101658391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:47.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:46 smithi130 bash[51302]: debug 2023-08-27T23:48:46.846+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:46.850571+0000) 2023-08-27T23:48:47.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:47 smithi006 bash[33918]: audit 2023-08-27T23:48:46.741197+0000 mon.c (mon.2) 1267 : audit [DBG] from='client.? 172.21.15.6:0/2101658391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:47.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:47 smithi006 bash[27292]: audit 2023-08-27T23:48:46.741197+0000 mon.c (mon.2) 1267 : audit [DBG] from='client.? 172.21.15.6:0/2101658391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:48.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:48 smithi130 bash[27579]: cluster 2023-08-27T23:48:47.618119+0000 mgr.y (mgr.14901) 3307 : cluster [DBG] pgmap v3314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:48.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:47 smithi130 bash[51302]: debug 2023-08-27T23:48:47.846+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:47.850899+0000) 2023-08-27T23:48:48.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:48 smithi006 bash[27292]: cluster 2023-08-27T23:48:47.618119+0000 mgr.y (mgr.14901) 3307 : cluster [DBG] pgmap v3314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:48.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:48 smithi006 bash[33918]: cluster 2023-08-27T23:48:47.618119+0000 mgr.y (mgr.14901) 3307 : cluster [DBG] pgmap v3314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:49.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:48 smithi130 bash[51302]: debug 2023-08-27T23:48:48.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:48.851137+0000) 2023-08-27T23:48:49.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:49 smithi006 bash[27292]: audit 2023-08-27T23:48:49.155675+0000 mon.c (mon.2) 1268 : audit [DBG] from='client.? 172.21.15.6:0/3628772330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:49.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:49 smithi006 bash[33918]: audit 2023-08-27T23:48:49.155675+0000 mon.c (mon.2) 1268 : audit [DBG] from='client.? 172.21.15.6:0/3628772330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:49.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:49 smithi130 bash[27579]: audit 2023-08-27T23:48:49.155675+0000 mon.c (mon.2) 1268 : audit [DBG] from='client.? 172.21.15.6:0/3628772330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:50.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:49 smithi130 bash[51302]: debug 2023-08-27T23:48:49.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:49.851424+0000) 2023-08-27T23:48:50.572 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:50 smithi006 bash[27292]: cluster 2023-08-27T23:48:49.619244+0000 mgr.y (mgr.14901) 3308 : cluster [DBG] pgmap v3315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:50.572 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:50 smithi006 bash[33918]: cluster 2023-08-27T23:48:49.619244+0000 mgr.y (mgr.14901) 3308 : cluster [DBG] pgmap v3315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:50.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:50 smithi130 bash[27579]: cluster 2023-08-27T23:48:49.619244+0000 mgr.y (mgr.14901) 3308 : cluster [DBG] pgmap v3315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:48:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:48:50] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-27T23:48:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:48:50.569Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:48:50.571Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:51.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:50 smithi130 bash[51302]: debug 2023-08-27T23:48:50.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:50.851714+0000) 2023-08-27T23:48:51.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:51 smithi006 bash[33918]: audit 2023-08-27T23:48:51.566674+0000 mon.a (mon.0) 2627 : audit [DBG] from='client.? 172.21.15.6:0/1366741319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:51.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:51 smithi006 bash[27292]: audit 2023-08-27T23:48:51.566674+0000 mon.a (mon.0) 2627 : audit [DBG] from='client.? 172.21.15.6:0/1366741319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:52.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:51 smithi130 bash[27579]: audit 2023-08-27T23:48:51.566674+0000 mon.a (mon.0) 2627 : audit [DBG] from='client.? 172.21.15.6:0/1366741319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:52.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:51 smithi130 bash[51302]: debug 2023-08-27T23:48:51.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:51.851997+0000) 2023-08-27T23:48:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:48:52.134Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:48:52.136Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:52 smithi006 bash[27292]: cluster 2023-08-27T23:48:51.619841+0000 mgr.y (mgr.14901) 3309 : cluster [DBG] pgmap v3316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:52 smithi006 bash[33918]: cluster 2023-08-27T23:48:51.619841+0000 mgr.y (mgr.14901) 3309 : cluster [DBG] pgmap v3316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:53.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:52 smithi130 bash[27579]: cluster 2023-08-27T23:48:51.619841+0000 mgr.y (mgr.14901) 3309 : cluster [DBG] pgmap v3316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:52 smithi130 bash[51302]: debug 2023-08-27T23:48:52.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:52.852329+0000) 2023-08-27T23:48:54.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:53 smithi130 bash[51302]: debug 2023-08-27T23:48:53.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:53.852613+0000) 2023-08-27T23:48:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:54 smithi006 bash[56298]: level=error ts=2023-08-27T23:48:54.033Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:48:54 smithi006 bash[56298]: level=warn ts=2023-08-27T23:48:54.035Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:48:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:54 smithi006 bash[27292]: cluster 2023-08-27T23:48:53.620433+0000 mgr.y (mgr.14901) 3310 : cluster [DBG] pgmap v3317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:54 smithi006 bash[27292]: audit 2023-08-27T23:48:53.975937+0000 mon.c (mon.2) 1269 : audit [DBG] from='client.? 172.21.15.6:0/3462096059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:54 smithi006 bash[33918]: cluster 2023-08-27T23:48:53.620433+0000 mgr.y (mgr.14901) 3310 : cluster [DBG] pgmap v3317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:54 smithi006 bash[33918]: audit 2023-08-27T23:48:53.975937+0000 mon.c (mon.2) 1269 : audit [DBG] from='client.? 172.21.15.6:0/3462096059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:55.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:54 smithi130 bash[51302]: debug 2023-08-27T23:48:54.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:54.852898+0000) 2023-08-27T23:48:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:54 smithi130 bash[27579]: cluster 2023-08-27T23:48:53.620433+0000 mgr.y (mgr.14901) 3310 : cluster [DBG] pgmap v3317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:54 smithi130 bash[27579]: audit 2023-08-27T23:48:53.975937+0000 mon.c (mon.2) 1269 : audit [DBG] from='client.? 172.21.15.6:0/3462096059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:56.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:56 smithi130 bash[27579]: cluster 2023-08-27T23:48:55.621673+0000 mgr.y (mgr.14901) 3311 : cluster [DBG] pgmap v3318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:56.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:55 smithi130 bash[51302]: debug 2023-08-27T23:48:55.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:55.853171+0000) 2023-08-27T23:48:56.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:56 smithi006 bash[33918]: cluster 2023-08-27T23:48:55.621673+0000 mgr.y (mgr.14901) 3311 : cluster [DBG] pgmap v3318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:56.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:56 smithi006 bash[27292]: cluster 2023-08-27T23:48:55.621673+0000 mgr.y (mgr.14901) 3311 : cluster [DBG] pgmap v3318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:57.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:56 smithi130 bash[51302]: debug 2023-08-27T23:48:56.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:56.853474+0000) 2023-08-27T23:48:57.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:57 smithi130 bash[27579]: audit 2023-08-27T23:48:56.388432+0000 mon.a (mon.0) 2628 : audit [DBG] from='client.? 172.21.15.6:0/3147843621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:57.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:57 smithi006 bash[33918]: audit 2023-08-27T23:48:56.388432+0000 mon.a (mon.0) 2628 : audit [DBG] from='client.? 172.21.15.6:0/3147843621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:57.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:57 smithi006 bash[27292]: audit 2023-08-27T23:48:56.388432+0000 mon.a (mon.0) 2628 : audit [DBG] from='client.? 172.21.15.6:0/3147843621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:58.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:58 smithi130 bash[27579]: cluster 2023-08-27T23:48:57.622390+0000 mgr.y (mgr.14901) 3312 : cluster [DBG] pgmap v3319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:58.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:57 smithi130 bash[51302]: debug 2023-08-27T23:48:57.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:57.853777+0000) 2023-08-27T23:48:58.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:58 smithi006 bash[27292]: cluster 2023-08-27T23:48:57.622390+0000 mgr.y (mgr.14901) 3312 : cluster [DBG] pgmap v3319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:58.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:58 smithi006 bash[33918]: cluster 2023-08-27T23:48:57.622390+0000 mgr.y (mgr.14901) 3312 : cluster [DBG] pgmap v3319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:48:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:48:59 smithi130 bash[27579]: audit 2023-08-27T23:48:58.799317+0000 mon.a (mon.0) 2629 : audit [DBG] from='client.? 172.21.15.6:0/2784417127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:59.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:58 smithi130 bash[51302]: debug 2023-08-27T23:48:58.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:58.854067+0000) 2023-08-27T23:48:59.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:48:59 smithi006 bash[27292]: audit 2023-08-27T23:48:58.799317+0000 mon.a (mon.0) 2629 : audit [DBG] from='client.? 172.21.15.6:0/2784417127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:48:59 smithi006 bash[33918]: audit 2023-08-27T23:48:58.799317+0000 mon.a (mon.0) 2629 : audit [DBG] from='client.? 172.21.15.6:0/2784417127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:00.110 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:48:59 smithi130 bash[51302]: debug 2023-08-27T23:48:59.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:59.854353+0000) 2023-08-27T23:49:00.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:00 smithi130 bash[27579]: cluster 2023-08-27T23:48:59.623512+0000 mgr.y (mgr.14901) 3313 : cluster [DBG] pgmap v3320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:00.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:00 smithi006 bash[27292]: cluster 2023-08-27T23:48:59.623512+0000 mgr.y (mgr.14901) 3313 : cluster [DBG] pgmap v3320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:00.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:00 smithi006 bash[33918]: cluster 2023-08-27T23:48:59.623512+0000 mgr.y (mgr.14901) 3313 : cluster [DBG] pgmap v3320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:49:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:49:00] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-27T23:49:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:49:00.569Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:49:00.571Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:01.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:00 smithi130 bash[51302]: debug 2023-08-27T23:49:00.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:00.854615+0000) 2023-08-27T23:49:01.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:01 smithi130 bash[27579]: audit 2023-08-27T23:49:01.213259+0000 mon.c (mon.2) 1270 : audit [DBG] from='client.? 172.21.15.6:0/2893850706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:01.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:01 smithi006 bash[33918]: audit 2023-08-27T23:49:01.213259+0000 mon.c (mon.2) 1270 : audit [DBG] from='client.? 172.21.15.6:0/2893850706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:01.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:01 smithi006 bash[27292]: audit 2023-08-27T23:49:01.213259+0000 mon.c (mon.2) 1270 : audit [DBG] from='client.? 172.21.15.6:0/2893850706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:02.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:01 smithi130 bash[51302]: debug 2023-08-27T23:49:01.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:01.854914+0000) 2023-08-27T23:49:02.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:02 smithi006 bash[27292]: cluster 2023-08-27T23:49:01.624144+0000 mgr.y (mgr.14901) 3314 : cluster [DBG] pgmap v3321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:02.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:02 smithi006 bash[33918]: cluster 2023-08-27T23:49:01.624144+0000 mgr.y (mgr.14901) 3314 : cluster [DBG] pgmap v3321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:49:02.135Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:49:02.137Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:02.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:02 smithi130 bash[27579]: cluster 2023-08-27T23:49:01.624144+0000 mgr.y (mgr.14901) 3314 : cluster [DBG] pgmap v3321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:02 smithi130 bash[51302]: debug 2023-08-27T23:49:02.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:02.855177+0000) 2023-08-27T23:49:03.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:03 smithi006 bash[33918]: audit 2023-08-27T23:49:03.637445+0000 mon.c (mon.2) 1271 : audit [DBG] from='client.? 172.21.15.6:0/4286314677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:03.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:03 smithi006 bash[27292]: audit 2023-08-27T23:49:03.637445+0000 mon.c (mon.2) 1271 : audit [DBG] from='client.? 172.21.15.6:0/4286314677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:04.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:03 smithi130 bash[27579]: audit 2023-08-27T23:49:03.637445+0000 mon.c (mon.2) 1271 : audit [DBG] from='client.? 172.21.15.6:0/4286314677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:04.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:03 smithi130 bash[51302]: debug 2023-08-27T23:49:03.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:03.855509+0000) 2023-08-27T23:49:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:04 smithi006 bash[56298]: level=error ts=2023-08-27T23:49:04.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:04 smithi006 bash[56298]: level=warn ts=2023-08-27T23:49:04.036Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:04 smithi006 bash[33918]: cluster 2023-08-27T23:49:03.624795+0000 mgr.y (mgr.14901) 3315 : cluster [DBG] pgmap v3322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:04.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:04 smithi006 bash[27292]: cluster 2023-08-27T23:49:03.624795+0000 mgr.y (mgr.14901) 3315 : cluster [DBG] pgmap v3322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:04 smithi130 bash[27579]: cluster 2023-08-27T23:49:03.624795+0000 mgr.y (mgr.14901) 3315 : cluster [DBG] pgmap v3322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:04 smithi130 bash[51302]: debug 2023-08-27T23:49:04.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:04.855787+0000) 2023-08-27T23:49:06.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:05 smithi130 bash[51302]: debug 2023-08-27T23:49:05.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:05.856070+0000) 2023-08-27T23:49:06.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:06 smithi130 bash[27579]: cluster 2023-08-27T23:49:05.625903+0000 mgr.y (mgr.14901) 3316 : cluster [DBG] pgmap v3323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:06 smithi130 bash[27579]: audit 2023-08-27T23:49:06.051325+0000 mon.a (mon.0) 2630 : audit [DBG] from='client.? 172.21.15.6:0/2525239435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:06.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:06 smithi006 bash[33918]: cluster 2023-08-27T23:49:05.625903+0000 mgr.y (mgr.14901) 3316 : cluster [DBG] pgmap v3323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:06.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:06 smithi006 bash[33918]: audit 2023-08-27T23:49:06.051325+0000 mon.a (mon.0) 2630 : audit [DBG] from='client.? 172.21.15.6:0/2525239435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:06.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:06 smithi006 bash[27292]: cluster 2023-08-27T23:49:05.625903+0000 mgr.y (mgr.14901) 3316 : cluster [DBG] pgmap v3323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:06.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:06 smithi006 bash[27292]: audit 2023-08-27T23:49:06.051325+0000 mon.a (mon.0) 2630 : audit [DBG] from='client.? 172.21.15.6:0/2525239435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:07.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:06 smithi130 bash[51302]: debug 2023-08-27T23:49:06.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:06.856323+0000) 2023-08-27T23:49:08.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:07 smithi130 bash[51302]: debug 2023-08-27T23:49:07.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:07.856588+0000) 2023-08-27T23:49:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:08 smithi006 bash[33918]: cluster 2023-08-27T23:49:07.626501+0000 mgr.y (mgr.14901) 3317 : cluster [DBG] pgmap v3324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:08 smithi006 bash[33918]: audit 2023-08-27T23:49:08.466285+0000 mon.a (mon.0) 2631 : audit [DBG] from='client.? 172.21.15.6:0/1617520219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:08 smithi006 bash[27292]: cluster 2023-08-27T23:49:07.626501+0000 mgr.y (mgr.14901) 3317 : cluster [DBG] pgmap v3324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:08 smithi006 bash[27292]: audit 2023-08-27T23:49:08.466285+0000 mon.a (mon.0) 2631 : audit [DBG] from='client.? 172.21.15.6:0/1617520219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:08 smithi130 bash[27579]: cluster 2023-08-27T23:49:07.626501+0000 mgr.y (mgr.14901) 3317 : cluster [DBG] pgmap v3324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:08 smithi130 bash[27579]: audit 2023-08-27T23:49:08.466285+0000 mon.a (mon.0) 2631 : audit [DBG] from='client.? 172.21.15.6:0/1617520219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:09.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:08 smithi130 bash[51302]: debug 2023-08-27T23:49:08.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:08.856916+0000) 2023-08-27T23:49:10.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:09 smithi130 bash[51302]: debug 2023-08-27T23:49:09.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:09.857195+0000) 2023-08-27T23:49:10.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:10 smithi006 bash[27292]: cluster 2023-08-27T23:49:09.627569+0000 mgr.y (mgr.14901) 3318 : cluster [DBG] pgmap v3325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:49:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:49:10] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-27T23:49:10.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:10 smithi006 bash[33918]: cluster 2023-08-27T23:49:09.627569+0000 mgr.y (mgr.14901) 3318 : cluster [DBG] pgmap v3325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:49:10.570Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:49:10.572Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:10 smithi130 bash[27579]: cluster 2023-08-27T23:49:09.627569+0000 mgr.y (mgr.14901) 3318 : cluster [DBG] pgmap v3325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:11.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:10 smithi130 bash[51302]: debug 2023-08-27T23:49:10.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:10.857495+0000) 2023-08-27T23:49:11.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:11 smithi006 bash[33918]: audit 2023-08-27T23:49:10.880668+0000 mon.a (mon.0) 2632 : audit [DBG] from='client.? 172.21.15.6:0/540899058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:11.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:11 smithi006 bash[27292]: audit 2023-08-27T23:49:10.880668+0000 mon.a (mon.0) 2632 : audit [DBG] from='client.? 172.21.15.6:0/540899058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:12.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:11 smithi130 bash[27579]: audit 2023-08-27T23:49:10.880668+0000 mon.a (mon.0) 2632 : audit [DBG] from='client.? 172.21.15.6:0/540899058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:12.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:11 smithi130 bash[51302]: debug 2023-08-27T23:49:11.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:11.857770+0000) 2023-08-27T23:49:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:49:12.136Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:49:12.138Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:12 smithi006 bash[27292]: cluster 2023-08-27T23:49:11.628191+0000 mgr.y (mgr.14901) 3319 : cluster [DBG] pgmap v3326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:12 smithi006 bash[33918]: cluster 2023-08-27T23:49:11.628191+0000 mgr.y (mgr.14901) 3319 : cluster [DBG] pgmap v3326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:12 smithi130 bash[27579]: cluster 2023-08-27T23:49:11.628191+0000 mgr.y (mgr.14901) 3319 : cluster [DBG] pgmap v3326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:13.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:12 smithi130 bash[51302]: debug 2023-08-27T23:49:12.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:12.858093+0000) 2023-08-27T23:49:13.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:13 smithi006 bash[27292]: audit 2023-08-27T23:49:13.289821+0000 mon.c (mon.2) 1272 : audit [DBG] from='client.? 172.21.15.6:0/4008991564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:13.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:13 smithi006 bash[33918]: audit 2023-08-27T23:49:13.289821+0000 mon.c (mon.2) 1272 : audit [DBG] from='client.? 172.21.15.6:0/4008991564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:14.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:13 smithi130 bash[27579]: audit 2023-08-27T23:49:13.289821+0000 mon.c (mon.2) 1272 : audit [DBG] from='client.? 172.21.15.6:0/4008991564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:14.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:13 smithi130 bash[51302]: debug 2023-08-27T23:49:13.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:13.858362+0000) 2023-08-27T23:49:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:14 smithi006 bash[56298]: level=error ts=2023-08-27T23:49:14.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:14 smithi006 bash[56298]: level=warn ts=2023-08-27T23:49:14.036Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:14 smithi006 bash[33918]: cluster 2023-08-27T23:49:13.628855+0000 mgr.y (mgr.14901) 3320 : cluster [DBG] pgmap v3327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:14 smithi006 bash[27292]: cluster 2023-08-27T23:49:13.628855+0000 mgr.y (mgr.14901) 3320 : cluster [DBG] pgmap v3327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:14 smithi130 bash[27579]: cluster 2023-08-27T23:49:13.628855+0000 mgr.y (mgr.14901) 3320 : cluster [DBG] pgmap v3327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:15.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:14 smithi130 bash[51302]: debug 2023-08-27T23:49:14.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:14.858689+0000) 2023-08-27T23:49:16.111 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:15 smithi130 bash[51302]: debug 2023-08-27T23:49:15.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:15.858986+0000) 2023-08-27T23:49:16.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:16 smithi130 bash[27579]: cluster 2023-08-27T23:49:15.630021+0000 mgr.y (mgr.14901) 3321 : cluster [DBG] pgmap v3328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:16 smithi130 bash[27579]: audit 2023-08-27T23:49:15.705265+0000 mon.a (mon.0) 2633 : audit [DBG] from='client.? 172.21.15.6:0/4057381329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:16.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:16 smithi006 bash[27292]: cluster 2023-08-27T23:49:15.630021+0000 mgr.y (mgr.14901) 3321 : cluster [DBG] pgmap v3328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:16.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:16 smithi006 bash[27292]: audit 2023-08-27T23:49:15.705265+0000 mon.a (mon.0) 2633 : audit [DBG] from='client.? 172.21.15.6:0/4057381329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:16.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:16 smithi006 bash[33918]: cluster 2023-08-27T23:49:15.630021+0000 mgr.y (mgr.14901) 3321 : cluster [DBG] pgmap v3328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:16.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:16 smithi006 bash[33918]: audit 2023-08-27T23:49:15.705265+0000 mon.a (mon.0) 2633 : audit [DBG] from='client.? 172.21.15.6:0/4057381329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:17.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:16 smithi130 bash[51302]: debug 2023-08-27T23:49:16.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:16.859293+0000) 2023-08-27T23:49:18.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:17 smithi130 bash[51302]: debug 2023-08-27T23:49:17.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:17.859516+0000) 2023-08-27T23:49:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:18 smithi006 bash[27292]: cluster 2023-08-27T23:49:17.630718+0000 mgr.y (mgr.14901) 3322 : cluster [DBG] pgmap v3329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:18 smithi006 bash[27292]: audit 2023-08-27T23:49:18.119869+0000 mon.c (mon.2) 1273 : audit [DBG] from='client.? 172.21.15.6:0/2692641750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:18 smithi006 bash[33918]: cluster 2023-08-27T23:49:17.630718+0000 mgr.y (mgr.14901) 3322 : cluster [DBG] pgmap v3329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:18 smithi006 bash[33918]: audit 2023-08-27T23:49:18.119869+0000 mon.c (mon.2) 1273 : audit [DBG] from='client.? 172.21.15.6:0/2692641750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:19.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:18 smithi130 bash[51302]: debug 2023-08-27T23:49:18.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:18.859835+0000) 2023-08-27T23:49:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:18 smithi130 bash[27579]: cluster 2023-08-27T23:49:17.630718+0000 mgr.y (mgr.14901) 3322 : cluster [DBG] pgmap v3329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:18 smithi130 bash[27579]: audit 2023-08-27T23:49:18.119869+0000 mon.c (mon.2) 1273 : audit [DBG] from='client.? 172.21.15.6:0/2692641750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:19.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:19 smithi006 bash[27292]: audit 2023-08-27T23:49:18.864448+0000 mon.b (mon.1) 433 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:49:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:19 smithi006 bash[33918]: audit 2023-08-27T23:49:18.864448+0000 mon.b (mon.1) 433 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:49:20.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:19 smithi130 bash[27579]: audit 2023-08-27T23:49:18.864448+0000 mon.b (mon.1) 433 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:49:20.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:19 smithi130 bash[51302]: debug 2023-08-27T23:49:19.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:19.860133+0000) 2023-08-27T23:49:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:49:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:49:20] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-27T23:49:20.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:20 smithi006 bash[27292]: cluster 2023-08-27T23:49:19.631838+0000 mgr.y (mgr.14901) 3323 : cluster [DBG] pgmap v3330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:20.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:20 smithi006 bash[27292]: audit 2023-08-27T23:49:20.545170+0000 mon.a (mon.0) 2634 : audit [DBG] from='client.? 172.21.15.6:0/2910404177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:20.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:49:20.570Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:20.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:49:20.572Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:20.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:20 smithi006 bash[33918]: cluster 2023-08-27T23:49:19.631838+0000 mgr.y (mgr.14901) 3323 : cluster [DBG] pgmap v3330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:20.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:20 smithi006 bash[33918]: audit 2023-08-27T23:49:20.545170+0000 mon.a (mon.0) 2634 : audit [DBG] from='client.? 172.21.15.6:0/2910404177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:21.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:20 smithi130 bash[51302]: debug 2023-08-27T23:49:20.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:20.860429+0000) 2023-08-27T23:49:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:20 smithi130 bash[27579]: cluster 2023-08-27T23:49:19.631838+0000 mgr.y (mgr.14901) 3323 : cluster [DBG] pgmap v3330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:20 smithi130 bash[27579]: audit 2023-08-27T23:49:20.545170+0000 mon.a (mon.0) 2634 : audit [DBG] from='client.? 172.21.15.6:0/2910404177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:22.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:21 smithi130 bash[51302]: debug 2023-08-27T23:49:21.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:21.860658+0000) 2023-08-27T23:49:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:49:22.137Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:49:22.139Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:22 smithi006 bash[27292]: cluster 2023-08-27T23:49:21.632486+0000 mgr.y (mgr.14901) 3324 : cluster [DBG] pgmap v3331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:22 smithi006 bash[33918]: cluster 2023-08-27T23:49:21.632486+0000 mgr.y (mgr.14901) 3324 : cluster [DBG] pgmap v3331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:22 smithi130 bash[27579]: cluster 2023-08-27T23:49:21.632486+0000 mgr.y (mgr.14901) 3324 : cluster [DBG] pgmap v3331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:23.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:22 smithi130 bash[51302]: debug 2023-08-27T23:49:22.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:22.860961+0000) 2023-08-27T23:49:24.038 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:23 smithi006 bash[33918]: audit 2023-08-27T23:49:22.963999+0000 mon.a (mon.0) 2635 : audit [DBG] from='client.? 172.21.15.6:0/646078651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:24.039 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:23 smithi006 bash[27292]: audit 2023-08-27T23:49:22.963999+0000 mon.a (mon.0) 2635 : audit [DBG] from='client.? 172.21.15.6:0/646078651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:24.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:23 smithi130 bash[27579]: audit 2023-08-27T23:49:22.963999+0000 mon.a (mon.0) 2635 : audit [DBG] from='client.? 172.21.15.6:0/646078651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:24.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:23 smithi130 bash[51302]: debug 2023-08-27T23:49:23.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:23.861182+0000) 2023-08-27T23:49:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:24 smithi006 bash[56298]: level=error ts=2023-08-27T23:49:24.035Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:24 smithi006 bash[56298]: level=warn ts=2023-08-27T23:49:24.038Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:24 smithi130 bash[27579]: cluster 2023-08-27T23:49:23.633105+0000 mgr.y (mgr.14901) 3325 : cluster [DBG] pgmap v3332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:24 smithi130 bash[51302]: debug 2023-08-27T23:49:24.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:24.861471+0000) 2023-08-27T23:49:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:24 smithi006 bash[33918]: cluster 2023-08-27T23:49:23.633105+0000 mgr.y (mgr.14901) 3325 : cluster [DBG] pgmap v3332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:24 smithi006 bash[27292]: cluster 2023-08-27T23:49:23.633105+0000 mgr.y (mgr.14901) 3325 : cluster [DBG] pgmap v3332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:25 smithi130 bash[27579]: audit 2023-08-27T23:49:24.789954+0000 mon.a (mon.0) 2636 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:49:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:25 smithi130 bash[27579]: audit 2023-08-27T23:49:25.226790+0000 mon.a (mon.0) 2637 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:49:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:25 smithi130 bash[27579]: audit 2023-08-27T23:49:25.384863+0000 mon.c (mon.2) 1274 : audit [DBG] from='client.? 172.21.15.6:0/2197918331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:25 smithi130 bash[27579]: audit 2023-08-27T23:49:25.520232+0000 mon.b (mon.1) 434 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:49:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:25 smithi130 bash[27579]: audit 2023-08-27T23:49:25.522631+0000 mon.b (mon.1) 435 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:49:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:25 smithi130 bash[27579]: audit 2023-08-27T23:49:25.530504+0000 mon.a (mon.0) 2638 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:49:26.120 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:25 smithi130 bash[51302]: debug 2023-08-27T23:49:25.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:25.861737+0000) 2023-08-27T23:49:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:25 smithi006 bash[27292]: audit 2023-08-27T23:49:24.789954+0000 mon.a (mon.0) 2636 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:49:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:25 smithi006 bash[27292]: audit 2023-08-27T23:49:25.226790+0000 mon.a (mon.0) 2637 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:49:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:25 smithi006 bash[27292]: audit 2023-08-27T23:49:25.384863+0000 mon.c (mon.2) 1274 : audit [DBG] from='client.? 172.21.15.6:0/2197918331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:25 smithi006 bash[27292]: audit 2023-08-27T23:49:25.520232+0000 mon.b (mon.1) 434 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:49:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:25 smithi006 bash[27292]: audit 2023-08-27T23:49:25.522631+0000 mon.b (mon.1) 435 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:49:26.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:25 smithi006 bash[27292]: audit 2023-08-27T23:49:25.530504+0000 mon.a (mon.0) 2638 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:49:26.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:25 smithi006 bash[33918]: audit 2023-08-27T23:49:24.789954+0000 mon.a (mon.0) 2636 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:49:26.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:25 smithi006 bash[33918]: audit 2023-08-27T23:49:25.226790+0000 mon.a (mon.0) 2637 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:49:26.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:25 smithi006 bash[33918]: audit 2023-08-27T23:49:25.384863+0000 mon.c (mon.2) 1274 : audit [DBG] from='client.? 172.21.15.6:0/2197918331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:26.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:25 smithi006 bash[33918]: audit 2023-08-27T23:49:25.520232+0000 mon.b (mon.1) 434 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:49:26.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:25 smithi006 bash[33918]: audit 2023-08-27T23:49:25.522631+0000 mon.b (mon.1) 435 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:49:26.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:25 smithi006 bash[33918]: audit 2023-08-27T23:49:25.530504+0000 mon.a (mon.0) 2638 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:49:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:26 smithi130 bash[27579]: cluster 2023-08-27T23:49:25.634195+0000 mgr.y (mgr.14901) 3326 : cluster [DBG] pgmap v3333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:27.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:26 smithi130 bash[51302]: debug 2023-08-27T23:49:26.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:26.862057+0000) 2023-08-27T23:49:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:26 smithi006 bash[33918]: cluster 2023-08-27T23:49:25.634195+0000 mgr.y (mgr.14901) 3326 : cluster [DBG] pgmap v3333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:26 smithi006 bash[27292]: cluster 2023-08-27T23:49:25.634195+0000 mgr.y (mgr.14901) 3326 : cluster [DBG] pgmap v3333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:28.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:27 smithi130 bash[51302]: debug 2023-08-27T23:49:27.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:27.862269+0000) 2023-08-27T23:49:28.801 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:28 smithi006 bash[27292]: cluster 2023-08-27T23:49:27.634771+0000 mgr.y (mgr.14901) 3327 : cluster [DBG] pgmap v3334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:28.802 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:28 smithi006 bash[27292]: audit 2023-08-27T23:49:27.798484+0000 mon.c (mon.2) 1275 : audit [DBG] from='client.? 172.21.15.6:0/2445815600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:28 smithi130 bash[27579]: cluster 2023-08-27T23:49:27.634771+0000 mgr.y (mgr.14901) 3327 : cluster [DBG] pgmap v3334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:28 smithi130 bash[27579]: audit 2023-08-27T23:49:27.798484+0000 mon.c (mon.2) 1275 : audit [DBG] from='client.? 172.21.15.6:0/2445815600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:29.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:28 smithi130 bash[51302]: debug 2023-08-27T23:49:28.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:28.862558+0000) 2023-08-27T23:49:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:28 smithi006 bash[33918]: cluster 2023-08-27T23:49:27.634771+0000 mgr.y (mgr.14901) 3327 : cluster [DBG] pgmap v3334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:28 smithi006 bash[33918]: audit 2023-08-27T23:49:27.798484+0000 mon.c (mon.2) 1275 : audit [DBG] from='client.? 172.21.15.6:0/2445815600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:30.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:29 smithi130 bash[51302]: debug 2023-08-27T23:49:29.862+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:29.862887+0000) 2023-08-27T23:49:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:49:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:49:30] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-27T23:49:30.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:30 smithi006 bash[33918]: cluster 2023-08-27T23:49:29.635920+0000 mgr.y (mgr.14901) 3328 : cluster [DBG] pgmap v3335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:30.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:30 smithi006 bash[33918]: audit 2023-08-27T23:49:30.220038+0000 mon.c (mon.2) 1276 : audit [DBG] from='client.? 172.21.15.6:0/354231395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:30.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:30 smithi006 bash[27292]: cluster 2023-08-27T23:49:29.635920+0000 mgr.y (mgr.14901) 3328 : cluster [DBG] pgmap v3335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:30.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:30 smithi006 bash[27292]: audit 2023-08-27T23:49:30.220038+0000 mon.c (mon.2) 1276 : audit [DBG] from='client.? 172.21.15.6:0/354231395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:49:30.571Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:49:30.574Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:31.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:30 smithi130 bash[51302]: debug 2023-08-27T23:49:30.862+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:30.863172+0000) 2023-08-27T23:49:31.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:30 smithi130 bash[27579]: cluster 2023-08-27T23:49:29.635920+0000 mgr.y (mgr.14901) 3328 : cluster [DBG] pgmap v3335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:31.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:30 smithi130 bash[27579]: audit 2023-08-27T23:49:30.220038+0000 mon.c (mon.2) 1276 : audit [DBG] from='client.? 172.21.15.6:0/354231395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:32.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:31 smithi130 bash[51302]: debug 2023-08-27T23:49:31.862+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:31.863473+0000) 2023-08-27T23:49:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:49:32.138Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:49:32.140Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:33.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:32 smithi130 bash[51302]: debug 2023-08-27T23:49:32.862+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:32.863697+0000) 2023-08-27T23:49:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:32 smithi130 bash[27579]: cluster 2023-08-27T23:49:31.636526+0000 mgr.y (mgr.14901) 3329 : cluster [DBG] pgmap v3336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:32 smithi130 bash[27579]: audit 2023-08-27T23:49:32.634809+0000 mon.c (mon.2) 1277 : audit [DBG] from='client.? 172.21.15.6:0/2045804559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:32 smithi006 bash[33918]: cluster 2023-08-27T23:49:31.636526+0000 mgr.y (mgr.14901) 3329 : cluster [DBG] pgmap v3336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:32 smithi006 bash[33918]: audit 2023-08-27T23:49:32.634809+0000 mon.c (mon.2) 1277 : audit [DBG] from='client.? 172.21.15.6:0/2045804559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:32 smithi006 bash[27292]: cluster 2023-08-27T23:49:31.636526+0000 mgr.y (mgr.14901) 3329 : cluster [DBG] pgmap v3336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:32 smithi006 bash[27292]: audit 2023-08-27T23:49:32.634809+0000 mon.c (mon.2) 1277 : audit [DBG] from='client.? 172.21.15.6:0/2045804559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:34.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:33 smithi130 bash[51302]: debug 2023-08-27T23:49:33.862+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:33.863979+0000) 2023-08-27T23:49:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:34 smithi006 bash[56298]: level=error ts=2023-08-27T23:49:34.036Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:34 smithi006 bash[56298]: level=warn ts=2023-08-27T23:49:34.038Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:34 smithi130 bash[27579]: cluster 2023-08-27T23:49:33.637162+0000 mgr.y (mgr.14901) 3330 : cluster [DBG] pgmap v3337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:35.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:34 smithi130 bash[51302]: debug 2023-08-27T23:49:34.862+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:34.864248+0000) 2023-08-27T23:49:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:34 smithi006 bash[27292]: cluster 2023-08-27T23:49:33.637162+0000 mgr.y (mgr.14901) 3330 : cluster [DBG] pgmap v3337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:34 smithi006 bash[33918]: cluster 2023-08-27T23:49:33.637162+0000 mgr.y (mgr.14901) 3330 : cluster [DBG] pgmap v3337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:36.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:35 smithi130 bash[27579]: audit 2023-08-27T23:49:35.053442+0000 mon.c (mon.2) 1278 : audit [DBG] from='client.? 172.21.15.6:0/715630527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:36.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:35 smithi130 bash[51302]: debug 2023-08-27T23:49:35.861+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:35.864531+0000) 2023-08-27T23:49:36.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:35 smithi006 bash[27292]: audit 2023-08-27T23:49:35.053442+0000 mon.c (mon.2) 1278 : audit [DBG] from='client.? 172.21.15.6:0/715630527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:36.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:35 smithi006 bash[33918]: audit 2023-08-27T23:49:35.053442+0000 mon.c (mon.2) 1278 : audit [DBG] from='client.? 172.21.15.6:0/715630527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:37.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:36 smithi130 bash[27579]: cluster 2023-08-27T23:49:35.638252+0000 mgr.y (mgr.14901) 3331 : cluster [DBG] pgmap v3338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:37.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:36 smithi130 bash[51302]: debug 2023-08-27T23:49:36.861+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:36.864829+0000) 2023-08-27T23:49:37.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:36 smithi006 bash[27292]: cluster 2023-08-27T23:49:35.638252+0000 mgr.y (mgr.14901) 3331 : cluster [DBG] pgmap v3338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:37.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:36 smithi006 bash[33918]: cluster 2023-08-27T23:49:35.638252+0000 mgr.y (mgr.14901) 3331 : cluster [DBG] pgmap v3338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:38.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:37 smithi130 bash[27579]: audit 2023-08-27T23:49:37.460632+0000 mon.c (mon.2) 1279 : audit [DBG] from='client.? 172.21.15.6:0/1749576023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:38.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:37 smithi130 bash[51302]: debug 2023-08-27T23:49:37.861+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:37.865138+0000) 2023-08-27T23:49:38.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:37 smithi006 bash[33918]: audit 2023-08-27T23:49:37.460632+0000 mon.c (mon.2) 1279 : audit [DBG] from='client.? 172.21.15.6:0/1749576023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:38.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:37 smithi006 bash[27292]: audit 2023-08-27T23:49:37.460632+0000 mon.c (mon.2) 1279 : audit [DBG] from='client.? 172.21.15.6:0/1749576023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:38 smithi130 bash[27579]: cluster 2023-08-27T23:49:37.638940+0000 mgr.y (mgr.14901) 3332 : cluster [DBG] pgmap v3339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:39.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:38 smithi130 bash[51302]: debug 2023-08-27T23:49:38.861+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:38.865423+0000) 2023-08-27T23:49:39.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:38 smithi006 bash[27292]: cluster 2023-08-27T23:49:37.638940+0000 mgr.y (mgr.14901) 3332 : cluster [DBG] pgmap v3339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:39.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:38 smithi006 bash[33918]: cluster 2023-08-27T23:49:37.638940+0000 mgr.y (mgr.14901) 3332 : cluster [DBG] pgmap v3339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:40.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:39 smithi130 bash[51302]: debug 2023-08-27T23:49:39.861+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:39.865694+0000) 2023-08-27T23:49:40.838 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:49:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:49:40] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-27T23:49:40.838 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:49:40.572Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:40.838 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:49:40.575Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:40 smithi130 bash[27579]: cluster 2023-08-27T23:49:39.640026+0000 mgr.y (mgr.14901) 3333 : cluster [DBG] pgmap v3340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:40 smithi130 bash[27579]: audit 2023-08-27T23:49:39.869194+0000 mon.a (mon.0) 2639 : audit [DBG] from='client.? 172.21.15.6:0/1043227374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:41.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:40 smithi130 bash[51302]: debug 2023-08-27T23:49:40.861+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:40.865985+0000) 2023-08-27T23:49:41.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:40 smithi006 bash[33918]: cluster 2023-08-27T23:49:39.640026+0000 mgr.y (mgr.14901) 3333 : cluster [DBG] pgmap v3340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:41.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:40 smithi006 bash[33918]: audit 2023-08-27T23:49:39.869194+0000 mon.a (mon.0) 2639 : audit [DBG] from='client.? 172.21.15.6:0/1043227374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:40 smithi006 bash[27292]: cluster 2023-08-27T23:49:39.640026+0000 mgr.y (mgr.14901) 3333 : cluster [DBG] pgmap v3340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:40 smithi006 bash[27292]: audit 2023-08-27T23:49:39.869194+0000 mon.a (mon.0) 2639 : audit [DBG] from='client.? 172.21.15.6:0/1043227374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:42.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:41 smithi130 bash[51302]: debug 2023-08-27T23:49:41.861+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:41.866265+0000) 2023-08-27T23:49:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:49:42.139Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:49:42.141Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:42 smithi130 bash[27579]: cluster 2023-08-27T23:49:41.640680+0000 mgr.y (mgr.14901) 3334 : cluster [DBG] pgmap v3341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:42 smithi130 bash[27579]: audit 2023-08-27T23:49:42.295651+0000 mon.a (mon.0) 2640 : audit [DBG] from='client.? 172.21.15.6:0/1862713922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:43.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:42 smithi130 bash[51302]: debug 2023-08-27T23:49:42.861+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:42.866557+0000) 2023-08-27T23:49:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:42 smithi006 bash[33918]: cluster 2023-08-27T23:49:41.640680+0000 mgr.y (mgr.14901) 3334 : cluster [DBG] pgmap v3341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:42 smithi006 bash[33918]: audit 2023-08-27T23:49:42.295651+0000 mon.a (mon.0) 2640 : audit [DBG] from='client.? 172.21.15.6:0/1862713922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:43.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:42 smithi006 bash[27292]: cluster 2023-08-27T23:49:41.640680+0000 mgr.y (mgr.14901) 3334 : cluster [DBG] pgmap v3341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:43.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:42 smithi006 bash[27292]: audit 2023-08-27T23:49:42.295651+0000 mon.a (mon.0) 2640 : audit [DBG] from='client.? 172.21.15.6:0/1862713922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:44.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:43 smithi130 bash[51302]: debug 2023-08-27T23:49:43.861+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:43.866785+0000) 2023-08-27T23:49:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:44 smithi006 bash[56298]: level=error ts=2023-08-27T23:49:44.037Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:44 smithi006 bash[56298]: level=warn ts=2023-08-27T23:49:44.039Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:44 smithi130 bash[51302]: debug 2023-08-27T23:49:44.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:44.867056+0000) 2023-08-27T23:49:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:44 smithi130 bash[27579]: cluster 2023-08-27T23:49:43.641333+0000 mgr.y (mgr.14901) 3335 : cluster [DBG] pgmap v3342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:44 smithi130 bash[27579]: audit 2023-08-27T23:49:44.719107+0000 mon.c (mon.2) 1280 : audit [DBG] from='client.? 172.21.15.6:0/3970215015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:44 smithi006 bash[27292]: cluster 2023-08-27T23:49:43.641333+0000 mgr.y (mgr.14901) 3335 : cluster [DBG] pgmap v3342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:44 smithi006 bash[27292]: audit 2023-08-27T23:49:44.719107+0000 mon.c (mon.2) 1280 : audit [DBG] from='client.? 172.21.15.6:0/3970215015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:44 smithi006 bash[33918]: cluster 2023-08-27T23:49:43.641333+0000 mgr.y (mgr.14901) 3335 : cluster [DBG] pgmap v3342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:45.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:44 smithi006 bash[33918]: audit 2023-08-27T23:49:44.719107+0000 mon.c (mon.2) 1280 : audit [DBG] from='client.? 172.21.15.6:0/3970215015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:46 smithi130 bash[27579]: cluster 2023-08-27T23:49:45.642516+0000 mgr.y (mgr.14901) 3336 : cluster [DBG] pgmap v3343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:45 smithi130 bash[51302]: debug 2023-08-27T23:49:45.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:45.867326+0000) 2023-08-27T23:49:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:46 smithi006 bash[33918]: cluster 2023-08-27T23:49:45.642516+0000 mgr.y (mgr.14901) 3336 : cluster [DBG] pgmap v3343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:46.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:46 smithi006 bash[27292]: cluster 2023-08-27T23:49:45.642516+0000 mgr.y (mgr.14901) 3336 : cluster [DBG] pgmap v3343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:47.193 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:46 smithi130 bash[51302]: debug 2023-08-27T23:49:46.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:46.867654+0000) 2023-08-27T23:49:47.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:47 smithi006 bash[33918]: audit 2023-08-27T23:49:47.136314+0000 mon.c (mon.2) 1281 : audit [DBG] from='client.? 172.21.15.6:0/1698891298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:47.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:47 smithi006 bash[27292]: audit 2023-08-27T23:49:47.136314+0000 mon.c (mon.2) 1281 : audit [DBG] from='client.? 172.21.15.6:0/1698891298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:47.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:47 smithi130 bash[27579]: audit 2023-08-27T23:49:47.136314+0000 mon.c (mon.2) 1281 : audit [DBG] from='client.? 172.21.15.6:0/1698891298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:48.203 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:47 smithi130 bash[51302]: debug 2023-08-27T23:49:47.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:47.867968+0000) 2023-08-27T23:49:48.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:48 smithi006 bash[27292]: cluster 2023-08-27T23:49:47.643187+0000 mgr.y (mgr.14901) 3337 : cluster [DBG] pgmap v3344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:48.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:48 smithi006 bash[33918]: cluster 2023-08-27T23:49:47.643187+0000 mgr.y (mgr.14901) 3337 : cluster [DBG] pgmap v3344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:48 smithi130 bash[27579]: cluster 2023-08-27T23:49:47.643187+0000 mgr.y (mgr.14901) 3337 : cluster [DBG] pgmap v3344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:48 smithi130 bash[51302]: debug 2023-08-27T23:49:48.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:48.868315+0000) 2023-08-27T23:49:49.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:49 smithi130 bash[27579]: audit 2023-08-27T23:49:49.552229+0000 mon.a (mon.0) 2641 : audit [DBG] from='client.? 172.21.15.6:0/482493135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:49.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:49 smithi006 bash[27292]: audit 2023-08-27T23:49:49.552229+0000 mon.a (mon.0) 2641 : audit [DBG] from='client.? 172.21.15.6:0/482493135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:49.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:49 smithi006 bash[33918]: audit 2023-08-27T23:49:49.552229+0000 mon.a (mon.0) 2641 : audit [DBG] from='client.? 172.21.15.6:0/482493135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:49 smithi130 bash[51302]: debug 2023-08-27T23:49:49.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:49.868560+0000) 2023-08-27T23:49:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:50 smithi130 bash[27579]: cluster 2023-08-27T23:49:49.644336+0000 mgr.y (mgr.14901) 3338 : cluster [DBG] pgmap v3345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:49:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:49:50] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:49:50.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:50 smithi006 bash[33918]: cluster 2023-08-27T23:49:49.644336+0000 mgr.y (mgr.14901) 3338 : cluster [DBG] pgmap v3345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:50.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:50 smithi006 bash[27292]: cluster 2023-08-27T23:49:49.644336+0000 mgr.y (mgr.14901) 3338 : cluster [DBG] pgmap v3345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:49:50.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:49:50.575Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:51.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:50 smithi130 bash[51302]: debug 2023-08-27T23:49:50.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:50.868856+0000) 2023-08-27T23:49:52.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:51 smithi130 bash[51302]: debug 2023-08-27T23:49:51.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:51.869120+0000) 2023-08-27T23:49:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:49:52.139Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:49:52.141Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:52 smithi006 bash[33918]: cluster 2023-08-27T23:49:51.644956+0000 mgr.y (mgr.14901) 3339 : cluster [DBG] pgmap v3346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:52 smithi006 bash[33918]: audit 2023-08-27T23:49:51.966502+0000 mon.c (mon.2) 1282 : audit [DBG] from='client.? 172.21.15.6:0/1694578771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:52 smithi006 bash[27292]: cluster 2023-08-27T23:49:51.644956+0000 mgr.y (mgr.14901) 3339 : cluster [DBG] pgmap v3346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:52 smithi006 bash[27292]: audit 2023-08-27T23:49:51.966502+0000 mon.c (mon.2) 1282 : audit [DBG] from='client.? 172.21.15.6:0/1694578771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:53.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:52 smithi130 bash[51302]: debug 2023-08-27T23:49:52.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:52.869393+0000) 2023-08-27T23:49:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:52 smithi130 bash[27579]: cluster 2023-08-27T23:49:51.644956+0000 mgr.y (mgr.14901) 3339 : cluster [DBG] pgmap v3346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:52 smithi130 bash[27579]: audit 2023-08-27T23:49:51.966502+0000 mon.c (mon.2) 1282 : audit [DBG] from='client.? 172.21.15.6:0/1694578771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:54.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:53 smithi130 bash[51302]: debug 2023-08-27T23:49:53.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:53.869684+0000) 2023-08-27T23:49:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:54 smithi006 bash[56298]: level=error ts=2023-08-27T23:49:54.038Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:49:54 smithi006 bash[56298]: level=warn ts=2023-08-27T23:49:54.040Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:49:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:54 smithi006 bash[33918]: cluster 2023-08-27T23:49:53.645604+0000 mgr.y (mgr.14901) 3340 : cluster [DBG] pgmap v3347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:54 smithi006 bash[33918]: audit 2023-08-27T23:49:54.400969+0000 mon.c (mon.2) 1283 : audit [DBG] from='client.? 172.21.15.6:0/3936947951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:54.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:54 smithi006 bash[27292]: cluster 2023-08-27T23:49:53.645604+0000 mgr.y (mgr.14901) 3340 : cluster [DBG] pgmap v3347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:54.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:54 smithi006 bash[27292]: audit 2023-08-27T23:49:54.400969+0000 mon.c (mon.2) 1283 : audit [DBG] from='client.? 172.21.15.6:0/3936947951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:54 smithi130 bash[27579]: cluster 2023-08-27T23:49:53.645604+0000 mgr.y (mgr.14901) 3340 : cluster [DBG] pgmap v3347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:54 smithi130 bash[27579]: audit 2023-08-27T23:49:54.400969+0000 mon.c (mon.2) 1283 : audit [DBG] from='client.? 172.21.15.6:0/3936947951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:54 smithi130 bash[51302]: debug 2023-08-27T23:49:54.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:54.869998+0000) 2023-08-27T23:49:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:56 smithi130 bash[27579]: cluster 2023-08-27T23:49:55.646652+0000 mgr.y (mgr.14901) 3341 : cluster [DBG] pgmap v3348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:56.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:55 smithi130 bash[51302]: debug 2023-08-27T23:49:55.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:55.870290+0000) 2023-08-27T23:49:56.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:56 smithi006 bash[27292]: cluster 2023-08-27T23:49:55.646652+0000 mgr.y (mgr.14901) 3341 : cluster [DBG] pgmap v3348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:56.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:56 smithi006 bash[33918]: cluster 2023-08-27T23:49:55.646652+0000 mgr.y (mgr.14901) 3341 : cluster [DBG] pgmap v3348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:57.124 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:56 smithi130 bash[51302]: debug 2023-08-27T23:49:56.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:56.870558+0000) 2023-08-27T23:49:57.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:57 smithi006 bash[33918]: audit 2023-08-27T23:49:56.809424+0000 mon.c (mon.2) 1284 : audit [DBG] from='client.? 172.21.15.6:0/2067088826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:57.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:57 smithi006 bash[27292]: audit 2023-08-27T23:49:56.809424+0000 mon.c (mon.2) 1284 : audit [DBG] from='client.? 172.21.15.6:0/2067088826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:57.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:57 smithi130 bash[27579]: audit 2023-08-27T23:49:56.809424+0000 mon.c (mon.2) 1284 : audit [DBG] from='client.? 172.21.15.6:0/2067088826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:58.129 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:57 smithi130 bash[51302]: debug 2023-08-27T23:49:57.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:57.870835+0000) 2023-08-27T23:49:58.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:58 smithi006 bash[33918]: cluster 2023-08-27T23:49:57.647309+0000 mgr.y (mgr.14901) 3342 : cluster [DBG] pgmap v3349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:58.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:58 smithi006 bash[27292]: cluster 2023-08-27T23:49:57.647309+0000 mgr.y (mgr.14901) 3342 : cluster [DBG] pgmap v3349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:58.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:58 smithi130 bash[27579]: cluster 2023-08-27T23:49:57.647309+0000 mgr.y (mgr.14901) 3342 : cluster [DBG] pgmap v3349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:49:59.292 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:58 smithi130 bash[51302]: debug 2023-08-27T23:49:58.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:58.871111+0000) 2023-08-27T23:49:59.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:49:59 smithi130 bash[27579]: audit 2023-08-27T23:49:59.235261+0000 mon.c (mon.2) 1285 : audit [DBG] from='client.? 172.21.15.6:0/3617856543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:59.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:49:59 smithi006 bash[27292]: audit 2023-08-27T23:49:59.235261+0000 mon.c (mon.2) 1285 : audit [DBG] from='client.? 172.21.15.6:0/3617856543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:59.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:49:59 smithi006 bash[33918]: audit 2023-08-27T23:49:59.235261+0000 mon.c (mon.2) 1285 : audit [DBG] from='client.? 172.21.15.6:0/3617856543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:00.301 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:49:59 smithi130 bash[51302]: debug 2023-08-27T23:49:59.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:59.871328+0000) 2023-08-27T23:50:00.577 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:00 smithi006 bash[27292]: cluster 2023-08-27T23:49:59.648454+0000 mgr.y (mgr.14901) 3343 : cluster [DBG] pgmap v3350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:00.577 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:00 smithi006 bash[27292]: cluster 2023-08-27T23:50:00.000168+0000 mon.a (mon.0) 2642 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:50:00.577 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:00 smithi006 bash[33918]: cluster 2023-08-27T23:49:59.648454+0000 mgr.y (mgr.14901) 3343 : cluster [DBG] pgmap v3350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:00.577 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:00 smithi006 bash[33918]: cluster 2023-08-27T23:50:00.000168+0000 mon.a (mon.0) 2642 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:50:00.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:00 smithi130 bash[27579]: cluster 2023-08-27T23:49:59.648454+0000 mgr.y (mgr.14901) 3343 : cluster [DBG] pgmap v3350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:00 smithi130 bash[27579]: cluster 2023-08-27T23:50:00.000168+0000 mon.a (mon.0) 2642 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:50:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:50:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:50:00] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:50:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:50:00.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:50:00.576Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:01.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:00 smithi130 bash[51302]: debug 2023-08-27T23:50:00.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:00.871603+0000) 2023-08-27T23:50:01.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:01 smithi006 bash[33918]: audit 2023-08-27T23:50:01.655909+0000 mon.a (mon.0) 2643 : audit [DBG] from='client.? 172.21.15.6:0/2450477896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:01.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:01 smithi006 bash[27292]: audit 2023-08-27T23:50:01.655909+0000 mon.a (mon.0) 2643 : audit [DBG] from='client.? 172.21.15.6:0/2450477896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:02.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:01 smithi130 bash[27579]: audit 2023-08-27T23:50:01.655909+0000 mon.a (mon.0) 2643 : audit [DBG] from='client.? 172.21.15.6:0/2450477896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:02.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:01 smithi130 bash[51302]: debug 2023-08-27T23:50:01.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:01.871886+0000) 2023-08-27T23:50:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:50:02.140Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:50:02.142Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:02 smithi130 bash[27579]: cluster 2023-08-27T23:50:01.648894+0000 mgr.y (mgr.14901) 3344 : cluster [DBG] pgmap v3351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:02 smithi130 bash[51302]: debug 2023-08-27T23:50:02.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:02.872175+0000) 2023-08-27T23:50:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:02 smithi006 bash[33918]: cluster 2023-08-27T23:50:01.648894+0000 mgr.y (mgr.14901) 3344 : cluster [DBG] pgmap v3351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:02 smithi006 bash[27292]: cluster 2023-08-27T23:50:01.648894+0000 mgr.y (mgr.14901) 3344 : cluster [DBG] pgmap v3351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:04.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:03 smithi130 bash[51302]: debug 2023-08-27T23:50:03.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:03.872455+0000) 2023-08-27T23:50:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:04 smithi006 bash[56298]: level=error ts=2023-08-27T23:50:04.039Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:04 smithi006 bash[56298]: level=warn ts=2023-08-27T23:50:04.041Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:04 smithi130 bash[27579]: cluster 2023-08-27T23:50:03.649490+0000 mgr.y (mgr.14901) 3345 : cluster [DBG] pgmap v3352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:04 smithi130 bash[27579]: audit 2023-08-27T23:50:04.073184+0000 mon.a (mon.0) 2644 : audit [DBG] from='client.? 172.21.15.6:0/1851383801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:04 smithi130 bash[51302]: debug 2023-08-27T23:50:04.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:04.872755+0000) 2023-08-27T23:50:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:04 smithi006 bash[27292]: cluster 2023-08-27T23:50:03.649490+0000 mgr.y (mgr.14901) 3345 : cluster [DBG] pgmap v3352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:04 smithi006 bash[27292]: audit 2023-08-27T23:50:04.073184+0000 mon.a (mon.0) 2644 : audit [DBG] from='client.? 172.21.15.6:0/1851383801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:04 smithi006 bash[33918]: cluster 2023-08-27T23:50:03.649490+0000 mgr.y (mgr.14901) 3345 : cluster [DBG] pgmap v3352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:04 smithi006 bash[33918]: audit 2023-08-27T23:50:04.073184+0000 mon.a (mon.0) 2644 : audit [DBG] from='client.? 172.21.15.6:0/1851383801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:06.124 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:05 smithi130 bash[51302]: debug 2023-08-27T23:50:05.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:05.873025+0000) 2023-08-27T23:50:06.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:06 smithi006 bash[33918]: cluster 2023-08-27T23:50:05.650621+0000 mgr.y (mgr.14901) 3346 : cluster [DBG] pgmap v3353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:06.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:06 smithi006 bash[27292]: cluster 2023-08-27T23:50:05.650621+0000 mgr.y (mgr.14901) 3346 : cluster [DBG] pgmap v3353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:06.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:06 smithi130 bash[27579]: cluster 2023-08-27T23:50:05.650621+0000 mgr.y (mgr.14901) 3346 : cluster [DBG] pgmap v3353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:07.127 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:06 smithi130 bash[51302]: debug 2023-08-27T23:50:06.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:06.873259+0000) 2023-08-27T23:50:07.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:07 smithi006 bash[33918]: audit 2023-08-27T23:50:06.496215+0000 mon.a (mon.0) 2645 : audit [DBG] from='client.? 172.21.15.6:0/841909143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:07.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:07 smithi006 bash[27292]: audit 2023-08-27T23:50:06.496215+0000 mon.a (mon.0) 2645 : audit [DBG] from='client.? 172.21.15.6:0/841909143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:07.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:07 smithi130 bash[27579]: audit 2023-08-27T23:50:06.496215+0000 mon.a (mon.0) 2645 : audit [DBG] from='client.? 172.21.15.6:0/841909143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:08.137 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:07 smithi130 bash[51302]: debug 2023-08-27T23:50:07.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:07.873541+0000) 2023-08-27T23:50:08.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:08 smithi006 bash[33918]: cluster 2023-08-27T23:50:07.651265+0000 mgr.y (mgr.14901) 3347 : cluster [DBG] pgmap v3354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:08.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:08 smithi006 bash[27292]: cluster 2023-08-27T23:50:07.651265+0000 mgr.y (mgr.14901) 3347 : cluster [DBG] pgmap v3354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:08 smithi130 bash[27579]: cluster 2023-08-27T23:50:07.651265+0000 mgr.y (mgr.14901) 3347 : cluster [DBG] pgmap v3354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:09.140 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:08 smithi130 bash[51302]: debug 2023-08-27T23:50:08.873+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:08.873837+0000) 2023-08-27T23:50:09.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:09 smithi006 bash[27292]: audit 2023-08-27T23:50:08.915985+0000 mon.a (mon.0) 2646 : audit [DBG] from='client.? 172.21.15.6:0/608562908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:09.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:09 smithi006 bash[33918]: audit 2023-08-27T23:50:08.915985+0000 mon.a (mon.0) 2646 : audit [DBG] from='client.? 172.21.15.6:0/608562908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:09.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:09 smithi130 bash[27579]: audit 2023-08-27T23:50:08.915985+0000 mon.a (mon.0) 2646 : audit [DBG] from='client.? 172.21.15.6:0/608562908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:10.148 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:09 smithi130 bash[51302]: debug 2023-08-27T23:50:09.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:09.874139+0000) 2023-08-27T23:50:10.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:10 smithi006 bash[27292]: cluster 2023-08-27T23:50:09.652382+0000 mgr.y (mgr.14901) 3348 : cluster [DBG] pgmap v3355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:10.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:10 smithi006 bash[33918]: cluster 2023-08-27T23:50:09.652382+0000 mgr.y (mgr.14901) 3348 : cluster [DBG] pgmap v3355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:10.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:10 smithi130 bash[27579]: cluster 2023-08-27T23:50:09.652382+0000 mgr.y (mgr.14901) 3348 : cluster [DBG] pgmap v3355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:10.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:50:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:50:10] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:50:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:50:10.575Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:50:10.577Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:11.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:10 smithi130 bash[51302]: debug 2023-08-27T23:50:10.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:10.874411+0000) 2023-08-27T23:50:11.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:11 smithi006 bash[33918]: audit 2023-08-27T23:50:11.337161+0000 mon.c (mon.2) 1286 : audit [DBG] from='client.? 172.21.15.6:0/947993146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:11.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:11 smithi006 bash[27292]: audit 2023-08-27T23:50:11.337161+0000 mon.c (mon.2) 1286 : audit [DBG] from='client.? 172.21.15.6:0/947993146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:11.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:11 smithi130 bash[27579]: audit 2023-08-27T23:50:11.337161+0000 mon.c (mon.2) 1286 : audit [DBG] from='client.? 172.21.15.6:0/947993146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:12.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:11 smithi130 bash[51302]: debug 2023-08-27T23:50:11.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:11.874674+0000) 2023-08-27T23:50:12.401 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:50:12.141Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:12.401 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:50:12.143Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:12.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:12 smithi006 bash[27292]: cluster 2023-08-27T23:50:11.653051+0000 mgr.y (mgr.14901) 3349 : cluster [DBG] pgmap v3356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:12.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:12 smithi006 bash[33918]: cluster 2023-08-27T23:50:11.653051+0000 mgr.y (mgr.14901) 3349 : cluster [DBG] pgmap v3356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:12 smithi130 bash[27579]: cluster 2023-08-27T23:50:11.653051+0000 mgr.y (mgr.14901) 3349 : cluster [DBG] pgmap v3356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:13.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:12 smithi130 bash[51302]: debug 2023-08-27T23:50:12.873+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:12.874955+0000) 2023-08-27T23:50:14.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:13 smithi130 bash[51302]: debug 2023-08-27T23:50:13.873+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:13.875281+0000) 2023-08-27T23:50:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:14 smithi006 bash[56298]: level=error ts=2023-08-27T23:50:14.039Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:14 smithi006 bash[56298]: level=warn ts=2023-08-27T23:50:14.042Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:14 smithi006 bash[27292]: cluster 2023-08-27T23:50:13.653640+0000 mgr.y (mgr.14901) 3350 : cluster [DBG] pgmap v3357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:14 smithi006 bash[27292]: audit 2023-08-27T23:50:13.760344+0000 mon.c (mon.2) 1287 : audit [DBG] from='client.? 172.21.15.6:0/3154342726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:14 smithi006 bash[33918]: cluster 2023-08-27T23:50:13.653640+0000 mgr.y (mgr.14901) 3350 : cluster [DBG] pgmap v3357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:14 smithi006 bash[33918]: audit 2023-08-27T23:50:13.760344+0000 mon.c (mon.2) 1287 : audit [DBG] from='client.? 172.21.15.6:0/3154342726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:14 smithi130 bash[27579]: cluster 2023-08-27T23:50:13.653640+0000 mgr.y (mgr.14901) 3350 : cluster [DBG] pgmap v3357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:14 smithi130 bash[27579]: audit 2023-08-27T23:50:13.760344+0000 mon.c (mon.2) 1287 : audit [DBG] from='client.? 172.21.15.6:0/3154342726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:15.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:14 smithi130 bash[51302]: debug 2023-08-27T23:50:14.873+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:14.875565+0000) 2023-08-27T23:50:16.131 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:15 smithi130 bash[51302]: debug 2023-08-27T23:50:15.873+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:15.875905+0000) 2023-08-27T23:50:16.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:16 smithi006 bash[33918]: cluster 2023-08-27T23:50:15.654534+0000 mgr.y (mgr.14901) 3351 : cluster [DBG] pgmap v3358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:16.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:16 smithi006 bash[27292]: cluster 2023-08-27T23:50:15.654534+0000 mgr.y (mgr.14901) 3351 : cluster [DBG] pgmap v3358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:16.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:16 smithi130 bash[27579]: cluster 2023-08-27T23:50:15.654534+0000 mgr.y (mgr.14901) 3351 : cluster [DBG] pgmap v3358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:17.134 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:16 smithi130 bash[51302]: debug 2023-08-27T23:50:16.873+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:16.876114+0000) 2023-08-27T23:50:17.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:17 smithi006 bash[33918]: audit 2023-08-27T23:50:16.177378+0000 mon.c (mon.2) 1288 : audit [DBG] from='client.? 172.21.15.6:0/3897766125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:17.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:17 smithi006 bash[27292]: audit 2023-08-27T23:50:16.177378+0000 mon.c (mon.2) 1288 : audit [DBG] from='client.? 172.21.15.6:0/3897766125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:17.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:17 smithi130 bash[27579]: audit 2023-08-27T23:50:16.177378+0000 mon.c (mon.2) 1288 : audit [DBG] from='client.? 172.21.15.6:0/3897766125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:18.142 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:17 smithi130 bash[51302]: debug 2023-08-27T23:50:17.873+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:17.876394+0000) 2023-08-27T23:50:18.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:18 smithi006 bash[27292]: cluster 2023-08-27T23:50:17.655170+0000 mgr.y (mgr.14901) 3352 : cluster [DBG] pgmap v3359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:18.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:18 smithi006 bash[33918]: cluster 2023-08-27T23:50:17.655170+0000 mgr.y (mgr.14901) 3352 : cluster [DBG] pgmap v3359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:18 smithi130 bash[27579]: cluster 2023-08-27T23:50:17.655170+0000 mgr.y (mgr.14901) 3352 : cluster [DBG] pgmap v3359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:19.145 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:18 smithi130 bash[51302]: debug 2023-08-27T23:50:18.873+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:18.876727+0000) 2023-08-27T23:50:19.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:19 smithi006 bash[33918]: audit 2023-08-27T23:50:18.597753+0000 mon.a (mon.0) 2647 : audit [DBG] from='client.? 172.21.15.6:0/2201906732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:19.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:19 smithi006 bash[27292]: audit 2023-08-27T23:50:18.597753+0000 mon.a (mon.0) 2647 : audit [DBG] from='client.? 172.21.15.6:0/2201906732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:19.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:19 smithi130 bash[27579]: audit 2023-08-27T23:50:18.597753+0000 mon.a (mon.0) 2647 : audit [DBG] from='client.? 172.21.15.6:0/2201906732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:20.151 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:19 smithi130 bash[51302]: debug 2023-08-27T23:50:19.873+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:19.877017+0000) 2023-08-27T23:50:20.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:20 smithi006 bash[27292]: cluster 2023-08-27T23:50:19.656260+0000 mgr.y (mgr.14901) 3353 : cluster [DBG] pgmap v3360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:20.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:20 smithi006 bash[33918]: cluster 2023-08-27T23:50:19.656260+0000 mgr.y (mgr.14901) 3353 : cluster [DBG] pgmap v3360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:20 smithi130 bash[27579]: cluster 2023-08-27T23:50:19.656260+0000 mgr.y (mgr.14901) 3353 : cluster [DBG] pgmap v3360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:50:20.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:50:20.578Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:50:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:50:20] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-27T23:50:21.154 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:20 smithi130 bash[51302]: debug 2023-08-27T23:50:20.873+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:20.877333+0000) 2023-08-27T23:50:21.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:21 smithi006 bash[27292]: audit 2023-08-27T23:50:21.008890+0000 mon.a (mon.0) 2648 : audit [DBG] from='client.? 172.21.15.6:0/121171205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:21.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:21 smithi006 bash[33918]: audit 2023-08-27T23:50:21.008890+0000 mon.a (mon.0) 2648 : audit [DBG] from='client.? 172.21.15.6:0/121171205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:21.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:21 smithi130 bash[27579]: audit 2023-08-27T23:50:21.008890+0000 mon.a (mon.0) 2648 : audit [DBG] from='client.? 172.21.15.6:0/121171205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:22.163 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:21 smithi130 bash[51302]: debug 2023-08-27T23:50:21.873+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:21.877615+0000) 2023-08-27T23:50:22.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:22 smithi006 bash[27292]: cluster 2023-08-27T23:50:21.656954+0000 mgr.y (mgr.14901) 3354 : cluster [DBG] pgmap v3361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:22.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:22 smithi006 bash[33918]: cluster 2023-08-27T23:50:21.656954+0000 mgr.y (mgr.14901) 3354 : cluster [DBG] pgmap v3361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:50:22.141Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:22.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:50:22.143Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:22.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:22 smithi130 bash[27579]: cluster 2023-08-27T23:50:21.656954+0000 mgr.y (mgr.14901) 3354 : cluster [DBG] pgmap v3361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:23.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:22 smithi130 bash[51302]: debug 2023-08-27T23:50:22.873+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:22.877878+0000) 2023-08-27T23:50:23.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:23 smithi130 bash[27579]: audit 2023-08-27T23:50:23.430381+0000 mon.a (mon.0) 2649 : audit [DBG] from='client.? 172.21.15.6:0/3353348054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:23.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:23 smithi006 bash[33918]: audit 2023-08-27T23:50:23.430381+0000 mon.a (mon.0) 2649 : audit [DBG] from='client.? 172.21.15.6:0/3353348054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:23.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:23 smithi006 bash[27292]: audit 2023-08-27T23:50:23.430381+0000 mon.a (mon.0) 2649 : audit [DBG] from='client.? 172.21.15.6:0/3353348054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:24.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:23 smithi130 bash[51302]: debug 2023-08-27T23:50:23.873+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:23.878162+0000) 2023-08-27T23:50:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:24 smithi006 bash[56298]: level=error ts=2023-08-27T23:50:24.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:24 smithi006 bash[56298]: level=warn ts=2023-08-27T23:50:24.042Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:24 smithi130 bash[27579]: cluster 2023-08-27T23:50:23.657593+0000 mgr.y (mgr.14901) 3355 : cluster [DBG] pgmap v3362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:24 smithi006 bash[27292]: cluster 2023-08-27T23:50:23.657593+0000 mgr.y (mgr.14901) 3355 : cluster [DBG] pgmap v3362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:24.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:24 smithi006 bash[33918]: cluster 2023-08-27T23:50:23.657593+0000 mgr.y (mgr.14901) 3355 : cluster [DBG] pgmap v3362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:25.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:24 smithi130 bash[51302]: debug 2023-08-27T23:50:24.873+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:24.878429+0000) 2023-08-27T23:50:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:26 smithi130 bash[27579]: audit 2023-08-27T23:50:25.542826+0000 mon.b (mon.1) 436 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:50:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:26 smithi130 bash[27579]: cluster 2023-08-27T23:50:25.658689+0000 mgr.y (mgr.14901) 3356 : cluster [DBG] pgmap v3363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:26 smithi130 bash[27579]: audit 2023-08-27T23:50:25.845021+0000 mon.c (mon.2) 1289 : audit [DBG] from='client.? 172.21.15.6:0/1165234442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:26.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:25 smithi130 bash[51302]: debug 2023-08-27T23:50:25.873+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:25.878699+0000) 2023-08-27T23:50:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:26 smithi006 bash[27292]: audit 2023-08-27T23:50:25.542826+0000 mon.b (mon.1) 436 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:50:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:26 smithi006 bash[27292]: cluster 2023-08-27T23:50:25.658689+0000 mgr.y (mgr.14901) 3356 : cluster [DBG] pgmap v3363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:26 smithi006 bash[27292]: audit 2023-08-27T23:50:25.845021+0000 mon.c (mon.2) 1289 : audit [DBG] from='client.? 172.21.15.6:0/1165234442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:26.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:26 smithi006 bash[33918]: audit 2023-08-27T23:50:25.542826+0000 mon.b (mon.1) 436 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:50:26.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:26 smithi006 bash[33918]: cluster 2023-08-27T23:50:25.658689+0000 mgr.y (mgr.14901) 3356 : cluster [DBG] pgmap v3363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:26 smithi006 bash[33918]: audit 2023-08-27T23:50:25.845021+0000 mon.c (mon.2) 1289 : audit [DBG] from='client.? 172.21.15.6:0/1165234442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:27.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:26 smithi130 bash[51302]: debug 2023-08-27T23:50:26.877+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:26.879076+0000) 2023-08-27T23:50:28.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:27 smithi130 bash[51302]: debug 2023-08-27T23:50:27.877+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:27.879383+0000) 2023-08-27T23:50:29.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:28 smithi130 bash[51302]: debug 2023-08-27T23:50:28.877+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:28.879691+0000) 2023-08-27T23:50:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:28 smithi130 bash[27579]: cluster 2023-08-27T23:50:27.659325+0000 mgr.y (mgr.14901) 3357 : cluster [DBG] pgmap v3364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:28 smithi130 bash[27579]: audit 2023-08-27T23:50:28.260356+0000 mon.c (mon.2) 1290 : audit [DBG] from='client.? 172.21.15.6:0/2106027611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:28 smithi006 bash[33918]: cluster 2023-08-27T23:50:27.659325+0000 mgr.y (mgr.14901) 3357 : cluster [DBG] pgmap v3364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:28 smithi006 bash[33918]: audit 2023-08-27T23:50:28.260356+0000 mon.c (mon.2) 1290 : audit [DBG] from='client.? 172.21.15.6:0/2106027611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:28 smithi006 bash[27292]: cluster 2023-08-27T23:50:27.659325+0000 mgr.y (mgr.14901) 3357 : cluster [DBG] pgmap v3364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:28 smithi006 bash[27292]: audit 2023-08-27T23:50:28.260356+0000 mon.c (mon.2) 1290 : audit [DBG] from='client.? 172.21.15.6:0/2106027611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:29 smithi130 bash[51302]: debug 2023-08-27T23:50:29.877+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:29.879901+0000) 2023-08-27T23:50:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:50:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:50:30] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-27T23:50:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:50:30.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:50:30.578Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:30.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:30 smithi006 bash[27292]: cluster 2023-08-27T23:50:29.660441+0000 mgr.y (mgr.14901) 3358 : cluster [DBG] pgmap v3365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:30.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:30 smithi006 bash[27292]: audit 2023-08-27T23:50:30.700096+0000 mon.a (mon.0) 2650 : audit [DBG] from='client.? 172.21.15.6:0/2692915166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:30.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:30 smithi006 bash[33918]: cluster 2023-08-27T23:50:29.660441+0000 mgr.y (mgr.14901) 3358 : cluster [DBG] pgmap v3365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:30.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:30 smithi006 bash[33918]: audit 2023-08-27T23:50:30.700096+0000 mon.a (mon.0) 2650 : audit [DBG] from='client.? 172.21.15.6:0/2692915166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:30 smithi130 bash[27579]: cluster 2023-08-27T23:50:29.660441+0000 mgr.y (mgr.14901) 3358 : cluster [DBG] pgmap v3365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:31.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:30 smithi130 bash[27579]: audit 2023-08-27T23:50:30.700096+0000 mon.a (mon.0) 2650 : audit [DBG] from='client.? 172.21.15.6:0/2692915166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:31.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:30 smithi130 bash[51302]: debug 2023-08-27T23:50:30.877+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:30.880152+0000) 2023-08-27T23:50:32.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:31 smithi130 bash[51302]: debug 2023-08-27T23:50:31.876+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:31.880340+0000) 2023-08-27T23:50:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:50:32.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:50:32.144Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:32 smithi130 bash[27579]: audit 2023-08-27T23:50:31.513364+0000 mon.a (mon.0) 2651 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:50:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:32 smithi130 bash[27579]: cluster 2023-08-27T23:50:31.661099+0000 mgr.y (mgr.14901) 3359 : cluster [DBG] pgmap v3366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:32 smithi130 bash[27579]: audit 2023-08-27T23:50:31.899357+0000 mon.a (mon.0) 2652 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:50:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:32 smithi006 bash[33918]: audit 2023-08-27T23:50:31.513364+0000 mon.a (mon.0) 2651 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:50:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:32 smithi006 bash[33918]: cluster 2023-08-27T23:50:31.661099+0000 mgr.y (mgr.14901) 3359 : cluster [DBG] pgmap v3366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:32 smithi006 bash[33918]: audit 2023-08-27T23:50:31.899357+0000 mon.a (mon.0) 2652 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:50:32.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:32 smithi006 bash[27292]: audit 2023-08-27T23:50:31.513364+0000 mon.a (mon.0) 2651 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:50:32.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:32 smithi006 bash[27292]: cluster 2023-08-27T23:50:31.661099+0000 mgr.y (mgr.14901) 3359 : cluster [DBG] pgmap v3366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:32.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:32 smithi006 bash[27292]: audit 2023-08-27T23:50:31.899357+0000 mon.a (mon.0) 2652 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:50:33.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:32 smithi130 bash[51302]: debug 2023-08-27T23:50:32.876+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:32.880578+0000) 2023-08-27T23:50:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:33 smithi130 bash[27579]: audit 2023-08-27T23:50:33.122348+0000 mon.c (mon.2) 1291 : audit [DBG] from='client.? 172.21.15.6:0/1803185473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:33.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:33 smithi006 bash[33918]: audit 2023-08-27T23:50:33.122348+0000 mon.c (mon.2) 1291 : audit [DBG] from='client.? 172.21.15.6:0/1803185473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:33.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:33 smithi006 bash[27292]: audit 2023-08-27T23:50:33.122348+0000 mon.c (mon.2) 1291 : audit [DBG] from='client.? 172.21.15.6:0/1803185473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:34.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:33 smithi130 bash[51302]: debug 2023-08-27T23:50:33.876+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:33.880873+0000) 2023-08-27T23:50:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:34 smithi006 bash[56298]: level=error ts=2023-08-27T23:50:34.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:34 smithi006 bash[56298]: level=warn ts=2023-08-27T23:50:34.043Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:34 smithi130 bash[27579]: cluster 2023-08-27T23:50:33.661765+0000 mgr.y (mgr.14901) 3360 : cluster [DBG] pgmap v3367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:34.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:34 smithi006 bash[33918]: cluster 2023-08-27T23:50:33.661765+0000 mgr.y (mgr.14901) 3360 : cluster [DBG] pgmap v3367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:34.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:34 smithi006 bash[27292]: cluster 2023-08-27T23:50:33.661765+0000 mgr.y (mgr.14901) 3360 : cluster [DBG] pgmap v3367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:35.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:34 smithi130 bash[51302]: debug 2023-08-27T23:50:34.876+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:34.881169+0000) 2023-08-27T23:50:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:36 smithi130 bash[27579]: audit 2023-08-27T23:50:35.532356+0000 mon.c (mon.2) 1292 : audit [DBG] from='client.? 172.21.15.6:0/3430242075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:36 smithi130 bash[27579]: cluster 2023-08-27T23:50:35.662873+0000 mgr.y (mgr.14901) 3361 : cluster [DBG] pgmap v3368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:36.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:35 smithi130 bash[51302]: debug 2023-08-27T23:50:35.876+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:35.881443+0000) 2023-08-27T23:50:36.394 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:36 smithi006 bash[27292]: audit 2023-08-27T23:50:35.532356+0000 mon.c (mon.2) 1292 : audit [DBG] from='client.? 172.21.15.6:0/3430242075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:36.394 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:36 smithi006 bash[27292]: cluster 2023-08-27T23:50:35.662873+0000 mgr.y (mgr.14901) 3361 : cluster [DBG] pgmap v3368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:36.394 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:36 smithi006 bash[33918]: audit 2023-08-27T23:50:35.532356+0000 mon.c (mon.2) 1292 : audit [DBG] from='client.? 172.21.15.6:0/3430242075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:36.395 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:36 smithi006 bash[33918]: cluster 2023-08-27T23:50:35.662873+0000 mgr.y (mgr.14901) 3361 : cluster [DBG] pgmap v3368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:36 smithi130 bash[51302]: debug 2023-08-27T23:50:36.880+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:36.881702+0000) 2023-08-27T23:50:38.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:37 smithi130 bash[51302]: debug 2023-08-27T23:50:37.876+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:37.881997+0000) 2023-08-27T23:50:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:38 smithi006 bash[33918]: cluster 2023-08-27T23:50:37.663514+0000 mgr.y (mgr.14901) 3362 : cluster [DBG] pgmap v3369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:38 smithi006 bash[33918]: audit 2023-08-27T23:50:37.954562+0000 mon.a (mon.0) 2653 : audit [DBG] from='client.? 172.21.15.6:0/327094036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:38 smithi006 bash[27292]: cluster 2023-08-27T23:50:37.663514+0000 mgr.y (mgr.14901) 3362 : cluster [DBG] pgmap v3369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:38 smithi006 bash[27292]: audit 2023-08-27T23:50:37.954562+0000 mon.a (mon.0) 2653 : audit [DBG] from='client.? 172.21.15.6:0/327094036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:38 smithi130 bash[27579]: cluster 2023-08-27T23:50:37.663514+0000 mgr.y (mgr.14901) 3362 : cluster [DBG] pgmap v3369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:38 smithi130 bash[27579]: audit 2023-08-27T23:50:37.954562+0000 mon.a (mon.0) 2653 : audit [DBG] from='client.? 172.21.15.6:0/327094036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:39.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:38 smithi130 bash[51302]: debug 2023-08-27T23:50:38.876+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:38.882320+0000) 2023-08-27T23:50:40.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:39 smithi130 bash[51302]: debug 2023-08-27T23:50:39.876+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:39.882609+0000) 2023-08-27T23:50:40.877 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:50:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:50:40] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-27T23:50:40.877 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:50:40.577Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:40.877 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:50:40.579Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:40.877 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:40 smithi006 bash[27292]: cluster 2023-08-27T23:50:39.664369+0000 mgr.y (mgr.14901) 3363 : cluster [DBG] pgmap v3370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:40.878 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:40 smithi006 bash[27292]: audit 2023-08-27T23:50:40.367463+0000 mon.a (mon.0) 2654 : audit [DBG] from='client.? 172.21.15.6:0/2046142800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:40.878 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:40 smithi006 bash[33918]: cluster 2023-08-27T23:50:39.664369+0000 mgr.y (mgr.14901) 3363 : cluster [DBG] pgmap v3370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:40.878 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:40 smithi006 bash[33918]: audit 2023-08-27T23:50:40.367463+0000 mon.a (mon.0) 2654 : audit [DBG] from='client.? 172.21.15.6:0/2046142800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:41.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:40 smithi130 bash[51302]: debug 2023-08-27T23:50:40.880+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:40.882912+0000) 2023-08-27T23:50:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:40 smithi130 bash[27579]: cluster 2023-08-27T23:50:39.664369+0000 mgr.y (mgr.14901) 3363 : cluster [DBG] pgmap v3370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:40 smithi130 bash[27579]: audit 2023-08-27T23:50:40.367463+0000 mon.a (mon.0) 2654 : audit [DBG] from='client.? 172.21.15.6:0/2046142800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:42.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:41 smithi130 bash[51302]: debug 2023-08-27T23:50:41.880+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:41.883182+0000) 2023-08-27T23:50:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:50:42.143Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:50:42.145Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:43.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:42 smithi130 bash[51302]: debug 2023-08-27T23:50:42.880+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:42.883506+0000) 2023-08-27T23:50:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:42 smithi130 bash[27579]: cluster 2023-08-27T23:50:41.665007+0000 mgr.y (mgr.14901) 3364 : cluster [DBG] pgmap v3371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:42 smithi130 bash[27579]: audit 2023-08-27T23:50:41.798086+0000 mon.a (mon.0) 2655 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:50:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:42 smithi130 bash[27579]: audit 2023-08-27T23:50:42.098216+0000 mon.b (mon.1) 437 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:50:43.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:42 smithi130 bash[27579]: audit 2023-08-27T23:50:42.100072+0000 mon.b (mon.1) 438 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:50:43.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:42 smithi130 bash[27579]: audit 2023-08-27T23:50:42.106350+0000 mon.a (mon.0) 2656 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:50:43.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:42 smithi130 bash[27579]: audit 2023-08-27T23:50:42.777295+0000 mon.c (mon.2) 1293 : audit [DBG] from='client.? 172.21.15.6:0/4279912568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:42 smithi006 bash[27292]: cluster 2023-08-27T23:50:41.665007+0000 mgr.y (mgr.14901) 3364 : cluster [DBG] pgmap v3371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:42 smithi006 bash[27292]: audit 2023-08-27T23:50:41.798086+0000 mon.a (mon.0) 2655 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:50:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:42 smithi006 bash[27292]: audit 2023-08-27T23:50:42.098216+0000 mon.b (mon.1) 437 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:50:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:42 smithi006 bash[27292]: audit 2023-08-27T23:50:42.100072+0000 mon.b (mon.1) 438 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:50:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:42 smithi006 bash[27292]: audit 2023-08-27T23:50:42.106350+0000 mon.a (mon.0) 2656 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:50:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:42 smithi006 bash[27292]: audit 2023-08-27T23:50:42.777295+0000 mon.c (mon.2) 1293 : audit [DBG] from='client.? 172.21.15.6:0/4279912568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:43.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:42 smithi006 bash[33918]: cluster 2023-08-27T23:50:41.665007+0000 mgr.y (mgr.14901) 3364 : cluster [DBG] pgmap v3371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:43.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:42 smithi006 bash[33918]: audit 2023-08-27T23:50:41.798086+0000 mon.a (mon.0) 2655 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:50:43.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:42 smithi006 bash[33918]: audit 2023-08-27T23:50:42.098216+0000 mon.b (mon.1) 437 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:50:43.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:42 smithi006 bash[33918]: audit 2023-08-27T23:50:42.100072+0000 mon.b (mon.1) 438 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:50:43.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:42 smithi006 bash[33918]: audit 2023-08-27T23:50:42.106350+0000 mon.a (mon.0) 2656 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:50:43.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:42 smithi006 bash[33918]: audit 2023-08-27T23:50:42.777295+0000 mon.c (mon.2) 1293 : audit [DBG] from='client.? 172.21.15.6:0/4279912568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:44.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:43 smithi130 bash[51302]: debug 2023-08-27T23:50:43.880+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:43.883762+0000) 2023-08-27T23:50:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:44 smithi006 bash[56298]: level=error ts=2023-08-27T23:50:44.041Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:44 smithi006 bash[56298]: level=warn ts=2023-08-27T23:50:44.043Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:44 smithi130 bash[27579]: cluster 2023-08-27T23:50:43.665622+0000 mgr.y (mgr.14901) 3365 : cluster [DBG] pgmap v3372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:44 smithi130 bash[51302]: debug 2023-08-27T23:50:44.880+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:44.884080+0000) 2023-08-27T23:50:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:44 smithi006 bash[27292]: cluster 2023-08-27T23:50:43.665622+0000 mgr.y (mgr.14901) 3365 : cluster [DBG] pgmap v3372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:44 smithi006 bash[33918]: cluster 2023-08-27T23:50:43.665622+0000 mgr.y (mgr.14901) 3365 : cluster [DBG] pgmap v3372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:46 smithi130 bash[27579]: audit 2023-08-27T23:50:45.194776+0000 mon.a (mon.0) 2657 : audit [DBG] from='client.? 172.21.15.6:0/3939866627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:46 smithi130 bash[27579]: cluster 2023-08-27T23:50:45.666769+0000 mgr.y (mgr.14901) 3366 : cluster [DBG] pgmap v3373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:45 smithi130 bash[51302]: debug 2023-08-27T23:50:45.880+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:45.884297+0000) 2023-08-27T23:50:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:46 smithi006 bash[33918]: audit 2023-08-27T23:50:45.194776+0000 mon.a (mon.0) 2657 : audit [DBG] from='client.? 172.21.15.6:0/3939866627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:46 smithi006 bash[33918]: cluster 2023-08-27T23:50:45.666769+0000 mgr.y (mgr.14901) 3366 : cluster [DBG] pgmap v3373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:46.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:46 smithi006 bash[27292]: audit 2023-08-27T23:50:45.194776+0000 mon.a (mon.0) 2657 : audit [DBG] from='client.? 172.21.15.6:0/3939866627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:46.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:46 smithi006 bash[27292]: cluster 2023-08-27T23:50:45.666769+0000 mgr.y (mgr.14901) 3366 : cluster [DBG] pgmap v3373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:47.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:46 smithi130 bash[51302]: debug 2023-08-27T23:50:46.880+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:46.884606+0000) 2023-08-27T23:50:47.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:47 smithi006 bash[27292]: audit 2023-08-27T23:50:47.610723+0000 mon.a (mon.0) 2658 : audit [DBG] from='client.? 172.21.15.6:0/579981006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:47.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:47 smithi006 bash[33918]: audit 2023-08-27T23:50:47.610723+0000 mon.a (mon.0) 2658 : audit [DBG] from='client.? 172.21.15.6:0/579981006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:48.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:47 smithi130 bash[27579]: audit 2023-08-27T23:50:47.610723+0000 mon.a (mon.0) 2658 : audit [DBG] from='client.? 172.21.15.6:0/579981006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:48.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:47 smithi130 bash[51302]: debug 2023-08-27T23:50:47.880+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:47.884910+0000) 2023-08-27T23:50:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:48 smithi006 bash[33918]: cluster 2023-08-27T23:50:47.667338+0000 mgr.y (mgr.14901) 3367 : cluster [DBG] pgmap v3374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:48 smithi006 bash[27292]: cluster 2023-08-27T23:50:47.667338+0000 mgr.y (mgr.14901) 3367 : cluster [DBG] pgmap v3374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:48 smithi130 bash[27579]: cluster 2023-08-27T23:50:47.667338+0000 mgr.y (mgr.14901) 3367 : cluster [DBG] pgmap v3374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:49.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:48 smithi130 bash[51302]: debug 2023-08-27T23:50:48.880+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:48.885231+0000) 2023-08-27T23:50:50.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:49 smithi130 bash[51302]: debug 2023-08-27T23:50:49.884+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:49.885527+0000) 2023-08-27T23:50:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:50:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:50:50] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-27T23:50:50.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:50 smithi006 bash[33918]: cluster 2023-08-27T23:50:49.668439+0000 mgr.y (mgr.14901) 3368 : cluster [DBG] pgmap v3375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:50.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:50 smithi006 bash[33918]: audit 2023-08-27T23:50:50.029556+0000 mon.a (mon.0) 2659 : audit [DBG] from='client.? 172.21.15.6:0/3917334192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:50:50.578Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:50:50.580Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:50.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:50 smithi006 bash[27292]: cluster 2023-08-27T23:50:49.668439+0000 mgr.y (mgr.14901) 3368 : cluster [DBG] pgmap v3375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:50.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:50 smithi006 bash[27292]: audit 2023-08-27T23:50:50.029556+0000 mon.a (mon.0) 2659 : audit [DBG] from='client.? 172.21.15.6:0/3917334192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:50 smithi130 bash[27579]: cluster 2023-08-27T23:50:49.668439+0000 mgr.y (mgr.14901) 3368 : cluster [DBG] pgmap v3375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:50 smithi130 bash[27579]: audit 2023-08-27T23:50:50.029556+0000 mon.a (mon.0) 2659 : audit [DBG] from='client.? 172.21.15.6:0/3917334192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:51.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:50 smithi130 bash[51302]: debug 2023-08-27T23:50:50.880+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:50.885791+0000) 2023-08-27T23:50:52.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:51 smithi130 bash[51302]: debug 2023-08-27T23:50:51.880+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:51.886048+0000) 2023-08-27T23:50:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:50:52.144Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:50:52.148Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:52 smithi006 bash[27292]: cluster 2023-08-27T23:50:51.669065+0000 mgr.y (mgr.14901) 3369 : cluster [DBG] pgmap v3376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:52 smithi006 bash[27292]: audit 2023-08-27T23:50:52.448725+0000 mon.a (mon.0) 2660 : audit [DBG] from='client.? 172.21.15.6:0/1518118488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:52 smithi006 bash[33918]: cluster 2023-08-27T23:50:51.669065+0000 mgr.y (mgr.14901) 3369 : cluster [DBG] pgmap v3376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:52 smithi006 bash[33918]: audit 2023-08-27T23:50:52.448725+0000 mon.a (mon.0) 2660 : audit [DBG] from='client.? 172.21.15.6:0/1518118488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:52 smithi130 bash[51302]: debug 2023-08-27T23:50:52.880+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:52.886324+0000) 2023-08-27T23:50:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:52 smithi130 bash[27579]: cluster 2023-08-27T23:50:51.669065+0000 mgr.y (mgr.14901) 3369 : cluster [DBG] pgmap v3376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:52 smithi130 bash[27579]: audit 2023-08-27T23:50:52.448725+0000 mon.a (mon.0) 2660 : audit [DBG] from='client.? 172.21.15.6:0/1518118488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:54.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:53 smithi130 bash[51302]: debug 2023-08-27T23:50:53.884+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:53.886609+0000) 2023-08-27T23:50:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:54 smithi006 bash[56298]: level=error ts=2023-08-27T23:50:54.042Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:50:54 smithi006 bash[56298]: level=warn ts=2023-08-27T23:50:54.044Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:50:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:54 smithi006 bash[33918]: cluster 2023-08-27T23:50:53.669814+0000 mgr.y (mgr.14901) 3370 : cluster [DBG] pgmap v3377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:54 smithi006 bash[27292]: cluster 2023-08-27T23:50:53.669814+0000 mgr.y (mgr.14901) 3370 : cluster [DBG] pgmap v3377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:54 smithi130 bash[27579]: cluster 2023-08-27T23:50:53.669814+0000 mgr.y (mgr.14901) 3370 : cluster [DBG] pgmap v3377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:54 smithi130 bash[51302]: debug 2023-08-27T23:50:54.884+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:54.886933+0000) 2023-08-27T23:50:55.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:55 smithi006 bash[27292]: audit 2023-08-27T23:50:54.866995+0000 mon.c (mon.2) 1294 : audit [DBG] from='client.? 172.21.15.6:0/2209052297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:55.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:55 smithi006 bash[33918]: audit 2023-08-27T23:50:54.866995+0000 mon.c (mon.2) 1294 : audit [DBG] from='client.? 172.21.15.6:0/2209052297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:56.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:55 smithi130 bash[27579]: audit 2023-08-27T23:50:54.866995+0000 mon.c (mon.2) 1294 : audit [DBG] from='client.? 172.21.15.6:0/2209052297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:56.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:55 smithi130 bash[51302]: debug 2023-08-27T23:50:55.884+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:55.887238+0000) 2023-08-27T23:50:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:56 smithi006 bash[27292]: cluster 2023-08-27T23:50:55.671063+0000 mgr.y (mgr.14901) 3371 : cluster [DBG] pgmap v3378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:56 smithi006 bash[33918]: cluster 2023-08-27T23:50:55.671063+0000 mgr.y (mgr.14901) 3371 : cluster [DBG] pgmap v3378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:56 smithi130 bash[27579]: cluster 2023-08-27T23:50:55.671063+0000 mgr.y (mgr.14901) 3371 : cluster [DBG] pgmap v3378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:57.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:56 smithi130 bash[51302]: debug 2023-08-27T23:50:56.884+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:56.887573+0000) 2023-08-27T23:50:57.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:57 smithi006 bash[27292]: audit 2023-08-27T23:50:57.280964+0000 mon.a (mon.0) 2661 : audit [DBG] from='client.? 172.21.15.6:0/2359569250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:57.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:57 smithi006 bash[33918]: audit 2023-08-27T23:50:57.280964+0000 mon.a (mon.0) 2661 : audit [DBG] from='client.? 172.21.15.6:0/2359569250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:58.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:57 smithi130 bash[27579]: audit 2023-08-27T23:50:57.280964+0000 mon.a (mon.0) 2661 : audit [DBG] from='client.? 172.21.15.6:0/2359569250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:58.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:57 smithi130 bash[51302]: debug 2023-08-27T23:50:57.884+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:57.887848+0000) 2023-08-27T23:50:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:58 smithi006 bash[27292]: cluster 2023-08-27T23:50:57.671780+0000 mgr.y (mgr.14901) 3372 : cluster [DBG] pgmap v3379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:58 smithi006 bash[33918]: cluster 2023-08-27T23:50:57.671780+0000 mgr.y (mgr.14901) 3372 : cluster [DBG] pgmap v3379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:58 smithi130 bash[27579]: cluster 2023-08-27T23:50:57.671780+0000 mgr.y (mgr.14901) 3372 : cluster [DBG] pgmap v3379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:50:59.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:58 smithi130 bash[51302]: debug 2023-08-27T23:50:58.884+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:58.888164+0000) 2023-08-27T23:51:00.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:50:59 smithi130 bash[27579]: audit 2023-08-27T23:50:59.696875+0000 mon.c (mon.2) 1295 : audit [DBG] from='client.? 172.21.15.6:0/1943126155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:00.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:50:59 smithi130 bash[51302]: debug 2023-08-27T23:50:59.884+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:59.888471+0000) 2023-08-27T23:51:00.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:50:59 smithi006 bash[33918]: audit 2023-08-27T23:50:59.696875+0000 mon.c (mon.2) 1295 : audit [DBG] from='client.? 172.21.15.6:0/1943126155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:00.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:50:59 smithi006 bash[27292]: audit 2023-08-27T23:50:59.696875+0000 mon.c (mon.2) 1295 : audit [DBG] from='client.? 172.21.15.6:0/1943126155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:00.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:00 smithi006 bash[33918]: cluster 2023-08-27T23:50:59.672909+0000 mgr.y (mgr.14901) 3373 : cluster [DBG] pgmap v3380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:00.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:00 smithi006 bash[27292]: cluster 2023-08-27T23:50:59.672909+0000 mgr.y (mgr.14901) 3373 : cluster [DBG] pgmap v3380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:51:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:51:00] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-27T23:51:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:51:00.579Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:51:00.581Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:00 smithi130 bash[27579]: cluster 2023-08-27T23:50:59.672909+0000 mgr.y (mgr.14901) 3373 : cluster [DBG] pgmap v3380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:01.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:00 smithi130 bash[51302]: debug 2023-08-27T23:51:00.888+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:00.888775+0000) 2023-08-27T23:51:02.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:01 smithi130 bash[51302]: debug 2023-08-27T23:51:01.888+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:01.889061+0000) 2023-08-27T23:51:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:51:02.144Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:51:02.147Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:02 smithi130 bash[27579]: cluster 2023-08-27T23:51:01.673526+0000 mgr.y (mgr.14901) 3374 : cluster [DBG] pgmap v3381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:02 smithi130 bash[27579]: audit 2023-08-27T23:51:02.108645+0000 mon.c (mon.2) 1296 : audit [DBG] from='client.? 172.21.15.6:0/3131293557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:02 smithi130 bash[51302]: debug 2023-08-27T23:51:02.888+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:02.889364+0000) 2023-08-27T23:51:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:02 smithi006 bash[27292]: cluster 2023-08-27T23:51:01.673526+0000 mgr.y (mgr.14901) 3374 : cluster [DBG] pgmap v3381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:02 smithi006 bash[27292]: audit 2023-08-27T23:51:02.108645+0000 mon.c (mon.2) 1296 : audit [DBG] from='client.? 172.21.15.6:0/3131293557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:02 smithi006 bash[33918]: cluster 2023-08-27T23:51:01.673526+0000 mgr.y (mgr.14901) 3374 : cluster [DBG] pgmap v3381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:02 smithi006 bash[33918]: audit 2023-08-27T23:51:02.108645+0000 mon.c (mon.2) 1296 : audit [DBG] from='client.? 172.21.15.6:0/3131293557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:03 smithi130 bash[51302]: debug 2023-08-27T23:51:03.888+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:03.889668+0000) 2023-08-27T23:51:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:04 smithi006 bash[56298]: level=error ts=2023-08-27T23:51:04.042Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:04 smithi006 bash[56298]: level=warn ts=2023-08-27T23:51:04.044Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:04 smithi130 bash[51302]: debug 2023-08-27T23:51:04.888+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:04.889943+0000) 2023-08-27T23:51:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:04 smithi130 bash[27579]: cluster 2023-08-27T23:51:03.674167+0000 mgr.y (mgr.14901) 3375 : cluster [DBG] pgmap v3382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:04 smithi130 bash[27579]: audit 2023-08-27T23:51:04.520452+0000 mon.a (mon.0) 2662 : audit [DBG] from='client.? 172.21.15.6:0/3928216378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:04 smithi006 bash[33918]: cluster 2023-08-27T23:51:03.674167+0000 mgr.y (mgr.14901) 3375 : cluster [DBG] pgmap v3382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:04 smithi006 bash[33918]: audit 2023-08-27T23:51:04.520452+0000 mon.a (mon.0) 2662 : audit [DBG] from='client.? 172.21.15.6:0/3928216378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:04 smithi006 bash[27292]: cluster 2023-08-27T23:51:03.674167+0000 mgr.y (mgr.14901) 3375 : cluster [DBG] pgmap v3382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:04 smithi006 bash[27292]: audit 2023-08-27T23:51:04.520452+0000 mon.a (mon.0) 2662 : audit [DBG] from='client.? 172.21.15.6:0/3928216378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:06 smithi130 bash[27579]: cluster 2023-08-27T23:51:05.675383+0000 mgr.y (mgr.14901) 3376 : cluster [DBG] pgmap v3383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:06.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:05 smithi130 bash[51302]: debug 2023-08-27T23:51:05.884+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:05.890204+0000) 2023-08-27T23:51:06.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:06 smithi006 bash[33918]: cluster 2023-08-27T23:51:05.675383+0000 mgr.y (mgr.14901) 3376 : cluster [DBG] pgmap v3383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:06.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:06 smithi006 bash[27292]: cluster 2023-08-27T23:51:05.675383+0000 mgr.y (mgr.14901) 3376 : cluster [DBG] pgmap v3383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:07.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:07 smithi130 bash[27579]: audit 2023-08-27T23:51:06.929476+0000 mon.c (mon.2) 1297 : audit [DBG] from='client.? 172.21.15.6:0/678249101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:07.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:06 smithi130 bash[51302]: debug 2023-08-27T23:51:06.884+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:06.890473+0000) 2023-08-27T23:51:07.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:07 smithi006 bash[33918]: audit 2023-08-27T23:51:06.929476+0000 mon.c (mon.2) 1297 : audit [DBG] from='client.? 172.21.15.6:0/678249101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:07.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:07 smithi006 bash[27292]: audit 2023-08-27T23:51:06.929476+0000 mon.c (mon.2) 1297 : audit [DBG] from='client.? 172.21.15.6:0/678249101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:08.146 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:07 smithi130 bash[51302]: debug 2023-08-27T23:51:07.884+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:07.890735+0000) 2023-08-27T23:51:08.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:08 smithi006 bash[27292]: cluster 2023-08-27T23:51:07.676047+0000 mgr.y (mgr.14901) 3377 : cluster [DBG] pgmap v3384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:08.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:08 smithi006 bash[33918]: cluster 2023-08-27T23:51:07.676047+0000 mgr.y (mgr.14901) 3377 : cluster [DBG] pgmap v3384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:08 smithi130 bash[27579]: cluster 2023-08-27T23:51:07.676047+0000 mgr.y (mgr.14901) 3377 : cluster [DBG] pgmap v3384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:09.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:08 smithi130 bash[51302]: debug 2023-08-27T23:51:08.888+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:08.891017+0000) 2023-08-27T23:51:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:09 smithi006 bash[27292]: audit 2023-08-27T23:51:09.348410+0000 mon.a (mon.0) 2663 : audit [DBG] from='client.? 172.21.15.6:0/3852993963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:09.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:09 smithi006 bash[33918]: audit 2023-08-27T23:51:09.348410+0000 mon.a (mon.0) 2663 : audit [DBG] from='client.? 172.21.15.6:0/3852993963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:09.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:09 smithi130 bash[27579]: audit 2023-08-27T23:51:09.348410+0000 mon.a (mon.0) 2663 : audit [DBG] from='client.? 172.21.15.6:0/3852993963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:10.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:09 smithi130 bash[51302]: debug 2023-08-27T23:51:09.888+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:09.891217+0000) 2023-08-27T23:51:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:10 smithi006 bash[27292]: cluster 2023-08-27T23:51:09.677292+0000 mgr.y (mgr.14901) 3378 : cluster [DBG] pgmap v3385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:10 smithi006 bash[33918]: cluster 2023-08-27T23:51:09.677292+0000 mgr.y (mgr.14901) 3378 : cluster [DBG] pgmap v3385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:51:10.580Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:51:10.582Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:10 smithi130 bash[27579]: cluster 2023-08-27T23:51:09.677292+0000 mgr.y (mgr.14901) 3378 : cluster [DBG] pgmap v3385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:51:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:51:10] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-27T23:51:11.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:10 smithi130 bash[51302]: debug 2023-08-27T23:51:10.888+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:10.891428+0000) 2023-08-27T23:51:12.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:11 smithi130 bash[51302]: debug 2023-08-27T23:51:11.888+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:11.891706+0000) 2023-08-27T23:51:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:51:12.146Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:51:12.147Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:13.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:12 smithi130 bash[27579]: cluster 2023-08-27T23:51:11.677910+0000 mgr.y (mgr.14901) 3379 : cluster [DBG] pgmap v3386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:13.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:12 smithi130 bash[27579]: audit 2023-08-27T23:51:11.765189+0000 mon.a (mon.0) 2664 : audit [DBG] from='client.? 172.21.15.6:0/1956182586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:13.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:12 smithi130 bash[51302]: debug 2023-08-27T23:51:12.888+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:12.892016+0000) 2023-08-27T23:51:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:12 smithi006 bash[27292]: cluster 2023-08-27T23:51:11.677910+0000 mgr.y (mgr.14901) 3379 : cluster [DBG] pgmap v3386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:12 smithi006 bash[27292]: audit 2023-08-27T23:51:11.765189+0000 mon.a (mon.0) 2664 : audit [DBG] from='client.? 172.21.15.6:0/1956182586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:12 smithi006 bash[33918]: cluster 2023-08-27T23:51:11.677910+0000 mgr.y (mgr.14901) 3379 : cluster [DBG] pgmap v3386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:12 smithi006 bash[33918]: audit 2023-08-27T23:51:11.765189+0000 mon.a (mon.0) 2664 : audit [DBG] from='client.? 172.21.15.6:0/1956182586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:14.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:13 smithi130 bash[51302]: debug 2023-08-27T23:51:13.888+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:13.892313+0000) 2023-08-27T23:51:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:14 smithi006 bash[56298]: level=error ts=2023-08-27T23:51:14.043Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:14 smithi006 bash[56298]: level=warn ts=2023-08-27T23:51:14.045Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:14 smithi130 bash[27579]: cluster 2023-08-27T23:51:13.678511+0000 mgr.y (mgr.14901) 3380 : cluster [DBG] pgmap v3387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:14 smithi130 bash[27579]: audit 2023-08-27T23:51:14.182710+0000 mon.a (mon.0) 2665 : audit [DBG] from='client.? 172.21.15.6:0/2427311629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:15.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:14 smithi130 bash[51302]: debug 2023-08-27T23:51:14.892+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:14.892609+0000) 2023-08-27T23:51:15.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:14 smithi006 bash[27292]: cluster 2023-08-27T23:51:13.678511+0000 mgr.y (mgr.14901) 3380 : cluster [DBG] pgmap v3387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:14 smithi006 bash[27292]: audit 2023-08-27T23:51:14.182710+0000 mon.a (mon.0) 2665 : audit [DBG] from='client.? 172.21.15.6:0/2427311629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:14 smithi006 bash[33918]: cluster 2023-08-27T23:51:13.678511+0000 mgr.y (mgr.14901) 3380 : cluster [DBG] pgmap v3387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:14 smithi006 bash[33918]: audit 2023-08-27T23:51:14.182710+0000 mon.a (mon.0) 2665 : audit [DBG] from='client.? 172.21.15.6:0/2427311629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:16.154 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:15 smithi130 bash[51302]: debug 2023-08-27T23:51:15.888+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:15.892909+0000) 2023-08-27T23:51:16.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:16 smithi006 bash[33918]: cluster 2023-08-27T23:51:15.679609+0000 mgr.y (mgr.14901) 3381 : cluster [DBG] pgmap v3388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:16.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:16 smithi006 bash[27292]: cluster 2023-08-27T23:51:15.679609+0000 mgr.y (mgr.14901) 3381 : cluster [DBG] pgmap v3388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:16.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:16 smithi130 bash[27579]: cluster 2023-08-27T23:51:15.679609+0000 mgr.y (mgr.14901) 3381 : cluster [DBG] pgmap v3388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:17.155 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:16 smithi130 bash[51302]: debug 2023-08-27T23:51:16.888+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:16.893174+0000) 2023-08-27T23:51:17.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:17 smithi006 bash[27292]: audit 2023-08-27T23:51:16.593858+0000 mon.a (mon.0) 2666 : audit [DBG] from='client.? 172.21.15.6:0/848496683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:17.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:17 smithi006 bash[33918]: audit 2023-08-27T23:51:16.593858+0000 mon.a (mon.0) 2666 : audit [DBG] from='client.? 172.21.15.6:0/848496683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:17.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:17 smithi130 bash[27579]: audit 2023-08-27T23:51:16.593858+0000 mon.a (mon.0) 2666 : audit [DBG] from='client.? 172.21.15.6:0/848496683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:18.164 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:17 smithi130 bash[51302]: debug 2023-08-27T23:51:17.888+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:17.893462+0000) 2023-08-27T23:51:18.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:18 smithi006 bash[27292]: cluster 2023-08-27T23:51:17.680250+0000 mgr.y (mgr.14901) 3382 : cluster [DBG] pgmap v3389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:18.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:18 smithi006 bash[33918]: cluster 2023-08-27T23:51:17.680250+0000 mgr.y (mgr.14901) 3382 : cluster [DBG] pgmap v3389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:18 smithi130 bash[27579]: cluster 2023-08-27T23:51:17.680250+0000 mgr.y (mgr.14901) 3382 : cluster [DBG] pgmap v3389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:19.165 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:18 smithi130 bash[51302]: debug 2023-08-27T23:51:18.888+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:18.893780+0000) 2023-08-27T23:51:19.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:19 smithi006 bash[27292]: audit 2023-08-27T23:51:19.007446+0000 mon.c (mon.2) 1298 : audit [DBG] from='client.? 172.21.15.6:0/3357920034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:19.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:19 smithi006 bash[33918]: audit 2023-08-27T23:51:19.007446+0000 mon.c (mon.2) 1298 : audit [DBG] from='client.? 172.21.15.6:0/3357920034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:19 smithi130 bash[27579]: audit 2023-08-27T23:51:19.007446+0000 mon.c (mon.2) 1298 : audit [DBG] from='client.? 172.21.15.6:0/3357920034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:20.171 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:19 smithi130 bash[51302]: debug 2023-08-27T23:51:19.888+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:19.894060+0000) 2023-08-27T23:51:20.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:20 smithi006 bash[27292]: cluster 2023-08-27T23:51:19.681466+0000 mgr.y (mgr.14901) 3383 : cluster [DBG] pgmap v3390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:20.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:20 smithi006 bash[33918]: cluster 2023-08-27T23:51:19.681466+0000 mgr.y (mgr.14901) 3383 : cluster [DBG] pgmap v3390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:20 smithi130 bash[27579]: cluster 2023-08-27T23:51:19.681466+0000 mgr.y (mgr.14901) 3383 : cluster [DBG] pgmap v3390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:51:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:51:20] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-27T23:51:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:51:20.580Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:51:20.582Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:21.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:20 smithi130 bash[51302]: debug 2023-08-27T23:51:20.892+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:20.894359+0000) 2023-08-27T23:51:21.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:21 smithi130 bash[27579]: audit 2023-08-27T23:51:21.418985+0000 mon.a (mon.0) 2667 : audit [DBG] from='client.? 172.21.15.6:0/4164443383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:21.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:21 smithi006 bash[33918]: audit 2023-08-27T23:51:21.418985+0000 mon.a (mon.0) 2667 : audit [DBG] from='client.? 172.21.15.6:0/4164443383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:21.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:21 smithi006 bash[27292]: audit 2023-08-27T23:51:21.418985+0000 mon.a (mon.0) 2667 : audit [DBG] from='client.? 172.21.15.6:0/4164443383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:22.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:21 smithi130 bash[51302]: debug 2023-08-27T23:51:21.888+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:21.894620+0000) 2023-08-27T23:51:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:51:22.146Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:51:22.148Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:22 smithi130 bash[27579]: cluster 2023-08-27T23:51:21.682143+0000 mgr.y (mgr.14901) 3384 : cluster [DBG] pgmap v3391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:22 smithi006 bash[33918]: cluster 2023-08-27T23:51:21.682143+0000 mgr.y (mgr.14901) 3384 : cluster [DBG] pgmap v3391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:22 smithi006 bash[27292]: cluster 2023-08-27T23:51:21.682143+0000 mgr.y (mgr.14901) 3384 : cluster [DBG] pgmap v3391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:23.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:22 smithi130 bash[51302]: debug 2023-08-27T23:51:22.892+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:22.894952+0000) 2023-08-27T23:51:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:23 smithi130 bash[51302]: debug 2023-08-27T23:51:23.892+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:23.895232+0000) 2023-08-27T23:51:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:24 smithi006 bash[56298]: level=error ts=2023-08-27T23:51:24.044Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:24 smithi006 bash[56298]: level=warn ts=2023-08-27T23:51:24.046Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:24 smithi130 bash[27579]: cluster 2023-08-27T23:51:23.682606+0000 mgr.y (mgr.14901) 3385 : cluster [DBG] pgmap v3392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:24 smithi130 bash[27579]: audit 2023-08-27T23:51:23.825335+0000 mon.c (mon.2) 1299 : audit [DBG] from='client.? 172.21.15.6:0/721954038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:24 smithi130 bash[51302]: debug 2023-08-27T23:51:24.892+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:24.895509+0000) 2023-08-27T23:51:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:24 smithi006 bash[33918]: cluster 2023-08-27T23:51:23.682606+0000 mgr.y (mgr.14901) 3385 : cluster [DBG] pgmap v3392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:24 smithi006 bash[33918]: audit 2023-08-27T23:51:23.825335+0000 mon.c (mon.2) 1299 : audit [DBG] from='client.? 172.21.15.6:0/721954038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:24 smithi006 bash[27292]: cluster 2023-08-27T23:51:23.682606+0000 mgr.y (mgr.14901) 3385 : cluster [DBG] pgmap v3392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:25.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:24 smithi006 bash[27292]: audit 2023-08-27T23:51:23.825335+0000 mon.c (mon.2) 1299 : audit [DBG] from='client.? 172.21.15.6:0/721954038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:26.147 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:25 smithi130 bash[51302]: debug 2023-08-27T23:51:25.892+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:25.895809+0000) 2023-08-27T23:51:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:26 smithi006 bash[27292]: cluster 2023-08-27T23:51:25.683653+0000 mgr.y (mgr.14901) 3386 : cluster [DBG] pgmap v3393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:26.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:26 smithi006 bash[33918]: cluster 2023-08-27T23:51:25.683653+0000 mgr.y (mgr.14901) 3386 : cluster [DBG] pgmap v3393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:26.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:26 smithi130 bash[27579]: cluster 2023-08-27T23:51:25.683653+0000 mgr.y (mgr.14901) 3386 : cluster [DBG] pgmap v3393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:27.149 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:26 smithi130 bash[51302]: debug 2023-08-27T23:51:26.891+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:26.896081+0000) 2023-08-27T23:51:27.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:27 smithi006 bash[27292]: audit 2023-08-27T23:51:26.240787+0000 mon.a (mon.0) 2668 : audit [DBG] from='client.? 172.21.15.6:0/3045557611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:27.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:27 smithi006 bash[33918]: audit 2023-08-27T23:51:26.240787+0000 mon.a (mon.0) 2668 : audit [DBG] from='client.? 172.21.15.6:0/3045557611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:27.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:27 smithi130 bash[27579]: audit 2023-08-27T23:51:26.240787+0000 mon.a (mon.0) 2668 : audit [DBG] from='client.? 172.21.15.6:0/3045557611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:28.159 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:27 smithi130 bash[51302]: debug 2023-08-27T23:51:27.891+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:27.896342+0000) 2023-08-27T23:51:28.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:28 smithi006 bash[33918]: cluster 2023-08-27T23:51:27.684362+0000 mgr.y (mgr.14901) 3387 : cluster [DBG] pgmap v3394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:28.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:28 smithi006 bash[27292]: cluster 2023-08-27T23:51:27.684362+0000 mgr.y (mgr.14901) 3387 : cluster [DBG] pgmap v3394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:28.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:28 smithi130 bash[27579]: cluster 2023-08-27T23:51:27.684362+0000 mgr.y (mgr.14901) 3387 : cluster [DBG] pgmap v3394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:29.162 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:28 smithi130 bash[51302]: debug 2023-08-27T23:51:28.895+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:28.896628+0000) 2023-08-27T23:51:29.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:29 smithi006 bash[27292]: audit 2023-08-27T23:51:28.650020+0000 mon.a (mon.0) 2669 : audit [DBG] from='client.? 172.21.15.6:0/772275074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:29.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:29 smithi006 bash[33918]: audit 2023-08-27T23:51:28.650020+0000 mon.a (mon.0) 2669 : audit [DBG] from='client.? 172.21.15.6:0/772275074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:29.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:29 smithi130 bash[27579]: audit 2023-08-27T23:51:28.650020+0000 mon.a (mon.0) 2669 : audit [DBG] from='client.? 172.21.15.6:0/772275074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:30.172 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:29 smithi130 bash[51302]: debug 2023-08-27T23:51:29.895+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:29.896944+0000) 2023-08-27T23:51:30.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:30 smithi006 bash[27292]: cluster 2023-08-27T23:51:29.685491+0000 mgr.y (mgr.14901) 3388 : cluster [DBG] pgmap v3395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:30.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:30 smithi006 bash[33918]: cluster 2023-08-27T23:51:29.685491+0000 mgr.y (mgr.14901) 3388 : cluster [DBG] pgmap v3395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:30.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:30 smithi130 bash[27579]: cluster 2023-08-27T23:51:29.685491+0000 mgr.y (mgr.14901) 3388 : cluster [DBG] pgmap v3395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:51:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:51:30] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-27T23:51:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:51:30.581Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:51:30.583Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:31.174 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:30 smithi130 bash[51302]: debug 2023-08-27T23:51:30.891+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:30.897233+0000) 2023-08-27T23:51:31.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:31 smithi006 bash[27292]: audit 2023-08-27T23:51:31.061863+0000 mon.c (mon.2) 1300 : audit [DBG] from='client.? 172.21.15.6:0/63952952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:31.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:31 smithi006 bash[33918]: audit 2023-08-27T23:51:31.061863+0000 mon.c (mon.2) 1300 : audit [DBG] from='client.? 172.21.15.6:0/63952952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:31.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:31 smithi130 bash[27579]: audit 2023-08-27T23:51:31.061863+0000 mon.c (mon.2) 1300 : audit [DBG] from='client.? 172.21.15.6:0/63952952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:32.183 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:31 smithi130 bash[51302]: debug 2023-08-27T23:51:31.895+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:31.897498+0000) 2023-08-27T23:51:32.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:32 smithi006 bash[27292]: cluster 2023-08-27T23:51:31.686250+0000 mgr.y (mgr.14901) 3389 : cluster [DBG] pgmap v3396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:32.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:32 smithi006 bash[33918]: cluster 2023-08-27T23:51:31.686250+0000 mgr.y (mgr.14901) 3389 : cluster [DBG] pgmap v3396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:51:32.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:51:32.149Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:32 smithi130 bash[27579]: cluster 2023-08-27T23:51:31.686250+0000 mgr.y (mgr.14901) 3389 : cluster [DBG] pgmap v3396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:33.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:32 smithi130 bash[51302]: debug 2023-08-27T23:51:32.895+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:32.897813+0000) 2023-08-27T23:51:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:33 smithi130 bash[27579]: audit 2023-08-27T23:51:33.477847+0000 mon.a (mon.0) 2670 : audit [DBG] from='client.? 172.21.15.6:0/1914577068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:33.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:33 smithi006 bash[27292]: audit 2023-08-27T23:51:33.477847+0000 mon.a (mon.0) 2670 : audit [DBG] from='client.? 172.21.15.6:0/1914577068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:33.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:33 smithi006 bash[33918]: audit 2023-08-27T23:51:33.477847+0000 mon.a (mon.0) 2670 : audit [DBG] from='client.? 172.21.15.6:0/1914577068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:34.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:33 smithi130 bash[51302]: debug 2023-08-27T23:51:33.891+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:33.898076+0000) 2023-08-27T23:51:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:34 smithi006 bash[56298]: level=error ts=2023-08-27T23:51:34.045Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:34 smithi006 bash[56298]: level=warn ts=2023-08-27T23:51:34.047Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:34 smithi130 bash[27579]: cluster 2023-08-27T23:51:33.686934+0000 mgr.y (mgr.14901) 3390 : cluster [DBG] pgmap v3397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:34.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:34 smithi006 bash[27292]: cluster 2023-08-27T23:51:33.686934+0000 mgr.y (mgr.14901) 3390 : cluster [DBG] pgmap v3397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:34.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:34 smithi006 bash[33918]: cluster 2023-08-27T23:51:33.686934+0000 mgr.y (mgr.14901) 3390 : cluster [DBG] pgmap v3397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:35.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:34 smithi130 bash[51302]: debug 2023-08-27T23:51:34.891+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:34.898416+0000) 2023-08-27T23:51:36.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:35 smithi130 bash[51302]: debug 2023-08-27T23:51:35.895+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:35.898690+0000) 2023-08-27T23:51:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:36 smithi130 bash[27579]: cluster 2023-08-27T23:51:35.687683+0000 mgr.y (mgr.14901) 3391 : cluster [DBG] pgmap v3398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:36 smithi130 bash[27579]: audit 2023-08-27T23:51:35.892916+0000 mon.c (mon.2) 1301 : audit [DBG] from='client.? 172.21.15.6:0/764410046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:36.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:36 smithi006 bash[33918]: cluster 2023-08-27T23:51:35.687683+0000 mgr.y (mgr.14901) 3391 : cluster [DBG] pgmap v3398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:36.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:36 smithi006 bash[33918]: audit 2023-08-27T23:51:35.892916+0000 mon.c (mon.2) 1301 : audit [DBG] from='client.? 172.21.15.6:0/764410046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:36.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:36 smithi006 bash[27292]: cluster 2023-08-27T23:51:35.687683+0000 mgr.y (mgr.14901) 3391 : cluster [DBG] pgmap v3398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:36.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:36 smithi006 bash[27292]: audit 2023-08-27T23:51:35.892916+0000 mon.c (mon.2) 1301 : audit [DBG] from='client.? 172.21.15.6:0/764410046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:36 smithi130 bash[51302]: debug 2023-08-27T23:51:36.895+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:36.898923+0000) 2023-08-27T23:51:38.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:37 smithi130 bash[51302]: debug 2023-08-27T23:51:37.895+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:37.899201+0000) 2023-08-27T23:51:39.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:38 smithi130 bash[51302]: debug 2023-08-27T23:51:38.895+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:38.899538+0000) 2023-08-27T23:51:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:38 smithi130 bash[27579]: cluster 2023-08-27T23:51:37.688309+0000 mgr.y (mgr.14901) 3392 : cluster [DBG] pgmap v3399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:38 smithi130 bash[27579]: audit 2023-08-27T23:51:38.312933+0000 mon.a (mon.0) 2671 : audit [DBG] from='client.? 172.21.15.6:0/1132902845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:39.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:38 smithi006 bash[27292]: cluster 2023-08-27T23:51:37.688309+0000 mgr.y (mgr.14901) 3392 : cluster [DBG] pgmap v3399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:39.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:38 smithi006 bash[27292]: audit 2023-08-27T23:51:38.312933+0000 mon.a (mon.0) 2671 : audit [DBG] from='client.? 172.21.15.6:0/1132902845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:39.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:38 smithi006 bash[33918]: cluster 2023-08-27T23:51:37.688309+0000 mgr.y (mgr.14901) 3392 : cluster [DBG] pgmap v3399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:39.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:38 smithi006 bash[33918]: audit 2023-08-27T23:51:38.312933+0000 mon.a (mon.0) 2671 : audit [DBG] from='client.? 172.21.15.6:0/1132902845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:40.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:39 smithi130 bash[51302]: debug 2023-08-27T23:51:39.895+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:39.899798+0000) 2023-08-27T23:51:40.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:40 smithi006 bash[27292]: cluster 2023-08-27T23:51:39.689433+0000 mgr.y (mgr.14901) 3393 : cluster [DBG] pgmap v3400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:40.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:40 smithi006 bash[27292]: audit 2023-08-27T23:51:40.732561+0000 mon.c (mon.2) 1302 : audit [DBG] from='client.? 172.21.15.6:0/1725183013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:51:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:51:40] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-27T23:51:40.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:40 smithi006 bash[33918]: cluster 2023-08-27T23:51:39.689433+0000 mgr.y (mgr.14901) 3393 : cluster [DBG] pgmap v3400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:40.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:40 smithi006 bash[33918]: audit 2023-08-27T23:51:40.732561+0000 mon.c (mon.2) 1302 : audit [DBG] from='client.? 172.21.15.6:0/1725183013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:51:40.582Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:51:40.583Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:40 smithi130 bash[27579]: cluster 2023-08-27T23:51:39.689433+0000 mgr.y (mgr.14901) 3393 : cluster [DBG] pgmap v3400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:40 smithi130 bash[27579]: audit 2023-08-27T23:51:40.732561+0000 mon.c (mon.2) 1302 : audit [DBG] from='client.? 172.21.15.6:0/1725183013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:41.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:40 smithi130 bash[51302]: debug 2023-08-27T23:51:40.895+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:40.900052+0000) 2023-08-27T23:51:42.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:41 smithi130 bash[51302]: debug 2023-08-27T23:51:41.895+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:41.900313+0000) 2023-08-27T23:51:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:51:42.148Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:51:42.151Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:43.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:42 smithi130 bash[51302]: debug 2023-08-27T23:51:42.895+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:42.900613+0000) 2023-08-27T23:51:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:42 smithi130 bash[27579]: cluster 2023-08-27T23:51:41.690077+0000 mgr.y (mgr.14901) 3394 : cluster [DBG] pgmap v3401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:42 smithi130 bash[27579]: audit 2023-08-27T23:51:42.116110+0000 mon.b (mon.1) 439 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:51:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:42 smithi006 bash[33918]: cluster 2023-08-27T23:51:41.690077+0000 mgr.y (mgr.14901) 3394 : cluster [DBG] pgmap v3401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:42 smithi006 bash[33918]: audit 2023-08-27T23:51:42.116110+0000 mon.b (mon.1) 439 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:51:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:42 smithi006 bash[27292]: cluster 2023-08-27T23:51:41.690077+0000 mgr.y (mgr.14901) 3394 : cluster [DBG] pgmap v3401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:42 smithi006 bash[27292]: audit 2023-08-27T23:51:42.116110+0000 mon.b (mon.1) 439 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:51:44.048 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:43 smithi006 bash[33918]: audit 2023-08-27T23:51:43.149706+0000 mon.c (mon.2) 1303 : audit [DBG] from='client.? 172.21.15.6:0/1695891423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:44.048 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:43 smithi006 bash[27292]: audit 2023-08-27T23:51:43.149706+0000 mon.c (mon.2) 1303 : audit [DBG] from='client.? 172.21.15.6:0/1695891423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:44.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:43 smithi130 bash[27579]: audit 2023-08-27T23:51:43.149706+0000 mon.c (mon.2) 1303 : audit [DBG] from='client.? 172.21.15.6:0/1695891423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:44.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:43 smithi130 bash[51302]: debug 2023-08-27T23:51:43.899+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:43.900898+0000) 2023-08-27T23:51:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:44 smithi006 bash[56298]: level=error ts=2023-08-27T23:51:44.045Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:44 smithi006 bash[56298]: level=warn ts=2023-08-27T23:51:44.047Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:44 smithi130 bash[27579]: cluster 2023-08-27T23:51:43.690735+0000 mgr.y (mgr.14901) 3395 : cluster [DBG] pgmap v3402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:44 smithi130 bash[51302]: debug 2023-08-27T23:51:44.895+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:44.901148+0000) 2023-08-27T23:51:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:44 smithi006 bash[33918]: cluster 2023-08-27T23:51:43.690735+0000 mgr.y (mgr.14901) 3395 : cluster [DBG] pgmap v3402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:44 smithi006 bash[27292]: cluster 2023-08-27T23:51:43.690735+0000 mgr.y (mgr.14901) 3395 : cluster [DBG] pgmap v3402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:46.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:46 smithi130 bash[27579]: audit 2023-08-27T23:51:45.564869+0000 mon.a (mon.0) 2672 : audit [DBG] from='client.? 172.21.15.6:0/2177256470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:46 smithi130 bash[27579]: cluster 2023-08-27T23:51:45.691885+0000 mgr.y (mgr.14901) 3396 : cluster [DBG] pgmap v3403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:45 smithi130 bash[51302]: debug 2023-08-27T23:51:45.899+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:45.901441+0000) 2023-08-27T23:51:46.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:46 smithi006 bash[27292]: audit 2023-08-27T23:51:45.564869+0000 mon.a (mon.0) 2672 : audit [DBG] from='client.? 172.21.15.6:0/2177256470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:46.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:46 smithi006 bash[27292]: cluster 2023-08-27T23:51:45.691885+0000 mgr.y (mgr.14901) 3396 : cluster [DBG] pgmap v3403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:46 smithi006 bash[33918]: audit 2023-08-27T23:51:45.564869+0000 mon.a (mon.0) 2672 : audit [DBG] from='client.? 172.21.15.6:0/2177256470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:46.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:46 smithi006 bash[33918]: cluster 2023-08-27T23:51:45.691885+0000 mgr.y (mgr.14901) 3396 : cluster [DBG] pgmap v3403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:47.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:46 smithi130 bash[51302]: debug 2023-08-27T23:51:46.895+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:46.901695+0000) 2023-08-27T23:51:48.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:47 smithi130 bash[51302]: debug 2023-08-27T23:51:47.895+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:47.901966+0000) 2023-08-27T23:51:49.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:48 smithi130 bash[51302]: debug 2023-08-27T23:51:48.899+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:48.902254+0000) 2023-08-27T23:51:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:48 smithi130 bash[27579]: cluster 2023-08-27T23:51:47.692401+0000 mgr.y (mgr.14901) 3397 : cluster [DBG] pgmap v3404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:48 smithi130 bash[27579]: audit 2023-08-27T23:51:48.018837+0000 mon.a (mon.0) 2673 : audit [DBG] from='client.? 172.21.15.6:0/1760665430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:48 smithi130 bash[27579]: audit 2023-08-27T23:51:48.034391+0000 mon.a (mon.0) 2674 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:51:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:48 smithi130 bash[27579]: audit 2023-08-27T23:51:48.453512+0000 mon.a (mon.0) 2675 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:51:49.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:48 smithi006 bash[27292]: cluster 2023-08-27T23:51:47.692401+0000 mgr.y (mgr.14901) 3397 : cluster [DBG] pgmap v3404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:49.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:48 smithi006 bash[27292]: audit 2023-08-27T23:51:48.018837+0000 mon.a (mon.0) 2673 : audit [DBG] from='client.? 172.21.15.6:0/1760665430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:49.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:48 smithi006 bash[27292]: audit 2023-08-27T23:51:48.034391+0000 mon.a (mon.0) 2674 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:51:49.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:48 smithi006 bash[27292]: audit 2023-08-27T23:51:48.453512+0000 mon.a (mon.0) 2675 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:51:49.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:48 smithi006 bash[33918]: cluster 2023-08-27T23:51:47.692401+0000 mgr.y (mgr.14901) 3397 : cluster [DBG] pgmap v3404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:49.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:48 smithi006 bash[33918]: audit 2023-08-27T23:51:48.018837+0000 mon.a (mon.0) 2673 : audit [DBG] from='client.? 172.21.15.6:0/1760665430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:49.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:48 smithi006 bash[33918]: audit 2023-08-27T23:51:48.034391+0000 mon.a (mon.0) 2674 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:51:49.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:48 smithi006 bash[33918]: audit 2023-08-27T23:51:48.453512+0000 mon.a (mon.0) 2675 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:51:50.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:49 smithi130 bash[51302]: debug 2023-08-27T23:51:49.899+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:49.902614+0000) 2023-08-27T23:51:50.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:49 smithi130 bash[27579]: audit 2023-08-27T23:51:48.754672+0000 mon.b (mon.1) 440 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:51:50.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:49 smithi130 bash[27579]: audit 2023-08-27T23:51:48.757196+0000 mon.b (mon.1) 441 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:51:50.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:49 smithi130 bash[27579]: audit 2023-08-27T23:51:48.763641+0000 mon.a (mon.0) 2676 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:51:50.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:49 smithi006 bash[33918]: audit 2023-08-27T23:51:48.754672+0000 mon.b (mon.1) 440 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:51:50.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:49 smithi006 bash[33918]: audit 2023-08-27T23:51:48.757196+0000 mon.b (mon.1) 441 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:51:50.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:49 smithi006 bash[33918]: audit 2023-08-27T23:51:48.763641+0000 mon.a (mon.0) 2676 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:51:50.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:49 smithi006 bash[27292]: audit 2023-08-27T23:51:48.754672+0000 mon.b (mon.1) 440 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:51:50.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:49 smithi006 bash[27292]: audit 2023-08-27T23:51:48.757196+0000 mon.b (mon.1) 441 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:51:50.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:49 smithi006 bash[27292]: audit 2023-08-27T23:51:48.763641+0000 mon.a (mon.0) 2676 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:51:50.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:51:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:51:50] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-27T23:51:50.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:50 smithi006 bash[33918]: cluster 2023-08-27T23:51:49.693623+0000 mgr.y (mgr.14901) 3398 : cluster [DBG] pgmap v3405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:50.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:50 smithi006 bash[33918]: audit 2023-08-27T23:51:50.439913+0000 mon.c (mon.2) 1304 : audit [DBG] from='client.? 172.21.15.6:0/1740030669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:50.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:50 smithi006 bash[27292]: cluster 2023-08-27T23:51:49.693623+0000 mgr.y (mgr.14901) 3398 : cluster [DBG] pgmap v3405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:50.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:50 smithi006 bash[27292]: audit 2023-08-27T23:51:50.439913+0000 mon.c (mon.2) 1304 : audit [DBG] from='client.? 172.21.15.6:0/1740030669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:51:50.582Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:51:50.584Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:51.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:50 smithi130 bash[51302]: debug 2023-08-27T23:51:50.899+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:50.902984+0000) 2023-08-27T23:51:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:50 smithi130 bash[27579]: cluster 2023-08-27T23:51:49.693623+0000 mgr.y (mgr.14901) 3398 : cluster [DBG] pgmap v3405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:50 smithi130 bash[27579]: audit 2023-08-27T23:51:50.439913+0000 mon.c (mon.2) 1304 : audit [DBG] from='client.? 172.21.15.6:0/1740030669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:52.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:51 smithi130 bash[51302]: debug 2023-08-27T23:51:51.899+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:51.903353+0000) 2023-08-27T23:51:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:51:52.150Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:51:52.152Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:52 smithi130 bash[27579]: cluster 2023-08-27T23:51:51.694377+0000 mgr.y (mgr.14901) 3399 : cluster [DBG] pgmap v3406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:52 smithi130 bash[51302]: debug 2023-08-27T23:51:52.899+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:52.903705+0000) 2023-08-27T23:51:53.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:52 smithi006 bash[33918]: cluster 2023-08-27T23:51:51.694377+0000 mgr.y (mgr.14901) 3399 : cluster [DBG] pgmap v3406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:52 smithi006 bash[27292]: cluster 2023-08-27T23:51:51.694377+0000 mgr.y (mgr.14901) 3399 : cluster [DBG] pgmap v3406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:54.048 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:53 smithi006 bash[33918]: audit 2023-08-27T23:51:52.856236+0000 mon.c (mon.2) 1305 : audit [DBG] from='client.? 172.21.15.6:0/3271235718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:54.049 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:53 smithi006 bash[27292]: audit 2023-08-27T23:51:52.856236+0000 mon.c (mon.2) 1305 : audit [DBG] from='client.? 172.21.15.6:0/3271235718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:54.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:53 smithi130 bash[27579]: audit 2023-08-27T23:51:52.856236+0000 mon.c (mon.2) 1305 : audit [DBG] from='client.? 172.21.15.6:0/3271235718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:54.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:53 smithi130 bash[51302]: debug 2023-08-27T23:51:53.903+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:53.904058+0000) 2023-08-27T23:51:54.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:54 smithi006 bash[56298]: level=error ts=2023-08-27T23:51:54.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:51:54 smithi006 bash[56298]: level=warn ts=2023-08-27T23:51:54.048Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:51:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:54 smithi130 bash[27579]: cluster 2023-08-27T23:51:53.695028+0000 mgr.y (mgr.14901) 3400 : cluster [DBG] pgmap v3407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:54 smithi130 bash[51302]: debug 2023-08-27T23:51:54.903+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:54.904372+0000) 2023-08-27T23:51:55.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:54 smithi006 bash[33918]: cluster 2023-08-27T23:51:53.695028+0000 mgr.y (mgr.14901) 3400 : cluster [DBG] pgmap v3407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:54 smithi006 bash[27292]: cluster 2023-08-27T23:51:53.695028+0000 mgr.y (mgr.14901) 3400 : cluster [DBG] pgmap v3407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:56.165 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:55 smithi130 bash[51302]: debug 2023-08-27T23:51:55.903+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:55.904676+0000) 2023-08-27T23:51:56.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:56 smithi006 bash[27292]: audit 2023-08-27T23:51:55.272521+0000 mon.a (mon.0) 2677 : audit [DBG] from='client.? 172.21.15.6:0/2974657114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:56.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:56 smithi006 bash[27292]: cluster 2023-08-27T23:51:55.696126+0000 mgr.y (mgr.14901) 3401 : cluster [DBG] pgmap v3408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:56.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:56 smithi006 bash[33918]: audit 2023-08-27T23:51:55.272521+0000 mon.a (mon.0) 2677 : audit [DBG] from='client.? 172.21.15.6:0/2974657114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:56.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:56 smithi006 bash[33918]: cluster 2023-08-27T23:51:55.696126+0000 mgr.y (mgr.14901) 3401 : cluster [DBG] pgmap v3408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:56 smithi130 bash[27579]: audit 2023-08-27T23:51:55.272521+0000 mon.a (mon.0) 2677 : audit [DBG] from='client.? 172.21.15.6:0/2974657114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:56 smithi130 bash[27579]: cluster 2023-08-27T23:51:55.696126+0000 mgr.y (mgr.14901) 3401 : cluster [DBG] pgmap v3408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:57.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:56 smithi130 bash[51302]: debug 2023-08-27T23:51:56.899+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:56.905110+0000) 2023-08-27T23:51:58.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:57 smithi130 bash[27579]: audit 2023-08-27T23:51:57.691485+0000 mon.c (mon.2) 1306 : audit [DBG] from='client.? 172.21.15.6:0/4123365215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:58.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:57 smithi130 bash[51302]: debug 2023-08-27T23:51:57.899+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:57.905367+0000) 2023-08-27T23:51:58.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:57 smithi006 bash[33918]: audit 2023-08-27T23:51:57.691485+0000 mon.c (mon.2) 1306 : audit [DBG] from='client.? 172.21.15.6:0/4123365215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:58.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:57 smithi006 bash[27292]: audit 2023-08-27T23:51:57.691485+0000 mon.c (mon.2) 1306 : audit [DBG] from='client.? 172.21.15.6:0/4123365215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:51:58 smithi130 bash[27579]: cluster 2023-08-27T23:51:57.696806+0000 mgr.y (mgr.14901) 3402 : cluster [DBG] pgmap v3409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:59.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:58 smithi130 bash[51302]: debug 2023-08-27T23:51:58.899+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:58.905648+0000) 2023-08-27T23:51:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:51:58 smithi006 bash[33918]: cluster 2023-08-27T23:51:57.696806+0000 mgr.y (mgr.14901) 3402 : cluster [DBG] pgmap v3409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:51:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:51:58 smithi006 bash[27292]: cluster 2023-08-27T23:51:57.696806+0000 mgr.y (mgr.14901) 3402 : cluster [DBG] pgmap v3409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:00.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:51:59 smithi130 bash[51302]: debug 2023-08-27T23:51:59.899+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:59.905921+0000) 2023-08-27T23:52:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:52:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:52:00] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-27T23:52:00.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:00 smithi006 bash[27292]: cluster 2023-08-27T23:51:59.697907+0000 mgr.y (mgr.14901) 3403 : cluster [DBG] pgmap v3410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:00.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:00 smithi006 bash[27292]: audit 2023-08-27T23:52:00.194885+0000 mon.c (mon.2) 1307 : audit [DBG] from='client.? 172.21.15.6:0/3911435697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:00.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:00 smithi006 bash[33918]: cluster 2023-08-27T23:51:59.697907+0000 mgr.y (mgr.14901) 3403 : cluster [DBG] pgmap v3410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:00.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:00 smithi006 bash[33918]: audit 2023-08-27T23:52:00.194885+0000 mon.c (mon.2) 1307 : audit [DBG] from='client.? 172.21.15.6:0/3911435697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:52:00.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:52:00.586Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:00 smithi130 bash[27579]: cluster 2023-08-27T23:51:59.697907+0000 mgr.y (mgr.14901) 3403 : cluster [DBG] pgmap v3410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:00 smithi130 bash[27579]: audit 2023-08-27T23:52:00.194885+0000 mon.c (mon.2) 1307 : audit [DBG] from='client.? 172.21.15.6:0/3911435697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:01.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:00 smithi130 bash[51302]: debug 2023-08-27T23:52:00.899+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:00.906192+0000) 2023-08-27T23:52:02.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:01 smithi130 bash[51302]: debug 2023-08-27T23:52:01.903+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:01.906450+0000) 2023-08-27T23:52:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:52:02.151Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:52:02.153Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:02 smithi130 bash[51302]: debug 2023-08-27T23:52:02.903+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:02.906743+0000) 2023-08-27T23:52:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:02 smithi130 bash[27579]: cluster 2023-08-27T23:52:01.698693+0000 mgr.y (mgr.14901) 3404 : cluster [DBG] pgmap v3411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:02 smithi130 bash[27579]: audit 2023-08-27T23:52:02.604186+0000 mon.c (mon.2) 1308 : audit [DBG] from='client.? 172.21.15.6:0/3060047121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:02 smithi006 bash[33918]: cluster 2023-08-27T23:52:01.698693+0000 mgr.y (mgr.14901) 3404 : cluster [DBG] pgmap v3411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:02 smithi006 bash[33918]: audit 2023-08-27T23:52:02.604186+0000 mon.c (mon.2) 1308 : audit [DBG] from='client.? 172.21.15.6:0/3060047121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:02 smithi006 bash[27292]: cluster 2023-08-27T23:52:01.698693+0000 mgr.y (mgr.14901) 3404 : cluster [DBG] pgmap v3411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:02 smithi006 bash[27292]: audit 2023-08-27T23:52:02.604186+0000 mon.c (mon.2) 1308 : audit [DBG] from='client.? 172.21.15.6:0/3060047121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:03 smithi130 bash[51302]: debug 2023-08-27T23:52:03.903+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:03.907009+0000) 2023-08-27T23:52:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:04 smithi006 bash[56298]: level=error ts=2023-08-27T23:52:04.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:04 smithi006 bash[56298]: level=warn ts=2023-08-27T23:52:04.048Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:04 smithi130 bash[27579]: cluster 2023-08-27T23:52:03.699353+0000 mgr.y (mgr.14901) 3405 : cluster [DBG] pgmap v3412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:04 smithi130 bash[51302]: debug 2023-08-27T23:52:04.903+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:04.907270+0000) 2023-08-27T23:52:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:04 smithi006 bash[27292]: cluster 2023-08-27T23:52:03.699353+0000 mgr.y (mgr.14901) 3405 : cluster [DBG] pgmap v3412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:04 smithi006 bash[33918]: cluster 2023-08-27T23:52:03.699353+0000 mgr.y (mgr.14901) 3405 : cluster [DBG] pgmap v3412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:05 smithi130 bash[27579]: audit 2023-08-27T23:52:05.019203+0000 mon.a (mon.0) 2678 : audit [DBG] from='client.? 172.21.15.6:0/1728546747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:06.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:05 smithi130 bash[51302]: debug 2023-08-27T23:52:05.903+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:05.907548+0000) 2023-08-27T23:52:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:05 smithi006 bash[33918]: audit 2023-08-27T23:52:05.019203+0000 mon.a (mon.0) 2678 : audit [DBG] from='client.? 172.21.15.6:0/1728546747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:06.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:05 smithi006 bash[27292]: audit 2023-08-27T23:52:05.019203+0000 mon.a (mon.0) 2678 : audit [DBG] from='client.? 172.21.15.6:0/1728546747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:06 smithi130 bash[27579]: cluster 2023-08-27T23:52:05.700490+0000 mgr.y (mgr.14901) 3406 : cluster [DBG] pgmap v3413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:07.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:06 smithi130 bash[51302]: debug 2023-08-27T23:52:06.903+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:06.907845+0000) 2023-08-27T23:52:07.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:06 smithi006 bash[33918]: cluster 2023-08-27T23:52:05.700490+0000 mgr.y (mgr.14901) 3406 : cluster [DBG] pgmap v3413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:07.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:06 smithi006 bash[27292]: cluster 2023-08-27T23:52:05.700490+0000 mgr.y (mgr.14901) 3406 : cluster [DBG] pgmap v3413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:08.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:07 smithi130 bash[27579]: audit 2023-08-27T23:52:07.430141+0000 mon.a (mon.0) 2679 : audit [DBG] from='client.? 172.21.15.6:0/2684327031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:08.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:07 smithi130 bash[51302]: debug 2023-08-27T23:52:07.903+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:07.908147+0000) 2023-08-27T23:52:08.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:07 smithi006 bash[33918]: audit 2023-08-27T23:52:07.430141+0000 mon.a (mon.0) 2679 : audit [DBG] from='client.? 172.21.15.6:0/2684327031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:08.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:07 smithi006 bash[27292]: audit 2023-08-27T23:52:07.430141+0000 mon.a (mon.0) 2679 : audit [DBG] from='client.? 172.21.15.6:0/2684327031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:08 smithi130 bash[27579]: cluster 2023-08-27T23:52:07.701181+0000 mgr.y (mgr.14901) 3407 : cluster [DBG] pgmap v3414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:09.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:08 smithi130 bash[51302]: debug 2023-08-27T23:52:08.907+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:08.908426+0000) 2023-08-27T23:52:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:08 smithi006 bash[27292]: cluster 2023-08-27T23:52:07.701181+0000 mgr.y (mgr.14901) 3407 : cluster [DBG] pgmap v3414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:08 smithi006 bash[33918]: cluster 2023-08-27T23:52:07.701181+0000 mgr.y (mgr.14901) 3407 : cluster [DBG] pgmap v3414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:10.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:09 smithi130 bash[51302]: debug 2023-08-27T23:52:09.903+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:09.908698+0000) 2023-08-27T23:52:10.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:10 smithi006 bash[27292]: cluster 2023-08-27T23:52:09.702230+0000 mgr.y (mgr.14901) 3408 : cluster [DBG] pgmap v3415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:10.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:10 smithi006 bash[27292]: audit 2023-08-27T23:52:09.850995+0000 mon.a (mon.0) 2680 : audit [DBG] from='client.? 172.21.15.6:0/1947670888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:52:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:52:10] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-27T23:52:10.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:10 smithi006 bash[33918]: cluster 2023-08-27T23:52:09.702230+0000 mgr.y (mgr.14901) 3408 : cluster [DBG] pgmap v3415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:10.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:10 smithi006 bash[33918]: audit 2023-08-27T23:52:09.850995+0000 mon.a (mon.0) 2680 : audit [DBG] from='client.? 172.21.15.6:0/1947670888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:52:10.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:52:10.586Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:10 smithi130 bash[27579]: cluster 2023-08-27T23:52:09.702230+0000 mgr.y (mgr.14901) 3408 : cluster [DBG] pgmap v3415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:10 smithi130 bash[27579]: audit 2023-08-27T23:52:09.850995+0000 mon.a (mon.0) 2680 : audit [DBG] from='client.? 172.21.15.6:0/1947670888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:11.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:10 smithi130 bash[51302]: debug 2023-08-27T23:52:10.907+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:10.908933+0000) 2023-08-27T23:52:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:11 smithi130 bash[51302]: debug 2023-08-27T23:52:11.903+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:11.909195+0000) 2023-08-27T23:52:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:52:12.151Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:52:12.153Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:12 smithi130 bash[27579]: cluster 2023-08-27T23:52:11.702870+0000 mgr.y (mgr.14901) 3409 : cluster [DBG] pgmap v3416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:12 smithi130 bash[27579]: audit 2023-08-27T23:52:12.261713+0000 mon.a (mon.0) 2681 : audit [DBG] from='client.? 172.21.15.6:0/1840206827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:13.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:12 smithi130 bash[51302]: debug 2023-08-27T23:52:12.903+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:12.909500+0000) 2023-08-27T23:52:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:12 smithi006 bash[27292]: cluster 2023-08-27T23:52:11.702870+0000 mgr.y (mgr.14901) 3409 : cluster [DBG] pgmap v3416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:12 smithi006 bash[27292]: audit 2023-08-27T23:52:12.261713+0000 mon.a (mon.0) 2681 : audit [DBG] from='client.? 172.21.15.6:0/1840206827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:12 smithi006 bash[33918]: cluster 2023-08-27T23:52:11.702870+0000 mgr.y (mgr.14901) 3409 : cluster [DBG] pgmap v3416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:12 smithi006 bash[33918]: audit 2023-08-27T23:52:12.261713+0000 mon.a (mon.0) 2681 : audit [DBG] from='client.? 172.21.15.6:0/1840206827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:14.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:13 smithi130 bash[51302]: debug 2023-08-27T23:52:13.907+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:13.909777+0000) 2023-08-27T23:52:14.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:14 smithi006 bash[56298]: level=error ts=2023-08-27T23:52:14.047Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:14 smithi006 bash[56298]: level=warn ts=2023-08-27T23:52:14.049Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:15.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:14 smithi130 bash[51302]: debug 2023-08-27T23:52:14.907+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:14.910091+0000) 2023-08-27T23:52:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:14 smithi130 bash[27579]: cluster 2023-08-27T23:52:13.703504+0000 mgr.y (mgr.14901) 3410 : cluster [DBG] pgmap v3417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:14 smithi130 bash[27579]: audit 2023-08-27T23:52:14.677851+0000 mon.c (mon.2) 1309 : audit [DBG] from='client.? 172.21.15.6:0/3471636949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:14 smithi006 bash[33918]: cluster 2023-08-27T23:52:13.703504+0000 mgr.y (mgr.14901) 3410 : cluster [DBG] pgmap v3417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:14 smithi006 bash[33918]: audit 2023-08-27T23:52:14.677851+0000 mon.c (mon.2) 1309 : audit [DBG] from='client.? 172.21.15.6:0/3471636949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:14 smithi006 bash[27292]: cluster 2023-08-27T23:52:13.703504+0000 mgr.y (mgr.14901) 3410 : cluster [DBG] pgmap v3417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:14 smithi006 bash[27292]: audit 2023-08-27T23:52:14.677851+0000 mon.c (mon.2) 1309 : audit [DBG] from='client.? 172.21.15.6:0/3471636949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:16 smithi130 bash[27579]: cluster 2023-08-27T23:52:15.704621+0000 mgr.y (mgr.14901) 3411 : cluster [DBG] pgmap v3418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:16.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:15 smithi130 bash[51302]: debug 2023-08-27T23:52:15.903+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:15.910372+0000) 2023-08-27T23:52:16.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:16 smithi006 bash[27292]: cluster 2023-08-27T23:52:15.704621+0000 mgr.y (mgr.14901) 3411 : cluster [DBG] pgmap v3418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:16.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:16 smithi006 bash[33918]: cluster 2023-08-27T23:52:15.704621+0000 mgr.y (mgr.14901) 3411 : cluster [DBG] pgmap v3418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:17.163 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:16 smithi130 bash[51302]: debug 2023-08-27T23:52:16.907+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:16.910638+0000) 2023-08-27T23:52:17.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:17 smithi006 bash[27292]: audit 2023-08-27T23:52:17.099259+0000 mon.c (mon.2) 1310 : audit [DBG] from='client.? 172.21.15.6:0/3611562182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:17.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:17 smithi006 bash[33918]: audit 2023-08-27T23:52:17.099259+0000 mon.c (mon.2) 1310 : audit [DBG] from='client.? 172.21.15.6:0/3611562182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:17.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:17 smithi130 bash[27579]: audit 2023-08-27T23:52:17.099259+0000 mon.c (mon.2) 1310 : audit [DBG] from='client.? 172.21.15.6:0/3611562182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:18.171 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:17 smithi130 bash[51302]: debug 2023-08-27T23:52:17.907+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:17.910944+0000) 2023-08-27T23:52:18.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:18 smithi006 bash[27292]: cluster 2023-08-27T23:52:17.705305+0000 mgr.y (mgr.14901) 3412 : cluster [DBG] pgmap v3419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:18.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:18 smithi006 bash[33918]: cluster 2023-08-27T23:52:17.705305+0000 mgr.y (mgr.14901) 3412 : cluster [DBG] pgmap v3419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:18 smithi130 bash[27579]: cluster 2023-08-27T23:52:17.705305+0000 mgr.y (mgr.14901) 3412 : cluster [DBG] pgmap v3419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:19.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:18 smithi130 bash[51302]: debug 2023-08-27T23:52:18.911+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:18.911251+0000) 2023-08-27T23:52:19.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:19 smithi130 bash[27579]: audit 2023-08-27T23:52:19.515280+0000 mon.c (mon.2) 1311 : audit [DBG] from='client.? 172.21.15.6:0/2318199319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:19.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:19 smithi006 bash[27292]: audit 2023-08-27T23:52:19.515280+0000 mon.c (mon.2) 1311 : audit [DBG] from='client.? 172.21.15.6:0/2318199319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:19 smithi006 bash[33918]: audit 2023-08-27T23:52:19.515280+0000 mon.c (mon.2) 1311 : audit [DBG] from='client.? 172.21.15.6:0/2318199319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:20.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:19 smithi130 bash[51302]: debug 2023-08-27T23:52:19.907+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:19.911536+0000) 2023-08-27T23:52:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:20 smithi130 bash[27579]: cluster 2023-08-27T23:52:19.706439+0000 mgr.y (mgr.14901) 3413 : cluster [DBG] pgmap v3420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:20.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:20 smithi006 bash[27292]: cluster 2023-08-27T23:52:19.706439+0000 mgr.y (mgr.14901) 3413 : cluster [DBG] pgmap v3420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:52:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:52:20] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:52:20.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:20 smithi006 bash[33918]: cluster 2023-08-27T23:52:19.706439+0000 mgr.y (mgr.14901) 3413 : cluster [DBG] pgmap v3420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:20.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:52:20.585Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:20.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:52:20.586Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:21.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:20 smithi130 bash[51302]: debug 2023-08-27T23:52:20.907+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:20.911824+0000) 2023-08-27T23:52:22.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:21 smithi130 bash[51302]: debug 2023-08-27T23:52:21.911+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:21.912132+0000) 2023-08-27T23:52:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:52:22.152Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:52:22.154Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:22 smithi130 bash[27579]: cluster 2023-08-27T23:52:21.707109+0000 mgr.y (mgr.14901) 3414 : cluster [DBG] pgmap v3421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:22 smithi130 bash[27579]: audit 2023-08-27T23:52:21.927308+0000 mon.a (mon.0) 2682 : audit [DBG] from='client.? 172.21.15.6:0/1623189021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:23.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:22 smithi130 bash[51302]: debug 2023-08-27T23:52:22.906+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:22.912411+0000) 2023-08-27T23:52:23.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:22 smithi006 bash[33918]: cluster 2023-08-27T23:52:21.707109+0000 mgr.y (mgr.14901) 3414 : cluster [DBG] pgmap v3421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:23.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:22 smithi006 bash[33918]: audit 2023-08-27T23:52:21.927308+0000 mon.a (mon.0) 2682 : audit [DBG] from='client.? 172.21.15.6:0/1623189021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:23.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:22 smithi006 bash[27292]: cluster 2023-08-27T23:52:21.707109+0000 mgr.y (mgr.14901) 3414 : cluster [DBG] pgmap v3421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:23.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:22 smithi006 bash[27292]: audit 2023-08-27T23:52:21.927308+0000 mon.a (mon.0) 2682 : audit [DBG] from='client.? 172.21.15.6:0/1623189021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:23 smithi130 bash[51302]: debug 2023-08-27T23:52:23.906+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:23.912680+0000) 2023-08-27T23:52:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:24 smithi006 bash[56298]: level=error ts=2023-08-27T23:52:24.047Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:24 smithi006 bash[56298]: level=warn ts=2023-08-27T23:52:24.049Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:24 smithi130 bash[51302]: debug 2023-08-27T23:52:24.910+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:24.912957+0000) 2023-08-27T23:52:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:24 smithi130 bash[27579]: cluster 2023-08-27T23:52:23.707827+0000 mgr.y (mgr.14901) 3415 : cluster [DBG] pgmap v3422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:24 smithi130 bash[27579]: audit 2023-08-27T23:52:24.339978+0000 mon.a (mon.0) 2683 : audit [DBG] from='client.? 172.21.15.6:0/2664179360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:24 smithi006 bash[33918]: cluster 2023-08-27T23:52:23.707827+0000 mgr.y (mgr.14901) 3415 : cluster [DBG] pgmap v3422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:24 smithi006 bash[33918]: audit 2023-08-27T23:52:24.339978+0000 mon.a (mon.0) 2683 : audit [DBG] from='client.? 172.21.15.6:0/2664179360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:24 smithi006 bash[27292]: cluster 2023-08-27T23:52:23.707827+0000 mgr.y (mgr.14901) 3415 : cluster [DBG] pgmap v3422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:24 smithi006 bash[27292]: audit 2023-08-27T23:52:24.339978+0000 mon.a (mon.0) 2683 : audit [DBG] from='client.? 172.21.15.6:0/2664179360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:26 smithi130 bash[27579]: cluster 2023-08-27T23:52:25.709005+0000 mgr.y (mgr.14901) 3416 : cluster [DBG] pgmap v3423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:26.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:25 smithi130 bash[51302]: debug 2023-08-27T23:52:25.906+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:25.913245+0000) 2023-08-27T23:52:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:26 smithi006 bash[27292]: cluster 2023-08-27T23:52:25.709005+0000 mgr.y (mgr.14901) 3416 : cluster [DBG] pgmap v3423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:26.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:26 smithi006 bash[33918]: cluster 2023-08-27T23:52:25.709005+0000 mgr.y (mgr.14901) 3416 : cluster [DBG] pgmap v3423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:27.166 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:26 smithi130 bash[51302]: debug 2023-08-27T23:52:26.910+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:26.913538+0000) 2023-08-27T23:52:27.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:27 smithi006 bash[27292]: audit 2023-08-27T23:52:26.754138+0000 mon.a (mon.0) 2684 : audit [DBG] from='client.? 172.21.15.6:0/3651487172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:27.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:27 smithi006 bash[33918]: audit 2023-08-27T23:52:26.754138+0000 mon.a (mon.0) 2684 : audit [DBG] from='client.? 172.21.15.6:0/3651487172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:27.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:27 smithi130 bash[27579]: audit 2023-08-27T23:52:26.754138+0000 mon.a (mon.0) 2684 : audit [DBG] from='client.? 172.21.15.6:0/3651487172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:28.175 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:27 smithi130 bash[51302]: debug 2023-08-27T23:52:27.910+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:27.913811+0000) 2023-08-27T23:52:28.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:28 smithi006 bash[33918]: cluster 2023-08-27T23:52:27.709636+0000 mgr.y (mgr.14901) 3417 : cluster [DBG] pgmap v3424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:28.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:28 smithi006 bash[27292]: cluster 2023-08-27T23:52:27.709636+0000 mgr.y (mgr.14901) 3417 : cluster [DBG] pgmap v3424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:28 smithi130 bash[27579]: cluster 2023-08-27T23:52:27.709636+0000 mgr.y (mgr.14901) 3417 : cluster [DBG] pgmap v3424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:29.176 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:28 smithi130 bash[51302]: debug 2023-08-27T23:52:28.906+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:28.914080+0000) 2023-08-27T23:52:29.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:29 smithi006 bash[33918]: audit 2023-08-27T23:52:29.164472+0000 mon.a (mon.0) 2685 : audit [DBG] from='client.? 172.21.15.6:0/1839709307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:29.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:29 smithi006 bash[27292]: audit 2023-08-27T23:52:29.164472+0000 mon.a (mon.0) 2685 : audit [DBG] from='client.? 172.21.15.6:0/1839709307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:29.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:29 smithi130 bash[27579]: audit 2023-08-27T23:52:29.164472+0000 mon.a (mon.0) 2685 : audit [DBG] from='client.? 172.21.15.6:0/1839709307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:30.186 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:29 smithi130 bash[51302]: debug 2023-08-27T23:52:29.910+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:29.914395+0000) 2023-08-27T23:52:30.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:30 smithi006 bash[27292]: cluster 2023-08-27T23:52:29.710758+0000 mgr.y (mgr.14901) 3418 : cluster [DBG] pgmap v3425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:30.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:30 smithi006 bash[33918]: cluster 2023-08-27T23:52:29.710758+0000 mgr.y (mgr.14901) 3418 : cluster [DBG] pgmap v3425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:30 smithi130 bash[27579]: cluster 2023-08-27T23:52:29.710758+0000 mgr.y (mgr.14901) 3418 : cluster [DBG] pgmap v3425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:52:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:52:30] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-27T23:52:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:52:30.586Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:52:30.588Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:31.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:30 smithi130 bash[51302]: debug 2023-08-27T23:52:30.910+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:30.914673+0000) 2023-08-27T23:52:31.916 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:31 smithi130 bash[27579]: audit 2023-08-27T23:52:31.582747+0000 mon.a (mon.0) 2686 : audit [DBG] from='client.? 172.21.15.6:0/2653917204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:31.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:31 smithi006 bash[27292]: audit 2023-08-27T23:52:31.582747+0000 mon.a (mon.0) 2686 : audit [DBG] from='client.? 172.21.15.6:0/2653917204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:31.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:31 smithi006 bash[33918]: audit 2023-08-27T23:52:31.582747+0000 mon.a (mon.0) 2686 : audit [DBG] from='client.? 172.21.15.6:0/2653917204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:32.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:31 smithi130 bash[51302]: debug 2023-08-27T23:52:31.914+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:31.914988+0000) 2023-08-27T23:52:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:52:32.154Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:52:32.156Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:32.916 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:32 smithi130 bash[27579]: cluster 2023-08-27T23:52:31.711431+0000 mgr.y (mgr.14901) 3419 : cluster [DBG] pgmap v3426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:32 smithi006 bash[33918]: cluster 2023-08-27T23:52:31.711431+0000 mgr.y (mgr.14901) 3419 : cluster [DBG] pgmap v3426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:32.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:32 smithi006 bash[27292]: cluster 2023-08-27T23:52:31.711431+0000 mgr.y (mgr.14901) 3419 : cluster [DBG] pgmap v3426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:33.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:32 smithi130 bash[51302]: debug 2023-08-27T23:52:32.914+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:32.915216+0000) 2023-08-27T23:52:34.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:33 smithi130 bash[51302]: debug 2023-08-27T23:52:33.914+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:33.915538+0000) 2023-08-27T23:52:34.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:34 smithi006 bash[56298]: level=error ts=2023-08-27T23:52:34.048Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:34 smithi006 bash[56298]: level=warn ts=2023-08-27T23:52:34.050Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:35.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:34 smithi130 bash[27579]: cluster 2023-08-27T23:52:33.712067+0000 mgr.y (mgr.14901) 3420 : cluster [DBG] pgmap v3427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:35.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:34 smithi130 bash[27579]: audit 2023-08-27T23:52:33.997248+0000 mon.a (mon.0) 2687 : audit [DBG] from='client.? 172.21.15.6:0/3714159899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:35.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:34 smithi130 bash[51302]: debug 2023-08-27T23:52:34.914+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:34.915801+0000) 2023-08-27T23:52:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:34 smithi006 bash[27292]: cluster 2023-08-27T23:52:33.712067+0000 mgr.y (mgr.14901) 3420 : cluster [DBG] pgmap v3427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:34 smithi006 bash[27292]: audit 2023-08-27T23:52:33.997248+0000 mon.a (mon.0) 2687 : audit [DBG] from='client.? 172.21.15.6:0/3714159899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:34 smithi006 bash[33918]: cluster 2023-08-27T23:52:33.712067+0000 mgr.y (mgr.14901) 3420 : cluster [DBG] pgmap v3427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:34 smithi006 bash[33918]: audit 2023-08-27T23:52:33.997248+0000 mon.a (mon.0) 2687 : audit [DBG] from='client.? 172.21.15.6:0/3714159899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:36 smithi130 bash[27579]: cluster 2023-08-27T23:52:35.713163+0000 mgr.y (mgr.14901) 3421 : cluster [DBG] pgmap v3428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:36.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:35 smithi130 bash[51302]: debug 2023-08-27T23:52:35.914+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:35.916105+0000) 2023-08-27T23:52:36.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:36 smithi006 bash[33918]: cluster 2023-08-27T23:52:35.713163+0000 mgr.y (mgr.14901) 3421 : cluster [DBG] pgmap v3428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:36.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:36 smithi006 bash[27292]: cluster 2023-08-27T23:52:35.713163+0000 mgr.y (mgr.14901) 3421 : cluster [DBG] pgmap v3428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:37.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:37 smithi130 bash[27579]: audit 2023-08-27T23:52:36.416605+0000 mon.c (mon.2) 1312 : audit [DBG] from='client.? 172.21.15.6:0/691902680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:36 smithi130 bash[51302]: debug 2023-08-27T23:52:36.914+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:36.916386+0000) 2023-08-27T23:52:37.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:37 smithi006 bash[27292]: audit 2023-08-27T23:52:36.416605+0000 mon.c (mon.2) 1312 : audit [DBG] from='client.? 172.21.15.6:0/691902680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:37.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:37 smithi006 bash[33918]: audit 2023-08-27T23:52:36.416605+0000 mon.c (mon.2) 1312 : audit [DBG] from='client.? 172.21.15.6:0/691902680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:38.174 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:37 smithi130 bash[51302]: debug 2023-08-27T23:52:37.914+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:37.916682+0000) 2023-08-27T23:52:38.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:38 smithi006 bash[27292]: cluster 2023-08-27T23:52:37.713822+0000 mgr.y (mgr.14901) 3422 : cluster [DBG] pgmap v3429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:38.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:38 smithi006 bash[33918]: cluster 2023-08-27T23:52:37.713822+0000 mgr.y (mgr.14901) 3422 : cluster [DBG] pgmap v3429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:38 smithi130 bash[27579]: cluster 2023-08-27T23:52:37.713822+0000 mgr.y (mgr.14901) 3422 : cluster [DBG] pgmap v3429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:39.178 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:38 smithi130 bash[51302]: debug 2023-08-27T23:52:38.914+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:38.916963+0000) 2023-08-27T23:52:39.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:39 smithi006 bash[27292]: audit 2023-08-27T23:52:38.825305+0000 mon.c (mon.2) 1313 : audit [DBG] from='client.? 172.21.15.6:0/340422595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:39.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:39 smithi006 bash[33918]: audit 2023-08-27T23:52:38.825305+0000 mon.c (mon.2) 1313 : audit [DBG] from='client.? 172.21.15.6:0/340422595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:39.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:39 smithi130 bash[27579]: audit 2023-08-27T23:52:38.825305+0000 mon.c (mon.2) 1313 : audit [DBG] from='client.? 172.21.15.6:0/340422595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:40.184 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:39 smithi130 bash[51302]: debug 2023-08-27T23:52:39.914+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:39.917281+0000) 2023-08-27T23:52:40.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:40 smithi006 bash[27292]: cluster 2023-08-27T23:52:39.714926+0000 mgr.y (mgr.14901) 3423 : cluster [DBG] pgmap v3430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:40.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:40 smithi006 bash[33918]: cluster 2023-08-27T23:52:39.714926+0000 mgr.y (mgr.14901) 3423 : cluster [DBG] pgmap v3430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:40 smithi130 bash[27579]: cluster 2023-08-27T23:52:39.714926+0000 mgr.y (mgr.14901) 3423 : cluster [DBG] pgmap v3430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:52:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:52:40] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-27T23:52:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:52:40.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:52:40.589Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:41.306 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:40 smithi130 bash[51302]: debug 2023-08-27T23:52:40.914+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:40.917498+0000) 2023-08-27T23:52:41.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:41 smithi130 bash[27579]: audit 2023-08-27T23:52:41.248439+0000 mon.a (mon.0) 2688 : audit [DBG] from='client.? 172.21.15.6:0/3341665921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:41.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:41 smithi006 bash[27292]: audit 2023-08-27T23:52:41.248439+0000 mon.a (mon.0) 2688 : audit [DBG] from='client.? 172.21.15.6:0/3341665921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:41.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:41 smithi006 bash[33918]: audit 2023-08-27T23:52:41.248439+0000 mon.a (mon.0) 2688 : audit [DBG] from='client.? 172.21.15.6:0/3341665921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:42.315 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:41 smithi130 bash[51302]: debug 2023-08-27T23:52:41.914+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:41.917767+0000) 2023-08-27T23:52:42.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:42 smithi006 bash[33918]: cluster 2023-08-27T23:52:41.715687+0000 mgr.y (mgr.14901) 3424 : cluster [DBG] pgmap v3431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:52:42.154Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:52:42.156Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:42.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:42 smithi006 bash[27292]: cluster 2023-08-27T23:52:41.715687+0000 mgr.y (mgr.14901) 3424 : cluster [DBG] pgmap v3431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:42.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:42 smithi130 bash[27579]: cluster 2023-08-27T23:52:41.715687+0000 mgr.y (mgr.14901) 3424 : cluster [DBG] pgmap v3431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:43.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:42 smithi130 bash[51302]: debug 2023-08-27T23:52:42.914+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:42.918074+0000) 2023-08-27T23:52:44.051 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:43 smithi006 bash[33918]: audit 2023-08-27T23:52:43.671999+0000 mon.c (mon.2) 1314 : audit [DBG] from='client.? 172.21.15.6:0/214176261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:44.051 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:43 smithi006 bash[27292]: audit 2023-08-27T23:52:43.671999+0000 mon.c (mon.2) 1314 : audit [DBG] from='client.? 172.21.15.6:0/214176261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:44.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:43 smithi130 bash[27579]: audit 2023-08-27T23:52:43.671999+0000 mon.c (mon.2) 1314 : audit [DBG] from='client.? 172.21.15.6:0/214176261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:44.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:43 smithi130 bash[51302]: debug 2023-08-27T23:52:43.914+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:43.918356+0000) 2023-08-27T23:52:44.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:44 smithi006 bash[56298]: level=error ts=2023-08-27T23:52:44.049Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:44.482 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:44 smithi006 bash[56298]: level=warn ts=2023-08-27T23:52:44.051Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:44 smithi130 bash[27579]: cluster 2023-08-27T23:52:43.716319+0000 mgr.y (mgr.14901) 3425 : cluster [DBG] pgmap v3432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:44 smithi130 bash[51302]: debug 2023-08-27T23:52:44.914+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:44.918644+0000) 2023-08-27T23:52:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:44 smithi006 bash[27292]: cluster 2023-08-27T23:52:43.716319+0000 mgr.y (mgr.14901) 3425 : cluster [DBG] pgmap v3432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:44 smithi006 bash[33918]: cluster 2023-08-27T23:52:43.716319+0000 mgr.y (mgr.14901) 3425 : cluster [DBG] pgmap v3432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:46 smithi130 bash[27579]: cluster 2023-08-27T23:52:45.717735+0000 mgr.y (mgr.14901) 3426 : cluster [DBG] pgmap v3433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:46 smithi130 bash[27579]: audit 2023-08-27T23:52:46.088526+0000 mon.a (mon.0) 2689 : audit [DBG] from='client.? 172.21.15.6:0/2369133368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:45 smithi130 bash[51302]: debug 2023-08-27T23:52:45.918+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:45.918928+0000) 2023-08-27T23:52:46.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:46 smithi006 bash[33918]: cluster 2023-08-27T23:52:45.717735+0000 mgr.y (mgr.14901) 3426 : cluster [DBG] pgmap v3433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:46 smithi006 bash[33918]: audit 2023-08-27T23:52:46.088526+0000 mon.a (mon.0) 2689 : audit [DBG] from='client.? 172.21.15.6:0/2369133368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:46.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:46 smithi006 bash[27292]: cluster 2023-08-27T23:52:45.717735+0000 mgr.y (mgr.14901) 3426 : cluster [DBG] pgmap v3433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:46.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:46 smithi006 bash[27292]: audit 2023-08-27T23:52:46.088526+0000 mon.a (mon.0) 2689 : audit [DBG] from='client.? 172.21.15.6:0/2369133368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:46 smithi130 bash[51302]: debug 2023-08-27T23:52:46.918+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:46.919196+0000) 2023-08-27T23:52:48.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:47 smithi130 bash[51302]: debug 2023-08-27T23:52:47.918+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:47.919535+0000) 2023-08-27T23:52:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:48 smithi130 bash[27579]: cluster 2023-08-27T23:52:47.718492+0000 mgr.y (mgr.14901) 3427 : cluster [DBG] pgmap v3434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:48 smithi130 bash[27579]: audit 2023-08-27T23:52:48.499176+0000 mon.a (mon.0) 2690 : audit [DBG] from='client.? 172.21.15.6:0/2741667585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:49.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:48 smithi130 bash[51302]: debug 2023-08-27T23:52:48.918+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:48.919857+0000) 2023-08-27T23:52:49.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:48 smithi006 bash[27292]: cluster 2023-08-27T23:52:47.718492+0000 mgr.y (mgr.14901) 3427 : cluster [DBG] pgmap v3434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:49.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:48 smithi006 bash[27292]: audit 2023-08-27T23:52:48.499176+0000 mon.a (mon.0) 2690 : audit [DBG] from='client.? 172.21.15.6:0/2741667585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:49.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:48 smithi006 bash[33918]: cluster 2023-08-27T23:52:47.718492+0000 mgr.y (mgr.14901) 3427 : cluster [DBG] pgmap v3434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:49.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:48 smithi006 bash[33918]: audit 2023-08-27T23:52:48.499176+0000 mon.a (mon.0) 2690 : audit [DBG] from='client.? 172.21.15.6:0/2741667585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:50.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:49 smithi130 bash[27579]: audit 2023-08-27T23:52:48.776354+0000 mon.b (mon.1) 442 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:52:50.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:49 smithi130 bash[27579]: audit 2023-08-27T23:52:49.054832+0000 mon.a (mon.0) 2691 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:52:50.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:49 smithi130 bash[27579]: audit 2023-08-27T23:52:49.060262+0000 mon.a (mon.0) 2692 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:52:50.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:49 smithi130 bash[51302]: debug 2023-08-27T23:52:49.918+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:49.920148+0000) 2023-08-27T23:52:50.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:49 smithi006 bash[33918]: audit 2023-08-27T23:52:48.776354+0000 mon.b (mon.1) 442 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:52:50.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:49 smithi006 bash[33918]: audit 2023-08-27T23:52:49.054832+0000 mon.a (mon.0) 2691 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:52:50.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:49 smithi006 bash[33918]: audit 2023-08-27T23:52:49.060262+0000 mon.a (mon.0) 2692 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:52:50.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:49 smithi006 bash[27292]: audit 2023-08-27T23:52:48.776354+0000 mon.b (mon.1) 442 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:52:50.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:49 smithi006 bash[27292]: audit 2023-08-27T23:52:49.054832+0000 mon.a (mon.0) 2691 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:52:50.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:49 smithi006 bash[27292]: audit 2023-08-27T23:52:49.060262+0000 mon.a (mon.0) 2692 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:52:50.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:52:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:52:50] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-27T23:52:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:52:50.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:52:50.589Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:51.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:51 smithi130 bash[27579]: cluster 2023-08-27T23:52:49.719630+0000 mgr.y (mgr.14901) 3428 : cluster [DBG] pgmap v3435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:51.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:51 smithi130 bash[27579]: audit 2023-08-27T23:52:50.917630+0000 mon.a (mon.0) 2693 : audit [DBG] from='client.? 172.21.15.6:0/801210892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:51.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:50 smithi130 bash[51302]: debug 2023-08-27T23:52:50.918+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:50.920412+0000) 2023-08-27T23:52:51.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:51 smithi006 bash[27292]: cluster 2023-08-27T23:52:49.719630+0000 mgr.y (mgr.14901) 3428 : cluster [DBG] pgmap v3435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:51.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:51 smithi006 bash[27292]: audit 2023-08-27T23:52:50.917630+0000 mon.a (mon.0) 2693 : audit [DBG] from='client.? 172.21.15.6:0/801210892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:51.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:51 smithi006 bash[33918]: cluster 2023-08-27T23:52:49.719630+0000 mgr.y (mgr.14901) 3428 : cluster [DBG] pgmap v3435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:51.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:51 smithi006 bash[33918]: audit 2023-08-27T23:52:50.917630+0000 mon.a (mon.0) 2693 : audit [DBG] from='client.? 172.21.15.6:0/801210892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:52.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:51 smithi130 bash[51302]: debug 2023-08-27T23:52:51.918+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:51.920717+0000) 2023-08-27T23:52:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:52:52.155Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:52:52.156Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:53 smithi130 bash[27579]: cluster 2023-08-27T23:52:51.720299+0000 mgr.y (mgr.14901) 3429 : cluster [DBG] pgmap v3436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:53.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:52 smithi130 bash[51302]: debug 2023-08-27T23:52:52.918+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:52.920992+0000) 2023-08-27T23:52:53.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:53 smithi006 bash[27292]: cluster 2023-08-27T23:52:51.720299+0000 mgr.y (mgr.14901) 3429 : cluster [DBG] pgmap v3436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:53.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:53 smithi006 bash[33918]: cluster 2023-08-27T23:52:51.720299+0000 mgr.y (mgr.14901) 3429 : cluster [DBG] pgmap v3436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:54 smithi130 bash[27579]: audit 2023-08-27T23:52:53.330958+0000 mon.a (mon.0) 2694 : audit [DBG] from='client.? 172.21.15.6:0/122468797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:54 smithi130 bash[27579]: cluster 2023-08-27T23:52:53.720976+0000 mgr.y (mgr.14901) 3430 : cluster [DBG] pgmap v3437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:53 smithi130 bash[51302]: debug 2023-08-27T23:52:53.918+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:53.921183+0000) 2023-08-27T23:52:54.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:54 smithi006 bash[27292]: audit 2023-08-27T23:52:53.330958+0000 mon.a (mon.0) 2694 : audit [DBG] from='client.? 172.21.15.6:0/122468797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:54.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:54 smithi006 bash[27292]: cluster 2023-08-27T23:52:53.720976+0000 mgr.y (mgr.14901) 3430 : cluster [DBG] pgmap v3437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:54.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:54 smithi006 bash[33918]: audit 2023-08-27T23:52:53.330958+0000 mon.a (mon.0) 2694 : audit [DBG] from='client.? 172.21.15.6:0/122468797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:54.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:54 smithi006 bash[33918]: cluster 2023-08-27T23:52:53.720976+0000 mgr.y (mgr.14901) 3430 : cluster [DBG] pgmap v3437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:54 smithi006 bash[56298]: level=error ts=2023-08-27T23:52:54.049Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:52:54 smithi006 bash[56298]: level=warn ts=2023-08-27T23:52:54.052Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:52:55.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:54 smithi130 bash[51302]: debug 2023-08-27T23:52:54.918+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:54.921438+0000) 2023-08-27T23:52:56.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:55 smithi006 bash[33918]: audit 2023-08-27T23:52:54.961557+0000 mon.a (mon.0) 2695 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:52:56.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:55 smithi006 bash[33918]: audit 2023-08-27T23:52:55.364805+0000 mon.a (mon.0) 2696 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:52:56.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:55 smithi006 bash[33918]: audit 2023-08-27T23:52:55.629295+0000 mon.b (mon.1) 443 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:52:56.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:55 smithi006 bash[33918]: audit 2023-08-27T23:52:55.630965+0000 mon.b (mon.1) 444 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:52:56.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:55 smithi006 bash[33918]: audit 2023-08-27T23:52:55.637437+0000 mon.a (mon.0) 2697 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:52:56.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:55 smithi006 bash[33918]: audit 2023-08-27T23:52:55.747407+0000 mon.a (mon.0) 2698 : audit [DBG] from='client.? 172.21.15.6:0/358981105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:56.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:55 smithi006 bash[27292]: audit 2023-08-27T23:52:54.961557+0000 mon.a (mon.0) 2695 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:52:56.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:55 smithi006 bash[27292]: audit 2023-08-27T23:52:55.364805+0000 mon.a (mon.0) 2696 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:52:56.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:55 smithi006 bash[27292]: audit 2023-08-27T23:52:55.629295+0000 mon.b (mon.1) 443 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:52:56.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:55 smithi006 bash[27292]: audit 2023-08-27T23:52:55.630965+0000 mon.b (mon.1) 444 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:52:56.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:55 smithi006 bash[27292]: audit 2023-08-27T23:52:55.637437+0000 mon.a (mon.0) 2697 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:52:56.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:55 smithi006 bash[27292]: audit 2023-08-27T23:52:55.747407+0000 mon.a (mon.0) 2698 : audit [DBG] from='client.? 172.21.15.6:0/358981105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:56.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:55 smithi130 bash[51302]: debug 2023-08-27T23:52:55.918+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:55.921759+0000) 2023-08-27T23:52:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:55 smithi130 bash[27579]: audit 2023-08-27T23:52:54.961557+0000 mon.a (mon.0) 2695 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:52:56.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:55 smithi130 bash[27579]: audit 2023-08-27T23:52:55.364805+0000 mon.a (mon.0) 2696 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:52:56.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:55 smithi130 bash[27579]: audit 2023-08-27T23:52:55.629295+0000 mon.b (mon.1) 443 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:52:56.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:55 smithi130 bash[27579]: audit 2023-08-27T23:52:55.630965+0000 mon.b (mon.1) 444 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:52:56.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:55 smithi130 bash[27579]: audit 2023-08-27T23:52:55.637437+0000 mon.a (mon.0) 2697 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:52:56.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:55 smithi130 bash[27579]: audit 2023-08-27T23:52:55.747407+0000 mon.a (mon.0) 2698 : audit [DBG] from='client.? 172.21.15.6:0/358981105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:57.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:56 smithi006 bash[33918]: cluster 2023-08-27T23:52:55.722018+0000 mgr.y (mgr.14901) 3431 : cluster [DBG] pgmap v3438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:57.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:56 smithi006 bash[27292]: cluster 2023-08-27T23:52:55.722018+0000 mgr.y (mgr.14901) 3431 : cluster [DBG] pgmap v3438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:57.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:56 smithi130 bash[27579]: cluster 2023-08-27T23:52:55.722018+0000 mgr.y (mgr.14901) 3431 : cluster [DBG] pgmap v3438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:57.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:56 smithi130 bash[51302]: debug 2023-08-27T23:52:56.918+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:56.922028+0000) 2023-08-27T23:52:58.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:57 smithi130 bash[51302]: debug 2023-08-27T23:52:57.918+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:57.922298+0000) 2023-08-27T23:52:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:58 smithi130 bash[27579]: cluster 2023-08-27T23:52:57.722646+0000 mgr.y (mgr.14901) 3432 : cluster [DBG] pgmap v3439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:52:58 smithi130 bash[27579]: audit 2023-08-27T23:52:58.159764+0000 mon.a (mon.0) 2699 : audit [DBG] from='client.? 172.21.15.6:0/1106101820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:58 smithi130 bash[51302]: debug 2023-08-27T23:52:58.922+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:58.922626+0000) 2023-08-27T23:52:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:58 smithi006 bash[33918]: cluster 2023-08-27T23:52:57.722646+0000 mgr.y (mgr.14901) 3432 : cluster [DBG] pgmap v3439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:52:58 smithi006 bash[33918]: audit 2023-08-27T23:52:58.159764+0000 mon.a (mon.0) 2699 : audit [DBG] from='client.? 172.21.15.6:0/1106101820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:59.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:58 smithi006 bash[27292]: cluster 2023-08-27T23:52:57.722646+0000 mgr.y (mgr.14901) 3432 : cluster [DBG] pgmap v3439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:52:59.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:52:58 smithi006 bash[27292]: audit 2023-08-27T23:52:58.159764+0000 mon.a (mon.0) 2699 : audit [DBG] from='client.? 172.21.15.6:0/1106101820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:00.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:52:59 smithi130 bash[51302]: debug 2023-08-27T23:52:59.922+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:59.922938+0000) 2023-08-27T23:53:00.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:53:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:53:00] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-27T23:53:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:53:00.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:53:00.590Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:01.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:00 smithi130 bash[27579]: cluster 2023-08-27T23:52:59.723749+0000 mgr.y (mgr.14901) 3433 : cluster [DBG] pgmap v3440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:01.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:00 smithi130 bash[27579]: audit 2023-08-27T23:53:00.575541+0000 mon.a (mon.0) 2700 : audit [DBG] from='client.? 172.21.15.6:0/1882017666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:01.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:00 smithi130 bash[51302]: debug 2023-08-27T23:53:00.922+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:00.923223+0000) 2023-08-27T23:53:01.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:00 smithi006 bash[27292]: cluster 2023-08-27T23:52:59.723749+0000 mgr.y (mgr.14901) 3433 : cluster [DBG] pgmap v3440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:01.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:00 smithi006 bash[27292]: audit 2023-08-27T23:53:00.575541+0000 mon.a (mon.0) 2700 : audit [DBG] from='client.? 172.21.15.6:0/1882017666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:01.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:00 smithi006 bash[33918]: cluster 2023-08-27T23:52:59.723749+0000 mgr.y (mgr.14901) 3433 : cluster [DBG] pgmap v3440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:01.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:00 smithi006 bash[33918]: audit 2023-08-27T23:53:00.575541+0000 mon.a (mon.0) 2700 : audit [DBG] from='client.? 172.21.15.6:0/1882017666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:02.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:01 smithi130 bash[51302]: debug 2023-08-27T23:53:01.922+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:01.923522+0000) 2023-08-27T23:53:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:53:02.155Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:53:02.157Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:03.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:02 smithi130 bash[27579]: cluster 2023-08-27T23:53:01.724419+0000 mgr.y (mgr.14901) 3434 : cluster [DBG] pgmap v3441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:02 smithi130 bash[51302]: debug 2023-08-27T23:53:02.922+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:02.923802+0000) 2023-08-27T23:53:03.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:02 smithi006 bash[33918]: cluster 2023-08-27T23:53:01.724419+0000 mgr.y (mgr.14901) 3434 : cluster [DBG] pgmap v3441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:03.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:02 smithi006 bash[27292]: cluster 2023-08-27T23:53:01.724419+0000 mgr.y (mgr.14901) 3434 : cluster [DBG] pgmap v3441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:04.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:03 smithi130 bash[27579]: audit 2023-08-27T23:53:02.993987+0000 mon.c (mon.2) 1315 : audit [DBG] from='client.? 172.21.15.6:0/4193614746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:03 smithi130 bash[51302]: debug 2023-08-27T23:53:03.922+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:03.924110+0000) 2023-08-27T23:53:04.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:03 smithi006 bash[33918]: audit 2023-08-27T23:53:02.993987+0000 mon.c (mon.2) 1315 : audit [DBG] from='client.? 172.21.15.6:0/4193614746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:04 smithi006 bash[56298]: level=error ts=2023-08-27T23:53:04.051Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:04 smithi006 bash[56298]: level=warn ts=2023-08-27T23:53:04.053Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:04.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:03 smithi006 bash[27292]: audit 2023-08-27T23:53:02.993987+0000 mon.c (mon.2) 1315 : audit [DBG] from='client.? 172.21.15.6:0/4193614746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:05 smithi130 bash[27579]: cluster 2023-08-27T23:53:03.725085+0000 mgr.y (mgr.14901) 3435 : cluster [DBG] pgmap v3442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:04 smithi130 bash[51302]: debug 2023-08-27T23:53:04.922+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:04.924392+0000) 2023-08-27T23:53:05.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:04 smithi006 bash[27292]: cluster 2023-08-27T23:53:03.725085+0000 mgr.y (mgr.14901) 3435 : cluster [DBG] pgmap v3442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:05.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:05 smithi006 bash[33918]: cluster 2023-08-27T23:53:03.725085+0000 mgr.y (mgr.14901) 3435 : cluster [DBG] pgmap v3442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:06 smithi130 bash[27579]: audit 2023-08-27T23:53:05.406391+0000 mon.c (mon.2) 1316 : audit [DBG] from='client.? 172.21.15.6:0/2131065409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:06 smithi130 bash[27579]: cluster 2023-08-27T23:53:05.726202+0000 mgr.y (mgr.14901) 3436 : cluster [DBG] pgmap v3443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:06.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:05 smithi130 bash[51302]: debug 2023-08-27T23:53:05.922+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:05.924679+0000) 2023-08-27T23:53:06.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:06 smithi006 bash[27292]: audit 2023-08-27T23:53:05.406391+0000 mon.c (mon.2) 1316 : audit [DBG] from='client.? 172.21.15.6:0/2131065409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:06.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:06 smithi006 bash[27292]: cluster 2023-08-27T23:53:05.726202+0000 mgr.y (mgr.14901) 3436 : cluster [DBG] pgmap v3443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:06.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:06 smithi006 bash[33918]: audit 2023-08-27T23:53:05.406391+0000 mon.c (mon.2) 1316 : audit [DBG] from='client.? 172.21.15.6:0/2131065409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:06.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:06 smithi006 bash[33918]: cluster 2023-08-27T23:53:05.726202+0000 mgr.y (mgr.14901) 3436 : cluster [DBG] pgmap v3443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:07.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:06 smithi130 bash[51302]: debug 2023-08-27T23:53:06.922+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:06.924982+0000) 2023-08-27T23:53:08.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:07 smithi130 bash[51302]: debug 2023-08-27T23:53:07.922+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:07.925294+0000) 2023-08-27T23:53:09.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:08 smithi130 bash[51302]: debug 2023-08-27T23:53:08.922+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:08.925713+0000) 2023-08-27T23:53:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:08 smithi130 bash[27579]: cluster 2023-08-27T23:53:07.726824+0000 mgr.y (mgr.14901) 3437 : cluster [DBG] pgmap v3444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:08 smithi130 bash[27579]: audit 2023-08-27T23:53:07.825171+0000 mon.c (mon.2) 1317 : audit [DBG] from='client.? 172.21.15.6:0/625733617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:08 smithi006 bash[27292]: cluster 2023-08-27T23:53:07.726824+0000 mgr.y (mgr.14901) 3437 : cluster [DBG] pgmap v3444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:08 smithi006 bash[27292]: audit 2023-08-27T23:53:07.825171+0000 mon.c (mon.2) 1317 : audit [DBG] from='client.? 172.21.15.6:0/625733617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:08 smithi006 bash[33918]: cluster 2023-08-27T23:53:07.726824+0000 mgr.y (mgr.14901) 3437 : cluster [DBG] pgmap v3444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:08 smithi006 bash[33918]: audit 2023-08-27T23:53:07.825171+0000 mon.c (mon.2) 1317 : audit [DBG] from='client.? 172.21.15.6:0/625733617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:10.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:09 smithi130 bash[51302]: debug 2023-08-27T23:53:09.922+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:09.926040+0000) 2023-08-27T23:53:10.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:53:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:53:10] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-27T23:53:10.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:10 smithi006 bash[27292]: cluster 2023-08-27T23:53:09.727918+0000 mgr.y (mgr.14901) 3438 : cluster [DBG] pgmap v3445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:10.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:10 smithi006 bash[27292]: audit 2023-08-27T23:53:10.247711+0000 mon.c (mon.2) 1318 : audit [DBG] from='client.? 172.21.15.6:0/2909599555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:10.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:10 smithi006 bash[33918]: cluster 2023-08-27T23:53:09.727918+0000 mgr.y (mgr.14901) 3438 : cluster [DBG] pgmap v3445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:10.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:10 smithi006 bash[33918]: audit 2023-08-27T23:53:10.247711+0000 mon.c (mon.2) 1318 : audit [DBG] from='client.? 172.21.15.6:0/2909599555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:53:10.589Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:53:10.592Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:11.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:10 smithi130 bash[51302]: debug 2023-08-27T23:53:10.926+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:10.926417+0000) 2023-08-27T23:53:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:10 smithi130 bash[27579]: cluster 2023-08-27T23:53:09.727918+0000 mgr.y (mgr.14901) 3438 : cluster [DBG] pgmap v3445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:10 smithi130 bash[27579]: audit 2023-08-27T23:53:10.247711+0000 mon.c (mon.2) 1318 : audit [DBG] from='client.? 172.21.15.6:0/2909599555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:11 smithi130 bash[51302]: debug 2023-08-27T23:53:11.922+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:11.926678+0000) 2023-08-27T23:53:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:53:12.155Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:53:12.158Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:12 smithi130 bash[27579]: cluster 2023-08-27T23:53:11.728603+0000 mgr.y (mgr.14901) 3439 : cluster [DBG] pgmap v3446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:12 smithi130 bash[27579]: audit 2023-08-27T23:53:12.653654+0000 mon.a (mon.0) 2701 : audit [DBG] from='client.? 172.21.15.6:0/359246740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:13.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:12 smithi130 bash[51302]: debug 2023-08-27T23:53:12.926+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:12.927029+0000) 2023-08-27T23:53:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:12 smithi006 bash[27292]: cluster 2023-08-27T23:53:11.728603+0000 mgr.y (mgr.14901) 3439 : cluster [DBG] pgmap v3446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:12 smithi006 bash[27292]: audit 2023-08-27T23:53:12.653654+0000 mon.a (mon.0) 2701 : audit [DBG] from='client.? 172.21.15.6:0/359246740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:12 smithi006 bash[33918]: cluster 2023-08-27T23:53:11.728603+0000 mgr.y (mgr.14901) 3439 : cluster [DBG] pgmap v3446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:12 smithi006 bash[33918]: audit 2023-08-27T23:53:12.653654+0000 mon.a (mon.0) 2701 : audit [DBG] from='client.? 172.21.15.6:0/359246740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:14.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:13 smithi130 bash[51302]: debug 2023-08-27T23:53:13.926+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:13.927288+0000) 2023-08-27T23:53:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:14 smithi006 bash[56298]: level=error ts=2023-08-27T23:53:14.051Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:14 smithi006 bash[56298]: level=warn ts=2023-08-27T23:53:14.053Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:14 smithi130 bash[27579]: cluster 2023-08-27T23:53:13.729264+0000 mgr.y (mgr.14901) 3440 : cluster [DBG] pgmap v3447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:15.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:14 smithi130 bash[51302]: debug 2023-08-27T23:53:14.926+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:14.927609+0000) 2023-08-27T23:53:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:14 smithi006 bash[27292]: cluster 2023-08-27T23:53:13.729264+0000 mgr.y (mgr.14901) 3440 : cluster [DBG] pgmap v3447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:14 smithi006 bash[33918]: cluster 2023-08-27T23:53:13.729264+0000 mgr.y (mgr.14901) 3440 : cluster [DBG] pgmap v3447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:16.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:15 smithi130 bash[27579]: audit 2023-08-27T23:53:15.078293+0000 mon.c (mon.2) 1319 : audit [DBG] from='client.? 172.21.15.6:0/1648349898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:16.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:15 smithi130 bash[51302]: debug 2023-08-27T23:53:15.926+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:15.927939+0000) 2023-08-27T23:53:16.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:15 smithi006 bash[27292]: audit 2023-08-27T23:53:15.078293+0000 mon.c (mon.2) 1319 : audit [DBG] from='client.? 172.21.15.6:0/1648349898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:16.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:15 smithi006 bash[33918]: audit 2023-08-27T23:53:15.078293+0000 mon.c (mon.2) 1319 : audit [DBG] from='client.? 172.21.15.6:0/1648349898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:17.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:16 smithi130 bash[27579]: cluster 2023-08-27T23:53:15.730412+0000 mgr.y (mgr.14901) 3441 : cluster [DBG] pgmap v3448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:17.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:16 smithi130 bash[51302]: debug 2023-08-27T23:53:16.926+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:16.928272+0000) 2023-08-27T23:53:17.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:16 smithi006 bash[27292]: cluster 2023-08-27T23:53:15.730412+0000 mgr.y (mgr.14901) 3441 : cluster [DBG] pgmap v3448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:16 smithi006 bash[33918]: cluster 2023-08-27T23:53:15.730412+0000 mgr.y (mgr.14901) 3441 : cluster [DBG] pgmap v3448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:18.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:17 smithi130 bash[27579]: audit 2023-08-27T23:53:17.505118+0000 mon.a (mon.0) 2702 : audit [DBG] from='client.? 172.21.15.6:0/1229813245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:18.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:17 smithi130 bash[51302]: debug 2023-08-27T23:53:17.925+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:17.928590+0000) 2023-08-27T23:53:18.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:17 smithi006 bash[27292]: audit 2023-08-27T23:53:17.505118+0000 mon.a (mon.0) 2702 : audit [DBG] from='client.? 172.21.15.6:0/1229813245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:18.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:17 smithi006 bash[33918]: audit 2023-08-27T23:53:17.505118+0000 mon.a (mon.0) 2702 : audit [DBG] from='client.? 172.21.15.6:0/1229813245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:18 smithi130 bash[27579]: cluster 2023-08-27T23:53:17.731038+0000 mgr.y (mgr.14901) 3442 : cluster [DBG] pgmap v3449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:19.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:18 smithi130 bash[51302]: debug 2023-08-27T23:53:18.925+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:18.928935+0000) 2023-08-27T23:53:19.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:18 smithi006 bash[33918]: cluster 2023-08-27T23:53:17.731038+0000 mgr.y (mgr.14901) 3442 : cluster [DBG] pgmap v3449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:18 smithi006 bash[27292]: cluster 2023-08-27T23:53:17.731038+0000 mgr.y (mgr.14901) 3442 : cluster [DBG] pgmap v3449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:20.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:19 smithi130 bash[51302]: debug 2023-08-27T23:53:19.925+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:19.929249+0000) 2023-08-27T23:53:20.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:53:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:53:20] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-27T23:53:20.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:20 smithi006 bash[33918]: cluster 2023-08-27T23:53:19.732122+0000 mgr.y (mgr.14901) 3443 : cluster [DBG] pgmap v3450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:20.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:20 smithi006 bash[33918]: audit 2023-08-27T23:53:19.916829+0000 mon.a (mon.0) 2703 : audit [DBG] from='client.? 172.21.15.6:0/1466521420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:53:20.590Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:53:20.591Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:20.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:20 smithi006 bash[27292]: cluster 2023-08-27T23:53:19.732122+0000 mgr.y (mgr.14901) 3443 : cluster [DBG] pgmap v3450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:20.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:20 smithi006 bash[27292]: audit 2023-08-27T23:53:19.916829+0000 mon.a (mon.0) 2703 : audit [DBG] from='client.? 172.21.15.6:0/1466521420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:21.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:20 smithi130 bash[51302]: debug 2023-08-27T23:53:20.925+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:20.929593+0000) 2023-08-27T23:53:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:20 smithi130 bash[27579]: cluster 2023-08-27T23:53:19.732122+0000 mgr.y (mgr.14901) 3443 : cluster [DBG] pgmap v3450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:20 smithi130 bash[27579]: audit 2023-08-27T23:53:19.916829+0000 mon.a (mon.0) 2703 : audit [DBG] from='client.? 172.21.15.6:0/1466521420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:22.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:21 smithi130 bash[51302]: debug 2023-08-27T23:53:21.925+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:21.929964+0000) 2023-08-27T23:53:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:53:22.156Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:53:22.158Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:22 smithi130 bash[27579]: cluster 2023-08-27T23:53:21.732773+0000 mgr.y (mgr.14901) 3444 : cluster [DBG] pgmap v3451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:22 smithi130 bash[27579]: audit 2023-08-27T23:53:22.330982+0000 mon.c (mon.2) 1320 : audit [DBG] from='client.? 172.21.15.6:0/2479046932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:23.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:22 smithi130 bash[51302]: debug 2023-08-27T23:53:22.925+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:22.930248+0000) 2023-08-27T23:53:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:22 smithi006 bash[27292]: cluster 2023-08-27T23:53:21.732773+0000 mgr.y (mgr.14901) 3444 : cluster [DBG] pgmap v3451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:22 smithi006 bash[27292]: audit 2023-08-27T23:53:22.330982+0000 mon.c (mon.2) 1320 : audit [DBG] from='client.? 172.21.15.6:0/2479046932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:22 smithi006 bash[33918]: cluster 2023-08-27T23:53:21.732773+0000 mgr.y (mgr.14901) 3444 : cluster [DBG] pgmap v3451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:22 smithi006 bash[33918]: audit 2023-08-27T23:53:22.330982+0000 mon.c (mon.2) 1320 : audit [DBG] from='client.? 172.21.15.6:0/2479046932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:23 smithi130 bash[51302]: debug 2023-08-27T23:53:23.929+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:23.930558+0000) 2023-08-27T23:53:24.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:24 smithi006 bash[56298]: level=error ts=2023-08-27T23:53:24.052Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:24 smithi006 bash[56298]: level=warn ts=2023-08-27T23:53:24.055Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:24 smithi130 bash[27579]: cluster 2023-08-27T23:53:23.733405+0000 mgr.y (mgr.14901) 3445 : cluster [DBG] pgmap v3452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:24 smithi130 bash[27579]: audit 2023-08-27T23:53:24.742587+0000 mon.c (mon.2) 1321 : audit [DBG] from='client.? 172.21.15.6:0/1259971677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:24 smithi130 bash[51302]: debug 2023-08-27T23:53:24.929+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:24.930891+0000) 2023-08-27T23:53:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:24 smithi006 bash[27292]: cluster 2023-08-27T23:53:23.733405+0000 mgr.y (mgr.14901) 3445 : cluster [DBG] pgmap v3452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:24 smithi006 bash[27292]: audit 2023-08-27T23:53:24.742587+0000 mon.c (mon.2) 1321 : audit [DBG] from='client.? 172.21.15.6:0/1259971677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:24 smithi006 bash[33918]: cluster 2023-08-27T23:53:23.733405+0000 mgr.y (mgr.14901) 3445 : cluster [DBG] pgmap v3452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:24 smithi006 bash[33918]: audit 2023-08-27T23:53:24.742587+0000 mon.c (mon.2) 1321 : audit [DBG] from='client.? 172.21.15.6:0/1259971677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:26.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:25 smithi130 bash[51302]: debug 2023-08-27T23:53:25.929+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:25.931174+0000) 2023-08-27T23:53:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:26 smithi130 bash[27579]: cluster 2023-08-27T23:53:25.734561+0000 mgr.y (mgr.14901) 3446 : cluster [DBG] pgmap v3453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:26.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:26 smithi006 bash[27292]: cluster 2023-08-27T23:53:25.734561+0000 mgr.y (mgr.14901) 3446 : cluster [DBG] pgmap v3453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:26.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:26 smithi006 bash[33918]: cluster 2023-08-27T23:53:25.734561+0000 mgr.y (mgr.14901) 3446 : cluster [DBG] pgmap v3453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:27.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:27 smithi130 bash[27579]: audit 2023-08-27T23:53:27.163927+0000 mon.a (mon.0) 2704 : audit [DBG] from='client.? 172.21.15.6:0/40047012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:27.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:26 smithi130 bash[51302]: debug 2023-08-27T23:53:26.929+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:26.931509+0000) 2023-08-27T23:53:27.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:27 smithi006 bash[33918]: audit 2023-08-27T23:53:27.163927+0000 mon.a (mon.0) 2704 : audit [DBG] from='client.? 172.21.15.6:0/40047012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:27.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:27 smithi006 bash[27292]: audit 2023-08-27T23:53:27.163927+0000 mon.a (mon.0) 2704 : audit [DBG] from='client.? 172.21.15.6:0/40047012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:28.187 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:27 smithi130 bash[51302]: debug 2023-08-27T23:53:27.929+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:27.931803+0000) 2023-08-27T23:53:28.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:28 smithi006 bash[33918]: cluster 2023-08-27T23:53:27.735265+0000 mgr.y (mgr.14901) 3447 : cluster [DBG] pgmap v3454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:28.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:28 smithi006 bash[27292]: cluster 2023-08-27T23:53:27.735265+0000 mgr.y (mgr.14901) 3447 : cluster [DBG] pgmap v3454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:28.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:28 smithi130 bash[27579]: cluster 2023-08-27T23:53:27.735265+0000 mgr.y (mgr.14901) 3447 : cluster [DBG] pgmap v3454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:29.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:28 smithi130 bash[51302]: debug 2023-08-27T23:53:28.929+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:28.932089+0000) 2023-08-27T23:53:29.933 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:29 smithi130 bash[27579]: audit 2023-08-27T23:53:29.581063+0000 mon.c (mon.2) 1322 : audit [DBG] from='client.? 172.21.15.6:0/2985491110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:29.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:29 smithi006 bash[27292]: audit 2023-08-27T23:53:29.581063+0000 mon.c (mon.2) 1322 : audit [DBG] from='client.? 172.21.15.6:0/2985491110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:29.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:29 smithi006 bash[33918]: audit 2023-08-27T23:53:29.581063+0000 mon.c (mon.2) 1322 : audit [DBG] from='client.? 172.21.15.6:0/2985491110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:29 smithi130 bash[51302]: debug 2023-08-27T23:53:29.929+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:29.932368+0000) 2023-08-27T23:53:30.933 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:30 smithi130 bash[27579]: cluster 2023-08-27T23:53:29.736564+0000 mgr.y (mgr.14901) 3448 : cluster [DBG] pgmap v3455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:53:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:53:30] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-27T23:53:30.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:30 smithi006 bash[27292]: cluster 2023-08-27T23:53:29.736564+0000 mgr.y (mgr.14901) 3448 : cluster [DBG] pgmap v3455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:30.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:30 smithi006 bash[33918]: cluster 2023-08-27T23:53:29.736564+0000 mgr.y (mgr.14901) 3448 : cluster [DBG] pgmap v3455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:53:30.591Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:53:30.592Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:31.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:30 smithi130 bash[51302]: debug 2023-08-27T23:53:30.929+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:30.932653+0000) 2023-08-27T23:53:32.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:31 smithi130 bash[51302]: debug 2023-08-27T23:53:31.929+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:31.932937+0000) 2023-08-27T23:53:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:53:32.157Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:53:32.159Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:33.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:32 smithi130 bash[51302]: debug 2023-08-27T23:53:32.929+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:32.933195+0000) 2023-08-27T23:53:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:32 smithi130 bash[27579]: cluster 2023-08-27T23:53:31.737151+0000 mgr.y (mgr.14901) 3449 : cluster [DBG] pgmap v3456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:32 smithi130 bash[27579]: audit 2023-08-27T23:53:32.003281+0000 mon.c (mon.2) 1323 : audit [DBG] from='client.? 172.21.15.6:0/4206164520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:32 smithi006 bash[33918]: cluster 2023-08-27T23:53:31.737151+0000 mgr.y (mgr.14901) 3449 : cluster [DBG] pgmap v3456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:32 smithi006 bash[33918]: audit 2023-08-27T23:53:32.003281+0000 mon.c (mon.2) 1323 : audit [DBG] from='client.? 172.21.15.6:0/4206164520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:32 smithi006 bash[27292]: cluster 2023-08-27T23:53:31.737151+0000 mgr.y (mgr.14901) 3449 : cluster [DBG] pgmap v3456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:32 smithi006 bash[27292]: audit 2023-08-27T23:53:32.003281+0000 mon.c (mon.2) 1323 : audit [DBG] from='client.? 172.21.15.6:0/4206164520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:34.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:33 smithi130 bash[51302]: debug 2023-08-27T23:53:33.929+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:33.933476+0000) 2023-08-27T23:53:34.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:34 smithi006 bash[56298]: level=error ts=2023-08-27T23:53:34.053Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:34 smithi006 bash[56298]: level=warn ts=2023-08-27T23:53:34.056Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:35.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:34 smithi130 bash[27579]: cluster 2023-08-27T23:53:33.737920+0000 mgr.y (mgr.14901) 3450 : cluster [DBG] pgmap v3457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:35.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:34 smithi130 bash[27579]: audit 2023-08-27T23:53:34.427845+0000 mon.c (mon.2) 1324 : audit [DBG] from='client.? 172.21.15.6:0/2522422413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:35.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:34 smithi130 bash[51302]: debug 2023-08-27T23:53:34.929+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:34.933762+0000) 2023-08-27T23:53:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:34 smithi006 bash[33918]: cluster 2023-08-27T23:53:33.737920+0000 mgr.y (mgr.14901) 3450 : cluster [DBG] pgmap v3457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:34 smithi006 bash[33918]: audit 2023-08-27T23:53:34.427845+0000 mon.c (mon.2) 1324 : audit [DBG] from='client.? 172.21.15.6:0/2522422413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:34 smithi006 bash[27292]: cluster 2023-08-27T23:53:33.737920+0000 mgr.y (mgr.14901) 3450 : cluster [DBG] pgmap v3457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:34 smithi006 bash[27292]: audit 2023-08-27T23:53:34.427845+0000 mon.c (mon.2) 1324 : audit [DBG] from='client.? 172.21.15.6:0/2522422413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:36 smithi130 bash[27579]: cluster 2023-08-27T23:53:35.739113+0000 mgr.y (mgr.14901) 3451 : cluster [DBG] pgmap v3458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:36.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:35 smithi130 bash[51302]: debug 2023-08-27T23:53:35.929+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:35.934068+0000) 2023-08-27T23:53:36.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:36 smithi006 bash[27292]: cluster 2023-08-27T23:53:35.739113+0000 mgr.y (mgr.14901) 3451 : cluster [DBG] pgmap v3458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:36.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:36 smithi006 bash[33918]: cluster 2023-08-27T23:53:35.739113+0000 mgr.y (mgr.14901) 3451 : cluster [DBG] pgmap v3458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:37.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:37 smithi130 bash[27579]: audit 2023-08-27T23:53:36.853241+0000 mon.c (mon.2) 1325 : audit [DBG] from='client.? 172.21.15.6:0/3796090993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:36 smithi130 bash[51302]: debug 2023-08-27T23:53:36.929+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:36.934335+0000) 2023-08-27T23:53:37.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:37 smithi006 bash[33918]: audit 2023-08-27T23:53:36.853241+0000 mon.c (mon.2) 1325 : audit [DBG] from='client.? 172.21.15.6:0/3796090993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:37.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:37 smithi006 bash[27292]: audit 2023-08-27T23:53:36.853241+0000 mon.c (mon.2) 1325 : audit [DBG] from='client.? 172.21.15.6:0/3796090993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:38.189 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:37 smithi130 bash[51302]: debug 2023-08-27T23:53:37.929+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:37.934617+0000) 2023-08-27T23:53:38.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:38 smithi006 bash[27292]: cluster 2023-08-27T23:53:37.739813+0000 mgr.y (mgr.14901) 3452 : cluster [DBG] pgmap v3459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:38.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:38 smithi006 bash[33918]: cluster 2023-08-27T23:53:37.739813+0000 mgr.y (mgr.14901) 3452 : cluster [DBG] pgmap v3459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:38 smithi130 bash[27579]: cluster 2023-08-27T23:53:37.739813+0000 mgr.y (mgr.14901) 3452 : cluster [DBG] pgmap v3459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:39.330 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:38 smithi130 bash[51302]: debug 2023-08-27T23:53:38.933+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:38.934927+0000) 2023-08-27T23:53:39.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:39 smithi130 bash[27579]: audit 2023-08-27T23:53:39.274185+0000 mon.a (mon.0) 2705 : audit [DBG] from='client.? 172.21.15.6:0/3083308269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:39.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:39 smithi006 bash[33918]: audit 2023-08-27T23:53:39.274185+0000 mon.a (mon.0) 2705 : audit [DBG] from='client.? 172.21.15.6:0/3083308269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:39.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:39 smithi006 bash[27292]: audit 2023-08-27T23:53:39.274185+0000 mon.a (mon.0) 2705 : audit [DBG] from='client.? 172.21.15.6:0/3083308269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:40.343 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:39 smithi130 bash[51302]: debug 2023-08-27T23:53:39.933+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:39.935215+0000) 2023-08-27T23:53:40.594 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:40 smithi006 bash[27292]: cluster 2023-08-27T23:53:39.740967+0000 mgr.y (mgr.14901) 3453 : cluster [DBG] pgmap v3460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:40.595 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:40 smithi006 bash[33918]: cluster 2023-08-27T23:53:39.740967+0000 mgr.y (mgr.14901) 3453 : cluster [DBG] pgmap v3460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:40.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:40 smithi130 bash[27579]: cluster 2023-08-27T23:53:39.740967+0000 mgr.y (mgr.14901) 3453 : cluster [DBG] pgmap v3460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:53:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:53:40] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-27T23:53:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:53:40.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:53:40.594Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:41.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:40 smithi130 bash[51302]: debug 2023-08-27T23:53:40.933+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:40.935504+0000) 2023-08-27T23:53:42.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:41 smithi130 bash[27579]: audit 2023-08-27T23:53:41.686319+0000 mon.c (mon.2) 1326 : audit [DBG] from='client.? 172.21.15.6:0/3314372051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:42.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:41 smithi130 bash[51302]: debug 2023-08-27T23:53:41.933+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:41.935775+0000) 2023-08-27T23:53:42.160 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:41 smithi006 bash[33918]: audit 2023-08-27T23:53:41.686319+0000 mon.c (mon.2) 1326 : audit [DBG] from='client.? 172.21.15.6:0/3314372051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:42.160 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:41 smithi006 bash[27292]: audit 2023-08-27T23:53:41.686319+0000 mon.c (mon.2) 1326 : audit [DBG] from='client.? 172.21.15.6:0/3314372051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:42.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:53:42.158Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:42.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:53:42.160Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:42 smithi130 bash[27579]: cluster 2023-08-27T23:53:41.741572+0000 mgr.y (mgr.14901) 3454 : cluster [DBG] pgmap v3461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:43.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:42 smithi130 bash[51302]: debug 2023-08-27T23:53:42.933+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:42.936027+0000) 2023-08-27T23:53:43.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:42 smithi006 bash[27292]: cluster 2023-08-27T23:53:41.741572+0000 mgr.y (mgr.14901) 3454 : cluster [DBG] pgmap v3461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:42 smithi006 bash[33918]: cluster 2023-08-27T23:53:41.741572+0000 mgr.y (mgr.14901) 3454 : cluster [DBG] pgmap v3461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:44.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:43 smithi130 bash[51302]: debug 2023-08-27T23:53:43.933+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:43.936320+0000) 2023-08-27T23:53:44.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:44 smithi006 bash[56298]: level=error ts=2023-08-27T23:53:44.053Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:44 smithi006 bash[56298]: level=warn ts=2023-08-27T23:53:44.055Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:44 smithi130 bash[27579]: cluster 2023-08-27T23:53:43.742372+0000 mgr.y (mgr.14901) 3455 : cluster [DBG] pgmap v3462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:44 smithi130 bash[27579]: audit 2023-08-27T23:53:44.102852+0000 mon.a (mon.0) 2706 : audit [DBG] from='client.? 172.21.15.6:0/2117307276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:44 smithi130 bash[51302]: debug 2023-08-27T23:53:44.933+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:44.936605+0000) 2023-08-27T23:53:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:44 smithi006 bash[33918]: cluster 2023-08-27T23:53:43.742372+0000 mgr.y (mgr.14901) 3455 : cluster [DBG] pgmap v3462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:44 smithi006 bash[33918]: audit 2023-08-27T23:53:44.102852+0000 mon.a (mon.0) 2706 : audit [DBG] from='client.? 172.21.15.6:0/2117307276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:44 smithi006 bash[27292]: cluster 2023-08-27T23:53:43.742372+0000 mgr.y (mgr.14901) 3455 : cluster [DBG] pgmap v3462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:44 smithi006 bash[27292]: audit 2023-08-27T23:53:44.102852+0000 mon.a (mon.0) 2706 : audit [DBG] from='client.? 172.21.15.6:0/2117307276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:46 smithi130 bash[27579]: cluster 2023-08-27T23:53:45.743552+0000 mgr.y (mgr.14901) 3456 : cluster [DBG] pgmap v3463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:45 smithi130 bash[51302]: debug 2023-08-27T23:53:45.933+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:45.936861+0000) 2023-08-27T23:53:46.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:46 smithi006 bash[27292]: cluster 2023-08-27T23:53:45.743552+0000 mgr.y (mgr.14901) 3456 : cluster [DBG] pgmap v3463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:46 smithi006 bash[33918]: cluster 2023-08-27T23:53:45.743552+0000 mgr.y (mgr.14901) 3456 : cluster [DBG] pgmap v3463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:47 smithi130 bash[27579]: audit 2023-08-27T23:53:46.519135+0000 mon.c (mon.2) 1327 : audit [DBG] from='client.? 172.21.15.6:0/2860328168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:46 smithi130 bash[51302]: debug 2023-08-27T23:53:46.933+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:46.937127+0000) 2023-08-27T23:53:47.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:47 smithi006 bash[33918]: audit 2023-08-27T23:53:46.519135+0000 mon.c (mon.2) 1327 : audit [DBG] from='client.? 172.21.15.6:0/2860328168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:47.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:47 smithi006 bash[27292]: audit 2023-08-27T23:53:46.519135+0000 mon.c (mon.2) 1327 : audit [DBG] from='client.? 172.21.15.6:0/2860328168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:48.189 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:47 smithi130 bash[51302]: debug 2023-08-27T23:53:47.933+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:47.937409+0000) 2023-08-27T23:53:48.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:48 smithi006 bash[33918]: cluster 2023-08-27T23:53:47.744328+0000 mgr.y (mgr.14901) 3457 : cluster [DBG] pgmap v3464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:48.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:48 smithi006 bash[27292]: cluster 2023-08-27T23:53:47.744328+0000 mgr.y (mgr.14901) 3457 : cluster [DBG] pgmap v3464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:48 smithi130 bash[27579]: cluster 2023-08-27T23:53:47.744328+0000 mgr.y (mgr.14901) 3457 : cluster [DBG] pgmap v3464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:49.192 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:48 smithi130 bash[51302]: debug 2023-08-27T23:53:48.933+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:48.937720+0000) 2023-08-27T23:53:49.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:49 smithi006 bash[33918]: audit 2023-08-27T23:53:48.936250+0000 mon.a (mon.0) 2707 : audit [DBG] from='client.? 172.21.15.6:0/4245390667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:49.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:49 smithi006 bash[27292]: audit 2023-08-27T23:53:48.936250+0000 mon.a (mon.0) 2707 : audit [DBG] from='client.? 172.21.15.6:0/4245390667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:49.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:49 smithi130 bash[27579]: audit 2023-08-27T23:53:48.936250+0000 mon.a (mon.0) 2707 : audit [DBG] from='client.? 172.21.15.6:0/4245390667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:50.202 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:49 smithi130 bash[51302]: debug 2023-08-27T23:53:49.937+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:49.938009+0000) 2023-08-27T23:53:50.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:50 smithi006 bash[27292]: cluster 2023-08-27T23:53:49.745637+0000 mgr.y (mgr.14901) 3458 : cluster [DBG] pgmap v3465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:50.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:50 smithi006 bash[33918]: cluster 2023-08-27T23:53:49.745637+0000 mgr.y (mgr.14901) 3458 : cluster [DBG] pgmap v3465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:50 smithi130 bash[27579]: cluster 2023-08-27T23:53:49.745637+0000 mgr.y (mgr.14901) 3458 : cluster [DBG] pgmap v3465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:50.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:53:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:53:50] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-27T23:53:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:53:50.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:53:50.594Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:51.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:50 smithi130 bash[51302]: debug 2023-08-27T23:53:50.933+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:50.938303+0000) 2023-08-27T23:53:51.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:51 smithi006 bash[27292]: audit 2023-08-27T23:53:51.359482+0000 mon.c (mon.2) 1328 : audit [DBG] from='client.? 172.21.15.6:0/1518316655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:51.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:51 smithi006 bash[33918]: audit 2023-08-27T23:53:51.359482+0000 mon.c (mon.2) 1328 : audit [DBG] from='client.? 172.21.15.6:0/1518316655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:51.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:51 smithi130 bash[27579]: audit 2023-08-27T23:53:51.359482+0000 mon.c (mon.2) 1328 : audit [DBG] from='client.? 172.21.15.6:0/1518316655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:52.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:51 smithi130 bash[51302]: debug 2023-08-27T23:53:51.937+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:51.938577+0000) 2023-08-27T23:53:52.423 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:53:52.158Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:52.423 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:53:52.160Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:52.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:52 smithi006 bash[27292]: cluster 2023-08-27T23:53:51.746339+0000 mgr.y (mgr.14901) 3459 : cluster [DBG] pgmap v3466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:52 smithi006 bash[33918]: cluster 2023-08-27T23:53:51.746339+0000 mgr.y (mgr.14901) 3459 : cluster [DBG] pgmap v3466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:52 smithi130 bash[27579]: cluster 2023-08-27T23:53:51.746339+0000 mgr.y (mgr.14901) 3459 : cluster [DBG] pgmap v3466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:53.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:52 smithi130 bash[51302]: debug 2023-08-27T23:53:52.937+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:52.938908+0000) 2023-08-27T23:53:54.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:53 smithi130 bash[27579]: audit 2023-08-27T23:53:53.772057+0000 mon.a (mon.0) 2708 : audit [DBG] from='client.? 172.21.15.6:0/2862472182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:54.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:53 smithi130 bash[51302]: debug 2023-08-27T23:53:53.937+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:53.939137+0000) 2023-08-27T23:53:54.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:53 smithi006 bash[27292]: audit 2023-08-27T23:53:53.772057+0000 mon.a (mon.0) 2708 : audit [DBG] from='client.? 172.21.15.6:0/2862472182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:54.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:53 smithi006 bash[33918]: audit 2023-08-27T23:53:53.772057+0000 mon.a (mon.0) 2708 : audit [DBG] from='client.? 172.21.15.6:0/2862472182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:54 smithi006 bash[56298]: level=error ts=2023-08-27T23:53:54.053Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:53:54 smithi006 bash[56298]: level=warn ts=2023-08-27T23:53:54.056Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:53:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:54 smithi130 bash[27579]: cluster 2023-08-27T23:53:53.746937+0000 mgr.y (mgr.14901) 3460 : cluster [DBG] pgmap v3467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:54 smithi130 bash[51302]: debug 2023-08-27T23:53:54.937+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:54.939397+0000) 2023-08-27T23:53:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:54 smithi006 bash[27292]: cluster 2023-08-27T23:53:53.746937+0000 mgr.y (mgr.14901) 3460 : cluster [DBG] pgmap v3467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:54 smithi006 bash[33918]: cluster 2023-08-27T23:53:53.746937+0000 mgr.y (mgr.14901) 3460 : cluster [DBG] pgmap v3467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:56.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:55 smithi130 bash[51302]: debug 2023-08-27T23:53:55.937+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:55.939685+0000) 2023-08-27T23:53:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:56 smithi130 bash[27579]: audit 2023-08-27T23:53:55.648360+0000 mon.b (mon.1) 445 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:53:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:56 smithi130 bash[27579]: cluster 2023-08-27T23:53:55.747967+0000 mgr.y (mgr.14901) 3461 : cluster [DBG] pgmap v3468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:56.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:56 smithi006 bash[27292]: audit 2023-08-27T23:53:55.648360+0000 mon.b (mon.1) 445 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:53:56.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:56 smithi006 bash[27292]: cluster 2023-08-27T23:53:55.747967+0000 mgr.y (mgr.14901) 3461 : cluster [DBG] pgmap v3468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:56.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:56 smithi006 bash[33918]: audit 2023-08-27T23:53:55.648360+0000 mon.b (mon.1) 445 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:53:56.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:56 smithi006 bash[33918]: cluster 2023-08-27T23:53:55.747967+0000 mgr.y (mgr.14901) 3461 : cluster [DBG] pgmap v3468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:57.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:57 smithi130 bash[27579]: audit 2023-08-27T23:53:56.186297+0000 mon.a (mon.0) 2709 : audit [DBG] from='client.? 172.21.15.6:0/1449507792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:57.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:56 smithi130 bash[51302]: debug 2023-08-27T23:53:56.937+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:56.939956+0000) 2023-08-27T23:53:57.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:57 smithi006 bash[27292]: audit 2023-08-27T23:53:56.186297+0000 mon.a (mon.0) 2709 : audit [DBG] from='client.? 172.21.15.6:0/1449507792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:57.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:57 smithi006 bash[33918]: audit 2023-08-27T23:53:56.186297+0000 mon.a (mon.0) 2709 : audit [DBG] from='client.? 172.21.15.6:0/1449507792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:58.193 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:57 smithi130 bash[51302]: debug 2023-08-27T23:53:57.937+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:57.940262+0000) 2023-08-27T23:53:58.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:58 smithi006 bash[27292]: cluster 2023-08-27T23:53:57.748640+0000 mgr.y (mgr.14901) 3462 : cluster [DBG] pgmap v3469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:58.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:58 smithi006 bash[33918]: cluster 2023-08-27T23:53:57.748640+0000 mgr.y (mgr.14901) 3462 : cluster [DBG] pgmap v3469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:58 smithi130 bash[27579]: cluster 2023-08-27T23:53:57.748640+0000 mgr.y (mgr.14901) 3462 : cluster [DBG] pgmap v3469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:53:59.196 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:58 smithi130 bash[51302]: debug 2023-08-27T23:53:58.937+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:58.940500+0000) 2023-08-27T23:53:59.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:53:59 smithi006 bash[33918]: audit 2023-08-27T23:53:58.605950+0000 mon.c (mon.2) 1329 : audit [DBG] from='client.? 172.21.15.6:0/1074603659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:59.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:53:59 smithi006 bash[27292]: audit 2023-08-27T23:53:58.605950+0000 mon.c (mon.2) 1329 : audit [DBG] from='client.? 172.21.15.6:0/1074603659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:59.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:53:59 smithi130 bash[27579]: audit 2023-08-27T23:53:58.605950+0000 mon.c (mon.2) 1329 : audit [DBG] from='client.? 172.21.15.6:0/1074603659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:00.205 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:53:59 smithi130 bash[51302]: debug 2023-08-27T23:53:59.937+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:59.940769+0000) 2023-08-27T23:54:00.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:00 smithi006 bash[33918]: cluster 2023-08-27T23:53:59.749846+0000 mgr.y (mgr.14901) 3463 : cluster [DBG] pgmap v3470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:00.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:00 smithi006 bash[27292]: cluster 2023-08-27T23:53:59.749846+0000 mgr.y (mgr.14901) 3463 : cluster [DBG] pgmap v3470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:00 smithi130 bash[27579]: cluster 2023-08-27T23:53:59.749846+0000 mgr.y (mgr.14901) 3463 : cluster [DBG] pgmap v3470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:00.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:54:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:54:00] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-27T23:54:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:54:00.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:54:00.595Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:01.205 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:00 smithi130 bash[51302]: debug 2023-08-27T23:54:00.937+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:00.940983+0000) 2023-08-27T23:54:01.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:01 smithi006 bash[27292]: audit 2023-08-27T23:54:01.042170+0000 mon.c (mon.2) 1330 : audit [DBG] from='client.? 172.21.15.6:0/3731188846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:01.480 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:01 smithi006 bash[33918]: audit 2023-08-27T23:54:01.042170+0000 mon.c (mon.2) 1330 : audit [DBG] from='client.? 172.21.15.6:0/3731188846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:01.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:01 smithi130 bash[27579]: audit 2023-08-27T23:54:01.042170+0000 mon.c (mon.2) 1330 : audit [DBG] from='client.? 172.21.15.6:0/3731188846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:02.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:01 smithi130 bash[51302]: debug 2023-08-27T23:54:01.937+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:01.941197+0000) 2023-08-27T23:54:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:54:02.159Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:54:02.161Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:02 smithi130 bash[27579]: audit 2023-08-27T23:54:01.503639+0000 mon.a (mon.0) 2710 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:54:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:02 smithi130 bash[27579]: cluster 2023-08-27T23:54:01.750523+0000 mgr.y (mgr.14901) 3464 : cluster [DBG] pgmap v3471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:02 smithi130 bash[27579]: audit 2023-08-27T23:54:01.940739+0000 mon.a (mon.0) 2711 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:54:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:02 smithi006 bash[33918]: audit 2023-08-27T23:54:01.503639+0000 mon.a (mon.0) 2710 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:54:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:02 smithi006 bash[33918]: cluster 2023-08-27T23:54:01.750523+0000 mgr.y (mgr.14901) 3464 : cluster [DBG] pgmap v3471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:02 smithi006 bash[33918]: audit 2023-08-27T23:54:01.940739+0000 mon.a (mon.0) 2711 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:54:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:02 smithi006 bash[27292]: audit 2023-08-27T23:54:01.503639+0000 mon.a (mon.0) 2710 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:54:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:02 smithi006 bash[27292]: cluster 2023-08-27T23:54:01.750523+0000 mgr.y (mgr.14901) 3464 : cluster [DBG] pgmap v3471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:02 smithi006 bash[27292]: audit 2023-08-27T23:54:01.940739+0000 mon.a (mon.0) 2711 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:54:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:02 smithi130 bash[51302]: debug 2023-08-27T23:54:02.941+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:02.941446+0000) 2023-08-27T23:54:03.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:03 smithi130 bash[27579]: audit 2023-08-27T23:54:03.460235+0000 mon.a (mon.0) 2712 : audit [DBG] from='client.? 172.21.15.6:0/1817334289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:03.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:03 smithi006 bash[33918]: audit 2023-08-27T23:54:03.460235+0000 mon.a (mon.0) 2712 : audit [DBG] from='client.? 172.21.15.6:0/1817334289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:03.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:03 smithi006 bash[27292]: audit 2023-08-27T23:54:03.460235+0000 mon.a (mon.0) 2712 : audit [DBG] from='client.? 172.21.15.6:0/1817334289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:03 smithi130 bash[51302]: debug 2023-08-27T23:54:03.941+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:03.941776+0000) 2023-08-27T23:54:04.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:04 smithi006 bash[56298]: level=error ts=2023-08-27T23:54:04.054Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:04 smithi006 bash[56298]: level=warn ts=2023-08-27T23:54:04.056Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:04 smithi130 bash[27579]: cluster 2023-08-27T23:54:03.751199+0000 mgr.y (mgr.14901) 3465 : cluster [DBG] pgmap v3472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:04.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:04 smithi006 bash[27292]: cluster 2023-08-27T23:54:03.751199+0000 mgr.y (mgr.14901) 3465 : cluster [DBG] pgmap v3472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:04 smithi006 bash[33918]: cluster 2023-08-27T23:54:03.751199+0000 mgr.y (mgr.14901) 3465 : cluster [DBG] pgmap v3472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:04 smithi130 bash[51302]: debug 2023-08-27T23:54:04.941+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:04.942058+0000) 2023-08-27T23:54:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:06 smithi130 bash[27579]: cluster 2023-08-27T23:54:05.752269+0000 mgr.y (mgr.14901) 3466 : cluster [DBG] pgmap v3473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:06 smithi130 bash[27579]: audit 2023-08-27T23:54:05.873080+0000 mon.c (mon.2) 1331 : audit [DBG] from='client.? 172.21.15.6:0/1482529299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:06.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:05 smithi130 bash[51302]: debug 2023-08-27T23:54:05.937+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:05.942375+0000) 2023-08-27T23:54:06.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:06 smithi006 bash[27292]: cluster 2023-08-27T23:54:05.752269+0000 mgr.y (mgr.14901) 3466 : cluster [DBG] pgmap v3473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:06.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:06 smithi006 bash[27292]: audit 2023-08-27T23:54:05.873080+0000 mon.c (mon.2) 1331 : audit [DBG] from='client.? 172.21.15.6:0/1482529299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:06.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:06 smithi006 bash[33918]: cluster 2023-08-27T23:54:05.752269+0000 mgr.y (mgr.14901) 3466 : cluster [DBG] pgmap v3473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:06.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:06 smithi006 bash[33918]: audit 2023-08-27T23:54:05.873080+0000 mon.c (mon.2) 1331 : audit [DBG] from='client.? 172.21.15.6:0/1482529299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:07.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:06 smithi130 bash[51302]: debug 2023-08-27T23:54:06.937+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:06.942691+0000) 2023-08-27T23:54:08.257 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:07 smithi130 bash[51302]: debug 2023-08-27T23:54:07.941+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:07.943003+0000) 2023-08-27T23:54:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:08 smithi130 bash[27579]: cluster 2023-08-27T23:54:07.752999+0000 mgr.y (mgr.14901) 3467 : cluster [DBG] pgmap v3474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:08 smithi130 bash[27579]: audit 2023-08-27T23:54:08.292565+0000 mon.a (mon.0) 2713 : audit [DBG] from='client.? 172.21.15.6:0/3299789034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:09.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:08 smithi130 bash[51302]: debug 2023-08-27T23:54:08.941+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:08.943285+0000) 2023-08-27T23:54:09.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:08 smithi006 bash[27292]: cluster 2023-08-27T23:54:07.752999+0000 mgr.y (mgr.14901) 3467 : cluster [DBG] pgmap v3474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:09.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:08 smithi006 bash[27292]: audit 2023-08-27T23:54:08.292565+0000 mon.a (mon.0) 2713 : audit [DBG] from='client.? 172.21.15.6:0/3299789034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:09.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:08 smithi006 bash[33918]: cluster 2023-08-27T23:54:07.752999+0000 mgr.y (mgr.14901) 3467 : cluster [DBG] pgmap v3474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:09.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:08 smithi006 bash[33918]: audit 2023-08-27T23:54:08.292565+0000 mon.a (mon.0) 2713 : audit [DBG] from='client.? 172.21.15.6:0/3299789034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:10.236 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:09 smithi130 bash[51302]: debug 2023-08-27T23:54:09.941+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:09.943533+0000) 2023-08-27T23:54:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:54:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:54:10] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-27T23:54:10.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:10 smithi006 bash[27292]: cluster 2023-08-27T23:54:09.754077+0000 mgr.y (mgr.14901) 3468 : cluster [DBG] pgmap v3475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:10.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:10 smithi006 bash[27292]: audit 2023-08-27T23:54:10.721777+0000 mon.a (mon.0) 2714 : audit [DBG] from='client.? 172.21.15.6:0/106871190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:10.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:10 smithi006 bash[33918]: cluster 2023-08-27T23:54:09.754077+0000 mgr.y (mgr.14901) 3468 : cluster [DBG] pgmap v3475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:10.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:10 smithi006 bash[33918]: audit 2023-08-27T23:54:10.721777+0000 mon.a (mon.0) 2714 : audit [DBG] from='client.? 172.21.15.6:0/106871190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:54:10.594Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:54:10.596Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:11.091 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:10 smithi130 bash[27579]: cluster 2023-08-27T23:54:09.754077+0000 mgr.y (mgr.14901) 3468 : cluster [DBG] pgmap v3475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:11.091 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:10 smithi130 bash[27579]: audit 2023-08-27T23:54:10.721777+0000 mon.a (mon.0) 2714 : audit [DBG] from='client.? 172.21.15.6:0/106871190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:11.091 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:10 smithi130 bash[51302]: debug 2023-08-27T23:54:10.941+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:10.943829+0000) 2023-08-27T23:54:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:11 smithi130 bash[51302]: debug 2023-08-27T23:54:11.941+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:11.944057+0000) 2023-08-27T23:54:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:54:12.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:54:12.162Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:12 smithi006 bash[33918]: cluster 2023-08-27T23:54:11.754748+0000 mgr.y (mgr.14901) 3469 : cluster [DBG] pgmap v3476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:12 smithi006 bash[33918]: audit 2023-08-27T23:54:11.919183+0000 mon.a (mon.0) 2715 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:54:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:12 smithi006 bash[33918]: audit 2023-08-27T23:54:12.208400+0000 mon.b (mon.1) 446 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:54:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:12 smithi006 bash[33918]: audit 2023-08-27T23:54:12.210675+0000 mon.b (mon.1) 447 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:54:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:12 smithi006 bash[33918]: audit 2023-08-27T23:54:12.219423+0000 mon.a (mon.0) 2716 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:54:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:12 smithi006 bash[27292]: cluster 2023-08-27T23:54:11.754748+0000 mgr.y (mgr.14901) 3469 : cluster [DBG] pgmap v3476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:12 smithi006 bash[27292]: audit 2023-08-27T23:54:11.919183+0000 mon.a (mon.0) 2715 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:54:13.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:12 smithi006 bash[27292]: audit 2023-08-27T23:54:12.208400+0000 mon.b (mon.1) 446 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:54:13.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:12 smithi006 bash[27292]: audit 2023-08-27T23:54:12.210675+0000 mon.b (mon.1) 447 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:54:13.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:12 smithi006 bash[27292]: audit 2023-08-27T23:54:12.219423+0000 mon.a (mon.0) 2716 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:54:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:12 smithi130 bash[51302]: debug 2023-08-27T23:54:12.941+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:12.944304+0000) 2023-08-27T23:54:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:12 smithi130 bash[27579]: cluster 2023-08-27T23:54:11.754748+0000 mgr.y (mgr.14901) 3469 : cluster [DBG] pgmap v3476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:12 smithi130 bash[27579]: audit 2023-08-27T23:54:11.919183+0000 mon.a (mon.0) 2715 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:54:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:12 smithi130 bash[27579]: audit 2023-08-27T23:54:12.208400+0000 mon.b (mon.1) 446 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:54:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:12 smithi130 bash[27579]: audit 2023-08-27T23:54:12.210675+0000 mon.b (mon.1) 447 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:54:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:12 smithi130 bash[27579]: audit 2023-08-27T23:54:12.219423+0000 mon.a (mon.0) 2716 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:54:14.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:13 smithi006 bash[33918]: audit 2023-08-27T23:54:13.135387+0000 mon.c (mon.2) 1332 : audit [DBG] from='client.? 172.21.15.6:0/2888022919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:14.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:13 smithi006 bash[27292]: audit 2023-08-27T23:54:13.135387+0000 mon.c (mon.2) 1332 : audit [DBG] from='client.? 172.21.15.6:0/2888022919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:14 smithi006 bash[56298]: level=error ts=2023-08-27T23:54:14.054Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:14.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:14 smithi006 bash[56298]: level=warn ts=2023-08-27T23:54:14.056Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:14.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:13 smithi130 bash[51302]: debug 2023-08-27T23:54:13.940+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:13.944617+0000) 2023-08-27T23:54:14.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:13 smithi130 bash[27579]: audit 2023-08-27T23:54:13.135387+0000 mon.c (mon.2) 1332 : audit [DBG] from='client.? 172.21.15.6:0/2888022919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:15.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:14 smithi006 bash[33918]: cluster 2023-08-27T23:54:13.755509+0000 mgr.y (mgr.14901) 3470 : cluster [DBG] pgmap v3477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:14 smithi006 bash[27292]: cluster 2023-08-27T23:54:13.755509+0000 mgr.y (mgr.14901) 3470 : cluster [DBG] pgmap v3477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:14 smithi130 bash[27579]: cluster 2023-08-27T23:54:13.755509+0000 mgr.y (mgr.14901) 3470 : cluster [DBG] pgmap v3477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:14 smithi130 bash[51302]: debug 2023-08-27T23:54:14.940+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:14.944920+0000) 2023-08-27T23:54:16.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:15 smithi130 bash[51302]: debug 2023-08-27T23:54:15.940+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:15.945186+0000) 2023-08-27T23:54:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:16 smithi130 bash[27579]: audit 2023-08-27T23:54:15.559406+0000 mon.c (mon.2) 1333 : audit [DBG] from='client.? 172.21.15.6:0/2445886093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:16 smithi130 bash[27579]: cluster 2023-08-27T23:54:15.756760+0000 mgr.y (mgr.14901) 3471 : cluster [DBG] pgmap v3478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:16.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:16 smithi006 bash[33918]: audit 2023-08-27T23:54:15.559406+0000 mon.c (mon.2) 1333 : audit [DBG] from='client.? 172.21.15.6:0/2445886093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:16.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:16 smithi006 bash[33918]: cluster 2023-08-27T23:54:15.756760+0000 mgr.y (mgr.14901) 3471 : cluster [DBG] pgmap v3478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:16.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:16 smithi006 bash[27292]: audit 2023-08-27T23:54:15.559406+0000 mon.c (mon.2) 1333 : audit [DBG] from='client.? 172.21.15.6:0/2445886093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:16.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:16 smithi006 bash[27292]: cluster 2023-08-27T23:54:15.756760+0000 mgr.y (mgr.14901) 3471 : cluster [DBG] pgmap v3478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:17.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:16 smithi130 bash[51302]: debug 2023-08-27T23:54:16.944+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:16.945471+0000) 2023-08-27T23:54:18.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:17 smithi130 bash[51302]: debug 2023-08-27T23:54:17.944+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:17.945771+0000) 2023-08-27T23:54:19.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:18 smithi130 bash[51302]: debug 2023-08-27T23:54:18.944+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:18.946064+0000) 2023-08-27T23:54:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:18 smithi130 bash[27579]: cluster 2023-08-27T23:54:17.757479+0000 mgr.y (mgr.14901) 3472 : cluster [DBG] pgmap v3479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:18 smithi130 bash[27579]: audit 2023-08-27T23:54:17.966408+0000 mon.a (mon.0) 2717 : audit [DBG] from='client.? 172.21.15.6:0/3487837558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:18 smithi006 bash[27292]: cluster 2023-08-27T23:54:17.757479+0000 mgr.y (mgr.14901) 3472 : cluster [DBG] pgmap v3479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:18 smithi006 bash[27292]: audit 2023-08-27T23:54:17.966408+0000 mon.a (mon.0) 2717 : audit [DBG] from='client.? 172.21.15.6:0/3487837558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:18 smithi006 bash[33918]: cluster 2023-08-27T23:54:17.757479+0000 mgr.y (mgr.14901) 3472 : cluster [DBG] pgmap v3479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:19.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:18 smithi006 bash[33918]: audit 2023-08-27T23:54:17.966408+0000 mon.a (mon.0) 2717 : audit [DBG] from='client.? 172.21.15.6:0/3487837558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:20.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:19 smithi130 bash[51302]: debug 2023-08-27T23:54:19.940+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:19.946398+0000) 2023-08-27T23:54:20.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:20 smithi006 bash[33918]: cluster 2023-08-27T23:54:19.758559+0000 mgr.y (mgr.14901) 3473 : cluster [DBG] pgmap v3480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:20.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:20 smithi006 bash[33918]: audit 2023-08-27T23:54:20.392802+0000 mon.a (mon.0) 2718 : audit [DBG] from='client.? 172.21.15.6:0/1233934354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:54:20.595Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:54:20.597Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:20.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:20 smithi006 bash[27292]: cluster 2023-08-27T23:54:19.758559+0000 mgr.y (mgr.14901) 3473 : cluster [DBG] pgmap v3480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:20.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:20 smithi006 bash[27292]: audit 2023-08-27T23:54:20.392802+0000 mon.a (mon.0) 2718 : audit [DBG] from='client.? 172.21.15.6:0/1233934354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:20.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:54:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:54:20] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-27T23:54:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:20 smithi130 bash[27579]: cluster 2023-08-27T23:54:19.758559+0000 mgr.y (mgr.14901) 3473 : cluster [DBG] pgmap v3480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:20 smithi130 bash[27579]: audit 2023-08-27T23:54:20.392802+0000 mon.a (mon.0) 2718 : audit [DBG] from='client.? 172.21.15.6:0/1233934354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:21.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:20 smithi130 bash[51302]: debug 2023-08-27T23:54:20.940+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:20.946672+0000) 2023-08-27T23:54:22.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:21 smithi130 bash[51302]: debug 2023-08-27T23:54:21.944+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:21.946978+0000) 2023-08-27T23:54:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:54:22.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:54:22.162Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:22 smithi130 bash[27579]: cluster 2023-08-27T23:54:21.759230+0000 mgr.y (mgr.14901) 3474 : cluster [DBG] pgmap v3481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:22 smithi130 bash[27579]: audit 2023-08-27T23:54:22.799001+0000 mon.a (mon.0) 2719 : audit [DBG] from='client.? 172.21.15.6:0/4140098167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:23.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:22 smithi130 bash[51302]: debug 2023-08-27T23:54:22.944+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:22.947193+0000) 2023-08-27T23:54:23.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:22 smithi006 bash[33918]: cluster 2023-08-27T23:54:21.759230+0000 mgr.y (mgr.14901) 3474 : cluster [DBG] pgmap v3481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:22 smithi006 bash[33918]: audit 2023-08-27T23:54:22.799001+0000 mon.a (mon.0) 2719 : audit [DBG] from='client.? 172.21.15.6:0/4140098167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:22 smithi006 bash[27292]: cluster 2023-08-27T23:54:21.759230+0000 mgr.y (mgr.14901) 3474 : cluster [DBG] pgmap v3481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:22 smithi006 bash[27292]: audit 2023-08-27T23:54:22.799001+0000 mon.a (mon.0) 2719 : audit [DBG] from='client.? 172.21.15.6:0/4140098167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:23 smithi130 bash[51302]: debug 2023-08-27T23:54:23.944+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:23.947419+0000) 2023-08-27T23:54:24.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:24 smithi006 bash[56298]: level=error ts=2023-08-27T23:54:24.055Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:24 smithi006 bash[56298]: level=warn ts=2023-08-27T23:54:24.057Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:24 smithi130 bash[27579]: cluster 2023-08-27T23:54:23.759850+0000 mgr.y (mgr.14901) 3475 : cluster [DBG] pgmap v3482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:24 smithi130 bash[51302]: debug 2023-08-27T23:54:24.944+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:24.947664+0000) 2023-08-27T23:54:25.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:24 smithi006 bash[27292]: cluster 2023-08-27T23:54:23.759850+0000 mgr.y (mgr.14901) 3475 : cluster [DBG] pgmap v3482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:24 smithi006 bash[33918]: cluster 2023-08-27T23:54:23.759850+0000 mgr.y (mgr.14901) 3475 : cluster [DBG] pgmap v3482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:26.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:25 smithi130 bash[51302]: debug 2023-08-27T23:54:25.944+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:25.947846+0000) 2023-08-27T23:54:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:26 smithi130 bash[27579]: audit 2023-08-27T23:54:25.213520+0000 mon.a (mon.0) 2720 : audit [DBG] from='client.? 172.21.15.6:0/36249502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:26 smithi130 bash[27579]: cluster 2023-08-27T23:54:25.761037+0000 mgr.y (mgr.14901) 3476 : cluster [DBG] pgmap v3483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:26 smithi006 bash[27292]: audit 2023-08-27T23:54:25.213520+0000 mon.a (mon.0) 2720 : audit [DBG] from='client.? 172.21.15.6:0/36249502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:26 smithi006 bash[27292]: cluster 2023-08-27T23:54:25.761037+0000 mgr.y (mgr.14901) 3476 : cluster [DBG] pgmap v3483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:26 smithi006 bash[33918]: audit 2023-08-27T23:54:25.213520+0000 mon.a (mon.0) 2720 : audit [DBG] from='client.? 172.21.15.6:0/36249502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:26 smithi006 bash[33918]: cluster 2023-08-27T23:54:25.761037+0000 mgr.y (mgr.14901) 3476 : cluster [DBG] pgmap v3483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:27.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:26 smithi130 bash[51302]: debug 2023-08-27T23:54:26.944+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:26.948092+0000) 2023-08-27T23:54:27.949 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:27 smithi130 bash[27579]: audit 2023-08-27T23:54:27.624642+0000 mon.a (mon.0) 2721 : audit [DBG] from='client.? 172.21.15.6:0/1492243370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:27.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:27 smithi006 bash[33918]: audit 2023-08-27T23:54:27.624642+0000 mon.a (mon.0) 2721 : audit [DBG] from='client.? 172.21.15.6:0/1492243370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:27.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:27 smithi006 bash[27292]: audit 2023-08-27T23:54:27.624642+0000 mon.a (mon.0) 2721 : audit [DBG] from='client.? 172.21.15.6:0/1492243370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:28.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:27 smithi130 bash[51302]: debug 2023-08-27T23:54:27.944+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:27.948335+0000) 2023-08-27T23:54:28.949 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:28 smithi130 bash[27579]: cluster 2023-08-27T23:54:27.761842+0000 mgr.y (mgr.14901) 3477 : cluster [DBG] pgmap v3484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:28.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:28 smithi006 bash[33918]: cluster 2023-08-27T23:54:27.761842+0000 mgr.y (mgr.14901) 3477 : cluster [DBG] pgmap v3484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:28 smithi006 bash[27292]: cluster 2023-08-27T23:54:27.761842+0000 mgr.y (mgr.14901) 3477 : cluster [DBG] pgmap v3484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:29.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:28 smithi130 bash[51302]: debug 2023-08-27T23:54:28.944+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:28.948645+0000) 2023-08-27T23:54:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:29 smithi130 bash[51302]: debug 2023-08-27T23:54:29.948+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:29.948923+0000) 2023-08-27T23:54:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:54:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:54:30] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-27T23:54:30.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:30 smithi006 bash[33918]: cluster 2023-08-27T23:54:29.762915+0000 mgr.y (mgr.14901) 3478 : cluster [DBG] pgmap v3485: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:30.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:30 smithi006 bash[33918]: audit 2023-08-27T23:54:30.042054+0000 mon.a (mon.0) 2722 : audit [DBG] from='client.? 172.21.15.6:0/2238000403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:54:30.596Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:54:30.598Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:30.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:30 smithi006 bash[27292]: cluster 2023-08-27T23:54:29.762915+0000 mgr.y (mgr.14901) 3478 : cluster [DBG] pgmap v3485: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:30.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:30 smithi006 bash[27292]: audit 2023-08-27T23:54:30.042054+0000 mon.a (mon.0) 2722 : audit [DBG] from='client.? 172.21.15.6:0/2238000403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:31.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:30 smithi130 bash[27579]: cluster 2023-08-27T23:54:29.762915+0000 mgr.y (mgr.14901) 3478 : cluster [DBG] pgmap v3485: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:31.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:30 smithi130 bash[27579]: audit 2023-08-27T23:54:30.042054+0000 mon.a (mon.0) 2722 : audit [DBG] from='client.? 172.21.15.6:0/2238000403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:31.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:30 smithi130 bash[51302]: debug 2023-08-27T23:54:30.944+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:30.949190+0000) 2023-08-27T23:54:32.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:31 smithi130 bash[51302]: debug 2023-08-27T23:54:31.944+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:31.949543+0000) 2023-08-27T23:54:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:54:32.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:54:32.163Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:32 smithi130 bash[27579]: cluster 2023-08-27T23:54:31.763675+0000 mgr.y (mgr.14901) 3479 : cluster [DBG] pgmap v3486: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:32 smithi130 bash[27579]: audit 2023-08-27T23:54:32.461479+0000 mon.a (mon.0) 2723 : audit [DBG] from='client.? 172.21.15.6:0/2312879989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:33.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:32 smithi130 bash[51302]: debug 2023-08-27T23:54:32.948+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:32.949831+0000) 2023-08-27T23:54:33.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:32 smithi006 bash[33918]: cluster 2023-08-27T23:54:31.763675+0000 mgr.y (mgr.14901) 3479 : cluster [DBG] pgmap v3486: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:33.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:32 smithi006 bash[33918]: audit 2023-08-27T23:54:32.461479+0000 mon.a (mon.0) 2723 : audit [DBG] from='client.? 172.21.15.6:0/2312879989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:32 smithi006 bash[27292]: cluster 2023-08-27T23:54:31.763675+0000 mgr.y (mgr.14901) 3479 : cluster [DBG] pgmap v3486: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:32 smithi006 bash[27292]: audit 2023-08-27T23:54:32.461479+0000 mon.a (mon.0) 2723 : audit [DBG] from='client.? 172.21.15.6:0/2312879989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:34.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:33 smithi130 bash[51302]: debug 2023-08-27T23:54:33.948+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:33.950145+0000) 2023-08-27T23:54:34.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:34 smithi006 bash[56298]: level=error ts=2023-08-27T23:54:34.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:34.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:34 smithi006 bash[56298]: level=warn ts=2023-08-27T23:54:34.058Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:35.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:34 smithi130 bash[27579]: cluster 2023-08-27T23:54:33.764499+0000 mgr.y (mgr.14901) 3480 : cluster [DBG] pgmap v3487: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:35.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:34 smithi130 bash[51302]: debug 2023-08-27T23:54:34.948+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:34.950437+0000) 2023-08-27T23:54:35.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:34 smithi006 bash[27292]: cluster 2023-08-27T23:54:33.764499+0000 mgr.y (mgr.14901) 3480 : cluster [DBG] pgmap v3487: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:34 smithi006 bash[33918]: cluster 2023-08-27T23:54:33.764499+0000 mgr.y (mgr.14901) 3480 : cluster [DBG] pgmap v3487: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:36.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:35 smithi130 bash[51302]: debug 2023-08-27T23:54:35.948+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:35.950669+0000) 2023-08-27T23:54:36.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:35 smithi130 bash[27579]: audit 2023-08-27T23:54:34.881041+0000 mon.a (mon.0) 2724 : audit [DBG] from='client.? 172.21.15.6:0/2681980218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:36.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:35 smithi006 bash[27292]: audit 2023-08-27T23:54:34.881041+0000 mon.a (mon.0) 2724 : audit [DBG] from='client.? 172.21.15.6:0/2681980218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:36.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:35 smithi006 bash[33918]: audit 2023-08-27T23:54:34.881041+0000 mon.a (mon.0) 2724 : audit [DBG] from='client.? 172.21.15.6:0/2681980218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:37.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:36 smithi130 bash[51302]: debug 2023-08-27T23:54:36.948+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:36.950976+0000) 2023-08-27T23:54:37.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:36 smithi130 bash[27579]: cluster 2023-08-27T23:54:35.765679+0000 mgr.y (mgr.14901) 3481 : cluster [DBG] pgmap v3488: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:37.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:36 smithi006 bash[27292]: cluster 2023-08-27T23:54:35.765679+0000 mgr.y (mgr.14901) 3481 : cluster [DBG] pgmap v3488: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:37.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:36 smithi006 bash[33918]: cluster 2023-08-27T23:54:35.765679+0000 mgr.y (mgr.14901) 3481 : cluster [DBG] pgmap v3488: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:38.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:37 smithi130 bash[27579]: audit 2023-08-27T23:54:37.293625+0000 mon.a (mon.0) 2725 : audit [DBG] from='client.? 172.21.15.6:0/3465853498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:38.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:37 smithi130 bash[51302]: debug 2023-08-27T23:54:37.948+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:37.951241+0000) 2023-08-27T23:54:38.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:37 smithi006 bash[33918]: audit 2023-08-27T23:54:37.293625+0000 mon.a (mon.0) 2725 : audit [DBG] from='client.? 172.21.15.6:0/3465853498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:38.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:37 smithi006 bash[27292]: audit 2023-08-27T23:54:37.293625+0000 mon.a (mon.0) 2725 : audit [DBG] from='client.? 172.21.15.6:0/3465853498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:39.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:38 smithi130 bash[51302]: debug 2023-08-27T23:54:38.948+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:38.951472+0000) 2023-08-27T23:54:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:38 smithi130 bash[27579]: cluster 2023-08-27T23:54:37.766352+0000 mgr.y (mgr.14901) 3482 : cluster [DBG] pgmap v3489: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:39.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:38 smithi006 bash[33918]: cluster 2023-08-27T23:54:37.766352+0000 mgr.y (mgr.14901) 3482 : cluster [DBG] pgmap v3489: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:39.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:38 smithi006 bash[27292]: cluster 2023-08-27T23:54:37.766352+0000 mgr.y (mgr.14901) 3482 : cluster [DBG] pgmap v3489: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:40.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:39 smithi130 bash[51302]: debug 2023-08-27T23:54:39.948+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:39.951737+0000) 2023-08-27T23:54:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:39 smithi130 bash[27579]: audit 2023-08-27T23:54:39.710905+0000 mon.a (mon.0) 2726 : audit [DBG] from='client.? 172.21.15.6:0/503882104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:40.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:39 smithi006 bash[33918]: audit 2023-08-27T23:54:39.710905+0000 mon.a (mon.0) 2726 : audit [DBG] from='client.? 172.21.15.6:0/503882104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:40.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:39 smithi006 bash[27292]: audit 2023-08-27T23:54:39.710905+0000 mon.a (mon.0) 2726 : audit [DBG] from='client.? 172.21.15.6:0/503882104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:40.863 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:54:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:54:40] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-27T23:54:40.863 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:54:40.596Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:40.863 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:54:40.598Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:41.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:40 smithi130 bash[51302]: debug 2023-08-27T23:54:40.948+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:40.952032+0000) 2023-08-27T23:54:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:40 smithi130 bash[27579]: cluster 2023-08-27T23:54:39.767466+0000 mgr.y (mgr.14901) 3483 : cluster [DBG] pgmap v3490: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:41.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:40 smithi006 bash[33918]: cluster 2023-08-27T23:54:39.767466+0000 mgr.y (mgr.14901) 3483 : cluster [DBG] pgmap v3490: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:40 smithi006 bash[27292]: cluster 2023-08-27T23:54:39.767466+0000 mgr.y (mgr.14901) 3483 : cluster [DBG] pgmap v3490: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:42.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:41 smithi130 bash[51302]: debug 2023-08-27T23:54:41.948+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:41.952321+0000) 2023-08-27T23:54:42.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:54:42.162Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:54:42.164Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:43.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:42 smithi006 bash[33918]: cluster 2023-08-27T23:54:41.768230+0000 mgr.y (mgr.14901) 3484 : cluster [DBG] pgmap v3491: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:42 smithi006 bash[33918]: audit 2023-08-27T23:54:42.116296+0000 mon.c (mon.2) 1334 : audit [DBG] from='client.? 172.21.15.6:0/2258783024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:42 smithi006 bash[27292]: cluster 2023-08-27T23:54:41.768230+0000 mgr.y (mgr.14901) 3484 : cluster [DBG] pgmap v3491: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:42 smithi006 bash[27292]: audit 2023-08-27T23:54:42.116296+0000 mon.c (mon.2) 1334 : audit [DBG] from='client.? 172.21.15.6:0/2258783024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:43.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:42 smithi130 bash[51302]: debug 2023-08-27T23:54:42.948+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:42.952595+0000) 2023-08-27T23:54:43.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:42 smithi130 bash[27579]: cluster 2023-08-27T23:54:41.768230+0000 mgr.y (mgr.14901) 3484 : cluster [DBG] pgmap v3491: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:43.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:42 smithi130 bash[27579]: audit 2023-08-27T23:54:42.116296+0000 mon.c (mon.2) 1334 : audit [DBG] from='client.? 172.21.15.6:0/2258783024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:44.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:43 smithi130 bash[51302]: debug 2023-08-27T23:54:43.948+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:43.952865+0000) 2023-08-27T23:54:44.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:44 smithi006 bash[56298]: level=error ts=2023-08-27T23:54:44.057Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:44 smithi006 bash[56298]: level=warn ts=2023-08-27T23:54:44.059Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:44 smithi006 bash[33918]: cluster 2023-08-27T23:54:43.768940+0000 mgr.y (mgr.14901) 3485 : cluster [DBG] pgmap v3492: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:44 smithi006 bash[33918]: audit 2023-08-27T23:54:44.530661+0000 mon.a (mon.0) 2727 : audit [DBG] from='client.? 172.21.15.6:0/836424246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:44 smithi006 bash[27292]: cluster 2023-08-27T23:54:43.768940+0000 mgr.y (mgr.14901) 3485 : cluster [DBG] pgmap v3492: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:44 smithi006 bash[27292]: audit 2023-08-27T23:54:44.530661+0000 mon.a (mon.0) 2727 : audit [DBG] from='client.? 172.21.15.6:0/836424246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:44 smithi130 bash[27579]: cluster 2023-08-27T23:54:43.768940+0000 mgr.y (mgr.14901) 3485 : cluster [DBG] pgmap v3492: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:44 smithi130 bash[27579]: audit 2023-08-27T23:54:44.530661+0000 mon.a (mon.0) 2727 : audit [DBG] from='client.? 172.21.15.6:0/836424246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:44 smithi130 bash[51302]: debug 2023-08-27T23:54:44.952+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:44.953158+0000) 2023-08-27T23:54:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:46 smithi130 bash[27579]: cluster 2023-08-27T23:54:45.770025+0000 mgr.y (mgr.14901) 3486 : cluster [DBG] pgmap v3493: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:45 smithi130 bash[51302]: debug 2023-08-27T23:54:45.948+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:45.953449+0000) 2023-08-27T23:54:46.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:46 smithi006 bash[33918]: cluster 2023-08-27T23:54:45.770025+0000 mgr.y (mgr.14901) 3486 : cluster [DBG] pgmap v3493: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:46.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:46 smithi006 bash[27292]: cluster 2023-08-27T23:54:45.770025+0000 mgr.y (mgr.14901) 3486 : cluster [DBG] pgmap v3493: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:47 smithi130 bash[27579]: audit 2023-08-27T23:54:46.941829+0000 mon.c (mon.2) 1335 : audit [DBG] from='client.? 172.21.15.6:0/1303615578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:46 smithi130 bash[51302]: debug 2023-08-27T23:54:46.948+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:46.953717+0000) 2023-08-27T23:54:47.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:47 smithi006 bash[27292]: audit 2023-08-27T23:54:46.941829+0000 mon.c (mon.2) 1335 : audit [DBG] from='client.? 172.21.15.6:0/1303615578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:47.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:47 smithi006 bash[33918]: audit 2023-08-27T23:54:46.941829+0000 mon.c (mon.2) 1335 : audit [DBG] from='client.? 172.21.15.6:0/1303615578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:48.206 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:47 smithi130 bash[51302]: debug 2023-08-27T23:54:47.952+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:47.953994+0000) 2023-08-27T23:54:48.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:48 smithi006 bash[27292]: cluster 2023-08-27T23:54:47.770788+0000 mgr.y (mgr.14901) 3487 : cluster [DBG] pgmap v3494: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:48.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:48 smithi006 bash[33918]: cluster 2023-08-27T23:54:47.770788+0000 mgr.y (mgr.14901) 3487 : cluster [DBG] pgmap v3494: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:48 smithi130 bash[27579]: cluster 2023-08-27T23:54:47.770788+0000 mgr.y (mgr.14901) 3487 : cluster [DBG] pgmap v3494: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:48 smithi130 bash[51302]: debug 2023-08-27T23:54:48.948+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:48.954271+0000) 2023-08-27T23:54:49.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:49 smithi006 bash[27292]: audit 2023-08-27T23:54:49.359214+0000 mon.a (mon.0) 2728 : audit [DBG] from='client.? 172.21.15.6:0/995803351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:49.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:49 smithi006 bash[33918]: audit 2023-08-27T23:54:49.359214+0000 mon.a (mon.0) 2728 : audit [DBG] from='client.? 172.21.15.6:0/995803351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:49.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:49 smithi130 bash[27579]: audit 2023-08-27T23:54:49.359214+0000 mon.a (mon.0) 2728 : audit [DBG] from='client.? 172.21.15.6:0/995803351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:49 smithi130 bash[51302]: debug 2023-08-27T23:54:49.952+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:49.954568+0000) 2023-08-27T23:54:50.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:50 smithi006 bash[33918]: cluster 2023-08-27T23:54:49.771927+0000 mgr.y (mgr.14901) 3488 : cluster [DBG] pgmap v3495: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:50 smithi006 bash[27292]: cluster 2023-08-27T23:54:49.771927+0000 mgr.y (mgr.14901) 3488 : cluster [DBG] pgmap v3495: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:54:50.597Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:54:50.599Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:50 smithi130 bash[27579]: cluster 2023-08-27T23:54:49.771927+0000 mgr.y (mgr.14901) 3488 : cluster [DBG] pgmap v3495: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:54:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:54:50] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-27T23:54:51.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:50 smithi130 bash[51302]: debug 2023-08-27T23:54:50.952+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:50.954844+0000) 2023-08-27T23:54:52.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:51 smithi130 bash[27579]: audit 2023-08-27T23:54:51.779367+0000 mon.a (mon.0) 2729 : audit [DBG] from='client.? 172.21.15.6:0/3056068833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:52.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:51 smithi130 bash[51302]: debug 2023-08-27T23:54:51.952+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:51.955151+0000) 2023-08-27T23:54:52.165 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:51 smithi006 bash[33918]: audit 2023-08-27T23:54:51.779367+0000 mon.a (mon.0) 2729 : audit [DBG] from='client.? 172.21.15.6:0/3056068833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:52.165 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:51 smithi006 bash[27292]: audit 2023-08-27T23:54:51.779367+0000 mon.a (mon.0) 2729 : audit [DBG] from='client.? 172.21.15.6:0/3056068833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:54:52.163Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:54:52.165Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:52 smithi130 bash[27579]: cluster 2023-08-27T23:54:51.772674+0000 mgr.y (mgr.14901) 3489 : cluster [DBG] pgmap v3496: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:52 smithi130 bash[51302]: debug 2023-08-27T23:54:52.952+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:52.955384+0000) 2023-08-27T23:54:53.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:52 smithi006 bash[33918]: cluster 2023-08-27T23:54:51.772674+0000 mgr.y (mgr.14901) 3489 : cluster [DBG] pgmap v3496: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:52 smithi006 bash[27292]: cluster 2023-08-27T23:54:51.772674+0000 mgr.y (mgr.14901) 3489 : cluster [DBG] pgmap v3496: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:54.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:53 smithi130 bash[51302]: debug 2023-08-27T23:54:53.952+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:53.955671+0000) 2023-08-27T23:54:54.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:54 smithi006 bash[56298]: level=error ts=2023-08-27T23:54:54.057Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:54:54 smithi006 bash[56298]: level=warn ts=2023-08-27T23:54:54.060Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:54:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:54 smithi130 bash[51302]: debug 2023-08-27T23:54:54.952+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:54.955969+0000) 2023-08-27T23:54:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:54 smithi130 bash[27579]: cluster 2023-08-27T23:54:53.773429+0000 mgr.y (mgr.14901) 3490 : cluster [DBG] pgmap v3497: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:54 smithi130 bash[27579]: audit 2023-08-27T23:54:54.197305+0000 mon.a (mon.0) 2730 : audit [DBG] from='client.? 172.21.15.6:0/1140475355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:55.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:54 smithi006 bash[33918]: cluster 2023-08-27T23:54:53.773429+0000 mgr.y (mgr.14901) 3490 : cluster [DBG] pgmap v3497: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:54 smithi006 bash[33918]: audit 2023-08-27T23:54:54.197305+0000 mon.a (mon.0) 2730 : audit [DBG] from='client.? 172.21.15.6:0/1140475355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:54 smithi006 bash[27292]: cluster 2023-08-27T23:54:53.773429+0000 mgr.y (mgr.14901) 3490 : cluster [DBG] pgmap v3497: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:54 smithi006 bash[27292]: audit 2023-08-27T23:54:54.197305+0000 mon.a (mon.0) 2730 : audit [DBG] from='client.? 172.21.15.6:0/1140475355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:56 smithi130 bash[27579]: cluster 2023-08-27T23:54:55.774562+0000 mgr.y (mgr.14901) 3491 : cluster [DBG] pgmap v3498: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:56.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:55 smithi130 bash[51302]: debug 2023-08-27T23:54:55.952+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:55.956251+0000) 2023-08-27T23:54:56.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:56 smithi006 bash[33918]: cluster 2023-08-27T23:54:55.774562+0000 mgr.y (mgr.14901) 3491 : cluster [DBG] pgmap v3498: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:56.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:56 smithi006 bash[27292]: cluster 2023-08-27T23:54:55.774562+0000 mgr.y (mgr.14901) 3491 : cluster [DBG] pgmap v3498: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:57.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:57 smithi130 bash[27579]: audit 2023-08-27T23:54:56.610150+0000 mon.c (mon.2) 1336 : audit [DBG] from='client.? 172.21.15.6:0/3933426591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:57.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:56 smithi130 bash[51302]: debug 2023-08-27T23:54:56.956+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:56.956545+0000) 2023-08-27T23:54:57.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:57 smithi006 bash[33918]: audit 2023-08-27T23:54:56.610150+0000 mon.c (mon.2) 1336 : audit [DBG] from='client.? 172.21.15.6:0/3933426591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:57.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:57 smithi006 bash[27292]: audit 2023-08-27T23:54:56.610150+0000 mon.c (mon.2) 1336 : audit [DBG] from='client.? 172.21.15.6:0/3933426591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:58.210 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:57 smithi130 bash[51302]: debug 2023-08-27T23:54:57.952+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:57.956804+0000) 2023-08-27T23:54:58.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:58 smithi006 bash[27292]: cluster 2023-08-27T23:54:57.775195+0000 mgr.y (mgr.14901) 3492 : cluster [DBG] pgmap v3499: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:58.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:58 smithi006 bash[33918]: cluster 2023-08-27T23:54:57.775195+0000 mgr.y (mgr.14901) 3492 : cluster [DBG] pgmap v3499: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:58 smithi130 bash[27579]: cluster 2023-08-27T23:54:57.775195+0000 mgr.y (mgr.14901) 3492 : cluster [DBG] pgmap v3499: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:54:59.213 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:58 smithi130 bash[51302]: debug 2023-08-27T23:54:58.952+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:58.957095+0000) 2023-08-27T23:54:59.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:54:59 smithi006 bash[33918]: audit 2023-08-27T23:54:59.026007+0000 mon.a (mon.0) 2731 : audit [DBG] from='client.? 172.21.15.6:0/2822277901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:59.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:54:59 smithi006 bash[27292]: audit 2023-08-27T23:54:59.026007+0000 mon.a (mon.0) 2731 : audit [DBG] from='client.? 172.21.15.6:0/2822277901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:54:59 smithi130 bash[27579]: audit 2023-08-27T23:54:59.026007+0000 mon.a (mon.0) 2731 : audit [DBG] from='client.? 172.21.15.6:0/2822277901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:00.233 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:54:59 smithi130 bash[51302]: debug 2023-08-27T23:54:59.952+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:59.957434+0000) 2023-08-27T23:55:00.600 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:00 smithi006 bash[33918]: cluster 2023-08-27T23:54:59.776289+0000 mgr.y (mgr.14901) 3493 : cluster [DBG] pgmap v3500: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:00.600 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:00 smithi006 bash[27292]: cluster 2023-08-27T23:54:59.776289+0000 mgr.y (mgr.14901) 3493 : cluster [DBG] pgmap v3500: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:00 smithi130 bash[27579]: cluster 2023-08-27T23:54:59.776289+0000 mgr.y (mgr.14901) 3493 : cluster [DBG] pgmap v3500: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:00.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:55:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:55:00] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:55:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:55:00.597Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:55:00.599Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:01.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:00 smithi130 bash[51302]: debug 2023-08-27T23:55:00.952+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:00.957702+0000) 2023-08-27T23:55:01.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:01 smithi130 bash[27579]: audit 2023-08-27T23:55:01.451056+0000 mon.a (mon.0) 2732 : audit [DBG] from='client.? 172.21.15.6:0/1948075363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:01.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:01 smithi006 bash[27292]: audit 2023-08-27T23:55:01.451056+0000 mon.a (mon.0) 2732 : audit [DBG] from='client.? 172.21.15.6:0/1948075363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:01.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:01 smithi006 bash[33918]: audit 2023-08-27T23:55:01.451056+0000 mon.a (mon.0) 2732 : audit [DBG] from='client.? 172.21.15.6:0/1948075363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:02.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:01 smithi130 bash[51302]: debug 2023-08-27T23:55:01.952+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:01.957982+0000) 2023-08-27T23:55:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:55:02.164Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:55:02.165Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:02 smithi130 bash[27579]: cluster 2023-08-27T23:55:01.776971+0000 mgr.y (mgr.14901) 3494 : cluster [DBG] pgmap v3501: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:02 smithi006 bash[27292]: cluster 2023-08-27T23:55:01.776971+0000 mgr.y (mgr.14901) 3494 : cluster [DBG] pgmap v3501: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:02 smithi006 bash[33918]: cluster 2023-08-27T23:55:01.776971+0000 mgr.y (mgr.14901) 3494 : cluster [DBG] pgmap v3501: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:02 smithi130 bash[51302]: debug 2023-08-27T23:55:02.952+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:02.958241+0000) 2023-08-27T23:55:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:03 smithi130 bash[51302]: debug 2023-08-27T23:55:03.956+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:03.958510+0000) 2023-08-27T23:55:04.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:04 smithi006 bash[56298]: level=error ts=2023-08-27T23:55:04.058Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:04 smithi006 bash[56298]: level=warn ts=2023-08-27T23:55:04.060Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:04 smithi130 bash[51302]: debug 2023-08-27T23:55:04.952+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:04.958781+0000) 2023-08-27T23:55:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:04 smithi130 bash[27579]: cluster 2023-08-27T23:55:03.777597+0000 mgr.y (mgr.14901) 3495 : cluster [DBG] pgmap v3502: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:04 smithi130 bash[27579]: audit 2023-08-27T23:55:03.872788+0000 mon.a (mon.0) 2733 : audit [DBG] from='client.? 172.21.15.6:0/1672716505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:05.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:04 smithi006 bash[33918]: cluster 2023-08-27T23:55:03.777597+0000 mgr.y (mgr.14901) 3495 : cluster [DBG] pgmap v3502: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:04 smithi006 bash[33918]: audit 2023-08-27T23:55:03.872788+0000 mon.a (mon.0) 2733 : audit [DBG] from='client.? 172.21.15.6:0/1672716505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:04 smithi006 bash[27292]: cluster 2023-08-27T23:55:03.777597+0000 mgr.y (mgr.14901) 3495 : cluster [DBG] pgmap v3502: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:04 smithi006 bash[27292]: audit 2023-08-27T23:55:03.872788+0000 mon.a (mon.0) 2733 : audit [DBG] from='client.? 172.21.15.6:0/1672716505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:06.217 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:05 smithi130 bash[51302]: debug 2023-08-27T23:55:05.956+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:05.959082+0000) 2023-08-27T23:55:06.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:06 smithi006 bash[33918]: cluster 2023-08-27T23:55:05.778713+0000 mgr.y (mgr.14901) 3496 : cluster [DBG] pgmap v3503: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:06.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:06 smithi006 bash[27292]: cluster 2023-08-27T23:55:05.778713+0000 mgr.y (mgr.14901) 3496 : cluster [DBG] pgmap v3503: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:06 smithi130 bash[27579]: cluster 2023-08-27T23:55:05.778713+0000 mgr.y (mgr.14901) 3496 : cluster [DBG] pgmap v3503: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:07.220 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:06 smithi130 bash[51302]: debug 2023-08-27T23:55:06.956+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:06.959363+0000) 2023-08-27T23:55:07.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:07 smithi130 bash[27579]: audit 2023-08-27T23:55:06.293953+0000 mon.a (mon.0) 2734 : audit [DBG] from='client.? 172.21.15.6:0/1384858622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:07.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:07 smithi006 bash[27292]: audit 2023-08-27T23:55:06.293953+0000 mon.a (mon.0) 2734 : audit [DBG] from='client.? 172.21.15.6:0/1384858622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:07 smithi006 bash[33918]: audit 2023-08-27T23:55:06.293953+0000 mon.a (mon.0) 2734 : audit [DBG] from='client.? 172.21.15.6:0/1384858622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:08.227 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:07 smithi130 bash[51302]: debug 2023-08-27T23:55:07.956+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:07.959603+0000) 2023-08-27T23:55:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:08 smithi130 bash[27579]: cluster 2023-08-27T23:55:07.779359+0000 mgr.y (mgr.14901) 3497 : cluster [DBG] pgmap v3504: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:08.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:08 smithi006 bash[27292]: cluster 2023-08-27T23:55:07.779359+0000 mgr.y (mgr.14901) 3497 : cluster [DBG] pgmap v3504: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:08 smithi006 bash[33918]: cluster 2023-08-27T23:55:07.779359+0000 mgr.y (mgr.14901) 3497 : cluster [DBG] pgmap v3504: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:09.231 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:08 smithi130 bash[51302]: debug 2023-08-27T23:55:08.956+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:08.959875+0000) 2023-08-27T23:55:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:09 smithi130 bash[27579]: audit 2023-08-27T23:55:08.711415+0000 mon.c (mon.2) 1337 : audit [DBG] from='client.? 172.21.15.6:0/2132804309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:09.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:09 smithi006 bash[33918]: audit 2023-08-27T23:55:08.711415+0000 mon.c (mon.2) 1337 : audit [DBG] from='client.? 172.21.15.6:0/2132804309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:09 smithi006 bash[27292]: audit 2023-08-27T23:55:08.711415+0000 mon.c (mon.2) 1337 : audit [DBG] from='client.? 172.21.15.6:0/2132804309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:10.237 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:09 smithi130 bash[51302]: debug 2023-08-27T23:55:09.956+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:09.960148+0000) 2023-08-27T23:55:10.601 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:10 smithi006 bash[27292]: cluster 2023-08-27T23:55:09.780458+0000 mgr.y (mgr.14901) 3498 : cluster [DBG] pgmap v3505: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:10.601 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:10 smithi006 bash[33918]: cluster 2023-08-27T23:55:09.780458+0000 mgr.y (mgr.14901) 3498 : cluster [DBG] pgmap v3505: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:10 smithi130 bash[27579]: cluster 2023-08-27T23:55:09.780458+0000 mgr.y (mgr.14901) 3498 : cluster [DBG] pgmap v3505: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:10.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:55:10.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:55:10.600Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:55:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:55:10] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:55:11.240 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:10 smithi130 bash[51302]: debug 2023-08-27T23:55:10.955+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:10.960424+0000) 2023-08-27T23:55:11.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:11 smithi130 bash[27579]: audit 2023-08-27T23:55:11.132396+0000 mon.a (mon.0) 2735 : audit [DBG] from='client.? 172.21.15.6:0/811421826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:11.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:11 smithi006 bash[27292]: audit 2023-08-27T23:55:11.132396+0000 mon.a (mon.0) 2735 : audit [DBG] from='client.? 172.21.15.6:0/811421826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:11.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:11 smithi006 bash[33918]: audit 2023-08-27T23:55:11.132396+0000 mon.a (mon.0) 2735 : audit [DBG] from='client.? 172.21.15.6:0/811421826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:12.248 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:11 smithi130 bash[51302]: debug 2023-08-27T23:55:11.955+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:11.960714+0000) 2023-08-27T23:55:12.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:12 smithi006 bash[33918]: cluster 2023-08-27T23:55:11.781117+0000 mgr.y (mgr.14901) 3499 : cluster [DBG] pgmap v3506: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:12.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:12 smithi006 bash[33918]: audit 2023-08-27T23:55:12.230549+0000 mon.b (mon.1) 448 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:55:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:55:12.164Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:55:12.166Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:12.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:12 smithi006 bash[27292]: cluster 2023-08-27T23:55:11.781117+0000 mgr.y (mgr.14901) 3499 : cluster [DBG] pgmap v3506: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:12.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:12 smithi006 bash[27292]: audit 2023-08-27T23:55:12.230549+0000 mon.b (mon.1) 448 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:55:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:12 smithi130 bash[27579]: cluster 2023-08-27T23:55:11.781117+0000 mgr.y (mgr.14901) 3499 : cluster [DBG] pgmap v3506: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:12 smithi130 bash[27579]: audit 2023-08-27T23:55:12.230549+0000 mon.b (mon.1) 448 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:55:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:12 smithi130 bash[51302]: debug 2023-08-27T23:55:12.959+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:12.961000+0000) 2023-08-27T23:55:13.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:13 smithi130 bash[27579]: audit 2023-08-27T23:55:13.554920+0000 mon.a (mon.0) 2736 : audit [DBG] from='client.? 172.21.15.6:0/1061529473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:13.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:13 smithi006 bash[27292]: audit 2023-08-27T23:55:13.554920+0000 mon.a (mon.0) 2736 : audit [DBG] from='client.? 172.21.15.6:0/1061529473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:13.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:13 smithi006 bash[33918]: audit 2023-08-27T23:55:13.554920+0000 mon.a (mon.0) 2736 : audit [DBG] from='client.? 172.21.15.6:0/1061529473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:14.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:13 smithi130 bash[51302]: debug 2023-08-27T23:55:13.955+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:13.961267+0000) 2023-08-27T23:55:14.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:14 smithi006 bash[56298]: level=error ts=2023-08-27T23:55:14.058Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:14 smithi006 bash[56298]: level=warn ts=2023-08-27T23:55:14.060Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:14.962 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:14 smithi130 bash[27579]: cluster 2023-08-27T23:55:13.781742+0000 mgr.y (mgr.14901) 3500 : cluster [DBG] pgmap v3507: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:14.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:14 smithi006 bash[33918]: cluster 2023-08-27T23:55:13.781742+0000 mgr.y (mgr.14901) 3500 : cluster [DBG] pgmap v3507: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:14 smithi006 bash[27292]: cluster 2023-08-27T23:55:13.781742+0000 mgr.y (mgr.14901) 3500 : cluster [DBG] pgmap v3507: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:14 smithi130 bash[51302]: debug 2023-08-27T23:55:14.955+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:14.961574+0000) 2023-08-27T23:55:16.219 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:15 smithi130 bash[51302]: debug 2023-08-27T23:55:15.955+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:15.961865+0000) 2023-08-27T23:55:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:16 smithi130 bash[27579]: cluster 2023-08-27T23:55:15.782763+0000 mgr.y (mgr.14901) 3501 : cluster [DBG] pgmap v3508: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:16 smithi130 bash[27579]: audit 2023-08-27T23:55:15.968945+0000 mon.a (mon.0) 2737 : audit [DBG] from='client.? 172.21.15.6:0/1127406503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:16.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:16 smithi006 bash[33918]: cluster 2023-08-27T23:55:15.782763+0000 mgr.y (mgr.14901) 3501 : cluster [DBG] pgmap v3508: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:16 smithi006 bash[33918]: audit 2023-08-27T23:55:15.968945+0000 mon.a (mon.0) 2737 : audit [DBG] from='client.? 172.21.15.6:0/1127406503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:16 smithi006 bash[27292]: cluster 2023-08-27T23:55:15.782763+0000 mgr.y (mgr.14901) 3501 : cluster [DBG] pgmap v3508: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:16 smithi006 bash[27292]: audit 2023-08-27T23:55:15.968945+0000 mon.a (mon.0) 2737 : audit [DBG] from='client.? 172.21.15.6:0/1127406503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:17.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:16 smithi130 bash[51302]: debug 2023-08-27T23:55:16.955+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:16.962141+0000) 2023-08-27T23:55:18.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:17 smithi130 bash[51302]: debug 2023-08-27T23:55:17.955+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:17.962395+0000) 2023-08-27T23:55:19.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:18 smithi130 bash[51302]: debug 2023-08-27T23:55:18.955+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:18.962695+0000) 2023-08-27T23:55:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:18 smithi130 bash[27579]: cluster 2023-08-27T23:55:17.783384+0000 mgr.y (mgr.14901) 3502 : cluster [DBG] pgmap v3509: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:18 smithi130 bash[27579]: audit 2023-08-27T23:55:18.145495+0000 mon.a (mon.0) 2738 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:55:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:18 smithi130 bash[27579]: audit 2023-08-27T23:55:18.423438+0000 mon.a (mon.0) 2739 : audit [DBG] from='client.? 172.21.15.6:0/4167021114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:18 smithi130 bash[27579]: audit 2023-08-27T23:55:18.566748+0000 mon.a (mon.0) 2740 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:55:19.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:18 smithi006 bash[33918]: cluster 2023-08-27T23:55:17.783384+0000 mgr.y (mgr.14901) 3502 : cluster [DBG] pgmap v3509: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:18 smithi006 bash[33918]: audit 2023-08-27T23:55:18.145495+0000 mon.a (mon.0) 2738 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:55:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:18 smithi006 bash[33918]: audit 2023-08-27T23:55:18.423438+0000 mon.a (mon.0) 2739 : audit [DBG] from='client.? 172.21.15.6:0/4167021114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:18 smithi006 bash[33918]: audit 2023-08-27T23:55:18.566748+0000 mon.a (mon.0) 2740 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:55:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:18 smithi006 bash[27292]: cluster 2023-08-27T23:55:17.783384+0000 mgr.y (mgr.14901) 3502 : cluster [DBG] pgmap v3509: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:18 smithi006 bash[27292]: audit 2023-08-27T23:55:18.145495+0000 mon.a (mon.0) 2738 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:55:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:18 smithi006 bash[27292]: audit 2023-08-27T23:55:18.423438+0000 mon.a (mon.0) 2739 : audit [DBG] from='client.? 172.21.15.6:0/4167021114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:18 smithi006 bash[27292]: audit 2023-08-27T23:55:18.566748+0000 mon.a (mon.0) 2740 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:55:20.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:19 smithi130 bash[51302]: debug 2023-08-27T23:55:19.959+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:19.962986+0000) 2023-08-27T23:55:20.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:19 smithi130 bash[27579]: audit 2023-08-27T23:55:18.862074+0000 mon.b (mon.1) 449 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:55:20.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:19 smithi130 bash[27579]: audit 2023-08-27T23:55:18.864091+0000 mon.b (mon.1) 450 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:55:20.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:19 smithi130 bash[27579]: audit 2023-08-27T23:55:18.874803+0000 mon.a (mon.0) 2741 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:55:20.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:19 smithi006 bash[33918]: audit 2023-08-27T23:55:18.862074+0000 mon.b (mon.1) 449 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:55:20.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:19 smithi006 bash[33918]: audit 2023-08-27T23:55:18.864091+0000 mon.b (mon.1) 450 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:55:20.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:19 smithi006 bash[33918]: audit 2023-08-27T23:55:18.874803+0000 mon.a (mon.0) 2741 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:55:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:19 smithi006 bash[27292]: audit 2023-08-27T23:55:18.862074+0000 mon.b (mon.1) 449 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:55:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:19 smithi006 bash[27292]: audit 2023-08-27T23:55:18.864091+0000 mon.b (mon.1) 450 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:55:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:19 smithi006 bash[27292]: audit 2023-08-27T23:55:18.874803+0000 mon.a (mon.0) 2741 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:55:20.881 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:55:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:55:20] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:55:20.881 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:55:20.600Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:20.881 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:55:20.601Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:21.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:20 smithi006 bash[33918]: cluster 2023-08-27T23:55:19.784476+0000 mgr.y (mgr.14901) 3503 : cluster [DBG] pgmap v3510: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:21.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:20 smithi006 bash[33918]: audit 2023-08-27T23:55:20.836970+0000 mon.c (mon.2) 1338 : audit [DBG] from='client.? 172.21.15.6:0/2092439512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:20 smithi006 bash[27292]: cluster 2023-08-27T23:55:19.784476+0000 mgr.y (mgr.14901) 3503 : cluster [DBG] pgmap v3510: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:20 smithi006 bash[27292]: audit 2023-08-27T23:55:20.836970+0000 mon.c (mon.2) 1338 : audit [DBG] from='client.? 172.21.15.6:0/2092439512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:21.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:20 smithi130 bash[51302]: debug 2023-08-27T23:55:20.959+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:20.963214+0000) 2023-08-27T23:55:21.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:20 smithi130 bash[27579]: cluster 2023-08-27T23:55:19.784476+0000 mgr.y (mgr.14901) 3503 : cluster [DBG] pgmap v3510: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:21.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:20 smithi130 bash[27579]: audit 2023-08-27T23:55:20.836970+0000 mon.c (mon.2) 1338 : audit [DBG] from='client.? 172.21.15.6:0/2092439512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:22.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:21 smithi130 bash[51302]: debug 2023-08-27T23:55:21.959+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:21.963421+0000) 2023-08-27T23:55:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:55:22.166Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:55:22.168Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:23.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:22 smithi006 bash[27292]: cluster 2023-08-27T23:55:21.785179+0000 mgr.y (mgr.14901) 3504 : cluster [DBG] pgmap v3511: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:22 smithi006 bash[33918]: cluster 2023-08-27T23:55:21.785179+0000 mgr.y (mgr.14901) 3504 : cluster [DBG] pgmap v3511: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:23.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:22 smithi130 bash[51302]: debug 2023-08-27T23:55:22.959+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:22.963737+0000) 2023-08-27T23:55:23.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:22 smithi130 bash[27579]: cluster 2023-08-27T23:55:21.785179+0000 mgr.y (mgr.14901) 3504 : cluster [DBG] pgmap v3511: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:24.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:23 smithi006 bash[27292]: audit 2023-08-27T23:55:23.251551+0000 mon.a (mon.0) 2742 : audit [DBG] from='client.? 172.21.15.6:0/501376938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:24.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:23 smithi006 bash[33918]: audit 2023-08-27T23:55:23.251551+0000 mon.a (mon.0) 2742 : audit [DBG] from='client.? 172.21.15.6:0/501376938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:24 smithi006 bash[56298]: level=error ts=2023-08-27T23:55:24.059Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:24 smithi006 bash[56298]: level=warn ts=2023-08-27T23:55:24.061Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:23 smithi130 bash[27579]: audit 2023-08-27T23:55:23.251551+0000 mon.a (mon.0) 2742 : audit [DBG] from='client.? 172.21.15.6:0/501376938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:23 smithi130 bash[51302]: debug 2023-08-27T23:55:23.959+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:23.963962+0000) 2023-08-27T23:55:25.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:24 smithi006 bash[27292]: cluster 2023-08-27T23:55:23.785929+0000 mgr.y (mgr.14901) 3505 : cluster [DBG] pgmap v3512: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:24 smithi006 bash[33918]: cluster 2023-08-27T23:55:23.785929+0000 mgr.y (mgr.14901) 3505 : cluster [DBG] pgmap v3512: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:24 smithi130 bash[27579]: cluster 2023-08-27T23:55:23.785929+0000 mgr.y (mgr.14901) 3505 : cluster [DBG] pgmap v3512: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:24 smithi130 bash[51302]: debug 2023-08-27T23:55:24.959+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:24.964234+0000) 2023-08-27T23:55:26.220 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:25 smithi130 bash[51302]: debug 2023-08-27T23:55:25.963+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:25.964441+0000) 2023-08-27T23:55:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:26 smithi130 bash[27579]: audit 2023-08-27T23:55:25.668812+0000 mon.c (mon.2) 1339 : audit [DBG] from='client.? 172.21.15.6:0/822901304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:26 smithi130 bash[27579]: cluster 2023-08-27T23:55:25.787163+0000 mgr.y (mgr.14901) 3506 : cluster [DBG] pgmap v3513: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:26.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:26 smithi006 bash[27292]: audit 2023-08-27T23:55:25.668812+0000 mon.c (mon.2) 1339 : audit [DBG] from='client.? 172.21.15.6:0/822901304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:26 smithi006 bash[27292]: cluster 2023-08-27T23:55:25.787163+0000 mgr.y (mgr.14901) 3506 : cluster [DBG] pgmap v3513: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:26 smithi006 bash[33918]: audit 2023-08-27T23:55:25.668812+0000 mon.c (mon.2) 1339 : audit [DBG] from='client.? 172.21.15.6:0/822901304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:26 smithi006 bash[33918]: cluster 2023-08-27T23:55:25.787163+0000 mgr.y (mgr.14901) 3506 : cluster [DBG] pgmap v3513: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:27.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:26 smithi130 bash[51302]: debug 2023-08-27T23:55:26.963+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:26.964677+0000) 2023-08-27T23:55:28.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:27 smithi130 bash[51302]: debug 2023-08-27T23:55:27.963+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:27.964982+0000) 2023-08-27T23:55:29.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:28 smithi130 bash[51302]: debug 2023-08-27T23:55:28.959+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:28.965303+0000) 2023-08-27T23:55:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:28 smithi130 bash[27579]: cluster 2023-08-27T23:55:27.787958+0000 mgr.y (mgr.14901) 3507 : cluster [DBG] pgmap v3514: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:28 smithi130 bash[27579]: audit 2023-08-27T23:55:28.084072+0000 mon.c (mon.2) 1340 : audit [DBG] from='client.? 172.21.15.6:0/1051712312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:29.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:28 smithi006 bash[33918]: cluster 2023-08-27T23:55:27.787958+0000 mgr.y (mgr.14901) 3507 : cluster [DBG] pgmap v3514: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:28 smithi006 bash[33918]: audit 2023-08-27T23:55:28.084072+0000 mon.c (mon.2) 1340 : audit [DBG] from='client.? 172.21.15.6:0/1051712312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:28 smithi006 bash[27292]: cluster 2023-08-27T23:55:27.787958+0000 mgr.y (mgr.14901) 3507 : cluster [DBG] pgmap v3514: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:28 smithi006 bash[27292]: audit 2023-08-27T23:55:28.084072+0000 mon.c (mon.2) 1340 : audit [DBG] from='client.? 172.21.15.6:0/1051712312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:29 smithi130 bash[51302]: debug 2023-08-27T23:55:29.959+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:29.965614+0000) 2023-08-27T23:55:30.853 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:55:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:55:30] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:55:30.853 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:55:30.600Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:30.853 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:55:30.602Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:30.853 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:30 smithi006 bash[27292]: cluster 2023-08-27T23:55:29.789279+0000 mgr.y (mgr.14901) 3508 : cluster [DBG] pgmap v3515: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:30.854 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:30 smithi006 bash[27292]: audit 2023-08-27T23:55:30.515138+0000 mon.c (mon.2) 1341 : audit [DBG] from='client.? 172.21.15.6:0/3025173504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:31.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:30 smithi130 bash[27579]: cluster 2023-08-27T23:55:29.789279+0000 mgr.y (mgr.14901) 3508 : cluster [DBG] pgmap v3515: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:31.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:30 smithi130 bash[27579]: audit 2023-08-27T23:55:30.515138+0000 mon.c (mon.2) 1341 : audit [DBG] from='client.? 172.21.15.6:0/3025173504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:31.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:30 smithi130 bash[51302]: debug 2023-08-27T23:55:30.963+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:30.965919+0000) 2023-08-27T23:55:31.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:30 smithi006 bash[33918]: cluster 2023-08-27T23:55:29.789279+0000 mgr.y (mgr.14901) 3508 : cluster [DBG] pgmap v3515: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:31.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:30 smithi006 bash[33918]: audit 2023-08-27T23:55:30.515138+0000 mon.c (mon.2) 1341 : audit [DBG] from='client.? 172.21.15.6:0/3025173504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:32.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:31 smithi130 bash[51302]: debug 2023-08-27T23:55:31.963+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:31.966177+0000) 2023-08-27T23:55:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:55:32.167Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:55:32.169Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:32 smithi130 bash[27579]: cluster 2023-08-27T23:55:31.789991+0000 mgr.y (mgr.14901) 3509 : cluster [DBG] pgmap v3516: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:33.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:32 smithi130 bash[51302]: debug 2023-08-27T23:55:32.959+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:32.966468+0000) 2023-08-27T23:55:33.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:32 smithi006 bash[27292]: cluster 2023-08-27T23:55:31.789991+0000 mgr.y (mgr.14901) 3509 : cluster [DBG] pgmap v3516: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:32 smithi006 bash[33918]: cluster 2023-08-27T23:55:31.789991+0000 mgr.y (mgr.14901) 3509 : cluster [DBG] pgmap v3516: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:33 smithi130 bash[27579]: audit 2023-08-27T23:55:32.936282+0000 mon.a (mon.0) 2743 : audit [DBG] from='client.? 172.21.15.6:0/3370499999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:34.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:33 smithi130 bash[51302]: debug 2023-08-27T23:55:33.959+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:33.966726+0000) 2023-08-27T23:55:34.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:33 smithi006 bash[33918]: audit 2023-08-27T23:55:32.936282+0000 mon.a (mon.0) 2743 : audit [DBG] from='client.? 172.21.15.6:0/3370499999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:34.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:33 smithi006 bash[27292]: audit 2023-08-27T23:55:32.936282+0000 mon.a (mon.0) 2743 : audit [DBG] from='client.? 172.21.15.6:0/3370499999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:34 smithi006 bash[56298]: level=error ts=2023-08-27T23:55:34.061Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:34.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:34 smithi006 bash[56298]: level=warn ts=2023-08-27T23:55:34.063Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:35.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:34 smithi130 bash[27579]: cluster 2023-08-27T23:55:33.790615+0000 mgr.y (mgr.14901) 3510 : cluster [DBG] pgmap v3517: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:35.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:34 smithi130 bash[51302]: debug 2023-08-27T23:55:34.963+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:34.967005+0000) 2023-08-27T23:55:35.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:34 smithi006 bash[27292]: cluster 2023-08-27T23:55:33.790615+0000 mgr.y (mgr.14901) 3510 : cluster [DBG] pgmap v3517: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:34 smithi006 bash[33918]: cluster 2023-08-27T23:55:33.790615+0000 mgr.y (mgr.14901) 3510 : cluster [DBG] pgmap v3517: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:36.227 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:35 smithi130 bash[51302]: debug 2023-08-27T23:55:35.963+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:35.967335+0000) 2023-08-27T23:55:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:36 smithi130 bash[27579]: audit 2023-08-27T23:55:35.350604+0000 mon.a (mon.0) 2744 : audit [DBG] from='client.? 172.21.15.6:0/3734024061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:36 smithi130 bash[27579]: cluster 2023-08-27T23:55:35.791762+0000 mgr.y (mgr.14901) 3511 : cluster [DBG] pgmap v3518: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:36.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:36 smithi006 bash[33918]: audit 2023-08-27T23:55:35.350604+0000 mon.a (mon.0) 2744 : audit [DBG] from='client.? 172.21.15.6:0/3734024061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:36 smithi006 bash[33918]: cluster 2023-08-27T23:55:35.791762+0000 mgr.y (mgr.14901) 3511 : cluster [DBG] pgmap v3518: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:36 smithi006 bash[27292]: audit 2023-08-27T23:55:35.350604+0000 mon.a (mon.0) 2744 : audit [DBG] from='client.? 172.21.15.6:0/3734024061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:36 smithi006 bash[27292]: cluster 2023-08-27T23:55:35.791762+0000 mgr.y (mgr.14901) 3511 : cluster [DBG] pgmap v3518: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:36 smithi130 bash[51302]: debug 2023-08-27T23:55:36.963+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:36.967538+0000) 2023-08-27T23:55:38.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:37 smithi130 bash[27579]: audit 2023-08-27T23:55:37.761206+0000 mon.c (mon.2) 1342 : audit [DBG] from='client.? 172.21.15.6:0/3572787626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:38.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:37 smithi130 bash[51302]: debug 2023-08-27T23:55:37.963+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:37.967806+0000) 2023-08-27T23:55:38.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:37 smithi006 bash[33918]: audit 2023-08-27T23:55:37.761206+0000 mon.c (mon.2) 1342 : audit [DBG] from='client.? 172.21.15.6:0/3572787626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:38.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:37 smithi006 bash[27292]: audit 2023-08-27T23:55:37.761206+0000 mon.c (mon.2) 1342 : audit [DBG] from='client.? 172.21.15.6:0/3572787626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:39.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:38 smithi130 bash[51302]: debug 2023-08-27T23:55:38.967+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:38.968149+0000) 2023-08-27T23:55:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:38 smithi130 bash[27579]: cluster 2023-08-27T23:55:37.792417+0000 mgr.y (mgr.14901) 3512 : cluster [DBG] pgmap v3519: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:39.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:38 smithi006 bash[33918]: cluster 2023-08-27T23:55:37.792417+0000 mgr.y (mgr.14901) 3512 : cluster [DBG] pgmap v3519: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:39.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:38 smithi006 bash[27292]: cluster 2023-08-27T23:55:37.792417+0000 mgr.y (mgr.14901) 3512 : cluster [DBG] pgmap v3519: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:40.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:39 smithi130 bash[51302]: debug 2023-08-27T23:55:39.967+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:39.968449+0000) 2023-08-27T23:55:40.858 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:55:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:55:40] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:55:40.858 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:55:40.601Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:40.858 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:55:40.603Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:41.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:40 smithi130 bash[51302]: debug 2023-08-27T23:55:40.967+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:40.968758+0000) 2023-08-27T23:55:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:40 smithi130 bash[27579]: cluster 2023-08-27T23:55:39.793542+0000 mgr.y (mgr.14901) 3513 : cluster [DBG] pgmap v3520: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:40 smithi130 bash[27579]: audit 2023-08-27T23:55:40.177026+0000 mon.a (mon.0) 2745 : audit [DBG] from='client.? 172.21.15.6:0/3512911738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:41.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:40 smithi006 bash[27292]: cluster 2023-08-27T23:55:39.793542+0000 mgr.y (mgr.14901) 3513 : cluster [DBG] pgmap v3520: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:40 smithi006 bash[27292]: audit 2023-08-27T23:55:40.177026+0000 mon.a (mon.0) 2745 : audit [DBG] from='client.? 172.21.15.6:0/3512911738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:41.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:40 smithi006 bash[33918]: cluster 2023-08-27T23:55:39.793542+0000 mgr.y (mgr.14901) 3513 : cluster [DBG] pgmap v3520: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:41.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:40 smithi006 bash[33918]: audit 2023-08-27T23:55:40.177026+0000 mon.a (mon.0) 2745 : audit [DBG] from='client.? 172.21.15.6:0/3512911738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:42.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:41 smithi130 bash[51302]: debug 2023-08-27T23:55:41.963+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:41.969051+0000) 2023-08-27T23:55:42.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:55:42.168Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:55:42.170Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:43.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:42 smithi130 bash[51302]: debug 2023-08-27T23:55:42.963+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:42.969385+0000) 2023-08-27T23:55:43.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:42 smithi130 bash[27579]: cluster 2023-08-27T23:55:41.794194+0000 mgr.y (mgr.14901) 3514 : cluster [DBG] pgmap v3521: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:43.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:42 smithi130 bash[27579]: audit 2023-08-27T23:55:42.592723+0000 mon.c (mon.2) 1343 : audit [DBG] from='client.? 172.21.15.6:0/3092174098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:43.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:42 smithi006 bash[33918]: cluster 2023-08-27T23:55:41.794194+0000 mgr.y (mgr.14901) 3514 : cluster [DBG] pgmap v3521: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:42 smithi006 bash[33918]: audit 2023-08-27T23:55:42.592723+0000 mon.c (mon.2) 1343 : audit [DBG] from='client.? 172.21.15.6:0/3092174098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:42 smithi006 bash[27292]: cluster 2023-08-27T23:55:41.794194+0000 mgr.y (mgr.14901) 3514 : cluster [DBG] pgmap v3521: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:42 smithi006 bash[27292]: audit 2023-08-27T23:55:42.592723+0000 mon.c (mon.2) 1343 : audit [DBG] from='client.? 172.21.15.6:0/3092174098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:44.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:43 smithi130 bash[51302]: debug 2023-08-27T23:55:43.963+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:43.969657+0000) 2023-08-27T23:55:44.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:44 smithi006 bash[56298]: level=error ts=2023-08-27T23:55:44.061Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:44 smithi006 bash[56298]: level=warn ts=2023-08-27T23:55:44.063Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:45.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:44 smithi006 bash[27292]: cluster 2023-08-27T23:55:43.794769+0000 mgr.y (mgr.14901) 3515 : cluster [DBG] pgmap v3522: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:44 smithi006 bash[33918]: cluster 2023-08-27T23:55:43.794769+0000 mgr.y (mgr.14901) 3515 : cluster [DBG] pgmap v3522: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:44 smithi130 bash[27579]: cluster 2023-08-27T23:55:43.794769+0000 mgr.y (mgr.14901) 3515 : cluster [DBG] pgmap v3522: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:44 smithi130 bash[51302]: debug 2023-08-27T23:55:44.963+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:44.969997+0000) 2023-08-27T23:55:46.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:45 smithi006 bash[27292]: audit 2023-08-27T23:55:45.004470+0000 mon.a (mon.0) 2746 : audit [DBG] from='client.? 172.21.15.6:0/3935740293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:46.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:45 smithi006 bash[33918]: audit 2023-08-27T23:55:45.004470+0000 mon.a (mon.0) 2746 : audit [DBG] from='client.? 172.21.15.6:0/3935740293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:45 smithi130 bash[51302]: debug 2023-08-27T23:55:45.963+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:45.970282+0000) 2023-08-27T23:55:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:45 smithi130 bash[27579]: audit 2023-08-27T23:55:45.004470+0000 mon.a (mon.0) 2746 : audit [DBG] from='client.? 172.21.15.6:0/3935740293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:47.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:46 smithi006 bash[27292]: cluster 2023-08-27T23:55:45.795871+0000 mgr.y (mgr.14901) 3516 : cluster [DBG] pgmap v3523: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:47.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:46 smithi006 bash[33918]: cluster 2023-08-27T23:55:45.795871+0000 mgr.y (mgr.14901) 3516 : cluster [DBG] pgmap v3523: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:46 smithi130 bash[27579]: cluster 2023-08-27T23:55:45.795871+0000 mgr.y (mgr.14901) 3516 : cluster [DBG] pgmap v3523: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:46 smithi130 bash[51302]: debug 2023-08-27T23:55:46.963+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:46.970553+0000) 2023-08-27T23:55:48.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:47 smithi006 bash[33918]: audit 2023-08-27T23:55:47.424925+0000 mon.a (mon.0) 2747 : audit [DBG] from='client.? 172.21.15.6:0/805805805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:48.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:47 smithi006 bash[27292]: audit 2023-08-27T23:55:47.424925+0000 mon.a (mon.0) 2747 : audit [DBG] from='client.? 172.21.15.6:0/805805805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:47 smithi130 bash[27579]: audit 2023-08-27T23:55:47.424925+0000 mon.a (mon.0) 2747 : audit [DBG] from='client.? 172.21.15.6:0/805805805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:48.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:47 smithi130 bash[51302]: debug 2023-08-27T23:55:47.967+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:47.970915+0000) 2023-08-27T23:55:49.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:48 smithi006 bash[33918]: cluster 2023-08-27T23:55:47.796532+0000 mgr.y (mgr.14901) 3517 : cluster [DBG] pgmap v3524: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:49.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:48 smithi006 bash[27292]: cluster 2023-08-27T23:55:47.796532+0000 mgr.y (mgr.14901) 3517 : cluster [DBG] pgmap v3524: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:49.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:48 smithi130 bash[27579]: cluster 2023-08-27T23:55:47.796532+0000 mgr.y (mgr.14901) 3517 : cluster [DBG] pgmap v3524: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:48 smithi130 bash[51302]: debug 2023-08-27T23:55:48.967+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:48.971212+0000) 2023-08-27T23:55:50.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:49 smithi006 bash[27292]: audit 2023-08-27T23:55:49.839421+0000 mon.c (mon.2) 1344 : audit [DBG] from='client.? 172.21.15.6:0/1672519627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:50.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:49 smithi006 bash[33918]: audit 2023-08-27T23:55:49.839421+0000 mon.c (mon.2) 1344 : audit [DBG] from='client.? 172.21.15.6:0/1672519627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:50.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:49 smithi130 bash[27579]: audit 2023-08-27T23:55:49.839421+0000 mon.c (mon.2) 1344 : audit [DBG] from='client.? 172.21.15.6:0/1672519627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:49 smithi130 bash[51302]: debug 2023-08-27T23:55:49.971+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:49.971521+0000) 2023-08-27T23:55:50.898 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:55:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:55:50] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:55:50.898 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:55:50.601Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:50.898 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:55:50.603Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:51.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:50 smithi006 bash[27292]: cluster 2023-08-27T23:55:49.797271+0000 mgr.y (mgr.14901) 3518 : cluster [DBG] pgmap v3525: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:50 smithi006 bash[33918]: cluster 2023-08-27T23:55:49.797271+0000 mgr.y (mgr.14901) 3518 : cluster [DBG] pgmap v3525: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:51.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:50 smithi130 bash[27579]: cluster 2023-08-27T23:55:49.797271+0000 mgr.y (mgr.14901) 3518 : cluster [DBG] pgmap v3525: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:51.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:50 smithi130 bash[51302]: debug 2023-08-27T23:55:50.967+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:50.971753+0000) 2023-08-27T23:55:52.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:51 smithi130 bash[51302]: debug 2023-08-27T23:55:51.971+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:51.972041+0000) 2023-08-27T23:55:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:55:52.168Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:55:52.171Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:53.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:52 smithi006 bash[33918]: cluster 2023-08-27T23:55:51.797929+0000 mgr.y (mgr.14901) 3519 : cluster [DBG] pgmap v3526: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:53.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:52 smithi006 bash[33918]: audit 2023-08-27T23:55:52.257977+0000 mon.c (mon.2) 1345 : audit [DBG] from='client.? 172.21.15.6:0/2520703522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:52 smithi006 bash[27292]: cluster 2023-08-27T23:55:51.797929+0000 mgr.y (mgr.14901) 3519 : cluster [DBG] pgmap v3526: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:52 smithi006 bash[27292]: audit 2023-08-27T23:55:52.257977+0000 mon.c (mon.2) 1345 : audit [DBG] from='client.? 172.21.15.6:0/2520703522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:53.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:52 smithi130 bash[27579]: cluster 2023-08-27T23:55:51.797929+0000 mgr.y (mgr.14901) 3519 : cluster [DBG] pgmap v3526: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:53.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:52 smithi130 bash[27579]: audit 2023-08-27T23:55:52.257977+0000 mon.c (mon.2) 1345 : audit [DBG] from='client.? 172.21.15.6:0/2520703522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:53.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:52 smithi130 bash[51302]: debug 2023-08-27T23:55:52.967+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:52.972355+0000) 2023-08-27T23:55:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:53 smithi130 bash[51302]: debug 2023-08-27T23:55:53.971+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:53.972634+0000) 2023-08-27T23:55:54.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:54 smithi006 bash[56298]: level=error ts=2023-08-27T23:55:54.062Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:55:54 smithi006 bash[56298]: level=warn ts=2023-08-27T23:55:54.064Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:55:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:54 smithi006 bash[33918]: cluster 2023-08-27T23:55:53.798566+0000 mgr.y (mgr.14901) 3520 : cluster [DBG] pgmap v3527: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:54 smithi006 bash[33918]: audit 2023-08-27T23:55:54.669299+0000 mon.c (mon.2) 1346 : audit [DBG] from='client.? 172.21.15.6:0/3444918029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:55.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:54 smithi006 bash[27292]: cluster 2023-08-27T23:55:53.798566+0000 mgr.y (mgr.14901) 3520 : cluster [DBG] pgmap v3527: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:55.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:54 smithi006 bash[27292]: audit 2023-08-27T23:55:54.669299+0000 mon.c (mon.2) 1346 : audit [DBG] from='client.? 172.21.15.6:0/3444918029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:54 smithi130 bash[51302]: debug 2023-08-27T23:55:54.971+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:54.972910+0000) 2023-08-27T23:55:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:54 smithi130 bash[27579]: cluster 2023-08-27T23:55:53.798566+0000 mgr.y (mgr.14901) 3520 : cluster [DBG] pgmap v3527: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:54 smithi130 bash[27579]: audit 2023-08-27T23:55:54.669299+0000 mon.c (mon.2) 1346 : audit [DBG] from='client.? 172.21.15.6:0/3444918029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:56.236 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:55 smithi130 bash[51302]: debug 2023-08-27T23:55:55.971+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:55.973256+0000) 2023-08-27T23:55:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:56 smithi130 bash[27579]: cluster 2023-08-27T23:55:55.799725+0000 mgr.y (mgr.14901) 3521 : cluster [DBG] pgmap v3528: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:56.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:56 smithi006 bash[27292]: cluster 2023-08-27T23:55:55.799725+0000 mgr.y (mgr.14901) 3521 : cluster [DBG] pgmap v3528: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:56 smithi006 bash[33918]: cluster 2023-08-27T23:55:55.799725+0000 mgr.y (mgr.14901) 3521 : cluster [DBG] pgmap v3528: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:57.239 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:56 smithi130 bash[51302]: debug 2023-08-27T23:55:56.971+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:56.973572+0000) 2023-08-27T23:55:57.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:57 smithi130 bash[27579]: audit 2023-08-27T23:55:57.081855+0000 mon.a (mon.0) 2748 : audit [DBG] from='client.? 172.21.15.6:0/1108544176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:57.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:57 smithi006 bash[27292]: audit 2023-08-27T23:55:57.081855+0000 mon.a (mon.0) 2748 : audit [DBG] from='client.? 172.21.15.6:0/1108544176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:57.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:57 smithi006 bash[33918]: audit 2023-08-27T23:55:57.081855+0000 mon.a (mon.0) 2748 : audit [DBG] from='client.? 172.21.15.6:0/1108544176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:58.247 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:57 smithi130 bash[51302]: debug 2023-08-27T23:55:57.967+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:57.973772+0000) 2023-08-27T23:55:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:58 smithi130 bash[27579]: cluster 2023-08-27T23:55:57.800326+0000 mgr.y (mgr.14901) 3522 : cluster [DBG] pgmap v3529: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:58.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:58 smithi006 bash[27292]: cluster 2023-08-27T23:55:57.800326+0000 mgr.y (mgr.14901) 3522 : cluster [DBG] pgmap v3529: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:58 smithi006 bash[33918]: cluster 2023-08-27T23:55:57.800326+0000 mgr.y (mgr.14901) 3522 : cluster [DBG] pgmap v3529: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:55:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:58 smithi130 bash[51302]: debug 2023-08-27T23:55:58.967+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:58.974069+0000) 2023-08-27T23:55:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:55:59 smithi130 bash[27579]: audit 2023-08-27T23:55:59.497164+0000 mon.c (mon.2) 1347 : audit [DBG] from='client.? 172.21.15.6:0/1632144063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:59.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:55:59 smithi006 bash[27292]: audit 2023-08-27T23:55:59.497164+0000 mon.c (mon.2) 1347 : audit [DBG] from='client.? 172.21.15.6:0/1632144063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:55:59 smithi006 bash[33918]: audit 2023-08-27T23:55:59.497164+0000 mon.c (mon.2) 1347 : audit [DBG] from='client.? 172.21.15.6:0/1632144063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:00.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:55:59 smithi130 bash[51302]: debug 2023-08-27T23:55:59.967+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:59.974371+0000) 2023-08-27T23:56:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:00 smithi130 bash[27579]: cluster 2023-08-27T23:55:59.801485+0000 mgr.y (mgr.14901) 3523 : cluster [DBG] pgmap v3530: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:00.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:56:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:56:00] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-27T23:56:00.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:00 smithi006 bash[33918]: cluster 2023-08-27T23:55:59.801485+0000 mgr.y (mgr.14901) 3523 : cluster [DBG] pgmap v3530: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:00.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:00 smithi006 bash[27292]: cluster 2023-08-27T23:55:59.801485+0000 mgr.y (mgr.14901) 3523 : cluster [DBG] pgmap v3530: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:56:00.602Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:56:00.604Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:01.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:00 smithi130 bash[51302]: debug 2023-08-27T23:56:00.967+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:00.974665+0000) 2023-08-27T23:56:02.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:01 smithi130 bash[51302]: debug 2023-08-27T23:56:01.971+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:01.974952+0000) 2023-08-27T23:56:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:56:02.169Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:56:02.171Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:02 smithi130 bash[51302]: debug 2023-08-27T23:56:02.975+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:02.975222+0000) 2023-08-27T23:56:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:02 smithi130 bash[27579]: cluster 2023-08-27T23:56:01.802106+0000 mgr.y (mgr.14901) 3524 : cluster [DBG] pgmap v3531: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:02 smithi130 bash[27579]: audit 2023-08-27T23:56:01.914900+0000 mon.c (mon.2) 1348 : audit [DBG] from='client.? 172.21.15.6:0/3056928863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:02 smithi006 bash[33918]: cluster 2023-08-27T23:56:01.802106+0000 mgr.y (mgr.14901) 3524 : cluster [DBG] pgmap v3531: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:02 smithi006 bash[33918]: audit 2023-08-27T23:56:01.914900+0000 mon.c (mon.2) 1348 : audit [DBG] from='client.? 172.21.15.6:0/3056928863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:02 smithi006 bash[27292]: cluster 2023-08-27T23:56:01.802106+0000 mgr.y (mgr.14901) 3524 : cluster [DBG] pgmap v3531: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:03.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:02 smithi006 bash[27292]: audit 2023-08-27T23:56:01.914900+0000 mon.c (mon.2) 1348 : audit [DBG] from='client.? 172.21.15.6:0/3056928863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:03 smithi130 bash[51302]: debug 2023-08-27T23:56:03.971+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:03.975510+0000) 2023-08-27T23:56:04.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:04 smithi006 bash[56298]: level=error ts=2023-08-27T23:56:04.062Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:04 smithi006 bash[56298]: level=warn ts=2023-08-27T23:56:04.064Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:04 smithi006 bash[33918]: cluster 2023-08-27T23:56:03.802743+0000 mgr.y (mgr.14901) 3525 : cluster [DBG] pgmap v3532: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:04 smithi006 bash[33918]: audit 2023-08-27T23:56:04.336225+0000 mon.c (mon.2) 1349 : audit [DBG] from='client.? 172.21.15.6:0/2601790755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:04 smithi006 bash[27292]: cluster 2023-08-27T23:56:03.802743+0000 mgr.y (mgr.14901) 3525 : cluster [DBG] pgmap v3532: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:04 smithi006 bash[27292]: audit 2023-08-27T23:56:04.336225+0000 mon.c (mon.2) 1349 : audit [DBG] from='client.? 172.21.15.6:0/2601790755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:04 smithi130 bash[27579]: cluster 2023-08-27T23:56:03.802743+0000 mgr.y (mgr.14901) 3525 : cluster [DBG] pgmap v3532: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:04 smithi130 bash[27579]: audit 2023-08-27T23:56:04.336225+0000 mon.c (mon.2) 1349 : audit [DBG] from='client.? 172.21.15.6:0/2601790755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:04 smithi130 bash[51302]: debug 2023-08-27T23:56:04.971+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:04.975781+0000) 2023-08-27T23:56:06.234 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:05 smithi130 bash[51302]: debug 2023-08-27T23:56:05.975+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:05.976073+0000) 2023-08-27T23:56:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:06 smithi130 bash[27579]: cluster 2023-08-27T23:56:05.803883+0000 mgr.y (mgr.14901) 3526 : cluster [DBG] pgmap v3533: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:06 smithi006 bash[27292]: cluster 2023-08-27T23:56:05.803883+0000 mgr.y (mgr.14901) 3526 : cluster [DBG] pgmap v3533: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:06 smithi006 bash[33918]: cluster 2023-08-27T23:56:05.803883+0000 mgr.y (mgr.14901) 3526 : cluster [DBG] pgmap v3533: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:07.239 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:06 smithi130 bash[51302]: debug 2023-08-27T23:56:06.970+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:06.976386+0000) 2023-08-27T23:56:07.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:07 smithi130 bash[27579]: audit 2023-08-27T23:56:06.750396+0000 mon.c (mon.2) 1350 : audit [DBG] from='client.? 172.21.15.6:0/2840539235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:07.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:07 smithi006 bash[27292]: audit 2023-08-27T23:56:06.750396+0000 mon.c (mon.2) 1350 : audit [DBG] from='client.? 172.21.15.6:0/2840539235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:07 smithi006 bash[33918]: audit 2023-08-27T23:56:06.750396+0000 mon.c (mon.2) 1350 : audit [DBG] from='client.? 172.21.15.6:0/2840539235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:08.247 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:07 smithi130 bash[51302]: debug 2023-08-27T23:56:07.974+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:07.976682+0000) 2023-08-27T23:56:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:08 smithi130 bash[27579]: cluster 2023-08-27T23:56:07.804567+0000 mgr.y (mgr.14901) 3527 : cluster [DBG] pgmap v3534: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:08.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:08 smithi006 bash[33918]: cluster 2023-08-27T23:56:07.804567+0000 mgr.y (mgr.14901) 3527 : cluster [DBG] pgmap v3534: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:08 smithi006 bash[27292]: cluster 2023-08-27T23:56:07.804567+0000 mgr.y (mgr.14901) 3527 : cluster [DBG] pgmap v3534: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:09.249 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:08 smithi130 bash[51302]: debug 2023-08-27T23:56:08.970+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:08.976979+0000) 2023-08-27T23:56:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:09 smithi130 bash[27579]: audit 2023-08-27T23:56:09.157075+0000 mon.c (mon.2) 1351 : audit [DBG] from='client.? 172.21.15.6:0/3560838236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:09.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:09 smithi006 bash[33918]: audit 2023-08-27T23:56:09.157075+0000 mon.c (mon.2) 1351 : audit [DBG] from='client.? 172.21.15.6:0/3560838236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:09 smithi006 bash[27292]: audit 2023-08-27T23:56:09.157075+0000 mon.c (mon.2) 1351 : audit [DBG] from='client.? 172.21.15.6:0/3560838236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:10.257 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:09 smithi130 bash[51302]: debug 2023-08-27T23:56:09.974+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:09.977250+0000) 2023-08-27T23:56:10.605 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:10 smithi006 bash[33918]: cluster 2023-08-27T23:56:09.805739+0000 mgr.y (mgr.14901) 3528 : cluster [DBG] pgmap v3535: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:10.605 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:10 smithi006 bash[27292]: cluster 2023-08-27T23:56:09.805739+0000 mgr.y (mgr.14901) 3528 : cluster [DBG] pgmap v3535: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:10.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:10 smithi130 bash[27579]: cluster 2023-08-27T23:56:09.805739+0000 mgr.y (mgr.14901) 3528 : cluster [DBG] pgmap v3535: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:10.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:56:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:56:10] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-27T23:56:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:56:10.602Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:56:10.604Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:11.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:10 smithi130 bash[51302]: debug 2023-08-27T23:56:10.974+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:10.977533+0000) 2023-08-27T23:56:11.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:11 smithi006 bash[27292]: audit 2023-08-27T23:56:11.572229+0000 mon.a (mon.0) 2749 : audit [DBG] from='client.? 172.21.15.6:0/627074916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:11.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:11 smithi006 bash[33918]: audit 2023-08-27T23:56:11.572229+0000 mon.a (mon.0) 2749 : audit [DBG] from='client.? 172.21.15.6:0/627074916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:11.979 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:11 smithi130 bash[27579]: audit 2023-08-27T23:56:11.572229+0000 mon.a (mon.0) 2749 : audit [DBG] from='client.? 172.21.15.6:0/627074916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:11 smithi130 bash[51302]: debug 2023-08-27T23:56:11.970+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:11.977837+0000) 2023-08-27T23:56:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:56:12.169Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:56:12.172Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:12 smithi006 bash[33918]: cluster 2023-08-27T23:56:11.806392+0000 mgr.y (mgr.14901) 3529 : cluster [DBG] pgmap v3536: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:12 smithi006 bash[27292]: cluster 2023-08-27T23:56:11.806392+0000 mgr.y (mgr.14901) 3529 : cluster [DBG] pgmap v3536: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:12.979 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:12 smithi130 bash[27579]: cluster 2023-08-27T23:56:11.806392+0000 mgr.y (mgr.14901) 3529 : cluster [DBG] pgmap v3536: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:12 smithi130 bash[51302]: debug 2023-08-27T23:56:12.974+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:12.978113+0000) 2023-08-27T23:56:14.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:13 smithi130 bash[51302]: debug 2023-08-27T23:56:13.974+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:13.978384+0000) 2023-08-27T23:56:14.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:14 smithi006 bash[56298]: level=error ts=2023-08-27T23:56:14.062Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:14 smithi006 bash[56298]: level=warn ts=2023-08-27T23:56:14.065Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:15.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:14 smithi130 bash[51302]: debug 2023-08-27T23:56:14.974+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:14.978702+0000) 2023-08-27T23:56:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:14 smithi130 bash[27579]: cluster 2023-08-27T23:56:13.807026+0000 mgr.y (mgr.14901) 3530 : cluster [DBG] pgmap v3537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:14 smithi130 bash[27579]: audit 2023-08-27T23:56:13.993189+0000 mon.c (mon.2) 1352 : audit [DBG] from='client.? 172.21.15.6:0/1518508706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:14 smithi006 bash[33918]: cluster 2023-08-27T23:56:13.807026+0000 mgr.y (mgr.14901) 3530 : cluster [DBG] pgmap v3537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:14 smithi006 bash[33918]: audit 2023-08-27T23:56:13.993189+0000 mon.c (mon.2) 1352 : audit [DBG] from='client.? 172.21.15.6:0/1518508706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:14 smithi006 bash[27292]: cluster 2023-08-27T23:56:13.807026+0000 mgr.y (mgr.14901) 3530 : cluster [DBG] pgmap v3537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:14 smithi006 bash[27292]: audit 2023-08-27T23:56:13.993189+0000 mon.c (mon.2) 1352 : audit [DBG] from='client.? 172.21.15.6:0/1518508706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:16.238 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:15 smithi130 bash[51302]: debug 2023-08-27T23:56:15.978+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:15.978969+0000) 2023-08-27T23:56:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:16 smithi130 bash[27579]: cluster 2023-08-27T23:56:15.808190+0000 mgr.y (mgr.14901) 3531 : cluster [DBG] pgmap v3538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:16.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:16 smithi006 bash[27292]: cluster 2023-08-27T23:56:15.808190+0000 mgr.y (mgr.14901) 3531 : cluster [DBG] pgmap v3538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:16 smithi006 bash[33918]: cluster 2023-08-27T23:56:15.808190+0000 mgr.y (mgr.14901) 3531 : cluster [DBG] pgmap v3538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:17.242 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:16 smithi130 bash[51302]: debug 2023-08-27T23:56:16.978+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:16.979232+0000) 2023-08-27T23:56:17.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:17 smithi130 bash[27579]: audit 2023-08-27T23:56:16.402903+0000 mon.c (mon.2) 1353 : audit [DBG] from='client.? 172.21.15.6:0/2470052287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:17.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:17 smithi006 bash[27292]: audit 2023-08-27T23:56:16.402903+0000 mon.c (mon.2) 1353 : audit [DBG] from='client.? 172.21.15.6:0/2470052287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:17.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:17 smithi006 bash[33918]: audit 2023-08-27T23:56:16.402903+0000 mon.c (mon.2) 1353 : audit [DBG] from='client.? 172.21.15.6:0/2470052287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:18.252 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:17 smithi130 bash[51302]: debug 2023-08-27T23:56:17.978+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:17.979523+0000) 2023-08-27T23:56:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:18 smithi130 bash[27579]: cluster 2023-08-27T23:56:17.808928+0000 mgr.y (mgr.14901) 3532 : cluster [DBG] pgmap v3539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:18.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:18 smithi006 bash[33918]: cluster 2023-08-27T23:56:17.808928+0000 mgr.y (mgr.14901) 3532 : cluster [DBG] pgmap v3539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:18.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:18 smithi006 bash[27292]: cluster 2023-08-27T23:56:17.808928+0000 mgr.y (mgr.14901) 3532 : cluster [DBG] pgmap v3539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:19.256 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:18 smithi130 bash[51302]: debug 2023-08-27T23:56:18.978+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:18.979811+0000) 2023-08-27T23:56:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:19 smithi130 bash[27579]: audit 2023-08-27T23:56:18.820717+0000 mon.a (mon.0) 2750 : audit [DBG] from='client.? 172.21.15.6:0/2521284904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:19 smithi130 bash[27579]: audit 2023-08-27T23:56:18.886765+0000 mon.b (mon.1) 451 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:56:19.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:19 smithi006 bash[33918]: audit 2023-08-27T23:56:18.820717+0000 mon.a (mon.0) 2750 : audit [DBG] from='client.? 172.21.15.6:0/2521284904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:19.744 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:19 smithi006 bash[33918]: audit 2023-08-27T23:56:18.886765+0000 mon.b (mon.1) 451 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:56:19.745 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:19 smithi006 bash[27292]: audit 2023-08-27T23:56:18.820717+0000 mon.a (mon.0) 2750 : audit [DBG] from='client.? 172.21.15.6:0/2521284904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:19.745 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:19 smithi006 bash[27292]: audit 2023-08-27T23:56:18.886765+0000 mon.b (mon.1) 451 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:56:20.264 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:19 smithi130 bash[51302]: debug 2023-08-27T23:56:19.978+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:19.980096+0000) 2023-08-27T23:56:20.605 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:20 smithi006 bash[27292]: cluster 2023-08-27T23:56:19.809994+0000 mgr.y (mgr.14901) 3533 : cluster [DBG] pgmap v3540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:20.605 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:20 smithi006 bash[33918]: cluster 2023-08-27T23:56:19.809994+0000 mgr.y (mgr.14901) 3533 : cluster [DBG] pgmap v3540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:20 smithi130 bash[27579]: cluster 2023-08-27T23:56:19.809994+0000 mgr.y (mgr.14901) 3533 : cluster [DBG] pgmap v3540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:20.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:56:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:56:20] "GET /metrics HTTP/1.1" 200 193113 "" "Prometheus/2.33.4" 2023-08-27T23:56:20.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:56:20.603Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:56:20.605Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:21.267 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:20 smithi130 bash[51302]: debug 2023-08-27T23:56:20.978+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:20.980359+0000) 2023-08-27T23:56:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:21 smithi130 bash[27579]: audit 2023-08-27T23:56:21.228087+0000 mon.c (mon.2) 1354 : audit [DBG] from='client.? 172.21.15.6:0/165604727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:21.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:21 smithi006 bash[33918]: audit 2023-08-27T23:56:21.228087+0000 mon.c (mon.2) 1354 : audit [DBG] from='client.? 172.21.15.6:0/165604727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:21.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:21 smithi006 bash[27292]: audit 2023-08-27T23:56:21.228087+0000 mon.c (mon.2) 1354 : audit [DBG] from='client.? 172.21.15.6:0/165604727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:22.276 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:21 smithi130 bash[51302]: debug 2023-08-27T23:56:21.978+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:21.980651+0000) 2023-08-27T23:56:22.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:22 smithi006 bash[33918]: cluster 2023-08-27T23:56:21.810849+0000 mgr.y (mgr.14901) 3534 : cluster [DBG] pgmap v3541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:22.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:22 smithi006 bash[27292]: cluster 2023-08-27T23:56:21.810849+0000 mgr.y (mgr.14901) 3534 : cluster [DBG] pgmap v3541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:56:22.170Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:56:22.172Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:22 smithi130 bash[27579]: cluster 2023-08-27T23:56:21.810849+0000 mgr.y (mgr.14901) 3534 : cluster [DBG] pgmap v3541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:23.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:22 smithi130 bash[51302]: debug 2023-08-27T23:56:22.978+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:22.980922+0000) 2023-08-27T23:56:23.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:23 smithi006 bash[33918]: audit 2023-08-27T23:56:23.643668+0000 mon.c (mon.2) 1355 : audit [DBG] from='client.? 172.21.15.6:0/1586235540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:23.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:23 smithi006 bash[27292]: audit 2023-08-27T23:56:23.643668+0000 mon.c (mon.2) 1355 : audit [DBG] from='client.? 172.21.15.6:0/1586235540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:23.982 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:23 smithi130 bash[27579]: audit 2023-08-27T23:56:23.643668+0000 mon.c (mon.2) 1355 : audit [DBG] from='client.? 172.21.15.6:0/1586235540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:23 smithi130 bash[51302]: debug 2023-08-27T23:56:23.978+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:23.981151+0000) 2023-08-27T23:56:24.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:24 smithi006 bash[56298]: level=error ts=2023-08-27T23:56:24.063Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:24 smithi006 bash[56298]: level=warn ts=2023-08-27T23:56:24.065Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:24.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:24 smithi006 bash[33918]: cluster 2023-08-27T23:56:23.811541+0000 mgr.y (mgr.14901) 3535 : cluster [DBG] pgmap v3542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:24 smithi006 bash[27292]: cluster 2023-08-27T23:56:23.811541+0000 mgr.y (mgr.14901) 3535 : cluster [DBG] pgmap v3542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:24.983 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:24 smithi130 bash[27579]: cluster 2023-08-27T23:56:23.811541+0000 mgr.y (mgr.14901) 3535 : cluster [DBG] pgmap v3542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:24 smithi130 bash[51302]: debug 2023-08-27T23:56:24.978+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:24.981377+0000) 2023-08-27T23:56:26.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:25 smithi130 bash[51302]: debug 2023-08-27T23:56:25.978+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:25.981677+0000) 2023-08-27T23:56:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:25 smithi130 bash[27579]: audit 2023-08-27T23:56:24.810004+0000 mon.a (mon.0) 2751 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:56:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:25 smithi130 bash[27579]: audit 2023-08-27T23:56:25.224693+0000 mon.a (mon.0) 2752 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:56:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:25 smithi130 bash[27579]: audit 2023-08-27T23:56:25.499048+0000 mon.b (mon.1) 452 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:56:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:25 smithi130 bash[27579]: audit 2023-08-27T23:56:25.501303+0000 mon.b (mon.1) 453 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:56:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:25 smithi130 bash[27579]: audit 2023-08-27T23:56:25.510168+0000 mon.a (mon.0) 2753 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:56:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:25 smithi006 bash[27292]: audit 2023-08-27T23:56:24.810004+0000 mon.a (mon.0) 2751 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:56:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:25 smithi006 bash[27292]: audit 2023-08-27T23:56:25.224693+0000 mon.a (mon.0) 2752 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:56:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:25 smithi006 bash[27292]: audit 2023-08-27T23:56:25.499048+0000 mon.b (mon.1) 452 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:56:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:25 smithi006 bash[27292]: audit 2023-08-27T23:56:25.501303+0000 mon.b (mon.1) 453 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:56:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:25 smithi006 bash[27292]: audit 2023-08-27T23:56:25.510168+0000 mon.a (mon.0) 2753 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:56:26.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:25 smithi006 bash[33918]: audit 2023-08-27T23:56:24.810004+0000 mon.a (mon.0) 2751 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:56:26.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:25 smithi006 bash[33918]: audit 2023-08-27T23:56:25.224693+0000 mon.a (mon.0) 2752 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:56:26.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:25 smithi006 bash[33918]: audit 2023-08-27T23:56:25.499048+0000 mon.b (mon.1) 452 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:56:26.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:25 smithi006 bash[33918]: audit 2023-08-27T23:56:25.501303+0000 mon.b (mon.1) 453 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:56:26.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:25 smithi006 bash[33918]: audit 2023-08-27T23:56:25.510168+0000 mon.a (mon.0) 2753 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:56:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:26 smithi130 bash[27579]: cluster 2023-08-27T23:56:25.812883+0000 mgr.y (mgr.14901) 3536 : cluster [DBG] pgmap v3543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:26 smithi130 bash[27579]: audit 2023-08-27T23:56:26.058024+0000 mon.a (mon.0) 2754 : audit [DBG] from='client.? 172.21.15.6:0/631494847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:27.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:26 smithi130 bash[51302]: debug 2023-08-27T23:56:26.978+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:26.981950+0000) 2023-08-27T23:56:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:26 smithi006 bash[33918]: cluster 2023-08-27T23:56:25.812883+0000 mgr.y (mgr.14901) 3536 : cluster [DBG] pgmap v3543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:26 smithi006 bash[33918]: audit 2023-08-27T23:56:26.058024+0000 mon.a (mon.0) 2754 : audit [DBG] from='client.? 172.21.15.6:0/631494847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:26 smithi006 bash[27292]: cluster 2023-08-27T23:56:25.812883+0000 mgr.y (mgr.14901) 3536 : cluster [DBG] pgmap v3543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:26 smithi006 bash[27292]: audit 2023-08-27T23:56:26.058024+0000 mon.a (mon.0) 2754 : audit [DBG] from='client.? 172.21.15.6:0/631494847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:28.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:27 smithi130 bash[51302]: debug 2023-08-27T23:56:27.978+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:27.982208+0000) 2023-08-27T23:56:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:28 smithi130 bash[27579]: cluster 2023-08-27T23:56:27.813501+0000 mgr.y (mgr.14901) 3537 : cluster [DBG] pgmap v3544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:28 smithi130 bash[27579]: audit 2023-08-27T23:56:28.482213+0000 mon.a (mon.0) 2755 : audit [DBG] from='client.? 172.21.15.6:0/2816398933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:29.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:28 smithi130 bash[51302]: debug 2023-08-27T23:56:28.978+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:28.982563+0000) 2023-08-27T23:56:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:28 smithi006 bash[33918]: cluster 2023-08-27T23:56:27.813501+0000 mgr.y (mgr.14901) 3537 : cluster [DBG] pgmap v3544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:28 smithi006 bash[33918]: audit 2023-08-27T23:56:28.482213+0000 mon.a (mon.0) 2755 : audit [DBG] from='client.? 172.21.15.6:0/2816398933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:28 smithi006 bash[27292]: cluster 2023-08-27T23:56:27.813501+0000 mgr.y (mgr.14901) 3537 : cluster [DBG] pgmap v3544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:28 smithi006 bash[27292]: audit 2023-08-27T23:56:28.482213+0000 mon.a (mon.0) 2755 : audit [DBG] from='client.? 172.21.15.6:0/2816398933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:29 smithi130 bash[51302]: debug 2023-08-27T23:56:29.982+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:29.982839+0000) 2023-08-27T23:56:30.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:56:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:56:30] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-27T23:56:30.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:30 smithi006 bash[33918]: cluster 2023-08-27T23:56:29.814644+0000 mgr.y (mgr.14901) 3538 : cluster [DBG] pgmap v3545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:30.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:30 smithi006 bash[27292]: cluster 2023-08-27T23:56:29.814644+0000 mgr.y (mgr.14901) 3538 : cluster [DBG] pgmap v3545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:56:30.603Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:56:30.605Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:31.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:30 smithi130 bash[27579]: cluster 2023-08-27T23:56:29.814644+0000 mgr.y (mgr.14901) 3538 : cluster [DBG] pgmap v3545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:31.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:30 smithi130 bash[51302]: debug 2023-08-27T23:56:30.982+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:30.983129+0000) 2023-08-27T23:56:32.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:31 smithi130 bash[27579]: audit 2023-08-27T23:56:30.901021+0000 mon.a (mon.0) 2756 : audit [DBG] from='client.? 172.21.15.6:0/1675017155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:32.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:31 smithi130 bash[51302]: debug 2023-08-27T23:56:31.982+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:31.983340+0000) 2023-08-27T23:56:32.174 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:31 smithi006 bash[27292]: audit 2023-08-27T23:56:30.901021+0000 mon.a (mon.0) 2756 : audit [DBG] from='client.? 172.21.15.6:0/1675017155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:32.174 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:31 smithi006 bash[33918]: audit 2023-08-27T23:56:30.901021+0000 mon.a (mon.0) 2756 : audit [DBG] from='client.? 172.21.15.6:0/1675017155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:56:32.171Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:56:32.173Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:32 smithi130 bash[27579]: cluster 2023-08-27T23:56:31.815307+0000 mgr.y (mgr.14901) 3539 : cluster [DBG] pgmap v3546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:33.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:32 smithi130 bash[51302]: debug 2023-08-27T23:56:32.982+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:32.983649+0000) 2023-08-27T23:56:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:32 smithi006 bash[27292]: cluster 2023-08-27T23:56:31.815307+0000 mgr.y (mgr.14901) 3539 : cluster [DBG] pgmap v3546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:32 smithi006 bash[33918]: cluster 2023-08-27T23:56:31.815307+0000 mgr.y (mgr.14901) 3539 : cluster [DBG] pgmap v3546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:33 smithi130 bash[27579]: audit 2023-08-27T23:56:33.317332+0000 mon.a (mon.0) 2757 : audit [DBG] from='client.? 172.21.15.6:0/1255794250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:34.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:33 smithi130 bash[51302]: debug 2023-08-27T23:56:33.982+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:33.983964+0000) 2023-08-27T23:56:34.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:33 smithi006 bash[27292]: audit 2023-08-27T23:56:33.317332+0000 mon.a (mon.0) 2757 : audit [DBG] from='client.? 172.21.15.6:0/1255794250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:34.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:33 smithi006 bash[33918]: audit 2023-08-27T23:56:33.317332+0000 mon.a (mon.0) 2757 : audit [DBG] from='client.? 172.21.15.6:0/1255794250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:34 smithi006 bash[56298]: level=error ts=2023-08-27T23:56:34.063Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:34 smithi006 bash[56298]: level=warn ts=2023-08-27T23:56:34.065Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:35.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:34 smithi130 bash[27579]: cluster 2023-08-27T23:56:33.815936+0000 mgr.y (mgr.14901) 3540 : cluster [DBG] pgmap v3547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:35.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:34 smithi130 bash[51302]: debug 2023-08-27T23:56:34.982+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:34.984264+0000) 2023-08-27T23:56:35.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:34 smithi006 bash[33918]: cluster 2023-08-27T23:56:33.815936+0000 mgr.y (mgr.14901) 3540 : cluster [DBG] pgmap v3547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:34 smithi006 bash[27292]: cluster 2023-08-27T23:56:33.815936+0000 mgr.y (mgr.14901) 3540 : cluster [DBG] pgmap v3547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:36.243 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:35 smithi130 bash[51302]: debug 2023-08-27T23:56:35.982+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:35.984556+0000) 2023-08-27T23:56:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:36 smithi130 bash[27579]: audit 2023-08-27T23:56:35.738548+0000 mon.c (mon.2) 1356 : audit [DBG] from='client.? 172.21.15.6:0/3978213076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:36 smithi130 bash[27579]: cluster 2023-08-27T23:56:35.817062+0000 mgr.y (mgr.14901) 3541 : cluster [DBG] pgmap v3548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:36.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:36 smithi006 bash[27292]: audit 2023-08-27T23:56:35.738548+0000 mon.c (mon.2) 1356 : audit [DBG] from='client.? 172.21.15.6:0/3978213076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:36 smithi006 bash[27292]: cluster 2023-08-27T23:56:35.817062+0000 mgr.y (mgr.14901) 3541 : cluster [DBG] pgmap v3548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:36 smithi006 bash[33918]: audit 2023-08-27T23:56:35.738548+0000 mon.c (mon.2) 1356 : audit [DBG] from='client.? 172.21.15.6:0/3978213076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:36 smithi006 bash[33918]: cluster 2023-08-27T23:56:35.817062+0000 mgr.y (mgr.14901) 3541 : cluster [DBG] pgmap v3548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:36 smithi130 bash[51302]: debug 2023-08-27T23:56:36.982+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:36.984853+0000) 2023-08-27T23:56:38.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:37 smithi130 bash[51302]: debug 2023-08-27T23:56:37.982+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:37.985157+0000) 2023-08-27T23:56:39.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:38 smithi006 bash[33918]: cluster 2023-08-27T23:56:37.817850+0000 mgr.y (mgr.14901) 3542 : cluster [DBG] pgmap v3549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:39.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:38 smithi006 bash[33918]: audit 2023-08-27T23:56:38.139932+0000 mon.c (mon.2) 1357 : audit [DBG] from='client.? 172.21.15.6:0/2237917664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:39.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:38 smithi006 bash[27292]: cluster 2023-08-27T23:56:37.817850+0000 mgr.y (mgr.14901) 3542 : cluster [DBG] pgmap v3549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:39.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:38 smithi006 bash[27292]: audit 2023-08-27T23:56:38.139932+0000 mon.c (mon.2) 1357 : audit [DBG] from='client.? 172.21.15.6:0/2237917664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:38 smithi130 bash[27579]: cluster 2023-08-27T23:56:37.817850+0000 mgr.y (mgr.14901) 3542 : cluster [DBG] pgmap v3549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:38 smithi130 bash[27579]: audit 2023-08-27T23:56:38.139932+0000 mon.c (mon.2) 1357 : audit [DBG] from='client.? 172.21.15.6:0/2237917664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:39.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:38 smithi130 bash[51302]: debug 2023-08-27T23:56:38.982+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:38.985459+0000) 2023-08-27T23:56:40.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:39 smithi130 bash[51302]: debug 2023-08-27T23:56:39.982+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:39.985752+0000) 2023-08-27T23:56:40.882 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:56:40.605Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:40.882 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:56:40.607Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:40.883 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:56:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:56:40] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-27T23:56:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:40 smithi006 bash[27292]: cluster 2023-08-27T23:56:39.819128+0000 mgr.y (mgr.14901) 3543 : cluster [DBG] pgmap v3550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:40 smithi006 bash[27292]: audit 2023-08-27T23:56:40.564248+0000 mon.a (mon.0) 2758 : audit [DBG] from='client.? 172.21.15.6:0/1504986865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:41.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:40 smithi006 bash[33918]: cluster 2023-08-27T23:56:39.819128+0000 mgr.y (mgr.14901) 3543 : cluster [DBG] pgmap v3550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:41.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:40 smithi006 bash[33918]: audit 2023-08-27T23:56:40.564248+0000 mon.a (mon.0) 2758 : audit [DBG] from='client.? 172.21.15.6:0/1504986865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:41.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:40 smithi130 bash[27579]: cluster 2023-08-27T23:56:39.819128+0000 mgr.y (mgr.14901) 3543 : cluster [DBG] pgmap v3550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:41.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:40 smithi130 bash[27579]: audit 2023-08-27T23:56:40.564248+0000 mon.a (mon.0) 2758 : audit [DBG] from='client.? 172.21.15.6:0/1504986865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:41.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:40 smithi130 bash[51302]: debug 2023-08-27T23:56:40.982+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:40.986046+0000) 2023-08-27T23:56:42.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:41 smithi130 bash[51302]: debug 2023-08-27T23:56:41.982+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:41.986323+0000) 2023-08-27T23:56:42.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:56:42.172Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:42.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:56:42.174Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:42 smithi006 bash[33918]: cluster 2023-08-27T23:56:41.819946+0000 mgr.y (mgr.14901) 3544 : cluster [DBG] pgmap v3551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:42 smithi006 bash[27292]: cluster 2023-08-27T23:56:41.819946+0000 mgr.y (mgr.14901) 3544 : cluster [DBG] pgmap v3551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:43.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:42 smithi130 bash[27579]: cluster 2023-08-27T23:56:41.819946+0000 mgr.y (mgr.14901) 3544 : cluster [DBG] pgmap v3551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:43.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:42 smithi130 bash[51302]: debug 2023-08-27T23:56:42.982+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:42.986666+0000) 2023-08-27T23:56:44.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:43 smithi006 bash[33918]: audit 2023-08-27T23:56:42.976637+0000 mon.c (mon.2) 1358 : audit [DBG] from='client.? 172.21.15.6:0/3134071968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:44 smithi006 bash[56298]: level=error ts=2023-08-27T23:56:44.064Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:44 smithi006 bash[56298]: level=warn ts=2023-08-27T23:56:44.067Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:44.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:43 smithi006 bash[27292]: audit 2023-08-27T23:56:42.976637+0000 mon.c (mon.2) 1358 : audit [DBG] from='client.? 172.21.15.6:0/3134071968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:43 smithi130 bash[27579]: audit 2023-08-27T23:56:42.976637+0000 mon.c (mon.2) 1358 : audit [DBG] from='client.? 172.21.15.6:0/3134071968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:44.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:43 smithi130 bash[51302]: debug 2023-08-27T23:56:43.986+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:43.986965+0000) 2023-08-27T23:56:45.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:44 smithi006 bash[27292]: cluster 2023-08-27T23:56:43.820599+0000 mgr.y (mgr.14901) 3545 : cluster [DBG] pgmap v3552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:44 smithi006 bash[33918]: cluster 2023-08-27T23:56:43.820599+0000 mgr.y (mgr.14901) 3545 : cluster [DBG] pgmap v3552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:44 smithi130 bash[27579]: cluster 2023-08-27T23:56:43.820599+0000 mgr.y (mgr.14901) 3545 : cluster [DBG] pgmap v3552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:44 smithi130 bash[51302]: debug 2023-08-27T23:56:44.986+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:44.987241+0000) 2023-08-27T23:56:46.243 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:45 smithi130 bash[51302]: debug 2023-08-27T23:56:45.986+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:45.987517+0000) 2023-08-27T23:56:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:46 smithi130 bash[27579]: audit 2023-08-27T23:56:45.389653+0000 mon.a (mon.0) 2759 : audit [DBG] from='client.? 172.21.15.6:0/3278266914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:46 smithi130 bash[27579]: cluster 2023-08-27T23:56:45.821729+0000 mgr.y (mgr.14901) 3546 : cluster [DBG] pgmap v3553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:46 smithi006 bash[27292]: audit 2023-08-27T23:56:45.389653+0000 mon.a (mon.0) 2759 : audit [DBG] from='client.? 172.21.15.6:0/3278266914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:46 smithi006 bash[27292]: cluster 2023-08-27T23:56:45.821729+0000 mgr.y (mgr.14901) 3546 : cluster [DBG] pgmap v3553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:46 smithi006 bash[33918]: audit 2023-08-27T23:56:45.389653+0000 mon.a (mon.0) 2759 : audit [DBG] from='client.? 172.21.15.6:0/3278266914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:46 smithi006 bash[33918]: cluster 2023-08-27T23:56:45.821729+0000 mgr.y (mgr.14901) 3546 : cluster [DBG] pgmap v3553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:46 smithi130 bash[51302]: debug 2023-08-27T23:56:46.986+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:46.987821+0000) 2023-08-27T23:56:48.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:47 smithi130 bash[27579]: audit 2023-08-27T23:56:47.802016+0000 mon.a (mon.0) 2760 : audit [DBG] from='client.? 172.21.15.6:0/597551453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:48.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:47 smithi130 bash[51302]: debug 2023-08-27T23:56:47.986+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:47.988091+0000) 2023-08-27T23:56:48.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:47 smithi006 bash[27292]: audit 2023-08-27T23:56:47.802016+0000 mon.a (mon.0) 2760 : audit [DBG] from='client.? 172.21.15.6:0/597551453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:48.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:47 smithi006 bash[33918]: audit 2023-08-27T23:56:47.802016+0000 mon.a (mon.0) 2760 : audit [DBG] from='client.? 172.21.15.6:0/597551453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:49.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:48 smithi006 bash[33918]: cluster 2023-08-27T23:56:47.822178+0000 mgr.y (mgr.14901) 3547 : cluster [DBG] pgmap v3554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:49.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:48 smithi006 bash[27292]: cluster 2023-08-27T23:56:47.822178+0000 mgr.y (mgr.14901) 3547 : cluster [DBG] pgmap v3554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:49.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:48 smithi130 bash[27579]: cluster 2023-08-27T23:56:47.822178+0000 mgr.y (mgr.14901) 3547 : cluster [DBG] pgmap v3554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:48 smithi130 bash[51302]: debug 2023-08-27T23:56:48.986+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:48.988352+0000) 2023-08-27T23:56:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:49 smithi130 bash[51302]: debug 2023-08-27T23:56:49.986+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:49.988630+0000) 2023-08-27T23:56:50.886 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:56:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:56:50] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:56:50.887 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:56:50.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:50.887 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:56:50.607Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:51.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:50 smithi006 bash[27292]: cluster 2023-08-27T23:56:49.823251+0000 mgr.y (mgr.14901) 3548 : cluster [DBG] pgmap v3555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:50 smithi006 bash[27292]: audit 2023-08-27T23:56:50.212651+0000 mon.c (mon.2) 1359 : audit [DBG] from='client.? 172.21.15.6:0/2425015400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:50 smithi006 bash[33918]: cluster 2023-08-27T23:56:49.823251+0000 mgr.y (mgr.14901) 3548 : cluster [DBG] pgmap v3555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:50 smithi006 bash[33918]: audit 2023-08-27T23:56:50.212651+0000 mon.c (mon.2) 1359 : audit [DBG] from='client.? 172.21.15.6:0/2425015400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:51.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:50 smithi130 bash[27579]: cluster 2023-08-27T23:56:49.823251+0000 mgr.y (mgr.14901) 3548 : cluster [DBG] pgmap v3555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:51.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:50 smithi130 bash[27579]: audit 2023-08-27T23:56:50.212651+0000 mon.c (mon.2) 1359 : audit [DBG] from='client.? 172.21.15.6:0/2425015400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:51.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:50 smithi130 bash[51302]: debug 2023-08-27T23:56:50.986+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:50.988939+0000) 2023-08-27T23:56:52.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:51 smithi130 bash[51302]: debug 2023-08-27T23:56:51.986+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:51.989239+0000) 2023-08-27T23:56:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:56:52.174Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:56:52.176Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:53.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:52 smithi006 bash[33918]: cluster 2023-08-27T23:56:51.823916+0000 mgr.y (mgr.14901) 3549 : cluster [DBG] pgmap v3556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:53.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:52 smithi006 bash[33918]: audit 2023-08-27T23:56:52.619278+0000 mon.c (mon.2) 1360 : audit [DBG] from='client.? 172.21.15.6:0/2776437050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:52 smithi006 bash[27292]: cluster 2023-08-27T23:56:51.823916+0000 mgr.y (mgr.14901) 3549 : cluster [DBG] pgmap v3556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:52 smithi006 bash[27292]: audit 2023-08-27T23:56:52.619278+0000 mon.c (mon.2) 1360 : audit [DBG] from='client.? 172.21.15.6:0/2776437050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:52 smithi130 bash[27579]: cluster 2023-08-27T23:56:51.823916+0000 mgr.y (mgr.14901) 3549 : cluster [DBG] pgmap v3556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:52 smithi130 bash[27579]: audit 2023-08-27T23:56:52.619278+0000 mon.c (mon.2) 1360 : audit [DBG] from='client.? 172.21.15.6:0/2776437050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:53.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:52 smithi130 bash[51302]: debug 2023-08-27T23:56:52.986+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:52.989497+0000) 2023-08-27T23:56:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:53 smithi130 bash[51302]: debug 2023-08-27T23:56:53.986+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:53.989774+0000) 2023-08-27T23:56:54.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:54 smithi006 bash[56298]: level=error ts=2023-08-27T23:56:54.065Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:56:54 smithi006 bash[56298]: level=warn ts=2023-08-27T23:56:54.067Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:56:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:54 smithi006 bash[33918]: cluster 2023-08-27T23:56:53.824576+0000 mgr.y (mgr.14901) 3550 : cluster [DBG] pgmap v3557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:54 smithi006 bash[27292]: cluster 2023-08-27T23:56:53.824576+0000 mgr.y (mgr.14901) 3550 : cluster [DBG] pgmap v3557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:54 smithi130 bash[27579]: cluster 2023-08-27T23:56:53.824576+0000 mgr.y (mgr.14901) 3550 : cluster [DBG] pgmap v3557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:54 smithi130 bash[51302]: debug 2023-08-27T23:56:54.986+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:54.990125+0000) 2023-08-27T23:56:56.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:55 smithi006 bash[33918]: audit 2023-08-27T23:56:55.035446+0000 mon.a (mon.0) 2761 : audit [DBG] from='client.? 172.21.15.6:0/3877658567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:56.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:55 smithi006 bash[27292]: audit 2023-08-27T23:56:55.035446+0000 mon.a (mon.0) 2761 : audit [DBG] from='client.? 172.21.15.6:0/3877658567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:56.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:55 smithi130 bash[51302]: debug 2023-08-27T23:56:55.986+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:55.990416+0000) 2023-08-27T23:56:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:55 smithi130 bash[27579]: audit 2023-08-27T23:56:55.035446+0000 mon.a (mon.0) 2761 : audit [DBG] from='client.? 172.21.15.6:0/3877658567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:57.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:56 smithi006 bash[27292]: cluster 2023-08-27T23:56:55.825708+0000 mgr.y (mgr.14901) 3551 : cluster [DBG] pgmap v3558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:57.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:56 smithi006 bash[33918]: cluster 2023-08-27T23:56:55.825708+0000 mgr.y (mgr.14901) 3551 : cluster [DBG] pgmap v3558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:57.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:56 smithi130 bash[27579]: cluster 2023-08-27T23:56:55.825708+0000 mgr.y (mgr.14901) 3551 : cluster [DBG] pgmap v3558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:57.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:56 smithi130 bash[51302]: debug 2023-08-27T23:56:56.986+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:56.990727+0000) 2023-08-27T23:56:58.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:57 smithi006 bash[33918]: audit 2023-08-27T23:56:57.454926+0000 mon.c (mon.2) 1361 : audit [DBG] from='client.? 172.21.15.6:0/1847453790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:58.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:57 smithi006 bash[27292]: audit 2023-08-27T23:56:57.454926+0000 mon.c (mon.2) 1361 : audit [DBG] from='client.? 172.21.15.6:0/1847453790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:58.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:57 smithi130 bash[51302]: debug 2023-08-27T23:56:57.990+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:57.991012+0000) 2023-08-27T23:56:58.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:57 smithi130 bash[27579]: audit 2023-08-27T23:56:57.454926+0000 mon.c (mon.2) 1361 : audit [DBG] from='client.? 172.21.15.6:0/1847453790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:59.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:58 smithi006 bash[27292]: cluster 2023-08-27T23:56:57.826334+0000 mgr.y (mgr.14901) 3552 : cluster [DBG] pgmap v3559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:58 smithi006 bash[33918]: cluster 2023-08-27T23:56:57.826334+0000 mgr.y (mgr.14901) 3552 : cluster [DBG] pgmap v3559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:56:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:58 smithi130 bash[51302]: debug 2023-08-27T23:56:58.990+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:58.991236+0000) 2023-08-27T23:56:59.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:58 smithi130 bash[27579]: cluster 2023-08-27T23:56:57.826334+0000 mgr.y (mgr.14901) 3552 : cluster [DBG] pgmap v3559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:00.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:56:59 smithi006 bash[27292]: audit 2023-08-27T23:56:59.868296+0000 mon.c (mon.2) 1362 : audit [DBG] from='client.? 172.21.15.6:0/381617695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:00.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:56:59 smithi006 bash[33918]: audit 2023-08-27T23:56:59.868296+0000 mon.c (mon.2) 1362 : audit [DBG] from='client.? 172.21.15.6:0/381617695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:00.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:56:59 smithi130 bash[27579]: audit 2023-08-27T23:56:59.868296+0000 mon.c (mon.2) 1362 : audit [DBG] from='client.? 172.21.15.6:0/381617695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:00.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:56:59 smithi130 bash[51302]: debug 2023-08-27T23:56:59.990+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:59.991463+0000) 2023-08-27T23:57:00.924 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:57:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:57:00] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:57:00.924 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:57:00.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:00.924 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:57:00.608Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:01.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:00 smithi006 bash[27292]: cluster 2023-08-27T23:56:59.827381+0000 mgr.y (mgr.14901) 3553 : cluster [DBG] pgmap v3560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:00 smithi006 bash[33918]: cluster 2023-08-27T23:56:59.827381+0000 mgr.y (mgr.14901) 3553 : cluster [DBG] pgmap v3560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:01.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:00 smithi130 bash[27579]: cluster 2023-08-27T23:56:59.827381+0000 mgr.y (mgr.14901) 3553 : cluster [DBG] pgmap v3560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:01.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:00 smithi130 bash[51302]: debug 2023-08-27T23:57:00.990+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:00.991745+0000) 2023-08-27T23:57:02.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:01 smithi130 bash[51302]: debug 2023-08-27T23:57:01.990+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:01.992008+0000) 2023-08-27T23:57:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:57:02.175Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:57:02.176Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:03.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:02 smithi006 bash[33918]: cluster 2023-08-27T23:57:01.828025+0000 mgr.y (mgr.14901) 3554 : cluster [DBG] pgmap v3561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:02 smithi006 bash[33918]: audit 2023-08-27T23:57:02.284532+0000 mon.a (mon.0) 2762 : audit [DBG] from='client.? 172.21.15.6:0/1852094802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:02 smithi006 bash[27292]: cluster 2023-08-27T23:57:01.828025+0000 mgr.y (mgr.14901) 3554 : cluster [DBG] pgmap v3561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:02 smithi006 bash[27292]: audit 2023-08-27T23:57:02.284532+0000 mon.a (mon.0) 2762 : audit [DBG] from='client.? 172.21.15.6:0/1852094802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:03.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:02 smithi130 bash[27579]: cluster 2023-08-27T23:57:01.828025+0000 mgr.y (mgr.14901) 3554 : cluster [DBG] pgmap v3561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:03.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:02 smithi130 bash[27579]: audit 2023-08-27T23:57:02.284532+0000 mon.a (mon.0) 2762 : audit [DBG] from='client.? 172.21.15.6:0/1852094802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:02 smithi130 bash[51302]: debug 2023-08-27T23:57:02.989+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:02.992296+0000) 2023-08-27T23:57:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:03 smithi130 bash[51302]: debug 2023-08-27T23:57:03.989+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:03.992584+0000) 2023-08-27T23:57:04.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:04 smithi006 bash[56298]: level=error ts=2023-08-27T23:57:04.066Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:04 smithi006 bash[56298]: level=warn ts=2023-08-27T23:57:04.068Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:05.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:04 smithi006 bash[33918]: cluster 2023-08-27T23:57:03.828663+0000 mgr.y (mgr.14901) 3555 : cluster [DBG] pgmap v3562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:04 smithi006 bash[33918]: audit 2023-08-27T23:57:04.699192+0000 mon.a (mon.0) 2763 : audit [DBG] from='client.? 172.21.15.6:0/33826848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:04 smithi006 bash[27292]: cluster 2023-08-27T23:57:03.828663+0000 mgr.y (mgr.14901) 3555 : cluster [DBG] pgmap v3562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:04 smithi006 bash[27292]: audit 2023-08-27T23:57:04.699192+0000 mon.a (mon.0) 2763 : audit [DBG] from='client.? 172.21.15.6:0/33826848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:04 smithi130 bash[51302]: debug 2023-08-27T23:57:04.989+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:04.992924+0000) 2023-08-27T23:57:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:04 smithi130 bash[27579]: cluster 2023-08-27T23:57:03.828663+0000 mgr.y (mgr.14901) 3555 : cluster [DBG] pgmap v3562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:04 smithi130 bash[27579]: audit 2023-08-27T23:57:04.699192+0000 mon.a (mon.0) 2763 : audit [DBG] from='client.? 172.21.15.6:0/33826848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:06.250 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:05 smithi130 bash[51302]: debug 2023-08-27T23:57:05.989+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:05.993187+0000) 2023-08-27T23:57:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:06 smithi130 bash[27579]: cluster 2023-08-27T23:57:05.829797+0000 mgr.y (mgr.14901) 3556 : cluster [DBG] pgmap v3563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:06.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:06 smithi006 bash[27292]: cluster 2023-08-27T23:57:05.829797+0000 mgr.y (mgr.14901) 3556 : cluster [DBG] pgmap v3563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:06 smithi006 bash[33918]: cluster 2023-08-27T23:57:05.829797+0000 mgr.y (mgr.14901) 3556 : cluster [DBG] pgmap v3563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:07.253 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:06 smithi130 bash[51302]: debug 2023-08-27T23:57:06.989+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:06.993457+0000) 2023-08-27T23:57:07.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:07 smithi130 bash[27579]: audit 2023-08-27T23:57:07.115282+0000 mon.c (mon.2) 1363 : audit [DBG] from='client.? 172.21.15.6:0/3171821613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:07.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:07 smithi006 bash[27292]: audit 2023-08-27T23:57:07.115282+0000 mon.c (mon.2) 1363 : audit [DBG] from='client.? 172.21.15.6:0/3171821613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:07 smithi006 bash[33918]: audit 2023-08-27T23:57:07.115282+0000 mon.c (mon.2) 1363 : audit [DBG] from='client.? 172.21.15.6:0/3171821613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:08.263 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:07 smithi130 bash[51302]: debug 2023-08-27T23:57:07.989+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:07.993745+0000) 2023-08-27T23:57:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:08 smithi130 bash[27579]: cluster 2023-08-27T23:57:07.830447+0000 mgr.y (mgr.14901) 3557 : cluster [DBG] pgmap v3564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:08.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:08 smithi006 bash[27292]: cluster 2023-08-27T23:57:07.830447+0000 mgr.y (mgr.14901) 3557 : cluster [DBG] pgmap v3564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:08 smithi006 bash[33918]: cluster 2023-08-27T23:57:07.830447+0000 mgr.y (mgr.14901) 3557 : cluster [DBG] pgmap v3564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:09.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:08 smithi130 bash[51302]: debug 2023-08-27T23:57:08.989+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:08.994066+0000) 2023-08-27T23:57:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:09 smithi130 bash[27579]: audit 2023-08-27T23:57:09.537502+0000 mon.a (mon.0) 2764 : audit [DBG] from='client.? 172.21.15.6:0/2492698435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:09.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:09 smithi006 bash[33918]: audit 2023-08-27T23:57:09.537502+0000 mon.a (mon.0) 2764 : audit [DBG] from='client.? 172.21.15.6:0/2492698435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:09.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:09 smithi006 bash[27292]: audit 2023-08-27T23:57:09.537502+0000 mon.a (mon.0) 2764 : audit [DBG] from='client.? 172.21.15.6:0/2492698435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:10.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:09 smithi130 bash[51302]: debug 2023-08-27T23:57:09.993+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:09.994373+0000) 2023-08-27T23:57:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:10 smithi130 bash[27579]: cluster 2023-08-27T23:57:09.831712+0000 mgr.y (mgr.14901) 3558 : cluster [DBG] pgmap v3565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:10.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:10 smithi006 bash[27292]: cluster 2023-08-27T23:57:09.831712+0000 mgr.y (mgr.14901) 3558 : cluster [DBG] pgmap v3565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:57:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:57:10] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:57:10.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:10 smithi006 bash[33918]: cluster 2023-08-27T23:57:09.831712+0000 mgr.y (mgr.14901) 3558 : cluster [DBG] pgmap v3565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:57:10.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:57:10.609Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:11.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:10 smithi130 bash[51302]: debug 2023-08-27T23:57:10.993+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:10.994654+0000) 2023-08-27T23:57:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:11 smithi130 bash[51302]: debug 2023-08-27T23:57:11.993+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:11.994987+0000) 2023-08-27T23:57:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:57:12.176Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:57:12.178Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:13.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:12 smithi006 bash[27292]: cluster 2023-08-27T23:57:11.832493+0000 mgr.y (mgr.14901) 3559 : cluster [DBG] pgmap v3566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:12 smithi006 bash[27292]: audit 2023-08-27T23:57:11.956075+0000 mon.a (mon.0) 2765 : audit [DBG] from='client.? 172.21.15.6:0/1807526287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:12 smithi006 bash[33918]: cluster 2023-08-27T23:57:11.832493+0000 mgr.y (mgr.14901) 3559 : cluster [DBG] pgmap v3566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:12 smithi006 bash[33918]: audit 2023-08-27T23:57:11.956075+0000 mon.a (mon.0) 2765 : audit [DBG] from='client.? 172.21.15.6:0/1807526287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:12 smithi130 bash[27579]: cluster 2023-08-27T23:57:11.832493+0000 mgr.y (mgr.14901) 3559 : cluster [DBG] pgmap v3566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:12 smithi130 bash[27579]: audit 2023-08-27T23:57:11.956075+0000 mon.a (mon.0) 2765 : audit [DBG] from='client.? 172.21.15.6:0/1807526287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:12 smithi130 bash[51302]: debug 2023-08-27T23:57:12.993+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:12.995210+0000) 2023-08-27T23:57:14.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:13 smithi130 bash[51302]: debug 2023-08-27T23:57:13.993+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:13.995514+0000) 2023-08-27T23:57:14.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:14 smithi006 bash[56298]: level=error ts=2023-08-27T23:57:14.066Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:14 smithi006 bash[56298]: level=warn ts=2023-08-27T23:57:14.068Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:14 smithi006 bash[27292]: cluster 2023-08-27T23:57:13.833292+0000 mgr.y (mgr.14901) 3560 : cluster [DBG] pgmap v3567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:14 smithi006 bash[27292]: audit 2023-08-27T23:57:14.376428+0000 mon.a (mon.0) 2766 : audit [DBG] from='client.? 172.21.15.6:0/4288696334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:14 smithi006 bash[33918]: cluster 2023-08-27T23:57:13.833292+0000 mgr.y (mgr.14901) 3560 : cluster [DBG] pgmap v3567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:14 smithi006 bash[33918]: audit 2023-08-27T23:57:14.376428+0000 mon.a (mon.0) 2766 : audit [DBG] from='client.? 172.21.15.6:0/4288696334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:14 smithi130 bash[27579]: cluster 2023-08-27T23:57:13.833292+0000 mgr.y (mgr.14901) 3560 : cluster [DBG] pgmap v3567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:14 smithi130 bash[27579]: audit 2023-08-27T23:57:14.376428+0000 mon.a (mon.0) 2766 : audit [DBG] from='client.? 172.21.15.6:0/4288696334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:14 smithi130 bash[51302]: debug 2023-08-27T23:57:14.993+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:14.995799+0000) 2023-08-27T23:57:16.250 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:15 smithi130 bash[51302]: debug 2023-08-27T23:57:15.993+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:15.996068+0000) 2023-08-27T23:57:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:16 smithi130 bash[27579]: cluster 2023-08-27T23:57:15.834477+0000 mgr.y (mgr.14901) 3561 : cluster [DBG] pgmap v3568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:16.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:16 smithi006 bash[27292]: cluster 2023-08-27T23:57:15.834477+0000 mgr.y (mgr.14901) 3561 : cluster [DBG] pgmap v3568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:16 smithi006 bash[33918]: cluster 2023-08-27T23:57:15.834477+0000 mgr.y (mgr.14901) 3561 : cluster [DBG] pgmap v3568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:17.253 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:16 smithi130 bash[51302]: debug 2023-08-27T23:57:16.993+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:16.996277+0000) 2023-08-27T23:57:17.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:17 smithi130 bash[27579]: audit 2023-08-27T23:57:16.796931+0000 mon.a (mon.0) 2767 : audit [DBG] from='client.? 172.21.15.6:0/3014343910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:17.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:17 smithi006 bash[27292]: audit 2023-08-27T23:57:16.796931+0000 mon.a (mon.0) 2767 : audit [DBG] from='client.? 172.21.15.6:0/3014343910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:17.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:17 smithi006 bash[33918]: audit 2023-08-27T23:57:16.796931+0000 mon.a (mon.0) 2767 : audit [DBG] from='client.? 172.21.15.6:0/3014343910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:18.261 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:17 smithi130 bash[51302]: debug 2023-08-27T23:57:17.993+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:17.996551+0000) 2023-08-27T23:57:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:18 smithi130 bash[27579]: cluster 2023-08-27T23:57:17.835162+0000 mgr.y (mgr.14901) 3562 : cluster [DBG] pgmap v3569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:18.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:18 smithi006 bash[33918]: cluster 2023-08-27T23:57:17.835162+0000 mgr.y (mgr.14901) 3562 : cluster [DBG] pgmap v3569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:18.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:18 smithi006 bash[27292]: cluster 2023-08-27T23:57:17.835162+0000 mgr.y (mgr.14901) 3562 : cluster [DBG] pgmap v3569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:19.264 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:18 smithi130 bash[51302]: debug 2023-08-27T23:57:18.993+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:18.996876+0000) 2023-08-27T23:57:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:19 smithi130 bash[27579]: audit 2023-08-27T23:57:19.207994+0000 mon.a (mon.0) 2768 : audit [DBG] from='client.? 172.21.15.6:0/2333154805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:19.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:19 smithi006 bash[27292]: audit 2023-08-27T23:57:19.207994+0000 mon.a (mon.0) 2768 : audit [DBG] from='client.? 172.21.15.6:0/2333154805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:19.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:19 smithi006 bash[33918]: audit 2023-08-27T23:57:19.207994+0000 mon.a (mon.0) 2768 : audit [DBG] from='client.? 172.21.15.6:0/2333154805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:20.275 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:19 smithi130 bash[51302]: debug 2023-08-27T23:57:19.993+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:19.997147+0000) 2023-08-27T23:57:20.609 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:20 smithi006 bash[33918]: cluster 2023-08-27T23:57:19.836268+0000 mgr.y (mgr.14901) 3563 : cluster [DBG] pgmap v3570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:20.609 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:20 smithi006 bash[27292]: cluster 2023-08-27T23:57:19.836268+0000 mgr.y (mgr.14901) 3563 : cluster [DBG] pgmap v3570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:20 smithi130 bash[27579]: cluster 2023-08-27T23:57:19.836268+0000 mgr.y (mgr.14901) 3563 : cluster [DBG] pgmap v3570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:20.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:57:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:57:20] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-27T23:57:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:57:20.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:57:20.609Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:21.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:20 smithi130 bash[51302]: debug 2023-08-27T23:57:20.993+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:20.997417+0000) 2023-08-27T23:57:21.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:21 smithi006 bash[27292]: audit 2023-08-27T23:57:21.624499+0000 mon.c (mon.2) 1364 : audit [DBG] from='client.? 172.21.15.6:0/1953807321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:21.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:21 smithi006 bash[33918]: audit 2023-08-27T23:57:21.624499+0000 mon.c (mon.2) 1364 : audit [DBG] from='client.? 172.21.15.6:0/1953807321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:21.998 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:21 smithi130 bash[27579]: audit 2023-08-27T23:57:21.624499+0000 mon.c (mon.2) 1364 : audit [DBG] from='client.? 172.21.15.6:0/1953807321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:22.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:21 smithi130 bash[51302]: debug 2023-08-27T23:57:21.993+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:21.997688+0000) 2023-08-27T23:57:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:57:22.177Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:57:22.179Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:22.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:22 smithi006 bash[33918]: cluster 2023-08-27T23:57:21.836976+0000 mgr.y (mgr.14901) 3564 : cluster [DBG] pgmap v3571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:22 smithi006 bash[27292]: cluster 2023-08-27T23:57:21.836976+0000 mgr.y (mgr.14901) 3564 : cluster [DBG] pgmap v3571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:22.999 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:22 smithi130 bash[27579]: cluster 2023-08-27T23:57:21.836976+0000 mgr.y (mgr.14901) 3564 : cluster [DBG] pgmap v3571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:23.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:22 smithi130 bash[51302]: debug 2023-08-27T23:57:22.997+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:22.997972+0000) 2023-08-27T23:57:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:23 smithi130 bash[51302]: debug 2023-08-27T23:57:23.997+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:23.998263+0000) 2023-08-27T23:57:24.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:24 smithi006 bash[56298]: level=error ts=2023-08-27T23:57:24.067Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:24 smithi006 bash[56298]: level=warn ts=2023-08-27T23:57:24.069Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:25.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:24 smithi006 bash[27292]: cluster 2023-08-27T23:57:23.837448+0000 mgr.y (mgr.14901) 3565 : cluster [DBG] pgmap v3572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:24 smithi006 bash[27292]: audit 2023-08-27T23:57:24.047963+0000 mon.a (mon.0) 2769 : audit [DBG] from='client.? 172.21.15.6:0/2583872460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:24 smithi006 bash[33918]: cluster 2023-08-27T23:57:23.837448+0000 mgr.y (mgr.14901) 3565 : cluster [DBG] pgmap v3572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:24 smithi006 bash[33918]: audit 2023-08-27T23:57:24.047963+0000 mon.a (mon.0) 2769 : audit [DBG] from='client.? 172.21.15.6:0/2583872460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:24 smithi130 bash[27579]: cluster 2023-08-27T23:57:23.837448+0000 mgr.y (mgr.14901) 3565 : cluster [DBG] pgmap v3572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:24 smithi130 bash[27579]: audit 2023-08-27T23:57:24.047963+0000 mon.a (mon.0) 2769 : audit [DBG] from='client.? 172.21.15.6:0/2583872460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:24 smithi130 bash[51302]: debug 2023-08-27T23:57:24.993+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:24.998530+0000) 2023-08-27T23:57:26.257 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:25 smithi130 bash[51302]: debug 2023-08-27T23:57:25.997+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:25.998887+0000) 2023-08-27T23:57:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:26 smithi130 bash[27579]: audit 2023-08-27T23:57:25.523048+0000 mon.b (mon.1) 454 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:57:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:26 smithi130 bash[27579]: cluster 2023-08-27T23:57:25.838719+0000 mgr.y (mgr.14901) 3566 : cluster [DBG] pgmap v3573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:26.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:26 smithi006 bash[33918]: audit 2023-08-27T23:57:25.523048+0000 mon.b (mon.1) 454 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:57:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:26 smithi006 bash[33918]: cluster 2023-08-27T23:57:25.838719+0000 mgr.y (mgr.14901) 3566 : cluster [DBG] pgmap v3573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:26 smithi006 bash[27292]: audit 2023-08-27T23:57:25.523048+0000 mon.b (mon.1) 454 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:57:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:26 smithi006 bash[27292]: cluster 2023-08-27T23:57:25.838719+0000 mgr.y (mgr.14901) 3566 : cluster [DBG] pgmap v3573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:27.261 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:26 smithi130 bash[51302]: debug 2023-08-27T23:57:26.997+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:26.999146+0000) 2023-08-27T23:57:27.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:27 smithi130 bash[27579]: audit 2023-08-27T23:57:26.469829+0000 mon.a (mon.0) 2770 : audit [DBG] from='client.? 172.21.15.6:0/2747446016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:27.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:27 smithi006 bash[27292]: audit 2023-08-27T23:57:26.469829+0000 mon.a (mon.0) 2770 : audit [DBG] from='client.? 172.21.15.6:0/2747446016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:27.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:27 smithi006 bash[33918]: audit 2023-08-27T23:57:26.469829+0000 mon.a (mon.0) 2770 : audit [DBG] from='client.? 172.21.15.6:0/2747446016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:28.269 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:27 smithi130 bash[51302]: debug 2023-08-27T23:57:27.997+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:27.999417+0000) 2023-08-27T23:57:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:28 smithi130 bash[27579]: cluster 2023-08-27T23:57:27.839567+0000 mgr.y (mgr.14901) 3567 : cluster [DBG] pgmap v3574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:28.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:28 smithi006 bash[33918]: cluster 2023-08-27T23:57:27.839567+0000 mgr.y (mgr.14901) 3567 : cluster [DBG] pgmap v3574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:28.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:28 smithi006 bash[27292]: cluster 2023-08-27T23:57:27.839567+0000 mgr.y (mgr.14901) 3567 : cluster [DBG] pgmap v3574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:29.272 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:29 smithi130 bash[51302]: debug 2023-08-27T23:57:28.997+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:28.999749+0000) 2023-08-27T23:57:29.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:29 smithi130 bash[27579]: audit 2023-08-27T23:57:28.898468+0000 mon.a (mon.0) 2771 : audit [DBG] from='client.? 172.21.15.6:0/1808112133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:29.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:29 smithi006 bash[27292]: audit 2023-08-27T23:57:28.898468+0000 mon.a (mon.0) 2771 : audit [DBG] from='client.? 172.21.15.6:0/1808112133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:29.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:29 smithi006 bash[33918]: audit 2023-08-27T23:57:28.898468+0000 mon.a (mon.0) 2771 : audit [DBG] from='client.? 172.21.15.6:0/1808112133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:30.282 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:30 smithi130 bash[51302]: debug 2023-08-27T23:57:29.997+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:30.000014+0000) 2023-08-27T23:57:30.610 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:30 smithi006 bash[27292]: cluster 2023-08-27T23:57:29.840593+0000 mgr.y (mgr.14901) 3568 : cluster [DBG] pgmap v3575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:30.610 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:30 smithi006 bash[33918]: cluster 2023-08-27T23:57:29.840593+0000 mgr.y (mgr.14901) 3568 : cluster [DBG] pgmap v3575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:30.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:30 smithi130 bash[27579]: cluster 2023-08-27T23:57:29.840593+0000 mgr.y (mgr.14901) 3568 : cluster [DBG] pgmap v3575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:30.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:57:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:57:30] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:57:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:57:30.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:57:30.610Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:31.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:31 smithi130 bash[51302]: debug 2023-08-27T23:57:30.997+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:31.000276+0000) 2023-08-27T23:57:32.360 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:32 smithi130 bash[51302]: debug 2023-08-27T23:57:31.997+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:32.000580+0000) 2023-08-27T23:57:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:57:32.178Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:32.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:57:32.180Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:32.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:32 smithi006 bash[33918]: audit 2023-08-27T23:57:31.350531+0000 mon.a (mon.0) 2772 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:57:32.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:32 smithi006 bash[33918]: audit 2023-08-27T23:57:31.358469+0000 mon.c (mon.2) 1365 : audit [DBG] from='client.? 172.21.15.6:0/3754247089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:32.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:32 smithi006 bash[33918]: cluster 2023-08-27T23:57:31.841383+0000 mgr.y (mgr.14901) 3569 : cluster [DBG] pgmap v3576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:32.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:32 smithi006 bash[33918]: audit 2023-08-27T23:57:31.862956+0000 mon.a (mon.0) 2773 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:57:32.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:32 smithi006 bash[33918]: audit 2023-08-27T23:57:32.135972+0000 mon.b (mon.1) 455 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:57:32.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:32 smithi006 bash[33918]: audit 2023-08-27T23:57:32.138096+0000 mon.b (mon.1) 456 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:57:32.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:32 smithi006 bash[33918]: audit 2023-08-27T23:57:32.147057+0000 mon.a (mon.0) 2774 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:57:32.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:32 smithi006 bash[27292]: audit 2023-08-27T23:57:31.350531+0000 mon.a (mon.0) 2772 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:57:32.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:32 smithi006 bash[27292]: audit 2023-08-27T23:57:31.358469+0000 mon.c (mon.2) 1365 : audit [DBG] from='client.? 172.21.15.6:0/3754247089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:32.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:32 smithi006 bash[27292]: cluster 2023-08-27T23:57:31.841383+0000 mgr.y (mgr.14901) 3569 : cluster [DBG] pgmap v3576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:32.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:32 smithi006 bash[27292]: audit 2023-08-27T23:57:31.862956+0000 mon.a (mon.0) 2773 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:57:32.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:32 smithi006 bash[27292]: audit 2023-08-27T23:57:32.135972+0000 mon.b (mon.1) 455 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:57:32.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:32 smithi006 bash[27292]: audit 2023-08-27T23:57:32.138096+0000 mon.b (mon.1) 456 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:57:32.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:32 smithi006 bash[27292]: audit 2023-08-27T23:57:32.147057+0000 mon.a (mon.0) 2774 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:57:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:32 smithi130 bash[27579]: audit 2023-08-27T23:57:31.350531+0000 mon.a (mon.0) 2772 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:57:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:32 smithi130 bash[27579]: audit 2023-08-27T23:57:31.358469+0000 mon.c (mon.2) 1365 : audit [DBG] from='client.? 172.21.15.6:0/3754247089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:32 smithi130 bash[27579]: cluster 2023-08-27T23:57:31.841383+0000 mgr.y (mgr.14901) 3569 : cluster [DBG] pgmap v3576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:32 smithi130 bash[27579]: audit 2023-08-27T23:57:31.862956+0000 mon.a (mon.0) 2773 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:57:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:32 smithi130 bash[27579]: audit 2023-08-27T23:57:32.135972+0000 mon.b (mon.1) 455 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:57:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:32 smithi130 bash[27579]: audit 2023-08-27T23:57:32.138096+0000 mon.b (mon.1) 456 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:57:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:32 smithi130 bash[27579]: audit 2023-08-27T23:57:32.147057+0000 mon.a (mon.0) 2774 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:57:33.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:33 smithi130 bash[51302]: debug 2023-08-27T23:57:32.997+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:33.000919+0000) 2023-08-27T23:57:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:33 smithi130 bash[27579]: audit 2023-08-27T23:57:33.775650+0000 mon.c (mon.2) 1366 : audit [DBG] from='client.? 172.21.15.6:0/696308289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:34.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:34 smithi130 bash[51302]: debug 2023-08-27T23:57:33.997+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:34.001259+0000) 2023-08-27T23:57:34.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:34 smithi006 bash[56298]: level=error ts=2023-08-27T23:57:34.068Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:34 smithi006 bash[56298]: level=warn ts=2023-08-27T23:57:34.070Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:34.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:33 smithi006 bash[27292]: audit 2023-08-27T23:57:33.775650+0000 mon.c (mon.2) 1366 : audit [DBG] from='client.? 172.21.15.6:0/696308289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:34.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:33 smithi006 bash[33918]: audit 2023-08-27T23:57:33.775650+0000 mon.c (mon.2) 1366 : audit [DBG] from='client.? 172.21.15.6:0/696308289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:35.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:34 smithi130 bash[27579]: cluster 2023-08-27T23:57:33.841919+0000 mgr.y (mgr.14901) 3570 : cluster [DBG] pgmap v3577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:35.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:35 smithi130 bash[51302]: debug 2023-08-27T23:57:34.997+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:35.001592+0000) 2023-08-27T23:57:35.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:34 smithi006 bash[27292]: cluster 2023-08-27T23:57:33.841919+0000 mgr.y (mgr.14901) 3570 : cluster [DBG] pgmap v3577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:34 smithi006 bash[33918]: cluster 2023-08-27T23:57:33.841919+0000 mgr.y (mgr.14901) 3570 : cluster [DBG] pgmap v3577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:36.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:36 smithi130 bash[51302]: debug 2023-08-27T23:57:35.997+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:36.001927+0000) 2023-08-27T23:57:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:36 smithi130 bash[27579]: cluster 2023-08-27T23:57:35.843010+0000 mgr.y (mgr.14901) 3571 : cluster [DBG] pgmap v3578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:36 smithi130 bash[27579]: audit 2023-08-27T23:57:36.185584+0000 mon.a (mon.0) 2775 : audit [DBG] from='client.? 172.21.15.6:0/2021921133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:36.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:36 smithi006 bash[33918]: cluster 2023-08-27T23:57:35.843010+0000 mgr.y (mgr.14901) 3571 : cluster [DBG] pgmap v3578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:36 smithi006 bash[33918]: audit 2023-08-27T23:57:36.185584+0000 mon.a (mon.0) 2775 : audit [DBG] from='client.? 172.21.15.6:0/2021921133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:36 smithi006 bash[27292]: cluster 2023-08-27T23:57:35.843010+0000 mgr.y (mgr.14901) 3571 : cluster [DBG] pgmap v3578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:36 smithi006 bash[27292]: audit 2023-08-27T23:57:36.185584+0000 mon.a (mon.0) 2775 : audit [DBG] from='client.? 172.21.15.6:0/2021921133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:37 smithi130 bash[51302]: debug 2023-08-27T23:57:36.997+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:37.002240+0000) 2023-08-27T23:57:38.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:38 smithi130 bash[51302]: debug 2023-08-27T23:57:37.997+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:38.002618+0000) 2023-08-27T23:57:39.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:38 smithi006 bash[33918]: cluster 2023-08-27T23:57:37.843663+0000 mgr.y (mgr.14901) 3572 : cluster [DBG] pgmap v3579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:39.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:38 smithi006 bash[33918]: audit 2023-08-27T23:57:38.605334+0000 mon.a (mon.0) 2776 : audit [DBG] from='client.? 172.21.15.6:0/3581395163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:39.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:38 smithi006 bash[27292]: cluster 2023-08-27T23:57:37.843663+0000 mgr.y (mgr.14901) 3572 : cluster [DBG] pgmap v3579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:39.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:38 smithi006 bash[27292]: audit 2023-08-27T23:57:38.605334+0000 mon.a (mon.0) 2776 : audit [DBG] from='client.? 172.21.15.6:0/3581395163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:38 smithi130 bash[27579]: cluster 2023-08-27T23:57:37.843663+0000 mgr.y (mgr.14901) 3572 : cluster [DBG] pgmap v3579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:38 smithi130 bash[27579]: audit 2023-08-27T23:57:38.605334+0000 mon.a (mon.0) 2776 : audit [DBG] from='client.? 172.21.15.6:0/3581395163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:39.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:39 smithi130 bash[51302]: debug 2023-08-27T23:57:39.001+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:39.002963+0000) 2023-08-27T23:57:40.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:40 smithi130 bash[51302]: debug 2023-08-27T23:57:40.001+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:40.003231+0000) 2023-08-27T23:57:40.908 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:57:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:57:40] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:57:40.908 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:57:40.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:40.908 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:57:40.610Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:41.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:40 smithi006 bash[33918]: cluster 2023-08-27T23:57:39.844789+0000 mgr.y (mgr.14901) 3573 : cluster [DBG] pgmap v3580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:40 smithi006 bash[27292]: cluster 2023-08-27T23:57:39.844789+0000 mgr.y (mgr.14901) 3573 : cluster [DBG] pgmap v3580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:41.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:40 smithi130 bash[27579]: cluster 2023-08-27T23:57:39.844789+0000 mgr.y (mgr.14901) 3573 : cluster [DBG] pgmap v3580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:41.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:41 smithi130 bash[51302]: debug 2023-08-27T23:57:41.001+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:41.003516+0000) 2023-08-27T23:57:42.181 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:41 smithi006 bash[27292]: audit 2023-08-27T23:57:41.029145+0000 mon.c (mon.2) 1367 : audit [DBG] from='client.? 172.21.15.6:0/222217374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:42.181 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:41 smithi006 bash[33918]: audit 2023-08-27T23:57:41.029145+0000 mon.c (mon.2) 1367 : audit [DBG] from='client.? 172.21.15.6:0/222217374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:41 smithi130 bash[27579]: audit 2023-08-27T23:57:41.029145+0000 mon.c (mon.2) 1367 : audit [DBG] from='client.? 172.21.15.6:0/222217374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:42.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:42 smithi130 bash[51302]: debug 2023-08-27T23:57:42.001+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:42.003870+0000) 2023-08-27T23:57:42.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:57:42.179Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:57:42.182Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:43.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:42 smithi006 bash[27292]: cluster 2023-08-27T23:57:41.845511+0000 mgr.y (mgr.14901) 3574 : cluster [DBG] pgmap v3581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:42 smithi006 bash[33918]: cluster 2023-08-27T23:57:41.845511+0000 mgr.y (mgr.14901) 3574 : cluster [DBG] pgmap v3581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:43.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:42 smithi130 bash[27579]: cluster 2023-08-27T23:57:41.845511+0000 mgr.y (mgr.14901) 3574 : cluster [DBG] pgmap v3581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:43.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:43 smithi130 bash[51302]: debug 2023-08-27T23:57:43.001+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:43.004235+0000) 2023-08-27T23:57:44.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:43 smithi006 bash[27292]: audit 2023-08-27T23:57:43.432251+0000 mon.a (mon.0) 2777 : audit [DBG] from='client.? 172.21.15.6:0/3582398203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:44.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:43 smithi006 bash[33918]: audit 2023-08-27T23:57:43.432251+0000 mon.a (mon.0) 2777 : audit [DBG] from='client.? 172.21.15.6:0/3582398203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:44 smithi006 bash[56298]: level=error ts=2023-08-27T23:57:44.068Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:44 smithi006 bash[56298]: level=warn ts=2023-08-27T23:57:44.070Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:43 smithi130 bash[27579]: audit 2023-08-27T23:57:43.432251+0000 mon.a (mon.0) 2777 : audit [DBG] from='client.? 172.21.15.6:0/3582398203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:44.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:44 smithi130 bash[51302]: debug 2023-08-27T23:57:44.001+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:44.004586+0000) 2023-08-27T23:57:45.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:44 smithi006 bash[27292]: cluster 2023-08-27T23:57:43.846168+0000 mgr.y (mgr.14901) 3575 : cluster [DBG] pgmap v3582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:44 smithi006 bash[33918]: cluster 2023-08-27T23:57:43.846168+0000 mgr.y (mgr.14901) 3575 : cluster [DBG] pgmap v3582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:44 smithi130 bash[27579]: cluster 2023-08-27T23:57:43.846168+0000 mgr.y (mgr.14901) 3575 : cluster [DBG] pgmap v3582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:45 smithi130 bash[51302]: debug 2023-08-27T23:57:45.001+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:45.004912+0000) 2023-08-27T23:57:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:46 smithi130 bash[51302]: debug 2023-08-27T23:57:46.001+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:46.005226+0000) 2023-08-27T23:57:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:46 smithi130 bash[27579]: cluster 2023-08-27T23:57:45.846930+0000 mgr.y (mgr.14901) 3576 : cluster [DBG] pgmap v3583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:46 smithi130 bash[27579]: audit 2023-08-27T23:57:45.853819+0000 mon.c (mon.2) 1368 : audit [DBG] from='client.? 172.21.15.6:0/649961311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:46.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:46 smithi006 bash[33918]: cluster 2023-08-27T23:57:45.846930+0000 mgr.y (mgr.14901) 3576 : cluster [DBG] pgmap v3583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:46 smithi006 bash[33918]: audit 2023-08-27T23:57:45.853819+0000 mon.c (mon.2) 1368 : audit [DBG] from='client.? 172.21.15.6:0/649961311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:46 smithi006 bash[27292]: cluster 2023-08-27T23:57:45.846930+0000 mgr.y (mgr.14901) 3576 : cluster [DBG] pgmap v3583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:46 smithi006 bash[27292]: audit 2023-08-27T23:57:45.853819+0000 mon.c (mon.2) 1368 : audit [DBG] from='client.? 172.21.15.6:0/649961311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:47 smithi130 bash[51302]: debug 2023-08-27T23:57:47.001+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:47.005535+0000) 2023-08-27T23:57:48.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:48 smithi130 bash[51302]: debug 2023-08-27T23:57:48.001+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:48.005832+0000) 2023-08-27T23:57:49.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:48 smithi006 bash[27292]: cluster 2023-08-27T23:57:47.847548+0000 mgr.y (mgr.14901) 3577 : cluster [DBG] pgmap v3584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:49.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:48 smithi006 bash[27292]: audit 2023-08-27T23:57:48.275685+0000 mon.c (mon.2) 1369 : audit [DBG] from='client.? 172.21.15.6:0/1562030079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:49.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:48 smithi006 bash[33918]: cluster 2023-08-27T23:57:47.847548+0000 mgr.y (mgr.14901) 3577 : cluster [DBG] pgmap v3584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:49.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:48 smithi006 bash[33918]: audit 2023-08-27T23:57:48.275685+0000 mon.c (mon.2) 1369 : audit [DBG] from='client.? 172.21.15.6:0/1562030079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:49.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:48 smithi130 bash[27579]: cluster 2023-08-27T23:57:47.847548+0000 mgr.y (mgr.14901) 3577 : cluster [DBG] pgmap v3584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:49.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:48 smithi130 bash[27579]: audit 2023-08-27T23:57:48.275685+0000 mon.c (mon.2) 1369 : audit [DBG] from='client.? 172.21.15.6:0/1562030079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:49 smithi130 bash[51302]: debug 2023-08-27T23:57:49.005+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:49.006188+0000) 2023-08-27T23:57:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:50 smithi130 bash[51302]: debug 2023-08-27T23:57:50.001+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:50.006510+0000) 2023-08-27T23:57:50.912 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:57:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:57:50] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-27T23:57:50.913 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:57:50.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:50.913 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:57:50.610Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:50 smithi006 bash[27292]: cluster 2023-08-27T23:57:49.848668+0000 mgr.y (mgr.14901) 3578 : cluster [DBG] pgmap v3585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:50 smithi006 bash[27292]: audit 2023-08-27T23:57:50.692377+0000 mon.a (mon.0) 2778 : audit [DBG] from='client.? 172.21.15.6:0/1982996878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:50 smithi006 bash[33918]: cluster 2023-08-27T23:57:49.848668+0000 mgr.y (mgr.14901) 3578 : cluster [DBG] pgmap v3585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:50 smithi006 bash[33918]: audit 2023-08-27T23:57:50.692377+0000 mon.a (mon.0) 2778 : audit [DBG] from='client.? 172.21.15.6:0/1982996878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:51.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:50 smithi130 bash[27579]: cluster 2023-08-27T23:57:49.848668+0000 mgr.y (mgr.14901) 3578 : cluster [DBG] pgmap v3585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:51.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:50 smithi130 bash[27579]: audit 2023-08-27T23:57:50.692377+0000 mon.a (mon.0) 2778 : audit [DBG] from='client.? 172.21.15.6:0/1982996878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:51.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:51 smithi130 bash[51302]: debug 2023-08-27T23:57:51.005+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:51.006886+0000) 2023-08-27T23:57:52.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:52 smithi130 bash[51302]: debug 2023-08-27T23:57:52.005+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:52.007129+0000) 2023-08-27T23:57:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:57:52.179Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:57:52.181Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:53.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:52 smithi006 bash[27292]: cluster 2023-08-27T23:57:51.849339+0000 mgr.y (mgr.14901) 3579 : cluster [DBG] pgmap v3586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:53.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:52 smithi006 bash[33918]: cluster 2023-08-27T23:57:51.849339+0000 mgr.y (mgr.14901) 3579 : cluster [DBG] pgmap v3586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:52 smithi130 bash[27579]: cluster 2023-08-27T23:57:51.849339+0000 mgr.y (mgr.14901) 3579 : cluster [DBG] pgmap v3586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:53.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:53 smithi130 bash[51302]: debug 2023-08-27T23:57:53.005+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:53.007443+0000) 2023-08-27T23:57:54.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:53 smithi006 bash[27292]: audit 2023-08-27T23:57:53.107189+0000 mon.a (mon.0) 2779 : audit [DBG] from='client.? 172.21.15.6:0/1307015975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:54.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:53 smithi006 bash[33918]: audit 2023-08-27T23:57:53.107189+0000 mon.a (mon.0) 2779 : audit [DBG] from='client.? 172.21.15.6:0/1307015975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:54.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:54 smithi006 bash[56298]: level=error ts=2023-08-27T23:57:54.069Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:54.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:57:54 smithi006 bash[56298]: level=warn ts=2023-08-27T23:57:54.071Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:57:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:53 smithi130 bash[27579]: audit 2023-08-27T23:57:53.107189+0000 mon.a (mon.0) 2779 : audit [DBG] from='client.? 172.21.15.6:0/1307015975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:54 smithi130 bash[51302]: debug 2023-08-27T23:57:54.005+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:54.007847+0000) 2023-08-27T23:57:55.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:54 smithi006 bash[27292]: cluster 2023-08-27T23:57:53.849967+0000 mgr.y (mgr.14901) 3580 : cluster [DBG] pgmap v3587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:54 smithi006 bash[33918]: cluster 2023-08-27T23:57:53.849967+0000 mgr.y (mgr.14901) 3580 : cluster [DBG] pgmap v3587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:54 smithi130 bash[27579]: cluster 2023-08-27T23:57:53.849967+0000 mgr.y (mgr.14901) 3580 : cluster [DBG] pgmap v3587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:55 smithi130 bash[51302]: debug 2023-08-27T23:57:55.005+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:55.008188+0000) 2023-08-27T23:57:56.262 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:56 smithi130 bash[51302]: debug 2023-08-27T23:57:56.005+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:56.008554+0000) 2023-08-27T23:57:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:56 smithi130 bash[27579]: audit 2023-08-27T23:57:55.526636+0000 mon.a (mon.0) 2780 : audit [DBG] from='client.? 172.21.15.6:0/1369740659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:56 smithi130 bash[27579]: cluster 2023-08-27T23:57:55.851093+0000 mgr.y (mgr.14901) 3581 : cluster [DBG] pgmap v3588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:56.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:56 smithi006 bash[33918]: audit 2023-08-27T23:57:55.526636+0000 mon.a (mon.0) 2780 : audit [DBG] from='client.? 172.21.15.6:0/1369740659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:56 smithi006 bash[33918]: cluster 2023-08-27T23:57:55.851093+0000 mgr.y (mgr.14901) 3581 : cluster [DBG] pgmap v3588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:56 smithi006 bash[27292]: audit 2023-08-27T23:57:55.526636+0000 mon.a (mon.0) 2780 : audit [DBG] from='client.? 172.21.15.6:0/1369740659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:56 smithi006 bash[27292]: cluster 2023-08-27T23:57:55.851093+0000 mgr.y (mgr.14901) 3581 : cluster [DBG] pgmap v3588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:57.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:57 smithi130 bash[51302]: debug 2023-08-27T23:57:57.005+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:57.008868+0000) 2023-08-27T23:57:58.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:58 smithi130 bash[51302]: debug 2023-08-27T23:57:58.009+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:58.009232+0000) 2023-08-27T23:57:59.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:58 smithi006 bash[33918]: cluster 2023-08-27T23:57:57.851569+0000 mgr.y (mgr.14901) 3582 : cluster [DBG] pgmap v3589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:57:58 smithi006 bash[33918]: audit 2023-08-27T23:57:57.937375+0000 mon.a (mon.0) 2781 : audit [DBG] from='client.? 172.21.15.6:0/1408888762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:58 smithi006 bash[27292]: cluster 2023-08-27T23:57:57.851569+0000 mgr.y (mgr.14901) 3582 : cluster [DBG] pgmap v3589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:57:58 smithi006 bash[27292]: audit 2023-08-27T23:57:57.937375+0000 mon.a (mon.0) 2781 : audit [DBG] from='client.? 172.21.15.6:0/1408888762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:57:59 smithi130 bash[51302]: debug 2023-08-27T23:57:59.005+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:59.009562+0000) 2023-08-27T23:57:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:58 smithi130 bash[27579]: cluster 2023-08-27T23:57:57.851569+0000 mgr.y (mgr.14901) 3582 : cluster [DBG] pgmap v3589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:57:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:57:58 smithi130 bash[27579]: audit 2023-08-27T23:57:57.937375+0000 mon.a (mon.0) 2781 : audit [DBG] from='client.? 172.21.15.6:0/1408888762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:00.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:00 smithi130 bash[51302]: debug 2023-08-27T23:58:00.004+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:00.009877+0000) 2023-08-27T23:58:00.916 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:58:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:58:00] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-27T23:58:00.916 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:58:00.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:00.917 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:58:00.612Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:01.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:00 smithi006 bash[33918]: cluster 2023-08-27T23:57:59.852699+0000 mgr.y (mgr.14901) 3583 : cluster [DBG] pgmap v3590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:00 smithi006 bash[33918]: audit 2023-08-27T23:58:00.369250+0000 mon.c (mon.2) 1370 : audit [DBG] from='client.? 172.21.15.6:0/1158275069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:00 smithi006 bash[27292]: cluster 2023-08-27T23:57:59.852699+0000 mgr.y (mgr.14901) 3583 : cluster [DBG] pgmap v3590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:00 smithi006 bash[27292]: audit 2023-08-27T23:58:00.369250+0000 mon.c (mon.2) 1370 : audit [DBG] from='client.? 172.21.15.6:0/1158275069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:01.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:01 smithi130 bash[51302]: debug 2023-08-27T23:58:01.004+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:01.010179+0000) 2023-08-27T23:58:01.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:00 smithi130 bash[27579]: cluster 2023-08-27T23:57:59.852699+0000 mgr.y (mgr.14901) 3583 : cluster [DBG] pgmap v3590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:01.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:00 smithi130 bash[27579]: audit 2023-08-27T23:58:00.369250+0000 mon.c (mon.2) 1370 : audit [DBG] from='client.? 172.21.15.6:0/1158275069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:02.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:02 smithi130 bash[51302]: debug 2023-08-27T23:58:02.004+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:02.010517+0000) 2023-08-27T23:58:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:58:02.181Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:58:02.183Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:03.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:02 smithi006 bash[33918]: cluster 2023-08-27T23:58:01.853351+0000 mgr.y (mgr.14901) 3584 : cluster [DBG] pgmap v3591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:02 smithi006 bash[33918]: audit 2023-08-27T23:58:02.777559+0000 mon.c (mon.2) 1371 : audit [DBG] from='client.? 172.21.15.6:0/663511182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:02 smithi006 bash[27292]: cluster 2023-08-27T23:58:01.853351+0000 mgr.y (mgr.14901) 3584 : cluster [DBG] pgmap v3591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:02 smithi006 bash[27292]: audit 2023-08-27T23:58:02.777559+0000 mon.c (mon.2) 1371 : audit [DBG] from='client.? 172.21.15.6:0/663511182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:03.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:02 smithi130 bash[27579]: cluster 2023-08-27T23:58:01.853351+0000 mgr.y (mgr.14901) 3584 : cluster [DBG] pgmap v3591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:03.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:02 smithi130 bash[27579]: audit 2023-08-27T23:58:02.777559+0000 mon.c (mon.2) 1371 : audit [DBG] from='client.? 172.21.15.6:0/663511182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:03 smithi130 bash[51302]: debug 2023-08-27T23:58:03.008+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:03.010845+0000) 2023-08-27T23:58:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:04 smithi130 bash[51302]: debug 2023-08-27T23:58:04.008+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:04.011194+0000) 2023-08-27T23:58:04.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:04 smithi006 bash[56298]: level=error ts=2023-08-27T23:58:04.069Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:04 smithi006 bash[56298]: level=warn ts=2023-08-27T23:58:04.071Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:04 smithi006 bash[33918]: cluster 2023-08-27T23:58:03.854137+0000 mgr.y (mgr.14901) 3585 : cluster [DBG] pgmap v3592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:04 smithi006 bash[27292]: cluster 2023-08-27T23:58:03.854137+0000 mgr.y (mgr.14901) 3585 : cluster [DBG] pgmap v3592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:04 smithi130 bash[27579]: cluster 2023-08-27T23:58:03.854137+0000 mgr.y (mgr.14901) 3585 : cluster [DBG] pgmap v3592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:05 smithi130 bash[51302]: debug 2023-08-27T23:58:05.008+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:05.011548+0000) 2023-08-27T23:58:06.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:05 smithi006 bash[27292]: audit 2023-08-27T23:58:05.200954+0000 mon.c (mon.2) 1372 : audit [DBG] from='client.? 172.21.15.6:0/4007022822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:05 smithi006 bash[33918]: audit 2023-08-27T23:58:05.200954+0000 mon.c (mon.2) 1372 : audit [DBG] from='client.? 172.21.15.6:0/4007022822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:05 smithi130 bash[27579]: audit 2023-08-27T23:58:05.200954+0000 mon.c (mon.2) 1372 : audit [DBG] from='client.? 172.21.15.6:0/4007022822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:06.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:06 smithi130 bash[51302]: debug 2023-08-27T23:58:06.008+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:06.011881+0000) 2023-08-27T23:58:07.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:06 smithi006 bash[33918]: cluster 2023-08-27T23:58:05.855283+0000 mgr.y (mgr.14901) 3586 : cluster [DBG] pgmap v3593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:07.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:06 smithi006 bash[27292]: cluster 2023-08-27T23:58:05.855283+0000 mgr.y (mgr.14901) 3586 : cluster [DBG] pgmap v3593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:07.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:06 smithi130 bash[27579]: cluster 2023-08-27T23:58:05.855283+0000 mgr.y (mgr.14901) 3586 : cluster [DBG] pgmap v3593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:07.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:07 smithi130 bash[51302]: debug 2023-08-27T23:58:07.008+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:07.012215+0000) 2023-08-27T23:58:08.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:07 smithi006 bash[27292]: audit 2023-08-27T23:58:07.617780+0000 mon.a (mon.0) 2782 : audit [DBG] from='client.? 172.21.15.6:0/559203697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:08.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:07 smithi006 bash[33918]: audit 2023-08-27T23:58:07.617780+0000 mon.a (mon.0) 2782 : audit [DBG] from='client.? 172.21.15.6:0/559203697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:08.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:08 smithi130 bash[51302]: debug 2023-08-27T23:58:08.008+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:08.012538+0000) 2023-08-27T23:58:08.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:07 smithi130 bash[27579]: audit 2023-08-27T23:58:07.617780+0000 mon.a (mon.0) 2782 : audit [DBG] from='client.? 172.21.15.6:0/559203697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:09.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:08 smithi006 bash[33918]: cluster 2023-08-27T23:58:07.855924+0000 mgr.y (mgr.14901) 3587 : cluster [DBG] pgmap v3594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:08 smithi006 bash[27292]: cluster 2023-08-27T23:58:07.855924+0000 mgr.y (mgr.14901) 3587 : cluster [DBG] pgmap v3594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:09.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:08 smithi130 bash[27579]: cluster 2023-08-27T23:58:07.855924+0000 mgr.y (mgr.14901) 3587 : cluster [DBG] pgmap v3594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:09.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:09 smithi130 bash[51302]: debug 2023-08-27T23:58:09.008+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:09.012929+0000) 2023-08-27T23:58:10.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:10 smithi130 bash[51302]: debug 2023-08-27T23:58:10.008+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:10.013255+0000) 2023-08-27T23:58:10.950 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:58:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:58:10] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-27T23:58:10.950 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:58:10.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:10.950 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:58:10.612Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:11.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:10 smithi006 bash[27292]: cluster 2023-08-27T23:58:09.857161+0000 mgr.y (mgr.14901) 3588 : cluster [DBG] pgmap v3595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:11.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:10 smithi006 bash[27292]: audit 2023-08-27T23:58:10.034433+0000 mon.a (mon.0) 2783 : audit [DBG] from='client.? 172.21.15.6:0/3667835576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:10 smithi006 bash[33918]: cluster 2023-08-27T23:58:09.857161+0000 mgr.y (mgr.14901) 3588 : cluster [DBG] pgmap v3595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:10 smithi006 bash[33918]: audit 2023-08-27T23:58:10.034433+0000 mon.a (mon.0) 2783 : audit [DBG] from='client.? 172.21.15.6:0/3667835576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:11.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:11 smithi130 bash[51302]: debug 2023-08-27T23:58:11.012+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:11.013603+0000) 2023-08-27T23:58:11.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:10 smithi130 bash[27579]: cluster 2023-08-27T23:58:09.857161+0000 mgr.y (mgr.14901) 3588 : cluster [DBG] pgmap v3595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:11.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:10 smithi130 bash[27579]: audit 2023-08-27T23:58:10.034433+0000 mon.a (mon.0) 2783 : audit [DBG] from='client.? 172.21.15.6:0/3667835576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:12 smithi130 bash[51302]: debug 2023-08-27T23:58:12.008+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:12.013959+0000) 2023-08-27T23:58:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:58:12.181Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:58:12.183Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:13.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:12 smithi006 bash[33918]: cluster 2023-08-27T23:58:11.857813+0000 mgr.y (mgr.14901) 3589 : cluster [DBG] pgmap v3596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:12 smithi006 bash[33918]: audit 2023-08-27T23:58:12.451643+0000 mon.a (mon.0) 2784 : audit [DBG] from='client.? 172.21.15.6:0/168810064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:12 smithi006 bash[27292]: cluster 2023-08-27T23:58:11.857813+0000 mgr.y (mgr.14901) 3589 : cluster [DBG] pgmap v3596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:12 smithi006 bash[27292]: audit 2023-08-27T23:58:12.451643+0000 mon.a (mon.0) 2784 : audit [DBG] from='client.? 172.21.15.6:0/168810064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:12 smithi130 bash[27579]: cluster 2023-08-27T23:58:11.857813+0000 mgr.y (mgr.14901) 3589 : cluster [DBG] pgmap v3596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:12 smithi130 bash[27579]: audit 2023-08-27T23:58:12.451643+0000 mon.a (mon.0) 2784 : audit [DBG] from='client.? 172.21.15.6:0/168810064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:13 smithi130 bash[51302]: debug 2023-08-27T23:58:13.012+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:13.014311+0000) 2023-08-27T23:58:14.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:14 smithi130 bash[51302]: debug 2023-08-27T23:58:14.012+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:14.014633+0000) 2023-08-27T23:58:14.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:14 smithi006 bash[56298]: level=error ts=2023-08-27T23:58:14.070Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:14 smithi006 bash[56298]: level=warn ts=2023-08-27T23:58:14.072Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:14 smithi006 bash[33918]: cluster 2023-08-27T23:58:13.858460+0000 mgr.y (mgr.14901) 3590 : cluster [DBG] pgmap v3597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:14 smithi006 bash[33918]: audit 2023-08-27T23:58:14.872095+0000 mon.a (mon.0) 2785 : audit [DBG] from='client.? 172.21.15.6:0/678355618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:14 smithi006 bash[27292]: cluster 2023-08-27T23:58:13.858460+0000 mgr.y (mgr.14901) 3590 : cluster [DBG] pgmap v3597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:14 smithi006 bash[27292]: audit 2023-08-27T23:58:14.872095+0000 mon.a (mon.0) 2785 : audit [DBG] from='client.? 172.21.15.6:0/678355618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:15 smithi130 bash[51302]: debug 2023-08-27T23:58:15.012+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:15.015024+0000) 2023-08-27T23:58:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:14 smithi130 bash[27579]: cluster 2023-08-27T23:58:13.858460+0000 mgr.y (mgr.14901) 3590 : cluster [DBG] pgmap v3597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:14 smithi130 bash[27579]: audit 2023-08-27T23:58:14.872095+0000 mon.a (mon.0) 2785 : audit [DBG] from='client.? 172.21.15.6:0/678355618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:16.267 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:16 smithi130 bash[51302]: debug 2023-08-27T23:58:16.012+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:16.015351+0000) 2023-08-27T23:58:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:16 smithi130 bash[27579]: cluster 2023-08-27T23:58:15.859593+0000 mgr.y (mgr.14901) 3591 : cluster [DBG] pgmap v3598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:16.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:16 smithi006 bash[27292]: cluster 2023-08-27T23:58:15.859593+0000 mgr.y (mgr.14901) 3591 : cluster [DBG] pgmap v3598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:16 smithi006 bash[33918]: cluster 2023-08-27T23:58:15.859593+0000 mgr.y (mgr.14901) 3591 : cluster [DBG] pgmap v3598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:17.343 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:17 smithi130 bash[51302]: debug 2023-08-27T23:58:17.012+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:17.015680+0000) 2023-08-27T23:58:17.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:17 smithi130 bash[27579]: audit 2023-08-27T23:58:17.287222+0000 mon.a (mon.0) 2786 : audit [DBG] from='client.? 172.21.15.6:0/3104353858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:17.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:17 smithi006 bash[33918]: audit 2023-08-27T23:58:17.287222+0000 mon.a (mon.0) 2786 : audit [DBG] from='client.? 172.21.15.6:0/3104353858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:17.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:17 smithi006 bash[27292]: audit 2023-08-27T23:58:17.287222+0000 mon.a (mon.0) 2786 : audit [DBG] from='client.? 172.21.15.6:0/3104353858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:18.352 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:18 smithi130 bash[51302]: debug 2023-08-27T23:58:18.012+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:18.016035+0000) 2023-08-27T23:58:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:18 smithi130 bash[27579]: cluster 2023-08-27T23:58:17.860419+0000 mgr.y (mgr.14901) 3592 : cluster [DBG] pgmap v3599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:18.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:18 smithi006 bash[27292]: cluster 2023-08-27T23:58:17.860419+0000 mgr.y (mgr.14901) 3592 : cluster [DBG] pgmap v3599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:18 smithi006 bash[33918]: cluster 2023-08-27T23:58:17.860419+0000 mgr.y (mgr.14901) 3592 : cluster [DBG] pgmap v3599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:19.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:19 smithi130 bash[51302]: debug 2023-08-27T23:58:19.012+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:19.016382+0000) 2023-08-27T23:58:20.018 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:19 smithi130 bash[27579]: audit 2023-08-27T23:58:19.695969+0000 mon.a (mon.0) 2787 : audit [DBG] from='client.? 172.21.15.6:0/664032824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:20.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:19 smithi006 bash[27292]: audit 2023-08-27T23:58:19.695969+0000 mon.a (mon.0) 2787 : audit [DBG] from='client.? 172.21.15.6:0/664032824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:20.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:19 smithi006 bash[33918]: audit 2023-08-27T23:58:19.695969+0000 mon.a (mon.0) 2787 : audit [DBG] from='client.? 172.21.15.6:0/664032824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:20.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:20 smithi130 bash[51302]: debug 2023-08-27T23:58:20.012+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:20.016766+0000) 2023-08-27T23:58:20.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:58:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:58:20] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-27T23:58:20.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:20 smithi006 bash[33918]: cluster 2023-08-27T23:58:19.861593+0000 mgr.y (mgr.14901) 3593 : cluster [DBG] pgmap v3600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:20.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:20 smithi006 bash[27292]: cluster 2023-08-27T23:58:19.861593+0000 mgr.y (mgr.14901) 3593 : cluster [DBG] pgmap v3600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:58:20.611Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:58:20.613Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:21.018 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:20 smithi130 bash[27579]: cluster 2023-08-27T23:58:19.861593+0000 mgr.y (mgr.14901) 3593 : cluster [DBG] pgmap v3600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:21.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:21 smithi130 bash[51302]: debug 2023-08-27T23:58:21.012+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:21.017124+0000) 2023-08-27T23:58:22.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:22 smithi130 bash[51302]: debug 2023-08-27T23:58:22.016+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:22.017377+0000) 2023-08-27T23:58:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:58:22.182Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:58:22.184Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:22 smithi006 bash[27292]: cluster 2023-08-27T23:58:21.862151+0000 mgr.y (mgr.14901) 3594 : cluster [DBG] pgmap v3601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:22 smithi006 bash[27292]: audit 2023-08-27T23:58:22.130350+0000 mon.a (mon.0) 2788 : audit [DBG] from='client.? 172.21.15.6:0/472975604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:22 smithi006 bash[33918]: cluster 2023-08-27T23:58:21.862151+0000 mgr.y (mgr.14901) 3594 : cluster [DBG] pgmap v3601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:22 smithi006 bash[33918]: audit 2023-08-27T23:58:22.130350+0000 mon.a (mon.0) 2788 : audit [DBG] from='client.? 172.21.15.6:0/472975604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:23.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:22 smithi130 bash[27579]: cluster 2023-08-27T23:58:21.862151+0000 mgr.y (mgr.14901) 3594 : cluster [DBG] pgmap v3601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:23.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:22 smithi130 bash[27579]: audit 2023-08-27T23:58:22.130350+0000 mon.a (mon.0) 2788 : audit [DBG] from='client.? 172.21.15.6:0/472975604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:23.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:23 smithi130 bash[51302]: debug 2023-08-27T23:58:23.012+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:23.017697+0000) 2023-08-27T23:58:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:24 smithi130 bash[51302]: debug 2023-08-27T23:58:24.016+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:24.018035+0000) 2023-08-27T23:58:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:24 smithi006 bash[56298]: level=error ts=2023-08-27T23:58:24.071Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:24 smithi006 bash[56298]: level=warn ts=2023-08-27T23:58:24.073Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:25.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:24 smithi006 bash[33918]: cluster 2023-08-27T23:58:23.862802+0000 mgr.y (mgr.14901) 3595 : cluster [DBG] pgmap v3602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:24 smithi006 bash[33918]: audit 2023-08-27T23:58:24.547121+0000 mon.c (mon.2) 1373 : audit [DBG] from='client.? 172.21.15.6:0/2166724729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:24 smithi006 bash[27292]: cluster 2023-08-27T23:58:23.862802+0000 mgr.y (mgr.14901) 3595 : cluster [DBG] pgmap v3602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:24 smithi006 bash[27292]: audit 2023-08-27T23:58:24.547121+0000 mon.c (mon.2) 1373 : audit [DBG] from='client.? 172.21.15.6:0/2166724729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:24 smithi130 bash[27579]: cluster 2023-08-27T23:58:23.862802+0000 mgr.y (mgr.14901) 3595 : cluster [DBG] pgmap v3602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:24 smithi130 bash[27579]: audit 2023-08-27T23:58:24.547121+0000 mon.c (mon.2) 1373 : audit [DBG] from='client.? 172.21.15.6:0/2166724729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:25 smithi130 bash[51302]: debug 2023-08-27T23:58:25.016+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:25.018357+0000) 2023-08-27T23:58:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:26 smithi130 bash[27579]: cluster 2023-08-27T23:58:25.863911+0000 mgr.y (mgr.14901) 3596 : cluster [DBG] pgmap v3603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:26.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:26 smithi130 bash[51302]: debug 2023-08-27T23:58:26.012+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:26.018716+0000) 2023-08-27T23:58:26.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:26 smithi006 bash[33918]: cluster 2023-08-27T23:58:25.863911+0000 mgr.y (mgr.14901) 3596 : cluster [DBG] pgmap v3603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:26 smithi006 bash[27292]: cluster 2023-08-27T23:58:25.863911+0000 mgr.y (mgr.14901) 3596 : cluster [DBG] pgmap v3603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:27.271 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:27 smithi130 bash[51302]: debug 2023-08-27T23:58:27.016+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:27.019034+0000) 2023-08-27T23:58:27.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:27 smithi130 bash[27579]: audit 2023-08-27T23:58:26.954839+0000 mon.c (mon.2) 1374 : audit [DBG] from='client.? 172.21.15.6:0/399309027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:27.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:27 smithi006 bash[27292]: audit 2023-08-27T23:58:26.954839+0000 mon.c (mon.2) 1374 : audit [DBG] from='client.? 172.21.15.6:0/399309027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:27.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:27 smithi006 bash[33918]: audit 2023-08-27T23:58:26.954839+0000 mon.c (mon.2) 1374 : audit [DBG] from='client.? 172.21.15.6:0/399309027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:28.278 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:28 smithi130 bash[51302]: debug 2023-08-27T23:58:28.016+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:28.019396+0000) 2023-08-27T23:58:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:28 smithi130 bash[27579]: cluster 2023-08-27T23:58:27.864542+0000 mgr.y (mgr.14901) 3597 : cluster [DBG] pgmap v3604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:28.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:28 smithi006 bash[33918]: cluster 2023-08-27T23:58:27.864542+0000 mgr.y (mgr.14901) 3597 : cluster [DBG] pgmap v3604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:28.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:28 smithi006 bash[27292]: cluster 2023-08-27T23:58:27.864542+0000 mgr.y (mgr.14901) 3597 : cluster [DBG] pgmap v3604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:29.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:29 smithi130 bash[51302]: debug 2023-08-27T23:58:29.016+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:29.019732+0000) 2023-08-27T23:58:29.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:29 smithi006 bash[33918]: audit 2023-08-27T23:58:29.363217+0000 mon.c (mon.2) 1375 : audit [DBG] from='client.? 172.21.15.6:0/1403380298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:29.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:29 smithi006 bash[27292]: audit 2023-08-27T23:58:29.363217+0000 mon.c (mon.2) 1375 : audit [DBG] from='client.? 172.21.15.6:0/1403380298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:29.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:29 smithi130 bash[27579]: audit 2023-08-27T23:58:29.363217+0000 mon.c (mon.2) 1375 : audit [DBG] from='client.? 172.21.15.6:0/1403380298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:30 smithi130 bash[51302]: debug 2023-08-27T23:58:30.016+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:30.020083+0000) 2023-08-27T23:58:30.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:30 smithi006 bash[33918]: cluster 2023-08-27T23:58:29.865479+0000 mgr.y (mgr.14901) 3598 : cluster [DBG] pgmap v3605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:30 smithi006 bash[27292]: cluster 2023-08-27T23:58:29.865479+0000 mgr.y (mgr.14901) 3598 : cluster [DBG] pgmap v3605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:58:30.612Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:30.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:58:30.614Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:30 smithi130 bash[27579]: cluster 2023-08-27T23:58:29.865479+0000 mgr.y (mgr.14901) 3598 : cluster [DBG] pgmap v3605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:58:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:58:30] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-27T23:58:31.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:31 smithi130 bash[51302]: debug 2023-08-27T23:58:31.016+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:31.020410+0000) 2023-08-27T23:58:32.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:31 smithi130 bash[27579]: audit 2023-08-27T23:58:31.774877+0000 mon.c (mon.2) 1376 : audit [DBG] from='client.? 172.21.15.6:0/3451801220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:32.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:32 smithi130 bash[51302]: debug 2023-08-27T23:58:32.020+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:32.020737+0000) 2023-08-27T23:58:32.184 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:31 smithi006 bash[27292]: audit 2023-08-27T23:58:31.774877+0000 mon.c (mon.2) 1376 : audit [DBG] from='client.? 172.21.15.6:0/3451801220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:32.185 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:31 smithi006 bash[33918]: audit 2023-08-27T23:58:31.774877+0000 mon.c (mon.2) 1376 : audit [DBG] from='client.? 172.21.15.6:0/3451801220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:58:32.182Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:58:32.184Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:33.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:33 smithi130 bash[51302]: debug 2023-08-27T23:58:33.020+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:33.021072+0000) 2023-08-27T23:58:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:32 smithi130 bash[27579]: cluster 2023-08-27T23:58:31.866145+0000 mgr.y (mgr.14901) 3599 : cluster [DBG] pgmap v3606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:32 smithi130 bash[27579]: audit 2023-08-27T23:58:32.162273+0000 mon.b (mon.1) 457 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:58:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:32 smithi006 bash[33918]: cluster 2023-08-27T23:58:31.866145+0000 mgr.y (mgr.14901) 3599 : cluster [DBG] pgmap v3606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:32 smithi006 bash[33918]: audit 2023-08-27T23:58:32.162273+0000 mon.b (mon.1) 457 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:58:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:32 smithi006 bash[27292]: cluster 2023-08-27T23:58:31.866145+0000 mgr.y (mgr.14901) 3599 : cluster [DBG] pgmap v3606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:32 smithi006 bash[27292]: audit 2023-08-27T23:58:32.162273+0000 mon.b (mon.1) 457 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:58:34.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:34 smithi130 bash[51302]: debug 2023-08-27T23:58:34.020+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:34.021429+0000) 2023-08-27T23:58:34.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:34 smithi006 bash[56298]: level=error ts=2023-08-27T23:58:34.071Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:34 smithi006 bash[56298]: level=warn ts=2023-08-27T23:58:34.073Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:34 smithi006 bash[33918]: cluster 2023-08-27T23:58:33.866758+0000 mgr.y (mgr.14901) 3600 : cluster [DBG] pgmap v3607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:34 smithi006 bash[33918]: audit 2023-08-27T23:58:34.194949+0000 mon.a (mon.0) 2789 : audit [DBG] from='client.? 172.21.15.6:0/1693931336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:34 smithi006 bash[27292]: cluster 2023-08-27T23:58:33.866758+0000 mgr.y (mgr.14901) 3600 : cluster [DBG] pgmap v3607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:34 smithi006 bash[27292]: audit 2023-08-27T23:58:34.194949+0000 mon.a (mon.0) 2789 : audit [DBG] from='client.? 172.21.15.6:0/1693931336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:34 smithi130 bash[27579]: cluster 2023-08-27T23:58:33.866758+0000 mgr.y (mgr.14901) 3600 : cluster [DBG] pgmap v3607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:34 smithi130 bash[27579]: audit 2023-08-27T23:58:34.194949+0000 mon.a (mon.0) 2789 : audit [DBG] from='client.? 172.21.15.6:0/1693931336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:35.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:35 smithi130 bash[51302]: debug 2023-08-27T23:58:35.016+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:35.021834+0000) 2023-08-27T23:58:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:36 smithi130 bash[27579]: cluster 2023-08-27T23:58:35.867873+0000 mgr.y (mgr.14901) 3601 : cluster [DBG] pgmap v3608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:36.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:36 smithi130 bash[51302]: debug 2023-08-27T23:58:36.020+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:36.022173+0000) 2023-08-27T23:58:36.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:36 smithi006 bash[33918]: cluster 2023-08-27T23:58:35.867873+0000 mgr.y (mgr.14901) 3601 : cluster [DBG] pgmap v3608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:36 smithi006 bash[27292]: cluster 2023-08-27T23:58:35.867873+0000 mgr.y (mgr.14901) 3601 : cluster [DBG] pgmap v3608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:37.276 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:37 smithi130 bash[51302]: debug 2023-08-27T23:58:37.016+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:37.022415+0000) 2023-08-27T23:58:37.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:37 smithi130 bash[27579]: audit 2023-08-27T23:58:36.618206+0000 mon.a (mon.0) 2790 : audit [DBG] from='client.? 172.21.15.6:0/2246793413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:37.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:37 smithi006 bash[27292]: audit 2023-08-27T23:58:36.618206+0000 mon.a (mon.0) 2790 : audit [DBG] from='client.? 172.21.15.6:0/2246793413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:37.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:37 smithi006 bash[33918]: audit 2023-08-27T23:58:36.618206+0000 mon.a (mon.0) 2790 : audit [DBG] from='client.? 172.21.15.6:0/2246793413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:38.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:38 smithi130 bash[51302]: debug 2023-08-27T23:58:38.016+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:38.022643+0000) 2023-08-27T23:58:39.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:39 smithi130 bash[51302]: debug 2023-08-27T23:58:39.020+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:39.022983+0000) 2023-08-27T23:58:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:39 smithi130 bash[27579]: cluster 2023-08-27T23:58:37.868531+0000 mgr.y (mgr.14901) 3602 : cluster [DBG] pgmap v3609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:39 smithi130 bash[27579]: audit 2023-08-27T23:58:38.060122+0000 mon.a (mon.0) 2791 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:58:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:39 smithi130 bash[27579]: audit 2023-08-27T23:58:38.500456+0000 mon.a (mon.0) 2792 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:58:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:39 smithi130 bash[27579]: audit 2023-08-27T23:58:38.792689+0000 mon.b (mon.1) 458 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:58:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:39 smithi130 bash[27579]: audit 2023-08-27T23:58:38.795007+0000 mon.b (mon.1) 459 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:58:39.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:39 smithi130 bash[27579]: audit 2023-08-27T23:58:38.802346+0000 mon.a (mon.0) 2793 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:58:39.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:39 smithi130 bash[27579]: audit 2023-08-27T23:58:39.032484+0000 mon.a (mon.0) 2794 : audit [DBG] from='client.? 172.21.15.6:0/3260170277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:39 smithi006 bash[27292]: cluster 2023-08-27T23:58:37.868531+0000 mgr.y (mgr.14901) 3602 : cluster [DBG] pgmap v3609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:39 smithi006 bash[27292]: audit 2023-08-27T23:58:38.060122+0000 mon.a (mon.0) 2791 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:58:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:39 smithi006 bash[27292]: audit 2023-08-27T23:58:38.500456+0000 mon.a (mon.0) 2792 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:58:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:39 smithi006 bash[27292]: audit 2023-08-27T23:58:38.792689+0000 mon.b (mon.1) 458 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:58:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:39 smithi006 bash[27292]: audit 2023-08-27T23:58:38.795007+0000 mon.b (mon.1) 459 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:58:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:39 smithi006 bash[27292]: audit 2023-08-27T23:58:38.802346+0000 mon.a (mon.0) 2793 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:58:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:39 smithi006 bash[27292]: audit 2023-08-27T23:58:39.032484+0000 mon.a (mon.0) 2794 : audit [DBG] from='client.? 172.21.15.6:0/3260170277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:39.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:39 smithi006 bash[33918]: cluster 2023-08-27T23:58:37.868531+0000 mgr.y (mgr.14901) 3602 : cluster [DBG] pgmap v3609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:39.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:39 smithi006 bash[33918]: audit 2023-08-27T23:58:38.060122+0000 mon.a (mon.0) 2791 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:58:39.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:39 smithi006 bash[33918]: audit 2023-08-27T23:58:38.500456+0000 mon.a (mon.0) 2792 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:58:39.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:39 smithi006 bash[33918]: audit 2023-08-27T23:58:38.792689+0000 mon.b (mon.1) 458 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:58:39.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:39 smithi006 bash[33918]: audit 2023-08-27T23:58:38.795007+0000 mon.b (mon.1) 459 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:58:39.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:39 smithi006 bash[33918]: audit 2023-08-27T23:58:38.802346+0000 mon.a (mon.0) 2793 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:58:39.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:39 smithi006 bash[33918]: audit 2023-08-27T23:58:39.032484+0000 mon.a (mon.0) 2794 : audit [DBG] from='client.? 172.21.15.6:0/3260170277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:40.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:40 smithi130 bash[51302]: debug 2023-08-27T23:58:40.020+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:40.023283+0000) 2023-08-27T23:58:40.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:58:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:58:40] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-27T23:58:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:58:40.612Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:58:40.614Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:41.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:41 smithi130 bash[27579]: cluster 2023-08-27T23:58:39.869670+0000 mgr.y (mgr.14901) 3603 : cluster [DBG] pgmap v3610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:41.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:41 smithi130 bash[51302]: debug 2023-08-27T23:58:41.020+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:41.023634+0000) 2023-08-27T23:58:41.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:41 smithi006 bash[27292]: cluster 2023-08-27T23:58:39.869670+0000 mgr.y (mgr.14901) 3603 : cluster [DBG] pgmap v3610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:41.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:41 smithi006 bash[33918]: cluster 2023-08-27T23:58:39.869670+0000 mgr.y (mgr.14901) 3603 : cluster [DBG] pgmap v3610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:42 smithi130 bash[27579]: audit 2023-08-27T23:58:41.442786+0000 mon.c (mon.2) 1377 : audit [DBG] from='client.? 172.21.15.6:0/336518078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:42.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:42 smithi130 bash[51302]: debug 2023-08-27T23:58:42.020+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:42.023955+0000) 2023-08-27T23:58:42.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:42 smithi006 bash[33918]: audit 2023-08-27T23:58:41.442786+0000 mon.c (mon.2) 1377 : audit [DBG] from='client.? 172.21.15.6:0/336518078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:42.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:42 smithi006 bash[27292]: audit 2023-08-27T23:58:41.442786+0000 mon.c (mon.2) 1377 : audit [DBG] from='client.? 172.21.15.6:0/336518078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:58:42.183Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:58:42.185Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:43.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:43 smithi130 bash[27579]: cluster 2023-08-27T23:58:41.870325+0000 mgr.y (mgr.14901) 3604 : cluster [DBG] pgmap v3611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:43.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:43 smithi130 bash[51302]: debug 2023-08-27T23:58:43.020+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:43.024314+0000) 2023-08-27T23:58:43.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:43 smithi006 bash[27292]: cluster 2023-08-27T23:58:41.870325+0000 mgr.y (mgr.14901) 3604 : cluster [DBG] pgmap v3611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:43.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:43 smithi006 bash[33918]: cluster 2023-08-27T23:58:41.870325+0000 mgr.y (mgr.14901) 3604 : cluster [DBG] pgmap v3611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:44 smithi130 bash[27579]: audit 2023-08-27T23:58:43.885782+0000 mon.c (mon.2) 1378 : audit [DBG] from='client.? 172.21.15.6:0/896543387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:44.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:44 smithi130 bash[51302]: debug 2023-08-27T23:58:44.020+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:44.024668+0000) 2023-08-27T23:58:44.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:44 smithi006 bash[27292]: audit 2023-08-27T23:58:43.885782+0000 mon.c (mon.2) 1378 : audit [DBG] from='client.? 172.21.15.6:0/896543387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:44.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:44 smithi006 bash[33918]: audit 2023-08-27T23:58:43.885782+0000 mon.c (mon.2) 1378 : audit [DBG] from='client.? 172.21.15.6:0/896543387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:44 smithi006 bash[56298]: level=error ts=2023-08-27T23:58:44.072Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:44 smithi006 bash[56298]: level=warn ts=2023-08-27T23:58:44.074Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:45 smithi130 bash[27579]: cluster 2023-08-27T23:58:43.870773+0000 mgr.y (mgr.14901) 3605 : cluster [DBG] pgmap v3612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:45 smithi130 bash[51302]: debug 2023-08-27T23:58:45.020+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:45.025000+0000) 2023-08-27T23:58:45.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:45 smithi006 bash[33918]: cluster 2023-08-27T23:58:43.870773+0000 mgr.y (mgr.14901) 3605 : cluster [DBG] pgmap v3612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:45.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:45 smithi006 bash[27292]: cluster 2023-08-27T23:58:43.870773+0000 mgr.y (mgr.14901) 3605 : cluster [DBG] pgmap v3612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:46 smithi130 bash[27579]: cluster 2023-08-27T23:58:45.871847+0000 mgr.y (mgr.14901) 3606 : cluster [DBG] pgmap v3613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:46 smithi130 bash[51302]: debug 2023-08-27T23:58:46.024+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:46.025349+0000) 2023-08-27T23:58:46.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:46 smithi006 bash[33918]: cluster 2023-08-27T23:58:45.871847+0000 mgr.y (mgr.14901) 3606 : cluster [DBG] pgmap v3613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:46 smithi006 bash[27292]: cluster 2023-08-27T23:58:45.871847+0000 mgr.y (mgr.14901) 3606 : cluster [DBG] pgmap v3613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:47 smithi130 bash[27579]: audit 2023-08-27T23:58:46.308695+0000 mon.a (mon.0) 2795 : audit [DBG] from='client.? 172.21.15.6:0/3370040454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:47 smithi130 bash[51302]: debug 2023-08-27T23:58:47.024+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:47.025707+0000) 2023-08-27T23:58:47.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:47 smithi006 bash[33918]: audit 2023-08-27T23:58:46.308695+0000 mon.a (mon.0) 2795 : audit [DBG] from='client.? 172.21.15.6:0/3370040454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:47.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:47 smithi006 bash[27292]: audit 2023-08-27T23:58:46.308695+0000 mon.a (mon.0) 2795 : audit [DBG] from='client.? 172.21.15.6:0/3370040454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:48.283 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:48 smithi130 bash[51302]: debug 2023-08-27T23:58:48.020+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:48.026072+0000) 2023-08-27T23:58:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:48 smithi130 bash[27579]: cluster 2023-08-27T23:58:47.872462+0000 mgr.y (mgr.14901) 3607 : cluster [DBG] pgmap v3614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:48.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:48 smithi006 bash[27292]: cluster 2023-08-27T23:58:47.872462+0000 mgr.y (mgr.14901) 3607 : cluster [DBG] pgmap v3614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:48 smithi006 bash[33918]: cluster 2023-08-27T23:58:47.872462+0000 mgr.y (mgr.14901) 3607 : cluster [DBG] pgmap v3614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:49.286 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:49 smithi130 bash[51302]: debug 2023-08-27T23:58:49.024+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:49.026445+0000) 2023-08-27T23:58:49.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:49 smithi130 bash[27579]: audit 2023-08-27T23:58:48.720182+0000 mon.c (mon.2) 1379 : audit [DBG] from='client.? 172.21.15.6:0/4025453570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:49.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:49 smithi006 bash[33918]: audit 2023-08-27T23:58:48.720182+0000 mon.c (mon.2) 1379 : audit [DBG] from='client.? 172.21.15.6:0/4025453570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:49.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:49 smithi006 bash[27292]: audit 2023-08-27T23:58:48.720182+0000 mon.c (mon.2) 1379 : audit [DBG] from='client.? 172.21.15.6:0/4025453570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:50.297 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:50 smithi130 bash[51302]: debug 2023-08-27T23:58:50.020+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:50.026766+0000) 2023-08-27T23:58:50.615 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:50 smithi006 bash[33918]: cluster 2023-08-27T23:58:49.873605+0000 mgr.y (mgr.14901) 3608 : cluster [DBG] pgmap v3615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:50.615 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:50 smithi006 bash[27292]: cluster 2023-08-27T23:58:49.873605+0000 mgr.y (mgr.14901) 3608 : cluster [DBG] pgmap v3615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:50 smithi130 bash[27579]: cluster 2023-08-27T23:58:49.873605+0000 mgr.y (mgr.14901) 3608 : cluster [DBG] pgmap v3615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:50.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:58:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:58:50] "GET /metrics HTTP/1.1" 200 193111 "" "Prometheus/2.33.4" 2023-08-27T23:58:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:58:50.612Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:58:50.615Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:51.299 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:51 smithi130 bash[51302]: debug 2023-08-27T23:58:51.024+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:51.027114+0000) 2023-08-27T23:58:51.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:51 smithi130 bash[27579]: audit 2023-08-27T23:58:51.133422+0000 mon.a (mon.0) 2796 : audit [DBG] from='client.? 172.21.15.6:0/3065153542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:51.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:51 smithi006 bash[27292]: audit 2023-08-27T23:58:51.133422+0000 mon.a (mon.0) 2796 : audit [DBG] from='client.? 172.21.15.6:0/3065153542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:51.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:51 smithi006 bash[33918]: audit 2023-08-27T23:58:51.133422+0000 mon.a (mon.0) 2796 : audit [DBG] from='client.? 172.21.15.6:0/3065153542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:52.311 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:52 smithi130 bash[51302]: debug 2023-08-27T23:58:52.024+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:52.027474+0000) 2023-08-27T23:58:52.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:52 smithi006 bash[27292]: cluster 2023-08-27T23:58:51.874222+0000 mgr.y (mgr.14901) 3609 : cluster [DBG] pgmap v3616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:52.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:52 smithi006 bash[33918]: cluster 2023-08-27T23:58:51.874222+0000 mgr.y (mgr.14901) 3609 : cluster [DBG] pgmap v3616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:58:52.183Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:58:52.185Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:52 smithi130 bash[27579]: cluster 2023-08-27T23:58:51.874222+0000 mgr.y (mgr.14901) 3609 : cluster [DBG] pgmap v3616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:53.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:53 smithi130 bash[51302]: debug 2023-08-27T23:58:53.024+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:53.027840+0000) 2023-08-27T23:58:53.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:53 smithi130 bash[27579]: audit 2023-08-27T23:58:53.543831+0000 mon.c (mon.2) 1380 : audit [DBG] from='client.? 172.21.15.6:0/3652502905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:53.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:53 smithi006 bash[27292]: audit 2023-08-27T23:58:53.543831+0000 mon.c (mon.2) 1380 : audit [DBG] from='client.? 172.21.15.6:0/3652502905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:53.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:53 smithi006 bash[33918]: audit 2023-08-27T23:58:53.543831+0000 mon.c (mon.2) 1380 : audit [DBG] from='client.? 172.21.15.6:0/3652502905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:54 smithi130 bash[51302]: debug 2023-08-27T23:58:54.024+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:54.028175+0000) 2023-08-27T23:58:54.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:54 smithi006 bash[56298]: level=error ts=2023-08-27T23:58:54.072Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:58:54 smithi006 bash[56298]: level=warn ts=2023-08-27T23:58:54.074Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:58:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:54 smithi130 bash[27579]: cluster 2023-08-27T23:58:53.874837+0000 mgr.y (mgr.14901) 3610 : cluster [DBG] pgmap v3617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:54.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:54 smithi006 bash[27292]: cluster 2023-08-27T23:58:53.874837+0000 mgr.y (mgr.14901) 3610 : cluster [DBG] pgmap v3617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:54 smithi006 bash[33918]: cluster 2023-08-27T23:58:53.874837+0000 mgr.y (mgr.14901) 3610 : cluster [DBG] pgmap v3617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:55 smithi130 bash[51302]: debug 2023-08-27T23:58:55.024+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:55.028499+0000) 2023-08-27T23:58:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:56 smithi130 bash[27579]: cluster 2023-08-27T23:58:55.875877+0000 mgr.y (mgr.14901) 3611 : cluster [DBG] pgmap v3618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:56 smithi130 bash[27579]: audit 2023-08-27T23:58:55.959146+0000 mon.a (mon.0) 2797 : audit [DBG] from='client.? 172.21.15.6:0/1060572154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:56.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:56 smithi130 bash[51302]: debug 2023-08-27T23:58:56.027+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:56.028813+0000) 2023-08-27T23:58:56.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:56 smithi006 bash[27292]: cluster 2023-08-27T23:58:55.875877+0000 mgr.y (mgr.14901) 3611 : cluster [DBG] pgmap v3618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:56 smithi006 bash[27292]: audit 2023-08-27T23:58:55.959146+0000 mon.a (mon.0) 2797 : audit [DBG] from='client.? 172.21.15.6:0/1060572154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:56 smithi006 bash[33918]: cluster 2023-08-27T23:58:55.875877+0000 mgr.y (mgr.14901) 3611 : cluster [DBG] pgmap v3618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:56 smithi006 bash[33918]: audit 2023-08-27T23:58:55.959146+0000 mon.a (mon.0) 2797 : audit [DBG] from='client.? 172.21.15.6:0/1060572154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:57.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:57 smithi130 bash[51302]: debug 2023-08-27T23:58:57.027+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:57.029153+0000) 2023-08-27T23:58:58.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:58 smithi130 bash[51302]: debug 2023-08-27T23:58:58.027+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:58.029527+0000) 2023-08-27T23:58:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:58 smithi006 bash[27292]: cluster 2023-08-27T23:58:57.876496+0000 mgr.y (mgr.14901) 3612 : cluster [DBG] pgmap v3619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:58:58 smithi006 bash[27292]: audit 2023-08-27T23:58:58.381158+0000 mon.a (mon.0) 2798 : audit [DBG] from='client.? 172.21.15.6:0/3208440180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:58 smithi006 bash[33918]: cluster 2023-08-27T23:58:57.876496+0000 mgr.y (mgr.14901) 3612 : cluster [DBG] pgmap v3619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:58:58 smithi006 bash[33918]: audit 2023-08-27T23:58:58.381158+0000 mon.a (mon.0) 2798 : audit [DBG] from='client.? 172.21.15.6:0/3208440180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:58:59 smithi130 bash[51302]: debug 2023-08-27T23:58:59.023+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:59.029848+0000) 2023-08-27T23:58:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:58 smithi130 bash[27579]: cluster 2023-08-27T23:58:57.876496+0000 mgr.y (mgr.14901) 3612 : cluster [DBG] pgmap v3619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:58:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:58:58 smithi130 bash[27579]: audit 2023-08-27T23:58:58.381158+0000 mon.a (mon.0) 2798 : audit [DBG] from='client.? 172.21.15.6:0/3208440180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:00.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:00 smithi130 bash[51302]: debug 2023-08-27T23:59:00.023+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:00.030167+0000) 2023-08-27T23:59:00.941 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:59:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:59:00] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:59:00.941 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:00 smithi006 bash[56298]: level=error ts=2023-08-27T23:59:00.613Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:00.941 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:00 smithi006 bash[56298]: level=warn ts=2023-08-27T23:59:00.614Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:00 smithi006 bash[27292]: cluster 2023-08-27T23:58:59.877630+0000 mgr.y (mgr.14901) 3613 : cluster [DBG] pgmap v3620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:00 smithi006 bash[27292]: audit 2023-08-27T23:59:00.797086+0000 mon.a (mon.0) 2799 : audit [DBG] from='client.? 172.21.15.6:0/1504934780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:00 smithi006 bash[33918]: cluster 2023-08-27T23:58:59.877630+0000 mgr.y (mgr.14901) 3613 : cluster [DBG] pgmap v3620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:00 smithi006 bash[33918]: audit 2023-08-27T23:59:00.797086+0000 mon.a (mon.0) 2799 : audit [DBG] from='client.? 172.21.15.6:0/1504934780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:01.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:00 smithi130 bash[27579]: cluster 2023-08-27T23:58:59.877630+0000 mgr.y (mgr.14901) 3613 : cluster [DBG] pgmap v3620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:01.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:00 smithi130 bash[27579]: audit 2023-08-27T23:59:00.797086+0000 mon.a (mon.0) 2799 : audit [DBG] from='client.? 172.21.15.6:0/1504934780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:01.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:01 smithi130 bash[51302]: debug 2023-08-27T23:59:01.023+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:01.030482+0000) 2023-08-27T23:59:02.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:02 smithi130 bash[51302]: debug 2023-08-27T23:59:02.027+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:02.030818+0000) 2023-08-27T23:59:02.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:02 smithi006 bash[56298]: level=error ts=2023-08-27T23:59:02.185Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:02 smithi006 bash[56298]: level=warn ts=2023-08-27T23:59:02.186Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:03.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:02 smithi006 bash[27292]: cluster 2023-08-27T23:59:01.878322+0000 mgr.y (mgr.14901) 3614 : cluster [DBG] pgmap v3621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:02 smithi006 bash[33918]: cluster 2023-08-27T23:59:01.878322+0000 mgr.y (mgr.14901) 3614 : cluster [DBG] pgmap v3621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:03.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:02 smithi130 bash[27579]: cluster 2023-08-27T23:59:01.878322+0000 mgr.y (mgr.14901) 3614 : cluster [DBG] pgmap v3621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:03 smithi130 bash[51302]: debug 2023-08-27T23:59:03.027+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:03.031157+0000) 2023-08-27T23:59:04.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:03 smithi006 bash[33918]: audit 2023-08-27T23:59:03.214434+0000 mon.a (mon.0) 2800 : audit [DBG] from='client.? 172.21.15.6:0/3273638403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:04.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:03 smithi006 bash[27292]: audit 2023-08-27T23:59:03.214434+0000 mon.a (mon.0) 2800 : audit [DBG] from='client.? 172.21.15.6:0/3273638403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:04 smithi006 bash[56298]: level=error ts=2023-08-27T23:59:04.074Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:04.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:04 smithi006 bash[56298]: level=warn ts=2023-08-27T23:59:04.076Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:04.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:03 smithi130 bash[27579]: audit 2023-08-27T23:59:03.214434+0000 mon.a (mon.0) 2800 : audit [DBG] from='client.? 172.21.15.6:0/3273638403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:04 smithi130 bash[51302]: debug 2023-08-27T23:59:04.027+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:04.031498+0000) 2023-08-27T23:59:05.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:04 smithi006 bash[27292]: cluster 2023-08-27T23:59:03.878979+0000 mgr.y (mgr.14901) 3615 : cluster [DBG] pgmap v3622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:04 smithi006 bash[33918]: cluster 2023-08-27T23:59:03.878979+0000 mgr.y (mgr.14901) 3615 : cluster [DBG] pgmap v3622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:04 smithi130 bash[27579]: cluster 2023-08-27T23:59:03.878979+0000 mgr.y (mgr.14901) 3615 : cluster [DBG] pgmap v3622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:05 smithi130 bash[51302]: debug 2023-08-27T23:59:05.027+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:05.031757+0000) 2023-08-27T23:59:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:06 smithi130 bash[27579]: audit 2023-08-27T23:59:05.625543+0000 mon.a (mon.0) 2801 : audit [DBG] from='client.? 172.21.15.6:0/411277035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:06 smithi130 bash[27579]: cluster 2023-08-27T23:59:05.880235+0000 mgr.y (mgr.14901) 3616 : cluster [DBG] pgmap v3623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:06.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:06 smithi130 bash[51302]: debug 2023-08-27T23:59:06.027+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:06.032085+0000) 2023-08-27T23:59:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:06 smithi006 bash[27292]: audit 2023-08-27T23:59:05.625543+0000 mon.a (mon.0) 2801 : audit [DBG] from='client.? 172.21.15.6:0/411277035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:06 smithi006 bash[27292]: cluster 2023-08-27T23:59:05.880235+0000 mgr.y (mgr.14901) 3616 : cluster [DBG] pgmap v3623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:06 smithi006 bash[33918]: audit 2023-08-27T23:59:05.625543+0000 mon.a (mon.0) 2801 : audit [DBG] from='client.? 172.21.15.6:0/411277035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:06 smithi006 bash[33918]: cluster 2023-08-27T23:59:05.880235+0000 mgr.y (mgr.14901) 3616 : cluster [DBG] pgmap v3623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:07.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:07 smithi130 bash[51302]: debug 2023-08-27T23:59:07.031+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:07.032438+0000) 2023-08-27T23:59:08.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:08 smithi130 bash[51302]: debug 2023-08-27T23:59:08.031+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:08.032803+0000) 2023-08-27T23:59:09.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:08 smithi006 bash[27292]: cluster 2023-08-27T23:59:07.880714+0000 mgr.y (mgr.14901) 3617 : cluster [DBG] pgmap v3624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:08 smithi006 bash[27292]: audit 2023-08-27T23:59:08.040074+0000 mon.a (mon.0) 2802 : audit [DBG] from='client.? 172.21.15.6:0/4130916990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:08 smithi006 bash[33918]: cluster 2023-08-27T23:59:07.880714+0000 mgr.y (mgr.14901) 3617 : cluster [DBG] pgmap v3624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:08 smithi006 bash[33918]: audit 2023-08-27T23:59:08.040074+0000 mon.a (mon.0) 2802 : audit [DBG] from='client.? 172.21.15.6:0/4130916990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:09.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:09 smithi130 bash[51302]: debug 2023-08-27T23:59:09.027+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:09.033157+0000) 2023-08-27T23:59:09.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:08 smithi130 bash[27579]: cluster 2023-08-27T23:59:07.880714+0000 mgr.y (mgr.14901) 3617 : cluster [DBG] pgmap v3624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:09.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:08 smithi130 bash[27579]: audit 2023-08-27T23:59:08.040074+0000 mon.a (mon.0) 2802 : audit [DBG] from='client.? 172.21.15.6:0/4130916990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:10.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:10 smithi130 bash[51302]: debug 2023-08-27T23:59:10.031+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:10.033467+0000) 2023-08-27T23:59:10.945 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:10 smithi006 bash[56298]: level=error ts=2023-08-27T23:59:10.613Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:10.945 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:10 smithi006 bash[56298]: level=warn ts=2023-08-27T23:59:10.616Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:10.945 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:59:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:59:10] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:59:11.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:10 smithi006 bash[33918]: cluster 2023-08-27T23:59:09.881756+0000 mgr.y (mgr.14901) 3618 : cluster [DBG] pgmap v3625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:10 smithi006 bash[33918]: audit 2023-08-27T23:59:10.475613+0000 mon.a (mon.0) 2803 : audit [DBG] from='client.? 172.21.15.6:0/3947197118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:11.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:10 smithi006 bash[27292]: cluster 2023-08-27T23:59:09.881756+0000 mgr.y (mgr.14901) 3618 : cluster [DBG] pgmap v3625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:11.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:10 smithi006 bash[27292]: audit 2023-08-27T23:59:10.475613+0000 mon.a (mon.0) 2803 : audit [DBG] from='client.? 172.21.15.6:0/3947197118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:11.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:10 smithi130 bash[27579]: cluster 2023-08-27T23:59:09.881756+0000 mgr.y (mgr.14901) 3618 : cluster [DBG] pgmap v3625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:11.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:10 smithi130 bash[27579]: audit 2023-08-27T23:59:10.475613+0000 mon.a (mon.0) 2803 : audit [DBG] from='client.? 172.21.15.6:0/3947197118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:11.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:11 smithi130 bash[51302]: debug 2023-08-27T23:59:11.027+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:11.033803+0000) 2023-08-27T23:59:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:12 smithi130 bash[51302]: debug 2023-08-27T23:59:12.031+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:12.034184+0000) 2023-08-27T23:59:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:12 smithi006 bash[56298]: level=error ts=2023-08-27T23:59:12.185Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:12 smithi006 bash[56298]: level=warn ts=2023-08-27T23:59:12.187Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:12 smithi006 bash[33918]: cluster 2023-08-27T23:59:11.882546+0000 mgr.y (mgr.14901) 3619 : cluster [DBG] pgmap v3626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:12 smithi006 bash[33918]: audit 2023-08-27T23:59:12.883979+0000 mon.a (mon.0) 2804 : audit [DBG] from='client.? 172.21.15.6:0/3554744939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:13.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:12 smithi006 bash[27292]: cluster 2023-08-27T23:59:11.882546+0000 mgr.y (mgr.14901) 3619 : cluster [DBG] pgmap v3626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:13.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:12 smithi006 bash[27292]: audit 2023-08-27T23:59:12.883979+0000 mon.a (mon.0) 2804 : audit [DBG] from='client.? 172.21.15.6:0/3554744939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:13 smithi130 bash[51302]: debug 2023-08-27T23:59:13.027+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:13.034518+0000) 2023-08-27T23:59:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:12 smithi130 bash[27579]: cluster 2023-08-27T23:59:11.882546+0000 mgr.y (mgr.14901) 3619 : cluster [DBG] pgmap v3626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:12 smithi130 bash[27579]: audit 2023-08-27T23:59:12.883979+0000 mon.a (mon.0) 2804 : audit [DBG] from='client.? 172.21.15.6:0/3554744939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:14.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:14 smithi130 bash[51302]: debug 2023-08-27T23:59:14.031+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:14.034847+0000) 2023-08-27T23:59:14.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:14 smithi006 bash[56298]: level=error ts=2023-08-27T23:59:14.074Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:14 smithi006 bash[56298]: level=warn ts=2023-08-27T23:59:14.076Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:14 smithi006 bash[33918]: cluster 2023-08-27T23:59:13.883214+0000 mgr.y (mgr.14901) 3620 : cluster [DBG] pgmap v3627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:14 smithi006 bash[27292]: cluster 2023-08-27T23:59:13.883214+0000 mgr.y (mgr.14901) 3620 : cluster [DBG] pgmap v3627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:14 smithi130 bash[27579]: cluster 2023-08-27T23:59:13.883214+0000 mgr.y (mgr.14901) 3620 : cluster [DBG] pgmap v3627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:15 smithi130 bash[51302]: debug 2023-08-27T23:59:15.031+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:15.035168+0000) 2023-08-27T23:59:16.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:16 smithi130 bash[51302]: debug 2023-08-27T23:59:16.031+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:16.035520+0000) 2023-08-27T23:59:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:16 smithi130 bash[27579]: audit 2023-08-27T23:59:15.303535+0000 mon.a (mon.0) 2805 : audit [DBG] from='client.? 172.21.15.6:0/940420403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:16 smithi130 bash[27579]: cluster 2023-08-27T23:59:15.884497+0000 mgr.y (mgr.14901) 3621 : cluster [DBG] pgmap v3628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:16.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:16 smithi006 bash[33918]: audit 2023-08-27T23:59:15.303535+0000 mon.a (mon.0) 2805 : audit [DBG] from='client.? 172.21.15.6:0/940420403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:16 smithi006 bash[33918]: cluster 2023-08-27T23:59:15.884497+0000 mgr.y (mgr.14901) 3621 : cluster [DBG] pgmap v3628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:16 smithi006 bash[27292]: audit 2023-08-27T23:59:15.303535+0000 mon.a (mon.0) 2805 : audit [DBG] from='client.? 172.21.15.6:0/940420403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:16 smithi006 bash[27292]: cluster 2023-08-27T23:59:15.884497+0000 mgr.y (mgr.14901) 3621 : cluster [DBG] pgmap v3628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:17.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:17 smithi130 bash[51302]: debug 2023-08-27T23:59:17.031+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:17.035893+0000) 2023-08-27T23:59:18.037 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:17 smithi130 bash[27579]: audit 2023-08-27T23:59:17.709350+0000 mon.c (mon.2) 1381 : audit [DBG] from='client.? 172.21.15.6:0/3426289048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:18.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:17 smithi006 bash[27292]: audit 2023-08-27T23:59:17.709350+0000 mon.c (mon.2) 1381 : audit [DBG] from='client.? 172.21.15.6:0/3426289048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:18.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:17 smithi006 bash[33918]: audit 2023-08-27T23:59:17.709350+0000 mon.c (mon.2) 1381 : audit [DBG] from='client.? 172.21.15.6:0/3426289048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:18.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:18 smithi130 bash[51302]: debug 2023-08-27T23:59:18.035+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:18.036246+0000) 2023-08-27T23:59:19.038 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:18 smithi130 bash[27579]: cluster 2023-08-27T23:59:17.885363+0000 mgr.y (mgr.14901) 3622 : cluster [DBG] pgmap v3629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:19.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:18 smithi006 bash[27292]: cluster 2023-08-27T23:59:17.885363+0000 mgr.y (mgr.14901) 3622 : cluster [DBG] pgmap v3629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:18 smithi006 bash[33918]: cluster 2023-08-27T23:59:17.885363+0000 mgr.y (mgr.14901) 3622 : cluster [DBG] pgmap v3629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:19.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:19 smithi130 bash[51302]: debug 2023-08-27T23:59:19.031+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:19.036625+0000) 2023-08-27T23:59:20.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:20 smithi130 bash[51302]: debug 2023-08-27T23:59:20.031+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:20.036939+0000) 2023-08-27T23:59:20.945 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:59:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:59:20] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-27T23:59:20.945 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:20 smithi006 bash[56298]: level=error ts=2023-08-27T23:59:20.614Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:20.946 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:20 smithi006 bash[56298]: level=warn ts=2023-08-27T23:59:20.616Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:21.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:20 smithi006 bash[33918]: cluster 2023-08-27T23:59:19.886197+0000 mgr.y (mgr.14901) 3623 : cluster [DBG] pgmap v3630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:21.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:20 smithi006 bash[33918]: audit 2023-08-27T23:59:20.121438+0000 mon.a (mon.0) 2806 : audit [DBG] from='client.? 172.21.15.6:0/2691645586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:20 smithi006 bash[27292]: cluster 2023-08-27T23:59:19.886197+0000 mgr.y (mgr.14901) 3623 : cluster [DBG] pgmap v3630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:20 smithi006 bash[27292]: audit 2023-08-27T23:59:20.121438+0000 mon.a (mon.0) 2806 : audit [DBG] from='client.? 172.21.15.6:0/2691645586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:21.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:20 smithi130 bash[27579]: cluster 2023-08-27T23:59:19.886197+0000 mgr.y (mgr.14901) 3623 : cluster [DBG] pgmap v3630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:21.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:20 smithi130 bash[27579]: audit 2023-08-27T23:59:20.121438+0000 mon.a (mon.0) 2806 : audit [DBG] from='client.? 172.21.15.6:0/2691645586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:21.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:21 smithi130 bash[51302]: debug 2023-08-27T23:59:21.035+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:21.037240+0000) 2023-08-27T23:59:22.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:22 smithi130 bash[51302]: debug 2023-08-27T23:59:22.031+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:22.037490+0000) 2023-08-27T23:59:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:22 smithi006 bash[56298]: level=error ts=2023-08-27T23:59:22.186Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:22 smithi006 bash[56298]: level=warn ts=2023-08-27T23:59:22.188Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:23.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:22 smithi006 bash[33918]: cluster 2023-08-27T23:59:21.886846+0000 mgr.y (mgr.14901) 3624 : cluster [DBG] pgmap v3631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:22 smithi006 bash[33918]: audit 2023-08-27T23:59:22.538393+0000 mon.a (mon.0) 2807 : audit [DBG] from='client.? 172.21.15.6:0/3856918576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:22 smithi006 bash[27292]: cluster 2023-08-27T23:59:21.886846+0000 mgr.y (mgr.14901) 3624 : cluster [DBG] pgmap v3631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:22 smithi006 bash[27292]: audit 2023-08-27T23:59:22.538393+0000 mon.a (mon.0) 2807 : audit [DBG] from='client.? 172.21.15.6:0/3856918576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:23.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:22 smithi130 bash[27579]: cluster 2023-08-27T23:59:21.886846+0000 mgr.y (mgr.14901) 3624 : cluster [DBG] pgmap v3631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:23.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:22 smithi130 bash[27579]: audit 2023-08-27T23:59:22.538393+0000 mon.a (mon.0) 2807 : audit [DBG] from='client.? 172.21.15.6:0/3856918576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:23.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:23 smithi130 bash[51302]: debug 2023-08-27T23:59:23.031+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:23.037759+0000) 2023-08-27T23:59:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:24 smithi130 bash[51302]: debug 2023-08-27T23:59:24.031+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:24.038063+0000) 2023-08-27T23:59:24.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:24 smithi006 bash[56298]: level=error ts=2023-08-27T23:59:24.074Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:24 smithi006 bash[56298]: level=warn ts=2023-08-27T23:59:24.077Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:24 smithi006 bash[33918]: cluster 2023-08-27T23:59:23.887526+0000 mgr.y (mgr.14901) 3625 : cluster [DBG] pgmap v3632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:24 smithi006 bash[27292]: cluster 2023-08-27T23:59:23.887526+0000 mgr.y (mgr.14901) 3625 : cluster [DBG] pgmap v3632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:24 smithi130 bash[27579]: cluster 2023-08-27T23:59:23.887526+0000 mgr.y (mgr.14901) 3625 : cluster [DBG] pgmap v3632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:25 smithi130 bash[51302]: debug 2023-08-27T23:59:25.035+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:25.038329+0000) 2023-08-27T23:59:26.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:25 smithi006 bash[27292]: audit 2023-08-27T23:59:24.952707+0000 mon.a (mon.0) 2808 : audit [DBG] from='client.? 172.21.15.6:0/170081486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:26.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:25 smithi006 bash[33918]: audit 2023-08-27T23:59:24.952707+0000 mon.a (mon.0) 2808 : audit [DBG] from='client.? 172.21.15.6:0/170081486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:25 smithi130 bash[27579]: audit 2023-08-27T23:59:24.952707+0000 mon.a (mon.0) 2808 : audit [DBG] from='client.? 172.21.15.6:0/170081486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:26.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:26 smithi130 bash[51302]: debug 2023-08-27T23:59:26.031+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:26.038618+0000) 2023-08-27T23:59:27.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:26 smithi006 bash[33918]: cluster 2023-08-27T23:59:25.888638+0000 mgr.y (mgr.14901) 3626 : cluster [DBG] pgmap v3633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:27.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:26 smithi006 bash[27292]: cluster 2023-08-27T23:59:25.888638+0000 mgr.y (mgr.14901) 3626 : cluster [DBG] pgmap v3633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:27.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:26 smithi130 bash[27579]: cluster 2023-08-27T23:59:25.888638+0000 mgr.y (mgr.14901) 3626 : cluster [DBG] pgmap v3633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:27.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:27 smithi130 bash[51302]: debug 2023-08-27T23:59:27.035+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:27.038918+0000) 2023-08-27T23:59:28.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:27 smithi130 bash[27579]: audit 2023-08-27T23:59:27.371556+0000 mon.a (mon.0) 2809 : audit [DBG] from='client.? 172.21.15.6:0/1622247777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:28.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:28 smithi130 bash[51302]: debug 2023-08-27T23:59:28.035+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:28.039178+0000) 2023-08-27T23:59:28.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:27 smithi006 bash[33918]: audit 2023-08-27T23:59:27.371556+0000 mon.a (mon.0) 2809 : audit [DBG] from='client.? 172.21.15.6:0/1622247777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:28.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:27 smithi006 bash[27292]: audit 2023-08-27T23:59:27.371556+0000 mon.a (mon.0) 2809 : audit [DBG] from='client.? 172.21.15.6:0/1622247777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:29.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:28 smithi130 bash[27579]: cluster 2023-08-27T23:59:27.889334+0000 mgr.y (mgr.14901) 3627 : cluster [DBG] pgmap v3634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:29.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:29 smithi130 bash[51302]: debug 2023-08-27T23:59:29.035+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:29.039504+0000) 2023-08-27T23:59:29.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:28 smithi006 bash[27292]: cluster 2023-08-27T23:59:27.889334+0000 mgr.y (mgr.14901) 3627 : cluster [DBG] pgmap v3634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:29.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:28 smithi006 bash[33918]: cluster 2023-08-27T23:59:27.889334+0000 mgr.y (mgr.14901) 3627 : cluster [DBG] pgmap v3634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:29 smithi130 bash[27579]: audit 2023-08-27T23:59:29.786899+0000 mon.c (mon.2) 1382 : audit [DBG] from='client.? 172.21.15.6:0/2353623856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:30 smithi130 bash[51302]: debug 2023-08-27T23:59:30.035+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:30.039813+0000) 2023-08-27T23:59:30.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:29 smithi006 bash[33918]: audit 2023-08-27T23:59:29.786899+0000 mon.c (mon.2) 1382 : audit [DBG] from='client.? 172.21.15.6:0/2353623856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:30.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:29 smithi006 bash[27292]: audit 2023-08-27T23:59:29.786899+0000 mon.c (mon.2) 1382 : audit [DBG] from='client.? 172.21.15.6:0/2353623856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:30.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:59:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:59:30] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:59:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:30 smithi006 bash[56298]: level=error ts=2023-08-27T23:59:30.615Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:30 smithi006 bash[56298]: level=warn ts=2023-08-27T23:59:30.618Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:31.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:30 smithi130 bash[27579]: cluster 2023-08-27T23:59:29.890629+0000 mgr.y (mgr.14901) 3628 : cluster [DBG] pgmap v3635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:31.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:31 smithi130 bash[51302]: debug 2023-08-27T23:59:31.035+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:31.040129+0000) 2023-08-27T23:59:31.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:30 smithi006 bash[27292]: cluster 2023-08-27T23:59:29.890629+0000 mgr.y (mgr.14901) 3628 : cluster [DBG] pgmap v3635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:31.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:30 smithi006 bash[33918]: cluster 2023-08-27T23:59:29.890629+0000 mgr.y (mgr.14901) 3628 : cluster [DBG] pgmap v3635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:32.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:32 smithi130 bash[51302]: debug 2023-08-27T23:59:32.039+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:32.040421+0000) 2023-08-27T23:59:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:32 smithi006 bash[56298]: level=error ts=2023-08-27T23:59:32.186Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:32 smithi006 bash[56298]: level=warn ts=2023-08-27T23:59:32.187Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:32 smithi130 bash[27579]: cluster 2023-08-27T23:59:31.891445+0000 mgr.y (mgr.14901) 3629 : cluster [DBG] pgmap v3636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:32 smithi130 bash[27579]: audit 2023-08-27T23:59:32.203313+0000 mon.a (mon.0) 2810 : audit [DBG] from='client.? 172.21.15.6:0/1054462527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:33.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:33 smithi130 bash[51302]: debug 2023-08-27T23:59:33.035+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:33.040681+0000) 2023-08-27T23:59:33.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:32 smithi006 bash[33918]: cluster 2023-08-27T23:59:31.891445+0000 mgr.y (mgr.14901) 3629 : cluster [DBG] pgmap v3636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:33.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:32 smithi006 bash[33918]: audit 2023-08-27T23:59:32.203313+0000 mon.a (mon.0) 2810 : audit [DBG] from='client.? 172.21.15.6:0/1054462527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:33.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:32 smithi006 bash[27292]: cluster 2023-08-27T23:59:31.891445+0000 mgr.y (mgr.14901) 3629 : cluster [DBG] pgmap v3636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:33.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:32 smithi006 bash[27292]: audit 2023-08-27T23:59:32.203313+0000 mon.a (mon.0) 2810 : audit [DBG] from='client.? 172.21.15.6:0/1054462527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:34.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:34 smithi130 bash[51302]: debug 2023-08-27T23:59:34.035+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:34.040943+0000) 2023-08-27T23:59:34.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:34 smithi006 bash[56298]: level=error ts=2023-08-27T23:59:34.075Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:34 smithi006 bash[56298]: level=warn ts=2023-08-27T23:59:34.077Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:34 smithi130 bash[27579]: cluster 2023-08-27T23:59:33.892149+0000 mgr.y (mgr.14901) 3630 : cluster [DBG] pgmap v3637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:34 smithi130 bash[27579]: audit 2023-08-27T23:59:34.623310+0000 mon.c (mon.2) 1383 : audit [DBG] from='client.? 172.21.15.6:0/2330364188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:35.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:35 smithi130 bash[51302]: debug 2023-08-27T23:59:35.035+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:35.041266+0000) 2023-08-27T23:59:35.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:34 smithi006 bash[27292]: cluster 2023-08-27T23:59:33.892149+0000 mgr.y (mgr.14901) 3630 : cluster [DBG] pgmap v3637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:35.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:34 smithi006 bash[27292]: audit 2023-08-27T23:59:34.623310+0000 mon.c (mon.2) 1383 : audit [DBG] from='client.? 172.21.15.6:0/2330364188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:35.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:34 smithi006 bash[33918]: cluster 2023-08-27T23:59:33.892149+0000 mgr.y (mgr.14901) 3630 : cluster [DBG] pgmap v3637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:35.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:34 smithi006 bash[33918]: audit 2023-08-27T23:59:34.623310+0000 mon.c (mon.2) 1383 : audit [DBG] from='client.? 172.21.15.6:0/2330364188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:36.302 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:36 smithi130 bash[51302]: debug 2023-08-27T23:59:36.039+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:36.041589+0000) 2023-08-27T23:59:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:36 smithi130 bash[27579]: cluster 2023-08-27T23:59:35.893340+0000 mgr.y (mgr.14901) 3631 : cluster [DBG] pgmap v3638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:36.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:36 smithi006 bash[27292]: cluster 2023-08-27T23:59:35.893340+0000 mgr.y (mgr.14901) 3631 : cluster [DBG] pgmap v3638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:36 smithi006 bash[33918]: cluster 2023-08-27T23:59:35.893340+0000 mgr.y (mgr.14901) 3631 : cluster [DBG] pgmap v3638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:37.307 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:37 smithi130 bash[51302]: debug 2023-08-27T23:59:37.039+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:37.041859+0000) 2023-08-27T23:59:37.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:37 smithi130 bash[27579]: audit 2023-08-27T23:59:37.039982+0000 mon.c (mon.2) 1384 : audit [DBG] from='client.? 172.21.15.6:0/2715689647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:37.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:37 smithi006 bash[33918]: audit 2023-08-27T23:59:37.039982+0000 mon.c (mon.2) 1384 : audit [DBG] from='client.? 172.21.15.6:0/2715689647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:37.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:37 smithi006 bash[27292]: audit 2023-08-27T23:59:37.039982+0000 mon.c (mon.2) 1384 : audit [DBG] from='client.? 172.21.15.6:0/2715689647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:38.315 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:38 smithi130 bash[51302]: debug 2023-08-27T23:59:38.039+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:38.042181+0000) 2023-08-27T23:59:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:38 smithi130 bash[27579]: cluster 2023-08-27T23:59:37.894027+0000 mgr.y (mgr.14901) 3632 : cluster [DBG] pgmap v3639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:38.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:38 smithi006 bash[33918]: cluster 2023-08-27T23:59:37.894027+0000 mgr.y (mgr.14901) 3632 : cluster [DBG] pgmap v3639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:38.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:38 smithi006 bash[27292]: cluster 2023-08-27T23:59:37.894027+0000 mgr.y (mgr.14901) 3632 : cluster [DBG] pgmap v3639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:39.318 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:39 smithi130 bash[51302]: debug 2023-08-27T23:59:39.035+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:39.042438+0000) 2023-08-27T23:59:39.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:39 smithi130 bash[27579]: audit 2023-08-27T23:59:38.812315+0000 mon.b (mon.1) 460 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:59:39.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:39 smithi006 bash[27292]: audit 2023-08-27T23:59:38.812315+0000 mon.b (mon.1) 460 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:59:39.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:39 smithi006 bash[33918]: audit 2023-08-27T23:59:38.812315+0000 mon.b (mon.1) 460 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:59:40.326 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:40 smithi130 bash[51302]: debug 2023-08-27T23:59:40.035+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:40.042672+0000) 2023-08-27T23:59:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:40 smithi130 bash[27579]: audit 2023-08-27T23:59:39.450018+0000 mon.c (mon.2) 1385 : audit [DBG] from='client.? 172.21.15.6:0/3951887761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:40 smithi130 bash[27579]: cluster 2023-08-27T23:59:39.895128+0000 mgr.y (mgr.14901) 3633 : cluster [DBG] pgmap v3640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:40.618 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:40 smithi006 bash[27292]: audit 2023-08-27T23:59:39.450018+0000 mon.c (mon.2) 1385 : audit [DBG] from='client.? 172.21.15.6:0/3951887761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:40.619 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:40 smithi006 bash[27292]: cluster 2023-08-27T23:59:39.895128+0000 mgr.y (mgr.14901) 3633 : cluster [DBG] pgmap v3640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:40.619 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:40 smithi006 bash[33918]: audit 2023-08-27T23:59:39.450018+0000 mon.c (mon.2) 1385 : audit [DBG] from='client.? 172.21.15.6:0/3951887761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:40.619 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:40 smithi006 bash[33918]: cluster 2023-08-27T23:59:39.895128+0000 mgr.y (mgr.14901) 3633 : cluster [DBG] pgmap v3640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:40.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:59:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:59:40] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-27T23:59:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:40 smithi006 bash[56298]: level=error ts=2023-08-27T23:59:40.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:40 smithi006 bash[56298]: level=warn ts=2023-08-27T23:59:40.618Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:41.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:41 smithi130 bash[51302]: debug 2023-08-27T23:59:41.039+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:41.042971+0000) 2023-08-27T23:59:42.189 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:41 smithi006 bash[27292]: audit 2023-08-27T23:59:41.876265+0000 mon.c (mon.2) 1386 : audit [DBG] from='client.? 172.21.15.6:0/1154284809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:42.189 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:41 smithi006 bash[33918]: audit 2023-08-27T23:59:41.876265+0000 mon.c (mon.2) 1386 : audit [DBG] from='client.? 172.21.15.6:0/1154284809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:41 smithi130 bash[27579]: audit 2023-08-27T23:59:41.876265+0000 mon.c (mon.2) 1386 : audit [DBG] from='client.? 172.21.15.6:0/1154284809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:42.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:42 smithi130 bash[51302]: debug 2023-08-27T23:59:42.039+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:42.043182+0000) 2023-08-27T23:59:42.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:42 smithi006 bash[56298]: level=error ts=2023-08-27T23:59:42.187Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:42 smithi006 bash[56298]: level=warn ts=2023-08-27T23:59:42.189Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:43.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:42 smithi006 bash[27292]: cluster 2023-08-27T23:59:41.895761+0000 mgr.y (mgr.14901) 3634 : cluster [DBG] pgmap v3641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:42 smithi006 bash[33918]: cluster 2023-08-27T23:59:41.895761+0000 mgr.y (mgr.14901) 3634 : cluster [DBG] pgmap v3641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:43.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:42 smithi130 bash[27579]: cluster 2023-08-27T23:59:41.895761+0000 mgr.y (mgr.14901) 3634 : cluster [DBG] pgmap v3641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:43.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:43 smithi130 bash[51302]: debug 2023-08-27T23:59:43.043+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:43.043455+0000) 2023-08-27T23:59:44.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:44 smithi130 bash[51302]: debug 2023-08-27T23:59:44.039+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:44.043672+0000) 2023-08-27T23:59:44.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:44 smithi006 bash[56298]: level=error ts=2023-08-27T23:59:44.075Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:44.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:44 smithi006 bash[56298]: level=warn ts=2023-08-27T23:59:44.076Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:45.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:44 smithi006 bash[33918]: cluster 2023-08-27T23:59:43.896448+0000 mgr.y (mgr.14901) 3635 : cluster [DBG] pgmap v3642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:44 smithi006 bash[33918]: audit 2023-08-27T23:59:44.305900+0000 mon.c (mon.2) 1387 : audit [DBG] from='client.? 172.21.15.6:0/4065077364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:44 smithi006 bash[33918]: audit 2023-08-27T23:59:44.697473+0000 mon.a (mon.0) 2811 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:59:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:44 smithi006 bash[27292]: cluster 2023-08-27T23:59:43.896448+0000 mgr.y (mgr.14901) 3635 : cluster [DBG] pgmap v3642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:44 smithi006 bash[27292]: audit 2023-08-27T23:59:44.305900+0000 mon.c (mon.2) 1387 : audit [DBG] from='client.? 172.21.15.6:0/4065077364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:44 smithi006 bash[27292]: audit 2023-08-27T23:59:44.697473+0000 mon.a (mon.0) 2811 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:59:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:45 smithi130 bash[51302]: debug 2023-08-27T23:59:45.039+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:45.043885+0000) 2023-08-27T23:59:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:44 smithi130 bash[27579]: cluster 2023-08-27T23:59:43.896448+0000 mgr.y (mgr.14901) 3635 : cluster [DBG] pgmap v3642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:44 smithi130 bash[27579]: audit 2023-08-27T23:59:44.305900+0000 mon.c (mon.2) 1387 : audit [DBG] from='client.? 172.21.15.6:0/4065077364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:44 smithi130 bash[27579]: audit 2023-08-27T23:59:44.697473+0000 mon.a (mon.0) 2811 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:59:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:46 smithi130 bash[51302]: debug 2023-08-27T23:59:46.039+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:46.044146+0000) 2023-08-27T23:59:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:46 smithi130 bash[27579]: audit 2023-08-27T23:59:45.163887+0000 mon.a (mon.0) 2812 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:59:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:46 smithi130 bash[27579]: audit 2023-08-27T23:59:45.459484+0000 mon.b (mon.1) 461 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:59:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:46 smithi130 bash[27579]: audit 2023-08-27T23:59:45.461745+0000 mon.b (mon.1) 462 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:59:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:46 smithi130 bash[27579]: audit 2023-08-27T23:59:45.470897+0000 mon.a (mon.0) 2813 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:59:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:46 smithi130 bash[27579]: cluster 2023-08-27T23:59:45.897591+0000 mgr.y (mgr.14901) 3636 : cluster [DBG] pgmap v3643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:46.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:46 smithi006 bash[27292]: audit 2023-08-27T23:59:45.163887+0000 mon.a (mon.0) 2812 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:59:46.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:46 smithi006 bash[27292]: audit 2023-08-27T23:59:45.459484+0000 mon.b (mon.1) 461 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:59:46.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:46 smithi006 bash[27292]: audit 2023-08-27T23:59:45.461745+0000 mon.b (mon.1) 462 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:59:46.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:46 smithi006 bash[27292]: audit 2023-08-27T23:59:45.470897+0000 mon.a (mon.0) 2813 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:59:46.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:46 smithi006 bash[27292]: cluster 2023-08-27T23:59:45.897591+0000 mgr.y (mgr.14901) 3636 : cluster [DBG] pgmap v3643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:46 smithi006 bash[33918]: audit 2023-08-27T23:59:45.163887+0000 mon.a (mon.0) 2812 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:59:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:46 smithi006 bash[33918]: audit 2023-08-27T23:59:45.459484+0000 mon.b (mon.1) 461 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:59:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:46 smithi006 bash[33918]: audit 2023-08-27T23:59:45.461745+0000 mon.b (mon.1) 462 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:59:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:46 smithi006 bash[33918]: audit 2023-08-27T23:59:45.470897+0000 mon.a (mon.0) 2813 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-27T23:59:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:46 smithi006 bash[33918]: cluster 2023-08-27T23:59:45.897591+0000 mgr.y (mgr.14901) 3636 : cluster [DBG] pgmap v3643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:47 smithi130 bash[51302]: debug 2023-08-27T23:59:47.043+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:47.044435+0000) 2023-08-27T23:59:47.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:47 smithi130 bash[27579]: audit 2023-08-27T23:59:46.721992+0000 mon.c (mon.2) 1388 : audit [DBG] from='client.? 172.21.15.6:0/2666477768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:47.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:47 smithi006 bash[33918]: audit 2023-08-27T23:59:46.721992+0000 mon.c (mon.2) 1388 : audit [DBG] from='client.? 172.21.15.6:0/2666477768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:47.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:47 smithi006 bash[27292]: audit 2023-08-27T23:59:46.721992+0000 mon.c (mon.2) 1388 : audit [DBG] from='client.? 172.21.15.6:0/2666477768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:48.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:48 smithi130 bash[51302]: debug 2023-08-27T23:59:48.043+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:48.044711+0000) 2023-08-27T23:59:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:48 smithi130 bash[27579]: cluster 2023-08-27T23:59:47.898297+0000 mgr.y (mgr.14901) 3637 : cluster [DBG] pgmap v3644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:48.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:48 smithi006 bash[27292]: cluster 2023-08-27T23:59:47.898297+0000 mgr.y (mgr.14901) 3637 : cluster [DBG] pgmap v3644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:48 smithi006 bash[33918]: cluster 2023-08-27T23:59:47.898297+0000 mgr.y (mgr.14901) 3637 : cluster [DBG] pgmap v3644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:49 smithi130 bash[51302]: debug 2023-08-27T23:59:49.043+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:49.045042+0000) 2023-08-27T23:59:49.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:49 smithi130 bash[27579]: audit 2023-08-27T23:59:49.131565+0000 mon.a (mon.0) 2814 : audit [DBG] from='client.? 172.21.15.6:0/3160314230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:49.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:49 smithi006 bash[27292]: audit 2023-08-27T23:59:49.131565+0000 mon.a (mon.0) 2814 : audit [DBG] from='client.? 172.21.15.6:0/3160314230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:49.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:49 smithi006 bash[33918]: audit 2023-08-27T23:59:49.131565+0000 mon.a (mon.0) 2814 : audit [DBG] from='client.? 172.21.15.6:0/3160314230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:50 smithi130 bash[51302]: debug 2023-08-27T23:59:50.039+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:50.045315+0000) 2023-08-27T23:59:50.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:50 smithi006 bash[56298]: level=error ts=2023-08-27T23:59:50.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:50.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:50 smithi006 bash[56298]: level=warn ts=2023-08-27T23:59:50.619Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:50.756 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:50 smithi006 bash[33918]: cluster 2023-08-27T23:59:49.899439+0000 mgr.y (mgr.14901) 3638 : cluster [DBG] pgmap v3645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:50.756 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:50 smithi006 bash[27292]: cluster 2023-08-27T23:59:49.899439+0000 mgr.y (mgr.14901) 3638 : cluster [DBG] pgmap v3645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:50 smithi130 bash[27579]: cluster 2023-08-27T23:59:49.899439+0000 mgr.y (mgr.14901) 3638 : cluster [DBG] pgmap v3645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 27 23:59:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [27/Aug/2023:23:59:50] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-27T23:59:51.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:51 smithi130 bash[51302]: debug 2023-08-27T23:59:51.043+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:51.045597+0000) 2023-08-27T23:59:51.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:51 smithi130 bash[27579]: audit 2023-08-27T23:59:51.548310+0000 mon.c (mon.2) 1389 : audit [DBG] from='client.? 172.21.15.6:0/3177862717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:51.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:51 smithi006 bash[33918]: audit 2023-08-27T23:59:51.548310+0000 mon.c (mon.2) 1389 : audit [DBG] from='client.? 172.21.15.6:0/3177862717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:51.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:51 smithi006 bash[27292]: audit 2023-08-27T23:59:51.548310+0000 mon.c (mon.2) 1389 : audit [DBG] from='client.? 172.21.15.6:0/3177862717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:52.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:52 smithi130 bash[51302]: debug 2023-08-27T23:59:52.042+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:52.045886+0000) 2023-08-27T23:59:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:52 smithi006 bash[56298]: level=error ts=2023-08-27T23:59:52.188Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:52 smithi006 bash[56298]: level=warn ts=2023-08-27T23:59:52.190Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:52 smithi130 bash[27579]: cluster 2023-08-27T23:59:51.900126+0000 mgr.y (mgr.14901) 3639 : cluster [DBG] pgmap v3646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:52 smithi006 bash[27292]: cluster 2023-08-27T23:59:51.900126+0000 mgr.y (mgr.14901) 3639 : cluster [DBG] pgmap v3646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:52 smithi006 bash[33918]: cluster 2023-08-27T23:59:51.900126+0000 mgr.y (mgr.14901) 3639 : cluster [DBG] pgmap v3646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:53.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:53 smithi130 bash[51302]: debug 2023-08-27T23:59:53.042+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:53.046172+0000) 2023-08-27T23:59:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:54 smithi130 bash[51302]: debug 2023-08-27T23:59:54.038+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:54.046460+0000) 2023-08-27T23:59:54.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:54 smithi006 bash[56298]: level=error ts=2023-08-27T23:59:54.076Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:54.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 27 23:59:54 smithi006 bash[56298]: level=warn ts=2023-08-27T23:59:54.078Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-27T23:59:55.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:54 smithi006 bash[27292]: cluster 2023-08-27T23:59:53.900761+0000 mgr.y (mgr.14901) 3640 : cluster [DBG] pgmap v3647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:54 smithi006 bash[27292]: audit 2023-08-27T23:59:53.973939+0000 mon.a (mon.0) 2815 : audit [DBG] from='client.? 172.21.15.6:0/1949412583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:54 smithi006 bash[33918]: cluster 2023-08-27T23:59:53.900761+0000 mgr.y (mgr.14901) 3640 : cluster [DBG] pgmap v3647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:54 smithi006 bash[33918]: audit 2023-08-27T23:59:53.973939+0000 mon.a (mon.0) 2815 : audit [DBG] from='client.? 172.21.15.6:0/1949412583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:54 smithi130 bash[27579]: cluster 2023-08-27T23:59:53.900761+0000 mgr.y (mgr.14901) 3640 : cluster [DBG] pgmap v3647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:54 smithi130 bash[27579]: audit 2023-08-27T23:59:53.973939+0000 mon.a (mon.0) 2815 : audit [DBG] from='client.? 172.21.15.6:0/1949412583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:55 smithi130 bash[51302]: debug 2023-08-27T23:59:55.038+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:55.046752+0000) 2023-08-27T23:59:56.300 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:56 smithi130 bash[51302]: debug 2023-08-27T23:59:56.042+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:56.047039+0000) 2023-08-27T23:59:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:56 smithi130 bash[27579]: cluster 2023-08-27T23:59:55.901759+0000 mgr.y (mgr.14901) 3641 : cluster [DBG] pgmap v3648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:56.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:56 smithi006 bash[27292]: cluster 2023-08-27T23:59:55.901759+0000 mgr.y (mgr.14901) 3641 : cluster [DBG] pgmap v3648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:56 smithi006 bash[33918]: cluster 2023-08-27T23:59:55.901759+0000 mgr.y (mgr.14901) 3641 : cluster [DBG] pgmap v3648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:57.303 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:57 smithi130 bash[51302]: debug 2023-08-27T23:59:57.042+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:57.047303+0000) 2023-08-27T23:59:57.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:57 smithi130 bash[27579]: audit 2023-08-27T23:59:56.388266+0000 mon.c (mon.2) 1390 : audit [DBG] from='client.? 172.21.15.6:0/4264026989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:57.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:57 smithi006 bash[27292]: audit 2023-08-27T23:59:56.388266+0000 mon.c (mon.2) 1390 : audit [DBG] from='client.? 172.21.15.6:0/4264026989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:57.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:57 smithi006 bash[33918]: audit 2023-08-27T23:59:56.388266+0000 mon.c (mon.2) 1390 : audit [DBG] from='client.? 172.21.15.6:0/4264026989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:58.311 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:58 smithi130 bash[51302]: debug 2023-08-27T23:59:58.046+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:58.047569+0000) 2023-08-27T23:59:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:58 smithi130 bash[27579]: cluster 2023-08-27T23:59:57.902428+0000 mgr.y (mgr.14901) 3642 : cluster [DBG] pgmap v3649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:58.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:58 smithi006 bash[27292]: cluster 2023-08-27T23:59:57.902428+0000 mgr.y (mgr.14901) 3642 : cluster [DBG] pgmap v3649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:58.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:58 smithi006 bash[33918]: cluster 2023-08-27T23:59:57.902428+0000 mgr.y (mgr.14901) 3642 : cluster [DBG] pgmap v3649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-27T23:59:59.313 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 27 23:59:59 smithi130 bash[51302]: debug 2023-08-27T23:59:59.046+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:59.047882+0000) 2023-08-27T23:59:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 27 23:59:59 smithi130 bash[27579]: audit 2023-08-27T23:59:58.798217+0000 mon.c (mon.2) 1391 : audit [DBG] from='client.? 172.21.15.6:0/3401793925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:59.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 27 23:59:59 smithi006 bash[27292]: audit 2023-08-27T23:59:58.798217+0000 mon.c (mon.2) 1391 : audit [DBG] from='client.? 172.21.15.6:0/3401793925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:59.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 27 23:59:59 smithi006 bash[33918]: audit 2023-08-27T23:59:58.798217+0000 mon.c (mon.2) 1391 : audit [DBG] from='client.? 172.21.15.6:0/3401793925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:00.322 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:00 smithi130 bash[51302]: debug 2023-08-28T00:00:00.046+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:00.048204+0000) 2023-08-28T00:00:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:00 smithi130 bash[27579]: cluster 2023-08-27T23:59:59.903660+0000 mgr.y (mgr.14901) 3643 : cluster [DBG] pgmap v3650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:00 smithi130 bash[27579]: cluster 2023-08-28T00:00:00.000150+0000 mon.a (mon.0) 2816 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:00:00.620 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:00 smithi006 bash[27292]: cluster 2023-08-27T23:59:59.903660+0000 mgr.y (mgr.14901) 3643 : cluster [DBG] pgmap v3650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:00.620 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:00 smithi006 bash[27292]: cluster 2023-08-28T00:00:00.000150+0000 mon.a (mon.0) 2816 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:00:00.621 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:00 smithi006 bash[33918]: cluster 2023-08-27T23:59:59.903660+0000 mgr.y (mgr.14901) 3643 : cluster [DBG] pgmap v3650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:00.621 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:00 smithi006 bash[33918]: cluster 2023-08-28T00:00:00.000150+0000 mon.a (mon.0) 2816 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:00:00.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:00:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:00:00] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T00:00:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:00:00.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:00:00.620Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:01.323 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:01 smithi130 bash[51302]: debug 2023-08-28T00:00:01.046+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:01.048480+0000) 2023-08-28T00:00:01.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:01 smithi130 bash[27579]: audit 2023-08-28T00:00:01.218782+0000 mon.c (mon.2) 1392 : audit [DBG] from='client.? 172.21.15.6:0/376198823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:01.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:01 smithi006 bash[33918]: audit 2023-08-28T00:00:01.218782+0000 mon.c (mon.2) 1392 : audit [DBG] from='client.? 172.21.15.6:0/376198823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:01.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:01 smithi006 bash[27292]: audit 2023-08-28T00:00:01.218782+0000 mon.c (mon.2) 1392 : audit [DBG] from='client.? 172.21.15.6:0/376198823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:02.330 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:02 smithi130 bash[51302]: debug 2023-08-28T00:00:02.046+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:02.048754+0000) 2023-08-28T00:00:02.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:02 smithi006 bash[33918]: cluster 2023-08-28T00:00:01.904327+0000 mgr.y (mgr.14901) 3644 : cluster [DBG] pgmap v3651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:02.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:02 smithi006 bash[27292]: cluster 2023-08-28T00:00:01.904327+0000 mgr.y (mgr.14901) 3644 : cluster [DBG] pgmap v3651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:00:02.189Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:00:02.191Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:02 smithi130 bash[27579]: cluster 2023-08-28T00:00:01.904327+0000 mgr.y (mgr.14901) 3644 : cluster [DBG] pgmap v3651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:03 smithi130 bash[51302]: debug 2023-08-28T00:00:03.046+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:03.049027+0000) 2023-08-28T00:00:03.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:03 smithi006 bash[33918]: audit 2023-08-28T00:00:03.633384+0000 mon.c (mon.2) 1393 : audit [DBG] from='client.? 172.21.15.6:0/1807135813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:03.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:03 smithi006 bash[27292]: audit 2023-08-28T00:00:03.633384+0000 mon.c (mon.2) 1393 : audit [DBG] from='client.? 172.21.15.6:0/1807135813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:04.050 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:03 smithi130 bash[27579]: audit 2023-08-28T00:00:03.633384+0000 mon.c (mon.2) 1393 : audit [DBG] from='client.? 172.21.15.6:0/1807135813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:04 smithi130 bash[51302]: debug 2023-08-28T00:00:04.046+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:04.049314+0000) 2023-08-28T00:00:04.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:00:04.076Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:00:04.078Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:04.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:04 smithi006 bash[27292]: cluster 2023-08-28T00:00:03.905014+0000 mgr.y (mgr.14901) 3645 : cluster [DBG] pgmap v3652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:04 smithi006 bash[33918]: cluster 2023-08-28T00:00:03.905014+0000 mgr.y (mgr.14901) 3645 : cluster [DBG] pgmap v3652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:05.051 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:04 smithi130 bash[27579]: cluster 2023-08-28T00:00:03.905014+0000 mgr.y (mgr.14901) 3645 : cluster [DBG] pgmap v3652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:05 smithi130 bash[51302]: debug 2023-08-28T00:00:05.046+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:05.049609+0000) 2023-08-28T00:00:06.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:06 smithi130 bash[51302]: debug 2023-08-28T00:00:06.046+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:06.049891+0000) 2023-08-28T00:00:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:06 smithi130 bash[27579]: cluster 2023-08-28T00:00:05.906213+0000 mgr.y (mgr.14901) 3646 : cluster [DBG] pgmap v3653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:06 smithi130 bash[27579]: audit 2023-08-28T00:00:06.046581+0000 mon.c (mon.2) 1394 : audit [DBG] from='client.? 172.21.15.6:0/4108596302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:06.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:06 smithi006 bash[33918]: cluster 2023-08-28T00:00:05.906213+0000 mgr.y (mgr.14901) 3646 : cluster [DBG] pgmap v3653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:06 smithi006 bash[33918]: audit 2023-08-28T00:00:06.046581+0000 mon.c (mon.2) 1394 : audit [DBG] from='client.? 172.21.15.6:0/4108596302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:06 smithi006 bash[27292]: cluster 2023-08-28T00:00:05.906213+0000 mgr.y (mgr.14901) 3646 : cluster [DBG] pgmap v3653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:06 smithi006 bash[27292]: audit 2023-08-28T00:00:06.046581+0000 mon.c (mon.2) 1394 : audit [DBG] from='client.? 172.21.15.6:0/4108596302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:07.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:07 smithi130 bash[51302]: debug 2023-08-28T00:00:07.046+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:07.050163+0000) 2023-08-28T00:00:08.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:08 smithi130 bash[51302]: debug 2023-08-28T00:00:08.046+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:08.050438+0000) 2023-08-28T00:00:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:08 smithi006 bash[27292]: cluster 2023-08-28T00:00:07.906927+0000 mgr.y (mgr.14901) 3647 : cluster [DBG] pgmap v3654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:08 smithi006 bash[27292]: audit 2023-08-28T00:00:08.461527+0000 mon.a (mon.0) 2817 : audit [DBG] from='client.? 172.21.15.6:0/1132968132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:08 smithi006 bash[33918]: cluster 2023-08-28T00:00:07.906927+0000 mgr.y (mgr.14901) 3647 : cluster [DBG] pgmap v3654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:08 smithi006 bash[33918]: audit 2023-08-28T00:00:08.461527+0000 mon.a (mon.0) 2817 : audit [DBG] from='client.? 172.21.15.6:0/1132968132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:09.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:09 smithi130 bash[51302]: debug 2023-08-28T00:00:09.046+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:09.050729+0000) 2023-08-28T00:00:09.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:08 smithi130 bash[27579]: cluster 2023-08-28T00:00:07.906927+0000 mgr.y (mgr.14901) 3647 : cluster [DBG] pgmap v3654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:09.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:08 smithi130 bash[27579]: audit 2023-08-28T00:00:08.461527+0000 mon.a (mon.0) 2817 : audit [DBG] from='client.? 172.21.15.6:0/1132968132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:10.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:10 smithi130 bash[51302]: debug 2023-08-28T00:00:10.050+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:10.051035+0000) 2023-08-28T00:00:10.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:00:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:00:10] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T00:00:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:00:10.619Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:00:10.620Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:11.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:11 smithi130 bash[51302]: debug 2023-08-28T00:00:11.050+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:11.051343+0000) 2023-08-28T00:00:11.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:10 smithi130 bash[27579]: cluster 2023-08-28T00:00:09.908028+0000 mgr.y (mgr.14901) 3648 : cluster [DBG] pgmap v3655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:11.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:10 smithi130 bash[27579]: audit 2023-08-28T00:00:10.878903+0000 mon.c (mon.2) 1395 : audit [DBG] from='client.? 172.21.15.6:0/2317459820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:11.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:10 smithi006 bash[27292]: cluster 2023-08-28T00:00:09.908028+0000 mgr.y (mgr.14901) 3648 : cluster [DBG] pgmap v3655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:11.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:10 smithi006 bash[27292]: audit 2023-08-28T00:00:10.878903+0000 mon.c (mon.2) 1395 : audit [DBG] from='client.? 172.21.15.6:0/2317459820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:11.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:10 smithi006 bash[33918]: cluster 2023-08-28T00:00:09.908028+0000 mgr.y (mgr.14901) 3648 : cluster [DBG] pgmap v3655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:11.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:10 smithi006 bash[33918]: audit 2023-08-28T00:00:10.878903+0000 mon.c (mon.2) 1395 : audit [DBG] from='client.? 172.21.15.6:0/2317459820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:12.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:12 smithi130 bash[51302]: debug 2023-08-28T00:00:12.050+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:12.051558+0000) 2023-08-28T00:00:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:00:12.190Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:00:12.192Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:13.245 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:12 smithi130 bash[27579]: cluster 2023-08-28T00:00:11.908711+0000 mgr.y (mgr.14901) 3649 : cluster [DBG] pgmap v3656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:13.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:12 smithi006 bash[27292]: cluster 2023-08-28T00:00:11.908711+0000 mgr.y (mgr.14901) 3649 : cluster [DBG] pgmap v3656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:13.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:12 smithi006 bash[33918]: cluster 2023-08-28T00:00:11.908711+0000 mgr.y (mgr.14901) 3649 : cluster [DBG] pgmap v3656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:13.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:13 smithi130 bash[51302]: debug 2023-08-28T00:00:13.050+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:13.051851+0000) 2023-08-28T00:00:14.301 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:14 smithi130 bash[51302]: debug 2023-08-28T00:00:14.050+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:14.052138+0000) 2023-08-28T00:00:14.301 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:13 smithi130 bash[27579]: audit 2023-08-28T00:00:13.298147+0000 mon.a (mon.0) 2818 : audit [DBG] from='client.? 172.21.15.6:0/1385041834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:14.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:13 smithi006 bash[27292]: audit 2023-08-28T00:00:13.298147+0000 mon.a (mon.0) 2818 : audit [DBG] from='client.? 172.21.15.6:0/1385041834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:14.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:13 smithi006 bash[33918]: audit 2023-08-28T00:00:13.298147+0000 mon.a (mon.0) 2818 : audit [DBG] from='client.? 172.21.15.6:0/1385041834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:00:14.076Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:00:14.078Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:14 smithi130 bash[27579]: cluster 2023-08-28T00:00:13.909333+0000 mgr.y (mgr.14901) 3650 : cluster [DBG] pgmap v3657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:15 smithi130 bash[51302]: debug 2023-08-28T00:00:15.050+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:15.052416+0000) 2023-08-28T00:00:15.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:14 smithi006 bash[27292]: cluster 2023-08-28T00:00:13.909333+0000 mgr.y (mgr.14901) 3650 : cluster [DBG] pgmap v3657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:15.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:14 smithi006 bash[33918]: cluster 2023-08-28T00:00:13.909333+0000 mgr.y (mgr.14901) 3650 : cluster [DBG] pgmap v3657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:16 smithi130 bash[27579]: audit 2023-08-28T00:00:15.713566+0000 mon.a (mon.0) 2819 : audit [DBG] from='client.? 172.21.15.6:0/1579586818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:16 smithi130 bash[27579]: cluster 2023-08-28T00:00:15.910489+0000 mgr.y (mgr.14901) 3651 : cluster [DBG] pgmap v3658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:16.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:16 smithi130 bash[51302]: debug 2023-08-28T00:00:16.050+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:16.052698+0000) 2023-08-28T00:00:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:16 smithi006 bash[33918]: audit 2023-08-28T00:00:15.713566+0000 mon.a (mon.0) 2819 : audit [DBG] from='client.? 172.21.15.6:0/1579586818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:16 smithi006 bash[33918]: cluster 2023-08-28T00:00:15.910489+0000 mgr.y (mgr.14901) 3651 : cluster [DBG] pgmap v3658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:16 smithi006 bash[27292]: audit 2023-08-28T00:00:15.713566+0000 mon.a (mon.0) 2819 : audit [DBG] from='client.? 172.21.15.6:0/1579586818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:16 smithi006 bash[27292]: cluster 2023-08-28T00:00:15.910489+0000 mgr.y (mgr.14901) 3651 : cluster [DBG] pgmap v3658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:17.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:17 smithi130 bash[51302]: debug 2023-08-28T00:00:17.050+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:17.052993+0000) 2023-08-28T00:00:18.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:18 smithi130 bash[51302]: debug 2023-08-28T00:00:18.050+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:18.053324+0000) 2023-08-28T00:00:19.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:19 smithi130 bash[51302]: debug 2023-08-28T00:00:19.050+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:19.053663+0000) 2023-08-28T00:00:19.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:18 smithi130 bash[27579]: cluster 2023-08-28T00:00:17.911145+0000 mgr.y (mgr.14901) 3652 : cluster [DBG] pgmap v3659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:19.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:18 smithi130 bash[27579]: audit 2023-08-28T00:00:18.120277+0000 mon.a (mon.0) 2820 : audit [DBG] from='client.? 172.21.15.6:0/827524909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:19.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:18 smithi006 bash[33918]: cluster 2023-08-28T00:00:17.911145+0000 mgr.y (mgr.14901) 3652 : cluster [DBG] pgmap v3659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:19.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:18 smithi006 bash[33918]: audit 2023-08-28T00:00:18.120277+0000 mon.a (mon.0) 2820 : audit [DBG] from='client.? 172.21.15.6:0/827524909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:19.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:18 smithi006 bash[27292]: cluster 2023-08-28T00:00:17.911145+0000 mgr.y (mgr.14901) 3652 : cluster [DBG] pgmap v3659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:19.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:18 smithi006 bash[27292]: audit 2023-08-28T00:00:18.120277+0000 mon.a (mon.0) 2820 : audit [DBG] from='client.? 172.21.15.6:0/827524909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:20.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:20 smithi130 bash[51302]: debug 2023-08-28T00:00:20.050+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:20.053930+0000) 2023-08-28T00:00:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:00:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:00:20] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:00:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:00:20.619Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:00:20.621Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:21.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:21 smithi130 bash[51302]: debug 2023-08-28T00:00:21.050+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:21.054295+0000) 2023-08-28T00:00:21.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:20 smithi130 bash[27579]: cluster 2023-08-28T00:00:19.912263+0000 mgr.y (mgr.14901) 3653 : cluster [DBG] pgmap v3660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:21.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:20 smithi130 bash[27579]: audit 2023-08-28T00:00:20.541785+0000 mon.a (mon.0) 2821 : audit [DBG] from='client.? 172.21.15.6:0/4281323807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:21.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:20 smithi006 bash[27292]: cluster 2023-08-28T00:00:19.912263+0000 mgr.y (mgr.14901) 3653 : cluster [DBG] pgmap v3660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:21.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:20 smithi006 bash[27292]: audit 2023-08-28T00:00:20.541785+0000 mon.a (mon.0) 2821 : audit [DBG] from='client.? 172.21.15.6:0/4281323807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:21.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:20 smithi006 bash[33918]: cluster 2023-08-28T00:00:19.912263+0000 mgr.y (mgr.14901) 3653 : cluster [DBG] pgmap v3660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:21.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:20 smithi006 bash[33918]: audit 2023-08-28T00:00:20.541785+0000 mon.a (mon.0) 2821 : audit [DBG] from='client.? 172.21.15.6:0/4281323807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:22.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:22 smithi130 bash[51302]: debug 2023-08-28T00:00:22.050+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:22.054576+0000) 2023-08-28T00:00:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:00:22.191Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:00:22.192Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:23.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:22 smithi130 bash[27579]: cluster 2023-08-28T00:00:21.912953+0000 mgr.y (mgr.14901) 3654 : cluster [DBG] pgmap v3661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:23.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:22 smithi130 bash[27579]: audit 2023-08-28T00:00:22.961891+0000 mon.a (mon.0) 2822 : audit [DBG] from='client.? 172.21.15.6:0/3833645959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:23.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:23 smithi130 bash[51302]: debug 2023-08-28T00:00:23.054+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:23.054885+0000) 2023-08-28T00:00:23.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:22 smithi006 bash[27292]: cluster 2023-08-28T00:00:21.912953+0000 mgr.y (mgr.14901) 3654 : cluster [DBG] pgmap v3661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:23.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:22 smithi006 bash[27292]: audit 2023-08-28T00:00:22.961891+0000 mon.a (mon.0) 2822 : audit [DBG] from='client.? 172.21.15.6:0/3833645959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:23.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:22 smithi006 bash[33918]: cluster 2023-08-28T00:00:21.912953+0000 mgr.y (mgr.14901) 3654 : cluster [DBG] pgmap v3661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:23.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:22 smithi006 bash[33918]: audit 2023-08-28T00:00:22.961891+0000 mon.a (mon.0) 2822 : audit [DBG] from='client.? 172.21.15.6:0/3833645959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:24 smithi130 bash[51302]: debug 2023-08-28T00:00:24.054+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:24.055174+0000) 2023-08-28T00:00:24.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:00:24.078Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:00:24.080Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:24 smithi130 bash[27579]: cluster 2023-08-28T00:00:23.913577+0000 mgr.y (mgr.14901) 3655 : cluster [DBG] pgmap v3662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:25 smithi130 bash[51302]: debug 2023-08-28T00:00:25.054+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:25.055475+0000) 2023-08-28T00:00:25.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:24 smithi006 bash[27292]: cluster 2023-08-28T00:00:23.913577+0000 mgr.y (mgr.14901) 3655 : cluster [DBG] pgmap v3662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:25.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:24 smithi006 bash[33918]: cluster 2023-08-28T00:00:23.913577+0000 mgr.y (mgr.14901) 3655 : cluster [DBG] pgmap v3662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:26.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:26 smithi130 bash[51302]: debug 2023-08-28T00:00:26.054+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:26.055800+0000) 2023-08-28T00:00:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:26 smithi130 bash[27579]: audit 2023-08-28T00:00:25.382283+0000 mon.b (mon.1) 463 : audit [DBG] from='client.? 172.21.15.6:0/3920759339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:26 smithi130 bash[27579]: cluster 2023-08-28T00:00:25.914783+0000 mgr.y (mgr.14901) 3656 : cluster [DBG] pgmap v3663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:26.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:26 smithi006 bash[33918]: audit 2023-08-28T00:00:25.382283+0000 mon.b (mon.1) 463 : audit [DBG] from='client.? 172.21.15.6:0/3920759339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:26 smithi006 bash[33918]: cluster 2023-08-28T00:00:25.914783+0000 mgr.y (mgr.14901) 3656 : cluster [DBG] pgmap v3663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:26 smithi006 bash[27292]: audit 2023-08-28T00:00:25.382283+0000 mon.b (mon.1) 463 : audit [DBG] from='client.? 172.21.15.6:0/3920759339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:26 smithi006 bash[27292]: cluster 2023-08-28T00:00:25.914783+0000 mgr.y (mgr.14901) 3656 : cluster [DBG] pgmap v3663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:27.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:27 smithi130 bash[51302]: debug 2023-08-28T00:00:27.054+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:27.056069+0000) 2023-08-28T00:00:28.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:28 smithi130 bash[51302]: debug 2023-08-28T00:00:28.054+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:28.056350+0000) 2023-08-28T00:00:28.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:27 smithi130 bash[27579]: audit 2023-08-28T00:00:27.783071+0000 mon.c (mon.2) 1396 : audit [DBG] from='client.? 172.21.15.6:0/1077760315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:28.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:27 smithi006 bash[33918]: audit 2023-08-28T00:00:27.783071+0000 mon.c (mon.2) 1396 : audit [DBG] from='client.? 172.21.15.6:0/1077760315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:28.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:27 smithi006 bash[27292]: audit 2023-08-28T00:00:27.783071+0000 mon.c (mon.2) 1396 : audit [DBG] from='client.? 172.21.15.6:0/1077760315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:28 smithi130 bash[27579]: cluster 2023-08-28T00:00:27.915397+0000 mgr.y (mgr.14901) 3657 : cluster [DBG] pgmap v3664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:29.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:29 smithi130 bash[51302]: debug 2023-08-28T00:00:29.054+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:29.056641+0000) 2023-08-28T00:00:29.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:28 smithi006 bash[27292]: cluster 2023-08-28T00:00:27.915397+0000 mgr.y (mgr.14901) 3657 : cluster [DBG] pgmap v3664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:28 smithi006 bash[33918]: cluster 2023-08-28T00:00:27.915397+0000 mgr.y (mgr.14901) 3657 : cluster [DBG] pgmap v3664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:30 smithi130 bash[51302]: debug 2023-08-28T00:00:30.054+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:30.056947+0000) 2023-08-28T00:00:30.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:00:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:00:30] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:00:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:00:30.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:00:30.622Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:31.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:30 smithi130 bash[27579]: cluster 2023-08-28T00:00:29.916593+0000 mgr.y (mgr.14901) 3658 : cluster [DBG] pgmap v3665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:31.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:30 smithi130 bash[27579]: audit 2023-08-28T00:00:30.200472+0000 mon.a (mon.0) 2823 : audit [DBG] from='client.? 172.21.15.6:0/1698435746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:31.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:31 smithi130 bash[51302]: debug 2023-08-28T00:00:31.054+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:31.057228+0000) 2023-08-28T00:00:31.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:30 smithi006 bash[33918]: cluster 2023-08-28T00:00:29.916593+0000 mgr.y (mgr.14901) 3658 : cluster [DBG] pgmap v3665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:31.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:30 smithi006 bash[33918]: audit 2023-08-28T00:00:30.200472+0000 mon.a (mon.0) 2823 : audit [DBG] from='client.? 172.21.15.6:0/1698435746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:31.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:30 smithi006 bash[27292]: cluster 2023-08-28T00:00:29.916593+0000 mgr.y (mgr.14901) 3658 : cluster [DBG] pgmap v3665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:31.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:30 smithi006 bash[27292]: audit 2023-08-28T00:00:30.200472+0000 mon.a (mon.0) 2823 : audit [DBG] from='client.? 172.21.15.6:0/1698435746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:32.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:32 smithi130 bash[51302]: debug 2023-08-28T00:00:32.054+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:32.057522+0000) 2023-08-28T00:00:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:00:32.191Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:00:32.193Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:33.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:33 smithi130 bash[51302]: debug 2023-08-28T00:00:33.054+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:33.057784+0000) 2023-08-28T00:00:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:32 smithi130 bash[27579]: cluster 2023-08-28T00:00:31.917406+0000 mgr.y (mgr.14901) 3659 : cluster [DBG] pgmap v3666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:32 smithi130 bash[27579]: audit 2023-08-28T00:00:32.613251+0000 mon.b (mon.1) 464 : audit [DBG] from='client.? 172.21.15.6:0/1191865099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:33.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:32 smithi006 bash[33918]: cluster 2023-08-28T00:00:31.917406+0000 mgr.y (mgr.14901) 3659 : cluster [DBG] pgmap v3666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:33.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:32 smithi006 bash[33918]: audit 2023-08-28T00:00:32.613251+0000 mon.b (mon.1) 464 : audit [DBG] from='client.? 172.21.15.6:0/1191865099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:33.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:32 smithi006 bash[27292]: cluster 2023-08-28T00:00:31.917406+0000 mgr.y (mgr.14901) 3659 : cluster [DBG] pgmap v3666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:33.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:32 smithi006 bash[27292]: audit 2023-08-28T00:00:32.613251+0000 mon.b (mon.1) 464 : audit [DBG] from='client.? 172.21.15.6:0/1191865099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:34.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:34 smithi130 bash[51302]: debug 2023-08-28T00:00:34.054+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:34.058074+0000) 2023-08-28T00:00:34.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:00:34.078Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:00:34.081Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:34 smithi130 bash[27579]: cluster 2023-08-28T00:00:33.918202+0000 mgr.y (mgr.14901) 3660 : cluster [DBG] pgmap v3667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:35.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:35 smithi130 bash[51302]: debug 2023-08-28T00:00:35.058+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:35.058389+0000) 2023-08-28T00:00:35.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:34 smithi006 bash[27292]: cluster 2023-08-28T00:00:33.918202+0000 mgr.y (mgr.14901) 3660 : cluster [DBG] pgmap v3667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:35.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:34 smithi006 bash[33918]: cluster 2023-08-28T00:00:33.918202+0000 mgr.y (mgr.14901) 3660 : cluster [DBG] pgmap v3667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:35 smithi130 bash[27579]: audit 2023-08-28T00:00:35.033286+0000 mon.c (mon.2) 1397 : audit [DBG] from='client.? 172.21.15.6:0/1072714612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:36.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:36 smithi130 bash[51302]: debug 2023-08-28T00:00:36.058+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:36.058668+0000) 2023-08-28T00:00:36.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:35 smithi006 bash[27292]: audit 2023-08-28T00:00:35.033286+0000 mon.c (mon.2) 1397 : audit [DBG] from='client.? 172.21.15.6:0/1072714612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:36.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:35 smithi006 bash[33918]: audit 2023-08-28T00:00:35.033286+0000 mon.c (mon.2) 1397 : audit [DBG] from='client.? 172.21.15.6:0/1072714612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:37.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:36 smithi130 bash[27579]: cluster 2023-08-28T00:00:35.919401+0000 mgr.y (mgr.14901) 3661 : cluster [DBG] pgmap v3668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:37 smithi130 bash[51302]: debug 2023-08-28T00:00:37.058+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:37.058965+0000) 2023-08-28T00:00:37.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:36 smithi006 bash[27292]: cluster 2023-08-28T00:00:35.919401+0000 mgr.y (mgr.14901) 3661 : cluster [DBG] pgmap v3668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:37.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:36 smithi006 bash[33918]: cluster 2023-08-28T00:00:35.919401+0000 mgr.y (mgr.14901) 3661 : cluster [DBG] pgmap v3668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:38.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:38 smithi130 bash[51302]: debug 2023-08-28T00:00:38.058+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:38.059260+0000) 2023-08-28T00:00:38.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:38 smithi130 bash[27579]: audit 2023-08-28T00:00:37.452542+0000 mon.a (mon.0) 2824 : audit [DBG] from='client.? 172.21.15.6:0/734825467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:38.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:37 smithi006 bash[27292]: audit 2023-08-28T00:00:37.452542+0000 mon.a (mon.0) 2824 : audit [DBG] from='client.? 172.21.15.6:0/734825467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:38.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:38 smithi006 bash[33918]: audit 2023-08-28T00:00:37.452542+0000 mon.a (mon.0) 2824 : audit [DBG] from='client.? 172.21.15.6:0/734825467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:39 smithi130 bash[27579]: cluster 2023-08-28T00:00:37.920074+0000 mgr.y (mgr.14901) 3662 : cluster [DBG] pgmap v3669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:39.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:39 smithi130 bash[51302]: debug 2023-08-28T00:00:39.058+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:39.059638+0000) 2023-08-28T00:00:39.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:39 smithi006 bash[27292]: cluster 2023-08-28T00:00:37.920074+0000 mgr.y (mgr.14901) 3662 : cluster [DBG] pgmap v3669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:39.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:39 smithi006 bash[33918]: cluster 2023-08-28T00:00:37.920074+0000 mgr.y (mgr.14901) 3662 : cluster [DBG] pgmap v3669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:40 smithi130 bash[27579]: audit 2023-08-28T00:00:39.859227+0000 mon.a (mon.0) 2825 : audit [DBG] from='client.? 172.21.15.6:0/1030369805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:40.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:40 smithi130 bash[51302]: debug 2023-08-28T00:00:40.058+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:40.059941+0000) 2023-08-28T00:00:40.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:40 smithi006 bash[33918]: audit 2023-08-28T00:00:39.859227+0000 mon.a (mon.0) 2825 : audit [DBG] from='client.? 172.21.15.6:0/1030369805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:40.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:40 smithi006 bash[27292]: audit 2023-08-28T00:00:39.859227+0000 mon.a (mon.0) 2825 : audit [DBG] from='client.? 172.21.15.6:0/1030369805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:40.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:00:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:00:40] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:00:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:00:40.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:00:40.624Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:41.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:41 smithi130 bash[27579]: cluster 2023-08-28T00:00:39.921179+0000 mgr.y (mgr.14901) 3663 : cluster [DBG] pgmap v3670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:41.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:41 smithi130 bash[51302]: debug 2023-08-28T00:00:41.058+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:41.060212+0000) 2023-08-28T00:00:41.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:41 smithi006 bash[27292]: cluster 2023-08-28T00:00:39.921179+0000 mgr.y (mgr.14901) 3663 : cluster [DBG] pgmap v3670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:41.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:41 smithi006 bash[33918]: cluster 2023-08-28T00:00:39.921179+0000 mgr.y (mgr.14901) 3663 : cluster [DBG] pgmap v3670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:42.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:42 smithi130 bash[51302]: debug 2023-08-28T00:00:42.058+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:42.060484+0000) 2023-08-28T00:00:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:00:42.192Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:00:42.194Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:43.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:43 smithi130 bash[27579]: cluster 2023-08-28T00:00:41.921779+0000 mgr.y (mgr.14901) 3664 : cluster [DBG] pgmap v3671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:43.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:43 smithi130 bash[27579]: audit 2023-08-28T00:00:42.272951+0000 mon.c (mon.2) 1398 : audit [DBG] from='client.? 172.21.15.6:0/4124807657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:43.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:43 smithi130 bash[51302]: debug 2023-08-28T00:00:43.058+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:43.060803+0000) 2023-08-28T00:00:43.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:43 smithi006 bash[27292]: cluster 2023-08-28T00:00:41.921779+0000 mgr.y (mgr.14901) 3664 : cluster [DBG] pgmap v3671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:43.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:43 smithi006 bash[27292]: audit 2023-08-28T00:00:42.272951+0000 mon.c (mon.2) 1398 : audit [DBG] from='client.? 172.21.15.6:0/4124807657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:43.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:43 smithi006 bash[33918]: cluster 2023-08-28T00:00:41.921779+0000 mgr.y (mgr.14901) 3664 : cluster [DBG] pgmap v3671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:43.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:43 smithi006 bash[33918]: audit 2023-08-28T00:00:42.272951+0000 mon.c (mon.2) 1398 : audit [DBG] from='client.? 172.21.15.6:0/4124807657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:44.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:44 smithi130 bash[51302]: debug 2023-08-28T00:00:44.058+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:44.061083+0000) 2023-08-28T00:00:44.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:00:44.079Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:00:44.081Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:45 smithi130 bash[51302]: debug 2023-08-28T00:00:45.058+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:45.061349+0000) 2023-08-28T00:00:45.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:45 smithi130 bash[27579]: cluster 2023-08-28T00:00:43.922413+0000 mgr.y (mgr.14901) 3665 : cluster [DBG] pgmap v3672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:45.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:45 smithi130 bash[27579]: audit 2023-08-28T00:00:44.687034+0000 mon.a (mon.0) 2826 : audit [DBG] from='client.? 172.21.15.6:0/3875485397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:45.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:45 smithi006 bash[27292]: cluster 2023-08-28T00:00:43.922413+0000 mgr.y (mgr.14901) 3665 : cluster [DBG] pgmap v3672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:45.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:45 smithi006 bash[27292]: audit 2023-08-28T00:00:44.687034+0000 mon.a (mon.0) 2826 : audit [DBG] from='client.? 172.21.15.6:0/3875485397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:45.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:45 smithi006 bash[33918]: cluster 2023-08-28T00:00:43.922413+0000 mgr.y (mgr.14901) 3665 : cluster [DBG] pgmap v3672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:45.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:45 smithi006 bash[33918]: audit 2023-08-28T00:00:44.687034+0000 mon.a (mon.0) 2826 : audit [DBG] from='client.? 172.21.15.6:0/3875485397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:46 smithi130 bash[27579]: audit 2023-08-28T00:00:45.482334+0000 mon.b (mon.1) 465 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:00:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:46 smithi130 bash[27579]: cluster 2023-08-28T00:00:45.923580+0000 mgr.y (mgr.14901) 3666 : cluster [DBG] pgmap v3673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:46 smithi130 bash[51302]: debug 2023-08-28T00:00:46.058+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:46.061619+0000) 2023-08-28T00:00:46.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:46 smithi006 bash[33918]: audit 2023-08-28T00:00:45.482334+0000 mon.b (mon.1) 465 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:00:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:46 smithi006 bash[33918]: cluster 2023-08-28T00:00:45.923580+0000 mgr.y (mgr.14901) 3666 : cluster [DBG] pgmap v3673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:46 smithi006 bash[27292]: audit 2023-08-28T00:00:45.482334+0000 mon.b (mon.1) 465 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:00:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:46 smithi006 bash[27292]: cluster 2023-08-28T00:00:45.923580+0000 mgr.y (mgr.14901) 3666 : cluster [DBG] pgmap v3673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:47 smithi130 bash[27579]: audit 2023-08-28T00:00:47.104190+0000 mon.a (mon.0) 2827 : audit [DBG] from='client.? 172.21.15.6:0/2136623357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:47 smithi130 bash[51302]: debug 2023-08-28T00:00:47.058+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:47.061886+0000) 2023-08-28T00:00:47.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:47 smithi006 bash[33918]: audit 2023-08-28T00:00:47.104190+0000 mon.a (mon.0) 2827 : audit [DBG] from='client.? 172.21.15.6:0/2136623357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:47.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:47 smithi006 bash[27292]: audit 2023-08-28T00:00:47.104190+0000 mon.a (mon.0) 2827 : audit [DBG] from='client.? 172.21.15.6:0/2136623357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:48.317 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:48 smithi130 bash[51302]: debug 2023-08-28T00:00:48.057+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:48.062191+0000) 2023-08-28T00:00:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:48 smithi130 bash[27579]: cluster 2023-08-28T00:00:47.924320+0000 mgr.y (mgr.14901) 3667 : cluster [DBG] pgmap v3674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:48 smithi006 bash[33918]: cluster 2023-08-28T00:00:47.924320+0000 mgr.y (mgr.14901) 3667 : cluster [DBG] pgmap v3674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:48 smithi006 bash[27292]: cluster 2023-08-28T00:00:47.924320+0000 mgr.y (mgr.14901) 3667 : cluster [DBG] pgmap v3674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:49 smithi130 bash[51302]: debug 2023-08-28T00:00:49.061+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:49.062505+0000) 2023-08-28T00:00:49.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:49 smithi130 bash[27579]: audit 2023-08-28T00:00:49.519265+0000 mon.c (mon.2) 1399 : audit [DBG] from='client.? 172.21.15.6:0/3699909772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:49.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:49 smithi006 bash[33918]: audit 2023-08-28T00:00:49.519265+0000 mon.c (mon.2) 1399 : audit [DBG] from='client.? 172.21.15.6:0/3699909772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:49.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:49 smithi006 bash[27292]: audit 2023-08-28T00:00:49.519265+0000 mon.c (mon.2) 1399 : audit [DBG] from='client.? 172.21.15.6:0/3699909772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:50 smithi130 bash[51302]: debug 2023-08-28T00:00:50.061+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:50.062785+0000) 2023-08-28T00:00:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:50 smithi130 bash[27579]: cluster 2023-08-28T00:00:49.925467+0000 mgr.y (mgr.14901) 3668 : cluster [DBG] pgmap v3675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:00:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:00:50] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T00:00:50.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:50 smithi006 bash[33918]: cluster 2023-08-28T00:00:49.925467+0000 mgr.y (mgr.14901) 3668 : cluster [DBG] pgmap v3675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:00:50.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:00:50.624Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:50.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:50 smithi006 bash[27292]: cluster 2023-08-28T00:00:49.925467+0000 mgr.y (mgr.14901) 3668 : cluster [DBG] pgmap v3675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:51.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:51 smithi130 bash[51302]: debug 2023-08-28T00:00:51.061+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:51.062985+0000) 2023-08-28T00:00:52.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:52 smithi130 bash[51302]: debug 2023-08-28T00:00:52.061+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:52.063161+0000) 2023-08-28T00:00:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:00:52.193Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:00:52.195Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:52.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:52 smithi006 bash[27292]: audit 2023-08-28T00:00:51.363169+0000 mon.a (mon.0) 2828 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:00:52.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:52 smithi006 bash[27292]: audit 2023-08-28T00:00:51.836024+0000 mon.a (mon.0) 2829 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:00:52.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:52 smithi006 bash[27292]: cluster 2023-08-28T00:00:51.925912+0000 mgr.y (mgr.14901) 3669 : cluster [DBG] pgmap v3676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:52.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:52 smithi006 bash[27292]: audit 2023-08-28T00:00:51.932487+0000 mon.a (mon.0) 2830 : audit [DBG] from='client.? 172.21.15.6:0/4219762460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:52.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:52 smithi006 bash[27292]: audit 2023-08-28T00:00:52.136911+0000 mon.b (mon.1) 466 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:00:52.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:52 smithi006 bash[27292]: audit 2023-08-28T00:00:52.139188+0000 mon.b (mon.1) 467 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:00:52.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:52 smithi006 bash[27292]: audit 2023-08-28T00:00:52.150590+0000 mon.a (mon.0) 2831 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:00:52.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:52 smithi006 bash[33918]: audit 2023-08-28T00:00:51.363169+0000 mon.a (mon.0) 2828 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:00:52.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:52 smithi006 bash[33918]: audit 2023-08-28T00:00:51.836024+0000 mon.a (mon.0) 2829 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:00:52.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:52 smithi006 bash[33918]: cluster 2023-08-28T00:00:51.925912+0000 mgr.y (mgr.14901) 3669 : cluster [DBG] pgmap v3676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:52.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:52 smithi006 bash[33918]: audit 2023-08-28T00:00:51.932487+0000 mon.a (mon.0) 2830 : audit [DBG] from='client.? 172.21.15.6:0/4219762460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:52.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:52 smithi006 bash[33918]: audit 2023-08-28T00:00:52.136911+0000 mon.b (mon.1) 466 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:00:52.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:52 smithi006 bash[33918]: audit 2023-08-28T00:00:52.139188+0000 mon.b (mon.1) 467 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:00:52.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:52 smithi006 bash[33918]: audit 2023-08-28T00:00:52.150590+0000 mon.a (mon.0) 2831 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:00:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:52 smithi130 bash[27579]: audit 2023-08-28T00:00:51.363169+0000 mon.a (mon.0) 2828 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:00:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:52 smithi130 bash[27579]: audit 2023-08-28T00:00:51.836024+0000 mon.a (mon.0) 2829 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:00:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:52 smithi130 bash[27579]: cluster 2023-08-28T00:00:51.925912+0000 mgr.y (mgr.14901) 3669 : cluster [DBG] pgmap v3676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:52 smithi130 bash[27579]: audit 2023-08-28T00:00:51.932487+0000 mon.a (mon.0) 2830 : audit [DBG] from='client.? 172.21.15.6:0/4219762460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:52 smithi130 bash[27579]: audit 2023-08-28T00:00:52.136911+0000 mon.b (mon.1) 466 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:00:52.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:52 smithi130 bash[27579]: audit 2023-08-28T00:00:52.139188+0000 mon.b (mon.1) 467 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:00:52.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:52 smithi130 bash[27579]: audit 2023-08-28T00:00:52.150590+0000 mon.a (mon.0) 2831 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:00:53.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:53 smithi130 bash[51302]: debug 2023-08-28T00:00:53.061+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:53.063410+0000) 2023-08-28T00:00:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:54 smithi130 bash[51302]: debug 2023-08-28T00:00:54.061+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:54.063706+0000) 2023-08-28T00:00:54.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:00:54.080Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:00:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:00:54.081Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:00:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:55 smithi130 bash[51302]: debug 2023-08-28T00:00:55.061+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:55.064034+0000) 2023-08-28T00:00:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:54 smithi130 bash[27579]: cluster 2023-08-28T00:00:53.926635+0000 mgr.y (mgr.14901) 3670 : cluster [DBG] pgmap v3677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:54 smithi130 bash[27579]: audit 2023-08-28T00:00:54.350184+0000 mon.a (mon.0) 2832 : audit [DBG] from='client.? 172.21.15.6:0/2489268340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:55.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:54 smithi006 bash[33918]: cluster 2023-08-28T00:00:53.926635+0000 mgr.y (mgr.14901) 3670 : cluster [DBG] pgmap v3677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:55.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:54 smithi006 bash[33918]: audit 2023-08-28T00:00:54.350184+0000 mon.a (mon.0) 2832 : audit [DBG] from='client.? 172.21.15.6:0/2489268340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:55.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:54 smithi006 bash[27292]: cluster 2023-08-28T00:00:53.926635+0000 mgr.y (mgr.14901) 3670 : cluster [DBG] pgmap v3677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:55.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:54 smithi006 bash[27292]: audit 2023-08-28T00:00:54.350184+0000 mon.a (mon.0) 2832 : audit [DBG] from='client.? 172.21.15.6:0/2489268340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:56.334 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:56 smithi130 bash[51302]: debug 2023-08-28T00:00:56.061+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:56.064324+0000) 2023-08-28T00:00:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:56 smithi130 bash[27579]: cluster 2023-08-28T00:00:55.927915+0000 mgr.y (mgr.14901) 3671 : cluster [DBG] pgmap v3678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:56.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:56 smithi006 bash[33918]: cluster 2023-08-28T00:00:55.927915+0000 mgr.y (mgr.14901) 3671 : cluster [DBG] pgmap v3678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:56 smithi006 bash[27292]: cluster 2023-08-28T00:00:55.927915+0000 mgr.y (mgr.14901) 3671 : cluster [DBG] pgmap v3678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:57.338 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:57 smithi130 bash[51302]: debug 2023-08-28T00:00:57.061+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:57.064592+0000) 2023-08-28T00:00:57.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:57 smithi130 bash[27579]: audit 2023-08-28T00:00:56.763742+0000 mon.a (mon.0) 2833 : audit [DBG] from='client.? 172.21.15.6:0/384505548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:57.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:57 smithi006 bash[27292]: audit 2023-08-28T00:00:56.763742+0000 mon.a (mon.0) 2833 : audit [DBG] from='client.? 172.21.15.6:0/384505548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:57.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:57 smithi006 bash[33918]: audit 2023-08-28T00:00:56.763742+0000 mon.a (mon.0) 2833 : audit [DBG] from='client.? 172.21.15.6:0/384505548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:58.348 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:58 smithi130 bash[51302]: debug 2023-08-28T00:00:58.061+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:58.064847+0000) 2023-08-28T00:00:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:58 smithi130 bash[27579]: cluster 2023-08-28T00:00:57.928623+0000 mgr.y (mgr.14901) 3672 : cluster [DBG] pgmap v3679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:58 smithi006 bash[27292]: cluster 2023-08-28T00:00:57.928623+0000 mgr.y (mgr.14901) 3672 : cluster [DBG] pgmap v3679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:58 smithi006 bash[33918]: cluster 2023-08-28T00:00:57.928623+0000 mgr.y (mgr.14901) 3672 : cluster [DBG] pgmap v3679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:00:59.352 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:00:59 smithi130 bash[51302]: debug 2023-08-28T00:00:59.061+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:59.065149+0000) 2023-08-28T00:00:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:00:59 smithi130 bash[27579]: audit 2023-08-28T00:00:59.180850+0000 mon.a (mon.0) 2834 : audit [DBG] from='client.? 172.21.15.6:0/1511453626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:59.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:00:59 smithi006 bash[33918]: audit 2023-08-28T00:00:59.180850+0000 mon.a (mon.0) 2834 : audit [DBG] from='client.? 172.21.15.6:0/1511453626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:59.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:00:59 smithi006 bash[27292]: audit 2023-08-28T00:00:59.180850+0000 mon.a (mon.0) 2834 : audit [DBG] from='client.? 172.21.15.6:0/1511453626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:00.361 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:00 smithi130 bash[51302]: debug 2023-08-28T00:01:00.061+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:00.065438+0000) 2023-08-28T00:01:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:00 smithi130 bash[27579]: cluster 2023-08-28T00:00:59.929781+0000 mgr.y (mgr.14901) 3673 : cluster [DBG] pgmap v3680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:00.626 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:00 smithi006 bash[33918]: cluster 2023-08-28T00:00:59.929781+0000 mgr.y (mgr.14901) 3673 : cluster [DBG] pgmap v3680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:00.626 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:00 smithi006 bash[27292]: cluster 2023-08-28T00:00:59.929781+0000 mgr.y (mgr.14901) 3673 : cluster [DBG] pgmap v3680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:00.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:01:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:01:00] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T00:01:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:01:00.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:01:00.625Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:01.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:01 smithi130 bash[51302]: debug 2023-08-28T00:01:01.061+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:01.065732+0000) 2023-08-28T00:01:01.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:01 smithi006 bash[27292]: audit 2023-08-28T00:01:01.604898+0000 mon.a (mon.0) 2835 : audit [DBG] from='client.? 172.21.15.6:0/4192384496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:01.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:01 smithi006 bash[33918]: audit 2023-08-28T00:01:01.604898+0000 mon.a (mon.0) 2835 : audit [DBG] from='client.? 172.21.15.6:0/4192384496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:02.067 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:01 smithi130 bash[27579]: audit 2023-08-28T00:01:01.604898+0000 mon.a (mon.0) 2835 : audit [DBG] from='client.? 172.21.15.6:0/4192384496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:02.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:02 smithi130 bash[51302]: debug 2023-08-28T00:01:02.065+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:02.066006+0000) 2023-08-28T00:01:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:01:02.193Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:01:02.195Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:02 smithi006 bash[33918]: cluster 2023-08-28T00:01:01.930439+0000 mgr.y (mgr.14901) 3674 : cluster [DBG] pgmap v3681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:02 smithi006 bash[27292]: cluster 2023-08-28T00:01:01.930439+0000 mgr.y (mgr.14901) 3674 : cluster [DBG] pgmap v3681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:03.069 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:02 smithi130 bash[27579]: cluster 2023-08-28T00:01:01.930439+0000 mgr.y (mgr.14901) 3674 : cluster [DBG] pgmap v3681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:03.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:03 smithi130 bash[51302]: debug 2023-08-28T00:01:03.061+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:03.066300+0000) 2023-08-28T00:01:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:04 smithi130 bash[51302]: debug 2023-08-28T00:01:04.061+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:04.066518+0000) 2023-08-28T00:01:04.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:01:04.081Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:01:04.083Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:05 smithi130 bash[51302]: debug 2023-08-28T00:01:05.061+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:05.066779+0000) 2023-08-28T00:01:05.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:04 smithi130 bash[27579]: cluster 2023-08-28T00:01:03.931044+0000 mgr.y (mgr.14901) 3675 : cluster [DBG] pgmap v3682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:05.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:04 smithi130 bash[27579]: audit 2023-08-28T00:01:04.028858+0000 mon.c (mon.2) 1400 : audit [DBG] from='client.? 172.21.15.6:0/3682447109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:05.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:04 smithi006 bash[27292]: cluster 2023-08-28T00:01:03.931044+0000 mgr.y (mgr.14901) 3675 : cluster [DBG] pgmap v3682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:05.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:04 smithi006 bash[27292]: audit 2023-08-28T00:01:04.028858+0000 mon.c (mon.2) 1400 : audit [DBG] from='client.? 172.21.15.6:0/3682447109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:05.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:04 smithi006 bash[33918]: cluster 2023-08-28T00:01:03.931044+0000 mgr.y (mgr.14901) 3675 : cluster [DBG] pgmap v3682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:05.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:04 smithi006 bash[33918]: audit 2023-08-28T00:01:04.028858+0000 mon.c (mon.2) 1400 : audit [DBG] from='client.? 172.21.15.6:0/3682447109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:06.332 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:06 smithi130 bash[51302]: debug 2023-08-28T00:01:06.065+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:06.067076+0000) 2023-08-28T00:01:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:06 smithi130 bash[27579]: cluster 2023-08-28T00:01:05.932188+0000 mgr.y (mgr.14901) 3676 : cluster [DBG] pgmap v3683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:06.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:06 smithi006 bash[33918]: cluster 2023-08-28T00:01:05.932188+0000 mgr.y (mgr.14901) 3676 : cluster [DBG] pgmap v3683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:06 smithi006 bash[27292]: cluster 2023-08-28T00:01:05.932188+0000 mgr.y (mgr.14901) 3676 : cluster [DBG] pgmap v3683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:07.335 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:07 smithi130 bash[51302]: debug 2023-08-28T00:01:07.065+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:07.067373+0000) 2023-08-28T00:01:07.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:07 smithi130 bash[27579]: audit 2023-08-28T00:01:06.437147+0000 mon.c (mon.2) 1401 : audit [DBG] from='client.? 172.21.15.6:0/1958984442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:07.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:07 smithi006 bash[33918]: audit 2023-08-28T00:01:06.437147+0000 mon.c (mon.2) 1401 : audit [DBG] from='client.? 172.21.15.6:0/1958984442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:07.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:07 smithi006 bash[27292]: audit 2023-08-28T00:01:06.437147+0000 mon.c (mon.2) 1401 : audit [DBG] from='client.? 172.21.15.6:0/1958984442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:08.342 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:08 smithi130 bash[51302]: debug 2023-08-28T00:01:08.065+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:08.067686+0000) 2023-08-28T00:01:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:08 smithi130 bash[27579]: cluster 2023-08-28T00:01:07.932868+0000 mgr.y (mgr.14901) 3677 : cluster [DBG] pgmap v3684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:08.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:08 smithi006 bash[27292]: cluster 2023-08-28T00:01:07.932868+0000 mgr.y (mgr.14901) 3677 : cluster [DBG] pgmap v3684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:08 smithi006 bash[33918]: cluster 2023-08-28T00:01:07.932868+0000 mgr.y (mgr.14901) 3677 : cluster [DBG] pgmap v3684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:09.345 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:09 smithi130 bash[51302]: debug 2023-08-28T00:01:09.065+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:09.067982+0000) 2023-08-28T00:01:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:09 smithi130 bash[27579]: audit 2023-08-28T00:01:08.855942+0000 mon.c (mon.2) 1402 : audit [DBG] from='client.? 172.21.15.6:0/1271768598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:09.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:09 smithi006 bash[33918]: audit 2023-08-28T00:01:08.855942+0000 mon.c (mon.2) 1402 : audit [DBG] from='client.? 172.21.15.6:0/1271768598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:09 smithi006 bash[27292]: audit 2023-08-28T00:01:08.855942+0000 mon.c (mon.2) 1402 : audit [DBG] from='client.? 172.21.15.6:0/1271768598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:10.357 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:10 smithi130 bash[51302]: debug 2023-08-28T00:01:10.065+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:10.068261+0000) 2023-08-28T00:01:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:10 smithi130 bash[27579]: cluster 2023-08-28T00:01:09.933941+0000 mgr.y (mgr.14901) 3678 : cluster [DBG] pgmap v3685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:10.626 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:10 smithi006 bash[27292]: cluster 2023-08-28T00:01:09.933941+0000 mgr.y (mgr.14901) 3678 : cluster [DBG] pgmap v3685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:10.626 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:10 smithi006 bash[33918]: cluster 2023-08-28T00:01:09.933941+0000 mgr.y (mgr.14901) 3678 : cluster [DBG] pgmap v3685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:10.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:01:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:01:10] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T00:01:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:01:10.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:01:10.626Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:11.362 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:11 smithi130 bash[51302]: debug 2023-08-28T00:01:11.065+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:11.068602+0000) 2023-08-28T00:01:11.362 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:11 smithi130 bash[27579]: audit 2023-08-28T00:01:11.266353+0000 mon.c (mon.2) 1403 : audit [DBG] from='client.? 172.21.15.6:0/1155513708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:11.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:11 smithi006 bash[27292]: audit 2023-08-28T00:01:11.266353+0000 mon.c (mon.2) 1403 : audit [DBG] from='client.? 172.21.15.6:0/1155513708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:11.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:11 smithi006 bash[33918]: audit 2023-08-28T00:01:11.266353+0000 mon.c (mon.2) 1403 : audit [DBG] from='client.? 172.21.15.6:0/1155513708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:12 smithi130 bash[51302]: debug 2023-08-28T00:01:12.065+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:12.068894+0000) 2023-08-28T00:01:12.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:12 smithi006 bash[33918]: cluster 2023-08-28T00:01:11.934539+0000 mgr.y (mgr.14901) 3679 : cluster [DBG] pgmap v3686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:12.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:12 smithi006 bash[27292]: cluster 2023-08-28T00:01:11.934539+0000 mgr.y (mgr.14901) 3679 : cluster [DBG] pgmap v3686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:01:12.194Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:01:12.196Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:12 smithi130 bash[27579]: cluster 2023-08-28T00:01:11.934539+0000 mgr.y (mgr.14901) 3679 : cluster [DBG] pgmap v3686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:13 smithi130 bash[51302]: debug 2023-08-28T00:01:13.065+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:13.069205+0000) 2023-08-28T00:01:14.070 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:13 smithi130 bash[27579]: audit 2023-08-28T00:01:13.680470+0000 mon.c (mon.2) 1404 : audit [DBG] from='client.? 172.21.15.6:0/2195738037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:14.084 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:13 smithi006 bash[33918]: audit 2023-08-28T00:01:13.680470+0000 mon.c (mon.2) 1404 : audit [DBG] from='client.? 172.21.15.6:0/2195738037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:14.084 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:13 smithi006 bash[27292]: audit 2023-08-28T00:01:13.680470+0000 mon.c (mon.2) 1404 : audit [DBG] from='client.? 172.21.15.6:0/2195738037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:14.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:14 smithi130 bash[51302]: debug 2023-08-28T00:01:14.065+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:14.069478+0000) 2023-08-28T00:01:14.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:01:14.081Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:01:14.084Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:15.071 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:14 smithi130 bash[27579]: cluster 2023-08-28T00:01:13.935171+0000 mgr.y (mgr.14901) 3680 : cluster [DBG] pgmap v3687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:15.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:14 smithi006 bash[33918]: cluster 2023-08-28T00:01:13.935171+0000 mgr.y (mgr.14901) 3680 : cluster [DBG] pgmap v3687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:14 smithi006 bash[27292]: cluster 2023-08-28T00:01:13.935171+0000 mgr.y (mgr.14901) 3680 : cluster [DBG] pgmap v3687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:15 smithi130 bash[51302]: debug 2023-08-28T00:01:15.069+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:15.069761+0000) 2023-08-28T00:01:16.328 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:16 smithi130 bash[51302]: debug 2023-08-28T00:01:16.069+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:16.070031+0000) 2023-08-28T00:01:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:16 smithi130 bash[27579]: cluster 2023-08-28T00:01:15.936333+0000 mgr.y (mgr.14901) 3681 : cluster [DBG] pgmap v3688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:16 smithi130 bash[27579]: audit 2023-08-28T00:01:16.092797+0000 mon.c (mon.2) 1405 : audit [DBG] from='client.? 172.21.15.6:0/751407600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:16 smithi006 bash[27292]: cluster 2023-08-28T00:01:15.936333+0000 mgr.y (mgr.14901) 3681 : cluster [DBG] pgmap v3688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:16 smithi006 bash[27292]: audit 2023-08-28T00:01:16.092797+0000 mon.c (mon.2) 1405 : audit [DBG] from='client.? 172.21.15.6:0/751407600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:16 smithi006 bash[33918]: cluster 2023-08-28T00:01:15.936333+0000 mgr.y (mgr.14901) 3681 : cluster [DBG] pgmap v3688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:16 smithi006 bash[33918]: audit 2023-08-28T00:01:16.092797+0000 mon.c (mon.2) 1405 : audit [DBG] from='client.? 172.21.15.6:0/751407600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:17.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:17 smithi130 bash[51302]: debug 2023-08-28T00:01:17.069+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:17.070307+0000) 2023-08-28T00:01:18.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:18 smithi130 bash[51302]: debug 2023-08-28T00:01:18.069+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:18.070587+0000) 2023-08-28T00:01:19.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:18 smithi130 bash[27579]: cluster 2023-08-28T00:01:17.936965+0000 mgr.y (mgr.14901) 3682 : cluster [DBG] pgmap v3689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:19.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:18 smithi130 bash[27579]: audit 2023-08-28T00:01:18.515254+0000 mon.c (mon.2) 1406 : audit [DBG] from='client.? 172.21.15.6:0/1237316704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:19.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:19 smithi130 bash[51302]: debug 2023-08-28T00:01:19.069+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:19.070906+0000) 2023-08-28T00:01:19.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:18 smithi006 bash[27292]: cluster 2023-08-28T00:01:17.936965+0000 mgr.y (mgr.14901) 3682 : cluster [DBG] pgmap v3689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:19.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:18 smithi006 bash[27292]: audit 2023-08-28T00:01:18.515254+0000 mon.c (mon.2) 1406 : audit [DBG] from='client.? 172.21.15.6:0/1237316704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:19.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:18 smithi006 bash[33918]: cluster 2023-08-28T00:01:17.936965+0000 mgr.y (mgr.14901) 3682 : cluster [DBG] pgmap v3689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:19.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:18 smithi006 bash[33918]: audit 2023-08-28T00:01:18.515254+0000 mon.c (mon.2) 1406 : audit [DBG] from='client.? 172.21.15.6:0/1237316704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:20.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:20 smithi130 bash[51302]: debug 2023-08-28T00:01:20.069+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:20.071128+0000) 2023-08-28T00:01:20.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:01:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:01:20] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T00:01:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:01:20.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:01:20.626Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:21.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:21 smithi130 bash[27579]: cluster 2023-08-28T00:01:19.938008+0000 mgr.y (mgr.14901) 3683 : cluster [DBG] pgmap v3690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:21.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:21 smithi130 bash[27579]: audit 2023-08-28T00:01:20.934214+0000 mon.a (mon.0) 2836 : audit [DBG] from='client.? 172.21.15.6:0/2119349467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:21.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:21 smithi130 bash[51302]: debug 2023-08-28T00:01:21.069+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:21.071381+0000) 2023-08-28T00:01:21.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:20 smithi006 bash[27292]: cluster 2023-08-28T00:01:19.938008+0000 mgr.y (mgr.14901) 3683 : cluster [DBG] pgmap v3690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:21.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:20 smithi006 bash[27292]: audit 2023-08-28T00:01:20.934214+0000 mon.a (mon.0) 2836 : audit [DBG] from='client.? 172.21.15.6:0/2119349467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:21.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:21 smithi006 bash[33918]: cluster 2023-08-28T00:01:19.938008+0000 mgr.y (mgr.14901) 3683 : cluster [DBG] pgmap v3690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:21.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:21 smithi006 bash[33918]: audit 2023-08-28T00:01:20.934214+0000 mon.a (mon.0) 2836 : audit [DBG] from='client.? 172.21.15.6:0/2119349467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:22.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:22 smithi130 bash[51302]: debug 2023-08-28T00:01:22.069+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:22.071588+0000) 2023-08-28T00:01:22.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:01:22.194Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:01:22.197Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:23.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:23 smithi130 bash[27579]: cluster 2023-08-28T00:01:21.938663+0000 mgr.y (mgr.14901) 3684 : cluster [DBG] pgmap v3691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:23.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:23 smithi130 bash[51302]: debug 2023-08-28T00:01:23.069+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:23.071863+0000) 2023-08-28T00:01:23.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:23 smithi006 bash[33918]: cluster 2023-08-28T00:01:21.938663+0000 mgr.y (mgr.14901) 3684 : cluster [DBG] pgmap v3691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:23.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:23 smithi006 bash[27292]: cluster 2023-08-28T00:01:21.938663+0000 mgr.y (mgr.14901) 3684 : cluster [DBG] pgmap v3691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:24 smithi130 bash[27579]: audit 2023-08-28T00:01:23.351232+0000 mon.c (mon.2) 1407 : audit [DBG] from='client.? 172.21.15.6:0/2517123307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:24 smithi130 bash[51302]: debug 2023-08-28T00:01:24.069+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:24.072173+0000) 2023-08-28T00:01:24.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:24 smithi006 bash[33918]: audit 2023-08-28T00:01:23.351232+0000 mon.c (mon.2) 1407 : audit [DBG] from='client.? 172.21.15.6:0/2517123307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:24.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:24 smithi006 bash[27292]: audit 2023-08-28T00:01:23.351232+0000 mon.c (mon.2) 1407 : audit [DBG] from='client.? 172.21.15.6:0/2517123307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:01:24.082Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:01:24.084Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:25 smithi130 bash[27579]: cluster 2023-08-28T00:01:23.939269+0000 mgr.y (mgr.14901) 3685 : cluster [DBG] pgmap v3692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:25 smithi130 bash[51302]: debug 2023-08-28T00:01:25.069+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:25.072481+0000) 2023-08-28T00:01:25.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:25 smithi006 bash[27292]: cluster 2023-08-28T00:01:23.939269+0000 mgr.y (mgr.14901) 3685 : cluster [DBG] pgmap v3692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:25.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:25 smithi006 bash[33918]: cluster 2023-08-28T00:01:23.939269+0000 mgr.y (mgr.14901) 3685 : cluster [DBG] pgmap v3692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:26.339 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:26 smithi130 bash[51302]: debug 2023-08-28T00:01:26.069+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:26.072779+0000) 2023-08-28T00:01:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:26 smithi130 bash[27579]: audit 2023-08-28T00:01:25.771780+0000 mon.c (mon.2) 1408 : audit [DBG] from='client.? 172.21.15.6:0/2099281109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:26 smithi130 bash[27579]: cluster 2023-08-28T00:01:25.940546+0000 mgr.y (mgr.14901) 3686 : cluster [DBG] pgmap v3693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:26 smithi006 bash[33918]: audit 2023-08-28T00:01:25.771780+0000 mon.c (mon.2) 1408 : audit [DBG] from='client.? 172.21.15.6:0/2099281109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:26 smithi006 bash[33918]: cluster 2023-08-28T00:01:25.940546+0000 mgr.y (mgr.14901) 3686 : cluster [DBG] pgmap v3693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:26 smithi006 bash[27292]: audit 2023-08-28T00:01:25.771780+0000 mon.c (mon.2) 1408 : audit [DBG] from='client.? 172.21.15.6:0/2099281109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:26 smithi006 bash[27292]: cluster 2023-08-28T00:01:25.940546+0000 mgr.y (mgr.14901) 3686 : cluster [DBG] pgmap v3693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:27.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:27 smithi130 bash[51302]: debug 2023-08-28T00:01:27.069+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:27.073050+0000) 2023-08-28T00:01:28.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:28 smithi130 bash[51302]: debug 2023-08-28T00:01:28.069+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:28.073311+0000) 2023-08-28T00:01:29.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:29 smithi130 bash[27579]: cluster 2023-08-28T00:01:27.941214+0000 mgr.y (mgr.14901) 3687 : cluster [DBG] pgmap v3694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:29.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:29 smithi130 bash[27579]: audit 2023-08-28T00:01:28.190740+0000 mon.a (mon.0) 2837 : audit [DBG] from='client.? 172.21.15.6:0/4143618682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:29.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:29 smithi130 bash[51302]: debug 2023-08-28T00:01:29.073+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:29.073626+0000) 2023-08-28T00:01:29.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:29 smithi006 bash[33918]: cluster 2023-08-28T00:01:27.941214+0000 mgr.y (mgr.14901) 3687 : cluster [DBG] pgmap v3694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:29.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:29 smithi006 bash[33918]: audit 2023-08-28T00:01:28.190740+0000 mon.a (mon.0) 2837 : audit [DBG] from='client.? 172.21.15.6:0/4143618682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:29.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:28 smithi006 bash[27292]: cluster 2023-08-28T00:01:27.941214+0000 mgr.y (mgr.14901) 3687 : cluster [DBG] pgmap v3694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:29.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:28 smithi006 bash[27292]: audit 2023-08-28T00:01:28.190740+0000 mon.a (mon.0) 2837 : audit [DBG] from='client.? 172.21.15.6:0/4143618682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:30 smithi130 bash[51302]: debug 2023-08-28T00:01:30.073+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:30.073957+0000) 2023-08-28T00:01:30.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:01:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:01:30] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T00:01:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:01:30.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:01:30.627Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:31.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:31 smithi130 bash[27579]: cluster 2023-08-28T00:01:29.942325+0000 mgr.y (mgr.14901) 3688 : cluster [DBG] pgmap v3695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:31.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:31 smithi130 bash[27579]: audit 2023-08-28T00:01:30.598411+0000 mon.a (mon.0) 2838 : audit [DBG] from='client.? 172.21.15.6:0/1407197780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:31.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:31 smithi130 bash[51302]: debug 2023-08-28T00:01:31.073+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:31.074256+0000) 2023-08-28T00:01:31.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:31 smithi006 bash[27292]: cluster 2023-08-28T00:01:29.942325+0000 mgr.y (mgr.14901) 3688 : cluster [DBG] pgmap v3695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:31.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:31 smithi006 bash[27292]: audit 2023-08-28T00:01:30.598411+0000 mon.a (mon.0) 2838 : audit [DBG] from='client.? 172.21.15.6:0/1407197780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:31.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:31 smithi006 bash[33918]: cluster 2023-08-28T00:01:29.942325+0000 mgr.y (mgr.14901) 3688 : cluster [DBG] pgmap v3695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:31.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:31 smithi006 bash[33918]: audit 2023-08-28T00:01:30.598411+0000 mon.a (mon.0) 2838 : audit [DBG] from='client.? 172.21.15.6:0/1407197780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:32.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:32 smithi130 bash[51302]: debug 2023-08-28T00:01:32.069+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:32.074538+0000) 2023-08-28T00:01:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:01:32.195Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:01:32.197Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:33 smithi130 bash[27579]: cluster 2023-08-28T00:01:31.942938+0000 mgr.y (mgr.14901) 3689 : cluster [DBG] pgmap v3696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:33.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:33 smithi130 bash[51302]: debug 2023-08-28T00:01:33.069+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:33.074806+0000) 2023-08-28T00:01:33.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:33 smithi006 bash[27292]: cluster 2023-08-28T00:01:31.942938+0000 mgr.y (mgr.14901) 3689 : cluster [DBG] pgmap v3696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:33.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:33 smithi006 bash[33918]: cluster 2023-08-28T00:01:31.942938+0000 mgr.y (mgr.14901) 3689 : cluster [DBG] pgmap v3696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:34.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:34 smithi130 bash[27579]: audit 2023-08-28T00:01:33.006854+0000 mon.a (mon.0) 2839 : audit [DBG] from='client.? 172.21.15.6:0/2723040449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:34.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:34 smithi130 bash[51302]: debug 2023-08-28T00:01:34.073+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:34.075128+0000) 2023-08-28T00:01:34.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:34 smithi006 bash[33918]: audit 2023-08-28T00:01:33.006854+0000 mon.a (mon.0) 2839 : audit [DBG] from='client.? 172.21.15.6:0/2723040449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:34.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:34 smithi006 bash[27292]: audit 2023-08-28T00:01:33.006854+0000 mon.a (mon.0) 2839 : audit [DBG] from='client.? 172.21.15.6:0/2723040449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:01:34.082Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:01:34.085Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:35.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:35 smithi130 bash[51302]: debug 2023-08-28T00:01:35.073+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:35.075398+0000) 2023-08-28T00:01:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:35 smithi130 bash[27579]: cluster 2023-08-28T00:01:33.943533+0000 mgr.y (mgr.14901) 3690 : cluster [DBG] pgmap v3697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:35.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:35 smithi006 bash[33918]: cluster 2023-08-28T00:01:33.943533+0000 mgr.y (mgr.14901) 3690 : cluster [DBG] pgmap v3697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:35.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:35 smithi006 bash[27292]: cluster 2023-08-28T00:01:33.943533+0000 mgr.y (mgr.14901) 3690 : cluster [DBG] pgmap v3697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:36.337 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:36 smithi130 bash[51302]: debug 2023-08-28T00:01:36.073+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:36.075704+0000) 2023-08-28T00:01:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:36 smithi130 bash[27579]: audit 2023-08-28T00:01:35.431602+0000 mon.a (mon.0) 2840 : audit [DBG] from='client.? 172.21.15.6:0/1436822195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:36 smithi130 bash[27579]: cluster 2023-08-28T00:01:35.944669+0000 mgr.y (mgr.14901) 3691 : cluster [DBG] pgmap v3698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:36 smithi006 bash[33918]: audit 2023-08-28T00:01:35.431602+0000 mon.a (mon.0) 2840 : audit [DBG] from='client.? 172.21.15.6:0/1436822195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:36 smithi006 bash[33918]: cluster 2023-08-28T00:01:35.944669+0000 mgr.y (mgr.14901) 3691 : cluster [DBG] pgmap v3698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:36 smithi006 bash[27292]: audit 2023-08-28T00:01:35.431602+0000 mon.a (mon.0) 2840 : audit [DBG] from='client.? 172.21.15.6:0/1436822195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:36 smithi006 bash[27292]: cluster 2023-08-28T00:01:35.944669+0000 mgr.y (mgr.14901) 3691 : cluster [DBG] pgmap v3698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:37 smithi130 bash[51302]: debug 2023-08-28T00:01:37.073+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:37.075945+0000) 2023-08-28T00:01:38.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:37 smithi006 bash[27292]: audit 2023-08-28T00:01:37.853164+0000 mon.c (mon.2) 1409 : audit [DBG] from='client.? 172.21.15.6:0/152141066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:38.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:37 smithi006 bash[33918]: audit 2023-08-28T00:01:37.853164+0000 mon.c (mon.2) 1409 : audit [DBG] from='client.? 172.21.15.6:0/152141066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:38.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:37 smithi130 bash[27579]: audit 2023-08-28T00:01:37.853164+0000 mon.c (mon.2) 1409 : audit [DBG] from='client.? 172.21.15.6:0/152141066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:38.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:38 smithi130 bash[51302]: debug 2023-08-28T00:01:38.073+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:38.076199+0000) 2023-08-28T00:01:39.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:38 smithi006 bash[33918]: cluster 2023-08-28T00:01:37.945366+0000 mgr.y (mgr.14901) 3692 : cluster [DBG] pgmap v3699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:39.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:38 smithi006 bash[27292]: cluster 2023-08-28T00:01:37.945366+0000 mgr.y (mgr.14901) 3692 : cluster [DBG] pgmap v3699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:38 smithi130 bash[27579]: cluster 2023-08-28T00:01:37.945366+0000 mgr.y (mgr.14901) 3692 : cluster [DBG] pgmap v3699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:39.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:39 smithi130 bash[51302]: debug 2023-08-28T00:01:39.073+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:39.076495+0000) 2023-08-28T00:01:40.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:40 smithi130 bash[51302]: debug 2023-08-28T00:01:40.073+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:40.076802+0000) 2023-08-28T00:01:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:01:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:01:40] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T00:01:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:01:40.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:01:40.628Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:41.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:41 smithi130 bash[51302]: debug 2023-08-28T00:01:41.073+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:41.077134+0000) 2023-08-28T00:01:41.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:41 smithi130 bash[27579]: cluster 2023-08-28T00:01:39.946369+0000 mgr.y (mgr.14901) 3693 : cluster [DBG] pgmap v3700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:41.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:41 smithi130 bash[27579]: audit 2023-08-28T00:01:40.271411+0000 mon.c (mon.2) 1410 : audit [DBG] from='client.? 172.21.15.6:0/3418837846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:41.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:41 smithi006 bash[27292]: cluster 2023-08-28T00:01:39.946369+0000 mgr.y (mgr.14901) 3693 : cluster [DBG] pgmap v3700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:41.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:41 smithi006 bash[27292]: audit 2023-08-28T00:01:40.271411+0000 mon.c (mon.2) 1410 : audit [DBG] from='client.? 172.21.15.6:0/3418837846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:41.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:41 smithi006 bash[33918]: cluster 2023-08-28T00:01:39.946369+0000 mgr.y (mgr.14901) 3693 : cluster [DBG] pgmap v3700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:41.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:41 smithi006 bash[33918]: audit 2023-08-28T00:01:40.271411+0000 mon.c (mon.2) 1410 : audit [DBG] from='client.? 172.21.15.6:0/3418837846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:42.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:42 smithi130 bash[51302]: debug 2023-08-28T00:01:42.077+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:42.077393+0000) 2023-08-28T00:01:42.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:01:42.196Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:01:42.198Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:43.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:43 smithi130 bash[27579]: cluster 2023-08-28T00:01:41.947031+0000 mgr.y (mgr.14901) 3694 : cluster [DBG] pgmap v3701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:43.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:43 smithi130 bash[27579]: audit 2023-08-28T00:01:42.691722+0000 mon.a (mon.0) 2841 : audit [DBG] from='client.? 172.21.15.6:0/440481149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:43.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:43 smithi130 bash[51302]: debug 2023-08-28T00:01:43.073+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:43.077684+0000) 2023-08-28T00:01:43.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:43 smithi006 bash[33918]: cluster 2023-08-28T00:01:41.947031+0000 mgr.y (mgr.14901) 3694 : cluster [DBG] pgmap v3701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:43.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:43 smithi006 bash[33918]: audit 2023-08-28T00:01:42.691722+0000 mon.a (mon.0) 2841 : audit [DBG] from='client.? 172.21.15.6:0/440481149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:43.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:43 smithi006 bash[27292]: cluster 2023-08-28T00:01:41.947031+0000 mgr.y (mgr.14901) 3694 : cluster [DBG] pgmap v3701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:43.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:43 smithi006 bash[27292]: audit 2023-08-28T00:01:42.691722+0000 mon.a (mon.0) 2841 : audit [DBG] from='client.? 172.21.15.6:0/440481149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:44.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:44 smithi130 bash[51302]: debug 2023-08-28T00:01:44.072+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:44.078014+0000) 2023-08-28T00:01:44.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:01:44.084Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:01:44.086Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:45 smithi130 bash[27579]: cluster 2023-08-28T00:01:43.947659+0000 mgr.y (mgr.14901) 3695 : cluster [DBG] pgmap v3702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:45 smithi130 bash[51302]: debug 2023-08-28T00:01:45.072+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:45.078292+0000) 2023-08-28T00:01:45.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:45 smithi006 bash[33918]: cluster 2023-08-28T00:01:43.947659+0000 mgr.y (mgr.14901) 3695 : cluster [DBG] pgmap v3702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:45.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:45 smithi006 bash[27292]: cluster 2023-08-28T00:01:43.947659+0000 mgr.y (mgr.14901) 3695 : cluster [DBG] pgmap v3702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:46 smithi130 bash[27579]: audit 2023-08-28T00:01:45.102902+0000 mon.c (mon.2) 1411 : audit [DBG] from='client.? 172.21.15.6:0/2064463260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:46 smithi130 bash[51302]: debug 2023-08-28T00:01:46.072+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:46.078607+0000) 2023-08-28T00:01:46.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:46 smithi006 bash[27292]: audit 2023-08-28T00:01:45.102902+0000 mon.c (mon.2) 1411 : audit [DBG] from='client.? 172.21.15.6:0/2064463260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:46 smithi006 bash[33918]: audit 2023-08-28T00:01:45.102902+0000 mon.c (mon.2) 1411 : audit [DBG] from='client.? 172.21.15.6:0/2064463260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:47 smithi130 bash[27579]: cluster 2023-08-28T00:01:45.948733+0000 mgr.y (mgr.14901) 3696 : cluster [DBG] pgmap v3703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:47 smithi130 bash[51302]: debug 2023-08-28T00:01:47.076+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:47.078948+0000) 2023-08-28T00:01:47.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:47 smithi006 bash[33918]: cluster 2023-08-28T00:01:45.948733+0000 mgr.y (mgr.14901) 3696 : cluster [DBG] pgmap v3703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:47.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:47 smithi006 bash[27292]: cluster 2023-08-28T00:01:45.948733+0000 mgr.y (mgr.14901) 3696 : cluster [DBG] pgmap v3703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:48 smithi130 bash[27579]: audit 2023-08-28T00:01:47.517349+0000 mon.c (mon.2) 1412 : audit [DBG] from='client.? 172.21.15.6:0/3168119728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:48.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:48 smithi130 bash[51302]: debug 2023-08-28T00:01:48.076+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:48.079230+0000) 2023-08-28T00:01:48.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:48 smithi006 bash[27292]: audit 2023-08-28T00:01:47.517349+0000 mon.c (mon.2) 1412 : audit [DBG] from='client.? 172.21.15.6:0/3168119728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:48.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:48 smithi006 bash[33918]: audit 2023-08-28T00:01:47.517349+0000 mon.c (mon.2) 1412 : audit [DBG] from='client.? 172.21.15.6:0/3168119728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:49.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:49 smithi130 bash[27579]: cluster 2023-08-28T00:01:47.949404+0000 mgr.y (mgr.14901) 3697 : cluster [DBG] pgmap v3704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:49 smithi130 bash[51302]: debug 2023-08-28T00:01:49.076+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:49.079468+0000) 2023-08-28T00:01:49.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:49 smithi006 bash[33918]: cluster 2023-08-28T00:01:47.949404+0000 mgr.y (mgr.14901) 3697 : cluster [DBG] pgmap v3704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:49.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:49 smithi006 bash[27292]: cluster 2023-08-28T00:01:47.949404+0000 mgr.y (mgr.14901) 3697 : cluster [DBG] pgmap v3704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:50.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:50 smithi130 bash[27579]: audit 2023-08-28T00:01:49.926728+0000 mon.c (mon.2) 1413 : audit [DBG] from='client.? 172.21.15.6:0/614307557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:50 smithi130 bash[51302]: debug 2023-08-28T00:01:50.076+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:50.079752+0000) 2023-08-28T00:01:50.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:50 smithi006 bash[33918]: audit 2023-08-28T00:01:49.926728+0000 mon.c (mon.2) 1413 : audit [DBG] from='client.? 172.21.15.6:0/614307557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:50.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:50 smithi006 bash[27292]: audit 2023-08-28T00:01:49.926728+0000 mon.c (mon.2) 1413 : audit [DBG] from='client.? 172.21.15.6:0/614307557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:50.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:01:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:01:50] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T00:01:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:01:50.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:01:50.629Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:51.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:51 smithi130 bash[27579]: cluster 2023-08-28T00:01:49.950571+0000 mgr.y (mgr.14901) 3698 : cluster [DBG] pgmap v3705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:51.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:51 smithi130 bash[51302]: debug 2023-08-28T00:01:51.076+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:51.080045+0000) 2023-08-28T00:01:51.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:51 smithi006 bash[33918]: cluster 2023-08-28T00:01:49.950571+0000 mgr.y (mgr.14901) 3698 : cluster [DBG] pgmap v3705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:51.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:51 smithi006 bash[27292]: cluster 2023-08-28T00:01:49.950571+0000 mgr.y (mgr.14901) 3698 : cluster [DBG] pgmap v3705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:52.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:52 smithi130 bash[51302]: debug 2023-08-28T00:01:52.076+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:52.080340+0000) 2023-08-28T00:01:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:01:52.196Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:01:52.198Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:53.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:53 smithi130 bash[51302]: debug 2023-08-28T00:01:53.076+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:53.080612+0000) 2023-08-28T00:01:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:53 smithi130 bash[27579]: cluster 2023-08-28T00:01:51.951353+0000 mgr.y (mgr.14901) 3699 : cluster [DBG] pgmap v3706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:53 smithi130 bash[27579]: audit 2023-08-28T00:01:52.160704+0000 mon.b (mon.1) 468 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:01:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:53 smithi130 bash[27579]: audit 2023-08-28T00:01:52.341636+0000 mon.c (mon.2) 1414 : audit [DBG] from='client.? 172.21.15.6:0/4088092156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:53.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:53 smithi006 bash[33918]: cluster 2023-08-28T00:01:51.951353+0000 mgr.y (mgr.14901) 3699 : cluster [DBG] pgmap v3706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:53.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:53 smithi006 bash[33918]: audit 2023-08-28T00:01:52.160704+0000 mon.b (mon.1) 468 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:01:53.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:53 smithi006 bash[33918]: audit 2023-08-28T00:01:52.341636+0000 mon.c (mon.2) 1414 : audit [DBG] from='client.? 172.21.15.6:0/4088092156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:53 smithi006 bash[27292]: cluster 2023-08-28T00:01:51.951353+0000 mgr.y (mgr.14901) 3699 : cluster [DBG] pgmap v3706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:53 smithi006 bash[27292]: audit 2023-08-28T00:01:52.160704+0000 mon.b (mon.1) 468 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:01:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:53 smithi006 bash[27292]: audit 2023-08-28T00:01:52.341636+0000 mon.c (mon.2) 1414 : audit [DBG] from='client.? 172.21.15.6:0/4088092156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:54 smithi130 bash[51302]: debug 2023-08-28T00:01:54.076+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:54.080876+0000) 2023-08-28T00:01:54.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:01:54.084Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:01:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:01:54.086Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:01:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:55 smithi130 bash[51302]: debug 2023-08-28T00:01:55.076+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:55.081181+0000) 2023-08-28T00:01:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:55 smithi130 bash[27579]: cluster 2023-08-28T00:01:53.952131+0000 mgr.y (mgr.14901) 3700 : cluster [DBG] pgmap v3707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:55 smithi130 bash[27579]: audit 2023-08-28T00:01:54.761867+0000 mon.a (mon.0) 2842 : audit [DBG] from='client.? 172.21.15.6:0/2104496143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:55.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:55 smithi006 bash[27292]: cluster 2023-08-28T00:01:53.952131+0000 mgr.y (mgr.14901) 3700 : cluster [DBG] pgmap v3707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:55.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:55 smithi006 bash[27292]: audit 2023-08-28T00:01:54.761867+0000 mon.a (mon.0) 2842 : audit [DBG] from='client.? 172.21.15.6:0/2104496143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:55.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:55 smithi006 bash[33918]: cluster 2023-08-28T00:01:53.952131+0000 mgr.y (mgr.14901) 3700 : cluster [DBG] pgmap v3707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:55.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:55 smithi006 bash[33918]: audit 2023-08-28T00:01:54.761867+0000 mon.a (mon.0) 2842 : audit [DBG] from='client.? 172.21.15.6:0/2104496143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:56.344 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:56 smithi130 bash[51302]: debug 2023-08-28T00:01:56.080+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:56.081603+0000) 2023-08-28T00:01:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:56 smithi130 bash[27579]: cluster 2023-08-28T00:01:55.953190+0000 mgr.y (mgr.14901) 3701 : cluster [DBG] pgmap v3708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:56.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:56 smithi006 bash[33918]: cluster 2023-08-28T00:01:55.953190+0000 mgr.y (mgr.14901) 3701 : cluster [DBG] pgmap v3708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:56 smithi006 bash[27292]: cluster 2023-08-28T00:01:55.953190+0000 mgr.y (mgr.14901) 3701 : cluster [DBG] pgmap v3708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:57.346 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:57 smithi130 bash[51302]: debug 2023-08-28T00:01:57.076+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:57.081852+0000) 2023-08-28T00:01:57.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:57 smithi130 bash[27579]: audit 2023-08-28T00:01:57.187538+0000 mon.a (mon.0) 2843 : audit [DBG] from='client.? 172.21.15.6:0/2144240230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:57.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:57 smithi006 bash[33918]: audit 2023-08-28T00:01:57.187538+0000 mon.a (mon.0) 2843 : audit [DBG] from='client.? 172.21.15.6:0/2144240230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:57.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:57 smithi006 bash[27292]: audit 2023-08-28T00:01:57.187538+0000 mon.a (mon.0) 2843 : audit [DBG] from='client.? 172.21.15.6:0/2144240230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:58.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:58 smithi130 bash[51302]: debug 2023-08-28T00:01:58.076+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:58.082139+0000) 2023-08-28T00:01:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:01:59 smithi130 bash[51302]: debug 2023-08-28T00:01:59.076+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:59.082381+0000) 2023-08-28T00:01:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:59 smithi130 bash[27579]: cluster 2023-08-28T00:01:57.953795+0000 mgr.y (mgr.14901) 3702 : cluster [DBG] pgmap v3709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:59 smithi130 bash[27579]: audit 2023-08-28T00:01:58.046167+0000 mon.a (mon.0) 2844 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:01:59.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:59 smithi130 bash[27579]: audit 2023-08-28T00:01:58.447761+0000 mon.a (mon.0) 2845 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:01:59.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:59 smithi130 bash[27579]: audit 2023-08-28T00:01:58.718452+0000 mon.b (mon.1) 469 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:01:59.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:59 smithi130 bash[27579]: audit 2023-08-28T00:01:58.720607+0000 mon.b (mon.1) 470 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:01:59.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:01:59 smithi130 bash[27579]: audit 2023-08-28T00:01:58.732354+0000 mon.a (mon.0) 2846 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:01:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:59 smithi006 bash[33918]: cluster 2023-08-28T00:01:57.953795+0000 mgr.y (mgr.14901) 3702 : cluster [DBG] pgmap v3709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:59 smithi006 bash[33918]: audit 2023-08-28T00:01:58.046167+0000 mon.a (mon.0) 2844 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:01:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:59 smithi006 bash[33918]: audit 2023-08-28T00:01:58.447761+0000 mon.a (mon.0) 2845 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:01:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:59 smithi006 bash[33918]: audit 2023-08-28T00:01:58.718452+0000 mon.b (mon.1) 469 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:01:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:59 smithi006 bash[33918]: audit 2023-08-28T00:01:58.720607+0000 mon.b (mon.1) 470 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:01:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:01:59 smithi006 bash[33918]: audit 2023-08-28T00:01:58.732354+0000 mon.a (mon.0) 2846 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:01:59.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:59 smithi006 bash[27292]: cluster 2023-08-28T00:01:57.953795+0000 mgr.y (mgr.14901) 3702 : cluster [DBG] pgmap v3709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:01:59.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:59 smithi006 bash[27292]: audit 2023-08-28T00:01:58.046167+0000 mon.a (mon.0) 2844 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:01:59.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:59 smithi006 bash[27292]: audit 2023-08-28T00:01:58.447761+0000 mon.a (mon.0) 2845 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:01:59.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:59 smithi006 bash[27292]: audit 2023-08-28T00:01:58.718452+0000 mon.b (mon.1) 469 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:01:59.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:59 smithi006 bash[27292]: audit 2023-08-28T00:01:58.720607+0000 mon.b (mon.1) 470 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:01:59.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:01:59 smithi006 bash[27292]: audit 2023-08-28T00:01:58.732354+0000 mon.a (mon.0) 2846 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:02:00.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:00 smithi130 bash[27579]: audit 2023-08-28T00:01:59.605046+0000 mon.a (mon.0) 2847 : audit [DBG] from='client.? 172.21.15.6:0/857009207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:00.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:00 smithi130 bash[51302]: debug 2023-08-28T00:02:00.076+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:00.082629+0000) 2023-08-28T00:02:00.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:00 smithi006 bash[27292]: audit 2023-08-28T00:01:59.605046+0000 mon.a (mon.0) 2847 : audit [DBG] from='client.? 172.21.15.6:0/857009207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:00.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:00 smithi006 bash[33918]: audit 2023-08-28T00:01:59.605046+0000 mon.a (mon.0) 2847 : audit [DBG] from='client.? 172.21.15.6:0/857009207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:00.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:02:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:02:00] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T00:02:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:02:00.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:02:00.630Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:01.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:01 smithi130 bash[27579]: cluster 2023-08-28T00:01:59.954913+0000 mgr.y (mgr.14901) 3703 : cluster [DBG] pgmap v3710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:01.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:01 smithi130 bash[51302]: debug 2023-08-28T00:02:01.080+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:01.082924+0000) 2023-08-28T00:02:01.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:01 smithi006 bash[27292]: cluster 2023-08-28T00:01:59.954913+0000 mgr.y (mgr.14901) 3703 : cluster [DBG] pgmap v3710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:01.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:01 smithi006 bash[33918]: cluster 2023-08-28T00:01:59.954913+0000 mgr.y (mgr.14901) 3703 : cluster [DBG] pgmap v3710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:02.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:02 smithi130 bash[27579]: audit 2023-08-28T00:02:02.014558+0000 mon.a (mon.0) 2848 : audit [DBG] from='client.? 172.21.15.6:0/1275835806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:02.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:02 smithi130 bash[51302]: debug 2023-08-28T00:02:02.080+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:02.083196+0000) 2023-08-28T00:02:02.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:02 smithi006 bash[27292]: audit 2023-08-28T00:02:02.014558+0000 mon.a (mon.0) 2848 : audit [DBG] from='client.? 172.21.15.6:0/1275835806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:02.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:02 smithi006 bash[33918]: audit 2023-08-28T00:02:02.014558+0000 mon.a (mon.0) 2848 : audit [DBG] from='client.? 172.21.15.6:0/1275835806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:02:02.198Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:02:02.199Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:03 smithi130 bash[51302]: debug 2023-08-28T00:02:03.080+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:03.083487+0000) 2023-08-28T00:02:03.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:03 smithi130 bash[27579]: cluster 2023-08-28T00:02:01.955685+0000 mgr.y (mgr.14901) 3704 : cluster [DBG] pgmap v3711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:03.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:03 smithi006 bash[27292]: cluster 2023-08-28T00:02:01.955685+0000 mgr.y (mgr.14901) 3704 : cluster [DBG] pgmap v3711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:03.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:03 smithi006 bash[33918]: cluster 2023-08-28T00:02:01.955685+0000 mgr.y (mgr.14901) 3704 : cluster [DBG] pgmap v3711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:04 smithi130 bash[51302]: debug 2023-08-28T00:02:04.080+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:04.083765+0000) 2023-08-28T00:02:04.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:02:04.084Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:02:04.086Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:05 smithi130 bash[27579]: cluster 2023-08-28T00:02:03.956410+0000 mgr.y (mgr.14901) 3705 : cluster [DBG] pgmap v3712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:05 smithi130 bash[27579]: audit 2023-08-28T00:02:04.437809+0000 mon.a (mon.0) 2849 : audit [DBG] from='client.? 172.21.15.6:0/3207376594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:05 smithi130 bash[51302]: debug 2023-08-28T00:02:05.080+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:05.084035+0000) 2023-08-28T00:02:05.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:05 smithi006 bash[27292]: cluster 2023-08-28T00:02:03.956410+0000 mgr.y (mgr.14901) 3705 : cluster [DBG] pgmap v3712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:05.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:05 smithi006 bash[27292]: audit 2023-08-28T00:02:04.437809+0000 mon.a (mon.0) 2849 : audit [DBG] from='client.? 172.21.15.6:0/3207376594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:05.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:05 smithi006 bash[33918]: cluster 2023-08-28T00:02:03.956410+0000 mgr.y (mgr.14901) 3705 : cluster [DBG] pgmap v3712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:05.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:05 smithi006 bash[33918]: audit 2023-08-28T00:02:04.437809+0000 mon.a (mon.0) 2849 : audit [DBG] from='client.? 172.21.15.6:0/3207376594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:06.349 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:06 smithi130 bash[51302]: debug 2023-08-28T00:02:06.080+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:06.084301+0000) 2023-08-28T00:02:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:06 smithi130 bash[27579]: cluster 2023-08-28T00:02:05.957529+0000 mgr.y (mgr.14901) 3706 : cluster [DBG] pgmap v3713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:06.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:06 smithi006 bash[27292]: cluster 2023-08-28T00:02:05.957529+0000 mgr.y (mgr.14901) 3706 : cluster [DBG] pgmap v3713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:06 smithi006 bash[33918]: cluster 2023-08-28T00:02:05.957529+0000 mgr.y (mgr.14901) 3706 : cluster [DBG] pgmap v3713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:07.351 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:07 smithi130 bash[51302]: debug 2023-08-28T00:02:07.080+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:07.084565+0000) 2023-08-28T00:02:07.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:07 smithi130 bash[27579]: audit 2023-08-28T00:02:06.858010+0000 mon.c (mon.2) 1415 : audit [DBG] from='client.? 172.21.15.6:0/3873567605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:07.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:07 smithi006 bash[27292]: audit 2023-08-28T00:02:06.858010+0000 mon.c (mon.2) 1415 : audit [DBG] from='client.? 172.21.15.6:0/3873567605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:07 smithi006 bash[33918]: audit 2023-08-28T00:02:06.858010+0000 mon.c (mon.2) 1415 : audit [DBG] from='client.? 172.21.15.6:0/3873567605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:08.359 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:08 smithi130 bash[51302]: debug 2023-08-28T00:02:08.080+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:08.084831+0000) 2023-08-28T00:02:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:08 smithi130 bash[27579]: cluster 2023-08-28T00:02:07.958173+0000 mgr.y (mgr.14901) 3707 : cluster [DBG] pgmap v3714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:08.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:08 smithi006 bash[27292]: cluster 2023-08-28T00:02:07.958173+0000 mgr.y (mgr.14901) 3707 : cluster [DBG] pgmap v3714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:08 smithi006 bash[33918]: cluster 2023-08-28T00:02:07.958173+0000 mgr.y (mgr.14901) 3707 : cluster [DBG] pgmap v3714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:09.361 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:09 smithi130 bash[51302]: debug 2023-08-28T00:02:09.080+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:09.085124+0000) 2023-08-28T00:02:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:09 smithi130 bash[27579]: audit 2023-08-28T00:02:09.279146+0000 mon.a (mon.0) 2850 : audit [DBG] from='client.? 172.21.15.6:0/2841740798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:09 smithi006 bash[27292]: audit 2023-08-28T00:02:09.279146+0000 mon.a (mon.0) 2850 : audit [DBG] from='client.? 172.21.15.6:0/2841740798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:09.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:09 smithi006 bash[33918]: audit 2023-08-28T00:02:09.279146+0000 mon.a (mon.0) 2850 : audit [DBG] from='client.? 172.21.15.6:0/2841740798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:10.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:10 smithi130 bash[51302]: debug 2023-08-28T00:02:10.080+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:10.085407+0000) 2023-08-28T00:02:10.631 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:10 smithi006 bash[27292]: cluster 2023-08-28T00:02:09.959272+0000 mgr.y (mgr.14901) 3708 : cluster [DBG] pgmap v3715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:10.631 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:10 smithi006 bash[33918]: cluster 2023-08-28T00:02:09.959272+0000 mgr.y (mgr.14901) 3708 : cluster [DBG] pgmap v3715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:10 smithi130 bash[27579]: cluster 2023-08-28T00:02:09.959272+0000 mgr.y (mgr.14901) 3708 : cluster [DBG] pgmap v3715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:10.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:02:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:02:10] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T00:02:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:02:10.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:02:10.630Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:11.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:11 smithi130 bash[51302]: debug 2023-08-28T00:02:11.084+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:11.085767+0000) 2023-08-28T00:02:12.087 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:11 smithi130 bash[27579]: audit 2023-08-28T00:02:11.688800+0000 mon.c (mon.2) 1416 : audit [DBG] from='client.? 172.21.15.6:0/3998195752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:12.201 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:11 smithi006 bash[27292]: audit 2023-08-28T00:02:11.688800+0000 mon.c (mon.2) 1416 : audit [DBG] from='client.? 172.21.15.6:0/3998195752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:12.201 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:11 smithi006 bash[33918]: audit 2023-08-28T00:02:11.688800+0000 mon.c (mon.2) 1416 : audit [DBG] from='client.? 172.21.15.6:0/3998195752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:12 smithi130 bash[51302]: debug 2023-08-28T00:02:12.084+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:12.086058+0000) 2023-08-28T00:02:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:02:12.198Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:02:12.201Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:13.087 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:12 smithi130 bash[27579]: cluster 2023-08-28T00:02:11.959958+0000 mgr.y (mgr.14901) 3709 : cluster [DBG] pgmap v3716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:12 smithi006 bash[27292]: cluster 2023-08-28T00:02:11.959958+0000 mgr.y (mgr.14901) 3709 : cluster [DBG] pgmap v3716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:12 smithi006 bash[33918]: cluster 2023-08-28T00:02:11.959958+0000 mgr.y (mgr.14901) 3709 : cluster [DBG] pgmap v3716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:13.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:13 smithi130 bash[51302]: debug 2023-08-28T00:02:13.084+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:13.086355+0000) 2023-08-28T00:02:14.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:14 smithi130 bash[51302]: debug 2023-08-28T00:02:14.084+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:14.086639+0000) 2023-08-28T00:02:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:02:14.086Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:02:14.088Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:15 smithi130 bash[51302]: debug 2023-08-28T00:02:15.084+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:15.086992+0000) 2023-08-28T00:02:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:15 smithi130 bash[27579]: cluster 2023-08-28T00:02:13.960581+0000 mgr.y (mgr.14901) 3710 : cluster [DBG] pgmap v3717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:15 smithi130 bash[27579]: audit 2023-08-28T00:02:14.104816+0000 mon.a (mon.0) 2851 : audit [DBG] from='client.? 172.21.15.6:0/3413572485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:15.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:15 smithi006 bash[27292]: cluster 2023-08-28T00:02:13.960581+0000 mgr.y (mgr.14901) 3710 : cluster [DBG] pgmap v3717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:15.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:15 smithi006 bash[27292]: audit 2023-08-28T00:02:14.104816+0000 mon.a (mon.0) 2851 : audit [DBG] from='client.? 172.21.15.6:0/3413572485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:15.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:15 smithi006 bash[33918]: cluster 2023-08-28T00:02:13.960581+0000 mgr.y (mgr.14901) 3710 : cluster [DBG] pgmap v3717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:15.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:15 smithi006 bash[33918]: audit 2023-08-28T00:02:14.104816+0000 mon.a (mon.0) 2851 : audit [DBG] from='client.? 172.21.15.6:0/3413572485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:16.345 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:16 smithi130 bash[51302]: debug 2023-08-28T00:02:16.084+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:16.087420+0000) 2023-08-28T00:02:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:16 smithi130 bash[27579]: cluster 2023-08-28T00:02:15.961707+0000 mgr.y (mgr.14901) 3711 : cluster [DBG] pgmap v3718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:16 smithi006 bash[27292]: cluster 2023-08-28T00:02:15.961707+0000 mgr.y (mgr.14901) 3711 : cluster [DBG] pgmap v3718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:16 smithi006 bash[33918]: cluster 2023-08-28T00:02:15.961707+0000 mgr.y (mgr.14901) 3711 : cluster [DBG] pgmap v3718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:17.358 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:17 smithi130 bash[51302]: debug 2023-08-28T00:02:17.084+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:17.087896+0000) 2023-08-28T00:02:17.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:17 smithi130 bash[27579]: audit 2023-08-28T00:02:16.521856+0000 mon.c (mon.2) 1417 : audit [DBG] from='client.? 172.21.15.6:0/358971732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:17.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:17 smithi006 bash[33918]: audit 2023-08-28T00:02:16.521856+0000 mon.c (mon.2) 1417 : audit [DBG] from='client.? 172.21.15.6:0/358971732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:17.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:17 smithi006 bash[27292]: audit 2023-08-28T00:02:16.521856+0000 mon.c (mon.2) 1417 : audit [DBG] from='client.? 172.21.15.6:0/358971732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:18.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:18 smithi130 bash[51302]: debug 2023-08-28T00:02:18.084+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:18.088216+0000) 2023-08-28T00:02:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:18 smithi130 bash[27579]: cluster 2023-08-28T00:02:17.962361+0000 mgr.y (mgr.14901) 3712 : cluster [DBG] pgmap v3719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:18.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:18 smithi006 bash[27292]: cluster 2023-08-28T00:02:17.962361+0000 mgr.y (mgr.14901) 3712 : cluster [DBG] pgmap v3719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:18 smithi006 bash[33918]: cluster 2023-08-28T00:02:17.962361+0000 mgr.y (mgr.14901) 3712 : cluster [DBG] pgmap v3719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:19.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:19 smithi130 bash[51302]: debug 2023-08-28T00:02:19.088+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:19.088508+0000) 2023-08-28T00:02:19.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:19 smithi006 bash[27292]: audit 2023-08-28T00:02:18.940608+0000 mon.a (mon.0) 2852 : audit [DBG] from='client.? 172.21.15.6:0/3840508945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:19.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:19 smithi006 bash[33918]: audit 2023-08-28T00:02:18.940608+0000 mon.a (mon.0) 2852 : audit [DBG] from='client.? 172.21.15.6:0/3840508945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:19.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:19 smithi130 bash[27579]: audit 2023-08-28T00:02:18.940608+0000 mon.a (mon.0) 2852 : audit [DBG] from='client.? 172.21.15.6:0/3840508945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:20.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:20 smithi130 bash[51302]: debug 2023-08-28T00:02:20.084+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:20.088805+0000) 2023-08-28T00:02:20.631 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:20 smithi006 bash[33918]: cluster 2023-08-28T00:02:19.963446+0000 mgr.y (mgr.14901) 3713 : cluster [DBG] pgmap v3720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:20.631 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:20 smithi006 bash[27292]: cluster 2023-08-28T00:02:19.963446+0000 mgr.y (mgr.14901) 3713 : cluster [DBG] pgmap v3720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:20 smithi130 bash[27579]: cluster 2023-08-28T00:02:19.963446+0000 mgr.y (mgr.14901) 3713 : cluster [DBG] pgmap v3720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:20.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:02:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:02:20] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T00:02:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:02:20.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:02:20.631Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:21.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:21 smithi130 bash[51302]: debug 2023-08-28T00:02:21.088+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:21.089115+0000) 2023-08-28T00:02:21.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:21 smithi006 bash[27292]: audit 2023-08-28T00:02:21.361480+0000 mon.c (mon.2) 1418 : audit [DBG] from='client.? 172.21.15.6:0/3256812914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:21.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:21 smithi006 bash[33918]: audit 2023-08-28T00:02:21.361480+0000 mon.c (mon.2) 1418 : audit [DBG] from='client.? 172.21.15.6:0/3256812914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:21.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:21 smithi130 bash[27579]: audit 2023-08-28T00:02:21.361480+0000 mon.c (mon.2) 1418 : audit [DBG] from='client.? 172.21.15.6:0/3256812914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:22.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:22 smithi130 bash[51302]: debug 2023-08-28T00:02:22.084+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:22.089684+0000) 2023-08-28T00:02:22.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:22 smithi006 bash[27292]: cluster 2023-08-28T00:02:21.964110+0000 mgr.y (mgr.14901) 3714 : cluster [DBG] pgmap v3721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:22.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:22 smithi006 bash[33918]: cluster 2023-08-28T00:02:21.964110+0000 mgr.y (mgr.14901) 3714 : cluster [DBG] pgmap v3721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:02:22.199Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:02:22.201Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:22 smithi130 bash[27579]: cluster 2023-08-28T00:02:21.964110+0000 mgr.y (mgr.14901) 3714 : cluster [DBG] pgmap v3721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:23.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:23 smithi130 bash[51302]: debug 2023-08-28T00:02:23.084+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:23.090009+0000) 2023-08-28T00:02:24.089 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:23 smithi006 bash[27292]: audit 2023-08-28T00:02:23.777242+0000 mon.a (mon.0) 2853 : audit [DBG] from='client.? 172.21.15.6:0/1266463440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:24.090 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:23 smithi006 bash[33918]: audit 2023-08-28T00:02:23.777242+0000 mon.a (mon.0) 2853 : audit [DBG] from='client.? 172.21.15.6:0/1266463440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:24.091 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:23 smithi130 bash[27579]: audit 2023-08-28T00:02:23.777242+0000 mon.a (mon.0) 2853 : audit [DBG] from='client.? 172.21.15.6:0/1266463440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:24 smithi130 bash[51302]: debug 2023-08-28T00:02:24.084+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:24.090328+0000) 2023-08-28T00:02:24.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:02:24.087Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:02:24.089Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:24 smithi130 bash[27579]: cluster 2023-08-28T00:02:23.964760+0000 mgr.y (mgr.14901) 3715 : cluster [DBG] pgmap v3722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:25 smithi130 bash[51302]: debug 2023-08-28T00:02:25.088+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:25.090604+0000) 2023-08-28T00:02:25.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:24 smithi006 bash[33918]: cluster 2023-08-28T00:02:23.964760+0000 mgr.y (mgr.14901) 3715 : cluster [DBG] pgmap v3722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:24 smithi006 bash[27292]: cluster 2023-08-28T00:02:23.964760+0000 mgr.y (mgr.14901) 3715 : cluster [DBG] pgmap v3722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:26.354 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:26 smithi130 bash[51302]: debug 2023-08-28T00:02:26.088+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:26.090909+0000) 2023-08-28T00:02:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:26 smithi130 bash[27579]: cluster 2023-08-28T00:02:25.965885+0000 mgr.y (mgr.14901) 3716 : cluster [DBG] pgmap v3723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:26 smithi130 bash[27579]: audit 2023-08-28T00:02:26.188511+0000 mon.c (mon.2) 1419 : audit [DBG] from='client.? 172.21.15.6:0/2130145618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:26 smithi006 bash[27292]: cluster 2023-08-28T00:02:25.965885+0000 mgr.y (mgr.14901) 3716 : cluster [DBG] pgmap v3723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:26 smithi006 bash[27292]: audit 2023-08-28T00:02:26.188511+0000 mon.c (mon.2) 1419 : audit [DBG] from='client.? 172.21.15.6:0/2130145618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:26 smithi006 bash[33918]: cluster 2023-08-28T00:02:25.965885+0000 mgr.y (mgr.14901) 3716 : cluster [DBG] pgmap v3723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:26 smithi006 bash[33918]: audit 2023-08-28T00:02:26.188511+0000 mon.c (mon.2) 1419 : audit [DBG] from='client.? 172.21.15.6:0/2130145618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:27.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:27 smithi130 bash[51302]: debug 2023-08-28T00:02:27.088+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:27.091175+0000) 2023-08-28T00:02:28.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:28 smithi130 bash[51302]: debug 2023-08-28T00:02:28.088+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:28.091432+0000) 2023-08-28T00:02:29.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:29 smithi130 bash[27579]: cluster 2023-08-28T00:02:27.966529+0000 mgr.y (mgr.14901) 3717 : cluster [DBG] pgmap v3724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:29.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:29 smithi130 bash[27579]: audit 2023-08-28T00:02:28.641735+0000 mon.c (mon.2) 1420 : audit [DBG] from='client.? 172.21.15.6:0/1371222267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:29.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:29 smithi130 bash[51302]: debug 2023-08-28T00:02:29.088+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:29.091720+0000) 2023-08-28T00:02:29.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:29 smithi006 bash[33918]: cluster 2023-08-28T00:02:27.966529+0000 mgr.y (mgr.14901) 3717 : cluster [DBG] pgmap v3724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:29.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:29 smithi006 bash[33918]: audit 2023-08-28T00:02:28.641735+0000 mon.c (mon.2) 1420 : audit [DBG] from='client.? 172.21.15.6:0/1371222267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:29.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:29 smithi006 bash[27292]: cluster 2023-08-28T00:02:27.966529+0000 mgr.y (mgr.14901) 3717 : cluster [DBG] pgmap v3724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:29.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:29 smithi006 bash[27292]: audit 2023-08-28T00:02:28.641735+0000 mon.c (mon.2) 1420 : audit [DBG] from='client.? 172.21.15.6:0/1371222267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:30 smithi130 bash[51302]: debug 2023-08-28T00:02:30.088+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:30.092011+0000) 2023-08-28T00:02:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:02:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:02:30] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:02:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:02:30.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:02:30.631Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:31.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:31 smithi130 bash[27579]: cluster 2023-08-28T00:02:29.967730+0000 mgr.y (mgr.14901) 3718 : cluster [DBG] pgmap v3725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:31.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:31 smithi130 bash[51302]: debug 2023-08-28T00:02:31.088+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:31.092306+0000) 2023-08-28T00:02:31.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:31 smithi006 bash[27292]: cluster 2023-08-28T00:02:29.967730+0000 mgr.y (mgr.14901) 3718 : cluster [DBG] pgmap v3725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:31.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:31 smithi006 bash[33918]: cluster 2023-08-28T00:02:29.967730+0000 mgr.y (mgr.14901) 3718 : cluster [DBG] pgmap v3725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:32.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:32 smithi130 bash[51302]: debug 2023-08-28T00:02:32.088+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:32.092594+0000) 2023-08-28T00:02:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:32 smithi130 bash[27579]: audit 2023-08-28T00:02:31.061679+0000 mon.b (mon.1) 471 : audit [DBG] from='client.? 172.21.15.6:0/3700582019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:32.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:32 smithi006 bash[27292]: audit 2023-08-28T00:02:31.061679+0000 mon.b (mon.1) 471 : audit [DBG] from='client.? 172.21.15.6:0/3700582019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:32.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:32 smithi006 bash[33918]: audit 2023-08-28T00:02:31.061679+0000 mon.b (mon.1) 471 : audit [DBG] from='client.? 172.21.15.6:0/3700582019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:02:32.200Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:02:32.202Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:33 smithi130 bash[27579]: cluster 2023-08-28T00:02:31.968443+0000 mgr.y (mgr.14901) 3719 : cluster [DBG] pgmap v3726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:33.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:33 smithi130 bash[51302]: debug 2023-08-28T00:02:33.088+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:33.092903+0000) 2023-08-28T00:02:33.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:33 smithi006 bash[27292]: cluster 2023-08-28T00:02:31.968443+0000 mgr.y (mgr.14901) 3719 : cluster [DBG] pgmap v3726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:33.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:33 smithi006 bash[33918]: cluster 2023-08-28T00:02:31.968443+0000 mgr.y (mgr.14901) 3719 : cluster [DBG] pgmap v3726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:34.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:34 smithi130 bash[27579]: audit 2023-08-28T00:02:33.473986+0000 mon.a (mon.0) 2854 : audit [DBG] from='client.? 172.21.15.6:0/30143599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:34.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:34 smithi130 bash[51302]: debug 2023-08-28T00:02:34.088+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:34.093213+0000) 2023-08-28T00:02:34.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:34 smithi006 bash[27292]: audit 2023-08-28T00:02:33.473986+0000 mon.a (mon.0) 2854 : audit [DBG] from='client.? 172.21.15.6:0/30143599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:34.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:34 smithi006 bash[33918]: audit 2023-08-28T00:02:33.473986+0000 mon.a (mon.0) 2854 : audit [DBG] from='client.? 172.21.15.6:0/30143599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:02:34.087Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:02:34.089Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:35 smithi130 bash[27579]: cluster 2023-08-28T00:02:33.969059+0000 mgr.y (mgr.14901) 3720 : cluster [DBG] pgmap v3727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:35.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:35 smithi130 bash[51302]: debug 2023-08-28T00:02:35.092+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:35.093482+0000) 2023-08-28T00:02:35.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:35 smithi006 bash[33918]: cluster 2023-08-28T00:02:33.969059+0000 mgr.y (mgr.14901) 3720 : cluster [DBG] pgmap v3727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:35.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:35 smithi006 bash[27292]: cluster 2023-08-28T00:02:33.969059+0000 mgr.y (mgr.14901) 3720 : cluster [DBG] pgmap v3727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:36.356 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:36 smithi130 bash[51302]: debug 2023-08-28T00:02:36.092+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:36.093793+0000) 2023-08-28T00:02:36.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:36 smithi130 bash[27579]: audit 2023-08-28T00:02:35.889303+0000 mon.c (mon.2) 1421 : audit [DBG] from='client.? 172.21.15.6:0/2748497252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:36.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:36 smithi130 bash[27579]: cluster 2023-08-28T00:02:35.970185+0000 mgr.y (mgr.14901) 3721 : cluster [DBG] pgmap v3728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:36.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:36 smithi006 bash[33918]: audit 2023-08-28T00:02:35.889303+0000 mon.c (mon.2) 1421 : audit [DBG] from='client.? 172.21.15.6:0/2748497252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:36 smithi006 bash[33918]: cluster 2023-08-28T00:02:35.970185+0000 mgr.y (mgr.14901) 3721 : cluster [DBG] pgmap v3728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:36 smithi006 bash[27292]: audit 2023-08-28T00:02:35.889303+0000 mon.c (mon.2) 1421 : audit [DBG] from='client.? 172.21.15.6:0/2748497252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:36 smithi006 bash[27292]: cluster 2023-08-28T00:02:35.970185+0000 mgr.y (mgr.14901) 3721 : cluster [DBG] pgmap v3728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:37 smithi130 bash[51302]: debug 2023-08-28T00:02:37.092+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:37.094082+0000) 2023-08-28T00:02:38.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:38 smithi130 bash[51302]: debug 2023-08-28T00:02:38.092+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:38.094365+0000) 2023-08-28T00:02:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:39 smithi130 bash[27579]: cluster 2023-08-28T00:02:37.970821+0000 mgr.y (mgr.14901) 3722 : cluster [DBG] pgmap v3729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:39 smithi130 bash[27579]: audit 2023-08-28T00:02:38.310014+0000 mon.c (mon.2) 1422 : audit [DBG] from='client.? 172.21.15.6:0/1917342450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:39.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:39 smithi130 bash[51302]: debug 2023-08-28T00:02:39.092+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:39.094695+0000) 2023-08-28T00:02:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:39 smithi006 bash[27292]: cluster 2023-08-28T00:02:37.970821+0000 mgr.y (mgr.14901) 3722 : cluster [DBG] pgmap v3729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:39 smithi006 bash[27292]: audit 2023-08-28T00:02:38.310014+0000 mon.c (mon.2) 1422 : audit [DBG] from='client.? 172.21.15.6:0/1917342450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:39.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:39 smithi006 bash[33918]: cluster 2023-08-28T00:02:37.970821+0000 mgr.y (mgr.14901) 3722 : cluster [DBG] pgmap v3729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:39.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:39 smithi006 bash[33918]: audit 2023-08-28T00:02:38.310014+0000 mon.c (mon.2) 1422 : audit [DBG] from='client.? 172.21.15.6:0/1917342450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:40.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:40 smithi130 bash[51302]: debug 2023-08-28T00:02:40.091+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:40.094983+0000) 2023-08-28T00:02:40.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:02:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:02:40] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:02:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:02:40.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:02:40.631Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:41.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:41 smithi130 bash[51302]: debug 2023-08-28T00:02:41.091+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:41.095263+0000) 2023-08-28T00:02:41.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:41 smithi130 bash[27579]: cluster 2023-08-28T00:02:39.971948+0000 mgr.y (mgr.14901) 3723 : cluster [DBG] pgmap v3730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:41.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:41 smithi130 bash[27579]: audit 2023-08-28T00:02:40.732095+0000 mon.a (mon.0) 2855 : audit [DBG] from='client.? 172.21.15.6:0/3185032485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:41.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:41 smithi006 bash[33918]: cluster 2023-08-28T00:02:39.971948+0000 mgr.y (mgr.14901) 3723 : cluster [DBG] pgmap v3730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:41.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:41 smithi006 bash[33918]: audit 2023-08-28T00:02:40.732095+0000 mon.a (mon.0) 2855 : audit [DBG] from='client.? 172.21.15.6:0/3185032485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:41.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:41 smithi006 bash[27292]: cluster 2023-08-28T00:02:39.971948+0000 mgr.y (mgr.14901) 3723 : cluster [DBG] pgmap v3730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:41.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:41 smithi006 bash[27292]: audit 2023-08-28T00:02:40.732095+0000 mon.a (mon.0) 2855 : audit [DBG] from='client.? 172.21.15.6:0/3185032485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:42.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:42 smithi130 bash[51302]: debug 2023-08-28T00:02:42.091+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:42.095565+0000) 2023-08-28T00:02:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:02:42.200Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:02:42.202Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:43.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:43 smithi130 bash[27579]: cluster 2023-08-28T00:02:41.972606+0000 mgr.y (mgr.14901) 3724 : cluster [DBG] pgmap v3731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:43.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:43 smithi130 bash[51302]: debug 2023-08-28T00:02:43.091+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:43.095871+0000) 2023-08-28T00:02:43.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:43 smithi006 bash[27292]: cluster 2023-08-28T00:02:41.972606+0000 mgr.y (mgr.14901) 3724 : cluster [DBG] pgmap v3731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:43.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:43 smithi006 bash[33918]: cluster 2023-08-28T00:02:41.972606+0000 mgr.y (mgr.14901) 3724 : cluster [DBG] pgmap v3731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:44.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:44 smithi130 bash[51302]: debug 2023-08-28T00:02:44.091+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:44.096144+0000) 2023-08-28T00:02:44.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:44 smithi130 bash[27579]: audit 2023-08-28T00:02:43.149402+0000 mon.c (mon.2) 1423 : audit [DBG] from='client.? 172.21.15.6:0/2403831688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:44.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:44 smithi006 bash[27292]: audit 2023-08-28T00:02:43.149402+0000 mon.c (mon.2) 1423 : audit [DBG] from='client.? 172.21.15.6:0/2403831688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:44.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:44 smithi006 bash[33918]: audit 2023-08-28T00:02:43.149402+0000 mon.c (mon.2) 1423 : audit [DBG] from='client.? 172.21.15.6:0/2403831688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:02:44.088Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:02:44.090Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:45 smithi130 bash[27579]: cluster 2023-08-28T00:02:43.973249+0000 mgr.y (mgr.14901) 3725 : cluster [DBG] pgmap v3732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:45 smithi130 bash[51302]: debug 2023-08-28T00:02:45.091+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:45.096456+0000) 2023-08-28T00:02:45.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:45 smithi006 bash[27292]: cluster 2023-08-28T00:02:43.973249+0000 mgr.y (mgr.14901) 3725 : cluster [DBG] pgmap v3732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:45.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:45 smithi006 bash[33918]: cluster 2023-08-28T00:02:43.973249+0000 mgr.y (mgr.14901) 3725 : cluster [DBG] pgmap v3732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:46.358 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:46 smithi130 bash[51302]: debug 2023-08-28T00:02:46.095+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:46.096743+0000) 2023-08-28T00:02:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:46 smithi130 bash[27579]: audit 2023-08-28T00:02:45.570191+0000 mon.c (mon.2) 1424 : audit [DBG] from='client.? 172.21.15.6:0/1345219218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:46 smithi130 bash[27579]: cluster 2023-08-28T00:02:45.974304+0000 mgr.y (mgr.14901) 3726 : cluster [DBG] pgmap v3733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:46 smithi006 bash[33918]: audit 2023-08-28T00:02:45.570191+0000 mon.c (mon.2) 1424 : audit [DBG] from='client.? 172.21.15.6:0/1345219218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:46 smithi006 bash[33918]: cluster 2023-08-28T00:02:45.974304+0000 mgr.y (mgr.14901) 3726 : cluster [DBG] pgmap v3733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:46 smithi006 bash[27292]: audit 2023-08-28T00:02:45.570191+0000 mon.c (mon.2) 1424 : audit [DBG] from='client.? 172.21.15.6:0/1345219218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:46 smithi006 bash[27292]: cluster 2023-08-28T00:02:45.974304+0000 mgr.y (mgr.14901) 3726 : cluster [DBG] pgmap v3733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:47.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:47 smithi130 bash[51302]: debug 2023-08-28T00:02:47.095+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:47.097028+0000) 2023-08-28T00:02:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:48 smithi130 bash[27579]: audit 2023-08-28T00:02:47.974606+0000 mon.a (mon.0) 2856 : audit [DBG] from='client.? 172.21.15.6:0/973997486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:48.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:48 smithi130 bash[51302]: debug 2023-08-28T00:02:48.091+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:48.097349+0000) 2023-08-28T00:02:48.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:48 smithi006 bash[33918]: audit 2023-08-28T00:02:47.974606+0000 mon.a (mon.0) 2856 : audit [DBG] from='client.? 172.21.15.6:0/973997486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:48.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:48 smithi006 bash[27292]: audit 2023-08-28T00:02:47.974606+0000 mon.a (mon.0) 2856 : audit [DBG] from='client.? 172.21.15.6:0/973997486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:49.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:49 smithi130 bash[27579]: cluster 2023-08-28T00:02:47.974775+0000 mgr.y (mgr.14901) 3727 : cluster [DBG] pgmap v3734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:49 smithi130 bash[51302]: debug 2023-08-28T00:02:49.091+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:49.097639+0000) 2023-08-28T00:02:49.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:49 smithi006 bash[33918]: cluster 2023-08-28T00:02:47.974775+0000 mgr.y (mgr.14901) 3727 : cluster [DBG] pgmap v3734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:49.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:49 smithi006 bash[27292]: cluster 2023-08-28T00:02:47.974775+0000 mgr.y (mgr.14901) 3727 : cluster [DBG] pgmap v3734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:50 smithi130 bash[51302]: debug 2023-08-28T00:02:50.095+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:50.097900+0000) 2023-08-28T00:02:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:02:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:02:50] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T00:02:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:02:50.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:02:50.632Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:51.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:51 smithi130 bash[27579]: cluster 2023-08-28T00:02:49.975792+0000 mgr.y (mgr.14901) 3728 : cluster [DBG] pgmap v3735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:51.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:51 smithi130 bash[27579]: audit 2023-08-28T00:02:50.399253+0000 mon.c (mon.2) 1425 : audit [DBG] from='client.? 172.21.15.6:0/665549737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:51.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:51 smithi130 bash[51302]: debug 2023-08-28T00:02:51.091+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:51.098196+0000) 2023-08-28T00:02:51.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:51 smithi006 bash[27292]: cluster 2023-08-28T00:02:49.975792+0000 mgr.y (mgr.14901) 3728 : cluster [DBG] pgmap v3735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:51.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:51 smithi006 bash[27292]: audit 2023-08-28T00:02:50.399253+0000 mon.c (mon.2) 1425 : audit [DBG] from='client.? 172.21.15.6:0/665549737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:51.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:51 smithi006 bash[33918]: cluster 2023-08-28T00:02:49.975792+0000 mgr.y (mgr.14901) 3728 : cluster [DBG] pgmap v3735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:51.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:51 smithi006 bash[33918]: audit 2023-08-28T00:02:50.399253+0000 mon.c (mon.2) 1425 : audit [DBG] from='client.? 172.21.15.6:0/665549737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:52.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:52 smithi130 bash[51302]: debug 2023-08-28T00:02:52.091+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:52.098411+0000) 2023-08-28T00:02:52.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:02:52.202Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:02:52.203Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:53.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:53 smithi130 bash[51302]: debug 2023-08-28T00:02:53.091+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:53.098677+0000) 2023-08-28T00:02:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:53 smithi130 bash[27579]: cluster 2023-08-28T00:02:51.976392+0000 mgr.y (mgr.14901) 3729 : cluster [DBG] pgmap v3736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:53 smithi130 bash[27579]: audit 2023-08-28T00:02:52.807951+0000 mon.a (mon.0) 2857 : audit [DBG] from='client.? 172.21.15.6:0/121315550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:53.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:53 smithi006 bash[33918]: cluster 2023-08-28T00:02:51.976392+0000 mgr.y (mgr.14901) 3729 : cluster [DBG] pgmap v3736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:53.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:53 smithi006 bash[33918]: audit 2023-08-28T00:02:52.807951+0000 mon.a (mon.0) 2857 : audit [DBG] from='client.? 172.21.15.6:0/121315550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:53 smithi006 bash[27292]: cluster 2023-08-28T00:02:51.976392+0000 mgr.y (mgr.14901) 3729 : cluster [DBG] pgmap v3736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:53 smithi006 bash[27292]: audit 2023-08-28T00:02:52.807951+0000 mon.a (mon.0) 2857 : audit [DBG] from='client.? 172.21.15.6:0/121315550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:54 smithi130 bash[51302]: debug 2023-08-28T00:02:54.095+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:54.098973+0000) 2023-08-28T00:02:54.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:02:54.088Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:02:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:02:54.090Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:02:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:55 smithi130 bash[27579]: cluster 2023-08-28T00:02:53.977051+0000 mgr.y (mgr.14901) 3730 : cluster [DBG] pgmap v3737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:55 smithi130 bash[51302]: debug 2023-08-28T00:02:55.095+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:55.099257+0000) 2023-08-28T00:02:55.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:55 smithi006 bash[33918]: cluster 2023-08-28T00:02:53.977051+0000 mgr.y (mgr.14901) 3730 : cluster [DBG] pgmap v3737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:55.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:55 smithi006 bash[27292]: cluster 2023-08-28T00:02:53.977051+0000 mgr.y (mgr.14901) 3730 : cluster [DBG] pgmap v3737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:56 smithi130 bash[27579]: audit 2023-08-28T00:02:55.224503+0000 mon.a (mon.0) 2858 : audit [DBG] from='client.? 172.21.15.6:0/3119269313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:56.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:56 smithi130 bash[51302]: debug 2023-08-28T00:02:56.095+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:56.099533+0000) 2023-08-28T00:02:56.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:56 smithi006 bash[27292]: audit 2023-08-28T00:02:55.224503+0000 mon.a (mon.0) 2858 : audit [DBG] from='client.? 172.21.15.6:0/3119269313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:56.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:56 smithi006 bash[33918]: audit 2023-08-28T00:02:55.224503+0000 mon.a (mon.0) 2858 : audit [DBG] from='client.? 172.21.15.6:0/3119269313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:57.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:57 smithi130 bash[27579]: cluster 2023-08-28T00:02:55.978031+0000 mgr.y (mgr.14901) 3731 : cluster [DBG] pgmap v3738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:57.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:57 smithi130 bash[51302]: debug 2023-08-28T00:02:57.095+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:57.099822+0000) 2023-08-28T00:02:57.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:57 smithi006 bash[33918]: cluster 2023-08-28T00:02:55.978031+0000 mgr.y (mgr.14901) 3731 : cluster [DBG] pgmap v3738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:57.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:57 smithi006 bash[27292]: cluster 2023-08-28T00:02:55.978031+0000 mgr.y (mgr.14901) 3731 : cluster [DBG] pgmap v3738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:58.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:58 smithi130 bash[51302]: debug 2023-08-28T00:02:58.095+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:58.100082+0000) 2023-08-28T00:02:58.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:58 smithi130 bash[27579]: audit 2023-08-28T00:02:57.637786+0000 mon.c (mon.2) 1426 : audit [DBG] from='client.? 172.21.15.6:0/871691615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:58.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:58 smithi006 bash[33918]: audit 2023-08-28T00:02:57.637786+0000 mon.c (mon.2) 1426 : audit [DBG] from='client.? 172.21.15.6:0/871691615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:58.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:58 smithi006 bash[27292]: audit 2023-08-28T00:02:57.637786+0000 mon.c (mon.2) 1426 : audit [DBG] from='client.? 172.21.15.6:0/871691615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:02:59 smithi130 bash[51302]: debug 2023-08-28T00:02:59.095+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:59.100416+0000) 2023-08-28T00:02:59.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:59 smithi130 bash[27579]: cluster 2023-08-28T00:02:57.978675+0000 mgr.y (mgr.14901) 3732 : cluster [DBG] pgmap v3739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:59.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:59 smithi130 bash[27579]: audit 2023-08-28T00:02:58.743514+0000 mon.b (mon.1) 472 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:02:59.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:59 smithi130 bash[27579]: audit 2023-08-28T00:02:59.018060+0000 mon.a (mon.0) 2859 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:02:59.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:02:59 smithi130 bash[27579]: audit 2023-08-28T00:02:59.024549+0000 mon.a (mon.0) 2860 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:02:59.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:59 smithi006 bash[27292]: cluster 2023-08-28T00:02:57.978675+0000 mgr.y (mgr.14901) 3732 : cluster [DBG] pgmap v3739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:59.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:59 smithi006 bash[27292]: audit 2023-08-28T00:02:58.743514+0000 mon.b (mon.1) 472 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:02:59.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:59 smithi006 bash[27292]: audit 2023-08-28T00:02:59.018060+0000 mon.a (mon.0) 2859 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:02:59.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:02:59 smithi006 bash[27292]: audit 2023-08-28T00:02:59.024549+0000 mon.a (mon.0) 2860 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:02:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:59 smithi006 bash[33918]: cluster 2023-08-28T00:02:57.978675+0000 mgr.y (mgr.14901) 3732 : cluster [DBG] pgmap v3739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:02:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:59 smithi006 bash[33918]: audit 2023-08-28T00:02:58.743514+0000 mon.b (mon.1) 472 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:02:59.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:59 smithi006 bash[33918]: audit 2023-08-28T00:02:59.018060+0000 mon.a (mon.0) 2859 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:02:59.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:02:59 smithi006 bash[33918]: audit 2023-08-28T00:02:59.024549+0000 mon.a (mon.0) 2860 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:03:00.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:00 smithi130 bash[27579]: audit 2023-08-28T00:03:00.052133+0000 mon.c (mon.2) 1427 : audit [DBG] from='client.? 172.21.15.6:0/1580855469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:00.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:00 smithi130 bash[51302]: debug 2023-08-28T00:03:00.095+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:00.100721+0000) 2023-08-28T00:03:00.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:00 smithi006 bash[33918]: audit 2023-08-28T00:03:00.052133+0000 mon.c (mon.2) 1427 : audit [DBG] from='client.? 172.21.15.6:0/1580855469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:00.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:00 smithi006 bash[27292]: audit 2023-08-28T00:03:00.052133+0000 mon.c (mon.2) 1427 : audit [DBG] from='client.? 172.21.15.6:0/1580855469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:03:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:03:00] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T00:03:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:03:00.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:03:00.633Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:01.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:01 smithi130 bash[27579]: cluster 2023-08-28T00:02:59.979351+0000 mgr.y (mgr.14901) 3733 : cluster [DBG] pgmap v3740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:01.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:01 smithi130 bash[51302]: debug 2023-08-28T00:03:01.095+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:01.101028+0000) 2023-08-28T00:03:01.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:01 smithi006 bash[27292]: cluster 2023-08-28T00:02:59.979351+0000 mgr.y (mgr.14901) 3733 : cluster [DBG] pgmap v3740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:01.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:01 smithi006 bash[33918]: cluster 2023-08-28T00:02:59.979351+0000 mgr.y (mgr.14901) 3733 : cluster [DBG] pgmap v3740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:02.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:02 smithi130 bash[51302]: debug 2023-08-28T00:03:02.095+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:02.101288+0000) 2023-08-28T00:03:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:03:02.202Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:03:02.204Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:03.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:03 smithi130 bash[27579]: cluster 2023-08-28T00:03:01.979968+0000 mgr.y (mgr.14901) 3734 : cluster [DBG] pgmap v3741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:03.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:03 smithi130 bash[27579]: audit 2023-08-28T00:03:02.480244+0000 mon.a (mon.0) 2861 : audit [DBG] from='client.? 172.21.15.6:0/3222163110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:03 smithi130 bash[51302]: debug 2023-08-28T00:03:03.095+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:03.101565+0000) 2023-08-28T00:03:03.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:03 smithi006 bash[27292]: cluster 2023-08-28T00:03:01.979968+0000 mgr.y (mgr.14901) 3734 : cluster [DBG] pgmap v3741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:03.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:03 smithi006 bash[27292]: audit 2023-08-28T00:03:02.480244+0000 mon.a (mon.0) 2861 : audit [DBG] from='client.? 172.21.15.6:0/3222163110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:03.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:03 smithi006 bash[33918]: cluster 2023-08-28T00:03:01.979968+0000 mgr.y (mgr.14901) 3734 : cluster [DBG] pgmap v3741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:03.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:03 smithi006 bash[33918]: audit 2023-08-28T00:03:02.480244+0000 mon.a (mon.0) 2861 : audit [DBG] from='client.? 172.21.15.6:0/3222163110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:04 smithi130 bash[51302]: debug 2023-08-28T00:03:04.095+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:04.101787+0000) 2023-08-28T00:03:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:03:04.089Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:03:04.091Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:05 smithi130 bash[51302]: debug 2023-08-28T00:03:05.095+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:05.102069+0000) 2023-08-28T00:03:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:05 smithi130 bash[27579]: cluster 2023-08-28T00:03:03.980588+0000 mgr.y (mgr.14901) 3735 : cluster [DBG] pgmap v3742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:05 smithi130 bash[27579]: audit 2023-08-28T00:03:04.867953+0000 mon.a (mon.0) 2862 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:03:05.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:05 smithi130 bash[27579]: audit 2023-08-28T00:03:04.915125+0000 mon.a (mon.0) 2863 : audit [DBG] from='client.? 172.21.15.6:0/1286410767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:05.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:05 smithi006 bash[33918]: cluster 2023-08-28T00:03:03.980588+0000 mgr.y (mgr.14901) 3735 : cluster [DBG] pgmap v3742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:05.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:05 smithi006 bash[33918]: audit 2023-08-28T00:03:04.867953+0000 mon.a (mon.0) 2862 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:03:05.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:05 smithi006 bash[33918]: audit 2023-08-28T00:03:04.915125+0000 mon.a (mon.0) 2863 : audit [DBG] from='client.? 172.21.15.6:0/1286410767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:05.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:05 smithi006 bash[27292]: cluster 2023-08-28T00:03:03.980588+0000 mgr.y (mgr.14901) 3735 : cluster [DBG] pgmap v3742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:05.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:05 smithi006 bash[27292]: audit 2023-08-28T00:03:04.867953+0000 mon.a (mon.0) 2862 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:03:05.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:05 smithi006 bash[27292]: audit 2023-08-28T00:03:04.915125+0000 mon.a (mon.0) 2863 : audit [DBG] from='client.? 172.21.15.6:0/1286410767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:06.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:06 smithi130 bash[51302]: debug 2023-08-28T00:03:06.099+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:06.102368+0000) 2023-08-28T00:03:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:06 smithi130 bash[27579]: audit 2023-08-28T00:03:05.306736+0000 mon.a (mon.0) 2864 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:03:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:06 smithi130 bash[27579]: audit 2023-08-28T00:03:05.581298+0000 mon.b (mon.1) 473 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:03:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:06 smithi130 bash[27579]: audit 2023-08-28T00:03:05.583540+0000 mon.b (mon.1) 474 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:03:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:06 smithi130 bash[27579]: audit 2023-08-28T00:03:05.594715+0000 mon.a (mon.0) 2865 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:03:06.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:06 smithi130 bash[27579]: cluster 2023-08-28T00:03:05.981794+0000 mgr.y (mgr.14901) 3736 : cluster [DBG] pgmap v3743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:06 smithi006 bash[27292]: audit 2023-08-28T00:03:05.306736+0000 mon.a (mon.0) 2864 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:03:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:06 smithi006 bash[27292]: audit 2023-08-28T00:03:05.581298+0000 mon.b (mon.1) 473 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:03:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:06 smithi006 bash[27292]: audit 2023-08-28T00:03:05.583540+0000 mon.b (mon.1) 474 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:03:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:06 smithi006 bash[27292]: audit 2023-08-28T00:03:05.594715+0000 mon.a (mon.0) 2865 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:03:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:06 smithi006 bash[27292]: cluster 2023-08-28T00:03:05.981794+0000 mgr.y (mgr.14901) 3736 : cluster [DBG] pgmap v3743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:06 smithi006 bash[33918]: audit 2023-08-28T00:03:05.306736+0000 mon.a (mon.0) 2864 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:03:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:06 smithi006 bash[33918]: audit 2023-08-28T00:03:05.581298+0000 mon.b (mon.1) 473 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:03:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:06 smithi006 bash[33918]: audit 2023-08-28T00:03:05.583540+0000 mon.b (mon.1) 474 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:03:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:06 smithi006 bash[33918]: audit 2023-08-28T00:03:05.594715+0000 mon.a (mon.0) 2865 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:03:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:06 smithi006 bash[33918]: cluster 2023-08-28T00:03:05.981794+0000 mgr.y (mgr.14901) 3736 : cluster [DBG] pgmap v3743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:07.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:07 smithi130 bash[51302]: debug 2023-08-28T00:03:07.099+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:07.102663+0000) 2023-08-28T00:03:07.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:07 smithi130 bash[27579]: audit 2023-08-28T00:03:07.328279+0000 mon.a (mon.0) 2866 : audit [DBG] from='client.? 172.21.15.6:0/3588741986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:07.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:07 smithi006 bash[33918]: audit 2023-08-28T00:03:07.328279+0000 mon.a (mon.0) 2866 : audit [DBG] from='client.? 172.21.15.6:0/3588741986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:07.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:07 smithi006 bash[27292]: audit 2023-08-28T00:03:07.328279+0000 mon.a (mon.0) 2866 : audit [DBG] from='client.? 172.21.15.6:0/3588741986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:08.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:08 smithi130 bash[51302]: debug 2023-08-28T00:03:08.099+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:08.103013+0000) 2023-08-28T00:03:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:08 smithi130 bash[27579]: cluster 2023-08-28T00:03:07.982493+0000 mgr.y (mgr.14901) 3737 : cluster [DBG] pgmap v3744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:08 smithi006 bash[33918]: cluster 2023-08-28T00:03:07.982493+0000 mgr.y (mgr.14901) 3737 : cluster [DBG] pgmap v3744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:08 smithi006 bash[27292]: cluster 2023-08-28T00:03:07.982493+0000 mgr.y (mgr.14901) 3737 : cluster [DBG] pgmap v3744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:09.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:09 smithi130 bash[51302]: debug 2023-08-28T00:03:09.099+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:09.103334+0000) 2023-08-28T00:03:10.104 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:09 smithi130 bash[27579]: audit 2023-08-28T00:03:09.740677+0000 mon.c (mon.2) 1428 : audit [DBG] from='client.? 172.21.15.6:0/219646074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:10.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:09 smithi006 bash[27292]: audit 2023-08-28T00:03:09.740677+0000 mon.c (mon.2) 1428 : audit [DBG] from='client.? 172.21.15.6:0/219646074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:10.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:09 smithi006 bash[33918]: audit 2023-08-28T00:03:09.740677+0000 mon.c (mon.2) 1428 : audit [DBG] from='client.? 172.21.15.6:0/219646074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:10.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:10 smithi130 bash[51302]: debug 2023-08-28T00:03:10.103+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:10.103633+0000) 2023-08-28T00:03:10.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:10 smithi006 bash[27292]: cluster 2023-08-28T00:03:09.983635+0000 mgr.y (mgr.14901) 3738 : cluster [DBG] pgmap v3745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:03:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:03:10] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T00:03:10.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:10 smithi006 bash[33918]: cluster 2023-08-28T00:03:09.983635+0000 mgr.y (mgr.14901) 3738 : cluster [DBG] pgmap v3745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:03:10.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:03:10.633Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:11.105 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:10 smithi130 bash[27579]: cluster 2023-08-28T00:03:09.983635+0000 mgr.y (mgr.14901) 3738 : cluster [DBG] pgmap v3745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:11.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:11 smithi130 bash[51302]: debug 2023-08-28T00:03:11.099+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:11.103917+0000) 2023-08-28T00:03:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:12 smithi130 bash[51302]: debug 2023-08-28T00:03:12.099+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:12.104148+0000) 2023-08-28T00:03:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:03:12.203Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:03:12.205Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:13 smithi130 bash[27579]: cluster 2023-08-28T00:03:11.984280+0000 mgr.y (mgr.14901) 3739 : cluster [DBG] pgmap v3746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:13 smithi130 bash[27579]: audit 2023-08-28T00:03:12.151523+0000 mon.a (mon.0) 2867 : audit [DBG] from='client.? 172.21.15.6:0/709175489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:13 smithi130 bash[51302]: debug 2023-08-28T00:03:13.103+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:13.104475+0000) 2023-08-28T00:03:13.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:13 smithi006 bash[33918]: cluster 2023-08-28T00:03:11.984280+0000 mgr.y (mgr.14901) 3739 : cluster [DBG] pgmap v3746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:13.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:13 smithi006 bash[33918]: audit 2023-08-28T00:03:12.151523+0000 mon.a (mon.0) 2867 : audit [DBG] from='client.? 172.21.15.6:0/709175489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:13.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:13 smithi006 bash[27292]: cluster 2023-08-28T00:03:11.984280+0000 mgr.y (mgr.14901) 3739 : cluster [DBG] pgmap v3746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:13.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:13 smithi006 bash[27292]: audit 2023-08-28T00:03:12.151523+0000 mon.a (mon.0) 2867 : audit [DBG] from='client.? 172.21.15.6:0/709175489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:14.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:14 smithi130 bash[51302]: debug 2023-08-28T00:03:14.103+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:14.104783+0000) 2023-08-28T00:03:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:03:14.090Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:03:14.092Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:15 smithi130 bash[51302]: debug 2023-08-28T00:03:15.103+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:15.105117+0000) 2023-08-28T00:03:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:15 smithi130 bash[27579]: cluster 2023-08-28T00:03:13.985028+0000 mgr.y (mgr.14901) 3740 : cluster [DBG] pgmap v3747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:15 smithi130 bash[27579]: audit 2023-08-28T00:03:14.559916+0000 mon.a (mon.0) 2868 : audit [DBG] from='client.? 172.21.15.6:0/2087718425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:15.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:15 smithi006 bash[27292]: cluster 2023-08-28T00:03:13.985028+0000 mgr.y (mgr.14901) 3740 : cluster [DBG] pgmap v3747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:15.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:15 smithi006 bash[27292]: audit 2023-08-28T00:03:14.559916+0000 mon.a (mon.0) 2868 : audit [DBG] from='client.? 172.21.15.6:0/2087718425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:15.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:15 smithi006 bash[33918]: cluster 2023-08-28T00:03:13.985028+0000 mgr.y (mgr.14901) 3740 : cluster [DBG] pgmap v3747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:15.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:15 smithi006 bash[33918]: audit 2023-08-28T00:03:14.559916+0000 mon.a (mon.0) 2868 : audit [DBG] from='client.? 172.21.15.6:0/2087718425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:16.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:16 smithi130 bash[51302]: debug 2023-08-28T00:03:16.099+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:16.105412+0000) 2023-08-28T00:03:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:16 smithi130 bash[27579]: cluster 2023-08-28T00:03:15.986134+0000 mgr.y (mgr.14901) 3741 : cluster [DBG] pgmap v3748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:16.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:16 smithi006 bash[27292]: cluster 2023-08-28T00:03:15.986134+0000 mgr.y (mgr.14901) 3741 : cluster [DBG] pgmap v3748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:16 smithi006 bash[33918]: cluster 2023-08-28T00:03:15.986134+0000 mgr.y (mgr.14901) 3741 : cluster [DBG] pgmap v3748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:17.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:17 smithi130 bash[51302]: debug 2023-08-28T00:03:17.103+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:17.105671+0000) 2023-08-28T00:03:17.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:17 smithi130 bash[27579]: audit 2023-08-28T00:03:16.973102+0000 mon.a (mon.0) 2869 : audit [DBG] from='client.? 172.21.15.6:0/3968421638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:17.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:17 smithi006 bash[33918]: audit 2023-08-28T00:03:16.973102+0000 mon.a (mon.0) 2869 : audit [DBG] from='client.? 172.21.15.6:0/3968421638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:17.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:17 smithi006 bash[27292]: audit 2023-08-28T00:03:16.973102+0000 mon.a (mon.0) 2869 : audit [DBG] from='client.? 172.21.15.6:0/3968421638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:18.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:18 smithi130 bash[51302]: debug 2023-08-28T00:03:18.103+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:18.105952+0000) 2023-08-28T00:03:18.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:18 smithi006 bash[27292]: cluster 2023-08-28T00:03:17.986753+0000 mgr.y (mgr.14901) 3742 : cluster [DBG] pgmap v3749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:18 smithi006 bash[33918]: cluster 2023-08-28T00:03:17.986753+0000 mgr.y (mgr.14901) 3742 : cluster [DBG] pgmap v3749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:18.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:18 smithi130 bash[27579]: cluster 2023-08-28T00:03:17.986753+0000 mgr.y (mgr.14901) 3742 : cluster [DBG] pgmap v3749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:19.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:19 smithi130 bash[51302]: debug 2023-08-28T00:03:19.103+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:19.106246+0000) 2023-08-28T00:03:19.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:19 smithi006 bash[27292]: audit 2023-08-28T00:03:19.389591+0000 mon.a (mon.0) 2870 : audit [DBG] from='client.? 172.21.15.6:0/2934052730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:19.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:19 smithi006 bash[33918]: audit 2023-08-28T00:03:19.389591+0000 mon.a (mon.0) 2870 : audit [DBG] from='client.? 172.21.15.6:0/2934052730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:19.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:19 smithi130 bash[27579]: audit 2023-08-28T00:03:19.389591+0000 mon.a (mon.0) 2870 : audit [DBG] from='client.? 172.21.15.6:0/2934052730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:20.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:20 smithi130 bash[51302]: debug 2023-08-28T00:03:20.103+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:20.106579+0000) 2023-08-28T00:03:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:20 smithi006 bash[33918]: cluster 2023-08-28T00:03:19.987882+0000 mgr.y (mgr.14901) 3743 : cluster [DBG] pgmap v3750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:20 smithi006 bash[27292]: cluster 2023-08-28T00:03:19.987882+0000 mgr.y (mgr.14901) 3743 : cluster [DBG] pgmap v3750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:03:20.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:20.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:03:20.634Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:20 smithi130 bash[27579]: cluster 2023-08-28T00:03:19.987882+0000 mgr.y (mgr.14901) 3743 : cluster [DBG] pgmap v3750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:03:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:03:20] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T00:03:21.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:21 smithi130 bash[51302]: debug 2023-08-28T00:03:21.103+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:21.106872+0000) 2023-08-28T00:03:22.206 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:21 smithi006 bash[33918]: audit 2023-08-28T00:03:21.813238+0000 mon.a (mon.0) 2871 : audit [DBG] from='client.? 172.21.15.6:0/480312352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:22.206 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:21 smithi006 bash[27292]: audit 2023-08-28T00:03:21.813238+0000 mon.a (mon.0) 2871 : audit [DBG] from='client.? 172.21.15.6:0/480312352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:22.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:21 smithi130 bash[27579]: audit 2023-08-28T00:03:21.813238+0000 mon.a (mon.0) 2871 : audit [DBG] from='client.? 172.21.15.6:0/480312352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:22.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:22 smithi130 bash[51302]: debug 2023-08-28T00:03:22.103+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:22.107073+0000) 2023-08-28T00:03:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:03:22.203Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:03:22.205Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:22 smithi006 bash[33918]: cluster 2023-08-28T00:03:21.988539+0000 mgr.y (mgr.14901) 3744 : cluster [DBG] pgmap v3751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:22 smithi006 bash[27292]: cluster 2023-08-28T00:03:21.988539+0000 mgr.y (mgr.14901) 3744 : cluster [DBG] pgmap v3751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:23.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:22 smithi130 bash[27579]: cluster 2023-08-28T00:03:21.988539+0000 mgr.y (mgr.14901) 3744 : cluster [DBG] pgmap v3751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:23.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:23 smithi130 bash[51302]: debug 2023-08-28T00:03:23.103+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:23.107277+0000) 2023-08-28T00:03:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:24 smithi130 bash[51302]: debug 2023-08-28T00:03:24.103+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:24.107508+0000) 2023-08-28T00:03:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:03:24.090Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:03:24.092Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:25 smithi130 bash[51302]: debug 2023-08-28T00:03:25.103+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:25.107779+0000) 2023-08-28T00:03:25.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:25 smithi130 bash[27579]: cluster 2023-08-28T00:03:23.989225+0000 mgr.y (mgr.14901) 3745 : cluster [DBG] pgmap v3752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:25.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:25 smithi130 bash[27579]: audit 2023-08-28T00:03:24.223529+0000 mon.a (mon.0) 2872 : audit [DBG] from='client.? 172.21.15.6:0/63667095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:25.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:25 smithi006 bash[33918]: cluster 2023-08-28T00:03:23.989225+0000 mgr.y (mgr.14901) 3745 : cluster [DBG] pgmap v3752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:25.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:25 smithi006 bash[33918]: audit 2023-08-28T00:03:24.223529+0000 mon.a (mon.0) 2872 : audit [DBG] from='client.? 172.21.15.6:0/63667095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:25.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:25 smithi006 bash[27292]: cluster 2023-08-28T00:03:23.989225+0000 mgr.y (mgr.14901) 3745 : cluster [DBG] pgmap v3752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:25.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:25 smithi006 bash[27292]: audit 2023-08-28T00:03:24.223529+0000 mon.a (mon.0) 2872 : audit [DBG] from='client.? 172.21.15.6:0/63667095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:26.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:26 smithi130 bash[51302]: debug 2023-08-28T00:03:26.103+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:26.107995+0000) 2023-08-28T00:03:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:26 smithi130 bash[27579]: cluster 2023-08-28T00:03:25.990303+0000 mgr.y (mgr.14901) 3746 : cluster [DBG] pgmap v3753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:26 smithi006 bash[27292]: cluster 2023-08-28T00:03:25.990303+0000 mgr.y (mgr.14901) 3746 : cluster [DBG] pgmap v3753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:26 smithi006 bash[33918]: cluster 2023-08-28T00:03:25.990303+0000 mgr.y (mgr.14901) 3746 : cluster [DBG] pgmap v3753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:27.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:27 smithi130 bash[51302]: debug 2023-08-28T00:03:27.103+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:27.108287+0000) 2023-08-28T00:03:27.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:27 smithi006 bash[27292]: audit 2023-08-28T00:03:26.635796+0000 mon.a (mon.0) 2873 : audit [DBG] from='client.? 172.21.15.6:0/1449589812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:27.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:27 smithi006 bash[33918]: audit 2023-08-28T00:03:26.635796+0000 mon.a (mon.0) 2873 : audit [DBG] from='client.? 172.21.15.6:0/1449589812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:27 smithi130 bash[27579]: audit 2023-08-28T00:03:26.635796+0000 mon.a (mon.0) 2873 : audit [DBG] from='client.? 172.21.15.6:0/1449589812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:28.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:28 smithi130 bash[51302]: debug 2023-08-28T00:03:28.103+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:28.108577+0000) 2023-08-28T00:03:28.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:28 smithi006 bash[27292]: cluster 2023-08-28T00:03:27.990974+0000 mgr.y (mgr.14901) 3747 : cluster [DBG] pgmap v3754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:28 smithi006 bash[33918]: cluster 2023-08-28T00:03:27.990974+0000 mgr.y (mgr.14901) 3747 : cluster [DBG] pgmap v3754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:28 smithi130 bash[27579]: cluster 2023-08-28T00:03:27.990974+0000 mgr.y (mgr.14901) 3747 : cluster [DBG] pgmap v3754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:29.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:29 smithi130 bash[51302]: debug 2023-08-28T00:03:29.107+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:29.108864+0000) 2023-08-28T00:03:29.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:29 smithi006 bash[33918]: audit 2023-08-28T00:03:29.047877+0000 mon.c (mon.2) 1429 : audit [DBG] from='client.? 172.21.15.6:0/926681674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:29.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:29 smithi006 bash[27292]: audit 2023-08-28T00:03:29.047877+0000 mon.c (mon.2) 1429 : audit [DBG] from='client.? 172.21.15.6:0/926681674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:29.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:29 smithi130 bash[27579]: audit 2023-08-28T00:03:29.047877+0000 mon.c (mon.2) 1429 : audit [DBG] from='client.? 172.21.15.6:0/926681674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:30 smithi130 bash[51302]: debug 2023-08-28T00:03:30.107+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:30.109154+0000) 2023-08-28T00:03:30.636 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:30 smithi006 bash[33918]: cluster 2023-08-28T00:03:29.992120+0000 mgr.y (mgr.14901) 3748 : cluster [DBG] pgmap v3755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:30.636 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:30 smithi006 bash[27292]: cluster 2023-08-28T00:03:29.992120+0000 mgr.y (mgr.14901) 3748 : cluster [DBG] pgmap v3755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:30 smithi130 bash[27579]: cluster 2023-08-28T00:03:29.992120+0000 mgr.y (mgr.14901) 3748 : cluster [DBG] pgmap v3755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:30.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:03:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:03:30] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T00:03:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:03:30.633Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:03:30.635Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:31.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:31 smithi130 bash[51302]: debug 2023-08-28T00:03:31.103+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:31.109449+0000) 2023-08-28T00:03:31.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:31 smithi130 bash[27579]: audit 2023-08-28T00:03:31.461495+0000 mon.a (mon.0) 2874 : audit [DBG] from='client.? 172.21.15.6:0/1521198583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:31.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:31 smithi006 bash[27292]: audit 2023-08-28T00:03:31.461495+0000 mon.a (mon.0) 2874 : audit [DBG] from='client.? 172.21.15.6:0/1521198583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:31.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:31 smithi006 bash[33918]: audit 2023-08-28T00:03:31.461495+0000 mon.a (mon.0) 2874 : audit [DBG] from='client.? 172.21.15.6:0/1521198583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:32.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:32 smithi130 bash[51302]: debug 2023-08-28T00:03:32.103+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:32.109709+0000) 2023-08-28T00:03:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:03:32.204Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:03:32.206Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:32 smithi130 bash[27579]: cluster 2023-08-28T00:03:31.992893+0000 mgr.y (mgr.14901) 3749 : cluster [DBG] pgmap v3756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:32 smithi006 bash[33918]: cluster 2023-08-28T00:03:31.992893+0000 mgr.y (mgr.14901) 3749 : cluster [DBG] pgmap v3756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:32.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:32 smithi006 bash[27292]: cluster 2023-08-28T00:03:31.992893+0000 mgr.y (mgr.14901) 3749 : cluster [DBG] pgmap v3756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:33.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:33 smithi130 bash[51302]: debug 2023-08-28T00:03:33.107+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:33.109980+0000) 2023-08-28T00:03:34.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:33 smithi006 bash[33918]: audit 2023-08-28T00:03:33.876107+0000 mon.a (mon.0) 2875 : audit [DBG] from='client.? 172.21.15.6:0/638939495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:34.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:33 smithi006 bash[27292]: audit 2023-08-28T00:03:33.876107+0000 mon.a (mon.0) 2875 : audit [DBG] from='client.? 172.21.15.6:0/638939495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:03:34.091Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:03:34.093Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:34.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:33 smithi130 bash[27579]: audit 2023-08-28T00:03:33.876107+0000 mon.a (mon.0) 2875 : audit [DBG] from='client.? 172.21.15.6:0/638939495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:34.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:34 smithi130 bash[51302]: debug 2023-08-28T00:03:34.107+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:34.110259+0000) 2023-08-28T00:03:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:34 smithi006 bash[33918]: cluster 2023-08-28T00:03:33.993499+0000 mgr.y (mgr.14901) 3750 : cluster [DBG] pgmap v3757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:34 smithi006 bash[27292]: cluster 2023-08-28T00:03:33.993499+0000 mgr.y (mgr.14901) 3750 : cluster [DBG] pgmap v3757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:35.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:35 smithi130 bash[51302]: debug 2023-08-28T00:03:35.107+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:35.110548+0000) 2023-08-28T00:03:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:34 smithi130 bash[27579]: cluster 2023-08-28T00:03:33.993499+0000 mgr.y (mgr.14901) 3750 : cluster [DBG] pgmap v3757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:36.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:36 smithi130 bash[51302]: debug 2023-08-28T00:03:36.106+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:36.110908+0000) 2023-08-28T00:03:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:36 smithi006 bash[33918]: cluster 2023-08-28T00:03:35.994602+0000 mgr.y (mgr.14901) 3751 : cluster [DBG] pgmap v3758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:36 smithi006 bash[33918]: audit 2023-08-28T00:03:36.309937+0000 mon.a (mon.0) 2876 : audit [DBG] from='client.? 172.21.15.6:0/1400431624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:36 smithi006 bash[27292]: cluster 2023-08-28T00:03:35.994602+0000 mgr.y (mgr.14901) 3751 : cluster [DBG] pgmap v3758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:36 smithi006 bash[27292]: audit 2023-08-28T00:03:36.309937+0000 mon.a (mon.0) 2876 : audit [DBG] from='client.? 172.21.15.6:0/1400431624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:36 smithi130 bash[27579]: cluster 2023-08-28T00:03:35.994602+0000 mgr.y (mgr.14901) 3751 : cluster [DBG] pgmap v3758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:36 smithi130 bash[27579]: audit 2023-08-28T00:03:36.309937+0000 mon.a (mon.0) 2876 : audit [DBG] from='client.? 172.21.15.6:0/1400431624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:37 smithi130 bash[51302]: debug 2023-08-28T00:03:37.110+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:37.111177+0000) 2023-08-28T00:03:38.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:38 smithi130 bash[51302]: debug 2023-08-28T00:03:38.110+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:38.111452+0000) 2023-08-28T00:03:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:39 smithi130 bash[27579]: cluster 2023-08-28T00:03:37.995221+0000 mgr.y (mgr.14901) 3752 : cluster [DBG] pgmap v3759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:39 smithi130 bash[27579]: audit 2023-08-28T00:03:38.717569+0000 mon.a (mon.0) 2877 : audit [DBG] from='client.? 172.21.15.6:0/1079026785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:39.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:39 smithi130 bash[51302]: debug 2023-08-28T00:03:39.106+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:39.111740+0000) 2023-08-28T00:03:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:39 smithi006 bash[27292]: cluster 2023-08-28T00:03:37.995221+0000 mgr.y (mgr.14901) 3752 : cluster [DBG] pgmap v3759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:39.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:39 smithi006 bash[27292]: audit 2023-08-28T00:03:38.717569+0000 mon.a (mon.0) 2877 : audit [DBG] from='client.? 172.21.15.6:0/1079026785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:39.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:39 smithi006 bash[33918]: cluster 2023-08-28T00:03:37.995221+0000 mgr.y (mgr.14901) 3752 : cluster [DBG] pgmap v3759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:39.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:39 smithi006 bash[33918]: audit 2023-08-28T00:03:38.717569+0000 mon.a (mon.0) 2877 : audit [DBG] from='client.? 172.21.15.6:0/1079026785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:40.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:40 smithi130 bash[51302]: debug 2023-08-28T00:03:40.110+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:40.112026+0000) 2023-08-28T00:03:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:03:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:03:40] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T00:03:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:03:40.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:03:40.636Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:41.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:41 smithi130 bash[27579]: cluster 2023-08-28T00:03:39.996374+0000 mgr.y (mgr.14901) 3753 : cluster [DBG] pgmap v3760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:41.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:41 smithi130 bash[51302]: debug 2023-08-28T00:03:41.110+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:41.112319+0000) 2023-08-28T00:03:41.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:41 smithi006 bash[27292]: cluster 2023-08-28T00:03:39.996374+0000 mgr.y (mgr.14901) 3753 : cluster [DBG] pgmap v3760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:41.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:41 smithi006 bash[33918]: cluster 2023-08-28T00:03:39.996374+0000 mgr.y (mgr.14901) 3753 : cluster [DBG] pgmap v3760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:42 smithi130 bash[27579]: audit 2023-08-28T00:03:41.137249+0000 mon.a (mon.0) 2878 : audit [DBG] from='client.? 172.21.15.6:0/2347544503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:42.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:42 smithi130 bash[51302]: debug 2023-08-28T00:03:42.106+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:42.112584+0000) 2023-08-28T00:03:42.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:42 smithi006 bash[27292]: audit 2023-08-28T00:03:41.137249+0000 mon.a (mon.0) 2878 : audit [DBG] from='client.? 172.21.15.6:0/2347544503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:42.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:42 smithi006 bash[33918]: audit 2023-08-28T00:03:41.137249+0000 mon.a (mon.0) 2878 : audit [DBG] from='client.? 172.21.15.6:0/2347544503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:03:42.206Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:03:42.208Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:43.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:43 smithi130 bash[27579]: cluster 2023-08-28T00:03:41.997073+0000 mgr.y (mgr.14901) 3754 : cluster [DBG] pgmap v3761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:43.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:43 smithi130 bash[51302]: debug 2023-08-28T00:03:43.110+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:43.112854+0000) 2023-08-28T00:03:43.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:43 smithi006 bash[33918]: cluster 2023-08-28T00:03:41.997073+0000 mgr.y (mgr.14901) 3754 : cluster [DBG] pgmap v3761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:43.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:43 smithi006 bash[27292]: cluster 2023-08-28T00:03:41.997073+0000 mgr.y (mgr.14901) 3754 : cluster [DBG] pgmap v3761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:44 smithi130 bash[27579]: audit 2023-08-28T00:03:43.557827+0000 mon.a (mon.0) 2879 : audit [DBG] from='client.? 172.21.15.6:0/2164570939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:44.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:44 smithi130 bash[51302]: debug 2023-08-28T00:03:44.110+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:44.113134+0000) 2023-08-28T00:03:44.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:44 smithi006 bash[33918]: audit 2023-08-28T00:03:43.557827+0000 mon.a (mon.0) 2879 : audit [DBG] from='client.? 172.21.15.6:0/2164570939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:44.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:44 smithi006 bash[27292]: audit 2023-08-28T00:03:43.557827+0000 mon.a (mon.0) 2879 : audit [DBG] from='client.? 172.21.15.6:0/2164570939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:03:44.092Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:03:44.094Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:45 smithi130 bash[27579]: cluster 2023-08-28T00:03:43.997680+0000 mgr.y (mgr.14901) 3755 : cluster [DBG] pgmap v3762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:45 smithi130 bash[51302]: debug 2023-08-28T00:03:45.110+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:45.113564+0000) 2023-08-28T00:03:45.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:45 smithi006 bash[27292]: cluster 2023-08-28T00:03:43.997680+0000 mgr.y (mgr.14901) 3755 : cluster [DBG] pgmap v3762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:45.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:45 smithi006 bash[33918]: cluster 2023-08-28T00:03:43.997680+0000 mgr.y (mgr.14901) 3755 : cluster [DBG] pgmap v3762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:46 smithi130 bash[51302]: debug 2023-08-28T00:03:46.110+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:46.114114+0000) 2023-08-28T00:03:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:46 smithi006 bash[33918]: audit 2023-08-28T00:03:45.971823+0000 mon.a (mon.0) 2880 : audit [DBG] from='client.? 172.21.15.6:0/2829711947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:46 smithi006 bash[33918]: cluster 2023-08-28T00:03:45.998782+0000 mgr.y (mgr.14901) 3756 : cluster [DBG] pgmap v3763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:46 smithi006 bash[27292]: audit 2023-08-28T00:03:45.971823+0000 mon.a (mon.0) 2880 : audit [DBG] from='client.? 172.21.15.6:0/2829711947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:46 smithi006 bash[27292]: cluster 2023-08-28T00:03:45.998782+0000 mgr.y (mgr.14901) 3756 : cluster [DBG] pgmap v3763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:46 smithi130 bash[27579]: audit 2023-08-28T00:03:45.971823+0000 mon.a (mon.0) 2880 : audit [DBG] from='client.? 172.21.15.6:0/2829711947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:46 smithi130 bash[27579]: cluster 2023-08-28T00:03:45.998782+0000 mgr.y (mgr.14901) 3756 : cluster [DBG] pgmap v3763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:47 smithi130 bash[51302]: debug 2023-08-28T00:03:47.110+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:47.114435+0000) 2023-08-28T00:03:48.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:48 smithi130 bash[51302]: debug 2023-08-28T00:03:48.110+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:48.114689+0000) 2023-08-28T00:03:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:49 smithi130 bash[51302]: debug 2023-08-28T00:03:49.114+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:49.114987+0000) 2023-08-28T00:03:49.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:49 smithi130 bash[27579]: cluster 2023-08-28T00:03:47.999408+0000 mgr.y (mgr.14901) 3757 : cluster [DBG] pgmap v3764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:49.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:49 smithi130 bash[27579]: audit 2023-08-28T00:03:48.386305+0000 mon.a (mon.0) 2881 : audit [DBG] from='client.? 172.21.15.6:0/2208329255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:49.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:49 smithi006 bash[33918]: cluster 2023-08-28T00:03:47.999408+0000 mgr.y (mgr.14901) 3757 : cluster [DBG] pgmap v3764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:49.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:49 smithi006 bash[33918]: audit 2023-08-28T00:03:48.386305+0000 mon.a (mon.0) 2881 : audit [DBG] from='client.? 172.21.15.6:0/2208329255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:49.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:49 smithi006 bash[27292]: cluster 2023-08-28T00:03:47.999408+0000 mgr.y (mgr.14901) 3757 : cluster [DBG] pgmap v3764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:49.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:49 smithi006 bash[27292]: audit 2023-08-28T00:03:48.386305+0000 mon.a (mon.0) 2881 : audit [DBG] from='client.? 172.21.15.6:0/2208329255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:50 smithi130 bash[51302]: debug 2023-08-28T00:03:50.114+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:50.115299+0000) 2023-08-28T00:03:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:03:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:03:50] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T00:03:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:03:50.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:03:50.636Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:51.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:51 smithi130 bash[51302]: debug 2023-08-28T00:03:51.114+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:51.115613+0000) 2023-08-28T00:03:51.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:51 smithi130 bash[27579]: cluster 2023-08-28T00:03:50.000533+0000 mgr.y (mgr.14901) 3758 : cluster [DBG] pgmap v3765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:51.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:51 smithi130 bash[27579]: audit 2023-08-28T00:03:50.815648+0000 mon.c (mon.2) 1430 : audit [DBG] from='client.? 172.21.15.6:0/382886005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:51.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:51 smithi006 bash[27292]: cluster 2023-08-28T00:03:50.000533+0000 mgr.y (mgr.14901) 3758 : cluster [DBG] pgmap v3765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:51.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:51 smithi006 bash[27292]: audit 2023-08-28T00:03:50.815648+0000 mon.c (mon.2) 1430 : audit [DBG] from='client.? 172.21.15.6:0/382886005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:51.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:51 smithi006 bash[33918]: cluster 2023-08-28T00:03:50.000533+0000 mgr.y (mgr.14901) 3758 : cluster [DBG] pgmap v3765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:51.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:51 smithi006 bash[33918]: audit 2023-08-28T00:03:50.815648+0000 mon.c (mon.2) 1430 : audit [DBG] from='client.? 172.21.15.6:0/382886005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:52.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:52 smithi130 bash[51302]: debug 2023-08-28T00:03:52.114+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:52.115924+0000) 2023-08-28T00:03:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:03:52.206Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:03:52.208Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:53 smithi130 bash[27579]: cluster 2023-08-28T00:03:52.001274+0000 mgr.y (mgr.14901) 3759 : cluster [DBG] pgmap v3766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:53.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:53 smithi130 bash[51302]: debug 2023-08-28T00:03:53.114+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:53.116227+0000) 2023-08-28T00:03:53.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:53 smithi006 bash[33918]: cluster 2023-08-28T00:03:52.001274+0000 mgr.y (mgr.14901) 3759 : cluster [DBG] pgmap v3766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:53 smithi006 bash[27292]: cluster 2023-08-28T00:03:52.001274+0000 mgr.y (mgr.14901) 3759 : cluster [DBG] pgmap v3766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:54 smithi130 bash[27579]: audit 2023-08-28T00:03:53.230391+0000 mon.a (mon.0) 2882 : audit [DBG] from='client.? 172.21.15.6:0/3895224699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:54 smithi130 bash[51302]: debug 2023-08-28T00:03:54.114+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:54.116510+0000) 2023-08-28T00:03:54.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:54 smithi006 bash[33918]: audit 2023-08-28T00:03:53.230391+0000 mon.a (mon.0) 2882 : audit [DBG] from='client.? 172.21.15.6:0/3895224699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:54.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:54 smithi006 bash[27292]: audit 2023-08-28T00:03:53.230391+0000 mon.a (mon.0) 2882 : audit [DBG] from='client.? 172.21.15.6:0/3895224699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:03:54.092Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:03:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:03:54.094Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:03:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:55 smithi130 bash[27579]: cluster 2023-08-28T00:03:54.001854+0000 mgr.y (mgr.14901) 3760 : cluster [DBG] pgmap v3767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:55 smithi130 bash[51302]: debug 2023-08-28T00:03:55.114+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:55.116772+0000) 2023-08-28T00:03:55.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:55 smithi006 bash[33918]: cluster 2023-08-28T00:03:54.001854+0000 mgr.y (mgr.14901) 3760 : cluster [DBG] pgmap v3767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:55.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:55 smithi006 bash[27292]: cluster 2023-08-28T00:03:54.001854+0000 mgr.y (mgr.14901) 3760 : cluster [DBG] pgmap v3767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:56.380 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:56 smithi130 bash[51302]: debug 2023-08-28T00:03:56.114+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:56.117085+0000) 2023-08-28T00:03:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:56 smithi006 bash[33918]: audit 2023-08-28T00:03:55.642427+0000 mon.a (mon.0) 2883 : audit [DBG] from='client.? 172.21.15.6:0/4291284978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:56 smithi006 bash[33918]: cluster 2023-08-28T00:03:56.002985+0000 mgr.y (mgr.14901) 3761 : cluster [DBG] pgmap v3768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:56 smithi006 bash[27292]: audit 2023-08-28T00:03:55.642427+0000 mon.a (mon.0) 2883 : audit [DBG] from='client.? 172.21.15.6:0/4291284978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:56 smithi006 bash[27292]: cluster 2023-08-28T00:03:56.002985+0000 mgr.y (mgr.14901) 3761 : cluster [DBG] pgmap v3768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:56 smithi130 bash[27579]: audit 2023-08-28T00:03:55.642427+0000 mon.a (mon.0) 2883 : audit [DBG] from='client.? 172.21.15.6:0/4291284978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:56 smithi130 bash[27579]: cluster 2023-08-28T00:03:56.002985+0000 mgr.y (mgr.14901) 3761 : cluster [DBG] pgmap v3768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:57.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:57 smithi130 bash[51302]: debug 2023-08-28T00:03:57.114+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:57.117353+0000) 2023-08-28T00:03:58.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:58 smithi130 bash[27579]: audit 2023-08-28T00:03:58.045384+0000 mon.a (mon.0) 2884 : audit [DBG] from='client.? 172.21.15.6:0/1091698799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:58.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:58 smithi130 bash[51302]: debug 2023-08-28T00:03:58.114+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:58.117699+0000) 2023-08-28T00:03:58.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:58 smithi006 bash[33918]: audit 2023-08-28T00:03:58.045384+0000 mon.a (mon.0) 2884 : audit [DBG] from='client.? 172.21.15.6:0/1091698799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:58.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:58 smithi006 bash[27292]: audit 2023-08-28T00:03:58.045384+0000 mon.a (mon.0) 2884 : audit [DBG] from='client.? 172.21.15.6:0/1091698799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:03:59 smithi130 bash[51302]: debug 2023-08-28T00:03:59.114+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:59.117985+0000) 2023-08-28T00:03:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:03:59 smithi130 bash[27579]: cluster 2023-08-28T00:03:58.003611+0000 mgr.y (mgr.14901) 3762 : cluster [DBG] pgmap v3769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:59.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:03:59 smithi006 bash[27292]: cluster 2023-08-28T00:03:58.003611+0000 mgr.y (mgr.14901) 3762 : cluster [DBG] pgmap v3769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:03:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:03:59 smithi006 bash[33918]: cluster 2023-08-28T00:03:58.003611+0000 mgr.y (mgr.14901) 3762 : cluster [DBG] pgmap v3769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:00.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:00 smithi130 bash[51302]: debug 2023-08-28T00:04:00.114+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:00.118248+0000) 2023-08-28T00:04:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:04:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:04:00] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:04:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:04:00.635Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:04:00.637Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:01.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:01 smithi130 bash[27579]: cluster 2023-08-28T00:04:00.004692+0000 mgr.y (mgr.14901) 3763 : cluster [DBG] pgmap v3770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:01.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:01 smithi130 bash[27579]: audit 2023-08-28T00:04:00.458359+0000 mon.a (mon.0) 2885 : audit [DBG] from='client.? 172.21.15.6:0/296517799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:01.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:01 smithi130 bash[51302]: debug 2023-08-28T00:04:01.114+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:01.118513+0000) 2023-08-28T00:04:01.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:01 smithi006 bash[27292]: cluster 2023-08-28T00:04:00.004692+0000 mgr.y (mgr.14901) 3763 : cluster [DBG] pgmap v3770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:01.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:01 smithi006 bash[27292]: audit 2023-08-28T00:04:00.458359+0000 mon.a (mon.0) 2885 : audit [DBG] from='client.? 172.21.15.6:0/296517799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:01.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:01 smithi006 bash[33918]: cluster 2023-08-28T00:04:00.004692+0000 mgr.y (mgr.14901) 3763 : cluster [DBG] pgmap v3770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:01.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:01 smithi006 bash[33918]: audit 2023-08-28T00:04:00.458359+0000 mon.a (mon.0) 2885 : audit [DBG] from='client.? 172.21.15.6:0/296517799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:04:02.206Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:04:02.209Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:02.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:02 smithi130 bash[51302]: debug 2023-08-28T00:04:02.114+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:02.118753+0000) 2023-08-28T00:04:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:03 smithi130 bash[51302]: debug 2023-08-28T00:04:03.118+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:03.119019+0000) 2023-08-28T00:04:03.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:03 smithi130 bash[27579]: cluster 2023-08-28T00:04:02.005359+0000 mgr.y (mgr.14901) 3764 : cluster [DBG] pgmap v3771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:03.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:03 smithi130 bash[27579]: audit 2023-08-28T00:04:02.870096+0000 mon.a (mon.0) 2886 : audit [DBG] from='client.? 172.21.15.6:0/3684131979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:03.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:03 smithi006 bash[33918]: cluster 2023-08-28T00:04:02.005359+0000 mgr.y (mgr.14901) 3764 : cluster [DBG] pgmap v3771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:03.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:03 smithi006 bash[33918]: audit 2023-08-28T00:04:02.870096+0000 mon.a (mon.0) 2886 : audit [DBG] from='client.? 172.21.15.6:0/3684131979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:03.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:03 smithi006 bash[27292]: cluster 2023-08-28T00:04:02.005359+0000 mgr.y (mgr.14901) 3764 : cluster [DBG] pgmap v3771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:03.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:03 smithi006 bash[27292]: audit 2023-08-28T00:04:02.870096+0000 mon.a (mon.0) 2886 : audit [DBG] from='client.? 172.21.15.6:0/3684131979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:04:04.094Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:04:04.096Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:04.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:04 smithi130 bash[51302]: debug 2023-08-28T00:04:04.118+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:04.119220+0000) 2023-08-28T00:04:05.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:05 smithi006 bash[33918]: cluster 2023-08-28T00:04:04.005984+0000 mgr.y (mgr.14901) 3765 : cluster [DBG] pgmap v3772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:05.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:05 smithi006 bash[27292]: cluster 2023-08-28T00:04:04.005984+0000 mgr.y (mgr.14901) 3765 : cluster [DBG] pgmap v3772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:05.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:05 smithi130 bash[27579]: cluster 2023-08-28T00:04:04.005984+0000 mgr.y (mgr.14901) 3765 : cluster [DBG] pgmap v3772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:05.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:05 smithi130 bash[51302]: debug 2023-08-28T00:04:05.118+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:05.119439+0000) 2023-08-28T00:04:06.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:06 smithi006 bash[27292]: audit 2023-08-28T00:04:05.285486+0000 mon.a (mon.0) 2887 : audit [DBG] from='client.? 172.21.15.6:0/934654686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:06.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:06 smithi006 bash[27292]: audit 2023-08-28T00:04:05.605929+0000 mon.b (mon.1) 475 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:04:06.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:06 smithi006 bash[33918]: audit 2023-08-28T00:04:05.285486+0000 mon.a (mon.0) 2887 : audit [DBG] from='client.? 172.21.15.6:0/934654686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:06.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:06 smithi006 bash[33918]: audit 2023-08-28T00:04:05.605929+0000 mon.b (mon.1) 475 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:04:06.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:06 smithi130 bash[51302]: debug 2023-08-28T00:04:06.118+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:06.119758+0000) 2023-08-28T00:04:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:06 smithi130 bash[27579]: audit 2023-08-28T00:04:05.285486+0000 mon.a (mon.0) 2887 : audit [DBG] from='client.? 172.21.15.6:0/934654686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:06 smithi130 bash[27579]: audit 2023-08-28T00:04:05.605929+0000 mon.b (mon.1) 475 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:04:07.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:07 smithi006 bash[27292]: cluster 2023-08-28T00:04:06.007123+0000 mgr.y (mgr.14901) 3766 : cluster [DBG] pgmap v3773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:07.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:07 smithi006 bash[33918]: cluster 2023-08-28T00:04:06.007123+0000 mgr.y (mgr.14901) 3766 : cluster [DBG] pgmap v3773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:07.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:07 smithi130 bash[51302]: debug 2023-08-28T00:04:07.118+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:07.120090+0000) 2023-08-28T00:04:07.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:07 smithi130 bash[27579]: cluster 2023-08-28T00:04:06.007123+0000 mgr.y (mgr.14901) 3766 : cluster [DBG] pgmap v3773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:08.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:08 smithi006 bash[33918]: audit 2023-08-28T00:04:07.705892+0000 mon.c (mon.2) 1431 : audit [DBG] from='client.? 172.21.15.6:0/3810968348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:08.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:08 smithi006 bash[27292]: audit 2023-08-28T00:04:07.705892+0000 mon.c (mon.2) 1431 : audit [DBG] from='client.? 172.21.15.6:0/3810968348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:08 smithi130 bash[27579]: audit 2023-08-28T00:04:07.705892+0000 mon.c (mon.2) 1431 : audit [DBG] from='client.? 172.21.15.6:0/3810968348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:08.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:08 smithi130 bash[51302]: debug 2023-08-28T00:04:08.118+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:08.120401+0000) 2023-08-28T00:04:09.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:09 smithi006 bash[27292]: cluster 2023-08-28T00:04:08.007798+0000 mgr.y (mgr.14901) 3767 : cluster [DBG] pgmap v3774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:09.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:09 smithi006 bash[33918]: cluster 2023-08-28T00:04:08.007798+0000 mgr.y (mgr.14901) 3767 : cluster [DBG] pgmap v3774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:09 smithi130 bash[27579]: cluster 2023-08-28T00:04:08.007798+0000 mgr.y (mgr.14901) 3767 : cluster [DBG] pgmap v3774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:09.757 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:09 smithi130 bash[51302]: debug 2023-08-28T00:04:09.118+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:09.120719+0000) 2023-08-28T00:04:10.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:10 smithi006 bash[27292]: audit 2023-08-28T00:04:10.132459+0000 mon.c (mon.2) 1432 : audit [DBG] from='client.? 172.21.15.6:0/3117359812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:10.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:10 smithi006 bash[33918]: audit 2023-08-28T00:04:10.132459+0000 mon.c (mon.2) 1432 : audit [DBG] from='client.? 172.21.15.6:0/3117359812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:10 smithi130 bash[27579]: audit 2023-08-28T00:04:10.132459+0000 mon.c (mon.2) 1432 : audit [DBG] from='client.? 172.21.15.6:0/3117359812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:10.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:10 smithi130 bash[51302]: debug 2023-08-28T00:04:10.118+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:10.120991+0000) 2023-08-28T00:04:10.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:04:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:04:10] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:04:10.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:04:10.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:10.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:04:10.638Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:11.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:11 smithi006 bash[27292]: cluster 2023-08-28T00:04:10.008943+0000 mgr.y (mgr.14901) 3768 : cluster [DBG] pgmap v3775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:11.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:11 smithi006 bash[33918]: cluster 2023-08-28T00:04:10.008943+0000 mgr.y (mgr.14901) 3768 : cluster [DBG] pgmap v3775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:11.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:11 smithi130 bash[27579]: cluster 2023-08-28T00:04:10.008943+0000 mgr.y (mgr.14901) 3768 : cluster [DBG] pgmap v3775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:11.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:11 smithi130 bash[51302]: debug 2023-08-28T00:04:11.118+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:11.121240+0000) 2023-08-28T00:04:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:04:12.207Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:04:12.209Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:12.471 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:12 smithi130 bash[51302]: debug 2023-08-28T00:04:12.118+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:12.121427+0000) 2023-08-28T00:04:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:12 smithi130 bash[27579]: audit 2023-08-28T00:04:11.469528+0000 mon.a (mon.0) 2888 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:04:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:12 smithi130 bash[27579]: audit 2023-08-28T00:04:11.973089+0000 mon.a (mon.0) 2889 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:04:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:12 smithi130 bash[27579]: cluster 2023-08-28T00:04:12.009625+0000 mgr.y (mgr.14901) 3769 : cluster [DBG] pgmap v3776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:12 smithi130 bash[27579]: audit 2023-08-28T00:04:12.246446+0000 mon.b (mon.1) 476 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:04:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:12 smithi130 bash[27579]: audit 2023-08-28T00:04:12.248597+0000 mon.b (mon.1) 477 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:04:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:12 smithi130 bash[27579]: audit 2023-08-28T00:04:12.256325+0000 mon.a (mon.0) 2890 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:04:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:12 smithi006 bash[33918]: audit 2023-08-28T00:04:11.469528+0000 mon.a (mon.0) 2888 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:04:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:12 smithi006 bash[33918]: audit 2023-08-28T00:04:11.973089+0000 mon.a (mon.0) 2889 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:04:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:12 smithi006 bash[33918]: cluster 2023-08-28T00:04:12.009625+0000 mgr.y (mgr.14901) 3769 : cluster [DBG] pgmap v3776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:12 smithi006 bash[33918]: audit 2023-08-28T00:04:12.246446+0000 mon.b (mon.1) 476 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:04:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:12 smithi006 bash[33918]: audit 2023-08-28T00:04:12.248597+0000 mon.b (mon.1) 477 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:04:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:12 smithi006 bash[33918]: audit 2023-08-28T00:04:12.256325+0000 mon.a (mon.0) 2890 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:04:12.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:12 smithi006 bash[27292]: audit 2023-08-28T00:04:11.469528+0000 mon.a (mon.0) 2888 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:04:12.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:12 smithi006 bash[27292]: audit 2023-08-28T00:04:11.973089+0000 mon.a (mon.0) 2889 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:04:12.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:12 smithi006 bash[27292]: cluster 2023-08-28T00:04:12.009625+0000 mgr.y (mgr.14901) 3769 : cluster [DBG] pgmap v3776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:12.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:12 smithi006 bash[27292]: audit 2023-08-28T00:04:12.246446+0000 mon.b (mon.1) 476 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:04:12.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:12 smithi006 bash[27292]: audit 2023-08-28T00:04:12.248597+0000 mon.b (mon.1) 477 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:04:12.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:12 smithi006 bash[27292]: audit 2023-08-28T00:04:12.256325+0000 mon.a (mon.0) 2890 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:04:13.473 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:13 smithi130 bash[51302]: debug 2023-08-28T00:04:13.118+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:13.121695+0000) 2023-08-28T00:04:13.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:13 smithi130 bash[27579]: audit 2023-08-28T00:04:12.550461+0000 mon.c (mon.2) 1433 : audit [DBG] from='client.? 172.21.15.6:0/4077797695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:13.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:13 smithi006 bash[27292]: audit 2023-08-28T00:04:12.550461+0000 mon.c (mon.2) 1433 : audit [DBG] from='client.? 172.21.15.6:0/4077797695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:13.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:13 smithi006 bash[33918]: audit 2023-08-28T00:04:12.550461+0000 mon.c (mon.2) 1433 : audit [DBG] from='client.? 172.21.15.6:0/4077797695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:04:14.094Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:04:14.097Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:14.480 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:14 smithi130 bash[51302]: debug 2023-08-28T00:04:14.118+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:14.121969+0000) 2023-08-28T00:04:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:14 smithi130 bash[27579]: cluster 2023-08-28T00:04:14.010308+0000 mgr.y (mgr.14901) 3770 : cluster [DBG] pgmap v3777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:14.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:14 smithi006 bash[27292]: cluster 2023-08-28T00:04:14.010308+0000 mgr.y (mgr.14901) 3770 : cluster [DBG] pgmap v3777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:14 smithi006 bash[33918]: cluster 2023-08-28T00:04:14.010308+0000 mgr.y (mgr.14901) 3770 : cluster [DBG] pgmap v3777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:15.484 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:15 smithi130 bash[51302]: debug 2023-08-28T00:04:15.118+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:15.122246+0000) 2023-08-28T00:04:15.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:15 smithi130 bash[27579]: audit 2023-08-28T00:04:14.964718+0000 mon.c (mon.2) 1434 : audit [DBG] from='client.? 172.21.15.6:0/1942453694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:15.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:15 smithi006 bash[33918]: audit 2023-08-28T00:04:14.964718+0000 mon.c (mon.2) 1434 : audit [DBG] from='client.? 172.21.15.6:0/1942453694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:15.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:15 smithi006 bash[27292]: audit 2023-08-28T00:04:14.964718+0000 mon.c (mon.2) 1434 : audit [DBG] from='client.? 172.21.15.6:0/1942453694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:16.493 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:16 smithi130 bash[51302]: debug 2023-08-28T00:04:16.122+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:16.122520+0000) 2023-08-28T00:04:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:16 smithi130 bash[27579]: cluster 2023-08-28T00:04:16.011453+0000 mgr.y (mgr.14901) 3771 : cluster [DBG] pgmap v3778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:16 smithi006 bash[33918]: cluster 2023-08-28T00:04:16.011453+0000 mgr.y (mgr.14901) 3771 : cluster [DBG] pgmap v3778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:16 smithi006 bash[27292]: cluster 2023-08-28T00:04:16.011453+0000 mgr.y (mgr.14901) 3771 : cluster [DBG] pgmap v3778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:17.498 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:17 smithi130 bash[51302]: debug 2023-08-28T00:04:17.118+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:17.122790+0000) 2023-08-28T00:04:17.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:17 smithi130 bash[27579]: audit 2023-08-28T00:04:17.378321+0000 mon.a (mon.0) 2891 : audit [DBG] from='client.? 172.21.15.6:0/452765715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:17.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:17 smithi006 bash[33918]: audit 2023-08-28T00:04:17.378321+0000 mon.a (mon.0) 2891 : audit [DBG] from='client.? 172.21.15.6:0/452765715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:17.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:17 smithi006 bash[27292]: audit 2023-08-28T00:04:17.378321+0000 mon.a (mon.0) 2891 : audit [DBG] from='client.? 172.21.15.6:0/452765715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:18.513 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:18 smithi130 bash[51302]: debug 2023-08-28T00:04:18.122+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:18.123035+0000) 2023-08-28T00:04:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:18 smithi130 bash[27579]: cluster 2023-08-28T00:04:18.012093+0000 mgr.y (mgr.14901) 3772 : cluster [DBG] pgmap v3779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:18 smithi006 bash[33918]: cluster 2023-08-28T00:04:18.012093+0000 mgr.y (mgr.14901) 3772 : cluster [DBG] pgmap v3779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:18 smithi006 bash[27292]: cluster 2023-08-28T00:04:18.012093+0000 mgr.y (mgr.14901) 3772 : cluster [DBG] pgmap v3779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:19.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:19 smithi130 bash[51302]: debug 2023-08-28T00:04:19.122+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:19.123378+0000) 2023-08-28T00:04:20.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:19 smithi130 bash[27579]: audit 2023-08-28T00:04:19.790699+0000 mon.a (mon.0) 2892 : audit [DBG] from='client.? 172.21.15.6:0/949961886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:19 smithi006 bash[27292]: audit 2023-08-28T00:04:19.790699+0000 mon.a (mon.0) 2892 : audit [DBG] from='client.? 172.21.15.6:0/949961886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:20.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:19 smithi006 bash[33918]: audit 2023-08-28T00:04:19.790699+0000 mon.a (mon.0) 2892 : audit [DBG] from='client.? 172.21.15.6:0/949961886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:20.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:20 smithi130 bash[51302]: debug 2023-08-28T00:04:20.122+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:20.123657+0000) 2023-08-28T00:04:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:04:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:04:20] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:04:20.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:20 smithi006 bash[33918]: cluster 2023-08-28T00:04:20.013263+0000 mgr.y (mgr.14901) 3773 : cluster [DBG] pgmap v3780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:20.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:20 smithi006 bash[27292]: cluster 2023-08-28T00:04:20.013263+0000 mgr.y (mgr.14901) 3773 : cluster [DBG] pgmap v3780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:04:20.637Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:20.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:04:20.639Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:20 smithi130 bash[27579]: cluster 2023-08-28T00:04:20.013263+0000 mgr.y (mgr.14901) 3773 : cluster [DBG] pgmap v3780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:21.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:21 smithi130 bash[51302]: debug 2023-08-28T00:04:21.122+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:21.123886+0000) 2023-08-28T00:04:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:04:22.208Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:04:22.210Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:22.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:22 smithi130 bash[51302]: debug 2023-08-28T00:04:22.122+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:22.124131+0000) 2023-08-28T00:04:23.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:23 smithi130 bash[51302]: debug 2023-08-28T00:04:23.122+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:23.124469+0000) 2023-08-28T00:04:23.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:23 smithi130 bash[27579]: cluster 2023-08-28T00:04:22.013891+0000 mgr.y (mgr.14901) 3774 : cluster [DBG] pgmap v3781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:23.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:23 smithi130 bash[27579]: audit 2023-08-28T00:04:22.207081+0000 mon.c (mon.2) 1435 : audit [DBG] from='client.? 172.21.15.6:0/4116825593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:23.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:23 smithi006 bash[33918]: cluster 2023-08-28T00:04:22.013891+0000 mgr.y (mgr.14901) 3774 : cluster [DBG] pgmap v3781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:23.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:23 smithi006 bash[33918]: audit 2023-08-28T00:04:22.207081+0000 mon.c (mon.2) 1435 : audit [DBG] from='client.? 172.21.15.6:0/4116825593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:23.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:23 smithi006 bash[27292]: cluster 2023-08-28T00:04:22.013891+0000 mgr.y (mgr.14901) 3774 : cluster [DBG] pgmap v3781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:23.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:23 smithi006 bash[27292]: audit 2023-08-28T00:04:22.207081+0000 mon.c (mon.2) 1435 : audit [DBG] from='client.? 172.21.15.6:0/4116825593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:04:24.095Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:04:24.097Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:24.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:24 smithi130 bash[51302]: debug 2023-08-28T00:04:24.122+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:24.124746+0000) 2023-08-28T00:04:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:25 smithi130 bash[27579]: cluster 2023-08-28T00:04:24.014521+0000 mgr.y (mgr.14901) 3775 : cluster [DBG] pgmap v3782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:25 smithi130 bash[27579]: audit 2023-08-28T00:04:24.617116+0000 mon.a (mon.0) 2893 : audit [DBG] from='client.? 172.21.15.6:0/3424187748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:25 smithi130 bash[51302]: debug 2023-08-28T00:04:25.122+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:25.125017+0000) 2023-08-28T00:04:25.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:25 smithi006 bash[33918]: cluster 2023-08-28T00:04:24.014521+0000 mgr.y (mgr.14901) 3775 : cluster [DBG] pgmap v3782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:25.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:25 smithi006 bash[33918]: audit 2023-08-28T00:04:24.617116+0000 mon.a (mon.0) 2893 : audit [DBG] from='client.? 172.21.15.6:0/3424187748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:25.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:25 smithi006 bash[27292]: cluster 2023-08-28T00:04:24.014521+0000 mgr.y (mgr.14901) 3775 : cluster [DBG] pgmap v3782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:25.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:25 smithi006 bash[27292]: audit 2023-08-28T00:04:24.617116+0000 mon.a (mon.0) 2893 : audit [DBG] from='client.? 172.21.15.6:0/3424187748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:26.386 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:26 smithi130 bash[51302]: debug 2023-08-28T00:04:26.122+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:26.125294+0000) 2023-08-28T00:04:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:26 smithi006 bash[27292]: cluster 2023-08-28T00:04:26.015638+0000 mgr.y (mgr.14901) 3776 : cluster [DBG] pgmap v3783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:26 smithi006 bash[33918]: cluster 2023-08-28T00:04:26.015638+0000 mgr.y (mgr.14901) 3776 : cluster [DBG] pgmap v3783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:26 smithi130 bash[27579]: cluster 2023-08-28T00:04:26.015638+0000 mgr.y (mgr.14901) 3776 : cluster [DBG] pgmap v3783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:27.389 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:27 smithi130 bash[51302]: debug 2023-08-28T00:04:27.122+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:27.125612+0000) 2023-08-28T00:04:27.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:27 smithi006 bash[27292]: audit 2023-08-28T00:04:27.020035+0000 mon.c (mon.2) 1436 : audit [DBG] from='client.? 172.21.15.6:0/2936046433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:27.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:27 smithi006 bash[33918]: audit 2023-08-28T00:04:27.020035+0000 mon.c (mon.2) 1436 : audit [DBG] from='client.? 172.21.15.6:0/2936046433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:27 smithi130 bash[27579]: audit 2023-08-28T00:04:27.020035+0000 mon.c (mon.2) 1436 : audit [DBG] from='client.? 172.21.15.6:0/2936046433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:28.397 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:28 smithi130 bash[51302]: debug 2023-08-28T00:04:28.122+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:28.125920+0000) 2023-08-28T00:04:28.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:28 smithi006 bash[27292]: cluster 2023-08-28T00:04:28.016300+0000 mgr.y (mgr.14901) 3777 : cluster [DBG] pgmap v3784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:28 smithi006 bash[33918]: cluster 2023-08-28T00:04:28.016300+0000 mgr.y (mgr.14901) 3777 : cluster [DBG] pgmap v3784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:28 smithi130 bash[27579]: cluster 2023-08-28T00:04:28.016300+0000 mgr.y (mgr.14901) 3777 : cluster [DBG] pgmap v3784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:29.486 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:29 smithi130 bash[51302]: debug 2023-08-28T00:04:29.126+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:29.126262+0000) 2023-08-28T00:04:29.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:29 smithi130 bash[27579]: audit 2023-08-28T00:04:29.428656+0000 mon.c (mon.2) 1437 : audit [DBG] from='client.? 172.21.15.6:0/3542117928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:29.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:29 smithi006 bash[27292]: audit 2023-08-28T00:04:29.428656+0000 mon.c (mon.2) 1437 : audit [DBG] from='client.? 172.21.15.6:0/3542117928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:29.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:29 smithi006 bash[33918]: audit 2023-08-28T00:04:29.428656+0000 mon.c (mon.2) 1437 : audit [DBG] from='client.? 172.21.15.6:0/3542117928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:30.496 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:30 smithi130 bash[51302]: debug 2023-08-28T00:04:30.126+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:30.126585+0000) 2023-08-28T00:04:30.754 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:30 smithi006 bash[27292]: cluster 2023-08-28T00:04:30.017457+0000 mgr.y (mgr.14901) 3778 : cluster [DBG] pgmap v3785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:30.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:30 smithi006 bash[33918]: cluster 2023-08-28T00:04:30.017457+0000 mgr.y (mgr.14901) 3778 : cluster [DBG] pgmap v3785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:30.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:04:30.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:30.756 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:04:30.639Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:30 smithi130 bash[27579]: cluster 2023-08-28T00:04:30.017457+0000 mgr.y (mgr.14901) 3778 : cluster [DBG] pgmap v3785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:04:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:04:30] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:04:31.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:31 smithi130 bash[51302]: debug 2023-08-28T00:04:31.125+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:31.126880+0000) 2023-08-28T00:04:32.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:31 smithi006 bash[33918]: audit 2023-08-28T00:04:31.837592+0000 mon.a (mon.0) 2894 : audit [DBG] from='client.? 172.21.15.6:0/2383288846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:32.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:31 smithi006 bash[27292]: audit 2023-08-28T00:04:31.837592+0000 mon.a (mon.0) 2894 : audit [DBG] from='client.? 172.21.15.6:0/2383288846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:32.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:32 smithi130 bash[51302]: debug 2023-08-28T00:04:32.125+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:32.127113+0000) 2023-08-28T00:04:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:31 smithi130 bash[27579]: audit 2023-08-28T00:04:31.837592+0000 mon.a (mon.0) 2894 : audit [DBG] from='client.? 172.21.15.6:0/2383288846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:32.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:04:32.209Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:04:32.211Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:32 smithi006 bash[27292]: cluster 2023-08-28T00:04:32.018168+0000 mgr.y (mgr.14901) 3779 : cluster [DBG] pgmap v3786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:32 smithi006 bash[33918]: cluster 2023-08-28T00:04:32.018168+0000 mgr.y (mgr.14901) 3779 : cluster [DBG] pgmap v3786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:33.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:33 smithi130 bash[51302]: debug 2023-08-28T00:04:33.125+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:33.127333+0000) 2023-08-28T00:04:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:32 smithi130 bash[27579]: cluster 2023-08-28T00:04:32.018168+0000 mgr.y (mgr.14901) 3779 : cluster [DBG] pgmap v3786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:04:34.096Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:04:34.098Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:34.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:34 smithi130 bash[51302]: debug 2023-08-28T00:04:34.125+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:34.127633+0000) 2023-08-28T00:04:35.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:35 smithi130 bash[51302]: debug 2023-08-28T00:04:35.125+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:35.127943+0000) 2023-08-28T00:04:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:35 smithi130 bash[27579]: cluster 2023-08-28T00:04:34.018809+0000 mgr.y (mgr.14901) 3780 : cluster [DBG] pgmap v3787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:35 smithi130 bash[27579]: audit 2023-08-28T00:04:34.259025+0000 mon.a (mon.0) 2895 : audit [DBG] from='client.? 172.21.15.6:0/3355297775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:35.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:35 smithi006 bash[33918]: cluster 2023-08-28T00:04:34.018809+0000 mgr.y (mgr.14901) 3780 : cluster [DBG] pgmap v3787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:35.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:35 smithi006 bash[33918]: audit 2023-08-28T00:04:34.259025+0000 mon.a (mon.0) 2895 : audit [DBG] from='client.? 172.21.15.6:0/3355297775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:35.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:35 smithi006 bash[27292]: cluster 2023-08-28T00:04:34.018809+0000 mgr.y (mgr.14901) 3780 : cluster [DBG] pgmap v3787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:35.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:35 smithi006 bash[27292]: audit 2023-08-28T00:04:34.259025+0000 mon.a (mon.0) 2895 : audit [DBG] from='client.? 172.21.15.6:0/3355297775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:36.384 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:36 smithi130 bash[51302]: debug 2023-08-28T00:04:36.125+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:36.128254+0000) 2023-08-28T00:04:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:36 smithi006 bash[27292]: cluster 2023-08-28T00:04:36.019941+0000 mgr.y (mgr.14901) 3781 : cluster [DBG] pgmap v3788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:36 smithi006 bash[33918]: cluster 2023-08-28T00:04:36.019941+0000 mgr.y (mgr.14901) 3781 : cluster [DBG] pgmap v3788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:36 smithi130 bash[27579]: cluster 2023-08-28T00:04:36.019941+0000 mgr.y (mgr.14901) 3781 : cluster [DBG] pgmap v3788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:37.387 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:37 smithi130 bash[51302]: debug 2023-08-28T00:04:37.125+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:37.128529+0000) 2023-08-28T00:04:37.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:37 smithi006 bash[27292]: audit 2023-08-28T00:04:36.674156+0000 mon.a (mon.0) 2896 : audit [DBG] from='client.? 172.21.15.6:0/4115784325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:37.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:37 smithi006 bash[33918]: audit 2023-08-28T00:04:36.674156+0000 mon.a (mon.0) 2896 : audit [DBG] from='client.? 172.21.15.6:0/4115784325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:37.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:37 smithi130 bash[27579]: audit 2023-08-28T00:04:36.674156+0000 mon.a (mon.0) 2896 : audit [DBG] from='client.? 172.21.15.6:0/4115784325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:38.397 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:38 smithi130 bash[51302]: debug 2023-08-28T00:04:38.125+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:38.128797+0000) 2023-08-28T00:04:38.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:38 smithi006 bash[33918]: cluster 2023-08-28T00:04:38.020615+0000 mgr.y (mgr.14901) 3782 : cluster [DBG] pgmap v3789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:38.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:38 smithi006 bash[27292]: cluster 2023-08-28T00:04:38.020615+0000 mgr.y (mgr.14901) 3782 : cluster [DBG] pgmap v3789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:38 smithi130 bash[27579]: cluster 2023-08-28T00:04:38.020615+0000 mgr.y (mgr.14901) 3782 : cluster [DBG] pgmap v3789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:39.400 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:39 smithi130 bash[51302]: debug 2023-08-28T00:04:39.125+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:39.129081+0000) 2023-08-28T00:04:39.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:39 smithi006 bash[33918]: audit 2023-08-28T00:04:39.087562+0000 mon.a (mon.0) 2897 : audit [DBG] from='client.? 172.21.15.6:0/3382531983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:39.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:39 smithi006 bash[27292]: audit 2023-08-28T00:04:39.087562+0000 mon.a (mon.0) 2897 : audit [DBG] from='client.? 172.21.15.6:0/3382531983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:39.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:39 smithi130 bash[27579]: audit 2023-08-28T00:04:39.087562+0000 mon.a (mon.0) 2897 : audit [DBG] from='client.? 172.21.15.6:0/3382531983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:40.411 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:40 smithi130 bash[51302]: debug 2023-08-28T00:04:40.125+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:40.129381+0000) 2023-08-28T00:04:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:40 smithi006 bash[27292]: cluster 2023-08-28T00:04:40.021901+0000 mgr.y (mgr.14901) 3783 : cluster [DBG] pgmap v3790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:40 smithi006 bash[33918]: cluster 2023-08-28T00:04:40.021901+0000 mgr.y (mgr.14901) 3783 : cluster [DBG] pgmap v3790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:04:40.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:40.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:04:40.641Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:40 smithi130 bash[27579]: cluster 2023-08-28T00:04:40.021901+0000 mgr.y (mgr.14901) 3783 : cluster [DBG] pgmap v3790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:04:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:04:40] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:04:41.557 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:41 smithi130 bash[51302]: debug 2023-08-28T00:04:41.125+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:41.129655+0000) 2023-08-28T00:04:41.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:41 smithi130 bash[27579]: audit 2023-08-28T00:04:41.500569+0000 mon.a (mon.0) 2898 : audit [DBG] from='client.? 172.21.15.6:0/12278724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:41.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:41 smithi006 bash[27292]: audit 2023-08-28T00:04:41.500569+0000 mon.a (mon.0) 2898 : audit [DBG] from='client.? 172.21.15.6:0/12278724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:41.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:41 smithi006 bash[33918]: audit 2023-08-28T00:04:41.500569+0000 mon.a (mon.0) 2898 : audit [DBG] from='client.? 172.21.15.6:0/12278724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:04:42.210Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:04:42.212Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:42.565 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:42 smithi130 bash[51302]: debug 2023-08-28T00:04:42.125+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:42.129942+0000) 2023-08-28T00:04:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:42 smithi130 bash[27579]: cluster 2023-08-28T00:04:42.022725+0000 mgr.y (mgr.14901) 3784 : cluster [DBG] pgmap v3791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:42 smithi006 bash[27292]: cluster 2023-08-28T00:04:42.022725+0000 mgr.y (mgr.14901) 3784 : cluster [DBG] pgmap v3791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:42 smithi006 bash[33918]: cluster 2023-08-28T00:04:42.022725+0000 mgr.y (mgr.14901) 3784 : cluster [DBG] pgmap v3791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:43.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:43 smithi130 bash[51302]: debug 2023-08-28T00:04:43.129+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:43.130219+0000) 2023-08-28T00:04:44.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:43 smithi006 bash[33918]: audit 2023-08-28T00:04:43.910047+0000 mon.c (mon.2) 1438 : audit [DBG] from='client.? 172.21.15.6:0/2283899769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:44.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:43 smithi006 bash[27292]: audit 2023-08-28T00:04:43.910047+0000 mon.c (mon.2) 1438 : audit [DBG] from='client.? 172.21.15.6:0/2283899769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:04:44.097Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:44.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:04:44.099Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:43 smithi130 bash[27579]: audit 2023-08-28T00:04:43.910047+0000 mon.c (mon.2) 1438 : audit [DBG] from='client.? 172.21.15.6:0/2283899769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:44.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:44 smithi130 bash[51302]: debug 2023-08-28T00:04:44.125+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:44.130534+0000) 2023-08-28T00:04:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:44 smithi130 bash[27579]: cluster 2023-08-28T00:04:44.023479+0000 mgr.y (mgr.14901) 3785 : cluster [DBG] pgmap v3792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:45 smithi130 bash[51302]: debug 2023-08-28T00:04:45.125+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:45.130799+0000) 2023-08-28T00:04:45.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:44 smithi006 bash[27292]: cluster 2023-08-28T00:04:44.023479+0000 mgr.y (mgr.14901) 3785 : cluster [DBG] pgmap v3792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:45.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:44 smithi006 bash[33918]: cluster 2023-08-28T00:04:44.023479+0000 mgr.y (mgr.14901) 3785 : cluster [DBG] pgmap v3792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:46.386 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:46 smithi130 bash[51302]: debug 2023-08-28T00:04:46.129+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:46.131024+0000) 2023-08-28T00:04:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:46 smithi006 bash[27292]: cluster 2023-08-28T00:04:46.024589+0000 mgr.y (mgr.14901) 3786 : cluster [DBG] pgmap v3793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:46 smithi006 bash[27292]: audit 2023-08-28T00:04:46.332970+0000 mon.c (mon.2) 1439 : audit [DBG] from='client.? 172.21.15.6:0/2370578351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:46 smithi006 bash[33918]: cluster 2023-08-28T00:04:46.024589+0000 mgr.y (mgr.14901) 3786 : cluster [DBG] pgmap v3793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:46 smithi006 bash[33918]: audit 2023-08-28T00:04:46.332970+0000 mon.c (mon.2) 1439 : audit [DBG] from='client.? 172.21.15.6:0/2370578351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:46 smithi130 bash[27579]: cluster 2023-08-28T00:04:46.024589+0000 mgr.y (mgr.14901) 3786 : cluster [DBG] pgmap v3793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:46 smithi130 bash[27579]: audit 2023-08-28T00:04:46.332970+0000 mon.c (mon.2) 1439 : audit [DBG] from='client.? 172.21.15.6:0/2370578351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:47.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:47 smithi130 bash[51302]: debug 2023-08-28T00:04:47.129+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:47.131335+0000) 2023-08-28T00:04:48.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:48 smithi130 bash[51302]: debug 2023-08-28T00:04:48.129+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:48.131613+0000) 2023-08-28T00:04:49.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:49 smithi130 bash[27579]: cluster 2023-08-28T00:04:48.025246+0000 mgr.y (mgr.14901) 3787 : cluster [DBG] pgmap v3794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:49.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:49 smithi130 bash[27579]: audit 2023-08-28T00:04:48.741824+0000 mon.a (mon.0) 2899 : audit [DBG] from='client.? 172.21.15.6:0/910152344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:49.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:49 smithi130 bash[51302]: debug 2023-08-28T00:04:49.129+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:49.131900+0000) 2023-08-28T00:04:49.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:49 smithi006 bash[33918]: cluster 2023-08-28T00:04:48.025246+0000 mgr.y (mgr.14901) 3787 : cluster [DBG] pgmap v3794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:49.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:49 smithi006 bash[33918]: audit 2023-08-28T00:04:48.741824+0000 mon.a (mon.0) 2899 : audit [DBG] from='client.? 172.21.15.6:0/910152344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:49.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:49 smithi006 bash[27292]: cluster 2023-08-28T00:04:48.025246+0000 mgr.y (mgr.14901) 3787 : cluster [DBG] pgmap v3794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:49.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:49 smithi006 bash[27292]: audit 2023-08-28T00:04:48.741824+0000 mon.a (mon.0) 2899 : audit [DBG] from='client.? 172.21.15.6:0/910152344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:50.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:50 smithi130 bash[51302]: debug 2023-08-28T00:04:50.129+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:50.132174+0000) 2023-08-28T00:04:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:04:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:04:50] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T00:04:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:04:50.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:04:50.642Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:51.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:51 smithi130 bash[27579]: cluster 2023-08-28T00:04:50.026373+0000 mgr.y (mgr.14901) 3788 : cluster [DBG] pgmap v3795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:51.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:51 smithi130 bash[51302]: debug 2023-08-28T00:04:51.129+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:51.132494+0000) 2023-08-28T00:04:51.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:51 smithi006 bash[33918]: cluster 2023-08-28T00:04:50.026373+0000 mgr.y (mgr.14901) 3788 : cluster [DBG] pgmap v3795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:51.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:51 smithi006 bash[27292]: cluster 2023-08-28T00:04:50.026373+0000 mgr.y (mgr.14901) 3788 : cluster [DBG] pgmap v3795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:52 smithi130 bash[27579]: audit 2023-08-28T00:04:51.151115+0000 mon.a (mon.0) 2900 : audit [DBG] from='client.? 172.21.15.6:0/2915808257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:52.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:52 smithi130 bash[51302]: debug 2023-08-28T00:04:52.129+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:52.132771+0000) 2023-08-28T00:04:52.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:52 smithi006 bash[27292]: audit 2023-08-28T00:04:51.151115+0000 mon.a (mon.0) 2900 : audit [DBG] from='client.? 172.21.15.6:0/2915808257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:52.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:52 smithi006 bash[33918]: audit 2023-08-28T00:04:51.151115+0000 mon.a (mon.0) 2900 : audit [DBG] from='client.? 172.21.15.6:0/2915808257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:52.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:04:52.211Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:52.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:04:52.213Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:53 smithi130 bash[27579]: cluster 2023-08-28T00:04:52.027017+0000 mgr.y (mgr.14901) 3789 : cluster [DBG] pgmap v3796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:53.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:53 smithi130 bash[51302]: debug 2023-08-28T00:04:53.129+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:53.133027+0000) 2023-08-28T00:04:53.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:53 smithi006 bash[33918]: cluster 2023-08-28T00:04:52.027017+0000 mgr.y (mgr.14901) 3789 : cluster [DBG] pgmap v3796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:53 smithi006 bash[27292]: cluster 2023-08-28T00:04:52.027017+0000 mgr.y (mgr.14901) 3789 : cluster [DBG] pgmap v3796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:54 smithi130 bash[27579]: audit 2023-08-28T00:04:53.555186+0000 mon.a (mon.0) 2901 : audit [DBG] from='client.? 172.21.15.6:0/4130603971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:54 smithi130 bash[51302]: debug 2023-08-28T00:04:54.129+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:54.133324+0000) 2023-08-28T00:04:54.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:54 smithi006 bash[33918]: audit 2023-08-28T00:04:53.555186+0000 mon.a (mon.0) 2901 : audit [DBG] from='client.? 172.21.15.6:0/4130603971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:04:54.097Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:04:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:04:54.099Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:04:54.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:54 smithi006 bash[27292]: audit 2023-08-28T00:04:53.555186+0000 mon.a (mon.0) 2901 : audit [DBG] from='client.? 172.21.15.6:0/4130603971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:55 smithi130 bash[27579]: cluster 2023-08-28T00:04:54.027685+0000 mgr.y (mgr.14901) 3790 : cluster [DBG] pgmap v3797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:55 smithi130 bash[51302]: debug 2023-08-28T00:04:55.129+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:55.133633+0000) 2023-08-28T00:04:55.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:55 smithi006 bash[27292]: cluster 2023-08-28T00:04:54.027685+0000 mgr.y (mgr.14901) 3790 : cluster [DBG] pgmap v3797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:55.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:55 smithi006 bash[33918]: cluster 2023-08-28T00:04:54.027685+0000 mgr.y (mgr.14901) 3790 : cluster [DBG] pgmap v3797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:56.393 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:56 smithi130 bash[51302]: debug 2023-08-28T00:04:56.129+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:56.133891+0000) 2023-08-28T00:04:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:56 smithi006 bash[27292]: audit 2023-08-28T00:04:55.966003+0000 mon.a (mon.0) 2902 : audit [DBG] from='client.? 172.21.15.6:0/3694761671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:56 smithi006 bash[27292]: cluster 2023-08-28T00:04:56.028744+0000 mgr.y (mgr.14901) 3791 : cluster [DBG] pgmap v3798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:56 smithi006 bash[33918]: audit 2023-08-28T00:04:55.966003+0000 mon.a (mon.0) 2902 : audit [DBG] from='client.? 172.21.15.6:0/3694761671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:56 smithi006 bash[33918]: cluster 2023-08-28T00:04:56.028744+0000 mgr.y (mgr.14901) 3791 : cluster [DBG] pgmap v3798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:56 smithi130 bash[27579]: audit 2023-08-28T00:04:55.966003+0000 mon.a (mon.0) 2902 : audit [DBG] from='client.? 172.21.15.6:0/3694761671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:56 smithi130 bash[27579]: cluster 2023-08-28T00:04:56.028744+0000 mgr.y (mgr.14901) 3791 : cluster [DBG] pgmap v3798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:57.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:57 smithi130 bash[51302]: debug 2023-08-28T00:04:57.133+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:57.134156+0000) 2023-08-28T00:04:58.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:58 smithi130 bash[51302]: debug 2023-08-28T00:04:58.129+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:58.134418+0000) 2023-08-28T00:04:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:04:59 smithi130 bash[51302]: debug 2023-08-28T00:04:59.129+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:59.134704+0000) 2023-08-28T00:04:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:59 smithi130 bash[27579]: cluster 2023-08-28T00:04:58.029302+0000 mgr.y (mgr.14901) 3792 : cluster [DBG] pgmap v3799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:04:59 smithi130 bash[27579]: audit 2023-08-28T00:04:58.380339+0000 mon.a (mon.0) 2903 : audit [DBG] from='client.? 172.21.15.6:0/3242857234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:59 smithi006 bash[33918]: cluster 2023-08-28T00:04:58.029302+0000 mgr.y (mgr.14901) 3792 : cluster [DBG] pgmap v3799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:04:59 smithi006 bash[33918]: audit 2023-08-28T00:04:58.380339+0000 mon.a (mon.0) 2903 : audit [DBG] from='client.? 172.21.15.6:0/3242857234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:59.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:59 smithi006 bash[27292]: cluster 2023-08-28T00:04:58.029302+0000 mgr.y (mgr.14901) 3792 : cluster [DBG] pgmap v3799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:04:59.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:04:59 smithi006 bash[27292]: audit 2023-08-28T00:04:58.380339+0000 mon.a (mon.0) 2903 : audit [DBG] from='client.? 172.21.15.6:0/3242857234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:00.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:00 smithi130 bash[51302]: debug 2023-08-28T00:05:00.133+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:00.135025+0000) 2023-08-28T00:05:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:05:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:05:00] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T00:05:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:05:00.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:05:00.643Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:01.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:01 smithi130 bash[51302]: debug 2023-08-28T00:05:01.133+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:01.135313+0000) 2023-08-28T00:05:01.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:01 smithi130 bash[27579]: cluster 2023-08-28T00:05:00.030373+0000 mgr.y (mgr.14901) 3793 : cluster [DBG] pgmap v3800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:01.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:01 smithi130 bash[27579]: audit 2023-08-28T00:05:00.792761+0000 mon.a (mon.0) 2904 : audit [DBG] from='client.? 172.21.15.6:0/3449158669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:01.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:01 smithi006 bash[33918]: cluster 2023-08-28T00:05:00.030373+0000 mgr.y (mgr.14901) 3793 : cluster [DBG] pgmap v3800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:01.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:01 smithi006 bash[33918]: audit 2023-08-28T00:05:00.792761+0000 mon.a (mon.0) 2904 : audit [DBG] from='client.? 172.21.15.6:0/3449158669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:01.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:01 smithi006 bash[27292]: cluster 2023-08-28T00:05:00.030373+0000 mgr.y (mgr.14901) 3793 : cluster [DBG] pgmap v3800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:01.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:01 smithi006 bash[27292]: audit 2023-08-28T00:05:00.792761+0000 mon.a (mon.0) 2904 : audit [DBG] from='client.? 172.21.15.6:0/3449158669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:02.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:02 smithi130 bash[51302]: debug 2023-08-28T00:05:02.133+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:02.135542+0000) 2023-08-28T00:05:02.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:05:02.212Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:05:02.213Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:03.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:03 smithi130 bash[27579]: cluster 2023-08-28T00:05:02.031042+0000 mgr.y (mgr.14901) 3794 : cluster [DBG] pgmap v3801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:03 smithi130 bash[51302]: debug 2023-08-28T00:05:03.133+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:03.135832+0000) 2023-08-28T00:05:03.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:03 smithi006 bash[33918]: cluster 2023-08-28T00:05:02.031042+0000 mgr.y (mgr.14901) 3794 : cluster [DBG] pgmap v3801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:03.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:03 smithi006 bash[27292]: cluster 2023-08-28T00:05:02.031042+0000 mgr.y (mgr.14901) 3794 : cluster [DBG] pgmap v3801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:04.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:04 smithi130 bash[27579]: audit 2023-08-28T00:05:03.203258+0000 mon.a (mon.0) 2905 : audit [DBG] from='client.? 172.21.15.6:0/3574562306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:04 smithi130 bash[51302]: debug 2023-08-28T00:05:04.133+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:04.136104+0000) 2023-08-28T00:05:04.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:04 smithi006 bash[27292]: audit 2023-08-28T00:05:03.203258+0000 mon.a (mon.0) 2905 : audit [DBG] from='client.? 172.21.15.6:0/3574562306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:05:04.098Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:05:04.100Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:04.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:04 smithi006 bash[33918]: audit 2023-08-28T00:05:03.203258+0000 mon.a (mon.0) 2905 : audit [DBG] from='client.? 172.21.15.6:0/3574562306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:05 smithi130 bash[51302]: debug 2023-08-28T00:05:05.133+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:05.136380+0000) 2023-08-28T00:05:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:05 smithi130 bash[27579]: cluster 2023-08-28T00:05:04.031672+0000 mgr.y (mgr.14901) 3795 : cluster [DBG] pgmap v3802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:05.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:05 smithi006 bash[33918]: cluster 2023-08-28T00:05:04.031672+0000 mgr.y (mgr.14901) 3795 : cluster [DBG] pgmap v3802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:05.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:05 smithi006 bash[27292]: cluster 2023-08-28T00:05:04.031672+0000 mgr.y (mgr.14901) 3795 : cluster [DBG] pgmap v3802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:06.392 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:06 smithi130 bash[51302]: debug 2023-08-28T00:05:06.133+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:06.136727+0000) 2023-08-28T00:05:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:06 smithi006 bash[27292]: audit 2023-08-28T00:05:05.613728+0000 mon.a (mon.0) 2906 : audit [DBG] from='client.? 172.21.15.6:0/3071520741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:06 smithi006 bash[27292]: cluster 2023-08-28T00:05:06.032816+0000 mgr.y (mgr.14901) 3796 : cluster [DBG] pgmap v3803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:06 smithi006 bash[33918]: audit 2023-08-28T00:05:05.613728+0000 mon.a (mon.0) 2906 : audit [DBG] from='client.? 172.21.15.6:0/3071520741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:06 smithi006 bash[33918]: cluster 2023-08-28T00:05:06.032816+0000 mgr.y (mgr.14901) 3796 : cluster [DBG] pgmap v3803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:06 smithi130 bash[27579]: audit 2023-08-28T00:05:05.613728+0000 mon.a (mon.0) 2906 : audit [DBG] from='client.? 172.21.15.6:0/3071520741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:06 smithi130 bash[27579]: cluster 2023-08-28T00:05:06.032816+0000 mgr.y (mgr.14901) 3796 : cluster [DBG] pgmap v3803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:07.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:07 smithi130 bash[51302]: debug 2023-08-28T00:05:07.133+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:07.136988+0000) 2023-08-28T00:05:08.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:08 smithi130 bash[27579]: audit 2023-08-28T00:05:08.025086+0000 mon.c (mon.2) 1440 : audit [DBG] from='client.? 172.21.15.6:0/1334134821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:08.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:08 smithi130 bash[51302]: debug 2023-08-28T00:05:08.133+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:08.137233+0000) 2023-08-28T00:05:08.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:08 smithi006 bash[27292]: audit 2023-08-28T00:05:08.025086+0000 mon.c (mon.2) 1440 : audit [DBG] from='client.? 172.21.15.6:0/1334134821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:08.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:08 smithi006 bash[33918]: audit 2023-08-28T00:05:08.025086+0000 mon.c (mon.2) 1440 : audit [DBG] from='client.? 172.21.15.6:0/1334134821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:09.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:09 smithi130 bash[51302]: debug 2023-08-28T00:05:09.137+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:09.137566+0000) 2023-08-28T00:05:09.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:09 smithi130 bash[27579]: cluster 2023-08-28T00:05:08.033298+0000 mgr.y (mgr.14901) 3797 : cluster [DBG] pgmap v3804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:09.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:09 smithi006 bash[27292]: cluster 2023-08-28T00:05:08.033298+0000 mgr.y (mgr.14901) 3797 : cluster [DBG] pgmap v3804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:09.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:09 smithi006 bash[33918]: cluster 2023-08-28T00:05:08.033298+0000 mgr.y (mgr.14901) 3797 : cluster [DBG] pgmap v3804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:10.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:10 smithi130 bash[51302]: debug 2023-08-28T00:05:10.133+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:10.137830+0000) 2023-08-28T00:05:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:05:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:05:10] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T00:05:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:05:10.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:05:10.644Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:11.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:11 smithi130 bash[27579]: cluster 2023-08-28T00:05:10.034304+0000 mgr.y (mgr.14901) 3798 : cluster [DBG] pgmap v3805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:11.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:11 smithi130 bash[27579]: audit 2023-08-28T00:05:10.452598+0000 mon.c (mon.2) 1441 : audit [DBG] from='client.? 172.21.15.6:0/1598770013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:11.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:11 smithi130 bash[51302]: debug 2023-08-28T00:05:11.133+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:11.138112+0000) 2023-08-28T00:05:11.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:11 smithi006 bash[33918]: cluster 2023-08-28T00:05:10.034304+0000 mgr.y (mgr.14901) 3798 : cluster [DBG] pgmap v3805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:11.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:11 smithi006 bash[33918]: audit 2023-08-28T00:05:10.452598+0000 mon.c (mon.2) 1441 : audit [DBG] from='client.? 172.21.15.6:0/1598770013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:11.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:11 smithi006 bash[27292]: cluster 2023-08-28T00:05:10.034304+0000 mgr.y (mgr.14901) 3798 : cluster [DBG] pgmap v3805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:11.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:11 smithi006 bash[27292]: audit 2023-08-28T00:05:10.452598+0000 mon.c (mon.2) 1441 : audit [DBG] from='client.? 172.21.15.6:0/1598770013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:12.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:05:12.212Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:05:12.214Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:12.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:12 smithi130 bash[51302]: debug 2023-08-28T00:05:12.133+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:12.138365+0000) 2023-08-28T00:05:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:13 smithi130 bash[27579]: cluster 2023-08-28T00:05:12.034938+0000 mgr.y (mgr.14901) 3799 : cluster [DBG] pgmap v3806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:13 smithi130 bash[27579]: audit 2023-08-28T00:05:12.265604+0000 mon.b (mon.1) 478 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:05:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:13 smithi130 bash[27579]: audit 2023-08-28T00:05:12.870832+0000 mon.a (mon.0) 2907 : audit [DBG] from='client.? 172.21.15.6:0/332367393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:13 smithi130 bash[51302]: debug 2023-08-28T00:05:13.133+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:13.138612+0000) 2023-08-28T00:05:13.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:13 smithi006 bash[33918]: cluster 2023-08-28T00:05:12.034938+0000 mgr.y (mgr.14901) 3799 : cluster [DBG] pgmap v3806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:13.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:13 smithi006 bash[33918]: audit 2023-08-28T00:05:12.265604+0000 mon.b (mon.1) 478 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:05:13.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:13 smithi006 bash[33918]: audit 2023-08-28T00:05:12.870832+0000 mon.a (mon.0) 2907 : audit [DBG] from='client.? 172.21.15.6:0/332367393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:13.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:13 smithi006 bash[27292]: cluster 2023-08-28T00:05:12.034938+0000 mgr.y (mgr.14901) 3799 : cluster [DBG] pgmap v3806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:13.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:13 smithi006 bash[27292]: audit 2023-08-28T00:05:12.265604+0000 mon.b (mon.1) 478 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:05:13.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:13 smithi006 bash[27292]: audit 2023-08-28T00:05:12.870832+0000 mon.a (mon.0) 2907 : audit [DBG] from='client.? 172.21.15.6:0/332367393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:05:14.099Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:05:14.101Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:14.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:14 smithi130 bash[51302]: debug 2023-08-28T00:05:14.137+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:14.138913+0000) 2023-08-28T00:05:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:15 smithi130 bash[27579]: cluster 2023-08-28T00:05:14.035597+0000 mgr.y (mgr.14901) 3800 : cluster [DBG] pgmap v3807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:15 smithi130 bash[51302]: debug 2023-08-28T00:05:15.137+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:15.139162+0000) 2023-08-28T00:05:15.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:15 smithi006 bash[33918]: cluster 2023-08-28T00:05:14.035597+0000 mgr.y (mgr.14901) 3800 : cluster [DBG] pgmap v3807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:15.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:15 smithi006 bash[27292]: cluster 2023-08-28T00:05:14.035597+0000 mgr.y (mgr.14901) 3800 : cluster [DBG] pgmap v3807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:16 smithi130 bash[27579]: audit 2023-08-28T00:05:15.302577+0000 mon.a (mon.0) 2908 : audit [DBG] from='client.? 172.21.15.6:0/2757353305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:16.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:16 smithi130 bash[51302]: debug 2023-08-28T00:05:16.137+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:16.139467+0000) 2023-08-28T00:05:16.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:16 smithi006 bash[27292]: audit 2023-08-28T00:05:15.302577+0000 mon.a (mon.0) 2908 : audit [DBG] from='client.? 172.21.15.6:0/2757353305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:16.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:16 smithi006 bash[33918]: audit 2023-08-28T00:05:15.302577+0000 mon.a (mon.0) 2908 : audit [DBG] from='client.? 172.21.15.6:0/2757353305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:17.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:17 smithi130 bash[51302]: debug 2023-08-28T00:05:17.137+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:17.139705+0000) 2023-08-28T00:05:17.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:17 smithi130 bash[27579]: cluster 2023-08-28T00:05:16.036712+0000 mgr.y (mgr.14901) 3801 : cluster [DBG] pgmap v3808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:17.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:17 smithi006 bash[33918]: cluster 2023-08-28T00:05:16.036712+0000 mgr.y (mgr.14901) 3801 : cluster [DBG] pgmap v3808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:17.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:17 smithi006 bash[27292]: cluster 2023-08-28T00:05:16.036712+0000 mgr.y (mgr.14901) 3801 : cluster [DBG] pgmap v3808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:18 smithi130 bash[27579]: audit 2023-08-28T00:05:17.743241+0000 mon.a (mon.0) 2909 : audit [DBG] from='client.? 172.21.15.6:0/3793828677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:18.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:18 smithi130 bash[51302]: debug 2023-08-28T00:05:18.137+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:18.139890+0000) 2023-08-28T00:05:18.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:18 smithi006 bash[27292]: audit 2023-08-28T00:05:17.743241+0000 mon.a (mon.0) 2909 : audit [DBG] from='client.? 172.21.15.6:0/3793828677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:18.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:18 smithi006 bash[33918]: audit 2023-08-28T00:05:17.743241+0000 mon.a (mon.0) 2909 : audit [DBG] from='client.? 172.21.15.6:0/3793828677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:19.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:19 smithi006 bash[33918]: cluster 2023-08-28T00:05:18.037306+0000 mgr.y (mgr.14901) 3802 : cluster [DBG] pgmap v3809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:19.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:19 smithi006 bash[33918]: audit 2023-08-28T00:05:18.145525+0000 mon.a (mon.0) 2910 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:05:19.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:19 smithi006 bash[33918]: audit 2023-08-28T00:05:18.599694+0000 mon.a (mon.0) 2911 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:05:19.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:19 smithi006 bash[33918]: audit 2023-08-28T00:05:18.899244+0000 mon.b (mon.1) 479 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:05:19.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:19 smithi006 bash[33918]: audit 2023-08-28T00:05:18.901737+0000 mon.b (mon.1) 480 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:05:19.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:19 smithi006 bash[33918]: audit 2023-08-28T00:05:18.909904+0000 mon.a (mon.0) 2912 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:05:19.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:19 smithi006 bash[27292]: cluster 2023-08-28T00:05:18.037306+0000 mgr.y (mgr.14901) 3802 : cluster [DBG] pgmap v3809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:19.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:19 smithi006 bash[27292]: audit 2023-08-28T00:05:18.145525+0000 mon.a (mon.0) 2910 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:05:19.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:19 smithi006 bash[27292]: audit 2023-08-28T00:05:18.599694+0000 mon.a (mon.0) 2911 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:05:19.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:19 smithi006 bash[27292]: audit 2023-08-28T00:05:18.899244+0000 mon.b (mon.1) 479 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:05:19.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:19 smithi006 bash[27292]: audit 2023-08-28T00:05:18.901737+0000 mon.b (mon.1) 480 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:05:19.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:19 smithi006 bash[27292]: audit 2023-08-28T00:05:18.909904+0000 mon.a (mon.0) 2912 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:05:19.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:19 smithi130 bash[51302]: debug 2023-08-28T00:05:19.137+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:19.140186+0000) 2023-08-28T00:05:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:19 smithi130 bash[27579]: cluster 2023-08-28T00:05:18.037306+0000 mgr.y (mgr.14901) 3802 : cluster [DBG] pgmap v3809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:19.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:19 smithi130 bash[27579]: audit 2023-08-28T00:05:18.145525+0000 mon.a (mon.0) 2910 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:05:19.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:19 smithi130 bash[27579]: audit 2023-08-28T00:05:18.599694+0000 mon.a (mon.0) 2911 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:05:19.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:19 smithi130 bash[27579]: audit 2023-08-28T00:05:18.899244+0000 mon.b (mon.1) 479 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:05:19.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:19 smithi130 bash[27579]: audit 2023-08-28T00:05:18.901737+0000 mon.b (mon.1) 480 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:05:19.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:19 smithi130 bash[27579]: audit 2023-08-28T00:05:18.909904+0000 mon.a (mon.0) 2912 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:05:20.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:20 smithi130 bash[51302]: debug 2023-08-28T00:05:20.137+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:20.140579+0000) 2023-08-28T00:05:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:05:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:05:20] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:05:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:05:20.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:05:20.644Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:21.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:21 smithi006 bash[33918]: cluster 2023-08-28T00:05:20.038366+0000 mgr.y (mgr.14901) 3803 : cluster [DBG] pgmap v3810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:21.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:21 smithi006 bash[33918]: audit 2023-08-28T00:05:20.168590+0000 mon.a (mon.0) 2913 : audit [DBG] from='client.? 172.21.15.6:0/2267750736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:21.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:21 smithi006 bash[27292]: cluster 2023-08-28T00:05:20.038366+0000 mgr.y (mgr.14901) 3803 : cluster [DBG] pgmap v3810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:21.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:21 smithi006 bash[27292]: audit 2023-08-28T00:05:20.168590+0000 mon.a (mon.0) 2913 : audit [DBG] from='client.? 172.21.15.6:0/2267750736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:21 smithi130 bash[27579]: cluster 2023-08-28T00:05:20.038366+0000 mgr.y (mgr.14901) 3803 : cluster [DBG] pgmap v3810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:21 smithi130 bash[27579]: audit 2023-08-28T00:05:20.168590+0000 mon.a (mon.0) 2913 : audit [DBG] from='client.? 172.21.15.6:0/2267750736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:21.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:21 smithi130 bash[51302]: debug 2023-08-28T00:05:21.137+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:21.140865+0000) 2023-08-28T00:05:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:05:22.213Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:05:22.215Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:22.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:22 smithi130 bash[51302]: debug 2023-08-28T00:05:22.137+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:22.141136+0000) 2023-08-28T00:05:23.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:23 smithi006 bash[33918]: cluster 2023-08-28T00:05:22.039015+0000 mgr.y (mgr.14901) 3804 : cluster [DBG] pgmap v3811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:23.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:23 smithi006 bash[33918]: audit 2023-08-28T00:05:22.589267+0000 mon.c (mon.2) 1442 : audit [DBG] from='client.? 172.21.15.6:0/729104450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:23.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:23 smithi006 bash[27292]: cluster 2023-08-28T00:05:22.039015+0000 mgr.y (mgr.14901) 3804 : cluster [DBG] pgmap v3811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:23.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:23 smithi006 bash[27292]: audit 2023-08-28T00:05:22.589267+0000 mon.c (mon.2) 1442 : audit [DBG] from='client.? 172.21.15.6:0/729104450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:23.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:23 smithi130 bash[27579]: cluster 2023-08-28T00:05:22.039015+0000 mgr.y (mgr.14901) 3804 : cluster [DBG] pgmap v3811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:23.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:23 smithi130 bash[27579]: audit 2023-08-28T00:05:22.589267+0000 mon.c (mon.2) 1442 : audit [DBG] from='client.? 172.21.15.6:0/729104450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:23.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:23 smithi130 bash[51302]: debug 2023-08-28T00:05:23.141+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:23.141443+0000) 2023-08-28T00:05:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:05:24.100Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:05:24.102Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:24.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:24 smithi130 bash[51302]: debug 2023-08-28T00:05:24.137+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:24.141755+0000) 2023-08-28T00:05:25.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:25 smithi006 bash[33918]: cluster 2023-08-28T00:05:24.039769+0000 mgr.y (mgr.14901) 3805 : cluster [DBG] pgmap v3812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:25.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:25 smithi006 bash[33918]: audit 2023-08-28T00:05:25.003935+0000 mon.c (mon.2) 1443 : audit [DBG] from='client.? 172.21.15.6:0/3210837643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:25.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:25 smithi006 bash[27292]: cluster 2023-08-28T00:05:24.039769+0000 mgr.y (mgr.14901) 3805 : cluster [DBG] pgmap v3812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:25.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:25 smithi006 bash[27292]: audit 2023-08-28T00:05:25.003935+0000 mon.c (mon.2) 1443 : audit [DBG] from='client.? 172.21.15.6:0/3210837643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:25.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:25 smithi130 bash[27579]: cluster 2023-08-28T00:05:24.039769+0000 mgr.y (mgr.14901) 3805 : cluster [DBG] pgmap v3812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:25.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:25 smithi130 bash[27579]: audit 2023-08-28T00:05:25.003935+0000 mon.c (mon.2) 1443 : audit [DBG] from='client.? 172.21.15.6:0/3210837643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:25.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:25 smithi130 bash[51302]: debug 2023-08-28T00:05:25.141+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:25.142027+0000) 2023-08-28T00:05:26.400 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:26 smithi130 bash[51302]: debug 2023-08-28T00:05:26.137+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:26.142331+0000) 2023-08-28T00:05:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:26 smithi006 bash[27292]: cluster 2023-08-28T00:05:26.040912+0000 mgr.y (mgr.14901) 3806 : cluster [DBG] pgmap v3813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:26 smithi006 bash[33918]: cluster 2023-08-28T00:05:26.040912+0000 mgr.y (mgr.14901) 3806 : cluster [DBG] pgmap v3813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:26 smithi130 bash[27579]: cluster 2023-08-28T00:05:26.040912+0000 mgr.y (mgr.14901) 3806 : cluster [DBG] pgmap v3813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:27.477 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:27 smithi130 bash[51302]: debug 2023-08-28T00:05:27.136+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:27.142542+0000) 2023-08-28T00:05:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:27 smithi130 bash[27579]: audit 2023-08-28T00:05:27.420864+0000 mon.a (mon.0) 2914 : audit [DBG] from='client.? 172.21.15.6:0/1859168460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:27.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:27 smithi006 bash[27292]: audit 2023-08-28T00:05:27.420864+0000 mon.a (mon.0) 2914 : audit [DBG] from='client.? 172.21.15.6:0/1859168460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:27.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:27 smithi006 bash[33918]: audit 2023-08-28T00:05:27.420864+0000 mon.a (mon.0) 2914 : audit [DBG] from='client.? 172.21.15.6:0/1859168460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:28.485 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:28 smithi130 bash[51302]: debug 2023-08-28T00:05:28.140+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:28.142814+0000) 2023-08-28T00:05:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:28 smithi130 bash[27579]: cluster 2023-08-28T00:05:28.041578+0000 mgr.y (mgr.14901) 3807 : cluster [DBG] pgmap v3814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:28 smithi006 bash[27292]: cluster 2023-08-28T00:05:28.041578+0000 mgr.y (mgr.14901) 3807 : cluster [DBG] pgmap v3814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:28 smithi006 bash[33918]: cluster 2023-08-28T00:05:28.041578+0000 mgr.y (mgr.14901) 3807 : cluster [DBG] pgmap v3814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:29.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:29 smithi130 bash[51302]: debug 2023-08-28T00:05:29.140+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:29.143223+0000) 2023-08-28T00:05:30.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:29 smithi006 bash[33918]: audit 2023-08-28T00:05:29.838638+0000 mon.a (mon.0) 2915 : audit [DBG] from='client.? 172.21.15.6:0/927378568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:30.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:29 smithi006 bash[27292]: audit 2023-08-28T00:05:29.838638+0000 mon.a (mon.0) 2915 : audit [DBG] from='client.? 172.21.15.6:0/927378568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:29 smithi130 bash[27579]: audit 2023-08-28T00:05:29.838638+0000 mon.a (mon.0) 2915 : audit [DBG] from='client.? 172.21.15.6:0/927378568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:30 smithi130 bash[51302]: debug 2023-08-28T00:05:30.140+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:30.143438+0000) 2023-08-28T00:05:30.905 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:05:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:05:30] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:05:30.906 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:05:30.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:30.906 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:05:30.646Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:31.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:30 smithi006 bash[33918]: cluster 2023-08-28T00:05:30.042850+0000 mgr.y (mgr.14901) 3808 : cluster [DBG] pgmap v3815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:31.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:30 smithi006 bash[27292]: cluster 2023-08-28T00:05:30.042850+0000 mgr.y (mgr.14901) 3808 : cluster [DBG] pgmap v3815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:31.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:30 smithi130 bash[27579]: cluster 2023-08-28T00:05:30.042850+0000 mgr.y (mgr.14901) 3808 : cluster [DBG] pgmap v3815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:31.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:31 smithi130 bash[51302]: debug 2023-08-28T00:05:31.140+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:31.143733+0000) 2023-08-28T00:05:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:05:32.214Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:05:32.216Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:32.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:32 smithi130 bash[51302]: debug 2023-08-28T00:05:32.140+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:32.144041+0000) 2023-08-28T00:05:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:33 smithi130 bash[27579]: cluster 2023-08-28T00:05:32.043372+0000 mgr.y (mgr.14901) 3809 : cluster [DBG] pgmap v3816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:33 smithi130 bash[27579]: audit 2023-08-28T00:05:32.257504+0000 mon.a (mon.0) 2916 : audit [DBG] from='client.? 172.21.15.6:0/479909232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:33.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:33 smithi130 bash[51302]: debug 2023-08-28T00:05:33.140+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:33.144348+0000) 2023-08-28T00:05:33.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:33 smithi006 bash[27292]: cluster 2023-08-28T00:05:32.043372+0000 mgr.y (mgr.14901) 3809 : cluster [DBG] pgmap v3816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:33.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:33 smithi006 bash[27292]: audit 2023-08-28T00:05:32.257504+0000 mon.a (mon.0) 2916 : audit [DBG] from='client.? 172.21.15.6:0/479909232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:33.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:33 smithi006 bash[33918]: cluster 2023-08-28T00:05:32.043372+0000 mgr.y (mgr.14901) 3809 : cluster [DBG] pgmap v3816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:33.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:33 smithi006 bash[33918]: audit 2023-08-28T00:05:32.257504+0000 mon.a (mon.0) 2916 : audit [DBG] from='client.? 172.21.15.6:0/479909232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:05:34.101Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:05:34.103Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:34.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:34 smithi130 bash[51302]: debug 2023-08-28T00:05:34.140+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:34.144692+0000) 2023-08-28T00:05:35.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:35 smithi130 bash[51302]: debug 2023-08-28T00:05:35.144+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:35.145004+0000) 2023-08-28T00:05:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:35 smithi130 bash[27579]: cluster 2023-08-28T00:05:34.044116+0000 mgr.y (mgr.14901) 3810 : cluster [DBG] pgmap v3817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:35 smithi130 bash[27579]: audit 2023-08-28T00:05:34.678636+0000 mon.a (mon.0) 2917 : audit [DBG] from='client.? 172.21.15.6:0/217756566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:35.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:35 smithi006 bash[33918]: cluster 2023-08-28T00:05:34.044116+0000 mgr.y (mgr.14901) 3810 : cluster [DBG] pgmap v3817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:35.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:35 smithi006 bash[33918]: audit 2023-08-28T00:05:34.678636+0000 mon.a (mon.0) 2917 : audit [DBG] from='client.? 172.21.15.6:0/217756566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:35.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:35 smithi006 bash[27292]: cluster 2023-08-28T00:05:34.044116+0000 mgr.y (mgr.14901) 3810 : cluster [DBG] pgmap v3817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:35.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:35 smithi006 bash[27292]: audit 2023-08-28T00:05:34.678636+0000 mon.a (mon.0) 2917 : audit [DBG] from='client.? 172.21.15.6:0/217756566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:36.400 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:36 smithi130 bash[51302]: debug 2023-08-28T00:05:36.144+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:36.145316+0000) 2023-08-28T00:05:36.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:36 smithi006 bash[27292]: cluster 2023-08-28T00:05:36.045182+0000 mgr.y (mgr.14901) 3811 : cluster [DBG] pgmap v3818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:36 smithi006 bash[33918]: cluster 2023-08-28T00:05:36.045182+0000 mgr.y (mgr.14901) 3811 : cluster [DBG] pgmap v3818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:36 smithi130 bash[27579]: cluster 2023-08-28T00:05:36.045182+0000 mgr.y (mgr.14901) 3811 : cluster [DBG] pgmap v3818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:37.406 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:37 smithi130 bash[51302]: debug 2023-08-28T00:05:37.144+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:37.145586+0000) 2023-08-28T00:05:37.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:37 smithi006 bash[33918]: audit 2023-08-28T00:05:37.088144+0000 mon.c (mon.2) 1444 : audit [DBG] from='client.? 172.21.15.6:0/3692302818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:37.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:37 smithi006 bash[27292]: audit 2023-08-28T00:05:37.088144+0000 mon.c (mon.2) 1444 : audit [DBG] from='client.? 172.21.15.6:0/3692302818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:37.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:37 smithi130 bash[27579]: audit 2023-08-28T00:05:37.088144+0000 mon.c (mon.2) 1444 : audit [DBG] from='client.? 172.21.15.6:0/3692302818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:38.411 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:38 smithi130 bash[51302]: debug 2023-08-28T00:05:38.144+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:38.145856+0000) 2023-08-28T00:05:38.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:38 smithi006 bash[33918]: cluster 2023-08-28T00:05:38.045854+0000 mgr.y (mgr.14901) 3812 : cluster [DBG] pgmap v3819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:38.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:38 smithi006 bash[27292]: cluster 2023-08-28T00:05:38.045854+0000 mgr.y (mgr.14901) 3812 : cluster [DBG] pgmap v3819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:38 smithi130 bash[27579]: cluster 2023-08-28T00:05:38.045854+0000 mgr.y (mgr.14901) 3812 : cluster [DBG] pgmap v3819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:39.556 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:39 smithi130 bash[51302]: debug 2023-08-28T00:05:39.144+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:39.146181+0000) 2023-08-28T00:05:39.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:39 smithi130 bash[27579]: audit 2023-08-28T00:05:39.498813+0000 mon.c (mon.2) 1445 : audit [DBG] from='client.? 172.21.15.6:0/547603627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:39 smithi006 bash[33918]: audit 2023-08-28T00:05:39.498813+0000 mon.c (mon.2) 1445 : audit [DBG] from='client.? 172.21.15.6:0/547603627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:39.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:39 smithi006 bash[27292]: audit 2023-08-28T00:05:39.498813+0000 mon.c (mon.2) 1445 : audit [DBG] from='client.? 172.21.15.6:0/547603627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:40.563 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:40 smithi130 bash[51302]: debug 2023-08-28T00:05:40.140+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:40.146431+0000) 2023-08-28T00:05:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:40 smithi130 bash[27579]: cluster 2023-08-28T00:05:40.046981+0000 mgr.y (mgr.14901) 3813 : cluster [DBG] pgmap v3820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:40.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:40 smithi006 bash[33918]: cluster 2023-08-28T00:05:40.046981+0000 mgr.y (mgr.14901) 3813 : cluster [DBG] pgmap v3820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:40.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:40 smithi006 bash[27292]: cluster 2023-08-28T00:05:40.046981+0000 mgr.y (mgr.14901) 3813 : cluster [DBG] pgmap v3820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:05:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:05:40] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:05:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:05:40.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:05:40.647Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:41.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:41 smithi130 bash[51302]: debug 2023-08-28T00:05:41.144+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:41.146712+0000) 2023-08-28T00:05:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:41 smithi130 bash[27579]: audit 2023-08-28T00:05:41.913092+0000 mon.a (mon.0) 2918 : audit [DBG] from='client.? 172.21.15.6:0/3323179640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:42.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:42 smithi130 bash[51302]: debug 2023-08-28T00:05:42.144+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:42.147018+0000) 2023-08-28T00:05:42.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:41 smithi006 bash[27292]: audit 2023-08-28T00:05:41.913092+0000 mon.a (mon.0) 2918 : audit [DBG] from='client.? 172.21.15.6:0/3323179640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:42.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:41 smithi006 bash[33918]: audit 2023-08-28T00:05:41.913092+0000 mon.a (mon.0) 2918 : audit [DBG] from='client.? 172.21.15.6:0/3323179640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:05:42.215Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:05:42.217Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:43.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:42 smithi130 bash[27579]: cluster 2023-08-28T00:05:42.047678+0000 mgr.y (mgr.14901) 3814 : cluster [DBG] pgmap v3821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:43.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:43 smithi130 bash[51302]: debug 2023-08-28T00:05:43.144+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:43.147231+0000) 2023-08-28T00:05:43.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:42 smithi006 bash[27292]: cluster 2023-08-28T00:05:42.047678+0000 mgr.y (mgr.14901) 3814 : cluster [DBG] pgmap v3821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:43.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:42 smithi006 bash[33918]: cluster 2023-08-28T00:05:42.047678+0000 mgr.y (mgr.14901) 3814 : cluster [DBG] pgmap v3821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:05:44.102Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:05:44.104Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:44.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:44 smithi130 bash[51302]: debug 2023-08-28T00:05:44.144+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:44.147548+0000) 2023-08-28T00:05:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:45 smithi130 bash[51302]: debug 2023-08-28T00:05:45.144+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:45.147836+0000) 2023-08-28T00:05:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:45 smithi130 bash[27579]: cluster 2023-08-28T00:05:44.048285+0000 mgr.y (mgr.14901) 3815 : cluster [DBG] pgmap v3822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:45 smithi130 bash[27579]: audit 2023-08-28T00:05:44.328774+0000 mon.c (mon.2) 1446 : audit [DBG] from='client.? 172.21.15.6:0/43579210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:45.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:45 smithi006 bash[27292]: cluster 2023-08-28T00:05:44.048285+0000 mgr.y (mgr.14901) 3815 : cluster [DBG] pgmap v3822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:45.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:45 smithi006 bash[27292]: audit 2023-08-28T00:05:44.328774+0000 mon.c (mon.2) 1446 : audit [DBG] from='client.? 172.21.15.6:0/43579210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:45.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:45 smithi006 bash[33918]: cluster 2023-08-28T00:05:44.048285+0000 mgr.y (mgr.14901) 3815 : cluster [DBG] pgmap v3822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:45.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:45 smithi006 bash[33918]: audit 2023-08-28T00:05:44.328774+0000 mon.c (mon.2) 1446 : audit [DBG] from='client.? 172.21.15.6:0/43579210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:46.405 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:46 smithi130 bash[51302]: debug 2023-08-28T00:05:46.144+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:46.148087+0000) 2023-08-28T00:05:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:46 smithi006 bash[27292]: cluster 2023-08-28T00:05:46.049354+0000 mgr.y (mgr.14901) 3816 : cluster [DBG] pgmap v3823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:46 smithi006 bash[33918]: cluster 2023-08-28T00:05:46.049354+0000 mgr.y (mgr.14901) 3816 : cluster [DBG] pgmap v3823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:46 smithi130 bash[27579]: cluster 2023-08-28T00:05:46.049354+0000 mgr.y (mgr.14901) 3816 : cluster [DBG] pgmap v3823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:47.409 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:47 smithi130 bash[51302]: debug 2023-08-28T00:05:47.144+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:47.148365+0000) 2023-08-28T00:05:47.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:47 smithi006 bash[27292]: audit 2023-08-28T00:05:46.739797+0000 mon.a (mon.0) 2919 : audit [DBG] from='client.? 172.21.15.6:0/4110404023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:47.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:47 smithi006 bash[33918]: audit 2023-08-28T00:05:46.739797+0000 mon.a (mon.0) 2919 : audit [DBG] from='client.? 172.21.15.6:0/4110404023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:47.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:47 smithi130 bash[27579]: audit 2023-08-28T00:05:46.739797+0000 mon.a (mon.0) 2919 : audit [DBG] from='client.? 172.21.15.6:0/4110404023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:48.418 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:48 smithi130 bash[51302]: debug 2023-08-28T00:05:48.144+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:48.148661+0000) 2023-08-28T00:05:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:48 smithi006 bash[33918]: cluster 2023-08-28T00:05:48.050052+0000 mgr.y (mgr.14901) 3817 : cluster [DBG] pgmap v3824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:48 smithi006 bash[27292]: cluster 2023-08-28T00:05:48.050052+0000 mgr.y (mgr.14901) 3817 : cluster [DBG] pgmap v3824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:48 smithi130 bash[27579]: cluster 2023-08-28T00:05:48.050052+0000 mgr.y (mgr.14901) 3817 : cluster [DBG] pgmap v3824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:49.421 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:49 smithi130 bash[51302]: debug 2023-08-28T00:05:49.144+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:49.148970+0000) 2023-08-28T00:05:49.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:49 smithi006 bash[33918]: audit 2023-08-28T00:05:49.154992+0000 mon.c (mon.2) 1447 : audit [DBG] from='client.? 172.21.15.6:0/2536998043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:49.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:49 smithi006 bash[27292]: audit 2023-08-28T00:05:49.154992+0000 mon.c (mon.2) 1447 : audit [DBG] from='client.? 172.21.15.6:0/2536998043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:49.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:49 smithi130 bash[27579]: audit 2023-08-28T00:05:49.154992+0000 mon.c (mon.2) 1447 : audit [DBG] from='client.? 172.21.15.6:0/2536998043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:50.427 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:50 smithi130 bash[51302]: debug 2023-08-28T00:05:50.144+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:50.149276+0000) 2023-08-28T00:05:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:50 smithi006 bash[33918]: cluster 2023-08-28T00:05:50.051212+0000 mgr.y (mgr.14901) 3818 : cluster [DBG] pgmap v3825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:05:50.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:50.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:05:50.648Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:50 smithi006 bash[27292]: cluster 2023-08-28T00:05:50.051212+0000 mgr.y (mgr.14901) 3818 : cluster [DBG] pgmap v3825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:50 smithi130 bash[27579]: cluster 2023-08-28T00:05:50.051212+0000 mgr.y (mgr.14901) 3818 : cluster [DBG] pgmap v3825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:05:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:05:50] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:05:51.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:51 smithi130 bash[51302]: debug 2023-08-28T00:05:51.144+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:51.149563+0000) 2023-08-28T00:05:51.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:51 smithi006 bash[27292]: audit 2023-08-28T00:05:51.565649+0000 mon.a (mon.0) 2920 : audit [DBG] from='client.? 172.21.15.6:0/3354851066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:51.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:51 smithi006 bash[33918]: audit 2023-08-28T00:05:51.565649+0000 mon.a (mon.0) 2920 : audit [DBG] from='client.? 172.21.15.6:0/3354851066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:52.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:51 smithi130 bash[27579]: audit 2023-08-28T00:05:51.565649+0000 mon.a (mon.0) 2920 : audit [DBG] from='client.? 172.21.15.6:0/3354851066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:52.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:52 smithi130 bash[51302]: debug 2023-08-28T00:05:52.144+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:52.149853+0000) 2023-08-28T00:05:52.628 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:05:52.216Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:52.628 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:05:52.217Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:52 smithi006 bash[27292]: cluster 2023-08-28T00:05:52.051853+0000 mgr.y (mgr.14901) 3819 : cluster [DBG] pgmap v3826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:52 smithi006 bash[33918]: cluster 2023-08-28T00:05:52.051853+0000 mgr.y (mgr.14901) 3819 : cluster [DBG] pgmap v3826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:52 smithi130 bash[27579]: cluster 2023-08-28T00:05:52.051853+0000 mgr.y (mgr.14901) 3819 : cluster [DBG] pgmap v3826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:53.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:53 smithi130 bash[51302]: debug 2023-08-28T00:05:53.148+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:53.150141+0000) 2023-08-28T00:05:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:54 smithi130 bash[27579]: audit 2023-08-28T00:05:53.983777+0000 mon.c (mon.2) 1448 : audit [DBG] from='client.? 172.21.15.6:0/847445873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:54 smithi130 bash[51302]: debug 2023-08-28T00:05:54.148+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:54.150422+0000) 2023-08-28T00:05:54.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:54 smithi006 bash[27292]: audit 2023-08-28T00:05:53.983777+0000 mon.c (mon.2) 1448 : audit [DBG] from='client.? 172.21.15.6:0/847445873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:54.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:54 smithi006 bash[33918]: audit 2023-08-28T00:05:53.983777+0000 mon.c (mon.2) 1448 : audit [DBG] from='client.? 172.21.15.6:0/847445873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:05:54.103Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:05:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:05:54.105Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:05:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:55 smithi130 bash[27579]: cluster 2023-08-28T00:05:54.052563+0000 mgr.y (mgr.14901) 3820 : cluster [DBG] pgmap v3827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:55 smithi130 bash[51302]: debug 2023-08-28T00:05:55.144+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:55.150747+0000) 2023-08-28T00:05:55.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:55 smithi006 bash[27292]: cluster 2023-08-28T00:05:54.052563+0000 mgr.y (mgr.14901) 3820 : cluster [DBG] pgmap v3827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:55.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:55 smithi006 bash[33918]: cluster 2023-08-28T00:05:54.052563+0000 mgr.y (mgr.14901) 3820 : cluster [DBG] pgmap v3827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:56.407 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:56 smithi130 bash[51302]: debug 2023-08-28T00:05:56.148+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:56.151035+0000) 2023-08-28T00:05:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:56 smithi006 bash[33918]: cluster 2023-08-28T00:05:56.053298+0000 mgr.y (mgr.14901) 3821 : cluster [DBG] pgmap v3828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:56 smithi006 bash[33918]: audit 2023-08-28T00:05:56.392790+0000 mon.a (mon.0) 2921 : audit [DBG] from='client.? 172.21.15.6:0/2672515364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:56 smithi006 bash[27292]: cluster 2023-08-28T00:05:56.053298+0000 mgr.y (mgr.14901) 3821 : cluster [DBG] pgmap v3828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:56 smithi006 bash[27292]: audit 2023-08-28T00:05:56.392790+0000 mon.a (mon.0) 2921 : audit [DBG] from='client.? 172.21.15.6:0/2672515364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:56 smithi130 bash[27579]: cluster 2023-08-28T00:05:56.053298+0000 mgr.y (mgr.14901) 3821 : cluster [DBG] pgmap v3828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:56 smithi130 bash[27579]: audit 2023-08-28T00:05:56.392790+0000 mon.a (mon.0) 2921 : audit [DBG] from='client.? 172.21.15.6:0/2672515364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:57.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:57 smithi130 bash[51302]: debug 2023-08-28T00:05:57.148+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:57.151373+0000) 2023-08-28T00:05:58.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:58 smithi130 bash[51302]: debug 2023-08-28T00:05:58.148+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:58.151681+0000) 2023-08-28T00:05:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:05:59 smithi130 bash[51302]: debug 2023-08-28T00:05:59.148+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:59.151975+0000) 2023-08-28T00:05:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:59 smithi130 bash[27579]: cluster 2023-08-28T00:05:58.053935+0000 mgr.y (mgr.14901) 3822 : cluster [DBG] pgmap v3829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:05:59 smithi130 bash[27579]: audit 2023-08-28T00:05:58.812449+0000 mon.c (mon.2) 1449 : audit [DBG] from='client.? 172.21.15.6:0/4252335420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:59 smithi006 bash[33918]: cluster 2023-08-28T00:05:58.053935+0000 mgr.y (mgr.14901) 3822 : cluster [DBG] pgmap v3829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:05:59 smithi006 bash[33918]: audit 2023-08-28T00:05:58.812449+0000 mon.c (mon.2) 1449 : audit [DBG] from='client.? 172.21.15.6:0/4252335420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:59.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:59 smithi006 bash[27292]: cluster 2023-08-28T00:05:58.053935+0000 mgr.y (mgr.14901) 3822 : cluster [DBG] pgmap v3829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:05:59.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:05:59 smithi006 bash[27292]: audit 2023-08-28T00:05:58.812449+0000 mon.c (mon.2) 1449 : audit [DBG] from='client.? 172.21.15.6:0/4252335420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:00.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:00 smithi130 bash[51302]: debug 2023-08-28T00:06:00.148+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:00.152232+0000) 2023-08-28T00:06:00.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:06:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:06:00] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T00:06:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:06:00.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:06:00.648Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:01.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:01 smithi006 bash[33918]: cluster 2023-08-28T00:06:00.055014+0000 mgr.y (mgr.14901) 3823 : cluster [DBG] pgmap v3830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:01.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:01 smithi006 bash[27292]: cluster 2023-08-28T00:06:00.055014+0000 mgr.y (mgr.14901) 3823 : cluster [DBG] pgmap v3830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:01.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:01 smithi130 bash[27579]: cluster 2023-08-28T00:06:00.055014+0000 mgr.y (mgr.14901) 3823 : cluster [DBG] pgmap v3830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:01.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:01 smithi130 bash[51302]: debug 2023-08-28T00:06:01.148+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:01.152509+0000) 2023-08-28T00:06:02.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:02 smithi006 bash[33918]: audit 2023-08-28T00:06:01.225751+0000 mon.a (mon.0) 2922 : audit [DBG] from='client.? 172.21.15.6:0/1248517052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:02.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:02 smithi006 bash[27292]: audit 2023-08-28T00:06:01.225751+0000 mon.a (mon.0) 2922 : audit [DBG] from='client.? 172.21.15.6:0/1248517052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:06:02.216Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:06:02.218Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:02 smithi130 bash[27579]: audit 2023-08-28T00:06:01.225751+0000 mon.a (mon.0) 2922 : audit [DBG] from='client.? 172.21.15.6:0/1248517052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:02.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:02 smithi130 bash[51302]: debug 2023-08-28T00:06:02.148+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:02.152748+0000) 2023-08-28T00:06:03.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:03 smithi006 bash[27292]: cluster 2023-08-28T00:06:02.055647+0000 mgr.y (mgr.14901) 3824 : cluster [DBG] pgmap v3831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:03.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:03 smithi006 bash[33918]: cluster 2023-08-28T00:06:02.055647+0000 mgr.y (mgr.14901) 3824 : cluster [DBG] pgmap v3831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:03.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:03 smithi130 bash[51302]: debug 2023-08-28T00:06:03.152+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:03.153018+0000) 2023-08-28T00:06:03.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:03 smithi130 bash[27579]: cluster 2023-08-28T00:06:02.055647+0000 mgr.y (mgr.14901) 3824 : cluster [DBG] pgmap v3831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:04.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:04 smithi006 bash[27292]: audit 2023-08-28T00:06:03.639273+0000 mon.c (mon.2) 1450 : audit [DBG] from='client.? 172.21.15.6:0/2311115662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:04.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:04 smithi006 bash[33918]: audit 2023-08-28T00:06:03.639273+0000 mon.c (mon.2) 1450 : audit [DBG] from='client.? 172.21.15.6:0/2311115662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:06:04.103Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:06:04.105Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:04 smithi130 bash[27579]: audit 2023-08-28T00:06:03.639273+0000 mon.c (mon.2) 1450 : audit [DBG] from='client.? 172.21.15.6:0/2311115662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:04.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:04 smithi130 bash[51302]: debug 2023-08-28T00:06:04.148+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:04.153371+0000) 2023-08-28T00:06:05.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:05 smithi006 bash[33918]: cluster 2023-08-28T00:06:04.056523+0000 mgr.y (mgr.14901) 3825 : cluster [DBG] pgmap v3832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:05.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:05 smithi006 bash[27292]: cluster 2023-08-28T00:06:04.056523+0000 mgr.y (mgr.14901) 3825 : cluster [DBG] pgmap v3832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:05.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:05 smithi130 bash[27579]: cluster 2023-08-28T00:06:04.056523+0000 mgr.y (mgr.14901) 3825 : cluster [DBG] pgmap v3832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:05.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:05 smithi130 bash[51302]: debug 2023-08-28T00:06:05.148+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:05.153641+0000) 2023-08-28T00:06:06.409 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:06 smithi130 bash[51302]: debug 2023-08-28T00:06:06.148+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:06.153912+0000) 2023-08-28T00:06:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:06 smithi006 bash[33918]: audit 2023-08-28T00:06:06.049670+0000 mon.a (mon.0) 2923 : audit [DBG] from='client.? 172.21.15.6:0/270601405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:06 smithi006 bash[33918]: cluster 2023-08-28T00:06:06.057354+0000 mgr.y (mgr.14901) 3826 : cluster [DBG] pgmap v3833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:06 smithi006 bash[27292]: audit 2023-08-28T00:06:06.049670+0000 mon.a (mon.0) 2923 : audit [DBG] from='client.? 172.21.15.6:0/270601405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:06 smithi006 bash[27292]: cluster 2023-08-28T00:06:06.057354+0000 mgr.y (mgr.14901) 3826 : cluster [DBG] pgmap v3833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:06 smithi130 bash[27579]: audit 2023-08-28T00:06:06.049670+0000 mon.a (mon.0) 2923 : audit [DBG] from='client.? 172.21.15.6:0/270601405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:06 smithi130 bash[27579]: cluster 2023-08-28T00:06:06.057354+0000 mgr.y (mgr.14901) 3826 : cluster [DBG] pgmap v3833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:07.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:07 smithi130 bash[51302]: debug 2023-08-28T00:06:07.152+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:07.154212+0000) 2023-08-28T00:06:08.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:08 smithi130 bash[51302]: debug 2023-08-28T00:06:08.152+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:08.154486+0000) 2023-08-28T00:06:09.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:09 smithi006 bash[33918]: cluster 2023-08-28T00:06:08.058085+0000 mgr.y (mgr.14901) 3827 : cluster [DBG] pgmap v3834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:09.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:09 smithi006 bash[33918]: audit 2023-08-28T00:06:08.478802+0000 mon.c (mon.2) 1451 : audit [DBG] from='client.? 172.21.15.6:0/804782119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:09.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:09 smithi006 bash[27292]: cluster 2023-08-28T00:06:08.058085+0000 mgr.y (mgr.14901) 3827 : cluster [DBG] pgmap v3834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:09.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:09 smithi006 bash[27292]: audit 2023-08-28T00:06:08.478802+0000 mon.c (mon.2) 1451 : audit [DBG] from='client.? 172.21.15.6:0/804782119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:09 smithi130 bash[27579]: cluster 2023-08-28T00:06:08.058085+0000 mgr.y (mgr.14901) 3827 : cluster [DBG] pgmap v3834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:09 smithi130 bash[27579]: audit 2023-08-28T00:06:08.478802+0000 mon.c (mon.2) 1451 : audit [DBG] from='client.? 172.21.15.6:0/804782119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:09.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:09 smithi130 bash[51302]: debug 2023-08-28T00:06:09.152+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:09.154873+0000) 2023-08-28T00:06:10.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:10 smithi130 bash[51302]: debug 2023-08-28T00:06:10.152+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:10.155179+0000) 2023-08-28T00:06:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:06:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:06:10] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T00:06:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:06:10.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:06:10.649Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:11.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:11 smithi006 bash[33918]: cluster 2023-08-28T00:06:10.059367+0000 mgr.y (mgr.14901) 3828 : cluster [DBG] pgmap v3835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:11.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:11 smithi006 bash[33918]: audit 2023-08-28T00:06:10.897661+0000 mon.c (mon.2) 1452 : audit [DBG] from='client.? 172.21.15.6:0/2897311355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:11.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:11 smithi006 bash[27292]: cluster 2023-08-28T00:06:10.059367+0000 mgr.y (mgr.14901) 3828 : cluster [DBG] pgmap v3835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:11.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:11 smithi006 bash[27292]: audit 2023-08-28T00:06:10.897661+0000 mon.c (mon.2) 1452 : audit [DBG] from='client.? 172.21.15.6:0/2897311355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:11.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:11 smithi130 bash[27579]: cluster 2023-08-28T00:06:10.059367+0000 mgr.y (mgr.14901) 3828 : cluster [DBG] pgmap v3835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:11.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:11 smithi130 bash[27579]: audit 2023-08-28T00:06:10.897661+0000 mon.c (mon.2) 1452 : audit [DBG] from='client.? 172.21.15.6:0/2897311355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:11.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:11 smithi130 bash[51302]: debug 2023-08-28T00:06:11.152+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:11.155464+0000) 2023-08-28T00:06:12.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:12 smithi130 bash[51302]: debug 2023-08-28T00:06:12.152+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:12.155730+0000) 2023-08-28T00:06:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:06:12.217Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:06:12.219Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:13.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:13 smithi006 bash[27292]: cluster 2023-08-28T00:06:12.060168+0000 mgr.y (mgr.14901) 3829 : cluster [DBG] pgmap v3836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:13.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:13 smithi006 bash[33918]: cluster 2023-08-28T00:06:12.060168+0000 mgr.y (mgr.14901) 3829 : cluster [DBG] pgmap v3836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:13 smithi130 bash[27579]: cluster 2023-08-28T00:06:12.060168+0000 mgr.y (mgr.14901) 3829 : cluster [DBG] pgmap v3836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:13.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:13 smithi130 bash[51302]: debug 2023-08-28T00:06:13.152+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:13.155993+0000) 2023-08-28T00:06:14.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:14 smithi006 bash[33918]: audit 2023-08-28T00:06:13.314086+0000 mon.a (mon.0) 2924 : audit [DBG] from='client.? 172.21.15.6:0/4112029486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:06:14.104Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:06:14.106Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:14.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:14 smithi006 bash[27292]: audit 2023-08-28T00:06:13.314086+0000 mon.a (mon.0) 2924 : audit [DBG] from='client.? 172.21.15.6:0/4112029486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:14 smithi130 bash[27579]: audit 2023-08-28T00:06:13.314086+0000 mon.a (mon.0) 2924 : audit [DBG] from='client.? 172.21.15.6:0/4112029486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:14.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:14 smithi130 bash[51302]: debug 2023-08-28T00:06:14.152+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:14.156260+0000) 2023-08-28T00:06:15.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:15 smithi006 bash[33918]: cluster 2023-08-28T00:06:14.060979+0000 mgr.y (mgr.14901) 3830 : cluster [DBG] pgmap v3837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:15.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:15 smithi006 bash[27292]: cluster 2023-08-28T00:06:14.060979+0000 mgr.y (mgr.14901) 3830 : cluster [DBG] pgmap v3837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:15.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:15 smithi130 bash[27579]: cluster 2023-08-28T00:06:14.060979+0000 mgr.y (mgr.14901) 3830 : cluster [DBG] pgmap v3837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:15.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:15 smithi130 bash[51302]: debug 2023-08-28T00:06:15.152+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:15.156542+0000) 2023-08-28T00:06:16.412 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:16 smithi130 bash[51302]: debug 2023-08-28T00:06:16.156+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:16.156808+0000) 2023-08-28T00:06:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:16 smithi006 bash[27292]: audit 2023-08-28T00:06:15.735913+0000 mon.a (mon.0) 2925 : audit [DBG] from='client.? 172.21.15.6:0/3504196689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:16 smithi006 bash[27292]: cluster 2023-08-28T00:06:16.061944+0000 mgr.y (mgr.14901) 3831 : cluster [DBG] pgmap v3838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:16 smithi006 bash[33918]: audit 2023-08-28T00:06:15.735913+0000 mon.a (mon.0) 2925 : audit [DBG] from='client.? 172.21.15.6:0/3504196689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:16 smithi006 bash[33918]: cluster 2023-08-28T00:06:16.061944+0000 mgr.y (mgr.14901) 3831 : cluster [DBG] pgmap v3838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:16 smithi130 bash[27579]: audit 2023-08-28T00:06:15.735913+0000 mon.a (mon.0) 2925 : audit [DBG] from='client.? 172.21.15.6:0/3504196689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:16 smithi130 bash[27579]: cluster 2023-08-28T00:06:16.061944+0000 mgr.y (mgr.14901) 3831 : cluster [DBG] pgmap v3838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:17.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:17 smithi130 bash[51302]: debug 2023-08-28T00:06:17.156+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:17.157081+0000) 2023-08-28T00:06:18.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:18 smithi130 bash[51302]: debug 2023-08-28T00:06:18.156+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:18.157373+0000) 2023-08-28T00:06:19.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:19 smithi006 bash[33918]: cluster 2023-08-28T00:06:18.062428+0000 mgr.y (mgr.14901) 3832 : cluster [DBG] pgmap v3839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:19.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:19 smithi006 bash[33918]: audit 2023-08-28T00:06:18.156980+0000 mon.c (mon.2) 1453 : audit [DBG] from='client.? 172.21.15.6:0/3437998089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:19.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:19 smithi006 bash[33918]: audit 2023-08-28T00:06:18.920989+0000 mon.b (mon.1) 481 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:06:19.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:19 smithi006 bash[27292]: cluster 2023-08-28T00:06:18.062428+0000 mgr.y (mgr.14901) 3832 : cluster [DBG] pgmap v3839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:19.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:19 smithi006 bash[27292]: audit 2023-08-28T00:06:18.156980+0000 mon.c (mon.2) 1453 : audit [DBG] from='client.? 172.21.15.6:0/3437998089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:19.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:19 smithi006 bash[27292]: audit 2023-08-28T00:06:18.920989+0000 mon.b (mon.1) 481 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:06:19.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:19 smithi130 bash[51302]: debug 2023-08-28T00:06:19.152+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:19.157675+0000) 2023-08-28T00:06:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:19 smithi130 bash[27579]: cluster 2023-08-28T00:06:18.062428+0000 mgr.y (mgr.14901) 3832 : cluster [DBG] pgmap v3839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:19 smithi130 bash[27579]: audit 2023-08-28T00:06:18.156980+0000 mon.c (mon.2) 1453 : audit [DBG] from='client.? 172.21.15.6:0/3437998089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:19 smithi130 bash[27579]: audit 2023-08-28T00:06:18.920989+0000 mon.b (mon.1) 481 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:06:20.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:20 smithi130 bash[51302]: debug 2023-08-28T00:06:20.152+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:20.157884+0000) 2023-08-28T00:06:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:06:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:06:20] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T00:06:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:06:20.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:06:20.649Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:21.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:21 smithi006 bash[27292]: cluster 2023-08-28T00:06:20.063522+0000 mgr.y (mgr.14901) 3833 : cluster [DBG] pgmap v3840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:21.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:21 smithi006 bash[27292]: audit 2023-08-28T00:06:20.580266+0000 mon.c (mon.2) 1454 : audit [DBG] from='client.? 172.21.15.6:0/2724710995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:21.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:21 smithi006 bash[33918]: cluster 2023-08-28T00:06:20.063522+0000 mgr.y (mgr.14901) 3833 : cluster [DBG] pgmap v3840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:21.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:21 smithi006 bash[33918]: audit 2023-08-28T00:06:20.580266+0000 mon.c (mon.2) 1454 : audit [DBG] from='client.? 172.21.15.6:0/2724710995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:21 smithi130 bash[27579]: cluster 2023-08-28T00:06:20.063522+0000 mgr.y (mgr.14901) 3833 : cluster [DBG] pgmap v3840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:21 smithi130 bash[27579]: audit 2023-08-28T00:06:20.580266+0000 mon.c (mon.2) 1454 : audit [DBG] from='client.? 172.21.15.6:0/2724710995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:21.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:21 smithi130 bash[51302]: debug 2023-08-28T00:06:21.152+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:21.158158+0000) 2023-08-28T00:06:22.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:22 smithi130 bash[51302]: debug 2023-08-28T00:06:22.152+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:22.158452+0000) 2023-08-28T00:06:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:06:22.218Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:06:22.219Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:23.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:23 smithi006 bash[33918]: cluster 2023-08-28T00:06:22.064183+0000 mgr.y (mgr.14901) 3834 : cluster [DBG] pgmap v3841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:23.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:23 smithi006 bash[33918]: audit 2023-08-28T00:06:23.000541+0000 mon.c (mon.2) 1455 : audit [DBG] from='client.? 172.21.15.6:0/859168773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:23.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:23 smithi006 bash[27292]: cluster 2023-08-28T00:06:22.064183+0000 mgr.y (mgr.14901) 3834 : cluster [DBG] pgmap v3841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:23.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:23 smithi006 bash[27292]: audit 2023-08-28T00:06:23.000541+0000 mon.c (mon.2) 1455 : audit [DBG] from='client.? 172.21.15.6:0/859168773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:23.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:23 smithi130 bash[27579]: cluster 2023-08-28T00:06:22.064183+0000 mgr.y (mgr.14901) 3834 : cluster [DBG] pgmap v3841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:23.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:23 smithi130 bash[27579]: audit 2023-08-28T00:06:23.000541+0000 mon.c (mon.2) 1455 : audit [DBG] from='client.? 172.21.15.6:0/859168773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:23.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:23 smithi130 bash[51302]: debug 2023-08-28T00:06:23.151+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:23.158727+0000) 2023-08-28T00:06:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:06:24.104Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:06:24.106Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:24.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:24 smithi130 bash[51302]: debug 2023-08-28T00:06:24.155+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:24.159060+0000) 2023-08-28T00:06:25.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:25 smithi006 bash[33918]: cluster 2023-08-28T00:06:24.064800+0000 mgr.y (mgr.14901) 3835 : cluster [DBG] pgmap v3842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:25.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:25 smithi006 bash[33918]: audit 2023-08-28T00:06:24.814788+0000 mon.a (mon.0) 2926 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:06:25.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:25 smithi006 bash[27292]: cluster 2023-08-28T00:06:24.064800+0000 mgr.y (mgr.14901) 3835 : cluster [DBG] pgmap v3842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:25.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:25 smithi006 bash[27292]: audit 2023-08-28T00:06:24.814788+0000 mon.a (mon.0) 2926 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:06:25.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:25 smithi130 bash[51302]: debug 2023-08-28T00:06:25.155+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:25.159265+0000) 2023-08-28T00:06:25.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:25 smithi130 bash[27579]: cluster 2023-08-28T00:06:24.064800+0000 mgr.y (mgr.14901) 3835 : cluster [DBG] pgmap v3842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:25.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:25 smithi130 bash[27579]: audit 2023-08-28T00:06:24.814788+0000 mon.a (mon.0) 2926 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:06:26.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:26 smithi130 bash[51302]: debug 2023-08-28T00:06:26.155+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:26.159470+0000) 2023-08-28T00:06:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:26 smithi130 bash[27579]: audit 2023-08-28T00:06:25.280454+0000 mon.a (mon.0) 2927 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:06:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:26 smithi130 bash[27579]: audit 2023-08-28T00:06:25.409908+0000 mon.c (mon.2) 1456 : audit [DBG] from='client.? 172.21.15.6:0/849280782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:26 smithi130 bash[27579]: audit 2023-08-28T00:06:25.577750+0000 mon.b (mon.1) 482 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:06:26.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:26 smithi130 bash[27579]: audit 2023-08-28T00:06:25.579921+0000 mon.b (mon.1) 483 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:06:26.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:26 smithi130 bash[27579]: audit 2023-08-28T00:06:25.589275+0000 mon.a (mon.0) 2928 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:06:26.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:26 smithi130 bash[27579]: cluster 2023-08-28T00:06:26.065609+0000 mgr.y (mgr.14901) 3836 : cluster [DBG] pgmap v3843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:26 smithi006 bash[27292]: audit 2023-08-28T00:06:25.280454+0000 mon.a (mon.0) 2927 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:06:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:26 smithi006 bash[27292]: audit 2023-08-28T00:06:25.409908+0000 mon.c (mon.2) 1456 : audit [DBG] from='client.? 172.21.15.6:0/849280782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:26 smithi006 bash[27292]: audit 2023-08-28T00:06:25.577750+0000 mon.b (mon.1) 482 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:06:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:26 smithi006 bash[27292]: audit 2023-08-28T00:06:25.579921+0000 mon.b (mon.1) 483 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:06:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:26 smithi006 bash[27292]: audit 2023-08-28T00:06:25.589275+0000 mon.a (mon.0) 2928 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:06:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:26 smithi006 bash[27292]: cluster 2023-08-28T00:06:26.065609+0000 mgr.y (mgr.14901) 3836 : cluster [DBG] pgmap v3843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:26 smithi006 bash[33918]: audit 2023-08-28T00:06:25.280454+0000 mon.a (mon.0) 2927 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:06:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:26 smithi006 bash[33918]: audit 2023-08-28T00:06:25.409908+0000 mon.c (mon.2) 1456 : audit [DBG] from='client.? 172.21.15.6:0/849280782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:26 smithi006 bash[33918]: audit 2023-08-28T00:06:25.577750+0000 mon.b (mon.1) 482 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:06:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:26 smithi006 bash[33918]: audit 2023-08-28T00:06:25.579921+0000 mon.b (mon.1) 483 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:06:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:26 smithi006 bash[33918]: audit 2023-08-28T00:06:25.589275+0000 mon.a (mon.0) 2928 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:06:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:26 smithi006 bash[33918]: cluster 2023-08-28T00:06:26.065609+0000 mgr.y (mgr.14901) 3836 : cluster [DBG] pgmap v3843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:27.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:27 smithi130 bash[51302]: debug 2023-08-28T00:06:27.155+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:27.159722+0000) 2023-08-28T00:06:28.161 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:27 smithi130 bash[27579]: audit 2023-08-28T00:06:27.830194+0000 mon.a (mon.0) 2929 : audit [DBG] from='client.? 172.21.15.6:0/1734930045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:28.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:27 smithi006 bash[27292]: audit 2023-08-28T00:06:27.830194+0000 mon.a (mon.0) 2929 : audit [DBG] from='client.? 172.21.15.6:0/1734930045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:28.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:27 smithi006 bash[33918]: audit 2023-08-28T00:06:27.830194+0000 mon.a (mon.0) 2929 : audit [DBG] from='client.? 172.21.15.6:0/1734930045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:28.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:28 smithi130 bash[51302]: debug 2023-08-28T00:06:28.155+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:28.160045+0000) 2023-08-28T00:06:29.161 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:28 smithi130 bash[27579]: cluster 2023-08-28T00:06:28.066277+0000 mgr.y (mgr.14901) 3837 : cluster [DBG] pgmap v3844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:28 smithi006 bash[27292]: cluster 2023-08-28T00:06:28.066277+0000 mgr.y (mgr.14901) 3837 : cluster [DBG] pgmap v3844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:28 smithi006 bash[33918]: cluster 2023-08-28T00:06:28.066277+0000 mgr.y (mgr.14901) 3837 : cluster [DBG] pgmap v3844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:29.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:29 smithi130 bash[51302]: debug 2023-08-28T00:06:29.155+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:29.160341+0000) 2023-08-28T00:06:30.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:30 smithi130 bash[51302]: debug 2023-08-28T00:06:30.155+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:30.160648+0000) 2023-08-28T00:06:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:06:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:06:30] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:06:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:06:30.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:06:30.650Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:31.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:31 smithi006 bash[33918]: cluster 2023-08-28T00:06:30.067387+0000 mgr.y (mgr.14901) 3838 : cluster [DBG] pgmap v3845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:31.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:31 smithi006 bash[33918]: audit 2023-08-28T00:06:30.256770+0000 mon.a (mon.0) 2930 : audit [DBG] from='client.? 172.21.15.6:0/943452180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:31.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:31 smithi006 bash[27292]: cluster 2023-08-28T00:06:30.067387+0000 mgr.y (mgr.14901) 3838 : cluster [DBG] pgmap v3845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:31.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:31 smithi006 bash[27292]: audit 2023-08-28T00:06:30.256770+0000 mon.a (mon.0) 2930 : audit [DBG] from='client.? 172.21.15.6:0/943452180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:31.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:31 smithi130 bash[51302]: debug 2023-08-28T00:06:31.155+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:31.160942+0000) 2023-08-28T00:06:31.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:31 smithi130 bash[27579]: cluster 2023-08-28T00:06:30.067387+0000 mgr.y (mgr.14901) 3838 : cluster [DBG] pgmap v3845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:31.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:31 smithi130 bash[27579]: audit 2023-08-28T00:06:30.256770+0000 mon.a (mon.0) 2930 : audit [DBG] from='client.? 172.21.15.6:0/943452180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:32.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:32 smithi130 bash[51302]: debug 2023-08-28T00:06:32.155+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:32.161240+0000) 2023-08-28T00:06:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:06:32.219Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:06:32.221Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:33.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:33 smithi006 bash[27292]: cluster 2023-08-28T00:06:32.068033+0000 mgr.y (mgr.14901) 3839 : cluster [DBG] pgmap v3846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:33.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:33 smithi006 bash[27292]: audit 2023-08-28T00:06:32.671472+0000 mon.a (mon.0) 2931 : audit [DBG] from='client.? 172.21.15.6:0/4172684141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:33.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:33 smithi006 bash[33918]: cluster 2023-08-28T00:06:32.068033+0000 mgr.y (mgr.14901) 3839 : cluster [DBG] pgmap v3846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:33.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:33 smithi006 bash[33918]: audit 2023-08-28T00:06:32.671472+0000 mon.a (mon.0) 2931 : audit [DBG] from='client.? 172.21.15.6:0/4172684141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:33.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:33 smithi130 bash[51302]: debug 2023-08-28T00:06:33.155+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:33.161460+0000) 2023-08-28T00:06:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:33 smithi130 bash[27579]: cluster 2023-08-28T00:06:32.068033+0000 mgr.y (mgr.14901) 3839 : cluster [DBG] pgmap v3846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:33 smithi130 bash[27579]: audit 2023-08-28T00:06:32.671472+0000 mon.a (mon.0) 2931 : audit [DBG] from='client.? 172.21.15.6:0/4172684141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:06:34.105Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:06:34.107Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:34.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:34 smithi130 bash[51302]: debug 2023-08-28T00:06:34.155+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:34.161746+0000) 2023-08-28T00:06:35.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:35 smithi006 bash[33918]: cluster 2023-08-28T00:06:34.068895+0000 mgr.y (mgr.14901) 3840 : cluster [DBG] pgmap v3847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:35.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:35 smithi006 bash[33918]: audit 2023-08-28T00:06:35.081191+0000 mon.a (mon.0) 2932 : audit [DBG] from='client.? 172.21.15.6:0/986335556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:35.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:35 smithi006 bash[27292]: cluster 2023-08-28T00:06:34.068895+0000 mgr.y (mgr.14901) 3840 : cluster [DBG] pgmap v3847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:35.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:35 smithi006 bash[27292]: audit 2023-08-28T00:06:35.081191+0000 mon.a (mon.0) 2932 : audit [DBG] from='client.? 172.21.15.6:0/986335556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:35.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:35 smithi130 bash[51302]: debug 2023-08-28T00:06:35.155+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:35.162032+0000) 2023-08-28T00:06:35.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:35 smithi130 bash[27579]: cluster 2023-08-28T00:06:34.068895+0000 mgr.y (mgr.14901) 3840 : cluster [DBG] pgmap v3847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:35.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:35 smithi130 bash[27579]: audit 2023-08-28T00:06:35.081191+0000 mon.a (mon.0) 2932 : audit [DBG] from='client.? 172.21.15.6:0/986335556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:36.417 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:36 smithi130 bash[51302]: debug 2023-08-28T00:06:36.159+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:36.162305+0000) 2023-08-28T00:06:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:36 smithi006 bash[33918]: cluster 2023-08-28T00:06:36.069682+0000 mgr.y (mgr.14901) 3841 : cluster [DBG] pgmap v3848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:36 smithi006 bash[27292]: cluster 2023-08-28T00:06:36.069682+0000 mgr.y (mgr.14901) 3841 : cluster [DBG] pgmap v3848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:36 smithi130 bash[27579]: cluster 2023-08-28T00:06:36.069682+0000 mgr.y (mgr.14901) 3841 : cluster [DBG] pgmap v3848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:37.553 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:37 smithi130 bash[51302]: debug 2023-08-28T00:06:37.155+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:37.162588+0000) 2023-08-28T00:06:37.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:37 smithi130 bash[27579]: audit 2023-08-28T00:06:37.497262+0000 mon.a (mon.0) 2933 : audit [DBG] from='client.? 172.21.15.6:0/4096790893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:37.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:37 smithi006 bash[27292]: audit 2023-08-28T00:06:37.497262+0000 mon.a (mon.0) 2933 : audit [DBG] from='client.? 172.21.15.6:0/4096790893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:37.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:37 smithi006 bash[33918]: audit 2023-08-28T00:06:37.497262+0000 mon.a (mon.0) 2933 : audit [DBG] from='client.? 172.21.15.6:0/4096790893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:38.563 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:38 smithi130 bash[51302]: debug 2023-08-28T00:06:38.159+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:38.162872+0000) 2023-08-28T00:06:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:38 smithi130 bash[27579]: cluster 2023-08-28T00:06:38.070320+0000 mgr.y (mgr.14901) 3842 : cluster [DBG] pgmap v3849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:38 smithi006 bash[27292]: cluster 2023-08-28T00:06:38.070320+0000 mgr.y (mgr.14901) 3842 : cluster [DBG] pgmap v3849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:38 smithi006 bash[33918]: cluster 2023-08-28T00:06:38.070320+0000 mgr.y (mgr.14901) 3842 : cluster [DBG] pgmap v3849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:39.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:39 smithi130 bash[51302]: debug 2023-08-28T00:06:39.159+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:39.163196+0000) 2023-08-28T00:06:40.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:39 smithi006 bash[27292]: audit 2023-08-28T00:06:39.904771+0000 mon.a (mon.0) 2934 : audit [DBG] from='client.? 172.21.15.6:0/1436896264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:40.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:39 smithi006 bash[33918]: audit 2023-08-28T00:06:39.904771+0000 mon.a (mon.0) 2934 : audit [DBG] from='client.? 172.21.15.6:0/1436896264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:39 smithi130 bash[27579]: audit 2023-08-28T00:06:39.904771+0000 mon.a (mon.0) 2934 : audit [DBG] from='client.? 172.21.15.6:0/1436896264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:40.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:40 smithi130 bash[51302]: debug 2023-08-28T00:06:40.159+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:40.163496+0000) 2023-08-28T00:06:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:06:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:06:40] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:06:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:06:40.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:06:40.651Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:41.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:40 smithi130 bash[27579]: cluster 2023-08-28T00:06:40.071477+0000 mgr.y (mgr.14901) 3843 : cluster [DBG] pgmap v3850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:41.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:41 smithi130 bash[51302]: debug 2023-08-28T00:06:41.159+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:41.163784+0000) 2023-08-28T00:06:41.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:40 smithi006 bash[33918]: cluster 2023-08-28T00:06:40.071477+0000 mgr.y (mgr.14901) 3843 : cluster [DBG] pgmap v3850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:41.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:40 smithi006 bash[27292]: cluster 2023-08-28T00:06:40.071477+0000 mgr.y (mgr.14901) 3843 : cluster [DBG] pgmap v3850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:42.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:42 smithi130 bash[51302]: debug 2023-08-28T00:06:42.159+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:42.164119+0000) 2023-08-28T00:06:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:06:42.219Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:06:42.221Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:43.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:43 smithi006 bash[33918]: cluster 2023-08-28T00:06:42.071936+0000 mgr.y (mgr.14901) 3844 : cluster [DBG] pgmap v3851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:43.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:43 smithi006 bash[33918]: audit 2023-08-28T00:06:42.339989+0000 mon.c (mon.2) 1457 : audit [DBG] from='client.? 172.21.15.6:0/420012547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:43.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:43 smithi006 bash[27292]: cluster 2023-08-28T00:06:42.071936+0000 mgr.y (mgr.14901) 3844 : cluster [DBG] pgmap v3851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:43.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:43 smithi006 bash[27292]: audit 2023-08-28T00:06:42.339989+0000 mon.c (mon.2) 1457 : audit [DBG] from='client.? 172.21.15.6:0/420012547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:43.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:43 smithi130 bash[51302]: debug 2023-08-28T00:06:43.159+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:43.164415+0000) 2023-08-28T00:06:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:43 smithi130 bash[27579]: cluster 2023-08-28T00:06:42.071936+0000 mgr.y (mgr.14901) 3844 : cluster [DBG] pgmap v3851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:43 smithi130 bash[27579]: audit 2023-08-28T00:06:42.339989+0000 mon.c (mon.2) 1457 : audit [DBG] from='client.? 172.21.15.6:0/420012547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:06:44.106Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:06:44.108Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:44.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:44 smithi130 bash[51302]: debug 2023-08-28T00:06:44.159+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:44.164679+0000) 2023-08-28T00:06:45.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:45 smithi006 bash[27292]: cluster 2023-08-28T00:06:44.072766+0000 mgr.y (mgr.14901) 3845 : cluster [DBG] pgmap v3852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:45.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:45 smithi006 bash[27292]: audit 2023-08-28T00:06:44.753907+0000 mon.c (mon.2) 1458 : audit [DBG] from='client.? 172.21.15.6:0/1250498441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:45.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:45 smithi006 bash[33918]: cluster 2023-08-28T00:06:44.072766+0000 mgr.y (mgr.14901) 3845 : cluster [DBG] pgmap v3852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:45.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:45 smithi006 bash[33918]: audit 2023-08-28T00:06:44.753907+0000 mon.c (mon.2) 1458 : audit [DBG] from='client.? 172.21.15.6:0/1250498441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:45.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:45 smithi130 bash[51302]: debug 2023-08-28T00:06:45.163+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:45.164959+0000) 2023-08-28T00:06:45.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:45 smithi130 bash[27579]: cluster 2023-08-28T00:06:44.072766+0000 mgr.y (mgr.14901) 3845 : cluster [DBG] pgmap v3852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:45.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:45 smithi130 bash[27579]: audit 2023-08-28T00:06:44.753907+0000 mon.c (mon.2) 1458 : audit [DBG] from='client.? 172.21.15.6:0/1250498441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:46.437 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:46 smithi130 bash[51302]: debug 2023-08-28T00:06:46.163+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:46.165261+0000) 2023-08-28T00:06:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:46 smithi006 bash[27292]: cluster 2023-08-28T00:06:46.073716+0000 mgr.y (mgr.14901) 3846 : cluster [DBG] pgmap v3853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:46 smithi006 bash[33918]: cluster 2023-08-28T00:06:46.073716+0000 mgr.y (mgr.14901) 3846 : cluster [DBG] pgmap v3853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:46 smithi130 bash[27579]: cluster 2023-08-28T00:06:46.073716+0000 mgr.y (mgr.14901) 3846 : cluster [DBG] pgmap v3853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:47.440 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:47 smithi130 bash[51302]: debug 2023-08-28T00:06:47.163+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:47.165567+0000) 2023-08-28T00:06:47.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:47 smithi006 bash[33918]: audit 2023-08-28T00:06:47.163158+0000 mon.a (mon.0) 2935 : audit [DBG] from='client.? 172.21.15.6:0/2155591123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:47.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:47 smithi006 bash[27292]: audit 2023-08-28T00:06:47.163158+0000 mon.a (mon.0) 2935 : audit [DBG] from='client.? 172.21.15.6:0/2155591123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:47.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:47 smithi130 bash[27579]: audit 2023-08-28T00:06:47.163158+0000 mon.a (mon.0) 2935 : audit [DBG] from='client.? 172.21.15.6:0/2155591123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:48.448 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:48 smithi130 bash[51302]: debug 2023-08-28T00:06:48.159+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:48.165837+0000) 2023-08-28T00:06:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:48 smithi006 bash[33918]: cluster 2023-08-28T00:06:48.074388+0000 mgr.y (mgr.14901) 3847 : cluster [DBG] pgmap v3854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:48 smithi006 bash[27292]: cluster 2023-08-28T00:06:48.074388+0000 mgr.y (mgr.14901) 3847 : cluster [DBG] pgmap v3854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:48 smithi130 bash[27579]: cluster 2023-08-28T00:06:48.074388+0000 mgr.y (mgr.14901) 3847 : cluster [DBG] pgmap v3854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:49.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:49 smithi130 bash[51302]: debug 2023-08-28T00:06:49.163+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:49.166143+0000) 2023-08-28T00:06:49.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:49 smithi006 bash[33918]: audit 2023-08-28T00:06:49.576633+0000 mon.a (mon.0) 2936 : audit [DBG] from='client.? 172.21.15.6:0/1595502627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:49.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:49 smithi006 bash[27292]: audit 2023-08-28T00:06:49.576633+0000 mon.a (mon.0) 2936 : audit [DBG] from='client.? 172.21.15.6:0/1595502627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:50.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:49 smithi130 bash[27579]: audit 2023-08-28T00:06:49.576633+0000 mon.a (mon.0) 2936 : audit [DBG] from='client.? 172.21.15.6:0/1595502627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:50.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:50 smithi130 bash[51302]: debug 2023-08-28T00:06:50.163+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:50.166415+0000) 2023-08-28T00:06:50.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:50 smithi006 bash[27292]: cluster 2023-08-28T00:06:50.075534+0000 mgr.y (mgr.14901) 3848 : cluster [DBG] pgmap v3855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:06:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:06:50] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T00:06:50.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:50 smithi006 bash[33918]: cluster 2023-08-28T00:06:50.075534+0000 mgr.y (mgr.14901) 3848 : cluster [DBG] pgmap v3855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:06:50.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:06:50.652Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:50 smithi130 bash[27579]: cluster 2023-08-28T00:06:50.075534+0000 mgr.y (mgr.14901) 3848 : cluster [DBG] pgmap v3855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:51.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:51 smithi130 bash[51302]: debug 2023-08-28T00:06:51.159+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:51.166733+0000) 2023-08-28T00:06:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:52 smithi130 bash[27579]: audit 2023-08-28T00:06:51.995320+0000 mon.a (mon.0) 2937 : audit [DBG] from='client.? 172.21.15.6:0/3088999268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:52.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:52 smithi130 bash[51302]: debug 2023-08-28T00:06:52.163+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:52.167058+0000) 2023-08-28T00:06:52.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:52 smithi006 bash[27292]: audit 2023-08-28T00:06:51.995320+0000 mon.a (mon.0) 2937 : audit [DBG] from='client.? 172.21.15.6:0/3088999268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:52.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:52 smithi006 bash[33918]: audit 2023-08-28T00:06:51.995320+0000 mon.a (mon.0) 2937 : audit [DBG] from='client.? 172.21.15.6:0/3088999268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:06:52.220Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:52.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:06:52.222Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:53 smithi130 bash[27579]: cluster 2023-08-28T00:06:52.076206+0000 mgr.y (mgr.14901) 3849 : cluster [DBG] pgmap v3856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:53.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:53 smithi130 bash[51302]: debug 2023-08-28T00:06:53.163+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:53.167347+0000) 2023-08-28T00:06:53.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:53 smithi006 bash[27292]: cluster 2023-08-28T00:06:52.076206+0000 mgr.y (mgr.14901) 3849 : cluster [DBG] pgmap v3856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:53.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:53 smithi006 bash[33918]: cluster 2023-08-28T00:06:52.076206+0000 mgr.y (mgr.14901) 3849 : cluster [DBG] pgmap v3856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:06:54.107Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:06:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:06:54.109Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:06:54.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:54 smithi130 bash[51302]: debug 2023-08-28T00:06:54.163+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:54.167574+0000) 2023-08-28T00:06:55.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:55 smithi006 bash[27292]: cluster 2023-08-28T00:06:54.077048+0000 mgr.y (mgr.14901) 3850 : cluster [DBG] pgmap v3857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:55.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:55 smithi006 bash[27292]: audit 2023-08-28T00:06:54.422291+0000 mon.a (mon.0) 2938 : audit [DBG] from='client.? 172.21.15.6:0/1899285798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:55.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:55 smithi006 bash[33918]: cluster 2023-08-28T00:06:54.077048+0000 mgr.y (mgr.14901) 3850 : cluster [DBG] pgmap v3857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:55.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:55 smithi006 bash[33918]: audit 2023-08-28T00:06:54.422291+0000 mon.a (mon.0) 2938 : audit [DBG] from='client.? 172.21.15.6:0/1899285798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:55.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:55 smithi130 bash[51302]: debug 2023-08-28T00:06:55.163+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:55.167860+0000) 2023-08-28T00:06:55.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:55 smithi130 bash[27579]: cluster 2023-08-28T00:06:54.077048+0000 mgr.y (mgr.14901) 3850 : cluster [DBG] pgmap v3857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:55.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:55 smithi130 bash[27579]: audit 2023-08-28T00:06:54.422291+0000 mon.a (mon.0) 2938 : audit [DBG] from='client.? 172.21.15.6:0/1899285798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:56.446 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:56 smithi130 bash[51302]: debug 2023-08-28T00:06:56.163+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:56.168300+0000) 2023-08-28T00:06:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:56 smithi006 bash[33918]: cluster 2023-08-28T00:06:56.077835+0000 mgr.y (mgr.14901) 3851 : cluster [DBG] pgmap v3858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:56 smithi006 bash[27292]: cluster 2023-08-28T00:06:56.077835+0000 mgr.y (mgr.14901) 3851 : cluster [DBG] pgmap v3858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:56 smithi130 bash[27579]: cluster 2023-08-28T00:06:56.077835+0000 mgr.y (mgr.14901) 3851 : cluster [DBG] pgmap v3858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:57.449 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:57 smithi130 bash[51302]: debug 2023-08-28T00:06:57.167+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:57.168567+0000) 2023-08-28T00:06:57.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:57 smithi006 bash[33918]: audit 2023-08-28T00:06:56.840310+0000 mon.c (mon.2) 1459 : audit [DBG] from='client.? 172.21.15.6:0/1051791033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:57.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:57 smithi006 bash[27292]: audit 2023-08-28T00:06:56.840310+0000 mon.c (mon.2) 1459 : audit [DBG] from='client.? 172.21.15.6:0/1051791033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:57.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:57 smithi130 bash[27579]: audit 2023-08-28T00:06:56.840310+0000 mon.c (mon.2) 1459 : audit [DBG] from='client.? 172.21.15.6:0/1051791033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:58.458 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:58 smithi130 bash[51302]: debug 2023-08-28T00:06:58.167+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:58.168859+0000) 2023-08-28T00:06:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:58 smithi006 bash[33918]: cluster 2023-08-28T00:06:58.078448+0000 mgr.y (mgr.14901) 3852 : cluster [DBG] pgmap v3859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:58 smithi006 bash[27292]: cluster 2023-08-28T00:06:58.078448+0000 mgr.y (mgr.14901) 3852 : cluster [DBG] pgmap v3859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:58 smithi130 bash[27579]: cluster 2023-08-28T00:06:58.078448+0000 mgr.y (mgr.14901) 3852 : cluster [DBG] pgmap v3859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:06:59.460 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:06:59 smithi130 bash[51302]: debug 2023-08-28T00:06:59.167+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:59.169183+0000) 2023-08-28T00:06:59.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:06:59 smithi006 bash[27292]: audit 2023-08-28T00:06:59.261220+0000 mon.c (mon.2) 1460 : audit [DBG] from='client.? 172.21.15.6:0/4133009259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:59.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:06:59 smithi006 bash[33918]: audit 2023-08-28T00:06:59.261220+0000 mon.c (mon.2) 1460 : audit [DBG] from='client.? 172.21.15.6:0/4133009259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:06:59 smithi130 bash[27579]: audit 2023-08-28T00:06:59.261220+0000 mon.c (mon.2) 1460 : audit [DBG] from='client.? 172.21.15.6:0/4133009259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:00.471 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:00 smithi130 bash[51302]: debug 2023-08-28T00:07:00.163+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:00.169484+0000) 2023-08-28T00:07:00.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:00 smithi006 bash[27292]: cluster 2023-08-28T00:07:00.079506+0000 mgr.y (mgr.14901) 3853 : cluster [DBG] pgmap v3860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:00.756 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:00 smithi006 bash[33918]: cluster 2023-08-28T00:07:00.079506+0000 mgr.y (mgr.14901) 3853 : cluster [DBG] pgmap v3860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:00.756 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:07:00.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:00.756 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:07:00.652Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:00 smithi130 bash[27579]: cluster 2023-08-28T00:07:00.079506+0000 mgr.y (mgr.14901) 3853 : cluster [DBG] pgmap v3860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:07:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:07:00] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:07:01.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:01 smithi130 bash[51302]: debug 2023-08-28T00:07:01.163+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:01.169758+0000) 2023-08-28T00:07:02.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:01 smithi130 bash[27579]: audit 2023-08-28T00:07:01.669640+0000 mon.a (mon.0) 2939 : audit [DBG] from='client.? 172.21.15.6:0/2813283172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:02.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:01 smithi006 bash[33918]: audit 2023-08-28T00:07:01.669640+0000 mon.a (mon.0) 2939 : audit [DBG] from='client.? 172.21.15.6:0/2813283172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:02.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:01 smithi006 bash[27292]: audit 2023-08-28T00:07:01.669640+0000 mon.a (mon.0) 2939 : audit [DBG] from='client.? 172.21.15.6:0/2813283172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:02.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:02 smithi130 bash[51302]: debug 2023-08-28T00:07:02.163+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:02.170096+0000) 2023-08-28T00:07:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:07:02.221Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:07:02.223Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:02 smithi130 bash[27579]: cluster 2023-08-28T00:07:02.080174+0000 mgr.y (mgr.14901) 3854 : cluster [DBG] pgmap v3861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:02 smithi006 bash[33918]: cluster 2023-08-28T00:07:02.080174+0000 mgr.y (mgr.14901) 3854 : cluster [DBG] pgmap v3861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:02 smithi006 bash[27292]: cluster 2023-08-28T00:07:02.080174+0000 mgr.y (mgr.14901) 3854 : cluster [DBG] pgmap v3861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:03.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:03 smithi130 bash[51302]: debug 2023-08-28T00:07:03.163+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:03.170432+0000) 2023-08-28T00:07:04.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:04 smithi006 bash[27292]: audit 2023-08-28T00:07:04.080859+0000 mon.a (mon.0) 2940 : audit [DBG] from='client.? 172.21.15.6:0/3394106135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:04.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:04 smithi006 bash[33918]: audit 2023-08-28T00:07:04.080859+0000 mon.a (mon.0) 2940 : audit [DBG] from='client.? 172.21.15.6:0/3394106135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:07:04.108Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:07:04.110Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:04 smithi130 bash[27579]: audit 2023-08-28T00:07:04.080859+0000 mon.a (mon.0) 2940 : audit [DBG] from='client.? 172.21.15.6:0/3394106135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:04.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:04 smithi130 bash[51302]: debug 2023-08-28T00:07:04.163+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:04.170688+0000) 2023-08-28T00:07:05.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:05 smithi006 bash[27292]: cluster 2023-08-28T00:07:04.081130+0000 mgr.y (mgr.14901) 3855 : cluster [DBG] pgmap v3862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:05.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:05 smithi006 bash[33918]: cluster 2023-08-28T00:07:04.081130+0000 mgr.y (mgr.14901) 3855 : cluster [DBG] pgmap v3862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:05.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:05 smithi130 bash[27579]: cluster 2023-08-28T00:07:04.081130+0000 mgr.y (mgr.14901) 3855 : cluster [DBG] pgmap v3862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:05.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:05 smithi130 bash[51302]: debug 2023-08-28T00:07:05.167+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:05.170983+0000) 2023-08-28T00:07:06.439 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:06 smithi130 bash[51302]: debug 2023-08-28T00:07:06.167+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:06.171257+0000) 2023-08-28T00:07:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:06 smithi006 bash[27292]: cluster 2023-08-28T00:07:06.081820+0000 mgr.y (mgr.14901) 3856 : cluster [DBG] pgmap v3863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:06 smithi006 bash[33918]: cluster 2023-08-28T00:07:06.081820+0000 mgr.y (mgr.14901) 3856 : cluster [DBG] pgmap v3863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:06 smithi130 bash[27579]: cluster 2023-08-28T00:07:06.081820+0000 mgr.y (mgr.14901) 3856 : cluster [DBG] pgmap v3863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:07.442 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:07 smithi130 bash[51302]: debug 2023-08-28T00:07:07.171+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:07.171517+0000) 2023-08-28T00:07:07.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:07 smithi006 bash[27292]: audit 2023-08-28T00:07:06.491181+0000 mon.c (mon.2) 1461 : audit [DBG] from='client.? 172.21.15.6:0/2269057574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:07 smithi006 bash[33918]: audit 2023-08-28T00:07:06.491181+0000 mon.c (mon.2) 1461 : audit [DBG] from='client.? 172.21.15.6:0/2269057574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:07.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:07 smithi130 bash[27579]: audit 2023-08-28T00:07:06.491181+0000 mon.c (mon.2) 1461 : audit [DBG] from='client.? 172.21.15.6:0/2269057574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:08.455 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:08 smithi130 bash[51302]: debug 2023-08-28T00:07:08.171+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:08.171804+0000) 2023-08-28T00:07:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:08 smithi006 bash[27292]: cluster 2023-08-28T00:07:08.082449+0000 mgr.y (mgr.14901) 3857 : cluster [DBG] pgmap v3864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:08 smithi006 bash[33918]: cluster 2023-08-28T00:07:08.082449+0000 mgr.y (mgr.14901) 3857 : cluster [DBG] pgmap v3864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:08 smithi130 bash[27579]: cluster 2023-08-28T00:07:08.082449+0000 mgr.y (mgr.14901) 3857 : cluster [DBG] pgmap v3864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:09.454 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:09 smithi130 bash[51302]: debug 2023-08-28T00:07:09.167+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:09.172073+0000) 2023-08-28T00:07:09.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:09 smithi006 bash[33918]: audit 2023-08-28T00:07:08.904288+0000 mon.a (mon.0) 2941 : audit [DBG] from='client.? 172.21.15.6:0/177576144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:09 smithi006 bash[27292]: audit 2023-08-28T00:07:08.904288+0000 mon.a (mon.0) 2941 : audit [DBG] from='client.? 172.21.15.6:0/177576144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:09 smithi130 bash[27579]: audit 2023-08-28T00:07:08.904288+0000 mon.a (mon.0) 2941 : audit [DBG] from='client.? 172.21.15.6:0/177576144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:10.460 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:10 smithi130 bash[51302]: debug 2023-08-28T00:07:10.167+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:10.172411+0000) 2023-08-28T00:07:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:10 smithi006 bash[33918]: cluster 2023-08-28T00:07:10.083702+0000 mgr.y (mgr.14901) 3858 : cluster [DBG] pgmap v3865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:10 smithi006 bash[27292]: cluster 2023-08-28T00:07:10.083702+0000 mgr.y (mgr.14901) 3858 : cluster [DBG] pgmap v3865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:07:10.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:10.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:07:10.653Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:10 smithi130 bash[27579]: cluster 2023-08-28T00:07:10.083702+0000 mgr.y (mgr.14901) 3858 : cluster [DBG] pgmap v3865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:07:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:07:10] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:07:11.463 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:11 smithi130 bash[51302]: debug 2023-08-28T00:07:11.167+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:11.172806+0000) 2023-08-28T00:07:11.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:11 smithi006 bash[33918]: audit 2023-08-28T00:07:11.317011+0000 mon.c (mon.2) 1462 : audit [DBG] from='client.? 172.21.15.6:0/879907664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:11.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:11 smithi006 bash[27292]: audit 2023-08-28T00:07:11.317011+0000 mon.c (mon.2) 1462 : audit [DBG] from='client.? 172.21.15.6:0/879907664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:11.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:11 smithi130 bash[27579]: audit 2023-08-28T00:07:11.317011+0000 mon.c (mon.2) 1462 : audit [DBG] from='client.? 172.21.15.6:0/879907664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:12.470 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:12 smithi130 bash[51302]: debug 2023-08-28T00:07:12.167+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:12.173162+0000) 2023-08-28T00:07:12.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:12 smithi006 bash[27292]: cluster 2023-08-28T00:07:12.084532+0000 mgr.y (mgr.14901) 3859 : cluster [DBG] pgmap v3866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:12.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:12 smithi006 bash[33918]: cluster 2023-08-28T00:07:12.084532+0000 mgr.y (mgr.14901) 3859 : cluster [DBG] pgmap v3866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:07:12.222Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:07:12.223Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:12 smithi130 bash[27579]: cluster 2023-08-28T00:07:12.084532+0000 mgr.y (mgr.14901) 3859 : cluster [DBG] pgmap v3866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:13.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:13 smithi130 bash[51302]: debug 2023-08-28T00:07:13.171+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:13.173486+0000) 2023-08-28T00:07:14.112 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:13 smithi006 bash[33918]: audit 2023-08-28T00:07:13.730138+0000 mon.a (mon.0) 2942 : audit [DBG] from='client.? 172.21.15.6:0/2256044488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:14.112 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:13 smithi006 bash[27292]: audit 2023-08-28T00:07:13.730138+0000 mon.a (mon.0) 2942 : audit [DBG] from='client.? 172.21.15.6:0/2256044488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:14.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:13 smithi130 bash[27579]: audit 2023-08-28T00:07:13.730138+0000 mon.a (mon.0) 2942 : audit [DBG] from='client.? 172.21.15.6:0/2256044488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:07:14.109Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:07:14.111Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:14.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:14 smithi130 bash[51302]: debug 2023-08-28T00:07:14.167+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:14.173793+0000) 2023-08-28T00:07:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:14 smithi130 bash[27579]: cluster 2023-08-28T00:07:14.085644+0000 mgr.y (mgr.14901) 3860 : cluster [DBG] pgmap v3867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:14 smithi006 bash[27292]: cluster 2023-08-28T00:07:14.085644+0000 mgr.y (mgr.14901) 3860 : cluster [DBG] pgmap v3867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:14 smithi006 bash[33918]: cluster 2023-08-28T00:07:14.085644+0000 mgr.y (mgr.14901) 3860 : cluster [DBG] pgmap v3867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:15.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:15 smithi130 bash[51302]: debug 2023-08-28T00:07:15.171+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:15.174142+0000) 2023-08-28T00:07:16.442 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:16 smithi130 bash[51302]: debug 2023-08-28T00:07:16.171+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:16.174463+0000) 2023-08-28T00:07:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:16 smithi006 bash[33918]: cluster 2023-08-28T00:07:16.086386+0000 mgr.y (mgr.14901) 3861 : cluster [DBG] pgmap v3868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:16 smithi006 bash[33918]: audit 2023-08-28T00:07:16.152502+0000 mon.a (mon.0) 2943 : audit [DBG] from='client.? 172.21.15.6:0/1071500776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:16 smithi006 bash[27292]: cluster 2023-08-28T00:07:16.086386+0000 mgr.y (mgr.14901) 3861 : cluster [DBG] pgmap v3868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:16 smithi006 bash[27292]: audit 2023-08-28T00:07:16.152502+0000 mon.a (mon.0) 2943 : audit [DBG] from='client.? 172.21.15.6:0/1071500776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:16 smithi130 bash[27579]: cluster 2023-08-28T00:07:16.086386+0000 mgr.y (mgr.14901) 3861 : cluster [DBG] pgmap v3868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:16 smithi130 bash[27579]: audit 2023-08-28T00:07:16.152502+0000 mon.a (mon.0) 2943 : audit [DBG] from='client.? 172.21.15.6:0/1071500776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:17.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:17 smithi130 bash[51302]: debug 2023-08-28T00:07:17.171+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:17.174776+0000) 2023-08-28T00:07:18.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:18 smithi130 bash[51302]: debug 2023-08-28T00:07:18.171+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:18.175138+0000) 2023-08-28T00:07:19.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:19 smithi006 bash[33918]: cluster 2023-08-28T00:07:18.087036+0000 mgr.y (mgr.14901) 3862 : cluster [DBG] pgmap v3869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:19.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:19 smithi006 bash[33918]: audit 2023-08-28T00:07:18.569290+0000 mon.a (mon.0) 2944 : audit [DBG] from='client.? 172.21.15.6:0/3128794741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:19.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:19 smithi006 bash[27292]: cluster 2023-08-28T00:07:18.087036+0000 mgr.y (mgr.14901) 3862 : cluster [DBG] pgmap v3869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:19.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:19 smithi006 bash[27292]: audit 2023-08-28T00:07:18.569290+0000 mon.a (mon.0) 2944 : audit [DBG] from='client.? 172.21.15.6:0/3128794741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:19.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:19 smithi130 bash[51302]: debug 2023-08-28T00:07:19.170+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:19.175512+0000) 2023-08-28T00:07:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:19 smithi130 bash[27579]: cluster 2023-08-28T00:07:18.087036+0000 mgr.y (mgr.14901) 3862 : cluster [DBG] pgmap v3869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:19.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:19 smithi130 bash[27579]: audit 2023-08-28T00:07:18.569290+0000 mon.a (mon.0) 2944 : audit [DBG] from='client.? 172.21.15.6:0/3128794741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:20.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:20 smithi130 bash[51302]: debug 2023-08-28T00:07:20.174+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:20.175872+0000) 2023-08-28T00:07:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:07:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:07:20] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:07:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:07:20.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:07:20.653Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:21.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:21 smithi006 bash[33918]: cluster 2023-08-28T00:07:20.088187+0000 mgr.y (mgr.14901) 3863 : cluster [DBG] pgmap v3870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:21.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:21 smithi006 bash[33918]: audit 2023-08-28T00:07:20.981417+0000 mon.a (mon.0) 2945 : audit [DBG] from='client.? 172.21.15.6:0/2639372988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:21.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:21 smithi006 bash[27292]: cluster 2023-08-28T00:07:20.088187+0000 mgr.y (mgr.14901) 3863 : cluster [DBG] pgmap v3870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:21.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:21 smithi006 bash[27292]: audit 2023-08-28T00:07:20.981417+0000 mon.a (mon.0) 2945 : audit [DBG] from='client.? 172.21.15.6:0/2639372988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:21 smithi130 bash[27579]: cluster 2023-08-28T00:07:20.088187+0000 mgr.y (mgr.14901) 3863 : cluster [DBG] pgmap v3870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:21 smithi130 bash[27579]: audit 2023-08-28T00:07:20.981417+0000 mon.a (mon.0) 2945 : audit [DBG] from='client.? 172.21.15.6:0/2639372988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:21.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:21 smithi130 bash[51302]: debug 2023-08-28T00:07:21.174+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:21.176239+0000) 2023-08-28T00:07:22.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:22 smithi130 bash[51302]: debug 2023-08-28T00:07:22.170+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:22.176514+0000) 2023-08-28T00:07:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:07:22.223Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:07:22.225Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:23.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:23 smithi006 bash[27292]: cluster 2023-08-28T00:07:22.088888+0000 mgr.y (mgr.14901) 3864 : cluster [DBG] pgmap v3871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:23.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:23 smithi006 bash[33918]: cluster 2023-08-28T00:07:22.088888+0000 mgr.y (mgr.14901) 3864 : cluster [DBG] pgmap v3871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:23.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:23 smithi130 bash[27579]: cluster 2023-08-28T00:07:22.088888+0000 mgr.y (mgr.14901) 3864 : cluster [DBG] pgmap v3871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:23.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:23 smithi130 bash[51302]: debug 2023-08-28T00:07:23.170+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:23.176872+0000) 2023-08-28T00:07:24.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:24 smithi006 bash[27292]: audit 2023-08-28T00:07:23.400554+0000 mon.c (mon.2) 1463 : audit [DBG] from='client.? 172.21.15.6:0/1291925642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:24.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:24 smithi006 bash[33918]: audit 2023-08-28T00:07:23.400554+0000 mon.c (mon.2) 1463 : audit [DBG] from='client.? 172.21.15.6:0/1291925642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:07:24.109Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:07:24.111Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:24 smithi130 bash[27579]: audit 2023-08-28T00:07:23.400554+0000 mon.c (mon.2) 1463 : audit [DBG] from='client.? 172.21.15.6:0/1291925642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:24.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:24 smithi130 bash[51302]: debug 2023-08-28T00:07:24.170+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:24.177144+0000) 2023-08-28T00:07:25.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:25 smithi006 bash[27292]: cluster 2023-08-28T00:07:24.090047+0000 mgr.y (mgr.14901) 3865 : cluster [DBG] pgmap v3872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:25.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:25 smithi006 bash[33918]: cluster 2023-08-28T00:07:24.090047+0000 mgr.y (mgr.14901) 3865 : cluster [DBG] pgmap v3872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:25.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:25 smithi130 bash[27579]: cluster 2023-08-28T00:07:24.090047+0000 mgr.y (mgr.14901) 3865 : cluster [DBG] pgmap v3872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:25.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:25 smithi130 bash[51302]: debug 2023-08-28T00:07:25.170+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:25.177403+0000) 2023-08-28T00:07:26.448 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:26 smithi130 bash[51302]: debug 2023-08-28T00:07:26.170+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:26.177674+0000) 2023-08-28T00:07:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:26 smithi006 bash[27292]: audit 2023-08-28T00:07:25.601815+0000 mon.b (mon.1) 484 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:07:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:26 smithi006 bash[27292]: audit 2023-08-28T00:07:25.817596+0000 mon.c (mon.2) 1464 : audit [DBG] from='client.? 172.21.15.6:0/493097847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:26 smithi006 bash[27292]: cluster 2023-08-28T00:07:26.090997+0000 mgr.y (mgr.14901) 3866 : cluster [DBG] pgmap v3873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:26 smithi006 bash[33918]: audit 2023-08-28T00:07:25.601815+0000 mon.b (mon.1) 484 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:07:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:26 smithi006 bash[33918]: audit 2023-08-28T00:07:25.817596+0000 mon.c (mon.2) 1464 : audit [DBG] from='client.? 172.21.15.6:0/493097847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:26 smithi006 bash[33918]: cluster 2023-08-28T00:07:26.090997+0000 mgr.y (mgr.14901) 3866 : cluster [DBG] pgmap v3873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:26 smithi130 bash[27579]: audit 2023-08-28T00:07:25.601815+0000 mon.b (mon.1) 484 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:07:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:26 smithi130 bash[27579]: audit 2023-08-28T00:07:25.817596+0000 mon.c (mon.2) 1464 : audit [DBG] from='client.? 172.21.15.6:0/493097847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:26 smithi130 bash[27579]: cluster 2023-08-28T00:07:26.090997+0000 mgr.y (mgr.14901) 3866 : cluster [DBG] pgmap v3873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:27.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:27 smithi130 bash[51302]: debug 2023-08-28T00:07:27.174+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:27.177999+0000) 2023-08-28T00:07:28.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:28 smithi130 bash[51302]: debug 2023-08-28T00:07:28.174+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:28.178339+0000) 2023-08-28T00:07:29.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:29 smithi006 bash[27292]: cluster 2023-08-28T00:07:28.091666+0000 mgr.y (mgr.14901) 3867 : cluster [DBG] pgmap v3874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:29.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:29 smithi006 bash[27292]: audit 2023-08-28T00:07:28.247110+0000 mon.a (mon.0) 2946 : audit [DBG] from='client.? 172.21.15.6:0/85859303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:29.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:29 smithi006 bash[33918]: cluster 2023-08-28T00:07:28.091666+0000 mgr.y (mgr.14901) 3867 : cluster [DBG] pgmap v3874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:29.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:29 smithi006 bash[33918]: audit 2023-08-28T00:07:28.247110+0000 mon.a (mon.0) 2946 : audit [DBG] from='client.? 172.21.15.6:0/85859303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:29.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:29 smithi130 bash[51302]: debug 2023-08-28T00:07:29.174+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:29.178691+0000) 2023-08-28T00:07:29.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:29 smithi130 bash[27579]: cluster 2023-08-28T00:07:28.091666+0000 mgr.y (mgr.14901) 3867 : cluster [DBG] pgmap v3874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:29.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:29 smithi130 bash[27579]: audit 2023-08-28T00:07:28.247110+0000 mon.a (mon.0) 2946 : audit [DBG] from='client.? 172.21.15.6:0/85859303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:30.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:30 smithi130 bash[51302]: debug 2023-08-28T00:07:30.178+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:30.179050+0000) 2023-08-28T00:07:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:07:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:07:30] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:07:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:07:30.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:07:30.653Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:31.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:31 smithi006 bash[27292]: cluster 2023-08-28T00:07:30.092821+0000 mgr.y (mgr.14901) 3868 : cluster [DBG] pgmap v3875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:31.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:31 smithi006 bash[27292]: audit 2023-08-28T00:07:30.680755+0000 mon.c (mon.2) 1465 : audit [DBG] from='client.? 172.21.15.6:0/3390963294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:31.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:31 smithi006 bash[33918]: cluster 2023-08-28T00:07:30.092821+0000 mgr.y (mgr.14901) 3868 : cluster [DBG] pgmap v3875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:31.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:31 smithi006 bash[33918]: audit 2023-08-28T00:07:30.680755+0000 mon.c (mon.2) 1465 : audit [DBG] from='client.? 172.21.15.6:0/3390963294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:31.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:31 smithi130 bash[27579]: cluster 2023-08-28T00:07:30.092821+0000 mgr.y (mgr.14901) 3868 : cluster [DBG] pgmap v3875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:31.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:31 smithi130 bash[27579]: audit 2023-08-28T00:07:30.680755+0000 mon.c (mon.2) 1465 : audit [DBG] from='client.? 172.21.15.6:0/3390963294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:31.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:31 smithi130 bash[51302]: debug 2023-08-28T00:07:31.178+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:31.179262+0000) 2023-08-28T00:07:32.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:32 smithi130 bash[51302]: debug 2023-08-28T00:07:32.178+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:32.179499+0000) 2023-08-28T00:07:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:32 smithi130 bash[27579]: audit 2023-08-28T00:07:31.414262+0000 mon.a (mon.0) 2947 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:07:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:32 smithi130 bash[27579]: audit 2023-08-28T00:07:31.919533+0000 mon.a (mon.0) 2948 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:07:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:32 smithi130 bash[27579]: cluster 2023-08-28T00:07:32.093473+0000 mgr.y (mgr.14901) 3869 : cluster [DBG] pgmap v3876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:32 smithi130 bash[27579]: audit 2023-08-28T00:07:32.185779+0000 mon.b (mon.1) 485 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:07:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:32 smithi130 bash[27579]: audit 2023-08-28T00:07:32.188012+0000 mon.b (mon.1) 486 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:07:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:32 smithi130 bash[27579]: audit 2023-08-28T00:07:32.196503+0000 mon.a (mon.0) 2949 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:07:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:07:32.223Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:07:32.225Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:32.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:32 smithi006 bash[33918]: audit 2023-08-28T00:07:31.414262+0000 mon.a (mon.0) 2947 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:07:32.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:32 smithi006 bash[33918]: audit 2023-08-28T00:07:31.919533+0000 mon.a (mon.0) 2948 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:07:32.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:32 smithi006 bash[33918]: cluster 2023-08-28T00:07:32.093473+0000 mgr.y (mgr.14901) 3869 : cluster [DBG] pgmap v3876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:32.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:32 smithi006 bash[33918]: audit 2023-08-28T00:07:32.185779+0000 mon.b (mon.1) 485 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:07:32.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:32 smithi006 bash[33918]: audit 2023-08-28T00:07:32.188012+0000 mon.b (mon.1) 486 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:07:32.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:32 smithi006 bash[33918]: audit 2023-08-28T00:07:32.196503+0000 mon.a (mon.0) 2949 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:07:32.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:32 smithi006 bash[27292]: audit 2023-08-28T00:07:31.414262+0000 mon.a (mon.0) 2947 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:07:32.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:32 smithi006 bash[27292]: audit 2023-08-28T00:07:31.919533+0000 mon.a (mon.0) 2948 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:07:32.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:32 smithi006 bash[27292]: cluster 2023-08-28T00:07:32.093473+0000 mgr.y (mgr.14901) 3869 : cluster [DBG] pgmap v3876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:32.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:32 smithi006 bash[27292]: audit 2023-08-28T00:07:32.185779+0000 mon.b (mon.1) 485 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:07:32.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:32 smithi006 bash[27292]: audit 2023-08-28T00:07:32.188012+0000 mon.b (mon.1) 486 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:07:32.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:32 smithi006 bash[27292]: audit 2023-08-28T00:07:32.196503+0000 mon.a (mon.0) 2949 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:07:33.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:33 smithi130 bash[51302]: debug 2023-08-28T00:07:33.178+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:33.179764+0000) 2023-08-28T00:07:34.181 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:33 smithi130 bash[27579]: audit 2023-08-28T00:07:33.103399+0000 mon.c (mon.2) 1466 : audit [DBG] from='client.? 172.21.15.6:0/349774732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:34.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:33 smithi006 bash[33918]: audit 2023-08-28T00:07:33.103399+0000 mon.c (mon.2) 1466 : audit [DBG] from='client.? 172.21.15.6:0/349774732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:34.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:33 smithi006 bash[27292]: audit 2023-08-28T00:07:33.103399+0000 mon.c (mon.2) 1466 : audit [DBG] from='client.? 172.21.15.6:0/349774732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:07:34.110Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:07:34.112Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:34.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:34 smithi130 bash[51302]: debug 2023-08-28T00:07:34.178+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:34.179989+0000) 2023-08-28T00:07:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:34 smithi006 bash[27292]: cluster 2023-08-28T00:07:34.094648+0000 mgr.y (mgr.14901) 3870 : cluster [DBG] pgmap v3877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:34 smithi006 bash[33918]: cluster 2023-08-28T00:07:34.094648+0000 mgr.y (mgr.14901) 3870 : cluster [DBG] pgmap v3877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:35.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:35 smithi130 bash[51302]: debug 2023-08-28T00:07:35.178+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:35.180279+0000) 2023-08-28T00:07:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:34 smithi130 bash[27579]: cluster 2023-08-28T00:07:34.094648+0000 mgr.y (mgr.14901) 3870 : cluster [DBG] pgmap v3877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:36.441 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:36 smithi130 bash[51302]: debug 2023-08-28T00:07:36.178+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:36.180557+0000) 2023-08-28T00:07:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:36 smithi006 bash[33918]: audit 2023-08-28T00:07:35.526121+0000 mon.c (mon.2) 1467 : audit [DBG] from='client.? 172.21.15.6:0/1591517634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:36 smithi006 bash[33918]: cluster 2023-08-28T00:07:36.095389+0000 mgr.y (mgr.14901) 3871 : cluster [DBG] pgmap v3878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:36 smithi006 bash[27292]: audit 2023-08-28T00:07:35.526121+0000 mon.c (mon.2) 1467 : audit [DBG] from='client.? 172.21.15.6:0/1591517634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:36 smithi006 bash[27292]: cluster 2023-08-28T00:07:36.095389+0000 mgr.y (mgr.14901) 3871 : cluster [DBG] pgmap v3878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:36 smithi130 bash[27579]: audit 2023-08-28T00:07:35.526121+0000 mon.c (mon.2) 1467 : audit [DBG] from='client.? 172.21.15.6:0/1591517634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:36 smithi130 bash[27579]: cluster 2023-08-28T00:07:36.095389+0000 mgr.y (mgr.14901) 3871 : cluster [DBG] pgmap v3878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:37.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:37 smithi130 bash[51302]: debug 2023-08-28T00:07:37.178+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:37.180819+0000) 2023-08-28T00:07:38.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:37 smithi130 bash[27579]: audit 2023-08-28T00:07:37.937579+0000 mon.a (mon.0) 2950 : audit [DBG] from='client.? 172.21.15.6:0/3513437019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:38.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:38 smithi130 bash[51302]: debug 2023-08-28T00:07:38.178+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:38.181159+0000) 2023-08-28T00:07:38.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:37 smithi006 bash[33918]: audit 2023-08-28T00:07:37.937579+0000 mon.a (mon.0) 2950 : audit [DBG] from='client.? 172.21.15.6:0/3513437019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:38.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:37 smithi006 bash[27292]: audit 2023-08-28T00:07:37.937579+0000 mon.a (mon.0) 2950 : audit [DBG] from='client.? 172.21.15.6:0/3513437019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:39 smithi130 bash[27579]: cluster 2023-08-28T00:07:38.096066+0000 mgr.y (mgr.14901) 3872 : cluster [DBG] pgmap v3879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:39.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:39 smithi130 bash[51302]: debug 2023-08-28T00:07:39.178+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:39.181453+0000) 2023-08-28T00:07:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:38 smithi006 bash[27292]: cluster 2023-08-28T00:07:38.096066+0000 mgr.y (mgr.14901) 3872 : cluster [DBG] pgmap v3879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:39.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:38 smithi006 bash[33918]: cluster 2023-08-28T00:07:38.096066+0000 mgr.y (mgr.14901) 3872 : cluster [DBG] pgmap v3879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:40.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:40 smithi130 bash[51302]: debug 2023-08-28T00:07:40.178+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:40.181737+0000) 2023-08-28T00:07:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:07:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:07:40] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:07:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:07:40.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:07:40.654Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:41.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:41 smithi006 bash[33918]: cluster 2023-08-28T00:07:40.097068+0000 mgr.y (mgr.14901) 3873 : cluster [DBG] pgmap v3880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:41.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:41 smithi006 bash[33918]: audit 2023-08-28T00:07:40.368198+0000 mon.a (mon.0) 2951 : audit [DBG] from='client.? 172.21.15.6:0/3955063797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:41.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:41 smithi006 bash[27292]: cluster 2023-08-28T00:07:40.097068+0000 mgr.y (mgr.14901) 3873 : cluster [DBG] pgmap v3880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:41.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:41 smithi006 bash[27292]: audit 2023-08-28T00:07:40.368198+0000 mon.a (mon.0) 2951 : audit [DBG] from='client.? 172.21.15.6:0/3955063797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:41.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:41 smithi130 bash[51302]: debug 2023-08-28T00:07:41.178+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:41.182053+0000) 2023-08-28T00:07:41.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:41 smithi130 bash[27579]: cluster 2023-08-28T00:07:40.097068+0000 mgr.y (mgr.14901) 3873 : cluster [DBG] pgmap v3880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:41.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:41 smithi130 bash[27579]: audit 2023-08-28T00:07:40.368198+0000 mon.a (mon.0) 2951 : audit [DBG] from='client.? 172.21.15.6:0/3955063797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:42.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:42 smithi130 bash[51302]: debug 2023-08-28T00:07:42.178+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:42.182251+0000) 2023-08-28T00:07:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:07:42.224Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:07:42.226Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:43.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:43 smithi006 bash[27292]: cluster 2023-08-28T00:07:42.097837+0000 mgr.y (mgr.14901) 3874 : cluster [DBG] pgmap v3881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:43.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:43 smithi006 bash[27292]: audit 2023-08-28T00:07:42.779671+0000 mon.a (mon.0) 2952 : audit [DBG] from='client.? 172.21.15.6:0/853502526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:43.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:43 smithi006 bash[33918]: cluster 2023-08-28T00:07:42.097837+0000 mgr.y (mgr.14901) 3874 : cluster [DBG] pgmap v3881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:43.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:43 smithi006 bash[33918]: audit 2023-08-28T00:07:42.779671+0000 mon.a (mon.0) 2952 : audit [DBG] from='client.? 172.21.15.6:0/853502526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:43 smithi130 bash[27579]: cluster 2023-08-28T00:07:42.097837+0000 mgr.y (mgr.14901) 3874 : cluster [DBG] pgmap v3881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:43 smithi130 bash[27579]: audit 2023-08-28T00:07:42.779671+0000 mon.a (mon.0) 2952 : audit [DBG] from='client.? 172.21.15.6:0/853502526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:43.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:43 smithi130 bash[51302]: debug 2023-08-28T00:07:43.178+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:43.182511+0000) 2023-08-28T00:07:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:07:44.111Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:07:44.113Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:44.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:44 smithi130 bash[51302]: debug 2023-08-28T00:07:44.182+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:44.182786+0000) 2023-08-28T00:07:45.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:45 smithi006 bash[33918]: cluster 2023-08-28T00:07:44.098921+0000 mgr.y (mgr.14901) 3875 : cluster [DBG] pgmap v3882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:45.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:45 smithi006 bash[27292]: cluster 2023-08-28T00:07:44.098921+0000 mgr.y (mgr.14901) 3875 : cluster [DBG] pgmap v3882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:45.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:45 smithi130 bash[27579]: cluster 2023-08-28T00:07:44.098921+0000 mgr.y (mgr.14901) 3875 : cluster [DBG] pgmap v3882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:45.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:45 smithi130 bash[51302]: debug 2023-08-28T00:07:45.182+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:45.183038+0000) 2023-08-28T00:07:46.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:46 smithi006 bash[27292]: audit 2023-08-28T00:07:45.191776+0000 mon.a (mon.0) 2953 : audit [DBG] from='client.? 172.21.15.6:0/1857699574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:46 smithi006 bash[33918]: audit 2023-08-28T00:07:45.191776+0000 mon.a (mon.0) 2953 : audit [DBG] from='client.? 172.21.15.6:0/1857699574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:46 smithi130 bash[27579]: audit 2023-08-28T00:07:45.191776+0000 mon.a (mon.0) 2953 : audit [DBG] from='client.? 172.21.15.6:0/1857699574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:46.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:46 smithi130 bash[51302]: debug 2023-08-28T00:07:46.182+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:46.183256+0000) 2023-08-28T00:07:47.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:47 smithi006 bash[33918]: cluster 2023-08-28T00:07:46.099694+0000 mgr.y (mgr.14901) 3876 : cluster [DBG] pgmap v3883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:47.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:47 smithi006 bash[27292]: cluster 2023-08-28T00:07:46.099694+0000 mgr.y (mgr.14901) 3876 : cluster [DBG] pgmap v3883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:47.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:47 smithi130 bash[27579]: cluster 2023-08-28T00:07:46.099694+0000 mgr.y (mgr.14901) 3876 : cluster [DBG] pgmap v3883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:47.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:47 smithi130 bash[51302]: debug 2023-08-28T00:07:47.182+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:47.183507+0000) 2023-08-28T00:07:48.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:48 smithi006 bash[33918]: audit 2023-08-28T00:07:47.602565+0000 mon.c (mon.2) 1468 : audit [DBG] from='client.? 172.21.15.6:0/3767162795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:48.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:48 smithi006 bash[27292]: audit 2023-08-28T00:07:47.602565+0000 mon.c (mon.2) 1468 : audit [DBG] from='client.? 172.21.15.6:0/3767162795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:48 smithi130 bash[27579]: audit 2023-08-28T00:07:47.602565+0000 mon.c (mon.2) 1468 : audit [DBG] from='client.? 172.21.15.6:0/3767162795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:48.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:48 smithi130 bash[51302]: debug 2023-08-28T00:07:48.182+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:48.183723+0000) 2023-08-28T00:07:49.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:49 smithi006 bash[27292]: cluster 2023-08-28T00:07:48.100357+0000 mgr.y (mgr.14901) 3877 : cluster [DBG] pgmap v3884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:49.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:49 smithi006 bash[33918]: cluster 2023-08-28T00:07:48.100357+0000 mgr.y (mgr.14901) 3877 : cluster [DBG] pgmap v3884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:49.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:49 smithi130 bash[27579]: cluster 2023-08-28T00:07:48.100357+0000 mgr.y (mgr.14901) 3877 : cluster [DBG] pgmap v3884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:49.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:49 smithi130 bash[51302]: debug 2023-08-28T00:07:49.182+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:49.184019+0000) 2023-08-28T00:07:50.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:50 smithi006 bash[27292]: audit 2023-08-28T00:07:50.015660+0000 mon.a (mon.0) 2954 : audit [DBG] from='client.? 172.21.15.6:0/1397612124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:50.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:50 smithi006 bash[33918]: audit 2023-08-28T00:07:50.015660+0000 mon.a (mon.0) 2954 : audit [DBG] from='client.? 172.21.15.6:0/1397612124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:50 smithi130 bash[27579]: audit 2023-08-28T00:07:50.015660+0000 mon.a (mon.0) 2954 : audit [DBG] from='client.? 172.21.15.6:0/1397612124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:50.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:50 smithi130 bash[51302]: debug 2023-08-28T00:07:50.182+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:50.184203+0000) 2023-08-28T00:07:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:07:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:07:50] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T00:07:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:07:50.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:07:50.654Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:51.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:51 smithi006 bash[27292]: cluster 2023-08-28T00:07:50.101482+0000 mgr.y (mgr.14901) 3878 : cluster [DBG] pgmap v3885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:51.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:51 smithi006 bash[33918]: cluster 2023-08-28T00:07:50.101482+0000 mgr.y (mgr.14901) 3878 : cluster [DBG] pgmap v3885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:51.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:51 smithi130 bash[27579]: cluster 2023-08-28T00:07:50.101482+0000 mgr.y (mgr.14901) 3878 : cluster [DBG] pgmap v3885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:51.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:51 smithi130 bash[51302]: debug 2023-08-28T00:07:51.182+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:51.184510+0000) 2023-08-28T00:07:52.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:52 smithi130 bash[51302]: debug 2023-08-28T00:07:52.182+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:52.184829+0000) 2023-08-28T00:07:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:07:52.224Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:07:52.226Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:53.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:53 smithi006 bash[33918]: cluster 2023-08-28T00:07:52.102132+0000 mgr.y (mgr.14901) 3879 : cluster [DBG] pgmap v3886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:53.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:53 smithi006 bash[33918]: audit 2023-08-28T00:07:52.427777+0000 mon.c (mon.2) 1469 : audit [DBG] from='client.? 172.21.15.6:0/3936356159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:53 smithi006 bash[27292]: cluster 2023-08-28T00:07:52.102132+0000 mgr.y (mgr.14901) 3879 : cluster [DBG] pgmap v3886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:53 smithi006 bash[27292]: audit 2023-08-28T00:07:52.427777+0000 mon.c (mon.2) 1469 : audit [DBG] from='client.? 172.21.15.6:0/3936356159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:53.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:53 smithi130 bash[51302]: debug 2023-08-28T00:07:53.182+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:53.185107+0000) 2023-08-28T00:07:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:53 smithi130 bash[27579]: cluster 2023-08-28T00:07:52.102132+0000 mgr.y (mgr.14901) 3879 : cluster [DBG] pgmap v3886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:53 smithi130 bash[27579]: audit 2023-08-28T00:07:52.427777+0000 mon.c (mon.2) 1469 : audit [DBG] from='client.? 172.21.15.6:0/3936356159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:07:54.111Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:07:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:07:54.114Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:07:54.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:54 smithi130 bash[51302]: debug 2023-08-28T00:07:54.182+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:54.185418+0000) 2023-08-28T00:07:55.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:55 smithi006 bash[27292]: cluster 2023-08-28T00:07:54.103120+0000 mgr.y (mgr.14901) 3880 : cluster [DBG] pgmap v3887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:55.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:55 smithi006 bash[27292]: audit 2023-08-28T00:07:54.836371+0000 mon.a (mon.0) 2955 : audit [DBG] from='client.? 172.21.15.6:0/1981671805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:55.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:55 smithi006 bash[33918]: cluster 2023-08-28T00:07:54.103120+0000 mgr.y (mgr.14901) 3880 : cluster [DBG] pgmap v3887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:55.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:55 smithi006 bash[33918]: audit 2023-08-28T00:07:54.836371+0000 mon.a (mon.0) 2955 : audit [DBG] from='client.? 172.21.15.6:0/1981671805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:55.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:55 smithi130 bash[51302]: debug 2023-08-28T00:07:55.182+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:55.185723+0000) 2023-08-28T00:07:55.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:55 smithi130 bash[27579]: cluster 2023-08-28T00:07:54.103120+0000 mgr.y (mgr.14901) 3880 : cluster [DBG] pgmap v3887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:55.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:55 smithi130 bash[27579]: audit 2023-08-28T00:07:54.836371+0000 mon.a (mon.0) 2955 : audit [DBG] from='client.? 172.21.15.6:0/1981671805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:56.466 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:56 smithi130 bash[51302]: debug 2023-08-28T00:07:56.182+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:56.186039+0000) 2023-08-28T00:07:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:56 smithi006 bash[33918]: cluster 2023-08-28T00:07:56.104000+0000 mgr.y (mgr.14901) 3881 : cluster [DBG] pgmap v3888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:56 smithi006 bash[27292]: cluster 2023-08-28T00:07:56.104000+0000 mgr.y (mgr.14901) 3881 : cluster [DBG] pgmap v3888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:56 smithi130 bash[27579]: cluster 2023-08-28T00:07:56.104000+0000 mgr.y (mgr.14901) 3881 : cluster [DBG] pgmap v3888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:57.470 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:57 smithi130 bash[51302]: debug 2023-08-28T00:07:57.182+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:57.186340+0000) 2023-08-28T00:07:57.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:57 smithi130 bash[27579]: audit 2023-08-28T00:07:57.242716+0000 mon.c (mon.2) 1470 : audit [DBG] from='client.? 172.21.15.6:0/1888514358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:57.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:57 smithi006 bash[33918]: audit 2023-08-28T00:07:57.242716+0000 mon.c (mon.2) 1470 : audit [DBG] from='client.? 172.21.15.6:0/1888514358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:57.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:57 smithi006 bash[27292]: audit 2023-08-28T00:07:57.242716+0000 mon.c (mon.2) 1470 : audit [DBG] from='client.? 172.21.15.6:0/1888514358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:58.478 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:58 smithi130 bash[51302]: debug 2023-08-28T00:07:58.182+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:58.186651+0000) 2023-08-28T00:07:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:58 smithi130 bash[27579]: cluster 2023-08-28T00:07:58.104883+0000 mgr.y (mgr.14901) 3882 : cluster [DBG] pgmap v3889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:58 smithi006 bash[27292]: cluster 2023-08-28T00:07:58.104883+0000 mgr.y (mgr.14901) 3882 : cluster [DBG] pgmap v3889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:58 smithi006 bash[33918]: cluster 2023-08-28T00:07:58.104883+0000 mgr.y (mgr.14901) 3882 : cluster [DBG] pgmap v3889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:07:59.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:07:59 smithi130 bash[51302]: debug 2023-08-28T00:07:59.186+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:59.186962+0000) 2023-08-28T00:07:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:07:59 smithi006 bash[33918]: audit 2023-08-28T00:07:59.654721+0000 mon.c (mon.2) 1471 : audit [DBG] from='client.? 172.21.15.6:0/1900665430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:59.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:07:59 smithi006 bash[27292]: audit 2023-08-28T00:07:59.654721+0000 mon.c (mon.2) 1471 : audit [DBG] from='client.? 172.21.15.6:0/1900665430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:07:59 smithi130 bash[27579]: audit 2023-08-28T00:07:59.654721+0000 mon.c (mon.2) 1471 : audit [DBG] from='client.? 172.21.15.6:0/1900665430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:00.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:00 smithi130 bash[51302]: debug 2023-08-28T00:08:00.186+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:00.187184+0000) 2023-08-28T00:08:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:08:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:08:00] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:08:00.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:00 smithi006 bash[27292]: cluster 2023-08-28T00:08:00.106081+0000 mgr.y (mgr.14901) 3883 : cluster [DBG] pgmap v3890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:00.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:00 smithi006 bash[33918]: cluster 2023-08-28T00:08:00.106081+0000 mgr.y (mgr.14901) 3883 : cluster [DBG] pgmap v3890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:08:00.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:08:00.656Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:00 smithi130 bash[27579]: cluster 2023-08-28T00:08:00.106081+0000 mgr.y (mgr.14901) 3883 : cluster [DBG] pgmap v3890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:01.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:01 smithi130 bash[51302]: debug 2023-08-28T00:08:01.186+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:01.187402+0000) 2023-08-28T00:08:02.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:02 smithi006 bash[27292]: audit 2023-08-28T00:08:02.079813+0000 mon.c (mon.2) 1472 : audit [DBG] from='client.? 172.21.15.6:0/851258871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:02.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:02 smithi006 bash[33918]: audit 2023-08-28T00:08:02.079813+0000 mon.c (mon.2) 1472 : audit [DBG] from='client.? 172.21.15.6:0/851258871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:08:02.226Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:08:02.228Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:02 smithi130 bash[27579]: audit 2023-08-28T00:08:02.079813+0000 mon.c (mon.2) 1472 : audit [DBG] from='client.? 172.21.15.6:0/851258871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:02.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:02 smithi130 bash[51302]: debug 2023-08-28T00:08:02.186+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:02.187638+0000) 2023-08-28T00:08:03.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:03 smithi006 bash[27292]: cluster 2023-08-28T00:08:02.106689+0000 mgr.y (mgr.14901) 3884 : cluster [DBG] pgmap v3891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:03.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:03 smithi006 bash[33918]: cluster 2023-08-28T00:08:02.106689+0000 mgr.y (mgr.14901) 3884 : cluster [DBG] pgmap v3891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:03.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:03 smithi130 bash[27579]: cluster 2023-08-28T00:08:02.106689+0000 mgr.y (mgr.14901) 3884 : cluster [DBG] pgmap v3891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:03.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:03 smithi130 bash[51302]: debug 2023-08-28T00:08:03.186+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:03.187964+0000) 2023-08-28T00:08:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:08:04.112Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:08:04.114Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:04.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:04 smithi130 bash[51302]: debug 2023-08-28T00:08:04.186+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:04.188231+0000) 2023-08-28T00:08:05.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:05 smithi006 bash[33918]: cluster 2023-08-28T00:08:04.107687+0000 mgr.y (mgr.14901) 3885 : cluster [DBG] pgmap v3892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:05.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:05 smithi006 bash[33918]: audit 2023-08-28T00:08:04.501413+0000 mon.a (mon.0) 2956 : audit [DBG] from='client.? 172.21.15.6:0/2597265929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:05.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:05 smithi006 bash[27292]: cluster 2023-08-28T00:08:04.107687+0000 mgr.y (mgr.14901) 3885 : cluster [DBG] pgmap v3892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:05.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:05 smithi006 bash[27292]: audit 2023-08-28T00:08:04.501413+0000 mon.a (mon.0) 2956 : audit [DBG] from='client.? 172.21.15.6:0/2597265929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:05.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:05 smithi130 bash[51302]: debug 2023-08-28T00:08:05.186+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:05.188503+0000) 2023-08-28T00:08:05.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:05 smithi130 bash[27579]: cluster 2023-08-28T00:08:04.107687+0000 mgr.y (mgr.14901) 3885 : cluster [DBG] pgmap v3892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:05.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:05 smithi130 bash[27579]: audit 2023-08-28T00:08:04.501413+0000 mon.a (mon.0) 2956 : audit [DBG] from='client.? 172.21.15.6:0/2597265929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:06.460 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:06 smithi130 bash[51302]: debug 2023-08-28T00:08:06.186+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:06.188729+0000) 2023-08-28T00:08:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:06 smithi006 bash[27292]: cluster 2023-08-28T00:08:06.108416+0000 mgr.y (mgr.14901) 3886 : cluster [DBG] pgmap v3893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:06 smithi006 bash[33918]: cluster 2023-08-28T00:08:06.108416+0000 mgr.y (mgr.14901) 3886 : cluster [DBG] pgmap v3893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:06 smithi130 bash[27579]: cluster 2023-08-28T00:08:06.108416+0000 mgr.y (mgr.14901) 3886 : cluster [DBG] pgmap v3893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:07.463 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:07 smithi130 bash[51302]: debug 2023-08-28T00:08:07.186+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:07.189014+0000) 2023-08-28T00:08:07.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:07 smithi006 bash[27292]: audit 2023-08-28T00:08:06.925936+0000 mon.a (mon.0) 2957 : audit [DBG] from='client.? 172.21.15.6:0/3994806188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:07.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:07 smithi006 bash[33918]: audit 2023-08-28T00:08:06.925936+0000 mon.a (mon.0) 2957 : audit [DBG] from='client.? 172.21.15.6:0/3994806188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:07.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:07 smithi130 bash[27579]: audit 2023-08-28T00:08:06.925936+0000 mon.a (mon.0) 2957 : audit [DBG] from='client.? 172.21.15.6:0/3994806188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:08.472 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:08 smithi130 bash[51302]: debug 2023-08-28T00:08:08.186+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:08.189283+0000) 2023-08-28T00:08:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:08 smithi130 bash[27579]: cluster 2023-08-28T00:08:08.109114+0000 mgr.y (mgr.14901) 3887 : cluster [DBG] pgmap v3894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:08 smithi006 bash[27292]: cluster 2023-08-28T00:08:08.109114+0000 mgr.y (mgr.14901) 3887 : cluster [DBG] pgmap v3894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:08 smithi006 bash[33918]: cluster 2023-08-28T00:08:08.109114+0000 mgr.y (mgr.14901) 3887 : cluster [DBG] pgmap v3894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:09.475 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:09 smithi130 bash[51302]: debug 2023-08-28T00:08:09.186+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:09.189635+0000) 2023-08-28T00:08:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:09 smithi130 bash[27579]: audit 2023-08-28T00:08:09.340073+0000 mon.a (mon.0) 2958 : audit [DBG] from='client.? 172.21.15.6:0/3528852875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:09.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:09 smithi006 bash[27292]: audit 2023-08-28T00:08:09.340073+0000 mon.a (mon.0) 2958 : audit [DBG] from='client.? 172.21.15.6:0/3528852875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:09.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:09 smithi006 bash[33918]: audit 2023-08-28T00:08:09.340073+0000 mon.a (mon.0) 2958 : audit [DBG] from='client.? 172.21.15.6:0/3528852875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:10.482 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:10 smithi130 bash[51302]: debug 2023-08-28T00:08:10.186+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:10.189940+0000) 2023-08-28T00:08:10.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:10 smithi006 bash[33918]: cluster 2023-08-28T00:08:10.110265+0000 mgr.y (mgr.14901) 3888 : cluster [DBG] pgmap v3895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:10.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:10 smithi006 bash[27292]: cluster 2023-08-28T00:08:10.110265+0000 mgr.y (mgr.14901) 3888 : cluster [DBG] pgmap v3895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:10.756 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:08:10.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:10.756 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:08:10.657Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:10 smithi130 bash[27579]: cluster 2023-08-28T00:08:10.110265+0000 mgr.y (mgr.14901) 3888 : cluster [DBG] pgmap v3895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:08:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:08:10] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:08:11.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:11 smithi130 bash[51302]: debug 2023-08-28T00:08:11.186+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:11.190266+0000) 2023-08-28T00:08:12.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:11 smithi130 bash[27579]: audit 2023-08-28T00:08:11.747669+0000 mon.c (mon.2) 1473 : audit [DBG] from='client.? 172.21.15.6:0/1844265550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:12.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:11 smithi006 bash[27292]: audit 2023-08-28T00:08:11.747669+0000 mon.c (mon.2) 1473 : audit [DBG] from='client.? 172.21.15.6:0/1844265550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:12.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:11 smithi006 bash[33918]: audit 2023-08-28T00:08:11.747669+0000 mon.c (mon.2) 1473 : audit [DBG] from='client.? 172.21.15.6:0/1844265550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:12.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:12 smithi130 bash[51302]: debug 2023-08-28T00:08:12.186+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:12.190547+0000) 2023-08-28T00:08:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:08:12.227Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:08:12.229Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:12 smithi130 bash[27579]: cluster 2023-08-28T00:08:12.110944+0000 mgr.y (mgr.14901) 3889 : cluster [DBG] pgmap v3896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:12 smithi006 bash[27292]: cluster 2023-08-28T00:08:12.110944+0000 mgr.y (mgr.14901) 3889 : cluster [DBG] pgmap v3896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:12 smithi006 bash[33918]: cluster 2023-08-28T00:08:12.110944+0000 mgr.y (mgr.14901) 3889 : cluster [DBG] pgmap v3896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:13.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:13 smithi130 bash[51302]: debug 2023-08-28T00:08:13.190+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:13.190847+0000) 2023-08-28T00:08:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:08:14.113Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:08:14.115Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:14.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:14 smithi130 bash[51302]: debug 2023-08-28T00:08:14.189+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:14.191153+0000) 2023-08-28T00:08:15.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:15 smithi006 bash[33918]: cluster 2023-08-28T00:08:14.111591+0000 mgr.y (mgr.14901) 3890 : cluster [DBG] pgmap v3897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:15.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:15 smithi006 bash[33918]: audit 2023-08-28T00:08:14.171099+0000 mon.c (mon.2) 1474 : audit [DBG] from='client.? 172.21.15.6:0/3667259863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:15.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:15 smithi006 bash[27292]: cluster 2023-08-28T00:08:14.111591+0000 mgr.y (mgr.14901) 3890 : cluster [DBG] pgmap v3897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:15.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:15 smithi006 bash[27292]: audit 2023-08-28T00:08:14.171099+0000 mon.c (mon.2) 1474 : audit [DBG] from='client.? 172.21.15.6:0/3667259863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:15.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:15 smithi130 bash[27579]: cluster 2023-08-28T00:08:14.111591+0000 mgr.y (mgr.14901) 3890 : cluster [DBG] pgmap v3897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:15.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:15 smithi130 bash[27579]: audit 2023-08-28T00:08:14.171099+0000 mon.c (mon.2) 1474 : audit [DBG] from='client.? 172.21.15.6:0/3667259863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:15.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:15 smithi130 bash[51302]: debug 2023-08-28T00:08:15.189+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:15.191337+0000) 2023-08-28T00:08:16.464 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:16 smithi130 bash[51302]: debug 2023-08-28T00:08:16.189+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:16.191603+0000) 2023-08-28T00:08:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:16 smithi006 bash[33918]: cluster 2023-08-28T00:08:16.112267+0000 mgr.y (mgr.14901) 3891 : cluster [DBG] pgmap v3898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:16 smithi006 bash[27292]: cluster 2023-08-28T00:08:16.112267+0000 mgr.y (mgr.14901) 3891 : cluster [DBG] pgmap v3898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:16 smithi130 bash[27579]: cluster 2023-08-28T00:08:16.112267+0000 mgr.y (mgr.14901) 3891 : cluster [DBG] pgmap v3898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:17.465 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:17 smithi130 bash[51302]: debug 2023-08-28T00:08:17.189+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:17.191893+0000) 2023-08-28T00:08:17.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:17 smithi006 bash[33918]: audit 2023-08-28T00:08:16.595501+0000 mon.a (mon.0) 2959 : audit [DBG] from='client.? 172.21.15.6:0/250548028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:17.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:17 smithi006 bash[27292]: audit 2023-08-28T00:08:16.595501+0000 mon.a (mon.0) 2959 : audit [DBG] from='client.? 172.21.15.6:0/250548028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:17.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:17 smithi130 bash[27579]: audit 2023-08-28T00:08:16.595501+0000 mon.a (mon.0) 2959 : audit [DBG] from='client.? 172.21.15.6:0/250548028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:18.474 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:18 smithi130 bash[51302]: debug 2023-08-28T00:08:18.189+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:18.192171+0000) 2023-08-28T00:08:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:18 smithi130 bash[27579]: cluster 2023-08-28T00:08:18.112958+0000 mgr.y (mgr.14901) 3892 : cluster [DBG] pgmap v3899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:18 smithi006 bash[27292]: cluster 2023-08-28T00:08:18.112958+0000 mgr.y (mgr.14901) 3892 : cluster [DBG] pgmap v3899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:18 smithi006 bash[33918]: cluster 2023-08-28T00:08:18.112958+0000 mgr.y (mgr.14901) 3892 : cluster [DBG] pgmap v3899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:19.475 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:19 smithi130 bash[51302]: debug 2023-08-28T00:08:19.189+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:19.192536+0000) 2023-08-28T00:08:19.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:19 smithi130 bash[27579]: audit 2023-08-28T00:08:19.009930+0000 mon.c (mon.2) 1475 : audit [DBG] from='client.? 172.21.15.6:0/1139810091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:19.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:19 smithi006 bash[27292]: audit 2023-08-28T00:08:19.009930+0000 mon.c (mon.2) 1475 : audit [DBG] from='client.? 172.21.15.6:0/1139810091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:19 smithi006 bash[33918]: audit 2023-08-28T00:08:19.009930+0000 mon.c (mon.2) 1475 : audit [DBG] from='client.? 172.21.15.6:0/1139810091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:20.486 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:20 smithi130 bash[51302]: debug 2023-08-28T00:08:20.189+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:20.192814+0000) 2023-08-28T00:08:20.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:20 smithi006 bash[27292]: cluster 2023-08-28T00:08:20.114024+0000 mgr.y (mgr.14901) 3893 : cluster [DBG] pgmap v3900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:20.756 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:20 smithi006 bash[33918]: cluster 2023-08-28T00:08:20.114024+0000 mgr.y (mgr.14901) 3893 : cluster [DBG] pgmap v3900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:20.756 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:08:20.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:20.756 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:08:20.658Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:20 smithi130 bash[27579]: cluster 2023-08-28T00:08:20.114024+0000 mgr.y (mgr.14901) 3893 : cluster [DBG] pgmap v3900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:08:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:08:20] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:08:21.489 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:21 smithi130 bash[51302]: debug 2023-08-28T00:08:21.189+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:21.193097+0000) 2023-08-28T00:08:21.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:21 smithi130 bash[27579]: audit 2023-08-28T00:08:21.421240+0000 mon.c (mon.2) 1476 : audit [DBG] from='client.? 172.21.15.6:0/900642090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:21.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:21 smithi006 bash[27292]: audit 2023-08-28T00:08:21.421240+0000 mon.c (mon.2) 1476 : audit [DBG] from='client.? 172.21.15.6:0/900642090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:21.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:21 smithi006 bash[33918]: audit 2023-08-28T00:08:21.421240+0000 mon.c (mon.2) 1476 : audit [DBG] from='client.? 172.21.15.6:0/900642090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:22.496 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:08:22.228Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:22.496 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:08:22.230Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:22.496 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:22 smithi130 bash[51302]: debug 2023-08-28T00:08:22.189+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:22.193351+0000) 2023-08-28T00:08:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:22 smithi130 bash[27579]: cluster 2023-08-28T00:08:22.114694+0000 mgr.y (mgr.14901) 3894 : cluster [DBG] pgmap v3901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:22 smithi006 bash[27292]: cluster 2023-08-28T00:08:22.114694+0000 mgr.y (mgr.14901) 3894 : cluster [DBG] pgmap v3901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:22 smithi006 bash[33918]: cluster 2023-08-28T00:08:22.114694+0000 mgr.y (mgr.14901) 3894 : cluster [DBG] pgmap v3901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:23.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:23 smithi130 bash[51302]: debug 2023-08-28T00:08:23.189+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:23.193652+0000) 2023-08-28T00:08:24.195 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:23 smithi130 bash[27579]: audit 2023-08-28T00:08:23.836184+0000 mon.c (mon.2) 1477 : audit [DBG] from='client.? 172.21.15.6:0/2107619756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:24.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:23 smithi006 bash[27292]: audit 2023-08-28T00:08:23.836184+0000 mon.c (mon.2) 1477 : audit [DBG] from='client.? 172.21.15.6:0/2107619756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:24.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:23 smithi006 bash[33918]: audit 2023-08-28T00:08:23.836184+0000 mon.c (mon.2) 1477 : audit [DBG] from='client.? 172.21.15.6:0/2107619756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:24.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:08:24.113Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:24.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:08:24.115Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:24.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:24 smithi130 bash[51302]: debug 2023-08-28T00:08:24.189+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:24.193916+0000) 2023-08-28T00:08:25.195 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:24 smithi130 bash[27579]: cluster 2023-08-28T00:08:24.116057+0000 mgr.y (mgr.14901) 3895 : cluster [DBG] pgmap v3902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:24 smithi006 bash[33918]: cluster 2023-08-28T00:08:24.116057+0000 mgr.y (mgr.14901) 3895 : cluster [DBG] pgmap v3902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:24 smithi006 bash[27292]: cluster 2023-08-28T00:08:24.116057+0000 mgr.y (mgr.14901) 3895 : cluster [DBG] pgmap v3902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:25.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:25 smithi130 bash[51302]: debug 2023-08-28T00:08:25.193+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:25.194199+0000) 2023-08-28T00:08:26.466 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:26 smithi130 bash[51302]: debug 2023-08-28T00:08:26.189+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:26.194466+0000) 2023-08-28T00:08:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:26 smithi006 bash[27292]: cluster 2023-08-28T00:08:26.117045+0000 mgr.y (mgr.14901) 3896 : cluster [DBG] pgmap v3903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:26 smithi006 bash[27292]: audit 2023-08-28T00:08:26.253975+0000 mon.c (mon.2) 1478 : audit [DBG] from='client.? 172.21.15.6:0/3425904375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:26 smithi006 bash[33918]: cluster 2023-08-28T00:08:26.117045+0000 mgr.y (mgr.14901) 3896 : cluster [DBG] pgmap v3903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:26 smithi006 bash[33918]: audit 2023-08-28T00:08:26.253975+0000 mon.c (mon.2) 1478 : audit [DBG] from='client.? 172.21.15.6:0/3425904375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:26 smithi130 bash[27579]: cluster 2023-08-28T00:08:26.117045+0000 mgr.y (mgr.14901) 3896 : cluster [DBG] pgmap v3903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:26 smithi130 bash[27579]: audit 2023-08-28T00:08:26.253975+0000 mon.c (mon.2) 1478 : audit [DBG] from='client.? 172.21.15.6:0/3425904375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:27.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:27 smithi130 bash[51302]: debug 2023-08-28T00:08:27.189+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:27.194669+0000) 2023-08-28T00:08:28.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:28 smithi130 bash[51302]: debug 2023-08-28T00:08:28.193+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:28.194955+0000) 2023-08-28T00:08:29.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:29 smithi006 bash[33918]: cluster 2023-08-28T00:08:28.117705+0000 mgr.y (mgr.14901) 3897 : cluster [DBG] pgmap v3904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:29.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:29 smithi006 bash[33918]: audit 2023-08-28T00:08:28.701832+0000 mon.a (mon.0) 2960 : audit [DBG] from='client.? 172.21.15.6:0/2390892329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:29.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:29 smithi006 bash[27292]: cluster 2023-08-28T00:08:28.117705+0000 mgr.y (mgr.14901) 3897 : cluster [DBG] pgmap v3904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:29.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:29 smithi006 bash[27292]: audit 2023-08-28T00:08:28.701832+0000 mon.a (mon.0) 2960 : audit [DBG] from='client.? 172.21.15.6:0/2390892329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:29.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:29 smithi130 bash[51302]: debug 2023-08-28T00:08:29.193+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:29.195202+0000) 2023-08-28T00:08:29.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:29 smithi130 bash[27579]: cluster 2023-08-28T00:08:28.117705+0000 mgr.y (mgr.14901) 3897 : cluster [DBG] pgmap v3904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:29.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:29 smithi130 bash[27579]: audit 2023-08-28T00:08:28.701832+0000 mon.a (mon.0) 2960 : audit [DBG] from='client.? 172.21.15.6:0/2390892329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:30.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:30 smithi130 bash[51302]: debug 2023-08-28T00:08:30.193+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:30.195534+0000) 2023-08-28T00:08:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:08:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:08:30] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:08:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:08:30.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:08:30.660Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:31.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:31 smithi006 bash[27292]: cluster 2023-08-28T00:08:30.118869+0000 mgr.y (mgr.14901) 3898 : cluster [DBG] pgmap v3905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:31.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:31 smithi006 bash[27292]: audit 2023-08-28T00:08:31.120431+0000 mon.a (mon.0) 2961 : audit [DBG] from='client.? 172.21.15.6:0/1221806896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:31.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:31 smithi006 bash[33918]: cluster 2023-08-28T00:08:30.118869+0000 mgr.y (mgr.14901) 3898 : cluster [DBG] pgmap v3905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:31.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:31 smithi006 bash[33918]: audit 2023-08-28T00:08:31.120431+0000 mon.a (mon.0) 2961 : audit [DBG] from='client.? 172.21.15.6:0/1221806896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:31.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:31 smithi130 bash[51302]: debug 2023-08-28T00:08:31.193+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:31.195809+0000) 2023-08-28T00:08:31.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:31 smithi130 bash[27579]: cluster 2023-08-28T00:08:30.118869+0000 mgr.y (mgr.14901) 3898 : cluster [DBG] pgmap v3905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:31.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:31 smithi130 bash[27579]: audit 2023-08-28T00:08:31.120431+0000 mon.a (mon.0) 2961 : audit [DBG] from='client.? 172.21.15.6:0/1221806896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:32.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:32 smithi130 bash[51302]: debug 2023-08-28T00:08:32.193+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:32.196120+0000) 2023-08-28T00:08:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:08:32.229Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:08:32.231Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:33.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:33 smithi006 bash[27292]: cluster 2023-08-28T00:08:32.119530+0000 mgr.y (mgr.14901) 3899 : cluster [DBG] pgmap v3906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:33.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:33 smithi006 bash[27292]: audit 2023-08-28T00:08:32.208149+0000 mon.b (mon.1) 487 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:08:33.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:33 smithi006 bash[33918]: cluster 2023-08-28T00:08:32.119530+0000 mgr.y (mgr.14901) 3899 : cluster [DBG] pgmap v3906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:33.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:33 smithi006 bash[33918]: audit 2023-08-28T00:08:32.208149+0000 mon.b (mon.1) 487 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:08:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:33 smithi130 bash[27579]: cluster 2023-08-28T00:08:32.119530+0000 mgr.y (mgr.14901) 3899 : cluster [DBG] pgmap v3906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:33 smithi130 bash[27579]: audit 2023-08-28T00:08:32.208149+0000 mon.b (mon.1) 487 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:08:33.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:33 smithi130 bash[51302]: debug 2023-08-28T00:08:33.193+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:33.196430+0000) 2023-08-28T00:08:34.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:34 smithi006 bash[33918]: audit 2023-08-28T00:08:33.545595+0000 mon.a (mon.0) 2962 : audit [DBG] from='client.? 172.21.15.6:0/968289921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:08:34.114Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:08:34.116Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:34.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:34 smithi006 bash[27292]: audit 2023-08-28T00:08:33.545595+0000 mon.a (mon.0) 2962 : audit [DBG] from='client.? 172.21.15.6:0/968289921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:34 smithi130 bash[27579]: audit 2023-08-28T00:08:33.545595+0000 mon.a (mon.0) 2962 : audit [DBG] from='client.? 172.21.15.6:0/968289921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:34.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:34 smithi130 bash[51302]: debug 2023-08-28T00:08:34.193+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:34.196676+0000) 2023-08-28T00:08:35.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:35 smithi006 bash[33918]: cluster 2023-08-28T00:08:34.120363+0000 mgr.y (mgr.14901) 3900 : cluster [DBG] pgmap v3907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:35.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:35 smithi006 bash[27292]: cluster 2023-08-28T00:08:34.120363+0000 mgr.y (mgr.14901) 3900 : cluster [DBG] pgmap v3907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:35.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:35 smithi130 bash[27579]: cluster 2023-08-28T00:08:34.120363+0000 mgr.y (mgr.14901) 3900 : cluster [DBG] pgmap v3907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:35.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:35 smithi130 bash[51302]: debug 2023-08-28T00:08:35.193+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:35.196889+0000) 2023-08-28T00:08:36.467 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:36 smithi130 bash[51302]: debug 2023-08-28T00:08:36.193+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:36.197164+0000) 2023-08-28T00:08:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:36 smithi006 bash[27292]: audit 2023-08-28T00:08:35.958766+0000 mon.c (mon.2) 1479 : audit [DBG] from='client.? 172.21.15.6:0/1431595149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:36 smithi006 bash[27292]: cluster 2023-08-28T00:08:36.121151+0000 mgr.y (mgr.14901) 3901 : cluster [DBG] pgmap v3908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:36 smithi006 bash[33918]: audit 2023-08-28T00:08:35.958766+0000 mon.c (mon.2) 1479 : audit [DBG] from='client.? 172.21.15.6:0/1431595149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:36 smithi006 bash[33918]: cluster 2023-08-28T00:08:36.121151+0000 mgr.y (mgr.14901) 3901 : cluster [DBG] pgmap v3908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:36 smithi130 bash[27579]: audit 2023-08-28T00:08:35.958766+0000 mon.c (mon.2) 1479 : audit [DBG] from='client.? 172.21.15.6:0/1431595149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:36 smithi130 bash[27579]: cluster 2023-08-28T00:08:36.121151+0000 mgr.y (mgr.14901) 3901 : cluster [DBG] pgmap v3908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:37.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:37 smithi130 bash[51302]: debug 2023-08-28T00:08:37.193+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:37.197500+0000) 2023-08-28T00:08:38.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:38 smithi130 bash[51302]: debug 2023-08-28T00:08:38.193+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:38.197820+0000) 2023-08-28T00:08:39.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:39 smithi006 bash[33918]: audit 2023-08-28T00:08:38.115901+0000 mon.a (mon.0) 2963 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:08:39.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:39 smithi006 bash[33918]: cluster 2023-08-28T00:08:38.121563+0000 mgr.y (mgr.14901) 3902 : cluster [DBG] pgmap v3909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:39.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:39 smithi006 bash[33918]: audit 2023-08-28T00:08:38.387039+0000 mon.a (mon.0) 2964 : audit [DBG] from='client.? 172.21.15.6:0/3723613202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:39.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:39 smithi006 bash[33918]: audit 2023-08-28T00:08:38.569680+0000 mon.a (mon.0) 2965 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:08:39.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:39 smithi006 bash[33918]: audit 2023-08-28T00:08:38.859545+0000 mon.b (mon.1) 488 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:08:39.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:39 smithi006 bash[33918]: audit 2023-08-28T00:08:38.861777+0000 mon.b (mon.1) 489 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:08:39.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:39 smithi006 bash[33918]: audit 2023-08-28T00:08:38.870487+0000 mon.a (mon.0) 2966 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:08:39.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:39 smithi006 bash[27292]: audit 2023-08-28T00:08:38.115901+0000 mon.a (mon.0) 2963 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:08:39.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:39 smithi006 bash[27292]: cluster 2023-08-28T00:08:38.121563+0000 mgr.y (mgr.14901) 3902 : cluster [DBG] pgmap v3909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:39.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:39 smithi006 bash[27292]: audit 2023-08-28T00:08:38.387039+0000 mon.a (mon.0) 2964 : audit [DBG] from='client.? 172.21.15.6:0/3723613202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:39.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:39 smithi006 bash[27292]: audit 2023-08-28T00:08:38.569680+0000 mon.a (mon.0) 2965 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:08:39.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:39 smithi006 bash[27292]: audit 2023-08-28T00:08:38.859545+0000 mon.b (mon.1) 488 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:08:39.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:39 smithi006 bash[27292]: audit 2023-08-28T00:08:38.861777+0000 mon.b (mon.1) 489 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:08:39.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:39 smithi006 bash[27292]: audit 2023-08-28T00:08:38.870487+0000 mon.a (mon.0) 2966 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:08:39.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:39 smithi130 bash[51302]: debug 2023-08-28T00:08:39.193+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:39.198087+0000) 2023-08-28T00:08:39.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:39 smithi130 bash[27579]: audit 2023-08-28T00:08:38.115901+0000 mon.a (mon.0) 2963 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:08:39.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:39 smithi130 bash[27579]: cluster 2023-08-28T00:08:38.121563+0000 mgr.y (mgr.14901) 3902 : cluster [DBG] pgmap v3909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:39.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:39 smithi130 bash[27579]: audit 2023-08-28T00:08:38.387039+0000 mon.a (mon.0) 2964 : audit [DBG] from='client.? 172.21.15.6:0/3723613202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:39.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:39 smithi130 bash[27579]: audit 2023-08-28T00:08:38.569680+0000 mon.a (mon.0) 2965 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:08:39.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:39 smithi130 bash[27579]: audit 2023-08-28T00:08:38.859545+0000 mon.b (mon.1) 488 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:08:39.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:39 smithi130 bash[27579]: audit 2023-08-28T00:08:38.861777+0000 mon.b (mon.1) 489 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:08:39.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:39 smithi130 bash[27579]: audit 2023-08-28T00:08:38.870487+0000 mon.a (mon.0) 2966 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:08:40.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:40 smithi130 bash[51302]: debug 2023-08-28T00:08:40.197+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:40.198356+0000) 2023-08-28T00:08:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:08:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:08:40] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:08:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:08:40.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:08:40.659Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:41.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:41 smithi006 bash[33918]: cluster 2023-08-28T00:08:40.122601+0000 mgr.y (mgr.14901) 3903 : cluster [DBG] pgmap v3910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:41.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:41 smithi006 bash[33918]: audit 2023-08-28T00:08:40.802473+0000 mon.c (mon.2) 1480 : audit [DBG] from='client.? 172.21.15.6:0/1968474345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:41.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:41 smithi006 bash[27292]: cluster 2023-08-28T00:08:40.122601+0000 mgr.y (mgr.14901) 3903 : cluster [DBG] pgmap v3910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:41.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:41 smithi006 bash[27292]: audit 2023-08-28T00:08:40.802473+0000 mon.c (mon.2) 1480 : audit [DBG] from='client.? 172.21.15.6:0/1968474345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:41.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:41 smithi130 bash[27579]: cluster 2023-08-28T00:08:40.122601+0000 mgr.y (mgr.14901) 3903 : cluster [DBG] pgmap v3910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:41.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:41 smithi130 bash[27579]: audit 2023-08-28T00:08:40.802473+0000 mon.c (mon.2) 1480 : audit [DBG] from='client.? 172.21.15.6:0/1968474345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:41.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:41 smithi130 bash[51302]: debug 2023-08-28T00:08:41.193+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:41.198625+0000) 2023-08-28T00:08:42.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:42 smithi130 bash[51302]: debug 2023-08-28T00:08:42.193+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:42.198820+0000) 2023-08-28T00:08:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:08:42.230Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:08:42.232Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:43.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:43 smithi006 bash[33918]: cluster 2023-08-28T00:08:42.123251+0000 mgr.y (mgr.14901) 3904 : cluster [DBG] pgmap v3911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:43.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:43 smithi006 bash[27292]: cluster 2023-08-28T00:08:42.123251+0000 mgr.y (mgr.14901) 3904 : cluster [DBG] pgmap v3911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:43 smithi130 bash[27579]: cluster 2023-08-28T00:08:42.123251+0000 mgr.y (mgr.14901) 3904 : cluster [DBG] pgmap v3911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:43.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:43 smithi130 bash[51302]: debug 2023-08-28T00:08:43.197+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:43.199061+0000) 2023-08-28T00:08:44.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:44 smithi006 bash[27292]: audit 2023-08-28T00:08:43.214677+0000 mon.a (mon.0) 2967 : audit [DBG] from='client.? 172.21.15.6:0/2325228782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:44.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:44 smithi006 bash[33918]: audit 2023-08-28T00:08:43.214677+0000 mon.a (mon.0) 2967 : audit [DBG] from='client.? 172.21.15.6:0/2325228782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:08:44.115Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:08:44.117Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:44 smithi130 bash[27579]: audit 2023-08-28T00:08:43.214677+0000 mon.a (mon.0) 2967 : audit [DBG] from='client.? 172.21.15.6:0/2325228782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:44.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:44 smithi130 bash[51302]: debug 2023-08-28T00:08:44.197+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:44.199255+0000) 2023-08-28T00:08:45.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:45 smithi006 bash[27292]: cluster 2023-08-28T00:08:44.124231+0000 mgr.y (mgr.14901) 3905 : cluster [DBG] pgmap v3912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:45.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:45 smithi006 bash[33918]: cluster 2023-08-28T00:08:44.124231+0000 mgr.y (mgr.14901) 3905 : cluster [DBG] pgmap v3912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:45.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:45 smithi130 bash[27579]: cluster 2023-08-28T00:08:44.124231+0000 mgr.y (mgr.14901) 3905 : cluster [DBG] pgmap v3912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:45.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:45 smithi130 bash[51302]: debug 2023-08-28T00:08:45.197+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:45.199505+0000) 2023-08-28T00:08:46.472 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:46 smithi130 bash[51302]: debug 2023-08-28T00:08:46.197+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:46.199803+0000) 2023-08-28T00:08:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:46 smithi130 bash[27579]: audit 2023-08-28T00:08:45.633384+0000 mon.a (mon.0) 2968 : audit [DBG] from='client.? 172.21.15.6:0/3340357434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:46 smithi130 bash[27579]: cluster 2023-08-28T00:08:46.125061+0000 mgr.y (mgr.14901) 3906 : cluster [DBG] pgmap v3913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:46 smithi006 bash[27292]: audit 2023-08-28T00:08:45.633384+0000 mon.a (mon.0) 2968 : audit [DBG] from='client.? 172.21.15.6:0/3340357434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:46 smithi006 bash[27292]: cluster 2023-08-28T00:08:46.125061+0000 mgr.y (mgr.14901) 3906 : cluster [DBG] pgmap v3913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:46 smithi006 bash[33918]: audit 2023-08-28T00:08:45.633384+0000 mon.a (mon.0) 2968 : audit [DBG] from='client.? 172.21.15.6:0/3340357434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:46 smithi006 bash[33918]: cluster 2023-08-28T00:08:46.125061+0000 mgr.y (mgr.14901) 3906 : cluster [DBG] pgmap v3913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:47.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:47 smithi130 bash[51302]: debug 2023-08-28T00:08:47.197+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:47.200058+0000) 2023-08-28T00:08:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:48 smithi130 bash[27579]: audit 2023-08-28T00:08:48.041719+0000 mon.a (mon.0) 2969 : audit [DBG] from='client.? 172.21.15.6:0/2508130077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:48.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:48 smithi130 bash[51302]: debug 2023-08-28T00:08:48.197+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:48.200386+0000) 2023-08-28T00:08:48.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:48 smithi006 bash[27292]: audit 2023-08-28T00:08:48.041719+0000 mon.a (mon.0) 2969 : audit [DBG] from='client.? 172.21.15.6:0/2508130077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:48.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:48 smithi006 bash[33918]: audit 2023-08-28T00:08:48.041719+0000 mon.a (mon.0) 2969 : audit [DBG] from='client.? 172.21.15.6:0/2508130077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:49 smithi130 bash[51302]: debug 2023-08-28T00:08:49.197+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:49.200702+0000) 2023-08-28T00:08:49.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:49 smithi130 bash[27579]: cluster 2023-08-28T00:08:48.125709+0000 mgr.y (mgr.14901) 3907 : cluster [DBG] pgmap v3914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:49.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:49 smithi006 bash[33918]: cluster 2023-08-28T00:08:48.125709+0000 mgr.y (mgr.14901) 3907 : cluster [DBG] pgmap v3914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:49.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:49 smithi006 bash[27292]: cluster 2023-08-28T00:08:48.125709+0000 mgr.y (mgr.14901) 3907 : cluster [DBG] pgmap v3914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:50.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:50 smithi130 bash[51302]: debug 2023-08-28T00:08:50.197+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:50.200989+0000) 2023-08-28T00:08:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:08:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:08:50] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:08:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:08:50.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:08:50.660Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:51.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:51 smithi006 bash[33918]: cluster 2023-08-28T00:08:50.126856+0000 mgr.y (mgr.14901) 3908 : cluster [DBG] pgmap v3915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:51.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:51 smithi006 bash[33918]: audit 2023-08-28T00:08:50.459753+0000 mon.a (mon.0) 2970 : audit [DBG] from='client.? 172.21.15.6:0/2060986031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:51.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:51 smithi006 bash[27292]: cluster 2023-08-28T00:08:50.126856+0000 mgr.y (mgr.14901) 3908 : cluster [DBG] pgmap v3915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:51.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:51 smithi006 bash[27292]: audit 2023-08-28T00:08:50.459753+0000 mon.a (mon.0) 2970 : audit [DBG] from='client.? 172.21.15.6:0/2060986031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:51.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:51 smithi130 bash[51302]: debug 2023-08-28T00:08:51.197+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:51.201268+0000) 2023-08-28T00:08:51.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:51 smithi130 bash[27579]: cluster 2023-08-28T00:08:50.126856+0000 mgr.y (mgr.14901) 3908 : cluster [DBG] pgmap v3915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:51.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:51 smithi130 bash[27579]: audit 2023-08-28T00:08:50.459753+0000 mon.a (mon.0) 2970 : audit [DBG] from='client.? 172.21.15.6:0/2060986031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:52.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:52 smithi130 bash[51302]: debug 2023-08-28T00:08:52.197+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:52.201552+0000) 2023-08-28T00:08:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:08:52.230Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:08:52.232Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:53.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:53 smithi006 bash[33918]: cluster 2023-08-28T00:08:52.127528+0000 mgr.y (mgr.14901) 3909 : cluster [DBG] pgmap v3916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:53.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:53 smithi006 bash[33918]: audit 2023-08-28T00:08:52.882784+0000 mon.c (mon.2) 1481 : audit [DBG] from='client.? 172.21.15.6:0/4089663462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:53 smithi006 bash[27292]: cluster 2023-08-28T00:08:52.127528+0000 mgr.y (mgr.14901) 3909 : cluster [DBG] pgmap v3916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:53 smithi006 bash[27292]: audit 2023-08-28T00:08:52.882784+0000 mon.c (mon.2) 1481 : audit [DBG] from='client.? 172.21.15.6:0/4089663462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:53 smithi130 bash[27579]: cluster 2023-08-28T00:08:52.127528+0000 mgr.y (mgr.14901) 3909 : cluster [DBG] pgmap v3916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:53 smithi130 bash[27579]: audit 2023-08-28T00:08:52.882784+0000 mon.c (mon.2) 1481 : audit [DBG] from='client.? 172.21.15.6:0/4089663462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:53.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:53 smithi130 bash[51302]: debug 2023-08-28T00:08:53.197+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:53.201788+0000) 2023-08-28T00:08:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:08:54.116Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:08:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:08:54.118Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:08:54.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:54 smithi130 bash[51302]: debug 2023-08-28T00:08:54.197+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:54.202047+0000) 2023-08-28T00:08:55.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:55 smithi006 bash[27292]: cluster 2023-08-28T00:08:54.128557+0000 mgr.y (mgr.14901) 3910 : cluster [DBG] pgmap v3917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:55.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:55 smithi006 bash[33918]: cluster 2023-08-28T00:08:54.128557+0000 mgr.y (mgr.14901) 3910 : cluster [DBG] pgmap v3917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:55.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:55 smithi130 bash[27579]: cluster 2023-08-28T00:08:54.128557+0000 mgr.y (mgr.14901) 3910 : cluster [DBG] pgmap v3917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:55.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:55 smithi130 bash[51302]: debug 2023-08-28T00:08:55.197+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:55.202262+0000) 2023-08-28T00:08:56.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:56 smithi006 bash[27292]: audit 2023-08-28T00:08:55.295241+0000 mon.c (mon.2) 1482 : audit [DBG] from='client.? 172.21.15.6:0/1500630272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:56.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:56 smithi006 bash[33918]: audit 2023-08-28T00:08:55.295241+0000 mon.c (mon.2) 1482 : audit [DBG] from='client.? 172.21.15.6:0/1500630272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:56 smithi130 bash[27579]: audit 2023-08-28T00:08:55.295241+0000 mon.c (mon.2) 1482 : audit [DBG] from='client.? 172.21.15.6:0/1500630272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:56.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:56 smithi130 bash[51302]: debug 2023-08-28T00:08:56.197+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:56.202518+0000) 2023-08-28T00:08:57.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:57 smithi006 bash[27292]: cluster 2023-08-28T00:08:56.129322+0000 mgr.y (mgr.14901) 3911 : cluster [DBG] pgmap v3918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:57.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:57 smithi006 bash[33918]: cluster 2023-08-28T00:08:56.129322+0000 mgr.y (mgr.14901) 3911 : cluster [DBG] pgmap v3918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:57.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:57 smithi130 bash[27579]: cluster 2023-08-28T00:08:56.129322+0000 mgr.y (mgr.14901) 3911 : cluster [DBG] pgmap v3918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:57.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:57 smithi130 bash[51302]: debug 2023-08-28T00:08:57.201+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:57.202878+0000) 2023-08-28T00:08:58.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:58 smithi006 bash[27292]: audit 2023-08-28T00:08:57.713169+0000 mon.a (mon.0) 2971 : audit [DBG] from='client.? 172.21.15.6:0/4124379127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:58.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:58 smithi006 bash[33918]: audit 2023-08-28T00:08:57.713169+0000 mon.a (mon.0) 2971 : audit [DBG] from='client.? 172.21.15.6:0/4124379127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:58 smithi130 bash[27579]: audit 2023-08-28T00:08:57.713169+0000 mon.a (mon.0) 2971 : audit [DBG] from='client.? 172.21.15.6:0/4124379127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:58.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:58 smithi130 bash[51302]: debug 2023-08-28T00:08:58.201+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:58.203132+0000) 2023-08-28T00:08:59.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:08:59 smithi006 bash[27292]: cluster 2023-08-28T00:08:58.129997+0000 mgr.y (mgr.14901) 3912 : cluster [DBG] pgmap v3919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:08:59 smithi006 bash[33918]: cluster 2023-08-28T00:08:58.129997+0000 mgr.y (mgr.14901) 3912 : cluster [DBG] pgmap v3919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:08:59 smithi130 bash[27579]: cluster 2023-08-28T00:08:58.129997+0000 mgr.y (mgr.14901) 3912 : cluster [DBG] pgmap v3919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:08:59.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:08:59 smithi130 bash[51302]: debug 2023-08-28T00:08:59.201+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:59.203362+0000) 2023-08-28T00:09:00.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:00 smithi130 bash[51302]: debug 2023-08-28T00:09:00.201+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:00.203709+0000) 2023-08-28T00:09:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:00 smithi130 bash[27579]: cluster 2023-08-28T00:08:59.219010+0000 mon.a (mon.0) 2972 : cluster [DBG] mgrmap e27: y(active, since 2h) 2023-08-28T00:09:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:00 smithi130 bash[27579]: audit 2023-08-28T00:08:59.478139+0000 mon.a (mon.0) 2973 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:09:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:00 smithi130 bash[27579]: audit 2023-08-28T00:09:00.097693+0000 mon.a (mon.0) 2974 : audit [DBG] from='client.? 172.21.15.6:0/213873162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:00.662 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:00 smithi006 bash[33918]: cluster 2023-08-28T00:08:59.219010+0000 mon.a (mon.0) 2972 : cluster [DBG] mgrmap e27: y(active, since 2h) 2023-08-28T00:09:00.662 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:00 smithi006 bash[33918]: audit 2023-08-28T00:08:59.478139+0000 mon.a (mon.0) 2973 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:09:00.662 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:00 smithi006 bash[33918]: audit 2023-08-28T00:09:00.097693+0000 mon.a (mon.0) 2974 : audit [DBG] from='client.? 172.21.15.6:0/213873162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:00.662 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:00 smithi006 bash[27292]: cluster 2023-08-28T00:08:59.219010+0000 mon.a (mon.0) 2972 : cluster [DBG] mgrmap e27: y(active, since 2h) 2023-08-28T00:09:00.663 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:00 smithi006 bash[27292]: audit 2023-08-28T00:08:59.478139+0000 mon.a (mon.0) 2973 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:09:00.663 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:00 smithi006 bash[27292]: audit 2023-08-28T00:09:00.097693+0000 mon.a (mon.0) 2974 : audit [DBG] from='client.? 172.21.15.6:0/213873162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:00.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:09:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:09:00] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:09:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:09:00.659Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:09:00.661Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:01.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:01 smithi130 bash[27579]: cluster 2023-08-28T00:09:00.131065+0000 mgr.y (mgr.14901) 3913 : cluster [DBG] pgmap v3920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:01.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:01 smithi130 bash[51302]: debug 2023-08-28T00:09:01.201+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:01.204129+0000) 2023-08-28T00:09:01.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:01 smithi006 bash[27292]: cluster 2023-08-28T00:09:00.131065+0000 mgr.y (mgr.14901) 3913 : cluster [DBG] pgmap v3920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:01.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:01 smithi006 bash[33918]: cluster 2023-08-28T00:09:00.131065+0000 mgr.y (mgr.14901) 3913 : cluster [DBG] pgmap v3920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:02.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:02 smithi130 bash[51302]: debug 2023-08-28T00:09:02.201+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:02.204356+0000) 2023-08-28T00:09:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:09:02.231Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:09:02.232Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:03.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:03 smithi130 bash[27579]: cluster 2023-08-28T00:09:02.131528+0000 mgr.y (mgr.14901) 3914 : cluster [DBG] pgmap v3921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:03.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:03 smithi130 bash[27579]: audit 2023-08-28T00:09:02.506789+0000 mon.c (mon.2) 1483 : audit [DBG] from='client.? 172.21.15.6:0/2875318493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:03.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:03 smithi130 bash[51302]: debug 2023-08-28T00:09:03.201+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:03.204554+0000) 2023-08-28T00:09:03.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:03 smithi006 bash[27292]: cluster 2023-08-28T00:09:02.131528+0000 mgr.y (mgr.14901) 3914 : cluster [DBG] pgmap v3921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:03.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:03 smithi006 bash[27292]: audit 2023-08-28T00:09:02.506789+0000 mon.c (mon.2) 1483 : audit [DBG] from='client.? 172.21.15.6:0/2875318493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:03.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:03 smithi006 bash[33918]: cluster 2023-08-28T00:09:02.131528+0000 mgr.y (mgr.14901) 3914 : cluster [DBG] pgmap v3921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:03.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:03 smithi006 bash[33918]: audit 2023-08-28T00:09:02.506789+0000 mon.c (mon.2) 1483 : audit [DBG] from='client.? 172.21.15.6:0/2875318493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:09:04.116Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:09:04.118Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:04.537 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:04 smithi130 bash[51302]: debug 2023-08-28T00:09:04.201+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:04.204815+0000) 2023-08-28T00:09:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:04 smithi130 bash[27579]: audit 2023-08-28T00:09:03.529429+0000 mon.a (mon.0) 2975 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:09:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:04 smithi130 bash[27579]: cluster 2023-08-28T00:09:04.132567+0000 mgr.y (mgr.14901) 3915 : cluster [DBG] pgmap v3922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2023-08-28T00:09:04.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:04 smithi006 bash[27292]: audit 2023-08-28T00:09:03.529429+0000 mon.a (mon.0) 2975 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:09:04.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:04 smithi006 bash[27292]: cluster 2023-08-28T00:09:04.132567+0000 mgr.y (mgr.14901) 3915 : cluster [DBG] pgmap v3922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2023-08-28T00:09:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:04 smithi006 bash[33918]: audit 2023-08-28T00:09:03.529429+0000 mon.a (mon.0) 2975 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:09:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:04 smithi006 bash[33918]: cluster 2023-08-28T00:09:04.132567+0000 mgr.y (mgr.14901) 3915 : cluster [DBG] pgmap v3922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2023-08-28T00:09:05.542 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:05 smithi130 bash[51302]: debug 2023-08-28T00:09:05.201+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:05.205092+0000) 2023-08-28T00:09:05.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:05 smithi130 bash[27579]: audit 2023-08-28T00:09:04.924300+0000 mon.c (mon.2) 1484 : audit [DBG] from='client.? 172.21.15.6:0/2822197002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:05.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:05 smithi006 bash[27292]: audit 2023-08-28T00:09:04.924300+0000 mon.c (mon.2) 1484 : audit [DBG] from='client.? 172.21.15.6:0/2822197002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:05.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:05 smithi006 bash[33918]: audit 2023-08-28T00:09:04.924300+0000 mon.c (mon.2) 1484 : audit [DBG] from='client.? 172.21.15.6:0/2822197002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:06.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:06 smithi130 bash[51302]: debug 2023-08-28T00:09:06.201+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:06.205393+0000) 2023-08-28T00:09:07.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:07 smithi130 bash[51302]: debug 2023-08-28T00:09:07.201+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:07.205718+0000) 2023-08-28T00:09:08.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:08 smithi130 bash[51302]: debug 2023-08-28T00:09:08.201+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:08.205981+0000) 2023-08-28T00:09:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:08 smithi130 bash[27579]: cluster 2023-08-28T00:09:06.133373+0000 mgr.y (mgr.14901) 3916 : cluster [DBG] pgmap v3923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2023-08-28T00:09:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:08 smithi130 bash[27579]: audit 2023-08-28T00:09:06.415251+0000 mon.a (mon.0) 2976 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T00:09:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:08 smithi006 bash[27292]: cluster 2023-08-28T00:09:06.133373+0000 mgr.y (mgr.14901) 3916 : cluster [DBG] pgmap v3923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2023-08-28T00:09:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:08 smithi006 bash[27292]: audit 2023-08-28T00:09:06.415251+0000 mon.a (mon.0) 2976 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T00:09:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:08 smithi006 bash[33918]: cluster 2023-08-28T00:09:06.133373+0000 mgr.y (mgr.14901) 3916 : cluster [DBG] pgmap v3923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2023-08-28T00:09:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:08 smithi006 bash[33918]: audit 2023-08-28T00:09:06.415251+0000 mon.a (mon.0) 2976 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T00:09:09.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:09 smithi130 bash[51302]: debug 2023-08-28T00:09:09.201+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:09.206269+0000) 2023-08-28T00:09:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:09 smithi130 bash[27579]: audit 2023-08-28T00:09:06.416798+0000 mon.b (mon.1) 490 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T00:09:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:09 smithi130 bash[27579]: audit 2023-08-28T00:09:06.417211+0000 mon.b (mon.1) 491 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T00:09:09.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:09 smithi130 bash[27579]: audit 2023-08-28T00:09:06.417604+0000 mon.b (mon.1) 492 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T00:09:09.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:09 smithi130 bash[27579]: audit 2023-08-28T00:09:07.328381+0000 mon.c (mon.2) 1485 : audit [DBG] from='client.? 172.21.15.6:0/943012331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:09.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:09 smithi130 bash[27579]: audit 2023-08-28T00:09:07.416520+0000 mon.a (mon.0) 2977 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T00:09:09.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:09 smithi130 bash[27579]: audit 2023-08-28T00:09:07.420876+0000 mon.b (mon.1) 493 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T00:09:09.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:09 smithi130 bash[27579]: cluster 2023-08-28T00:09:08.134021+0000 mgr.y (mgr.14901) 3917 : cluster [DBG] pgmap v3924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2023-08-28T00:09:09.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:09 smithi130 bash[27579]: audit 2023-08-28T00:09:08.338718+0000 mon.b (mon.1) 494 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T00:09:09.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:09 smithi130 bash[27579]: audit 2023-08-28T00:09:08.342376+0000 mon.b (mon.1) 495 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T00:09:09.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:09 smithi130 bash[27579]: audit 2023-08-28T00:09:08.342560+0000 mon.b (mon.1) 496 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T00:09:09.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:09 smithi130 bash[27579]: audit 2023-08-28T00:09:08.342737+0000 mon.b (mon.1) 497 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T00:09:09.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:09 smithi130 bash[27579]: audit 2023-08-28T00:09:08.349589+0000 mon.c (mon.2) 1486 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T00:09:09.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:09 smithi130 bash[27579]: audit 2023-08-28T00:09:08.354232+0000 mon.b (mon.1) 498 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T00:09:09.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:09 smithi130 bash[27579]: audit 2023-08-28T00:09:08.354398+0000 mon.b (mon.1) 499 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T00:09:09.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:09 smithi130 bash[27579]: audit 2023-08-28T00:09:08.354541+0000 mon.b (mon.1) 500 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T00:09:09.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:09 smithi130 bash[27579]: audit 2023-08-28T00:09:09.316429+0000 mon.c (mon.2) 1487 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T00:09:09.620 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:09 smithi130 bash[27579]: audit 2023-08-28T00:09:09.324384+0000 mon.a (mon.0) 2978 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:09:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[27292]: audit 2023-08-28T00:09:06.416798+0000 mon.b (mon.1) 490 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T00:09:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[27292]: audit 2023-08-28T00:09:06.417211+0000 mon.b (mon.1) 491 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T00:09:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[27292]: audit 2023-08-28T00:09:06.417604+0000 mon.b (mon.1) 492 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T00:09:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[27292]: audit 2023-08-28T00:09:07.328381+0000 mon.c (mon.2) 1485 : audit [DBG] from='client.? 172.21.15.6:0/943012331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[27292]: audit 2023-08-28T00:09:07.416520+0000 mon.a (mon.0) 2977 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T00:09:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[27292]: audit 2023-08-28T00:09:07.420876+0000 mon.b (mon.1) 493 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T00:09:09.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[27292]: cluster 2023-08-28T00:09:08.134021+0000 mgr.y (mgr.14901) 3917 : cluster [DBG] pgmap v3924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2023-08-28T00:09:09.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[27292]: audit 2023-08-28T00:09:08.338718+0000 mon.b (mon.1) 494 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T00:09:09.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[27292]: audit 2023-08-28T00:09:08.342376+0000 mon.b (mon.1) 495 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T00:09:09.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[27292]: audit 2023-08-28T00:09:08.342560+0000 mon.b (mon.1) 496 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T00:09:09.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[27292]: audit 2023-08-28T00:09:08.342737+0000 mon.b (mon.1) 497 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T00:09:09.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[27292]: audit 2023-08-28T00:09:08.349589+0000 mon.c (mon.2) 1486 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T00:09:09.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[27292]: audit 2023-08-28T00:09:08.354232+0000 mon.b (mon.1) 498 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T00:09:09.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[27292]: audit 2023-08-28T00:09:08.354398+0000 mon.b (mon.1) 499 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T00:09:09.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[27292]: audit 2023-08-28T00:09:08.354541+0000 mon.b (mon.1) 500 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T00:09:09.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[27292]: audit 2023-08-28T00:09:09.316429+0000 mon.c (mon.2) 1487 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T00:09:09.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[27292]: audit 2023-08-28T00:09:09.324384+0000 mon.a (mon.0) 2978 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:09:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[33918]: audit 2023-08-28T00:09:06.416798+0000 mon.b (mon.1) 490 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T00:09:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[33918]: audit 2023-08-28T00:09:06.417211+0000 mon.b (mon.1) 491 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T00:09:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[33918]: audit 2023-08-28T00:09:06.417604+0000 mon.b (mon.1) 492 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T00:09:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[33918]: audit 2023-08-28T00:09:07.328381+0000 mon.c (mon.2) 1485 : audit [DBG] from='client.? 172.21.15.6:0/943012331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[33918]: audit 2023-08-28T00:09:07.416520+0000 mon.a (mon.0) 2977 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T00:09:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[33918]: audit 2023-08-28T00:09:07.420876+0000 mon.b (mon.1) 493 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T00:09:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[33918]: cluster 2023-08-28T00:09:08.134021+0000 mgr.y (mgr.14901) 3917 : cluster [DBG] pgmap v3924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2023-08-28T00:09:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[33918]: audit 2023-08-28T00:09:08.338718+0000 mon.b (mon.1) 494 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T00:09:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[33918]: audit 2023-08-28T00:09:08.342376+0000 mon.b (mon.1) 495 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T00:09:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[33918]: audit 2023-08-28T00:09:08.342560+0000 mon.b (mon.1) 496 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T00:09:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[33918]: audit 2023-08-28T00:09:08.342737+0000 mon.b (mon.1) 497 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T00:09:09.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[33918]: audit 2023-08-28T00:09:08.349589+0000 mon.c (mon.2) 1486 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T00:09:09.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[33918]: audit 2023-08-28T00:09:08.354232+0000 mon.b (mon.1) 498 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T00:09:09.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[33918]: audit 2023-08-28T00:09:08.354398+0000 mon.b (mon.1) 499 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T00:09:09.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[33918]: audit 2023-08-28T00:09:08.354541+0000 mon.b (mon.1) 500 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T00:09:09.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[33918]: audit 2023-08-28T00:09:09.316429+0000 mon.c (mon.2) 1487 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T00:09:09.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:09 smithi006 bash[33918]: audit 2023-08-28T00:09:09.324384+0000 mon.a (mon.0) 2978 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:09:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:10 smithi130 bash[27579]: audit 2023-08-28T00:09:09.738741+0000 mon.c (mon.2) 1488 : audit [DBG] from='client.? 172.21.15.6:0/2874910930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:10 smithi130 bash[27579]: cluster 2023-08-28T00:09:10.135193+0000 mgr.y (mgr.14901) 3918 : cluster [DBG] pgmap v3925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2023-08-28T00:09:10.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:10 smithi130 bash[51302]: debug 2023-08-28T00:09:10.200+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:10.206536+0000) 2023-08-28T00:09:10.663 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:10 smithi006 bash[27292]: audit 2023-08-28T00:09:09.738741+0000 mon.c (mon.2) 1488 : audit [DBG] from='client.? 172.21.15.6:0/2874910930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:10.663 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:10 smithi006 bash[27292]: cluster 2023-08-28T00:09:10.135193+0000 mgr.y (mgr.14901) 3918 : cluster [DBG] pgmap v3925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2023-08-28T00:09:10.663 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:10 smithi006 bash[33918]: audit 2023-08-28T00:09:09.738741+0000 mon.c (mon.2) 1488 : audit [DBG] from='client.? 172.21.15.6:0/2874910930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:10.664 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:10 smithi006 bash[33918]: cluster 2023-08-28T00:09:10.135193+0000 mgr.y (mgr.14901) 3918 : cluster [DBG] pgmap v3925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2023-08-28T00:09:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:09:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:09:10] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:09:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:09:10.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:09:10.662Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:11.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:11 smithi130 bash[51302]: debug 2023-08-28T00:09:11.200+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:11.206762+0000) 2023-08-28T00:09:12.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:12 smithi006 bash[27292]: audit 2023-08-28T00:09:12.155214+0000 mon.c (mon.2) 1489 : audit [DBG] from='client.? 172.21.15.6:0/3562238600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:12.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:12 smithi006 bash[33918]: audit 2023-08-28T00:09:12.155214+0000 mon.c (mon.2) 1489 : audit [DBG] from='client.? 172.21.15.6:0/3562238600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:09:12.231Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:12.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:09:12.234Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:12 smithi130 bash[27579]: audit 2023-08-28T00:09:12.155214+0000 mon.c (mon.2) 1489 : audit [DBG] from='client.? 172.21.15.6:0/3562238600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:12.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:12 smithi130 bash[51302]: debug 2023-08-28T00:09:12.204+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:12.206996+0000) 2023-08-28T00:09:13.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:13 smithi006 bash[27292]: cluster 2023-08-28T00:09:12.135971+0000 mgr.y (mgr.14901) 3919 : cluster [DBG] pgmap v3926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2023-08-28T00:09:13.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:13 smithi006 bash[33918]: cluster 2023-08-28T00:09:12.135971+0000 mgr.y (mgr.14901) 3919 : cluster [DBG] pgmap v3926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2023-08-28T00:09:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:13 smithi130 bash[27579]: cluster 2023-08-28T00:09:12.135971+0000 mgr.y (mgr.14901) 3919 : cluster [DBG] pgmap v3926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2023-08-28T00:09:13.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:13 smithi130 bash[51302]: debug 2023-08-28T00:09:13.204+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:13.207265+0000) 2023-08-28T00:09:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:09:14.117Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:09:14.119Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:14.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:14 smithi130 bash[51302]: debug 2023-08-28T00:09:14.204+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:14.207539+0000) 2023-08-28T00:09:15.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:15 smithi130 bash[51302]: debug 2023-08-28T00:09:15.204+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:15.207887+0000) 2023-08-28T00:09:15.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:15 smithi130 bash[27579]: cluster 2023-08-28T00:09:14.137027+0000 mgr.y (mgr.14901) 3920 : cluster [DBG] pgmap v3927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2023-08-28T00:09:15.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:15 smithi130 bash[27579]: audit 2023-08-28T00:09:14.569874+0000 mon.a (mon.0) 2979 : audit [DBG] from='client.? 172.21.15.6:0/3924093262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:15.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:15 smithi006 bash[27292]: cluster 2023-08-28T00:09:14.137027+0000 mgr.y (mgr.14901) 3920 : cluster [DBG] pgmap v3927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2023-08-28T00:09:15.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:15 smithi006 bash[27292]: audit 2023-08-28T00:09:14.569874+0000 mon.a (mon.0) 2979 : audit [DBG] from='client.? 172.21.15.6:0/3924093262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:15.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:15 smithi006 bash[33918]: cluster 2023-08-28T00:09:14.137027+0000 mgr.y (mgr.14901) 3920 : cluster [DBG] pgmap v3927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2023-08-28T00:09:15.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:15 smithi006 bash[33918]: audit 2023-08-28T00:09:14.569874+0000 mon.a (mon.0) 2979 : audit [DBG] from='client.? 172.21.15.6:0/3924093262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:16.479 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:16 smithi130 bash[51302]: debug 2023-08-28T00:09:16.204+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:16.208160+0000) 2023-08-28T00:09:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:16 smithi130 bash[27579]: cluster 2023-08-28T00:09:16.137745+0000 mgr.y (mgr.14901) 3921 : cluster [DBG] pgmap v3928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:16 smithi006 bash[27292]: cluster 2023-08-28T00:09:16.137745+0000 mgr.y (mgr.14901) 3921 : cluster [DBG] pgmap v3928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:16 smithi006 bash[33918]: cluster 2023-08-28T00:09:16.137745+0000 mgr.y (mgr.14901) 3921 : cluster [DBG] pgmap v3928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:17.482 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:17 smithi130 bash[51302]: debug 2023-08-28T00:09:17.204+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:17.208422+0000) 2023-08-28T00:09:17.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:17 smithi130 bash[27579]: audit 2023-08-28T00:09:16.986088+0000 mon.a (mon.0) 2980 : audit [DBG] from='client.? 172.21.15.6:0/4281680815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:17.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:17 smithi006 bash[33918]: audit 2023-08-28T00:09:16.986088+0000 mon.a (mon.0) 2980 : audit [DBG] from='client.? 172.21.15.6:0/4281680815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:17.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:17 smithi006 bash[27292]: audit 2023-08-28T00:09:16.986088+0000 mon.a (mon.0) 2980 : audit [DBG] from='client.? 172.21.15.6:0/4281680815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:18.491 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:18 smithi130 bash[51302]: debug 2023-08-28T00:09:18.204+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:18.208663+0000) 2023-08-28T00:09:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:18 smithi130 bash[27579]: cluster 2023-08-28T00:09:18.138454+0000 mgr.y (mgr.14901) 3922 : cluster [DBG] pgmap v3929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:18 smithi006 bash[33918]: cluster 2023-08-28T00:09:18.138454+0000 mgr.y (mgr.14901) 3922 : cluster [DBG] pgmap v3929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:18 smithi006 bash[27292]: cluster 2023-08-28T00:09:18.138454+0000 mgr.y (mgr.14901) 3922 : cluster [DBG] pgmap v3929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:19.494 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:19 smithi130 bash[51302]: debug 2023-08-28T00:09:19.204+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:19.208959+0000) 2023-08-28T00:09:19.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:19 smithi130 bash[27579]: audit 2023-08-28T00:09:19.403021+0000 mon.c (mon.2) 1490 : audit [DBG] from='client.? 172.21.15.6:0/1644327904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:19 smithi006 bash[33918]: audit 2023-08-28T00:09:19.403021+0000 mon.c (mon.2) 1490 : audit [DBG] from='client.? 172.21.15.6:0/1644327904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:19.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:19 smithi006 bash[27292]: audit 2023-08-28T00:09:19.403021+0000 mon.c (mon.2) 1490 : audit [DBG] from='client.? 172.21.15.6:0/1644327904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:20.500 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:20 smithi130 bash[51302]: debug 2023-08-28T00:09:20.204+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:20.209232+0000) 2023-08-28T00:09:20.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:20 smithi006 bash[33918]: cluster 2023-08-28T00:09:20.139601+0000 mgr.y (mgr.14901) 3923 : cluster [DBG] pgmap v3930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:20.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:09:20.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:20.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:09:20.663Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:20.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:20 smithi006 bash[27292]: cluster 2023-08-28T00:09:20.139601+0000 mgr.y (mgr.14901) 3923 : cluster [DBG] pgmap v3930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:20 smithi130 bash[27579]: cluster 2023-08-28T00:09:20.139601+0000 mgr.y (mgr.14901) 3923 : cluster [DBG] pgmap v3930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:09:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:09:20] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:09:21.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:21 smithi130 bash[51302]: debug 2023-08-28T00:09:21.204+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:21.209567+0000) 2023-08-28T00:09:22.211 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:21 smithi130 bash[27579]: audit 2023-08-28T00:09:21.818555+0000 mon.a (mon.0) 2981 : audit [DBG] from='client.? 172.21.15.6:0/2986640021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:22.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:21 smithi006 bash[33918]: audit 2023-08-28T00:09:21.818555+0000 mon.a (mon.0) 2981 : audit [DBG] from='client.? 172.21.15.6:0/2986640021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:22.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:21 smithi006 bash[27292]: audit 2023-08-28T00:09:21.818555+0000 mon.a (mon.0) 2981 : audit [DBG] from='client.? 172.21.15.6:0/2986640021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:22.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:22 smithi130 bash[51302]: debug 2023-08-28T00:09:22.204+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:22.209800+0000) 2023-08-28T00:09:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:09:22.232Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:09:22.235Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:23.212 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:22 smithi130 bash[27579]: cluster 2023-08-28T00:09:22.140307+0000 mgr.y (mgr.14901) 3924 : cluster [DBG] pgmap v3931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:23.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:22 smithi006 bash[33918]: cluster 2023-08-28T00:09:22.140307+0000 mgr.y (mgr.14901) 3924 : cluster [DBG] pgmap v3931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:22 smithi006 bash[27292]: cluster 2023-08-28T00:09:22.140307+0000 mgr.y (mgr.14901) 3924 : cluster [DBG] pgmap v3931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:23.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:23 smithi130 bash[51302]: debug 2023-08-28T00:09:23.208+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:23.210092+0000) 2023-08-28T00:09:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:09:24.118Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:09:24.120Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:24.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:24 smithi130 bash[51302]: debug 2023-08-28T00:09:24.204+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:24.210316+0000) 2023-08-28T00:09:25.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:25 smithi006 bash[27292]: cluster 2023-08-28T00:09:24.141212+0000 mgr.y (mgr.14901) 3925 : cluster [DBG] pgmap v3932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:25.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:25 smithi006 bash[27292]: audit 2023-08-28T00:09:24.231966+0000 mon.c (mon.2) 1491 : audit [DBG] from='client.? 172.21.15.6:0/1249384905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:25.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:25 smithi006 bash[33918]: cluster 2023-08-28T00:09:24.141212+0000 mgr.y (mgr.14901) 3925 : cluster [DBG] pgmap v3932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:25.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:25 smithi006 bash[33918]: audit 2023-08-28T00:09:24.231966+0000 mon.c (mon.2) 1491 : audit [DBG] from='client.? 172.21.15.6:0/1249384905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:25.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:25 smithi130 bash[51302]: debug 2023-08-28T00:09:25.204+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:25.210572+0000) 2023-08-28T00:09:25.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:25 smithi130 bash[27579]: cluster 2023-08-28T00:09:24.141212+0000 mgr.y (mgr.14901) 3925 : cluster [DBG] pgmap v3932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:25.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:25 smithi130 bash[27579]: audit 2023-08-28T00:09:24.231966+0000 mon.c (mon.2) 1491 : audit [DBG] from='client.? 172.21.15.6:0/1249384905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:26.480 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:26 smithi130 bash[51302]: debug 2023-08-28T00:09:26.204+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:26.210787+0000) 2023-08-28T00:09:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:26 smithi130 bash[27579]: cluster 2023-08-28T00:09:26.142107+0000 mgr.y (mgr.14901) 3926 : cluster [DBG] pgmap v3933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:26 smithi006 bash[27292]: cluster 2023-08-28T00:09:26.142107+0000 mgr.y (mgr.14901) 3926 : cluster [DBG] pgmap v3933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:26 smithi006 bash[33918]: cluster 2023-08-28T00:09:26.142107+0000 mgr.y (mgr.14901) 3926 : cluster [DBG] pgmap v3933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:27.483 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:27 smithi130 bash[51302]: debug 2023-08-28T00:09:27.208+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:27.211040+0000) 2023-08-28T00:09:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:27 smithi130 bash[27579]: audit 2023-08-28T00:09:26.647314+0000 mon.c (mon.2) 1492 : audit [DBG] from='client.? 172.21.15.6:0/1549856007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:27.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:27 smithi006 bash[27292]: audit 2023-08-28T00:09:26.647314+0000 mon.c (mon.2) 1492 : audit [DBG] from='client.? 172.21.15.6:0/1549856007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:27.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:27 smithi006 bash[33918]: audit 2023-08-28T00:09:26.647314+0000 mon.c (mon.2) 1492 : audit [DBG] from='client.? 172.21.15.6:0/1549856007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:28.491 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:28 smithi130 bash[51302]: debug 2023-08-28T00:09:28.208+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:28.211287+0000) 2023-08-28T00:09:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:28 smithi130 bash[27579]: cluster 2023-08-28T00:09:28.142840+0000 mgr.y (mgr.14901) 3927 : cluster [DBG] pgmap v3934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:28 smithi006 bash[27292]: cluster 2023-08-28T00:09:28.142840+0000 mgr.y (mgr.14901) 3927 : cluster [DBG] pgmap v3934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:28 smithi006 bash[33918]: cluster 2023-08-28T00:09:28.142840+0000 mgr.y (mgr.14901) 3927 : cluster [DBG] pgmap v3934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:29.494 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:29 smithi130 bash[51302]: debug 2023-08-28T00:09:29.208+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:29.211575+0000) 2023-08-28T00:09:29.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:29 smithi130 bash[27579]: audit 2023-08-28T00:09:29.059748+0000 mon.a (mon.0) 2982 : audit [DBG] from='client.? 172.21.15.6:0/1102186071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:29.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:29 smithi006 bash[27292]: audit 2023-08-28T00:09:29.059748+0000 mon.a (mon.0) 2982 : audit [DBG] from='client.? 172.21.15.6:0/1102186071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:29.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:29 smithi006 bash[33918]: audit 2023-08-28T00:09:29.059748+0000 mon.a (mon.0) 2982 : audit [DBG] from='client.? 172.21.15.6:0/1102186071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:30.501 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:30 smithi130 bash[51302]: debug 2023-08-28T00:09:30.208+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:30.211885+0000) 2023-08-28T00:09:30.754 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:30 smithi006 bash[33918]: cluster 2023-08-28T00:09:30.143993+0000 mgr.y (mgr.14901) 3928 : cluster [DBG] pgmap v3935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:30.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:09:30.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:30.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:09:30.663Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:30.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:30 smithi006 bash[27292]: cluster 2023-08-28T00:09:30.143993+0000 mgr.y (mgr.14901) 3928 : cluster [DBG] pgmap v3935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:30 smithi130 bash[27579]: cluster 2023-08-28T00:09:30.143993+0000 mgr.y (mgr.14901) 3928 : cluster [DBG] pgmap v3935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:09:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:09:30] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T00:09:31.503 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:31 smithi130 bash[51302]: debug 2023-08-28T00:09:31.208+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:31.212166+0000) 2023-08-28T00:09:31.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:31 smithi130 bash[27579]: audit 2023-08-28T00:09:31.476360+0000 mon.a (mon.0) 2983 : audit [DBG] from='client.? 172.21.15.6:0/3793182004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:31.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:31 smithi006 bash[27292]: audit 2023-08-28T00:09:31.476360+0000 mon.a (mon.0) 2983 : audit [DBG] from='client.? 172.21.15.6:0/3793182004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:31.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:31 smithi006 bash[33918]: audit 2023-08-28T00:09:31.476360+0000 mon.a (mon.0) 2983 : audit [DBG] from='client.? 172.21.15.6:0/3793182004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:32.514 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:09:32.233Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:32.514 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:09:32.235Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:32.514 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:32 smithi130 bash[51302]: debug 2023-08-28T00:09:32.208+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:32.212437+0000) 2023-08-28T00:09:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:32 smithi130 bash[27579]: cluster 2023-08-28T00:09:32.144705+0000 mgr.y (mgr.14901) 3929 : cluster [DBG] pgmap v3936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:32 smithi006 bash[27292]: cluster 2023-08-28T00:09:32.144705+0000 mgr.y (mgr.14901) 3929 : cluster [DBG] pgmap v3936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:32 smithi006 bash[33918]: cluster 2023-08-28T00:09:32.144705+0000 mgr.y (mgr.14901) 3929 : cluster [DBG] pgmap v3936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:33.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:33 smithi130 bash[51302]: debug 2023-08-28T00:09:33.208+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:33.212708+0000) 2023-08-28T00:09:34.214 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:33 smithi130 bash[27579]: audit 2023-08-28T00:09:33.890089+0000 mon.c (mon.2) 1493 : audit [DBG] from='client.? 172.21.15.6:0/4232729888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:34.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:33 smithi006 bash[27292]: audit 2023-08-28T00:09:33.890089+0000 mon.c (mon.2) 1493 : audit [DBG] from='client.? 172.21.15.6:0/4232729888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:34.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:33 smithi006 bash[33918]: audit 2023-08-28T00:09:33.890089+0000 mon.c (mon.2) 1493 : audit [DBG] from='client.? 172.21.15.6:0/4232729888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:09:34.118Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:34.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:09:34.122Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:34.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:34 smithi130 bash[51302]: debug 2023-08-28T00:09:34.208+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:34.212942+0000) 2023-08-28T00:09:35.214 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:34 smithi130 bash[27579]: cluster 2023-08-28T00:09:34.145762+0000 mgr.y (mgr.14901) 3930 : cluster [DBG] pgmap v3937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:35.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:34 smithi006 bash[27292]: cluster 2023-08-28T00:09:34.145762+0000 mgr.y (mgr.14901) 3930 : cluster [DBG] pgmap v3937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:34 smithi006 bash[33918]: cluster 2023-08-28T00:09:34.145762+0000 mgr.y (mgr.14901) 3930 : cluster [DBG] pgmap v3937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:35.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:35 smithi130 bash[51302]: debug 2023-08-28T00:09:35.212+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:35.213215+0000) 2023-08-28T00:09:36.484 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:36 smithi130 bash[51302]: debug 2023-08-28T00:09:36.212+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:36.213505+0000) 2023-08-28T00:09:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:36 smithi130 bash[27579]: cluster 2023-08-28T00:09:36.146284+0000 mgr.y (mgr.14901) 3931 : cluster [DBG] pgmap v3938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:36 smithi130 bash[27579]: audit 2023-08-28T00:09:36.311689+0000 mon.a (mon.0) 2984 : audit [DBG] from='client.? 172.21.15.6:0/4150237153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:36 smithi006 bash[33918]: cluster 2023-08-28T00:09:36.146284+0000 mgr.y (mgr.14901) 3931 : cluster [DBG] pgmap v3938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:36 smithi006 bash[33918]: audit 2023-08-28T00:09:36.311689+0000 mon.a (mon.0) 2984 : audit [DBG] from='client.? 172.21.15.6:0/4150237153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:36 smithi006 bash[27292]: cluster 2023-08-28T00:09:36.146284+0000 mgr.y (mgr.14901) 3931 : cluster [DBG] pgmap v3938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:36 smithi006 bash[27292]: audit 2023-08-28T00:09:36.311689+0000 mon.a (mon.0) 2984 : audit [DBG] from='client.? 172.21.15.6:0/4150237153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:37.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:37 smithi130 bash[51302]: debug 2023-08-28T00:09:37.212+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:37.213762+0000) 2023-08-28T00:09:38.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:38 smithi130 bash[51302]: debug 2023-08-28T00:09:38.208+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:38.214006+0000) 2023-08-28T00:09:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:39 smithi006 bash[27292]: cluster 2023-08-28T00:09:38.146903+0000 mgr.y (mgr.14901) 3932 : cluster [DBG] pgmap v3939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:39 smithi006 bash[27292]: audit 2023-08-28T00:09:38.724143+0000 mon.c (mon.2) 1494 : audit [DBG] from='client.? 172.21.15.6:0/1219701997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:39 smithi006 bash[27292]: audit 2023-08-28T00:09:38.881324+0000 mon.b (mon.1) 501 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:09:39.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:39 smithi006 bash[33918]: cluster 2023-08-28T00:09:38.146903+0000 mgr.y (mgr.14901) 3932 : cluster [DBG] pgmap v3939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:39.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:39 smithi006 bash[33918]: audit 2023-08-28T00:09:38.724143+0000 mon.c (mon.2) 1494 : audit [DBG] from='client.? 172.21.15.6:0/1219701997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:39.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:39 smithi006 bash[33918]: audit 2023-08-28T00:09:38.881324+0000 mon.b (mon.1) 501 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:09:39.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:39 smithi130 bash[51302]: debug 2023-08-28T00:09:39.208+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:39.214281+0000) 2023-08-28T00:09:39.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:39 smithi130 bash[27579]: cluster 2023-08-28T00:09:38.146903+0000 mgr.y (mgr.14901) 3932 : cluster [DBG] pgmap v3939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:39.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:39 smithi130 bash[27579]: audit 2023-08-28T00:09:38.724143+0000 mon.c (mon.2) 1494 : audit [DBG] from='client.? 172.21.15.6:0/1219701997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:39.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:39 smithi130 bash[27579]: audit 2023-08-28T00:09:38.881324+0000 mon.b (mon.1) 501 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:09:40.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:40 smithi130 bash[51302]: debug 2023-08-28T00:09:40.208+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:40.214538+0000) 2023-08-28T00:09:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:09:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:09:40] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T00:09:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:09:40.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:09:40.664Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:41.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:41 smithi006 bash[33918]: cluster 2023-08-28T00:09:40.148052+0000 mgr.y (mgr.14901) 3933 : cluster [DBG] pgmap v3940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:41.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:41 smithi006 bash[33918]: audit 2023-08-28T00:09:41.147732+0000 mon.c (mon.2) 1495 : audit [DBG] from='client.? 172.21.15.6:0/3303609350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:41.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:41 smithi006 bash[27292]: cluster 2023-08-28T00:09:40.148052+0000 mgr.y (mgr.14901) 3933 : cluster [DBG] pgmap v3940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:41.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:41 smithi006 bash[27292]: audit 2023-08-28T00:09:41.147732+0000 mon.c (mon.2) 1495 : audit [DBG] from='client.? 172.21.15.6:0/3303609350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:41.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:41 smithi130 bash[51302]: debug 2023-08-28T00:09:41.208+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:41.214769+0000) 2023-08-28T00:09:41.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:41 smithi130 bash[27579]: cluster 2023-08-28T00:09:40.148052+0000 mgr.y (mgr.14901) 3933 : cluster [DBG] pgmap v3940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:41.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:41 smithi130 bash[27579]: audit 2023-08-28T00:09:41.147732+0000 mon.c (mon.2) 1495 : audit [DBG] from='client.? 172.21.15.6:0/3303609350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:42.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:42 smithi130 bash[51302]: debug 2023-08-28T00:09:42.212+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:42.214987+0000) 2023-08-28T00:09:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:09:42.234Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:09:42.236Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:43.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:43 smithi006 bash[33918]: cluster 2023-08-28T00:09:42.148739+0000 mgr.y (mgr.14901) 3934 : cluster [DBG] pgmap v3941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:43.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:43 smithi006 bash[27292]: cluster 2023-08-28T00:09:42.148739+0000 mgr.y (mgr.14901) 3934 : cluster [DBG] pgmap v3941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:43 smithi130 bash[27579]: cluster 2023-08-28T00:09:42.148739+0000 mgr.y (mgr.14901) 3934 : cluster [DBG] pgmap v3941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:43.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:43 smithi130 bash[51302]: debug 2023-08-28T00:09:43.212+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:43.215246+0000) 2023-08-28T00:09:44.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:44 smithi006 bash[33918]: audit 2023-08-28T00:09:43.572239+0000 mon.a (mon.0) 2985 : audit [DBG] from='client.? 172.21.15.6:0/2086194637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:09:44.119Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:09:44.121Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:44.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:44 smithi006 bash[27292]: audit 2023-08-28T00:09:43.572239+0000 mon.a (mon.0) 2985 : audit [DBG] from='client.? 172.21.15.6:0/2086194637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:44 smithi130 bash[27579]: audit 2023-08-28T00:09:43.572239+0000 mon.a (mon.0) 2985 : audit [DBG] from='client.? 172.21.15.6:0/2086194637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:44.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:44 smithi130 bash[51302]: debug 2023-08-28T00:09:44.212+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:44.215523+0000) 2023-08-28T00:09:45.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:45 smithi130 bash[27579]: cluster 2023-08-28T00:09:44.149670+0000 mgr.y (mgr.14901) 3935 : cluster [DBG] pgmap v3942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:45.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:45 smithi130 bash[27579]: audit 2023-08-28T00:09:44.819311+0000 mon.a (mon.0) 2986 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:09:45.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:45 smithi130 bash[51302]: debug 2023-08-28T00:09:45.212+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:45.215718+0000) 2023-08-28T00:09:45.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:45 smithi006 bash[33918]: cluster 2023-08-28T00:09:44.149670+0000 mgr.y (mgr.14901) 3935 : cluster [DBG] pgmap v3942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:45.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:45 smithi006 bash[33918]: audit 2023-08-28T00:09:44.819311+0000 mon.a (mon.0) 2986 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:09:45.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:45 smithi006 bash[27292]: cluster 2023-08-28T00:09:44.149670+0000 mgr.y (mgr.14901) 3935 : cluster [DBG] pgmap v3942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:45.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:45 smithi006 bash[27292]: audit 2023-08-28T00:09:44.819311+0000 mon.a (mon.0) 2986 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:09:46.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:46 smithi130 bash[51302]: debug 2023-08-28T00:09:46.212+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:46.216011+0000) 2023-08-28T00:09:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:46 smithi130 bash[27579]: audit 2023-08-28T00:09:45.278142+0000 mon.a (mon.0) 2987 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:09:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:46 smithi130 bash[27579]: audit 2023-08-28T00:09:45.548181+0000 mon.b (mon.1) 502 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:09:46.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:46 smithi130 bash[27579]: audit 2023-08-28T00:09:45.550599+0000 mon.b (mon.1) 503 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:09:46.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:46 smithi130 bash[27579]: audit 2023-08-28T00:09:45.560678+0000 mon.a (mon.0) 2988 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:09:46.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:46 smithi130 bash[27579]: audit 2023-08-28T00:09:45.983533+0000 mon.a (mon.0) 2989 : audit [DBG] from='client.? 172.21.15.6:0/681297952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:46.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:46 smithi130 bash[27579]: cluster 2023-08-28T00:09:46.150382+0000 mgr.y (mgr.14901) 3936 : cluster [DBG] pgmap v3943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:46 smithi006 bash[33918]: audit 2023-08-28T00:09:45.278142+0000 mon.a (mon.0) 2987 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:09:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:46 smithi006 bash[33918]: audit 2023-08-28T00:09:45.548181+0000 mon.b (mon.1) 502 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:09:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:46 smithi006 bash[33918]: audit 2023-08-28T00:09:45.550599+0000 mon.b (mon.1) 503 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:09:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:46 smithi006 bash[33918]: audit 2023-08-28T00:09:45.560678+0000 mon.a (mon.0) 2988 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:09:46.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:46 smithi006 bash[33918]: audit 2023-08-28T00:09:45.983533+0000 mon.a (mon.0) 2989 : audit [DBG] from='client.? 172.21.15.6:0/681297952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:46.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:46 smithi006 bash[33918]: cluster 2023-08-28T00:09:46.150382+0000 mgr.y (mgr.14901) 3936 : cluster [DBG] pgmap v3943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:46 smithi006 bash[27292]: audit 2023-08-28T00:09:45.278142+0000 mon.a (mon.0) 2987 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:09:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:46 smithi006 bash[27292]: audit 2023-08-28T00:09:45.548181+0000 mon.b (mon.1) 502 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:09:46.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:46 smithi006 bash[27292]: audit 2023-08-28T00:09:45.550599+0000 mon.b (mon.1) 503 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:09:46.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:46 smithi006 bash[27292]: audit 2023-08-28T00:09:45.560678+0000 mon.a (mon.0) 2988 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:09:46.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:46 smithi006 bash[27292]: audit 2023-08-28T00:09:45.983533+0000 mon.a (mon.0) 2989 : audit [DBG] from='client.? 172.21.15.6:0/681297952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:46.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:46 smithi006 bash[27292]: cluster 2023-08-28T00:09:46.150382+0000 mgr.y (mgr.14901) 3936 : cluster [DBG] pgmap v3943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:47.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:47 smithi130 bash[51302]: debug 2023-08-28T00:09:47.212+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:47.216363+0000) 2023-08-28T00:09:48.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:48 smithi130 bash[51302]: debug 2023-08-28T00:09:48.212+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:48.216715+0000) 2023-08-28T00:09:49.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:49 smithi006 bash[27292]: cluster 2023-08-28T00:09:48.150927+0000 mgr.y (mgr.14901) 3937 : cluster [DBG] pgmap v3944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:49.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:49 smithi006 bash[27292]: audit 2023-08-28T00:09:48.406063+0000 mon.a (mon.0) 2990 : audit [DBG] from='client.? 172.21.15.6:0/4162604289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:49.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:49 smithi006 bash[33918]: cluster 2023-08-28T00:09:48.150927+0000 mgr.y (mgr.14901) 3937 : cluster [DBG] pgmap v3944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:49.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:49 smithi006 bash[33918]: audit 2023-08-28T00:09:48.406063+0000 mon.a (mon.0) 2990 : audit [DBG] from='client.? 172.21.15.6:0/4162604289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:49.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:49 smithi130 bash[27579]: cluster 2023-08-28T00:09:48.150927+0000 mgr.y (mgr.14901) 3937 : cluster [DBG] pgmap v3944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:49.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:49 smithi130 bash[27579]: audit 2023-08-28T00:09:48.406063+0000 mon.a (mon.0) 2990 : audit [DBG] from='client.? 172.21.15.6:0/4162604289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:49.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:49 smithi130 bash[51302]: debug 2023-08-28T00:09:49.212+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:49.217056+0000) 2023-08-28T00:09:50.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:50 smithi130 bash[51302]: debug 2023-08-28T00:09:50.212+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:50.217306+0000) 2023-08-28T00:09:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:09:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:09:50] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:09:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:09:50.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:09:50.664Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:51.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:51 smithi006 bash[27292]: cluster 2023-08-28T00:09:50.152044+0000 mgr.y (mgr.14901) 3938 : cluster [DBG] pgmap v3945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:51.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:51 smithi006 bash[27292]: audit 2023-08-28T00:09:50.825566+0000 mon.a (mon.0) 2991 : audit [DBG] from='client.? 172.21.15.6:0/2760345307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:51.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:51 smithi006 bash[33918]: cluster 2023-08-28T00:09:50.152044+0000 mgr.y (mgr.14901) 3938 : cluster [DBG] pgmap v3945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:51.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:51 smithi006 bash[33918]: audit 2023-08-28T00:09:50.825566+0000 mon.a (mon.0) 2991 : audit [DBG] from='client.? 172.21.15.6:0/2760345307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:51.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:51 smithi130 bash[27579]: cluster 2023-08-28T00:09:50.152044+0000 mgr.y (mgr.14901) 3938 : cluster [DBG] pgmap v3945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:51.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:51 smithi130 bash[27579]: audit 2023-08-28T00:09:50.825566+0000 mon.a (mon.0) 2991 : audit [DBG] from='client.? 172.21.15.6:0/2760345307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:51.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:51 smithi130 bash[51302]: debug 2023-08-28T00:09:51.212+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:51.217545+0000) 2023-08-28T00:09:52.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:52 smithi130 bash[51302]: debug 2023-08-28T00:09:52.212+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:52.217871+0000) 2023-08-28T00:09:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:09:52.235Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:09:52.236Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:53 smithi130 bash[27579]: cluster 2023-08-28T00:09:52.152697+0000 mgr.y (mgr.14901) 3939 : cluster [DBG] pgmap v3946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:53.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:53 smithi130 bash[51302]: debug 2023-08-28T00:09:53.212+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:53.218211+0000) 2023-08-28T00:09:53.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:53 smithi006 bash[27292]: cluster 2023-08-28T00:09:52.152697+0000 mgr.y (mgr.14901) 3939 : cluster [DBG] pgmap v3946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:53.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:53 smithi006 bash[33918]: cluster 2023-08-28T00:09:52.152697+0000 mgr.y (mgr.14901) 3939 : cluster [DBG] pgmap v3946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:09:54.120Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:09:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:09:54.122Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:09:54.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:54 smithi006 bash[27292]: audit 2023-08-28T00:09:53.245831+0000 mon.c (mon.2) 1496 : audit [DBG] from='client.? 172.21.15.6:0/645615018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:54.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:54 smithi006 bash[33918]: audit 2023-08-28T00:09:53.245831+0000 mon.c (mon.2) 1496 : audit [DBG] from='client.? 172.21.15.6:0/645615018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:54 smithi130 bash[27579]: audit 2023-08-28T00:09:53.245831+0000 mon.c (mon.2) 1496 : audit [DBG] from='client.? 172.21.15.6:0/645615018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:54.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:54 smithi130 bash[51302]: debug 2023-08-28T00:09:54.212+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:54.218738+0000) 2023-08-28T00:09:55.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:55 smithi130 bash[27579]: cluster 2023-08-28T00:09:54.153732+0000 mgr.y (mgr.14901) 3940 : cluster [DBG] pgmap v3947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:55.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:55 smithi130 bash[51302]: debug 2023-08-28T00:09:55.216+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:55.219091+0000) 2023-08-28T00:09:55.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:55 smithi006 bash[27292]: cluster 2023-08-28T00:09:54.153732+0000 mgr.y (mgr.14901) 3940 : cluster [DBG] pgmap v3947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:55.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:55 smithi006 bash[33918]: cluster 2023-08-28T00:09:54.153732+0000 mgr.y (mgr.14901) 3940 : cluster [DBG] pgmap v3947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:56.496 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:56 smithi130 bash[51302]: debug 2023-08-28T00:09:56.216+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:56.219413+0000) 2023-08-28T00:09:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:56 smithi130 bash[27579]: audit 2023-08-28T00:09:55.663682+0000 mon.c (mon.2) 1497 : audit [DBG] from='client.? 172.21.15.6:0/4270486745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:56 smithi130 bash[27579]: cluster 2023-08-28T00:09:56.154483+0000 mgr.y (mgr.14901) 3941 : cluster [DBG] pgmap v3948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:56.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:56 smithi006 bash[33918]: audit 2023-08-28T00:09:55.663682+0000 mon.c (mon.2) 1497 : audit [DBG] from='client.? 172.21.15.6:0/4270486745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:56 smithi006 bash[33918]: cluster 2023-08-28T00:09:56.154483+0000 mgr.y (mgr.14901) 3941 : cluster [DBG] pgmap v3948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:56 smithi006 bash[27292]: audit 2023-08-28T00:09:55.663682+0000 mon.c (mon.2) 1497 : audit [DBG] from='client.? 172.21.15.6:0/4270486745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:56 smithi006 bash[27292]: cluster 2023-08-28T00:09:56.154483+0000 mgr.y (mgr.14901) 3941 : cluster [DBG] pgmap v3948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:57.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:57 smithi130 bash[51302]: debug 2023-08-28T00:09:57.216+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:57.219758+0000) 2023-08-28T00:09:58.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:58 smithi006 bash[27292]: audit 2023-08-28T00:09:58.078284+0000 mon.c (mon.2) 1498 : audit [DBG] from='client.? 172.21.15.6:0/1728769305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:58.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:58 smithi006 bash[33918]: audit 2023-08-28T00:09:58.078284+0000 mon.c (mon.2) 1498 : audit [DBG] from='client.? 172.21.15.6:0/1728769305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:58 smithi130 bash[27579]: audit 2023-08-28T00:09:58.078284+0000 mon.c (mon.2) 1498 : audit [DBG] from='client.? 172.21.15.6:0/1728769305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:58.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:58 smithi130 bash[51302]: debug 2023-08-28T00:09:58.216+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:58.220092+0000) 2023-08-28T00:09:59.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:09:59 smithi006 bash[27292]: cluster 2023-08-28T00:09:58.155125+0000 mgr.y (mgr.14901) 3942 : cluster [DBG] pgmap v3949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:09:59 smithi006 bash[33918]: cluster 2023-08-28T00:09:58.155125+0000 mgr.y (mgr.14901) 3942 : cluster [DBG] pgmap v3949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:09:59 smithi130 bash[27579]: cluster 2023-08-28T00:09:58.155125+0000 mgr.y (mgr.14901) 3942 : cluster [DBG] pgmap v3949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:09:59.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:09:59 smithi130 bash[51302]: debug 2023-08-28T00:09:59.220+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:59.220476+0000) 2023-08-28T00:10:00.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:00 smithi006 bash[33918]: cluster 2023-08-28T00:10:00.000151+0000 mon.a (mon.0) 2992 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:10:00.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:00 smithi006 bash[27292]: cluster 2023-08-28T00:10:00.000151+0000 mon.a (mon.0) 2992 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:10:00.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:00 smithi130 bash[51302]: debug 2023-08-28T00:10:00.216+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:00.220779+0000) 2023-08-28T00:10:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:00 smithi130 bash[27579]: cluster 2023-08-28T00:10:00.000151+0000 mon.a (mon.0) 2992 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:10:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:10:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:10:00] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:10:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:10:00.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:10:00.665Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:01.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:01 smithi006 bash[33918]: cluster 2023-08-28T00:10:00.155857+0000 mgr.y (mgr.14901) 3943 : cluster [DBG] pgmap v3950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:01.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:01 smithi006 bash[33918]: audit 2023-08-28T00:10:00.496656+0000 mon.c (mon.2) 1499 : audit [DBG] from='client.? 172.21.15.6:0/2175252994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:01.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:01 smithi006 bash[27292]: cluster 2023-08-28T00:10:00.155857+0000 mgr.y (mgr.14901) 3943 : cluster [DBG] pgmap v3950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:01.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:01 smithi006 bash[27292]: audit 2023-08-28T00:10:00.496656+0000 mon.c (mon.2) 1499 : audit [DBG] from='client.? 172.21.15.6:0/2175252994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:01.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:01 smithi130 bash[27579]: cluster 2023-08-28T00:10:00.155857+0000 mgr.y (mgr.14901) 3943 : cluster [DBG] pgmap v3950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:01.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:01 smithi130 bash[27579]: audit 2023-08-28T00:10:00.496656+0000 mon.c (mon.2) 1499 : audit [DBG] from='client.? 172.21.15.6:0/2175252994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:01.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:01 smithi130 bash[51302]: debug 2023-08-28T00:10:01.220+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:01.221048+0000) 2023-08-28T00:10:02.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:02 smithi130 bash[51302]: debug 2023-08-28T00:10:02.216+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:02.221222+0000) 2023-08-28T00:10:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:10:02.236Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:10:02.238Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:03.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:03 smithi006 bash[33918]: cluster 2023-08-28T00:10:02.156476+0000 mgr.y (mgr.14901) 3944 : cluster [DBG] pgmap v3951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:03.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:03 smithi006 bash[33918]: audit 2023-08-28T00:10:02.910407+0000 mon.c (mon.2) 1500 : audit [DBG] from='client.? 172.21.15.6:0/3768394845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:03.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:03 smithi006 bash[27292]: cluster 2023-08-28T00:10:02.156476+0000 mgr.y (mgr.14901) 3944 : cluster [DBG] pgmap v3951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:03.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:03 smithi006 bash[27292]: audit 2023-08-28T00:10:02.910407+0000 mon.c (mon.2) 1500 : audit [DBG] from='client.? 172.21.15.6:0/3768394845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:03.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:03 smithi130 bash[51302]: debug 2023-08-28T00:10:03.216+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:03.221506+0000) 2023-08-28T00:10:03.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:03 smithi130 bash[27579]: cluster 2023-08-28T00:10:02.156476+0000 mgr.y (mgr.14901) 3944 : cluster [DBG] pgmap v3951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:03.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:03 smithi130 bash[27579]: audit 2023-08-28T00:10:02.910407+0000 mon.c (mon.2) 1500 : audit [DBG] from='client.? 172.21.15.6:0/3768394845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:10:04.120Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:10:04.123Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:04.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:04 smithi130 bash[51302]: debug 2023-08-28T00:10:04.216+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:04.221748+0000) 2023-08-28T00:10:05.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:05 smithi130 bash[27579]: cluster 2023-08-28T00:10:04.157548+0000 mgr.y (mgr.14901) 3945 : cluster [DBG] pgmap v3952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:05.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:05 smithi130 bash[51302]: debug 2023-08-28T00:10:05.216+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:05.222050+0000) 2023-08-28T00:10:05.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:05 smithi006 bash[33918]: cluster 2023-08-28T00:10:04.157548+0000 mgr.y (mgr.14901) 3945 : cluster [DBG] pgmap v3952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:05.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:05 smithi006 bash[27292]: cluster 2023-08-28T00:10:04.157548+0000 mgr.y (mgr.14901) 3945 : cluster [DBG] pgmap v3952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:06 smithi130 bash[27579]: audit 2023-08-28T00:10:05.328952+0000 mon.c (mon.2) 1501 : audit [DBG] from='client.? 172.21.15.6:0/1153886042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:06.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:06 smithi130 bash[51302]: debug 2023-08-28T00:10:06.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:06.222335+0000) 2023-08-28T00:10:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:06 smithi006 bash[33918]: audit 2023-08-28T00:10:05.328952+0000 mon.c (mon.2) 1501 : audit [DBG] from='client.? 172.21.15.6:0/1153886042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:06 smithi006 bash[27292]: audit 2023-08-28T00:10:05.328952+0000 mon.c (mon.2) 1501 : audit [DBG] from='client.? 172.21.15.6:0/1153886042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:07.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:07 smithi130 bash[27579]: cluster 2023-08-28T00:10:06.158263+0000 mgr.y (mgr.14901) 3946 : cluster [DBG] pgmap v3953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:07.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:07 smithi130 bash[51302]: debug 2023-08-28T00:10:07.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:07.222619+0000) 2023-08-28T00:10:07.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:07 smithi006 bash[33918]: cluster 2023-08-28T00:10:06.158263+0000 mgr.y (mgr.14901) 3946 : cluster [DBG] pgmap v3953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:07.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:07 smithi006 bash[27292]: cluster 2023-08-28T00:10:06.158263+0000 mgr.y (mgr.14901) 3946 : cluster [DBG] pgmap v3953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:08 smithi130 bash[27579]: audit 2023-08-28T00:10:07.745779+0000 mon.a (mon.0) 2993 : audit [DBG] from='client.? 172.21.15.6:0/3307449002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:08.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:08 smithi130 bash[51302]: debug 2023-08-28T00:10:08.219+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:08.222961+0000) 2023-08-28T00:10:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:08 smithi006 bash[27292]: audit 2023-08-28T00:10:07.745779+0000 mon.a (mon.0) 2993 : audit [DBG] from='client.? 172.21.15.6:0/3307449002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:08 smithi006 bash[33918]: audit 2023-08-28T00:10:07.745779+0000 mon.a (mon.0) 2993 : audit [DBG] from='client.? 172.21.15.6:0/3307449002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:09 smithi130 bash[27579]: cluster 2023-08-28T00:10:08.158961+0000 mgr.y (mgr.14901) 3947 : cluster [DBG] pgmap v3954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:09.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:09 smithi130 bash[51302]: debug 2023-08-28T00:10:09.219+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:09.223289+0000) 2023-08-28T00:10:09.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:09 smithi006 bash[33918]: cluster 2023-08-28T00:10:08.158961+0000 mgr.y (mgr.14901) 3947 : cluster [DBG] pgmap v3954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:09 smithi006 bash[27292]: cluster 2023-08-28T00:10:08.158961+0000 mgr.y (mgr.14901) 3947 : cluster [DBG] pgmap v3954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:10 smithi130 bash[27579]: audit 2023-08-28T00:10:10.162224+0000 mon.a (mon.0) 2994 : audit [DBG] from='client.? 172.21.15.6:0/519992314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:10.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:10 smithi130 bash[51302]: debug 2023-08-28T00:10:10.219+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:10.223590+0000) 2023-08-28T00:10:10.667 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:10 smithi006 bash[27292]: audit 2023-08-28T00:10:10.162224+0000 mon.a (mon.0) 2994 : audit [DBG] from='client.? 172.21.15.6:0/519992314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:10.667 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:10 smithi006 bash[33918]: audit 2023-08-28T00:10:10.162224+0000 mon.a (mon.0) 2994 : audit [DBG] from='client.? 172.21.15.6:0/519992314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:10.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:10:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:10:10] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:10:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:10:10.664Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:10:10.667Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:11.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:11 smithi130 bash[27579]: cluster 2023-08-28T00:10:10.159720+0000 mgr.y (mgr.14901) 3948 : cluster [DBG] pgmap v3955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:11.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:11 smithi130 bash[51302]: debug 2023-08-28T00:10:11.219+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:11.223923+0000) 2023-08-28T00:10:11.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:11 smithi006 bash[27292]: cluster 2023-08-28T00:10:10.159720+0000 mgr.y (mgr.14901) 3948 : cluster [DBG] pgmap v3955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:11.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:11 smithi006 bash[33918]: cluster 2023-08-28T00:10:10.159720+0000 mgr.y (mgr.14901) 3948 : cluster [DBG] pgmap v3955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:12.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:12 smithi130 bash[51302]: debug 2023-08-28T00:10:12.219+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:12.224217+0000) 2023-08-28T00:10:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:10:12.237Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:10:12.239Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:13.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:13 smithi130 bash[51302]: debug 2023-08-28T00:10:13.219+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:13.224480+0000) 2023-08-28T00:10:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:13 smithi130 bash[27579]: cluster 2023-08-28T00:10:12.160308+0000 mgr.y (mgr.14901) 3949 : cluster [DBG] pgmap v3956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:13 smithi130 bash[27579]: audit 2023-08-28T00:10:12.580322+0000 mon.a (mon.0) 2995 : audit [DBG] from='client.? 172.21.15.6:0/2233657305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:13.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:13 smithi006 bash[33918]: cluster 2023-08-28T00:10:12.160308+0000 mgr.y (mgr.14901) 3949 : cluster [DBG] pgmap v3956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:13.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:13 smithi006 bash[33918]: audit 2023-08-28T00:10:12.580322+0000 mon.a (mon.0) 2995 : audit [DBG] from='client.? 172.21.15.6:0/2233657305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:13.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:13 smithi006 bash[27292]: cluster 2023-08-28T00:10:12.160308+0000 mgr.y (mgr.14901) 3949 : cluster [DBG] pgmap v3956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:13.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:13 smithi006 bash[27292]: audit 2023-08-28T00:10:12.580322+0000 mon.a (mon.0) 2995 : audit [DBG] from='client.? 172.21.15.6:0/2233657305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:10:14.121Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:10:14.123Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:14.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:14 smithi130 bash[51302]: debug 2023-08-28T00:10:14.219+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:14.224809+0000) 2023-08-28T00:10:15.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:15 smithi130 bash[51302]: debug 2023-08-28T00:10:15.223+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:15.225182+0000) 2023-08-28T00:10:15.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:15 smithi130 bash[27579]: cluster 2023-08-28T00:10:14.161381+0000 mgr.y (mgr.14901) 3950 : cluster [DBG] pgmap v3957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:15.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:15 smithi130 bash[27579]: audit 2023-08-28T00:10:14.992088+0000 mon.a (mon.0) 2996 : audit [DBG] from='client.? 172.21.15.6:0/2128540822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:15.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:15 smithi006 bash[27292]: cluster 2023-08-28T00:10:14.161381+0000 mgr.y (mgr.14901) 3950 : cluster [DBG] pgmap v3957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:15.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:15 smithi006 bash[27292]: audit 2023-08-28T00:10:14.992088+0000 mon.a (mon.0) 2996 : audit [DBG] from='client.? 172.21.15.6:0/2128540822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:15.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:15 smithi006 bash[33918]: cluster 2023-08-28T00:10:14.161381+0000 mgr.y (mgr.14901) 3950 : cluster [DBG] pgmap v3957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:15.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:15 smithi006 bash[33918]: audit 2023-08-28T00:10:14.992088+0000 mon.a (mon.0) 2996 : audit [DBG] from='client.? 172.21.15.6:0/2128540822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:16.493 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:16 smithi130 bash[51302]: debug 2023-08-28T00:10:16.223+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:16.225529+0000) 2023-08-28T00:10:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:16 smithi130 bash[27579]: cluster 2023-08-28T00:10:16.162106+0000 mgr.y (mgr.14901) 3951 : cluster [DBG] pgmap v3958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:17.052 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:16 smithi006 bash[27292]: cluster 2023-08-28T00:10:16.162106+0000 mgr.y (mgr.14901) 3951 : cluster [DBG] pgmap v3958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:17.053 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:16 smithi006 bash[33918]: cluster 2023-08-28T00:10:16.162106+0000 mgr.y (mgr.14901) 3951 : cluster [DBG] pgmap v3958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:17.496 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:17 smithi130 bash[51302]: debug 2023-08-28T00:10:17.219+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:17.225861+0000) 2023-08-28T00:10:17.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:17 smithi130 bash[27579]: audit 2023-08-28T00:10:17.406473+0000 mon.a (mon.0) 2997 : audit [DBG] from='client.? 172.21.15.6:0/286722703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:17.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:17 smithi006 bash[33918]: audit 2023-08-28T00:10:17.406473+0000 mon.a (mon.0) 2997 : audit [DBG] from='client.? 172.21.15.6:0/286722703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:17.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:17 smithi006 bash[27292]: audit 2023-08-28T00:10:17.406473+0000 mon.a (mon.0) 2997 : audit [DBG] from='client.? 172.21.15.6:0/286722703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:18.510 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:18 smithi130 bash[51302]: debug 2023-08-28T00:10:18.219+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:18.226198+0000) 2023-08-28T00:10:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:18 smithi130 bash[27579]: cluster 2023-08-28T00:10:18.162771+0000 mgr.y (mgr.14901) 3952 : cluster [DBG] pgmap v3959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:18 smithi006 bash[33918]: cluster 2023-08-28T00:10:18.162771+0000 mgr.y (mgr.14901) 3952 : cluster [DBG] pgmap v3959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:18 smithi006 bash[27292]: cluster 2023-08-28T00:10:18.162771+0000 mgr.y (mgr.14901) 3952 : cluster [DBG] pgmap v3959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:19.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:19 smithi130 bash[51302]: debug 2023-08-28T00:10:19.219+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:19.226538+0000) 2023-08-28T00:10:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:19 smithi006 bash[27292]: audit 2023-08-28T00:10:19.816193+0000 mon.c (mon.2) 1502 : audit [DBG] from='client.? 172.21.15.6:0/3831049147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:20.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:19 smithi006 bash[33918]: audit 2023-08-28T00:10:19.816193+0000 mon.c (mon.2) 1502 : audit [DBG] from='client.? 172.21.15.6:0/3831049147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:20.228 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:19 smithi130 bash[27579]: audit 2023-08-28T00:10:19.816193+0000 mon.c (mon.2) 1502 : audit [DBG] from='client.? 172.21.15.6:0/3831049147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:20.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:20 smithi130 bash[51302]: debug 2023-08-28T00:10:20.223+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:20.226902+0000) 2023-08-28T00:10:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:10:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:10:20] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T00:10:20.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:20 smithi006 bash[33918]: cluster 2023-08-28T00:10:20.163913+0000 mgr.y (mgr.14901) 3953 : cluster [DBG] pgmap v3960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:20.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:20 smithi006 bash[27292]: cluster 2023-08-28T00:10:20.163913+0000 mgr.y (mgr.14901) 3953 : cluster [DBG] pgmap v3960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:20.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:10:20.664Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:20.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:10:20.667Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:21.228 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:20 smithi130 bash[27579]: cluster 2023-08-28T00:10:20.163913+0000 mgr.y (mgr.14901) 3953 : cluster [DBG] pgmap v3960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:21.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:21 smithi130 bash[51302]: debug 2023-08-28T00:10:21.223+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:21.227248+0000) 2023-08-28T00:10:22.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:22 smithi130 bash[51302]: debug 2023-08-28T00:10:22.223+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:22.227516+0000) 2023-08-28T00:10:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:10:22.237Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:10:22.239Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:23.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:23 smithi130 bash[27579]: cluster 2023-08-28T00:10:22.164499+0000 mgr.y (mgr.14901) 3954 : cluster [DBG] pgmap v3961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:23.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:23 smithi130 bash[27579]: audit 2023-08-28T00:10:22.247936+0000 mon.a (mon.0) 2998 : audit [DBG] from='client.? 172.21.15.6:0/3572639578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:23.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:23 smithi130 bash[51302]: debug 2023-08-28T00:10:23.223+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:23.227818+0000) 2023-08-28T00:10:23.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:23 smithi006 bash[27292]: cluster 2023-08-28T00:10:22.164499+0000 mgr.y (mgr.14901) 3954 : cluster [DBG] pgmap v3961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:23.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:23 smithi006 bash[27292]: audit 2023-08-28T00:10:22.247936+0000 mon.a (mon.0) 2998 : audit [DBG] from='client.? 172.21.15.6:0/3572639578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:23.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:23 smithi006 bash[33918]: cluster 2023-08-28T00:10:22.164499+0000 mgr.y (mgr.14901) 3954 : cluster [DBG] pgmap v3961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:23.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:23 smithi006 bash[33918]: audit 2023-08-28T00:10:22.247936+0000 mon.a (mon.0) 2998 : audit [DBG] from='client.? 172.21.15.6:0/3572639578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:24.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:10:24.123Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:10:24.125Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:24.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:24 smithi130 bash[51302]: debug 2023-08-28T00:10:24.223+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:24.228163+0000) 2023-08-28T00:10:25.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:25 smithi130 bash[27579]: cluster 2023-08-28T00:10:24.165573+0000 mgr.y (mgr.14901) 3955 : cluster [DBG] pgmap v3962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:25.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:25 smithi130 bash[27579]: audit 2023-08-28T00:10:24.661642+0000 mon.c (mon.2) 1503 : audit [DBG] from='client.? 172.21.15.6:0/987726716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:25.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:25 smithi130 bash[51302]: debug 2023-08-28T00:10:25.223+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:25.228410+0000) 2023-08-28T00:10:25.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:25 smithi006 bash[27292]: cluster 2023-08-28T00:10:24.165573+0000 mgr.y (mgr.14901) 3955 : cluster [DBG] pgmap v3962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:25.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:25 smithi006 bash[27292]: audit 2023-08-28T00:10:24.661642+0000 mon.c (mon.2) 1503 : audit [DBG] from='client.? 172.21.15.6:0/987726716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:25.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:25 smithi006 bash[33918]: cluster 2023-08-28T00:10:24.165573+0000 mgr.y (mgr.14901) 3955 : cluster [DBG] pgmap v3962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:25.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:25 smithi006 bash[33918]: audit 2023-08-28T00:10:24.661642+0000 mon.c (mon.2) 1503 : audit [DBG] from='client.? 172.21.15.6:0/987726716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:26.498 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:26 smithi130 bash[51302]: debug 2023-08-28T00:10:26.223+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:26.228758+0000) 2023-08-28T00:10:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:26 smithi130 bash[27579]: cluster 2023-08-28T00:10:26.166326+0000 mgr.y (mgr.14901) 3956 : cluster [DBG] pgmap v3963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:26.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:26 smithi006 bash[27292]: cluster 2023-08-28T00:10:26.166326+0000 mgr.y (mgr.14901) 3956 : cluster [DBG] pgmap v3963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:26 smithi006 bash[33918]: cluster 2023-08-28T00:10:26.166326+0000 mgr.y (mgr.14901) 3956 : cluster [DBG] pgmap v3963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:27.501 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:27 smithi130 bash[51302]: debug 2023-08-28T00:10:27.223+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:27.229021+0000) 2023-08-28T00:10:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:27 smithi130 bash[27579]: audit 2023-08-28T00:10:27.076030+0000 mon.c (mon.2) 1504 : audit [DBG] from='client.? 172.21.15.6:0/163325483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:27.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:27 smithi006 bash[33918]: audit 2023-08-28T00:10:27.076030+0000 mon.c (mon.2) 1504 : audit [DBG] from='client.? 172.21.15.6:0/163325483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:27.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:27 smithi006 bash[27292]: audit 2023-08-28T00:10:27.076030+0000 mon.c (mon.2) 1504 : audit [DBG] from='client.? 172.21.15.6:0/163325483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:28.509 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:28 smithi130 bash[51302]: debug 2023-08-28T00:10:28.227+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:28.229291+0000) 2023-08-28T00:10:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:28 smithi130 bash[27579]: cluster 2023-08-28T00:10:28.167005+0000 mgr.y (mgr.14901) 3957 : cluster [DBG] pgmap v3964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:28 smithi006 bash[33918]: cluster 2023-08-28T00:10:28.167005+0000 mgr.y (mgr.14901) 3957 : cluster [DBG] pgmap v3964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:28 smithi006 bash[27292]: cluster 2023-08-28T00:10:28.167005+0000 mgr.y (mgr.14901) 3957 : cluster [DBG] pgmap v3964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:29.512 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:29 smithi130 bash[51302]: debug 2023-08-28T00:10:29.223+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:29.229598+0000) 2023-08-28T00:10:29.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:29 smithi130 bash[27579]: audit 2023-08-28T00:10:29.488038+0000 mon.a (mon.0) 2999 : audit [DBG] from='client.? 172.21.15.6:0/2304746726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:29.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:29 smithi006 bash[33918]: audit 2023-08-28T00:10:29.488038+0000 mon.a (mon.0) 2999 : audit [DBG] from='client.? 172.21.15.6:0/2304746726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:29.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:29 smithi006 bash[27292]: audit 2023-08-28T00:10:29.488038+0000 mon.a (mon.0) 2999 : audit [DBG] from='client.? 172.21.15.6:0/2304746726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:30.519 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:30 smithi130 bash[51302]: debug 2023-08-28T00:10:30.223+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:30.229888+0000) 2023-08-28T00:10:30.828 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:30 smithi006 bash[33918]: cluster 2023-08-28T00:10:30.168155+0000 mgr.y (mgr.14901) 3958 : cluster [DBG] pgmap v3965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:30.828 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:10:30.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:30.828 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:30 smithi006 bash[27292]: cluster 2023-08-28T00:10:30.168155+0000 mgr.y (mgr.14901) 3958 : cluster [DBG] pgmap v3965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:30 smithi130 bash[27579]: cluster 2023-08-28T00:10:30.168155+0000 mgr.y (mgr.14901) 3958 : cluster [DBG] pgmap v3965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:10:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:10:30] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T00:10:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:10:30.668Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:31.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:31 smithi130 bash[51302]: debug 2023-08-28T00:10:31.227+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:31.230148+0000) 2023-08-28T00:10:32.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:31 smithi006 bash[27292]: audit 2023-08-28T00:10:31.899155+0000 mon.c (mon.2) 1505 : audit [DBG] from='client.? 172.21.15.6:0/2424130494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:32.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:31 smithi006 bash[33918]: audit 2023-08-28T00:10:31.899155+0000 mon.c (mon.2) 1505 : audit [DBG] from='client.? 172.21.15.6:0/2424130494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:32.232 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:31 smithi130 bash[27579]: audit 2023-08-28T00:10:31.899155+0000 mon.c (mon.2) 1505 : audit [DBG] from='client.? 172.21.15.6:0/2424130494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:32.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:32 smithi130 bash[51302]: debug 2023-08-28T00:10:32.223+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:32.230444+0000) 2023-08-28T00:10:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:10:32.238Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:10:32.240Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:32 smithi006 bash[33918]: cluster 2023-08-28T00:10:32.168869+0000 mgr.y (mgr.14901) 3959 : cluster [DBG] pgmap v3966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:32 smithi006 bash[27292]: cluster 2023-08-28T00:10:32.168869+0000 mgr.y (mgr.14901) 3959 : cluster [DBG] pgmap v3966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:33.232 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:32 smithi130 bash[27579]: cluster 2023-08-28T00:10:32.168869+0000 mgr.y (mgr.14901) 3959 : cluster [DBG] pgmap v3966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:33.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:33 smithi130 bash[51302]: debug 2023-08-28T00:10:33.227+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:33.230720+0000) 2023-08-28T00:10:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:10:34.123Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:10:34.126Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:34.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:34 smithi130 bash[51302]: debug 2023-08-28T00:10:34.227+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:34.231017+0000) 2023-08-28T00:10:35.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:35 smithi130 bash[27579]: cluster 2023-08-28T00:10:34.169584+0000 mgr.y (mgr.14901) 3960 : cluster [DBG] pgmap v3967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:35.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:35 smithi130 bash[27579]: audit 2023-08-28T00:10:34.324769+0000 mon.a (mon.0) 3000 : audit [DBG] from='client.? 172.21.15.6:0/3047224586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:35.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:35 smithi130 bash[51302]: debug 2023-08-28T00:10:35.227+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:35.231243+0000) 2023-08-28T00:10:35.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:35 smithi006 bash[27292]: cluster 2023-08-28T00:10:34.169584+0000 mgr.y (mgr.14901) 3960 : cluster [DBG] pgmap v3967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:35.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:35 smithi006 bash[27292]: audit 2023-08-28T00:10:34.324769+0000 mon.a (mon.0) 3000 : audit [DBG] from='client.? 172.21.15.6:0/3047224586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:35.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:35 smithi006 bash[33918]: cluster 2023-08-28T00:10:34.169584+0000 mgr.y (mgr.14901) 3960 : cluster [DBG] pgmap v3967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:35.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:35 smithi006 bash[33918]: audit 2023-08-28T00:10:34.324769+0000 mon.a (mon.0) 3000 : audit [DBG] from='client.? 172.21.15.6:0/3047224586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:36.496 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:36 smithi130 bash[51302]: debug 2023-08-28T00:10:36.227+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:36.231535+0000) 2023-08-28T00:10:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:36 smithi130 bash[27579]: cluster 2023-08-28T00:10:36.170420+0000 mgr.y (mgr.14901) 3961 : cluster [DBG] pgmap v3968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:36 smithi006 bash[33918]: cluster 2023-08-28T00:10:36.170420+0000 mgr.y (mgr.14901) 3961 : cluster [DBG] pgmap v3968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:36 smithi006 bash[27292]: cluster 2023-08-28T00:10:36.170420+0000 mgr.y (mgr.14901) 3961 : cluster [DBG] pgmap v3968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:37.499 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:37 smithi130 bash[51302]: debug 2023-08-28T00:10:37.227+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:37.231833+0000) 2023-08-28T00:10:37.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:37 smithi130 bash[27579]: audit 2023-08-28T00:10:36.742652+0000 mon.a (mon.0) 3001 : audit [DBG] from='client.? 172.21.15.6:0/1036470696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:37.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:37 smithi006 bash[33918]: audit 2023-08-28T00:10:36.742652+0000 mon.a (mon.0) 3001 : audit [DBG] from='client.? 172.21.15.6:0/1036470696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:37.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:37 smithi006 bash[27292]: audit 2023-08-28T00:10:36.742652+0000 mon.a (mon.0) 3001 : audit [DBG] from='client.? 172.21.15.6:0/1036470696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:38.507 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:38 smithi130 bash[51302]: debug 2023-08-28T00:10:38.227+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:38.232111+0000) 2023-08-28T00:10:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:38 smithi130 bash[27579]: cluster 2023-08-28T00:10:38.171099+0000 mgr.y (mgr.14901) 3962 : cluster [DBG] pgmap v3969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:38 smithi006 bash[33918]: cluster 2023-08-28T00:10:38.171099+0000 mgr.y (mgr.14901) 3962 : cluster [DBG] pgmap v3969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:38 smithi006 bash[27292]: cluster 2023-08-28T00:10:38.171099+0000 mgr.y (mgr.14901) 3962 : cluster [DBG] pgmap v3969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:39.510 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:39 smithi130 bash[51302]: debug 2023-08-28T00:10:39.227+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:39.232476+0000) 2023-08-28T00:10:39.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:39 smithi130 bash[27579]: audit 2023-08-28T00:10:39.162551+0000 mon.c (mon.2) 1506 : audit [DBG] from='client.? 172.21.15.6:0/303896858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:39 smithi006 bash[33918]: audit 2023-08-28T00:10:39.162551+0000 mon.c (mon.2) 1506 : audit [DBG] from='client.? 172.21.15.6:0/303896858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:39.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:39 smithi006 bash[27292]: audit 2023-08-28T00:10:39.162551+0000 mon.c (mon.2) 1506 : audit [DBG] from='client.? 172.21.15.6:0/303896858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:40.517 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:40 smithi130 bash[51302]: debug 2023-08-28T00:10:40.227+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:40.232735+0000) 2023-08-28T00:10:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:40 smithi130 bash[27579]: cluster 2023-08-28T00:10:40.172177+0000 mgr.y (mgr.14901) 3963 : cluster [DBG] pgmap v3970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:10:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:10:40] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T00:10:40.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:40 smithi006 bash[27292]: cluster 2023-08-28T00:10:40.172177+0000 mgr.y (mgr.14901) 3963 : cluster [DBG] pgmap v3970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:40.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:40 smithi006 bash[33918]: cluster 2023-08-28T00:10:40.172177+0000 mgr.y (mgr.14901) 3963 : cluster [DBG] pgmap v3970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:10:40.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:10:40.668Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:41.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:41 smithi130 bash[51302]: debug 2023-08-28T00:10:41.227+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:41.233028+0000) 2023-08-28T00:10:41.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:41 smithi006 bash[33918]: audit 2023-08-28T00:10:41.577376+0000 mon.c (mon.2) 1507 : audit [DBG] from='client.? 172.21.15.6:0/4131647204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:41.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:41 smithi006 bash[27292]: audit 2023-08-28T00:10:41.577376+0000 mon.c (mon.2) 1507 : audit [DBG] from='client.? 172.21.15.6:0/4131647204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:42.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:41 smithi130 bash[27579]: audit 2023-08-28T00:10:41.577376+0000 mon.c (mon.2) 1507 : audit [DBG] from='client.? 172.21.15.6:0/4131647204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:42.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:42 smithi130 bash[51302]: debug 2023-08-28T00:10:42.227+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:42.233319+0000) 2023-08-28T00:10:42.640 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:10:42.239Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:42.640 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:10:42.241Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:42 smithi006 bash[27292]: cluster 2023-08-28T00:10:42.172862+0000 mgr.y (mgr.14901) 3964 : cluster [DBG] pgmap v3971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:42 smithi006 bash[33918]: cluster 2023-08-28T00:10:42.172862+0000 mgr.y (mgr.14901) 3964 : cluster [DBG] pgmap v3971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:42 smithi130 bash[27579]: cluster 2023-08-28T00:10:42.172862+0000 mgr.y (mgr.14901) 3964 : cluster [DBG] pgmap v3971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:43.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:43 smithi130 bash[51302]: debug 2023-08-28T00:10:43.231+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:43.233583+0000) 2023-08-28T00:10:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:44 smithi130 bash[27579]: audit 2023-08-28T00:10:43.987866+0000 mon.c (mon.2) 1508 : audit [DBG] from='client.? 172.21.15.6:0/2806957179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:44.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:44 smithi130 bash[51302]: debug 2023-08-28T00:10:44.227+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:44.233817+0000) 2023-08-28T00:10:44.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:44 smithi006 bash[27292]: audit 2023-08-28T00:10:43.987866+0000 mon.c (mon.2) 1508 : audit [DBG] from='client.? 172.21.15.6:0/2806957179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:44.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:44 smithi006 bash[33918]: audit 2023-08-28T00:10:43.987866+0000 mon.c (mon.2) 1508 : audit [DBG] from='client.? 172.21.15.6:0/2806957179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:10:44.123Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:10:44.126Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:45 smithi130 bash[27579]: cluster 2023-08-28T00:10:44.173838+0000 mgr.y (mgr.14901) 3965 : cluster [DBG] pgmap v3972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:45 smithi130 bash[51302]: debug 2023-08-28T00:10:45.231+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:45.234130+0000) 2023-08-28T00:10:45.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:45 smithi006 bash[33918]: cluster 2023-08-28T00:10:44.173838+0000 mgr.y (mgr.14901) 3965 : cluster [DBG] pgmap v3972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:45.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:45 smithi006 bash[27292]: cluster 2023-08-28T00:10:44.173838+0000 mgr.y (mgr.14901) 3965 : cluster [DBG] pgmap v3972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:46.496 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:46 smithi130 bash[51302]: debug 2023-08-28T00:10:46.227+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:46.234399+0000) 2023-08-28T00:10:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:46 smithi130 bash[27579]: audit 2023-08-28T00:10:45.571070+0000 mon.b (mon.1) 504 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:10:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:46 smithi130 bash[27579]: cluster 2023-08-28T00:10:46.174557+0000 mgr.y (mgr.14901) 3966 : cluster [DBG] pgmap v3973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:46 smithi130 bash[27579]: audit 2023-08-28T00:10:46.400326+0000 mon.a (mon.0) 3002 : audit [DBG] from='client.? 172.21.15.6:0/925428996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:46 smithi006 bash[27292]: audit 2023-08-28T00:10:45.571070+0000 mon.b (mon.1) 504 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:10:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:46 smithi006 bash[27292]: cluster 2023-08-28T00:10:46.174557+0000 mgr.y (mgr.14901) 3966 : cluster [DBG] pgmap v3973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:46 smithi006 bash[27292]: audit 2023-08-28T00:10:46.400326+0000 mon.a (mon.0) 3002 : audit [DBG] from='client.? 172.21.15.6:0/925428996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:46 smithi006 bash[33918]: audit 2023-08-28T00:10:45.571070+0000 mon.b (mon.1) 504 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:10:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:46 smithi006 bash[33918]: cluster 2023-08-28T00:10:46.174557+0000 mgr.y (mgr.14901) 3966 : cluster [DBG] pgmap v3973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:46 smithi006 bash[33918]: audit 2023-08-28T00:10:46.400326+0000 mon.a (mon.0) 3002 : audit [DBG] from='client.? 172.21.15.6:0/925428996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:47.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:47 smithi130 bash[51302]: debug 2023-08-28T00:10:47.227+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:47.234713+0000) 2023-08-28T00:10:48.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:48 smithi130 bash[51302]: debug 2023-08-28T00:10:48.231+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:48.234995+0000) 2023-08-28T00:10:49.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:49 smithi130 bash[27579]: cluster 2023-08-28T00:10:48.175203+0000 mgr.y (mgr.14901) 3967 : cluster [DBG] pgmap v3974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:49.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:49 smithi130 bash[27579]: audit 2023-08-28T00:10:48.815148+0000 mon.a (mon.0) 3003 : audit [DBG] from='client.? 172.21.15.6:0/2354725056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:49.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:49 smithi130 bash[51302]: debug 2023-08-28T00:10:49.231+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:49.235229+0000) 2023-08-28T00:10:49.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:49 smithi006 bash[27292]: cluster 2023-08-28T00:10:48.175203+0000 mgr.y (mgr.14901) 3967 : cluster [DBG] pgmap v3974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:49.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:49 smithi006 bash[27292]: audit 2023-08-28T00:10:48.815148+0000 mon.a (mon.0) 3003 : audit [DBG] from='client.? 172.21.15.6:0/2354725056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:49.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:49 smithi006 bash[33918]: cluster 2023-08-28T00:10:48.175203+0000 mgr.y (mgr.14901) 3967 : cluster [DBG] pgmap v3974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:49.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:49 smithi006 bash[33918]: audit 2023-08-28T00:10:48.815148+0000 mon.a (mon.0) 3003 : audit [DBG] from='client.? 172.21.15.6:0/2354725056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:50.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:50 smithi130 bash[51302]: debug 2023-08-28T00:10:50.231+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:50.235447+0000) 2023-08-28T00:10:50.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:10:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:10:50] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:10:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:10:50.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:10:50.668Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:51.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:51 smithi130 bash[27579]: cluster 2023-08-28T00:10:50.176310+0000 mgr.y (mgr.14901) 3968 : cluster [DBG] pgmap v3975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:51.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:51 smithi130 bash[51302]: debug 2023-08-28T00:10:51.231+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:51.235698+0000) 2023-08-28T00:10:51.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:51 smithi006 bash[27292]: cluster 2023-08-28T00:10:50.176310+0000 mgr.y (mgr.14901) 3968 : cluster [DBG] pgmap v3975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:51.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:51 smithi006 bash[33918]: cluster 2023-08-28T00:10:50.176310+0000 mgr.y (mgr.14901) 3968 : cluster [DBG] pgmap v3975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:52.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:52 smithi130 bash[51302]: debug 2023-08-28T00:10:52.231+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:52.235890+0000) 2023-08-28T00:10:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:52 smithi130 bash[27579]: audit 2023-08-28T00:10:51.280433+0000 mon.c (mon.2) 1509 : audit [DBG] from='client.? 172.21.15.6:0/4195435541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:52 smithi130 bash[27579]: audit 2023-08-28T00:10:51.379890+0000 mon.a (mon.0) 3004 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:10:52.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:52 smithi130 bash[27579]: audit 2023-08-28T00:10:51.949230+0000 mon.a (mon.0) 3005 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:10:52.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:52 smithi130 bash[27579]: audit 2023-08-28T00:10:52.219027+0000 mon.b (mon.1) 505 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:10:52.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:52 smithi130 bash[27579]: audit 2023-08-28T00:10:52.221090+0000 mon.b (mon.1) 506 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:10:52.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:52 smithi130 bash[27579]: audit 2023-08-28T00:10:52.232000+0000 mon.a (mon.0) 3006 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:10:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:10:52.240Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:10:52.242Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:52 smithi006 bash[33918]: audit 2023-08-28T00:10:51.280433+0000 mon.c (mon.2) 1509 : audit [DBG] from='client.? 172.21.15.6:0/4195435541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:52 smithi006 bash[33918]: audit 2023-08-28T00:10:51.379890+0000 mon.a (mon.0) 3004 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:10:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:52 smithi006 bash[33918]: audit 2023-08-28T00:10:51.949230+0000 mon.a (mon.0) 3005 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:10:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:52 smithi006 bash[33918]: audit 2023-08-28T00:10:52.219027+0000 mon.b (mon.1) 505 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:10:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:52 smithi006 bash[33918]: audit 2023-08-28T00:10:52.221090+0000 mon.b (mon.1) 506 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:10:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:52 smithi006 bash[33918]: audit 2023-08-28T00:10:52.232000+0000 mon.a (mon.0) 3006 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:10:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:52 smithi006 bash[27292]: audit 2023-08-28T00:10:51.280433+0000 mon.c (mon.2) 1509 : audit [DBG] from='client.? 172.21.15.6:0/4195435541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:52 smithi006 bash[27292]: audit 2023-08-28T00:10:51.379890+0000 mon.a (mon.0) 3004 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:10:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:52 smithi006 bash[27292]: audit 2023-08-28T00:10:51.949230+0000 mon.a (mon.0) 3005 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:10:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:52 smithi006 bash[27292]: audit 2023-08-28T00:10:52.219027+0000 mon.b (mon.1) 505 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:10:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:52 smithi006 bash[27292]: audit 2023-08-28T00:10:52.221090+0000 mon.b (mon.1) 506 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:10:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:52 smithi006 bash[27292]: audit 2023-08-28T00:10:52.232000+0000 mon.a (mon.0) 3006 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:10:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:53 smithi130 bash[27579]: cluster 2023-08-28T00:10:52.176954+0000 mgr.y (mgr.14901) 3969 : cluster [DBG] pgmap v3976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:53.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:53 smithi130 bash[51302]: debug 2023-08-28T00:10:53.235+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:53.236154+0000) 2023-08-28T00:10:53.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:53 smithi006 bash[27292]: cluster 2023-08-28T00:10:52.176954+0000 mgr.y (mgr.14901) 3969 : cluster [DBG] pgmap v3976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:53.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:53 smithi006 bash[33918]: cluster 2023-08-28T00:10:52.176954+0000 mgr.y (mgr.14901) 3969 : cluster [DBG] pgmap v3976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:54.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:54 smithi006 bash[27292]: audit 2023-08-28T00:10:53.702859+0000 mon.c (mon.2) 1510 : audit [DBG] from='client.? 172.21.15.6:0/2701697408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:54.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:54 smithi006 bash[33918]: audit 2023-08-28T00:10:53.702859+0000 mon.c (mon.2) 1510 : audit [DBG] from='client.? 172.21.15.6:0/2701697408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:10:54.125Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:10:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:10:54.127Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:10:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:54 smithi130 bash[27579]: audit 2023-08-28T00:10:53.702859+0000 mon.c (mon.2) 1510 : audit [DBG] from='client.? 172.21.15.6:0/2701697408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:54.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:54 smithi130 bash[51302]: debug 2023-08-28T00:10:54.231+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:54.236573+0000) 2023-08-28T00:10:55.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:55 smithi130 bash[27579]: cluster 2023-08-28T00:10:54.177919+0000 mgr.y (mgr.14901) 3970 : cluster [DBG] pgmap v3977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:55.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:55 smithi130 bash[51302]: debug 2023-08-28T00:10:55.235+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:55.236879+0000) 2023-08-28T00:10:55.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:55 smithi006 bash[27292]: cluster 2023-08-28T00:10:54.177919+0000 mgr.y (mgr.14901) 3970 : cluster [DBG] pgmap v3977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:55.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:55 smithi006 bash[33918]: cluster 2023-08-28T00:10:54.177919+0000 mgr.y (mgr.14901) 3970 : cluster [DBG] pgmap v3977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:56.502 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:56 smithi130 bash[51302]: debug 2023-08-28T00:10:56.235+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:56.237156+0000) 2023-08-28T00:10:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:56 smithi130 bash[27579]: audit 2023-08-28T00:10:56.115916+0000 mon.c (mon.2) 1511 : audit [DBG] from='client.? 172.21.15.6:0/1320824537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:56.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:56 smithi130 bash[27579]: cluster 2023-08-28T00:10:56.178630+0000 mgr.y (mgr.14901) 3971 : cluster [DBG] pgmap v3978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:56 smithi006 bash[27292]: audit 2023-08-28T00:10:56.115916+0000 mon.c (mon.2) 1511 : audit [DBG] from='client.? 172.21.15.6:0/1320824537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:56 smithi006 bash[27292]: cluster 2023-08-28T00:10:56.178630+0000 mgr.y (mgr.14901) 3971 : cluster [DBG] pgmap v3978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:56 smithi006 bash[33918]: audit 2023-08-28T00:10:56.115916+0000 mon.c (mon.2) 1511 : audit [DBG] from='client.? 172.21.15.6:0/1320824537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:56 smithi006 bash[33918]: cluster 2023-08-28T00:10:56.178630+0000 mgr.y (mgr.14901) 3971 : cluster [DBG] pgmap v3978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:57.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:57 smithi130 bash[51302]: debug 2023-08-28T00:10:57.235+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:57.237445+0000) 2023-08-28T00:10:58.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:58 smithi130 bash[51302]: debug 2023-08-28T00:10:58.231+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:58.237687+0000) 2023-08-28T00:10:59.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:10:59 smithi130 bash[51302]: debug 2023-08-28T00:10:59.235+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:59.238005+0000) 2023-08-28T00:10:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:59 smithi130 bash[27579]: cluster 2023-08-28T00:10:58.179269+0000 mgr.y (mgr.14901) 3972 : cluster [DBG] pgmap v3979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:10:59 smithi130 bash[27579]: audit 2023-08-28T00:10:58.540884+0000 mon.a (mon.0) 3007 : audit [DBG] from='client.? 172.21.15.6:0/2997587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:59.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:59 smithi006 bash[27292]: cluster 2023-08-28T00:10:58.179269+0000 mgr.y (mgr.14901) 3972 : cluster [DBG] pgmap v3979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:59.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:10:59 smithi006 bash[27292]: audit 2023-08-28T00:10:58.540884+0000 mon.a (mon.0) 3007 : audit [DBG] from='client.? 172.21.15.6:0/2997587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:59.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:59 smithi006 bash[33918]: cluster 2023-08-28T00:10:58.179269+0000 mgr.y (mgr.14901) 3972 : cluster [DBG] pgmap v3979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:10:59.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:10:59 smithi006 bash[33918]: audit 2023-08-28T00:10:58.540884+0000 mon.a (mon.0) 3007 : audit [DBG] from='client.? 172.21.15.6:0/2997587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:00.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:00 smithi130 bash[51302]: debug 2023-08-28T00:11:00.235+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:00.238305+0000) 2023-08-28T00:11:00.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:11:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:11:00] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:11:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:11:00.668Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:11:00.669Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:01.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:01 smithi130 bash[27579]: cluster 2023-08-28T00:11:00.180355+0000 mgr.y (mgr.14901) 3973 : cluster [DBG] pgmap v3980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:01.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:01 smithi130 bash[27579]: audit 2023-08-28T00:11:00.959513+0000 mon.a (mon.0) 3008 : audit [DBG] from='client.? 172.21.15.6:0/125534697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:01.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:01 smithi130 bash[51302]: debug 2023-08-28T00:11:01.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:01.238589+0000) 2023-08-28T00:11:01.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:01 smithi006 bash[33918]: cluster 2023-08-28T00:11:00.180355+0000 mgr.y (mgr.14901) 3973 : cluster [DBG] pgmap v3980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:01.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:01 smithi006 bash[33918]: audit 2023-08-28T00:11:00.959513+0000 mon.a (mon.0) 3008 : audit [DBG] from='client.? 172.21.15.6:0/125534697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:01.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:01 smithi006 bash[27292]: cluster 2023-08-28T00:11:00.180355+0000 mgr.y (mgr.14901) 3973 : cluster [DBG] pgmap v3980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:01.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:01 smithi006 bash[27292]: audit 2023-08-28T00:11:00.959513+0000 mon.a (mon.0) 3008 : audit [DBG] from='client.? 172.21.15.6:0/125534697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:02.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:02 smithi130 bash[51302]: debug 2023-08-28T00:11:02.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:02.238826+0000) 2023-08-28T00:11:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:11:02.241Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:11:02.243Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:03.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:03 smithi130 bash[27579]: cluster 2023-08-28T00:11:02.181055+0000 mgr.y (mgr.14901) 3974 : cluster [DBG] pgmap v3981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:03.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:03 smithi130 bash[51302]: debug 2023-08-28T00:11:03.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:03.239169+0000) 2023-08-28T00:11:03.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:03 smithi006 bash[27292]: cluster 2023-08-28T00:11:02.181055+0000 mgr.y (mgr.14901) 3974 : cluster [DBG] pgmap v3981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:03.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:03 smithi006 bash[33918]: cluster 2023-08-28T00:11:02.181055+0000 mgr.y (mgr.14901) 3974 : cluster [DBG] pgmap v3981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:04.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:04 smithi006 bash[33918]: audit 2023-08-28T00:11:03.370901+0000 mon.c (mon.2) 1512 : audit [DBG] from='client.? 172.21.15.6:0/1875392046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:11:04.125Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:11:04.127Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:04.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:04 smithi006 bash[27292]: audit 2023-08-28T00:11:03.370901+0000 mon.c (mon.2) 1512 : audit [DBG] from='client.? 172.21.15.6:0/1875392046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:04 smithi130 bash[27579]: audit 2023-08-28T00:11:03.370901+0000 mon.c (mon.2) 1512 : audit [DBG] from='client.? 172.21.15.6:0/1875392046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:04.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:04 smithi130 bash[51302]: debug 2023-08-28T00:11:04.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:04.239391+0000) 2023-08-28T00:11:05.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:05 smithi130 bash[27579]: cluster 2023-08-28T00:11:04.182064+0000 mgr.y (mgr.14901) 3975 : cluster [DBG] pgmap v3982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:05.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:05 smithi130 bash[51302]: debug 2023-08-28T00:11:05.238+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:05.239660+0000) 2023-08-28T00:11:05.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:05 smithi006 bash[27292]: cluster 2023-08-28T00:11:04.182064+0000 mgr.y (mgr.14901) 3975 : cluster [DBG] pgmap v3982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:05.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:05 smithi006 bash[33918]: cluster 2023-08-28T00:11:04.182064+0000 mgr.y (mgr.14901) 3975 : cluster [DBG] pgmap v3982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:06.508 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:06 smithi130 bash[51302]: debug 2023-08-28T00:11:06.238+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:06.239919+0000) 2023-08-28T00:11:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:06 smithi130 bash[27579]: audit 2023-08-28T00:11:05.780290+0000 mon.c (mon.2) 1513 : audit [DBG] from='client.? 172.21.15.6:0/4094818452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:06 smithi130 bash[27579]: cluster 2023-08-28T00:11:06.182841+0000 mgr.y (mgr.14901) 3976 : cluster [DBG] pgmap v3983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:06 smithi006 bash[27292]: audit 2023-08-28T00:11:05.780290+0000 mon.c (mon.2) 1513 : audit [DBG] from='client.? 172.21.15.6:0/4094818452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:06 smithi006 bash[27292]: cluster 2023-08-28T00:11:06.182841+0000 mgr.y (mgr.14901) 3976 : cluster [DBG] pgmap v3983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:06 smithi006 bash[33918]: audit 2023-08-28T00:11:05.780290+0000 mon.c (mon.2) 1513 : audit [DBG] from='client.? 172.21.15.6:0/4094818452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:06 smithi006 bash[33918]: cluster 2023-08-28T00:11:06.182841+0000 mgr.y (mgr.14901) 3976 : cluster [DBG] pgmap v3983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:07.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:07 smithi130 bash[51302]: debug 2023-08-28T00:11:07.238+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:07.240184+0000) 2023-08-28T00:11:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:08 smithi130 bash[27579]: audit 2023-08-28T00:11:08.192905+0000 mon.c (mon.2) 1514 : audit [DBG] from='client.? 172.21.15.6:0/4211237809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:08.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:08 smithi130 bash[51302]: debug 2023-08-28T00:11:08.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:08.240453+0000) 2023-08-28T00:11:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:08 smithi006 bash[27292]: audit 2023-08-28T00:11:08.192905+0000 mon.c (mon.2) 1514 : audit [DBG] from='client.? 172.21.15.6:0/4211237809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:08 smithi006 bash[33918]: audit 2023-08-28T00:11:08.192905+0000 mon.c (mon.2) 1514 : audit [DBG] from='client.? 172.21.15.6:0/4211237809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:09 smithi130 bash[27579]: cluster 2023-08-28T00:11:08.183471+0000 mgr.y (mgr.14901) 3977 : cluster [DBG] pgmap v3984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:09.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:09 smithi130 bash[51302]: debug 2023-08-28T00:11:09.238+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:09.240754+0000) 2023-08-28T00:11:09.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:09 smithi006 bash[33918]: cluster 2023-08-28T00:11:08.183471+0000 mgr.y (mgr.14901) 3977 : cluster [DBG] pgmap v3984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:09 smithi006 bash[27292]: cluster 2023-08-28T00:11:08.183471+0000 mgr.y (mgr.14901) 3977 : cluster [DBG] pgmap v3984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:10.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:10 smithi130 bash[51302]: debug 2023-08-28T00:11:10.238+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:10.240972+0000) 2023-08-28T00:11:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:11:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:11:10] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:11:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:11:10.668Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:11:10.670Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:11.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:11 smithi130 bash[27579]: cluster 2023-08-28T00:11:10.184541+0000 mgr.y (mgr.14901) 3978 : cluster [DBG] pgmap v3985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:11.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:11 smithi130 bash[27579]: audit 2023-08-28T00:11:10.610682+0000 mon.a (mon.0) 3009 : audit [DBG] from='client.? 172.21.15.6:0/3318528383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:11.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:11 smithi130 bash[51302]: debug 2023-08-28T00:11:11.238+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:11.241252+0000) 2023-08-28T00:11:11.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:11 smithi006 bash[33918]: cluster 2023-08-28T00:11:10.184541+0000 mgr.y (mgr.14901) 3978 : cluster [DBG] pgmap v3985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:11.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:11 smithi006 bash[33918]: audit 2023-08-28T00:11:10.610682+0000 mon.a (mon.0) 3009 : audit [DBG] from='client.? 172.21.15.6:0/3318528383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:11.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:11 smithi006 bash[27292]: cluster 2023-08-28T00:11:10.184541+0000 mgr.y (mgr.14901) 3978 : cluster [DBG] pgmap v3985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:11.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:11 smithi006 bash[27292]: audit 2023-08-28T00:11:10.610682+0000 mon.a (mon.0) 3009 : audit [DBG] from='client.? 172.21.15.6:0/3318528383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:12.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:12 smithi130 bash[51302]: debug 2023-08-28T00:11:12.238+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:12.241557+0000) 2023-08-28T00:11:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:11:12.242Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:11:12.244Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:13.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:13 smithi130 bash[51302]: debug 2023-08-28T00:11:13.238+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:13.241874+0000) 2023-08-28T00:11:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:13 smithi130 bash[27579]: cluster 2023-08-28T00:11:12.185252+0000 mgr.y (mgr.14901) 3979 : cluster [DBG] pgmap v3986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:13 smithi130 bash[27579]: audit 2023-08-28T00:11:13.018918+0000 mon.a (mon.0) 3010 : audit [DBG] from='client.? 172.21.15.6:0/902325289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:13.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:13 smithi006 bash[27292]: cluster 2023-08-28T00:11:12.185252+0000 mgr.y (mgr.14901) 3979 : cluster [DBG] pgmap v3986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:13.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:13 smithi006 bash[27292]: audit 2023-08-28T00:11:13.018918+0000 mon.a (mon.0) 3010 : audit [DBG] from='client.? 172.21.15.6:0/902325289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:13.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:13 smithi006 bash[33918]: cluster 2023-08-28T00:11:12.185252+0000 mgr.y (mgr.14901) 3979 : cluster [DBG] pgmap v3986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:13.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:13 smithi006 bash[33918]: audit 2023-08-28T00:11:13.018918+0000 mon.a (mon.0) 3010 : audit [DBG] from='client.? 172.21.15.6:0/902325289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:11:14.126Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:11:14.128Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:14.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:14 smithi130 bash[51302]: debug 2023-08-28T00:11:14.238+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:14.242157+0000) 2023-08-28T00:11:15.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:15 smithi130 bash[27579]: cluster 2023-08-28T00:11:14.186251+0000 mgr.y (mgr.14901) 3980 : cluster [DBG] pgmap v3987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:15.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:15 smithi130 bash[51302]: debug 2023-08-28T00:11:15.238+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:15.242470+0000) 2023-08-28T00:11:15.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:15 smithi006 bash[27292]: cluster 2023-08-28T00:11:14.186251+0000 mgr.y (mgr.14901) 3980 : cluster [DBG] pgmap v3987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:15.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:15 smithi006 bash[33918]: cluster 2023-08-28T00:11:14.186251+0000 mgr.y (mgr.14901) 3980 : cluster [DBG] pgmap v3987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:16 smithi130 bash[27579]: audit 2023-08-28T00:11:15.426865+0000 mon.c (mon.2) 1515 : audit [DBG] from='client.? 172.21.15.6:0/2095482752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:16.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:16 smithi130 bash[51302]: debug 2023-08-28T00:11:16.238+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:16.242761+0000) 2023-08-28T00:11:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:16 smithi006 bash[33918]: audit 2023-08-28T00:11:15.426865+0000 mon.c (mon.2) 1515 : audit [DBG] from='client.? 172.21.15.6:0/2095482752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:16 smithi006 bash[27292]: audit 2023-08-28T00:11:15.426865+0000 mon.c (mon.2) 1515 : audit [DBG] from='client.? 172.21.15.6:0/2095482752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:17.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:17 smithi130 bash[27579]: cluster 2023-08-28T00:11:16.187062+0000 mgr.y (mgr.14901) 3981 : cluster [DBG] pgmap v3988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:17.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:17 smithi130 bash[51302]: debug 2023-08-28T00:11:17.242+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:17.243023+0000) 2023-08-28T00:11:17.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:17 smithi006 bash[27292]: cluster 2023-08-28T00:11:16.187062+0000 mgr.y (mgr.14901) 3981 : cluster [DBG] pgmap v3988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:17.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:17 smithi006 bash[33918]: cluster 2023-08-28T00:11:16.187062+0000 mgr.y (mgr.14901) 3981 : cluster [DBG] pgmap v3988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:18.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:18 smithi130 bash[27579]: audit 2023-08-28T00:11:17.838104+0000 mon.a (mon.0) 3011 : audit [DBG] from='client.? 172.21.15.6:0/1667087492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:18.625 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:18 smithi130 bash[51302]: debug 2023-08-28T00:11:18.242+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:18.243245+0000) 2023-08-28T00:11:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:18 smithi006 bash[33918]: audit 2023-08-28T00:11:17.838104+0000 mon.a (mon.0) 3011 : audit [DBG] from='client.? 172.21.15.6:0/1667087492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:18.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:18 smithi006 bash[27292]: audit 2023-08-28T00:11:17.838104+0000 mon.a (mon.0) 3011 : audit [DBG] from='client.? 172.21.15.6:0/1667087492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:19 smithi130 bash[27579]: cluster 2023-08-28T00:11:18.187733+0000 mgr.y (mgr.14901) 3982 : cluster [DBG] pgmap v3989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:19.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:19 smithi130 bash[51302]: debug 2023-08-28T00:11:19.242+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:19.243543+0000) 2023-08-28T00:11:19.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:19 smithi006 bash[27292]: cluster 2023-08-28T00:11:18.187733+0000 mgr.y (mgr.14901) 3982 : cluster [DBG] pgmap v3989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:19.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:19 smithi006 bash[33918]: cluster 2023-08-28T00:11:18.187733+0000 mgr.y (mgr.14901) 3982 : cluster [DBG] pgmap v3989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:20 smithi130 bash[27579]: audit 2023-08-28T00:11:20.252655+0000 mon.c (mon.2) 1516 : audit [DBG] from='client.? 172.21.15.6:0/1342898424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:20.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:20 smithi130 bash[51302]: debug 2023-08-28T00:11:20.242+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:20.243808+0000) 2023-08-28T00:11:20.671 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:20 smithi006 bash[27292]: audit 2023-08-28T00:11:20.252655+0000 mon.c (mon.2) 1516 : audit [DBG] from='client.? 172.21.15.6:0/1342898424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:20.671 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:20 smithi006 bash[33918]: audit 2023-08-28T00:11:20.252655+0000 mon.c (mon.2) 1516 : audit [DBG] from='client.? 172.21.15.6:0/1342898424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:11:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:11:20] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T00:11:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:11:20.668Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:11:20.670Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:21 smithi130 bash[27579]: cluster 2023-08-28T00:11:20.188814+0000 mgr.y (mgr.14901) 3983 : cluster [DBG] pgmap v3990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:21.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:21 smithi130 bash[51302]: debug 2023-08-28T00:11:21.242+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:21.244073+0000) 2023-08-28T00:11:21.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:21 smithi006 bash[27292]: cluster 2023-08-28T00:11:20.188814+0000 mgr.y (mgr.14901) 3983 : cluster [DBG] pgmap v3990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:21.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:21 smithi006 bash[33918]: cluster 2023-08-28T00:11:20.188814+0000 mgr.y (mgr.14901) 3983 : cluster [DBG] pgmap v3990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:22 smithi130 bash[27579]: cluster 2023-08-28T00:11:22.189431+0000 mgr.y (mgr.14901) 3984 : cluster [DBG] pgmap v3991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:22.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:22 smithi130 bash[51302]: debug 2023-08-28T00:11:22.242+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:22.244371+0000) 2023-08-28T00:11:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:22 smithi006 bash[33918]: cluster 2023-08-28T00:11:22.189431+0000 mgr.y (mgr.14901) 3984 : cluster [DBG] pgmap v3991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:22.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:11:22.243Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:22.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:11:22.245Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:22.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:22 smithi006 bash[27292]: cluster 2023-08-28T00:11:22.189431+0000 mgr.y (mgr.14901) 3984 : cluster [DBG] pgmap v3991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:23.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:23 smithi130 bash[27579]: audit 2023-08-28T00:11:22.665696+0000 mon.c (mon.2) 1517 : audit [DBG] from='client.? 172.21.15.6:0/1298866903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:23.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:23 smithi130 bash[51302]: debug 2023-08-28T00:11:23.242+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:23.244688+0000) 2023-08-28T00:11:23.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:23 smithi006 bash[33918]: audit 2023-08-28T00:11:22.665696+0000 mon.c (mon.2) 1517 : audit [DBG] from='client.? 172.21.15.6:0/1298866903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:23.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:23 smithi006 bash[27292]: audit 2023-08-28T00:11:22.665696+0000 mon.c (mon.2) 1517 : audit [DBG] from='client.? 172.21.15.6:0/1298866903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:24.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:24 smithi006 bash[27292]: cluster 2023-08-28T00:11:24.190390+0000 mgr.y (mgr.14901) 3985 : cluster [DBG] pgmap v3992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:24.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:24 smithi006 bash[33918]: cluster 2023-08-28T00:11:24.190390+0000 mgr.y (mgr.14901) 3985 : cluster [DBG] pgmap v3992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:11:24.126Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:11:24.128Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:24 smithi130 bash[27579]: cluster 2023-08-28T00:11:24.190390+0000 mgr.y (mgr.14901) 3985 : cluster [DBG] pgmap v3992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:24.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:24 smithi130 bash[51302]: debug 2023-08-28T00:11:24.242+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:24.244990+0000) 2023-08-28T00:11:25.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:25 smithi130 bash[27579]: audit 2023-08-28T00:11:25.081239+0000 mon.c (mon.2) 1518 : audit [DBG] from='client.? 172.21.15.6:0/1824626950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:25.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:25 smithi130 bash[51302]: debug 2023-08-28T00:11:25.242+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:25.245271+0000) 2023-08-28T00:11:25.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:25 smithi006 bash[27292]: audit 2023-08-28T00:11:25.081239+0000 mon.c (mon.2) 1518 : audit [DBG] from='client.? 172.21.15.6:0/1824626950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:25.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:25 smithi006 bash[33918]: audit 2023-08-28T00:11:25.081239+0000 mon.c (mon.2) 1518 : audit [DBG] from='client.? 172.21.15.6:0/1824626950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:26.510 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:26 smithi130 bash[51302]: debug 2023-08-28T00:11:26.242+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:26.245532+0000) 2023-08-28T00:11:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:26 smithi130 bash[27579]: cluster 2023-08-28T00:11:26.191157+0000 mgr.y (mgr.14901) 3986 : cluster [DBG] pgmap v3993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:26.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:26 smithi006 bash[33918]: cluster 2023-08-28T00:11:26.191157+0000 mgr.y (mgr.14901) 3986 : cluster [DBG] pgmap v3993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:26 smithi006 bash[27292]: cluster 2023-08-28T00:11:26.191157+0000 mgr.y (mgr.14901) 3986 : cluster [DBG] pgmap v3993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:27.512 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:27 smithi130 bash[51302]: debug 2023-08-28T00:11:27.242+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:27.245788+0000) 2023-08-28T00:11:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:27 smithi130 bash[27579]: audit 2023-08-28T00:11:27.498167+0000 mon.a (mon.0) 3012 : audit [DBG] from='client.? 172.21.15.6:0/3293506566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:27.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:27 smithi006 bash[33918]: audit 2023-08-28T00:11:27.498167+0000 mon.a (mon.0) 3012 : audit [DBG] from='client.? 172.21.15.6:0/3293506566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:27.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:27 smithi006 bash[27292]: audit 2023-08-28T00:11:27.498167+0000 mon.a (mon.0) 3012 : audit [DBG] from='client.? 172.21.15.6:0/3293506566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:28.521 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:28 smithi130 bash[51302]: debug 2023-08-28T00:11:28.242+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:28.246079+0000) 2023-08-28T00:11:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:28 smithi130 bash[27579]: cluster 2023-08-28T00:11:28.191876+0000 mgr.y (mgr.14901) 3987 : cluster [DBG] pgmap v3994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:28 smithi006 bash[27292]: cluster 2023-08-28T00:11:28.191876+0000 mgr.y (mgr.14901) 3987 : cluster [DBG] pgmap v3994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:28 smithi006 bash[33918]: cluster 2023-08-28T00:11:28.191876+0000 mgr.y (mgr.14901) 3987 : cluster [DBG] pgmap v3994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:29.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:29 smithi130 bash[51302]: debug 2023-08-28T00:11:29.242+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:29.246362+0000) 2023-08-28T00:11:30.248 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:29 smithi130 bash[27579]: audit 2023-08-28T00:11:29.915201+0000 mon.a (mon.0) 3013 : audit [DBG] from='client.? 172.21.15.6:0/545754188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:30.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:29 smithi006 bash[33918]: audit 2023-08-28T00:11:29.915201+0000 mon.a (mon.0) 3013 : audit [DBG] from='client.? 172.21.15.6:0/545754188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:30.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:29 smithi006 bash[27292]: audit 2023-08-28T00:11:29.915201+0000 mon.a (mon.0) 3013 : audit [DBG] from='client.? 172.21.15.6:0/545754188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:30.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:30 smithi130 bash[51302]: debug 2023-08-28T00:11:30.242+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:30.246661+0000) 2023-08-28T00:11:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:11:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:11:30] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:11:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:11:30.669Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:11:30.671Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:31.248 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:30 smithi130 bash[27579]: cluster 2023-08-28T00:11:30.193044+0000 mgr.y (mgr.14901) 3988 : cluster [DBG] pgmap v3995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:31.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:30 smithi006 bash[33918]: cluster 2023-08-28T00:11:30.193044+0000 mgr.y (mgr.14901) 3988 : cluster [DBG] pgmap v3995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:31.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:30 smithi006 bash[27292]: cluster 2023-08-28T00:11:30.193044+0000 mgr.y (mgr.14901) 3988 : cluster [DBG] pgmap v3995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:31.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:31 smithi130 bash[51302]: debug 2023-08-28T00:11:31.246+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:31.246981+0000) 2023-08-28T00:11:32.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:32 smithi130 bash[51302]: debug 2023-08-28T00:11:32.246+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:32.247206+0000) 2023-08-28T00:11:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:11:32.244Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:11:32.245Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:33 smithi130 bash[27579]: cluster 2023-08-28T00:11:32.193648+0000 mgr.y (mgr.14901) 3989 : cluster [DBG] pgmap v3996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:33 smithi130 bash[27579]: audit 2023-08-28T00:11:32.333187+0000 mon.a (mon.0) 3014 : audit [DBG] from='client.? 172.21.15.6:0/3185190221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:33.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:33 smithi130 bash[51302]: debug 2023-08-28T00:11:33.246+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:33.247561+0000) 2023-08-28T00:11:33.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:33 smithi006 bash[33918]: cluster 2023-08-28T00:11:32.193648+0000 mgr.y (mgr.14901) 3989 : cluster [DBG] pgmap v3996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:33.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:33 smithi006 bash[33918]: audit 2023-08-28T00:11:32.333187+0000 mon.a (mon.0) 3014 : audit [DBG] from='client.? 172.21.15.6:0/3185190221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:33.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:33 smithi006 bash[27292]: cluster 2023-08-28T00:11:32.193648+0000 mgr.y (mgr.14901) 3989 : cluster [DBG] pgmap v3996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:33.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:33 smithi006 bash[27292]: audit 2023-08-28T00:11:32.333187+0000 mon.a (mon.0) 3014 : audit [DBG] from='client.? 172.21.15.6:0/3185190221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:34.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:11:34.127Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:11:34.128Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:34.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:34 smithi130 bash[51302]: debug 2023-08-28T00:11:34.246+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:34.247851+0000) 2023-08-28T00:11:35.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:35 smithi130 bash[27579]: cluster 2023-08-28T00:11:34.194506+0000 mgr.y (mgr.14901) 3990 : cluster [DBG] pgmap v3997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:35.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:35 smithi130 bash[27579]: audit 2023-08-28T00:11:34.751210+0000 mon.a (mon.0) 3015 : audit [DBG] from='client.? 172.21.15.6:0/4745628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:35.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:35 smithi130 bash[51302]: debug 2023-08-28T00:11:35.246+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:35.248139+0000) 2023-08-28T00:11:35.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:35 smithi006 bash[33918]: cluster 2023-08-28T00:11:34.194506+0000 mgr.y (mgr.14901) 3990 : cluster [DBG] pgmap v3997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:35.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:35 smithi006 bash[33918]: audit 2023-08-28T00:11:34.751210+0000 mon.a (mon.0) 3015 : audit [DBG] from='client.? 172.21.15.6:0/4745628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:35.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:35 smithi006 bash[27292]: cluster 2023-08-28T00:11:34.194506+0000 mgr.y (mgr.14901) 3990 : cluster [DBG] pgmap v3997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:35.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:35 smithi006 bash[27292]: audit 2023-08-28T00:11:34.751210+0000 mon.a (mon.0) 3015 : audit [DBG] from='client.? 172.21.15.6:0/4745628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:36.517 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:36 smithi130 bash[51302]: debug 2023-08-28T00:11:36.246+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:36.248456+0000) 2023-08-28T00:11:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:36 smithi130 bash[27579]: cluster 2023-08-28T00:11:36.195369+0000 mgr.y (mgr.14901) 3991 : cluster [DBG] pgmap v3998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:36.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:36 smithi006 bash[33918]: cluster 2023-08-28T00:11:36.195369+0000 mgr.y (mgr.14901) 3991 : cluster [DBG] pgmap v3998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:36 smithi006 bash[27292]: cluster 2023-08-28T00:11:36.195369+0000 mgr.y (mgr.14901) 3991 : cluster [DBG] pgmap v3998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:37.520 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:37 smithi130 bash[51302]: debug 2023-08-28T00:11:37.246+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:37.248723+0000) 2023-08-28T00:11:37.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:37 smithi130 bash[27579]: audit 2023-08-28T00:11:37.164225+0000 mon.a (mon.0) 3016 : audit [DBG] from='client.? 172.21.15.6:0/115430006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:37.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:37 smithi006 bash[33918]: audit 2023-08-28T00:11:37.164225+0000 mon.a (mon.0) 3016 : audit [DBG] from='client.? 172.21.15.6:0/115430006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:37.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:37 smithi006 bash[27292]: audit 2023-08-28T00:11:37.164225+0000 mon.a (mon.0) 3016 : audit [DBG] from='client.? 172.21.15.6:0/115430006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:38.527 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:38 smithi130 bash[51302]: debug 2023-08-28T00:11:38.246+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:38.248976+0000) 2023-08-28T00:11:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:38 smithi130 bash[27579]: cluster 2023-08-28T00:11:38.196209+0000 mgr.y (mgr.14901) 3992 : cluster [DBG] pgmap v3999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:38 smithi006 bash[27292]: cluster 2023-08-28T00:11:38.196209+0000 mgr.y (mgr.14901) 3992 : cluster [DBG] pgmap v3999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:38 smithi006 bash[33918]: cluster 2023-08-28T00:11:38.196209+0000 mgr.y (mgr.14901) 3992 : cluster [DBG] pgmap v3999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:39.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:39 smithi130 bash[51302]: debug 2023-08-28T00:11:39.246+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:39.249277+0000) 2023-08-28T00:11:39.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:39 smithi006 bash[27292]: audit 2023-08-28T00:11:39.578306+0000 mon.a (mon.0) 3017 : audit [DBG] from='client.? 172.21.15.6:0/1488697128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:39 smithi006 bash[33918]: audit 2023-08-28T00:11:39.578306+0000 mon.a (mon.0) 3017 : audit [DBG] from='client.? 172.21.15.6:0/1488697128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:39 smithi130 bash[27579]: audit 2023-08-28T00:11:39.578306+0000 mon.a (mon.0) 3017 : audit [DBG] from='client.? 172.21.15.6:0/1488697128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:40.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:40 smithi130 bash[51302]: debug 2023-08-28T00:11:40.246+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:40.249501+0000) 2023-08-28T00:11:40.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:40 smithi006 bash[27292]: cluster 2023-08-28T00:11:40.197428+0000 mgr.y (mgr.14901) 3993 : cluster [DBG] pgmap v4000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:11:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:11:40] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:11:40.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:40 smithi006 bash[33918]: cluster 2023-08-28T00:11:40.197428+0000 mgr.y (mgr.14901) 3993 : cluster [DBG] pgmap v4000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:11:40.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:11:40.672Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:40 smithi130 bash[27579]: cluster 2023-08-28T00:11:40.197428+0000 mgr.y (mgr.14901) 3993 : cluster [DBG] pgmap v4000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:41.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:41 smithi130 bash[51302]: debug 2023-08-28T00:11:41.246+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:41.249753+0000) 2023-08-28T00:11:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:42 smithi130 bash[27579]: audit 2023-08-28T00:11:41.992421+0000 mon.c (mon.2) 1519 : audit [DBG] from='client.? 172.21.15.6:0/242143287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:42.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:42 smithi130 bash[51302]: debug 2023-08-28T00:11:42.246+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:42.250009+0000) 2023-08-28T00:11:42.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:42 smithi006 bash[27292]: audit 2023-08-28T00:11:41.992421+0000 mon.c (mon.2) 1519 : audit [DBG] from='client.? 172.21.15.6:0/242143287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:42.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:42 smithi006 bash[33918]: audit 2023-08-28T00:11:41.992421+0000 mon.c (mon.2) 1519 : audit [DBG] from='client.? 172.21.15.6:0/242143287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:11:42.244Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:42.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:11:42.246Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:43.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:43 smithi130 bash[27579]: cluster 2023-08-28T00:11:42.198095+0000 mgr.y (mgr.14901) 3994 : cluster [DBG] pgmap v4001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:43.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:43 smithi130 bash[51302]: debug 2023-08-28T00:11:43.246+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:43.250324+0000) 2023-08-28T00:11:43.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:43 smithi006 bash[27292]: cluster 2023-08-28T00:11:42.198095+0000 mgr.y (mgr.14901) 3994 : cluster [DBG] pgmap v4001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:43.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:43 smithi006 bash[33918]: cluster 2023-08-28T00:11:42.198095+0000 mgr.y (mgr.14901) 3994 : cluster [DBG] pgmap v4001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:11:44.128Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:11:44.130Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:44.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:44 smithi130 bash[51302]: debug 2023-08-28T00:11:44.246+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:44.250628+0000) 2023-08-28T00:11:45.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:45 smithi130 bash[27579]: cluster 2023-08-28T00:11:44.199129+0000 mgr.y (mgr.14901) 3995 : cluster [DBG] pgmap v4002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:45.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:45 smithi130 bash[27579]: audit 2023-08-28T00:11:44.411573+0000 mon.a (mon.0) 3018 : audit [DBG] from='client.? 172.21.15.6:0/3683609899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:45.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:45 smithi130 bash[51302]: debug 2023-08-28T00:11:45.250+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:45.250968+0000) 2023-08-28T00:11:45.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:45 smithi006 bash[27292]: cluster 2023-08-28T00:11:44.199129+0000 mgr.y (mgr.14901) 3995 : cluster [DBG] pgmap v4002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:45.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:45 smithi006 bash[27292]: audit 2023-08-28T00:11:44.411573+0000 mon.a (mon.0) 3018 : audit [DBG] from='client.? 172.21.15.6:0/3683609899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:45.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:45 smithi006 bash[33918]: cluster 2023-08-28T00:11:44.199129+0000 mgr.y (mgr.14901) 3995 : cluster [DBG] pgmap v4002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:45.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:45 smithi006 bash[33918]: audit 2023-08-28T00:11:44.411573+0000 mon.a (mon.0) 3018 : audit [DBG] from='client.? 172.21.15.6:0/3683609899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:46.515 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:46 smithi130 bash[51302]: debug 2023-08-28T00:11:46.250+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:46.251299+0000) 2023-08-28T00:11:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:46 smithi130 bash[27579]: cluster 2023-08-28T00:11:46.199850+0000 mgr.y (mgr.14901) 3996 : cluster [DBG] pgmap v4003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:46 smithi006 bash[27292]: cluster 2023-08-28T00:11:46.199850+0000 mgr.y (mgr.14901) 3996 : cluster [DBG] pgmap v4003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:46 smithi006 bash[33918]: cluster 2023-08-28T00:11:46.199850+0000 mgr.y (mgr.14901) 3996 : cluster [DBG] pgmap v4003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:47.517 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:47 smithi130 bash[51302]: debug 2023-08-28T00:11:47.250+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:47.251548+0000) 2023-08-28T00:11:47.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:47 smithi130 bash[27579]: audit 2023-08-28T00:11:46.831189+0000 mon.a (mon.0) 3019 : audit [DBG] from='client.? 172.21.15.6:0/3387136461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:47.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:47 smithi006 bash[33918]: audit 2023-08-28T00:11:46.831189+0000 mon.a (mon.0) 3019 : audit [DBG] from='client.? 172.21.15.6:0/3387136461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:47.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:47 smithi006 bash[27292]: audit 2023-08-28T00:11:46.831189+0000 mon.a (mon.0) 3019 : audit [DBG] from='client.? 172.21.15.6:0/3387136461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:48.526 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:48 smithi130 bash[51302]: debug 2023-08-28T00:11:48.250+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:48.251806+0000) 2023-08-28T00:11:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:48 smithi130 bash[27579]: cluster 2023-08-28T00:11:48.200640+0000 mgr.y (mgr.14901) 3997 : cluster [DBG] pgmap v4004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:48 smithi006 bash[27292]: cluster 2023-08-28T00:11:48.200640+0000 mgr.y (mgr.14901) 3997 : cluster [DBG] pgmap v4004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:48 smithi006 bash[33918]: cluster 2023-08-28T00:11:48.200640+0000 mgr.y (mgr.14901) 3997 : cluster [DBG] pgmap v4004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:49.528 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:49 smithi130 bash[51302]: debug 2023-08-28T00:11:49.250+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:49.252145+0000) 2023-08-28T00:11:49.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:49 smithi130 bash[27579]: audit 2023-08-28T00:11:49.250309+0000 mon.a (mon.0) 3020 : audit [DBG] from='client.? 172.21.15.6:0/1406607978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:49.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:49 smithi006 bash[27292]: audit 2023-08-28T00:11:49.250309+0000 mon.a (mon.0) 3020 : audit [DBG] from='client.? 172.21.15.6:0/1406607978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:49.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:49 smithi006 bash[33918]: audit 2023-08-28T00:11:49.250309+0000 mon.a (mon.0) 3020 : audit [DBG] from='client.? 172.21.15.6:0/1406607978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:50.535 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:50 smithi130 bash[51302]: debug 2023-08-28T00:11:50.250+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:50.252414+0000) 2023-08-28T00:11:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:50 smithi130 bash[27579]: cluster 2023-08-28T00:11:50.201785+0000 mgr.y (mgr.14901) 3998 : cluster [DBG] pgmap v4005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:11:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:11:50] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:11:50.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:50 smithi006 bash[27292]: cluster 2023-08-28T00:11:50.201785+0000 mgr.y (mgr.14901) 3998 : cluster [DBG] pgmap v4005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:50.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:50 smithi006 bash[33918]: cluster 2023-08-28T00:11:50.201785+0000 mgr.y (mgr.14901) 3998 : cluster [DBG] pgmap v4005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:11:50.671Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:11:50.673Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:51.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:51 smithi130 bash[51302]: debug 2023-08-28T00:11:51.250+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:51.252767+0000) 2023-08-28T00:11:51.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:51 smithi006 bash[27292]: audit 2023-08-28T00:11:51.656686+0000 mon.a (mon.0) 3021 : audit [DBG] from='client.? 172.21.15.6:0/2969770190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:51.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:51 smithi006 bash[33918]: audit 2023-08-28T00:11:51.656686+0000 mon.a (mon.0) 3021 : audit [DBG] from='client.? 172.21.15.6:0/2969770190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:52.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:51 smithi130 bash[27579]: audit 2023-08-28T00:11:51.656686+0000 mon.a (mon.0) 3021 : audit [DBG] from='client.? 172.21.15.6:0/2969770190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:52.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:52 smithi130 bash[51302]: debug 2023-08-28T00:11:52.250+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:52.253062+0000) 2023-08-28T00:11:52.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:11:52.245Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:11:52.247Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:52 smithi130 bash[27579]: cluster 2023-08-28T00:11:52.202459+0000 mgr.y (mgr.14901) 3999 : cluster [DBG] pgmap v4006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:52 smithi130 bash[27579]: audit 2023-08-28T00:11:52.246104+0000 mon.b (mon.1) 507 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:11:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:52 smithi006 bash[27292]: cluster 2023-08-28T00:11:52.202459+0000 mgr.y (mgr.14901) 3999 : cluster [DBG] pgmap v4006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:52 smithi006 bash[27292]: audit 2023-08-28T00:11:52.246104+0000 mon.b (mon.1) 507 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:11:53.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:52 smithi006 bash[33918]: cluster 2023-08-28T00:11:52.202459+0000 mgr.y (mgr.14901) 3999 : cluster [DBG] pgmap v4006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:53.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:52 smithi006 bash[33918]: audit 2023-08-28T00:11:52.246104+0000 mon.b (mon.1) 507 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:11:53.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:53 smithi130 bash[51302]: debug 2023-08-28T00:11:53.250+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:53.253384+0000) 2023-08-28T00:11:54.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:54 smithi006 bash[33918]: audit 2023-08-28T00:11:54.070909+0000 mon.a (mon.0) 3022 : audit [DBG] from='client.? 172.21.15.6:0/311701282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:54.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:54 smithi006 bash[27292]: audit 2023-08-28T00:11:54.070909+0000 mon.a (mon.0) 3022 : audit [DBG] from='client.? 172.21.15.6:0/311701282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:11:54.128Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:11:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:11:54.130Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:11:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:54 smithi130 bash[27579]: audit 2023-08-28T00:11:54.070909+0000 mon.a (mon.0) 3022 : audit [DBG] from='client.? 172.21.15.6:0/311701282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:54.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:54 smithi130 bash[51302]: debug 2023-08-28T00:11:54.250+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:54.253822+0000) 2023-08-28T00:11:55.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:55 smithi006 bash[33918]: cluster 2023-08-28T00:11:54.203620+0000 mgr.y (mgr.14901) 4000 : cluster [DBG] pgmap v4007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:55.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:55 smithi006 bash[27292]: cluster 2023-08-28T00:11:54.203620+0000 mgr.y (mgr.14901) 4000 : cluster [DBG] pgmap v4007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:55.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:55 smithi130 bash[27579]: cluster 2023-08-28T00:11:54.203620+0000 mgr.y (mgr.14901) 4000 : cluster [DBG] pgmap v4007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:55.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:55 smithi130 bash[51302]: debug 2023-08-28T00:11:55.250+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:55.254134+0000) 2023-08-28T00:11:56.526 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:56 smithi130 bash[51302]: debug 2023-08-28T00:11:56.250+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:56.254616+0000) 2023-08-28T00:11:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:56 smithi130 bash[27579]: cluster 2023-08-28T00:11:56.204306+0000 mgr.y (mgr.14901) 4001 : cluster [DBG] pgmap v4008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:56 smithi130 bash[27579]: audit 2023-08-28T00:11:56.495554+0000 mon.a (mon.0) 3023 : audit [DBG] from='client.? 172.21.15.6:0/1408721147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:56 smithi006 bash[33918]: cluster 2023-08-28T00:11:56.204306+0000 mgr.y (mgr.14901) 4001 : cluster [DBG] pgmap v4008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:56 smithi006 bash[33918]: audit 2023-08-28T00:11:56.495554+0000 mon.a (mon.0) 3023 : audit [DBG] from='client.? 172.21.15.6:0/1408721147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:56 smithi006 bash[27292]: cluster 2023-08-28T00:11:56.204306+0000 mgr.y (mgr.14901) 4001 : cluster [DBG] pgmap v4008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:56 smithi006 bash[27292]: audit 2023-08-28T00:11:56.495554+0000 mon.a (mon.0) 3023 : audit [DBG] from='client.? 172.21.15.6:0/1408721147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:57.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:57 smithi130 bash[51302]: debug 2023-08-28T00:11:57.253+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:57.254903+0000) 2023-08-28T00:11:58.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:58 smithi130 bash[51302]: debug 2023-08-28T00:11:58.253+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:58.255198+0000) 2023-08-28T00:11:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:59 smithi006 bash[33918]: audit 2023-08-28T00:11:58.182136+0000 mon.a (mon.0) 3024 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:11:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:59 smithi006 bash[33918]: cluster 2023-08-28T00:11:58.204942+0000 mgr.y (mgr.14901) 4002 : cluster [DBG] pgmap v4009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:59 smithi006 bash[33918]: audit 2023-08-28T00:11:58.553767+0000 mon.a (mon.0) 3025 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:11:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:59 smithi006 bash[33918]: audit 2023-08-28T00:11:58.851759+0000 mon.b (mon.1) 508 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:11:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:59 smithi006 bash[33918]: audit 2023-08-28T00:11:58.853751+0000 mon.b (mon.1) 509 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:11:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:59 smithi006 bash[33918]: audit 2023-08-28T00:11:58.863863+0000 mon.a (mon.0) 3026 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:11:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:11:59 smithi006 bash[33918]: audit 2023-08-28T00:11:58.923562+0000 mon.a (mon.0) 3027 : audit [DBG] from='client.? 172.21.15.6:0/463851103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:59.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:59 smithi006 bash[27292]: audit 2023-08-28T00:11:58.182136+0000 mon.a (mon.0) 3024 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:11:59.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:59 smithi006 bash[27292]: cluster 2023-08-28T00:11:58.204942+0000 mgr.y (mgr.14901) 4002 : cluster [DBG] pgmap v4009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:59.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:59 smithi006 bash[27292]: audit 2023-08-28T00:11:58.553767+0000 mon.a (mon.0) 3025 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:11:59.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:59 smithi006 bash[27292]: audit 2023-08-28T00:11:58.851759+0000 mon.b (mon.1) 508 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:11:59.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:59 smithi006 bash[27292]: audit 2023-08-28T00:11:58.853751+0000 mon.b (mon.1) 509 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:11:59.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:59 smithi006 bash[27292]: audit 2023-08-28T00:11:58.863863+0000 mon.a (mon.0) 3026 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:11:59.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:11:59 smithi006 bash[27292]: audit 2023-08-28T00:11:58.923562+0000 mon.a (mon.0) 3027 : audit [DBG] from='client.? 172.21.15.6:0/463851103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:59.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:11:59 smithi130 bash[51302]: debug 2023-08-28T00:11:59.253+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:59.255443+0000) 2023-08-28T00:11:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:59 smithi130 bash[27579]: audit 2023-08-28T00:11:58.182136+0000 mon.a (mon.0) 3024 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:11:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:59 smithi130 bash[27579]: cluster 2023-08-28T00:11:58.204942+0000 mgr.y (mgr.14901) 4002 : cluster [DBG] pgmap v4009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:11:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:59 smithi130 bash[27579]: audit 2023-08-28T00:11:58.553767+0000 mon.a (mon.0) 3025 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:11:59.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:59 smithi130 bash[27579]: audit 2023-08-28T00:11:58.851759+0000 mon.b (mon.1) 508 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:11:59.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:59 smithi130 bash[27579]: audit 2023-08-28T00:11:58.853751+0000 mon.b (mon.1) 509 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:11:59.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:59 smithi130 bash[27579]: audit 2023-08-28T00:11:58.863863+0000 mon.a (mon.0) 3026 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:11:59.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:11:59 smithi130 bash[27579]: audit 2023-08-28T00:11:58.923562+0000 mon.a (mon.0) 3027 : audit [DBG] from='client.? 172.21.15.6:0/463851103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:00.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:00 smithi130 bash[51302]: debug 2023-08-28T00:12:00.253+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:00.255712+0000) 2023-08-28T00:12:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:12:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:12:00] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:12:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:12:00.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:12:00.674Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:01.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:01 smithi130 bash[27579]: cluster 2023-08-28T00:12:00.206043+0000 mgr.y (mgr.14901) 4003 : cluster [DBG] pgmap v4010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:01.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:01 smithi130 bash[51302]: debug 2023-08-28T00:12:01.253+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:01.256004+0000) 2023-08-28T00:12:01.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:01 smithi006 bash[27292]: cluster 2023-08-28T00:12:00.206043+0000 mgr.y (mgr.14901) 4003 : cluster [DBG] pgmap v4010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:01.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:01 smithi006 bash[33918]: cluster 2023-08-28T00:12:00.206043+0000 mgr.y (mgr.14901) 4003 : cluster [DBG] pgmap v4010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:02 smithi130 bash[27579]: audit 2023-08-28T00:12:01.344397+0000 mon.c (mon.2) 1520 : audit [DBG] from='client.? 172.21.15.6:0/1467006521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:02.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:02 smithi130 bash[51302]: debug 2023-08-28T00:12:02.253+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:02.256294+0000) 2023-08-28T00:12:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:02 smithi006 bash[33918]: audit 2023-08-28T00:12:01.344397+0000 mon.c (mon.2) 1520 : audit [DBG] from='client.? 172.21.15.6:0/1467006521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:12:02.246Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:12:02.247Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:02 smithi006 bash[27292]: audit 2023-08-28T00:12:01.344397+0000 mon.c (mon.2) 1520 : audit [DBG] from='client.? 172.21.15.6:0/1467006521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:03.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:03 smithi130 bash[27579]: cluster 2023-08-28T00:12:02.206680+0000 mgr.y (mgr.14901) 4004 : cluster [DBG] pgmap v4011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:03.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:03 smithi130 bash[51302]: debug 2023-08-28T00:12:03.253+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:03.256604+0000) 2023-08-28T00:12:03.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:03 smithi006 bash[33918]: cluster 2023-08-28T00:12:02.206680+0000 mgr.y (mgr.14901) 4004 : cluster [DBG] pgmap v4011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:03.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:03 smithi006 bash[27292]: cluster 2023-08-28T00:12:02.206680+0000 mgr.y (mgr.14901) 4004 : cluster [DBG] pgmap v4011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:04.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:04 smithi006 bash[27292]: audit 2023-08-28T00:12:03.764411+0000 mon.c (mon.2) 1521 : audit [DBG] from='client.? 172.21.15.6:0/2855455265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:04.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:04 smithi006 bash[33918]: audit 2023-08-28T00:12:03.764411+0000 mon.c (mon.2) 1521 : audit [DBG] from='client.? 172.21.15.6:0/2855455265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:12:04.129Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:12:04.131Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:04 smithi130 bash[27579]: audit 2023-08-28T00:12:03.764411+0000 mon.c (mon.2) 1521 : audit [DBG] from='client.? 172.21.15.6:0/2855455265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:04.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:04 smithi130 bash[51302]: debug 2023-08-28T00:12:04.253+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:04.256882+0000) 2023-08-28T00:12:05.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:05 smithi130 bash[27579]: cluster 2023-08-28T00:12:04.207808+0000 mgr.y (mgr.14901) 4005 : cluster [DBG] pgmap v4012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:05.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:05 smithi130 bash[51302]: debug 2023-08-28T00:12:05.253+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:05.257206+0000) 2023-08-28T00:12:05.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:05 smithi006 bash[27292]: cluster 2023-08-28T00:12:04.207808+0000 mgr.y (mgr.14901) 4005 : cluster [DBG] pgmap v4012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:05.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:05 smithi006 bash[33918]: cluster 2023-08-28T00:12:04.207808+0000 mgr.y (mgr.14901) 4005 : cluster [DBG] pgmap v4012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:06.522 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:06 smithi130 bash[51302]: debug 2023-08-28T00:12:06.253+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:06.257455+0000) 2023-08-28T00:12:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:06 smithi130 bash[27579]: audit 2023-08-28T00:12:06.177938+0000 mon.c (mon.2) 1522 : audit [DBG] from='client.? 172.21.15.6:0/4019408133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:06 smithi130 bash[27579]: cluster 2023-08-28T00:12:06.208453+0000 mgr.y (mgr.14901) 4006 : cluster [DBG] pgmap v4013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:06.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:06 smithi006 bash[27292]: audit 2023-08-28T00:12:06.177938+0000 mon.c (mon.2) 1522 : audit [DBG] from='client.? 172.21.15.6:0/4019408133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:06 smithi006 bash[27292]: cluster 2023-08-28T00:12:06.208453+0000 mgr.y (mgr.14901) 4006 : cluster [DBG] pgmap v4013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:06 smithi006 bash[33918]: audit 2023-08-28T00:12:06.177938+0000 mon.c (mon.2) 1522 : audit [DBG] from='client.? 172.21.15.6:0/4019408133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:06 smithi006 bash[33918]: cluster 2023-08-28T00:12:06.208453+0000 mgr.y (mgr.14901) 4006 : cluster [DBG] pgmap v4013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:07.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:07 smithi130 bash[51302]: debug 2023-08-28T00:12:07.253+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:07.257715+0000) 2023-08-28T00:12:08.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:08 smithi130 bash[51302]: debug 2023-08-28T00:12:08.257+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:08.258040+0000) 2023-08-28T00:12:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:09 smithi130 bash[27579]: cluster 2023-08-28T00:12:08.209063+0000 mgr.y (mgr.14901) 4007 : cluster [DBG] pgmap v4014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:09 smithi130 bash[27579]: audit 2023-08-28T00:12:08.596291+0000 mon.a (mon.0) 3028 : audit [DBG] from='client.? 172.21.15.6:0/3024013700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:09.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:09 smithi130 bash[51302]: debug 2023-08-28T00:12:09.253+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:09.258360+0000) 2023-08-28T00:12:09.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:09 smithi006 bash[33918]: cluster 2023-08-28T00:12:08.209063+0000 mgr.y (mgr.14901) 4007 : cluster [DBG] pgmap v4014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:09.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:09 smithi006 bash[33918]: audit 2023-08-28T00:12:08.596291+0000 mon.a (mon.0) 3028 : audit [DBG] from='client.? 172.21.15.6:0/3024013700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:09 smithi006 bash[27292]: cluster 2023-08-28T00:12:08.209063+0000 mgr.y (mgr.14901) 4007 : cluster [DBG] pgmap v4014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:09 smithi006 bash[27292]: audit 2023-08-28T00:12:08.596291+0000 mon.a (mon.0) 3028 : audit [DBG] from='client.? 172.21.15.6:0/3024013700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:10.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:10 smithi130 bash[51302]: debug 2023-08-28T00:12:10.257+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:10.258630+0000) 2023-08-28T00:12:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:12:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:12:10] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:12:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:12:10.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:12:10.674Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:11.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:11 smithi130 bash[27579]: cluster 2023-08-28T00:12:10.210139+0000 mgr.y (mgr.14901) 4008 : cluster [DBG] pgmap v4015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:11.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:11 smithi130 bash[27579]: audit 2023-08-28T00:12:11.011409+0000 mon.c (mon.2) 1523 : audit [DBG] from='client.? 172.21.15.6:0/3680733109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:11.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:11 smithi130 bash[51302]: debug 2023-08-28T00:12:11.257+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:11.258975+0000) 2023-08-28T00:12:11.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:11 smithi006 bash[33918]: cluster 2023-08-28T00:12:10.210139+0000 mgr.y (mgr.14901) 4008 : cluster [DBG] pgmap v4015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:11.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:11 smithi006 bash[33918]: audit 2023-08-28T00:12:11.011409+0000 mon.c (mon.2) 1523 : audit [DBG] from='client.? 172.21.15.6:0/3680733109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:11.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:11 smithi006 bash[27292]: cluster 2023-08-28T00:12:10.210139+0000 mgr.y (mgr.14901) 4008 : cluster [DBG] pgmap v4015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:11.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:11 smithi006 bash[27292]: audit 2023-08-28T00:12:11.011409+0000 mon.c (mon.2) 1523 : audit [DBG] from='client.? 172.21.15.6:0/3680733109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:12.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:12 smithi130 bash[51302]: debug 2023-08-28T00:12:12.257+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:12.259215+0000) 2023-08-28T00:12:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:12:12.246Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:12:12.248Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:13.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:13 smithi130 bash[27579]: cluster 2023-08-28T00:12:12.210883+0000 mgr.y (mgr.14901) 4009 : cluster [DBG] pgmap v4016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:13.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:13 smithi130 bash[51302]: debug 2023-08-28T00:12:13.257+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:13.259424+0000) 2023-08-28T00:12:13.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:13 smithi006 bash[33918]: cluster 2023-08-28T00:12:12.210883+0000 mgr.y (mgr.14901) 4009 : cluster [DBG] pgmap v4016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:13.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:13 smithi006 bash[27292]: cluster 2023-08-28T00:12:12.210883+0000 mgr.y (mgr.14901) 4009 : cluster [DBG] pgmap v4016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:14.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:14 smithi006 bash[33918]: audit 2023-08-28T00:12:13.427247+0000 mon.a (mon.0) 3029 : audit [DBG] from='client.? 172.21.15.6:0/304318728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:14.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:14 smithi006 bash[27292]: audit 2023-08-28T00:12:13.427247+0000 mon.a (mon.0) 3029 : audit [DBG] from='client.? 172.21.15.6:0/304318728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:12:14.130Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:12:14.132Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:14 smithi130 bash[27579]: audit 2023-08-28T00:12:13.427247+0000 mon.a (mon.0) 3029 : audit [DBG] from='client.? 172.21.15.6:0/304318728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:14.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:14 smithi130 bash[51302]: debug 2023-08-28T00:12:14.257+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:14.259741+0000) 2023-08-28T00:12:15.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:15 smithi130 bash[27579]: cluster 2023-08-28T00:12:14.211983+0000 mgr.y (mgr.14901) 4010 : cluster [DBG] pgmap v4017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:15.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:15 smithi130 bash[51302]: debug 2023-08-28T00:12:15.257+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:15.260007+0000) 2023-08-28T00:12:15.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:15 smithi006 bash[33918]: cluster 2023-08-28T00:12:14.211983+0000 mgr.y (mgr.14901) 4010 : cluster [DBG] pgmap v4017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:15.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:15 smithi006 bash[27292]: cluster 2023-08-28T00:12:14.211983+0000 mgr.y (mgr.14901) 4010 : cluster [DBG] pgmap v4017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:16.522 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:16 smithi130 bash[51302]: debug 2023-08-28T00:12:16.257+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:16.260290+0000) 2023-08-28T00:12:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:16 smithi130 bash[27579]: audit 2023-08-28T00:12:15.843713+0000 mon.a (mon.0) 3030 : audit [DBG] from='client.? 172.21.15.6:0/1399596117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:16 smithi130 bash[27579]: cluster 2023-08-28T00:12:16.212646+0000 mgr.y (mgr.14901) 4011 : cluster [DBG] pgmap v4018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:16 smithi006 bash[33918]: audit 2023-08-28T00:12:15.843713+0000 mon.a (mon.0) 3030 : audit [DBG] from='client.? 172.21.15.6:0/1399596117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:16 smithi006 bash[33918]: cluster 2023-08-28T00:12:16.212646+0000 mgr.y (mgr.14901) 4011 : cluster [DBG] pgmap v4018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:16 smithi006 bash[27292]: audit 2023-08-28T00:12:15.843713+0000 mon.a (mon.0) 3030 : audit [DBG] from='client.? 172.21.15.6:0/1399596117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:16 smithi006 bash[27292]: cluster 2023-08-28T00:12:16.212646+0000 mgr.y (mgr.14901) 4011 : cluster [DBG] pgmap v4018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:17.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:17 smithi130 bash[51302]: debug 2023-08-28T00:12:17.257+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:17.260592+0000) 2023-08-28T00:12:18.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:18 smithi130 bash[51302]: debug 2023-08-28T00:12:18.257+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:18.260857+0000) 2023-08-28T00:12:18.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:18 smithi130 bash[27579]: cluster 2023-08-28T00:12:18.213306+0000 mgr.y (mgr.14901) 4012 : cluster [DBG] pgmap v4019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:18.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:18 smithi130 bash[27579]: audit 2023-08-28T00:12:18.264088+0000 mon.c (mon.2) 1524 : audit [DBG] from='client.? 172.21.15.6:0/89580843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:18 smithi006 bash[33918]: cluster 2023-08-28T00:12:18.213306+0000 mgr.y (mgr.14901) 4012 : cluster [DBG] pgmap v4019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:18 smithi006 bash[33918]: audit 2023-08-28T00:12:18.264088+0000 mon.c (mon.2) 1524 : audit [DBG] from='client.? 172.21.15.6:0/89580843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:18 smithi006 bash[27292]: cluster 2023-08-28T00:12:18.213306+0000 mgr.y (mgr.14901) 4012 : cluster [DBG] pgmap v4019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:18 smithi006 bash[27292]: audit 2023-08-28T00:12:18.264088+0000 mon.c (mon.2) 1524 : audit [DBG] from='client.? 172.21.15.6:0/89580843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:19.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:19 smithi130 bash[51302]: debug 2023-08-28T00:12:19.257+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:19.261173+0000) 2023-08-28T00:12:20.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:20 smithi130 bash[51302]: debug 2023-08-28T00:12:20.257+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:20.261439+0000) 2023-08-28T00:12:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:12:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:12:20] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:12:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:12:20.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:12:20.675Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:21.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:21 smithi130 bash[51302]: debug 2023-08-28T00:12:21.261+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:21.261754+0000) 2023-08-28T00:12:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:21 smithi130 bash[27579]: cluster 2023-08-28T00:12:20.214434+0000 mgr.y (mgr.14901) 4013 : cluster [DBG] pgmap v4020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:21 smithi130 bash[27579]: audit 2023-08-28T00:12:20.692121+0000 mon.a (mon.0) 3031 : audit [DBG] from='client.? 172.21.15.6:0/1748528829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:21.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:21 smithi006 bash[27292]: cluster 2023-08-28T00:12:20.214434+0000 mgr.y (mgr.14901) 4013 : cluster [DBG] pgmap v4020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:21.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:21 smithi006 bash[27292]: audit 2023-08-28T00:12:20.692121+0000 mon.a (mon.0) 3031 : audit [DBG] from='client.? 172.21.15.6:0/1748528829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:21.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:21 smithi006 bash[33918]: cluster 2023-08-28T00:12:20.214434+0000 mgr.y (mgr.14901) 4013 : cluster [DBG] pgmap v4020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:21.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:21 smithi006 bash[33918]: audit 2023-08-28T00:12:20.692121+0000 mon.a (mon.0) 3031 : audit [DBG] from='client.? 172.21.15.6:0/1748528829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:22.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:22 smithi130 bash[51302]: debug 2023-08-28T00:12:22.257+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:22.261970+0000) 2023-08-28T00:12:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:12:22.246Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:12:22.248Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:23.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:23 smithi130 bash[27579]: cluster 2023-08-28T00:12:22.215117+0000 mgr.y (mgr.14901) 4014 : cluster [DBG] pgmap v4021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:23.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:23 smithi130 bash[27579]: audit 2023-08-28T00:12:23.107107+0000 mon.a (mon.0) 3032 : audit [DBG] from='client.? 172.21.15.6:0/1612311321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:23.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:23 smithi130 bash[51302]: debug 2023-08-28T00:12:23.257+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:23.262229+0000) 2023-08-28T00:12:23.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:23 smithi006 bash[33918]: cluster 2023-08-28T00:12:22.215117+0000 mgr.y (mgr.14901) 4014 : cluster [DBG] pgmap v4021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:23.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:23 smithi006 bash[33918]: audit 2023-08-28T00:12:23.107107+0000 mon.a (mon.0) 3032 : audit [DBG] from='client.? 172.21.15.6:0/1612311321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:23.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:23 smithi006 bash[27292]: cluster 2023-08-28T00:12:22.215117+0000 mgr.y (mgr.14901) 4014 : cluster [DBG] pgmap v4021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:23.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:23 smithi006 bash[27292]: audit 2023-08-28T00:12:23.107107+0000 mon.a (mon.0) 3032 : audit [DBG] from='client.? 172.21.15.6:0/1612311321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:12:24.131Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:12:24.133Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:24.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:24 smithi130 bash[51302]: debug 2023-08-28T00:12:24.257+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:24.262544+0000) 2023-08-28T00:12:25.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:25 smithi130 bash[27579]: cluster 2023-08-28T00:12:24.216241+0000 mgr.y (mgr.14901) 4015 : cluster [DBG] pgmap v4022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:25.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:25 smithi130 bash[51302]: debug 2023-08-28T00:12:25.261+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:25.262835+0000) 2023-08-28T00:12:25.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:25 smithi006 bash[27292]: cluster 2023-08-28T00:12:24.216241+0000 mgr.y (mgr.14901) 4015 : cluster [DBG] pgmap v4022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:25.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:25 smithi006 bash[33918]: cluster 2023-08-28T00:12:24.216241+0000 mgr.y (mgr.14901) 4015 : cluster [DBG] pgmap v4022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:26 smithi130 bash[27579]: audit 2023-08-28T00:12:25.528325+0000 mon.a (mon.0) 3033 : audit [DBG] from='client.? 172.21.15.6:0/1877061727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:26.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:26 smithi130 bash[51302]: debug 2023-08-28T00:12:26.261+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:26.263157+0000) 2023-08-28T00:12:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:26 smithi006 bash[27292]: audit 2023-08-28T00:12:25.528325+0000 mon.a (mon.0) 3033 : audit [DBG] from='client.? 172.21.15.6:0/1877061727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:26 smithi006 bash[33918]: audit 2023-08-28T00:12:25.528325+0000 mon.a (mon.0) 3033 : audit [DBG] from='client.? 172.21.15.6:0/1877061727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:27.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:27 smithi130 bash[27579]: cluster 2023-08-28T00:12:26.217003+0000 mgr.y (mgr.14901) 4016 : cluster [DBG] pgmap v4023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:27.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:27 smithi130 bash[51302]: debug 2023-08-28T00:12:27.261+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:27.263485+0000) 2023-08-28T00:12:27.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:27 smithi006 bash[33918]: cluster 2023-08-28T00:12:26.217003+0000 mgr.y (mgr.14901) 4016 : cluster [DBG] pgmap v4023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:27.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:27 smithi006 bash[27292]: cluster 2023-08-28T00:12:26.217003+0000 mgr.y (mgr.14901) 4016 : cluster [DBG] pgmap v4023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:28 smithi130 bash[27579]: audit 2023-08-28T00:12:27.955737+0000 mon.c (mon.2) 1525 : audit [DBG] from='client.? 172.21.15.6:0/1394596472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:28.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:28 smithi130 bash[51302]: debug 2023-08-28T00:12:28.261+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:28.263847+0000) 2023-08-28T00:12:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:28 smithi006 bash[33918]: audit 2023-08-28T00:12:27.955737+0000 mon.c (mon.2) 1525 : audit [DBG] from='client.? 172.21.15.6:0/1394596472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:28.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:28 smithi006 bash[27292]: audit 2023-08-28T00:12:27.955737+0000 mon.c (mon.2) 1525 : audit [DBG] from='client.? 172.21.15.6:0/1394596472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:29.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:29 smithi130 bash[27579]: cluster 2023-08-28T00:12:28.217621+0000 mgr.y (mgr.14901) 4017 : cluster [DBG] pgmap v4024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:29.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:29 smithi130 bash[51302]: debug 2023-08-28T00:12:29.261+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:29.264076+0000) 2023-08-28T00:12:29.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:29 smithi006 bash[33918]: cluster 2023-08-28T00:12:28.217621+0000 mgr.y (mgr.14901) 4017 : cluster [DBG] pgmap v4024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:29.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:29 smithi006 bash[27292]: cluster 2023-08-28T00:12:28.217621+0000 mgr.y (mgr.14901) 4017 : cluster [DBG] pgmap v4024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:30.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:30 smithi130 bash[51302]: debug 2023-08-28T00:12:30.261+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:30.264324+0000) 2023-08-28T00:12:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:12:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:12:30] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T00:12:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:12:30.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:12:30.676Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:31.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:31 smithi130 bash[27579]: cluster 2023-08-28T00:12:30.218701+0000 mgr.y (mgr.14901) 4018 : cluster [DBG] pgmap v4025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:31.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:31 smithi130 bash[27579]: audit 2023-08-28T00:12:30.379880+0000 mon.c (mon.2) 1526 : audit [DBG] from='client.? 172.21.15.6:0/2390554130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:31.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:31 smithi130 bash[51302]: debug 2023-08-28T00:12:31.261+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:31.264602+0000) 2023-08-28T00:12:31.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:31 smithi006 bash[33918]: cluster 2023-08-28T00:12:30.218701+0000 mgr.y (mgr.14901) 4018 : cluster [DBG] pgmap v4025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:31.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:31 smithi006 bash[33918]: audit 2023-08-28T00:12:30.379880+0000 mon.c (mon.2) 1526 : audit [DBG] from='client.? 172.21.15.6:0/2390554130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:31.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:31 smithi006 bash[27292]: cluster 2023-08-28T00:12:30.218701+0000 mgr.y (mgr.14901) 4018 : cluster [DBG] pgmap v4025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:31.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:31 smithi006 bash[27292]: audit 2023-08-28T00:12:30.379880+0000 mon.c (mon.2) 1526 : audit [DBG] from='client.? 172.21.15.6:0/2390554130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:32 smithi130 bash[27579]: cluster 2023-08-28T00:12:32.219372+0000 mgr.y (mgr.14901) 4019 : cluster [DBG] pgmap v4026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:32.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:32 smithi130 bash[51302]: debug 2023-08-28T00:12:32.261+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:32.264848+0000) 2023-08-28T00:12:32.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:32 smithi006 bash[33918]: cluster 2023-08-28T00:12:32.219372+0000 mgr.y (mgr.14901) 4019 : cluster [DBG] pgmap v4026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:32 smithi006 bash[27292]: cluster 2023-08-28T00:12:32.219372+0000 mgr.y (mgr.14901) 4019 : cluster [DBG] pgmap v4026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:12:32.247Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:12:32.249Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:33 smithi130 bash[27579]: audit 2023-08-28T00:12:32.801459+0000 mon.c (mon.2) 1527 : audit [DBG] from='client.? 172.21.15.6:0/553523721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:33.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:33 smithi130 bash[51302]: debug 2023-08-28T00:12:33.261+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:33.265106+0000) 2023-08-28T00:12:33.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:33 smithi006 bash[27292]: audit 2023-08-28T00:12:32.801459+0000 mon.c (mon.2) 1527 : audit [DBG] from='client.? 172.21.15.6:0/553523721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:33.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:33 smithi006 bash[33918]: audit 2023-08-28T00:12:32.801459+0000 mon.c (mon.2) 1527 : audit [DBG] from='client.? 172.21.15.6:0/553523721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:34.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:34 smithi006 bash[27292]: cluster 2023-08-28T00:12:34.220473+0000 mgr.y (mgr.14901) 4020 : cluster [DBG] pgmap v4027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:34.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:34 smithi006 bash[33918]: cluster 2023-08-28T00:12:34.220473+0000 mgr.y (mgr.14901) 4020 : cluster [DBG] pgmap v4027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:12:34.131Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:12:34.134Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:34 smithi130 bash[27579]: cluster 2023-08-28T00:12:34.220473+0000 mgr.y (mgr.14901) 4020 : cluster [DBG] pgmap v4027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:34.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:34 smithi130 bash[51302]: debug 2023-08-28T00:12:34.261+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:34.265388+0000) 2023-08-28T00:12:35.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:35 smithi130 bash[27579]: audit 2023-08-28T00:12:35.216201+0000 mon.a (mon.0) 3034 : audit [DBG] from='client.? 172.21.15.6:0/3617284767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:35.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:35 smithi130 bash[51302]: debug 2023-08-28T00:12:35.265+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:35.265698+0000) 2023-08-28T00:12:35.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:35 smithi006 bash[27292]: audit 2023-08-28T00:12:35.216201+0000 mon.a (mon.0) 3034 : audit [DBG] from='client.? 172.21.15.6:0/3617284767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:35.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:35 smithi006 bash[33918]: audit 2023-08-28T00:12:35.216201+0000 mon.a (mon.0) 3034 : audit [DBG] from='client.? 172.21.15.6:0/3617284767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:36.546 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:36 smithi130 bash[51302]: debug 2023-08-28T00:12:36.261+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:36.265983+0000) 2023-08-28T00:12:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:36 smithi130 bash[27579]: cluster 2023-08-28T00:12:36.221204+0000 mgr.y (mgr.14901) 4021 : cluster [DBG] pgmap v4028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:36 smithi006 bash[33918]: cluster 2023-08-28T00:12:36.221204+0000 mgr.y (mgr.14901) 4021 : cluster [DBG] pgmap v4028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:36 smithi006 bash[27292]: cluster 2023-08-28T00:12:36.221204+0000 mgr.y (mgr.14901) 4021 : cluster [DBG] pgmap v4028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:37.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:37 smithi130 bash[51302]: debug 2023-08-28T00:12:37.265+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:37.266283+0000) 2023-08-28T00:12:37.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:37 smithi006 bash[33918]: audit 2023-08-28T00:12:37.634512+0000 mon.a (mon.0) 3035 : audit [DBG] from='client.? 172.21.15.6:0/107188697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:37.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:37 smithi006 bash[27292]: audit 2023-08-28T00:12:37.634512+0000 mon.a (mon.0) 3035 : audit [DBG] from='client.? 172.21.15.6:0/107188697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:38.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:37 smithi130 bash[27579]: audit 2023-08-28T00:12:37.634512+0000 mon.a (mon.0) 3035 : audit [DBG] from='client.? 172.21.15.6:0/107188697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:38.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:38 smithi130 bash[51302]: debug 2023-08-28T00:12:38.261+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:38.266574+0000) 2023-08-28T00:12:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:38 smithi006 bash[27292]: cluster 2023-08-28T00:12:38.221833+0000 mgr.y (mgr.14901) 4022 : cluster [DBG] pgmap v4029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:38 smithi006 bash[33918]: cluster 2023-08-28T00:12:38.221833+0000 mgr.y (mgr.14901) 4022 : cluster [DBG] pgmap v4029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:38 smithi130 bash[27579]: cluster 2023-08-28T00:12:38.221833+0000 mgr.y (mgr.14901) 4022 : cluster [DBG] pgmap v4029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:39.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:39 smithi130 bash[51302]: debug 2023-08-28T00:12:39.265+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:39.266872+0000) 2023-08-28T00:12:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:40 smithi130 bash[27579]: audit 2023-08-28T00:12:40.052063+0000 mon.a (mon.0) 3036 : audit [DBG] from='client.? 172.21.15.6:0/1342054413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:40.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:40 smithi130 bash[51302]: debug 2023-08-28T00:12:40.265+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:40.267175+0000) 2023-08-28T00:12:40.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:40 smithi006 bash[33918]: audit 2023-08-28T00:12:40.052063+0000 mon.a (mon.0) 3036 : audit [DBG] from='client.? 172.21.15.6:0/1342054413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:40.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:40 smithi006 bash[27292]: audit 2023-08-28T00:12:40.052063+0000 mon.a (mon.0) 3036 : audit [DBG] from='client.? 172.21.15.6:0/1342054413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:12:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:12:40] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T00:12:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:12:40.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:12:40.676Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:41.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:41 smithi130 bash[27579]: cluster 2023-08-28T00:12:40.222974+0000 mgr.y (mgr.14901) 4023 : cluster [DBG] pgmap v4030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:41.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:41 smithi130 bash[51302]: debug 2023-08-28T00:12:41.265+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:41.267589+0000) 2023-08-28T00:12:41.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:41 smithi006 bash[33918]: cluster 2023-08-28T00:12:40.222974+0000 mgr.y (mgr.14901) 4023 : cluster [DBG] pgmap v4030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:41.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:41 smithi006 bash[27292]: cluster 2023-08-28T00:12:40.222974+0000 mgr.y (mgr.14901) 4023 : cluster [DBG] pgmap v4030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:42.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:42 smithi130 bash[51302]: debug 2023-08-28T00:12:42.265+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:42.267873+0000) 2023-08-28T00:12:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:12:42.249Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:12:42.251Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:43 smithi130 bash[27579]: cluster 2023-08-28T00:12:42.223605+0000 mgr.y (mgr.14901) 4024 : cluster [DBG] pgmap v4031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:43 smithi130 bash[27579]: audit 2023-08-28T00:12:42.464189+0000 mon.a (mon.0) 3037 : audit [DBG] from='client.? 172.21.15.6:0/3938459217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:43.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:43 smithi130 bash[51302]: debug 2023-08-28T00:12:43.265+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:43.268144+0000) 2023-08-28T00:12:43.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:43 smithi006 bash[27292]: cluster 2023-08-28T00:12:42.223605+0000 mgr.y (mgr.14901) 4024 : cluster [DBG] pgmap v4031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:43.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:43 smithi006 bash[27292]: audit 2023-08-28T00:12:42.464189+0000 mon.a (mon.0) 3037 : audit [DBG] from='client.? 172.21.15.6:0/3938459217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:43.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:43 smithi006 bash[33918]: cluster 2023-08-28T00:12:42.223605+0000 mgr.y (mgr.14901) 4024 : cluster [DBG] pgmap v4031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:43.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:43 smithi006 bash[33918]: audit 2023-08-28T00:12:42.464189+0000 mon.a (mon.0) 3037 : audit [DBG] from='client.? 172.21.15.6:0/3938459217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:12:44.132Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:12:44.135Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:44.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:44 smithi130 bash[51302]: debug 2023-08-28T00:12:44.265+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:44.268516+0000) 2023-08-28T00:12:45.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:45 smithi130 bash[51302]: debug 2023-08-28T00:12:45.265+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:45.268937+0000) 2023-08-28T00:12:45.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:45 smithi130 bash[27579]: cluster 2023-08-28T00:12:44.224702+0000 mgr.y (mgr.14901) 4025 : cluster [DBG] pgmap v4032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:45.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:45 smithi130 bash[27579]: audit 2023-08-28T00:12:44.881811+0000 mon.a (mon.0) 3038 : audit [DBG] from='client.? 172.21.15.6:0/2064267803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:45.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:45 smithi006 bash[33918]: cluster 2023-08-28T00:12:44.224702+0000 mgr.y (mgr.14901) 4025 : cluster [DBG] pgmap v4032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:45.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:45 smithi006 bash[33918]: audit 2023-08-28T00:12:44.881811+0000 mon.a (mon.0) 3038 : audit [DBG] from='client.? 172.21.15.6:0/2064267803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:45.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:45 smithi006 bash[27292]: cluster 2023-08-28T00:12:44.224702+0000 mgr.y (mgr.14901) 4025 : cluster [DBG] pgmap v4032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:45.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:45 smithi006 bash[27292]: audit 2023-08-28T00:12:44.881811+0000 mon.a (mon.0) 3038 : audit [DBG] from='client.? 172.21.15.6:0/2064267803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:46.549 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:46 smithi130 bash[51302]: debug 2023-08-28T00:12:46.265+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:46.269228+0000) 2023-08-28T00:12:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:46 smithi130 bash[27579]: cluster 2023-08-28T00:12:46.225409+0000 mgr.y (mgr.14901) 4026 : cluster [DBG] pgmap v4033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:46 smithi006 bash[33918]: cluster 2023-08-28T00:12:46.225409+0000 mgr.y (mgr.14901) 4026 : cluster [DBG] pgmap v4033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:46 smithi006 bash[27292]: cluster 2023-08-28T00:12:46.225409+0000 mgr.y (mgr.14901) 4026 : cluster [DBG] pgmap v4033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:47.555 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:47 smithi130 bash[51302]: debug 2023-08-28T00:12:47.265+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:47.269491+0000) 2023-08-28T00:12:47.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:47 smithi130 bash[27579]: audit 2023-08-28T00:12:47.289658+0000 mon.c (mon.2) 1528 : audit [DBG] from='client.? 172.21.15.6:0/1929629320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:47.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:47 smithi006 bash[33918]: audit 2023-08-28T00:12:47.289658+0000 mon.c (mon.2) 1528 : audit [DBG] from='client.? 172.21.15.6:0/1929629320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:47.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:47 smithi006 bash[27292]: audit 2023-08-28T00:12:47.289658+0000 mon.c (mon.2) 1528 : audit [DBG] from='client.? 172.21.15.6:0/1929629320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:48.563 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:48 smithi130 bash[51302]: debug 2023-08-28T00:12:48.265+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:48.269779+0000) 2023-08-28T00:12:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:48 smithi130 bash[27579]: cluster 2023-08-28T00:12:48.226024+0000 mgr.y (mgr.14901) 4027 : cluster [DBG] pgmap v4034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:48 smithi006 bash[33918]: cluster 2023-08-28T00:12:48.226024+0000 mgr.y (mgr.14901) 4027 : cluster [DBG] pgmap v4034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:48 smithi006 bash[27292]: cluster 2023-08-28T00:12:48.226024+0000 mgr.y (mgr.14901) 4027 : cluster [DBG] pgmap v4034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:49.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:49 smithi130 bash[51302]: debug 2023-08-28T00:12:49.265+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:49.270071+0000) 2023-08-28T00:12:50.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:49 smithi130 bash[27579]: audit 2023-08-28T00:12:49.705819+0000 mon.c (mon.2) 1529 : audit [DBG] from='client.? 172.21.15.6:0/3336839607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:50.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:49 smithi006 bash[27292]: audit 2023-08-28T00:12:49.705819+0000 mon.c (mon.2) 1529 : audit [DBG] from='client.? 172.21.15.6:0/3336839607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:50.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:49 smithi006 bash[33918]: audit 2023-08-28T00:12:49.705819+0000 mon.c (mon.2) 1529 : audit [DBG] from='client.? 172.21.15.6:0/3336839607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:50.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:50 smithi130 bash[51302]: debug 2023-08-28T00:12:50.269+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:50.270347+0000) 2023-08-28T00:12:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:12:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:12:50] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:12:50.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:50 smithi006 bash[33918]: cluster 2023-08-28T00:12:50.227133+0000 mgr.y (mgr.14901) 4028 : cluster [DBG] pgmap v4035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:50.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:50 smithi006 bash[27292]: cluster 2023-08-28T00:12:50.227133+0000 mgr.y (mgr.14901) 4028 : cluster [DBG] pgmap v4035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:12:50.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:12:50.677Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:50 smithi130 bash[27579]: cluster 2023-08-28T00:12:50.227133+0000 mgr.y (mgr.14901) 4028 : cluster [DBG] pgmap v4035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:51.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:51 smithi130 bash[51302]: debug 2023-08-28T00:12:51.269+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:51.270629+0000) 2023-08-28T00:12:52.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:52 smithi006 bash[27292]: audit 2023-08-28T00:12:52.118674+0000 mon.c (mon.2) 1530 : audit [DBG] from='client.? 172.21.15.6:0/1879384888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:52.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:52 smithi006 bash[33918]: audit 2023-08-28T00:12:52.118674+0000 mon.c (mon.2) 1530 : audit [DBG] from='client.? 172.21.15.6:0/1879384888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:12:52.249Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:12:52.251Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:52 smithi130 bash[27579]: audit 2023-08-28T00:12:52.118674+0000 mon.c (mon.2) 1530 : audit [DBG] from='client.? 172.21.15.6:0/1879384888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:52.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:52 smithi130 bash[51302]: debug 2023-08-28T00:12:52.269+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:52.270890+0000) 2023-08-28T00:12:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:53 smithi006 bash[27292]: cluster 2023-08-28T00:12:52.227842+0000 mgr.y (mgr.14901) 4029 : cluster [DBG] pgmap v4036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:53.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:53 smithi006 bash[33918]: cluster 2023-08-28T00:12:52.227842+0000 mgr.y (mgr.14901) 4029 : cluster [DBG] pgmap v4036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:53 smithi130 bash[27579]: cluster 2023-08-28T00:12:52.227842+0000 mgr.y (mgr.14901) 4029 : cluster [DBG] pgmap v4036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:53.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:53 smithi130 bash[51302]: debug 2023-08-28T00:12:53.268+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:53.271192+0000) 2023-08-28T00:12:54.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:12:54.133Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:12:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:12:54.135Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:12:54.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:54 smithi130 bash[51302]: debug 2023-08-28T00:12:54.268+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:54.271436+0000) 2023-08-28T00:12:55.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:55 smithi130 bash[51302]: debug 2023-08-28T00:12:55.268+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:55.271725+0000) 2023-08-28T00:12:55.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:55 smithi130 bash[27579]: cluster 2023-08-28T00:12:54.228855+0000 mgr.y (mgr.14901) 4030 : cluster [DBG] pgmap v4037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:55.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:55 smithi130 bash[27579]: audit 2023-08-28T00:12:54.535173+0000 mon.c (mon.2) 1531 : audit [DBG] from='client.? 172.21.15.6:0/3516254547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:55.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:55 smithi006 bash[27292]: cluster 2023-08-28T00:12:54.228855+0000 mgr.y (mgr.14901) 4030 : cluster [DBG] pgmap v4037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:55.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:55 smithi006 bash[27292]: audit 2023-08-28T00:12:54.535173+0000 mon.c (mon.2) 1531 : audit [DBG] from='client.? 172.21.15.6:0/3516254547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:55.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:55 smithi006 bash[33918]: cluster 2023-08-28T00:12:54.228855+0000 mgr.y (mgr.14901) 4030 : cluster [DBG] pgmap v4037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:55.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:55 smithi006 bash[33918]: audit 2023-08-28T00:12:54.535173+0000 mon.c (mon.2) 1531 : audit [DBG] from='client.? 172.21.15.6:0/3516254547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:56.548 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:56 smithi130 bash[51302]: debug 2023-08-28T00:12:56.268+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:56.272069+0000) 2023-08-28T00:12:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:56 smithi130 bash[27579]: cluster 2023-08-28T00:12:56.229461+0000 mgr.y (mgr.14901) 4031 : cluster [DBG] pgmap v4038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:56 smithi006 bash[27292]: cluster 2023-08-28T00:12:56.229461+0000 mgr.y (mgr.14901) 4031 : cluster [DBG] pgmap v4038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:56 smithi006 bash[33918]: cluster 2023-08-28T00:12:56.229461+0000 mgr.y (mgr.14901) 4031 : cluster [DBG] pgmap v4038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:57.551 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:57 smithi130 bash[51302]: debug 2023-08-28T00:12:57.268+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:57.272364+0000) 2023-08-28T00:12:57.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:57 smithi130 bash[27579]: audit 2023-08-28T00:12:56.946128+0000 mon.a (mon.0) 3039 : audit [DBG] from='client.? 172.21.15.6:0/1057492885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:57.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:57 smithi006 bash[27292]: audit 2023-08-28T00:12:56.946128+0000 mon.a (mon.0) 3039 : audit [DBG] from='client.? 172.21.15.6:0/1057492885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:57.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:57 smithi006 bash[33918]: audit 2023-08-28T00:12:56.946128+0000 mon.a (mon.0) 3039 : audit [DBG] from='client.? 172.21.15.6:0/1057492885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:58.558 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:58 smithi130 bash[51302]: debug 2023-08-28T00:12:58.268+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:58.272625+0000) 2023-08-28T00:12:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:58 smithi130 bash[27579]: cluster 2023-08-28T00:12:58.230110+0000 mgr.y (mgr.14901) 4032 : cluster [DBG] pgmap v4039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:58 smithi006 bash[27292]: cluster 2023-08-28T00:12:58.230110+0000 mgr.y (mgr.14901) 4032 : cluster [DBG] pgmap v4039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:58 smithi006 bash[33918]: cluster 2023-08-28T00:12:58.230110+0000 mgr.y (mgr.14901) 4032 : cluster [DBG] pgmap v4039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:12:59.561 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:12:59 smithi130 bash[51302]: debug 2023-08-28T00:12:59.268+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:59.272917+0000) 2023-08-28T00:12:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:59 smithi130 bash[27579]: audit 2023-08-28T00:12:58.873441+0000 mon.b (mon.1) 510 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:12:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:12:59 smithi130 bash[27579]: audit 2023-08-28T00:12:59.354211+0000 mon.c (mon.2) 1532 : audit [DBG] from='client.? 172.21.15.6:0/1840864351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:59.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:59 smithi006 bash[27292]: audit 2023-08-28T00:12:58.873441+0000 mon.b (mon.1) 510 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:12:59.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:12:59 smithi006 bash[27292]: audit 2023-08-28T00:12:59.354211+0000 mon.c (mon.2) 1532 : audit [DBG] from='client.? 172.21.15.6:0/1840864351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:59 smithi006 bash[33918]: audit 2023-08-28T00:12:58.873441+0000 mon.b (mon.1) 510 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:12:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:12:59 smithi006 bash[33918]: audit 2023-08-28T00:12:59.354211+0000 mon.c (mon.2) 1532 : audit [DBG] from='client.? 172.21.15.6:0/1840864351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:00.568 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:00 smithi130 bash[51302]: debug 2023-08-28T00:13:00.272+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:00.273203+0000) 2023-08-28T00:13:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:00 smithi130 bash[27579]: cluster 2023-08-28T00:13:00.231242+0000 mgr.y (mgr.14901) 4033 : cluster [DBG] pgmap v4040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:00.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:00 smithi006 bash[27292]: cluster 2023-08-28T00:13:00.231242+0000 mgr.y (mgr.14901) 4033 : cluster [DBG] pgmap v4040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:13:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:13:00] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:13:00.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:00 smithi006 bash[33918]: cluster 2023-08-28T00:13:00.231242+0000 mgr.y (mgr.14901) 4033 : cluster [DBG] pgmap v4040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:13:00.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:13:00.677Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:01.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:01 smithi130 bash[51302]: debug 2023-08-28T00:13:01.272+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:01.273508+0000) 2023-08-28T00:13:02.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:01 smithi130 bash[27579]: audit 2023-08-28T00:13:01.785813+0000 mon.c (mon.2) 1533 : audit [DBG] from='client.? 172.21.15.6:0/1088235972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:02.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:01 smithi006 bash[27292]: audit 2023-08-28T00:13:01.785813+0000 mon.c (mon.2) 1533 : audit [DBG] from='client.? 172.21.15.6:0/1088235972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:02.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:01 smithi006 bash[33918]: audit 2023-08-28T00:13:01.785813+0000 mon.c (mon.2) 1533 : audit [DBG] from='client.? 172.21.15.6:0/1088235972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:02.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:02 smithi130 bash[51302]: debug 2023-08-28T00:13:02.268+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:02.273783+0000) 2023-08-28T00:13:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:13:02.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:13:02.252Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:02 smithi130 bash[27579]: cluster 2023-08-28T00:13:02.232078+0000 mgr.y (mgr.14901) 4034 : cluster [DBG] pgmap v4041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:02 smithi006 bash[33918]: cluster 2023-08-28T00:13:02.232078+0000 mgr.y (mgr.14901) 4034 : cluster [DBG] pgmap v4041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:02 smithi006 bash[27292]: cluster 2023-08-28T00:13:02.232078+0000 mgr.y (mgr.14901) 4034 : cluster [DBG] pgmap v4041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:03.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:03 smithi130 bash[51302]: debug 2023-08-28T00:13:03.268+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:03.274030+0000) 2023-08-28T00:13:04.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:04 smithi006 bash[27292]: audit 2023-08-28T00:13:04.245901+0000 mon.a (mon.0) 3040 : audit [DBG] from='client.? 172.21.15.6:0/1182995696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:13:04.134Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:13:04.135Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:04.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:04 smithi006 bash[33918]: audit 2023-08-28T00:13:04.245901+0000 mon.a (mon.0) 3040 : audit [DBG] from='client.? 172.21.15.6:0/1182995696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:04 smithi130 bash[27579]: audit 2023-08-28T00:13:04.245901+0000 mon.a (mon.0) 3040 : audit [DBG] from='client.? 172.21.15.6:0/1182995696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:04.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:04 smithi130 bash[51302]: debug 2023-08-28T00:13:04.268+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:04.274269+0000) 2023-08-28T00:13:05.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:05 smithi130 bash[51302]: debug 2023-08-28T00:13:05.268+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:05.274514+0000) 2023-08-28T00:13:05.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:05 smithi130 bash[27579]: cluster 2023-08-28T00:13:04.233137+0000 mgr.y (mgr.14901) 4035 : cluster [DBG] pgmap v4042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:05.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:05 smithi130 bash[27579]: audit 2023-08-28T00:13:04.777475+0000 mon.a (mon.0) 3041 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:13:05.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:05 smithi130 bash[27579]: audit 2023-08-28T00:13:05.155124+0000 mon.a (mon.0) 3042 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:13:05.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:05 smithi006 bash[33918]: cluster 2023-08-28T00:13:04.233137+0000 mgr.y (mgr.14901) 4035 : cluster [DBG] pgmap v4042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:05.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:05 smithi006 bash[33918]: audit 2023-08-28T00:13:04.777475+0000 mon.a (mon.0) 3041 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:13:05.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:05 smithi006 bash[33918]: audit 2023-08-28T00:13:05.155124+0000 mon.a (mon.0) 3042 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:13:05.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:05 smithi006 bash[27292]: cluster 2023-08-28T00:13:04.233137+0000 mgr.y (mgr.14901) 4035 : cluster [DBG] pgmap v4042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:05.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:05 smithi006 bash[27292]: audit 2023-08-28T00:13:04.777475+0000 mon.a (mon.0) 3041 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:13:05.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:05 smithi006 bash[27292]: audit 2023-08-28T00:13:05.155124+0000 mon.a (mon.0) 3042 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:13:06.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:06 smithi130 bash[51302]: debug 2023-08-28T00:13:06.272+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:06.274814+0000) 2023-08-28T00:13:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:06 smithi130 bash[27579]: audit 2023-08-28T00:13:05.436181+0000 mon.b (mon.1) 511 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:13:06.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:06 smithi130 bash[27579]: audit 2023-08-28T00:13:05.438462+0000 mon.b (mon.1) 512 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:13:06.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:06 smithi130 bash[27579]: audit 2023-08-28T00:13:05.446704+0000 mon.a (mon.0) 3043 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:13:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:06 smithi006 bash[33918]: audit 2023-08-28T00:13:05.436181+0000 mon.b (mon.1) 511 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:13:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:06 smithi006 bash[33918]: audit 2023-08-28T00:13:05.438462+0000 mon.b (mon.1) 512 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:13:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:06 smithi006 bash[33918]: audit 2023-08-28T00:13:05.446704+0000 mon.a (mon.0) 3043 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:13:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:06 smithi006 bash[27292]: audit 2023-08-28T00:13:05.436181+0000 mon.b (mon.1) 511 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:13:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:06 smithi006 bash[27292]: audit 2023-08-28T00:13:05.438462+0000 mon.b (mon.1) 512 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:13:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:06 smithi006 bash[27292]: audit 2023-08-28T00:13:05.446704+0000 mon.a (mon.0) 3043 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:13:07.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:07 smithi130 bash[27579]: cluster 2023-08-28T00:13:06.233789+0000 mgr.y (mgr.14901) 4036 : cluster [DBG] pgmap v4043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:07.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:07 smithi130 bash[27579]: audit 2023-08-28T00:13:06.662995+0000 mon.a (mon.0) 3044 : audit [DBG] from='client.? 172.21.15.6:0/599413544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:07.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:07 smithi130 bash[51302]: debug 2023-08-28T00:13:07.272+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:07.275085+0000) 2023-08-28T00:13:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:07 smithi006 bash[33918]: cluster 2023-08-28T00:13:06.233789+0000 mgr.y (mgr.14901) 4036 : cluster [DBG] pgmap v4043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:07 smithi006 bash[33918]: audit 2023-08-28T00:13:06.662995+0000 mon.a (mon.0) 3044 : audit [DBG] from='client.? 172.21.15.6:0/599413544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:07.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:07 smithi006 bash[27292]: cluster 2023-08-28T00:13:06.233789+0000 mgr.y (mgr.14901) 4036 : cluster [DBG] pgmap v4043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:07.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:07 smithi006 bash[27292]: audit 2023-08-28T00:13:06.662995+0000 mon.a (mon.0) 3044 : audit [DBG] from='client.? 172.21.15.6:0/599413544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:08 smithi130 bash[27579]: cluster 2023-08-28T00:13:08.234480+0000 mgr.y (mgr.14901) 4037 : cluster [DBG] pgmap v4044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:08.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:08 smithi130 bash[51302]: debug 2023-08-28T00:13:08.272+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:08.275346+0000) 2023-08-28T00:13:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:08 smithi006 bash[33918]: cluster 2023-08-28T00:13:08.234480+0000 mgr.y (mgr.14901) 4037 : cluster [DBG] pgmap v4044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:08 smithi006 bash[27292]: cluster 2023-08-28T00:13:08.234480+0000 mgr.y (mgr.14901) 4037 : cluster [DBG] pgmap v4044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:09 smithi130 bash[27579]: audit 2023-08-28T00:13:09.081324+0000 mon.a (mon.0) 3045 : audit [DBG] from='client.? 172.21.15.6:0/201547195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:09.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:09 smithi130 bash[51302]: debug 2023-08-28T00:13:09.272+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:09.275626+0000) 2023-08-28T00:13:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:09 smithi006 bash[27292]: audit 2023-08-28T00:13:09.081324+0000 mon.a (mon.0) 3045 : audit [DBG] from='client.? 172.21.15.6:0/201547195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:09.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:09 smithi006 bash[33918]: audit 2023-08-28T00:13:09.081324+0000 mon.a (mon.0) 3045 : audit [DBG] from='client.? 172.21.15.6:0/201547195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:10 smithi130 bash[27579]: cluster 2023-08-28T00:13:10.235594+0000 mgr.y (mgr.14901) 4038 : cluster [DBG] pgmap v4045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:10.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:10 smithi130 bash[51302]: debug 2023-08-28T00:13:10.272+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:10.275928+0000) 2023-08-28T00:13:10.679 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:10 smithi006 bash[27292]: cluster 2023-08-28T00:13:10.235594+0000 mgr.y (mgr.14901) 4038 : cluster [DBG] pgmap v4045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:10.679 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:10 smithi006 bash[33918]: cluster 2023-08-28T00:13:10.235594+0000 mgr.y (mgr.14901) 4038 : cluster [DBG] pgmap v4045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:10.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:13:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:13:10] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:13:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:13:10.676Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:13:10.679Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:11.546 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:11 smithi130 bash[51302]: debug 2023-08-28T00:13:11.272+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:11.276258+0000) 2023-08-28T00:13:11.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:11 smithi130 bash[27579]: audit 2023-08-28T00:13:11.504469+0000 mon.a (mon.0) 3046 : audit [DBG] from='client.? 172.21.15.6:0/3260224416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:11.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:11 smithi006 bash[27292]: audit 2023-08-28T00:13:11.504469+0000 mon.a (mon.0) 3046 : audit [DBG] from='client.? 172.21.15.6:0/3260224416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:11.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:11 smithi006 bash[33918]: audit 2023-08-28T00:13:11.504469+0000 mon.a (mon.0) 3046 : audit [DBG] from='client.? 172.21.15.6:0/3260224416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:12.552 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:13:12.251Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:12.553 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:13:12.253Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:12.553 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:12 smithi130 bash[51302]: debug 2023-08-28T00:13:12.272+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:12.276556+0000) 2023-08-28T00:13:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:12 smithi130 bash[27579]: cluster 2023-08-28T00:13:12.236225+0000 mgr.y (mgr.14901) 4039 : cluster [DBG] pgmap v4046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:12 smithi006 bash[33918]: cluster 2023-08-28T00:13:12.236225+0000 mgr.y (mgr.14901) 4039 : cluster [DBG] pgmap v4046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:12 smithi006 bash[27292]: cluster 2023-08-28T00:13:12.236225+0000 mgr.y (mgr.14901) 4039 : cluster [DBG] pgmap v4046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:13.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:13 smithi130 bash[51302]: debug 2023-08-28T00:13:13.276+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:13.276824+0000) 2023-08-28T00:13:14.278 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:13 smithi130 bash[27579]: audit 2023-08-28T00:13:13.913206+0000 mon.a (mon.0) 3047 : audit [DBG] from='client.? 172.21.15.6:0/3824806151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:14.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:13 smithi006 bash[33918]: audit 2023-08-28T00:13:13.913206+0000 mon.a (mon.0) 3047 : audit [DBG] from='client.? 172.21.15.6:0/3824806151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:13:14.135Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:13:14.137Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:14.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:13 smithi006 bash[27292]: audit 2023-08-28T00:13:13.913206+0000 mon.a (mon.0) 3047 : audit [DBG] from='client.? 172.21.15.6:0/3824806151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:14.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:14 smithi130 bash[51302]: debug 2023-08-28T00:13:14.276+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:14.277106+0000) 2023-08-28T00:13:15.279 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:14 smithi130 bash[27579]: cluster 2023-08-28T00:13:14.237366+0000 mgr.y (mgr.14901) 4040 : cluster [DBG] pgmap v4047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:15.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:14 smithi006 bash[33918]: cluster 2023-08-28T00:13:14.237366+0000 mgr.y (mgr.14901) 4040 : cluster [DBG] pgmap v4047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:15.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:14 smithi006 bash[27292]: cluster 2023-08-28T00:13:14.237366+0000 mgr.y (mgr.14901) 4040 : cluster [DBG] pgmap v4047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:15.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:15 smithi130 bash[51302]: debug 2023-08-28T00:13:15.272+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:15.277416+0000) 2023-08-28T00:13:16.551 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:16 smithi130 bash[51302]: debug 2023-08-28T00:13:16.276+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:16.277694+0000) 2023-08-28T00:13:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:16 smithi130 bash[27579]: cluster 2023-08-28T00:13:16.237976+0000 mgr.y (mgr.14901) 4041 : cluster [DBG] pgmap v4048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:16 smithi130 bash[27579]: audit 2023-08-28T00:13:16.317935+0000 mon.a (mon.0) 3048 : audit [DBG] from='client.? 172.21.15.6:0/817935088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:16 smithi006 bash[33918]: cluster 2023-08-28T00:13:16.237976+0000 mgr.y (mgr.14901) 4041 : cluster [DBG] pgmap v4048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:16 smithi006 bash[33918]: audit 2023-08-28T00:13:16.317935+0000 mon.a (mon.0) 3048 : audit [DBG] from='client.? 172.21.15.6:0/817935088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:16 smithi006 bash[27292]: cluster 2023-08-28T00:13:16.237976+0000 mgr.y (mgr.14901) 4041 : cluster [DBG] pgmap v4048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:16 smithi006 bash[27292]: audit 2023-08-28T00:13:16.317935+0000 mon.a (mon.0) 3048 : audit [DBG] from='client.? 172.21.15.6:0/817935088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:17.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:17 smithi130 bash[51302]: debug 2023-08-28T00:13:17.276+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:17.277963+0000) 2023-08-28T00:13:18.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:18 smithi130 bash[51302]: debug 2023-08-28T00:13:18.272+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:18.278233+0000) 2023-08-28T00:13:19.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:19 smithi130 bash[51302]: debug 2023-08-28T00:13:19.272+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:19.278538+0000) 2023-08-28T00:13:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:19 smithi130 bash[27579]: cluster 2023-08-28T00:13:18.238639+0000 mgr.y (mgr.14901) 4042 : cluster [DBG] pgmap v4049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:19 smithi130 bash[27579]: audit 2023-08-28T00:13:18.730216+0000 mon.a (mon.0) 3049 : audit [DBG] from='client.? 172.21.15.6:0/1243420091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:19.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:19 smithi006 bash[33918]: cluster 2023-08-28T00:13:18.238639+0000 mgr.y (mgr.14901) 4042 : cluster [DBG] pgmap v4049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:19.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:19 smithi006 bash[33918]: audit 2023-08-28T00:13:18.730216+0000 mon.a (mon.0) 3049 : audit [DBG] from='client.? 172.21.15.6:0/1243420091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:19.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:19 smithi006 bash[27292]: cluster 2023-08-28T00:13:18.238639+0000 mgr.y (mgr.14901) 4042 : cluster [DBG] pgmap v4049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:19.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:19 smithi006 bash[27292]: audit 2023-08-28T00:13:18.730216+0000 mon.a (mon.0) 3049 : audit [DBG] from='client.? 172.21.15.6:0/1243420091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:20.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:20 smithi130 bash[51302]: debug 2023-08-28T00:13:20.276+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:20.278902+0000) 2023-08-28T00:13:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:13:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:13:20] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:13:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:13:20.676Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:13:20.678Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:21 smithi130 bash[27579]: cluster 2023-08-28T00:13:20.239759+0000 mgr.y (mgr.14901) 4043 : cluster [DBG] pgmap v4050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:21 smithi130 bash[27579]: audit 2023-08-28T00:13:21.143504+0000 mon.c (mon.2) 1534 : audit [DBG] from='client.? 172.21.15.6:0/3630378669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:21.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:21 smithi130 bash[51302]: debug 2023-08-28T00:13:21.276+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:21.279204+0000) 2023-08-28T00:13:21.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:21 smithi006 bash[27292]: cluster 2023-08-28T00:13:20.239759+0000 mgr.y (mgr.14901) 4043 : cluster [DBG] pgmap v4050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:21.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:21 smithi006 bash[27292]: audit 2023-08-28T00:13:21.143504+0000 mon.c (mon.2) 1534 : audit [DBG] from='client.? 172.21.15.6:0/3630378669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:21.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:21 smithi006 bash[33918]: cluster 2023-08-28T00:13:20.239759+0000 mgr.y (mgr.14901) 4043 : cluster [DBG] pgmap v4050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:21.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:21 smithi006 bash[33918]: audit 2023-08-28T00:13:21.143504+0000 mon.c (mon.2) 1534 : audit [DBG] from='client.? 172.21.15.6:0/3630378669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:22.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:22 smithi130 bash[51302]: debug 2023-08-28T00:13:22.276+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:22.279486+0000) 2023-08-28T00:13:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:13:22.252Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:13:22.253Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:23.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:23 smithi130 bash[27579]: cluster 2023-08-28T00:13:22.240432+0000 mgr.y (mgr.14901) 4044 : cluster [DBG] pgmap v4051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:23.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:23 smithi130 bash[51302]: debug 2023-08-28T00:13:23.276+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:23.279753+0000) 2023-08-28T00:13:23.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:23 smithi006 bash[27292]: cluster 2023-08-28T00:13:22.240432+0000 mgr.y (mgr.14901) 4044 : cluster [DBG] pgmap v4051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:23.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:23 smithi006 bash[33918]: cluster 2023-08-28T00:13:22.240432+0000 mgr.y (mgr.14901) 4044 : cluster [DBG] pgmap v4051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:24.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:24 smithi006 bash[27292]: audit 2023-08-28T00:13:23.555904+0000 mon.c (mon.2) 1535 : audit [DBG] from='client.? 172.21.15.6:0/2167951555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:24.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:24 smithi006 bash[33918]: audit 2023-08-28T00:13:23.555904+0000 mon.c (mon.2) 1535 : audit [DBG] from='client.? 172.21.15.6:0/2167951555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:13:24.135Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:13:24.138Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:24 smithi130 bash[27579]: audit 2023-08-28T00:13:23.555904+0000 mon.c (mon.2) 1535 : audit [DBG] from='client.? 172.21.15.6:0/2167951555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:24.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:24 smithi130 bash[51302]: debug 2023-08-28T00:13:24.276+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:24.280067+0000) 2023-08-28T00:13:25.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:25 smithi130 bash[27579]: cluster 2023-08-28T00:13:24.241524+0000 mgr.y (mgr.14901) 4045 : cluster [DBG] pgmap v4052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:25.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:25 smithi130 bash[51302]: debug 2023-08-28T00:13:25.276+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:25.280335+0000) 2023-08-28T00:13:25.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:25 smithi006 bash[27292]: cluster 2023-08-28T00:13:24.241524+0000 mgr.y (mgr.14901) 4045 : cluster [DBG] pgmap v4052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:25.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:25 smithi006 bash[33918]: cluster 2023-08-28T00:13:24.241524+0000 mgr.y (mgr.14901) 4045 : cluster [DBG] pgmap v4052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:26.558 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:26 smithi130 bash[51302]: debug 2023-08-28T00:13:26.280+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:26.280644+0000) 2023-08-28T00:13:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:26 smithi130 bash[27579]: audit 2023-08-28T00:13:25.969375+0000 mon.c (mon.2) 1536 : audit [DBG] from='client.? 172.21.15.6:0/4003921657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:26 smithi130 bash[27579]: cluster 2023-08-28T00:13:26.242199+0000 mgr.y (mgr.14901) 4046 : cluster [DBG] pgmap v4053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:26 smithi006 bash[33918]: audit 2023-08-28T00:13:25.969375+0000 mon.c (mon.2) 1536 : audit [DBG] from='client.? 172.21.15.6:0/4003921657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:26 smithi006 bash[33918]: cluster 2023-08-28T00:13:26.242199+0000 mgr.y (mgr.14901) 4046 : cluster [DBG] pgmap v4053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:26 smithi006 bash[27292]: audit 2023-08-28T00:13:25.969375+0000 mon.c (mon.2) 1536 : audit [DBG] from='client.? 172.21.15.6:0/4003921657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:26 smithi006 bash[27292]: cluster 2023-08-28T00:13:26.242199+0000 mgr.y (mgr.14901) 4046 : cluster [DBG] pgmap v4053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:27.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:27 smithi130 bash[51302]: debug 2023-08-28T00:13:27.280+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:27.280911+0000) 2023-08-28T00:13:28.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:28 smithi130 bash[51302]: debug 2023-08-28T00:13:28.280+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:28.281162+0000) 2023-08-28T00:13:29.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:29 smithi130 bash[27579]: cluster 2023-08-28T00:13:28.242839+0000 mgr.y (mgr.14901) 4047 : cluster [DBG] pgmap v4054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:29.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:29 smithi130 bash[27579]: audit 2023-08-28T00:13:28.388958+0000 mon.a (mon.0) 3050 : audit [DBG] from='client.? 172.21.15.6:0/3162061019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:29.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:29 smithi130 bash[51302]: debug 2023-08-28T00:13:29.276+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:29.281458+0000) 2023-08-28T00:13:29.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:29 smithi006 bash[33918]: cluster 2023-08-28T00:13:28.242839+0000 mgr.y (mgr.14901) 4047 : cluster [DBG] pgmap v4054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:29.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:29 smithi006 bash[33918]: audit 2023-08-28T00:13:28.388958+0000 mon.a (mon.0) 3050 : audit [DBG] from='client.? 172.21.15.6:0/3162061019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:29.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:29 smithi006 bash[27292]: cluster 2023-08-28T00:13:28.242839+0000 mgr.y (mgr.14901) 4047 : cluster [DBG] pgmap v4054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:29.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:29 smithi006 bash[27292]: audit 2023-08-28T00:13:28.388958+0000 mon.a (mon.0) 3050 : audit [DBG] from='client.? 172.21.15.6:0/3162061019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:30.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:30 smithi130 bash[51302]: debug 2023-08-28T00:13:30.276+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:30.281728+0000) 2023-08-28T00:13:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:13:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:13:30] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:13:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:13:30.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:13:30.679Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:31.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:31 smithi130 bash[27579]: cluster 2023-08-28T00:13:30.243898+0000 mgr.y (mgr.14901) 4048 : cluster [DBG] pgmap v4055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:31.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:31 smithi130 bash[27579]: audit 2023-08-28T00:13:30.816151+0000 mon.c (mon.2) 1537 : audit [DBG] from='client.? 172.21.15.6:0/3755076375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:31.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:31 smithi130 bash[51302]: debug 2023-08-28T00:13:31.280+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:31.281993+0000) 2023-08-28T00:13:31.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:31 smithi006 bash[27292]: cluster 2023-08-28T00:13:30.243898+0000 mgr.y (mgr.14901) 4048 : cluster [DBG] pgmap v4055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:31.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:31 smithi006 bash[27292]: audit 2023-08-28T00:13:30.816151+0000 mon.c (mon.2) 1537 : audit [DBG] from='client.? 172.21.15.6:0/3755076375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:31.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:31 smithi006 bash[33918]: cluster 2023-08-28T00:13:30.243898+0000 mgr.y (mgr.14901) 4048 : cluster [DBG] pgmap v4055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:31.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:31 smithi006 bash[33918]: audit 2023-08-28T00:13:30.816151+0000 mon.c (mon.2) 1537 : audit [DBG] from='client.? 172.21.15.6:0/3755076375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:32.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:32 smithi130 bash[51302]: debug 2023-08-28T00:13:32.276+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:32.282308+0000) 2023-08-28T00:13:32.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:13:32.253Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:13:32.254Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:33 smithi130 bash[27579]: cluster 2023-08-28T00:13:32.244525+0000 mgr.y (mgr.14901) 4049 : cluster [DBG] pgmap v4056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:33 smithi130 bash[27579]: audit 2023-08-28T00:13:33.227740+0000 mon.a (mon.0) 3051 : audit [DBG] from='client.? 172.21.15.6:0/2201689146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:33.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:33 smithi130 bash[51302]: debug 2023-08-28T00:13:33.276+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:33.282643+0000) 2023-08-28T00:13:33.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:33 smithi006 bash[27292]: cluster 2023-08-28T00:13:32.244525+0000 mgr.y (mgr.14901) 4049 : cluster [DBG] pgmap v4056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:33.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:33 smithi006 bash[27292]: audit 2023-08-28T00:13:33.227740+0000 mon.a (mon.0) 3051 : audit [DBG] from='client.? 172.21.15.6:0/2201689146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:33.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:33 smithi006 bash[33918]: cluster 2023-08-28T00:13:32.244525+0000 mgr.y (mgr.14901) 4049 : cluster [DBG] pgmap v4056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:33.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:33 smithi006 bash[33918]: audit 2023-08-28T00:13:33.227740+0000 mon.a (mon.0) 3051 : audit [DBG] from='client.? 172.21.15.6:0/2201689146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:13:34.136Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:13:34.138Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:34.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:34 smithi130 bash[51302]: debug 2023-08-28T00:13:34.280+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:34.282937+0000) 2023-08-28T00:13:35.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:35 smithi130 bash[27579]: cluster 2023-08-28T00:13:34.245642+0000 mgr.y (mgr.14901) 4050 : cluster [DBG] pgmap v4057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:35.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:35 smithi130 bash[51302]: debug 2023-08-28T00:13:35.280+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:35.283215+0000) 2023-08-28T00:13:35.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:35 smithi006 bash[33918]: cluster 2023-08-28T00:13:34.245642+0000 mgr.y (mgr.14901) 4050 : cluster [DBG] pgmap v4057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:35.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:35 smithi006 bash[27292]: cluster 2023-08-28T00:13:34.245642+0000 mgr.y (mgr.14901) 4050 : cluster [DBG] pgmap v4057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:36.559 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:36 smithi130 bash[51302]: debug 2023-08-28T00:13:36.280+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:36.283538+0000) 2023-08-28T00:13:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:36 smithi130 bash[27579]: audit 2023-08-28T00:13:35.643263+0000 mon.a (mon.0) 3052 : audit [DBG] from='client.? 172.21.15.6:0/2916755042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:36 smithi130 bash[27579]: cluster 2023-08-28T00:13:36.246317+0000 mgr.y (mgr.14901) 4051 : cluster [DBG] pgmap v4058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:36 smithi006 bash[27292]: audit 2023-08-28T00:13:35.643263+0000 mon.a (mon.0) 3052 : audit [DBG] from='client.? 172.21.15.6:0/2916755042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:36 smithi006 bash[27292]: cluster 2023-08-28T00:13:36.246317+0000 mgr.y (mgr.14901) 4051 : cluster [DBG] pgmap v4058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:36 smithi006 bash[33918]: audit 2023-08-28T00:13:35.643263+0000 mon.a (mon.0) 3052 : audit [DBG] from='client.? 172.21.15.6:0/2916755042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:36 smithi006 bash[33918]: cluster 2023-08-28T00:13:36.246317+0000 mgr.y (mgr.14901) 4051 : cluster [DBG] pgmap v4058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:37.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:37 smithi130 bash[51302]: debug 2023-08-28T00:13:37.280+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:37.283801+0000) 2023-08-28T00:13:38.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:38 smithi006 bash[27292]: audit 2023-08-28T00:13:38.068891+0000 mon.c (mon.2) 1538 : audit [DBG] from='client.? 172.21.15.6:0/3309755708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:38.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:38 smithi006 bash[33918]: audit 2023-08-28T00:13:38.068891+0000 mon.c (mon.2) 1538 : audit [DBG] from='client.? 172.21.15.6:0/3309755708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:38 smithi130 bash[27579]: audit 2023-08-28T00:13:38.068891+0000 mon.c (mon.2) 1538 : audit [DBG] from='client.? 172.21.15.6:0/3309755708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:38.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:38 smithi130 bash[51302]: debug 2023-08-28T00:13:38.280+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:38.284081+0000) 2023-08-28T00:13:39.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:39 smithi006 bash[33918]: cluster 2023-08-28T00:13:38.246987+0000 mgr.y (mgr.14901) 4052 : cluster [DBG] pgmap v4059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:39 smithi006 bash[27292]: cluster 2023-08-28T00:13:38.246987+0000 mgr.y (mgr.14901) 4052 : cluster [DBG] pgmap v4059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:39.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:39 smithi130 bash[51302]: debug 2023-08-28T00:13:39.280+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:39.284381+0000) 2023-08-28T00:13:39.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:39 smithi130 bash[27579]: cluster 2023-08-28T00:13:38.246987+0000 mgr.y (mgr.14901) 4052 : cluster [DBG] pgmap v4059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:40.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:40 smithi130 bash[51302]: debug 2023-08-28T00:13:40.280+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:40.284685+0000) 2023-08-28T00:13:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:13:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:13:40] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:13:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:13:40.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:13:40.679Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:41.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:41 smithi130 bash[51302]: debug 2023-08-28T00:13:41.284+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:41.285002+0000) 2023-08-28T00:13:41.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:41 smithi130 bash[27579]: cluster 2023-08-28T00:13:40.248104+0000 mgr.y (mgr.14901) 4053 : cluster [DBG] pgmap v4060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:41.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:41 smithi130 bash[27579]: audit 2023-08-28T00:13:40.490793+0000 mon.c (mon.2) 1539 : audit [DBG] from='client.? 172.21.15.6:0/3006935043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:41.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:41 smithi006 bash[33918]: cluster 2023-08-28T00:13:40.248104+0000 mgr.y (mgr.14901) 4053 : cluster [DBG] pgmap v4060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:41.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:41 smithi006 bash[33918]: audit 2023-08-28T00:13:40.490793+0000 mon.c (mon.2) 1539 : audit [DBG] from='client.? 172.21.15.6:0/3006935043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:41.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:41 smithi006 bash[27292]: cluster 2023-08-28T00:13:40.248104+0000 mgr.y (mgr.14901) 4053 : cluster [DBG] pgmap v4060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:41.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:41 smithi006 bash[27292]: audit 2023-08-28T00:13:40.490793+0000 mon.c (mon.2) 1539 : audit [DBG] from='client.? 172.21.15.6:0/3006935043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:42.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:42 smithi130 bash[51302]: debug 2023-08-28T00:13:42.280+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:42.285302+0000) 2023-08-28T00:13:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:13:42.253Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:13:42.255Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:43 smithi130 bash[27579]: cluster 2023-08-28T00:13:42.248755+0000 mgr.y (mgr.14901) 4054 : cluster [DBG] pgmap v4061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:43 smithi130 bash[27579]: audit 2023-08-28T00:13:42.901982+0000 mon.c (mon.2) 1540 : audit [DBG] from='client.? 172.21.15.6:0/261230587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:43.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:43 smithi130 bash[51302]: debug 2023-08-28T00:13:43.284+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:43.285621+0000) 2023-08-28T00:13:43.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:43 smithi006 bash[27292]: cluster 2023-08-28T00:13:42.248755+0000 mgr.y (mgr.14901) 4054 : cluster [DBG] pgmap v4061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:43.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:43 smithi006 bash[27292]: audit 2023-08-28T00:13:42.901982+0000 mon.c (mon.2) 1540 : audit [DBG] from='client.? 172.21.15.6:0/261230587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:43.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:43 smithi006 bash[33918]: cluster 2023-08-28T00:13:42.248755+0000 mgr.y (mgr.14901) 4054 : cluster [DBG] pgmap v4061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:43.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:43 smithi006 bash[33918]: audit 2023-08-28T00:13:42.901982+0000 mon.c (mon.2) 1540 : audit [DBG] from='client.? 172.21.15.6:0/261230587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:13:44.137Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:13:44.139Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:44.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:44 smithi130 bash[51302]: debug 2023-08-28T00:13:44.284+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:44.285905+0000) 2023-08-28T00:13:45.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:45 smithi130 bash[27579]: cluster 2023-08-28T00:13:44.249914+0000 mgr.y (mgr.14901) 4055 : cluster [DBG] pgmap v4062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:45.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:45 smithi130 bash[51302]: debug 2023-08-28T00:13:45.284+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:45.286200+0000) 2023-08-28T00:13:45.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:45 smithi006 bash[27292]: cluster 2023-08-28T00:13:44.249914+0000 mgr.y (mgr.14901) 4055 : cluster [DBG] pgmap v4062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:45.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:45 smithi006 bash[33918]: cluster 2023-08-28T00:13:44.249914+0000 mgr.y (mgr.14901) 4055 : cluster [DBG] pgmap v4062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:46 smithi130 bash[27579]: audit 2023-08-28T00:13:45.313747+0000 mon.a (mon.0) 3053 : audit [DBG] from='client.? 172.21.15.6:0/3920332263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:46.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:46 smithi130 bash[51302]: debug 2023-08-28T00:13:46.284+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:46.286468+0000) 2023-08-28T00:13:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:46 smithi006 bash[27292]: audit 2023-08-28T00:13:45.313747+0000 mon.a (mon.0) 3053 : audit [DBG] from='client.? 172.21.15.6:0/3920332263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:46 smithi006 bash[33918]: audit 2023-08-28T00:13:45.313747+0000 mon.a (mon.0) 3053 : audit [DBG] from='client.? 172.21.15.6:0/3920332263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:47.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:47 smithi130 bash[27579]: cluster 2023-08-28T00:13:46.250551+0000 mgr.y (mgr.14901) 4056 : cluster [DBG] pgmap v4063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:47.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:47 smithi130 bash[51302]: debug 2023-08-28T00:13:47.284+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:47.286744+0000) 2023-08-28T00:13:47.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:47 smithi006 bash[27292]: cluster 2023-08-28T00:13:46.250551+0000 mgr.y (mgr.14901) 4056 : cluster [DBG] pgmap v4063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:47.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:47 smithi006 bash[33918]: cluster 2023-08-28T00:13:46.250551+0000 mgr.y (mgr.14901) 4056 : cluster [DBG] pgmap v4063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:48 smithi130 bash[27579]: audit 2023-08-28T00:13:47.727927+0000 mon.a (mon.0) 3054 : audit [DBG] from='client.? 172.21.15.6:0/3262120866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:48 smithi130 bash[27579]: cluster 2023-08-28T00:13:48.251205+0000 mgr.y (mgr.14901) 4057 : cluster [DBG] pgmap v4064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:48.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:48 smithi130 bash[51302]: debug 2023-08-28T00:13:48.283+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:48.287030+0000) 2023-08-28T00:13:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:48 smithi006 bash[27292]: audit 2023-08-28T00:13:47.727927+0000 mon.a (mon.0) 3054 : audit [DBG] from='client.? 172.21.15.6:0/3262120866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:48 smithi006 bash[27292]: cluster 2023-08-28T00:13:48.251205+0000 mgr.y (mgr.14901) 4057 : cluster [DBG] pgmap v4064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:48 smithi006 bash[33918]: audit 2023-08-28T00:13:47.727927+0000 mon.a (mon.0) 3054 : audit [DBG] from='client.? 172.21.15.6:0/3262120866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:48 smithi006 bash[33918]: cluster 2023-08-28T00:13:48.251205+0000 mgr.y (mgr.14901) 4057 : cluster [DBG] pgmap v4064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:49.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:49 smithi130 bash[51302]: debug 2023-08-28T00:13:49.283+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:49.287311+0000) 2023-08-28T00:13:50.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:50 smithi006 bash[27292]: audit 2023-08-28T00:13:50.139732+0000 mon.a (mon.0) 3055 : audit [DBG] from='client.? 172.21.15.6:0/1404486262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:50.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:50 smithi006 bash[33918]: audit 2023-08-28T00:13:50.139732+0000 mon.a (mon.0) 3055 : audit [DBG] from='client.? 172.21.15.6:0/1404486262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:50 smithi130 bash[27579]: audit 2023-08-28T00:13:50.139732+0000 mon.a (mon.0) 3055 : audit [DBG] from='client.? 172.21.15.6:0/1404486262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:50.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:50 smithi130 bash[51302]: debug 2023-08-28T00:13:50.283+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:50.287546+0000) 2023-08-28T00:13:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:13:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:13:50] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:13:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:13:50.678Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:13:50.680Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:51.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:51 smithi006 bash[27292]: cluster 2023-08-28T00:13:50.252324+0000 mgr.y (mgr.14901) 4058 : cluster [DBG] pgmap v4065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:51.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:51 smithi006 bash[33918]: cluster 2023-08-28T00:13:50.252324+0000 mgr.y (mgr.14901) 4058 : cluster [DBG] pgmap v4065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:51.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:51 smithi130 bash[27579]: cluster 2023-08-28T00:13:50.252324+0000 mgr.y (mgr.14901) 4058 : cluster [DBG] pgmap v4065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:51.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:51 smithi130 bash[51302]: debug 2023-08-28T00:13:51.283+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:51.287855+0000) 2023-08-28T00:13:52.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:52 smithi130 bash[51302]: debug 2023-08-28T00:13:52.283+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:52.288117+0000) 2023-08-28T00:13:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:13:52.254Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:13:52.256Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:53.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:53 smithi130 bash[51302]: debug 2023-08-28T00:13:53.287+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:53.288411+0000) 2023-08-28T00:13:53.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:53 smithi130 bash[27579]: cluster 2023-08-28T00:13:52.253020+0000 mgr.y (mgr.14901) 4059 : cluster [DBG] pgmap v4066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:53.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:53 smithi130 bash[27579]: audit 2023-08-28T00:13:52.552620+0000 mon.c (mon.2) 1541 : audit [DBG] from='client.? 172.21.15.6:0/1645977046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:53.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:53 smithi006 bash[33918]: cluster 2023-08-28T00:13:52.253020+0000 mgr.y (mgr.14901) 4059 : cluster [DBG] pgmap v4066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:53.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:53 smithi006 bash[33918]: audit 2023-08-28T00:13:52.552620+0000 mon.c (mon.2) 1541 : audit [DBG] from='client.? 172.21.15.6:0/1645977046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:53.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:53 smithi006 bash[27292]: cluster 2023-08-28T00:13:52.253020+0000 mgr.y (mgr.14901) 4059 : cluster [DBG] pgmap v4066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:53.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:53 smithi006 bash[27292]: audit 2023-08-28T00:13:52.552620+0000 mon.c (mon.2) 1541 : audit [DBG] from='client.? 172.21.15.6:0/1645977046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:13:54.138Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:13:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:13:54.140Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:13:54.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:54 smithi130 bash[51302]: debug 2023-08-28T00:13:54.287+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:54.288774+0000) 2023-08-28T00:13:55.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:55 smithi130 bash[51302]: debug 2023-08-28T00:13:55.283+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:55.289083+0000) 2023-08-28T00:13:55.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:55 smithi130 bash[27579]: cluster 2023-08-28T00:13:54.254066+0000 mgr.y (mgr.14901) 4060 : cluster [DBG] pgmap v4067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:55.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:55 smithi130 bash[27579]: audit 2023-08-28T00:13:54.962694+0000 mon.c (mon.2) 1542 : audit [DBG] from='client.? 172.21.15.6:0/4149829248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:55.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:55 smithi006 bash[27292]: cluster 2023-08-28T00:13:54.254066+0000 mgr.y (mgr.14901) 4060 : cluster [DBG] pgmap v4067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:55.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:55 smithi006 bash[27292]: audit 2023-08-28T00:13:54.962694+0000 mon.c (mon.2) 1542 : audit [DBG] from='client.? 172.21.15.6:0/4149829248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:55.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:55 smithi006 bash[33918]: cluster 2023-08-28T00:13:54.254066+0000 mgr.y (mgr.14901) 4060 : cluster [DBG] pgmap v4067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:55.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:55 smithi006 bash[33918]: audit 2023-08-28T00:13:54.962694+0000 mon.c (mon.2) 1542 : audit [DBG] from='client.? 172.21.15.6:0/4149829248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:56.564 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:56 smithi130 bash[51302]: debug 2023-08-28T00:13:56.287+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:56.289433+0000) 2023-08-28T00:13:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:56 smithi130 bash[27579]: cluster 2023-08-28T00:13:56.254716+0000 mgr.y (mgr.14901) 4061 : cluster [DBG] pgmap v4068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:56 smithi006 bash[27292]: cluster 2023-08-28T00:13:56.254716+0000 mgr.y (mgr.14901) 4061 : cluster [DBG] pgmap v4068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:56 smithi006 bash[33918]: cluster 2023-08-28T00:13:56.254716+0000 mgr.y (mgr.14901) 4061 : cluster [DBG] pgmap v4068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:57.570 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:57 smithi130 bash[51302]: debug 2023-08-28T00:13:57.287+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:57.289787+0000) 2023-08-28T00:13:57.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:57 smithi130 bash[27579]: audit 2023-08-28T00:13:57.376633+0000 mon.a (mon.0) 3056 : audit [DBG] from='client.? 172.21.15.6:0/1450995593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:57.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:57 smithi006 bash[33918]: audit 2023-08-28T00:13:57.376633+0000 mon.a (mon.0) 3056 : audit [DBG] from='client.? 172.21.15.6:0/1450995593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:57.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:57 smithi006 bash[27292]: audit 2023-08-28T00:13:57.376633+0000 mon.a (mon.0) 3056 : audit [DBG] from='client.? 172.21.15.6:0/1450995593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:58.577 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:58 smithi130 bash[51302]: debug 2023-08-28T00:13:58.283+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:58.290099+0000) 2023-08-28T00:13:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:58 smithi130 bash[27579]: cluster 2023-08-28T00:13:58.255369+0000 mgr.y (mgr.14901) 4062 : cluster [DBG] pgmap v4069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:58 smithi006 bash[33918]: cluster 2023-08-28T00:13:58.255369+0000 mgr.y (mgr.14901) 4062 : cluster [DBG] pgmap v4069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:58 smithi006 bash[27292]: cluster 2023-08-28T00:13:58.255369+0000 mgr.y (mgr.14901) 4062 : cluster [DBG] pgmap v4069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:13:59.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:13:59 smithi130 bash[51302]: debug 2023-08-28T00:13:59.283+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:59.290460+0000) 2023-08-28T00:14:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:13:59 smithi130 bash[27579]: audit 2023-08-28T00:13:59.789083+0000 mon.a (mon.0) 3057 : audit [DBG] from='client.? 172.21.15.6:0/2323577233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:00.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:13:59 smithi006 bash[33918]: audit 2023-08-28T00:13:59.789083+0000 mon.a (mon.0) 3057 : audit [DBG] from='client.? 172.21.15.6:0/2323577233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:00.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:13:59 smithi006 bash[27292]: audit 2023-08-28T00:13:59.789083+0000 mon.a (mon.0) 3057 : audit [DBG] from='client.? 172.21.15.6:0/2323577233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:00.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:00 smithi130 bash[51302]: debug 2023-08-28T00:14:00.287+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:00.290950+0000) 2023-08-28T00:14:00.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:00 smithi006 bash[27292]: cluster 2023-08-28T00:14:00.256489+0000 mgr.y (mgr.14901) 4063 : cluster [DBG] pgmap v4070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:14:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:14:00] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:14:00.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:00 smithi006 bash[33918]: cluster 2023-08-28T00:14:00.256489+0000 mgr.y (mgr.14901) 4063 : cluster [DBG] pgmap v4070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:14:00.678Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:14:00.680Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:00 smithi130 bash[27579]: cluster 2023-08-28T00:14:00.256489+0000 mgr.y (mgr.14901) 4063 : cluster [DBG] pgmap v4070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:01.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:01 smithi130 bash[51302]: debug 2023-08-28T00:14:01.287+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:01.291231+0000) 2023-08-28T00:14:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:02 smithi130 bash[27579]: audit 2023-08-28T00:14:02.204163+0000 mon.a (mon.0) 3058 : audit [DBG] from='client.? 172.21.15.6:0/265960666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:02.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:02 smithi130 bash[51302]: debug 2023-08-28T00:14:02.287+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:02.291456+0000) 2023-08-28T00:14:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:02 smithi006 bash[27292]: audit 2023-08-28T00:14:02.204163+0000 mon.a (mon.0) 3058 : audit [DBG] from='client.? 172.21.15.6:0/265960666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:02 smithi006 bash[33918]: audit 2023-08-28T00:14:02.204163+0000 mon.a (mon.0) 3058 : audit [DBG] from='client.? 172.21.15.6:0/265960666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:14:02.254Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:14:02.257Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:03.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:03 smithi130 bash[27579]: cluster 2023-08-28T00:14:02.257302+0000 mgr.y (mgr.14901) 4064 : cluster [DBG] pgmap v4071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:03.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:03 smithi130 bash[51302]: debug 2023-08-28T00:14:03.287+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:03.291723+0000) 2023-08-28T00:14:03.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:03 smithi006 bash[33918]: cluster 2023-08-28T00:14:02.257302+0000 mgr.y (mgr.14901) 4064 : cluster [DBG] pgmap v4071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:03.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:03 smithi006 bash[27292]: cluster 2023-08-28T00:14:02.257302+0000 mgr.y (mgr.14901) 4064 : cluster [DBG] pgmap v4071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:14:04.138Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:14:04.140Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:04.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:04 smithi130 bash[51302]: debug 2023-08-28T00:14:04.291+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:04.292004+0000) 2023-08-28T00:14:05.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:05 smithi130 bash[27579]: cluster 2023-08-28T00:14:04.258462+0000 mgr.y (mgr.14901) 4065 : cluster [DBG] pgmap v4072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:05.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:05 smithi130 bash[27579]: audit 2023-08-28T00:14:04.630831+0000 mon.c (mon.2) 1543 : audit [DBG] from='client.? 172.21.15.6:0/3201127273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:05.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:05 smithi130 bash[51302]: debug 2023-08-28T00:14:05.287+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:05.292278+0000) 2023-08-28T00:14:05.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:05 smithi006 bash[27292]: cluster 2023-08-28T00:14:04.258462+0000 mgr.y (mgr.14901) 4065 : cluster [DBG] pgmap v4072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:05.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:05 smithi006 bash[27292]: audit 2023-08-28T00:14:04.630831+0000 mon.c (mon.2) 1543 : audit [DBG] from='client.? 172.21.15.6:0/3201127273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:05.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:05 smithi006 bash[33918]: cluster 2023-08-28T00:14:04.258462+0000 mgr.y (mgr.14901) 4065 : cluster [DBG] pgmap v4072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:05.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:05 smithi006 bash[33918]: audit 2023-08-28T00:14:04.630831+0000 mon.c (mon.2) 1543 : audit [DBG] from='client.? 172.21.15.6:0/3201127273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:06.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:06 smithi130 bash[51302]: debug 2023-08-28T00:14:06.287+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:06.292527+0000) 2023-08-28T00:14:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:06 smithi130 bash[27579]: audit 2023-08-28T00:14:05.457916+0000 mon.b (mon.1) 513 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:14:06.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:06 smithi130 bash[27579]: audit 2023-08-28T00:14:05.739096+0000 mon.a (mon.0) 3059 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:14:06.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:06 smithi130 bash[27579]: audit 2023-08-28T00:14:05.745072+0000 mon.a (mon.0) 3060 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:14:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:06 smithi006 bash[27292]: audit 2023-08-28T00:14:05.457916+0000 mon.b (mon.1) 513 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:14:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:06 smithi006 bash[27292]: audit 2023-08-28T00:14:05.739096+0000 mon.a (mon.0) 3059 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:14:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:06 smithi006 bash[27292]: audit 2023-08-28T00:14:05.745072+0000 mon.a (mon.0) 3060 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:14:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:06 smithi006 bash[33918]: audit 2023-08-28T00:14:05.457916+0000 mon.b (mon.1) 513 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:14:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:06 smithi006 bash[33918]: audit 2023-08-28T00:14:05.739096+0000 mon.a (mon.0) 3059 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:14:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:06 smithi006 bash[33918]: audit 2023-08-28T00:14:05.745072+0000 mon.a (mon.0) 3060 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:14:07.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:07 smithi130 bash[51302]: debug 2023-08-28T00:14:07.287+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:07.292776+0000) 2023-08-28T00:14:07.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:07 smithi130 bash[27579]: cluster 2023-08-28T00:14:06.259072+0000 mgr.y (mgr.14901) 4066 : cluster [DBG] pgmap v4073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:07.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:07 smithi130 bash[27579]: audit 2023-08-28T00:14:07.055087+0000 mon.a (mon.0) 3061 : audit [DBG] from='client.? 172.21.15.6:0/1466119076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:07 smithi006 bash[33918]: cluster 2023-08-28T00:14:06.259072+0000 mgr.y (mgr.14901) 4066 : cluster [DBG] pgmap v4073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:07 smithi006 bash[33918]: audit 2023-08-28T00:14:07.055087+0000 mon.a (mon.0) 3061 : audit [DBG] from='client.? 172.21.15.6:0/1466119076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:07.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:07 smithi006 bash[27292]: cluster 2023-08-28T00:14:06.259072+0000 mgr.y (mgr.14901) 4066 : cluster [DBG] pgmap v4073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:07.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:07 smithi006 bash[27292]: audit 2023-08-28T00:14:07.055087+0000 mon.a (mon.0) 3061 : audit [DBG] from='client.? 172.21.15.6:0/1466119076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:08.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:08 smithi130 bash[51302]: debug 2023-08-28T00:14:08.287+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:08.293035+0000) 2023-08-28T00:14:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:09 smithi130 bash[27579]: cluster 2023-08-28T00:14:08.259916+0000 mgr.y (mgr.14901) 4067 : cluster [DBG] pgmap v4074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:09.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:09 smithi130 bash[51302]: debug 2023-08-28T00:14:09.287+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:09.293314+0000) 2023-08-28T00:14:09.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:09 smithi006 bash[33918]: cluster 2023-08-28T00:14:08.259916+0000 mgr.y (mgr.14901) 4067 : cluster [DBG] pgmap v4074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:09 smithi006 bash[27292]: cluster 2023-08-28T00:14:08.259916+0000 mgr.y (mgr.14901) 4067 : cluster [DBG] pgmap v4074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:10 smithi130 bash[27579]: audit 2023-08-28T00:14:09.475489+0000 mon.a (mon.0) 3062 : audit [DBG] from='client.? 172.21.15.6:0/1668830626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:10.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:10 smithi130 bash[51302]: debug 2023-08-28T00:14:10.287+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:10.293584+0000) 2023-08-28T00:14:10.682 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:10 smithi006 bash[33918]: audit 2023-08-28T00:14:09.475489+0000 mon.a (mon.0) 3062 : audit [DBG] from='client.? 172.21.15.6:0/1668830626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:10.682 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:10 smithi006 bash[27292]: audit 2023-08-28T00:14:09.475489+0000 mon.a (mon.0) 3062 : audit [DBG] from='client.? 172.21.15.6:0/1668830626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:10.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:14:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:14:10] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:14:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:14:10.679Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:14:10.681Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:11.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:11 smithi130 bash[27579]: cluster 2023-08-28T00:14:10.261140+0000 mgr.y (mgr.14901) 4068 : cluster [DBG] pgmap v4075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:11.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:11 smithi130 bash[51302]: debug 2023-08-28T00:14:11.287+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:11.293789+0000) 2023-08-28T00:14:11.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:11 smithi006 bash[27292]: cluster 2023-08-28T00:14:10.261140+0000 mgr.y (mgr.14901) 4068 : cluster [DBG] pgmap v4075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:11.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:11 smithi006 bash[33918]: cluster 2023-08-28T00:14:10.261140+0000 mgr.y (mgr.14901) 4068 : cluster [DBG] pgmap v4075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:12.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:12 smithi130 bash[51302]: debug 2023-08-28T00:14:12.287+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:12.293982+0000) 2023-08-28T00:14:12.619 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:14:12.255Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:12.619 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:14:12.258Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:12 smithi006 bash[27292]: audit 2023-08-28T00:14:11.616973+0000 mon.a (mon.0) 3063 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:14:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:12 smithi006 bash[27292]: audit 2023-08-28T00:14:11.915426+0000 mon.a (mon.0) 3064 : audit [DBG] from='client.? 172.21.15.6:0/3445095888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:12 smithi006 bash[27292]: audit 2023-08-28T00:14:12.064624+0000 mon.a (mon.0) 3065 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:14:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:12 smithi006 bash[27292]: cluster 2023-08-28T00:14:12.261952+0000 mgr.y (mgr.14901) 4069 : cluster [DBG] pgmap v4076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:12 smithi006 bash[27292]: audit 2023-08-28T00:14:12.337267+0000 mon.b (mon.1) 514 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:14:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:12 smithi006 bash[27292]: audit 2023-08-28T00:14:12.339252+0000 mon.b (mon.1) 515 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:14:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:12 smithi006 bash[27292]: audit 2023-08-28T00:14:12.348273+0000 mon.a (mon.0) 3066 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:14:12.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:12 smithi006 bash[33918]: audit 2023-08-28T00:14:11.616973+0000 mon.a (mon.0) 3063 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:14:12.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:12 smithi006 bash[33918]: audit 2023-08-28T00:14:11.915426+0000 mon.a (mon.0) 3064 : audit [DBG] from='client.? 172.21.15.6:0/3445095888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:12.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:12 smithi006 bash[33918]: audit 2023-08-28T00:14:12.064624+0000 mon.a (mon.0) 3065 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:14:12.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:12 smithi006 bash[33918]: cluster 2023-08-28T00:14:12.261952+0000 mgr.y (mgr.14901) 4069 : cluster [DBG] pgmap v4076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:12.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:12 smithi006 bash[33918]: audit 2023-08-28T00:14:12.337267+0000 mon.b (mon.1) 514 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:14:12.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:12 smithi006 bash[33918]: audit 2023-08-28T00:14:12.339252+0000 mon.b (mon.1) 515 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:14:12.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:12 smithi006 bash[33918]: audit 2023-08-28T00:14:12.348273+0000 mon.a (mon.0) 3066 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:14:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:12 smithi130 bash[27579]: audit 2023-08-28T00:14:11.616973+0000 mon.a (mon.0) 3063 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:14:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:12 smithi130 bash[27579]: audit 2023-08-28T00:14:11.915426+0000 mon.a (mon.0) 3064 : audit [DBG] from='client.? 172.21.15.6:0/3445095888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:12 smithi130 bash[27579]: audit 2023-08-28T00:14:12.064624+0000 mon.a (mon.0) 3065 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:14:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:12 smithi130 bash[27579]: cluster 2023-08-28T00:14:12.261952+0000 mgr.y (mgr.14901) 4069 : cluster [DBG] pgmap v4076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:12 smithi130 bash[27579]: audit 2023-08-28T00:14:12.337267+0000 mon.b (mon.1) 514 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:14:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:12 smithi130 bash[27579]: audit 2023-08-28T00:14:12.339252+0000 mon.b (mon.1) 515 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:14:13.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:12 smithi130 bash[27579]: audit 2023-08-28T00:14:12.348273+0000 mon.a (mon.0) 3066 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:14:13.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:13 smithi130 bash[51302]: debug 2023-08-28T00:14:13.291+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:13.294250+0000) 2023-08-28T00:14:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:14:14.140Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:14:14.142Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:14.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:14 smithi130 bash[51302]: debug 2023-08-28T00:14:14.287+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:14.294536+0000) 2023-08-28T00:14:15.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:15 smithi130 bash[51302]: debug 2023-08-28T00:14:15.291+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:15.294835+0000) 2023-08-28T00:14:15.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:15 smithi130 bash[27579]: cluster 2023-08-28T00:14:14.262959+0000 mgr.y (mgr.14901) 4070 : cluster [DBG] pgmap v4077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:15.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:15 smithi130 bash[27579]: audit 2023-08-28T00:14:14.327309+0000 mon.c (mon.2) 1544 : audit [DBG] from='client.? 172.21.15.6:0/1121121155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:15.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:15 smithi006 bash[33918]: cluster 2023-08-28T00:14:14.262959+0000 mgr.y (mgr.14901) 4070 : cluster [DBG] pgmap v4077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:15.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:15 smithi006 bash[33918]: audit 2023-08-28T00:14:14.327309+0000 mon.c (mon.2) 1544 : audit [DBG] from='client.? 172.21.15.6:0/1121121155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:15.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:15 smithi006 bash[27292]: cluster 2023-08-28T00:14:14.262959+0000 mgr.y (mgr.14901) 4070 : cluster [DBG] pgmap v4077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:15.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:15 smithi006 bash[27292]: audit 2023-08-28T00:14:14.327309+0000 mon.c (mon.2) 1544 : audit [DBG] from='client.? 172.21.15.6:0/1121121155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:16.572 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:16 smithi130 bash[51302]: debug 2023-08-28T00:14:16.291+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:16.295131+0000) 2023-08-28T00:14:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:16 smithi130 bash[27579]: cluster 2023-08-28T00:14:16.263647+0000 mgr.y (mgr.14901) 4071 : cluster [DBG] pgmap v4078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:16 smithi006 bash[27292]: cluster 2023-08-28T00:14:16.263647+0000 mgr.y (mgr.14901) 4071 : cluster [DBG] pgmap v4078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:16 smithi006 bash[33918]: cluster 2023-08-28T00:14:16.263647+0000 mgr.y (mgr.14901) 4071 : cluster [DBG] pgmap v4078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:17.575 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:17 smithi130 bash[51302]: debug 2023-08-28T00:14:17.291+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:17.295407+0000) 2023-08-28T00:14:17.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:17 smithi130 bash[27579]: audit 2023-08-28T00:14:16.748252+0000 mon.a (mon.0) 3067 : audit [DBG] from='client.? 172.21.15.6:0/467994639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:17.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:17 smithi006 bash[33918]: audit 2023-08-28T00:14:16.748252+0000 mon.a (mon.0) 3067 : audit [DBG] from='client.? 172.21.15.6:0/467994639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:17.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:17 smithi006 bash[27292]: audit 2023-08-28T00:14:16.748252+0000 mon.a (mon.0) 3067 : audit [DBG] from='client.? 172.21.15.6:0/467994639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:18.585 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:18 smithi130 bash[51302]: debug 2023-08-28T00:14:18.295+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:18.295738+0000) 2023-08-28T00:14:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:18 smithi130 bash[27579]: cluster 2023-08-28T00:14:18.264350+0000 mgr.y (mgr.14901) 4072 : cluster [DBG] pgmap v4079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:18 smithi006 bash[27292]: cluster 2023-08-28T00:14:18.264350+0000 mgr.y (mgr.14901) 4072 : cluster [DBG] pgmap v4079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:18 smithi006 bash[33918]: cluster 2023-08-28T00:14:18.264350+0000 mgr.y (mgr.14901) 4072 : cluster [DBG] pgmap v4079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:19.588 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:19 smithi130 bash[51302]: debug 2023-08-28T00:14:19.295+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:19.296087+0000) 2023-08-28T00:14:19.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:19 smithi130 bash[27579]: audit 2023-08-28T00:14:19.164795+0000 mon.a (mon.0) 3068 : audit [DBG] from='client.? 172.21.15.6:0/55212724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:19 smithi006 bash[33918]: audit 2023-08-28T00:14:19.164795+0000 mon.a (mon.0) 3068 : audit [DBG] from='client.? 172.21.15.6:0/55212724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:19.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:19 smithi006 bash[27292]: audit 2023-08-28T00:14:19.164795+0000 mon.a (mon.0) 3068 : audit [DBG] from='client.? 172.21.15.6:0/55212724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:20.595 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:20 smithi130 bash[51302]: debug 2023-08-28T00:14:20.291+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:20.296446+0000) 2023-08-28T00:14:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:20 smithi130 bash[27579]: cluster 2023-08-28T00:14:20.265625+0000 mgr.y (mgr.14901) 4073 : cluster [DBG] pgmap v4080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:14:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:14:20] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:14:20.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:20 smithi006 bash[33918]: cluster 2023-08-28T00:14:20.265625+0000 mgr.y (mgr.14901) 4073 : cluster [DBG] pgmap v4080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:14:20.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:14:20.682Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:20.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:20 smithi006 bash[27292]: cluster 2023-08-28T00:14:20.265625+0000 mgr.y (mgr.14901) 4073 : cluster [DBG] pgmap v4080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:21.599 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:21 smithi130 bash[51302]: debug 2023-08-28T00:14:21.295+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:21.296808+0000) 2023-08-28T00:14:21.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:21 smithi130 bash[27579]: audit 2023-08-28T00:14:21.568414+0000 mon.a (mon.0) 3069 : audit [DBG] from='client.? 172.21.15.6:0/3853239157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:21.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:21 smithi006 bash[27292]: audit 2023-08-28T00:14:21.568414+0000 mon.a (mon.0) 3069 : audit [DBG] from='client.? 172.21.15.6:0/3853239157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:21.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:21 smithi006 bash[33918]: audit 2023-08-28T00:14:21.568414+0000 mon.a (mon.0) 3069 : audit [DBG] from='client.? 172.21.15.6:0/3853239157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:22.605 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:14:22.257Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:22.605 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:14:22.259Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:22.606 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:22 smithi130 bash[51302]: debug 2023-08-28T00:14:22.291+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:22.297188+0000) 2023-08-28T00:14:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:22 smithi130 bash[27579]: cluster 2023-08-28T00:14:22.266231+0000 mgr.y (mgr.14901) 4074 : cluster [DBG] pgmap v4081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:22 smithi006 bash[33918]: cluster 2023-08-28T00:14:22.266231+0000 mgr.y (mgr.14901) 4074 : cluster [DBG] pgmap v4081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:22 smithi006 bash[27292]: cluster 2023-08-28T00:14:22.266231+0000 mgr.y (mgr.14901) 4074 : cluster [DBG] pgmap v4081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:23.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:23 smithi130 bash[51302]: debug 2023-08-28T00:14:23.291+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:23.297488+0000) 2023-08-28T00:14:24.299 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:24 smithi130 bash[27579]: audit 2023-08-28T00:14:23.988739+0000 mon.a (mon.0) 3070 : audit [DBG] from='client.? 172.21.15.6:0/3620532055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:24.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:24 smithi006 bash[33918]: audit 2023-08-28T00:14:23.988739+0000 mon.a (mon.0) 3070 : audit [DBG] from='client.? 172.21.15.6:0/3620532055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:24.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:24 smithi006 bash[27292]: audit 2023-08-28T00:14:23.988739+0000 mon.a (mon.0) 3070 : audit [DBG] from='client.? 172.21.15.6:0/3620532055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:14:24.141Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:14:24.143Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:24.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:24 smithi130 bash[51302]: debug 2023-08-28T00:14:24.291+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:24.297828+0000) 2023-08-28T00:14:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:25 smithi130 bash[27579]: cluster 2023-08-28T00:14:24.267474+0000 mgr.y (mgr.14901) 4075 : cluster [DBG] pgmap v4082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:25.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:25 smithi130 bash[51302]: debug 2023-08-28T00:14:25.295+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:25.298166+0000) 2023-08-28T00:14:25.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:25 smithi006 bash[33918]: cluster 2023-08-28T00:14:24.267474+0000 mgr.y (mgr.14901) 4075 : cluster [DBG] pgmap v4082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:25.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:25 smithi006 bash[27292]: cluster 2023-08-28T00:14:24.267474+0000 mgr.y (mgr.14901) 4075 : cluster [DBG] pgmap v4082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:26.570 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:26 smithi130 bash[51302]: debug 2023-08-28T00:14:26.295+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:26.298468+0000) 2023-08-28T00:14:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:26 smithi130 bash[27579]: cluster 2023-08-28T00:14:26.268168+0000 mgr.y (mgr.14901) 4076 : cluster [DBG] pgmap v4083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:26 smithi130 bash[27579]: audit 2023-08-28T00:14:26.399900+0000 mon.a (mon.0) 3071 : audit [DBG] from='client.? 172.21.15.6:0/419460084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:26 smithi006 bash[33918]: cluster 2023-08-28T00:14:26.268168+0000 mgr.y (mgr.14901) 4076 : cluster [DBG] pgmap v4083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:26 smithi006 bash[33918]: audit 2023-08-28T00:14:26.399900+0000 mon.a (mon.0) 3071 : audit [DBG] from='client.? 172.21.15.6:0/419460084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:26 smithi006 bash[27292]: cluster 2023-08-28T00:14:26.268168+0000 mgr.y (mgr.14901) 4076 : cluster [DBG] pgmap v4083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:26 smithi006 bash[27292]: audit 2023-08-28T00:14:26.399900+0000 mon.a (mon.0) 3071 : audit [DBG] from='client.? 172.21.15.6:0/419460084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:27.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:27 smithi130 bash[51302]: debug 2023-08-28T00:14:27.291+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:27.298763+0000) 2023-08-28T00:14:28.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:28 smithi130 bash[51302]: debug 2023-08-28T00:14:28.295+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:28.299021+0000) 2023-08-28T00:14:29.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:29 smithi130 bash[51302]: debug 2023-08-28T00:14:29.295+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:29.299323+0000) 2023-08-28T00:14:29.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:29 smithi130 bash[27579]: cluster 2023-08-28T00:14:28.268830+0000 mgr.y (mgr.14901) 4077 : cluster [DBG] pgmap v4084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:29.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:29 smithi130 bash[27579]: audit 2023-08-28T00:14:28.806716+0000 mon.a (mon.0) 3072 : audit [DBG] from='client.? 172.21.15.6:0/2622587254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:29.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:29 smithi006 bash[27292]: cluster 2023-08-28T00:14:28.268830+0000 mgr.y (mgr.14901) 4077 : cluster [DBG] pgmap v4084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:29.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:29 smithi006 bash[27292]: audit 2023-08-28T00:14:28.806716+0000 mon.a (mon.0) 3072 : audit [DBG] from='client.? 172.21.15.6:0/2622587254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:29.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:29 smithi006 bash[33918]: cluster 2023-08-28T00:14:28.268830+0000 mgr.y (mgr.14901) 4077 : cluster [DBG] pgmap v4084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:29.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:29 smithi006 bash[33918]: audit 2023-08-28T00:14:28.806716+0000 mon.a (mon.0) 3072 : audit [DBG] from='client.? 172.21.15.6:0/2622587254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:30.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:30 smithi130 bash[51302]: debug 2023-08-28T00:14:30.295+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:30.299580+0000) 2023-08-28T00:14:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:30 smithi130 bash[27579]: cluster 2023-08-28T00:14:30.269911+0000 mgr.y (mgr.14901) 4078 : cluster [DBG] pgmap v4085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:30.683 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:30 smithi006 bash[27292]: cluster 2023-08-28T00:14:30.269911+0000 mgr.y (mgr.14901) 4078 : cluster [DBG] pgmap v4085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:30.683 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:30 smithi006 bash[33918]: cluster 2023-08-28T00:14:30.269911+0000 mgr.y (mgr.14901) 4078 : cluster [DBG] pgmap v4085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:14:30.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:14:30.683Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:14:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:14:30] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:14:31.567 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:31 smithi130 bash[51302]: debug 2023-08-28T00:14:31.295+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:31.299956+0000) 2023-08-28T00:14:31.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:31 smithi130 bash[27579]: audit 2023-08-28T00:14:31.233083+0000 mon.c (mon.2) 1545 : audit [DBG] from='client.? 172.21.15.6:0/1747358216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:31.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:31 smithi006 bash[27292]: audit 2023-08-28T00:14:31.233083+0000 mon.c (mon.2) 1545 : audit [DBG] from='client.? 172.21.15.6:0/1747358216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:31.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:31 smithi006 bash[33918]: audit 2023-08-28T00:14:31.233083+0000 mon.c (mon.2) 1545 : audit [DBG] from='client.? 172.21.15.6:0/1747358216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:32.573 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:14:32.258Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:32.573 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:14:32.260Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:32.574 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:32 smithi130 bash[51302]: debug 2023-08-28T00:14:32.295+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:32.300280+0000) 2023-08-28T00:14:32.574 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:32 smithi006 bash[27292]: cluster 2023-08-28T00:14:32.270567+0000 mgr.y (mgr.14901) 4079 : cluster [DBG] pgmap v4086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:32 smithi130 bash[27579]: cluster 2023-08-28T00:14:32.270567+0000 mgr.y (mgr.14901) 4079 : cluster [DBG] pgmap v4086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:32 smithi006 bash[33918]: cluster 2023-08-28T00:14:32.270567+0000 mgr.y (mgr.14901) 4079 : cluster [DBG] pgmap v4086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:33.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:33 smithi130 bash[51302]: debug 2023-08-28T00:14:33.299+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:33.300542+0000) 2023-08-28T00:14:33.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:33 smithi006 bash[27292]: audit 2023-08-28T00:14:33.650736+0000 mon.a (mon.0) 3073 : audit [DBG] from='client.? 172.21.15.6:0/2708175974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:33.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:33 smithi006 bash[33918]: audit 2023-08-28T00:14:33.650736+0000 mon.a (mon.0) 3073 : audit [DBG] from='client.? 172.21.15.6:0/2708175974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:33 smithi130 bash[27579]: audit 2023-08-28T00:14:33.650736+0000 mon.a (mon.0) 3073 : audit [DBG] from='client.? 172.21.15.6:0/2708175974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:14:34.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:14:34.144Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:34.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:34 smithi130 bash[51302]: debug 2023-08-28T00:14:34.295+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:34.300804+0000) 2023-08-28T00:14:34.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:34 smithi006 bash[33918]: cluster 2023-08-28T00:14:34.271672+0000 mgr.y (mgr.14901) 4080 : cluster [DBG] pgmap v4087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:34.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:34 smithi006 bash[27292]: cluster 2023-08-28T00:14:34.271672+0000 mgr.y (mgr.14901) 4080 : cluster [DBG] pgmap v4087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:35.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:34 smithi130 bash[27579]: cluster 2023-08-28T00:14:34.271672+0000 mgr.y (mgr.14901) 4080 : cluster [DBG] pgmap v4087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:35.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:35 smithi130 bash[51302]: debug 2023-08-28T00:14:35.299+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:35.301105+0000) 2023-08-28T00:14:36.575 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:36 smithi130 bash[51302]: debug 2023-08-28T00:14:36.295+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:36.301412+0000) 2023-08-28T00:14:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:36 smithi130 bash[27579]: audit 2023-08-28T00:14:36.061791+0000 mon.a (mon.0) 3074 : audit [DBG] from='client.? 172.21.15.6:0/4130210025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:36 smithi130 bash[27579]: cluster 2023-08-28T00:14:36.272396+0000 mgr.y (mgr.14901) 4081 : cluster [DBG] pgmap v4088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:36 smithi006 bash[33918]: audit 2023-08-28T00:14:36.061791+0000 mon.a (mon.0) 3074 : audit [DBG] from='client.? 172.21.15.6:0/4130210025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:36 smithi006 bash[33918]: cluster 2023-08-28T00:14:36.272396+0000 mgr.y (mgr.14901) 4081 : cluster [DBG] pgmap v4088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:36 smithi006 bash[27292]: audit 2023-08-28T00:14:36.061791+0000 mon.a (mon.0) 3074 : audit [DBG] from='client.? 172.21.15.6:0/4130210025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:36 smithi006 bash[27292]: cluster 2023-08-28T00:14:36.272396+0000 mgr.y (mgr.14901) 4081 : cluster [DBG] pgmap v4088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:37.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:37 smithi130 bash[51302]: debug 2023-08-28T00:14:37.299+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:37.301679+0000) 2023-08-28T00:14:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:38 smithi130 bash[27579]: cluster 2023-08-28T00:14:38.273014+0000 mgr.y (mgr.14901) 4082 : cluster [DBG] pgmap v4089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:38.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:38 smithi130 bash[51302]: debug 2023-08-28T00:14:38.295+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:38.301955+0000) 2023-08-28T00:14:38.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:38 smithi006 bash[27292]: cluster 2023-08-28T00:14:38.273014+0000 mgr.y (mgr.14901) 4082 : cluster [DBG] pgmap v4089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:38.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:38 smithi006 bash[33918]: cluster 2023-08-28T00:14:38.273014+0000 mgr.y (mgr.14901) 4082 : cluster [DBG] pgmap v4089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:39.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:39 smithi130 bash[27579]: audit 2023-08-28T00:14:38.490118+0000 mon.a (mon.0) 3075 : audit [DBG] from='client.? 172.21.15.6:0/2068100666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:39.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:39 smithi130 bash[51302]: debug 2023-08-28T00:14:39.299+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:39.302246+0000) 2023-08-28T00:14:39.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:39 smithi006 bash[33918]: audit 2023-08-28T00:14:38.490118+0000 mon.a (mon.0) 3075 : audit [DBG] from='client.? 172.21.15.6:0/2068100666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:39.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:39 smithi006 bash[27292]: audit 2023-08-28T00:14:38.490118+0000 mon.a (mon.0) 3075 : audit [DBG] from='client.? 172.21.15.6:0/2068100666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:40 smithi130 bash[27579]: cluster 2023-08-28T00:14:40.274111+0000 mgr.y (mgr.14901) 4083 : cluster [DBG] pgmap v4090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:40.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:40 smithi130 bash[51302]: debug 2023-08-28T00:14:40.299+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:40.302541+0000) 2023-08-28T00:14:40.684 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:40 smithi006 bash[27292]: cluster 2023-08-28T00:14:40.274111+0000 mgr.y (mgr.14901) 4083 : cluster [DBG] pgmap v4090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:40.684 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:40 smithi006 bash[33918]: cluster 2023-08-28T00:14:40.274111+0000 mgr.y (mgr.14901) 4083 : cluster [DBG] pgmap v4090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:14:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:14:40] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:14:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:14:40.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:14:40.683Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:41.569 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:41 smithi130 bash[51302]: debug 2023-08-28T00:14:41.299+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:41.302838+0000) 2023-08-28T00:14:41.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:41 smithi130 bash[27579]: audit 2023-08-28T00:14:40.915553+0000 mon.a (mon.0) 3076 : audit [DBG] from='client.? 172.21.15.6:0/2689173488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:41.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:41 smithi006 bash[27292]: audit 2023-08-28T00:14:40.915553+0000 mon.a (mon.0) 3076 : audit [DBG] from='client.? 172.21.15.6:0/2689173488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:41.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:41 smithi006 bash[33918]: audit 2023-08-28T00:14:40.915553+0000 mon.a (mon.0) 3076 : audit [DBG] from='client.? 172.21.15.6:0/2689173488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:42.579 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:14:42.258Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:42.579 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:14:42.260Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:42.580 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:42 smithi130 bash[51302]: debug 2023-08-28T00:14:42.299+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:42.303112+0000) 2023-08-28T00:14:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:42 smithi130 bash[27579]: cluster 2023-08-28T00:14:42.274781+0000 mgr.y (mgr.14901) 4084 : cluster [DBG] pgmap v4091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:42.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:42 smithi006 bash[33918]: cluster 2023-08-28T00:14:42.274781+0000 mgr.y (mgr.14901) 4084 : cluster [DBG] pgmap v4091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:42 smithi006 bash[27292]: cluster 2023-08-28T00:14:42.274781+0000 mgr.y (mgr.14901) 4084 : cluster [DBG] pgmap v4091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:43.583 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:43 smithi130 bash[51302]: debug 2023-08-28T00:14:43.303+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:43.303389+0000) 2023-08-28T00:14:43.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:43 smithi130 bash[27579]: audit 2023-08-28T00:14:43.340659+0000 mon.a (mon.0) 3077 : audit [DBG] from='client.? 172.21.15.6:0/3018499692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:43.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:43 smithi006 bash[27292]: audit 2023-08-28T00:14:43.340659+0000 mon.a (mon.0) 3077 : audit [DBG] from='client.? 172.21.15.6:0/3018499692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:43.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:43 smithi006 bash[33918]: audit 2023-08-28T00:14:43.340659+0000 mon.a (mon.0) 3077 : audit [DBG] from='client.? 172.21.15.6:0/3018499692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:14:44.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:14:44.144Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:44.590 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:44 smithi130 bash[51302]: debug 2023-08-28T00:14:44.302+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:44.303670+0000) 2023-08-28T00:14:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:44 smithi130 bash[27579]: cluster 2023-08-28T00:14:44.275879+0000 mgr.y (mgr.14901) 4085 : cluster [DBG] pgmap v4092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:44.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:44 smithi006 bash[27292]: cluster 2023-08-28T00:14:44.275879+0000 mgr.y (mgr.14901) 4085 : cluster [DBG] pgmap v4092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:44 smithi006 bash[33918]: cluster 2023-08-28T00:14:44.275879+0000 mgr.y (mgr.14901) 4085 : cluster [DBG] pgmap v4092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:45.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:45 smithi130 bash[51302]: debug 2023-08-28T00:14:45.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:45.303955+0000) 2023-08-28T00:14:46.577 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:46 smithi130 bash[51302]: debug 2023-08-28T00:14:46.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:46.304165+0000) 2023-08-28T00:14:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:46 smithi130 bash[27579]: audit 2023-08-28T00:14:45.766335+0000 mon.a (mon.0) 3078 : audit [DBG] from='client.? 172.21.15.6:0/127936810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:46 smithi130 bash[27579]: cluster 2023-08-28T00:14:46.276474+0000 mgr.y (mgr.14901) 4086 : cluster [DBG] pgmap v4093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:46 smithi006 bash[27292]: audit 2023-08-28T00:14:45.766335+0000 mon.a (mon.0) 3078 : audit [DBG] from='client.? 172.21.15.6:0/127936810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:46 smithi006 bash[27292]: cluster 2023-08-28T00:14:46.276474+0000 mgr.y (mgr.14901) 4086 : cluster [DBG] pgmap v4093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:46 smithi006 bash[33918]: audit 2023-08-28T00:14:45.766335+0000 mon.a (mon.0) 3078 : audit [DBG] from='client.? 172.21.15.6:0/127936810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:46 smithi006 bash[33918]: cluster 2023-08-28T00:14:46.276474+0000 mgr.y (mgr.14901) 4086 : cluster [DBG] pgmap v4093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:47.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:47 smithi130 bash[51302]: debug 2023-08-28T00:14:47.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:47.304457+0000) 2023-08-28T00:14:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:48 smithi130 bash[27579]: audit 2023-08-28T00:14:48.169892+0000 mon.c (mon.2) 1546 : audit [DBG] from='client.? 172.21.15.6:0/3715901341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:48.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:48 smithi130 bash[51302]: debug 2023-08-28T00:14:48.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:48.304772+0000) 2023-08-28T00:14:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:48 smithi006 bash[27292]: audit 2023-08-28T00:14:48.169892+0000 mon.c (mon.2) 1546 : audit [DBG] from='client.? 172.21.15.6:0/3715901341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:48 smithi006 bash[33918]: audit 2023-08-28T00:14:48.169892+0000 mon.c (mon.2) 1546 : audit [DBG] from='client.? 172.21.15.6:0/3715901341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:49.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:49 smithi130 bash[27579]: cluster 2023-08-28T00:14:48.277060+0000 mgr.y (mgr.14901) 4087 : cluster [DBG] pgmap v4094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:49.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:49 smithi130 bash[51302]: debug 2023-08-28T00:14:49.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:49.305063+0000) 2023-08-28T00:14:49.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:49 smithi006 bash[33918]: cluster 2023-08-28T00:14:48.277060+0000 mgr.y (mgr.14901) 4087 : cluster [DBG] pgmap v4094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:49.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:49 smithi006 bash[27292]: cluster 2023-08-28T00:14:48.277060+0000 mgr.y (mgr.14901) 4087 : cluster [DBG] pgmap v4094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:50 smithi130 bash[27579]: cluster 2023-08-28T00:14:50.278178+0000 mgr.y (mgr.14901) 4088 : cluster [DBG] pgmap v4095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:50.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:50 smithi130 bash[51302]: debug 2023-08-28T00:14:50.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:50.305320+0000) 2023-08-28T00:14:50.685 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:50 smithi006 bash[27292]: cluster 2023-08-28T00:14:50.278178+0000 mgr.y (mgr.14901) 4088 : cluster [DBG] pgmap v4095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:50.685 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:50 smithi006 bash[33918]: cluster 2023-08-28T00:14:50.278178+0000 mgr.y (mgr.14901) 4088 : cluster [DBG] pgmap v4095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:14:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:14:50] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:14:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:14:50.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:14:50.684Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:51.571 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:51 smithi130 bash[51302]: debug 2023-08-28T00:14:51.302+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:51.305616+0000) 2023-08-28T00:14:51.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:51 smithi130 bash[27579]: audit 2023-08-28T00:14:50.591595+0000 mon.c (mon.2) 1547 : audit [DBG] from='client.? 172.21.15.6:0/530093664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:51.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:51 smithi006 bash[27292]: audit 2023-08-28T00:14:50.591595+0000 mon.c (mon.2) 1547 : audit [DBG] from='client.? 172.21.15.6:0/530093664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:51.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:51 smithi006 bash[33918]: audit 2023-08-28T00:14:50.591595+0000 mon.c (mon.2) 1547 : audit [DBG] from='client.? 172.21.15.6:0/530093664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:52.577 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:14:52.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:52.578 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:14:52.261Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:52.578 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:52 smithi130 bash[51302]: debug 2023-08-28T00:14:52.302+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:52.305909+0000) 2023-08-28T00:14:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:52 smithi130 bash[27579]: cluster 2023-08-28T00:14:52.278842+0000 mgr.y (mgr.14901) 4089 : cluster [DBG] pgmap v4096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:52 smithi006 bash[27292]: cluster 2023-08-28T00:14:52.278842+0000 mgr.y (mgr.14901) 4089 : cluster [DBG] pgmap v4096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:52 smithi006 bash[33918]: cluster 2023-08-28T00:14:52.278842+0000 mgr.y (mgr.14901) 4089 : cluster [DBG] pgmap v4096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:53.582 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:53 smithi130 bash[51302]: debug 2023-08-28T00:14:53.302+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:53.306251+0000) 2023-08-28T00:14:53.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:53 smithi130 bash[27579]: audit 2023-08-28T00:14:53.009969+0000 mon.c (mon.2) 1548 : audit [DBG] from='client.? 172.21.15.6:0/196167929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:53.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:53 smithi006 bash[27292]: audit 2023-08-28T00:14:53.009969+0000 mon.c (mon.2) 1548 : audit [DBG] from='client.? 172.21.15.6:0/196167929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:53.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:53 smithi006 bash[33918]: audit 2023-08-28T00:14:53.009969+0000 mon.c (mon.2) 1548 : audit [DBG] from='client.? 172.21.15.6:0/196167929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:14:54.143Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:14:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:14:54.146Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:14:54.589 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:54 smithi130 bash[51302]: debug 2023-08-28T00:14:54.302+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:54.306525+0000) 2023-08-28T00:14:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:54 smithi130 bash[27579]: cluster 2023-08-28T00:14:54.279990+0000 mgr.y (mgr.14901) 4090 : cluster [DBG] pgmap v4097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:54 smithi006 bash[33918]: cluster 2023-08-28T00:14:54.279990+0000 mgr.y (mgr.14901) 4090 : cluster [DBG] pgmap v4097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:54 smithi006 bash[27292]: cluster 2023-08-28T00:14:54.279990+0000 mgr.y (mgr.14901) 4090 : cluster [DBG] pgmap v4097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:55.592 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:55 smithi130 bash[51302]: debug 2023-08-28T00:14:55.306+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:55.306848+0000) 2023-08-28T00:14:55.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:55 smithi130 bash[27579]: audit 2023-08-28T00:14:55.434802+0000 mon.a (mon.0) 3079 : audit [DBG] from='client.? 172.21.15.6:0/2896688231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:55.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:55 smithi006 bash[33918]: audit 2023-08-28T00:14:55.434802+0000 mon.a (mon.0) 3079 : audit [DBG] from='client.? 172.21.15.6:0/2896688231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:55.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:55 smithi006 bash[27292]: audit 2023-08-28T00:14:55.434802+0000 mon.a (mon.0) 3079 : audit [DBG] from='client.? 172.21.15.6:0/2896688231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:56.603 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:56 smithi130 bash[51302]: debug 2023-08-28T00:14:56.306+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:56.307166+0000) 2023-08-28T00:14:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:56 smithi130 bash[27579]: cluster 2023-08-28T00:14:56.280857+0000 mgr.y (mgr.14901) 4091 : cluster [DBG] pgmap v4098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:56.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:56 smithi006 bash[33918]: cluster 2023-08-28T00:14:56.280857+0000 mgr.y (mgr.14901) 4091 : cluster [DBG] pgmap v4098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:56 smithi006 bash[27292]: cluster 2023-08-28T00:14:56.280857+0000 mgr.y (mgr.14901) 4091 : cluster [DBG] pgmap v4098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:57.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:57 smithi130 bash[51302]: debug 2023-08-28T00:14:57.306+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:57.307380+0000) 2023-08-28T00:14:58.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:57 smithi006 bash[33918]: audit 2023-08-28T00:14:57.853385+0000 mon.a (mon.0) 3080 : audit [DBG] from='client.? 172.21.15.6:0/3383734926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:58.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:57 smithi006 bash[27292]: audit 2023-08-28T00:14:57.853385+0000 mon.a (mon.0) 3080 : audit [DBG] from='client.? 172.21.15.6:0/3383734926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:58.309 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:57 smithi130 bash[27579]: audit 2023-08-28T00:14:57.853385+0000 mon.a (mon.0) 3080 : audit [DBG] from='client.? 172.21.15.6:0/3383734926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:58.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:58 smithi130 bash[51302]: debug 2023-08-28T00:14:58.306+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:58.307653+0000) 2023-08-28T00:14:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:14:58 smithi006 bash[33918]: cluster 2023-08-28T00:14:58.281530+0000 mgr.y (mgr.14901) 4092 : cluster [DBG] pgmap v4099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:59.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:14:58 smithi006 bash[27292]: cluster 2023-08-28T00:14:58.281530+0000 mgr.y (mgr.14901) 4092 : cluster [DBG] pgmap v4099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:59.309 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:14:58 smithi130 bash[27579]: cluster 2023-08-28T00:14:58.281530+0000 mgr.y (mgr.14901) 4092 : cluster [DBG] pgmap v4099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:14:59.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:14:59 smithi130 bash[51302]: debug 2023-08-28T00:14:59.306+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:59.307948+0000) 2023-08-28T00:15:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:00 smithi130 bash[27579]: audit 2023-08-28T00:15:00.270883+0000 mon.a (mon.0) 3081 : audit [DBG] from='client.? 172.21.15.6:0/3679808155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:00.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:00 smithi130 bash[51302]: debug 2023-08-28T00:15:00.306+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:00.308239+0000) 2023-08-28T00:15:00.685 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:00 smithi006 bash[33918]: audit 2023-08-28T00:15:00.270883+0000 mon.a (mon.0) 3081 : audit [DBG] from='client.? 172.21.15.6:0/3679808155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:00.685 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:00 smithi006 bash[27292]: audit 2023-08-28T00:15:00.270883+0000 mon.a (mon.0) 3081 : audit [DBG] from='client.? 172.21.15.6:0/3679808155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:15:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:15:00] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:15:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:15:00.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:15:00.685Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:01.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:01 smithi006 bash[27292]: cluster 2023-08-28T00:15:00.282743+0000 mgr.y (mgr.14901) 4093 : cluster [DBG] pgmap v4100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:01.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:01 smithi006 bash[33918]: cluster 2023-08-28T00:15:00.282743+0000 mgr.y (mgr.14901) 4093 : cluster [DBG] pgmap v4100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:01.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:01 smithi130 bash[27579]: cluster 2023-08-28T00:15:00.282743+0000 mgr.y (mgr.14901) 4093 : cluster [DBG] pgmap v4100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:01.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:01 smithi130 bash[51302]: debug 2023-08-28T00:15:01.306+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:01.308513+0000) 2023-08-28T00:15:02.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:02 smithi130 bash[51302]: debug 2023-08-28T00:15:02.306+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:02.308788+0000) 2023-08-28T00:15:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:15:02.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:15:02.261Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:03.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:03 smithi130 bash[27579]: cluster 2023-08-28T00:15:02.283419+0000 mgr.y (mgr.14901) 4094 : cluster [DBG] pgmap v4101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:03.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:03 smithi130 bash[27579]: audit 2023-08-28T00:15:02.682747+0000 mon.a (mon.0) 3082 : audit [DBG] from='client.? 172.21.15.6:0/1701772380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:03.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:03 smithi130 bash[51302]: debug 2023-08-28T00:15:03.306+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:03.309068+0000) 2023-08-28T00:15:03.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:03 smithi006 bash[33918]: cluster 2023-08-28T00:15:02.283419+0000 mgr.y (mgr.14901) 4094 : cluster [DBG] pgmap v4101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:03.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:03 smithi006 bash[33918]: audit 2023-08-28T00:15:02.682747+0000 mon.a (mon.0) 3082 : audit [DBG] from='client.? 172.21.15.6:0/1701772380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:03.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:03 smithi006 bash[27292]: cluster 2023-08-28T00:15:02.283419+0000 mgr.y (mgr.14901) 4094 : cluster [DBG] pgmap v4101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:03.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:03 smithi006 bash[27292]: audit 2023-08-28T00:15:02.682747+0000 mon.a (mon.0) 3082 : audit [DBG] from='client.? 172.21.15.6:0/1701772380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:04.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:04 smithi006 bash[27292]: cluster 2023-08-28T00:15:04.284575+0000 mgr.y (mgr.14901) 4095 : cluster [DBG] pgmap v4102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:04.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:04 smithi006 bash[33918]: cluster 2023-08-28T00:15:04.284575+0000 mgr.y (mgr.14901) 4095 : cluster [DBG] pgmap v4102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:15:04.145Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:15:04.147Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:04 smithi130 bash[27579]: cluster 2023-08-28T00:15:04.284575+0000 mgr.y (mgr.14901) 4095 : cluster [DBG] pgmap v4102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:04.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:04 smithi130 bash[51302]: debug 2023-08-28T00:15:04.306+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:04.309409+0000) 2023-08-28T00:15:05.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:05 smithi130 bash[27579]: audit 2023-08-28T00:15:05.097125+0000 mon.c (mon.2) 1549 : audit [DBG] from='client.? 172.21.15.6:0/4066580736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:05.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:05 smithi130 bash[51302]: debug 2023-08-28T00:15:05.306+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:05.309732+0000) 2023-08-28T00:15:05.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:05 smithi006 bash[27292]: audit 2023-08-28T00:15:05.097125+0000 mon.c (mon.2) 1549 : audit [DBG] from='client.? 172.21.15.6:0/4066580736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:05.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:05 smithi006 bash[33918]: audit 2023-08-28T00:15:05.097125+0000 mon.c (mon.2) 1549 : audit [DBG] from='client.? 172.21.15.6:0/4066580736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:06.582 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:06 smithi130 bash[51302]: debug 2023-08-28T00:15:06.306+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:06.310023+0000) 2023-08-28T00:15:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:06 smithi130 bash[27579]: cluster 2023-08-28T00:15:06.285277+0000 mgr.y (mgr.14901) 4096 : cluster [DBG] pgmap v4103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:06 smithi006 bash[33918]: cluster 2023-08-28T00:15:06.285277+0000 mgr.y (mgr.14901) 4096 : cluster [DBG] pgmap v4103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:06 smithi006 bash[27292]: cluster 2023-08-28T00:15:06.285277+0000 mgr.y (mgr.14901) 4096 : cluster [DBG] pgmap v4103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:07.585 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:07 smithi130 bash[51302]: debug 2023-08-28T00:15:07.306+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:07.310314+0000) 2023-08-28T00:15:07.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:07 smithi130 bash[27579]: audit 2023-08-28T00:15:07.505997+0000 mon.c (mon.2) 1550 : audit [DBG] from='client.? 172.21.15.6:0/2110621959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:07.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:07 smithi006 bash[27292]: audit 2023-08-28T00:15:07.505997+0000 mon.c (mon.2) 1550 : audit [DBG] from='client.? 172.21.15.6:0/2110621959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:07.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:07 smithi006 bash[33918]: audit 2023-08-28T00:15:07.505997+0000 mon.c (mon.2) 1550 : audit [DBG] from='client.? 172.21.15.6:0/2110621959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:08.593 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:08 smithi130 bash[51302]: debug 2023-08-28T00:15:08.306+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:08.310607+0000) 2023-08-28T00:15:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:08 smithi130 bash[27579]: cluster 2023-08-28T00:15:08.285880+0000 mgr.y (mgr.14901) 4097 : cluster [DBG] pgmap v4104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:08 smithi006 bash[27292]: cluster 2023-08-28T00:15:08.285880+0000 mgr.y (mgr.14901) 4097 : cluster [DBG] pgmap v4104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:08 smithi006 bash[33918]: cluster 2023-08-28T00:15:08.285880+0000 mgr.y (mgr.14901) 4097 : cluster [DBG] pgmap v4104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:09.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:09 smithi130 bash[51302]: debug 2023-08-28T00:15:09.310+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:09.310945+0000) 2023-08-28T00:15:10.312 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:09 smithi130 bash[27579]: audit 2023-08-28T00:15:09.924131+0000 mon.c (mon.2) 1551 : audit [DBG] from='client.? 172.21.15.6:0/1391239299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:10.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:09 smithi006 bash[27292]: audit 2023-08-28T00:15:09.924131+0000 mon.c (mon.2) 1551 : audit [DBG] from='client.? 172.21.15.6:0/1391239299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:10.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:09 smithi006 bash[33918]: audit 2023-08-28T00:15:09.924131+0000 mon.c (mon.2) 1551 : audit [DBG] from='client.? 172.21.15.6:0/1391239299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:10.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:10 smithi130 bash[51302]: debug 2023-08-28T00:15:10.310+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:10.311251+0000) 2023-08-28T00:15:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:15:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:15:10] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:15:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:15:10.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:15:10.685Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:11.313 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:10 smithi130 bash[27579]: cluster 2023-08-28T00:15:10.286986+0000 mgr.y (mgr.14901) 4098 : cluster [DBG] pgmap v4105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:11.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:10 smithi006 bash[27292]: cluster 2023-08-28T00:15:10.286986+0000 mgr.y (mgr.14901) 4098 : cluster [DBG] pgmap v4105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:11.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:10 smithi006 bash[33918]: cluster 2023-08-28T00:15:10.286986+0000 mgr.y (mgr.14901) 4098 : cluster [DBG] pgmap v4105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:11.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:11 smithi130 bash[51302]: debug 2023-08-28T00:15:11.310+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:11.311496+0000) 2023-08-28T00:15:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:12 smithi130 bash[27579]: cluster 2023-08-28T00:15:12.287613+0000 mgr.y (mgr.14901) 4099 : cluster [DBG] pgmap v4106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:12 smithi130 bash[27579]: audit 2023-08-28T00:15:12.336270+0000 mon.a (mon.0) 3083 : audit [DBG] from='client.? 172.21.15.6:0/2206084092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:12 smithi130 bash[27579]: audit 2023-08-28T00:15:12.359941+0000 mon.b (mon.1) 516 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:15:12.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:12 smithi130 bash[51302]: debug 2023-08-28T00:15:12.310+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:12.311803+0000) 2023-08-28T00:15:12.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:12 smithi006 bash[27292]: cluster 2023-08-28T00:15:12.287613+0000 mgr.y (mgr.14901) 4099 : cluster [DBG] pgmap v4106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:12.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:12 smithi006 bash[27292]: audit 2023-08-28T00:15:12.336270+0000 mon.a (mon.0) 3083 : audit [DBG] from='client.? 172.21.15.6:0/2206084092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:12.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:12 smithi006 bash[27292]: audit 2023-08-28T00:15:12.359941+0000 mon.b (mon.1) 516 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:15:12.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:12 smithi006 bash[33918]: cluster 2023-08-28T00:15:12.287613+0000 mgr.y (mgr.14901) 4099 : cluster [DBG] pgmap v4106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:12.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:12 smithi006 bash[33918]: audit 2023-08-28T00:15:12.336270+0000 mon.a (mon.0) 3083 : audit [DBG] from='client.? 172.21.15.6:0/2206084092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:12 smithi006 bash[33918]: audit 2023-08-28T00:15:12.359941+0000 mon.b (mon.1) 516 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:15:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:15:12.261Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:15:12.263Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:13.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:13 smithi130 bash[51302]: debug 2023-08-28T00:15:13.310+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:13.312018+0000) 2023-08-28T00:15:14.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:14 smithi006 bash[33918]: cluster 2023-08-28T00:15:14.288813+0000 mgr.y (mgr.14901) 4100 : cluster [DBG] pgmap v4107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:14.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:14 smithi006 bash[27292]: cluster 2023-08-28T00:15:14.288813+0000 mgr.y (mgr.14901) 4100 : cluster [DBG] pgmap v4107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:15:14.145Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:15:14.147Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:14 smithi130 bash[27579]: cluster 2023-08-28T00:15:14.288813+0000 mgr.y (mgr.14901) 4100 : cluster [DBG] pgmap v4107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:14.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:14 smithi130 bash[51302]: debug 2023-08-28T00:15:14.310+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:14.312325+0000) 2023-08-28T00:15:15.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:15 smithi130 bash[27579]: audit 2023-08-28T00:15:14.752904+0000 mon.c (mon.2) 1552 : audit [DBG] from='client.? 172.21.15.6:0/1203170469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:15.631 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:15 smithi130 bash[51302]: debug 2023-08-28T00:15:15.310+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:15.312595+0000) 2023-08-28T00:15:15.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:15 smithi006 bash[33918]: audit 2023-08-28T00:15:14.752904+0000 mon.c (mon.2) 1552 : audit [DBG] from='client.? 172.21.15.6:0/1203170469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:15.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:15 smithi006 bash[27292]: audit 2023-08-28T00:15:14.752904+0000 mon.c (mon.2) 1552 : audit [DBG] from='client.? 172.21.15.6:0/1203170469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:16.584 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:16 smithi130 bash[51302]: debug 2023-08-28T00:15:16.310+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:16.312871+0000) 2023-08-28T00:15:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:16 smithi130 bash[27579]: cluster 2023-08-28T00:15:16.289489+0000 mgr.y (mgr.14901) 4101 : cluster [DBG] pgmap v4108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:16 smithi006 bash[27292]: cluster 2023-08-28T00:15:16.289489+0000 mgr.y (mgr.14901) 4101 : cluster [DBG] pgmap v4108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:16 smithi006 bash[33918]: cluster 2023-08-28T00:15:16.289489+0000 mgr.y (mgr.14901) 4101 : cluster [DBG] pgmap v4108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:17.586 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:17 smithi130 bash[51302]: debug 2023-08-28T00:15:17.310+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:17.313094+0000) 2023-08-28T00:15:17.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:17 smithi130 bash[27579]: audit 2023-08-28T00:15:17.167413+0000 mon.c (mon.2) 1553 : audit [DBG] from='client.? 172.21.15.6:0/4699422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:17.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:17 smithi006 bash[33918]: audit 2023-08-28T00:15:17.167413+0000 mon.c (mon.2) 1553 : audit [DBG] from='client.? 172.21.15.6:0/4699422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:17.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:17 smithi006 bash[27292]: audit 2023-08-28T00:15:17.167413+0000 mon.c (mon.2) 1553 : audit [DBG] from='client.? 172.21.15.6:0/4699422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:18.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:18 smithi130 bash[51302]: debug 2023-08-28T00:15:18.310+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:18.313400+0000) 2023-08-28T00:15:19.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:19 smithi130 bash[51302]: debug 2023-08-28T00:15:19.310+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:19.313711+0000) 2023-08-28T00:15:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:19 smithi130 bash[27579]: audit 2023-08-28T00:15:18.245759+0000 mon.a (mon.0) 3084 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:15:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:19 smithi130 bash[27579]: cluster 2023-08-28T00:15:18.290135+0000 mgr.y (mgr.14901) 4102 : cluster [DBG] pgmap v4109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:19.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:19 smithi130 bash[27579]: audit 2023-08-28T00:15:18.659223+0000 mon.a (mon.0) 3085 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:15:19.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:19 smithi130 bash[27579]: audit 2023-08-28T00:15:18.953353+0000 mon.b (mon.1) 517 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:15:19.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:19 smithi130 bash[27579]: audit 2023-08-28T00:15:18.955452+0000 mon.b (mon.1) 518 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:15:19.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:19 smithi130 bash[27579]: audit 2023-08-28T00:15:18.964918+0000 mon.a (mon.0) 3086 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:15:19.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:19 smithi006 bash[27292]: audit 2023-08-28T00:15:18.245759+0000 mon.a (mon.0) 3084 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:15:19.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:19 smithi006 bash[27292]: cluster 2023-08-28T00:15:18.290135+0000 mgr.y (mgr.14901) 4102 : cluster [DBG] pgmap v4109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:19.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:19 smithi006 bash[27292]: audit 2023-08-28T00:15:18.659223+0000 mon.a (mon.0) 3085 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:15:19.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:19 smithi006 bash[27292]: audit 2023-08-28T00:15:18.953353+0000 mon.b (mon.1) 517 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:15:19.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:19 smithi006 bash[27292]: audit 2023-08-28T00:15:18.955452+0000 mon.b (mon.1) 518 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:15:19.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:19 smithi006 bash[27292]: audit 2023-08-28T00:15:18.964918+0000 mon.a (mon.0) 3086 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:15:19.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:19 smithi006 bash[33918]: audit 2023-08-28T00:15:18.245759+0000 mon.a (mon.0) 3084 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:15:19.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:19 smithi006 bash[33918]: cluster 2023-08-28T00:15:18.290135+0000 mgr.y (mgr.14901) 4102 : cluster [DBG] pgmap v4109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:19.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:19 smithi006 bash[33918]: audit 2023-08-28T00:15:18.659223+0000 mon.a (mon.0) 3085 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:15:19.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:19 smithi006 bash[33918]: audit 2023-08-28T00:15:18.953353+0000 mon.b (mon.1) 517 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:15:19.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:19 smithi006 bash[33918]: audit 2023-08-28T00:15:18.955452+0000 mon.b (mon.1) 518 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:15:19.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:19 smithi006 bash[33918]: audit 2023-08-28T00:15:18.964918+0000 mon.a (mon.0) 3086 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:15:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:20 smithi130 bash[27579]: audit 2023-08-28T00:15:19.582566+0000 mon.c (mon.2) 1554 : audit [DBG] from='client.? 172.21.15.6:0/3596044484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:20.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:20 smithi130 bash[51302]: debug 2023-08-28T00:15:20.310+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:20.314054+0000) 2023-08-28T00:15:20.687 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:20 smithi006 bash[33918]: audit 2023-08-28T00:15:19.582566+0000 mon.c (mon.2) 1554 : audit [DBG] from='client.? 172.21.15.6:0/3596044484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:20.687 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:20 smithi006 bash[27292]: audit 2023-08-28T00:15:19.582566+0000 mon.c (mon.2) 1554 : audit [DBG] from='client.? 172.21.15.6:0/3596044484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:15:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:15:20] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T00:15:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:15:20.684Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:15:20.687Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:21 smithi130 bash[27579]: cluster 2023-08-28T00:15:20.291419+0000 mgr.y (mgr.14901) 4103 : cluster [DBG] pgmap v4110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:21.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:21 smithi130 bash[51302]: debug 2023-08-28T00:15:21.310+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:21.314343+0000) 2023-08-28T00:15:21.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:21 smithi006 bash[33918]: cluster 2023-08-28T00:15:20.291419+0000 mgr.y (mgr.14901) 4103 : cluster [DBG] pgmap v4110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:21.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:21 smithi006 bash[27292]: cluster 2023-08-28T00:15:20.291419+0000 mgr.y (mgr.14901) 4103 : cluster [DBG] pgmap v4110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:22 smithi130 bash[27579]: audit 2023-08-28T00:15:22.003944+0000 mon.c (mon.2) 1555 : audit [DBG] from='client.? 172.21.15.6:0/4082492912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:22.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:22 smithi130 bash[51302]: debug 2023-08-28T00:15:22.310+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:22.314609+0000) 2023-08-28T00:15:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:22 smithi006 bash[27292]: audit 2023-08-28T00:15:22.003944+0000 mon.c (mon.2) 1555 : audit [DBG] from='client.? 172.21.15.6:0/4082492912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:22 smithi006 bash[33918]: audit 2023-08-28T00:15:22.003944+0000 mon.c (mon.2) 1555 : audit [DBG] from='client.? 172.21.15.6:0/4082492912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:15:22.262Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:15:22.263Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:23.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:23 smithi130 bash[27579]: cluster 2023-08-28T00:15:22.292160+0000 mgr.y (mgr.14901) 4104 : cluster [DBG] pgmap v4111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:23.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:23 smithi130 bash[51302]: debug 2023-08-28T00:15:23.314+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:23.314901+0000) 2023-08-28T00:15:23.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:23 smithi006 bash[27292]: cluster 2023-08-28T00:15:22.292160+0000 mgr.y (mgr.14901) 4104 : cluster [DBG] pgmap v4111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:23.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:23 smithi006 bash[33918]: cluster 2023-08-28T00:15:22.292160+0000 mgr.y (mgr.14901) 4104 : cluster [DBG] pgmap v4111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:24.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:24 smithi006 bash[27292]: cluster 2023-08-28T00:15:24.293262+0000 mgr.y (mgr.14901) 4105 : cluster [DBG] pgmap v4112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:15:24.145Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:15:24.148Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:24.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:24 smithi006 bash[33918]: cluster 2023-08-28T00:15:24.293262+0000 mgr.y (mgr.14901) 4105 : cluster [DBG] pgmap v4112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:24 smithi130 bash[27579]: cluster 2023-08-28T00:15:24.293262+0000 mgr.y (mgr.14901) 4105 : cluster [DBG] pgmap v4112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:24.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:24 smithi130 bash[51302]: debug 2023-08-28T00:15:24.314+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:24.315157+0000) 2023-08-28T00:15:25.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:25 smithi130 bash[27579]: audit 2023-08-28T00:15:24.438913+0000 mon.c (mon.2) 1556 : audit [DBG] from='client.? 172.21.15.6:0/2694310962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:25.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:25 smithi130 bash[51302]: debug 2023-08-28T00:15:25.314+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:25.315431+0000) 2023-08-28T00:15:25.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:25 smithi006 bash[27292]: audit 2023-08-28T00:15:24.438913+0000 mon.c (mon.2) 1556 : audit [DBG] from='client.? 172.21.15.6:0/2694310962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:25.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:25 smithi006 bash[33918]: audit 2023-08-28T00:15:24.438913+0000 mon.c (mon.2) 1556 : audit [DBG] from='client.? 172.21.15.6:0/2694310962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:26.582 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:26 smithi130 bash[51302]: debug 2023-08-28T00:15:26.314+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:26.315654+0000) 2023-08-28T00:15:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:26 smithi130 bash[27579]: cluster 2023-08-28T00:15:26.293955+0000 mgr.y (mgr.14901) 4106 : cluster [DBG] pgmap v4113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:26 smithi006 bash[27292]: cluster 2023-08-28T00:15:26.293955+0000 mgr.y (mgr.14901) 4106 : cluster [DBG] pgmap v4113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:26 smithi006 bash[33918]: cluster 2023-08-28T00:15:26.293955+0000 mgr.y (mgr.14901) 4106 : cluster [DBG] pgmap v4113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:27.592 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:27 smithi130 bash[51302]: debug 2023-08-28T00:15:27.314+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:27.315924+0000) 2023-08-28T00:15:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:27 smithi130 bash[27579]: audit 2023-08-28T00:15:26.869013+0000 mon.c (mon.2) 1557 : audit [DBG] from='client.? 172.21.15.6:0/3539700576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:27.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:27 smithi006 bash[27292]: audit 2023-08-28T00:15:26.869013+0000 mon.c (mon.2) 1557 : audit [DBG] from='client.? 172.21.15.6:0/3539700576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:27.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:27 smithi006 bash[33918]: audit 2023-08-28T00:15:26.869013+0000 mon.c (mon.2) 1557 : audit [DBG] from='client.? 172.21.15.6:0/3539700576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:28.600 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:28 smithi130 bash[51302]: debug 2023-08-28T00:15:28.314+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:28.316214+0000) 2023-08-28T00:15:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:28 smithi130 bash[27579]: cluster 2023-08-28T00:15:28.294627+0000 mgr.y (mgr.14901) 4107 : cluster [DBG] pgmap v4114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:28 smithi006 bash[33918]: cluster 2023-08-28T00:15:28.294627+0000 mgr.y (mgr.14901) 4107 : cluster [DBG] pgmap v4114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:28 smithi006 bash[27292]: cluster 2023-08-28T00:15:28.294627+0000 mgr.y (mgr.14901) 4107 : cluster [DBG] pgmap v4114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:29.605 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:29 smithi130 bash[51302]: debug 2023-08-28T00:15:29.314+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:29.316519+0000) 2023-08-28T00:15:29.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:29 smithi130 bash[27579]: audit 2023-08-28T00:15:29.283432+0000 mon.a (mon.0) 3087 : audit [DBG] from='client.? 172.21.15.6:0/2967505710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:29.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:29 smithi006 bash[27292]: audit 2023-08-28T00:15:29.283432+0000 mon.a (mon.0) 3087 : audit [DBG] from='client.? 172.21.15.6:0/2967505710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:29.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:29 smithi006 bash[33918]: audit 2023-08-28T00:15:29.283432+0000 mon.a (mon.0) 3087 : audit [DBG] from='client.? 172.21.15.6:0/2967505710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:30.613 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:30 smithi130 bash[51302]: debug 2023-08-28T00:15:30.314+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:30.316787+0000) 2023-08-28T00:15:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:30 smithi130 bash[27579]: cluster 2023-08-28T00:15:30.295744+0000 mgr.y (mgr.14901) 4108 : cluster [DBG] pgmap v4115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:30.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:30 smithi006 bash[27292]: cluster 2023-08-28T00:15:30.295744+0000 mgr.y (mgr.14901) 4108 : cluster [DBG] pgmap v4115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:15:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:15:30] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:15:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:15:30.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:15:30.687Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:30.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:30 smithi006 bash[33918]: cluster 2023-08-28T00:15:30.295744+0000 mgr.y (mgr.14901) 4108 : cluster [DBG] pgmap v4115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:31.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:31 smithi130 bash[51302]: debug 2023-08-28T00:15:31.314+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:31.317089+0000) 2023-08-28T00:15:32.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:31 smithi130 bash[27579]: audit 2023-08-28T00:15:31.692141+0000 mon.c (mon.2) 1558 : audit [DBG] from='client.? 172.21.15.6:0/1356860583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:32.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:31 smithi006 bash[27292]: audit 2023-08-28T00:15:31.692141+0000 mon.c (mon.2) 1558 : audit [DBG] from='client.? 172.21.15.6:0/1356860583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:32.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:31 smithi006 bash[33918]: audit 2023-08-28T00:15:31.692141+0000 mon.c (mon.2) 1558 : audit [DBG] from='client.? 172.21.15.6:0/1356860583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:32.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:32 smithi130 bash[51302]: debug 2023-08-28T00:15:32.314+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:32.317375+0000) 2023-08-28T00:15:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:15:32.263Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:15:32.265Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:32 smithi130 bash[27579]: cluster 2023-08-28T00:15:32.296436+0000 mgr.y (mgr.14901) 4109 : cluster [DBG] pgmap v4116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:32 smithi006 bash[33918]: cluster 2023-08-28T00:15:32.296436+0000 mgr.y (mgr.14901) 4109 : cluster [DBG] pgmap v4116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:32 smithi006 bash[27292]: cluster 2023-08-28T00:15:32.296436+0000 mgr.y (mgr.14901) 4109 : cluster [DBG] pgmap v4116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:33.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:33 smithi130 bash[51302]: debug 2023-08-28T00:15:33.314+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:33.317644+0000) 2023-08-28T00:15:34.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:34 smithi006 bash[33918]: audit 2023-08-28T00:15:34.102038+0000 mon.c (mon.2) 1559 : audit [DBG] from='client.? 172.21.15.6:0/3602347263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:15:34.146Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:15:34.149Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:34.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:34 smithi006 bash[27292]: audit 2023-08-28T00:15:34.102038+0000 mon.c (mon.2) 1559 : audit [DBG] from='client.? 172.21.15.6:0/3602347263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:34 smithi130 bash[27579]: audit 2023-08-28T00:15:34.102038+0000 mon.c (mon.2) 1559 : audit [DBG] from='client.? 172.21.15.6:0/3602347263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:34.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:34 smithi130 bash[51302]: debug 2023-08-28T00:15:34.314+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:34.317906+0000) 2023-08-28T00:15:35.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:35 smithi006 bash[33918]: cluster 2023-08-28T00:15:34.297661+0000 mgr.y (mgr.14901) 4110 : cluster [DBG] pgmap v4117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:35.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:35 smithi006 bash[27292]: cluster 2023-08-28T00:15:34.297661+0000 mgr.y (mgr.14901) 4110 : cluster [DBG] pgmap v4117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:35.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:35 smithi130 bash[51302]: debug 2023-08-28T00:15:35.314+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:35.318205+0000) 2023-08-28T00:15:35.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:35 smithi130 bash[27579]: cluster 2023-08-28T00:15:34.297661+0000 mgr.y (mgr.14901) 4110 : cluster [DBG] pgmap v4117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:36.586 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:36 smithi130 bash[51302]: debug 2023-08-28T00:15:36.314+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:36.318506+0000) 2023-08-28T00:15:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:36 smithi130 bash[27579]: cluster 2023-08-28T00:15:36.298295+0000 mgr.y (mgr.14901) 4111 : cluster [DBG] pgmap v4118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:36 smithi130 bash[27579]: audit 2023-08-28T00:15:36.520745+0000 mon.c (mon.2) 1560 : audit [DBG] from='client.? 172.21.15.6:0/2381686929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:36 smithi006 bash[27292]: cluster 2023-08-28T00:15:36.298295+0000 mgr.y (mgr.14901) 4111 : cluster [DBG] pgmap v4118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:36 smithi006 bash[27292]: audit 2023-08-28T00:15:36.520745+0000 mon.c (mon.2) 1560 : audit [DBG] from='client.? 172.21.15.6:0/2381686929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:36 smithi006 bash[33918]: cluster 2023-08-28T00:15:36.298295+0000 mgr.y (mgr.14901) 4111 : cluster [DBG] pgmap v4118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:36 smithi006 bash[33918]: audit 2023-08-28T00:15:36.520745+0000 mon.c (mon.2) 1560 : audit [DBG] from='client.? 172.21.15.6:0/2381686929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:37.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:37 smithi130 bash[51302]: debug 2023-08-28T00:15:37.314+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:37.318775+0000) 2023-08-28T00:15:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:38 smithi130 bash[27579]: cluster 2023-08-28T00:15:38.298969+0000 mgr.y (mgr.14901) 4112 : cluster [DBG] pgmap v4119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:38.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:38 smithi130 bash[51302]: debug 2023-08-28T00:15:38.318+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:38.319051+0000) 2023-08-28T00:15:38.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:38 smithi006 bash[33918]: cluster 2023-08-28T00:15:38.298969+0000 mgr.y (mgr.14901) 4112 : cluster [DBG] pgmap v4119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:38.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:38 smithi006 bash[27292]: cluster 2023-08-28T00:15:38.298969+0000 mgr.y (mgr.14901) 4112 : cluster [DBG] pgmap v4119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:39.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:39 smithi130 bash[27579]: audit 2023-08-28T00:15:38.926561+0000 mon.a (mon.0) 3088 : audit [DBG] from='client.? 172.21.15.6:0/454113149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:39.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:39 smithi130 bash[51302]: debug 2023-08-28T00:15:39.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:39.319356+0000) 2023-08-28T00:15:39.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:39 smithi006 bash[33918]: audit 2023-08-28T00:15:38.926561+0000 mon.a (mon.0) 3088 : audit [DBG] from='client.? 172.21.15.6:0/454113149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:39.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:39 smithi006 bash[27292]: audit 2023-08-28T00:15:38.926561+0000 mon.a (mon.0) 3088 : audit [DBG] from='client.? 172.21.15.6:0/454113149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:40 smithi130 bash[27579]: cluster 2023-08-28T00:15:40.300075+0000 mgr.y (mgr.14901) 4113 : cluster [DBG] pgmap v4120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:40.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:40 smithi130 bash[51302]: debug 2023-08-28T00:15:40.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:40.319648+0000) 2023-08-28T00:15:40.688 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:40 smithi006 bash[27292]: cluster 2023-08-28T00:15:40.300075+0000 mgr.y (mgr.14901) 4113 : cluster [DBG] pgmap v4120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:40.689 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:40 smithi006 bash[33918]: cluster 2023-08-28T00:15:40.300075+0000 mgr.y (mgr.14901) 4113 : cluster [DBG] pgmap v4120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:15:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:15:40] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:15:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:15:40.686Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:15:40.688Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:41.584 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:41 smithi130 bash[51302]: debug 2023-08-28T00:15:41.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:41.319919+0000) 2023-08-28T00:15:41.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:41 smithi130 bash[27579]: audit 2023-08-28T00:15:41.334256+0000 mon.c (mon.2) 1561 : audit [DBG] from='client.? 172.21.15.6:0/2491424984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:41.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:41 smithi006 bash[27292]: audit 2023-08-28T00:15:41.334256+0000 mon.c (mon.2) 1561 : audit [DBG] from='client.? 172.21.15.6:0/2491424984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:41.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:41 smithi006 bash[33918]: audit 2023-08-28T00:15:41.334256+0000 mon.c (mon.2) 1561 : audit [DBG] from='client.? 172.21.15.6:0/2491424984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:42.591 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:15:42.264Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:42.591 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:15:42.265Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:42.591 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:42 smithi130 bash[51302]: debug 2023-08-28T00:15:42.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:42.320200+0000) 2023-08-28T00:15:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:42 smithi130 bash[27579]: cluster 2023-08-28T00:15:42.300860+0000 mgr.y (mgr.14901) 4114 : cluster [DBG] pgmap v4121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:42 smithi006 bash[33918]: cluster 2023-08-28T00:15:42.300860+0000 mgr.y (mgr.14901) 4114 : cluster [DBG] pgmap v4121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:42 smithi006 bash[27292]: cluster 2023-08-28T00:15:42.300860+0000 mgr.y (mgr.14901) 4114 : cluster [DBG] pgmap v4121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:43.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:43 smithi130 bash[51302]: debug 2023-08-28T00:15:43.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:43.320583+0000) 2023-08-28T00:15:44.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:43 smithi130 bash[27579]: audit 2023-08-28T00:15:43.753409+0000 mon.a (mon.0) 3089 : audit [DBG] from='client.? 172.21.15.6:0/501747314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:44.150 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:43 smithi006 bash[33918]: audit 2023-08-28T00:15:43.753409+0000 mon.a (mon.0) 3089 : audit [DBG] from='client.? 172.21.15.6:0/501747314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:44.151 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:43 smithi006 bash[27292]: audit 2023-08-28T00:15:43.753409+0000 mon.a (mon.0) 3089 : audit [DBG] from='client.? 172.21.15.6:0/501747314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:44.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:15:44.148Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:15:44.150Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:44.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:44 smithi130 bash[51302]: debug 2023-08-28T00:15:44.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:44.321028+0000) 2023-08-28T00:15:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:44 smithi130 bash[27579]: cluster 2023-08-28T00:15:44.302073+0000 mgr.y (mgr.14901) 4115 : cluster [DBG] pgmap v4122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:44 smithi006 bash[27292]: cluster 2023-08-28T00:15:44.302073+0000 mgr.y (mgr.14901) 4115 : cluster [DBG] pgmap v4122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:44 smithi006 bash[33918]: cluster 2023-08-28T00:15:44.302073+0000 mgr.y (mgr.14901) 4115 : cluster [DBG] pgmap v4122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:45.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:45 smithi130 bash[51302]: debug 2023-08-28T00:15:45.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:45.321319+0000) 2023-08-28T00:15:46.585 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:46 smithi130 bash[51302]: debug 2023-08-28T00:15:46.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:46.321581+0000) 2023-08-28T00:15:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:46 smithi130 bash[27579]: audit 2023-08-28T00:15:46.168973+0000 mon.c (mon.2) 1562 : audit [DBG] from='client.? 172.21.15.6:0/2321939476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:46 smithi130 bash[27579]: cluster 2023-08-28T00:15:46.302881+0000 mgr.y (mgr.14901) 4116 : cluster [DBG] pgmap v4123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:46 smithi006 bash[33918]: audit 2023-08-28T00:15:46.168973+0000 mon.c (mon.2) 1562 : audit [DBG] from='client.? 172.21.15.6:0/2321939476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:46 smithi006 bash[33918]: cluster 2023-08-28T00:15:46.302881+0000 mgr.y (mgr.14901) 4116 : cluster [DBG] pgmap v4123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:46 smithi006 bash[27292]: audit 2023-08-28T00:15:46.168973+0000 mon.c (mon.2) 1562 : audit [DBG] from='client.? 172.21.15.6:0/2321939476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:46 smithi006 bash[27292]: cluster 2023-08-28T00:15:46.302881+0000 mgr.y (mgr.14901) 4116 : cluster [DBG] pgmap v4123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:47.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:47 smithi130 bash[51302]: debug 2023-08-28T00:15:47.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:47.321885+0000) 2023-08-28T00:15:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:48 smithi130 bash[27579]: cluster 2023-08-28T00:15:48.303568+0000 mgr.y (mgr.14901) 4117 : cluster [DBG] pgmap v4124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:48.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:48 smithi130 bash[51302]: debug 2023-08-28T00:15:48.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:48.322194+0000) 2023-08-28T00:15:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:48 smithi006 bash[27292]: cluster 2023-08-28T00:15:48.303568+0000 mgr.y (mgr.14901) 4117 : cluster [DBG] pgmap v4124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:48 smithi006 bash[33918]: cluster 2023-08-28T00:15:48.303568+0000 mgr.y (mgr.14901) 4117 : cluster [DBG] pgmap v4124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:49.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:49 smithi130 bash[27579]: audit 2023-08-28T00:15:48.585719+0000 mon.a (mon.0) 3090 : audit [DBG] from='client.? 172.21.15.6:0/2457566234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:49.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:49 smithi130 bash[51302]: debug 2023-08-28T00:15:49.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:49.322493+0000) 2023-08-28T00:15:49.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:49 smithi006 bash[33918]: audit 2023-08-28T00:15:48.585719+0000 mon.a (mon.0) 3090 : audit [DBG] from='client.? 172.21.15.6:0/2457566234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:49.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:49 smithi006 bash[27292]: audit 2023-08-28T00:15:48.585719+0000 mon.a (mon.0) 3090 : audit [DBG] from='client.? 172.21.15.6:0/2457566234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:50 smithi130 bash[27579]: cluster 2023-08-28T00:15:50.304473+0000 mgr.y (mgr.14901) 4118 : cluster [DBG] pgmap v4125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:50.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:50 smithi130 bash[51302]: debug 2023-08-28T00:15:50.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:50.322758+0000) 2023-08-28T00:15:50.689 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:50 smithi006 bash[27292]: cluster 2023-08-28T00:15:50.304473+0000 mgr.y (mgr.14901) 4118 : cluster [DBG] pgmap v4125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:50.690 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:50 smithi006 bash[33918]: cluster 2023-08-28T00:15:50.304473+0000 mgr.y (mgr.14901) 4118 : cluster [DBG] pgmap v4125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:15:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:15:50] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:15:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:15:50.687Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:15:50.689Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:51.586 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:51 smithi130 bash[51302]: debug 2023-08-28T00:15:51.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:51.323032+0000) 2023-08-28T00:15:51.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:51 smithi130 bash[27579]: audit 2023-08-28T00:15:51.000389+0000 mon.c (mon.2) 1563 : audit [DBG] from='client.? 172.21.15.6:0/3006637529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:51.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:51 smithi006 bash[33918]: audit 2023-08-28T00:15:51.000389+0000 mon.c (mon.2) 1563 : audit [DBG] from='client.? 172.21.15.6:0/3006637529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:51.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:51 smithi006 bash[27292]: audit 2023-08-28T00:15:51.000389+0000 mon.c (mon.2) 1563 : audit [DBG] from='client.? 172.21.15.6:0/3006637529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:52.595 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:15:52.265Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:52.595 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:15:52.266Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:52.595 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:52 smithi130 bash[51302]: debug 2023-08-28T00:15:52.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:52.323233+0000) 2023-08-28T00:15:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:52 smithi130 bash[27579]: cluster 2023-08-28T00:15:52.305122+0000 mgr.y (mgr.14901) 4119 : cluster [DBG] pgmap v4126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:52 smithi006 bash[33918]: cluster 2023-08-28T00:15:52.305122+0000 mgr.y (mgr.14901) 4119 : cluster [DBG] pgmap v4126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:52 smithi006 bash[27292]: cluster 2023-08-28T00:15:52.305122+0000 mgr.y (mgr.14901) 4119 : cluster [DBG] pgmap v4126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:53.598 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:53 smithi130 bash[51302]: debug 2023-08-28T00:15:53.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:53.323521+0000) 2023-08-28T00:15:53.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:53 smithi130 bash[27579]: audit 2023-08-28T00:15:53.412361+0000 mon.c (mon.2) 1564 : audit [DBG] from='client.? 172.21.15.6:0/2263891706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:53.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:53 smithi006 bash[27292]: audit 2023-08-28T00:15:53.412361+0000 mon.c (mon.2) 1564 : audit [DBG] from='client.? 172.21.15.6:0/2263891706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:53.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:53 smithi006 bash[33918]: audit 2023-08-28T00:15:53.412361+0000 mon.c (mon.2) 1564 : audit [DBG] from='client.? 172.21.15.6:0/2263891706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:15:54.149Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:15:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:15:54.151Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:15:54.608 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:54 smithi130 bash[51302]: debug 2023-08-28T00:15:54.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:54.323862+0000) 2023-08-28T00:15:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:54 smithi130 bash[27579]: cluster 2023-08-28T00:15:54.306193+0000 mgr.y (mgr.14901) 4120 : cluster [DBG] pgmap v4127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:54 smithi006 bash[33918]: cluster 2023-08-28T00:15:54.306193+0000 mgr.y (mgr.14901) 4120 : cluster [DBG] pgmap v4127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:54 smithi006 bash[27292]: cluster 2023-08-28T00:15:54.306193+0000 mgr.y (mgr.14901) 4120 : cluster [DBG] pgmap v4127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:55.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:55 smithi130 bash[51302]: debug 2023-08-28T00:15:55.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:55.324134+0000) 2023-08-28T00:15:56.600 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:56 smithi130 bash[51302]: debug 2023-08-28T00:15:56.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:56.324426+0000) 2023-08-28T00:15:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:56 smithi130 bash[27579]: audit 2023-08-28T00:15:55.836952+0000 mon.c (mon.2) 1565 : audit [DBG] from='client.? 172.21.15.6:0/1373345188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:56 smithi130 bash[27579]: cluster 2023-08-28T00:15:56.306862+0000 mgr.y (mgr.14901) 4121 : cluster [DBG] pgmap v4128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:56 smithi006 bash[27292]: audit 2023-08-28T00:15:55.836952+0000 mon.c (mon.2) 1565 : audit [DBG] from='client.? 172.21.15.6:0/1373345188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:56 smithi006 bash[27292]: cluster 2023-08-28T00:15:56.306862+0000 mgr.y (mgr.14901) 4121 : cluster [DBG] pgmap v4128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:56 smithi006 bash[33918]: audit 2023-08-28T00:15:55.836952+0000 mon.c (mon.2) 1565 : audit [DBG] from='client.? 172.21.15.6:0/1373345188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:56 smithi006 bash[33918]: cluster 2023-08-28T00:15:56.306862+0000 mgr.y (mgr.14901) 4121 : cluster [DBG] pgmap v4128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:57.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:57 smithi130 bash[51302]: debug 2023-08-28T00:15:57.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:57.324698+0000) 2023-08-28T00:15:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:58 smithi130 bash[27579]: audit 2023-08-28T00:15:58.257444+0000 mon.a (mon.0) 3091 : audit [DBG] from='client.? 172.21.15.6:0/909787175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:58.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:58 smithi130 bash[51302]: debug 2023-08-28T00:15:58.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:58.324958+0000) 2023-08-28T00:15:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:58 smithi006 bash[33918]: audit 2023-08-28T00:15:58.257444+0000 mon.a (mon.0) 3091 : audit [DBG] from='client.? 172.21.15.6:0/909787175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:58 smithi006 bash[27292]: audit 2023-08-28T00:15:58.257444+0000 mon.a (mon.0) 3091 : audit [DBG] from='client.? 172.21.15.6:0/909787175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:15:59 smithi130 bash[27579]: cluster 2023-08-28T00:15:58.307384+0000 mgr.y (mgr.14901) 4122 : cluster [DBG] pgmap v4129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:59.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:15:59 smithi130 bash[51302]: debug 2023-08-28T00:15:59.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:59.325233+0000) 2023-08-28T00:15:59.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:15:59 smithi006 bash[33918]: cluster 2023-08-28T00:15:58.307384+0000 mgr.y (mgr.14901) 4122 : cluster [DBG] pgmap v4129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:15:59.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:15:59 smithi006 bash[27292]: cluster 2023-08-28T00:15:58.307384+0000 mgr.y (mgr.14901) 4122 : cluster [DBG] pgmap v4129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:00.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:00 smithi130 bash[51302]: debug 2023-08-28T00:16:00.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:00.325499+0000) 2023-08-28T00:16:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:16:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:16:00] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:16:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:16:00.687Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:16:00.689Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:01.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:01 smithi130 bash[51302]: debug 2023-08-28T00:16:01.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:01.325792+0000) 2023-08-28T00:16:01.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:01 smithi130 bash[27579]: cluster 2023-08-28T00:16:00.308506+0000 mgr.y (mgr.14901) 4123 : cluster [DBG] pgmap v4130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:01.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:01 smithi130 bash[27579]: audit 2023-08-28T00:16:00.687172+0000 mon.a (mon.0) 3092 : audit [DBG] from='client.? 172.21.15.6:0/307263172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:01.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:01 smithi006 bash[33918]: cluster 2023-08-28T00:16:00.308506+0000 mgr.y (mgr.14901) 4123 : cluster [DBG] pgmap v4130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:01.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:01 smithi006 bash[33918]: audit 2023-08-28T00:16:00.687172+0000 mon.a (mon.0) 3092 : audit [DBG] from='client.? 172.21.15.6:0/307263172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:01.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:01 smithi006 bash[27292]: cluster 2023-08-28T00:16:00.308506+0000 mgr.y (mgr.14901) 4123 : cluster [DBG] pgmap v4130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:01.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:01 smithi006 bash[27292]: audit 2023-08-28T00:16:00.687172+0000 mon.a (mon.0) 3092 : audit [DBG] from='client.? 172.21.15.6:0/307263172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:02.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:02 smithi130 bash[51302]: debug 2023-08-28T00:16:02.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:02.326014+0000) 2023-08-28T00:16:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:16:02.265Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:16:02.268Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:03.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:03 smithi130 bash[51302]: debug 2023-08-28T00:16:03.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:03.326268+0000) 2023-08-28T00:16:03.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:03 smithi130 bash[27579]: cluster 2023-08-28T00:16:02.309224+0000 mgr.y (mgr.14901) 4124 : cluster [DBG] pgmap v4131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:03.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:03 smithi130 bash[27579]: audit 2023-08-28T00:16:03.112065+0000 mon.a (mon.0) 3093 : audit [DBG] from='client.? 172.21.15.6:0/162070650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:03.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:03 smithi006 bash[33918]: cluster 2023-08-28T00:16:02.309224+0000 mgr.y (mgr.14901) 4124 : cluster [DBG] pgmap v4131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:03.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:03 smithi006 bash[33918]: audit 2023-08-28T00:16:03.112065+0000 mon.a (mon.0) 3093 : audit [DBG] from='client.? 172.21.15.6:0/162070650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:03.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:03 smithi006 bash[27292]: cluster 2023-08-28T00:16:02.309224+0000 mgr.y (mgr.14901) 4124 : cluster [DBG] pgmap v4131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:03.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:03 smithi006 bash[27292]: audit 2023-08-28T00:16:03.112065+0000 mon.a (mon.0) 3093 : audit [DBG] from='client.? 172.21.15.6:0/162070650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:16:04.150Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:16:04.152Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:04.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:04 smithi130 bash[51302]: debug 2023-08-28T00:16:04.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:04.326537+0000) 2023-08-28T00:16:05.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:05 smithi130 bash[27579]: cluster 2023-08-28T00:16:04.310308+0000 mgr.y (mgr.14901) 4125 : cluster [DBG] pgmap v4132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:05.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:05 smithi130 bash[51302]: debug 2023-08-28T00:16:05.325+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:05.326823+0000) 2023-08-28T00:16:05.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:05 smithi006 bash[33918]: cluster 2023-08-28T00:16:04.310308+0000 mgr.y (mgr.14901) 4125 : cluster [DBG] pgmap v4132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:05.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:05 smithi006 bash[27292]: cluster 2023-08-28T00:16:04.310308+0000 mgr.y (mgr.14901) 4125 : cluster [DBG] pgmap v4132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:06.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:06 smithi130 bash[51302]: debug 2023-08-28T00:16:06.325+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:06.327149+0000) 2023-08-28T00:16:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:06 smithi130 bash[27579]: audit 2023-08-28T00:16:05.523721+0000 mon.c (mon.2) 1566 : audit [DBG] from='client.? 172.21.15.6:0/3091908299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:06 smithi006 bash[33918]: audit 2023-08-28T00:16:05.523721+0000 mon.c (mon.2) 1566 : audit [DBG] from='client.? 172.21.15.6:0/3091908299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:06 smithi006 bash[27292]: audit 2023-08-28T00:16:05.523721+0000 mon.c (mon.2) 1566 : audit [DBG] from='client.? 172.21.15.6:0/3091908299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:07.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:07 smithi130 bash[27579]: cluster 2023-08-28T00:16:06.310819+0000 mgr.y (mgr.14901) 4126 : cluster [DBG] pgmap v4133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:07.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:07 smithi130 bash[51302]: debug 2023-08-28T00:16:07.325+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:07.327415+0000) 2023-08-28T00:16:07.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:07 smithi006 bash[27292]: cluster 2023-08-28T00:16:06.310819+0000 mgr.y (mgr.14901) 4126 : cluster [DBG] pgmap v4133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:07 smithi006 bash[33918]: cluster 2023-08-28T00:16:06.310819+0000 mgr.y (mgr.14901) 4126 : cluster [DBG] pgmap v4133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:08 smithi130 bash[27579]: audit 2023-08-28T00:16:07.937021+0000 mon.c (mon.2) 1567 : audit [DBG] from='client.? 172.21.15.6:0/3034789094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:08.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:08 smithi130 bash[51302]: debug 2023-08-28T00:16:08.325+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:08.327695+0000) 2023-08-28T00:16:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:08 smithi006 bash[33918]: audit 2023-08-28T00:16:07.937021+0000 mon.c (mon.2) 1567 : audit [DBG] from='client.? 172.21.15.6:0/3034789094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:08 smithi006 bash[27292]: audit 2023-08-28T00:16:07.937021+0000 mon.c (mon.2) 1567 : audit [DBG] from='client.? 172.21.15.6:0/3034789094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:09 smithi130 bash[27579]: cluster 2023-08-28T00:16:08.311448+0000 mgr.y (mgr.14901) 4127 : cluster [DBG] pgmap v4134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:09.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:09 smithi130 bash[51302]: debug 2023-08-28T00:16:09.325+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:09.328012+0000) 2023-08-28T00:16:09.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:09 smithi006 bash[33918]: cluster 2023-08-28T00:16:08.311448+0000 mgr.y (mgr.14901) 4127 : cluster [DBG] pgmap v4134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:09 smithi006 bash[27292]: cluster 2023-08-28T00:16:08.311448+0000 mgr.y (mgr.14901) 4127 : cluster [DBG] pgmap v4134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:10.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:10 smithi130 bash[51302]: debug 2023-08-28T00:16:10.325+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:10.328288+0000) 2023-08-28T00:16:10.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:10 smithi130 bash[27579]: cluster 2023-08-28T00:16:10.312576+0000 mgr.y (mgr.14901) 4128 : cluster [DBG] pgmap v4135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:10.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:10 smithi130 bash[27579]: audit 2023-08-28T00:16:10.355970+0000 mon.c (mon.2) 1568 : audit [DBG] from='client.? 172.21.15.6:0/3293334166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:10.691 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:10 smithi006 bash[33918]: cluster 2023-08-28T00:16:10.312576+0000 mgr.y (mgr.14901) 4128 : cluster [DBG] pgmap v4135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:10.691 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:10 smithi006 bash[33918]: audit 2023-08-28T00:16:10.355970+0000 mon.c (mon.2) 1568 : audit [DBG] from='client.? 172.21.15.6:0/3293334166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:10.691 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:10 smithi006 bash[27292]: cluster 2023-08-28T00:16:10.312576+0000 mgr.y (mgr.14901) 4128 : cluster [DBG] pgmap v4135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:10.691 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:10 smithi006 bash[27292]: audit 2023-08-28T00:16:10.355970+0000 mon.c (mon.2) 1568 : audit [DBG] from='client.? 172.21.15.6:0/3293334166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:16:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:16:10] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:16:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:16:10.688Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:16:10.690Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:11.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:11 smithi130 bash[51302]: debug 2023-08-28T00:16:11.325+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:11.328548+0000) 2023-08-28T00:16:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:12 smithi130 bash[27579]: cluster 2023-08-28T00:16:12.313309+0000 mgr.y (mgr.14901) 4129 : cluster [DBG] pgmap v4136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:12.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:12 smithi130 bash[51302]: debug 2023-08-28T00:16:12.325+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:12.328777+0000) 2023-08-28T00:16:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:16:12.266Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:16:12.268Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:12.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:12 smithi006 bash[33918]: cluster 2023-08-28T00:16:12.313309+0000 mgr.y (mgr.14901) 4129 : cluster [DBG] pgmap v4136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:12 smithi006 bash[27292]: cluster 2023-08-28T00:16:12.313309+0000 mgr.y (mgr.14901) 4129 : cluster [DBG] pgmap v4136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:13 smithi130 bash[27579]: audit 2023-08-28T00:16:12.766543+0000 mon.a (mon.0) 3094 : audit [DBG] from='client.? 172.21.15.6:0/58700152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:13.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:13 smithi130 bash[51302]: debug 2023-08-28T00:16:13.325+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:13.329041+0000) 2023-08-28T00:16:13.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:13 smithi006 bash[27292]: audit 2023-08-28T00:16:12.766543+0000 mon.a (mon.0) 3094 : audit [DBG] from='client.? 172.21.15.6:0/58700152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:13.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:13 smithi006 bash[33918]: audit 2023-08-28T00:16:12.766543+0000 mon.a (mon.0) 3094 : audit [DBG] from='client.? 172.21.15.6:0/58700152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:14.413 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:16:14.151Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:14.413 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:16:14.153Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:14 smithi130 bash[27579]: cluster 2023-08-28T00:16:14.314176+0000 mgr.y (mgr.14901) 4130 : cluster [DBG] pgmap v4137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:14.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:14 smithi130 bash[51302]: debug 2023-08-28T00:16:14.325+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:14.329323+0000) 2023-08-28T00:16:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:14 smithi006 bash[33918]: cluster 2023-08-28T00:16:14.314176+0000 mgr.y (mgr.14901) 4130 : cluster [DBG] pgmap v4137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:14.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:14 smithi006 bash[27292]: cluster 2023-08-28T00:16:14.314176+0000 mgr.y (mgr.14901) 4130 : cluster [DBG] pgmap v4137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:15.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:15 smithi130 bash[27579]: audit 2023-08-28T00:16:15.186615+0000 mon.c (mon.2) 1569 : audit [DBG] from='client.? 172.21.15.6:0/2577350823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:15.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:15 smithi130 bash[51302]: debug 2023-08-28T00:16:15.329+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:15.329596+0000) 2023-08-28T00:16:15.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:15 smithi006 bash[33918]: audit 2023-08-28T00:16:15.186615+0000 mon.c (mon.2) 1569 : audit [DBG] from='client.? 172.21.15.6:0/2577350823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:15.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:15 smithi006 bash[27292]: audit 2023-08-28T00:16:15.186615+0000 mon.c (mon.2) 1569 : audit [DBG] from='client.? 172.21.15.6:0/2577350823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:16.611 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:16 smithi130 bash[51302]: debug 2023-08-28T00:16:16.325+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:16.329881+0000) 2023-08-28T00:16:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:16 smithi130 bash[27579]: cluster 2023-08-28T00:16:16.314860+0000 mgr.y (mgr.14901) 4131 : cluster [DBG] pgmap v4138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:16 smithi006 bash[27292]: cluster 2023-08-28T00:16:16.314860+0000 mgr.y (mgr.14901) 4131 : cluster [DBG] pgmap v4138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:16 smithi006 bash[33918]: cluster 2023-08-28T00:16:16.314860+0000 mgr.y (mgr.14901) 4131 : cluster [DBG] pgmap v4138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:17.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:17 smithi130 bash[51302]: debug 2023-08-28T00:16:17.325+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:17.330145+0000) 2023-08-28T00:16:17.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:17 smithi130 bash[27579]: audit 2023-08-28T00:16:17.602508+0000 mon.c (mon.2) 1570 : audit [DBG] from='client.? 172.21.15.6:0/1907119073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:17.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:17 smithi006 bash[33918]: audit 2023-08-28T00:16:17.602508+0000 mon.c (mon.2) 1570 : audit [DBG] from='client.? 172.21.15.6:0/1907119073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:17.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:17 smithi006 bash[27292]: audit 2023-08-28T00:16:17.602508+0000 mon.c (mon.2) 1570 : audit [DBG] from='client.? 172.21.15.6:0/1907119073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:18.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:18 smithi130 bash[51302]: debug 2023-08-28T00:16:18.325+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:18.330454+0000) 2023-08-28T00:16:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:18 smithi006 bash[33918]: cluster 2023-08-28T00:16:18.315512+0000 mgr.y (mgr.14901) 4132 : cluster [DBG] pgmap v4139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:18 smithi006 bash[27292]: cluster 2023-08-28T00:16:18.315512+0000 mgr.y (mgr.14901) 4132 : cluster [DBG] pgmap v4139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:18 smithi130 bash[27579]: cluster 2023-08-28T00:16:18.315512+0000 mgr.y (mgr.14901) 4132 : cluster [DBG] pgmap v4139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:19.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:19 smithi130 bash[51302]: debug 2023-08-28T00:16:19.325+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:19.330741+0000) 2023-08-28T00:16:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:19 smithi006 bash[33918]: audit 2023-08-28T00:16:18.974623+0000 mon.b (mon.1) 519 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:16:19.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:19 smithi006 bash[27292]: audit 2023-08-28T00:16:18.974623+0000 mon.b (mon.1) 519 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:16:20.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:19 smithi130 bash[27579]: audit 2023-08-28T00:16:18.974623+0000 mon.b (mon.1) 519 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:16:20.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:20 smithi130 bash[51302]: debug 2023-08-28T00:16:20.329+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:20.331047+0000) 2023-08-28T00:16:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:16:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:16:20] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:16:20.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:20 smithi006 bash[27292]: audit 2023-08-28T00:16:20.013211+0000 mon.c (mon.2) 1571 : audit [DBG] from='client.? 172.21.15.6:0/740586974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:20.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:20 smithi006 bash[27292]: cluster 2023-08-28T00:16:20.316651+0000 mgr.y (mgr.14901) 4133 : cluster [DBG] pgmap v4140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:16:20.688Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:20.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:16:20.690Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:20.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:20 smithi006 bash[33918]: audit 2023-08-28T00:16:20.013211+0000 mon.c (mon.2) 1571 : audit [DBG] from='client.? 172.21.15.6:0/740586974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:20.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:20 smithi006 bash[33918]: cluster 2023-08-28T00:16:20.316651+0000 mgr.y (mgr.14901) 4133 : cluster [DBG] pgmap v4140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:20 smithi130 bash[27579]: audit 2023-08-28T00:16:20.013211+0000 mon.c (mon.2) 1571 : audit [DBG] from='client.? 172.21.15.6:0/740586974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:20 smithi130 bash[27579]: cluster 2023-08-28T00:16:20.316651+0000 mgr.y (mgr.14901) 4133 : cluster [DBG] pgmap v4140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:21.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:21 smithi130 bash[51302]: debug 2023-08-28T00:16:21.329+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:21.331438+0000) 2023-08-28T00:16:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:22 smithi130 bash[27579]: cluster 2023-08-28T00:16:22.317274+0000 mgr.y (mgr.14901) 4134 : cluster [DBG] pgmap v4141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:22.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:22 smithi130 bash[51302]: debug 2023-08-28T00:16:22.329+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:22.331721+0000) 2023-08-28T00:16:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:22 smithi006 bash[33918]: cluster 2023-08-28T00:16:22.317274+0000 mgr.y (mgr.14901) 4134 : cluster [DBG] pgmap v4141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:22 smithi006 bash[27292]: cluster 2023-08-28T00:16:22.317274+0000 mgr.y (mgr.14901) 4134 : cluster [DBG] pgmap v4141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:16:22.267Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:16:22.268Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:23.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:23 smithi130 bash[27579]: audit 2023-08-28T00:16:22.440043+0000 mon.a (mon.0) 3095 : audit [DBG] from='client.? 172.21.15.6:0/1819525820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:23.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:23 smithi130 bash[51302]: debug 2023-08-28T00:16:23.329+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:23.331974+0000) 2023-08-28T00:16:23.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:23 smithi006 bash[27292]: audit 2023-08-28T00:16:22.440043+0000 mon.a (mon.0) 3095 : audit [DBG] from='client.? 172.21.15.6:0/1819525820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:23.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:23 smithi006 bash[33918]: audit 2023-08-28T00:16:22.440043+0000 mon.a (mon.0) 3095 : audit [DBG] from='client.? 172.21.15.6:0/1819525820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:24.409 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:16:24.152Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:24.410 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:16:24.154Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:24.410 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:24 smithi006 bash[27292]: cluster 2023-08-28T00:16:24.318380+0000 mgr.y (mgr.14901) 4135 : cluster [DBG] pgmap v4142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:24 smithi130 bash[27579]: cluster 2023-08-28T00:16:24.318380+0000 mgr.y (mgr.14901) 4135 : cluster [DBG] pgmap v4142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:24.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:24 smithi130 bash[51302]: debug 2023-08-28T00:16:24.329+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:24.332156+0000) 2023-08-28T00:16:24.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:24 smithi006 bash[33918]: cluster 2023-08-28T00:16:24.318380+0000 mgr.y (mgr.14901) 4135 : cluster [DBG] pgmap v4142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:25.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:25 smithi130 bash[51302]: debug 2023-08-28T00:16:25.329+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:25.332443+0000) 2023-08-28T00:16:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:25 smithi006 bash[27292]: audit 2023-08-28T00:16:24.872139+0000 mon.a (mon.0) 3096 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:16:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:25 smithi006 bash[27292]: audit 2023-08-28T00:16:24.873481+0000 mon.c (mon.2) 1572 : audit [DBG] from='client.? 172.21.15.6:0/1720066100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:25 smithi006 bash[27292]: audit 2023-08-28T00:16:25.335726+0000 mon.a (mon.0) 3097 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:16:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:25 smithi006 bash[27292]: audit 2023-08-28T00:16:25.631070+0000 mon.b (mon.1) 520 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:16:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:25 smithi006 bash[27292]: audit 2023-08-28T00:16:25.633466+0000 mon.b (mon.1) 521 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:16:26.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:25 smithi006 bash[27292]: audit 2023-08-28T00:16:25.644298+0000 mon.a (mon.0) 3098 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:16:26.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:25 smithi006 bash[33918]: audit 2023-08-28T00:16:24.872139+0000 mon.a (mon.0) 3096 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:16:26.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:25 smithi006 bash[33918]: audit 2023-08-28T00:16:24.873481+0000 mon.c (mon.2) 1572 : audit [DBG] from='client.? 172.21.15.6:0/1720066100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:26.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:25 smithi006 bash[33918]: audit 2023-08-28T00:16:25.335726+0000 mon.a (mon.0) 3097 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:16:26.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:25 smithi006 bash[33918]: audit 2023-08-28T00:16:25.631070+0000 mon.b (mon.1) 520 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:16:26.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:25 smithi006 bash[33918]: audit 2023-08-28T00:16:25.633466+0000 mon.b (mon.1) 521 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:16:26.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:25 smithi006 bash[33918]: audit 2023-08-28T00:16:25.644298+0000 mon.a (mon.0) 3098 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:16:26.334 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:25 smithi130 bash[27579]: audit 2023-08-28T00:16:24.872139+0000 mon.a (mon.0) 3096 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:16:26.334 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:25 smithi130 bash[27579]: audit 2023-08-28T00:16:24.873481+0000 mon.c (mon.2) 1572 : audit [DBG] from='client.? 172.21.15.6:0/1720066100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:26.334 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:25 smithi130 bash[27579]: audit 2023-08-28T00:16:25.335726+0000 mon.a (mon.0) 3097 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:16:26.334 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:25 smithi130 bash[27579]: audit 2023-08-28T00:16:25.631070+0000 mon.b (mon.1) 520 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:16:26.335 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:25 smithi130 bash[27579]: audit 2023-08-28T00:16:25.633466+0000 mon.b (mon.1) 521 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:16:26.335 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:25 smithi130 bash[27579]: audit 2023-08-28T00:16:25.644298+0000 mon.a (mon.0) 3098 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:16:26.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:26 smithi130 bash[51302]: debug 2023-08-28T00:16:26.329+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:26.332730+0000) 2023-08-28T00:16:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:26 smithi006 bash[33918]: cluster 2023-08-28T00:16:26.319113+0000 mgr.y (mgr.14901) 4136 : cluster [DBG] pgmap v4143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:26 smithi006 bash[27292]: cluster 2023-08-28T00:16:26.319113+0000 mgr.y (mgr.14901) 4136 : cluster [DBG] pgmap v4143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:27.334 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:26 smithi130 bash[27579]: cluster 2023-08-28T00:16:26.319113+0000 mgr.y (mgr.14901) 4136 : cluster [DBG] pgmap v4143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:27.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:27 smithi130 bash[51302]: debug 2023-08-28T00:16:27.329+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:27.333030+0000) 2023-08-28T00:16:28.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:27 smithi006 bash[27292]: audit 2023-08-28T00:16:27.292590+0000 mon.c (mon.2) 1573 : audit [DBG] from='client.? 172.21.15.6:0/3560088177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:28.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:27 smithi006 bash[33918]: audit 2023-08-28T00:16:27.292590+0000 mon.c (mon.2) 1573 : audit [DBG] from='client.? 172.21.15.6:0/3560088177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:28.334 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:27 smithi130 bash[27579]: audit 2023-08-28T00:16:27.292590+0000 mon.c (mon.2) 1573 : audit [DBG] from='client.? 172.21.15.6:0/3560088177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:28.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:28 smithi130 bash[51302]: debug 2023-08-28T00:16:28.333+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:28.333304+0000) 2023-08-28T00:16:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:28 smithi006 bash[33918]: cluster 2023-08-28T00:16:28.319742+0000 mgr.y (mgr.14901) 4137 : cluster [DBG] pgmap v4144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:28 smithi006 bash[27292]: cluster 2023-08-28T00:16:28.319742+0000 mgr.y (mgr.14901) 4137 : cluster [DBG] pgmap v4144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:29.335 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:28 smithi130 bash[27579]: cluster 2023-08-28T00:16:28.319742+0000 mgr.y (mgr.14901) 4137 : cluster [DBG] pgmap v4144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:29.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:29 smithi130 bash[51302]: debug 2023-08-28T00:16:29.333+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:29.333588+0000) 2023-08-28T00:16:30.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:29 smithi006 bash[27292]: audit 2023-08-28T00:16:29.712182+0000 mon.a (mon.0) 3099 : audit [DBG] from='client.? 172.21.15.6:0/3020757081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:30.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:29 smithi006 bash[33918]: audit 2023-08-28T00:16:29.712182+0000 mon.a (mon.0) 3099 : audit [DBG] from='client.? 172.21.15.6:0/3020757081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:30.335 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:29 smithi130 bash[27579]: audit 2023-08-28T00:16:29.712182+0000 mon.a (mon.0) 3099 : audit [DBG] from='client.? 172.21.15.6:0/3020757081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:30.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:30 smithi130 bash[51302]: debug 2023-08-28T00:16:30.329+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:30.333855+0000) 2023-08-28T00:16:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:16:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:16:30] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:16:30.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:30 smithi006 bash[33918]: cluster 2023-08-28T00:16:30.320962+0000 mgr.y (mgr.14901) 4138 : cluster [DBG] pgmap v4145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:30.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:30 smithi006 bash[27292]: cluster 2023-08-28T00:16:30.320962+0000 mgr.y (mgr.14901) 4138 : cluster [DBG] pgmap v4145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:16:30.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:16:30.691Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:31.335 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:30 smithi130 bash[27579]: cluster 2023-08-28T00:16:30.320962+0000 mgr.y (mgr.14901) 4138 : cluster [DBG] pgmap v4145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:31.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:31 smithi130 bash[51302]: debug 2023-08-28T00:16:31.329+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:31.334079+0000) 2023-08-28T00:16:32.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:32 smithi006 bash[27292]: audit 2023-08-28T00:16:32.124793+0000 mon.a (mon.0) 3100 : audit [DBG] from='client.? 172.21.15.6:0/2795991955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:32.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:32 smithi006 bash[33918]: audit 2023-08-28T00:16:32.124793+0000 mon.a (mon.0) 3100 : audit [DBG] from='client.? 172.21.15.6:0/2795991955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:16:32.268Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:32.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:16:32.270Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:32 smithi130 bash[27579]: audit 2023-08-28T00:16:32.124793+0000 mon.a (mon.0) 3100 : audit [DBG] from='client.? 172.21.15.6:0/2795991955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:32.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:32 smithi130 bash[51302]: debug 2023-08-28T00:16:32.329+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:32.334375+0000) 2023-08-28T00:16:33.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:33 smithi006 bash[33918]: cluster 2023-08-28T00:16:32.321647+0000 mgr.y (mgr.14901) 4139 : cluster [DBG] pgmap v4146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:33.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:33 smithi006 bash[27292]: cluster 2023-08-28T00:16:32.321647+0000 mgr.y (mgr.14901) 4139 : cluster [DBG] pgmap v4146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:33 smithi130 bash[27579]: cluster 2023-08-28T00:16:32.321647+0000 mgr.y (mgr.14901) 4139 : cluster [DBG] pgmap v4146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:33.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:33 smithi130 bash[51302]: debug 2023-08-28T00:16:33.333+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:33.334646+0000) 2023-08-28T00:16:34.406 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:16:34.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:34.406 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:16:34.155Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:34 smithi130 bash[27579]: cluster 2023-08-28T00:16:34.322751+0000 mgr.y (mgr.14901) 4140 : cluster [DBG] pgmap v4147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:34.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:34 smithi130 bash[51302]: debug 2023-08-28T00:16:34.333+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:34.334985+0000) 2023-08-28T00:16:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:34 smithi006 bash[27292]: cluster 2023-08-28T00:16:34.322751+0000 mgr.y (mgr.14901) 4140 : cluster [DBG] pgmap v4147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:34 smithi006 bash[33918]: cluster 2023-08-28T00:16:34.322751+0000 mgr.y (mgr.14901) 4140 : cluster [DBG] pgmap v4147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:35.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:35 smithi130 bash[27579]: audit 2023-08-28T00:16:34.545089+0000 mon.a (mon.0) 3101 : audit [DBG] from='client.? 172.21.15.6:0/3960713033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:35.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:35 smithi130 bash[51302]: debug 2023-08-28T00:16:35.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:35.335247+0000) 2023-08-28T00:16:35.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:35 smithi006 bash[27292]: audit 2023-08-28T00:16:34.545089+0000 mon.a (mon.0) 3101 : audit [DBG] from='client.? 172.21.15.6:0/3960713033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:35.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:35 smithi006 bash[33918]: audit 2023-08-28T00:16:34.545089+0000 mon.a (mon.0) 3101 : audit [DBG] from='client.? 172.21.15.6:0/3960713033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:36.610 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:36 smithi130 bash[51302]: debug 2023-08-28T00:16:36.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:36.335520+0000) 2023-08-28T00:16:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:36 smithi130 bash[27579]: cluster 2023-08-28T00:16:36.323419+0000 mgr.y (mgr.14901) 4141 : cluster [DBG] pgmap v4148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:36 smithi006 bash[27292]: cluster 2023-08-28T00:16:36.323419+0000 mgr.y (mgr.14901) 4141 : cluster [DBG] pgmap v4148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:36 smithi006 bash[33918]: cluster 2023-08-28T00:16:36.323419+0000 mgr.y (mgr.14901) 4141 : cluster [DBG] pgmap v4148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:37.612 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:37 smithi130 bash[51302]: debug 2023-08-28T00:16:37.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:37.335787+0000) 2023-08-28T00:16:37.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:37 smithi130 bash[27579]: audit 2023-08-28T00:16:36.957122+0000 mon.a (mon.0) 3102 : audit [DBG] from='client.? 172.21.15.6:0/2298030538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:37.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:37 smithi006 bash[33918]: audit 2023-08-28T00:16:36.957122+0000 mon.a (mon.0) 3102 : audit [DBG] from='client.? 172.21.15.6:0/2298030538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:37.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:37 smithi006 bash[27292]: audit 2023-08-28T00:16:36.957122+0000 mon.a (mon.0) 3102 : audit [DBG] from='client.? 172.21.15.6:0/2298030538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:38.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:38 smithi130 bash[51302]: debug 2023-08-28T00:16:38.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:38.336102+0000) 2023-08-28T00:16:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:38 smithi006 bash[27292]: cluster 2023-08-28T00:16:38.324194+0000 mgr.y (mgr.14901) 4142 : cluster [DBG] pgmap v4149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:38 smithi006 bash[33918]: cluster 2023-08-28T00:16:38.324194+0000 mgr.y (mgr.14901) 4142 : cluster [DBG] pgmap v4149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:38 smithi130 bash[27579]: cluster 2023-08-28T00:16:38.324194+0000 mgr.y (mgr.14901) 4142 : cluster [DBG] pgmap v4149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:39.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:39 smithi130 bash[51302]: debug 2023-08-28T00:16:39.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:39.336343+0000) 2023-08-28T00:16:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:39 smithi006 bash[33918]: audit 2023-08-28T00:16:39.378403+0000 mon.c (mon.2) 1574 : audit [DBG] from='client.? 172.21.15.6:0/3598387066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:39.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:39 smithi006 bash[27292]: audit 2023-08-28T00:16:39.378403+0000 mon.c (mon.2) 1574 : audit [DBG] from='client.? 172.21.15.6:0/3598387066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:39 smithi130 bash[27579]: audit 2023-08-28T00:16:39.378403+0000 mon.c (mon.2) 1574 : audit [DBG] from='client.? 172.21.15.6:0/3598387066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:40.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:40 smithi130 bash[51302]: debug 2023-08-28T00:16:40.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:40.336600+0000) 2023-08-28T00:16:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:16:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:16:40] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:16:40.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:40 smithi006 bash[33918]: cluster 2023-08-28T00:16:40.325365+0000 mgr.y (mgr.14901) 4143 : cluster [DBG] pgmap v4150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:40.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:40 smithi006 bash[27292]: cluster 2023-08-28T00:16:40.325365+0000 mgr.y (mgr.14901) 4143 : cluster [DBG] pgmap v4150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:16:40.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:16:40.692Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:40 smithi130 bash[27579]: cluster 2023-08-28T00:16:40.325365+0000 mgr.y (mgr.14901) 4143 : cluster [DBG] pgmap v4150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:41.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:41 smithi130 bash[51302]: debug 2023-08-28T00:16:41.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:41.336922+0000) 2023-08-28T00:16:42.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:41 smithi130 bash[27579]: audit 2023-08-28T00:16:41.798544+0000 mon.a (mon.0) 3103 : audit [DBG] from='client.? 172.21.15.6:0/2806716828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:42.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:41 smithi006 bash[27292]: audit 2023-08-28T00:16:41.798544+0000 mon.a (mon.0) 3103 : audit [DBG] from='client.? 172.21.15.6:0/2806716828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:42.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:41 smithi006 bash[33918]: audit 2023-08-28T00:16:41.798544+0000 mon.a (mon.0) 3103 : audit [DBG] from='client.? 172.21.15.6:0/2806716828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:42.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:42 smithi130 bash[51302]: debug 2023-08-28T00:16:42.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:42.337210+0000) 2023-08-28T00:16:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:16:42.269Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:16:42.271Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:42 smithi130 bash[27579]: cluster 2023-08-28T00:16:42.326054+0000 mgr.y (mgr.14901) 4144 : cluster [DBG] pgmap v4151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:42 smithi006 bash[33918]: cluster 2023-08-28T00:16:42.326054+0000 mgr.y (mgr.14901) 4144 : cluster [DBG] pgmap v4151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:42 smithi006 bash[27292]: cluster 2023-08-28T00:16:42.326054+0000 mgr.y (mgr.14901) 4144 : cluster [DBG] pgmap v4151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:43.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:43 smithi130 bash[51302]: debug 2023-08-28T00:16:43.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:43.337483+0000) 2023-08-28T00:16:44.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:44 smithi006 bash[33918]: audit 2023-08-28T00:16:44.213262+0000 mon.c (mon.2) 1575 : audit [DBG] from='client.? 172.21.15.6:0/767127700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:44.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:44 smithi006 bash[27292]: audit 2023-08-28T00:16:44.213262+0000 mon.c (mon.2) 1575 : audit [DBG] from='client.? 172.21.15.6:0/767127700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:16:44.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:16:44.156Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:44 smithi130 bash[27579]: audit 2023-08-28T00:16:44.213262+0000 mon.c (mon.2) 1575 : audit [DBG] from='client.? 172.21.15.6:0/767127700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:44.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:44 smithi130 bash[51302]: debug 2023-08-28T00:16:44.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:44.337778+0000) 2023-08-28T00:16:45.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:45 smithi130 bash[27579]: cluster 2023-08-28T00:16:44.327121+0000 mgr.y (mgr.14901) 4145 : cluster [DBG] pgmap v4152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:45.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:45 smithi130 bash[51302]: debug 2023-08-28T00:16:45.336+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:45.338068+0000) 2023-08-28T00:16:45.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:45 smithi006 bash[27292]: cluster 2023-08-28T00:16:44.327121+0000 mgr.y (mgr.14901) 4145 : cluster [DBG] pgmap v4152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:45.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:45 smithi006 bash[33918]: cluster 2023-08-28T00:16:44.327121+0000 mgr.y (mgr.14901) 4145 : cluster [DBG] pgmap v4152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:46.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:46 smithi130 bash[51302]: debug 2023-08-28T00:16:46.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:46.338348+0000) 2023-08-28T00:16:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:46 smithi130 bash[27579]: cluster 2023-08-28T00:16:46.327776+0000 mgr.y (mgr.14901) 4146 : cluster [DBG] pgmap v4153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:46 smithi006 bash[27292]: cluster 2023-08-28T00:16:46.327776+0000 mgr.y (mgr.14901) 4146 : cluster [DBG] pgmap v4153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:46 smithi006 bash[33918]: cluster 2023-08-28T00:16:46.327776+0000 mgr.y (mgr.14901) 4146 : cluster [DBG] pgmap v4153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:47.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:47 smithi130 bash[51302]: debug 2023-08-28T00:16:47.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:47.338665+0000) 2023-08-28T00:16:47.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:47 smithi006 bash[27292]: audit 2023-08-28T00:16:46.632574+0000 mon.a (mon.0) 3104 : audit [DBG] from='client.? 172.21.15.6:0/3424310451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:47.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:47 smithi006 bash[33918]: audit 2023-08-28T00:16:46.632574+0000 mon.a (mon.0) 3104 : audit [DBG] from='client.? 172.21.15.6:0/3424310451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:48.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:47 smithi130 bash[27579]: audit 2023-08-28T00:16:46.632574+0000 mon.a (mon.0) 3104 : audit [DBG] from='client.? 172.21.15.6:0/3424310451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:48.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:48 smithi130 bash[51302]: debug 2023-08-28T00:16:48.336+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:48.338977+0000) 2023-08-28T00:16:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:48 smithi006 bash[33918]: cluster 2023-08-28T00:16:48.328466+0000 mgr.y (mgr.14901) 4147 : cluster [DBG] pgmap v4154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:48 smithi006 bash[27292]: cluster 2023-08-28T00:16:48.328466+0000 mgr.y (mgr.14901) 4147 : cluster [DBG] pgmap v4154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:48 smithi130 bash[27579]: cluster 2023-08-28T00:16:48.328466+0000 mgr.y (mgr.14901) 4147 : cluster [DBG] pgmap v4154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:49.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:49 smithi130 bash[51302]: debug 2023-08-28T00:16:49.336+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:49.339212+0000) 2023-08-28T00:16:49.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:49 smithi006 bash[27292]: audit 2023-08-28T00:16:49.047777+0000 mon.c (mon.2) 1576 : audit [DBG] from='client.? 172.21.15.6:0/2510031854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:49.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:49 smithi006 bash[33918]: audit 2023-08-28T00:16:49.047777+0000 mon.c (mon.2) 1576 : audit [DBG] from='client.? 172.21.15.6:0/2510031854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:50.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:49 smithi130 bash[27579]: audit 2023-08-28T00:16:49.047777+0000 mon.c (mon.2) 1576 : audit [DBG] from='client.? 172.21.15.6:0/2510031854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:50.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:50 smithi130 bash[51302]: debug 2023-08-28T00:16:50.336+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:50.339514+0000) 2023-08-28T00:16:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:16:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:16:50] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T00:16:50.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:50 smithi006 bash[27292]: cluster 2023-08-28T00:16:50.329873+0000 mgr.y (mgr.14901) 4148 : cluster [DBG] pgmap v4155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:50.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:50 smithi006 bash[33918]: cluster 2023-08-28T00:16:50.329873+0000 mgr.y (mgr.14901) 4148 : cluster [DBG] pgmap v4155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:16:50.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:16:50.692Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:50 smithi130 bash[27579]: cluster 2023-08-28T00:16:50.329873+0000 mgr.y (mgr.14901) 4148 : cluster [DBG] pgmap v4155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:51.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:51 smithi130 bash[51302]: debug 2023-08-28T00:16:51.336+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:51.339808+0000) 2023-08-28T00:16:51.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:51 smithi006 bash[33918]: audit 2023-08-28T00:16:51.463620+0000 mon.c (mon.2) 1577 : audit [DBG] from='client.? 172.21.15.6:0/2303022716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:51.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:51 smithi006 bash[27292]: audit 2023-08-28T00:16:51.463620+0000 mon.c (mon.2) 1577 : audit [DBG] from='client.? 172.21.15.6:0/2303022716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:52.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:51 smithi130 bash[27579]: audit 2023-08-28T00:16:51.463620+0000 mon.c (mon.2) 1577 : audit [DBG] from='client.? 172.21.15.6:0/2303022716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:52.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:52 smithi130 bash[51302]: debug 2023-08-28T00:16:52.336+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:52.340049+0000) 2023-08-28T00:16:52.651 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:16:52.270Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:52.651 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:16:52.272Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:52 smithi006 bash[33918]: cluster 2023-08-28T00:16:52.330603+0000 mgr.y (mgr.14901) 4149 : cluster [DBG] pgmap v4156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:52 smithi006 bash[27292]: cluster 2023-08-28T00:16:52.330603+0000 mgr.y (mgr.14901) 4149 : cluster [DBG] pgmap v4156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:52 smithi130 bash[27579]: cluster 2023-08-28T00:16:52.330603+0000 mgr.y (mgr.14901) 4149 : cluster [DBG] pgmap v4156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:53.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:53 smithi130 bash[51302]: debug 2023-08-28T00:16:53.336+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:53.340314+0000) 2023-08-28T00:16:54.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:53 smithi006 bash[33918]: audit 2023-08-28T00:16:53.882453+0000 mon.a (mon.0) 3105 : audit [DBG] from='client.? 172.21.15.6:0/537818392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:54.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:53 smithi006 bash[27292]: audit 2023-08-28T00:16:53.882453+0000 mon.a (mon.0) 3105 : audit [DBG] from='client.? 172.21.15.6:0/537818392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:16:54.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:16:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:16:54.155Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:16:54.342 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:53 smithi130 bash[27579]: audit 2023-08-28T00:16:53.882453+0000 mon.a (mon.0) 3105 : audit [DBG] from='client.? 172.21.15.6:0/537818392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:54.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:54 smithi130 bash[51302]: debug 2023-08-28T00:16:54.336+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:54.340595+0000) 2023-08-28T00:16:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:54 smithi006 bash[33918]: cluster 2023-08-28T00:16:54.331773+0000 mgr.y (mgr.14901) 4150 : cluster [DBG] pgmap v4157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:54 smithi006 bash[27292]: cluster 2023-08-28T00:16:54.331773+0000 mgr.y (mgr.14901) 4150 : cluster [DBG] pgmap v4157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:55.342 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:54 smithi130 bash[27579]: cluster 2023-08-28T00:16:54.331773+0000 mgr.y (mgr.14901) 4150 : cluster [DBG] pgmap v4157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:55.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:55 smithi130 bash[51302]: debug 2023-08-28T00:16:55.336+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:55.340870+0000) 2023-08-28T00:16:56.610 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:56 smithi130 bash[51302]: debug 2023-08-28T00:16:56.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:56.341169+0000) 2023-08-28T00:16:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:56 smithi130 bash[27579]: audit 2023-08-28T00:16:56.300747+0000 mon.a (mon.0) 3106 : audit [DBG] from='client.? 172.21.15.6:0/1029074877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:56 smithi130 bash[27579]: cluster 2023-08-28T00:16:56.332431+0000 mgr.y (mgr.14901) 4151 : cluster [DBG] pgmap v4158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:56 smithi006 bash[33918]: audit 2023-08-28T00:16:56.300747+0000 mon.a (mon.0) 3106 : audit [DBG] from='client.? 172.21.15.6:0/1029074877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:56 smithi006 bash[33918]: cluster 2023-08-28T00:16:56.332431+0000 mgr.y (mgr.14901) 4151 : cluster [DBG] pgmap v4158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:56 smithi006 bash[27292]: audit 2023-08-28T00:16:56.300747+0000 mon.a (mon.0) 3106 : audit [DBG] from='client.? 172.21.15.6:0/1029074877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:56 smithi006 bash[27292]: cluster 2023-08-28T00:16:56.332431+0000 mgr.y (mgr.14901) 4151 : cluster [DBG] pgmap v4158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:57.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:57 smithi130 bash[51302]: debug 2023-08-28T00:16:57.336+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:57.341427+0000) 2023-08-28T00:16:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:58 smithi130 bash[27579]: cluster 2023-08-28T00:16:58.333256+0000 mgr.y (mgr.14901) 4152 : cluster [DBG] pgmap v4159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:58.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:58 smithi130 bash[51302]: debug 2023-08-28T00:16:58.336+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:58.341667+0000) 2023-08-28T00:16:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:58 smithi006 bash[27292]: cluster 2023-08-28T00:16:58.333256+0000 mgr.y (mgr.14901) 4152 : cluster [DBG] pgmap v4159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:58 smithi006 bash[33918]: cluster 2023-08-28T00:16:58.333256+0000 mgr.y (mgr.14901) 4152 : cluster [DBG] pgmap v4159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:16:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:16:59 smithi130 bash[27579]: audit 2023-08-28T00:16:58.717602+0000 mon.a (mon.0) 3107 : audit [DBG] from='client.? 172.21.15.6:0/4250296701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:59.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:16:59 smithi130 bash[51302]: debug 2023-08-28T00:16:59.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:59.342018+0000) 2023-08-28T00:16:59.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:16:59 smithi006 bash[27292]: audit 2023-08-28T00:16:58.717602+0000 mon.a (mon.0) 3107 : audit [DBG] from='client.? 172.21.15.6:0/4250296701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:59.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:16:59 smithi006 bash[33918]: audit 2023-08-28T00:16:58.717602+0000 mon.a (mon.0) 3107 : audit [DBG] from='client.? 172.21.15.6:0/4250296701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:00 smithi130 bash[27579]: cluster 2023-08-28T00:17:00.334669+0000 mgr.y (mgr.14901) 4153 : cluster [DBG] pgmap v4160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:00.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:00 smithi130 bash[51302]: debug 2023-08-28T00:17:00.336+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:00.342303+0000) 2023-08-28T00:17:00.694 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:00 smithi006 bash[27292]: cluster 2023-08-28T00:17:00.334669+0000 mgr.y (mgr.14901) 4153 : cluster [DBG] pgmap v4160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:00.694 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:00 smithi006 bash[33918]: cluster 2023-08-28T00:17:00.334669+0000 mgr.y (mgr.14901) 4153 : cluster [DBG] pgmap v4160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:17:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:17:00] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T00:17:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:17:00.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:17:00.694Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:01.611 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:01 smithi130 bash[51302]: debug 2023-08-28T00:17:01.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:01.342598+0000) 2023-08-28T00:17:01.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:01 smithi130 bash[27579]: audit 2023-08-28T00:17:01.135607+0000 mon.c (mon.2) 1578 : audit [DBG] from='client.? 172.21.15.6:0/1547281665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:01.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:01 smithi006 bash[27292]: audit 2023-08-28T00:17:01.135607+0000 mon.c (mon.2) 1578 : audit [DBG] from='client.? 172.21.15.6:0/1547281665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:01.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:01 smithi006 bash[33918]: audit 2023-08-28T00:17:01.135607+0000 mon.c (mon.2) 1578 : audit [DBG] from='client.? 172.21.15.6:0/1547281665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:02.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:02 smithi130 bash[51302]: debug 2023-08-28T00:17:02.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:02.342840+0000) 2023-08-28T00:17:02.619 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:17:02.271Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:02.619 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:17:02.274Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:02 smithi006 bash[27292]: cluster 2023-08-28T00:17:02.335340+0000 mgr.y (mgr.14901) 4154 : cluster [DBG] pgmap v4161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:02 smithi006 bash[33918]: cluster 2023-08-28T00:17:02.335340+0000 mgr.y (mgr.14901) 4154 : cluster [DBG] pgmap v4161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:02 smithi130 bash[27579]: cluster 2023-08-28T00:17:02.335340+0000 mgr.y (mgr.14901) 4154 : cluster [DBG] pgmap v4161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:03.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:03 smithi130 bash[51302]: debug 2023-08-28T00:17:03.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:03.343127+0000) 2023-08-28T00:17:03.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:03 smithi006 bash[27292]: audit 2023-08-28T00:17:03.550164+0000 mon.c (mon.2) 1579 : audit [DBG] from='client.? 172.21.15.6:0/2990745726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:03.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:03 smithi006 bash[33918]: audit 2023-08-28T00:17:03.550164+0000 mon.c (mon.2) 1579 : audit [DBG] from='client.? 172.21.15.6:0/2990745726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:04.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:03 smithi130 bash[27579]: audit 2023-08-28T00:17:03.550164+0000 mon.c (mon.2) 1579 : audit [DBG] from='client.? 172.21.15.6:0/2990745726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:17:04.154Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:17:04.157Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:04.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:04 smithi130 bash[51302]: debug 2023-08-28T00:17:04.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:04.343351+0000) 2023-08-28T00:17:04.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:04 smithi006 bash[33918]: cluster 2023-08-28T00:17:04.336444+0000 mgr.y (mgr.14901) 4155 : cluster [DBG] pgmap v4162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:04.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:04 smithi006 bash[27292]: cluster 2023-08-28T00:17:04.336444+0000 mgr.y (mgr.14901) 4155 : cluster [DBG] pgmap v4162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:04 smithi130 bash[27579]: cluster 2023-08-28T00:17:04.336444+0000 mgr.y (mgr.14901) 4155 : cluster [DBG] pgmap v4162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:05.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:05 smithi130 bash[51302]: debug 2023-08-28T00:17:05.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:05.343641+0000) 2023-08-28T00:17:06.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:06 smithi130 bash[51302]: debug 2023-08-28T00:17:06.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:06.343895+0000) 2023-08-28T00:17:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:06 smithi006 bash[33918]: audit 2023-08-28T00:17:05.966644+0000 mon.a (mon.0) 3108 : audit [DBG] from='client.? 172.21.15.6:0/3022406367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:06 smithi006 bash[33918]: cluster 2023-08-28T00:17:06.337129+0000 mgr.y (mgr.14901) 4156 : cluster [DBG] pgmap v4163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:06 smithi006 bash[27292]: audit 2023-08-28T00:17:05.966644+0000 mon.a (mon.0) 3108 : audit [DBG] from='client.? 172.21.15.6:0/3022406367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:06 smithi006 bash[27292]: cluster 2023-08-28T00:17:06.337129+0000 mgr.y (mgr.14901) 4156 : cluster [DBG] pgmap v4163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:06 smithi130 bash[27579]: audit 2023-08-28T00:17:05.966644+0000 mon.a (mon.0) 3108 : audit [DBG] from='client.? 172.21.15.6:0/3022406367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:06 smithi130 bash[27579]: cluster 2023-08-28T00:17:06.337129+0000 mgr.y (mgr.14901) 4156 : cluster [DBG] pgmap v4163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:07.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:07 smithi130 bash[51302]: debug 2023-08-28T00:17:07.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:07.344185+0000) 2023-08-28T00:17:08.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:08 smithi130 bash[51302]: debug 2023-08-28T00:17:08.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:08.344457+0000) 2023-08-28T00:17:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:08 smithi130 bash[27579]: cluster 2023-08-28T00:17:08.337726+0000 mgr.y (mgr.14901) 4157 : cluster [DBG] pgmap v4164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:08.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:08 smithi130 bash[27579]: audit 2023-08-28T00:17:08.377467+0000 mon.a (mon.0) 3109 : audit [DBG] from='client.? 172.21.15.6:0/1814021628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:08 smithi006 bash[27292]: cluster 2023-08-28T00:17:08.337726+0000 mgr.y (mgr.14901) 4157 : cluster [DBG] pgmap v4164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:08 smithi006 bash[27292]: audit 2023-08-28T00:17:08.377467+0000 mon.a (mon.0) 3109 : audit [DBG] from='client.? 172.21.15.6:0/1814021628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:08 smithi006 bash[33918]: cluster 2023-08-28T00:17:08.337726+0000 mgr.y (mgr.14901) 4157 : cluster [DBG] pgmap v4164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:08 smithi006 bash[33918]: audit 2023-08-28T00:17:08.377467+0000 mon.a (mon.0) 3109 : audit [DBG] from='client.? 172.21.15.6:0/1814021628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:09.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:09 smithi130 bash[51302]: debug 2023-08-28T00:17:09.344+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:09.344788+0000) 2023-08-28T00:17:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:10 smithi130 bash[27579]: cluster 2023-08-28T00:17:10.338492+0000 mgr.y (mgr.14901) 4158 : cluster [DBG] pgmap v4165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:10.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:10 smithi130 bash[51302]: debug 2023-08-28T00:17:10.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:10.345050+0000) 2023-08-28T00:17:10.694 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:10 smithi006 bash[33918]: cluster 2023-08-28T00:17:10.338492+0000 mgr.y (mgr.14901) 4158 : cluster [DBG] pgmap v4165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:10.695 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:10 smithi006 bash[27292]: cluster 2023-08-28T00:17:10.338492+0000 mgr.y (mgr.14901) 4158 : cluster [DBG] pgmap v4165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:17:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:17:10] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T00:17:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:17:10.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:17:10.693Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:11.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:11 smithi130 bash[51302]: debug 2023-08-28T00:17:11.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:11.345334+0000) 2023-08-28T00:17:11.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:11 smithi130 bash[27579]: audit 2023-08-28T00:17:10.800849+0000 mon.c (mon.2) 1580 : audit [DBG] from='client.? 172.21.15.6:0/777642475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:11.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:11 smithi006 bash[33918]: audit 2023-08-28T00:17:10.800849+0000 mon.c (mon.2) 1580 : audit [DBG] from='client.? 172.21.15.6:0/777642475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:11.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:11 smithi006 bash[27292]: audit 2023-08-28T00:17:10.800849+0000 mon.c (mon.2) 1580 : audit [DBG] from='client.? 172.21.15.6:0/777642475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:12.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:12 smithi130 bash[51302]: debug 2023-08-28T00:17:12.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:12.345596+0000) 2023-08-28T00:17:12.628 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:17:12.272Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:12.628 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:17:12.274Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:12 smithi006 bash[33918]: cluster 2023-08-28T00:17:12.339136+0000 mgr.y (mgr.14901) 4159 : cluster [DBG] pgmap v4166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:12 smithi006 bash[27292]: cluster 2023-08-28T00:17:12.339136+0000 mgr.y (mgr.14901) 4159 : cluster [DBG] pgmap v4166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:12 smithi130 bash[27579]: cluster 2023-08-28T00:17:12.339136+0000 mgr.y (mgr.14901) 4159 : cluster [DBG] pgmap v4166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:13.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:13 smithi130 bash[51302]: debug 2023-08-28T00:17:13.344+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:13.345891+0000) 2023-08-28T00:17:13.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:13 smithi006 bash[27292]: audit 2023-08-28T00:17:13.219758+0000 mon.a (mon.0) 3110 : audit [DBG] from='client.? 172.21.15.6:0/66562878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:13.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:13 smithi006 bash[33918]: audit 2023-08-28T00:17:13.219758+0000 mon.a (mon.0) 3110 : audit [DBG] from='client.? 172.21.15.6:0/66562878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:14.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:13 smithi130 bash[27579]: audit 2023-08-28T00:17:13.219758+0000 mon.a (mon.0) 3110 : audit [DBG] from='client.? 172.21.15.6:0/66562878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:17:14.156Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:17:14.158Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:14.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:14 smithi130 bash[51302]: debug 2023-08-28T00:17:14.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:14.346155+0000) 2023-08-28T00:17:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:14 smithi006 bash[33918]: cluster 2023-08-28T00:17:14.340284+0000 mgr.y (mgr.14901) 4160 : cluster [DBG] pgmap v4167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:14 smithi006 bash[27292]: cluster 2023-08-28T00:17:14.340284+0000 mgr.y (mgr.14901) 4160 : cluster [DBG] pgmap v4167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:14 smithi130 bash[27579]: cluster 2023-08-28T00:17:14.340284+0000 mgr.y (mgr.14901) 4160 : cluster [DBG] pgmap v4167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:15.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:15 smithi130 bash[51302]: debug 2023-08-28T00:17:15.344+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:15.346450+0000) 2023-08-28T00:17:16.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:16 smithi130 bash[51302]: debug 2023-08-28T00:17:16.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:16.346780+0000) 2023-08-28T00:17:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:16 smithi006 bash[33918]: audit 2023-08-28T00:17:15.632927+0000 mon.a (mon.0) 3111 : audit [DBG] from='client.? 172.21.15.6:0/852450723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:16 smithi006 bash[33918]: cluster 2023-08-28T00:17:16.340998+0000 mgr.y (mgr.14901) 4161 : cluster [DBG] pgmap v4168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:16 smithi006 bash[27292]: audit 2023-08-28T00:17:15.632927+0000 mon.a (mon.0) 3111 : audit [DBG] from='client.? 172.21.15.6:0/852450723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:16 smithi006 bash[27292]: cluster 2023-08-28T00:17:16.340998+0000 mgr.y (mgr.14901) 4161 : cluster [DBG] pgmap v4168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:17.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:16 smithi130 bash[27579]: audit 2023-08-28T00:17:15.632927+0000 mon.a (mon.0) 3111 : audit [DBG] from='client.? 172.21.15.6:0/852450723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:17.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:16 smithi130 bash[27579]: cluster 2023-08-28T00:17:16.340998+0000 mgr.y (mgr.14901) 4161 : cluster [DBG] pgmap v4168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:17.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:17 smithi130 bash[51302]: debug 2023-08-28T00:17:17.344+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:17.347039+0000) 2023-08-28T00:17:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:18 smithi130 bash[27579]: audit 2023-08-28T00:17:18.045964+0000 mon.a (mon.0) 3112 : audit [DBG] from='client.? 172.21.15.6:0/451583657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:18.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:18 smithi130 bash[51302]: debug 2023-08-28T00:17:18.344+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:18.347265+0000) 2023-08-28T00:17:18.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:18 smithi006 bash[27292]: audit 2023-08-28T00:17:18.045964+0000 mon.a (mon.0) 3112 : audit [DBG] from='client.? 172.21.15.6:0/451583657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:18.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:18 smithi006 bash[33918]: audit 2023-08-28T00:17:18.045964+0000 mon.a (mon.0) 3112 : audit [DBG] from='client.? 172.21.15.6:0/451583657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:19.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:19 smithi130 bash[27579]: cluster 2023-08-28T00:17:18.341599+0000 mgr.y (mgr.14901) 4162 : cluster [DBG] pgmap v4169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:19.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:19 smithi130 bash[51302]: debug 2023-08-28T00:17:19.344+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:19.347582+0000) 2023-08-28T00:17:19.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:19 smithi006 bash[27292]: cluster 2023-08-28T00:17:18.341599+0000 mgr.y (mgr.14901) 4162 : cluster [DBG] pgmap v4169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:19.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:19 smithi006 bash[33918]: cluster 2023-08-28T00:17:18.341599+0000 mgr.y (mgr.14901) 4162 : cluster [DBG] pgmap v4169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:20 smithi130 bash[27579]: cluster 2023-08-28T00:17:20.342297+0000 mgr.y (mgr.14901) 4163 : cluster [DBG] pgmap v4170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:20.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:20 smithi130 bash[51302]: debug 2023-08-28T00:17:20.344+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:20.347845+0000) 2023-08-28T00:17:20.695 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:20 smithi006 bash[33918]: cluster 2023-08-28T00:17:20.342297+0000 mgr.y (mgr.14901) 4163 : cluster [DBG] pgmap v4170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:20.696 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:20 smithi006 bash[27292]: cluster 2023-08-28T00:17:20.342297+0000 mgr.y (mgr.14901) 4163 : cluster [DBG] pgmap v4170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:17:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:17:20] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:17:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:17:20.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:17:20.695Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:21 smithi130 bash[27579]: audit 2023-08-28T00:17:20.465172+0000 mon.c (mon.2) 1581 : audit [DBG] from='client.? 172.21.15.6:0/1813959958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:21.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:21 smithi130 bash[51302]: debug 2023-08-28T00:17:21.344+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:21.348124+0000) 2023-08-28T00:17:21.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:21 smithi006 bash[33918]: audit 2023-08-28T00:17:20.465172+0000 mon.c (mon.2) 1581 : audit [DBG] from='client.? 172.21.15.6:0/1813959958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:21.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:21 smithi006 bash[27292]: audit 2023-08-28T00:17:20.465172+0000 mon.c (mon.2) 1581 : audit [DBG] from='client.? 172.21.15.6:0/1813959958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:22 smithi130 bash[27579]: cluster 2023-08-28T00:17:22.343012+0000 mgr.y (mgr.14901) 4164 : cluster [DBG] pgmap v4171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:22.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:22 smithi130 bash[51302]: debug 2023-08-28T00:17:22.344+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:22.348449+0000) 2023-08-28T00:17:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:22 smithi006 bash[27292]: cluster 2023-08-28T00:17:22.343012+0000 mgr.y (mgr.14901) 4164 : cluster [DBG] pgmap v4171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:22 smithi006 bash[33918]: cluster 2023-08-28T00:17:22.343012+0000 mgr.y (mgr.14901) 4164 : cluster [DBG] pgmap v4171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:17:22.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:17:22.274Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:23.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:23 smithi130 bash[27579]: audit 2023-08-28T00:17:22.882079+0000 mon.c (mon.2) 1582 : audit [DBG] from='client.? 172.21.15.6:0/582137375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:23.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:23 smithi130 bash[51302]: debug 2023-08-28T00:17:23.344+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:23.348702+0000) 2023-08-28T00:17:23.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:23 smithi006 bash[27292]: audit 2023-08-28T00:17:22.882079+0000 mon.c (mon.2) 1582 : audit [DBG] from='client.? 172.21.15.6:0/582137375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:23.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:23 smithi006 bash[33918]: audit 2023-08-28T00:17:22.882079+0000 mon.c (mon.2) 1582 : audit [DBG] from='client.? 172.21.15.6:0/582137375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:24.433 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:17:24.156Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:24.433 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:17:24.158Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:24.434 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:24 smithi006 bash[27292]: cluster 2023-08-28T00:17:24.343976+0000 mgr.y (mgr.14901) 4165 : cluster [DBG] pgmap v4172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:24 smithi130 bash[27579]: cluster 2023-08-28T00:17:24.343976+0000 mgr.y (mgr.14901) 4165 : cluster [DBG] pgmap v4172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:24.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:24 smithi130 bash[51302]: debug 2023-08-28T00:17:24.348+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:24.349006+0000) 2023-08-28T00:17:24.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:24 smithi006 bash[33918]: cluster 2023-08-28T00:17:24.343976+0000 mgr.y (mgr.14901) 4165 : cluster [DBG] pgmap v4172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:25.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:25 smithi130 bash[27579]: audit 2023-08-28T00:17:25.299076+0000 mon.c (mon.2) 1583 : audit [DBG] from='client.? 172.21.15.6:0/3225544565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:25.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:25 smithi130 bash[51302]: debug 2023-08-28T00:17:25.348+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:25.349292+0000) 2023-08-28T00:17:25.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:25 smithi006 bash[33918]: audit 2023-08-28T00:17:25.299076+0000 mon.c (mon.2) 1583 : audit [DBG] from='client.? 172.21.15.6:0/3225544565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:25.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:25 smithi006 bash[27292]: audit 2023-08-28T00:17:25.299076+0000 mon.c (mon.2) 1583 : audit [DBG] from='client.? 172.21.15.6:0/3225544565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:26.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:26 smithi130 bash[51302]: debug 2023-08-28T00:17:26.344+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:26.349585+0000) 2023-08-28T00:17:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:26 smithi006 bash[27292]: audit 2023-08-28T00:17:25.657560+0000 mon.b (mon.1) 522 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:17:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:26 smithi006 bash[27292]: cluster 2023-08-28T00:17:26.344608+0000 mgr.y (mgr.14901) 4166 : cluster [DBG] pgmap v4173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:26 smithi006 bash[33918]: audit 2023-08-28T00:17:25.657560+0000 mon.b (mon.1) 522 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:17:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:26 smithi006 bash[33918]: cluster 2023-08-28T00:17:26.344608+0000 mgr.y (mgr.14901) 4166 : cluster [DBG] pgmap v4173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:26 smithi130 bash[27579]: audit 2023-08-28T00:17:25.657560+0000 mon.b (mon.1) 522 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:17:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:26 smithi130 bash[27579]: cluster 2023-08-28T00:17:26.344608+0000 mgr.y (mgr.14901) 4166 : cluster [DBG] pgmap v4173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:27.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:27 smithi130 bash[51302]: debug 2023-08-28T00:17:27.344+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:27.349892+0000) 2023-08-28T00:17:28.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:27 smithi130 bash[27579]: audit 2023-08-28T00:17:27.709828+0000 mon.a (mon.0) 3113 : audit [DBG] from='client.? 172.21.15.6:0/3794196914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:28.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:27 smithi006 bash[33918]: audit 2023-08-28T00:17:27.709828+0000 mon.a (mon.0) 3113 : audit [DBG] from='client.? 172.21.15.6:0/3794196914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:28.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:27 smithi006 bash[27292]: audit 2023-08-28T00:17:27.709828+0000 mon.a (mon.0) 3113 : audit [DBG] from='client.? 172.21.15.6:0/3794196914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:28.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:28 smithi130 bash[51302]: debug 2023-08-28T00:17:28.344+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:28.350203+0000) 2023-08-28T00:17:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:28 smithi130 bash[27579]: cluster 2023-08-28T00:17:28.345329+0000 mgr.y (mgr.14901) 4167 : cluster [DBG] pgmap v4174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:28 smithi006 bash[27292]: cluster 2023-08-28T00:17:28.345329+0000 mgr.y (mgr.14901) 4167 : cluster [DBG] pgmap v4174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:28 smithi006 bash[33918]: cluster 2023-08-28T00:17:28.345329+0000 mgr.y (mgr.14901) 4167 : cluster [DBG] pgmap v4174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:29.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:29 smithi130 bash[51302]: debug 2023-08-28T00:17:29.344+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:29.350434+0000) 2023-08-28T00:17:30.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:30 smithi006 bash[27292]: audit 2023-08-28T00:17:30.138576+0000 mon.c (mon.2) 1584 : audit [DBG] from='client.? 172.21.15.6:0/928152236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:30.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:30 smithi006 bash[33918]: audit 2023-08-28T00:17:30.138576+0000 mon.c (mon.2) 1584 : audit [DBG] from='client.? 172.21.15.6:0/928152236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:30 smithi130 bash[27579]: audit 2023-08-28T00:17:30.138576+0000 mon.c (mon.2) 1584 : audit [DBG] from='client.? 172.21.15.6:0/928152236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:30.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:30 smithi130 bash[51302]: debug 2023-08-28T00:17:30.343+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:30.350683+0000) 2023-08-28T00:17:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:17:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:17:30] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:17:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:17:30.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:17:30.695Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:31.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:31 smithi006 bash[27292]: cluster 2023-08-28T00:17:30.346097+0000 mgr.y (mgr.14901) 4168 : cluster [DBG] pgmap v4175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:31.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:31 smithi006 bash[33918]: cluster 2023-08-28T00:17:30.346097+0000 mgr.y (mgr.14901) 4168 : cluster [DBG] pgmap v4175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:31.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:31 smithi130 bash[27579]: cluster 2023-08-28T00:17:30.346097+0000 mgr.y (mgr.14901) 4168 : cluster [DBG] pgmap v4175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:31.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:31 smithi130 bash[51302]: debug 2023-08-28T00:17:31.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:31.350942+0000) 2023-08-28T00:17:32.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:32 smithi130 bash[51302]: debug 2023-08-28T00:17:32.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:32.351179+0000) 2023-08-28T00:17:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:32 smithi130 bash[27579]: audit 2023-08-28T00:17:31.493153+0000 mon.a (mon.0) 3114 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:17:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:32 smithi130 bash[27579]: audit 2023-08-28T00:17:32.046848+0000 mon.a (mon.0) 3115 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:17:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:32 smithi130 bash[27579]: audit 2023-08-28T00:17:32.317771+0000 mon.b (mon.1) 523 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:17:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:32 smithi130 bash[27579]: audit 2023-08-28T00:17:32.319727+0000 mon.b (mon.1) 524 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:17:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:32 smithi130 bash[27579]: audit 2023-08-28T00:17:32.330347+0000 mon.a (mon.0) 3116 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:17:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:32 smithi130 bash[27579]: cluster 2023-08-28T00:17:32.346652+0000 mgr.y (mgr.14901) 4169 : cluster [DBG] pgmap v4176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:17:32.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:17:32.275Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:32 smithi006 bash[33918]: audit 2023-08-28T00:17:31.493153+0000 mon.a (mon.0) 3114 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:17:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:32 smithi006 bash[33918]: audit 2023-08-28T00:17:32.046848+0000 mon.a (mon.0) 3115 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:17:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:32 smithi006 bash[33918]: audit 2023-08-28T00:17:32.317771+0000 mon.b (mon.1) 523 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:17:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:32 smithi006 bash[33918]: audit 2023-08-28T00:17:32.319727+0000 mon.b (mon.1) 524 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:17:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:32 smithi006 bash[33918]: audit 2023-08-28T00:17:32.330347+0000 mon.a (mon.0) 3116 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:17:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:32 smithi006 bash[33918]: cluster 2023-08-28T00:17:32.346652+0000 mgr.y (mgr.14901) 4169 : cluster [DBG] pgmap v4176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:32 smithi006 bash[27292]: audit 2023-08-28T00:17:31.493153+0000 mon.a (mon.0) 3114 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:17:32.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:32 smithi006 bash[27292]: audit 2023-08-28T00:17:32.046848+0000 mon.a (mon.0) 3115 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:17:32.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:32 smithi006 bash[27292]: audit 2023-08-28T00:17:32.317771+0000 mon.b (mon.1) 523 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:17:32.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:32 smithi006 bash[27292]: audit 2023-08-28T00:17:32.319727+0000 mon.b (mon.1) 524 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:17:32.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:32 smithi006 bash[27292]: audit 2023-08-28T00:17:32.330347+0000 mon.a (mon.0) 3116 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:17:32.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:32 smithi006 bash[27292]: cluster 2023-08-28T00:17:32.346652+0000 mgr.y (mgr.14901) 4169 : cluster [DBG] pgmap v4176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:33 smithi130 bash[27579]: audit 2023-08-28T00:17:32.558177+0000 mon.a (mon.0) 3117 : audit [DBG] from='client.? 172.21.15.6:0/1244302607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:33.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:33 smithi130 bash[51302]: debug 2023-08-28T00:17:33.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:33.351447+0000) 2023-08-28T00:17:33.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:33 smithi006 bash[27292]: audit 2023-08-28T00:17:32.558177+0000 mon.a (mon.0) 3117 : audit [DBG] from='client.? 172.21.15.6:0/1244302607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:33.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:33 smithi006 bash[33918]: audit 2023-08-28T00:17:32.558177+0000 mon.a (mon.0) 3117 : audit [DBG] from='client.? 172.21.15.6:0/1244302607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:17:34.157Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:17:34.159Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:34 smithi130 bash[27579]: cluster 2023-08-28T00:17:34.347795+0000 mgr.y (mgr.14901) 4170 : cluster [DBG] pgmap v4177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:34.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:34 smithi130 bash[51302]: debug 2023-08-28T00:17:34.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:34.351711+0000) 2023-08-28T00:17:34.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:34 smithi006 bash[27292]: cluster 2023-08-28T00:17:34.347795+0000 mgr.y (mgr.14901) 4170 : cluster [DBG] pgmap v4177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:34.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:34 smithi006 bash[33918]: cluster 2023-08-28T00:17:34.347795+0000 mgr.y (mgr.14901) 4170 : cluster [DBG] pgmap v4177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:35.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:35 smithi130 bash[27579]: audit 2023-08-28T00:17:34.976507+0000 mon.a (mon.0) 3118 : audit [DBG] from='client.? 172.21.15.6:0/2736063094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:35.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:35 smithi130 bash[51302]: debug 2023-08-28T00:17:35.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:35.352096+0000) 2023-08-28T00:17:35.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:35 smithi006 bash[27292]: audit 2023-08-28T00:17:34.976507+0000 mon.a (mon.0) 3118 : audit [DBG] from='client.? 172.21.15.6:0/2736063094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:35.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:35 smithi006 bash[33918]: audit 2023-08-28T00:17:34.976507+0000 mon.a (mon.0) 3118 : audit [DBG] from='client.? 172.21.15.6:0/2736063094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:36.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:36 smithi130 bash[51302]: debug 2023-08-28T00:17:36.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:36.352417+0000) 2023-08-28T00:17:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:36 smithi006 bash[33918]: cluster 2023-08-28T00:17:36.348450+0000 mgr.y (mgr.14901) 4171 : cluster [DBG] pgmap v4178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:36 smithi006 bash[27292]: cluster 2023-08-28T00:17:36.348450+0000 mgr.y (mgr.14901) 4171 : cluster [DBG] pgmap v4178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:37.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:36 smithi130 bash[27579]: cluster 2023-08-28T00:17:36.348450+0000 mgr.y (mgr.14901) 4171 : cluster [DBG] pgmap v4178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:37.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:37 smithi130 bash[51302]: debug 2023-08-28T00:17:37.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:37.352758+0000) 2023-08-28T00:17:37.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:37 smithi006 bash[33918]: audit 2023-08-28T00:17:37.393532+0000 mon.a (mon.0) 3119 : audit [DBG] from='client.? 172.21.15.6:0/2784915005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:37.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:37 smithi006 bash[27292]: audit 2023-08-28T00:17:37.393532+0000 mon.a (mon.0) 3119 : audit [DBG] from='client.? 172.21.15.6:0/2784915005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:38.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:37 smithi130 bash[27579]: audit 2023-08-28T00:17:37.393532+0000 mon.a (mon.0) 3119 : audit [DBG] from='client.? 172.21.15.6:0/2784915005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:38.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:38 smithi130 bash[51302]: debug 2023-08-28T00:17:38.351+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:38.356290+0000) 2023-08-28T00:17:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:38 smithi006 bash[27292]: cluster 2023-08-28T00:17:38.349143+0000 mgr.y (mgr.14901) 4172 : cluster [DBG] pgmap v4179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:38 smithi006 bash[33918]: cluster 2023-08-28T00:17:38.349143+0000 mgr.y (mgr.14901) 4172 : cluster [DBG] pgmap v4179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:38 smithi130 bash[27579]: cluster 2023-08-28T00:17:38.349143+0000 mgr.y (mgr.14901) 4172 : cluster [DBG] pgmap v4179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:39.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:39 smithi130 bash[51302]: debug 2023-08-28T00:17:39.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:39.356573+0000) 2023-08-28T00:17:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:39 smithi130 bash[27579]: audit 2023-08-28T00:17:39.806343+0000 mon.c (mon.2) 1585 : audit [DBG] from='client.? 172.21.15.6:0/2972980157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:40.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:39 smithi006 bash[27292]: audit 2023-08-28T00:17:39.806343+0000 mon.c (mon.2) 1585 : audit [DBG] from='client.? 172.21.15.6:0/2972980157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:40.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:39 smithi006 bash[33918]: audit 2023-08-28T00:17:39.806343+0000 mon.c (mon.2) 1585 : audit [DBG] from='client.? 172.21.15.6:0/2972980157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:40.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:40 smithi130 bash[51302]: debug 2023-08-28T00:17:40.351+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:40.356880+0000) 2023-08-28T00:17:40.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:40 smithi006 bash[27292]: cluster 2023-08-28T00:17:40.350096+0000 mgr.y (mgr.14901) 4173 : cluster [DBG] pgmap v4180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:40.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:40 smithi006 bash[33918]: cluster 2023-08-28T00:17:40.350096+0000 mgr.y (mgr.14901) 4173 : cluster [DBG] pgmap v4180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:40.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:17:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:17:40] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:17:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:17:40.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:17:40.696Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:41.358 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:40 smithi130 bash[27579]: cluster 2023-08-28T00:17:40.350096+0000 mgr.y (mgr.14901) 4173 : cluster [DBG] pgmap v4180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:41.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:41 smithi130 bash[51302]: debug 2023-08-28T00:17:41.351+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:41.357184+0000) 2023-08-28T00:17:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:42 smithi130 bash[27579]: audit 2023-08-28T00:17:42.275669+0000 mon.a (mon.0) 3120 : audit [DBG] from='client.? 172.21.15.6:0/2429390233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:42.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:42 smithi130 bash[51302]: debug 2023-08-28T00:17:42.351+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:42.357457+0000) 2023-08-28T00:17:42.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:42 smithi006 bash[33918]: audit 2023-08-28T00:17:42.275669+0000 mon.a (mon.0) 3120 : audit [DBG] from='client.? 172.21.15.6:0/2429390233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:42 smithi006 bash[27292]: audit 2023-08-28T00:17:42.275669+0000 mon.a (mon.0) 3120 : audit [DBG] from='client.? 172.21.15.6:0/2429390233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:17:42.274Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:17:42.276Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:43.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:43 smithi130 bash[27579]: cluster 2023-08-28T00:17:42.350869+0000 mgr.y (mgr.14901) 4174 : cluster [DBG] pgmap v4181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:43.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:43 smithi130 bash[51302]: debug 2023-08-28T00:17:43.351+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:43.357727+0000) 2023-08-28T00:17:43.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:43 smithi006 bash[27292]: cluster 2023-08-28T00:17:42.350869+0000 mgr.y (mgr.14901) 4174 : cluster [DBG] pgmap v4181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:43.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:43 smithi006 bash[33918]: cluster 2023-08-28T00:17:42.350869+0000 mgr.y (mgr.14901) 4174 : cluster [DBG] pgmap v4181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:44.417 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:17:44.157Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:44.417 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:17:44.160Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:44.418 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:44 smithi006 bash[27292]: cluster 2023-08-28T00:17:44.352025+0000 mgr.y (mgr.14901) 4175 : cluster [DBG] pgmap v4182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:44 smithi130 bash[27579]: cluster 2023-08-28T00:17:44.352025+0000 mgr.y (mgr.14901) 4175 : cluster [DBG] pgmap v4182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:44.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:44 smithi130 bash[51302]: debug 2023-08-28T00:17:44.351+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:44.358006+0000) 2023-08-28T00:17:44.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:44 smithi006 bash[33918]: cluster 2023-08-28T00:17:44.352025+0000 mgr.y (mgr.14901) 4175 : cluster [DBG] pgmap v4182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:45.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:45 smithi130 bash[27579]: audit 2023-08-28T00:17:44.688829+0000 mon.c (mon.2) 1586 : audit [DBG] from='client.? 172.21.15.6:0/4234948753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:45.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:45 smithi130 bash[51302]: debug 2023-08-28T00:17:45.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:45.358316+0000) 2023-08-28T00:17:45.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:45 smithi006 bash[27292]: audit 2023-08-28T00:17:44.688829+0000 mon.c (mon.2) 1586 : audit [DBG] from='client.? 172.21.15.6:0/4234948753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:45.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:45 smithi006 bash[33918]: audit 2023-08-28T00:17:44.688829+0000 mon.c (mon.2) 1586 : audit [DBG] from='client.? 172.21.15.6:0/4234948753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:46.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:46 smithi130 bash[51302]: debug 2023-08-28T00:17:46.351+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:46.358654+0000) 2023-08-28T00:17:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:46 smithi006 bash[27292]: cluster 2023-08-28T00:17:46.352666+0000 mgr.y (mgr.14901) 4176 : cluster [DBG] pgmap v4183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:46 smithi006 bash[33918]: cluster 2023-08-28T00:17:46.352666+0000 mgr.y (mgr.14901) 4176 : cluster [DBG] pgmap v4183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:47.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:46 smithi130 bash[27579]: cluster 2023-08-28T00:17:46.352666+0000 mgr.y (mgr.14901) 4176 : cluster [DBG] pgmap v4183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:47.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:47 smithi130 bash[51302]: debug 2023-08-28T00:17:47.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:47.358875+0000) 2023-08-28T00:17:47.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:47 smithi006 bash[33918]: audit 2023-08-28T00:17:47.098777+0000 mon.c (mon.2) 1587 : audit [DBG] from='client.? 172.21.15.6:0/1849869404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:47.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:47 smithi006 bash[27292]: audit 2023-08-28T00:17:47.098777+0000 mon.c (mon.2) 1587 : audit [DBG] from='client.? 172.21.15.6:0/1849869404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:48.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:47 smithi130 bash[27579]: audit 2023-08-28T00:17:47.098777+0000 mon.c (mon.2) 1587 : audit [DBG] from='client.? 172.21.15.6:0/1849869404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:48.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:48 smithi130 bash[51302]: debug 2023-08-28T00:17:48.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:48.359092+0000) 2023-08-28T00:17:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:48 smithi006 bash[27292]: cluster 2023-08-28T00:17:48.353451+0000 mgr.y (mgr.14901) 4177 : cluster [DBG] pgmap v4184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:48 smithi006 bash[33918]: cluster 2023-08-28T00:17:48.353451+0000 mgr.y (mgr.14901) 4177 : cluster [DBG] pgmap v4184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:48 smithi130 bash[27579]: cluster 2023-08-28T00:17:48.353451+0000 mgr.y (mgr.14901) 4177 : cluster [DBG] pgmap v4184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:49.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:49 smithi130 bash[51302]: debug 2023-08-28T00:17:49.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:49.359406+0000) 2023-08-28T00:17:49.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:49 smithi006 bash[27292]: audit 2023-08-28T00:17:49.512869+0000 mon.a (mon.0) 3121 : audit [DBG] from='client.? 172.21.15.6:0/2965109966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:49.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:49 smithi006 bash[33918]: audit 2023-08-28T00:17:49.512869+0000 mon.a (mon.0) 3121 : audit [DBG] from='client.? 172.21.15.6:0/2965109966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:50.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:49 smithi130 bash[27579]: audit 2023-08-28T00:17:49.512869+0000 mon.a (mon.0) 3121 : audit [DBG] from='client.? 172.21.15.6:0/2965109966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:50.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:50 smithi130 bash[51302]: debug 2023-08-28T00:17:50.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:50.359626+0000) 2023-08-28T00:17:50.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:50 smithi006 bash[33918]: cluster 2023-08-28T00:17:50.354304+0000 mgr.y (mgr.14901) 4178 : cluster [DBG] pgmap v4185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:50.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:50 smithi006 bash[27292]: cluster 2023-08-28T00:17:50.354304+0000 mgr.y (mgr.14901) 4178 : cluster [DBG] pgmap v4185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:50.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:17:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:17:50] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:17:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:17:50.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:17:50.696Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:51.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:50 smithi130 bash[27579]: cluster 2023-08-28T00:17:50.354304+0000 mgr.y (mgr.14901) 4178 : cluster [DBG] pgmap v4185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:51.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:51 smithi130 bash[51302]: debug 2023-08-28T00:17:51.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:51.359925+0000) 2023-08-28T00:17:52.277 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:51 smithi006 bash[33918]: audit 2023-08-28T00:17:51.922498+0000 mon.c (mon.2) 1588 : audit [DBG] from='client.? 172.21.15.6:0/3667703817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:52.278 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:51 smithi006 bash[27292]: audit 2023-08-28T00:17:51.922498+0000 mon.c (mon.2) 1588 : audit [DBG] from='client.? 172.21.15.6:0/3667703817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:52.361 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:51 smithi130 bash[27579]: audit 2023-08-28T00:17:51.922498+0000 mon.c (mon.2) 1588 : audit [DBG] from='client.? 172.21.15.6:0/3667703817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:52.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:52 smithi130 bash[51302]: debug 2023-08-28T00:17:52.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:52.360193+0000) 2023-08-28T00:17:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:17:52.275Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:17:52.277Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:53.362 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:52 smithi130 bash[27579]: cluster 2023-08-28T00:17:52.354984+0000 mgr.y (mgr.14901) 4179 : cluster [DBG] pgmap v4186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:52 smithi006 bash[27292]: cluster 2023-08-28T00:17:52.354984+0000 mgr.y (mgr.14901) 4179 : cluster [DBG] pgmap v4186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:53.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:52 smithi006 bash[33918]: cluster 2023-08-28T00:17:52.354984+0000 mgr.y (mgr.14901) 4179 : cluster [DBG] pgmap v4186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:53.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:53 smithi130 bash[51302]: debug 2023-08-28T00:17:53.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:53.360458+0000) 2023-08-28T00:17:54.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:54 smithi006 bash[27292]: audit 2023-08-28T00:17:54.338595+0000 mon.a (mon.0) 3122 : audit [DBG] from='client.? 172.21.15.6:0/2469024665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:54.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:54 smithi006 bash[27292]: cluster 2023-08-28T00:17:54.355788+0000 mgr.y (mgr.14901) 4180 : cluster [DBG] pgmap v4187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:17:54.158Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:17:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:17:54.161Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:17:54.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:54 smithi006 bash[33918]: audit 2023-08-28T00:17:54.338595+0000 mon.a (mon.0) 3122 : audit [DBG] from='client.? 172.21.15.6:0/2469024665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:54.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:54 smithi006 bash[33918]: cluster 2023-08-28T00:17:54.355788+0000 mgr.y (mgr.14901) 4180 : cluster [DBG] pgmap v4187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:54.620 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:54 smithi130 bash[51302]: debug 2023-08-28T00:17:54.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:54.360735+0000) 2023-08-28T00:17:54.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:54 smithi130 bash[27579]: audit 2023-08-28T00:17:54.338595+0000 mon.a (mon.0) 3122 : audit [DBG] from='client.? 172.21.15.6:0/2469024665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:54.621 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:54 smithi130 bash[27579]: cluster 2023-08-28T00:17:54.355788+0000 mgr.y (mgr.14901) 4180 : cluster [DBG] pgmap v4187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:55.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:55 smithi130 bash[51302]: debug 2023-08-28T00:17:55.359+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:55.361007+0000) 2023-08-28T00:17:56.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:56 smithi130 bash[51302]: debug 2023-08-28T00:17:56.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:56.361230+0000) 2023-08-28T00:17:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:56 smithi006 bash[27292]: cluster 2023-08-28T00:17:56.356407+0000 mgr.y (mgr.14901) 4181 : cluster [DBG] pgmap v4188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:56 smithi006 bash[33918]: cluster 2023-08-28T00:17:56.356407+0000 mgr.y (mgr.14901) 4181 : cluster [DBG] pgmap v4188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:57.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:56 smithi130 bash[27579]: cluster 2023-08-28T00:17:56.356407+0000 mgr.y (mgr.14901) 4181 : cluster [DBG] pgmap v4188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:57.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:57 smithi130 bash[51302]: debug 2023-08-28T00:17:57.359+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:57.361502+0000) 2023-08-28T00:17:57.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:57 smithi006 bash[27292]: audit 2023-08-28T00:17:56.756781+0000 mon.a (mon.0) 3123 : audit [DBG] from='client.? 172.21.15.6:0/3875023842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:57.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:57 smithi006 bash[33918]: audit 2023-08-28T00:17:56.756781+0000 mon.a (mon.0) 3123 : audit [DBG] from='client.? 172.21.15.6:0/3875023842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:58.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:57 smithi130 bash[27579]: audit 2023-08-28T00:17:56.756781+0000 mon.a (mon.0) 3123 : audit [DBG] from='client.? 172.21.15.6:0/3875023842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:58.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:58 smithi130 bash[51302]: debug 2023-08-28T00:17:58.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:58.361812+0000) 2023-08-28T00:17:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:58 smithi006 bash[27292]: cluster 2023-08-28T00:17:58.357074+0000 mgr.y (mgr.14901) 4182 : cluster [DBG] pgmap v4189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:58 smithi006 bash[33918]: cluster 2023-08-28T00:17:58.357074+0000 mgr.y (mgr.14901) 4182 : cluster [DBG] pgmap v4189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:58 smithi130 bash[27579]: cluster 2023-08-28T00:17:58.357074+0000 mgr.y (mgr.14901) 4182 : cluster [DBG] pgmap v4189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:17:59.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:17:59 smithi130 bash[51302]: debug 2023-08-28T00:17:59.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:59.362223+0000) 2023-08-28T00:17:59.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:17:59 smithi006 bash[27292]: audit 2023-08-28T00:17:59.178002+0000 mon.a (mon.0) 3124 : audit [DBG] from='client.? 172.21.15.6:0/4037170125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:59.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:17:59 smithi006 bash[33918]: audit 2023-08-28T00:17:59.178002+0000 mon.a (mon.0) 3124 : audit [DBG] from='client.? 172.21.15.6:0/4037170125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:17:59 smithi130 bash[27579]: audit 2023-08-28T00:17:59.178002+0000 mon.a (mon.0) 3124 : audit [DBG] from='client.? 172.21.15.6:0/4037170125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:00.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:00 smithi130 bash[51302]: debug 2023-08-28T00:18:00.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:00.362526+0000) 2023-08-28T00:18:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:18:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:18:00] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T00:18:00.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:00 smithi006 bash[33918]: cluster 2023-08-28T00:18:00.358071+0000 mgr.y (mgr.14901) 4183 : cluster [DBG] pgmap v4190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:00.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:00 smithi006 bash[27292]: cluster 2023-08-28T00:18:00.358071+0000 mgr.y (mgr.14901) 4183 : cluster [DBG] pgmap v4190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:18:00.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:00.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:18:00.697Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:00 smithi130 bash[27579]: cluster 2023-08-28T00:18:00.358071+0000 mgr.y (mgr.14901) 4183 : cluster [DBG] pgmap v4190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:01.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:01 smithi130 bash[51302]: debug 2023-08-28T00:18:01.359+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:01.362828+0000) 2023-08-28T00:18:01.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:01 smithi006 bash[33918]: audit 2023-08-28T00:18:01.591418+0000 mon.a (mon.0) 3125 : audit [DBG] from='client.? 172.21.15.6:0/383477818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:01.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:01 smithi006 bash[27292]: audit 2023-08-28T00:18:01.591418+0000 mon.a (mon.0) 3125 : audit [DBG] from='client.? 172.21.15.6:0/383477818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:02.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:01 smithi130 bash[27579]: audit 2023-08-28T00:18:01.591418+0000 mon.a (mon.0) 3125 : audit [DBG] from='client.? 172.21.15.6:0/383477818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:02.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:02 smithi130 bash[51302]: debug 2023-08-28T00:18:02.359+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:02.363135+0000) 2023-08-28T00:18:02.695 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:18:02.276Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:02.695 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:18:02.278Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:02.695 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:02 smithi006 bash[27292]: cluster 2023-08-28T00:18:02.358734+0000 mgr.y (mgr.14901) 4184 : cluster [DBG] pgmap v4191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:02 smithi006 bash[33918]: cluster 2023-08-28T00:18:02.358734+0000 mgr.y (mgr.14901) 4184 : cluster [DBG] pgmap v4191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:02 smithi130 bash[27579]: cluster 2023-08-28T00:18:02.358734+0000 mgr.y (mgr.14901) 4184 : cluster [DBG] pgmap v4191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:03.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:03 smithi130 bash[51302]: debug 2023-08-28T00:18:03.359+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:03.363421+0000) 2023-08-28T00:18:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:04 smithi130 bash[27579]: audit 2023-08-28T00:18:04.000571+0000 mon.c (mon.2) 1589 : audit [DBG] from='client.? 172.21.15.6:0/1546519801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:04.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:04 smithi006 bash[27292]: audit 2023-08-28T00:18:04.000571+0000 mon.c (mon.2) 1589 : audit [DBG] from='client.? 172.21.15.6:0/1546519801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:04.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:04 smithi006 bash[33918]: audit 2023-08-28T00:18:04.000571+0000 mon.c (mon.2) 1589 : audit [DBG] from='client.? 172.21.15.6:0/1546519801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:18:04.159Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:18:04.162Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:04.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:04 smithi130 bash[51302]: debug 2023-08-28T00:18:04.359+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:04.363672+0000) 2023-08-28T00:18:05.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:05 smithi130 bash[27579]: cluster 2023-08-28T00:18:04.359849+0000 mgr.y (mgr.14901) 4185 : cluster [DBG] pgmap v4192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:05.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:05 smithi006 bash[27292]: cluster 2023-08-28T00:18:04.359849+0000 mgr.y (mgr.14901) 4185 : cluster [DBG] pgmap v4192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:05.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:05 smithi006 bash[33918]: cluster 2023-08-28T00:18:04.359849+0000 mgr.y (mgr.14901) 4185 : cluster [DBG] pgmap v4192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:05.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:05 smithi130 bash[51302]: debug 2023-08-28T00:18:05.359+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:05.364081+0000) 2023-08-28T00:18:06.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:06 smithi130 bash[51302]: debug 2023-08-28T00:18:06.359+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:06.364390+0000) 2023-08-28T00:18:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:06 smithi006 bash[33918]: cluster 2023-08-28T00:18:06.360500+0000 mgr.y (mgr.14901) 4186 : cluster [DBG] pgmap v4193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:06 smithi006 bash[33918]: audit 2023-08-28T00:18:06.413847+0000 mon.c (mon.2) 1590 : audit [DBG] from='client.? 172.21.15.6:0/349423840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:06 smithi006 bash[27292]: cluster 2023-08-28T00:18:06.360500+0000 mgr.y (mgr.14901) 4186 : cluster [DBG] pgmap v4193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:06 smithi006 bash[27292]: audit 2023-08-28T00:18:06.413847+0000 mon.c (mon.2) 1590 : audit [DBG] from='client.? 172.21.15.6:0/349423840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:06 smithi130 bash[27579]: cluster 2023-08-28T00:18:06.360500+0000 mgr.y (mgr.14901) 4186 : cluster [DBG] pgmap v4193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:06 smithi130 bash[27579]: audit 2023-08-28T00:18:06.413847+0000 mon.c (mon.2) 1590 : audit [DBG] from='client.? 172.21.15.6:0/349423840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:07.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:07 smithi130 bash[51302]: debug 2023-08-28T00:18:07.363+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:07.364713+0000) 2023-08-28T00:18:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:08 smithi130 bash[27579]: cluster 2023-08-28T00:18:08.361164+0000 mgr.y (mgr.14901) 4187 : cluster [DBG] pgmap v4194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:08.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:08 smithi130 bash[51302]: debug 2023-08-28T00:18:08.359+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:08.364934+0000) 2023-08-28T00:18:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:08 smithi006 bash[27292]: cluster 2023-08-28T00:18:08.361164+0000 mgr.y (mgr.14901) 4187 : cluster [DBG] pgmap v4194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:08 smithi006 bash[33918]: cluster 2023-08-28T00:18:08.361164+0000 mgr.y (mgr.14901) 4187 : cluster [DBG] pgmap v4194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:09.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:09 smithi130 bash[51302]: debug 2023-08-28T00:18:09.363+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:09.365217+0000) 2023-08-28T00:18:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:09 smithi130 bash[27579]: audit 2023-08-28T00:18:08.825847+0000 mon.c (mon.2) 1591 : audit [DBG] from='client.? 172.21.15.6:0/1267260572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:09.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:09 smithi006 bash[33918]: audit 2023-08-28T00:18:08.825847+0000 mon.c (mon.2) 1591 : audit [DBG] from='client.? 172.21.15.6:0/1267260572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:09 smithi006 bash[27292]: audit 2023-08-28T00:18:08.825847+0000 mon.c (mon.2) 1591 : audit [DBG] from='client.? 172.21.15.6:0/1267260572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:10 smithi130 bash[27579]: cluster 2023-08-28T00:18:10.362428+0000 mgr.y (mgr.14901) 4188 : cluster [DBG] pgmap v4195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:10.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:10 smithi130 bash[51302]: debug 2023-08-28T00:18:10.359+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:10.365499+0000) 2023-08-28T00:18:10.698 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:10 smithi006 bash[33918]: cluster 2023-08-28T00:18:10.362428+0000 mgr.y (mgr.14901) 4188 : cluster [DBG] pgmap v4195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:10.699 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:10 smithi006 bash[27292]: cluster 2023-08-28T00:18:10.362428+0000 mgr.y (mgr.14901) 4188 : cluster [DBG] pgmap v4195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:18:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:18:10] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T00:18:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:18:10.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:18:10.698Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:11.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:11 smithi130 bash[51302]: debug 2023-08-28T00:18:11.359+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:11.365773+0000) 2023-08-28T00:18:11.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:11 smithi006 bash[33918]: audit 2023-08-28T00:18:11.238084+0000 mon.c (mon.2) 1592 : audit [DBG] from='client.? 172.21.15.6:0/3562687444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:11.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:11 smithi006 bash[27292]: audit 2023-08-28T00:18:11.238084+0000 mon.c (mon.2) 1592 : audit [DBG] from='client.? 172.21.15.6:0/3562687444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:12.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:11 smithi130 bash[27579]: audit 2023-08-28T00:18:11.238084+0000 mon.c (mon.2) 1592 : audit [DBG] from='client.? 172.21.15.6:0/3562687444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:12.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:12 smithi130 bash[51302]: debug 2023-08-28T00:18:12.359+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:12.365987+0000) 2023-08-28T00:18:12.647 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:18:12.277Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:12.647 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:18:12.279Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:12 smithi006 bash[27292]: cluster 2023-08-28T00:18:12.363107+0000 mgr.y (mgr.14901) 4189 : cluster [DBG] pgmap v4196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:12 smithi006 bash[33918]: cluster 2023-08-28T00:18:12.363107+0000 mgr.y (mgr.14901) 4189 : cluster [DBG] pgmap v4196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:12 smithi130 bash[27579]: cluster 2023-08-28T00:18:12.363107+0000 mgr.y (mgr.14901) 4189 : cluster [DBG] pgmap v4196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:13.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:13 smithi130 bash[51302]: debug 2023-08-28T00:18:13.363+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:13.366279+0000) 2023-08-28T00:18:13.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:13 smithi006 bash[33918]: audit 2023-08-28T00:18:13.655200+0000 mon.c (mon.2) 1593 : audit [DBG] from='client.? 172.21.15.6:0/3557914525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:13.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:13 smithi006 bash[27292]: audit 2023-08-28T00:18:13.655200+0000 mon.c (mon.2) 1593 : audit [DBG] from='client.? 172.21.15.6:0/3557914525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:14.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:13 smithi130 bash[27579]: audit 2023-08-28T00:18:13.655200+0000 mon.c (mon.2) 1593 : audit [DBG] from='client.? 172.21.15.6:0/3557914525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:18:14.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:18:14.162Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:14.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:14 smithi130 bash[51302]: debug 2023-08-28T00:18:14.359+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:14.366583+0000) 2023-08-28T00:18:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:14 smithi130 bash[27579]: cluster 2023-08-28T00:18:14.364230+0000 mgr.y (mgr.14901) 4190 : cluster [DBG] pgmap v4197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:14 smithi006 bash[33918]: cluster 2023-08-28T00:18:14.364230+0000 mgr.y (mgr.14901) 4190 : cluster [DBG] pgmap v4197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:14 smithi006 bash[27292]: cluster 2023-08-28T00:18:14.364230+0000 mgr.y (mgr.14901) 4190 : cluster [DBG] pgmap v4197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:15.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:15 smithi130 bash[51302]: debug 2023-08-28T00:18:15.363+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:15.366965+0000) 2023-08-28T00:18:16.649 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:16 smithi130 bash[51302]: debug 2023-08-28T00:18:16.363+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:16.367276+0000) 2023-08-28T00:18:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:16 smithi006 bash[27292]: audit 2023-08-28T00:18:16.071734+0000 mon.a (mon.0) 3126 : audit [DBG] from='client.? 172.21.15.6:0/4291835709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:16 smithi006 bash[27292]: cluster 2023-08-28T00:18:16.364951+0000 mgr.y (mgr.14901) 4191 : cluster [DBG] pgmap v4198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:16 smithi006 bash[33918]: audit 2023-08-28T00:18:16.071734+0000 mon.a (mon.0) 3126 : audit [DBG] from='client.? 172.21.15.6:0/4291835709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:16 smithi006 bash[33918]: cluster 2023-08-28T00:18:16.364951+0000 mgr.y (mgr.14901) 4191 : cluster [DBG] pgmap v4198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:17.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:16 smithi130 bash[27579]: audit 2023-08-28T00:18:16.071734+0000 mon.a (mon.0) 3126 : audit [DBG] from='client.? 172.21.15.6:0/4291835709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:17.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:16 smithi130 bash[27579]: cluster 2023-08-28T00:18:16.364951+0000 mgr.y (mgr.14901) 4191 : cluster [DBG] pgmap v4198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:17.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:17 smithi130 bash[51302]: debug 2023-08-28T00:18:17.363+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:17.367549+0000) 2023-08-28T00:18:18.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:18 smithi006 bash[27292]: cluster 2023-08-28T00:18:18.365565+0000 mgr.y (mgr.14901) 4192 : cluster [DBG] pgmap v4199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:18 smithi006 bash[33918]: cluster 2023-08-28T00:18:18.365565+0000 mgr.y (mgr.14901) 4192 : cluster [DBG] pgmap v4199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:18 smithi130 bash[27579]: cluster 2023-08-28T00:18:18.365565+0000 mgr.y (mgr.14901) 4192 : cluster [DBG] pgmap v4199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:18.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:18 smithi130 bash[51302]: debug 2023-08-28T00:18:18.363+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:18.367853+0000) 2023-08-28T00:18:19.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:19 smithi006 bash[33918]: audit 2023-08-28T00:18:18.497210+0000 mon.c (mon.2) 1594 : audit [DBG] from='client.? 172.21.15.6:0/823617411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:19.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:19 smithi006 bash[27292]: audit 2023-08-28T00:18:18.497210+0000 mon.c (mon.2) 1594 : audit [DBG] from='client.? 172.21.15.6:0/823617411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:19.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:19 smithi130 bash[27579]: audit 2023-08-28T00:18:18.497210+0000 mon.c (mon.2) 1594 : audit [DBG] from='client.? 172.21.15.6:0/823617411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:19 smithi130 bash[51302]: debug 2023-08-28T00:18:19.363+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:19.368199+0000) 2023-08-28T00:18:20.698 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:20 smithi006 bash[27292]: cluster 2023-08-28T00:18:20.366531+0000 mgr.y (mgr.14901) 4193 : cluster [DBG] pgmap v4200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:20.699 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:20 smithi006 bash[33918]: cluster 2023-08-28T00:18:20.366531+0000 mgr.y (mgr.14901) 4193 : cluster [DBG] pgmap v4200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:20 smithi130 bash[27579]: cluster 2023-08-28T00:18:20.366531+0000 mgr.y (mgr.14901) 4193 : cluster [DBG] pgmap v4200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:20 smithi130 bash[51302]: debug 2023-08-28T00:18:20.363+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:20.368428+0000) 2023-08-28T00:18:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:18:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:18:20] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:18:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:18:20.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:18:20.698Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:21.645 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:21 smithi130 bash[51302]: debug 2023-08-28T00:18:21.367+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:21.368754+0000) 2023-08-28T00:18:21.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:21 smithi006 bash[27292]: audit 2023-08-28T00:18:20.915232+0000 mon.a (mon.0) 3127 : audit [DBG] from='client.? 172.21.15.6:0/2154965125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:21.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:21 smithi006 bash[33918]: audit 2023-08-28T00:18:20.915232+0000 mon.a (mon.0) 3127 : audit [DBG] from='client.? 172.21.15.6:0/2154965125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:22.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:21 smithi130 bash[27579]: audit 2023-08-28T00:18:20.915232+0000 mon.a (mon.0) 3127 : audit [DBG] from='client.? 172.21.15.6:0/2154965125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:18:22.278Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:18:22.279Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:22 smithi130 bash[51302]: debug 2023-08-28T00:18:22.363+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:22.369033+0000) 2023-08-28T00:18:23.650 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:23 smithi130 bash[51302]: debug 2023-08-28T00:18:23.363+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:23.369388+0000) 2023-08-28T00:18:23.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:23 smithi006 bash[33918]: cluster 2023-08-28T00:18:22.367177+0000 mgr.y (mgr.14901) 4194 : cluster [DBG] pgmap v4201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:23.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:23 smithi006 bash[33918]: audit 2023-08-28T00:18:23.329782+0000 mon.a (mon.0) 3128 : audit [DBG] from='client.? 172.21.15.6:0/3352320327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:23.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:23 smithi006 bash[27292]: cluster 2023-08-28T00:18:22.367177+0000 mgr.y (mgr.14901) 4194 : cluster [DBG] pgmap v4201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:23.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:23 smithi006 bash[27292]: audit 2023-08-28T00:18:23.329782+0000 mon.a (mon.0) 3128 : audit [DBG] from='client.? 172.21.15.6:0/3352320327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:24.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:23 smithi130 bash[27579]: cluster 2023-08-28T00:18:22.367177+0000 mgr.y (mgr.14901) 4194 : cluster [DBG] pgmap v4201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:24.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:23 smithi130 bash[27579]: audit 2023-08-28T00:18:23.329782+0000 mon.a (mon.0) 3128 : audit [DBG] from='client.? 172.21.15.6:0/3352320327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:18:24.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:18:24.163Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:24 smithi130 bash[51302]: debug 2023-08-28T00:18:24.367+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:24.369769+0000) 2023-08-28T00:18:25.653 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:25 smithi130 bash[51302]: debug 2023-08-28T00:18:25.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:25.370140+0000) 2023-08-28T00:18:25.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:25 smithi006 bash[27292]: cluster 2023-08-28T00:18:24.368387+0000 mgr.y (mgr.14901) 4195 : cluster [DBG] pgmap v4202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:25.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:25 smithi006 bash[33918]: cluster 2023-08-28T00:18:24.368387+0000 mgr.y (mgr.14901) 4195 : cluster [DBG] pgmap v4202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:25 smithi130 bash[27579]: cluster 2023-08-28T00:18:24.368387+0000 mgr.y (mgr.14901) 4195 : cluster [DBG] pgmap v4202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:26.663 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:26 smithi130 bash[51302]: debug 2023-08-28T00:18:26.362+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:26.370477+0000) 2023-08-28T00:18:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:26 smithi006 bash[27292]: audit 2023-08-28T00:18:25.741799+0000 mon.a (mon.0) 3129 : audit [DBG] from='client.? 172.21.15.6:0/193057095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:26 smithi006 bash[33918]: audit 2023-08-28T00:18:25.741799+0000 mon.a (mon.0) 3129 : audit [DBG] from='client.? 172.21.15.6:0/193057095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:26 smithi130 bash[27579]: audit 2023-08-28T00:18:25.741799+0000 mon.a (mon.0) 3129 : audit [DBG] from='client.? 172.21.15.6:0/193057095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:27.665 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:27 smithi130 bash[51302]: debug 2023-08-28T00:18:27.362+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:27.370762+0000) 2023-08-28T00:18:27.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:27 smithi006 bash[33918]: cluster 2023-08-28T00:18:26.369008+0000 mgr.y (mgr.14901) 4196 : cluster [DBG] pgmap v4203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:27.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:27 smithi006 bash[27292]: cluster 2023-08-28T00:18:26.369008+0000 mgr.y (mgr.14901) 4196 : cluster [DBG] pgmap v4203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:28.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:27 smithi130 bash[27579]: cluster 2023-08-28T00:18:26.369008+0000 mgr.y (mgr.14901) 4196 : cluster [DBG] pgmap v4203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:28.674 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:28 smithi130 bash[51302]: debug 2023-08-28T00:18:28.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:28.371072+0000) 2023-08-28T00:18:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:28 smithi006 bash[27292]: audit 2023-08-28T00:18:28.148822+0000 mon.a (mon.0) 3130 : audit [DBG] from='client.? 172.21.15.6:0/3325527656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:28 smithi006 bash[33918]: audit 2023-08-28T00:18:28.148822+0000 mon.a (mon.0) 3130 : audit [DBG] from='client.? 172.21.15.6:0/3325527656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:28 smithi130 bash[27579]: audit 2023-08-28T00:18:28.148822+0000 mon.a (mon.0) 3130 : audit [DBG] from='client.? 172.21.15.6:0/3325527656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:29.679 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:29 smithi130 bash[51302]: debug 2023-08-28T00:18:29.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:29.371438+0000) 2023-08-28T00:18:29.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:29 smithi006 bash[27292]: cluster 2023-08-28T00:18:28.369619+0000 mgr.y (mgr.14901) 4197 : cluster [DBG] pgmap v4204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:29.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:29 smithi006 bash[33918]: cluster 2023-08-28T00:18:28.369619+0000 mgr.y (mgr.14901) 4197 : cluster [DBG] pgmap v4204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:30.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:29 smithi130 bash[27579]: cluster 2023-08-28T00:18:28.369619+0000 mgr.y (mgr.14901) 4197 : cluster [DBG] pgmap v4204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:30.686 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:30 smithi130 bash[51302]: debug 2023-08-28T00:18:30.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:30.371723+0000) 2023-08-28T00:18:30.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:18:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:18:30] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:18:30.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:30 smithi006 bash[33918]: audit 2023-08-28T00:18:30.576161+0000 mon.c (mon.2) 1595 : audit [DBG] from='client.? 172.21.15.6:0/3020405347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:30.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:30 smithi006 bash[27292]: audit 2023-08-28T00:18:30.576161+0000 mon.c (mon.2) 1595 : audit [DBG] from='client.? 172.21.15.6:0/3020405347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:30.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:18:30.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:30.971 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:18:30.699Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:31.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:30 smithi130 bash[27579]: audit 2023-08-28T00:18:30.576161+0000 mon.c (mon.2) 1595 : audit [DBG] from='client.? 172.21.15.6:0/3020405347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:31.689 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:31 smithi130 bash[51302]: debug 2023-08-28T00:18:31.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:31.371997+0000) 2023-08-28T00:18:31.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:31 smithi006 bash[27292]: cluster 2023-08-28T00:18:30.370971+0000 mgr.y (mgr.14901) 4198 : cluster [DBG] pgmap v4205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:31.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:31 smithi006 bash[33918]: cluster 2023-08-28T00:18:30.370971+0000 mgr.y (mgr.14901) 4198 : cluster [DBG] pgmap v4205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:32.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:31 smithi130 bash[27579]: cluster 2023-08-28T00:18:30.370971+0000 mgr.y (mgr.14901) 4198 : cluster [DBG] pgmap v4205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:32.696 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:18:32.278Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:32.696 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:18:32.280Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:32.697 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:32 smithi130 bash[51302]: debug 2023-08-28T00:18:32.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:32.372259+0000) 2023-08-28T00:18:32.697 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:32 smithi006 bash[27292]: audit 2023-08-28T00:18:32.340211+0000 mon.b (mon.1) 525 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:18:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:32 smithi006 bash[33918]: audit 2023-08-28T00:18:32.340211+0000 mon.b (mon.1) 525 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:18:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:32 smithi130 bash[27579]: audit 2023-08-28T00:18:32.340211+0000 mon.b (mon.1) 525 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:18:33.700 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:33 smithi130 bash[51302]: debug 2023-08-28T00:18:33.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:33.372546+0000) 2023-08-28T00:18:33.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:33 smithi006 bash[27292]: cluster 2023-08-28T00:18:32.371638+0000 mgr.y (mgr.14901) 4199 : cluster [DBG] pgmap v4206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:33.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:33 smithi006 bash[27292]: audit 2023-08-28T00:18:32.982389+0000 mon.a (mon.0) 3131 : audit [DBG] from='client.? 172.21.15.6:0/3960811642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:33.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:33 smithi006 bash[33918]: cluster 2023-08-28T00:18:32.371638+0000 mgr.y (mgr.14901) 4199 : cluster [DBG] pgmap v4206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:33.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:33 smithi006 bash[33918]: audit 2023-08-28T00:18:32.982389+0000 mon.a (mon.0) 3131 : audit [DBG] from='client.? 172.21.15.6:0/3960811642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:33 smithi130 bash[27579]: cluster 2023-08-28T00:18:32.371638+0000 mgr.y (mgr.14901) 4199 : cluster [DBG] pgmap v4206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:33 smithi130 bash[27579]: audit 2023-08-28T00:18:32.982389+0000 mon.a (mon.0) 3131 : audit [DBG] from='client.? 172.21.15.6:0/3960811642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:18:34.162Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:18:34.164Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:34 smithi130 bash[51302]: debug 2023-08-28T00:18:34.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:34.372826+0000) 2023-08-28T00:18:35.705 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:35 smithi130 bash[51302]: debug 2023-08-28T00:18:35.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:35.373129+0000) 2023-08-28T00:18:35.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:35 smithi006 bash[33918]: cluster 2023-08-28T00:18:34.372892+0000 mgr.y (mgr.14901) 4200 : cluster [DBG] pgmap v4207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:35.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:35 smithi006 bash[33918]: audit 2023-08-28T00:18:35.404543+0000 mon.c (mon.2) 1596 : audit [DBG] from='client.? 172.21.15.6:0/317410705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:35.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:35 smithi006 bash[27292]: cluster 2023-08-28T00:18:34.372892+0000 mgr.y (mgr.14901) 4200 : cluster [DBG] pgmap v4207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:35.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:35 smithi006 bash[27292]: audit 2023-08-28T00:18:35.404543+0000 mon.c (mon.2) 1596 : audit [DBG] from='client.? 172.21.15.6:0/317410705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:36.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:35 smithi130 bash[27579]: cluster 2023-08-28T00:18:34.372892+0000 mgr.y (mgr.14901) 4200 : cluster [DBG] pgmap v4207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:36.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:35 smithi130 bash[27579]: audit 2023-08-28T00:18:35.404543+0000 mon.c (mon.2) 1596 : audit [DBG] from='client.? 172.21.15.6:0/317410705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:36 smithi130 bash[51302]: debug 2023-08-28T00:18:36.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:36.373486+0000) 2023-08-28T00:18:37.709 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:37 smithi130 bash[51302]: debug 2023-08-28T00:18:37.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:37.373734+0000) 2023-08-28T00:18:37.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:37 smithi006 bash[27292]: cluster 2023-08-28T00:18:36.373589+0000 mgr.y (mgr.14901) 4201 : cluster [DBG] pgmap v4208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:37.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:37 smithi006 bash[33918]: cluster 2023-08-28T00:18:36.373589+0000 mgr.y (mgr.14901) 4201 : cluster [DBG] pgmap v4208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:38.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:37 smithi130 bash[27579]: cluster 2023-08-28T00:18:36.373589+0000 mgr.y (mgr.14901) 4201 : cluster [DBG] pgmap v4208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:38.711 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:38 smithi130 bash[51302]: debug 2023-08-28T00:18:38.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:38.373926+0000) 2023-08-28T00:18:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:38 smithi006 bash[27292]: audit 2023-08-28T00:18:37.832629+0000 mon.a (mon.0) 3132 : audit [DBG] from='client.? 172.21.15.6:0/1614671440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:38 smithi006 bash[27292]: audit 2023-08-28T00:18:38.255719+0000 mon.a (mon.0) 3133 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:18:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:38 smithi006 bash[27292]: audit 2023-08-28T00:18:38.661375+0000 mon.a (mon.0) 3134 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:18:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:38 smithi006 bash[33918]: audit 2023-08-28T00:18:37.832629+0000 mon.a (mon.0) 3132 : audit [DBG] from='client.? 172.21.15.6:0/1614671440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:38 smithi006 bash[33918]: audit 2023-08-28T00:18:38.255719+0000 mon.a (mon.0) 3133 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:18:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:38 smithi006 bash[33918]: audit 2023-08-28T00:18:38.661375+0000 mon.a (mon.0) 3134 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:18:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:38 smithi130 bash[27579]: audit 2023-08-28T00:18:37.832629+0000 mon.a (mon.0) 3132 : audit [DBG] from='client.? 172.21.15.6:0/1614671440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:38 smithi130 bash[27579]: audit 2023-08-28T00:18:38.255719+0000 mon.a (mon.0) 3133 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:18:39.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:38 smithi130 bash[27579]: audit 2023-08-28T00:18:38.661375+0000 mon.a (mon.0) 3134 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:18:39.716 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:39 smithi130 bash[51302]: debug 2023-08-28T00:18:39.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:39.374188+0000) 2023-08-28T00:18:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:39 smithi006 bash[33918]: cluster 2023-08-28T00:18:38.374182+0000 mgr.y (mgr.14901) 4202 : cluster [DBG] pgmap v4209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:39 smithi006 bash[33918]: audit 2023-08-28T00:18:38.958428+0000 mon.b (mon.1) 526 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:18:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:39 smithi006 bash[33918]: audit 2023-08-28T00:18:38.960759+0000 mon.b (mon.1) 527 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:18:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:39 smithi006 bash[33918]: audit 2023-08-28T00:18:38.969305+0000 mon.a (mon.0) 3135 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:18:39.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:39 smithi006 bash[27292]: cluster 2023-08-28T00:18:38.374182+0000 mgr.y (mgr.14901) 4202 : cluster [DBG] pgmap v4209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:39.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:39 smithi006 bash[27292]: audit 2023-08-28T00:18:38.958428+0000 mon.b (mon.1) 526 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:18:39.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:39 smithi006 bash[27292]: audit 2023-08-28T00:18:38.960759+0000 mon.b (mon.1) 527 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:18:39.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:39 smithi006 bash[27292]: audit 2023-08-28T00:18:38.969305+0000 mon.a (mon.0) 3135 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:18:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:39 smithi130 bash[27579]: cluster 2023-08-28T00:18:38.374182+0000 mgr.y (mgr.14901) 4202 : cluster [DBG] pgmap v4209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:39 smithi130 bash[27579]: audit 2023-08-28T00:18:38.958428+0000 mon.b (mon.1) 526 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:18:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:39 smithi130 bash[27579]: audit 2023-08-28T00:18:38.960759+0000 mon.b (mon.1) 527 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:18:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:39 smithi130 bash[27579]: audit 2023-08-28T00:18:38.969305+0000 mon.a (mon.0) 3135 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:18:40.722 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:40 smithi130 bash[51302]: debug 2023-08-28T00:18:40.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:40.374453+0000) 2023-08-28T00:18:40.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:40 smithi006 bash[27292]: audit 2023-08-28T00:18:40.259550+0000 mon.a (mon.0) 3136 : audit [DBG] from='client.? 172.21.15.6:0/3891111080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:18:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:18:40] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:18:40.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:40 smithi006 bash[33918]: audit 2023-08-28T00:18:40.259550+0000 mon.a (mon.0) 3136 : audit [DBG] from='client.? 172.21.15.6:0/3891111080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:18:40.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:18:40.700Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:40 smithi130 bash[27579]: audit 2023-08-28T00:18:40.259550+0000 mon.a (mon.0) 3136 : audit [DBG] from='client.? 172.21.15.6:0/3891111080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:41.726 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:41 smithi130 bash[51302]: debug 2023-08-28T00:18:41.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:41.374721+0000) 2023-08-28T00:18:42.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:41 smithi130 bash[27579]: cluster 2023-08-28T00:18:40.375243+0000 mgr.y (mgr.14901) 4203 : cluster [DBG] pgmap v4210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:42.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:41 smithi006 bash[33918]: cluster 2023-08-28T00:18:40.375243+0000 mgr.y (mgr.14901) 4203 : cluster [DBG] pgmap v4210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:42.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:41 smithi006 bash[27292]: cluster 2023-08-28T00:18:40.375243+0000 mgr.y (mgr.14901) 4203 : cluster [DBG] pgmap v4210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:18:42.279Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:18:42.281Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:42.733 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:42 smithi130 bash[51302]: debug 2023-08-28T00:18:42.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:42.375035+0000) 2023-08-28T00:18:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:42 smithi130 bash[27579]: audit 2023-08-28T00:18:42.667170+0000 mon.a (mon.0) 3137 : audit [DBG] from='client.? 172.21.15.6:0/1479704893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:42 smithi006 bash[33918]: audit 2023-08-28T00:18:42.667170+0000 mon.a (mon.0) 3137 : audit [DBG] from='client.? 172.21.15.6:0/1479704893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:42 smithi006 bash[27292]: audit 2023-08-28T00:18:42.667170+0000 mon.a (mon.0) 3137 : audit [DBG] from='client.? 172.21.15.6:0/1479704893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:43.736 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:43 smithi130 bash[51302]: debug 2023-08-28T00:18:43.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:43.375288+0000) 2023-08-28T00:18:44.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:43 smithi130 bash[27579]: cluster 2023-08-28T00:18:42.375931+0000 mgr.y (mgr.14901) 4204 : cluster [DBG] pgmap v4211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:44.165 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:43 smithi006 bash[33918]: cluster 2023-08-28T00:18:42.375931+0000 mgr.y (mgr.14901) 4204 : cluster [DBG] pgmap v4211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:44.166 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:43 smithi006 bash[27292]: cluster 2023-08-28T00:18:42.375931+0000 mgr.y (mgr.14901) 4204 : cluster [DBG] pgmap v4211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:18:44.163Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:18:44.165Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:44 smithi130 bash[51302]: debug 2023-08-28T00:18:44.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:44.375603+0000) 2023-08-28T00:18:45.740 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:45 smithi130 bash[51302]: debug 2023-08-28T00:18:45.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:45.375898+0000) 2023-08-28T00:18:46.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:45 smithi130 bash[27579]: cluster 2023-08-28T00:18:44.377078+0000 mgr.y (mgr.14901) 4205 : cluster [DBG] pgmap v4212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:46.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:45 smithi130 bash[27579]: audit 2023-08-28T00:18:45.077914+0000 mon.a (mon.0) 3138 : audit [DBG] from='client.? 172.21.15.6:0/1338864860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:46.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:45 smithi006 bash[27292]: cluster 2023-08-28T00:18:44.377078+0000 mgr.y (mgr.14901) 4205 : cluster [DBG] pgmap v4212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:46.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:45 smithi006 bash[27292]: audit 2023-08-28T00:18:45.077914+0000 mon.a (mon.0) 3138 : audit [DBG] from='client.? 172.21.15.6:0/1338864860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:46.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:45 smithi006 bash[33918]: cluster 2023-08-28T00:18:44.377078+0000 mgr.y (mgr.14901) 4205 : cluster [DBG] pgmap v4212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:46.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:45 smithi006 bash[33918]: audit 2023-08-28T00:18:45.077914+0000 mon.a (mon.0) 3138 : audit [DBG] from='client.? 172.21.15.6:0/1338864860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:46.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:46 smithi130 bash[51302]: debug 2023-08-28T00:18:46.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:46.376197+0000) 2023-08-28T00:18:47.745 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:47 smithi130 bash[51302]: debug 2023-08-28T00:18:47.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:47.376464+0000) 2023-08-28T00:18:48.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:47 smithi130 bash[27579]: cluster 2023-08-28T00:18:46.377634+0000 mgr.y (mgr.14901) 4206 : cluster [DBG] pgmap v4213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:48.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:47 smithi130 bash[27579]: audit 2023-08-28T00:18:47.486756+0000 mon.a (mon.0) 3139 : audit [DBG] from='client.? 172.21.15.6:0/1134548787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:48.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:47 smithi006 bash[27292]: cluster 2023-08-28T00:18:46.377634+0000 mgr.y (mgr.14901) 4206 : cluster [DBG] pgmap v4213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:48.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:47 smithi006 bash[27292]: audit 2023-08-28T00:18:47.486756+0000 mon.a (mon.0) 3139 : audit [DBG] from='client.? 172.21.15.6:0/1134548787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:48.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:47 smithi006 bash[33918]: cluster 2023-08-28T00:18:46.377634+0000 mgr.y (mgr.14901) 4206 : cluster [DBG] pgmap v4213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:48.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:47 smithi006 bash[33918]: audit 2023-08-28T00:18:47.486756+0000 mon.a (mon.0) 3139 : audit [DBG] from='client.? 172.21.15.6:0/1134548787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:48 smithi130 bash[51302]: debug 2023-08-28T00:18:48.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:48.376731+0000) 2023-08-28T00:18:49.752 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:49 smithi130 bash[51302]: debug 2023-08-28T00:18:49.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:49.377018+0000) 2023-08-28T00:18:50.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:49 smithi130 bash[27579]: cluster 2023-08-28T00:18:48.378282+0000 mgr.y (mgr.14901) 4207 : cluster [DBG] pgmap v4214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:50.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:49 smithi006 bash[27292]: cluster 2023-08-28T00:18:48.378282+0000 mgr.y (mgr.14901) 4207 : cluster [DBG] pgmap v4214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:50.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:49 smithi006 bash[33918]: cluster 2023-08-28T00:18:48.378282+0000 mgr.y (mgr.14901) 4207 : cluster [DBG] pgmap v4214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:50.761 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:50 smithi130 bash[51302]: debug 2023-08-28T00:18:50.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:50.377307+0000) 2023-08-28T00:18:50.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:50 smithi006 bash[27292]: audit 2023-08-28T00:18:49.907045+0000 mon.a (mon.0) 3140 : audit [DBG] from='client.? 172.21.15.6:0/2487948578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:18:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:18:50] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:18:50.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:50 smithi006 bash[33918]: audit 2023-08-28T00:18:49.907045+0000 mon.a (mon.0) 3140 : audit [DBG] from='client.? 172.21.15.6:0/2487948578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:18:50.699Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:18:50.701Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:50 smithi130 bash[27579]: audit 2023-08-28T00:18:49.907045+0000 mon.a (mon.0) 3140 : audit [DBG] from='client.? 172.21.15.6:0/2487948578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:51.764 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:51 smithi130 bash[51302]: debug 2023-08-28T00:18:51.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:51.377606+0000) 2023-08-28T00:18:52.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:51 smithi130 bash[27579]: cluster 2023-08-28T00:18:50.379455+0000 mgr.y (mgr.14901) 4208 : cluster [DBG] pgmap v4215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:52.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:51 smithi006 bash[27292]: cluster 2023-08-28T00:18:50.379455+0000 mgr.y (mgr.14901) 4208 : cluster [DBG] pgmap v4215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:52.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:51 smithi006 bash[33918]: cluster 2023-08-28T00:18:50.379455+0000 mgr.y (mgr.14901) 4208 : cluster [DBG] pgmap v4215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:18:52.280Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:18:52.281Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:52.773 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:52 smithi130 bash[51302]: debug 2023-08-28T00:18:52.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:52.377875+0000) 2023-08-28T00:18:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:52 smithi130 bash[27579]: audit 2023-08-28T00:18:52.322325+0000 mon.c (mon.2) 1597 : audit [DBG] from='client.? 172.21.15.6:0/1871118276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:53.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:52 smithi006 bash[33918]: audit 2023-08-28T00:18:52.322325+0000 mon.c (mon.2) 1597 : audit [DBG] from='client.? 172.21.15.6:0/1871118276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:52 smithi006 bash[27292]: audit 2023-08-28T00:18:52.322325+0000 mon.c (mon.2) 1597 : audit [DBG] from='client.? 172.21.15.6:0/1871118276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:53.776 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:53 smithi130 bash[51302]: debug 2023-08-28T00:18:53.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:53.378151+0000) 2023-08-28T00:18:54.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:53 smithi130 bash[27579]: cluster 2023-08-28T00:18:52.380102+0000 mgr.y (mgr.14901) 4209 : cluster [DBG] pgmap v4216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:54.166 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:53 smithi006 bash[27292]: cluster 2023-08-28T00:18:52.380102+0000 mgr.y (mgr.14901) 4209 : cluster [DBG] pgmap v4216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:54.166 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:53 smithi006 bash[33918]: cluster 2023-08-28T00:18:52.380102+0000 mgr.y (mgr.14901) 4209 : cluster [DBG] pgmap v4216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:18:54.163Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:18:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:18:54.165Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:18:54.784 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:54 smithi130 bash[51302]: debug 2023-08-28T00:18:54.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:54.378403+0000) 2023-08-28T00:18:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:54 smithi130 bash[27579]: audit 2023-08-28T00:18:54.743015+0000 mon.c (mon.2) 1598 : audit [DBG] from='client.? 172.21.15.6:0/3307120496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:54 smithi006 bash[33918]: audit 2023-08-28T00:18:54.743015+0000 mon.c (mon.2) 1598 : audit [DBG] from='client.? 172.21.15.6:0/3307120496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:54 smithi006 bash[27292]: audit 2023-08-28T00:18:54.743015+0000 mon.c (mon.2) 1598 : audit [DBG] from='client.? 172.21.15.6:0/3307120496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:55.788 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:55 smithi130 bash[51302]: debug 2023-08-28T00:18:55.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:55.378674+0000) 2023-08-28T00:18:56.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:55 smithi130 bash[27579]: cluster 2023-08-28T00:18:54.380883+0000 mgr.y (mgr.14901) 4210 : cluster [DBG] pgmap v4217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:56.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:55 smithi006 bash[33918]: cluster 2023-08-28T00:18:54.380883+0000 mgr.y (mgr.14901) 4210 : cluster [DBG] pgmap v4217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:56.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:55 smithi006 bash[27292]: cluster 2023-08-28T00:18:54.380883+0000 mgr.y (mgr.14901) 4210 : cluster [DBG] pgmap v4217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:56.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:56 smithi130 bash[51302]: debug 2023-08-28T00:18:56.378+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:56.378947+0000) 2023-08-28T00:18:57.793 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:57 smithi130 bash[51302]: debug 2023-08-28T00:18:57.378+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:57.379185+0000) 2023-08-28T00:18:58.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:57 smithi130 bash[27579]: cluster 2023-08-28T00:18:56.381461+0000 mgr.y (mgr.14901) 4211 : cluster [DBG] pgmap v4218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:58.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:57 smithi130 bash[27579]: audit 2023-08-28T00:18:57.157953+0000 mon.c (mon.2) 1599 : audit [DBG] from='client.? 172.21.15.6:0/402521856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:58.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:57 smithi006 bash[33918]: cluster 2023-08-28T00:18:56.381461+0000 mgr.y (mgr.14901) 4211 : cluster [DBG] pgmap v4218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:58.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:57 smithi006 bash[33918]: audit 2023-08-28T00:18:57.157953+0000 mon.c (mon.2) 1599 : audit [DBG] from='client.? 172.21.15.6:0/402521856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:58.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:57 smithi006 bash[27292]: cluster 2023-08-28T00:18:56.381461+0000 mgr.y (mgr.14901) 4211 : cluster [DBG] pgmap v4218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:18:58.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:57 smithi006 bash[27292]: audit 2023-08-28T00:18:57.157953+0000 mon.c (mon.2) 1599 : audit [DBG] from='client.? 172.21.15.6:0/402521856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:58.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:58 smithi130 bash[51302]: debug 2023-08-28T00:18:58.378+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:58.379404+0000) 2023-08-28T00:18:59.799 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:18:59 smithi130 bash[51302]: debug 2023-08-28T00:18:59.378+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:59.379706+0000) 2023-08-28T00:19:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:59 smithi130 bash[27579]: cluster 2023-08-28T00:18:58.382110+0000 mgr.y (mgr.14901) 4212 : cluster [DBG] pgmap v4219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:18:59 smithi130 bash[27579]: audit 2023-08-28T00:18:59.581623+0000 mon.c (mon.2) 1600 : audit [DBG] from='client.? 172.21.15.6:0/2892489795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:00.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:59 smithi006 bash[33918]: cluster 2023-08-28T00:18:58.382110+0000 mgr.y (mgr.14901) 4212 : cluster [DBG] pgmap v4219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:00.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:18:59 smithi006 bash[33918]: audit 2023-08-28T00:18:59.581623+0000 mon.c (mon.2) 1600 : audit [DBG] from='client.? 172.21.15.6:0/2892489795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:00.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:59 smithi006 bash[27292]: cluster 2023-08-28T00:18:58.382110+0000 mgr.y (mgr.14901) 4212 : cluster [DBG] pgmap v4219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:00.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:18:59 smithi006 bash[27292]: audit 2023-08-28T00:18:59.581623+0000 mon.c (mon.2) 1600 : audit [DBG] from='client.? 172.21.15.6:0/2892489795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:00.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:00 smithi130 bash[51302]: debug 2023-08-28T00:19:00.378+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:00.380014+0000) 2023-08-28T00:19:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:19:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:19:00] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:19:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:19:00.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:19:00.702Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:01.807 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:01 smithi130 bash[51302]: debug 2023-08-28T00:19:01.378+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:01.380296+0000) 2023-08-28T00:19:02.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:01 smithi130 bash[27579]: cluster 2023-08-28T00:19:00.383225+0000 mgr.y (mgr.14901) 4213 : cluster [DBG] pgmap v4220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:02.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:01 smithi006 bash[27292]: cluster 2023-08-28T00:19:00.383225+0000 mgr.y (mgr.14901) 4213 : cluster [DBG] pgmap v4220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:02.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:01 smithi006 bash[33918]: cluster 2023-08-28T00:19:00.383225+0000 mgr.y (mgr.14901) 4213 : cluster [DBG] pgmap v4220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:19:02.280Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:19:02.283Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:02.818 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:02 smithi130 bash[51302]: debug 2023-08-28T00:19:02.378+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:02.380573+0000) 2023-08-28T00:19:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:02 smithi130 bash[27579]: audit 2023-08-28T00:19:01.998710+0000 mon.c (mon.2) 1601 : audit [DBG] from='client.? 172.21.15.6:0/3882616111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:03.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:02 smithi006 bash[27292]: audit 2023-08-28T00:19:01.998710+0000 mon.c (mon.2) 1601 : audit [DBG] from='client.? 172.21.15.6:0/3882616111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:02 smithi006 bash[33918]: audit 2023-08-28T00:19:01.998710+0000 mon.c (mon.2) 1601 : audit [DBG] from='client.? 172.21.15.6:0/3882616111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:03.821 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:03 smithi130 bash[51302]: debug 2023-08-28T00:19:03.378+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:03.380883+0000) 2023-08-28T00:19:04.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:03 smithi130 bash[27579]: cluster 2023-08-28T00:19:02.383927+0000 mgr.y (mgr.14901) 4214 : cluster [DBG] pgmap v4221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:04.166 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:03 smithi006 bash[33918]: cluster 2023-08-28T00:19:02.383927+0000 mgr.y (mgr.14901) 4214 : cluster [DBG] pgmap v4221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:04.167 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:03 smithi006 bash[27292]: cluster 2023-08-28T00:19:02.383927+0000 mgr.y (mgr.14901) 4214 : cluster [DBG] pgmap v4221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:19:04.164Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:19:04.166Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:04.830 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:04 smithi130 bash[51302]: debug 2023-08-28T00:19:04.378+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:04.381151+0000) 2023-08-28T00:19:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:04 smithi130 bash[27579]: audit 2023-08-28T00:19:04.410761+0000 mon.a (mon.0) 3141 : audit [DBG] from='client.? 172.21.15.6:0/2055936987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:05.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:04 smithi006 bash[27292]: audit 2023-08-28T00:19:04.410761+0000 mon.a (mon.0) 3141 : audit [DBG] from='client.? 172.21.15.6:0/2055936987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:05.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:04 smithi006 bash[33918]: audit 2023-08-28T00:19:04.410761+0000 mon.a (mon.0) 3141 : audit [DBG] from='client.? 172.21.15.6:0/2055936987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:05.832 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:05 smithi130 bash[51302]: debug 2023-08-28T00:19:05.378+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:05.381446+0000) 2023-08-28T00:19:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:05 smithi130 bash[27579]: cluster 2023-08-28T00:19:04.385042+0000 mgr.y (mgr.14901) 4215 : cluster [DBG] pgmap v4222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:06.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:05 smithi006 bash[27292]: cluster 2023-08-28T00:19:04.385042+0000 mgr.y (mgr.14901) 4215 : cluster [DBG] pgmap v4222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:05 smithi006 bash[33918]: cluster 2023-08-28T00:19:04.385042+0000 mgr.y (mgr.14901) 4215 : cluster [DBG] pgmap v4222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:06.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:06 smithi130 bash[51302]: debug 2023-08-28T00:19:06.378+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:06.381707+0000) 2023-08-28T00:19:07.837 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:07 smithi130 bash[51302]: debug 2023-08-28T00:19:07.378+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:07.381959+0000) 2023-08-28T00:19:08.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:07 smithi130 bash[27579]: cluster 2023-08-28T00:19:06.385633+0000 mgr.y (mgr.14901) 4216 : cluster [DBG] pgmap v4223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:08.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:07 smithi130 bash[27579]: audit 2023-08-28T00:19:06.836341+0000 mon.a (mon.0) 3142 : audit [DBG] from='client.? 172.21.15.6:0/4082251057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:08.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:07 smithi006 bash[33918]: cluster 2023-08-28T00:19:06.385633+0000 mgr.y (mgr.14901) 4216 : cluster [DBG] pgmap v4223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:08.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:07 smithi006 bash[33918]: audit 2023-08-28T00:19:06.836341+0000 mon.a (mon.0) 3142 : audit [DBG] from='client.? 172.21.15.6:0/4082251057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:08.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:07 smithi006 bash[27292]: cluster 2023-08-28T00:19:06.385633+0000 mgr.y (mgr.14901) 4216 : cluster [DBG] pgmap v4223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:08.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:07 smithi006 bash[27292]: audit 2023-08-28T00:19:06.836341+0000 mon.a (mon.0) 3142 : audit [DBG] from='client.? 172.21.15.6:0/4082251057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:08.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:08 smithi130 bash[51302]: debug 2023-08-28T00:19:08.378+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:08.382276+0000) 2023-08-28T00:19:09.841 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:09 smithi130 bash[51302]: debug 2023-08-28T00:19:09.378+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:09.382569+0000) 2023-08-28T00:19:10.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:09 smithi130 bash[27579]: cluster 2023-08-28T00:19:08.386327+0000 mgr.y (mgr.14901) 4217 : cluster [DBG] pgmap v4224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:10.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:09 smithi130 bash[27579]: audit 2023-08-28T00:19:09.241360+0000 mon.a (mon.0) 3143 : audit [DBG] from='client.? 172.21.15.6:0/3646351748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:10.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:09 smithi006 bash[33918]: cluster 2023-08-28T00:19:08.386327+0000 mgr.y (mgr.14901) 4217 : cluster [DBG] pgmap v4224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:10.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:09 smithi006 bash[33918]: audit 2023-08-28T00:19:09.241360+0000 mon.a (mon.0) 3143 : audit [DBG] from='client.? 172.21.15.6:0/3646351748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:10.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:09 smithi006 bash[27292]: cluster 2023-08-28T00:19:08.386327+0000 mgr.y (mgr.14901) 4217 : cluster [DBG] pgmap v4224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:10.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:09 smithi006 bash[27292]: audit 2023-08-28T00:19:09.241360+0000 mon.a (mon.0) 3143 : audit [DBG] from='client.? 172.21.15.6:0/3646351748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:10.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:10 smithi130 bash[51302]: debug 2023-08-28T00:19:10.382+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:10.382903+0000) 2023-08-28T00:19:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:19:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:19:10] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:19:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:19:10.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:10.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:19:10.703Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:11.846 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:11 smithi130 bash[51302]: debug 2023-08-28T00:19:11.382+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:11.383208+0000) 2023-08-28T00:19:12.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:11 smithi130 bash[27579]: cluster 2023-08-28T00:19:10.387358+0000 mgr.y (mgr.14901) 4218 : cluster [DBG] pgmap v4225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:12.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:11 smithi130 bash[27579]: audit 2023-08-28T00:19:11.655422+0000 mon.c (mon.2) 1602 : audit [DBG] from='client.? 172.21.15.6:0/3749889397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:12.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:11 smithi006 bash[27292]: cluster 2023-08-28T00:19:10.387358+0000 mgr.y (mgr.14901) 4218 : cluster [DBG] pgmap v4225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:12.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:11 smithi006 bash[27292]: audit 2023-08-28T00:19:11.655422+0000 mon.c (mon.2) 1602 : audit [DBG] from='client.? 172.21.15.6:0/3749889397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:12.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:11 smithi006 bash[33918]: cluster 2023-08-28T00:19:10.387358+0000 mgr.y (mgr.14901) 4218 : cluster [DBG] pgmap v4225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:12.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:11 smithi006 bash[33918]: audit 2023-08-28T00:19:11.655422+0000 mon.c (mon.2) 1602 : audit [DBG] from='client.? 172.21.15.6:0/3749889397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:19:12.281Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:19:12.283Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:12 smithi130 bash[51302]: debug 2023-08-28T00:19:12.382+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:12.383473+0000) 2023-08-28T00:19:13.850 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:13 smithi130 bash[51302]: debug 2023-08-28T00:19:13.382+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:13.383753+0000) 2023-08-28T00:19:14.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:13 smithi130 bash[27579]: cluster 2023-08-28T00:19:12.388075+0000 mgr.y (mgr.14901) 4219 : cluster [DBG] pgmap v4226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:14.167 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:13 smithi006 bash[27292]: cluster 2023-08-28T00:19:12.388075+0000 mgr.y (mgr.14901) 4219 : cluster [DBG] pgmap v4226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:14.168 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:13 smithi006 bash[33918]: cluster 2023-08-28T00:19:12.388075+0000 mgr.y (mgr.14901) 4219 : cluster [DBG] pgmap v4226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:19:14.165Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:19:14.167Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:14.858 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:14 smithi130 bash[51302]: debug 2023-08-28T00:19:14.382+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:14.384059+0000) 2023-08-28T00:19:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:14 smithi130 bash[27579]: audit 2023-08-28T00:19:14.075954+0000 mon.c (mon.2) 1603 : audit [DBG] from='client.? 172.21.15.6:0/474575724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:14 smithi006 bash[33918]: audit 2023-08-28T00:19:14.075954+0000 mon.c (mon.2) 1603 : audit [DBG] from='client.? 172.21.15.6:0/474575724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:14 smithi006 bash[27292]: audit 2023-08-28T00:19:14.075954+0000 mon.c (mon.2) 1603 : audit [DBG] from='client.? 172.21.15.6:0/474575724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:15.862 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:15 smithi130 bash[51302]: debug 2023-08-28T00:19:15.382+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:15.384277+0000) 2023-08-28T00:19:16.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:15 smithi130 bash[27579]: cluster 2023-08-28T00:19:14.389381+0000 mgr.y (mgr.14901) 4220 : cluster [DBG] pgmap v4227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:16.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:15 smithi006 bash[33918]: cluster 2023-08-28T00:19:14.389381+0000 mgr.y (mgr.14901) 4220 : cluster [DBG] pgmap v4227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:16.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:15 smithi006 bash[27292]: cluster 2023-08-28T00:19:14.389381+0000 mgr.y (mgr.14901) 4220 : cluster [DBG] pgmap v4227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:16.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:16 smithi130 bash[51302]: debug 2023-08-28T00:19:16.382+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:16.384611+0000) 2023-08-28T00:19:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:16 smithi006 bash[33918]: audit 2023-08-28T00:19:16.485195+0000 mon.c (mon.2) 1604 : audit [DBG] from='client.? 172.21.15.6:0/4086192323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:17.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:16 smithi006 bash[27292]: audit 2023-08-28T00:19:16.485195+0000 mon.c (mon.2) 1604 : audit [DBG] from='client.? 172.21.15.6:0/4086192323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:17.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:16 smithi130 bash[27579]: audit 2023-08-28T00:19:16.485195+0000 mon.c (mon.2) 1604 : audit [DBG] from='client.? 172.21.15.6:0/4086192323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:17.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:17 smithi130 bash[51302]: debug 2023-08-28T00:19:17.382+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:17.384895+0000) 2023-08-28T00:19:18.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:17 smithi006 bash[33918]: cluster 2023-08-28T00:19:16.389860+0000 mgr.y (mgr.14901) 4221 : cluster [DBG] pgmap v4228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:18.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:17 smithi006 bash[27292]: cluster 2023-08-28T00:19:16.389860+0000 mgr.y (mgr.14901) 4221 : cluster [DBG] pgmap v4228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:17 smithi130 bash[27579]: cluster 2023-08-28T00:19:16.389860+0000 mgr.y (mgr.14901) 4221 : cluster [DBG] pgmap v4228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:18 smithi130 bash[51302]: debug 2023-08-28T00:19:18.382+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:18.385170+0000) 2023-08-28T00:19:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:19 smithi130 bash[51302]: debug 2023-08-28T00:19:19.382+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:19.385483+0000) 2023-08-28T00:19:20.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:19 smithi006 bash[33918]: cluster 2023-08-28T00:19:18.390471+0000 mgr.y (mgr.14901) 4222 : cluster [DBG] pgmap v4229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:20.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:19 smithi006 bash[33918]: audit 2023-08-28T00:19:18.907597+0000 mon.c (mon.2) 1605 : audit [DBG] from='client.? 172.21.15.6:0/461483876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:19 smithi006 bash[27292]: cluster 2023-08-28T00:19:18.390471+0000 mgr.y (mgr.14901) 4222 : cluster [DBG] pgmap v4229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:19 smithi006 bash[27292]: audit 2023-08-28T00:19:18.907597+0000 mon.c (mon.2) 1605 : audit [DBG] from='client.? 172.21.15.6:0/461483876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:19 smithi130 bash[27579]: cluster 2023-08-28T00:19:18.390471+0000 mgr.y (mgr.14901) 4222 : cluster [DBG] pgmap v4229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:19 smithi130 bash[27579]: audit 2023-08-28T00:19:18.907597+0000 mon.c (mon.2) 1605 : audit [DBG] from='client.? 172.21.15.6:0/461483876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:20 smithi130 bash[51302]: debug 2023-08-28T00:19:20.382+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:20.385707+0000) 2023-08-28T00:19:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:19:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:19:20] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:19:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:19:20.702Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:19:20.704Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:21.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:21 smithi130 bash[51302]: debug 2023-08-28T00:19:21.381+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:21.385971+0000) 2023-08-28T00:19:22.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:21 smithi006 bash[33918]: cluster 2023-08-28T00:19:20.391622+0000 mgr.y (mgr.14901) 4223 : cluster [DBG] pgmap v4230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:22.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:21 smithi006 bash[33918]: audit 2023-08-28T00:19:21.315380+0000 mon.a (mon.0) 3144 : audit [DBG] from='client.? 172.21.15.6:0/3078005992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:22.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:21 smithi006 bash[27292]: cluster 2023-08-28T00:19:20.391622+0000 mgr.y (mgr.14901) 4223 : cluster [DBG] pgmap v4230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:22.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:21 smithi006 bash[27292]: audit 2023-08-28T00:19:21.315380+0000 mon.a (mon.0) 3144 : audit [DBG] from='client.? 172.21.15.6:0/3078005992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:22.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:21 smithi130 bash[27579]: cluster 2023-08-28T00:19:20.391622+0000 mgr.y (mgr.14901) 4223 : cluster [DBG] pgmap v4230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:22.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:21 smithi130 bash[27579]: audit 2023-08-28T00:19:21.315380+0000 mon.a (mon.0) 3144 : audit [DBG] from='client.? 172.21.15.6:0/3078005992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:19:22.282Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:19:22.283Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:22 smithi130 bash[51302]: debug 2023-08-28T00:19:22.381+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:22.386271+0000) 2023-08-28T00:19:23.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:23 smithi130 bash[51302]: debug 2023-08-28T00:19:23.381+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:23.386500+0000) 2023-08-28T00:19:24.168 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:23 smithi006 bash[27292]: cluster 2023-08-28T00:19:22.392227+0000 mgr.y (mgr.14901) 4224 : cluster [DBG] pgmap v4231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:24.168 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:23 smithi006 bash[27292]: audit 2023-08-28T00:19:23.727712+0000 mon.c (mon.2) 1606 : audit [DBG] from='client.? 172.21.15.6:0/843543535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:24.168 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:23 smithi006 bash[33918]: cluster 2023-08-28T00:19:22.392227+0000 mgr.y (mgr.14901) 4224 : cluster [DBG] pgmap v4231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:24.169 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:23 smithi006 bash[33918]: audit 2023-08-28T00:19:23.727712+0000 mon.c (mon.2) 1606 : audit [DBG] from='client.? 172.21.15.6:0/843543535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:23 smithi130 bash[27579]: cluster 2023-08-28T00:19:22.392227+0000 mgr.y (mgr.14901) 4224 : cluster [DBG] pgmap v4231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:23 smithi130 bash[27579]: audit 2023-08-28T00:19:23.727712+0000 mon.c (mon.2) 1606 : audit [DBG] from='client.? 172.21.15.6:0/843543535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:19:24.165Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:19:24.168Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:24 smithi130 bash[51302]: debug 2023-08-28T00:19:24.381+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:24.386756+0000) 2023-08-28T00:19:25.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:25 smithi130 bash[51302]: debug 2023-08-28T00:19:25.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:25.386961+0000) 2023-08-28T00:19:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:25 smithi006 bash[27292]: cluster 2023-08-28T00:19:24.393345+0000 mgr.y (mgr.14901) 4225 : cluster [DBG] pgmap v4232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:26.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:25 smithi006 bash[33918]: cluster 2023-08-28T00:19:24.393345+0000 mgr.y (mgr.14901) 4225 : cluster [DBG] pgmap v4232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:25 smithi130 bash[27579]: cluster 2023-08-28T00:19:24.393345+0000 mgr.y (mgr.14901) 4225 : cluster [DBG] pgmap v4232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:26.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:26 smithi130 bash[51302]: debug 2023-08-28T00:19:26.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:26.387233+0000) 2023-08-28T00:19:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:26 smithi006 bash[27292]: audit 2023-08-28T00:19:26.143451+0000 mon.a (mon.0) 3145 : audit [DBG] from='client.? 172.21.15.6:0/1828859253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:26 smithi006 bash[33918]: audit 2023-08-28T00:19:26.143451+0000 mon.a (mon.0) 3145 : audit [DBG] from='client.? 172.21.15.6:0/1828859253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:27.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:26 smithi130 bash[27579]: audit 2023-08-28T00:19:26.143451+0000 mon.a (mon.0) 3145 : audit [DBG] from='client.? 172.21.15.6:0/1828859253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:27.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:27 smithi130 bash[51302]: debug 2023-08-28T00:19:27.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:27.387485+0000) 2023-08-28T00:19:28.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:27 smithi006 bash[27292]: cluster 2023-08-28T00:19:26.394158+0000 mgr.y (mgr.14901) 4226 : cluster [DBG] pgmap v4233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:28.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:27 smithi006 bash[33918]: cluster 2023-08-28T00:19:26.394158+0000 mgr.y (mgr.14901) 4226 : cluster [DBG] pgmap v4233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:28.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:27 smithi130 bash[27579]: cluster 2023-08-28T00:19:26.394158+0000 mgr.y (mgr.14901) 4226 : cluster [DBG] pgmap v4233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:28 smithi130 bash[51302]: debug 2023-08-28T00:19:28.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:28.387688+0000) 2023-08-28T00:19:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:28 smithi006 bash[33918]: audit 2023-08-28T00:19:28.556212+0000 mon.c (mon.2) 1607 : audit [DBG] from='client.? 172.21.15.6:0/1852318274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:29.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:28 smithi006 bash[27292]: audit 2023-08-28T00:19:28.556212+0000 mon.c (mon.2) 1607 : audit [DBG] from='client.? 172.21.15.6:0/1852318274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:29.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:28 smithi130 bash[27579]: audit 2023-08-28T00:19:28.556212+0000 mon.c (mon.2) 1607 : audit [DBG] from='client.? 172.21.15.6:0/1852318274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:29.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:29 smithi130 bash[51302]: debug 2023-08-28T00:19:29.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:29.387968+0000) 2023-08-28T00:19:30.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:29 smithi006 bash[33918]: cluster 2023-08-28T00:19:28.394923+0000 mgr.y (mgr.14901) 4227 : cluster [DBG] pgmap v4234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:30.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:29 smithi006 bash[27292]: cluster 2023-08-28T00:19:28.394923+0000 mgr.y (mgr.14901) 4227 : cluster [DBG] pgmap v4234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:29 smithi130 bash[27579]: cluster 2023-08-28T00:19:28.394923+0000 mgr.y (mgr.14901) 4227 : cluster [DBG] pgmap v4234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:30 smithi130 bash[51302]: debug 2023-08-28T00:19:30.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:30.388165+0000) 2023-08-28T00:19:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:19:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:19:30] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:19:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:19:30.703Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:19:30.705Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:31.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:31 smithi130 bash[51302]: debug 2023-08-28T00:19:31.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:31.388444+0000) 2023-08-28T00:19:32.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:31 smithi006 bash[27292]: cluster 2023-08-28T00:19:30.396085+0000 mgr.y (mgr.14901) 4228 : cluster [DBG] pgmap v4235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:32.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:31 smithi006 bash[27292]: audit 2023-08-28T00:19:30.978139+0000 mon.c (mon.2) 1608 : audit [DBG] from='client.? 172.21.15.6:0/4231148111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:32.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:31 smithi006 bash[33918]: cluster 2023-08-28T00:19:30.396085+0000 mgr.y (mgr.14901) 4228 : cluster [DBG] pgmap v4235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:32.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:31 smithi006 bash[33918]: audit 2023-08-28T00:19:30.978139+0000 mon.c (mon.2) 1608 : audit [DBG] from='client.? 172.21.15.6:0/4231148111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:31 smithi130 bash[27579]: cluster 2023-08-28T00:19:30.396085+0000 mgr.y (mgr.14901) 4228 : cluster [DBG] pgmap v4235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:31 smithi130 bash[27579]: audit 2023-08-28T00:19:30.978139+0000 mon.c (mon.2) 1608 : audit [DBG] from='client.? 172.21.15.6:0/4231148111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:19:32.282Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:19:32.284Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:32 smithi130 bash[51302]: debug 2023-08-28T00:19:32.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:32.388700+0000) 2023-08-28T00:19:33.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:33 smithi130 bash[51302]: debug 2023-08-28T00:19:33.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:33.388993+0000) 2023-08-28T00:19:34.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:33 smithi006 bash[27292]: cluster 2023-08-28T00:19:32.396778+0000 mgr.y (mgr.14901) 4229 : cluster [DBG] pgmap v4236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:34.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:33 smithi006 bash[27292]: audit 2023-08-28T00:19:33.391468+0000 mon.c (mon.2) 1609 : audit [DBG] from='client.? 172.21.15.6:0/3918854404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:34.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:33 smithi006 bash[33918]: cluster 2023-08-28T00:19:32.396778+0000 mgr.y (mgr.14901) 4229 : cluster [DBG] pgmap v4236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:34.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:33 smithi006 bash[33918]: audit 2023-08-28T00:19:33.391468+0000 mon.c (mon.2) 1609 : audit [DBG] from='client.? 172.21.15.6:0/3918854404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:34.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:19:34.166Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:34.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:19:34.168Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:34.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:33 smithi130 bash[27579]: cluster 2023-08-28T00:19:32.396778+0000 mgr.y (mgr.14901) 4229 : cluster [DBG] pgmap v4236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:34.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:33 smithi130 bash[27579]: audit 2023-08-28T00:19:33.391468+0000 mon.c (mon.2) 1609 : audit [DBG] from='client.? 172.21.15.6:0/3918854404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:34 smithi130 bash[51302]: debug 2023-08-28T00:19:34.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:34.389259+0000) 2023-08-28T00:19:35.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:35 smithi130 bash[51302]: debug 2023-08-28T00:19:35.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:35.389505+0000) 2023-08-28T00:19:36.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:35 smithi006 bash[27292]: cluster 2023-08-28T00:19:34.397969+0000 mgr.y (mgr.14901) 4230 : cluster [DBG] pgmap v4237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:36.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:35 smithi006 bash[27292]: audit 2023-08-28T00:19:35.805442+0000 mon.c (mon.2) 1610 : audit [DBG] from='client.? 172.21.15.6:0/3058775700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:36.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:35 smithi006 bash[33918]: cluster 2023-08-28T00:19:34.397969+0000 mgr.y (mgr.14901) 4230 : cluster [DBG] pgmap v4237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:36.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:35 smithi006 bash[33918]: audit 2023-08-28T00:19:35.805442+0000 mon.c (mon.2) 1610 : audit [DBG] from='client.? 172.21.15.6:0/3058775700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:35 smithi130 bash[27579]: cluster 2023-08-28T00:19:34.397969+0000 mgr.y (mgr.14901) 4230 : cluster [DBG] pgmap v4237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:35 smithi130 bash[27579]: audit 2023-08-28T00:19:35.805442+0000 mon.c (mon.2) 1610 : audit [DBG] from='client.? 172.21.15.6:0/3058775700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:36 smithi130 bash[51302]: debug 2023-08-28T00:19:36.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:36.389769+0000) 2023-08-28T00:19:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:37 smithi130 bash[51302]: debug 2023-08-28T00:19:37.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:37.390054+0000) 2023-08-28T00:19:38.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:37 smithi006 bash[27292]: cluster 2023-08-28T00:19:36.398721+0000 mgr.y (mgr.14901) 4231 : cluster [DBG] pgmap v4238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:38.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:37 smithi006 bash[33918]: cluster 2023-08-28T00:19:36.398721+0000 mgr.y (mgr.14901) 4231 : cluster [DBG] pgmap v4238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:38.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:37 smithi130 bash[27579]: cluster 2023-08-28T00:19:36.398721+0000 mgr.y (mgr.14901) 4231 : cluster [DBG] pgmap v4238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:38 smithi130 bash[51302]: debug 2023-08-28T00:19:38.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:38.390385+0000) 2023-08-28T00:19:39.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:38 smithi006 bash[27292]: audit 2023-08-28T00:19:38.213122+0000 mon.c (mon.2) 1611 : audit [DBG] from='client.? 172.21.15.6:0/2425975514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:39.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:38 smithi006 bash[33918]: audit 2023-08-28T00:19:38.213122+0000 mon.c (mon.2) 1611 : audit [DBG] from='client.? 172.21.15.6:0/2425975514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:38 smithi130 bash[27579]: audit 2023-08-28T00:19:38.213122+0000 mon.c (mon.2) 1611 : audit [DBG] from='client.? 172.21.15.6:0/2425975514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:39 smithi130 bash[51302]: debug 2023-08-28T00:19:39.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:39.390703+0000) 2023-08-28T00:19:40.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:39 smithi006 bash[27292]: cluster 2023-08-28T00:19:38.399446+0000 mgr.y (mgr.14901) 4232 : cluster [DBG] pgmap v4239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:40.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:39 smithi006 bash[27292]: audit 2023-08-28T00:19:38.983619+0000 mon.b (mon.1) 528 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:19:40.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:39 smithi006 bash[33918]: cluster 2023-08-28T00:19:38.399446+0000 mgr.y (mgr.14901) 4232 : cluster [DBG] pgmap v4239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:40.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:39 smithi006 bash[33918]: audit 2023-08-28T00:19:38.983619+0000 mon.b (mon.1) 528 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:19:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:39 smithi130 bash[27579]: cluster 2023-08-28T00:19:38.399446+0000 mgr.y (mgr.14901) 4232 : cluster [DBG] pgmap v4239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:39 smithi130 bash[27579]: audit 2023-08-28T00:19:38.983619+0000 mon.b (mon.1) 528 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:19:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:40 smithi130 bash[51302]: debug 2023-08-28T00:19:40.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:40.390993+0000) 2023-08-28T00:19:40.959 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:19:40.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:40.959 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:19:40.706Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:40.959 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:19:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:19:40] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:19:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:40 smithi006 bash[27292]: audit 2023-08-28T00:19:40.623448+0000 mon.a (mon.0) 3146 : audit [DBG] from='client.? 172.21.15.6:0/641680355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:41.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:40 smithi006 bash[33918]: audit 2023-08-28T00:19:40.623448+0000 mon.a (mon.0) 3146 : audit [DBG] from='client.? 172.21.15.6:0/641680355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:41.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:40 smithi130 bash[27579]: audit 2023-08-28T00:19:40.623448+0000 mon.a (mon.0) 3146 : audit [DBG] from='client.? 172.21.15.6:0/641680355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:41.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:41 smithi130 bash[51302]: debug 2023-08-28T00:19:41.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:41.391225+0000) 2023-08-28T00:19:42.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:41 smithi006 bash[33918]: cluster 2023-08-28T00:19:40.400489+0000 mgr.y (mgr.14901) 4233 : cluster [DBG] pgmap v4240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:42.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:41 smithi006 bash[27292]: cluster 2023-08-28T00:19:40.400489+0000 mgr.y (mgr.14901) 4233 : cluster [DBG] pgmap v4240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:41 smithi130 bash[27579]: cluster 2023-08-28T00:19:40.400489+0000 mgr.y (mgr.14901) 4233 : cluster [DBG] pgmap v4240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:19:42.283Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:19:42.285Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:42 smithi130 bash[51302]: debug 2023-08-28T00:19:42.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:42.391446+0000) 2023-08-28T00:19:43.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:43 smithi130 bash[51302]: debug 2023-08-28T00:19:43.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:43.391731+0000) 2023-08-28T00:19:44.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:43 smithi006 bash[27292]: cluster 2023-08-28T00:19:42.401171+0000 mgr.y (mgr.14901) 4234 : cluster [DBG] pgmap v4241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:44.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:43 smithi006 bash[27292]: audit 2023-08-28T00:19:43.039412+0000 mon.a (mon.0) 3147 : audit [DBG] from='client.? 172.21.15.6:0/3269530749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:44.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:43 smithi006 bash[33918]: cluster 2023-08-28T00:19:42.401171+0000 mgr.y (mgr.14901) 4234 : cluster [DBG] pgmap v4241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:44.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:43 smithi006 bash[33918]: audit 2023-08-28T00:19:43.039412+0000 mon.a (mon.0) 3147 : audit [DBG] from='client.? 172.21.15.6:0/3269530749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:44.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:19:44.167Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:44.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:19:44.168Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:43 smithi130 bash[27579]: cluster 2023-08-28T00:19:42.401171+0000 mgr.y (mgr.14901) 4234 : cluster [DBG] pgmap v4241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:43 smithi130 bash[27579]: audit 2023-08-28T00:19:43.039412+0000 mon.a (mon.0) 3147 : audit [DBG] from='client.? 172.21.15.6:0/3269530749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:44 smithi130 bash[51302]: debug 2023-08-28T00:19:44.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:44.392001+0000) 2023-08-28T00:19:45.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:45 smithi130 bash[51302]: debug 2023-08-28T00:19:45.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:45.392294+0000) 2023-08-28T00:19:46.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:45 smithi006 bash[27292]: cluster 2023-08-28T00:19:44.402055+0000 mgr.y (mgr.14901) 4235 : cluster [DBG] pgmap v4242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:46.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:45 smithi006 bash[27292]: audit 2023-08-28T00:19:44.915598+0000 mon.a (mon.0) 3148 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:19:46.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:45 smithi006 bash[27292]: audit 2023-08-28T00:19:45.289783+0000 mon.a (mon.0) 3149 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:19:46.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:45 smithi006 bash[27292]: audit 2023-08-28T00:19:45.466926+0000 mon.c (mon.2) 1612 : audit [DBG] from='client.? 172.21.15.6:0/3854743759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:46.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:45 smithi006 bash[27292]: audit 2023-08-28T00:19:45.582925+0000 mon.b (mon.1) 529 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:19:46.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:45 smithi006 bash[27292]: audit 2023-08-28T00:19:45.585257+0000 mon.b (mon.1) 530 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:19:46.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:45 smithi006 bash[27292]: audit 2023-08-28T00:19:45.594985+0000 mon.a (mon.0) 3150 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:19:46.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:45 smithi006 bash[33918]: cluster 2023-08-28T00:19:44.402055+0000 mgr.y (mgr.14901) 4235 : cluster [DBG] pgmap v4242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:46.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:45 smithi006 bash[33918]: audit 2023-08-28T00:19:44.915598+0000 mon.a (mon.0) 3148 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:19:46.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:45 smithi006 bash[33918]: audit 2023-08-28T00:19:45.289783+0000 mon.a (mon.0) 3149 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:19:46.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:45 smithi006 bash[33918]: audit 2023-08-28T00:19:45.466926+0000 mon.c (mon.2) 1612 : audit [DBG] from='client.? 172.21.15.6:0/3854743759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:46.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:45 smithi006 bash[33918]: audit 2023-08-28T00:19:45.582925+0000 mon.b (mon.1) 529 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:19:46.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:45 smithi006 bash[33918]: audit 2023-08-28T00:19:45.585257+0000 mon.b (mon.1) 530 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:19:46.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:45 smithi006 bash[33918]: audit 2023-08-28T00:19:45.594985+0000 mon.a (mon.0) 3150 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:19:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:45 smithi130 bash[27579]: cluster 2023-08-28T00:19:44.402055+0000 mgr.y (mgr.14901) 4235 : cluster [DBG] pgmap v4242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:45 smithi130 bash[27579]: audit 2023-08-28T00:19:44.915598+0000 mon.a (mon.0) 3148 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:19:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:45 smithi130 bash[27579]: audit 2023-08-28T00:19:45.289783+0000 mon.a (mon.0) 3149 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:19:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:45 smithi130 bash[27579]: audit 2023-08-28T00:19:45.466926+0000 mon.c (mon.2) 1612 : audit [DBG] from='client.? 172.21.15.6:0/3854743759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:46.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:45 smithi130 bash[27579]: audit 2023-08-28T00:19:45.582925+0000 mon.b (mon.1) 529 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:19:46.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:45 smithi130 bash[27579]: audit 2023-08-28T00:19:45.585257+0000 mon.b (mon.1) 530 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:19:46.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:45 smithi130 bash[27579]: audit 2023-08-28T00:19:45.594985+0000 mon.a (mon.0) 3150 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:19:46.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:46 smithi130 bash[51302]: debug 2023-08-28T00:19:46.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:46.392585+0000) 2023-08-28T00:19:47.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:47 smithi130 bash[51302]: debug 2023-08-28T00:19:47.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:47.392846+0000) 2023-08-28T00:19:48.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:47 smithi006 bash[33918]: cluster 2023-08-28T00:19:46.402626+0000 mgr.y (mgr.14901) 4236 : cluster [DBG] pgmap v4243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:48.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:47 smithi006 bash[33918]: audit 2023-08-28T00:19:47.882693+0000 mon.c (mon.2) 1613 : audit [DBG] from='client.? 172.21.15.6:0/1475292004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:48.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:47 smithi006 bash[27292]: cluster 2023-08-28T00:19:46.402626+0000 mgr.y (mgr.14901) 4236 : cluster [DBG] pgmap v4243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:48.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:47 smithi006 bash[27292]: audit 2023-08-28T00:19:47.882693+0000 mon.c (mon.2) 1613 : audit [DBG] from='client.? 172.21.15.6:0/1475292004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:47 smithi130 bash[27579]: cluster 2023-08-28T00:19:46.402626+0000 mgr.y (mgr.14901) 4236 : cluster [DBG] pgmap v4243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:47 smithi130 bash[27579]: audit 2023-08-28T00:19:47.882693+0000 mon.c (mon.2) 1613 : audit [DBG] from='client.? 172.21.15.6:0/1475292004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:48 smithi130 bash[51302]: debug 2023-08-28T00:19:48.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:48.393106+0000) 2023-08-28T00:19:49.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:49 smithi130 bash[51302]: debug 2023-08-28T00:19:49.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:49.393389+0000) 2023-08-28T00:19:50.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:49 smithi006 bash[27292]: cluster 2023-08-28T00:19:48.403293+0000 mgr.y (mgr.14901) 4237 : cluster [DBG] pgmap v4244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:50.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:49 smithi006 bash[33918]: cluster 2023-08-28T00:19:48.403293+0000 mgr.y (mgr.14901) 4237 : cluster [DBG] pgmap v4244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:50.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:49 smithi130 bash[27579]: cluster 2023-08-28T00:19:48.403293+0000 mgr.y (mgr.14901) 4237 : cluster [DBG] pgmap v4244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:50 smithi130 bash[51302]: debug 2023-08-28T00:19:50.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:50.393702+0000) 2023-08-28T00:19:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:19:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:19:50] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:19:50.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:50 smithi006 bash[33918]: audit 2023-08-28T00:19:50.308478+0000 mon.c (mon.2) 1614 : audit [DBG] from='client.? 172.21.15.6:0/2761153120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:50.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:50 smithi006 bash[27292]: audit 2023-08-28T00:19:50.308478+0000 mon.c (mon.2) 1614 : audit [DBG] from='client.? 172.21.15.6:0/2761153120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:19:50.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:50.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:19:50.706Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:51.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:50 smithi130 bash[27579]: audit 2023-08-28T00:19:50.308478+0000 mon.c (mon.2) 1614 : audit [DBG] from='client.? 172.21.15.6:0/2761153120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:51.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:51 smithi130 bash[51302]: debug 2023-08-28T00:19:51.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:51.393981+0000) 2023-08-28T00:19:52.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:51 smithi006 bash[33918]: cluster 2023-08-28T00:19:50.404745+0000 mgr.y (mgr.14901) 4238 : cluster [DBG] pgmap v4245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:52.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:51 smithi006 bash[27292]: cluster 2023-08-28T00:19:50.404745+0000 mgr.y (mgr.14901) 4238 : cluster [DBG] pgmap v4245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:51 smithi130 bash[27579]: cluster 2023-08-28T00:19:50.404745+0000 mgr.y (mgr.14901) 4238 : cluster [DBG] pgmap v4245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:19:52.284Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:19:52.286Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:52.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:52 smithi130 bash[51302]: debug 2023-08-28T00:19:52.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:52.394257+0000) 2023-08-28T00:19:53.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:52 smithi006 bash[33918]: audit 2023-08-28T00:19:52.736250+0000 mon.c (mon.2) 1615 : audit [DBG] from='client.? 172.21.15.6:0/3209656957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:52 smithi006 bash[27292]: audit 2023-08-28T00:19:52.736250+0000 mon.c (mon.2) 1615 : audit [DBG] from='client.? 172.21.15.6:0/3209656957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:52 smithi130 bash[27579]: audit 2023-08-28T00:19:52.736250+0000 mon.c (mon.2) 1615 : audit [DBG] from='client.? 172.21.15.6:0/3209656957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:53.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:53 smithi130 bash[51302]: debug 2023-08-28T00:19:53.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:53.394524+0000) 2023-08-28T00:19:54.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:53 smithi006 bash[33918]: cluster 2023-08-28T00:19:52.405444+0000 mgr.y (mgr.14901) 4239 : cluster [DBG] pgmap v4246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:54.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:53 smithi006 bash[27292]: cluster 2023-08-28T00:19:52.405444+0000 mgr.y (mgr.14901) 4239 : cluster [DBG] pgmap v4246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:54.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:19:54.168Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:54.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:19:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:19:54.170Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:19:54.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:53 smithi130 bash[27579]: cluster 2023-08-28T00:19:52.405444+0000 mgr.y (mgr.14901) 4239 : cluster [DBG] pgmap v4246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:54.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:54 smithi130 bash[51302]: debug 2023-08-28T00:19:54.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:54.394802+0000) 2023-08-28T00:19:55.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:55 smithi130 bash[51302]: debug 2023-08-28T00:19:55.393+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:55.395067+0000) 2023-08-28T00:19:56.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:55 smithi006 bash[27292]: cluster 2023-08-28T00:19:54.406517+0000 mgr.y (mgr.14901) 4240 : cluster [DBG] pgmap v4247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:56.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:55 smithi006 bash[27292]: audit 2023-08-28T00:19:55.153178+0000 mon.a (mon.0) 3151 : audit [DBG] from='client.? 172.21.15.6:0/3333435528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:56.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:55 smithi006 bash[33918]: cluster 2023-08-28T00:19:54.406517+0000 mgr.y (mgr.14901) 4240 : cluster [DBG] pgmap v4247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:56.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:55 smithi006 bash[33918]: audit 2023-08-28T00:19:55.153178+0000 mon.a (mon.0) 3151 : audit [DBG] from='client.? 172.21.15.6:0/3333435528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:55 smithi130 bash[27579]: cluster 2023-08-28T00:19:54.406517+0000 mgr.y (mgr.14901) 4240 : cluster [DBG] pgmap v4247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:55 smithi130 bash[27579]: audit 2023-08-28T00:19:55.153178+0000 mon.a (mon.0) 3151 : audit [DBG] from='client.? 172.21.15.6:0/3333435528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:56.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:56 smithi130 bash[51302]: debug 2023-08-28T00:19:56.393+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:56.395301+0000) 2023-08-28T00:19:57.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:57 smithi130 bash[51302]: debug 2023-08-28T00:19:57.393+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:57.395573+0000) 2023-08-28T00:19:58.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:57 smithi006 bash[33918]: cluster 2023-08-28T00:19:56.407192+0000 mgr.y (mgr.14901) 4241 : cluster [DBG] pgmap v4248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:58.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:57 smithi006 bash[33918]: audit 2023-08-28T00:19:57.567704+0000 mon.c (mon.2) 1616 : audit [DBG] from='client.? 172.21.15.6:0/1271939709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:58.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:57 smithi006 bash[27292]: cluster 2023-08-28T00:19:56.407192+0000 mgr.y (mgr.14901) 4241 : cluster [DBG] pgmap v4248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:58.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:57 smithi006 bash[27292]: audit 2023-08-28T00:19:57.567704+0000 mon.c (mon.2) 1616 : audit [DBG] from='client.? 172.21.15.6:0/1271939709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:58.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:57 smithi130 bash[27579]: cluster 2023-08-28T00:19:56.407192+0000 mgr.y (mgr.14901) 4241 : cluster [DBG] pgmap v4248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:19:58.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:57 smithi130 bash[27579]: audit 2023-08-28T00:19:57.567704+0000 mon.c (mon.2) 1616 : audit [DBG] from='client.? 172.21.15.6:0/1271939709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:58.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:58 smithi130 bash[51302]: debug 2023-08-28T00:19:58.393+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:58.395789+0000) 2023-08-28T00:19:59.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:19:59 smithi130 bash[51302]: debug 2023-08-28T00:19:59.393+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:59.396102+0000) 2023-08-28T00:20:00.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:19:59 smithi130 bash[27579]: cluster 2023-08-28T00:19:58.407851+0000 mgr.y (mgr.14901) 4242 : cluster [DBG] pgmap v4249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:00.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:19:59 smithi006 bash[27292]: cluster 2023-08-28T00:19:58.407851+0000 mgr.y (mgr.14901) 4242 : cluster [DBG] pgmap v4249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:00.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:19:59 smithi006 bash[33918]: cluster 2023-08-28T00:19:58.407851+0000 mgr.y (mgr.14901) 4242 : cluster [DBG] pgmap v4249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:00.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:00 smithi130 bash[51302]: debug 2023-08-28T00:20:00.393+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:00.396469+0000) 2023-08-28T00:20:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:20:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:20:00] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:20:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:20:00.705Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:20:00.707Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:01.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:00 smithi130 bash[27579]: audit 2023-08-28T00:19:59.978410+0000 mon.c (mon.2) 1617 : audit [DBG] from='client.? 172.21.15.6:0/1928283406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:01.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:00 smithi130 bash[27579]: cluster 2023-08-28T00:20:00.000134+0000 mon.a (mon.0) 3152 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:20:01.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:00 smithi006 bash[27292]: audit 2023-08-28T00:19:59.978410+0000 mon.c (mon.2) 1617 : audit [DBG] from='client.? 172.21.15.6:0/1928283406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:01.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:00 smithi006 bash[27292]: cluster 2023-08-28T00:20:00.000134+0000 mon.a (mon.0) 3152 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:20:01.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:00 smithi006 bash[33918]: audit 2023-08-28T00:19:59.978410+0000 mon.c (mon.2) 1617 : audit [DBG] from='client.? 172.21.15.6:0/1928283406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:01.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:00 smithi006 bash[33918]: cluster 2023-08-28T00:20:00.000134+0000 mon.a (mon.0) 3152 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:20:01.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:01 smithi130 bash[51302]: debug 2023-08-28T00:20:01.393+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:01.396770+0000) 2023-08-28T00:20:02.288 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:01 smithi006 bash[27292]: cluster 2023-08-28T00:20:00.409060+0000 mgr.y (mgr.14901) 4243 : cluster [DBG] pgmap v4250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:02.288 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:01 smithi006 bash[33918]: cluster 2023-08-28T00:20:00.409060+0000 mgr.y (mgr.14901) 4243 : cluster [DBG] pgmap v4250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:02.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:01 smithi130 bash[27579]: cluster 2023-08-28T00:20:00.409060+0000 mgr.y (mgr.14901) 4243 : cluster [DBG] pgmap v4250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:20:02.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:20:02.287Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:02.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:02 smithi130 bash[51302]: debug 2023-08-28T00:20:02.393+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:02.397119+0000) 2023-08-28T00:20:03.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:02 smithi130 bash[27579]: audit 2023-08-28T00:20:02.389854+0000 mon.c (mon.2) 1618 : audit [DBG] from='client.? 172.21.15.6:0/1655972612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:03.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:02 smithi006 bash[27292]: audit 2023-08-28T00:20:02.389854+0000 mon.c (mon.2) 1618 : audit [DBG] from='client.? 172.21.15.6:0/1655972612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:03.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:02 smithi006 bash[33918]: audit 2023-08-28T00:20:02.389854+0000 mon.c (mon.2) 1618 : audit [DBG] from='client.? 172.21.15.6:0/1655972612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:03.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:03 smithi130 bash[51302]: debug 2023-08-28T00:20:03.393+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:03.397432+0000) 2023-08-28T00:20:04.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:03 smithi130 bash[27579]: cluster 2023-08-28T00:20:02.409680+0000 mgr.y (mgr.14901) 4244 : cluster [DBG] pgmap v4251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:04.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:03 smithi006 bash[27292]: cluster 2023-08-28T00:20:02.409680+0000 mgr.y (mgr.14901) 4244 : cluster [DBG] pgmap v4251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:04.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:03 smithi006 bash[33918]: cluster 2023-08-28T00:20:02.409680+0000 mgr.y (mgr.14901) 4244 : cluster [DBG] pgmap v4251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:20:04.169Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:20:04.171Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:04.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:04 smithi130 bash[51302]: debug 2023-08-28T00:20:04.393+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:04.397769+0000) 2023-08-28T00:20:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:05 smithi130 bash[27579]: audit 2023-08-28T00:20:04.806620+0000 mon.c (mon.2) 1619 : audit [DBG] from='client.? 172.21.15.6:0/3297985082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:05.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:04 smithi006 bash[27292]: audit 2023-08-28T00:20:04.806620+0000 mon.c (mon.2) 1619 : audit [DBG] from='client.? 172.21.15.6:0/3297985082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:05.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:04 smithi006 bash[33918]: audit 2023-08-28T00:20:04.806620+0000 mon.c (mon.2) 1619 : audit [DBG] from='client.? 172.21.15.6:0/3297985082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:05.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:05 smithi130 bash[51302]: debug 2023-08-28T00:20:05.393+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:05.398124+0000) 2023-08-28T00:20:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:06 smithi130 bash[27579]: cluster 2023-08-28T00:20:04.410781+0000 mgr.y (mgr.14901) 4245 : cluster [DBG] pgmap v4252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:06.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:06 smithi006 bash[33918]: cluster 2023-08-28T00:20:04.410781+0000 mgr.y (mgr.14901) 4245 : cluster [DBG] pgmap v4252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:06.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:06 smithi006 bash[27292]: cluster 2023-08-28T00:20:04.410781+0000 mgr.y (mgr.14901) 4245 : cluster [DBG] pgmap v4252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:06.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:06 smithi130 bash[51302]: debug 2023-08-28T00:20:06.393+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:06.398443+0000) 2023-08-28T00:20:07.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:07 smithi130 bash[51302]: debug 2023-08-28T00:20:07.397+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:07.398788+0000) 2023-08-28T00:20:08.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:08 smithi130 bash[27579]: cluster 2023-08-28T00:20:06.411253+0000 mgr.y (mgr.14901) 4246 : cluster [DBG] pgmap v4253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:08.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:08 smithi130 bash[27579]: audit 2023-08-28T00:20:07.220523+0000 mon.a (mon.0) 3153 : audit [DBG] from='client.? 172.21.15.6:0/3179882737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:08.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:08 smithi006 bash[27292]: cluster 2023-08-28T00:20:06.411253+0000 mgr.y (mgr.14901) 4246 : cluster [DBG] pgmap v4253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:08.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:08 smithi006 bash[27292]: audit 2023-08-28T00:20:07.220523+0000 mon.a (mon.0) 3153 : audit [DBG] from='client.? 172.21.15.6:0/3179882737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:08.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:08 smithi006 bash[33918]: cluster 2023-08-28T00:20:06.411253+0000 mgr.y (mgr.14901) 4246 : cluster [DBG] pgmap v4253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:08.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:08 smithi006 bash[33918]: audit 2023-08-28T00:20:07.220523+0000 mon.a (mon.0) 3153 : audit [DBG] from='client.? 172.21.15.6:0/3179882737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:08.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:08 smithi130 bash[51302]: debug 2023-08-28T00:20:08.397+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:08.399050+0000) 2023-08-28T00:20:09.870 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:09 smithi130 bash[51302]: debug 2023-08-28T00:20:09.397+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:09.399323+0000) 2023-08-28T00:20:10.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:10 smithi130 bash[27579]: cluster 2023-08-28T00:20:08.411916+0000 mgr.y (mgr.14901) 4247 : cluster [DBG] pgmap v4254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:10.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:10 smithi130 bash[27579]: audit 2023-08-28T00:20:09.645967+0000 mon.c (mon.2) 1620 : audit [DBG] from='client.? 172.21.15.6:0/2598664155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:10.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:10 smithi006 bash[27292]: cluster 2023-08-28T00:20:08.411916+0000 mgr.y (mgr.14901) 4247 : cluster [DBG] pgmap v4254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:10.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:10 smithi006 bash[27292]: audit 2023-08-28T00:20:09.645967+0000 mon.c (mon.2) 1620 : audit [DBG] from='client.? 172.21.15.6:0/2598664155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:10.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:10 smithi006 bash[33918]: cluster 2023-08-28T00:20:08.411916+0000 mgr.y (mgr.14901) 4247 : cluster [DBG] pgmap v4254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:10.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:10 smithi006 bash[33918]: audit 2023-08-28T00:20:09.645967+0000 mon.c (mon.2) 1620 : audit [DBG] from='client.? 172.21.15.6:0/2598664155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:10 smithi130 bash[51302]: debug 2023-08-28T00:20:10.397+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:10.399641+0000) 2023-08-28T00:20:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:20:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:20:10] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:20:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:20:10.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:20:10.708Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:11.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:11 smithi130 bash[51302]: debug 2023-08-28T00:20:11.397+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:11.399937+0000) 2023-08-28T00:20:12.288 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:12 smithi006 bash[27292]: cluster 2023-08-28T00:20:10.412994+0000 mgr.y (mgr.14901) 4248 : cluster [DBG] pgmap v4255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:12.288 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:12 smithi006 bash[33918]: cluster 2023-08-28T00:20:10.412994+0000 mgr.y (mgr.14901) 4248 : cluster [DBG] pgmap v4255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:12.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:12 smithi130 bash[27579]: cluster 2023-08-28T00:20:10.412994+0000 mgr.y (mgr.14901) 4248 : cluster [DBG] pgmap v4255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:20:12.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:20:12.287Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:12 smithi130 bash[51302]: debug 2023-08-28T00:20:12.397+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:12.400241+0000) 2023-08-28T00:20:13.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:13 smithi130 bash[27579]: audit 2023-08-28T00:20:12.062493+0000 mon.c (mon.2) 1621 : audit [DBG] from='client.? 172.21.15.6:0/595922375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:13.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:13 smithi006 bash[33918]: audit 2023-08-28T00:20:12.062493+0000 mon.c (mon.2) 1621 : audit [DBG] from='client.? 172.21.15.6:0/595922375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:13.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:13 smithi006 bash[27292]: audit 2023-08-28T00:20:12.062493+0000 mon.c (mon.2) 1621 : audit [DBG] from='client.? 172.21.15.6:0/595922375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:13.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:13 smithi130 bash[51302]: debug 2023-08-28T00:20:13.397+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:13.400538+0000) 2023-08-28T00:20:14.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:14 smithi130 bash[27579]: cluster 2023-08-28T00:20:12.413737+0000 mgr.y (mgr.14901) 4249 : cluster [DBG] pgmap v4256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:14.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:14 smithi006 bash[27292]: cluster 2023-08-28T00:20:12.413737+0000 mgr.y (mgr.14901) 4249 : cluster [DBG] pgmap v4256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:20:14.169Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:20:14.171Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:14.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:14 smithi006 bash[33918]: cluster 2023-08-28T00:20:12.413737+0000 mgr.y (mgr.14901) 4249 : cluster [DBG] pgmap v4256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:14 smithi130 bash[51302]: debug 2023-08-28T00:20:14.397+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:14.400844+0000) 2023-08-28T00:20:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:15 smithi130 bash[27579]: audit 2023-08-28T00:20:14.483755+0000 mon.a (mon.0) 3154 : audit [DBG] from='client.? 172.21.15.6:0/1691917071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:15.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:15 smithi006 bash[27292]: audit 2023-08-28T00:20:14.483755+0000 mon.a (mon.0) 3154 : audit [DBG] from='client.? 172.21.15.6:0/1691917071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:15.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:15 smithi006 bash[33918]: audit 2023-08-28T00:20:14.483755+0000 mon.a (mon.0) 3154 : audit [DBG] from='client.? 172.21.15.6:0/1691917071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:15.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:15 smithi130 bash[51302]: debug 2023-08-28T00:20:15.397+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:15.401143+0000) 2023-08-28T00:20:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:16 smithi130 bash[27579]: cluster 2023-08-28T00:20:14.414656+0000 mgr.y (mgr.14901) 4250 : cluster [DBG] pgmap v4257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:16.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:16 smithi006 bash[27292]: cluster 2023-08-28T00:20:14.414656+0000 mgr.y (mgr.14901) 4250 : cluster [DBG] pgmap v4257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:16.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:16 smithi006 bash[33918]: cluster 2023-08-28T00:20:14.414656+0000 mgr.y (mgr.14901) 4250 : cluster [DBG] pgmap v4257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:16.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:16 smithi130 bash[51302]: debug 2023-08-28T00:20:16.396+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:16.401475+0000) 2023-08-28T00:20:17.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:17 smithi130 bash[27579]: audit 2023-08-28T00:20:16.892737+0000 mon.a (mon.0) 3155 : audit [DBG] from='client.? 172.21.15.6:0/557432364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:17.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:17 smithi006 bash[27292]: audit 2023-08-28T00:20:16.892737+0000 mon.a (mon.0) 3155 : audit [DBG] from='client.? 172.21.15.6:0/557432364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:17.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:17 smithi006 bash[33918]: audit 2023-08-28T00:20:16.892737+0000 mon.a (mon.0) 3155 : audit [DBG] from='client.? 172.21.15.6:0/557432364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:17.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:17 smithi130 bash[51302]: debug 2023-08-28T00:20:17.396+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:17.401847+0000) 2023-08-28T00:20:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:18 smithi130 bash[27579]: cluster 2023-08-28T00:20:16.415287+0000 mgr.y (mgr.14901) 4251 : cluster [DBG] pgmap v4258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:18.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:18 smithi006 bash[27292]: cluster 2023-08-28T00:20:16.415287+0000 mgr.y (mgr.14901) 4251 : cluster [DBG] pgmap v4258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:18.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:18 smithi006 bash[33918]: cluster 2023-08-28T00:20:16.415287+0000 mgr.y (mgr.14901) 4251 : cluster [DBG] pgmap v4258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:18 smithi130 bash[51302]: debug 2023-08-28T00:20:18.396+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:18.402115+0000) 2023-08-28T00:20:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:19 smithi130 bash[51302]: debug 2023-08-28T00:20:19.396+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:19.402399+0000) 2023-08-28T00:20:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:20 smithi130 bash[27579]: cluster 2023-08-28T00:20:18.415939+0000 mgr.y (mgr.14901) 4252 : cluster [DBG] pgmap v4259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:20 smithi130 bash[27579]: audit 2023-08-28T00:20:19.304676+0000 mon.a (mon.0) 3156 : audit [DBG] from='client.? 172.21.15.6:0/3262842428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:20.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:20 smithi006 bash[27292]: cluster 2023-08-28T00:20:18.415939+0000 mgr.y (mgr.14901) 4252 : cluster [DBG] pgmap v4259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:20.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:20 smithi006 bash[27292]: audit 2023-08-28T00:20:19.304676+0000 mon.a (mon.0) 3156 : audit [DBG] from='client.? 172.21.15.6:0/3262842428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:20.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:20 smithi006 bash[33918]: cluster 2023-08-28T00:20:18.415939+0000 mgr.y (mgr.14901) 4252 : cluster [DBG] pgmap v4259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:20.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:20 smithi006 bash[33918]: audit 2023-08-28T00:20:19.304676+0000 mon.a (mon.0) 3156 : audit [DBG] from='client.? 172.21.15.6:0/3262842428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:20 smithi130 bash[51302]: debug 2023-08-28T00:20:20.396+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:20.402668+0000) 2023-08-28T00:20:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:20:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:20:20] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:20:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:20:20.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:20.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:20:20.708Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:21.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:21 smithi130 bash[51302]: debug 2023-08-28T00:20:21.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:21.402964+0000) 2023-08-28T00:20:22.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:22 smithi130 bash[27579]: cluster 2023-08-28T00:20:20.417090+0000 mgr.y (mgr.14901) 4253 : cluster [DBG] pgmap v4260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:22.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:22 smithi130 bash[27579]: audit 2023-08-28T00:20:21.721599+0000 mon.c (mon.2) 1622 : audit [DBG] from='client.? 172.21.15.6:0/1524437368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:22.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:22 smithi006 bash[27292]: cluster 2023-08-28T00:20:20.417090+0000 mgr.y (mgr.14901) 4253 : cluster [DBG] pgmap v4260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:22.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:22 smithi006 bash[27292]: audit 2023-08-28T00:20:21.721599+0000 mon.c (mon.2) 1622 : audit [DBG] from='client.? 172.21.15.6:0/1524437368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:22.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:20:22.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:22.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:20:22.287Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:22.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:22 smithi006 bash[33918]: cluster 2023-08-28T00:20:20.417090+0000 mgr.y (mgr.14901) 4253 : cluster [DBG] pgmap v4260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:22.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:22 smithi006 bash[33918]: audit 2023-08-28T00:20:21.721599+0000 mon.c (mon.2) 1622 : audit [DBG] from='client.? 172.21.15.6:0/1524437368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:22 smithi130 bash[51302]: debug 2023-08-28T00:20:22.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:22.403247+0000) 2023-08-28T00:20:23.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:23 smithi130 bash[51302]: debug 2023-08-28T00:20:23.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:23.403513+0000) 2023-08-28T00:20:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:24 smithi130 bash[27579]: cluster 2023-08-28T00:20:22.417713+0000 mgr.y (mgr.14901) 4254 : cluster [DBG] pgmap v4261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:24.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:24 smithi006 bash[27292]: cluster 2023-08-28T00:20:22.417713+0000 mgr.y (mgr.14901) 4254 : cluster [DBG] pgmap v4261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:24.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:24 smithi006 bash[33918]: cluster 2023-08-28T00:20:22.417713+0000 mgr.y (mgr.14901) 4254 : cluster [DBG] pgmap v4261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:20:24.169Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 9 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:20:24.172Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:24 smithi130 bash[51302]: debug 2023-08-28T00:20:24.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:24.403831+0000) 2023-08-28T00:20:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:25 smithi130 bash[27579]: audit 2023-08-28T00:20:24.136411+0000 mon.a (mon.0) 3157 : audit [DBG] from='client.? 172.21.15.6:0/1245678135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:25.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:25 smithi006 bash[33918]: audit 2023-08-28T00:20:24.136411+0000 mon.a (mon.0) 3157 : audit [DBG] from='client.? 172.21.15.6:0/1245678135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:25.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:25 smithi006 bash[27292]: audit 2023-08-28T00:20:24.136411+0000 mon.a (mon.0) 3157 : audit [DBG] from='client.? 172.21.15.6:0/1245678135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:25.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:25 smithi130 bash[51302]: debug 2023-08-28T00:20:25.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:25.404138+0000) 2023-08-28T00:20:26.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:26 smithi130 bash[27579]: cluster 2023-08-28T00:20:24.418908+0000 mgr.y (mgr.14901) 4255 : cluster [DBG] pgmap v4262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:26.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:26 smithi006 bash[33918]: cluster 2023-08-28T00:20:24.418908+0000 mgr.y (mgr.14901) 4255 : cluster [DBG] pgmap v4262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:26 smithi006 bash[27292]: cluster 2023-08-28T00:20:24.418908+0000 mgr.y (mgr.14901) 4255 : cluster [DBG] pgmap v4262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:26.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:26 smithi130 bash[51302]: debug 2023-08-28T00:20:26.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:26.404417+0000) 2023-08-28T00:20:27.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:27 smithi130 bash[27579]: audit 2023-08-28T00:20:26.547798+0000 mon.c (mon.2) 1623 : audit [DBG] from='client.? 172.21.15.6:0/574205512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:27.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:27 smithi006 bash[33918]: audit 2023-08-28T00:20:26.547798+0000 mon.c (mon.2) 1623 : audit [DBG] from='client.? 172.21.15.6:0/574205512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:27.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:27 smithi006 bash[27292]: audit 2023-08-28T00:20:26.547798+0000 mon.c (mon.2) 1623 : audit [DBG] from='client.? 172.21.15.6:0/574205512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:27.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:27 smithi130 bash[51302]: debug 2023-08-28T00:20:27.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:27.404737+0000) 2023-08-28T00:20:28.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:28 smithi130 bash[27579]: cluster 2023-08-28T00:20:26.419508+0000 mgr.y (mgr.14901) 4256 : cluster [DBG] pgmap v4263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:28.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:28 smithi006 bash[33918]: cluster 2023-08-28T00:20:26.419508+0000 mgr.y (mgr.14901) 4256 : cluster [DBG] pgmap v4263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:28.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:28 smithi006 bash[27292]: cluster 2023-08-28T00:20:26.419508+0000 mgr.y (mgr.14901) 4256 : cluster [DBG] pgmap v4263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:28 smithi130 bash[51302]: debug 2023-08-28T00:20:28.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:28.404998+0000) 2023-08-28T00:20:29.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:29 smithi130 bash[27579]: audit 2023-08-28T00:20:28.954361+0000 mon.c (mon.2) 1624 : audit [DBG] from='client.? 172.21.15.6:0/234929900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:29.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:29 smithi006 bash[33918]: audit 2023-08-28T00:20:28.954361+0000 mon.c (mon.2) 1624 : audit [DBG] from='client.? 172.21.15.6:0/234929900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:29.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:29 smithi006 bash[27292]: audit 2023-08-28T00:20:28.954361+0000 mon.c (mon.2) 1624 : audit [DBG] from='client.? 172.21.15.6:0/234929900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:29.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:29 smithi130 bash[51302]: debug 2023-08-28T00:20:29.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:29.405324+0000) 2023-08-28T00:20:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:30 smithi130 bash[27579]: cluster 2023-08-28T00:20:28.420130+0000 mgr.y (mgr.14901) 4257 : cluster [DBG] pgmap v4264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:30.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:30 smithi006 bash[33918]: cluster 2023-08-28T00:20:28.420130+0000 mgr.y (mgr.14901) 4257 : cluster [DBG] pgmap v4264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:30.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:30 smithi006 bash[27292]: cluster 2023-08-28T00:20:28.420130+0000 mgr.y (mgr.14901) 4257 : cluster [DBG] pgmap v4264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:30 smithi130 bash[51302]: debug 2023-08-28T00:20:30.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:30.405625+0000) 2023-08-28T00:20:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:20:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:20:30] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:20:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:20:30.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:20:30.709Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:31.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:31 smithi130 bash[51302]: debug 2023-08-28T00:20:31.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:31.405897+0000) 2023-08-28T00:20:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:32 smithi130 bash[27579]: cluster 2023-08-28T00:20:30.421315+0000 mgr.y (mgr.14901) 4258 : cluster [DBG] pgmap v4265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:32 smithi130 bash[27579]: audit 2023-08-28T00:20:31.367365+0000 mon.a (mon.0) 3158 : audit [DBG] from='client.? 172.21.15.6:0/1076526657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:32.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:32 smithi006 bash[33918]: cluster 2023-08-28T00:20:30.421315+0000 mgr.y (mgr.14901) 4258 : cluster [DBG] pgmap v4265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:32.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:32 smithi006 bash[33918]: audit 2023-08-28T00:20:31.367365+0000 mon.a (mon.0) 3158 : audit [DBG] from='client.? 172.21.15.6:0/1076526657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:20:32.286Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:32.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:20:32.288Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:32.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:32 smithi006 bash[27292]: cluster 2023-08-28T00:20:30.421315+0000 mgr.y (mgr.14901) 4258 : cluster [DBG] pgmap v4265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:32.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:32 smithi006 bash[27292]: audit 2023-08-28T00:20:31.367365+0000 mon.a (mon.0) 3158 : audit [DBG] from='client.? 172.21.15.6:0/1076526657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:32 smithi130 bash[51302]: debug 2023-08-28T00:20:32.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:32.406113+0000) 2023-08-28T00:20:33.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:33 smithi130 bash[51302]: debug 2023-08-28T00:20:33.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:33.406312+0000) 2023-08-28T00:20:34.408 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:34 smithi130 bash[27579]: cluster 2023-08-28T00:20:32.421983+0000 mgr.y (mgr.14901) 4259 : cluster [DBG] pgmap v4266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:34.408 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:34 smithi130 bash[27579]: audit 2023-08-28T00:20:33.780499+0000 mon.c (mon.2) 1625 : audit [DBG] from='client.? 172.21.15.6:0/2908485894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:34.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:34 smithi006 bash[33918]: cluster 2023-08-28T00:20:32.421983+0000 mgr.y (mgr.14901) 4259 : cluster [DBG] pgmap v4266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:34.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:34 smithi006 bash[33918]: audit 2023-08-28T00:20:33.780499+0000 mon.c (mon.2) 1625 : audit [DBG] from='client.? 172.21.15.6:0/2908485894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:20:34.170Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:20:34.173Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:34.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:34 smithi006 bash[27292]: cluster 2023-08-28T00:20:32.421983+0000 mgr.y (mgr.14901) 4259 : cluster [DBG] pgmap v4266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:34.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:34 smithi006 bash[27292]: audit 2023-08-28T00:20:33.780499+0000 mon.c (mon.2) 1625 : audit [DBG] from='client.? 172.21.15.6:0/2908485894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:34 smithi130 bash[51302]: debug 2023-08-28T00:20:34.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:34.406607+0000) 2023-08-28T00:20:35.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:35 smithi130 bash[51302]: debug 2023-08-28T00:20:35.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:35.406936+0000) 2023-08-28T00:20:36.408 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:36 smithi130 bash[27579]: cluster 2023-08-28T00:20:34.423104+0000 mgr.y (mgr.14901) 4260 : cluster [DBG] pgmap v4267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:36.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:36 smithi006 bash[33918]: cluster 2023-08-28T00:20:34.423104+0000 mgr.y (mgr.14901) 4260 : cluster [DBG] pgmap v4267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:36.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:36 smithi006 bash[27292]: cluster 2023-08-28T00:20:34.423104+0000 mgr.y (mgr.14901) 4260 : cluster [DBG] pgmap v4267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:36 smithi130 bash[51302]: debug 2023-08-28T00:20:36.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:36.407146+0000) 2023-08-28T00:20:37.409 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:37 smithi130 bash[27579]: audit 2023-08-28T00:20:36.191960+0000 mon.c (mon.2) 1626 : audit [DBG] from='client.? 172.21.15.6:0/2031318776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:37.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:37 smithi006 bash[27292]: audit 2023-08-28T00:20:36.191960+0000 mon.c (mon.2) 1626 : audit [DBG] from='client.? 172.21.15.6:0/2031318776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:37.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:37 smithi006 bash[33918]: audit 2023-08-28T00:20:36.191960+0000 mon.c (mon.2) 1626 : audit [DBG] from='client.? 172.21.15.6:0/2031318776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:37 smithi130 bash[51302]: debug 2023-08-28T00:20:37.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:37.407361+0000) 2023-08-28T00:20:38.409 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:38 smithi130 bash[27579]: cluster 2023-08-28T00:20:36.423872+0000 mgr.y (mgr.14901) 4261 : cluster [DBG] pgmap v4268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:38.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:38 smithi006 bash[27292]: cluster 2023-08-28T00:20:36.423872+0000 mgr.y (mgr.14901) 4261 : cluster [DBG] pgmap v4268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:38.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:38 smithi006 bash[33918]: cluster 2023-08-28T00:20:36.423872+0000 mgr.y (mgr.14901) 4261 : cluster [DBG] pgmap v4268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:38 smithi130 bash[51302]: debug 2023-08-28T00:20:38.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:38.407638+0000) 2023-08-28T00:20:39.409 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:39 smithi130 bash[27579]: audit 2023-08-28T00:20:38.610043+0000 mon.c (mon.2) 1627 : audit [DBG] from='client.? 172.21.15.6:0/623040271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:39 smithi006 bash[27292]: audit 2023-08-28T00:20:38.610043+0000 mon.c (mon.2) 1627 : audit [DBG] from='client.? 172.21.15.6:0/623040271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:39.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:39 smithi006 bash[33918]: audit 2023-08-28T00:20:38.610043+0000 mon.c (mon.2) 1627 : audit [DBG] from='client.? 172.21.15.6:0/623040271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:39 smithi130 bash[51302]: debug 2023-08-28T00:20:39.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:39.407936+0000) 2023-08-28T00:20:40.409 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:40 smithi130 bash[27579]: cluster 2023-08-28T00:20:38.424657+0000 mgr.y (mgr.14901) 4262 : cluster [DBG] pgmap v4269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:40.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:40 smithi006 bash[27292]: cluster 2023-08-28T00:20:38.424657+0000 mgr.y (mgr.14901) 4262 : cluster [DBG] pgmap v4269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:40.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:40 smithi006 bash[33918]: cluster 2023-08-28T00:20:38.424657+0000 mgr.y (mgr.14901) 4262 : cluster [DBG] pgmap v4269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:40.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:40 smithi130 bash[51302]: debug 2023-08-28T00:20:40.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:40.408213+0000) 2023-08-28T00:20:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:20:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:20:40] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:20:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:20:40.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:20:40.710Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:41.681 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:41 smithi130 bash[51302]: debug 2023-08-28T00:20:41.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:41.408472+0000) 2023-08-28T00:20:41.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:41 smithi006 bash[33918]: cluster 2023-08-28T00:20:40.425804+0000 mgr.y (mgr.14901) 4263 : cluster [DBG] pgmap v4270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:41.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:41 smithi006 bash[33918]: audit 2023-08-28T00:20:41.024681+0000 mon.c (mon.2) 1628 : audit [DBG] from='client.? 172.21.15.6:0/3329248945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:41.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:41 smithi006 bash[27292]: cluster 2023-08-28T00:20:40.425804+0000 mgr.y (mgr.14901) 4263 : cluster [DBG] pgmap v4270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:41.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:41 smithi006 bash[27292]: audit 2023-08-28T00:20:41.024681+0000 mon.c (mon.2) 1628 : audit [DBG] from='client.? 172.21.15.6:0/3329248945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:42.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:41 smithi130 bash[27579]: cluster 2023-08-28T00:20:40.425804+0000 mgr.y (mgr.14901) 4263 : cluster [DBG] pgmap v4270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:42.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:41 smithi130 bash[27579]: audit 2023-08-28T00:20:41.024681+0000 mon.c (mon.2) 1628 : audit [DBG] from='client.? 172.21.15.6:0/3329248945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:20:42.287Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:20:42.289Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:42 smithi130 bash[51302]: debug 2023-08-28T00:20:42.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:42.408750+0000) 2023-08-28T00:20:43.686 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:43 smithi130 bash[51302]: debug 2023-08-28T00:20:43.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:43.409010+0000) 2023-08-28T00:20:43.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:43 smithi006 bash[33918]: cluster 2023-08-28T00:20:42.426485+0000 mgr.y (mgr.14901) 4264 : cluster [DBG] pgmap v4271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:43.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:43 smithi006 bash[33918]: audit 2023-08-28T00:20:43.439244+0000 mon.a (mon.0) 3159 : audit [DBG] from='client.? 172.21.15.6:0/2848707872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:43.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:43 smithi006 bash[27292]: cluster 2023-08-28T00:20:42.426485+0000 mgr.y (mgr.14901) 4264 : cluster [DBG] pgmap v4271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:43.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:43 smithi006 bash[27292]: audit 2023-08-28T00:20:43.439244+0000 mon.a (mon.0) 3159 : audit [DBG] from='client.? 172.21.15.6:0/2848707872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:44.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:43 smithi130 bash[27579]: cluster 2023-08-28T00:20:42.426485+0000 mgr.y (mgr.14901) 4264 : cluster [DBG] pgmap v4271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:44.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:43 smithi130 bash[27579]: audit 2023-08-28T00:20:43.439244+0000 mon.a (mon.0) 3159 : audit [DBG] from='client.? 172.21.15.6:0/2848707872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:20:44.171Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:20:44.173Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:44 smithi130 bash[51302]: debug 2023-08-28T00:20:44.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:44.409314+0000) 2023-08-28T00:20:45.691 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:45 smithi130 bash[51302]: debug 2023-08-28T00:20:45.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:45.409586+0000) 2023-08-28T00:20:45.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:45 smithi006 bash[33918]: cluster 2023-08-28T00:20:44.427580+0000 mgr.y (mgr.14901) 4265 : cluster [DBG] pgmap v4272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:45.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:45 smithi006 bash[33918]: audit 2023-08-28T00:20:45.605728+0000 mon.b (mon.1) 531 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:20:45.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:45 smithi006 bash[27292]: cluster 2023-08-28T00:20:44.427580+0000 mgr.y (mgr.14901) 4265 : cluster [DBG] pgmap v4272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:45.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:45 smithi006 bash[27292]: audit 2023-08-28T00:20:45.605728+0000 mon.b (mon.1) 531 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:20:46.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:45 smithi130 bash[27579]: cluster 2023-08-28T00:20:44.427580+0000 mgr.y (mgr.14901) 4265 : cluster [DBG] pgmap v4272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:46.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:45 smithi130 bash[27579]: audit 2023-08-28T00:20:45.605728+0000 mon.b (mon.1) 531 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:20:46.700 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:46 smithi130 bash[51302]: debug 2023-08-28T00:20:46.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:46.409855+0000) 2023-08-28T00:20:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:46 smithi006 bash[33918]: audit 2023-08-28T00:20:45.863878+0000 mon.c (mon.2) 1629 : audit [DBG] from='client.? 172.21.15.6:0/274167659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:46 smithi006 bash[27292]: audit 2023-08-28T00:20:45.863878+0000 mon.c (mon.2) 1629 : audit [DBG] from='client.? 172.21.15.6:0/274167659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:47.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:46 smithi130 bash[27579]: audit 2023-08-28T00:20:45.863878+0000 mon.c (mon.2) 1629 : audit [DBG] from='client.? 172.21.15.6:0/274167659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:47.702 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:47 smithi130 bash[51302]: debug 2023-08-28T00:20:47.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:47.410124+0000) 2023-08-28T00:20:47.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:47 smithi006 bash[27292]: cluster 2023-08-28T00:20:46.428229+0000 mgr.y (mgr.14901) 4266 : cluster [DBG] pgmap v4273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:47.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:47 smithi006 bash[33918]: cluster 2023-08-28T00:20:46.428229+0000 mgr.y (mgr.14901) 4266 : cluster [DBG] pgmap v4273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:48.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:47 smithi130 bash[27579]: cluster 2023-08-28T00:20:46.428229+0000 mgr.y (mgr.14901) 4266 : cluster [DBG] pgmap v4273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:48.710 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:48 smithi130 bash[51302]: debug 2023-08-28T00:20:48.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:48.410320+0000) 2023-08-28T00:20:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:48 smithi006 bash[27292]: audit 2023-08-28T00:20:48.283166+0000 mon.a (mon.0) 3160 : audit [DBG] from='client.? 172.21.15.6:0/1894074615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:48 smithi006 bash[33918]: audit 2023-08-28T00:20:48.283166+0000 mon.a (mon.0) 3160 : audit [DBG] from='client.? 172.21.15.6:0/1894074615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:48 smithi130 bash[27579]: audit 2023-08-28T00:20:48.283166+0000 mon.a (mon.0) 3160 : audit [DBG] from='client.? 172.21.15.6:0/1894074615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:49.714 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:49 smithi130 bash[51302]: debug 2023-08-28T00:20:49.408+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:49.410621+0000) 2023-08-28T00:20:49.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:49 smithi006 bash[27292]: cluster 2023-08-28T00:20:48.428935+0000 mgr.y (mgr.14901) 4267 : cluster [DBG] pgmap v4274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:49.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:49 smithi006 bash[33918]: cluster 2023-08-28T00:20:48.428935+0000 mgr.y (mgr.14901) 4267 : cluster [DBG] pgmap v4274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:50.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:49 smithi130 bash[27579]: cluster 2023-08-28T00:20:48.428935+0000 mgr.y (mgr.14901) 4267 : cluster [DBG] pgmap v4274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:50 smithi130 bash[51302]: debug 2023-08-28T00:20:50.408+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:50.410928+0000) 2023-08-28T00:20:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:20:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:20:50] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T00:20:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:20:50.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:20:50.710Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:51.720 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:51 smithi130 bash[51302]: debug 2023-08-28T00:20:51.408+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:51.411127+0000) 2023-08-28T00:20:52.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:51 smithi130 bash[27579]: cluster 2023-08-28T00:20:50.430024+0000 mgr.y (mgr.14901) 4268 : cluster [DBG] pgmap v4275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:52.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:51 smithi130 bash[27579]: audit 2023-08-28T00:20:50.725733+0000 mon.c (mon.2) 1630 : audit [DBG] from='client.? 172.21.15.6:0/3045838816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:52.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:51 smithi130 bash[27579]: audit 2023-08-28T00:20:51.447644+0000 mon.a (mon.0) 3161 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:20:52.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:51 smithi006 bash[33918]: cluster 2023-08-28T00:20:50.430024+0000 mgr.y (mgr.14901) 4268 : cluster [DBG] pgmap v4275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:52.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:51 smithi006 bash[33918]: audit 2023-08-28T00:20:50.725733+0000 mon.c (mon.2) 1630 : audit [DBG] from='client.? 172.21.15.6:0/3045838816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:52.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:51 smithi006 bash[33918]: audit 2023-08-28T00:20:51.447644+0000 mon.a (mon.0) 3161 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:20:52.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:51 smithi006 bash[27292]: cluster 2023-08-28T00:20:50.430024+0000 mgr.y (mgr.14901) 4268 : cluster [DBG] pgmap v4275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:52.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:51 smithi006 bash[27292]: audit 2023-08-28T00:20:50.725733+0000 mon.c (mon.2) 1630 : audit [DBG] from='client.? 172.21.15.6:0/3045838816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:52.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:51 smithi006 bash[27292]: audit 2023-08-28T00:20:51.447644+0000 mon.a (mon.0) 3161 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:20:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:20:52.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:20:52.290Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:52.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:52 smithi130 bash[51302]: debug 2023-08-28T00:20:52.408+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:52.411409+0000) 2023-08-28T00:20:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:52 smithi006 bash[27292]: audit 2023-08-28T00:20:51.961760+0000 mon.a (mon.0) 3162 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:20:53.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:52 smithi006 bash[33918]: audit 2023-08-28T00:20:51.961760+0000 mon.a (mon.0) 3162 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:20:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:52 smithi130 bash[27579]: audit 2023-08-28T00:20:51.961760+0000 mon.a (mon.0) 3162 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:20:53.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:53 smithi130 bash[51302]: debug 2023-08-28T00:20:53.408+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:53.411690+0000) 2023-08-28T00:20:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:53 smithi130 bash[27579]: cluster 2023-08-28T00:20:52.430749+0000 mgr.y (mgr.14901) 4269 : cluster [DBG] pgmap v4276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:53 smithi130 bash[27579]: audit 2023-08-28T00:20:53.164118+0000 mon.c (mon.2) 1631 : audit [DBG] from='client.? 172.21.15.6:0/1006052195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:54.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:53 smithi006 bash[27292]: cluster 2023-08-28T00:20:52.430749+0000 mgr.y (mgr.14901) 4269 : cluster [DBG] pgmap v4276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:54.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:53 smithi006 bash[27292]: audit 2023-08-28T00:20:53.164118+0000 mon.c (mon.2) 1631 : audit [DBG] from='client.? 172.21.15.6:0/1006052195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:20:54.172Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:20:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:20:54.174Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:20:54.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:53 smithi006 bash[33918]: cluster 2023-08-28T00:20:52.430749+0000 mgr.y (mgr.14901) 4269 : cluster [DBG] pgmap v4276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:54.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:53 smithi006 bash[33918]: audit 2023-08-28T00:20:53.164118+0000 mon.c (mon.2) 1631 : audit [DBG] from='client.? 172.21.15.6:0/1006052195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:54.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:54 smithi130 bash[51302]: debug 2023-08-28T00:20:54.408+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:54.411957+0000) 2023-08-28T00:20:55.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:55 smithi130 bash[51302]: debug 2023-08-28T00:20:55.408+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:55.412253+0000) 2023-08-28T00:20:56.325 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:55 smithi006 bash[27292]: cluster 2023-08-28T00:20:54.431820+0000 mgr.y (mgr.14901) 4270 : cluster [DBG] pgmap v4277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:56.325 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:55 smithi006 bash[27292]: audit 2023-08-28T00:20:55.580775+0000 mon.c (mon.2) 1632 : audit [DBG] from='client.? 172.21.15.6:0/3286216548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:56.325 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:55 smithi006 bash[33918]: cluster 2023-08-28T00:20:54.431820+0000 mgr.y (mgr.14901) 4270 : cluster [DBG] pgmap v4277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:56.326 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:55 smithi006 bash[33918]: audit 2023-08-28T00:20:55.580775+0000 mon.c (mon.2) 1632 : audit [DBG] from='client.? 172.21.15.6:0/3286216548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:55 smithi130 bash[27579]: cluster 2023-08-28T00:20:54.431820+0000 mgr.y (mgr.14901) 4270 : cluster [DBG] pgmap v4277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:55 smithi130 bash[27579]: audit 2023-08-28T00:20:55.580775+0000 mon.c (mon.2) 1632 : audit [DBG] from='client.? 172.21.15.6:0/3286216548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:56.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:56 smithi130 bash[51302]: debug 2023-08-28T00:20:56.408+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:56.412549+0000) 2023-08-28T00:20:57.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:57 smithi130 bash[51302]: debug 2023-08-28T00:20:57.412+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:57.412810+0000) 2023-08-28T00:20:58.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:57 smithi006 bash[27292]: cluster 2023-08-28T00:20:56.432479+0000 mgr.y (mgr.14901) 4271 : cluster [DBG] pgmap v4278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:58.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:57 smithi006 bash[33918]: cluster 2023-08-28T00:20:56.432479+0000 mgr.y (mgr.14901) 4271 : cluster [DBG] pgmap v4278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:58.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:57 smithi130 bash[27579]: cluster 2023-08-28T00:20:56.432479+0000 mgr.y (mgr.14901) 4271 : cluster [DBG] pgmap v4278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:20:58.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:58 smithi130 bash[51302]: debug 2023-08-28T00:20:58.408+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:58.413108+0000) 2023-08-28T00:20:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:58 smithi130 bash[27579]: audit 2023-08-28T00:20:58.000501+0000 mon.c (mon.2) 1633 : audit [DBG] from='client.? 172.21.15.6:0/873729786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:59.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:58 smithi006 bash[33918]: audit 2023-08-28T00:20:58.000501+0000 mon.c (mon.2) 1633 : audit [DBG] from='client.? 172.21.15.6:0/873729786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:59.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:58 smithi006 bash[27292]: audit 2023-08-28T00:20:58.000501+0000 mon.c (mon.2) 1633 : audit [DBG] from='client.? 172.21.15.6:0/873729786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:59.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:20:59 smithi130 bash[51302]: debug 2023-08-28T00:20:59.408+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:59.413379+0000) 2023-08-28T00:21:00.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:20:59 smithi006 bash[33918]: cluster 2023-08-28T00:20:58.432990+0000 mgr.y (mgr.14901) 4272 : cluster [DBG] pgmap v4279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:00.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:20:59 smithi006 bash[27292]: cluster 2023-08-28T00:20:58.432990+0000 mgr.y (mgr.14901) 4272 : cluster [DBG] pgmap v4279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:00.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:20:59 smithi130 bash[27579]: cluster 2023-08-28T00:20:58.432990+0000 mgr.y (mgr.14901) 4272 : cluster [DBG] pgmap v4279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:00.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:00 smithi130 bash[51302]: debug 2023-08-28T00:21:00.408+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:00.413689+0000) 2023-08-28T00:21:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:21:00.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:21:00.710Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:21:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:21:00] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:21:01.263 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:01 smithi006 bash[33918]: audit 2023-08-28T00:21:00.406073+0000 mon.c (mon.2) 1634 : audit [DBG] from='client.? 172.21.15.6:0/3517986700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:01.263 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:01 smithi006 bash[27292]: audit 2023-08-28T00:21:00.406073+0000 mon.c (mon.2) 1634 : audit [DBG] from='client.? 172.21.15.6:0/3517986700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:01.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:01 smithi130 bash[27579]: audit 2023-08-28T00:21:00.406073+0000 mon.c (mon.2) 1634 : audit [DBG] from='client.? 172.21.15.6:0/3517986700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:01.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:01 smithi130 bash[51302]: debug 2023-08-28T00:21:01.412+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:01.413995+0000) 2023-08-28T00:21:02.291 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:02 smithi006 bash[33918]: cluster 2023-08-28T00:21:00.433697+0000 mgr.y (mgr.14901) 4273 : cluster [DBG] pgmap v4280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:02.292 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:02 smithi006 bash[27292]: cluster 2023-08-28T00:21:00.433697+0000 mgr.y (mgr.14901) 4273 : cluster [DBG] pgmap v4280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:02.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:02 smithi130 bash[27579]: cluster 2023-08-28T00:21:00.433697+0000 mgr.y (mgr.14901) 4273 : cluster [DBG] pgmap v4280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:21:02.289Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:21:02.290Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:02.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:02 smithi130 bash[51302]: debug 2023-08-28T00:21:02.408+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:02.414246+0000) 2023-08-28T00:21:03.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:03 smithi006 bash[33918]: audit 2023-08-28T00:21:02.375128+0000 mon.a (mon.0) 3163 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:21:03.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:03 smithi006 bash[33918]: audit 2023-08-28T00:21:02.661619+0000 mon.b (mon.1) 532 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:21:03.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:03 smithi006 bash[33918]: audit 2023-08-28T00:21:02.663643+0000 mon.b (mon.1) 533 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:21:03.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:03 smithi006 bash[33918]: audit 2023-08-28T00:21:02.673063+0000 mon.a (mon.0) 3164 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:21:03.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:03 smithi006 bash[33918]: audit 2023-08-28T00:21:02.826384+0000 mon.a (mon.0) 3165 : audit [DBG] from='client.? 172.21.15.6:0/2804290027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:03.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:03 smithi006 bash[27292]: audit 2023-08-28T00:21:02.375128+0000 mon.a (mon.0) 3163 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:21:03.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:03 smithi006 bash[27292]: audit 2023-08-28T00:21:02.661619+0000 mon.b (mon.1) 532 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:21:03.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:03 smithi006 bash[27292]: audit 2023-08-28T00:21:02.663643+0000 mon.b (mon.1) 533 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:21:03.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:03 smithi006 bash[27292]: audit 2023-08-28T00:21:02.673063+0000 mon.a (mon.0) 3164 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:21:03.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:03 smithi006 bash[27292]: audit 2023-08-28T00:21:02.826384+0000 mon.a (mon.0) 3165 : audit [DBG] from='client.? 172.21.15.6:0/2804290027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:03.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:03 smithi130 bash[51302]: debug 2023-08-28T00:21:03.408+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:03.414565+0000) 2023-08-28T00:21:03.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:03 smithi130 bash[27579]: audit 2023-08-28T00:21:02.375128+0000 mon.a (mon.0) 3163 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:21:03.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:03 smithi130 bash[27579]: audit 2023-08-28T00:21:02.661619+0000 mon.b (mon.1) 532 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:21:03.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:03 smithi130 bash[27579]: audit 2023-08-28T00:21:02.663643+0000 mon.b (mon.1) 533 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:21:03.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:03 smithi130 bash[27579]: audit 2023-08-28T00:21:02.673063+0000 mon.a (mon.0) 3164 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:21:03.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:03 smithi130 bash[27579]: audit 2023-08-28T00:21:02.826384+0000 mon.a (mon.0) 3165 : audit [DBG] from='client.? 172.21.15.6:0/2804290027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:04.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:04 smithi006 bash[33918]: cluster 2023-08-28T00:21:02.434252+0000 mgr.y (mgr.14901) 4274 : cluster [DBG] pgmap v4281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:04.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:04 smithi006 bash[27292]: cluster 2023-08-28T00:21:02.434252+0000 mgr.y (mgr.14901) 4274 : cluster [DBG] pgmap v4281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:21:04.172Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:21:04.174Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:04 smithi130 bash[27579]: cluster 2023-08-28T00:21:02.434252+0000 mgr.y (mgr.14901) 4274 : cluster [DBG] pgmap v4281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:04.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:04 smithi130 bash[51302]: debug 2023-08-28T00:21:04.412+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:04.414893+0000) 2023-08-28T00:21:05.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:05 smithi006 bash[33918]: audit 2023-08-28T00:21:05.235169+0000 mon.c (mon.2) 1635 : audit [DBG] from='client.? 172.21.15.6:0/2088407025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:05.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:05 smithi006 bash[27292]: audit 2023-08-28T00:21:05.235169+0000 mon.c (mon.2) 1635 : audit [DBG] from='client.? 172.21.15.6:0/2088407025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:05.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:05 smithi130 bash[27579]: audit 2023-08-28T00:21:05.235169+0000 mon.c (mon.2) 1635 : audit [DBG] from='client.? 172.21.15.6:0/2088407025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:05.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:05 smithi130 bash[51302]: debug 2023-08-28T00:21:05.412+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:05.415192+0000) 2023-08-28T00:21:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:06 smithi006 bash[27292]: cluster 2023-08-28T00:21:04.435322+0000 mgr.y (mgr.14901) 4275 : cluster [DBG] pgmap v4282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:06 smithi006 bash[33918]: cluster 2023-08-28T00:21:04.435322+0000 mgr.y (mgr.14901) 4275 : cluster [DBG] pgmap v4282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:06 smithi130 bash[27579]: cluster 2023-08-28T00:21:04.435322+0000 mgr.y (mgr.14901) 4275 : cluster [DBG] pgmap v4282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:06.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:06 smithi130 bash[51302]: debug 2023-08-28T00:21:06.412+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:06.415508+0000) 2023-08-28T00:21:07.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:07 smithi130 bash[51302]: debug 2023-08-28T00:21:07.412+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:07.415730+0000) 2023-08-28T00:21:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:08 smithi006 bash[33918]: cluster 2023-08-28T00:21:06.436139+0000 mgr.y (mgr.14901) 4276 : cluster [DBG] pgmap v4283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:08 smithi006 bash[33918]: audit 2023-08-28T00:21:07.661185+0000 mon.b (mon.1) 534 : audit [DBG] from='client.? 172.21.15.6:0/3633433354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:08 smithi006 bash[27292]: cluster 2023-08-28T00:21:06.436139+0000 mgr.y (mgr.14901) 4276 : cluster [DBG] pgmap v4283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:08 smithi006 bash[27292]: audit 2023-08-28T00:21:07.661185+0000 mon.b (mon.1) 534 : audit [DBG] from='client.? 172.21.15.6:0/3633433354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:08.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:08 smithi130 bash[51302]: debug 2023-08-28T00:21:08.412+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:08.416005+0000) 2023-08-28T00:21:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:08 smithi130 bash[27579]: cluster 2023-08-28T00:21:06.436139+0000 mgr.y (mgr.14901) 4276 : cluster [DBG] pgmap v4283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:08.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:08 smithi130 bash[27579]: audit 2023-08-28T00:21:07.661185+0000 mon.b (mon.1) 534 : audit [DBG] from='client.? 172.21.15.6:0/3633433354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:09.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:09 smithi006 bash[33918]: cluster 2023-08-28T00:21:08.436881+0000 mgr.y (mgr.14901) 4277 : cluster [DBG] pgmap v4284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:09 smithi006 bash[27292]: cluster 2023-08-28T00:21:08.436881+0000 mgr.y (mgr.14901) 4277 : cluster [DBG] pgmap v4284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:09 smithi130 bash[27579]: cluster 2023-08-28T00:21:08.436881+0000 mgr.y (mgr.14901) 4277 : cluster [DBG] pgmap v4284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:09.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:09 smithi130 bash[51302]: debug 2023-08-28T00:21:09.412+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:09.416206+0000) 2023-08-28T00:21:10.712 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:10 smithi006 bash[33918]: audit 2023-08-28T00:21:10.078430+0000 mon.c (mon.2) 1636 : audit [DBG] from='client.? 172.21.15.6:0/1840074161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:10.712 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:10 smithi006 bash[27292]: audit 2023-08-28T00:21:10.078430+0000 mon.c (mon.2) 1636 : audit [DBG] from='client.? 172.21.15.6:0/1840074161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:10 smithi130 bash[27579]: audit 2023-08-28T00:21:10.078430+0000 mon.c (mon.2) 1636 : audit [DBG] from='client.? 172.21.15.6:0/1840074161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:10 smithi130 bash[51302]: debug 2023-08-28T00:21:10.412+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:10.416425+0000) 2023-08-28T00:21:10.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:21:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:21:10] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:21:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:21:10.709Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:10.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:21:10.712Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:11.688 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:11 smithi130 bash[51302]: debug 2023-08-28T00:21:11.412+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:11.416696+0000) 2023-08-28T00:21:11.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:11 smithi006 bash[27292]: cluster 2023-08-28T00:21:10.437971+0000 mgr.y (mgr.14901) 4278 : cluster [DBG] pgmap v4285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:11.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:11 smithi006 bash[33918]: cluster 2023-08-28T00:21:10.437971+0000 mgr.y (mgr.14901) 4278 : cluster [DBG] pgmap v4285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:12.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:11 smithi130 bash[27579]: cluster 2023-08-28T00:21:10.437971+0000 mgr.y (mgr.14901) 4278 : cluster [DBG] pgmap v4285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:12.695 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:21:12.289Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:12.696 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:21:12.291Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:12.696 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:12 smithi130 bash[51302]: debug 2023-08-28T00:21:12.411+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:12.416964+0000) 2023-08-28T00:21:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:12 smithi006 bash[33918]: audit 2023-08-28T00:21:12.492916+0000 mon.c (mon.2) 1637 : audit [DBG] from='client.? 172.21.15.6:0/767987091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:12.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:12 smithi006 bash[27292]: audit 2023-08-28T00:21:12.492916+0000 mon.c (mon.2) 1637 : audit [DBG] from='client.? 172.21.15.6:0/767987091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:12 smithi130 bash[27579]: audit 2023-08-28T00:21:12.492916+0000 mon.c (mon.2) 1637 : audit [DBG] from='client.? 172.21.15.6:0/767987091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:13.702 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:13 smithi130 bash[51302]: debug 2023-08-28T00:21:13.411+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:13.417266+0000) 2023-08-28T00:21:13.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:13 smithi006 bash[27292]: cluster 2023-08-28T00:21:12.438435+0000 mgr.y (mgr.14901) 4279 : cluster [DBG] pgmap v4286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:13.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:13 smithi006 bash[33918]: cluster 2023-08-28T00:21:12.438435+0000 mgr.y (mgr.14901) 4279 : cluster [DBG] pgmap v4286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:14.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:13 smithi130 bash[27579]: cluster 2023-08-28T00:21:12.438435+0000 mgr.y (mgr.14901) 4279 : cluster [DBG] pgmap v4286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:21:14.173Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:21:14.175Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:14 smithi130 bash[51302]: debug 2023-08-28T00:21:14.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:14.417674+0000) 2023-08-28T00:21:15.704 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:15 smithi130 bash[51302]: debug 2023-08-28T00:21:15.411+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:15.417986+0000) 2023-08-28T00:21:15.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:15 smithi006 bash[33918]: cluster 2023-08-28T00:21:14.439512+0000 mgr.y (mgr.14901) 4280 : cluster [DBG] pgmap v4287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:15.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:15 smithi006 bash[33918]: audit 2023-08-28T00:21:14.917054+0000 mon.c (mon.2) 1638 : audit [DBG] from='client.? 172.21.15.6:0/3837207033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:15.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:15 smithi006 bash[27292]: cluster 2023-08-28T00:21:14.439512+0000 mgr.y (mgr.14901) 4280 : cluster [DBG] pgmap v4287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:15.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:15 smithi006 bash[27292]: audit 2023-08-28T00:21:14.917054+0000 mon.c (mon.2) 1638 : audit [DBG] from='client.? 172.21.15.6:0/3837207033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:16.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:15 smithi130 bash[27579]: cluster 2023-08-28T00:21:14.439512+0000 mgr.y (mgr.14901) 4280 : cluster [DBG] pgmap v4287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:16.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:15 smithi130 bash[27579]: audit 2023-08-28T00:21:14.917054+0000 mon.c (mon.2) 1638 : audit [DBG] from='client.? 172.21.15.6:0/3837207033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:16.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:16 smithi130 bash[51302]: debug 2023-08-28T00:21:16.411+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:16.418356+0000) 2023-08-28T00:21:17.709 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:17 smithi130 bash[51302]: debug 2023-08-28T00:21:17.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:17.418689+0000) 2023-08-28T00:21:17.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:17 smithi006 bash[33918]: cluster 2023-08-28T00:21:16.440204+0000 mgr.y (mgr.14901) 4281 : cluster [DBG] pgmap v4288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:17.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:17 smithi006 bash[33918]: audit 2023-08-28T00:21:17.330211+0000 mon.a (mon.0) 3166 : audit [DBG] from='client.? 172.21.15.6:0/942873017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:17.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:17 smithi006 bash[27292]: cluster 2023-08-28T00:21:16.440204+0000 mgr.y (mgr.14901) 4281 : cluster [DBG] pgmap v4288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:17.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:17 smithi006 bash[27292]: audit 2023-08-28T00:21:17.330211+0000 mon.a (mon.0) 3166 : audit [DBG] from='client.? 172.21.15.6:0/942873017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:18.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:17 smithi130 bash[27579]: cluster 2023-08-28T00:21:16.440204+0000 mgr.y (mgr.14901) 4281 : cluster [DBG] pgmap v4288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:18.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:17 smithi130 bash[27579]: audit 2023-08-28T00:21:17.330211+0000 mon.a (mon.0) 3166 : audit [DBG] from='client.? 172.21.15.6:0/942873017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:18 smithi130 bash[51302]: debug 2023-08-28T00:21:18.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:18.419034+0000) 2023-08-28T00:21:19.715 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:19 smithi130 bash[51302]: debug 2023-08-28T00:21:19.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:19.419388+0000) 2023-08-28T00:21:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:19 smithi006 bash[33918]: cluster 2023-08-28T00:21:18.440861+0000 mgr.y (mgr.14901) 4282 : cluster [DBG] pgmap v4289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:19.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:19 smithi006 bash[27292]: cluster 2023-08-28T00:21:18.440861+0000 mgr.y (mgr.14901) 4282 : cluster [DBG] pgmap v4289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:20.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:19 smithi130 bash[27579]: cluster 2023-08-28T00:21:18.440861+0000 mgr.y (mgr.14901) 4282 : cluster [DBG] pgmap v4289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:20.725 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:20 smithi130 bash[51302]: debug 2023-08-28T00:21:20.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:20.419685+0000) 2023-08-28T00:21:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:21:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:21:20] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T00:21:20.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:20 smithi006 bash[33918]: audit 2023-08-28T00:21:19.747197+0000 mon.a (mon.0) 3167 : audit [DBG] from='client.? 172.21.15.6:0/2036314317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:20.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:20 smithi006 bash[27292]: audit 2023-08-28T00:21:19.747197+0000 mon.a (mon.0) 3167 : audit [DBG] from='client.? 172.21.15.6:0/2036314317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:20.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:21:20.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:20.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:21:20.712Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:20 smithi130 bash[27579]: audit 2023-08-28T00:21:19.747197+0000 mon.a (mon.0) 3167 : audit [DBG] from='client.? 172.21.15.6:0/2036314317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:21.727 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:21 smithi130 bash[51302]: debug 2023-08-28T00:21:21.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:21.419983+0000) 2023-08-28T00:21:22.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:21 smithi130 bash[27579]: cluster 2023-08-28T00:21:20.441945+0000 mgr.y (mgr.14901) 4283 : cluster [DBG] pgmap v4290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:22.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:21 smithi006 bash[33918]: cluster 2023-08-28T00:21:20.441945+0000 mgr.y (mgr.14901) 4283 : cluster [DBG] pgmap v4290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:22.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:21 smithi006 bash[27292]: cluster 2023-08-28T00:21:20.441945+0000 mgr.y (mgr.14901) 4283 : cluster [DBG] pgmap v4290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:21:22.289Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:21:22.291Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:22.730 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:22 smithi130 bash[51302]: debug 2023-08-28T00:21:22.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:22.420281+0000) 2023-08-28T00:21:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:22 smithi130 bash[27579]: audit 2023-08-28T00:21:22.159928+0000 mon.c (mon.2) 1639 : audit [DBG] from='client.? 172.21.15.6:0/2101613476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:22 smithi006 bash[33918]: audit 2023-08-28T00:21:22.159928+0000 mon.c (mon.2) 1639 : audit [DBG] from='client.? 172.21.15.6:0/2101613476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:23.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:22 smithi006 bash[27292]: audit 2023-08-28T00:21:22.159928+0000 mon.c (mon.2) 1639 : audit [DBG] from='client.? 172.21.15.6:0/2101613476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:23.740 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:23 smithi130 bash[51302]: debug 2023-08-28T00:21:23.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:23.420586+0000) 2023-08-28T00:21:24.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:23 smithi130 bash[27579]: cluster 2023-08-28T00:21:22.442739+0000 mgr.y (mgr.14901) 4284 : cluster [DBG] pgmap v4291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:24.176 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:23 smithi006 bash[27292]: cluster 2023-08-28T00:21:22.442739+0000 mgr.y (mgr.14901) 4284 : cluster [DBG] pgmap v4291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:24.177 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:23 smithi006 bash[33918]: cluster 2023-08-28T00:21:22.442739+0000 mgr.y (mgr.14901) 4284 : cluster [DBG] pgmap v4291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:21:24.174Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:21:24.176Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:24.749 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:24 smithi130 bash[51302]: debug 2023-08-28T00:21:24.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:24.420868+0000) 2023-08-28T00:21:25.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:24 smithi130 bash[27579]: audit 2023-08-28T00:21:24.575363+0000 mon.a (mon.0) 3168 : audit [DBG] from='client.? 172.21.15.6:0/2898453157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:24 smithi006 bash[33918]: audit 2023-08-28T00:21:24.575363+0000 mon.a (mon.0) 3168 : audit [DBG] from='client.? 172.21.15.6:0/2898453157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:24 smithi006 bash[27292]: audit 2023-08-28T00:21:24.575363+0000 mon.a (mon.0) 3168 : audit [DBG] from='client.? 172.21.15.6:0/2898453157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:25.751 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:25 smithi130 bash[51302]: debug 2023-08-28T00:21:25.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:25.421139+0000) 2023-08-28T00:21:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:25 smithi130 bash[27579]: cluster 2023-08-28T00:21:24.443742+0000 mgr.y (mgr.14901) 4285 : cluster [DBG] pgmap v4292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:26.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:25 smithi006 bash[33918]: cluster 2023-08-28T00:21:24.443742+0000 mgr.y (mgr.14901) 4285 : cluster [DBG] pgmap v4292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:26.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:25 smithi006 bash[27292]: cluster 2023-08-28T00:21:24.443742+0000 mgr.y (mgr.14901) 4285 : cluster [DBG] pgmap v4292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:26.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:26 smithi130 bash[51302]: debug 2023-08-28T00:21:26.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:26.421358+0000) 2023-08-28T00:21:27.759 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:27 smithi130 bash[51302]: debug 2023-08-28T00:21:27.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:27.421613+0000) 2023-08-28T00:21:28.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:27 smithi130 bash[27579]: cluster 2023-08-28T00:21:26.444405+0000 mgr.y (mgr.14901) 4286 : cluster [DBG] pgmap v4293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:28.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:27 smithi130 bash[27579]: audit 2023-08-28T00:21:26.991353+0000 mon.a (mon.0) 3169 : audit [DBG] from='client.? 172.21.15.6:0/101499117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:28.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:27 smithi006 bash[27292]: cluster 2023-08-28T00:21:26.444405+0000 mgr.y (mgr.14901) 4286 : cluster [DBG] pgmap v4293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:28.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:27 smithi006 bash[27292]: audit 2023-08-28T00:21:26.991353+0000 mon.a (mon.0) 3169 : audit [DBG] from='client.? 172.21.15.6:0/101499117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:28.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:27 smithi006 bash[33918]: cluster 2023-08-28T00:21:26.444405+0000 mgr.y (mgr.14901) 4286 : cluster [DBG] pgmap v4293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:28.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:27 smithi006 bash[33918]: audit 2023-08-28T00:21:26.991353+0000 mon.a (mon.0) 3169 : audit [DBG] from='client.? 172.21.15.6:0/101499117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:28 smithi130 bash[51302]: debug 2023-08-28T00:21:28.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:28.421875+0000) 2023-08-28T00:21:29.764 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:29 smithi130 bash[51302]: debug 2023-08-28T00:21:29.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:29.422208+0000) 2023-08-28T00:21:30.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:29 smithi130 bash[27579]: cluster 2023-08-28T00:21:28.445140+0000 mgr.y (mgr.14901) 4287 : cluster [DBG] pgmap v4294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:30.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:29 smithi130 bash[27579]: audit 2023-08-28T00:21:29.400088+0000 mon.c (mon.2) 1640 : audit [DBG] from='client.? 172.21.15.6:0/4176312043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:30.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:29 smithi006 bash[27292]: cluster 2023-08-28T00:21:28.445140+0000 mgr.y (mgr.14901) 4287 : cluster [DBG] pgmap v4294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:30.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:29 smithi006 bash[27292]: audit 2023-08-28T00:21:29.400088+0000 mon.c (mon.2) 1640 : audit [DBG] from='client.? 172.21.15.6:0/4176312043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:30.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:29 smithi006 bash[33918]: cluster 2023-08-28T00:21:28.445140+0000 mgr.y (mgr.14901) 4287 : cluster [DBG] pgmap v4294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:30.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:29 smithi006 bash[33918]: audit 2023-08-28T00:21:29.400088+0000 mon.c (mon.2) 1640 : audit [DBG] from='client.? 172.21.15.6:0/4176312043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:30 smithi130 bash[51302]: debug 2023-08-28T00:21:30.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:30.422513+0000) 2023-08-28T00:21:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:21:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:21:30] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T00:21:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:21:30.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:21:30.713Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:31.770 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:31 smithi130 bash[51302]: debug 2023-08-28T00:21:31.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:31.422842+0000) 2023-08-28T00:21:32.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:31 smithi130 bash[27579]: cluster 2023-08-28T00:21:30.446239+0000 mgr.y (mgr.14901) 4288 : cluster [DBG] pgmap v4295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:32.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:31 smithi006 bash[27292]: cluster 2023-08-28T00:21:30.446239+0000 mgr.y (mgr.14901) 4288 : cluster [DBG] pgmap v4295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:32.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:31 smithi006 bash[33918]: cluster 2023-08-28T00:21:30.446239+0000 mgr.y (mgr.14901) 4288 : cluster [DBG] pgmap v4295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:21:32.290Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:21:32.292Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:32.778 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:32 smithi130 bash[51302]: debug 2023-08-28T00:21:32.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:32.423139+0000) 2023-08-28T00:21:33.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:32 smithi130 bash[27579]: audit 2023-08-28T00:21:31.819534+0000 mon.a (mon.0) 3170 : audit [DBG] from='client.? 172.21.15.6:0/3390873211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:32 smithi006 bash[27292]: audit 2023-08-28T00:21:31.819534+0000 mon.a (mon.0) 3170 : audit [DBG] from='client.? 172.21.15.6:0/3390873211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:32 smithi006 bash[33918]: audit 2023-08-28T00:21:31.819534+0000 mon.a (mon.0) 3170 : audit [DBG] from='client.? 172.21.15.6:0/3390873211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:33.781 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:33 smithi130 bash[51302]: debug 2023-08-28T00:21:33.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:33.423407+0000) 2023-08-28T00:21:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:33 smithi130 bash[27579]: cluster 2023-08-28T00:21:32.446915+0000 mgr.y (mgr.14901) 4289 : cluster [DBG] pgmap v4296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:34.177 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:33 smithi006 bash[27292]: cluster 2023-08-28T00:21:32.446915+0000 mgr.y (mgr.14901) 4289 : cluster [DBG] pgmap v4296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:34.178 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:33 smithi006 bash[33918]: cluster 2023-08-28T00:21:32.446915+0000 mgr.y (mgr.14901) 4289 : cluster [DBG] pgmap v4296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:21:34.175Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:21:34.177Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:34.788 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:34 smithi130 bash[51302]: debug 2023-08-28T00:21:34.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:34.423758+0000) 2023-08-28T00:21:35.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:34 smithi130 bash[27579]: audit 2023-08-28T00:21:34.236548+0000 mon.a (mon.0) 3171 : audit [DBG] from='client.? 172.21.15.6:0/2251211169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:34 smithi006 bash[33918]: audit 2023-08-28T00:21:34.236548+0000 mon.a (mon.0) 3171 : audit [DBG] from='client.? 172.21.15.6:0/2251211169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:34 smithi006 bash[27292]: audit 2023-08-28T00:21:34.236548+0000 mon.a (mon.0) 3171 : audit [DBG] from='client.? 172.21.15.6:0/2251211169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:35.791 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:35 smithi130 bash[51302]: debug 2023-08-28T00:21:35.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:35.424008+0000) 2023-08-28T00:21:36.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:35 smithi130 bash[27579]: cluster 2023-08-28T00:21:34.448040+0000 mgr.y (mgr.14901) 4290 : cluster [DBG] pgmap v4297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:36.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:35 smithi006 bash[27292]: cluster 2023-08-28T00:21:34.448040+0000 mgr.y (mgr.14901) 4290 : cluster [DBG] pgmap v4297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:36.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:35 smithi006 bash[33918]: cluster 2023-08-28T00:21:34.448040+0000 mgr.y (mgr.14901) 4290 : cluster [DBG] pgmap v4297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:36.799 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:36 smithi130 bash[51302]: debug 2023-08-28T00:21:36.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:36.424275+0000) 2023-08-28T00:21:37.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:36 smithi130 bash[27579]: audit 2023-08-28T00:21:36.657515+0000 mon.c (mon.2) 1641 : audit [DBG] from='client.? 172.21.15.6:0/4029874734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:37.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:36 smithi006 bash[33918]: audit 2023-08-28T00:21:36.657515+0000 mon.c (mon.2) 1641 : audit [DBG] from='client.? 172.21.15.6:0/4029874734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:37.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:36 smithi006 bash[27292]: audit 2023-08-28T00:21:36.657515+0000 mon.c (mon.2) 1641 : audit [DBG] from='client.? 172.21.15.6:0/4029874734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:37.801 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:37 smithi130 bash[51302]: debug 2023-08-28T00:21:37.423+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:37.424623+0000) 2023-08-28T00:21:38.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:37 smithi130 bash[27579]: cluster 2023-08-28T00:21:36.448685+0000 mgr.y (mgr.14901) 4291 : cluster [DBG] pgmap v4298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:38.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:37 smithi006 bash[33918]: cluster 2023-08-28T00:21:36.448685+0000 mgr.y (mgr.14901) 4291 : cluster [DBG] pgmap v4298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:38.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:37 smithi006 bash[27292]: cluster 2023-08-28T00:21:36.448685+0000 mgr.y (mgr.14901) 4291 : cluster [DBG] pgmap v4298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:38 smithi130 bash[51302]: debug 2023-08-28T00:21:38.423+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:38.424906+0000) 2023-08-28T00:21:39.812 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:39 smithi130 bash[51302]: debug 2023-08-28T00:21:39.423+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:39.425220+0000) 2023-08-28T00:21:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:39 smithi130 bash[27579]: cluster 2023-08-28T00:21:38.449362+0000 mgr.y (mgr.14901) 4292 : cluster [DBG] pgmap v4299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:39 smithi130 bash[27579]: audit 2023-08-28T00:21:39.069163+0000 mon.a (mon.0) 3172 : audit [DBG] from='client.? 172.21.15.6:0/3739156249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:40.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:39 smithi006 bash[33918]: cluster 2023-08-28T00:21:38.449362+0000 mgr.y (mgr.14901) 4292 : cluster [DBG] pgmap v4299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:40.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:39 smithi006 bash[33918]: audit 2023-08-28T00:21:39.069163+0000 mon.a (mon.0) 3172 : audit [DBG] from='client.? 172.21.15.6:0/3739156249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:40.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:39 smithi006 bash[27292]: cluster 2023-08-28T00:21:38.449362+0000 mgr.y (mgr.14901) 4292 : cluster [DBG] pgmap v4299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:40.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:39 smithi006 bash[27292]: audit 2023-08-28T00:21:39.069163+0000 mon.a (mon.0) 3172 : audit [DBG] from='client.? 172.21.15.6:0/3739156249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:40.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:40 smithi130 bash[51302]: debug 2023-08-28T00:21:40.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:40.425493+0000) 2023-08-28T00:21:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:21:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:21:40] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T00:21:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:21:40.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:21:40.715Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:41.817 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:41 smithi130 bash[51302]: debug 2023-08-28T00:21:41.423+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:41.425777+0000) 2023-08-28T00:21:42.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:41 smithi130 bash[27579]: cluster 2023-08-28T00:21:40.450644+0000 mgr.y (mgr.14901) 4293 : cluster [DBG] pgmap v4300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:42.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:41 smithi130 bash[27579]: audit 2023-08-28T00:21:41.484225+0000 mon.a (mon.0) 3173 : audit [DBG] from='client.? 172.21.15.6:0/1064634684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:42.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:41 smithi006 bash[33918]: cluster 2023-08-28T00:21:40.450644+0000 mgr.y (mgr.14901) 4293 : cluster [DBG] pgmap v4300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:42.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:41 smithi006 bash[33918]: audit 2023-08-28T00:21:41.484225+0000 mon.a (mon.0) 3173 : audit [DBG] from='client.? 172.21.15.6:0/1064634684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:42.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:41 smithi006 bash[27292]: cluster 2023-08-28T00:21:40.450644+0000 mgr.y (mgr.14901) 4293 : cluster [DBG] pgmap v4300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:42.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:41 smithi006 bash[27292]: audit 2023-08-28T00:21:41.484225+0000 mon.a (mon.0) 3173 : audit [DBG] from='client.? 172.21.15.6:0/1064634684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:21:42.291Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:21:42.293Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:42 smithi130 bash[51302]: debug 2023-08-28T00:21:42.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:42.426092+0000) 2023-08-28T00:21:43.822 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:43 smithi130 bash[51302]: debug 2023-08-28T00:21:43.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:43.426329+0000) 2023-08-28T00:21:44.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:43 smithi130 bash[27579]: cluster 2023-08-28T00:21:42.451499+0000 mgr.y (mgr.14901) 4294 : cluster [DBG] pgmap v4301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:44.179 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:43 smithi006 bash[27292]: cluster 2023-08-28T00:21:42.451499+0000 mgr.y (mgr.14901) 4294 : cluster [DBG] pgmap v4301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:44.179 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:43 smithi006 bash[33918]: cluster 2023-08-28T00:21:42.451499+0000 mgr.y (mgr.14901) 4294 : cluster [DBG] pgmap v4301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:21:44.176Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:21:44.179Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:44.824 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:44 smithi130 bash[51302]: debug 2023-08-28T00:21:44.423+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:44.426589+0000) 2023-08-28T00:21:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:44 smithi130 bash[27579]: audit 2023-08-28T00:21:43.903463+0000 mon.a (mon.0) 3174 : audit [DBG] from='client.? 172.21.15.6:0/1010661011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:45.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:44 smithi006 bash[27292]: audit 2023-08-28T00:21:43.903463+0000 mon.a (mon.0) 3174 : audit [DBG] from='client.? 172.21.15.6:0/1010661011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:45.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:44 smithi006 bash[33918]: audit 2023-08-28T00:21:43.903463+0000 mon.a (mon.0) 3174 : audit [DBG] from='client.? 172.21.15.6:0/1010661011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:45.837 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:45 smithi130 bash[51302]: debug 2023-08-28T00:21:45.423+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:45.426886+0000) 2023-08-28T00:21:46.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:45 smithi130 bash[27579]: cluster 2023-08-28T00:21:44.452690+0000 mgr.y (mgr.14901) 4295 : cluster [DBG] pgmap v4302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:46.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:45 smithi006 bash[33918]: cluster 2023-08-28T00:21:44.452690+0000 mgr.y (mgr.14901) 4295 : cluster [DBG] pgmap v4302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:46.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:45 smithi006 bash[27292]: cluster 2023-08-28T00:21:44.452690+0000 mgr.y (mgr.14901) 4295 : cluster [DBG] pgmap v4302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:46.846 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:46 smithi130 bash[51302]: debug 2023-08-28T00:21:46.423+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:46.427109+0000) 2023-08-28T00:21:47.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:46 smithi130 bash[27579]: audit 2023-08-28T00:21:46.321237+0000 mon.a (mon.0) 3175 : audit [DBG] from='client.? 172.21.15.6:0/4057352447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:47.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:46 smithi006 bash[33918]: audit 2023-08-28T00:21:46.321237+0000 mon.a (mon.0) 3175 : audit [DBG] from='client.? 172.21.15.6:0/4057352447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:47.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:46 smithi006 bash[27292]: audit 2023-08-28T00:21:46.321237+0000 mon.a (mon.0) 3175 : audit [DBG] from='client.? 172.21.15.6:0/4057352447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:47.852 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:47 smithi130 bash[51302]: debug 2023-08-28T00:21:47.423+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:47.427391+0000) 2023-08-28T00:21:48.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:47 smithi130 bash[27579]: cluster 2023-08-28T00:21:46.453355+0000 mgr.y (mgr.14901) 4296 : cluster [DBG] pgmap v4303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:48.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:47 smithi006 bash[33918]: cluster 2023-08-28T00:21:46.453355+0000 mgr.y (mgr.14901) 4296 : cluster [DBG] pgmap v4303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:48.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:47 smithi006 bash[27292]: cluster 2023-08-28T00:21:46.453355+0000 mgr.y (mgr.14901) 4296 : cluster [DBG] pgmap v4303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:48.859 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:48 smithi130 bash[51302]: debug 2023-08-28T00:21:48.427+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:48.427668+0000) 2023-08-28T00:21:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:48 smithi130 bash[27579]: audit 2023-08-28T00:21:48.733869+0000 mon.c (mon.2) 1642 : audit [DBG] from='client.? 172.21.15.6:0/2905115627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:49.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:48 smithi006 bash[33918]: audit 2023-08-28T00:21:48.733869+0000 mon.c (mon.2) 1642 : audit [DBG] from='client.? 172.21.15.6:0/2905115627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:49.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:48 smithi006 bash[27292]: audit 2023-08-28T00:21:48.733869+0000 mon.c (mon.2) 1642 : audit [DBG] from='client.? 172.21.15.6:0/2905115627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:49.862 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:49 smithi130 bash[51302]: debug 2023-08-28T00:21:49.427+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:49.427961+0000) 2023-08-28T00:21:50.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:49 smithi130 bash[27579]: cluster 2023-08-28T00:21:48.453983+0000 mgr.y (mgr.14901) 4297 : cluster [DBG] pgmap v4304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:50.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:49 smithi006 bash[33918]: cluster 2023-08-28T00:21:48.453983+0000 mgr.y (mgr.14901) 4297 : cluster [DBG] pgmap v4304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:50.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:49 smithi006 bash[27292]: cluster 2023-08-28T00:21:48.453983+0000 mgr.y (mgr.14901) 4297 : cluster [DBG] pgmap v4304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:50 smithi130 bash[51302]: debug 2023-08-28T00:21:50.427+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:50.428230+0000) 2023-08-28T00:21:50.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:21:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:21:50] "GET /metrics HTTP/1.1" 200 193131 "" "Prometheus/2.33.4" 2023-08-28T00:21:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:21:50.713Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:21:50.715Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:51.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:51 smithi130 bash[51302]: debug 2023-08-28T00:21:51.423+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:51.428512+0000) 2023-08-28T00:21:52.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:51 smithi130 bash[27579]: cluster 2023-08-28T00:21:50.455119+0000 mgr.y (mgr.14901) 4298 : cluster [DBG] pgmap v4305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:52.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:51 smithi130 bash[27579]: audit 2023-08-28T00:21:51.146784+0000 mon.c (mon.2) 1643 : audit [DBG] from='client.? 172.21.15.6:0/443472327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:52.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:51 smithi006 bash[33918]: cluster 2023-08-28T00:21:50.455119+0000 mgr.y (mgr.14901) 4298 : cluster [DBG] pgmap v4305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:52.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:51 smithi006 bash[33918]: audit 2023-08-28T00:21:51.146784+0000 mon.c (mon.2) 1643 : audit [DBG] from='client.? 172.21.15.6:0/443472327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:52.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:51 smithi006 bash[27292]: cluster 2023-08-28T00:21:50.455119+0000 mgr.y (mgr.14901) 4298 : cluster [DBG] pgmap v4305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:52.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:51 smithi006 bash[27292]: audit 2023-08-28T00:21:51.146784+0000 mon.c (mon.2) 1643 : audit [DBG] from='client.? 172.21.15.6:0/443472327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:21:52.292Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:21:52.294Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:52.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:52 smithi130 bash[51302]: debug 2023-08-28T00:21:52.423+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:52.428787+0000) 2023-08-28T00:21:53.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:53 smithi130 bash[51302]: debug 2023-08-28T00:21:53.427+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:53.429074+0000) 2023-08-28T00:21:54.180 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:53 smithi006 bash[33918]: cluster 2023-08-28T00:21:52.455778+0000 mgr.y (mgr.14901) 4299 : cluster [DBG] pgmap v4306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:54.180 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:53 smithi006 bash[33918]: audit 2023-08-28T00:21:53.564556+0000 mon.a (mon.0) 3176 : audit [DBG] from='client.? 172.21.15.6:0/780539066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:54.180 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:53 smithi006 bash[27292]: cluster 2023-08-28T00:21:52.455778+0000 mgr.y (mgr.14901) 4299 : cluster [DBG] pgmap v4306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:54.180 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:53 smithi006 bash[27292]: audit 2023-08-28T00:21:53.564556+0000 mon.a (mon.0) 3176 : audit [DBG] from='client.? 172.21.15.6:0/780539066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:53 smithi130 bash[27579]: cluster 2023-08-28T00:21:52.455778+0000 mgr.y (mgr.14901) 4299 : cluster [DBG] pgmap v4306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:53 smithi130 bash[27579]: audit 2023-08-28T00:21:53.564556+0000 mon.a (mon.0) 3176 : audit [DBG] from='client.? 172.21.15.6:0/780539066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:21:54.177Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:21:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:21:54.179Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:21:54.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:54 smithi130 bash[51302]: debug 2023-08-28T00:21:54.427+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:54.429359+0000) 2023-08-28T00:21:55.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:55 smithi130 bash[51302]: debug 2023-08-28T00:21:55.423+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:55.429664+0000) 2023-08-28T00:21:56.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:55 smithi006 bash[27292]: cluster 2023-08-28T00:21:54.456927+0000 mgr.y (mgr.14901) 4300 : cluster [DBG] pgmap v4307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:56.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:55 smithi006 bash[33918]: cluster 2023-08-28T00:21:54.456927+0000 mgr.y (mgr.14901) 4300 : cluster [DBG] pgmap v4307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:55 smithi130 bash[27579]: cluster 2023-08-28T00:21:54.456927+0000 mgr.y (mgr.14901) 4300 : cluster [DBG] pgmap v4307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:56.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:56 smithi130 bash[51302]: debug 2023-08-28T00:21:56.423+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:56.429973+0000) 2023-08-28T00:21:57.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:56 smithi006 bash[33918]: audit 2023-08-28T00:21:55.977503+0000 mon.a (mon.0) 3177 : audit [DBG] from='client.? 172.21.15.6:0/855471265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:57.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:56 smithi006 bash[27292]: audit 2023-08-28T00:21:55.977503+0000 mon.a (mon.0) 3177 : audit [DBG] from='client.? 172.21.15.6:0/855471265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:57.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:56 smithi130 bash[27579]: audit 2023-08-28T00:21:55.977503+0000 mon.a (mon.0) 3177 : audit [DBG] from='client.? 172.21.15.6:0/855471265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:57.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:57 smithi130 bash[51302]: debug 2023-08-28T00:21:57.423+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:57.430177+0000) 2023-08-28T00:21:58.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:57 smithi006 bash[33918]: cluster 2023-08-28T00:21:56.457584+0000 mgr.y (mgr.14901) 4301 : cluster [DBG] pgmap v4308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:58.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:57 smithi006 bash[27292]: cluster 2023-08-28T00:21:56.457584+0000 mgr.y (mgr.14901) 4301 : cluster [DBG] pgmap v4308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:58.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:57 smithi130 bash[27579]: cluster 2023-08-28T00:21:56.457584+0000 mgr.y (mgr.14901) 4301 : cluster [DBG] pgmap v4308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:21:58.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:58 smithi130 bash[51302]: debug 2023-08-28T00:21:58.423+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:58.430467+0000) 2023-08-28T00:21:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:58 smithi006 bash[33918]: audit 2023-08-28T00:21:58.405683+0000 mon.c (mon.2) 1644 : audit [DBG] from='client.? 172.21.15.6:0/746318739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:58 smithi006 bash[27292]: audit 2023-08-28T00:21:58.405683+0000 mon.c (mon.2) 1644 : audit [DBG] from='client.? 172.21.15.6:0/746318739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:58 smithi130 bash[27579]: audit 2023-08-28T00:21:58.405683+0000 mon.c (mon.2) 1644 : audit [DBG] from='client.? 172.21.15.6:0/746318739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:59.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:21:59 smithi130 bash[51302]: debug 2023-08-28T00:21:59.423+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:59.430691+0000) 2023-08-28T00:22:00.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:21:59 smithi006 bash[33918]: cluster 2023-08-28T00:21:58.458242+0000 mgr.y (mgr.14901) 4302 : cluster [DBG] pgmap v4309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:00.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:21:59 smithi006 bash[27292]: cluster 2023-08-28T00:21:58.458242+0000 mgr.y (mgr.14901) 4302 : cluster [DBG] pgmap v4309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:00.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:21:59 smithi130 bash[27579]: cluster 2023-08-28T00:21:58.458242+0000 mgr.y (mgr.14901) 4302 : cluster [DBG] pgmap v4309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:00.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:00 smithi130 bash[51302]: debug 2023-08-28T00:22:00.427+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:00.431010+0000) 2023-08-28T00:22:00.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:22:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:22:00] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:22:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:22:00.713Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:00.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:22:00.715Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:01.701 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:01 smithi130 bash[51302]: debug 2023-08-28T00:22:01.431+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:01.431288+0000) 2023-08-28T00:22:01.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:01 smithi006 bash[27292]: cluster 2023-08-28T00:22:00.459214+0000 mgr.y (mgr.14901) 4303 : cluster [DBG] pgmap v4310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:01.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:01 smithi006 bash[27292]: audit 2023-08-28T00:22:00.829181+0000 mon.c (mon.2) 1645 : audit [DBG] from='client.? 172.21.15.6:0/2448227745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:01.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:01 smithi006 bash[33918]: cluster 2023-08-28T00:22:00.459214+0000 mgr.y (mgr.14901) 4303 : cluster [DBG] pgmap v4310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:01.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:01 smithi006 bash[33918]: audit 2023-08-28T00:22:00.829181+0000 mon.c (mon.2) 1645 : audit [DBG] from='client.? 172.21.15.6:0/2448227745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:02.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:01 smithi130 bash[27579]: cluster 2023-08-28T00:22:00.459214+0000 mgr.y (mgr.14901) 4303 : cluster [DBG] pgmap v4310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:02.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:01 smithi130 bash[27579]: audit 2023-08-28T00:22:00.829181+0000 mon.c (mon.2) 1645 : audit [DBG] from='client.? 172.21.15.6:0/2448227745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:02.710 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:22:02.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:02.710 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:22:02.295Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:02.711 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:02 smithi130 bash[51302]: debug 2023-08-28T00:22:02.427+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:02.431572+0000) 2023-08-28T00:22:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:02 smithi006 bash[33918]: audit 2023-08-28T00:22:02.682847+0000 mon.b (mon.1) 535 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:22:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:02 smithi006 bash[27292]: audit 2023-08-28T00:22:02.682847+0000 mon.b (mon.1) 535 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:22:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:02 smithi130 bash[27579]: audit 2023-08-28T00:22:02.682847+0000 mon.b (mon.1) 535 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:22:03.714 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:03 smithi130 bash[51302]: debug 2023-08-28T00:22:03.427+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:03.431863+0000) 2023-08-28T00:22:03.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:03 smithi006 bash[33918]: cluster 2023-08-28T00:22:02.459885+0000 mgr.y (mgr.14901) 4304 : cluster [DBG] pgmap v4311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:03.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:03 smithi006 bash[33918]: audit 2023-08-28T00:22:03.247057+0000 mon.c (mon.2) 1646 : audit [DBG] from='client.? 172.21.15.6:0/1272714687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:03.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:03 smithi006 bash[27292]: cluster 2023-08-28T00:22:02.459885+0000 mgr.y (mgr.14901) 4304 : cluster [DBG] pgmap v4311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:03.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:03 smithi006 bash[27292]: audit 2023-08-28T00:22:03.247057+0000 mon.c (mon.2) 1646 : audit [DBG] from='client.? 172.21.15.6:0/1272714687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:04.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:03 smithi130 bash[27579]: cluster 2023-08-28T00:22:02.459885+0000 mgr.y (mgr.14901) 4304 : cluster [DBG] pgmap v4311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:04.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:03 smithi130 bash[27579]: audit 2023-08-28T00:22:03.247057+0000 mon.c (mon.2) 1646 : audit [DBG] from='client.? 172.21.15.6:0/1272714687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:22:04.177Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:22:04.179Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:04.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:04 smithi130 bash[51302]: debug 2023-08-28T00:22:04.431+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:04.432214+0000) 2023-08-28T00:22:05.720 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:05 smithi130 bash[51302]: debug 2023-08-28T00:22:05.431+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:05.432559+0000) 2023-08-28T00:22:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:05 smithi130 bash[27579]: cluster 2023-08-28T00:22:04.461008+0000 mgr.y (mgr.14901) 4305 : cluster [DBG] pgmap v4312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:05 smithi130 bash[27579]: audit 2023-08-28T00:22:05.666797+0000 mon.a (mon.0) 3178 : audit [DBG] from='client.? 172.21.15.6:0/3843529930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:05 smithi006 bash[33918]: cluster 2023-08-28T00:22:04.461008+0000 mgr.y (mgr.14901) 4305 : cluster [DBG] pgmap v4312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:05 smithi006 bash[33918]: audit 2023-08-28T00:22:05.666797+0000 mon.a (mon.0) 3178 : audit [DBG] from='client.? 172.21.15.6:0/3843529930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:06.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:05 smithi006 bash[27292]: cluster 2023-08-28T00:22:04.461008+0000 mgr.y (mgr.14901) 4305 : cluster [DBG] pgmap v4312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:06.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:05 smithi006 bash[27292]: audit 2023-08-28T00:22:05.666797+0000 mon.a (mon.0) 3178 : audit [DBG] from='client.? 172.21.15.6:0/3843529930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:06.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:06 smithi130 bash[51302]: debug 2023-08-28T00:22:06.427+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:06.432867+0000) 2023-08-28T00:22:07.727 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:07 smithi130 bash[51302]: debug 2023-08-28T00:22:07.426+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:07.433188+0000) 2023-08-28T00:22:08.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:07 smithi130 bash[27579]: cluster 2023-08-28T00:22:06.461690+0000 mgr.y (mgr.14901) 4306 : cluster [DBG] pgmap v4313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:08.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:07 smithi006 bash[33918]: cluster 2023-08-28T00:22:06.461690+0000 mgr.y (mgr.14901) 4306 : cluster [DBG] pgmap v4313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:08.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:07 smithi006 bash[27292]: cluster 2023-08-28T00:22:06.461690+0000 mgr.y (mgr.14901) 4306 : cluster [DBG] pgmap v4313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:08.737 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:08 smithi130 bash[51302]: debug 2023-08-28T00:22:08.426+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:08.433445+0000) 2023-08-28T00:22:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:08 smithi130 bash[27579]: audit 2023-08-28T00:22:08.120652+0000 mon.c (mon.2) 1647 : audit [DBG] from='client.? 172.21.15.6:0/1666205467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:08 smithi130 bash[27579]: audit 2023-08-28T00:22:08.576296+0000 mon.a (mon.0) 3179 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:22:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:08 smithi006 bash[33918]: audit 2023-08-28T00:22:08.120652+0000 mon.c (mon.2) 1647 : audit [DBG] from='client.? 172.21.15.6:0/1666205467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:08 smithi006 bash[33918]: audit 2023-08-28T00:22:08.576296+0000 mon.a (mon.0) 3179 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:22:09.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:08 smithi006 bash[27292]: audit 2023-08-28T00:22:08.120652+0000 mon.c (mon.2) 1647 : audit [DBG] from='client.? 172.21.15.6:0/1666205467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:09.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:08 smithi006 bash[27292]: audit 2023-08-28T00:22:08.576296+0000 mon.a (mon.0) 3179 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:22:09.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:09 smithi130 bash[51302]: debug 2023-08-28T00:22:09.430+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:09.433723+0000) 2023-08-28T00:22:10.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:10 smithi130 bash[27579]: cluster 2023-08-28T00:22:08.462352+0000 mgr.y (mgr.14901) 4307 : cluster [DBG] pgmap v4314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:10.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:10 smithi130 bash[27579]: audit 2023-08-28T00:22:09.019607+0000 mon.a (mon.0) 3180 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:22:10.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:10 smithi130 bash[27579]: audit 2023-08-28T00:22:09.287260+0000 mon.b (mon.1) 536 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:22:10.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:10 smithi130 bash[27579]: audit 2023-08-28T00:22:09.289619+0000 mon.b (mon.1) 537 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:22:10.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:10 smithi130 bash[27579]: audit 2023-08-28T00:22:09.299051+0000 mon.a (mon.0) 3181 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:22:10.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:10 smithi006 bash[33918]: cluster 2023-08-28T00:22:08.462352+0000 mgr.y (mgr.14901) 4307 : cluster [DBG] pgmap v4314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:10.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:10 smithi006 bash[33918]: audit 2023-08-28T00:22:09.019607+0000 mon.a (mon.0) 3180 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:22:10.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:10 smithi006 bash[33918]: audit 2023-08-28T00:22:09.287260+0000 mon.b (mon.1) 536 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:22:10.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:10 smithi006 bash[33918]: audit 2023-08-28T00:22:09.289619+0000 mon.b (mon.1) 537 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:22:10.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:10 smithi006 bash[33918]: audit 2023-08-28T00:22:09.299051+0000 mon.a (mon.0) 3181 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:22:10.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:10 smithi006 bash[27292]: cluster 2023-08-28T00:22:08.462352+0000 mgr.y (mgr.14901) 4307 : cluster [DBG] pgmap v4314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:10.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:10 smithi006 bash[27292]: audit 2023-08-28T00:22:09.019607+0000 mon.a (mon.0) 3180 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:22:10.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:10 smithi006 bash[27292]: audit 2023-08-28T00:22:09.287260+0000 mon.b (mon.1) 536 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:22:10.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:10 smithi006 bash[27292]: audit 2023-08-28T00:22:09.289619+0000 mon.b (mon.1) 537 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:22:10.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:10 smithi006 bash[27292]: audit 2023-08-28T00:22:09.299051+0000 mon.a (mon.0) 3181 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:22:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:10 smithi130 bash[51302]: debug 2023-08-28T00:22:10.426+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:10.434040+0000) 2023-08-28T00:22:11.027 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:22:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:22:10] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:22:11.027 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:22:10.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:11.027 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:22:10.716Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:11.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:11 smithi130 bash[27579]: audit 2023-08-28T00:22:10.546104+0000 mon.a (mon.0) 3182 : audit [DBG] from='client.? 172.21.15.6:0/1921422305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:11.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:11 smithi006 bash[27292]: audit 2023-08-28T00:22:10.546104+0000 mon.a (mon.0) 3182 : audit [DBG] from='client.? 172.21.15.6:0/1921422305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:11.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:11 smithi006 bash[33918]: audit 2023-08-28T00:22:10.546104+0000 mon.a (mon.0) 3182 : audit [DBG] from='client.? 172.21.15.6:0/1921422305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:11.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:11 smithi130 bash[51302]: debug 2023-08-28T00:22:11.426+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:11.434340+0000) 2023-08-28T00:22:12.296 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:12 smithi006 bash[33918]: cluster 2023-08-28T00:22:10.463349+0000 mgr.y (mgr.14901) 4308 : cluster [DBG] pgmap v4315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:12.297 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:12 smithi006 bash[27292]: cluster 2023-08-28T00:22:10.463349+0000 mgr.y (mgr.14901) 4308 : cluster [DBG] pgmap v4315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:12.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:12 smithi130 bash[27579]: cluster 2023-08-28T00:22:10.463349+0000 mgr.y (mgr.14901) 4308 : cluster [DBG] pgmap v4315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:22:12.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:22:12.296Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:12 smithi130 bash[51302]: debug 2023-08-28T00:22:12.430+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:12.434650+0000) 2023-08-28T00:22:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:13 smithi130 bash[27579]: audit 2023-08-28T00:22:12.960626+0000 mon.c (mon.2) 1648 : audit [DBG] from='client.? 172.21.15.6:0/3463639523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:13.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:13 smithi006 bash[33918]: audit 2023-08-28T00:22:12.960626+0000 mon.c (mon.2) 1648 : audit [DBG] from='client.? 172.21.15.6:0/3463639523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:13.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:13 smithi006 bash[27292]: audit 2023-08-28T00:22:12.960626+0000 mon.c (mon.2) 1648 : audit [DBG] from='client.? 172.21.15.6:0/3463639523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:13.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:13 smithi130 bash[51302]: debug 2023-08-28T00:22:13.430+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:13.434971+0000) 2023-08-28T00:22:14.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:14 smithi130 bash[27579]: cluster 2023-08-28T00:22:12.463921+0000 mgr.y (mgr.14901) 4309 : cluster [DBG] pgmap v4316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:14.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:14 smithi006 bash[33918]: cluster 2023-08-28T00:22:12.463921+0000 mgr.y (mgr.14901) 4309 : cluster [DBG] pgmap v4316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:14.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:14 smithi006 bash[27292]: cluster 2023-08-28T00:22:12.463921+0000 mgr.y (mgr.14901) 4309 : cluster [DBG] pgmap v4316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:22:14.178Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:22:14.180Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:14 smithi130 bash[51302]: debug 2023-08-28T00:22:14.430+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:14.435207+0000) 2023-08-28T00:22:15.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:15 smithi130 bash[51302]: debug 2023-08-28T00:22:15.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:15.435504+0000) 2023-08-28T00:22:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:16 smithi130 bash[27579]: cluster 2023-08-28T00:22:14.464979+0000 mgr.y (mgr.14901) 4310 : cluster [DBG] pgmap v4317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:16 smithi130 bash[27579]: audit 2023-08-28T00:22:15.368780+0000 mon.c (mon.2) 1649 : audit [DBG] from='client.? 172.21.15.6:0/26177031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:16.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:16 smithi006 bash[27292]: cluster 2023-08-28T00:22:14.464979+0000 mgr.y (mgr.14901) 4310 : cluster [DBG] pgmap v4317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:16.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:16 smithi006 bash[27292]: audit 2023-08-28T00:22:15.368780+0000 mon.c (mon.2) 1649 : audit [DBG] from='client.? 172.21.15.6:0/26177031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:16.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:16 smithi006 bash[33918]: cluster 2023-08-28T00:22:14.464979+0000 mgr.y (mgr.14901) 4310 : cluster [DBG] pgmap v4317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:16.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:16 smithi006 bash[33918]: audit 2023-08-28T00:22:15.368780+0000 mon.c (mon.2) 1649 : audit [DBG] from='client.? 172.21.15.6:0/26177031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:16.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:16 smithi130 bash[51302]: debug 2023-08-28T00:22:16.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:16.435810+0000) 2023-08-28T00:22:17.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:17 smithi130 bash[51302]: debug 2023-08-28T00:22:17.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:17.436072+0000) 2023-08-28T00:22:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:18 smithi130 bash[27579]: cluster 2023-08-28T00:22:16.465703+0000 mgr.y (mgr.14901) 4311 : cluster [DBG] pgmap v4318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:18 smithi130 bash[27579]: audit 2023-08-28T00:22:17.785569+0000 mon.c (mon.2) 1650 : audit [DBG] from='client.? 172.21.15.6:0/3431473366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:18.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:18 smithi006 bash[33918]: cluster 2023-08-28T00:22:16.465703+0000 mgr.y (mgr.14901) 4311 : cluster [DBG] pgmap v4318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:18.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:18 smithi006 bash[33918]: audit 2023-08-28T00:22:17.785569+0000 mon.c (mon.2) 1650 : audit [DBG] from='client.? 172.21.15.6:0/3431473366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:18.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:18 smithi006 bash[27292]: cluster 2023-08-28T00:22:16.465703+0000 mgr.y (mgr.14901) 4311 : cluster [DBG] pgmap v4318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:18.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:18 smithi006 bash[27292]: audit 2023-08-28T00:22:17.785569+0000 mon.c (mon.2) 1650 : audit [DBG] from='client.? 172.21.15.6:0/3431473366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:18 smithi130 bash[51302]: debug 2023-08-28T00:22:18.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:18.436343+0000) 2023-08-28T00:22:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:19 smithi130 bash[51302]: debug 2023-08-28T00:22:19.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:19.436614+0000) 2023-08-28T00:22:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:20 smithi130 bash[27579]: cluster 2023-08-28T00:22:18.466289+0000 mgr.y (mgr.14901) 4312 : cluster [DBG] pgmap v4319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:20.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:20 smithi006 bash[27292]: cluster 2023-08-28T00:22:18.466289+0000 mgr.y (mgr.14901) 4312 : cluster [DBG] pgmap v4319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:20.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:20 smithi006 bash[33918]: cluster 2023-08-28T00:22:18.466289+0000 mgr.y (mgr.14901) 4312 : cluster [DBG] pgmap v4319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:20 smithi130 bash[51302]: debug 2023-08-28T00:22:20.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:20.436897+0000) 2023-08-28T00:22:21.074 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:22:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:22:20] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:22:21.074 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:22:20.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:21.075 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:22:20.717Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:21.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:21 smithi130 bash[27579]: audit 2023-08-28T00:22:20.205776+0000 mon.c (mon.2) 1651 : audit [DBG] from='client.? 172.21.15.6:0/2184603248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:21.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:21 smithi006 bash[33918]: audit 2023-08-28T00:22:20.205776+0000 mon.c (mon.2) 1651 : audit [DBG] from='client.? 172.21.15.6:0/2184603248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:21.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:21 smithi006 bash[27292]: audit 2023-08-28T00:22:20.205776+0000 mon.c (mon.2) 1651 : audit [DBG] from='client.? 172.21.15.6:0/2184603248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:21.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:21 smithi130 bash[51302]: debug 2023-08-28T00:22:21.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:21.437212+0000) 2023-08-28T00:22:22.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:22 smithi130 bash[27579]: cluster 2023-08-28T00:22:20.467492+0000 mgr.y (mgr.14901) 4313 : cluster [DBG] pgmap v4320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:22.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:22 smithi006 bash[33918]: cluster 2023-08-28T00:22:20.467492+0000 mgr.y (mgr.14901) 4313 : cluster [DBG] pgmap v4320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:22.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:22 smithi006 bash[27292]: cluster 2023-08-28T00:22:20.467492+0000 mgr.y (mgr.14901) 4313 : cluster [DBG] pgmap v4320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:22.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:22:22.295Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:22.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:22:22.297Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:22 smithi130 bash[51302]: debug 2023-08-28T00:22:22.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:22.437496+0000) 2023-08-28T00:22:23.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:23 smithi130 bash[27579]: audit 2023-08-28T00:22:22.631088+0000 mon.a (mon.0) 3183 : audit [DBG] from='client.? 172.21.15.6:0/245855224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:23.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:23 smithi006 bash[27292]: audit 2023-08-28T00:22:22.631088+0000 mon.a (mon.0) 3183 : audit [DBG] from='client.? 172.21.15.6:0/245855224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:23.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:23 smithi006 bash[33918]: audit 2023-08-28T00:22:22.631088+0000 mon.a (mon.0) 3183 : audit [DBG] from='client.? 172.21.15.6:0/245855224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:23.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:23 smithi130 bash[51302]: debug 2023-08-28T00:22:23.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:23.437778+0000) 2023-08-28T00:22:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:24 smithi130 bash[27579]: cluster 2023-08-28T00:22:22.468169+0000 mgr.y (mgr.14901) 4314 : cluster [DBG] pgmap v4321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:24.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:24 smithi006 bash[27292]: cluster 2023-08-28T00:22:22.468169+0000 mgr.y (mgr.14901) 4314 : cluster [DBG] pgmap v4321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:24.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:24 smithi006 bash[33918]: cluster 2023-08-28T00:22:22.468169+0000 mgr.y (mgr.14901) 4314 : cluster [DBG] pgmap v4321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:22:24.179Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:22:24.181Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:24 smithi130 bash[51302]: debug 2023-08-28T00:22:24.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:24.438058+0000) 2023-08-28T00:22:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:25 smithi130 bash[27579]: audit 2023-08-28T00:22:25.050426+0000 mon.c (mon.2) 1652 : audit [DBG] from='client.? 172.21.15.6:0/3691477150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:25.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:25 smithi006 bash[27292]: audit 2023-08-28T00:22:25.050426+0000 mon.c (mon.2) 1652 : audit [DBG] from='client.? 172.21.15.6:0/3691477150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:25.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:25 smithi006 bash[33918]: audit 2023-08-28T00:22:25.050426+0000 mon.c (mon.2) 1652 : audit [DBG] from='client.? 172.21.15.6:0/3691477150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:25.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:25 smithi130 bash[51302]: debug 2023-08-28T00:22:25.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:25.438333+0000) 2023-08-28T00:22:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:26 smithi130 bash[27579]: cluster 2023-08-28T00:22:24.469320+0000 mgr.y (mgr.14901) 4315 : cluster [DBG] pgmap v4322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:26.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:26 smithi006 bash[33918]: cluster 2023-08-28T00:22:24.469320+0000 mgr.y (mgr.14901) 4315 : cluster [DBG] pgmap v4322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:26 smithi006 bash[27292]: cluster 2023-08-28T00:22:24.469320+0000 mgr.y (mgr.14901) 4315 : cluster [DBG] pgmap v4322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:26.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:26 smithi130 bash[51302]: debug 2023-08-28T00:22:26.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:26.438650+0000) 2023-08-28T00:22:27.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:27 smithi130 bash[51302]: debug 2023-08-28T00:22:27.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:27.438977+0000) 2023-08-28T00:22:28.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:28 smithi130 bash[27579]: cluster 2023-08-28T00:22:26.469960+0000 mgr.y (mgr.14901) 4316 : cluster [DBG] pgmap v4323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:28.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:28 smithi130 bash[27579]: audit 2023-08-28T00:22:27.458996+0000 mon.c (mon.2) 1653 : audit [DBG] from='client.? 172.21.15.6:0/3355343573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:28.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:28 smithi006 bash[33918]: cluster 2023-08-28T00:22:26.469960+0000 mgr.y (mgr.14901) 4316 : cluster [DBG] pgmap v4323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:28.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:28 smithi006 bash[33918]: audit 2023-08-28T00:22:27.458996+0000 mon.c (mon.2) 1653 : audit [DBG] from='client.? 172.21.15.6:0/3355343573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:28.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:28 smithi006 bash[27292]: cluster 2023-08-28T00:22:26.469960+0000 mgr.y (mgr.14901) 4316 : cluster [DBG] pgmap v4323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:28.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:28 smithi006 bash[27292]: audit 2023-08-28T00:22:27.458996+0000 mon.c (mon.2) 1653 : audit [DBG] from='client.? 172.21.15.6:0/3355343573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:28 smithi130 bash[51302]: debug 2023-08-28T00:22:28.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:28.439288+0000) 2023-08-28T00:22:29.870 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:29 smithi130 bash[51302]: debug 2023-08-28T00:22:29.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:29.439581+0000) 2023-08-28T00:22:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:30 smithi130 bash[27579]: cluster 2023-08-28T00:22:28.470638+0000 mgr.y (mgr.14901) 4317 : cluster [DBG] pgmap v4324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:30 smithi130 bash[27579]: audit 2023-08-28T00:22:29.871135+0000 mon.a (mon.0) 3184 : audit [DBG] from='client.? 172.21.15.6:0/930017903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:30.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:30 smithi006 bash[27292]: cluster 2023-08-28T00:22:28.470638+0000 mgr.y (mgr.14901) 4317 : cluster [DBG] pgmap v4324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:30.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:30 smithi006 bash[27292]: audit 2023-08-28T00:22:29.871135+0000 mon.a (mon.0) 3184 : audit [DBG] from='client.? 172.21.15.6:0/930017903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:30.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:30 smithi006 bash[33918]: cluster 2023-08-28T00:22:28.470638+0000 mgr.y (mgr.14901) 4317 : cluster [DBG] pgmap v4324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:30.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:30 smithi006 bash[33918]: audit 2023-08-28T00:22:29.871135+0000 mon.a (mon.0) 3184 : audit [DBG] from='client.? 172.21.15.6:0/930017903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:30 smithi130 bash[51302]: debug 2023-08-28T00:22:30.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:30.439805+0000) 2023-08-28T00:22:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:22:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:22:30] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:22:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:22:30.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:22:30.718Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:31.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:31 smithi130 bash[51302]: debug 2023-08-28T00:22:31.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:31.440092+0000) 2023-08-28T00:22:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:32 smithi130 bash[27579]: cluster 2023-08-28T00:22:30.471769+0000 mgr.y (mgr.14901) 4318 : cluster [DBG] pgmap v4325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:32.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:32 smithi006 bash[33918]: cluster 2023-08-28T00:22:30.471769+0000 mgr.y (mgr.14901) 4318 : cluster [DBG] pgmap v4325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:32.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:32 smithi006 bash[27292]: cluster 2023-08-28T00:22:30.471769+0000 mgr.y (mgr.14901) 4318 : cluster [DBG] pgmap v4325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:32.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:22:32.296Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:32.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:22:32.298Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:32 smithi130 bash[51302]: debug 2023-08-28T00:22:32.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:32.440395+0000) 2023-08-28T00:22:33.442 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:33 smithi130 bash[27579]: audit 2023-08-28T00:22:32.281814+0000 mon.c (mon.2) 1654 : audit [DBG] from='client.? 172.21.15.6:0/3898789388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:33.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:33 smithi006 bash[33918]: audit 2023-08-28T00:22:32.281814+0000 mon.c (mon.2) 1654 : audit [DBG] from='client.? 172.21.15.6:0/3898789388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:33.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:33 smithi006 bash[27292]: audit 2023-08-28T00:22:32.281814+0000 mon.c (mon.2) 1654 : audit [DBG] from='client.? 172.21.15.6:0/3898789388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:33.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:33 smithi130 bash[51302]: debug 2023-08-28T00:22:33.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:33.440645+0000) 2023-08-28T00:22:34.442 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:34 smithi130 bash[27579]: cluster 2023-08-28T00:22:32.472371+0000 mgr.y (mgr.14901) 4319 : cluster [DBG] pgmap v4326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:34.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:34 smithi006 bash[33918]: cluster 2023-08-28T00:22:32.472371+0000 mgr.y (mgr.14901) 4319 : cluster [DBG] pgmap v4326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:34.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:34 smithi006 bash[27292]: cluster 2023-08-28T00:22:32.472371+0000 mgr.y (mgr.14901) 4319 : cluster [DBG] pgmap v4326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:22:34.180Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:22:34.182Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:34 smithi130 bash[51302]: debug 2023-08-28T00:22:34.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:34.440896+0000) 2023-08-28T00:22:35.442 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:35 smithi130 bash[27579]: audit 2023-08-28T00:22:34.702469+0000 mon.c (mon.2) 1655 : audit [DBG] from='client.? 172.21.15.6:0/2169400808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:35.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:35 smithi006 bash[27292]: audit 2023-08-28T00:22:34.702469+0000 mon.c (mon.2) 1655 : audit [DBG] from='client.? 172.21.15.6:0/2169400808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:35.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:35 smithi006 bash[33918]: audit 2023-08-28T00:22:34.702469+0000 mon.c (mon.2) 1655 : audit [DBG] from='client.? 172.21.15.6:0/2169400808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:35.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:35 smithi130 bash[51302]: debug 2023-08-28T00:22:35.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:35.441176+0000) 2023-08-28T00:22:36.443 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:36 smithi130 bash[27579]: cluster 2023-08-28T00:22:34.473171+0000 mgr.y (mgr.14901) 4320 : cluster [DBG] pgmap v4327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:36.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:36 smithi006 bash[27292]: cluster 2023-08-28T00:22:34.473171+0000 mgr.y (mgr.14901) 4320 : cluster [DBG] pgmap v4327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:36.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:36 smithi006 bash[33918]: cluster 2023-08-28T00:22:34.473171+0000 mgr.y (mgr.14901) 4320 : cluster [DBG] pgmap v4327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:36 smithi130 bash[51302]: debug 2023-08-28T00:22:36.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:36.441475+0000) 2023-08-28T00:22:37.443 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:37 smithi130 bash[27579]: audit 2023-08-28T00:22:37.114895+0000 mon.a (mon.0) 3185 : audit [DBG] from='client.? 172.21.15.6:0/3696873913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:37.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:37 smithi006 bash[27292]: audit 2023-08-28T00:22:37.114895+0000 mon.a (mon.0) 3185 : audit [DBG] from='client.? 172.21.15.6:0/3696873913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:37.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:37 smithi006 bash[33918]: audit 2023-08-28T00:22:37.114895+0000 mon.a (mon.0) 3185 : audit [DBG] from='client.? 172.21.15.6:0/3696873913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:37 smithi130 bash[51302]: debug 2023-08-28T00:22:37.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:37.441788+0000) 2023-08-28T00:22:38.444 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:38 smithi130 bash[27579]: cluster 2023-08-28T00:22:36.473776+0000 mgr.y (mgr.14901) 4321 : cluster [DBG] pgmap v4328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:38.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:38 smithi006 bash[27292]: cluster 2023-08-28T00:22:36.473776+0000 mgr.y (mgr.14901) 4321 : cluster [DBG] pgmap v4328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:38.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:38 smithi006 bash[33918]: cluster 2023-08-28T00:22:36.473776+0000 mgr.y (mgr.14901) 4321 : cluster [DBG] pgmap v4328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:38 smithi130 bash[51302]: debug 2023-08-28T00:22:38.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:38.442045+0000) 2023-08-28T00:22:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:39 smithi130 bash[51302]: debug 2023-08-28T00:22:39.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:39.442307+0000) 2023-08-28T00:22:40.444 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:40 smithi130 bash[27579]: cluster 2023-08-28T00:22:38.474428+0000 mgr.y (mgr.14901) 4322 : cluster [DBG] pgmap v4329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:40.444 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:40 smithi130 bash[27579]: audit 2023-08-28T00:22:39.533239+0000 mon.c (mon.2) 1656 : audit [DBG] from='client.? 172.21.15.6:0/2881205501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:40.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:40 smithi006 bash[27292]: cluster 2023-08-28T00:22:38.474428+0000 mgr.y (mgr.14901) 4322 : cluster [DBG] pgmap v4329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:40.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:40 smithi006 bash[27292]: audit 2023-08-28T00:22:39.533239+0000 mon.c (mon.2) 1656 : audit [DBG] from='client.? 172.21.15.6:0/2881205501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:40.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:40 smithi006 bash[33918]: cluster 2023-08-28T00:22:38.474428+0000 mgr.y (mgr.14901) 4322 : cluster [DBG] pgmap v4329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:40.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:40 smithi006 bash[33918]: audit 2023-08-28T00:22:39.533239+0000 mon.c (mon.2) 1656 : audit [DBG] from='client.? 172.21.15.6:0/2881205501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:40 smithi130 bash[51302]: debug 2023-08-28T00:22:40.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:40.442563+0000) 2023-08-28T00:22:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:22:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:22:40] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:22:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:22:40.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:22:40.718Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:41.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:41 smithi130 bash[51302]: debug 2023-08-28T00:22:41.442+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:41.442885+0000) 2023-08-28T00:22:42.445 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:42 smithi130 bash[27579]: cluster 2023-08-28T00:22:40.475473+0000 mgr.y (mgr.14901) 4323 : cluster [DBG] pgmap v4330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:42.445 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:42 smithi130 bash[27579]: audit 2023-08-28T00:22:41.943429+0000 mon.a (mon.0) 3186 : audit [DBG] from='client.? 172.21.15.6:0/2110070129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:42.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:42 smithi006 bash[27292]: cluster 2023-08-28T00:22:40.475473+0000 mgr.y (mgr.14901) 4323 : cluster [DBG] pgmap v4330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:42.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:42 smithi006 bash[27292]: audit 2023-08-28T00:22:41.943429+0000 mon.a (mon.0) 3186 : audit [DBG] from='client.? 172.21.15.6:0/2110070129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:42.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:42 smithi006 bash[33918]: cluster 2023-08-28T00:22:40.475473+0000 mgr.y (mgr.14901) 4323 : cluster [DBG] pgmap v4330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:42.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:42 smithi006 bash[33918]: audit 2023-08-28T00:22:41.943429+0000 mon.a (mon.0) 3186 : audit [DBG] from='client.? 172.21.15.6:0/2110070129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:42.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:22:42.297Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:42.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:22:42.299Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:42 smithi130 bash[51302]: debug 2023-08-28T00:22:42.442+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:42.443128+0000) 2023-08-28T00:22:43.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:43 smithi130 bash[51302]: debug 2023-08-28T00:22:43.442+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:43.443335+0000) 2023-08-28T00:22:44.445 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:44 smithi130 bash[27579]: cluster 2023-08-28T00:22:42.476070+0000 mgr.y (mgr.14901) 4324 : cluster [DBG] pgmap v4331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:44.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:44 smithi006 bash[33918]: cluster 2023-08-28T00:22:42.476070+0000 mgr.y (mgr.14901) 4324 : cluster [DBG] pgmap v4331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:44.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:44 smithi006 bash[27292]: cluster 2023-08-28T00:22:42.476070+0000 mgr.y (mgr.14901) 4324 : cluster [DBG] pgmap v4331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:22:44.180Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:22:44.182Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:44 smithi130 bash[51302]: debug 2023-08-28T00:22:44.442+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:44.443560+0000) 2023-08-28T00:22:45.445 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:45 smithi130 bash[27579]: audit 2023-08-28T00:22:44.361733+0000 mon.a (mon.0) 3187 : audit [DBG] from='client.? 172.21.15.6:0/404013805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:45.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:45 smithi006 bash[33918]: audit 2023-08-28T00:22:44.361733+0000 mon.a (mon.0) 3187 : audit [DBG] from='client.? 172.21.15.6:0/404013805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:45.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:45 smithi006 bash[27292]: audit 2023-08-28T00:22:44.361733+0000 mon.a (mon.0) 3187 : audit [DBG] from='client.? 172.21.15.6:0/404013805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:45.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:45 smithi130 bash[51302]: debug 2023-08-28T00:22:45.442+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:45.443833+0000) 2023-08-28T00:22:46.446 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:46 smithi130 bash[27579]: cluster 2023-08-28T00:22:44.477108+0000 mgr.y (mgr.14901) 4325 : cluster [DBG] pgmap v4332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:46 smithi006 bash[33918]: cluster 2023-08-28T00:22:44.477108+0000 mgr.y (mgr.14901) 4325 : cluster [DBG] pgmap v4332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:46.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:46 smithi006 bash[27292]: cluster 2023-08-28T00:22:44.477108+0000 mgr.y (mgr.14901) 4325 : cluster [DBG] pgmap v4332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:46.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:46 smithi130 bash[51302]: debug 2023-08-28T00:22:46.442+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:46.444095+0000) 2023-08-28T00:22:47.446 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:47 smithi130 bash[27579]: audit 2023-08-28T00:22:46.772859+0000 mon.a (mon.0) 3188 : audit [DBG] from='client.? 172.21.15.6:0/1962291453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:47.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:47 smithi006 bash[27292]: audit 2023-08-28T00:22:46.772859+0000 mon.a (mon.0) 3188 : audit [DBG] from='client.? 172.21.15.6:0/1962291453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:47.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:47 smithi006 bash[33918]: audit 2023-08-28T00:22:46.772859+0000 mon.a (mon.0) 3188 : audit [DBG] from='client.? 172.21.15.6:0/1962291453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:47.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:47 smithi130 bash[51302]: debug 2023-08-28T00:22:47.442+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:47.444395+0000) 2023-08-28T00:22:48.446 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:48 smithi130 bash[27579]: cluster 2023-08-28T00:22:46.477662+0000 mgr.y (mgr.14901) 4326 : cluster [DBG] pgmap v4333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:48.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:48 smithi006 bash[33918]: cluster 2023-08-28T00:22:46.477662+0000 mgr.y (mgr.14901) 4326 : cluster [DBG] pgmap v4333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:48.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:48 smithi006 bash[27292]: cluster 2023-08-28T00:22:46.477662+0000 mgr.y (mgr.14901) 4326 : cluster [DBG] pgmap v4333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:48 smithi130 bash[51302]: debug 2023-08-28T00:22:48.442+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:48.444643+0000) 2023-08-28T00:22:49.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:49 smithi130 bash[51302]: debug 2023-08-28T00:22:49.442+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:49.444933+0000) 2023-08-28T00:22:50.447 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:50 smithi130 bash[27579]: cluster 2023-08-28T00:22:48.478290+0000 mgr.y (mgr.14901) 4327 : cluster [DBG] pgmap v4334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:50.447 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:50 smithi130 bash[27579]: audit 2023-08-28T00:22:49.195536+0000 mon.c (mon.2) 1657 : audit [DBG] from='client.? 172.21.15.6:0/868954138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:50.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:50 smithi006 bash[27292]: cluster 2023-08-28T00:22:48.478290+0000 mgr.y (mgr.14901) 4327 : cluster [DBG] pgmap v4334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:50.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:50 smithi006 bash[27292]: audit 2023-08-28T00:22:49.195536+0000 mon.c (mon.2) 1657 : audit [DBG] from='client.? 172.21.15.6:0/868954138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:50.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:50 smithi006 bash[33918]: cluster 2023-08-28T00:22:48.478290+0000 mgr.y (mgr.14901) 4327 : cluster [DBG] pgmap v4334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:50.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:50 smithi006 bash[33918]: audit 2023-08-28T00:22:49.195536+0000 mon.c (mon.2) 1657 : audit [DBG] from='client.? 172.21.15.6:0/868954138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:50 smithi130 bash[51302]: debug 2023-08-28T00:22:50.442+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:50.445219+0000) 2023-08-28T00:22:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:22:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:22:50] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:22:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:22:50.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:22:50.719Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:51.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:51 smithi130 bash[51302]: debug 2023-08-28T00:22:51.442+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:51.445486+0000) 2023-08-28T00:22:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:22:52.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:52.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:22:52.300Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:52.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:52 smithi006 bash[27292]: cluster 2023-08-28T00:22:50.479358+0000 mgr.y (mgr.14901) 4328 : cluster [DBG] pgmap v4335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:52.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:52 smithi006 bash[27292]: audit 2023-08-28T00:22:51.607138+0000 mon.a (mon.0) 3189 : audit [DBG] from='client.? 172.21.15.6:0/425698933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:52.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:52 smithi006 bash[33918]: cluster 2023-08-28T00:22:50.479358+0000 mgr.y (mgr.14901) 4328 : cluster [DBG] pgmap v4335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:52.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:52 smithi006 bash[33918]: audit 2023-08-28T00:22:51.607138+0000 mon.a (mon.0) 3189 : audit [DBG] from='client.? 172.21.15.6:0/425698933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:52.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:52 smithi130 bash[51302]: debug 2023-08-28T00:22:52.442+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:52.445784+0000) 2023-08-28T00:22:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:52 smithi130 bash[27579]: cluster 2023-08-28T00:22:50.479358+0000 mgr.y (mgr.14901) 4328 : cluster [DBG] pgmap v4335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:52 smithi130 bash[27579]: audit 2023-08-28T00:22:51.607138+0000 mon.a (mon.0) 3189 : audit [DBG] from='client.? 172.21.15.6:0/425698933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:53.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:53 smithi130 bash[51302]: debug 2023-08-28T00:22:53.442+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:53.446045+0000) 2023-08-28T00:22:54.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:54 smithi006 bash[27292]: cluster 2023-08-28T00:22:52.479979+0000 mgr.y (mgr.14901) 4329 : cluster [DBG] pgmap v4336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:54.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:54 smithi006 bash[27292]: audit 2023-08-28T00:22:54.025209+0000 mon.a (mon.0) 3190 : audit [DBG] from='client.? 172.21.15.6:0/3385876229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:54.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:54 smithi006 bash[33918]: cluster 2023-08-28T00:22:52.479979+0000 mgr.y (mgr.14901) 4329 : cluster [DBG] pgmap v4336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:54.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:54 smithi006 bash[33918]: audit 2023-08-28T00:22:54.025209+0000 mon.a (mon.0) 3190 : audit [DBG] from='client.? 172.21.15.6:0/3385876229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:22:54.182Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:22:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:22:54.184Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:22:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:54 smithi130 bash[27579]: cluster 2023-08-28T00:22:52.479979+0000 mgr.y (mgr.14901) 4329 : cluster [DBG] pgmap v4336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:54 smithi130 bash[27579]: audit 2023-08-28T00:22:54.025209+0000 mon.a (mon.0) 3190 : audit [DBG] from='client.? 172.21.15.6:0/3385876229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:54.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:54 smithi130 bash[51302]: debug 2023-08-28T00:22:54.442+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:54.446315+0000) 2023-08-28T00:22:55.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:55 smithi130 bash[51302]: debug 2023-08-28T00:22:55.442+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:55.446628+0000) 2023-08-28T00:22:56.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:56 smithi006 bash[27292]: cluster 2023-08-28T00:22:54.481167+0000 mgr.y (mgr.14901) 4330 : cluster [DBG] pgmap v4337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:56.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:56 smithi006 bash[33918]: cluster 2023-08-28T00:22:54.481167+0000 mgr.y (mgr.14901) 4330 : cluster [DBG] pgmap v4337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:56 smithi130 bash[27579]: cluster 2023-08-28T00:22:54.481167+0000 mgr.y (mgr.14901) 4330 : cluster [DBG] pgmap v4337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:56.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:56 smithi130 bash[51302]: debug 2023-08-28T00:22:56.446+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:56.446910+0000) 2023-08-28T00:22:57.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:57 smithi006 bash[27292]: audit 2023-08-28T00:22:56.439534+0000 mon.c (mon.2) 1658 : audit [DBG] from='client.? 172.21.15.6:0/2176585661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:57.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:57 smithi006 bash[33918]: audit 2023-08-28T00:22:56.439534+0000 mon.c (mon.2) 1658 : audit [DBG] from='client.? 172.21.15.6:0/2176585661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:57.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:57 smithi130 bash[27579]: audit 2023-08-28T00:22:56.439534+0000 mon.c (mon.2) 1658 : audit [DBG] from='client.? 172.21.15.6:0/2176585661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:57.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:57 smithi130 bash[51302]: debug 2023-08-28T00:22:57.446+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:57.447120+0000) 2023-08-28T00:22:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:58 smithi130 bash[27579]: cluster 2023-08-28T00:22:56.481889+0000 mgr.y (mgr.14901) 4331 : cluster [DBG] pgmap v4338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:58.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:58 smithi130 bash[51302]: debug 2023-08-28T00:22:58.446+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:58.447328+0000) 2023-08-28T00:22:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:58 smithi006 bash[27292]: cluster 2023-08-28T00:22:56.481889+0000 mgr.y (mgr.14901) 4331 : cluster [DBG] pgmap v4338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:58 smithi006 bash[33918]: cluster 2023-08-28T00:22:56.481889+0000 mgr.y (mgr.14901) 4331 : cluster [DBG] pgmap v4338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:22:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:22:59 smithi130 bash[27579]: audit 2023-08-28T00:22:58.849183+0000 mon.a (mon.0) 3191 : audit [DBG] from='client.? 172.21.15.6:0/1652712459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:59.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:22:59 smithi130 bash[51302]: debug 2023-08-28T00:22:59.446+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:59.447562+0000) 2023-08-28T00:22:59.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:22:59 smithi006 bash[27292]: audit 2023-08-28T00:22:58.849183+0000 mon.a (mon.0) 3191 : audit [DBG] from='client.? 172.21.15.6:0/1652712459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:59.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:22:59 smithi006 bash[33918]: audit 2023-08-28T00:22:58.849183+0000 mon.a (mon.0) 3191 : audit [DBG] from='client.? 172.21.15.6:0/1652712459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:00 smithi130 bash[27579]: cluster 2023-08-28T00:22:58.482603+0000 mgr.y (mgr.14901) 4332 : cluster [DBG] pgmap v4339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:00.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:00 smithi130 bash[51302]: debug 2023-08-28T00:23:00.446+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:00.447872+0000) 2023-08-28T00:23:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:00 smithi006 bash[33918]: cluster 2023-08-28T00:22:58.482603+0000 mgr.y (mgr.14901) 4332 : cluster [DBG] pgmap v4339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:00 smithi006 bash[27292]: cluster 2023-08-28T00:22:58.482603+0000 mgr.y (mgr.14901) 4332 : cluster [DBG] pgmap v4339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:23:00.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:23:00.720Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:23:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:23:00] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T00:23:01.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:01 smithi130 bash[51302]: debug 2023-08-28T00:23:01.446+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:01.448152+0000) 2023-08-28T00:23:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:02 smithi130 bash[27579]: cluster 2023-08-28T00:23:00.483841+0000 mgr.y (mgr.14901) 4333 : cluster [DBG] pgmap v4340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:02 smithi130 bash[27579]: audit 2023-08-28T00:23:01.263197+0000 mon.c (mon.2) 1659 : audit [DBG] from='client.? 172.21.15.6:0/2623026278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:02.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:02 smithi130 bash[51302]: debug 2023-08-28T00:23:02.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:02.448477+0000) 2023-08-28T00:23:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:02 smithi006 bash[27292]: cluster 2023-08-28T00:23:00.483841+0000 mgr.y (mgr.14901) 4333 : cluster [DBG] pgmap v4340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:02 smithi006 bash[27292]: audit 2023-08-28T00:23:01.263197+0000 mon.c (mon.2) 1659 : audit [DBG] from='client.? 172.21.15.6:0/2623026278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:02 smithi006 bash[33918]: cluster 2023-08-28T00:23:00.483841+0000 mgr.y (mgr.14901) 4333 : cluster [DBG] pgmap v4340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:02 smithi006 bash[33918]: audit 2023-08-28T00:23:01.263197+0000 mon.c (mon.2) 1659 : audit [DBG] from='client.? 172.21.15.6:0/2623026278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:23:02.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:23:02.300Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:03.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:03 smithi130 bash[51302]: debug 2023-08-28T00:23:03.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:03.448785+0000) 2023-08-28T00:23:04.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:04 smithi006 bash[33918]: cluster 2023-08-28T00:23:02.484529+0000 mgr.y (mgr.14901) 4334 : cluster [DBG] pgmap v4341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:04.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:04 smithi006 bash[33918]: audit 2023-08-28T00:23:03.683786+0000 mon.c (mon.2) 1660 : audit [DBG] from='client.? 172.21.15.6:0/670609673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:23:04.182Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:23:04.184Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:04.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:04 smithi006 bash[27292]: cluster 2023-08-28T00:23:02.484529+0000 mgr.y (mgr.14901) 4334 : cluster [DBG] pgmap v4341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:04.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:04 smithi006 bash[27292]: audit 2023-08-28T00:23:03.683786+0000 mon.c (mon.2) 1660 : audit [DBG] from='client.? 172.21.15.6:0/670609673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:04 smithi130 bash[27579]: cluster 2023-08-28T00:23:02.484529+0000 mgr.y (mgr.14901) 4334 : cluster [DBG] pgmap v4341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:04 smithi130 bash[27579]: audit 2023-08-28T00:23:03.683786+0000 mon.c (mon.2) 1660 : audit [DBG] from='client.? 172.21.15.6:0/670609673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:04.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:04 smithi130 bash[51302]: debug 2023-08-28T00:23:04.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:04.449080+0000) 2023-08-28T00:23:05.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:05 smithi130 bash[51302]: debug 2023-08-28T00:23:05.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:05.449389+0000) 2023-08-28T00:23:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:06 smithi130 bash[27579]: cluster 2023-08-28T00:23:04.485686+0000 mgr.y (mgr.14901) 4335 : cluster [DBG] pgmap v4342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:06 smithi130 bash[27579]: audit 2023-08-28T00:23:06.106438+0000 mon.a (mon.0) 3192 : audit [DBG] from='client.? 172.21.15.6:0/2944660822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:06.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:06 smithi130 bash[51302]: debug 2023-08-28T00:23:06.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:06.449669+0000) 2023-08-28T00:23:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:06 smithi006 bash[27292]: cluster 2023-08-28T00:23:04.485686+0000 mgr.y (mgr.14901) 4335 : cluster [DBG] pgmap v4342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:06 smithi006 bash[27292]: audit 2023-08-28T00:23:06.106438+0000 mon.a (mon.0) 3192 : audit [DBG] from='client.? 172.21.15.6:0/2944660822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:06 smithi006 bash[33918]: cluster 2023-08-28T00:23:04.485686+0000 mgr.y (mgr.14901) 4335 : cluster [DBG] pgmap v4342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:06 smithi006 bash[33918]: audit 2023-08-28T00:23:06.106438+0000 mon.a (mon.0) 3192 : audit [DBG] from='client.? 172.21.15.6:0/2944660822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:07.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:07 smithi130 bash[51302]: debug 2023-08-28T00:23:07.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:07.449996+0000) 2023-08-28T00:23:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:08 smithi130 bash[27579]: cluster 2023-08-28T00:23:06.486301+0000 mgr.y (mgr.14901) 4336 : cluster [DBG] pgmap v4343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:08.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:08 smithi130 bash[51302]: debug 2023-08-28T00:23:08.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:08.450302+0000) 2023-08-28T00:23:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:08 smithi006 bash[27292]: cluster 2023-08-28T00:23:06.486301+0000 mgr.y (mgr.14901) 4336 : cluster [DBG] pgmap v4343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:08 smithi006 bash[33918]: cluster 2023-08-28T00:23:06.486301+0000 mgr.y (mgr.14901) 4336 : cluster [DBG] pgmap v4343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:09 smithi130 bash[27579]: audit 2023-08-28T00:23:08.524800+0000 mon.c (mon.2) 1661 : audit [DBG] from='client.? 172.21.15.6:0/523566630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:09.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:09 smithi130 bash[51302]: debug 2023-08-28T00:23:09.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:09.450700+0000) 2023-08-28T00:23:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:09 smithi006 bash[27292]: audit 2023-08-28T00:23:08.524800+0000 mon.c (mon.2) 1661 : audit [DBG] from='client.? 172.21.15.6:0/523566630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:09.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:09 smithi006 bash[33918]: audit 2023-08-28T00:23:08.524800+0000 mon.c (mon.2) 1661 : audit [DBG] from='client.? 172.21.15.6:0/523566630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:10 smithi130 bash[27579]: cluster 2023-08-28T00:23:08.486835+0000 mgr.y (mgr.14901) 4337 : cluster [DBG] pgmap v4344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:10 smithi130 bash[27579]: audit 2023-08-28T00:23:09.308901+0000 mon.b (mon.1) 538 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:23:10.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:10 smithi130 bash[51302]: debug 2023-08-28T00:23:10.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:10.450991+0000) 2023-08-28T00:23:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:10 smithi006 bash[33918]: cluster 2023-08-28T00:23:08.486835+0000 mgr.y (mgr.14901) 4337 : cluster [DBG] pgmap v4344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:10 smithi006 bash[33918]: audit 2023-08-28T00:23:09.308901+0000 mon.b (mon.1) 538 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:23:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:10 smithi006 bash[27292]: cluster 2023-08-28T00:23:08.486835+0000 mgr.y (mgr.14901) 4337 : cluster [DBG] pgmap v4344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:10 smithi006 bash[27292]: audit 2023-08-28T00:23:09.308901+0000 mon.b (mon.1) 538 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:23:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:23:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:23:10] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T00:23:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:23:10.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:23:10.721Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:11.720 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:11 smithi130 bash[51302]: debug 2023-08-28T00:23:11.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:11.451286+0000) 2023-08-28T00:23:12.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:11 smithi130 bash[27579]: cluster 2023-08-28T00:23:10.488112+0000 mgr.y (mgr.14901) 4338 : cluster [DBG] pgmap v4345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:12.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:11 smithi130 bash[27579]: audit 2023-08-28T00:23:10.951676+0000 mon.c (mon.2) 1662 : audit [DBG] from='client.? 172.21.15.6:0/610335784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:12.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:11 smithi006 bash[33918]: cluster 2023-08-28T00:23:10.488112+0000 mgr.y (mgr.14901) 4338 : cluster [DBG] pgmap v4345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:12.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:11 smithi006 bash[33918]: audit 2023-08-28T00:23:10.951676+0000 mon.c (mon.2) 1662 : audit [DBG] from='client.? 172.21.15.6:0/610335784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:12.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:11 smithi006 bash[27292]: cluster 2023-08-28T00:23:10.488112+0000 mgr.y (mgr.14901) 4338 : cluster [DBG] pgmap v4345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:12.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:11 smithi006 bash[27292]: audit 2023-08-28T00:23:10.951676+0000 mon.c (mon.2) 1662 : audit [DBG] from='client.? 172.21.15.6:0/610335784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:23:12.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:23:12.300Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:12 smithi130 bash[51302]: debug 2023-08-28T00:23:12.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:12.451628+0000) 2023-08-28T00:23:13.726 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:13 smithi130 bash[51302]: debug 2023-08-28T00:23:13.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:13.451927+0000) 2023-08-28T00:23:14.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:13 smithi130 bash[27579]: cluster 2023-08-28T00:23:12.488872+0000 mgr.y (mgr.14901) 4339 : cluster [DBG] pgmap v4346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:14.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:13 smithi130 bash[27579]: audit 2023-08-28T00:23:13.373498+0000 mon.c (mon.2) 1663 : audit [DBG] from='client.? 172.21.15.6:0/552452262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:14.186 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:13 smithi006 bash[33918]: cluster 2023-08-28T00:23:12.488872+0000 mgr.y (mgr.14901) 4339 : cluster [DBG] pgmap v4346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:14.186 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:13 smithi006 bash[33918]: audit 2023-08-28T00:23:13.373498+0000 mon.c (mon.2) 1663 : audit [DBG] from='client.? 172.21.15.6:0/552452262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:14.186 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:13 smithi006 bash[27292]: cluster 2023-08-28T00:23:12.488872+0000 mgr.y (mgr.14901) 4339 : cluster [DBG] pgmap v4346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:14.187 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:13 smithi006 bash[27292]: audit 2023-08-28T00:23:13.373498+0000 mon.c (mon.2) 1663 : audit [DBG] from='client.? 172.21.15.6:0/552452262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:23:14.183Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:23:14.185Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:14 smithi130 bash[51302]: debug 2023-08-28T00:23:14.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:14.452132+0000) 2023-08-28T00:23:15.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:15 smithi130 bash[51302]: debug 2023-08-28T00:23:15.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:15.452382+0000) 2023-08-28T00:23:16.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:16 smithi130 bash[51302]: debug 2023-08-28T00:23:16.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:16.452672+0000) 2023-08-28T00:23:16.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:16 smithi130 bash[27579]: cluster 2023-08-28T00:23:14.489892+0000 mgr.y (mgr.14901) 4340 : cluster [DBG] pgmap v4347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:16.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:16 smithi130 bash[27579]: audit 2023-08-28T00:23:15.236304+0000 mon.a (mon.0) 3193 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:23:16.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:16 smithi130 bash[27579]: audit 2023-08-28T00:23:15.676809+0000 mon.a (mon.0) 3194 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:23:16.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:16 smithi130 bash[27579]: audit 2023-08-28T00:23:15.780152+0000 mon.c (mon.2) 1664 : audit [DBG] from='client.? 172.21.15.6:0/2873644796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:16.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:16 smithi130 bash[27579]: audit 2023-08-28T00:23:15.974590+0000 mon.b (mon.1) 539 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:23:16.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:16 smithi130 bash[27579]: audit 2023-08-28T00:23:15.977236+0000 mon.b (mon.1) 540 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:23:16.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:16 smithi130 bash[27579]: audit 2023-08-28T00:23:15.988357+0000 mon.a (mon.0) 3195 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:23:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:16 smithi006 bash[33918]: cluster 2023-08-28T00:23:14.489892+0000 mgr.y (mgr.14901) 4340 : cluster [DBG] pgmap v4347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:16 smithi006 bash[33918]: audit 2023-08-28T00:23:15.236304+0000 mon.a (mon.0) 3193 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:23:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:16 smithi006 bash[33918]: audit 2023-08-28T00:23:15.676809+0000 mon.a (mon.0) 3194 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:23:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:16 smithi006 bash[33918]: audit 2023-08-28T00:23:15.780152+0000 mon.c (mon.2) 1664 : audit [DBG] from='client.? 172.21.15.6:0/2873644796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:16 smithi006 bash[33918]: audit 2023-08-28T00:23:15.974590+0000 mon.b (mon.1) 539 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:23:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:16 smithi006 bash[33918]: audit 2023-08-28T00:23:15.977236+0000 mon.b (mon.1) 540 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:23:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:16 smithi006 bash[33918]: audit 2023-08-28T00:23:15.988357+0000 mon.a (mon.0) 3195 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:23:16.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:16 smithi006 bash[27292]: cluster 2023-08-28T00:23:14.489892+0000 mgr.y (mgr.14901) 4340 : cluster [DBG] pgmap v4347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:16.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:16 smithi006 bash[27292]: audit 2023-08-28T00:23:15.236304+0000 mon.a (mon.0) 3193 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:23:16.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:16 smithi006 bash[27292]: audit 2023-08-28T00:23:15.676809+0000 mon.a (mon.0) 3194 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:23:16.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:16 smithi006 bash[27292]: audit 2023-08-28T00:23:15.780152+0000 mon.c (mon.2) 1664 : audit [DBG] from='client.? 172.21.15.6:0/2873644796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:16.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:16 smithi006 bash[27292]: audit 2023-08-28T00:23:15.974590+0000 mon.b (mon.1) 539 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:23:16.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:16 smithi006 bash[27292]: audit 2023-08-28T00:23:15.977236+0000 mon.b (mon.1) 540 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:23:16.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:16 smithi006 bash[27292]: audit 2023-08-28T00:23:15.988357+0000 mon.a (mon.0) 3195 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:23:17.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:17 smithi130 bash[51302]: debug 2023-08-28T00:23:17.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:17.452981+0000) 2023-08-28T00:23:18.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:18 smithi130 bash[51302]: debug 2023-08-28T00:23:18.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:18.453205+0000) 2023-08-28T00:23:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:18 smithi130 bash[27579]: cluster 2023-08-28T00:23:16.490523+0000 mgr.y (mgr.14901) 4341 : cluster [DBG] pgmap v4348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:18.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:18 smithi130 bash[27579]: audit 2023-08-28T00:23:18.193145+0000 mon.c (mon.2) 1665 : audit [DBG] from='client.? 172.21.15.6:0/566448568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:18.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:18 smithi006 bash[27292]: cluster 2023-08-28T00:23:16.490523+0000 mgr.y (mgr.14901) 4341 : cluster [DBG] pgmap v4348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:18.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:18 smithi006 bash[27292]: audit 2023-08-28T00:23:18.193145+0000 mon.c (mon.2) 1665 : audit [DBG] from='client.? 172.21.15.6:0/566448568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:18.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:18 smithi006 bash[33918]: cluster 2023-08-28T00:23:16.490523+0000 mgr.y (mgr.14901) 4341 : cluster [DBG] pgmap v4348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:18.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:18 smithi006 bash[33918]: audit 2023-08-28T00:23:18.193145+0000 mon.c (mon.2) 1665 : audit [DBG] from='client.? 172.21.15.6:0/566448568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:19 smithi130 bash[51302]: debug 2023-08-28T00:23:19.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:19.453520+0000) 2023-08-28T00:23:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:20 smithi130 bash[27579]: cluster 2023-08-28T00:23:18.491221+0000 mgr.y (mgr.14901) 4342 : cluster [DBG] pgmap v4349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:20.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:20 smithi130 bash[51302]: debug 2023-08-28T00:23:20.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:20.453790+0000) 2023-08-28T00:23:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:20 smithi006 bash[27292]: cluster 2023-08-28T00:23:18.491221+0000 mgr.y (mgr.14901) 4342 : cluster [DBG] pgmap v4349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:20 smithi006 bash[33918]: cluster 2023-08-28T00:23:18.491221+0000 mgr.y (mgr.14901) 4342 : cluster [DBG] pgmap v4349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:23:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:23:20] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:23:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:23:20.720Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:23:20.722Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:21 smithi130 bash[27579]: audit 2023-08-28T00:23:20.620135+0000 mon.a (mon.0) 3196 : audit [DBG] from='client.? 172.21.15.6:0/1747426234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:21.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:21 smithi130 bash[51302]: debug 2023-08-28T00:23:21.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:21.454089+0000) 2023-08-28T00:23:21.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:21 smithi006 bash[27292]: audit 2023-08-28T00:23:20.620135+0000 mon.a (mon.0) 3196 : audit [DBG] from='client.? 172.21.15.6:0/1747426234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:21.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:21 smithi006 bash[33918]: audit 2023-08-28T00:23:20.620135+0000 mon.a (mon.0) 3196 : audit [DBG] from='client.? 172.21.15.6:0/1747426234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:22 smithi130 bash[27579]: cluster 2023-08-28T00:23:20.492269+0000 mgr.y (mgr.14901) 4343 : cluster [DBG] pgmap v4350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:22.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:22 smithi130 bash[51302]: debug 2023-08-28T00:23:22.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:22.454399+0000) 2023-08-28T00:23:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:22 smithi006 bash[27292]: cluster 2023-08-28T00:23:20.492269+0000 mgr.y (mgr.14901) 4343 : cluster [DBG] pgmap v4350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:22 smithi006 bash[33918]: cluster 2023-08-28T00:23:20.492269+0000 mgr.y (mgr.14901) 4343 : cluster [DBG] pgmap v4350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:23:22.300Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:23:22.301Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:23.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:23 smithi130 bash[27579]: audit 2023-08-28T00:23:23.032667+0000 mon.c (mon.2) 1666 : audit [DBG] from='client.? 172.21.15.6:0/2854670976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:23.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:23 smithi130 bash[51302]: debug 2023-08-28T00:23:23.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:23.454712+0000) 2023-08-28T00:23:23.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:23 smithi006 bash[33918]: audit 2023-08-28T00:23:23.032667+0000 mon.c (mon.2) 1666 : audit [DBG] from='client.? 172.21.15.6:0/2854670976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:23.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:23 smithi006 bash[27292]: audit 2023-08-28T00:23:23.032667+0000 mon.c (mon.2) 1666 : audit [DBG] from='client.? 172.21.15.6:0/2854670976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:24.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:24 smithi006 bash[27292]: cluster 2023-08-28T00:23:22.493008+0000 mgr.y (mgr.14901) 4344 : cluster [DBG] pgmap v4351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:24.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:24 smithi006 bash[33918]: cluster 2023-08-28T00:23:22.493008+0000 mgr.y (mgr.14901) 4344 : cluster [DBG] pgmap v4351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:23:24.184Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:23:24.186Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:24 smithi130 bash[27579]: cluster 2023-08-28T00:23:22.493008+0000 mgr.y (mgr.14901) 4344 : cluster [DBG] pgmap v4351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:24.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:24 smithi130 bash[51302]: debug 2023-08-28T00:23:24.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:24.455002+0000) 2023-08-28T00:23:25.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:25 smithi130 bash[51302]: debug 2023-08-28T00:23:25.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:25.455296+0000) 2023-08-28T00:23:26.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:26 smithi130 bash[51302]: debug 2023-08-28T00:23:26.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:26.455624+0000) 2023-08-28T00:23:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:26 smithi130 bash[27579]: cluster 2023-08-28T00:23:24.494161+0000 mgr.y (mgr.14901) 4345 : cluster [DBG] pgmap v4352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:26.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:26 smithi130 bash[27579]: audit 2023-08-28T00:23:25.440832+0000 mon.a (mon.0) 3197 : audit [DBG] from='client.? 172.21.15.6:0/3343544789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:26 smithi006 bash[33918]: cluster 2023-08-28T00:23:24.494161+0000 mgr.y (mgr.14901) 4345 : cluster [DBG] pgmap v4352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:26 smithi006 bash[33918]: audit 2023-08-28T00:23:25.440832+0000 mon.a (mon.0) 3197 : audit [DBG] from='client.? 172.21.15.6:0/3343544789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:26 smithi006 bash[27292]: cluster 2023-08-28T00:23:24.494161+0000 mgr.y (mgr.14901) 4345 : cluster [DBG] pgmap v4352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:26 smithi006 bash[27292]: audit 2023-08-28T00:23:25.440832+0000 mon.a (mon.0) 3197 : audit [DBG] from='client.? 172.21.15.6:0/3343544789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:27.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:27 smithi130 bash[51302]: debug 2023-08-28T00:23:27.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:27.455973+0000) 2023-08-28T00:23:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:28 smithi130 bash[27579]: cluster 2023-08-28T00:23:26.494930+0000 mgr.y (mgr.14901) 4346 : cluster [DBG] pgmap v4353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:28 smithi130 bash[27579]: audit 2023-08-28T00:23:27.850364+0000 mon.a (mon.0) 3198 : audit [DBG] from='client.? 172.21.15.6:0/2778401982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:28.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:28 smithi130 bash[51302]: debug 2023-08-28T00:23:28.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:28.456291+0000) 2023-08-28T00:23:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:28 smithi006 bash[33918]: cluster 2023-08-28T00:23:26.494930+0000 mgr.y (mgr.14901) 4346 : cluster [DBG] pgmap v4353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:28 smithi006 bash[33918]: audit 2023-08-28T00:23:27.850364+0000 mon.a (mon.0) 3198 : audit [DBG] from='client.? 172.21.15.6:0/2778401982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:28.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:28 smithi006 bash[27292]: cluster 2023-08-28T00:23:26.494930+0000 mgr.y (mgr.14901) 4346 : cluster [DBG] pgmap v4353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:28.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:28 smithi006 bash[27292]: audit 2023-08-28T00:23:27.850364+0000 mon.a (mon.0) 3198 : audit [DBG] from='client.? 172.21.15.6:0/2778401982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:29.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:29 smithi130 bash[51302]: debug 2023-08-28T00:23:29.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:29.456647+0000) 2023-08-28T00:23:30.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:30 smithi130 bash[51302]: debug 2023-08-28T00:23:30.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:30.457021+0000) 2023-08-28T00:23:30.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:30 smithi130 bash[27579]: cluster 2023-08-28T00:23:28.495647+0000 mgr.y (mgr.14901) 4347 : cluster [DBG] pgmap v4354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:30.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:30 smithi130 bash[27579]: audit 2023-08-28T00:23:30.260995+0000 mon.a (mon.0) 3199 : audit [DBG] from='client.? 172.21.15.6:0/2977420996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:30 smithi006 bash[33918]: cluster 2023-08-28T00:23:28.495647+0000 mgr.y (mgr.14901) 4347 : cluster [DBG] pgmap v4354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:30 smithi006 bash[33918]: audit 2023-08-28T00:23:30.260995+0000 mon.a (mon.0) 3199 : audit [DBG] from='client.? 172.21.15.6:0/2977420996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:30 smithi006 bash[27292]: cluster 2023-08-28T00:23:28.495647+0000 mgr.y (mgr.14901) 4347 : cluster [DBG] pgmap v4354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:30 smithi006 bash[27292]: audit 2023-08-28T00:23:30.260995+0000 mon.a (mon.0) 3199 : audit [DBG] from='client.? 172.21.15.6:0/2977420996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:23:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:23:30] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:23:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:23:30.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:23:30.724Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:31.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:31 smithi130 bash[51302]: debug 2023-08-28T00:23:31.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:31.457321+0000) 2023-08-28T00:23:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:32 smithi130 bash[27579]: cluster 2023-08-28T00:23:30.496873+0000 mgr.y (mgr.14901) 4348 : cluster [DBG] pgmap v4355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:32.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:32 smithi130 bash[51302]: debug 2023-08-28T00:23:32.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:32.457658+0000) 2023-08-28T00:23:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:32 smithi006 bash[27292]: cluster 2023-08-28T00:23:30.496873+0000 mgr.y (mgr.14901) 4348 : cluster [DBG] pgmap v4355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:32.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:32 smithi006 bash[33918]: cluster 2023-08-28T00:23:30.496873+0000 mgr.y (mgr.14901) 4348 : cluster [DBG] pgmap v4355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:23:32.301Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:23:32.302Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:33 smithi130 bash[27579]: audit 2023-08-28T00:23:32.674857+0000 mon.c (mon.2) 1667 : audit [DBG] from='client.? 172.21.15.6:0/4164035393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:33.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:33 smithi130 bash[51302]: debug 2023-08-28T00:23:33.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:33.457911+0000) 2023-08-28T00:23:33.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:33 smithi006 bash[33918]: audit 2023-08-28T00:23:32.674857+0000 mon.c (mon.2) 1667 : audit [DBG] from='client.? 172.21.15.6:0/4164035393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:33.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:33 smithi006 bash[27292]: audit 2023-08-28T00:23:32.674857+0000 mon.c (mon.2) 1667 : audit [DBG] from='client.? 172.21.15.6:0/4164035393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:34.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:34 smithi006 bash[33918]: cluster 2023-08-28T00:23:32.497466+0000 mgr.y (mgr.14901) 4349 : cluster [DBG] pgmap v4356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:34.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:34 smithi006 bash[27292]: cluster 2023-08-28T00:23:32.497466+0000 mgr.y (mgr.14901) 4349 : cluster [DBG] pgmap v4356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:23:34.184Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:23:34.187Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:34 smithi130 bash[27579]: cluster 2023-08-28T00:23:32.497466+0000 mgr.y (mgr.14901) 4349 : cluster [DBG] pgmap v4356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:34.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:34 smithi130 bash[51302]: debug 2023-08-28T00:23:34.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:34.458133+0000) 2023-08-28T00:23:35.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:35 smithi130 bash[27579]: audit 2023-08-28T00:23:35.087390+0000 mon.c (mon.2) 1668 : audit [DBG] from='client.? 172.21.15.6:0/188883290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:35.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:35 smithi130 bash[51302]: debug 2023-08-28T00:23:35.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:35.458411+0000) 2023-08-28T00:23:35.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:35 smithi006 bash[27292]: audit 2023-08-28T00:23:35.087390+0000 mon.c (mon.2) 1668 : audit [DBG] from='client.? 172.21.15.6:0/188883290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:35.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:35 smithi006 bash[33918]: audit 2023-08-28T00:23:35.087390+0000 mon.c (mon.2) 1668 : audit [DBG] from='client.? 172.21.15.6:0/188883290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:36.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:36 smithi130 bash[27579]: cluster 2023-08-28T00:23:34.498522+0000 mgr.y (mgr.14901) 4350 : cluster [DBG] pgmap v4357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:36.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:36 smithi130 bash[51302]: debug 2023-08-28T00:23:36.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:36.458685+0000) 2023-08-28T00:23:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:36 smithi006 bash[27292]: cluster 2023-08-28T00:23:34.498522+0000 mgr.y (mgr.14901) 4350 : cluster [DBG] pgmap v4357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:36 smithi006 bash[33918]: cluster 2023-08-28T00:23:34.498522+0000 mgr.y (mgr.14901) 4350 : cluster [DBG] pgmap v4357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:37 smithi130 bash[51302]: debug 2023-08-28T00:23:37.457+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:37.459003+0000) 2023-08-28T00:23:38.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:38 smithi130 bash[51302]: debug 2023-08-28T00:23:38.457+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:38.459307+0000) 2023-08-28T00:23:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:38 smithi130 bash[27579]: cluster 2023-08-28T00:23:36.499168+0000 mgr.y (mgr.14901) 4351 : cluster [DBG] pgmap v4358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:38.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:38 smithi130 bash[27579]: audit 2023-08-28T00:23:37.507602+0000 mon.a (mon.0) 3200 : audit [DBG] from='client.? 172.21.15.6:0/14589361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:38.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:38 smithi006 bash[33918]: cluster 2023-08-28T00:23:36.499168+0000 mgr.y (mgr.14901) 4351 : cluster [DBG] pgmap v4358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:38.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:38 smithi006 bash[33918]: audit 2023-08-28T00:23:37.507602+0000 mon.a (mon.0) 3200 : audit [DBG] from='client.? 172.21.15.6:0/14589361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:38.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:38 smithi006 bash[27292]: cluster 2023-08-28T00:23:36.499168+0000 mgr.y (mgr.14901) 4351 : cluster [DBG] pgmap v4358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:38.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:38 smithi006 bash[27292]: audit 2023-08-28T00:23:37.507602+0000 mon.a (mon.0) 3200 : audit [DBG] from='client.? 172.21.15.6:0/14589361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:39 smithi130 bash[51302]: debug 2023-08-28T00:23:39.457+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:39.459648+0000) 2023-08-28T00:23:40.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:40 smithi130 bash[51302]: debug 2023-08-28T00:23:40.457+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:40.459920+0000) 2023-08-28T00:23:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:40 smithi130 bash[27579]: cluster 2023-08-28T00:23:38.499819+0000 mgr.y (mgr.14901) 4352 : cluster [DBG] pgmap v4359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:40.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:40 smithi130 bash[27579]: audit 2023-08-28T00:23:39.926125+0000 mon.a (mon.0) 3201 : audit [DBG] from='client.? 172.21.15.6:0/4257480662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:40 smithi006 bash[27292]: cluster 2023-08-28T00:23:38.499819+0000 mgr.y (mgr.14901) 4352 : cluster [DBG] pgmap v4359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:40 smithi006 bash[27292]: audit 2023-08-28T00:23:39.926125+0000 mon.a (mon.0) 3201 : audit [DBG] from='client.? 172.21.15.6:0/4257480662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:40 smithi006 bash[33918]: cluster 2023-08-28T00:23:38.499819+0000 mgr.y (mgr.14901) 4352 : cluster [DBG] pgmap v4359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:40 smithi006 bash[33918]: audit 2023-08-28T00:23:39.926125+0000 mon.a (mon.0) 3201 : audit [DBG] from='client.? 172.21.15.6:0/4257480662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:23:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:23:40] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:23:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:23:40.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:23:40.724Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:41.870 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:41 smithi130 bash[51302]: debug 2023-08-28T00:23:41.457+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:41.460226+0000) 2023-08-28T00:23:42.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:42 smithi130 bash[51302]: debug 2023-08-28T00:23:42.457+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:42.460545+0000) 2023-08-28T00:23:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:42 smithi130 bash[27579]: cluster 2023-08-28T00:23:40.500910+0000 mgr.y (mgr.14901) 4353 : cluster [DBG] pgmap v4360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:42 smithi006 bash[27292]: cluster 2023-08-28T00:23:40.500910+0000 mgr.y (mgr.14901) 4353 : cluster [DBG] pgmap v4360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:42.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:42 smithi006 bash[33918]: cluster 2023-08-28T00:23:40.500910+0000 mgr.y (mgr.14901) 4353 : cluster [DBG] pgmap v4360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:23:42.301Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:23:42.303Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:43.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:43 smithi130 bash[51302]: debug 2023-08-28T00:23:43.457+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:43.460827+0000) 2023-08-28T00:23:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:43 smithi130 bash[27579]: audit 2023-08-28T00:23:42.368258+0000 mon.c (mon.2) 1669 : audit [DBG] from='client.? 172.21.15.6:0/3664433554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:43.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:43 smithi006 bash[27292]: audit 2023-08-28T00:23:42.368258+0000 mon.c (mon.2) 1669 : audit [DBG] from='client.? 172.21.15.6:0/3664433554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:43.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:43 smithi006 bash[33918]: audit 2023-08-28T00:23:42.368258+0000 mon.c (mon.2) 1669 : audit [DBG] from='client.? 172.21.15.6:0/3664433554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:44.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:44 smithi006 bash[27292]: cluster 2023-08-28T00:23:42.501514+0000 mgr.y (mgr.14901) 4354 : cluster [DBG] pgmap v4361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:44.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:44 smithi006 bash[33918]: cluster 2023-08-28T00:23:42.501514+0000 mgr.y (mgr.14901) 4354 : cluster [DBG] pgmap v4361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:23:44.185Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:23:44.186Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:44 smithi130 bash[27579]: cluster 2023-08-28T00:23:42.501514+0000 mgr.y (mgr.14901) 4354 : cluster [DBG] pgmap v4361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:44.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:44 smithi130 bash[51302]: debug 2023-08-28T00:23:44.457+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:44.461146+0000) 2023-08-28T00:23:45.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:45 smithi130 bash[27579]: audit 2023-08-28T00:23:44.782219+0000 mon.c (mon.2) 1670 : audit [DBG] from='client.? 172.21.15.6:0/834131867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:45.619 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:45 smithi130 bash[51302]: debug 2023-08-28T00:23:45.461+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:45.461420+0000) 2023-08-28T00:23:45.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:45 smithi006 bash[33918]: audit 2023-08-28T00:23:44.782219+0000 mon.c (mon.2) 1670 : audit [DBG] from='client.? 172.21.15.6:0/834131867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:45.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:45 smithi006 bash[27292]: audit 2023-08-28T00:23:44.782219+0000 mon.c (mon.2) 1670 : audit [DBG] from='client.? 172.21.15.6:0/834131867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:46 smithi130 bash[27579]: cluster 2023-08-28T00:23:44.502134+0000 mgr.y (mgr.14901) 4355 : cluster [DBG] pgmap v4362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:46.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:46 smithi130 bash[51302]: debug 2023-08-28T00:23:46.457+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:46.461766+0000) 2023-08-28T00:23:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:46 smithi006 bash[33918]: cluster 2023-08-28T00:23:44.502134+0000 mgr.y (mgr.14901) 4355 : cluster [DBG] pgmap v4362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:46 smithi006 bash[27292]: cluster 2023-08-28T00:23:44.502134+0000 mgr.y (mgr.14901) 4355 : cluster [DBG] pgmap v4362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:47.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:47 smithi130 bash[27579]: audit 2023-08-28T00:23:47.193378+0000 mon.c (mon.2) 1671 : audit [DBG] from='client.? 172.21.15.6:0/3729478480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:47.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:47 smithi130 bash[51302]: debug 2023-08-28T00:23:47.461+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:47.462054+0000) 2023-08-28T00:23:47.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:47 smithi006 bash[27292]: audit 2023-08-28T00:23:47.193378+0000 mon.c (mon.2) 1671 : audit [DBG] from='client.? 172.21.15.6:0/3729478480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:47.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:47 smithi006 bash[33918]: audit 2023-08-28T00:23:47.193378+0000 mon.c (mon.2) 1671 : audit [DBG] from='client.? 172.21.15.6:0/3729478480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:48 smithi006 bash[27292]: cluster 2023-08-28T00:23:46.502736+0000 mgr.y (mgr.14901) 4356 : cluster [DBG] pgmap v4363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:48 smithi006 bash[33918]: cluster 2023-08-28T00:23:46.502736+0000 mgr.y (mgr.14901) 4356 : cluster [DBG] pgmap v4363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:48 smithi130 bash[27579]: cluster 2023-08-28T00:23:46.502736+0000 mgr.y (mgr.14901) 4356 : cluster [DBG] pgmap v4363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:48 smithi130 bash[51302]: debug 2023-08-28T00:23:48.457+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:48.462311+0000) 2023-08-28T00:23:49.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:49 smithi130 bash[51302]: debug 2023-08-28T00:23:49.457+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:49.462596+0000) 2023-08-28T00:23:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:50 smithi006 bash[33918]: cluster 2023-08-28T00:23:48.503400+0000 mgr.y (mgr.14901) 4357 : cluster [DBG] pgmap v4364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:50 smithi006 bash[33918]: audit 2023-08-28T00:23:49.611679+0000 mon.c (mon.2) 1672 : audit [DBG] from='client.? 172.21.15.6:0/3056989153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:50 smithi006 bash[27292]: cluster 2023-08-28T00:23:48.503400+0000 mgr.y (mgr.14901) 4357 : cluster [DBG] pgmap v4364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:50 smithi006 bash[27292]: audit 2023-08-28T00:23:49.611679+0000 mon.c (mon.2) 1672 : audit [DBG] from='client.? 172.21.15.6:0/3056989153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:50.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:50 smithi130 bash[27579]: cluster 2023-08-28T00:23:48.503400+0000 mgr.y (mgr.14901) 4357 : cluster [DBG] pgmap v4364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:50.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:50 smithi130 bash[27579]: audit 2023-08-28T00:23:49.611679+0000 mon.c (mon.2) 1672 : audit [DBG] from='client.? 172.21.15.6:0/3056989153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:50.870 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:50 smithi130 bash[51302]: debug 2023-08-28T00:23:50.461+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:50.462890+0000) 2023-08-28T00:23:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:23:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:23:50] "GET /metrics HTTP/1.1" 200 193132 "" "Prometheus/2.33.4" 2023-08-28T00:23:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:23:50.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:23:50.725Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:51.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:51 smithi130 bash[51302]: debug 2023-08-28T00:23:51.461+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:51.463167+0000) 2023-08-28T00:23:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:23:52.302Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:23:52.304Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:52 smithi006 bash[27292]: cluster 2023-08-28T00:23:50.504539+0000 mgr.y (mgr.14901) 4358 : cluster [DBG] pgmap v4365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:52 smithi006 bash[27292]: audit 2023-08-28T00:23:52.025610+0000 mon.a (mon.0) 3202 : audit [DBG] from='client.? 172.21.15.6:0/3460908371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:52 smithi006 bash[33918]: cluster 2023-08-28T00:23:50.504539+0000 mgr.y (mgr.14901) 4358 : cluster [DBG] pgmap v4365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:52 smithi006 bash[33918]: audit 2023-08-28T00:23:52.025610+0000 mon.a (mon.0) 3202 : audit [DBG] from='client.? 172.21.15.6:0/3460908371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:52 smithi130 bash[27579]: cluster 2023-08-28T00:23:50.504539+0000 mgr.y (mgr.14901) 4358 : cluster [DBG] pgmap v4365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:52 smithi130 bash[27579]: audit 2023-08-28T00:23:52.025610+0000 mon.a (mon.0) 3202 : audit [DBG] from='client.? 172.21.15.6:0/3460908371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:52.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:52 smithi130 bash[51302]: debug 2023-08-28T00:23:52.461+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:52.463472+0000) 2023-08-28T00:23:53.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:53 smithi130 bash[51302]: debug 2023-08-28T00:23:53.461+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:53.463742+0000) 2023-08-28T00:23:54.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:54 smithi006 bash[33918]: cluster 2023-08-28T00:23:52.505241+0000 mgr.y (mgr.14901) 4359 : cluster [DBG] pgmap v4366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:54.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:54 smithi006 bash[27292]: cluster 2023-08-28T00:23:52.505241+0000 mgr.y (mgr.14901) 4359 : cluster [DBG] pgmap v4366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:23:54.185Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:23:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:23:54.187Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:23:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:54 smithi130 bash[27579]: cluster 2023-08-28T00:23:52.505241+0000 mgr.y (mgr.14901) 4359 : cluster [DBG] pgmap v4366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:54.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:54 smithi130 bash[51302]: debug 2023-08-28T00:23:54.461+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:54.464020+0000) 2023-08-28T00:23:55.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:55 smithi006 bash[27292]: audit 2023-08-28T00:23:54.449665+0000 mon.a (mon.0) 3203 : audit [DBG] from='client.? 172.21.15.6:0/3613089909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:55.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:55 smithi006 bash[33918]: audit 2023-08-28T00:23:54.449665+0000 mon.a (mon.0) 3203 : audit [DBG] from='client.? 172.21.15.6:0/3613089909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:55.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:55 smithi130 bash[27579]: audit 2023-08-28T00:23:54.449665+0000 mon.a (mon.0) 3203 : audit [DBG] from='client.? 172.21.15.6:0/3613089909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:55.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:55 smithi130 bash[51302]: debug 2023-08-28T00:23:55.461+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:55.464277+0000) 2023-08-28T00:23:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:56 smithi006 bash[27292]: cluster 2023-08-28T00:23:54.506409+0000 mgr.y (mgr.14901) 4360 : cluster [DBG] pgmap v4367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:56 smithi006 bash[33918]: cluster 2023-08-28T00:23:54.506409+0000 mgr.y (mgr.14901) 4360 : cluster [DBG] pgmap v4367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:56 smithi130 bash[27579]: cluster 2023-08-28T00:23:54.506409+0000 mgr.y (mgr.14901) 4360 : cluster [DBG] pgmap v4367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:56.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:56 smithi130 bash[51302]: debug 2023-08-28T00:23:56.461+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:56.464561+0000) 2023-08-28T00:23:57.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:57 smithi006 bash[27292]: audit 2023-08-28T00:23:56.861222+0000 mon.a (mon.0) 3204 : audit [DBG] from='client.? 172.21.15.6:0/3301332349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:57.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:57 smithi006 bash[33918]: audit 2023-08-28T00:23:56.861222+0000 mon.a (mon.0) 3204 : audit [DBG] from='client.? 172.21.15.6:0/3301332349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:57.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:57 smithi130 bash[27579]: audit 2023-08-28T00:23:56.861222+0000 mon.a (mon.0) 3204 : audit [DBG] from='client.? 172.21.15.6:0/3301332349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:57.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:57 smithi130 bash[51302]: debug 2023-08-28T00:23:57.461+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:57.464878+0000) 2023-08-28T00:23:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:58 smithi006 bash[33918]: cluster 2023-08-28T00:23:56.507200+0000 mgr.y (mgr.14901) 4361 : cluster [DBG] pgmap v4368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:58 smithi006 bash[27292]: cluster 2023-08-28T00:23:56.507200+0000 mgr.y (mgr.14901) 4361 : cluster [DBG] pgmap v4368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:58 smithi130 bash[27579]: cluster 2023-08-28T00:23:56.507200+0000 mgr.y (mgr.14901) 4361 : cluster [DBG] pgmap v4368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:23:58.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:58 smithi130 bash[51302]: debug 2023-08-28T00:23:58.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:58.465150+0000) 2023-08-28T00:23:59.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:23:59 smithi006 bash[27292]: audit 2023-08-28T00:23:59.270612+0000 mon.a (mon.0) 3205 : audit [DBG] from='client.? 172.21.15.6:0/188143954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:59.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:23:59 smithi006 bash[33918]: audit 2023-08-28T00:23:59.270612+0000 mon.a (mon.0) 3205 : audit [DBG] from='client.? 172.21.15.6:0/188143954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:59.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:23:59 smithi130 bash[27579]: audit 2023-08-28T00:23:59.270612+0000 mon.a (mon.0) 3205 : audit [DBG] from='client.? 172.21.15.6:0/188143954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:59.871 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:23:59 smithi130 bash[51302]: debug 2023-08-28T00:23:59.460+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:59.465446+0000) 2023-08-28T00:24:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:00 smithi006 bash[27292]: cluster 2023-08-28T00:23:58.507930+0000 mgr.y (mgr.14901) 4362 : cluster [DBG] pgmap v4369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:00 smithi006 bash[33918]: cluster 2023-08-28T00:23:58.507930+0000 mgr.y (mgr.14901) 4362 : cluster [DBG] pgmap v4369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:00 smithi130 bash[27579]: cluster 2023-08-28T00:23:58.507930+0000 mgr.y (mgr.14901) 4362 : cluster [DBG] pgmap v4369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:00.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:00 smithi130 bash[51302]: debug 2023-08-28T00:24:00.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:00.465760+0000) 2023-08-28T00:24:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:24:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:24:00] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:24:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:24:00.725Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:24:00.726Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:01.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:01 smithi130 bash[51302]: debug 2023-08-28T00:24:01.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:01.466040+0000) 2023-08-28T00:24:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:02 smithi006 bash[33918]: cluster 2023-08-28T00:24:00.509023+0000 mgr.y (mgr.14901) 4363 : cluster [DBG] pgmap v4370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:02 smithi006 bash[33918]: audit 2023-08-28T00:24:01.685990+0000 mon.a (mon.0) 3206 : audit [DBG] from='client.? 172.21.15.6:0/1685602784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:02.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:02 smithi006 bash[27292]: cluster 2023-08-28T00:24:00.509023+0000 mgr.y (mgr.14901) 4363 : cluster [DBG] pgmap v4370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:02.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:02 smithi006 bash[27292]: audit 2023-08-28T00:24:01.685990+0000 mon.a (mon.0) 3206 : audit [DBG] from='client.? 172.21.15.6:0/1685602784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:24:02.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:24:02.305Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:02.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:02 smithi130 bash[51302]: debug 2023-08-28T00:24:02.460+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:02.466332+0000) 2023-08-28T00:24:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:02 smithi130 bash[27579]: cluster 2023-08-28T00:24:00.509023+0000 mgr.y (mgr.14901) 4363 : cluster [DBG] pgmap v4370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:02.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:02 smithi130 bash[27579]: audit 2023-08-28T00:24:01.685990+0000 mon.a (mon.0) 3206 : audit [DBG] from='client.? 172.21.15.6:0/1685602784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:03.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:03 smithi130 bash[51302]: debug 2023-08-28T00:24:03.460+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:03.466660+0000) 2023-08-28T00:24:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:24:04.186Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:24:04.188Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:04.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:04 smithi006 bash[33918]: cluster 2023-08-28T00:24:02.509637+0000 mgr.y (mgr.14901) 4364 : cluster [DBG] pgmap v4371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:04.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:04 smithi006 bash[33918]: audit 2023-08-28T00:24:04.101385+0000 mon.a (mon.0) 3207 : audit [DBG] from='client.? 172.21.15.6:0/2845416178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:04.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:04 smithi006 bash[27292]: cluster 2023-08-28T00:24:02.509637+0000 mgr.y (mgr.14901) 4364 : cluster [DBG] pgmap v4371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:04.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:04 smithi006 bash[27292]: audit 2023-08-28T00:24:04.101385+0000 mon.a (mon.0) 3207 : audit [DBG] from='client.? 172.21.15.6:0/2845416178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:04.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:04 smithi130 bash[27579]: cluster 2023-08-28T00:24:02.509637+0000 mgr.y (mgr.14901) 4364 : cluster [DBG] pgmap v4371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:04.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:04 smithi130 bash[27579]: audit 2023-08-28T00:24:04.101385+0000 mon.a (mon.0) 3207 : audit [DBG] from='client.? 172.21.15.6:0/2845416178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:04.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:04 smithi130 bash[51302]: debug 2023-08-28T00:24:04.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:04.466940+0000) 2023-08-28T00:24:05.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:05 smithi130 bash[51302]: debug 2023-08-28T00:24:05.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:05.467208+0000) 2023-08-28T00:24:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:06 smithi006 bash[33918]: cluster 2023-08-28T00:24:04.510754+0000 mgr.y (mgr.14901) 4365 : cluster [DBG] pgmap v4372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:06 smithi006 bash[27292]: cluster 2023-08-28T00:24:04.510754+0000 mgr.y (mgr.14901) 4365 : cluster [DBG] pgmap v4372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:06 smithi130 bash[27579]: cluster 2023-08-28T00:24:04.510754+0000 mgr.y (mgr.14901) 4365 : cluster [DBG] pgmap v4372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:06.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:06 smithi130 bash[51302]: debug 2023-08-28T00:24:06.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:06.467551+0000) 2023-08-28T00:24:07.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:07 smithi006 bash[27292]: audit 2023-08-28T00:24:06.520192+0000 mon.a (mon.0) 3208 : audit [DBG] from='client.? 172.21.15.6:0/3849248918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:07 smithi006 bash[33918]: audit 2023-08-28T00:24:06.520192+0000 mon.a (mon.0) 3208 : audit [DBG] from='client.? 172.21.15.6:0/3849248918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:07.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:07 smithi130 bash[27579]: audit 2023-08-28T00:24:06.520192+0000 mon.a (mon.0) 3208 : audit [DBG] from='client.? 172.21.15.6:0/3849248918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:07.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:07 smithi130 bash[51302]: debug 2023-08-28T00:24:07.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:07.467909+0000) 2023-08-28T00:24:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:08 smithi006 bash[27292]: cluster 2023-08-28T00:24:06.511362+0000 mgr.y (mgr.14901) 4366 : cluster [DBG] pgmap v4373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:08 smithi006 bash[33918]: cluster 2023-08-28T00:24:06.511362+0000 mgr.y (mgr.14901) 4366 : cluster [DBG] pgmap v4373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:08 smithi130 bash[27579]: cluster 2023-08-28T00:24:06.511362+0000 mgr.y (mgr.14901) 4366 : cluster [DBG] pgmap v4373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:08.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:08 smithi130 bash[51302]: debug 2023-08-28T00:24:08.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:08.468183+0000) 2023-08-28T00:24:09.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:09 smithi006 bash[33918]: cluster 2023-08-28T00:24:08.512008+0000 mgr.y (mgr.14901) 4367 : cluster [DBG] pgmap v4374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:09.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:09 smithi006 bash[33918]: audit 2023-08-28T00:24:08.936037+0000 mon.a (mon.0) 3209 : audit [DBG] from='client.? 172.21.15.6:0/2479843556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:09.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:09 smithi006 bash[27292]: cluster 2023-08-28T00:24:08.512008+0000 mgr.y (mgr.14901) 4367 : cluster [DBG] pgmap v4374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:09.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:09 smithi006 bash[27292]: audit 2023-08-28T00:24:08.936037+0000 mon.a (mon.0) 3209 : audit [DBG] from='client.? 172.21.15.6:0/2479843556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:09.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:09 smithi130 bash[51302]: debug 2023-08-28T00:24:09.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:09.468554+0000) 2023-08-28T00:24:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:09 smithi130 bash[27579]: cluster 2023-08-28T00:24:08.512008+0000 mgr.y (mgr.14901) 4367 : cluster [DBG] pgmap v4374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:09.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:09 smithi130 bash[27579]: audit 2023-08-28T00:24:08.936037+0000 mon.a (mon.0) 3209 : audit [DBG] from='client.? 172.21.15.6:0/2479843556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:10 smithi130 bash[51302]: debug 2023-08-28T00:24:10.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:10.468890+0000) 2023-08-28T00:24:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:24:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:24:10] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:24:11.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:24:10.725Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:11.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:24:10.727Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:11.751 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:11 smithi130 bash[51302]: debug 2023-08-28T00:24:11.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:11.469245+0000) 2023-08-28T00:24:12.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:11 smithi130 bash[27579]: cluster 2023-08-28T00:24:10.513147+0000 mgr.y (mgr.14901) 4368 : cluster [DBG] pgmap v4375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:12.256 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:11 smithi130 bash[27579]: audit 2023-08-28T00:24:11.353996+0000 mon.a (mon.0) 3210 : audit [DBG] from='client.? 172.21.15.6:0/762609559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:12.256 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:11 smithi006 bash[27292]: cluster 2023-08-28T00:24:10.513147+0000 mgr.y (mgr.14901) 4368 : cluster [DBG] pgmap v4375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:12.257 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:11 smithi006 bash[27292]: audit 2023-08-28T00:24:11.353996+0000 mon.a (mon.0) 3210 : audit [DBG] from='client.? 172.21.15.6:0/762609559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:12.257 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:11 smithi006 bash[33918]: cluster 2023-08-28T00:24:10.513147+0000 mgr.y (mgr.14901) 4368 : cluster [DBG] pgmap v4375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:12.257 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:11 smithi006 bash[33918]: audit 2023-08-28T00:24:11.353996+0000 mon.a (mon.0) 3210 : audit [DBG] from='client.? 172.21.15.6:0/762609559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:24:12.304Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:12.755 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:24:12.305Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:12 smithi130 bash[51302]: debug 2023-08-28T00:24:12.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:12.469512+0000) 2023-08-28T00:24:13.757 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:13 smithi130 bash[51302]: debug 2023-08-28T00:24:13.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:13.469755+0000) 2023-08-28T00:24:14.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:13 smithi130 bash[27579]: cluster 2023-08-28T00:24:12.513798+0000 mgr.y (mgr.14901) 4369 : cluster [DBG] pgmap v4376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:14.190 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:13 smithi006 bash[27292]: cluster 2023-08-28T00:24:12.513798+0000 mgr.y (mgr.14901) 4369 : cluster [DBG] pgmap v4376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:14.190 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:13 smithi006 bash[33918]: cluster 2023-08-28T00:24:12.513798+0000 mgr.y (mgr.14901) 4369 : cluster [DBG] pgmap v4376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:24:14.187Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:24:14.189Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:14.764 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:14 smithi130 bash[51302]: debug 2023-08-28T00:24:14.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:14.470030+0000) 2023-08-28T00:24:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:14 smithi130 bash[27579]: audit 2023-08-28T00:24:13.775376+0000 mon.c (mon.2) 1673 : audit [DBG] from='client.? 172.21.15.6:0/2201111337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:14 smithi006 bash[27292]: audit 2023-08-28T00:24:13.775376+0000 mon.c (mon.2) 1673 : audit [DBG] from='client.? 172.21.15.6:0/2201111337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:14 smithi006 bash[33918]: audit 2023-08-28T00:24:13.775376+0000 mon.c (mon.2) 1673 : audit [DBG] from='client.? 172.21.15.6:0/2201111337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:15.767 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:15 smithi130 bash[51302]: debug 2023-08-28T00:24:15.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:15.470307+0000) 2023-08-28T00:24:16.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:15 smithi130 bash[27579]: cluster 2023-08-28T00:24:14.515055+0000 mgr.y (mgr.14901) 4370 : cluster [DBG] pgmap v4377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:16.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:15 smithi006 bash[33918]: cluster 2023-08-28T00:24:14.515055+0000 mgr.y (mgr.14901) 4370 : cluster [DBG] pgmap v4377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:16.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:15 smithi006 bash[27292]: cluster 2023-08-28T00:24:14.515055+0000 mgr.y (mgr.14901) 4370 : cluster [DBG] pgmap v4377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:16.770 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:16 smithi130 bash[51302]: debug 2023-08-28T00:24:16.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:16.470538+0000) 2023-08-28T00:24:17.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:16 smithi130 bash[27579]: audit 2023-08-28T00:24:15.999862+0000 mon.b (mon.1) 541 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:24:17.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:16 smithi130 bash[27579]: audit 2023-08-28T00:24:16.189591+0000 mon.c (mon.2) 1674 : audit [DBG] from='client.? 172.21.15.6:0/1460811888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:17.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:16 smithi130 bash[27579]: audit 2023-08-28T00:24:16.271727+0000 mon.a (mon.0) 3211 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:24:17.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:16 smithi130 bash[27579]: audit 2023-08-28T00:24:16.280634+0000 mon.a (mon.0) 3212 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:24:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:16 smithi006 bash[33918]: audit 2023-08-28T00:24:15.999862+0000 mon.b (mon.1) 541 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:24:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:16 smithi006 bash[33918]: audit 2023-08-28T00:24:16.189591+0000 mon.c (mon.2) 1674 : audit [DBG] from='client.? 172.21.15.6:0/1460811888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:16 smithi006 bash[33918]: audit 2023-08-28T00:24:16.271727+0000 mon.a (mon.0) 3211 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:24:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:16 smithi006 bash[33918]: audit 2023-08-28T00:24:16.280634+0000 mon.a (mon.0) 3212 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:24:17.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:16 smithi006 bash[27292]: audit 2023-08-28T00:24:15.999862+0000 mon.b (mon.1) 541 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:24:17.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:16 smithi006 bash[27292]: audit 2023-08-28T00:24:16.189591+0000 mon.c (mon.2) 1674 : audit [DBG] from='client.? 172.21.15.6:0/1460811888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:17.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:16 smithi006 bash[27292]: audit 2023-08-28T00:24:16.271727+0000 mon.a (mon.0) 3211 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:24:17.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:16 smithi006 bash[27292]: audit 2023-08-28T00:24:16.280634+0000 mon.a (mon.0) 3212 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:24:17.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:17 smithi130 bash[51302]: debug 2023-08-28T00:24:17.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:17.470822+0000) 2023-08-28T00:24:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:18 smithi130 bash[27579]: cluster 2023-08-28T00:24:16.515732+0000 mgr.y (mgr.14901) 4371 : cluster [DBG] pgmap v4378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:18.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:18 smithi130 bash[51302]: debug 2023-08-28T00:24:18.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:18.471086+0000) 2023-08-28T00:24:18.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:18 smithi006 bash[27292]: cluster 2023-08-28T00:24:16.515732+0000 mgr.y (mgr.14901) 4371 : cluster [DBG] pgmap v4378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:18.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:18 smithi006 bash[33918]: cluster 2023-08-28T00:24:16.515732+0000 mgr.y (mgr.14901) 4371 : cluster [DBG] pgmap v4378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:19 smithi130 bash[27579]: audit 2023-08-28T00:24:18.618928+0000 mon.a (mon.0) 3213 : audit [DBG] from='client.? 172.21.15.6:0/3127687443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:19.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:19 smithi130 bash[51302]: debug 2023-08-28T00:24:19.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:19.471375+0000) 2023-08-28T00:24:19.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:19 smithi006 bash[27292]: audit 2023-08-28T00:24:18.618928+0000 mon.a (mon.0) 3213 : audit [DBG] from='client.? 172.21.15.6:0/3127687443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:19.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:19 smithi006 bash[33918]: audit 2023-08-28T00:24:18.618928+0000 mon.a (mon.0) 3213 : audit [DBG] from='client.? 172.21.15.6:0/3127687443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:20 smithi130 bash[27579]: cluster 2023-08-28T00:24:18.516319+0000 mgr.y (mgr.14901) 4372 : cluster [DBG] pgmap v4379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:20.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:20 smithi130 bash[51302]: debug 2023-08-28T00:24:20.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:20.471655+0000) 2023-08-28T00:24:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:20 smithi006 bash[33918]: cluster 2023-08-28T00:24:18.516319+0000 mgr.y (mgr.14901) 4372 : cluster [DBG] pgmap v4379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:20.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:20 smithi006 bash[27292]: cluster 2023-08-28T00:24:18.516319+0000 mgr.y (mgr.14901) 4372 : cluster [DBG] pgmap v4379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:24:20.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:24:20.728Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:24:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:24:20] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:24:21.751 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:21 smithi130 bash[51302]: debug 2023-08-28T00:24:21.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:21.471866+0000) 2023-08-28T00:24:22.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:21 smithi130 bash[27579]: cluster 2023-08-28T00:24:20.517447+0000 mgr.y (mgr.14901) 4373 : cluster [DBG] pgmap v4380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:22.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:21 smithi130 bash[27579]: audit 2023-08-28T00:24:21.034982+0000 mon.a (mon.0) 3214 : audit [DBG] from='client.? 172.21.15.6:0/1676838292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:22.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:21 smithi006 bash[33918]: cluster 2023-08-28T00:24:20.517447+0000 mgr.y (mgr.14901) 4373 : cluster [DBG] pgmap v4380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:22.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:21 smithi006 bash[33918]: audit 2023-08-28T00:24:21.034982+0000 mon.a (mon.0) 3214 : audit [DBG] from='client.? 172.21.15.6:0/1676838292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:22.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:21 smithi006 bash[27292]: cluster 2023-08-28T00:24:20.517447+0000 mgr.y (mgr.14901) 4373 : cluster [DBG] pgmap v4380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:22.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:21 smithi006 bash[27292]: audit 2023-08-28T00:24:21.034982+0000 mon.a (mon.0) 3214 : audit [DBG] from='client.? 172.21.15.6:0/1676838292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:24:22.305Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:24:22.307Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:22 smithi130 bash[51302]: debug 2023-08-28T00:24:22.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:22.472183+0000) 2023-08-28T00:24:23.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:23 smithi006 bash[27292]: audit 2023-08-28T00:24:22.135592+0000 mon.a (mon.0) 3215 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:24:23.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:23 smithi006 bash[27292]: audit 2023-08-28T00:24:22.604190+0000 mon.a (mon.0) 3216 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:24:23.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:23 smithi006 bash[33918]: audit 2023-08-28T00:24:22.135592+0000 mon.a (mon.0) 3215 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:24:23.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:23 smithi006 bash[33918]: audit 2023-08-28T00:24:22.604190+0000 mon.a (mon.0) 3216 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:24:23.474 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:23 smithi130 bash[27579]: audit 2023-08-28T00:24:22.135592+0000 mon.a (mon.0) 3215 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:24:23.474 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:23 smithi130 bash[27579]: audit 2023-08-28T00:24:22.604190+0000 mon.a (mon.0) 3216 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:24:23.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:23 smithi130 bash[51302]: debug 2023-08-28T00:24:23.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:23.473261+0000) 2023-08-28T00:24:24.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:24 smithi006 bash[27292]: cluster 2023-08-28T00:24:22.518075+0000 mgr.y (mgr.14901) 4374 : cluster [DBG] pgmap v4381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:24.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:24 smithi006 bash[27292]: audit 2023-08-28T00:24:23.457662+0000 mon.a (mon.0) 3217 : audit [DBG] from='client.? 172.21.15.6:0/3367694739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:24:24.188Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:24:24.190Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:24.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:24 smithi006 bash[33918]: cluster 2023-08-28T00:24:22.518075+0000 mgr.y (mgr.14901) 4374 : cluster [DBG] pgmap v4381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:24.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:24 smithi006 bash[33918]: audit 2023-08-28T00:24:23.457662+0000 mon.a (mon.0) 3217 : audit [DBG] from='client.? 172.21.15.6:0/3367694739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:24.475 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:24 smithi130 bash[27579]: cluster 2023-08-28T00:24:22.518075+0000 mgr.y (mgr.14901) 4374 : cluster [DBG] pgmap v4381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:24.475 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:24 smithi130 bash[27579]: audit 2023-08-28T00:24:23.457662+0000 mon.a (mon.0) 3217 : audit [DBG] from='client.? 172.21.15.6:0/3367694739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:24 smithi130 bash[51302]: debug 2023-08-28T00:24:24.472+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:24.473563+0000) 2023-08-28T00:24:25.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:25 smithi130 bash[51302]: debug 2023-08-28T00:24:25.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:25.473857+0000) 2023-08-28T00:24:26.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:26 smithi006 bash[33918]: cluster 2023-08-28T00:24:24.519198+0000 mgr.y (mgr.14901) 4375 : cluster [DBG] pgmap v4382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:26.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:26 smithi006 bash[33918]: audit 2023-08-28T00:24:25.872906+0000 mon.a (mon.0) 3218 : audit [DBG] from='client.? 172.21.15.6:0/68177954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:26 smithi006 bash[27292]: cluster 2023-08-28T00:24:24.519198+0000 mgr.y (mgr.14901) 4375 : cluster [DBG] pgmap v4382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:26.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:26 smithi006 bash[27292]: audit 2023-08-28T00:24:25.872906+0000 mon.a (mon.0) 3218 : audit [DBG] from='client.? 172.21.15.6:0/68177954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:26.475 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:26 smithi130 bash[27579]: cluster 2023-08-28T00:24:24.519198+0000 mgr.y (mgr.14901) 4375 : cluster [DBG] pgmap v4382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:26.476 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:26 smithi130 bash[27579]: audit 2023-08-28T00:24:25.872906+0000 mon.a (mon.0) 3218 : audit [DBG] from='client.? 172.21.15.6:0/68177954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:26.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:26 smithi130 bash[51302]: debug 2023-08-28T00:24:26.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:26.474169+0000) 2023-08-28T00:24:27.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:27 smithi130 bash[51302]: debug 2023-08-28T00:24:27.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:27.474496+0000) 2023-08-28T00:24:28.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:28 smithi006 bash[33918]: cluster 2023-08-28T00:24:26.519843+0000 mgr.y (mgr.14901) 4376 : cluster [DBG] pgmap v4383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:28.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:28 smithi006 bash[27292]: cluster 2023-08-28T00:24:26.519843+0000 mgr.y (mgr.14901) 4376 : cluster [DBG] pgmap v4383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:28.476 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:28 smithi130 bash[27579]: cluster 2023-08-28T00:24:26.519843+0000 mgr.y (mgr.14901) 4376 : cluster [DBG] pgmap v4383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:28.740 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:28 smithi130 bash[51302]: debug 2023-08-28T00:24:28.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:28.474764+0000) 2023-08-28T00:24:29.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:29 smithi130 bash[27579]: audit 2023-08-28T00:24:28.292212+0000 mon.c (mon.2) 1675 : audit [DBG] from='client.? 172.21.15.6:0/1575460183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:29.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:29 smithi006 bash[33918]: audit 2023-08-28T00:24:28.292212+0000 mon.c (mon.2) 1675 : audit [DBG] from='client.? 172.21.15.6:0/1575460183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:29.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:29 smithi006 bash[27292]: audit 2023-08-28T00:24:28.292212+0000 mon.c (mon.2) 1675 : audit [DBG] from='client.? 172.21.15.6:0/1575460183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:29.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:29 smithi130 bash[51302]: debug 2023-08-28T00:24:29.472+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:29.475053+0000) 2023-08-28T00:24:30.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:30 smithi006 bash[33918]: cluster 2023-08-28T00:24:28.520525+0000 mgr.y (mgr.14901) 4377 : cluster [DBG] pgmap v4384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:30.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:30 smithi006 bash[27292]: cluster 2023-08-28T00:24:28.520525+0000 mgr.y (mgr.14901) 4377 : cluster [DBG] pgmap v4384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:30.476 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:30 smithi130 bash[27579]: cluster 2023-08-28T00:24:28.520525+0000 mgr.y (mgr.14901) 4377 : cluster [DBG] pgmap v4384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:30 smithi130 bash[51302]: debug 2023-08-28T00:24:30.472+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:30.475283+0000) 2023-08-28T00:24:31.175 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:24:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:24:30] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T00:24:31.175 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:24:30.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:31.175 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:24:30.730Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:31.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:31 smithi130 bash[27579]: audit 2023-08-28T00:24:30.723739+0000 mon.c (mon.2) 1676 : audit [DBG] from='client.? 172.21.15.6:0/2913816397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:31.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:31 smithi006 bash[33918]: audit 2023-08-28T00:24:30.723739+0000 mon.c (mon.2) 1676 : audit [DBG] from='client.? 172.21.15.6:0/2913816397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:31.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:31 smithi006 bash[27292]: audit 2023-08-28T00:24:30.723739+0000 mon.c (mon.2) 1676 : audit [DBG] from='client.? 172.21.15.6:0/2913816397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:31.811 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:31 smithi130 bash[51302]: debug 2023-08-28T00:24:31.472+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:31.475531+0000) 2023-08-28T00:24:32.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:32 smithi130 bash[27579]: cluster 2023-08-28T00:24:30.521474+0000 mgr.y (mgr.14901) 4378 : cluster [DBG] pgmap v4385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:32.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:32 smithi006 bash[33918]: cluster 2023-08-28T00:24:30.521474+0000 mgr.y (mgr.14901) 4378 : cluster [DBG] pgmap v4385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:24:32.305Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:32.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:24:32.308Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:32.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:32 smithi006 bash[27292]: cluster 2023-08-28T00:24:30.521474+0000 mgr.y (mgr.14901) 4378 : cluster [DBG] pgmap v4385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:32 smithi130 bash[51302]: debug 2023-08-28T00:24:32.472+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:32.475850+0000) 2023-08-28T00:24:33.803 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:33 smithi130 bash[51302]: debug 2023-08-28T00:24:33.472+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:33.476130+0000) 2023-08-28T00:24:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:33 smithi130 bash[27579]: cluster 2023-08-28T00:24:32.522111+0000 mgr.y (mgr.14901) 4379 : cluster [DBG] pgmap v4386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:33 smithi130 bash[27579]: audit 2023-08-28T00:24:32.800320+0000 mon.a (mon.0) 3219 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:24:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:33 smithi130 bash[27579]: audit 2023-08-28T00:24:33.064710+0000 mon.b (mon.1) 542 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:24:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:33 smithi130 bash[27579]: audit 2023-08-28T00:24:33.066711+0000 mon.b (mon.1) 543 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:24:34.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:33 smithi130 bash[27579]: audit 2023-08-28T00:24:33.072881+0000 mon.a (mon.0) 3220 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:24:34.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:33 smithi130 bash[27579]: audit 2023-08-28T00:24:33.140775+0000 mon.c (mon.2) 1677 : audit [DBG] from='client.? 172.21.15.6:0/2371824664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:34.191 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:33 smithi006 bash[27292]: cluster 2023-08-28T00:24:32.522111+0000 mgr.y (mgr.14901) 4379 : cluster [DBG] pgmap v4386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:34.191 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:33 smithi006 bash[27292]: audit 2023-08-28T00:24:32.800320+0000 mon.a (mon.0) 3219 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:24:34.191 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:33 smithi006 bash[27292]: audit 2023-08-28T00:24:33.064710+0000 mon.b (mon.1) 542 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:24:34.191 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:33 smithi006 bash[27292]: audit 2023-08-28T00:24:33.066711+0000 mon.b (mon.1) 543 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:24:34.192 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:33 smithi006 bash[27292]: audit 2023-08-28T00:24:33.072881+0000 mon.a (mon.0) 3220 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:24:34.192 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:33 smithi006 bash[27292]: audit 2023-08-28T00:24:33.140775+0000 mon.c (mon.2) 1677 : audit [DBG] from='client.? 172.21.15.6:0/2371824664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:34.192 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:33 smithi006 bash[33918]: cluster 2023-08-28T00:24:32.522111+0000 mgr.y (mgr.14901) 4379 : cluster [DBG] pgmap v4386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:34.192 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:33 smithi006 bash[33918]: audit 2023-08-28T00:24:32.800320+0000 mon.a (mon.0) 3219 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:24:34.192 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:33 smithi006 bash[33918]: audit 2023-08-28T00:24:33.064710+0000 mon.b (mon.1) 542 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:24:34.192 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:33 smithi006 bash[33918]: audit 2023-08-28T00:24:33.066711+0000 mon.b (mon.1) 543 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:24:34.192 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:33 smithi006 bash[33918]: audit 2023-08-28T00:24:33.072881+0000 mon.a (mon.0) 3220 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:24:34.192 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:33 smithi006 bash[33918]: audit 2023-08-28T00:24:33.140775+0000 mon.c (mon.2) 1677 : audit [DBG] from='client.? 172.21.15.6:0/2371824664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:24:34.188Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:24:34.190Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:34 smithi130 bash[51302]: debug 2023-08-28T00:24:34.472+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:34.476409+0000) 2023-08-28T00:24:35.862 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:35 smithi130 bash[51302]: debug 2023-08-28T00:24:35.472+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:35.476750+0000) 2023-08-28T00:24:36.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:35 smithi130 bash[27579]: cluster 2023-08-28T00:24:34.523197+0000 mgr.y (mgr.14901) 4380 : cluster [DBG] pgmap v4387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:36.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:35 smithi130 bash[27579]: audit 2023-08-28T00:24:35.563581+0000 mon.a (mon.0) 3221 : audit [DBG] from='client.? 172.21.15.6:0/2033858340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:36.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:35 smithi006 bash[27292]: cluster 2023-08-28T00:24:34.523197+0000 mgr.y (mgr.14901) 4380 : cluster [DBG] pgmap v4387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:36.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:35 smithi006 bash[27292]: audit 2023-08-28T00:24:35.563581+0000 mon.a (mon.0) 3221 : audit [DBG] from='client.? 172.21.15.6:0/2033858340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:36.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:35 smithi006 bash[33918]: cluster 2023-08-28T00:24:34.523197+0000 mgr.y (mgr.14901) 4380 : cluster [DBG] pgmap v4387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:36.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:35 smithi006 bash[33918]: audit 2023-08-28T00:24:35.563581+0000 mon.a (mon.0) 3221 : audit [DBG] from='client.? 172.21.15.6:0/2033858340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:36 smithi130 bash[51302]: debug 2023-08-28T00:24:36.476+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:36.477028+0000) 2023-08-28T00:24:37.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:37 smithi130 bash[51302]: debug 2023-08-28T00:24:37.476+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:37.477287+0000) 2023-08-28T00:24:38.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:37 smithi130 bash[27579]: cluster 2023-08-28T00:24:36.523838+0000 mgr.y (mgr.14901) 4381 : cluster [DBG] pgmap v4388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:38.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:37 smithi006 bash[27292]: cluster 2023-08-28T00:24:36.523838+0000 mgr.y (mgr.14901) 4381 : cluster [DBG] pgmap v4388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:38.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:37 smithi006 bash[33918]: cluster 2023-08-28T00:24:36.523838+0000 mgr.y (mgr.14901) 4381 : cluster [DBG] pgmap v4388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:38 smithi130 bash[51302]: debug 2023-08-28T00:24:38.472+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:38.477599+0000) 2023-08-28T00:24:39.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:38 smithi006 bash[33918]: audit 2023-08-28T00:24:37.987773+0000 mon.a (mon.0) 3222 : audit [DBG] from='client.? 172.21.15.6:0/3619275379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:39.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:38 smithi006 bash[27292]: audit 2023-08-28T00:24:37.987773+0000 mon.a (mon.0) 3222 : audit [DBG] from='client.? 172.21.15.6:0/3619275379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:38 smithi130 bash[27579]: audit 2023-08-28T00:24:37.987773+0000 mon.a (mon.0) 3222 : audit [DBG] from='client.? 172.21.15.6:0/3619275379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:39 smithi130 bash[51302]: debug 2023-08-28T00:24:39.472+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:39.477910+0000) 2023-08-28T00:24:40.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:39 smithi006 bash[27292]: cluster 2023-08-28T00:24:38.524506+0000 mgr.y (mgr.14901) 4382 : cluster [DBG] pgmap v4389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:40.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:39 smithi006 bash[33918]: cluster 2023-08-28T00:24:38.524506+0000 mgr.y (mgr.14901) 4382 : cluster [DBG] pgmap v4389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:39 smithi130 bash[27579]: cluster 2023-08-28T00:24:38.524506+0000 mgr.y (mgr.14901) 4382 : cluster [DBG] pgmap v4389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:40 smithi130 bash[51302]: debug 2023-08-28T00:24:40.476+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:40.478185+0000) 2023-08-28T00:24:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:40 smithi006 bash[27292]: audit 2023-08-28T00:24:40.417110+0000 mon.c (mon.2) 1678 : audit [DBG] from='client.? 172.21.15.6:0/1407242482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:24:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:24:40] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T00:24:41.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:40 smithi006 bash[33918]: audit 2023-08-28T00:24:40.417110+0000 mon.c (mon.2) 1678 : audit [DBG] from='client.? 172.21.15.6:0/1407242482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:24:40.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:24:40.730Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:41.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:40 smithi130 bash[27579]: audit 2023-08-28T00:24:40.417110+0000 mon.c (mon.2) 1678 : audit [DBG] from='client.? 172.21.15.6:0/1407242482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:41.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:41 smithi130 bash[51302]: debug 2023-08-28T00:24:41.476+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:41.478470+0000) 2023-08-28T00:24:42.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:41 smithi006 bash[33918]: cluster 2023-08-28T00:24:40.525649+0000 mgr.y (mgr.14901) 4383 : cluster [DBG] pgmap v4390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:42.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:41 smithi006 bash[27292]: cluster 2023-08-28T00:24:40.525649+0000 mgr.y (mgr.14901) 4383 : cluster [DBG] pgmap v4390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:41 smithi130 bash[27579]: cluster 2023-08-28T00:24:40.525649+0000 mgr.y (mgr.14901) 4383 : cluster [DBG] pgmap v4390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:24:42.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:24:42.308Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:42 smithi130 bash[51302]: debug 2023-08-28T00:24:42.472+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:42.478799+0000) 2023-08-28T00:24:43.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:42 smithi006 bash[27292]: audit 2023-08-28T00:24:42.826843+0000 mon.a (mon.0) 3223 : audit [DBG] from='client.? 172.21.15.6:0/2917236170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:43.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:42 smithi006 bash[33918]: audit 2023-08-28T00:24:42.826843+0000 mon.a (mon.0) 3223 : audit [DBG] from='client.? 172.21.15.6:0/2917236170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:43.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:42 smithi130 bash[27579]: audit 2023-08-28T00:24:42.826843+0000 mon.a (mon.0) 3223 : audit [DBG] from='client.? 172.21.15.6:0/2917236170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:43.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:43 smithi130 bash[51302]: debug 2023-08-28T00:24:43.476+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:43.479066+0000) 2023-08-28T00:24:44.192 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:43 smithi006 bash[27292]: cluster 2023-08-28T00:24:42.526249+0000 mgr.y (mgr.14901) 4384 : cluster [DBG] pgmap v4391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:44.192 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:43 smithi006 bash[33918]: cluster 2023-08-28T00:24:42.526249+0000 mgr.y (mgr.14901) 4384 : cluster [DBG] pgmap v4391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:43 smithi130 bash[27579]: cluster 2023-08-28T00:24:42.526249+0000 mgr.y (mgr.14901) 4384 : cluster [DBG] pgmap v4391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:24:44.189Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:24:44.191Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:44 smithi130 bash[51302]: debug 2023-08-28T00:24:44.476+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:44.479346+0000) 2023-08-28T00:24:45.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:45 smithi130 bash[51302]: debug 2023-08-28T00:24:45.476+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:45.479641+0000) 2023-08-28T00:24:46.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:45 smithi006 bash[33918]: cluster 2023-08-28T00:24:44.527376+0000 mgr.y (mgr.14901) 4385 : cluster [DBG] pgmap v4392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:46.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:45 smithi006 bash[33918]: audit 2023-08-28T00:24:45.243621+0000 mon.a (mon.0) 3224 : audit [DBG] from='client.? 172.21.15.6:0/1230654081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:46.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:45 smithi006 bash[27292]: cluster 2023-08-28T00:24:44.527376+0000 mgr.y (mgr.14901) 4385 : cluster [DBG] pgmap v4392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:46.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:45 smithi006 bash[27292]: audit 2023-08-28T00:24:45.243621+0000 mon.a (mon.0) 3224 : audit [DBG] from='client.? 172.21.15.6:0/1230654081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:45 smithi130 bash[27579]: cluster 2023-08-28T00:24:44.527376+0000 mgr.y (mgr.14901) 4385 : cluster [DBG] pgmap v4392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:45 smithi130 bash[27579]: audit 2023-08-28T00:24:45.243621+0000 mon.a (mon.0) 3224 : audit [DBG] from='client.? 172.21.15.6:0/1230654081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:46.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:46 smithi130 bash[51302]: debug 2023-08-28T00:24:46.476+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:46.479951+0000) 2023-08-28T00:24:47.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:47 smithi130 bash[51302]: debug 2023-08-28T00:24:47.476+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:47.480282+0000) 2023-08-28T00:24:48.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:47 smithi006 bash[27292]: cluster 2023-08-28T00:24:46.528075+0000 mgr.y (mgr.14901) 4386 : cluster [DBG] pgmap v4393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:48.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:47 smithi006 bash[27292]: audit 2023-08-28T00:24:47.657949+0000 mon.a (mon.0) 3225 : audit [DBG] from='client.? 172.21.15.6:0/1020973408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:48.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:47 smithi006 bash[33918]: cluster 2023-08-28T00:24:46.528075+0000 mgr.y (mgr.14901) 4386 : cluster [DBG] pgmap v4393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:48.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:47 smithi006 bash[33918]: audit 2023-08-28T00:24:47.657949+0000 mon.a (mon.0) 3225 : audit [DBG] from='client.? 172.21.15.6:0/1020973408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:47 smithi130 bash[27579]: cluster 2023-08-28T00:24:46.528075+0000 mgr.y (mgr.14901) 4386 : cluster [DBG] pgmap v4393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:47 smithi130 bash[27579]: audit 2023-08-28T00:24:47.657949+0000 mon.a (mon.0) 3225 : audit [DBG] from='client.? 172.21.15.6:0/1020973408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:48 smithi130 bash[51302]: debug 2023-08-28T00:24:48.480+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:48.480547+0000) 2023-08-28T00:24:49.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:49 smithi130 bash[51302]: debug 2023-08-28T00:24:49.476+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:49.480838+0000) 2023-08-28T00:24:50.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:49 smithi006 bash[27292]: cluster 2023-08-28T00:24:48.528722+0000 mgr.y (mgr.14901) 4387 : cluster [DBG] pgmap v4394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:50.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:49 smithi006 bash[33918]: cluster 2023-08-28T00:24:48.528722+0000 mgr.y (mgr.14901) 4387 : cluster [DBG] pgmap v4394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:50.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:49 smithi130 bash[27579]: cluster 2023-08-28T00:24:48.528722+0000 mgr.y (mgr.14901) 4387 : cluster [DBG] pgmap v4394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:50 smithi130 bash[51302]: debug 2023-08-28T00:24:50.476+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:50.481141+0000) 2023-08-28T00:24:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:24:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:24:50] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:24:51.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:50 smithi006 bash[33918]: audit 2023-08-28T00:24:50.075136+0000 mon.c (mon.2) 1679 : audit [DBG] from='client.? 172.21.15.6:0/3123832391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:50 smithi006 bash[27292]: audit 2023-08-28T00:24:50.075136+0000 mon.c (mon.2) 1679 : audit [DBG] from='client.? 172.21.15.6:0/3123832391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:24:50.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:51.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:24:50.731Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:51.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:50 smithi130 bash[27579]: audit 2023-08-28T00:24:50.075136+0000 mon.c (mon.2) 1679 : audit [DBG] from='client.? 172.21.15.6:0/3123832391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:51.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:51 smithi130 bash[51302]: debug 2023-08-28T00:24:51.476+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:51.481412+0000) 2023-08-28T00:24:52.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:51 smithi006 bash[27292]: cluster 2023-08-28T00:24:50.529830+0000 mgr.y (mgr.14901) 4388 : cluster [DBG] pgmap v4395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:52.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:51 smithi006 bash[33918]: cluster 2023-08-28T00:24:50.529830+0000 mgr.y (mgr.14901) 4388 : cluster [DBG] pgmap v4395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:51 smithi130 bash[27579]: cluster 2023-08-28T00:24:50.529830+0000 mgr.y (mgr.14901) 4388 : cluster [DBG] pgmap v4395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:24:52.308Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:24:52.310Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:52.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:52 smithi130 bash[51302]: debug 2023-08-28T00:24:52.476+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:52.481735+0000) 2023-08-28T00:24:53.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:52 smithi006 bash[33918]: audit 2023-08-28T00:24:52.499962+0000 mon.c (mon.2) 1680 : audit [DBG] from='client.? 172.21.15.6:0/2442527033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:53.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:52 smithi006 bash[27292]: audit 2023-08-28T00:24:52.499962+0000 mon.c (mon.2) 1680 : audit [DBG] from='client.? 172.21.15.6:0/2442527033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:52 smithi130 bash[27579]: audit 2023-08-28T00:24:52.499962+0000 mon.c (mon.2) 1680 : audit [DBG] from='client.? 172.21.15.6:0/2442527033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:53.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:53 smithi130 bash[51302]: debug 2023-08-28T00:24:53.475+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:53.482011+0000) 2023-08-28T00:24:54.193 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:53 smithi006 bash[33918]: cluster 2023-08-28T00:24:52.530458+0000 mgr.y (mgr.14901) 4389 : cluster [DBG] pgmap v4396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:54.194 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:53 smithi006 bash[27292]: cluster 2023-08-28T00:24:52.530458+0000 mgr.y (mgr.14901) 4389 : cluster [DBG] pgmap v4396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:53 smithi130 bash[27579]: cluster 2023-08-28T00:24:52.530458+0000 mgr.y (mgr.14901) 4389 : cluster [DBG] pgmap v4396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:24:54.190Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:24:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:24:54.192Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:24:54.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:54 smithi130 bash[51302]: debug 2023-08-28T00:24:54.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:54.482294+0000) 2023-08-28T00:24:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:54 smithi006 bash[27292]: audit 2023-08-28T00:24:54.906185+0000 mon.c (mon.2) 1681 : audit [DBG] from='client.? 172.21.15.6:0/2434371198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:54 smithi006 bash[33918]: audit 2023-08-28T00:24:54.906185+0000 mon.c (mon.2) 1681 : audit [DBG] from='client.? 172.21.15.6:0/2434371198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:54 smithi130 bash[27579]: audit 2023-08-28T00:24:54.906185+0000 mon.c (mon.2) 1681 : audit [DBG] from='client.? 172.21.15.6:0/2434371198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:55.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:55 smithi130 bash[51302]: debug 2023-08-28T00:24:55.475+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:55.482607+0000) 2023-08-28T00:24:56.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:55 smithi006 bash[27292]: cluster 2023-08-28T00:24:54.531420+0000 mgr.y (mgr.14901) 4390 : cluster [DBG] pgmap v4397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:56.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:55 smithi006 bash[33918]: cluster 2023-08-28T00:24:54.531420+0000 mgr.y (mgr.14901) 4390 : cluster [DBG] pgmap v4397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:55 smithi130 bash[27579]: cluster 2023-08-28T00:24:54.531420+0000 mgr.y (mgr.14901) 4390 : cluster [DBG] pgmap v4397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:56.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:56 smithi130 bash[51302]: debug 2023-08-28T00:24:56.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:56.483050+0000) 2023-08-28T00:24:57.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:57 smithi130 bash[51302]: debug 2023-08-28T00:24:57.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:57.483271+0000) 2023-08-28T00:24:58.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:57 smithi006 bash[27292]: cluster 2023-08-28T00:24:56.532029+0000 mgr.y (mgr.14901) 4391 : cluster [DBG] pgmap v4398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:58.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:57 smithi006 bash[27292]: audit 2023-08-28T00:24:57.321482+0000 mon.a (mon.0) 3226 : audit [DBG] from='client.? 172.21.15.6:0/540891906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:58.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:57 smithi006 bash[33918]: cluster 2023-08-28T00:24:56.532029+0000 mgr.y (mgr.14901) 4391 : cluster [DBG] pgmap v4398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:58.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:57 smithi006 bash[33918]: audit 2023-08-28T00:24:57.321482+0000 mon.a (mon.0) 3226 : audit [DBG] from='client.? 172.21.15.6:0/540891906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:58.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:57 smithi130 bash[27579]: cluster 2023-08-28T00:24:56.532029+0000 mgr.y (mgr.14901) 4391 : cluster [DBG] pgmap v4398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:24:58.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:57 smithi130 bash[27579]: audit 2023-08-28T00:24:57.321482+0000 mon.a (mon.0) 3226 : audit [DBG] from='client.? 172.21.15.6:0/540891906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:58.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:58 smithi130 bash[51302]: debug 2023-08-28T00:24:58.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:58.483567+0000) 2023-08-28T00:24:59.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:24:59 smithi130 bash[51302]: debug 2023-08-28T00:24:59.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:59.483858+0000) 2023-08-28T00:25:00.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:59 smithi006 bash[33918]: cluster 2023-08-28T00:24:58.532751+0000 mgr.y (mgr.14901) 4392 : cluster [DBG] pgmap v4399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:00.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:24:59 smithi006 bash[33918]: audit 2023-08-28T00:24:59.737411+0000 mon.a (mon.0) 3227 : audit [DBG] from='client.? 172.21.15.6:0/2213898774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:00.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:59 smithi006 bash[27292]: cluster 2023-08-28T00:24:58.532751+0000 mgr.y (mgr.14901) 4392 : cluster [DBG] pgmap v4399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:00.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:24:59 smithi006 bash[27292]: audit 2023-08-28T00:24:59.737411+0000 mon.a (mon.0) 3227 : audit [DBG] from='client.? 172.21.15.6:0/2213898774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:00.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:59 smithi130 bash[27579]: cluster 2023-08-28T00:24:58.532751+0000 mgr.y (mgr.14901) 4392 : cluster [DBG] pgmap v4399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:00.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:24:59 smithi130 bash[27579]: audit 2023-08-28T00:24:59.737411+0000 mon.a (mon.0) 3227 : audit [DBG] from='client.? 172.21.15.6:0/2213898774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:00.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:00 smithi130 bash[51302]: debug 2023-08-28T00:25:00.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:00.484138+0000) 2023-08-28T00:25:01.150 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:25:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:25:00] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:25:01.150 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:25:00.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:01.151 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:25:00.731Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:01.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:01 smithi130 bash[51302]: debug 2023-08-28T00:25:01.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:01.484379+0000) 2023-08-28T00:25:02.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:01 smithi006 bash[27292]: cluster 2023-08-28T00:25:00.533749+0000 mgr.y (mgr.14901) 4393 : cluster [DBG] pgmap v4400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:02.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:01 smithi006 bash[33918]: cluster 2023-08-28T00:25:00.533749+0000 mgr.y (mgr.14901) 4393 : cluster [DBG] pgmap v4400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:02.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:01 smithi130 bash[27579]: cluster 2023-08-28T00:25:00.533749+0000 mgr.y (mgr.14901) 4393 : cluster [DBG] pgmap v4400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:25:02.309Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:25:02.311Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:02.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:02 smithi130 bash[51302]: debug 2023-08-28T00:25:02.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:02.484668+0000) 2023-08-28T00:25:03.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:02 smithi130 bash[27579]: audit 2023-08-28T00:25:02.146300+0000 mon.c (mon.2) 1682 : audit [DBG] from='client.? 172.21.15.6:0/3910001755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:03.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:02 smithi006 bash[27292]: audit 2023-08-28T00:25:02.146300+0000 mon.c (mon.2) 1682 : audit [DBG] from='client.? 172.21.15.6:0/3910001755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:03.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:02 smithi006 bash[33918]: audit 2023-08-28T00:25:02.146300+0000 mon.c (mon.2) 1682 : audit [DBG] from='client.? 172.21.15.6:0/3910001755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:03.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:03 smithi130 bash[51302]: debug 2023-08-28T00:25:03.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:03.484972+0000) 2023-08-28T00:25:04.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:03 smithi130 bash[27579]: cluster 2023-08-28T00:25:02.534468+0000 mgr.y (mgr.14901) 4394 : cluster [DBG] pgmap v4401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:04.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:03 smithi006 bash[33918]: cluster 2023-08-28T00:25:02.534468+0000 mgr.y (mgr.14901) 4394 : cluster [DBG] pgmap v4401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:04.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:03 smithi006 bash[27292]: cluster 2023-08-28T00:25:02.534468+0000 mgr.y (mgr.14901) 4394 : cluster [DBG] pgmap v4401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:25:04.190Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:25:04.192Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:04.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:04 smithi130 bash[51302]: debug 2023-08-28T00:25:04.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:04.485282+0000) 2023-08-28T00:25:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:04 smithi130 bash[27579]: audit 2023-08-28T00:25:04.554575+0000 mon.a (mon.0) 3228 : audit [DBG] from='client.? 172.21.15.6:0/3639086928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:05.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:04 smithi006 bash[27292]: audit 2023-08-28T00:25:04.554575+0000 mon.a (mon.0) 3228 : audit [DBG] from='client.? 172.21.15.6:0/3639086928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:05.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:04 smithi006 bash[33918]: audit 2023-08-28T00:25:04.554575+0000 mon.a (mon.0) 3228 : audit [DBG] from='client.? 172.21.15.6:0/3639086928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:05.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:05 smithi130 bash[51302]: debug 2023-08-28T00:25:05.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:05.485563+0000) 2023-08-28T00:25:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:05 smithi130 bash[27579]: cluster 2023-08-28T00:25:04.535578+0000 mgr.y (mgr.14901) 4395 : cluster [DBG] pgmap v4402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:06.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:05 smithi006 bash[27292]: cluster 2023-08-28T00:25:04.535578+0000 mgr.y (mgr.14901) 4395 : cluster [DBG] pgmap v4402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:06.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:05 smithi006 bash[33918]: cluster 2023-08-28T00:25:04.535578+0000 mgr.y (mgr.14901) 4395 : cluster [DBG] pgmap v4402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:06.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:06 smithi130 bash[51302]: debug 2023-08-28T00:25:06.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:06.485839+0000) 2023-08-28T00:25:07.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:06 smithi130 bash[27579]: audit 2023-08-28T00:25:06.963070+0000 mon.a (mon.0) 3229 : audit [DBG] from='client.? 172.21.15.6:0/158363981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:07.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:06 smithi006 bash[27292]: audit 2023-08-28T00:25:06.963070+0000 mon.a (mon.0) 3229 : audit [DBG] from='client.? 172.21.15.6:0/158363981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:07.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:06 smithi006 bash[33918]: audit 2023-08-28T00:25:06.963070+0000 mon.a (mon.0) 3229 : audit [DBG] from='client.? 172.21.15.6:0/158363981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:07.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:07 smithi130 bash[51302]: debug 2023-08-28T00:25:07.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:07.486143+0000) 2023-08-28T00:25:08.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:07 smithi130 bash[27579]: cluster 2023-08-28T00:25:06.536150+0000 mgr.y (mgr.14901) 4396 : cluster [DBG] pgmap v4403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:08.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:07 smithi006 bash[27292]: cluster 2023-08-28T00:25:06.536150+0000 mgr.y (mgr.14901) 4396 : cluster [DBG] pgmap v4403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:08.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:07 smithi006 bash[33918]: cluster 2023-08-28T00:25:06.536150+0000 mgr.y (mgr.14901) 4396 : cluster [DBG] pgmap v4403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:08.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:08 smithi130 bash[51302]: debug 2023-08-28T00:25:08.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:08.486446+0000) 2023-08-28T00:25:09.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:09 smithi130 bash[51302]: debug 2023-08-28T00:25:09.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:09.486729+0000) 2023-08-28T00:25:10.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:10 smithi130 bash[27579]: cluster 2023-08-28T00:25:08.536719+0000 mgr.y (mgr.14901) 4397 : cluster [DBG] pgmap v4404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:10.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:10 smithi130 bash[27579]: audit 2023-08-28T00:25:09.380781+0000 mon.a (mon.0) 3230 : audit [DBG] from='client.? 172.21.15.6:0/2809472755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:10.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:10 smithi006 bash[33918]: cluster 2023-08-28T00:25:08.536719+0000 mgr.y (mgr.14901) 4397 : cluster [DBG] pgmap v4404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:10.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:10 smithi006 bash[33918]: audit 2023-08-28T00:25:09.380781+0000 mon.a (mon.0) 3230 : audit [DBG] from='client.? 172.21.15.6:0/2809472755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:10.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:10 smithi006 bash[27292]: cluster 2023-08-28T00:25:08.536719+0000 mgr.y (mgr.14901) 4397 : cluster [DBG] pgmap v4404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:10.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:10 smithi006 bash[27292]: audit 2023-08-28T00:25:09.380781+0000 mon.a (mon.0) 3230 : audit [DBG] from='client.? 172.21.15.6:0/2809472755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:10 smithi130 bash[51302]: debug 2023-08-28T00:25:10.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:10.487008+0000) 2023-08-28T00:25:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:25:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:25:10] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:25:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:25:10.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:25:10.733Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:11.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:11 smithi130 bash[51302]: debug 2023-08-28T00:25:11.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:11.487230+0000) 2023-08-28T00:25:12.312 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:12 smithi006 bash[27292]: cluster 2023-08-28T00:25:10.537694+0000 mgr.y (mgr.14901) 4398 : cluster [DBG] pgmap v4405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:12.312 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:12 smithi006 bash[27292]: audit 2023-08-28T00:25:11.799237+0000 mon.c (mon.2) 1683 : audit [DBG] from='client.? 172.21.15.6:0/1042993389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:12.313 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:12 smithi006 bash[33918]: cluster 2023-08-28T00:25:10.537694+0000 mgr.y (mgr.14901) 4398 : cluster [DBG] pgmap v4405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:12.313 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:12 smithi006 bash[33918]: audit 2023-08-28T00:25:11.799237+0000 mon.c (mon.2) 1683 : audit [DBG] from='client.? 172.21.15.6:0/1042993389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:12.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:12 smithi130 bash[27579]: cluster 2023-08-28T00:25:10.537694+0000 mgr.y (mgr.14901) 4398 : cluster [DBG] pgmap v4405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:12.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:12 smithi130 bash[27579]: audit 2023-08-28T00:25:11.799237+0000 mon.c (mon.2) 1683 : audit [DBG] from='client.? 172.21.15.6:0/1042993389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:25:12.309Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:25:12.311Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:12 smithi130 bash[51302]: debug 2023-08-28T00:25:12.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:12.487511+0000) 2023-08-28T00:25:13.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:13 smithi130 bash[51302]: debug 2023-08-28T00:25:13.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:13.487802+0000) 2023-08-28T00:25:14.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:14 smithi130 bash[27579]: cluster 2023-08-28T00:25:12.538400+0000 mgr.y (mgr.14901) 4399 : cluster [DBG] pgmap v4406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:14.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:14 smithi006 bash[33918]: cluster 2023-08-28T00:25:12.538400+0000 mgr.y (mgr.14901) 4399 : cluster [DBG] pgmap v4406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:14.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:14 smithi006 bash[27292]: cluster 2023-08-28T00:25:12.538400+0000 mgr.y (mgr.14901) 4399 : cluster [DBG] pgmap v4406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:25:14.191Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:25:14.193Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:14 smithi130 bash[51302]: debug 2023-08-28T00:25:14.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:14.488106+0000) 2023-08-28T00:25:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:15 smithi130 bash[27579]: audit 2023-08-28T00:25:14.209144+0000 mon.c (mon.2) 1684 : audit [DBG] from='client.? 172.21.15.6:0/2163758259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:15.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:15 smithi006 bash[27292]: audit 2023-08-28T00:25:14.209144+0000 mon.c (mon.2) 1684 : audit [DBG] from='client.? 172.21.15.6:0/2163758259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:15.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:15 smithi006 bash[33918]: audit 2023-08-28T00:25:14.209144+0000 mon.c (mon.2) 1684 : audit [DBG] from='client.? 172.21.15.6:0/2163758259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:15.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:15 smithi130 bash[51302]: debug 2023-08-28T00:25:15.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:15.488428+0000) 2023-08-28T00:25:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:16 smithi130 bash[27579]: cluster 2023-08-28T00:25:14.539516+0000 mgr.y (mgr.14901) 4400 : cluster [DBG] pgmap v4407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:16.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:16 smithi006 bash[27292]: cluster 2023-08-28T00:25:14.539516+0000 mgr.y (mgr.14901) 4400 : cluster [DBG] pgmap v4407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:16.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:16 smithi006 bash[33918]: cluster 2023-08-28T00:25:14.539516+0000 mgr.y (mgr.14901) 4400 : cluster [DBG] pgmap v4407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:16.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:16 smithi130 bash[51302]: debug 2023-08-28T00:25:16.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:16.488643+0000) 2023-08-28T00:25:17.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:17 smithi130 bash[27579]: audit 2023-08-28T00:25:16.621742+0000 mon.c (mon.2) 1685 : audit [DBG] from='client.? 172.21.15.6:0/1541080518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:17.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:17 smithi006 bash[27292]: audit 2023-08-28T00:25:16.621742+0000 mon.c (mon.2) 1685 : audit [DBG] from='client.? 172.21.15.6:0/1541080518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:17.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:17 smithi006 bash[33918]: audit 2023-08-28T00:25:16.621742+0000 mon.c (mon.2) 1685 : audit [DBG] from='client.? 172.21.15.6:0/1541080518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:17.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:17 smithi130 bash[51302]: debug 2023-08-28T00:25:17.487+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:17.488929+0000) 2023-08-28T00:25:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:18 smithi130 bash[27579]: cluster 2023-08-28T00:25:16.540116+0000 mgr.y (mgr.14901) 4401 : cluster [DBG] pgmap v4408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:18.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:18 smithi006 bash[33918]: cluster 2023-08-28T00:25:16.540116+0000 mgr.y (mgr.14901) 4401 : cluster [DBG] pgmap v4408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:18.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:18 smithi006 bash[27292]: cluster 2023-08-28T00:25:16.540116+0000 mgr.y (mgr.14901) 4401 : cluster [DBG] pgmap v4408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:18 smithi130 bash[51302]: debug 2023-08-28T00:25:18.487+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:18.489255+0000) 2023-08-28T00:25:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:19 smithi130 bash[51302]: debug 2023-08-28T00:25:19.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:19.489537+0000) 2023-08-28T00:25:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:20 smithi130 bash[27579]: cluster 2023-08-28T00:25:18.540776+0000 mgr.y (mgr.14901) 4402 : cluster [DBG] pgmap v4409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:20 smithi130 bash[27579]: audit 2023-08-28T00:25:19.041571+0000 mon.c (mon.2) 1686 : audit [DBG] from='client.? 172.21.15.6:0/2171473974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:20.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:20 smithi006 bash[27292]: cluster 2023-08-28T00:25:18.540776+0000 mgr.y (mgr.14901) 4402 : cluster [DBG] pgmap v4409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:20.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:20 smithi006 bash[27292]: audit 2023-08-28T00:25:19.041571+0000 mon.c (mon.2) 1686 : audit [DBG] from='client.? 172.21.15.6:0/2171473974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:20.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:20 smithi006 bash[33918]: cluster 2023-08-28T00:25:18.540776+0000 mgr.y (mgr.14901) 4402 : cluster [DBG] pgmap v4409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:20.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:20 smithi006 bash[33918]: audit 2023-08-28T00:25:19.041571+0000 mon.c (mon.2) 1686 : audit [DBG] from='client.? 172.21.15.6:0/2171473974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:20.870 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:20 smithi130 bash[51302]: debug 2023-08-28T00:25:20.487+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:20.489822+0000) 2023-08-28T00:25:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:25:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:25:20] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T00:25:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:25:20.732Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:25:20.734Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:21.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:21 smithi130 bash[51302]: debug 2023-08-28T00:25:21.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:21.490062+0000) 2023-08-28T00:25:22.313 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:22 smithi006 bash[33918]: cluster 2023-08-28T00:25:20.542083+0000 mgr.y (mgr.14901) 4403 : cluster [DBG] pgmap v4410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:22.313 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:22 smithi006 bash[33918]: audit 2023-08-28T00:25:21.455055+0000 mon.c (mon.2) 1687 : audit [DBG] from='client.? 172.21.15.6:0/3667791997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:22.313 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:22 smithi006 bash[27292]: cluster 2023-08-28T00:25:20.542083+0000 mgr.y (mgr.14901) 4403 : cluster [DBG] pgmap v4410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:22.313 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:22 smithi006 bash[27292]: audit 2023-08-28T00:25:21.455055+0000 mon.c (mon.2) 1687 : audit [DBG] from='client.? 172.21.15.6:0/3667791997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:22.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:22 smithi130 bash[27579]: cluster 2023-08-28T00:25:20.542083+0000 mgr.y (mgr.14901) 4403 : cluster [DBG] pgmap v4410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:22.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:22 smithi130 bash[27579]: audit 2023-08-28T00:25:21.455055+0000 mon.c (mon.2) 1687 : audit [DBG] from='client.? 172.21.15.6:0/3667791997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:25:22.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:25:22.311Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:22 smithi130 bash[51302]: debug 2023-08-28T00:25:22.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:22.490334+0000) 2023-08-28T00:25:23.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:23 smithi130 bash[51302]: debug 2023-08-28T00:25:23.487+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:23.490693+0000) 2023-08-28T00:25:24.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:24 smithi130 bash[27579]: cluster 2023-08-28T00:25:22.542787+0000 mgr.y (mgr.14901) 4404 : cluster [DBG] pgmap v4411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:24.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:24 smithi130 bash[27579]: audit 2023-08-28T00:25:23.877240+0000 mon.a (mon.0) 3231 : audit [DBG] from='client.? 172.21.15.6:0/200637517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:24.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:24 smithi006 bash[27292]: cluster 2023-08-28T00:25:22.542787+0000 mgr.y (mgr.14901) 4404 : cluster [DBG] pgmap v4411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:24.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:24 smithi006 bash[27292]: audit 2023-08-28T00:25:23.877240+0000 mon.a (mon.0) 3231 : audit [DBG] from='client.? 172.21.15.6:0/200637517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:25:24.191Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:25:24.194Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:24.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:24 smithi006 bash[33918]: cluster 2023-08-28T00:25:22.542787+0000 mgr.y (mgr.14901) 4404 : cluster [DBG] pgmap v4411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:24.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:24 smithi006 bash[33918]: audit 2023-08-28T00:25:23.877240+0000 mon.a (mon.0) 3231 : audit [DBG] from='client.? 172.21.15.6:0/200637517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:24 smithi130 bash[51302]: debug 2023-08-28T00:25:24.487+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:24.490969+0000) 2023-08-28T00:25:25.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:25 smithi130 bash[51302]: debug 2023-08-28T00:25:25.487+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:25.491224+0000) 2023-08-28T00:25:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:26 smithi130 bash[27579]: cluster 2023-08-28T00:25:24.543707+0000 mgr.y (mgr.14901) 4405 : cluster [DBG] pgmap v4412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:26 smithi006 bash[27292]: cluster 2023-08-28T00:25:24.543707+0000 mgr.y (mgr.14901) 4405 : cluster [DBG] pgmap v4412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:26.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:26 smithi006 bash[33918]: cluster 2023-08-28T00:25:24.543707+0000 mgr.y (mgr.14901) 4405 : cluster [DBG] pgmap v4412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:26.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:26 smithi130 bash[51302]: debug 2023-08-28T00:25:26.487+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:26.491465+0000) 2023-08-28T00:25:27.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:27 smithi130 bash[27579]: audit 2023-08-28T00:25:26.285742+0000 mon.c (mon.2) 1688 : audit [DBG] from='client.? 172.21.15.6:0/2641262353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:27.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:27 smithi006 bash[33918]: audit 2023-08-28T00:25:26.285742+0000 mon.c (mon.2) 1688 : audit [DBG] from='client.? 172.21.15.6:0/2641262353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:27.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:27 smithi006 bash[27292]: audit 2023-08-28T00:25:26.285742+0000 mon.c (mon.2) 1688 : audit [DBG] from='client.? 172.21.15.6:0/2641262353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:27.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:27 smithi130 bash[51302]: debug 2023-08-28T00:25:27.487+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:27.491723+0000) 2023-08-28T00:25:28.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:28 smithi130 bash[27579]: cluster 2023-08-28T00:25:26.544381+0000 mgr.y (mgr.14901) 4406 : cluster [DBG] pgmap v4413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:28.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:28 smithi006 bash[27292]: cluster 2023-08-28T00:25:26.544381+0000 mgr.y (mgr.14901) 4406 : cluster [DBG] pgmap v4413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:28.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:28 smithi006 bash[33918]: cluster 2023-08-28T00:25:26.544381+0000 mgr.y (mgr.14901) 4406 : cluster [DBG] pgmap v4413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:28 smithi130 bash[51302]: debug 2023-08-28T00:25:28.491+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:28.491985+0000) 2023-08-28T00:25:29.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:29 smithi130 bash[27579]: audit 2023-08-28T00:25:28.703749+0000 mon.c (mon.2) 1689 : audit [DBG] from='client.? 172.21.15.6:0/2961701659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:29.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:29 smithi006 bash[27292]: audit 2023-08-28T00:25:28.703749+0000 mon.c (mon.2) 1689 : audit [DBG] from='client.? 172.21.15.6:0/2961701659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:29.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:29 smithi006 bash[33918]: audit 2023-08-28T00:25:28.703749+0000 mon.c (mon.2) 1689 : audit [DBG] from='client.? 172.21.15.6:0/2961701659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:29.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:29 smithi130 bash[51302]: debug 2023-08-28T00:25:29.487+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:29.492280+0000) 2023-08-28T00:25:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:30 smithi130 bash[27579]: cluster 2023-08-28T00:25:28.545040+0000 mgr.y (mgr.14901) 4407 : cluster [DBG] pgmap v4414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:30.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:30 smithi006 bash[33918]: cluster 2023-08-28T00:25:28.545040+0000 mgr.y (mgr.14901) 4407 : cluster [DBG] pgmap v4414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:30.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:30 smithi006 bash[27292]: cluster 2023-08-28T00:25:28.545040+0000 mgr.y (mgr.14901) 4407 : cluster [DBG] pgmap v4414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:30 smithi130 bash[51302]: debug 2023-08-28T00:25:30.491+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:30.492578+0000) 2023-08-28T00:25:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:25:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:25:30] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:25:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:25:30.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:25:30.735Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:31.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:31 smithi130 bash[51302]: debug 2023-08-28T00:25:31.491+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:31.492898+0000) 2023-08-28T00:25:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:32 smithi130 bash[27579]: cluster 2023-08-28T00:25:30.546127+0000 mgr.y (mgr.14901) 4408 : cluster [DBG] pgmap v4415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:32 smithi130 bash[27579]: audit 2023-08-28T00:25:31.109563+0000 mon.a (mon.0) 3232 : audit [DBG] from='client.? 172.21.15.6:0/1694694872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:32.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:32 smithi006 bash[27292]: cluster 2023-08-28T00:25:30.546127+0000 mgr.y (mgr.14901) 4408 : cluster [DBG] pgmap v4415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:32.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:32 smithi006 bash[27292]: audit 2023-08-28T00:25:31.109563+0000 mon.a (mon.0) 3232 : audit [DBG] from='client.? 172.21.15.6:0/1694694872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:32.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:32 smithi006 bash[33918]: cluster 2023-08-28T00:25:30.546127+0000 mgr.y (mgr.14901) 4408 : cluster [DBG] pgmap v4415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:32.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:32 smithi006 bash[33918]: audit 2023-08-28T00:25:31.109563+0000 mon.a (mon.0) 3232 : audit [DBG] from='client.? 172.21.15.6:0/1694694872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:32.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:25:32.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:32.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:25:32.312Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:32 smithi130 bash[51302]: debug 2023-08-28T00:25:32.487+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:32.493125+0000) 2023-08-28T00:25:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:33 smithi130 bash[27579]: audit 2023-08-28T00:25:33.083408+0000 mon.b (mon.1) 544 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:25:33.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:33 smithi006 bash[33918]: audit 2023-08-28T00:25:33.083408+0000 mon.b (mon.1) 544 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:25:33.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:33 smithi006 bash[27292]: audit 2023-08-28T00:25:33.083408+0000 mon.b (mon.1) 544 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:25:33.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:33 smithi130 bash[51302]: debug 2023-08-28T00:25:33.491+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:33.493404+0000) 2023-08-28T00:25:34.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:34 smithi130 bash[27579]: cluster 2023-08-28T00:25:32.546811+0000 mgr.y (mgr.14901) 4409 : cluster [DBG] pgmap v4416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:34.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:34 smithi130 bash[27579]: audit 2023-08-28T00:25:33.528446+0000 mon.c (mon.2) 1690 : audit [DBG] from='client.? 172.21.15.6:0/2864215228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:25:34.193Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:25:34.195Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:34.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:34 smithi006 bash[27292]: cluster 2023-08-28T00:25:32.546811+0000 mgr.y (mgr.14901) 4409 : cluster [DBG] pgmap v4416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:34.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:34 smithi006 bash[27292]: audit 2023-08-28T00:25:33.528446+0000 mon.c (mon.2) 1690 : audit [DBG] from='client.? 172.21.15.6:0/2864215228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:34.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:34 smithi006 bash[33918]: cluster 2023-08-28T00:25:32.546811+0000 mgr.y (mgr.14901) 4409 : cluster [DBG] pgmap v4416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:34.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:34 smithi006 bash[33918]: audit 2023-08-28T00:25:33.528446+0000 mon.c (mon.2) 1690 : audit [DBG] from='client.? 172.21.15.6:0/2864215228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:34 smithi130 bash[51302]: debug 2023-08-28T00:25:34.487+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:34.493692+0000) 2023-08-28T00:25:35.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:35 smithi130 bash[51302]: debug 2023-08-28T00:25:35.487+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:35.493981+0000) 2023-08-28T00:25:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:36 smithi130 bash[27579]: cluster 2023-08-28T00:25:34.547855+0000 mgr.y (mgr.14901) 4410 : cluster [DBG] pgmap v4417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:36 smithi130 bash[27579]: audit 2023-08-28T00:25:35.948505+0000 mon.a (mon.0) 3233 : audit [DBG] from='client.? 172.21.15.6:0/1997336684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:36.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:36 smithi006 bash[27292]: cluster 2023-08-28T00:25:34.547855+0000 mgr.y (mgr.14901) 4410 : cluster [DBG] pgmap v4417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:36.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:36 smithi006 bash[27292]: audit 2023-08-28T00:25:35.948505+0000 mon.a (mon.0) 3233 : audit [DBG] from='client.? 172.21.15.6:0/1997336684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:36.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:36 smithi006 bash[33918]: cluster 2023-08-28T00:25:34.547855+0000 mgr.y (mgr.14901) 4410 : cluster [DBG] pgmap v4417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:36.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:36 smithi006 bash[33918]: audit 2023-08-28T00:25:35.948505+0000 mon.a (mon.0) 3233 : audit [DBG] from='client.? 172.21.15.6:0/1997336684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:36 smithi130 bash[51302]: debug 2023-08-28T00:25:36.487+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:36.494243+0000) 2023-08-28T00:25:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:37 smithi130 bash[51302]: debug 2023-08-28T00:25:37.487+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:37.494506+0000) 2023-08-28T00:25:38.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:38 smithi006 bash[27292]: cluster 2023-08-28T00:25:36.548576+0000 mgr.y (mgr.14901) 4411 : cluster [DBG] pgmap v4418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:38.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:38 smithi006 bash[33918]: cluster 2023-08-28T00:25:36.548576+0000 mgr.y (mgr.14901) 4411 : cluster [DBG] pgmap v4418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:38.496 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:38 smithi130 bash[27579]: cluster 2023-08-28T00:25:36.548576+0000 mgr.y (mgr.14901) 4411 : cluster [DBG] pgmap v4418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:38.497 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:38 smithi130 bash[51302]: debug 2023-08-28T00:25:38.487+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:38.494749+0000) 2023-08-28T00:25:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:39 smithi006 bash[27292]: audit 2023-08-28T00:25:38.385987+0000 mon.a (mon.0) 3234 : audit [DBG] from='client.? 172.21.15.6:0/435406843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:39 smithi006 bash[27292]: audit 2023-08-28T00:25:38.909543+0000 mon.a (mon.0) 3235 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:25:39.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:39 smithi006 bash[33918]: audit 2023-08-28T00:25:38.385987+0000 mon.a (mon.0) 3234 : audit [DBG] from='client.? 172.21.15.6:0/435406843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:39.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:39 smithi006 bash[33918]: audit 2023-08-28T00:25:38.909543+0000 mon.a (mon.0) 3235 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:25:39.496 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:39 smithi130 bash[27579]: audit 2023-08-28T00:25:38.385987+0000 mon.a (mon.0) 3234 : audit [DBG] from='client.? 172.21.15.6:0/435406843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:39.496 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:39 smithi130 bash[27579]: audit 2023-08-28T00:25:38.909543+0000 mon.a (mon.0) 3235 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:25:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:39 smithi130 bash[51302]: debug 2023-08-28T00:25:39.491+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:39.495038+0000) 2023-08-28T00:25:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:40 smithi006 bash[33918]: cluster 2023-08-28T00:25:38.549111+0000 mgr.y (mgr.14901) 4412 : cluster [DBG] pgmap v4419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:40 smithi006 bash[33918]: audit 2023-08-28T00:25:39.396937+0000 mon.a (mon.0) 3236 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:25:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:40 smithi006 bash[33918]: audit 2023-08-28T00:25:39.692747+0000 mon.b (mon.1) 545 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:25:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:40 smithi006 bash[33918]: audit 2023-08-28T00:25:39.694826+0000 mon.b (mon.1) 546 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:25:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:40 smithi006 bash[33918]: audit 2023-08-28T00:25:39.704976+0000 mon.a (mon.0) 3237 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:25:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:40 smithi006 bash[27292]: cluster 2023-08-28T00:25:38.549111+0000 mgr.y (mgr.14901) 4412 : cluster [DBG] pgmap v4419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:40 smithi006 bash[27292]: audit 2023-08-28T00:25:39.396937+0000 mon.a (mon.0) 3236 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:25:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:40 smithi006 bash[27292]: audit 2023-08-28T00:25:39.692747+0000 mon.b (mon.1) 545 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:25:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:40 smithi006 bash[27292]: audit 2023-08-28T00:25:39.694826+0000 mon.b (mon.1) 546 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:25:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:40 smithi006 bash[27292]: audit 2023-08-28T00:25:39.704976+0000 mon.a (mon.0) 3237 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:25:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:40 smithi130 bash[51302]: debug 2023-08-28T00:25:40.491+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:40.495241+0000) 2023-08-28T00:25:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:40 smithi130 bash[27579]: cluster 2023-08-28T00:25:38.549111+0000 mgr.y (mgr.14901) 4412 : cluster [DBG] pgmap v4419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:40.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:40 smithi130 bash[27579]: audit 2023-08-28T00:25:39.396937+0000 mon.a (mon.0) 3236 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:25:40.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:40 smithi130 bash[27579]: audit 2023-08-28T00:25:39.692747+0000 mon.b (mon.1) 545 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:25:40.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:40 smithi130 bash[27579]: audit 2023-08-28T00:25:39.694826+0000 mon.b (mon.1) 546 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:25:40.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:40 smithi130 bash[27579]: audit 2023-08-28T00:25:39.704976+0000 mon.a (mon.0) 3237 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:25:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:25:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:25:40] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:25:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:25:40.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:25:40.735Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:41.774 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:41 smithi130 bash[51302]: debug 2023-08-28T00:25:41.491+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:41.495494+0000) 2023-08-28T00:25:42.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:41 smithi130 bash[27579]: cluster 2023-08-28T00:25:40.550062+0000 mgr.y (mgr.14901) 4413 : cluster [DBG] pgmap v4420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:42.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:41 smithi130 bash[27579]: audit 2023-08-28T00:25:40.803260+0000 mon.c (mon.2) 1691 : audit [DBG] from='client.? 172.21.15.6:0/633615952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:42.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:41 smithi006 bash[27292]: cluster 2023-08-28T00:25:40.550062+0000 mgr.y (mgr.14901) 4413 : cluster [DBG] pgmap v4420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:42.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:41 smithi006 bash[27292]: audit 2023-08-28T00:25:40.803260+0000 mon.c (mon.2) 1691 : audit [DBG] from='client.? 172.21.15.6:0/633615952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:42.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:41 smithi006 bash[33918]: cluster 2023-08-28T00:25:40.550062+0000 mgr.y (mgr.14901) 4413 : cluster [DBG] pgmap v4420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:42.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:41 smithi006 bash[33918]: audit 2023-08-28T00:25:40.803260+0000 mon.c (mon.2) 1691 : audit [DBG] from='client.? 172.21.15.6:0/633615952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:25:42.311Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:25:42.313Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:42 smithi130 bash[51302]: debug 2023-08-28T00:25:42.491+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:42.495755+0000) 2023-08-28T00:25:43.780 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:43 smithi130 bash[51302]: debug 2023-08-28T00:25:43.491+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:43.496041+0000) 2023-08-28T00:25:44.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:43 smithi130 bash[27579]: cluster 2023-08-28T00:25:42.550754+0000 mgr.y (mgr.14901) 4414 : cluster [DBG] pgmap v4421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:44.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:43 smithi130 bash[27579]: audit 2023-08-28T00:25:43.222467+0000 mon.c (mon.2) 1692 : audit [DBG] from='client.? 172.21.15.6:0/3420045146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:44.196 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:43 smithi006 bash[33918]: cluster 2023-08-28T00:25:42.550754+0000 mgr.y (mgr.14901) 4414 : cluster [DBG] pgmap v4421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:44.196 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:43 smithi006 bash[33918]: audit 2023-08-28T00:25:43.222467+0000 mon.c (mon.2) 1692 : audit [DBG] from='client.? 172.21.15.6:0/3420045146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:44.197 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:43 smithi006 bash[27292]: cluster 2023-08-28T00:25:42.550754+0000 mgr.y (mgr.14901) 4414 : cluster [DBG] pgmap v4421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:44.197 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:43 smithi006 bash[27292]: audit 2023-08-28T00:25:43.222467+0000 mon.c (mon.2) 1692 : audit [DBG] from='client.? 172.21.15.6:0/3420045146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:25:44.193Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:25:44.196Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:44 smithi130 bash[51302]: debug 2023-08-28T00:25:44.491+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:44.496370+0000) 2023-08-28T00:25:45.785 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:45 smithi130 bash[51302]: debug 2023-08-28T00:25:45.491+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:45.496638+0000) 2023-08-28T00:25:46.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:45 smithi130 bash[27579]: cluster 2023-08-28T00:25:44.551888+0000 mgr.y (mgr.14901) 4415 : cluster [DBG] pgmap v4422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:46.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:45 smithi130 bash[27579]: audit 2023-08-28T00:25:45.643429+0000 mon.c (mon.2) 1693 : audit [DBG] from='client.? 172.21.15.6:0/1866836318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:46.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:45 smithi006 bash[33918]: cluster 2023-08-28T00:25:44.551888+0000 mgr.y (mgr.14901) 4415 : cluster [DBG] pgmap v4422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:46.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:45 smithi006 bash[33918]: audit 2023-08-28T00:25:45.643429+0000 mon.c (mon.2) 1693 : audit [DBG] from='client.? 172.21.15.6:0/1866836318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:46.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:45 smithi006 bash[27292]: cluster 2023-08-28T00:25:44.551888+0000 mgr.y (mgr.14901) 4415 : cluster [DBG] pgmap v4422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:46.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:45 smithi006 bash[27292]: audit 2023-08-28T00:25:45.643429+0000 mon.c (mon.2) 1693 : audit [DBG] from='client.? 172.21.15.6:0/1866836318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:46.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:46 smithi130 bash[51302]: debug 2023-08-28T00:25:46.491+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:46.496894+0000) 2023-08-28T00:25:47.788 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:47 smithi130 bash[51302]: debug 2023-08-28T00:25:47.491+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:47.497164+0000) 2023-08-28T00:25:48.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:47 smithi130 bash[27579]: cluster 2023-08-28T00:25:46.552553+0000 mgr.y (mgr.14901) 4416 : cluster [DBG] pgmap v4423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:48.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:47 smithi006 bash[33918]: cluster 2023-08-28T00:25:46.552553+0000 mgr.y (mgr.14901) 4416 : cluster [DBG] pgmap v4423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:48.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:47 smithi006 bash[27292]: cluster 2023-08-28T00:25:46.552553+0000 mgr.y (mgr.14901) 4416 : cluster [DBG] pgmap v4423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:48.797 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:48 smithi130 bash[51302]: debug 2023-08-28T00:25:48.494+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:48.497470+0000) 2023-08-28T00:25:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:48 smithi130 bash[27579]: audit 2023-08-28T00:25:48.066178+0000 mon.a (mon.0) 3238 : audit [DBG] from='client.? 172.21.15.6:0/878907247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:49.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:48 smithi006 bash[27292]: audit 2023-08-28T00:25:48.066178+0000 mon.a (mon.0) 3238 : audit [DBG] from='client.? 172.21.15.6:0/878907247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:49.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:48 smithi006 bash[33918]: audit 2023-08-28T00:25:48.066178+0000 mon.a (mon.0) 3238 : audit [DBG] from='client.? 172.21.15.6:0/878907247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:49.798 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:49 smithi130 bash[51302]: debug 2023-08-28T00:25:49.494+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:49.497821+0000) 2023-08-28T00:25:50.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:49 smithi130 bash[27579]: cluster 2023-08-28T00:25:48.553255+0000 mgr.y (mgr.14901) 4417 : cluster [DBG] pgmap v4424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:50.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:49 smithi006 bash[27292]: cluster 2023-08-28T00:25:48.553255+0000 mgr.y (mgr.14901) 4417 : cluster [DBG] pgmap v4424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:50.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:49 smithi006 bash[33918]: cluster 2023-08-28T00:25:48.553255+0000 mgr.y (mgr.14901) 4417 : cluster [DBG] pgmap v4424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:50.805 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:50 smithi130 bash[51302]: debug 2023-08-28T00:25:50.490+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:50.498133+0000) 2023-08-28T00:25:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:50 smithi130 bash[27579]: audit 2023-08-28T00:25:50.506575+0000 mon.a (mon.0) 3239 : audit [DBG] from='client.? 172.21.15.6:0/2139635182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:25:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:25:50] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:25:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:50 smithi006 bash[33918]: audit 2023-08-28T00:25:50.506575+0000 mon.a (mon.0) 3239 : audit [DBG] from='client.? 172.21.15.6:0/2139635182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:50 smithi006 bash[27292]: audit 2023-08-28T00:25:50.506575+0000 mon.a (mon.0) 3239 : audit [DBG] from='client.? 172.21.15.6:0/2139635182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:25:50.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:25:50.736Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:51.808 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:51 smithi130 bash[51302]: debug 2023-08-28T00:25:51.490+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:51.498427+0000) 2023-08-28T00:25:52.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:51 smithi130 bash[27579]: cluster 2023-08-28T00:25:50.554351+0000 mgr.y (mgr.14901) 4418 : cluster [DBG] pgmap v4425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:52.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:51 smithi006 bash[33918]: cluster 2023-08-28T00:25:50.554351+0000 mgr.y (mgr.14901) 4418 : cluster [DBG] pgmap v4425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:52.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:51 smithi006 bash[27292]: cluster 2023-08-28T00:25:50.554351+0000 mgr.y (mgr.14901) 4418 : cluster [DBG] pgmap v4425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:25:52.312Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:25:52.314Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:52.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:52 smithi130 bash[51302]: debug 2023-08-28T00:25:52.490+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:52.498731+0000) 2023-08-28T00:25:53.813 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:53 smithi130 bash[51302]: debug 2023-08-28T00:25:53.494+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:53.499016+0000) 2023-08-28T00:25:54.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:53 smithi130 bash[27579]: cluster 2023-08-28T00:25:52.554944+0000 mgr.y (mgr.14901) 4419 : cluster [DBG] pgmap v4426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:54.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:53 smithi130 bash[27579]: audit 2023-08-28T00:25:52.923381+0000 mon.c (mon.2) 1694 : audit [DBG] from='client.? 172.21.15.6:0/4042049775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:54.196 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:53 smithi006 bash[33918]: cluster 2023-08-28T00:25:52.554944+0000 mgr.y (mgr.14901) 4419 : cluster [DBG] pgmap v4426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:54.196 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:53 smithi006 bash[33918]: audit 2023-08-28T00:25:52.923381+0000 mon.c (mon.2) 1694 : audit [DBG] from='client.? 172.21.15.6:0/4042049775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:54.197 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:53 smithi006 bash[27292]: cluster 2023-08-28T00:25:52.554944+0000 mgr.y (mgr.14901) 4419 : cluster [DBG] pgmap v4426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:54.197 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:53 smithi006 bash[27292]: audit 2023-08-28T00:25:52.923381+0000 mon.c (mon.2) 1694 : audit [DBG] from='client.? 172.21.15.6:0/4042049775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:25:54.193Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:25:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:25:54.196Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:25:54.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:54 smithi130 bash[51302]: debug 2023-08-28T00:25:54.494+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:54.499238+0000) 2023-08-28T00:25:55.818 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:55 smithi130 bash[51302]: debug 2023-08-28T00:25:55.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:55.499527+0000) 2023-08-28T00:25:56.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:55 smithi130 bash[27579]: cluster 2023-08-28T00:25:54.556160+0000 mgr.y (mgr.14901) 4420 : cluster [DBG] pgmap v4427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:56.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:55 smithi130 bash[27579]: audit 2023-08-28T00:25:55.334503+0000 mon.a (mon.0) 3240 : audit [DBG] from='client.? 172.21.15.6:0/3409763065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:56.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:55 smithi006 bash[33918]: cluster 2023-08-28T00:25:54.556160+0000 mgr.y (mgr.14901) 4420 : cluster [DBG] pgmap v4427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:56.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:55 smithi006 bash[33918]: audit 2023-08-28T00:25:55.334503+0000 mon.a (mon.0) 3240 : audit [DBG] from='client.? 172.21.15.6:0/3409763065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:56.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:55 smithi006 bash[27292]: cluster 2023-08-28T00:25:54.556160+0000 mgr.y (mgr.14901) 4420 : cluster [DBG] pgmap v4427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:56.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:55 smithi006 bash[27292]: audit 2023-08-28T00:25:55.334503+0000 mon.a (mon.0) 3240 : audit [DBG] from='client.? 172.21.15.6:0/3409763065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:56.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:56 smithi130 bash[51302]: debug 2023-08-28T00:25:56.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:56.499761+0000) 2023-08-28T00:25:57.823 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:57 smithi130 bash[51302]: debug 2023-08-28T00:25:57.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:57.499987+0000) 2023-08-28T00:25:58.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:57 smithi130 bash[27579]: cluster 2023-08-28T00:25:56.556861+0000 mgr.y (mgr.14901) 4421 : cluster [DBG] pgmap v4428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:58.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:57 smithi130 bash[27579]: audit 2023-08-28T00:25:57.743996+0000 mon.c (mon.2) 1695 : audit [DBG] from='client.? 172.21.15.6:0/2627004797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:58.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:57 smithi006 bash[33918]: cluster 2023-08-28T00:25:56.556861+0000 mgr.y (mgr.14901) 4421 : cluster [DBG] pgmap v4428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:58.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:57 smithi006 bash[33918]: audit 2023-08-28T00:25:57.743996+0000 mon.c (mon.2) 1695 : audit [DBG] from='client.? 172.21.15.6:0/2627004797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:58.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:57 smithi006 bash[27292]: cluster 2023-08-28T00:25:56.556861+0000 mgr.y (mgr.14901) 4421 : cluster [DBG] pgmap v4428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:25:58.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:57 smithi006 bash[27292]: audit 2023-08-28T00:25:57.743996+0000 mon.c (mon.2) 1695 : audit [DBG] from='client.? 172.21.15.6:0/2627004797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:58.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:58 smithi130 bash[51302]: debug 2023-08-28T00:25:58.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:58.500205+0000) 2023-08-28T00:25:59.828 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:25:59 smithi130 bash[51302]: debug 2023-08-28T00:25:59.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:59.500438+0000) 2023-08-28T00:26:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:25:59 smithi130 bash[27579]: cluster 2023-08-28T00:25:58.557450+0000 mgr.y (mgr.14901) 4422 : cluster [DBG] pgmap v4429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:00.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:25:59 smithi006 bash[33918]: cluster 2023-08-28T00:25:58.557450+0000 mgr.y (mgr.14901) 4422 : cluster [DBG] pgmap v4429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:00.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:25:59 smithi006 bash[27292]: cluster 2023-08-28T00:25:58.557450+0000 mgr.y (mgr.14901) 4422 : cluster [DBG] pgmap v4429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:00.836 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:00 smithi130 bash[51302]: debug 2023-08-28T00:26:00.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:00.500663+0000) 2023-08-28T00:26:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:00 smithi130 bash[27579]: audit 2023-08-28T00:26:00.166482+0000 mon.c (mon.2) 1696 : audit [DBG] from='client.? 172.21.15.6:0/628611588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:26:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:26:00] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:26:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:00 smithi006 bash[33918]: audit 2023-08-28T00:26:00.166482+0000 mon.c (mon.2) 1696 : audit [DBG] from='client.? 172.21.15.6:0/628611588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:01.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:00 smithi006 bash[27292]: audit 2023-08-28T00:26:00.166482+0000 mon.c (mon.2) 1696 : audit [DBG] from='client.? 172.21.15.6:0/628611588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:26:00.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:26:00.738Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:01.839 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:01 smithi130 bash[51302]: debug 2023-08-28T00:26:01.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:01.500947+0000) 2023-08-28T00:26:02.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:01 smithi130 bash[27579]: cluster 2023-08-28T00:26:00.558591+0000 mgr.y (mgr.14901) 4423 : cluster [DBG] pgmap v4430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:02.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:01 smithi006 bash[33918]: cluster 2023-08-28T00:26:00.558591+0000 mgr.y (mgr.14901) 4423 : cluster [DBG] pgmap v4430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:02.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:01 smithi006 bash[27292]: cluster 2023-08-28T00:26:00.558591+0000 mgr.y (mgr.14901) 4423 : cluster [DBG] pgmap v4430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:26:02.312Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:26:02.314Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:02.849 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:02 smithi130 bash[51302]: debug 2023-08-28T00:26:02.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:02.501209+0000) 2023-08-28T00:26:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:02 smithi130 bash[27579]: audit 2023-08-28T00:26:02.576964+0000 mon.c (mon.2) 1697 : audit [DBG] from='client.? 172.21.15.6:0/1153495571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:03.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:02 smithi006 bash[27292]: audit 2023-08-28T00:26:02.576964+0000 mon.c (mon.2) 1697 : audit [DBG] from='client.? 172.21.15.6:0/1153495571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:03.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:02 smithi006 bash[33918]: audit 2023-08-28T00:26:02.576964+0000 mon.c (mon.2) 1697 : audit [DBG] from='client.? 172.21.15.6:0/1153495571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:03.851 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:03 smithi130 bash[51302]: debug 2023-08-28T00:26:03.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:03.501420+0000) 2023-08-28T00:26:04.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:03 smithi130 bash[27579]: cluster 2023-08-28T00:26:02.559036+0000 mgr.y (mgr.14901) 4424 : cluster [DBG] pgmap v4431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:04.198 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:03 smithi006 bash[27292]: cluster 2023-08-28T00:26:02.559036+0000 mgr.y (mgr.14901) 4424 : cluster [DBG] pgmap v4431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:04.198 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:03 smithi006 bash[33918]: cluster 2023-08-28T00:26:02.559036+0000 mgr.y (mgr.14901) 4424 : cluster [DBG] pgmap v4431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:26:04.195Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:26:04.197Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:04.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:04 smithi130 bash[51302]: debug 2023-08-28T00:26:04.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:04.501693+0000) 2023-08-28T00:26:05.856 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:05 smithi130 bash[51302]: debug 2023-08-28T00:26:05.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:05.502031+0000) 2023-08-28T00:26:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:05 smithi130 bash[27579]: cluster 2023-08-28T00:26:04.560102+0000 mgr.y (mgr.14901) 4425 : cluster [DBG] pgmap v4432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:05 smithi130 bash[27579]: audit 2023-08-28T00:26:04.992050+0000 mon.a (mon.0) 3241 : audit [DBG] from='client.? 172.21.15.6:0/1513992512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:05 smithi006 bash[33918]: cluster 2023-08-28T00:26:04.560102+0000 mgr.y (mgr.14901) 4425 : cluster [DBG] pgmap v4432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:05 smithi006 bash[33918]: audit 2023-08-28T00:26:04.992050+0000 mon.a (mon.0) 3241 : audit [DBG] from='client.? 172.21.15.6:0/1513992512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:06.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:05 smithi006 bash[27292]: cluster 2023-08-28T00:26:04.560102+0000 mgr.y (mgr.14901) 4425 : cluster [DBG] pgmap v4432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:06.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:05 smithi006 bash[27292]: audit 2023-08-28T00:26:04.992050+0000 mon.a (mon.0) 3241 : audit [DBG] from='client.? 172.21.15.6:0/1513992512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:06.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:06 smithi130 bash[51302]: debug 2023-08-28T00:26:06.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:06.502347+0000) 2023-08-28T00:26:07.861 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:07 smithi130 bash[51302]: debug 2023-08-28T00:26:07.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:07.502651+0000) 2023-08-28T00:26:08.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:07 smithi130 bash[27579]: cluster 2023-08-28T00:26:06.560770+0000 mgr.y (mgr.14901) 4426 : cluster [DBG] pgmap v4433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:08.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:07 smithi130 bash[27579]: audit 2023-08-28T00:26:07.404806+0000 mon.c (mon.2) 1698 : audit [DBG] from='client.? 172.21.15.6:0/3934132515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:08.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:07 smithi006 bash[33918]: cluster 2023-08-28T00:26:06.560770+0000 mgr.y (mgr.14901) 4426 : cluster [DBG] pgmap v4433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:08.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:07 smithi006 bash[33918]: audit 2023-08-28T00:26:07.404806+0000 mon.c (mon.2) 1698 : audit [DBG] from='client.? 172.21.15.6:0/3934132515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:08.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:07 smithi006 bash[27292]: cluster 2023-08-28T00:26:06.560770+0000 mgr.y (mgr.14901) 4426 : cluster [DBG] pgmap v4433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:08.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:07 smithi006 bash[27292]: audit 2023-08-28T00:26:07.404806+0000 mon.c (mon.2) 1698 : audit [DBG] from='client.? 172.21.15.6:0/3934132515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:08.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:08 smithi130 bash[51302]: debug 2023-08-28T00:26:08.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:08.502968+0000) 2023-08-28T00:26:09.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:09 smithi130 bash[51302]: debug 2023-08-28T00:26:09.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:09.503229+0000) 2023-08-28T00:26:10.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:09 smithi130 bash[27579]: cluster 2023-08-28T00:26:08.561424+0000 mgr.y (mgr.14901) 4427 : cluster [DBG] pgmap v4434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:10.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:09 smithi130 bash[27579]: audit 2023-08-28T00:26:09.817628+0000 mon.a (mon.0) 3242 : audit [DBG] from='client.? 172.21.15.6:0/1085294851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:10.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:09 smithi006 bash[33918]: cluster 2023-08-28T00:26:08.561424+0000 mgr.y (mgr.14901) 4427 : cluster [DBG] pgmap v4434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:10.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:09 smithi006 bash[33918]: audit 2023-08-28T00:26:09.817628+0000 mon.a (mon.0) 3242 : audit [DBG] from='client.? 172.21.15.6:0/1085294851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:10.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:09 smithi006 bash[27292]: cluster 2023-08-28T00:26:08.561424+0000 mgr.y (mgr.14901) 4427 : cluster [DBG] pgmap v4434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:10.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:09 smithi006 bash[27292]: audit 2023-08-28T00:26:09.817628+0000 mon.a (mon.0) 3242 : audit [DBG] from='client.? 172.21.15.6:0/1085294851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:10 smithi130 bash[51302]: debug 2023-08-28T00:26:10.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:10.503451+0000) 2023-08-28T00:26:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:26:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:26:10] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:26:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:26:10.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:26:10.739Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:11.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:11 smithi130 bash[51302]: debug 2023-08-28T00:26:11.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:11.503761+0000) 2023-08-28T00:26:12.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:11 smithi006 bash[27292]: cluster 2023-08-28T00:26:10.562683+0000 mgr.y (mgr.14901) 4428 : cluster [DBG] pgmap v4435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:12.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:11 smithi006 bash[33918]: cluster 2023-08-28T00:26:10.562683+0000 mgr.y (mgr.14901) 4428 : cluster [DBG] pgmap v4435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:12.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:11 smithi130 bash[27579]: cluster 2023-08-28T00:26:10.562683+0000 mgr.y (mgr.14901) 4428 : cluster [DBG] pgmap v4435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:26:12.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:26:12.315Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:12 smithi130 bash[51302]: debug 2023-08-28T00:26:12.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:12.504047+0000) 2023-08-28T00:26:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:12 smithi006 bash[27292]: audit 2023-08-28T00:26:12.227707+0000 mon.c (mon.2) 1699 : audit [DBG] from='client.? 172.21.15.6:0/4190141215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:12 smithi006 bash[33918]: audit 2023-08-28T00:26:12.227707+0000 mon.c (mon.2) 1699 : audit [DBG] from='client.? 172.21.15.6:0/4190141215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:12 smithi130 bash[27579]: audit 2023-08-28T00:26:12.227707+0000 mon.c (mon.2) 1699 : audit [DBG] from='client.? 172.21.15.6:0/4190141215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:13.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:13 smithi130 bash[51302]: debug 2023-08-28T00:26:13.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:13.504312+0000) 2023-08-28T00:26:14.199 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:13 smithi006 bash[27292]: cluster 2023-08-28T00:26:12.563401+0000 mgr.y (mgr.14901) 4429 : cluster [DBG] pgmap v4436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:14.199 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:13 smithi006 bash[33918]: cluster 2023-08-28T00:26:12.563401+0000 mgr.y (mgr.14901) 4429 : cluster [DBG] pgmap v4436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:14.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:13 smithi130 bash[27579]: cluster 2023-08-28T00:26:12.563401+0000 mgr.y (mgr.14901) 4429 : cluster [DBG] pgmap v4436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:26:14.196Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:26:14.197Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:14 smithi130 bash[51302]: debug 2023-08-28T00:26:14.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:14.504602+0000) 2023-08-28T00:26:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:14 smithi006 bash[27292]: audit 2023-08-28T00:26:14.645770+0000 mon.c (mon.2) 1700 : audit [DBG] from='client.? 172.21.15.6:0/1283757360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:14 smithi006 bash[33918]: audit 2023-08-28T00:26:14.645770+0000 mon.c (mon.2) 1700 : audit [DBG] from='client.? 172.21.15.6:0/1283757360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:14 smithi130 bash[27579]: audit 2023-08-28T00:26:14.645770+0000 mon.c (mon.2) 1700 : audit [DBG] from='client.? 172.21.15.6:0/1283757360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:15.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:15 smithi130 bash[51302]: debug 2023-08-28T00:26:15.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:15.504831+0000) 2023-08-28T00:26:16.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:15 smithi006 bash[33918]: cluster 2023-08-28T00:26:14.564507+0000 mgr.y (mgr.14901) 4430 : cluster [DBG] pgmap v4437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:16.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:15 smithi006 bash[27292]: cluster 2023-08-28T00:26:14.564507+0000 mgr.y (mgr.14901) 4430 : cluster [DBG] pgmap v4437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:15 smithi130 bash[27579]: cluster 2023-08-28T00:26:14.564507+0000 mgr.y (mgr.14901) 4430 : cluster [DBG] pgmap v4437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:16.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:16 smithi130 bash[51302]: debug 2023-08-28T00:26:16.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:16.505084+0000) 2023-08-28T00:26:17.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:17 smithi130 bash[51302]: debug 2023-08-28T00:26:17.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:17.505392+0000) 2023-08-28T00:26:18.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:17 smithi006 bash[33918]: cluster 2023-08-28T00:26:16.565106+0000 mgr.y (mgr.14901) 4431 : cluster [DBG] pgmap v4438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:18.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:17 smithi006 bash[33918]: audit 2023-08-28T00:26:17.054521+0000 mon.c (mon.2) 1701 : audit [DBG] from='client.? 172.21.15.6:0/2538124508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:18.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:17 smithi006 bash[27292]: cluster 2023-08-28T00:26:16.565106+0000 mgr.y (mgr.14901) 4431 : cluster [DBG] pgmap v4438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:18.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:17 smithi006 bash[27292]: audit 2023-08-28T00:26:17.054521+0000 mon.c (mon.2) 1701 : audit [DBG] from='client.? 172.21.15.6:0/2538124508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:17 smithi130 bash[27579]: cluster 2023-08-28T00:26:16.565106+0000 mgr.y (mgr.14901) 4431 : cluster [DBG] pgmap v4438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:17 smithi130 bash[27579]: audit 2023-08-28T00:26:17.054521+0000 mon.c (mon.2) 1701 : audit [DBG] from='client.? 172.21.15.6:0/2538124508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:18 smithi130 bash[51302]: debug 2023-08-28T00:26:18.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:18.505712+0000) 2023-08-28T00:26:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:19 smithi130 bash[51302]: debug 2023-08-28T00:26:19.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:19.506001+0000) 2023-08-28T00:26:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:19 smithi006 bash[27292]: cluster 2023-08-28T00:26:18.565712+0000 mgr.y (mgr.14901) 4432 : cluster [DBG] pgmap v4439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:19 smithi006 bash[27292]: audit 2023-08-28T00:26:19.470947+0000 mon.c (mon.2) 1702 : audit [DBG] from='client.? 172.21.15.6:0/3185415212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:20.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:19 smithi006 bash[33918]: cluster 2023-08-28T00:26:18.565712+0000 mgr.y (mgr.14901) 4432 : cluster [DBG] pgmap v4439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:20.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:19 smithi006 bash[33918]: audit 2023-08-28T00:26:19.470947+0000 mon.c (mon.2) 1702 : audit [DBG] from='client.? 172.21.15.6:0/3185415212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:19 smithi130 bash[27579]: cluster 2023-08-28T00:26:18.565712+0000 mgr.y (mgr.14901) 4432 : cluster [DBG] pgmap v4439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:19 smithi130 bash[27579]: audit 2023-08-28T00:26:19.470947+0000 mon.c (mon.2) 1702 : audit [DBG] from='client.? 172.21.15.6:0/3185415212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:20 smithi130 bash[51302]: debug 2023-08-28T00:26:20.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:20.506321+0000) 2023-08-28T00:26:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:26:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:26:20] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:26:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:26:20.737Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:26:20.739Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:21.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:21 smithi130 bash[51302]: debug 2023-08-28T00:26:21.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:21.506596+0000) 2023-08-28T00:26:22.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:21 smithi006 bash[33918]: cluster 2023-08-28T00:26:20.566913+0000 mgr.y (mgr.14901) 4433 : cluster [DBG] pgmap v4440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:22.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:21 smithi006 bash[33918]: audit 2023-08-28T00:26:21.896638+0000 mon.c (mon.2) 1703 : audit [DBG] from='client.? 172.21.15.6:0/3250443875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:22.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:21 smithi006 bash[27292]: cluster 2023-08-28T00:26:20.566913+0000 mgr.y (mgr.14901) 4433 : cluster [DBG] pgmap v4440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:22.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:21 smithi006 bash[27292]: audit 2023-08-28T00:26:21.896638+0000 mon.c (mon.2) 1703 : audit [DBG] from='client.? 172.21.15.6:0/3250443875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:22.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:21 smithi130 bash[27579]: cluster 2023-08-28T00:26:20.566913+0000 mgr.y (mgr.14901) 4433 : cluster [DBG] pgmap v4440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:22.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:21 smithi130 bash[27579]: audit 2023-08-28T00:26:21.896638+0000 mon.c (mon.2) 1703 : audit [DBG] from='client.? 172.21.15.6:0/3250443875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:26:22.314Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:26:22.316Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:22 smithi130 bash[51302]: debug 2023-08-28T00:26:22.506+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:22.506908+0000) 2023-08-28T00:26:23.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:23 smithi130 bash[51302]: debug 2023-08-28T00:26:23.506+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:23.507163+0000) 2023-08-28T00:26:24.199 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:23 smithi006 bash[27292]: cluster 2023-08-28T00:26:22.567600+0000 mgr.y (mgr.14901) 4434 : cluster [DBG] pgmap v4441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:24.200 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:23 smithi006 bash[33918]: cluster 2023-08-28T00:26:22.567600+0000 mgr.y (mgr.14901) 4434 : cluster [DBG] pgmap v4441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:23 smithi130 bash[27579]: cluster 2023-08-28T00:26:22.567600+0000 mgr.y (mgr.14901) 4434 : cluster [DBG] pgmap v4441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:26:24.196Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:26:24.199Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:24 smithi130 bash[51302]: debug 2023-08-28T00:26:24.506+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:24.507407+0000) 2023-08-28T00:26:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:24 smithi006 bash[33918]: audit 2023-08-28T00:26:24.316346+0000 mon.a (mon.0) 3243 : audit [DBG] from='client.? 172.21.15.6:0/3569551211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:24 smithi006 bash[27292]: audit 2023-08-28T00:26:24.316346+0000 mon.a (mon.0) 3243 : audit [DBG] from='client.? 172.21.15.6:0/3569551211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:24 smithi130 bash[27579]: audit 2023-08-28T00:26:24.316346+0000 mon.a (mon.0) 3243 : audit [DBG] from='client.? 172.21.15.6:0/3569551211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:25.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:25 smithi130 bash[51302]: debug 2023-08-28T00:26:25.506+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:25.507676+0000) 2023-08-28T00:26:26.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:25 smithi006 bash[33918]: cluster 2023-08-28T00:26:24.568649+0000 mgr.y (mgr.14901) 4435 : cluster [DBG] pgmap v4442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:25 smithi006 bash[27292]: cluster 2023-08-28T00:26:24.568649+0000 mgr.y (mgr.14901) 4435 : cluster [DBG] pgmap v4442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:25 smithi130 bash[27579]: cluster 2023-08-28T00:26:24.568649+0000 mgr.y (mgr.14901) 4435 : cluster [DBG] pgmap v4442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:26.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:26 smithi130 bash[51302]: debug 2023-08-28T00:26:26.506+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:26.507898+0000) 2023-08-28T00:26:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:26 smithi006 bash[33918]: audit 2023-08-28T00:26:26.733088+0000 mon.c (mon.2) 1704 : audit [DBG] from='client.? 172.21.15.6:0/4238982998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:26 smithi006 bash[27292]: audit 2023-08-28T00:26:26.733088+0000 mon.c (mon.2) 1704 : audit [DBG] from='client.? 172.21.15.6:0/4238982998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:27.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:26 smithi130 bash[27579]: audit 2023-08-28T00:26:26.733088+0000 mon.c (mon.2) 1704 : audit [DBG] from='client.? 172.21.15.6:0/4238982998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:27.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:27 smithi130 bash[51302]: debug 2023-08-28T00:26:27.506+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:27.508215+0000) 2023-08-28T00:26:28.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:27 smithi006 bash[33918]: cluster 2023-08-28T00:26:26.569287+0000 mgr.y (mgr.14901) 4436 : cluster [DBG] pgmap v4443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:28.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:27 smithi006 bash[27292]: cluster 2023-08-28T00:26:26.569287+0000 mgr.y (mgr.14901) 4436 : cluster [DBG] pgmap v4443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:28.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:27 smithi130 bash[27579]: cluster 2023-08-28T00:26:26.569287+0000 mgr.y (mgr.14901) 4436 : cluster [DBG] pgmap v4443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:28 smithi130 bash[51302]: debug 2023-08-28T00:26:28.506+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:28.508514+0000) 2023-08-28T00:26:29.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:29 smithi130 bash[51302]: debug 2023-08-28T00:26:29.506+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:29.508881+0000) 2023-08-28T00:26:30.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:29 smithi006 bash[33918]: cluster 2023-08-28T00:26:28.569975+0000 mgr.y (mgr.14901) 4437 : cluster [DBG] pgmap v4444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:30.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:29 smithi006 bash[33918]: audit 2023-08-28T00:26:29.151539+0000 mon.a (mon.0) 3244 : audit [DBG] from='client.? 172.21.15.6:0/2114251956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:30.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:29 smithi006 bash[27292]: cluster 2023-08-28T00:26:28.569975+0000 mgr.y (mgr.14901) 4437 : cluster [DBG] pgmap v4444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:30.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:29 smithi006 bash[27292]: audit 2023-08-28T00:26:29.151539+0000 mon.a (mon.0) 3244 : audit [DBG] from='client.? 172.21.15.6:0/2114251956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:29 smithi130 bash[27579]: cluster 2023-08-28T00:26:28.569975+0000 mgr.y (mgr.14901) 4437 : cluster [DBG] pgmap v4444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:29 smithi130 bash[27579]: audit 2023-08-28T00:26:29.151539+0000 mon.a (mon.0) 3244 : audit [DBG] from='client.? 172.21.15.6:0/2114251956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:30 smithi130 bash[51302]: debug 2023-08-28T00:26:30.506+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:30.509146+0000) 2023-08-28T00:26:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:26:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:26:30] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:26:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:26:30.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:26:30.739Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:31.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:31 smithi130 bash[51302]: debug 2023-08-28T00:26:31.506+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:31.509417+0000) 2023-08-28T00:26:32.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:31 smithi006 bash[33918]: cluster 2023-08-28T00:26:30.571046+0000 mgr.y (mgr.14901) 4438 : cluster [DBG] pgmap v4445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:32.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:31 smithi006 bash[33918]: audit 2023-08-28T00:26:31.569074+0000 mon.c (mon.2) 1705 : audit [DBG] from='client.? 172.21.15.6:0/1104238081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:32.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:31 smithi006 bash[27292]: cluster 2023-08-28T00:26:30.571046+0000 mgr.y (mgr.14901) 4438 : cluster [DBG] pgmap v4445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:32.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:31 smithi006 bash[27292]: audit 2023-08-28T00:26:31.569074+0000 mon.c (mon.2) 1705 : audit [DBG] from='client.? 172.21.15.6:0/1104238081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:31 smithi130 bash[27579]: cluster 2023-08-28T00:26:30.571046+0000 mgr.y (mgr.14901) 4438 : cluster [DBG] pgmap v4445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:31 smithi130 bash[27579]: audit 2023-08-28T00:26:31.569074+0000 mon.c (mon.2) 1705 : audit [DBG] from='client.? 172.21.15.6:0/1104238081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:26:32.315Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:26:32.317Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:32 smithi130 bash[51302]: debug 2023-08-28T00:26:32.506+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:32.509714+0000) 2023-08-28T00:26:33.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:33 smithi130 bash[51302]: debug 2023-08-28T00:26:33.506+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:33.510001+0000) 2023-08-28T00:26:34.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:33 smithi006 bash[33918]: cluster 2023-08-28T00:26:32.571676+0000 mgr.y (mgr.14901) 4439 : cluster [DBG] pgmap v4446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:34.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:33 smithi006 bash[27292]: cluster 2023-08-28T00:26:32.571676+0000 mgr.y (mgr.14901) 4439 : cluster [DBG] pgmap v4446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:34.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:26:34.197Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:34.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:26:34.199Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:34.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:33 smithi130 bash[27579]: cluster 2023-08-28T00:26:32.571676+0000 mgr.y (mgr.14901) 4439 : cluster [DBG] pgmap v4446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:34 smithi130 bash[51302]: debug 2023-08-28T00:26:34.506+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:34.510395+0000) 2023-08-28T00:26:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:34 smithi130 bash[27579]: audit 2023-08-28T00:26:33.986415+0000 mon.a (mon.0) 3245 : audit [DBG] from='client.? 172.21.15.6:0/237867236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:35.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:34 smithi006 bash[27292]: audit 2023-08-28T00:26:33.986415+0000 mon.a (mon.0) 3245 : audit [DBG] from='client.? 172.21.15.6:0/237867236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:35.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:34 smithi006 bash[33918]: audit 2023-08-28T00:26:33.986415+0000 mon.a (mon.0) 3245 : audit [DBG] from='client.? 172.21.15.6:0/237867236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:35.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:35 smithi130 bash[51302]: debug 2023-08-28T00:26:35.506+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:35.510704+0000) 2023-08-28T00:26:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:35 smithi130 bash[27579]: cluster 2023-08-28T00:26:34.572774+0000 mgr.y (mgr.14901) 4440 : cluster [DBG] pgmap v4447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:36.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:35 smithi006 bash[33918]: cluster 2023-08-28T00:26:34.572774+0000 mgr.y (mgr.14901) 4440 : cluster [DBG] pgmap v4447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:36.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:35 smithi006 bash[27292]: cluster 2023-08-28T00:26:34.572774+0000 mgr.y (mgr.14901) 4440 : cluster [DBG] pgmap v4447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:36 smithi130 bash[51302]: debug 2023-08-28T00:26:36.510+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:36.511000+0000) 2023-08-28T00:26:37.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:36 smithi130 bash[27579]: audit 2023-08-28T00:26:36.397032+0000 mon.c (mon.2) 1706 : audit [DBG] from='client.? 172.21.15.6:0/2139732624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:37.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:36 smithi006 bash[33918]: audit 2023-08-28T00:26:36.397032+0000 mon.c (mon.2) 1706 : audit [DBG] from='client.? 172.21.15.6:0/2139732624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:37.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:36 smithi006 bash[27292]: audit 2023-08-28T00:26:36.397032+0000 mon.c (mon.2) 1706 : audit [DBG] from='client.? 172.21.15.6:0/2139732624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:37 smithi130 bash[51302]: debug 2023-08-28T00:26:37.510+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:37.511262+0000) 2023-08-28T00:26:38.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:37 smithi130 bash[27579]: cluster 2023-08-28T00:26:36.573399+0000 mgr.y (mgr.14901) 4441 : cluster [DBG] pgmap v4448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:38.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:37 smithi006 bash[33918]: cluster 2023-08-28T00:26:36.573399+0000 mgr.y (mgr.14901) 4441 : cluster [DBG] pgmap v4448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:38.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:37 smithi006 bash[27292]: cluster 2023-08-28T00:26:36.573399+0000 mgr.y (mgr.14901) 4441 : cluster [DBG] pgmap v4448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:38 smithi130 bash[51302]: debug 2023-08-28T00:26:38.510+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:38.511541+0000) 2023-08-28T00:26:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:38 smithi130 bash[27579]: audit 2023-08-28T00:26:38.813041+0000 mon.c (mon.2) 1707 : audit [DBG] from='client.? 172.21.15.6:0/2064364697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:38 smithi006 bash[27292]: audit 2023-08-28T00:26:38.813041+0000 mon.c (mon.2) 1707 : audit [DBG] from='client.? 172.21.15.6:0/2064364697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:39.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:38 smithi006 bash[33918]: audit 2023-08-28T00:26:38.813041+0000 mon.c (mon.2) 1707 : audit [DBG] from='client.? 172.21.15.6:0/2064364697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:39 smithi130 bash[51302]: debug 2023-08-28T00:26:39.510+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:39.511896+0000) 2023-08-28T00:26:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:39 smithi130 bash[27579]: cluster 2023-08-28T00:26:38.574134+0000 mgr.y (mgr.14901) 4442 : cluster [DBG] pgmap v4449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:39 smithi130 bash[27579]: audit 2023-08-28T00:26:39.716442+0000 mon.b (mon.1) 547 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:26:40.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:39 smithi006 bash[27292]: cluster 2023-08-28T00:26:38.574134+0000 mgr.y (mgr.14901) 4442 : cluster [DBG] pgmap v4449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:40.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:39 smithi006 bash[27292]: audit 2023-08-28T00:26:39.716442+0000 mon.b (mon.1) 547 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:26:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:39 smithi006 bash[33918]: cluster 2023-08-28T00:26:38.574134+0000 mgr.y (mgr.14901) 4442 : cluster [DBG] pgmap v4449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:39 smithi006 bash[33918]: audit 2023-08-28T00:26:39.716442+0000 mon.b (mon.1) 547 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:26:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:40 smithi130 bash[51302]: debug 2023-08-28T00:26:40.510+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:40.512249+0000) 2023-08-28T00:26:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:26:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:26:40] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:26:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:26:40.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:26:40.740Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:41.870 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:41 smithi130 bash[51302]: debug 2023-08-28T00:26:41.510+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:41.512585+0000) 2023-08-28T00:26:42.318 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:42 smithi006 bash[27292]: cluster 2023-08-28T00:26:40.575242+0000 mgr.y (mgr.14901) 4443 : cluster [DBG] pgmap v4450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:42.319 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:42 smithi006 bash[27292]: audit 2023-08-28T00:26:41.237725+0000 mon.c (mon.2) 1708 : audit [DBG] from='client.? 172.21.15.6:0/2294743898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:42.319 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:42 smithi006 bash[33918]: cluster 2023-08-28T00:26:40.575242+0000 mgr.y (mgr.14901) 4443 : cluster [DBG] pgmap v4450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:42.319 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:42 smithi006 bash[33918]: audit 2023-08-28T00:26:41.237725+0000 mon.c (mon.2) 1708 : audit [DBG] from='client.? 172.21.15.6:0/2294743898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:42 smithi130 bash[27579]: cluster 2023-08-28T00:26:40.575242+0000 mgr.y (mgr.14901) 4443 : cluster [DBG] pgmap v4450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:42 smithi130 bash[27579]: audit 2023-08-28T00:26:41.237725+0000 mon.c (mon.2) 1708 : audit [DBG] from='client.? 172.21.15.6:0/2294743898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:26:42.315Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:26:42.318Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:42 smithi130 bash[51302]: debug 2023-08-28T00:26:42.510+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:42.512876+0000) 2023-08-28T00:26:43.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:43 smithi130 bash[51302]: debug 2023-08-28T00:26:43.510+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:43.513070+0000) 2023-08-28T00:26:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:44 smithi130 bash[27579]: cluster 2023-08-28T00:26:42.575919+0000 mgr.y (mgr.14901) 4444 : cluster [DBG] pgmap v4451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:44 smithi130 bash[27579]: audit 2023-08-28T00:26:43.649233+0000 mon.c (mon.2) 1709 : audit [DBG] from='client.? 172.21.15.6:0/4230244737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:26:44.198Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:44.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:26:44.200Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:44.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:44 smithi006 bash[33918]: cluster 2023-08-28T00:26:42.575919+0000 mgr.y (mgr.14901) 4444 : cluster [DBG] pgmap v4451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:44.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:44 smithi006 bash[33918]: audit 2023-08-28T00:26:43.649233+0000 mon.c (mon.2) 1709 : audit [DBG] from='client.? 172.21.15.6:0/4230244737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:44.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:44 smithi006 bash[27292]: cluster 2023-08-28T00:26:42.575919+0000 mgr.y (mgr.14901) 4444 : cluster [DBG] pgmap v4451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:44.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:44 smithi006 bash[27292]: audit 2023-08-28T00:26:43.649233+0000 mon.c (mon.2) 1709 : audit [DBG] from='client.? 172.21.15.6:0/4230244737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:44 smithi130 bash[51302]: debug 2023-08-28T00:26:44.509+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:44.513414+0000) 2023-08-28T00:26:45.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:45 smithi130 bash[51302]: debug 2023-08-28T00:26:45.509+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:45.513769+0000) 2023-08-28T00:26:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:46 smithi130 bash[27579]: cluster 2023-08-28T00:26:44.577357+0000 mgr.y (mgr.14901) 4445 : cluster [DBG] pgmap v4452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:46.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:46 smithi130 bash[27579]: audit 2023-08-28T00:26:45.654546+0000 mon.a (mon.0) 3246 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:26:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:46 smithi006 bash[33918]: cluster 2023-08-28T00:26:44.577357+0000 mgr.y (mgr.14901) 4445 : cluster [DBG] pgmap v4452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:46 smithi006 bash[33918]: audit 2023-08-28T00:26:45.654546+0000 mon.a (mon.0) 3246 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:26:46.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:46 smithi006 bash[27292]: cluster 2023-08-28T00:26:44.577357+0000 mgr.y (mgr.14901) 4445 : cluster [DBG] pgmap v4452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:46.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:46 smithi006 bash[27292]: audit 2023-08-28T00:26:45.654546+0000 mon.a (mon.0) 3246 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:26:46.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:46 smithi130 bash[51302]: debug 2023-08-28T00:26:46.509+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:46.514046+0000) 2023-08-28T00:26:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:47 smithi130 bash[27579]: audit 2023-08-28T00:26:46.052994+0000 mon.a (mon.0) 3247 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:26:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:47 smithi130 bash[27579]: audit 2023-08-28T00:26:46.104706+0000 mon.a (mon.0) 3248 : audit [DBG] from='client.? 172.21.15.6:0/3630473192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:47 smithi130 bash[27579]: audit 2023-08-28T00:26:46.343247+0000 mon.b (mon.1) 548 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:26:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:47 smithi130 bash[27579]: audit 2023-08-28T00:26:46.345051+0000 mon.b (mon.1) 549 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:26:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:47 smithi130 bash[27579]: audit 2023-08-28T00:26:46.352538+0000 mon.a (mon.0) 3249 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:26:47.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:47 smithi006 bash[33918]: audit 2023-08-28T00:26:46.052994+0000 mon.a (mon.0) 3247 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:26:47.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:47 smithi006 bash[33918]: audit 2023-08-28T00:26:46.104706+0000 mon.a (mon.0) 3248 : audit [DBG] from='client.? 172.21.15.6:0/3630473192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:47.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:47 smithi006 bash[33918]: audit 2023-08-28T00:26:46.343247+0000 mon.b (mon.1) 548 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:26:47.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:47 smithi006 bash[33918]: audit 2023-08-28T00:26:46.345051+0000 mon.b (mon.1) 549 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:26:47.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:47 smithi006 bash[33918]: audit 2023-08-28T00:26:46.352538+0000 mon.a (mon.0) 3249 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:26:47.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:47 smithi006 bash[27292]: audit 2023-08-28T00:26:46.052994+0000 mon.a (mon.0) 3247 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:26:47.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:47 smithi006 bash[27292]: audit 2023-08-28T00:26:46.104706+0000 mon.a (mon.0) 3248 : audit [DBG] from='client.? 172.21.15.6:0/3630473192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:47.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:47 smithi006 bash[27292]: audit 2023-08-28T00:26:46.343247+0000 mon.b (mon.1) 548 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:26:47.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:47 smithi006 bash[27292]: audit 2023-08-28T00:26:46.345051+0000 mon.b (mon.1) 549 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:26:47.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:47 smithi006 bash[27292]: audit 2023-08-28T00:26:46.352538+0000 mon.a (mon.0) 3249 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:26:47.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:47 smithi130 bash[51302]: debug 2023-08-28T00:26:47.509+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:47.514289+0000) 2023-08-28T00:26:48.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:48 smithi130 bash[51302]: debug 2023-08-28T00:26:48.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:48.514577+0000) 2023-08-28T00:26:48.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:48 smithi130 bash[27579]: cluster 2023-08-28T00:26:46.578005+0000 mgr.y (mgr.14901) 4446 : cluster [DBG] pgmap v4453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:48 smithi006 bash[33918]: cluster 2023-08-28T00:26:46.578005+0000 mgr.y (mgr.14901) 4446 : cluster [DBG] pgmap v4453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:48.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:48 smithi006 bash[27292]: cluster 2023-08-28T00:26:46.578005+0000 mgr.y (mgr.14901) 4446 : cluster [DBG] pgmap v4453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:49.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:49 smithi130 bash[51302]: debug 2023-08-28T00:26:49.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:49.514875+0000) 2023-08-28T00:26:49.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:49 smithi130 bash[27579]: audit 2023-08-28T00:26:48.516877+0000 mon.a (mon.0) 3250 : audit [DBG] from='client.? 172.21.15.6:0/1600916992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:49.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:49 smithi006 bash[27292]: audit 2023-08-28T00:26:48.516877+0000 mon.a (mon.0) 3250 : audit [DBG] from='client.? 172.21.15.6:0/1600916992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:49.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:49 smithi006 bash[33918]: audit 2023-08-28T00:26:48.516877+0000 mon.a (mon.0) 3250 : audit [DBG] from='client.? 172.21.15.6:0/1600916992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:50.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:50 smithi130 bash[51302]: debug 2023-08-28T00:26:50.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:50.515137+0000) 2023-08-28T00:26:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:50 smithi130 bash[27579]: cluster 2023-08-28T00:26:48.578551+0000 mgr.y (mgr.14901) 4447 : cluster [DBG] pgmap v4454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:50 smithi006 bash[27292]: cluster 2023-08-28T00:26:48.578551+0000 mgr.y (mgr.14901) 4447 : cluster [DBG] pgmap v4454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:50 smithi006 bash[33918]: cluster 2023-08-28T00:26:48.578551+0000 mgr.y (mgr.14901) 4447 : cluster [DBG] pgmap v4454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:26:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:26:50] "GET /metrics HTTP/1.1" 200 193132 "" "Prometheus/2.33.4" 2023-08-28T00:26:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:26:50.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:26:50.741Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:51.791 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:51 smithi130 bash[51302]: debug 2023-08-28T00:26:51.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:51.515438+0000) 2023-08-28T00:26:52.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:51 smithi130 bash[27579]: cluster 2023-08-28T00:26:50.579553+0000 mgr.y (mgr.14901) 4448 : cluster [DBG] pgmap v4455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:52.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:51 smithi130 bash[27579]: audit 2023-08-28T00:26:50.934476+0000 mon.c (mon.2) 1710 : audit [DBG] from='client.? 172.21.15.6:0/2859437530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:52.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:51 smithi006 bash[33918]: cluster 2023-08-28T00:26:50.579553+0000 mgr.y (mgr.14901) 4448 : cluster [DBG] pgmap v4455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:52.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:51 smithi006 bash[33918]: audit 2023-08-28T00:26:50.934476+0000 mon.c (mon.2) 1710 : audit [DBG] from='client.? 172.21.15.6:0/2859437530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:52.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:51 smithi006 bash[27292]: cluster 2023-08-28T00:26:50.579553+0000 mgr.y (mgr.14901) 4448 : cluster [DBG] pgmap v4455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:52.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:51 smithi006 bash[27292]: audit 2023-08-28T00:26:50.934476+0000 mon.c (mon.2) 1710 : audit [DBG] from='client.? 172.21.15.6:0/2859437530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:26:52.316Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:26:52.318Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:52.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:52 smithi130 bash[51302]: debug 2023-08-28T00:26:52.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:52.515711+0000) 2023-08-28T00:26:53.796 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:53 smithi130 bash[51302]: debug 2023-08-28T00:26:53.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:53.515955+0000) 2023-08-28T00:26:54.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:53 smithi130 bash[27579]: cluster 2023-08-28T00:26:52.580211+0000 mgr.y (mgr.14901) 4449 : cluster [DBG] pgmap v4456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:54.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:53 smithi130 bash[27579]: audit 2023-08-28T00:26:53.344086+0000 mon.c (mon.2) 1711 : audit [DBG] from='client.? 172.21.15.6:0/4048705372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:54.202 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:53 smithi006 bash[33918]: cluster 2023-08-28T00:26:52.580211+0000 mgr.y (mgr.14901) 4449 : cluster [DBG] pgmap v4456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:54.202 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:53 smithi006 bash[33918]: audit 2023-08-28T00:26:53.344086+0000 mon.c (mon.2) 1711 : audit [DBG] from='client.? 172.21.15.6:0/4048705372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:54.202 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:53 smithi006 bash[27292]: cluster 2023-08-28T00:26:52.580211+0000 mgr.y (mgr.14901) 4449 : cluster [DBG] pgmap v4456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:54.202 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:53 smithi006 bash[27292]: audit 2023-08-28T00:26:53.344086+0000 mon.c (mon.2) 1711 : audit [DBG] from='client.? 172.21.15.6:0/4048705372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:26:54.199Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:26:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:26:54.201Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:26:54.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:54 smithi130 bash[51302]: debug 2023-08-28T00:26:54.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:54.516255+0000) 2023-08-28T00:26:55.802 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:55 smithi130 bash[51302]: debug 2023-08-28T00:26:55.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:55.516504+0000) 2023-08-28T00:26:56.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:55 smithi130 bash[27579]: cluster 2023-08-28T00:26:54.581258+0000 mgr.y (mgr.14901) 4450 : cluster [DBG] pgmap v4457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:56.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:55 smithi130 bash[27579]: audit 2023-08-28T00:26:55.754579+0000 mon.a (mon.0) 3251 : audit [DBG] from='client.? 172.21.15.6:0/3186867669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:56.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:55 smithi006 bash[27292]: cluster 2023-08-28T00:26:54.581258+0000 mgr.y (mgr.14901) 4450 : cluster [DBG] pgmap v4457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:56.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:55 smithi006 bash[27292]: audit 2023-08-28T00:26:55.754579+0000 mon.a (mon.0) 3251 : audit [DBG] from='client.? 172.21.15.6:0/3186867669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:56.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:55 smithi006 bash[33918]: cluster 2023-08-28T00:26:54.581258+0000 mgr.y (mgr.14901) 4450 : cluster [DBG] pgmap v4457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:56.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:55 smithi006 bash[33918]: audit 2023-08-28T00:26:55.754579+0000 mon.a (mon.0) 3251 : audit [DBG] from='client.? 172.21.15.6:0/3186867669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:56.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:56 smithi130 bash[51302]: debug 2023-08-28T00:26:56.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:56.516769+0000) 2023-08-28T00:26:57.808 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:57 smithi130 bash[51302]: debug 2023-08-28T00:26:57.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:57.517012+0000) 2023-08-28T00:26:58.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:57 smithi130 bash[27579]: cluster 2023-08-28T00:26:56.581911+0000 mgr.y (mgr.14901) 4451 : cluster [DBG] pgmap v4458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:58.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:57 smithi006 bash[27292]: cluster 2023-08-28T00:26:56.581911+0000 mgr.y (mgr.14901) 4451 : cluster [DBG] pgmap v4458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:58.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:57 smithi006 bash[33918]: cluster 2023-08-28T00:26:56.581911+0000 mgr.y (mgr.14901) 4451 : cluster [DBG] pgmap v4458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:26:58.816 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:58 smithi130 bash[51302]: debug 2023-08-28T00:26:58.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:58.517272+0000) 2023-08-28T00:26:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:58 smithi130 bash[27579]: audit 2023-08-28T00:26:58.176654+0000 mon.c (mon.2) 1712 : audit [DBG] from='client.? 172.21.15.6:0/1603396707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:58 smithi006 bash[33918]: audit 2023-08-28T00:26:58.176654+0000 mon.c (mon.2) 1712 : audit [DBG] from='client.? 172.21.15.6:0/1603396707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:58 smithi006 bash[27292]: audit 2023-08-28T00:26:58.176654+0000 mon.c (mon.2) 1712 : audit [DBG] from='client.? 172.21.15.6:0/1603396707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:59.820 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:26:59 smithi130 bash[51302]: debug 2023-08-28T00:26:59.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:59.517585+0000) 2023-08-28T00:27:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:26:59 smithi130 bash[27579]: cluster 2023-08-28T00:26:58.582597+0000 mgr.y (mgr.14901) 4452 : cluster [DBG] pgmap v4459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:00.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:26:59 smithi006 bash[27292]: cluster 2023-08-28T00:26:58.582597+0000 mgr.y (mgr.14901) 4452 : cluster [DBG] pgmap v4459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:00.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:26:59 smithi006 bash[33918]: cluster 2023-08-28T00:26:58.582597+0000 mgr.y (mgr.14901) 4452 : cluster [DBG] pgmap v4459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:00.827 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:00 smithi130 bash[51302]: debug 2023-08-28T00:27:00.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:00.517879+0000) 2023-08-28T00:27:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:00 smithi130 bash[27579]: audit 2023-08-28T00:27:00.596446+0000 mon.a (mon.0) 3252 : audit [DBG] from='client.? 172.21.15.6:0/2863737910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:27:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:27:00] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:27:01.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:00 smithi006 bash[27292]: audit 2023-08-28T00:27:00.596446+0000 mon.a (mon.0) 3252 : audit [DBG] from='client.? 172.21.15.6:0/2863737910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:01.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:00 smithi006 bash[33918]: audit 2023-08-28T00:27:00.596446+0000 mon.a (mon.0) 3252 : audit [DBG] from='client.? 172.21.15.6:0/2863737910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:01.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:27:00.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:01.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:27:00.741Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:01.832 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:01 smithi130 bash[51302]: debug 2023-08-28T00:27:01.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:01.518155+0000) 2023-08-28T00:27:02.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:01 smithi130 bash[27579]: cluster 2023-08-28T00:27:00.583694+0000 mgr.y (mgr.14901) 4453 : cluster [DBG] pgmap v4460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:02.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:01 smithi006 bash[33918]: cluster 2023-08-28T00:27:00.583694+0000 mgr.y (mgr.14901) 4453 : cluster [DBG] pgmap v4460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:02.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:01 smithi006 bash[27292]: cluster 2023-08-28T00:27:00.583694+0000 mgr.y (mgr.14901) 4453 : cluster [DBG] pgmap v4460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:27:02.316Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:27:02.318Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:02.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:02 smithi130 bash[51302]: debug 2023-08-28T00:27:02.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:02.518430+0000) 2023-08-28T00:27:03.837 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:03 smithi130 bash[51302]: debug 2023-08-28T00:27:03.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:03.518752+0000) 2023-08-28T00:27:04.148 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:03 smithi130 bash[27579]: cluster 2023-08-28T00:27:02.584505+0000 mgr.y (mgr.14901) 4454 : cluster [DBG] pgmap v4461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:04.148 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:03 smithi130 bash[27579]: audit 2023-08-28T00:27:03.019288+0000 mon.c (mon.2) 1713 : audit [DBG] from='client.? 172.21.15.6:0/693580905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:04.203 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:03 smithi006 bash[27292]: cluster 2023-08-28T00:27:02.584505+0000 mgr.y (mgr.14901) 4454 : cluster [DBG] pgmap v4461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:04.203 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:03 smithi006 bash[27292]: audit 2023-08-28T00:27:03.019288+0000 mon.c (mon.2) 1713 : audit [DBG] from='client.? 172.21.15.6:0/693580905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:04.204 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:03 smithi006 bash[33918]: cluster 2023-08-28T00:27:02.584505+0000 mgr.y (mgr.14901) 4454 : cluster [DBG] pgmap v4461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:04.204 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:03 smithi006 bash[33918]: audit 2023-08-28T00:27:03.019288+0000 mon.c (mon.2) 1713 : audit [DBG] from='client.? 172.21.15.6:0/693580905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:27:04.200Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:27:04.202Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:04.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:04 smithi130 bash[51302]: debug 2023-08-28T00:27:04.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:04.519052+0000) 2023-08-28T00:27:05.841 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:05 smithi130 bash[51302]: debug 2023-08-28T00:27:05.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:05.519344+0000) 2023-08-28T00:27:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:05 smithi130 bash[27579]: cluster 2023-08-28T00:27:04.585634+0000 mgr.y (mgr.14901) 4455 : cluster [DBG] pgmap v4462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:05 smithi130 bash[27579]: audit 2023-08-28T00:27:05.432120+0000 mon.c (mon.2) 1714 : audit [DBG] from='client.? 172.21.15.6:0/2011214855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:05 smithi006 bash[33918]: cluster 2023-08-28T00:27:04.585634+0000 mgr.y (mgr.14901) 4455 : cluster [DBG] pgmap v4462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:05 smithi006 bash[33918]: audit 2023-08-28T00:27:05.432120+0000 mon.c (mon.2) 1714 : audit [DBG] from='client.? 172.21.15.6:0/2011214855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:06.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:05 smithi006 bash[27292]: cluster 2023-08-28T00:27:04.585634+0000 mgr.y (mgr.14901) 4455 : cluster [DBG] pgmap v4462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:06.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:05 smithi006 bash[27292]: audit 2023-08-28T00:27:05.432120+0000 mon.c (mon.2) 1714 : audit [DBG] from='client.? 172.21.15.6:0/2011214855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:06.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:06 smithi130 bash[51302]: debug 2023-08-28T00:27:06.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:06.519641+0000) 2023-08-28T00:27:07.845 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:07 smithi130 bash[51302]: debug 2023-08-28T00:27:07.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:07.519935+0000) 2023-08-28T00:27:08.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:07 smithi130 bash[27579]: cluster 2023-08-28T00:27:06.586269+0000 mgr.y (mgr.14901) 4456 : cluster [DBG] pgmap v4463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:08.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:07 smithi006 bash[27292]: cluster 2023-08-28T00:27:06.586269+0000 mgr.y (mgr.14901) 4456 : cluster [DBG] pgmap v4463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:08.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:07 smithi006 bash[33918]: cluster 2023-08-28T00:27:06.586269+0000 mgr.y (mgr.14901) 4456 : cluster [DBG] pgmap v4463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:08.848 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:08 smithi130 bash[51302]: debug 2023-08-28T00:27:08.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:08.520211+0000) 2023-08-28T00:27:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:08 smithi130 bash[27579]: audit 2023-08-28T00:27:07.855135+0000 mon.a (mon.0) 3253 : audit [DBG] from='client.? 172.21.15.6:0/1180124071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:08 smithi006 bash[33918]: audit 2023-08-28T00:27:07.855135+0000 mon.a (mon.0) 3253 : audit [DBG] from='client.? 172.21.15.6:0/1180124071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:08 smithi006 bash[27292]: audit 2023-08-28T00:27:07.855135+0000 mon.a (mon.0) 3253 : audit [DBG] from='client.? 172.21.15.6:0/1180124071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:09.857 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:09 smithi130 bash[51302]: debug 2023-08-28T00:27:09.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:09.520504+0000) 2023-08-28T00:27:10.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:09 smithi130 bash[27579]: cluster 2023-08-28T00:27:08.586924+0000 mgr.y (mgr.14901) 4457 : cluster [DBG] pgmap v4464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:10.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:09 smithi006 bash[33918]: cluster 2023-08-28T00:27:08.586924+0000 mgr.y (mgr.14901) 4457 : cluster [DBG] pgmap v4464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:10.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:09 smithi006 bash[27292]: cluster 2023-08-28T00:27:08.586924+0000 mgr.y (mgr.14901) 4457 : cluster [DBG] pgmap v4464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:10.864 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:10 smithi130 bash[51302]: debug 2023-08-28T00:27:10.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:10.520807+0000) 2023-08-28T00:27:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:10 smithi130 bash[27579]: audit 2023-08-28T00:27:10.270982+0000 mon.c (mon.2) 1715 : audit [DBG] from='client.? 172.21.15.6:0/3684104838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:27:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:27:10] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:27:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:10 smithi006 bash[33918]: audit 2023-08-28T00:27:10.270982+0000 mon.c (mon.2) 1715 : audit [DBG] from='client.? 172.21.15.6:0/3684104838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:11.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:10 smithi006 bash[27292]: audit 2023-08-28T00:27:10.270982+0000 mon.c (mon.2) 1715 : audit [DBG] from='client.? 172.21.15.6:0/3684104838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:27:10.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:27:10.741Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:11.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:11 smithi130 bash[51302]: debug 2023-08-28T00:27:11.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:11.521083+0000) 2023-08-28T00:27:12.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:11 smithi130 bash[27579]: cluster 2023-08-28T00:27:10.588035+0000 mgr.y (mgr.14901) 4458 : cluster [DBG] pgmap v4465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:12.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:11 smithi006 bash[33918]: cluster 2023-08-28T00:27:10.588035+0000 mgr.y (mgr.14901) 4458 : cluster [DBG] pgmap v4465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:12.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:11 smithi006 bash[27292]: cluster 2023-08-28T00:27:10.588035+0000 mgr.y (mgr.14901) 4458 : cluster [DBG] pgmap v4465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:27:12.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:27:12.319Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:12 smithi130 bash[51302]: debug 2023-08-28T00:27:12.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:12.521364+0000) 2023-08-28T00:27:13.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:12 smithi006 bash[27292]: audit 2023-08-28T00:27:12.685813+0000 mon.c (mon.2) 1716 : audit [DBG] from='client.? 172.21.15.6:0/2515377951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:13.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:12 smithi006 bash[33918]: audit 2023-08-28T00:27:12.685813+0000 mon.c (mon.2) 1716 : audit [DBG] from='client.? 172.21.15.6:0/2515377951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:12 smithi130 bash[27579]: audit 2023-08-28T00:27:12.685813+0000 mon.c (mon.2) 1716 : audit [DBG] from='client.? 172.21.15.6:0/2515377951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:13.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:13 smithi130 bash[51302]: debug 2023-08-28T00:27:13.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:13.521634+0000) 2023-08-28T00:27:14.204 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:13 smithi006 bash[33918]: cluster 2023-08-28T00:27:12.588652+0000 mgr.y (mgr.14901) 4459 : cluster [DBG] pgmap v4466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:14.205 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:13 smithi006 bash[27292]: cluster 2023-08-28T00:27:12.588652+0000 mgr.y (mgr.14901) 4459 : cluster [DBG] pgmap v4466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:14.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:13 smithi130 bash[27579]: cluster 2023-08-28T00:27:12.588652+0000 mgr.y (mgr.14901) 4459 : cluster [DBG] pgmap v4466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:27:14.201Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:14.474 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:27:14.204Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:14 smithi130 bash[51302]: debug 2023-08-28T00:27:14.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:14.521911+0000) 2023-08-28T00:27:15.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:15 smithi130 bash[51302]: debug 2023-08-28T00:27:15.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:15.522164+0000) 2023-08-28T00:27:16.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:15 smithi006 bash[33918]: cluster 2023-08-28T00:27:14.589780+0000 mgr.y (mgr.14901) 4460 : cluster [DBG] pgmap v4467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:16.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:15 smithi006 bash[33918]: audit 2023-08-28T00:27:15.097955+0000 mon.c (mon.2) 1717 : audit [DBG] from='client.? 172.21.15.6:0/2390690123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:16.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:15 smithi006 bash[27292]: cluster 2023-08-28T00:27:14.589780+0000 mgr.y (mgr.14901) 4460 : cluster [DBG] pgmap v4467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:16.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:15 smithi006 bash[27292]: audit 2023-08-28T00:27:15.097955+0000 mon.c (mon.2) 1717 : audit [DBG] from='client.? 172.21.15.6:0/2390690123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:15 smithi130 bash[27579]: cluster 2023-08-28T00:27:14.589780+0000 mgr.y (mgr.14901) 4460 : cluster [DBG] pgmap v4467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:15 smithi130 bash[27579]: audit 2023-08-28T00:27:15.097955+0000 mon.c (mon.2) 1717 : audit [DBG] from='client.? 172.21.15.6:0/2390690123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:16.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:16 smithi130 bash[51302]: debug 2023-08-28T00:27:16.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:16.522483+0000) 2023-08-28T00:27:17.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:17 smithi130 bash[51302]: debug 2023-08-28T00:27:17.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:17.522758+0000) 2023-08-28T00:27:18.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:17 smithi006 bash[33918]: cluster 2023-08-28T00:27:16.590458+0000 mgr.y (mgr.14901) 4461 : cluster [DBG] pgmap v4468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:18.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:17 smithi006 bash[33918]: audit 2023-08-28T00:27:17.513056+0000 mon.c (mon.2) 1718 : audit [DBG] from='client.? 172.21.15.6:0/2765061992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:18.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:17 smithi006 bash[27292]: cluster 2023-08-28T00:27:16.590458+0000 mgr.y (mgr.14901) 4461 : cluster [DBG] pgmap v4468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:18.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:17 smithi006 bash[27292]: audit 2023-08-28T00:27:17.513056+0000 mon.c (mon.2) 1718 : audit [DBG] from='client.? 172.21.15.6:0/2765061992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:17 smithi130 bash[27579]: cluster 2023-08-28T00:27:16.590458+0000 mgr.y (mgr.14901) 4461 : cluster [DBG] pgmap v4468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:17 smithi130 bash[27579]: audit 2023-08-28T00:27:17.513056+0000 mon.c (mon.2) 1718 : audit [DBG] from='client.? 172.21.15.6:0/2765061992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:18 smithi130 bash[51302]: debug 2023-08-28T00:27:18.521+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:18.523067+0000) 2023-08-28T00:27:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:19 smithi130 bash[51302]: debug 2023-08-28T00:27:19.521+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:19.523261+0000) 2023-08-28T00:27:20.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:19 smithi006 bash[33918]: cluster 2023-08-28T00:27:18.591117+0000 mgr.y (mgr.14901) 4462 : cluster [DBG] pgmap v4469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:20.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:19 smithi006 bash[27292]: cluster 2023-08-28T00:27:18.591117+0000 mgr.y (mgr.14901) 4462 : cluster [DBG] pgmap v4469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:19 smithi130 bash[27579]: cluster 2023-08-28T00:27:18.591117+0000 mgr.y (mgr.14901) 4462 : cluster [DBG] pgmap v4469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:20 smithi130 bash[51302]: debug 2023-08-28T00:27:20.521+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:20.523537+0000) 2023-08-28T00:27:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:27:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:27:20] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T00:27:21.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:20 smithi006 bash[33918]: audit 2023-08-28T00:27:19.922845+0000 mon.c (mon.2) 1719 : audit [DBG] from='client.? 172.21.15.6:0/1550745252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:21.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:20 smithi006 bash[27292]: audit 2023-08-28T00:27:19.922845+0000 mon.c (mon.2) 1719 : audit [DBG] from='client.? 172.21.15.6:0/1550745252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:27:20.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:27:20.742Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:21.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:20 smithi130 bash[27579]: audit 2023-08-28T00:27:19.922845+0000 mon.c (mon.2) 1719 : audit [DBG] from='client.? 172.21.15.6:0/1550745252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:21.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:21 smithi130 bash[51302]: debug 2023-08-28T00:27:21.521+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:21.523827+0000) 2023-08-28T00:27:22.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:21 smithi006 bash[27292]: cluster 2023-08-28T00:27:20.592211+0000 mgr.y (mgr.14901) 4463 : cluster [DBG] pgmap v4470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:22.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:21 smithi006 bash[33918]: cluster 2023-08-28T00:27:20.592211+0000 mgr.y (mgr.14901) 4463 : cluster [DBG] pgmap v4470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:22.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:21 smithi130 bash[27579]: cluster 2023-08-28T00:27:20.592211+0000 mgr.y (mgr.14901) 4463 : cluster [DBG] pgmap v4470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:27:22.318Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:27:22.320Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:22 smithi130 bash[51302]: debug 2023-08-28T00:27:22.521+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:22.524094+0000) 2023-08-28T00:27:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:22 smithi006 bash[33918]: audit 2023-08-28T00:27:22.337844+0000 mon.c (mon.2) 1720 : audit [DBG] from='client.? 172.21.15.6:0/2778432107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:23.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:22 smithi006 bash[27292]: audit 2023-08-28T00:27:22.337844+0000 mon.c (mon.2) 1720 : audit [DBG] from='client.? 172.21.15.6:0/2778432107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:23.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:22 smithi130 bash[27579]: audit 2023-08-28T00:27:22.337844+0000 mon.c (mon.2) 1720 : audit [DBG] from='client.? 172.21.15.6:0/2778432107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:23.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:23 smithi130 bash[51302]: debug 2023-08-28T00:27:23.521+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:23.524395+0000) 2023-08-28T00:27:24.205 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:23 smithi006 bash[33918]: cluster 2023-08-28T00:27:22.593031+0000 mgr.y (mgr.14901) 4464 : cluster [DBG] pgmap v4471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:24.206 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:23 smithi006 bash[27292]: cluster 2023-08-28T00:27:22.593031+0000 mgr.y (mgr.14901) 4464 : cluster [DBG] pgmap v4471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:23 smithi130 bash[27579]: cluster 2023-08-28T00:27:22.593031+0000 mgr.y (mgr.14901) 4464 : cluster [DBG] pgmap v4471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:27:24.202Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:27:24.204Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:24 smithi130 bash[51302]: debug 2023-08-28T00:27:24.521+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:24.524666+0000) 2023-08-28T00:27:25.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:24 smithi006 bash[27292]: audit 2023-08-28T00:27:24.749125+0000 mon.a (mon.0) 3254 : audit [DBG] from='client.? 172.21.15.6:0/917126420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:24 smithi006 bash[33918]: audit 2023-08-28T00:27:24.749125+0000 mon.a (mon.0) 3254 : audit [DBG] from='client.? 172.21.15.6:0/917126420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:24 smithi130 bash[27579]: audit 2023-08-28T00:27:24.749125+0000 mon.a (mon.0) 3254 : audit [DBG] from='client.? 172.21.15.6:0/917126420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:25.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:25 smithi130 bash[51302]: debug 2023-08-28T00:27:25.521+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:25.524920+0000) 2023-08-28T00:27:26.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:25 smithi006 bash[33918]: cluster 2023-08-28T00:27:24.594239+0000 mgr.y (mgr.14901) 4465 : cluster [DBG] pgmap v4472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:25 smithi006 bash[27292]: cluster 2023-08-28T00:27:24.594239+0000 mgr.y (mgr.14901) 4465 : cluster [DBG] pgmap v4472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:25 smithi130 bash[27579]: cluster 2023-08-28T00:27:24.594239+0000 mgr.y (mgr.14901) 4465 : cluster [DBG] pgmap v4472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:26.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:26 smithi130 bash[51302]: debug 2023-08-28T00:27:26.521+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:26.525269+0000) 2023-08-28T00:27:27.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:27 smithi130 bash[51302]: debug 2023-08-28T00:27:27.521+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:27.525540+0000) 2023-08-28T00:27:28.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:27 smithi006 bash[27292]: cluster 2023-08-28T00:27:26.594942+0000 mgr.y (mgr.14901) 4466 : cluster [DBG] pgmap v4473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:28.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:27 smithi006 bash[27292]: audit 2023-08-28T00:27:27.162308+0000 mon.a (mon.0) 3255 : audit [DBG] from='client.? 172.21.15.6:0/2796166045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:28.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:27 smithi006 bash[33918]: cluster 2023-08-28T00:27:26.594942+0000 mgr.y (mgr.14901) 4466 : cluster [DBG] pgmap v4473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:28.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:27 smithi006 bash[33918]: audit 2023-08-28T00:27:27.162308+0000 mon.a (mon.0) 3255 : audit [DBG] from='client.? 172.21.15.6:0/2796166045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:28.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:27 smithi130 bash[27579]: cluster 2023-08-28T00:27:26.594942+0000 mgr.y (mgr.14901) 4466 : cluster [DBG] pgmap v4473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:28.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:27 smithi130 bash[27579]: audit 2023-08-28T00:27:27.162308+0000 mon.a (mon.0) 3255 : audit [DBG] from='client.? 172.21.15.6:0/2796166045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:28 smithi130 bash[51302]: debug 2023-08-28T00:27:28.525+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:28.525802+0000) 2023-08-28T00:27:29.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:29 smithi130 bash[51302]: debug 2023-08-28T00:27:29.525+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:29.526092+0000) 2023-08-28T00:27:30.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:29 smithi006 bash[33918]: cluster 2023-08-28T00:27:28.595408+0000 mgr.y (mgr.14901) 4467 : cluster [DBG] pgmap v4474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:30.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:29 smithi006 bash[33918]: audit 2023-08-28T00:27:29.580212+0000 mon.c (mon.2) 1721 : audit [DBG] from='client.? 172.21.15.6:0/209192672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:30.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:29 smithi006 bash[27292]: cluster 2023-08-28T00:27:28.595408+0000 mgr.y (mgr.14901) 4467 : cluster [DBG] pgmap v4474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:30.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:29 smithi006 bash[27292]: audit 2023-08-28T00:27:29.580212+0000 mon.c (mon.2) 1721 : audit [DBG] from='client.? 172.21.15.6:0/209192672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:29 smithi130 bash[27579]: cluster 2023-08-28T00:27:28.595408+0000 mgr.y (mgr.14901) 4467 : cluster [DBG] pgmap v4474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:30.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:29 smithi130 bash[27579]: audit 2023-08-28T00:27:29.580212+0000 mon.c (mon.2) 1721 : audit [DBG] from='client.? 172.21.15.6:0/209192672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:30 smithi130 bash[51302]: debug 2023-08-28T00:27:30.521+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:30.526367+0000) 2023-08-28T00:27:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:27:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:27:30] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:27:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:27:30.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:27:30.742Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:31.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:31 smithi130 bash[51302]: debug 2023-08-28T00:27:31.521+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:31.526668+0000) 2023-08-28T00:27:32.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:31 smithi006 bash[33918]: cluster 2023-08-28T00:27:30.596621+0000 mgr.y (mgr.14901) 4468 : cluster [DBG] pgmap v4475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:32.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:31 smithi006 bash[27292]: cluster 2023-08-28T00:27:30.596621+0000 mgr.y (mgr.14901) 4468 : cluster [DBG] pgmap v4475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:31 smithi130 bash[27579]: cluster 2023-08-28T00:27:30.596621+0000 mgr.y (mgr.14901) 4468 : cluster [DBG] pgmap v4475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:27:32.318Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:27:32.320Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:32.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:32 smithi130 bash[51302]: debug 2023-08-28T00:27:32.525+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:32.526911+0000) 2023-08-28T00:27:33.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:32 smithi006 bash[33918]: audit 2023-08-28T00:27:32.001687+0000 mon.c (mon.2) 1722 : audit [DBG] from='client.? 172.21.15.6:0/1484100266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:32 smithi006 bash[27292]: audit 2023-08-28T00:27:32.001687+0000 mon.c (mon.2) 1722 : audit [DBG] from='client.? 172.21.15.6:0/1484100266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:32 smithi130 bash[27579]: audit 2023-08-28T00:27:32.001687+0000 mon.c (mon.2) 1722 : audit [DBG] from='client.? 172.21.15.6:0/1484100266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:33.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:33 smithi130 bash[51302]: debug 2023-08-28T00:27:33.525+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:33.527121+0000) 2023-08-28T00:27:34.208 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:27:34.203Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:34.208 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:33 smithi006 bash[33918]: cluster 2023-08-28T00:27:32.597357+0000 mgr.y (mgr.14901) 4469 : cluster [DBG] pgmap v4476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:34.208 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:33 smithi006 bash[27292]: cluster 2023-08-28T00:27:32.597357+0000 mgr.y (mgr.14901) 4469 : cluster [DBG] pgmap v4476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:34.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:33 smithi130 bash[27579]: cluster 2023-08-28T00:27:32.597357+0000 mgr.y (mgr.14901) 4469 : cluster [DBG] pgmap v4476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:27:34.205Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:34 smithi130 bash[51302]: debug 2023-08-28T00:27:34.525+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:34.527356+0000) 2023-08-28T00:27:35.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:34 smithi006 bash[33918]: audit 2023-08-28T00:27:34.425865+0000 mon.c (mon.2) 1723 : audit [DBG] from='client.? 172.21.15.6:0/1520981420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:34 smithi006 bash[27292]: audit 2023-08-28T00:27:34.425865+0000 mon.c (mon.2) 1723 : audit [DBG] from='client.? 172.21.15.6:0/1520981420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:34 smithi130 bash[27579]: audit 2023-08-28T00:27:34.425865+0000 mon.c (mon.2) 1723 : audit [DBG] from='client.? 172.21.15.6:0/1520981420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:35.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:35 smithi130 bash[51302]: debug 2023-08-28T00:27:35.525+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:35.527620+0000) 2023-08-28T00:27:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:35 smithi130 bash[27579]: cluster 2023-08-28T00:27:34.598491+0000 mgr.y (mgr.14901) 4470 : cluster [DBG] pgmap v4477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:36.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:35 smithi006 bash[33918]: cluster 2023-08-28T00:27:34.598491+0000 mgr.y (mgr.14901) 4470 : cluster [DBG] pgmap v4477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:36.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:35 smithi006 bash[27292]: cluster 2023-08-28T00:27:34.598491+0000 mgr.y (mgr.14901) 4470 : cluster [DBG] pgmap v4477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:36 smithi130 bash[51302]: debug 2023-08-28T00:27:36.525+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:36.527871+0000) 2023-08-28T00:27:37.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:36 smithi130 bash[27579]: audit 2023-08-28T00:27:36.837431+0000 mon.c (mon.2) 1724 : audit [DBG] from='client.? 172.21.15.6:0/3805587795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:37.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:36 smithi006 bash[33918]: audit 2023-08-28T00:27:36.837431+0000 mon.c (mon.2) 1724 : audit [DBG] from='client.? 172.21.15.6:0/3805587795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:37.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:36 smithi006 bash[27292]: audit 2023-08-28T00:27:36.837431+0000 mon.c (mon.2) 1724 : audit [DBG] from='client.? 172.21.15.6:0/3805587795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:37 smithi130 bash[51302]: debug 2023-08-28T00:27:37.525+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:37.528150+0000) 2023-08-28T00:27:38.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:37 smithi130 bash[27579]: cluster 2023-08-28T00:27:36.599114+0000 mgr.y (mgr.14901) 4471 : cluster [DBG] pgmap v4478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:38.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:37 smithi006 bash[27292]: cluster 2023-08-28T00:27:36.599114+0000 mgr.y (mgr.14901) 4471 : cluster [DBG] pgmap v4478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:38.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:37 smithi006 bash[33918]: cluster 2023-08-28T00:27:36.599114+0000 mgr.y (mgr.14901) 4471 : cluster [DBG] pgmap v4478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:38 smithi130 bash[51302]: debug 2023-08-28T00:27:38.525+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:38.528471+0000) 2023-08-28T00:27:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:39 smithi130 bash[51302]: debug 2023-08-28T00:27:39.524+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:39.528763+0000) 2023-08-28T00:27:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:39 smithi130 bash[27579]: cluster 2023-08-28T00:27:38.599780+0000 mgr.y (mgr.14901) 4472 : cluster [DBG] pgmap v4479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:39 smithi130 bash[27579]: audit 2023-08-28T00:27:39.254573+0000 mon.c (mon.2) 1725 : audit [DBG] from='client.? 172.21.15.6:0/1668383137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:40.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:39 smithi006 bash[27292]: cluster 2023-08-28T00:27:38.599780+0000 mgr.y (mgr.14901) 4472 : cluster [DBG] pgmap v4479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:40.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:39 smithi006 bash[27292]: audit 2023-08-28T00:27:39.254573+0000 mon.c (mon.2) 1725 : audit [DBG] from='client.? 172.21.15.6:0/1668383137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:39 smithi006 bash[33918]: cluster 2023-08-28T00:27:38.599780+0000 mgr.y (mgr.14901) 4472 : cluster [DBG] pgmap v4479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:39 smithi006 bash[33918]: audit 2023-08-28T00:27:39.254573+0000 mon.c (mon.2) 1725 : audit [DBG] from='client.? 172.21.15.6:0/1668383137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:40 smithi130 bash[51302]: debug 2023-08-28T00:27:40.524+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:40.529088+0000) 2023-08-28T00:27:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:27:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:27:40] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:27:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:27:40.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:27:40.743Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:41.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:41 smithi130 bash[51302]: debug 2023-08-28T00:27:41.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:41.529440+0000) 2023-08-28T00:27:42.322 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:41 smithi006 bash[33918]: cluster 2023-08-28T00:27:40.600884+0000 mgr.y (mgr.14901) 4473 : cluster [DBG] pgmap v4480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:42.322 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:41 smithi006 bash[33918]: audit 2023-08-28T00:27:41.663545+0000 mon.c (mon.2) 1726 : audit [DBG] from='client.? 172.21.15.6:0/3979067727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:42.323 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:41 smithi006 bash[27292]: cluster 2023-08-28T00:27:40.600884+0000 mgr.y (mgr.14901) 4473 : cluster [DBG] pgmap v4480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:42.323 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:41 smithi006 bash[27292]: audit 2023-08-28T00:27:41.663545+0000 mon.c (mon.2) 1726 : audit [DBG] from='client.? 172.21.15.6:0/3979067727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:41 smithi130 bash[27579]: cluster 2023-08-28T00:27:40.600884+0000 mgr.y (mgr.14901) 4473 : cluster [DBG] pgmap v4480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:41 smithi130 bash[27579]: audit 2023-08-28T00:27:41.663545+0000 mon.c (mon.2) 1726 : audit [DBG] from='client.? 172.21.15.6:0/3979067727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:27:42.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:27:42.321Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:42 smithi130 bash[51302]: debug 2023-08-28T00:27:42.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:42.529728+0000) 2023-08-28T00:27:43.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:43 smithi130 bash[51302]: debug 2023-08-28T00:27:43.524+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:43.529997+0000) 2023-08-28T00:27:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:43 smithi130 bash[27579]: cluster 2023-08-28T00:27:42.601492+0000 mgr.y (mgr.14901) 4474 : cluster [DBG] pgmap v4481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:44.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:43 smithi006 bash[27292]: cluster 2023-08-28T00:27:42.601492+0000 mgr.y (mgr.14901) 4474 : cluster [DBG] pgmap v4481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:44.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:43 smithi006 bash[33918]: cluster 2023-08-28T00:27:42.601492+0000 mgr.y (mgr.14901) 4474 : cluster [DBG] pgmap v4481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:27:44.204Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:27:44.206Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:44 smithi130 bash[51302]: debug 2023-08-28T00:27:44.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:44.530278+0000) 2023-08-28T00:27:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:45 smithi130 bash[27579]: audit 2023-08-28T00:27:44.076409+0000 mon.c (mon.2) 1727 : audit [DBG] from='client.? 172.21.15.6:0/3464085866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:45.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:45 smithi006 bash[27292]: audit 2023-08-28T00:27:44.076409+0000 mon.c (mon.2) 1727 : audit [DBG] from='client.? 172.21.15.6:0/3464085866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:45.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:45 smithi006 bash[33918]: audit 2023-08-28T00:27:44.076409+0000 mon.c (mon.2) 1727 : audit [DBG] from='client.? 172.21.15.6:0/3464085866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:45.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:45 smithi130 bash[51302]: debug 2023-08-28T00:27:45.524+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:45.530570+0000) 2023-08-28T00:27:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:46 smithi130 bash[27579]: cluster 2023-08-28T00:27:44.602589+0000 mgr.y (mgr.14901) 4475 : cluster [DBG] pgmap v4482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:46.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:46 smithi006 bash[27292]: cluster 2023-08-28T00:27:44.602589+0000 mgr.y (mgr.14901) 4475 : cluster [DBG] pgmap v4482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:46 smithi006 bash[33918]: cluster 2023-08-28T00:27:44.602589+0000 mgr.y (mgr.14901) 4475 : cluster [DBG] pgmap v4482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:46.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:46 smithi130 bash[51302]: debug 2023-08-28T00:27:46.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:46.530838+0000) 2023-08-28T00:27:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:47 smithi130 bash[27579]: audit 2023-08-28T00:27:46.361957+0000 mon.b (mon.1) 550 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:27:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:47 smithi130 bash[27579]: audit 2023-08-28T00:27:46.495915+0000 mon.a (mon.0) 3256 : audit [DBG] from='client.? 172.21.15.6:0/3013755230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:47.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:47 smithi006 bash[27292]: audit 2023-08-28T00:27:46.361957+0000 mon.b (mon.1) 550 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:27:47.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:47 smithi006 bash[27292]: audit 2023-08-28T00:27:46.495915+0000 mon.a (mon.0) 3256 : audit [DBG] from='client.? 172.21.15.6:0/3013755230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:47.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:47 smithi006 bash[33918]: audit 2023-08-28T00:27:46.361957+0000 mon.b (mon.1) 550 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:27:47.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:47 smithi006 bash[33918]: audit 2023-08-28T00:27:46.495915+0000 mon.a (mon.0) 3256 : audit [DBG] from='client.? 172.21.15.6:0/3013755230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:47.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:47 smithi130 bash[51302]: debug 2023-08-28T00:27:47.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:47.531098+0000) 2023-08-28T00:27:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:48 smithi130 bash[27579]: cluster 2023-08-28T00:27:46.603284+0000 mgr.y (mgr.14901) 4476 : cluster [DBG] pgmap v4483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:48.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:48 smithi006 bash[33918]: cluster 2023-08-28T00:27:46.603284+0000 mgr.y (mgr.14901) 4476 : cluster [DBG] pgmap v4483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:48.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:48 smithi006 bash[27292]: cluster 2023-08-28T00:27:46.603284+0000 mgr.y (mgr.14901) 4476 : cluster [DBG] pgmap v4483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:48 smithi130 bash[51302]: debug 2023-08-28T00:27:48.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:48.531377+0000) 2023-08-28T00:27:49.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:49 smithi130 bash[27579]: audit 2023-08-28T00:27:48.917089+0000 mon.c (mon.2) 1728 : audit [DBG] from='client.? 172.21.15.6:0/751423975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:49.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:49 smithi006 bash[27292]: audit 2023-08-28T00:27:48.917089+0000 mon.c (mon.2) 1728 : audit [DBG] from='client.? 172.21.15.6:0/751423975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:49.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:49 smithi006 bash[33918]: audit 2023-08-28T00:27:48.917089+0000 mon.c (mon.2) 1728 : audit [DBG] from='client.? 172.21.15.6:0/751423975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:49.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:49 smithi130 bash[51302]: debug 2023-08-28T00:27:49.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:49.531685+0000) 2023-08-28T00:27:50.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:50 smithi130 bash[27579]: cluster 2023-08-28T00:27:48.603917+0000 mgr.y (mgr.14901) 4477 : cluster [DBG] pgmap v4484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:50.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:50 smithi006 bash[27292]: cluster 2023-08-28T00:27:48.603917+0000 mgr.y (mgr.14901) 4477 : cluster [DBG] pgmap v4484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:50.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:50 smithi006 bash[33918]: cluster 2023-08-28T00:27:48.603917+0000 mgr.y (mgr.14901) 4477 : cluster [DBG] pgmap v4484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:50 smithi130 bash[51302]: debug 2023-08-28T00:27:50.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:50.531984+0000) 2023-08-28T00:27:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:27:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:27:50] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T00:27:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:27:50.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:27:50.744Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:51.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:51 smithi130 bash[51302]: debug 2023-08-28T00:27:51.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:51.532364+0000) 2023-08-28T00:27:52.323 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:52 smithi006 bash[27292]: cluster 2023-08-28T00:27:50.605240+0000 mgr.y (mgr.14901) 4478 : cluster [DBG] pgmap v4485: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:52.323 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:52 smithi006 bash[27292]: audit 2023-08-28T00:27:51.341005+0000 mon.c (mon.2) 1729 : audit [DBG] from='client.? 172.21.15.6:0/293286589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:52.324 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:52 smithi006 bash[33918]: cluster 2023-08-28T00:27:50.605240+0000 mgr.y (mgr.14901) 4478 : cluster [DBG] pgmap v4485: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:52.324 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:52 smithi006 bash[33918]: audit 2023-08-28T00:27:51.341005+0000 mon.c (mon.2) 1729 : audit [DBG] from='client.? 172.21.15.6:0/293286589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:52 smithi130 bash[27579]: cluster 2023-08-28T00:27:50.605240+0000 mgr.y (mgr.14901) 4478 : cluster [DBG] pgmap v4485: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:52 smithi130 bash[27579]: audit 2023-08-28T00:27:51.341005+0000 mon.c (mon.2) 1729 : audit [DBG] from='client.? 172.21.15.6:0/293286589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:27:52.320Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:27:52.321Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:52.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:52 smithi130 bash[51302]: debug 2023-08-28T00:27:52.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:52.532604+0000) 2023-08-28T00:27:53.534 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:53 smithi130 bash[27579]: audit 2023-08-28T00:27:52.245430+0000 mon.a (mon.0) 3257 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:27:53.534 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:53 smithi130 bash[27579]: audit 2023-08-28T00:27:52.626835+0000 mon.a (mon.0) 3258 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:27:53.534 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:53 smithi130 bash[27579]: audit 2023-08-28T00:27:52.894004+0000 mon.b (mon.1) 551 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:27:53.535 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:53 smithi130 bash[27579]: audit 2023-08-28T00:27:52.895943+0000 mon.b (mon.1) 552 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:27:53.535 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:53 smithi130 bash[27579]: audit 2023-08-28T00:27:52.906032+0000 mon.a (mon.0) 3259 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:27:53.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:53 smithi006 bash[33918]: audit 2023-08-28T00:27:52.245430+0000 mon.a (mon.0) 3257 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:27:53.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:53 smithi006 bash[33918]: audit 2023-08-28T00:27:52.626835+0000 mon.a (mon.0) 3258 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:27:53.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:53 smithi006 bash[33918]: audit 2023-08-28T00:27:52.894004+0000 mon.b (mon.1) 551 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:27:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:53 smithi006 bash[33918]: audit 2023-08-28T00:27:52.895943+0000 mon.b (mon.1) 552 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:27:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:53 smithi006 bash[33918]: audit 2023-08-28T00:27:52.906032+0000 mon.a (mon.0) 3259 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:27:53.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:53 smithi006 bash[27292]: audit 2023-08-28T00:27:52.245430+0000 mon.a (mon.0) 3257 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:27:53.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:53 smithi006 bash[27292]: audit 2023-08-28T00:27:52.626835+0000 mon.a (mon.0) 3258 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:27:53.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:53 smithi006 bash[27292]: audit 2023-08-28T00:27:52.894004+0000 mon.b (mon.1) 551 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:27:53.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:53 smithi006 bash[27292]: audit 2023-08-28T00:27:52.895943+0000 mon.b (mon.1) 552 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:27:53.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:53 smithi006 bash[27292]: audit 2023-08-28T00:27:52.906032+0000 mon.a (mon.0) 3259 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:27:53.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:53 smithi130 bash[51302]: debug 2023-08-28T00:27:53.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:53.532927+0000) 2023-08-28T00:27:54.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:54 smithi006 bash[33918]: cluster 2023-08-28T00:27:52.605962+0000 mgr.y (mgr.14901) 4479 : cluster [DBG] pgmap v4486: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:54.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:54 smithi006 bash[33918]: audit 2023-08-28T00:27:53.752897+0000 mon.a (mon.0) 3260 : audit [DBG] from='client.? 172.21.15.6:0/6658727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:27:54.204Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:27:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:27:54.206Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:27:54.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:54 smithi006 bash[27292]: cluster 2023-08-28T00:27:52.605962+0000 mgr.y (mgr.14901) 4479 : cluster [DBG] pgmap v4486: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:54.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:54 smithi006 bash[27292]: audit 2023-08-28T00:27:53.752897+0000 mon.a (mon.0) 3260 : audit [DBG] from='client.? 172.21.15.6:0/6658727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:54.535 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:54 smithi130 bash[27579]: cluster 2023-08-28T00:27:52.605962+0000 mgr.y (mgr.14901) 4479 : cluster [DBG] pgmap v4486: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:54.535 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:54 smithi130 bash[27579]: audit 2023-08-28T00:27:53.752897+0000 mon.a (mon.0) 3260 : audit [DBG] from='client.? 172.21.15.6:0/6658727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:54.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:54 smithi130 bash[51302]: debug 2023-08-28T00:27:54.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:54.533255+0000) 2023-08-28T00:27:55.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:55 smithi130 bash[51302]: debug 2023-08-28T00:27:55.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:55.533517+0000) 2023-08-28T00:27:56.535 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:56 smithi130 bash[27579]: cluster 2023-08-28T00:27:54.607096+0000 mgr.y (mgr.14901) 4480 : cluster [DBG] pgmap v4487: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:56.535 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:56 smithi130 bash[27579]: audit 2023-08-28T00:27:56.169654+0000 mon.c (mon.2) 1730 : audit [DBG] from='client.? 172.21.15.6:0/3303562385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:56 smithi006 bash[27292]: cluster 2023-08-28T00:27:54.607096+0000 mgr.y (mgr.14901) 4480 : cluster [DBG] pgmap v4487: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:56 smithi006 bash[27292]: audit 2023-08-28T00:27:56.169654+0000 mon.c (mon.2) 1730 : audit [DBG] from='client.? 172.21.15.6:0/3303562385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:56.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:56 smithi006 bash[33918]: cluster 2023-08-28T00:27:54.607096+0000 mgr.y (mgr.14901) 4480 : cluster [DBG] pgmap v4487: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:56.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:56 smithi006 bash[33918]: audit 2023-08-28T00:27:56.169654+0000 mon.c (mon.2) 1730 : audit [DBG] from='client.? 172.21.15.6:0/3303562385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:56.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:56 smithi130 bash[51302]: debug 2023-08-28T00:27:56.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:56.533793+0000) 2023-08-28T00:27:57.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:57 smithi130 bash[51302]: debug 2023-08-28T00:27:57.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:57.534045+0000) 2023-08-28T00:27:58.536 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:58 smithi130 bash[27579]: cluster 2023-08-28T00:27:56.607751+0000 mgr.y (mgr.14901) 4481 : cluster [DBG] pgmap v4488: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:58 smithi006 bash[33918]: cluster 2023-08-28T00:27:56.607751+0000 mgr.y (mgr.14901) 4481 : cluster [DBG] pgmap v4488: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:58 smithi006 bash[27292]: cluster 2023-08-28T00:27:56.607751+0000 mgr.y (mgr.14901) 4481 : cluster [DBG] pgmap v4488: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:27:58.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:58 smithi130 bash[51302]: debug 2023-08-28T00:27:58.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:58.534335+0000) 2023-08-28T00:27:59.537 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:27:59 smithi130 bash[27579]: audit 2023-08-28T00:27:58.592429+0000 mon.c (mon.2) 1731 : audit [DBG] from='client.? 172.21.15.6:0/4278751622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:59.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:27:59 smithi006 bash[33918]: audit 2023-08-28T00:27:58.592429+0000 mon.c (mon.2) 1731 : audit [DBG] from='client.? 172.21.15.6:0/4278751622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:59.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:27:59 smithi006 bash[27292]: audit 2023-08-28T00:27:58.592429+0000 mon.c (mon.2) 1731 : audit [DBG] from='client.? 172.21.15.6:0/4278751622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:59.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:27:59 smithi130 bash[51302]: debug 2023-08-28T00:27:59.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:59.534618+0000) 2023-08-28T00:28:00.536 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:00 smithi130 bash[27579]: cluster 2023-08-28T00:27:58.608304+0000 mgr.y (mgr.14901) 4482 : cluster [DBG] pgmap v4489: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:00 smithi006 bash[33918]: cluster 2023-08-28T00:27:58.608304+0000 mgr.y (mgr.14901) 4482 : cluster [DBG] pgmap v4489: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:00 smithi006 bash[27292]: cluster 2023-08-28T00:27:58.608304+0000 mgr.y (mgr.14901) 4482 : cluster [DBG] pgmap v4489: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:00.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:00 smithi130 bash[51302]: debug 2023-08-28T00:28:00.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:00.534914+0000) 2023-08-28T00:28:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:28:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:28:00] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:28:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:28:00.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:28:00.744Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:01.810 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:01 smithi130 bash[51302]: debug 2023-08-28T00:28:01.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:01.535189+0000) 2023-08-28T00:28:02.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:01 smithi130 bash[27579]: cluster 2023-08-28T00:28:00.609451+0000 mgr.y (mgr.14901) 4483 : cluster [DBG] pgmap v4490: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:02.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:01 smithi130 bash[27579]: audit 2023-08-28T00:28:01.009321+0000 mon.a (mon.0) 3261 : audit [DBG] from='client.? 172.21.15.6:0/2179073838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:02.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:01 smithi006 bash[33918]: cluster 2023-08-28T00:28:00.609451+0000 mgr.y (mgr.14901) 4483 : cluster [DBG] pgmap v4490: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:02.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:01 smithi006 bash[33918]: audit 2023-08-28T00:28:01.009321+0000 mon.a (mon.0) 3261 : audit [DBG] from='client.? 172.21.15.6:0/2179073838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:02.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:01 smithi006 bash[27292]: cluster 2023-08-28T00:28:00.609451+0000 mgr.y (mgr.14901) 4483 : cluster [DBG] pgmap v4490: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:02.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:01 smithi006 bash[27292]: audit 2023-08-28T00:28:01.009321+0000 mon.a (mon.0) 3261 : audit [DBG] from='client.? 172.21.15.6:0/2179073838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:28:02.321Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:28:02.323Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:02.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:02 smithi130 bash[51302]: debug 2023-08-28T00:28:02.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:02.535474+0000) 2023-08-28T00:28:03.815 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:03 smithi130 bash[51302]: debug 2023-08-28T00:28:03.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:03.535777+0000) 2023-08-28T00:28:04.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:03 smithi130 bash[27579]: cluster 2023-08-28T00:28:02.610120+0000 mgr.y (mgr.14901) 4484 : cluster [DBG] pgmap v4491: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:04.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:03 smithi130 bash[27579]: audit 2023-08-28T00:28:03.428282+0000 mon.c (mon.2) 1732 : audit [DBG] from='client.? 172.21.15.6:0/578968638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:04.208 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:03 smithi006 bash[33918]: cluster 2023-08-28T00:28:02.610120+0000 mgr.y (mgr.14901) 4484 : cluster [DBG] pgmap v4491: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:04.208 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:03 smithi006 bash[33918]: audit 2023-08-28T00:28:03.428282+0000 mon.c (mon.2) 1732 : audit [DBG] from='client.? 172.21.15.6:0/578968638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:04.209 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:03 smithi006 bash[27292]: cluster 2023-08-28T00:28:02.610120+0000 mgr.y (mgr.14901) 4484 : cluster [DBG] pgmap v4491: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:04.209 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:03 smithi006 bash[27292]: audit 2023-08-28T00:28:03.428282+0000 mon.c (mon.2) 1732 : audit [DBG] from='client.? 172.21.15.6:0/578968638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:28:04.205Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:28:04.207Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:04.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:04 smithi130 bash[51302]: debug 2023-08-28T00:28:04.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:04.536088+0000) 2023-08-28T00:28:05.819 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:05 smithi130 bash[51302]: debug 2023-08-28T00:28:05.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:05.536316+0000) 2023-08-28T00:28:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:05 smithi130 bash[27579]: cluster 2023-08-28T00:28:04.611349+0000 mgr.y (mgr.14901) 4485 : cluster [DBG] pgmap v4492: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:06.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:05 smithi006 bash[27292]: cluster 2023-08-28T00:28:04.611349+0000 mgr.y (mgr.14901) 4485 : cluster [DBG] pgmap v4492: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:05 smithi006 bash[33918]: cluster 2023-08-28T00:28:04.611349+0000 mgr.y (mgr.14901) 4485 : cluster [DBG] pgmap v4492: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:06.826 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:06 smithi130 bash[51302]: debug 2023-08-28T00:28:06.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:06.536596+0000) 2023-08-28T00:28:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:06 smithi130 bash[27579]: audit 2023-08-28T00:28:05.841220+0000 mon.c (mon.2) 1733 : audit [DBG] from='client.? 172.21.15.6:0/2625937509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:07.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:06 smithi006 bash[27292]: audit 2023-08-28T00:28:05.841220+0000 mon.c (mon.2) 1733 : audit [DBG] from='client.? 172.21.15.6:0/2625937509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:07.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:06 smithi006 bash[33918]: audit 2023-08-28T00:28:05.841220+0000 mon.c (mon.2) 1733 : audit [DBG] from='client.? 172.21.15.6:0/2625937509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:07.829 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:07 smithi130 bash[51302]: debug 2023-08-28T00:28:07.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:07.536866+0000) 2023-08-28T00:28:08.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:07 smithi130 bash[27579]: cluster 2023-08-28T00:28:06.612116+0000 mgr.y (mgr.14901) 4486 : cluster [DBG] pgmap v4493: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:08.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:07 smithi006 bash[27292]: cluster 2023-08-28T00:28:06.612116+0000 mgr.y (mgr.14901) 4486 : cluster [DBG] pgmap v4493: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:08.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:07 smithi006 bash[33918]: cluster 2023-08-28T00:28:06.612116+0000 mgr.y (mgr.14901) 4486 : cluster [DBG] pgmap v4493: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:08.832 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:08 smithi130 bash[51302]: debug 2023-08-28T00:28:08.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:08.537225+0000) 2023-08-28T00:28:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:08 smithi130 bash[27579]: audit 2023-08-28T00:28:08.250146+0000 mon.c (mon.2) 1734 : audit [DBG] from='client.? 172.21.15.6:0/4188533310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:08 smithi006 bash[33918]: audit 2023-08-28T00:28:08.250146+0000 mon.c (mon.2) 1734 : audit [DBG] from='client.? 172.21.15.6:0/4188533310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:08 smithi006 bash[27292]: audit 2023-08-28T00:28:08.250146+0000 mon.c (mon.2) 1734 : audit [DBG] from='client.? 172.21.15.6:0/4188533310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:09.843 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:09 smithi130 bash[51302]: debug 2023-08-28T00:28:09.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:09.537546+0000) 2023-08-28T00:28:10.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:09 smithi130 bash[27579]: cluster 2023-08-28T00:28:08.612785+0000 mgr.y (mgr.14901) 4487 : cluster [DBG] pgmap v4494: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:10.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:09 smithi006 bash[27292]: cluster 2023-08-28T00:28:08.612785+0000 mgr.y (mgr.14901) 4487 : cluster [DBG] pgmap v4494: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:10.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:09 smithi006 bash[33918]: cluster 2023-08-28T00:28:08.612785+0000 mgr.y (mgr.14901) 4487 : cluster [DBG] pgmap v4494: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:10.855 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:10 smithi130 bash[51302]: debug 2023-08-28T00:28:10.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:10.537822+0000) 2023-08-28T00:28:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:10 smithi130 bash[27579]: audit 2023-08-28T00:28:10.666060+0000 mon.a (mon.0) 3262 : audit [DBG] from='client.? 172.21.15.6:0/1881443666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:10 smithi006 bash[33918]: audit 2023-08-28T00:28:10.666060+0000 mon.a (mon.0) 3262 : audit [DBG] from='client.? 172.21.15.6:0/1881443666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:11.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:10 smithi006 bash[27292]: audit 2023-08-28T00:28:10.666060+0000 mon.a (mon.0) 3262 : audit [DBG] from='client.? 172.21.15.6:0/1881443666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:11.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:28:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:28:10] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:28:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:28:10.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:28:10.745Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:11.858 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:11 smithi130 bash[51302]: debug 2023-08-28T00:28:11.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:11.538136+0000) 2023-08-28T00:28:12.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:11 smithi130 bash[27579]: cluster 2023-08-28T00:28:10.613849+0000 mgr.y (mgr.14901) 4488 : cluster [DBG] pgmap v4495: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:12.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:11 smithi006 bash[27292]: cluster 2023-08-28T00:28:10.613849+0000 mgr.y (mgr.14901) 4488 : cluster [DBG] pgmap v4495: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:12.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:11 smithi006 bash[33918]: cluster 2023-08-28T00:28:10.613849+0000 mgr.y (mgr.14901) 4488 : cluster [DBG] pgmap v4495: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:28:12.322Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:28:12.324Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:12 smithi130 bash[51302]: debug 2023-08-28T00:28:12.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:12.538412+0000) 2023-08-28T00:28:13.863 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:13 smithi130 bash[51302]: debug 2023-08-28T00:28:13.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:13.538685+0000) 2023-08-28T00:28:14.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:13 smithi130 bash[27579]: cluster 2023-08-28T00:28:12.614622+0000 mgr.y (mgr.14901) 4489 : cluster [DBG] pgmap v4496: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:14.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:13 smithi130 bash[27579]: audit 2023-08-28T00:28:13.091552+0000 mon.c (mon.2) 1735 : audit [DBG] from='client.? 172.21.15.6:0/2559487431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:14.210 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:13 smithi006 bash[27292]: cluster 2023-08-28T00:28:12.614622+0000 mgr.y (mgr.14901) 4489 : cluster [DBG] pgmap v4496: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:14.210 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:13 smithi006 bash[27292]: audit 2023-08-28T00:28:13.091552+0000 mon.c (mon.2) 1735 : audit [DBG] from='client.? 172.21.15.6:0/2559487431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:14.210 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:13 smithi006 bash[33918]: cluster 2023-08-28T00:28:12.614622+0000 mgr.y (mgr.14901) 4489 : cluster [DBG] pgmap v4496: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:14.210 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:13 smithi006 bash[33918]: audit 2023-08-28T00:28:13.091552+0000 mon.c (mon.2) 1735 : audit [DBG] from='client.? 172.21.15.6:0/2559487431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:28:14.206Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:28:14.209Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:14 smithi130 bash[51302]: debug 2023-08-28T00:28:14.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:14.539034+0000) 2023-08-28T00:28:15.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:15 smithi130 bash[51302]: debug 2023-08-28T00:28:15.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:15.539335+0000) 2023-08-28T00:28:16.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:15 smithi006 bash[33918]: cluster 2023-08-28T00:28:14.615831+0000 mgr.y (mgr.14901) 4490 : cluster [DBG] pgmap v4497: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:16.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:15 smithi006 bash[33918]: audit 2023-08-28T00:28:15.503742+0000 mon.a (mon.0) 3263 : audit [DBG] from='client.? 172.21.15.6:0/4196575459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:16.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:15 smithi006 bash[27292]: cluster 2023-08-28T00:28:14.615831+0000 mgr.y (mgr.14901) 4490 : cluster [DBG] pgmap v4497: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:16.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:15 smithi006 bash[27292]: audit 2023-08-28T00:28:15.503742+0000 mon.a (mon.0) 3263 : audit [DBG] from='client.? 172.21.15.6:0/4196575459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:15 smithi130 bash[27579]: cluster 2023-08-28T00:28:14.615831+0000 mgr.y (mgr.14901) 4490 : cluster [DBG] pgmap v4497: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:16.396 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:15 smithi130 bash[27579]: audit 2023-08-28T00:28:15.503742+0000 mon.a (mon.0) 3263 : audit [DBG] from='client.? 172.21.15.6:0/4196575459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:16.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:16 smithi130 bash[51302]: debug 2023-08-28T00:28:16.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:16.539613+0000) 2023-08-28T00:28:17.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:17 smithi130 bash[51302]: debug 2023-08-28T00:28:17.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:17.539903+0000) 2023-08-28T00:28:18.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:17 smithi006 bash[27292]: cluster 2023-08-28T00:28:16.616640+0000 mgr.y (mgr.14901) 4491 : cluster [DBG] pgmap v4498: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:18.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:17 smithi006 bash[33918]: cluster 2023-08-28T00:28:16.616640+0000 mgr.y (mgr.14901) 4491 : cluster [DBG] pgmap v4498: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:17 smithi130 bash[27579]: cluster 2023-08-28T00:28:16.616640+0000 mgr.y (mgr.14901) 4491 : cluster [DBG] pgmap v4498: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:18 smithi130 bash[51302]: debug 2023-08-28T00:28:18.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:18.540169+0000) 2023-08-28T00:28:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:18 smithi006 bash[27292]: audit 2023-08-28T00:28:17.921991+0000 mon.c (mon.2) 1736 : audit [DBG] from='client.? 172.21.15.6:0/3823104720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:19.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:18 smithi006 bash[33918]: audit 2023-08-28T00:28:17.921991+0000 mon.c (mon.2) 1736 : audit [DBG] from='client.? 172.21.15.6:0/3823104720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:19.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:18 smithi130 bash[27579]: audit 2023-08-28T00:28:17.921991+0000 mon.c (mon.2) 1736 : audit [DBG] from='client.? 172.21.15.6:0/3823104720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:19 smithi130 bash[51302]: debug 2023-08-28T00:28:19.540+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:19.540500+0000) 2023-08-28T00:28:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:19 smithi006 bash[27292]: cluster 2023-08-28T00:28:18.617424+0000 mgr.y (mgr.14901) 4492 : cluster [DBG] pgmap v4499: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:20.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:19 smithi006 bash[33918]: cluster 2023-08-28T00:28:18.617424+0000 mgr.y (mgr.14901) 4492 : cluster [DBG] pgmap v4499: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:19 smithi130 bash[27579]: cluster 2023-08-28T00:28:18.617424+0000 mgr.y (mgr.14901) 4492 : cluster [DBG] pgmap v4499: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:20 smithi130 bash[51302]: debug 2023-08-28T00:28:20.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:20.540798+0000) 2023-08-28T00:28:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:20 smithi006 bash[27292]: audit 2023-08-28T00:28:20.329162+0000 mon.a (mon.0) 3264 : audit [DBG] from='client.? 172.21.15.6:0/3770119848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:21.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:28:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:28:20] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T00:28:21.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:20 smithi006 bash[33918]: audit 2023-08-28T00:28:20.329162+0000 mon.a (mon.0) 3264 : audit [DBG] from='client.? 172.21.15.6:0/3770119848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:28:20.744Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:28:20.746Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:21.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:20 smithi130 bash[27579]: audit 2023-08-28T00:28:20.329162+0000 mon.a (mon.0) 3264 : audit [DBG] from='client.? 172.21.15.6:0/3770119848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:21.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:21 smithi130 bash[51302]: debug 2023-08-28T00:28:21.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:21.541132+0000) 2023-08-28T00:28:22.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:21 smithi006 bash[33918]: cluster 2023-08-28T00:28:20.618539+0000 mgr.y (mgr.14901) 4493 : cluster [DBG] pgmap v4500: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:22.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:21 smithi006 bash[27292]: cluster 2023-08-28T00:28:20.618539+0000 mgr.y (mgr.14901) 4493 : cluster [DBG] pgmap v4500: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:22.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:21 smithi130 bash[27579]: cluster 2023-08-28T00:28:20.618539+0000 mgr.y (mgr.14901) 4493 : cluster [DBG] pgmap v4500: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:28:22.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:28:22.325Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:22 smithi130 bash[51302]: debug 2023-08-28T00:28:22.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:22.541419+0000) 2023-08-28T00:28:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:22 smithi006 bash[33918]: audit 2023-08-28T00:28:22.754378+0000 mon.a (mon.0) 3265 : audit [DBG] from='client.? 172.21.15.6:0/2041848847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:23.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:22 smithi006 bash[27292]: audit 2023-08-28T00:28:22.754378+0000 mon.a (mon.0) 3265 : audit [DBG] from='client.? 172.21.15.6:0/2041848847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:23.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:22 smithi130 bash[27579]: audit 2023-08-28T00:28:22.754378+0000 mon.a (mon.0) 3265 : audit [DBG] from='client.? 172.21.15.6:0/2041848847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:23.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:23 smithi130 bash[51302]: debug 2023-08-28T00:28:23.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:23.541694+0000) 2023-08-28T00:28:24.210 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:23 smithi006 bash[33918]: cluster 2023-08-28T00:28:22.619163+0000 mgr.y (mgr.14901) 4494 : cluster [DBG] pgmap v4501: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:24.211 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:23 smithi006 bash[27292]: cluster 2023-08-28T00:28:22.619163+0000 mgr.y (mgr.14901) 4494 : cluster [DBG] pgmap v4501: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:23 smithi130 bash[27579]: cluster 2023-08-28T00:28:22.619163+0000 mgr.y (mgr.14901) 4494 : cluster [DBG] pgmap v4501: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:28:24.207Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:28:24.210Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:24 smithi130 bash[51302]: debug 2023-08-28T00:28:24.540+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:24.542022+0000) 2023-08-28T00:28:25.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:25 smithi130 bash[51302]: debug 2023-08-28T00:28:25.540+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:25.542295+0000) 2023-08-28T00:28:26.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:25 smithi006 bash[33918]: cluster 2023-08-28T00:28:24.620416+0000 mgr.y (mgr.14901) 4495 : cluster [DBG] pgmap v4502: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:26.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:25 smithi006 bash[33918]: audit 2023-08-28T00:28:25.176112+0000 mon.c (mon.2) 1737 : audit [DBG] from='client.? 172.21.15.6:0/2526093683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:26.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:25 smithi006 bash[27292]: cluster 2023-08-28T00:28:24.620416+0000 mgr.y (mgr.14901) 4495 : cluster [DBG] pgmap v4502: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:26.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:25 smithi006 bash[27292]: audit 2023-08-28T00:28:25.176112+0000 mon.c (mon.2) 1737 : audit [DBG] from='client.? 172.21.15.6:0/2526093683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:25 smithi130 bash[27579]: cluster 2023-08-28T00:28:24.620416+0000 mgr.y (mgr.14901) 4495 : cluster [DBG] pgmap v4502: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:25 smithi130 bash[27579]: audit 2023-08-28T00:28:25.176112+0000 mon.c (mon.2) 1737 : audit [DBG] from='client.? 172.21.15.6:0/2526093683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:26.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:26 smithi130 bash[51302]: debug 2023-08-28T00:28:26.540+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:26.542580+0000) 2023-08-28T00:28:27.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:27 smithi130 bash[51302]: debug 2023-08-28T00:28:27.540+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:27.542890+0000) 2023-08-28T00:28:28.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:27 smithi006 bash[27292]: cluster 2023-08-28T00:28:26.621141+0000 mgr.y (mgr.14901) 4496 : cluster [DBG] pgmap v4503: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:28.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:27 smithi006 bash[27292]: audit 2023-08-28T00:28:27.593360+0000 mon.c (mon.2) 1738 : audit [DBG] from='client.? 172.21.15.6:0/3184131311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:28.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:27 smithi006 bash[33918]: cluster 2023-08-28T00:28:26.621141+0000 mgr.y (mgr.14901) 4496 : cluster [DBG] pgmap v4503: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:28.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:27 smithi006 bash[33918]: audit 2023-08-28T00:28:27.593360+0000 mon.c (mon.2) 1738 : audit [DBG] from='client.? 172.21.15.6:0/3184131311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:28.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:27 smithi130 bash[27579]: cluster 2023-08-28T00:28:26.621141+0000 mgr.y (mgr.14901) 4496 : cluster [DBG] pgmap v4503: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:28.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:27 smithi130 bash[27579]: audit 2023-08-28T00:28:27.593360+0000 mon.c (mon.2) 1738 : audit [DBG] from='client.? 172.21.15.6:0/3184131311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:28 smithi130 bash[51302]: debug 2023-08-28T00:28:28.540+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:28.543148+0000) 2023-08-28T00:28:29.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:29 smithi130 bash[51302]: debug 2023-08-28T00:28:29.540+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:29.543481+0000) 2023-08-28T00:28:30.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:29 smithi006 bash[27292]: cluster 2023-08-28T00:28:28.621791+0000 mgr.y (mgr.14901) 4497 : cluster [DBG] pgmap v4504: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:30.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:29 smithi006 bash[33918]: cluster 2023-08-28T00:28:28.621791+0000 mgr.y (mgr.14901) 4497 : cluster [DBG] pgmap v4504: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:29 smithi130 bash[27579]: cluster 2023-08-28T00:28:28.621791+0000 mgr.y (mgr.14901) 4497 : cluster [DBG] pgmap v4504: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:30 smithi130 bash[51302]: debug 2023-08-28T00:28:30.540+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:30.543768+0000) 2023-08-28T00:28:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:28:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:28:30] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:28:31.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:30 smithi006 bash[33918]: audit 2023-08-28T00:28:30.005553+0000 mon.c (mon.2) 1739 : audit [DBG] from='client.? 172.21.15.6:0/1646938387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:31.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:30 smithi006 bash[27292]: audit 2023-08-28T00:28:30.005553+0000 mon.c (mon.2) 1739 : audit [DBG] from='client.? 172.21.15.6:0/1646938387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:31.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:28:30.744Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:31.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:28:30.746Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:31.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:30 smithi130 bash[27579]: audit 2023-08-28T00:28:30.005553+0000 mon.c (mon.2) 1739 : audit [DBG] from='client.? 172.21.15.6:0/1646938387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:31.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:31 smithi130 bash[51302]: debug 2023-08-28T00:28:31.540+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:31.544038+0000) 2023-08-28T00:28:32.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:31 smithi006 bash[33918]: cluster 2023-08-28T00:28:30.622899+0000 mgr.y (mgr.14901) 4498 : cluster [DBG] pgmap v4505: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:32.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:31 smithi006 bash[27292]: cluster 2023-08-28T00:28:30.622899+0000 mgr.y (mgr.14901) 4498 : cluster [DBG] pgmap v4505: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:31 smithi130 bash[27579]: cluster 2023-08-28T00:28:30.622899+0000 mgr.y (mgr.14901) 4498 : cluster [DBG] pgmap v4505: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:28:32.324Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:28:32.326Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:32 smithi130 bash[51302]: debug 2023-08-28T00:28:32.544+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:32.544321+0000) 2023-08-28T00:28:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:32 smithi006 bash[27292]: audit 2023-08-28T00:28:32.426687+0000 mon.c (mon.2) 1740 : audit [DBG] from='client.? 172.21.15.6:0/898849091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:33.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:32 smithi006 bash[33918]: audit 2023-08-28T00:28:32.426687+0000 mon.c (mon.2) 1740 : audit [DBG] from='client.? 172.21.15.6:0/898849091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:32 smithi130 bash[27579]: audit 2023-08-28T00:28:32.426687+0000 mon.c (mon.2) 1740 : audit [DBG] from='client.? 172.21.15.6:0/898849091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:33.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:33 smithi130 bash[51302]: debug 2023-08-28T00:28:33.540+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:33.544653+0000) 2023-08-28T00:28:34.211 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:33 smithi006 bash[33918]: cluster 2023-08-28T00:28:32.623563+0000 mgr.y (mgr.14901) 4499 : cluster [DBG] pgmap v4506: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:34.211 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:33 smithi006 bash[27292]: cluster 2023-08-28T00:28:32.623563+0000 mgr.y (mgr.14901) 4499 : cluster [DBG] pgmap v4506: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:34.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:33 smithi130 bash[27579]: cluster 2023-08-28T00:28:32.623563+0000 mgr.y (mgr.14901) 4499 : cluster [DBG] pgmap v4506: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:28:34.208Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:34.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:28:34.210Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:34 smithi130 bash[51302]: debug 2023-08-28T00:28:34.539+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:34.544917+0000) 2023-08-28T00:28:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:34 smithi006 bash[27292]: audit 2023-08-28T00:28:34.845643+0000 mon.c (mon.2) 1741 : audit [DBG] from='client.? 172.21.15.6:0/3254087855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:35.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:34 smithi006 bash[33918]: audit 2023-08-28T00:28:34.845643+0000 mon.c (mon.2) 1741 : audit [DBG] from='client.? 172.21.15.6:0/3254087855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:34 smithi130 bash[27579]: audit 2023-08-28T00:28:34.845643+0000 mon.c (mon.2) 1741 : audit [DBG] from='client.? 172.21.15.6:0/3254087855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:35.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:35 smithi130 bash[51302]: debug 2023-08-28T00:28:35.539+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:35.545191+0000) 2023-08-28T00:28:36.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:35 smithi006 bash[33918]: cluster 2023-08-28T00:28:34.624646+0000 mgr.y (mgr.14901) 4500 : cluster [DBG] pgmap v4507: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:36.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:35 smithi006 bash[27292]: cluster 2023-08-28T00:28:34.624646+0000 mgr.y (mgr.14901) 4500 : cluster [DBG] pgmap v4507: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:35 smithi130 bash[27579]: cluster 2023-08-28T00:28:34.624646+0000 mgr.y (mgr.14901) 4500 : cluster [DBG] pgmap v4507: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:36 smithi130 bash[51302]: debug 2023-08-28T00:28:36.539+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:36.545461+0000) 2023-08-28T00:28:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:37 smithi130 bash[51302]: debug 2023-08-28T00:28:37.539+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:37.545725+0000) 2023-08-28T00:28:38.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:37 smithi006 bash[27292]: cluster 2023-08-28T00:28:36.625320+0000 mgr.y (mgr.14901) 4501 : cluster [DBG] pgmap v4508: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:38.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:37 smithi006 bash[27292]: audit 2023-08-28T00:28:37.259552+0000 mon.a (mon.0) 3266 : audit [DBG] from='client.? 172.21.15.6:0/1158032964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:38.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:37 smithi006 bash[33918]: cluster 2023-08-28T00:28:36.625320+0000 mgr.y (mgr.14901) 4501 : cluster [DBG] pgmap v4508: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:38.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:37 smithi006 bash[33918]: audit 2023-08-28T00:28:37.259552+0000 mon.a (mon.0) 3266 : audit [DBG] from='client.? 172.21.15.6:0/1158032964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:38.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:37 smithi130 bash[27579]: cluster 2023-08-28T00:28:36.625320+0000 mgr.y (mgr.14901) 4501 : cluster [DBG] pgmap v4508: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:38.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:37 smithi130 bash[27579]: audit 2023-08-28T00:28:37.259552+0000 mon.a (mon.0) 3266 : audit [DBG] from='client.? 172.21.15.6:0/1158032964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:38 smithi130 bash[51302]: debug 2023-08-28T00:28:38.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:38.546007+0000) 2023-08-28T00:28:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:39 smithi130 bash[51302]: debug 2023-08-28T00:28:39.539+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:39.546253+0000) 2023-08-28T00:28:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:39 smithi130 bash[27579]: cluster 2023-08-28T00:28:38.625958+0000 mgr.y (mgr.14901) 4502 : cluster [DBG] pgmap v4509: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:39 smithi130 bash[27579]: audit 2023-08-28T00:28:39.679394+0000 mon.a (mon.0) 3267 : audit [DBG] from='client.? 172.21.15.6:0/3526398142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:40.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:39 smithi006 bash[27292]: cluster 2023-08-28T00:28:38.625958+0000 mgr.y (mgr.14901) 4502 : cluster [DBG] pgmap v4509: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:40.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:39 smithi006 bash[27292]: audit 2023-08-28T00:28:39.679394+0000 mon.a (mon.0) 3267 : audit [DBG] from='client.? 172.21.15.6:0/3526398142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:39 smithi006 bash[33918]: cluster 2023-08-28T00:28:38.625958+0000 mgr.y (mgr.14901) 4502 : cluster [DBG] pgmap v4509: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:39 smithi006 bash[33918]: audit 2023-08-28T00:28:39.679394+0000 mon.a (mon.0) 3267 : audit [DBG] from='client.? 172.21.15.6:0/3526398142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:40 smithi130 bash[51302]: debug 2023-08-28T00:28:40.539+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:40.546539+0000) 2023-08-28T00:28:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:28:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:28:40] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:28:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:28:40.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:28:40.747Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:41.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:41 smithi130 bash[51302]: debug 2023-08-28T00:28:41.539+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:41.546834+0000) 2023-08-28T00:28:42.328 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:41 smithi006 bash[33918]: cluster 2023-08-28T00:28:40.627165+0000 mgr.y (mgr.14901) 4503 : cluster [DBG] pgmap v4510: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:42.328 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:41 smithi006 bash[27292]: cluster 2023-08-28T00:28:40.627165+0000 mgr.y (mgr.14901) 4503 : cluster [DBG] pgmap v4510: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:41 smithi130 bash[27579]: cluster 2023-08-28T00:28:40.627165+0000 mgr.y (mgr.14901) 4503 : cluster [DBG] pgmap v4510: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:28:42.324Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:28:42.327Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:42 smithi130 bash[51302]: debug 2023-08-28T00:28:42.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:42.547076+0000) 2023-08-28T00:28:43.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:42 smithi130 bash[27579]: audit 2023-08-28T00:28:42.093446+0000 mon.c (mon.2) 1742 : audit [DBG] from='client.? 172.21.15.6:0/1325373082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:43.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:42 smithi006 bash[33918]: audit 2023-08-28T00:28:42.093446+0000 mon.c (mon.2) 1742 : audit [DBG] from='client.? 172.21.15.6:0/1325373082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:43.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:42 smithi006 bash[27292]: audit 2023-08-28T00:28:42.093446+0000 mon.c (mon.2) 1742 : audit [DBG] from='client.? 172.21.15.6:0/1325373082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:43.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:43 smithi130 bash[51302]: debug 2023-08-28T00:28:43.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:43.547340+0000) 2023-08-28T00:28:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:43 smithi130 bash[27579]: cluster 2023-08-28T00:28:42.627857+0000 mgr.y (mgr.14901) 4504 : cluster [DBG] pgmap v4511: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:44.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:43 smithi006 bash[27292]: cluster 2023-08-28T00:28:42.627857+0000 mgr.y (mgr.14901) 4504 : cluster [DBG] pgmap v4511: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:44.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:43 smithi006 bash[33918]: cluster 2023-08-28T00:28:42.627857+0000 mgr.y (mgr.14901) 4504 : cluster [DBG] pgmap v4511: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:28:44.209Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:28:44.211Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:44 smithi130 bash[51302]: debug 2023-08-28T00:28:44.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:44.547659+0000) 2023-08-28T00:28:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:44 smithi130 bash[27579]: audit 2023-08-28T00:28:44.513601+0000 mon.c (mon.2) 1743 : audit [DBG] from='client.? 172.21.15.6:0/3759509554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:45.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:44 smithi006 bash[33918]: audit 2023-08-28T00:28:44.513601+0000 mon.c (mon.2) 1743 : audit [DBG] from='client.? 172.21.15.6:0/3759509554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:45.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:44 smithi006 bash[27292]: audit 2023-08-28T00:28:44.513601+0000 mon.c (mon.2) 1743 : audit [DBG] from='client.? 172.21.15.6:0/3759509554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:45.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:45 smithi130 bash[51302]: debug 2023-08-28T00:28:45.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:45.547948+0000) 2023-08-28T00:28:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:46 smithi130 bash[27579]: cluster 2023-08-28T00:28:44.628998+0000 mgr.y (mgr.14901) 4505 : cluster [DBG] pgmap v4512: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:46.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:45 smithi006 bash[27292]: cluster 2023-08-28T00:28:44.628998+0000 mgr.y (mgr.14901) 4505 : cluster [DBG] pgmap v4512: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:46.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:45 smithi006 bash[33918]: cluster 2023-08-28T00:28:44.628998+0000 mgr.y (mgr.14901) 4505 : cluster [DBG] pgmap v4512: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:46.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:46 smithi130 bash[51302]: debug 2023-08-28T00:28:46.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:46.548225+0000) 2023-08-28T00:28:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:47 smithi130 bash[27579]: audit 2023-08-28T00:28:46.924747+0000 mon.c (mon.2) 1744 : audit [DBG] from='client.? 172.21.15.6:0/388622401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:47.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:47 smithi006 bash[27292]: audit 2023-08-28T00:28:46.924747+0000 mon.c (mon.2) 1744 : audit [DBG] from='client.? 172.21.15.6:0/388622401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:47.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:47 smithi006 bash[33918]: audit 2023-08-28T00:28:46.924747+0000 mon.c (mon.2) 1744 : audit [DBG] from='client.? 172.21.15.6:0/388622401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:47.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:47 smithi130 bash[51302]: debug 2023-08-28T00:28:47.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:47.548428+0000) 2023-08-28T00:28:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:48 smithi130 bash[27579]: cluster 2023-08-28T00:28:46.629624+0000 mgr.y (mgr.14901) 4506 : cluster [DBG] pgmap v4513: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:48.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:48 smithi006 bash[33918]: cluster 2023-08-28T00:28:46.629624+0000 mgr.y (mgr.14901) 4506 : cluster [DBG] pgmap v4513: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:48.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:48 smithi006 bash[27292]: cluster 2023-08-28T00:28:46.629624+0000 mgr.y (mgr.14901) 4506 : cluster [DBG] pgmap v4513: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:48 smithi130 bash[51302]: debug 2023-08-28T00:28:48.547+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:48.548695+0000) 2023-08-28T00:28:49.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:49 smithi130 bash[51302]: debug 2023-08-28T00:28:49.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:49.548976+0000) 2023-08-28T00:28:50.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:50 smithi130 bash[27579]: cluster 2023-08-28T00:28:48.630279+0000 mgr.y (mgr.14901) 4507 : cluster [DBG] pgmap v4514: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:50.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:50 smithi130 bash[27579]: audit 2023-08-28T00:28:49.337452+0000 mon.c (mon.2) 1745 : audit [DBG] from='client.? 172.21.15.6:0/3292038140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:50.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:50 smithi006 bash[27292]: cluster 2023-08-28T00:28:48.630279+0000 mgr.y (mgr.14901) 4507 : cluster [DBG] pgmap v4514: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:50.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:50 smithi006 bash[27292]: audit 2023-08-28T00:28:49.337452+0000 mon.c (mon.2) 1745 : audit [DBG] from='client.? 172.21.15.6:0/3292038140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:50.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:50 smithi006 bash[33918]: cluster 2023-08-28T00:28:48.630279+0000 mgr.y (mgr.14901) 4507 : cluster [DBG] pgmap v4514: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:50.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:50 smithi006 bash[33918]: audit 2023-08-28T00:28:49.337452+0000 mon.c (mon.2) 1745 : audit [DBG] from='client.? 172.21.15.6:0/3292038140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:50 smithi130 bash[51302]: debug 2023-08-28T00:28:50.547+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:50.549253+0000) 2023-08-28T00:28:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:28:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:28:50] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:28:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:28:50.746Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:28:50.748Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:51.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:51 smithi130 bash[51302]: debug 2023-08-28T00:28:51.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:51.549549+0000) 2023-08-28T00:28:52.329 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:52 smithi006 bash[33918]: cluster 2023-08-28T00:28:50.631416+0000 mgr.y (mgr.14901) 4508 : cluster [DBG] pgmap v4515: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:52.329 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:52 smithi006 bash[33918]: audit 2023-08-28T00:28:51.751422+0000 mon.c (mon.2) 1746 : audit [DBG] from='client.? 172.21.15.6:0/1545132068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:52.329 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:52 smithi006 bash[27292]: cluster 2023-08-28T00:28:50.631416+0000 mgr.y (mgr.14901) 4508 : cluster [DBG] pgmap v4515: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:52.329 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:52 smithi006 bash[27292]: audit 2023-08-28T00:28:51.751422+0000 mon.c (mon.2) 1746 : audit [DBG] from='client.? 172.21.15.6:0/1545132068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:52 smithi130 bash[27579]: cluster 2023-08-28T00:28:50.631416+0000 mgr.y (mgr.14901) 4508 : cluster [DBG] pgmap v4515: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:52.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:52 smithi130 bash[27579]: audit 2023-08-28T00:28:51.751422+0000 mon.c (mon.2) 1746 : audit [DBG] from='client.? 172.21.15.6:0/1545132068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:28:52.325Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:28:52.327Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:52.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:52 smithi130 bash[51302]: debug 2023-08-28T00:28:52.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:52.549760+0000) 2023-08-28T00:28:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:53 smithi130 bash[27579]: audit 2023-08-28T00:28:52.916569+0000 mon.b (mon.1) 553 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:28:53.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:53 smithi006 bash[27292]: audit 2023-08-28T00:28:52.916569+0000 mon.b (mon.1) 553 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:28:53.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:53 smithi006 bash[33918]: audit 2023-08-28T00:28:52.916569+0000 mon.b (mon.1) 553 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:28:53.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:53 smithi130 bash[51302]: debug 2023-08-28T00:28:53.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:53.550058+0000) 2023-08-28T00:28:54.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:54 smithi130 bash[27579]: cluster 2023-08-28T00:28:52.632058+0000 mgr.y (mgr.14901) 4509 : cluster [DBG] pgmap v4516: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:54.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:54 smithi006 bash[27292]: cluster 2023-08-28T00:28:52.632058+0000 mgr.y (mgr.14901) 4509 : cluster [DBG] pgmap v4516: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:28:54.209Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:28:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:28:54.212Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:28:54.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:54 smithi006 bash[33918]: cluster 2023-08-28T00:28:52.632058+0000 mgr.y (mgr.14901) 4509 : cluster [DBG] pgmap v4516: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:54.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:54 smithi130 bash[51302]: debug 2023-08-28T00:28:54.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:54.550330+0000) 2023-08-28T00:28:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:55 smithi130 bash[27579]: audit 2023-08-28T00:28:54.178272+0000 mon.a (mon.0) 3268 : audit [DBG] from='client.? 172.21.15.6:0/3642424320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:55.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:55 smithi006 bash[27292]: audit 2023-08-28T00:28:54.178272+0000 mon.a (mon.0) 3268 : audit [DBG] from='client.? 172.21.15.6:0/3642424320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:55.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:55 smithi006 bash[33918]: audit 2023-08-28T00:28:54.178272+0000 mon.a (mon.0) 3268 : audit [DBG] from='client.? 172.21.15.6:0/3642424320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:55.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:55 smithi130 bash[51302]: debug 2023-08-28T00:28:55.547+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:55.550624+0000) 2023-08-28T00:28:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:56 smithi130 bash[27579]: cluster 2023-08-28T00:28:54.633214+0000 mgr.y (mgr.14901) 4510 : cluster [DBG] pgmap v4517: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:56.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:56 smithi006 bash[27292]: cluster 2023-08-28T00:28:54.633214+0000 mgr.y (mgr.14901) 4510 : cluster [DBG] pgmap v4517: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:56.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:56 smithi006 bash[33918]: cluster 2023-08-28T00:28:54.633214+0000 mgr.y (mgr.14901) 4510 : cluster [DBG] pgmap v4517: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:56.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:56 smithi130 bash[51302]: debug 2023-08-28T00:28:56.547+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:56.550977+0000) 2023-08-28T00:28:57.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:57 smithi130 bash[27579]: audit 2023-08-28T00:28:56.591885+0000 mon.a (mon.0) 3269 : audit [DBG] from='client.? 172.21.15.6:0/2566889192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:57.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:57 smithi006 bash[27292]: audit 2023-08-28T00:28:56.591885+0000 mon.a (mon.0) 3269 : audit [DBG] from='client.? 172.21.15.6:0/2566889192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:57.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:57 smithi006 bash[33918]: audit 2023-08-28T00:28:56.591885+0000 mon.a (mon.0) 3269 : audit [DBG] from='client.? 172.21.15.6:0/2566889192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:57.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:57 smithi130 bash[51302]: debug 2023-08-28T00:28:57.547+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:57.551273+0000) 2023-08-28T00:28:58.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:58 smithi130 bash[27579]: cluster 2023-08-28T00:28:56.633830+0000 mgr.y (mgr.14901) 4511 : cluster [DBG] pgmap v4518: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:58.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:58 smithi006 bash[27292]: cluster 2023-08-28T00:28:56.633830+0000 mgr.y (mgr.14901) 4511 : cluster [DBG] pgmap v4518: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:58.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:58 smithi006 bash[33918]: cluster 2023-08-28T00:28:56.633830+0000 mgr.y (mgr.14901) 4511 : cluster [DBG] pgmap v4518: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:28:58.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:58 smithi130 bash[51302]: debug 2023-08-28T00:28:58.547+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:58.551474+0000) 2023-08-28T00:28:59.839 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:28:59 smithi130 bash[51302]: debug 2023-08-28T00:28:59.547+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:59.551686+0000) 2023-08-28T00:29:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:59 smithi130 bash[27579]: cluster 2023-08-28T00:28:58.634485+0000 mgr.y (mgr.14901) 4512 : cluster [DBG] pgmap v4519: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:59 smithi130 bash[27579]: audit 2023-08-28T00:28:58.834853+0000 mon.a (mon.0) 3270 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:29:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:59 smithi130 bash[27579]: audit 2023-08-28T00:28:59.013312+0000 mon.a (mon.0) 3271 : audit [DBG] from='client.? 172.21.15.6:0/1692352311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:00.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:59 smithi130 bash[27579]: audit 2023-08-28T00:28:59.286737+0000 mon.a (mon.0) 3272 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:29:00.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:59 smithi130 bash[27579]: audit 2023-08-28T00:28:59.594308+0000 mon.b (mon.1) 554 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:29:00.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:59 smithi130 bash[27579]: audit 2023-08-28T00:28:59.596308+0000 mon.b (mon.1) 555 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:29:00.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:28:59 smithi130 bash[27579]: audit 2023-08-28T00:28:59.604220+0000 mon.a (mon.0) 3273 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:29:00.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:59 smithi006 bash[27292]: cluster 2023-08-28T00:28:58.634485+0000 mgr.y (mgr.14901) 4512 : cluster [DBG] pgmap v4519: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:00.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:59 smithi006 bash[27292]: audit 2023-08-28T00:28:58.834853+0000 mon.a (mon.0) 3270 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:29:00.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:59 smithi006 bash[27292]: audit 2023-08-28T00:28:59.013312+0000 mon.a (mon.0) 3271 : audit [DBG] from='client.? 172.21.15.6:0/1692352311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:00.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:59 smithi006 bash[27292]: audit 2023-08-28T00:28:59.286737+0000 mon.a (mon.0) 3272 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:29:00.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:59 smithi006 bash[27292]: audit 2023-08-28T00:28:59.594308+0000 mon.b (mon.1) 554 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:29:00.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:59 smithi006 bash[27292]: audit 2023-08-28T00:28:59.596308+0000 mon.b (mon.1) 555 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:29:00.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:28:59 smithi006 bash[27292]: audit 2023-08-28T00:28:59.604220+0000 mon.a (mon.0) 3273 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:29:00.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:59 smithi006 bash[33918]: cluster 2023-08-28T00:28:58.634485+0000 mgr.y (mgr.14901) 4512 : cluster [DBG] pgmap v4519: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:00.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:59 smithi006 bash[33918]: audit 2023-08-28T00:28:58.834853+0000 mon.a (mon.0) 3270 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:29:00.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:59 smithi006 bash[33918]: audit 2023-08-28T00:28:59.013312+0000 mon.a (mon.0) 3271 : audit [DBG] from='client.? 172.21.15.6:0/1692352311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:00.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:59 smithi006 bash[33918]: audit 2023-08-28T00:28:59.286737+0000 mon.a (mon.0) 3272 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:29:00.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:59 smithi006 bash[33918]: audit 2023-08-28T00:28:59.594308+0000 mon.b (mon.1) 554 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:29:00.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:59 smithi006 bash[33918]: audit 2023-08-28T00:28:59.596308+0000 mon.b (mon.1) 555 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:29:00.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:28:59 smithi006 bash[33918]: audit 2023-08-28T00:28:59.604220+0000 mon.a (mon.0) 3273 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:29:00.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:00 smithi130 bash[51302]: debug 2023-08-28T00:29:00.547+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:00.552014+0000) 2023-08-28T00:29:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:29:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:29:00] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:29:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:29:00.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:29:00.749Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:01.841 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:01 smithi130 bash[51302]: debug 2023-08-28T00:29:01.547+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:01.552400+0000) 2023-08-28T00:29:02.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:01 smithi130 bash[27579]: cluster 2023-08-28T00:29:00.635734+0000 mgr.y (mgr.14901) 4513 : cluster [DBG] pgmap v4520: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:02.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:01 smithi130 bash[27579]: audit 2023-08-28T00:29:01.429283+0000 mon.a (mon.0) 3274 : audit [DBG] from='client.? 172.21.15.6:0/1535507315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:02.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:01 smithi006 bash[33918]: cluster 2023-08-28T00:29:00.635734+0000 mgr.y (mgr.14901) 4513 : cluster [DBG] pgmap v4520: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:02.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:01 smithi006 bash[33918]: audit 2023-08-28T00:29:01.429283+0000 mon.a (mon.0) 3274 : audit [DBG] from='client.? 172.21.15.6:0/1535507315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:02.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:01 smithi006 bash[27292]: cluster 2023-08-28T00:29:00.635734+0000 mgr.y (mgr.14901) 4513 : cluster [DBG] pgmap v4520: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:02.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:01 smithi006 bash[27292]: audit 2023-08-28T00:29:01.429283+0000 mon.a (mon.0) 3274 : audit [DBG] from='client.? 172.21.15.6:0/1535507315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:29:02.326Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:29:02.328Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:02.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:02 smithi130 bash[51302]: debug 2023-08-28T00:29:02.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:02.552737+0000) 2023-08-28T00:29:03.846 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:03 smithi130 bash[51302]: debug 2023-08-28T00:29:03.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:03.553062+0000) 2023-08-28T00:29:04.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:03 smithi130 bash[27579]: cluster 2023-08-28T00:29:02.636473+0000 mgr.y (mgr.14901) 4514 : cluster [DBG] pgmap v4521: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:04.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:03 smithi130 bash[27579]: audit 2023-08-28T00:29:03.836780+0000 mon.a (mon.0) 3275 : audit [DBG] from='client.? 172.21.15.6:0/940390045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:04.213 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:03 smithi006 bash[27292]: cluster 2023-08-28T00:29:02.636473+0000 mgr.y (mgr.14901) 4514 : cluster [DBG] pgmap v4521: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:04.213 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:03 smithi006 bash[27292]: audit 2023-08-28T00:29:03.836780+0000 mon.a (mon.0) 3275 : audit [DBG] from='client.? 172.21.15.6:0/940390045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:04.213 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:03 smithi006 bash[33918]: cluster 2023-08-28T00:29:02.636473+0000 mgr.y (mgr.14901) 4514 : cluster [DBG] pgmap v4521: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:04.213 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:03 smithi006 bash[33918]: audit 2023-08-28T00:29:03.836780+0000 mon.a (mon.0) 3275 : audit [DBG] from='client.? 172.21.15.6:0/940390045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:29:04.209Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:04.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:29:04.212Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:04.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:04 smithi130 bash[51302]: debug 2023-08-28T00:29:04.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:04.553392+0000) 2023-08-28T00:29:05.851 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:05 smithi130 bash[51302]: debug 2023-08-28T00:29:05.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:05.553752+0000) 2023-08-28T00:29:06.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:05 smithi130 bash[27579]: cluster 2023-08-28T00:29:04.637641+0000 mgr.y (mgr.14901) 4515 : cluster [DBG] pgmap v4522: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:06.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:05 smithi006 bash[33918]: cluster 2023-08-28T00:29:04.637641+0000 mgr.y (mgr.14901) 4515 : cluster [DBG] pgmap v4522: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:06.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:05 smithi006 bash[27292]: cluster 2023-08-28T00:29:04.637641+0000 mgr.y (mgr.14901) 4515 : cluster [DBG] pgmap v4522: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:06.858 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:06 smithi130 bash[51302]: debug 2023-08-28T00:29:06.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:06.554078+0000) 2023-08-28T00:29:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:06 smithi130 bash[27579]: audit 2023-08-28T00:29:06.246601+0000 mon.a (mon.0) 3276 : audit [DBG] from='client.? 172.21.15.6:0/3137691793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:07.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:06 smithi006 bash[27292]: audit 2023-08-28T00:29:06.246601+0000 mon.a (mon.0) 3276 : audit [DBG] from='client.? 172.21.15.6:0/3137691793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:07.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:06 smithi006 bash[33918]: audit 2023-08-28T00:29:06.246601+0000 mon.a (mon.0) 3276 : audit [DBG] from='client.? 172.21.15.6:0/3137691793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:07.862 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:07 smithi130 bash[51302]: debug 2023-08-28T00:29:07.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:07.554442+0000) 2023-08-28T00:29:08.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:07 smithi130 bash[27579]: cluster 2023-08-28T00:29:06.638346+0000 mgr.y (mgr.14901) 4516 : cluster [DBG] pgmap v4523: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:08.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:07 smithi006 bash[27292]: cluster 2023-08-28T00:29:06.638346+0000 mgr.y (mgr.14901) 4516 : cluster [DBG] pgmap v4523: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:08.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:07 smithi006 bash[33918]: cluster 2023-08-28T00:29:06.638346+0000 mgr.y (mgr.14901) 4516 : cluster [DBG] pgmap v4523: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:08.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:08 smithi130 bash[51302]: debug 2023-08-28T00:29:08.547+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:08.554783+0000) 2023-08-28T00:29:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:08 smithi006 bash[27292]: audit 2023-08-28T00:29:08.656897+0000 mon.c (mon.2) 1747 : audit [DBG] from='client.? 172.21.15.6:0/2569906220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:09.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:08 smithi006 bash[33918]: audit 2023-08-28T00:29:08.656897+0000 mon.c (mon.2) 1747 : audit [DBG] from='client.? 172.21.15.6:0/2569906220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:09.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:08 smithi130 bash[27579]: audit 2023-08-28T00:29:08.656897+0000 mon.c (mon.2) 1747 : audit [DBG] from='client.? 172.21.15.6:0/2569906220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:09.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:09 smithi130 bash[51302]: debug 2023-08-28T00:29:09.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:09.554990+0000) 2023-08-28T00:29:10.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:09 smithi006 bash[27292]: cluster 2023-08-28T00:29:08.638983+0000 mgr.y (mgr.14901) 4517 : cluster [DBG] pgmap v4524: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:10.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:09 smithi006 bash[33918]: cluster 2023-08-28T00:29:08.638983+0000 mgr.y (mgr.14901) 4517 : cluster [DBG] pgmap v4524: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:10.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:09 smithi130 bash[27579]: cluster 2023-08-28T00:29:08.638983+0000 mgr.y (mgr.14901) 4517 : cluster [DBG] pgmap v4524: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:10 smithi130 bash[51302]: debug 2023-08-28T00:29:10.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:10.555209+0000) 2023-08-28T00:29:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:29:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:29:10] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:29:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:29:10.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:29:10.749Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:11.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:11 smithi130 bash[51302]: debug 2023-08-28T00:29:11.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:11.555436+0000) 2023-08-28T00:29:12.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:11 smithi006 bash[33918]: cluster 2023-08-28T00:29:10.640114+0000 mgr.y (mgr.14901) 4518 : cluster [DBG] pgmap v4525: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:12.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:11 smithi006 bash[33918]: audit 2023-08-28T00:29:11.082252+0000 mon.a (mon.0) 3277 : audit [DBG] from='client.? 172.21.15.6:0/2915077936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:12.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:11 smithi006 bash[27292]: cluster 2023-08-28T00:29:10.640114+0000 mgr.y (mgr.14901) 4518 : cluster [DBG] pgmap v4525: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:12.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:11 smithi006 bash[27292]: audit 2023-08-28T00:29:11.082252+0000 mon.a (mon.0) 3277 : audit [DBG] from='client.? 172.21.15.6:0/2915077936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:12.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:11 smithi130 bash[27579]: cluster 2023-08-28T00:29:10.640114+0000 mgr.y (mgr.14901) 4518 : cluster [DBG] pgmap v4525: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:12.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:11 smithi130 bash[27579]: audit 2023-08-28T00:29:11.082252+0000 mon.a (mon.0) 3277 : audit [DBG] from='client.? 172.21.15.6:0/2915077936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:29:12.327Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:29:12.329Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:12 smithi130 bash[51302]: debug 2023-08-28T00:29:12.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:12.555725+0000) 2023-08-28T00:29:13.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:13 smithi130 bash[51302]: debug 2023-08-28T00:29:13.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:13.556009+0000) 2023-08-28T00:29:14.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:13 smithi006 bash[27292]: cluster 2023-08-28T00:29:12.640811+0000 mgr.y (mgr.14901) 4519 : cluster [DBG] pgmap v4526: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:14.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:13 smithi006 bash[27292]: audit 2023-08-28T00:29:13.491574+0000 mon.a (mon.0) 3278 : audit [DBG] from='client.? 172.21.15.6:0/1710509791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:14.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:13 smithi006 bash[33918]: cluster 2023-08-28T00:29:12.640811+0000 mgr.y (mgr.14901) 4519 : cluster [DBG] pgmap v4526: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:13 smithi006 bash[33918]: audit 2023-08-28T00:29:13.491574+0000 mon.a (mon.0) 3278 : audit [DBG] from='client.? 172.21.15.6:0/1710509791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:14.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:13 smithi130 bash[27579]: cluster 2023-08-28T00:29:12.640811+0000 mgr.y (mgr.14901) 4519 : cluster [DBG] pgmap v4526: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:14.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:13 smithi130 bash[27579]: audit 2023-08-28T00:29:13.491574+0000 mon.a (mon.0) 3278 : audit [DBG] from='client.? 172.21.15.6:0/1710509791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:29:14.210Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:14.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:29:14.213Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:14 smithi130 bash[51302]: debug 2023-08-28T00:29:14.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:14.556286+0000) 2023-08-28T00:29:15.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:15 smithi130 bash[51302]: debug 2023-08-28T00:29:15.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:15.556568+0000) 2023-08-28T00:29:16.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:15 smithi006 bash[27292]: cluster 2023-08-28T00:29:14.641951+0000 mgr.y (mgr.14901) 4520 : cluster [DBG] pgmap v4527: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:16.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:15 smithi006 bash[33918]: cluster 2023-08-28T00:29:14.641951+0000 mgr.y (mgr.14901) 4520 : cluster [DBG] pgmap v4527: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:15 smithi130 bash[27579]: cluster 2023-08-28T00:29:14.641951+0000 mgr.y (mgr.14901) 4520 : cluster [DBG] pgmap v4527: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:16.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:16 smithi130 bash[51302]: debug 2023-08-28T00:29:16.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:16.556848+0000) 2023-08-28T00:29:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:16 smithi006 bash[33918]: audit 2023-08-28T00:29:15.899485+0000 mon.c (mon.2) 1748 : audit [DBG] from='client.? 172.21.15.6:0/36332750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:17.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:16 smithi006 bash[27292]: audit 2023-08-28T00:29:15.899485+0000 mon.c (mon.2) 1748 : audit [DBG] from='client.? 172.21.15.6:0/36332750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:17.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:16 smithi130 bash[27579]: audit 2023-08-28T00:29:15.899485+0000 mon.c (mon.2) 1748 : audit [DBG] from='client.? 172.21.15.6:0/36332750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:17.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:17 smithi130 bash[51302]: debug 2023-08-28T00:29:17.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:17.557109+0000) 2023-08-28T00:29:18.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:17 smithi006 bash[27292]: cluster 2023-08-28T00:29:16.642649+0000 mgr.y (mgr.14901) 4521 : cluster [DBG] pgmap v4528: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:18.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:17 smithi006 bash[33918]: cluster 2023-08-28T00:29:16.642649+0000 mgr.y (mgr.14901) 4521 : cluster [DBG] pgmap v4528: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:17 smithi130 bash[27579]: cluster 2023-08-28T00:29:16.642649+0000 mgr.y (mgr.14901) 4521 : cluster [DBG] pgmap v4528: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:18 smithi130 bash[51302]: debug 2023-08-28T00:29:18.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:18.557384+0000) 2023-08-28T00:29:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:18 smithi006 bash[33918]: audit 2023-08-28T00:29:18.310496+0000 mon.c (mon.2) 1749 : audit [DBG] from='client.? 172.21.15.6:0/746062647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:19.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:18 smithi006 bash[27292]: audit 2023-08-28T00:29:18.310496+0000 mon.c (mon.2) 1749 : audit [DBG] from='client.? 172.21.15.6:0/746062647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:19.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:18 smithi130 bash[27579]: audit 2023-08-28T00:29:18.310496+0000 mon.c (mon.2) 1749 : audit [DBG] from='client.? 172.21.15.6:0/746062647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:19 smithi130 bash[51302]: debug 2023-08-28T00:29:19.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:19.557712+0000) 2023-08-28T00:29:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:19 smithi006 bash[27292]: cluster 2023-08-28T00:29:18.643325+0000 mgr.y (mgr.14901) 4522 : cluster [DBG] pgmap v4529: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:20.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:19 smithi006 bash[33918]: cluster 2023-08-28T00:29:18.643325+0000 mgr.y (mgr.14901) 4522 : cluster [DBG] pgmap v4529: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:19 smithi130 bash[27579]: cluster 2023-08-28T00:29:18.643325+0000 mgr.y (mgr.14901) 4522 : cluster [DBG] pgmap v4529: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:20 smithi130 bash[51302]: debug 2023-08-28T00:29:20.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:20.557993+0000) 2023-08-28T00:29:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:29:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:29:20] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:29:21.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:20 smithi006 bash[33918]: audit 2023-08-28T00:29:20.736964+0000 mon.a (mon.0) 3279 : audit [DBG] from='client.? 172.21.15.6:0/3332370861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:21.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:20 smithi006 bash[27292]: audit 2023-08-28T00:29:20.736964+0000 mon.a (mon.0) 3279 : audit [DBG] from='client.? 172.21.15.6:0/3332370861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:29:20.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:29:20.749Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:21.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:20 smithi130 bash[27579]: audit 2023-08-28T00:29:20.736964+0000 mon.a (mon.0) 3279 : audit [DBG] from='client.? 172.21.15.6:0/3332370861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:21.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:21 smithi130 bash[51302]: debug 2023-08-28T00:29:21.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:21.558290+0000) 2023-08-28T00:29:22.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:21 smithi006 bash[33918]: cluster 2023-08-28T00:29:20.644352+0000 mgr.y (mgr.14901) 4523 : cluster [DBG] pgmap v4530: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:22.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:21 smithi006 bash[27292]: cluster 2023-08-28T00:29:20.644352+0000 mgr.y (mgr.14901) 4523 : cluster [DBG] pgmap v4530: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:22.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:21 smithi130 bash[27579]: cluster 2023-08-28T00:29:20.644352+0000 mgr.y (mgr.14901) 4523 : cluster [DBG] pgmap v4530: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:29:22.328Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:29:22.330Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:22 smithi130 bash[51302]: debug 2023-08-28T00:29:22.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:22.558584+0000) 2023-08-28T00:29:23.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:23 smithi130 bash[51302]: debug 2023-08-28T00:29:23.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:23.558851+0000) 2023-08-28T00:29:24.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:23 smithi006 bash[33918]: cluster 2023-08-28T00:29:22.645052+0000 mgr.y (mgr.14901) 4524 : cluster [DBG] pgmap v4531: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:24.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:23 smithi006 bash[33918]: audit 2023-08-28T00:29:23.148538+0000 mon.a (mon.0) 3280 : audit [DBG] from='client.? 172.21.15.6:0/3813646191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:23 smithi006 bash[27292]: cluster 2023-08-28T00:29:22.645052+0000 mgr.y (mgr.14901) 4524 : cluster [DBG] pgmap v4531: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:23 smithi006 bash[27292]: audit 2023-08-28T00:29:23.148538+0000 mon.a (mon.0) 3280 : audit [DBG] from='client.? 172.21.15.6:0/3813646191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:23 smithi130 bash[27579]: cluster 2023-08-28T00:29:22.645052+0000 mgr.y (mgr.14901) 4524 : cluster [DBG] pgmap v4531: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:23 smithi130 bash[27579]: audit 2023-08-28T00:29:23.148538+0000 mon.a (mon.0) 3280 : audit [DBG] from='client.? 172.21.15.6:0/3813646191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:29:24.211Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:24.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:29:24.213Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:24 smithi130 bash[51302]: debug 2023-08-28T00:29:24.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:24.559139+0000) 2023-08-28T00:29:25.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:25 smithi130 bash[51302]: debug 2023-08-28T00:29:25.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:25.559351+0000) 2023-08-28T00:29:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:25 smithi006 bash[27292]: cluster 2023-08-28T00:29:24.646180+0000 mgr.y (mgr.14901) 4525 : cluster [DBG] pgmap v4532: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:26.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:25 smithi006 bash[27292]: audit 2023-08-28T00:29:25.563624+0000 mon.a (mon.0) 3281 : audit [DBG] from='client.? 172.21.15.6:0/1668916751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:26.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:25 smithi006 bash[33918]: cluster 2023-08-28T00:29:24.646180+0000 mgr.y (mgr.14901) 4525 : cluster [DBG] pgmap v4532: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:26.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:25 smithi006 bash[33918]: audit 2023-08-28T00:29:25.563624+0000 mon.a (mon.0) 3281 : audit [DBG] from='client.? 172.21.15.6:0/1668916751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:25 smithi130 bash[27579]: cluster 2023-08-28T00:29:24.646180+0000 mgr.y (mgr.14901) 4525 : cluster [DBG] pgmap v4532: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:25 smithi130 bash[27579]: audit 2023-08-28T00:29:25.563624+0000 mon.a (mon.0) 3281 : audit [DBG] from='client.? 172.21.15.6:0/1668916751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:26.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:26 smithi130 bash[51302]: debug 2023-08-28T00:29:26.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:26.559663+0000) 2023-08-28T00:29:27.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:27 smithi130 bash[51302]: debug 2023-08-28T00:29:27.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:27.559944+0000) 2023-08-28T00:29:28.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:27 smithi006 bash[27292]: cluster 2023-08-28T00:29:26.646863+0000 mgr.y (mgr.14901) 4526 : cluster [DBG] pgmap v4533: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:28.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:27 smithi006 bash[33918]: cluster 2023-08-28T00:29:26.646863+0000 mgr.y (mgr.14901) 4526 : cluster [DBG] pgmap v4533: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:28.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:27 smithi130 bash[27579]: cluster 2023-08-28T00:29:26.646863+0000 mgr.y (mgr.14901) 4526 : cluster [DBG] pgmap v4533: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:28 smithi130 bash[51302]: debug 2023-08-28T00:29:28.559+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:28.560218+0000) 2023-08-28T00:29:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:28 smithi006 bash[27292]: audit 2023-08-28T00:29:27.976946+0000 mon.a (mon.0) 3282 : audit [DBG] from='client.? 172.21.15.6:0/3109712073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:29.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:28 smithi006 bash[33918]: audit 2023-08-28T00:29:27.976946+0000 mon.a (mon.0) 3282 : audit [DBG] from='client.? 172.21.15.6:0/3109712073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:29.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:28 smithi130 bash[27579]: audit 2023-08-28T00:29:27.976946+0000 mon.a (mon.0) 3282 : audit [DBG] from='client.? 172.21.15.6:0/3109712073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:29.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:29 smithi130 bash[51302]: debug 2023-08-28T00:29:29.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:29.560545+0000) 2023-08-28T00:29:30.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:29 smithi006 bash[33918]: cluster 2023-08-28T00:29:28.647555+0000 mgr.y (mgr.14901) 4527 : cluster [DBG] pgmap v4534: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:30.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:29 smithi006 bash[27292]: cluster 2023-08-28T00:29:28.647555+0000 mgr.y (mgr.14901) 4527 : cluster [DBG] pgmap v4534: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:29 smithi130 bash[27579]: cluster 2023-08-28T00:29:28.647555+0000 mgr.y (mgr.14901) 4527 : cluster [DBG] pgmap v4534: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:30 smithi130 bash[51302]: debug 2023-08-28T00:29:30.558+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:30.560848+0000) 2023-08-28T00:29:31.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:30 smithi006 bash[27292]: audit 2023-08-28T00:29:30.403581+0000 mon.c (mon.2) 1750 : audit [DBG] from='client.? 172.21.15.6:0/205784451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:31.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:29:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:29:30] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:29:31.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:30 smithi006 bash[33918]: audit 2023-08-28T00:29:30.403581+0000 mon.c (mon.2) 1750 : audit [DBG] from='client.? 172.21.15.6:0/205784451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:29:30.749Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:29:30.751Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:31.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:30 smithi130 bash[27579]: audit 2023-08-28T00:29:30.403581+0000 mon.c (mon.2) 1750 : audit [DBG] from='client.? 172.21.15.6:0/205784451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:31.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:31 smithi130 bash[51302]: debug 2023-08-28T00:29:31.558+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:31.561128+0000) 2023-08-28T00:29:32.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:31 smithi006 bash[27292]: cluster 2023-08-28T00:29:30.648706+0000 mgr.y (mgr.14901) 4528 : cluster [DBG] pgmap v4535: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:32.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:31 smithi006 bash[33918]: cluster 2023-08-28T00:29:30.648706+0000 mgr.y (mgr.14901) 4528 : cluster [DBG] pgmap v4535: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:31 smithi130 bash[27579]: cluster 2023-08-28T00:29:30.648706+0000 mgr.y (mgr.14901) 4528 : cluster [DBG] pgmap v4535: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:29:32.329Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:29:32.331Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:32 smithi130 bash[51302]: debug 2023-08-28T00:29:32.554+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:32.561352+0000) 2023-08-28T00:29:33.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:32 smithi006 bash[27292]: audit 2023-08-28T00:29:32.816343+0000 mon.c (mon.2) 1751 : audit [DBG] from='client.? 172.21.15.6:0/1261650283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:33.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:32 smithi006 bash[33918]: audit 2023-08-28T00:29:32.816343+0000 mon.c (mon.2) 1751 : audit [DBG] from='client.? 172.21.15.6:0/1261650283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:32 smithi130 bash[27579]: audit 2023-08-28T00:29:32.816343+0000 mon.c (mon.2) 1751 : audit [DBG] from='client.? 172.21.15.6:0/1261650283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:33.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:33 smithi130 bash[51302]: debug 2023-08-28T00:29:33.554+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:33.561611+0000) 2023-08-28T00:29:34.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:33 smithi130 bash[27579]: cluster 2023-08-28T00:29:32.649374+0000 mgr.y (mgr.14901) 4529 : cluster [DBG] pgmap v4536: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:34.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:33 smithi006 bash[33918]: cluster 2023-08-28T00:29:32.649374+0000 mgr.y (mgr.14901) 4529 : cluster [DBG] pgmap v4536: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:34.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:33 smithi006 bash[27292]: cluster 2023-08-28T00:29:32.649374+0000 mgr.y (mgr.14901) 4529 : cluster [DBG] pgmap v4536: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:29:34.212Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:29:34.215Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:34 smithi130 bash[51302]: debug 2023-08-28T00:29:34.558+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:34.561927+0000) 2023-08-28T00:29:35.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:35 smithi130 bash[51302]: debug 2023-08-28T00:29:35.558+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:35.562224+0000) 2023-08-28T00:29:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:35 smithi130 bash[27579]: cluster 2023-08-28T00:29:34.650482+0000 mgr.y (mgr.14901) 4530 : cluster [DBG] pgmap v4537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:35 smithi130 bash[27579]: audit 2023-08-28T00:29:35.234128+0000 mon.c (mon.2) 1752 : audit [DBG] from='client.? 172.21.15.6:0/2488141241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:36.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:35 smithi006 bash[33918]: cluster 2023-08-28T00:29:34.650482+0000 mgr.y (mgr.14901) 4530 : cluster [DBG] pgmap v4537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:36.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:35 smithi006 bash[33918]: audit 2023-08-28T00:29:35.234128+0000 mon.c (mon.2) 1752 : audit [DBG] from='client.? 172.21.15.6:0/2488141241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:36.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:35 smithi006 bash[27292]: cluster 2023-08-28T00:29:34.650482+0000 mgr.y (mgr.14901) 4530 : cluster [DBG] pgmap v4537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:36.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:35 smithi006 bash[27292]: audit 2023-08-28T00:29:35.234128+0000 mon.c (mon.2) 1752 : audit [DBG] from='client.? 172.21.15.6:0/2488141241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:36 smithi130 bash[51302]: debug 2023-08-28T00:29:36.558+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:36.562486+0000) 2023-08-28T00:29:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:37 smithi130 bash[51302]: debug 2023-08-28T00:29:37.558+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:37.562763+0000) 2023-08-28T00:29:38.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:37 smithi130 bash[27579]: cluster 2023-08-28T00:29:36.651168+0000 mgr.y (mgr.14901) 4531 : cluster [DBG] pgmap v4538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:38.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:37 smithi130 bash[27579]: audit 2023-08-28T00:29:37.639620+0000 mon.a (mon.0) 3283 : audit [DBG] from='client.? 172.21.15.6:0/3657962372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:38.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:37 smithi006 bash[27292]: cluster 2023-08-28T00:29:36.651168+0000 mgr.y (mgr.14901) 4531 : cluster [DBG] pgmap v4538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:38.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:37 smithi006 bash[27292]: audit 2023-08-28T00:29:37.639620+0000 mon.a (mon.0) 3283 : audit [DBG] from='client.? 172.21.15.6:0/3657962372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:38.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:37 smithi006 bash[33918]: cluster 2023-08-28T00:29:36.651168+0000 mgr.y (mgr.14901) 4531 : cluster [DBG] pgmap v4538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:38.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:37 smithi006 bash[33918]: audit 2023-08-28T00:29:37.639620+0000 mon.a (mon.0) 3283 : audit [DBG] from='client.? 172.21.15.6:0/3657962372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:38 smithi130 bash[51302]: debug 2023-08-28T00:29:38.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:38.563032+0000) 2023-08-28T00:29:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:39 smithi130 bash[51302]: debug 2023-08-28T00:29:39.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:39.563270+0000) 2023-08-28T00:29:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:39 smithi130 bash[27579]: cluster 2023-08-28T00:29:38.651859+0000 mgr.y (mgr.14901) 4532 : cluster [DBG] pgmap v4539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:40.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:39 smithi006 bash[27292]: cluster 2023-08-28T00:29:38.651859+0000 mgr.y (mgr.14901) 4532 : cluster [DBG] pgmap v4539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:39 smithi006 bash[33918]: cluster 2023-08-28T00:29:38.651859+0000 mgr.y (mgr.14901) 4532 : cluster [DBG] pgmap v4539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:40 smithi130 bash[51302]: debug 2023-08-28T00:29:40.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:40.563543+0000) 2023-08-28T00:29:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:40 smithi006 bash[27292]: audit 2023-08-28T00:29:40.055417+0000 mon.a (mon.0) 3284 : audit [DBG] from='client.? 172.21.15.6:0/1520551661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:41.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:29:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:29:40] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:29:41.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:40 smithi006 bash[33918]: audit 2023-08-28T00:29:40.055417+0000 mon.a (mon.0) 3284 : audit [DBG] from='client.? 172.21.15.6:0/1520551661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:29:40.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:29:40.752Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:41.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:40 smithi130 bash[27579]: audit 2023-08-28T00:29:40.055417+0000 mon.a (mon.0) 3284 : audit [DBG] from='client.? 172.21.15.6:0/1520551661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:41.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:41 smithi130 bash[51302]: debug 2023-08-28T00:29:41.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:41.563850+0000) 2023-08-28T00:29:42.333 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:41 smithi006 bash[33918]: cluster 2023-08-28T00:29:40.653039+0000 mgr.y (mgr.14901) 4533 : cluster [DBG] pgmap v4540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:42.334 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:41 smithi006 bash[27292]: cluster 2023-08-28T00:29:40.653039+0000 mgr.y (mgr.14901) 4533 : cluster [DBG] pgmap v4540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:41 smithi130 bash[27579]: cluster 2023-08-28T00:29:40.653039+0000 mgr.y (mgr.14901) 4533 : cluster [DBG] pgmap v4540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:29:42.330Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:29:42.332Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:42 smithi130 bash[51302]: debug 2023-08-28T00:29:42.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:42.564112+0000) 2023-08-28T00:29:43.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:43 smithi130 bash[27579]: audit 2023-08-28T00:29:42.469491+0000 mon.a (mon.0) 3285 : audit [DBG] from='client.? 172.21.15.6:0/1655098031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:43.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:43 smithi006 bash[33918]: audit 2023-08-28T00:29:42.469491+0000 mon.a (mon.0) 3285 : audit [DBG] from='client.? 172.21.15.6:0/1655098031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:43.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:43 smithi006 bash[27292]: audit 2023-08-28T00:29:42.469491+0000 mon.a (mon.0) 3285 : audit [DBG] from='client.? 172.21.15.6:0/1655098031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:43.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:43 smithi130 bash[51302]: debug 2023-08-28T00:29:43.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:43.564431+0000) 2023-08-28T00:29:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:44 smithi130 bash[27579]: cluster 2023-08-28T00:29:42.653670+0000 mgr.y (mgr.14901) 4534 : cluster [DBG] pgmap v4541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:44.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:44 smithi006 bash[27292]: cluster 2023-08-28T00:29:42.653670+0000 mgr.y (mgr.14901) 4534 : cluster [DBG] pgmap v4541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:29:44.212Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:29:44.215Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:44.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:44 smithi006 bash[33918]: cluster 2023-08-28T00:29:42.653670+0000 mgr.y (mgr.14901) 4534 : cluster [DBG] pgmap v4541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:44 smithi130 bash[51302]: debug 2023-08-28T00:29:44.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:44.564718+0000) 2023-08-28T00:29:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:45 smithi130 bash[27579]: audit 2023-08-28T00:29:44.896166+0000 mon.c (mon.2) 1753 : audit [DBG] from='client.? 172.21.15.6:0/2269291367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:45.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:45 smithi006 bash[33918]: audit 2023-08-28T00:29:44.896166+0000 mon.c (mon.2) 1753 : audit [DBG] from='client.? 172.21.15.6:0/2269291367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:45.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:45 smithi006 bash[27292]: audit 2023-08-28T00:29:44.896166+0000 mon.c (mon.2) 1753 : audit [DBG] from='client.? 172.21.15.6:0/2269291367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:45.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:45 smithi130 bash[51302]: debug 2023-08-28T00:29:45.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:45.564998+0000) 2023-08-28T00:29:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:46 smithi130 bash[27579]: cluster 2023-08-28T00:29:44.654791+0000 mgr.y (mgr.14901) 4535 : cluster [DBG] pgmap v4542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:46.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:46 smithi006 bash[27292]: cluster 2023-08-28T00:29:44.654791+0000 mgr.y (mgr.14901) 4535 : cluster [DBG] pgmap v4542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:46.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:46 smithi006 bash[33918]: cluster 2023-08-28T00:29:44.654791+0000 mgr.y (mgr.14901) 4535 : cluster [DBG] pgmap v4542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:46.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:46 smithi130 bash[51302]: debug 2023-08-28T00:29:46.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:46.565280+0000) 2023-08-28T00:29:47.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:47 smithi130 bash[51302]: debug 2023-08-28T00:29:47.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:47.565560+0000) 2023-08-28T00:29:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:48 smithi130 bash[27579]: cluster 2023-08-28T00:29:46.655476+0000 mgr.y (mgr.14901) 4536 : cluster [DBG] pgmap v4543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:48 smithi130 bash[27579]: audit 2023-08-28T00:29:47.311547+0000 mon.c (mon.2) 1754 : audit [DBG] from='client.? 172.21.15.6:0/1534887405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:48.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:48 smithi006 bash[27292]: cluster 2023-08-28T00:29:46.655476+0000 mgr.y (mgr.14901) 4536 : cluster [DBG] pgmap v4543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:48.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:48 smithi006 bash[27292]: audit 2023-08-28T00:29:47.311547+0000 mon.c (mon.2) 1754 : audit [DBG] from='client.? 172.21.15.6:0/1534887405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:48.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:48 smithi006 bash[33918]: cluster 2023-08-28T00:29:46.655476+0000 mgr.y (mgr.14901) 4536 : cluster [DBG] pgmap v4543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:48.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:48 smithi006 bash[33918]: audit 2023-08-28T00:29:47.311547+0000 mon.c (mon.2) 1754 : audit [DBG] from='client.? 172.21.15.6:0/1534887405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:48 smithi130 bash[51302]: debug 2023-08-28T00:29:48.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:48.565866+0000) 2023-08-28T00:29:49.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:49 smithi130 bash[51302]: debug 2023-08-28T00:29:49.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:49.566080+0000) 2023-08-28T00:29:50.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:50 smithi130 bash[27579]: cluster 2023-08-28T00:29:48.656137+0000 mgr.y (mgr.14901) 4537 : cluster [DBG] pgmap v4544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:50.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:50 smithi130 bash[27579]: audit 2023-08-28T00:29:49.723820+0000 mon.c (mon.2) 1755 : audit [DBG] from='client.? 172.21.15.6:0/2812541341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:50.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:50 smithi006 bash[33918]: cluster 2023-08-28T00:29:48.656137+0000 mgr.y (mgr.14901) 4537 : cluster [DBG] pgmap v4544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:50.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:50 smithi006 bash[33918]: audit 2023-08-28T00:29:49.723820+0000 mon.c (mon.2) 1755 : audit [DBG] from='client.? 172.21.15.6:0/2812541341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:50.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:50 smithi006 bash[27292]: cluster 2023-08-28T00:29:48.656137+0000 mgr.y (mgr.14901) 4537 : cluster [DBG] pgmap v4544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:50.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:50 smithi006 bash[27292]: audit 2023-08-28T00:29:49.723820+0000 mon.c (mon.2) 1755 : audit [DBG] from='client.? 172.21.15.6:0/2812541341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:50 smithi130 bash[51302]: debug 2023-08-28T00:29:50.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:50.566400+0000) 2023-08-28T00:29:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:29:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:29:50] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:29:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:29:50.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:29:50.753Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:51.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:51 smithi130 bash[51302]: debug 2023-08-28T00:29:51.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:51.566715+0000) 2023-08-28T00:29:52.335 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:52 smithi006 bash[33918]: cluster 2023-08-28T00:29:50.657343+0000 mgr.y (mgr.14901) 4538 : cluster [DBG] pgmap v4545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:52.335 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:52 smithi006 bash[27292]: cluster 2023-08-28T00:29:50.657343+0000 mgr.y (mgr.14901) 4538 : cluster [DBG] pgmap v4545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:52 smithi130 bash[27579]: cluster 2023-08-28T00:29:50.657343+0000 mgr.y (mgr.14901) 4538 : cluster [DBG] pgmap v4545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:29:52.331Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:29:52.333Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:52.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:52 smithi130 bash[51302]: debug 2023-08-28T00:29:52.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:52.567072+0000) 2023-08-28T00:29:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:53 smithi130 bash[27579]: audit 2023-08-28T00:29:52.138318+0000 mon.a (mon.0) 3286 : audit [DBG] from='client.? 172.21.15.6:0/4041398685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:53.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:53 smithi006 bash[33918]: audit 2023-08-28T00:29:52.138318+0000 mon.a (mon.0) 3286 : audit [DBG] from='client.? 172.21.15.6:0/4041398685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:53.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:53 smithi006 bash[27292]: audit 2023-08-28T00:29:52.138318+0000 mon.a (mon.0) 3286 : audit [DBG] from='client.? 172.21.15.6:0/4041398685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:53.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:53 smithi130 bash[51302]: debug 2023-08-28T00:29:53.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:53.567255+0000) 2023-08-28T00:29:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:54 smithi130 bash[27579]: cluster 2023-08-28T00:29:52.657998+0000 mgr.y (mgr.14901) 4539 : cluster [DBG] pgmap v4546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:29:54.213Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:54.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:29:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:29:54.215Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:29:54.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:54 smithi006 bash[33918]: cluster 2023-08-28T00:29:52.657998+0000 mgr.y (mgr.14901) 4539 : cluster [DBG] pgmap v4546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:54.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:54 smithi006 bash[27292]: cluster 2023-08-28T00:29:52.657998+0000 mgr.y (mgr.14901) 4539 : cluster [DBG] pgmap v4546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:54.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:54 smithi130 bash[51302]: debug 2023-08-28T00:29:54.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:54.567468+0000) 2023-08-28T00:29:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:55 smithi130 bash[27579]: audit 2023-08-28T00:29:54.562200+0000 mon.a (mon.0) 3287 : audit [DBG] from='client.? 172.21.15.6:0/2295984028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:55.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:55 smithi006 bash[33918]: audit 2023-08-28T00:29:54.562200+0000 mon.a (mon.0) 3287 : audit [DBG] from='client.? 172.21.15.6:0/2295984028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:55.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:55 smithi006 bash[27292]: audit 2023-08-28T00:29:54.562200+0000 mon.a (mon.0) 3287 : audit [DBG] from='client.? 172.21.15.6:0/2295984028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:55.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:55 smithi130 bash[51302]: debug 2023-08-28T00:29:55.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:55.567680+0000) 2023-08-28T00:29:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:56 smithi130 bash[27579]: cluster 2023-08-28T00:29:54.659137+0000 mgr.y (mgr.14901) 4540 : cluster [DBG] pgmap v4547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:56.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:56 smithi006 bash[33918]: cluster 2023-08-28T00:29:54.659137+0000 mgr.y (mgr.14901) 4540 : cluster [DBG] pgmap v4547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:56.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:56 smithi006 bash[27292]: cluster 2023-08-28T00:29:54.659137+0000 mgr.y (mgr.14901) 4540 : cluster [DBG] pgmap v4547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:56.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:56 smithi130 bash[51302]: debug 2023-08-28T00:29:56.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:56.567984+0000) 2023-08-28T00:29:57.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:57 smithi130 bash[27579]: audit 2023-08-28T00:29:56.974523+0000 mon.c (mon.2) 1756 : audit [DBG] from='client.? 172.21.15.6:0/3134129790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:57.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:57 smithi006 bash[27292]: audit 2023-08-28T00:29:56.974523+0000 mon.c (mon.2) 1756 : audit [DBG] from='client.? 172.21.15.6:0/3134129790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:57.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:57 smithi006 bash[33918]: audit 2023-08-28T00:29:56.974523+0000 mon.c (mon.2) 1756 : audit [DBG] from='client.? 172.21.15.6:0/3134129790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:57.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:57 smithi130 bash[51302]: debug 2023-08-28T00:29:57.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:57.568286+0000) 2023-08-28T00:29:58.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:29:58 smithi130 bash[27579]: cluster 2023-08-28T00:29:56.659805+0000 mgr.y (mgr.14901) 4541 : cluster [DBG] pgmap v4548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:58.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:29:58 smithi006 bash[27292]: cluster 2023-08-28T00:29:56.659805+0000 mgr.y (mgr.14901) 4541 : cluster [DBG] pgmap v4548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:58.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:29:58 smithi006 bash[33918]: cluster 2023-08-28T00:29:56.659805+0000 mgr.y (mgr.14901) 4541 : cluster [DBG] pgmap v4548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:29:58.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:58 smithi130 bash[51302]: debug 2023-08-28T00:29:58.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:58.568616+0000) 2023-08-28T00:29:59.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:29:59 smithi130 bash[51302]: debug 2023-08-28T00:29:59.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:59.568896+0000) 2023-08-28T00:30:00.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:00 smithi130 bash[27579]: cluster 2023-08-28T00:29:58.660435+0000 mgr.y (mgr.14901) 4542 : cluster [DBG] pgmap v4549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:00.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:00 smithi130 bash[27579]: audit 2023-08-28T00:29:59.390366+0000 mon.a (mon.0) 3288 : audit [DBG] from='client.? 172.21.15.6:0/2898650307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:00.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:00 smithi130 bash[27579]: audit 2023-08-28T00:29:59.615939+0000 mon.b (mon.1) 556 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:30:00.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:00 smithi130 bash[27579]: cluster 2023-08-28T00:30:00.000173+0000 mon.a (mon.0) 3289 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:30:00.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:00 smithi006 bash[27292]: cluster 2023-08-28T00:29:58.660435+0000 mgr.y (mgr.14901) 4542 : cluster [DBG] pgmap v4549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:00.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:00 smithi006 bash[27292]: audit 2023-08-28T00:29:59.390366+0000 mon.a (mon.0) 3288 : audit [DBG] from='client.? 172.21.15.6:0/2898650307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:00.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:00 smithi006 bash[27292]: audit 2023-08-28T00:29:59.615939+0000 mon.b (mon.1) 556 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:30:00.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:00 smithi006 bash[27292]: cluster 2023-08-28T00:30:00.000173+0000 mon.a (mon.0) 3289 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:30:00.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:00 smithi006 bash[33918]: cluster 2023-08-28T00:29:58.660435+0000 mgr.y (mgr.14901) 4542 : cluster [DBG] pgmap v4549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:00.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:00 smithi006 bash[33918]: audit 2023-08-28T00:29:59.390366+0000 mon.a (mon.0) 3288 : audit [DBG] from='client.? 172.21.15.6:0/2898650307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:00.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:00 smithi006 bash[33918]: audit 2023-08-28T00:29:59.615939+0000 mon.b (mon.1) 556 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:30:00.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:00 smithi006 bash[33918]: cluster 2023-08-28T00:30:00.000173+0000 mon.a (mon.0) 3289 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:30:00.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:00 smithi130 bash[51302]: debug 2023-08-28T00:30:00.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:00.569161+0000) 2023-08-28T00:30:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:30:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:30:00] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:30:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:30:00.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:30:00.754Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:01.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:01 smithi130 bash[51302]: debug 2023-08-28T00:30:01.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:01.569426+0000) 2023-08-28T00:30:02.336 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:02 smithi006 bash[27292]: cluster 2023-08-28T00:30:00.661669+0000 mgr.y (mgr.14901) 4543 : cluster [DBG] pgmap v4550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:02.336 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:02 smithi006 bash[27292]: audit 2023-08-28T00:30:01.807841+0000 mon.c (mon.2) 1757 : audit [DBG] from='client.? 172.21.15.6:0/70149212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:02.336 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:02 smithi006 bash[33918]: cluster 2023-08-28T00:30:00.661669+0000 mgr.y (mgr.14901) 4543 : cluster [DBG] pgmap v4550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:02.336 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:02 smithi006 bash[33918]: audit 2023-08-28T00:30:01.807841+0000 mon.c (mon.2) 1757 : audit [DBG] from='client.? 172.21.15.6:0/70149212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:02.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:02 smithi130 bash[27579]: cluster 2023-08-28T00:30:00.661669+0000 mgr.y (mgr.14901) 4543 : cluster [DBG] pgmap v4550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:02.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:02 smithi130 bash[27579]: audit 2023-08-28T00:30:01.807841+0000 mon.c (mon.2) 1757 : audit [DBG] from='client.? 172.21.15.6:0/70149212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:30:02.332Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:30:02.334Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:02.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:02 smithi130 bash[51302]: debug 2023-08-28T00:30:02.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:02.569704+0000) 2023-08-28T00:30:03.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:03 smithi130 bash[51302]: debug 2023-08-28T00:30:03.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:03.569983+0000) 2023-08-28T00:30:04.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:04 smithi130 bash[27579]: cluster 2023-08-28T00:30:02.662334+0000 mgr.y (mgr.14901) 4544 : cluster [DBG] pgmap v4551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:04.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:04 smithi006 bash[33918]: cluster 2023-08-28T00:30:02.662334+0000 mgr.y (mgr.14901) 4544 : cluster [DBG] pgmap v4551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:04.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:04 smithi006 bash[27292]: cluster 2023-08-28T00:30:02.662334+0000 mgr.y (mgr.14901) 4544 : cluster [DBG] pgmap v4551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:30:04.213Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:30:04.215Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:04.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:04 smithi130 bash[51302]: debug 2023-08-28T00:30:04.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:04.570233+0000) 2023-08-28T00:30:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:05 smithi130 bash[27579]: audit 2023-08-28T00:30:04.217398+0000 mon.c (mon.2) 1758 : audit [DBG] from='client.? 172.21.15.6:0/1504298646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:05.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:05 smithi006 bash[33918]: audit 2023-08-28T00:30:04.217398+0000 mon.c (mon.2) 1758 : audit [DBG] from='client.? 172.21.15.6:0/1504298646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:05.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:05 smithi006 bash[27292]: audit 2023-08-28T00:30:04.217398+0000 mon.c (mon.2) 1758 : audit [DBG] from='client.? 172.21.15.6:0/1504298646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:05.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:05 smithi130 bash[51302]: debug 2023-08-28T00:30:05.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:05.570541+0000) 2023-08-28T00:30:06.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:06 smithi130 bash[51302]: debug 2023-08-28T00:30:06.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:06.570839+0000) 2023-08-28T00:30:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:06 smithi130 bash[27579]: cluster 2023-08-28T00:30:04.663131+0000 mgr.y (mgr.14901) 4545 : cluster [DBG] pgmap v4552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:06 smithi130 bash[27579]: audit 2023-08-28T00:30:05.499115+0000 mon.a (mon.0) 3290 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:30:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:06 smithi130 bash[27579]: audit 2023-08-28T00:30:05.959244+0000 mon.a (mon.0) 3291 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:30:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:06 smithi130 bash[27579]: audit 2023-08-28T00:30:06.223800+0000 mon.b (mon.1) 557 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:30:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:06 smithi130 bash[27579]: audit 2023-08-28T00:30:06.226129+0000 mon.b (mon.1) 558 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:30:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:06 smithi130 bash[27579]: audit 2023-08-28T00:30:06.235360+0000 mon.a (mon.0) 3292 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:30:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:06 smithi006 bash[33918]: cluster 2023-08-28T00:30:04.663131+0000 mgr.y (mgr.14901) 4545 : cluster [DBG] pgmap v4552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:06.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:06 smithi006 bash[33918]: audit 2023-08-28T00:30:05.499115+0000 mon.a (mon.0) 3290 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:30:06.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:06 smithi006 bash[33918]: audit 2023-08-28T00:30:05.959244+0000 mon.a (mon.0) 3291 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:30:06.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:06 smithi006 bash[33918]: audit 2023-08-28T00:30:06.223800+0000 mon.b (mon.1) 557 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:30:06.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:06 smithi006 bash[33918]: audit 2023-08-28T00:30:06.226129+0000 mon.b (mon.1) 558 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:30:06.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:06 smithi006 bash[33918]: audit 2023-08-28T00:30:06.235360+0000 mon.a (mon.0) 3292 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:30:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:06 smithi006 bash[27292]: cluster 2023-08-28T00:30:04.663131+0000 mgr.y (mgr.14901) 4545 : cluster [DBG] pgmap v4552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:06 smithi006 bash[27292]: audit 2023-08-28T00:30:05.499115+0000 mon.a (mon.0) 3290 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:30:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:06 smithi006 bash[27292]: audit 2023-08-28T00:30:05.959244+0000 mon.a (mon.0) 3291 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:30:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:06 smithi006 bash[27292]: audit 2023-08-28T00:30:06.223800+0000 mon.b (mon.1) 557 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:30:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:06 smithi006 bash[27292]: audit 2023-08-28T00:30:06.226129+0000 mon.b (mon.1) 558 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:30:06.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:06 smithi006 bash[27292]: audit 2023-08-28T00:30:06.235360+0000 mon.a (mon.0) 3292 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:30:07.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:07 smithi130 bash[27579]: audit 2023-08-28T00:30:06.635115+0000 mon.a (mon.0) 3293 : audit [DBG] from='client.? 172.21.15.6:0/1169738834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:07.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:07 smithi130 bash[51302]: debug 2023-08-28T00:30:07.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:07.571094+0000) 2023-08-28T00:30:07.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:07 smithi006 bash[27292]: audit 2023-08-28T00:30:06.635115+0000 mon.a (mon.0) 3293 : audit [DBG] from='client.? 172.21.15.6:0/1169738834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:07.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:07 smithi006 bash[33918]: audit 2023-08-28T00:30:06.635115+0000 mon.a (mon.0) 3293 : audit [DBG] from='client.? 172.21.15.6:0/1169738834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:08 smithi130 bash[27579]: cluster 2023-08-28T00:30:06.663902+0000 mgr.y (mgr.14901) 4546 : cluster [DBG] pgmap v4553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:08.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:08 smithi130 bash[51302]: debug 2023-08-28T00:30:08.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:08.571390+0000) 2023-08-28T00:30:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:08 smithi006 bash[27292]: cluster 2023-08-28T00:30:06.663902+0000 mgr.y (mgr.14901) 4546 : cluster [DBG] pgmap v4553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:08.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:08 smithi006 bash[33918]: cluster 2023-08-28T00:30:06.663902+0000 mgr.y (mgr.14901) 4546 : cluster [DBG] pgmap v4553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:09.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:09 smithi130 bash[51302]: debug 2023-08-28T00:30:09.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:09.571714+0000) 2023-08-28T00:30:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:09 smithi130 bash[27579]: cluster 2023-08-28T00:30:08.664489+0000 mgr.y (mgr.14901) 4547 : cluster [DBG] pgmap v4554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:09.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:09 smithi130 bash[27579]: audit 2023-08-28T00:30:09.054679+0000 mon.c (mon.2) 1759 : audit [DBG] from='client.? 172.21.15.6:0/285304057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:09.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:09 smithi006 bash[33918]: cluster 2023-08-28T00:30:08.664489+0000 mgr.y (mgr.14901) 4547 : cluster [DBG] pgmap v4554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:09.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:09 smithi006 bash[33918]: audit 2023-08-28T00:30:09.054679+0000 mon.c (mon.2) 1759 : audit [DBG] from='client.? 172.21.15.6:0/285304057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:09.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:09 smithi006 bash[27292]: cluster 2023-08-28T00:30:08.664489+0000 mgr.y (mgr.14901) 4547 : cluster [DBG] pgmap v4554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:09.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:09 smithi006 bash[27292]: audit 2023-08-28T00:30:09.054679+0000 mon.c (mon.2) 1759 : audit [DBG] from='client.? 172.21.15.6:0/285304057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:10 smithi130 bash[51302]: debug 2023-08-28T00:30:10.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:10.571969+0000) 2023-08-28T00:30:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:30:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:30:10] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:30:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:30:10.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:30:10.755Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:11.860 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:11 smithi130 bash[51302]: debug 2023-08-28T00:30:11.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:11.572256+0000) 2023-08-28T00:30:12.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:11 smithi130 bash[27579]: cluster 2023-08-28T00:30:10.665586+0000 mgr.y (mgr.14901) 4548 : cluster [DBG] pgmap v4555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:12.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:11 smithi130 bash[27579]: audit 2023-08-28T00:30:11.466180+0000 mon.c (mon.2) 1760 : audit [DBG] from='client.? 172.21.15.6:0/3563518168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:12.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:11 smithi006 bash[33918]: cluster 2023-08-28T00:30:10.665586+0000 mgr.y (mgr.14901) 4548 : cluster [DBG] pgmap v4555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:12.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:11 smithi006 bash[33918]: audit 2023-08-28T00:30:11.466180+0000 mon.c (mon.2) 1760 : audit [DBG] from='client.? 172.21.15.6:0/3563518168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:12.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:11 smithi006 bash[27292]: cluster 2023-08-28T00:30:10.665586+0000 mgr.y (mgr.14901) 4548 : cluster [DBG] pgmap v4555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:12.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:11 smithi006 bash[27292]: audit 2023-08-28T00:30:11.466180+0000 mon.c (mon.2) 1760 : audit [DBG] from='client.? 172.21.15.6:0/3563518168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:30:12.333Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:30:12.335Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:12 smithi130 bash[51302]: debug 2023-08-28T00:30:12.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:12.572536+0000) 2023-08-28T00:30:13.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:13 smithi130 bash[51302]: debug 2023-08-28T00:30:13.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:13.572818+0000) 2023-08-28T00:30:14.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:13 smithi130 bash[27579]: cluster 2023-08-28T00:30:12.666214+0000 mgr.y (mgr.14901) 4549 : cluster [DBG] pgmap v4556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:14.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:13 smithi006 bash[33918]: cluster 2023-08-28T00:30:12.666214+0000 mgr.y (mgr.14901) 4549 : cluster [DBG] pgmap v4556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:14.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:13 smithi006 bash[27292]: cluster 2023-08-28T00:30:12.666214+0000 mgr.y (mgr.14901) 4549 : cluster [DBG] pgmap v4556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:30:14.214Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:30:14.217Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:14 smithi130 bash[51302]: debug 2023-08-28T00:30:14.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:14.573130+0000) 2023-08-28T00:30:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:14 smithi006 bash[33918]: audit 2023-08-28T00:30:13.905457+0000 mon.a (mon.0) 3294 : audit [DBG] from='client.? 172.21.15.6:0/434237142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:15.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:14 smithi006 bash[27292]: audit 2023-08-28T00:30:13.905457+0000 mon.a (mon.0) 3294 : audit [DBG] from='client.? 172.21.15.6:0/434237142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:14 smithi130 bash[27579]: audit 2023-08-28T00:30:13.905457+0000 mon.a (mon.0) 3294 : audit [DBG] from='client.? 172.21.15.6:0/434237142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:15.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:15 smithi130 bash[51302]: debug 2023-08-28T00:30:15.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:15.573440+0000) 2023-08-28T00:30:16.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:15 smithi006 bash[27292]: cluster 2023-08-28T00:30:14.667351+0000 mgr.y (mgr.14901) 4550 : cluster [DBG] pgmap v4557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:16.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:15 smithi006 bash[33918]: cluster 2023-08-28T00:30:14.667351+0000 mgr.y (mgr.14901) 4550 : cluster [DBG] pgmap v4557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:15 smithi130 bash[27579]: cluster 2023-08-28T00:30:14.667351+0000 mgr.y (mgr.14901) 4550 : cluster [DBG] pgmap v4557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:16.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:16 smithi130 bash[51302]: debug 2023-08-28T00:30:16.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:16.573822+0000) 2023-08-28T00:30:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:16 smithi006 bash[33918]: audit 2023-08-28T00:30:16.320747+0000 mon.c (mon.2) 1761 : audit [DBG] from='client.? 172.21.15.6:0/1230346097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:17.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:16 smithi006 bash[27292]: audit 2023-08-28T00:30:16.320747+0000 mon.c (mon.2) 1761 : audit [DBG] from='client.? 172.21.15.6:0/1230346097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:17.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:16 smithi130 bash[27579]: audit 2023-08-28T00:30:16.320747+0000 mon.c (mon.2) 1761 : audit [DBG] from='client.? 172.21.15.6:0/1230346097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:17.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:17 smithi130 bash[51302]: debug 2023-08-28T00:30:17.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:17.574136+0000) 2023-08-28T00:30:18.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:17 smithi006 bash[33918]: cluster 2023-08-28T00:30:16.668012+0000 mgr.y (mgr.14901) 4551 : cluster [DBG] pgmap v4558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:18.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:17 smithi006 bash[27292]: cluster 2023-08-28T00:30:16.668012+0000 mgr.y (mgr.14901) 4551 : cluster [DBG] pgmap v4558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:17 smithi130 bash[27579]: cluster 2023-08-28T00:30:16.668012+0000 mgr.y (mgr.14901) 4551 : cluster [DBG] pgmap v4558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:18 smithi130 bash[51302]: debug 2023-08-28T00:30:18.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:18.574571+0000) 2023-08-28T00:30:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:18 smithi006 bash[27292]: audit 2023-08-28T00:30:18.749381+0000 mon.a (mon.0) 3295 : audit [DBG] from='client.? 172.21.15.6:0/1292349922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:19.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:18 smithi006 bash[33918]: audit 2023-08-28T00:30:18.749381+0000 mon.a (mon.0) 3295 : audit [DBG] from='client.? 172.21.15.6:0/1292349922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:19.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:18 smithi130 bash[27579]: audit 2023-08-28T00:30:18.749381+0000 mon.a (mon.0) 3295 : audit [DBG] from='client.? 172.21.15.6:0/1292349922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:19 smithi130 bash[51302]: debug 2023-08-28T00:30:19.574+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:19.574936+0000) 2023-08-28T00:30:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:19 smithi006 bash[27292]: cluster 2023-08-28T00:30:18.668657+0000 mgr.y (mgr.14901) 4552 : cluster [DBG] pgmap v4559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:20.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:19 smithi006 bash[33918]: cluster 2023-08-28T00:30:18.668657+0000 mgr.y (mgr.14901) 4552 : cluster [DBG] pgmap v4559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:19 smithi130 bash[27579]: cluster 2023-08-28T00:30:18.668657+0000 mgr.y (mgr.14901) 4552 : cluster [DBG] pgmap v4559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:20 smithi130 bash[51302]: debug 2023-08-28T00:30:20.574+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:20.575178+0000) 2023-08-28T00:30:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:30:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:30:20] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:30:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:30:20.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:30:20.755Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:21.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:21 smithi130 bash[51302]: debug 2023-08-28T00:30:21.574+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:21.575468+0000) 2023-08-28T00:30:22.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:21 smithi006 bash[33918]: cluster 2023-08-28T00:30:20.669906+0000 mgr.y (mgr.14901) 4553 : cluster [DBG] pgmap v4560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:22.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:21 smithi006 bash[33918]: audit 2023-08-28T00:30:21.168897+0000 mon.a (mon.0) 3296 : audit [DBG] from='client.? 172.21.15.6:0/1880563035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:22.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:21 smithi006 bash[27292]: cluster 2023-08-28T00:30:20.669906+0000 mgr.y (mgr.14901) 4553 : cluster [DBG] pgmap v4560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:22.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:21 smithi006 bash[27292]: audit 2023-08-28T00:30:21.168897+0000 mon.a (mon.0) 3296 : audit [DBG] from='client.? 172.21.15.6:0/1880563035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:22.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:21 smithi130 bash[27579]: cluster 2023-08-28T00:30:20.669906+0000 mgr.y (mgr.14901) 4553 : cluster [DBG] pgmap v4560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:22.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:21 smithi130 bash[27579]: audit 2023-08-28T00:30:21.168897+0000 mon.a (mon.0) 3296 : audit [DBG] from='client.? 172.21.15.6:0/1880563035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:30:22.333Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:30:22.336Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:22 smithi130 bash[51302]: debug 2023-08-28T00:30:22.574+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:22.575894+0000) 2023-08-28T00:30:23.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:23 smithi130 bash[51302]: debug 2023-08-28T00:30:23.574+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:23.576225+0000) 2023-08-28T00:30:24.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:23 smithi006 bash[33918]: cluster 2023-08-28T00:30:22.670577+0000 mgr.y (mgr.14901) 4554 : cluster [DBG] pgmap v4561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:24.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:23 smithi006 bash[33918]: audit 2023-08-28T00:30:23.590002+0000 mon.c (mon.2) 1762 : audit [DBG] from='client.? 172.21.15.6:0/3122848706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:24.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:23 smithi006 bash[27292]: cluster 2023-08-28T00:30:22.670577+0000 mgr.y (mgr.14901) 4554 : cluster [DBG] pgmap v4561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:24.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:23 smithi006 bash[27292]: audit 2023-08-28T00:30:23.590002+0000 mon.c (mon.2) 1762 : audit [DBG] from='client.? 172.21.15.6:0/3122848706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:23 smithi130 bash[27579]: cluster 2023-08-28T00:30:22.670577+0000 mgr.y (mgr.14901) 4554 : cluster [DBG] pgmap v4561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:23 smithi130 bash[27579]: audit 2023-08-28T00:30:23.590002+0000 mon.c (mon.2) 1762 : audit [DBG] from='client.? 172.21.15.6:0/3122848706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:24.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:30:24.215Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:24.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:30:24.217Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:24 smithi130 bash[51302]: debug 2023-08-28T00:30:24.574+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:24.576562+0000) 2023-08-28T00:30:25.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:25 smithi130 bash[51302]: debug 2023-08-28T00:30:25.574+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:25.576905+0000) 2023-08-28T00:30:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:25 smithi006 bash[27292]: cluster 2023-08-28T00:30:24.671689+0000 mgr.y (mgr.14901) 4555 : cluster [DBG] pgmap v4562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:26.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:25 smithi006 bash[33918]: cluster 2023-08-28T00:30:24.671689+0000 mgr.y (mgr.14901) 4555 : cluster [DBG] pgmap v4562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:25 smithi130 bash[27579]: cluster 2023-08-28T00:30:24.671689+0000 mgr.y (mgr.14901) 4555 : cluster [DBG] pgmap v4562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:26.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:26 smithi130 bash[51302]: debug 2023-08-28T00:30:26.574+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:26.577257+0000) 2023-08-28T00:30:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:26 smithi006 bash[27292]: audit 2023-08-28T00:30:26.006316+0000 mon.a (mon.0) 3297 : audit [DBG] from='client.? 172.21.15.6:0/1119046869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:27.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:26 smithi006 bash[33918]: audit 2023-08-28T00:30:26.006316+0000 mon.a (mon.0) 3297 : audit [DBG] from='client.? 172.21.15.6:0/1119046869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:27.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:26 smithi130 bash[27579]: audit 2023-08-28T00:30:26.006316+0000 mon.a (mon.0) 3297 : audit [DBG] from='client.? 172.21.15.6:0/1119046869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:27.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:27 smithi130 bash[51302]: debug 2023-08-28T00:30:27.573+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:27.577582+0000) 2023-08-28T00:30:28.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:27 smithi006 bash[27292]: cluster 2023-08-28T00:30:26.672348+0000 mgr.y (mgr.14901) 4556 : cluster [DBG] pgmap v4563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:28.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:27 smithi006 bash[33918]: cluster 2023-08-28T00:30:26.672348+0000 mgr.y (mgr.14901) 4556 : cluster [DBG] pgmap v4563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:28.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:27 smithi130 bash[27579]: cluster 2023-08-28T00:30:26.672348+0000 mgr.y (mgr.14901) 4556 : cluster [DBG] pgmap v4563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:28 smithi130 bash[51302]: debug 2023-08-28T00:30:28.573+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:28.577918+0000) 2023-08-28T00:30:29.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:28 smithi006 bash[33918]: audit 2023-08-28T00:30:28.424646+0000 mon.a (mon.0) 3298 : audit [DBG] from='client.? 172.21.15.6:0/3488392285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:29.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:28 smithi006 bash[27292]: audit 2023-08-28T00:30:28.424646+0000 mon.a (mon.0) 3298 : audit [DBG] from='client.? 172.21.15.6:0/3488392285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:29.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:28 smithi130 bash[27579]: audit 2023-08-28T00:30:28.424646+0000 mon.a (mon.0) 3298 : audit [DBG] from='client.? 172.21.15.6:0/3488392285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:29.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:29 smithi130 bash[51302]: debug 2023-08-28T00:30:29.573+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:29.578324+0000) 2023-08-28T00:30:30.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:29 smithi006 bash[33918]: cluster 2023-08-28T00:30:28.672964+0000 mgr.y (mgr.14901) 4557 : cluster [DBG] pgmap v4564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:30.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:29 smithi006 bash[27292]: cluster 2023-08-28T00:30:28.672964+0000 mgr.y (mgr.14901) 4557 : cluster [DBG] pgmap v4564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:29 smithi130 bash[27579]: cluster 2023-08-28T00:30:28.672964+0000 mgr.y (mgr.14901) 4557 : cluster [DBG] pgmap v4564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:30 smithi130 bash[51302]: debug 2023-08-28T00:30:30.573+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:30.578613+0000) 2023-08-28T00:30:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:30:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:30:30] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:30:31.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:30 smithi006 bash[33918]: audit 2023-08-28T00:30:30.843270+0000 mon.c (mon.2) 1763 : audit [DBG] from='client.? 172.21.15.6:0/3389145651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:31.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:30 smithi006 bash[27292]: audit 2023-08-28T00:30:30.843270+0000 mon.c (mon.2) 1763 : audit [DBG] from='client.? 172.21.15.6:0/3389145651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:30:30.755Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:30:30.757Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:31.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:30 smithi130 bash[27579]: audit 2023-08-28T00:30:30.843270+0000 mon.c (mon.2) 1763 : audit [DBG] from='client.? 172.21.15.6:0/3389145651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:31.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:31 smithi130 bash[51302]: debug 2023-08-28T00:30:31.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:31.578964+0000) 2023-08-28T00:30:32.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:31 smithi006 bash[27292]: cluster 2023-08-28T00:30:30.673696+0000 mgr.y (mgr.14901) 4558 : cluster [DBG] pgmap v4565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:32.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:31 smithi006 bash[33918]: cluster 2023-08-28T00:30:30.673696+0000 mgr.y (mgr.14901) 4558 : cluster [DBG] pgmap v4565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:31 smithi130 bash[27579]: cluster 2023-08-28T00:30:30.673696+0000 mgr.y (mgr.14901) 4558 : cluster [DBG] pgmap v4565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:30:32.334Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:30:32.337Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:32 smithi130 bash[51302]: debug 2023-08-28T00:30:32.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:32.579255+0000) 2023-08-28T00:30:33.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:33 smithi130 bash[51302]: debug 2023-08-28T00:30:33.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:33.579535+0000) 2023-08-28T00:30:34.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:33 smithi006 bash[33918]: cluster 2023-08-28T00:30:32.674299+0000 mgr.y (mgr.14901) 4559 : cluster [DBG] pgmap v4566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:34.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:33 smithi006 bash[33918]: audit 2023-08-28T00:30:33.250691+0000 mon.a (mon.0) 3299 : audit [DBG] from='client.? 172.21.15.6:0/1782297560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:34.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:33 smithi006 bash[27292]: cluster 2023-08-28T00:30:32.674299+0000 mgr.y (mgr.14901) 4559 : cluster [DBG] pgmap v4566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:34.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:33 smithi006 bash[27292]: audit 2023-08-28T00:30:33.250691+0000 mon.a (mon.0) 3299 : audit [DBG] from='client.? 172.21.15.6:0/1782297560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:34.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:33 smithi130 bash[27579]: cluster 2023-08-28T00:30:32.674299+0000 mgr.y (mgr.14901) 4559 : cluster [DBG] pgmap v4566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:34.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:33 smithi130 bash[27579]: audit 2023-08-28T00:30:33.250691+0000 mon.a (mon.0) 3299 : audit [DBG] from='client.? 172.21.15.6:0/1782297560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:34.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:30:34.215Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:30:34.217Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:34 smithi130 bash[51302]: debug 2023-08-28T00:30:34.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:34.579858+0000) 2023-08-28T00:30:35.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:35 smithi130 bash[51302]: debug 2023-08-28T00:30:35.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:35.580157+0000) 2023-08-28T00:30:36.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:35 smithi006 bash[33918]: cluster 2023-08-28T00:30:34.675395+0000 mgr.y (mgr.14901) 4560 : cluster [DBG] pgmap v4567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:36.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:35 smithi006 bash[33918]: audit 2023-08-28T00:30:35.670287+0000 mon.a (mon.0) 3300 : audit [DBG] from='client.? 172.21.15.6:0/3319229004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:36.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:35 smithi006 bash[27292]: cluster 2023-08-28T00:30:34.675395+0000 mgr.y (mgr.14901) 4560 : cluster [DBG] pgmap v4567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:36.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:35 smithi006 bash[27292]: audit 2023-08-28T00:30:35.670287+0000 mon.a (mon.0) 3300 : audit [DBG] from='client.? 172.21.15.6:0/3319229004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:35 smithi130 bash[27579]: cluster 2023-08-28T00:30:34.675395+0000 mgr.y (mgr.14901) 4560 : cluster [DBG] pgmap v4567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:35 smithi130 bash[27579]: audit 2023-08-28T00:30:35.670287+0000 mon.a (mon.0) 3300 : audit [DBG] from='client.? 172.21.15.6:0/3319229004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:36 smithi130 bash[51302]: debug 2023-08-28T00:30:36.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:36.580423+0000) 2023-08-28T00:30:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:37 smithi130 bash[51302]: debug 2023-08-28T00:30:37.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:37.580687+0000) 2023-08-28T00:30:38.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:37 smithi006 bash[27292]: cluster 2023-08-28T00:30:36.676063+0000 mgr.y (mgr.14901) 4561 : cluster [DBG] pgmap v4568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:38.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:37 smithi006 bash[33918]: cluster 2023-08-28T00:30:36.676063+0000 mgr.y (mgr.14901) 4561 : cluster [DBG] pgmap v4568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:38.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:37 smithi130 bash[27579]: cluster 2023-08-28T00:30:36.676063+0000 mgr.y (mgr.14901) 4561 : cluster [DBG] pgmap v4568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:38 smithi130 bash[51302]: debug 2023-08-28T00:30:38.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:38.580990+0000) 2023-08-28T00:30:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:38 smithi130 bash[27579]: audit 2023-08-28T00:30:38.088170+0000 mon.a (mon.0) 3301 : audit [DBG] from='client.? 172.21.15.6:0/2435398480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:38 smithi006 bash[27292]: audit 2023-08-28T00:30:38.088170+0000 mon.a (mon.0) 3301 : audit [DBG] from='client.? 172.21.15.6:0/2435398480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:39.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:38 smithi006 bash[33918]: audit 2023-08-28T00:30:38.088170+0000 mon.a (mon.0) 3301 : audit [DBG] from='client.? 172.21.15.6:0/2435398480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:39 smithi130 bash[51302]: debug 2023-08-28T00:30:39.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:39.581287+0000) 2023-08-28T00:30:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:39 smithi130 bash[27579]: cluster 2023-08-28T00:30:38.676679+0000 mgr.y (mgr.14901) 4562 : cluster [DBG] pgmap v4569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:40.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:39 smithi006 bash[27292]: cluster 2023-08-28T00:30:38.676679+0000 mgr.y (mgr.14901) 4562 : cluster [DBG] pgmap v4569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:40.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:39 smithi006 bash[33918]: cluster 2023-08-28T00:30:38.676679+0000 mgr.y (mgr.14901) 4562 : cluster [DBG] pgmap v4569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:40 smithi130 bash[51302]: debug 2023-08-28T00:30:40.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:40.581563+0000) 2023-08-28T00:30:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:30:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:30:40] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:30:41.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:40 smithi006 bash[33918]: audit 2023-08-28T00:30:40.509763+0000 mon.a (mon.0) 3302 : audit [DBG] from='client.? 172.21.15.6:0/3586230175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:41.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:40 smithi006 bash[27292]: audit 2023-08-28T00:30:40.509763+0000 mon.a (mon.0) 3302 : audit [DBG] from='client.? 172.21.15.6:0/3586230175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:30:40.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:30:40.757Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:41.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:40 smithi130 bash[27579]: audit 2023-08-28T00:30:40.509763+0000 mon.a (mon.0) 3302 : audit [DBG] from='client.? 172.21.15.6:0/3586230175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:41.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:41 smithi130 bash[51302]: debug 2023-08-28T00:30:41.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:41.581841+0000) 2023-08-28T00:30:42.338 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:41 smithi006 bash[27292]: cluster 2023-08-28T00:30:40.677788+0000 mgr.y (mgr.14901) 4563 : cluster [DBG] pgmap v4570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:42.338 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:41 smithi006 bash[33918]: cluster 2023-08-28T00:30:40.677788+0000 mgr.y (mgr.14901) 4563 : cluster [DBG] pgmap v4570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:41 smithi130 bash[27579]: cluster 2023-08-28T00:30:40.677788+0000 mgr.y (mgr.14901) 4563 : cluster [DBG] pgmap v4570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:30:42.335Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:30:42.337Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:42 smithi130 bash[51302]: debug 2023-08-28T00:30:42.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:42.582177+0000) 2023-08-28T00:30:43.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:42 smithi130 bash[27579]: audit 2023-08-28T00:30:42.928897+0000 mon.c (mon.2) 1764 : audit [DBG] from='client.? 172.21.15.6:0/1733087079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:43.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:42 smithi006 bash[27292]: audit 2023-08-28T00:30:42.928897+0000 mon.c (mon.2) 1764 : audit [DBG] from='client.? 172.21.15.6:0/1733087079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:43.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:42 smithi006 bash[33918]: audit 2023-08-28T00:30:42.928897+0000 mon.c (mon.2) 1764 : audit [DBG] from='client.? 172.21.15.6:0/1733087079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:43.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:43 smithi130 bash[51302]: debug 2023-08-28T00:30:43.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:43.582514+0000) 2023-08-28T00:30:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:43 smithi130 bash[27579]: cluster 2023-08-28T00:30:42.678378+0000 mgr.y (mgr.14901) 4564 : cluster [DBG] pgmap v4571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:44.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:43 smithi006 bash[27292]: cluster 2023-08-28T00:30:42.678378+0000 mgr.y (mgr.14901) 4564 : cluster [DBG] pgmap v4571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:44.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:43 smithi006 bash[33918]: cluster 2023-08-28T00:30:42.678378+0000 mgr.y (mgr.14901) 4564 : cluster [DBG] pgmap v4571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:30:44.216Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:30:44.218Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:44 smithi130 bash[51302]: debug 2023-08-28T00:30:44.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:44.582748+0000) 2023-08-28T00:30:45.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:45 smithi130 bash[51302]: debug 2023-08-28T00:30:45.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:45.583023+0000) 2023-08-28T00:30:46.007 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:46 smithi006 bash[27292]: cluster 2023-08-28T00:30:44.679476+0000 mgr.y (mgr.14901) 4565 : cluster [DBG] pgmap v4572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:46.007 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:46 smithi006 bash[27292]: audit 2023-08-28T00:30:45.346015+0000 mon.a (mon.0) 3303 : audit [DBG] from='client.? 172.21.15.6:0/154635946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:46 smithi130 bash[27579]: cluster 2023-08-28T00:30:44.679476+0000 mgr.y (mgr.14901) 4565 : cluster [DBG] pgmap v4572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:46 smithi130 bash[27579]: audit 2023-08-28T00:30:45.346015+0000 mon.a (mon.0) 3303 : audit [DBG] from='client.? 172.21.15.6:0/154635946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:46 smithi006 bash[33918]: cluster 2023-08-28T00:30:44.679476+0000 mgr.y (mgr.14901) 4565 : cluster [DBG] pgmap v4572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:46 smithi006 bash[33918]: audit 2023-08-28T00:30:45.346015+0000 mon.a (mon.0) 3303 : audit [DBG] from='client.? 172.21.15.6:0/154635946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:46.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:46 smithi130 bash[51302]: debug 2023-08-28T00:30:46.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:46.583320+0000) 2023-08-28T00:30:47.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:47 smithi130 bash[51302]: debug 2023-08-28T00:30:47.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:47.583612+0000) 2023-08-28T00:30:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:48 smithi130 bash[27579]: cluster 2023-08-28T00:30:46.679990+0000 mgr.y (mgr.14901) 4566 : cluster [DBG] pgmap v4573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:48 smithi130 bash[27579]: audit 2023-08-28T00:30:47.756345+0000 mon.c (mon.2) 1765 : audit [DBG] from='client.? 172.21.15.6:0/4181024438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:48.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:48 smithi006 bash[27292]: cluster 2023-08-28T00:30:46.679990+0000 mgr.y (mgr.14901) 4566 : cluster [DBG] pgmap v4573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:48.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:48 smithi006 bash[27292]: audit 2023-08-28T00:30:47.756345+0000 mon.c (mon.2) 1765 : audit [DBG] from='client.? 172.21.15.6:0/4181024438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:48.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:48 smithi006 bash[33918]: cluster 2023-08-28T00:30:46.679990+0000 mgr.y (mgr.14901) 4566 : cluster [DBG] pgmap v4573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:48.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:48 smithi006 bash[33918]: audit 2023-08-28T00:30:47.756345+0000 mon.c (mon.2) 1765 : audit [DBG] from='client.? 172.21.15.6:0/4181024438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:48 smithi130 bash[51302]: debug 2023-08-28T00:30:48.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:48.583917+0000) 2023-08-28T00:30:49.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:49 smithi130 bash[51302]: debug 2023-08-28T00:30:49.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:49.584198+0000) 2023-08-28T00:30:50.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:50 smithi130 bash[27579]: cluster 2023-08-28T00:30:48.680617+0000 mgr.y (mgr.14901) 4567 : cluster [DBG] pgmap v4574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:50.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:50 smithi006 bash[33918]: cluster 2023-08-28T00:30:48.680617+0000 mgr.y (mgr.14901) 4567 : cluster [DBG] pgmap v4574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:50.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:50 smithi006 bash[27292]: cluster 2023-08-28T00:30:48.680617+0000 mgr.y (mgr.14901) 4567 : cluster [DBG] pgmap v4574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:50 smithi130 bash[51302]: debug 2023-08-28T00:30:50.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:50.584479+0000) 2023-08-28T00:30:51.018 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:30:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:30:50] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T00:30:51.065 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:30:50.757Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:51.065 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:30:50.759Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:51.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:51 smithi130 bash[27579]: audit 2023-08-28T00:30:50.172779+0000 mon.a (mon.0) 3304 : audit [DBG] from='client.? 172.21.15.6:0/2277530451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:51.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:51 smithi006 bash[27292]: audit 2023-08-28T00:30:50.172779+0000 mon.a (mon.0) 3304 : audit [DBG] from='client.? 172.21.15.6:0/2277530451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:51.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:51 smithi006 bash[33918]: audit 2023-08-28T00:30:50.172779+0000 mon.a (mon.0) 3304 : audit [DBG] from='client.? 172.21.15.6:0/2277530451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:51.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:51 smithi130 bash[51302]: debug 2023-08-28T00:30:51.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:51.584759+0000) 2023-08-28T00:30:52.339 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:52 smithi006 bash[27292]: cluster 2023-08-28T00:30:50.681819+0000 mgr.y (mgr.14901) 4568 : cluster [DBG] pgmap v4575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:52.339 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:52 smithi006 bash[33918]: cluster 2023-08-28T00:30:50.681819+0000 mgr.y (mgr.14901) 4568 : cluster [DBG] pgmap v4575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:52 smithi130 bash[27579]: cluster 2023-08-28T00:30:50.681819+0000 mgr.y (mgr.14901) 4568 : cluster [DBG] pgmap v4575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:30:52.335Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:30:52.337Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:52.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:52 smithi130 bash[51302]: debug 2023-08-28T00:30:52.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:52.585034+0000) 2023-08-28T00:30:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:53 smithi130 bash[27579]: audit 2023-08-28T00:30:52.585905+0000 mon.a (mon.0) 3305 : audit [DBG] from='client.? 172.21.15.6:0/1912596948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:53.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:53 smithi006 bash[33918]: audit 2023-08-28T00:30:52.585905+0000 mon.a (mon.0) 3305 : audit [DBG] from='client.? 172.21.15.6:0/1912596948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:53.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:53 smithi006 bash[27292]: audit 2023-08-28T00:30:52.585905+0000 mon.a (mon.0) 3305 : audit [DBG] from='client.? 172.21.15.6:0/1912596948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:53.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:53 smithi130 bash[51302]: debug 2023-08-28T00:30:53.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:53.585258+0000) 2023-08-28T00:30:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:54 smithi130 bash[27579]: cluster 2023-08-28T00:30:52.682632+0000 mgr.y (mgr.14901) 4569 : cluster [DBG] pgmap v4576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:54.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:54 smithi006 bash[27292]: cluster 2023-08-28T00:30:52.682632+0000 mgr.y (mgr.14901) 4569 : cluster [DBG] pgmap v4576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:30:54.217Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:30:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:30:54.219Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:30:54.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:54 smithi006 bash[33918]: cluster 2023-08-28T00:30:52.682632+0000 mgr.y (mgr.14901) 4569 : cluster [DBG] pgmap v4576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:54.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:54 smithi130 bash[51302]: debug 2023-08-28T00:30:54.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:54.585540+0000) 2023-08-28T00:30:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:55 smithi130 bash[27579]: audit 2023-08-28T00:30:55.000242+0000 mon.c (mon.2) 1766 : audit [DBG] from='client.? 172.21.15.6:0/706140115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:55.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:55 smithi006 bash[27292]: audit 2023-08-28T00:30:55.000242+0000 mon.c (mon.2) 1766 : audit [DBG] from='client.? 172.21.15.6:0/706140115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:55.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:55 smithi006 bash[33918]: audit 2023-08-28T00:30:55.000242+0000 mon.c (mon.2) 1766 : audit [DBG] from='client.? 172.21.15.6:0/706140115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:55.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:55 smithi130 bash[51302]: debug 2023-08-28T00:30:55.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:55.585827+0000) 2023-08-28T00:30:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:56 smithi130 bash[27579]: cluster 2023-08-28T00:30:54.683817+0000 mgr.y (mgr.14901) 4570 : cluster [DBG] pgmap v4577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:56.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:56 smithi006 bash[33918]: cluster 2023-08-28T00:30:54.683817+0000 mgr.y (mgr.14901) 4570 : cluster [DBG] pgmap v4577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:56.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:56 smithi006 bash[27292]: cluster 2023-08-28T00:30:54.683817+0000 mgr.y (mgr.14901) 4570 : cluster [DBG] pgmap v4577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:56.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:56 smithi130 bash[51302]: debug 2023-08-28T00:30:56.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:56.586096+0000) 2023-08-28T00:30:57.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:57 smithi130 bash[51302]: debug 2023-08-28T00:30:57.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:57.586391+0000) 2023-08-28T00:30:58.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:58 smithi130 bash[27579]: cluster 2023-08-28T00:30:56.684528+0000 mgr.y (mgr.14901) 4571 : cluster [DBG] pgmap v4578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:58.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:30:58 smithi130 bash[27579]: audit 2023-08-28T00:30:57.412340+0000 mon.a (mon.0) 3306 : audit [DBG] from='client.? 172.21.15.6:0/2430998282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:58.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:58 smithi006 bash[33918]: cluster 2023-08-28T00:30:56.684528+0000 mgr.y (mgr.14901) 4571 : cluster [DBG] pgmap v4578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:58.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:30:58 smithi006 bash[33918]: audit 2023-08-28T00:30:57.412340+0000 mon.a (mon.0) 3306 : audit [DBG] from='client.? 172.21.15.6:0/2430998282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:58.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:58 smithi006 bash[27292]: cluster 2023-08-28T00:30:56.684528+0000 mgr.y (mgr.14901) 4571 : cluster [DBG] pgmap v4578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:30:58.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:30:58 smithi006 bash[27292]: audit 2023-08-28T00:30:57.412340+0000 mon.a (mon.0) 3306 : audit [DBG] from='client.? 172.21.15.6:0/2430998282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:58.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:58 smithi130 bash[51302]: debug 2023-08-28T00:30:58.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:58.586654+0000) 2023-08-28T00:30:59.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:30:59 smithi130 bash[51302]: debug 2023-08-28T00:30:59.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:59.587006+0000) 2023-08-28T00:31:00.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:00 smithi130 bash[27579]: cluster 2023-08-28T00:30:58.685188+0000 mgr.y (mgr.14901) 4572 : cluster [DBG] pgmap v4579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:00.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:00 smithi130 bash[27579]: audit 2023-08-28T00:30:59.822881+0000 mon.c (mon.2) 1767 : audit [DBG] from='client.? 172.21.15.6:0/3977864997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:00.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:00 smithi006 bash[33918]: cluster 2023-08-28T00:30:58.685188+0000 mgr.y (mgr.14901) 4572 : cluster [DBG] pgmap v4579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:00.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:00 smithi006 bash[33918]: audit 2023-08-28T00:30:59.822881+0000 mon.c (mon.2) 1767 : audit [DBG] from='client.? 172.21.15.6:0/3977864997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:00.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:00 smithi006 bash[27292]: cluster 2023-08-28T00:30:58.685188+0000 mgr.y (mgr.14901) 4572 : cluster [DBG] pgmap v4579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:00.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:00 smithi006 bash[27292]: audit 2023-08-28T00:30:59.822881+0000 mon.c (mon.2) 1767 : audit [DBG] from='client.? 172.21.15.6:0/3977864997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:00.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:00 smithi130 bash[51302]: debug 2023-08-28T00:31:00.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:00.587309+0000) 2023-08-28T00:31:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:31:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:31:00] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:31:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:31:00.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:31:00.760Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:01.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:01 smithi130 bash[51302]: debug 2023-08-28T00:31:01.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:01.587539+0000) 2023-08-28T00:31:02.340 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:02 smithi006 bash[33918]: cluster 2023-08-28T00:31:00.686451+0000 mgr.y (mgr.14901) 4573 : cluster [DBG] pgmap v4580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:02.340 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:02 smithi006 bash[27292]: cluster 2023-08-28T00:31:00.686451+0000 mgr.y (mgr.14901) 4573 : cluster [DBG] pgmap v4580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:02.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:02 smithi130 bash[27579]: cluster 2023-08-28T00:31:00.686451+0000 mgr.y (mgr.14901) 4573 : cluster [DBG] pgmap v4580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:31:02.336Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:31:02.338Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:02.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:02 smithi130 bash[51302]: debug 2023-08-28T00:31:02.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:02.587792+0000) 2023-08-28T00:31:03.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:03 smithi130 bash[27579]: audit 2023-08-28T00:31:02.231318+0000 mon.a (mon.0) 3307 : audit [DBG] from='client.? 172.21.15.6:0/32626211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:03.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:03 smithi006 bash[27292]: audit 2023-08-28T00:31:02.231318+0000 mon.a (mon.0) 3307 : audit [DBG] from='client.? 172.21.15.6:0/32626211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:03.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:03 smithi006 bash[33918]: audit 2023-08-28T00:31:02.231318+0000 mon.a (mon.0) 3307 : audit [DBG] from='client.? 172.21.15.6:0/32626211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:03.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:03 smithi130 bash[51302]: debug 2023-08-28T00:31:03.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:03.588063+0000) 2023-08-28T00:31:04.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:04 smithi130 bash[27579]: cluster 2023-08-28T00:31:02.687183+0000 mgr.y (mgr.14901) 4574 : cluster [DBG] pgmap v4581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:04.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:04 smithi006 bash[33918]: cluster 2023-08-28T00:31:02.687183+0000 mgr.y (mgr.14901) 4574 : cluster [DBG] pgmap v4581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:04.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:04 smithi006 bash[27292]: cluster 2023-08-28T00:31:02.687183+0000 mgr.y (mgr.14901) 4574 : cluster [DBG] pgmap v4581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:31:04.217Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:31:04.219Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:04.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:04 smithi130 bash[51302]: debug 2023-08-28T00:31:04.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:04.588334+0000) 2023-08-28T00:31:05.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:05 smithi130 bash[27579]: audit 2023-08-28T00:31:04.650652+0000 mon.c (mon.2) 1768 : audit [DBG] from='client.? 172.21.15.6:0/385384148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:05.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:05 smithi006 bash[33918]: audit 2023-08-28T00:31:04.650652+0000 mon.c (mon.2) 1768 : audit [DBG] from='client.? 172.21.15.6:0/385384148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:05.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:05 smithi006 bash[27292]: audit 2023-08-28T00:31:04.650652+0000 mon.c (mon.2) 1768 : audit [DBG] from='client.? 172.21.15.6:0/385384148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:05.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:05 smithi130 bash[51302]: debug 2023-08-28T00:31:05.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:05.588634+0000) 2023-08-28T00:31:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:06 smithi130 bash[27579]: cluster 2023-08-28T00:31:04.688299+0000 mgr.y (mgr.14901) 4575 : cluster [DBG] pgmap v4582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:06.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:06 smithi006 bash[27292]: cluster 2023-08-28T00:31:04.688299+0000 mgr.y (mgr.14901) 4575 : cluster [DBG] pgmap v4582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:06 smithi006 bash[33918]: cluster 2023-08-28T00:31:04.688299+0000 mgr.y (mgr.14901) 4575 : cluster [DBG] pgmap v4582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:06.870 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:06 smithi130 bash[51302]: debug 2023-08-28T00:31:06.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:06.588929+0000) 2023-08-28T00:31:07.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:07 smithi130 bash[27579]: audit 2023-08-28T00:31:06.251971+0000 mon.b (mon.1) 559 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:31:07.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:07 smithi130 bash[27579]: audit 2023-08-28T00:31:07.059320+0000 mon.c (mon.2) 1769 : audit [DBG] from='client.? 172.21.15.6:0/2870225528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:07.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:07 smithi006 bash[33918]: audit 2023-08-28T00:31:06.251971+0000 mon.b (mon.1) 559 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:31:07.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:07 smithi006 bash[33918]: audit 2023-08-28T00:31:07.059320+0000 mon.c (mon.2) 1769 : audit [DBG] from='client.? 172.21.15.6:0/2870225528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:07.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:07 smithi006 bash[27292]: audit 2023-08-28T00:31:06.251971+0000 mon.b (mon.1) 559 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:31:07.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:07 smithi006 bash[27292]: audit 2023-08-28T00:31:07.059320+0000 mon.c (mon.2) 1769 : audit [DBG] from='client.? 172.21.15.6:0/2870225528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:07.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:07 smithi130 bash[51302]: debug 2023-08-28T00:31:07.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:07.589243+0000) 2023-08-28T00:31:08.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:08 smithi130 bash[27579]: cluster 2023-08-28T00:31:06.689006+0000 mgr.y (mgr.14901) 4576 : cluster [DBG] pgmap v4583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:08.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:08 smithi006 bash[27292]: cluster 2023-08-28T00:31:06.689006+0000 mgr.y (mgr.14901) 4576 : cluster [DBG] pgmap v4583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:08.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:08 smithi006 bash[33918]: cluster 2023-08-28T00:31:06.689006+0000 mgr.y (mgr.14901) 4576 : cluster [DBG] pgmap v4583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:08.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:08 smithi130 bash[51302]: debug 2023-08-28T00:31:08.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:08.589533+0000) 2023-08-28T00:31:09.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:09 smithi130 bash[51302]: debug 2023-08-28T00:31:09.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:09.589770+0000) 2023-08-28T00:31:10.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:10 smithi130 bash[27579]: cluster 2023-08-28T00:31:08.689712+0000 mgr.y (mgr.14901) 4577 : cluster [DBG] pgmap v4584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:10.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:10 smithi130 bash[27579]: audit 2023-08-28T00:31:09.469242+0000 mon.a (mon.0) 3308 : audit [DBG] from='client.? 172.21.15.6:0/2214300797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:10.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:10 smithi006 bash[27292]: cluster 2023-08-28T00:31:08.689712+0000 mgr.y (mgr.14901) 4577 : cluster [DBG] pgmap v4584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:10.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:10 smithi006 bash[27292]: audit 2023-08-28T00:31:09.469242+0000 mon.a (mon.0) 3308 : audit [DBG] from='client.? 172.21.15.6:0/2214300797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:10.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:10 smithi006 bash[33918]: cluster 2023-08-28T00:31:08.689712+0000 mgr.y (mgr.14901) 4577 : cluster [DBG] pgmap v4584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:10.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:10 smithi006 bash[33918]: audit 2023-08-28T00:31:09.469242+0000 mon.a (mon.0) 3308 : audit [DBG] from='client.? 172.21.15.6:0/2214300797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:10 smithi130 bash[51302]: debug 2023-08-28T00:31:10.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:10.590051+0000) 2023-08-28T00:31:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:31:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:31:10] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:31:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:31:10.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:31:10.760Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:11.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:11 smithi130 bash[51302]: debug 2023-08-28T00:31:11.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:11.590304+0000) 2023-08-28T00:31:12.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:12 smithi130 bash[27579]: cluster 2023-08-28T00:31:10.690761+0000 mgr.y (mgr.14901) 4578 : cluster [DBG] pgmap v4585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:12.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:12 smithi130 bash[27579]: audit 2023-08-28T00:31:11.914354+0000 mon.a (mon.0) 3309 : audit [DBG] from='client.? 172.21.15.6:0/535310303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:12.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:12 smithi006 bash[27292]: cluster 2023-08-28T00:31:10.690761+0000 mgr.y (mgr.14901) 4578 : cluster [DBG] pgmap v4585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:12.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:12 smithi006 bash[27292]: audit 2023-08-28T00:31:11.914354+0000 mon.a (mon.0) 3309 : audit [DBG] from='client.? 172.21.15.6:0/535310303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:12.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:12 smithi006 bash[33918]: cluster 2023-08-28T00:31:10.690761+0000 mgr.y (mgr.14901) 4578 : cluster [DBG] pgmap v4585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:12.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:12 smithi006 bash[33918]: audit 2023-08-28T00:31:11.914354+0000 mon.a (mon.0) 3309 : audit [DBG] from='client.? 172.21.15.6:0/535310303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:12.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:31:12.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:12.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:31:12.340Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:12 smithi130 bash[51302]: debug 2023-08-28T00:31:12.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:12.590507+0000) 2023-08-28T00:31:13.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:13 smithi006 bash[33918]: audit 2023-08-28T00:31:12.133162+0000 mon.a (mon.0) 3310 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:31:13.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:13 smithi006 bash[33918]: audit 2023-08-28T00:31:12.578069+0000 mon.a (mon.0) 3311 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:31:13.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:13 smithi006 bash[33918]: audit 2023-08-28T00:31:12.852646+0000 mon.b (mon.1) 560 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:31:13.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:13 smithi006 bash[33918]: audit 2023-08-28T00:31:12.854769+0000 mon.b (mon.1) 561 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:31:13.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:13 smithi006 bash[33918]: audit 2023-08-28T00:31:12.863155+0000 mon.a (mon.0) 3312 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:31:13.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:13 smithi006 bash[27292]: audit 2023-08-28T00:31:12.133162+0000 mon.a (mon.0) 3310 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:31:13.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:13 smithi006 bash[27292]: audit 2023-08-28T00:31:12.578069+0000 mon.a (mon.0) 3311 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:31:13.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:13 smithi006 bash[27292]: audit 2023-08-28T00:31:12.852646+0000 mon.b (mon.1) 560 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:31:13.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:13 smithi006 bash[27292]: audit 2023-08-28T00:31:12.854769+0000 mon.b (mon.1) 561 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:31:13.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:13 smithi006 bash[27292]: audit 2023-08-28T00:31:12.863155+0000 mon.a (mon.0) 3312 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:31:13.592 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:13 smithi130 bash[27579]: audit 2023-08-28T00:31:12.133162+0000 mon.a (mon.0) 3310 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:31:13.592 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:13 smithi130 bash[27579]: audit 2023-08-28T00:31:12.578069+0000 mon.a (mon.0) 3311 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:31:13.592 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:13 smithi130 bash[27579]: audit 2023-08-28T00:31:12.852646+0000 mon.b (mon.1) 560 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:31:13.593 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:13 smithi130 bash[27579]: audit 2023-08-28T00:31:12.854769+0000 mon.b (mon.1) 561 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:31:13.593 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:13 smithi130 bash[27579]: audit 2023-08-28T00:31:12.863155+0000 mon.a (mon.0) 3312 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:31:13.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:13 smithi130 bash[51302]: debug 2023-08-28T00:31:13.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:13.590768+0000) 2023-08-28T00:31:14.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:14 smithi006 bash[27292]: cluster 2023-08-28T00:31:12.691352+0000 mgr.y (mgr.14901) 4579 : cluster [DBG] pgmap v4586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:14.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:14 smithi006 bash[33918]: cluster 2023-08-28T00:31:12.691352+0000 mgr.y (mgr.14901) 4579 : cluster [DBG] pgmap v4586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:31:14.217Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:31:14.219Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:14.592 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:14 smithi130 bash[27579]: cluster 2023-08-28T00:31:12.691352+0000 mgr.y (mgr.14901) 4579 : cluster [DBG] pgmap v4586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:14 smithi130 bash[51302]: debug 2023-08-28T00:31:14.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:14.591044+0000) 2023-08-28T00:31:15.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:15 smithi006 bash[33918]: audit 2023-08-28T00:31:14.333788+0000 mon.c (mon.2) 1770 : audit [DBG] from='client.? 172.21.15.6:0/2070441979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:15.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:15 smithi006 bash[27292]: audit 2023-08-28T00:31:14.333788+0000 mon.c (mon.2) 1770 : audit [DBG] from='client.? 172.21.15.6:0/2070441979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:15.592 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:15 smithi130 bash[27579]: audit 2023-08-28T00:31:14.333788+0000 mon.c (mon.2) 1770 : audit [DBG] from='client.? 172.21.15.6:0/2070441979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:15.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:15 smithi130 bash[51302]: debug 2023-08-28T00:31:15.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:15.591308+0000) 2023-08-28T00:31:16.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:16 smithi006 bash[33918]: cluster 2023-08-28T00:31:14.692484+0000 mgr.y (mgr.14901) 4580 : cluster [DBG] pgmap v4587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:16.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:16 smithi006 bash[27292]: cluster 2023-08-28T00:31:14.692484+0000 mgr.y (mgr.14901) 4580 : cluster [DBG] pgmap v4587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:16.593 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:16 smithi130 bash[27579]: cluster 2023-08-28T00:31:14.692484+0000 mgr.y (mgr.14901) 4580 : cluster [DBG] pgmap v4587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:16.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:16 smithi130 bash[51302]: debug 2023-08-28T00:31:16.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:16.591518+0000) 2023-08-28T00:31:17.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:17 smithi006 bash[27292]: audit 2023-08-28T00:31:16.758703+0000 mon.a (mon.0) 3313 : audit [DBG] from='client.? 172.21.15.6:0/3370237833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:17.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:17 smithi006 bash[33918]: audit 2023-08-28T00:31:16.758703+0000 mon.a (mon.0) 3313 : audit [DBG] from='client.? 172.21.15.6:0/3370237833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:17.593 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:17 smithi130 bash[27579]: audit 2023-08-28T00:31:16.758703+0000 mon.a (mon.0) 3313 : audit [DBG] from='client.? 172.21.15.6:0/3370237833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:17.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:17 smithi130 bash[51302]: debug 2023-08-28T00:31:17.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:17.591782+0000) 2023-08-28T00:31:18.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:18 smithi006 bash[27292]: cluster 2023-08-28T00:31:16.693133+0000 mgr.y (mgr.14901) 4581 : cluster [DBG] pgmap v4588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:18.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:18 smithi006 bash[33918]: cluster 2023-08-28T00:31:16.693133+0000 mgr.y (mgr.14901) 4581 : cluster [DBG] pgmap v4588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:18.593 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:18 smithi130 bash[27579]: cluster 2023-08-28T00:31:16.693133+0000 mgr.y (mgr.14901) 4581 : cluster [DBG] pgmap v4588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:18 smithi130 bash[51302]: debug 2023-08-28T00:31:18.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:18.592052+0000) 2023-08-28T00:31:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:19 smithi130 bash[51302]: debug 2023-08-28T00:31:19.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:19.592343+0000) 2023-08-28T00:31:20.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:20 smithi006 bash[27292]: cluster 2023-08-28T00:31:18.693788+0000 mgr.y (mgr.14901) 4582 : cluster [DBG] pgmap v4589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:20.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:20 smithi006 bash[27292]: audit 2023-08-28T00:31:19.175526+0000 mon.a (mon.0) 3314 : audit [DBG] from='client.? 172.21.15.6:0/2775042882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:20.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:20 smithi006 bash[33918]: cluster 2023-08-28T00:31:18.693788+0000 mgr.y (mgr.14901) 4582 : cluster [DBG] pgmap v4589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:20.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:20 smithi006 bash[33918]: audit 2023-08-28T00:31:19.175526+0000 mon.a (mon.0) 3314 : audit [DBG] from='client.? 172.21.15.6:0/2775042882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:20.594 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:20 smithi130 bash[27579]: cluster 2023-08-28T00:31:18.693788+0000 mgr.y (mgr.14901) 4582 : cluster [DBG] pgmap v4589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:20.594 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:20 smithi130 bash[27579]: audit 2023-08-28T00:31:19.175526+0000 mon.a (mon.0) 3314 : audit [DBG] from='client.? 172.21.15.6:0/2775042882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:20 smithi130 bash[51302]: debug 2023-08-28T00:31:20.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:20.592628+0000) 2023-08-28T00:31:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:31:20.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:31:20.762Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:21.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:31:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:31:20] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T00:31:21.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:21 smithi130 bash[51302]: debug 2023-08-28T00:31:21.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:21.592929+0000) 2023-08-28T00:31:22.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:22 smithi006 bash[33918]: cluster 2023-08-28T00:31:20.694839+0000 mgr.y (mgr.14901) 4583 : cluster [DBG] pgmap v4590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:22.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:22 smithi006 bash[33918]: audit 2023-08-28T00:31:21.588585+0000 mon.a (mon.0) 3315 : audit [DBG] from='client.? 172.21.15.6:0/3840824166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:22.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:31:22.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:22.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:31:22.340Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:22.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:22 smithi006 bash[27292]: cluster 2023-08-28T00:31:20.694839+0000 mgr.y (mgr.14901) 4583 : cluster [DBG] pgmap v4590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:22.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:22 smithi006 bash[27292]: audit 2023-08-28T00:31:21.588585+0000 mon.a (mon.0) 3315 : audit [DBG] from='client.? 172.21.15.6:0/3840824166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:22.594 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:22 smithi130 bash[27579]: cluster 2023-08-28T00:31:20.694839+0000 mgr.y (mgr.14901) 4583 : cluster [DBG] pgmap v4590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:22.595 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:22 smithi130 bash[27579]: audit 2023-08-28T00:31:21.588585+0000 mon.a (mon.0) 3315 : audit [DBG] from='client.? 172.21.15.6:0/3840824166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:22 smithi130 bash[51302]: debug 2023-08-28T00:31:22.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:22.593226+0000) 2023-08-28T00:31:23.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:23 smithi130 bash[51302]: debug 2023-08-28T00:31:23.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:23.593521+0000) 2023-08-28T00:31:24.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:24 smithi006 bash[27292]: cluster 2023-08-28T00:31:22.695617+0000 mgr.y (mgr.14901) 4584 : cluster [DBG] pgmap v4591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:24.471 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:24 smithi006 bash[27292]: audit 2023-08-28T00:31:24.015681+0000 mon.c (mon.2) 1771 : audit [DBG] from='client.? 172.21.15.6:0/2688108717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:24.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:24 smithi006 bash[33918]: cluster 2023-08-28T00:31:22.695617+0000 mgr.y (mgr.14901) 4584 : cluster [DBG] pgmap v4591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:24.471 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:24 smithi006 bash[33918]: audit 2023-08-28T00:31:24.015681+0000 mon.c (mon.2) 1771 : audit [DBG] from='client.? 172.21.15.6:0/2688108717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:24.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:31:24.219Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:24.471 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:31:24.221Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:24.595 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:24 smithi130 bash[27579]: cluster 2023-08-28T00:31:22.695617+0000 mgr.y (mgr.14901) 4584 : cluster [DBG] pgmap v4591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:24.595 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:24 smithi130 bash[27579]: audit 2023-08-28T00:31:24.015681+0000 mon.c (mon.2) 1771 : audit [DBG] from='client.? 172.21.15.6:0/2688108717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:24 smithi130 bash[51302]: debug 2023-08-28T00:31:24.592+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:24.593801+0000) 2023-08-28T00:31:25.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:25 smithi130 bash[51302]: debug 2023-08-28T00:31:25.588+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:25.594080+0000) 2023-08-28T00:31:26.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:26 smithi006 bash[33918]: cluster 2023-08-28T00:31:24.696798+0000 mgr.y (mgr.14901) 4585 : cluster [DBG] pgmap v4592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:26.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:26 smithi006 bash[27292]: cluster 2023-08-28T00:31:24.696798+0000 mgr.y (mgr.14901) 4585 : cluster [DBG] pgmap v4592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:26.596 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:26 smithi130 bash[27579]: cluster 2023-08-28T00:31:24.696798+0000 mgr.y (mgr.14901) 4585 : cluster [DBG] pgmap v4592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:26.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:26 smithi130 bash[51302]: debug 2023-08-28T00:31:26.588+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:26.594404+0000) 2023-08-28T00:31:27.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:27 smithi006 bash[33918]: audit 2023-08-28T00:31:26.441193+0000 mon.c (mon.2) 1772 : audit [DBG] from='client.? 172.21.15.6:0/41146016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:27.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:27 smithi006 bash[27292]: audit 2023-08-28T00:31:26.441193+0000 mon.c (mon.2) 1772 : audit [DBG] from='client.? 172.21.15.6:0/41146016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:27.596 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:27 smithi130 bash[27579]: audit 2023-08-28T00:31:26.441193+0000 mon.c (mon.2) 1772 : audit [DBG] from='client.? 172.21.15.6:0/41146016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:27.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:27 smithi130 bash[51302]: debug 2023-08-28T00:31:27.588+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:27.594677+0000) 2023-08-28T00:31:28.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:28 smithi006 bash[27292]: cluster 2023-08-28T00:31:26.697466+0000 mgr.y (mgr.14901) 4586 : cluster [DBG] pgmap v4593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:28.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:28 smithi006 bash[33918]: cluster 2023-08-28T00:31:26.697466+0000 mgr.y (mgr.14901) 4586 : cluster [DBG] pgmap v4593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:28.596 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:28 smithi130 bash[27579]: cluster 2023-08-28T00:31:26.697466+0000 mgr.y (mgr.14901) 4586 : cluster [DBG] pgmap v4593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:28 smithi130 bash[51302]: debug 2023-08-28T00:31:28.592+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:28.594966+0000) 2023-08-28T00:31:29.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:29 smithi006 bash[27292]: audit 2023-08-28T00:31:28.864673+0000 mon.c (mon.2) 1773 : audit [DBG] from='client.? 172.21.15.6:0/3051235154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:29.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:29 smithi006 bash[33918]: audit 2023-08-28T00:31:28.864673+0000 mon.c (mon.2) 1773 : audit [DBG] from='client.? 172.21.15.6:0/3051235154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:29.597 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:29 smithi130 bash[27579]: audit 2023-08-28T00:31:28.864673+0000 mon.c (mon.2) 1773 : audit [DBG] from='client.? 172.21.15.6:0/3051235154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:29.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:29 smithi130 bash[51302]: debug 2023-08-28T00:31:29.592+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:29.595333+0000) 2023-08-28T00:31:30.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:30 smithi006 bash[33918]: cluster 2023-08-28T00:31:28.697929+0000 mgr.y (mgr.14901) 4587 : cluster [DBG] pgmap v4594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:30.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:30 smithi006 bash[27292]: cluster 2023-08-28T00:31:28.697929+0000 mgr.y (mgr.14901) 4587 : cluster [DBG] pgmap v4594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:30.597 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:30 smithi130 bash[27579]: cluster 2023-08-28T00:31:28.697929+0000 mgr.y (mgr.14901) 4587 : cluster [DBG] pgmap v4594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:30 smithi130 bash[51302]: debug 2023-08-28T00:31:30.592+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:30.595598+0000) 2023-08-28T00:31:31.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:31:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:31:30] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:31:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:31:30.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:31:30.762Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:31.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:31 smithi130 bash[51302]: debug 2023-08-28T00:31:31.592+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:31.595934+0000) 2023-08-28T00:31:32.598 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:32 smithi130 bash[27579]: cluster 2023-08-28T00:31:30.699034+0000 mgr.y (mgr.14901) 4588 : cluster [DBG] pgmap v4595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:32.598 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:32 smithi130 bash[27579]: audit 2023-08-28T00:31:31.284416+0000 mon.a (mon.0) 3316 : audit [DBG] from='client.? 172.21.15.6:0/4034678116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:32 smithi006 bash[27292]: cluster 2023-08-28T00:31:30.699034+0000 mgr.y (mgr.14901) 4588 : cluster [DBG] pgmap v4595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:32 smithi006 bash[27292]: audit 2023-08-28T00:31:31.284416+0000 mon.a (mon.0) 3316 : audit [DBG] from='client.? 172.21.15.6:0/4034678116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:31:32.339Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:31:32.341Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:32 smithi006 bash[33918]: cluster 2023-08-28T00:31:30.699034+0000 mgr.y (mgr.14901) 4588 : cluster [DBG] pgmap v4595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:32 smithi006 bash[33918]: audit 2023-08-28T00:31:31.284416+0000 mon.a (mon.0) 3316 : audit [DBG] from='client.? 172.21.15.6:0/4034678116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:32 smithi130 bash[51302]: debug 2023-08-28T00:31:32.592+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:32.596266+0000) 2023-08-28T00:31:33.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:33 smithi130 bash[51302]: debug 2023-08-28T00:31:33.592+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:33.596629+0000) 2023-08-28T00:31:34.598 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:34 smithi130 bash[27579]: cluster 2023-08-28T00:31:32.699713+0000 mgr.y (mgr.14901) 4589 : cluster [DBG] pgmap v4596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:34.599 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:34 smithi130 bash[27579]: audit 2023-08-28T00:31:33.693109+0000 mon.a (mon.0) 3317 : audit [DBG] from='client.? 172.21.15.6:0/2283757750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:34.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:31:34.219Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:31:34.222Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:34 smithi006 bash[27292]: cluster 2023-08-28T00:31:32.699713+0000 mgr.y (mgr.14901) 4589 : cluster [DBG] pgmap v4596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:34 smithi006 bash[27292]: audit 2023-08-28T00:31:33.693109+0000 mon.a (mon.0) 3317 : audit [DBG] from='client.? 172.21.15.6:0/2283757750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:34 smithi006 bash[33918]: cluster 2023-08-28T00:31:32.699713+0000 mgr.y (mgr.14901) 4589 : cluster [DBG] pgmap v4596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:34 smithi006 bash[33918]: audit 2023-08-28T00:31:33.693109+0000 mon.a (mon.0) 3317 : audit [DBG] from='client.? 172.21.15.6:0/2283757750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:34 smithi130 bash[51302]: debug 2023-08-28T00:31:34.592+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:34.596974+0000) 2023-08-28T00:31:35.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:35 smithi130 bash[51302]: debug 2023-08-28T00:31:35.592+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:35.597306+0000) 2023-08-28T00:31:36.599 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:36 smithi130 bash[27579]: cluster 2023-08-28T00:31:34.700909+0000 mgr.y (mgr.14901) 4590 : cluster [DBG] pgmap v4597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:36.599 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:36 smithi130 bash[27579]: audit 2023-08-28T00:31:36.111937+0000 mon.a (mon.0) 3318 : audit [DBG] from='client.? 172.21.15.6:0/1970890464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:36 smithi006 bash[27292]: cluster 2023-08-28T00:31:34.700909+0000 mgr.y (mgr.14901) 4590 : cluster [DBG] pgmap v4597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:36.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:36 smithi006 bash[27292]: audit 2023-08-28T00:31:36.111937+0000 mon.a (mon.0) 3318 : audit [DBG] from='client.? 172.21.15.6:0/1970890464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:36.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:36 smithi006 bash[33918]: cluster 2023-08-28T00:31:34.700909+0000 mgr.y (mgr.14901) 4590 : cluster [DBG] pgmap v4597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:36.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:36 smithi006 bash[33918]: audit 2023-08-28T00:31:36.111937+0000 mon.a (mon.0) 3318 : audit [DBG] from='client.? 172.21.15.6:0/1970890464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:36 smithi130 bash[51302]: debug 2023-08-28T00:31:36.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:36.597597+0000) 2023-08-28T00:31:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:37 smithi130 bash[51302]: debug 2023-08-28T00:31:37.592+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:37.597950+0000) 2023-08-28T00:31:38.600 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:38 smithi130 bash[27579]: cluster 2023-08-28T00:31:36.701516+0000 mgr.y (mgr.14901) 4591 : cluster [DBG] pgmap v4598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:38.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:38 smithi006 bash[33918]: cluster 2023-08-28T00:31:36.701516+0000 mgr.y (mgr.14901) 4591 : cluster [DBG] pgmap v4598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:38.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:38 smithi006 bash[27292]: cluster 2023-08-28T00:31:36.701516+0000 mgr.y (mgr.14901) 4591 : cluster [DBG] pgmap v4598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:38 smithi130 bash[51302]: debug 2023-08-28T00:31:38.592+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:38.598239+0000) 2023-08-28T00:31:39.600 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:39 smithi130 bash[27579]: audit 2023-08-28T00:31:38.523867+0000 mon.c (mon.2) 1774 : audit [DBG] from='client.? 172.21.15.6:0/4140737072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:39.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:39 smithi006 bash[33918]: audit 2023-08-28T00:31:38.523867+0000 mon.c (mon.2) 1774 : audit [DBG] from='client.? 172.21.15.6:0/4140737072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:39.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:39 smithi006 bash[27292]: audit 2023-08-28T00:31:38.523867+0000 mon.c (mon.2) 1774 : audit [DBG] from='client.? 172.21.15.6:0/4140737072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:39 smithi130 bash[51302]: debug 2023-08-28T00:31:39.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:39.598580+0000) 2023-08-28T00:31:40.600 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:40 smithi130 bash[27579]: cluster 2023-08-28T00:31:38.702134+0000 mgr.y (mgr.14901) 4592 : cluster [DBG] pgmap v4599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:40 smithi006 bash[33918]: cluster 2023-08-28T00:31:38.702134+0000 mgr.y (mgr.14901) 4592 : cluster [DBG] pgmap v4599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:40 smithi006 bash[27292]: cluster 2023-08-28T00:31:38.702134+0000 mgr.y (mgr.14901) 4592 : cluster [DBG] pgmap v4599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:40 smithi130 bash[51302]: debug 2023-08-28T00:31:40.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:40.598949+0000) 2023-08-28T00:31:41.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:31:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:31:40] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:31:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:31:40.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:31:40.763Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:41.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:41 smithi130 bash[51302]: debug 2023-08-28T00:31:41.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:41.599303+0000) 2023-08-28T00:31:42.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:41 smithi006 bash[27292]: cluster 2023-08-28T00:31:40.703354+0000 mgr.y (mgr.14901) 4593 : cluster [DBG] pgmap v4600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:42.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:41 smithi006 bash[27292]: audit 2023-08-28T00:31:40.941526+0000 mon.a (mon.0) 3319 : audit [DBG] from='client.? 172.21.15.6:0/2008551868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:42.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:41 smithi006 bash[33918]: cluster 2023-08-28T00:31:40.703354+0000 mgr.y (mgr.14901) 4593 : cluster [DBG] pgmap v4600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:42.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:41 smithi006 bash[33918]: audit 2023-08-28T00:31:40.941526+0000 mon.a (mon.0) 3319 : audit [DBG] from='client.? 172.21.15.6:0/2008551868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:41 smithi130 bash[27579]: cluster 2023-08-28T00:31:40.703354+0000 mgr.y (mgr.14901) 4593 : cluster [DBG] pgmap v4600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:41 smithi130 bash[27579]: audit 2023-08-28T00:31:40.941526+0000 mon.a (mon.0) 3319 : audit [DBG] from='client.? 172.21.15.6:0/2008551868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:31:42.339Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:31:42.341Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:42 smithi130 bash[51302]: debug 2023-08-28T00:31:42.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:42.599655+0000) 2023-08-28T00:31:43.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:43 smithi130 bash[51302]: debug 2023-08-28T00:31:43.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:43.599980+0000) 2023-08-28T00:31:44.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:43 smithi006 bash[27292]: cluster 2023-08-28T00:31:42.704017+0000 mgr.y (mgr.14901) 4594 : cluster [DBG] pgmap v4601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:44.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:43 smithi006 bash[27292]: audit 2023-08-28T00:31:43.359567+0000 mon.c (mon.2) 1775 : audit [DBG] from='client.? 172.21.15.6:0/3370754903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:44.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:43 smithi006 bash[33918]: cluster 2023-08-28T00:31:42.704017+0000 mgr.y (mgr.14901) 4594 : cluster [DBG] pgmap v4601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:44.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:43 smithi006 bash[33918]: audit 2023-08-28T00:31:43.359567+0000 mon.c (mon.2) 1775 : audit [DBG] from='client.? 172.21.15.6:0/3370754903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:43 smithi130 bash[27579]: cluster 2023-08-28T00:31:42.704017+0000 mgr.y (mgr.14901) 4594 : cluster [DBG] pgmap v4601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:43 smithi130 bash[27579]: audit 2023-08-28T00:31:43.359567+0000 mon.c (mon.2) 1775 : audit [DBG] from='client.? 172.21.15.6:0/3370754903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:44.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:31:44.221Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:31:44.223Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:44 smithi130 bash[51302]: debug 2023-08-28T00:31:44.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:44.600370+0000) 2023-08-28T00:31:45.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:45 smithi130 bash[51302]: debug 2023-08-28T00:31:45.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:45.600681+0000) 2023-08-28T00:31:46.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:45 smithi006 bash[27292]: cluster 2023-08-28T00:31:44.705117+0000 mgr.y (mgr.14901) 4595 : cluster [DBG] pgmap v4602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:46.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:45 smithi006 bash[27292]: audit 2023-08-28T00:31:45.774750+0000 mon.a (mon.0) 3320 : audit [DBG] from='client.? 172.21.15.6:0/1505239466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:46.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:45 smithi006 bash[33918]: cluster 2023-08-28T00:31:44.705117+0000 mgr.y (mgr.14901) 4595 : cluster [DBG] pgmap v4602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:46.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:45 smithi006 bash[33918]: audit 2023-08-28T00:31:45.774750+0000 mon.a (mon.0) 3320 : audit [DBG] from='client.? 172.21.15.6:0/1505239466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:45 smithi130 bash[27579]: cluster 2023-08-28T00:31:44.705117+0000 mgr.y (mgr.14901) 4595 : cluster [DBG] pgmap v4602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:45 smithi130 bash[27579]: audit 2023-08-28T00:31:45.774750+0000 mon.a (mon.0) 3320 : audit [DBG] from='client.? 172.21.15.6:0/1505239466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:46.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:46 smithi130 bash[51302]: debug 2023-08-28T00:31:46.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:46.601053+0000) 2023-08-28T00:31:47.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:47 smithi130 bash[51302]: debug 2023-08-28T00:31:47.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:47.601377+0000) 2023-08-28T00:31:48.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:47 smithi006 bash[27292]: cluster 2023-08-28T00:31:46.705790+0000 mgr.y (mgr.14901) 4596 : cluster [DBG] pgmap v4603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:48.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:47 smithi006 bash[33918]: cluster 2023-08-28T00:31:46.705790+0000 mgr.y (mgr.14901) 4596 : cluster [DBG] pgmap v4603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:47 smithi130 bash[27579]: cluster 2023-08-28T00:31:46.705790+0000 mgr.y (mgr.14901) 4596 : cluster [DBG] pgmap v4603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:48 smithi130 bash[51302]: debug 2023-08-28T00:31:48.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:48.601721+0000) 2023-08-28T00:31:49.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:48 smithi006 bash[27292]: audit 2023-08-28T00:31:48.179701+0000 mon.a (mon.0) 3321 : audit [DBG] from='client.? 172.21.15.6:0/2049452900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:49.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:48 smithi006 bash[33918]: audit 2023-08-28T00:31:48.179701+0000 mon.a (mon.0) 3321 : audit [DBG] from='client.? 172.21.15.6:0/2049452900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:49.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:48 smithi130 bash[27579]: audit 2023-08-28T00:31:48.179701+0000 mon.a (mon.0) 3321 : audit [DBG] from='client.? 172.21.15.6:0/2049452900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:49.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:49 smithi130 bash[51302]: debug 2023-08-28T00:31:49.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:49.602070+0000) 2023-08-28T00:31:50.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:49 smithi006 bash[27292]: cluster 2023-08-28T00:31:48.706469+0000 mgr.y (mgr.14901) 4597 : cluster [DBG] pgmap v4604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:50.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:49 smithi006 bash[33918]: cluster 2023-08-28T00:31:48.706469+0000 mgr.y (mgr.14901) 4597 : cluster [DBG] pgmap v4604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:50.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:49 smithi130 bash[27579]: cluster 2023-08-28T00:31:48.706469+0000 mgr.y (mgr.14901) 4597 : cluster [DBG] pgmap v4604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:50 smithi130 bash[51302]: debug 2023-08-28T00:31:50.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:50.602422+0000) 2023-08-28T00:31:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:31:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:31:50] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:31:51.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:50 smithi006 bash[33918]: audit 2023-08-28T00:31:50.603813+0000 mon.c (mon.2) 1776 : audit [DBG] from='client.? 172.21.15.6:0/4081438738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:50 smithi006 bash[27292]: audit 2023-08-28T00:31:50.603813+0000 mon.c (mon.2) 1776 : audit [DBG] from='client.? 172.21.15.6:0/4081438738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:31:50.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:31:50.764Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:51.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:50 smithi130 bash[27579]: audit 2023-08-28T00:31:50.603813+0000 mon.c (mon.2) 1776 : audit [DBG] from='client.? 172.21.15.6:0/4081438738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:51.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:51 smithi130 bash[51302]: debug 2023-08-28T00:31:51.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:51.602765+0000) 2023-08-28T00:31:52.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:51 smithi006 bash[27292]: cluster 2023-08-28T00:31:50.707598+0000 mgr.y (mgr.14901) 4598 : cluster [DBG] pgmap v4605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:52.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:51 smithi006 bash[33918]: cluster 2023-08-28T00:31:50.707598+0000 mgr.y (mgr.14901) 4598 : cluster [DBG] pgmap v4605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:51 smithi130 bash[27579]: cluster 2023-08-28T00:31:50.707598+0000 mgr.y (mgr.14901) 4598 : cluster [DBG] pgmap v4605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:31:52.340Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:31:52.342Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:52.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:52 smithi130 bash[51302]: debug 2023-08-28T00:31:52.600+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:52.603086+0000) 2023-08-28T00:31:53.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:53 smithi130 bash[51302]: debug 2023-08-28T00:31:53.600+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:53.603413+0000) 2023-08-28T00:31:54.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:53 smithi006 bash[27292]: cluster 2023-08-28T00:31:52.708328+0000 mgr.y (mgr.14901) 4599 : cluster [DBG] pgmap v4606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:54.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:53 smithi006 bash[27292]: audit 2023-08-28T00:31:53.017266+0000 mon.a (mon.0) 3322 : audit [DBG] from='client.? 172.21.15.6:0/1273434808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:54.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:53 smithi006 bash[33918]: cluster 2023-08-28T00:31:52.708328+0000 mgr.y (mgr.14901) 4599 : cluster [DBG] pgmap v4606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:54.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:53 smithi006 bash[33918]: audit 2023-08-28T00:31:53.017266+0000 mon.a (mon.0) 3322 : audit [DBG] from='client.? 172.21.15.6:0/1273434808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:53 smithi130 bash[27579]: cluster 2023-08-28T00:31:52.708328+0000 mgr.y (mgr.14901) 4599 : cluster [DBG] pgmap v4606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:53 smithi130 bash[27579]: audit 2023-08-28T00:31:53.017266+0000 mon.a (mon.0) 3322 : audit [DBG] from='client.? 172.21.15.6:0/1273434808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:54.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:31:54.221Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:31:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:31:54.223Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:31:54.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:54 smithi130 bash[51302]: debug 2023-08-28T00:31:54.600+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:54.603779+0000) 2023-08-28T00:31:55.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:55 smithi130 bash[51302]: debug 2023-08-28T00:31:55.600+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:55.604090+0000) 2023-08-28T00:31:56.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:55 smithi006 bash[33918]: cluster 2023-08-28T00:31:54.709482+0000 mgr.y (mgr.14901) 4600 : cluster [DBG] pgmap v4607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:56.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:55 smithi006 bash[33918]: audit 2023-08-28T00:31:55.438751+0000 mon.c (mon.2) 1777 : audit [DBG] from='client.? 172.21.15.6:0/791503207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:56.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:55 smithi006 bash[27292]: cluster 2023-08-28T00:31:54.709482+0000 mgr.y (mgr.14901) 4600 : cluster [DBG] pgmap v4607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:56.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:55 smithi006 bash[27292]: audit 2023-08-28T00:31:55.438751+0000 mon.c (mon.2) 1777 : audit [DBG] from='client.? 172.21.15.6:0/791503207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:55 smithi130 bash[27579]: cluster 2023-08-28T00:31:54.709482+0000 mgr.y (mgr.14901) 4600 : cluster [DBG] pgmap v4607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:55 smithi130 bash[27579]: audit 2023-08-28T00:31:55.438751+0000 mon.c (mon.2) 1777 : audit [DBG] from='client.? 172.21.15.6:0/791503207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:56.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:56 smithi130 bash[51302]: debug 2023-08-28T00:31:56.600+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:56.604385+0000) 2023-08-28T00:31:57.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:57 smithi130 bash[51302]: debug 2023-08-28T00:31:57.600+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:57.604722+0000) 2023-08-28T00:31:58.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:57 smithi006 bash[33918]: cluster 2023-08-28T00:31:56.710112+0000 mgr.y (mgr.14901) 4601 : cluster [DBG] pgmap v4608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:58.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:57 smithi006 bash[33918]: audit 2023-08-28T00:31:57.847196+0000 mon.a (mon.0) 3323 : audit [DBG] from='client.? 172.21.15.6:0/2617611830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:58.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:57 smithi006 bash[27292]: cluster 2023-08-28T00:31:56.710112+0000 mgr.y (mgr.14901) 4601 : cluster [DBG] pgmap v4608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:58.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:57 smithi006 bash[27292]: audit 2023-08-28T00:31:57.847196+0000 mon.a (mon.0) 3323 : audit [DBG] from='client.? 172.21.15.6:0/2617611830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:58.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:57 smithi130 bash[27579]: cluster 2023-08-28T00:31:56.710112+0000 mgr.y (mgr.14901) 4601 : cluster [DBG] pgmap v4608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:31:58.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:57 smithi130 bash[27579]: audit 2023-08-28T00:31:57.847196+0000 mon.a (mon.0) 3323 : audit [DBG] from='client.? 172.21.15.6:0/2617611830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:58.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:58 smithi130 bash[51302]: debug 2023-08-28T00:31:58.600+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:58.605054+0000) 2023-08-28T00:31:59.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:31:59 smithi130 bash[51302]: debug 2023-08-28T00:31:59.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:59.605420+0000) 2023-08-28T00:32:00.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:31:59 smithi006 bash[33918]: cluster 2023-08-28T00:31:58.710760+0000 mgr.y (mgr.14901) 4602 : cluster [DBG] pgmap v4609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:00.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:31:59 smithi006 bash[27292]: cluster 2023-08-28T00:31:58.710760+0000 mgr.y (mgr.14901) 4602 : cluster [DBG] pgmap v4609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:00.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:31:59 smithi130 bash[27579]: cluster 2023-08-28T00:31:58.710760+0000 mgr.y (mgr.14901) 4602 : cluster [DBG] pgmap v4609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:00.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:00 smithi130 bash[51302]: debug 2023-08-28T00:32:00.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:00.605753+0000) 2023-08-28T00:32:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:32:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:32:00] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:32:01.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:00 smithi006 bash[33918]: audit 2023-08-28T00:32:00.265376+0000 mon.c (mon.2) 1778 : audit [DBG] from='client.? 172.21.15.6:0/2809773463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:32:00.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:32:00.764Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:01.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:00 smithi006 bash[27292]: audit 2023-08-28T00:32:00.265376+0000 mon.c (mon.2) 1778 : audit [DBG] from='client.? 172.21.15.6:0/2809773463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:01.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:00 smithi130 bash[27579]: audit 2023-08-28T00:32:00.265376+0000 mon.c (mon.2) 1778 : audit [DBG] from='client.? 172.21.15.6:0/2809773463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:01.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:01 smithi130 bash[51302]: debug 2023-08-28T00:32:01.600+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:01.606082+0000) 2023-08-28T00:32:02.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:01 smithi006 bash[27292]: cluster 2023-08-28T00:32:00.712045+0000 mgr.y (mgr.14901) 4603 : cluster [DBG] pgmap v4610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:02.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:01 smithi006 bash[33918]: cluster 2023-08-28T00:32:00.712045+0000 mgr.y (mgr.14901) 4603 : cluster [DBG] pgmap v4610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:02.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:01 smithi130 bash[27579]: cluster 2023-08-28T00:32:00.712045+0000 mgr.y (mgr.14901) 4603 : cluster [DBG] pgmap v4610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:32:02.341Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:32:02.343Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:02.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:02 smithi130 bash[51302]: debug 2023-08-28T00:32:02.600+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:02.606394+0000) 2023-08-28T00:32:03.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:02 smithi006 bash[33918]: audit 2023-08-28T00:32:02.680000+0000 mon.a (mon.0) 3324 : audit [DBG] from='client.? 172.21.15.6:0/2460754013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:03.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:02 smithi006 bash[27292]: audit 2023-08-28T00:32:02.680000+0000 mon.a (mon.0) 3324 : audit [DBG] from='client.? 172.21.15.6:0/2460754013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:03.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:02 smithi130 bash[27579]: audit 2023-08-28T00:32:02.680000+0000 mon.a (mon.0) 3324 : audit [DBG] from='client.? 172.21.15.6:0/2460754013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:03.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:03 smithi130 bash[51302]: debug 2023-08-28T00:32:03.600+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:03.606725+0000) 2023-08-28T00:32:04.225 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:03 smithi006 bash[27292]: cluster 2023-08-28T00:32:02.712749+0000 mgr.y (mgr.14901) 4604 : cluster [DBG] pgmap v4611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:04.226 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:03 smithi006 bash[33918]: cluster 2023-08-28T00:32:02.712749+0000 mgr.y (mgr.14901) 4604 : cluster [DBG] pgmap v4611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:04.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:03 smithi130 bash[27579]: cluster 2023-08-28T00:32:02.712749+0000 mgr.y (mgr.14901) 4604 : cluster [DBG] pgmap v4611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:04.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:32:04.222Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:32:04.224Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:04.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:04 smithi130 bash[51302]: debug 2023-08-28T00:32:04.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:04.607050+0000) 2023-08-28T00:32:05.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:05 smithi130 bash[51302]: debug 2023-08-28T00:32:05.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:05.607336+0000) 2023-08-28T00:32:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:05 smithi130 bash[27579]: cluster 2023-08-28T00:32:04.713850+0000 mgr.y (mgr.14901) 4605 : cluster [DBG] pgmap v4612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:05 smithi130 bash[27579]: audit 2023-08-28T00:32:05.094956+0000 mon.c (mon.2) 1779 : audit [DBG] from='client.? 172.21.15.6:0/1996753232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:06.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:05 smithi006 bash[27292]: cluster 2023-08-28T00:32:04.713850+0000 mgr.y (mgr.14901) 4605 : cluster [DBG] pgmap v4612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:06.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:05 smithi006 bash[27292]: audit 2023-08-28T00:32:05.094956+0000 mon.c (mon.2) 1779 : audit [DBG] from='client.? 172.21.15.6:0/1996753232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:05 smithi006 bash[33918]: cluster 2023-08-28T00:32:04.713850+0000 mgr.y (mgr.14901) 4605 : cluster [DBG] pgmap v4612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:05 smithi006 bash[33918]: audit 2023-08-28T00:32:05.094956+0000 mon.c (mon.2) 1779 : audit [DBG] from='client.? 172.21.15.6:0/1996753232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:06.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:06 smithi130 bash[51302]: debug 2023-08-28T00:32:06.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:06.607690+0000) 2023-08-28T00:32:07.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:07 smithi130 bash[51302]: debug 2023-08-28T00:32:07.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:07.608015+0000) 2023-08-28T00:32:08.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:07 smithi130 bash[27579]: cluster 2023-08-28T00:32:06.714535+0000 mgr.y (mgr.14901) 4606 : cluster [DBG] pgmap v4613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:08.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:07 smithi130 bash[27579]: audit 2023-08-28T00:32:07.508925+0000 mon.a (mon.0) 3325 : audit [DBG] from='client.? 172.21.15.6:0/866885395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:08.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:07 smithi006 bash[33918]: cluster 2023-08-28T00:32:06.714535+0000 mgr.y (mgr.14901) 4606 : cluster [DBG] pgmap v4613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:08.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:07 smithi006 bash[33918]: audit 2023-08-28T00:32:07.508925+0000 mon.a (mon.0) 3325 : audit [DBG] from='client.? 172.21.15.6:0/866885395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:08.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:07 smithi006 bash[27292]: cluster 2023-08-28T00:32:06.714535+0000 mgr.y (mgr.14901) 4606 : cluster [DBG] pgmap v4613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:08.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:07 smithi006 bash[27292]: audit 2023-08-28T00:32:07.508925+0000 mon.a (mon.0) 3325 : audit [DBG] from='client.? 172.21.15.6:0/866885395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:08.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:08 smithi130 bash[51302]: debug 2023-08-28T00:32:08.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:08.608380+0000) 2023-08-28T00:32:09.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:09 smithi130 bash[51302]: debug 2023-08-28T00:32:09.608+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:09.608701+0000) 2023-08-28T00:32:10.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:09 smithi130 bash[27579]: cluster 2023-08-28T00:32:08.715248+0000 mgr.y (mgr.14901) 4607 : cluster [DBG] pgmap v4614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:10.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:09 smithi130 bash[27579]: audit 2023-08-28T00:32:09.923102+0000 mon.c (mon.2) 1780 : audit [DBG] from='client.? 172.21.15.6:0/514597922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:10.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:09 smithi006 bash[27292]: cluster 2023-08-28T00:32:08.715248+0000 mgr.y (mgr.14901) 4607 : cluster [DBG] pgmap v4614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:10.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:09 smithi006 bash[27292]: audit 2023-08-28T00:32:09.923102+0000 mon.c (mon.2) 1780 : audit [DBG] from='client.? 172.21.15.6:0/514597922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:10.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:09 smithi006 bash[33918]: cluster 2023-08-28T00:32:08.715248+0000 mgr.y (mgr.14901) 4607 : cluster [DBG] pgmap v4614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:10.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:09 smithi006 bash[33918]: audit 2023-08-28T00:32:09.923102+0000 mon.c (mon.2) 1780 : audit [DBG] from='client.? 172.21.15.6:0/514597922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:10 smithi130 bash[51302]: debug 2023-08-28T00:32:10.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:10.608954+0000) 2023-08-28T00:32:11.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:32:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:32:10] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:32:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:32:10.763Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:32:10.765Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:11.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:11 smithi130 bash[51302]: debug 2023-08-28T00:32:11.608+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:11.609204+0000) 2023-08-28T00:32:12.345 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:11 smithi006 bash[27292]: cluster 2023-08-28T00:32:10.716461+0000 mgr.y (mgr.14901) 4608 : cluster [DBG] pgmap v4615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:12.345 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:11 smithi006 bash[33918]: cluster 2023-08-28T00:32:10.716461+0000 mgr.y (mgr.14901) 4608 : cluster [DBG] pgmap v4615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:12.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:11 smithi130 bash[27579]: cluster 2023-08-28T00:32:10.716461+0000 mgr.y (mgr.14901) 4608 : cluster [DBG] pgmap v4615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:32:12.341Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:32:12.343Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:12.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:12 smithi130 bash[51302]: debug 2023-08-28T00:32:12.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:12.609438+0000) 2023-08-28T00:32:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:13 smithi130 bash[27579]: audit 2023-08-28T00:32:12.337814+0000 mon.a (mon.0) 3326 : audit [DBG] from='client.? 172.21.15.6:0/1807361799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:13.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:13 smithi130 bash[27579]: audit 2023-08-28T00:32:12.884152+0000 mon.b (mon.1) 562 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:32:13.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:13 smithi006 bash[33918]: audit 2023-08-28T00:32:12.337814+0000 mon.a (mon.0) 3326 : audit [DBG] from='client.? 172.21.15.6:0/1807361799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:13.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:13 smithi006 bash[33918]: audit 2023-08-28T00:32:12.884152+0000 mon.b (mon.1) 562 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:32:13.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:13 smithi006 bash[27292]: audit 2023-08-28T00:32:12.337814+0000 mon.a (mon.0) 3326 : audit [DBG] from='client.? 172.21.15.6:0/1807361799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:13.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:13 smithi006 bash[27292]: audit 2023-08-28T00:32:12.884152+0000 mon.b (mon.1) 562 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:32:13.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:13 smithi130 bash[51302]: debug 2023-08-28T00:32:13.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:13.609680+0000) 2023-08-28T00:32:14.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:14 smithi130 bash[27579]: cluster 2023-08-28T00:32:12.717383+0000 mgr.y (mgr.14901) 4609 : cluster [DBG] pgmap v4616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:14.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:14 smithi006 bash[27292]: cluster 2023-08-28T00:32:12.717383+0000 mgr.y (mgr.14901) 4609 : cluster [DBG] pgmap v4616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:14.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:14 smithi006 bash[33918]: cluster 2023-08-28T00:32:12.717383+0000 mgr.y (mgr.14901) 4609 : cluster [DBG] pgmap v4616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:32:14.222Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:32:14.224Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:14 smithi130 bash[51302]: debug 2023-08-28T00:32:14.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:14.609967+0000) 2023-08-28T00:32:15.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:15 smithi130 bash[27579]: audit 2023-08-28T00:32:14.746373+0000 mon.a (mon.0) 3327 : audit [DBG] from='client.? 172.21.15.6:0/4152379433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:15.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:15 smithi006 bash[27292]: audit 2023-08-28T00:32:14.746373+0000 mon.a (mon.0) 3327 : audit [DBG] from='client.? 172.21.15.6:0/4152379433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:15.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:15 smithi006 bash[33918]: audit 2023-08-28T00:32:14.746373+0000 mon.a (mon.0) 3327 : audit [DBG] from='client.? 172.21.15.6:0/4152379433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:15.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:15 smithi130 bash[51302]: debug 2023-08-28T00:32:15.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:15.610248+0000) 2023-08-28T00:32:16.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:16 smithi130 bash[27579]: cluster 2023-08-28T00:32:14.718482+0000 mgr.y (mgr.14901) 4610 : cluster [DBG] pgmap v4617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:16.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:16 smithi006 bash[33918]: cluster 2023-08-28T00:32:14.718482+0000 mgr.y (mgr.14901) 4610 : cluster [DBG] pgmap v4617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:16.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:16 smithi006 bash[27292]: cluster 2023-08-28T00:32:14.718482+0000 mgr.y (mgr.14901) 4610 : cluster [DBG] pgmap v4617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:16.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:16 smithi130 bash[51302]: debug 2023-08-28T00:32:16.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:16.610514+0000) 2023-08-28T00:32:17.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:17 smithi130 bash[51302]: debug 2023-08-28T00:32:17.608+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:17.610785+0000) 2023-08-28T00:32:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:18 smithi130 bash[27579]: cluster 2023-08-28T00:32:16.719147+0000 mgr.y (mgr.14901) 4611 : cluster [DBG] pgmap v4618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:18 smithi130 bash[27579]: audit 2023-08-28T00:32:17.158025+0000 mon.a (mon.0) 3328 : audit [DBG] from='client.? 172.21.15.6:0/49201853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:18.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:18 smithi006 bash[33918]: cluster 2023-08-28T00:32:16.719147+0000 mgr.y (mgr.14901) 4611 : cluster [DBG] pgmap v4618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:18.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:18 smithi006 bash[33918]: audit 2023-08-28T00:32:17.158025+0000 mon.a (mon.0) 3328 : audit [DBG] from='client.? 172.21.15.6:0/49201853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:18.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:18 smithi006 bash[27292]: cluster 2023-08-28T00:32:16.719147+0000 mgr.y (mgr.14901) 4611 : cluster [DBG] pgmap v4618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:18.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:18 smithi006 bash[27292]: audit 2023-08-28T00:32:17.158025+0000 mon.a (mon.0) 3328 : audit [DBG] from='client.? 172.21.15.6:0/49201853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:18.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:18 smithi130 bash[51302]: debug 2023-08-28T00:32:18.608+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:18.610980+0000) 2023-08-28T00:32:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:19 smithi130 bash[51302]: debug 2023-08-28T00:32:19.608+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:19.611227+0000) 2023-08-28T00:32:19.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:19 smithi130 bash[27579]: cluster 2023-08-28T00:32:18.719804+0000 mgr.y (mgr.14901) 4612 : cluster [DBG] pgmap v4619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:19.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:19 smithi130 bash[27579]: audit 2023-08-28T00:32:18.820978+0000 mon.a (mon.0) 3329 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:32:19.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:19 smithi130 bash[27579]: audit 2023-08-28T00:32:19.228735+0000 mon.a (mon.0) 3330 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:32:19.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:19 smithi130 bash[27579]: audit 2023-08-28T00:32:19.515474+0000 mon.b (mon.1) 563 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:32:19.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:19 smithi130 bash[27579]: audit 2023-08-28T00:32:19.517459+0000 mon.b (mon.1) 564 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:32:19.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:19 smithi130 bash[27579]: audit 2023-08-28T00:32:19.524315+0000 mon.a (mon.0) 3331 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:32:19.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:19 smithi130 bash[27579]: audit 2023-08-28T00:32:19.581270+0000 mon.a (mon.0) 3332 : audit [DBG] from='client.? 172.21.15.6:0/615008322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:20.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:19 smithi006 bash[33918]: cluster 2023-08-28T00:32:18.719804+0000 mgr.y (mgr.14901) 4612 : cluster [DBG] pgmap v4619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:20.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:19 smithi006 bash[33918]: audit 2023-08-28T00:32:18.820978+0000 mon.a (mon.0) 3329 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:32:20.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:19 smithi006 bash[33918]: audit 2023-08-28T00:32:19.228735+0000 mon.a (mon.0) 3330 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:32:20.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:19 smithi006 bash[33918]: audit 2023-08-28T00:32:19.515474+0000 mon.b (mon.1) 563 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:32:20.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:19 smithi006 bash[33918]: audit 2023-08-28T00:32:19.517459+0000 mon.b (mon.1) 564 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:32:20.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:19 smithi006 bash[33918]: audit 2023-08-28T00:32:19.524315+0000 mon.a (mon.0) 3331 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:32:20.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:19 smithi006 bash[33918]: audit 2023-08-28T00:32:19.581270+0000 mon.a (mon.0) 3332 : audit [DBG] from='client.? 172.21.15.6:0/615008322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:20.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:19 smithi006 bash[27292]: cluster 2023-08-28T00:32:18.719804+0000 mgr.y (mgr.14901) 4612 : cluster [DBG] pgmap v4619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:20.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:19 smithi006 bash[27292]: audit 2023-08-28T00:32:18.820978+0000 mon.a (mon.0) 3329 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:32:20.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:19 smithi006 bash[27292]: audit 2023-08-28T00:32:19.228735+0000 mon.a (mon.0) 3330 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:32:20.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:19 smithi006 bash[27292]: audit 2023-08-28T00:32:19.515474+0000 mon.b (mon.1) 563 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:32:20.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:19 smithi006 bash[27292]: audit 2023-08-28T00:32:19.517459+0000 mon.b (mon.1) 564 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:32:20.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:19 smithi006 bash[27292]: audit 2023-08-28T00:32:19.524315+0000 mon.a (mon.0) 3331 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:32:20.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:19 smithi006 bash[27292]: audit 2023-08-28T00:32:19.581270+0000 mon.a (mon.0) 3332 : audit [DBG] from='client.? 172.21.15.6:0/615008322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:20 smithi130 bash[51302]: debug 2023-08-28T00:32:20.607+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:20.611518+0000) 2023-08-28T00:32:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:32:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:32:20] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:32:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:32:20.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:32:20.766Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:21.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:21 smithi130 bash[51302]: debug 2023-08-28T00:32:21.607+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:21.611793+0000) 2023-08-28T00:32:22.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:21 smithi006 bash[33918]: cluster 2023-08-28T00:32:20.720986+0000 mgr.y (mgr.14901) 4613 : cluster [DBG] pgmap v4620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:22.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:21 smithi006 bash[27292]: cluster 2023-08-28T00:32:20.720986+0000 mgr.y (mgr.14901) 4613 : cluster [DBG] pgmap v4620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:22.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:21 smithi130 bash[27579]: cluster 2023-08-28T00:32:20.720986+0000 mgr.y (mgr.14901) 4613 : cluster [DBG] pgmap v4620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:32:22.342Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:32:22.344Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:22 smithi130 bash[51302]: debug 2023-08-28T00:32:22.607+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:22.612091+0000) 2023-08-28T00:32:23.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:22 smithi006 bash[33918]: audit 2023-08-28T00:32:21.996494+0000 mon.a (mon.0) 3333 : audit [DBG] from='client.? 172.21.15.6:0/4254197005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:23.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:22 smithi006 bash[27292]: audit 2023-08-28T00:32:21.996494+0000 mon.a (mon.0) 3333 : audit [DBG] from='client.? 172.21.15.6:0/4254197005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:23.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:22 smithi130 bash[27579]: audit 2023-08-28T00:32:21.996494+0000 mon.a (mon.0) 3333 : audit [DBG] from='client.? 172.21.15.6:0/4254197005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:23.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:23 smithi130 bash[51302]: debug 2023-08-28T00:32:23.607+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:23.612421+0000) 2023-08-28T00:32:24.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:23 smithi006 bash[33918]: cluster 2023-08-28T00:32:22.721615+0000 mgr.y (mgr.14901) 4614 : cluster [DBG] pgmap v4621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:24.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:23 smithi006 bash[27292]: cluster 2023-08-28T00:32:22.721615+0000 mgr.y (mgr.14901) 4614 : cluster [DBG] pgmap v4621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:23 smithi130 bash[27579]: cluster 2023-08-28T00:32:22.721615+0000 mgr.y (mgr.14901) 4614 : cluster [DBG] pgmap v4621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:24.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:32:24.223Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:32:24.225Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:24 smithi130 bash[51302]: debug 2023-08-28T00:32:24.607+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:24.612699+0000) 2023-08-28T00:32:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:24 smithi006 bash[33918]: audit 2023-08-28T00:32:24.405607+0000 mon.c (mon.2) 1781 : audit [DBG] from='client.? 172.21.15.6:0/3447325814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:25.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:24 smithi006 bash[27292]: audit 2023-08-28T00:32:24.405607+0000 mon.c (mon.2) 1781 : audit [DBG] from='client.? 172.21.15.6:0/3447325814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:24 smithi130 bash[27579]: audit 2023-08-28T00:32:24.405607+0000 mon.c (mon.2) 1781 : audit [DBG] from='client.? 172.21.15.6:0/3447325814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:25.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:25 smithi130 bash[51302]: debug 2023-08-28T00:32:25.607+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:25.612960+0000) 2023-08-28T00:32:26.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:25 smithi006 bash[27292]: cluster 2023-08-28T00:32:24.722723+0000 mgr.y (mgr.14901) 4615 : cluster [DBG] pgmap v4622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:26.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:25 smithi006 bash[33918]: cluster 2023-08-28T00:32:24.722723+0000 mgr.y (mgr.14901) 4615 : cluster [DBG] pgmap v4622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:26.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:25 smithi130 bash[27579]: cluster 2023-08-28T00:32:24.722723+0000 mgr.y (mgr.14901) 4615 : cluster [DBG] pgmap v4622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:26.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:26 smithi130 bash[51302]: debug 2023-08-28T00:32:26.607+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:26.613231+0000) 2023-08-28T00:32:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:26 smithi006 bash[33918]: audit 2023-08-28T00:32:26.817178+0000 mon.c (mon.2) 1782 : audit [DBG] from='client.? 172.21.15.6:0/3709816465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:27.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:26 smithi006 bash[27292]: audit 2023-08-28T00:32:26.817178+0000 mon.c (mon.2) 1782 : audit [DBG] from='client.? 172.21.15.6:0/3709816465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:27.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:26 smithi130 bash[27579]: audit 2023-08-28T00:32:26.817178+0000 mon.c (mon.2) 1782 : audit [DBG] from='client.? 172.21.15.6:0/3709816465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:27.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:27 smithi130 bash[51302]: debug 2023-08-28T00:32:27.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:27.613517+0000) 2023-08-28T00:32:28.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:27 smithi006 bash[27292]: cluster 2023-08-28T00:32:26.723509+0000 mgr.y (mgr.14901) 4616 : cluster [DBG] pgmap v4623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:28.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:27 smithi006 bash[33918]: cluster 2023-08-28T00:32:26.723509+0000 mgr.y (mgr.14901) 4616 : cluster [DBG] pgmap v4623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:28.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:27 smithi130 bash[27579]: cluster 2023-08-28T00:32:26.723509+0000 mgr.y (mgr.14901) 4616 : cluster [DBG] pgmap v4623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:28 smithi130 bash[51302]: debug 2023-08-28T00:32:28.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:28.613775+0000) 2023-08-28T00:32:29.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:29 smithi130 bash[51302]: debug 2023-08-28T00:32:29.607+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:29.614072+0000) 2023-08-28T00:32:30.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:29 smithi006 bash[27292]: cluster 2023-08-28T00:32:28.724315+0000 mgr.y (mgr.14901) 4617 : cluster [DBG] pgmap v4624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:30.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:29 smithi006 bash[27292]: audit 2023-08-28T00:32:29.228925+0000 mon.c (mon.2) 1783 : audit [DBG] from='client.? 172.21.15.6:0/1358738267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:30.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:29 smithi006 bash[33918]: cluster 2023-08-28T00:32:28.724315+0000 mgr.y (mgr.14901) 4617 : cluster [DBG] pgmap v4624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:30.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:29 smithi006 bash[33918]: audit 2023-08-28T00:32:29.228925+0000 mon.c (mon.2) 1783 : audit [DBG] from='client.? 172.21.15.6:0/1358738267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:29 smithi130 bash[27579]: cluster 2023-08-28T00:32:28.724315+0000 mgr.y (mgr.14901) 4617 : cluster [DBG] pgmap v4624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:29 smithi130 bash[27579]: audit 2023-08-28T00:32:29.228925+0000 mon.c (mon.2) 1783 : audit [DBG] from='client.? 172.21.15.6:0/1358738267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:30 smithi130 bash[51302]: debug 2023-08-28T00:32:30.607+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:30.614340+0000) 2023-08-28T00:32:31.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:32:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:32:30] "GET /metrics HTTP/1.1" 200 193134 "" "Prometheus/2.33.4" 2023-08-28T00:32:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:32:30.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": context deadline exceeded" 2023-08-28T00:32:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:32:30.766Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:31.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:31 smithi130 bash[51302]: debug 2023-08-28T00:32:31.607+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:31.614604+0000) 2023-08-28T00:32:32.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:31 smithi006 bash[33918]: cluster 2023-08-28T00:32:30.725616+0000 mgr.y (mgr.14901) 4618 : cluster [DBG] pgmap v4625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:32.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:31 smithi006 bash[33918]: audit 2023-08-28T00:32:31.651538+0000 mon.a (mon.0) 3334 : audit [DBG] from='client.? 172.21.15.6:0/4015869950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:32.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:31 smithi006 bash[27292]: cluster 2023-08-28T00:32:30.725616+0000 mgr.y (mgr.14901) 4618 : cluster [DBG] pgmap v4625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:32.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:31 smithi006 bash[27292]: audit 2023-08-28T00:32:31.651538+0000 mon.a (mon.0) 3334 : audit [DBG] from='client.? 172.21.15.6:0/4015869950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:31 smithi130 bash[27579]: cluster 2023-08-28T00:32:30.725616+0000 mgr.y (mgr.14901) 4618 : cluster [DBG] pgmap v4625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:31 smithi130 bash[27579]: audit 2023-08-28T00:32:31.651538+0000 mon.a (mon.0) 3334 : audit [DBG] from='client.? 172.21.15.6:0/4015869950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:32:32.343Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:32:32.345Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:32 smithi130 bash[51302]: debug 2023-08-28T00:32:32.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:32.614927+0000) 2023-08-28T00:32:33.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:33 smithi130 bash[51302]: debug 2023-08-28T00:32:33.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:33.615215+0000) 2023-08-28T00:32:34.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:33 smithi006 bash[27292]: cluster 2023-08-28T00:32:32.726331+0000 mgr.y (mgr.14901) 4619 : cluster [DBG] pgmap v4626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:34.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:33 smithi006 bash[33918]: cluster 2023-08-28T00:32:32.726331+0000 mgr.y (mgr.14901) 4619 : cluster [DBG] pgmap v4626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:34.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:33 smithi130 bash[27579]: cluster 2023-08-28T00:32:32.726331+0000 mgr.y (mgr.14901) 4619 : cluster [DBG] pgmap v4626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:34.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:32:34.225Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:32:34.227Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:34 smithi130 bash[51302]: debug 2023-08-28T00:32:34.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:34.615437+0000) 2023-08-28T00:32:35.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:34 smithi006 bash[27292]: audit 2023-08-28T00:32:34.069450+0000 mon.c (mon.2) 1784 : audit [DBG] from='client.? 172.21.15.6:0/190324528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:35.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:34 smithi006 bash[33918]: audit 2023-08-28T00:32:34.069450+0000 mon.c (mon.2) 1784 : audit [DBG] from='client.? 172.21.15.6:0/190324528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:34 smithi130 bash[27579]: audit 2023-08-28T00:32:34.069450+0000 mon.c (mon.2) 1784 : audit [DBG] from='client.? 172.21.15.6:0/190324528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:35.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:35 smithi130 bash[51302]: debug 2023-08-28T00:32:35.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:35.615665+0000) 2023-08-28T00:32:36.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:35 smithi006 bash[33918]: cluster 2023-08-28T00:32:34.727471+0000 mgr.y (mgr.14901) 4620 : cluster [DBG] pgmap v4627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:36.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:35 smithi006 bash[27292]: cluster 2023-08-28T00:32:34.727471+0000 mgr.y (mgr.14901) 4620 : cluster [DBG] pgmap v4627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:35 smithi130 bash[27579]: cluster 2023-08-28T00:32:34.727471+0000 mgr.y (mgr.14901) 4620 : cluster [DBG] pgmap v4627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:36 smithi130 bash[51302]: debug 2023-08-28T00:32:36.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:36.615934+0000) 2023-08-28T00:32:37.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:36 smithi006 bash[27292]: audit 2023-08-28T00:32:36.486951+0000 mon.a (mon.0) 3335 : audit [DBG] from='client.? 172.21.15.6:0/229208078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:37.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:36 smithi006 bash[33918]: audit 2023-08-28T00:32:36.486951+0000 mon.a (mon.0) 3335 : audit [DBG] from='client.? 172.21.15.6:0/229208078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:37.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:36 smithi130 bash[27579]: audit 2023-08-28T00:32:36.486951+0000 mon.a (mon.0) 3335 : audit [DBG] from='client.? 172.21.15.6:0/229208078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:37 smithi130 bash[51302]: debug 2023-08-28T00:32:37.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:37.616153+0000) 2023-08-28T00:32:38.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:37 smithi006 bash[27292]: cluster 2023-08-28T00:32:36.728187+0000 mgr.y (mgr.14901) 4621 : cluster [DBG] pgmap v4628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:38.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:37 smithi006 bash[33918]: cluster 2023-08-28T00:32:36.728187+0000 mgr.y (mgr.14901) 4621 : cluster [DBG] pgmap v4628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:38.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:37 smithi130 bash[27579]: cluster 2023-08-28T00:32:36.728187+0000 mgr.y (mgr.14901) 4621 : cluster [DBG] pgmap v4628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:38 smithi130 bash[51302]: debug 2023-08-28T00:32:38.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:38.616436+0000) 2023-08-28T00:32:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:38 smithi130 bash[27579]: audit 2023-08-28T00:32:38.901475+0000 mon.c (mon.2) 1785 : audit [DBG] from='client.? 172.21.15.6:0/2051329312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:39.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:38 smithi006 bash[27292]: audit 2023-08-28T00:32:38.901475+0000 mon.c (mon.2) 1785 : audit [DBG] from='client.? 172.21.15.6:0/2051329312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:39.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:38 smithi006 bash[33918]: audit 2023-08-28T00:32:38.901475+0000 mon.c (mon.2) 1785 : audit [DBG] from='client.? 172.21.15.6:0/2051329312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:39.978 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:39 smithi130 bash[51302]: debug 2023-08-28T00:32:39.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:39.616740+0000) 2023-08-28T00:32:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:39 smithi130 bash[27579]: cluster 2023-08-28T00:32:38.728877+0000 mgr.y (mgr.14901) 4622 : cluster [DBG] pgmap v4629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:40.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:39 smithi006 bash[27292]: cluster 2023-08-28T00:32:38.728877+0000 mgr.y (mgr.14901) 4622 : cluster [DBG] pgmap v4629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:39 smithi006 bash[33918]: cluster 2023-08-28T00:32:38.728877+0000 mgr.y (mgr.14901) 4622 : cluster [DBG] pgmap v4629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:41.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:40 smithi130 bash[51302]: debug 2023-08-28T00:32:40.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:40.617027+0000) 2023-08-28T00:32:41.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:32:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:32:40] "GET /metrics HTTP/1.1" 200 193134 "" "Prometheus/2.33.4" 2023-08-28T00:32:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:32:40.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:32:40.767Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:41.982 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:41 smithi130 bash[51302]: debug 2023-08-28T00:32:41.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:41.617293+0000) 2023-08-28T00:32:42.347 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:41 smithi006 bash[27292]: cluster 2023-08-28T00:32:40.730090+0000 mgr.y (mgr.14901) 4623 : cluster [DBG] pgmap v4630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:42.347 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:41 smithi006 bash[27292]: audit 2023-08-28T00:32:41.310548+0000 mon.a (mon.0) 3336 : audit [DBG] from='client.? 172.21.15.6:0/2645855849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:42.347 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:41 smithi006 bash[33918]: cluster 2023-08-28T00:32:40.730090+0000 mgr.y (mgr.14901) 4623 : cluster [DBG] pgmap v4630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:42.348 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:41 smithi006 bash[33918]: audit 2023-08-28T00:32:41.310548+0000 mon.a (mon.0) 3336 : audit [DBG] from='client.? 172.21.15.6:0/2645855849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:41 smithi130 bash[27579]: cluster 2023-08-28T00:32:40.730090+0000 mgr.y (mgr.14901) 4623 : cluster [DBG] pgmap v4630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:41 smithi130 bash[27579]: audit 2023-08-28T00:32:41.310548+0000 mon.a (mon.0) 3336 : audit [DBG] from='client.? 172.21.15.6:0/2645855849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:32:42.343Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:32:42.346Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:43.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:42 smithi130 bash[51302]: debug 2023-08-28T00:32:42.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:42.617517+0000) 2023-08-28T00:32:43.988 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:43 smithi130 bash[51302]: debug 2023-08-28T00:32:43.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:43.617764+0000) 2023-08-28T00:32:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:43 smithi130 bash[27579]: cluster 2023-08-28T00:32:42.730788+0000 mgr.y (mgr.14901) 4624 : cluster [DBG] pgmap v4631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:43 smithi130 bash[27579]: audit 2023-08-28T00:32:43.733155+0000 mon.a (mon.0) 3337 : audit [DBG] from='client.? 172.21.15.6:0/3852151051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:44.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:43 smithi006 bash[27292]: cluster 2023-08-28T00:32:42.730788+0000 mgr.y (mgr.14901) 4624 : cluster [DBG] pgmap v4631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:44.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:43 smithi006 bash[27292]: audit 2023-08-28T00:32:43.733155+0000 mon.a (mon.0) 3337 : audit [DBG] from='client.? 172.21.15.6:0/3852151051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:44.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:43 smithi006 bash[33918]: cluster 2023-08-28T00:32:42.730788+0000 mgr.y (mgr.14901) 4624 : cluster [DBG] pgmap v4631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:44.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:43 smithi006 bash[33918]: audit 2023-08-28T00:32:43.733155+0000 mon.a (mon.0) 3337 : audit [DBG] from='client.? 172.21.15.6:0/3852151051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:32:44.225Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:32:44.228Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:44 smithi130 bash[51302]: debug 2023-08-28T00:32:44.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:44.618063+0000) 2023-08-28T00:32:45.993 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:45 smithi130 bash[51302]: debug 2023-08-28T00:32:45.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:45.618330+0000) 2023-08-28T00:32:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:45 smithi130 bash[27579]: cluster 2023-08-28T00:32:44.731953+0000 mgr.y (mgr.14901) 4625 : cluster [DBG] pgmap v4632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:46.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:45 smithi006 bash[33918]: cluster 2023-08-28T00:32:44.731953+0000 mgr.y (mgr.14901) 4625 : cluster [DBG] pgmap v4632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:46.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:45 smithi006 bash[27292]: cluster 2023-08-28T00:32:44.731953+0000 mgr.y (mgr.14901) 4625 : cluster [DBG] pgmap v4632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:47.000 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:46 smithi130 bash[51302]: debug 2023-08-28T00:32:46.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:46.618594+0000) 2023-08-28T00:32:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:46 smithi130 bash[27579]: audit 2023-08-28T00:32:46.159736+0000 mon.a (mon.0) 3338 : audit [DBG] from='client.? 172.21.15.6:0/4169043479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:47.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:46 smithi006 bash[33918]: audit 2023-08-28T00:32:46.159736+0000 mon.a (mon.0) 3338 : audit [DBG] from='client.? 172.21.15.6:0/4169043479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:47.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:46 smithi006 bash[27292]: audit 2023-08-28T00:32:46.159736+0000 mon.a (mon.0) 3338 : audit [DBG] from='client.? 172.21.15.6:0/4169043479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:48.003 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:47 smithi130 bash[51302]: debug 2023-08-28T00:32:47.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:47.618910+0000) 2023-08-28T00:32:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:48 smithi130 bash[27579]: cluster 2023-08-28T00:32:46.732634+0000 mgr.y (mgr.14901) 4626 : cluster [DBG] pgmap v4633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:48.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:48 smithi006 bash[33918]: cluster 2023-08-28T00:32:46.732634+0000 mgr.y (mgr.14901) 4626 : cluster [DBG] pgmap v4633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:48.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:47 smithi006 bash[27292]: cluster 2023-08-28T00:32:46.732634+0000 mgr.y (mgr.14901) 4626 : cluster [DBG] pgmap v4633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:49.015 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:48 smithi130 bash[51302]: debug 2023-08-28T00:32:48.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:48.619116+0000) 2023-08-28T00:32:49.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:49 smithi130 bash[27579]: audit 2023-08-28T00:32:48.573892+0000 mon.c (mon.2) 1786 : audit [DBG] from='client.? 172.21.15.6:0/1874648993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:49.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:49 smithi006 bash[27292]: audit 2023-08-28T00:32:48.573892+0000 mon.c (mon.2) 1786 : audit [DBG] from='client.? 172.21.15.6:0/1874648993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:49.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:49 smithi006 bash[33918]: audit 2023-08-28T00:32:48.573892+0000 mon.c (mon.2) 1786 : audit [DBG] from='client.? 172.21.15.6:0/1874648993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:50.017 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:49 smithi130 bash[51302]: debug 2023-08-28T00:32:49.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:49.619336+0000) 2023-08-28T00:32:50.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:50 smithi130 bash[27579]: cluster 2023-08-28T00:32:48.733443+0000 mgr.y (mgr.14901) 4627 : cluster [DBG] pgmap v4634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:50.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:50 smithi006 bash[33918]: cluster 2023-08-28T00:32:48.733443+0000 mgr.y (mgr.14901) 4627 : cluster [DBG] pgmap v4634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:50.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:50 smithi006 bash[27292]: cluster 2023-08-28T00:32:48.733443+0000 mgr.y (mgr.14901) 4627 : cluster [DBG] pgmap v4634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:51.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:50 smithi130 bash[51302]: debug 2023-08-28T00:32:50.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:50.619563+0000) 2023-08-28T00:32:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:32:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:32:50] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T00:32:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:32:50.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:32:50.767Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:51.903 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:51 smithi130 bash[51302]: debug 2023-08-28T00:32:51.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:51.619851+0000) 2023-08-28T00:32:52.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:51 smithi006 bash[33918]: cluster 2023-08-28T00:32:50.734334+0000 mgr.y (mgr.14901) 4628 : cluster [DBG] pgmap v4635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:52.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:51 smithi006 bash[33918]: audit 2023-08-28T00:32:50.993415+0000 mon.a (mon.0) 3339 : audit [DBG] from='client.? 172.21.15.6:0/2348964797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:52.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:51 smithi006 bash[27292]: cluster 2023-08-28T00:32:50.734334+0000 mgr.y (mgr.14901) 4628 : cluster [DBG] pgmap v4635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:52.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:51 smithi006 bash[27292]: audit 2023-08-28T00:32:50.993415+0000 mon.a (mon.0) 3339 : audit [DBG] from='client.? 172.21.15.6:0/2348964797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:51 smithi130 bash[27579]: cluster 2023-08-28T00:32:50.734334+0000 mgr.y (mgr.14901) 4628 : cluster [DBG] pgmap v4635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:51 smithi130 bash[27579]: audit 2023-08-28T00:32:50.993415+0000 mon.a (mon.0) 3339 : audit [DBG] from='client.? 172.21.15.6:0/2348964797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:32:52.344Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:32:52.346Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:52 smithi130 bash[51302]: debug 2023-08-28T00:32:52.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:52.620071+0000) 2023-08-28T00:32:53.910 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:53 smithi130 bash[51302]: debug 2023-08-28T00:32:53.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:53.620298+0000) 2023-08-28T00:32:54.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:53 smithi006 bash[33918]: cluster 2023-08-28T00:32:52.734982+0000 mgr.y (mgr.14901) 4629 : cluster [DBG] pgmap v4636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:54.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:53 smithi006 bash[33918]: audit 2023-08-28T00:32:53.409852+0000 mon.c (mon.2) 1787 : audit [DBG] from='client.? 172.21.15.6:0/1714934145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:54.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:53 smithi006 bash[27292]: cluster 2023-08-28T00:32:52.734982+0000 mgr.y (mgr.14901) 4629 : cluster [DBG] pgmap v4636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:54.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:53 smithi006 bash[27292]: audit 2023-08-28T00:32:53.409852+0000 mon.c (mon.2) 1787 : audit [DBG] from='client.? 172.21.15.6:0/1714934145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:53 smithi130 bash[27579]: cluster 2023-08-28T00:32:52.734982+0000 mgr.y (mgr.14901) 4629 : cluster [DBG] pgmap v4636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:53 smithi130 bash[27579]: audit 2023-08-28T00:32:53.409852+0000 mon.c (mon.2) 1787 : audit [DBG] from='client.? 172.21.15.6:0/1714934145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:54.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:32:54.226Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:32:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:32:54.228Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:32:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:54 smithi130 bash[51302]: debug 2023-08-28T00:32:54.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:54.620557+0000) 2023-08-28T00:32:55.919 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:55 smithi130 bash[51302]: debug 2023-08-28T00:32:55.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:55.620813+0000) 2023-08-28T00:32:56.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:55 smithi006 bash[27292]: cluster 2023-08-28T00:32:54.736132+0000 mgr.y (mgr.14901) 4630 : cluster [DBG] pgmap v4637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:56.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:55 smithi006 bash[27292]: audit 2023-08-28T00:32:55.822841+0000 mon.c (mon.2) 1788 : audit [DBG] from='client.? 172.21.15.6:0/710751762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:56.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:55 smithi006 bash[33918]: cluster 2023-08-28T00:32:54.736132+0000 mgr.y (mgr.14901) 4630 : cluster [DBG] pgmap v4637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:56.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:55 smithi006 bash[33918]: audit 2023-08-28T00:32:55.822841+0000 mon.c (mon.2) 1788 : audit [DBG] from='client.? 172.21.15.6:0/710751762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:55 smithi130 bash[27579]: cluster 2023-08-28T00:32:54.736132+0000 mgr.y (mgr.14901) 4630 : cluster [DBG] pgmap v4637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:55 smithi130 bash[27579]: audit 2023-08-28T00:32:55.822841+0000 mon.c (mon.2) 1788 : audit [DBG] from='client.? 172.21.15.6:0/710751762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:57.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:56 smithi130 bash[51302]: debug 2023-08-28T00:32:56.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:56.621099+0000) 2023-08-28T00:32:57.924 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:57 smithi130 bash[51302]: debug 2023-08-28T00:32:57.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:57.621360+0000) 2023-08-28T00:32:58.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:57 smithi006 bash[33918]: cluster 2023-08-28T00:32:56.736808+0000 mgr.y (mgr.14901) 4631 : cluster [DBG] pgmap v4638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:58.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:57 smithi006 bash[27292]: cluster 2023-08-28T00:32:56.736808+0000 mgr.y (mgr.14901) 4631 : cluster [DBG] pgmap v4638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:58.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:57 smithi130 bash[27579]: cluster 2023-08-28T00:32:56.736808+0000 mgr.y (mgr.14901) 4631 : cluster [DBG] pgmap v4638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:32:58.935 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:58 smithi130 bash[51302]: debug 2023-08-28T00:32:58.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:58.621628+0000) 2023-08-28T00:32:59.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:58 smithi006 bash[33918]: audit 2023-08-28T00:32:58.237322+0000 mon.a (mon.0) 3340 : audit [DBG] from='client.? 172.21.15.6:0/1550740426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:59.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:58 smithi006 bash[27292]: audit 2023-08-28T00:32:58.237322+0000 mon.a (mon.0) 3340 : audit [DBG] from='client.? 172.21.15.6:0/1550740426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:58 smithi130 bash[27579]: audit 2023-08-28T00:32:58.237322+0000 mon.a (mon.0) 3340 : audit [DBG] from='client.? 172.21.15.6:0/1550740426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:59.938 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:32:59 smithi130 bash[51302]: debug 2023-08-28T00:32:59.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:59.621970+0000) 2023-08-28T00:33:00.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:32:59 smithi006 bash[27292]: cluster 2023-08-28T00:32:58.737491+0000 mgr.y (mgr.14901) 4632 : cluster [DBG] pgmap v4639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:00.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:32:59 smithi006 bash[33918]: cluster 2023-08-28T00:32:58.737491+0000 mgr.y (mgr.14901) 4632 : cluster [DBG] pgmap v4639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:00.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:32:59 smithi130 bash[27579]: cluster 2023-08-28T00:32:58.737491+0000 mgr.y (mgr.14901) 4632 : cluster [DBG] pgmap v4639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:00.946 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:00 smithi130 bash[51302]: debug 2023-08-28T00:33:00.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:00.622268+0000) 2023-08-28T00:33:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:00 smithi006 bash[33918]: audit 2023-08-28T00:33:00.649524+0000 mon.a (mon.0) 3341 : audit [DBG] from='client.? 172.21.15.6:0/2207549662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:01.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:33:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:33:00] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:33:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:33:00.766Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:33:00.767Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:01.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:00 smithi006 bash[27292]: audit 2023-08-28T00:33:00.649524+0000 mon.a (mon.0) 3341 : audit [DBG] from='client.? 172.21.15.6:0/2207549662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:01.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:00 smithi130 bash[27579]: audit 2023-08-28T00:33:00.649524+0000 mon.a (mon.0) 3341 : audit [DBG] from='client.? 172.21.15.6:0/2207549662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:01.949 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:01 smithi130 bash[51302]: debug 2023-08-28T00:33:01.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:01.622547+0000) 2023-08-28T00:33:02.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:01 smithi006 bash[33918]: cluster 2023-08-28T00:33:00.738662+0000 mgr.y (mgr.14901) 4633 : cluster [DBG] pgmap v4640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:02.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:01 smithi006 bash[27292]: cluster 2023-08-28T00:33:00.738662+0000 mgr.y (mgr.14901) 4633 : cluster [DBG] pgmap v4640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:02.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:01 smithi130 bash[27579]: cluster 2023-08-28T00:33:00.738662+0000 mgr.y (mgr.14901) 4633 : cluster [DBG] pgmap v4640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:33:02.345Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:33:02.347Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:02 smithi130 bash[51302]: debug 2023-08-28T00:33:02.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:02.622936+0000) 2023-08-28T00:33:03.955 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:03 smithi130 bash[51302]: debug 2023-08-28T00:33:03.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:03.623178+0000) 2023-08-28T00:33:04.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:03 smithi006 bash[33918]: cluster 2023-08-28T00:33:02.739332+0000 mgr.y (mgr.14901) 4634 : cluster [DBG] pgmap v4641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:04.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:03 smithi006 bash[33918]: audit 2023-08-28T00:33:03.060929+0000 mon.a (mon.0) 3342 : audit [DBG] from='client.? 172.21.15.6:0/2779504240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:04.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:03 smithi006 bash[27292]: cluster 2023-08-28T00:33:02.739332+0000 mgr.y (mgr.14901) 4634 : cluster [DBG] pgmap v4641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:04.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:03 smithi006 bash[27292]: audit 2023-08-28T00:33:03.060929+0000 mon.a (mon.0) 3342 : audit [DBG] from='client.? 172.21.15.6:0/2779504240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:04.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:03 smithi130 bash[27579]: cluster 2023-08-28T00:33:02.739332+0000 mgr.y (mgr.14901) 4634 : cluster [DBG] pgmap v4641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:04.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:03 smithi130 bash[27579]: audit 2023-08-28T00:33:03.060929+0000 mon.a (mon.0) 3342 : audit [DBG] from='client.? 172.21.15.6:0/2779504240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:04.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:33:04.227Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:33:04.230Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:04 smithi130 bash[51302]: debug 2023-08-28T00:33:04.623+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:04.623449+0000) 2023-08-28T00:33:05.960 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:05 smithi130 bash[51302]: debug 2023-08-28T00:33:05.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:05.623754+0000) 2023-08-28T00:33:06.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:05 smithi006 bash[27292]: cluster 2023-08-28T00:33:04.740486+0000 mgr.y (mgr.14901) 4635 : cluster [DBG] pgmap v4642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:06.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:05 smithi006 bash[27292]: audit 2023-08-28T00:33:05.472413+0000 mon.c (mon.2) 1789 : audit [DBG] from='client.? 172.21.15.6:0/2601090474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:06.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:05 smithi006 bash[33918]: cluster 2023-08-28T00:33:04.740486+0000 mgr.y (mgr.14901) 4635 : cluster [DBG] pgmap v4642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:06.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:05 smithi006 bash[33918]: audit 2023-08-28T00:33:05.472413+0000 mon.c (mon.2) 1789 : audit [DBG] from='client.? 172.21.15.6:0/2601090474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:05 smithi130 bash[27579]: cluster 2023-08-28T00:33:04.740486+0000 mgr.y (mgr.14901) 4635 : cluster [DBG] pgmap v4642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:05 smithi130 bash[27579]: audit 2023-08-28T00:33:05.472413+0000 mon.c (mon.2) 1789 : audit [DBG] from='client.? 172.21.15.6:0/2601090474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:07.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:06 smithi130 bash[51302]: debug 2023-08-28T00:33:06.623+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:06.624041+0000) 2023-08-28T00:33:07.967 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:07 smithi130 bash[51302]: debug 2023-08-28T00:33:07.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:07.624347+0000) 2023-08-28T00:33:08.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:07 smithi006 bash[33918]: cluster 2023-08-28T00:33:06.741192+0000 mgr.y (mgr.14901) 4636 : cluster [DBG] pgmap v4643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:08.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:07 smithi006 bash[33918]: audit 2023-08-28T00:33:07.883370+0000 mon.a (mon.0) 3343 : audit [DBG] from='client.? 172.21.15.6:0/306252541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:08.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:07 smithi006 bash[27292]: cluster 2023-08-28T00:33:06.741192+0000 mgr.y (mgr.14901) 4636 : cluster [DBG] pgmap v4643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:08.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:07 smithi006 bash[27292]: audit 2023-08-28T00:33:07.883370+0000 mon.a (mon.0) 3343 : audit [DBG] from='client.? 172.21.15.6:0/306252541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:08.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:07 smithi130 bash[27579]: cluster 2023-08-28T00:33:06.741192+0000 mgr.y (mgr.14901) 4636 : cluster [DBG] pgmap v4643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:08.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:07 smithi130 bash[27579]: audit 2023-08-28T00:33:07.883370+0000 mon.a (mon.0) 3343 : audit [DBG] from='client.? 172.21.15.6:0/306252541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:09.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:08 smithi130 bash[51302]: debug 2023-08-28T00:33:08.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:08.624611+0000) 2023-08-28T00:33:09.970 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:09 smithi130 bash[51302]: debug 2023-08-28T00:33:09.623+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:09.624921+0000) 2023-08-28T00:33:10.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:09 smithi130 bash[27579]: cluster 2023-08-28T00:33:08.741824+0000 mgr.y (mgr.14901) 4637 : cluster [DBG] pgmap v4644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:10.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:09 smithi006 bash[33918]: cluster 2023-08-28T00:33:08.741824+0000 mgr.y (mgr.14901) 4637 : cluster [DBG] pgmap v4644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:10.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:09 smithi006 bash[27292]: cluster 2023-08-28T00:33:08.741824+0000 mgr.y (mgr.14901) 4637 : cluster [DBG] pgmap v4644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:10.978 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:10 smithi130 bash[51302]: debug 2023-08-28T00:33:10.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:10.625260+0000) 2023-08-28T00:33:11.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:10 smithi006 bash[27292]: audit 2023-08-28T00:33:10.305839+0000 mon.a (mon.0) 3344 : audit [DBG] from='client.? 172.21.15.6:0/987058680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:11.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:10 smithi006 bash[33918]: audit 2023-08-28T00:33:10.305839+0000 mon.a (mon.0) 3344 : audit [DBG] from='client.? 172.21.15.6:0/987058680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:11.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:33:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:33:10] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:33:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:33:10.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:33:10.769Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:11.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:10 smithi130 bash[27579]: audit 2023-08-28T00:33:10.305839+0000 mon.a (mon.0) 3344 : audit [DBG] from='client.? 172.21.15.6:0/987058680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:11.981 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:11 smithi130 bash[51302]: debug 2023-08-28T00:33:11.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:11.625531+0000) 2023-08-28T00:33:12.349 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:11 smithi006 bash[27292]: cluster 2023-08-28T00:33:10.742837+0000 mgr.y (mgr.14901) 4638 : cluster [DBG] pgmap v4645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:12.350 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:11 smithi006 bash[33918]: cluster 2023-08-28T00:33:10.742837+0000 mgr.y (mgr.14901) 4638 : cluster [DBG] pgmap v4645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:12.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:11 smithi130 bash[27579]: cluster 2023-08-28T00:33:10.742837+0000 mgr.y (mgr.14901) 4638 : cluster [DBG] pgmap v4645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:33:12.346Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:33:12.348Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:12.990 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:12 smithi130 bash[51302]: debug 2023-08-28T00:33:12.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:12.625841+0000) 2023-08-28T00:33:13.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:12 smithi130 bash[27579]: audit 2023-08-28T00:33:12.718609+0000 mon.c (mon.2) 1790 : audit [DBG] from='client.? 172.21.15.6:0/1010526713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:13.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:12 smithi006 bash[27292]: audit 2023-08-28T00:33:12.718609+0000 mon.c (mon.2) 1790 : audit [DBG] from='client.? 172.21.15.6:0/1010526713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:13.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:12 smithi006 bash[33918]: audit 2023-08-28T00:33:12.718609+0000 mon.c (mon.2) 1790 : audit [DBG] from='client.? 172.21.15.6:0/1010526713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:13.992 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:13 smithi130 bash[51302]: debug 2023-08-28T00:33:13.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:13.626111+0000) 2023-08-28T00:33:14.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:13 smithi130 bash[27579]: cluster 2023-08-28T00:33:12.743605+0000 mgr.y (mgr.14901) 4639 : cluster [DBG] pgmap v4646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:14.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:13 smithi006 bash[33918]: cluster 2023-08-28T00:33:12.743605+0000 mgr.y (mgr.14901) 4639 : cluster [DBG] pgmap v4646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:33:14.229Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:33:14.231Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:14.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:13 smithi006 bash[27292]: cluster 2023-08-28T00:33:12.743605+0000 mgr.y (mgr.14901) 4639 : cluster [DBG] pgmap v4646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:15.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:14 smithi130 bash[51302]: debug 2023-08-28T00:33:14.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:14.626442+0000) 2023-08-28T00:33:15.998 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:15 smithi130 bash[51302]: debug 2023-08-28T00:33:15.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:15.626715+0000) 2023-08-28T00:33:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:15 smithi130 bash[27579]: cluster 2023-08-28T00:33:14.744739+0000 mgr.y (mgr.14901) 4640 : cluster [DBG] pgmap v4647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:15 smithi130 bash[27579]: audit 2023-08-28T00:33:15.128215+0000 mon.c (mon.2) 1791 : audit [DBG] from='client.? 172.21.15.6:0/4214267976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:16.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:15 smithi006 bash[27292]: cluster 2023-08-28T00:33:14.744739+0000 mgr.y (mgr.14901) 4640 : cluster [DBG] pgmap v4647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:16.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:15 smithi006 bash[27292]: audit 2023-08-28T00:33:15.128215+0000 mon.c (mon.2) 1791 : audit [DBG] from='client.? 172.21.15.6:0/4214267976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:16.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:15 smithi006 bash[33918]: cluster 2023-08-28T00:33:14.744739+0000 mgr.y (mgr.14901) 4640 : cluster [DBG] pgmap v4647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:16.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:15 smithi006 bash[33918]: audit 2023-08-28T00:33:15.128215+0000 mon.c (mon.2) 1791 : audit [DBG] from='client.? 172.21.15.6:0/4214267976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:17.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:16 smithi130 bash[51302]: debug 2023-08-28T00:33:16.623+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:16.627014+0000) 2023-08-28T00:33:18.003 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:17 smithi130 bash[51302]: debug 2023-08-28T00:33:17.622+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:17.627284+0000) 2023-08-28T00:33:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:18 smithi130 bash[27579]: cluster 2023-08-28T00:33:16.745355+0000 mgr.y (mgr.14901) 4641 : cluster [DBG] pgmap v4648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:18 smithi130 bash[27579]: audit 2023-08-28T00:33:17.558596+0000 mon.c (mon.2) 1792 : audit [DBG] from='client.? 172.21.15.6:0/2961328915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:18.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:17 smithi006 bash[27292]: cluster 2023-08-28T00:33:16.745355+0000 mgr.y (mgr.14901) 4641 : cluster [DBG] pgmap v4648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:18.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:18 smithi006 bash[27292]: audit 2023-08-28T00:33:17.558596+0000 mon.c (mon.2) 1792 : audit [DBG] from='client.? 172.21.15.6:0/2961328915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:18.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:18 smithi006 bash[33918]: cluster 2023-08-28T00:33:16.745355+0000 mgr.y (mgr.14901) 4641 : cluster [DBG] pgmap v4648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:18.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:18 smithi006 bash[33918]: audit 2023-08-28T00:33:17.558596+0000 mon.c (mon.2) 1792 : audit [DBG] from='client.? 172.21.15.6:0/2961328915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:19.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:18 smithi130 bash[51302]: debug 2023-08-28T00:33:18.622+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:18.627547+0000) 2023-08-28T00:33:20.008 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:19 smithi130 bash[51302]: debug 2023-08-28T00:33:19.622+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:19.627816+0000) 2023-08-28T00:33:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:20 smithi130 bash[27579]: cluster 2023-08-28T00:33:18.746018+0000 mgr.y (mgr.14901) 4642 : cluster [DBG] pgmap v4649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:20 smithi130 bash[27579]: audit 2023-08-28T00:33:19.535968+0000 mon.b (mon.1) 565 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:33:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:20 smithi130 bash[27579]: audit 2023-08-28T00:33:19.964669+0000 mon.c (mon.2) 1793 : audit [DBG] from='client.? 172.21.15.6:0/2216492624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:20.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:20 smithi006 bash[27292]: cluster 2023-08-28T00:33:18.746018+0000 mgr.y (mgr.14901) 4642 : cluster [DBG] pgmap v4649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:20.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:20 smithi006 bash[27292]: audit 2023-08-28T00:33:19.535968+0000 mon.b (mon.1) 565 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:33:20.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:20 smithi006 bash[27292]: audit 2023-08-28T00:33:19.964669+0000 mon.c (mon.2) 1793 : audit [DBG] from='client.? 172.21.15.6:0/2216492624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:20.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:20 smithi006 bash[33918]: cluster 2023-08-28T00:33:18.746018+0000 mgr.y (mgr.14901) 4642 : cluster [DBG] pgmap v4649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:20.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:20 smithi006 bash[33918]: audit 2023-08-28T00:33:19.535968+0000 mon.b (mon.1) 565 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:33:20.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:20 smithi006 bash[33918]: audit 2023-08-28T00:33:19.964669+0000 mon.c (mon.2) 1793 : audit [DBG] from='client.? 172.21.15.6:0/2216492624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:21.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:20 smithi130 bash[51302]: debug 2023-08-28T00:33:20.622+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:20.628078+0000) 2023-08-28T00:33:21.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:33:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:33:20] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:33:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:33:20.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:33:20.770Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:22.012 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:21 smithi130 bash[51302]: debug 2023-08-28T00:33:21.622+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:21.628334+0000) 2023-08-28T00:33:22.350 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:22 smithi006 bash[33918]: cluster 2023-08-28T00:33:20.747159+0000 mgr.y (mgr.14901) 4643 : cluster [DBG] pgmap v4650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:22.350 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:22 smithi006 bash[27292]: cluster 2023-08-28T00:33:20.747159+0000 mgr.y (mgr.14901) 4643 : cluster [DBG] pgmap v4650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:22.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:22 smithi130 bash[27579]: cluster 2023-08-28T00:33:20.747159+0000 mgr.y (mgr.14901) 4643 : cluster [DBG] pgmap v4650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:33:22.347Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:33:22.348Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:23.021 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:22 smithi130 bash[51302]: debug 2023-08-28T00:33:22.626+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:22.628607+0000) 2023-08-28T00:33:23.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:23 smithi130 bash[27579]: audit 2023-08-28T00:33:22.393058+0000 mon.c (mon.2) 1794 : audit [DBG] from='client.? 172.21.15.6:0/3057322275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:23.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:23 smithi006 bash[27292]: audit 2023-08-28T00:33:22.393058+0000 mon.c (mon.2) 1794 : audit [DBG] from='client.? 172.21.15.6:0/3057322275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:23.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:23 smithi006 bash[33918]: audit 2023-08-28T00:33:22.393058+0000 mon.c (mon.2) 1794 : audit [DBG] from='client.? 172.21.15.6:0/3057322275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:24.023 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:23 smithi130 bash[51302]: debug 2023-08-28T00:33:23.626+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:23.628881+0000) 2023-08-28T00:33:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:24 smithi130 bash[27579]: cluster 2023-08-28T00:33:22.747880+0000 mgr.y (mgr.14901) 4644 : cluster [DBG] pgmap v4651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:24.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:24 smithi006 bash[33918]: cluster 2023-08-28T00:33:22.747880+0000 mgr.y (mgr.14901) 4644 : cluster [DBG] pgmap v4651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:33:24.230Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:33:24.232Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:24.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:24 smithi006 bash[27292]: cluster 2023-08-28T00:33:22.747880+0000 mgr.y (mgr.14901) 4644 : cluster [DBG] pgmap v4651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:25.032 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:24 smithi130 bash[51302]: debug 2023-08-28T00:33:24.622+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:24.629161+0000) 2023-08-28T00:33:25.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:25 smithi130 bash[27579]: audit 2023-08-28T00:33:24.839781+0000 mon.a (mon.0) 3345 : audit [DBG] from='client.? 172.21.15.6:0/1195190075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:25.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:25 smithi006 bash[27292]: audit 2023-08-28T00:33:24.839781+0000 mon.a (mon.0) 3345 : audit [DBG] from='client.? 172.21.15.6:0/1195190075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:25.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:25 smithi006 bash[33918]: audit 2023-08-28T00:33:24.839781+0000 mon.a (mon.0) 3345 : audit [DBG] from='client.? 172.21.15.6:0/1195190075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:26.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:25 smithi130 bash[51302]: debug 2023-08-28T00:33:25.626+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:25.629416+0000) 2023-08-28T00:33:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:26 smithi006 bash[33918]: cluster 2023-08-28T00:33:24.748737+0000 mgr.y (mgr.14901) 4645 : cluster [DBG] pgmap v4652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:26 smithi006 bash[33918]: audit 2023-08-28T00:33:25.379259+0000 mon.a (mon.0) 3346 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:33:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:26 smithi006 bash[33918]: audit 2023-08-28T00:33:25.917349+0000 mon.a (mon.0) 3347 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:33:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:26 smithi006 bash[33918]: audit 2023-08-28T00:33:26.215759+0000 mon.b (mon.1) 566 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:33:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:26 smithi006 bash[33918]: audit 2023-08-28T00:33:26.217968+0000 mon.b (mon.1) 567 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:33:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:26 smithi006 bash[33918]: audit 2023-08-28T00:33:26.226412+0000 mon.a (mon.0) 3348 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:33:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:26 smithi006 bash[27292]: cluster 2023-08-28T00:33:24.748737+0000 mgr.y (mgr.14901) 4645 : cluster [DBG] pgmap v4652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:26.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:26 smithi006 bash[27292]: audit 2023-08-28T00:33:25.379259+0000 mon.a (mon.0) 3346 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:33:26.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:26 smithi006 bash[27292]: audit 2023-08-28T00:33:25.917349+0000 mon.a (mon.0) 3347 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:33:26.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:26 smithi006 bash[27292]: audit 2023-08-28T00:33:26.215759+0000 mon.b (mon.1) 566 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:33:26.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:26 smithi006 bash[27292]: audit 2023-08-28T00:33:26.217968+0000 mon.b (mon.1) 567 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:33:26.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:26 smithi006 bash[27292]: audit 2023-08-28T00:33:26.226412+0000 mon.a (mon.0) 3348 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:33:26.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:26 smithi130 bash[51302]: debug 2023-08-28T00:33:26.626+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:26.629729+0000) 2023-08-28T00:33:26.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:26 smithi130 bash[27579]: cluster 2023-08-28T00:33:24.748737+0000 mgr.y (mgr.14901) 4645 : cluster [DBG] pgmap v4652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:26.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:26 smithi130 bash[27579]: audit 2023-08-28T00:33:25.379259+0000 mon.a (mon.0) 3346 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:33:26.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:26 smithi130 bash[27579]: audit 2023-08-28T00:33:25.917349+0000 mon.a (mon.0) 3347 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:33:26.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:26 smithi130 bash[27579]: audit 2023-08-28T00:33:26.215759+0000 mon.b (mon.1) 566 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:33:26.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:26 smithi130 bash[27579]: audit 2023-08-28T00:33:26.217968+0000 mon.b (mon.1) 567 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:33:26.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:26 smithi130 bash[27579]: audit 2023-08-28T00:33:26.226412+0000 mon.a (mon.0) 3348 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:33:27.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:27 smithi006 bash[27292]: audit 2023-08-28T00:33:27.265631+0000 mon.c (mon.2) 1795 : audit [DBG] from='client.? 172.21.15.6:0/3472518575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:27.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:27 smithi006 bash[33918]: audit 2023-08-28T00:33:27.265631+0000 mon.c (mon.2) 1795 : audit [DBG] from='client.? 172.21.15.6:0/3472518575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:27 smithi130 bash[27579]: audit 2023-08-28T00:33:27.265631+0000 mon.c (mon.2) 1795 : audit [DBG] from='client.? 172.21.15.6:0/3472518575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:27.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:27 smithi130 bash[51302]: debug 2023-08-28T00:33:27.626+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:27.630042+0000) 2023-08-28T00:33:28.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:28 smithi006 bash[27292]: cluster 2023-08-28T00:33:26.749343+0000 mgr.y (mgr.14901) 4646 : cluster [DBG] pgmap v4653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:28.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:28 smithi006 bash[33918]: cluster 2023-08-28T00:33:26.749343+0000 mgr.y (mgr.14901) 4646 : cluster [DBG] pgmap v4653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:28 smithi130 bash[51302]: debug 2023-08-28T00:33:28.626+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:28.630309+0000) 2023-08-28T00:33:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:28 smithi130 bash[27579]: cluster 2023-08-28T00:33:26.749343+0000 mgr.y (mgr.14901) 4646 : cluster [DBG] pgmap v4653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:30.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:29 smithi130 bash[51302]: debug 2023-08-28T00:33:29.626+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:29.630606+0000) 2023-08-28T00:33:30.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:30 smithi006 bash[27292]: cluster 2023-08-28T00:33:28.750000+0000 mgr.y (mgr.14901) 4647 : cluster [DBG] pgmap v4654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:30.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:30 smithi006 bash[27292]: audit 2023-08-28T00:33:29.679015+0000 mon.a (mon.0) 3349 : audit [DBG] from='client.? 172.21.15.6:0/3889841391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:30.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:30 smithi006 bash[33918]: cluster 2023-08-28T00:33:28.750000+0000 mgr.y (mgr.14901) 4647 : cluster [DBG] pgmap v4654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:30.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:30 smithi006 bash[33918]: audit 2023-08-28T00:33:29.679015+0000 mon.a (mon.0) 3349 : audit [DBG] from='client.? 172.21.15.6:0/3889841391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:30 smithi130 bash[51302]: debug 2023-08-28T00:33:30.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:30.630937+0000) 2023-08-28T00:33:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:30 smithi130 bash[27579]: cluster 2023-08-28T00:33:28.750000+0000 mgr.y (mgr.14901) 4647 : cluster [DBG] pgmap v4654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:30.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:30 smithi130 bash[27579]: audit 2023-08-28T00:33:29.679015+0000 mon.a (mon.0) 3349 : audit [DBG] from='client.? 172.21.15.6:0/3889841391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:31.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:33:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:33:30] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:33:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:33:30.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:33:30.771Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:32.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:31 smithi130 bash[51302]: debug 2023-08-28T00:33:31.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:31.631165+0000) 2023-08-28T00:33:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:32 smithi006 bash[27292]: cluster 2023-08-28T00:33:30.751035+0000 mgr.y (mgr.14901) 4648 : cluster [DBG] pgmap v4655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:32 smithi006 bash[27292]: audit 2023-08-28T00:33:32.103333+0000 mon.c (mon.2) 1796 : audit [DBG] from='client.? 172.21.15.6:0/1912785891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:32 smithi006 bash[33918]: cluster 2023-08-28T00:33:30.751035+0000 mgr.y (mgr.14901) 4648 : cluster [DBG] pgmap v4655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:32 smithi006 bash[33918]: audit 2023-08-28T00:33:32.103333+0000 mon.c (mon.2) 1796 : audit [DBG] from='client.? 172.21.15.6:0/1912785891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:32.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:33:32.347Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:32.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:33:32.349Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:32 smithi130 bash[51302]: debug 2023-08-28T00:33:32.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:32.631449+0000) 2023-08-28T00:33:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:32 smithi130 bash[27579]: cluster 2023-08-28T00:33:30.751035+0000 mgr.y (mgr.14901) 4648 : cluster [DBG] pgmap v4655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:32.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:32 smithi130 bash[27579]: audit 2023-08-28T00:33:32.103333+0000 mon.c (mon.2) 1796 : audit [DBG] from='client.? 172.21.15.6:0/1912785891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:34.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:33 smithi130 bash[51302]: debug 2023-08-28T00:33:33.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:33.631726+0000) 2023-08-28T00:33:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:34 smithi006 bash[33918]: cluster 2023-08-28T00:33:32.751690+0000 mgr.y (mgr.14901) 4649 : cluster [DBG] pgmap v4656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:34 smithi006 bash[27292]: cluster 2023-08-28T00:33:32.751690+0000 mgr.y (mgr.14901) 4649 : cluster [DBG] pgmap v4656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:33:34.231Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:34.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:33:34.233Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:34 smithi130 bash[27579]: cluster 2023-08-28T00:33:32.751690+0000 mgr.y (mgr.14901) 4649 : cluster [DBG] pgmap v4656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:34 smithi130 bash[51302]: debug 2023-08-28T00:33:34.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:34.632013+0000) 2023-08-28T00:33:35.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:35 smithi006 bash[27292]: audit 2023-08-28T00:33:34.525443+0000 mon.c (mon.2) 1797 : audit [DBG] from='client.? 172.21.15.6:0/3470861694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:35.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:35 smithi006 bash[33918]: audit 2023-08-28T00:33:34.525443+0000 mon.c (mon.2) 1797 : audit [DBG] from='client.? 172.21.15.6:0/3470861694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:35.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:35 smithi130 bash[27579]: audit 2023-08-28T00:33:34.525443+0000 mon.c (mon.2) 1797 : audit [DBG] from='client.? 172.21.15.6:0/3470861694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:35.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:35 smithi130 bash[51302]: debug 2023-08-28T00:33:35.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:35.632302+0000) 2023-08-28T00:33:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:36 smithi006 bash[33918]: cluster 2023-08-28T00:33:34.752825+0000 mgr.y (mgr.14901) 4650 : cluster [DBG] pgmap v4657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:36.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:36 smithi006 bash[27292]: cluster 2023-08-28T00:33:34.752825+0000 mgr.y (mgr.14901) 4650 : cluster [DBG] pgmap v4657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:36 smithi130 bash[27579]: cluster 2023-08-28T00:33:34.752825+0000 mgr.y (mgr.14901) 4650 : cluster [DBG] pgmap v4657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:36 smithi130 bash[51302]: debug 2023-08-28T00:33:36.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:36.632558+0000) 2023-08-28T00:33:37.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:37 smithi006 bash[33918]: audit 2023-08-28T00:33:36.936895+0000 mon.a (mon.0) 3350 : audit [DBG] from='client.? 172.21.15.6:0/4001106032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:37.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:37 smithi006 bash[27292]: audit 2023-08-28T00:33:36.936895+0000 mon.a (mon.0) 3350 : audit [DBG] from='client.? 172.21.15.6:0/4001106032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:37.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:37 smithi130 bash[27579]: audit 2023-08-28T00:33:36.936895+0000 mon.a (mon.0) 3350 : audit [DBG] from='client.? 172.21.15.6:0/4001106032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:37 smithi130 bash[51302]: debug 2023-08-28T00:33:37.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:37.632841+0000) 2023-08-28T00:33:38.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:38 smithi006 bash[27292]: cluster 2023-08-28T00:33:36.753488+0000 mgr.y (mgr.14901) 4651 : cluster [DBG] pgmap v4658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:38.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:38 smithi006 bash[33918]: cluster 2023-08-28T00:33:36.753488+0000 mgr.y (mgr.14901) 4651 : cluster [DBG] pgmap v4658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:38 smithi130 bash[27579]: cluster 2023-08-28T00:33:36.753488+0000 mgr.y (mgr.14901) 4651 : cluster [DBG] pgmap v4658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:38 smithi130 bash[51302]: debug 2023-08-28T00:33:38.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:38.633090+0000) 2023-08-28T00:33:39.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:39 smithi006 bash[33918]: audit 2023-08-28T00:33:39.355053+0000 mon.a (mon.0) 3351 : audit [DBG] from='client.? 172.21.15.6:0/2865330671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:39.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:39 smithi006 bash[27292]: audit 2023-08-28T00:33:39.355053+0000 mon.a (mon.0) 3351 : audit [DBG] from='client.? 172.21.15.6:0/2865330671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:39.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:39 smithi130 bash[27579]: audit 2023-08-28T00:33:39.355053+0000 mon.a (mon.0) 3351 : audit [DBG] from='client.? 172.21.15.6:0/2865330671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:39 smithi130 bash[51302]: debug 2023-08-28T00:33:39.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:39.633464+0000) 2023-08-28T00:33:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:40 smithi006 bash[27292]: cluster 2023-08-28T00:33:38.754150+0000 mgr.y (mgr.14901) 4652 : cluster [DBG] pgmap v4659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:40 smithi006 bash[33918]: cluster 2023-08-28T00:33:38.754150+0000 mgr.y (mgr.14901) 4652 : cluster [DBG] pgmap v4659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:40 smithi130 bash[27579]: cluster 2023-08-28T00:33:38.754150+0000 mgr.y (mgr.14901) 4652 : cluster [DBG] pgmap v4659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:40.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:40 smithi130 bash[51302]: debug 2023-08-28T00:33:40.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:40.633734+0000) 2023-08-28T00:33:41.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:33:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:33:40] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:33:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:33:40.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:33:40.771Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:42.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:41 smithi130 bash[51302]: debug 2023-08-28T00:33:41.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:41.634016+0000) 2023-08-28T00:33:42.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:42 smithi006 bash[33918]: cluster 2023-08-28T00:33:40.755231+0000 mgr.y (mgr.14901) 4653 : cluster [DBG] pgmap v4660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:42.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:42 smithi006 bash[33918]: audit 2023-08-28T00:33:41.773732+0000 mon.a (mon.0) 3352 : audit [DBG] from='client.? 172.21.15.6:0/2391552497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:42.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:42 smithi006 bash[27292]: cluster 2023-08-28T00:33:40.755231+0000 mgr.y (mgr.14901) 4653 : cluster [DBG] pgmap v4660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:42.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:42 smithi006 bash[27292]: audit 2023-08-28T00:33:41.773732+0000 mon.a (mon.0) 3352 : audit [DBG] from='client.? 172.21.15.6:0/2391552497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:42.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:33:42.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:42.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:33:42.350Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:42 smithi130 bash[27579]: cluster 2023-08-28T00:33:40.755231+0000 mgr.y (mgr.14901) 4653 : cluster [DBG] pgmap v4660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:42 smithi130 bash[27579]: audit 2023-08-28T00:33:41.773732+0000 mon.a (mon.0) 3352 : audit [DBG] from='client.? 172.21.15.6:0/2391552497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:42 smithi130 bash[51302]: debug 2023-08-28T00:33:42.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:42.634327+0000) 2023-08-28T00:33:44.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:43 smithi130 bash[51302]: debug 2023-08-28T00:33:43.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:43.634639+0000) 2023-08-28T00:33:44.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:44 smithi006 bash[33918]: cluster 2023-08-28T00:33:42.755894+0000 mgr.y (mgr.14901) 4654 : cluster [DBG] pgmap v4661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:44.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:44 smithi006 bash[33918]: audit 2023-08-28T00:33:44.191753+0000 mon.a (mon.0) 3353 : audit [DBG] from='client.? 172.21.15.6:0/905197401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:33:44.231Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:33:44.233Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:44.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:44 smithi006 bash[27292]: cluster 2023-08-28T00:33:42.755894+0000 mgr.y (mgr.14901) 4654 : cluster [DBG] pgmap v4661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:44.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:44 smithi006 bash[27292]: audit 2023-08-28T00:33:44.191753+0000 mon.a (mon.0) 3353 : audit [DBG] from='client.? 172.21.15.6:0/905197401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:44 smithi130 bash[27579]: cluster 2023-08-28T00:33:42.755894+0000 mgr.y (mgr.14901) 4654 : cluster [DBG] pgmap v4661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:44 smithi130 bash[27579]: audit 2023-08-28T00:33:44.191753+0000 mon.a (mon.0) 3353 : audit [DBG] from='client.? 172.21.15.6:0/905197401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:44.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:44 smithi130 bash[51302]: debug 2023-08-28T00:33:44.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:44.634986+0000) 2023-08-28T00:33:46.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:45 smithi130 bash[51302]: debug 2023-08-28T00:33:45.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:45.635244+0000) 2023-08-28T00:33:46.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:46 smithi006 bash[33918]: cluster 2023-08-28T00:33:44.756998+0000 mgr.y (mgr.14901) 4655 : cluster [DBG] pgmap v4662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:46.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:46 smithi006 bash[27292]: cluster 2023-08-28T00:33:44.756998+0000 mgr.y (mgr.14901) 4655 : cluster [DBG] pgmap v4662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:46 smithi130 bash[27579]: cluster 2023-08-28T00:33:44.756998+0000 mgr.y (mgr.14901) 4655 : cluster [DBG] pgmap v4662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:46.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:46 smithi130 bash[51302]: debug 2023-08-28T00:33:46.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:46.635561+0000) 2023-08-28T00:33:47.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:47 smithi130 bash[27579]: audit 2023-08-28T00:33:46.600511+0000 mon.c (mon.2) 1798 : audit [DBG] from='client.? 172.21.15.6:0/2399431152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:47.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:47 smithi130 bash[51302]: debug 2023-08-28T00:33:47.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:47.635835+0000) 2023-08-28T00:33:47.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:47 smithi006 bash[27292]: audit 2023-08-28T00:33:46.600511+0000 mon.c (mon.2) 1798 : audit [DBG] from='client.? 172.21.15.6:0/2399431152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:47.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:47 smithi006 bash[33918]: audit 2023-08-28T00:33:46.600511+0000 mon.c (mon.2) 1798 : audit [DBG] from='client.? 172.21.15.6:0/2399431152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:48 smithi130 bash[27579]: cluster 2023-08-28T00:33:46.757655+0000 mgr.y (mgr.14901) 4656 : cluster [DBG] pgmap v4663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:48 smithi130 bash[51302]: debug 2023-08-28T00:33:48.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:48.636142+0000) 2023-08-28T00:33:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:48 smithi006 bash[27292]: cluster 2023-08-28T00:33:46.757655+0000 mgr.y (mgr.14901) 4656 : cluster [DBG] pgmap v4663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:48.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:48 smithi006 bash[33918]: cluster 2023-08-28T00:33:46.757655+0000 mgr.y (mgr.14901) 4656 : cluster [DBG] pgmap v4663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:49.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:49 smithi130 bash[27579]: audit 2023-08-28T00:33:49.010829+0000 mon.c (mon.2) 1799 : audit [DBG] from='client.? 172.21.15.6:0/1986231199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:49.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:49 smithi130 bash[51302]: debug 2023-08-28T00:33:49.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:49.636483+0000) 2023-08-28T00:33:49.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:49 smithi006 bash[27292]: audit 2023-08-28T00:33:49.010829+0000 mon.c (mon.2) 1799 : audit [DBG] from='client.? 172.21.15.6:0/1986231199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:49.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:49 smithi006 bash[33918]: audit 2023-08-28T00:33:49.010829+0000 mon.c (mon.2) 1799 : audit [DBG] from='client.? 172.21.15.6:0/1986231199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:50.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:50 smithi006 bash[27292]: cluster 2023-08-28T00:33:48.758255+0000 mgr.y (mgr.14901) 4657 : cluster [DBG] pgmap v4664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:50.756 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:50 smithi006 bash[33918]: cluster 2023-08-28T00:33:48.758255+0000 mgr.y (mgr.14901) 4657 : cluster [DBG] pgmap v4664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:50 smithi130 bash[27579]: cluster 2023-08-28T00:33:48.758255+0000 mgr.y (mgr.14901) 4657 : cluster [DBG] pgmap v4664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:50 smithi130 bash[51302]: debug 2023-08-28T00:33:50.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:50.636754+0000) 2023-08-28T00:33:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:33:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:33:50] "GET /metrics HTTP/1.1" 200 193132 "" "Prometheus/2.33.4" 2023-08-28T00:33:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:33:50.770Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:33:50.772Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:51.921 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:51 smithi130 bash[51302]: debug 2023-08-28T00:33:51.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:51.637101+0000) 2023-08-28T00:33:52.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:51 smithi006 bash[33918]: cluster 2023-08-28T00:33:50.759294+0000 mgr.y (mgr.14901) 4658 : cluster [DBG] pgmap v4665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:52.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:51 smithi006 bash[33918]: audit 2023-08-28T00:33:51.423161+0000 mon.c (mon.2) 1800 : audit [DBG] from='client.? 172.21.15.6:0/2802243222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:52.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:51 smithi006 bash[27292]: cluster 2023-08-28T00:33:50.759294+0000 mgr.y (mgr.14901) 4658 : cluster [DBG] pgmap v4665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:52.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:51 smithi006 bash[27292]: audit 2023-08-28T00:33:51.423161+0000 mon.c (mon.2) 1800 : audit [DBG] from='client.? 172.21.15.6:0/2802243222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:51 smithi130 bash[27579]: cluster 2023-08-28T00:33:50.759294+0000 mgr.y (mgr.14901) 4658 : cluster [DBG] pgmap v4665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:51 smithi130 bash[27579]: audit 2023-08-28T00:33:51.423161+0000 mon.c (mon.2) 1800 : audit [DBG] from='client.? 172.21.15.6:0/2802243222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:33:52.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:33:52.350Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:52 smithi130 bash[51302]: debug 2023-08-28T00:33:52.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:52.637378+0000) 2023-08-28T00:33:53.926 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:53 smithi130 bash[51302]: debug 2023-08-28T00:33:53.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:53.637651+0000) 2023-08-28T00:33:54.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:53 smithi006 bash[33918]: cluster 2023-08-28T00:33:52.759955+0000 mgr.y (mgr.14901) 4659 : cluster [DBG] pgmap v4666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:54.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:53 smithi006 bash[33918]: audit 2023-08-28T00:33:53.843803+0000 mon.a (mon.0) 3354 : audit [DBG] from='client.? 172.21.15.6:0/1617649310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:54.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:53 smithi006 bash[27292]: cluster 2023-08-28T00:33:52.759955+0000 mgr.y (mgr.14901) 4659 : cluster [DBG] pgmap v4666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:54.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:53 smithi006 bash[27292]: audit 2023-08-28T00:33:53.843803+0000 mon.a (mon.0) 3354 : audit [DBG] from='client.? 172.21.15.6:0/1617649310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:53 smithi130 bash[27579]: cluster 2023-08-28T00:33:52.759955+0000 mgr.y (mgr.14901) 4659 : cluster [DBG] pgmap v4666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:53 smithi130 bash[27579]: audit 2023-08-28T00:33:53.843803+0000 mon.a (mon.0) 3354 : audit [DBG] from='client.? 172.21.15.6:0/1617649310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:33:54.233Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:33:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:33:54.235Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:33:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:54 smithi130 bash[51302]: debug 2023-08-28T00:33:54.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:54.637926+0000) 2023-08-28T00:33:55.929 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:55 smithi130 bash[51302]: debug 2023-08-28T00:33:55.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:55.638203+0000) 2023-08-28T00:33:56.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:55 smithi006 bash[27292]: cluster 2023-08-28T00:33:54.761078+0000 mgr.y (mgr.14901) 4660 : cluster [DBG] pgmap v4667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:56.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:55 smithi006 bash[33918]: cluster 2023-08-28T00:33:54.761078+0000 mgr.y (mgr.14901) 4660 : cluster [DBG] pgmap v4667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:55 smithi130 bash[27579]: cluster 2023-08-28T00:33:54.761078+0000 mgr.y (mgr.14901) 4660 : cluster [DBG] pgmap v4667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:56.941 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:56 smithi130 bash[51302]: debug 2023-08-28T00:33:56.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:56.638474+0000) 2023-08-28T00:33:57.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:56 smithi006 bash[33918]: audit 2023-08-28T00:33:56.266499+0000 mon.c (mon.2) 1801 : audit [DBG] from='client.? 172.21.15.6:0/1613675255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:57.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:56 smithi006 bash[27292]: audit 2023-08-28T00:33:56.266499+0000 mon.c (mon.2) 1801 : audit [DBG] from='client.? 172.21.15.6:0/1613675255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:57.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:56 smithi130 bash[27579]: audit 2023-08-28T00:33:56.266499+0000 mon.c (mon.2) 1801 : audit [DBG] from='client.? 172.21.15.6:0/1613675255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:57.945 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:57 smithi130 bash[51302]: debug 2023-08-28T00:33:57.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:57.638697+0000) 2023-08-28T00:33:58.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:57 smithi006 bash[27292]: cluster 2023-08-28T00:33:56.761724+0000 mgr.y (mgr.14901) 4661 : cluster [DBG] pgmap v4668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:58.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:57 smithi006 bash[33918]: cluster 2023-08-28T00:33:56.761724+0000 mgr.y (mgr.14901) 4661 : cluster [DBG] pgmap v4668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:58.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:57 smithi130 bash[27579]: cluster 2023-08-28T00:33:56.761724+0000 mgr.y (mgr.14901) 4661 : cluster [DBG] pgmap v4668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:33:58.955 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:58 smithi130 bash[51302]: debug 2023-08-28T00:33:58.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:58.639008+0000) 2023-08-28T00:33:59.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:58 smithi006 bash[27292]: audit 2023-08-28T00:33:58.689257+0000 mon.c (mon.2) 1802 : audit [DBG] from='client.? 172.21.15.6:0/3607666956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:59.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:58 smithi006 bash[33918]: audit 2023-08-28T00:33:58.689257+0000 mon.c (mon.2) 1802 : audit [DBG] from='client.? 172.21.15.6:0/3607666956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:58 smithi130 bash[27579]: audit 2023-08-28T00:33:58.689257+0000 mon.c (mon.2) 1802 : audit [DBG] from='client.? 172.21.15.6:0/3607666956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:59.958 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:33:59 smithi130 bash[51302]: debug 2023-08-28T00:33:59.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:59.639304+0000) 2023-08-28T00:34:00.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:33:59 smithi006 bash[33918]: cluster 2023-08-28T00:33:58.762390+0000 mgr.y (mgr.14901) 4662 : cluster [DBG] pgmap v4669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:00.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:33:59 smithi006 bash[27292]: cluster 2023-08-28T00:33:58.762390+0000 mgr.y (mgr.14901) 4662 : cluster [DBG] pgmap v4669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:00.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:33:59 smithi130 bash[27579]: cluster 2023-08-28T00:33:58.762390+0000 mgr.y (mgr.14901) 4662 : cluster [DBG] pgmap v4669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:01.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:00 smithi130 bash[51302]: debug 2023-08-28T00:34:00.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:00.639588+0000) 2023-08-28T00:34:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:34:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:34:00] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:34:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:34:00.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:34:00.773Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:01.962 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:01 smithi130 bash[51302]: debug 2023-08-28T00:34:01.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:01.639863+0000) 2023-08-28T00:34:02.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:01 smithi006 bash[27292]: cluster 2023-08-28T00:34:00.763428+0000 mgr.y (mgr.14901) 4663 : cluster [DBG] pgmap v4670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:02.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:01 smithi006 bash[27292]: audit 2023-08-28T00:34:01.110271+0000 mon.c (mon.2) 1803 : audit [DBG] from='client.? 172.21.15.6:0/2055319109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:02.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:01 smithi006 bash[33918]: cluster 2023-08-28T00:34:00.763428+0000 mgr.y (mgr.14901) 4663 : cluster [DBG] pgmap v4670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:02.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:01 smithi006 bash[33918]: audit 2023-08-28T00:34:01.110271+0000 mon.c (mon.2) 1803 : audit [DBG] from='client.? 172.21.15.6:0/2055319109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:02.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:01 smithi130 bash[27579]: cluster 2023-08-28T00:34:00.763428+0000 mgr.y (mgr.14901) 4663 : cluster [DBG] pgmap v4670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:02.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:01 smithi130 bash[27579]: audit 2023-08-28T00:34:01.110271+0000 mon.c (mon.2) 1803 : audit [DBG] from='client.? 172.21.15.6:0/2055319109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:34:02.349Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:34:02.351Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:02 smithi130 bash[51302]: debug 2023-08-28T00:34:02.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:02.640142+0000) 2023-08-28T00:34:03.969 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:03 smithi130 bash[51302]: debug 2023-08-28T00:34:03.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:03.640408+0000) 2023-08-28T00:34:04.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:03 smithi006 bash[33918]: cluster 2023-08-28T00:34:02.764059+0000 mgr.y (mgr.14901) 4664 : cluster [DBG] pgmap v4671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:04.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:03 smithi006 bash[33918]: audit 2023-08-28T00:34:03.526914+0000 mon.a (mon.0) 3355 : audit [DBG] from='client.? 172.21.15.6:0/827719499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:04.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:03 smithi006 bash[27292]: cluster 2023-08-28T00:34:02.764059+0000 mgr.y (mgr.14901) 4664 : cluster [DBG] pgmap v4671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:04.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:03 smithi006 bash[27292]: audit 2023-08-28T00:34:03.526914+0000 mon.a (mon.0) 3355 : audit [DBG] from='client.? 172.21.15.6:0/827719499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:04.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:03 smithi130 bash[27579]: cluster 2023-08-28T00:34:02.764059+0000 mgr.y (mgr.14901) 4664 : cluster [DBG] pgmap v4671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:04.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:03 smithi130 bash[27579]: audit 2023-08-28T00:34:03.526914+0000 mon.a (mon.0) 3355 : audit [DBG] from='client.? 172.21.15.6:0/827719499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:04.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:34:04.233Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:34:04.235Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:04 smithi130 bash[51302]: debug 2023-08-28T00:34:04.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:04.640669+0000) 2023-08-28T00:34:05.972 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:05 smithi130 bash[51302]: debug 2023-08-28T00:34:05.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:05.640962+0000) 2023-08-28T00:34:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:05 smithi130 bash[27579]: cluster 2023-08-28T00:34:04.765212+0000 mgr.y (mgr.14901) 4665 : cluster [DBG] pgmap v4672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:05 smithi130 bash[27579]: audit 2023-08-28T00:34:05.943682+0000 mon.c (mon.2) 1804 : audit [DBG] from='client.? 172.21.15.6:0/2820459493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:06.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:05 smithi006 bash[27292]: cluster 2023-08-28T00:34:04.765212+0000 mgr.y (mgr.14901) 4665 : cluster [DBG] pgmap v4672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:06.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:05 smithi006 bash[27292]: audit 2023-08-28T00:34:05.943682+0000 mon.c (mon.2) 1804 : audit [DBG] from='client.? 172.21.15.6:0/2820459493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:05 smithi006 bash[33918]: cluster 2023-08-28T00:34:04.765212+0000 mgr.y (mgr.14901) 4665 : cluster [DBG] pgmap v4672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:05 smithi006 bash[33918]: audit 2023-08-28T00:34:05.943682+0000 mon.c (mon.2) 1804 : audit [DBG] from='client.? 172.21.15.6:0/2820459493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:07.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:06 smithi130 bash[51302]: debug 2023-08-28T00:34:06.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:06.641244+0000) 2023-08-28T00:34:07.977 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:07 smithi130 bash[51302]: debug 2023-08-28T00:34:07.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:07.641526+0000) 2023-08-28T00:34:08.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:07 smithi130 bash[27579]: cluster 2023-08-28T00:34:06.765848+0000 mgr.y (mgr.14901) 4666 : cluster [DBG] pgmap v4673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:08.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:07 smithi006 bash[33918]: cluster 2023-08-28T00:34:06.765848+0000 mgr.y (mgr.14901) 4666 : cluster [DBG] pgmap v4673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:08.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:07 smithi006 bash[27292]: cluster 2023-08-28T00:34:06.765848+0000 mgr.y (mgr.14901) 4666 : cluster [DBG] pgmap v4673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:08.984 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:08 smithi130 bash[51302]: debug 2023-08-28T00:34:08.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:08.641863+0000) 2023-08-28T00:34:09.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:08 smithi130 bash[27579]: audit 2023-08-28T00:34:08.353187+0000 mon.a (mon.0) 3356 : audit [DBG] from='client.? 172.21.15.6:0/1471086079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:09.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:08 smithi006 bash[27292]: audit 2023-08-28T00:34:08.353187+0000 mon.a (mon.0) 3356 : audit [DBG] from='client.? 172.21.15.6:0/1471086079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:09.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:08 smithi006 bash[33918]: audit 2023-08-28T00:34:08.353187+0000 mon.a (mon.0) 3356 : audit [DBG] from='client.? 172.21.15.6:0/1471086079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:09.988 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:09 smithi130 bash[51302]: debug 2023-08-28T00:34:09.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:09.642182+0000) 2023-08-28T00:34:10.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:09 smithi130 bash[27579]: cluster 2023-08-28T00:34:08.766457+0000 mgr.y (mgr.14901) 4667 : cluster [DBG] pgmap v4674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:10.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:09 smithi006 bash[33918]: cluster 2023-08-28T00:34:08.766457+0000 mgr.y (mgr.14901) 4667 : cluster [DBG] pgmap v4674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:10.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:09 smithi006 bash[27292]: cluster 2023-08-28T00:34:08.766457+0000 mgr.y (mgr.14901) 4667 : cluster [DBG] pgmap v4674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:10.996 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:10 smithi130 bash[51302]: debug 2023-08-28T00:34:10.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:10.642468+0000) 2023-08-28T00:34:11.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:34:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:34:10] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:34:11.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:10 smithi006 bash[33918]: audit 2023-08-28T00:34:10.771097+0000 mon.c (mon.2) 1805 : audit [DBG] from='client.? 172.21.15.6:0/969848174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:11.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:10 smithi006 bash[27292]: audit 2023-08-28T00:34:10.771097+0000 mon.c (mon.2) 1805 : audit [DBG] from='client.? 172.21.15.6:0/969848174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:11.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:34:10.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:11.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:34:10.774Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:11.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:10 smithi130 bash[27579]: audit 2023-08-28T00:34:10.771097+0000 mon.c (mon.2) 1805 : audit [DBG] from='client.? 172.21.15.6:0/969848174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:11.997 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:11 smithi130 bash[51302]: debug 2023-08-28T00:34:11.642+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:11.642795+0000) 2023-08-28T00:34:12.354 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:11 smithi006 bash[27292]: cluster 2023-08-28T00:34:10.767405+0000 mgr.y (mgr.14901) 4668 : cluster [DBG] pgmap v4675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:12.354 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:11 smithi006 bash[33918]: cluster 2023-08-28T00:34:10.767405+0000 mgr.y (mgr.14901) 4668 : cluster [DBG] pgmap v4675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:12.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:11 smithi130 bash[27579]: cluster 2023-08-28T00:34:10.767405+0000 mgr.y (mgr.14901) 4668 : cluster [DBG] pgmap v4675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:34:12.351Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:34:12.352Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:13.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:12 smithi130 bash[51302]: debug 2023-08-28T00:34:12.642+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:12.643085+0000) 2023-08-28T00:34:14.005 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:13 smithi130 bash[51302]: debug 2023-08-28T00:34:13.641+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:13.643405+0000) 2023-08-28T00:34:14.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:14 smithi130 bash[27579]: cluster 2023-08-28T00:34:12.768043+0000 mgr.y (mgr.14901) 4669 : cluster [DBG] pgmap v4676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:14.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:14 smithi130 bash[27579]: audit 2023-08-28T00:34:13.182455+0000 mon.c (mon.2) 1806 : audit [DBG] from='client.? 172.21.15.6:0/4260800095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:14.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:14 smithi006 bash[33918]: cluster 2023-08-28T00:34:12.768043+0000 mgr.y (mgr.14901) 4669 : cluster [DBG] pgmap v4676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:14.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:14 smithi006 bash[33918]: audit 2023-08-28T00:34:13.182455+0000 mon.c (mon.2) 1806 : audit [DBG] from='client.? 172.21.15.6:0/4260800095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:14.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:14 smithi006 bash[27292]: cluster 2023-08-28T00:34:12.768043+0000 mgr.y (mgr.14901) 4669 : cluster [DBG] pgmap v4676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:14.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:14 smithi006 bash[27292]: audit 2023-08-28T00:34:13.182455+0000 mon.c (mon.2) 1806 : audit [DBG] from='client.? 172.21.15.6:0/4260800095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:34:14.233Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:34:14.236Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:15.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:14 smithi130 bash[51302]: debug 2023-08-28T00:34:14.641+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:14.643676+0000) 2023-08-28T00:34:16.011 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:15 smithi130 bash[51302]: debug 2023-08-28T00:34:15.641+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:15.643935+0000) 2023-08-28T00:34:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:16 smithi130 bash[27579]: cluster 2023-08-28T00:34:14.769225+0000 mgr.y (mgr.14901) 4670 : cluster [DBG] pgmap v4677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:16 smithi130 bash[27579]: audit 2023-08-28T00:34:15.601059+0000 mon.c (mon.2) 1807 : audit [DBG] from='client.? 172.21.15.6:0/1191567607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:16.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:16 smithi006 bash[27292]: cluster 2023-08-28T00:34:14.769225+0000 mgr.y (mgr.14901) 4670 : cluster [DBG] pgmap v4677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:16.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:16 smithi006 bash[27292]: audit 2023-08-28T00:34:15.601059+0000 mon.c (mon.2) 1807 : audit [DBG] from='client.? 172.21.15.6:0/1191567607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:16.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:16 smithi006 bash[33918]: cluster 2023-08-28T00:34:14.769225+0000 mgr.y (mgr.14901) 4670 : cluster [DBG] pgmap v4677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:16.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:16 smithi006 bash[33918]: audit 2023-08-28T00:34:15.601059+0000 mon.c (mon.2) 1807 : audit [DBG] from='client.? 172.21.15.6:0/1191567607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:17.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:16 smithi130 bash[51302]: debug 2023-08-28T00:34:16.641+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:16.644198+0000) 2023-08-28T00:34:18.016 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:17 smithi130 bash[51302]: debug 2023-08-28T00:34:17.641+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:17.644521+0000) 2023-08-28T00:34:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:18 smithi130 bash[27579]: cluster 2023-08-28T00:34:16.769851+0000 mgr.y (mgr.14901) 4671 : cluster [DBG] pgmap v4678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:18.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:18 smithi006 bash[27292]: cluster 2023-08-28T00:34:16.769851+0000 mgr.y (mgr.14901) 4671 : cluster [DBG] pgmap v4678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:18.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:18 smithi006 bash[33918]: cluster 2023-08-28T00:34:16.769851+0000 mgr.y (mgr.14901) 4671 : cluster [DBG] pgmap v4678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:19.025 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:18 smithi130 bash[51302]: debug 2023-08-28T00:34:18.641+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:18.644835+0000) 2023-08-28T00:34:19.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:19 smithi130 bash[27579]: audit 2023-08-28T00:34:18.015934+0000 mon.c (mon.2) 1808 : audit [DBG] from='client.? 172.21.15.6:0/4142202940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:19.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:19 smithi006 bash[27292]: audit 2023-08-28T00:34:18.015934+0000 mon.c (mon.2) 1808 : audit [DBG] from='client.? 172.21.15.6:0/4142202940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:19.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:19 smithi006 bash[33918]: audit 2023-08-28T00:34:18.015934+0000 mon.c (mon.2) 1808 : audit [DBG] from='client.? 172.21.15.6:0/4142202940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:20.027 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:19 smithi130 bash[51302]: debug 2023-08-28T00:34:19.641+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:19.645151+0000) 2023-08-28T00:34:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:20 smithi130 bash[27579]: cluster 2023-08-28T00:34:18.770495+0000 mgr.y (mgr.14901) 4672 : cluster [DBG] pgmap v4679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:20.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:20 smithi006 bash[27292]: cluster 2023-08-28T00:34:18.770495+0000 mgr.y (mgr.14901) 4672 : cluster [DBG] pgmap v4679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:20.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:20 smithi006 bash[33918]: cluster 2023-08-28T00:34:18.770495+0000 mgr.y (mgr.14901) 4672 : cluster [DBG] pgmap v4679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:21.034 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:34:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:34:20] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T00:34:21.034 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:34:20.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:21.034 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:34:20.776Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:21.034 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:20 smithi130 bash[51302]: debug 2023-08-28T00:34:20.641+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:20.645471+0000) 2023-08-28T00:34:21.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:21 smithi130 bash[27579]: audit 2023-08-28T00:34:20.432812+0000 mon.c (mon.2) 1809 : audit [DBG] from='client.? 172.21.15.6:0/1141220866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:21.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:21 smithi006 bash[27292]: audit 2023-08-28T00:34:20.432812+0000 mon.c (mon.2) 1809 : audit [DBG] from='client.? 172.21.15.6:0/1141220866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:21.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:21 smithi006 bash[33918]: audit 2023-08-28T00:34:20.432812+0000 mon.c (mon.2) 1809 : audit [DBG] from='client.? 172.21.15.6:0/1141220866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:22.037 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:21 smithi130 bash[51302]: debug 2023-08-28T00:34:21.641+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:21.645734+0000) 2023-08-28T00:34:22.355 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:22 smithi006 bash[27292]: cluster 2023-08-28T00:34:20.771552+0000 mgr.y (mgr.14901) 4673 : cluster [DBG] pgmap v4680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:22.356 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:22 smithi006 bash[33918]: cluster 2023-08-28T00:34:20.771552+0000 mgr.y (mgr.14901) 4673 : cluster [DBG] pgmap v4680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:22.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:22 smithi130 bash[27579]: cluster 2023-08-28T00:34:20.771552+0000 mgr.y (mgr.14901) 4673 : cluster [DBG] pgmap v4680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:34:22.352Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:34:22.354Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:23.045 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:22 smithi130 bash[51302]: debug 2023-08-28T00:34:22.641+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:22.646045+0000) 2023-08-28T00:34:23.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:23 smithi130 bash[27579]: audit 2023-08-28T00:34:22.850398+0000 mon.a (mon.0) 3357 : audit [DBG] from='client.? 172.21.15.6:0/3881724502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:23.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:23 smithi006 bash[27292]: audit 2023-08-28T00:34:22.850398+0000 mon.a (mon.0) 3357 : audit [DBG] from='client.? 172.21.15.6:0/3881724502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:23.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:23 smithi006 bash[33918]: audit 2023-08-28T00:34:22.850398+0000 mon.a (mon.0) 3357 : audit [DBG] from='client.? 172.21.15.6:0/3881724502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:24.049 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:23 smithi130 bash[51302]: debug 2023-08-28T00:34:23.641+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:23.646306+0000) 2023-08-28T00:34:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:24 smithi130 bash[27579]: cluster 2023-08-28T00:34:22.772169+0000 mgr.y (mgr.14901) 4674 : cluster [DBG] pgmap v4681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:24.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:24 smithi006 bash[33918]: cluster 2023-08-28T00:34:22.772169+0000 mgr.y (mgr.14901) 4674 : cluster [DBG] pgmap v4681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:34:24.235Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:34:24.237Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:24.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:24 smithi006 bash[27292]: cluster 2023-08-28T00:34:22.772169+0000 mgr.y (mgr.14901) 4674 : cluster [DBG] pgmap v4681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:24 smithi130 bash[51302]: debug 2023-08-28T00:34:24.641+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:24.646584+0000) 2023-08-28T00:34:26.053 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:25 smithi130 bash[51302]: debug 2023-08-28T00:34:25.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:25.646871+0000) 2023-08-28T00:34:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:26 smithi130 bash[27579]: cluster 2023-08-28T00:34:24.773337+0000 mgr.y (mgr.14901) 4675 : cluster [DBG] pgmap v4682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:26 smithi130 bash[27579]: audit 2023-08-28T00:34:25.268800+0000 mon.c (mon.2) 1810 : audit [DBG] from='client.? 172.21.15.6:0/2307858173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:26 smithi006 bash[27292]: cluster 2023-08-28T00:34:24.773337+0000 mgr.y (mgr.14901) 4675 : cluster [DBG] pgmap v4682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:26.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:26 smithi006 bash[27292]: audit 2023-08-28T00:34:25.268800+0000 mon.c (mon.2) 1810 : audit [DBG] from='client.? 172.21.15.6:0/2307858173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:26 smithi006 bash[33918]: cluster 2023-08-28T00:34:24.773337+0000 mgr.y (mgr.14901) 4675 : cluster [DBG] pgmap v4682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:26 smithi006 bash[33918]: audit 2023-08-28T00:34:25.268800+0000 mon.c (mon.2) 1810 : audit [DBG] from='client.? 172.21.15.6:0/2307858173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:27.056 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:26 smithi130 bash[51302]: debug 2023-08-28T00:34:26.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:26.647126+0000) 2023-08-28T00:34:27.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:27 smithi130 bash[27579]: audit 2023-08-28T00:34:26.237736+0000 mon.b (mon.1) 568 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:34:27.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:27 smithi130 bash[27579]: audit 2023-08-28T00:34:26.513637+0000 mon.a (mon.0) 3358 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:34:27.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:27 smithi130 bash[27579]: audit 2023-08-28T00:34:26.521409+0000 mon.a (mon.0) 3359 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:34:27.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:27 smithi006 bash[33918]: audit 2023-08-28T00:34:26.237736+0000 mon.b (mon.1) 568 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:34:27.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:27 smithi006 bash[33918]: audit 2023-08-28T00:34:26.513637+0000 mon.a (mon.0) 3358 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:34:27.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:27 smithi006 bash[33918]: audit 2023-08-28T00:34:26.521409+0000 mon.a (mon.0) 3359 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:34:27.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:27 smithi006 bash[27292]: audit 2023-08-28T00:34:26.237736+0000 mon.b (mon.1) 568 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:34:27.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:27 smithi006 bash[27292]: audit 2023-08-28T00:34:26.513637+0000 mon.a (mon.0) 3358 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:34:27.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:27 smithi006 bash[27292]: audit 2023-08-28T00:34:26.521409+0000 mon.a (mon.0) 3359 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:34:28.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:27 smithi130 bash[51302]: debug 2023-08-28T00:34:27.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:27.647356+0000) 2023-08-28T00:34:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:28 smithi130 bash[27579]: cluster 2023-08-28T00:34:26.773963+0000 mgr.y (mgr.14901) 4676 : cluster [DBG] pgmap v4683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:28 smithi130 bash[27579]: audit 2023-08-28T00:34:27.679274+0000 mon.c (mon.2) 1811 : audit [DBG] from='client.? 172.21.15.6:0/2932301475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:28 smithi130 bash[51302]: debug 2023-08-28T00:34:28.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:28.647579+0000) 2023-08-28T00:34:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:28 smithi006 bash[33918]: cluster 2023-08-28T00:34:26.773963+0000 mgr.y (mgr.14901) 4676 : cluster [DBG] pgmap v4683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:28.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:28 smithi006 bash[33918]: audit 2023-08-28T00:34:27.679274+0000 mon.c (mon.2) 1811 : audit [DBG] from='client.? 172.21.15.6:0/2932301475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:28.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:28 smithi006 bash[27292]: cluster 2023-08-28T00:34:26.773963+0000 mgr.y (mgr.14901) 4676 : cluster [DBG] pgmap v4683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:28.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:28 smithi006 bash[27292]: audit 2023-08-28T00:34:27.679274+0000 mon.c (mon.2) 1811 : audit [DBG] from='client.? 172.21.15.6:0/2932301475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:30.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:29 smithi130 bash[51302]: debug 2023-08-28T00:34:29.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:29.647816+0000) 2023-08-28T00:34:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:30 smithi130 bash[27579]: cluster 2023-08-28T00:34:28.774533+0000 mgr.y (mgr.14901) 4677 : cluster [DBG] pgmap v4684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:30 smithi130 bash[27579]: audit 2023-08-28T00:34:30.094075+0000 mon.a (mon.0) 3360 : audit [DBG] from='client.? 172.21.15.6:0/4164394556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:30.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:30 smithi130 bash[51302]: debug 2023-08-28T00:34:30.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:30.648113+0000) 2023-08-28T00:34:30.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:34:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:34:30] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:34:30.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:30 smithi006 bash[27292]: cluster 2023-08-28T00:34:28.774533+0000 mgr.y (mgr.14901) 4677 : cluster [DBG] pgmap v4684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:30.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:30 smithi006 bash[27292]: audit 2023-08-28T00:34:30.094075+0000 mon.a (mon.0) 3360 : audit [DBG] from='client.? 172.21.15.6:0/4164394556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:30.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:30 smithi006 bash[33918]: cluster 2023-08-28T00:34:28.774533+0000 mgr.y (mgr.14901) 4677 : cluster [DBG] pgmap v4684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:30.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:30 smithi006 bash[33918]: audit 2023-08-28T00:34:30.094075+0000 mon.a (mon.0) 3360 : audit [DBG] from='client.? 172.21.15.6:0/4164394556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:30.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:34:30.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:30.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:34:30.777Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:32.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:31 smithi130 bash[51302]: debug 2023-08-28T00:34:31.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:31.648299+0000) 2023-08-28T00:34:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:32 smithi006 bash[27292]: cluster 2023-08-28T00:34:30.775695+0000 mgr.y (mgr.14901) 4678 : cluster [DBG] pgmap v4685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:32 smithi006 bash[27292]: audit 2023-08-28T00:34:32.427357+0000 mon.a (mon.0) 3361 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:34:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:32 smithi006 bash[27292]: audit 2023-08-28T00:34:32.519667+0000 mon.c (mon.2) 1812 : audit [DBG] from='client.? 172.21.15.6:0/3843738974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:32 smithi006 bash[33918]: cluster 2023-08-28T00:34:30.775695+0000 mgr.y (mgr.14901) 4678 : cluster [DBG] pgmap v4685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:32 smithi006 bash[33918]: audit 2023-08-28T00:34:32.427357+0000 mon.a (mon.0) 3361 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:34:32.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:32 smithi006 bash[33918]: audit 2023-08-28T00:34:32.519667+0000 mon.c (mon.2) 1812 : audit [DBG] from='client.? 172.21.15.6:0/3843738974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:32.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:34:32.352Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:32.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:34:32.354Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:32 smithi130 bash[27579]: cluster 2023-08-28T00:34:30.775695+0000 mgr.y (mgr.14901) 4678 : cluster [DBG] pgmap v4685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:32 smithi130 bash[27579]: audit 2023-08-28T00:34:32.427357+0000 mon.a (mon.0) 3361 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:34:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:32 smithi130 bash[27579]: audit 2023-08-28T00:34:32.519667+0000 mon.c (mon.2) 1812 : audit [DBG] from='client.? 172.21.15.6:0/3843738974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:32.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:32 smithi130 bash[51302]: debug 2023-08-28T00:34:32.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:32.648501+0000) 2023-08-28T00:34:34.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:33 smithi130 bash[51302]: debug 2023-08-28T00:34:33.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:33.648749+0000) 2023-08-28T00:34:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:33 smithi130 bash[27579]: cluster 2023-08-28T00:34:32.776504+0000 mgr.y (mgr.14901) 4679 : cluster [DBG] pgmap v4686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:34.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:33 smithi130 bash[27579]: audit 2023-08-28T00:34:32.860288+0000 mon.a (mon.0) 3362 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:34:34.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:33 smithi130 bash[27579]: audit 2023-08-28T00:34:33.131012+0000 mon.b (mon.1) 569 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:34:34.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:33 smithi130 bash[27579]: audit 2023-08-28T00:34:33.132984+0000 mon.b (mon.1) 570 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:34:34.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:33 smithi130 bash[27579]: audit 2023-08-28T00:34:33.141342+0000 mon.a (mon.0) 3363 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:34:34.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:33 smithi006 bash[33918]: cluster 2023-08-28T00:34:32.776504+0000 mgr.y (mgr.14901) 4679 : cluster [DBG] pgmap v4686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:34.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:33 smithi006 bash[33918]: audit 2023-08-28T00:34:32.860288+0000 mon.a (mon.0) 3362 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:34:34.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:33 smithi006 bash[33918]: audit 2023-08-28T00:34:33.131012+0000 mon.b (mon.1) 569 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:34:34.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:33 smithi006 bash[33918]: audit 2023-08-28T00:34:33.132984+0000 mon.b (mon.1) 570 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:34:34.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:33 smithi006 bash[33918]: audit 2023-08-28T00:34:33.141342+0000 mon.a (mon.0) 3363 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:34:34.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:33 smithi006 bash[27292]: cluster 2023-08-28T00:34:32.776504+0000 mgr.y (mgr.14901) 4679 : cluster [DBG] pgmap v4686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:34.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:33 smithi006 bash[27292]: audit 2023-08-28T00:34:32.860288+0000 mon.a (mon.0) 3362 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:34:34.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:33 smithi006 bash[27292]: audit 2023-08-28T00:34:33.131012+0000 mon.b (mon.1) 569 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:34:34.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:33 smithi006 bash[27292]: audit 2023-08-28T00:34:33.132984+0000 mon.b (mon.1) 570 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:34:34.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:33 smithi006 bash[27292]: audit 2023-08-28T00:34:33.141342+0000 mon.a (mon.0) 3363 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:34:34.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:34:34.235Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:34:34.237Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:35.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:34 smithi130 bash[51302]: debug 2023-08-28T00:34:34.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:34.648997+0000) 2023-08-28T00:34:36.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:35 smithi130 bash[27579]: cluster 2023-08-28T00:34:34.777684+0000 mgr.y (mgr.14901) 4680 : cluster [DBG] pgmap v4687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:36.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:35 smithi130 bash[27579]: audit 2023-08-28T00:34:34.944445+0000 mon.c (mon.2) 1813 : audit [DBG] from='client.? 172.21.15.6:0/1712471020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:36.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:35 smithi130 bash[51302]: debug 2023-08-28T00:34:35.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:35.649242+0000) 2023-08-28T00:34:36.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:35 smithi006 bash[33918]: cluster 2023-08-28T00:34:34.777684+0000 mgr.y (mgr.14901) 4680 : cluster [DBG] pgmap v4687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:36.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:35 smithi006 bash[33918]: audit 2023-08-28T00:34:34.944445+0000 mon.c (mon.2) 1813 : audit [DBG] from='client.? 172.21.15.6:0/1712471020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:36.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:35 smithi006 bash[27292]: cluster 2023-08-28T00:34:34.777684+0000 mgr.y (mgr.14901) 4680 : cluster [DBG] pgmap v4687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:36.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:35 smithi006 bash[27292]: audit 2023-08-28T00:34:34.944445+0000 mon.c (mon.2) 1813 : audit [DBG] from='client.? 172.21.15.6:0/1712471020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:37.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:36 smithi130 bash[51302]: debug 2023-08-28T00:34:36.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:36.649522+0000) 2023-08-28T00:34:37.938 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:37 smithi130 bash[51302]: debug 2023-08-28T00:34:37.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:37.649772+0000) 2023-08-28T00:34:38.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:37 smithi006 bash[27292]: cluster 2023-08-28T00:34:36.778349+0000 mgr.y (mgr.14901) 4681 : cluster [DBG] pgmap v4688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:38.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:37 smithi006 bash[27292]: audit 2023-08-28T00:34:37.357324+0000 mon.c (mon.2) 1814 : audit [DBG] from='client.? 172.21.15.6:0/1735129630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:38.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:37 smithi006 bash[33918]: cluster 2023-08-28T00:34:36.778349+0000 mgr.y (mgr.14901) 4681 : cluster [DBG] pgmap v4688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:38.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:37 smithi006 bash[33918]: audit 2023-08-28T00:34:37.357324+0000 mon.c (mon.2) 1814 : audit [DBG] from='client.? 172.21.15.6:0/1735129630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:38.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:37 smithi130 bash[27579]: cluster 2023-08-28T00:34:36.778349+0000 mgr.y (mgr.14901) 4681 : cluster [DBG] pgmap v4688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:38.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:37 smithi130 bash[27579]: audit 2023-08-28T00:34:37.357324+0000 mon.c (mon.2) 1814 : audit [DBG] from='client.? 172.21.15.6:0/1735129630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:39.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:38 smithi130 bash[51302]: debug 2023-08-28T00:34:38.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:38.650083+0000) 2023-08-28T00:34:39.941 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:39 smithi130 bash[51302]: debug 2023-08-28T00:34:39.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:39.650373+0000) 2023-08-28T00:34:40.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:39 smithi006 bash[33918]: cluster 2023-08-28T00:34:38.779000+0000 mgr.y (mgr.14901) 4682 : cluster [DBG] pgmap v4689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:40.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:39 smithi006 bash[33918]: audit 2023-08-28T00:34:39.769968+0000 mon.a (mon.0) 3364 : audit [DBG] from='client.? 172.21.15.6:0/3891876877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:40.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:39 smithi006 bash[27292]: cluster 2023-08-28T00:34:38.779000+0000 mgr.y (mgr.14901) 4682 : cluster [DBG] pgmap v4689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:40.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:39 smithi006 bash[27292]: audit 2023-08-28T00:34:39.769968+0000 mon.a (mon.0) 3364 : audit [DBG] from='client.? 172.21.15.6:0/3891876877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:39 smithi130 bash[27579]: cluster 2023-08-28T00:34:38.779000+0000 mgr.y (mgr.14901) 4682 : cluster [DBG] pgmap v4689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:39 smithi130 bash[27579]: audit 2023-08-28T00:34:39.769968+0000 mon.a (mon.0) 3364 : audit [DBG] from='client.? 172.21.15.6:0/3891876877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:41.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:40 smithi130 bash[51302]: debug 2023-08-28T00:34:40.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:40.650585+0000) 2023-08-28T00:34:41.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:34:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:34:40] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:34:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:34:40.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:34:40.777Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:41.945 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:41 smithi130 bash[51302]: debug 2023-08-28T00:34:41.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:41.650849+0000) 2023-08-28T00:34:42.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:41 smithi006 bash[27292]: cluster 2023-08-28T00:34:40.780129+0000 mgr.y (mgr.14901) 4683 : cluster [DBG] pgmap v4690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:42.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:41 smithi006 bash[33918]: cluster 2023-08-28T00:34:40.780129+0000 mgr.y (mgr.14901) 4683 : cluster [DBG] pgmap v4690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:41 smithi130 bash[27579]: cluster 2023-08-28T00:34:40.780129+0000 mgr.y (mgr.14901) 4683 : cluster [DBG] pgmap v4690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:34:42.354Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:34:42.356Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:42.955 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:42 smithi130 bash[51302]: debug 2023-08-28T00:34:42.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:42.651101+0000) 2023-08-28T00:34:43.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:42 smithi006 bash[27292]: audit 2023-08-28T00:34:42.188574+0000 mon.c (mon.2) 1815 : audit [DBG] from='client.? 172.21.15.6:0/4105092605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:43.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:42 smithi006 bash[33918]: audit 2023-08-28T00:34:42.188574+0000 mon.c (mon.2) 1815 : audit [DBG] from='client.? 172.21.15.6:0/4105092605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:43.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:42 smithi130 bash[27579]: audit 2023-08-28T00:34:42.188574+0000 mon.c (mon.2) 1815 : audit [DBG] from='client.? 172.21.15.6:0/4105092605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:43.958 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:43 smithi130 bash[51302]: debug 2023-08-28T00:34:43.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:43.651337+0000) 2023-08-28T00:34:44.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:43 smithi006 bash[27292]: cluster 2023-08-28T00:34:42.780705+0000 mgr.y (mgr.14901) 4684 : cluster [DBG] pgmap v4691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:44.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:43 smithi006 bash[33918]: cluster 2023-08-28T00:34:42.780705+0000 mgr.y (mgr.14901) 4684 : cluster [DBG] pgmap v4691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:43 smithi130 bash[27579]: cluster 2023-08-28T00:34:42.780705+0000 mgr.y (mgr.14901) 4684 : cluster [DBG] pgmap v4691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:34:44.236Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:34:44.238Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:44.967 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:44 smithi130 bash[51302]: debug 2023-08-28T00:34:44.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:44.651588+0000) 2023-08-28T00:34:45.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:44 smithi006 bash[33918]: audit 2023-08-28T00:34:44.608207+0000 mon.c (mon.2) 1816 : audit [DBG] from='client.? 172.21.15.6:0/2054956457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:45.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:44 smithi006 bash[27292]: audit 2023-08-28T00:34:44.608207+0000 mon.c (mon.2) 1816 : audit [DBG] from='client.? 172.21.15.6:0/2054956457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:44 smithi130 bash[27579]: audit 2023-08-28T00:34:44.608207+0000 mon.c (mon.2) 1816 : audit [DBG] from='client.? 172.21.15.6:0/2054956457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:45.972 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:45 smithi130 bash[51302]: debug 2023-08-28T00:34:45.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:45.651838+0000) 2023-08-28T00:34:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:45 smithi130 bash[27579]: cluster 2023-08-28T00:34:44.781782+0000 mgr.y (mgr.14901) 4685 : cluster [DBG] pgmap v4692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:46.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:45 smithi006 bash[33918]: cluster 2023-08-28T00:34:44.781782+0000 mgr.y (mgr.14901) 4685 : cluster [DBG] pgmap v4692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:46.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:45 smithi006 bash[27292]: cluster 2023-08-28T00:34:44.781782+0000 mgr.y (mgr.14901) 4685 : cluster [DBG] pgmap v4692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:47.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:46 smithi130 bash[51302]: debug 2023-08-28T00:34:46.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:46.652118+0000) 2023-08-28T00:34:47.977 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:47 smithi130 bash[51302]: debug 2023-08-28T00:34:47.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:47.652374+0000) 2023-08-28T00:34:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:47 smithi130 bash[27579]: cluster 2023-08-28T00:34:46.782460+0000 mgr.y (mgr.14901) 4686 : cluster [DBG] pgmap v4693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:47 smithi130 bash[27579]: audit 2023-08-28T00:34:47.031876+0000 mon.a (mon.0) 3365 : audit [DBG] from='client.? 172.21.15.6:0/3459885679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:48.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:47 smithi006 bash[27292]: cluster 2023-08-28T00:34:46.782460+0000 mgr.y (mgr.14901) 4686 : cluster [DBG] pgmap v4693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:48.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:47 smithi006 bash[27292]: audit 2023-08-28T00:34:47.031876+0000 mon.a (mon.0) 3365 : audit [DBG] from='client.? 172.21.15.6:0/3459885679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:48.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:47 smithi006 bash[33918]: cluster 2023-08-28T00:34:46.782460+0000 mgr.y (mgr.14901) 4686 : cluster [DBG] pgmap v4693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:48.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:47 smithi006 bash[33918]: audit 2023-08-28T00:34:47.031876+0000 mon.a (mon.0) 3365 : audit [DBG] from='client.? 172.21.15.6:0/3459885679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:49.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:48 smithi130 bash[51302]: debug 2023-08-28T00:34:48.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:48.652642+0000) 2023-08-28T00:34:49.982 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:49 smithi130 bash[51302]: debug 2023-08-28T00:34:49.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:49.652902+0000) 2023-08-28T00:34:50.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:49 smithi130 bash[27579]: cluster 2023-08-28T00:34:48.783295+0000 mgr.y (mgr.14901) 4687 : cluster [DBG] pgmap v4694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:50.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:49 smithi130 bash[27579]: audit 2023-08-28T00:34:49.440783+0000 mon.c (mon.2) 1817 : audit [DBG] from='client.? 172.21.15.6:0/708970719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:50.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:49 smithi006 bash[27292]: cluster 2023-08-28T00:34:48.783295+0000 mgr.y (mgr.14901) 4687 : cluster [DBG] pgmap v4694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:50.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:49 smithi006 bash[27292]: audit 2023-08-28T00:34:49.440783+0000 mon.c (mon.2) 1817 : audit [DBG] from='client.? 172.21.15.6:0/708970719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:50.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:49 smithi006 bash[33918]: cluster 2023-08-28T00:34:48.783295+0000 mgr.y (mgr.14901) 4687 : cluster [DBG] pgmap v4694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:50.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:49 smithi006 bash[33918]: audit 2023-08-28T00:34:49.440783+0000 mon.c (mon.2) 1817 : audit [DBG] from='client.? 172.21.15.6:0/708970719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:51.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:50 smithi130 bash[51302]: debug 2023-08-28T00:34:50.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:50.653163+0000) 2023-08-28T00:34:51.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:34:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:34:50] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:34:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:34:50.776Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:34:50.778Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:51.986 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:51 smithi130 bash[51302]: debug 2023-08-28T00:34:51.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:51.653403+0000) 2023-08-28T00:34:52.358 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:51 smithi006 bash[27292]: cluster 2023-08-28T00:34:50.784597+0000 mgr.y (mgr.14901) 4688 : cluster [DBG] pgmap v4695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:52.358 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:51 smithi006 bash[27292]: audit 2023-08-28T00:34:51.850983+0000 mon.c (mon.2) 1818 : audit [DBG] from='client.? 172.21.15.6:0/1542957119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:52.359 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:51 smithi006 bash[33918]: cluster 2023-08-28T00:34:50.784597+0000 mgr.y (mgr.14901) 4688 : cluster [DBG] pgmap v4695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:52.359 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:51 smithi006 bash[33918]: audit 2023-08-28T00:34:51.850983+0000 mon.c (mon.2) 1818 : audit [DBG] from='client.? 172.21.15.6:0/1542957119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:51 smithi130 bash[27579]: cluster 2023-08-28T00:34:50.784597+0000 mgr.y (mgr.14901) 4688 : cluster [DBG] pgmap v4695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:51 smithi130 bash[27579]: audit 2023-08-28T00:34:51.850983+0000 mon.c (mon.2) 1818 : audit [DBG] from='client.? 172.21.15.6:0/1542957119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:34:52.354Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:34:52.356Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:53.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:52 smithi130 bash[51302]: debug 2023-08-28T00:34:52.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:52.653686+0000) 2023-08-28T00:34:53.992 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:53 smithi130 bash[51302]: debug 2023-08-28T00:34:53.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:53.653952+0000) 2023-08-28T00:34:54.241 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:34:54.236Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:54.242 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:34:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:34:54.238Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:34:54.242 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:53 smithi006 bash[33918]: cluster 2023-08-28T00:34:52.785321+0000 mgr.y (mgr.14901) 4689 : cluster [DBG] pgmap v4696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:54.242 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:53 smithi006 bash[27292]: cluster 2023-08-28T00:34:52.785321+0000 mgr.y (mgr.14901) 4689 : cluster [DBG] pgmap v4696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:53 smithi130 bash[27579]: cluster 2023-08-28T00:34:52.785321+0000 mgr.y (mgr.14901) 4689 : cluster [DBG] pgmap v4696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:55.000 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:54 smithi130 bash[51302]: debug 2023-08-28T00:34:54.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:54.654229+0000) 2023-08-28T00:34:55.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:54 smithi130 bash[27579]: audit 2023-08-28T00:34:54.288704+0000 mon.b (mon.1) 571 : audit [DBG] from='client.? 172.21.15.6:0/4047307628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:55.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:54 smithi006 bash[27292]: audit 2023-08-28T00:34:54.288704+0000 mon.b (mon.1) 571 : audit [DBG] from='client.? 172.21.15.6:0/4047307628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:55.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:54 smithi006 bash[33918]: audit 2023-08-28T00:34:54.288704+0000 mon.b (mon.1) 571 : audit [DBG] from='client.? 172.21.15.6:0/4047307628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:56.002 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:55 smithi130 bash[51302]: debug 2023-08-28T00:34:55.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:55.654484+0000) 2023-08-28T00:34:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:56 smithi130 bash[27579]: cluster 2023-08-28T00:34:54.786449+0000 mgr.y (mgr.14901) 4690 : cluster [DBG] pgmap v4697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:56.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:55 smithi006 bash[27292]: cluster 2023-08-28T00:34:54.786449+0000 mgr.y (mgr.14901) 4690 : cluster [DBG] pgmap v4697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:56.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:56 smithi006 bash[33918]: cluster 2023-08-28T00:34:54.786449+0000 mgr.y (mgr.14901) 4690 : cluster [DBG] pgmap v4697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:57.010 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:56 smithi130 bash[51302]: debug 2023-08-28T00:34:56.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:56.654718+0000) 2023-08-28T00:34:57.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:57 smithi130 bash[27579]: audit 2023-08-28T00:34:56.699424+0000 mon.c (mon.2) 1819 : audit [DBG] from='client.? 172.21.15.6:0/1218094982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:57.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:57 smithi006 bash[33918]: audit 2023-08-28T00:34:56.699424+0000 mon.c (mon.2) 1819 : audit [DBG] from='client.? 172.21.15.6:0/1218094982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:57.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:57 smithi006 bash[27292]: audit 2023-08-28T00:34:56.699424+0000 mon.c (mon.2) 1819 : audit [DBG] from='client.? 172.21.15.6:0/1218094982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:58.013 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:57 smithi130 bash[51302]: debug 2023-08-28T00:34:57.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:57.654987+0000) 2023-08-28T00:34:58.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:34:58 smithi130 bash[27579]: cluster 2023-08-28T00:34:56.787100+0000 mgr.y (mgr.14901) 4691 : cluster [DBG] pgmap v4698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:58.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:34:58 smithi006 bash[33918]: cluster 2023-08-28T00:34:56.787100+0000 mgr.y (mgr.14901) 4691 : cluster [DBG] pgmap v4698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:58.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:34:58 smithi006 bash[27292]: cluster 2023-08-28T00:34:56.787100+0000 mgr.y (mgr.14901) 4691 : cluster [DBG] pgmap v4698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:34:59.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:58 smithi130 bash[51302]: debug 2023-08-28T00:34:58.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:58.655211+0000) 2023-08-28T00:35:00.026 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:34:59 smithi130 bash[51302]: debug 2023-08-28T00:34:59.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:59.655530+0000) 2023-08-28T00:35:00.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:00 smithi130 bash[27579]: cluster 2023-08-28T00:34:58.787760+0000 mgr.y (mgr.14901) 4692 : cluster [DBG] pgmap v4699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:00.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:00 smithi130 bash[27579]: audit 2023-08-28T00:34:59.114452+0000 mon.c (mon.2) 1820 : audit [DBG] from='client.? 172.21.15.6:0/2402519688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:00.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:00 smithi006 bash[27292]: cluster 2023-08-28T00:34:58.787760+0000 mgr.y (mgr.14901) 4692 : cluster [DBG] pgmap v4699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:00.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:00 smithi006 bash[27292]: audit 2023-08-28T00:34:59.114452+0000 mon.c (mon.2) 1820 : audit [DBG] from='client.? 172.21.15.6:0/2402519688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:00.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:00 smithi006 bash[33918]: cluster 2023-08-28T00:34:58.787760+0000 mgr.y (mgr.14901) 4692 : cluster [DBG] pgmap v4699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:00.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:00 smithi006 bash[33918]: audit 2023-08-28T00:34:59.114452+0000 mon.c (mon.2) 1820 : audit [DBG] from='client.? 172.21.15.6:0/2402519688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:01.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:00 smithi130 bash[51302]: debug 2023-08-28T00:35:00.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:00.655862+0000) 2023-08-28T00:35:01.161 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:35:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:35:00] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:35:01.161 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:35:00.777Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:01.161 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:35:00.778Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:01.940 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:01 smithi130 bash[51302]: debug 2023-08-28T00:35:01.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:01.656143+0000) 2023-08-28T00:35:02.359 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:02 smithi006 bash[33918]: cluster 2023-08-28T00:35:00.788759+0000 mgr.y (mgr.14901) 4693 : cluster [DBG] pgmap v4700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:02.359 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:02 smithi006 bash[33918]: audit 2023-08-28T00:35:01.531084+0000 mon.a (mon.0) 3366 : audit [DBG] from='client.? 172.21.15.6:0/2236290505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:02.360 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:02 smithi006 bash[27292]: cluster 2023-08-28T00:35:00.788759+0000 mgr.y (mgr.14901) 4693 : cluster [DBG] pgmap v4700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:02.360 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:02 smithi006 bash[27292]: audit 2023-08-28T00:35:01.531084+0000 mon.a (mon.0) 3366 : audit [DBG] from='client.? 172.21.15.6:0/2236290505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:02.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:02 smithi130 bash[27579]: cluster 2023-08-28T00:35:00.788759+0000 mgr.y (mgr.14901) 4693 : cluster [DBG] pgmap v4700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:02.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:02 smithi130 bash[27579]: audit 2023-08-28T00:35:01.531084+0000 mon.a (mon.0) 3366 : audit [DBG] from='client.? 172.21.15.6:0/2236290505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:35:02.355Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:35:02.358Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:02 smithi130 bash[51302]: debug 2023-08-28T00:35:02.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:02.656394+0000) 2023-08-28T00:35:04.037 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:03 smithi130 bash[51302]: debug 2023-08-28T00:35:03.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:03.656673+0000) 2023-08-28T00:35:04.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:04 smithi130 bash[27579]: cluster 2023-08-28T00:35:02.789406+0000 mgr.y (mgr.14901) 4694 : cluster [DBG] pgmap v4701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:04.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:04 smithi130 bash[27579]: audit 2023-08-28T00:35:03.944544+0000 mon.c (mon.2) 1821 : audit [DBG] from='client.? 172.21.15.6:0/2688156126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:04.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:04 smithi006 bash[27292]: cluster 2023-08-28T00:35:02.789406+0000 mgr.y (mgr.14901) 4694 : cluster [DBG] pgmap v4701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:04.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:04 smithi006 bash[27292]: audit 2023-08-28T00:35:03.944544+0000 mon.c (mon.2) 1821 : audit [DBG] from='client.? 172.21.15.6:0/2688156126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:35:04.237Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:35:04.239Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:04.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:04 smithi006 bash[33918]: cluster 2023-08-28T00:35:02.789406+0000 mgr.y (mgr.14901) 4694 : cluster [DBG] pgmap v4701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:04.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:04 smithi006 bash[33918]: audit 2023-08-28T00:35:03.944544+0000 mon.c (mon.2) 1821 : audit [DBG] from='client.? 172.21.15.6:0/2688156126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:04 smithi130 bash[51302]: debug 2023-08-28T00:35:04.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:04.656951+0000) 2023-08-28T00:35:06.043 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:05 smithi130 bash[51302]: debug 2023-08-28T00:35:05.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:05.657212+0000) 2023-08-28T00:35:06.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:06 smithi130 bash[27579]: cluster 2023-08-28T00:35:04.790505+0000 mgr.y (mgr.14901) 4695 : cluster [DBG] pgmap v4702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:06.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:06 smithi006 bash[33918]: cluster 2023-08-28T00:35:04.790505+0000 mgr.y (mgr.14901) 4695 : cluster [DBG] pgmap v4702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:06.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:06 smithi006 bash[27292]: cluster 2023-08-28T00:35:04.790505+0000 mgr.y (mgr.14901) 4695 : cluster [DBG] pgmap v4702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:07.050 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:06 smithi130 bash[51302]: debug 2023-08-28T00:35:06.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:06.657470+0000) 2023-08-28T00:35:07.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:07 smithi130 bash[27579]: audit 2023-08-28T00:35:06.353723+0000 mon.a (mon.0) 3367 : audit [DBG] from='client.? 172.21.15.6:0/2019607807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:07.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:07 smithi006 bash[27292]: audit 2023-08-28T00:35:06.353723+0000 mon.a (mon.0) 3367 : audit [DBG] from='client.? 172.21.15.6:0/2019607807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:07.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:07 smithi006 bash[33918]: audit 2023-08-28T00:35:06.353723+0000 mon.a (mon.0) 3367 : audit [DBG] from='client.? 172.21.15.6:0/2019607807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:08.053 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:07 smithi130 bash[51302]: debug 2023-08-28T00:35:07.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:07.657732+0000) 2023-08-28T00:35:08.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:08 smithi130 bash[27579]: cluster 2023-08-28T00:35:06.791168+0000 mgr.y (mgr.14901) 4696 : cluster [DBG] pgmap v4703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:08.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:08 smithi006 bash[27292]: cluster 2023-08-28T00:35:06.791168+0000 mgr.y (mgr.14901) 4696 : cluster [DBG] pgmap v4703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:08.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:08 smithi006 bash[33918]: cluster 2023-08-28T00:35:06.791168+0000 mgr.y (mgr.14901) 4696 : cluster [DBG] pgmap v4703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:09.062 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:08 smithi130 bash[51302]: debug 2023-08-28T00:35:08.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:08.658031+0000) 2023-08-28T00:35:09.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:09 smithi130 bash[27579]: audit 2023-08-28T00:35:08.765396+0000 mon.c (mon.2) 1822 : audit [DBG] from='client.? 172.21.15.6:0/368866752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:09.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:09 smithi006 bash[27292]: audit 2023-08-28T00:35:08.765396+0000 mon.c (mon.2) 1822 : audit [DBG] from='client.? 172.21.15.6:0/368866752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:09.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:09 smithi006 bash[33918]: audit 2023-08-28T00:35:08.765396+0000 mon.c (mon.2) 1822 : audit [DBG] from='client.? 172.21.15.6:0/368866752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:10.065 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:09 smithi130 bash[51302]: debug 2023-08-28T00:35:09.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:09.658319+0000) 2023-08-28T00:35:10.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:10 smithi130 bash[27579]: cluster 2023-08-28T00:35:08.791807+0000 mgr.y (mgr.14901) 4697 : cluster [DBG] pgmap v4704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:10.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:10 smithi006 bash[27292]: cluster 2023-08-28T00:35:08.791807+0000 mgr.y (mgr.14901) 4697 : cluster [DBG] pgmap v4704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:10.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:10 smithi006 bash[33918]: cluster 2023-08-28T00:35:08.791807+0000 mgr.y (mgr.14901) 4697 : cluster [DBG] pgmap v4704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:11.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:10 smithi130 bash[51302]: debug 2023-08-28T00:35:10.652+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:10.658582+0000) 2023-08-28T00:35:11.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:35:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:35:10] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:35:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:35:10.777Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:35:10.779Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:12.070 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:11 smithi130 bash[51302]: debug 2023-08-28T00:35:11.656+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:11.658848+0000) 2023-08-28T00:35:12.360 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:12 smithi006 bash[27292]: cluster 2023-08-28T00:35:10.792906+0000 mgr.y (mgr.14901) 4698 : cluster [DBG] pgmap v4705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:12.360 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:12 smithi006 bash[27292]: audit 2023-08-28T00:35:11.185576+0000 mon.c (mon.2) 1823 : audit [DBG] from='client.? 172.21.15.6:0/2207675520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:12.360 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:12 smithi006 bash[33918]: cluster 2023-08-28T00:35:10.792906+0000 mgr.y (mgr.14901) 4698 : cluster [DBG] pgmap v4705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:12.361 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:12 smithi006 bash[33918]: audit 2023-08-28T00:35:11.185576+0000 mon.c (mon.2) 1823 : audit [DBG] from='client.? 172.21.15.6:0/2207675520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:12.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:12 smithi130 bash[27579]: cluster 2023-08-28T00:35:10.792906+0000 mgr.y (mgr.14901) 4698 : cluster [DBG] pgmap v4705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:12.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:12 smithi130 bash[27579]: audit 2023-08-28T00:35:11.185576+0000 mon.c (mon.2) 1823 : audit [DBG] from='client.? 172.21.15.6:0/2207675520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:35:12.357Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:35:12.358Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:13.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:12 smithi130 bash[51302]: debug 2023-08-28T00:35:12.656+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:12.659071+0000) 2023-08-28T00:35:14.075 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:13 smithi130 bash[51302]: debug 2023-08-28T00:35:13.656+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:13.659298+0000) 2023-08-28T00:35:14.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:14 smithi130 bash[27579]: cluster 2023-08-28T00:35:12.793529+0000 mgr.y (mgr.14901) 4699 : cluster [DBG] pgmap v4706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:14.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:14 smithi130 bash[27579]: audit 2023-08-28T00:35:13.600237+0000 mon.c (mon.2) 1824 : audit [DBG] from='client.? 172.21.15.6:0/373291295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:14.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:14 smithi006 bash[33918]: cluster 2023-08-28T00:35:12.793529+0000 mgr.y (mgr.14901) 4699 : cluster [DBG] pgmap v4706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:14.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:14 smithi006 bash[33918]: audit 2023-08-28T00:35:13.600237+0000 mon.c (mon.2) 1824 : audit [DBG] from='client.? 172.21.15.6:0/373291295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:35:14.238Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:35:14.241Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:14.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:14 smithi006 bash[27292]: cluster 2023-08-28T00:35:12.793529+0000 mgr.y (mgr.14901) 4699 : cluster [DBG] pgmap v4706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:14.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:14 smithi006 bash[27292]: audit 2023-08-28T00:35:13.600237+0000 mon.c (mon.2) 1824 : audit [DBG] from='client.? 172.21.15.6:0/373291295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:15.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:14 smithi130 bash[51302]: debug 2023-08-28T00:35:14.656+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:14.659507+0000) 2023-08-28T00:35:16.082 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:15 smithi130 bash[51302]: debug 2023-08-28T00:35:15.656+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:15.659748+0000) 2023-08-28T00:35:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:16 smithi130 bash[27579]: cluster 2023-08-28T00:35:14.794661+0000 mgr.y (mgr.14901) 4700 : cluster [DBG] pgmap v4707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:16.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:16 smithi130 bash[27579]: audit 2023-08-28T00:35:16.015141+0000 mon.c (mon.2) 1825 : audit [DBG] from='client.? 172.21.15.6:0/2853086659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:16.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:16 smithi006 bash[27292]: cluster 2023-08-28T00:35:14.794661+0000 mgr.y (mgr.14901) 4700 : cluster [DBG] pgmap v4707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:16.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:16 smithi006 bash[27292]: audit 2023-08-28T00:35:16.015141+0000 mon.c (mon.2) 1825 : audit [DBG] from='client.? 172.21.15.6:0/2853086659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:16.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:16 smithi006 bash[33918]: cluster 2023-08-28T00:35:14.794661+0000 mgr.y (mgr.14901) 4700 : cluster [DBG] pgmap v4707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:16.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:16 smithi006 bash[33918]: audit 2023-08-28T00:35:16.015141+0000 mon.c (mon.2) 1825 : audit [DBG] from='client.? 172.21.15.6:0/2853086659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:17.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:16 smithi130 bash[51302]: debug 2023-08-28T00:35:16.656+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:16.659988+0000) 2023-08-28T00:35:18.084 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:17 smithi130 bash[51302]: debug 2023-08-28T00:35:17.656+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:17.660256+0000) 2023-08-28T00:35:18.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:18 smithi130 bash[27579]: cluster 2023-08-28T00:35:16.795345+0000 mgr.y (mgr.14901) 4701 : cluster [DBG] pgmap v4708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:18.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:18 smithi006 bash[27292]: cluster 2023-08-28T00:35:16.795345+0000 mgr.y (mgr.14901) 4701 : cluster [DBG] pgmap v4708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:18.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:18 smithi006 bash[33918]: cluster 2023-08-28T00:35:16.795345+0000 mgr.y (mgr.14901) 4701 : cluster [DBG] pgmap v4708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:19.093 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:18 smithi130 bash[51302]: debug 2023-08-28T00:35:18.656+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:18.660489+0000) 2023-08-28T00:35:19.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:19 smithi130 bash[27579]: audit 2023-08-28T00:35:18.438672+0000 mon.a (mon.0) 3368 : audit [DBG] from='client.? 172.21.15.6:0/2733467550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:19.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:19 smithi006 bash[27292]: audit 2023-08-28T00:35:18.438672+0000 mon.a (mon.0) 3368 : audit [DBG] from='client.? 172.21.15.6:0/2733467550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:19.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:19 smithi006 bash[33918]: audit 2023-08-28T00:35:18.438672+0000 mon.a (mon.0) 3368 : audit [DBG] from='client.? 172.21.15.6:0/2733467550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:20.095 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:19 smithi130 bash[51302]: debug 2023-08-28T00:35:19.656+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:19.660797+0000) 2023-08-28T00:35:20.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:20 smithi130 bash[27579]: cluster 2023-08-28T00:35:18.795952+0000 mgr.y (mgr.14901) 4702 : cluster [DBG] pgmap v4709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:20.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:20 smithi006 bash[27292]: cluster 2023-08-28T00:35:18.795952+0000 mgr.y (mgr.14901) 4702 : cluster [DBG] pgmap v4709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:20.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:20 smithi006 bash[33918]: cluster 2023-08-28T00:35:18.795952+0000 mgr.y (mgr.14901) 4702 : cluster [DBG] pgmap v4709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:21.097 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:35:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:35:20] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:35:21.097 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:35:20.777Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:21.097 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:35:20.779Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:21.098 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:20 smithi130 bash[51302]: debug 2023-08-28T00:35:20.656+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:20.661067+0000) 2023-08-28T00:35:21.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:21 smithi130 bash[27579]: audit 2023-08-28T00:35:20.852778+0000 mon.a (mon.0) 3369 : audit [DBG] from='client.? 172.21.15.6:0/4279183413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:21.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:21 smithi006 bash[27292]: audit 2023-08-28T00:35:20.852778+0000 mon.a (mon.0) 3369 : audit [DBG] from='client.? 172.21.15.6:0/4279183413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:21.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:21 smithi006 bash[33918]: audit 2023-08-28T00:35:20.852778+0000 mon.a (mon.0) 3369 : audit [DBG] from='client.? 172.21.15.6:0/4279183413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:22.108 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:21 smithi130 bash[51302]: debug 2023-08-28T00:35:21.656+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:21.661362+0000) 2023-08-28T00:35:22.361 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:22 smithi006 bash[33918]: cluster 2023-08-28T00:35:20.797029+0000 mgr.y (mgr.14901) 4703 : cluster [DBG] pgmap v4710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:22.415 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:22 smithi006 bash[27292]: cluster 2023-08-28T00:35:20.797029+0000 mgr.y (mgr.14901) 4703 : cluster [DBG] pgmap v4710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:22.415 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:22 smithi130 bash[27579]: cluster 2023-08-28T00:35:20.797029+0000 mgr.y (mgr.14901) 4703 : cluster [DBG] pgmap v4710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:35:22.358Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:35:22.360Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:23.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:22 smithi130 bash[51302]: debug 2023-08-28T00:35:22.656+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:22.661712+0000) 2023-08-28T00:35:24.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:23 smithi130 bash[51302]: debug 2023-08-28T00:35:23.656+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:23.662022+0000) 2023-08-28T00:35:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:24 smithi130 bash[27579]: cluster 2023-08-28T00:35:22.797708+0000 mgr.y (mgr.14901) 4704 : cluster [DBG] pgmap v4711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:24 smithi130 bash[27579]: audit 2023-08-28T00:35:23.269310+0000 mon.a (mon.0) 3370 : audit [DBG] from='client.? 172.21.15.6:0/2660920838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:24.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:24 smithi006 bash[27292]: cluster 2023-08-28T00:35:22.797708+0000 mgr.y (mgr.14901) 4704 : cluster [DBG] pgmap v4711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:24.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:24 smithi006 bash[27292]: audit 2023-08-28T00:35:23.269310+0000 mon.a (mon.0) 3370 : audit [DBG] from='client.? 172.21.15.6:0/2660920838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:24.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:24 smithi006 bash[33918]: cluster 2023-08-28T00:35:22.797708+0000 mgr.y (mgr.14901) 4704 : cluster [DBG] pgmap v4711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:24.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:24 smithi006 bash[33918]: audit 2023-08-28T00:35:23.269310+0000 mon.a (mon.0) 3370 : audit [DBG] from='client.? 172.21.15.6:0/2660920838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:24.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:35:24.239Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:24.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:35:24.241Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:24 smithi130 bash[51302]: debug 2023-08-28T00:35:24.656+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:24.662286+0000) 2023-08-28T00:35:26.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:25 smithi130 bash[51302]: debug 2023-08-28T00:35:25.656+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:25.662595+0000) 2023-08-28T00:35:26.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:26 smithi006 bash[27292]: cluster 2023-08-28T00:35:24.798959+0000 mgr.y (mgr.14901) 4705 : cluster [DBG] pgmap v4712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:26.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:26 smithi006 bash[27292]: audit 2023-08-28T00:35:25.689022+0000 mon.a (mon.0) 3371 : audit [DBG] from='client.? 172.21.15.6:0/2951150561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:26 smithi006 bash[33918]: cluster 2023-08-28T00:35:24.798959+0000 mgr.y (mgr.14901) 4705 : cluster [DBG] pgmap v4712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:26 smithi006 bash[33918]: audit 2023-08-28T00:35:25.689022+0000 mon.a (mon.0) 3371 : audit [DBG] from='client.? 172.21.15.6:0/2951150561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:26 smithi130 bash[27579]: cluster 2023-08-28T00:35:24.798959+0000 mgr.y (mgr.14901) 4705 : cluster [DBG] pgmap v4712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:26 smithi130 bash[27579]: audit 2023-08-28T00:35:25.689022+0000 mon.a (mon.0) 3371 : audit [DBG] from='client.? 172.21.15.6:0/2951150561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:27.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:26 smithi130 bash[51302]: debug 2023-08-28T00:35:26.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:26.662937+0000) 2023-08-28T00:35:28.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:27 smithi130 bash[51302]: debug 2023-08-28T00:35:27.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:27.663146+0000) 2023-08-28T00:35:28.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:28 smithi006 bash[27292]: cluster 2023-08-28T00:35:26.799744+0000 mgr.y (mgr.14901) 4706 : cluster [DBG] pgmap v4713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:28.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:28 smithi006 bash[27292]: audit 2023-08-28T00:35:28.101498+0000 mon.c (mon.2) 1826 : audit [DBG] from='client.? 172.21.15.6:0/3578975638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:28.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:28 smithi006 bash[33918]: cluster 2023-08-28T00:35:26.799744+0000 mgr.y (mgr.14901) 4706 : cluster [DBG] pgmap v4713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:28.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:28 smithi006 bash[33918]: audit 2023-08-28T00:35:28.101498+0000 mon.c (mon.2) 1826 : audit [DBG] from='client.? 172.21.15.6:0/3578975638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:28 smithi130 bash[27579]: cluster 2023-08-28T00:35:26.799744+0000 mgr.y (mgr.14901) 4706 : cluster [DBG] pgmap v4713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:28 smithi130 bash[27579]: audit 2023-08-28T00:35:28.101498+0000 mon.c (mon.2) 1826 : audit [DBG] from='client.? 172.21.15.6:0/3578975638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:29.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:28 smithi130 bash[51302]: debug 2023-08-28T00:35:28.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:28.663364+0000) 2023-08-28T00:35:30.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:29 smithi130 bash[51302]: debug 2023-08-28T00:35:29.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:29.663600+0000) 2023-08-28T00:35:30.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:30 smithi006 bash[33918]: cluster 2023-08-28T00:35:28.800556+0000 mgr.y (mgr.14901) 4707 : cluster [DBG] pgmap v4714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:30.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:30 smithi006 bash[27292]: cluster 2023-08-28T00:35:28.800556+0000 mgr.y (mgr.14901) 4707 : cluster [DBG] pgmap v4714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:30 smithi130 bash[27579]: cluster 2023-08-28T00:35:28.800556+0000 mgr.y (mgr.14901) 4707 : cluster [DBG] pgmap v4714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:31.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:30 smithi130 bash[51302]: debug 2023-08-28T00:35:30.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:30.663882+0000) 2023-08-28T00:35:31.137 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:35:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:35:30] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:35:31.138 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:35:30.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:31.138 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:35:30.780Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:31.139 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:31 smithi006 bash[27292]: audit 2023-08-28T00:35:30.528430+0000 mon.c (mon.2) 1827 : audit [DBG] from='client.? 172.21.15.6:0/2552247977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:31.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:31 smithi006 bash[33918]: audit 2023-08-28T00:35:30.528430+0000 mon.c (mon.2) 1827 : audit [DBG] from='client.? 172.21.15.6:0/2552247977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:31.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:31 smithi130 bash[27579]: audit 2023-08-28T00:35:30.528430+0000 mon.c (mon.2) 1827 : audit [DBG] from='client.? 172.21.15.6:0/2552247977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:32.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:31 smithi130 bash[51302]: debug 2023-08-28T00:35:31.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:31.664141+0000) 2023-08-28T00:35:32.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:32 smithi006 bash[33918]: cluster 2023-08-28T00:35:30.801711+0000 mgr.y (mgr.14901) 4708 : cluster [DBG] pgmap v4715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:32.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:32 smithi006 bash[27292]: cluster 2023-08-28T00:35:30.801711+0000 mgr.y (mgr.14901) 4708 : cluster [DBG] pgmap v4715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:32.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:35:32.359Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:32.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:35:32.361Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:32 smithi130 bash[27579]: cluster 2023-08-28T00:35:30.801711+0000 mgr.y (mgr.14901) 4708 : cluster [DBG] pgmap v4715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:33.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:32 smithi130 bash[51302]: debug 2023-08-28T00:35:32.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:32.664444+0000) 2023-08-28T00:35:33.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:33 smithi006 bash[27292]: audit 2023-08-28T00:35:32.945495+0000 mon.a (mon.0) 3372 : audit [DBG] from='client.? 172.21.15.6:0/4092703315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:33.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:33 smithi006 bash[33918]: audit 2023-08-28T00:35:32.945495+0000 mon.a (mon.0) 3372 : audit [DBG] from='client.? 172.21.15.6:0/4092703315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:33 smithi130 bash[27579]: audit 2023-08-28T00:35:32.945495+0000 mon.a (mon.0) 3372 : audit [DBG] from='client.? 172.21.15.6:0/4092703315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:34.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:33 smithi130 bash[51302]: debug 2023-08-28T00:35:33.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:33.664745+0000) 2023-08-28T00:35:34.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:34 smithi006 bash[27292]: cluster 2023-08-28T00:35:32.802466+0000 mgr.y (mgr.14901) 4709 : cluster [DBG] pgmap v4716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:34.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:34 smithi006 bash[27292]: audit 2023-08-28T00:35:33.151872+0000 mon.b (mon.1) 572 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:35:34.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:34 smithi006 bash[33918]: cluster 2023-08-28T00:35:32.802466+0000 mgr.y (mgr.14901) 4709 : cluster [DBG] pgmap v4716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:34.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:34 smithi006 bash[33918]: audit 2023-08-28T00:35:33.151872+0000 mon.b (mon.1) 572 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:35:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:35:34.239Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:34.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:35:34.241Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:34 smithi130 bash[27579]: cluster 2023-08-28T00:35:32.802466+0000 mgr.y (mgr.14901) 4709 : cluster [DBG] pgmap v4716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:34 smithi130 bash[27579]: audit 2023-08-28T00:35:33.151872+0000 mon.b (mon.1) 572 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:35:35.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:34 smithi130 bash[51302]: debug 2023-08-28T00:35:34.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:34.665055+0000) 2023-08-28T00:35:36.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:35 smithi130 bash[51302]: debug 2023-08-28T00:35:35.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:35.665371+0000) 2023-08-28T00:35:36.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:36 smithi006 bash[27292]: cluster 2023-08-28T00:35:34.803792+0000 mgr.y (mgr.14901) 4710 : cluster [DBG] pgmap v4717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:36.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:36 smithi006 bash[27292]: audit 2023-08-28T00:35:35.369816+0000 mon.a (mon.0) 3373 : audit [DBG] from='client.? 172.21.15.6:0/201398152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:36.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:36 smithi006 bash[33918]: cluster 2023-08-28T00:35:34.803792+0000 mgr.y (mgr.14901) 4710 : cluster [DBG] pgmap v4717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:36.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:36 smithi006 bash[33918]: audit 2023-08-28T00:35:35.369816+0000 mon.a (mon.0) 3373 : audit [DBG] from='client.? 172.21.15.6:0/201398152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:36 smithi130 bash[27579]: cluster 2023-08-28T00:35:34.803792+0000 mgr.y (mgr.14901) 4710 : cluster [DBG] pgmap v4717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:36 smithi130 bash[27579]: audit 2023-08-28T00:35:35.369816+0000 mon.a (mon.0) 3373 : audit [DBG] from='client.? 172.21.15.6:0/201398152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:37.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:36 smithi130 bash[51302]: debug 2023-08-28T00:35:36.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:36.665688+0000) 2023-08-28T00:35:38.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:37 smithi130 bash[51302]: debug 2023-08-28T00:35:37.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:37.665956+0000) 2023-08-28T00:35:38.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:38 smithi006 bash[33918]: cluster 2023-08-28T00:35:36.804480+0000 mgr.y (mgr.14901) 4711 : cluster [DBG] pgmap v4718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:38.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:38 smithi006 bash[33918]: audit 2023-08-28T00:35:37.793642+0000 mon.c (mon.2) 1828 : audit [DBG] from='client.? 172.21.15.6:0/3773190824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:38.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:38 smithi006 bash[27292]: cluster 2023-08-28T00:35:36.804480+0000 mgr.y (mgr.14901) 4711 : cluster [DBG] pgmap v4718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:38.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:38 smithi006 bash[27292]: audit 2023-08-28T00:35:37.793642+0000 mon.c (mon.2) 1828 : audit [DBG] from='client.? 172.21.15.6:0/3773190824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:38 smithi130 bash[27579]: cluster 2023-08-28T00:35:36.804480+0000 mgr.y (mgr.14901) 4711 : cluster [DBG] pgmap v4718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:38 smithi130 bash[27579]: audit 2023-08-28T00:35:37.793642+0000 mon.c (mon.2) 1828 : audit [DBG] from='client.? 172.21.15.6:0/3773190824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:39.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:38 smithi130 bash[51302]: debug 2023-08-28T00:35:38.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:38.666188+0000) 2023-08-28T00:35:40.043 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:39 smithi130 bash[51302]: debug 2023-08-28T00:35:39.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:39.666499+0000) 2023-08-28T00:35:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:40 smithi130 bash[27579]: cluster 2023-08-28T00:35:38.805018+0000 mgr.y (mgr.14901) 4712 : cluster [DBG] pgmap v4719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:40 smithi130 bash[27579]: audit 2023-08-28T00:35:39.036681+0000 mon.a (mon.0) 3374 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:35:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:40 smithi130 bash[27579]: audit 2023-08-28T00:35:39.494633+0000 mon.a (mon.0) 3375 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:35:40.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:40 smithi130 bash[27579]: audit 2023-08-28T00:35:39.788898+0000 mon.b (mon.1) 573 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:35:40.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:40 smithi130 bash[27579]: audit 2023-08-28T00:35:39.791218+0000 mon.b (mon.1) 574 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:35:40.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:40 smithi130 bash[27579]: audit 2023-08-28T00:35:39.799069+0000 mon.a (mon.0) 3376 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:35:40.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:40 smithi006 bash[27292]: cluster 2023-08-28T00:35:38.805018+0000 mgr.y (mgr.14901) 4712 : cluster [DBG] pgmap v4719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:40.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:40 smithi006 bash[27292]: audit 2023-08-28T00:35:39.036681+0000 mon.a (mon.0) 3374 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:35:40.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:40 smithi006 bash[27292]: audit 2023-08-28T00:35:39.494633+0000 mon.a (mon.0) 3375 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:35:40.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:40 smithi006 bash[27292]: audit 2023-08-28T00:35:39.788898+0000 mon.b (mon.1) 573 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:35:40.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:40 smithi006 bash[27292]: audit 2023-08-28T00:35:39.791218+0000 mon.b (mon.1) 574 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:35:40.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:40 smithi006 bash[27292]: audit 2023-08-28T00:35:39.799069+0000 mon.a (mon.0) 3376 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:35:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:40 smithi006 bash[33918]: cluster 2023-08-28T00:35:38.805018+0000 mgr.y (mgr.14901) 4712 : cluster [DBG] pgmap v4719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:40.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:40 smithi006 bash[33918]: audit 2023-08-28T00:35:39.036681+0000 mon.a (mon.0) 3374 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:35:40.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:40 smithi006 bash[33918]: audit 2023-08-28T00:35:39.494633+0000 mon.a (mon.0) 3375 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:35:40.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:40 smithi006 bash[33918]: audit 2023-08-28T00:35:39.788898+0000 mon.b (mon.1) 573 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:35:40.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:40 smithi006 bash[33918]: audit 2023-08-28T00:35:39.791218+0000 mon.b (mon.1) 574 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:35:40.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:40 smithi006 bash[33918]: audit 2023-08-28T00:35:39.799069+0000 mon.a (mon.0) 3376 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:35:41.045 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:35:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:35:40] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:35:41.046 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:35:40.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:41.046 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:35:40.781Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:41.046 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:40 smithi130 bash[51302]: debug 2023-08-28T00:35:40.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:40.666759+0000) 2023-08-28T00:35:41.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:41 smithi130 bash[27579]: audit 2023-08-28T00:35:40.217423+0000 mon.a (mon.0) 3377 : audit [DBG] from='client.? 172.21.15.6:0/357963069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:41.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:41 smithi006 bash[27292]: audit 2023-08-28T00:35:40.217423+0000 mon.a (mon.0) 3377 : audit [DBG] from='client.? 172.21.15.6:0/357963069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:41.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:41 smithi006 bash[33918]: audit 2023-08-28T00:35:40.217423+0000 mon.a (mon.0) 3377 : audit [DBG] from='client.? 172.21.15.6:0/357963069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:42.057 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:41 smithi130 bash[51302]: debug 2023-08-28T00:35:41.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:41.667027+0000) 2023-08-28T00:35:42.363 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:42 smithi006 bash[27292]: cluster 2023-08-28T00:35:40.806247+0000 mgr.y (mgr.14901) 4713 : cluster [DBG] pgmap v4720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:42.363 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:42 smithi006 bash[33918]: cluster 2023-08-28T00:35:40.806247+0000 mgr.y (mgr.14901) 4713 : cluster [DBG] pgmap v4720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:42.364 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:35:42.359Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:42 smithi130 bash[27579]: cluster 2023-08-28T00:35:40.806247+0000 mgr.y (mgr.14901) 4713 : cluster [DBG] pgmap v4720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:35:42.361Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:43.066 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:42 smithi130 bash[51302]: debug 2023-08-28T00:35:42.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:42.667280+0000) 2023-08-28T00:35:43.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:43 smithi130 bash[27579]: audit 2023-08-28T00:35:42.637157+0000 mon.b (mon.1) 575 : audit [DBG] from='client.? 172.21.15.6:0/260745547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:43.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:43 smithi006 bash[27292]: audit 2023-08-28T00:35:42.637157+0000 mon.b (mon.1) 575 : audit [DBG] from='client.? 172.21.15.6:0/260745547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:43.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:43 smithi006 bash[33918]: audit 2023-08-28T00:35:42.637157+0000 mon.b (mon.1) 575 : audit [DBG] from='client.? 172.21.15.6:0/260745547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:44.067 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:43 smithi130 bash[51302]: debug 2023-08-28T00:35:43.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:43.667499+0000) 2023-08-28T00:35:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:44 smithi130 bash[27579]: cluster 2023-08-28T00:35:42.807054+0000 mgr.y (mgr.14901) 4714 : cluster [DBG] pgmap v4721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:44.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:44 smithi006 bash[27292]: cluster 2023-08-28T00:35:42.807054+0000 mgr.y (mgr.14901) 4714 : cluster [DBG] pgmap v4721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:44.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:44 smithi006 bash[33918]: cluster 2023-08-28T00:35:42.807054+0000 mgr.y (mgr.14901) 4714 : cluster [DBG] pgmap v4721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:35:44.240Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:35:44.242Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:45.076 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:44 smithi130 bash[51302]: debug 2023-08-28T00:35:44.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:44.667783+0000) 2023-08-28T00:35:45.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:45 smithi130 bash[27579]: audit 2023-08-28T00:35:45.053663+0000 mon.a (mon.0) 3378 : audit [DBG] from='client.? 172.21.15.6:0/3958210132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:45.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:45 smithi006 bash[33918]: audit 2023-08-28T00:35:45.053663+0000 mon.a (mon.0) 3378 : audit [DBG] from='client.? 172.21.15.6:0/3958210132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:45.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:45 smithi006 bash[27292]: audit 2023-08-28T00:35:45.053663+0000 mon.a (mon.0) 3378 : audit [DBG] from='client.? 172.21.15.6:0/3958210132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:46.079 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:45 smithi130 bash[51302]: debug 2023-08-28T00:35:45.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:45.668084+0000) 2023-08-28T00:35:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:46 smithi130 bash[27579]: cluster 2023-08-28T00:35:44.808116+0000 mgr.y (mgr.14901) 4715 : cluster [DBG] pgmap v4722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:46.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:46 smithi006 bash[27292]: cluster 2023-08-28T00:35:44.808116+0000 mgr.y (mgr.14901) 4715 : cluster [DBG] pgmap v4722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:46.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:46 smithi006 bash[33918]: cluster 2023-08-28T00:35:44.808116+0000 mgr.y (mgr.14901) 4715 : cluster [DBG] pgmap v4722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:47.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:46 smithi130 bash[51302]: debug 2023-08-28T00:35:46.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:46.668364+0000) 2023-08-28T00:35:48.084 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:47 smithi130 bash[51302]: debug 2023-08-28T00:35:47.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:47.668644+0000) 2023-08-28T00:35:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:48 smithi130 bash[27579]: cluster 2023-08-28T00:35:46.808812+0000 mgr.y (mgr.14901) 4716 : cluster [DBG] pgmap v4723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:48 smithi130 bash[27579]: audit 2023-08-28T00:35:47.469141+0000 mon.a (mon.0) 3379 : audit [DBG] from='client.? 172.21.15.6:0/1415048829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:48.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:48 smithi006 bash[27292]: cluster 2023-08-28T00:35:46.808812+0000 mgr.y (mgr.14901) 4716 : cluster [DBG] pgmap v4723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:48.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:48 smithi006 bash[27292]: audit 2023-08-28T00:35:47.469141+0000 mon.a (mon.0) 3379 : audit [DBG] from='client.? 172.21.15.6:0/1415048829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:48.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:48 smithi006 bash[33918]: cluster 2023-08-28T00:35:46.808812+0000 mgr.y (mgr.14901) 4716 : cluster [DBG] pgmap v4723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:48.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:48 smithi006 bash[33918]: audit 2023-08-28T00:35:47.469141+0000 mon.a (mon.0) 3379 : audit [DBG] from='client.? 172.21.15.6:0/1415048829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:49.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:48 smithi130 bash[51302]: debug 2023-08-28T00:35:48.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:48.668921+0000) 2023-08-28T00:35:50.090 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:49 smithi130 bash[51302]: debug 2023-08-28T00:35:49.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:49.669213+0000) 2023-08-28T00:35:50.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:50 smithi130 bash[27579]: cluster 2023-08-28T00:35:48.809474+0000 mgr.y (mgr.14901) 4717 : cluster [DBG] pgmap v4724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:50.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:50 smithi130 bash[27579]: audit 2023-08-28T00:35:49.879212+0000 mon.c (mon.2) 1829 : audit [DBG] from='client.? 172.21.15.6:0/2712726355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:50.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:50 smithi006 bash[27292]: cluster 2023-08-28T00:35:48.809474+0000 mgr.y (mgr.14901) 4717 : cluster [DBG] pgmap v4724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:50.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:50 smithi006 bash[27292]: audit 2023-08-28T00:35:49.879212+0000 mon.c (mon.2) 1829 : audit [DBG] from='client.? 172.21.15.6:0/2712726355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:50.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:50 smithi006 bash[33918]: cluster 2023-08-28T00:35:48.809474+0000 mgr.y (mgr.14901) 4717 : cluster [DBG] pgmap v4724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:50.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:50 smithi006 bash[33918]: audit 2023-08-28T00:35:49.879212+0000 mon.c (mon.2) 1829 : audit [DBG] from='client.? 172.21.15.6:0/2712726355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:51.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:50 smithi130 bash[51302]: debug 2023-08-28T00:35:50.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:50.669494+0000) 2023-08-28T00:35:51.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:35:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:35:50] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:35:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:35:50.780Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:35:50.782Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:52.096 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:51 smithi130 bash[51302]: debug 2023-08-28T00:35:51.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:51.669863+0000) 2023-08-28T00:35:52.097 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:52 smithi130 bash[27579]: cluster 2023-08-28T00:35:50.810623+0000 mgr.y (mgr.14901) 4718 : cluster [DBG] pgmap v4725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:52.363 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:52 smithi006 bash[33918]: cluster 2023-08-28T00:35:50.810623+0000 mgr.y (mgr.14901) 4718 : cluster [DBG] pgmap v4725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:52.364 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:52 smithi006 bash[27292]: cluster 2023-08-28T00:35:50.810623+0000 mgr.y (mgr.14901) 4718 : cluster [DBG] pgmap v4725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:35:52.360Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:35:52.362Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:53.105 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:52 smithi130 bash[51302]: debug 2023-08-28T00:35:52.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:52.670173+0000) 2023-08-28T00:35:53.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:53 smithi130 bash[27579]: audit 2023-08-28T00:35:52.300111+0000 mon.a (mon.0) 3380 : audit [DBG] from='client.? 172.21.15.6:0/3779057245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:53.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:53 smithi006 bash[27292]: audit 2023-08-28T00:35:52.300111+0000 mon.a (mon.0) 3380 : audit [DBG] from='client.? 172.21.15.6:0/3779057245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:53.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:53 smithi006 bash[33918]: audit 2023-08-28T00:35:52.300111+0000 mon.a (mon.0) 3380 : audit [DBG] from='client.? 172.21.15.6:0/3779057245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:54.107 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:53 smithi130 bash[51302]: debug 2023-08-28T00:35:53.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:53.670397+0000) 2023-08-28T00:35:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:54 smithi130 bash[27579]: cluster 2023-08-28T00:35:52.811483+0000 mgr.y (mgr.14901) 4719 : cluster [DBG] pgmap v4726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:54.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:54 smithi006 bash[33918]: cluster 2023-08-28T00:35:52.811483+0000 mgr.y (mgr.14901) 4719 : cluster [DBG] pgmap v4726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:54.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:54 smithi006 bash[27292]: cluster 2023-08-28T00:35:52.811483+0000 mgr.y (mgr.14901) 4719 : cluster [DBG] pgmap v4726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:35:54.241Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:35:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:35:54.243Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:35:55.110 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:54 smithi130 bash[51302]: debug 2023-08-28T00:35:54.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:54.670667+0000) 2023-08-28T00:35:55.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:55 smithi130 bash[27579]: audit 2023-08-28T00:35:54.707397+0000 mon.a (mon.0) 3381 : audit [DBG] from='client.? 172.21.15.6:0/293605958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:55.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:55 smithi006 bash[33918]: audit 2023-08-28T00:35:54.707397+0000 mon.a (mon.0) 3381 : audit [DBG] from='client.? 172.21.15.6:0/293605958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:55.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:55 smithi006 bash[27292]: audit 2023-08-28T00:35:54.707397+0000 mon.a (mon.0) 3381 : audit [DBG] from='client.? 172.21.15.6:0/293605958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:56.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:55 smithi130 bash[51302]: debug 2023-08-28T00:35:55.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:55.670966+0000) 2023-08-28T00:35:56.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:56 smithi006 bash[27292]: cluster 2023-08-28T00:35:54.812634+0000 mgr.y (mgr.14901) 4720 : cluster [DBG] pgmap v4727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:56.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:56 smithi006 bash[33918]: cluster 2023-08-28T00:35:54.812634+0000 mgr.y (mgr.14901) 4720 : cluster [DBG] pgmap v4727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:56 smithi130 bash[27579]: cluster 2023-08-28T00:35:54.812634+0000 mgr.y (mgr.14901) 4720 : cluster [DBG] pgmap v4727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:57.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:56 smithi130 bash[51302]: debug 2023-08-28T00:35:56.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:56.671193+0000) 2023-08-28T00:35:57.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:57 smithi006 bash[33918]: audit 2023-08-28T00:35:57.112712+0000 mon.c (mon.2) 1830 : audit [DBG] from='client.? 172.21.15.6:0/722743446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:57.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:57 smithi006 bash[27292]: audit 2023-08-28T00:35:57.112712+0000 mon.c (mon.2) 1830 : audit [DBG] from='client.? 172.21.15.6:0/722743446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:57.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:57 smithi130 bash[27579]: audit 2023-08-28T00:35:57.112712+0000 mon.c (mon.2) 1830 : audit [DBG] from='client.? 172.21.15.6:0/722743446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:58.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:57 smithi130 bash[51302]: debug 2023-08-28T00:35:57.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:57.671412+0000) 2023-08-28T00:35:58.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:35:58 smithi006 bash[33918]: cluster 2023-08-28T00:35:56.813306+0000 mgr.y (mgr.14901) 4721 : cluster [DBG] pgmap v4728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:58.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:35:58 smithi006 bash[27292]: cluster 2023-08-28T00:35:56.813306+0000 mgr.y (mgr.14901) 4721 : cluster [DBG] pgmap v4728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:35:58 smithi130 bash[27579]: cluster 2023-08-28T00:35:56.813306+0000 mgr.y (mgr.14901) 4721 : cluster [DBG] pgmap v4728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:35:59.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:58 smithi130 bash[51302]: debug 2023-08-28T00:35:58.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:58.671636+0000) 2023-08-28T00:36:00.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:35:59 smithi130 bash[51302]: debug 2023-08-28T00:35:59.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:59.671936+0000) 2023-08-28T00:36:00.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:00 smithi006 bash[33918]: cluster 2023-08-28T00:35:58.813919+0000 mgr.y (mgr.14901) 4722 : cluster [DBG] pgmap v4729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:00.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:00 smithi006 bash[33918]: audit 2023-08-28T00:35:59.526586+0000 mon.c (mon.2) 1831 : audit [DBG] from='client.? 172.21.15.6:0/2490258349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:00.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:00 smithi006 bash[27292]: cluster 2023-08-28T00:35:58.813919+0000 mgr.y (mgr.14901) 4722 : cluster [DBG] pgmap v4729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:00.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:00 smithi006 bash[27292]: audit 2023-08-28T00:35:59.526586+0000 mon.c (mon.2) 1831 : audit [DBG] from='client.? 172.21.15.6:0/2490258349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:00 smithi130 bash[27579]: cluster 2023-08-28T00:35:58.813919+0000 mgr.y (mgr.14901) 4722 : cluster [DBG] pgmap v4729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:00 smithi130 bash[27579]: audit 2023-08-28T00:35:59.526586+0000 mon.c (mon.2) 1831 : audit [DBG] from='client.? 172.21.15.6:0/2490258349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:01.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:00 smithi130 bash[51302]: debug 2023-08-28T00:36:00.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:00.672267+0000) 2023-08-28T00:36:01.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:36:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:36:00] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:36:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:36:00.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:36:00.783Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:02.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:01 smithi130 bash[51302]: debug 2023-08-28T00:36:01.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:01.672495+0000) 2023-08-28T00:36:02.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:02 smithi006 bash[33918]: cluster 2023-08-28T00:36:00.815096+0000 mgr.y (mgr.14901) 4723 : cluster [DBG] pgmap v4730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:02.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:02 smithi006 bash[33918]: audit 2023-08-28T00:36:01.940984+0000 mon.a (mon.0) 3382 : audit [DBG] from='client.? 172.21.15.6:0/3580186596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:36:02.360Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:36:02.362Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:02.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:02 smithi006 bash[27292]: cluster 2023-08-28T00:36:00.815096+0000 mgr.y (mgr.14901) 4723 : cluster [DBG] pgmap v4730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:02.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:02 smithi006 bash[27292]: audit 2023-08-28T00:36:01.940984+0000 mon.a (mon.0) 3382 : audit [DBG] from='client.? 172.21.15.6:0/3580186596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:02 smithi130 bash[27579]: cluster 2023-08-28T00:36:00.815096+0000 mgr.y (mgr.14901) 4723 : cluster [DBG] pgmap v4730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:02 smithi130 bash[27579]: audit 2023-08-28T00:36:01.940984+0000 mon.a (mon.0) 3382 : audit [DBG] from='client.? 172.21.15.6:0/3580186596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:02 smithi130 bash[51302]: debug 2023-08-28T00:36:02.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:02.672756+0000) 2023-08-28T00:36:04.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:03 smithi130 bash[51302]: debug 2023-08-28T00:36:03.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:03.672973+0000) 2023-08-28T00:36:04.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:04 smithi006 bash[33918]: cluster 2023-08-28T00:36:02.815780+0000 mgr.y (mgr.14901) 4724 : cluster [DBG] pgmap v4731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:04.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:04 smithi006 bash[27292]: cluster 2023-08-28T00:36:02.815780+0000 mgr.y (mgr.14901) 4724 : cluster [DBG] pgmap v4731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:36:04.242Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:36:04.244Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:04 smithi130 bash[27579]: cluster 2023-08-28T00:36:02.815780+0000 mgr.y (mgr.14901) 4724 : cluster [DBG] pgmap v4731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:04 smithi130 bash[51302]: debug 2023-08-28T00:36:04.672+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:04.673287+0000) 2023-08-28T00:36:05.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:05 smithi006 bash[33918]: audit 2023-08-28T00:36:04.363214+0000 mon.a (mon.0) 3383 : audit [DBG] from='client.? 172.21.15.6:0/2254110297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:05.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:05 smithi006 bash[27292]: audit 2023-08-28T00:36:04.363214+0000 mon.a (mon.0) 3383 : audit [DBG] from='client.? 172.21.15.6:0/2254110297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:05.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:05 smithi130 bash[27579]: audit 2023-08-28T00:36:04.363214+0000 mon.a (mon.0) 3383 : audit [DBG] from='client.? 172.21.15.6:0/2254110297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:06.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:05 smithi130 bash[51302]: debug 2023-08-28T00:36:05.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:05.673585+0000) 2023-08-28T00:36:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:06 smithi006 bash[33918]: cluster 2023-08-28T00:36:04.816965+0000 mgr.y (mgr.14901) 4725 : cluster [DBG] pgmap v4732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:06.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:06 smithi006 bash[27292]: cluster 2023-08-28T00:36:04.816965+0000 mgr.y (mgr.14901) 4725 : cluster [DBG] pgmap v4732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:06 smithi130 bash[27579]: cluster 2023-08-28T00:36:04.816965+0000 mgr.y (mgr.14901) 4725 : cluster [DBG] pgmap v4732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:07.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:06 smithi130 bash[51302]: debug 2023-08-28T00:36:06.671+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:06.673847+0000) 2023-08-28T00:36:07.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:07 smithi006 bash[27292]: audit 2023-08-28T00:36:06.783423+0000 mon.a (mon.0) 3384 : audit [DBG] from='client.? 172.21.15.6:0/260125959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:07.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:07 smithi006 bash[33918]: audit 2023-08-28T00:36:06.783423+0000 mon.a (mon.0) 3384 : audit [DBG] from='client.? 172.21.15.6:0/260125959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:07.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:07 smithi130 bash[27579]: audit 2023-08-28T00:36:06.783423+0000 mon.a (mon.0) 3384 : audit [DBG] from='client.? 172.21.15.6:0/260125959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:08.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:07 smithi130 bash[51302]: debug 2023-08-28T00:36:07.671+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:07.674118+0000) 2023-08-28T00:36:08.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:08 smithi006 bash[33918]: cluster 2023-08-28T00:36:06.817549+0000 mgr.y (mgr.14901) 4726 : cluster [DBG] pgmap v4733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:08.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:08 smithi006 bash[27292]: cluster 2023-08-28T00:36:06.817549+0000 mgr.y (mgr.14901) 4726 : cluster [DBG] pgmap v4733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:08 smithi130 bash[27579]: cluster 2023-08-28T00:36:06.817549+0000 mgr.y (mgr.14901) 4726 : cluster [DBG] pgmap v4733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:09.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:08 smithi130 bash[51302]: debug 2023-08-28T00:36:08.667+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:08.674417+0000) 2023-08-28T00:36:10.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:09 smithi130 bash[51302]: debug 2023-08-28T00:36:09.671+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:09.674705+0000) 2023-08-28T00:36:10.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:10 smithi006 bash[33918]: cluster 2023-08-28T00:36:08.818213+0000 mgr.y (mgr.14901) 4727 : cluster [DBG] pgmap v4734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:10.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:10 smithi006 bash[33918]: audit 2023-08-28T00:36:09.195064+0000 mon.a (mon.0) 3385 : audit [DBG] from='client.? 172.21.15.6:0/57556303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:10.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:10 smithi006 bash[27292]: cluster 2023-08-28T00:36:08.818213+0000 mgr.y (mgr.14901) 4727 : cluster [DBG] pgmap v4734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:10.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:10 smithi006 bash[27292]: audit 2023-08-28T00:36:09.195064+0000 mon.a (mon.0) 3385 : audit [DBG] from='client.? 172.21.15.6:0/57556303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:10 smithi130 bash[27579]: cluster 2023-08-28T00:36:08.818213+0000 mgr.y (mgr.14901) 4727 : cluster [DBG] pgmap v4734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:10 smithi130 bash[27579]: audit 2023-08-28T00:36:09.195064+0000 mon.a (mon.0) 3385 : audit [DBG] from='client.? 172.21.15.6:0/57556303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:11.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:10 smithi130 bash[51302]: debug 2023-08-28T00:36:10.671+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:10.674982+0000) 2023-08-28T00:36:11.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:36:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:36:10] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:36:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:36:10.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:36:10.783Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:12.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:11 smithi130 bash[51302]: debug 2023-08-28T00:36:11.671+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:11.675237+0000) 2023-08-28T00:36:12.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:12 smithi006 bash[33918]: cluster 2023-08-28T00:36:10.819311+0000 mgr.y (mgr.14901) 4728 : cluster [DBG] pgmap v4735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:12.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:12 smithi006 bash[33918]: audit 2023-08-28T00:36:11.611247+0000 mon.a (mon.0) 3386 : audit [DBG] from='client.? 172.21.15.6:0/1642446584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:12.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:12 smithi006 bash[27292]: cluster 2023-08-28T00:36:10.819311+0000 mgr.y (mgr.14901) 4728 : cluster [DBG] pgmap v4735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:12.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:12 smithi006 bash[27292]: audit 2023-08-28T00:36:11.611247+0000 mon.a (mon.0) 3386 : audit [DBG] from='client.? 172.21.15.6:0/1642446584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:12.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:36:12.361Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:12.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:36:12.363Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:12 smithi130 bash[27579]: cluster 2023-08-28T00:36:10.819311+0000 mgr.y (mgr.14901) 4728 : cluster [DBG] pgmap v4735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:12 smithi130 bash[27579]: audit 2023-08-28T00:36:11.611247+0000 mon.a (mon.0) 3386 : audit [DBG] from='client.? 172.21.15.6:0/1642446584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:13.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:12 smithi130 bash[51302]: debug 2023-08-28T00:36:12.671+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:12.675459+0000) 2023-08-28T00:36:14.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:13 smithi130 bash[51302]: debug 2023-08-28T00:36:13.671+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:13.675737+0000) 2023-08-28T00:36:14.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:14 smithi006 bash[27292]: cluster 2023-08-28T00:36:12.820103+0000 mgr.y (mgr.14901) 4729 : cluster [DBG] pgmap v4736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:14.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:14 smithi006 bash[27292]: audit 2023-08-28T00:36:14.021068+0000 mon.c (mon.2) 1832 : audit [DBG] from='client.? 172.21.15.6:0/1284062599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:36:14.243Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:36:14.245Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:14.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:14 smithi006 bash[33918]: cluster 2023-08-28T00:36:12.820103+0000 mgr.y (mgr.14901) 4729 : cluster [DBG] pgmap v4736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:14.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:14 smithi006 bash[33918]: audit 2023-08-28T00:36:14.021068+0000 mon.c (mon.2) 1832 : audit [DBG] from='client.? 172.21.15.6:0/1284062599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:14 smithi130 bash[27579]: cluster 2023-08-28T00:36:12.820103+0000 mgr.y (mgr.14901) 4729 : cluster [DBG] pgmap v4736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:14 smithi130 bash[27579]: audit 2023-08-28T00:36:14.021068+0000 mon.c (mon.2) 1832 : audit [DBG] from='client.? 172.21.15.6:0/1284062599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:15.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:14 smithi130 bash[51302]: debug 2023-08-28T00:36:14.671+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:14.676057+0000) 2023-08-28T00:36:16.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:15 smithi130 bash[51302]: debug 2023-08-28T00:36:15.671+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:15.676310+0000) 2023-08-28T00:36:16.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:16 smithi006 bash[33918]: cluster 2023-08-28T00:36:14.821384+0000 mgr.y (mgr.14901) 4730 : cluster [DBG] pgmap v4737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:16.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:16 smithi006 bash[27292]: cluster 2023-08-28T00:36:14.821384+0000 mgr.y (mgr.14901) 4730 : cluster [DBG] pgmap v4737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:16 smithi130 bash[27579]: cluster 2023-08-28T00:36:14.821384+0000 mgr.y (mgr.14901) 4730 : cluster [DBG] pgmap v4737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:17.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:16 smithi130 bash[51302]: debug 2023-08-28T00:36:16.671+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:16.676576+0000) 2023-08-28T00:36:17.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:17 smithi006 bash[33918]: audit 2023-08-28T00:36:16.437384+0000 mon.a (mon.0) 3387 : audit [DBG] from='client.? 172.21.15.6:0/4225578127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:17.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:17 smithi006 bash[27292]: audit 2023-08-28T00:36:16.437384+0000 mon.a (mon.0) 3387 : audit [DBG] from='client.? 172.21.15.6:0/4225578127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:17.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:17 smithi130 bash[27579]: audit 2023-08-28T00:36:16.437384+0000 mon.a (mon.0) 3387 : audit [DBG] from='client.? 172.21.15.6:0/4225578127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:18.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:17 smithi130 bash[51302]: debug 2023-08-28T00:36:17.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:17.676871+0000) 2023-08-28T00:36:18.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:18 smithi006 bash[27292]: cluster 2023-08-28T00:36:16.822155+0000 mgr.y (mgr.14901) 4731 : cluster [DBG] pgmap v4738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:18.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:18 smithi006 bash[33918]: cluster 2023-08-28T00:36:16.822155+0000 mgr.y (mgr.14901) 4731 : cluster [DBG] pgmap v4738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:18 smithi130 bash[27579]: cluster 2023-08-28T00:36:16.822155+0000 mgr.y (mgr.14901) 4731 : cluster [DBG] pgmap v4738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:19.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:18 smithi130 bash[51302]: debug 2023-08-28T00:36:18.671+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:18.677170+0000) 2023-08-28T00:36:19.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:19 smithi006 bash[27292]: audit 2023-08-28T00:36:18.854222+0000 mon.a (mon.0) 3388 : audit [DBG] from='client.? 172.21.15.6:0/1364528378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:19.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:19 smithi006 bash[33918]: audit 2023-08-28T00:36:18.854222+0000 mon.a (mon.0) 3388 : audit [DBG] from='client.? 172.21.15.6:0/1364528378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:19 smithi130 bash[27579]: audit 2023-08-28T00:36:18.854222+0000 mon.a (mon.0) 3388 : audit [DBG] from='client.? 172.21.15.6:0/1364528378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:20.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:19 smithi130 bash[51302]: debug 2023-08-28T00:36:19.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:19.677451+0000) 2023-08-28T00:36:20.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:20 smithi006 bash[27292]: cluster 2023-08-28T00:36:18.822665+0000 mgr.y (mgr.14901) 4732 : cluster [DBG] pgmap v4739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:20.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:20 smithi006 bash[33918]: cluster 2023-08-28T00:36:18.822665+0000 mgr.y (mgr.14901) 4732 : cluster [DBG] pgmap v4739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:20 smithi130 bash[27579]: cluster 2023-08-28T00:36:18.822665+0000 mgr.y (mgr.14901) 4732 : cluster [DBG] pgmap v4739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:21.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:20 smithi130 bash[51302]: debug 2023-08-28T00:36:20.671+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:20.677752+0000) 2023-08-28T00:36:21.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:36:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:36:20] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:36:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:36:20.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:36:20.785Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:22.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:21 smithi130 bash[51302]: debug 2023-08-28T00:36:21.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:21.678066+0000) 2023-08-28T00:36:22.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:22 smithi006 bash[33918]: cluster 2023-08-28T00:36:20.823670+0000 mgr.y (mgr.14901) 4733 : cluster [DBG] pgmap v4740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:22.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:22 smithi006 bash[33918]: audit 2023-08-28T00:36:21.267666+0000 mon.a (mon.0) 3389 : audit [DBG] from='client.? 172.21.15.6:0/1066543082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:22.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:36:22.362Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:22.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:36:22.364Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:22.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:22 smithi006 bash[27292]: cluster 2023-08-28T00:36:20.823670+0000 mgr.y (mgr.14901) 4733 : cluster [DBG] pgmap v4740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:22.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:22 smithi006 bash[27292]: audit 2023-08-28T00:36:21.267666+0000 mon.a (mon.0) 3389 : audit [DBG] from='client.? 172.21.15.6:0/1066543082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:22 smithi130 bash[27579]: cluster 2023-08-28T00:36:20.823670+0000 mgr.y (mgr.14901) 4733 : cluster [DBG] pgmap v4740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:22 smithi130 bash[27579]: audit 2023-08-28T00:36:21.267666+0000 mon.a (mon.0) 3389 : audit [DBG] from='client.? 172.21.15.6:0/1066543082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:23.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:22 smithi130 bash[51302]: debug 2023-08-28T00:36:22.671+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:22.678350+0000) 2023-08-28T00:36:24.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:23 smithi130 bash[51302]: debug 2023-08-28T00:36:23.671+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:23.678687+0000) 2023-08-28T00:36:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:24 smithi130 bash[27579]: cluster 2023-08-28T00:36:22.824385+0000 mgr.y (mgr.14901) 4734 : cluster [DBG] pgmap v4741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:24 smithi130 bash[27579]: audit 2023-08-28T00:36:23.692269+0000 mon.a (mon.0) 3390 : audit [DBG] from='client.? 172.21.15.6:0/733838715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:24.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:24 smithi006 bash[27292]: cluster 2023-08-28T00:36:22.824385+0000 mgr.y (mgr.14901) 4734 : cluster [DBG] pgmap v4741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:24.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:24 smithi006 bash[27292]: audit 2023-08-28T00:36:23.692269+0000 mon.a (mon.0) 3390 : audit [DBG] from='client.? 172.21.15.6:0/733838715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:36:24.244Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:36:24.246Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:24.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:24 smithi006 bash[33918]: cluster 2023-08-28T00:36:22.824385+0000 mgr.y (mgr.14901) 4734 : cluster [DBG] pgmap v4741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:24.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:24 smithi006 bash[33918]: audit 2023-08-28T00:36:23.692269+0000 mon.a (mon.0) 3390 : audit [DBG] from='client.? 172.21.15.6:0/733838715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:24 smithi130 bash[51302]: debug 2023-08-28T00:36:24.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:24.678991+0000) 2023-08-28T00:36:26.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:25 smithi130 bash[51302]: debug 2023-08-28T00:36:25.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:25.679283+0000) 2023-08-28T00:36:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:26 smithi130 bash[27579]: cluster 2023-08-28T00:36:24.825624+0000 mgr.y (mgr.14901) 4735 : cluster [DBG] pgmap v4742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:26 smithi130 bash[27579]: audit 2023-08-28T00:36:26.104776+0000 mon.c (mon.2) 1833 : audit [DBG] from='client.? 172.21.15.6:0/3256277568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:26 smithi006 bash[33918]: cluster 2023-08-28T00:36:24.825624+0000 mgr.y (mgr.14901) 4735 : cluster [DBG] pgmap v4742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:26 smithi006 bash[33918]: audit 2023-08-28T00:36:26.104776+0000 mon.c (mon.2) 1833 : audit [DBG] from='client.? 172.21.15.6:0/3256277568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:26 smithi006 bash[27292]: cluster 2023-08-28T00:36:24.825624+0000 mgr.y (mgr.14901) 4735 : cluster [DBG] pgmap v4742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:26 smithi006 bash[27292]: audit 2023-08-28T00:36:26.104776+0000 mon.c (mon.2) 1833 : audit [DBG] from='client.? 172.21.15.6:0/3256277568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:27.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:26 smithi130 bash[51302]: debug 2023-08-28T00:36:26.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:26.679619+0000) 2023-08-28T00:36:28.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:27 smithi130 bash[51302]: debug 2023-08-28T00:36:27.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:27.679889+0000) 2023-08-28T00:36:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:28 smithi130 bash[27579]: cluster 2023-08-28T00:36:26.826291+0000 mgr.y (mgr.14901) 4736 : cluster [DBG] pgmap v4743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:28.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:28 smithi006 bash[27292]: cluster 2023-08-28T00:36:26.826291+0000 mgr.y (mgr.14901) 4736 : cluster [DBG] pgmap v4743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:28.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:28 smithi006 bash[33918]: cluster 2023-08-28T00:36:26.826291+0000 mgr.y (mgr.14901) 4736 : cluster [DBG] pgmap v4743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:29.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:28 smithi130 bash[51302]: debug 2023-08-28T00:36:28.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:28.680159+0000) 2023-08-28T00:36:29.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:29 smithi130 bash[27579]: audit 2023-08-28T00:36:28.516433+0000 mon.a (mon.0) 3391 : audit [DBG] from='client.? 172.21.15.6:0/1788509837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:29.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:29 smithi006 bash[27292]: audit 2023-08-28T00:36:28.516433+0000 mon.a (mon.0) 3391 : audit [DBG] from='client.? 172.21.15.6:0/1788509837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:29.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:29 smithi006 bash[33918]: audit 2023-08-28T00:36:28.516433+0000 mon.a (mon.0) 3391 : audit [DBG] from='client.? 172.21.15.6:0/1788509837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:30.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:29 smithi130 bash[51302]: debug 2023-08-28T00:36:29.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:29.680441+0000) 2023-08-28T00:36:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:30 smithi130 bash[27579]: cluster 2023-08-28T00:36:28.826930+0000 mgr.y (mgr.14901) 4737 : cluster [DBG] pgmap v4744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:30.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:30 smithi006 bash[33918]: cluster 2023-08-28T00:36:28.826930+0000 mgr.y (mgr.14901) 4737 : cluster [DBG] pgmap v4744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:30.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:30 smithi006 bash[27292]: cluster 2023-08-28T00:36:28.826930+0000 mgr.y (mgr.14901) 4737 : cluster [DBG] pgmap v4744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:31.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:30 smithi130 bash[51302]: debug 2023-08-28T00:36:30.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:30.680756+0000) 2023-08-28T00:36:31.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:36:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:36:30] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:36:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:36:30.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:36:30.784Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:31.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:31 smithi130 bash[27579]: audit 2023-08-28T00:36:30.932134+0000 mon.c (mon.2) 1834 : audit [DBG] from='client.? 172.21.15.6:0/33899770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:31.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:31 smithi006 bash[33918]: audit 2023-08-28T00:36:30.932134+0000 mon.c (mon.2) 1834 : audit [DBG] from='client.? 172.21.15.6:0/33899770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:31.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:31 smithi006 bash[27292]: audit 2023-08-28T00:36:30.932134+0000 mon.c (mon.2) 1834 : audit [DBG] from='client.? 172.21.15.6:0/33899770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:32.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:31 smithi130 bash[51302]: debug 2023-08-28T00:36:31.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:31.681041+0000) 2023-08-28T00:36:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:32 smithi130 bash[27579]: cluster 2023-08-28T00:36:30.827687+0000 mgr.y (mgr.14901) 4738 : cluster [DBG] pgmap v4745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:32 smithi006 bash[27292]: cluster 2023-08-28T00:36:30.827687+0000 mgr.y (mgr.14901) 4738 : cluster [DBG] pgmap v4745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:32 smithi006 bash[33918]: cluster 2023-08-28T00:36:30.827687+0000 mgr.y (mgr.14901) 4738 : cluster [DBG] pgmap v4745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:36:32.362Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:36:32.364Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:33.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:32 smithi130 bash[51302]: debug 2023-08-28T00:36:32.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:32.681334+0000) 2023-08-28T00:36:34.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:33 smithi130 bash[51302]: debug 2023-08-28T00:36:33.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:33.681596+0000) 2023-08-28T00:36:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:34 smithi130 bash[27579]: cluster 2023-08-28T00:36:32.828323+0000 mgr.y (mgr.14901) 4739 : cluster [DBG] pgmap v4746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:34 smithi130 bash[27579]: audit 2023-08-28T00:36:33.341367+0000 mon.a (mon.0) 3392 : audit [DBG] from='client.? 172.21.15.6:0/1381200868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:34 smithi006 bash[27292]: cluster 2023-08-28T00:36:32.828323+0000 mgr.y (mgr.14901) 4739 : cluster [DBG] pgmap v4746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:34 smithi006 bash[27292]: audit 2023-08-28T00:36:33.341367+0000 mon.a (mon.0) 3392 : audit [DBG] from='client.? 172.21.15.6:0/1381200868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:34 smithi006 bash[33918]: cluster 2023-08-28T00:36:32.828323+0000 mgr.y (mgr.14901) 4739 : cluster [DBG] pgmap v4746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:34 smithi006 bash[33918]: audit 2023-08-28T00:36:33.341367+0000 mon.a (mon.0) 3392 : audit [DBG] from='client.? 172.21.15.6:0/1381200868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:34.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:36:34.245Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:34.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:36:34.247Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:35.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:34 smithi130 bash[51302]: debug 2023-08-28T00:36:34.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:34.681872+0000) 2023-08-28T00:36:36.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:35 smithi130 bash[51302]: debug 2023-08-28T00:36:35.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:35.682132+0000) 2023-08-28T00:36:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:36 smithi130 bash[27579]: cluster 2023-08-28T00:36:34.829435+0000 mgr.y (mgr.14901) 4740 : cluster [DBG] pgmap v4747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:36 smithi130 bash[27579]: audit 2023-08-28T00:36:35.750644+0000 mon.c (mon.2) 1835 : audit [DBG] from='client.? 172.21.15.6:0/3728776067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:36.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:36 smithi006 bash[33918]: cluster 2023-08-28T00:36:34.829435+0000 mgr.y (mgr.14901) 4740 : cluster [DBG] pgmap v4747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:36.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:36 smithi006 bash[33918]: audit 2023-08-28T00:36:35.750644+0000 mon.c (mon.2) 1835 : audit [DBG] from='client.? 172.21.15.6:0/3728776067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:36.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:36 smithi006 bash[27292]: cluster 2023-08-28T00:36:34.829435+0000 mgr.y (mgr.14901) 4740 : cluster [DBG] pgmap v4747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:36.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:36 smithi006 bash[27292]: audit 2023-08-28T00:36:35.750644+0000 mon.c (mon.2) 1835 : audit [DBG] from='client.? 172.21.15.6:0/3728776067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:37.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:36 smithi130 bash[51302]: debug 2023-08-28T00:36:36.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:36.682373+0000) 2023-08-28T00:36:38.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:37 smithi130 bash[51302]: debug 2023-08-28T00:36:37.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:37.682668+0000) 2023-08-28T00:36:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:38 smithi130 bash[27579]: cluster 2023-08-28T00:36:36.830087+0000 mgr.y (mgr.14901) 4741 : cluster [DBG] pgmap v4748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:38 smithi130 bash[27579]: audit 2023-08-28T00:36:38.167941+0000 mon.c (mon.2) 1836 : audit [DBG] from='client.? 172.21.15.6:0/973568216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:38.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:38 smithi006 bash[27292]: cluster 2023-08-28T00:36:36.830087+0000 mgr.y (mgr.14901) 4741 : cluster [DBG] pgmap v4748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:38.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:38 smithi006 bash[27292]: audit 2023-08-28T00:36:38.167941+0000 mon.c (mon.2) 1836 : audit [DBG] from='client.? 172.21.15.6:0/973568216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:38.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:38 smithi006 bash[33918]: cluster 2023-08-28T00:36:36.830087+0000 mgr.y (mgr.14901) 4741 : cluster [DBG] pgmap v4748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:38.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:38 smithi006 bash[33918]: audit 2023-08-28T00:36:38.167941+0000 mon.c (mon.2) 1836 : audit [DBG] from='client.? 172.21.15.6:0/973568216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:39.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:38 smithi130 bash[51302]: debug 2023-08-28T00:36:38.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:38.682902+0000) 2023-08-28T00:36:40.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:39 smithi130 bash[51302]: debug 2023-08-28T00:36:39.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:39.683175+0000) 2023-08-28T00:36:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:40 smithi130 bash[27579]: cluster 2023-08-28T00:36:38.830767+0000 mgr.y (mgr.14901) 4742 : cluster [DBG] pgmap v4749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:40 smithi130 bash[27579]: audit 2023-08-28T00:36:39.811458+0000 mon.b (mon.1) 576 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:36:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:40 smithi006 bash[33918]: cluster 2023-08-28T00:36:38.830767+0000 mgr.y (mgr.14901) 4742 : cluster [DBG] pgmap v4749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:40 smithi006 bash[33918]: audit 2023-08-28T00:36:39.811458+0000 mon.b (mon.1) 576 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:36:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:40 smithi006 bash[27292]: cluster 2023-08-28T00:36:38.830767+0000 mgr.y (mgr.14901) 4742 : cluster [DBG] pgmap v4749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:40 smithi006 bash[27292]: audit 2023-08-28T00:36:39.811458+0000 mon.b (mon.1) 576 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:36:41.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:40 smithi130 bash[51302]: debug 2023-08-28T00:36:40.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:40.683474+0000) 2023-08-28T00:36:41.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:36:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:36:40] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:36:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:36:40.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:36:40.785Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:41.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:41 smithi130 bash[27579]: audit 2023-08-28T00:36:40.591146+0000 mon.c (mon.2) 1837 : audit [DBG] from='client.? 172.21.15.6:0/3747043051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:41.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:41 smithi006 bash[27292]: audit 2023-08-28T00:36:40.591146+0000 mon.c (mon.2) 1837 : audit [DBG] from='client.? 172.21.15.6:0/3747043051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:41.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:41 smithi006 bash[33918]: audit 2023-08-28T00:36:40.591146+0000 mon.c (mon.2) 1837 : audit [DBG] from='client.? 172.21.15.6:0/3747043051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:42.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:41 smithi130 bash[51302]: debug 2023-08-28T00:36:41.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:41.683776+0000) 2023-08-28T00:36:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:42 smithi130 bash[27579]: cluster 2023-08-28T00:36:40.831867+0000 mgr.y (mgr.14901) 4743 : cluster [DBG] pgmap v4750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:42.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:42 smithi006 bash[27292]: cluster 2023-08-28T00:36:40.831867+0000 mgr.y (mgr.14901) 4743 : cluster [DBG] pgmap v4750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:42.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:42 smithi006 bash[33918]: cluster 2023-08-28T00:36:40.831867+0000 mgr.y (mgr.14901) 4743 : cluster [DBG] pgmap v4750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:36:42.363Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:36:42.366Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:43.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:42 smithi130 bash[51302]: debug 2023-08-28T00:36:42.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:42.684074+0000) 2023-08-28T00:36:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:43 smithi130 bash[27579]: audit 2023-08-28T00:36:43.018164+0000 mon.c (mon.2) 1838 : audit [DBG] from='client.? 172.21.15.6:0/2153096135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:43.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:43 smithi006 bash[33918]: audit 2023-08-28T00:36:43.018164+0000 mon.c (mon.2) 1838 : audit [DBG] from='client.? 172.21.15.6:0/2153096135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:43.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:43 smithi006 bash[27292]: audit 2023-08-28T00:36:43.018164+0000 mon.c (mon.2) 1838 : audit [DBG] from='client.? 172.21.15.6:0/2153096135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:44.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:43 smithi130 bash[51302]: debug 2023-08-28T00:36:43.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:43.684387+0000) 2023-08-28T00:36:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:44 smithi130 bash[27579]: cluster 2023-08-28T00:36:42.832493+0000 mgr.y (mgr.14901) 4744 : cluster [DBG] pgmap v4751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:44.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:44 smithi006 bash[33918]: cluster 2023-08-28T00:36:42.832493+0000 mgr.y (mgr.14901) 4744 : cluster [DBG] pgmap v4751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:36:44.245Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:36:44.247Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:44.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:44 smithi006 bash[27292]: cluster 2023-08-28T00:36:42.832493+0000 mgr.y (mgr.14901) 4744 : cluster [DBG] pgmap v4751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:44 smithi130 bash[51302]: debug 2023-08-28T00:36:44.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:44.684631+0000) 2023-08-28T00:36:46.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:45 smithi130 bash[51302]: debug 2023-08-28T00:36:45.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:45.684870+0000) 2023-08-28T00:36:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:46 smithi130 bash[27579]: cluster 2023-08-28T00:36:44.833733+0000 mgr.y (mgr.14901) 4745 : cluster [DBG] pgmap v4752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:46 smithi130 bash[27579]: audit 2023-08-28T00:36:45.455102+0000 mon.c (mon.2) 1839 : audit [DBG] from='client.? 172.21.15.6:0/235578094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:46 smithi130 bash[27579]: audit 2023-08-28T00:36:45.698322+0000 mon.a (mon.0) 3393 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:36:46.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:46 smithi130 bash[27579]: audit 2023-08-28T00:36:46.175842+0000 mon.a (mon.0) 3394 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:36:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:46 smithi006 bash[27292]: cluster 2023-08-28T00:36:44.833733+0000 mgr.y (mgr.14901) 4745 : cluster [DBG] pgmap v4752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:46 smithi006 bash[27292]: audit 2023-08-28T00:36:45.455102+0000 mon.c (mon.2) 1839 : audit [DBG] from='client.? 172.21.15.6:0/235578094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:46 smithi006 bash[27292]: audit 2023-08-28T00:36:45.698322+0000 mon.a (mon.0) 3393 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:36:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:46 smithi006 bash[27292]: audit 2023-08-28T00:36:46.175842+0000 mon.a (mon.0) 3394 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:36:46.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:46 smithi006 bash[33918]: cluster 2023-08-28T00:36:44.833733+0000 mgr.y (mgr.14901) 4745 : cluster [DBG] pgmap v4752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:46.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:46 smithi006 bash[33918]: audit 2023-08-28T00:36:45.455102+0000 mon.c (mon.2) 1839 : audit [DBG] from='client.? 172.21.15.6:0/235578094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:46.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:46 smithi006 bash[33918]: audit 2023-08-28T00:36:45.698322+0000 mon.a (mon.0) 3393 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:36:46.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:46 smithi006 bash[33918]: audit 2023-08-28T00:36:46.175842+0000 mon.a (mon.0) 3394 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:36:47.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:46 smithi130 bash[51302]: debug 2023-08-28T00:36:46.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:46.685202+0000) 2023-08-28T00:36:47.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:47 smithi130 bash[27579]: audit 2023-08-28T00:36:46.444423+0000 mon.b (mon.1) 577 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:36:47.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:47 smithi130 bash[27579]: audit 2023-08-28T00:36:46.446586+0000 mon.b (mon.1) 578 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:36:47.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:47 smithi130 bash[27579]: audit 2023-08-28T00:36:46.454907+0000 mon.a (mon.0) 3395 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:36:47.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:47 smithi006 bash[33918]: audit 2023-08-28T00:36:46.444423+0000 mon.b (mon.1) 577 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:36:47.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:47 smithi006 bash[33918]: audit 2023-08-28T00:36:46.446586+0000 mon.b (mon.1) 578 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:36:47.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:47 smithi006 bash[33918]: audit 2023-08-28T00:36:46.454907+0000 mon.a (mon.0) 3395 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:36:47.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:47 smithi006 bash[27292]: audit 2023-08-28T00:36:46.444423+0000 mon.b (mon.1) 577 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:36:47.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:47 smithi006 bash[27292]: audit 2023-08-28T00:36:46.446586+0000 mon.b (mon.1) 578 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:36:47.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:47 smithi006 bash[27292]: audit 2023-08-28T00:36:46.454907+0000 mon.a (mon.0) 3395 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:36:48.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:47 smithi130 bash[51302]: debug 2023-08-28T00:36:47.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:47.685571+0000) 2023-08-28T00:36:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:48 smithi006 bash[33918]: cluster 2023-08-28T00:36:46.834387+0000 mgr.y (mgr.14901) 4746 : cluster [DBG] pgmap v4753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:48 smithi006 bash[33918]: audit 2023-08-28T00:36:47.866407+0000 mon.c (mon.2) 1840 : audit [DBG] from='client.? 172.21.15.6:0/2478861903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:48.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:48 smithi006 bash[27292]: cluster 2023-08-28T00:36:46.834387+0000 mgr.y (mgr.14901) 4746 : cluster [DBG] pgmap v4753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:48.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:48 smithi006 bash[27292]: audit 2023-08-28T00:36:47.866407+0000 mon.c (mon.2) 1840 : audit [DBG] from='client.? 172.21.15.6:0/2478861903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:48 smithi130 bash[27579]: cluster 2023-08-28T00:36:46.834387+0000 mgr.y (mgr.14901) 4746 : cluster [DBG] pgmap v4753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:48 smithi130 bash[27579]: audit 2023-08-28T00:36:47.866407+0000 mon.c (mon.2) 1840 : audit [DBG] from='client.? 172.21.15.6:0/2478861903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:48 smithi130 bash[51302]: debug 2023-08-28T00:36:48.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:48.685915+0000) 2023-08-28T00:36:50.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:49 smithi130 bash[51302]: debug 2023-08-28T00:36:49.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:49.686249+0000) 2023-08-28T00:36:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:50 smithi006 bash[27292]: cluster 2023-08-28T00:36:48.835067+0000 mgr.y (mgr.14901) 4747 : cluster [DBG] pgmap v4754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:50 smithi006 bash[27292]: audit 2023-08-28T00:36:50.288656+0000 mon.a (mon.0) 3396 : audit [DBG] from='client.? 172.21.15.6:0/2028628305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:50 smithi006 bash[33918]: cluster 2023-08-28T00:36:48.835067+0000 mgr.y (mgr.14901) 4747 : cluster [DBG] pgmap v4754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:50 smithi006 bash[33918]: audit 2023-08-28T00:36:50.288656+0000 mon.a (mon.0) 3396 : audit [DBG] from='client.? 172.21.15.6:0/2028628305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:50.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:50 smithi130 bash[51302]: debug 2023-08-28T00:36:50.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:50.686526+0000) 2023-08-28T00:36:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:50 smithi130 bash[27579]: cluster 2023-08-28T00:36:48.835067+0000 mgr.y (mgr.14901) 4747 : cluster [DBG] pgmap v4754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:50 smithi130 bash[27579]: audit 2023-08-28T00:36:50.288656+0000 mon.a (mon.0) 3396 : audit [DBG] from='client.? 172.21.15.6:0/2028628305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:51.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:36:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:36:50] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:36:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:36:50.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:36:50.785Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:52.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:51 smithi130 bash[51302]: debug 2023-08-28T00:36:51.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:51.686847+0000) 2023-08-28T00:36:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:52 smithi006 bash[33918]: cluster 2023-08-28T00:36:50.836204+0000 mgr.y (mgr.14901) 4748 : cluster [DBG] pgmap v4755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:36:52.364Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:36:52.366Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:52 smithi006 bash[27292]: cluster 2023-08-28T00:36:50.836204+0000 mgr.y (mgr.14901) 4748 : cluster [DBG] pgmap v4755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:52 smithi130 bash[27579]: cluster 2023-08-28T00:36:50.836204+0000 mgr.y (mgr.14901) 4748 : cluster [DBG] pgmap v4755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:52.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:52 smithi130 bash[51302]: debug 2023-08-28T00:36:52.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:52.687229+0000) 2023-08-28T00:36:53.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:53 smithi130 bash[27579]: audit 2023-08-28T00:36:52.698175+0000 mon.a (mon.0) 3397 : audit [DBG] from='client.? 172.21.15.6:0/2348648833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:53.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:53 smithi130 bash[51302]: debug 2023-08-28T00:36:53.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:53.687597+0000) 2023-08-28T00:36:53.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:53 smithi006 bash[33918]: audit 2023-08-28T00:36:52.698175+0000 mon.a (mon.0) 3397 : audit [DBG] from='client.? 172.21.15.6:0/2348648833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:53.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:53 smithi006 bash[27292]: audit 2023-08-28T00:36:52.698175+0000 mon.a (mon.0) 3397 : audit [DBG] from='client.? 172.21.15.6:0/2348648833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:54.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:54 smithi006 bash[33918]: cluster 2023-08-28T00:36:52.836903+0000 mgr.y (mgr.14901) 4749 : cluster [DBG] pgmap v4756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:54.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:54 smithi006 bash[27292]: cluster 2023-08-28T00:36:52.836903+0000 mgr.y (mgr.14901) 4749 : cluster [DBG] pgmap v4756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:36:54.246Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:54.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:36:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:36:54.249Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:36:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:54 smithi130 bash[27579]: cluster 2023-08-28T00:36:52.836903+0000 mgr.y (mgr.14901) 4749 : cluster [DBG] pgmap v4756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:54.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:54 smithi130 bash[51302]: debug 2023-08-28T00:36:54.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:54.687898+0000) 2023-08-28T00:36:55.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:55 smithi130 bash[27579]: audit 2023-08-28T00:36:55.115135+0000 mon.a (mon.0) 3398 : audit [DBG] from='client.? 172.21.15.6:0/1349929135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:55.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:55 smithi130 bash[51302]: debug 2023-08-28T00:36:55.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:55.688196+0000) 2023-08-28T00:36:55.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:55 smithi006 bash[33918]: audit 2023-08-28T00:36:55.115135+0000 mon.a (mon.0) 3398 : audit [DBG] from='client.? 172.21.15.6:0/1349929135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:55.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:55 smithi006 bash[27292]: audit 2023-08-28T00:36:55.115135+0000 mon.a (mon.0) 3398 : audit [DBG] from='client.? 172.21.15.6:0/1349929135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:56 smithi130 bash[27579]: cluster 2023-08-28T00:36:54.837991+0000 mgr.y (mgr.14901) 4750 : cluster [DBG] pgmap v4757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:56.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:56 smithi130 bash[51302]: debug 2023-08-28T00:36:56.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:56.688539+0000) 2023-08-28T00:36:56.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:56 smithi006 bash[27292]: cluster 2023-08-28T00:36:54.837991+0000 mgr.y (mgr.14901) 4750 : cluster [DBG] pgmap v4757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:56.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:56 smithi006 bash[33918]: cluster 2023-08-28T00:36:54.837991+0000 mgr.y (mgr.14901) 4750 : cluster [DBG] pgmap v4757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:58.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:57 smithi130 bash[51302]: debug 2023-08-28T00:36:57.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:57.688901+0000) 2023-08-28T00:36:58.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:58 smithi130 bash[51302]: debug 2023-08-28T00:36:58.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:58.689232+0000) 2023-08-28T00:36:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:58 smithi130 bash[27579]: cluster 2023-08-28T00:36:56.838790+0000 mgr.y (mgr.14901) 4751 : cluster [DBG] pgmap v4758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:58.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:36:58 smithi130 bash[27579]: audit 2023-08-28T00:36:57.526084+0000 mon.a (mon.0) 3399 : audit [DBG] from='client.? 172.21.15.6:0/1241525312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:58.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:58 smithi006 bash[27292]: cluster 2023-08-28T00:36:56.838790+0000 mgr.y (mgr.14901) 4751 : cluster [DBG] pgmap v4758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:58.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:36:58 smithi006 bash[27292]: audit 2023-08-28T00:36:57.526084+0000 mon.a (mon.0) 3399 : audit [DBG] from='client.? 172.21.15.6:0/1241525312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:58.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:58 smithi006 bash[33918]: cluster 2023-08-28T00:36:56.838790+0000 mgr.y (mgr.14901) 4751 : cluster [DBG] pgmap v4758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:36:58.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:36:58 smithi006 bash[33918]: audit 2023-08-28T00:36:57.526084+0000 mon.a (mon.0) 3399 : audit [DBG] from='client.? 172.21.15.6:0/1241525312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:00.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:36:59 smithi130 bash[51302]: debug 2023-08-28T00:36:59.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:59.689567+0000) 2023-08-28T00:37:00.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:00 smithi006 bash[33918]: cluster 2023-08-28T00:36:58.839484+0000 mgr.y (mgr.14901) 4752 : cluster [DBG] pgmap v4759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:00.756 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:00 smithi006 bash[33918]: audit 2023-08-28T00:36:59.941023+0000 mon.a (mon.0) 3400 : audit [DBG] from='client.? 172.21.15.6:0/869195843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:00.756 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:00 smithi006 bash[27292]: cluster 2023-08-28T00:36:58.839484+0000 mgr.y (mgr.14901) 4752 : cluster [DBG] pgmap v4759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:00.756 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:00 smithi006 bash[27292]: audit 2023-08-28T00:36:59.941023+0000 mon.a (mon.0) 3400 : audit [DBG] from='client.? 172.21.15.6:0/869195843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:00.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:00 smithi130 bash[51302]: debug 2023-08-28T00:37:00.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:00.689879+0000) 2023-08-28T00:37:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:00 smithi130 bash[27579]: cluster 2023-08-28T00:36:58.839484+0000 mgr.y (mgr.14901) 4752 : cluster [DBG] pgmap v4759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:00.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:00 smithi130 bash[27579]: audit 2023-08-28T00:36:59.941023+0000 mon.a (mon.0) 3400 : audit [DBG] from='client.? 172.21.15.6:0/869195843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:01.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:37:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:37:00] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:37:01.477 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:37:00.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:01.477 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:37:00.786Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:02.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:01 smithi130 bash[51302]: debug 2023-08-28T00:37:01.687+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:01.690210+0000) 2023-08-28T00:37:02.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:02 smithi006 bash[27292]: cluster 2023-08-28T00:37:00.840670+0000 mgr.y (mgr.14901) 4753 : cluster [DBG] pgmap v4760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:02.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:02 smithi006 bash[27292]: audit 2023-08-28T00:37:02.356913+0000 mon.a (mon.0) 3401 : audit [DBG] from='client.? 172.21.15.6:0/2822330536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:02 smithi006 bash[33918]: cluster 2023-08-28T00:37:00.840670+0000 mgr.y (mgr.14901) 4753 : cluster [DBG] pgmap v4760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:02 smithi006 bash[33918]: audit 2023-08-28T00:37:02.356913+0000 mon.a (mon.0) 3401 : audit [DBG] from='client.? 172.21.15.6:0/2822330536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:02.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:37:02.364Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:02.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:37:02.366Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:02.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:02 smithi130 bash[51302]: debug 2023-08-28T00:37:02.682+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:02.690519+0000) 2023-08-28T00:37:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:02 smithi130 bash[27579]: cluster 2023-08-28T00:37:00.840670+0000 mgr.y (mgr.14901) 4753 : cluster [DBG] pgmap v4760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:02.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:02 smithi130 bash[27579]: audit 2023-08-28T00:37:02.356913+0000 mon.a (mon.0) 3401 : audit [DBG] from='client.? 172.21.15.6:0/2822330536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:04.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:03 smithi130 bash[51302]: debug 2023-08-28T00:37:03.682+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:03.690814+0000) 2023-08-28T00:37:04.517 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:37:04.247Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:04.517 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:37:04.250Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:04 smithi130 bash[27579]: cluster 2023-08-28T00:37:02.841406+0000 mgr.y (mgr.14901) 4754 : cluster [DBG] pgmap v4761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:04.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:04 smithi130 bash[51302]: debug 2023-08-28T00:37:04.690+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:04.691109+0000) 2023-08-28T00:37:04.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:04 smithi006 bash[33918]: cluster 2023-08-28T00:37:02.841406+0000 mgr.y (mgr.14901) 4754 : cluster [DBG] pgmap v4761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:04.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:04 smithi006 bash[27292]: cluster 2023-08-28T00:37:02.841406+0000 mgr.y (mgr.14901) 4754 : cluster [DBG] pgmap v4761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:05.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:05 smithi130 bash[27579]: audit 2023-08-28T00:37:04.771140+0000 mon.a (mon.0) 3402 : audit [DBG] from='client.? 172.21.15.6:0/2597609392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:05.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:05 smithi130 bash[51302]: debug 2023-08-28T00:37:05.690+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:05.691390+0000) 2023-08-28T00:37:05.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:05 smithi006 bash[27292]: audit 2023-08-28T00:37:04.771140+0000 mon.a (mon.0) 3402 : audit [DBG] from='client.? 172.21.15.6:0/2597609392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:05.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:05 smithi006 bash[33918]: audit 2023-08-28T00:37:04.771140+0000 mon.a (mon.0) 3402 : audit [DBG] from='client.? 172.21.15.6:0/2597609392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:06 smithi130 bash[27579]: cluster 2023-08-28T00:37:04.842543+0000 mgr.y (mgr.14901) 4755 : cluster [DBG] pgmap v4762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:06.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:06 smithi130 bash[51302]: debug 2023-08-28T00:37:06.690+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:06.691681+0000) 2023-08-28T00:37:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:06 smithi006 bash[27292]: cluster 2023-08-28T00:37:04.842543+0000 mgr.y (mgr.14901) 4755 : cluster [DBG] pgmap v4762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:06.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:06 smithi006 bash[33918]: cluster 2023-08-28T00:37:04.842543+0000 mgr.y (mgr.14901) 4755 : cluster [DBG] pgmap v4762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:07.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:07 smithi130 bash[27579]: audit 2023-08-28T00:37:07.186283+0000 mon.c (mon.2) 1841 : audit [DBG] from='client.? 172.21.15.6:0/1262054890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:07.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:07 smithi130 bash[51302]: debug 2023-08-28T00:37:07.686+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:07.692029+0000) 2023-08-28T00:37:07.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:07 smithi006 bash[27292]: audit 2023-08-28T00:37:07.186283+0000 mon.c (mon.2) 1841 : audit [DBG] from='client.? 172.21.15.6:0/1262054890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:07.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:07 smithi006 bash[33918]: audit 2023-08-28T00:37:07.186283+0000 mon.c (mon.2) 1841 : audit [DBG] from='client.? 172.21.15.6:0/1262054890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:08 smithi130 bash[27579]: cluster 2023-08-28T00:37:06.843158+0000 mgr.y (mgr.14901) 4756 : cluster [DBG] pgmap v4763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:08.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:08 smithi130 bash[51302]: debug 2023-08-28T00:37:08.690+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:08.692350+0000) 2023-08-28T00:37:08.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:08 smithi006 bash[27292]: cluster 2023-08-28T00:37:06.843158+0000 mgr.y (mgr.14901) 4756 : cluster [DBG] pgmap v4763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:08.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:08 smithi006 bash[33918]: cluster 2023-08-28T00:37:06.843158+0000 mgr.y (mgr.14901) 4756 : cluster [DBG] pgmap v4763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:10.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:09 smithi130 bash[51302]: debug 2023-08-28T00:37:09.690+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:09.692646+0000) 2023-08-28T00:37:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:10 smithi130 bash[51302]: debug 2023-08-28T00:37:10.690+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:10.692955+0000) 2023-08-28T00:37:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:10 smithi130 bash[27579]: cluster 2023-08-28T00:37:08.843817+0000 mgr.y (mgr.14901) 4757 : cluster [DBG] pgmap v4764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:10 smithi130 bash[27579]: audit 2023-08-28T00:37:09.602400+0000 mon.a (mon.0) 3403 : audit [DBG] from='client.? 172.21.15.6:0/132821708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:10.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:37:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:37:10] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:37:10.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:10 smithi006 bash[27292]: cluster 2023-08-28T00:37:08.843817+0000 mgr.y (mgr.14901) 4757 : cluster [DBG] pgmap v4764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:10.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:10 smithi006 bash[27292]: audit 2023-08-28T00:37:09.602400+0000 mon.a (mon.0) 3403 : audit [DBG] from='client.? 172.21.15.6:0/132821708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:10.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:10 smithi006 bash[33918]: cluster 2023-08-28T00:37:08.843817+0000 mgr.y (mgr.14901) 4757 : cluster [DBG] pgmap v4764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:10.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:10 smithi006 bash[33918]: audit 2023-08-28T00:37:09.602400+0000 mon.a (mon.0) 3403 : audit [DBG] from='client.? 172.21.15.6:0/132821708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:10.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:37:10.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:10.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:37:10.786Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:12.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:11 smithi130 bash[51302]: debug 2023-08-28T00:37:11.690+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:11.693234+0000) 2023-08-28T00:37:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:12 smithi006 bash[27292]: cluster 2023-08-28T00:37:10.844980+0000 mgr.y (mgr.14901) 4758 : cluster [DBG] pgmap v4765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:12 smithi006 bash[27292]: audit 2023-08-28T00:37:12.027712+0000 mon.c (mon.2) 1842 : audit [DBG] from='client.? 172.21.15.6:0/470829220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:12 smithi006 bash[33918]: cluster 2023-08-28T00:37:10.844980+0000 mgr.y (mgr.14901) 4758 : cluster [DBG] pgmap v4765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:12 smithi006 bash[33918]: audit 2023-08-28T00:37:12.027712+0000 mon.c (mon.2) 1842 : audit [DBG] from='client.? 172.21.15.6:0/470829220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:12.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:37:12.365Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:12.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:37:12.367Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:12 smithi130 bash[27579]: cluster 2023-08-28T00:37:10.844980+0000 mgr.y (mgr.14901) 4758 : cluster [DBG] pgmap v4765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:12 smithi130 bash[27579]: audit 2023-08-28T00:37:12.027712+0000 mon.c (mon.2) 1842 : audit [DBG] from='client.? 172.21.15.6:0/470829220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:12.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:12 smithi130 bash[51302]: debug 2023-08-28T00:37:12.690+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:12.693516+0000) 2023-08-28T00:37:14.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:13 smithi130 bash[51302]: debug 2023-08-28T00:37:13.690+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:13.693810+0000) 2023-08-28T00:37:14.555 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:37:14.248Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:14.555 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:37:14.250Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:14 smithi130 bash[27579]: cluster 2023-08-28T00:37:12.845684+0000 mgr.y (mgr.14901) 4759 : cluster [DBG] pgmap v4766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:14 smithi130 bash[27579]: audit 2023-08-28T00:37:14.438959+0000 mon.a (mon.0) 3404 : audit [DBG] from='client.? 172.21.15.6:0/4117895802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:14.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:14 smithi130 bash[51302]: debug 2023-08-28T00:37:14.690+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:14.694078+0000) 2023-08-28T00:37:14.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:14 smithi006 bash[33918]: cluster 2023-08-28T00:37:12.845684+0000 mgr.y (mgr.14901) 4759 : cluster [DBG] pgmap v4766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:14.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:14 smithi006 bash[33918]: audit 2023-08-28T00:37:14.438959+0000 mon.a (mon.0) 3404 : audit [DBG] from='client.? 172.21.15.6:0/4117895802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:14.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:14 smithi006 bash[27292]: cluster 2023-08-28T00:37:12.845684+0000 mgr.y (mgr.14901) 4759 : cluster [DBG] pgmap v4766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:14.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:14 smithi006 bash[27292]: audit 2023-08-28T00:37:14.438959+0000 mon.a (mon.0) 3404 : audit [DBG] from='client.? 172.21.15.6:0/4117895802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:16.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:15 smithi130 bash[51302]: debug 2023-08-28T00:37:15.690+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:15.694369+0000) 2023-08-28T00:37:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:16 smithi130 bash[27579]: cluster 2023-08-28T00:37:14.846829+0000 mgr.y (mgr.14901) 4760 : cluster [DBG] pgmap v4767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:16.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:16 smithi130 bash[51302]: debug 2023-08-28T00:37:16.690+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:16.694656+0000) 2023-08-28T00:37:16.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:16 smithi006 bash[33918]: cluster 2023-08-28T00:37:14.846829+0000 mgr.y (mgr.14901) 4760 : cluster [DBG] pgmap v4767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:16.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:16 smithi006 bash[27292]: cluster 2023-08-28T00:37:14.846829+0000 mgr.y (mgr.14901) 4760 : cluster [DBG] pgmap v4767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:17.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:17 smithi130 bash[27579]: audit 2023-08-28T00:37:16.851951+0000 mon.a (mon.0) 3405 : audit [DBG] from='client.? 172.21.15.6:0/1490170460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:17.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:17 smithi130 bash[51302]: debug 2023-08-28T00:37:17.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:17.694998+0000) 2023-08-28T00:37:17.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:17 smithi006 bash[27292]: audit 2023-08-28T00:37:16.851951+0000 mon.a (mon.0) 3405 : audit [DBG] from='client.? 172.21.15.6:0/1490170460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:17.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:17 smithi006 bash[33918]: audit 2023-08-28T00:37:16.851951+0000 mon.a (mon.0) 3405 : audit [DBG] from='client.? 172.21.15.6:0/1490170460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:18 smithi130 bash[27579]: cluster 2023-08-28T00:37:16.847452+0000 mgr.y (mgr.14901) 4761 : cluster [DBG] pgmap v4768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:18 smithi130 bash[51302]: debug 2023-08-28T00:37:18.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:18.695278+0000) 2023-08-28T00:37:18.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:18 smithi006 bash[33918]: cluster 2023-08-28T00:37:16.847452+0000 mgr.y (mgr.14901) 4761 : cluster [DBG] pgmap v4768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:18.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:18 smithi006 bash[27292]: cluster 2023-08-28T00:37:16.847452+0000 mgr.y (mgr.14901) 4761 : cluster [DBG] pgmap v4768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:19.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:19 smithi130 bash[51302]: debug 2023-08-28T00:37:19.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:19.695628+0000) 2023-08-28T00:37:19.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:19 smithi130 bash[27579]: audit 2023-08-28T00:37:19.262943+0000 mon.c (mon.2) 1843 : audit [DBG] from='client.? 172.21.15.6:0/3657972568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:19.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:19 smithi006 bash[27292]: audit 2023-08-28T00:37:19.262943+0000 mon.c (mon.2) 1843 : audit [DBG] from='client.? 172.21.15.6:0/3657972568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:19.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:19 smithi006 bash[33918]: audit 2023-08-28T00:37:19.262943+0000 mon.c (mon.2) 1843 : audit [DBG] from='client.? 172.21.15.6:0/3657972568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:20 smithi130 bash[27579]: cluster 2023-08-28T00:37:18.848145+0000 mgr.y (mgr.14901) 4762 : cluster [DBG] pgmap v4769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:20.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:20 smithi130 bash[51302]: debug 2023-08-28T00:37:20.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:20.695928+0000) 2023-08-28T00:37:20.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:37:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:37:20] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T00:37:20.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:20 smithi006 bash[27292]: cluster 2023-08-28T00:37:18.848145+0000 mgr.y (mgr.14901) 4762 : cluster [DBG] pgmap v4769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:20.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:20 smithi006 bash[33918]: cluster 2023-08-28T00:37:18.848145+0000 mgr.y (mgr.14901) 4762 : cluster [DBG] pgmap v4769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:20.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:37:20.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:20.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:37:20.787Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:22.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:21 smithi130 bash[51302]: debug 2023-08-28T00:37:21.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:21.696216+0000) 2023-08-28T00:37:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:22 smithi006 bash[33918]: cluster 2023-08-28T00:37:20.849326+0000 mgr.y (mgr.14901) 4763 : cluster [DBG] pgmap v4770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:22 smithi006 bash[33918]: audit 2023-08-28T00:37:21.678788+0000 mon.a (mon.0) 3406 : audit [DBG] from='client.? 172.21.15.6:0/2821260432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:37:22.366Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:22.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:37:22.368Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:22.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:22 smithi006 bash[27292]: cluster 2023-08-28T00:37:20.849326+0000 mgr.y (mgr.14901) 4763 : cluster [DBG] pgmap v4770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:22.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:22 smithi006 bash[27292]: audit 2023-08-28T00:37:21.678788+0000 mon.a (mon.0) 3406 : audit [DBG] from='client.? 172.21.15.6:0/2821260432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:22 smithi130 bash[27579]: cluster 2023-08-28T00:37:20.849326+0000 mgr.y (mgr.14901) 4763 : cluster [DBG] pgmap v4770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:22 smithi130 bash[27579]: audit 2023-08-28T00:37:21.678788+0000 mon.a (mon.0) 3406 : audit [DBG] from='client.? 172.21.15.6:0/2821260432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:22.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:22 smithi130 bash[51302]: debug 2023-08-28T00:37:22.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:22.696485+0000) 2023-08-28T00:37:24.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:23 smithi130 bash[51302]: debug 2023-08-28T00:37:23.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:23.696763+0000) 2023-08-28T00:37:24.594 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:37:24.248Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:24.594 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:37:24.252Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:24 smithi130 bash[27579]: cluster 2023-08-28T00:37:22.850019+0000 mgr.y (mgr.14901) 4764 : cluster [DBG] pgmap v4771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:24 smithi130 bash[27579]: audit 2023-08-28T00:37:24.100441+0000 mon.c (mon.2) 1844 : audit [DBG] from='client.? 172.21.15.6:0/1643241324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:24 smithi130 bash[51302]: debug 2023-08-28T00:37:24.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:24.697065+0000) 2023-08-28T00:37:24.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:24 smithi006 bash[33918]: cluster 2023-08-28T00:37:22.850019+0000 mgr.y (mgr.14901) 4764 : cluster [DBG] pgmap v4771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:24.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:24 smithi006 bash[33918]: audit 2023-08-28T00:37:24.100441+0000 mon.c (mon.2) 1844 : audit [DBG] from='client.? 172.21.15.6:0/1643241324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:24.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:24 smithi006 bash[27292]: cluster 2023-08-28T00:37:22.850019+0000 mgr.y (mgr.14901) 4764 : cluster [DBG] pgmap v4771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:24.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:24 smithi006 bash[27292]: audit 2023-08-28T00:37:24.100441+0000 mon.c (mon.2) 1844 : audit [DBG] from='client.? 172.21.15.6:0/1643241324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:25.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:25 smithi006 bash[27292]: cluster 2023-08-28T00:37:24.851199+0000 mgr.y (mgr.14901) 4765 : cluster [DBG] pgmap v4772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:25.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:25 smithi006 bash[33918]: cluster 2023-08-28T00:37:24.851199+0000 mgr.y (mgr.14901) 4765 : cluster [DBG] pgmap v4772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:25 smithi130 bash[27579]: cluster 2023-08-28T00:37:24.851199+0000 mgr.y (mgr.14901) 4765 : cluster [DBG] pgmap v4772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:26.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:25 smithi130 bash[51302]: debug 2023-08-28T00:37:25.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:25.697337+0000) 2023-08-28T00:37:27.002 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:26 smithi130 bash[51302]: debug 2023-08-28T00:37:26.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:26.697608+0000) 2023-08-28T00:37:27.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:27 smithi130 bash[27579]: audit 2023-08-28T00:37:26.515730+0000 mon.c (mon.2) 1845 : audit [DBG] from='client.? 172.21.15.6:0/338214250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:27.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:26 smithi006 bash[33918]: audit 2023-08-28T00:37:26.515730+0000 mon.c (mon.2) 1845 : audit [DBG] from='client.? 172.21.15.6:0/338214250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:27.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:26 smithi006 bash[27292]: audit 2023-08-28T00:37:26.515730+0000 mon.c (mon.2) 1845 : audit [DBG] from='client.? 172.21.15.6:0/338214250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:28.003 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:27 smithi130 bash[51302]: debug 2023-08-28T00:37:27.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:27.697890+0000) 2023-08-28T00:37:28.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:28 smithi130 bash[27579]: cluster 2023-08-28T00:37:26.851871+0000 mgr.y (mgr.14901) 4766 : cluster [DBG] pgmap v4773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:28.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:28 smithi006 bash[33918]: cluster 2023-08-28T00:37:26.851871+0000 mgr.y (mgr.14901) 4766 : cluster [DBG] pgmap v4773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:28.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:27 smithi006 bash[27292]: cluster 2023-08-28T00:37:26.851871+0000 mgr.y (mgr.14901) 4766 : cluster [DBG] pgmap v4773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:29.014 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:28 smithi130 bash[51302]: debug 2023-08-28T00:37:28.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:28.698147+0000) 2023-08-28T00:37:29.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:29 smithi130 bash[27579]: audit 2023-08-28T00:37:28.949983+0000 mon.a (mon.0) 3407 : audit [DBG] from='client.? 172.21.15.6:0/148286454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:29.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:29 smithi006 bash[33918]: audit 2023-08-28T00:37:28.949983+0000 mon.a (mon.0) 3407 : audit [DBG] from='client.? 172.21.15.6:0/148286454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:29.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:29 smithi006 bash[27292]: audit 2023-08-28T00:37:28.949983+0000 mon.a (mon.0) 3407 : audit [DBG] from='client.? 172.21.15.6:0/148286454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:30.015 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:29 smithi130 bash[51302]: debug 2023-08-28T00:37:29.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:29.698485+0000) 2023-08-28T00:37:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:30 smithi130 bash[27579]: cluster 2023-08-28T00:37:28.852480+0000 mgr.y (mgr.14901) 4767 : cluster [DBG] pgmap v4774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:30.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:30 smithi006 bash[27292]: cluster 2023-08-28T00:37:28.852480+0000 mgr.y (mgr.14901) 4767 : cluster [DBG] pgmap v4774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:30.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:30 smithi006 bash[33918]: cluster 2023-08-28T00:37:28.852480+0000 mgr.y (mgr.14901) 4767 : cluster [DBG] pgmap v4774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:31.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:30 smithi130 bash[51302]: debug 2023-08-28T00:37:30.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:30.698767+0000) 2023-08-28T00:37:31.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:37:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:37:30] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:37:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:37:30.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:37:30.788Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:32.020 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:31 smithi130 bash[51302]: debug 2023-08-28T00:37:31.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:31.699137+0000) 2023-08-28T00:37:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:32 smithi130 bash[27579]: cluster 2023-08-28T00:37:30.853344+0000 mgr.y (mgr.14901) 4768 : cluster [DBG] pgmap v4775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:32 smithi130 bash[27579]: audit 2023-08-28T00:37:31.381865+0000 mon.c (mon.2) 1846 : audit [DBG] from='client.? 172.21.15.6:0/2270594841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:32.370 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:32 smithi006 bash[27292]: cluster 2023-08-28T00:37:30.853344+0000 mgr.y (mgr.14901) 4768 : cluster [DBG] pgmap v4775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:32.370 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:32 smithi006 bash[27292]: audit 2023-08-28T00:37:31.381865+0000 mon.c (mon.2) 1846 : audit [DBG] from='client.? 172.21.15.6:0/2270594841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:32.370 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:32 smithi006 bash[33918]: cluster 2023-08-28T00:37:30.853344+0000 mgr.y (mgr.14901) 4768 : cluster [DBG] pgmap v4775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:32.370 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:32 smithi006 bash[33918]: audit 2023-08-28T00:37:31.381865+0000 mon.c (mon.2) 1846 : audit [DBG] from='client.? 172.21.15.6:0/2270594841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:37:32.366Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:37:32.368Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:33.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:32 smithi130 bash[51302]: debug 2023-08-28T00:37:32.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:32.699447+0000) 2023-08-28T00:37:34.025 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:33 smithi130 bash[51302]: debug 2023-08-28T00:37:33.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:33.699724+0000) 2023-08-28T00:37:34.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:34 smithi130 bash[27579]: cluster 2023-08-28T00:37:32.853955+0000 mgr.y (mgr.14901) 4769 : cluster [DBG] pgmap v4776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:34.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:34 smithi130 bash[27579]: audit 2023-08-28T00:37:33.799815+0000 mon.c (mon.2) 1847 : audit [DBG] from='client.? 172.21.15.6:0/698943014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:34.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:34 smithi006 bash[33918]: cluster 2023-08-28T00:37:32.853955+0000 mgr.y (mgr.14901) 4769 : cluster [DBG] pgmap v4776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:34.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:34 smithi006 bash[33918]: audit 2023-08-28T00:37:33.799815+0000 mon.c (mon.2) 1847 : audit [DBG] from='client.? 172.21.15.6:0/698943014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:37:34.249Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:37:34.251Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:34.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:34 smithi006 bash[27292]: cluster 2023-08-28T00:37:32.853955+0000 mgr.y (mgr.14901) 4769 : cluster [DBG] pgmap v4776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:34.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:34 smithi006 bash[27292]: audit 2023-08-28T00:37:33.799815+0000 mon.c (mon.2) 1847 : audit [DBG] from='client.? 172.21.15.6:0/698943014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:35.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:34 smithi130 bash[51302]: debug 2023-08-28T00:37:34.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:34.700035+0000) 2023-08-28T00:37:36.033 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:35 smithi130 bash[51302]: debug 2023-08-28T00:37:35.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:35.700347+0000) 2023-08-28T00:37:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:36 smithi130 bash[27579]: cluster 2023-08-28T00:37:34.855059+0000 mgr.y (mgr.14901) 4770 : cluster [DBG] pgmap v4777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:36.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:36 smithi006 bash[27292]: cluster 2023-08-28T00:37:34.855059+0000 mgr.y (mgr.14901) 4770 : cluster [DBG] pgmap v4777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:36.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:36 smithi006 bash[33918]: cluster 2023-08-28T00:37:34.855059+0000 mgr.y (mgr.14901) 4770 : cluster [DBG] pgmap v4777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:37.043 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:36 smithi130 bash[51302]: debug 2023-08-28T00:37:36.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:36.700631+0000) 2023-08-28T00:37:37.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:37 smithi130 bash[27579]: audit 2023-08-28T00:37:36.218366+0000 mon.c (mon.2) 1848 : audit [DBG] from='client.? 172.21.15.6:0/1365900417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:37.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:37 smithi006 bash[33918]: audit 2023-08-28T00:37:36.218366+0000 mon.c (mon.2) 1848 : audit [DBG] from='client.? 172.21.15.6:0/1365900417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:37.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:37 smithi006 bash[27292]: audit 2023-08-28T00:37:36.218366+0000 mon.c (mon.2) 1848 : audit [DBG] from='client.? 172.21.15.6:0/1365900417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:38.049 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:37 smithi130 bash[51302]: debug 2023-08-28T00:37:37.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:37.700893+0000) 2023-08-28T00:37:38.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:38 smithi130 bash[27579]: cluster 2023-08-28T00:37:36.855699+0000 mgr.y (mgr.14901) 4771 : cluster [DBG] pgmap v4778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:38.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:38 smithi006 bash[33918]: cluster 2023-08-28T00:37:36.855699+0000 mgr.y (mgr.14901) 4771 : cluster [DBG] pgmap v4778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:38.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:38 smithi006 bash[27292]: cluster 2023-08-28T00:37:36.855699+0000 mgr.y (mgr.14901) 4771 : cluster [DBG] pgmap v4778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:39.057 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:38 smithi130 bash[51302]: debug 2023-08-28T00:37:38.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:38.701179+0000) 2023-08-28T00:37:39.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:39 smithi130 bash[27579]: audit 2023-08-28T00:37:38.634238+0000 mon.a (mon.0) 3408 : audit [DBG] from='client.? 172.21.15.6:0/1452964545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:39.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:39 smithi006 bash[27292]: audit 2023-08-28T00:37:38.634238+0000 mon.a (mon.0) 3408 : audit [DBG] from='client.? 172.21.15.6:0/1452964545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:39.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:39 smithi006 bash[33918]: audit 2023-08-28T00:37:38.634238+0000 mon.a (mon.0) 3408 : audit [DBG] from='client.? 172.21.15.6:0/1452964545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:40.060 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:39 smithi130 bash[51302]: debug 2023-08-28T00:37:39.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:39.701495+0000) 2023-08-28T00:37:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:40 smithi130 bash[27579]: cluster 2023-08-28T00:37:38.856352+0000 mgr.y (mgr.14901) 4772 : cluster [DBG] pgmap v4779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:40.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:40 smithi006 bash[27292]: cluster 2023-08-28T00:37:38.856352+0000 mgr.y (mgr.14901) 4772 : cluster [DBG] pgmap v4779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:40.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:40 smithi006 bash[33918]: cluster 2023-08-28T00:37:38.856352+0000 mgr.y (mgr.14901) 4772 : cluster [DBG] pgmap v4779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:41.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:40 smithi130 bash[51302]: debug 2023-08-28T00:37:40.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:40.701751+0000) 2023-08-28T00:37:41.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:37:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:37:40] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:37:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:37:40.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:37:40.788Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:41.998 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:41 smithi130 bash[51302]: debug 2023-08-28T00:37:41.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:41.702116+0000) 2023-08-28T00:37:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:41 smithi130 bash[27579]: cluster 2023-08-28T00:37:40.857490+0000 mgr.y (mgr.14901) 4773 : cluster [DBG] pgmap v4780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:41 smithi130 bash[27579]: audit 2023-08-28T00:37:41.051616+0000 mon.c (mon.2) 1849 : audit [DBG] from='client.? 172.21.15.6:0/965322821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:42.371 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:41 smithi006 bash[33918]: cluster 2023-08-28T00:37:40.857490+0000 mgr.y (mgr.14901) 4773 : cluster [DBG] pgmap v4780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:42.371 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:41 smithi006 bash[33918]: audit 2023-08-28T00:37:41.051616+0000 mon.c (mon.2) 1849 : audit [DBG] from='client.? 172.21.15.6:0/965322821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:42.371 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:41 smithi006 bash[27292]: cluster 2023-08-28T00:37:40.857490+0000 mgr.y (mgr.14901) 4773 : cluster [DBG] pgmap v4780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:42.371 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:41 smithi006 bash[27292]: audit 2023-08-28T00:37:41.051616+0000 mon.c (mon.2) 1849 : audit [DBG] from='client.? 172.21.15.6:0/965322821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:37:42.367Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:37:42.369Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:43.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:42 smithi130 bash[51302]: debug 2023-08-28T00:37:42.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:42.702335+0000) 2023-08-28T00:37:44.005 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:43 smithi130 bash[51302]: debug 2023-08-28T00:37:43.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:43.702608+0000) 2023-08-28T00:37:44.255 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:37:44.249Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:44.256 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:44 smithi006 bash[33918]: cluster 2023-08-28T00:37:42.858146+0000 mgr.y (mgr.14901) 4774 : cluster [DBG] pgmap v4781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:44.256 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:44 smithi006 bash[33918]: audit 2023-08-28T00:37:43.463295+0000 mon.c (mon.2) 1850 : audit [DBG] from='client.? 172.21.15.6:0/707576616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:44.256 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:44 smithi006 bash[27292]: cluster 2023-08-28T00:37:42.858146+0000 mgr.y (mgr.14901) 4774 : cluster [DBG] pgmap v4781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:44.256 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:44 smithi006 bash[27292]: audit 2023-08-28T00:37:43.463295+0000 mon.c (mon.2) 1850 : audit [DBG] from='client.? 172.21.15.6:0/707576616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:44 smithi130 bash[27579]: cluster 2023-08-28T00:37:42.858146+0000 mgr.y (mgr.14901) 4774 : cluster [DBG] pgmap v4781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:44 smithi130 bash[27579]: audit 2023-08-28T00:37:43.463295+0000 mon.c (mon.2) 1850 : audit [DBG] from='client.? 172.21.15.6:0/707576616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:37:44.251Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:44 smithi130 bash[51302]: debug 2023-08-28T00:37:44.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:44.702908+0000) 2023-08-28T00:37:46.011 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:45 smithi130 bash[51302]: debug 2023-08-28T00:37:45.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:45.703180+0000) 2023-08-28T00:37:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:46 smithi130 bash[27579]: cluster 2023-08-28T00:37:44.859221+0000 mgr.y (mgr.14901) 4775 : cluster [DBG] pgmap v4782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:46 smithi130 bash[27579]: audit 2023-08-28T00:37:45.879566+0000 mon.c (mon.2) 1851 : audit [DBG] from='client.? 172.21.15.6:0/1049442758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:46.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:46 smithi006 bash[27292]: cluster 2023-08-28T00:37:44.859221+0000 mgr.y (mgr.14901) 4775 : cluster [DBG] pgmap v4782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:46.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:46 smithi006 bash[27292]: audit 2023-08-28T00:37:45.879566+0000 mon.c (mon.2) 1851 : audit [DBG] from='client.? 172.21.15.6:0/1049442758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:46.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:46 smithi006 bash[33918]: cluster 2023-08-28T00:37:44.859221+0000 mgr.y (mgr.14901) 4775 : cluster [DBG] pgmap v4782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:46.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:46 smithi006 bash[33918]: audit 2023-08-28T00:37:45.879566+0000 mon.c (mon.2) 1851 : audit [DBG] from='client.? 172.21.15.6:0/1049442758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:47.019 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:46 smithi130 bash[51302]: debug 2023-08-28T00:37:46.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:46.703394+0000) 2023-08-28T00:37:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:47 smithi130 bash[27579]: audit 2023-08-28T00:37:46.466235+0000 mon.b (mon.1) 579 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:37:47.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:47 smithi006 bash[33918]: audit 2023-08-28T00:37:46.466235+0000 mon.b (mon.1) 579 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:37:47.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:47 smithi006 bash[27292]: audit 2023-08-28T00:37:46.466235+0000 mon.b (mon.1) 579 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:37:48.022 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:47 smithi130 bash[51302]: debug 2023-08-28T00:37:47.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:47.703608+0000) 2023-08-28T00:37:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:48 smithi130 bash[27579]: cluster 2023-08-28T00:37:46.859895+0000 mgr.y (mgr.14901) 4776 : cluster [DBG] pgmap v4783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:48.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:48 smithi006 bash[33918]: cluster 2023-08-28T00:37:46.859895+0000 mgr.y (mgr.14901) 4776 : cluster [DBG] pgmap v4783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:48.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:48 smithi006 bash[27292]: cluster 2023-08-28T00:37:46.859895+0000 mgr.y (mgr.14901) 4776 : cluster [DBG] pgmap v4783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:49.029 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:48 smithi130 bash[51302]: debug 2023-08-28T00:37:48.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:48.703904+0000) 2023-08-28T00:37:49.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:49 smithi130 bash[27579]: audit 2023-08-28T00:37:48.289822+0000 mon.c (mon.2) 1852 : audit [DBG] from='client.? 172.21.15.6:0/1087502779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:49.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:49 smithi006 bash[27292]: audit 2023-08-28T00:37:48.289822+0000 mon.c (mon.2) 1852 : audit [DBG] from='client.? 172.21.15.6:0/1087502779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:49.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:49 smithi006 bash[33918]: audit 2023-08-28T00:37:48.289822+0000 mon.c (mon.2) 1852 : audit [DBG] from='client.? 172.21.15.6:0/1087502779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:50.032 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:49 smithi130 bash[51302]: debug 2023-08-28T00:37:49.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:49.704218+0000) 2023-08-28T00:37:50.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:50 smithi130 bash[27579]: cluster 2023-08-28T00:37:48.860679+0000 mgr.y (mgr.14901) 4777 : cluster [DBG] pgmap v4784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:50.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:50 smithi006 bash[33918]: cluster 2023-08-28T00:37:48.860679+0000 mgr.y (mgr.14901) 4777 : cluster [DBG] pgmap v4784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:50.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:50 smithi006 bash[27292]: cluster 2023-08-28T00:37:48.860679+0000 mgr.y (mgr.14901) 4777 : cluster [DBG] pgmap v4784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:51.039 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:37:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:37:50] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:37:51.039 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:37:50.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:51.039 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:37:50.789Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:51.039 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:50 smithi130 bash[51302]: debug 2023-08-28T00:37:50.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:50.704529+0000) 2023-08-28T00:37:51.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:51 smithi130 bash[27579]: audit 2023-08-28T00:37:50.717595+0000 mon.c (mon.2) 1853 : audit [DBG] from='client.? 172.21.15.6:0/1619212003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:51.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:51 smithi006 bash[33918]: audit 2023-08-28T00:37:50.717595+0000 mon.c (mon.2) 1853 : audit [DBG] from='client.? 172.21.15.6:0/1619212003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:51.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:51 smithi006 bash[27292]: audit 2023-08-28T00:37:50.717595+0000 mon.c (mon.2) 1853 : audit [DBG] from='client.? 172.21.15.6:0/1619212003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:52.049 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:51 smithi130 bash[51302]: debug 2023-08-28T00:37:51.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:51.704745+0000) 2023-08-28T00:37:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:52 smithi130 bash[27579]: cluster 2023-08-28T00:37:50.861828+0000 mgr.y (mgr.14901) 4778 : cluster [DBG] pgmap v4785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:52.372 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:52 smithi006 bash[33918]: cluster 2023-08-28T00:37:50.861828+0000 mgr.y (mgr.14901) 4778 : cluster [DBG] pgmap v4785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:52.372 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:52 smithi006 bash[27292]: cluster 2023-08-28T00:37:50.861828+0000 mgr.y (mgr.14901) 4778 : cluster [DBG] pgmap v4785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:37:52.368Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:37:52.370Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:52 smithi130 bash[51302]: debug 2023-08-28T00:37:52.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:52.704978+0000) 2023-08-28T00:37:53.707 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:53 smithi130 bash[27579]: audit 2023-08-28T00:37:52.386977+0000 mon.a (mon.0) 3409 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:37:53.707 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:53 smithi130 bash[27579]: audit 2023-08-28T00:37:52.780874+0000 mon.a (mon.0) 3410 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:37:53.707 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:53 smithi130 bash[27579]: audit 2023-08-28T00:37:53.044448+0000 mon.b (mon.1) 580 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:37:53.707 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:53 smithi130 bash[27579]: audit 2023-08-28T00:37:53.046288+0000 mon.b (mon.1) 581 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:37:53.707 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:53 smithi130 bash[27579]: audit 2023-08-28T00:37:53.053197+0000 mon.a (mon.0) 3411 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:37:53.707 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:53 smithi130 bash[27579]: audit 2023-08-28T00:37:53.146982+0000 mon.a (mon.0) 3412 : audit [DBG] from='client.? 172.21.15.6:0/2858672427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:53 smithi006 bash[33918]: audit 2023-08-28T00:37:52.386977+0000 mon.a (mon.0) 3409 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:37:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:53 smithi006 bash[33918]: audit 2023-08-28T00:37:52.780874+0000 mon.a (mon.0) 3410 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:37:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:53 smithi006 bash[33918]: audit 2023-08-28T00:37:53.044448+0000 mon.b (mon.1) 580 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:37:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:53 smithi006 bash[33918]: audit 2023-08-28T00:37:53.046288+0000 mon.b (mon.1) 581 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:37:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:53 smithi006 bash[33918]: audit 2023-08-28T00:37:53.053197+0000 mon.a (mon.0) 3411 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:37:53.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:53 smithi006 bash[33918]: audit 2023-08-28T00:37:53.146982+0000 mon.a (mon.0) 3412 : audit [DBG] from='client.? 172.21.15.6:0/2858672427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:53.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:53 smithi006 bash[27292]: audit 2023-08-28T00:37:52.386977+0000 mon.a (mon.0) 3409 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:37:53.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:53 smithi006 bash[27292]: audit 2023-08-28T00:37:52.780874+0000 mon.a (mon.0) 3410 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:37:53.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:53 smithi006 bash[27292]: audit 2023-08-28T00:37:53.044448+0000 mon.b (mon.1) 580 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:37:53.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:53 smithi006 bash[27292]: audit 2023-08-28T00:37:53.046288+0000 mon.b (mon.1) 581 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:37:53.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:53 smithi006 bash[27292]: audit 2023-08-28T00:37:53.053197+0000 mon.a (mon.0) 3411 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:37:53.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:53 smithi006 bash[27292]: audit 2023-08-28T00:37:53.146982+0000 mon.a (mon.0) 3412 : audit [DBG] from='client.? 172.21.15.6:0/2858672427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:54.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:53 smithi130 bash[51302]: debug 2023-08-28T00:37:53.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:53.705241+0000) 2023-08-28T00:37:54.707 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:54 smithi130 bash[27579]: cluster 2023-08-28T00:37:52.862412+0000 mgr.y (mgr.14901) 4779 : cluster [DBG] pgmap v4786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:54.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:54 smithi006 bash[27292]: cluster 2023-08-28T00:37:52.862412+0000 mgr.y (mgr.14901) 4779 : cluster [DBG] pgmap v4786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:54.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:54 smithi006 bash[33918]: cluster 2023-08-28T00:37:52.862412+0000 mgr.y (mgr.14901) 4779 : cluster [DBG] pgmap v4786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:37:54.249Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:37:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:37:54.252Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:37:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:54 smithi130 bash[51302]: debug 2023-08-28T00:37:54.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:54.705542+0000) 2023-08-28T00:37:56.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:55 smithi130 bash[51302]: debug 2023-08-28T00:37:55.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:55.705801+0000) 2023-08-28T00:37:56.707 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:56 smithi130 bash[27579]: cluster 2023-08-28T00:37:54.863521+0000 mgr.y (mgr.14901) 4780 : cluster [DBG] pgmap v4787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:56.708 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:56 smithi130 bash[27579]: audit 2023-08-28T00:37:55.568499+0000 mon.a (mon.0) 3413 : audit [DBG] from='client.? 172.21.15.6:0/1287860297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:56.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:56 smithi006 bash[33918]: cluster 2023-08-28T00:37:54.863521+0000 mgr.y (mgr.14901) 4780 : cluster [DBG] pgmap v4787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:56.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:56 smithi006 bash[33918]: audit 2023-08-28T00:37:55.568499+0000 mon.a (mon.0) 3413 : audit [DBG] from='client.? 172.21.15.6:0/1287860297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:56 smithi006 bash[27292]: cluster 2023-08-28T00:37:54.863521+0000 mgr.y (mgr.14901) 4780 : cluster [DBG] pgmap v4787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:56 smithi006 bash[27292]: audit 2023-08-28T00:37:55.568499+0000 mon.a (mon.0) 3413 : audit [DBG] from='client.? 172.21.15.6:0/1287860297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:57.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:56 smithi130 bash[51302]: debug 2023-08-28T00:37:56.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:56.706098+0000) 2023-08-28T00:37:58.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:57 smithi130 bash[51302]: debug 2023-08-28T00:37:57.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:57.706359+0000) 2023-08-28T00:37:58.708 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:58 smithi130 bash[27579]: cluster 2023-08-28T00:37:56.864117+0000 mgr.y (mgr.14901) 4781 : cluster [DBG] pgmap v4788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:58.708 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:37:58 smithi130 bash[27579]: audit 2023-08-28T00:37:57.984654+0000 mon.a (mon.0) 3414 : audit [DBG] from='client.? 172.21.15.6:0/2023720856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:58.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:58 smithi006 bash[33918]: cluster 2023-08-28T00:37:56.864117+0000 mgr.y (mgr.14901) 4781 : cluster [DBG] pgmap v4788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:58.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:37:58 smithi006 bash[33918]: audit 2023-08-28T00:37:57.984654+0000 mon.a (mon.0) 3414 : audit [DBG] from='client.? 172.21.15.6:0/2023720856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:58.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:58 smithi006 bash[27292]: cluster 2023-08-28T00:37:56.864117+0000 mgr.y (mgr.14901) 4781 : cluster [DBG] pgmap v4788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:37:58.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:37:58 smithi006 bash[27292]: audit 2023-08-28T00:37:57.984654+0000 mon.a (mon.0) 3414 : audit [DBG] from='client.? 172.21.15.6:0/2023720856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:59.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:58 smithi130 bash[51302]: debug 2023-08-28T00:37:58.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:58.706631+0000) 2023-08-28T00:38:00.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:37:59 smithi130 bash[51302]: debug 2023-08-28T00:37:59.705+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:59.706945+0000) 2023-08-28T00:38:00.709 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:00 smithi130 bash[27579]: cluster 2023-08-28T00:37:58.864768+0000 mgr.y (mgr.14901) 4782 : cluster [DBG] pgmap v4789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:00 smithi006 bash[27292]: cluster 2023-08-28T00:37:58.864768+0000 mgr.y (mgr.14901) 4782 : cluster [DBG] pgmap v4789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:00 smithi006 bash[33918]: cluster 2023-08-28T00:37:58.864768+0000 mgr.y (mgr.14901) 4782 : cluster [DBG] pgmap v4789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:01.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:00 smithi130 bash[51302]: debug 2023-08-28T00:38:00.705+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:00.707234+0000) 2023-08-28T00:38:01.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:38:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:38:00] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:38:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:38:00.788Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:38:00.790Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:01.709 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:01 smithi130 bash[27579]: audit 2023-08-28T00:38:00.416464+0000 mon.c (mon.2) 1854 : audit [DBG] from='client.? 172.21.15.6:0/2024575741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:01.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:01 smithi006 bash[27292]: audit 2023-08-28T00:38:00.416464+0000 mon.c (mon.2) 1854 : audit [DBG] from='client.? 172.21.15.6:0/2024575741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:01.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:01 smithi006 bash[33918]: audit 2023-08-28T00:38:00.416464+0000 mon.c (mon.2) 1854 : audit [DBG] from='client.? 172.21.15.6:0/2024575741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:02.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:01 smithi130 bash[51302]: debug 2023-08-28T00:38:01.705+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:01.707474+0000) 2023-08-28T00:38:02.709 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:02 smithi130 bash[27579]: cluster 2023-08-28T00:38:00.865840+0000 mgr.y (mgr.14901) 4783 : cluster [DBG] pgmap v4790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:02.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:02 smithi006 bash[27292]: cluster 2023-08-28T00:38:00.865840+0000 mgr.y (mgr.14901) 4783 : cluster [DBG] pgmap v4790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:02 smithi006 bash[33918]: cluster 2023-08-28T00:38:00.865840+0000 mgr.y (mgr.14901) 4783 : cluster [DBG] pgmap v4790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:38:02.369Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:38:02.371Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:02 smithi130 bash[51302]: debug 2023-08-28T00:38:02.705+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:02.707745+0000) 2023-08-28T00:38:03.709 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:03 smithi130 bash[27579]: audit 2023-08-28T00:38:02.830731+0000 mon.c (mon.2) 1855 : audit [DBG] from='client.? 172.21.15.6:0/6198501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:03.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:03 smithi006 bash[27292]: audit 2023-08-28T00:38:02.830731+0000 mon.c (mon.2) 1855 : audit [DBG] from='client.? 172.21.15.6:0/6198501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:03.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:03 smithi006 bash[33918]: audit 2023-08-28T00:38:02.830731+0000 mon.c (mon.2) 1855 : audit [DBG] from='client.? 172.21.15.6:0/6198501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:04.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:03 smithi130 bash[51302]: debug 2023-08-28T00:38:03.705+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:03.707971+0000) 2023-08-28T00:38:04.710 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:04 smithi130 bash[27579]: cluster 2023-08-28T00:38:02.866510+0000 mgr.y (mgr.14901) 4784 : cluster [DBG] pgmap v4791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:04.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:04 smithi006 bash[33918]: cluster 2023-08-28T00:38:02.866510+0000 mgr.y (mgr.14901) 4784 : cluster [DBG] pgmap v4791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:04.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:04 smithi006 bash[27292]: cluster 2023-08-28T00:38:02.866510+0000 mgr.y (mgr.14901) 4784 : cluster [DBG] pgmap v4791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:38:04.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:38:04.252Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:04 smithi130 bash[51302]: debug 2023-08-28T00:38:04.705+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:04.708264+0000) 2023-08-28T00:38:05.710 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:05 smithi130 bash[27579]: audit 2023-08-28T00:38:05.247297+0000 mon.c (mon.2) 1856 : audit [DBG] from='client.? 172.21.15.6:0/1547461636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:05.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:05 smithi006 bash[27292]: audit 2023-08-28T00:38:05.247297+0000 mon.c (mon.2) 1856 : audit [DBG] from='client.? 172.21.15.6:0/1547461636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:05.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:05 smithi006 bash[33918]: audit 2023-08-28T00:38:05.247297+0000 mon.c (mon.2) 1856 : audit [DBG] from='client.? 172.21.15.6:0/1547461636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:06.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:05 smithi130 bash[51302]: debug 2023-08-28T00:38:05.705+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:05.708569+0000) 2023-08-28T00:38:06.710 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:06 smithi130 bash[27579]: cluster 2023-08-28T00:38:04.867505+0000 mgr.y (mgr.14901) 4785 : cluster [DBG] pgmap v4792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:06 smithi006 bash[27292]: cluster 2023-08-28T00:38:04.867505+0000 mgr.y (mgr.14901) 4785 : cluster [DBG] pgmap v4792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:06 smithi006 bash[33918]: cluster 2023-08-28T00:38:04.867505+0000 mgr.y (mgr.14901) 4785 : cluster [DBG] pgmap v4792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:07.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:06 smithi130 bash[51302]: debug 2023-08-28T00:38:06.705+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:06.708877+0000) 2023-08-28T00:38:08.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:07 smithi130 bash[51302]: debug 2023-08-28T00:38:07.705+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:07.709206+0000) 2023-08-28T00:38:08.711 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:08 smithi130 bash[27579]: cluster 2023-08-28T00:38:06.868119+0000 mgr.y (mgr.14901) 4786 : cluster [DBG] pgmap v4793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:08.711 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:08 smithi130 bash[27579]: audit 2023-08-28T00:38:07.663318+0000 mon.c (mon.2) 1857 : audit [DBG] from='client.? 172.21.15.6:0/999813687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:08 smithi006 bash[33918]: cluster 2023-08-28T00:38:06.868119+0000 mgr.y (mgr.14901) 4786 : cluster [DBG] pgmap v4793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:08 smithi006 bash[33918]: audit 2023-08-28T00:38:07.663318+0000 mon.c (mon.2) 1857 : audit [DBG] from='client.? 172.21.15.6:0/999813687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:08.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:08 smithi006 bash[27292]: cluster 2023-08-28T00:38:06.868119+0000 mgr.y (mgr.14901) 4786 : cluster [DBG] pgmap v4793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:08.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:08 smithi006 bash[27292]: audit 2023-08-28T00:38:07.663318+0000 mon.c (mon.2) 1857 : audit [DBG] from='client.? 172.21.15.6:0/999813687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:09.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:08 smithi130 bash[51302]: debug 2023-08-28T00:38:08.705+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:08.709543+0000) 2023-08-28T00:38:10.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:09 smithi130 bash[51302]: debug 2023-08-28T00:38:09.705+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:09.709828+0000) 2023-08-28T00:38:10.711 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:10 smithi130 bash[27579]: cluster 2023-08-28T00:38:08.868780+0000 mgr.y (mgr.14901) 4787 : cluster [DBG] pgmap v4794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:10.712 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:10 smithi130 bash[27579]: audit 2023-08-28T00:38:10.088605+0000 mon.c (mon.2) 1858 : audit [DBG] from='client.? 172.21.15.6:0/356825773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:10 smithi006 bash[33918]: cluster 2023-08-28T00:38:08.868780+0000 mgr.y (mgr.14901) 4787 : cluster [DBG] pgmap v4794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:10 smithi006 bash[33918]: audit 2023-08-28T00:38:10.088605+0000 mon.c (mon.2) 1858 : audit [DBG] from='client.? 172.21.15.6:0/356825773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:10 smithi006 bash[27292]: cluster 2023-08-28T00:38:08.868780+0000 mgr.y (mgr.14901) 4787 : cluster [DBG] pgmap v4794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:10 smithi006 bash[27292]: audit 2023-08-28T00:38:10.088605+0000 mon.c (mon.2) 1858 : audit [DBG] from='client.? 172.21.15.6:0/356825773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:11.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:10 smithi130 bash[51302]: debug 2023-08-28T00:38:10.705+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:10.710119+0000) 2023-08-28T00:38:11.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:38:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:38:10] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:38:11.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:38:10.789Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:11.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:38:10.791Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:12.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:11 smithi130 bash[51302]: debug 2023-08-28T00:38:11.705+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:11.710433+0000) 2023-08-28T00:38:12.712 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:12 smithi130 bash[27579]: cluster 2023-08-28T00:38:10.869976+0000 mgr.y (mgr.14901) 4788 : cluster [DBG] pgmap v4795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:12 smithi006 bash[27292]: cluster 2023-08-28T00:38:10.869976+0000 mgr.y (mgr.14901) 4788 : cluster [DBG] pgmap v4795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:12 smithi006 bash[33918]: cluster 2023-08-28T00:38:10.869976+0000 mgr.y (mgr.14901) 4788 : cluster [DBG] pgmap v4795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:38:12.370Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:38:12.371Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:13.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:12 smithi130 bash[51302]: debug 2023-08-28T00:38:12.705+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:12.710745+0000) 2023-08-28T00:38:13.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:13 smithi006 bash[27292]: audit 2023-08-28T00:38:12.502840+0000 mon.a (mon.0) 3415 : audit [DBG] from='client.? 172.21.15.6:0/363558418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:13.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:13 smithi006 bash[33918]: audit 2023-08-28T00:38:12.502840+0000 mon.a (mon.0) 3415 : audit [DBG] from='client.? 172.21.15.6:0/363558418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:13.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:13 smithi130 bash[27579]: audit 2023-08-28T00:38:12.502840+0000 mon.a (mon.0) 3415 : audit [DBG] from='client.? 172.21.15.6:0/363558418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:13.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:13 smithi130 bash[51302]: debug 2023-08-28T00:38:13.709+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:13.711056+0000) 2023-08-28T00:38:14.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:14 smithi006 bash[33918]: cluster 2023-08-28T00:38:12.870638+0000 mgr.y (mgr.14901) 4789 : cluster [DBG] pgmap v4796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:14.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:14 smithi006 bash[27292]: cluster 2023-08-28T00:38:12.870638+0000 mgr.y (mgr.14901) 4789 : cluster [DBG] pgmap v4796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:38:14.250Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:38:14.252Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:14 smithi130 bash[27579]: cluster 2023-08-28T00:38:12.870638+0000 mgr.y (mgr.14901) 4789 : cluster [DBG] pgmap v4796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:14.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:14 smithi130 bash[51302]: debug 2023-08-28T00:38:14.709+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:14.711344+0000) 2023-08-28T00:38:15.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:15 smithi130 bash[27579]: audit 2023-08-28T00:38:14.930151+0000 mon.a (mon.0) 3416 : audit [DBG] from='client.? 172.21.15.6:0/1193787761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:15.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:15 smithi130 bash[51302]: debug 2023-08-28T00:38:15.709+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:15.711678+0000) 2023-08-28T00:38:15.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:15 smithi006 bash[27292]: audit 2023-08-28T00:38:14.930151+0000 mon.a (mon.0) 3416 : audit [DBG] from='client.? 172.21.15.6:0/1193787761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:15.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:15 smithi006 bash[33918]: audit 2023-08-28T00:38:14.930151+0000 mon.a (mon.0) 3416 : audit [DBG] from='client.? 172.21.15.6:0/1193787761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:16 smithi130 bash[27579]: cluster 2023-08-28T00:38:14.871575+0000 mgr.y (mgr.14901) 4790 : cluster [DBG] pgmap v4797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:16.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:16 smithi130 bash[51302]: debug 2023-08-28T00:38:16.709+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:16.711952+0000) 2023-08-28T00:38:16.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:16 smithi006 bash[27292]: cluster 2023-08-28T00:38:14.871575+0000 mgr.y (mgr.14901) 4790 : cluster [DBG] pgmap v4797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:16.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:16 smithi006 bash[33918]: cluster 2023-08-28T00:38:14.871575+0000 mgr.y (mgr.14901) 4790 : cluster [DBG] pgmap v4797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:17.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:17 smithi130 bash[27579]: audit 2023-08-28T00:38:17.347051+0000 mon.c (mon.2) 1859 : audit [DBG] from='client.? 172.21.15.6:0/733319174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:17.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:17 smithi130 bash[51302]: debug 2023-08-28T00:38:17.709+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:17.712261+0000) 2023-08-28T00:38:17.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:17 smithi006 bash[33918]: audit 2023-08-28T00:38:17.347051+0000 mon.c (mon.2) 1859 : audit [DBG] from='client.? 172.21.15.6:0/733319174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:17.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:17 smithi006 bash[27292]: audit 2023-08-28T00:38:17.347051+0000 mon.c (mon.2) 1859 : audit [DBG] from='client.? 172.21.15.6:0/733319174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:18 smithi130 bash[27579]: cluster 2023-08-28T00:38:16.872176+0000 mgr.y (mgr.14901) 4791 : cluster [DBG] pgmap v4798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:18.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:18 smithi130 bash[51302]: debug 2023-08-28T00:38:18.709+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:18.712546+0000) 2023-08-28T00:38:18.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:18 smithi006 bash[27292]: cluster 2023-08-28T00:38:16.872176+0000 mgr.y (mgr.14901) 4791 : cluster [DBG] pgmap v4798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:18.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:18 smithi006 bash[33918]: cluster 2023-08-28T00:38:16.872176+0000 mgr.y (mgr.14901) 4791 : cluster [DBG] pgmap v4798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:20.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:19 smithi130 bash[51302]: debug 2023-08-28T00:38:19.709+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:19.712877+0000) 2023-08-28T00:38:20.756 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:20 smithi006 bash[27292]: cluster 2023-08-28T00:38:18.872805+0000 mgr.y (mgr.14901) 4792 : cluster [DBG] pgmap v4799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:20.756 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:20 smithi006 bash[27292]: audit 2023-08-28T00:38:19.762845+0000 mon.a (mon.0) 3417 : audit [DBG] from='client.? 172.21.15.6:0/1043649855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:20.757 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:20 smithi006 bash[33918]: cluster 2023-08-28T00:38:18.872805+0000 mgr.y (mgr.14901) 4792 : cluster [DBG] pgmap v4799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:20.757 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:20 smithi006 bash[33918]: audit 2023-08-28T00:38:19.762845+0000 mon.a (mon.0) 3417 : audit [DBG] from='client.? 172.21.15.6:0/1043649855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:20 smithi130 bash[27579]: cluster 2023-08-28T00:38:18.872805+0000 mgr.y (mgr.14901) 4792 : cluster [DBG] pgmap v4799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:20 smithi130 bash[27579]: audit 2023-08-28T00:38:19.762845+0000 mon.a (mon.0) 3417 : audit [DBG] from='client.? 172.21.15.6:0/1043649855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:20 smithi130 bash[51302]: debug 2023-08-28T00:38:20.709+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:20.713210+0000) 2023-08-28T00:38:21.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:38:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:38:20] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T00:38:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:38:20.789Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:38:20.791Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:22.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:21 smithi130 bash[51302]: debug 2023-08-28T00:38:21.709+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:21.713482+0000) 2023-08-28T00:38:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:22 smithi006 bash[33918]: cluster 2023-08-28T00:38:20.873904+0000 mgr.y (mgr.14901) 4793 : cluster [DBG] pgmap v4800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:22 smithi006 bash[33918]: audit 2023-08-28T00:38:22.179125+0000 mon.c (mon.2) 1860 : audit [DBG] from='client.? 172.21.15.6:0/484172571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:22 smithi006 bash[27292]: cluster 2023-08-28T00:38:20.873904+0000 mgr.y (mgr.14901) 4793 : cluster [DBG] pgmap v4800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:22 smithi006 bash[27292]: audit 2023-08-28T00:38:22.179125+0000 mon.c (mon.2) 1860 : audit [DBG] from='client.? 172.21.15.6:0/484172571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:22.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:38:22.370Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:22.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:38:22.372Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:22 smithi130 bash[27579]: cluster 2023-08-28T00:38:20.873904+0000 mgr.y (mgr.14901) 4793 : cluster [DBG] pgmap v4800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:22.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:22 smithi130 bash[27579]: audit 2023-08-28T00:38:22.179125+0000 mon.c (mon.2) 1860 : audit [DBG] from='client.? 172.21.15.6:0/484172571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:22.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:22 smithi130 bash[51302]: debug 2023-08-28T00:38:22.709+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:22.713759+0000) 2023-08-28T00:38:24.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:23 smithi130 bash[51302]: debug 2023-08-28T00:38:23.709+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:23.714051+0000) 2023-08-28T00:38:24.511 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:38:24.251Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:24.512 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:38:24.253Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:24 smithi130 bash[27579]: cluster 2023-08-28T00:38:22.874596+0000 mgr.y (mgr.14901) 4794 : cluster [DBG] pgmap v4801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:24.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:24 smithi130 bash[51302]: debug 2023-08-28T00:38:24.709+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:24.714399+0000) 2023-08-28T00:38:24.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:24 smithi006 bash[27292]: cluster 2023-08-28T00:38:22.874596+0000 mgr.y (mgr.14901) 4794 : cluster [DBG] pgmap v4801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:24.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:24 smithi006 bash[33918]: cluster 2023-08-28T00:38:22.874596+0000 mgr.y (mgr.14901) 4794 : cluster [DBG] pgmap v4801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:25.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:25 smithi130 bash[27579]: audit 2023-08-28T00:38:24.603466+0000 mon.a (mon.0) 3418 : audit [DBG] from='client.? 172.21.15.6:0/2788516064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:25.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:25 smithi130 bash[51302]: debug 2023-08-28T00:38:25.709+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:25.714658+0000) 2023-08-28T00:38:25.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:25 smithi006 bash[33918]: audit 2023-08-28T00:38:24.603466+0000 mon.a (mon.0) 3418 : audit [DBG] from='client.? 172.21.15.6:0/2788516064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:25.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:25 smithi006 bash[27292]: audit 2023-08-28T00:38:24.603466+0000 mon.a (mon.0) 3418 : audit [DBG] from='client.? 172.21.15.6:0/2788516064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:26 smithi130 bash[27579]: cluster 2023-08-28T00:38:24.875704+0000 mgr.y (mgr.14901) 4795 : cluster [DBG] pgmap v4802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:26.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:26 smithi130 bash[51302]: debug 2023-08-28T00:38:26.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:26.714983+0000) 2023-08-28T00:38:26.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:26 smithi006 bash[27292]: cluster 2023-08-28T00:38:24.875704+0000 mgr.y (mgr.14901) 4795 : cluster [DBG] pgmap v4802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:26.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:26 smithi006 bash[33918]: cluster 2023-08-28T00:38:24.875704+0000 mgr.y (mgr.14901) 4795 : cluster [DBG] pgmap v4802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:27 smithi130 bash[27579]: audit 2023-08-28T00:38:27.015715+0000 mon.a (mon.0) 3419 : audit [DBG] from='client.? 172.21.15.6:0/596728112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:27.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:27 smithi130 bash[51302]: debug 2023-08-28T00:38:27.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:27.715309+0000) 2023-08-28T00:38:27.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:27 smithi006 bash[33918]: audit 2023-08-28T00:38:27.015715+0000 mon.a (mon.0) 3419 : audit [DBG] from='client.? 172.21.15.6:0/596728112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:27.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:27 smithi006 bash[27292]: audit 2023-08-28T00:38:27.015715+0000 mon.a (mon.0) 3419 : audit [DBG] from='client.? 172.21.15.6:0/596728112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:28 smithi130 bash[27579]: cluster 2023-08-28T00:38:26.876299+0000 mgr.y (mgr.14901) 4796 : cluster [DBG] pgmap v4803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:28 smithi130 bash[51302]: debug 2023-08-28T00:38:28.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:28.715571+0000) 2023-08-28T00:38:28.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:28 smithi006 bash[27292]: cluster 2023-08-28T00:38:26.876299+0000 mgr.y (mgr.14901) 4796 : cluster [DBG] pgmap v4803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:28.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:28 smithi006 bash[33918]: cluster 2023-08-28T00:38:26.876299+0000 mgr.y (mgr.14901) 4796 : cluster [DBG] pgmap v4803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:29.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:29 smithi130 bash[51302]: debug 2023-08-28T00:38:29.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:29.715934+0000) 2023-08-28T00:38:29.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:29 smithi130 bash[27579]: audit 2023-08-28T00:38:29.427532+0000 mon.a (mon.0) 3420 : audit [DBG] from='client.? 172.21.15.6:0/490872974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:29.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:29 smithi006 bash[27292]: audit 2023-08-28T00:38:29.427532+0000 mon.a (mon.0) 3420 : audit [DBG] from='client.? 172.21.15.6:0/490872974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:29.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:29 smithi006 bash[33918]: audit 2023-08-28T00:38:29.427532+0000 mon.a (mon.0) 3420 : audit [DBG] from='client.? 172.21.15.6:0/490872974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:30.795 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:30 smithi006 bash[27292]: cluster 2023-08-28T00:38:28.876986+0000 mgr.y (mgr.14901) 4797 : cluster [DBG] pgmap v4804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:30.796 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:38:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:38:30] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T00:38:30.796 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:38:30.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:30.797 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:30 smithi006 bash[33918]: cluster 2023-08-28T00:38:28.876986+0000 mgr.y (mgr.14901) 4797 : cluster [DBG] pgmap v4804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:30 smithi130 bash[27579]: cluster 2023-08-28T00:38:28.876986+0000 mgr.y (mgr.14901) 4797 : cluster [DBG] pgmap v4804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:30.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:30 smithi130 bash[51302]: debug 2023-08-28T00:38:30.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:30.716216+0000) 2023-08-28T00:38:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:38:30.792Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:32.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:31 smithi130 bash[51302]: debug 2023-08-28T00:38:31.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:31.716502+0000) 2023-08-28T00:38:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:32 smithi006 bash[33918]: cluster 2023-08-28T00:38:30.877928+0000 mgr.y (mgr.14901) 4798 : cluster [DBG] pgmap v4805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:32 smithi006 bash[33918]: audit 2023-08-28T00:38:31.836346+0000 mon.a (mon.0) 3421 : audit [DBG] from='client.? 172.21.15.6:0/1110620494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:38:32.371Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:32.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:38:32.373Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:32.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:32 smithi006 bash[27292]: cluster 2023-08-28T00:38:30.877928+0000 mgr.y (mgr.14901) 4798 : cluster [DBG] pgmap v4805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:32.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:32 smithi006 bash[27292]: audit 2023-08-28T00:38:31.836346+0000 mon.a (mon.0) 3421 : audit [DBG] from='client.? 172.21.15.6:0/1110620494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:32 smithi130 bash[51302]: debug 2023-08-28T00:38:32.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:32.716780+0000) 2023-08-28T00:38:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:32 smithi130 bash[27579]: cluster 2023-08-28T00:38:30.877928+0000 mgr.y (mgr.14901) 4798 : cluster [DBG] pgmap v4805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:32 smithi130 bash[27579]: audit 2023-08-28T00:38:31.836346+0000 mon.a (mon.0) 3421 : audit [DBG] from='client.? 172.21.15.6:0/1110620494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:34.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:33 smithi130 bash[51302]: debug 2023-08-28T00:38:33.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:33.717027+0000) 2023-08-28T00:38:34.553 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:38:34.252Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:34.553 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:38:34.253Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:34 smithi130 bash[51302]: debug 2023-08-28T00:38:34.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:34.717294+0000) 2023-08-28T00:38:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:34 smithi130 bash[27579]: cluster 2023-08-28T00:38:32.878551+0000 mgr.y (mgr.14901) 4799 : cluster [DBG] pgmap v4806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:34.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:34 smithi130 bash[27579]: audit 2023-08-28T00:38:34.261709+0000 mon.a (mon.0) 3422 : audit [DBG] from='client.? 172.21.15.6:0/1980764598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:34.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:34 smithi006 bash[33918]: cluster 2023-08-28T00:38:32.878551+0000 mgr.y (mgr.14901) 4799 : cluster [DBG] pgmap v4806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:34.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:34 smithi006 bash[33918]: audit 2023-08-28T00:38:34.261709+0000 mon.a (mon.0) 3422 : audit [DBG] from='client.? 172.21.15.6:0/1980764598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:34.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:34 smithi006 bash[27292]: cluster 2023-08-28T00:38:32.878551+0000 mgr.y (mgr.14901) 4799 : cluster [DBG] pgmap v4806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:34.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:34 smithi006 bash[27292]: audit 2023-08-28T00:38:34.261709+0000 mon.a (mon.0) 3422 : audit [DBG] from='client.? 172.21.15.6:0/1980764598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:36.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:35 smithi130 bash[51302]: debug 2023-08-28T00:38:35.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:35.717581+0000) 2023-08-28T00:38:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:36 smithi130 bash[27579]: cluster 2023-08-28T00:38:34.879693+0000 mgr.y (mgr.14901) 4800 : cluster [DBG] pgmap v4807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:36 smithi130 bash[51302]: debug 2023-08-28T00:38:36.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:36.717887+0000) 2023-08-28T00:38:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:36 smithi006 bash[33918]: cluster 2023-08-28T00:38:34.879693+0000 mgr.y (mgr.14901) 4800 : cluster [DBG] pgmap v4807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:36.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:36 smithi006 bash[27292]: cluster 2023-08-28T00:38:34.879693+0000 mgr.y (mgr.14901) 4800 : cluster [DBG] pgmap v4807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:37 smithi130 bash[51302]: debug 2023-08-28T00:38:37.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:37.718134+0000) 2023-08-28T00:38:37.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:37 smithi130 bash[27579]: audit 2023-08-28T00:38:36.684419+0000 mon.a (mon.0) 3423 : audit [DBG] from='client.? 172.21.15.6:0/3175443044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:37.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:37 smithi006 bash[27292]: audit 2023-08-28T00:38:36.684419+0000 mon.a (mon.0) 3423 : audit [DBG] from='client.? 172.21.15.6:0/3175443044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:37.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:37 smithi006 bash[33918]: audit 2023-08-28T00:38:36.684419+0000 mon.a (mon.0) 3423 : audit [DBG] from='client.? 172.21.15.6:0/3175443044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:38 smithi130 bash[27579]: cluster 2023-08-28T00:38:36.880446+0000 mgr.y (mgr.14901) 4801 : cluster [DBG] pgmap v4808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:38.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:38 smithi130 bash[51302]: debug 2023-08-28T00:38:38.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:38.718464+0000) 2023-08-28T00:38:38.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:38 smithi006 bash[33918]: cluster 2023-08-28T00:38:36.880446+0000 mgr.y (mgr.14901) 4801 : cluster [DBG] pgmap v4808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:38.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:38 smithi006 bash[27292]: cluster 2023-08-28T00:38:36.880446+0000 mgr.y (mgr.14901) 4801 : cluster [DBG] pgmap v4808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:39.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:39 smithi130 bash[27579]: audit 2023-08-28T00:38:39.105531+0000 mon.a (mon.0) 3424 : audit [DBG] from='client.? 172.21.15.6:0/3675140636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:39.871 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:39 smithi130 bash[51302]: debug 2023-08-28T00:38:39.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:39.718754+0000) 2023-08-28T00:38:39.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:39 smithi006 bash[27292]: audit 2023-08-28T00:38:39.105531+0000 mon.a (mon.0) 3424 : audit [DBG] from='client.? 172.21.15.6:0/3675140636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:39.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:39 smithi006 bash[33918]: audit 2023-08-28T00:38:39.105531+0000 mon.a (mon.0) 3424 : audit [DBG] from='client.? 172.21.15.6:0/3675140636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:40 smithi130 bash[27579]: cluster 2023-08-28T00:38:38.880959+0000 mgr.y (mgr.14901) 4802 : cluster [DBG] pgmap v4809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:40.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:40 smithi130 bash[51302]: debug 2023-08-28T00:38:40.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:40.719073+0000) 2023-08-28T00:38:40.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:38:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:38:40] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T00:38:40.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:40 smithi006 bash[27292]: cluster 2023-08-28T00:38:38.880959+0000 mgr.y (mgr.14901) 4802 : cluster [DBG] pgmap v4809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:40.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:40 smithi006 bash[33918]: cluster 2023-08-28T00:38:38.880959+0000 mgr.y (mgr.14901) 4802 : cluster [DBG] pgmap v4809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:40.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:38:40.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:40.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:38:40.792Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:42.012 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:41 smithi130 bash[51302]: debug 2023-08-28T00:38:41.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:41.719383+0000) 2023-08-28T00:38:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:42 smithi130 bash[27579]: cluster 2023-08-28T00:38:40.882009+0000 mgr.y (mgr.14901) 4803 : cluster [DBG] pgmap v4810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:42.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:42 smithi130 bash[27579]: audit 2023-08-28T00:38:41.511776+0000 mon.a (mon.0) 3425 : audit [DBG] from='client.? 172.21.15.6:0/2459325341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:42.375 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:42 smithi006 bash[27292]: cluster 2023-08-28T00:38:40.882009+0000 mgr.y (mgr.14901) 4803 : cluster [DBG] pgmap v4810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:42.375 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:42 smithi006 bash[27292]: audit 2023-08-28T00:38:41.511776+0000 mon.a (mon.0) 3425 : audit [DBG] from='client.? 172.21.15.6:0/2459325341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:42.376 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:42 smithi006 bash[33918]: cluster 2023-08-28T00:38:40.882009+0000 mgr.y (mgr.14901) 4803 : cluster [DBG] pgmap v4810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:42.376 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:42 smithi006 bash[33918]: audit 2023-08-28T00:38:41.511776+0000 mon.a (mon.0) 3425 : audit [DBG] from='client.? 172.21.15.6:0/2459325341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:38:42.372Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:38:42.373Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:43.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:42 smithi130 bash[51302]: debug 2023-08-28T00:38:42.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:42.719683+0000) 2023-08-28T00:38:44.017 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:43 smithi130 bash[51302]: debug 2023-08-28T00:38:43.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:43.719981+0000) 2023-08-28T00:38:44.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:44 smithi130 bash[27579]: cluster 2023-08-28T00:38:42.882662+0000 mgr.y (mgr.14901) 4804 : cluster [DBG] pgmap v4811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:44.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:44 smithi130 bash[27579]: audit 2023-08-28T00:38:43.921522+0000 mon.c (mon.2) 1861 : audit [DBG] from='client.? 172.21.15.6:0/2518897877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:38:44.252Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:38:44.254Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:44.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:44 smithi006 bash[27292]: cluster 2023-08-28T00:38:42.882662+0000 mgr.y (mgr.14901) 4804 : cluster [DBG] pgmap v4811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:44.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:44 smithi006 bash[27292]: audit 2023-08-28T00:38:43.921522+0000 mon.c (mon.2) 1861 : audit [DBG] from='client.? 172.21.15.6:0/2518897877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:44.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:44 smithi006 bash[33918]: cluster 2023-08-28T00:38:42.882662+0000 mgr.y (mgr.14901) 4804 : cluster [DBG] pgmap v4811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:44.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:44 smithi006 bash[33918]: audit 2023-08-28T00:38:43.921522+0000 mon.c (mon.2) 1861 : audit [DBG] from='client.? 172.21.15.6:0/2518897877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:44 smithi130 bash[51302]: debug 2023-08-28T00:38:44.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:44.720192+0000) 2023-08-28T00:38:46.022 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:45 smithi130 bash[51302]: debug 2023-08-28T00:38:45.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:45.720497+0000) 2023-08-28T00:38:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:46 smithi130 bash[27579]: cluster 2023-08-28T00:38:44.883816+0000 mgr.y (mgr.14901) 4805 : cluster [DBG] pgmap v4812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:46.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:46 smithi006 bash[33918]: cluster 2023-08-28T00:38:44.883816+0000 mgr.y (mgr.14901) 4805 : cluster [DBG] pgmap v4812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:46.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:46 smithi006 bash[27292]: cluster 2023-08-28T00:38:44.883816+0000 mgr.y (mgr.14901) 4805 : cluster [DBG] pgmap v4812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:47.031 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:46 smithi130 bash[51302]: debug 2023-08-28T00:38:46.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:46.720777+0000) 2023-08-28T00:38:47.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:47 smithi130 bash[27579]: audit 2023-08-28T00:38:46.335723+0000 mon.a (mon.0) 3426 : audit [DBG] from='client.? 172.21.15.6:0/2741958983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:47.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:47 smithi006 bash[33918]: audit 2023-08-28T00:38:46.335723+0000 mon.a (mon.0) 3426 : audit [DBG] from='client.? 172.21.15.6:0/2741958983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:47.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:47 smithi006 bash[27292]: audit 2023-08-28T00:38:46.335723+0000 mon.a (mon.0) 3426 : audit [DBG] from='client.? 172.21.15.6:0/2741958983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:48.034 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:47 smithi130 bash[51302]: debug 2023-08-28T00:38:47.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:47.721053+0000) 2023-08-28T00:38:48.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:48 smithi130 bash[27579]: cluster 2023-08-28T00:38:46.884464+0000 mgr.y (mgr.14901) 4806 : cluster [DBG] pgmap v4813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:48.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:48 smithi006 bash[27292]: cluster 2023-08-28T00:38:46.884464+0000 mgr.y (mgr.14901) 4806 : cluster [DBG] pgmap v4813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:48.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:48 smithi006 bash[33918]: cluster 2023-08-28T00:38:46.884464+0000 mgr.y (mgr.14901) 4806 : cluster [DBG] pgmap v4813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:49.042 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:48 smithi130 bash[51302]: debug 2023-08-28T00:38:48.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:48.721332+0000) 2023-08-28T00:38:49.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:49 smithi130 bash[27579]: audit 2023-08-28T00:38:48.757611+0000 mon.c (mon.2) 1862 : audit [DBG] from='client.? 172.21.15.6:0/2655973889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:49.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:49 smithi006 bash[27292]: audit 2023-08-28T00:38:48.757611+0000 mon.c (mon.2) 1862 : audit [DBG] from='client.? 172.21.15.6:0/2655973889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:49.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:49 smithi006 bash[33918]: audit 2023-08-28T00:38:48.757611+0000 mon.c (mon.2) 1862 : audit [DBG] from='client.? 172.21.15.6:0/2655973889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:50.045 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:49 smithi130 bash[51302]: debug 2023-08-28T00:38:49.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:49.721651+0000) 2023-08-28T00:38:50.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:50 smithi130 bash[27579]: cluster 2023-08-28T00:38:48.885180+0000 mgr.y (mgr.14901) 4807 : cluster [DBG] pgmap v4814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:50.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:50 smithi006 bash[33918]: cluster 2023-08-28T00:38:48.885180+0000 mgr.y (mgr.14901) 4807 : cluster [DBG] pgmap v4814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:50.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:50 smithi006 bash[27292]: cluster 2023-08-28T00:38:48.885180+0000 mgr.y (mgr.14901) 4807 : cluster [DBG] pgmap v4814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:51.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:50 smithi130 bash[51302]: debug 2023-08-28T00:38:50.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:50.721958+0000) 2023-08-28T00:38:51.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:38:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:38:50] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:38:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:38:50.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:38:50.793Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:52.049 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:51 smithi130 bash[51302]: debug 2023-08-28T00:38:51.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:51.722265+0000) 2023-08-28T00:38:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:52 smithi130 bash[27579]: cluster 2023-08-28T00:38:50.885782+0000 mgr.y (mgr.14901) 4808 : cluster [DBG] pgmap v4815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:52 smithi130 bash[27579]: audit 2023-08-28T00:38:51.174923+0000 mon.a (mon.0) 3427 : audit [DBG] from='client.? 172.21.15.6:0/4253083215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:52.376 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:52 smithi006 bash[33918]: cluster 2023-08-28T00:38:50.885782+0000 mgr.y (mgr.14901) 4808 : cluster [DBG] pgmap v4815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:52.376 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:52 smithi006 bash[33918]: audit 2023-08-28T00:38:51.174923+0000 mon.a (mon.0) 3427 : audit [DBG] from='client.? 172.21.15.6:0/4253083215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:52.377 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:52 smithi006 bash[27292]: cluster 2023-08-28T00:38:50.885782+0000 mgr.y (mgr.14901) 4808 : cluster [DBG] pgmap v4815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:52.377 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:52 smithi006 bash[27292]: audit 2023-08-28T00:38:51.174923+0000 mon.a (mon.0) 3427 : audit [DBG] from='client.? 172.21.15.6:0/4253083215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:38:52.373Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:38:52.375Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:52 smithi130 bash[51302]: debug 2023-08-28T00:38:52.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:52.722549+0000) 2023-08-28T00:38:54.054 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:53 smithi130 bash[51302]: debug 2023-08-28T00:38:53.721+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:53.722903+0000) 2023-08-28T00:38:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:54 smithi130 bash[27579]: cluster 2023-08-28T00:38:52.886414+0000 mgr.y (mgr.14901) 4809 : cluster [DBG] pgmap v4816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:54 smithi130 bash[27579]: audit 2023-08-28T00:38:53.063687+0000 mon.b (mon.1) 582 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:38:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:54 smithi130 bash[27579]: audit 2023-08-28T00:38:53.572064+0000 mon.c (mon.2) 1863 : audit [DBG] from='client.? 172.21.15.6:0/2736121229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:54.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:54 smithi006 bash[27292]: cluster 2023-08-28T00:38:52.886414+0000 mgr.y (mgr.14901) 4809 : cluster [DBG] pgmap v4816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:54.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:54 smithi006 bash[27292]: audit 2023-08-28T00:38:53.063687+0000 mon.b (mon.1) 582 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:38:54.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:54 smithi006 bash[27292]: audit 2023-08-28T00:38:53.572064+0000 mon.c (mon.2) 1863 : audit [DBG] from='client.? 172.21.15.6:0/2736121229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:54.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:54 smithi006 bash[33918]: cluster 2023-08-28T00:38:52.886414+0000 mgr.y (mgr.14901) 4809 : cluster [DBG] pgmap v4816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:54.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:54 smithi006 bash[33918]: audit 2023-08-28T00:38:53.063687+0000 mon.b (mon.1) 582 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:38:54.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:54 smithi006 bash[33918]: audit 2023-08-28T00:38:53.572064+0000 mon.c (mon.2) 1863 : audit [DBG] from='client.? 172.21.15.6:0/2736121229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:54.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:38:54.253Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:54.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:38:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:38:54.255Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:38:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:54 smithi130 bash[51302]: debug 2023-08-28T00:38:54.721+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:54.723176+0000) 2023-08-28T00:38:56.059 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:55 smithi130 bash[51302]: debug 2023-08-28T00:38:55.721+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:55.723442+0000) 2023-08-28T00:38:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:56 smithi130 bash[27579]: cluster 2023-08-28T00:38:54.887515+0000 mgr.y (mgr.14901) 4810 : cluster [DBG] pgmap v4817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:56 smithi130 bash[27579]: audit 2023-08-28T00:38:55.990408+0000 mon.c (mon.2) 1864 : audit [DBG] from='client.? 172.21.15.6:0/1717409884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:56.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:56 smithi006 bash[27292]: cluster 2023-08-28T00:38:54.887515+0000 mgr.y (mgr.14901) 4810 : cluster [DBG] pgmap v4817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:56.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:56 smithi006 bash[27292]: audit 2023-08-28T00:38:55.990408+0000 mon.c (mon.2) 1864 : audit [DBG] from='client.? 172.21.15.6:0/1717409884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:56.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:56 smithi006 bash[33918]: cluster 2023-08-28T00:38:54.887515+0000 mgr.y (mgr.14901) 4810 : cluster [DBG] pgmap v4817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:56.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:56 smithi006 bash[33918]: audit 2023-08-28T00:38:55.990408+0000 mon.c (mon.2) 1864 : audit [DBG] from='client.? 172.21.15.6:0/1717409884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:57.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:56 smithi130 bash[51302]: debug 2023-08-28T00:38:56.720+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:56.723669+0000) 2023-08-28T00:38:58.064 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:57 smithi130 bash[51302]: debug 2023-08-28T00:38:57.720+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:57.723981+0000) 2023-08-28T00:38:58.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:58 smithi130 bash[27579]: cluster 2023-08-28T00:38:56.887995+0000 mgr.y (mgr.14901) 4811 : cluster [DBG] pgmap v4818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:58.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:58 smithi006 bash[27292]: cluster 2023-08-28T00:38:56.887995+0000 mgr.y (mgr.14901) 4811 : cluster [DBG] pgmap v4818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:58.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:58 smithi006 bash[33918]: cluster 2023-08-28T00:38:56.887995+0000 mgr.y (mgr.14901) 4811 : cluster [DBG] pgmap v4818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:38:59.070 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:58 smithi130 bash[51302]: debug 2023-08-28T00:38:58.720+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:58.724198+0000) 2023-08-28T00:38:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:59 smithi130 bash[27579]: audit 2023-08-28T00:38:58.423197+0000 mon.c (mon.2) 1865 : audit [DBG] from='client.? 172.21.15.6:0/198388403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:38:59 smithi130 bash[27579]: audit 2023-08-28T00:38:58.919622+0000 mon.a (mon.0) 3428 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:38:59.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:59 smithi006 bash[27292]: audit 2023-08-28T00:38:58.423197+0000 mon.c (mon.2) 1865 : audit [DBG] from='client.? 172.21.15.6:0/198388403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:59.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:38:59 smithi006 bash[27292]: audit 2023-08-28T00:38:58.919622+0000 mon.a (mon.0) 3428 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:38:59.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:59 smithi006 bash[33918]: audit 2023-08-28T00:38:58.423197+0000 mon.c (mon.2) 1865 : audit [DBG] from='client.? 172.21.15.6:0/198388403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:59.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:38:59 smithi006 bash[33918]: audit 2023-08-28T00:38:58.919622+0000 mon.a (mon.0) 3428 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:39:00.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:38:59 smithi130 bash[51302]: debug 2023-08-28T00:38:59.720+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:59.724458+0000) 2023-08-28T00:39:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:00 smithi006 bash[33918]: cluster 2023-08-28T00:38:58.888589+0000 mgr.y (mgr.14901) 4812 : cluster [DBG] pgmap v4819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:00 smithi006 bash[33918]: audit 2023-08-28T00:38:59.388730+0000 mon.a (mon.0) 3429 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:39:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:00 smithi006 bash[33918]: audit 2023-08-28T00:38:59.688290+0000 mon.b (mon.1) 583 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:39:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:00 smithi006 bash[33918]: audit 2023-08-28T00:38:59.690784+0000 mon.b (mon.1) 584 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:39:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:00 smithi006 bash[33918]: audit 2023-08-28T00:38:59.698849+0000 mon.a (mon.0) 3430 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:39:00.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:00 smithi006 bash[27292]: cluster 2023-08-28T00:38:58.888589+0000 mgr.y (mgr.14901) 4812 : cluster [DBG] pgmap v4819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:00.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:00 smithi006 bash[27292]: audit 2023-08-28T00:38:59.388730+0000 mon.a (mon.0) 3429 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:39:00.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:00 smithi006 bash[27292]: audit 2023-08-28T00:38:59.688290+0000 mon.b (mon.1) 583 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:39:00.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:00 smithi006 bash[27292]: audit 2023-08-28T00:38:59.690784+0000 mon.b (mon.1) 584 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:39:00.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:00 smithi006 bash[27292]: audit 2023-08-28T00:38:59.698849+0000 mon.a (mon.0) 3430 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:39:00.726 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:00 smithi130 bash[27579]: cluster 2023-08-28T00:38:58.888589+0000 mgr.y (mgr.14901) 4812 : cluster [DBG] pgmap v4819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:00.726 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:00 smithi130 bash[27579]: audit 2023-08-28T00:38:59.388730+0000 mon.a (mon.0) 3429 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:39:00.726 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:00 smithi130 bash[27579]: audit 2023-08-28T00:38:59.688290+0000 mon.b (mon.1) 583 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:39:00.727 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:00 smithi130 bash[27579]: audit 2023-08-28T00:38:59.690784+0000 mon.b (mon.1) 584 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:39:00.727 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:00 smithi130 bash[27579]: audit 2023-08-28T00:38:59.698849+0000 mon.a (mon.0) 3430 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:39:01.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:00 smithi130 bash[51302]: debug 2023-08-28T00:39:00.720+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:00.724733+0000) 2023-08-28T00:39:01.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:39:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:39:00] "GET /metrics HTTP/1.1" 200 193132 "" "Prometheus/2.33.4" 2023-08-28T00:39:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:39:00.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:01.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:39:00.794Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:01.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:01 smithi006 bash[33918]: audit 2023-08-28T00:39:00.845442+0000 mon.a (mon.0) 3431 : audit [DBG] from='client.? 172.21.15.6:0/3231429691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:01.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:01 smithi006 bash[27292]: audit 2023-08-28T00:39:00.845442+0000 mon.a (mon.0) 3431 : audit [DBG] from='client.? 172.21.15.6:0/3231429691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:01.726 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:01 smithi130 bash[27579]: audit 2023-08-28T00:39:00.845442+0000 mon.a (mon.0) 3431 : audit [DBG] from='client.? 172.21.15.6:0/3231429691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:02.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:01 smithi130 bash[51302]: debug 2023-08-28T00:39:01.720+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:01.725004+0000) 2023-08-28T00:39:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:02 smithi006 bash[33918]: cluster 2023-08-28T00:39:00.889802+0000 mgr.y (mgr.14901) 4813 : cluster [DBG] pgmap v4820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:02.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:39:02.373Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:02.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:39:02.375Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:02.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:02 smithi006 bash[27292]: cluster 2023-08-28T00:39:00.889802+0000 mgr.y (mgr.14901) 4813 : cluster [DBG] pgmap v4820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:02.726 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:02 smithi130 bash[27579]: cluster 2023-08-28T00:39:00.889802+0000 mgr.y (mgr.14901) 4813 : cluster [DBG] pgmap v4820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:02 smithi130 bash[51302]: debug 2023-08-28T00:39:02.720+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:02.725212+0000) 2023-08-28T00:39:03.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:03 smithi006 bash[33918]: audit 2023-08-28T00:39:03.254319+0000 mon.c (mon.2) 1866 : audit [DBG] from='client.? 172.21.15.6:0/146105502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:03.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:03 smithi006 bash[27292]: audit 2023-08-28T00:39:03.254319+0000 mon.c (mon.2) 1866 : audit [DBG] from='client.? 172.21.15.6:0/146105502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:03.727 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:03 smithi130 bash[27579]: audit 2023-08-28T00:39:03.254319+0000 mon.c (mon.2) 1866 : audit [DBG] from='client.? 172.21.15.6:0/146105502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:04.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:03 smithi130 bash[51302]: debug 2023-08-28T00:39:03.724+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:03.725494+0000) 2023-08-28T00:39:04.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:04 smithi006 bash[33918]: cluster 2023-08-28T00:39:02.890584+0000 mgr.y (mgr.14901) 4814 : cluster [DBG] pgmap v4821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:04.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:04 smithi006 bash[27292]: cluster 2023-08-28T00:39:02.890584+0000 mgr.y (mgr.14901) 4814 : cluster [DBG] pgmap v4821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:04.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:39:04.253Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:04.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:39:04.256Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:04.727 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:04 smithi130 bash[27579]: cluster 2023-08-28T00:39:02.890584+0000 mgr.y (mgr.14901) 4814 : cluster [DBG] pgmap v4821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:04 smithi130 bash[51302]: debug 2023-08-28T00:39:04.720+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:04.725768+0000) 2023-08-28T00:39:06.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:05 smithi130 bash[51302]: debug 2023-08-28T00:39:05.724+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:05.726036+0000) 2023-08-28T00:39:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:06 smithi006 bash[27292]: cluster 2023-08-28T00:39:04.891727+0000 mgr.y (mgr.14901) 4815 : cluster [DBG] pgmap v4822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:06.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:06 smithi006 bash[27292]: audit 2023-08-28T00:39:05.673698+0000 mon.c (mon.2) 1867 : audit [DBG] from='client.? 172.21.15.6:0/1112864931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:06.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:06 smithi006 bash[33918]: cluster 2023-08-28T00:39:04.891727+0000 mgr.y (mgr.14901) 4815 : cluster [DBG] pgmap v4822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:06.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:06 smithi006 bash[33918]: audit 2023-08-28T00:39:05.673698+0000 mon.c (mon.2) 1867 : audit [DBG] from='client.? 172.21.15.6:0/1112864931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:06.728 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:06 smithi130 bash[27579]: cluster 2023-08-28T00:39:04.891727+0000 mgr.y (mgr.14901) 4815 : cluster [DBG] pgmap v4822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:06.728 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:06 smithi130 bash[27579]: audit 2023-08-28T00:39:05.673698+0000 mon.c (mon.2) 1867 : audit [DBG] from='client.? 172.21.15.6:0/1112864931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:07.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:06 smithi130 bash[51302]: debug 2023-08-28T00:39:06.720+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:06.726311+0000) 2023-08-28T00:39:08.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:07 smithi130 bash[51302]: debug 2023-08-28T00:39:07.720+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:07.726602+0000) 2023-08-28T00:39:08.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:08 smithi006 bash[27292]: cluster 2023-08-28T00:39:06.892365+0000 mgr.y (mgr.14901) 4816 : cluster [DBG] pgmap v4823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:08.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:08 smithi006 bash[27292]: audit 2023-08-28T00:39:08.083788+0000 mon.a (mon.0) 3432 : audit [DBG] from='client.? 172.21.15.6:0/2510290063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:08 smithi006 bash[33918]: cluster 2023-08-28T00:39:06.892365+0000 mgr.y (mgr.14901) 4816 : cluster [DBG] pgmap v4823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:08 smithi006 bash[33918]: audit 2023-08-28T00:39:08.083788+0000 mon.a (mon.0) 3432 : audit [DBG] from='client.? 172.21.15.6:0/2510290063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:08.728 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:08 smithi130 bash[27579]: cluster 2023-08-28T00:39:06.892365+0000 mgr.y (mgr.14901) 4816 : cluster [DBG] pgmap v4823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:08.732 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:08 smithi130 bash[27579]: audit 2023-08-28T00:39:08.083788+0000 mon.a (mon.0) 3432 : audit [DBG] from='client.? 172.21.15.6:0/2510290063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:09.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:08 smithi130 bash[51302]: debug 2023-08-28T00:39:08.724+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:08.726958+0000) 2023-08-28T00:39:10.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:09 smithi130 bash[51302]: debug 2023-08-28T00:39:09.724+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:09.727262+0000) 2023-08-28T00:39:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:10 smithi006 bash[27292]: cluster 2023-08-28T00:39:08.893067+0000 mgr.y (mgr.14901) 4817 : cluster [DBG] pgmap v4824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:10 smithi006 bash[33918]: cluster 2023-08-28T00:39:08.893067+0000 mgr.y (mgr.14901) 4817 : cluster [DBG] pgmap v4824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:10.729 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:10 smithi130 bash[27579]: cluster 2023-08-28T00:39:08.893067+0000 mgr.y (mgr.14901) 4817 : cluster [DBG] pgmap v4824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:11.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:10 smithi130 bash[51302]: debug 2023-08-28T00:39:10.724+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:10.727583+0000) 2023-08-28T00:39:11.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:39:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:39:10] "GET /metrics HTTP/1.1" 200 193132 "" "Prometheus/2.33.4" 2023-08-28T00:39:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:39:10.793Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:39:10.794Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:11.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:11 smithi006 bash[27292]: audit 2023-08-28T00:39:10.518073+0000 mon.a (mon.0) 3433 : audit [DBG] from='client.? 172.21.15.6:0/4253764389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:11.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:11 smithi006 bash[33918]: audit 2023-08-28T00:39:10.518073+0000 mon.a (mon.0) 3433 : audit [DBG] from='client.? 172.21.15.6:0/4253764389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:11.729 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:11 smithi130 bash[27579]: audit 2023-08-28T00:39:10.518073+0000 mon.a (mon.0) 3433 : audit [DBG] from='client.? 172.21.15.6:0/4253764389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:12.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:11 smithi130 bash[51302]: debug 2023-08-28T00:39:11.724+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:11.727882+0000) 2023-08-28T00:39:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:12 smithi006 bash[33918]: cluster 2023-08-28T00:39:10.894175+0000 mgr.y (mgr.14901) 4818 : cluster [DBG] pgmap v4825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:12 smithi006 bash[27292]: cluster 2023-08-28T00:39:10.894175+0000 mgr.y (mgr.14901) 4818 : cluster [DBG] pgmap v4825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:39:12.374Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:39:12.376Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:12.729 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:12 smithi130 bash[27579]: cluster 2023-08-28T00:39:10.894175+0000 mgr.y (mgr.14901) 4818 : cluster [DBG] pgmap v4825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:13.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:12 smithi130 bash[51302]: debug 2023-08-28T00:39:12.724+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:12.728183+0000) 2023-08-28T00:39:13.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:13 smithi006 bash[33918]: audit 2023-08-28T00:39:12.936619+0000 mon.c (mon.2) 1868 : audit [DBG] from='client.? 172.21.15.6:0/2008788261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:13.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:13 smithi006 bash[27292]: audit 2023-08-28T00:39:12.936619+0000 mon.c (mon.2) 1868 : audit [DBG] from='client.? 172.21.15.6:0/2008788261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:13.730 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:13 smithi130 bash[27579]: audit 2023-08-28T00:39:12.936619+0000 mon.c (mon.2) 1868 : audit [DBG] from='client.? 172.21.15.6:0/2008788261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:14.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:13 smithi130 bash[51302]: debug 2023-08-28T00:39:13.724+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:13.728479+0000) 2023-08-28T00:39:14.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:14 smithi006 bash[27292]: cluster 2023-08-28T00:39:12.894647+0000 mgr.y (mgr.14901) 4819 : cluster [DBG] pgmap v4826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:39:14.255Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:39:14.257Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:14.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:14 smithi006 bash[33918]: cluster 2023-08-28T00:39:12.894647+0000 mgr.y (mgr.14901) 4819 : cluster [DBG] pgmap v4826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:14.730 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:14 smithi130 bash[27579]: cluster 2023-08-28T00:39:12.894647+0000 mgr.y (mgr.14901) 4819 : cluster [DBG] pgmap v4826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:15.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:14 smithi130 bash[51302]: debug 2023-08-28T00:39:14.724+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:14.728799+0000) 2023-08-28T00:39:15.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:15 smithi006 bash[27292]: audit 2023-08-28T00:39:15.338903+0000 mon.a (mon.0) 3434 : audit [DBG] from='client.? 172.21.15.6:0/3122214037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:15.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:15 smithi006 bash[33918]: audit 2023-08-28T00:39:15.338903+0000 mon.a (mon.0) 3434 : audit [DBG] from='client.? 172.21.15.6:0/3122214037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:15.730 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:15 smithi130 bash[27579]: audit 2023-08-28T00:39:15.338903+0000 mon.a (mon.0) 3434 : audit [DBG] from='client.? 172.21.15.6:0/3122214037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:16.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:15 smithi130 bash[51302]: debug 2023-08-28T00:39:15.724+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:15.729086+0000) 2023-08-28T00:39:16.731 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:16 smithi130 bash[27579]: cluster 2023-08-28T00:39:14.895692+0000 mgr.y (mgr.14901) 4820 : cluster [DBG] pgmap v4827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:16.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:16 smithi006 bash[33918]: cluster 2023-08-28T00:39:14.895692+0000 mgr.y (mgr.14901) 4820 : cluster [DBG] pgmap v4827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:16.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:16 smithi006 bash[27292]: cluster 2023-08-28T00:39:14.895692+0000 mgr.y (mgr.14901) 4820 : cluster [DBG] pgmap v4827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:17.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:16 smithi130 bash[51302]: debug 2023-08-28T00:39:16.724+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:16.729374+0000) 2023-08-28T00:39:18.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:17 smithi130 bash[51302]: debug 2023-08-28T00:39:17.724+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:17.729673+0000) 2023-08-28T00:39:18.732 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:18 smithi130 bash[27579]: cluster 2023-08-28T00:39:16.896286+0000 mgr.y (mgr.14901) 4821 : cluster [DBG] pgmap v4828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:18.732 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:18 smithi130 bash[27579]: audit 2023-08-28T00:39:17.751967+0000 mon.c (mon.2) 1869 : audit [DBG] from='client.? 172.21.15.6:0/493370313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:18.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:18 smithi006 bash[33918]: cluster 2023-08-28T00:39:16.896286+0000 mgr.y (mgr.14901) 4821 : cluster [DBG] pgmap v4828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:18.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:18 smithi006 bash[33918]: audit 2023-08-28T00:39:17.751967+0000 mon.c (mon.2) 1869 : audit [DBG] from='client.? 172.21.15.6:0/493370313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:18.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:18 smithi006 bash[27292]: cluster 2023-08-28T00:39:16.896286+0000 mgr.y (mgr.14901) 4821 : cluster [DBG] pgmap v4828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:18.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:18 smithi006 bash[27292]: audit 2023-08-28T00:39:17.751967+0000 mon.c (mon.2) 1869 : audit [DBG] from='client.? 172.21.15.6:0/493370313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:19.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:18 smithi130 bash[51302]: debug 2023-08-28T00:39:18.724+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:18.729986+0000) 2023-08-28T00:39:20.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:19 smithi130 bash[51302]: debug 2023-08-28T00:39:19.724+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:19.730268+0000) 2023-08-28T00:39:20.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:20 smithi006 bash[33918]: cluster 2023-08-28T00:39:18.896869+0000 mgr.y (mgr.14901) 4822 : cluster [DBG] pgmap v4829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:20.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:20 smithi006 bash[33918]: audit 2023-08-28T00:39:20.164085+0000 mon.a (mon.0) 3435 : audit [DBG] from='client.? 172.21.15.6:0/2177556314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:20.756 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:20 smithi006 bash[27292]: cluster 2023-08-28T00:39:18.896869+0000 mgr.y (mgr.14901) 4822 : cluster [DBG] pgmap v4829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:20.756 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:20 smithi006 bash[27292]: audit 2023-08-28T00:39:20.164085+0000 mon.a (mon.0) 3435 : audit [DBG] from='client.? 172.21.15.6:0/2177556314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:20.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:20 smithi130 bash[51302]: debug 2023-08-28T00:39:20.724+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:20.730501+0000) 2023-08-28T00:39:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:20 smithi130 bash[27579]: cluster 2023-08-28T00:39:18.896869+0000 mgr.y (mgr.14901) 4822 : cluster [DBG] pgmap v4829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:20 smithi130 bash[27579]: audit 2023-08-28T00:39:20.164085+0000 mon.a (mon.0) 3435 : audit [DBG] from='client.? 172.21.15.6:0/2177556314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:21.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:39:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:39:20] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:39:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:39:20.793Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:39:20.795Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:22.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:21 smithi130 bash[51302]: debug 2023-08-28T00:39:21.728+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:21.730755+0000) 2023-08-28T00:39:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:22 smithi006 bash[33918]: cluster 2023-08-28T00:39:20.897988+0000 mgr.y (mgr.14901) 4823 : cluster [DBG] pgmap v4830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:39:22.375Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:39:22.377Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:22 smithi006 bash[27292]: cluster 2023-08-28T00:39:20.897988+0000 mgr.y (mgr.14901) 4823 : cluster [DBG] pgmap v4830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:22 smithi130 bash[27579]: cluster 2023-08-28T00:39:20.897988+0000 mgr.y (mgr.14901) 4823 : cluster [DBG] pgmap v4830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:22.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:22 smithi130 bash[51302]: debug 2023-08-28T00:39:22.728+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:22.731079+0000) 2023-08-28T00:39:23.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:23 smithi130 bash[51302]: debug 2023-08-28T00:39:23.728+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:23.731432+0000) 2023-08-28T00:39:23.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:23 smithi130 bash[27579]: audit 2023-08-28T00:39:22.580295+0000 mon.a (mon.0) 3436 : audit [DBG] from='client.? 172.21.15.6:0/1638190076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:23.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:23 smithi006 bash[33918]: audit 2023-08-28T00:39:22.580295+0000 mon.a (mon.0) 3436 : audit [DBG] from='client.? 172.21.15.6:0/1638190076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:23.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:23 smithi006 bash[27292]: audit 2023-08-28T00:39:22.580295+0000 mon.a (mon.0) 3436 : audit [DBG] from='client.? 172.21.15.6:0/1638190076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:24.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:24 smithi006 bash[27292]: cluster 2023-08-28T00:39:22.898680+0000 mgr.y (mgr.14901) 4824 : cluster [DBG] pgmap v4831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:24.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:24 smithi006 bash[33918]: cluster 2023-08-28T00:39:22.898680+0000 mgr.y (mgr.14901) 4824 : cluster [DBG] pgmap v4831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:39:24.256Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:39:24.258Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:24 smithi130 bash[27579]: cluster 2023-08-28T00:39:22.898680+0000 mgr.y (mgr.14901) 4824 : cluster [DBG] pgmap v4831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:24.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:24 smithi130 bash[51302]: debug 2023-08-28T00:39:24.728+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:24.731721+0000) 2023-08-28T00:39:25.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:25 smithi130 bash[27579]: audit 2023-08-28T00:39:24.991941+0000 mon.c (mon.2) 1870 : audit [DBG] from='client.? 172.21.15.6:0/2115369297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:25.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:25 smithi130 bash[51302]: debug 2023-08-28T00:39:25.728+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:25.732058+0000) 2023-08-28T00:39:25.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:25 smithi006 bash[33918]: audit 2023-08-28T00:39:24.991941+0000 mon.c (mon.2) 1870 : audit [DBG] from='client.? 172.21.15.6:0/2115369297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:25.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:25 smithi006 bash[27292]: audit 2023-08-28T00:39:24.991941+0000 mon.c (mon.2) 1870 : audit [DBG] from='client.? 172.21.15.6:0/2115369297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:26 smithi130 bash[27579]: cluster 2023-08-28T00:39:24.899767+0000 mgr.y (mgr.14901) 4825 : cluster [DBG] pgmap v4832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:26.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:26 smithi130 bash[51302]: debug 2023-08-28T00:39:26.728+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:26.732408+0000) 2023-08-28T00:39:26.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:26 smithi006 bash[27292]: cluster 2023-08-28T00:39:24.899767+0000 mgr.y (mgr.14901) 4825 : cluster [DBG] pgmap v4832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:26.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:26 smithi006 bash[33918]: cluster 2023-08-28T00:39:24.899767+0000 mgr.y (mgr.14901) 4825 : cluster [DBG] pgmap v4832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:27.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:27 smithi130 bash[51302]: debug 2023-08-28T00:39:27.728+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:27.732685+0000) 2023-08-28T00:39:27.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:27 smithi130 bash[27579]: audit 2023-08-28T00:39:27.410392+0000 mon.a (mon.0) 3437 : audit [DBG] from='client.? 172.21.15.6:0/309600510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:27.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:27 smithi006 bash[27292]: audit 2023-08-28T00:39:27.410392+0000 mon.a (mon.0) 3437 : audit [DBG] from='client.? 172.21.15.6:0/309600510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:27.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:27 smithi006 bash[33918]: audit 2023-08-28T00:39:27.410392+0000 mon.a (mon.0) 3437 : audit [DBG] from='client.? 172.21.15.6:0/309600510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:28 smithi130 bash[27579]: cluster 2023-08-28T00:39:26.900463+0000 mgr.y (mgr.14901) 4826 : cluster [DBG] pgmap v4833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:28 smithi130 bash[51302]: debug 2023-08-28T00:39:28.728+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:28.733029+0000) 2023-08-28T00:39:28.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:28 smithi006 bash[33918]: cluster 2023-08-28T00:39:26.900463+0000 mgr.y (mgr.14901) 4826 : cluster [DBG] pgmap v4833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:28.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:28 smithi006 bash[27292]: cluster 2023-08-28T00:39:26.900463+0000 mgr.y (mgr.14901) 4826 : cluster [DBG] pgmap v4833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:30.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:29 smithi130 bash[51302]: debug 2023-08-28T00:39:29.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:29.733362+0000) 2023-08-28T00:39:30.798 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:39:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:39:30] "GET /metrics HTTP/1.1" 200 193132 "" "Prometheus/2.33.4" 2023-08-28T00:39:30.798 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:30 smithi006 bash[27292]: cluster 2023-08-28T00:39:28.901329+0000 mgr.y (mgr.14901) 4827 : cluster [DBG] pgmap v4834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:30.798 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:30 smithi006 bash[27292]: audit 2023-08-28T00:39:29.831385+0000 mon.c (mon.2) 1871 : audit [DBG] from='client.? 172.21.15.6:0/2334951188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:30.799 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:30 smithi006 bash[33918]: cluster 2023-08-28T00:39:28.901329+0000 mgr.y (mgr.14901) 4827 : cluster [DBG] pgmap v4834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:30.799 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:30 smithi006 bash[33918]: audit 2023-08-28T00:39:29.831385+0000 mon.c (mon.2) 1871 : audit [DBG] from='client.? 172.21.15.6:0/2334951188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:30 smithi130 bash[27579]: cluster 2023-08-28T00:39:28.901329+0000 mgr.y (mgr.14901) 4827 : cluster [DBG] pgmap v4834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:30 smithi130 bash[27579]: audit 2023-08-28T00:39:29.831385+0000 mon.c (mon.2) 1871 : audit [DBG] from='client.? 172.21.15.6:0/2334951188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:30.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:30 smithi130 bash[51302]: debug 2023-08-28T00:39:30.728+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:30.733684+0000) 2023-08-28T00:39:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:39:30.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:39:30.796Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:32.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:31 smithi130 bash[51302]: debug 2023-08-28T00:39:31.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:31.734050+0000) 2023-08-28T00:39:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:32 smithi006 bash[27292]: cluster 2023-08-28T00:39:30.902640+0000 mgr.y (mgr.14901) 4828 : cluster [DBG] pgmap v4835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:32 smithi006 bash[27292]: audit 2023-08-28T00:39:32.242552+0000 mon.a (mon.0) 3438 : audit [DBG] from='client.? 172.21.15.6:0/43858251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:32.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:39:32.376Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:32.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:39:32.378Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:32.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:32 smithi006 bash[33918]: cluster 2023-08-28T00:39:30.902640+0000 mgr.y (mgr.14901) 4828 : cluster [DBG] pgmap v4835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:32.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:32 smithi006 bash[33918]: audit 2023-08-28T00:39:32.242552+0000 mon.a (mon.0) 3438 : audit [DBG] from='client.? 172.21.15.6:0/43858251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:32 smithi130 bash[51302]: debug 2023-08-28T00:39:32.728+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:32.734366+0000) 2023-08-28T00:39:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:32 smithi130 bash[27579]: cluster 2023-08-28T00:39:30.902640+0000 mgr.y (mgr.14901) 4828 : cluster [DBG] pgmap v4835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:32 smithi130 bash[27579]: audit 2023-08-28T00:39:32.242552+0000 mon.a (mon.0) 3438 : audit [DBG] from='client.? 172.21.15.6:0/43858251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:34.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:33 smithi130 bash[51302]: debug 2023-08-28T00:39:33.728+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:33.734716+0000) 2023-08-28T00:39:34.536 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:39:34.257Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:34.537 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:39:34.259Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:34 smithi130 bash[27579]: cluster 2023-08-28T00:39:32.903466+0000 mgr.y (mgr.14901) 4829 : cluster [DBG] pgmap v4836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:34 smithi130 bash[51302]: debug 2023-08-28T00:39:34.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:34.735025+0000) 2023-08-28T00:39:34.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:34 smithi006 bash[27292]: cluster 2023-08-28T00:39:32.903466+0000 mgr.y (mgr.14901) 4829 : cluster [DBG] pgmap v4836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:34.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:34 smithi006 bash[33918]: cluster 2023-08-28T00:39:32.903466+0000 mgr.y (mgr.14901) 4829 : cluster [DBG] pgmap v4836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:35.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:35 smithi130 bash[27579]: audit 2023-08-28T00:39:34.658624+0000 mon.a (mon.0) 3439 : audit [DBG] from='client.? 172.21.15.6:0/3709490873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:35.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:35 smithi130 bash[51302]: debug 2023-08-28T00:39:35.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:35.735286+0000) 2023-08-28T00:39:35.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:35 smithi006 bash[27292]: audit 2023-08-28T00:39:34.658624+0000 mon.a (mon.0) 3439 : audit [DBG] from='client.? 172.21.15.6:0/3709490873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:35.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:35 smithi006 bash[33918]: audit 2023-08-28T00:39:34.658624+0000 mon.a (mon.0) 3439 : audit [DBG] from='client.? 172.21.15.6:0/3709490873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:36 smithi130 bash[27579]: cluster 2023-08-28T00:39:34.904608+0000 mgr.y (mgr.14901) 4830 : cluster [DBG] pgmap v4837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:36 smithi130 bash[51302]: debug 2023-08-28T00:39:36.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:36.735652+0000) 2023-08-28T00:39:36.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:36 smithi006 bash[27292]: cluster 2023-08-28T00:39:34.904608+0000 mgr.y (mgr.14901) 4830 : cluster [DBG] pgmap v4837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:36 smithi006 bash[33918]: cluster 2023-08-28T00:39:34.904608+0000 mgr.y (mgr.14901) 4830 : cluster [DBG] pgmap v4837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:37.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:37 smithi130 bash[27579]: audit 2023-08-28T00:39:37.074399+0000 mon.c (mon.2) 1872 : audit [DBG] from='client.? 172.21.15.6:0/3640156078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:37 smithi130 bash[51302]: debug 2023-08-28T00:39:37.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:37.735999+0000) 2023-08-28T00:39:37.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:37 smithi006 bash[27292]: audit 2023-08-28T00:39:37.074399+0000 mon.c (mon.2) 1872 : audit [DBG] from='client.? 172.21.15.6:0/3640156078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:37.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:37 smithi006 bash[33918]: audit 2023-08-28T00:39:37.074399+0000 mon.c (mon.2) 1872 : audit [DBG] from='client.? 172.21.15.6:0/3640156078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:38 smithi130 bash[27579]: cluster 2023-08-28T00:39:36.905269+0000 mgr.y (mgr.14901) 4831 : cluster [DBG] pgmap v4838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:38 smithi130 bash[51302]: debug 2023-08-28T00:39:38.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:38.736377+0000) 2023-08-28T00:39:38.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:38 smithi006 bash[33918]: cluster 2023-08-28T00:39:36.905269+0000 mgr.y (mgr.14901) 4831 : cluster [DBG] pgmap v4838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:38.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:38 smithi006 bash[27292]: cluster 2023-08-28T00:39:36.905269+0000 mgr.y (mgr.14901) 4831 : cluster [DBG] pgmap v4838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:39.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:39 smithi130 bash[27579]: audit 2023-08-28T00:39:39.483104+0000 mon.a (mon.0) 3440 : audit [DBG] from='client.? 172.21.15.6:0/114055152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:39 smithi130 bash[51302]: debug 2023-08-28T00:39:39.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:39.736740+0000) 2023-08-28T00:39:39.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:39 smithi006 bash[33918]: audit 2023-08-28T00:39:39.483104+0000 mon.a (mon.0) 3440 : audit [DBG] from='client.? 172.21.15.6:0/114055152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:39.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:39 smithi006 bash[27292]: audit 2023-08-28T00:39:39.483104+0000 mon.a (mon.0) 3440 : audit [DBG] from='client.? 172.21.15.6:0/114055152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:40 smithi130 bash[27579]: cluster 2023-08-28T00:39:38.906049+0000 mgr.y (mgr.14901) 4832 : cluster [DBG] pgmap v4839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:40 smithi130 bash[51302]: debug 2023-08-28T00:39:40.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:40.737073+0000) 2023-08-28T00:39:40.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:39:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:39:40] "GET /metrics HTTP/1.1" 200 193132 "" "Prometheus/2.33.4" 2023-08-28T00:39:40.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:40 smithi006 bash[33918]: cluster 2023-08-28T00:39:38.906049+0000 mgr.y (mgr.14901) 4832 : cluster [DBG] pgmap v4839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:40.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:40 smithi006 bash[27292]: cluster 2023-08-28T00:39:38.906049+0000 mgr.y (mgr.14901) 4832 : cluster [DBG] pgmap v4839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:40.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:39:40.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:40.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:39:40.797Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:42.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:41 smithi130 bash[51302]: debug 2023-08-28T00:39:41.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:41.737437+0000) 2023-08-28T00:39:42.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:42 smithi006 bash[27292]: cluster 2023-08-28T00:39:40.907222+0000 mgr.y (mgr.14901) 4833 : cluster [DBG] pgmap v4840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:42.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:42 smithi006 bash[27292]: audit 2023-08-28T00:39:41.901109+0000 mon.a (mon.0) 3441 : audit [DBG] from='client.? 172.21.15.6:0/1769852129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:42.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:42 smithi006 bash[33918]: cluster 2023-08-28T00:39:40.907222+0000 mgr.y (mgr.14901) 4833 : cluster [DBG] pgmap v4840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:42.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:42 smithi006 bash[33918]: audit 2023-08-28T00:39:41.901109+0000 mon.a (mon.0) 3441 : audit [DBG] from='client.? 172.21.15.6:0/1769852129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:42.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:39:42.376Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:42.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:39:42.378Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:42 smithi130 bash[27579]: cluster 2023-08-28T00:39:40.907222+0000 mgr.y (mgr.14901) 4833 : cluster [DBG] pgmap v4840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:42 smithi130 bash[27579]: audit 2023-08-28T00:39:41.901109+0000 mon.a (mon.0) 3441 : audit [DBG] from='client.? 172.21.15.6:0/1769852129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:42 smithi130 bash[51302]: debug 2023-08-28T00:39:42.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:42.737763+0000) 2023-08-28T00:39:44.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:43 smithi130 bash[51302]: debug 2023-08-28T00:39:43.736+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:43.738093+0000) 2023-08-28T00:39:44.580 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:39:44.258Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:44.580 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:39:44.259Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:44.580 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:44 smithi006 bash[27292]: cluster 2023-08-28T00:39:42.907898+0000 mgr.y (mgr.14901) 4834 : cluster [DBG] pgmap v4841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:44.580 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:44 smithi006 bash[27292]: audit 2023-08-28T00:39:44.323526+0000 mon.c (mon.2) 1873 : audit [DBG] from='client.? 172.21.15.6:0/2185662666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:44 smithi130 bash[51302]: debug 2023-08-28T00:39:44.736+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:44.738451+0000) 2023-08-28T00:39:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:44 smithi130 bash[27579]: cluster 2023-08-28T00:39:42.907898+0000 mgr.y (mgr.14901) 4834 : cluster [DBG] pgmap v4841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:44 smithi130 bash[27579]: audit 2023-08-28T00:39:44.323526+0000 mon.c (mon.2) 1873 : audit [DBG] from='client.? 172.21.15.6:0/2185662666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:44.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:44 smithi006 bash[33918]: cluster 2023-08-28T00:39:42.907898+0000 mgr.y (mgr.14901) 4834 : cluster [DBG] pgmap v4841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:44.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:44 smithi006 bash[33918]: audit 2023-08-28T00:39:44.323526+0000 mon.c (mon.2) 1873 : audit [DBG] from='client.? 172.21.15.6:0/2185662666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:46.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:45 smithi130 bash[51302]: debug 2023-08-28T00:39:45.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:45.738817+0000) 2023-08-28T00:39:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:46 smithi130 bash[27579]: cluster 2023-08-28T00:39:44.909067+0000 mgr.y (mgr.14901) 4835 : cluster [DBG] pgmap v4842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:46.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:46 smithi130 bash[51302]: debug 2023-08-28T00:39:46.736+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:46.739177+0000) 2023-08-28T00:39:46.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:46 smithi006 bash[27292]: cluster 2023-08-28T00:39:44.909067+0000 mgr.y (mgr.14901) 4835 : cluster [DBG] pgmap v4842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:46.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:46 smithi006 bash[33918]: cluster 2023-08-28T00:39:44.909067+0000 mgr.y (mgr.14901) 4835 : cluster [DBG] pgmap v4842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:47.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:47 smithi130 bash[27579]: audit 2023-08-28T00:39:46.742924+0000 mon.a (mon.0) 3442 : audit [DBG] from='client.? 172.21.15.6:0/815367255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:47.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:47 smithi130 bash[51302]: debug 2023-08-28T00:39:47.736+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:47.739461+0000) 2023-08-28T00:39:47.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:47 smithi006 bash[33918]: audit 2023-08-28T00:39:46.742924+0000 mon.a (mon.0) 3442 : audit [DBG] from='client.? 172.21.15.6:0/815367255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:47.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:47 smithi006 bash[27292]: audit 2023-08-28T00:39:46.742924+0000 mon.a (mon.0) 3442 : audit [DBG] from='client.? 172.21.15.6:0/815367255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:48 smithi130 bash[27579]: cluster 2023-08-28T00:39:46.909727+0000 mgr.y (mgr.14901) 4836 : cluster [DBG] pgmap v4843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:48.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:48 smithi130 bash[51302]: debug 2023-08-28T00:39:48.736+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:48.739775+0000) 2023-08-28T00:39:48.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:48 smithi006 bash[27292]: cluster 2023-08-28T00:39:46.909727+0000 mgr.y (mgr.14901) 4836 : cluster [DBG] pgmap v4843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:48.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:48 smithi006 bash[33918]: cluster 2023-08-28T00:39:46.909727+0000 mgr.y (mgr.14901) 4836 : cluster [DBG] pgmap v4843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:49.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:49 smithi130 bash[27579]: audit 2023-08-28T00:39:49.160314+0000 mon.c (mon.2) 1874 : audit [DBG] from='client.? 172.21.15.6:0/3112681904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:49.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:49 smithi130 bash[51302]: debug 2023-08-28T00:39:49.736+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:49.740146+0000) 2023-08-28T00:39:49.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:49 smithi006 bash[33918]: audit 2023-08-28T00:39:49.160314+0000 mon.c (mon.2) 1874 : audit [DBG] from='client.? 172.21.15.6:0/3112681904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:49.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:49 smithi006 bash[27292]: audit 2023-08-28T00:39:49.160314+0000 mon.c (mon.2) 1874 : audit [DBG] from='client.? 172.21.15.6:0/3112681904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:50 smithi130 bash[27579]: cluster 2023-08-28T00:39:48.910398+0000 mgr.y (mgr.14901) 4837 : cluster [DBG] pgmap v4844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:50.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:50 smithi130 bash[51302]: debug 2023-08-28T00:39:50.740+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:50.740480+0000) 2023-08-28T00:39:50.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:39:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:39:50] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:39:50.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:50 smithi006 bash[33918]: cluster 2023-08-28T00:39:48.910398+0000 mgr.y (mgr.14901) 4837 : cluster [DBG] pgmap v4844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:50.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:50 smithi006 bash[27292]: cluster 2023-08-28T00:39:48.910398+0000 mgr.y (mgr.14901) 4837 : cluster [DBG] pgmap v4844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:50.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:39:50.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:50.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:39:50.798Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:52.030 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:51 smithi130 bash[51302]: debug 2023-08-28T00:39:51.736+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:51.740884+0000) 2023-08-28T00:39:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:52 smithi130 bash[27579]: cluster 2023-08-28T00:39:50.911558+0000 mgr.y (mgr.14901) 4838 : cluster [DBG] pgmap v4845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:52.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:52 smithi130 bash[27579]: audit 2023-08-28T00:39:51.577627+0000 mon.c (mon.2) 1875 : audit [DBG] from='client.? 172.21.15.6:0/2288992207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:52.380 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:52 smithi006 bash[33918]: cluster 2023-08-28T00:39:50.911558+0000 mgr.y (mgr.14901) 4838 : cluster [DBG] pgmap v4845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:52.380 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:52 smithi006 bash[33918]: audit 2023-08-28T00:39:51.577627+0000 mon.c (mon.2) 1875 : audit [DBG] from='client.? 172.21.15.6:0/2288992207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:52.380 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:52 smithi006 bash[27292]: cluster 2023-08-28T00:39:50.911558+0000 mgr.y (mgr.14901) 4838 : cluster [DBG] pgmap v4845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:52.381 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:52 smithi006 bash[27292]: audit 2023-08-28T00:39:51.577627+0000 mon.c (mon.2) 1875 : audit [DBG] from='client.? 172.21.15.6:0/2288992207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:39:52.377Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:39:52.379Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:52 smithi130 bash[51302]: debug 2023-08-28T00:39:52.740+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:52.741247+0000) 2023-08-28T00:39:54.034 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:53 smithi130 bash[51302]: debug 2023-08-28T00:39:53.735+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:53.741567+0000) 2023-08-28T00:39:54.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:54 smithi130 bash[27579]: cluster 2023-08-28T00:39:52.912225+0000 mgr.y (mgr.14901) 4839 : cluster [DBG] pgmap v4846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:54.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:54 smithi130 bash[27579]: audit 2023-08-28T00:39:53.992669+0000 mon.c (mon.2) 1876 : audit [DBG] from='client.? 172.21.15.6:0/4250177378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:54.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:54 smithi006 bash[27292]: cluster 2023-08-28T00:39:52.912225+0000 mgr.y (mgr.14901) 4839 : cluster [DBG] pgmap v4846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:54.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:54 smithi006 bash[27292]: audit 2023-08-28T00:39:53.992669+0000 mon.c (mon.2) 1876 : audit [DBG] from='client.? 172.21.15.6:0/4250177378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:54.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:54 smithi006 bash[33918]: cluster 2023-08-28T00:39:52.912225+0000 mgr.y (mgr.14901) 4839 : cluster [DBG] pgmap v4846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:54.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:54 smithi006 bash[33918]: audit 2023-08-28T00:39:53.992669+0000 mon.c (mon.2) 1876 : audit [DBG] from='client.? 172.21.15.6:0/4250177378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:54.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:39:54.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:54.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:39:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:39:54.261Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:39:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:54 smithi130 bash[51302]: debug 2023-08-28T00:39:54.739+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:54.741951+0000) 2023-08-28T00:39:56.039 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:55 smithi130 bash[51302]: debug 2023-08-28T00:39:55.735+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:55.742281+0000) 2023-08-28T00:39:56.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:56 smithi130 bash[27579]: cluster 2023-08-28T00:39:54.913380+0000 mgr.y (mgr.14901) 4840 : cluster [DBG] pgmap v4847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:56.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:56 smithi006 bash[27292]: cluster 2023-08-28T00:39:54.913380+0000 mgr.y (mgr.14901) 4840 : cluster [DBG] pgmap v4847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:56.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:56 smithi006 bash[33918]: cluster 2023-08-28T00:39:54.913380+0000 mgr.y (mgr.14901) 4840 : cluster [DBG] pgmap v4847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:57.047 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:56 smithi130 bash[51302]: debug 2023-08-28T00:39:56.739+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:56.742649+0000) 2023-08-28T00:39:57.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:57 smithi130 bash[27579]: audit 2023-08-28T00:39:56.407555+0000 mon.a (mon.0) 3443 : audit [DBG] from='client.? 172.21.15.6:0/2972517354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:57.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:57 smithi006 bash[27292]: audit 2023-08-28T00:39:56.407555+0000 mon.a (mon.0) 3443 : audit [DBG] from='client.? 172.21.15.6:0/2972517354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:57.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:57 smithi006 bash[33918]: audit 2023-08-28T00:39:56.407555+0000 mon.a (mon.0) 3443 : audit [DBG] from='client.? 172.21.15.6:0/2972517354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:58.052 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:57 smithi130 bash[51302]: debug 2023-08-28T00:39:57.739+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:57.742998+0000) 2023-08-28T00:39:58.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:58 smithi130 bash[27579]: cluster 2023-08-28T00:39:56.914048+0000 mgr.y (mgr.14901) 4841 : cluster [DBG] pgmap v4848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:58.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:58 smithi006 bash[27292]: cluster 2023-08-28T00:39:56.914048+0000 mgr.y (mgr.14901) 4841 : cluster [DBG] pgmap v4848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:58.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:58 smithi006 bash[33918]: cluster 2023-08-28T00:39:56.914048+0000 mgr.y (mgr.14901) 4841 : cluster [DBG] pgmap v4848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:39:59.055 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:58 smithi130 bash[51302]: debug 2023-08-28T00:39:58.739+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:58.743295+0000) 2023-08-28T00:39:59.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:39:59 smithi130 bash[27579]: audit 2023-08-28T00:39:58.829001+0000 mon.c (mon.2) 1877 : audit [DBG] from='client.? 172.21.15.6:0/2293102018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:59.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:39:59 smithi006 bash[27292]: audit 2023-08-28T00:39:58.829001+0000 mon.c (mon.2) 1877 : audit [DBG] from='client.? 172.21.15.6:0/2293102018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:59.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:39:59 smithi006 bash[33918]: audit 2023-08-28T00:39:58.829001+0000 mon.c (mon.2) 1877 : audit [DBG] from='client.? 172.21.15.6:0/2293102018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:00.064 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:39:59 smithi130 bash[51302]: debug 2023-08-28T00:39:59.739+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:59.743626+0000) 2023-08-28T00:40:00.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:00 smithi130 bash[27579]: cluster 2023-08-28T00:39:58.914708+0000 mgr.y (mgr.14901) 4842 : cluster [DBG] pgmap v4849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:00.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:00 smithi130 bash[27579]: audit 2023-08-28T00:39:59.710431+0000 mon.b (mon.1) 585 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:40:00.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:00 smithi130 bash[27579]: cluster 2023-08-28T00:40:00.000162+0000 mon.a (mon.0) 3444 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:40:00.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:00 smithi006 bash[27292]: cluster 2023-08-28T00:39:58.914708+0000 mgr.y (mgr.14901) 4842 : cluster [DBG] pgmap v4849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:00.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:00 smithi006 bash[27292]: audit 2023-08-28T00:39:59.710431+0000 mon.b (mon.1) 585 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:40:00.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:00 smithi006 bash[27292]: cluster 2023-08-28T00:40:00.000162+0000 mon.a (mon.0) 3444 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:40:00.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:00 smithi006 bash[33918]: cluster 2023-08-28T00:39:58.914708+0000 mgr.y (mgr.14901) 4842 : cluster [DBG] pgmap v4849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:00.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:00 smithi006 bash[33918]: audit 2023-08-28T00:39:59.710431+0000 mon.b (mon.1) 585 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:40:00.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:00 smithi006 bash[33918]: cluster 2023-08-28T00:40:00.000162+0000 mon.a (mon.0) 3444 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:40:01.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:00 smithi130 bash[51302]: debug 2023-08-28T00:40:00.739+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:00.743954+0000) 2023-08-28T00:40:01.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:40:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:40:00] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:40:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:40:00.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:40:00.799Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:02.071 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:01 smithi130 bash[51302]: debug 2023-08-28T00:40:01.743+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:01.744310+0000) 2023-08-28T00:40:02.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:02 smithi130 bash[27579]: cluster 2023-08-28T00:40:00.915827+0000 mgr.y (mgr.14901) 4843 : cluster [DBG] pgmap v4850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:02.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:02 smithi130 bash[27579]: audit 2023-08-28T00:40:01.247489+0000 mon.c (mon.2) 1878 : audit [DBG] from='client.? 172.21.15.6:0/4225270646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:02.381 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:02 smithi006 bash[33918]: cluster 2023-08-28T00:40:00.915827+0000 mgr.y (mgr.14901) 4843 : cluster [DBG] pgmap v4850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:02.381 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:02 smithi006 bash[33918]: audit 2023-08-28T00:40:01.247489+0000 mon.c (mon.2) 1878 : audit [DBG] from='client.? 172.21.15.6:0/4225270646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:02.381 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:02 smithi006 bash[27292]: cluster 2023-08-28T00:40:00.915827+0000 mgr.y (mgr.14901) 4843 : cluster [DBG] pgmap v4850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:02.381 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:02 smithi006 bash[27292]: audit 2023-08-28T00:40:01.247489+0000 mon.c (mon.2) 1878 : audit [DBG] from='client.? 172.21.15.6:0/4225270646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:40:02.377Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:40:02.379Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:02 smithi130 bash[51302]: debug 2023-08-28T00:40:02.739+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:02.744671+0000) 2023-08-28T00:40:04.075 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:03 smithi130 bash[51302]: debug 2023-08-28T00:40:03.743+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:03.745026+0000) 2023-08-28T00:40:04.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:04 smithi130 bash[27579]: cluster 2023-08-28T00:40:02.916519+0000 mgr.y (mgr.14901) 4844 : cluster [DBG] pgmap v4851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:04.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:04 smithi130 bash[27579]: audit 2023-08-28T00:40:03.660150+0000 mon.c (mon.2) 1879 : audit [DBG] from='client.? 172.21.15.6:0/2665265437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:04.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:04 smithi006 bash[33918]: cluster 2023-08-28T00:40:02.916519+0000 mgr.y (mgr.14901) 4844 : cluster [DBG] pgmap v4851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:04.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:04 smithi006 bash[33918]: audit 2023-08-28T00:40:03.660150+0000 mon.c (mon.2) 1879 : audit [DBG] from='client.? 172.21.15.6:0/2665265437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:40:04.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:04.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:40:04.262Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:04.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:04 smithi006 bash[27292]: cluster 2023-08-28T00:40:02.916519+0000 mgr.y (mgr.14901) 4844 : cluster [DBG] pgmap v4851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:04.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:04 smithi006 bash[27292]: audit 2023-08-28T00:40:03.660150+0000 mon.c (mon.2) 1879 : audit [DBG] from='client.? 172.21.15.6:0/2665265437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:04 smithi130 bash[51302]: debug 2023-08-28T00:40:04.739+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:04.745339+0000) 2023-08-28T00:40:06.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:05 smithi130 bash[51302]: debug 2023-08-28T00:40:05.739+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:05.745579+0000) 2023-08-28T00:40:06.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:06 smithi130 bash[51302]: debug 2023-08-28T00:40:06.743+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:06.745875+0000) 2023-08-28T00:40:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:06 smithi130 bash[27579]: cluster 2023-08-28T00:40:04.917747+0000 mgr.y (mgr.14901) 4845 : cluster [DBG] pgmap v4852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:06 smithi130 bash[27579]: audit 2023-08-28T00:40:05.600558+0000 mon.a (mon.0) 3445 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:40:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:06 smithi130 bash[27579]: audit 2023-08-28T00:40:06.068297+0000 mon.a (mon.0) 3446 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:40:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:06 smithi130 bash[27579]: audit 2023-08-28T00:40:06.085503+0000 mon.a (mon.0) 3447 : audit [DBG] from='client.? 172.21.15.6:0/2260983034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:06 smithi130 bash[27579]: audit 2023-08-28T00:40:06.338027+0000 mon.b (mon.1) 586 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:40:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:06 smithi130 bash[27579]: audit 2023-08-28T00:40:06.339919+0000 mon.b (mon.1) 587 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:40:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:06 smithi130 bash[27579]: audit 2023-08-28T00:40:06.349286+0000 mon.a (mon.0) 3448 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:40:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:06 smithi006 bash[27292]: cluster 2023-08-28T00:40:04.917747+0000 mgr.y (mgr.14901) 4845 : cluster [DBG] pgmap v4852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:06 smithi006 bash[27292]: audit 2023-08-28T00:40:05.600558+0000 mon.a (mon.0) 3445 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:40:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:06 smithi006 bash[27292]: audit 2023-08-28T00:40:06.068297+0000 mon.a (mon.0) 3446 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:40:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:06 smithi006 bash[27292]: audit 2023-08-28T00:40:06.085503+0000 mon.a (mon.0) 3447 : audit [DBG] from='client.? 172.21.15.6:0/2260983034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:06 smithi006 bash[27292]: audit 2023-08-28T00:40:06.338027+0000 mon.b (mon.1) 586 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:40:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:06 smithi006 bash[27292]: audit 2023-08-28T00:40:06.339919+0000 mon.b (mon.1) 587 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:40:06.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:06 smithi006 bash[27292]: audit 2023-08-28T00:40:06.349286+0000 mon.a (mon.0) 3448 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:40:06.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:06 smithi006 bash[33918]: cluster 2023-08-28T00:40:04.917747+0000 mgr.y (mgr.14901) 4845 : cluster [DBG] pgmap v4852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:06.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:06 smithi006 bash[33918]: audit 2023-08-28T00:40:05.600558+0000 mon.a (mon.0) 3445 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:40:06.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:06 smithi006 bash[33918]: audit 2023-08-28T00:40:06.068297+0000 mon.a (mon.0) 3446 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:40:06.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:06 smithi006 bash[33918]: audit 2023-08-28T00:40:06.085503+0000 mon.a (mon.0) 3447 : audit [DBG] from='client.? 172.21.15.6:0/2260983034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:06.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:06 smithi006 bash[33918]: audit 2023-08-28T00:40:06.338027+0000 mon.b (mon.1) 586 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:40:06.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:06 smithi006 bash[33918]: audit 2023-08-28T00:40:06.339919+0000 mon.b (mon.1) 587 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:40:06.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:06 smithi006 bash[33918]: audit 2023-08-28T00:40:06.349286+0000 mon.a (mon.0) 3448 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:40:08.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:07 smithi130 bash[51302]: debug 2023-08-28T00:40:07.739+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:07.746206+0000) 2023-08-28T00:40:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:08 smithi130 bash[27579]: cluster 2023-08-28T00:40:06.918495+0000 mgr.y (mgr.14901) 4846 : cluster [DBG] pgmap v4853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:08 smithi130 bash[27579]: audit 2023-08-28T00:40:08.501147+0000 mon.c (mon.2) 1880 : audit [DBG] from='client.? 172.21.15.6:0/1414012998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:08.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:08 smithi130 bash[51302]: debug 2023-08-28T00:40:08.739+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:08.746541+0000) 2023-08-28T00:40:08.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:08 smithi006 bash[33918]: cluster 2023-08-28T00:40:06.918495+0000 mgr.y (mgr.14901) 4846 : cluster [DBG] pgmap v4853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:08.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:08 smithi006 bash[33918]: audit 2023-08-28T00:40:08.501147+0000 mon.c (mon.2) 1880 : audit [DBG] from='client.? 172.21.15.6:0/1414012998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:08.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:08 smithi006 bash[27292]: cluster 2023-08-28T00:40:06.918495+0000 mgr.y (mgr.14901) 4846 : cluster [DBG] pgmap v4853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:08.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:08 smithi006 bash[27292]: audit 2023-08-28T00:40:08.501147+0000 mon.c (mon.2) 1880 : audit [DBG] from='client.? 172.21.15.6:0/1414012998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:10.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:09 smithi130 bash[51302]: debug 2023-08-28T00:40:09.743+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:09.746916+0000) 2023-08-28T00:40:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:10 smithi130 bash[27579]: cluster 2023-08-28T00:40:08.919205+0000 mgr.y (mgr.14901) 4847 : cluster [DBG] pgmap v4854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:10.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:10 smithi130 bash[51302]: debug 2023-08-28T00:40:10.743+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:10.747254+0000) 2023-08-28T00:40:10.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:40:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:40:10] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:40:10.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:10 smithi006 bash[33918]: cluster 2023-08-28T00:40:08.919205+0000 mgr.y (mgr.14901) 4847 : cluster [DBG] pgmap v4854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:10.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:10 smithi006 bash[27292]: cluster 2023-08-28T00:40:08.919205+0000 mgr.y (mgr.14901) 4847 : cluster [DBG] pgmap v4854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:10.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:40:10.798Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:10.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:40:10.800Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:11.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:11 smithi006 bash[33918]: audit 2023-08-28T00:40:10.919621+0000 mon.a (mon.0) 3449 : audit [DBG] from='client.? 172.21.15.6:0/4201092712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:11.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:11 smithi006 bash[27292]: audit 2023-08-28T00:40:10.919621+0000 mon.a (mon.0) 3449 : audit [DBG] from='client.? 172.21.15.6:0/4201092712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:12.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:11 smithi130 bash[27579]: audit 2023-08-28T00:40:10.919621+0000 mon.a (mon.0) 3449 : audit [DBG] from='client.? 172.21.15.6:0/4201092712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:12.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:11 smithi130 bash[51302]: debug 2023-08-28T00:40:11.743+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:11.747622+0000) 2023-08-28T00:40:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:12 smithi006 bash[33918]: cluster 2023-08-28T00:40:10.920215+0000 mgr.y (mgr.14901) 4848 : cluster [DBG] pgmap v4855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:40:12.377Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:40:12.379Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:12.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:12 smithi006 bash[27292]: cluster 2023-08-28T00:40:10.920215+0000 mgr.y (mgr.14901) 4848 : cluster [DBG] pgmap v4855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:13.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:12 smithi130 bash[27579]: cluster 2023-08-28T00:40:10.920215+0000 mgr.y (mgr.14901) 4848 : cluster [DBG] pgmap v4855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:13.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:12 smithi130 bash[51302]: debug 2023-08-28T00:40:12.743+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:12.747904+0000) 2023-08-28T00:40:13.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:13 smithi006 bash[33918]: audit 2023-08-28T00:40:13.338172+0000 mon.a (mon.0) 3450 : audit [DBG] from='client.? 172.21.15.6:0/1508854507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:13.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:13 smithi006 bash[27292]: audit 2023-08-28T00:40:13.338172+0000 mon.a (mon.0) 3450 : audit [DBG] from='client.? 172.21.15.6:0/1508854507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:14.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:13 smithi130 bash[27579]: audit 2023-08-28T00:40:13.338172+0000 mon.a (mon.0) 3450 : audit [DBG] from='client.? 172.21.15.6:0/1508854507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:14.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:13 smithi130 bash[51302]: debug 2023-08-28T00:40:13.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:13.748239+0000) 2023-08-28T00:40:14.634 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:40:14.261Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:14.634 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:40:14.263Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:14.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:14 smithi006 bash[33918]: cluster 2023-08-28T00:40:12.920900+0000 mgr.y (mgr.14901) 4849 : cluster [DBG] pgmap v4856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:14.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:14 smithi006 bash[27292]: cluster 2023-08-28T00:40:12.920900+0000 mgr.y (mgr.14901) 4849 : cluster [DBG] pgmap v4856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:14 smithi130 bash[27579]: cluster 2023-08-28T00:40:12.920900+0000 mgr.y (mgr.14901) 4849 : cluster [DBG] pgmap v4856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:15.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:14 smithi130 bash[51302]: debug 2023-08-28T00:40:14.743+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:14.748615+0000) 2023-08-28T00:40:15.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:15 smithi006 bash[33918]: cluster 2023-08-28T00:40:14.921969+0000 mgr.y (mgr.14901) 4850 : cluster [DBG] pgmap v4857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:15.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:15 smithi006 bash[27292]: cluster 2023-08-28T00:40:14.921969+0000 mgr.y (mgr.14901) 4850 : cluster [DBG] pgmap v4857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:16.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:15 smithi130 bash[27579]: cluster 2023-08-28T00:40:14.921969+0000 mgr.y (mgr.14901) 4850 : cluster [DBG] pgmap v4857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:16.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:15 smithi130 bash[51302]: debug 2023-08-28T00:40:15.743+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:15.748962+0000) 2023-08-28T00:40:16.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:16 smithi006 bash[33918]: audit 2023-08-28T00:40:15.751906+0000 mon.a (mon.0) 3451 : audit [DBG] from='client.? 172.21.15.6:0/209312349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:16.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:16 smithi006 bash[27292]: audit 2023-08-28T00:40:15.751906+0000 mon.a (mon.0) 3451 : audit [DBG] from='client.? 172.21.15.6:0/209312349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:17.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:16 smithi130 bash[27579]: audit 2023-08-28T00:40:15.751906+0000 mon.a (mon.0) 3451 : audit [DBG] from='client.? 172.21.15.6:0/209312349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:17.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:16 smithi130 bash[51302]: debug 2023-08-28T00:40:16.743+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:16.749305+0000) 2023-08-28T00:40:17.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:17 smithi006 bash[33918]: cluster 2023-08-28T00:40:16.922643+0000 mgr.y (mgr.14901) 4851 : cluster [DBG] pgmap v4858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:17.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:17 smithi006 bash[27292]: cluster 2023-08-28T00:40:16.922643+0000 mgr.y (mgr.14901) 4851 : cluster [DBG] pgmap v4858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:18.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:17 smithi130 bash[27579]: cluster 2023-08-28T00:40:16.922643+0000 mgr.y (mgr.14901) 4851 : cluster [DBG] pgmap v4858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:18.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:17 smithi130 bash[51302]: debug 2023-08-28T00:40:17.743+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:17.749613+0000) 2023-08-28T00:40:18.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:18 smithi006 bash[27292]: audit 2023-08-28T00:40:18.160057+0000 mon.c (mon.2) 1881 : audit [DBG] from='client.? 172.21.15.6:0/3438710933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:18.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:18 smithi006 bash[33918]: audit 2023-08-28T00:40:18.160057+0000 mon.c (mon.2) 1881 : audit [DBG] from='client.? 172.21.15.6:0/3438710933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:18 smithi130 bash[27579]: audit 2023-08-28T00:40:18.160057+0000 mon.c (mon.2) 1881 : audit [DBG] from='client.? 172.21.15.6:0/3438710933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:19.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:18 smithi130 bash[51302]: debug 2023-08-28T00:40:18.743+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:18.749880+0000) 2023-08-28T00:40:19.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:19 smithi006 bash[27292]: cluster 2023-08-28T00:40:18.923338+0000 mgr.y (mgr.14901) 4852 : cluster [DBG] pgmap v4859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:20.239 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:19 smithi006 bash[33918]: cluster 2023-08-28T00:40:18.923338+0000 mgr.y (mgr.14901) 4852 : cluster [DBG] pgmap v4859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:20.239 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:19 smithi130 bash[27579]: cluster 2023-08-28T00:40:18.923338+0000 mgr.y (mgr.14901) 4852 : cluster [DBG] pgmap v4859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:20.240 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:19 smithi130 bash[51302]: debug 2023-08-28T00:40:19.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:19.750152+0000) 2023-08-28T00:40:20.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:20 smithi006 bash[27292]: audit 2023-08-28T00:40:20.582016+0000 mon.a (mon.0) 3452 : audit [DBG] from='client.? 172.21.15.6:0/1318729287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:20.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:40:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:40:20] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:40:20.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:20 smithi006 bash[33918]: audit 2023-08-28T00:40:20.582016+0000 mon.a (mon.0) 3452 : audit [DBG] from='client.? 172.21.15.6:0/1318729287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:20.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:40:20.799Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:20.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:40:20.801Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:21.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:20 smithi130 bash[27579]: audit 2023-08-28T00:40:20.582016+0000 mon.a (mon.0) 3452 : audit [DBG] from='client.? 172.21.15.6:0/1318729287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:21.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:20 smithi130 bash[51302]: debug 2023-08-28T00:40:20.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:20.750429+0000) 2023-08-28T00:40:22.039 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:21 smithi130 bash[51302]: debug 2023-08-28T00:40:21.743+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:21.750697+0000) 2023-08-28T00:40:22.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:22 smithi130 bash[27579]: cluster 2023-08-28T00:40:20.924522+0000 mgr.y (mgr.14901) 4853 : cluster [DBG] pgmap v4860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:22.381 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:22 smithi006 bash[33918]: cluster 2023-08-28T00:40:20.924522+0000 mgr.y (mgr.14901) 4853 : cluster [DBG] pgmap v4860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:22.382 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:22 smithi006 bash[27292]: cluster 2023-08-28T00:40:20.924522+0000 mgr.y (mgr.14901) 4853 : cluster [DBG] pgmap v4860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:40:22.378Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:40:22.379Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:23.045 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:22 smithi130 bash[51302]: debug 2023-08-28T00:40:22.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:22.751033+0000) 2023-08-28T00:40:23.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:23 smithi130 bash[27579]: audit 2023-08-28T00:40:23.001615+0000 mon.c (mon.2) 1882 : audit [DBG] from='client.? 172.21.15.6:0/3578874409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:23.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:23 smithi006 bash[33918]: audit 2023-08-28T00:40:23.001615+0000 mon.c (mon.2) 1882 : audit [DBG] from='client.? 172.21.15.6:0/3578874409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:23.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:23 smithi006 bash[27292]: audit 2023-08-28T00:40:23.001615+0000 mon.c (mon.2) 1882 : audit [DBG] from='client.? 172.21.15.6:0/3578874409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:24.049 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:23 smithi130 bash[51302]: debug 2023-08-28T00:40:23.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:23.751262+0000) 2023-08-28T00:40:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:24 smithi130 bash[27579]: cluster 2023-08-28T00:40:22.925159+0000 mgr.y (mgr.14901) 4854 : cluster [DBG] pgmap v4861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:24.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:24 smithi006 bash[33918]: cluster 2023-08-28T00:40:22.925159+0000 mgr.y (mgr.14901) 4854 : cluster [DBG] pgmap v4861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:24.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:24 smithi006 bash[27292]: cluster 2023-08-28T00:40:22.925159+0000 mgr.y (mgr.14901) 4854 : cluster [DBG] pgmap v4861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:40:24.262Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:24.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:40:24.264Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:24 smithi130 bash[51302]: debug 2023-08-28T00:40:24.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:24.751491+0000) 2023-08-28T00:40:26.053 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:25 smithi130 bash[51302]: debug 2023-08-28T00:40:25.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:25.751781+0000) 2023-08-28T00:40:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:26 smithi130 bash[27579]: cluster 2023-08-28T00:40:24.926291+0000 mgr.y (mgr.14901) 4855 : cluster [DBG] pgmap v4862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:26 smithi130 bash[27579]: audit 2023-08-28T00:40:25.417561+0000 mon.a (mon.0) 3453 : audit [DBG] from='client.? 172.21.15.6:0/1278003258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:26.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:26 smithi006 bash[27292]: cluster 2023-08-28T00:40:24.926291+0000 mgr.y (mgr.14901) 4855 : cluster [DBG] pgmap v4862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:26.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:26 smithi006 bash[27292]: audit 2023-08-28T00:40:25.417561+0000 mon.a (mon.0) 3453 : audit [DBG] from='client.? 172.21.15.6:0/1278003258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:26 smithi006 bash[33918]: cluster 2023-08-28T00:40:24.926291+0000 mgr.y (mgr.14901) 4855 : cluster [DBG] pgmap v4862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:26 smithi006 bash[33918]: audit 2023-08-28T00:40:25.417561+0000 mon.a (mon.0) 3453 : audit [DBG] from='client.? 172.21.15.6:0/1278003258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:27.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:26 smithi130 bash[51302]: debug 2023-08-28T00:40:26.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:26.752062+0000) 2023-08-28T00:40:28.059 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:27 smithi130 bash[51302]: debug 2023-08-28T00:40:27.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:27.752315+0000) 2023-08-28T00:40:28.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:28 smithi130 bash[27579]: cluster 2023-08-28T00:40:26.926986+0000 mgr.y (mgr.14901) 4856 : cluster [DBG] pgmap v4863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:28.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:28 smithi130 bash[27579]: audit 2023-08-28T00:40:27.828974+0000 mon.a (mon.0) 3454 : audit [DBG] from='client.? 172.21.15.6:0/3607276801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:28.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:28 smithi006 bash[33918]: cluster 2023-08-28T00:40:26.926986+0000 mgr.y (mgr.14901) 4856 : cluster [DBG] pgmap v4863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:28.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:28 smithi006 bash[33918]: audit 2023-08-28T00:40:27.828974+0000 mon.a (mon.0) 3454 : audit [DBG] from='client.? 172.21.15.6:0/3607276801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:28.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:28 smithi006 bash[27292]: cluster 2023-08-28T00:40:26.926986+0000 mgr.y (mgr.14901) 4856 : cluster [DBG] pgmap v4863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:28.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:28 smithi006 bash[27292]: audit 2023-08-28T00:40:27.828974+0000 mon.a (mon.0) 3454 : audit [DBG] from='client.? 172.21.15.6:0/3607276801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:29.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:28 smithi130 bash[51302]: debug 2023-08-28T00:40:28.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:28.752586+0000) 2023-08-28T00:40:30.063 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:29 smithi130 bash[51302]: debug 2023-08-28T00:40:29.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:29.752988+0000) 2023-08-28T00:40:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:30 smithi130 bash[27579]: cluster 2023-08-28T00:40:28.927640+0000 mgr.y (mgr.14901) 4857 : cluster [DBG] pgmap v4864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:30.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:30 smithi006 bash[27292]: cluster 2023-08-28T00:40:28.927640+0000 mgr.y (mgr.14901) 4857 : cluster [DBG] pgmap v4864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:30.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:30 smithi006 bash[33918]: cluster 2023-08-28T00:40:28.927640+0000 mgr.y (mgr.14901) 4857 : cluster [DBG] pgmap v4864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:31.072 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:40:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:40:30] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:40:31.073 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:40:30.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:31.073 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:40:30.802Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:31.073 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:30 smithi130 bash[51302]: debug 2023-08-28T00:40:30.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:30.753303+0000) 2023-08-28T00:40:31.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:31 smithi130 bash[27579]: audit 2023-08-28T00:40:30.249799+0000 mon.c (mon.2) 1883 : audit [DBG] from='client.? 172.21.15.6:0/2173937590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:31.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:31 smithi006 bash[33918]: audit 2023-08-28T00:40:30.249799+0000 mon.c (mon.2) 1883 : audit [DBG] from='client.? 172.21.15.6:0/2173937590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:31.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:31 smithi006 bash[27292]: audit 2023-08-28T00:40:30.249799+0000 mon.c (mon.2) 1883 : audit [DBG] from='client.? 172.21.15.6:0/2173937590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:32.076 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:31 smithi130 bash[51302]: debug 2023-08-28T00:40:31.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:31.753562+0000) 2023-08-28T00:40:32.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:32 smithi130 bash[27579]: cluster 2023-08-28T00:40:30.928777+0000 mgr.y (mgr.14901) 4858 : cluster [DBG] pgmap v4865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:32.382 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:32 smithi006 bash[27292]: cluster 2023-08-28T00:40:30.928777+0000 mgr.y (mgr.14901) 4858 : cluster [DBG] pgmap v4865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:32.382 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:32 smithi006 bash[33918]: cluster 2023-08-28T00:40:30.928777+0000 mgr.y (mgr.14901) 4858 : cluster [DBG] pgmap v4865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:40:32.378Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:40:32.380Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:33.084 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:32 smithi130 bash[51302]: debug 2023-08-28T00:40:32.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:32.753844+0000) 2023-08-28T00:40:33.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:33 smithi130 bash[27579]: audit 2023-08-28T00:40:32.661041+0000 mon.a (mon.0) 3455 : audit [DBG] from='client.? 172.21.15.6:0/2163582498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:33.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:33 smithi006 bash[33918]: audit 2023-08-28T00:40:32.661041+0000 mon.a (mon.0) 3455 : audit [DBG] from='client.? 172.21.15.6:0/2163582498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:33.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:33 smithi006 bash[27292]: audit 2023-08-28T00:40:32.661041+0000 mon.a (mon.0) 3455 : audit [DBG] from='client.? 172.21.15.6:0/2163582498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:34.087 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:33 smithi130 bash[51302]: debug 2023-08-28T00:40:33.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:33.754131+0000) 2023-08-28T00:40:34.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:34 smithi130 bash[27579]: cluster 2023-08-28T00:40:32.929587+0000 mgr.y (mgr.14901) 4859 : cluster [DBG] pgmap v4866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:34.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:34 smithi006 bash[33918]: cluster 2023-08-28T00:40:32.929587+0000 mgr.y (mgr.14901) 4859 : cluster [DBG] pgmap v4866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:34.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:34 smithi006 bash[27292]: cluster 2023-08-28T00:40:32.929587+0000 mgr.y (mgr.14901) 4859 : cluster [DBG] pgmap v4866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:40:34.262Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:34.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:40:34.264Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:35.095 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:34 smithi130 bash[51302]: debug 2023-08-28T00:40:34.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:34.754449+0000) 2023-08-28T00:40:35.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:35 smithi130 bash[27579]: audit 2023-08-28T00:40:35.081441+0000 mon.a (mon.0) 3456 : audit [DBG] from='client.? 172.21.15.6:0/2456720846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:35.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:35 smithi006 bash[33918]: audit 2023-08-28T00:40:35.081441+0000 mon.a (mon.0) 3456 : audit [DBG] from='client.? 172.21.15.6:0/2456720846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:35.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:35 smithi006 bash[27292]: audit 2023-08-28T00:40:35.081441+0000 mon.a (mon.0) 3456 : audit [DBG] from='client.? 172.21.15.6:0/2456720846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:36.098 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:35 smithi130 bash[51302]: debug 2023-08-28T00:40:35.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:35.754738+0000) 2023-08-28T00:40:36.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:36 smithi130 bash[27579]: cluster 2023-08-28T00:40:34.930682+0000 mgr.y (mgr.14901) 4860 : cluster [DBG] pgmap v4867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:36.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:36 smithi006 bash[27292]: cluster 2023-08-28T00:40:34.930682+0000 mgr.y (mgr.14901) 4860 : cluster [DBG] pgmap v4867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:36.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:36 smithi006 bash[33918]: cluster 2023-08-28T00:40:34.930682+0000 mgr.y (mgr.14901) 4860 : cluster [DBG] pgmap v4867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:37.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:36 smithi130 bash[51302]: debug 2023-08-28T00:40:36.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:36.755048+0000) 2023-08-28T00:40:38.103 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:37 smithi130 bash[51302]: debug 2023-08-28T00:40:37.755+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:37.755373+0000) 2023-08-28T00:40:38.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:38 smithi130 bash[27579]: cluster 2023-08-28T00:40:36.931333+0000 mgr.y (mgr.14901) 4861 : cluster [DBG] pgmap v4868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:38.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:38 smithi130 bash[27579]: audit 2023-08-28T00:40:37.495833+0000 mon.a (mon.0) 3457 : audit [DBG] from='client.? 172.21.15.6:0/99250424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:38.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:38 smithi006 bash[27292]: cluster 2023-08-28T00:40:36.931333+0000 mgr.y (mgr.14901) 4861 : cluster [DBG] pgmap v4868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:38.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:38 smithi006 bash[27292]: audit 2023-08-28T00:40:37.495833+0000 mon.a (mon.0) 3457 : audit [DBG] from='client.? 172.21.15.6:0/99250424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:38.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:38 smithi006 bash[33918]: cluster 2023-08-28T00:40:36.931333+0000 mgr.y (mgr.14901) 4861 : cluster [DBG] pgmap v4868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:38.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:38 smithi006 bash[33918]: audit 2023-08-28T00:40:37.495833+0000 mon.a (mon.0) 3457 : audit [DBG] from='client.? 172.21.15.6:0/99250424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:39.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:38 smithi130 bash[51302]: debug 2023-08-28T00:40:38.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:38.755596+0000) 2023-08-28T00:40:40.114 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:39 smithi130 bash[51302]: debug 2023-08-28T00:40:39.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:39.755905+0000) 2023-08-28T00:40:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:40 smithi130 bash[27579]: cluster 2023-08-28T00:40:38.931999+0000 mgr.y (mgr.14901) 4862 : cluster [DBG] pgmap v4869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:40.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:40 smithi130 bash[27579]: audit 2023-08-28T00:40:39.913335+0000 mon.a (mon.0) 3458 : audit [DBG] from='client.? 172.21.15.6:0/965477955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:40.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:40 smithi006 bash[27292]: cluster 2023-08-28T00:40:38.931999+0000 mgr.y (mgr.14901) 4862 : cluster [DBG] pgmap v4869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:40.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:40 smithi006 bash[27292]: audit 2023-08-28T00:40:39.913335+0000 mon.a (mon.0) 3458 : audit [DBG] from='client.? 172.21.15.6:0/965477955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:40 smithi006 bash[33918]: cluster 2023-08-28T00:40:38.931999+0000 mgr.y (mgr.14901) 4862 : cluster [DBG] pgmap v4869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:40 smithi006 bash[33918]: audit 2023-08-28T00:40:39.913335+0000 mon.a (mon.0) 3458 : audit [DBG] from='client.? 172.21.15.6:0/965477955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:41.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:40 smithi130 bash[51302]: debug 2023-08-28T00:40:40.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:40.756215+0000) 2023-08-28T00:40:41.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:40:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:40:40] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:40:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:40:40.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:40:40.802Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:42.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:41 smithi130 bash[51302]: debug 2023-08-28T00:40:41.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:41.756487+0000) 2023-08-28T00:40:42.382 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:42 smithi006 bash[27292]: cluster 2023-08-28T00:40:40.933096+0000 mgr.y (mgr.14901) 4863 : cluster [DBG] pgmap v4870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:42.383 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:42 smithi006 bash[33918]: cluster 2023-08-28T00:40:40.933096+0000 mgr.y (mgr.14901) 4863 : cluster [DBG] pgmap v4870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:42 smithi130 bash[27579]: cluster 2023-08-28T00:40:40.933096+0000 mgr.y (mgr.14901) 4863 : cluster [DBG] pgmap v4870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:40:42.379Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:40:42.381Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:43.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:42 smithi130 bash[51302]: debug 2023-08-28T00:40:42.755+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:42.756781+0000) 2023-08-28T00:40:43.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:43 smithi006 bash[27292]: audit 2023-08-28T00:40:42.331653+0000 mon.a (mon.0) 3459 : audit [DBG] from='client.? 172.21.15.6:0/4233684568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:43.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:43 smithi006 bash[33918]: audit 2023-08-28T00:40:42.331653+0000 mon.a (mon.0) 3459 : audit [DBG] from='client.? 172.21.15.6:0/4233684568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:43 smithi130 bash[27579]: audit 2023-08-28T00:40:42.331653+0000 mon.a (mon.0) 3459 : audit [DBG] from='client.? 172.21.15.6:0/4233684568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:44.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:43 smithi130 bash[51302]: debug 2023-08-28T00:40:43.755+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:43.757095+0000) 2023-08-28T00:40:44.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:44 smithi006 bash[33918]: cluster 2023-08-28T00:40:42.933736+0000 mgr.y (mgr.14901) 4864 : cluster [DBG] pgmap v4871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:44.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:44 smithi006 bash[27292]: cluster 2023-08-28T00:40:42.933736+0000 mgr.y (mgr.14901) 4864 : cluster [DBG] pgmap v4871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:40:44.263Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:40:44.265Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:44 smithi130 bash[27579]: cluster 2023-08-28T00:40:42.933736+0000 mgr.y (mgr.14901) 4864 : cluster [DBG] pgmap v4871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:44 smithi130 bash[51302]: debug 2023-08-28T00:40:44.755+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:44.757377+0000) 2023-08-28T00:40:45.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:45 smithi006 bash[33918]: audit 2023-08-28T00:40:44.749231+0000 mon.c (mon.2) 1884 : audit [DBG] from='client.? 172.21.15.6:0/3813582871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:45.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:45 smithi006 bash[27292]: audit 2023-08-28T00:40:44.749231+0000 mon.c (mon.2) 1884 : audit [DBG] from='client.? 172.21.15.6:0/3813582871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:45.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:45 smithi130 bash[27579]: audit 2023-08-28T00:40:44.749231+0000 mon.c (mon.2) 1884 : audit [DBG] from='client.? 172.21.15.6:0/3813582871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:46.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:45 smithi130 bash[51302]: debug 2023-08-28T00:40:45.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:45.757672+0000) 2023-08-28T00:40:46.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:46 smithi006 bash[27292]: cluster 2023-08-28T00:40:44.934888+0000 mgr.y (mgr.14901) 4865 : cluster [DBG] pgmap v4872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:46.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:46 smithi006 bash[33918]: cluster 2023-08-28T00:40:44.934888+0000 mgr.y (mgr.14901) 4865 : cluster [DBG] pgmap v4872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:46 smithi130 bash[27579]: cluster 2023-08-28T00:40:44.934888+0000 mgr.y (mgr.14901) 4865 : cluster [DBG] pgmap v4872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:47.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:46 smithi130 bash[51302]: debug 2023-08-28T00:40:46.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:46.757956+0000) 2023-08-28T00:40:48.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:47 smithi130 bash[51302]: debug 2023-08-28T00:40:47.755+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:47.758252+0000) 2023-08-28T00:40:48.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:48 smithi006 bash[33918]: cluster 2023-08-28T00:40:46.935544+0000 mgr.y (mgr.14901) 4866 : cluster [DBG] pgmap v4873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:48.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:48 smithi006 bash[33918]: audit 2023-08-28T00:40:47.169454+0000 mon.a (mon.0) 3460 : audit [DBG] from='client.? 172.21.15.6:0/1287917273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:48.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:48 smithi006 bash[27292]: cluster 2023-08-28T00:40:46.935544+0000 mgr.y (mgr.14901) 4866 : cluster [DBG] pgmap v4873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:48.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:48 smithi006 bash[27292]: audit 2023-08-28T00:40:47.169454+0000 mon.a (mon.0) 3460 : audit [DBG] from='client.? 172.21.15.6:0/1287917273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:48 smithi130 bash[27579]: cluster 2023-08-28T00:40:46.935544+0000 mgr.y (mgr.14901) 4866 : cluster [DBG] pgmap v4873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:48 smithi130 bash[27579]: audit 2023-08-28T00:40:47.169454+0000 mon.a (mon.0) 3460 : audit [DBG] from='client.? 172.21.15.6:0/1287917273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:49.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:48 smithi130 bash[51302]: debug 2023-08-28T00:40:48.755+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:48.758526+0000) 2023-08-28T00:40:50.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:49 smithi130 bash[51302]: debug 2023-08-28T00:40:49.754+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:49.758798+0000) 2023-08-28T00:40:50.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:50 smithi006 bash[27292]: cluster 2023-08-28T00:40:48.936253+0000 mgr.y (mgr.14901) 4867 : cluster [DBG] pgmap v4874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:50.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:50 smithi006 bash[27292]: audit 2023-08-28T00:40:49.587276+0000 mon.c (mon.2) 1885 : audit [DBG] from='client.? 172.21.15.6:0/3303985609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:50.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:50 smithi006 bash[33918]: cluster 2023-08-28T00:40:48.936253+0000 mgr.y (mgr.14901) 4867 : cluster [DBG] pgmap v4874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:50.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:50 smithi006 bash[33918]: audit 2023-08-28T00:40:49.587276+0000 mon.c (mon.2) 1885 : audit [DBG] from='client.? 172.21.15.6:0/3303985609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:50 smithi130 bash[27579]: cluster 2023-08-28T00:40:48.936253+0000 mgr.y (mgr.14901) 4867 : cluster [DBG] pgmap v4874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:50 smithi130 bash[27579]: audit 2023-08-28T00:40:49.587276+0000 mon.c (mon.2) 1885 : audit [DBG] from='client.? 172.21.15.6:0/3303985609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:51.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:50 smithi130 bash[51302]: debug 2023-08-28T00:40:50.754+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:50.759010+0000) 2023-08-28T00:40:51.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:40:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:40:50] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:40:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:40:50.801Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:40:50.803Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:52.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:51 smithi130 bash[51302]: debug 2023-08-28T00:40:51.754+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:51.759293+0000) 2023-08-28T00:40:52.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:52 smithi006 bash[27292]: cluster 2023-08-28T00:40:50.937390+0000 mgr.y (mgr.14901) 4868 : cluster [DBG] pgmap v4875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:52.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:52 smithi006 bash[27292]: audit 2023-08-28T00:40:52.001759+0000 mon.a (mon.0) 3461 : audit [DBG] from='client.? 172.21.15.6:0/3462157853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:52.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:52 smithi006 bash[33918]: cluster 2023-08-28T00:40:50.937390+0000 mgr.y (mgr.14901) 4868 : cluster [DBG] pgmap v4875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:52.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:52 smithi006 bash[33918]: audit 2023-08-28T00:40:52.001759+0000 mon.a (mon.0) 3461 : audit [DBG] from='client.? 172.21.15.6:0/3462157853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:52.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:40:52.380Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:52.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:40:52.382Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:52 smithi130 bash[27579]: cluster 2023-08-28T00:40:50.937390+0000 mgr.y (mgr.14901) 4868 : cluster [DBG] pgmap v4875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:52 smithi130 bash[27579]: audit 2023-08-28T00:40:52.001759+0000 mon.a (mon.0) 3461 : audit [DBG] from='client.? 172.21.15.6:0/3462157853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:52 smithi130 bash[51302]: debug 2023-08-28T00:40:52.754+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:52.759656+0000) 2023-08-28T00:40:54.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:53 smithi130 bash[51302]: debug 2023-08-28T00:40:53.758+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:53.759948+0000) 2023-08-28T00:40:54.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:54 smithi006 bash[27292]: cluster 2023-08-28T00:40:52.938076+0000 mgr.y (mgr.14901) 4869 : cluster [DBG] pgmap v4876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:54.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:54 smithi006 bash[33918]: cluster 2023-08-28T00:40:52.938076+0000 mgr.y (mgr.14901) 4869 : cluster [DBG] pgmap v4876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:40:54.263Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:40:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:40:54.265Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:40:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:54 smithi130 bash[27579]: cluster 2023-08-28T00:40:52.938076+0000 mgr.y (mgr.14901) 4869 : cluster [DBG] pgmap v4876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:54 smithi130 bash[51302]: debug 2023-08-28T00:40:54.754+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:54.760287+0000) 2023-08-28T00:40:55.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:55 smithi006 bash[33918]: audit 2023-08-28T00:40:54.415489+0000 mon.c (mon.2) 1886 : audit [DBG] from='client.? 172.21.15.6:0/3929898963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:55.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:55 smithi006 bash[27292]: audit 2023-08-28T00:40:54.415489+0000 mon.c (mon.2) 1886 : audit [DBG] from='client.? 172.21.15.6:0/3929898963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:55.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:55 smithi130 bash[27579]: audit 2023-08-28T00:40:54.415489+0000 mon.c (mon.2) 1886 : audit [DBG] from='client.? 172.21.15.6:0/3929898963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:56.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:55 smithi130 bash[51302]: debug 2023-08-28T00:40:55.758+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:55.760551+0000) 2023-08-28T00:40:56.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:56 smithi006 bash[27292]: cluster 2023-08-28T00:40:54.939227+0000 mgr.y (mgr.14901) 4870 : cluster [DBG] pgmap v4877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:56.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:56 smithi006 bash[33918]: cluster 2023-08-28T00:40:54.939227+0000 mgr.y (mgr.14901) 4870 : cluster [DBG] pgmap v4877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:56 smithi130 bash[27579]: cluster 2023-08-28T00:40:54.939227+0000 mgr.y (mgr.14901) 4870 : cluster [DBG] pgmap v4877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:57.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:56 smithi130 bash[51302]: debug 2023-08-28T00:40:56.758+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:56.760875+0000) 2023-08-28T00:40:57.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:57 smithi006 bash[33918]: audit 2023-08-28T00:40:56.827636+0000 mon.c (mon.2) 1887 : audit [DBG] from='client.? 172.21.15.6:0/2581745733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:57.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:57 smithi006 bash[27292]: audit 2023-08-28T00:40:56.827636+0000 mon.c (mon.2) 1887 : audit [DBG] from='client.? 172.21.15.6:0/2581745733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:57.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:57 smithi130 bash[27579]: audit 2023-08-28T00:40:56.827636+0000 mon.c (mon.2) 1887 : audit [DBG] from='client.? 172.21.15.6:0/2581745733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:58.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:57 smithi130 bash[51302]: debug 2023-08-28T00:40:57.758+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:57.761211+0000) 2023-08-28T00:40:58.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:40:58 smithi006 bash[33918]: cluster 2023-08-28T00:40:56.940019+0000 mgr.y (mgr.14901) 4871 : cluster [DBG] pgmap v4878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:58.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:40:58 smithi006 bash[27292]: cluster 2023-08-28T00:40:56.940019+0000 mgr.y (mgr.14901) 4871 : cluster [DBG] pgmap v4878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:40:58 smithi130 bash[27579]: cluster 2023-08-28T00:40:56.940019+0000 mgr.y (mgr.14901) 4871 : cluster [DBG] pgmap v4878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:40:59.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:58 smithi130 bash[51302]: debug 2023-08-28T00:40:58.758+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:58.761469+0000) 2023-08-28T00:41:00.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:40:59 smithi130 bash[51302]: debug 2023-08-28T00:40:59.758+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:59.761773+0000) 2023-08-28T00:41:00.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:00 smithi006 bash[27292]: cluster 2023-08-28T00:40:58.940688+0000 mgr.y (mgr.14901) 4872 : cluster [DBG] pgmap v4879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:00.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:00 smithi006 bash[27292]: audit 2023-08-28T00:40:59.241203+0000 mon.a (mon.0) 3462 : audit [DBG] from='client.? 172.21.15.6:0/4076058499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:00.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:00 smithi006 bash[33918]: cluster 2023-08-28T00:40:58.940688+0000 mgr.y (mgr.14901) 4872 : cluster [DBG] pgmap v4879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:00.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:00 smithi006 bash[33918]: audit 2023-08-28T00:40:59.241203+0000 mon.a (mon.0) 3462 : audit [DBG] from='client.? 172.21.15.6:0/4076058499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:00 smithi130 bash[27579]: cluster 2023-08-28T00:40:58.940688+0000 mgr.y (mgr.14901) 4872 : cluster [DBG] pgmap v4879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:00 smithi130 bash[27579]: audit 2023-08-28T00:40:59.241203+0000 mon.a (mon.0) 3462 : audit [DBG] from='client.? 172.21.15.6:0/4076058499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:01.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:00 smithi130 bash[51302]: debug 2023-08-28T00:41:00.758+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:00.761987+0000) 2023-08-28T00:41:01.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:41:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:41:00] "GET /metrics HTTP/1.1" 200 193131 "" "Prometheus/2.33.4" 2023-08-28T00:41:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:41:00.801Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:41:00.803Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:02.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:01 smithi130 bash[51302]: debug 2023-08-28T00:41:01.758+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:01.762297+0000) 2023-08-28T00:41:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:41:02.380Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:41:02.382Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:02.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:02 smithi006 bash[27292]: cluster 2023-08-28T00:41:00.941856+0000 mgr.y (mgr.14901) 4873 : cluster [DBG] pgmap v4880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:02.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:02 smithi006 bash[27292]: audit 2023-08-28T00:41:01.653904+0000 mon.a (mon.0) 3463 : audit [DBG] from='client.? 172.21.15.6:0/489998626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:02.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:02 smithi006 bash[33918]: cluster 2023-08-28T00:41:00.941856+0000 mgr.y (mgr.14901) 4873 : cluster [DBG] pgmap v4880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:02.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:02 smithi006 bash[33918]: audit 2023-08-28T00:41:01.653904+0000 mon.a (mon.0) 3463 : audit [DBG] from='client.? 172.21.15.6:0/489998626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:02 smithi130 bash[27579]: cluster 2023-08-28T00:41:00.941856+0000 mgr.y (mgr.14901) 4873 : cluster [DBG] pgmap v4880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:02 smithi130 bash[27579]: audit 2023-08-28T00:41:01.653904+0000 mon.a (mon.0) 3463 : audit [DBG] from='client.? 172.21.15.6:0/489998626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:02 smithi130 bash[51302]: debug 2023-08-28T00:41:02.758+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:02.762611+0000) 2023-08-28T00:41:04.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:03 smithi130 bash[51302]: debug 2023-08-28T00:41:03.762+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:03.762923+0000) 2023-08-28T00:41:04.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:04 smithi006 bash[33918]: cluster 2023-08-28T00:41:02.942570+0000 mgr.y (mgr.14901) 4874 : cluster [DBG] pgmap v4881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:04.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:04 smithi006 bash[33918]: audit 2023-08-28T00:41:04.058821+0000 mon.c (mon.2) 1888 : audit [DBG] from='client.? 172.21.15.6:0/1223144810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:41:04.264Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:41:04.266Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:04.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:04 smithi006 bash[27292]: cluster 2023-08-28T00:41:02.942570+0000 mgr.y (mgr.14901) 4874 : cluster [DBG] pgmap v4881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:04.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:04 smithi006 bash[27292]: audit 2023-08-28T00:41:04.058821+0000 mon.c (mon.2) 1888 : audit [DBG] from='client.? 172.21.15.6:0/1223144810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:04 smithi130 bash[27579]: cluster 2023-08-28T00:41:02.942570+0000 mgr.y (mgr.14901) 4874 : cluster [DBG] pgmap v4881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:04 smithi130 bash[27579]: audit 2023-08-28T00:41:04.058821+0000 mon.c (mon.2) 1888 : audit [DBG] from='client.? 172.21.15.6:0/1223144810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:04 smithi130 bash[51302]: debug 2023-08-28T00:41:04.762+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:04.763218+0000) 2023-08-28T00:41:06.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:05 smithi130 bash[51302]: debug 2023-08-28T00:41:05.762+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:05.763542+0000) 2023-08-28T00:41:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:06 smithi006 bash[33918]: cluster 2023-08-28T00:41:04.943716+0000 mgr.y (mgr.14901) 4875 : cluster [DBG] pgmap v4882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:06.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:06 smithi006 bash[27292]: cluster 2023-08-28T00:41:04.943716+0000 mgr.y (mgr.14901) 4875 : cluster [DBG] pgmap v4882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:06 smithi130 bash[27579]: cluster 2023-08-28T00:41:04.943716+0000 mgr.y (mgr.14901) 4875 : cluster [DBG] pgmap v4882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:07.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:06 smithi130 bash[51302]: debug 2023-08-28T00:41:06.762+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:06.763817+0000) 2023-08-28T00:41:07.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:07 smithi006 bash[33918]: audit 2023-08-28T00:41:06.360757+0000 mon.b (mon.1) 588 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:41:07.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:07 smithi006 bash[33918]: audit 2023-08-28T00:41:06.480880+0000 mon.a (mon.0) 3464 : audit [DBG] from='client.? 172.21.15.6:0/801152372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:07.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:07 smithi006 bash[27292]: audit 2023-08-28T00:41:06.360757+0000 mon.b (mon.1) 588 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:41:07.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:07 smithi006 bash[27292]: audit 2023-08-28T00:41:06.480880+0000 mon.a (mon.0) 3464 : audit [DBG] from='client.? 172.21.15.6:0/801152372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:07.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:07 smithi130 bash[27579]: audit 2023-08-28T00:41:06.360757+0000 mon.b (mon.1) 588 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:41:07.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:07 smithi130 bash[27579]: audit 2023-08-28T00:41:06.480880+0000 mon.a (mon.0) 3464 : audit [DBG] from='client.? 172.21.15.6:0/801152372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:08.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:07 smithi130 bash[51302]: debug 2023-08-28T00:41:07.762+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:07.764115+0000) 2023-08-28T00:41:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:08 smithi130 bash[27579]: cluster 2023-08-28T00:41:06.944487+0000 mgr.y (mgr.14901) 4876 : cluster [DBG] pgmap v4883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:08 smithi006 bash[33918]: cluster 2023-08-28T00:41:06.944487+0000 mgr.y (mgr.14901) 4876 : cluster [DBG] pgmap v4883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:08.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:08 smithi006 bash[27292]: cluster 2023-08-28T00:41:06.944487+0000 mgr.y (mgr.14901) 4876 : cluster [DBG] pgmap v4883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:09.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:08 smithi130 bash[51302]: debug 2023-08-28T00:41:08.762+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:08.764399+0000) 2023-08-28T00:41:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:09 smithi130 bash[27579]: audit 2023-08-28T00:41:08.901356+0000 mon.c (mon.2) 1889 : audit [DBG] from='client.? 172.21.15.6:0/2450797328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:09.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:09 smithi006 bash[33918]: audit 2023-08-28T00:41:08.901356+0000 mon.c (mon.2) 1889 : audit [DBG] from='client.? 172.21.15.6:0/2450797328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:09.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:09 smithi006 bash[27292]: audit 2023-08-28T00:41:08.901356+0000 mon.c (mon.2) 1889 : audit [DBG] from='client.? 172.21.15.6:0/2450797328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:10.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:09 smithi130 bash[51302]: debug 2023-08-28T00:41:09.762+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:09.764682+0000) 2023-08-28T00:41:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:10 smithi130 bash[27579]: cluster 2023-08-28T00:41:08.945206+0000 mgr.y (mgr.14901) 4877 : cluster [DBG] pgmap v4884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:10 smithi006 bash[33918]: cluster 2023-08-28T00:41:08.945206+0000 mgr.y (mgr.14901) 4877 : cluster [DBG] pgmap v4884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:10 smithi006 bash[27292]: cluster 2023-08-28T00:41:08.945206+0000 mgr.y (mgr.14901) 4877 : cluster [DBG] pgmap v4884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:11.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:10 smithi130 bash[51302]: debug 2023-08-28T00:41:10.762+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:10.764902+0000) 2023-08-28T00:41:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:41:10.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:41:10.804Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:11.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:41:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:41:10] "GET /metrics HTTP/1.1" 200 193131 "" "Prometheus/2.33.4" 2023-08-28T00:41:12.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:11 smithi130 bash[51302]: debug 2023-08-28T00:41:11.762+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:11.765213+0000) 2023-08-28T00:41:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:12 smithi130 bash[27579]: cluster 2023-08-28T00:41:10.946227+0000 mgr.y (mgr.14901) 4878 : cluster [DBG] pgmap v4885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:12 smithi130 bash[27579]: audit 2023-08-28T00:41:11.320624+0000 mon.a (mon.0) 3465 : audit [DBG] from='client.? 172.21.15.6:0/994816896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:41:12.381Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:41:12.382Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:12 smithi006 bash[27292]: cluster 2023-08-28T00:41:10.946227+0000 mgr.y (mgr.14901) 4878 : cluster [DBG] pgmap v4885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:12 smithi006 bash[27292]: audit 2023-08-28T00:41:11.320624+0000 mon.a (mon.0) 3465 : audit [DBG] from='client.? 172.21.15.6:0/994816896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:12.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:12 smithi006 bash[33918]: cluster 2023-08-28T00:41:10.946227+0000 mgr.y (mgr.14901) 4878 : cluster [DBG] pgmap v4885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:12.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:12 smithi006 bash[33918]: audit 2023-08-28T00:41:11.320624+0000 mon.a (mon.0) 3465 : audit [DBG] from='client.? 172.21.15.6:0/994816896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:13.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:12 smithi130 bash[51302]: debug 2023-08-28T00:41:12.762+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:12.765557+0000) 2023-08-28T00:41:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:13 smithi130 bash[27579]: audit 2023-08-28T00:41:12.231226+0000 mon.a (mon.0) 3466 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:41:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:13 smithi130 bash[27579]: audit 2023-08-28T00:41:12.690663+0000 mon.a (mon.0) 3467 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:41:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:13 smithi130 bash[27579]: audit 2023-08-28T00:41:12.960516+0000 mon.b (mon.1) 589 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:41:13.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:13 smithi130 bash[27579]: audit 2023-08-28T00:41:12.962638+0000 mon.b (mon.1) 590 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:41:13.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:13 smithi130 bash[27579]: audit 2023-08-28T00:41:12.972083+0000 mon.a (mon.0) 3468 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:41:13.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:13 smithi006 bash[27292]: audit 2023-08-28T00:41:12.231226+0000 mon.a (mon.0) 3466 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:41:13.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:13 smithi006 bash[27292]: audit 2023-08-28T00:41:12.690663+0000 mon.a (mon.0) 3467 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:41:13.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:13 smithi006 bash[27292]: audit 2023-08-28T00:41:12.960516+0000 mon.b (mon.1) 589 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:41:13.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:13 smithi006 bash[27292]: audit 2023-08-28T00:41:12.962638+0000 mon.b (mon.1) 590 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:41:13.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:13 smithi006 bash[27292]: audit 2023-08-28T00:41:12.972083+0000 mon.a (mon.0) 3468 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:41:13.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:13 smithi006 bash[33918]: audit 2023-08-28T00:41:12.231226+0000 mon.a (mon.0) 3466 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:41:13.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:13 smithi006 bash[33918]: audit 2023-08-28T00:41:12.690663+0000 mon.a (mon.0) 3467 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:41:13.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:13 smithi006 bash[33918]: audit 2023-08-28T00:41:12.960516+0000 mon.b (mon.1) 589 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:41:13.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:13 smithi006 bash[33918]: audit 2023-08-28T00:41:12.962638+0000 mon.b (mon.1) 590 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:41:13.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:13 smithi006 bash[33918]: audit 2023-08-28T00:41:12.972083+0000 mon.a (mon.0) 3468 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:41:14.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:13 smithi130 bash[51302]: debug 2023-08-28T00:41:13.762+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:13.765827+0000) 2023-08-28T00:41:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:14 smithi130 bash[27579]: cluster 2023-08-28T00:41:12.946921+0000 mgr.y (mgr.14901) 4879 : cluster [DBG] pgmap v4886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:14 smithi130 bash[27579]: audit 2023-08-28T00:41:13.748088+0000 mon.c (mon.2) 1890 : audit [DBG] from='client.? 172.21.15.6:0/1840116050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:14.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:14 smithi006 bash[33918]: cluster 2023-08-28T00:41:12.946921+0000 mgr.y (mgr.14901) 4879 : cluster [DBG] pgmap v4886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:14.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:14 smithi006 bash[33918]: audit 2023-08-28T00:41:13.748088+0000 mon.c (mon.2) 1890 : audit [DBG] from='client.? 172.21.15.6:0/1840116050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:41:14.264Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:41:14.266Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:14.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:14 smithi006 bash[27292]: cluster 2023-08-28T00:41:12.946921+0000 mgr.y (mgr.14901) 4879 : cluster [DBG] pgmap v4886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:14.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:14 smithi006 bash[27292]: audit 2023-08-28T00:41:13.748088+0000 mon.c (mon.2) 1890 : audit [DBG] from='client.? 172.21.15.6:0/1840116050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:15.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:14 smithi130 bash[51302]: debug 2023-08-28T00:41:14.762+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:14.766110+0000) 2023-08-28T00:41:16.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:15 smithi130 bash[51302]: debug 2023-08-28T00:41:15.762+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:15.766374+0000) 2023-08-28T00:41:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:16 smithi130 bash[27579]: cluster 2023-08-28T00:41:14.948046+0000 mgr.y (mgr.14901) 4880 : cluster [DBG] pgmap v4887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:16 smithi130 bash[27579]: audit 2023-08-28T00:41:16.193868+0000 mon.b (mon.1) 591 : audit [DBG] from='client.? 172.21.15.6:0/2396452201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:16 smithi006 bash[33918]: cluster 2023-08-28T00:41:14.948046+0000 mgr.y (mgr.14901) 4880 : cluster [DBG] pgmap v4887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:16 smithi006 bash[33918]: audit 2023-08-28T00:41:16.193868+0000 mon.b (mon.1) 591 : audit [DBG] from='client.? 172.21.15.6:0/2396452201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:16.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:16 smithi006 bash[27292]: cluster 2023-08-28T00:41:14.948046+0000 mgr.y (mgr.14901) 4880 : cluster [DBG] pgmap v4887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:16.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:16 smithi006 bash[27292]: audit 2023-08-28T00:41:16.193868+0000 mon.b (mon.1) 591 : audit [DBG] from='client.? 172.21.15.6:0/2396452201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:17.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:16 smithi130 bash[51302]: debug 2023-08-28T00:41:16.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:16.766715+0000) 2023-08-28T00:41:18.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:17 smithi130 bash[51302]: debug 2023-08-28T00:41:17.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:17.767003+0000) 2023-08-28T00:41:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:18 smithi130 bash[27579]: cluster 2023-08-28T00:41:16.948921+0000 mgr.y (mgr.14901) 4881 : cluster [DBG] pgmap v4888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:18.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:18 smithi006 bash[33918]: cluster 2023-08-28T00:41:16.948921+0000 mgr.y (mgr.14901) 4881 : cluster [DBG] pgmap v4888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:18 smithi006 bash[27292]: cluster 2023-08-28T00:41:16.948921+0000 mgr.y (mgr.14901) 4881 : cluster [DBG] pgmap v4888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:19.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:18 smithi130 bash[51302]: debug 2023-08-28T00:41:18.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:18.767230+0000) 2023-08-28T00:41:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:19 smithi130 bash[27579]: audit 2023-08-28T00:41:18.620146+0000 mon.c (mon.2) 1891 : audit [DBG] from='client.? 172.21.15.6:0/2239105177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:19.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:19 smithi006 bash[33918]: audit 2023-08-28T00:41:18.620146+0000 mon.c (mon.2) 1891 : audit [DBG] from='client.? 172.21.15.6:0/2239105177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:19.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:19 smithi006 bash[27292]: audit 2023-08-28T00:41:18.620146+0000 mon.c (mon.2) 1891 : audit [DBG] from='client.? 172.21.15.6:0/2239105177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:20.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:19 smithi130 bash[51302]: debug 2023-08-28T00:41:19.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:19.767537+0000) 2023-08-28T00:41:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:20 smithi130 bash[27579]: cluster 2023-08-28T00:41:18.949589+0000 mgr.y (mgr.14901) 4882 : cluster [DBG] pgmap v4889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:20 smithi006 bash[27292]: cluster 2023-08-28T00:41:18.949589+0000 mgr.y (mgr.14901) 4882 : cluster [DBG] pgmap v4889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:20.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:20 smithi006 bash[33918]: cluster 2023-08-28T00:41:18.949589+0000 mgr.y (mgr.14901) 4882 : cluster [DBG] pgmap v4889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:21.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:20 smithi130 bash[51302]: debug 2023-08-28T00:41:20.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:20.767781+0000) 2023-08-28T00:41:21.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:41:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:41:20] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:41:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:41:20.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:41:20.805Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:21 smithi130 bash[27579]: audit 2023-08-28T00:41:21.039072+0000 mon.c (mon.2) 1892 : audit [DBG] from='client.? 172.21.15.6:0/3450975973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:21.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:21 smithi006 bash[33918]: audit 2023-08-28T00:41:21.039072+0000 mon.c (mon.2) 1892 : audit [DBG] from='client.? 172.21.15.6:0/3450975973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:21.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:21 smithi006 bash[27292]: audit 2023-08-28T00:41:21.039072+0000 mon.c (mon.2) 1892 : audit [DBG] from='client.? 172.21.15.6:0/3450975973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:22.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:21 smithi130 bash[51302]: debug 2023-08-28T00:41:21.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:21.768091+0000) 2023-08-28T00:41:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:22 smithi130 bash[27579]: cluster 2023-08-28T00:41:20.950647+0000 mgr.y (mgr.14901) 4883 : cluster [DBG] pgmap v4890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:22 smithi006 bash[33918]: cluster 2023-08-28T00:41:20.950647+0000 mgr.y (mgr.14901) 4883 : cluster [DBG] pgmap v4890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:41:22.381Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:41:22.383Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:22.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:22 smithi006 bash[27292]: cluster 2023-08-28T00:41:20.950647+0000 mgr.y (mgr.14901) 4883 : cluster [DBG] pgmap v4890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:23.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:22 smithi130 bash[51302]: debug 2023-08-28T00:41:22.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:22.768411+0000) 2023-08-28T00:41:24.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:23 smithi130 bash[51302]: debug 2023-08-28T00:41:23.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:23.768749+0000) 2023-08-28T00:41:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:24 smithi130 bash[27579]: cluster 2023-08-28T00:41:22.951260+0000 mgr.y (mgr.14901) 4884 : cluster [DBG] pgmap v4891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:24 smithi130 bash[27579]: audit 2023-08-28T00:41:23.459875+0000 mon.c (mon.2) 1893 : audit [DBG] from='client.? 172.21.15.6:0/1043099994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:24.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:24 smithi006 bash[27292]: cluster 2023-08-28T00:41:22.951260+0000 mgr.y (mgr.14901) 4884 : cluster [DBG] pgmap v4891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:24.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:24 smithi006 bash[27292]: audit 2023-08-28T00:41:23.459875+0000 mon.c (mon.2) 1893 : audit [DBG] from='client.? 172.21.15.6:0/1043099994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:41:24.265Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:41:24.266Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:24.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:24 smithi006 bash[33918]: cluster 2023-08-28T00:41:22.951260+0000 mgr.y (mgr.14901) 4884 : cluster [DBG] pgmap v4891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:24.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:24 smithi006 bash[33918]: audit 2023-08-28T00:41:23.459875+0000 mon.c (mon.2) 1893 : audit [DBG] from='client.? 172.21.15.6:0/1043099994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:24 smithi130 bash[51302]: debug 2023-08-28T00:41:24.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:24.769026+0000) 2023-08-28T00:41:26.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:25 smithi130 bash[51302]: debug 2023-08-28T00:41:25.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:25.769294+0000) 2023-08-28T00:41:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:26 smithi130 bash[27579]: cluster 2023-08-28T00:41:24.952363+0000 mgr.y (mgr.14901) 4885 : cluster [DBG] pgmap v4892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:26 smithi130 bash[27579]: audit 2023-08-28T00:41:25.882339+0000 mon.a (mon.0) 3469 : audit [DBG] from='client.? 172.21.15.6:0/2404265127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:26 smithi006 bash[33918]: cluster 2023-08-28T00:41:24.952363+0000 mgr.y (mgr.14901) 4885 : cluster [DBG] pgmap v4892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:26.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:26 smithi006 bash[33918]: audit 2023-08-28T00:41:25.882339+0000 mon.a (mon.0) 3469 : audit [DBG] from='client.? 172.21.15.6:0/2404265127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:26.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:26 smithi006 bash[27292]: cluster 2023-08-28T00:41:24.952363+0000 mgr.y (mgr.14901) 4885 : cluster [DBG] pgmap v4892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:26.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:26 smithi006 bash[27292]: audit 2023-08-28T00:41:25.882339+0000 mon.a (mon.0) 3469 : audit [DBG] from='client.? 172.21.15.6:0/2404265127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:27.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:26 smithi130 bash[51302]: debug 2023-08-28T00:41:26.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:26.769580+0000) 2023-08-28T00:41:28.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:27 smithi130 bash[51302]: debug 2023-08-28T00:41:27.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:27.769846+0000) 2023-08-28T00:41:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:28 smithi130 bash[27579]: cluster 2023-08-28T00:41:26.953043+0000 mgr.y (mgr.14901) 4886 : cluster [DBG] pgmap v4893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:28.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:28 smithi006 bash[33918]: cluster 2023-08-28T00:41:26.953043+0000 mgr.y (mgr.14901) 4886 : cluster [DBG] pgmap v4893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:28.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:28 smithi006 bash[27292]: cluster 2023-08-28T00:41:26.953043+0000 mgr.y (mgr.14901) 4886 : cluster [DBG] pgmap v4893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:29.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:28 smithi130 bash[51302]: debug 2023-08-28T00:41:28.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:28.770127+0000) 2023-08-28T00:41:29.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:29 smithi130 bash[27579]: audit 2023-08-28T00:41:28.295298+0000 mon.c (mon.2) 1894 : audit [DBG] from='client.? 172.21.15.6:0/1463565380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:29.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:29 smithi006 bash[27292]: audit 2023-08-28T00:41:28.295298+0000 mon.c (mon.2) 1894 : audit [DBG] from='client.? 172.21.15.6:0/1463565380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:29.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:29 smithi006 bash[33918]: audit 2023-08-28T00:41:28.295298+0000 mon.c (mon.2) 1894 : audit [DBG] from='client.? 172.21.15.6:0/1463565380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:30.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:29 smithi130 bash[51302]: debug 2023-08-28T00:41:29.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:29.770438+0000) 2023-08-28T00:41:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:30 smithi130 bash[27579]: cluster 2023-08-28T00:41:28.953656+0000 mgr.y (mgr.14901) 4887 : cluster [DBG] pgmap v4894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:30.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:30 smithi006 bash[27292]: cluster 2023-08-28T00:41:28.953656+0000 mgr.y (mgr.14901) 4887 : cluster [DBG] pgmap v4894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:30.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:30 smithi006 bash[33918]: cluster 2023-08-28T00:41:28.953656+0000 mgr.y (mgr.14901) 4887 : cluster [DBG] pgmap v4894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:31.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:30 smithi130 bash[51302]: debug 2023-08-28T00:41:30.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:30.770713+0000) 2023-08-28T00:41:31.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:41:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:41:30] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T00:41:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:41:30.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:41:30.806Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:31.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:31 smithi130 bash[27579]: audit 2023-08-28T00:41:30.711665+0000 mon.a (mon.0) 3470 : audit [DBG] from='client.? 172.21.15.6:0/2136648256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:31.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:31 smithi006 bash[33918]: audit 2023-08-28T00:41:30.711665+0000 mon.a (mon.0) 3470 : audit [DBG] from='client.? 172.21.15.6:0/2136648256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:31.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:31 smithi006 bash[27292]: audit 2023-08-28T00:41:30.711665+0000 mon.a (mon.0) 3470 : audit [DBG] from='client.? 172.21.15.6:0/2136648256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:32.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:31 smithi130 bash[51302]: debug 2023-08-28T00:41:31.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:31.771041+0000) 2023-08-28T00:41:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:32 smithi130 bash[27579]: cluster 2023-08-28T00:41:30.954811+0000 mgr.y (mgr.14901) 4888 : cluster [DBG] pgmap v4895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:32 smithi006 bash[27292]: cluster 2023-08-28T00:41:30.954811+0000 mgr.y (mgr.14901) 4888 : cluster [DBG] pgmap v4895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:32 smithi006 bash[33918]: cluster 2023-08-28T00:41:30.954811+0000 mgr.y (mgr.14901) 4888 : cluster [DBG] pgmap v4895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:41:32.382Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:32.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:41:32.384Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:33.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:32 smithi130 bash[51302]: debug 2023-08-28T00:41:32.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:32.771256+0000) 2023-08-28T00:41:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:33 smithi130 bash[27579]: audit 2023-08-28T00:41:33.132369+0000 mon.c (mon.2) 1895 : audit [DBG] from='client.? 172.21.15.6:0/313182651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:33.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:33 smithi006 bash[27292]: audit 2023-08-28T00:41:33.132369+0000 mon.c (mon.2) 1895 : audit [DBG] from='client.? 172.21.15.6:0/313182651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:33.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:33 smithi006 bash[33918]: audit 2023-08-28T00:41:33.132369+0000 mon.c (mon.2) 1895 : audit [DBG] from='client.? 172.21.15.6:0/313182651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:34.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:33 smithi130 bash[51302]: debug 2023-08-28T00:41:33.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:33.771558+0000) 2023-08-28T00:41:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:34 smithi130 bash[27579]: cluster 2023-08-28T00:41:32.955465+0000 mgr.y (mgr.14901) 4889 : cluster [DBG] pgmap v4896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:34 smithi006 bash[27292]: cluster 2023-08-28T00:41:32.955465+0000 mgr.y (mgr.14901) 4889 : cluster [DBG] pgmap v4896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:34 smithi006 bash[33918]: cluster 2023-08-28T00:41:32.955465+0000 mgr.y (mgr.14901) 4889 : cluster [DBG] pgmap v4896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:34.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:41:34.265Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:34.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:41:34.267Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:35.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:34 smithi130 bash[51302]: debug 2023-08-28T00:41:34.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:34.771839+0000) 2023-08-28T00:41:36.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:35 smithi130 bash[51302]: debug 2023-08-28T00:41:35.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:35.772154+0000) 2023-08-28T00:41:36.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:36 smithi130 bash[27579]: cluster 2023-08-28T00:41:34.956624+0000 mgr.y (mgr.14901) 4890 : cluster [DBG] pgmap v4897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:36.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:36 smithi130 bash[27579]: audit 2023-08-28T00:41:35.546958+0000 mon.a (mon.0) 3471 : audit [DBG] from='client.? 172.21.15.6:0/2096220649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:36.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:36 smithi006 bash[27292]: cluster 2023-08-28T00:41:34.956624+0000 mgr.y (mgr.14901) 4890 : cluster [DBG] pgmap v4897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:36.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:36 smithi006 bash[27292]: audit 2023-08-28T00:41:35.546958+0000 mon.a (mon.0) 3471 : audit [DBG] from='client.? 172.21.15.6:0/2096220649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:36.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:36 smithi006 bash[33918]: cluster 2023-08-28T00:41:34.956624+0000 mgr.y (mgr.14901) 4890 : cluster [DBG] pgmap v4897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:36.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:36 smithi006 bash[33918]: audit 2023-08-28T00:41:35.546958+0000 mon.a (mon.0) 3471 : audit [DBG] from='client.? 172.21.15.6:0/2096220649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:37.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:36 smithi130 bash[51302]: debug 2023-08-28T00:41:36.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:36.772432+0000) 2023-08-28T00:41:38.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:37 smithi130 bash[51302]: debug 2023-08-28T00:41:37.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:37.772682+0000) 2023-08-28T00:41:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:38 smithi130 bash[27579]: cluster 2023-08-28T00:41:36.957266+0000 mgr.y (mgr.14901) 4891 : cluster [DBG] pgmap v4898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:38 smithi130 bash[27579]: audit 2023-08-28T00:41:37.968457+0000 mon.c (mon.2) 1896 : audit [DBG] from='client.? 172.21.15.6:0/1348222898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:38.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:38 smithi006 bash[27292]: cluster 2023-08-28T00:41:36.957266+0000 mgr.y (mgr.14901) 4891 : cluster [DBG] pgmap v4898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:38.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:38 smithi006 bash[27292]: audit 2023-08-28T00:41:37.968457+0000 mon.c (mon.2) 1896 : audit [DBG] from='client.? 172.21.15.6:0/1348222898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:38.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:38 smithi006 bash[33918]: cluster 2023-08-28T00:41:36.957266+0000 mgr.y (mgr.14901) 4891 : cluster [DBG] pgmap v4898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:38.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:38 smithi006 bash[33918]: audit 2023-08-28T00:41:37.968457+0000 mon.c (mon.2) 1896 : audit [DBG] from='client.? 172.21.15.6:0/1348222898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:39.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:38 smithi130 bash[51302]: debug 2023-08-28T00:41:38.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:38.773029+0000) 2023-08-28T00:41:40.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:39 smithi130 bash[51302]: debug 2023-08-28T00:41:39.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:39.773319+0000) 2023-08-28T00:41:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:40 smithi130 bash[27579]: cluster 2023-08-28T00:41:38.957882+0000 mgr.y (mgr.14901) 4892 : cluster [DBG] pgmap v4899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:40 smithi006 bash[27292]: cluster 2023-08-28T00:41:38.957882+0000 mgr.y (mgr.14901) 4892 : cluster [DBG] pgmap v4899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:40 smithi006 bash[33918]: cluster 2023-08-28T00:41:38.957882+0000 mgr.y (mgr.14901) 4892 : cluster [DBG] pgmap v4899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:41.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:40 smithi130 bash[51302]: debug 2023-08-28T00:41:40.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:40.773606+0000) 2023-08-28T00:41:41.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:41:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:41:40] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T00:41:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:41:40.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:41:40.807Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:41.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:41 smithi130 bash[27579]: audit 2023-08-28T00:41:40.393074+0000 mon.a (mon.0) 3472 : audit [DBG] from='client.? 172.21.15.6:0/3597476137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:41.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:41 smithi006 bash[33918]: audit 2023-08-28T00:41:40.393074+0000 mon.a (mon.0) 3472 : audit [DBG] from='client.? 172.21.15.6:0/3597476137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:41.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:41 smithi006 bash[27292]: audit 2023-08-28T00:41:40.393074+0000 mon.a (mon.0) 3472 : audit [DBG] from='client.? 172.21.15.6:0/3597476137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:42.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:41 smithi130 bash[51302]: debug 2023-08-28T00:41:41.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:41.773911+0000) 2023-08-28T00:41:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:42 smithi130 bash[27579]: cluster 2023-08-28T00:41:40.959150+0000 mgr.y (mgr.14901) 4893 : cluster [DBG] pgmap v4900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:42.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:42 smithi006 bash[27292]: cluster 2023-08-28T00:41:40.959150+0000 mgr.y (mgr.14901) 4893 : cluster [DBG] pgmap v4900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:42.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:42 smithi006 bash[33918]: cluster 2023-08-28T00:41:40.959150+0000 mgr.y (mgr.14901) 4893 : cluster [DBG] pgmap v4900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:41:42.383Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:42.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:41:42.384Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:43.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:42 smithi130 bash[51302]: debug 2023-08-28T00:41:42.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:42.774179+0000) 2023-08-28T00:41:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:43 smithi130 bash[27579]: audit 2023-08-28T00:41:42.810128+0000 mon.a (mon.0) 3473 : audit [DBG] from='client.? 172.21.15.6:0/2256986705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:43.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:43 smithi006 bash[33918]: audit 2023-08-28T00:41:42.810128+0000 mon.a (mon.0) 3473 : audit [DBG] from='client.? 172.21.15.6:0/2256986705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:43.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:43 smithi006 bash[27292]: audit 2023-08-28T00:41:42.810128+0000 mon.a (mon.0) 3473 : audit [DBG] from='client.? 172.21.15.6:0/2256986705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:44.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:43 smithi130 bash[51302]: debug 2023-08-28T00:41:43.774+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:43.774457+0000) 2023-08-28T00:41:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:44 smithi130 bash[27579]: cluster 2023-08-28T00:41:42.959865+0000 mgr.y (mgr.14901) 4894 : cluster [DBG] pgmap v4901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:44.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:44 smithi006 bash[27292]: cluster 2023-08-28T00:41:42.959865+0000 mgr.y (mgr.14901) 4894 : cluster [DBG] pgmap v4901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:44.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:44 smithi006 bash[33918]: cluster 2023-08-28T00:41:42.959865+0000 mgr.y (mgr.14901) 4894 : cluster [DBG] pgmap v4901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:41:44.266Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:44.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:41:44.268Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:44 smithi130 bash[51302]: debug 2023-08-28T00:41:44.774+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:44.774765+0000) 2023-08-28T00:41:45.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:45 smithi006 bash[27292]: audit 2023-08-28T00:41:45.226068+0000 mon.c (mon.2) 1897 : audit [DBG] from='client.? 172.21.15.6:0/1557987246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:45.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:45 smithi006 bash[33918]: audit 2023-08-28T00:41:45.226068+0000 mon.c (mon.2) 1897 : audit [DBG] from='client.? 172.21.15.6:0/1557987246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:45.776 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:45 smithi130 bash[27579]: audit 2023-08-28T00:41:45.226068+0000 mon.c (mon.2) 1897 : audit [DBG] from='client.? 172.21.15.6:0/1557987246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:46.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:45 smithi130 bash[51302]: debug 2023-08-28T00:41:45.774+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:45.775057+0000) 2023-08-28T00:41:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:46 smithi006 bash[27292]: cluster 2023-08-28T00:41:44.960901+0000 mgr.y (mgr.14901) 4895 : cluster [DBG] pgmap v4902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:46.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:46 smithi006 bash[33918]: cluster 2023-08-28T00:41:44.960901+0000 mgr.y (mgr.14901) 4895 : cluster [DBG] pgmap v4902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:46.777 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:46 smithi130 bash[27579]: cluster 2023-08-28T00:41:44.960901+0000 mgr.y (mgr.14901) 4895 : cluster [DBG] pgmap v4902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:47.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:46 smithi130 bash[51302]: debug 2023-08-28T00:41:46.773+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:46.775339+0000) 2023-08-28T00:41:48.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:47 smithi130 bash[51302]: debug 2023-08-28T00:41:47.773+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:47.775608+0000) 2023-08-28T00:41:48.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:48 smithi006 bash[27292]: cluster 2023-08-28T00:41:46.961487+0000 mgr.y (mgr.14901) 4896 : cluster [DBG] pgmap v4903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:48.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:48 smithi006 bash[27292]: audit 2023-08-28T00:41:47.642310+0000 mon.a (mon.0) 3474 : audit [DBG] from='client.? 172.21.15.6:0/3576938014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:48 smithi006 bash[33918]: cluster 2023-08-28T00:41:46.961487+0000 mgr.y (mgr.14901) 4896 : cluster [DBG] pgmap v4903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:48 smithi006 bash[33918]: audit 2023-08-28T00:41:47.642310+0000 mon.a (mon.0) 3474 : audit [DBG] from='client.? 172.21.15.6:0/3576938014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:48.777 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:48 smithi130 bash[27579]: cluster 2023-08-28T00:41:46.961487+0000 mgr.y (mgr.14901) 4896 : cluster [DBG] pgmap v4903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:48.778 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:48 smithi130 bash[27579]: audit 2023-08-28T00:41:47.642310+0000 mon.a (mon.0) 3474 : audit [DBG] from='client.? 172.21.15.6:0/3576938014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:49.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:48 smithi130 bash[51302]: debug 2023-08-28T00:41:48.773+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:48.775920+0000) 2023-08-28T00:41:50.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:49 smithi130 bash[51302]: debug 2023-08-28T00:41:49.773+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:49.776214+0000) 2023-08-28T00:41:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:50 smithi006 bash[33918]: cluster 2023-08-28T00:41:48.962197+0000 mgr.y (mgr.14901) 4897 : cluster [DBG] pgmap v4904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:50 smithi006 bash[33918]: audit 2023-08-28T00:41:50.056912+0000 mon.c (mon.2) 1898 : audit [DBG] from='client.? 172.21.15.6:0/4018769689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:50 smithi006 bash[27292]: cluster 2023-08-28T00:41:48.962197+0000 mgr.y (mgr.14901) 4897 : cluster [DBG] pgmap v4904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:50.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:50 smithi006 bash[27292]: audit 2023-08-28T00:41:50.056912+0000 mon.c (mon.2) 1898 : audit [DBG] from='client.? 172.21.15.6:0/4018769689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:50.778 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:50 smithi130 bash[27579]: cluster 2023-08-28T00:41:48.962197+0000 mgr.y (mgr.14901) 4897 : cluster [DBG] pgmap v4904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:50.778 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:50 smithi130 bash[27579]: audit 2023-08-28T00:41:50.056912+0000 mon.c (mon.2) 1898 : audit [DBG] from='client.? 172.21.15.6:0/4018769689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:51.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:50 smithi130 bash[51302]: debug 2023-08-28T00:41:50.773+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:50.776487+0000) 2023-08-28T00:41:51.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:41:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:41:50] "GET /metrics HTTP/1.1" 200 193133 "" "Prometheus/2.33.4" 2023-08-28T00:41:51.220 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:41:50.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:51.221 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:41:50.807Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:52.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:51 smithi130 bash[51302]: debug 2023-08-28T00:41:51.773+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:51.776818+0000) 2023-08-28T00:41:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:52 smithi006 bash[33918]: cluster 2023-08-28T00:41:50.963341+0000 mgr.y (mgr.14901) 4898 : cluster [DBG] pgmap v4905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:52 smithi006 bash[27292]: cluster 2023-08-28T00:41:50.963341+0000 mgr.y (mgr.14901) 4898 : cluster [DBG] pgmap v4905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:52.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:41:52.383Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:52.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:41:52.385Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:52.778 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:52 smithi130 bash[27579]: cluster 2023-08-28T00:41:50.963341+0000 mgr.y (mgr.14901) 4898 : cluster [DBG] pgmap v4905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:52 smithi130 bash[51302]: debug 2023-08-28T00:41:52.773+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:52.777102+0000) 2023-08-28T00:41:53.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:53 smithi006 bash[27292]: audit 2023-08-28T00:41:52.476219+0000 mon.a (mon.0) 3475 : audit [DBG] from='client.? 172.21.15.6:0/2453912966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:53 smithi006 bash[33918]: audit 2023-08-28T00:41:52.476219+0000 mon.a (mon.0) 3475 : audit [DBG] from='client.? 172.21.15.6:0/2453912966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:53.779 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:53 smithi130 bash[27579]: audit 2023-08-28T00:41:52.476219+0000 mon.a (mon.0) 3475 : audit [DBG] from='client.? 172.21.15.6:0/2453912966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:54.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:53 smithi130 bash[51302]: debug 2023-08-28T00:41:53.773+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:53.777383+0000) 2023-08-28T00:41:54.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:54 smithi006 bash[27292]: cluster 2023-08-28T00:41:52.964110+0000 mgr.y (mgr.14901) 4899 : cluster [DBG] pgmap v4906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:54.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:54 smithi006 bash[33918]: cluster 2023-08-28T00:41:52.964110+0000 mgr.y (mgr.14901) 4899 : cluster [DBG] pgmap v4906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:41:54.267Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:54.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:41:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:41:54.269Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:41:54.779 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:54 smithi130 bash[27579]: cluster 2023-08-28T00:41:52.964110+0000 mgr.y (mgr.14901) 4899 : cluster [DBG] pgmap v4906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:54 smithi130 bash[51302]: debug 2023-08-28T00:41:54.773+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:54.777662+0000) 2023-08-28T00:41:55.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:55 smithi006 bash[33918]: audit 2023-08-28T00:41:54.892582+0000 mon.c (mon.2) 1899 : audit [DBG] from='client.? 172.21.15.6:0/3527157468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:55.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:55 smithi006 bash[27292]: audit 2023-08-28T00:41:54.892582+0000 mon.c (mon.2) 1899 : audit [DBG] from='client.? 172.21.15.6:0/3527157468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:55.779 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:55 smithi130 bash[27579]: audit 2023-08-28T00:41:54.892582+0000 mon.c (mon.2) 1899 : audit [DBG] from='client.? 172.21.15.6:0/3527157468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:56.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:55 smithi130 bash[51302]: debug 2023-08-28T00:41:55.773+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:55.778008+0000) 2023-08-28T00:41:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:56 smithi006 bash[27292]: cluster 2023-08-28T00:41:54.965289+0000 mgr.y (mgr.14901) 4900 : cluster [DBG] pgmap v4907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:56.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:56 smithi006 bash[33918]: cluster 2023-08-28T00:41:54.965289+0000 mgr.y (mgr.14901) 4900 : cluster [DBG] pgmap v4907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:56.780 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:56 smithi130 bash[27579]: cluster 2023-08-28T00:41:54.965289+0000 mgr.y (mgr.14901) 4900 : cluster [DBG] pgmap v4907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:57.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:56 smithi130 bash[51302]: debug 2023-08-28T00:41:56.777+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:56.778289+0000) 2023-08-28T00:41:57.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:57 smithi006 bash[27292]: audit 2023-08-28T00:41:57.310602+0000 mon.c (mon.2) 1900 : audit [DBG] from='client.? 172.21.15.6:0/1677391113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:57.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:57 smithi006 bash[33918]: audit 2023-08-28T00:41:57.310602+0000 mon.c (mon.2) 1900 : audit [DBG] from='client.? 172.21.15.6:0/1677391113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:57.780 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:57 smithi130 bash[27579]: audit 2023-08-28T00:41:57.310602+0000 mon.c (mon.2) 1900 : audit [DBG] from='client.? 172.21.15.6:0/1677391113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:58.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:57 smithi130 bash[51302]: debug 2023-08-28T00:41:57.777+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:57.778571+0000) 2023-08-28T00:41:58.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:41:58 smithi006 bash[27292]: cluster 2023-08-28T00:41:56.965992+0000 mgr.y (mgr.14901) 4901 : cluster [DBG] pgmap v4908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:58.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:41:58 smithi006 bash[33918]: cluster 2023-08-28T00:41:56.965992+0000 mgr.y (mgr.14901) 4901 : cluster [DBG] pgmap v4908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:58.780 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:41:58 smithi130 bash[27579]: cluster 2023-08-28T00:41:56.965992+0000 mgr.y (mgr.14901) 4901 : cluster [DBG] pgmap v4908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:41:59.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:58 smithi130 bash[51302]: debug 2023-08-28T00:41:58.777+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:58.778919+0000) 2023-08-28T00:42:00.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:41:59 smithi130 bash[51302]: debug 2023-08-28T00:41:59.777+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:59.779205+0000) 2023-08-28T00:42:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:00 smithi006 bash[33918]: cluster 2023-08-28T00:41:58.966701+0000 mgr.y (mgr.14901) 4902 : cluster [DBG] pgmap v4909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:00 smithi006 bash[33918]: audit 2023-08-28T00:41:59.727415+0000 mon.c (mon.2) 1901 : audit [DBG] from='client.? 172.21.15.6:0/3424306603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:00 smithi006 bash[27292]: cluster 2023-08-28T00:41:58.966701+0000 mgr.y (mgr.14901) 4902 : cluster [DBG] pgmap v4909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:00 smithi006 bash[27292]: audit 2023-08-28T00:41:59.727415+0000 mon.c (mon.2) 1901 : audit [DBG] from='client.? 172.21.15.6:0/3424306603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:00.780 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:00 smithi130 bash[27579]: cluster 2023-08-28T00:41:58.966701+0000 mgr.y (mgr.14901) 4902 : cluster [DBG] pgmap v4909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:00.781 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:00 smithi130 bash[27579]: audit 2023-08-28T00:41:59.727415+0000 mon.c (mon.2) 1901 : audit [DBG] from='client.? 172.21.15.6:0/3424306603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:01.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:00 smithi130 bash[51302]: debug 2023-08-28T00:42:00.777+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:00.779445+0000) 2023-08-28T00:42:01.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:42:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:42:00] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T00:42:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:42:00.806Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:42:00.808Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:02.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:01 smithi130 bash[51302]: debug 2023-08-28T00:42:01.777+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:01.779726+0000) 2023-08-28T00:42:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:02 smithi006 bash[33918]: cluster 2023-08-28T00:42:00.967963+0000 mgr.y (mgr.14901) 4903 : cluster [DBG] pgmap v4910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:02 smithi006 bash[33918]: audit 2023-08-28T00:42:02.143871+0000 mon.a (mon.0) 3476 : audit [DBG] from='client.? 172.21.15.6:0/1635596459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:02.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:42:02.384Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:02.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:42:02.386Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:02.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:02 smithi006 bash[27292]: cluster 2023-08-28T00:42:00.967963+0000 mgr.y (mgr.14901) 4903 : cluster [DBG] pgmap v4910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:02.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:02 smithi006 bash[27292]: audit 2023-08-28T00:42:02.143871+0000 mon.a (mon.0) 3476 : audit [DBG] from='client.? 172.21.15.6:0/1635596459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:02.781 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:02 smithi130 bash[27579]: cluster 2023-08-28T00:42:00.967963+0000 mgr.y (mgr.14901) 4903 : cluster [DBG] pgmap v4910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:02.782 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:02 smithi130 bash[27579]: audit 2023-08-28T00:42:02.143871+0000 mon.a (mon.0) 3476 : audit [DBG] from='client.? 172.21.15.6:0/1635596459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:02 smithi130 bash[51302]: debug 2023-08-28T00:42:02.777+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:02.780001+0000) 2023-08-28T00:42:04.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:03 smithi130 bash[51302]: debug 2023-08-28T00:42:03.777+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:03.780300+0000) 2023-08-28T00:42:04.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:04 smithi006 bash[27292]: cluster 2023-08-28T00:42:02.968791+0000 mgr.y (mgr.14901) 4904 : cluster [DBG] pgmap v4911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:04.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:04 smithi006 bash[33918]: cluster 2023-08-28T00:42:02.968791+0000 mgr.y (mgr.14901) 4904 : cluster [DBG] pgmap v4911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:04.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:42:04.267Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:04.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:42:04.269Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:04.782 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:04 smithi130 bash[27579]: cluster 2023-08-28T00:42:02.968791+0000 mgr.y (mgr.14901) 4904 : cluster [DBG] pgmap v4911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:04 smithi130 bash[51302]: debug 2023-08-28T00:42:04.777+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:04.780582+0000) 2023-08-28T00:42:05.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:05 smithi006 bash[33918]: audit 2023-08-28T00:42:04.564855+0000 mon.c (mon.2) 1902 : audit [DBG] from='client.? 172.21.15.6:0/529848680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:05.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:05 smithi006 bash[27292]: audit 2023-08-28T00:42:04.564855+0000 mon.c (mon.2) 1902 : audit [DBG] from='client.? 172.21.15.6:0/529848680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:05.782 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:05 smithi130 bash[27579]: audit 2023-08-28T00:42:04.564855+0000 mon.c (mon.2) 1902 : audit [DBG] from='client.? 172.21.15.6:0/529848680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:06.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:05 smithi130 bash[51302]: debug 2023-08-28T00:42:05.777+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:05.780895+0000) 2023-08-28T00:42:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:06 smithi006 bash[33918]: cluster 2023-08-28T00:42:04.969969+0000 mgr.y (mgr.14901) 4905 : cluster [DBG] pgmap v4912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:06 smithi006 bash[27292]: cluster 2023-08-28T00:42:04.969969+0000 mgr.y (mgr.14901) 4905 : cluster [DBG] pgmap v4912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:06.782 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:06 smithi130 bash[27579]: cluster 2023-08-28T00:42:04.969969+0000 mgr.y (mgr.14901) 4905 : cluster [DBG] pgmap v4912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:07.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:06 smithi130 bash[51302]: debug 2023-08-28T00:42:06.777+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:06.781175+0000) 2023-08-28T00:42:07.783 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:07 smithi130 bash[27579]: audit 2023-08-28T00:42:06.983686+0000 mon.a (mon.0) 3477 : audit [DBG] from='client.? 172.21.15.6:0/2530707872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:07.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:07 smithi006 bash[33918]: audit 2023-08-28T00:42:06.983686+0000 mon.a (mon.0) 3477 : audit [DBG] from='client.? 172.21.15.6:0/2530707872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:07.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:07 smithi006 bash[27292]: audit 2023-08-28T00:42:06.983686+0000 mon.a (mon.0) 3477 : audit [DBG] from='client.? 172.21.15.6:0/2530707872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:08.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:07 smithi130 bash[51302]: debug 2023-08-28T00:42:07.777+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:07.781479+0000) 2023-08-28T00:42:08.783 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:08 smithi130 bash[27579]: cluster 2023-08-28T00:42:06.970572+0000 mgr.y (mgr.14901) 4906 : cluster [DBG] pgmap v4913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:08.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:08 smithi006 bash[33918]: cluster 2023-08-28T00:42:06.970572+0000 mgr.y (mgr.14901) 4906 : cluster [DBG] pgmap v4913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:08.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:08 smithi006 bash[27292]: cluster 2023-08-28T00:42:06.970572+0000 mgr.y (mgr.14901) 4906 : cluster [DBG] pgmap v4913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:09.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:08 smithi130 bash[51302]: debug 2023-08-28T00:42:08.777+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:08.781797+0000) 2023-08-28T00:42:09.783 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:09 smithi130 bash[27579]: audit 2023-08-28T00:42:09.407047+0000 mon.a (mon.0) 3478 : audit [DBG] from='client.? 172.21.15.6:0/2000416741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:09.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:09 smithi006 bash[27292]: audit 2023-08-28T00:42:09.407047+0000 mon.a (mon.0) 3478 : audit [DBG] from='client.? 172.21.15.6:0/2000416741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:09.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:09 smithi006 bash[33918]: audit 2023-08-28T00:42:09.407047+0000 mon.a (mon.0) 3478 : audit [DBG] from='client.? 172.21.15.6:0/2000416741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:10.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:09 smithi130 bash[51302]: debug 2023-08-28T00:42:09.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:09.782138+0000) 2023-08-28T00:42:10.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:10 smithi006 bash[27292]: cluster 2023-08-28T00:42:08.971204+0000 mgr.y (mgr.14901) 4907 : cluster [DBG] pgmap v4914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:10.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:10 smithi006 bash[33918]: cluster 2023-08-28T00:42:08.971204+0000 mgr.y (mgr.14901) 4907 : cluster [DBG] pgmap v4914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:10.784 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:10 smithi130 bash[27579]: cluster 2023-08-28T00:42:08.971204+0000 mgr.y (mgr.14901) 4907 : cluster [DBG] pgmap v4914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:11.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:10 smithi130 bash[51302]: debug 2023-08-28T00:42:10.777+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:10.782416+0000) 2023-08-28T00:42:11.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:42:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:42:10] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T00:42:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:42:10.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:42:10.809Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:12.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:11 smithi130 bash[51302]: debug 2023-08-28T00:42:11.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:11.782708+0000) 2023-08-28T00:42:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:12 smithi006 bash[33918]: cluster 2023-08-28T00:42:10.972412+0000 mgr.y (mgr.14901) 4908 : cluster [DBG] pgmap v4915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:12 smithi006 bash[33918]: audit 2023-08-28T00:42:11.830728+0000 mon.c (mon.2) 1903 : audit [DBG] from='client.? 172.21.15.6:0/2369882454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:42:12.384Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:42:12.386Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:12.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:12 smithi006 bash[27292]: cluster 2023-08-28T00:42:10.972412+0000 mgr.y (mgr.14901) 4908 : cluster [DBG] pgmap v4915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:12.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:12 smithi006 bash[27292]: audit 2023-08-28T00:42:11.830728+0000 mon.c (mon.2) 1903 : audit [DBG] from='client.? 172.21.15.6:0/2369882454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:12.784 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:12 smithi130 bash[27579]: cluster 2023-08-28T00:42:10.972412+0000 mgr.y (mgr.14901) 4908 : cluster [DBG] pgmap v4915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:12.785 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:12 smithi130 bash[27579]: audit 2023-08-28T00:42:11.830728+0000 mon.c (mon.2) 1903 : audit [DBG] from='client.? 172.21.15.6:0/2369882454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:13.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:12 smithi130 bash[51302]: debug 2023-08-28T00:42:12.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:12.783044+0000) 2023-08-28T00:42:13.784 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:13 smithi130 bash[27579]: audit 2023-08-28T00:42:12.983417+0000 mon.b (mon.1) 592 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:42:13.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:13 smithi006 bash[33918]: audit 2023-08-28T00:42:12.983417+0000 mon.b (mon.1) 592 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:42:13.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:13 smithi006 bash[27292]: audit 2023-08-28T00:42:12.983417+0000 mon.b (mon.1) 592 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:42:14.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:13 smithi130 bash[51302]: debug 2023-08-28T00:42:13.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:13.783273+0000) 2023-08-28T00:42:14.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:14 smithi006 bash[27292]: cluster 2023-08-28T00:42:12.973207+0000 mgr.y (mgr.14901) 4909 : cluster [DBG] pgmap v4916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:14.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:14 smithi006 bash[27292]: audit 2023-08-28T00:42:14.250905+0000 mon.c (mon.2) 1904 : audit [DBG] from='client.? 172.21.15.6:0/3686729044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:42:14.268Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:42:14.270Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:14.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:14 smithi006 bash[33918]: cluster 2023-08-28T00:42:12.973207+0000 mgr.y (mgr.14901) 4909 : cluster [DBG] pgmap v4916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:14.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:14 smithi006 bash[33918]: audit 2023-08-28T00:42:14.250905+0000 mon.c (mon.2) 1904 : audit [DBG] from='client.? 172.21.15.6:0/3686729044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:14.785 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:14 smithi130 bash[27579]: cluster 2023-08-28T00:42:12.973207+0000 mgr.y (mgr.14901) 4909 : cluster [DBG] pgmap v4916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:14.785 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:14 smithi130 bash[27579]: audit 2023-08-28T00:42:14.250905+0000 mon.c (mon.2) 1904 : audit [DBG] from='client.? 172.21.15.6:0/3686729044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:15.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:14 smithi130 bash[51302]: debug 2023-08-28T00:42:14.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:14.783510+0000) 2023-08-28T00:42:16.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:15 smithi130 bash[51302]: debug 2023-08-28T00:42:15.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:15.783728+0000) 2023-08-28T00:42:16.785 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:16 smithi130 bash[27579]: cluster 2023-08-28T00:42:14.974414+0000 mgr.y (mgr.14901) 4910 : cluster [DBG] pgmap v4917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:16.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:16 smithi006 bash[27292]: cluster 2023-08-28T00:42:14.974414+0000 mgr.y (mgr.14901) 4910 : cluster [DBG] pgmap v4917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:16.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:16 smithi006 bash[33918]: cluster 2023-08-28T00:42:14.974414+0000 mgr.y (mgr.14901) 4910 : cluster [DBG] pgmap v4917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:17.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:16 smithi130 bash[51302]: debug 2023-08-28T00:42:16.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:16.784034+0000) 2023-08-28T00:42:17.785 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:17 smithi130 bash[27579]: audit 2023-08-28T00:42:16.659738+0000 mon.a (mon.0) 3479 : audit [DBG] from='client.? 172.21.15.6:0/3741566952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:17.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:17 smithi006 bash[27292]: audit 2023-08-28T00:42:16.659738+0000 mon.a (mon.0) 3479 : audit [DBG] from='client.? 172.21.15.6:0/3741566952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:17.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:17 smithi006 bash[33918]: audit 2023-08-28T00:42:16.659738+0000 mon.a (mon.0) 3479 : audit [DBG] from='client.? 172.21.15.6:0/3741566952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:18.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:17 smithi130 bash[51302]: debug 2023-08-28T00:42:17.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:17.784302+0000) 2023-08-28T00:42:18.786 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:18 smithi130 bash[27579]: cluster 2023-08-28T00:42:16.975052+0000 mgr.y (mgr.14901) 4911 : cluster [DBG] pgmap v4918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:18.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:18 smithi006 bash[27292]: cluster 2023-08-28T00:42:16.975052+0000 mgr.y (mgr.14901) 4911 : cluster [DBG] pgmap v4918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:18.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:18 smithi006 bash[33918]: cluster 2023-08-28T00:42:16.975052+0000 mgr.y (mgr.14901) 4911 : cluster [DBG] pgmap v4918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:19.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:18 smithi130 bash[51302]: debug 2023-08-28T00:42:18.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:18.784569+0000) 2023-08-28T00:42:20.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:19 smithi130 bash[51302]: debug 2023-08-28T00:42:19.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:19.784857+0000) 2023-08-28T00:42:20.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:19 smithi130 bash[27579]: audit 2023-08-28T00:42:18.878271+0000 mon.a (mon.0) 3480 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:42:20.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:19 smithi130 bash[27579]: cluster 2023-08-28T00:42:18.975545+0000 mgr.y (mgr.14901) 4912 : cluster [DBG] pgmap v4919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:20.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:19 smithi130 bash[27579]: audit 2023-08-28T00:42:19.079979+0000 mon.c (mon.2) 1905 : audit [DBG] from='client.? 172.21.15.6:0/1938477164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:20.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:19 smithi130 bash[27579]: audit 2023-08-28T00:42:19.321712+0000 mon.a (mon.0) 3481 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:42:20.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:19 smithi130 bash[27579]: audit 2023-08-28T00:42:19.589267+0000 mon.b (mon.1) 593 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:42:20.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:19 smithi130 bash[27579]: audit 2023-08-28T00:42:19.591942+0000 mon.b (mon.1) 594 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:42:20.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:19 smithi130 bash[27579]: audit 2023-08-28T00:42:19.602110+0000 mon.a (mon.0) 3482 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:42:20.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:19 smithi006 bash[27292]: audit 2023-08-28T00:42:18.878271+0000 mon.a (mon.0) 3480 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:42:20.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:19 smithi006 bash[27292]: cluster 2023-08-28T00:42:18.975545+0000 mgr.y (mgr.14901) 4912 : cluster [DBG] pgmap v4919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:20.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:19 smithi006 bash[27292]: audit 2023-08-28T00:42:19.079979+0000 mon.c (mon.2) 1905 : audit [DBG] from='client.? 172.21.15.6:0/1938477164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:20.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:19 smithi006 bash[27292]: audit 2023-08-28T00:42:19.321712+0000 mon.a (mon.0) 3481 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:42:20.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:19 smithi006 bash[27292]: audit 2023-08-28T00:42:19.589267+0000 mon.b (mon.1) 593 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:42:20.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:19 smithi006 bash[27292]: audit 2023-08-28T00:42:19.591942+0000 mon.b (mon.1) 594 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:42:20.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:19 smithi006 bash[27292]: audit 2023-08-28T00:42:19.602110+0000 mon.a (mon.0) 3482 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:42:20.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:19 smithi006 bash[33918]: audit 2023-08-28T00:42:18.878271+0000 mon.a (mon.0) 3480 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:42:20.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:19 smithi006 bash[33918]: cluster 2023-08-28T00:42:18.975545+0000 mgr.y (mgr.14901) 4912 : cluster [DBG] pgmap v4919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:20.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:19 smithi006 bash[33918]: audit 2023-08-28T00:42:19.079979+0000 mon.c (mon.2) 1905 : audit [DBG] from='client.? 172.21.15.6:0/1938477164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:20.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:19 smithi006 bash[33918]: audit 2023-08-28T00:42:19.321712+0000 mon.a (mon.0) 3481 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:42:20.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:19 smithi006 bash[33918]: audit 2023-08-28T00:42:19.589267+0000 mon.b (mon.1) 593 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:42:20.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:19 smithi006 bash[33918]: audit 2023-08-28T00:42:19.591942+0000 mon.b (mon.1) 594 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:42:20.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:19 smithi006 bash[33918]: audit 2023-08-28T00:42:19.602110+0000 mon.a (mon.0) 3482 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:42:21.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:20 smithi130 bash[51302]: debug 2023-08-28T00:42:20.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:20.785180+0000) 2023-08-28T00:42:21.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:42:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:42:20] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T00:42:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:42:20.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:42:20.810Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:22.088 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:21 smithi130 bash[51302]: debug 2023-08-28T00:42:21.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:21.785524+0000) 2023-08-28T00:42:22.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:22 smithi130 bash[27579]: cluster 2023-08-28T00:42:20.976653+0000 mgr.y (mgr.14901) 4913 : cluster [DBG] pgmap v4920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:22.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:22 smithi130 bash[27579]: audit 2023-08-28T00:42:21.495172+0000 mon.a (mon.0) 3483 : audit [DBG] from='client.? 172.21.15.6:0/3773623862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:22.388 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:22 smithi006 bash[27292]: cluster 2023-08-28T00:42:20.976653+0000 mgr.y (mgr.14901) 4913 : cluster [DBG] pgmap v4920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:22.388 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:22 smithi006 bash[27292]: audit 2023-08-28T00:42:21.495172+0000 mon.a (mon.0) 3483 : audit [DBG] from='client.? 172.21.15.6:0/3773623862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:22.389 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:22 smithi006 bash[33918]: cluster 2023-08-28T00:42:20.976653+0000 mgr.y (mgr.14901) 4913 : cluster [DBG] pgmap v4920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:22.389 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:22 smithi006 bash[33918]: audit 2023-08-28T00:42:21.495172+0000 mon.a (mon.0) 3483 : audit [DBG] from='client.? 172.21.15.6:0/3773623862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:42:22.385Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:42:22.386Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:23.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:22 smithi130 bash[51302]: debug 2023-08-28T00:42:22.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:22.785806+0000) 2023-08-28T00:42:24.093 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:23 smithi130 bash[51302]: debug 2023-08-28T00:42:23.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:23.786123+0000) 2023-08-28T00:42:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:24 smithi130 bash[27579]: cluster 2023-08-28T00:42:22.977300+0000 mgr.y (mgr.14901) 4914 : cluster [DBG] pgmap v4921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:24.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:24 smithi130 bash[27579]: audit 2023-08-28T00:42:23.906542+0000 mon.a (mon.0) 3484 : audit [DBG] from='client.? 172.21.15.6:0/1226924758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:24.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:24 smithi006 bash[33918]: cluster 2023-08-28T00:42:22.977300+0000 mgr.y (mgr.14901) 4914 : cluster [DBG] pgmap v4921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:24.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:24 smithi006 bash[33918]: audit 2023-08-28T00:42:23.906542+0000 mon.a (mon.0) 3484 : audit [DBG] from='client.? 172.21.15.6:0/1226924758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:42:24.269Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:42:24.271Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:24.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:24 smithi006 bash[27292]: cluster 2023-08-28T00:42:22.977300+0000 mgr.y (mgr.14901) 4914 : cluster [DBG] pgmap v4921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:24.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:24 smithi006 bash[27292]: audit 2023-08-28T00:42:23.906542+0000 mon.a (mon.0) 3484 : audit [DBG] from='client.? 172.21.15.6:0/1226924758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:24 smithi130 bash[51302]: debug 2023-08-28T00:42:24.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:24.786402+0000) 2023-08-28T00:42:26.096 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:25 smithi130 bash[51302]: debug 2023-08-28T00:42:25.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:25.786677+0000) 2023-08-28T00:42:26.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:26 smithi130 bash[27579]: cluster 2023-08-28T00:42:24.978316+0000 mgr.y (mgr.14901) 4915 : cluster [DBG] pgmap v4922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:26.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:26 smithi006 bash[27292]: cluster 2023-08-28T00:42:24.978316+0000 mgr.y (mgr.14901) 4915 : cluster [DBG] pgmap v4922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:26 smithi006 bash[33918]: cluster 2023-08-28T00:42:24.978316+0000 mgr.y (mgr.14901) 4915 : cluster [DBG] pgmap v4922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:27.104 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:26 smithi130 bash[51302]: debug 2023-08-28T00:42:26.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:26.787027+0000) 2023-08-28T00:42:27.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:27 smithi130 bash[27579]: audit 2023-08-28T00:42:26.327523+0000 mon.c (mon.2) 1906 : audit [DBG] from='client.? 172.21.15.6:0/3249061429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:27.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:27 smithi006 bash[27292]: audit 2023-08-28T00:42:26.327523+0000 mon.c (mon.2) 1906 : audit [DBG] from='client.? 172.21.15.6:0/3249061429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:27.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:27 smithi006 bash[33918]: audit 2023-08-28T00:42:26.327523+0000 mon.c (mon.2) 1906 : audit [DBG] from='client.? 172.21.15.6:0/3249061429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:28.107 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:27 smithi130 bash[51302]: debug 2023-08-28T00:42:27.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:27.787277+0000) 2023-08-28T00:42:28.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:28 smithi130 bash[27579]: cluster 2023-08-28T00:42:26.978979+0000 mgr.y (mgr.14901) 4916 : cluster [DBG] pgmap v4923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:28.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:28 smithi006 bash[33918]: cluster 2023-08-28T00:42:26.978979+0000 mgr.y (mgr.14901) 4916 : cluster [DBG] pgmap v4923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:28.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:28 smithi006 bash[27292]: cluster 2023-08-28T00:42:26.978979+0000 mgr.y (mgr.14901) 4916 : cluster [DBG] pgmap v4923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:29.110 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:28 smithi130 bash[51302]: debug 2023-08-28T00:42:28.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:28.787502+0000) 2023-08-28T00:42:29.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:29 smithi130 bash[27579]: audit 2023-08-28T00:42:28.740893+0000 mon.c (mon.2) 1907 : audit [DBG] from='client.? 172.21.15.6:0/372734284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:29.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:29 smithi006 bash[27292]: audit 2023-08-28T00:42:28.740893+0000 mon.c (mon.2) 1907 : audit [DBG] from='client.? 172.21.15.6:0/372734284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:29.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:29 smithi006 bash[33918]: audit 2023-08-28T00:42:28.740893+0000 mon.c (mon.2) 1907 : audit [DBG] from='client.? 172.21.15.6:0/372734284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:30.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:29 smithi130 bash[51302]: debug 2023-08-28T00:42:29.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:29.787801+0000) 2023-08-28T00:42:30.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:30 smithi006 bash[33918]: cluster 2023-08-28T00:42:28.979661+0000 mgr.y (mgr.14901) 4917 : cluster [DBG] pgmap v4924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:30.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:30 smithi006 bash[27292]: cluster 2023-08-28T00:42:28.979661+0000 mgr.y (mgr.14901) 4917 : cluster [DBG] pgmap v4924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:30 smithi130 bash[27579]: cluster 2023-08-28T00:42:28.979661+0000 mgr.y (mgr.14901) 4917 : cluster [DBG] pgmap v4924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:31.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:30 smithi130 bash[51302]: debug 2023-08-28T00:42:30.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:30.788122+0000) 2023-08-28T00:42:31.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:42:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:42:30] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:42:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:42:30.809Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:42:30.810Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:32.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:31 smithi130 bash[51302]: debug 2023-08-28T00:42:31.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:31.788382+0000) 2023-08-28T00:42:32.389 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:32 smithi006 bash[27292]: cluster 2023-08-28T00:42:30.980390+0000 mgr.y (mgr.14901) 4918 : cluster [DBG] pgmap v4925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:32.390 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:32 smithi006 bash[27292]: audit 2023-08-28T00:42:31.165676+0000 mon.a (mon.0) 3485 : audit [DBG] from='client.? 172.21.15.6:0/269078673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:32.390 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:32 smithi006 bash[33918]: cluster 2023-08-28T00:42:30.980390+0000 mgr.y (mgr.14901) 4918 : cluster [DBG] pgmap v4925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:32.390 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:32 smithi006 bash[33918]: audit 2023-08-28T00:42:31.165676+0000 mon.a (mon.0) 3485 : audit [DBG] from='client.? 172.21.15.6:0/269078673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:32 smithi130 bash[27579]: cluster 2023-08-28T00:42:30.980390+0000 mgr.y (mgr.14901) 4918 : cluster [DBG] pgmap v4925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:32 smithi130 bash[27579]: audit 2023-08-28T00:42:31.165676+0000 mon.a (mon.0) 3485 : audit [DBG] from='client.? 172.21.15.6:0/269078673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:42:32.386Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:42:32.388Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:33.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:32 smithi130 bash[51302]: debug 2023-08-28T00:42:32.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:32.788686+0000) 2023-08-28T00:42:34.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:33 smithi130 bash[51302]: debug 2023-08-28T00:42:33.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:33.788982+0000) 2023-08-28T00:42:34.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:34 smithi006 bash[27292]: cluster 2023-08-28T00:42:32.981066+0000 mgr.y (mgr.14901) 4919 : cluster [DBG] pgmap v4926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:34.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:34 smithi006 bash[27292]: audit 2023-08-28T00:42:33.582218+0000 mon.c (mon.2) 1908 : audit [DBG] from='client.? 172.21.15.6:0/366709936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:34.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:34 smithi006 bash[33918]: cluster 2023-08-28T00:42:32.981066+0000 mgr.y (mgr.14901) 4919 : cluster [DBG] pgmap v4926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:34.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:34 smithi006 bash[33918]: audit 2023-08-28T00:42:33.582218+0000 mon.c (mon.2) 1908 : audit [DBG] from='client.? 172.21.15.6:0/366709936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:34.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:42:34.270Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:34.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:42:34.272Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:34 smithi130 bash[27579]: cluster 2023-08-28T00:42:32.981066+0000 mgr.y (mgr.14901) 4919 : cluster [DBG] pgmap v4926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:34 smithi130 bash[27579]: audit 2023-08-28T00:42:33.582218+0000 mon.c (mon.2) 1908 : audit [DBG] from='client.? 172.21.15.6:0/366709936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:35.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:34 smithi130 bash[51302]: debug 2023-08-28T00:42:34.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:34.789258+0000) 2023-08-28T00:42:36.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:35 smithi130 bash[51302]: debug 2023-08-28T00:42:35.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:35.789476+0000) 2023-08-28T00:42:36.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:36 smithi006 bash[33918]: cluster 2023-08-28T00:42:34.982129+0000 mgr.y (mgr.14901) 4920 : cluster [DBG] pgmap v4927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:36.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:36 smithi006 bash[33918]: audit 2023-08-28T00:42:35.994268+0000 mon.a (mon.0) 3486 : audit [DBG] from='client.? 172.21.15.6:0/2409120957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:36.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:36 smithi006 bash[27292]: cluster 2023-08-28T00:42:34.982129+0000 mgr.y (mgr.14901) 4920 : cluster [DBG] pgmap v4927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:36.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:36 smithi006 bash[27292]: audit 2023-08-28T00:42:35.994268+0000 mon.a (mon.0) 3486 : audit [DBG] from='client.? 172.21.15.6:0/2409120957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:36 smithi130 bash[27579]: cluster 2023-08-28T00:42:34.982129+0000 mgr.y (mgr.14901) 4920 : cluster [DBG] pgmap v4927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:36 smithi130 bash[27579]: audit 2023-08-28T00:42:35.994268+0000 mon.a (mon.0) 3486 : audit [DBG] from='client.? 172.21.15.6:0/2409120957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:37.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:36 smithi130 bash[51302]: debug 2023-08-28T00:42:36.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:36.789742+0000) 2023-08-28T00:42:38.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:37 smithi130 bash[51302]: debug 2023-08-28T00:42:37.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:37.790014+0000) 2023-08-28T00:42:38.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:38 smithi006 bash[27292]: cluster 2023-08-28T00:42:36.982770+0000 mgr.y (mgr.14901) 4921 : cluster [DBG] pgmap v4928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:38.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:38 smithi006 bash[33918]: cluster 2023-08-28T00:42:36.982770+0000 mgr.y (mgr.14901) 4921 : cluster [DBG] pgmap v4928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:38 smithi130 bash[27579]: cluster 2023-08-28T00:42:36.982770+0000 mgr.y (mgr.14901) 4921 : cluster [DBG] pgmap v4928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:39.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:38 smithi130 bash[51302]: debug 2023-08-28T00:42:38.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:38.790359+0000) 2023-08-28T00:42:39.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:39 smithi006 bash[27292]: audit 2023-08-28T00:42:38.416823+0000 mon.a (mon.0) 3487 : audit [DBG] from='client.? 172.21.15.6:0/876656418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:39.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:39 smithi006 bash[33918]: audit 2023-08-28T00:42:38.416823+0000 mon.a (mon.0) 3487 : audit [DBG] from='client.? 172.21.15.6:0/876656418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:39.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:39 smithi130 bash[27579]: audit 2023-08-28T00:42:38.416823+0000 mon.a (mon.0) 3487 : audit [DBG] from='client.? 172.21.15.6:0/876656418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:40.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:39 smithi130 bash[51302]: debug 2023-08-28T00:42:39.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:39.790639+0000) 2023-08-28T00:42:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:40 smithi006 bash[33918]: cluster 2023-08-28T00:42:38.983420+0000 mgr.y (mgr.14901) 4922 : cluster [DBG] pgmap v4929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:40.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:40 smithi006 bash[27292]: cluster 2023-08-28T00:42:38.983420+0000 mgr.y (mgr.14901) 4922 : cluster [DBG] pgmap v4929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:40 smithi130 bash[27579]: cluster 2023-08-28T00:42:38.983420+0000 mgr.y (mgr.14901) 4922 : cluster [DBG] pgmap v4929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:41.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:40 smithi130 bash[51302]: debug 2023-08-28T00:42:40.789+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:40.791064+0000) 2023-08-28T00:42:41.155 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:42:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:42:40] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:42:41.156 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:42:40.809Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:41.156 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:42:40.811Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:41.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:41 smithi006 bash[33918]: audit 2023-08-28T00:42:40.841757+0000 mon.a (mon.0) 3488 : audit [DBG] from='client.? 172.21.15.6:0/2213031535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:41.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:41 smithi006 bash[27292]: audit 2023-08-28T00:42:40.841757+0000 mon.a (mon.0) 3488 : audit [DBG] from='client.? 172.21.15.6:0/2213031535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:41.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:41 smithi130 bash[27579]: audit 2023-08-28T00:42:40.841757+0000 mon.a (mon.0) 3488 : audit [DBG] from='client.? 172.21.15.6:0/2213031535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:42.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:41 smithi130 bash[51302]: debug 2023-08-28T00:42:41.789+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:41.791335+0000) 2023-08-28T00:42:42.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:42 smithi006 bash[33918]: cluster 2023-08-28T00:42:40.984508+0000 mgr.y (mgr.14901) 4923 : cluster [DBG] pgmap v4930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:42.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:42:42.386Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:42.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:42:42.388Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:42.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:42 smithi006 bash[27292]: cluster 2023-08-28T00:42:40.984508+0000 mgr.y (mgr.14901) 4923 : cluster [DBG] pgmap v4930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:42 smithi130 bash[27579]: cluster 2023-08-28T00:42:40.984508+0000 mgr.y (mgr.14901) 4923 : cluster [DBG] pgmap v4930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:43.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:42 smithi130 bash[51302]: debug 2023-08-28T00:42:42.789+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:42.791595+0000) 2023-08-28T00:42:44.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:43 smithi130 bash[51302]: debug 2023-08-28T00:42:43.788+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:43.791879+0000) 2023-08-28T00:42:44.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:44 smithi006 bash[27292]: cluster 2023-08-28T00:42:42.984974+0000 mgr.y (mgr.14901) 4924 : cluster [DBG] pgmap v4931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:44.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:44 smithi006 bash[27292]: audit 2023-08-28T00:42:43.258366+0000 mon.c (mon.2) 1909 : audit [DBG] from='client.? 172.21.15.6:0/1165894590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:44.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:44 smithi006 bash[33918]: cluster 2023-08-28T00:42:42.984974+0000 mgr.y (mgr.14901) 4924 : cluster [DBG] pgmap v4931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:44.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:44 smithi006 bash[33918]: audit 2023-08-28T00:42:43.258366+0000 mon.c (mon.2) 1909 : audit [DBG] from='client.? 172.21.15.6:0/1165894590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:44.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:42:44.270Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:44.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:42:44.272Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:44 smithi130 bash[27579]: cluster 2023-08-28T00:42:42.984974+0000 mgr.y (mgr.14901) 4924 : cluster [DBG] pgmap v4931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:44 smithi130 bash[27579]: audit 2023-08-28T00:42:43.258366+0000 mon.c (mon.2) 1909 : audit [DBG] from='client.? 172.21.15.6:0/1165894590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:44 smithi130 bash[51302]: debug 2023-08-28T00:42:44.788+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:44.792191+0000) 2023-08-28T00:42:46.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:45 smithi130 bash[51302]: debug 2023-08-28T00:42:45.788+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:45.792463+0000) 2023-08-28T00:42:46.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:46 smithi006 bash[27292]: cluster 2023-08-28T00:42:44.986013+0000 mgr.y (mgr.14901) 4925 : cluster [DBG] pgmap v4932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:46.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:46 smithi006 bash[27292]: audit 2023-08-28T00:42:45.681267+0000 mon.c (mon.2) 1910 : audit [DBG] from='client.? 172.21.15.6:0/3704687092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:46.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:46 smithi006 bash[33918]: cluster 2023-08-28T00:42:44.986013+0000 mgr.y (mgr.14901) 4925 : cluster [DBG] pgmap v4932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:46.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:46 smithi006 bash[33918]: audit 2023-08-28T00:42:45.681267+0000 mon.c (mon.2) 1910 : audit [DBG] from='client.? 172.21.15.6:0/3704687092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:46 smithi130 bash[27579]: cluster 2023-08-28T00:42:44.986013+0000 mgr.y (mgr.14901) 4925 : cluster [DBG] pgmap v4932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:46 smithi130 bash[27579]: audit 2023-08-28T00:42:45.681267+0000 mon.c (mon.2) 1910 : audit [DBG] from='client.? 172.21.15.6:0/3704687092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:47.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:46 smithi130 bash[51302]: debug 2023-08-28T00:42:46.788+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:46.792741+0000) 2023-08-28T00:42:48.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:47 smithi130 bash[51302]: debug 2023-08-28T00:42:47.788+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:47.793039+0000) 2023-08-28T00:42:48.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:48 smithi006 bash[33918]: cluster 2023-08-28T00:42:46.986715+0000 mgr.y (mgr.14901) 4926 : cluster [DBG] pgmap v4933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:48.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:48 smithi006 bash[33918]: audit 2023-08-28T00:42:48.096678+0000 mon.a (mon.0) 3489 : audit [DBG] from='client.? 172.21.15.6:0/3259408478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:48.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:48 smithi006 bash[27292]: cluster 2023-08-28T00:42:46.986715+0000 mgr.y (mgr.14901) 4926 : cluster [DBG] pgmap v4933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:48.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:48 smithi006 bash[27292]: audit 2023-08-28T00:42:48.096678+0000 mon.a (mon.0) 3489 : audit [DBG] from='client.? 172.21.15.6:0/3259408478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:48.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:48 smithi130 bash[27579]: cluster 2023-08-28T00:42:46.986715+0000 mgr.y (mgr.14901) 4926 : cluster [DBG] pgmap v4933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:48.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:48 smithi130 bash[27579]: audit 2023-08-28T00:42:48.096678+0000 mon.a (mon.0) 3489 : audit [DBG] from='client.? 172.21.15.6:0/3259408478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:49.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:48 smithi130 bash[51302]: debug 2023-08-28T00:42:48.788+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:48.793329+0000) 2023-08-28T00:42:50.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:49 smithi130 bash[51302]: debug 2023-08-28T00:42:49.788+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:49.793561+0000) 2023-08-28T00:42:50.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:50 smithi006 bash[33918]: cluster 2023-08-28T00:42:48.987403+0000 mgr.y (mgr.14901) 4927 : cluster [DBG] pgmap v4934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:50.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:50 smithi006 bash[27292]: cluster 2023-08-28T00:42:48.987403+0000 mgr.y (mgr.14901) 4927 : cluster [DBG] pgmap v4934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:50 smithi130 bash[27579]: cluster 2023-08-28T00:42:48.987403+0000 mgr.y (mgr.14901) 4927 : cluster [DBG] pgmap v4934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:51.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:50 smithi130 bash[51302]: debug 2023-08-28T00:42:50.788+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:50.793843+0000) 2023-08-28T00:42:51.186 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:42:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:42:50] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T00:42:51.187 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:42:50.810Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:51.187 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:42:50.812Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:51.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:51 smithi006 bash[33918]: audit 2023-08-28T00:42:50.512243+0000 mon.a (mon.0) 3490 : audit [DBG] from='client.? 172.21.15.6:0/1647298454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:51.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:51 smithi006 bash[27292]: audit 2023-08-28T00:42:50.512243+0000 mon.a (mon.0) 3490 : audit [DBG] from='client.? 172.21.15.6:0/1647298454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:51.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:51 smithi130 bash[27579]: audit 2023-08-28T00:42:50.512243+0000 mon.a (mon.0) 3490 : audit [DBG] from='client.? 172.21.15.6:0/1647298454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:52.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:51 smithi130 bash[51302]: debug 2023-08-28T00:42:51.788+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:51.794127+0000) 2023-08-28T00:42:52.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:52 smithi006 bash[33918]: cluster 2023-08-28T00:42:50.988550+0000 mgr.y (mgr.14901) 4928 : cluster [DBG] pgmap v4935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:52.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:42:52.387Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:52.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:42:52.389Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:52.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:52 smithi006 bash[27292]: cluster 2023-08-28T00:42:50.988550+0000 mgr.y (mgr.14901) 4928 : cluster [DBG] pgmap v4935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:52 smithi130 bash[27579]: cluster 2023-08-28T00:42:50.988550+0000 mgr.y (mgr.14901) 4928 : cluster [DBG] pgmap v4935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:52 smithi130 bash[51302]: debug 2023-08-28T00:42:52.788+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:52.794408+0000) 2023-08-28T00:42:53.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:53 smithi006 bash[27292]: audit 2023-08-28T00:42:52.927253+0000 mon.a (mon.0) 3491 : audit [DBG] from='client.? 172.21.15.6:0/1761089539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:53.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:53 smithi006 bash[33918]: audit 2023-08-28T00:42:52.927253+0000 mon.a (mon.0) 3491 : audit [DBG] from='client.? 172.21.15.6:0/1761089539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:53 smithi130 bash[27579]: audit 2023-08-28T00:42:52.927253+0000 mon.a (mon.0) 3491 : audit [DBG] from='client.? 172.21.15.6:0/1761089539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:54.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:53 smithi130 bash[51302]: debug 2023-08-28T00:42:53.788+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:53.794687+0000) 2023-08-28T00:42:54.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:54 smithi006 bash[33918]: cluster 2023-08-28T00:42:52.989364+0000 mgr.y (mgr.14901) 4929 : cluster [DBG] pgmap v4936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:42:54.271Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:42:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:42:54.273Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:42:54.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:54 smithi006 bash[27292]: cluster 2023-08-28T00:42:52.989364+0000 mgr.y (mgr.14901) 4929 : cluster [DBG] pgmap v4936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:54 smithi130 bash[27579]: cluster 2023-08-28T00:42:52.989364+0000 mgr.y (mgr.14901) 4929 : cluster [DBG] pgmap v4936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:54 smithi130 bash[51302]: debug 2023-08-28T00:42:54.792+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:54.794984+0000) 2023-08-28T00:42:56.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:55 smithi130 bash[51302]: debug 2023-08-28T00:42:55.792+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:55.795269+0000) 2023-08-28T00:42:56.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:56 smithi006 bash[33918]: cluster 2023-08-28T00:42:54.990201+0000 mgr.y (mgr.14901) 4930 : cluster [DBG] pgmap v4937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:56.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:56 smithi006 bash[33918]: audit 2023-08-28T00:42:55.344386+0000 mon.c (mon.2) 1911 : audit [DBG] from='client.? 172.21.15.6:0/92461927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:56.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:56 smithi006 bash[27292]: cluster 2023-08-28T00:42:54.990201+0000 mgr.y (mgr.14901) 4930 : cluster [DBG] pgmap v4937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:56.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:56 smithi006 bash[27292]: audit 2023-08-28T00:42:55.344386+0000 mon.c (mon.2) 1911 : audit [DBG] from='client.? 172.21.15.6:0/92461927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:56 smithi130 bash[27579]: cluster 2023-08-28T00:42:54.990201+0000 mgr.y (mgr.14901) 4930 : cluster [DBG] pgmap v4937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:56 smithi130 bash[27579]: audit 2023-08-28T00:42:55.344386+0000 mon.c (mon.2) 1911 : audit [DBG] from='client.? 172.21.15.6:0/92461927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:57.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:56 smithi130 bash[51302]: debug 2023-08-28T00:42:56.792+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:56.795570+0000) 2023-08-28T00:42:58.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:57 smithi130 bash[51302]: debug 2023-08-28T00:42:57.792+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:57.795875+0000) 2023-08-28T00:42:58.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:58 smithi006 bash[33918]: cluster 2023-08-28T00:42:56.990800+0000 mgr.y (mgr.14901) 4931 : cluster [DBG] pgmap v4938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:58.557 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:42:58 smithi006 bash[33918]: audit 2023-08-28T00:42:57.769222+0000 mon.c (mon.2) 1912 : audit [DBG] from='client.? 172.21.15.6:0/123591469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:58.557 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:58 smithi006 bash[27292]: cluster 2023-08-28T00:42:56.990800+0000 mgr.y (mgr.14901) 4931 : cluster [DBG] pgmap v4938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:58.557 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:42:58 smithi006 bash[27292]: audit 2023-08-28T00:42:57.769222+0000 mon.c (mon.2) 1912 : audit [DBG] from='client.? 172.21.15.6:0/123591469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:58 smithi130 bash[27579]: cluster 2023-08-28T00:42:56.990800+0000 mgr.y (mgr.14901) 4931 : cluster [DBG] pgmap v4938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:42:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:42:58 smithi130 bash[27579]: audit 2023-08-28T00:42:57.769222+0000 mon.c (mon.2) 1912 : audit [DBG] from='client.? 172.21.15.6:0/123591469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:59.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:58 smithi130 bash[51302]: debug 2023-08-28T00:42:58.792+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:58.796165+0000) 2023-08-28T00:43:00.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:42:59 smithi130 bash[51302]: debug 2023-08-28T00:42:59.792+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:59.796474+0000) 2023-08-28T00:43:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:00 smithi130 bash[27579]: cluster 2023-08-28T00:42:58.991406+0000 mgr.y (mgr.14901) 4932 : cluster [DBG] pgmap v4939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:00 smithi130 bash[27579]: audit 2023-08-28T00:43:00.175935+0000 mon.c (mon.2) 1913 : audit [DBG] from='client.? 172.21.15.6:0/2842861359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:00 smithi006 bash[27292]: cluster 2023-08-28T00:42:58.991406+0000 mgr.y (mgr.14901) 4932 : cluster [DBG] pgmap v4939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:00 smithi006 bash[27292]: audit 2023-08-28T00:43:00.175935+0000 mon.c (mon.2) 1913 : audit [DBG] from='client.? 172.21.15.6:0/2842861359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:00 smithi006 bash[33918]: cluster 2023-08-28T00:42:58.991406+0000 mgr.y (mgr.14901) 4932 : cluster [DBG] pgmap v4939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:00 smithi006 bash[33918]: audit 2023-08-28T00:43:00.175935+0000 mon.c (mon.2) 1913 : audit [DBG] from='client.? 172.21.15.6:0/2842861359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:01.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:00 smithi130 bash[51302]: debug 2023-08-28T00:43:00.792+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:00.796762+0000) 2023-08-28T00:43:01.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:43:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:43:00] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:43:01.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:43:00.810Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:01.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:43:00.812Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:02.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:01 smithi130 bash[51302]: debug 2023-08-28T00:43:01.792+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:01.797037+0000) 2023-08-28T00:43:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:02 smithi130 bash[27579]: cluster 2023-08-28T00:43:00.992466+0000 mgr.y (mgr.14901) 4933 : cluster [DBG] pgmap v4940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:02 smithi006 bash[33918]: cluster 2023-08-28T00:43:00.992466+0000 mgr.y (mgr.14901) 4933 : cluster [DBG] pgmap v4940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:02.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:02 smithi006 bash[27292]: cluster 2023-08-28T00:43:00.992466+0000 mgr.y (mgr.14901) 4933 : cluster [DBG] pgmap v4940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:43:02.388Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:02.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:43:02.389Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:02 smithi130 bash[51302]: debug 2023-08-28T00:43:02.792+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:02.797362+0000) 2023-08-28T00:43:03.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:03 smithi130 bash[27579]: audit 2023-08-28T00:43:02.605088+0000 mon.a (mon.0) 3492 : audit [DBG] from='client.? 172.21.15.6:0/3236111346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:03.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:03 smithi006 bash[33918]: audit 2023-08-28T00:43:02.605088+0000 mon.a (mon.0) 3492 : audit [DBG] from='client.? 172.21.15.6:0/3236111346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:03.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:03 smithi006 bash[27292]: audit 2023-08-28T00:43:02.605088+0000 mon.a (mon.0) 3492 : audit [DBG] from='client.? 172.21.15.6:0/3236111346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:04.127 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:03 smithi130 bash[51302]: debug 2023-08-28T00:43:03.792+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:03.797641+0000) 2023-08-28T00:43:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:04 smithi130 bash[27579]: cluster 2023-08-28T00:43:02.993173+0000 mgr.y (mgr.14901) 4934 : cluster [DBG] pgmap v4941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:04.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:04 smithi006 bash[27292]: cluster 2023-08-28T00:43:02.993173+0000 mgr.y (mgr.14901) 4934 : cluster [DBG] pgmap v4941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:04.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:04 smithi006 bash[33918]: cluster 2023-08-28T00:43:02.993173+0000 mgr.y (mgr.14901) 4934 : cluster [DBG] pgmap v4941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:43:04.272Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:04.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:43:04.274Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:04 smithi130 bash[51302]: debug 2023-08-28T00:43:04.792+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:04.797893+0000) 2023-08-28T00:43:05.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:05 smithi130 bash[27579]: audit 2023-08-28T00:43:05.018379+0000 mon.a (mon.0) 3493 : audit [DBG] from='client.? 172.21.15.6:0/1689416202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:05.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:05 smithi006 bash[33918]: audit 2023-08-28T00:43:05.018379+0000 mon.a (mon.0) 3493 : audit [DBG] from='client.? 172.21.15.6:0/1689416202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:05.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:05 smithi006 bash[27292]: audit 2023-08-28T00:43:05.018379+0000 mon.a (mon.0) 3493 : audit [DBG] from='client.? 172.21.15.6:0/1689416202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:06.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:05 smithi130 bash[51302]: debug 2023-08-28T00:43:05.792+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:05.798170+0000) 2023-08-28T00:43:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:06 smithi130 bash[27579]: cluster 2023-08-28T00:43:04.994336+0000 mgr.y (mgr.14901) 4935 : cluster [DBG] pgmap v4942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:06 smithi006 bash[33918]: cluster 2023-08-28T00:43:04.994336+0000 mgr.y (mgr.14901) 4935 : cluster [DBG] pgmap v4942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:06 smithi006 bash[27292]: cluster 2023-08-28T00:43:04.994336+0000 mgr.y (mgr.14901) 4935 : cluster [DBG] pgmap v4942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:07.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:06 smithi130 bash[51302]: debug 2023-08-28T00:43:06.792+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:06.798482+0000) 2023-08-28T00:43:08.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:07 smithi130 bash[51302]: debug 2023-08-28T00:43:07.792+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:07.798727+0000) 2023-08-28T00:43:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:08 smithi130 bash[27579]: cluster 2023-08-28T00:43:06.995043+0000 mgr.y (mgr.14901) 4936 : cluster [DBG] pgmap v4943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:08.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:08 smithi130 bash[27579]: audit 2023-08-28T00:43:07.434403+0000 mon.a (mon.0) 3494 : audit [DBG] from='client.? 172.21.15.6:0/126060500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:08.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:08 smithi006 bash[27292]: cluster 2023-08-28T00:43:06.995043+0000 mgr.y (mgr.14901) 4936 : cluster [DBG] pgmap v4943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:08.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:08 smithi006 bash[27292]: audit 2023-08-28T00:43:07.434403+0000 mon.a (mon.0) 3494 : audit [DBG] from='client.? 172.21.15.6:0/126060500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:08 smithi006 bash[33918]: cluster 2023-08-28T00:43:06.995043+0000 mgr.y (mgr.14901) 4936 : cluster [DBG] pgmap v4943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:08 smithi006 bash[33918]: audit 2023-08-28T00:43:07.434403+0000 mon.a (mon.0) 3494 : audit [DBG] from='client.? 172.21.15.6:0/126060500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:09.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:08 smithi130 bash[51302]: debug 2023-08-28T00:43:08.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:08.799033+0000) 2023-08-28T00:43:10.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:09 smithi130 bash[51302]: debug 2023-08-28T00:43:09.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:09.799267+0000) 2023-08-28T00:43:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:10 smithi130 bash[27579]: cluster 2023-08-28T00:43:08.995722+0000 mgr.y (mgr.14901) 4937 : cluster [DBG] pgmap v4944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:10 smithi130 bash[27579]: audit 2023-08-28T00:43:09.848050+0000 mon.c (mon.2) 1914 : audit [DBG] from='client.? 172.21.15.6:0/365024159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:10 smithi006 bash[33918]: cluster 2023-08-28T00:43:08.995722+0000 mgr.y (mgr.14901) 4937 : cluster [DBG] pgmap v4944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:10 smithi006 bash[33918]: audit 2023-08-28T00:43:09.848050+0000 mon.c (mon.2) 1914 : audit [DBG] from='client.? 172.21.15.6:0/365024159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:10 smithi006 bash[27292]: cluster 2023-08-28T00:43:08.995722+0000 mgr.y (mgr.14901) 4937 : cluster [DBG] pgmap v4944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:10 smithi006 bash[27292]: audit 2023-08-28T00:43:09.848050+0000 mon.c (mon.2) 1914 : audit [DBG] from='client.? 172.21.15.6:0/365024159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:11.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:10 smithi130 bash[51302]: debug 2023-08-28T00:43:10.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:10.799491+0000) 2023-08-28T00:43:11.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:43:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:43:10] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:43:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:43:10.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:43:10.813Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:12.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:11 smithi130 bash[51302]: debug 2023-08-28T00:43:11.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:11.799808+0000) 2023-08-28T00:43:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:12 smithi130 bash[27579]: cluster 2023-08-28T00:43:10.996750+0000 mgr.y (mgr.14901) 4938 : cluster [DBG] pgmap v4945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:12 smithi006 bash[33918]: cluster 2023-08-28T00:43:10.996750+0000 mgr.y (mgr.14901) 4938 : cluster [DBG] pgmap v4945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:12 smithi006 bash[27292]: cluster 2023-08-28T00:43:10.996750+0000 mgr.y (mgr.14901) 4938 : cluster [DBG] pgmap v4945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:12.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:43:12.389Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:12.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:43:12.391Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:13.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:12 smithi130 bash[51302]: debug 2023-08-28T00:43:12.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:12.800077+0000) 2023-08-28T00:43:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:13 smithi130 bash[27579]: audit 2023-08-28T00:43:12.260900+0000 mon.a (mon.0) 3495 : audit [DBG] from='client.? 172.21.15.6:0/3711640284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:13.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:13 smithi006 bash[27292]: audit 2023-08-28T00:43:12.260900+0000 mon.a (mon.0) 3495 : audit [DBG] from='client.? 172.21.15.6:0/3711640284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:13.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:13 smithi006 bash[33918]: audit 2023-08-28T00:43:12.260900+0000 mon.a (mon.0) 3495 : audit [DBG] from='client.? 172.21.15.6:0/3711640284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:14.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:13 smithi130 bash[51302]: debug 2023-08-28T00:43:13.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:13.800338+0000) 2023-08-28T00:43:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:14 smithi130 bash[27579]: cluster 2023-08-28T00:43:12.997395+0000 mgr.y (mgr.14901) 4939 : cluster [DBG] pgmap v4946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:14.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:14 smithi006 bash[27292]: cluster 2023-08-28T00:43:12.997395+0000 mgr.y (mgr.14901) 4939 : cluster [DBG] pgmap v4946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:14.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:14 smithi006 bash[33918]: cluster 2023-08-28T00:43:12.997395+0000 mgr.y (mgr.14901) 4939 : cluster [DBG] pgmap v4946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:43:14.272Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:43:14.275Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:15.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:14 smithi130 bash[51302]: debug 2023-08-28T00:43:14.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:14.800636+0000) 2023-08-28T00:43:15.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:15 smithi130 bash[27579]: audit 2023-08-28T00:43:14.674962+0000 mon.c (mon.2) 1915 : audit [DBG] from='client.? 172.21.15.6:0/2907408028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:15.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:15 smithi006 bash[27292]: audit 2023-08-28T00:43:14.674962+0000 mon.c (mon.2) 1915 : audit [DBG] from='client.? 172.21.15.6:0/2907408028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:15.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:15 smithi006 bash[33918]: audit 2023-08-28T00:43:14.674962+0000 mon.c (mon.2) 1915 : audit [DBG] from='client.? 172.21.15.6:0/2907408028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:16.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:15 smithi130 bash[51302]: debug 2023-08-28T00:43:15.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:15.800959+0000) 2023-08-28T00:43:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:16 smithi130 bash[27579]: cluster 2023-08-28T00:43:14.998630+0000 mgr.y (mgr.14901) 4940 : cluster [DBG] pgmap v4947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:16.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:16 smithi006 bash[27292]: cluster 2023-08-28T00:43:14.998630+0000 mgr.y (mgr.14901) 4940 : cluster [DBG] pgmap v4947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:16 smithi006 bash[33918]: cluster 2023-08-28T00:43:14.998630+0000 mgr.y (mgr.14901) 4940 : cluster [DBG] pgmap v4947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:17.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:16 smithi130 bash[51302]: debug 2023-08-28T00:43:16.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:16.801193+0000) 2023-08-28T00:43:17.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:17 smithi130 bash[27579]: audit 2023-08-28T00:43:17.087846+0000 mon.a (mon.0) 3496 : audit [DBG] from='client.? 172.21.15.6:0/2588362378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:17.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:17 smithi006 bash[33918]: audit 2023-08-28T00:43:17.087846+0000 mon.a (mon.0) 3496 : audit [DBG] from='client.? 172.21.15.6:0/2588362378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:17.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:17 smithi006 bash[27292]: audit 2023-08-28T00:43:17.087846+0000 mon.a (mon.0) 3496 : audit [DBG] from='client.? 172.21.15.6:0/2588362378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:18.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:17 smithi130 bash[51302]: debug 2023-08-28T00:43:17.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:17.801477+0000) 2023-08-28T00:43:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:18 smithi130 bash[27579]: cluster 2023-08-28T00:43:16.999378+0000 mgr.y (mgr.14901) 4941 : cluster [DBG] pgmap v4948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:18.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:18 smithi006 bash[33918]: cluster 2023-08-28T00:43:16.999378+0000 mgr.y (mgr.14901) 4941 : cluster [DBG] pgmap v4948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:18 smithi006 bash[27292]: cluster 2023-08-28T00:43:16.999378+0000 mgr.y (mgr.14901) 4941 : cluster [DBG] pgmap v4948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:19.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:18 smithi130 bash[51302]: debug 2023-08-28T00:43:18.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:18.801749+0000) 2023-08-28T00:43:20.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:19 smithi130 bash[51302]: debug 2023-08-28T00:43:19.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:19.801996+0000) 2023-08-28T00:43:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:20 smithi130 bash[27579]: cluster 2023-08-28T00:43:19.000018+0000 mgr.y (mgr.14901) 4942 : cluster [DBG] pgmap v4949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:20 smithi130 bash[27579]: audit 2023-08-28T00:43:19.508498+0000 mon.a (mon.0) 3497 : audit [DBG] from='client.? 172.21.15.6:0/1512052441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:20 smithi130 bash[27579]: audit 2023-08-28T00:43:19.615050+0000 mon.b (mon.1) 595 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:43:20.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:20 smithi006 bash[33918]: cluster 2023-08-28T00:43:19.000018+0000 mgr.y (mgr.14901) 4942 : cluster [DBG] pgmap v4949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:20.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:20 smithi006 bash[33918]: audit 2023-08-28T00:43:19.508498+0000 mon.a (mon.0) 3497 : audit [DBG] from='client.? 172.21.15.6:0/1512052441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:20.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:20 smithi006 bash[33918]: audit 2023-08-28T00:43:19.615050+0000 mon.b (mon.1) 595 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:43:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:20 smithi006 bash[27292]: cluster 2023-08-28T00:43:19.000018+0000 mgr.y (mgr.14901) 4942 : cluster [DBG] pgmap v4949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:20.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:20 smithi006 bash[27292]: audit 2023-08-28T00:43:19.508498+0000 mon.a (mon.0) 3497 : audit [DBG] from='client.? 172.21.15.6:0/1512052441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:20.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:20 smithi006 bash[27292]: audit 2023-08-28T00:43:19.615050+0000 mon.b (mon.1) 595 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:43:21.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:20 smithi130 bash[51302]: debug 2023-08-28T00:43:20.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:20.802300+0000) 2023-08-28T00:43:21.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:43:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:43:20] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:43:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:43:20.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:43:20.814Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:22.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:21 smithi130 bash[51302]: debug 2023-08-28T00:43:21.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:21.802606+0000) 2023-08-28T00:43:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:22 smithi130 bash[27579]: cluster 2023-08-28T00:43:21.001007+0000 mgr.y (mgr.14901) 4943 : cluster [DBG] pgmap v4950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:22 smithi130 bash[27579]: audit 2023-08-28T00:43:21.928792+0000 mon.c (mon.2) 1916 : audit [DBG] from='client.? 172.21.15.6:0/2054552906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:43:22.390Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:43:22.392Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:22 smithi006 bash[27292]: cluster 2023-08-28T00:43:21.001007+0000 mgr.y (mgr.14901) 4943 : cluster [DBG] pgmap v4950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:22.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:22 smithi006 bash[27292]: audit 2023-08-28T00:43:21.928792+0000 mon.c (mon.2) 1916 : audit [DBG] from='client.? 172.21.15.6:0/2054552906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:22.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:22 smithi006 bash[33918]: cluster 2023-08-28T00:43:21.001007+0000 mgr.y (mgr.14901) 4943 : cluster [DBG] pgmap v4950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:22.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:22 smithi006 bash[33918]: audit 2023-08-28T00:43:21.928792+0000 mon.c (mon.2) 1916 : audit [DBG] from='client.? 172.21.15.6:0/2054552906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:23.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:22 smithi130 bash[51302]: debug 2023-08-28T00:43:22.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:22.803025+0000) 2023-08-28T00:43:24.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:23 smithi130 bash[51302]: debug 2023-08-28T00:43:23.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:23.803323+0000) 2023-08-28T00:43:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:24 smithi130 bash[27579]: cluster 2023-08-28T00:43:23.001625+0000 mgr.y (mgr.14901) 4944 : cluster [DBG] pgmap v4951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:24.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:24 smithi006 bash[33918]: cluster 2023-08-28T00:43:23.001625+0000 mgr.y (mgr.14901) 4944 : cluster [DBG] pgmap v4951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:24.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:24 smithi006 bash[27292]: cluster 2023-08-28T00:43:23.001625+0000 mgr.y (mgr.14901) 4944 : cluster [DBG] pgmap v4951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:43:24.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:24.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:43:24.276Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:24 smithi130 bash[51302]: debug 2023-08-28T00:43:24.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:24.803595+0000) 2023-08-28T00:43:25.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:25 smithi130 bash[27579]: audit 2023-08-28T00:43:24.360425+0000 mon.c (mon.2) 1917 : audit [DBG] from='client.? 172.21.15.6:0/2956653164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:25.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:25 smithi006 bash[33918]: audit 2023-08-28T00:43:24.360425+0000 mon.c (mon.2) 1917 : audit [DBG] from='client.? 172.21.15.6:0/2956653164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:25.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:25 smithi006 bash[27292]: audit 2023-08-28T00:43:24.360425+0000 mon.c (mon.2) 1917 : audit [DBG] from='client.? 172.21.15.6:0/2956653164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:26.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:25 smithi130 bash[51302]: debug 2023-08-28T00:43:25.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:25.803872+0000) 2023-08-28T00:43:26.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:26 smithi130 bash[51302]: debug 2023-08-28T00:43:26.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:26.804094+0000) 2023-08-28T00:43:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:26 smithi130 bash[27579]: cluster 2023-08-28T00:43:25.002597+0000 mgr.y (mgr.14901) 4945 : cluster [DBG] pgmap v4952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:26 smithi130 bash[27579]: audit 2023-08-28T00:43:25.554763+0000 mon.a (mon.0) 3498 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:43:26.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:26 smithi130 bash[27579]: audit 2023-08-28T00:43:25.942085+0000 mon.a (mon.0) 3499 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:43:26.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:26 smithi130 bash[27579]: audit 2023-08-28T00:43:26.215031+0000 mon.b (mon.1) 596 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:43:26.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:26 smithi130 bash[27579]: audit 2023-08-28T00:43:26.217284+0000 mon.b (mon.1) 597 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:43:26.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:26 smithi130 bash[27579]: audit 2023-08-28T00:43:26.224511+0000 mon.a (mon.0) 3500 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:43:26.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:26 smithi006 bash[33918]: cluster 2023-08-28T00:43:25.002597+0000 mgr.y (mgr.14901) 4945 : cluster [DBG] pgmap v4952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:26.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:26 smithi006 bash[33918]: audit 2023-08-28T00:43:25.554763+0000 mon.a (mon.0) 3498 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:43:26.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:26 smithi006 bash[33918]: audit 2023-08-28T00:43:25.942085+0000 mon.a (mon.0) 3499 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:43:26.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:26 smithi006 bash[33918]: audit 2023-08-28T00:43:26.215031+0000 mon.b (mon.1) 596 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:43:26.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:26 smithi006 bash[33918]: audit 2023-08-28T00:43:26.217284+0000 mon.b (mon.1) 597 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:43:26.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:26 smithi006 bash[33918]: audit 2023-08-28T00:43:26.224511+0000 mon.a (mon.0) 3500 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:43:26.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:26 smithi006 bash[27292]: cluster 2023-08-28T00:43:25.002597+0000 mgr.y (mgr.14901) 4945 : cluster [DBG] pgmap v4952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:26.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:26 smithi006 bash[27292]: audit 2023-08-28T00:43:25.554763+0000 mon.a (mon.0) 3498 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:43:26.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:26 smithi006 bash[27292]: audit 2023-08-28T00:43:25.942085+0000 mon.a (mon.0) 3499 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:43:26.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:26 smithi006 bash[27292]: audit 2023-08-28T00:43:26.215031+0000 mon.b (mon.1) 596 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:43:26.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:26 smithi006 bash[27292]: audit 2023-08-28T00:43:26.217284+0000 mon.b (mon.1) 597 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:43:26.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:26 smithi006 bash[27292]: audit 2023-08-28T00:43:26.224511+0000 mon.a (mon.0) 3500 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:43:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:27 smithi130 bash[27579]: audit 2023-08-28T00:43:26.774823+0000 mon.a (mon.0) 3501 : audit [DBG] from='client.? 172.21.15.6:0/2313965855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:27.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:27 smithi130 bash[51302]: debug 2023-08-28T00:43:27.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:27.804397+0000) 2023-08-28T00:43:27.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:27 smithi006 bash[33918]: audit 2023-08-28T00:43:26.774823+0000 mon.a (mon.0) 3501 : audit [DBG] from='client.? 172.21.15.6:0/2313965855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:27.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:27 smithi006 bash[27292]: audit 2023-08-28T00:43:26.774823+0000 mon.a (mon.0) 3501 : audit [DBG] from='client.? 172.21.15.6:0/2313965855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:28 smithi130 bash[27579]: cluster 2023-08-28T00:43:27.003283+0000 mgr.y (mgr.14901) 4946 : cluster [DBG] pgmap v4953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:28.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:28 smithi130 bash[51302]: debug 2023-08-28T00:43:28.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:28.804706+0000) 2023-08-28T00:43:28.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:28 smithi006 bash[27292]: cluster 2023-08-28T00:43:27.003283+0000 mgr.y (mgr.14901) 4946 : cluster [DBG] pgmap v4953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:28.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:28 smithi006 bash[33918]: cluster 2023-08-28T00:43:27.003283+0000 mgr.y (mgr.14901) 4946 : cluster [DBG] pgmap v4953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:29.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:29 smithi130 bash[27579]: audit 2023-08-28T00:43:29.182202+0000 mon.c (mon.2) 1918 : audit [DBG] from='client.? 172.21.15.6:0/614401421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:29.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:29 smithi130 bash[51302]: debug 2023-08-28T00:43:29.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:29.804999+0000) 2023-08-28T00:43:29.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:29 smithi006 bash[27292]: audit 2023-08-28T00:43:29.182202+0000 mon.c (mon.2) 1918 : audit [DBG] from='client.? 172.21.15.6:0/614401421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:29.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:29 smithi006 bash[33918]: audit 2023-08-28T00:43:29.182202+0000 mon.c (mon.2) 1918 : audit [DBG] from='client.? 172.21.15.6:0/614401421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:30 smithi130 bash[27579]: cluster 2023-08-28T00:43:29.003896+0000 mgr.y (mgr.14901) 4947 : cluster [DBG] pgmap v4954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:30.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:30 smithi130 bash[51302]: debug 2023-08-28T00:43:30.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:30.805299+0000) 2023-08-28T00:43:30.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:30 smithi006 bash[27292]: cluster 2023-08-28T00:43:29.003896+0000 mgr.y (mgr.14901) 4947 : cluster [DBG] pgmap v4954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:30.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:30 smithi006 bash[33918]: cluster 2023-08-28T00:43:29.003896+0000 mgr.y (mgr.14901) 4947 : cluster [DBG] pgmap v4954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:30.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:43:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:43:30] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:43:30.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:43:30.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:30.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:43:30.815Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:32.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:31 smithi130 bash[51302]: debug 2023-08-28T00:43:31.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:31.805622+0000) 2023-08-28T00:43:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:32 smithi006 bash[27292]: cluster 2023-08-28T00:43:31.005080+0000 mgr.y (mgr.14901) 4948 : cluster [DBG] pgmap v4955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:32 smithi006 bash[27292]: audit 2023-08-28T00:43:31.597073+0000 mon.c (mon.2) 1919 : audit [DBG] from='client.? 172.21.15.6:0/1847846022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:32 smithi006 bash[33918]: cluster 2023-08-28T00:43:31.005080+0000 mgr.y (mgr.14901) 4948 : cluster [DBG] pgmap v4955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:32.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:32 smithi006 bash[33918]: audit 2023-08-28T00:43:31.597073+0000 mon.c (mon.2) 1919 : audit [DBG] from='client.? 172.21.15.6:0/1847846022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:32.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:43:32.391Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:32.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:43:32.392Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:32 smithi130 bash[27579]: cluster 2023-08-28T00:43:31.005080+0000 mgr.y (mgr.14901) 4948 : cluster [DBG] pgmap v4955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:32 smithi130 bash[27579]: audit 2023-08-28T00:43:31.597073+0000 mon.c (mon.2) 1919 : audit [DBG] from='client.? 172.21.15.6:0/1847846022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:32.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:32 smithi130 bash[51302]: debug 2023-08-28T00:43:32.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:32.805925+0000) 2023-08-28T00:43:34.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:33 smithi130 bash[51302]: debug 2023-08-28T00:43:33.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:33.806185+0000) 2023-08-28T00:43:34.596 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:43:34.275Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:34.596 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:43:34.277Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:34 smithi130 bash[27579]: cluster 2023-08-28T00:43:33.005694+0000 mgr.y (mgr.14901) 4949 : cluster [DBG] pgmap v4956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:34 smithi130 bash[27579]: audit 2023-08-28T00:43:34.011595+0000 mon.a (mon.0) 3502 : audit [DBG] from='client.? 172.21.15.6:0/974008698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:34.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:34 smithi130 bash[51302]: debug 2023-08-28T00:43:34.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:34.806478+0000) 2023-08-28T00:43:34.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:34 smithi006 bash[27292]: cluster 2023-08-28T00:43:33.005694+0000 mgr.y (mgr.14901) 4949 : cluster [DBG] pgmap v4956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:34.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:34 smithi006 bash[27292]: audit 2023-08-28T00:43:34.011595+0000 mon.a (mon.0) 3502 : audit [DBG] from='client.? 172.21.15.6:0/974008698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:34.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:34 smithi006 bash[33918]: cluster 2023-08-28T00:43:33.005694+0000 mgr.y (mgr.14901) 4949 : cluster [DBG] pgmap v4956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:34.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:34 smithi006 bash[33918]: audit 2023-08-28T00:43:34.011595+0000 mon.a (mon.0) 3502 : audit [DBG] from='client.? 172.21.15.6:0/974008698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:36.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:35 smithi130 bash[51302]: debug 2023-08-28T00:43:35.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:35.806793+0000) 2023-08-28T00:43:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:36 smithi130 bash[27579]: cluster 2023-08-28T00:43:35.006775+0000 mgr.y (mgr.14901) 4950 : cluster [DBG] pgmap v4957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:36 smithi130 bash[27579]: audit 2023-08-28T00:43:36.412386+0000 mon.c (mon.2) 1920 : audit [DBG] from='client.? 172.21.15.6:0/2393279994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:36.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:36 smithi130 bash[51302]: debug 2023-08-28T00:43:36.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:36.807069+0000) 2023-08-28T00:43:36.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:36 smithi006 bash[27292]: cluster 2023-08-28T00:43:35.006775+0000 mgr.y (mgr.14901) 4950 : cluster [DBG] pgmap v4957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:36.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:36 smithi006 bash[27292]: audit 2023-08-28T00:43:36.412386+0000 mon.c (mon.2) 1920 : audit [DBG] from='client.? 172.21.15.6:0/2393279994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:36 smithi006 bash[33918]: cluster 2023-08-28T00:43:35.006775+0000 mgr.y (mgr.14901) 4950 : cluster [DBG] pgmap v4957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:36 smithi006 bash[33918]: audit 2023-08-28T00:43:36.412386+0000 mon.c (mon.2) 1920 : audit [DBG] from='client.? 172.21.15.6:0/2393279994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:38.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:37 smithi130 bash[51302]: debug 2023-08-28T00:43:37.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:37.807340+0000) 2023-08-28T00:43:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:38 smithi130 bash[27579]: cluster 2023-08-28T00:43:37.007465+0000 mgr.y (mgr.14901) 4951 : cluster [DBG] pgmap v4958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:38 smithi130 bash[51302]: debug 2023-08-28T00:43:38.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:38.807603+0000) 2023-08-28T00:43:38.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:38 smithi006 bash[33918]: cluster 2023-08-28T00:43:37.007465+0000 mgr.y (mgr.14901) 4951 : cluster [DBG] pgmap v4958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:38.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:38 smithi006 bash[27292]: cluster 2023-08-28T00:43:37.007465+0000 mgr.y (mgr.14901) 4951 : cluster [DBG] pgmap v4958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:39.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:39 smithi130 bash[27579]: audit 2023-08-28T00:43:38.829667+0000 mon.a (mon.0) 3503 : audit [DBG] from='client.? 172.21.15.6:0/1835271243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:39 smithi130 bash[51302]: debug 2023-08-28T00:43:39.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:39.807860+0000) 2023-08-28T00:43:39.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:39 smithi006 bash[33918]: audit 2023-08-28T00:43:38.829667+0000 mon.a (mon.0) 3503 : audit [DBG] from='client.? 172.21.15.6:0/1835271243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:39.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:39 smithi006 bash[27292]: audit 2023-08-28T00:43:38.829667+0000 mon.a (mon.0) 3503 : audit [DBG] from='client.? 172.21.15.6:0/1835271243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:40.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:40 smithi006 bash[33918]: cluster 2023-08-28T00:43:39.008125+0000 mgr.y (mgr.14901) 4952 : cluster [DBG] pgmap v4959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:40.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:40 smithi006 bash[27292]: cluster 2023-08-28T00:43:39.008125+0000 mgr.y (mgr.14901) 4952 : cluster [DBG] pgmap v4959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:40.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:43:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:43:40] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:43:40.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:43:40.814Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:40.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:43:40.816Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:40 smithi130 bash[27579]: cluster 2023-08-28T00:43:39.008125+0000 mgr.y (mgr.14901) 4952 : cluster [DBG] pgmap v4959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:41.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:40 smithi130 bash[51302]: debug 2023-08-28T00:43:40.807+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:40.808155+0000) 2023-08-28T00:43:42.107 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:41 smithi130 bash[51302]: debug 2023-08-28T00:43:41.803+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:41.808494+0000) 2023-08-28T00:43:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:42 smithi130 bash[27579]: cluster 2023-08-28T00:43:41.009092+0000 mgr.y (mgr.14901) 4953 : cluster [DBG] pgmap v4960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:42.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:42 smithi130 bash[27579]: audit 2023-08-28T00:43:41.251216+0000 mon.a (mon.0) 3504 : audit [DBG] from='client.? 172.21.15.6:0/1184486821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:42.395 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:42 smithi006 bash[27292]: cluster 2023-08-28T00:43:41.009092+0000 mgr.y (mgr.14901) 4953 : cluster [DBG] pgmap v4960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:42.395 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:42 smithi006 bash[27292]: audit 2023-08-28T00:43:41.251216+0000 mon.a (mon.0) 3504 : audit [DBG] from='client.? 172.21.15.6:0/1184486821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:42.395 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:42 smithi006 bash[33918]: cluster 2023-08-28T00:43:41.009092+0000 mgr.y (mgr.14901) 4953 : cluster [DBG] pgmap v4960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:42.395 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:42 smithi006 bash[33918]: audit 2023-08-28T00:43:41.251216+0000 mon.a (mon.0) 3504 : audit [DBG] from='client.? 172.21.15.6:0/1184486821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:43:42.391Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:43:42.393Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:43.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:42 smithi130 bash[51302]: debug 2023-08-28T00:43:42.807+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:42.808777+0000) 2023-08-28T00:43:44.112 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:43 smithi130 bash[51302]: debug 2023-08-28T00:43:43.807+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:43.809081+0000) 2023-08-28T00:43:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:44 smithi130 bash[27579]: cluster 2023-08-28T00:43:43.009716+0000 mgr.y (mgr.14901) 4954 : cluster [DBG] pgmap v4961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:44.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:44 smithi130 bash[27579]: audit 2023-08-28T00:43:43.677950+0000 mon.c (mon.2) 1921 : audit [DBG] from='client.? 172.21.15.6:0/228847414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:44.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:44 smithi006 bash[33918]: cluster 2023-08-28T00:43:43.009716+0000 mgr.y (mgr.14901) 4954 : cluster [DBG] pgmap v4961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:44.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:44 smithi006 bash[33918]: audit 2023-08-28T00:43:43.677950+0000 mon.c (mon.2) 1921 : audit [DBG] from='client.? 172.21.15.6:0/228847414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:43:44.275Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:43:44.277Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:44.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:44 smithi006 bash[27292]: cluster 2023-08-28T00:43:43.009716+0000 mgr.y (mgr.14901) 4954 : cluster [DBG] pgmap v4961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:44.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:44 smithi006 bash[27292]: audit 2023-08-28T00:43:43.677950+0000 mon.c (mon.2) 1921 : audit [DBG] from='client.? 172.21.15.6:0/228847414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:44 smithi130 bash[51302]: debug 2023-08-28T00:43:44.803+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:44.809350+0000) 2023-08-28T00:43:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:45 smithi130 bash[51302]: debug 2023-08-28T00:43:45.803+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:45.809641+0000) 2023-08-28T00:43:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:46 smithi130 bash[27579]: cluster 2023-08-28T00:43:45.010897+0000 mgr.y (mgr.14901) 4955 : cluster [DBG] pgmap v4962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:46.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:46 smithi130 bash[27579]: audit 2023-08-28T00:43:46.091268+0000 mon.a (mon.0) 3505 : audit [DBG] from='client.? 172.21.15.6:0/3000649566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:46.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:46 smithi006 bash[27292]: cluster 2023-08-28T00:43:45.010897+0000 mgr.y (mgr.14901) 4955 : cluster [DBG] pgmap v4962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:46.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:46 smithi006 bash[27292]: audit 2023-08-28T00:43:46.091268+0000 mon.a (mon.0) 3505 : audit [DBG] from='client.? 172.21.15.6:0/3000649566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:46.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:46 smithi006 bash[33918]: cluster 2023-08-28T00:43:45.010897+0000 mgr.y (mgr.14901) 4955 : cluster [DBG] pgmap v4962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:46.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:46 smithi006 bash[33918]: audit 2023-08-28T00:43:46.091268+0000 mon.a (mon.0) 3505 : audit [DBG] from='client.? 172.21.15.6:0/3000649566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:47.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:46 smithi130 bash[51302]: debug 2023-08-28T00:43:46.807+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:46.809914+0000) 2023-08-28T00:43:48.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:47 smithi130 bash[51302]: debug 2023-08-28T00:43:47.803+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:47.810227+0000) 2023-08-28T00:43:48.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:48 smithi006 bash[33918]: cluster 2023-08-28T00:43:47.011550+0000 mgr.y (mgr.14901) 4956 : cluster [DBG] pgmap v4963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:48.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:48 smithi006 bash[27292]: cluster 2023-08-28T00:43:47.011550+0000 mgr.y (mgr.14901) 4956 : cluster [DBG] pgmap v4963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:48 smithi130 bash[27579]: cluster 2023-08-28T00:43:47.011550+0000 mgr.y (mgr.14901) 4956 : cluster [DBG] pgmap v4963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:49.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:48 smithi130 bash[51302]: debug 2023-08-28T00:43:48.807+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:48.810528+0000) 2023-08-28T00:43:49.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:49 smithi006 bash[27292]: audit 2023-08-28T00:43:48.507135+0000 mon.c (mon.2) 1922 : audit [DBG] from='client.? 172.21.15.6:0/1947406590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:49.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:49 smithi006 bash[33918]: audit 2023-08-28T00:43:48.507135+0000 mon.c (mon.2) 1922 : audit [DBG] from='client.? 172.21.15.6:0/1947406590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:49.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:49 smithi130 bash[27579]: audit 2023-08-28T00:43:48.507135+0000 mon.c (mon.2) 1922 : audit [DBG] from='client.? 172.21.15.6:0/1947406590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:50.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:49 smithi130 bash[51302]: debug 2023-08-28T00:43:49.807+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:49.810800+0000) 2023-08-28T00:43:50.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:50 smithi006 bash[27292]: cluster 2023-08-28T00:43:49.012307+0000 mgr.y (mgr.14901) 4957 : cluster [DBG] pgmap v4964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:50.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:50 smithi006 bash[33918]: cluster 2023-08-28T00:43:49.012307+0000 mgr.y (mgr.14901) 4957 : cluster [DBG] pgmap v4964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:50 smithi130 bash[27579]: cluster 2023-08-28T00:43:49.012307+0000 mgr.y (mgr.14901) 4957 : cluster [DBG] pgmap v4964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:51.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:50 smithi130 bash[51302]: debug 2023-08-28T00:43:50.807+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:50.811067+0000) 2023-08-28T00:43:51.137 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:43:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:43:50] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:43:51.137 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:43:50.815Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:51.137 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:43:50.817Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:51.137 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:51 smithi006 bash[27292]: audit 2023-08-28T00:43:50.926709+0000 mon.a (mon.0) 3506 : audit [DBG] from='client.? 172.21.15.6:0/3410754066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:51.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:51 smithi006 bash[33918]: audit 2023-08-28T00:43:50.926709+0000 mon.a (mon.0) 3506 : audit [DBG] from='client.? 172.21.15.6:0/3410754066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:51.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:51 smithi130 bash[27579]: audit 2023-08-28T00:43:50.926709+0000 mon.a (mon.0) 3506 : audit [DBG] from='client.? 172.21.15.6:0/3410754066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:52.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:51 smithi130 bash[51302]: debug 2023-08-28T00:43:51.807+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:51.811360+0000) 2023-08-28T00:43:52.396 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:52 smithi006 bash[33918]: cluster 2023-08-28T00:43:51.013294+0000 mgr.y (mgr.14901) 4958 : cluster [DBG] pgmap v4965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:52.396 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:52 smithi006 bash[27292]: cluster 2023-08-28T00:43:51.013294+0000 mgr.y (mgr.14901) 4958 : cluster [DBG] pgmap v4965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:52 smithi130 bash[27579]: cluster 2023-08-28T00:43:51.013294+0000 mgr.y (mgr.14901) 4958 : cluster [DBG] pgmap v4965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:43:52.392Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:43:52.396Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:52 smithi130 bash[51302]: debug 2023-08-28T00:43:52.807+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:52.811665+0000) 2023-08-28T00:43:54.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:53 smithi130 bash[51302]: debug 2023-08-28T00:43:53.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:53.811962+0000) 2023-08-28T00:43:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:43:54.276Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:43:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:43:54.279Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:43:54.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:54 smithi006 bash[27292]: cluster 2023-08-28T00:43:53.013882+0000 mgr.y (mgr.14901) 4959 : cluster [DBG] pgmap v4966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:54.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:54 smithi006 bash[27292]: audit 2023-08-28T00:43:53.337785+0000 mon.a (mon.0) 3507 : audit [DBG] from='client.? 172.21.15.6:0/6075250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:54.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:54 smithi006 bash[33918]: cluster 2023-08-28T00:43:53.013882+0000 mgr.y (mgr.14901) 4959 : cluster [DBG] pgmap v4966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:54.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:54 smithi006 bash[33918]: audit 2023-08-28T00:43:53.337785+0000 mon.a (mon.0) 3507 : audit [DBG] from='client.? 172.21.15.6:0/6075250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:54.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:54 smithi130 bash[27579]: cluster 2023-08-28T00:43:53.013882+0000 mgr.y (mgr.14901) 4959 : cluster [DBG] pgmap v4966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:54.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:54 smithi130 bash[27579]: audit 2023-08-28T00:43:53.337785+0000 mon.a (mon.0) 3507 : audit [DBG] from='client.? 172.21.15.6:0/6075250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:54 smithi130 bash[51302]: debug 2023-08-28T00:43:54.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:54.812257+0000) 2023-08-28T00:43:56.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:55 smithi130 bash[51302]: debug 2023-08-28T00:43:55.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:55.812517+0000) 2023-08-28T00:43:56.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:56 smithi006 bash[27292]: cluster 2023-08-28T00:43:55.014998+0000 mgr.y (mgr.14901) 4960 : cluster [DBG] pgmap v4967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:56.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:56 smithi006 bash[27292]: audit 2023-08-28T00:43:55.754335+0000 mon.c (mon.2) 1923 : audit [DBG] from='client.? 172.21.15.6:0/3200069248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:56.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:56 smithi006 bash[33918]: cluster 2023-08-28T00:43:55.014998+0000 mgr.y (mgr.14901) 4960 : cluster [DBG] pgmap v4967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:56.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:56 smithi006 bash[33918]: audit 2023-08-28T00:43:55.754335+0000 mon.c (mon.2) 1923 : audit [DBG] from='client.? 172.21.15.6:0/3200069248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:56 smithi130 bash[27579]: cluster 2023-08-28T00:43:55.014998+0000 mgr.y (mgr.14901) 4960 : cluster [DBG] pgmap v4967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:56 smithi130 bash[27579]: audit 2023-08-28T00:43:55.754335+0000 mon.c (mon.2) 1923 : audit [DBG] from='client.? 172.21.15.6:0/3200069248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:57.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:56 smithi130 bash[51302]: debug 2023-08-28T00:43:56.807+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:56.812803+0000) 2023-08-28T00:43:58.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:57 smithi130 bash[51302]: debug 2023-08-28T00:43:57.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:57.813070+0000) 2023-08-28T00:43:58.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:58 smithi006 bash[27292]: cluster 2023-08-28T00:43:57.015598+0000 mgr.y (mgr.14901) 4961 : cluster [DBG] pgmap v4968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:58.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:58 smithi006 bash[33918]: cluster 2023-08-28T00:43:57.015598+0000 mgr.y (mgr.14901) 4961 : cluster [DBG] pgmap v4968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:58 smithi130 bash[27579]: cluster 2023-08-28T00:43:57.015598+0000 mgr.y (mgr.14901) 4961 : cluster [DBG] pgmap v4968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:43:59.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:58 smithi130 bash[51302]: debug 2023-08-28T00:43:58.807+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:58.813418+0000) 2023-08-28T00:43:59.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:43:59 smithi006 bash[33918]: audit 2023-08-28T00:43:58.173236+0000 mon.a (mon.0) 3508 : audit [DBG] from='client.? 172.21.15.6:0/346951894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:59.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:43:59 smithi006 bash[27292]: audit 2023-08-28T00:43:58.173236+0000 mon.a (mon.0) 3508 : audit [DBG] from='client.? 172.21.15.6:0/346951894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:43:59 smithi130 bash[27579]: audit 2023-08-28T00:43:58.173236+0000 mon.a (mon.0) 3508 : audit [DBG] from='client.? 172.21.15.6:0/346951894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:00.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:43:59 smithi130 bash[51302]: debug 2023-08-28T00:43:59.807+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:59.813725+0000) 2023-08-28T00:44:00.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:00 smithi006 bash[33918]: cluster 2023-08-28T00:43:59.016486+0000 mgr.y (mgr.14901) 4962 : cluster [DBG] pgmap v4969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:00.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:00 smithi006 bash[27292]: cluster 2023-08-28T00:43:59.016486+0000 mgr.y (mgr.14901) 4962 : cluster [DBG] pgmap v4969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:00 smithi130 bash[27579]: cluster 2023-08-28T00:43:59.016486+0000 mgr.y (mgr.14901) 4962 : cluster [DBG] pgmap v4969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:01.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:00 smithi130 bash[51302]: debug 2023-08-28T00:44:00.807+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:00.813983+0000) 2023-08-28T00:44:01.171 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:44:00.815Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:01.172 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:44:00.818Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:01.172 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:44:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:44:00] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:44:01.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:01 smithi006 bash[33918]: audit 2023-08-28T00:44:00.593993+0000 mon.c (mon.2) 1924 : audit [DBG] from='client.? 172.21.15.6:0/1389675544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:01.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:01 smithi006 bash[27292]: audit 2023-08-28T00:44:00.593993+0000 mon.c (mon.2) 1924 : audit [DBG] from='client.? 172.21.15.6:0/1389675544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:01.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:01 smithi130 bash[27579]: audit 2023-08-28T00:44:00.593993+0000 mon.c (mon.2) 1924 : audit [DBG] from='client.? 172.21.15.6:0/1389675544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:02.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:01 smithi130 bash[51302]: debug 2023-08-28T00:44:01.807+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:01.814280+0000) 2023-08-28T00:44:02.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:02 smithi006 bash[33918]: cluster 2023-08-28T00:44:01.017552+0000 mgr.y (mgr.14901) 4963 : cluster [DBG] pgmap v4970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:02.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:02 smithi006 bash[27292]: cluster 2023-08-28T00:44:01.017552+0000 mgr.y (mgr.14901) 4963 : cluster [DBG] pgmap v4970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:02.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:44:02.393Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:02.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:44:02.395Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:02 smithi130 bash[27579]: cluster 2023-08-28T00:44:01.017552+0000 mgr.y (mgr.14901) 4963 : cluster [DBG] pgmap v4970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:02 smithi130 bash[51302]: debug 2023-08-28T00:44:02.807+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:02.814611+0000) 2023-08-28T00:44:03.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:03 smithi006 bash[27292]: audit 2023-08-28T00:44:03.003809+0000 mon.c (mon.2) 1925 : audit [DBG] from='client.? 172.21.15.6:0/2480198338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:03.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:03 smithi006 bash[33918]: audit 2023-08-28T00:44:03.003809+0000 mon.c (mon.2) 1925 : audit [DBG] from='client.? 172.21.15.6:0/2480198338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:03.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:03 smithi130 bash[27579]: audit 2023-08-28T00:44:03.003809+0000 mon.c (mon.2) 1925 : audit [DBG] from='client.? 172.21.15.6:0/2480198338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:04.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:03 smithi130 bash[51302]: debug 2023-08-28T00:44:03.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:03.814920+0000) 2023-08-28T00:44:04.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:04 smithi006 bash[33918]: cluster 2023-08-28T00:44:03.018135+0000 mgr.y (mgr.14901) 4964 : cluster [DBG] pgmap v4971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:44:04.277Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:04.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:44:04.279Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:04.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:04 smithi006 bash[27292]: cluster 2023-08-28T00:44:03.018135+0000 mgr.y (mgr.14901) 4964 : cluster [DBG] pgmap v4971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:04 smithi130 bash[27579]: cluster 2023-08-28T00:44:03.018135+0000 mgr.y (mgr.14901) 4964 : cluster [DBG] pgmap v4971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:04 smithi130 bash[51302]: debug 2023-08-28T00:44:04.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:04.815213+0000) 2023-08-28T00:44:06.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:05 smithi130 bash[51302]: debug 2023-08-28T00:44:05.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:05.815441+0000) 2023-08-28T00:44:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:06 smithi006 bash[33918]: cluster 2023-08-28T00:44:05.019263+0000 mgr.y (mgr.14901) 4965 : cluster [DBG] pgmap v4972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:06 smithi006 bash[33918]: audit 2023-08-28T00:44:05.416065+0000 mon.a (mon.0) 3509 : audit [DBG] from='client.? 172.21.15.6:0/2152523236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:06.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:06 smithi006 bash[27292]: cluster 2023-08-28T00:44:05.019263+0000 mgr.y (mgr.14901) 4965 : cluster [DBG] pgmap v4972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:06.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:06 smithi006 bash[27292]: audit 2023-08-28T00:44:05.416065+0000 mon.a (mon.0) 3509 : audit [DBG] from='client.? 172.21.15.6:0/2152523236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:06 smithi130 bash[27579]: cluster 2023-08-28T00:44:05.019263+0000 mgr.y (mgr.14901) 4965 : cluster [DBG] pgmap v4972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:06 smithi130 bash[27579]: audit 2023-08-28T00:44:05.416065+0000 mon.a (mon.0) 3509 : audit [DBG] from='client.? 172.21.15.6:0/2152523236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:07.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:06 smithi130 bash[51302]: debug 2023-08-28T00:44:06.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:06.815666+0000) 2023-08-28T00:44:08.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:07 smithi130 bash[51302]: debug 2023-08-28T00:44:07.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:07.815896+0000) 2023-08-28T00:44:08.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:08 smithi006 bash[27292]: cluster 2023-08-28T00:44:07.019902+0000 mgr.y (mgr.14901) 4966 : cluster [DBG] pgmap v4973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:08.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:08 smithi006 bash[27292]: audit 2023-08-28T00:44:07.830090+0000 mon.c (mon.2) 1926 : audit [DBG] from='client.? 172.21.15.6:0/4105277655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:08.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:08 smithi006 bash[33918]: cluster 2023-08-28T00:44:07.019902+0000 mgr.y (mgr.14901) 4966 : cluster [DBG] pgmap v4973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:08.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:08 smithi006 bash[33918]: audit 2023-08-28T00:44:07.830090+0000 mon.c (mon.2) 1926 : audit [DBG] from='client.? 172.21.15.6:0/4105277655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:08 smithi130 bash[27579]: cluster 2023-08-28T00:44:07.019902+0000 mgr.y (mgr.14901) 4966 : cluster [DBG] pgmap v4973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:08 smithi130 bash[27579]: audit 2023-08-28T00:44:07.830090+0000 mon.c (mon.2) 1926 : audit [DBG] from='client.? 172.21.15.6:0/4105277655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:09.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:08 smithi130 bash[51302]: debug 2023-08-28T00:44:08.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:08.816159+0000) 2023-08-28T00:44:10.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:09 smithi130 bash[51302]: debug 2023-08-28T00:44:09.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:09.816501+0000) 2023-08-28T00:44:10.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:10 smithi006 bash[27292]: cluster 2023-08-28T00:44:09.020732+0000 mgr.y (mgr.14901) 4967 : cluster [DBG] pgmap v4974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:10.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:10 smithi006 bash[33918]: cluster 2023-08-28T00:44:09.020732+0000 mgr.y (mgr.14901) 4967 : cluster [DBG] pgmap v4974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:10 smithi130 bash[27579]: cluster 2023-08-28T00:44:09.020732+0000 mgr.y (mgr.14901) 4967 : cluster [DBG] pgmap v4974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:11.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:10 smithi130 bash[51302]: debug 2023-08-28T00:44:10.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:10.816765+0000) 2023-08-28T00:44:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:44:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:44:10] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:44:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:44:10.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:44:10.819Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:11.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:11 smithi006 bash[27292]: audit 2023-08-28T00:44:10.249742+0000 mon.a (mon.0) 3510 : audit [DBG] from='client.? 172.21.15.6:0/1505991536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:11.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:11 smithi006 bash[33918]: audit 2023-08-28T00:44:10.249742+0000 mon.a (mon.0) 3510 : audit [DBG] from='client.? 172.21.15.6:0/1505991536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:11.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:11 smithi130 bash[27579]: audit 2023-08-28T00:44:10.249742+0000 mon.a (mon.0) 3510 : audit [DBG] from='client.? 172.21.15.6:0/1505991536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:12.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:11 smithi130 bash[51302]: debug 2023-08-28T00:44:11.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:11.817031+0000) 2023-08-28T00:44:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:12 smithi130 bash[27579]: cluster 2023-08-28T00:44:11.021780+0000 mgr.y (mgr.14901) 4968 : cluster [DBG] pgmap v4975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:12 smithi006 bash[33918]: cluster 2023-08-28T00:44:11.021780+0000 mgr.y (mgr.14901) 4968 : cluster [DBG] pgmap v4975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:12 smithi006 bash[27292]: cluster 2023-08-28T00:44:11.021780+0000 mgr.y (mgr.14901) 4968 : cluster [DBG] pgmap v4975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:44:12.394Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:44:12.396Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:13.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:12 smithi130 bash[51302]: debug 2023-08-28T00:44:12.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:12.817312+0000) 2023-08-28T00:44:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:13 smithi130 bash[27579]: audit 2023-08-28T00:44:12.663761+0000 mon.c (mon.2) 1927 : audit [DBG] from='client.? 172.21.15.6:0/3690322106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:13.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:13 smithi006 bash[33918]: audit 2023-08-28T00:44:12.663761+0000 mon.c (mon.2) 1927 : audit [DBG] from='client.? 172.21.15.6:0/3690322106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:13.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:13 smithi006 bash[27292]: audit 2023-08-28T00:44:12.663761+0000 mon.c (mon.2) 1927 : audit [DBG] from='client.? 172.21.15.6:0/3690322106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:14.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:13 smithi130 bash[51302]: debug 2023-08-28T00:44:13.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:13.817607+0000) 2023-08-28T00:44:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:14 smithi130 bash[27579]: cluster 2023-08-28T00:44:13.022594+0000 mgr.y (mgr.14901) 4969 : cluster [DBG] pgmap v4976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:14.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:14 smithi006 bash[27292]: cluster 2023-08-28T00:44:13.022594+0000 mgr.y (mgr.14901) 4969 : cluster [DBG] pgmap v4976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:14.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:14 smithi006 bash[33918]: cluster 2023-08-28T00:44:13.022594+0000 mgr.y (mgr.14901) 4969 : cluster [DBG] pgmap v4976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:44:14.277Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:14.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:44:14.279Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:15.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:14 smithi130 bash[51302]: debug 2023-08-28T00:44:14.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:14.817878+0000) 2023-08-28T00:44:15.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:15 smithi130 bash[27579]: audit 2023-08-28T00:44:15.094658+0000 mon.c (mon.2) 1928 : audit [DBG] from='client.? 172.21.15.6:0/1483759192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:15.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:15 smithi006 bash[27292]: audit 2023-08-28T00:44:15.094658+0000 mon.c (mon.2) 1928 : audit [DBG] from='client.? 172.21.15.6:0/1483759192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:15.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:15 smithi006 bash[33918]: audit 2023-08-28T00:44:15.094658+0000 mon.c (mon.2) 1928 : audit [DBG] from='client.? 172.21.15.6:0/1483759192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:16.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:15 smithi130 bash[51302]: debug 2023-08-28T00:44:15.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:15.818136+0000) 2023-08-28T00:44:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:16 smithi130 bash[27579]: cluster 2023-08-28T00:44:15.023641+0000 mgr.y (mgr.14901) 4970 : cluster [DBG] pgmap v4977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:16.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:16 smithi006 bash[27292]: cluster 2023-08-28T00:44:15.023641+0000 mgr.y (mgr.14901) 4970 : cluster [DBG] pgmap v4977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:16 smithi006 bash[33918]: cluster 2023-08-28T00:44:15.023641+0000 mgr.y (mgr.14901) 4970 : cluster [DBG] pgmap v4977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:17.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:16 smithi130 bash[51302]: debug 2023-08-28T00:44:16.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:16.818395+0000) 2023-08-28T00:44:18.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:17 smithi130 bash[51302]: debug 2023-08-28T00:44:17.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:17.818712+0000) 2023-08-28T00:44:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:18 smithi130 bash[27579]: cluster 2023-08-28T00:44:17.024298+0000 mgr.y (mgr.14901) 4971 : cluster [DBG] pgmap v4978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:18 smithi130 bash[27579]: audit 2023-08-28T00:44:17.500228+0000 mon.a (mon.0) 3511 : audit [DBG] from='client.? 172.21.15.6:0/2686263508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:18 smithi006 bash[27292]: cluster 2023-08-28T00:44:17.024298+0000 mgr.y (mgr.14901) 4971 : cluster [DBG] pgmap v4978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:18 smithi006 bash[27292]: audit 2023-08-28T00:44:17.500228+0000 mon.a (mon.0) 3511 : audit [DBG] from='client.? 172.21.15.6:0/2686263508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:18.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:18 smithi006 bash[33918]: cluster 2023-08-28T00:44:17.024298+0000 mgr.y (mgr.14901) 4971 : cluster [DBG] pgmap v4978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:18.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:18 smithi006 bash[33918]: audit 2023-08-28T00:44:17.500228+0000 mon.a (mon.0) 3511 : audit [DBG] from='client.? 172.21.15.6:0/2686263508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:19.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:18 smithi130 bash[51302]: debug 2023-08-28T00:44:18.815+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:18.818981+0000) 2023-08-28T00:44:20.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:19 smithi130 bash[51302]: debug 2023-08-28T00:44:19.815+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:19.819233+0000) 2023-08-28T00:44:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:20 smithi130 bash[27579]: cluster 2023-08-28T00:44:19.025274+0000 mgr.y (mgr.14901) 4972 : cluster [DBG] pgmap v4979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:20 smithi130 bash[27579]: audit 2023-08-28T00:44:19.922370+0000 mon.a (mon.0) 3512 : audit [DBG] from='client.? 172.21.15.6:0/202994404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:20 smithi006 bash[27292]: cluster 2023-08-28T00:44:19.025274+0000 mgr.y (mgr.14901) 4972 : cluster [DBG] pgmap v4979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:20 smithi006 bash[27292]: audit 2023-08-28T00:44:19.922370+0000 mon.a (mon.0) 3512 : audit [DBG] from='client.? 172.21.15.6:0/202994404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:20.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:20 smithi006 bash[33918]: cluster 2023-08-28T00:44:19.025274+0000 mgr.y (mgr.14901) 4972 : cluster [DBG] pgmap v4979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:20.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:20 smithi006 bash[33918]: audit 2023-08-28T00:44:19.922370+0000 mon.a (mon.0) 3512 : audit [DBG] from='client.? 172.21.15.6:0/202994404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:21.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:20 smithi130 bash[51302]: debug 2023-08-28T00:44:20.815+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:20.819490+0000) 2023-08-28T00:44:21.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:44:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:44:20] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:44:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:44:20.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:44:20.819Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:22.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:21 smithi130 bash[51302]: debug 2023-08-28T00:44:21.815+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:21.819798+0000) 2023-08-28T00:44:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:22 smithi130 bash[27579]: cluster 2023-08-28T00:44:21.026146+0000 mgr.y (mgr.14901) 4973 : cluster [DBG] pgmap v4980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:22 smithi006 bash[33918]: cluster 2023-08-28T00:44:21.026146+0000 mgr.y (mgr.14901) 4973 : cluster [DBG] pgmap v4980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:22 smithi006 bash[27292]: cluster 2023-08-28T00:44:21.026146+0000 mgr.y (mgr.14901) 4973 : cluster [DBG] pgmap v4980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:44:22.395Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:44:22.397Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:23.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:22 smithi130 bash[51302]: debug 2023-08-28T00:44:22.815+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:22.820094+0000) 2023-08-28T00:44:23.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:23 smithi130 bash[27579]: audit 2023-08-28T00:44:22.346589+0000 mon.a (mon.0) 3513 : audit [DBG] from='client.? 172.21.15.6:0/3173205945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:23.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:23 smithi006 bash[27292]: audit 2023-08-28T00:44:22.346589+0000 mon.a (mon.0) 3513 : audit [DBG] from='client.? 172.21.15.6:0/3173205945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:23.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:23 smithi006 bash[33918]: audit 2023-08-28T00:44:22.346589+0000 mon.a (mon.0) 3513 : audit [DBG] from='client.? 172.21.15.6:0/3173205945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:24.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:23 smithi130 bash[51302]: debug 2023-08-28T00:44:23.815+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:23.820372+0000) 2023-08-28T00:44:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:24 smithi130 bash[27579]: cluster 2023-08-28T00:44:23.026844+0000 mgr.y (mgr.14901) 4974 : cluster [DBG] pgmap v4981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:24.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:24 smithi006 bash[27292]: cluster 2023-08-28T00:44:23.026844+0000 mgr.y (mgr.14901) 4974 : cluster [DBG] pgmap v4981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:24.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:24 smithi006 bash[33918]: cluster 2023-08-28T00:44:23.026844+0000 mgr.y (mgr.14901) 4974 : cluster [DBG] pgmap v4981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:44:24.278Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:44:24.280Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:24 smithi130 bash[51302]: debug 2023-08-28T00:44:24.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:24.820648+0000) 2023-08-28T00:44:25.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:25 smithi130 bash[27579]: audit 2023-08-28T00:44:24.763745+0000 mon.a (mon.0) 3514 : audit [DBG] from='client.? 172.21.15.6:0/1273014420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:25.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:25 smithi006 bash[33918]: audit 2023-08-28T00:44:24.763745+0000 mon.a (mon.0) 3514 : audit [DBG] from='client.? 172.21.15.6:0/1273014420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:25.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:25 smithi006 bash[27292]: audit 2023-08-28T00:44:24.763745+0000 mon.a (mon.0) 3514 : audit [DBG] from='client.? 172.21.15.6:0/1273014420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:26.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:25 smithi130 bash[51302]: debug 2023-08-28T00:44:25.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:25.821022+0000) 2023-08-28T00:44:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:26 smithi130 bash[27579]: cluster 2023-08-28T00:44:25.028000+0000 mgr.y (mgr.14901) 4975 : cluster [DBG] pgmap v4982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:26.677 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:26 smithi130 bash[27579]: audit 2023-08-28T00:44:26.237911+0000 mon.b (mon.1) 598 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:44:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:26 smithi006 bash[33918]: cluster 2023-08-28T00:44:25.028000+0000 mgr.y (mgr.14901) 4975 : cluster [DBG] pgmap v4982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:26 smithi006 bash[33918]: audit 2023-08-28T00:44:26.237911+0000 mon.b (mon.1) 598 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:44:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:26 smithi006 bash[27292]: cluster 2023-08-28T00:44:25.028000+0000 mgr.y (mgr.14901) 4975 : cluster [DBG] pgmap v4982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:26 smithi006 bash[27292]: audit 2023-08-28T00:44:26.237911+0000 mon.b (mon.1) 598 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:44:27.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:26 smithi130 bash[51302]: debug 2023-08-28T00:44:26.815+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:26.821287+0000) 2023-08-28T00:44:27.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:27 smithi130 bash[27579]: audit 2023-08-28T00:44:27.180665+0000 mon.a (mon.0) 3515 : audit [DBG] from='client.? 172.21.15.6:0/446186975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:27.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:27 smithi006 bash[27292]: audit 2023-08-28T00:44:27.180665+0000 mon.a (mon.0) 3515 : audit [DBG] from='client.? 172.21.15.6:0/446186975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:27.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:27 smithi006 bash[33918]: audit 2023-08-28T00:44:27.180665+0000 mon.a (mon.0) 3515 : audit [DBG] from='client.? 172.21.15.6:0/446186975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:28.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:27 smithi130 bash[51302]: debug 2023-08-28T00:44:27.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:27.821619+0000) 2023-08-28T00:44:28.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:28 smithi130 bash[27579]: cluster 2023-08-28T00:44:27.028620+0000 mgr.y (mgr.14901) 4976 : cluster [DBG] pgmap v4983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:28.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:28 smithi006 bash[33918]: cluster 2023-08-28T00:44:27.028620+0000 mgr.y (mgr.14901) 4976 : cluster [DBG] pgmap v4983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:28.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:28 smithi006 bash[27292]: cluster 2023-08-28T00:44:27.028620+0000 mgr.y (mgr.14901) 4976 : cluster [DBG] pgmap v4983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:29.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:28 smithi130 bash[51302]: debug 2023-08-28T00:44:28.815+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:28.821998+0000) 2023-08-28T00:44:30.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:29 smithi130 bash[51302]: debug 2023-08-28T00:44:29.815+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:29.822338+0000) 2023-08-28T00:44:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:30 smithi130 bash[27579]: cluster 2023-08-28T00:44:29.029412+0000 mgr.y (mgr.14901) 4977 : cluster [DBG] pgmap v4984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:30 smithi130 bash[27579]: audit 2023-08-28T00:44:29.597703+0000 mon.c (mon.2) 1929 : audit [DBG] from='client.? 172.21.15.6:0/2304122650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:30.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:30 smithi006 bash[33918]: cluster 2023-08-28T00:44:29.029412+0000 mgr.y (mgr.14901) 4977 : cluster [DBG] pgmap v4984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:30.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:30 smithi006 bash[33918]: audit 2023-08-28T00:44:29.597703+0000 mon.c (mon.2) 1929 : audit [DBG] from='client.? 172.21.15.6:0/2304122650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:30.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:30 smithi006 bash[27292]: cluster 2023-08-28T00:44:29.029412+0000 mgr.y (mgr.14901) 4977 : cluster [DBG] pgmap v4984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:30.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:30 smithi006 bash[27292]: audit 2023-08-28T00:44:29.597703+0000 mon.c (mon.2) 1929 : audit [DBG] from='client.? 172.21.15.6:0/2304122650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:31.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:30 smithi130 bash[51302]: debug 2023-08-28T00:44:30.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:30.822657+0000) 2023-08-28T00:44:31.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:44:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:44:30] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:44:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:44:30.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:44:30.820Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:32.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:31 smithi130 bash[51302]: debug 2023-08-28T00:44:31.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:31.822913+0000) 2023-08-28T00:44:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:32 smithi130 bash[27579]: cluster 2023-08-28T00:44:31.030229+0000 mgr.y (mgr.14901) 4978 : cluster [DBG] pgmap v4985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:32 smithi130 bash[27579]: audit 2023-08-28T00:44:32.044451+0000 mon.c (mon.2) 1930 : audit [DBG] from='client.? 172.21.15.6:0/4134929676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:32 smithi130 bash[27579]: audit 2023-08-28T00:44:32.118165+0000 mon.a (mon.0) 3516 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:44:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:32 smithi006 bash[27292]: cluster 2023-08-28T00:44:31.030229+0000 mgr.y (mgr.14901) 4978 : cluster [DBG] pgmap v4985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:32 smithi006 bash[27292]: audit 2023-08-28T00:44:32.044451+0000 mon.c (mon.2) 1930 : audit [DBG] from='client.? 172.21.15.6:0/4134929676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:32 smithi006 bash[27292]: audit 2023-08-28T00:44:32.118165+0000 mon.a (mon.0) 3516 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:44:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:32 smithi006 bash[33918]: cluster 2023-08-28T00:44:31.030229+0000 mgr.y (mgr.14901) 4978 : cluster [DBG] pgmap v4985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:32 smithi006 bash[33918]: audit 2023-08-28T00:44:32.044451+0000 mon.c (mon.2) 1930 : audit [DBG] from='client.? 172.21.15.6:0/4134929676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:32.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:32 smithi006 bash[33918]: audit 2023-08-28T00:44:32.118165+0000 mon.a (mon.0) 3516 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:44:32.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:44:32.396Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:32.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:44:32.398Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:33.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:32 smithi130 bash[51302]: debug 2023-08-28T00:44:32.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:32.823184+0000) 2023-08-28T00:44:33.825 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:33 smithi130 bash[27579]: audit 2023-08-28T00:44:32.570416+0000 mon.a (mon.0) 3517 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:44:33.825 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:33 smithi130 bash[27579]: audit 2023-08-28T00:44:32.842766+0000 mon.b (mon.1) 599 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:44:33.825 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:33 smithi130 bash[27579]: audit 2023-08-28T00:44:32.844987+0000 mon.b (mon.1) 600 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:44:33.825 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:33 smithi130 bash[27579]: audit 2023-08-28T00:44:32.852607+0000 mon.a (mon.0) 3518 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:44:33.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:33 smithi006 bash[33918]: audit 2023-08-28T00:44:32.570416+0000 mon.a (mon.0) 3517 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:44:33.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:33 smithi006 bash[33918]: audit 2023-08-28T00:44:32.842766+0000 mon.b (mon.1) 599 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:44:33.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:33 smithi006 bash[33918]: audit 2023-08-28T00:44:32.844987+0000 mon.b (mon.1) 600 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:44:33.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:33 smithi006 bash[33918]: audit 2023-08-28T00:44:32.852607+0000 mon.a (mon.0) 3518 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:44:33.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:33 smithi006 bash[27292]: audit 2023-08-28T00:44:32.570416+0000 mon.a (mon.0) 3517 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:44:33.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:33 smithi006 bash[27292]: audit 2023-08-28T00:44:32.842766+0000 mon.b (mon.1) 599 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:44:33.970 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:33 smithi006 bash[27292]: audit 2023-08-28T00:44:32.844987+0000 mon.b (mon.1) 600 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:44:33.971 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:33 smithi006 bash[27292]: audit 2023-08-28T00:44:32.852607+0000 mon.a (mon.0) 3518 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:44:34.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:33 smithi130 bash[51302]: debug 2023-08-28T00:44:33.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:33.823449+0000) 2023-08-28T00:44:34.575 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:44:34.278Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:34.575 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:44:34.281Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:34 smithi130 bash[27579]: cluster 2023-08-28T00:44:33.030901+0000 mgr.y (mgr.14901) 4979 : cluster [DBG] pgmap v4986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:34 smithi130 bash[27579]: audit 2023-08-28T00:44:34.469372+0000 mon.a (mon.0) 3519 : audit [DBG] from='client.? 172.21.15.6:0/3141100501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:34.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:34 smithi130 bash[51302]: debug 2023-08-28T00:44:34.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:34.823776+0000) 2023-08-28T00:44:34.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:34 smithi006 bash[27292]: cluster 2023-08-28T00:44:33.030901+0000 mgr.y (mgr.14901) 4979 : cluster [DBG] pgmap v4986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:34.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:34 smithi006 bash[27292]: audit 2023-08-28T00:44:34.469372+0000 mon.a (mon.0) 3519 : audit [DBG] from='client.? 172.21.15.6:0/3141100501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:34.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:34 smithi006 bash[33918]: cluster 2023-08-28T00:44:33.030901+0000 mgr.y (mgr.14901) 4979 : cluster [DBG] pgmap v4986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:34.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:34 smithi006 bash[33918]: audit 2023-08-28T00:44:34.469372+0000 mon.a (mon.0) 3519 : audit [DBG] from='client.? 172.21.15.6:0/3141100501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:36.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:35 smithi130 bash[51302]: debug 2023-08-28T00:44:35.823+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:35.824061+0000) 2023-08-28T00:44:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:36 smithi130 bash[27579]: cluster 2023-08-28T00:44:35.032080+0000 mgr.y (mgr.14901) 4980 : cluster [DBG] pgmap v4987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:36.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:36 smithi130 bash[51302]: debug 2023-08-28T00:44:36.818+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:36.824348+0000) 2023-08-28T00:44:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:36 smithi006 bash[33918]: cluster 2023-08-28T00:44:35.032080+0000 mgr.y (mgr.14901) 4980 : cluster [DBG] pgmap v4987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:36.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:36 smithi006 bash[27292]: cluster 2023-08-28T00:44:35.032080+0000 mgr.y (mgr.14901) 4980 : cluster [DBG] pgmap v4987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:37.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:37 smithi130 bash[27579]: audit 2023-08-28T00:44:36.882909+0000 mon.c (mon.2) 1931 : audit [DBG] from='client.? 172.21.15.6:0/483258427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:37.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:37 smithi130 bash[51302]: debug 2023-08-28T00:44:37.822+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:37.824651+0000) 2023-08-28T00:44:37.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:37 smithi006 bash[33918]: audit 2023-08-28T00:44:36.882909+0000 mon.c (mon.2) 1931 : audit [DBG] from='client.? 172.21.15.6:0/483258427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:37.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:37 smithi006 bash[27292]: audit 2023-08-28T00:44:36.882909+0000 mon.c (mon.2) 1931 : audit [DBG] from='client.? 172.21.15.6:0/483258427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:38 smithi130 bash[27579]: cluster 2023-08-28T00:44:37.032910+0000 mgr.y (mgr.14901) 4981 : cluster [DBG] pgmap v4988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:38.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:38 smithi130 bash[51302]: debug 2023-08-28T00:44:38.818+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:38.824977+0000) 2023-08-28T00:44:38.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:38 smithi006 bash[27292]: cluster 2023-08-28T00:44:37.032910+0000 mgr.y (mgr.14901) 4981 : cluster [DBG] pgmap v4988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:38.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:38 smithi006 bash[33918]: cluster 2023-08-28T00:44:37.032910+0000 mgr.y (mgr.14901) 4981 : cluster [DBG] pgmap v4988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:39.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:39 smithi130 bash[51302]: debug 2023-08-28T00:44:39.818+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:39.825257+0000) 2023-08-28T00:44:39.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:39 smithi130 bash[27579]: audit 2023-08-28T00:44:39.300291+0000 mon.a (mon.0) 3520 : audit [DBG] from='client.? 172.21.15.6:0/1513509041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:39.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:39 smithi006 bash[33918]: audit 2023-08-28T00:44:39.300291+0000 mon.a (mon.0) 3520 : audit [DBG] from='client.? 172.21.15.6:0/1513509041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:39.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:39 smithi006 bash[27292]: audit 2023-08-28T00:44:39.300291+0000 mon.a (mon.0) 3520 : audit [DBG] from='client.? 172.21.15.6:0/1513509041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:40 smithi130 bash[27579]: cluster 2023-08-28T00:44:39.033874+0000 mgr.y (mgr.14901) 4982 : cluster [DBG] pgmap v4989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:40.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:40 smithi130 bash[51302]: debug 2023-08-28T00:44:40.822+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:40.825607+0000) 2023-08-28T00:44:40.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:40 smithi006 bash[27292]: cluster 2023-08-28T00:44:39.033874+0000 mgr.y (mgr.14901) 4982 : cluster [DBG] pgmap v4989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:40.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:44:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:44:40] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:44:40.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:40 smithi006 bash[33918]: cluster 2023-08-28T00:44:39.033874+0000 mgr.y (mgr.14901) 4982 : cluster [DBG] pgmap v4989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:40.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:44:40.819Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:40.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:44:40.820Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:42.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:41 smithi130 bash[51302]: debug 2023-08-28T00:44:41.822+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:41.825887+0000) 2023-08-28T00:44:42.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:42 smithi006 bash[33918]: cluster 2023-08-28T00:44:41.034555+0000 mgr.y (mgr.14901) 4983 : cluster [DBG] pgmap v4990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:42.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:42 smithi006 bash[33918]: audit 2023-08-28T00:44:41.720809+0000 mon.a (mon.0) 3521 : audit [DBG] from='client.? 172.21.15.6:0/4003493040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:44:42.396Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:44:42.398Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:42.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:42 smithi006 bash[27292]: cluster 2023-08-28T00:44:41.034555+0000 mgr.y (mgr.14901) 4983 : cluster [DBG] pgmap v4990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:42.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:42 smithi006 bash[27292]: audit 2023-08-28T00:44:41.720809+0000 mon.a (mon.0) 3521 : audit [DBG] from='client.? 172.21.15.6:0/4003493040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:42 smithi130 bash[27579]: cluster 2023-08-28T00:44:41.034555+0000 mgr.y (mgr.14901) 4983 : cluster [DBG] pgmap v4990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:42 smithi130 bash[27579]: audit 2023-08-28T00:44:41.720809+0000 mon.a (mon.0) 3521 : audit [DBG] from='client.? 172.21.15.6:0/4003493040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:42.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:42 smithi130 bash[51302]: debug 2023-08-28T00:44:42.822+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:42.826172+0000) 2023-08-28T00:44:44.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:43 smithi130 bash[51302]: debug 2023-08-28T00:44:43.818+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:43.826448+0000) 2023-08-28T00:44:44.612 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:44:44.279Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:44.612 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:44:44.281Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:44 smithi130 bash[27579]: cluster 2023-08-28T00:44:43.035196+0000 mgr.y (mgr.14901) 4984 : cluster [DBG] pgmap v4991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:44 smithi130 bash[27579]: audit 2023-08-28T00:44:44.134469+0000 mon.a (mon.0) 3522 : audit [DBG] from='client.? 172.21.15.6:0/305910472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:44.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:44 smithi130 bash[51302]: debug 2023-08-28T00:44:44.822+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:44.826725+0000) 2023-08-28T00:44:44.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:44 smithi006 bash[33918]: cluster 2023-08-28T00:44:43.035196+0000 mgr.y (mgr.14901) 4984 : cluster [DBG] pgmap v4991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:44.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:44 smithi006 bash[33918]: audit 2023-08-28T00:44:44.134469+0000 mon.a (mon.0) 3522 : audit [DBG] from='client.? 172.21.15.6:0/305910472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:44.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:44 smithi006 bash[27292]: cluster 2023-08-28T00:44:43.035196+0000 mgr.y (mgr.14901) 4984 : cluster [DBG] pgmap v4991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:44.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:44 smithi006 bash[27292]: audit 2023-08-28T00:44:44.134469+0000 mon.a (mon.0) 3522 : audit [DBG] from='client.? 172.21.15.6:0/305910472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:46.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:45 smithi130 bash[51302]: debug 2023-08-28T00:44:45.826+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:45.827022+0000) 2023-08-28T00:44:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:46 smithi130 bash[27579]: cluster 2023-08-28T00:44:45.036376+0000 mgr.y (mgr.14901) 4985 : cluster [DBG] pgmap v4992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:46 smithi130 bash[27579]: audit 2023-08-28T00:44:46.553666+0000 mon.c (mon.2) 1932 : audit [DBG] from='client.? 172.21.15.6:0/1169750870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:46.869 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:46 smithi130 bash[51302]: debug 2023-08-28T00:44:46.826+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:46.827307+0000) 2023-08-28T00:44:46.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:46 smithi006 bash[33918]: cluster 2023-08-28T00:44:45.036376+0000 mgr.y (mgr.14901) 4985 : cluster [DBG] pgmap v4992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:46.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:46 smithi006 bash[33918]: audit 2023-08-28T00:44:46.553666+0000 mon.c (mon.2) 1932 : audit [DBG] from='client.? 172.21.15.6:0/1169750870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:46.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:46 smithi006 bash[27292]: cluster 2023-08-28T00:44:45.036376+0000 mgr.y (mgr.14901) 4985 : cluster [DBG] pgmap v4992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:46.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:46 smithi006 bash[27292]: audit 2023-08-28T00:44:46.553666+0000 mon.c (mon.2) 1932 : audit [DBG] from='client.? 172.21.15.6:0/1169750870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:48.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:47 smithi130 bash[51302]: debug 2023-08-28T00:44:47.826+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:47.827655+0000) 2023-08-28T00:44:48.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:48 smithi006 bash[33918]: cluster 2023-08-28T00:44:47.037118+0000 mgr.y (mgr.14901) 4986 : cluster [DBG] pgmap v4993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:48.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:48 smithi006 bash[27292]: cluster 2023-08-28T00:44:47.037118+0000 mgr.y (mgr.14901) 4986 : cluster [DBG] pgmap v4993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:48 smithi130 bash[27579]: cluster 2023-08-28T00:44:47.037118+0000 mgr.y (mgr.14901) 4986 : cluster [DBG] pgmap v4993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:49.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:48 smithi130 bash[51302]: debug 2023-08-28T00:44:48.826+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:48.827945+0000) 2023-08-28T00:44:49.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:49 smithi006 bash[33918]: audit 2023-08-28T00:44:48.974969+0000 mon.c (mon.2) 1933 : audit [DBG] from='client.? 172.21.15.6:0/700694524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:49.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:49 smithi006 bash[27292]: audit 2023-08-28T00:44:48.974969+0000 mon.c (mon.2) 1933 : audit [DBG] from='client.? 172.21.15.6:0/700694524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:50.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:49 smithi130 bash[27579]: audit 2023-08-28T00:44:48.974969+0000 mon.c (mon.2) 1933 : audit [DBG] from='client.? 172.21.15.6:0/700694524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:50.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:49 smithi130 bash[51302]: debug 2023-08-28T00:44:49.826+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:49.828257+0000) 2023-08-28T00:44:50.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:44:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:44:50] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:44:50.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:50 smithi006 bash[33918]: cluster 2023-08-28T00:44:49.038271+0000 mgr.y (mgr.14901) 4987 : cluster [DBG] pgmap v4994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:50.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:50 smithi006 bash[27292]: cluster 2023-08-28T00:44:49.038271+0000 mgr.y (mgr.14901) 4987 : cluster [DBG] pgmap v4994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:50.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:44:50.820Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:50.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:44:50.822Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:50 smithi130 bash[27579]: cluster 2023-08-28T00:44:49.038271+0000 mgr.y (mgr.14901) 4987 : cluster [DBG] pgmap v4994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:51.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:50 smithi130 bash[51302]: debug 2023-08-28T00:44:50.826+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:50.828544+0000) 2023-08-28T00:44:52.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:51 smithi130 bash[51302]: debug 2023-08-28T00:44:51.826+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:51.828831+0000) 2023-08-28T00:44:52.401 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:52 smithi006 bash[33918]: cluster 2023-08-28T00:44:51.038934+0000 mgr.y (mgr.14901) 4988 : cluster [DBG] pgmap v4995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:52.401 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:52 smithi006 bash[33918]: audit 2023-08-28T00:44:51.391909+0000 mon.c (mon.2) 1934 : audit [DBG] from='client.? 172.21.15.6:0/3852252661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:52.402 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:52 smithi006 bash[27292]: cluster 2023-08-28T00:44:51.038934+0000 mgr.y (mgr.14901) 4988 : cluster [DBG] pgmap v4995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:52.402 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:52 smithi006 bash[27292]: audit 2023-08-28T00:44:51.391909+0000 mon.c (mon.2) 1934 : audit [DBG] from='client.? 172.21.15.6:0/3852252661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:52 smithi130 bash[27579]: cluster 2023-08-28T00:44:51.038934+0000 mgr.y (mgr.14901) 4988 : cluster [DBG] pgmap v4995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:52 smithi130 bash[27579]: audit 2023-08-28T00:44:51.391909+0000 mon.c (mon.2) 1934 : audit [DBG] from='client.? 172.21.15.6:0/3852252661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:44:52.398Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:44:52.400Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:52 smithi130 bash[51302]: debug 2023-08-28T00:44:52.826+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:52.829115+0000) 2023-08-28T00:44:54.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:53 smithi130 bash[51302]: debug 2023-08-28T00:44:53.826+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:53.829390+0000) 2023-08-28T00:44:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:44:54.279Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:44:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:44:54.281Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:44:54.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:54 smithi006 bash[27292]: cluster 2023-08-28T00:44:53.039617+0000 mgr.y (mgr.14901) 4989 : cluster [DBG] pgmap v4996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:54.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:54 smithi006 bash[27292]: audit 2023-08-28T00:44:53.801860+0000 mon.c (mon.2) 1935 : audit [DBG] from='client.? 172.21.15.6:0/54119116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:54.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:54 smithi006 bash[33918]: cluster 2023-08-28T00:44:53.039617+0000 mgr.y (mgr.14901) 4989 : cluster [DBG] pgmap v4996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:54.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:54 smithi006 bash[33918]: audit 2023-08-28T00:44:53.801860+0000 mon.c (mon.2) 1935 : audit [DBG] from='client.? 172.21.15.6:0/54119116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:54 smithi130 bash[27579]: cluster 2023-08-28T00:44:53.039617+0000 mgr.y (mgr.14901) 4989 : cluster [DBG] pgmap v4996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:54 smithi130 bash[27579]: audit 2023-08-28T00:44:53.801860+0000 mon.c (mon.2) 1935 : audit [DBG] from='client.? 172.21.15.6:0/54119116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:54 smithi130 bash[51302]: debug 2023-08-28T00:44:54.826+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:54.829674+0000) 2023-08-28T00:44:56.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:55 smithi130 bash[51302]: debug 2023-08-28T00:44:55.826+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:55.829950+0000) 2023-08-28T00:44:56.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:56 smithi006 bash[33918]: cluster 2023-08-28T00:44:55.040743+0000 mgr.y (mgr.14901) 4990 : cluster [DBG] pgmap v4997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:56.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:56 smithi006 bash[27292]: cluster 2023-08-28T00:44:55.040743+0000 mgr.y (mgr.14901) 4990 : cluster [DBG] pgmap v4997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:56 smithi130 bash[27579]: cluster 2023-08-28T00:44:55.040743+0000 mgr.y (mgr.14901) 4990 : cluster [DBG] pgmap v4997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:57.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:56 smithi130 bash[51302]: debug 2023-08-28T00:44:56.826+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:56.830266+0000) 2023-08-28T00:44:57.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:57 smithi006 bash[33918]: audit 2023-08-28T00:44:56.216727+0000 mon.c (mon.2) 1936 : audit [DBG] from='client.? 172.21.15.6:0/2938736563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:57.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:57 smithi006 bash[27292]: audit 2023-08-28T00:44:56.216727+0000 mon.c (mon.2) 1936 : audit [DBG] from='client.? 172.21.15.6:0/2938736563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:57.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:57 smithi130 bash[27579]: audit 2023-08-28T00:44:56.216727+0000 mon.c (mon.2) 1936 : audit [DBG] from='client.? 172.21.15.6:0/2938736563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:58.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:57 smithi130 bash[51302]: debug 2023-08-28T00:44:57.826+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:57.830562+0000) 2023-08-28T00:44:58.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:58 smithi006 bash[27292]: cluster 2023-08-28T00:44:57.041375+0000 mgr.y (mgr.14901) 4991 : cluster [DBG] pgmap v4998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:58.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:58 smithi006 bash[33918]: cluster 2023-08-28T00:44:57.041375+0000 mgr.y (mgr.14901) 4991 : cluster [DBG] pgmap v4998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:58 smithi130 bash[27579]: cluster 2023-08-28T00:44:57.041375+0000 mgr.y (mgr.14901) 4991 : cluster [DBG] pgmap v4998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:44:59.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:58 smithi130 bash[51302]: debug 2023-08-28T00:44:58.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:58.830876+0000) 2023-08-28T00:44:59.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:44:59 smithi006 bash[33918]: audit 2023-08-28T00:44:58.632547+0000 mon.c (mon.2) 1937 : audit [DBG] from='client.? 172.21.15.6:0/1708128175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:59.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:44:59 smithi006 bash[27292]: audit 2023-08-28T00:44:58.632547+0000 mon.c (mon.2) 1937 : audit [DBG] from='client.? 172.21.15.6:0/1708128175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:44:59 smithi130 bash[27579]: audit 2023-08-28T00:44:58.632547+0000 mon.c (mon.2) 1937 : audit [DBG] from='client.? 172.21.15.6:0/1708128175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:00.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:44:59 smithi130 bash[51302]: debug 2023-08-28T00:44:59.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:59.831159+0000) 2023-08-28T00:45:00.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:00 smithi006 bash[33918]: cluster 2023-08-28T00:44:59.042604+0000 mgr.y (mgr.14901) 4992 : cluster [DBG] pgmap v4999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:00.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:00 smithi006 bash[27292]: cluster 2023-08-28T00:44:59.042604+0000 mgr.y (mgr.14901) 4992 : cluster [DBG] pgmap v4999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:00 smithi130 bash[27579]: cluster 2023-08-28T00:44:59.042604+0000 mgr.y (mgr.14901) 4992 : cluster [DBG] pgmap v4999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:01.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:00 smithi130 bash[51302]: debug 2023-08-28T00:45:00.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:00.831446+0000) 2023-08-28T00:45:01.164 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:45:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:45:00] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:45:01.164 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:45:00.820Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:01.164 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:45:00.822Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:01.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:01 smithi006 bash[27292]: audit 2023-08-28T00:45:01.049883+0000 mon.c (mon.2) 1938 : audit [DBG] from='client.? 172.21.15.6:0/2207868669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:01.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:01 smithi006 bash[33918]: audit 2023-08-28T00:45:01.049883+0000 mon.c (mon.2) 1938 : audit [DBG] from='client.? 172.21.15.6:0/2207868669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:01.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:01 smithi130 bash[27579]: audit 2023-08-28T00:45:01.049883+0000 mon.c (mon.2) 1938 : audit [DBG] from='client.? 172.21.15.6:0/2207868669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:02.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:01 smithi130 bash[51302]: debug 2023-08-28T00:45:01.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:01.831724+0000) 2023-08-28T00:45:02.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:02 smithi006 bash[33918]: cluster 2023-08-28T00:45:01.043262+0000 mgr.y (mgr.14901) 4993 : cluster [DBG] pgmap v5000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:02.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:02 smithi006 bash[27292]: cluster 2023-08-28T00:45:01.043262+0000 mgr.y (mgr.14901) 4993 : cluster [DBG] pgmap v5000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:45:02.398Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:45:02.400Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:02 smithi130 bash[27579]: cluster 2023-08-28T00:45:01.043262+0000 mgr.y (mgr.14901) 4993 : cluster [DBG] pgmap v5000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:02 smithi130 bash[51302]: debug 2023-08-28T00:45:02.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:02.831950+0000) 2023-08-28T00:45:04.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:03 smithi130 bash[51302]: debug 2023-08-28T00:45:03.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:03.832272+0000) 2023-08-28T00:45:04.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:04 smithi006 bash[27292]: cluster 2023-08-28T00:45:03.043860+0000 mgr.y (mgr.14901) 4994 : cluster [DBG] pgmap v5001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:04.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:04 smithi006 bash[27292]: audit 2023-08-28T00:45:03.460905+0000 mon.a (mon.0) 3523 : audit [DBG] from='client.? 172.21.15.6:0/3704012093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:45:04.280Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:45:04.282Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:04.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:04 smithi006 bash[33918]: cluster 2023-08-28T00:45:03.043860+0000 mgr.y (mgr.14901) 4994 : cluster [DBG] pgmap v5001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:04.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:04 smithi006 bash[33918]: audit 2023-08-28T00:45:03.460905+0000 mon.a (mon.0) 3523 : audit [DBG] from='client.? 172.21.15.6:0/3704012093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:04 smithi130 bash[27579]: cluster 2023-08-28T00:45:03.043860+0000 mgr.y (mgr.14901) 4994 : cluster [DBG] pgmap v5001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:04 smithi130 bash[27579]: audit 2023-08-28T00:45:03.460905+0000 mon.a (mon.0) 3523 : audit [DBG] from='client.? 172.21.15.6:0/3704012093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:04 smithi130 bash[51302]: debug 2023-08-28T00:45:04.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:04.832558+0000) 2023-08-28T00:45:06.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:05 smithi130 bash[51302]: debug 2023-08-28T00:45:05.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:05.832829+0000) 2023-08-28T00:45:06.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:06 smithi006 bash[27292]: cluster 2023-08-28T00:45:05.045017+0000 mgr.y (mgr.14901) 4995 : cluster [DBG] pgmap v5002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:06.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:06 smithi006 bash[27292]: audit 2023-08-28T00:45:05.879508+0000 mon.c (mon.2) 1939 : audit [DBG] from='client.? 172.21.15.6:0/2148015673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:06 smithi006 bash[33918]: cluster 2023-08-28T00:45:05.045017+0000 mgr.y (mgr.14901) 4995 : cluster [DBG] pgmap v5002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:06.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:06 smithi006 bash[33918]: audit 2023-08-28T00:45:05.879508+0000 mon.c (mon.2) 1939 : audit [DBG] from='client.? 172.21.15.6:0/2148015673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:06 smithi130 bash[27579]: cluster 2023-08-28T00:45:05.045017+0000 mgr.y (mgr.14901) 4995 : cluster [DBG] pgmap v5002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:06 smithi130 bash[27579]: audit 2023-08-28T00:45:05.879508+0000 mon.c (mon.2) 1939 : audit [DBG] from='client.? 172.21.15.6:0/2148015673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:07.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:06 smithi130 bash[51302]: debug 2023-08-28T00:45:06.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:06.833145+0000) 2023-08-28T00:45:08.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:07 smithi130 bash[51302]: debug 2023-08-28T00:45:07.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:07.833428+0000) 2023-08-28T00:45:08.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:08 smithi006 bash[33918]: cluster 2023-08-28T00:45:07.045647+0000 mgr.y (mgr.14901) 4996 : cluster [DBG] pgmap v5003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:08.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:08 smithi006 bash[27292]: cluster 2023-08-28T00:45:07.045647+0000 mgr.y (mgr.14901) 4996 : cluster [DBG] pgmap v5003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:08 smithi130 bash[27579]: cluster 2023-08-28T00:45:07.045647+0000 mgr.y (mgr.14901) 4996 : cluster [DBG] pgmap v5003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:09.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:08 smithi130 bash[51302]: debug 2023-08-28T00:45:08.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:08.833703+0000) 2023-08-28T00:45:09.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:09 smithi006 bash[27292]: audit 2023-08-28T00:45:08.300825+0000 mon.a (mon.0) 3524 : audit [DBG] from='client.? 172.21.15.6:0/1816126336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:09.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:09 smithi006 bash[33918]: audit 2023-08-28T00:45:08.300825+0000 mon.a (mon.0) 3524 : audit [DBG] from='client.? 172.21.15.6:0/1816126336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:09 smithi130 bash[27579]: audit 2023-08-28T00:45:08.300825+0000 mon.a (mon.0) 3524 : audit [DBG] from='client.? 172.21.15.6:0/1816126336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:10.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:09 smithi130 bash[51302]: debug 2023-08-28T00:45:09.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:09.834010+0000) 2023-08-28T00:45:10.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:10 smithi006 bash[27292]: cluster 2023-08-28T00:45:09.046728+0000 mgr.y (mgr.14901) 4997 : cluster [DBG] pgmap v5004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:10.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:10 smithi006 bash[33918]: cluster 2023-08-28T00:45:09.046728+0000 mgr.y (mgr.14901) 4997 : cluster [DBG] pgmap v5004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:10 smithi130 bash[27579]: cluster 2023-08-28T00:45:09.046728+0000 mgr.y (mgr.14901) 4997 : cluster [DBG] pgmap v5004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:11.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:10 smithi130 bash[51302]: debug 2023-08-28T00:45:10.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:10.834311+0000) 2023-08-28T00:45:11.209 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:45:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:45:10] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:45:11.210 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:45:10.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:11.210 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:45:10.823Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:11.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:11 smithi006 bash[27292]: audit 2023-08-28T00:45:10.719568+0000 mon.a (mon.0) 3525 : audit [DBG] from='client.? 172.21.15.6:0/587392309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:11.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:11 smithi006 bash[33918]: audit 2023-08-28T00:45:10.719568+0000 mon.a (mon.0) 3525 : audit [DBG] from='client.? 172.21.15.6:0/587392309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:11.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:11 smithi130 bash[27579]: audit 2023-08-28T00:45:10.719568+0000 mon.a (mon.0) 3525 : audit [DBG] from='client.? 172.21.15.6:0/587392309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:12.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:11 smithi130 bash[51302]: debug 2023-08-28T00:45:11.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:11.834610+0000) 2023-08-28T00:45:12.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:12 smithi006 bash[27292]: cluster 2023-08-28T00:45:11.047150+0000 mgr.y (mgr.14901) 4998 : cluster [DBG] pgmap v5005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:12.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:12 smithi006 bash[33918]: cluster 2023-08-28T00:45:11.047150+0000 mgr.y (mgr.14901) 4998 : cluster [DBG] pgmap v5005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:12.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:45:12.399Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:12.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:45:12.401Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:12 smithi130 bash[27579]: cluster 2023-08-28T00:45:11.047150+0000 mgr.y (mgr.14901) 4998 : cluster [DBG] pgmap v5005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:13.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:12 smithi130 bash[51302]: debug 2023-08-28T00:45:12.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:12.834910+0000) 2023-08-28T00:45:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:13 smithi130 bash[27579]: audit 2023-08-28T00:45:13.129454+0000 mon.c (mon.2) 1940 : audit [DBG] from='client.? 172.21.15.6:0/279982799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:13.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:13 smithi006 bash[27292]: audit 2023-08-28T00:45:13.129454+0000 mon.c (mon.2) 1940 : audit [DBG] from='client.? 172.21.15.6:0/279982799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:13.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:13 smithi006 bash[33918]: audit 2023-08-28T00:45:13.129454+0000 mon.c (mon.2) 1940 : audit [DBG] from='client.? 172.21.15.6:0/279982799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:14.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:13 smithi130 bash[51302]: debug 2023-08-28T00:45:13.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:13.835180+0000) 2023-08-28T00:45:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:14 smithi130 bash[27579]: cluster 2023-08-28T00:45:13.047731+0000 mgr.y (mgr.14901) 4999 : cluster [DBG] pgmap v5006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:14.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:14 smithi006 bash[27292]: cluster 2023-08-28T00:45:13.047731+0000 mgr.y (mgr.14901) 4999 : cluster [DBG] pgmap v5006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:14.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:14 smithi006 bash[33918]: cluster 2023-08-28T00:45:13.047731+0000 mgr.y (mgr.14901) 4999 : cluster [DBG] pgmap v5006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:45:14.280Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:45:14.282Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:15.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:14 smithi130 bash[51302]: debug 2023-08-28T00:45:14.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:14.835439+0000) 2023-08-28T00:45:16.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:15 smithi130 bash[51302]: debug 2023-08-28T00:45:15.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:15.835708+0000) 2023-08-28T00:45:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:16 smithi130 bash[27579]: cluster 2023-08-28T00:45:15.048787+0000 mgr.y (mgr.14901) 5000 : cluster [DBG] pgmap v5007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:16 smithi130 bash[27579]: audit 2023-08-28T00:45:15.548202+0000 mon.a (mon.0) 3526 : audit [DBG] from='client.? 172.21.15.6:0/398599886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:16 smithi006 bash[33918]: cluster 2023-08-28T00:45:15.048787+0000 mgr.y (mgr.14901) 5000 : cluster [DBG] pgmap v5007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:16 smithi006 bash[33918]: audit 2023-08-28T00:45:15.548202+0000 mon.a (mon.0) 3526 : audit [DBG] from='client.? 172.21.15.6:0/398599886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:16.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:16 smithi006 bash[27292]: cluster 2023-08-28T00:45:15.048787+0000 mgr.y (mgr.14901) 5000 : cluster [DBG] pgmap v5007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:16.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:16 smithi006 bash[27292]: audit 2023-08-28T00:45:15.548202+0000 mon.a (mon.0) 3526 : audit [DBG] from='client.? 172.21.15.6:0/398599886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:17.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:16 smithi130 bash[51302]: debug 2023-08-28T00:45:16.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:16.835999+0000) 2023-08-28T00:45:18.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:17 smithi130 bash[51302]: debug 2023-08-28T00:45:17.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:17.836313+0000) 2023-08-28T00:45:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:18 smithi130 bash[27579]: cluster 2023-08-28T00:45:17.049439+0000 mgr.y (mgr.14901) 5001 : cluster [DBG] pgmap v5008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:18 smithi130 bash[27579]: audit 2023-08-28T00:45:17.973169+0000 mon.c (mon.2) 1941 : audit [DBG] from='client.? 172.21.15.6:0/1442402952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:18 smithi006 bash[27292]: cluster 2023-08-28T00:45:17.049439+0000 mgr.y (mgr.14901) 5001 : cluster [DBG] pgmap v5008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:18 smithi006 bash[27292]: audit 2023-08-28T00:45:17.973169+0000 mon.c (mon.2) 1941 : audit [DBG] from='client.? 172.21.15.6:0/1442402952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:18.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:18 smithi006 bash[33918]: cluster 2023-08-28T00:45:17.049439+0000 mgr.y (mgr.14901) 5001 : cluster [DBG] pgmap v5008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:18.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:18 smithi006 bash[33918]: audit 2023-08-28T00:45:17.973169+0000 mon.c (mon.2) 1941 : audit [DBG] from='client.? 172.21.15.6:0/1442402952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:19.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:18 smithi130 bash[51302]: debug 2023-08-28T00:45:18.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:18.836583+0000) 2023-08-28T00:45:20.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:19 smithi130 bash[51302]: debug 2023-08-28T00:45:19.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:19.836850+0000) 2023-08-28T00:45:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:20 smithi130 bash[27579]: cluster 2023-08-28T00:45:19.050433+0000 mgr.y (mgr.14901) 5002 : cluster [DBG] pgmap v5009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:20 smithi006 bash[27292]: cluster 2023-08-28T00:45:19.050433+0000 mgr.y (mgr.14901) 5002 : cluster [DBG] pgmap v5009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:20.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:20 smithi006 bash[33918]: cluster 2023-08-28T00:45:19.050433+0000 mgr.y (mgr.14901) 5002 : cluster [DBG] pgmap v5009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:21.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:20 smithi130 bash[51302]: debug 2023-08-28T00:45:20.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:20.837105+0000) 2023-08-28T00:45:21.479 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:45:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:45:20] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:45:21.480 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:45:20.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:21.480 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:45:20.823Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:21 smithi130 bash[27579]: audit 2023-08-28T00:45:20.388288+0000 mon.a (mon.0) 3527 : audit [DBG] from='client.? 172.21.15.6:0/3511644761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:21.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:21 smithi006 bash[27292]: audit 2023-08-28T00:45:20.388288+0000 mon.a (mon.0) 3527 : audit [DBG] from='client.? 172.21.15.6:0/3511644761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:21.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:21 smithi006 bash[33918]: audit 2023-08-28T00:45:20.388288+0000 mon.a (mon.0) 3527 : audit [DBG] from='client.? 172.21.15.6:0/3511644761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:22.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:21 smithi130 bash[51302]: debug 2023-08-28T00:45:21.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:21.837363+0000) 2023-08-28T00:45:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:22 smithi130 bash[27579]: cluster 2023-08-28T00:45:21.051063+0000 mgr.y (mgr.14901) 5003 : cluster [DBG] pgmap v5010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:22 smithi006 bash[33918]: cluster 2023-08-28T00:45:21.051063+0000 mgr.y (mgr.14901) 5003 : cluster [DBG] pgmap v5010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:22 smithi006 bash[27292]: cluster 2023-08-28T00:45:21.051063+0000 mgr.y (mgr.14901) 5003 : cluster [DBG] pgmap v5010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:45:22.400Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:22.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:45:22.402Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:23.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:22 smithi130 bash[51302]: debug 2023-08-28T00:45:22.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:22.837646+0000) 2023-08-28T00:45:23.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:23 smithi130 bash[27579]: audit 2023-08-28T00:45:22.800115+0000 mon.c (mon.2) 1942 : audit [DBG] from='client.? 172.21.15.6:0/1075911212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:23.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:23 smithi006 bash[27292]: audit 2023-08-28T00:45:22.800115+0000 mon.c (mon.2) 1942 : audit [DBG] from='client.? 172.21.15.6:0/1075911212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:23.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:23 smithi006 bash[33918]: audit 2023-08-28T00:45:22.800115+0000 mon.c (mon.2) 1942 : audit [DBG] from='client.? 172.21.15.6:0/1075911212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:24.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:23 smithi130 bash[51302]: debug 2023-08-28T00:45:23.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:23.837965+0000) 2023-08-28T00:45:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:24 smithi130 bash[27579]: cluster 2023-08-28T00:45:23.051669+0000 mgr.y (mgr.14901) 5004 : cluster [DBG] pgmap v5011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:24.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:24 smithi006 bash[27292]: cluster 2023-08-28T00:45:23.051669+0000 mgr.y (mgr.14901) 5004 : cluster [DBG] pgmap v5011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:24.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:24 smithi006 bash[33918]: cluster 2023-08-28T00:45:23.051669+0000 mgr.y (mgr.14901) 5004 : cluster [DBG] pgmap v5011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:45:24.281Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:45:24.283Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:24 smithi130 bash[51302]: debug 2023-08-28T00:45:24.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:24.838260+0000) 2023-08-28T00:45:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:26 smithi130 bash[27579]: cluster 2023-08-28T00:45:25.052415+0000 mgr.y (mgr.14901) 5005 : cluster [DBG] pgmap v5012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:26.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:26 smithi130 bash[27579]: audit 2023-08-28T00:45:25.215039+0000 mon.a (mon.0) 3528 : audit [DBG] from='client.? 172.21.15.6:0/2591680983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:26.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:25 smithi130 bash[51302]: debug 2023-08-28T00:45:25.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:25.838565+0000) 2023-08-28T00:45:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:26 smithi006 bash[33918]: cluster 2023-08-28T00:45:25.052415+0000 mgr.y (mgr.14901) 5005 : cluster [DBG] pgmap v5012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:26 smithi006 bash[33918]: audit 2023-08-28T00:45:25.215039+0000 mon.a (mon.0) 3528 : audit [DBG] from='client.? 172.21.15.6:0/2591680983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:26.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:26 smithi006 bash[27292]: cluster 2023-08-28T00:45:25.052415+0000 mgr.y (mgr.14901) 5005 : cluster [DBG] pgmap v5012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:26.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:26 smithi006 bash[27292]: audit 2023-08-28T00:45:25.215039+0000 mon.a (mon.0) 3528 : audit [DBG] from='client.? 172.21.15.6:0/2591680983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:27.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:26 smithi130 bash[51302]: debug 2023-08-28T00:45:26.838+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:26.838846+0000) 2023-08-28T00:45:28.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:27 smithi130 bash[51302]: debug 2023-08-28T00:45:27.838+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:27.839148+0000) 2023-08-28T00:45:28.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:28 smithi006 bash[27292]: cluster 2023-08-28T00:45:27.053100+0000 mgr.y (mgr.14901) 5006 : cluster [DBG] pgmap v5013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:28.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:28 smithi006 bash[27292]: audit 2023-08-28T00:45:27.632792+0000 mon.c (mon.2) 1943 : audit [DBG] from='client.? 172.21.15.6:0/3218770394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:28.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:28 smithi006 bash[33918]: cluster 2023-08-28T00:45:27.053100+0000 mgr.y (mgr.14901) 5006 : cluster [DBG] pgmap v5013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:28.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:28 smithi006 bash[33918]: audit 2023-08-28T00:45:27.632792+0000 mon.c (mon.2) 1943 : audit [DBG] from='client.? 172.21.15.6:0/3218770394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:28 smithi130 bash[27579]: cluster 2023-08-28T00:45:27.053100+0000 mgr.y (mgr.14901) 5006 : cluster [DBG] pgmap v5013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:28 smithi130 bash[27579]: audit 2023-08-28T00:45:27.632792+0000 mon.c (mon.2) 1943 : audit [DBG] from='client.? 172.21.15.6:0/3218770394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:29.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:28 smithi130 bash[51302]: debug 2023-08-28T00:45:28.838+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:28.839381+0000) 2023-08-28T00:45:30.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:29 smithi130 bash[51302]: debug 2023-08-28T00:45:29.838+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:29.839677+0000) 2023-08-28T00:45:30.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:30 smithi006 bash[27292]: cluster 2023-08-28T00:45:29.054062+0000 mgr.y (mgr.14901) 5007 : cluster [DBG] pgmap v5014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:30.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:30 smithi006 bash[27292]: audit 2023-08-28T00:45:30.042232+0000 mon.c (mon.2) 1944 : audit [DBG] from='client.? 172.21.15.6:0/3031865149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:30.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:30 smithi006 bash[33918]: cluster 2023-08-28T00:45:29.054062+0000 mgr.y (mgr.14901) 5007 : cluster [DBG] pgmap v5014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:30.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:30 smithi006 bash[33918]: audit 2023-08-28T00:45:30.042232+0000 mon.c (mon.2) 1944 : audit [DBG] from='client.? 172.21.15.6:0/3031865149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:30 smithi130 bash[27579]: cluster 2023-08-28T00:45:29.054062+0000 mgr.y (mgr.14901) 5007 : cluster [DBG] pgmap v5014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:30 smithi130 bash[27579]: audit 2023-08-28T00:45:30.042232+0000 mon.c (mon.2) 1944 : audit [DBG] from='client.? 172.21.15.6:0/3031865149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:31.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:30 smithi130 bash[51302]: debug 2023-08-28T00:45:30.838+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:30.839953+0000) 2023-08-28T00:45:31.164 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:45:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:45:30] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:45:31.164 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:45:30.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:31.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:31 smithi006 bash[56298]: level=warn ts=2023-08-28T00:45:30.825Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:32.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:31 smithi130 bash[51302]: debug 2023-08-28T00:45:31.838+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:31.840313+0000) 2023-08-28T00:45:32.405 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:32 smithi006 bash[33918]: cluster 2023-08-28T00:45:31.054714+0000 mgr.y (mgr.14901) 5008 : cluster [DBG] pgmap v5015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:32.405 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:32 smithi006 bash[27292]: cluster 2023-08-28T00:45:31.054714+0000 mgr.y (mgr.14901) 5008 : cluster [DBG] pgmap v5015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:32 smithi130 bash[27579]: cluster 2023-08-28T00:45:31.054714+0000 mgr.y (mgr.14901) 5008 : cluster [DBG] pgmap v5015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:45:32.401Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:45:32.402Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:33.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:32 smithi130 bash[51302]: debug 2023-08-28T00:45:32.838+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:32.840627+0000) 2023-08-28T00:45:33.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:33 smithi006 bash[33918]: audit 2023-08-28T00:45:32.459713+0000 mon.c (mon.2) 1945 : audit [DBG] from='client.? 172.21.15.6:0/3975400706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:33.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:33 smithi006 bash[33918]: audit 2023-08-28T00:45:32.864091+0000 mon.b (mon.1) 601 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:45:33.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:33 smithi006 bash[33918]: audit 2023-08-28T00:45:33.138418+0000 mon.a (mon.0) 3529 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:45:33.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:33 smithi006 bash[27292]: audit 2023-08-28T00:45:32.459713+0000 mon.c (mon.2) 1945 : audit [DBG] from='client.? 172.21.15.6:0/3975400706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:33.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:33 smithi006 bash[27292]: audit 2023-08-28T00:45:32.864091+0000 mon.b (mon.1) 601 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:45:33.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:33 smithi006 bash[27292]: audit 2023-08-28T00:45:33.138418+0000 mon.a (mon.0) 3529 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:45:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:33 smithi130 bash[27579]: audit 2023-08-28T00:45:32.459713+0000 mon.c (mon.2) 1945 : audit [DBG] from='client.? 172.21.15.6:0/3975400706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:33 smithi130 bash[27579]: audit 2023-08-28T00:45:32.864091+0000 mon.b (mon.1) 601 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:45:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:33 smithi130 bash[27579]: audit 2023-08-28T00:45:33.138418+0000 mon.a (mon.0) 3529 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:45:34.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:33 smithi130 bash[51302]: debug 2023-08-28T00:45:33.837+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:33.840930+0000) 2023-08-28T00:45:34.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:34 smithi006 bash[27292]: cluster 2023-08-28T00:45:33.055343+0000 mgr.y (mgr.14901) 5009 : cluster [DBG] pgmap v5016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:34.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:34 smithi006 bash[27292]: audit 2023-08-28T00:45:33.143546+0000 mon.a (mon.0) 3530 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:45:34.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:34 smithi006 bash[33918]: cluster 2023-08-28T00:45:33.055343+0000 mgr.y (mgr.14901) 5009 : cluster [DBG] pgmap v5016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:34.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:34 smithi006 bash[33918]: audit 2023-08-28T00:45:33.143546+0000 mon.a (mon.0) 3530 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:45:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:45:34.282Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:34.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:45:34.284Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:34 smithi130 bash[27579]: cluster 2023-08-28T00:45:33.055343+0000 mgr.y (mgr.14901) 5009 : cluster [DBG] pgmap v5016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:34 smithi130 bash[27579]: audit 2023-08-28T00:45:33.143546+0000 mon.a (mon.0) 3530 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:45:35.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:34 smithi130 bash[51302]: debug 2023-08-28T00:45:34.837+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:34.841193+0000) 2023-08-28T00:45:35.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:35 smithi006 bash[27292]: audit 2023-08-28T00:45:34.886170+0000 mon.a (mon.0) 3531 : audit [DBG] from='client.? 172.21.15.6:0/2670147348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:35.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:35 smithi006 bash[33918]: audit 2023-08-28T00:45:34.886170+0000 mon.a (mon.0) 3531 : audit [DBG] from='client.? 172.21.15.6:0/2670147348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:35.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:35 smithi130 bash[27579]: audit 2023-08-28T00:45:34.886170+0000 mon.a (mon.0) 3531 : audit [DBG] from='client.? 172.21.15.6:0/2670147348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:36.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:35 smithi130 bash[51302]: debug 2023-08-28T00:45:35.837+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:35.841478+0000) 2023-08-28T00:45:36.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:36 smithi006 bash[33918]: cluster 2023-08-28T00:45:35.056412+0000 mgr.y (mgr.14901) 5010 : cluster [DBG] pgmap v5017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:36.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:36 smithi006 bash[27292]: cluster 2023-08-28T00:45:35.056412+0000 mgr.y (mgr.14901) 5010 : cluster [DBG] pgmap v5017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:36 smithi130 bash[27579]: cluster 2023-08-28T00:45:35.056412+0000 mgr.y (mgr.14901) 5010 : cluster [DBG] pgmap v5017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:37.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:36 smithi130 bash[51302]: debug 2023-08-28T00:45:36.837+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:36.841759+0000) 2023-08-28T00:45:38.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:37 smithi130 bash[51302]: debug 2023-08-28T00:45:37.841+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:37.842073+0000) 2023-08-28T00:45:38.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:38 smithi006 bash[27292]: cluster 2023-08-28T00:45:37.056912+0000 mgr.y (mgr.14901) 5011 : cluster [DBG] pgmap v5018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:38.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:38 smithi006 bash[27292]: audit 2023-08-28T00:45:37.311600+0000 mon.a (mon.0) 3532 : audit [DBG] from='client.? 172.21.15.6:0/3721330641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:38.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:38 smithi006 bash[33918]: cluster 2023-08-28T00:45:37.056912+0000 mgr.y (mgr.14901) 5011 : cluster [DBG] pgmap v5018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:38.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:38 smithi006 bash[33918]: audit 2023-08-28T00:45:37.311600+0000 mon.a (mon.0) 3532 : audit [DBG] from='client.? 172.21.15.6:0/3721330641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:38 smithi130 bash[27579]: cluster 2023-08-28T00:45:37.056912+0000 mgr.y (mgr.14901) 5011 : cluster [DBG] pgmap v5018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:38 smithi130 bash[27579]: audit 2023-08-28T00:45:37.311600+0000 mon.a (mon.0) 3532 : audit [DBG] from='client.? 172.21.15.6:0/3721330641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:39.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:38 smithi130 bash[51302]: debug 2023-08-28T00:45:38.837+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:38.842335+0000) 2023-08-28T00:45:40.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:39 smithi130 bash[51302]: debug 2023-08-28T00:45:39.837+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:39.842630+0000) 2023-08-28T00:45:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:39 smithi130 bash[27579]: audit 2023-08-28T00:45:38.984919+0000 mon.a (mon.0) 3533 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:45:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:39 smithi130 bash[27579]: cluster 2023-08-28T00:45:39.057926+0000 mgr.y (mgr.14901) 5012 : cluster [DBG] pgmap v5019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:40.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:39 smithi130 bash[27579]: audit 2023-08-28T00:45:39.451503+0000 mon.a (mon.0) 3534 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:45:40.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:39 smithi130 bash[27579]: audit 2023-08-28T00:45:39.713564+0000 mon.b (mon.1) 602 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:45:40.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:39 smithi130 bash[27579]: audit 2023-08-28T00:45:39.715582+0000 mon.b (mon.1) 603 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:45:40.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:39 smithi130 bash[27579]: audit 2023-08-28T00:45:39.722553+0000 mon.a (mon.0) 3535 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:45:40.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:39 smithi130 bash[27579]: audit 2023-08-28T00:45:39.731250+0000 mon.a (mon.0) 3536 : audit [DBG] from='client.? 172.21.15.6:0/166402807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:39 smithi006 bash[33918]: audit 2023-08-28T00:45:38.984919+0000 mon.a (mon.0) 3533 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:45:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:39 smithi006 bash[33918]: cluster 2023-08-28T00:45:39.057926+0000 mgr.y (mgr.14901) 5012 : cluster [DBG] pgmap v5019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:39 smithi006 bash[33918]: audit 2023-08-28T00:45:39.451503+0000 mon.a (mon.0) 3534 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:45:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:39 smithi006 bash[33918]: audit 2023-08-28T00:45:39.713564+0000 mon.b (mon.1) 602 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:45:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:39 smithi006 bash[33918]: audit 2023-08-28T00:45:39.715582+0000 mon.b (mon.1) 603 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:45:40.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:39 smithi006 bash[33918]: audit 2023-08-28T00:45:39.722553+0000 mon.a (mon.0) 3535 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:45:40.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:39 smithi006 bash[33918]: audit 2023-08-28T00:45:39.731250+0000 mon.a (mon.0) 3536 : audit [DBG] from='client.? 172.21.15.6:0/166402807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:40.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:39 smithi006 bash[27292]: audit 2023-08-28T00:45:38.984919+0000 mon.a (mon.0) 3533 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:45:40.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:39 smithi006 bash[27292]: cluster 2023-08-28T00:45:39.057926+0000 mgr.y (mgr.14901) 5012 : cluster [DBG] pgmap v5019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:40.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:39 smithi006 bash[27292]: audit 2023-08-28T00:45:39.451503+0000 mon.a (mon.0) 3534 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:45:40.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:39 smithi006 bash[27292]: audit 2023-08-28T00:45:39.713564+0000 mon.b (mon.1) 602 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:45:40.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:39 smithi006 bash[27292]: audit 2023-08-28T00:45:39.715582+0000 mon.b (mon.1) 603 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:45:40.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:39 smithi006 bash[27292]: audit 2023-08-28T00:45:39.722553+0000 mon.a (mon.0) 3535 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:45:40.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:39 smithi006 bash[27292]: audit 2023-08-28T00:45:39.731250+0000 mon.a (mon.0) 3536 : audit [DBG] from='client.? 172.21.15.6:0/166402807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:41.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:40 smithi130 bash[51302]: debug 2023-08-28T00:45:40.841+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:40.842916+0000) 2023-08-28T00:45:41.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:45:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:45:40] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:45:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:45:40.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:45:40.826Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:42.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:41 smithi130 bash[51302]: debug 2023-08-28T00:45:41.841+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:41.843136+0000) 2023-08-28T00:45:42.405 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:42 smithi006 bash[33918]: cluster 2023-08-28T00:45:41.058588+0000 mgr.y (mgr.14901) 5013 : cluster [DBG] pgmap v5020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:42.405 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:42 smithi006 bash[27292]: cluster 2023-08-28T00:45:41.058588+0000 mgr.y (mgr.14901) 5013 : cluster [DBG] pgmap v5020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:42 smithi130 bash[27579]: cluster 2023-08-28T00:45:41.058588+0000 mgr.y (mgr.14901) 5013 : cluster [DBG] pgmap v5020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:45:42.401Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:45:42.403Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:43.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:42 smithi130 bash[51302]: debug 2023-08-28T00:45:42.841+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:42.843379+0000) 2023-08-28T00:45:43.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:43 smithi006 bash[27292]: audit 2023-08-28T00:45:42.151080+0000 mon.a (mon.0) 3537 : audit [DBG] from='client.? 172.21.15.6:0/51455940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:43.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:43 smithi006 bash[33918]: audit 2023-08-28T00:45:42.151080+0000 mon.a (mon.0) 3537 : audit [DBG] from='client.? 172.21.15.6:0/51455940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:43 smithi130 bash[27579]: audit 2023-08-28T00:45:42.151080+0000 mon.a (mon.0) 3537 : audit [DBG] from='client.? 172.21.15.6:0/51455940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:44.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:43 smithi130 bash[51302]: debug 2023-08-28T00:45:43.841+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:43.843696+0000) 2023-08-28T00:45:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:45:44.283Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:44.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:45:44.284Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:44.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:44 smithi006 bash[33918]: cluster 2023-08-28T00:45:43.059260+0000 mgr.y (mgr.14901) 5014 : cluster [DBG] pgmap v5021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:44.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:44 smithi006 bash[27292]: cluster 2023-08-28T00:45:43.059260+0000 mgr.y (mgr.14901) 5014 : cluster [DBG] pgmap v5021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:44 smithi130 bash[27579]: cluster 2023-08-28T00:45:43.059260+0000 mgr.y (mgr.14901) 5014 : cluster [DBG] pgmap v5021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:44 smithi130 bash[51302]: debug 2023-08-28T00:45:44.841+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:44.844027+0000) 2023-08-28T00:45:45.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:45 smithi006 bash[33918]: audit 2023-08-28T00:45:44.571552+0000 mon.c (mon.2) 1946 : audit [DBG] from='client.? 172.21.15.6:0/1296102202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:45.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:45 smithi006 bash[27292]: audit 2023-08-28T00:45:44.571552+0000 mon.c (mon.2) 1946 : audit [DBG] from='client.? 172.21.15.6:0/1296102202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:45.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:45 smithi130 bash[27579]: audit 2023-08-28T00:45:44.571552+0000 mon.c (mon.2) 1946 : audit [DBG] from='client.? 172.21.15.6:0/1296102202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:46.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:45 smithi130 bash[51302]: debug 2023-08-28T00:45:45.841+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:45.844322+0000) 2023-08-28T00:45:46.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:46 smithi006 bash[33918]: cluster 2023-08-28T00:45:45.060484+0000 mgr.y (mgr.14901) 5015 : cluster [DBG] pgmap v5022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:46.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:46 smithi006 bash[27292]: cluster 2023-08-28T00:45:45.060484+0000 mgr.y (mgr.14901) 5015 : cluster [DBG] pgmap v5022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:46 smithi130 bash[27579]: cluster 2023-08-28T00:45:45.060484+0000 mgr.y (mgr.14901) 5015 : cluster [DBG] pgmap v5022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:47.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:46 smithi130 bash[51302]: debug 2023-08-28T00:45:46.841+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:46.844620+0000) 2023-08-28T00:45:47.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:47 smithi006 bash[27292]: audit 2023-08-28T00:45:46.996013+0000 mon.a (mon.0) 3538 : audit [DBG] from='client.? 172.21.15.6:0/3869767578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:47.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:47 smithi006 bash[33918]: audit 2023-08-28T00:45:46.996013+0000 mon.a (mon.0) 3538 : audit [DBG] from='client.? 172.21.15.6:0/3869767578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:47.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:47 smithi130 bash[27579]: audit 2023-08-28T00:45:46.996013+0000 mon.a (mon.0) 3538 : audit [DBG] from='client.? 172.21.15.6:0/3869767578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:48.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:47 smithi130 bash[51302]: debug 2023-08-28T00:45:47.841+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:47.844904+0000) 2023-08-28T00:45:48.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:48 smithi006 bash[33918]: cluster 2023-08-28T00:45:47.061188+0000 mgr.y (mgr.14901) 5016 : cluster [DBG] pgmap v5023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:48.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:48 smithi006 bash[27292]: cluster 2023-08-28T00:45:47.061188+0000 mgr.y (mgr.14901) 5016 : cluster [DBG] pgmap v5023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:48 smithi130 bash[27579]: cluster 2023-08-28T00:45:47.061188+0000 mgr.y (mgr.14901) 5016 : cluster [DBG] pgmap v5023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:49.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:48 smithi130 bash[51302]: debug 2023-08-28T00:45:48.841+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:48.845185+0000) 2023-08-28T00:45:50.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:49 smithi130 bash[51302]: debug 2023-08-28T00:45:49.841+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:49.845463+0000) 2023-08-28T00:45:50.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:50 smithi006 bash[33918]: cluster 2023-08-28T00:45:49.062300+0000 mgr.y (mgr.14901) 5017 : cluster [DBG] pgmap v5024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:50.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:50 smithi006 bash[33918]: audit 2023-08-28T00:45:49.409108+0000 mon.a (mon.0) 3539 : audit [DBG] from='client.? 172.21.15.6:0/2960605873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:50.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:50 smithi006 bash[27292]: cluster 2023-08-28T00:45:49.062300+0000 mgr.y (mgr.14901) 5017 : cluster [DBG] pgmap v5024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:50.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:50 smithi006 bash[27292]: audit 2023-08-28T00:45:49.409108+0000 mon.a (mon.0) 3539 : audit [DBG] from='client.? 172.21.15.6:0/2960605873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:50 smithi130 bash[27579]: cluster 2023-08-28T00:45:49.062300+0000 mgr.y (mgr.14901) 5017 : cluster [DBG] pgmap v5024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:50 smithi130 bash[27579]: audit 2023-08-28T00:45:49.409108+0000 mon.a (mon.0) 3539 : audit [DBG] from='client.? 172.21.15.6:0/2960605873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:51.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:50 smithi130 bash[51302]: debug 2023-08-28T00:45:50.841+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:50.845749+0000) 2023-08-28T00:45:51.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:45:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:45:50] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T00:45:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:45:50.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:45:50.826Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:52.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:51 smithi130 bash[51302]: debug 2023-08-28T00:45:51.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:51.846045+0000) 2023-08-28T00:45:52.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:52 smithi006 bash[27292]: cluster 2023-08-28T00:45:51.062968+0000 mgr.y (mgr.14901) 5018 : cluster [DBG] pgmap v5025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:52.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:52 smithi006 bash[27292]: audit 2023-08-28T00:45:51.826408+0000 mon.a (mon.0) 3540 : audit [DBG] from='client.? 172.21.15.6:0/887656525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:52.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:52 smithi006 bash[33918]: cluster 2023-08-28T00:45:51.062968+0000 mgr.y (mgr.14901) 5018 : cluster [DBG] pgmap v5025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:52.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:52 smithi006 bash[33918]: audit 2023-08-28T00:45:51.826408+0000 mon.a (mon.0) 3540 : audit [DBG] from='client.? 172.21.15.6:0/887656525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:52.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:45:52.402Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:52.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:45:52.404Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:52 smithi130 bash[27579]: cluster 2023-08-28T00:45:51.062968+0000 mgr.y (mgr.14901) 5018 : cluster [DBG] pgmap v5025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:52 smithi130 bash[27579]: audit 2023-08-28T00:45:51.826408+0000 mon.a (mon.0) 3540 : audit [DBG] from='client.? 172.21.15.6:0/887656525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:52 smithi130 bash[51302]: debug 2023-08-28T00:45:52.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:52.846344+0000) 2023-08-28T00:45:54.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:53 smithi130 bash[51302]: debug 2023-08-28T00:45:53.841+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:53.846615+0000) 2023-08-28T00:45:54.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:54 smithi006 bash[33918]: cluster 2023-08-28T00:45:53.063581+0000 mgr.y (mgr.14901) 5019 : cluster [DBG] pgmap v5026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:54.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:54 smithi006 bash[27292]: cluster 2023-08-28T00:45:53.063581+0000 mgr.y (mgr.14901) 5019 : cluster [DBG] pgmap v5026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:45:54.284Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:54.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:45:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:45:54.286Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:45:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:54 smithi130 bash[27579]: cluster 2023-08-28T00:45:53.063581+0000 mgr.y (mgr.14901) 5019 : cluster [DBG] pgmap v5026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:54 smithi130 bash[51302]: debug 2023-08-28T00:45:54.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:54.846917+0000) 2023-08-28T00:45:55.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:55 smithi006 bash[27292]: audit 2023-08-28T00:45:54.234447+0000 mon.a (mon.0) 3541 : audit [DBG] from='client.? 172.21.15.6:0/3028062647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:55.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:55 smithi006 bash[33918]: audit 2023-08-28T00:45:54.234447+0000 mon.a (mon.0) 3541 : audit [DBG] from='client.? 172.21.15.6:0/3028062647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:55.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:55 smithi130 bash[27579]: audit 2023-08-28T00:45:54.234447+0000 mon.a (mon.0) 3541 : audit [DBG] from='client.? 172.21.15.6:0/3028062647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:56.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:55 smithi130 bash[51302]: debug 2023-08-28T00:45:55.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:55.847199+0000) 2023-08-28T00:45:56.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:56 smithi006 bash[33918]: cluster 2023-08-28T00:45:55.064710+0000 mgr.y (mgr.14901) 5020 : cluster [DBG] pgmap v5027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:56.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:56 smithi006 bash[27292]: cluster 2023-08-28T00:45:55.064710+0000 mgr.y (mgr.14901) 5020 : cluster [DBG] pgmap v5027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:56 smithi130 bash[27579]: cluster 2023-08-28T00:45:55.064710+0000 mgr.y (mgr.14901) 5020 : cluster [DBG] pgmap v5027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:57.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:56 smithi130 bash[51302]: debug 2023-08-28T00:45:56.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:56.847525+0000) 2023-08-28T00:45:57.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:57 smithi006 bash[27292]: audit 2023-08-28T00:45:56.646817+0000 mon.c (mon.2) 1947 : audit [DBG] from='client.? 172.21.15.6:0/2108750855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:57.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:57 smithi006 bash[33918]: audit 2023-08-28T00:45:56.646817+0000 mon.c (mon.2) 1947 : audit [DBG] from='client.? 172.21.15.6:0/2108750855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:57.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:57 smithi130 bash[27579]: audit 2023-08-28T00:45:56.646817+0000 mon.c (mon.2) 1947 : audit [DBG] from='client.? 172.21.15.6:0/2108750855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:58.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:57 smithi130 bash[51302]: debug 2023-08-28T00:45:57.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:57.847826+0000) 2023-08-28T00:45:58.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:58 smithi006 bash[27292]: cluster 2023-08-28T00:45:57.065402+0000 mgr.y (mgr.14901) 5021 : cluster [DBG] pgmap v5028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:58.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:58 smithi006 bash[33918]: cluster 2023-08-28T00:45:57.065402+0000 mgr.y (mgr.14901) 5021 : cluster [DBG] pgmap v5028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:58 smithi130 bash[27579]: cluster 2023-08-28T00:45:57.065402+0000 mgr.y (mgr.14901) 5021 : cluster [DBG] pgmap v5028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:45:59.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:58 smithi130 bash[51302]: debug 2023-08-28T00:45:58.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:58.848120+0000) 2023-08-28T00:45:59.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:45:59 smithi006 bash[27292]: audit 2023-08-28T00:45:59.068749+0000 mon.a (mon.0) 3542 : audit [DBG] from='client.? 172.21.15.6:0/23271918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:59.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:45:59 smithi006 bash[33918]: audit 2023-08-28T00:45:59.068749+0000 mon.a (mon.0) 3542 : audit [DBG] from='client.? 172.21.15.6:0/23271918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:45:59 smithi130 bash[27579]: audit 2023-08-28T00:45:59.068749+0000 mon.a (mon.0) 3542 : audit [DBG] from='client.? 172.21.15.6:0/23271918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:00.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:45:59 smithi130 bash[51302]: debug 2023-08-28T00:45:59.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:59.848403+0000) 2023-08-28T00:46:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:00 smithi130 bash[27579]: cluster 2023-08-28T00:45:59.066443+0000 mgr.y (mgr.14901) 5022 : cluster [DBG] pgmap v5029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:00 smithi006 bash[27292]: cluster 2023-08-28T00:45:59.066443+0000 mgr.y (mgr.14901) 5022 : cluster [DBG] pgmap v5029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:00 smithi006 bash[33918]: cluster 2023-08-28T00:45:59.066443+0000 mgr.y (mgr.14901) 5022 : cluster [DBG] pgmap v5029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:01.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:00 smithi130 bash[51302]: debug 2023-08-28T00:46:00.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:00.848669+0000) 2023-08-28T00:46:01.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:46:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:46:00] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:46:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:46:00.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:46:00.827Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:02.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:01 smithi130 bash[51302]: debug 2023-08-28T00:46:01.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:01.848927+0000) 2023-08-28T00:46:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:02 smithi130 bash[27579]: cluster 2023-08-28T00:46:01.067016+0000 mgr.y (mgr.14901) 5023 : cluster [DBG] pgmap v5030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:02 smithi130 bash[27579]: audit 2023-08-28T00:46:01.490965+0000 mon.c (mon.2) 1948 : audit [DBG] from='client.? 172.21.15.6:0/3257325859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:02.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:02 smithi006 bash[27292]: cluster 2023-08-28T00:46:01.067016+0000 mgr.y (mgr.14901) 5023 : cluster [DBG] pgmap v5030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:02.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:02 smithi006 bash[27292]: audit 2023-08-28T00:46:01.490965+0000 mon.c (mon.2) 1948 : audit [DBG] from='client.? 172.21.15.6:0/3257325859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:02 smithi006 bash[33918]: cluster 2023-08-28T00:46:01.067016+0000 mgr.y (mgr.14901) 5023 : cluster [DBG] pgmap v5030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:02 smithi006 bash[33918]: audit 2023-08-28T00:46:01.490965+0000 mon.c (mon.2) 1948 : audit [DBG] from='client.? 172.21.15.6:0/3257325859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:02.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:46:02.403Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:02.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:46:02.405Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:02 smithi130 bash[51302]: debug 2023-08-28T00:46:02.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:02.849250+0000) 2023-08-28T00:46:04.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:03 smithi130 bash[51302]: debug 2023-08-28T00:46:03.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:03.849533+0000) 2023-08-28T00:46:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:04 smithi130 bash[27579]: cluster 2023-08-28T00:46:03.067660+0000 mgr.y (mgr.14901) 5024 : cluster [DBG] pgmap v5031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:04 smithi130 bash[27579]: audit 2023-08-28T00:46:03.910242+0000 mon.a (mon.0) 3543 : audit [DBG] from='client.? 172.21.15.6:0/1470484579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:04.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:04 smithi006 bash[27292]: cluster 2023-08-28T00:46:03.067660+0000 mgr.y (mgr.14901) 5024 : cluster [DBG] pgmap v5031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:04.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:04 smithi006 bash[27292]: audit 2023-08-28T00:46:03.910242+0000 mon.a (mon.0) 3543 : audit [DBG] from='client.? 172.21.15.6:0/1470484579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:46:04.284Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:46:04.287Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:04.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:04 smithi006 bash[33918]: cluster 2023-08-28T00:46:03.067660+0000 mgr.y (mgr.14901) 5024 : cluster [DBG] pgmap v5031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:04.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:04 smithi006 bash[33918]: audit 2023-08-28T00:46:03.910242+0000 mon.a (mon.0) 3543 : audit [DBG] from='client.? 172.21.15.6:0/1470484579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:04 smithi130 bash[51302]: debug 2023-08-28T00:46:04.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:04.849863+0000) 2023-08-28T00:46:06.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:05 smithi130 bash[51302]: debug 2023-08-28T00:46:05.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:05.850186+0000) 2023-08-28T00:46:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:06 smithi130 bash[27579]: cluster 2023-08-28T00:46:05.068790+0000 mgr.y (mgr.14901) 5025 : cluster [DBG] pgmap v5032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:06 smithi006 bash[33918]: cluster 2023-08-28T00:46:05.068790+0000 mgr.y (mgr.14901) 5025 : cluster [DBG] pgmap v5032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:06 smithi006 bash[27292]: cluster 2023-08-28T00:46:05.068790+0000 mgr.y (mgr.14901) 5025 : cluster [DBG] pgmap v5032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:07.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:06 smithi130 bash[51302]: debug 2023-08-28T00:46:06.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:06.850495+0000) 2023-08-28T00:46:07.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:07 smithi130 bash[27579]: audit 2023-08-28T00:46:06.329984+0000 mon.a (mon.0) 3544 : audit [DBG] from='client.? 172.21.15.6:0/2543858259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:07.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:07 smithi006 bash[33918]: audit 2023-08-28T00:46:06.329984+0000 mon.a (mon.0) 3544 : audit [DBG] from='client.? 172.21.15.6:0/2543858259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:07.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:07 smithi006 bash[27292]: audit 2023-08-28T00:46:06.329984+0000 mon.a (mon.0) 3544 : audit [DBG] from='client.? 172.21.15.6:0/2543858259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:08.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:07 smithi130 bash[51302]: debug 2023-08-28T00:46:07.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:07.850750+0000) 2023-08-28T00:46:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:08 smithi130 bash[27579]: cluster 2023-08-28T00:46:07.069419+0000 mgr.y (mgr.14901) 5026 : cluster [DBG] pgmap v5033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:08 smithi006 bash[33918]: cluster 2023-08-28T00:46:07.069419+0000 mgr.y (mgr.14901) 5026 : cluster [DBG] pgmap v5033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:08.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:08 smithi006 bash[27292]: cluster 2023-08-28T00:46:07.069419+0000 mgr.y (mgr.14901) 5026 : cluster [DBG] pgmap v5033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:09.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:08 smithi130 bash[51302]: debug 2023-08-28T00:46:08.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:08.851034+0000) 2023-08-28T00:46:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:09 smithi130 bash[27579]: audit 2023-08-28T00:46:08.735430+0000 mon.a (mon.0) 3545 : audit [DBG] from='client.? 172.21.15.6:0/1379207210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:09.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:09 smithi006 bash[33918]: audit 2023-08-28T00:46:08.735430+0000 mon.a (mon.0) 3545 : audit [DBG] from='client.? 172.21.15.6:0/1379207210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:09.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:09 smithi006 bash[27292]: audit 2023-08-28T00:46:08.735430+0000 mon.a (mon.0) 3545 : audit [DBG] from='client.? 172.21.15.6:0/1379207210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:10.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:09 smithi130 bash[51302]: debug 2023-08-28T00:46:09.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:09.851264+0000) 2023-08-28T00:46:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:10 smithi130 bash[27579]: cluster 2023-08-28T00:46:09.070400+0000 mgr.y (mgr.14901) 5027 : cluster [DBG] pgmap v5034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:10 smithi006 bash[33918]: cluster 2023-08-28T00:46:09.070400+0000 mgr.y (mgr.14901) 5027 : cluster [DBG] pgmap v5034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:10 smithi006 bash[27292]: cluster 2023-08-28T00:46:09.070400+0000 mgr.y (mgr.14901) 5027 : cluster [DBG] pgmap v5034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:11.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:10 smithi130 bash[51302]: debug 2023-08-28T00:46:10.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:10.851492+0000) 2023-08-28T00:46:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:46:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:46:10] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:46:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:46:10.826Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:46:10.828Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:12.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:11 smithi130 bash[51302]: debug 2023-08-28T00:46:11.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:11.851794+0000) 2023-08-28T00:46:12.408 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:12 smithi006 bash[33918]: cluster 2023-08-28T00:46:11.071015+0000 mgr.y (mgr.14901) 5028 : cluster [DBG] pgmap v5035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:12.408 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:12 smithi006 bash[33918]: audit 2023-08-28T00:46:11.158245+0000 mon.c (mon.2) 1949 : audit [DBG] from='client.? 172.21.15.6:0/599227631' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:12.408 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:12 smithi006 bash[27292]: cluster 2023-08-28T00:46:11.071015+0000 mgr.y (mgr.14901) 5028 : cluster [DBG] pgmap v5035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:12.408 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:12 smithi006 bash[27292]: audit 2023-08-28T00:46:11.158245+0000 mon.c (mon.2) 1949 : audit [DBG] from='client.? 172.21.15.6:0/599227631' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:12 smithi130 bash[27579]: cluster 2023-08-28T00:46:11.071015+0000 mgr.y (mgr.14901) 5028 : cluster [DBG] pgmap v5035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:12 smithi130 bash[27579]: audit 2023-08-28T00:46:11.158245+0000 mon.c (mon.2) 1949 : audit [DBG] from='client.? 172.21.15.6:0/599227631' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:46:12.404Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:46:12.406Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:13.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:12 smithi130 bash[51302]: debug 2023-08-28T00:46:12.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:12.852019+0000) 2023-08-28T00:46:14.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:13 smithi130 bash[51302]: debug 2023-08-28T00:46:13.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:13.852283+0000) 2023-08-28T00:46:14.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:14 smithi006 bash[27292]: cluster 2023-08-28T00:46:13.071670+0000 mgr.y (mgr.14901) 5029 : cluster [DBG] pgmap v5036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:14.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:14 smithi006 bash[27292]: audit 2023-08-28T00:46:13.572655+0000 mon.a (mon.0) 3546 : audit [DBG] from='client.? 172.21.15.6:0/1142841803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:14.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:14 smithi006 bash[33918]: cluster 2023-08-28T00:46:13.071670+0000 mgr.y (mgr.14901) 5029 : cluster [DBG] pgmap v5036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:14.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:14 smithi006 bash[33918]: audit 2023-08-28T00:46:13.572655+0000 mon.a (mon.0) 3546 : audit [DBG] from='client.? 172.21.15.6:0/1142841803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:14.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:46:14.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:14.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:46:14.287Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:14 smithi130 bash[27579]: cluster 2023-08-28T00:46:13.071670+0000 mgr.y (mgr.14901) 5029 : cluster [DBG] pgmap v5036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:14 smithi130 bash[27579]: audit 2023-08-28T00:46:13.572655+0000 mon.a (mon.0) 3546 : audit [DBG] from='client.? 172.21.15.6:0/1142841803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:15.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:14 smithi130 bash[51302]: debug 2023-08-28T00:46:14.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:14.852631+0000) 2023-08-28T00:46:16.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:15 smithi130 bash[51302]: debug 2023-08-28T00:46:15.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:15.852920+0000) 2023-08-28T00:46:16.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:16 smithi006 bash[33918]: cluster 2023-08-28T00:46:15.072774+0000 mgr.y (mgr.14901) 5030 : cluster [DBG] pgmap v5037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:16.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:16 smithi006 bash[33918]: audit 2023-08-28T00:46:15.992971+0000 mon.a (mon.0) 3547 : audit [DBG] from='client.? 172.21.15.6:0/3986816530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:16.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:16 smithi006 bash[27292]: cluster 2023-08-28T00:46:15.072774+0000 mgr.y (mgr.14901) 5030 : cluster [DBG] pgmap v5037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:16.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:16 smithi006 bash[27292]: audit 2023-08-28T00:46:15.992971+0000 mon.a (mon.0) 3547 : audit [DBG] from='client.? 172.21.15.6:0/3986816530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:16 smithi130 bash[27579]: cluster 2023-08-28T00:46:15.072774+0000 mgr.y (mgr.14901) 5030 : cluster [DBG] pgmap v5037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:16 smithi130 bash[27579]: audit 2023-08-28T00:46:15.992971+0000 mon.a (mon.0) 3547 : audit [DBG] from='client.? 172.21.15.6:0/3986816530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:17.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:16 smithi130 bash[51302]: debug 2023-08-28T00:46:16.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:16.853207+0000) 2023-08-28T00:46:18.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:17 smithi130 bash[51302]: debug 2023-08-28T00:46:17.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:17.853494+0000) 2023-08-28T00:46:18.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:18 smithi006 bash[27292]: cluster 2023-08-28T00:46:17.073379+0000 mgr.y (mgr.14901) 5031 : cluster [DBG] pgmap v5038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:18.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:18 smithi006 bash[33918]: cluster 2023-08-28T00:46:17.073379+0000 mgr.y (mgr.14901) 5031 : cluster [DBG] pgmap v5038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:18 smithi130 bash[27579]: cluster 2023-08-28T00:46:17.073379+0000 mgr.y (mgr.14901) 5031 : cluster [DBG] pgmap v5038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:19.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:18 smithi130 bash[51302]: debug 2023-08-28T00:46:18.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:18.853751+0000) 2023-08-28T00:46:19.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:19 smithi006 bash[27292]: audit 2023-08-28T00:46:18.403452+0000 mon.c (mon.2) 1950 : audit [DBG] from='client.? 172.21.15.6:0/1342427160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:19.470 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:19 smithi006 bash[33918]: audit 2023-08-28T00:46:18.403452+0000 mon.c (mon.2) 1950 : audit [DBG] from='client.? 172.21.15.6:0/1342427160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:19 smithi130 bash[27579]: audit 2023-08-28T00:46:18.403452+0000 mon.c (mon.2) 1950 : audit [DBG] from='client.? 172.21.15.6:0/1342427160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:20.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:19 smithi130 bash[51302]: debug 2023-08-28T00:46:19.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:19.854038+0000) 2023-08-28T00:46:20.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:20 smithi006 bash[33918]: cluster 2023-08-28T00:46:19.074569+0000 mgr.y (mgr.14901) 5032 : cluster [DBG] pgmap v5039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:20.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:20 smithi006 bash[27292]: cluster 2023-08-28T00:46:19.074569+0000 mgr.y (mgr.14901) 5032 : cluster [DBG] pgmap v5039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:20 smithi130 bash[27579]: cluster 2023-08-28T00:46:19.074569+0000 mgr.y (mgr.14901) 5032 : cluster [DBG] pgmap v5039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:21.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:20 smithi130 bash[51302]: debug 2023-08-28T00:46:20.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:20.854309+0000) 2023-08-28T00:46:21.189 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:46:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:46:20] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:46:21.189 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:46:20.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:21.189 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:46:20.828Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:21.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:21 smithi006 bash[33918]: audit 2023-08-28T00:46:20.818515+0000 mon.c (mon.2) 1951 : audit [DBG] from='client.? 172.21.15.6:0/2298420121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:21.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:21 smithi006 bash[27292]: audit 2023-08-28T00:46:20.818515+0000 mon.c (mon.2) 1951 : audit [DBG] from='client.? 172.21.15.6:0/2298420121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:21 smithi130 bash[27579]: audit 2023-08-28T00:46:20.818515+0000 mon.c (mon.2) 1951 : audit [DBG] from='client.? 172.21.15.6:0/2298420121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:22.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:21 smithi130 bash[51302]: debug 2023-08-28T00:46:21.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:21.854580+0000) 2023-08-28T00:46:22.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:22 smithi006 bash[33918]: cluster 2023-08-28T00:46:21.075263+0000 mgr.y (mgr.14901) 5033 : cluster [DBG] pgmap v5040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:22.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:46:22.404Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:22.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:46:22.406Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:22.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:22 smithi006 bash[27292]: cluster 2023-08-28T00:46:21.075263+0000 mgr.y (mgr.14901) 5033 : cluster [DBG] pgmap v5040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:22 smithi130 bash[27579]: cluster 2023-08-28T00:46:21.075263+0000 mgr.y (mgr.14901) 5033 : cluster [DBG] pgmap v5040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:23.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:22 smithi130 bash[51302]: debug 2023-08-28T00:46:22.853+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:22.854875+0000) 2023-08-28T00:46:24.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:23 smithi130 bash[51302]: debug 2023-08-28T00:46:23.853+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:23.855204+0000) 2023-08-28T00:46:24.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:24 smithi006 bash[33918]: cluster 2023-08-28T00:46:23.075892+0000 mgr.y (mgr.14901) 5034 : cluster [DBG] pgmap v5041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:24.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:24 smithi006 bash[33918]: audit 2023-08-28T00:46:23.230182+0000 mon.c (mon.2) 1952 : audit [DBG] from='client.? 172.21.15.6:0/4137761508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:46:24.286Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:46:24.289Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:24.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:24 smithi006 bash[27292]: cluster 2023-08-28T00:46:23.075892+0000 mgr.y (mgr.14901) 5034 : cluster [DBG] pgmap v5041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:24.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:24 smithi006 bash[27292]: audit 2023-08-28T00:46:23.230182+0000 mon.c (mon.2) 1952 : audit [DBG] from='client.? 172.21.15.6:0/4137761508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:24 smithi130 bash[27579]: cluster 2023-08-28T00:46:23.075892+0000 mgr.y (mgr.14901) 5034 : cluster [DBG] pgmap v5041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:24 smithi130 bash[27579]: audit 2023-08-28T00:46:23.230182+0000 mon.c (mon.2) 1952 : audit [DBG] from='client.? 172.21.15.6:0/4137761508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:24 smithi130 bash[51302]: debug 2023-08-28T00:46:24.853+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:24.855437+0000) 2023-08-28T00:46:26.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:25 smithi130 bash[51302]: debug 2023-08-28T00:46:25.853+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:25.855749+0000) 2023-08-28T00:46:26.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:26 smithi006 bash[27292]: cluster 2023-08-28T00:46:25.077035+0000 mgr.y (mgr.14901) 5035 : cluster [DBG] pgmap v5042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:26.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:26 smithi006 bash[27292]: audit 2023-08-28T00:46:25.640386+0000 mon.c (mon.2) 1953 : audit [DBG] from='client.? 172.21.15.6:0/124345124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:26 smithi006 bash[33918]: cluster 2023-08-28T00:46:25.077035+0000 mgr.y (mgr.14901) 5035 : cluster [DBG] pgmap v5042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:26.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:26 smithi006 bash[33918]: audit 2023-08-28T00:46:25.640386+0000 mon.c (mon.2) 1953 : audit [DBG] from='client.? 172.21.15.6:0/124345124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:26 smithi130 bash[27579]: cluster 2023-08-28T00:46:25.077035+0000 mgr.y (mgr.14901) 5035 : cluster [DBG] pgmap v5042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:26 smithi130 bash[27579]: audit 2023-08-28T00:46:25.640386+0000 mon.c (mon.2) 1953 : audit [DBG] from='client.? 172.21.15.6:0/124345124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:27.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:26 smithi130 bash[51302]: debug 2023-08-28T00:46:26.853+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:26.856028+0000) 2023-08-28T00:46:28.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:27 smithi130 bash[51302]: debug 2023-08-28T00:46:27.853+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:27.856362+0000) 2023-08-28T00:46:28.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:28 smithi006 bash[27292]: cluster 2023-08-28T00:46:27.077685+0000 mgr.y (mgr.14901) 5036 : cluster [DBG] pgmap v5043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:28.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:28 smithi006 bash[27292]: audit 2023-08-28T00:46:28.047646+0000 mon.c (mon.2) 1954 : audit [DBG] from='client.? 172.21.15.6:0/2205250092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:28.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:28 smithi006 bash[33918]: cluster 2023-08-28T00:46:27.077685+0000 mgr.y (mgr.14901) 5036 : cluster [DBG] pgmap v5043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:28.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:28 smithi006 bash[33918]: audit 2023-08-28T00:46:28.047646+0000 mon.c (mon.2) 1954 : audit [DBG] from='client.? 172.21.15.6:0/2205250092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:28 smithi130 bash[27579]: cluster 2023-08-28T00:46:27.077685+0000 mgr.y (mgr.14901) 5036 : cluster [DBG] pgmap v5043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:28 smithi130 bash[27579]: audit 2023-08-28T00:46:28.047646+0000 mon.c (mon.2) 1954 : audit [DBG] from='client.? 172.21.15.6:0/2205250092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:29.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:28 smithi130 bash[51302]: debug 2023-08-28T00:46:28.853+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:28.856673+0000) 2023-08-28T00:46:30.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:29 smithi130 bash[51302]: debug 2023-08-28T00:46:29.852+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:29.856970+0000) 2023-08-28T00:46:30.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:30 smithi006 bash[33918]: cluster 2023-08-28T00:46:29.078783+0000 mgr.y (mgr.14901) 5037 : cluster [DBG] pgmap v5044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:30.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:30 smithi006 bash[27292]: cluster 2023-08-28T00:46:29.078783+0000 mgr.y (mgr.14901) 5037 : cluster [DBG] pgmap v5044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:30 smithi130 bash[27579]: cluster 2023-08-28T00:46:29.078783+0000 mgr.y (mgr.14901) 5037 : cluster [DBG] pgmap v5044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:31.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:30 smithi130 bash[51302]: debug 2023-08-28T00:46:30.852+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:30.857269+0000) 2023-08-28T00:46:31.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:46:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:46:30] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T00:46:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:46:30.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:46:30.829Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:31.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:31 smithi130 bash[27579]: audit 2023-08-28T00:46:30.473167+0000 mon.a (mon.0) 3548 : audit [DBG] from='client.? 172.21.15.6:0/1918678501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:31.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:31 smithi006 bash[27292]: audit 2023-08-28T00:46:30.473167+0000 mon.a (mon.0) 3548 : audit [DBG] from='client.? 172.21.15.6:0/1918678501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:31.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:31 smithi006 bash[33918]: audit 2023-08-28T00:46:30.473167+0000 mon.a (mon.0) 3548 : audit [DBG] from='client.? 172.21.15.6:0/1918678501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:32.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:31 smithi130 bash[51302]: debug 2023-08-28T00:46:31.852+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:31.857560+0000) 2023-08-28T00:46:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:32 smithi130 bash[27579]: cluster 2023-08-28T00:46:31.079432+0000 mgr.y (mgr.14901) 5038 : cluster [DBG] pgmap v5045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:46:32.405Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:46:32.407Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:32 smithi006 bash[27292]: cluster 2023-08-28T00:46:31.079432+0000 mgr.y (mgr.14901) 5038 : cluster [DBG] pgmap v5045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:32 smithi006 bash[33918]: cluster 2023-08-28T00:46:31.079432+0000 mgr.y (mgr.14901) 5038 : cluster [DBG] pgmap v5045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:33.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:32 smithi130 bash[51302]: debug 2023-08-28T00:46:32.856+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:32.857882+0000) 2023-08-28T00:46:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:33 smithi130 bash[27579]: audit 2023-08-28T00:46:32.886626+0000 mon.a (mon.0) 3549 : audit [DBG] from='client.? 172.21.15.6:0/216386450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:33.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:33 smithi006 bash[33918]: audit 2023-08-28T00:46:32.886626+0000 mon.a (mon.0) 3549 : audit [DBG] from='client.? 172.21.15.6:0/216386450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:33.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:33 smithi006 bash[27292]: audit 2023-08-28T00:46:32.886626+0000 mon.a (mon.0) 3549 : audit [DBG] from='client.? 172.21.15.6:0/216386450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:34.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:33 smithi130 bash[51302]: debug 2023-08-28T00:46:33.852+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:33.858174+0000) 2023-08-28T00:46:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:34 smithi130 bash[27579]: cluster 2023-08-28T00:46:33.080079+0000 mgr.y (mgr.14901) 5039 : cluster [DBG] pgmap v5046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:34 smithi006 bash[33918]: cluster 2023-08-28T00:46:33.080079+0000 mgr.y (mgr.14901) 5039 : cluster [DBG] pgmap v5046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:34 smithi006 bash[27292]: cluster 2023-08-28T00:46:33.080079+0000 mgr.y (mgr.14901) 5039 : cluster [DBG] pgmap v5046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:46:34.287Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:46:34.290Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:35.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:34 smithi130 bash[51302]: debug 2023-08-28T00:46:34.852+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:34.858451+0000) 2023-08-28T00:46:36.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:35 smithi130 bash[51302]: debug 2023-08-28T00:46:35.856+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:35.858776+0000) 2023-08-28T00:46:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:36 smithi130 bash[27579]: cluster 2023-08-28T00:46:35.081194+0000 mgr.y (mgr.14901) 5040 : cluster [DBG] pgmap v5047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:36 smithi130 bash[27579]: audit 2023-08-28T00:46:35.308344+0000 mon.c (mon.2) 1955 : audit [DBG] from='client.? 172.21.15.6:0/2737898908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:36.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:36 smithi006 bash[33918]: cluster 2023-08-28T00:46:35.081194+0000 mgr.y (mgr.14901) 5040 : cluster [DBG] pgmap v5047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:36.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:36 smithi006 bash[33918]: audit 2023-08-28T00:46:35.308344+0000 mon.c (mon.2) 1955 : audit [DBG] from='client.? 172.21.15.6:0/2737898908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:36.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:36 smithi006 bash[27292]: cluster 2023-08-28T00:46:35.081194+0000 mgr.y (mgr.14901) 5040 : cluster [DBG] pgmap v5047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:36.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:36 smithi006 bash[27292]: audit 2023-08-28T00:46:35.308344+0000 mon.c (mon.2) 1955 : audit [DBG] from='client.? 172.21.15.6:0/2737898908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:37.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:36 smithi130 bash[51302]: debug 2023-08-28T00:46:36.856+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:36.859039+0000) 2023-08-28T00:46:38.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:37 smithi130 bash[51302]: debug 2023-08-28T00:46:37.856+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:37.859246+0000) 2023-08-28T00:46:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:38 smithi130 bash[27579]: cluster 2023-08-28T00:46:37.081852+0000 mgr.y (mgr.14901) 5041 : cluster [DBG] pgmap v5048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:38 smithi130 bash[27579]: audit 2023-08-28T00:46:37.724057+0000 mon.c (mon.2) 1956 : audit [DBG] from='client.? 172.21.15.6:0/351460387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:38.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:38 smithi006 bash[27292]: cluster 2023-08-28T00:46:37.081852+0000 mgr.y (mgr.14901) 5041 : cluster [DBG] pgmap v5048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:38.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:38 smithi006 bash[27292]: audit 2023-08-28T00:46:37.724057+0000 mon.c (mon.2) 1956 : audit [DBG] from='client.? 172.21.15.6:0/351460387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:38.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:38 smithi006 bash[33918]: cluster 2023-08-28T00:46:37.081852+0000 mgr.y (mgr.14901) 5041 : cluster [DBG] pgmap v5048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:38.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:38 smithi006 bash[33918]: audit 2023-08-28T00:46:37.724057+0000 mon.c (mon.2) 1956 : audit [DBG] from='client.? 172.21.15.6:0/351460387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:39.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:38 smithi130 bash[51302]: debug 2023-08-28T00:46:38.856+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:38.859506+0000) 2023-08-28T00:46:40.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:39 smithi130 bash[51302]: debug 2023-08-28T00:46:39.856+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:39.859752+0000) 2023-08-28T00:46:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:40 smithi130 bash[27579]: cluster 2023-08-28T00:46:39.082941+0000 mgr.y (mgr.14901) 5042 : cluster [DBG] pgmap v5049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:40 smithi130 bash[27579]: audit 2023-08-28T00:46:39.733029+0000 mon.b (mon.1) 604 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:46:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:40 smithi130 bash[27579]: audit 2023-08-28T00:46:40.137328+0000 mon.c (mon.2) 1957 : audit [DBG] from='client.? 172.21.15.6:0/483995135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:40 smithi006 bash[33918]: cluster 2023-08-28T00:46:39.082941+0000 mgr.y (mgr.14901) 5042 : cluster [DBG] pgmap v5049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:40 smithi006 bash[33918]: audit 2023-08-28T00:46:39.733029+0000 mon.b (mon.1) 604 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:46:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:40 smithi006 bash[33918]: audit 2023-08-28T00:46:40.137328+0000 mon.c (mon.2) 1957 : audit [DBG] from='client.? 172.21.15.6:0/483995135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:40 smithi006 bash[27292]: cluster 2023-08-28T00:46:39.082941+0000 mgr.y (mgr.14901) 5042 : cluster [DBG] pgmap v5049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:40 smithi006 bash[27292]: audit 2023-08-28T00:46:39.733029+0000 mon.b (mon.1) 604 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:46:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:40 smithi006 bash[27292]: audit 2023-08-28T00:46:40.137328+0000 mon.c (mon.2) 1957 : audit [DBG] from='client.? 172.21.15.6:0/483995135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:41.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:40 smithi130 bash[51302]: debug 2023-08-28T00:46:40.856+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:40.860033+0000) 2023-08-28T00:46:41.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:46:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:46:40] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T00:46:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:46:40.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:46:40.830Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:42.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:41 smithi130 bash[51302]: debug 2023-08-28T00:46:41.856+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:41.860375+0000) 2023-08-28T00:46:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:42 smithi130 bash[27579]: cluster 2023-08-28T00:46:41.083554+0000 mgr.y (mgr.14901) 5043 : cluster [DBG] pgmap v5050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:42 smithi006 bash[27292]: cluster 2023-08-28T00:46:41.083554+0000 mgr.y (mgr.14901) 5043 : cluster [DBG] pgmap v5050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:42.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:42 smithi006 bash[33918]: cluster 2023-08-28T00:46:41.083554+0000 mgr.y (mgr.14901) 5043 : cluster [DBG] pgmap v5050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:46:42.405Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:46:42.407Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:43.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:42 smithi130 bash[51302]: debug 2023-08-28T00:46:42.856+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:42.860648+0000) 2023-08-28T00:46:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:43 smithi130 bash[27579]: audit 2023-08-28T00:46:42.558568+0000 mon.c (mon.2) 1958 : audit [DBG] from='client.? 172.21.15.6:0/2287291192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:43.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:43 smithi006 bash[33918]: audit 2023-08-28T00:46:42.558568+0000 mon.c (mon.2) 1958 : audit [DBG] from='client.? 172.21.15.6:0/2287291192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:43.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:43 smithi006 bash[27292]: audit 2023-08-28T00:46:42.558568+0000 mon.c (mon.2) 1958 : audit [DBG] from='client.? 172.21.15.6:0/2287291192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:44.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:43 smithi130 bash[51302]: debug 2023-08-28T00:46:43.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:43.860932+0000) 2023-08-28T00:46:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:44 smithi130 bash[27579]: cluster 2023-08-28T00:46:43.084182+0000 mgr.y (mgr.14901) 5044 : cluster [DBG] pgmap v5051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:44.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:44 smithi006 bash[27292]: cluster 2023-08-28T00:46:43.084182+0000 mgr.y (mgr.14901) 5044 : cluster [DBG] pgmap v5051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:44.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:44 smithi006 bash[33918]: cluster 2023-08-28T00:46:43.084182+0000 mgr.y (mgr.14901) 5044 : cluster [DBG] pgmap v5051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:46:44.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:46:44.290Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:44 smithi130 bash[51302]: debug 2023-08-28T00:46:44.856+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:44.861157+0000) 2023-08-28T00:46:45.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:45 smithi130 bash[27579]: audit 2023-08-28T00:46:44.993287+0000 mon.a (mon.0) 3550 : audit [DBG] from='client.? 172.21.15.6:0/470384250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:45.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:45 smithi006 bash[27292]: audit 2023-08-28T00:46:44.993287+0000 mon.a (mon.0) 3550 : audit [DBG] from='client.? 172.21.15.6:0/470384250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:45.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:45 smithi006 bash[33918]: audit 2023-08-28T00:46:44.993287+0000 mon.a (mon.0) 3550 : audit [DBG] from='client.? 172.21.15.6:0/470384250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:46.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:45 smithi130 bash[51302]: debug 2023-08-28T00:46:45.856+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:45.861444+0000) 2023-08-28T00:46:46.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:46 smithi006 bash[27292]: cluster 2023-08-28T00:46:45.085300+0000 mgr.y (mgr.14901) 5045 : cluster [DBG] pgmap v5052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:46.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:46 smithi006 bash[27292]: audit 2023-08-28T00:46:45.626001+0000 mon.a (mon.0) 3551 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:46:46.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:46 smithi006 bash[27292]: audit 2023-08-28T00:46:46.117673+0000 mon.a (mon.0) 3552 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:46:46.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:46 smithi006 bash[27292]: audit 2023-08-28T00:46:46.407719+0000 mon.b (mon.1) 605 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:46:46.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:46 smithi006 bash[27292]: audit 2023-08-28T00:46:46.409824+0000 mon.b (mon.1) 606 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:46:46.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:46 smithi006 bash[27292]: audit 2023-08-28T00:46:46.419669+0000 mon.a (mon.0) 3553 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:46:46.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:46 smithi006 bash[33918]: cluster 2023-08-28T00:46:45.085300+0000 mgr.y (mgr.14901) 5045 : cluster [DBG] pgmap v5052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:46.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:46 smithi006 bash[33918]: audit 2023-08-28T00:46:45.626001+0000 mon.a (mon.0) 3551 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:46:46.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:46 smithi006 bash[33918]: audit 2023-08-28T00:46:46.117673+0000 mon.a (mon.0) 3552 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:46:46.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:46 smithi006 bash[33918]: audit 2023-08-28T00:46:46.407719+0000 mon.b (mon.1) 605 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:46:46.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:46 smithi006 bash[33918]: audit 2023-08-28T00:46:46.409824+0000 mon.b (mon.1) 606 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:46:46.970 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:46 smithi006 bash[33918]: audit 2023-08-28T00:46:46.419669+0000 mon.a (mon.0) 3553 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:46:47.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:46 smithi130 bash[51302]: debug 2023-08-28T00:46:46.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:46.861786+0000) 2023-08-28T00:46:47.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:46 smithi130 bash[27579]: cluster 2023-08-28T00:46:45.085300+0000 mgr.y (mgr.14901) 5045 : cluster [DBG] pgmap v5052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:47.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:46 smithi130 bash[27579]: audit 2023-08-28T00:46:45.626001+0000 mon.a (mon.0) 3551 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:46:47.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:46 smithi130 bash[27579]: audit 2023-08-28T00:46:46.117673+0000 mon.a (mon.0) 3552 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:46:47.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:46 smithi130 bash[27579]: audit 2023-08-28T00:46:46.407719+0000 mon.b (mon.1) 605 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:46:47.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:46 smithi130 bash[27579]: audit 2023-08-28T00:46:46.409824+0000 mon.b (mon.1) 606 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:46:47.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:46 smithi130 bash[27579]: audit 2023-08-28T00:46:46.419669+0000 mon.a (mon.0) 3553 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:46:47.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:47 smithi006 bash[33918]: audit 2023-08-28T00:46:47.409847+0000 mon.c (mon.2) 1959 : audit [DBG] from='client.? 172.21.15.6:0/3166892309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:47.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:47 smithi006 bash[27292]: audit 2023-08-28T00:46:47.409847+0000 mon.c (mon.2) 1959 : audit [DBG] from='client.? 172.21.15.6:0/3166892309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:48.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:47 smithi130 bash[27579]: audit 2023-08-28T00:46:47.409847+0000 mon.c (mon.2) 1959 : audit [DBG] from='client.? 172.21.15.6:0/3166892309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:48.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:47 smithi130 bash[51302]: debug 2023-08-28T00:46:47.856+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:47.862069+0000) 2023-08-28T00:46:48.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:48 smithi006 bash[33918]: cluster 2023-08-28T00:46:47.086061+0000 mgr.y (mgr.14901) 5046 : cluster [DBG] pgmap v5053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:48.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:48 smithi006 bash[27292]: cluster 2023-08-28T00:46:47.086061+0000 mgr.y (mgr.14901) 5046 : cluster [DBG] pgmap v5053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:48 smithi130 bash[27579]: cluster 2023-08-28T00:46:47.086061+0000 mgr.y (mgr.14901) 5046 : cluster [DBG] pgmap v5053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:49.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:48 smithi130 bash[51302]: debug 2023-08-28T00:46:48.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:48.862379+0000) 2023-08-28T00:46:50.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:49 smithi130 bash[51302]: debug 2023-08-28T00:46:49.856+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:49.862665+0000) 2023-08-28T00:46:50.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:46:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:46:50] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:46:50.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:50 smithi006 bash[27292]: cluster 2023-08-28T00:46:49.087168+0000 mgr.y (mgr.14901) 5047 : cluster [DBG] pgmap v5054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:50.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:50 smithi006 bash[27292]: audit 2023-08-28T00:46:49.825146+0000 mon.a (mon.0) 3554 : audit [DBG] from='client.? 172.21.15.6:0/3243122063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:50.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:50 smithi006 bash[33918]: cluster 2023-08-28T00:46:49.087168+0000 mgr.y (mgr.14901) 5047 : cluster [DBG] pgmap v5054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:50.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:50 smithi006 bash[33918]: audit 2023-08-28T00:46:49.825146+0000 mon.a (mon.0) 3554 : audit [DBG] from='client.? 172.21.15.6:0/3243122063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:50.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:46:50.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:50.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:46:50.830Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:51.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:50 smithi130 bash[51302]: debug 2023-08-28T00:46:50.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:50.862968+0000) 2023-08-28T00:46:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:50 smithi130 bash[27579]: cluster 2023-08-28T00:46:49.087168+0000 mgr.y (mgr.14901) 5047 : cluster [DBG] pgmap v5054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:51.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:50 smithi130 bash[27579]: audit 2023-08-28T00:46:49.825146+0000 mon.a (mon.0) 3554 : audit [DBG] from='client.? 172.21.15.6:0/3243122063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:52.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:51 smithi130 bash[51302]: debug 2023-08-28T00:46:51.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:51.863238+0000) 2023-08-28T00:46:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:52 smithi006 bash[27292]: cluster 2023-08-28T00:46:51.087780+0000 mgr.y (mgr.14901) 5048 : cluster [DBG] pgmap v5055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:52 smithi006 bash[27292]: audit 2023-08-28T00:46:52.248823+0000 mon.c (mon.2) 1960 : audit [DBG] from='client.? 172.21.15.6:0/2358776565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:52 smithi006 bash[33918]: cluster 2023-08-28T00:46:51.087780+0000 mgr.y (mgr.14901) 5048 : cluster [DBG] pgmap v5055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:52 smithi006 bash[33918]: audit 2023-08-28T00:46:52.248823+0000 mon.c (mon.2) 1960 : audit [DBG] from='client.? 172.21.15.6:0/2358776565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:52.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:46:52.406Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:52.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:46:52.408Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:52 smithi130 bash[51302]: debug 2023-08-28T00:46:52.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:52.863592+0000) 2023-08-28T00:46:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:52 smithi130 bash[27579]: cluster 2023-08-28T00:46:51.087780+0000 mgr.y (mgr.14901) 5048 : cluster [DBG] pgmap v5055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:53.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:52 smithi130 bash[27579]: audit 2023-08-28T00:46:52.248823+0000 mon.c (mon.2) 1960 : audit [DBG] from='client.? 172.21.15.6:0/2358776565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:54.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:53 smithi130 bash[51302]: debug 2023-08-28T00:46:53.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:53.863870+0000) 2023-08-28T00:46:54.657 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:46:54.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:54.657 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:46:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:46:54.291Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:46:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:54 smithi006 bash[27292]: cluster 2023-08-28T00:46:53.088454+0000 mgr.y (mgr.14901) 5049 : cluster [DBG] pgmap v5056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:54.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:54 smithi006 bash[33918]: cluster 2023-08-28T00:46:53.088454+0000 mgr.y (mgr.14901) 5049 : cluster [DBG] pgmap v5056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:54 smithi130 bash[27579]: cluster 2023-08-28T00:46:53.088454+0000 mgr.y (mgr.14901) 5049 : cluster [DBG] pgmap v5056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:54 smithi130 bash[51302]: debug 2023-08-28T00:46:54.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:54.864162+0000) 2023-08-28T00:46:55.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:55 smithi006 bash[27292]: audit 2023-08-28T00:46:54.658341+0000 mon.a (mon.0) 3555 : audit [DBG] from='client.? 172.21.15.6:0/4202751283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:55.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:55 smithi006 bash[33918]: audit 2023-08-28T00:46:54.658341+0000 mon.a (mon.0) 3555 : audit [DBG] from='client.? 172.21.15.6:0/4202751283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:56.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:55 smithi130 bash[27579]: audit 2023-08-28T00:46:54.658341+0000 mon.a (mon.0) 3555 : audit [DBG] from='client.? 172.21.15.6:0/4202751283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:56.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:55 smithi130 bash[51302]: debug 2023-08-28T00:46:55.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:55.864525+0000) 2023-08-28T00:46:56.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:56 smithi006 bash[33918]: cluster 2023-08-28T00:46:55.089618+0000 mgr.y (mgr.14901) 5050 : cluster [DBG] pgmap v5057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:56.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:56 smithi006 bash[27292]: cluster 2023-08-28T00:46:55.089618+0000 mgr.y (mgr.14901) 5050 : cluster [DBG] pgmap v5057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:57.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:56 smithi130 bash[27579]: cluster 2023-08-28T00:46:55.089618+0000 mgr.y (mgr.14901) 5050 : cluster [DBG] pgmap v5057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:57.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:56 smithi130 bash[51302]: debug 2023-08-28T00:46:56.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:56.864852+0000) 2023-08-28T00:46:57.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:57 smithi006 bash[27292]: audit 2023-08-28T00:46:57.077739+0000 mon.c (mon.2) 1961 : audit [DBG] from='client.? 172.21.15.6:0/2815308204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:57.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:57 smithi006 bash[33918]: audit 2023-08-28T00:46:57.077739+0000 mon.c (mon.2) 1961 : audit [DBG] from='client.? 172.21.15.6:0/2815308204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:58.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:57 smithi130 bash[27579]: audit 2023-08-28T00:46:57.077739+0000 mon.c (mon.2) 1961 : audit [DBG] from='client.? 172.21.15.6:0/2815308204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:58.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:57 smithi130 bash[51302]: debug 2023-08-28T00:46:57.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:57.865121+0000) 2023-08-28T00:46:58.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:58 smithi006 bash[27292]: cluster 2023-08-28T00:46:57.090181+0000 mgr.y (mgr.14901) 5051 : cluster [DBG] pgmap v5058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:58.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:58 smithi006 bash[33918]: cluster 2023-08-28T00:46:57.090181+0000 mgr.y (mgr.14901) 5051 : cluster [DBG] pgmap v5058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:59.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:58 smithi130 bash[27579]: cluster 2023-08-28T00:46:57.090181+0000 mgr.y (mgr.14901) 5051 : cluster [DBG] pgmap v5058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:46:59.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:58 smithi130 bash[51302]: debug 2023-08-28T00:46:58.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:58.865381+0000) 2023-08-28T00:46:59.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:46:59 smithi006 bash[27292]: audit 2023-08-28T00:46:59.482988+0000 mon.a (mon.0) 3556 : audit [DBG] from='client.? 172.21.15.6:0/1402057447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:59.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:46:59 smithi006 bash[33918]: audit 2023-08-28T00:46:59.482988+0000 mon.a (mon.0) 3556 : audit [DBG] from='client.? 172.21.15.6:0/1402057447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:00.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:46:59 smithi130 bash[27579]: audit 2023-08-28T00:46:59.482988+0000 mon.a (mon.0) 3556 : audit [DBG] from='client.? 172.21.15.6:0/1402057447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:00.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:46:59 smithi130 bash[51302]: debug 2023-08-28T00:46:59.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:59.865695+0000) 2023-08-28T00:47:00.968 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:47:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:47:00] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:47:00.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:00 smithi006 bash[33918]: cluster 2023-08-28T00:46:59.091190+0000 mgr.y (mgr.14901) 5052 : cluster [DBG] pgmap v5059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:00.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:00 smithi006 bash[27292]: cluster 2023-08-28T00:46:59.091190+0000 mgr.y (mgr.14901) 5052 : cluster [DBG] pgmap v5059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:00.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:47:00.829Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:00.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:47:00.831Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:01.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:00 smithi130 bash[27579]: cluster 2023-08-28T00:46:59.091190+0000 mgr.y (mgr.14901) 5052 : cluster [DBG] pgmap v5059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:01.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:00 smithi130 bash[51302]: debug 2023-08-28T00:47:00.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:00.865971+0000) 2023-08-28T00:47:02.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:01 smithi130 bash[51302]: debug 2023-08-28T00:47:01.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:01.866244+0000) 2023-08-28T00:47:02.695 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:47:02.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:02.695 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:47:02.409Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:02.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:02 smithi006 bash[33918]: cluster 2023-08-28T00:47:01.091810+0000 mgr.y (mgr.14901) 5053 : cluster [DBG] pgmap v5060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:02.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:02 smithi006 bash[33918]: audit 2023-08-28T00:47:01.891233+0000 mon.a (mon.0) 3557 : audit [DBG] from='client.? 172.21.15.6:0/4223622611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:02.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:02 smithi006 bash[27292]: cluster 2023-08-28T00:47:01.091810+0000 mgr.y (mgr.14901) 5053 : cluster [DBG] pgmap v5060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:02.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:02 smithi006 bash[27292]: audit 2023-08-28T00:47:01.891233+0000 mon.a (mon.0) 3557 : audit [DBG] from='client.? 172.21.15.6:0/4223622611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:02 smithi130 bash[51302]: debug 2023-08-28T00:47:02.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:02.866531+0000) 2023-08-28T00:47:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:02 smithi130 bash[27579]: cluster 2023-08-28T00:47:01.091810+0000 mgr.y (mgr.14901) 5053 : cluster [DBG] pgmap v5060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:03.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:02 smithi130 bash[27579]: audit 2023-08-28T00:47:01.891233+0000 mon.a (mon.0) 3557 : audit [DBG] from='client.? 172.21.15.6:0/4223622611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:03 smithi130 bash[51302]: debug 2023-08-28T00:47:03.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:03.866806+0000) 2023-08-28T00:47:04.700 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:47:04.289Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:04.700 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:47:04.290Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:04.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:04 smithi006 bash[27292]: cluster 2023-08-28T00:47:03.092414+0000 mgr.y (mgr.14901) 5054 : cluster [DBG] pgmap v5061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:04.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:04 smithi006 bash[27292]: audit 2023-08-28T00:47:04.311979+0000 mon.a (mon.0) 3558 : audit [DBG] from='client.? 172.21.15.6:0/3268603556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:04.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:04 smithi006 bash[33918]: cluster 2023-08-28T00:47:03.092414+0000 mgr.y (mgr.14901) 5054 : cluster [DBG] pgmap v5061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:04.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:04 smithi006 bash[33918]: audit 2023-08-28T00:47:04.311979+0000 mon.a (mon.0) 3558 : audit [DBG] from='client.? 172.21.15.6:0/3268603556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:05.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:04 smithi130 bash[51302]: debug 2023-08-28T00:47:04.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:04.867130+0000) 2023-08-28T00:47:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:04 smithi130 bash[27579]: cluster 2023-08-28T00:47:03.092414+0000 mgr.y (mgr.14901) 5054 : cluster [DBG] pgmap v5061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:05.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:04 smithi130 bash[27579]: audit 2023-08-28T00:47:04.311979+0000 mon.a (mon.0) 3558 : audit [DBG] from='client.? 172.21.15.6:0/3268603556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:06.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:05 smithi130 bash[51302]: debug 2023-08-28T00:47:05.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:05.867425+0000) 2023-08-28T00:47:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:06 smithi006 bash[27292]: cluster 2023-08-28T00:47:05.093587+0000 mgr.y (mgr.14901) 5055 : cluster [DBG] pgmap v5062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:06.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:06 smithi006 bash[33918]: cluster 2023-08-28T00:47:05.093587+0000 mgr.y (mgr.14901) 5055 : cluster [DBG] pgmap v5062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:07.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:06 smithi130 bash[27579]: cluster 2023-08-28T00:47:05.093587+0000 mgr.y (mgr.14901) 5055 : cluster [DBG] pgmap v5062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:07.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:06 smithi130 bash[51302]: debug 2023-08-28T00:47:06.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:06.867717+0000) 2023-08-28T00:47:07.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:07 smithi006 bash[33918]: audit 2023-08-28T00:47:06.726958+0000 mon.c (mon.2) 1962 : audit [DBG] from='client.? 172.21.15.6:0/3990926829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:07.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:07 smithi006 bash[27292]: audit 2023-08-28T00:47:06.726958+0000 mon.c (mon.2) 1962 : audit [DBG] from='client.? 172.21.15.6:0/3990926829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:08.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:07 smithi130 bash[27579]: audit 2023-08-28T00:47:06.726958+0000 mon.c (mon.2) 1962 : audit [DBG] from='client.? 172.21.15.6:0/3990926829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:08.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:07 smithi130 bash[51302]: debug 2023-08-28T00:47:07.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:07.868009+0000) 2023-08-28T00:47:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:08 smithi006 bash[33918]: cluster 2023-08-28T00:47:07.094304+0000 mgr.y (mgr.14901) 5056 : cluster [DBG] pgmap v5063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:08.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:08 smithi006 bash[27292]: cluster 2023-08-28T00:47:07.094304+0000 mgr.y (mgr.14901) 5056 : cluster [DBG] pgmap v5063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:09.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:08 smithi130 bash[51302]: debug 2023-08-28T00:47:08.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:08.868274+0000) 2023-08-28T00:47:09.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:08 smithi130 bash[27579]: cluster 2023-08-28T00:47:07.094304+0000 mgr.y (mgr.14901) 5056 : cluster [DBG] pgmap v5063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:10.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:09 smithi130 bash[51302]: debug 2023-08-28T00:47:09.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:09.868561+0000) 2023-08-28T00:47:10.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:09 smithi130 bash[27579]: cluster 2023-08-28T00:47:09.095380+0000 mgr.y (mgr.14901) 5057 : cluster [DBG] pgmap v5064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:10.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:09 smithi130 bash[27579]: audit 2023-08-28T00:47:09.151324+0000 mon.c (mon.2) 1963 : audit [DBG] from='client.? 172.21.15.6:0/433846583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:10.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:09 smithi006 bash[27292]: cluster 2023-08-28T00:47:09.095380+0000 mgr.y (mgr.14901) 5057 : cluster [DBG] pgmap v5064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:10.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:09 smithi006 bash[27292]: audit 2023-08-28T00:47:09.151324+0000 mon.c (mon.2) 1963 : audit [DBG] from='client.? 172.21.15.6:0/433846583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:10.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:09 smithi006 bash[33918]: cluster 2023-08-28T00:47:09.095380+0000 mgr.y (mgr.14901) 5057 : cluster [DBG] pgmap v5064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:10.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:09 smithi006 bash[33918]: audit 2023-08-28T00:47:09.151324+0000 mon.c (mon.2) 1963 : audit [DBG] from='client.? 172.21.15.6:0/433846583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:47:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:47:10] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:47:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:47:10.830Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:47:10.834Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:11.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:10 smithi130 bash[51302]: debug 2023-08-28T00:47:10.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:10.868871+0000) 2023-08-28T00:47:12.174 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:11 smithi130 bash[51302]: debug 2023-08-28T00:47:11.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:11.869154+0000) 2023-08-28T00:47:12.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:12 smithi006 bash[33918]: cluster 2023-08-28T00:47:11.096046+0000 mgr.y (mgr.14901) 5058 : cluster [DBG] pgmap v5065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:12.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:12 smithi006 bash[33918]: audit 2023-08-28T00:47:11.567724+0000 mon.a (mon.0) 3559 : audit [DBG] from='client.? 172.21.15.6:0/3449237937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:12.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:47:12.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:12.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:47:12.408Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:12.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:12 smithi006 bash[27292]: cluster 2023-08-28T00:47:11.096046+0000 mgr.y (mgr.14901) 5058 : cluster [DBG] pgmap v5065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:12.469 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:12 smithi006 bash[27292]: audit 2023-08-28T00:47:11.567724+0000 mon.a (mon.0) 3559 : audit [DBG] from='client.? 172.21.15.6:0/3449237937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:12 smithi130 bash[27579]: cluster 2023-08-28T00:47:11.096046+0000 mgr.y (mgr.14901) 5058 : cluster [DBG] pgmap v5065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:12 smithi130 bash[27579]: audit 2023-08-28T00:47:11.567724+0000 mon.a (mon.0) 3559 : audit [DBG] from='client.? 172.21.15.6:0/3449237937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:12 smithi130 bash[51302]: debug 2023-08-28T00:47:12.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:12.869441+0000) 2023-08-28T00:47:14.182 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:13 smithi130 bash[51302]: debug 2023-08-28T00:47:13.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:13.869714+0000) 2023-08-28T00:47:14.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:14 smithi006 bash[27292]: cluster 2023-08-28T00:47:13.096826+0000 mgr.y (mgr.14901) 5059 : cluster [DBG] pgmap v5066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:14.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:14 smithi006 bash[27292]: audit 2023-08-28T00:47:13.979816+0000 mon.c (mon.2) 1964 : audit [DBG] from='client.? 172.21.15.6:0/2076245713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:14.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:14 smithi006 bash[33918]: cluster 2023-08-28T00:47:13.096826+0000 mgr.y (mgr.14901) 5059 : cluster [DBG] pgmap v5066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:14.469 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:14 smithi006 bash[33918]: audit 2023-08-28T00:47:13.979816+0000 mon.c (mon.2) 1964 : audit [DBG] from='client.? 172.21.15.6:0/2076245713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:14.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:47:14.289Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:14.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:47:14.291Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:14 smithi130 bash[27579]: cluster 2023-08-28T00:47:13.096826+0000 mgr.y (mgr.14901) 5059 : cluster [DBG] pgmap v5066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:14 smithi130 bash[27579]: audit 2023-08-28T00:47:13.979816+0000 mon.c (mon.2) 1964 : audit [DBG] from='client.? 172.21.15.6:0/2076245713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:14 smithi130 bash[51302]: debug 2023-08-28T00:47:14.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:14.870046+0000) 2023-08-28T00:47:16.186 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:15 smithi130 bash[51302]: debug 2023-08-28T00:47:15.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:15.870374+0000) 2023-08-28T00:47:16.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:16 smithi006 bash[27292]: cluster 2023-08-28T00:47:15.098078+0000 mgr.y (mgr.14901) 5060 : cluster [DBG] pgmap v5067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:16.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:16 smithi006 bash[33918]: cluster 2023-08-28T00:47:15.098078+0000 mgr.y (mgr.14901) 5060 : cluster [DBG] pgmap v5067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:16 smithi130 bash[27579]: cluster 2023-08-28T00:47:15.098078+0000 mgr.y (mgr.14901) 5060 : cluster [DBG] pgmap v5067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:17.194 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:16 smithi130 bash[51302]: debug 2023-08-28T00:47:16.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:16.870707+0000) 2023-08-28T00:47:17.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:17 smithi006 bash[33918]: audit 2023-08-28T00:47:16.396341+0000 mon.c (mon.2) 1965 : audit [DBG] from='client.? 172.21.15.6:0/3032438452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:17.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:17 smithi006 bash[27292]: audit 2023-08-28T00:47:16.396341+0000 mon.c (mon.2) 1965 : audit [DBG] from='client.? 172.21.15.6:0/3032438452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:17.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:17 smithi130 bash[27579]: audit 2023-08-28T00:47:16.396341+0000 mon.c (mon.2) 1965 : audit [DBG] from='client.? 172.21.15.6:0/3032438452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:18.197 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:17 smithi130 bash[51302]: debug 2023-08-28T00:47:17.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:17.870985+0000) 2023-08-28T00:47:18.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:18 smithi006 bash[27292]: cluster 2023-08-28T00:47:17.098810+0000 mgr.y (mgr.14901) 5061 : cluster [DBG] pgmap v5068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:18.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:18 smithi006 bash[33918]: cluster 2023-08-28T00:47:17.098810+0000 mgr.y (mgr.14901) 5061 : cluster [DBG] pgmap v5068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:18 smithi130 bash[27579]: cluster 2023-08-28T00:47:17.098810+0000 mgr.y (mgr.14901) 5061 : cluster [DBG] pgmap v5068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:19.206 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:18 smithi130 bash[51302]: debug 2023-08-28T00:47:18.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:18.871211+0000) 2023-08-28T00:47:19.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:19 smithi006 bash[27292]: audit 2023-08-28T00:47:18.814006+0000 mon.a (mon.0) 3560 : audit [DBG] from='client.? 172.21.15.6:0/3546103107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:19.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:19 smithi006 bash[33918]: audit 2023-08-28T00:47:18.814006+0000 mon.a (mon.0) 3560 : audit [DBG] from='client.? 172.21.15.6:0/3546103107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:19 smithi130 bash[27579]: audit 2023-08-28T00:47:18.814006+0000 mon.a (mon.0) 3560 : audit [DBG] from='client.? 172.21.15.6:0/3546103107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:20.207 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:19 smithi130 bash[51302]: debug 2023-08-28T00:47:19.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:19.871442+0000) 2023-08-28T00:47:20.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:20 smithi006 bash[27292]: cluster 2023-08-28T00:47:19.100012+0000 mgr.y (mgr.14901) 5062 : cluster [DBG] pgmap v5069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:20.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:20 smithi006 bash[33918]: cluster 2023-08-28T00:47:19.100012+0000 mgr.y (mgr.14901) 5062 : cluster [DBG] pgmap v5069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:20 smithi130 bash[27579]: cluster 2023-08-28T00:47:19.100012+0000 mgr.y (mgr.14901) 5062 : cluster [DBG] pgmap v5069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:47:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:47:20] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:47:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:47:20.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:47:20.833Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:21.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:20 smithi130 bash[51302]: debug 2023-08-28T00:47:20.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:20.871751+0000) 2023-08-28T00:47:22.212 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:21 smithi130 bash[51302]: debug 2023-08-28T00:47:21.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:21.872054+0000) 2023-08-28T00:47:22.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:22 smithi006 bash[33918]: cluster 2023-08-28T00:47:21.100816+0000 mgr.y (mgr.14901) 5063 : cluster [DBG] pgmap v5070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:22.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:22 smithi006 bash[33918]: audit 2023-08-28T00:47:21.233041+0000 mon.a (mon.0) 3561 : audit [DBG] from='client.? 172.21.15.6:0/169323729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:22.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:47:22.408Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:22.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:47:22.410Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:22.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:22 smithi006 bash[27292]: cluster 2023-08-28T00:47:21.100816+0000 mgr.y (mgr.14901) 5063 : cluster [DBG] pgmap v5070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:22.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:22 smithi006 bash[27292]: audit 2023-08-28T00:47:21.233041+0000 mon.a (mon.0) 3561 : audit [DBG] from='client.? 172.21.15.6:0/169323729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:22 smithi130 bash[27579]: cluster 2023-08-28T00:47:21.100816+0000 mgr.y (mgr.14901) 5063 : cluster [DBG] pgmap v5070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:22 smithi130 bash[27579]: audit 2023-08-28T00:47:21.233041+0000 mon.a (mon.0) 3561 : audit [DBG] from='client.? 172.21.15.6:0/169323729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:23.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:22 smithi130 bash[51302]: debug 2023-08-28T00:47:22.872+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:22.872332+0000) 2023-08-28T00:47:24.216 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:23 smithi130 bash[51302]: debug 2023-08-28T00:47:23.872+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:23.872617+0000) 2023-08-28T00:47:24.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:24 smithi006 bash[33918]: cluster 2023-08-28T00:47:23.101620+0000 mgr.y (mgr.14901) 5064 : cluster [DBG] pgmap v5071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:24.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:24 smithi006 bash[33918]: audit 2023-08-28T00:47:23.646938+0000 mon.c (mon.2) 1966 : audit [DBG] from='client.? 172.21.15.6:0/1815200067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:24.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:24 smithi006 bash[27292]: cluster 2023-08-28T00:47:23.101620+0000 mgr.y (mgr.14901) 5064 : cluster [DBG] pgmap v5071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:24.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:24 smithi006 bash[27292]: audit 2023-08-28T00:47:23.646938+0000 mon.c (mon.2) 1966 : audit [DBG] from='client.? 172.21.15.6:0/1815200067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:24.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:47:24.290Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:24.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:47:24.292Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:24 smithi130 bash[27579]: cluster 2023-08-28T00:47:23.101620+0000 mgr.y (mgr.14901) 5064 : cluster [DBG] pgmap v5071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:24 smithi130 bash[27579]: audit 2023-08-28T00:47:23.646938+0000 mon.c (mon.2) 1966 : audit [DBG] from='client.? 172.21.15.6:0/1815200067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:24 smithi130 bash[51302]: debug 2023-08-28T00:47:24.872+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:24.872892+0000) 2023-08-28T00:47:26.222 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:25 smithi130 bash[51302]: debug 2023-08-28T00:47:25.867+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:25.873180+0000) 2023-08-28T00:47:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:26 smithi130 bash[27579]: cluster 2023-08-28T00:47:25.102798+0000 mgr.y (mgr.14901) 5065 : cluster [DBG] pgmap v5072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:26 smithi130 bash[27579]: audit 2023-08-28T00:47:26.057883+0000 mon.a (mon.0) 3562 : audit [DBG] from='client.? 172.21.15.6:0/356691390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:26 smithi006 bash[33918]: cluster 2023-08-28T00:47:25.102798+0000 mgr.y (mgr.14901) 5065 : cluster [DBG] pgmap v5072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:26 smithi006 bash[33918]: audit 2023-08-28T00:47:26.057883+0000 mon.a (mon.0) 3562 : audit [DBG] from='client.? 172.21.15.6:0/356691390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:26 smithi006 bash[27292]: cluster 2023-08-28T00:47:25.102798+0000 mgr.y (mgr.14901) 5065 : cluster [DBG] pgmap v5072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:26 smithi006 bash[27292]: audit 2023-08-28T00:47:26.057883+0000 mon.a (mon.0) 3562 : audit [DBG] from='client.? 172.21.15.6:0/356691390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:27.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:26 smithi130 bash[51302]: debug 2023-08-28T00:47:26.867+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:26.873457+0000) 2023-08-28T00:47:28.226 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:27 smithi130 bash[51302]: debug 2023-08-28T00:47:27.871+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:27.873758+0000) 2023-08-28T00:47:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:28 smithi130 bash[27579]: cluster 2023-08-28T00:47:27.103410+0000 mgr.y (mgr.14901) 5066 : cluster [DBG] pgmap v5073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:28.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:28 smithi006 bash[33918]: cluster 2023-08-28T00:47:27.103410+0000 mgr.y (mgr.14901) 5066 : cluster [DBG] pgmap v5073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:28.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:28 smithi006 bash[27292]: cluster 2023-08-28T00:47:27.103410+0000 mgr.y (mgr.14901) 5066 : cluster [DBG] pgmap v5073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:29.233 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:28 smithi130 bash[51302]: debug 2023-08-28T00:47:28.867+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:28.874077+0000) 2023-08-28T00:47:29.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:29 smithi130 bash[27579]: audit 2023-08-28T00:47:28.472038+0000 mon.c (mon.2) 1967 : audit [DBG] from='client.? 172.21.15.6:0/3242233579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:29.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:29 smithi006 bash[33918]: audit 2023-08-28T00:47:28.472038+0000 mon.c (mon.2) 1967 : audit [DBG] from='client.? 172.21.15.6:0/3242233579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:29.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:29 smithi006 bash[27292]: audit 2023-08-28T00:47:28.472038+0000 mon.c (mon.2) 1967 : audit [DBG] from='client.? 172.21.15.6:0/3242233579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:30.236 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:29 smithi130 bash[51302]: debug 2023-08-28T00:47:29.871+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:29.874456+0000) 2023-08-28T00:47:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:30 smithi130 bash[27579]: cluster 2023-08-28T00:47:29.104441+0000 mgr.y (mgr.14901) 5067 : cluster [DBG] pgmap v5074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:30.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:30 smithi006 bash[33918]: cluster 2023-08-28T00:47:29.104441+0000 mgr.y (mgr.14901) 5067 : cluster [DBG] pgmap v5074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:30.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:30 smithi006 bash[27292]: cluster 2023-08-28T00:47:29.104441+0000 mgr.y (mgr.14901) 5067 : cluster [DBG] pgmap v5074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:31.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:47:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:47:30] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:47:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:47:30.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:47:30.834Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:31.238 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:30 smithi130 bash[51302]: debug 2023-08-28T00:47:30.871+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:30.874769+0000) 2023-08-28T00:47:31.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:31 smithi130 bash[27579]: audit 2023-08-28T00:47:30.891400+0000 mon.a (mon.0) 3563 : audit [DBG] from='client.? 172.21.15.6:0/518185146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:31.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:31 smithi006 bash[33918]: audit 2023-08-28T00:47:30.891400+0000 mon.a (mon.0) 3563 : audit [DBG] from='client.? 172.21.15.6:0/518185146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:31.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:31 smithi006 bash[27292]: audit 2023-08-28T00:47:30.891400+0000 mon.a (mon.0) 3563 : audit [DBG] from='client.? 172.21.15.6:0/518185146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:32.248 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:31 smithi130 bash[51302]: debug 2023-08-28T00:47:31.871+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:31.875116+0000) 2023-08-28T00:47:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:32 smithi130 bash[27579]: cluster 2023-08-28T00:47:31.105156+0000 mgr.y (mgr.14901) 5068 : cluster [DBG] pgmap v5075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:32 smithi006 bash[27292]: cluster 2023-08-28T00:47:31.105156+0000 mgr.y (mgr.14901) 5068 : cluster [DBG] pgmap v5075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:32 smithi006 bash[33918]: cluster 2023-08-28T00:47:31.105156+0000 mgr.y (mgr.14901) 5068 : cluster [DBG] pgmap v5075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:47:32.409Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:47:32.411Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:33.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:32 smithi130 bash[51302]: debug 2023-08-28T00:47:32.871+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:32.875397+0000) 2023-08-28T00:47:34.252 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:33 smithi130 bash[51302]: debug 2023-08-28T00:47:33.871+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:33.875692+0000) 2023-08-28T00:47:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:34 smithi130 bash[27579]: cluster 2023-08-28T00:47:33.105806+0000 mgr.y (mgr.14901) 5069 : cluster [DBG] pgmap v5076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:34 smithi130 bash[27579]: audit 2023-08-28T00:47:33.310648+0000 mon.a (mon.0) 3564 : audit [DBG] from='client.? 172.21.15.6:0/1462628008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:34 smithi006 bash[33918]: cluster 2023-08-28T00:47:33.105806+0000 mgr.y (mgr.14901) 5069 : cluster [DBG] pgmap v5076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:34 smithi006 bash[33918]: audit 2023-08-28T00:47:33.310648+0000 mon.a (mon.0) 3564 : audit [DBG] from='client.? 172.21.15.6:0/1462628008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:47:34.290Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:47:34.292Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:34 smithi006 bash[27292]: cluster 2023-08-28T00:47:33.105806+0000 mgr.y (mgr.14901) 5069 : cluster [DBG] pgmap v5076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:34 smithi006 bash[27292]: audit 2023-08-28T00:47:33.310648+0000 mon.a (mon.0) 3564 : audit [DBG] from='client.? 172.21.15.6:0/1462628008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:35.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:34 smithi130 bash[51302]: debug 2023-08-28T00:47:34.871+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:34.875965+0000) 2023-08-28T00:47:36.257 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:35 smithi130 bash[51302]: debug 2023-08-28T00:47:35.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:35.876268+0000) 2023-08-28T00:47:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:36 smithi130 bash[27579]: cluster 2023-08-28T00:47:35.106968+0000 mgr.y (mgr.14901) 5070 : cluster [DBG] pgmap v5077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:36 smithi130 bash[27579]: audit 2023-08-28T00:47:35.722479+0000 mon.c (mon.2) 1968 : audit [DBG] from='client.? 172.21.15.6:0/1018327727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:36.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:36 smithi006 bash[33918]: cluster 2023-08-28T00:47:35.106968+0000 mgr.y (mgr.14901) 5070 : cluster [DBG] pgmap v5077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:36.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:36 smithi006 bash[33918]: audit 2023-08-28T00:47:35.722479+0000 mon.c (mon.2) 1968 : audit [DBG] from='client.? 172.21.15.6:0/1018327727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:36.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:36 smithi006 bash[27292]: cluster 2023-08-28T00:47:35.106968+0000 mgr.y (mgr.14901) 5070 : cluster [DBG] pgmap v5077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:36.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:36 smithi006 bash[27292]: audit 2023-08-28T00:47:35.722479+0000 mon.c (mon.2) 1968 : audit [DBG] from='client.? 172.21.15.6:0/1018327727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:36 smithi130 bash[51302]: debug 2023-08-28T00:47:36.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:36.876556+0000) 2023-08-28T00:47:38.261 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:37 smithi130 bash[51302]: debug 2023-08-28T00:47:37.871+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:37.876830+0000) 2023-08-28T00:47:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:38 smithi130 bash[27579]: cluster 2023-08-28T00:47:37.107609+0000 mgr.y (mgr.14901) 5071 : cluster [DBG] pgmap v5078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:38 smithi130 bash[27579]: audit 2023-08-28T00:47:38.134401+0000 mon.a (mon.0) 3565 : audit [DBG] from='client.? 172.21.15.6:0/753782023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:38.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:38 smithi006 bash[33918]: cluster 2023-08-28T00:47:37.107609+0000 mgr.y (mgr.14901) 5071 : cluster [DBG] pgmap v5078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:38.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:38 smithi006 bash[33918]: audit 2023-08-28T00:47:38.134401+0000 mon.a (mon.0) 3565 : audit [DBG] from='client.? 172.21.15.6:0/753782023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:38.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:38 smithi006 bash[27292]: cluster 2023-08-28T00:47:37.107609+0000 mgr.y (mgr.14901) 5071 : cluster [DBG] pgmap v5078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:38.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:38 smithi006 bash[27292]: audit 2023-08-28T00:47:38.134401+0000 mon.a (mon.0) 3565 : audit [DBG] from='client.? 172.21.15.6:0/753782023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:39.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:38 smithi130 bash[51302]: debug 2023-08-28T00:47:38.871+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:38.877157+0000) 2023-08-28T00:47:40.266 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:39 smithi130 bash[51302]: debug 2023-08-28T00:47:39.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:39.877455+0000) 2023-08-28T00:47:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:40 smithi130 bash[27579]: cluster 2023-08-28T00:47:39.108805+0000 mgr.y (mgr.14901) 5072 : cluster [DBG] pgmap v5079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:40 smithi006 bash[33918]: cluster 2023-08-28T00:47:39.108805+0000 mgr.y (mgr.14901) 5072 : cluster [DBG] pgmap v5079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:40 smithi006 bash[27292]: cluster 2023-08-28T00:47:39.108805+0000 mgr.y (mgr.14901) 5072 : cluster [DBG] pgmap v5079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:41.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:47:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:47:40] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:47:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:47:40.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:47:40.835Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:41.268 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:40 smithi130 bash[51302]: debug 2023-08-28T00:47:40.871+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:40.877719+0000) 2023-08-28T00:47:41.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:41 smithi130 bash[27579]: audit 2023-08-28T00:47:40.553313+0000 mon.a (mon.0) 3566 : audit [DBG] from='client.? 172.21.15.6:0/500583026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:41.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:41 smithi006 bash[27292]: audit 2023-08-28T00:47:40.553313+0000 mon.a (mon.0) 3566 : audit [DBG] from='client.? 172.21.15.6:0/500583026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:41.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:41 smithi006 bash[33918]: audit 2023-08-28T00:47:40.553313+0000 mon.a (mon.0) 3566 : audit [DBG] from='client.? 172.21.15.6:0/500583026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:42.280 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:41 smithi130 bash[51302]: debug 2023-08-28T00:47:41.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:41.877997+0000) 2023-08-28T00:47:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:42 smithi130 bash[27579]: cluster 2023-08-28T00:47:41.109584+0000 mgr.y (mgr.14901) 5073 : cluster [DBG] pgmap v5080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:42.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:42 smithi006 bash[33918]: cluster 2023-08-28T00:47:41.109584+0000 mgr.y (mgr.14901) 5073 : cluster [DBG] pgmap v5080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:42.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:42 smithi006 bash[27292]: cluster 2023-08-28T00:47:41.109584+0000 mgr.y (mgr.14901) 5073 : cluster [DBG] pgmap v5080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:47:42.409Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:47:42.411Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:43.285 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:42 smithi130 bash[51302]: debug 2023-08-28T00:47:42.871+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:42.878320+0000) 2023-08-28T00:47:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:43 smithi130 bash[27579]: audit 2023-08-28T00:47:42.976627+0000 mon.a (mon.0) 3567 : audit [DBG] from='client.? 172.21.15.6:0/4237639707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:43.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:43 smithi006 bash[33918]: audit 2023-08-28T00:47:42.976627+0000 mon.a (mon.0) 3567 : audit [DBG] from='client.? 172.21.15.6:0/4237639707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:43.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:43 smithi006 bash[27292]: audit 2023-08-28T00:47:42.976627+0000 mon.a (mon.0) 3567 : audit [DBG] from='client.? 172.21.15.6:0/4237639707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:44.296 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:43 smithi130 bash[51302]: debug 2023-08-28T00:47:43.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:43.878588+0000) 2023-08-28T00:47:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:44 smithi130 bash[27579]: cluster 2023-08-28T00:47:43.110266+0000 mgr.y (mgr.14901) 5074 : cluster [DBG] pgmap v5081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:44.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:44 smithi006 bash[33918]: cluster 2023-08-28T00:47:43.110266+0000 mgr.y (mgr.14901) 5074 : cluster [DBG] pgmap v5081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:44.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:44 smithi006 bash[27292]: cluster 2023-08-28T00:47:43.110266+0000 mgr.y (mgr.14901) 5074 : cluster [DBG] pgmap v5081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:47:44.291Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:47:44.293Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:44 smithi130 bash[51302]: debug 2023-08-28T00:47:44.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:44.878890+0000) 2023-08-28T00:47:46.301 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:45 smithi130 bash[51302]: debug 2023-08-28T00:47:45.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:45.879182+0000) 2023-08-28T00:47:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:46 smithi130 bash[27579]: cluster 2023-08-28T00:47:45.111350+0000 mgr.y (mgr.14901) 5075 : cluster [DBG] pgmap v5082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:46 smithi130 bash[27579]: audit 2023-08-28T00:47:45.395340+0000 mon.c (mon.2) 1969 : audit [DBG] from='client.? 172.21.15.6:0/1688725104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:46.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:46 smithi006 bash[33918]: cluster 2023-08-28T00:47:45.111350+0000 mgr.y (mgr.14901) 5075 : cluster [DBG] pgmap v5082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:46.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:46 smithi006 bash[33918]: audit 2023-08-28T00:47:45.395340+0000 mon.c (mon.2) 1969 : audit [DBG] from='client.? 172.21.15.6:0/1688725104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:46 smithi006 bash[27292]: cluster 2023-08-28T00:47:45.111350+0000 mgr.y (mgr.14901) 5075 : cluster [DBG] pgmap v5082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:46 smithi006 bash[27292]: audit 2023-08-28T00:47:45.395340+0000 mon.c (mon.2) 1969 : audit [DBG] from='client.? 172.21.15.6:0/1688725104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:47.309 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:46 smithi130 bash[51302]: debug 2023-08-28T00:47:46.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:46.879513+0000) 2023-08-28T00:47:47.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:47 smithi130 bash[27579]: audit 2023-08-28T00:47:46.432696+0000 mon.b (mon.1) 607 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:47:47.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:47 smithi006 bash[33918]: audit 2023-08-28T00:47:46.432696+0000 mon.b (mon.1) 607 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:47:47.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:47 smithi006 bash[27292]: audit 2023-08-28T00:47:46.432696+0000 mon.b (mon.1) 607 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:47:48.313 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:47 smithi130 bash[51302]: debug 2023-08-28T00:47:47.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:47.879744+0000) 2023-08-28T00:47:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:48 smithi130 bash[27579]: cluster 2023-08-28T00:47:47.112029+0000 mgr.y (mgr.14901) 5076 : cluster [DBG] pgmap v5083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:48 smithi130 bash[27579]: audit 2023-08-28T00:47:47.813251+0000 mon.a (mon.0) 3568 : audit [DBG] from='client.? 172.21.15.6:0/3848438820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:48 smithi006 bash[27292]: cluster 2023-08-28T00:47:47.112029+0000 mgr.y (mgr.14901) 5076 : cluster [DBG] pgmap v5083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:48.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:48 smithi006 bash[27292]: audit 2023-08-28T00:47:47.813251+0000 mon.a (mon.0) 3568 : audit [DBG] from='client.? 172.21.15.6:0/3848438820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:48 smithi006 bash[33918]: cluster 2023-08-28T00:47:47.112029+0000 mgr.y (mgr.14901) 5076 : cluster [DBG] pgmap v5083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:48 smithi006 bash[33918]: audit 2023-08-28T00:47:47.813251+0000 mon.a (mon.0) 3568 : audit [DBG] from='client.? 172.21.15.6:0/3848438820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:48 smithi130 bash[51302]: debug 2023-08-28T00:47:48.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:48.880026+0000) 2023-08-28T00:47:50.316 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:49 smithi130 bash[51302]: debug 2023-08-28T00:47:49.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:49.880339+0000) 2023-08-28T00:47:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:50 smithi130 bash[27579]: cluster 2023-08-28T00:47:49.113113+0000 mgr.y (mgr.14901) 5077 : cluster [DBG] pgmap v5084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:50 smithi130 bash[27579]: audit 2023-08-28T00:47:50.229317+0000 mon.a (mon.0) 3569 : audit [DBG] from='client.? 172.21.15.6:0/936298235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:50 smithi006 bash[33918]: cluster 2023-08-28T00:47:49.113113+0000 mgr.y (mgr.14901) 5077 : cluster [DBG] pgmap v5084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:50 smithi006 bash[33918]: audit 2023-08-28T00:47:50.229317+0000 mon.a (mon.0) 3569 : audit [DBG] from='client.? 172.21.15.6:0/936298235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:50 smithi006 bash[27292]: cluster 2023-08-28T00:47:49.113113+0000 mgr.y (mgr.14901) 5077 : cluster [DBG] pgmap v5084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:50 smithi006 bash[27292]: audit 2023-08-28T00:47:50.229317+0000 mon.a (mon.0) 3569 : audit [DBG] from='client.? 172.21.15.6:0/936298235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:51.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:47:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:47:50] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:47:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:47:50.833Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:47:50.835Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:51.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:50 smithi130 bash[51302]: debug 2023-08-28T00:47:50.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:50.880615+0000) 2023-08-28T00:47:52.323 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:51 smithi130 bash[51302]: debug 2023-08-28T00:47:51.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:51.880843+0000) 2023-08-28T00:47:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:52 smithi130 bash[27579]: cluster 2023-08-28T00:47:51.113744+0000 mgr.y (mgr.14901) 5078 : cluster [DBG] pgmap v5085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:52.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:52 smithi006 bash[27292]: cluster 2023-08-28T00:47:51.113744+0000 mgr.y (mgr.14901) 5078 : cluster [DBG] pgmap v5085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:52 smithi006 bash[33918]: cluster 2023-08-28T00:47:51.113744+0000 mgr.y (mgr.14901) 5078 : cluster [DBG] pgmap v5085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:47:52.410Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:47:52.412Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:53.353 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:52 smithi130 bash[51302]: debug 2023-08-28T00:47:52.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:52.881102+0000) 2023-08-28T00:47:53.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:53 smithi130 bash[27579]: audit 2023-08-28T00:47:52.349005+0000 mon.a (mon.0) 3570 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:47:53.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:53 smithi130 bash[27579]: audit 2023-08-28T00:47:52.647659+0000 mon.a (mon.0) 3571 : audit [DBG] from='client.? 172.21.15.6:0/3601408749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:53.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:53 smithi130 bash[27579]: audit 2023-08-28T00:47:52.741107+0000 mon.a (mon.0) 3572 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:47:53.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:53 smithi130 bash[27579]: audit 2023-08-28T00:47:53.009620+0000 mon.b (mon.1) 608 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:47:53.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:53 smithi130 bash[27579]: audit 2023-08-28T00:47:53.011882+0000 mon.b (mon.1) 609 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:47:53.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:53 smithi130 bash[27579]: audit 2023-08-28T00:47:53.020315+0000 mon.a (mon.0) 3573 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:47:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:53 smithi006 bash[33918]: audit 2023-08-28T00:47:52.349005+0000 mon.a (mon.0) 3570 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:47:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:53 smithi006 bash[33918]: audit 2023-08-28T00:47:52.647659+0000 mon.a (mon.0) 3571 : audit [DBG] from='client.? 172.21.15.6:0/3601408749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:53 smithi006 bash[33918]: audit 2023-08-28T00:47:52.741107+0000 mon.a (mon.0) 3572 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:47:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:53 smithi006 bash[33918]: audit 2023-08-28T00:47:53.009620+0000 mon.b (mon.1) 608 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:47:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:53 smithi006 bash[33918]: audit 2023-08-28T00:47:53.011882+0000 mon.b (mon.1) 609 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:47:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:53 smithi006 bash[33918]: audit 2023-08-28T00:47:53.020315+0000 mon.a (mon.0) 3573 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:47:53.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:53 smithi006 bash[27292]: audit 2023-08-28T00:47:52.349005+0000 mon.a (mon.0) 3570 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:47:53.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:53 smithi006 bash[27292]: audit 2023-08-28T00:47:52.647659+0000 mon.a (mon.0) 3571 : audit [DBG] from='client.? 172.21.15.6:0/3601408749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:53.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:53 smithi006 bash[27292]: audit 2023-08-28T00:47:52.741107+0000 mon.a (mon.0) 3572 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:47:53.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:53 smithi006 bash[27292]: audit 2023-08-28T00:47:53.009620+0000 mon.b (mon.1) 608 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:47:53.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:53 smithi006 bash[27292]: audit 2023-08-28T00:47:53.011882+0000 mon.b (mon.1) 609 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:47:53.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:53 smithi006 bash[27292]: audit 2023-08-28T00:47:53.020315+0000 mon.a (mon.0) 3573 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:47:54.357 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:53 smithi130 bash[51302]: debug 2023-08-28T00:47:53.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:53.881371+0000) 2023-08-28T00:47:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:54 smithi130 bash[27579]: cluster 2023-08-28T00:47:53.114422+0000 mgr.y (mgr.14901) 5079 : cluster [DBG] pgmap v5086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:54.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:54 smithi006 bash[33918]: cluster 2023-08-28T00:47:53.114422+0000 mgr.y (mgr.14901) 5079 : cluster [DBG] pgmap v5086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:54.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:54 smithi006 bash[27292]: cluster 2023-08-28T00:47:53.114422+0000 mgr.y (mgr.14901) 5079 : cluster [DBG] pgmap v5086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:54.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:47:54.292Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:54.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:47:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:47:54.294Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:47:55.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:54 smithi130 bash[51302]: debug 2023-08-28T00:47:54.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:54.881636+0000) 2023-08-28T00:47:55.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:55 smithi130 bash[27579]: audit 2023-08-28T00:47:55.060108+0000 mon.a (mon.0) 3574 : audit [DBG] from='client.? 172.21.15.6:0/16585011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:55.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:55 smithi006 bash[27292]: audit 2023-08-28T00:47:55.060108+0000 mon.a (mon.0) 3574 : audit [DBG] from='client.? 172.21.15.6:0/16585011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:55.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:55 smithi006 bash[33918]: audit 2023-08-28T00:47:55.060108+0000 mon.a (mon.0) 3574 : audit [DBG] from='client.? 172.21.15.6:0/16585011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:56.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:55 smithi130 bash[51302]: debug 2023-08-28T00:47:55.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:55.881935+0000) 2023-08-28T00:47:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:56 smithi006 bash[27292]: cluster 2023-08-28T00:47:55.115546+0000 mgr.y (mgr.14901) 5080 : cluster [DBG] pgmap v5087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:56.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:56 smithi006 bash[33918]: cluster 2023-08-28T00:47:55.115546+0000 mgr.y (mgr.14901) 5080 : cluster [DBG] pgmap v5087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:56 smithi130 bash[27579]: cluster 2023-08-28T00:47:55.115546+0000 mgr.y (mgr.14901) 5080 : cluster [DBG] pgmap v5087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:57.652 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:56 smithi130 bash[51302]: debug 2023-08-28T00:47:56.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:56.882214+0000) 2023-08-28T00:47:58.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:57 smithi130 bash[51302]: debug 2023-08-28T00:47:57.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:57.882479+0000) 2023-08-28T00:47:58.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:58 smithi006 bash[33918]: cluster 2023-08-28T00:47:57.116193+0000 mgr.y (mgr.14901) 5081 : cluster [DBG] pgmap v5088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:58.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:47:58 smithi006 bash[33918]: audit 2023-08-28T00:47:57.471945+0000 mon.c (mon.2) 1970 : audit [DBG] from='client.? 172.21.15.6:0/1634221754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:58.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:58 smithi006 bash[27292]: cluster 2023-08-28T00:47:57.116193+0000 mgr.y (mgr.14901) 5081 : cluster [DBG] pgmap v5088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:58.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:47:58 smithi006 bash[27292]: audit 2023-08-28T00:47:57.471945+0000 mon.c (mon.2) 1970 : audit [DBG] from='client.? 172.21.15.6:0/1634221754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:58 smithi130 bash[27579]: cluster 2023-08-28T00:47:57.116193+0000 mgr.y (mgr.14901) 5081 : cluster [DBG] pgmap v5088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:47:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:47:58 smithi130 bash[27579]: audit 2023-08-28T00:47:57.471945+0000 mon.c (mon.2) 1970 : audit [DBG] from='client.? 172.21.15.6:0/1634221754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:58 smithi130 bash[51302]: debug 2023-08-28T00:47:58.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:58.882787+0000) 2023-08-28T00:48:00.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:47:59 smithi130 bash[51302]: debug 2023-08-28T00:47:59.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:59.883083+0000) 2023-08-28T00:48:00.792 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:00 smithi006 bash[33918]: cluster 2023-08-28T00:47:59.117291+0000 mgr.y (mgr.14901) 5082 : cluster [DBG] pgmap v5089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:00.793 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:00 smithi006 bash[33918]: audit 2023-08-28T00:47:59.894165+0000 mon.c (mon.2) 1971 : audit [DBG] from='client.? 172.21.15.6:0/3558762912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:00.793 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:00 smithi006 bash[27292]: cluster 2023-08-28T00:47:59.117291+0000 mgr.y (mgr.14901) 5082 : cluster [DBG] pgmap v5089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:00.793 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:00 smithi006 bash[27292]: audit 2023-08-28T00:47:59.894165+0000 mon.c (mon.2) 1971 : audit [DBG] from='client.? 172.21.15.6:0/3558762912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:00 smithi130 bash[27579]: cluster 2023-08-28T00:47:59.117291+0000 mgr.y (mgr.14901) 5082 : cluster [DBG] pgmap v5089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:00.873 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:00 smithi130 bash[27579]: audit 2023-08-28T00:47:59.894165+0000 mon.c (mon.2) 1971 : audit [DBG] from='client.? 172.21.15.6:0/3558762912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:01.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:48:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:48:00] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:48:01.232 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:48:00.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:01.232 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:48:00.837Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:01.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:00 smithi130 bash[51302]: debug 2023-08-28T00:48:00.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:00.883309+0000) 2023-08-28T00:48:02.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:01 smithi130 bash[51302]: debug 2023-08-28T00:48:01.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:01.883566+0000) 2023-08-28T00:48:02.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:02 smithi006 bash[27292]: cluster 2023-08-28T00:48:01.117922+0000 mgr.y (mgr.14901) 5083 : cluster [DBG] pgmap v5090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:02.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:02 smithi006 bash[27292]: audit 2023-08-28T00:48:02.299309+0000 mon.a (mon.0) 3575 : audit [DBG] from='client.? 172.21.15.6:0/1335721147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:02 smithi006 bash[33918]: cluster 2023-08-28T00:48:01.117922+0000 mgr.y (mgr.14901) 5083 : cluster [DBG] pgmap v5090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:02 smithi006 bash[33918]: audit 2023-08-28T00:48:02.299309+0000 mon.a (mon.0) 3575 : audit [DBG] from='client.? 172.21.15.6:0/1335721147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:48:02.411Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:02.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:48:02.412Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:02 smithi130 bash[27579]: cluster 2023-08-28T00:48:01.117922+0000 mgr.y (mgr.14901) 5083 : cluster [DBG] pgmap v5090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:02 smithi130 bash[27579]: audit 2023-08-28T00:48:02.299309+0000 mon.a (mon.0) 3575 : audit [DBG] from='client.? 172.21.15.6:0/1335721147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:02 smithi130 bash[51302]: debug 2023-08-28T00:48:02.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:02.883792+0000) 2023-08-28T00:48:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:03 smithi130 bash[51302]: debug 2023-08-28T00:48:03.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:03.884083+0000) 2023-08-28T00:48:04.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:04 smithi006 bash[27292]: cluster 2023-08-28T00:48:03.118602+0000 mgr.y (mgr.14901) 5084 : cluster [DBG] pgmap v5091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:04.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:04 smithi006 bash[33918]: cluster 2023-08-28T00:48:03.118602+0000 mgr.y (mgr.14901) 5084 : cluster [DBG] pgmap v5091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:48:04.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:48:04.295Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:04 smithi130 bash[27579]: cluster 2023-08-28T00:48:03.118602+0000 mgr.y (mgr.14901) 5084 : cluster [DBG] pgmap v5091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:04 smithi130 bash[51302]: debug 2023-08-28T00:48:04.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:04.884364+0000) 2023-08-28T00:48:05.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:05 smithi006 bash[27292]: audit 2023-08-28T00:48:04.709232+0000 mon.c (mon.2) 1972 : audit [DBG] from='client.? 172.21.15.6:0/3046285714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:05.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:05 smithi006 bash[33918]: audit 2023-08-28T00:48:04.709232+0000 mon.c (mon.2) 1972 : audit [DBG] from='client.? 172.21.15.6:0/3046285714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:05.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:05 smithi130 bash[27579]: audit 2023-08-28T00:48:04.709232+0000 mon.c (mon.2) 1972 : audit [DBG] from='client.? 172.21.15.6:0/3046285714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:06.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:05 smithi130 bash[51302]: debug 2023-08-28T00:48:05.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:05.884643+0000) 2023-08-28T00:48:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:06 smithi006 bash[27292]: cluster 2023-08-28T00:48:05.119737+0000 mgr.y (mgr.14901) 5085 : cluster [DBG] pgmap v5092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:06 smithi006 bash[33918]: cluster 2023-08-28T00:48:05.119737+0000 mgr.y (mgr.14901) 5085 : cluster [DBG] pgmap v5092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:06 smithi130 bash[27579]: cluster 2023-08-28T00:48:05.119737+0000 mgr.y (mgr.14901) 5085 : cluster [DBG] pgmap v5092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:07.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:06 smithi130 bash[51302]: debug 2023-08-28T00:48:06.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:06.884920+0000) 2023-08-28T00:48:07.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:07 smithi006 bash[27292]: audit 2023-08-28T00:48:07.131122+0000 mon.a (mon.0) 3576 : audit [DBG] from='client.? 172.21.15.6:0/1892616218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:07.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:07 smithi006 bash[33918]: audit 2023-08-28T00:48:07.131122+0000 mon.a (mon.0) 3576 : audit [DBG] from='client.? 172.21.15.6:0/1892616218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:07.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:07 smithi130 bash[27579]: audit 2023-08-28T00:48:07.131122+0000 mon.a (mon.0) 3576 : audit [DBG] from='client.? 172.21.15.6:0/1892616218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:08.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:07 smithi130 bash[51302]: debug 2023-08-28T00:48:07.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:07.885180+0000) 2023-08-28T00:48:08.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:08 smithi006 bash[27292]: cluster 2023-08-28T00:48:07.120356+0000 mgr.y (mgr.14901) 5086 : cluster [DBG] pgmap v5093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:08 smithi006 bash[33918]: cluster 2023-08-28T00:48:07.120356+0000 mgr.y (mgr.14901) 5086 : cluster [DBG] pgmap v5093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:08 smithi130 bash[27579]: cluster 2023-08-28T00:48:07.120356+0000 mgr.y (mgr.14901) 5086 : cluster [DBG] pgmap v5093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:09.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:08 smithi130 bash[51302]: debug 2023-08-28T00:48:08.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:08.885444+0000) 2023-08-28T00:48:10.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:09 smithi130 bash[51302]: debug 2023-08-28T00:48:09.883+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:09.885722+0000) 2023-08-28T00:48:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:10 smithi006 bash[33918]: cluster 2023-08-28T00:48:09.121463+0000 mgr.y (mgr.14901) 5087 : cluster [DBG] pgmap v5094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:10 smithi006 bash[33918]: audit 2023-08-28T00:48:09.544691+0000 mon.a (mon.0) 3577 : audit [DBG] from='client.? 172.21.15.6:0/4040616575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:10 smithi006 bash[27292]: cluster 2023-08-28T00:48:09.121463+0000 mgr.y (mgr.14901) 5087 : cluster [DBG] pgmap v5094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:10 smithi006 bash[27292]: audit 2023-08-28T00:48:09.544691+0000 mon.a (mon.0) 3577 : audit [DBG] from='client.? 172.21.15.6:0/4040616575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:10 smithi130 bash[27579]: cluster 2023-08-28T00:48:09.121463+0000 mgr.y (mgr.14901) 5087 : cluster [DBG] pgmap v5094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:10 smithi130 bash[27579]: audit 2023-08-28T00:48:09.544691+0000 mon.a (mon.0) 3577 : audit [DBG] from='client.? 172.21.15.6:0/4040616575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:11.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:48:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:48:10] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:48:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:48:10.835Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:11.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:48:10.837Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:11.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:10 smithi130 bash[51302]: debug 2023-08-28T00:48:10.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:10.886003+0000) 2023-08-28T00:48:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:11 smithi130 bash[51302]: debug 2023-08-28T00:48:11.883+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:11.886278+0000) 2023-08-28T00:48:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:12 smithi006 bash[27292]: cluster 2023-08-28T00:48:11.122113+0000 mgr.y (mgr.14901) 5088 : cluster [DBG] pgmap v5095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:12 smithi006 bash[27292]: audit 2023-08-28T00:48:11.957151+0000 mon.a (mon.0) 3578 : audit [DBG] from='client.? 172.21.15.6:0/3869415311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:12 smithi006 bash[33918]: cluster 2023-08-28T00:48:11.122113+0000 mgr.y (mgr.14901) 5088 : cluster [DBG] pgmap v5095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:12 smithi006 bash[33918]: audit 2023-08-28T00:48:11.957151+0000 mon.a (mon.0) 3578 : audit [DBG] from='client.? 172.21.15.6:0/3869415311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:12.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:48:12.412Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:12.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:48:12.413Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:12 smithi130 bash[27579]: cluster 2023-08-28T00:48:11.122113+0000 mgr.y (mgr.14901) 5088 : cluster [DBG] pgmap v5095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:12 smithi130 bash[27579]: audit 2023-08-28T00:48:11.957151+0000 mon.a (mon.0) 3578 : audit [DBG] from='client.? 172.21.15.6:0/3869415311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:12 smithi130 bash[51302]: debug 2023-08-28T00:48:12.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:12.886580+0000) 2023-08-28T00:48:14.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:13 smithi130 bash[51302]: debug 2023-08-28T00:48:13.883+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:13.886870+0000) 2023-08-28T00:48:14.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:14 smithi006 bash[33918]: cluster 2023-08-28T00:48:13.122767+0000 mgr.y (mgr.14901) 5089 : cluster [DBG] pgmap v5096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:14.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:14 smithi006 bash[33918]: audit 2023-08-28T00:48:14.374147+0000 mon.a (mon.0) 3579 : audit [DBG] from='client.? 172.21.15.6:0/3743209449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:14.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:14 smithi006 bash[27292]: cluster 2023-08-28T00:48:13.122767+0000 mgr.y (mgr.14901) 5089 : cluster [DBG] pgmap v5096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:14.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:14 smithi006 bash[27292]: audit 2023-08-28T00:48:14.374147+0000 mon.a (mon.0) 3579 : audit [DBG] from='client.? 172.21.15.6:0/3743209449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:14.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:48:14.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:14.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:48:14.295Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:14 smithi130 bash[27579]: cluster 2023-08-28T00:48:13.122767+0000 mgr.y (mgr.14901) 5089 : cluster [DBG] pgmap v5096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:14 smithi130 bash[27579]: audit 2023-08-28T00:48:14.374147+0000 mon.a (mon.0) 3579 : audit [DBG] from='client.? 172.21.15.6:0/3743209449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:14 smithi130 bash[51302]: debug 2023-08-28T00:48:14.883+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:14.887142+0000) 2023-08-28T00:48:16.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:15 smithi130 bash[51302]: debug 2023-08-28T00:48:15.887+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:15.887419+0000) 2023-08-28T00:48:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:16 smithi006 bash[33918]: cluster 2023-08-28T00:48:15.123915+0000 mgr.y (mgr.14901) 5090 : cluster [DBG] pgmap v5097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:16.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:16 smithi006 bash[27292]: cluster 2023-08-28T00:48:15.123915+0000 mgr.y (mgr.14901) 5090 : cluster [DBG] pgmap v5097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:16 smithi130 bash[27579]: cluster 2023-08-28T00:48:15.123915+0000 mgr.y (mgr.14901) 5090 : cluster [DBG] pgmap v5097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:17.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:16 smithi130 bash[51302]: debug 2023-08-28T00:48:16.883+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:16.887640+0000) 2023-08-28T00:48:17.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:17 smithi006 bash[27292]: audit 2023-08-28T00:48:16.784240+0000 mon.a (mon.0) 3580 : audit [DBG] from='client.? 172.21.15.6:0/775758760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:17.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:17 smithi006 bash[33918]: audit 2023-08-28T00:48:16.784240+0000 mon.a (mon.0) 3580 : audit [DBG] from='client.? 172.21.15.6:0/775758760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:17.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:17 smithi130 bash[27579]: audit 2023-08-28T00:48:16.784240+0000 mon.a (mon.0) 3580 : audit [DBG] from='client.? 172.21.15.6:0/775758760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:18.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:17 smithi130 bash[51302]: debug 2023-08-28T00:48:17.883+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:17.887925+0000) 2023-08-28T00:48:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:18 smithi006 bash[27292]: cluster 2023-08-28T00:48:17.124562+0000 mgr.y (mgr.14901) 5091 : cluster [DBG] pgmap v5098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:18.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:18 smithi006 bash[33918]: cluster 2023-08-28T00:48:17.124562+0000 mgr.y (mgr.14901) 5091 : cluster [DBG] pgmap v5098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:18 smithi130 bash[27579]: cluster 2023-08-28T00:48:17.124562+0000 mgr.y (mgr.14901) 5091 : cluster [DBG] pgmap v5098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:19.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:18 smithi130 bash[51302]: debug 2023-08-28T00:48:18.883+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:18.888219+0000) 2023-08-28T00:48:19.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:19 smithi006 bash[27292]: audit 2023-08-28T00:48:19.203371+0000 mon.c (mon.2) 1973 : audit [DBG] from='client.? 172.21.15.6:0/158211344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:19.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:19 smithi006 bash[33918]: audit 2023-08-28T00:48:19.203371+0000 mon.c (mon.2) 1973 : audit [DBG] from='client.? 172.21.15.6:0/158211344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:19.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:19 smithi130 bash[27579]: audit 2023-08-28T00:48:19.203371+0000 mon.c (mon.2) 1973 : audit [DBG] from='client.? 172.21.15.6:0/158211344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:20.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:19 smithi130 bash[51302]: debug 2023-08-28T00:48:19.883+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:19.888535+0000) 2023-08-28T00:48:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:20 smithi006 bash[27292]: cluster 2023-08-28T00:48:19.125587+0000 mgr.y (mgr.14901) 5092 : cluster [DBG] pgmap v5099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:20.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:20 smithi006 bash[33918]: cluster 2023-08-28T00:48:19.125587+0000 mgr.y (mgr.14901) 5092 : cluster [DBG] pgmap v5099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:20 smithi130 bash[27579]: cluster 2023-08-28T00:48:19.125587+0000 mgr.y (mgr.14901) 5092 : cluster [DBG] pgmap v5099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:21.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:48:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:48:20] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:48:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:48:20.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:48:20.838Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:21.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:20 smithi130 bash[51302]: debug 2023-08-28T00:48:20.883+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:20.888783+0000) 2023-08-28T00:48:22.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:21 smithi130 bash[51302]: debug 2023-08-28T00:48:21.887+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:21.889054+0000) 2023-08-28T00:48:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:22 smithi006 bash[27292]: cluster 2023-08-28T00:48:21.126194+0000 mgr.y (mgr.14901) 5093 : cluster [DBG] pgmap v5100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:22 smithi006 bash[27292]: audit 2023-08-28T00:48:21.619233+0000 mon.a (mon.0) 3581 : audit [DBG] from='client.? 172.21.15.6:0/533207458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:22 smithi006 bash[33918]: cluster 2023-08-28T00:48:21.126194+0000 mgr.y (mgr.14901) 5093 : cluster [DBG] pgmap v5100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:22 smithi006 bash[33918]: audit 2023-08-28T00:48:21.619233+0000 mon.a (mon.0) 3581 : audit [DBG] from='client.? 172.21.15.6:0/533207458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:22.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:48:22.412Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:22.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:48:22.414Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:22 smithi130 bash[27579]: cluster 2023-08-28T00:48:21.126194+0000 mgr.y (mgr.14901) 5093 : cluster [DBG] pgmap v5100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:22 smithi130 bash[27579]: audit 2023-08-28T00:48:21.619233+0000 mon.a (mon.0) 3581 : audit [DBG] from='client.? 172.21.15.6:0/533207458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:23.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:22 smithi130 bash[51302]: debug 2023-08-28T00:48:22.882+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:22.889354+0000) 2023-08-28T00:48:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:23 smithi130 bash[51302]: debug 2023-08-28T00:48:23.886+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:23.889637+0000) 2023-08-28T00:48:24.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:24 smithi006 bash[27292]: cluster 2023-08-28T00:48:23.126857+0000 mgr.y (mgr.14901) 5094 : cluster [DBG] pgmap v5101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:24.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:24 smithi006 bash[27292]: audit 2023-08-28T00:48:24.040075+0000 mon.a (mon.0) 3582 : audit [DBG] from='client.? 172.21.15.6:0/521955510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:24.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:24 smithi006 bash[33918]: cluster 2023-08-28T00:48:23.126857+0000 mgr.y (mgr.14901) 5094 : cluster [DBG] pgmap v5101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:24.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:24 smithi006 bash[33918]: audit 2023-08-28T00:48:24.040075+0000 mon.a (mon.0) 3582 : audit [DBG] from='client.? 172.21.15.6:0/521955510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:24.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:48:24.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:24.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:48:24.296Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:24 smithi130 bash[27579]: cluster 2023-08-28T00:48:23.126857+0000 mgr.y (mgr.14901) 5094 : cluster [DBG] pgmap v5101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:24 smithi130 bash[27579]: audit 2023-08-28T00:48:24.040075+0000 mon.a (mon.0) 3582 : audit [DBG] from='client.? 172.21.15.6:0/521955510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:24 smithi130 bash[51302]: debug 2023-08-28T00:48:24.882+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:24.889905+0000) 2023-08-28T00:48:26.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:25 smithi130 bash[51302]: debug 2023-08-28T00:48:25.886+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:25.890164+0000) 2023-08-28T00:48:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:26 smithi130 bash[27579]: cluster 2023-08-28T00:48:25.128006+0000 mgr.y (mgr.14901) 5095 : cluster [DBG] pgmap v5102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:26 smithi130 bash[27579]: audit 2023-08-28T00:48:26.462893+0000 mon.c (mon.2) 1974 : audit [DBG] from='client.? 172.21.15.6:0/2091989487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:26.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:26 smithi006 bash[27292]: cluster 2023-08-28T00:48:25.128006+0000 mgr.y (mgr.14901) 5095 : cluster [DBG] pgmap v5102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:26.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:26 smithi006 bash[27292]: audit 2023-08-28T00:48:26.462893+0000 mon.c (mon.2) 1974 : audit [DBG] from='client.? 172.21.15.6:0/2091989487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:26.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:26 smithi006 bash[33918]: cluster 2023-08-28T00:48:25.128006+0000 mgr.y (mgr.14901) 5095 : cluster [DBG] pgmap v5102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:26.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:26 smithi006 bash[33918]: audit 2023-08-28T00:48:26.462893+0000 mon.c (mon.2) 1974 : audit [DBG] from='client.? 172.21.15.6:0/2091989487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:27.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:26 smithi130 bash[51302]: debug 2023-08-28T00:48:26.882+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:26.890456+0000) 2023-08-28T00:48:28.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:27 smithi130 bash[51302]: debug 2023-08-28T00:48:27.882+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:27.890716+0000) 2023-08-28T00:48:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:28 smithi130 bash[27579]: cluster 2023-08-28T00:48:27.128786+0000 mgr.y (mgr.14901) 5096 : cluster [DBG] pgmap v5103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:28.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:28 smithi006 bash[27292]: cluster 2023-08-28T00:48:27.128786+0000 mgr.y (mgr.14901) 5096 : cluster [DBG] pgmap v5103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:28.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:28 smithi006 bash[33918]: cluster 2023-08-28T00:48:27.128786+0000 mgr.y (mgr.14901) 5096 : cluster [DBG] pgmap v5103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:29.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:28 smithi130 bash[51302]: debug 2023-08-28T00:48:28.890+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:28.891015+0000) 2023-08-28T00:48:29.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:29 smithi130 bash[27579]: audit 2023-08-28T00:48:28.884963+0000 mon.a (mon.0) 3583 : audit [DBG] from='client.? 172.21.15.6:0/587788915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:29.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:29 smithi006 bash[27292]: audit 2023-08-28T00:48:28.884963+0000 mon.a (mon.0) 3583 : audit [DBG] from='client.? 172.21.15.6:0/587788915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:29.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:29 smithi006 bash[33918]: audit 2023-08-28T00:48:28.884963+0000 mon.a (mon.0) 3583 : audit [DBG] from='client.? 172.21.15.6:0/587788915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:29 smithi130 bash[51302]: debug 2023-08-28T00:48:29.890+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:29.891261+0000) 2023-08-28T00:48:30.754 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:30 smithi006 bash[27292]: cluster 2023-08-28T00:48:29.129873+0000 mgr.y (mgr.14901) 5097 : cluster [DBG] pgmap v5104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:30.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:30 smithi006 bash[33918]: cluster 2023-08-28T00:48:29.129873+0000 mgr.y (mgr.14901) 5097 : cluster [DBG] pgmap v5104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:30 smithi130 bash[27579]: cluster 2023-08-28T00:48:29.129873+0000 mgr.y (mgr.14901) 5097 : cluster [DBG] pgmap v5104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:48:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:48:30] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T00:48:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:48:30.837Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:48:30.839Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:31.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:30 smithi130 bash[51302]: debug 2023-08-28T00:48:30.890+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:30.891489+0000) 2023-08-28T00:48:32.194 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:31 smithi130 bash[51302]: debug 2023-08-28T00:48:31.890+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:31.891757+0000) 2023-08-28T00:48:32.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:32 smithi006 bash[27292]: cluster 2023-08-28T00:48:31.130474+0000 mgr.y (mgr.14901) 5098 : cluster [DBG] pgmap v5105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:32.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:32 smithi006 bash[27292]: audit 2023-08-28T00:48:31.299419+0000 mon.c (mon.2) 1975 : audit [DBG] from='client.? 172.21.15.6:0/1190035545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:32.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:32 smithi006 bash[33918]: cluster 2023-08-28T00:48:31.130474+0000 mgr.y (mgr.14901) 5098 : cluster [DBG] pgmap v5105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:32.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:32 smithi006 bash[33918]: audit 2023-08-28T00:48:31.299419+0000 mon.c (mon.2) 1975 : audit [DBG] from='client.? 172.21.15.6:0/1190035545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:32.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:48:32.413Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:32.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:48:32.415Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:32 smithi130 bash[27579]: cluster 2023-08-28T00:48:31.130474+0000 mgr.y (mgr.14901) 5098 : cluster [DBG] pgmap v5105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:32 smithi130 bash[27579]: audit 2023-08-28T00:48:31.299419+0000 mon.c (mon.2) 1975 : audit [DBG] from='client.? 172.21.15.6:0/1190035545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:33.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:32 smithi130 bash[51302]: debug 2023-08-28T00:48:32.890+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:32.892084+0000) 2023-08-28T00:48:34.199 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:33 smithi130 bash[51302]: debug 2023-08-28T00:48:33.890+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:33.892384+0000) 2023-08-28T00:48:34.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:34 smithi006 bash[27292]: cluster 2023-08-28T00:48:33.131152+0000 mgr.y (mgr.14901) 5099 : cluster [DBG] pgmap v5106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:34.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:34 smithi006 bash[27292]: audit 2023-08-28T00:48:33.717774+0000 mon.a (mon.0) 3584 : audit [DBG] from='client.? 172.21.15.6:0/908848851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:34.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:34 smithi006 bash[33918]: cluster 2023-08-28T00:48:33.131152+0000 mgr.y (mgr.14901) 5099 : cluster [DBG] pgmap v5106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:34.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:34 smithi006 bash[33918]: audit 2023-08-28T00:48:33.717774+0000 mon.a (mon.0) 3584 : audit [DBG] from='client.? 172.21.15.6:0/908848851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:34.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:48:34.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:34.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:48:34.296Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:34 smithi130 bash[27579]: cluster 2023-08-28T00:48:33.131152+0000 mgr.y (mgr.14901) 5099 : cluster [DBG] pgmap v5106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:34 smithi130 bash[27579]: audit 2023-08-28T00:48:33.717774+0000 mon.a (mon.0) 3584 : audit [DBG] from='client.? 172.21.15.6:0/908848851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:35.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:34 smithi130 bash[51302]: debug 2023-08-28T00:48:34.890+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:34.892694+0000) 2023-08-28T00:48:36.203 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:35 smithi130 bash[51302]: debug 2023-08-28T00:48:35.890+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:35.892989+0000) 2023-08-28T00:48:36.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:36 smithi006 bash[33918]: cluster 2023-08-28T00:48:35.132340+0000 mgr.y (mgr.14901) 5100 : cluster [DBG] pgmap v5107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:36.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:36 smithi006 bash[33918]: audit 2023-08-28T00:48:36.132025+0000 mon.c (mon.2) 1976 : audit [DBG] from='client.? 172.21.15.6:0/1738247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:36.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:36 smithi006 bash[27292]: cluster 2023-08-28T00:48:35.132340+0000 mgr.y (mgr.14901) 5100 : cluster [DBG] pgmap v5107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:36.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:36 smithi006 bash[27292]: audit 2023-08-28T00:48:36.132025+0000 mon.c (mon.2) 1976 : audit [DBG] from='client.? 172.21.15.6:0/1738247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:36 smithi130 bash[27579]: cluster 2023-08-28T00:48:35.132340+0000 mgr.y (mgr.14901) 5100 : cluster [DBG] pgmap v5107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:36 smithi130 bash[27579]: audit 2023-08-28T00:48:36.132025+0000 mon.c (mon.2) 1976 : audit [DBG] from='client.? 172.21.15.6:0/1738247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:36 smithi130 bash[51302]: debug 2023-08-28T00:48:36.890+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:36.893258+0000) 2023-08-28T00:48:38.208 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:37 smithi130 bash[51302]: debug 2023-08-28T00:48:37.890+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:37.893506+0000) 2023-08-28T00:48:38.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:38 smithi006 bash[33918]: cluster 2023-08-28T00:48:37.133018+0000 mgr.y (mgr.14901) 5101 : cluster [DBG] pgmap v5108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:38.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:38 smithi006 bash[27292]: cluster 2023-08-28T00:48:37.133018+0000 mgr.y (mgr.14901) 5101 : cluster [DBG] pgmap v5108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:38 smithi130 bash[27579]: cluster 2023-08-28T00:48:37.133018+0000 mgr.y (mgr.14901) 5101 : cluster [DBG] pgmap v5108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:39.219 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:38 smithi130 bash[51302]: debug 2023-08-28T00:48:38.890+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:38.893773+0000) 2023-08-28T00:48:39.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:39 smithi006 bash[27292]: audit 2023-08-28T00:48:38.550452+0000 mon.c (mon.2) 1977 : audit [DBG] from='client.? 172.21.15.6:0/51966915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:39.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:39 smithi006 bash[33918]: audit 2023-08-28T00:48:38.550452+0000 mon.c (mon.2) 1977 : audit [DBG] from='client.? 172.21.15.6:0/51966915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:39.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:39 smithi130 bash[27579]: audit 2023-08-28T00:48:38.550452+0000 mon.c (mon.2) 1977 : audit [DBG] from='client.? 172.21.15.6:0/51966915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:40.220 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:39 smithi130 bash[51302]: debug 2023-08-28T00:48:39.890+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:39.894052+0000) 2023-08-28T00:48:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:40 smithi130 bash[27579]: cluster 2023-08-28T00:48:39.134105+0000 mgr.y (mgr.14901) 5102 : cluster [DBG] pgmap v5109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:40 smithi006 bash[33918]: cluster 2023-08-28T00:48:39.134105+0000 mgr.y (mgr.14901) 5102 : cluster [DBG] pgmap v5109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:40 smithi006 bash[27292]: cluster 2023-08-28T00:48:39.134105+0000 mgr.y (mgr.14901) 5102 : cluster [DBG] pgmap v5109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:41.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:48:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:48:40] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T00:48:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:48:40.837Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:48:40.840Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:41.226 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:40 smithi130 bash[51302]: debug 2023-08-28T00:48:40.890+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:40.894357+0000) 2023-08-28T00:48:41.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:41 smithi130 bash[27579]: audit 2023-08-28T00:48:40.965371+0000 mon.a (mon.0) 3585 : audit [DBG] from='client.? 172.21.15.6:0/1136730334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:41.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:41 smithi006 bash[33918]: audit 2023-08-28T00:48:40.965371+0000 mon.a (mon.0) 3585 : audit [DBG] from='client.? 172.21.15.6:0/1136730334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:41.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:41 smithi006 bash[27292]: audit 2023-08-28T00:48:40.965371+0000 mon.a (mon.0) 3585 : audit [DBG] from='client.? 172.21.15.6:0/1136730334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:42.229 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:41 smithi130 bash[51302]: debug 2023-08-28T00:48:41.890+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:41.894698+0000) 2023-08-28T00:48:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:42 smithi130 bash[27579]: cluster 2023-08-28T00:48:41.134753+0000 mgr.y (mgr.14901) 5103 : cluster [DBG] pgmap v5110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:42.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:42 smithi006 bash[33918]: cluster 2023-08-28T00:48:41.134753+0000 mgr.y (mgr.14901) 5103 : cluster [DBG] pgmap v5110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:42.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:42 smithi006 bash[27292]: cluster 2023-08-28T00:48:41.134753+0000 mgr.y (mgr.14901) 5103 : cluster [DBG] pgmap v5110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:48:42.414Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:48:42.416Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:43.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:42 smithi130 bash[51302]: debug 2023-08-28T00:48:42.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:42.894980+0000) 2023-08-28T00:48:44.236 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:43 smithi130 bash[51302]: debug 2023-08-28T00:48:43.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:43.895298+0000) 2023-08-28T00:48:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:44 smithi130 bash[27579]: cluster 2023-08-28T00:48:43.135362+0000 mgr.y (mgr.14901) 5104 : cluster [DBG] pgmap v5111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:44 smithi130 bash[27579]: audit 2023-08-28T00:48:43.385191+0000 mon.c (mon.2) 1978 : audit [DBG] from='client.? 172.21.15.6:0/2467606910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:44.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:44 smithi006 bash[33918]: cluster 2023-08-28T00:48:43.135362+0000 mgr.y (mgr.14901) 5104 : cluster [DBG] pgmap v5111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:44.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:44 smithi006 bash[33918]: audit 2023-08-28T00:48:43.385191+0000 mon.c (mon.2) 1978 : audit [DBG] from='client.? 172.21.15.6:0/2467606910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:48:44.295Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:48:44.298Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:44.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:44 smithi006 bash[27292]: cluster 2023-08-28T00:48:43.135362+0000 mgr.y (mgr.14901) 5104 : cluster [DBG] pgmap v5111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:44.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:44 smithi006 bash[27292]: audit 2023-08-28T00:48:43.385191+0000 mon.c (mon.2) 1978 : audit [DBG] from='client.? 172.21.15.6:0/2467606910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:44 smithi130 bash[51302]: debug 2023-08-28T00:48:44.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:44.895580+0000) 2023-08-28T00:48:46.242 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:45 smithi130 bash[51302]: debug 2023-08-28T00:48:45.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:45.895860+0000) 2023-08-28T00:48:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:46 smithi130 bash[27579]: cluster 2023-08-28T00:48:45.136480+0000 mgr.y (mgr.14901) 5105 : cluster [DBG] pgmap v5112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:46 smithi130 bash[27579]: audit 2023-08-28T00:48:45.796203+0000 mon.c (mon.2) 1979 : audit [DBG] from='client.? 172.21.15.6:0/3259947867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:46 smithi006 bash[27292]: cluster 2023-08-28T00:48:45.136480+0000 mgr.y (mgr.14901) 5105 : cluster [DBG] pgmap v5112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:46 smithi006 bash[27292]: audit 2023-08-28T00:48:45.796203+0000 mon.c (mon.2) 1979 : audit [DBG] from='client.? 172.21.15.6:0/3259947867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:46.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:46 smithi006 bash[33918]: cluster 2023-08-28T00:48:45.136480+0000 mgr.y (mgr.14901) 5105 : cluster [DBG] pgmap v5112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:46.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:46 smithi006 bash[33918]: audit 2023-08-28T00:48:45.796203+0000 mon.c (mon.2) 1979 : audit [DBG] from='client.? 172.21.15.6:0/3259947867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:46 smithi130 bash[51302]: debug 2023-08-28T00:48:46.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:46.896170+0000) 2023-08-28T00:48:48.247 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:47 smithi130 bash[51302]: debug 2023-08-28T00:48:47.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:47.896494+0000) 2023-08-28T00:48:48.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:48 smithi130 bash[27579]: cluster 2023-08-28T00:48:47.137045+0000 mgr.y (mgr.14901) 5106 : cluster [DBG] pgmap v5113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:48.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:48 smithi130 bash[27579]: audit 2023-08-28T00:48:48.208483+0000 mon.c (mon.2) 1980 : audit [DBG] from='client.? 172.21.15.6:0/4031474559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:48 smithi006 bash[33918]: cluster 2023-08-28T00:48:47.137045+0000 mgr.y (mgr.14901) 5106 : cluster [DBG] pgmap v5113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:48 smithi006 bash[33918]: audit 2023-08-28T00:48:48.208483+0000 mon.c (mon.2) 1980 : audit [DBG] from='client.? 172.21.15.6:0/4031474559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:48.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:48 smithi006 bash[27292]: cluster 2023-08-28T00:48:47.137045+0000 mgr.y (mgr.14901) 5106 : cluster [DBG] pgmap v5113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:48.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:48 smithi006 bash[27292]: audit 2023-08-28T00:48:48.208483+0000 mon.c (mon.2) 1980 : audit [DBG] from='client.? 172.21.15.6:0/4031474559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:48 smithi130 bash[51302]: debug 2023-08-28T00:48:48.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:48.896767+0000) 2023-08-28T00:48:50.253 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:49 smithi130 bash[51302]: debug 2023-08-28T00:48:49.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:49.897068+0000) 2023-08-28T00:48:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:50 smithi130 bash[27579]: cluster 2023-08-28T00:48:49.138087+0000 mgr.y (mgr.14901) 5107 : cluster [DBG] pgmap v5114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:50 smithi006 bash[33918]: cluster 2023-08-28T00:48:49.138087+0000 mgr.y (mgr.14901) 5107 : cluster [DBG] pgmap v5114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:50 smithi006 bash[27292]: cluster 2023-08-28T00:48:49.138087+0000 mgr.y (mgr.14901) 5107 : cluster [DBG] pgmap v5114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:51.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:48:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:48:50] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:48:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:48:50.839Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:48:50.840Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:51.264 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:50 smithi130 bash[51302]: debug 2023-08-28T00:48:50.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:50.897348+0000) 2023-08-28T00:48:51.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:51 smithi130 bash[27579]: audit 2023-08-28T00:48:50.635864+0000 mon.a (mon.0) 3586 : audit [DBG] from='client.? 172.21.15.6:0/364935875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:51.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:51 smithi006 bash[27292]: audit 2023-08-28T00:48:50.635864+0000 mon.a (mon.0) 3586 : audit [DBG] from='client.? 172.21.15.6:0/364935875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:51.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:51 smithi006 bash[33918]: audit 2023-08-28T00:48:50.635864+0000 mon.a (mon.0) 3586 : audit [DBG] from='client.? 172.21.15.6:0/364935875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:52.267 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:51 smithi130 bash[51302]: debug 2023-08-28T00:48:51.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:51.897630+0000) 2023-08-28T00:48:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:52 smithi130 bash[27579]: cluster 2023-08-28T00:48:51.138751+0000 mgr.y (mgr.14901) 5108 : cluster [DBG] pgmap v5115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:52.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:52 smithi006 bash[27292]: cluster 2023-08-28T00:48:51.138751+0000 mgr.y (mgr.14901) 5108 : cluster [DBG] pgmap v5115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:48:52.414Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:48:52.417Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:52 smithi006 bash[33918]: cluster 2023-08-28T00:48:51.138751+0000 mgr.y (mgr.14901) 5108 : cluster [DBG] pgmap v5115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:53.279 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:52 smithi130 bash[51302]: debug 2023-08-28T00:48:52.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:52.897893+0000) 2023-08-28T00:48:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:53 smithi130 bash[27579]: audit 2023-08-28T00:48:53.030505+0000 mon.b (mon.1) 610 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:48:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:53 smithi130 bash[27579]: audit 2023-08-28T00:48:53.051991+0000 mon.c (mon.2) 1981 : audit [DBG] from='client.? 172.21.15.6:0/166915707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:53.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:53 smithi006 bash[27292]: audit 2023-08-28T00:48:53.030505+0000 mon.b (mon.1) 610 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:48:53.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:53 smithi006 bash[27292]: audit 2023-08-28T00:48:53.051991+0000 mon.c (mon.2) 1981 : audit [DBG] from='client.? 172.21.15.6:0/166915707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:53 smithi006 bash[33918]: audit 2023-08-28T00:48:53.030505+0000 mon.b (mon.1) 610 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:48:53.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:53 smithi006 bash[33918]: audit 2023-08-28T00:48:53.051991+0000 mon.c (mon.2) 1981 : audit [DBG] from='client.? 172.21.15.6:0/166915707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:54.282 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:53 smithi130 bash[51302]: debug 2023-08-28T00:48:53.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:53.898180+0000) 2023-08-28T00:48:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:54 smithi130 bash[27579]: cluster 2023-08-28T00:48:53.139378+0000 mgr.y (mgr.14901) 5109 : cluster [DBG] pgmap v5116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:54 smithi006 bash[27292]: cluster 2023-08-28T00:48:53.139378+0000 mgr.y (mgr.14901) 5109 : cluster [DBG] pgmap v5116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:54.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:54 smithi006 bash[33918]: cluster 2023-08-28T00:48:53.139378+0000 mgr.y (mgr.14901) 5109 : cluster [DBG] pgmap v5116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:48:54.296Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:48:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:48:54.298Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:48:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:54 smithi130 bash[51302]: debug 2023-08-28T00:48:54.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:54.898498+0000) 2023-08-28T00:48:56.288 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:55 smithi130 bash[51302]: debug 2023-08-28T00:48:55.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:55.898780+0000) 2023-08-28T00:48:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:56 smithi130 bash[27579]: cluster 2023-08-28T00:48:55.140164+0000 mgr.y (mgr.14901) 5110 : cluster [DBG] pgmap v5117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:56 smithi130 bash[27579]: audit 2023-08-28T00:48:55.479993+0000 mon.a (mon.0) 3587 : audit [DBG] from='client.? 172.21.15.6:0/3579935561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:56 smithi006 bash[27292]: cluster 2023-08-28T00:48:55.140164+0000 mgr.y (mgr.14901) 5110 : cluster [DBG] pgmap v5117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:56.830 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:56 smithi006 bash[27292]: audit 2023-08-28T00:48:55.479993+0000 mon.a (mon.0) 3587 : audit [DBG] from='client.? 172.21.15.6:0/3579935561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:56.831 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:56 smithi006 bash[33918]: cluster 2023-08-28T00:48:55.140164+0000 mgr.y (mgr.14901) 5110 : cluster [DBG] pgmap v5117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:56.831 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:56 smithi006 bash[33918]: audit 2023-08-28T00:48:55.479993+0000 mon.a (mon.0) 3587 : audit [DBG] from='client.? 172.21.15.6:0/3579935561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:57.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:56 smithi130 bash[51302]: debug 2023-08-28T00:48:56.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:56.899055+0000) 2023-08-28T00:48:58.291 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:57 smithi130 bash[51302]: debug 2023-08-28T00:48:57.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:57.899278+0000) 2023-08-28T00:48:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:58 smithi130 bash[27579]: cluster 2023-08-28T00:48:57.140775+0000 mgr.y (mgr.14901) 5111 : cluster [DBG] pgmap v5118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:58 smithi130 bash[27579]: audit 2023-08-28T00:48:57.888427+0000 mon.c (mon.2) 1982 : audit [DBG] from='client.? 172.21.15.6:0/4293497772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:58 smithi006 bash[27292]: cluster 2023-08-28T00:48:57.140775+0000 mgr.y (mgr.14901) 5111 : cluster [DBG] pgmap v5118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:58.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:58 smithi006 bash[27292]: audit 2023-08-28T00:48:57.888427+0000 mon.c (mon.2) 1982 : audit [DBG] from='client.? 172.21.15.6:0/4293497772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:58.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:58 smithi006 bash[33918]: cluster 2023-08-28T00:48:57.140775+0000 mgr.y (mgr.14901) 5111 : cluster [DBG] pgmap v5118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:48:58.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:58 smithi006 bash[33918]: audit 2023-08-28T00:48:57.888427+0000 mon.c (mon.2) 1982 : audit [DBG] from='client.? 172.21.15.6:0/4293497772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:58 smithi130 bash[51302]: debug 2023-08-28T00:48:58.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:58.899495+0000) 2023-08-28T00:49:00.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:59 smithi006 bash[27292]: audit 2023-08-28T00:48:58.937403+0000 mon.a (mon.0) 3588 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:49:00.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:59 smithi006 bash[27292]: cluster 2023-08-28T00:48:59.141575+0000 mgr.y (mgr.14901) 5112 : cluster [DBG] pgmap v5119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:00.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:59 smithi006 bash[27292]: audit 2023-08-28T00:48:59.343500+0000 mon.a (mon.0) 3589 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:49:00.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:59 smithi006 bash[27292]: audit 2023-08-28T00:48:59.637220+0000 mon.b (mon.1) 611 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:49:00.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:59 smithi006 bash[27292]: audit 2023-08-28T00:48:59.639577+0000 mon.b (mon.1) 612 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:49:00.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:48:59 smithi006 bash[27292]: audit 2023-08-28T00:48:59.647476+0000 mon.a (mon.0) 3590 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:49:00.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:59 smithi006 bash[33918]: audit 2023-08-28T00:48:58.937403+0000 mon.a (mon.0) 3588 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:49:00.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:59 smithi006 bash[33918]: cluster 2023-08-28T00:48:59.141575+0000 mgr.y (mgr.14901) 5112 : cluster [DBG] pgmap v5119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:00.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:59 smithi006 bash[33918]: audit 2023-08-28T00:48:59.343500+0000 mon.a (mon.0) 3589 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:49:00.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:59 smithi006 bash[33918]: audit 2023-08-28T00:48:59.637220+0000 mon.b (mon.1) 611 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:49:00.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:59 smithi006 bash[33918]: audit 2023-08-28T00:48:59.639577+0000 mon.b (mon.1) 612 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:49:00.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:48:59 smithi006 bash[33918]: audit 2023-08-28T00:48:59.647476+0000 mon.a (mon.0) 3590 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:49:00.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:48:59 smithi130 bash[51302]: debug 2023-08-28T00:48:59.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:59.899798+0000) 2023-08-28T00:49:00.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:59 smithi130 bash[27579]: audit 2023-08-28T00:48:58.937403+0000 mon.a (mon.0) 3588 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:49:00.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:59 smithi130 bash[27579]: cluster 2023-08-28T00:48:59.141575+0000 mgr.y (mgr.14901) 5112 : cluster [DBG] pgmap v5119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:00.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:59 smithi130 bash[27579]: audit 2023-08-28T00:48:59.343500+0000 mon.a (mon.0) 3589 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:49:00.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:59 smithi130 bash[27579]: audit 2023-08-28T00:48:59.637220+0000 mon.b (mon.1) 611 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:49:00.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:59 smithi130 bash[27579]: audit 2023-08-28T00:48:59.639577+0000 mon.b (mon.1) 612 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:49:00.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:48:59 smithi130 bash[27579]: audit 2023-08-28T00:48:59.647476+0000 mon.a (mon.0) 3590 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:49:01.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:00 smithi006 bash[27292]: audit 2023-08-28T00:49:00.300003+0000 mon.c (mon.2) 1983 : audit [DBG] from='client.? 172.21.15.6:0/575899079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:01.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:49:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:49:00] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:49:01.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:00 smithi006 bash[33918]: audit 2023-08-28T00:49:00.300003+0000 mon.c (mon.2) 1983 : audit [DBG] from='client.? 172.21.15.6:0/575899079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:01.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:49:00.839Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:01.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:49:00.841Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:01.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:00 smithi130 bash[27579]: audit 2023-08-28T00:49:00.300003+0000 mon.c (mon.2) 1983 : audit [DBG] from='client.? 172.21.15.6:0/575899079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:01.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:00 smithi130 bash[51302]: debug 2023-08-28T00:49:00.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:00.900082+0000) 2023-08-28T00:49:02.203 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:01 smithi130 bash[51302]: debug 2023-08-28T00:49:01.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:01.900517+0000) 2023-08-28T00:49:02.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:02 smithi006 bash[27292]: cluster 2023-08-28T00:49:01.142224+0000 mgr.y (mgr.14901) 5113 : cluster [DBG] pgmap v5120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:02.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:02 smithi006 bash[33918]: cluster 2023-08-28T00:49:01.142224+0000 mgr.y (mgr.14901) 5113 : cluster [DBG] pgmap v5120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:49:02.415Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:02.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:49:02.416Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:02 smithi130 bash[27579]: cluster 2023-08-28T00:49:01.142224+0000 mgr.y (mgr.14901) 5113 : cluster [DBG] pgmap v5120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:03.210 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:02 smithi130 bash[51302]: debug 2023-08-28T00:49:02.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:02.900808+0000) 2023-08-28T00:49:03.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:03 smithi006 bash[27292]: audit 2023-08-28T00:49:02.719823+0000 mon.a (mon.0) 3591 : audit [DBG] from='client.? 172.21.15.6:0/1273364870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:03.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:03 smithi006 bash[33918]: audit 2023-08-28T00:49:02.719823+0000 mon.a (mon.0) 3591 : audit [DBG] from='client.? 172.21.15.6:0/1273364870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:03.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:03 smithi130 bash[27579]: audit 2023-08-28T00:49:02.719823+0000 mon.a (mon.0) 3591 : audit [DBG] from='client.? 172.21.15.6:0/1273364870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:04.212 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:03 smithi130 bash[51302]: debug 2023-08-28T00:49:03.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:03.901120+0000) 2023-08-28T00:49:04.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:04 smithi006 bash[27292]: cluster 2023-08-28T00:49:03.142916+0000 mgr.y (mgr.14901) 5114 : cluster [DBG] pgmap v5121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:04.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:04 smithi006 bash[33918]: cluster 2023-08-28T00:49:03.142916+0000 mgr.y (mgr.14901) 5114 : cluster [DBG] pgmap v5121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:49:04.296Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:04.468 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:49:04.298Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:04 smithi130 bash[27579]: cluster 2023-08-28T00:49:03.142916+0000 mgr.y (mgr.14901) 5114 : cluster [DBG] pgmap v5121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:05.220 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:04 smithi130 bash[51302]: debug 2023-08-28T00:49:04.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:04.901402+0000) 2023-08-28T00:49:05.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:05 smithi130 bash[27579]: audit 2023-08-28T00:49:05.128920+0000 mon.c (mon.2) 1984 : audit [DBG] from='client.? 172.21.15.6:0/1578529620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:05.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:05 smithi006 bash[27292]: audit 2023-08-28T00:49:05.128920+0000 mon.c (mon.2) 1984 : audit [DBG] from='client.? 172.21.15.6:0/1578529620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:05.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:05 smithi006 bash[33918]: audit 2023-08-28T00:49:05.128920+0000 mon.c (mon.2) 1984 : audit [DBG] from='client.? 172.21.15.6:0/1578529620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:06.223 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:05 smithi130 bash[51302]: debug 2023-08-28T00:49:05.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:05.901686+0000) 2023-08-28T00:49:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:06 smithi130 bash[27579]: cluster 2023-08-28T00:49:05.143893+0000 mgr.y (mgr.14901) 5115 : cluster [DBG] pgmap v5122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:06 smithi006 bash[27292]: cluster 2023-08-28T00:49:05.143893+0000 mgr.y (mgr.14901) 5115 : cluster [DBG] pgmap v5122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:06 smithi006 bash[33918]: cluster 2023-08-28T00:49:05.143893+0000 mgr.y (mgr.14901) 5115 : cluster [DBG] pgmap v5122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:07.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:06 smithi130 bash[51302]: debug 2023-08-28T00:49:06.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:06.901983+0000) 2023-08-28T00:49:08.228 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:07 smithi130 bash[51302]: debug 2023-08-28T00:49:07.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:07.902279+0000) 2023-08-28T00:49:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:08 smithi130 bash[27579]: cluster 2023-08-28T00:49:07.144551+0000 mgr.y (mgr.14901) 5116 : cluster [DBG] pgmap v5123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:08.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:08 smithi130 bash[27579]: audit 2023-08-28T00:49:07.548752+0000 mon.a (mon.0) 3592 : audit [DBG] from='client.? 172.21.15.6:0/2493555838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:08.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:08 smithi006 bash[27292]: cluster 2023-08-28T00:49:07.144551+0000 mgr.y (mgr.14901) 5116 : cluster [DBG] pgmap v5123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:08.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:08 smithi006 bash[27292]: audit 2023-08-28T00:49:07.548752+0000 mon.a (mon.0) 3592 : audit [DBG] from='client.? 172.21.15.6:0/2493555838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:08 smithi006 bash[33918]: cluster 2023-08-28T00:49:07.144551+0000 mgr.y (mgr.14901) 5116 : cluster [DBG] pgmap v5123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:08 smithi006 bash[33918]: audit 2023-08-28T00:49:07.548752+0000 mon.a (mon.0) 3592 : audit [DBG] from='client.? 172.21.15.6:0/2493555838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:09.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:08 smithi130 bash[51302]: debug 2023-08-28T00:49:08.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:08.902557+0000) 2023-08-28T00:49:10.233 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:09 smithi130 bash[51302]: debug 2023-08-28T00:49:09.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:09.902848+0000) 2023-08-28T00:49:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:10 smithi130 bash[27579]: cluster 2023-08-28T00:49:09.145679+0000 mgr.y (mgr.14901) 5117 : cluster [DBG] pgmap v5124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:10 smithi130 bash[27579]: audit 2023-08-28T00:49:09.953801+0000 mon.c (mon.2) 1985 : audit [DBG] from='client.? 172.21.15.6:0/3717665394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:10 smithi006 bash[27292]: cluster 2023-08-28T00:49:09.145679+0000 mgr.y (mgr.14901) 5117 : cluster [DBG] pgmap v5124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:10 smithi006 bash[27292]: audit 2023-08-28T00:49:09.953801+0000 mon.c (mon.2) 1985 : audit [DBG] from='client.? 172.21.15.6:0/3717665394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:10 smithi006 bash[33918]: cluster 2023-08-28T00:49:09.145679+0000 mgr.y (mgr.14901) 5117 : cluster [DBG] pgmap v5124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:10 smithi006 bash[33918]: audit 2023-08-28T00:49:09.953801+0000 mon.c (mon.2) 1985 : audit [DBG] from='client.? 172.21.15.6:0/3717665394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:11.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:49:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:49:10] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:49:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:49:10.840Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:49:10.841Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:11.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:10 smithi130 bash[51302]: debug 2023-08-28T00:49:10.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:10.903138+0000) 2023-08-28T00:49:12.237 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:11 smithi130 bash[51302]: debug 2023-08-28T00:49:11.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:11.903412+0000) 2023-08-28T00:49:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:12 smithi130 bash[27579]: cluster 2023-08-28T00:49:11.146306+0000 mgr.y (mgr.14901) 5118 : cluster [DBG] pgmap v5125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:12.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:12 smithi006 bash[27292]: cluster 2023-08-28T00:49:11.146306+0000 mgr.y (mgr.14901) 5118 : cluster [DBG] pgmap v5125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:12 smithi006 bash[33918]: cluster 2023-08-28T00:49:11.146306+0000 mgr.y (mgr.14901) 5118 : cluster [DBG] pgmap v5125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:49:12.416Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:49:12.418Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:13.253 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:12 smithi130 bash[51302]: debug 2023-08-28T00:49:12.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:12.903695+0000) 2023-08-28T00:49:13.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:13 smithi130 bash[27579]: audit 2023-08-28T00:49:12.387096+0000 mon.a (mon.0) 3593 : audit [DBG] from='client.? 172.21.15.6:0/2763867687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:13.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:13 smithi006 bash[33918]: audit 2023-08-28T00:49:12.387096+0000 mon.a (mon.0) 3593 : audit [DBG] from='client.? 172.21.15.6:0/2763867687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:13.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:13 smithi006 bash[27292]: audit 2023-08-28T00:49:12.387096+0000 mon.a (mon.0) 3593 : audit [DBG] from='client.? 172.21.15.6:0/2763867687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:14.256 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:13 smithi130 bash[51302]: debug 2023-08-28T00:49:13.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:13.903986+0000) 2023-08-28T00:49:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:14 smithi130 bash[27579]: cluster 2023-08-28T00:49:13.146986+0000 mgr.y (mgr.14901) 5119 : cluster [DBG] pgmap v5126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:14.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:14 smithi006 bash[27292]: cluster 2023-08-28T00:49:13.146986+0000 mgr.y (mgr.14901) 5119 : cluster [DBG] pgmap v5126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:14.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:14 smithi006 bash[33918]: cluster 2023-08-28T00:49:13.146986+0000 mgr.y (mgr.14901) 5119 : cluster [DBG] pgmap v5126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:49:14.297Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:49:14.299Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:15.268 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:14 smithi130 bash[51302]: debug 2023-08-28T00:49:14.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:14.904263+0000) 2023-08-28T00:49:15.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:15 smithi130 bash[27579]: audit 2023-08-28T00:49:14.797197+0000 mon.c (mon.2) 1986 : audit [DBG] from='client.? 172.21.15.6:0/227153701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:15.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:15 smithi006 bash[33918]: audit 2023-08-28T00:49:14.797197+0000 mon.c (mon.2) 1986 : audit [DBG] from='client.? 172.21.15.6:0/227153701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:15.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:15 smithi006 bash[27292]: audit 2023-08-28T00:49:14.797197+0000 mon.c (mon.2) 1986 : audit [DBG] from='client.? 172.21.15.6:0/227153701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:16.271 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:15 smithi130 bash[51302]: debug 2023-08-28T00:49:15.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:15.904543+0000) 2023-08-28T00:49:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:16 smithi130 bash[27579]: cluster 2023-08-28T00:49:15.148007+0000 mgr.y (mgr.14901) 5120 : cluster [DBG] pgmap v5127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:16 smithi006 bash[33918]: cluster 2023-08-28T00:49:15.148007+0000 mgr.y (mgr.14901) 5120 : cluster [DBG] pgmap v5127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:16.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:16 smithi006 bash[27292]: cluster 2023-08-28T00:49:15.148007+0000 mgr.y (mgr.14901) 5120 : cluster [DBG] pgmap v5127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:17.282 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:16 smithi130 bash[51302]: debug 2023-08-28T00:49:16.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:16.904861+0000) 2023-08-28T00:49:17.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:17 smithi130 bash[27579]: audit 2023-08-28T00:49:17.204036+0000 mon.a (mon.0) 3594 : audit [DBG] from='client.? 172.21.15.6:0/2235911119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:17.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:17 smithi006 bash[27292]: audit 2023-08-28T00:49:17.204036+0000 mon.a (mon.0) 3594 : audit [DBG] from='client.? 172.21.15.6:0/2235911119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:17.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:17 smithi006 bash[33918]: audit 2023-08-28T00:49:17.204036+0000 mon.a (mon.0) 3594 : audit [DBG] from='client.? 172.21.15.6:0/2235911119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:18.286 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:17 smithi130 bash[51302]: debug 2023-08-28T00:49:17.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:17.905160+0000) 2023-08-28T00:49:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:18 smithi130 bash[27579]: cluster 2023-08-28T00:49:17.148620+0000 mgr.y (mgr.14901) 5121 : cluster [DBG] pgmap v5128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:18 smithi006 bash[33918]: cluster 2023-08-28T00:49:17.148620+0000 mgr.y (mgr.14901) 5121 : cluster [DBG] pgmap v5128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:18 smithi006 bash[27292]: cluster 2023-08-28T00:49:17.148620+0000 mgr.y (mgr.14901) 5121 : cluster [DBG] pgmap v5128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:19.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:18 smithi130 bash[51302]: debug 2023-08-28T00:49:18.901+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:18.905429+0000) 2023-08-28T00:49:20.291 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:19 smithi130 bash[51302]: debug 2023-08-28T00:49:19.901+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:19.905735+0000) 2023-08-28T00:49:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:20 smithi130 bash[27579]: cluster 2023-08-28T00:49:19.149864+0000 mgr.y (mgr.14901) 5122 : cluster [DBG] pgmap v5129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:20 smithi130 bash[27579]: audit 2023-08-28T00:49:19.631966+0000 mon.c (mon.2) 1987 : audit [DBG] from='client.? 172.21.15.6:0/3637784132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:20 smithi006 bash[33918]: cluster 2023-08-28T00:49:19.149864+0000 mgr.y (mgr.14901) 5122 : cluster [DBG] pgmap v5129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:20.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:20 smithi006 bash[33918]: audit 2023-08-28T00:49:19.631966+0000 mon.c (mon.2) 1987 : audit [DBG] from='client.? 172.21.15.6:0/3637784132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:20 smithi006 bash[27292]: cluster 2023-08-28T00:49:19.149864+0000 mgr.y (mgr.14901) 5122 : cluster [DBG] pgmap v5129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:20 smithi006 bash[27292]: audit 2023-08-28T00:49:19.631966+0000 mon.c (mon.2) 1987 : audit [DBG] from='client.? 172.21.15.6:0/3637784132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:49:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:49:20] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:49:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:49:20.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:49:20.842Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:21.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:20 smithi130 bash[51302]: debug 2023-08-28T00:49:20.901+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:20.906031+0000) 2023-08-28T00:49:22.296 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:21 smithi130 bash[51302]: debug 2023-08-28T00:49:21.905+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:21.906371+0000) 2023-08-28T00:49:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:22 smithi130 bash[27579]: cluster 2023-08-28T00:49:21.150492+0000 mgr.y (mgr.14901) 5123 : cluster [DBG] pgmap v5130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:22 smithi130 bash[27579]: audit 2023-08-28T00:49:22.045465+0000 mon.c (mon.2) 1988 : audit [DBG] from='client.? 172.21.15.6:0/3442537812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:22 smithi006 bash[27292]: cluster 2023-08-28T00:49:21.150492+0000 mgr.y (mgr.14901) 5123 : cluster [DBG] pgmap v5130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:22 smithi006 bash[27292]: audit 2023-08-28T00:49:22.045465+0000 mon.c (mon.2) 1988 : audit [DBG] from='client.? 172.21.15.6:0/3442537812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:22 smithi006 bash[33918]: cluster 2023-08-28T00:49:21.150492+0000 mgr.y (mgr.14901) 5123 : cluster [DBG] pgmap v5130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:22 smithi006 bash[33918]: audit 2023-08-28T00:49:22.045465+0000 mon.c (mon.2) 1988 : audit [DBG] from='client.? 172.21.15.6:0/3442537812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:49:22.416Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:22.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:49:22.418Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:23.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:22 smithi130 bash[51302]: debug 2023-08-28T00:49:22.901+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:22.906718+0000) 2023-08-28T00:49:24.300 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:23 smithi130 bash[51302]: debug 2023-08-28T00:49:23.905+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:23.907032+0000) 2023-08-28T00:49:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:24 smithi130 bash[27579]: cluster 2023-08-28T00:49:23.151136+0000 mgr.y (mgr.14901) 5124 : cluster [DBG] pgmap v5131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:24.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:24 smithi006 bash[33918]: cluster 2023-08-28T00:49:23.151136+0000 mgr.y (mgr.14901) 5124 : cluster [DBG] pgmap v5131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:24.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:24 smithi006 bash[27292]: cluster 2023-08-28T00:49:23.151136+0000 mgr.y (mgr.14901) 5124 : cluster [DBG] pgmap v5131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:49:24.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:24.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:49:24.300Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:25.308 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:24 smithi130 bash[51302]: debug 2023-08-28T00:49:24.905+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:24.907353+0000) 2023-08-28T00:49:25.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:25 smithi130 bash[27579]: audit 2023-08-28T00:49:24.457946+0000 mon.c (mon.2) 1989 : audit [DBG] from='client.? 172.21.15.6:0/2186862944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:25.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:25 smithi006 bash[33918]: audit 2023-08-28T00:49:24.457946+0000 mon.c (mon.2) 1989 : audit [DBG] from='client.? 172.21.15.6:0/2186862944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:25.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:25 smithi006 bash[27292]: audit 2023-08-28T00:49:24.457946+0000 mon.c (mon.2) 1989 : audit [DBG] from='client.? 172.21.15.6:0/2186862944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:26.314 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:25 smithi130 bash[51302]: debug 2023-08-28T00:49:25.905+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:25.907672+0000) 2023-08-28T00:49:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:26 smithi130 bash[27579]: cluster 2023-08-28T00:49:25.152266+0000 mgr.y (mgr.14901) 5125 : cluster [DBG] pgmap v5132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:26 smithi006 bash[33918]: cluster 2023-08-28T00:49:25.152266+0000 mgr.y (mgr.14901) 5125 : cluster [DBG] pgmap v5132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:26 smithi006 bash[27292]: cluster 2023-08-28T00:49:25.152266+0000 mgr.y (mgr.14901) 5125 : cluster [DBG] pgmap v5132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:27.322 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:26 smithi130 bash[51302]: debug 2023-08-28T00:49:26.905+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:26.907982+0000) 2023-08-28T00:49:27.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:27 smithi130 bash[27579]: audit 2023-08-28T00:49:26.880019+0000 mon.a (mon.0) 3595 : audit [DBG] from='client.? 172.21.15.6:0/1067433107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:27.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:27 smithi006 bash[27292]: audit 2023-08-28T00:49:26.880019+0000 mon.a (mon.0) 3595 : audit [DBG] from='client.? 172.21.15.6:0/1067433107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:27.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:27 smithi006 bash[33918]: audit 2023-08-28T00:49:26.880019+0000 mon.a (mon.0) 3595 : audit [DBG] from='client.? 172.21.15.6:0/1067433107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:28.325 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:27 smithi130 bash[51302]: debug 2023-08-28T00:49:27.905+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:27.908307+0000) 2023-08-28T00:49:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:28 smithi130 bash[27579]: cluster 2023-08-28T00:49:27.153077+0000 mgr.y (mgr.14901) 5126 : cluster [DBG] pgmap v5133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:28.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:28 smithi006 bash[27292]: cluster 2023-08-28T00:49:27.153077+0000 mgr.y (mgr.14901) 5126 : cluster [DBG] pgmap v5133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:28.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:28 smithi006 bash[33918]: cluster 2023-08-28T00:49:27.153077+0000 mgr.y (mgr.14901) 5126 : cluster [DBG] pgmap v5133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:29.334 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:28 smithi130 bash[51302]: debug 2023-08-28T00:49:28.905+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:28.908583+0000) 2023-08-28T00:49:29.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:29 smithi130 bash[27579]: audit 2023-08-28T00:49:29.305251+0000 mon.c (mon.2) 1990 : audit [DBG] from='client.? 172.21.15.6:0/724578844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:29.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:29 smithi006 bash[27292]: audit 2023-08-28T00:49:29.305251+0000 mon.c (mon.2) 1990 : audit [DBG] from='client.? 172.21.15.6:0/724578844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:29.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:29 smithi006 bash[33918]: audit 2023-08-28T00:49:29.305251+0000 mon.c (mon.2) 1990 : audit [DBG] from='client.? 172.21.15.6:0/724578844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:30.335 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:29 smithi130 bash[51302]: debug 2023-08-28T00:49:29.905+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:29.908948+0000) 2023-08-28T00:49:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:30 smithi130 bash[27579]: cluster 2023-08-28T00:49:29.154292+0000 mgr.y (mgr.14901) 5127 : cluster [DBG] pgmap v5134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:30 smithi006 bash[27292]: cluster 2023-08-28T00:49:29.154292+0000 mgr.y (mgr.14901) 5127 : cluster [DBG] pgmap v5134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:30.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:30 smithi006 bash[33918]: cluster 2023-08-28T00:49:29.154292+0000 mgr.y (mgr.14901) 5127 : cluster [DBG] pgmap v5134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:49:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:49:30] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:49:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:49:30.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:49:30.843Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:31.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:30 smithi130 bash[51302]: debug 2023-08-28T00:49:30.905+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:30.909237+0000) 2023-08-28T00:49:32.340 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:31 smithi130 bash[51302]: debug 2023-08-28T00:49:31.905+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:31.909526+0000) 2023-08-28T00:49:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:32 smithi130 bash[27579]: cluster 2023-08-28T00:49:31.155027+0000 mgr.y (mgr.14901) 5128 : cluster [DBG] pgmap v5135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:32 smithi130 bash[27579]: audit 2023-08-28T00:49:31.715031+0000 mon.a (mon.0) 3596 : audit [DBG] from='client.? 172.21.15.6:0/408152852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:32.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:32 smithi006 bash[33918]: cluster 2023-08-28T00:49:31.155027+0000 mgr.y (mgr.14901) 5128 : cluster [DBG] pgmap v5135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:32 smithi006 bash[33918]: audit 2023-08-28T00:49:31.715031+0000 mon.a (mon.0) 3596 : audit [DBG] from='client.? 172.21.15.6:0/408152852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:49:32.417Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:49:32.419Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:32 smithi006 bash[27292]: cluster 2023-08-28T00:49:31.155027+0000 mgr.y (mgr.14901) 5128 : cluster [DBG] pgmap v5135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:32 smithi006 bash[27292]: audit 2023-08-28T00:49:31.715031+0000 mon.a (mon.0) 3596 : audit [DBG] from='client.? 172.21.15.6:0/408152852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:33.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:32 smithi130 bash[51302]: debug 2023-08-28T00:49:32.905+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:32.909825+0000) 2023-08-28T00:49:34.345 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:33 smithi130 bash[51302]: debug 2023-08-28T00:49:33.905+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:33.910113+0000) 2023-08-28T00:49:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:34 smithi130 bash[27579]: cluster 2023-08-28T00:49:33.155728+0000 mgr.y (mgr.14901) 5129 : cluster [DBG] pgmap v5136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:34 smithi130 bash[27579]: audit 2023-08-28T00:49:34.140675+0000 mon.c (mon.2) 1991 : audit [DBG] from='client.? 172.21.15.6:0/314455028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:34 smithi006 bash[33918]: cluster 2023-08-28T00:49:33.155728+0000 mgr.y (mgr.14901) 5129 : cluster [DBG] pgmap v5136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:34 smithi006 bash[33918]: audit 2023-08-28T00:49:34.140675+0000 mon.c (mon.2) 1991 : audit [DBG] from='client.? 172.21.15.6:0/314455028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:49:34.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:49:34.300Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:34 smithi006 bash[27292]: cluster 2023-08-28T00:49:33.155728+0000 mgr.y (mgr.14901) 5129 : cluster [DBG] pgmap v5136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:34 smithi006 bash[27292]: audit 2023-08-28T00:49:34.140675+0000 mon.c (mon.2) 1991 : audit [DBG] from='client.? 172.21.15.6:0/314455028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:35.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:34 smithi130 bash[51302]: debug 2023-08-28T00:49:34.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:34.910454+0000) 2023-08-28T00:49:36.349 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:35 smithi130 bash[51302]: debug 2023-08-28T00:49:35.905+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:35.910761+0000) 2023-08-28T00:49:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:36 smithi130 bash[27579]: cluster 2023-08-28T00:49:35.156875+0000 mgr.y (mgr.14901) 5130 : cluster [DBG] pgmap v5137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:36.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:36 smithi006 bash[33918]: cluster 2023-08-28T00:49:35.156875+0000 mgr.y (mgr.14901) 5130 : cluster [DBG] pgmap v5137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:36.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:36 smithi006 bash[27292]: cluster 2023-08-28T00:49:35.156875+0000 mgr.y (mgr.14901) 5130 : cluster [DBG] pgmap v5137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:37.358 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:36 smithi130 bash[51302]: debug 2023-08-28T00:49:36.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:36.911041+0000) 2023-08-28T00:49:37.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:37 smithi130 bash[27579]: audit 2023-08-28T00:49:36.557433+0000 mon.c (mon.2) 1992 : audit [DBG] from='client.? 172.21.15.6:0/2749434282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:37.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:37 smithi006 bash[27292]: audit 2023-08-28T00:49:36.557433+0000 mon.c (mon.2) 1992 : audit [DBG] from='client.? 172.21.15.6:0/2749434282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:37.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:37 smithi006 bash[33918]: audit 2023-08-28T00:49:36.557433+0000 mon.c (mon.2) 1992 : audit [DBG] from='client.? 172.21.15.6:0/2749434282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:38.359 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:37 smithi130 bash[51302]: debug 2023-08-28T00:49:37.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:37.911362+0000) 2023-08-28T00:49:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:38 smithi130 bash[27579]: cluster 2023-08-28T00:49:37.157510+0000 mgr.y (mgr.14901) 5131 : cluster [DBG] pgmap v5138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:38.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:38 smithi006 bash[27292]: cluster 2023-08-28T00:49:37.157510+0000 mgr.y (mgr.14901) 5131 : cluster [DBG] pgmap v5138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:38.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:38 smithi006 bash[33918]: cluster 2023-08-28T00:49:37.157510+0000 mgr.y (mgr.14901) 5131 : cluster [DBG] pgmap v5138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:39.362 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:38 smithi130 bash[51302]: debug 2023-08-28T00:49:38.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:38.911673+0000) 2023-08-28T00:49:39.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:39 smithi130 bash[27579]: audit 2023-08-28T00:49:38.969944+0000 mon.a (mon.0) 3597 : audit [DBG] from='client.? 172.21.15.6:0/2017190123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:39.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:39 smithi006 bash[33918]: audit 2023-08-28T00:49:38.969944+0000 mon.a (mon.0) 3597 : audit [DBG] from='client.? 172.21.15.6:0/2017190123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:39.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:39 smithi006 bash[27292]: audit 2023-08-28T00:49:38.969944+0000 mon.a (mon.0) 3597 : audit [DBG] from='client.? 172.21.15.6:0/2017190123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:40.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:39 smithi130 bash[51302]: debug 2023-08-28T00:49:39.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:39.912010+0000) 2023-08-28T00:49:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:40 smithi006 bash[27292]: cluster 2023-08-28T00:49:39.158568+0000 mgr.y (mgr.14901) 5132 : cluster [DBG] pgmap v5139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:40 smithi006 bash[33918]: cluster 2023-08-28T00:49:39.158568+0000 mgr.y (mgr.14901) 5132 : cluster [DBG] pgmap v5139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:40 smithi130 bash[27579]: cluster 2023-08-28T00:49:39.158568+0000 mgr.y (mgr.14901) 5132 : cluster [DBG] pgmap v5139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:41.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:49:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:49:40] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:49:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:49:40.842Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:49:40.844Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:41.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:40 smithi130 bash[51302]: debug 2023-08-28T00:49:40.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:40.912333+0000) 2023-08-28T00:49:42.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:41 smithi130 bash[51302]: debug 2023-08-28T00:49:41.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:41.912597+0000) 2023-08-28T00:49:42.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:42 smithi006 bash[33918]: cluster 2023-08-28T00:49:41.159029+0000 mgr.y (mgr.14901) 5133 : cluster [DBG] pgmap v5140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:42.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:42 smithi006 bash[33918]: audit 2023-08-28T00:49:41.388606+0000 mon.c (mon.2) 1993 : audit [DBG] from='client.? 172.21.15.6:0/2606851963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:42.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:42 smithi006 bash[27292]: cluster 2023-08-28T00:49:41.159029+0000 mgr.y (mgr.14901) 5133 : cluster [DBG] pgmap v5140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:42.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:42 smithi006 bash[27292]: audit 2023-08-28T00:49:41.388606+0000 mon.c (mon.2) 1993 : audit [DBG] from='client.? 172.21.15.6:0/2606851963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:42.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:49:42.418Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:42.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:49:42.420Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:42 smithi130 bash[27579]: cluster 2023-08-28T00:49:41.159029+0000 mgr.y (mgr.14901) 5133 : cluster [DBG] pgmap v5140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:42 smithi130 bash[27579]: audit 2023-08-28T00:49:41.388606+0000 mon.c (mon.2) 1993 : audit [DBG] from='client.? 172.21.15.6:0/2606851963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:43.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:42 smithi130 bash[51302]: debug 2023-08-28T00:49:42.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:42.912905+0000) 2023-08-28T00:49:44.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:43 smithi130 bash[51302]: debug 2023-08-28T00:49:43.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:43.913206+0000) 2023-08-28T00:49:44.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:44 smithi006 bash[33918]: cluster 2023-08-28T00:49:43.159653+0000 mgr.y (mgr.14901) 5134 : cluster [DBG] pgmap v5141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:44.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:44 smithi006 bash[33918]: audit 2023-08-28T00:49:43.807524+0000 mon.a (mon.0) 3598 : audit [DBG] from='client.? 172.21.15.6:0/3938276514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:44.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:44 smithi006 bash[27292]: cluster 2023-08-28T00:49:43.159653+0000 mgr.y (mgr.14901) 5134 : cluster [DBG] pgmap v5141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:44.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:44 smithi006 bash[27292]: audit 2023-08-28T00:49:43.807524+0000 mon.a (mon.0) 3598 : audit [DBG] from='client.? 172.21.15.6:0/3938276514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:44.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:49:44.299Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:44.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:49:44.301Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:44 smithi130 bash[27579]: cluster 2023-08-28T00:49:43.159653+0000 mgr.y (mgr.14901) 5134 : cluster [DBG] pgmap v5141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:44 smithi130 bash[27579]: audit 2023-08-28T00:49:43.807524+0000 mon.a (mon.0) 3598 : audit [DBG] from='client.? 172.21.15.6:0/3938276514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:44 smithi130 bash[51302]: debug 2023-08-28T00:49:44.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:44.913484+0000) 2023-08-28T00:49:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:45 smithi130 bash[51302]: debug 2023-08-28T00:49:45.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:45.913899+0000) 2023-08-28T00:49:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:46 smithi006 bash[27292]: cluster 2023-08-28T00:49:45.160782+0000 mgr.y (mgr.14901) 5135 : cluster [DBG] pgmap v5142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:46 smithi006 bash[27292]: audit 2023-08-28T00:49:46.231036+0000 mon.c (mon.2) 1994 : audit [DBG] from='client.? 172.21.15.6:0/984891650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:46.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:46 smithi006 bash[33918]: cluster 2023-08-28T00:49:45.160782+0000 mgr.y (mgr.14901) 5135 : cluster [DBG] pgmap v5142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:46.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:46 smithi006 bash[33918]: audit 2023-08-28T00:49:46.231036+0000 mon.c (mon.2) 1994 : audit [DBG] from='client.? 172.21.15.6:0/984891650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:46 smithi130 bash[27579]: cluster 2023-08-28T00:49:45.160782+0000 mgr.y (mgr.14901) 5135 : cluster [DBG] pgmap v5142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:46 smithi130 bash[27579]: audit 2023-08-28T00:49:46.231036+0000 mon.c (mon.2) 1994 : audit [DBG] from='client.? 172.21.15.6:0/984891650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:46 smithi130 bash[51302]: debug 2023-08-28T00:49:46.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:46.914179+0000) 2023-08-28T00:49:48.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:47 smithi130 bash[51302]: debug 2023-08-28T00:49:47.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:47.914550+0000) 2023-08-28T00:49:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:48 smithi006 bash[27292]: cluster 2023-08-28T00:49:47.161472+0000 mgr.y (mgr.14901) 5136 : cluster [DBG] pgmap v5143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:48 smithi006 bash[33918]: cluster 2023-08-28T00:49:47.161472+0000 mgr.y (mgr.14901) 5136 : cluster [DBG] pgmap v5143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:48 smithi130 bash[27579]: cluster 2023-08-28T00:49:47.161472+0000 mgr.y (mgr.14901) 5136 : cluster [DBG] pgmap v5143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:48 smithi130 bash[51302]: debug 2023-08-28T00:49:48.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:48.914819+0000) 2023-08-28T00:49:49.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:49 smithi006 bash[33918]: audit 2023-08-28T00:49:48.642131+0000 mon.a (mon.0) 3599 : audit [DBG] from='client.? 172.21.15.6:0/2153689767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:49.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:49 smithi006 bash[27292]: audit 2023-08-28T00:49:48.642131+0000 mon.a (mon.0) 3599 : audit [DBG] from='client.? 172.21.15.6:0/2153689767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:49.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:49 smithi130 bash[27579]: audit 2023-08-28T00:49:48.642131+0000 mon.a (mon.0) 3599 : audit [DBG] from='client.? 172.21.15.6:0/2153689767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:49 smithi130 bash[51302]: debug 2023-08-28T00:49:49.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:49.915151+0000) 2023-08-28T00:49:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:50 smithi006 bash[27292]: cluster 2023-08-28T00:49:49.162546+0000 mgr.y (mgr.14901) 5137 : cluster [DBG] pgmap v5144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:50 smithi006 bash[33918]: cluster 2023-08-28T00:49:49.162546+0000 mgr.y (mgr.14901) 5137 : cluster [DBG] pgmap v5144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:50 smithi130 bash[27579]: cluster 2023-08-28T00:49:49.162546+0000 mgr.y (mgr.14901) 5137 : cluster [DBG] pgmap v5144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:51.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:49:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:49:50] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:49:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:49:50.842Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:49:50.845Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:51.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:50 smithi130 bash[51302]: debug 2023-08-28T00:49:50.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:50.915433+0000) 2023-08-28T00:49:51.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:51 smithi006 bash[33918]: audit 2023-08-28T00:49:51.059529+0000 mon.c (mon.2) 1995 : audit [DBG] from='client.? 172.21.15.6:0/4278376866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:51.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:51 smithi006 bash[27292]: audit 2023-08-28T00:49:51.059529+0000 mon.c (mon.2) 1995 : audit [DBG] from='client.? 172.21.15.6:0/4278376866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:51.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:51 smithi130 bash[27579]: audit 2023-08-28T00:49:51.059529+0000 mon.c (mon.2) 1995 : audit [DBG] from='client.? 172.21.15.6:0/4278376866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:52.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:51 smithi130 bash[51302]: debug 2023-08-28T00:49:51.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:51.915758+0000) 2023-08-28T00:49:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:52 smithi006 bash[33918]: cluster 2023-08-28T00:49:51.163197+0000 mgr.y (mgr.14901) 5138 : cluster [DBG] pgmap v5145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:49:52.418Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:49:52.420Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:52 smithi006 bash[27292]: cluster 2023-08-28T00:49:51.163197+0000 mgr.y (mgr.14901) 5138 : cluster [DBG] pgmap v5145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:52 smithi130 bash[27579]: cluster 2023-08-28T00:49:51.163197+0000 mgr.y (mgr.14901) 5138 : cluster [DBG] pgmap v5145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:53.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:52 smithi130 bash[51302]: debug 2023-08-28T00:49:52.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:52.916045+0000) 2023-08-28T00:49:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:53 smithi130 bash[51302]: debug 2023-08-28T00:49:53.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:53.916327+0000) 2023-08-28T00:49:54.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:54 smithi006 bash[27292]: cluster 2023-08-28T00:49:53.163887+0000 mgr.y (mgr.14901) 5139 : cluster [DBG] pgmap v5146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:54.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:54 smithi006 bash[27292]: audit 2023-08-28T00:49:53.477968+0000 mon.a (mon.0) 3600 : audit [DBG] from='client.? 172.21.15.6:0/3887929476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:54.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:54 smithi006 bash[33918]: cluster 2023-08-28T00:49:53.163887+0000 mgr.y (mgr.14901) 5139 : cluster [DBG] pgmap v5146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:54.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:54 smithi006 bash[33918]: audit 2023-08-28T00:49:53.477968+0000 mon.a (mon.0) 3600 : audit [DBG] from='client.? 172.21.15.6:0/3887929476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:49:54.299Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:54.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:49:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:49:54.301Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:49:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:54 smithi130 bash[27579]: cluster 2023-08-28T00:49:53.163887+0000 mgr.y (mgr.14901) 5139 : cluster [DBG] pgmap v5146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:54 smithi130 bash[27579]: audit 2023-08-28T00:49:53.477968+0000 mon.a (mon.0) 3600 : audit [DBG] from='client.? 172.21.15.6:0/3887929476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:54 smithi130 bash[51302]: debug 2023-08-28T00:49:54.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:54.916606+0000) 2023-08-28T00:49:56.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:55 smithi130 bash[51302]: debug 2023-08-28T00:49:55.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:55.916871+0000) 2023-08-28T00:49:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:56 smithi006 bash[27292]: cluster 2023-08-28T00:49:55.165113+0000 mgr.y (mgr.14901) 5140 : cluster [DBG] pgmap v5147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:56 smithi006 bash[27292]: audit 2023-08-28T00:49:55.892039+0000 mon.a (mon.0) 3601 : audit [DBG] from='client.? 172.21.15.6:0/831987576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:56.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:56 smithi006 bash[33918]: cluster 2023-08-28T00:49:55.165113+0000 mgr.y (mgr.14901) 5140 : cluster [DBG] pgmap v5147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:56.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:56 smithi006 bash[33918]: audit 2023-08-28T00:49:55.892039+0000 mon.a (mon.0) 3601 : audit [DBG] from='client.? 172.21.15.6:0/831987576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:56 smithi130 bash[27579]: cluster 2023-08-28T00:49:55.165113+0000 mgr.y (mgr.14901) 5140 : cluster [DBG] pgmap v5147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:56 smithi130 bash[27579]: audit 2023-08-28T00:49:55.892039+0000 mon.a (mon.0) 3601 : audit [DBG] from='client.? 172.21.15.6:0/831987576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:57.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:56 smithi130 bash[51302]: debug 2023-08-28T00:49:56.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:56.917182+0000) 2023-08-28T00:49:58.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:57 smithi130 bash[51302]: debug 2023-08-28T00:49:57.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:57.917449+0000) 2023-08-28T00:49:58.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:58 smithi006 bash[27292]: cluster 2023-08-28T00:49:57.165865+0000 mgr.y (mgr.14901) 5141 : cluster [DBG] pgmap v5148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:58.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:49:58 smithi006 bash[27292]: audit 2023-08-28T00:49:58.303941+0000 mon.a (mon.0) 3602 : audit [DBG] from='client.? 172.21.15.6:0/3872099244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:58.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:58 smithi006 bash[33918]: cluster 2023-08-28T00:49:57.165865+0000 mgr.y (mgr.14901) 5141 : cluster [DBG] pgmap v5148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:58.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:49:58 smithi006 bash[33918]: audit 2023-08-28T00:49:58.303941+0000 mon.a (mon.0) 3602 : audit [DBG] from='client.? 172.21.15.6:0/3872099244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:58 smithi130 bash[27579]: cluster 2023-08-28T00:49:57.165865+0000 mgr.y (mgr.14901) 5141 : cluster [DBG] pgmap v5148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:49:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:49:58 smithi130 bash[27579]: audit 2023-08-28T00:49:58.303941+0000 mon.a (mon.0) 3602 : audit [DBG] from='client.? 172.21.15.6:0/3872099244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:58 smithi130 bash[51302]: debug 2023-08-28T00:49:58.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:58.917726+0000) 2023-08-28T00:50:00.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:49:59 smithi130 bash[51302]: debug 2023-08-28T00:49:59.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:59.918031+0000) 2023-08-28T00:50:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:00 smithi006 bash[27292]: cluster 2023-08-28T00:49:59.167121+0000 mgr.y (mgr.14901) 5142 : cluster [DBG] pgmap v5149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:00 smithi006 bash[27292]: audit 2023-08-28T00:49:59.660971+0000 mon.b (mon.1) 613 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:50:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:00 smithi006 bash[27292]: cluster 2023-08-28T00:50:00.000156+0000 mon.a (mon.0) 3603 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:50:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:00 smithi006 bash[33918]: cluster 2023-08-28T00:49:59.167121+0000 mgr.y (mgr.14901) 5142 : cluster [DBG] pgmap v5149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:00 smithi006 bash[33918]: audit 2023-08-28T00:49:59.660971+0000 mon.b (mon.1) 613 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:50:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:00 smithi006 bash[33918]: cluster 2023-08-28T00:50:00.000156+0000 mon.a (mon.0) 3603 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:50:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:00 smithi130 bash[27579]: cluster 2023-08-28T00:49:59.167121+0000 mgr.y (mgr.14901) 5142 : cluster [DBG] pgmap v5149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:00 smithi130 bash[27579]: audit 2023-08-28T00:49:59.660971+0000 mon.b (mon.1) 613 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:50:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:00 smithi130 bash[27579]: cluster 2023-08-28T00:50:00.000156+0000 mon.a (mon.0) 3603 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:50:01.219 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:50:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:50:00] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:50:01.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:50:00.844Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:01.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:50:00.846Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:01.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:00 smithi130 bash[51302]: debug 2023-08-28T00:50:00.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:00.918315+0000) 2023-08-28T00:50:01.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:01 smithi006 bash[27292]: audit 2023-08-28T00:50:00.723547+0000 mon.c (mon.2) 1996 : audit [DBG] from='client.? 172.21.15.6:0/3169516539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:01.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:01 smithi006 bash[33918]: audit 2023-08-28T00:50:00.723547+0000 mon.c (mon.2) 1996 : audit [DBG] from='client.? 172.21.15.6:0/3169516539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:01.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:01 smithi130 bash[27579]: audit 2023-08-28T00:50:00.723547+0000 mon.c (mon.2) 1996 : audit [DBG] from='client.? 172.21.15.6:0/3169516539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:02.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:01 smithi130 bash[51302]: debug 2023-08-28T00:50:01.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:01.918528+0000) 2023-08-28T00:50:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:02 smithi006 bash[27292]: cluster 2023-08-28T00:50:01.167822+0000 mgr.y (mgr.14901) 5143 : cluster [DBG] pgmap v5150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:02 smithi006 bash[33918]: cluster 2023-08-28T00:50:01.167822+0000 mgr.y (mgr.14901) 5143 : cluster [DBG] pgmap v5150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:50:02.419Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:50:02.421Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:02 smithi130 bash[27579]: cluster 2023-08-28T00:50:01.167822+0000 mgr.y (mgr.14901) 5143 : cluster [DBG] pgmap v5150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:02 smithi130 bash[51302]: debug 2023-08-28T00:50:02.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:02.918805+0000) 2023-08-28T00:50:03.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:03 smithi006 bash[27292]: audit 2023-08-28T00:50:03.154187+0000 mon.c (mon.2) 1997 : audit [DBG] from='client.? 172.21.15.6:0/4028029756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:03.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:03 smithi006 bash[33918]: audit 2023-08-28T00:50:03.154187+0000 mon.c (mon.2) 1997 : audit [DBG] from='client.? 172.21.15.6:0/4028029756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:03.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:03 smithi130 bash[27579]: audit 2023-08-28T00:50:03.154187+0000 mon.c (mon.2) 1997 : audit [DBG] from='client.? 172.21.15.6:0/4028029756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:03 smithi130 bash[51302]: debug 2023-08-28T00:50:03.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:03.919114+0000) 2023-08-28T00:50:04.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:04 smithi006 bash[33918]: cluster 2023-08-28T00:50:03.168466+0000 mgr.y (mgr.14901) 5144 : cluster [DBG] pgmap v5151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:04.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:04 smithi006 bash[27292]: cluster 2023-08-28T00:50:03.168466+0000 mgr.y (mgr.14901) 5144 : cluster [DBG] pgmap v5151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:04.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:50:04.299Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:04.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:50:04.301Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:04 smithi130 bash[27579]: cluster 2023-08-28T00:50:03.168466+0000 mgr.y (mgr.14901) 5144 : cluster [DBG] pgmap v5151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:04 smithi130 bash[51302]: debug 2023-08-28T00:50:04.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:04.919299+0000) 2023-08-28T00:50:06.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:05 smithi130 bash[51302]: debug 2023-08-28T00:50:05.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:05.919486+0000) 2023-08-28T00:50:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:06 smithi130 bash[27579]: cluster 2023-08-28T00:50:05.169603+0000 mgr.y (mgr.14901) 5145 : cluster [DBG] pgmap v5152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:06 smithi130 bash[27579]: audit 2023-08-28T00:50:05.564192+0000 mon.a (mon.0) 3604 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:50:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:06 smithi130 bash[27579]: audit 2023-08-28T00:50:05.614823+0000 mon.a (mon.0) 3605 : audit [DBG] from='client.? 172.21.15.6:0/1979180600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:06 smithi130 bash[27579]: audit 2023-08-28T00:50:05.994320+0000 mon.a (mon.0) 3606 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:50:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:06 smithi130 bash[27579]: audit 2023-08-28T00:50:06.254432+0000 mon.b (mon.1) 614 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:50:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:06 smithi130 bash[27579]: audit 2023-08-28T00:50:06.256560+0000 mon.b (mon.1) 615 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:50:06.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:06 smithi130 bash[27579]: audit 2023-08-28T00:50:06.265378+0000 mon.a (mon.0) 3607 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:50:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:06 smithi006 bash[27292]: cluster 2023-08-28T00:50:05.169603+0000 mgr.y (mgr.14901) 5145 : cluster [DBG] pgmap v5152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:06 smithi006 bash[27292]: audit 2023-08-28T00:50:05.564192+0000 mon.a (mon.0) 3604 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:50:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:06 smithi006 bash[27292]: audit 2023-08-28T00:50:05.614823+0000 mon.a (mon.0) 3605 : audit [DBG] from='client.? 172.21.15.6:0/1979180600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:06 smithi006 bash[27292]: audit 2023-08-28T00:50:05.994320+0000 mon.a (mon.0) 3606 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:50:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:06 smithi006 bash[27292]: audit 2023-08-28T00:50:06.254432+0000 mon.b (mon.1) 614 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:50:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:06 smithi006 bash[27292]: audit 2023-08-28T00:50:06.256560+0000 mon.b (mon.1) 615 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:50:06.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:06 smithi006 bash[27292]: audit 2023-08-28T00:50:06.265378+0000 mon.a (mon.0) 3607 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:50:06.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:06 smithi006 bash[33918]: cluster 2023-08-28T00:50:05.169603+0000 mgr.y (mgr.14901) 5145 : cluster [DBG] pgmap v5152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:06.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:06 smithi006 bash[33918]: audit 2023-08-28T00:50:05.564192+0000 mon.a (mon.0) 3604 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:50:06.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:06 smithi006 bash[33918]: audit 2023-08-28T00:50:05.614823+0000 mon.a (mon.0) 3605 : audit [DBG] from='client.? 172.21.15.6:0/1979180600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:06.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:06 smithi006 bash[33918]: audit 2023-08-28T00:50:05.994320+0000 mon.a (mon.0) 3606 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:50:06.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:06 smithi006 bash[33918]: audit 2023-08-28T00:50:06.254432+0000 mon.b (mon.1) 614 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:50:06.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:06 smithi006 bash[33918]: audit 2023-08-28T00:50:06.256560+0000 mon.b (mon.1) 615 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:50:06.969 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:06 smithi006 bash[33918]: audit 2023-08-28T00:50:06.265378+0000 mon.a (mon.0) 3607 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:50:07.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:06 smithi130 bash[51302]: debug 2023-08-28T00:50:06.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:06.919775+0000) 2023-08-28T00:50:08.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:07 smithi130 bash[51302]: debug 2023-08-28T00:50:07.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:07.920047+0000) 2023-08-28T00:50:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:08 smithi130 bash[27579]: cluster 2023-08-28T00:50:07.170247+0000 mgr.y (mgr.14901) 5146 : cluster [DBG] pgmap v5153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:08 smithi130 bash[27579]: audit 2023-08-28T00:50:08.034389+0000 mon.c (mon.2) 1998 : audit [DBG] from='client.? 172.21.15.6:0/3212549949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:08 smithi006 bash[27292]: cluster 2023-08-28T00:50:07.170247+0000 mgr.y (mgr.14901) 5146 : cluster [DBG] pgmap v5153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:08.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:08 smithi006 bash[27292]: audit 2023-08-28T00:50:08.034389+0000 mon.c (mon.2) 1998 : audit [DBG] from='client.? 172.21.15.6:0/3212549949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:08.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:08 smithi006 bash[33918]: cluster 2023-08-28T00:50:07.170247+0000 mgr.y (mgr.14901) 5146 : cluster [DBG] pgmap v5153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:08.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:08 smithi006 bash[33918]: audit 2023-08-28T00:50:08.034389+0000 mon.c (mon.2) 1998 : audit [DBG] from='client.? 172.21.15.6:0/3212549949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:09.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:08 smithi130 bash[51302]: debug 2023-08-28T00:50:08.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:08.920334+0000) 2023-08-28T00:50:10.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:09 smithi130 bash[51302]: debug 2023-08-28T00:50:09.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:09.920647+0000) 2023-08-28T00:50:10.848 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:50:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:50:10] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:50:10.848 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:10 smithi006 bash[33918]: cluster 2023-08-28T00:50:09.171334+0000 mgr.y (mgr.14901) 5147 : cluster [DBG] pgmap v5154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:10.848 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:10 smithi006 bash[33918]: audit 2023-08-28T00:50:10.450045+0000 mon.a (mon.0) 3608 : audit [DBG] from='client.? 172.21.15.6:0/1549445628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:10.849 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:10 smithi006 bash[27292]: cluster 2023-08-28T00:50:09.171334+0000 mgr.y (mgr.14901) 5147 : cluster [DBG] pgmap v5154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:10.849 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:10 smithi006 bash[27292]: audit 2023-08-28T00:50:10.450045+0000 mon.a (mon.0) 3608 : audit [DBG] from='client.? 172.21.15.6:0/1549445628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:10 smithi130 bash[27579]: cluster 2023-08-28T00:50:09.171334+0000 mgr.y (mgr.14901) 5147 : cluster [DBG] pgmap v5154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:10 smithi130 bash[27579]: audit 2023-08-28T00:50:10.450045+0000 mon.a (mon.0) 3608 : audit [DBG] from='client.? 172.21.15.6:0/1549445628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:50:10.844Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:50:10.847Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:11.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:10 smithi130 bash[51302]: debug 2023-08-28T00:50:10.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:10.920929+0000) 2023-08-28T00:50:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:11 smithi130 bash[51302]: debug 2023-08-28T00:50:11.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:11.921205+0000) 2023-08-28T00:50:12.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:12 smithi006 bash[27292]: cluster 2023-08-28T00:50:11.171933+0000 mgr.y (mgr.14901) 5148 : cluster [DBG] pgmap v5155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:12 smithi006 bash[33918]: cluster 2023-08-28T00:50:11.171933+0000 mgr.y (mgr.14901) 5148 : cluster [DBG] pgmap v5155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:50:12.420Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:50:12.422Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:12 smithi130 bash[27579]: cluster 2023-08-28T00:50:11.171933+0000 mgr.y (mgr.14901) 5148 : cluster [DBG] pgmap v5155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:12 smithi130 bash[51302]: debug 2023-08-28T00:50:12.921+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:12.921533+0000) 2023-08-28T00:50:13.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:13 smithi130 bash[27579]: audit 2023-08-28T00:50:12.869308+0000 mon.c (mon.2) 1999 : audit [DBG] from='client.? 172.21.15.6:0/1449060070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:13.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:13 smithi006 bash[33918]: audit 2023-08-28T00:50:12.869308+0000 mon.c (mon.2) 1999 : audit [DBG] from='client.? 172.21.15.6:0/1449060070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:13.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:13 smithi006 bash[27292]: audit 2023-08-28T00:50:12.869308+0000 mon.c (mon.2) 1999 : audit [DBG] from='client.? 172.21.15.6:0/1449060070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:14.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:13 smithi130 bash[51302]: debug 2023-08-28T00:50:13.921+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:13.921830+0000) 2023-08-28T00:50:14.593 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:50:14.300Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:14.593 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:50:14.302Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:14 smithi130 bash[27579]: cluster 2023-08-28T00:50:13.172613+0000 mgr.y (mgr.14901) 5149 : cluster [DBG] pgmap v5156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:14 smithi006 bash[27292]: cluster 2023-08-28T00:50:13.172613+0000 mgr.y (mgr.14901) 5149 : cluster [DBG] pgmap v5156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:14.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:14 smithi006 bash[33918]: cluster 2023-08-28T00:50:13.172613+0000 mgr.y (mgr.14901) 5149 : cluster [DBG] pgmap v5156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:14 smithi130 bash[51302]: debug 2023-08-28T00:50:14.916+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:14.922179+0000) 2023-08-28T00:50:15.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:15 smithi130 bash[27579]: audit 2023-08-28T00:50:15.288465+0000 mon.a (mon.0) 3609 : audit [DBG] from='client.? 172.21.15.6:0/1201848038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:15.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:15 smithi006 bash[27292]: audit 2023-08-28T00:50:15.288465+0000 mon.a (mon.0) 3609 : audit [DBG] from='client.? 172.21.15.6:0/1201848038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:15.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:15 smithi006 bash[33918]: audit 2023-08-28T00:50:15.288465+0000 mon.a (mon.0) 3609 : audit [DBG] from='client.? 172.21.15.6:0/1201848038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:16.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:15 smithi130 bash[51302]: debug 2023-08-28T00:50:15.916+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:15.922508+0000) 2023-08-28T00:50:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:16 smithi130 bash[27579]: cluster 2023-08-28T00:50:15.173671+0000 mgr.y (mgr.14901) 5150 : cluster [DBG] pgmap v5157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:16.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:16 smithi006 bash[33918]: cluster 2023-08-28T00:50:15.173671+0000 mgr.y (mgr.14901) 5150 : cluster [DBG] pgmap v5157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:16.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:16 smithi006 bash[27292]: cluster 2023-08-28T00:50:15.173671+0000 mgr.y (mgr.14901) 5150 : cluster [DBG] pgmap v5157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:17.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:16 smithi130 bash[51302]: debug 2023-08-28T00:50:16.916+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:16.922792+0000) 2023-08-28T00:50:18.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:17 smithi130 bash[51302]: debug 2023-08-28T00:50:17.920+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:17.923070+0000) 2023-08-28T00:50:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:18 smithi130 bash[27579]: cluster 2023-08-28T00:50:17.174322+0000 mgr.y (mgr.14901) 5151 : cluster [DBG] pgmap v5158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:18 smithi130 bash[27579]: audit 2023-08-28T00:50:17.700199+0000 mon.a (mon.0) 3610 : audit [DBG] from='client.? 172.21.15.6:0/3966609497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:18 smithi006 bash[33918]: cluster 2023-08-28T00:50:17.174322+0000 mgr.y (mgr.14901) 5151 : cluster [DBG] pgmap v5158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:18.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:18 smithi006 bash[33918]: audit 2023-08-28T00:50:17.700199+0000 mon.a (mon.0) 3610 : audit [DBG] from='client.? 172.21.15.6:0/3966609497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:18.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:18 smithi006 bash[27292]: cluster 2023-08-28T00:50:17.174322+0000 mgr.y (mgr.14901) 5151 : cluster [DBG] pgmap v5158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:18.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:18 smithi006 bash[27292]: audit 2023-08-28T00:50:17.700199+0000 mon.a (mon.0) 3610 : audit [DBG] from='client.? 172.21.15.6:0/3966609497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:19.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:18 smithi130 bash[51302]: debug 2023-08-28T00:50:18.920+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:18.923342+0000) 2023-08-28T00:50:20.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:19 smithi130 bash[51302]: debug 2023-08-28T00:50:19.920+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:19.923678+0000) 2023-08-28T00:50:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:20 smithi130 bash[27579]: cluster 2023-08-28T00:50:19.175427+0000 mgr.y (mgr.14901) 5152 : cluster [DBG] pgmap v5159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:20 smithi130 bash[27579]: audit 2023-08-28T00:50:20.120964+0000 mon.c (mon.2) 2000 : audit [DBG] from='client.? 172.21.15.6:0/293013363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:20.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:50:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:50:20] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:50:20.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:20 smithi006 bash[27292]: cluster 2023-08-28T00:50:19.175427+0000 mgr.y (mgr.14901) 5152 : cluster [DBG] pgmap v5159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:20.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:20 smithi006 bash[27292]: audit 2023-08-28T00:50:20.120964+0000 mon.c (mon.2) 2000 : audit [DBG] from='client.? 172.21.15.6:0/293013363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:20.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:20 smithi006 bash[33918]: cluster 2023-08-28T00:50:19.175427+0000 mgr.y (mgr.14901) 5152 : cluster [DBG] pgmap v5159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:20.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:20 smithi006 bash[33918]: audit 2023-08-28T00:50:20.120964+0000 mon.c (mon.2) 2000 : audit [DBG] from='client.? 172.21.15.6:0/293013363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:20.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:50:20.845Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:20.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:50:20.847Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:21.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:20 smithi130 bash[51302]: debug 2023-08-28T00:50:20.920+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:20.923980+0000) 2023-08-28T00:50:22.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:21 smithi130 bash[51302]: debug 2023-08-28T00:50:21.920+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:21.924316+0000) 2023-08-28T00:50:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:22 smithi006 bash[33918]: cluster 2023-08-28T00:50:21.176030+0000 mgr.y (mgr.14901) 5153 : cluster [DBG] pgmap v5160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:22 smithi006 bash[33918]: audit 2023-08-28T00:50:22.539587+0000 mon.c (mon.2) 2001 : audit [DBG] from='client.? 172.21.15.6:0/1129452185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:22 smithi006 bash[27292]: cluster 2023-08-28T00:50:21.176030+0000 mgr.y (mgr.14901) 5153 : cluster [DBG] pgmap v5160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:22 smithi006 bash[27292]: audit 2023-08-28T00:50:22.539587+0000 mon.c (mon.2) 2001 : audit [DBG] from='client.? 172.21.15.6:0/1129452185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:22.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:50:22.421Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:22.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:50:22.423Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:22 smithi130 bash[27579]: cluster 2023-08-28T00:50:21.176030+0000 mgr.y (mgr.14901) 5153 : cluster [DBG] pgmap v5160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:22.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:22 smithi130 bash[27579]: audit 2023-08-28T00:50:22.539587+0000 mon.c (mon.2) 2001 : audit [DBG] from='client.? 172.21.15.6:0/1129452185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:23.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:22 smithi130 bash[51302]: debug 2023-08-28T00:50:22.920+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:22.924669+0000) 2023-08-28T00:50:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:23 smithi130 bash[51302]: debug 2023-08-28T00:50:23.920+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:23.924959+0000) 2023-08-28T00:50:24.621 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:50:24.301Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:24.621 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:50:24.304Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:24.926 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:24 smithi130 bash[27579]: cluster 2023-08-28T00:50:23.176674+0000 mgr.y (mgr.14901) 5154 : cluster [DBG] pgmap v5161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:25.334 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:24 smithi006 bash[33918]: cluster 2023-08-28T00:50:23.176674+0000 mgr.y (mgr.14901) 5154 : cluster [DBG] pgmap v5161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:25.334 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:24 smithi006 bash[27292]: cluster 2023-08-28T00:50:23.176674+0000 mgr.y (mgr.14901) 5154 : cluster [DBG] pgmap v5161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:24 smithi130 bash[51302]: debug 2023-08-28T00:50:24.920+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:24.925267+0000) 2023-08-28T00:50:25.926 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:25 smithi130 bash[27579]: audit 2023-08-28T00:50:24.967728+0000 mon.c (mon.2) 2002 : audit [DBG] from='client.? 172.21.15.6:0/3694711978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:26.353 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:25 smithi006 bash[33918]: audit 2023-08-28T00:50:24.967728+0000 mon.c (mon.2) 2002 : audit [DBG] from='client.? 172.21.15.6:0/3694711978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:26.353 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:25 smithi006 bash[27292]: audit 2023-08-28T00:50:24.967728+0000 mon.c (mon.2) 2002 : audit [DBG] from='client.? 172.21.15.6:0/3694711978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:26.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:25 smithi130 bash[51302]: debug 2023-08-28T00:50:25.920+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:25.925528+0000) 2023-08-28T00:50:26.927 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:26 smithi130 bash[27579]: cluster 2023-08-28T00:50:25.177802+0000 mgr.y (mgr.14901) 5155 : cluster [DBG] pgmap v5162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:27.202 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:26 smithi006 bash[33918]: cluster 2023-08-28T00:50:25.177802+0000 mgr.y (mgr.14901) 5155 : cluster [DBG] pgmap v5162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:27.202 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:26 smithi006 bash[27292]: cluster 2023-08-28T00:50:25.177802+0000 mgr.y (mgr.14901) 5155 : cluster [DBG] pgmap v5162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:27.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:26 smithi130 bash[51302]: debug 2023-08-28T00:50:26.920+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:26.925823+0000) 2023-08-28T00:50:27.927 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:27 smithi130 bash[27579]: audit 2023-08-28T00:50:27.383071+0000 mon.a (mon.0) 3611 : audit [DBG] from='client.? 172.21.15.6:0/3706346977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:27.988 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:27 smithi006 bash[27292]: audit 2023-08-28T00:50:27.383071+0000 mon.a (mon.0) 3611 : audit [DBG] from='client.? 172.21.15.6:0/3706346977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:27.988 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:27 smithi006 bash[33918]: audit 2023-08-28T00:50:27.383071+0000 mon.a (mon.0) 3611 : audit [DBG] from='client.? 172.21.15.6:0/3706346977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:28.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:27 smithi130 bash[51302]: debug 2023-08-28T00:50:27.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:27.926092+0000) 2023-08-28T00:50:28.928 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:28 smithi130 bash[27579]: cluster 2023-08-28T00:50:27.178441+0000 mgr.y (mgr.14901) 5156 : cluster [DBG] pgmap v5163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:29.136 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:28 smithi006 bash[27292]: cluster 2023-08-28T00:50:27.178441+0000 mgr.y (mgr.14901) 5156 : cluster [DBG] pgmap v5163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:29.198 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:28 smithi006 bash[33918]: cluster 2023-08-28T00:50:27.178441+0000 mgr.y (mgr.14901) 5156 : cluster [DBG] pgmap v5163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:29.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:28 smithi130 bash[51302]: debug 2023-08-28T00:50:28.920+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:28.926396+0000) 2023-08-28T00:50:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:29 smithi130 bash[51302]: debug 2023-08-28T00:50:29.920+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:29.926687+0000) 2023-08-28T00:50:30.928 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:30 smithi130 bash[27579]: cluster 2023-08-28T00:50:29.179464+0000 mgr.y (mgr.14901) 5157 : cluster [DBG] pgmap v5164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:30.929 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:30 smithi130 bash[27579]: audit 2023-08-28T00:50:29.801473+0000 mon.a (mon.0) 3612 : audit [DBG] from='client.? 172.21.15.6:0/9548948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:31.062 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:50:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:50:30] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:50:31.410 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:50:30.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:31.410 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:31 smithi006 bash[56298]: level=warn ts=2023-08-28T00:50:30.848Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:31.411 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:30 smithi006 bash[33918]: cluster 2023-08-28T00:50:29.179464+0000 mgr.y (mgr.14901) 5157 : cluster [DBG] pgmap v5164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:31.416 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:30 smithi006 bash[33918]: audit 2023-08-28T00:50:29.801473+0000 mon.a (mon.0) 3612 : audit [DBG] from='client.? 172.21.15.6:0/9548948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:31.416 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:30 smithi006 bash[27292]: cluster 2023-08-28T00:50:29.179464+0000 mgr.y (mgr.14901) 5157 : cluster [DBG] pgmap v5164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:31.417 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:30 smithi006 bash[27292]: audit 2023-08-28T00:50:29.801473+0000 mon.a (mon.0) 3612 : audit [DBG] from='client.? 172.21.15.6:0/9548948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:31.417 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:30 smithi130 bash[51302]: debug 2023-08-28T00:50:30.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:30.926980+0000) 2023-08-28T00:50:32.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:31 smithi130 bash[51302]: debug 2023-08-28T00:50:31.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:31.927251+0000) 2023-08-28T00:50:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:32 smithi006 bash[33918]: cluster 2023-08-28T00:50:31.180079+0000 mgr.y (mgr.14901) 5158 : cluster [DBG] pgmap v5165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:33.284 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:32 smithi006 bash[33918]: audit 2023-08-28T00:50:32.215961+0000 mon.a (mon.0) 3613 : audit [DBG] from='client.? 172.21.15.6:0/878500950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:33.285 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:32 smithi006 bash[27292]: cluster 2023-08-28T00:50:31.180079+0000 mgr.y (mgr.14901) 5158 : cluster [DBG] pgmap v5165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:33.285 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:32 smithi006 bash[27292]: audit 2023-08-28T00:50:32.215961+0000 mon.a (mon.0) 3613 : audit [DBG] from='client.? 172.21.15.6:0/878500950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:33.285 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:50:32.422Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:33.285 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:50:32.424Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:33.286 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:32 smithi130 bash[27579]: cluster 2023-08-28T00:50:31.180079+0000 mgr.y (mgr.14901) 5158 : cluster [DBG] pgmap v5165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:33.286 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:32 smithi130 bash[27579]: audit 2023-08-28T00:50:32.215961+0000 mon.a (mon.0) 3613 : audit [DBG] from='client.? 172.21.15.6:0/878500950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:33.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:32 smithi130 bash[51302]: debug 2023-08-28T00:50:32.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:32.927542+0000) 2023-08-28T00:50:34.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:33 smithi130 bash[51302]: debug 2023-08-28T00:50:33.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:33.927858+0000) 2023-08-28T00:50:35.055 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:50:34.302Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:35.055 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:50:34.304Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:35.055 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:34 smithi130 bash[27579]: cluster 2023-08-28T00:50:33.180819+0000 mgr.y (mgr.14901) 5159 : cluster [DBG] pgmap v5166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:35.055 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:34 smithi130 bash[27579]: audit 2023-08-28T00:50:34.631950+0000 mon.a (mon.0) 3614 : audit [DBG] from='client.? 172.21.15.6:0/1438703984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:35.056 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:34 smithi006 bash[27292]: cluster 2023-08-28T00:50:33.180819+0000 mgr.y (mgr.14901) 5159 : cluster [DBG] pgmap v5166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:35.056 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:34 smithi006 bash[27292]: audit 2023-08-28T00:50:34.631950+0000 mon.a (mon.0) 3614 : audit [DBG] from='client.? 172.21.15.6:0/1438703984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:35.056 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:34 smithi006 bash[33918]: cluster 2023-08-28T00:50:33.180819+0000 mgr.y (mgr.14901) 5159 : cluster [DBG] pgmap v5166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:35.056 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:34 smithi006 bash[33918]: audit 2023-08-28T00:50:34.631950+0000 mon.a (mon.0) 3614 : audit [DBG] from='client.? 172.21.15.6:0/1438703984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:35.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:34 smithi130 bash[51302]: debug 2023-08-28T00:50:34.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:34.928145+0000) 2023-08-28T00:50:36.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:35 smithi130 bash[51302]: debug 2023-08-28T00:50:35.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:35.928478+0000) 2023-08-28T00:50:36.930 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:36 smithi130 bash[27579]: cluster 2023-08-28T00:50:35.182092+0000 mgr.y (mgr.14901) 5160 : cluster [DBG] pgmap v5167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:36 smithi006 bash[33918]: cluster 2023-08-28T00:50:35.182092+0000 mgr.y (mgr.14901) 5160 : cluster [DBG] pgmap v5167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:36.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:36 smithi006 bash[27292]: cluster 2023-08-28T00:50:35.182092+0000 mgr.y (mgr.14901) 5160 : cluster [DBG] pgmap v5167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:36 smithi130 bash[51302]: debug 2023-08-28T00:50:36.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:36.928752+0000) 2023-08-28T00:50:37.930 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:37 smithi130 bash[27579]: audit 2023-08-28T00:50:37.050048+0000 mon.a (mon.0) 3615 : audit [DBG] from='client.? 172.21.15.6:0/2723082502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:37.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:37 smithi006 bash[27292]: audit 2023-08-28T00:50:37.050048+0000 mon.a (mon.0) 3615 : audit [DBG] from='client.? 172.21.15.6:0/2723082502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:37.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:37 smithi006 bash[33918]: audit 2023-08-28T00:50:37.050048+0000 mon.a (mon.0) 3615 : audit [DBG] from='client.? 172.21.15.6:0/2723082502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:38.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:37 smithi130 bash[51302]: debug 2023-08-28T00:50:37.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:37.929058+0000) 2023-08-28T00:50:38.931 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:38 smithi130 bash[27579]: cluster 2023-08-28T00:50:37.182770+0000 mgr.y (mgr.14901) 5161 : cluster [DBG] pgmap v5168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:38 smithi006 bash[33918]: cluster 2023-08-28T00:50:37.182770+0000 mgr.y (mgr.14901) 5161 : cluster [DBG] pgmap v5168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:38.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:38 smithi006 bash[27292]: cluster 2023-08-28T00:50:37.182770+0000 mgr.y (mgr.14901) 5161 : cluster [DBG] pgmap v5168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:39.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:38 smithi130 bash[51302]: debug 2023-08-28T00:50:38.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:38.929348+0000) 2023-08-28T00:50:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:39 smithi006 bash[33918]: audit 2023-08-28T00:50:39.466471+0000 mon.a (mon.0) 3616 : audit [DBG] from='client.? 172.21.15.6:0/2993294114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:39.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:39 smithi006 bash[27292]: audit 2023-08-28T00:50:39.466471+0000 mon.a (mon.0) 3616 : audit [DBG] from='client.? 172.21.15.6:0/2993294114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:40.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:39 smithi130 bash[27579]: audit 2023-08-28T00:50:39.466471+0000 mon.a (mon.0) 3616 : audit [DBG] from='client.? 172.21.15.6:0/2993294114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:40.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:39 smithi130 bash[51302]: debug 2023-08-28T00:50:39.928+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:39.929634+0000) 2023-08-28T00:50:40.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:50:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:50:40] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:50:40.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:40 smithi006 bash[33918]: cluster 2023-08-28T00:50:39.183929+0000 mgr.y (mgr.14901) 5162 : cluster [DBG] pgmap v5169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:40.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:40 smithi006 bash[27292]: cluster 2023-08-28T00:50:39.183929+0000 mgr.y (mgr.14901) 5162 : cluster [DBG] pgmap v5169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:50:40.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:40.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:50:40.849Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:41.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:40 smithi130 bash[27579]: cluster 2023-08-28T00:50:39.183929+0000 mgr.y (mgr.14901) 5162 : cluster [DBG] pgmap v5169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:41.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:40 smithi130 bash[51302]: debug 2023-08-28T00:50:40.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:40.929934+0000) 2023-08-28T00:50:42.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:41 smithi130 bash[51302]: debug 2023-08-28T00:50:41.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:41.930158+0000) 2023-08-28T00:50:42.695 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:50:42.422Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:42.695 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:50:42.425Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:42.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:42 smithi006 bash[27292]: cluster 2023-08-28T00:50:41.184751+0000 mgr.y (mgr.14901) 5163 : cluster [DBG] pgmap v5170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:42.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:42 smithi006 bash[27292]: audit 2023-08-28T00:50:41.887251+0000 mon.a (mon.0) 3617 : audit [DBG] from='client.? 172.21.15.6:0/465028595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:42.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:42 smithi006 bash[33918]: cluster 2023-08-28T00:50:41.184751+0000 mgr.y (mgr.14901) 5163 : cluster [DBG] pgmap v5170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:42.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:42 smithi006 bash[33918]: audit 2023-08-28T00:50:41.887251+0000 mon.a (mon.0) 3617 : audit [DBG] from='client.? 172.21.15.6:0/465028595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:42 smithi130 bash[27579]: cluster 2023-08-28T00:50:41.184751+0000 mgr.y (mgr.14901) 5163 : cluster [DBG] pgmap v5170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:43.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:42 smithi130 bash[27579]: audit 2023-08-28T00:50:41.887251+0000 mon.a (mon.0) 3617 : audit [DBG] from='client.? 172.21.15.6:0/465028595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:43.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:42 smithi130 bash[51302]: debug 2023-08-28T00:50:42.928+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:42.930434+0000) 2023-08-28T00:50:44.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:43 smithi130 bash[51302]: debug 2023-08-28T00:50:43.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:43.930743+0000) 2023-08-28T00:50:44.699 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:50:44.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:44.699 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:50:44.304Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:44.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:44 smithi006 bash[33918]: cluster 2023-08-28T00:50:43.185413+0000 mgr.y (mgr.14901) 5164 : cluster [DBG] pgmap v5171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:44.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:44 smithi006 bash[33918]: audit 2023-08-28T00:50:44.302192+0000 mon.c (mon.2) 2003 : audit [DBG] from='client.? 172.21.15.6:0/341633154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:44.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:44 smithi006 bash[27292]: cluster 2023-08-28T00:50:43.185413+0000 mgr.y (mgr.14901) 5164 : cluster [DBG] pgmap v5171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:44.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:44 smithi006 bash[27292]: audit 2023-08-28T00:50:44.302192+0000 mon.c (mon.2) 2003 : audit [DBG] from='client.? 172.21.15.6:0/341633154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:45.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:44 smithi130 bash[51302]: debug 2023-08-28T00:50:44.928+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:44.931034+0000) 2023-08-28T00:50:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:44 smithi130 bash[27579]: cluster 2023-08-28T00:50:43.185413+0000 mgr.y (mgr.14901) 5164 : cluster [DBG] pgmap v5171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:44 smithi130 bash[27579]: audit 2023-08-28T00:50:44.302192+0000 mon.c (mon.2) 2003 : audit [DBG] from='client.? 172.21.15.6:0/341633154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:45 smithi130 bash[51302]: debug 2023-08-28T00:50:45.928+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:45.931315+0000) 2023-08-28T00:50:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:46 smithi006 bash[33918]: cluster 2023-08-28T00:50:45.186541+0000 mgr.y (mgr.14901) 5165 : cluster [DBG] pgmap v5172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:46.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:46 smithi006 bash[27292]: cluster 2023-08-28T00:50:45.186541+0000 mgr.y (mgr.14901) 5165 : cluster [DBG] pgmap v5172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:47.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:46 smithi130 bash[27579]: cluster 2023-08-28T00:50:45.186541+0000 mgr.y (mgr.14901) 5165 : cluster [DBG] pgmap v5172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:47.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:46 smithi130 bash[51302]: debug 2023-08-28T00:50:46.928+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:46.931634+0000) 2023-08-28T00:50:47.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:47 smithi006 bash[33918]: audit 2023-08-28T00:50:46.717076+0000 mon.a (mon.0) 3618 : audit [DBG] from='client.? 172.21.15.6:0/3326821122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:47.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:47 smithi006 bash[27292]: audit 2023-08-28T00:50:46.717076+0000 mon.a (mon.0) 3618 : audit [DBG] from='client.? 172.21.15.6:0/3326821122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:48.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:47 smithi130 bash[27579]: audit 2023-08-28T00:50:46.717076+0000 mon.a (mon.0) 3618 : audit [DBG] from='client.? 172.21.15.6:0/3326821122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:48.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:47 smithi130 bash[51302]: debug 2023-08-28T00:50:47.928+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:47.931917+0000) 2023-08-28T00:50:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:48 smithi006 bash[27292]: cluster 2023-08-28T00:50:47.187216+0000 mgr.y (mgr.14901) 5166 : cluster [DBG] pgmap v5173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:48.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:48 smithi006 bash[33918]: cluster 2023-08-28T00:50:47.187216+0000 mgr.y (mgr.14901) 5166 : cluster [DBG] pgmap v5173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:48 smithi130 bash[27579]: cluster 2023-08-28T00:50:47.187216+0000 mgr.y (mgr.14901) 5166 : cluster [DBG] pgmap v5173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:49.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:48 smithi130 bash[51302]: debug 2023-08-28T00:50:48.928+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:48.932249+0000) 2023-08-28T00:50:50.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:49 smithi130 bash[27579]: audit 2023-08-28T00:50:49.131720+0000 mon.c (mon.2) 2004 : audit [DBG] from='client.? 172.21.15.6:0/3486179408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:50.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:49 smithi130 bash[51302]: debug 2023-08-28T00:50:49.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:49.932595+0000) 2023-08-28T00:50:50.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:49 smithi006 bash[27292]: audit 2023-08-28T00:50:49.131720+0000 mon.c (mon.2) 2004 : audit [DBG] from='client.? 172.21.15.6:0/3486179408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:50.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:49 smithi006 bash[33918]: audit 2023-08-28T00:50:49.131720+0000 mon.c (mon.2) 2004 : audit [DBG] from='client.? 172.21.15.6:0/3486179408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:50 smithi130 bash[27579]: cluster 2023-08-28T00:50:49.188429+0000 mgr.y (mgr.14901) 5167 : cluster [DBG] pgmap v5174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:51.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:50 smithi130 bash[51302]: debug 2023-08-28T00:50:50.928+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:50.932877+0000) 2023-08-28T00:50:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:50 smithi006 bash[27292]: cluster 2023-08-28T00:50:49.188429+0000 mgr.y (mgr.14901) 5167 : cluster [DBG] pgmap v5174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:51.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:50:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:50:50] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:50:51.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:50 smithi006 bash[33918]: cluster 2023-08-28T00:50:49.188429+0000 mgr.y (mgr.14901) 5167 : cluster [DBG] pgmap v5174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:50:50.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:50:50.850Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:52.234 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:51 smithi130 bash[51302]: debug 2023-08-28T00:50:51.928+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:51.933153+0000) 2023-08-28T00:50:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:52 smithi130 bash[27579]: cluster 2023-08-28T00:50:51.189266+0000 mgr.y (mgr.14901) 5168 : cluster [DBG] pgmap v5175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:52 smithi130 bash[27579]: audit 2023-08-28T00:50:51.547270+0000 mon.c (mon.2) 2005 : audit [DBG] from='client.? 172.21.15.6:0/2960807718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:52 smithi006 bash[27292]: cluster 2023-08-28T00:50:51.189266+0000 mgr.y (mgr.14901) 5168 : cluster [DBG] pgmap v5175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:52 smithi006 bash[27292]: audit 2023-08-28T00:50:51.547270+0000 mon.c (mon.2) 2005 : audit [DBG] from='client.? 172.21.15.6:0/2960807718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:52 smithi006 bash[33918]: cluster 2023-08-28T00:50:51.189266+0000 mgr.y (mgr.14901) 5168 : cluster [DBG] pgmap v5175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:52 smithi006 bash[33918]: audit 2023-08-28T00:50:51.547270+0000 mon.c (mon.2) 2005 : audit [DBG] from='client.? 172.21.15.6:0/2960807718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:52.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:50:52.423Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:52.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:50:52.425Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:53.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:52 smithi130 bash[51302]: debug 2023-08-28T00:50:52.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:52.933428+0000) 2023-08-28T00:50:54.238 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:53 smithi130 bash[51302]: debug 2023-08-28T00:50:53.928+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:53.933690+0000) 2023-08-28T00:50:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:54 smithi130 bash[27579]: cluster 2023-08-28T00:50:53.189988+0000 mgr.y (mgr.14901) 5169 : cluster [DBG] pgmap v5176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:54 smithi130 bash[27579]: audit 2023-08-28T00:50:53.960695+0000 mon.a (mon.0) 3619 : audit [DBG] from='client.? 172.21.15.6:0/2283223376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:54.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:54 smithi006 bash[33918]: cluster 2023-08-28T00:50:53.189988+0000 mgr.y (mgr.14901) 5169 : cluster [DBG] pgmap v5176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:54.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:54 smithi006 bash[33918]: audit 2023-08-28T00:50:53.960695+0000 mon.a (mon.0) 3619 : audit [DBG] from='client.? 172.21.15.6:0/2283223376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:50:54.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:50:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:50:54.305Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:50:54.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:54 smithi006 bash[27292]: cluster 2023-08-28T00:50:53.189988+0000 mgr.y (mgr.14901) 5169 : cluster [DBG] pgmap v5176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:54.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:54 smithi006 bash[27292]: audit 2023-08-28T00:50:53.960695+0000 mon.a (mon.0) 3619 : audit [DBG] from='client.? 172.21.15.6:0/2283223376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:54 smithi130 bash[51302]: debug 2023-08-28T00:50:54.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:54.933983+0000) 2023-08-28T00:50:56.246 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:55 smithi130 bash[51302]: debug 2023-08-28T00:50:55.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:55.934320+0000) 2023-08-28T00:50:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:56 smithi130 bash[27579]: cluster 2023-08-28T00:50:55.191122+0000 mgr.y (mgr.14901) 5170 : cluster [DBG] pgmap v5177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:56 smithi006 bash[27292]: cluster 2023-08-28T00:50:55.191122+0000 mgr.y (mgr.14901) 5170 : cluster [DBG] pgmap v5177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:56.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:56 smithi006 bash[33918]: cluster 2023-08-28T00:50:55.191122+0000 mgr.y (mgr.14901) 5170 : cluster [DBG] pgmap v5177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:57.258 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:56 smithi130 bash[51302]: debug 2023-08-28T00:50:56.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:56.934595+0000) 2023-08-28T00:50:57.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:57 smithi130 bash[27579]: audit 2023-08-28T00:50:56.385747+0000 mon.c (mon.2) 2006 : audit [DBG] from='client.? 172.21.15.6:0/1313195437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:57.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:57 smithi006 bash[27292]: audit 2023-08-28T00:50:56.385747+0000 mon.c (mon.2) 2006 : audit [DBG] from='client.? 172.21.15.6:0/1313195437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:57.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:57 smithi006 bash[33918]: audit 2023-08-28T00:50:56.385747+0000 mon.c (mon.2) 2006 : audit [DBG] from='client.? 172.21.15.6:0/1313195437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:58.261 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:57 smithi130 bash[51302]: debug 2023-08-28T00:50:57.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:57.934915+0000) 2023-08-28T00:50:58.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:58 smithi130 bash[27579]: cluster 2023-08-28T00:50:57.191876+0000 mgr.y (mgr.14901) 5171 : cluster [DBG] pgmap v5178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:58 smithi006 bash[27292]: cluster 2023-08-28T00:50:57.191876+0000 mgr.y (mgr.14901) 5171 : cluster [DBG] pgmap v5178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:58.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:58 smithi006 bash[33918]: cluster 2023-08-28T00:50:57.191876+0000 mgr.y (mgr.14901) 5171 : cluster [DBG] pgmap v5178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:50:59.269 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:58 smithi130 bash[51302]: debug 2023-08-28T00:50:58.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:58.935186+0000) 2023-08-28T00:50:59.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:50:59 smithi130 bash[27579]: audit 2023-08-28T00:50:58.804587+0000 mon.a (mon.0) 3620 : audit [DBG] from='client.? 172.21.15.6:0/1189100434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:59.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:50:59 smithi006 bash[27292]: audit 2023-08-28T00:50:58.804587+0000 mon.a (mon.0) 3620 : audit [DBG] from='client.? 172.21.15.6:0/1189100434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:59.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:50:59 smithi006 bash[33918]: audit 2023-08-28T00:50:58.804587+0000 mon.a (mon.0) 3620 : audit [DBG] from='client.? 172.21.15.6:0/1189100434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:00.272 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:50:59 smithi130 bash[51302]: debug 2023-08-28T00:50:59.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:59.935523+0000) 2023-08-28T00:51:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:00 smithi130 bash[27579]: cluster 2023-08-28T00:50:59.193034+0000 mgr.y (mgr.14901) 5172 : cluster [DBG] pgmap v5179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:00 smithi006 bash[27292]: cluster 2023-08-28T00:50:59.193034+0000 mgr.y (mgr.14901) 5172 : cluster [DBG] pgmap v5179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:00 smithi006 bash[33918]: cluster 2023-08-28T00:50:59.193034+0000 mgr.y (mgr.14901) 5172 : cluster [DBG] pgmap v5179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:51:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:51:00] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:51:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:51:00.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:51:00.850Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:01.282 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:00 smithi130 bash[51302]: debug 2023-08-28T00:51:00.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:00.935843+0000) 2023-08-28T00:51:01.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:01 smithi130 bash[27579]: audit 2023-08-28T00:51:01.214845+0000 mon.a (mon.0) 3621 : audit [DBG] from='client.? 172.21.15.6:0/1437206915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:01.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:01 smithi006 bash[33918]: audit 2023-08-28T00:51:01.214845+0000 mon.a (mon.0) 3621 : audit [DBG] from='client.? 172.21.15.6:0/1437206915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:01.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:01 smithi006 bash[27292]: audit 2023-08-28T00:51:01.214845+0000 mon.a (mon.0) 3621 : audit [DBG] from='client.? 172.21.15.6:0/1437206915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:02.285 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:01 smithi130 bash[51302]: debug 2023-08-28T00:51:01.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:01.936083+0000) 2023-08-28T00:51:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:02 smithi130 bash[27579]: cluster 2023-08-28T00:51:01.193602+0000 mgr.y (mgr.14901) 5173 : cluster [DBG] pgmap v5180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:02.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:02 smithi006 bash[27292]: cluster 2023-08-28T00:51:01.193602+0000 mgr.y (mgr.14901) 5173 : cluster [DBG] pgmap v5180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:51:02.424Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:51:02.426Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:02 smithi006 bash[33918]: cluster 2023-08-28T00:51:01.193602+0000 mgr.y (mgr.14901) 5173 : cluster [DBG] pgmap v5180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:02 smithi130 bash[51302]: debug 2023-08-28T00:51:02.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:02.936399+0000) 2023-08-28T00:51:04.290 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:03 smithi130 bash[51302]: debug 2023-08-28T00:51:03.936+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:03.936727+0000) 2023-08-28T00:51:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:04 smithi130 bash[27579]: cluster 2023-08-28T00:51:03.194260+0000 mgr.y (mgr.14901) 5174 : cluster [DBG] pgmap v5181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:04 smithi130 bash[27579]: audit 2023-08-28T00:51:03.637590+0000 mon.a (mon.0) 3622 : audit [DBG] from='client.? 172.21.15.6:0/2932333357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:04.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:04 smithi006 bash[33918]: cluster 2023-08-28T00:51:03.194260+0000 mgr.y (mgr.14901) 5174 : cluster [DBG] pgmap v5181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:04.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:04 smithi006 bash[33918]: audit 2023-08-28T00:51:03.637590+0000 mon.a (mon.0) 3622 : audit [DBG] from='client.? 172.21.15.6:0/2932333357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:51:04.304Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:51:04.306Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:04.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:04 smithi006 bash[27292]: cluster 2023-08-28T00:51:03.194260+0000 mgr.y (mgr.14901) 5174 : cluster [DBG] pgmap v5181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:04.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:04 smithi006 bash[27292]: audit 2023-08-28T00:51:03.637590+0000 mon.a (mon.0) 3622 : audit [DBG] from='client.? 172.21.15.6:0/2932333357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:04 smithi130 bash[51302]: debug 2023-08-28T00:51:04.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:04.937018+0000) 2023-08-28T00:51:06.294 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:05 smithi130 bash[51302]: debug 2023-08-28T00:51:05.936+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:05.937301+0000) 2023-08-28T00:51:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:06 smithi130 bash[27579]: cluster 2023-08-28T00:51:05.195503+0000 mgr.y (mgr.14901) 5175 : cluster [DBG] pgmap v5182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:06 smithi130 bash[27579]: audit 2023-08-28T00:51:06.063831+0000 mon.a (mon.0) 3623 : audit [DBG] from='client.? 172.21.15.6:0/125226561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:06 smithi130 bash[27579]: audit 2023-08-28T00:51:06.277017+0000 mon.b (mon.1) 616 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:51:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:06 smithi006 bash[27292]: cluster 2023-08-28T00:51:05.195503+0000 mgr.y (mgr.14901) 5175 : cluster [DBG] pgmap v5182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:06 smithi006 bash[27292]: audit 2023-08-28T00:51:06.063831+0000 mon.a (mon.0) 3623 : audit [DBG] from='client.? 172.21.15.6:0/125226561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:06 smithi006 bash[27292]: audit 2023-08-28T00:51:06.277017+0000 mon.b (mon.1) 616 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:51:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:06 smithi006 bash[33918]: cluster 2023-08-28T00:51:05.195503+0000 mgr.y (mgr.14901) 5175 : cluster [DBG] pgmap v5182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:06 smithi006 bash[33918]: audit 2023-08-28T00:51:06.063831+0000 mon.a (mon.0) 3623 : audit [DBG] from='client.? 172.21.15.6:0/125226561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:06 smithi006 bash[33918]: audit 2023-08-28T00:51:06.277017+0000 mon.b (mon.1) 616 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:51:07.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:06 smithi130 bash[51302]: debug 2023-08-28T00:51:06.936+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:06.937571+0000) 2023-08-28T00:51:08.299 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:07 smithi130 bash[51302]: debug 2023-08-28T00:51:07.936+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:07.937849+0000) 2023-08-28T00:51:08.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:08 smithi130 bash[27579]: cluster 2023-08-28T00:51:07.196234+0000 mgr.y (mgr.14901) 5176 : cluster [DBG] pgmap v5183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:08 smithi006 bash[33918]: cluster 2023-08-28T00:51:07.196234+0000 mgr.y (mgr.14901) 5176 : cluster [DBG] pgmap v5183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:08.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:08 smithi006 bash[27292]: cluster 2023-08-28T00:51:07.196234+0000 mgr.y (mgr.14901) 5176 : cluster [DBG] pgmap v5183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:09.307 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:08 smithi130 bash[51302]: debug 2023-08-28T00:51:08.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:08.938174+0000) 2023-08-28T00:51:09.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:09 smithi130 bash[27579]: audit 2023-08-28T00:51:08.485810+0000 mon.a (mon.0) 3624 : audit [DBG] from='client.? 172.21.15.6:0/3794734079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:09.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:09 smithi006 bash[33918]: audit 2023-08-28T00:51:08.485810+0000 mon.a (mon.0) 3624 : audit [DBG] from='client.? 172.21.15.6:0/3794734079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:09.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:09 smithi006 bash[27292]: audit 2023-08-28T00:51:08.485810+0000 mon.a (mon.0) 3624 : audit [DBG] from='client.? 172.21.15.6:0/3794734079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:10.310 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:09 smithi130 bash[51302]: debug 2023-08-28T00:51:09.936+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:09.938465+0000) 2023-08-28T00:51:10.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:10 smithi130 bash[27579]: cluster 2023-08-28T00:51:09.197412+0000 mgr.y (mgr.14901) 5177 : cluster [DBG] pgmap v5184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:10 smithi006 bash[33918]: cluster 2023-08-28T00:51:09.197412+0000 mgr.y (mgr.14901) 5177 : cluster [DBG] pgmap v5184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:10 smithi006 bash[27292]: cluster 2023-08-28T00:51:09.197412+0000 mgr.y (mgr.14901) 5177 : cluster [DBG] pgmap v5184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:51:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:51:10] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:51:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:51:10.849Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:51:10.851Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:11.317 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:10 smithi130 bash[51302]: debug 2023-08-28T00:51:10.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:10.938755+0000) 2023-08-28T00:51:11.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:11 smithi130 bash[27579]: audit 2023-08-28T00:51:10.910539+0000 mon.c (mon.2) 2007 : audit [DBG] from='client.? 172.21.15.6:0/1229648465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:11.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:11 smithi006 bash[33918]: audit 2023-08-28T00:51:10.910539+0000 mon.c (mon.2) 2007 : audit [DBG] from='client.? 172.21.15.6:0/1229648465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:11.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:11 smithi006 bash[27292]: audit 2023-08-28T00:51:10.910539+0000 mon.c (mon.2) 2007 : audit [DBG] from='client.? 172.21.15.6:0/1229648465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:12.320 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:11 smithi130 bash[51302]: debug 2023-08-28T00:51:11.935+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:11.938995+0000) 2023-08-28T00:51:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:12 smithi130 bash[27579]: cluster 2023-08-28T00:51:11.198236+0000 mgr.y (mgr.14901) 5178 : cluster [DBG] pgmap v5185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:12 smithi130 bash[27579]: audit 2023-08-28T00:51:12.193431+0000 mon.a (mon.0) 3625 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:51:12.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:12 smithi006 bash[27292]: cluster 2023-08-28T00:51:11.198236+0000 mgr.y (mgr.14901) 5178 : cluster [DBG] pgmap v5185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:12 smithi006 bash[27292]: audit 2023-08-28T00:51:12.193431+0000 mon.a (mon.0) 3625 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:51:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:12 smithi006 bash[33918]: cluster 2023-08-28T00:51:11.198236+0000 mgr.y (mgr.14901) 5178 : cluster [DBG] pgmap v5185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:12 smithi006 bash[33918]: audit 2023-08-28T00:51:12.193431+0000 mon.a (mon.0) 3625 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:51:12.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:51:12.424Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:12.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:51:12.426Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:12 smithi130 bash[51302]: debug 2023-08-28T00:51:12.935+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:12.939258+0000) 2023-08-28T00:51:13.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:13 smithi130 bash[27579]: audit 2023-08-28T00:51:12.552073+0000 mon.a (mon.0) 3626 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:51:13.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:13 smithi130 bash[27579]: audit 2023-08-28T00:51:13.325914+0000 mon.a (mon.0) 3627 : audit [DBG] from='client.? 172.21.15.6:0/1121511946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:13.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:13 smithi006 bash[27292]: audit 2023-08-28T00:51:12.552073+0000 mon.a (mon.0) 3626 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:51:13.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:13 smithi006 bash[27292]: audit 2023-08-28T00:51:13.325914+0000 mon.a (mon.0) 3627 : audit [DBG] from='client.? 172.21.15.6:0/1121511946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:13.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:13 smithi006 bash[33918]: audit 2023-08-28T00:51:12.552073+0000 mon.a (mon.0) 3626 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:51:13.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:13 smithi006 bash[33918]: audit 2023-08-28T00:51:13.325914+0000 mon.a (mon.0) 3627 : audit [DBG] from='client.? 172.21.15.6:0/1121511946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:14.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:13 smithi130 bash[51302]: debug 2023-08-28T00:51:13.935+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:13.939559+0000) 2023-08-28T00:51:14.562 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:51:14.304Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:14.562 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:51:14.307Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:14 smithi130 bash[27579]: cluster 2023-08-28T00:51:13.198904+0000 mgr.y (mgr.14901) 5179 : cluster [DBG] pgmap v5186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:14 smithi006 bash[27292]: cluster 2023-08-28T00:51:13.198904+0000 mgr.y (mgr.14901) 5179 : cluster [DBG] pgmap v5186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:14.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:14 smithi006 bash[33918]: cluster 2023-08-28T00:51:13.198904+0000 mgr.y (mgr.14901) 5179 : cluster [DBG] pgmap v5186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:14 smithi130 bash[51302]: debug 2023-08-28T00:51:14.935+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:14.939858+0000) 2023-08-28T00:51:16.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:15 smithi130 bash[51302]: debug 2023-08-28T00:51:15.935+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:15.940118+0000) 2023-08-28T00:51:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:16 smithi130 bash[27579]: cluster 2023-08-28T00:51:15.200044+0000 mgr.y (mgr.14901) 5180 : cluster [DBG] pgmap v5187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:16 smithi130 bash[27579]: audit 2023-08-28T00:51:15.743683+0000 mon.c (mon.2) 2008 : audit [DBG] from='client.? 172.21.15.6:0/1117999828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:16 smithi006 bash[27292]: cluster 2023-08-28T00:51:15.200044+0000 mgr.y (mgr.14901) 5180 : cluster [DBG] pgmap v5187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:16.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:16 smithi006 bash[27292]: audit 2023-08-28T00:51:15.743683+0000 mon.c (mon.2) 2008 : audit [DBG] from='client.? 172.21.15.6:0/1117999828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:16.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:16 smithi006 bash[33918]: cluster 2023-08-28T00:51:15.200044+0000 mgr.y (mgr.14901) 5180 : cluster [DBG] pgmap v5187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:16.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:16 smithi006 bash[33918]: audit 2023-08-28T00:51:15.743683+0000 mon.c (mon.2) 2008 : audit [DBG] from='client.? 172.21.15.6:0/1117999828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:17.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:16 smithi130 bash[51302]: debug 2023-08-28T00:51:16.935+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:16.940384+0000) 2023-08-28T00:51:18.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:17 smithi130 bash[51302]: debug 2023-08-28T00:51:17.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:17.940692+0000) 2023-08-28T00:51:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:18 smithi006 bash[33918]: cluster 2023-08-28T00:51:17.200735+0000 mgr.y (mgr.14901) 5181 : cluster [DBG] pgmap v5188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:18.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:18 smithi006 bash[33918]: audit 2023-08-28T00:51:18.146097+0000 mon.a (mon.0) 3628 : audit [DBG] from='client.? 172.21.15.6:0/323853725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:18 smithi006 bash[27292]: cluster 2023-08-28T00:51:17.200735+0000 mgr.y (mgr.14901) 5181 : cluster [DBG] pgmap v5188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:18 smithi006 bash[27292]: audit 2023-08-28T00:51:18.146097+0000 mon.a (mon.0) 3628 : audit [DBG] from='client.? 172.21.15.6:0/323853725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:18.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:18 smithi130 bash[27579]: cluster 2023-08-28T00:51:17.200735+0000 mgr.y (mgr.14901) 5181 : cluster [DBG] pgmap v5188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:18.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:18 smithi130 bash[27579]: audit 2023-08-28T00:51:18.146097+0000 mon.a (mon.0) 3628 : audit [DBG] from='client.? 172.21.15.6:0/323853725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:19.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:18 smithi130 bash[51302]: debug 2023-08-28T00:51:18.935+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:18.941005+0000) 2023-08-28T00:51:20.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:19 smithi130 bash[51302]: debug 2023-08-28T00:51:19.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:19.941301+0000) 2023-08-28T00:51:20.853 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:51:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:51:20] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:51:20.854 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:20 smithi006 bash[33918]: cluster 2023-08-28T00:51:19.201545+0000 mgr.y (mgr.14901) 5182 : cluster [DBG] pgmap v5189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:20.854 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:20 smithi006 bash[33918]: audit 2023-08-28T00:51:20.564403+0000 mon.a (mon.0) 3629 : audit [DBG] from='client.? 172.21.15.6:0/4040891979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:20.854 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:20 smithi006 bash[27292]: cluster 2023-08-28T00:51:19.201545+0000 mgr.y (mgr.14901) 5182 : cluster [DBG] pgmap v5189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:20.854 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:20 smithi006 bash[27292]: audit 2023-08-28T00:51:20.564403+0000 mon.a (mon.0) 3629 : audit [DBG] from='client.? 172.21.15.6:0/4040891979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:20 smithi130 bash[27579]: cluster 2023-08-28T00:51:19.201545+0000 mgr.y (mgr.14901) 5182 : cluster [DBG] pgmap v5189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:20 smithi130 bash[27579]: audit 2023-08-28T00:51:20.564403+0000 mon.a (mon.0) 3629 : audit [DBG] from='client.? 172.21.15.6:0/4040891979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:51:20.850Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:51:20.852Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:21.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:20 smithi130 bash[51302]: debug 2023-08-28T00:51:20.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:20.941605+0000) 2023-08-28T00:51:22.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:21 smithi130 bash[51302]: debug 2023-08-28T00:51:21.935+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:21.941870+0000) 2023-08-28T00:51:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:22 smithi006 bash[27292]: cluster 2023-08-28T00:51:21.202168+0000 mgr.y (mgr.14901) 5183 : cluster [DBG] pgmap v5190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:22 smithi006 bash[33918]: cluster 2023-08-28T00:51:21.202168+0000 mgr.y (mgr.14901) 5183 : cluster [DBG] pgmap v5190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:51:22.425Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:51:22.427Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:22 smithi130 bash[27579]: cluster 2023-08-28T00:51:21.202168+0000 mgr.y (mgr.14901) 5183 : cluster [DBG] pgmap v5190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:23.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:22 smithi130 bash[51302]: debug 2023-08-28T00:51:22.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:22.942135+0000) 2023-08-28T00:51:24.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:23 smithi130 bash[51302]: debug 2023-08-28T00:51:23.935+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:23.942404+0000) 2023-08-28T00:51:24.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:23 smithi130 bash[27579]: audit 2023-08-28T00:51:22.836791+0000 mon.a (mon.0) 3630 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:51:24.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:23 smithi130 bash[27579]: audit 2023-08-28T00:51:22.983917+0000 mon.c (mon.2) 2009 : audit [DBG] from='client.? 172.21.15.6:0/1469706997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:24.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:23 smithi130 bash[27579]: audit 2023-08-28T00:51:23.127839+0000 mon.b (mon.1) 617 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:51:24.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:23 smithi130 bash[27579]: audit 2023-08-28T00:51:23.129992+0000 mon.b (mon.1) 618 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:51:24.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:23 smithi130 bash[27579]: audit 2023-08-28T00:51:23.137710+0000 mon.a (mon.0) 3631 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:51:24.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:23 smithi130 bash[27579]: cluster 2023-08-28T00:51:23.202832+0000 mgr.y (mgr.14901) 5184 : cluster [DBG] pgmap v5191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:24.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:23 smithi006 bash[33918]: audit 2023-08-28T00:51:22.836791+0000 mon.a (mon.0) 3630 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:51:24.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:23 smithi006 bash[33918]: audit 2023-08-28T00:51:22.983917+0000 mon.c (mon.2) 2009 : audit [DBG] from='client.? 172.21.15.6:0/1469706997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:24.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:23 smithi006 bash[33918]: audit 2023-08-28T00:51:23.127839+0000 mon.b (mon.1) 617 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:51:24.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:23 smithi006 bash[33918]: audit 2023-08-28T00:51:23.129992+0000 mon.b (mon.1) 618 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:51:24.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:23 smithi006 bash[33918]: audit 2023-08-28T00:51:23.137710+0000 mon.a (mon.0) 3631 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:51:24.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:23 smithi006 bash[33918]: cluster 2023-08-28T00:51:23.202832+0000 mgr.y (mgr.14901) 5184 : cluster [DBG] pgmap v5191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:24.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:23 smithi006 bash[27292]: audit 2023-08-28T00:51:22.836791+0000 mon.a (mon.0) 3630 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:51:24.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:23 smithi006 bash[27292]: audit 2023-08-28T00:51:22.983917+0000 mon.c (mon.2) 2009 : audit [DBG] from='client.? 172.21.15.6:0/1469706997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:24.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:23 smithi006 bash[27292]: audit 2023-08-28T00:51:23.127839+0000 mon.b (mon.1) 617 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:51:24.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:23 smithi006 bash[27292]: audit 2023-08-28T00:51:23.129992+0000 mon.b (mon.1) 618 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:51:24.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:23 smithi006 bash[27292]: audit 2023-08-28T00:51:23.137710+0000 mon.a (mon.0) 3631 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:51:24.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:23 smithi006 bash[27292]: cluster 2023-08-28T00:51:23.202832+0000 mgr.y (mgr.14901) 5184 : cluster [DBG] pgmap v5191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:24.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:51:24.306Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:51:24.308Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:24 smithi130 bash[51302]: debug 2023-08-28T00:51:24.935+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:24.942726+0000) 2023-08-28T00:51:26.268 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:25 smithi130 bash[51302]: debug 2023-08-28T00:51:25.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:25.943007+0000) 2023-08-28T00:51:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:26 smithi130 bash[27579]: cluster 2023-08-28T00:51:25.203887+0000 mgr.y (mgr.14901) 5185 : cluster [DBG] pgmap v5192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:26 smithi130 bash[27579]: audit 2023-08-28T00:51:25.398166+0000 mon.a (mon.0) 3632 : audit [DBG] from='client.? 172.21.15.6:0/2566614526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:26 smithi006 bash[27292]: cluster 2023-08-28T00:51:25.203887+0000 mgr.y (mgr.14901) 5185 : cluster [DBG] pgmap v5192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:26 smithi006 bash[27292]: audit 2023-08-28T00:51:25.398166+0000 mon.a (mon.0) 3632 : audit [DBG] from='client.? 172.21.15.6:0/2566614526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:26 smithi006 bash[33918]: cluster 2023-08-28T00:51:25.203887+0000 mgr.y (mgr.14901) 5185 : cluster [DBG] pgmap v5192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:26 smithi006 bash[33918]: audit 2023-08-28T00:51:25.398166+0000 mon.a (mon.0) 3632 : audit [DBG] from='client.? 172.21.15.6:0/2566614526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:27.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:26 smithi130 bash[51302]: debug 2023-08-28T00:51:26.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:26.943236+0000) 2023-08-28T00:51:28.273 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:27 smithi130 bash[51302]: debug 2023-08-28T00:51:27.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:27.943504+0000) 2023-08-28T00:51:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:28 smithi130 bash[27579]: cluster 2023-08-28T00:51:27.204529+0000 mgr.y (mgr.14901) 5186 : cluster [DBG] pgmap v5193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:28 smithi130 bash[27579]: audit 2023-08-28T00:51:27.812658+0000 mon.c (mon.2) 2010 : audit [DBG] from='client.? 172.21.15.6:0/1438732546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:28 smithi006 bash[33918]: cluster 2023-08-28T00:51:27.204529+0000 mgr.y (mgr.14901) 5186 : cluster [DBG] pgmap v5193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:28.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:28 smithi006 bash[33918]: audit 2023-08-28T00:51:27.812658+0000 mon.c (mon.2) 2010 : audit [DBG] from='client.? 172.21.15.6:0/1438732546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:28.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:28 smithi006 bash[27292]: cluster 2023-08-28T00:51:27.204529+0000 mgr.y (mgr.14901) 5186 : cluster [DBG] pgmap v5193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:28.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:28 smithi006 bash[27292]: audit 2023-08-28T00:51:27.812658+0000 mon.c (mon.2) 2010 : audit [DBG] from='client.? 172.21.15.6:0/1438732546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:29.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:28 smithi130 bash[51302]: debug 2023-08-28T00:51:28.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:28.943777+0000) 2023-08-28T00:51:30.277 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:29 smithi130 bash[51302]: debug 2023-08-28T00:51:29.943+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:29.944067+0000) 2023-08-28T00:51:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:30 smithi130 bash[27579]: cluster 2023-08-28T00:51:29.205633+0000 mgr.y (mgr.14901) 5187 : cluster [DBG] pgmap v5194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:30 smithi130 bash[27579]: audit 2023-08-28T00:51:30.223317+0000 mon.c (mon.2) 2011 : audit [DBG] from='client.? 172.21.15.6:0/2492952222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:30 smithi006 bash[33918]: cluster 2023-08-28T00:51:29.205633+0000 mgr.y (mgr.14901) 5187 : cluster [DBG] pgmap v5194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:30.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:30 smithi006 bash[33918]: audit 2023-08-28T00:51:30.223317+0000 mon.c (mon.2) 2011 : audit [DBG] from='client.? 172.21.15.6:0/2492952222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:30.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:30 smithi006 bash[27292]: cluster 2023-08-28T00:51:29.205633+0000 mgr.y (mgr.14901) 5187 : cluster [DBG] pgmap v5194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:30.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:30 smithi006 bash[27292]: audit 2023-08-28T00:51:30.223317+0000 mon.c (mon.2) 2011 : audit [DBG] from='client.? 172.21.15.6:0/2492952222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:51:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:51:30] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:51:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:51:30.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:51:30.854Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:31.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:30 smithi130 bash[51302]: debug 2023-08-28T00:51:30.943+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:30.944396+0000) 2023-08-28T00:51:32.282 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:31 smithi130 bash[51302]: debug 2023-08-28T00:51:31.943+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:31.944682+0000) 2023-08-28T00:51:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:32 smithi130 bash[27579]: cluster 2023-08-28T00:51:31.206381+0000 mgr.y (mgr.14901) 5188 : cluster [DBG] pgmap v5195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:32.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:32 smithi006 bash[33918]: cluster 2023-08-28T00:51:31.206381+0000 mgr.y (mgr.14901) 5188 : cluster [DBG] pgmap v5195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:51:32.426Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:51:32.427Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:32 smithi006 bash[27292]: cluster 2023-08-28T00:51:31.206381+0000 mgr.y (mgr.14901) 5188 : cluster [DBG] pgmap v5195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:33.291 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:32 smithi130 bash[51302]: debug 2023-08-28T00:51:32.943+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:32.944990+0000) 2023-08-28T00:51:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:33 smithi130 bash[27579]: audit 2023-08-28T00:51:32.640911+0000 mon.c (mon.2) 2012 : audit [DBG] from='client.? 172.21.15.6:0/3578471705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:33.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:33 smithi006 bash[33918]: audit 2023-08-28T00:51:32.640911+0000 mon.c (mon.2) 2012 : audit [DBG] from='client.? 172.21.15.6:0/3578471705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:33.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:33 smithi006 bash[27292]: audit 2023-08-28T00:51:32.640911+0000 mon.c (mon.2) 2012 : audit [DBG] from='client.? 172.21.15.6:0/3578471705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:34.293 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:33 smithi130 bash[51302]: debug 2023-08-28T00:51:33.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:33.945269+0000) 2023-08-28T00:51:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:34 smithi130 bash[27579]: cluster 2023-08-28T00:51:33.207181+0000 mgr.y (mgr.14901) 5189 : cluster [DBG] pgmap v5196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:34 smithi006 bash[27292]: cluster 2023-08-28T00:51:33.207181+0000 mgr.y (mgr.14901) 5189 : cluster [DBG] pgmap v5196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:34 smithi006 bash[33918]: cluster 2023-08-28T00:51:33.207181+0000 mgr.y (mgr.14901) 5189 : cluster [DBG] pgmap v5196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:51:34.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:51:34.309Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:35.301 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:34 smithi130 bash[51302]: debug 2023-08-28T00:51:34.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:34.945575+0000) 2023-08-28T00:51:35.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:35 smithi130 bash[27579]: audit 2023-08-28T00:51:35.058127+0000 mon.c (mon.2) 2013 : audit [DBG] from='client.? 172.21.15.6:0/2579473828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:35.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:35 smithi006 bash[33918]: audit 2023-08-28T00:51:35.058127+0000 mon.c (mon.2) 2013 : audit [DBG] from='client.? 172.21.15.6:0/2579473828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:35.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:35 smithi006 bash[27292]: audit 2023-08-28T00:51:35.058127+0000 mon.c (mon.2) 2013 : audit [DBG] from='client.? 172.21.15.6:0/2579473828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:36.303 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:35 smithi130 bash[51302]: debug 2023-08-28T00:51:35.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:35.945857+0000) 2023-08-28T00:51:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:36 smithi130 bash[27579]: cluster 2023-08-28T00:51:35.208346+0000 mgr.y (mgr.14901) 5190 : cluster [DBG] pgmap v5197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:36 smithi006 bash[33918]: cluster 2023-08-28T00:51:35.208346+0000 mgr.y (mgr.14901) 5190 : cluster [DBG] pgmap v5197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:36.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:36 smithi006 bash[27292]: cluster 2023-08-28T00:51:35.208346+0000 mgr.y (mgr.14901) 5190 : cluster [DBG] pgmap v5197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:36 smithi130 bash[51302]: debug 2023-08-28T00:51:36.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:36.946098+0000) 2023-08-28T00:51:38.308 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:37 smithi130 bash[51302]: debug 2023-08-28T00:51:37.943+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:37.946400+0000) 2023-08-28T00:51:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:38 smithi130 bash[27579]: cluster 2023-08-28T00:51:37.208937+0000 mgr.y (mgr.14901) 5191 : cluster [DBG] pgmap v5198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:38 smithi130 bash[27579]: audit 2023-08-28T00:51:37.477072+0000 mon.a (mon.0) 3633 : audit [DBG] from='client.? 172.21.15.6:0/125434203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:38.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:38 smithi006 bash[33918]: cluster 2023-08-28T00:51:37.208937+0000 mgr.y (mgr.14901) 5191 : cluster [DBG] pgmap v5198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:38.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:38 smithi006 bash[33918]: audit 2023-08-28T00:51:37.477072+0000 mon.a (mon.0) 3633 : audit [DBG] from='client.? 172.21.15.6:0/125434203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:38.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:38 smithi006 bash[27292]: cluster 2023-08-28T00:51:37.208937+0000 mgr.y (mgr.14901) 5191 : cluster [DBG] pgmap v5198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:38.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:38 smithi006 bash[27292]: audit 2023-08-28T00:51:37.477072+0000 mon.a (mon.0) 3633 : audit [DBG] from='client.? 172.21.15.6:0/125434203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:39.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:38 smithi130 bash[51302]: debug 2023-08-28T00:51:38.943+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:38.946713+0000) 2023-08-28T00:51:40.313 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:39 smithi130 bash[51302]: debug 2023-08-28T00:51:39.943+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:39.947007+0000) 2023-08-28T00:51:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:40 smithi130 bash[27579]: cluster 2023-08-28T00:51:39.210012+0000 mgr.y (mgr.14901) 5192 : cluster [DBG] pgmap v5199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:40 smithi130 bash[27579]: audit 2023-08-28T00:51:39.889458+0000 mon.c (mon.2) 2014 : audit [DBG] from='client.? 172.21.15.6:0/1702963556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:40.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:40 smithi006 bash[27292]: cluster 2023-08-28T00:51:39.210012+0000 mgr.y (mgr.14901) 5192 : cluster [DBG] pgmap v5199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:40.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:40 smithi006 bash[27292]: audit 2023-08-28T00:51:39.889458+0000 mon.c (mon.2) 2014 : audit [DBG] from='client.? 172.21.15.6:0/1702963556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:40.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:40 smithi006 bash[33918]: cluster 2023-08-28T00:51:39.210012+0000 mgr.y (mgr.14901) 5192 : cluster [DBG] pgmap v5199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:40.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:40 smithi006 bash[33918]: audit 2023-08-28T00:51:39.889458+0000 mon.c (mon.2) 2014 : audit [DBG] from='client.? 172.21.15.6:0/1702963556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:51:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:51:40] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:51:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:51:40.852Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:51:40.855Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:41.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:40 smithi130 bash[51302]: debug 2023-08-28T00:51:40.943+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:40.947233+0000) 2023-08-28T00:51:42.322 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:41 smithi130 bash[51302]: debug 2023-08-28T00:51:41.943+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:41.947441+0000) 2023-08-28T00:51:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:42 smithi130 bash[27579]: cluster 2023-08-28T00:51:41.210646+0000 mgr.y (mgr.14901) 5193 : cluster [DBG] pgmap v5200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:42 smithi130 bash[27579]: audit 2023-08-28T00:51:42.307229+0000 mon.a (mon.0) 3634 : audit [DBG] from='client.? 172.21.15.6:0/1918223526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:42.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:42 smithi006 bash[33918]: cluster 2023-08-28T00:51:41.210646+0000 mgr.y (mgr.14901) 5193 : cluster [DBG] pgmap v5200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:42.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:42 smithi006 bash[33918]: audit 2023-08-28T00:51:42.307229+0000 mon.a (mon.0) 3634 : audit [DBG] from='client.? 172.21.15.6:0/1918223526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:51:42.426Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:51:42.428Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:42.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:42 smithi006 bash[27292]: cluster 2023-08-28T00:51:41.210646+0000 mgr.y (mgr.14901) 5193 : cluster [DBG] pgmap v5200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:42.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:42 smithi006 bash[27292]: audit 2023-08-28T00:51:42.307229+0000 mon.a (mon.0) 3634 : audit [DBG] from='client.? 172.21.15.6:0/1918223526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:43.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:42 smithi130 bash[51302]: debug 2023-08-28T00:51:42.943+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:42.947683+0000) 2023-08-28T00:51:44.324 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:43 smithi130 bash[51302]: debug 2023-08-28T00:51:43.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:43.947960+0000) 2023-08-28T00:51:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:44 smithi130 bash[27579]: cluster 2023-08-28T00:51:43.211365+0000 mgr.y (mgr.14901) 5194 : cluster [DBG] pgmap v5201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:44.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:44 smithi006 bash[33918]: cluster 2023-08-28T00:51:43.211365+0000 mgr.y (mgr.14901) 5194 : cluster [DBG] pgmap v5201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:44.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:44 smithi006 bash[27292]: cluster 2023-08-28T00:51:43.211365+0000 mgr.y (mgr.14901) 5194 : cluster [DBG] pgmap v5201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:51:44.308Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:51:44.310Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:45.332 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:44 smithi130 bash[51302]: debug 2023-08-28T00:51:44.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:44.948270+0000) 2023-08-28T00:51:45.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:45 smithi130 bash[27579]: audit 2023-08-28T00:51:44.722609+0000 mon.a (mon.0) 3635 : audit [DBG] from='client.? 172.21.15.6:0/54712664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:45.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:45 smithi006 bash[33918]: audit 2023-08-28T00:51:44.722609+0000 mon.a (mon.0) 3635 : audit [DBG] from='client.? 172.21.15.6:0/54712664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:45.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:45 smithi006 bash[27292]: audit 2023-08-28T00:51:44.722609+0000 mon.a (mon.0) 3635 : audit [DBG] from='client.? 172.21.15.6:0/54712664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:46.336 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:45 smithi130 bash[51302]: debug 2023-08-28T00:51:45.943+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:45.948567+0000) 2023-08-28T00:51:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:46 smithi130 bash[27579]: cluster 2023-08-28T00:51:45.212669+0000 mgr.y (mgr.14901) 5195 : cluster [DBG] pgmap v5202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:46 smithi006 bash[33918]: cluster 2023-08-28T00:51:45.212669+0000 mgr.y (mgr.14901) 5195 : cluster [DBG] pgmap v5202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:46 smithi006 bash[27292]: cluster 2023-08-28T00:51:45.212669+0000 mgr.y (mgr.14901) 5195 : cluster [DBG] pgmap v5202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:47.345 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:46 smithi130 bash[51302]: debug 2023-08-28T00:51:46.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:46.948826+0000) 2023-08-28T00:51:47.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:47 smithi130 bash[27579]: audit 2023-08-28T00:51:47.139083+0000 mon.a (mon.0) 3636 : audit [DBG] from='client.? 172.21.15.6:0/1889634983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:47.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:47 smithi006 bash[27292]: audit 2023-08-28T00:51:47.139083+0000 mon.a (mon.0) 3636 : audit [DBG] from='client.? 172.21.15.6:0/1889634983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:47.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:47 smithi006 bash[33918]: audit 2023-08-28T00:51:47.139083+0000 mon.a (mon.0) 3636 : audit [DBG] from='client.? 172.21.15.6:0/1889634983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:48.348 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:47 smithi130 bash[51302]: debug 2023-08-28T00:51:47.943+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:47.949137+0000) 2023-08-28T00:51:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:48 smithi130 bash[27579]: cluster 2023-08-28T00:51:47.213464+0000 mgr.y (mgr.14901) 5196 : cluster [DBG] pgmap v5203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:48 smithi006 bash[27292]: cluster 2023-08-28T00:51:47.213464+0000 mgr.y (mgr.14901) 5196 : cluster [DBG] pgmap v5203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:48 smithi006 bash[33918]: cluster 2023-08-28T00:51:47.213464+0000 mgr.y (mgr.14901) 5196 : cluster [DBG] pgmap v5203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:48 smithi130 bash[51302]: debug 2023-08-28T00:51:48.943+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:48.949435+0000) 2023-08-28T00:51:50.353 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:49 smithi130 bash[51302]: debug 2023-08-28T00:51:49.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:49.949708+0000) 2023-08-28T00:51:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:50 smithi130 bash[27579]: cluster 2023-08-28T00:51:49.214289+0000 mgr.y (mgr.14901) 5197 : cluster [DBG] pgmap v5204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:50 smithi130 bash[27579]: audit 2023-08-28T00:51:49.550741+0000 mon.a (mon.0) 3637 : audit [DBG] from='client.? 172.21.15.6:0/3462497364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:50 smithi006 bash[33918]: cluster 2023-08-28T00:51:49.214289+0000 mgr.y (mgr.14901) 5197 : cluster [DBG] pgmap v5204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:50 smithi006 bash[33918]: audit 2023-08-28T00:51:49.550741+0000 mon.a (mon.0) 3637 : audit [DBG] from='client.? 172.21.15.6:0/3462497364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:50 smithi006 bash[27292]: cluster 2023-08-28T00:51:49.214289+0000 mgr.y (mgr.14901) 5197 : cluster [DBG] pgmap v5204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:50 smithi006 bash[27292]: audit 2023-08-28T00:51:49.550741+0000 mon.a (mon.0) 3637 : audit [DBG] from='client.? 172.21.15.6:0/3462497364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:51.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:51:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:51:50] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:51:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:51:50.854Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:51:50.856Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:51.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:50 smithi130 bash[51302]: debug 2023-08-28T00:51:50.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:50.949992+0000) 2023-08-28T00:51:52.358 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:51 smithi130 bash[51302]: debug 2023-08-28T00:51:51.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:51.950256+0000) 2023-08-28T00:51:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:52 smithi130 bash[27579]: cluster 2023-08-28T00:51:51.214945+0000 mgr.y (mgr.14901) 5198 : cluster [DBG] pgmap v5205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:52.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:52 smithi130 bash[27579]: audit 2023-08-28T00:51:51.965004+0000 mon.c (mon.2) 2015 : audit [DBG] from='client.? 172.21.15.6:0/2128817072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:52 smithi006 bash[33918]: cluster 2023-08-28T00:51:51.214945+0000 mgr.y (mgr.14901) 5198 : cluster [DBG] pgmap v5205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:52 smithi006 bash[33918]: audit 2023-08-28T00:51:51.965004+0000 mon.c (mon.2) 2015 : audit [DBG] from='client.? 172.21.15.6:0/2128817072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:51:52.427Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:51:52.429Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:52 smithi006 bash[27292]: cluster 2023-08-28T00:51:51.214945+0000 mgr.y (mgr.14901) 5198 : cluster [DBG] pgmap v5205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:52 smithi006 bash[27292]: audit 2023-08-28T00:51:51.965004+0000 mon.c (mon.2) 2015 : audit [DBG] from='client.? 172.21.15.6:0/2128817072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:53.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:52 smithi130 bash[51302]: debug 2023-08-28T00:51:52.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:52.950573+0000) 2023-08-28T00:51:54.363 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:53 smithi130 bash[51302]: debug 2023-08-28T00:51:53.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:53.950909+0000) 2023-08-28T00:51:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:54 smithi130 bash[27579]: cluster 2023-08-28T00:51:53.215613+0000 mgr.y (mgr.14901) 5199 : cluster [DBG] pgmap v5206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:54 smithi006 bash[27292]: cluster 2023-08-28T00:51:53.215613+0000 mgr.y (mgr.14901) 5199 : cluster [DBG] pgmap v5206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:54.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:54 smithi006 bash[33918]: cluster 2023-08-28T00:51:53.215613+0000 mgr.y (mgr.14901) 5199 : cluster [DBG] pgmap v5206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:51:54.309Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:51:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:51:54.310Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:51:55.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:54 smithi130 bash[51302]: debug 2023-08-28T00:51:54.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:54.951158+0000) 2023-08-28T00:51:55.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:55 smithi130 bash[27579]: audit 2023-08-28T00:51:54.384603+0000 mon.a (mon.0) 3638 : audit [DBG] from='client.? 172.21.15.6:0/2608851888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:55.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:55 smithi006 bash[27292]: audit 2023-08-28T00:51:54.384603+0000 mon.a (mon.0) 3638 : audit [DBG] from='client.? 172.21.15.6:0/2608851888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:55.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:55 smithi006 bash[33918]: audit 2023-08-28T00:51:54.384603+0000 mon.a (mon.0) 3638 : audit [DBG] from='client.? 172.21.15.6:0/2608851888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:56.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:55 smithi130 bash[51302]: debug 2023-08-28T00:51:55.951+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:55.951461+0000) 2023-08-28T00:51:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:56 smithi006 bash[33918]: cluster 2023-08-28T00:51:55.216754+0000 mgr.y (mgr.14901) 5200 : cluster [DBG] pgmap v5207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:56 smithi006 bash[27292]: cluster 2023-08-28T00:51:55.216754+0000 mgr.y (mgr.14901) 5200 : cluster [DBG] pgmap v5207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:56 smithi130 bash[27579]: cluster 2023-08-28T00:51:55.216754+0000 mgr.y (mgr.14901) 5200 : cluster [DBG] pgmap v5207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:57.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:56 smithi130 bash[51302]: debug 2023-08-28T00:51:56.951+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:56.951727+0000) 2023-08-28T00:51:57.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:57 smithi006 bash[27292]: audit 2023-08-28T00:51:56.800619+0000 mon.a (mon.0) 3639 : audit [DBG] from='client.? 172.21.15.6:0/3671433272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:57.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:57 smithi006 bash[33918]: audit 2023-08-28T00:51:56.800619+0000 mon.a (mon.0) 3639 : audit [DBG] from='client.? 172.21.15.6:0/3671433272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:57.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:57 smithi130 bash[27579]: audit 2023-08-28T00:51:56.800619+0000 mon.a (mon.0) 3639 : audit [DBG] from='client.? 172.21.15.6:0/3671433272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:58.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:57 smithi130 bash[51302]: debug 2023-08-28T00:51:57.951+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:57.952013+0000) 2023-08-28T00:51:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:58 smithi006 bash[27292]: cluster 2023-08-28T00:51:57.217357+0000 mgr.y (mgr.14901) 5201 : cluster [DBG] pgmap v5208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:58.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:58 smithi006 bash[33918]: cluster 2023-08-28T00:51:57.217357+0000 mgr.y (mgr.14901) 5201 : cluster [DBG] pgmap v5208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:58 smithi130 bash[27579]: cluster 2023-08-28T00:51:57.217357+0000 mgr.y (mgr.14901) 5201 : cluster [DBG] pgmap v5208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:51:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:58 smithi130 bash[51302]: debug 2023-08-28T00:51:58.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:58.952336+0000) 2023-08-28T00:51:59.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:51:59 smithi006 bash[33918]: audit 2023-08-28T00:51:59.222272+0000 mon.a (mon.0) 3640 : audit [DBG] from='client.? 172.21.15.6:0/4290982514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:59.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:51:59 smithi006 bash[27292]: audit 2023-08-28T00:51:59.222272+0000 mon.a (mon.0) 3640 : audit [DBG] from='client.? 172.21.15.6:0/4290982514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:51:59 smithi130 bash[27579]: audit 2023-08-28T00:51:59.222272+0000 mon.a (mon.0) 3640 : audit [DBG] from='client.? 172.21.15.6:0/4290982514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:00.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:51:59 smithi130 bash[51302]: debug 2023-08-28T00:51:59.951+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:59.952655+0000) 2023-08-28T00:52:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:00 smithi006 bash[27292]: cluster 2023-08-28T00:51:59.218545+0000 mgr.y (mgr.14901) 5202 : cluster [DBG] pgmap v5209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:00 smithi006 bash[33918]: cluster 2023-08-28T00:51:59.218545+0000 mgr.y (mgr.14901) 5202 : cluster [DBG] pgmap v5209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:00 smithi130 bash[27579]: cluster 2023-08-28T00:51:59.218545+0000 mgr.y (mgr.14901) 5202 : cluster [DBG] pgmap v5209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:52:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:52:00] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:52:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:52:00.854Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:52:00.856Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:01.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:00 smithi130 bash[51302]: debug 2023-08-28T00:52:00.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:00.952924+0000) 2023-08-28T00:52:02.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:01 smithi130 bash[51302]: debug 2023-08-28T00:52:01.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:01.953181+0000) 2023-08-28T00:52:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:02 smithi006 bash[27292]: cluster 2023-08-28T00:52:01.219235+0000 mgr.y (mgr.14901) 5203 : cluster [DBG] pgmap v5210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:02.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:02 smithi006 bash[27292]: audit 2023-08-28T00:52:01.635607+0000 mon.c (mon.2) 2016 : audit [DBG] from='client.? 172.21.15.6:0/740379923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:02 smithi006 bash[33918]: cluster 2023-08-28T00:52:01.219235+0000 mgr.y (mgr.14901) 5203 : cluster [DBG] pgmap v5210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:02 smithi006 bash[33918]: audit 2023-08-28T00:52:01.635607+0000 mon.c (mon.2) 2016 : audit [DBG] from='client.? 172.21.15.6:0/740379923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:52:02.427Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:52:02.430Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:02 smithi130 bash[27579]: cluster 2023-08-28T00:52:01.219235+0000 mgr.y (mgr.14901) 5203 : cluster [DBG] pgmap v5210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:02 smithi130 bash[27579]: audit 2023-08-28T00:52:01.635607+0000 mon.c (mon.2) 2016 : audit [DBG] from='client.? 172.21.15.6:0/740379923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:02 smithi130 bash[51302]: debug 2023-08-28T00:52:02.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:02.953481+0000) 2023-08-28T00:52:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:03 smithi130 bash[51302]: debug 2023-08-28T00:52:03.951+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:03.953781+0000) 2023-08-28T00:52:04.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:04 smithi006 bash[27292]: cluster 2023-08-28T00:52:03.219829+0000 mgr.y (mgr.14901) 5204 : cluster [DBG] pgmap v5211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:04.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:04 smithi006 bash[27292]: audit 2023-08-28T00:52:04.043354+0000 mon.a (mon.0) 3641 : audit [DBG] from='client.? 172.21.15.6:0/305633182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:04.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:04 smithi006 bash[33918]: cluster 2023-08-28T00:52:03.219829+0000 mgr.y (mgr.14901) 5204 : cluster [DBG] pgmap v5211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:04.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:04 smithi006 bash[33918]: audit 2023-08-28T00:52:04.043354+0000 mon.a (mon.0) 3641 : audit [DBG] from='client.? 172.21.15.6:0/305633182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:04.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:52:04.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:04.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:52:04.312Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:04 smithi130 bash[27579]: cluster 2023-08-28T00:52:03.219829+0000 mgr.y (mgr.14901) 5204 : cluster [DBG] pgmap v5211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:04 smithi130 bash[27579]: audit 2023-08-28T00:52:04.043354+0000 mon.a (mon.0) 3641 : audit [DBG] from='client.? 172.21.15.6:0/305633182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:04 smithi130 bash[51302]: debug 2023-08-28T00:52:04.951+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:04.954080+0000) 2023-08-28T00:52:06.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:05 smithi130 bash[51302]: debug 2023-08-28T00:52:05.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:05.954345+0000) 2023-08-28T00:52:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:06 smithi006 bash[33918]: cluster 2023-08-28T00:52:05.221008+0000 mgr.y (mgr.14901) 5205 : cluster [DBG] pgmap v5212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:06 smithi006 bash[27292]: cluster 2023-08-28T00:52:05.221008+0000 mgr.y (mgr.14901) 5205 : cluster [DBG] pgmap v5212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:06 smithi130 bash[27579]: cluster 2023-08-28T00:52:05.221008+0000 mgr.y (mgr.14901) 5205 : cluster [DBG] pgmap v5212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:07.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:06 smithi130 bash[51302]: debug 2023-08-28T00:52:06.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:06.954585+0000) 2023-08-28T00:52:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:07 smithi006 bash[33918]: audit 2023-08-28T00:52:06.456812+0000 mon.a (mon.0) 3642 : audit [DBG] from='client.? 172.21.15.6:0/3332128292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:07.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:07 smithi006 bash[27292]: audit 2023-08-28T00:52:06.456812+0000 mon.a (mon.0) 3642 : audit [DBG] from='client.? 172.21.15.6:0/3332128292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:07.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:07 smithi130 bash[27579]: audit 2023-08-28T00:52:06.456812+0000 mon.a (mon.0) 3642 : audit [DBG] from='client.? 172.21.15.6:0/3332128292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:08.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:07 smithi130 bash[51302]: debug 2023-08-28T00:52:07.950+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:07.954872+0000) 2023-08-28T00:52:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:08 smithi006 bash[27292]: cluster 2023-08-28T00:52:07.221802+0000 mgr.y (mgr.14901) 5206 : cluster [DBG] pgmap v5213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:08 smithi006 bash[33918]: cluster 2023-08-28T00:52:07.221802+0000 mgr.y (mgr.14901) 5206 : cluster [DBG] pgmap v5213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:08 smithi130 bash[27579]: cluster 2023-08-28T00:52:07.221802+0000 mgr.y (mgr.14901) 5206 : cluster [DBG] pgmap v5213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:09.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:08 smithi130 bash[51302]: debug 2023-08-28T00:52:08.954+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:08.955195+0000) 2023-08-28T00:52:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:09 smithi006 bash[27292]: audit 2023-08-28T00:52:08.870639+0000 mon.c (mon.2) 2017 : audit [DBG] from='client.? 172.21.15.6:0/3672523510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:09.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:09 smithi006 bash[33918]: audit 2023-08-28T00:52:08.870639+0000 mon.c (mon.2) 2017 : audit [DBG] from='client.? 172.21.15.6:0/3672523510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:09 smithi130 bash[27579]: audit 2023-08-28T00:52:08.870639+0000 mon.c (mon.2) 2017 : audit [DBG] from='client.? 172.21.15.6:0/3672523510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:10.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:09 smithi130 bash[51302]: debug 2023-08-28T00:52:09.954+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:09.955483+0000) 2023-08-28T00:52:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:10 smithi006 bash[27292]: cluster 2023-08-28T00:52:09.222939+0000 mgr.y (mgr.14901) 5207 : cluster [DBG] pgmap v5214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:10 smithi006 bash[33918]: cluster 2023-08-28T00:52:09.222939+0000 mgr.y (mgr.14901) 5207 : cluster [DBG] pgmap v5214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:10 smithi130 bash[27579]: cluster 2023-08-28T00:52:09.222939+0000 mgr.y (mgr.14901) 5207 : cluster [DBG] pgmap v5214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:52:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:52:10] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:52:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:52:10.854Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:52:10.856Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:11.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:10 smithi130 bash[51302]: debug 2023-08-28T00:52:10.954+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:10.955769+0000) 2023-08-28T00:52:12.256 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:11 smithi130 bash[51302]: debug 2023-08-28T00:52:11.954+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:11.956037+0000) 2023-08-28T00:52:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:12 smithi130 bash[27579]: cluster 2023-08-28T00:52:11.223568+0000 mgr.y (mgr.14901) 5208 : cluster [DBG] pgmap v5215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:12.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:12 smithi130 bash[27579]: audit 2023-08-28T00:52:11.286670+0000 mon.c (mon.2) 2018 : audit [DBG] from='client.? 172.21.15.6:0/4097358676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:12.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:12 smithi006 bash[27292]: cluster 2023-08-28T00:52:11.223568+0000 mgr.y (mgr.14901) 5208 : cluster [DBG] pgmap v5215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:12.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:12 smithi006 bash[27292]: audit 2023-08-28T00:52:11.286670+0000 mon.c (mon.2) 2018 : audit [DBG] from='client.? 172.21.15.6:0/4097358676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:12 smithi006 bash[33918]: cluster 2023-08-28T00:52:11.223568+0000 mgr.y (mgr.14901) 5208 : cluster [DBG] pgmap v5215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:12 smithi006 bash[33918]: audit 2023-08-28T00:52:11.286670+0000 mon.c (mon.2) 2018 : audit [DBG] from='client.? 172.21.15.6:0/4097358676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:52:12.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:52:12.430Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:12 smithi130 bash[51302]: debug 2023-08-28T00:52:12.954+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:12.956362+0000) 2023-08-28T00:52:14.264 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:13 smithi130 bash[51302]: debug 2023-08-28T00:52:13.954+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:13.956631+0000) 2023-08-28T00:52:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:14 smithi130 bash[27579]: cluster 2023-08-28T00:52:13.224206+0000 mgr.y (mgr.14901) 5209 : cluster [DBG] pgmap v5216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:14.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:14 smithi130 bash[27579]: audit 2023-08-28T00:52:13.701145+0000 mon.a (mon.0) 3643 : audit [DBG] from='client.? 172.21.15.6:0/4153984945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:14.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:52:14.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:52:14.312Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:14.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:14 smithi006 bash[27292]: cluster 2023-08-28T00:52:13.224206+0000 mgr.y (mgr.14901) 5209 : cluster [DBG] pgmap v5216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:14.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:14 smithi006 bash[27292]: audit 2023-08-28T00:52:13.701145+0000 mon.a (mon.0) 3643 : audit [DBG] from='client.? 172.21.15.6:0/4153984945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:14.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:14 smithi006 bash[33918]: cluster 2023-08-28T00:52:13.224206+0000 mgr.y (mgr.14901) 5209 : cluster [DBG] pgmap v5216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:14.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:14 smithi006 bash[33918]: audit 2023-08-28T00:52:13.701145+0000 mon.a (mon.0) 3643 : audit [DBG] from='client.? 172.21.15.6:0/4153984945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:14 smithi130 bash[51302]: debug 2023-08-28T00:52:14.950+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:14.956951+0000) 2023-08-28T00:52:16.270 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:15 smithi130 bash[51302]: debug 2023-08-28T00:52:15.954+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:15.957245+0000) 2023-08-28T00:52:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:16 smithi130 bash[27579]: cluster 2023-08-28T00:52:15.225358+0000 mgr.y (mgr.14901) 5210 : cluster [DBG] pgmap v5217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:16 smithi130 bash[27579]: audit 2023-08-28T00:52:16.113322+0000 mon.a (mon.0) 3644 : audit [DBG] from='client.? 172.21.15.6:0/3019352402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:16 smithi006 bash[27292]: cluster 2023-08-28T00:52:15.225358+0000 mgr.y (mgr.14901) 5210 : cluster [DBG] pgmap v5217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:16.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:16 smithi006 bash[27292]: audit 2023-08-28T00:52:16.113322+0000 mon.a (mon.0) 3644 : audit [DBG] from='client.? 172.21.15.6:0/3019352402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:16 smithi006 bash[33918]: cluster 2023-08-28T00:52:15.225358+0000 mgr.y (mgr.14901) 5210 : cluster [DBG] pgmap v5217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:16.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:16 smithi006 bash[33918]: audit 2023-08-28T00:52:16.113322+0000 mon.a (mon.0) 3644 : audit [DBG] from='client.? 172.21.15.6:0/3019352402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:17.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:16 smithi130 bash[51302]: debug 2023-08-28T00:52:16.954+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:16.957516+0000) 2023-08-28T00:52:18.280 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:17 smithi130 bash[51302]: debug 2023-08-28T00:52:17.954+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:17.957842+0000) 2023-08-28T00:52:18.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:18 smithi130 bash[27579]: cluster 2023-08-28T00:52:17.226034+0000 mgr.y (mgr.14901) 5211 : cluster [DBG] pgmap v5218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:18.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:18 smithi006 bash[27292]: cluster 2023-08-28T00:52:17.226034+0000 mgr.y (mgr.14901) 5211 : cluster [DBG] pgmap v5218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:18.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:18 smithi006 bash[33918]: cluster 2023-08-28T00:52:17.226034+0000 mgr.y (mgr.14901) 5211 : cluster [DBG] pgmap v5218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:19.289 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:18 smithi130 bash[51302]: debug 2023-08-28T00:52:18.954+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:18.958162+0000) 2023-08-28T00:52:19.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:19 smithi130 bash[27579]: audit 2023-08-28T00:52:18.537036+0000 mon.a (mon.0) 3645 : audit [DBG] from='client.? 172.21.15.6:0/821105620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:19.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:19 smithi006 bash[27292]: audit 2023-08-28T00:52:18.537036+0000 mon.a (mon.0) 3645 : audit [DBG] from='client.? 172.21.15.6:0/821105620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:19.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:19 smithi006 bash[33918]: audit 2023-08-28T00:52:18.537036+0000 mon.a (mon.0) 3645 : audit [DBG] from='client.? 172.21.15.6:0/821105620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:20.291 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:19 smithi130 bash[51302]: debug 2023-08-28T00:52:19.954+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:19.958449+0000) 2023-08-28T00:52:20.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:20 smithi130 bash[27579]: cluster 2023-08-28T00:52:19.227208+0000 mgr.y (mgr.14901) 5212 : cluster [DBG] pgmap v5219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:20 smithi006 bash[27292]: cluster 2023-08-28T00:52:19.227208+0000 mgr.y (mgr.14901) 5212 : cluster [DBG] pgmap v5219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:20.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:20 smithi006 bash[33918]: cluster 2023-08-28T00:52:19.227208+0000 mgr.y (mgr.14901) 5212 : cluster [DBG] pgmap v5219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:52:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:52:20] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:52:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:52:20.856Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:52:20.857Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:21.294 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:20 smithi130 bash[51302]: debug 2023-08-28T00:52:20.954+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:20.958725+0000) 2023-08-28T00:52:21.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:21 smithi130 bash[27579]: audit 2023-08-28T00:52:20.952238+0000 mon.c (mon.2) 2019 : audit [DBG] from='client.? 172.21.15.6:0/3890760338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:21.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:21 smithi006 bash[27292]: audit 2023-08-28T00:52:20.952238+0000 mon.c (mon.2) 2019 : audit [DBG] from='client.? 172.21.15.6:0/3890760338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:21.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:21 smithi006 bash[33918]: audit 2023-08-28T00:52:20.952238+0000 mon.c (mon.2) 2019 : audit [DBG] from='client.? 172.21.15.6:0/3890760338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:22.303 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:21 smithi130 bash[51302]: debug 2023-08-28T00:52:21.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:21.958994+0000) 2023-08-28T00:52:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:22 smithi130 bash[27579]: cluster 2023-08-28T00:52:21.227901+0000 mgr.y (mgr.14901) 5213 : cluster [DBG] pgmap v5220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:22 smithi006 bash[33918]: cluster 2023-08-28T00:52:21.227901+0000 mgr.y (mgr.14901) 5213 : cluster [DBG] pgmap v5220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:52:22.429Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:52:22.431Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:22 smithi006 bash[27292]: cluster 2023-08-28T00:52:21.227901+0000 mgr.y (mgr.14901) 5213 : cluster [DBG] pgmap v5220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:23.309 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:22 smithi130 bash[51302]: debug 2023-08-28T00:52:22.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:22.959233+0000) 2023-08-28T00:52:23.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:23 smithi130 bash[27579]: audit 2023-08-28T00:52:23.151807+0000 mon.b (mon.1) 619 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:52:23.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:23 smithi006 bash[33918]: audit 2023-08-28T00:52:23.151807+0000 mon.b (mon.1) 619 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:52:23.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:23 smithi006 bash[27292]: audit 2023-08-28T00:52:23.151807+0000 mon.b (mon.1) 619 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:52:24.313 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:23 smithi130 bash[51302]: debug 2023-08-28T00:52:23.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:23.959579+0000) 2023-08-28T00:52:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:24 smithi130 bash[27579]: cluster 2023-08-28T00:52:23.228537+0000 mgr.y (mgr.14901) 5214 : cluster [DBG] pgmap v5221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:24 smithi130 bash[27579]: audit 2023-08-28T00:52:23.380959+0000 mon.a (mon.0) 3646 : audit [DBG] from='client.? 172.21.15.6:0/1620060735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:24.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:24 smithi006 bash[33918]: cluster 2023-08-28T00:52:23.228537+0000 mgr.y (mgr.14901) 5214 : cluster [DBG] pgmap v5221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:24.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:24 smithi006 bash[33918]: audit 2023-08-28T00:52:23.380959+0000 mon.a (mon.0) 3646 : audit [DBG] from='client.? 172.21.15.6:0/1620060735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:52:24.311Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:52:24.315Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:24.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:24 smithi006 bash[27292]: cluster 2023-08-28T00:52:23.228537+0000 mgr.y (mgr.14901) 5214 : cluster [DBG] pgmap v5221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:24.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:24 smithi006 bash[27292]: audit 2023-08-28T00:52:23.380959+0000 mon.a (mon.0) 3646 : audit [DBG] from='client.? 172.21.15.6:0/1620060735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:24 smithi130 bash[51302]: debug 2023-08-28T00:52:24.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:24.959910+0000) 2023-08-28T00:52:26.317 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:25 smithi130 bash[51302]: debug 2023-08-28T00:52:25.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:25.960179+0000) 2023-08-28T00:52:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:26 smithi130 bash[27579]: cluster 2023-08-28T00:52:25.229791+0000 mgr.y (mgr.14901) 5215 : cluster [DBG] pgmap v5222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:26 smithi130 bash[27579]: audit 2023-08-28T00:52:25.791482+0000 mon.a (mon.0) 3647 : audit [DBG] from='client.? 172.21.15.6:0/3522949240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:26 smithi006 bash[33918]: cluster 2023-08-28T00:52:25.229791+0000 mgr.y (mgr.14901) 5215 : cluster [DBG] pgmap v5222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:26.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:26 smithi006 bash[33918]: audit 2023-08-28T00:52:25.791482+0000 mon.a (mon.0) 3647 : audit [DBG] from='client.? 172.21.15.6:0/3522949240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:26 smithi006 bash[27292]: cluster 2023-08-28T00:52:25.229791+0000 mgr.y (mgr.14901) 5215 : cluster [DBG] pgmap v5222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:26 smithi006 bash[27292]: audit 2023-08-28T00:52:25.791482+0000 mon.a (mon.0) 3647 : audit [DBG] from='client.? 172.21.15.6:0/3522949240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:27.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:26 smithi130 bash[51302]: debug 2023-08-28T00:52:26.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:26.960473+0000) 2023-08-28T00:52:28.321 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:27 smithi130 bash[51302]: debug 2023-08-28T00:52:27.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:27.960784+0000) 2023-08-28T00:52:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:28 smithi130 bash[27579]: cluster 2023-08-28T00:52:27.230515+0000 mgr.y (mgr.14901) 5216 : cluster [DBG] pgmap v5223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:28.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:28 smithi130 bash[27579]: audit 2023-08-28T00:52:28.230069+0000 mon.a (mon.0) 3648 : audit [DBG] from='client.? 172.21.15.6:0/609659941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:28.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:28 smithi006 bash[27292]: cluster 2023-08-28T00:52:27.230515+0000 mgr.y (mgr.14901) 5216 : cluster [DBG] pgmap v5223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:28.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:28 smithi006 bash[27292]: audit 2023-08-28T00:52:28.230069+0000 mon.a (mon.0) 3648 : audit [DBG] from='client.? 172.21.15.6:0/609659941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:28.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:28 smithi006 bash[33918]: cluster 2023-08-28T00:52:27.230515+0000 mgr.y (mgr.14901) 5216 : cluster [DBG] pgmap v5223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:28.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:28 smithi006 bash[33918]: audit 2023-08-28T00:52:28.230069+0000 mon.a (mon.0) 3648 : audit [DBG] from='client.? 172.21.15.6:0/609659941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:29.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:28 smithi130 bash[51302]: debug 2023-08-28T00:52:28.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:28.960991+0000) 2023-08-28T00:52:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:29 smithi130 bash[51302]: debug 2023-08-28T00:52:29.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:29.961266+0000) 2023-08-28T00:52:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:30 smithi130 bash[27579]: audit 2023-08-28T00:52:29.053293+0000 mon.a (mon.0) 3649 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:52:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:30 smithi130 bash[27579]: cluster 2023-08-28T00:52:29.231550+0000 mgr.y (mgr.14901) 5217 : cluster [DBG] pgmap v5224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:30 smithi130 bash[27579]: audit 2023-08-28T00:52:29.493372+0000 mon.a (mon.0) 3650 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:52:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:30 smithi130 bash[27579]: audit 2023-08-28T00:52:29.785498+0000 mon.b (mon.1) 620 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:52:30.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:30 smithi130 bash[27579]: audit 2023-08-28T00:52:29.787565+0000 mon.b (mon.1) 621 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:52:30.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:30 smithi130 bash[27579]: audit 2023-08-28T00:52:29.796402+0000 mon.a (mon.0) 3651 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:52:30.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:30 smithi006 bash[33918]: audit 2023-08-28T00:52:29.053293+0000 mon.a (mon.0) 3649 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:52:30.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:30 smithi006 bash[33918]: cluster 2023-08-28T00:52:29.231550+0000 mgr.y (mgr.14901) 5217 : cluster [DBG] pgmap v5224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:30.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:30 smithi006 bash[33918]: audit 2023-08-28T00:52:29.493372+0000 mon.a (mon.0) 3650 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:52:30.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:30 smithi006 bash[33918]: audit 2023-08-28T00:52:29.785498+0000 mon.b (mon.1) 620 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:52:30.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:30 smithi006 bash[33918]: audit 2023-08-28T00:52:29.787565+0000 mon.b (mon.1) 621 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:52:30.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:30 smithi006 bash[33918]: audit 2023-08-28T00:52:29.796402+0000 mon.a (mon.0) 3651 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:52:30.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:30 smithi006 bash[27292]: audit 2023-08-28T00:52:29.053293+0000 mon.a (mon.0) 3649 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:52:30.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:30 smithi006 bash[27292]: cluster 2023-08-28T00:52:29.231550+0000 mgr.y (mgr.14901) 5217 : cluster [DBG] pgmap v5224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:30.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:30 smithi006 bash[27292]: audit 2023-08-28T00:52:29.493372+0000 mon.a (mon.0) 3650 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:52:30.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:30 smithi006 bash[27292]: audit 2023-08-28T00:52:29.785498+0000 mon.b (mon.1) 620 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:52:30.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:30 smithi006 bash[27292]: audit 2023-08-28T00:52:29.787565+0000 mon.b (mon.1) 621 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:52:30.468 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:30 smithi006 bash[27292]: audit 2023-08-28T00:52:29.796402+0000 mon.a (mon.0) 3651 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:52:31.063 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:52:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:52:30] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:52:31.064 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:52:30.856Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:31.064 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:52:30.858Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:31.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:31 smithi130 bash[27579]: audit 2023-08-28T00:52:30.659002+0000 mon.a (mon.0) 3652 : audit [DBG] from='client.? 172.21.15.6:0/61386178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:31.369 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:30 smithi130 bash[51302]: debug 2023-08-28T00:52:30.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:30.961534+0000) 2023-08-28T00:52:31.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:31 smithi006 bash[27292]: audit 2023-08-28T00:52:30.659002+0000 mon.a (mon.0) 3652 : audit [DBG] from='client.? 172.21.15.6:0/61386178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:31.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:31 smithi006 bash[33918]: audit 2023-08-28T00:52:30.659002+0000 mon.a (mon.0) 3652 : audit [DBG] from='client.? 172.21.15.6:0/61386178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:32.262 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:31 smithi130 bash[51302]: debug 2023-08-28T00:52:31.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:31.961864+0000) 2023-08-28T00:52:32.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:32 smithi130 bash[27579]: cluster 2023-08-28T00:52:31.232191+0000 mgr.y (mgr.14901) 5218 : cluster [DBG] pgmap v5225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:32 smithi006 bash[27292]: cluster 2023-08-28T00:52:31.232191+0000 mgr.y (mgr.14901) 5218 : cluster [DBG] pgmap v5225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:52:32.430Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:52:32.432Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:32 smithi006 bash[33918]: cluster 2023-08-28T00:52:31.232191+0000 mgr.y (mgr.14901) 5218 : cluster [DBG] pgmap v5225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:33.271 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:32 smithi130 bash[51302]: debug 2023-08-28T00:52:32.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:32.962139+0000) 2023-08-28T00:52:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:33 smithi130 bash[27579]: audit 2023-08-28T00:52:33.070891+0000 mon.c (mon.2) 2020 : audit [DBG] from='client.? 172.21.15.6:0/589210103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:33.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:33 smithi006 bash[33918]: audit 2023-08-28T00:52:33.070891+0000 mon.c (mon.2) 2020 : audit [DBG] from='client.? 172.21.15.6:0/589210103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:33.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:33 smithi006 bash[27292]: audit 2023-08-28T00:52:33.070891+0000 mon.c (mon.2) 2020 : audit [DBG] from='client.? 172.21.15.6:0/589210103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:34.274 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:33 smithi130 bash[51302]: debug 2023-08-28T00:52:33.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:33.962398+0000) 2023-08-28T00:52:34.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:34 smithi130 bash[27579]: cluster 2023-08-28T00:52:33.232863+0000 mgr.y (mgr.14901) 5219 : cluster [DBG] pgmap v5226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:34 smithi006 bash[27292]: cluster 2023-08-28T00:52:33.232863+0000 mgr.y (mgr.14901) 5219 : cluster [DBG] pgmap v5226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:34 smithi006 bash[33918]: cluster 2023-08-28T00:52:33.232863+0000 mgr.y (mgr.14901) 5219 : cluster [DBG] pgmap v5226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:52:34.312Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:52:34.314Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:35.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:34 smithi130 bash[51302]: debug 2023-08-28T00:52:34.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:34.962758+0000) 2023-08-28T00:52:36.280 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:35 smithi130 bash[51302]: debug 2023-08-28T00:52:35.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:35.962985+0000) 2023-08-28T00:52:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:36 smithi130 bash[27579]: cluster 2023-08-28T00:52:35.233883+0000 mgr.y (mgr.14901) 5220 : cluster [DBG] pgmap v5227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:36 smithi130 bash[27579]: audit 2023-08-28T00:52:35.486342+0000 mon.a (mon.0) 3653 : audit [DBG] from='client.? 172.21.15.6:0/2827116594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:36 smithi006 bash[27292]: cluster 2023-08-28T00:52:35.233883+0000 mgr.y (mgr.14901) 5220 : cluster [DBG] pgmap v5227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:36 smithi006 bash[27292]: audit 2023-08-28T00:52:35.486342+0000 mon.a (mon.0) 3653 : audit [DBG] from='client.? 172.21.15.6:0/2827116594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:36.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:36 smithi006 bash[33918]: cluster 2023-08-28T00:52:35.233883+0000 mgr.y (mgr.14901) 5220 : cluster [DBG] pgmap v5227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:36.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:36 smithi006 bash[33918]: audit 2023-08-28T00:52:35.486342+0000 mon.a (mon.0) 3653 : audit [DBG] from='client.? 172.21.15.6:0/2827116594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:36 smithi130 bash[51302]: debug 2023-08-28T00:52:36.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:36.963295+0000) 2023-08-28T00:52:38.286 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:37 smithi130 bash[51302]: debug 2023-08-28T00:52:37.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:37.963565+0000) 2023-08-28T00:52:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:38 smithi130 bash[27579]: cluster 2023-08-28T00:52:37.234541+0000 mgr.y (mgr.14901) 5221 : cluster [DBG] pgmap v5228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:38 smithi130 bash[27579]: audit 2023-08-28T00:52:37.897934+0000 mon.a (mon.0) 3654 : audit [DBG] from='client.? 172.21.15.6:0/1146874821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:38.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:38 smithi006 bash[27292]: cluster 2023-08-28T00:52:37.234541+0000 mgr.y (mgr.14901) 5221 : cluster [DBG] pgmap v5228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:38.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:38 smithi006 bash[27292]: audit 2023-08-28T00:52:37.897934+0000 mon.a (mon.0) 3654 : audit [DBG] from='client.? 172.21.15.6:0/1146874821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:38.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:38 smithi006 bash[33918]: cluster 2023-08-28T00:52:37.234541+0000 mgr.y (mgr.14901) 5221 : cluster [DBG] pgmap v5228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:38.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:38 smithi006 bash[33918]: audit 2023-08-28T00:52:37.897934+0000 mon.a (mon.0) 3654 : audit [DBG] from='client.? 172.21.15.6:0/1146874821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:39.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:38 smithi130 bash[51302]: debug 2023-08-28T00:52:38.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:38.963842+0000) 2023-08-28T00:52:40.292 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:39 smithi130 bash[51302]: debug 2023-08-28T00:52:39.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:39.964187+0000) 2023-08-28T00:52:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:40 smithi130 bash[27579]: cluster 2023-08-28T00:52:39.235625+0000 mgr.y (mgr.14901) 5222 : cluster [DBG] pgmap v5229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:40 smithi006 bash[27292]: cluster 2023-08-28T00:52:39.235625+0000 mgr.y (mgr.14901) 5222 : cluster [DBG] pgmap v5229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:40 smithi006 bash[33918]: cluster 2023-08-28T00:52:39.235625+0000 mgr.y (mgr.14901) 5222 : cluster [DBG] pgmap v5229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:52:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:52:40] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:52:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:52:40.857Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:52:40.859Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:41.302 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:40 smithi130 bash[51302]: debug 2023-08-28T00:52:40.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:40.964464+0000) 2023-08-28T00:52:41.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:41 smithi130 bash[27579]: audit 2023-08-28T00:52:40.315186+0000 mon.a (mon.0) 3655 : audit [DBG] from='client.? 172.21.15.6:0/1711346146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:41.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:41 smithi006 bash[27292]: audit 2023-08-28T00:52:40.315186+0000 mon.a (mon.0) 3655 : audit [DBG] from='client.? 172.21.15.6:0/1711346146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:41.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:41 smithi006 bash[33918]: audit 2023-08-28T00:52:40.315186+0000 mon.a (mon.0) 3655 : audit [DBG] from='client.? 172.21.15.6:0/1711346146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:42.305 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:41 smithi130 bash[51302]: debug 2023-08-28T00:52:41.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:41.964738+0000) 2023-08-28T00:52:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:42 smithi130 bash[27579]: cluster 2023-08-28T00:52:41.236309+0000 mgr.y (mgr.14901) 5223 : cluster [DBG] pgmap v5230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:42 smithi006 bash[27292]: cluster 2023-08-28T00:52:41.236309+0000 mgr.y (mgr.14901) 5223 : cluster [DBG] pgmap v5230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:42.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:42 smithi006 bash[33918]: cluster 2023-08-28T00:52:41.236309+0000 mgr.y (mgr.14901) 5223 : cluster [DBG] pgmap v5230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:52:42.431Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:52:42.432Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:43.314 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:42 smithi130 bash[51302]: debug 2023-08-28T00:52:42.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:42.965038+0000) 2023-08-28T00:52:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:43 smithi130 bash[27579]: audit 2023-08-28T00:52:42.724239+0000 mon.c (mon.2) 2021 : audit [DBG] from='client.? 172.21.15.6:0/3321007468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:43.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:43 smithi006 bash[27292]: audit 2023-08-28T00:52:42.724239+0000 mon.c (mon.2) 2021 : audit [DBG] from='client.? 172.21.15.6:0/3321007468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:43.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:43 smithi006 bash[33918]: audit 2023-08-28T00:52:42.724239+0000 mon.c (mon.2) 2021 : audit [DBG] from='client.? 172.21.15.6:0/3321007468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:44.317 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:43 smithi130 bash[51302]: debug 2023-08-28T00:52:43.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:43.965330+0000) 2023-08-28T00:52:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:44 smithi130 bash[27579]: cluster 2023-08-28T00:52:43.237012+0000 mgr.y (mgr.14901) 5224 : cluster [DBG] pgmap v5231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:44.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:44 smithi006 bash[33918]: cluster 2023-08-28T00:52:43.237012+0000 mgr.y (mgr.14901) 5224 : cluster [DBG] pgmap v5231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:52:44.312Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:52:44.315Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:44.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:44 smithi006 bash[27292]: cluster 2023-08-28T00:52:43.237012+0000 mgr.y (mgr.14901) 5224 : cluster [DBG] pgmap v5231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:45.327 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:44 smithi130 bash[51302]: debug 2023-08-28T00:52:44.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:44.965637+0000) 2023-08-28T00:52:45.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:45 smithi130 bash[27579]: audit 2023-08-28T00:52:45.139261+0000 mon.c (mon.2) 2022 : audit [DBG] from='client.? 172.21.15.6:0/2618036068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:45.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:45 smithi006 bash[33918]: audit 2023-08-28T00:52:45.139261+0000 mon.c (mon.2) 2022 : audit [DBG] from='client.? 172.21.15.6:0/2618036068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:45.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:45 smithi006 bash[27292]: audit 2023-08-28T00:52:45.139261+0000 mon.c (mon.2) 2022 : audit [DBG] from='client.? 172.21.15.6:0/2618036068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:46.330 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:45 smithi130 bash[51302]: debug 2023-08-28T00:52:45.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:45.965917+0000) 2023-08-28T00:52:46.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:46 smithi130 bash[27579]: cluster 2023-08-28T00:52:45.238091+0000 mgr.y (mgr.14901) 5225 : cluster [DBG] pgmap v5232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:46.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:46 smithi006 bash[33918]: cluster 2023-08-28T00:52:45.238091+0000 mgr.y (mgr.14901) 5225 : cluster [DBG] pgmap v5232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:46 smithi006 bash[27292]: cluster 2023-08-28T00:52:45.238091+0000 mgr.y (mgr.14901) 5225 : cluster [DBG] pgmap v5232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:47.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:46 smithi130 bash[51302]: debug 2023-08-28T00:52:46.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:46.966176+0000) 2023-08-28T00:52:48.334 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:47 smithi130 bash[51302]: debug 2023-08-28T00:52:47.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:47.966437+0000) 2023-08-28T00:52:48.874 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:48 smithi006 bash[33918]: cluster 2023-08-28T00:52:47.238851+0000 mgr.y (mgr.14901) 5226 : cluster [DBG] pgmap v5233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:48.882 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:48 smithi006 bash[33918]: audit 2023-08-28T00:52:47.556571+0000 mon.c (mon.2) 2023 : audit [DBG] from='client.? 172.21.15.6:0/839798537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:48.882 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:48 smithi130 bash[27579]: cluster 2023-08-28T00:52:47.238851+0000 mgr.y (mgr.14901) 5226 : cluster [DBG] pgmap v5233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:48.882 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:48 smithi130 bash[27579]: audit 2023-08-28T00:52:47.556571+0000 mon.c (mon.2) 2023 : audit [DBG] from='client.? 172.21.15.6:0/839798537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:48.882 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:48 smithi006 bash[27292]: cluster 2023-08-28T00:52:47.238851+0000 mgr.y (mgr.14901) 5226 : cluster [DBG] pgmap v5233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:48.882 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:48 smithi006 bash[27292]: audit 2023-08-28T00:52:47.556571+0000 mon.c (mon.2) 2023 : audit [DBG] from='client.? 172.21.15.6:0/839798537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:48 smithi130 bash[51302]: debug 2023-08-28T00:52:48.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:48.966716+0000) 2023-08-28T00:52:50.339 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:49 smithi130 bash[51302]: debug 2023-08-28T00:52:49.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:49.967025+0000) 2023-08-28T00:52:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:50 smithi130 bash[27579]: cluster 2023-08-28T00:52:49.240122+0000 mgr.y (mgr.14901) 5227 : cluster [DBG] pgmap v5234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:50.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:50 smithi130 bash[27579]: audit 2023-08-28T00:52:49.970512+0000 mon.a (mon.0) 3656 : audit [DBG] from='client.? 172.21.15.6:0/2431557983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:50 smithi006 bash[33918]: cluster 2023-08-28T00:52:49.240122+0000 mgr.y (mgr.14901) 5227 : cluster [DBG] pgmap v5234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:50 smithi006 bash[33918]: audit 2023-08-28T00:52:49.970512+0000 mon.a (mon.0) 3656 : audit [DBG] from='client.? 172.21.15.6:0/2431557983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:50 smithi006 bash[27292]: cluster 2023-08-28T00:52:49.240122+0000 mgr.y (mgr.14901) 5227 : cluster [DBG] pgmap v5234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:50 smithi006 bash[27292]: audit 2023-08-28T00:52:49.970512+0000 mon.a (mon.0) 3656 : audit [DBG] from='client.? 172.21.15.6:0/2431557983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:52:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:52:50] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T00:52:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:52:50.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:52:50.860Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:51.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:50 smithi130 bash[51302]: debug 2023-08-28T00:52:50.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:50.967328+0000) 2023-08-28T00:52:52.344 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:51 smithi130 bash[51302]: debug 2023-08-28T00:52:51.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:51.967621+0000) 2023-08-28T00:52:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:52 smithi130 bash[27579]: cluster 2023-08-28T00:52:51.240792+0000 mgr.y (mgr.14901) 5228 : cluster [DBG] pgmap v5235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:52.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:52 smithi006 bash[27292]: cluster 2023-08-28T00:52:51.240792+0000 mgr.y (mgr.14901) 5228 : cluster [DBG] pgmap v5235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:52 smithi006 bash[33918]: cluster 2023-08-28T00:52:51.240792+0000 mgr.y (mgr.14901) 5228 : cluster [DBG] pgmap v5235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:52:52.432Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:52:52.433Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:53.351 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:52 smithi130 bash[51302]: debug 2023-08-28T00:52:52.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:52.967892+0000) 2023-08-28T00:52:53.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:53 smithi130 bash[27579]: audit 2023-08-28T00:52:52.382298+0000 mon.c (mon.2) 2024 : audit [DBG] from='client.? 172.21.15.6:0/2230719654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:53.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:53 smithi006 bash[33918]: audit 2023-08-28T00:52:52.382298+0000 mon.c (mon.2) 2024 : audit [DBG] from='client.? 172.21.15.6:0/2230719654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:53.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:53 smithi006 bash[27292]: audit 2023-08-28T00:52:52.382298+0000 mon.c (mon.2) 2024 : audit [DBG] from='client.? 172.21.15.6:0/2230719654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:54.354 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:53 smithi130 bash[51302]: debug 2023-08-28T00:52:53.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:53.968162+0000) 2023-08-28T00:52:54.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:54 smithi130 bash[27579]: cluster 2023-08-28T00:52:53.241502+0000 mgr.y (mgr.14901) 5229 : cluster [DBG] pgmap v5236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:54.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:54 smithi006 bash[33918]: cluster 2023-08-28T00:52:53.241502+0000 mgr.y (mgr.14901) 5229 : cluster [DBG] pgmap v5236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:54.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:54 smithi006 bash[27292]: cluster 2023-08-28T00:52:53.241502+0000 mgr.y (mgr.14901) 5229 : cluster [DBG] pgmap v5236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:52:54.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:52:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:52:54.315Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:52:55.362 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:54 smithi130 bash[51302]: debug 2023-08-28T00:52:54.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:54.968424+0000) 2023-08-28T00:52:55.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:55 smithi130 bash[27579]: audit 2023-08-28T00:52:54.800131+0000 mon.a (mon.0) 3657 : audit [DBG] from='client.? 172.21.15.6:0/2058395923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:55.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:55 smithi006 bash[33918]: audit 2023-08-28T00:52:54.800131+0000 mon.a (mon.0) 3657 : audit [DBG] from='client.? 172.21.15.6:0/2058395923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:55.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:55 smithi006 bash[27292]: audit 2023-08-28T00:52:54.800131+0000 mon.a (mon.0) 3657 : audit [DBG] from='client.? 172.21.15.6:0/2058395923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:56.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:55 smithi130 bash[51302]: debug 2023-08-28T00:52:55.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:55.968701+0000) 2023-08-28T00:52:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:56 smithi130 bash[27579]: cluster 2023-08-28T00:52:55.242630+0000 mgr.y (mgr.14901) 5230 : cluster [DBG] pgmap v5237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:56 smithi006 bash[33918]: cluster 2023-08-28T00:52:55.242630+0000 mgr.y (mgr.14901) 5230 : cluster [DBG] pgmap v5237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:56 smithi006 bash[27292]: cluster 2023-08-28T00:52:55.242630+0000 mgr.y (mgr.14901) 5230 : cluster [DBG] pgmap v5237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:57.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:56 smithi130 bash[51302]: debug 2023-08-28T00:52:56.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:56.969003+0000) 2023-08-28T00:52:57.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:57 smithi006 bash[33918]: audit 2023-08-28T00:52:57.217359+0000 mon.c (mon.2) 2025 : audit [DBG] from='client.? 172.21.15.6:0/1058836087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:57.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:57 smithi006 bash[27292]: audit 2023-08-28T00:52:57.217359+0000 mon.c (mon.2) 2025 : audit [DBG] from='client.? 172.21.15.6:0/1058836087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:57.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:57 smithi130 bash[27579]: audit 2023-08-28T00:52:57.217359+0000 mon.c (mon.2) 2025 : audit [DBG] from='client.? 172.21.15.6:0/1058836087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:58.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:57 smithi130 bash[51302]: debug 2023-08-28T00:52:57.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:57.969291+0000) 2023-08-28T00:52:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:52:58 smithi006 bash[33918]: cluster 2023-08-28T00:52:57.243269+0000 mgr.y (mgr.14901) 5231 : cluster [DBG] pgmap v5238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:58.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:52:58 smithi006 bash[27292]: cluster 2023-08-28T00:52:57.243269+0000 mgr.y (mgr.14901) 5231 : cluster [DBG] pgmap v5238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:52:58 smithi130 bash[27579]: cluster 2023-08-28T00:52:57.243269+0000 mgr.y (mgr.14901) 5231 : cluster [DBG] pgmap v5238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:52:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:58 smithi130 bash[51302]: debug 2023-08-28T00:52:58.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:58.969611+0000) 2023-08-28T00:53:00.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:52:59 smithi130 bash[51302]: debug 2023-08-28T00:52:59.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:59.969931+0000) 2023-08-28T00:53:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:00 smithi006 bash[33918]: cluster 2023-08-28T00:52:59.244495+0000 mgr.y (mgr.14901) 5232 : cluster [DBG] pgmap v5239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:00 smithi006 bash[33918]: audit 2023-08-28T00:52:59.635733+0000 mon.c (mon.2) 2026 : audit [DBG] from='client.? 172.21.15.6:0/274672101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:00 smithi006 bash[27292]: cluster 2023-08-28T00:52:59.244495+0000 mgr.y (mgr.14901) 5232 : cluster [DBG] pgmap v5239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:00 smithi006 bash[27292]: audit 2023-08-28T00:52:59.635733+0000 mon.c (mon.2) 2026 : audit [DBG] from='client.? 172.21.15.6:0/274672101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:00 smithi130 bash[27579]: cluster 2023-08-28T00:52:59.244495+0000 mgr.y (mgr.14901) 5232 : cluster [DBG] pgmap v5239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:00 smithi130 bash[27579]: audit 2023-08-28T00:52:59.635733+0000 mon.c (mon.2) 2026 : audit [DBG] from='client.? 172.21.15.6:0/274672101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:53:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:53:00] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:53:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:53:00.859Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:53:00.862Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:01.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:00 smithi130 bash[51302]: debug 2023-08-28T00:53:00.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:00.970200+0000) 2023-08-28T00:53:02.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:01 smithi130 bash[51302]: debug 2023-08-28T00:53:01.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:01.970443+0000) 2023-08-28T00:53:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:02 smithi006 bash[33918]: cluster 2023-08-28T00:53:01.245217+0000 mgr.y (mgr.14901) 5233 : cluster [DBG] pgmap v5240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:02 smithi006 bash[33918]: audit 2023-08-28T00:53:02.059185+0000 mon.c (mon.2) 2027 : audit [DBG] from='client.? 172.21.15.6:0/4043238860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:53:02.433Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:53:02.435Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:02.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:02 smithi006 bash[27292]: cluster 2023-08-28T00:53:01.245217+0000 mgr.y (mgr.14901) 5233 : cluster [DBG] pgmap v5240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:02.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:02 smithi006 bash[27292]: audit 2023-08-28T00:53:02.059185+0000 mon.c (mon.2) 2027 : audit [DBG] from='client.? 172.21.15.6:0/4043238860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:02 smithi130 bash[27579]: cluster 2023-08-28T00:53:01.245217+0000 mgr.y (mgr.14901) 5233 : cluster [DBG] pgmap v5240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:02 smithi130 bash[27579]: audit 2023-08-28T00:53:02.059185+0000 mon.c (mon.2) 2027 : audit [DBG] from='client.? 172.21.15.6:0/4043238860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:02 smithi130 bash[51302]: debug 2023-08-28T00:53:02.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:02.970743+0000) 2023-08-28T00:53:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:03 smithi130 bash[51302]: debug 2023-08-28T00:53:03.969+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:03.971012+0000) 2023-08-28T00:53:04.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:04 smithi006 bash[27292]: cluster 2023-08-28T00:53:03.245892+0000 mgr.y (mgr.14901) 5234 : cluster [DBG] pgmap v5241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:04.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:04 smithi006 bash[33918]: cluster 2023-08-28T00:53:03.245892+0000 mgr.y (mgr.14901) 5234 : cluster [DBG] pgmap v5241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:53:04.313Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:53:04.316Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:04 smithi130 bash[27579]: cluster 2023-08-28T00:53:03.245892+0000 mgr.y (mgr.14901) 5234 : cluster [DBG] pgmap v5241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:04 smithi130 bash[51302]: debug 2023-08-28T00:53:04.969+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:04.971242+0000) 2023-08-28T00:53:05.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:05 smithi006 bash[33918]: audit 2023-08-28T00:53:04.474394+0000 mon.a (mon.0) 3658 : audit [DBG] from='client.? 172.21.15.6:0/311145412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:05.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:05 smithi006 bash[27292]: audit 2023-08-28T00:53:04.474394+0000 mon.a (mon.0) 3658 : audit [DBG] from='client.? 172.21.15.6:0/311145412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:05.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:05 smithi130 bash[27579]: audit 2023-08-28T00:53:04.474394+0000 mon.a (mon.0) 3658 : audit [DBG] from='client.? 172.21.15.6:0/311145412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:06.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:05 smithi130 bash[51302]: debug 2023-08-28T00:53:05.969+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:05.971522+0000) 2023-08-28T00:53:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:06 smithi006 bash[33918]: cluster 2023-08-28T00:53:05.246965+0000 mgr.y (mgr.14901) 5235 : cluster [DBG] pgmap v5242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:06 smithi006 bash[27292]: cluster 2023-08-28T00:53:05.246965+0000 mgr.y (mgr.14901) 5235 : cluster [DBG] pgmap v5242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:06 smithi130 bash[27579]: cluster 2023-08-28T00:53:05.246965+0000 mgr.y (mgr.14901) 5235 : cluster [DBG] pgmap v5242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:07.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:06 smithi130 bash[51302]: debug 2023-08-28T00:53:06.969+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:06.971792+0000) 2023-08-28T00:53:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:07 smithi006 bash[33918]: audit 2023-08-28T00:53:06.888757+0000 mon.c (mon.2) 2028 : audit [DBG] from='client.? 172.21.15.6:0/370432004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:07.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:07 smithi006 bash[27292]: audit 2023-08-28T00:53:06.888757+0000 mon.c (mon.2) 2028 : audit [DBG] from='client.? 172.21.15.6:0/370432004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:07.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:07 smithi130 bash[27579]: audit 2023-08-28T00:53:06.888757+0000 mon.c (mon.2) 2028 : audit [DBG] from='client.? 172.21.15.6:0/370432004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:08.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:07 smithi130 bash[51302]: debug 2023-08-28T00:53:07.969+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:07.972104+0000) 2023-08-28T00:53:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:08 smithi006 bash[33918]: cluster 2023-08-28T00:53:07.247645+0000 mgr.y (mgr.14901) 5236 : cluster [DBG] pgmap v5243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:08.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:08 smithi006 bash[27292]: cluster 2023-08-28T00:53:07.247645+0000 mgr.y (mgr.14901) 5236 : cluster [DBG] pgmap v5243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:08 smithi130 bash[27579]: cluster 2023-08-28T00:53:07.247645+0000 mgr.y (mgr.14901) 5236 : cluster [DBG] pgmap v5243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:09.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:08 smithi130 bash[51302]: debug 2023-08-28T00:53:08.969+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:08.972385+0000) 2023-08-28T00:53:09.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:09 smithi006 bash[33918]: audit 2023-08-28T00:53:09.308492+0000 mon.a (mon.0) 3659 : audit [DBG] from='client.? 172.21.15.6:0/3908121387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:09.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:09 smithi006 bash[27292]: audit 2023-08-28T00:53:09.308492+0000 mon.a (mon.0) 3659 : audit [DBG] from='client.? 172.21.15.6:0/3908121387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:09 smithi130 bash[27579]: audit 2023-08-28T00:53:09.308492+0000 mon.a (mon.0) 3659 : audit [DBG] from='client.? 172.21.15.6:0/3908121387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:10.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:09 smithi130 bash[51302]: debug 2023-08-28T00:53:09.969+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:09.972678+0000) 2023-08-28T00:53:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:10 smithi006 bash[33918]: cluster 2023-08-28T00:53:09.248625+0000 mgr.y (mgr.14901) 5237 : cluster [DBG] pgmap v5244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:10 smithi006 bash[27292]: cluster 2023-08-28T00:53:09.248625+0000 mgr.y (mgr.14901) 5237 : cluster [DBG] pgmap v5244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:10 smithi130 bash[27579]: cluster 2023-08-28T00:53:09.248625+0000 mgr.y (mgr.14901) 5237 : cluster [DBG] pgmap v5244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:53:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:53:10] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:53:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:53:10.860Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:53:10.862Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:11.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:10 smithi130 bash[51302]: debug 2023-08-28T00:53:10.969+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:10.972956+0000) 2023-08-28T00:53:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:11 smithi130 bash[51302]: debug 2023-08-28T00:53:11.969+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:11.973209+0000) 2023-08-28T00:53:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:53:12.433Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:53:12.435Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:12 smithi006 bash[27292]: cluster 2023-08-28T00:53:11.249269+0000 mgr.y (mgr.14901) 5238 : cluster [DBG] pgmap v5245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:12 smithi006 bash[27292]: audit 2023-08-28T00:53:11.724283+0000 mon.a (mon.0) 3660 : audit [DBG] from='client.? 172.21.15.6:0/2275040605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:12 smithi006 bash[33918]: cluster 2023-08-28T00:53:11.249269+0000 mgr.y (mgr.14901) 5238 : cluster [DBG] pgmap v5245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:12 smithi006 bash[33918]: audit 2023-08-28T00:53:11.724283+0000 mon.a (mon.0) 3660 : audit [DBG] from='client.? 172.21.15.6:0/2275040605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:12 smithi130 bash[27579]: cluster 2023-08-28T00:53:11.249269+0000 mgr.y (mgr.14901) 5238 : cluster [DBG] pgmap v5245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:12 smithi130 bash[27579]: audit 2023-08-28T00:53:11.724283+0000 mon.a (mon.0) 3660 : audit [DBG] from='client.? 172.21.15.6:0/2275040605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:12 smithi130 bash[51302]: debug 2023-08-28T00:53:12.969+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:12.973504+0000) 2023-08-28T00:53:14.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:13 smithi130 bash[51302]: debug 2023-08-28T00:53:13.969+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:13.973772+0000) 2023-08-28T00:53:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:14 smithi006 bash[33918]: cluster 2023-08-28T00:53:13.250034+0000 mgr.y (mgr.14901) 5239 : cluster [DBG] pgmap v5246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:14.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:14 smithi006 bash[33918]: audit 2023-08-28T00:53:14.144918+0000 mon.a (mon.0) 3661 : audit [DBG] from='client.? 172.21.15.6:0/701563582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:53:14.314Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:53:14.316Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:14.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:14 smithi006 bash[27292]: cluster 2023-08-28T00:53:13.250034+0000 mgr.y (mgr.14901) 5239 : cluster [DBG] pgmap v5246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:14.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:14 smithi006 bash[27292]: audit 2023-08-28T00:53:14.144918+0000 mon.a (mon.0) 3661 : audit [DBG] from='client.? 172.21.15.6:0/701563582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:14 smithi130 bash[27579]: cluster 2023-08-28T00:53:13.250034+0000 mgr.y (mgr.14901) 5239 : cluster [DBG] pgmap v5246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:14 smithi130 bash[27579]: audit 2023-08-28T00:53:14.144918+0000 mon.a (mon.0) 3661 : audit [DBG] from='client.? 172.21.15.6:0/701563582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:14 smithi130 bash[51302]: debug 2023-08-28T00:53:14.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:14.974054+0000) 2023-08-28T00:53:16.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:15 smithi130 bash[51302]: debug 2023-08-28T00:53:15.969+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:15.974322+0000) 2023-08-28T00:53:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:16 smithi006 bash[33918]: cluster 2023-08-28T00:53:15.251314+0000 mgr.y (mgr.14901) 5240 : cluster [DBG] pgmap v5247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:16.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:16 smithi006 bash[27292]: cluster 2023-08-28T00:53:15.251314+0000 mgr.y (mgr.14901) 5240 : cluster [DBG] pgmap v5247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:16 smithi130 bash[27579]: cluster 2023-08-28T00:53:15.251314+0000 mgr.y (mgr.14901) 5240 : cluster [DBG] pgmap v5247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:17.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:16 smithi130 bash[51302]: debug 2023-08-28T00:53:16.969+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:16.974606+0000) 2023-08-28T00:53:17.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:17 smithi006 bash[27292]: audit 2023-08-28T00:53:16.556582+0000 mon.c (mon.2) 2029 : audit [DBG] from='client.? 172.21.15.6:0/3800175059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:17.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:17 smithi006 bash[33918]: audit 2023-08-28T00:53:16.556582+0000 mon.c (mon.2) 2029 : audit [DBG] from='client.? 172.21.15.6:0/3800175059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:17.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:17 smithi130 bash[27579]: audit 2023-08-28T00:53:16.556582+0000 mon.c (mon.2) 2029 : audit [DBG] from='client.? 172.21.15.6:0/3800175059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:18.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:17 smithi130 bash[51302]: debug 2023-08-28T00:53:17.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:17.974925+0000) 2023-08-28T00:53:18.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:18 smithi006 bash[27292]: cluster 2023-08-28T00:53:17.252111+0000 mgr.y (mgr.14901) 5241 : cluster [DBG] pgmap v5248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:18 smithi006 bash[33918]: cluster 2023-08-28T00:53:17.252111+0000 mgr.y (mgr.14901) 5241 : cluster [DBG] pgmap v5248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:18 smithi130 bash[27579]: cluster 2023-08-28T00:53:17.252111+0000 mgr.y (mgr.14901) 5241 : cluster [DBG] pgmap v5248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:19.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:18 smithi130 bash[51302]: debug 2023-08-28T00:53:18.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:18.975197+0000) 2023-08-28T00:53:19.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:19 smithi130 bash[27579]: audit 2023-08-28T00:53:18.972388+0000 mon.c (mon.2) 2030 : audit [DBG] from='client.? 172.21.15.6:0/276574176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:19.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:19 smithi006 bash[27292]: audit 2023-08-28T00:53:18.972388+0000 mon.c (mon.2) 2030 : audit [DBG] from='client.? 172.21.15.6:0/276574176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:19.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:19 smithi006 bash[33918]: audit 2023-08-28T00:53:18.972388+0000 mon.c (mon.2) 2030 : audit [DBG] from='client.? 172.21.15.6:0/276574176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:20.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:19 smithi130 bash[51302]: debug 2023-08-28T00:53:19.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:19.975575+0000) 2023-08-28T00:53:20.369 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 28 00:53:20 smithi130 bash[48907]: ts=2023-08-28T00:53:20.184Z caller=compact.go:519 level=info component=tsdb msg="write block" mint=1693173196995 maxt=1693173600000 ulid=01H8WT304DBXX3H70ES6AV0PWR duration=43.682276ms 2023-08-28T00:53:20.369 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 28 00:53:20 smithi130 bash[48907]: ts=2023-08-28T00:53:20.189Z caller=head.go:827 level=info component=tsdb msg="Head GC completed" duration=3.674387ms 2023-08-28T00:53:20.754 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:20 smithi006 bash[33918]: cluster 2023-08-28T00:53:19.253180+0000 mgr.y (mgr.14901) 5242 : cluster [DBG] pgmap v5249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:20.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:20 smithi006 bash[27292]: cluster 2023-08-28T00:53:19.253180+0000 mgr.y (mgr.14901) 5242 : cluster [DBG] pgmap v5249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:20 smithi130 bash[27579]: cluster 2023-08-28T00:53:19.253180+0000 mgr.y (mgr.14901) 5242 : cluster [DBG] pgmap v5249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:53:20.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:53:20.863Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:21.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:53:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:53:20] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:53:21.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:20 smithi130 bash[51302]: debug 2023-08-28T00:53:20.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:20.975896+0000) 2023-08-28T00:53:22.287 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:21 smithi130 bash[51302]: debug 2023-08-28T00:53:21.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:21.976147+0000) 2023-08-28T00:53:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:22 smithi130 bash[27579]: cluster 2023-08-28T00:53:21.253779+0000 mgr.y (mgr.14901) 5243 : cluster [DBG] pgmap v5250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:22 smithi130 bash[27579]: audit 2023-08-28T00:53:21.391721+0000 mon.c (mon.2) 2031 : audit [DBG] from='client.? 172.21.15.6:0/1700476647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:22 smithi006 bash[27292]: cluster 2023-08-28T00:53:21.253779+0000 mgr.y (mgr.14901) 5243 : cluster [DBG] pgmap v5250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:22 smithi006 bash[27292]: audit 2023-08-28T00:53:21.391721+0000 mon.c (mon.2) 2031 : audit [DBG] from='client.? 172.21.15.6:0/1700476647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:53:22.434Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:53:22.435Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:22 smithi006 bash[33918]: cluster 2023-08-28T00:53:21.253779+0000 mgr.y (mgr.14901) 5243 : cluster [DBG] pgmap v5250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:22 smithi006 bash[33918]: audit 2023-08-28T00:53:21.391721+0000 mon.c (mon.2) 2031 : audit [DBG] from='client.? 172.21.15.6:0/1700476647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:23.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:22 smithi130 bash[51302]: debug 2023-08-28T00:53:22.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:22.976410+0000) 2023-08-28T00:53:24.292 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:23 smithi130 bash[51302]: debug 2023-08-28T00:53:23.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:23.976689+0000) 2023-08-28T00:53:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:24 smithi130 bash[27579]: cluster 2023-08-28T00:53:23.254435+0000 mgr.y (mgr.14901) 5244 : cluster [DBG] pgmap v5251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:24 smithi130 bash[27579]: audit 2023-08-28T00:53:23.797462+0000 mon.a (mon.0) 3662 : audit [DBG] from='client.? 172.21.15.6:0/4110142916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:24.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:53:24.315Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:53:24.317Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:24.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:24 smithi006 bash[33918]: cluster 2023-08-28T00:53:23.254435+0000 mgr.y (mgr.14901) 5244 : cluster [DBG] pgmap v5251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:24.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:24 smithi006 bash[33918]: audit 2023-08-28T00:53:23.797462+0000 mon.a (mon.0) 3662 : audit [DBG] from='client.? 172.21.15.6:0/4110142916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:24.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:24 smithi006 bash[27292]: cluster 2023-08-28T00:53:23.254435+0000 mgr.y (mgr.14901) 5244 : cluster [DBG] pgmap v5251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:24.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:24 smithi006 bash[27292]: audit 2023-08-28T00:53:23.797462+0000 mon.a (mon.0) 3662 : audit [DBG] from='client.? 172.21.15.6:0/4110142916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:24 smithi130 bash[51302]: debug 2023-08-28T00:53:24.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:24.976966+0000) 2023-08-28T00:53:26.295 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:25 smithi130 bash[51302]: debug 2023-08-28T00:53:25.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:25.977266+0000) 2023-08-28T00:53:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:26 smithi130 bash[27579]: cluster 2023-08-28T00:53:25.255580+0000 mgr.y (mgr.14901) 5245 : cluster [DBG] pgmap v5252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:26 smithi130 bash[27579]: audit 2023-08-28T00:53:26.209797+0000 mon.b (mon.1) 622 : audit [DBG] from='client.? 172.21.15.6:0/452765312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:26 smithi006 bash[33918]: cluster 2023-08-28T00:53:25.255580+0000 mgr.y (mgr.14901) 5245 : cluster [DBG] pgmap v5252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:26 smithi006 bash[33918]: audit 2023-08-28T00:53:26.209797+0000 mon.b (mon.1) 622 : audit [DBG] from='client.? 172.21.15.6:0/452765312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:26 smithi006 bash[27292]: cluster 2023-08-28T00:53:25.255580+0000 mgr.y (mgr.14901) 5245 : cluster [DBG] pgmap v5252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:26 smithi006 bash[27292]: audit 2023-08-28T00:53:26.209797+0000 mon.b (mon.1) 622 : audit [DBG] from='client.? 172.21.15.6:0/452765312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:27.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:26 smithi130 bash[51302]: debug 2023-08-28T00:53:26.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:26.977471+0000) 2023-08-28T00:53:28.300 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:27 smithi130 bash[51302]: debug 2023-08-28T00:53:27.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:27.977771+0000) 2023-08-28T00:53:28.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:28 smithi130 bash[27579]: cluster 2023-08-28T00:53:27.256252+0000 mgr.y (mgr.14901) 5246 : cluster [DBG] pgmap v5253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:28 smithi006 bash[33918]: cluster 2023-08-28T00:53:27.256252+0000 mgr.y (mgr.14901) 5246 : cluster [DBG] pgmap v5253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:28.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:28 smithi006 bash[27292]: cluster 2023-08-28T00:53:27.256252+0000 mgr.y (mgr.14901) 5246 : cluster [DBG] pgmap v5253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:29.307 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:28 smithi130 bash[51302]: debug 2023-08-28T00:53:28.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:28.978055+0000) 2023-08-28T00:53:29.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:29 smithi130 bash[27579]: audit 2023-08-28T00:53:28.624929+0000 mon.a (mon.0) 3663 : audit [DBG] from='client.? 172.21.15.6:0/795594342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:29.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:29 smithi006 bash[27292]: audit 2023-08-28T00:53:28.624929+0000 mon.a (mon.0) 3663 : audit [DBG] from='client.? 172.21.15.6:0/795594342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:29.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:29 smithi006 bash[33918]: audit 2023-08-28T00:53:28.624929+0000 mon.a (mon.0) 3663 : audit [DBG] from='client.? 172.21.15.6:0/795594342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:30.310 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:29 smithi130 bash[51302]: debug 2023-08-28T00:53:29.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:29.978301+0000) 2023-08-28T00:53:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:30 smithi130 bash[27579]: cluster 2023-08-28T00:53:29.257366+0000 mgr.y (mgr.14901) 5247 : cluster [DBG] pgmap v5254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:30.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:30 smithi130 bash[27579]: audit 2023-08-28T00:53:29.807213+0000 mon.b (mon.1) 623 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:53:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:30 smithi006 bash[33918]: cluster 2023-08-28T00:53:29.257366+0000 mgr.y (mgr.14901) 5247 : cluster [DBG] pgmap v5254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:30 smithi006 bash[33918]: audit 2023-08-28T00:53:29.807213+0000 mon.b (mon.1) 623 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:53:30.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:30 smithi006 bash[27292]: cluster 2023-08-28T00:53:29.257366+0000 mgr.y (mgr.14901) 5247 : cluster [DBG] pgmap v5254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:30.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:30 smithi006 bash[27292]: audit 2023-08-28T00:53:29.807213+0000 mon.b (mon.1) 623 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:53:31.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:53:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:53:30] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:53:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:53:30.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:53:30.863Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:31.317 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:30 smithi130 bash[51302]: debug 2023-08-28T00:53:30.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:30.978617+0000) 2023-08-28T00:53:31.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:31 smithi130 bash[27579]: audit 2023-08-28T00:53:31.042799+0000 mon.c (mon.2) 2032 : audit [DBG] from='client.? 172.21.15.6:0/3601660188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:31.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:31 smithi006 bash[33918]: audit 2023-08-28T00:53:31.042799+0000 mon.c (mon.2) 2032 : audit [DBG] from='client.? 172.21.15.6:0/3601660188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:31.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:31 smithi006 bash[27292]: audit 2023-08-28T00:53:31.042799+0000 mon.c (mon.2) 2032 : audit [DBG] from='client.? 172.21.15.6:0/3601660188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:32.320 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:31 smithi130 bash[51302]: debug 2023-08-28T00:53:31.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:31.978977+0000) 2023-08-28T00:53:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:32 smithi130 bash[27579]: cluster 2023-08-28T00:53:31.258036+0000 mgr.y (mgr.14901) 5248 : cluster [DBG] pgmap v5255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:32 smithi006 bash[27292]: cluster 2023-08-28T00:53:31.258036+0000 mgr.y (mgr.14901) 5248 : cluster [DBG] pgmap v5255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:32 smithi006 bash[33918]: cluster 2023-08-28T00:53:31.258036+0000 mgr.y (mgr.14901) 5248 : cluster [DBG] pgmap v5255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:53:32.434Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:32.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:53:32.436Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:33.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:32 smithi130 bash[51302]: debug 2023-08-28T00:53:32.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:32.979272+0000) 2023-08-28T00:53:34.326 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:33 smithi130 bash[51302]: debug 2023-08-28T00:53:33.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:33.979619+0000) 2023-08-28T00:53:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:34 smithi130 bash[27579]: cluster 2023-08-28T00:53:33.258694+0000 mgr.y (mgr.14901) 5249 : cluster [DBG] pgmap v5256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:34 smithi130 bash[27579]: audit 2023-08-28T00:53:33.456609+0000 mon.c (mon.2) 2033 : audit [DBG] from='client.? 172.21.15.6:0/474810574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:34 smithi006 bash[33918]: cluster 2023-08-28T00:53:33.258694+0000 mgr.y (mgr.14901) 5249 : cluster [DBG] pgmap v5256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:34 smithi006 bash[33918]: audit 2023-08-28T00:53:33.456609+0000 mon.c (mon.2) 2033 : audit [DBG] from='client.? 172.21.15.6:0/474810574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:34 smithi006 bash[27292]: cluster 2023-08-28T00:53:33.258694+0000 mgr.y (mgr.14901) 5249 : cluster [DBG] pgmap v5256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:34 smithi006 bash[27292]: audit 2023-08-28T00:53:33.456609+0000 mon.c (mon.2) 2033 : audit [DBG] from='client.? 172.21.15.6:0/474810574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:53:34.316Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:53:34.318Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:35.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:34 smithi130 bash[51302]: debug 2023-08-28T00:53:34.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:34.979851+0000) 2023-08-28T00:53:36.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:35 smithi130 bash[51302]: debug 2023-08-28T00:53:35.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:35.980108+0000) 2023-08-28T00:53:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:36 smithi006 bash[33918]: cluster 2023-08-28T00:53:35.259732+0000 mgr.y (mgr.14901) 5250 : cluster [DBG] pgmap v5257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:36 smithi006 bash[33918]: audit 2023-08-28T00:53:35.706470+0000 mon.a (mon.0) 3664 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:53:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:36 smithi006 bash[33918]: audit 2023-08-28T00:53:35.894489+0000 mon.a (mon.0) 3665 : audit [DBG] from='client.? 172.21.15.6:0/118939287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:36 smithi006 bash[33918]: audit 2023-08-28T00:53:36.135760+0000 mon.a (mon.0) 3666 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:53:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:36 smithi006 bash[33918]: audit 2023-08-28T00:53:36.428643+0000 mon.b (mon.1) 624 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:53:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:36 smithi006 bash[33918]: audit 2023-08-28T00:53:36.430780+0000 mon.b (mon.1) 625 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:53:36.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:36 smithi006 bash[33918]: audit 2023-08-28T00:53:36.438548+0000 mon.a (mon.0) 3667 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:53:36.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:36 smithi006 bash[27292]: cluster 2023-08-28T00:53:35.259732+0000 mgr.y (mgr.14901) 5250 : cluster [DBG] pgmap v5257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:36.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:36 smithi006 bash[27292]: audit 2023-08-28T00:53:35.706470+0000 mon.a (mon.0) 3664 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:53:36.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:36 smithi006 bash[27292]: audit 2023-08-28T00:53:35.894489+0000 mon.a (mon.0) 3665 : audit [DBG] from='client.? 172.21.15.6:0/118939287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:36.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:36 smithi006 bash[27292]: audit 2023-08-28T00:53:36.135760+0000 mon.a (mon.0) 3666 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:53:36.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:36 smithi006 bash[27292]: audit 2023-08-28T00:53:36.428643+0000 mon.b (mon.1) 624 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:53:36.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:36 smithi006 bash[27292]: audit 2023-08-28T00:53:36.430780+0000 mon.b (mon.1) 625 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:53:36.969 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:36 smithi006 bash[27292]: audit 2023-08-28T00:53:36.438548+0000 mon.a (mon.0) 3667 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:53:36.981 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:36 smithi130 bash[27579]: cluster 2023-08-28T00:53:35.259732+0000 mgr.y (mgr.14901) 5250 : cluster [DBG] pgmap v5257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:36.982 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:36 smithi130 bash[27579]: audit 2023-08-28T00:53:35.706470+0000 mon.a (mon.0) 3664 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:53:36.982 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:36 smithi130 bash[27579]: audit 2023-08-28T00:53:35.894489+0000 mon.a (mon.0) 3665 : audit [DBG] from='client.? 172.21.15.6:0/118939287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:36.982 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:36 smithi130 bash[27579]: audit 2023-08-28T00:53:36.135760+0000 mon.a (mon.0) 3666 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:53:36.982 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:36 smithi130 bash[27579]: audit 2023-08-28T00:53:36.428643+0000 mon.b (mon.1) 624 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:53:36.982 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:36 smithi130 bash[27579]: audit 2023-08-28T00:53:36.430780+0000 mon.b (mon.1) 625 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:53:36.982 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:36 smithi130 bash[27579]: audit 2023-08-28T00:53:36.438548+0000 mon.a (mon.0) 3667 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:53:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:36 smithi130 bash[51302]: debug 2023-08-28T00:53:36.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:36.980405+0000) 2023-08-28T00:53:38.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:37 smithi130 bash[51302]: debug 2023-08-28T00:53:37.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:37.980696+0000) 2023-08-28T00:53:38.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:38 smithi006 bash[33918]: cluster 2023-08-28T00:53:37.260394+0000 mgr.y (mgr.14901) 5251 : cluster [DBG] pgmap v5258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:38.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:38 smithi006 bash[33918]: audit 2023-08-28T00:53:38.296987+0000 mon.a (mon.0) 3668 : audit [DBG] from='client.? 172.21.15.6:0/1131503112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:38.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:38 smithi006 bash[27292]: cluster 2023-08-28T00:53:37.260394+0000 mgr.y (mgr.14901) 5251 : cluster [DBG] pgmap v5258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:38.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:38 smithi006 bash[27292]: audit 2023-08-28T00:53:38.296987+0000 mon.a (mon.0) 3668 : audit [DBG] from='client.? 172.21.15.6:0/1131503112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:38.982 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:38 smithi130 bash[27579]: cluster 2023-08-28T00:53:37.260394+0000 mgr.y (mgr.14901) 5251 : cluster [DBG] pgmap v5258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:38.982 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:38 smithi130 bash[27579]: audit 2023-08-28T00:53:38.296987+0000 mon.a (mon.0) 3668 : audit [DBG] from='client.? 172.21.15.6:0/1131503112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:39.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:38 smithi130 bash[51302]: debug 2023-08-28T00:53:38.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:38.981008+0000) 2023-08-28T00:53:40.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:39 smithi130 bash[51302]: debug 2023-08-28T00:53:39.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:39.981301+0000) 2023-08-28T00:53:40.983 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:40 smithi130 bash[27579]: cluster 2023-08-28T00:53:39.261514+0000 mgr.y (mgr.14901) 5252 : cluster [DBG] pgmap v5259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:53:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:53:40] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:53:41.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:40 smithi006 bash[33918]: cluster 2023-08-28T00:53:39.261514+0000 mgr.y (mgr.14901) 5252 : cluster [DBG] pgmap v5259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:41.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:40 smithi006 bash[27292]: cluster 2023-08-28T00:53:39.261514+0000 mgr.y (mgr.14901) 5252 : cluster [DBG] pgmap v5259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:53:40.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:53:40.866Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:41.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:40 smithi130 bash[51302]: debug 2023-08-28T00:53:40.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:40.981668+0000) 2023-08-28T00:53:41.983 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:41 smithi130 bash[27579]: audit 2023-08-28T00:53:40.713701+0000 mon.c (mon.2) 2034 : audit [DBG] from='client.? 172.21.15.6:0/1175188653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:42.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:41 smithi006 bash[27292]: audit 2023-08-28T00:53:40.713701+0000 mon.c (mon.2) 2034 : audit [DBG] from='client.? 172.21.15.6:0/1175188653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:42.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:41 smithi006 bash[33918]: audit 2023-08-28T00:53:40.713701+0000 mon.c (mon.2) 2034 : audit [DBG] from='client.? 172.21.15.6:0/1175188653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:42.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:41 smithi130 bash[51302]: debug 2023-08-28T00:53:41.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:41.981921+0000) 2023-08-28T00:53:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:53:42.435Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:53:42.437Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:42.983 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:42 smithi130 bash[27579]: cluster 2023-08-28T00:53:41.262164+0000 mgr.y (mgr.14901) 5253 : cluster [DBG] pgmap v5260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:43.944 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:42 smithi006 bash[33918]: cluster 2023-08-28T00:53:41.262164+0000 mgr.y (mgr.14901) 5253 : cluster [DBG] pgmap v5260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:43.945 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:42 smithi130 bash[51302]: debug 2023-08-28T00:53:42.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:42.982221+0000) 2023-08-28T00:53:43.945 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:42 smithi006 bash[27292]: cluster 2023-08-28T00:53:41.262164+0000 mgr.y (mgr.14901) 5253 : cluster [DBG] pgmap v5260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:44.175 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:43 smithi130 bash[27579]: audit 2023-08-28T00:53:43.142098+0000 mon.a (mon.0) 3669 : audit [DBG] from='client.? 172.21.15.6:0/1014081651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:44.175 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:43 smithi130 bash[51302]: debug 2023-08-28T00:53:43.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:43.982511+0000) 2023-08-28T00:53:44.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:43 smithi006 bash[27292]: audit 2023-08-28T00:53:43.142098+0000 mon.a (mon.0) 3669 : audit [DBG] from='client.? 172.21.15.6:0/1014081651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:44.589 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:43 smithi006 bash[33918]: audit 2023-08-28T00:53:43.142098+0000 mon.a (mon.0) 3669 : audit [DBG] from='client.? 172.21.15.6:0/1014081651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:44.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:53:44.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:44.791 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:53:44.319Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:45.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:44 smithi130 bash[27579]: cluster 2023-08-28T00:53:43.262857+0000 mgr.y (mgr.14901) 5254 : cluster [DBG] pgmap v5261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:45.410 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:44 smithi130 bash[51302]: debug 2023-08-28T00:53:44.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:44.982790+0000) 2023-08-28T00:53:45.411 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:44 smithi006 bash[33918]: cluster 2023-08-28T00:53:43.262857+0000 mgr.y (mgr.14901) 5254 : cluster [DBG] pgmap v5261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:45.411 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:44 smithi006 bash[27292]: cluster 2023-08-28T00:53:43.262857+0000 mgr.y (mgr.14901) 5254 : cluster [DBG] pgmap v5261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:46.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:45 smithi130 bash[27579]: audit 2023-08-28T00:53:45.569946+0000 mon.a (mon.0) 3670 : audit [DBG] from='client.? 172.21.15.6:0/3464266965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:46.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:45 smithi130 bash[51302]: debug 2023-08-28T00:53:45.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:45.983070+0000) 2023-08-28T00:53:46.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:45 smithi006 bash[33918]: audit 2023-08-28T00:53:45.569946+0000 mon.a (mon.0) 3670 : audit [DBG] from='client.? 172.21.15.6:0/3464266965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:46.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:45 smithi006 bash[27292]: audit 2023-08-28T00:53:45.569946+0000 mon.a (mon.0) 3670 : audit [DBG] from='client.? 172.21.15.6:0/3464266965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:47.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:46 smithi130 bash[27579]: cluster 2023-08-28T00:53:45.263951+0000 mgr.y (mgr.14901) 5255 : cluster [DBG] pgmap v5262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:47.136 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:46 smithi130 bash[51302]: debug 2023-08-28T00:53:46.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:46.983292+0000) 2023-08-28T00:53:47.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:46 smithi006 bash[27292]: cluster 2023-08-28T00:53:45.263951+0000 mgr.y (mgr.14901) 5255 : cluster [DBG] pgmap v5262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:47.344 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:46 smithi006 bash[33918]: cluster 2023-08-28T00:53:45.263951+0000 mgr.y (mgr.14901) 5255 : cluster [DBG] pgmap v5262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:48.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:47 smithi130 bash[51302]: debug 2023-08-28T00:53:47.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:47.983562+0000) 2023-08-28T00:53:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:48 smithi130 bash[27579]: cluster 2023-08-28T00:53:47.264592+0000 mgr.y (mgr.14901) 5256 : cluster [DBG] pgmap v5263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:49.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:48 smithi130 bash[27579]: audit 2023-08-28T00:53:47.992544+0000 mon.a (mon.0) 3671 : audit [DBG] from='client.? 172.21.15.6:0/949694689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:49.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:48 smithi130 bash[51302]: debug 2023-08-28T00:53:48.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:48.983902+0000) 2023-08-28T00:53:49.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:48 smithi006 bash[33918]: cluster 2023-08-28T00:53:47.264592+0000 mgr.y (mgr.14901) 5256 : cluster [DBG] pgmap v5263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:49.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:48 smithi006 bash[33918]: audit 2023-08-28T00:53:47.992544+0000 mon.a (mon.0) 3671 : audit [DBG] from='client.? 172.21.15.6:0/949694689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:49.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:48 smithi006 bash[27292]: cluster 2023-08-28T00:53:47.264592+0000 mgr.y (mgr.14901) 5256 : cluster [DBG] pgmap v5263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:49.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:48 smithi006 bash[27292]: audit 2023-08-28T00:53:47.992544+0000 mon.a (mon.0) 3671 : audit [DBG] from='client.? 172.21.15.6:0/949694689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:49 smithi130 bash[51302]: debug 2023-08-28T00:53:49.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:49.984232+0000) 2023-08-28T00:53:51.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:50 smithi130 bash[51302]: debug 2023-08-28T00:53:50.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:50.984508+0000) 2023-08-28T00:53:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:50 smithi130 bash[27579]: cluster 2023-08-28T00:53:49.265684+0000 mgr.y (mgr.14901) 5257 : cluster [DBG] pgmap v5264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:50 smithi130 bash[27579]: audit 2023-08-28T00:53:50.408932+0000 mon.a (mon.0) 3672 : audit [DBG] from='client.? 172.21.15.6:0/3930244120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:53:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:53:50] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:53:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:50 smithi006 bash[27292]: cluster 2023-08-28T00:53:49.265684+0000 mgr.y (mgr.14901) 5257 : cluster [DBG] pgmap v5264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:51.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:50 smithi006 bash[27292]: audit 2023-08-28T00:53:50.408932+0000 mon.a (mon.0) 3672 : audit [DBG] from='client.? 172.21.15.6:0/3930244120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:51.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:50 smithi006 bash[33918]: cluster 2023-08-28T00:53:49.265684+0000 mgr.y (mgr.14901) 5257 : cluster [DBG] pgmap v5264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:51.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:50 smithi006 bash[33918]: audit 2023-08-28T00:53:50.408932+0000 mon.a (mon.0) 3672 : audit [DBG] from='client.? 172.21.15.6:0/3930244120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:53:50.863Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:51.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:53:50.865Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:52.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:51 smithi130 bash[51302]: debug 2023-08-28T00:53:51.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:51.984809+0000) 2023-08-28T00:53:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:53:52.436Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:53:52.438Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:53.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:52 smithi130 bash[27579]: cluster 2023-08-28T00:53:51.266279+0000 mgr.y (mgr.14901) 5258 : cluster [DBG] pgmap v5265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:53.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:52 smithi130 bash[51302]: debug 2023-08-28T00:53:52.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:52.985123+0000) 2023-08-28T00:53:53.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:52 smithi006 bash[27292]: cluster 2023-08-28T00:53:51.266279+0000 mgr.y (mgr.14901) 5258 : cluster [DBG] pgmap v5265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:53.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:52 smithi006 bash[33918]: cluster 2023-08-28T00:53:51.266279+0000 mgr.y (mgr.14901) 5258 : cluster [DBG] pgmap v5265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:54.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:53 smithi130 bash[51302]: debug 2023-08-28T00:53:53.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:53.985407+0000) 2023-08-28T00:53:54.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:53 smithi130 bash[27579]: audit 2023-08-28T00:53:52.825740+0000 mon.a (mon.0) 3673 : audit [DBG] from='client.? 172.21.15.6:0/3049399383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:54.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:53 smithi006 bash[33918]: audit 2023-08-28T00:53:52.825740+0000 mon.a (mon.0) 3673 : audit [DBG] from='client.? 172.21.15.6:0/3049399383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:54.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:53 smithi006 bash[27292]: audit 2023-08-28T00:53:52.825740+0000 mon.a (mon.0) 3673 : audit [DBG] from='client.? 172.21.15.6:0/3049399383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:54.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:53:54.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:54.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:53:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:53:54.320Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:53:55.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:54 smithi130 bash[27579]: cluster 2023-08-28T00:53:53.266925+0000 mgr.y (mgr.14901) 5259 : cluster [DBG] pgmap v5266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:55.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:54 smithi130 bash[51302]: debug 2023-08-28T00:53:54.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:54.985702+0000) 2023-08-28T00:53:55.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:54 smithi006 bash[27292]: cluster 2023-08-28T00:53:53.266925+0000 mgr.y (mgr.14901) 5259 : cluster [DBG] pgmap v5266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:55.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:54 smithi006 bash[33918]: cluster 2023-08-28T00:53:53.266925+0000 mgr.y (mgr.14901) 5259 : cluster [DBG] pgmap v5266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:56.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:55 smithi130 bash[27579]: audit 2023-08-28T00:53:55.239773+0000 mon.c (mon.2) 2035 : audit [DBG] from='client.? 172.21.15.6:0/329239910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:56.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:55 smithi130 bash[51302]: debug 2023-08-28T00:53:55.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:55.985991+0000) 2023-08-28T00:53:56.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:55 smithi006 bash[27292]: audit 2023-08-28T00:53:55.239773+0000 mon.c (mon.2) 2035 : audit [DBG] from='client.? 172.21.15.6:0/329239910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:56.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:55 smithi006 bash[33918]: audit 2023-08-28T00:53:55.239773+0000 mon.c (mon.2) 2035 : audit [DBG] from='client.? 172.21.15.6:0/329239910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:57.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:56 smithi130 bash[27579]: cluster 2023-08-28T00:53:55.268031+0000 mgr.y (mgr.14901) 5260 : cluster [DBG] pgmap v5267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:57.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:56 smithi130 bash[51302]: debug 2023-08-28T00:53:56.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:56.986258+0000) 2023-08-28T00:53:57.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:56 smithi006 bash[27292]: cluster 2023-08-28T00:53:55.268031+0000 mgr.y (mgr.14901) 5260 : cluster [DBG] pgmap v5267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:57.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:56 smithi006 bash[33918]: cluster 2023-08-28T00:53:55.268031+0000 mgr.y (mgr.14901) 5260 : cluster [DBG] pgmap v5267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:58.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:57 smithi130 bash[27579]: cluster 2023-08-28T00:53:57.268659+0000 mgr.y (mgr.14901) 5261 : cluster [DBG] pgmap v5268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:58.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:53:57 smithi130 bash[27579]: audit 2023-08-28T00:53:57.649496+0000 mon.a (mon.0) 3674 : audit [DBG] from='client.? 172.21.15.6:0/3886242832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:58.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:57 smithi130 bash[51302]: debug 2023-08-28T00:53:57.985+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:57.986529+0000) 2023-08-28T00:53:58.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:57 smithi006 bash[27292]: cluster 2023-08-28T00:53:57.268659+0000 mgr.y (mgr.14901) 5261 : cluster [DBG] pgmap v5268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:58.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:53:57 smithi006 bash[27292]: audit 2023-08-28T00:53:57.649496+0000 mon.a (mon.0) 3674 : audit [DBG] from='client.? 172.21.15.6:0/3886242832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:58.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:57 smithi006 bash[33918]: cluster 2023-08-28T00:53:57.268659+0000 mgr.y (mgr.14901) 5261 : cluster [DBG] pgmap v5268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:53:58.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:53:57 smithi006 bash[33918]: audit 2023-08-28T00:53:57.649496+0000 mon.a (mon.0) 3674 : audit [DBG] from='client.? 172.21.15.6:0/3886242832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:58 smithi130 bash[51302]: debug 2023-08-28T00:53:58.985+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:58.986878+0000) 2023-08-28T00:54:00.333 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:53:59 smithi130 bash[51302]: debug 2023-08-28T00:53:59.984+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:59.987170+0000) 2023-08-28T00:54:00.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:00 smithi130 bash[27579]: cluster 2023-08-28T00:53:59.269782+0000 mgr.y (mgr.14901) 5262 : cluster [DBG] pgmap v5269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:00.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:00 smithi130 bash[27579]: audit 2023-08-28T00:54:00.074713+0000 mon.a (mon.0) 3675 : audit [DBG] from='client.? 172.21.15.6:0/1671250575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:00 smithi006 bash[33918]: cluster 2023-08-28T00:53:59.269782+0000 mgr.y (mgr.14901) 5262 : cluster [DBG] pgmap v5269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:00 smithi006 bash[33918]: audit 2023-08-28T00:54:00.074713+0000 mon.a (mon.0) 3675 : audit [DBG] from='client.? 172.21.15.6:0/1671250575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:00 smithi006 bash[27292]: cluster 2023-08-28T00:53:59.269782+0000 mgr.y (mgr.14901) 5262 : cluster [DBG] pgmap v5269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:00 smithi006 bash[27292]: audit 2023-08-28T00:54:00.074713+0000 mon.a (mon.0) 3675 : audit [DBG] from='client.? 172.21.15.6:0/1671250575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:54:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:54:00] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:54:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:54:00.864Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:54:00.866Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:01.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:00 smithi130 bash[51302]: debug 2023-08-28T00:54:00.984+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:00.987484+0000) 2023-08-28T00:54:02.338 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:01 smithi130 bash[51302]: debug 2023-08-28T00:54:01.984+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:01.987808+0000) 2023-08-28T00:54:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:02 smithi130 bash[27579]: cluster 2023-08-28T00:54:01.270374+0000 mgr.y (mgr.14901) 5263 : cluster [DBG] pgmap v5270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:02 smithi006 bash[33918]: cluster 2023-08-28T00:54:01.270374+0000 mgr.y (mgr.14901) 5263 : cluster [DBG] pgmap v5270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:02.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:02 smithi006 bash[27292]: cluster 2023-08-28T00:54:01.270374+0000 mgr.y (mgr.14901) 5263 : cluster [DBG] pgmap v5270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:54:02.436Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:54:02.438Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:03.350 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:02 smithi130 bash[51302]: debug 2023-08-28T00:54:02.984+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:02.988121+0000) 2023-08-28T00:54:03.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:03 smithi130 bash[27579]: audit 2023-08-28T00:54:02.476065+0000 mon.a (mon.0) 3676 : audit [DBG] from='client.? 172.21.15.6:0/327486286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:03.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:03 smithi006 bash[33918]: audit 2023-08-28T00:54:02.476065+0000 mon.a (mon.0) 3676 : audit [DBG] from='client.? 172.21.15.6:0/327486286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:03.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:03 smithi006 bash[27292]: audit 2023-08-28T00:54:02.476065+0000 mon.a (mon.0) 3676 : audit [DBG] from='client.? 172.21.15.6:0/327486286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:04.354 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:03 smithi130 bash[51302]: debug 2023-08-28T00:54:03.984+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:03.988436+0000) 2023-08-28T00:54:04.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:04 smithi130 bash[27579]: cluster 2023-08-28T00:54:03.270936+0000 mgr.y (mgr.14901) 5264 : cluster [DBG] pgmap v5271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:04.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:04 smithi006 bash[33918]: cluster 2023-08-28T00:54:03.270936+0000 mgr.y (mgr.14901) 5264 : cluster [DBG] pgmap v5271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:04.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:04 smithi006 bash[27292]: cluster 2023-08-28T00:54:03.270936+0000 mgr.y (mgr.14901) 5264 : cluster [DBG] pgmap v5271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:54:04.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:54:04.319Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:05.362 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:04 smithi130 bash[51302]: debug 2023-08-28T00:54:04.984+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:04.988750+0000) 2023-08-28T00:54:05.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:05 smithi130 bash[27579]: audit 2023-08-28T00:54:04.870312+0000 mon.a (mon.0) 3677 : audit [DBG] from='client.? 172.21.15.6:0/2166455605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:05.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:05 smithi006 bash[33918]: audit 2023-08-28T00:54:04.870312+0000 mon.a (mon.0) 3677 : audit [DBG] from='client.? 172.21.15.6:0/2166455605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:05.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:05 smithi006 bash[27292]: audit 2023-08-28T00:54:04.870312+0000 mon.a (mon.0) 3677 : audit [DBG] from='client.? 172.21.15.6:0/2166455605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:06.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:05 smithi130 bash[51302]: debug 2023-08-28T00:54:05.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:05.989104+0000) 2023-08-28T00:54:06.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:06 smithi130 bash[27579]: cluster 2023-08-28T00:54:05.272116+0000 mgr.y (mgr.14901) 5265 : cluster [DBG] pgmap v5272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:06 smithi006 bash[33918]: cluster 2023-08-28T00:54:05.272116+0000 mgr.y (mgr.14901) 5265 : cluster [DBG] pgmap v5272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:06 smithi006 bash[27292]: cluster 2023-08-28T00:54:05.272116+0000 mgr.y (mgr.14901) 5265 : cluster [DBG] pgmap v5272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:07.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:06 smithi130 bash[51302]: debug 2023-08-28T00:54:06.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:06.989398+0000) 2023-08-28T00:54:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:07 smithi006 bash[33918]: audit 2023-08-28T00:54:07.284930+0000 mon.a (mon.0) 3678 : audit [DBG] from='client.? 172.21.15.6:0/4227621268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:07.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:07 smithi006 bash[27292]: audit 2023-08-28T00:54:07.284930+0000 mon.a (mon.0) 3678 : audit [DBG] from='client.? 172.21.15.6:0/4227621268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:07.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:07 smithi130 bash[27579]: audit 2023-08-28T00:54:07.284930+0000 mon.a (mon.0) 3678 : audit [DBG] from='client.? 172.21.15.6:0/4227621268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:08.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:07 smithi130 bash[51302]: debug 2023-08-28T00:54:07.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:07.989707+0000) 2023-08-28T00:54:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:08 smithi006 bash[27292]: cluster 2023-08-28T00:54:07.272903+0000 mgr.y (mgr.14901) 5266 : cluster [DBG] pgmap v5273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:08 smithi006 bash[33918]: cluster 2023-08-28T00:54:07.272903+0000 mgr.y (mgr.14901) 5266 : cluster [DBG] pgmap v5273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:08 smithi130 bash[27579]: cluster 2023-08-28T00:54:07.272903+0000 mgr.y (mgr.14901) 5266 : cluster [DBG] pgmap v5273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:09.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:08 smithi130 bash[51302]: debug 2023-08-28T00:54:08.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:08.990003+0000) 2023-08-28T00:54:10.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:09 smithi130 bash[51302]: debug 2023-08-28T00:54:09.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:09.990320+0000) 2023-08-28T00:54:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:10 smithi006 bash[27292]: cluster 2023-08-28T00:54:09.273972+0000 mgr.y (mgr.14901) 5267 : cluster [DBG] pgmap v5274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:10 smithi006 bash[27292]: audit 2023-08-28T00:54:09.692520+0000 mon.c (mon.2) 2036 : audit [DBG] from='client.? 172.21.15.6:0/62105900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:10 smithi006 bash[33918]: cluster 2023-08-28T00:54:09.273972+0000 mgr.y (mgr.14901) 5267 : cluster [DBG] pgmap v5274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:10.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:10 smithi006 bash[33918]: audit 2023-08-28T00:54:09.692520+0000 mon.c (mon.2) 2036 : audit [DBG] from='client.? 172.21.15.6:0/62105900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:10 smithi130 bash[27579]: cluster 2023-08-28T00:54:09.273972+0000 mgr.y (mgr.14901) 5267 : cluster [DBG] pgmap v5274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:10 smithi130 bash[27579]: audit 2023-08-28T00:54:09.692520+0000 mon.c (mon.2) 2036 : audit [DBG] from='client.? 172.21.15.6:0/62105900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:54:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:54:10] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:54:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:54:10.864Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:54:10.866Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:11.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:10 smithi130 bash[51302]: debug 2023-08-28T00:54:10.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:10.990620+0000) 2023-08-28T00:54:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:11 smithi130 bash[51302]: debug 2023-08-28T00:54:11.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:11.990932+0000) 2023-08-28T00:54:12.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:12 smithi006 bash[27292]: cluster 2023-08-28T00:54:11.274616+0000 mgr.y (mgr.14901) 5268 : cluster [DBG] pgmap v5275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:12 smithi006 bash[27292]: audit 2023-08-28T00:54:12.103312+0000 mon.a (mon.0) 3679 : audit [DBG] from='client.? 172.21.15.6:0/328015539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:54:12.436Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:54:12.438Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:12 smithi006 bash[33918]: cluster 2023-08-28T00:54:11.274616+0000 mgr.y (mgr.14901) 5268 : cluster [DBG] pgmap v5275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:12 smithi006 bash[33918]: audit 2023-08-28T00:54:12.103312+0000 mon.a (mon.0) 3679 : audit [DBG] from='client.? 172.21.15.6:0/328015539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:12 smithi130 bash[27579]: cluster 2023-08-28T00:54:11.274616+0000 mgr.y (mgr.14901) 5268 : cluster [DBG] pgmap v5275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:12 smithi130 bash[27579]: audit 2023-08-28T00:54:12.103312+0000 mon.a (mon.0) 3679 : audit [DBG] from='client.? 172.21.15.6:0/328015539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:12 smithi130 bash[51302]: debug 2023-08-28T00:54:12.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:12.991206+0000) 2023-08-28T00:54:14.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:13 smithi130 bash[51302]: debug 2023-08-28T00:54:13.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:13.991500+0000) 2023-08-28T00:54:14.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:14 smithi006 bash[27292]: cluster 2023-08-28T00:54:13.275391+0000 mgr.y (mgr.14901) 5269 : cluster [DBG] pgmap v5276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:14.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:14 smithi006 bash[33918]: cluster 2023-08-28T00:54:13.275391+0000 mgr.y (mgr.14901) 5269 : cluster [DBG] pgmap v5276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:54:14.318Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:54:14.320Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:14 smithi130 bash[27579]: cluster 2023-08-28T00:54:13.275391+0000 mgr.y (mgr.14901) 5269 : cluster [DBG] pgmap v5276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:14 smithi130 bash[51302]: debug 2023-08-28T00:54:14.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:14.991815+0000) 2023-08-28T00:54:15.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:15 smithi006 bash[27292]: audit 2023-08-28T00:54:14.531880+0000 mon.c (mon.2) 2037 : audit [DBG] from='client.? 172.21.15.6:0/3713205331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:15.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:15 smithi006 bash[33918]: audit 2023-08-28T00:54:14.531880+0000 mon.c (mon.2) 2037 : audit [DBG] from='client.? 172.21.15.6:0/3713205331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:15.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:15 smithi130 bash[27579]: audit 2023-08-28T00:54:14.531880+0000 mon.c (mon.2) 2037 : audit [DBG] from='client.? 172.21.15.6:0/3713205331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:16.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:15 smithi130 bash[51302]: debug 2023-08-28T00:54:15.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:15.992083+0000) 2023-08-28T00:54:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:16 smithi006 bash[33918]: cluster 2023-08-28T00:54:15.276539+0000 mgr.y (mgr.14901) 5270 : cluster [DBG] pgmap v5277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:16.830 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:16 smithi006 bash[27292]: cluster 2023-08-28T00:54:15.276539+0000 mgr.y (mgr.14901) 5270 : cluster [DBG] pgmap v5277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:16 smithi130 bash[27579]: cluster 2023-08-28T00:54:15.276539+0000 mgr.y (mgr.14901) 5270 : cluster [DBG] pgmap v5277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:17.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:16 smithi130 bash[51302]: debug 2023-08-28T00:54:16.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:16.992354+0000) 2023-08-28T00:54:17.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:17 smithi006 bash[33918]: audit 2023-08-28T00:54:16.948682+0000 mon.c (mon.2) 2038 : audit [DBG] from='client.? 172.21.15.6:0/1821488253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:21.340 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:17 smithi006 bash[27292]: audit 2023-08-28T00:54:16.948682+0000 mon.c (mon.2) 2038 : audit [DBG] from='client.? 172.21.15.6:0/1821488253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:21.340 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:18 smithi006 bash[27292]: cluster 2023-08-28T00:54:17.277141+0000 mgr.y (mgr.14901) 5271 : cluster [DBG] pgmap v5278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:21.340 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:18 smithi006 bash[33918]: cluster 2023-08-28T00:54:17.277141+0000 mgr.y (mgr.14901) 5271 : cluster [DBG] pgmap v5278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:21.340 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:19 smithi006 bash[27292]: audit 2023-08-28T00:54:19.368598+0000 mon.a (mon.0) 3680 : audit [DBG] from='client.? 172.21.15.6:0/3608884916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:21.341 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:19 smithi006 bash[33918]: audit 2023-08-28T00:54:19.368598+0000 mon.a (mon.0) 3680 : audit [DBG] from='client.? 172.21.15.6:0/3608884916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:21.341 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:20 smithi006 bash[27292]: cluster 2023-08-28T00:54:19.278211+0000 mgr.y (mgr.14901) 5272 : cluster [DBG] pgmap v5279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:21.341 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:20 smithi006 bash[33918]: cluster 2023-08-28T00:54:19.278211+0000 mgr.y (mgr.14901) 5272 : cluster [DBG] pgmap v5279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:21.341 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:54:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:54:20] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:54:21.342 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:54:20.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:21.342 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:54:20.867Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:21.342 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:17 smithi130 bash[27579]: audit 2023-08-28T00:54:16.948682+0000 mon.c (mon.2) 2038 : audit [DBG] from='client.? 172.21.15.6:0/1821488253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:21.342 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:17 smithi130 bash[51302]: debug 2023-08-28T00:54:17.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:17.992638+0000) 2023-08-28T00:54:21.343 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:18 smithi130 bash[27579]: cluster 2023-08-28T00:54:17.277141+0000 mgr.y (mgr.14901) 5271 : cluster [DBG] pgmap v5278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:21.343 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:18 smithi130 bash[51302]: debug 2023-08-28T00:54:18.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:18.992950+0000) 2023-08-28T00:54:21.343 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:19 smithi130 bash[27579]: audit 2023-08-28T00:54:19.368598+0000 mon.a (mon.0) 3680 : audit [DBG] from='client.? 172.21.15.6:0/3608884916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:21.343 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:19 smithi130 bash[51302]: debug 2023-08-28T00:54:19.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:19.993184+0000) 2023-08-28T00:54:21.552 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:20 smithi130 bash[27579]: cluster 2023-08-28T00:54:19.278211+0000 mgr.y (mgr.14901) 5272 : cluster [DBG] pgmap v5279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:21.552 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:20 smithi130 bash[51302]: debug 2023-08-28T00:54:20.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:20.993459+0000) 2023-08-28T00:54:22.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:21 smithi130 bash[51302]: debug 2023-08-28T00:54:21.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:21.993705+0000) 2023-08-28T00:54:22.838 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:22 smithi006 bash[33918]: cluster 2023-08-28T00:54:21.278781+0000 mgr.y (mgr.14901) 5273 : cluster [DBG] pgmap v5280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:22.838 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:22 smithi006 bash[33918]: audit 2023-08-28T00:54:21.776735+0000 mon.a (mon.0) 3681 : audit [DBG] from='client.? 172.21.15.6:0/1951121994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:22.838 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:22 smithi006 bash[27292]: cluster 2023-08-28T00:54:21.278781+0000 mgr.y (mgr.14901) 5273 : cluster [DBG] pgmap v5280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:22.857 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:22 smithi006 bash[27292]: audit 2023-08-28T00:54:21.776735+0000 mon.a (mon.0) 3681 : audit [DBG] from='client.? 172.21.15.6:0/1951121994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:22.857 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:54:22.437Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:22.857 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:54:22.439Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:22 smithi130 bash[27579]: cluster 2023-08-28T00:54:21.278781+0000 mgr.y (mgr.14901) 5273 : cluster [DBG] pgmap v5280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:22.993 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:22 smithi130 bash[27579]: audit 2023-08-28T00:54:21.776735+0000 mon.a (mon.0) 3681 : audit [DBG] from='client.? 172.21.15.6:0/1951121994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:23.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:22 smithi130 bash[51302]: debug 2023-08-28T00:54:22.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:22.993999+0000) 2023-08-28T00:54:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:23 smithi130 bash[51302]: debug 2023-08-28T00:54:23.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:23.994350+0000) 2023-08-28T00:54:24.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:24 smithi006 bash[27292]: cluster 2023-08-28T00:54:23.279411+0000 mgr.y (mgr.14901) 5274 : cluster [DBG] pgmap v5281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:24.756 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:24 smithi006 bash[27292]: audit 2023-08-28T00:54:24.187302+0000 mon.a (mon.0) 3682 : audit [DBG] from='client.? 172.21.15.6:0/263256235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:24.756 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:24 smithi006 bash[33918]: cluster 2023-08-28T00:54:23.279411+0000 mgr.y (mgr.14901) 5274 : cluster [DBG] pgmap v5281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:24.756 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:24 smithi006 bash[33918]: audit 2023-08-28T00:54:24.187302+0000 mon.a (mon.0) 3682 : audit [DBG] from='client.? 172.21.15.6:0/263256235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:24.756 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:54:24.318Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:24.757 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:54:24.321Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:24 smithi130 bash[27579]: cluster 2023-08-28T00:54:23.279411+0000 mgr.y (mgr.14901) 5274 : cluster [DBG] pgmap v5281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:24.921 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:24 smithi130 bash[27579]: audit 2023-08-28T00:54:24.187302+0000 mon.a (mon.0) 3682 : audit [DBG] from='client.? 172.21.15.6:0/263256235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:24 smithi130 bash[51302]: debug 2023-08-28T00:54:24.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:24.994660+0000) 2023-08-28T00:54:26.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:25 smithi130 bash[51302]: debug 2023-08-28T00:54:25.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:25.994954+0000) 2023-08-28T00:54:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:26 smithi006 bash[33918]: cluster 2023-08-28T00:54:25.280486+0000 mgr.y (mgr.14901) 5275 : cluster [DBG] pgmap v5282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:26.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:26 smithi006 bash[27292]: cluster 2023-08-28T00:54:25.280486+0000 mgr.y (mgr.14901) 5275 : cluster [DBG] pgmap v5282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:26 smithi130 bash[27579]: cluster 2023-08-28T00:54:25.280486+0000 mgr.y (mgr.14901) 5275 : cluster [DBG] pgmap v5282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:27.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:26 smithi130 bash[51302]: debug 2023-08-28T00:54:26.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:26.995214+0000) 2023-08-28T00:54:27.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:27 smithi006 bash[27292]: audit 2023-08-28T00:54:26.603789+0000 mon.c (mon.2) 2039 : audit [DBG] from='client.? 172.21.15.6:0/669662066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:27.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:27 smithi006 bash[33918]: audit 2023-08-28T00:54:26.603789+0000 mon.c (mon.2) 2039 : audit [DBG] from='client.? 172.21.15.6:0/669662066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:27 smithi130 bash[27579]: audit 2023-08-28T00:54:26.603789+0000 mon.c (mon.2) 2039 : audit [DBG] from='client.? 172.21.15.6:0/669662066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:28.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:27 smithi130 bash[51302]: debug 2023-08-28T00:54:27.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:27.995535+0000) 2023-08-28T00:54:28.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:28 smithi006 bash[27292]: cluster 2023-08-28T00:54:27.281152+0000 mgr.y (mgr.14901) 5276 : cluster [DBG] pgmap v5283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:28 smithi006 bash[33918]: cluster 2023-08-28T00:54:27.281152+0000 mgr.y (mgr.14901) 5276 : cluster [DBG] pgmap v5283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:28 smithi130 bash[27579]: cluster 2023-08-28T00:54:27.281152+0000 mgr.y (mgr.14901) 5276 : cluster [DBG] pgmap v5283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:29.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:28 smithi130 bash[51302]: debug 2023-08-28T00:54:28.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:28.995830+0000) 2023-08-28T00:54:29.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:29 smithi006 bash[27292]: audit 2023-08-28T00:54:29.020227+0000 mon.a (mon.0) 3683 : audit [DBG] from='client.? 172.21.15.6:0/661060726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:29.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:29 smithi006 bash[33918]: audit 2023-08-28T00:54:29.020227+0000 mon.a (mon.0) 3683 : audit [DBG] from='client.? 172.21.15.6:0/661060726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:29.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:29 smithi130 bash[27579]: audit 2023-08-28T00:54:29.020227+0000 mon.a (mon.0) 3683 : audit [DBG] from='client.? 172.21.15.6:0/661060726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:29 smithi130 bash[51302]: debug 2023-08-28T00:54:29.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:29.996125+0000) 2023-08-28T00:54:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:30 smithi006 bash[27292]: cluster 2023-08-28T00:54:29.282191+0000 mgr.y (mgr.14901) 5277 : cluster [DBG] pgmap v5284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:30.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:30 smithi006 bash[33918]: cluster 2023-08-28T00:54:29.282191+0000 mgr.y (mgr.14901) 5277 : cluster [DBG] pgmap v5284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:30 smithi130 bash[27579]: cluster 2023-08-28T00:54:29.282191+0000 mgr.y (mgr.14901) 5277 : cluster [DBG] pgmap v5284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:54:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:54:30] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:54:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:54:30.866Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:54:30.868Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:31.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:30 smithi130 bash[51302]: debug 2023-08-28T00:54:30.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:30.996404+0000) 2023-08-28T00:54:32.305 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:31 smithi130 bash[51302]: debug 2023-08-28T00:54:31.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:31.996705+0000) 2023-08-28T00:54:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:32 smithi130 bash[27579]: cluster 2023-08-28T00:54:31.282753+0000 mgr.y (mgr.14901) 5278 : cluster [DBG] pgmap v5285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:32 smithi130 bash[27579]: audit 2023-08-28T00:54:31.432857+0000 mon.a (mon.0) 3684 : audit [DBG] from='client.? 172.21.15.6:0/4152094906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:32.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:32 smithi006 bash[33918]: cluster 2023-08-28T00:54:31.282753+0000 mgr.y (mgr.14901) 5278 : cluster [DBG] pgmap v5285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:32 smithi006 bash[33918]: audit 2023-08-28T00:54:31.432857+0000 mon.a (mon.0) 3684 : audit [DBG] from='client.? 172.21.15.6:0/4152094906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:54:32.438Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:54:32.440Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:32.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:32 smithi006 bash[27292]: cluster 2023-08-28T00:54:31.282753+0000 mgr.y (mgr.14901) 5278 : cluster [DBG] pgmap v5285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:32.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:32 smithi006 bash[27292]: audit 2023-08-28T00:54:31.432857+0000 mon.a (mon.0) 3684 : audit [DBG] from='client.? 172.21.15.6:0/4152094906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:33.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:32 smithi130 bash[51302]: debug 2023-08-28T00:54:32.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:32.997009+0000) 2023-08-28T00:54:34.310 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:33 smithi130 bash[51302]: debug 2023-08-28T00:54:33.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:33.997252+0000) 2023-08-28T00:54:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:34 smithi130 bash[27579]: cluster 2023-08-28T00:54:33.283367+0000 mgr.y (mgr.14901) 5279 : cluster [DBG] pgmap v5286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:34 smithi130 bash[27579]: audit 2023-08-28T00:54:33.849978+0000 mon.c (mon.2) 2040 : audit [DBG] from='client.? 172.21.15.6:0/4237687902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:34 smithi006 bash[27292]: cluster 2023-08-28T00:54:33.283367+0000 mgr.y (mgr.14901) 5279 : cluster [DBG] pgmap v5286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:34 smithi006 bash[27292]: audit 2023-08-28T00:54:33.849978+0000 mon.c (mon.2) 2040 : audit [DBG] from='client.? 172.21.15.6:0/4237687902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:34 smithi006 bash[33918]: cluster 2023-08-28T00:54:33.283367+0000 mgr.y (mgr.14901) 5279 : cluster [DBG] pgmap v5286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:34 smithi006 bash[33918]: audit 2023-08-28T00:54:33.849978+0000 mon.c (mon.2) 2040 : audit [DBG] from='client.? 172.21.15.6:0/4237687902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:54:34.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:54:34.321Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:35.371 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:34 smithi130 bash[51302]: debug 2023-08-28T00:54:34.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:34.997553+0000) 2023-08-28T00:54:36.314 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:35 smithi130 bash[51302]: debug 2023-08-28T00:54:35.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:35.997845+0000) 2023-08-28T00:54:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:36 smithi130 bash[27579]: cluster 2023-08-28T00:54:35.284525+0000 mgr.y (mgr.14901) 5280 : cluster [DBG] pgmap v5287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:36.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:36 smithi130 bash[27579]: audit 2023-08-28T00:54:36.262099+0000 mon.a (mon.0) 3685 : audit [DBG] from='client.? 172.21.15.6:0/1735689052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:36 smithi006 bash[33918]: cluster 2023-08-28T00:54:35.284525+0000 mgr.y (mgr.14901) 5280 : cluster [DBG] pgmap v5287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:36 smithi006 bash[33918]: audit 2023-08-28T00:54:36.262099+0000 mon.a (mon.0) 3685 : audit [DBG] from='client.? 172.21.15.6:0/1735689052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:36.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:36 smithi006 bash[27292]: cluster 2023-08-28T00:54:35.284525+0000 mgr.y (mgr.14901) 5280 : cluster [DBG] pgmap v5287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:36.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:36 smithi006 bash[27292]: audit 2023-08-28T00:54:36.262099+0000 mon.a (mon.0) 3685 : audit [DBG] from='client.? 172.21.15.6:0/1735689052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:37.324 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:36 smithi130 bash[51302]: debug 2023-08-28T00:54:36.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:36.998130+0000) 2023-08-28T00:54:37.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:37 smithi130 bash[27579]: audit 2023-08-28T00:54:36.448563+0000 mon.b (mon.1) 626 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:54:37.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:37 smithi006 bash[27292]: audit 2023-08-28T00:54:36.448563+0000 mon.b (mon.1) 626 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:54:37.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:37 smithi006 bash[33918]: audit 2023-08-28T00:54:36.448563+0000 mon.b (mon.1) 626 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:54:38.328 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:37 smithi130 bash[51302]: debug 2023-08-28T00:54:37.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:37.998455+0000) 2023-08-28T00:54:38.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:38 smithi130 bash[27579]: cluster 2023-08-28T00:54:37.285220+0000 mgr.y (mgr.14901) 5281 : cluster [DBG] pgmap v5288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:38.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:38 smithi006 bash[27292]: cluster 2023-08-28T00:54:37.285220+0000 mgr.y (mgr.14901) 5281 : cluster [DBG] pgmap v5288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:38.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:38 smithi006 bash[33918]: cluster 2023-08-28T00:54:37.285220+0000 mgr.y (mgr.14901) 5281 : cluster [DBG] pgmap v5288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:39.335 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:38 smithi130 bash[51302]: debug 2023-08-28T00:54:38.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:38.998739+0000) 2023-08-28T00:54:39.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:39 smithi130 bash[27579]: audit 2023-08-28T00:54:38.692455+0000 mon.b (mon.1) 627 : audit [DBG] from='client.? 172.21.15.6:0/1362184757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:39.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:39 smithi006 bash[33918]: audit 2023-08-28T00:54:38.692455+0000 mon.b (mon.1) 627 : audit [DBG] from='client.? 172.21.15.6:0/1362184757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:39.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:39 smithi006 bash[27292]: audit 2023-08-28T00:54:38.692455+0000 mon.b (mon.1) 627 : audit [DBG] from='client.? 172.21.15.6:0/1362184757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:40.341 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:39 smithi130 bash[51302]: debug 2023-08-28T00:54:39.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:39.999092+0000) 2023-08-28T00:54:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:40 smithi130 bash[27579]: cluster 2023-08-28T00:54:39.286143+0000 mgr.y (mgr.14901) 5282 : cluster [DBG] pgmap v5289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:40 smithi006 bash[27292]: cluster 2023-08-28T00:54:39.286143+0000 mgr.y (mgr.14901) 5282 : cluster [DBG] pgmap v5289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:40 smithi006 bash[33918]: cluster 2023-08-28T00:54:39.286143+0000 mgr.y (mgr.14901) 5282 : cluster [DBG] pgmap v5289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:54:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:54:40] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:54:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:54:40.867Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:41.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:54:40.869Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:41.348 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:40 smithi130 bash[51302]: debug 2023-08-28T00:54:40.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:40.999376+0000) 2023-08-28T00:54:41.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:41 smithi130 bash[27579]: audit 2023-08-28T00:54:41.113784+0000 mon.c (mon.2) 2041 : audit [DBG] from='client.? 172.21.15.6:0/890782173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:41.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:41 smithi006 bash[27292]: audit 2023-08-28T00:54:41.113784+0000 mon.c (mon.2) 2041 : audit [DBG] from='client.? 172.21.15.6:0/890782173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:41.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:41 smithi006 bash[33918]: audit 2023-08-28T00:54:41.113784+0000 mon.c (mon.2) 2041 : audit [DBG] from='client.? 172.21.15.6:0/890782173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:42.355 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:41 smithi130 bash[51302]: debug 2023-08-28T00:54:41.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:41.999591+0000) 2023-08-28T00:54:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:42 smithi130 bash[27579]: cluster 2023-08-28T00:54:41.286758+0000 mgr.y (mgr.14901) 5283 : cluster [DBG] pgmap v5290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:42 smithi006 bash[27292]: cluster 2023-08-28T00:54:41.286758+0000 mgr.y (mgr.14901) 5283 : cluster [DBG] pgmap v5290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:42.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:42 smithi006 bash[33918]: cluster 2023-08-28T00:54:41.286758+0000 mgr.y (mgr.14901) 5283 : cluster [DBG] pgmap v5290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:54:42.438Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:42.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:54:42.440Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:43.358 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:43 smithi130 bash[51302]: debug 2023-08-28T00:54:42.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:42.999866+0000) 2023-08-28T00:54:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:43 smithi130 bash[27579]: audit 2023-08-28T00:54:42.348901+0000 mon.a (mon.0) 3686 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:54:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:43 smithi130 bash[27579]: audit 2023-08-28T00:54:42.725486+0000 mon.a (mon.0) 3687 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:54:43.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:43 smithi006 bash[33918]: audit 2023-08-28T00:54:42.348901+0000 mon.a (mon.0) 3686 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:54:43.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:43 smithi006 bash[33918]: audit 2023-08-28T00:54:42.725486+0000 mon.a (mon.0) 3687 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:54:43.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:43 smithi006 bash[27292]: audit 2023-08-28T00:54:42.348901+0000 mon.a (mon.0) 3686 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:54:43.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:43 smithi006 bash[27292]: audit 2023-08-28T00:54:42.725486+0000 mon.a (mon.0) 3687 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:54:44.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:44 smithi130 bash[51302]: debug 2023-08-28T00:54:43.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:44.000132+0000) 2023-08-28T00:54:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:44 smithi130 bash[27579]: cluster 2023-08-28T00:54:43.287235+0000 mgr.y (mgr.14901) 5284 : cluster [DBG] pgmap v5291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:44.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:44 smithi130 bash[27579]: audit 2023-08-28T00:54:43.531060+0000 mon.a (mon.0) 3688 : audit [DBG] from='client.? 172.21.15.6:0/814101274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:44.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:44 smithi006 bash[27292]: cluster 2023-08-28T00:54:43.287235+0000 mgr.y (mgr.14901) 5284 : cluster [DBG] pgmap v5291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:44.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:44 smithi006 bash[27292]: audit 2023-08-28T00:54:43.531060+0000 mon.a (mon.0) 3688 : audit [DBG] from='client.? 172.21.15.6:0/814101274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:44.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:44 smithi006 bash[33918]: cluster 2023-08-28T00:54:43.287235+0000 mgr.y (mgr.14901) 5284 : cluster [DBG] pgmap v5291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:44.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:44 smithi006 bash[33918]: audit 2023-08-28T00:54:43.531060+0000 mon.a (mon.0) 3688 : audit [DBG] from='client.? 172.21.15.6:0/814101274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:54:44.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:54:44.322Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:45 smithi130 bash[51302]: debug 2023-08-28T00:54:44.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:45.000368+0000) 2023-08-28T00:54:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:46 smithi130 bash[51302]: debug 2023-08-28T00:54:45.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:46.000615+0000) 2023-08-28T00:54:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:46 smithi006 bash[27292]: cluster 2023-08-28T00:54:45.288282+0000 mgr.y (mgr.14901) 5285 : cluster [DBG] pgmap v5292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:46.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:46 smithi006 bash[27292]: audit 2023-08-28T00:54:45.944424+0000 mon.a (mon.0) 3689 : audit [DBG] from='client.? 172.21.15.6:0/3929170378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:46.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:46 smithi006 bash[33918]: cluster 2023-08-28T00:54:45.288282+0000 mgr.y (mgr.14901) 5285 : cluster [DBG] pgmap v5292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:46.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:46 smithi006 bash[33918]: audit 2023-08-28T00:54:45.944424+0000 mon.a (mon.0) 3689 : audit [DBG] from='client.? 172.21.15.6:0/3929170378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:46 smithi130 bash[27579]: cluster 2023-08-28T00:54:45.288282+0000 mgr.y (mgr.14901) 5285 : cluster [DBG] pgmap v5292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:46 smithi130 bash[27579]: audit 2023-08-28T00:54:45.944424+0000 mon.a (mon.0) 3689 : audit [DBG] from='client.? 172.21.15.6:0/3929170378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:47 smithi130 bash[51302]: debug 2023-08-28T00:54:46.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:47.000881+0000) 2023-08-28T00:54:48.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:48 smithi130 bash[51302]: debug 2023-08-28T00:54:47.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:48.001185+0000) 2023-08-28T00:54:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:48 smithi006 bash[33918]: cluster 2023-08-28T00:54:47.288953+0000 mgr.y (mgr.14901) 5286 : cluster [DBG] pgmap v5293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:48.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:48 smithi006 bash[33918]: audit 2023-08-28T00:54:48.357191+0000 mon.c (mon.2) 2042 : audit [DBG] from='client.? 172.21.15.6:0/3450321838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:48.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:48 smithi006 bash[27292]: cluster 2023-08-28T00:54:47.288953+0000 mgr.y (mgr.14901) 5286 : cluster [DBG] pgmap v5293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:48.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:48 smithi006 bash[27292]: audit 2023-08-28T00:54:48.357191+0000 mon.c (mon.2) 2042 : audit [DBG] from='client.? 172.21.15.6:0/3450321838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:48.871 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:48 smithi130 bash[27579]: cluster 2023-08-28T00:54:47.288953+0000 mgr.y (mgr.14901) 5286 : cluster [DBG] pgmap v5293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:48.872 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:48 smithi130 bash[27579]: audit 2023-08-28T00:54:48.357191+0000 mon.c (mon.2) 2042 : audit [DBG] from='client.? 172.21.15.6:0/3450321838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:49.307 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:49 smithi130 bash[51302]: debug 2023-08-28T00:54:48.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:49.001507+0000) 2023-08-28T00:54:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:50 smithi130 bash[51302]: debug 2023-08-28T00:54:49.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:50.001750+0000) 2023-08-28T00:54:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:50 smithi006 bash[27292]: cluster 2023-08-28T00:54:49.290029+0000 mgr.y (mgr.14901) 5287 : cluster [DBG] pgmap v5294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:50.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:50 smithi006 bash[33918]: cluster 2023-08-28T00:54:49.290029+0000 mgr.y (mgr.14901) 5287 : cluster [DBG] pgmap v5294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:50 smithi130 bash[27579]: cluster 2023-08-28T00:54:49.290029+0000 mgr.y (mgr.14901) 5287 : cluster [DBG] pgmap v5294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:54:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:54:50] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:54:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:54:50.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:51.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:54:50.870Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:51.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:51 smithi130 bash[51302]: debug 2023-08-28T00:54:50.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:51.002007+0000) 2023-08-28T00:54:51.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:51 smithi006 bash[27292]: audit 2023-08-28T00:54:50.777401+0000 mon.c (mon.2) 2043 : audit [DBG] from='client.? 172.21.15.6:0/1416288621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:51.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:51 smithi006 bash[33918]: audit 2023-08-28T00:54:50.777401+0000 mon.c (mon.2) 2043 : audit [DBG] from='client.? 172.21.15.6:0/1416288621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:51.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:51 smithi130 bash[27579]: audit 2023-08-28T00:54:50.777401+0000 mon.c (mon.2) 2043 : audit [DBG] from='client.? 172.21.15.6:0/1416288621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:52.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:52 smithi130 bash[51302]: debug 2023-08-28T00:54:51.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:52.002275+0000) 2023-08-28T00:54:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:52 smithi006 bash[33918]: cluster 2023-08-28T00:54:51.290778+0000 mgr.y (mgr.14901) 5288 : cluster [DBG] pgmap v5295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:52 smithi006 bash[27292]: cluster 2023-08-28T00:54:51.290778+0000 mgr.y (mgr.14901) 5288 : cluster [DBG] pgmap v5295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:54:52.439Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:54:52.442Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:52 smithi130 bash[27579]: cluster 2023-08-28T00:54:51.290778+0000 mgr.y (mgr.14901) 5288 : cluster [DBG] pgmap v5295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:53.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:53 smithi130 bash[51302]: debug 2023-08-28T00:54:52.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:53.002527+0000) 2023-08-28T00:54:54.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:53 smithi006 bash[27292]: audit 2023-08-28T00:54:52.949949+0000 mon.a (mon.0) 3690 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:54:54.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:53 smithi006 bash[27292]: audit 2023-08-28T00:54:53.199047+0000 mon.a (mon.0) 3691 : audit [DBG] from='client.? 172.21.15.6:0/4196260088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:54.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:53 smithi006 bash[27292]: audit 2023-08-28T00:54:53.239296+0000 mon.b (mon.1) 628 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:54:54.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:53 smithi006 bash[27292]: audit 2023-08-28T00:54:53.241323+0000 mon.b (mon.1) 629 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:54:54.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:53 smithi006 bash[27292]: audit 2023-08-28T00:54:53.249073+0000 mon.a (mon.0) 3692 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:54:54.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:53 smithi006 bash[27292]: cluster 2023-08-28T00:54:53.291349+0000 mgr.y (mgr.14901) 5289 : cluster [DBG] pgmap v5296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:54.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:53 smithi006 bash[33918]: audit 2023-08-28T00:54:52.949949+0000 mon.a (mon.0) 3690 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:54:54.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:53 smithi006 bash[33918]: audit 2023-08-28T00:54:53.199047+0000 mon.a (mon.0) 3691 : audit [DBG] from='client.? 172.21.15.6:0/4196260088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:54.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:53 smithi006 bash[33918]: audit 2023-08-28T00:54:53.239296+0000 mon.b (mon.1) 628 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:54:54.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:53 smithi006 bash[33918]: audit 2023-08-28T00:54:53.241323+0000 mon.b (mon.1) 629 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:54:54.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:53 smithi006 bash[33918]: audit 2023-08-28T00:54:53.249073+0000 mon.a (mon.0) 3692 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:54:54.219 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:53 smithi006 bash[33918]: cluster 2023-08-28T00:54:53.291349+0000 mgr.y (mgr.14901) 5289 : cluster [DBG] pgmap v5296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:54 smithi130 bash[51302]: debug 2023-08-28T00:54:54.000+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:54.002872+0000) 2023-08-28T00:54:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:53 smithi130 bash[27579]: audit 2023-08-28T00:54:52.949949+0000 mon.a (mon.0) 3690 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:54:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:53 smithi130 bash[27579]: audit 2023-08-28T00:54:53.199047+0000 mon.a (mon.0) 3691 : audit [DBG] from='client.? 172.21.15.6:0/4196260088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:53 smithi130 bash[27579]: audit 2023-08-28T00:54:53.239296+0000 mon.b (mon.1) 628 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:54:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:53 smithi130 bash[27579]: audit 2023-08-28T00:54:53.241323+0000 mon.b (mon.1) 629 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:54:54.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:53 smithi130 bash[27579]: audit 2023-08-28T00:54:53.249073+0000 mon.a (mon.0) 3692 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:54:54.369 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:53 smithi130 bash[27579]: cluster 2023-08-28T00:54:53.291349+0000 mgr.y (mgr.14901) 5289 : cluster [DBG] pgmap v5296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:54.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:54:54.320Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:54.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:54:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:54:54.322Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:54:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:55 smithi130 bash[51302]: debug 2023-08-28T00:54:55.000+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:55.003167+0000) 2023-08-28T00:54:56.354 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:56 smithi130 bash[51302]: debug 2023-08-28T00:54:55.999+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:56.003492+0000) 2023-08-28T00:54:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:56 smithi130 bash[27579]: cluster 2023-08-28T00:54:55.292448+0000 mgr.y (mgr.14901) 5290 : cluster [DBG] pgmap v5297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:56.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:56 smithi130 bash[27579]: audit 2023-08-28T00:54:55.616411+0000 mon.a (mon.0) 3693 : audit [DBG] from='client.? 172.21.15.6:0/1944704103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:56 smithi006 bash[27292]: cluster 2023-08-28T00:54:55.292448+0000 mgr.y (mgr.14901) 5290 : cluster [DBG] pgmap v5297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:56 smithi006 bash[27292]: audit 2023-08-28T00:54:55.616411+0000 mon.a (mon.0) 3693 : audit [DBG] from='client.? 172.21.15.6:0/1944704103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:56.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:56 smithi006 bash[33918]: cluster 2023-08-28T00:54:55.292448+0000 mgr.y (mgr.14901) 5290 : cluster [DBG] pgmap v5297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:56.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:56 smithi006 bash[33918]: audit 2023-08-28T00:54:55.616411+0000 mon.a (mon.0) 3693 : audit [DBG] from='client.? 172.21.15.6:0/1944704103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:57.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:57 smithi130 bash[51302]: debug 2023-08-28T00:54:56.999+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:57.003704+0000) 2023-08-28T00:54:58.358 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:58 smithi130 bash[51302]: debug 2023-08-28T00:54:57.999+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:58.003977+0000) 2023-08-28T00:54:58.687 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:58 smithi130 bash[27579]: cluster 2023-08-28T00:54:57.293131+0000 mgr.y (mgr.14901) 5291 : cluster [DBG] pgmap v5298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:58.688 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:54:58 smithi130 bash[27579]: audit 2023-08-28T00:54:58.033953+0000 mon.a (mon.0) 3694 : audit [DBG] from='client.? 172.21.15.6:0/3449159746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:58 smithi006 bash[27292]: cluster 2023-08-28T00:54:57.293131+0000 mgr.y (mgr.14901) 5291 : cluster [DBG] pgmap v5298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:59.237 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:54:58 smithi006 bash[27292]: audit 2023-08-28T00:54:58.033953+0000 mon.a (mon.0) 3694 : audit [DBG] from='client.? 172.21.15.6:0/3449159746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:59.238 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:58 smithi006 bash[33918]: cluster 2023-08-28T00:54:57.293131+0000 mgr.y (mgr.14901) 5291 : cluster [DBG] pgmap v5298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:54:59.238 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:54:58 smithi006 bash[33918]: audit 2023-08-28T00:54:58.033953+0000 mon.a (mon.0) 3694 : audit [DBG] from='client.? 172.21.15.6:0/3449159746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:54:59 smithi130 bash[51302]: debug 2023-08-28T00:54:59.003+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:59.004285+0000) 2023-08-28T00:55:00.363 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:00 smithi130 bash[51302]: debug 2023-08-28T00:54:59.999+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:00.004566+0000) 2023-08-28T00:55:00.791 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:00 smithi006 bash[33918]: cluster 2023-08-28T00:54:59.294183+0000 mgr.y (mgr.14901) 5292 : cluster [DBG] pgmap v5299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:00.791 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:00 smithi130 bash[27579]: cluster 2023-08-28T00:54:59.294183+0000 mgr.y (mgr.14901) 5292 : cluster [DBG] pgmap v5299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:00.799 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:00 smithi006 bash[27292]: cluster 2023-08-28T00:54:59.294183+0000 mgr.y (mgr.14901) 5292 : cluster [DBG] pgmap v5299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:01.180 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:55:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:55:00] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:55:01.276 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:55:00.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:01.276 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:55:00.871Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:01.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:01 smithi130 bash[51302]: debug 2023-08-28T00:55:01.003+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:01.004857+0000) 2023-08-28T00:55:01.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:01 smithi006 bash[27292]: audit 2023-08-28T00:55:00.454895+0000 mon.c (mon.2) 2044 : audit [DBG] from='client.? 172.21.15.6:0/2641777874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:01.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:01 smithi006 bash[33918]: audit 2023-08-28T00:55:00.454895+0000 mon.c (mon.2) 2044 : audit [DBG] from='client.? 172.21.15.6:0/2641777874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:01.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:01 smithi130 bash[27579]: audit 2023-08-28T00:55:00.454895+0000 mon.c (mon.2) 2044 : audit [DBG] from='client.? 172.21.15.6:0/2641777874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:02.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:02 smithi130 bash[51302]: debug 2023-08-28T00:55:01.999+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:02.005135+0000) 2023-08-28T00:55:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:02 smithi006 bash[27292]: cluster 2023-08-28T00:55:01.294828+0000 mgr.y (mgr.14901) 5293 : cluster [DBG] pgmap v5300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:02 smithi006 bash[33918]: cluster 2023-08-28T00:55:01.294828+0000 mgr.y (mgr.14901) 5293 : cluster [DBG] pgmap v5300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:55:02.440Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:55:02.442Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:02 smithi130 bash[27579]: cluster 2023-08-28T00:55:01.294828+0000 mgr.y (mgr.14901) 5293 : cluster [DBG] pgmap v5300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:03 smithi130 bash[51302]: debug 2023-08-28T00:55:03.003+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:03.005441+0000) 2023-08-28T00:55:03.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:03 smithi006 bash[33918]: audit 2023-08-28T00:55:02.877823+0000 mon.c (mon.2) 2045 : audit [DBG] from='client.? 172.21.15.6:0/127401905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:03.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:03 smithi006 bash[27292]: audit 2023-08-28T00:55:02.877823+0000 mon.c (mon.2) 2045 : audit [DBG] from='client.? 172.21.15.6:0/127401905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:03.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:03 smithi130 bash[27579]: audit 2023-08-28T00:55:02.877823+0000 mon.c (mon.2) 2045 : audit [DBG] from='client.? 172.21.15.6:0/127401905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:04 smithi130 bash[51302]: debug 2023-08-28T00:55:03.999+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:04.005704+0000) 2023-08-28T00:55:04.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:04 smithi006 bash[33918]: cluster 2023-08-28T00:55:03.295447+0000 mgr.y (mgr.14901) 5294 : cluster [DBG] pgmap v5301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:55:04.321Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:55:04.323Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:04.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:04 smithi006 bash[27292]: cluster 2023-08-28T00:55:03.295447+0000 mgr.y (mgr.14901) 5294 : cluster [DBG] pgmap v5301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:04 smithi130 bash[27579]: cluster 2023-08-28T00:55:03.295447+0000 mgr.y (mgr.14901) 5294 : cluster [DBG] pgmap v5301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:05 smithi130 bash[51302]: debug 2023-08-28T00:55:04.999+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:05.005978+0000) 2023-08-28T00:55:05.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:05 smithi006 bash[33918]: audit 2023-08-28T00:55:05.286097+0000 mon.c (mon.2) 2046 : audit [DBG] from='client.? 172.21.15.6:0/1213118752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:05.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:05 smithi006 bash[27292]: audit 2023-08-28T00:55:05.286097+0000 mon.c (mon.2) 2046 : audit [DBG] from='client.? 172.21.15.6:0/1213118752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:05.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:05 smithi130 bash[27579]: audit 2023-08-28T00:55:05.286097+0000 mon.c (mon.2) 2046 : audit [DBG] from='client.? 172.21.15.6:0/1213118752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:06.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:06 smithi130 bash[51302]: debug 2023-08-28T00:55:06.003+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:06.006293+0000) 2023-08-28T00:55:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:06 smithi006 bash[27292]: cluster 2023-08-28T00:55:05.296514+0000 mgr.y (mgr.14901) 5295 : cluster [DBG] pgmap v5302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:06 smithi006 bash[33918]: cluster 2023-08-28T00:55:05.296514+0000 mgr.y (mgr.14901) 5295 : cluster [DBG] pgmap v5302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:06 smithi130 bash[27579]: cluster 2023-08-28T00:55:05.296514+0000 mgr.y (mgr.14901) 5295 : cluster [DBG] pgmap v5302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:07.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:07 smithi130 bash[51302]: debug 2023-08-28T00:55:06.999+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:07.006574+0000) 2023-08-28T00:55:08.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:08 smithi130 bash[51302]: debug 2023-08-28T00:55:08.003+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:08.006890+0000) 2023-08-28T00:55:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:08 smithi006 bash[27292]: cluster 2023-08-28T00:55:07.297227+0000 mgr.y (mgr.14901) 5296 : cluster [DBG] pgmap v5303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:08.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:08 smithi006 bash[27292]: audit 2023-08-28T00:55:07.711269+0000 mon.c (mon.2) 2047 : audit [DBG] from='client.? 172.21.15.6:0/574123599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:08 smithi006 bash[33918]: cluster 2023-08-28T00:55:07.297227+0000 mgr.y (mgr.14901) 5296 : cluster [DBG] pgmap v5303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:08.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:08 smithi006 bash[33918]: audit 2023-08-28T00:55:07.711269+0000 mon.c (mon.2) 2047 : audit [DBG] from='client.? 172.21.15.6:0/574123599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:08 smithi130 bash[27579]: cluster 2023-08-28T00:55:07.297227+0000 mgr.y (mgr.14901) 5296 : cluster [DBG] pgmap v5303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:08 smithi130 bash[27579]: audit 2023-08-28T00:55:07.711269+0000 mon.c (mon.2) 2047 : audit [DBG] from='client.? 172.21.15.6:0/574123599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:09.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:09 smithi130 bash[51302]: debug 2023-08-28T00:55:09.003+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:09.007110+0000) 2023-08-28T00:55:10.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:10 smithi130 bash[51302]: debug 2023-08-28T00:55:10.003+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:10.007399+0000) 2023-08-28T00:55:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:10 smithi006 bash[33918]: cluster 2023-08-28T00:55:09.298293+0000 mgr.y (mgr.14901) 5297 : cluster [DBG] pgmap v5304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:10 smithi006 bash[33918]: audit 2023-08-28T00:55:10.128464+0000 mon.c (mon.2) 2048 : audit [DBG] from='client.? 172.21.15.6:0/4176242953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:10 smithi006 bash[27292]: cluster 2023-08-28T00:55:09.298293+0000 mgr.y (mgr.14901) 5297 : cluster [DBG] pgmap v5304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:10 smithi006 bash[27292]: audit 2023-08-28T00:55:10.128464+0000 mon.c (mon.2) 2048 : audit [DBG] from='client.? 172.21.15.6:0/4176242953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:10 smithi130 bash[27579]: cluster 2023-08-28T00:55:09.298293+0000 mgr.y (mgr.14901) 5297 : cluster [DBG] pgmap v5304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:10 smithi130 bash[27579]: audit 2023-08-28T00:55:10.128464+0000 mon.c (mon.2) 2048 : audit [DBG] from='client.? 172.21.15.6:0/4176242953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:55:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:55:10] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:55:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:55:10.869Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:55:10.871Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:11.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:11 smithi130 bash[51302]: debug 2023-08-28T00:55:11.003+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:11.007701+0000) 2023-08-28T00:55:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:12 smithi130 bash[51302]: debug 2023-08-28T00:55:12.003+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:12.008017+0000) 2023-08-28T00:55:12.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:12 smithi006 bash[27292]: cluster 2023-08-28T00:55:11.298933+0000 mgr.y (mgr.14901) 5298 : cluster [DBG] pgmap v5305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:12 smithi006 bash[33918]: cluster 2023-08-28T00:55:11.298933+0000 mgr.y (mgr.14901) 5298 : cluster [DBG] pgmap v5305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:55:12.441Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:55:12.443Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:12 smithi130 bash[27579]: cluster 2023-08-28T00:55:11.298933+0000 mgr.y (mgr.14901) 5298 : cluster [DBG] pgmap v5305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:13 smithi130 bash[51302]: debug 2023-08-28T00:55:13.003+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:13.008297+0000) 2023-08-28T00:55:13.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:13 smithi006 bash[33918]: audit 2023-08-28T00:55:12.546983+0000 mon.c (mon.2) 2049 : audit [DBG] from='client.? 172.21.15.6:0/784016234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:13.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:13 smithi006 bash[27292]: audit 2023-08-28T00:55:12.546983+0000 mon.c (mon.2) 2049 : audit [DBG] from='client.? 172.21.15.6:0/784016234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:13.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:13 smithi130 bash[27579]: audit 2023-08-28T00:55:12.546983+0000 mon.c (mon.2) 2049 : audit [DBG] from='client.? 172.21.15.6:0/784016234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:14.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:14 smithi130 bash[51302]: debug 2023-08-28T00:55:14.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:14.008577+0000) 2023-08-28T00:55:14.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:14 smithi006 bash[27292]: cluster 2023-08-28T00:55:13.299572+0000 mgr.y (mgr.14901) 5299 : cluster [DBG] pgmap v5306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:14.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:14 smithi006 bash[33918]: cluster 2023-08-28T00:55:13.299572+0000 mgr.y (mgr.14901) 5299 : cluster [DBG] pgmap v5306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:55:14.322Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:55:14.324Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:14 smithi130 bash[27579]: cluster 2023-08-28T00:55:13.299572+0000 mgr.y (mgr.14901) 5299 : cluster [DBG] pgmap v5306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:15 smithi130 bash[51302]: debug 2023-08-28T00:55:15.003+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:15.008849+0000) 2023-08-28T00:55:15.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:15 smithi006 bash[27292]: audit 2023-08-28T00:55:14.959271+0000 mon.a (mon.0) 3695 : audit [DBG] from='client.? 172.21.15.6:0/3072848057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:15.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:15 smithi006 bash[33918]: audit 2023-08-28T00:55:14.959271+0000 mon.a (mon.0) 3695 : audit [DBG] from='client.? 172.21.15.6:0/3072848057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:15.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:15 smithi130 bash[27579]: audit 2023-08-28T00:55:14.959271+0000 mon.a (mon.0) 3695 : audit [DBG] from='client.? 172.21.15.6:0/3072848057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:16.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:16 smithi130 bash[51302]: debug 2023-08-28T00:55:16.003+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:16.009135+0000) 2023-08-28T00:55:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:16 smithi006 bash[27292]: cluster 2023-08-28T00:55:15.300642+0000 mgr.y (mgr.14901) 5300 : cluster [DBG] pgmap v5307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:16 smithi006 bash[33918]: cluster 2023-08-28T00:55:15.300642+0000 mgr.y (mgr.14901) 5300 : cluster [DBG] pgmap v5307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:16 smithi130 bash[27579]: cluster 2023-08-28T00:55:15.300642+0000 mgr.y (mgr.14901) 5300 : cluster [DBG] pgmap v5307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:17.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:17 smithi130 bash[51302]: debug 2023-08-28T00:55:17.003+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:17.009455+0000) 2023-08-28T00:55:17.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:17 smithi006 bash[33918]: audit 2023-08-28T00:55:17.385101+0000 mon.a (mon.0) 3696 : audit [DBG] from='client.? 172.21.15.6:0/3534635114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:17.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:17 smithi006 bash[27292]: audit 2023-08-28T00:55:17.385101+0000 mon.a (mon.0) 3696 : audit [DBG] from='client.? 172.21.15.6:0/3534635114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:17.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:17 smithi130 bash[27579]: audit 2023-08-28T00:55:17.385101+0000 mon.a (mon.0) 3696 : audit [DBG] from='client.? 172.21.15.6:0/3534635114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:18.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:18 smithi130 bash[51302]: debug 2023-08-28T00:55:18.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:18.009723+0000) 2023-08-28T00:55:18.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:18 smithi006 bash[27292]: cluster 2023-08-28T00:55:17.301274+0000 mgr.y (mgr.14901) 5301 : cluster [DBG] pgmap v5308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:18.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:18 smithi006 bash[33918]: cluster 2023-08-28T00:55:17.301274+0000 mgr.y (mgr.14901) 5301 : cluster [DBG] pgmap v5308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:18 smithi130 bash[27579]: cluster 2023-08-28T00:55:17.301274+0000 mgr.y (mgr.14901) 5301 : cluster [DBG] pgmap v5308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:19.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:19 smithi130 bash[51302]: debug 2023-08-28T00:55:19.003+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:19.010025+0000) 2023-08-28T00:55:20.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:20 smithi130 bash[51302]: debug 2023-08-28T00:55:20.003+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:20.010308+0000) 2023-08-28T00:55:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:20 smithi006 bash[33918]: cluster 2023-08-28T00:55:19.302337+0000 mgr.y (mgr.14901) 5302 : cluster [DBG] pgmap v5309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:20.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:20 smithi006 bash[33918]: audit 2023-08-28T00:55:19.797943+0000 mon.c (mon.2) 2050 : audit [DBG] from='client.? 172.21.15.6:0/3512984284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:20 smithi006 bash[27292]: cluster 2023-08-28T00:55:19.302337+0000 mgr.y (mgr.14901) 5302 : cluster [DBG] pgmap v5309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:20.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:20 smithi006 bash[27292]: audit 2023-08-28T00:55:19.797943+0000 mon.c (mon.2) 2050 : audit [DBG] from='client.? 172.21.15.6:0/3512984284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:20 smithi130 bash[27579]: cluster 2023-08-28T00:55:19.302337+0000 mgr.y (mgr.14901) 5302 : cluster [DBG] pgmap v5309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:20 smithi130 bash[27579]: audit 2023-08-28T00:55:19.797943+0000 mon.c (mon.2) 2050 : audit [DBG] from='client.? 172.21.15.6:0/3512984284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:55:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:55:20] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:55:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:55:20.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:55:20.872Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:21.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:21 smithi130 bash[51302]: debug 2023-08-28T00:55:21.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:21.010595+0000) 2023-08-28T00:55:22.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:22 smithi130 bash[51302]: debug 2023-08-28T00:55:22.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:22.010883+0000) 2023-08-28T00:55:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:55:22.441Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:55:22.443Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:22 smithi006 bash[27292]: cluster 2023-08-28T00:55:21.302970+0000 mgr.y (mgr.14901) 5303 : cluster [DBG] pgmap v5310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:22 smithi006 bash[27292]: audit 2023-08-28T00:55:22.220513+0000 mon.a (mon.0) 3697 : audit [DBG] from='client.? 172.21.15.6:0/214989872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:22 smithi006 bash[33918]: cluster 2023-08-28T00:55:21.302970+0000 mgr.y (mgr.14901) 5303 : cluster [DBG] pgmap v5310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:22 smithi006 bash[33918]: audit 2023-08-28T00:55:22.220513+0000 mon.a (mon.0) 3697 : audit [DBG] from='client.? 172.21.15.6:0/214989872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:22 smithi130 bash[27579]: cluster 2023-08-28T00:55:21.302970+0000 mgr.y (mgr.14901) 5303 : cluster [DBG] pgmap v5310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:22 smithi130 bash[27579]: audit 2023-08-28T00:55:22.220513+0000 mon.a (mon.0) 3697 : audit [DBG] from='client.? 172.21.15.6:0/214989872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:23.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:23 smithi130 bash[51302]: debug 2023-08-28T00:55:23.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:23.011130+0000) 2023-08-28T00:55:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:24 smithi130 bash[51302]: debug 2023-08-28T00:55:24.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:24.011409+0000) 2023-08-28T00:55:24.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:24 smithi006 bash[33918]: cluster 2023-08-28T00:55:23.303649+0000 mgr.y (mgr.14901) 5304 : cluster [DBG] pgmap v5311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:24.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:24 smithi006 bash[27292]: cluster 2023-08-28T00:55:23.303649+0000 mgr.y (mgr.14901) 5304 : cluster [DBG] pgmap v5311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:55:24.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:55:24.325Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:24 smithi130 bash[27579]: cluster 2023-08-28T00:55:23.303649+0000 mgr.y (mgr.14901) 5304 : cluster [DBG] pgmap v5311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:25 smithi130 bash[51302]: debug 2023-08-28T00:55:25.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:25.011696+0000) 2023-08-28T00:55:25.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:25 smithi130 bash[27579]: audit 2023-08-28T00:55:24.628721+0000 mon.c (mon.2) 2051 : audit [DBG] from='client.? 172.21.15.6:0/1260026595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:25.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:25 smithi006 bash[33918]: audit 2023-08-28T00:55:24.628721+0000 mon.c (mon.2) 2051 : audit [DBG] from='client.? 172.21.15.6:0/1260026595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:25.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:25 smithi006 bash[27292]: audit 2023-08-28T00:55:24.628721+0000 mon.c (mon.2) 2051 : audit [DBG] from='client.? 172.21.15.6:0/1260026595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:26.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:26 smithi130 bash[51302]: debug 2023-08-28T00:55:26.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:26.011969+0000) 2023-08-28T00:55:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:26 smithi130 bash[27579]: cluster 2023-08-28T00:55:25.304786+0000 mgr.y (mgr.14901) 5305 : cluster [DBG] pgmap v5312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:26 smithi006 bash[33918]: cluster 2023-08-28T00:55:25.304786+0000 mgr.y (mgr.14901) 5305 : cluster [DBG] pgmap v5312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:26.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:26 smithi006 bash[27292]: cluster 2023-08-28T00:55:25.304786+0000 mgr.y (mgr.14901) 5305 : cluster [DBG] pgmap v5312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:27.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:27 smithi130 bash[51302]: debug 2023-08-28T00:55:27.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:27.012230+0000) 2023-08-28T00:55:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:27 smithi130 bash[27579]: audit 2023-08-28T00:55:27.046156+0000 mon.c (mon.2) 2052 : audit [DBG] from='client.? 172.21.15.6:0/2439093639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:27.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:27 smithi006 bash[27292]: audit 2023-08-28T00:55:27.046156+0000 mon.c (mon.2) 2052 : audit [DBG] from='client.? 172.21.15.6:0/2439093639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:27.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:27 smithi006 bash[33918]: audit 2023-08-28T00:55:27.046156+0000 mon.c (mon.2) 2052 : audit [DBG] from='client.? 172.21.15.6:0/2439093639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:28.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:28 smithi130 bash[51302]: debug 2023-08-28T00:55:28.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:28.012520+0000) 2023-08-28T00:55:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:28 smithi130 bash[27579]: cluster 2023-08-28T00:55:27.305420+0000 mgr.y (mgr.14901) 5306 : cluster [DBG] pgmap v5313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:28 smithi006 bash[27292]: cluster 2023-08-28T00:55:27.305420+0000 mgr.y (mgr.14901) 5306 : cluster [DBG] pgmap v5313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:28.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:28 smithi006 bash[33918]: cluster 2023-08-28T00:55:27.305420+0000 mgr.y (mgr.14901) 5306 : cluster [DBG] pgmap v5313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:29.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:29 smithi130 bash[51302]: debug 2023-08-28T00:55:29.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:29.012854+0000) 2023-08-28T00:55:29.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:29 smithi130 bash[27579]: audit 2023-08-28T00:55:29.454188+0000 mon.c (mon.2) 2053 : audit [DBG] from='client.? 172.21.15.6:0/3929885190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:29.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:29 smithi006 bash[33918]: audit 2023-08-28T00:55:29.454188+0000 mon.c (mon.2) 2053 : audit [DBG] from='client.? 172.21.15.6:0/3929885190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:29.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:29 smithi006 bash[27292]: audit 2023-08-28T00:55:29.454188+0000 mon.c (mon.2) 2053 : audit [DBG] from='client.? 172.21.15.6:0/3929885190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:30 smithi130 bash[51302]: debug 2023-08-28T00:55:30.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:30.013147+0000) 2023-08-28T00:55:30.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:30 smithi006 bash[27292]: cluster 2023-08-28T00:55:29.306502+0000 mgr.y (mgr.14901) 5307 : cluster [DBG] pgmap v5314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:30.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:30 smithi006 bash[33918]: cluster 2023-08-28T00:55:29.306502+0000 mgr.y (mgr.14901) 5307 : cluster [DBG] pgmap v5314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:30 smithi130 bash[27579]: cluster 2023-08-28T00:55:29.306502+0000 mgr.y (mgr.14901) 5307 : cluster [DBG] pgmap v5314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:31.094 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:55:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:55:30] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:55:31.094 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:55:30.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:31.094 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:31 smithi006 bash[56298]: level=warn ts=2023-08-28T00:55:30.872Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:31.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:31 smithi130 bash[51302]: debug 2023-08-28T00:55:31.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:31.013450+0000) 2023-08-28T00:55:32.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:32 smithi130 bash[51302]: debug 2023-08-28T00:55:32.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:32.013713+0000) 2023-08-28T00:55:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:32 smithi006 bash[27292]: cluster 2023-08-28T00:55:31.307142+0000 mgr.y (mgr.14901) 5308 : cluster [DBG] pgmap v5315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:32 smithi006 bash[27292]: audit 2023-08-28T00:55:31.870482+0000 mon.a (mon.0) 3698 : audit [DBG] from='client.? 172.21.15.6:0/1492463705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:32 smithi006 bash[33918]: cluster 2023-08-28T00:55:31.307142+0000 mgr.y (mgr.14901) 5308 : cluster [DBG] pgmap v5315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:32 smithi006 bash[33918]: audit 2023-08-28T00:55:31.870482+0000 mon.a (mon.0) 3698 : audit [DBG] from='client.? 172.21.15.6:0/1492463705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:55:32.442Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:55:32.443Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:32 smithi130 bash[27579]: cluster 2023-08-28T00:55:31.307142+0000 mgr.y (mgr.14901) 5308 : cluster [DBG] pgmap v5315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:32 smithi130 bash[27579]: audit 2023-08-28T00:55:31.870482+0000 mon.a (mon.0) 3698 : audit [DBG] from='client.? 172.21.15.6:0/1492463705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:33.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:33 smithi130 bash[51302]: debug 2023-08-28T00:55:33.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:33.013995+0000) 2023-08-28T00:55:34.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:34 smithi130 bash[51302]: debug 2023-08-28T00:55:34.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:34.014304+0000) 2023-08-28T00:55:34.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:55:34.324Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:55:34.326Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:34 smithi006 bash[27292]: cluster 2023-08-28T00:55:33.307791+0000 mgr.y (mgr.14901) 5309 : cluster [DBG] pgmap v5316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:34.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:34 smithi006 bash[27292]: audit 2023-08-28T00:55:34.290742+0000 mon.a (mon.0) 3699 : audit [DBG] from='client.? 172.21.15.6:0/342855029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:34 smithi006 bash[33918]: cluster 2023-08-28T00:55:33.307791+0000 mgr.y (mgr.14901) 5309 : cluster [DBG] pgmap v5316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:34 smithi006 bash[33918]: audit 2023-08-28T00:55:34.290742+0000 mon.a (mon.0) 3699 : audit [DBG] from='client.? 172.21.15.6:0/342855029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:34 smithi130 bash[27579]: cluster 2023-08-28T00:55:33.307791+0000 mgr.y (mgr.14901) 5309 : cluster [DBG] pgmap v5316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:34 smithi130 bash[27579]: audit 2023-08-28T00:55:34.290742+0000 mon.a (mon.0) 3699 : audit [DBG] from='client.? 172.21.15.6:0/342855029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:35.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:35 smithi130 bash[51302]: debug 2023-08-28T00:55:35.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:35.014608+0000) 2023-08-28T00:55:36.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:36 smithi130 bash[51302]: debug 2023-08-28T00:55:36.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:36.014913+0000) 2023-08-28T00:55:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:36 smithi130 bash[27579]: cluster 2023-08-28T00:55:35.308945+0000 mgr.y (mgr.14901) 5310 : cluster [DBG] pgmap v5317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:36 smithi006 bash[27292]: cluster 2023-08-28T00:55:35.308945+0000 mgr.y (mgr.14901) 5310 : cluster [DBG] pgmap v5317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:36 smithi006 bash[33918]: cluster 2023-08-28T00:55:35.308945+0000 mgr.y (mgr.14901) 5310 : cluster [DBG] pgmap v5317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:37 smithi130 bash[51302]: debug 2023-08-28T00:55:37.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:37.015222+0000) 2023-08-28T00:55:37.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:37 smithi130 bash[27579]: audit 2023-08-28T00:55:36.706087+0000 mon.a (mon.0) 3700 : audit [DBG] from='client.? 172.21.15.6:0/4238676661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:37.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:37 smithi006 bash[33918]: audit 2023-08-28T00:55:36.706087+0000 mon.a (mon.0) 3700 : audit [DBG] from='client.? 172.21.15.6:0/4238676661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:37.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:37 smithi006 bash[27292]: audit 2023-08-28T00:55:36.706087+0000 mon.a (mon.0) 3700 : audit [DBG] from='client.? 172.21.15.6:0/4238676661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:38.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:38 smithi130 bash[51302]: debug 2023-08-28T00:55:38.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:38.015431+0000) 2023-08-28T00:55:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:38 smithi130 bash[27579]: cluster 2023-08-28T00:55:37.309546+0000 mgr.y (mgr.14901) 5311 : cluster [DBG] pgmap v5318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:38 smithi006 bash[27292]: cluster 2023-08-28T00:55:37.309546+0000 mgr.y (mgr.14901) 5311 : cluster [DBG] pgmap v5318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:38.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:38 smithi006 bash[33918]: cluster 2023-08-28T00:55:37.309546+0000 mgr.y (mgr.14901) 5311 : cluster [DBG] pgmap v5318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:39.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:39 smithi130 bash[51302]: debug 2023-08-28T00:55:39.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:39.015720+0000) 2023-08-28T00:55:39.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:39 smithi130 bash[27579]: audit 2023-08-28T00:55:39.116089+0000 mon.c (mon.2) 2054 : audit [DBG] from='client.? 172.21.15.6:0/766503771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:39.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:39 smithi006 bash[27292]: audit 2023-08-28T00:55:39.116089+0000 mon.c (mon.2) 2054 : audit [DBG] from='client.? 172.21.15.6:0/766503771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:39 smithi006 bash[33918]: audit 2023-08-28T00:55:39.116089+0000 mon.c (mon.2) 2054 : audit [DBG] from='client.? 172.21.15.6:0/766503771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:40.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:40 smithi130 bash[51302]: debug 2023-08-28T00:55:40.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:40.016005+0000) 2023-08-28T00:55:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:40 smithi130 bash[27579]: cluster 2023-08-28T00:55:39.310643+0000 mgr.y (mgr.14901) 5312 : cluster [DBG] pgmap v5319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:40.873 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:55:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:55:40] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:55:40.874 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:40 smithi006 bash[27292]: cluster 2023-08-28T00:55:39.310643+0000 mgr.y (mgr.14901) 5312 : cluster [DBG] pgmap v5319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:40.874 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:40 smithi006 bash[33918]: cluster 2023-08-28T00:55:39.310643+0000 mgr.y (mgr.14901) 5312 : cluster [DBG] pgmap v5319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:55:40.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:55:40.872Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:41.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:41 smithi130 bash[51302]: debug 2023-08-28T00:55:41.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:41.016314+0000) 2023-08-28T00:55:42.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:42 smithi130 bash[51302]: debug 2023-08-28T00:55:42.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:42.016611+0000) 2023-08-28T00:55:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:42 smithi006 bash[27292]: cluster 2023-08-28T00:55:41.311262+0000 mgr.y (mgr.14901) 5313 : cluster [DBG] pgmap v5320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:42.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:42 smithi006 bash[27292]: audit 2023-08-28T00:55:41.535493+0000 mon.a (mon.0) 3701 : audit [DBG] from='client.? 172.21.15.6:0/3435514863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:42.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:42 smithi006 bash[33918]: cluster 2023-08-28T00:55:41.311262+0000 mgr.y (mgr.14901) 5313 : cluster [DBG] pgmap v5320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:42.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:42 smithi006 bash[33918]: audit 2023-08-28T00:55:41.535493+0000 mon.a (mon.0) 3701 : audit [DBG] from='client.? 172.21.15.6:0/3435514863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:55:42.443Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:55:42.444Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:42 smithi130 bash[27579]: cluster 2023-08-28T00:55:41.311262+0000 mgr.y (mgr.14901) 5313 : cluster [DBG] pgmap v5320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:42.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:42 smithi130 bash[27579]: audit 2023-08-28T00:55:41.535493+0000 mon.a (mon.0) 3701 : audit [DBG] from='client.? 172.21.15.6:0/3435514863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:43.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:43 smithi130 bash[51302]: debug 2023-08-28T00:55:43.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:43.016926+0000) 2023-08-28T00:55:44.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:44 smithi130 bash[51302]: debug 2023-08-28T00:55:44.015+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:44.017224+0000) 2023-08-28T00:55:44.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:44 smithi006 bash[33918]: cluster 2023-08-28T00:55:43.311888+0000 mgr.y (mgr.14901) 5314 : cluster [DBG] pgmap v5321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:44.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:44 smithi006 bash[33918]: audit 2023-08-28T00:55:43.944677+0000 mon.a (mon.0) 3702 : audit [DBG] from='client.? 172.21.15.6:0/2342590784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:44.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:44 smithi006 bash[27292]: cluster 2023-08-28T00:55:43.311888+0000 mgr.y (mgr.14901) 5314 : cluster [DBG] pgmap v5321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:44.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:44 smithi006 bash[27292]: audit 2023-08-28T00:55:43.944677+0000 mon.a (mon.0) 3702 : audit [DBG] from='client.? 172.21.15.6:0/2342590784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:55:44.324Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:55:44.327Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:44 smithi130 bash[27579]: cluster 2023-08-28T00:55:43.311888+0000 mgr.y (mgr.14901) 5314 : cluster [DBG] pgmap v5321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:44 smithi130 bash[27579]: audit 2023-08-28T00:55:43.944677+0000 mon.a (mon.0) 3702 : audit [DBG] from='client.? 172.21.15.6:0/2342590784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:45 smithi130 bash[51302]: debug 2023-08-28T00:55:45.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:45.017539+0000) 2023-08-28T00:55:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:46 smithi130 bash[51302]: debug 2023-08-28T00:55:46.015+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:46.017835+0000) 2023-08-28T00:55:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:46 smithi130 bash[27579]: cluster 2023-08-28T00:55:45.313024+0000 mgr.y (mgr.14901) 5315 : cluster [DBG] pgmap v5322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:46.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:46 smithi130 bash[27579]: audit 2023-08-28T00:55:46.367370+0000 mon.a (mon.0) 3703 : audit [DBG] from='client.? 172.21.15.6:0/2287045023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:46 smithi006 bash[27292]: cluster 2023-08-28T00:55:45.313024+0000 mgr.y (mgr.14901) 5315 : cluster [DBG] pgmap v5322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:46 smithi006 bash[27292]: audit 2023-08-28T00:55:46.367370+0000 mon.a (mon.0) 3703 : audit [DBG] from='client.? 172.21.15.6:0/2287045023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:46.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:46 smithi006 bash[33918]: cluster 2023-08-28T00:55:45.313024+0000 mgr.y (mgr.14901) 5315 : cluster [DBG] pgmap v5322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:46.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:46 smithi006 bash[33918]: audit 2023-08-28T00:55:46.367370+0000 mon.a (mon.0) 3703 : audit [DBG] from='client.? 172.21.15.6:0/2287045023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:47 smithi130 bash[51302]: debug 2023-08-28T00:55:47.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:47.018087+0000) 2023-08-28T00:55:48.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:48 smithi130 bash[51302]: debug 2023-08-28T00:55:48.015+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:48.018325+0000) 2023-08-28T00:55:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:48 smithi130 bash[27579]: cluster 2023-08-28T00:55:47.313678+0000 mgr.y (mgr.14901) 5316 : cluster [DBG] pgmap v5323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:48 smithi006 bash[33918]: cluster 2023-08-28T00:55:47.313678+0000 mgr.y (mgr.14901) 5316 : cluster [DBG] pgmap v5323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:48 smithi006 bash[27292]: cluster 2023-08-28T00:55:47.313678+0000 mgr.y (mgr.14901) 5316 : cluster [DBG] pgmap v5323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:49 smithi130 bash[51302]: debug 2023-08-28T00:55:49.015+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:49.018621+0000) 2023-08-28T00:55:49.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:49 smithi130 bash[27579]: audit 2023-08-28T00:55:48.781244+0000 mon.c (mon.2) 2055 : audit [DBG] from='client.? 172.21.15.6:0/535946821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:49.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:49 smithi006 bash[33918]: audit 2023-08-28T00:55:48.781244+0000 mon.c (mon.2) 2055 : audit [DBG] from='client.? 172.21.15.6:0/535946821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:49.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:49 smithi006 bash[27292]: audit 2023-08-28T00:55:48.781244+0000 mon.c (mon.2) 2055 : audit [DBG] from='client.? 172.21.15.6:0/535946821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:50 smithi130 bash[51302]: debug 2023-08-28T00:55:50.015+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:50.018921+0000) 2023-08-28T00:55:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:50 smithi130 bash[27579]: cluster 2023-08-28T00:55:49.314737+0000 mgr.y (mgr.14901) 5317 : cluster [DBG] pgmap v5324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:50.943 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:50 smithi006 bash[27292]: cluster 2023-08-28T00:55:49.314737+0000 mgr.y (mgr.14901) 5317 : cluster [DBG] pgmap v5324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:50.944 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:55:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:55:50] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T00:55:50.944 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:50 smithi006 bash[33918]: cluster 2023-08-28T00:55:49.314737+0000 mgr.y (mgr.14901) 5317 : cluster [DBG] pgmap v5324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:55:50.871Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:51.555 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:55:50.873Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:51.555 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:51 smithi130 bash[51302]: debug 2023-08-28T00:55:51.015+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:51.019198+0000) 2023-08-28T00:55:51.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:51 smithi130 bash[27579]: audit 2023-08-28T00:55:51.195352+0000 mon.c (mon.2) 2056 : audit [DBG] from='client.? 172.21.15.6:0/513835467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:51.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:51 smithi006 bash[27292]: audit 2023-08-28T00:55:51.195352+0000 mon.c (mon.2) 2056 : audit [DBG] from='client.? 172.21.15.6:0/513835467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:52.070 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:51 smithi006 bash[33918]: audit 2023-08-28T00:55:51.195352+0000 mon.c (mon.2) 2056 : audit [DBG] from='client.? 172.21.15.6:0/513835467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:52.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:52 smithi130 bash[51302]: debug 2023-08-28T00:55:52.018+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:52.019462+0000) 2023-08-28T00:55:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:52 smithi006 bash[33918]: cluster 2023-08-28T00:55:51.315387+0000 mgr.y (mgr.14901) 5318 : cluster [DBG] pgmap v5325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:52.753 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:52 smithi006 bash[27292]: cluster 2023-08-28T00:55:51.315387+0000 mgr.y (mgr.14901) 5318 : cluster [DBG] pgmap v5325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:52.753 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:55:52.444Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:52.753 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:55:52.446Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:52 smithi130 bash[27579]: cluster 2023-08-28T00:55:51.315387+0000 mgr.y (mgr.14901) 5318 : cluster [DBG] pgmap v5325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:53.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:53 smithi130 bash[51302]: debug 2023-08-28T00:55:53.014+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:53.019728+0000) 2023-08-28T00:55:53.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:53 smithi130 bash[27579]: audit 2023-08-28T00:55:53.261419+0000 mon.b (mon.1) 630 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:55:53.941 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:53 smithi130 bash[27579]: audit 2023-08-28T00:55:53.528651+0000 mon.a (mon.0) 3704 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:55:53.941 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:53 smithi130 bash[27579]: audit 2023-08-28T00:55:53.541297+0000 mon.a (mon.0) 3705 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:55:53.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:53 smithi006 bash[33918]: audit 2023-08-28T00:55:53.261419+0000 mon.b (mon.1) 630 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:55:54.106 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:53 smithi006 bash[33918]: audit 2023-08-28T00:55:53.528651+0000 mon.a (mon.0) 3704 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:55:54.106 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:53 smithi006 bash[33918]: audit 2023-08-28T00:55:53.541297+0000 mon.a (mon.0) 3705 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:55:54.106 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:53 smithi006 bash[27292]: audit 2023-08-28T00:55:53.261419+0000 mon.b (mon.1) 630 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:55:54.107 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:53 smithi006 bash[27292]: audit 2023-08-28T00:55:53.528651+0000 mon.a (mon.0) 3704 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:55:54.107 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:53 smithi006 bash[27292]: audit 2023-08-28T00:55:53.541297+0000 mon.a (mon.0) 3705 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:55:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:54 smithi130 bash[51302]: debug 2023-08-28T00:55:54.018+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:54.020069+0000) 2023-08-28T00:55:54.742 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:55:54.326Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:54.742 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:55:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:55:54.327Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:55:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:54 smithi130 bash[27579]: cluster 2023-08-28T00:55:53.316178+0000 mgr.y (mgr.14901) 5319 : cluster [DBG] pgmap v5326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:54.922 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:54 smithi130 bash[27579]: audit 2023-08-28T00:55:53.624809+0000 mon.c (mon.2) 2057 : audit [DBG] from='client.? 172.21.15.6:0/2604204420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:54 smithi006 bash[27292]: cluster 2023-08-28T00:55:53.316178+0000 mgr.y (mgr.14901) 5319 : cluster [DBG] pgmap v5326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:54.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:54 smithi006 bash[27292]: audit 2023-08-28T00:55:53.624809+0000 mon.c (mon.2) 2057 : audit [DBG] from='client.? 172.21.15.6:0/2604204420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:54.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:54 smithi006 bash[33918]: cluster 2023-08-28T00:55:53.316178+0000 mgr.y (mgr.14901) 5319 : cluster [DBG] pgmap v5326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:54.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:54 smithi006 bash[33918]: audit 2023-08-28T00:55:53.624809+0000 mon.c (mon.2) 2057 : audit [DBG] from='client.? 172.21.15.6:0/2604204420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:55 smithi130 bash[51302]: debug 2023-08-28T00:55:55.014+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:55.020355+0000) 2023-08-28T00:55:56.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:56 smithi130 bash[51302]: debug 2023-08-28T00:55:56.014+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:56.020641+0000) 2023-08-28T00:55:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:56 smithi130 bash[27579]: cluster 2023-08-28T00:55:55.317390+0000 mgr.y (mgr.14901) 5320 : cluster [DBG] pgmap v5327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:56 smithi130 bash[27579]: audit 2023-08-28T00:55:56.035335+0000 mon.a (mon.0) 3706 : audit [DBG] from='client.? 172.21.15.6:0/1508743016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:56 smithi006 bash[33918]: cluster 2023-08-28T00:55:55.317390+0000 mgr.y (mgr.14901) 5320 : cluster [DBG] pgmap v5327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:56 smithi006 bash[33918]: audit 2023-08-28T00:55:56.035335+0000 mon.a (mon.0) 3706 : audit [DBG] from='client.? 172.21.15.6:0/1508743016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:56.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:56 smithi006 bash[27292]: cluster 2023-08-28T00:55:55.317390+0000 mgr.y (mgr.14901) 5320 : cluster [DBG] pgmap v5327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:56.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:56 smithi006 bash[27292]: audit 2023-08-28T00:55:56.035335+0000 mon.a (mon.0) 3706 : audit [DBG] from='client.? 172.21.15.6:0/1508743016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:57.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:57 smithi130 bash[51302]: debug 2023-08-28T00:55:57.018+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:57.020947+0000) 2023-08-28T00:55:58.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:58 smithi130 bash[51302]: debug 2023-08-28T00:55:58.018+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:58.021220+0000) 2023-08-28T00:55:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:58 smithi130 bash[27579]: cluster 2023-08-28T00:55:57.318055+0000 mgr.y (mgr.14901) 5321 : cluster [DBG] pgmap v5328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:55:58 smithi130 bash[27579]: audit 2023-08-28T00:55:58.461968+0000 mon.c (mon.2) 2058 : audit [DBG] from='client.? 172.21.15.6:0/1944392563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:58 smithi006 bash[27292]: cluster 2023-08-28T00:55:57.318055+0000 mgr.y (mgr.14901) 5321 : cluster [DBG] pgmap v5328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:55:58 smithi006 bash[27292]: audit 2023-08-28T00:55:58.461968+0000 mon.c (mon.2) 2058 : audit [DBG] from='client.? 172.21.15.6:0/1944392563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:58.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:58 smithi006 bash[33918]: cluster 2023-08-28T00:55:57.318055+0000 mgr.y (mgr.14901) 5321 : cluster [DBG] pgmap v5328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:55:58.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:55:58 smithi006 bash[33918]: audit 2023-08-28T00:55:58.461968+0000 mon.c (mon.2) 2058 : audit [DBG] from='client.? 172.21.15.6:0/1944392563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:55:59 smithi130 bash[51302]: debug 2023-08-28T00:55:59.018+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:59.021482+0000) 2023-08-28T00:56:00.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:00 smithi130 bash[51302]: debug 2023-08-28T00:56:00.018+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:00.021755+0000) 2023-08-28T00:56:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:00 smithi006 bash[33918]: cluster 2023-08-28T00:55:59.319312+0000 mgr.y (mgr.14901) 5322 : cluster [DBG] pgmap v5329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:00 smithi006 bash[33918]: audit 2023-08-28T00:55:59.394927+0000 mon.a (mon.0) 3707 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:56:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:00 smithi006 bash[33918]: audit 2023-08-28T00:55:59.893828+0000 mon.a (mon.0) 3708 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:56:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:00 smithi006 bash[33918]: audit 2023-08-28T00:56:00.190832+0000 mon.b (mon.1) 631 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:56:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:00 smithi006 bash[33918]: audit 2023-08-28T00:56:00.193077+0000 mon.b (mon.1) 632 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:56:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:00 smithi006 bash[33918]: audit 2023-08-28T00:56:00.200472+0000 mon.a (mon.0) 3709 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:56:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:00 smithi006 bash[27292]: cluster 2023-08-28T00:55:59.319312+0000 mgr.y (mgr.14901) 5322 : cluster [DBG] pgmap v5329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:00 smithi006 bash[27292]: audit 2023-08-28T00:55:59.394927+0000 mon.a (mon.0) 3707 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:56:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:00 smithi006 bash[27292]: audit 2023-08-28T00:55:59.893828+0000 mon.a (mon.0) 3708 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:56:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:00 smithi006 bash[27292]: audit 2023-08-28T00:56:00.190832+0000 mon.b (mon.1) 631 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:56:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:00 smithi006 bash[27292]: audit 2023-08-28T00:56:00.193077+0000 mon.b (mon.1) 632 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:56:00.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:00 smithi006 bash[27292]: audit 2023-08-28T00:56:00.200472+0000 mon.a (mon.0) 3709 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:56:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:00 smithi130 bash[27579]: cluster 2023-08-28T00:55:59.319312+0000 mgr.y (mgr.14901) 5322 : cluster [DBG] pgmap v5329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:00 smithi130 bash[27579]: audit 2023-08-28T00:55:59.394927+0000 mon.a (mon.0) 3707 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:56:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:00 smithi130 bash[27579]: audit 2023-08-28T00:55:59.893828+0000 mon.a (mon.0) 3708 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:56:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:00 smithi130 bash[27579]: audit 2023-08-28T00:56:00.190832+0000 mon.b (mon.1) 631 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:56:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:00 smithi130 bash[27579]: audit 2023-08-28T00:56:00.193077+0000 mon.b (mon.1) 632 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:56:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:00 smithi130 bash[27579]: audit 2023-08-28T00:56:00.200472+0000 mon.a (mon.0) 3709 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:56:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:56:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:56:00] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:56:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:56:00.872Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:01.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:56:00.874Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:01.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:01 smithi130 bash[51302]: debug 2023-08-28T00:56:01.018+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:01.021985+0000) 2023-08-28T00:56:01.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:01 smithi006 bash[33918]: audit 2023-08-28T00:56:00.892142+0000 mon.a (mon.0) 3710 : audit [DBG] from='client.? 172.21.15.6:0/2340258292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:01.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:01 smithi006 bash[27292]: audit 2023-08-28T00:56:00.892142+0000 mon.a (mon.0) 3710 : audit [DBG] from='client.? 172.21.15.6:0/2340258292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:01.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:01 smithi130 bash[27579]: audit 2023-08-28T00:56:00.892142+0000 mon.a (mon.0) 3710 : audit [DBG] from='client.? 172.21.15.6:0/2340258292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:02.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:02 smithi130 bash[51302]: debug 2023-08-28T00:56:02.018+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:02.022271+0000) 2023-08-28T00:56:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:02 smithi006 bash[27292]: cluster 2023-08-28T00:56:01.320018+0000 mgr.y (mgr.14901) 5323 : cluster [DBG] pgmap v5330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:02 smithi006 bash[33918]: cluster 2023-08-28T00:56:01.320018+0000 mgr.y (mgr.14901) 5323 : cluster [DBG] pgmap v5330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:56:02.445Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:56:02.446Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:02 smithi130 bash[27579]: cluster 2023-08-28T00:56:01.320018+0000 mgr.y (mgr.14901) 5323 : cluster [DBG] pgmap v5330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:03 smithi130 bash[51302]: debug 2023-08-28T00:56:03.018+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:03.022575+0000) 2023-08-28T00:56:03.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:03 smithi006 bash[27292]: audit 2023-08-28T00:56:03.298634+0000 mon.a (mon.0) 3711 : audit [DBG] from='client.? 172.21.15.6:0/48178946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:03.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:03 smithi006 bash[33918]: audit 2023-08-28T00:56:03.298634+0000 mon.a (mon.0) 3711 : audit [DBG] from='client.? 172.21.15.6:0/48178946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:03.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:03 smithi130 bash[27579]: audit 2023-08-28T00:56:03.298634+0000 mon.a (mon.0) 3711 : audit [DBG] from='client.? 172.21.15.6:0/48178946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:04 smithi130 bash[51302]: debug 2023-08-28T00:56:04.022+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:04.022909+0000) 2023-08-28T00:56:04.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:04 smithi006 bash[33918]: cluster 2023-08-28T00:56:03.320697+0000 mgr.y (mgr.14901) 5324 : cluster [DBG] pgmap v5331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:04.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:04 smithi006 bash[27292]: cluster 2023-08-28T00:56:03.320697+0000 mgr.y (mgr.14901) 5324 : cluster [DBG] pgmap v5331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:56:04.327Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:56:04.329Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:04 smithi130 bash[27579]: cluster 2023-08-28T00:56:03.320697+0000 mgr.y (mgr.14901) 5324 : cluster [DBG] pgmap v5331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:05 smithi130 bash[51302]: debug 2023-08-28T00:56:05.022+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:05.023180+0000) 2023-08-28T00:56:06.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:06 smithi130 bash[51302]: debug 2023-08-28T00:56:06.022+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:06.023476+0000) 2023-08-28T00:56:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:06 smithi006 bash[33918]: cluster 2023-08-28T00:56:05.321939+0000 mgr.y (mgr.14901) 5325 : cluster [DBG] pgmap v5332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:06 smithi006 bash[33918]: audit 2023-08-28T00:56:05.720250+0000 mon.a (mon.0) 3712 : audit [DBG] from='client.? 172.21.15.6:0/2405968634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:06 smithi006 bash[27292]: cluster 2023-08-28T00:56:05.321939+0000 mgr.y (mgr.14901) 5325 : cluster [DBG] pgmap v5332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:06.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:06 smithi006 bash[27292]: audit 2023-08-28T00:56:05.720250+0000 mon.a (mon.0) 3712 : audit [DBG] from='client.? 172.21.15.6:0/2405968634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:06 smithi130 bash[27579]: cluster 2023-08-28T00:56:05.321939+0000 mgr.y (mgr.14901) 5325 : cluster [DBG] pgmap v5332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:06 smithi130 bash[27579]: audit 2023-08-28T00:56:05.720250+0000 mon.a (mon.0) 3712 : audit [DBG] from='client.? 172.21.15.6:0/2405968634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:07.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:07 smithi130 bash[51302]: debug 2023-08-28T00:56:07.022+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:07.023767+0000) 2023-08-28T00:56:08.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:08 smithi130 bash[51302]: debug 2023-08-28T00:56:08.022+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:08.024056+0000) 2023-08-28T00:56:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:08 smithi006 bash[33918]: cluster 2023-08-28T00:56:07.322772+0000 mgr.y (mgr.14901) 5326 : cluster [DBG] pgmap v5333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:08 smithi006 bash[33918]: audit 2023-08-28T00:56:08.134209+0000 mon.a (mon.0) 3713 : audit [DBG] from='client.? 172.21.15.6:0/416762506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:08 smithi006 bash[27292]: cluster 2023-08-28T00:56:07.322772+0000 mgr.y (mgr.14901) 5326 : cluster [DBG] pgmap v5333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:08.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:08 smithi006 bash[27292]: audit 2023-08-28T00:56:08.134209+0000 mon.a (mon.0) 3713 : audit [DBG] from='client.? 172.21.15.6:0/416762506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:08 smithi130 bash[27579]: cluster 2023-08-28T00:56:07.322772+0000 mgr.y (mgr.14901) 5326 : cluster [DBG] pgmap v5333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:08 smithi130 bash[27579]: audit 2023-08-28T00:56:08.134209+0000 mon.a (mon.0) 3713 : audit [DBG] from='client.? 172.21.15.6:0/416762506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:09.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:09 smithi130 bash[51302]: debug 2023-08-28T00:56:09.022+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:09.024350+0000) 2023-08-28T00:56:10.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:10 smithi130 bash[51302]: debug 2023-08-28T00:56:10.022+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:10.024638+0000) 2023-08-28T00:56:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:10 smithi006 bash[33918]: cluster 2023-08-28T00:56:09.323892+0000 mgr.y (mgr.14901) 5327 : cluster [DBG] pgmap v5334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:10.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:10 smithi006 bash[27292]: cluster 2023-08-28T00:56:09.323892+0000 mgr.y (mgr.14901) 5327 : cluster [DBG] pgmap v5334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:10 smithi130 bash[27579]: cluster 2023-08-28T00:56:09.323892+0000 mgr.y (mgr.14901) 5327 : cluster [DBG] pgmap v5334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:56:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:56:10] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:56:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:56:10.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:56:10.875Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:11.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:11 smithi130 bash[51302]: debug 2023-08-28T00:56:11.022+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:11.024913+0000) 2023-08-28T00:56:11.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:11 smithi006 bash[27292]: audit 2023-08-28T00:56:10.556466+0000 mon.c (mon.2) 2059 : audit [DBG] from='client.? 172.21.15.6:0/2118642316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:11.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:11 smithi006 bash[33918]: audit 2023-08-28T00:56:10.556466+0000 mon.c (mon.2) 2059 : audit [DBG] from='client.? 172.21.15.6:0/2118642316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:11.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:11 smithi130 bash[27579]: audit 2023-08-28T00:56:10.556466+0000 mon.c (mon.2) 2059 : audit [DBG] from='client.? 172.21.15.6:0/2118642316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:12 smithi130 bash[51302]: debug 2023-08-28T00:56:12.022+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:12.025183+0000) 2023-08-28T00:56:12.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:12 smithi006 bash[33918]: cluster 2023-08-28T00:56:11.324549+0000 mgr.y (mgr.14901) 5328 : cluster [DBG] pgmap v5335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:56:12.445Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:56:12.447Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:12 smithi006 bash[27292]: cluster 2023-08-28T00:56:11.324549+0000 mgr.y (mgr.14901) 5328 : cluster [DBG] pgmap v5335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:12 smithi130 bash[27579]: cluster 2023-08-28T00:56:11.324549+0000 mgr.y (mgr.14901) 5328 : cluster [DBG] pgmap v5335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:13 smithi130 bash[51302]: debug 2023-08-28T00:56:13.022+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:13.025457+0000) 2023-08-28T00:56:13.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:13 smithi006 bash[33918]: audit 2023-08-28T00:56:12.976989+0000 mon.c (mon.2) 2060 : audit [DBG] from='client.? 172.21.15.6:0/4180577966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:13.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:13 smithi006 bash[27292]: audit 2023-08-28T00:56:12.976989+0000 mon.c (mon.2) 2060 : audit [DBG] from='client.? 172.21.15.6:0/4180577966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:13.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:13 smithi130 bash[27579]: audit 2023-08-28T00:56:12.976989+0000 mon.c (mon.2) 2060 : audit [DBG] from='client.? 172.21.15.6:0/4180577966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:14.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:14 smithi130 bash[51302]: debug 2023-08-28T00:56:14.022+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:14.025766+0000) 2023-08-28T00:56:14.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:14 smithi006 bash[27292]: cluster 2023-08-28T00:56:13.325279+0000 mgr.y (mgr.14901) 5329 : cluster [DBG] pgmap v5336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:14.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:14 smithi006 bash[33918]: cluster 2023-08-28T00:56:13.325279+0000 mgr.y (mgr.14901) 5329 : cluster [DBG] pgmap v5336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:56:14.328Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:14.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:56:14.330Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:14 smithi130 bash[27579]: cluster 2023-08-28T00:56:13.325279+0000 mgr.y (mgr.14901) 5329 : cluster [DBG] pgmap v5336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:15 smithi130 bash[51302]: debug 2023-08-28T00:56:15.022+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:15.026104+0000) 2023-08-28T00:56:15.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:15 smithi006 bash[27292]: audit 2023-08-28T00:56:15.387841+0000 mon.a (mon.0) 3714 : audit [DBG] from='client.? 172.21.15.6:0/2061393365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:15.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:15 smithi006 bash[33918]: audit 2023-08-28T00:56:15.387841+0000 mon.a (mon.0) 3714 : audit [DBG] from='client.? 172.21.15.6:0/2061393365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:15.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:15 smithi130 bash[27579]: audit 2023-08-28T00:56:15.387841+0000 mon.a (mon.0) 3714 : audit [DBG] from='client.? 172.21.15.6:0/2061393365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:16.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:16 smithi130 bash[51302]: debug 2023-08-28T00:56:16.022+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:16.026417+0000) 2023-08-28T00:56:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:16 smithi006 bash[33918]: cluster 2023-08-28T00:56:15.326459+0000 mgr.y (mgr.14901) 5330 : cluster [DBG] pgmap v5337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:16.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:16 smithi006 bash[27292]: cluster 2023-08-28T00:56:15.326459+0000 mgr.y (mgr.14901) 5330 : cluster [DBG] pgmap v5337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:16 smithi130 bash[27579]: cluster 2023-08-28T00:56:15.326459+0000 mgr.y (mgr.14901) 5330 : cluster [DBG] pgmap v5337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:17.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:17 smithi130 bash[51302]: debug 2023-08-28T00:56:17.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:17.026690+0000) 2023-08-28T00:56:18.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:18 smithi130 bash[51302]: debug 2023-08-28T00:56:18.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:18.026977+0000) 2023-08-28T00:56:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:18 smithi006 bash[33918]: cluster 2023-08-28T00:56:17.327147+0000 mgr.y (mgr.14901) 5331 : cluster [DBG] pgmap v5338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:18.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:18 smithi006 bash[33918]: audit 2023-08-28T00:56:17.812582+0000 mon.a (mon.0) 3715 : audit [DBG] from='client.? 172.21.15.6:0/3802967605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:18 smithi006 bash[27292]: cluster 2023-08-28T00:56:17.327147+0000 mgr.y (mgr.14901) 5331 : cluster [DBG] pgmap v5338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:18.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:18 smithi006 bash[27292]: audit 2023-08-28T00:56:17.812582+0000 mon.a (mon.0) 3715 : audit [DBG] from='client.? 172.21.15.6:0/3802967605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:18 smithi130 bash[27579]: cluster 2023-08-28T00:56:17.327147+0000 mgr.y (mgr.14901) 5331 : cluster [DBG] pgmap v5338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:18 smithi130 bash[27579]: audit 2023-08-28T00:56:17.812582+0000 mon.a (mon.0) 3715 : audit [DBG] from='client.? 172.21.15.6:0/3802967605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:19.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:19 smithi130 bash[51302]: debug 2023-08-28T00:56:19.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:19.027248+0000) 2023-08-28T00:56:20.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:20 smithi130 bash[51302]: debug 2023-08-28T00:56:20.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:20.027554+0000) 2023-08-28T00:56:20.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:20 smithi006 bash[27292]: cluster 2023-08-28T00:56:19.328225+0000 mgr.y (mgr.14901) 5332 : cluster [DBG] pgmap v5339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:20.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:20 smithi006 bash[27292]: audit 2023-08-28T00:56:20.230452+0000 mon.c (mon.2) 2061 : audit [DBG] from='client.? 172.21.15.6:0/3873909352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:20.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:20 smithi006 bash[33918]: cluster 2023-08-28T00:56:19.328225+0000 mgr.y (mgr.14901) 5332 : cluster [DBG] pgmap v5339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:20.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:20 smithi006 bash[33918]: audit 2023-08-28T00:56:20.230452+0000 mon.c (mon.2) 2061 : audit [DBG] from='client.? 172.21.15.6:0/3873909352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:20 smithi130 bash[27579]: cluster 2023-08-28T00:56:19.328225+0000 mgr.y (mgr.14901) 5332 : cluster [DBG] pgmap v5339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:20 smithi130 bash[27579]: audit 2023-08-28T00:56:20.230452+0000 mon.c (mon.2) 2061 : audit [DBG] from='client.? 172.21.15.6:0/3873909352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:56:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:56:20] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:56:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:56:20.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:56:20.875Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:21.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:21 smithi130 bash[51302]: debug 2023-08-28T00:56:21.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:21.027852+0000) 2023-08-28T00:56:22.339 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:22 smithi130 bash[51302]: debug 2023-08-28T00:56:22.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:22.028118+0000) 2023-08-28T00:56:22.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:22 smithi130 bash[27579]: cluster 2023-08-28T00:56:21.328784+0000 mgr.y (mgr.14901) 5333 : cluster [DBG] pgmap v5340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:22 smithi006 bash[33918]: cluster 2023-08-28T00:56:21.328784+0000 mgr.y (mgr.14901) 5333 : cluster [DBG] pgmap v5340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:56:22.446Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:56:22.447Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:22 smithi006 bash[27292]: cluster 2023-08-28T00:56:21.328784+0000 mgr.y (mgr.14901) 5333 : cluster [DBG] pgmap v5340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:23.350 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:23 smithi130 bash[51302]: debug 2023-08-28T00:56:23.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:23.028442+0000) 2023-08-28T00:56:23.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:23 smithi130 bash[27579]: audit 2023-08-28T00:56:22.647183+0000 mon.c (mon.2) 2062 : audit [DBG] from='client.? 172.21.15.6:0/1012855946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:23.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:23 smithi006 bash[27292]: audit 2023-08-28T00:56:22.647183+0000 mon.c (mon.2) 2062 : audit [DBG] from='client.? 172.21.15.6:0/1012855946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:23.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:23 smithi006 bash[33918]: audit 2023-08-28T00:56:22.647183+0000 mon.c (mon.2) 2062 : audit [DBG] from='client.? 172.21.15.6:0/1012855946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:24.352 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:24 smithi130 bash[51302]: debug 2023-08-28T00:56:24.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:24.028744+0000) 2023-08-28T00:56:24.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:24 smithi130 bash[27579]: cluster 2023-08-28T00:56:23.329558+0000 mgr.y (mgr.14901) 5334 : cluster [DBG] pgmap v5341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:24.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:24 smithi006 bash[27292]: cluster 2023-08-28T00:56:23.329558+0000 mgr.y (mgr.14901) 5334 : cluster [DBG] pgmap v5341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:24.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:24 smithi006 bash[33918]: cluster 2023-08-28T00:56:23.329558+0000 mgr.y (mgr.14901) 5334 : cluster [DBG] pgmap v5341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:56:24.329Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:56:24.331Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:25.355 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:25 smithi130 bash[51302]: debug 2023-08-28T00:56:25.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:25.028973+0000) 2023-08-28T00:56:25.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:25 smithi130 bash[27579]: audit 2023-08-28T00:56:25.070964+0000 mon.c (mon.2) 2063 : audit [DBG] from='client.? 172.21.15.6:0/2945904384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:25.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:25 smithi006 bash[33918]: audit 2023-08-28T00:56:25.070964+0000 mon.c (mon.2) 2063 : audit [DBG] from='client.? 172.21.15.6:0/2945904384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:25.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:25 smithi006 bash[27292]: audit 2023-08-28T00:56:25.070964+0000 mon.c (mon.2) 2063 : audit [DBG] from='client.? 172.21.15.6:0/2945904384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:26.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:26 smithi130 bash[51302]: debug 2023-08-28T00:56:26.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:26.029272+0000) 2023-08-28T00:56:26.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:26 smithi130 bash[27579]: cluster 2023-08-28T00:56:25.330743+0000 mgr.y (mgr.14901) 5335 : cluster [DBG] pgmap v5342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:26 smithi006 bash[33918]: cluster 2023-08-28T00:56:25.330743+0000 mgr.y (mgr.14901) 5335 : cluster [DBG] pgmap v5342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:26 smithi006 bash[27292]: cluster 2023-08-28T00:56:25.330743+0000 mgr.y (mgr.14901) 5335 : cluster [DBG] pgmap v5342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:27.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:27 smithi130 bash[51302]: debug 2023-08-28T00:56:27.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:27.029540+0000) 2023-08-28T00:56:28.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:28 smithi130 bash[51302]: debug 2023-08-28T00:56:28.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:28.029835+0000) 2023-08-28T00:56:28.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:28 smithi006 bash[33918]: cluster 2023-08-28T00:56:27.331384+0000 mgr.y (mgr.14901) 5336 : cluster [DBG] pgmap v5343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:28.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:28 smithi006 bash[33918]: audit 2023-08-28T00:56:27.485973+0000 mon.c (mon.2) 2064 : audit [DBG] from='client.? 172.21.15.6:0/3434881015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:28.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:28 smithi006 bash[27292]: cluster 2023-08-28T00:56:27.331384+0000 mgr.y (mgr.14901) 5336 : cluster [DBG] pgmap v5343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:28.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:28 smithi006 bash[27292]: audit 2023-08-28T00:56:27.485973+0000 mon.c (mon.2) 2064 : audit [DBG] from='client.? 172.21.15.6:0/3434881015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:28 smithi130 bash[27579]: cluster 2023-08-28T00:56:27.331384+0000 mgr.y (mgr.14901) 5336 : cluster [DBG] pgmap v5343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:28 smithi130 bash[27579]: audit 2023-08-28T00:56:27.485973+0000 mon.c (mon.2) 2064 : audit [DBG] from='client.? 172.21.15.6:0/3434881015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:29.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:29 smithi130 bash[51302]: debug 2023-08-28T00:56:29.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:29.030111+0000) 2023-08-28T00:56:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:30 smithi130 bash[51302]: debug 2023-08-28T00:56:30.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:30.030401+0000) 2023-08-28T00:56:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:30 smithi006 bash[27292]: cluster 2023-08-28T00:56:29.332525+0000 mgr.y (mgr.14901) 5337 : cluster [DBG] pgmap v5344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:30 smithi006 bash[27292]: audit 2023-08-28T00:56:29.902826+0000 mon.c (mon.2) 2065 : audit [DBG] from='client.? 172.21.15.6:0/2184616365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:30.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:30 smithi006 bash[33918]: cluster 2023-08-28T00:56:29.332525+0000 mgr.y (mgr.14901) 5337 : cluster [DBG] pgmap v5344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:30.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:30 smithi006 bash[33918]: audit 2023-08-28T00:56:29.902826+0000 mon.c (mon.2) 2065 : audit [DBG] from='client.? 172.21.15.6:0/2184616365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:30 smithi130 bash[27579]: cluster 2023-08-28T00:56:29.332525+0000 mgr.y (mgr.14901) 5337 : cluster [DBG] pgmap v5344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:30 smithi130 bash[27579]: audit 2023-08-28T00:56:29.902826+0000 mon.c (mon.2) 2065 : audit [DBG] from='client.? 172.21.15.6:0/2184616365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:56:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:56:30] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:56:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:56:30.875Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:31.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:56:30.877Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:31.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:31 smithi130 bash[51302]: debug 2023-08-28T00:56:31.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:31.030701+0000) 2023-08-28T00:56:32.339 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:32 smithi130 bash[51302]: debug 2023-08-28T00:56:32.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:32.030990+0000) 2023-08-28T00:56:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:32 smithi130 bash[27579]: cluster 2023-08-28T00:56:31.333225+0000 mgr.y (mgr.14901) 5338 : cluster [DBG] pgmap v5345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:32 smithi130 bash[27579]: audit 2023-08-28T00:56:32.311642+0000 mon.a (mon.0) 3716 : audit [DBG] from='client.? 172.21.15.6:0/478755266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:32 smithi006 bash[27292]: cluster 2023-08-28T00:56:31.333225+0000 mgr.y (mgr.14901) 5338 : cluster [DBG] pgmap v5345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:32 smithi006 bash[27292]: audit 2023-08-28T00:56:32.311642+0000 mon.a (mon.0) 3716 : audit [DBG] from='client.? 172.21.15.6:0/478755266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:32 smithi006 bash[33918]: cluster 2023-08-28T00:56:31.333225+0000 mgr.y (mgr.14901) 5338 : cluster [DBG] pgmap v5345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:32.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:32 smithi006 bash[33918]: audit 2023-08-28T00:56:32.311642+0000 mon.a (mon.0) 3716 : audit [DBG] from='client.? 172.21.15.6:0/478755266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:56:32.447Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:56:32.449Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:33.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:33 smithi130 bash[51302]: debug 2023-08-28T00:56:33.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:33.031290+0000) 2023-08-28T00:56:34.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:34 smithi130 bash[51302]: debug 2023-08-28T00:56:34.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:34.031575+0000) 2023-08-28T00:56:34.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:56:34.330Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:34.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:56:34.332Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:34 smithi006 bash[27292]: cluster 2023-08-28T00:56:33.334018+0000 mgr.y (mgr.14901) 5339 : cluster [DBG] pgmap v5346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:34.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:34 smithi006 bash[33918]: cluster 2023-08-28T00:56:33.334018+0000 mgr.y (mgr.14901) 5339 : cluster [DBG] pgmap v5346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:34 smithi130 bash[27579]: cluster 2023-08-28T00:56:33.334018+0000 mgr.y (mgr.14901) 5339 : cluster [DBG] pgmap v5346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:35.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:35 smithi130 bash[51302]: debug 2023-08-28T00:56:35.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:35.031853+0000) 2023-08-28T00:56:35.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:35 smithi006 bash[27292]: audit 2023-08-28T00:56:34.726663+0000 mon.c (mon.2) 2066 : audit [DBG] from='client.? 172.21.15.6:0/1455052324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:35.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:35 smithi006 bash[33918]: audit 2023-08-28T00:56:34.726663+0000 mon.c (mon.2) 2066 : audit [DBG] from='client.? 172.21.15.6:0/1455052324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:35.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:35 smithi130 bash[27579]: audit 2023-08-28T00:56:34.726663+0000 mon.c (mon.2) 2066 : audit [DBG] from='client.? 172.21.15.6:0/1455052324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:36.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:36 smithi130 bash[51302]: debug 2023-08-28T00:56:36.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:36.032130+0000) 2023-08-28T00:56:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:36 smithi006 bash[27292]: cluster 2023-08-28T00:56:35.335223+0000 mgr.y (mgr.14901) 5340 : cluster [DBG] pgmap v5347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:36 smithi006 bash[33918]: cluster 2023-08-28T00:56:35.335223+0000 mgr.y (mgr.14901) 5340 : cluster [DBG] pgmap v5347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:36 smithi130 bash[27579]: cluster 2023-08-28T00:56:35.335223+0000 mgr.y (mgr.14901) 5340 : cluster [DBG] pgmap v5347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:37 smithi130 bash[51302]: debug 2023-08-28T00:56:37.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:37.032465+0000) 2023-08-28T00:56:37.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:37 smithi006 bash[27292]: audit 2023-08-28T00:56:37.143284+0000 mon.a (mon.0) 3717 : audit [DBG] from='client.? 172.21.15.6:0/2988388864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:37.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:37 smithi006 bash[33918]: audit 2023-08-28T00:56:37.143284+0000 mon.a (mon.0) 3717 : audit [DBG] from='client.? 172.21.15.6:0/2988388864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:37.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:37 smithi130 bash[27579]: audit 2023-08-28T00:56:37.143284+0000 mon.a (mon.0) 3717 : audit [DBG] from='client.? 172.21.15.6:0/2988388864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:38.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:38 smithi130 bash[51302]: debug 2023-08-28T00:56:38.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:38.032761+0000) 2023-08-28T00:56:38.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:38 smithi006 bash[33918]: cluster 2023-08-28T00:56:37.335942+0000 mgr.y (mgr.14901) 5341 : cluster [DBG] pgmap v5348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:38.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:38 smithi006 bash[27292]: cluster 2023-08-28T00:56:37.335942+0000 mgr.y (mgr.14901) 5341 : cluster [DBG] pgmap v5348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:38 smithi130 bash[27579]: cluster 2023-08-28T00:56:37.335942+0000 mgr.y (mgr.14901) 5341 : cluster [DBG] pgmap v5348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:39.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:39 smithi130 bash[51302]: debug 2023-08-28T00:56:39.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:39.033080+0000) 2023-08-28T00:56:40.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:40 smithi130 bash[51302]: debug 2023-08-28T00:56:40.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:40.033402+0000) 2023-08-28T00:56:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:40 smithi006 bash[27292]: cluster 2023-08-28T00:56:39.337080+0000 mgr.y (mgr.14901) 5342 : cluster [DBG] pgmap v5349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:40 smithi006 bash[27292]: audit 2023-08-28T00:56:39.555455+0000 mon.a (mon.0) 3718 : audit [DBG] from='client.? 172.21.15.6:0/1511444064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:40 smithi006 bash[33918]: cluster 2023-08-28T00:56:39.337080+0000 mgr.y (mgr.14901) 5342 : cluster [DBG] pgmap v5349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:40 smithi006 bash[33918]: audit 2023-08-28T00:56:39.555455+0000 mon.a (mon.0) 3718 : audit [DBG] from='client.? 172.21.15.6:0/1511444064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:40 smithi130 bash[27579]: cluster 2023-08-28T00:56:39.337080+0000 mgr.y (mgr.14901) 5342 : cluster [DBG] pgmap v5349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:40 smithi130 bash[27579]: audit 2023-08-28T00:56:39.555455+0000 mon.a (mon.0) 3718 : audit [DBG] from='client.? 172.21.15.6:0/1511444064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:56:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:56:40] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:56:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:56:40.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:56:40.877Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:41.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:41 smithi130 bash[51302]: debug 2023-08-28T00:56:41.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:41.033731+0000) 2023-08-28T00:56:42.341 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:42 smithi130 bash[51302]: debug 2023-08-28T00:56:42.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:42.034016+0000) 2023-08-28T00:56:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:42 smithi130 bash[27579]: cluster 2023-08-28T00:56:41.337607+0000 mgr.y (mgr.14901) 5343 : cluster [DBG] pgmap v5350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:42 smithi130 bash[27579]: audit 2023-08-28T00:56:41.965563+0000 mon.a (mon.0) 3719 : audit [DBG] from='client.? 172.21.15.6:0/482928885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:42 smithi006 bash[27292]: cluster 2023-08-28T00:56:41.337607+0000 mgr.y (mgr.14901) 5343 : cluster [DBG] pgmap v5350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:42 smithi006 bash[27292]: audit 2023-08-28T00:56:41.965563+0000 mon.a (mon.0) 3719 : audit [DBG] from='client.? 172.21.15.6:0/482928885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:42.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:42 smithi006 bash[33918]: cluster 2023-08-28T00:56:41.337607+0000 mgr.y (mgr.14901) 5343 : cluster [DBG] pgmap v5350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:42.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:42 smithi006 bash[33918]: audit 2023-08-28T00:56:41.965563+0000 mon.a (mon.0) 3719 : audit [DBG] from='client.? 172.21.15.6:0/482928885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:56:42.448Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:56:42.450Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:43.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:43 smithi130 bash[51302]: debug 2023-08-28T00:56:43.034+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:43.034298+0000) 2023-08-28T00:56:44.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:44 smithi130 bash[51302]: debug 2023-08-28T00:56:44.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:44.034561+0000) 2023-08-28T00:56:44.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:44 smithi006 bash[33918]: cluster 2023-08-28T00:56:43.338234+0000 mgr.y (mgr.14901) 5344 : cluster [DBG] pgmap v5351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:44.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:44 smithi006 bash[33918]: audit 2023-08-28T00:56:44.380803+0000 mon.c (mon.2) 2067 : audit [DBG] from='client.? 172.21.15.6:0/386510440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:56:44.330Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:56:44.332Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:44.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:44 smithi006 bash[27292]: cluster 2023-08-28T00:56:43.338234+0000 mgr.y (mgr.14901) 5344 : cluster [DBG] pgmap v5351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:44.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:44 smithi006 bash[27292]: audit 2023-08-28T00:56:44.380803+0000 mon.c (mon.2) 2067 : audit [DBG] from='client.? 172.21.15.6:0/386510440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:44 smithi130 bash[27579]: cluster 2023-08-28T00:56:43.338234+0000 mgr.y (mgr.14901) 5344 : cluster [DBG] pgmap v5351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:44 smithi130 bash[27579]: audit 2023-08-28T00:56:44.380803+0000 mon.c (mon.2) 2067 : audit [DBG] from='client.? 172.21.15.6:0/386510440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:45 smithi130 bash[51302]: debug 2023-08-28T00:56:45.034+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:45.034889+0000) 2023-08-28T00:56:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:46 smithi130 bash[51302]: debug 2023-08-28T00:56:46.034+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:46.035178+0000) 2023-08-28T00:56:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:46 smithi006 bash[27292]: cluster 2023-08-28T00:56:45.339381+0000 mgr.y (mgr.14901) 5345 : cluster [DBG] pgmap v5352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:46 smithi006 bash[33918]: cluster 2023-08-28T00:56:45.339381+0000 mgr.y (mgr.14901) 5345 : cluster [DBG] pgmap v5352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:46 smithi130 bash[27579]: cluster 2023-08-28T00:56:45.339381+0000 mgr.y (mgr.14901) 5345 : cluster [DBG] pgmap v5352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:47 smithi130 bash[51302]: debug 2023-08-28T00:56:47.034+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:47.035483+0000) 2023-08-28T00:56:47.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:47 smithi006 bash[27292]: audit 2023-08-28T00:56:46.802284+0000 mon.a (mon.0) 3720 : audit [DBG] from='client.? 172.21.15.6:0/979947008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:47.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:47 smithi006 bash[33918]: audit 2023-08-28T00:56:46.802284+0000 mon.a (mon.0) 3720 : audit [DBG] from='client.? 172.21.15.6:0/979947008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:47.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:47 smithi130 bash[27579]: audit 2023-08-28T00:56:46.802284+0000 mon.a (mon.0) 3720 : audit [DBG] from='client.? 172.21.15.6:0/979947008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:48.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:48 smithi130 bash[51302]: debug 2023-08-28T00:56:48.033+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:48.035763+0000) 2023-08-28T00:56:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:48 smithi006 bash[27292]: cluster 2023-08-28T00:56:47.340096+0000 mgr.y (mgr.14901) 5346 : cluster [DBG] pgmap v5353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:48 smithi006 bash[33918]: cluster 2023-08-28T00:56:47.340096+0000 mgr.y (mgr.14901) 5346 : cluster [DBG] pgmap v5353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:48 smithi130 bash[27579]: cluster 2023-08-28T00:56:47.340096+0000 mgr.y (mgr.14901) 5346 : cluster [DBG] pgmap v5353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:49 smithi130 bash[51302]: debug 2023-08-28T00:56:49.033+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:49.036030+0000) 2023-08-28T00:56:49.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:49 smithi006 bash[33918]: audit 2023-08-28T00:56:49.223217+0000 mon.c (mon.2) 2068 : audit [DBG] from='client.? 172.21.15.6:0/916366738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:49.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:49 smithi006 bash[27292]: audit 2023-08-28T00:56:49.223217+0000 mon.c (mon.2) 2068 : audit [DBG] from='client.? 172.21.15.6:0/916366738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:49.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:49 smithi130 bash[27579]: audit 2023-08-28T00:56:49.223217+0000 mon.c (mon.2) 2068 : audit [DBG] from='client.? 172.21.15.6:0/916366738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:50 smithi130 bash[51302]: debug 2023-08-28T00:56:50.033+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:50.036319+0000) 2023-08-28T00:56:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:50 smithi006 bash[33918]: cluster 2023-08-28T00:56:49.341255+0000 mgr.y (mgr.14901) 5347 : cluster [DBG] pgmap v5354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:50 smithi006 bash[27292]: cluster 2023-08-28T00:56:49.341255+0000 mgr.y (mgr.14901) 5347 : cluster [DBG] pgmap v5354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:50 smithi130 bash[27579]: cluster 2023-08-28T00:56:49.341255+0000 mgr.y (mgr.14901) 5347 : cluster [DBG] pgmap v5354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:56:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:56:50] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T00:56:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:56:50.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:56:50.879Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:51.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:51 smithi130 bash[51302]: debug 2023-08-28T00:56:51.033+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:51.036627+0000) 2023-08-28T00:56:52.346 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:52 smithi130 bash[51302]: debug 2023-08-28T00:56:52.033+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:52.036898+0000) 2023-08-28T00:56:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:52 smithi130 bash[27579]: cluster 2023-08-28T00:56:51.341732+0000 mgr.y (mgr.14901) 5348 : cluster [DBG] pgmap v5355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:52 smithi130 bash[27579]: audit 2023-08-28T00:56:51.638765+0000 mon.c (mon.2) 2069 : audit [DBG] from='client.? 172.21.15.6:0/2604896384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:52.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:52 smithi006 bash[27292]: cluster 2023-08-28T00:56:51.341732+0000 mgr.y (mgr.14901) 5348 : cluster [DBG] pgmap v5355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:52.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:52 smithi006 bash[27292]: audit 2023-08-28T00:56:51.638765+0000 mon.c (mon.2) 2069 : audit [DBG] from='client.? 172.21.15.6:0/2604896384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:52 smithi006 bash[33918]: cluster 2023-08-28T00:56:51.341732+0000 mgr.y (mgr.14901) 5348 : cluster [DBG] pgmap v5355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:52.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:52 smithi006 bash[33918]: audit 2023-08-28T00:56:51.638765+0000 mon.c (mon.2) 2069 : audit [DBG] from='client.? 172.21.15.6:0/2604896384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:56:52.449Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:56:52.451Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:53.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:53 smithi130 bash[51302]: debug 2023-08-28T00:56:53.033+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:53.037184+0000) 2023-08-28T00:56:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:54 smithi130 bash[51302]: debug 2023-08-28T00:56:54.033+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:54.037494+0000) 2023-08-28T00:56:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:54 smithi006 bash[27292]: cluster 2023-08-28T00:56:53.342357+0000 mgr.y (mgr.14901) 5349 : cluster [DBG] pgmap v5356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:54.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:54 smithi006 bash[27292]: audit 2023-08-28T00:56:54.059573+0000 mon.c (mon.2) 2070 : audit [DBG] from='client.? 172.21.15.6:0/1121289240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:54.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:54 smithi006 bash[33918]: cluster 2023-08-28T00:56:53.342357+0000 mgr.y (mgr.14901) 5349 : cluster [DBG] pgmap v5356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:54.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:54 smithi006 bash[33918]: audit 2023-08-28T00:56:54.059573+0000 mon.c (mon.2) 2070 : audit [DBG] from='client.? 172.21.15.6:0/1121289240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:56:54.331Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:54.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:56:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:56:54.334Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:56:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:54 smithi130 bash[27579]: cluster 2023-08-28T00:56:53.342357+0000 mgr.y (mgr.14901) 5349 : cluster [DBG] pgmap v5356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:54 smithi130 bash[27579]: audit 2023-08-28T00:56:54.059573+0000 mon.c (mon.2) 2070 : audit [DBG] from='client.? 172.21.15.6:0/1121289240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:55 smithi130 bash[51302]: debug 2023-08-28T00:56:55.033+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:55.037798+0000) 2023-08-28T00:56:56.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:56 smithi130 bash[51302]: debug 2023-08-28T00:56:56.033+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:56.038104+0000) 2023-08-28T00:56:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:56 smithi006 bash[33918]: cluster 2023-08-28T00:56:55.343490+0000 mgr.y (mgr.14901) 5350 : cluster [DBG] pgmap v5357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:56 smithi006 bash[27292]: cluster 2023-08-28T00:56:55.343490+0000 mgr.y (mgr.14901) 5350 : cluster [DBG] pgmap v5357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:56 smithi130 bash[27579]: cluster 2023-08-28T00:56:55.343490+0000 mgr.y (mgr.14901) 5350 : cluster [DBG] pgmap v5357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:57.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:57 smithi130 bash[51302]: debug 2023-08-28T00:56:57.037+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:57.038448+0000) 2023-08-28T00:56:57.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:57 smithi006 bash[27292]: audit 2023-08-28T00:56:56.478551+0000 mon.c (mon.2) 2071 : audit [DBG] from='client.? 172.21.15.6:0/627802253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:57.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:57 smithi006 bash[33918]: audit 2023-08-28T00:56:56.478551+0000 mon.c (mon.2) 2071 : audit [DBG] from='client.? 172.21.15.6:0/627802253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:57.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:57 smithi130 bash[27579]: audit 2023-08-28T00:56:56.478551+0000 mon.c (mon.2) 2071 : audit [DBG] from='client.? 172.21.15.6:0/627802253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:58.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:58 smithi130 bash[51302]: debug 2023-08-28T00:56:58.033+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:58.038733+0000) 2023-08-28T00:56:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:58 smithi006 bash[27292]: cluster 2023-08-28T00:56:57.344303+0000 mgr.y (mgr.14901) 5351 : cluster [DBG] pgmap v5358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:58 smithi006 bash[33918]: cluster 2023-08-28T00:56:57.344303+0000 mgr.y (mgr.14901) 5351 : cluster [DBG] pgmap v5358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:58 smithi130 bash[27579]: cluster 2023-08-28T00:56:57.344303+0000 mgr.y (mgr.14901) 5351 : cluster [DBG] pgmap v5358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:56:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:56:59 smithi130 bash[51302]: debug 2023-08-28T00:56:59.037+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:59.039036+0000) 2023-08-28T00:56:59.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:56:59 smithi006 bash[27292]: audit 2023-08-28T00:56:58.888637+0000 mon.c (mon.2) 2072 : audit [DBG] from='client.? 172.21.15.6:0/150169024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:59.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:56:59 smithi006 bash[33918]: audit 2023-08-28T00:56:58.888637+0000 mon.c (mon.2) 2072 : audit [DBG] from='client.? 172.21.15.6:0/150169024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:56:59 smithi130 bash[27579]: audit 2023-08-28T00:56:58.888637+0000 mon.c (mon.2) 2072 : audit [DBG] from='client.? 172.21.15.6:0/150169024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:00.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:00 smithi130 bash[51302]: debug 2023-08-28T00:57:00.037+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:00.039324+0000) 2023-08-28T00:57:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:00 smithi006 bash[27292]: cluster 2023-08-28T00:56:59.345492+0000 mgr.y (mgr.14901) 5352 : cluster [DBG] pgmap v5359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:00 smithi006 bash[27292]: audit 2023-08-28T00:57:00.212495+0000 mon.b (mon.1) 633 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:57:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:00 smithi006 bash[33918]: cluster 2023-08-28T00:56:59.345492+0000 mgr.y (mgr.14901) 5352 : cluster [DBG] pgmap v5359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:00 smithi006 bash[33918]: audit 2023-08-28T00:57:00.212495+0000 mon.b (mon.1) 633 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:57:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:00 smithi130 bash[27579]: cluster 2023-08-28T00:56:59.345492+0000 mgr.y (mgr.14901) 5352 : cluster [DBG] pgmap v5359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:00 smithi130 bash[27579]: audit 2023-08-28T00:57:00.212495+0000 mon.b (mon.1) 633 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:57:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:57:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:57:00] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:57:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:57:00.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:57:00.879Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:01.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:01 smithi130 bash[51302]: debug 2023-08-28T00:57:01.037+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:01.039541+0000) 2023-08-28T00:57:01.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:01 smithi006 bash[27292]: audit 2023-08-28T00:57:01.301005+0000 mon.a (mon.0) 3721 : audit [DBG] from='client.? 172.21.15.6:0/3192620396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:01.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:01 smithi006 bash[33918]: audit 2023-08-28T00:57:01.301005+0000 mon.a (mon.0) 3721 : audit [DBG] from='client.? 172.21.15.6:0/3192620396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:01.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:01 smithi130 bash[27579]: audit 2023-08-28T00:57:01.301005+0000 mon.a (mon.0) 3721 : audit [DBG] from='client.? 172.21.15.6:0/3192620396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:02.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:02 smithi130 bash[51302]: debug 2023-08-28T00:57:02.037+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:02.039829+0000) 2023-08-28T00:57:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:02 smithi006 bash[33918]: cluster 2023-08-28T00:57:01.346109+0000 mgr.y (mgr.14901) 5353 : cluster [DBG] pgmap v5360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:02 smithi006 bash[27292]: cluster 2023-08-28T00:57:01.346109+0000 mgr.y (mgr.14901) 5353 : cluster [DBG] pgmap v5360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:57:02.450Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:57:02.452Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:02 smithi130 bash[27579]: cluster 2023-08-28T00:57:01.346109+0000 mgr.y (mgr.14901) 5353 : cluster [DBG] pgmap v5360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:03 smithi130 bash[51302]: debug 2023-08-28T00:57:03.037+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:03.040132+0000) 2023-08-28T00:57:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:04 smithi130 bash[51302]: debug 2023-08-28T00:57:04.037+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:04.040408+0000) 2023-08-28T00:57:04.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:04 smithi006 bash[33918]: cluster 2023-08-28T00:57:03.346726+0000 mgr.y (mgr.14901) 5354 : cluster [DBG] pgmap v5361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:04.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:04 smithi006 bash[33918]: audit 2023-08-28T00:57:03.728699+0000 mon.a (mon.0) 3722 : audit [DBG] from='client.? 172.21.15.6:0/2122722169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:04.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:57:04.332Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:57:04.335Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:04.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:04 smithi006 bash[27292]: cluster 2023-08-28T00:57:03.346726+0000 mgr.y (mgr.14901) 5354 : cluster [DBG] pgmap v5361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:04.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:04 smithi006 bash[27292]: audit 2023-08-28T00:57:03.728699+0000 mon.a (mon.0) 3722 : audit [DBG] from='client.? 172.21.15.6:0/2122722169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:04 smithi130 bash[27579]: cluster 2023-08-28T00:57:03.346726+0000 mgr.y (mgr.14901) 5354 : cluster [DBG] pgmap v5361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:04 smithi130 bash[27579]: audit 2023-08-28T00:57:03.728699+0000 mon.a (mon.0) 3722 : audit [DBG] from='client.? 172.21.15.6:0/2122722169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:05.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:05 smithi130 bash[51302]: debug 2023-08-28T00:57:05.037+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:05.040658+0000) 2023-08-28T00:57:06.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:06 smithi130 bash[51302]: debug 2023-08-28T00:57:06.037+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:06.040846+0000) 2023-08-28T00:57:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:06 smithi006 bash[27292]: cluster 2023-08-28T00:57:05.347600+0000 mgr.y (mgr.14901) 5355 : cluster [DBG] pgmap v5362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:06 smithi006 bash[27292]: audit 2023-08-28T00:57:06.092273+0000 mon.a (mon.0) 3723 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:57:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:06 smithi006 bash[27292]: audit 2023-08-28T00:57:06.161454+0000 mon.c (mon.2) 2073 : audit [DBG] from='client.? 172.21.15.6:0/3402119451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:06 smithi006 bash[33918]: cluster 2023-08-28T00:57:05.347600+0000 mgr.y (mgr.14901) 5355 : cluster [DBG] pgmap v5362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:06 smithi006 bash[33918]: audit 2023-08-28T00:57:06.092273+0000 mon.a (mon.0) 3723 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:57:06.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:06 smithi006 bash[33918]: audit 2023-08-28T00:57:06.161454+0000 mon.c (mon.2) 2073 : audit [DBG] from='client.? 172.21.15.6:0/3402119451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:06 smithi130 bash[27579]: cluster 2023-08-28T00:57:05.347600+0000 mgr.y (mgr.14901) 5355 : cluster [DBG] pgmap v5362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:06 smithi130 bash[27579]: audit 2023-08-28T00:57:06.092273+0000 mon.a (mon.0) 3723 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:57:06.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:06 smithi130 bash[27579]: audit 2023-08-28T00:57:06.161454+0000 mon.c (mon.2) 2073 : audit [DBG] from='client.? 172.21.15.6:0/3402119451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:07.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:07 smithi130 bash[51302]: debug 2023-08-28T00:57:07.037+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:07.041094+0000) 2023-08-28T00:57:07.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:07 smithi130 bash[27579]: audit 2023-08-28T00:57:06.566727+0000 mon.a (mon.0) 3724 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:57:07.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:07 smithi130 bash[27579]: audit 2023-08-28T00:57:06.860839+0000 mon.b (mon.1) 634 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:57:07.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:07 smithi130 bash[27579]: audit 2023-08-28T00:57:06.863103+0000 mon.b (mon.1) 635 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:57:07.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:07 smithi130 bash[27579]: audit 2023-08-28T00:57:06.872100+0000 mon.a (mon.0) 3725 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:57:07.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:07 smithi006 bash[33918]: audit 2023-08-28T00:57:06.566727+0000 mon.a (mon.0) 3724 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:57:07.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:07 smithi006 bash[33918]: audit 2023-08-28T00:57:06.860839+0000 mon.b (mon.1) 634 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:57:07.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:07 smithi006 bash[33918]: audit 2023-08-28T00:57:06.863103+0000 mon.b (mon.1) 635 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:57:07.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:07 smithi006 bash[33918]: audit 2023-08-28T00:57:06.872100+0000 mon.a (mon.0) 3725 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:57:07.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:07 smithi006 bash[27292]: audit 2023-08-28T00:57:06.566727+0000 mon.a (mon.0) 3724 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:57:07.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:07 smithi006 bash[27292]: audit 2023-08-28T00:57:06.860839+0000 mon.b (mon.1) 634 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:57:07.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:07 smithi006 bash[27292]: audit 2023-08-28T00:57:06.863103+0000 mon.b (mon.1) 635 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:57:07.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:07 smithi006 bash[27292]: audit 2023-08-28T00:57:06.872100+0000 mon.a (mon.0) 3725 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:57:08.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:08 smithi130 bash[51302]: debug 2023-08-28T00:57:08.037+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:08.041376+0000) 2023-08-28T00:57:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:08 smithi130 bash[27579]: cluster 2023-08-28T00:57:07.348182+0000 mgr.y (mgr.14901) 5356 : cluster [DBG] pgmap v5363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:08 smithi006 bash[27292]: cluster 2023-08-28T00:57:07.348182+0000 mgr.y (mgr.14901) 5356 : cluster [DBG] pgmap v5363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:08 smithi006 bash[33918]: cluster 2023-08-28T00:57:07.348182+0000 mgr.y (mgr.14901) 5356 : cluster [DBG] pgmap v5363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:09.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:09 smithi130 bash[51302]: debug 2023-08-28T00:57:09.037+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:09.041648+0000) 2023-08-28T00:57:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:09 smithi130 bash[27579]: audit 2023-08-28T00:57:08.578538+0000 mon.a (mon.0) 3726 : audit [DBG] from='client.? 172.21.15.6:0/258352847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:09.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:09 smithi006 bash[33918]: audit 2023-08-28T00:57:08.578538+0000 mon.a (mon.0) 3726 : audit [DBG] from='client.? 172.21.15.6:0/258352847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:10.051 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:09 smithi006 bash[27292]: audit 2023-08-28T00:57:08.578538+0000 mon.a (mon.0) 3726 : audit [DBG] from='client.? 172.21.15.6:0/258352847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:10.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:10 smithi130 bash[51302]: debug 2023-08-28T00:57:10.037+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:10.041942+0000) 2023-08-28T00:57:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:10 smithi130 bash[27579]: cluster 2023-08-28T00:57:09.349295+0000 mgr.y (mgr.14901) 5357 : cluster [DBG] pgmap v5364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:10.881 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:57:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:57:10] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:57:10.881 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:10 smithi006 bash[33918]: cluster 2023-08-28T00:57:09.349295+0000 mgr.y (mgr.14901) 5357 : cluster [DBG] pgmap v5364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:10.881 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:10 smithi006 bash[27292]: cluster 2023-08-28T00:57:09.349295+0000 mgr.y (mgr.14901) 5357 : cluster [DBG] pgmap v5364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:57:10.878Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:57:10.880Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:11.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:11 smithi130 bash[51302]: debug 2023-08-28T00:57:11.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:11.042221+0000) 2023-08-28T00:57:11.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:11 smithi130 bash[27579]: audit 2023-08-28T00:57:10.991113+0000 mon.a (mon.0) 3727 : audit [DBG] from='client.? 172.21.15.6:0/873380333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:11.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:11 smithi006 bash[27292]: audit 2023-08-28T00:57:10.991113+0000 mon.a (mon.0) 3727 : audit [DBG] from='client.? 172.21.15.6:0/873380333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:11.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:11 smithi006 bash[33918]: audit 2023-08-28T00:57:10.991113+0000 mon.a (mon.0) 3727 : audit [DBG] from='client.? 172.21.15.6:0/873380333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:12 smithi130 bash[51302]: debug 2023-08-28T00:57:12.037+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:12.042560+0000) 2023-08-28T00:57:12.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:12 smithi006 bash[27292]: cluster 2023-08-28T00:57:11.349812+0000 mgr.y (mgr.14901) 5358 : cluster [DBG] pgmap v5365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:12.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:12 smithi006 bash[33918]: cluster 2023-08-28T00:57:11.349812+0000 mgr.y (mgr.14901) 5358 : cluster [DBG] pgmap v5365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:57:12.451Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:12.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:57:12.454Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:12 smithi130 bash[27579]: cluster 2023-08-28T00:57:11.349812+0000 mgr.y (mgr.14901) 5358 : cluster [DBG] pgmap v5365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:13.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:13 smithi130 bash[51302]: debug 2023-08-28T00:57:13.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:13.042877+0000) 2023-08-28T00:57:13.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:13 smithi130 bash[27579]: audit 2023-08-28T00:57:13.406965+0000 mon.a (mon.0) 3728 : audit [DBG] from='client.? 172.21.15.6:0/412032702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:13.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:13 smithi006 bash[27292]: audit 2023-08-28T00:57:13.406965+0000 mon.a (mon.0) 3728 : audit [DBG] from='client.? 172.21.15.6:0/412032702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:13.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:13 smithi006 bash[33918]: audit 2023-08-28T00:57:13.406965+0000 mon.a (mon.0) 3728 : audit [DBG] from='client.? 172.21.15.6:0/412032702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:14.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:14 smithi130 bash[51302]: debug 2023-08-28T00:57:14.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:14.043129+0000) 2023-08-28T00:57:14.605 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:57:14.333Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:14.605 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:57:14.336Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:14 smithi130 bash[27579]: cluster 2023-08-28T00:57:13.350417+0000 mgr.y (mgr.14901) 5359 : cluster [DBG] pgmap v5366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:14.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:14 smithi006 bash[33918]: cluster 2023-08-28T00:57:13.350417+0000 mgr.y (mgr.14901) 5359 : cluster [DBG] pgmap v5366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:14.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:14 smithi006 bash[27292]: cluster 2023-08-28T00:57:13.350417+0000 mgr.y (mgr.14901) 5359 : cluster [DBG] pgmap v5366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:15.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:15 smithi130 bash[51302]: debug 2023-08-28T00:57:15.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:15.043425+0000) 2023-08-28T00:57:16.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:16 smithi130 bash[51302]: debug 2023-08-28T00:57:16.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:16.043730+0000) 2023-08-28T00:57:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:16 smithi130 bash[27579]: cluster 2023-08-28T00:57:15.351497+0000 mgr.y (mgr.14901) 5360 : cluster [DBG] pgmap v5367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:16.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:16 smithi130 bash[27579]: audit 2023-08-28T00:57:15.822262+0000 mon.c (mon.2) 2074 : audit [DBG] from='client.? 172.21.15.6:0/2464285445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:16 smithi006 bash[27292]: cluster 2023-08-28T00:57:15.351497+0000 mgr.y (mgr.14901) 5360 : cluster [DBG] pgmap v5367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:16 smithi006 bash[27292]: audit 2023-08-28T00:57:15.822262+0000 mon.c (mon.2) 2074 : audit [DBG] from='client.? 172.21.15.6:0/2464285445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:16 smithi006 bash[33918]: cluster 2023-08-28T00:57:15.351497+0000 mgr.y (mgr.14901) 5360 : cluster [DBG] pgmap v5367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:16 smithi006 bash[33918]: audit 2023-08-28T00:57:15.822262+0000 mon.c (mon.2) 2074 : audit [DBG] from='client.? 172.21.15.6:0/2464285445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:17.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:17 smithi130 bash[51302]: debug 2023-08-28T00:57:17.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:17.044008+0000) 2023-08-28T00:57:18.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:18 smithi130 bash[51302]: debug 2023-08-28T00:57:18.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:18.044312+0000) 2023-08-28T00:57:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:18 smithi130 bash[27579]: cluster 2023-08-28T00:57:17.352136+0000 mgr.y (mgr.14901) 5361 : cluster [DBG] pgmap v5368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:18.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:18 smithi130 bash[27579]: audit 2023-08-28T00:57:18.241213+0000 mon.a (mon.0) 3729 : audit [DBG] from='client.? 172.21.15.6:0/2349094396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:18 smithi006 bash[27292]: cluster 2023-08-28T00:57:17.352136+0000 mgr.y (mgr.14901) 5361 : cluster [DBG] pgmap v5368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:18 smithi006 bash[27292]: audit 2023-08-28T00:57:18.241213+0000 mon.a (mon.0) 3729 : audit [DBG] from='client.? 172.21.15.6:0/2349094396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:18 smithi006 bash[33918]: cluster 2023-08-28T00:57:17.352136+0000 mgr.y (mgr.14901) 5361 : cluster [DBG] pgmap v5368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:18 smithi006 bash[33918]: audit 2023-08-28T00:57:18.241213+0000 mon.a (mon.0) 3729 : audit [DBG] from='client.? 172.21.15.6:0/2349094396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:19.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:19 smithi130 bash[51302]: debug 2023-08-28T00:57:19.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:19.044595+0000) 2023-08-28T00:57:20.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:20 smithi130 bash[51302]: debug 2023-08-28T00:57:20.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:20.044888+0000) 2023-08-28T00:57:20.882 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:57:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:57:20] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T00:57:20.882 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:20 smithi006 bash[33918]: cluster 2023-08-28T00:57:19.353236+0000 mgr.y (mgr.14901) 5362 : cluster [DBG] pgmap v5369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:20.882 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:20 smithi006 bash[27292]: cluster 2023-08-28T00:57:19.353236+0000 mgr.y (mgr.14901) 5362 : cluster [DBG] pgmap v5369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:21.046 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:20 smithi130 bash[27579]: cluster 2023-08-28T00:57:19.353236+0000 mgr.y (mgr.14901) 5362 : cluster [DBG] pgmap v5369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:57:20.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:57:20.881Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:21.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:21 smithi130 bash[51302]: debug 2023-08-28T00:57:21.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:21.045117+0000) 2023-08-28T00:57:21.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:21 smithi006 bash[33918]: audit 2023-08-28T00:57:20.673435+0000 mon.c (mon.2) 2075 : audit [DBG] from='client.? 172.21.15.6:0/1249779040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:21.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:21 smithi006 bash[27292]: audit 2023-08-28T00:57:20.673435+0000 mon.c (mon.2) 2075 : audit [DBG] from='client.? 172.21.15.6:0/1249779040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:22.046 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:21 smithi130 bash[27579]: audit 2023-08-28T00:57:20.673435+0000 mon.c (mon.2) 2075 : audit [DBG] from='client.? 172.21.15.6:0/1249779040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:22.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:22 smithi130 bash[51302]: debug 2023-08-28T00:57:22.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:22.045420+0000) 2023-08-28T00:57:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:22 smithi006 bash[33918]: cluster 2023-08-28T00:57:21.353898+0000 mgr.y (mgr.14901) 5363 : cluster [DBG] pgmap v5370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:57:22.452Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:57:22.454Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:22 smithi006 bash[27292]: cluster 2023-08-28T00:57:21.353898+0000 mgr.y (mgr.14901) 5363 : cluster [DBG] pgmap v5370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:23.047 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:22 smithi130 bash[27579]: cluster 2023-08-28T00:57:21.353898+0000 mgr.y (mgr.14901) 5363 : cluster [DBG] pgmap v5370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:23.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:23 smithi130 bash[51302]: debug 2023-08-28T00:57:23.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:23.045787+0000) 2023-08-28T00:57:23.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:23 smithi006 bash[33918]: audit 2023-08-28T00:57:23.083992+0000 mon.a (mon.0) 3730 : audit [DBG] from='client.? 172.21.15.6:0/3473830085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:23.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:23 smithi006 bash[27292]: audit 2023-08-28T00:57:23.083992+0000 mon.a (mon.0) 3730 : audit [DBG] from='client.? 172.21.15.6:0/3473830085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:24.047 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:23 smithi130 bash[27579]: audit 2023-08-28T00:57:23.083992+0000 mon.a (mon.0) 3730 : audit [DBG] from='client.? 172.21.15.6:0/3473830085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:24 smithi130 bash[51302]: debug 2023-08-28T00:57:24.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:24.046212+0000) 2023-08-28T00:57:24.641 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:57:24.334Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:24.641 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:57:24.336Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:24.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:24 smithi006 bash[33918]: cluster 2023-08-28T00:57:23.354595+0000 mgr.y (mgr.14901) 5364 : cluster [DBG] pgmap v5371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:24 smithi006 bash[27292]: cluster 2023-08-28T00:57:23.354595+0000 mgr.y (mgr.14901) 5364 : cluster [DBG] pgmap v5371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:25.047 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:24 smithi130 bash[27579]: cluster 2023-08-28T00:57:23.354595+0000 mgr.y (mgr.14901) 5364 : cluster [DBG] pgmap v5371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:25.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:25 smithi130 bash[51302]: debug 2023-08-28T00:57:25.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:25.046497+0000) 2023-08-28T00:57:25.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:25 smithi006 bash[33918]: audit 2023-08-28T00:57:25.498555+0000 mon.c (mon.2) 2076 : audit [DBG] from='client.? 172.21.15.6:0/4234584153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:25.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:25 smithi006 bash[27292]: audit 2023-08-28T00:57:25.498555+0000 mon.c (mon.2) 2076 : audit [DBG] from='client.? 172.21.15.6:0/4234584153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:26.048 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:25 smithi130 bash[27579]: audit 2023-08-28T00:57:25.498555+0000 mon.c (mon.2) 2076 : audit [DBG] from='client.? 172.21.15.6:0/4234584153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:26.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:26 smithi130 bash[51302]: debug 2023-08-28T00:57:26.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:26.046801+0000) 2023-08-28T00:57:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:26 smithi006 bash[27292]: cluster 2023-08-28T00:57:25.355673+0000 mgr.y (mgr.14901) 5365 : cluster [DBG] pgmap v5372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:26 smithi006 bash[33918]: cluster 2023-08-28T00:57:25.355673+0000 mgr.y (mgr.14901) 5365 : cluster [DBG] pgmap v5372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:27.048 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:26 smithi130 bash[27579]: cluster 2023-08-28T00:57:25.355673+0000 mgr.y (mgr.14901) 5365 : cluster [DBG] pgmap v5372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:27.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:27 smithi130 bash[51302]: debug 2023-08-28T00:57:27.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:27.047061+0000) 2023-08-28T00:57:28.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:28 smithi130 bash[51302]: debug 2023-08-28T00:57:28.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:28.047302+0000) 2023-08-28T00:57:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:28 smithi006 bash[33918]: cluster 2023-08-28T00:57:27.356312+0000 mgr.y (mgr.14901) 5366 : cluster [DBG] pgmap v5373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:28 smithi006 bash[33918]: audit 2023-08-28T00:57:27.914698+0000 mon.a (mon.0) 3731 : audit [DBG] from='client.? 172.21.15.6:0/2158257601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:28 smithi006 bash[27292]: cluster 2023-08-28T00:57:27.356312+0000 mgr.y (mgr.14901) 5366 : cluster [DBG] pgmap v5373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:28 smithi006 bash[27292]: audit 2023-08-28T00:57:27.914698+0000 mon.a (mon.0) 3731 : audit [DBG] from='client.? 172.21.15.6:0/2158257601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:29.048 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:28 smithi130 bash[27579]: cluster 2023-08-28T00:57:27.356312+0000 mgr.y (mgr.14901) 5366 : cluster [DBG] pgmap v5373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:29.049 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:28 smithi130 bash[27579]: audit 2023-08-28T00:57:27.914698+0000 mon.a (mon.0) 3731 : audit [DBG] from='client.? 172.21.15.6:0/2158257601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:29.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:29 smithi130 bash[51302]: debug 2023-08-28T00:57:29.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:29.047615+0000) 2023-08-28T00:57:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:30 smithi130 bash[51302]: debug 2023-08-28T00:57:30.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:30.047901+0000) 2023-08-28T00:57:30.967 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:57:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:57:30] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:57:30.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:30 smithi006 bash[27292]: cluster 2023-08-28T00:57:29.357404+0000 mgr.y (mgr.14901) 5367 : cluster [DBG] pgmap v5374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:30.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:30 smithi006 bash[27292]: audit 2023-08-28T00:57:30.326936+0000 mon.c (mon.2) 2077 : audit [DBG] from='client.? 172.21.15.6:0/155303033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:30.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:30 smithi006 bash[33918]: cluster 2023-08-28T00:57:29.357404+0000 mgr.y (mgr.14901) 5367 : cluster [DBG] pgmap v5374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:30.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:30 smithi006 bash[33918]: audit 2023-08-28T00:57:30.326936+0000 mon.c (mon.2) 2077 : audit [DBG] from='client.? 172.21.15.6:0/155303033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:57:30.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:30.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:57:30.881Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:31.050 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:30 smithi130 bash[27579]: cluster 2023-08-28T00:57:29.357404+0000 mgr.y (mgr.14901) 5367 : cluster [DBG] pgmap v5374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:31.050 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:30 smithi130 bash[27579]: audit 2023-08-28T00:57:30.326936+0000 mon.c (mon.2) 2077 : audit [DBG] from='client.? 172.21.15.6:0/155303033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:31.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:31 smithi130 bash[51302]: debug 2023-08-28T00:57:31.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:31.048185+0000) 2023-08-28T00:57:32.354 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:32 smithi130 bash[51302]: debug 2023-08-28T00:57:32.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:32.048447+0000) 2023-08-28T00:57:32.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:32 smithi130 bash[27579]: cluster 2023-08-28T00:57:31.358059+0000 mgr.y (mgr.14901) 5368 : cluster [DBG] pgmap v5375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:32.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:32 smithi006 bash[33918]: cluster 2023-08-28T00:57:31.358059+0000 mgr.y (mgr.14901) 5368 : cluster [DBG] pgmap v5375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:32 smithi006 bash[27292]: cluster 2023-08-28T00:57:31.358059+0000 mgr.y (mgr.14901) 5368 : cluster [DBG] pgmap v5375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:57:32.452Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:57:32.454Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:33.358 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:33 smithi130 bash[51302]: debug 2023-08-28T00:57:33.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:33.048757+0000) 2023-08-28T00:57:33.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:33 smithi130 bash[27579]: audit 2023-08-28T00:57:32.736401+0000 mon.a (mon.0) 3732 : audit [DBG] from='client.? 172.21.15.6:0/4084776111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:33.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:33 smithi006 bash[27292]: audit 2023-08-28T00:57:32.736401+0000 mon.a (mon.0) 3732 : audit [DBG] from='client.? 172.21.15.6:0/4084776111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:33.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:33 smithi006 bash[33918]: audit 2023-08-28T00:57:32.736401+0000 mon.a (mon.0) 3732 : audit [DBG] from='client.? 172.21.15.6:0/4084776111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:34.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:34 smithi130 bash[51302]: debug 2023-08-28T00:57:34.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:34.049028+0000) 2023-08-28T00:57:34.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:34 smithi130 bash[27579]: cluster 2023-08-28T00:57:33.358801+0000 mgr.y (mgr.14901) 5369 : cluster [DBG] pgmap v5376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:34 smithi006 bash[33918]: cluster 2023-08-28T00:57:33.358801+0000 mgr.y (mgr.14901) 5369 : cluster [DBG] pgmap v5376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:34 smithi006 bash[27292]: cluster 2023-08-28T00:57:33.358801+0000 mgr.y (mgr.14901) 5369 : cluster [DBG] pgmap v5376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:57:34.335Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:57:34.337Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:35.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:35 smithi130 bash[51302]: debug 2023-08-28T00:57:35.049+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:35.049343+0000) 2023-08-28T00:57:35.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:35 smithi006 bash[33918]: audit 2023-08-28T00:57:35.161127+0000 mon.c (mon.2) 2078 : audit [DBG] from='client.? 172.21.15.6:0/3705325426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:35.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:35 smithi006 bash[27292]: audit 2023-08-28T00:57:35.161127+0000 mon.c (mon.2) 2078 : audit [DBG] from='client.? 172.21.15.6:0/3705325426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:35.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:35 smithi130 bash[27579]: audit 2023-08-28T00:57:35.161127+0000 mon.c (mon.2) 2078 : audit [DBG] from='client.? 172.21.15.6:0/3705325426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:36.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:36 smithi130 bash[51302]: debug 2023-08-28T00:57:36.049+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:36.049668+0000) 2023-08-28T00:57:36.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:36 smithi006 bash[27292]: cluster 2023-08-28T00:57:35.359955+0000 mgr.y (mgr.14901) 5370 : cluster [DBG] pgmap v5377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:36.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:36 smithi006 bash[33918]: cluster 2023-08-28T00:57:35.359955+0000 mgr.y (mgr.14901) 5370 : cluster [DBG] pgmap v5377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:36 smithi130 bash[27579]: cluster 2023-08-28T00:57:35.359955+0000 mgr.y (mgr.14901) 5370 : cluster [DBG] pgmap v5377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:37 smithi130 bash[51302]: debug 2023-08-28T00:57:37.049+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:37.049936+0000) 2023-08-28T00:57:38.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:38 smithi130 bash[51302]: debug 2023-08-28T00:57:38.049+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:38.050242+0000) 2023-08-28T00:57:38.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:38 smithi006 bash[27292]: cluster 2023-08-28T00:57:37.360598+0000 mgr.y (mgr.14901) 5371 : cluster [DBG] pgmap v5378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:38.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:38 smithi006 bash[27292]: audit 2023-08-28T00:57:37.574460+0000 mon.a (mon.0) 3733 : audit [DBG] from='client.? 172.21.15.6:0/2192326770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:38.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:38 smithi006 bash[33918]: cluster 2023-08-28T00:57:37.360598+0000 mgr.y (mgr.14901) 5371 : cluster [DBG] pgmap v5378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:38.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:38 smithi006 bash[33918]: audit 2023-08-28T00:57:37.574460+0000 mon.a (mon.0) 3733 : audit [DBG] from='client.? 172.21.15.6:0/2192326770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:38 smithi130 bash[27579]: cluster 2023-08-28T00:57:37.360598+0000 mgr.y (mgr.14901) 5371 : cluster [DBG] pgmap v5378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:38 smithi130 bash[27579]: audit 2023-08-28T00:57:37.574460+0000 mon.a (mon.0) 3733 : audit [DBG] from='client.? 172.21.15.6:0/2192326770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:39.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:39 smithi130 bash[51302]: debug 2023-08-28T00:57:39.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:39.050533+0000) 2023-08-28T00:57:40.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:40 smithi130 bash[51302]: debug 2023-08-28T00:57:40.049+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:40.050885+0000) 2023-08-28T00:57:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:40 smithi006 bash[33918]: cluster 2023-08-28T00:57:39.361674+0000 mgr.y (mgr.14901) 5372 : cluster [DBG] pgmap v5379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:40 smithi006 bash[33918]: audit 2023-08-28T00:57:39.989459+0000 mon.c (mon.2) 2079 : audit [DBG] from='client.? 172.21.15.6:0/3090544700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:40 smithi006 bash[27292]: cluster 2023-08-28T00:57:39.361674+0000 mgr.y (mgr.14901) 5372 : cluster [DBG] pgmap v5379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:40 smithi006 bash[27292]: audit 2023-08-28T00:57:39.989459+0000 mon.c (mon.2) 2079 : audit [DBG] from='client.? 172.21.15.6:0/3090544700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:40 smithi130 bash[27579]: cluster 2023-08-28T00:57:39.361674+0000 mgr.y (mgr.14901) 5372 : cluster [DBG] pgmap v5379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:40 smithi130 bash[27579]: audit 2023-08-28T00:57:39.989459+0000 mon.c (mon.2) 2079 : audit [DBG] from='client.? 172.21.15.6:0/3090544700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:57:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:57:40] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:57:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:57:40.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:57:40.882Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:41.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:41 smithi130 bash[51302]: debug 2023-08-28T00:57:41.049+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:41.051226+0000) 2023-08-28T00:57:42.356 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:42 smithi130 bash[51302]: debug 2023-08-28T00:57:42.049+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:42.051528+0000) 2023-08-28T00:57:42.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:42 smithi130 bash[27579]: cluster 2023-08-28T00:57:41.362275+0000 mgr.y (mgr.14901) 5373 : cluster [DBG] pgmap v5380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:42 smithi006 bash[27292]: cluster 2023-08-28T00:57:41.362275+0000 mgr.y (mgr.14901) 5373 : cluster [DBG] pgmap v5380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:42.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:42 smithi006 bash[33918]: cluster 2023-08-28T00:57:41.362275+0000 mgr.y (mgr.14901) 5373 : cluster [DBG] pgmap v5380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:57:42.453Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:57:42.454Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:43.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:43 smithi130 bash[51302]: debug 2023-08-28T00:57:43.049+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:43.051810+0000) 2023-08-28T00:57:43.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:43 smithi130 bash[27579]: audit 2023-08-28T00:57:42.434008+0000 mon.c (mon.2) 2080 : audit [DBG] from='client.? 172.21.15.6:0/4082591582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:43.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:43 smithi006 bash[27292]: audit 2023-08-28T00:57:42.434008+0000 mon.c (mon.2) 2080 : audit [DBG] from='client.? 172.21.15.6:0/4082591582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:43.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:43 smithi006 bash[33918]: audit 2023-08-28T00:57:42.434008+0000 mon.c (mon.2) 2080 : audit [DBG] from='client.? 172.21.15.6:0/4082591582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:44.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:44 smithi130 bash[51302]: debug 2023-08-28T00:57:44.048+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:44.052156+0000) 2023-08-28T00:57:44.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:44 smithi006 bash[27292]: cluster 2023-08-28T00:57:43.362878+0000 mgr.y (mgr.14901) 5374 : cluster [DBG] pgmap v5381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:44.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:44 smithi006 bash[33918]: cluster 2023-08-28T00:57:43.362878+0000 mgr.y (mgr.14901) 5374 : cluster [DBG] pgmap v5381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:44.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:57:44.335Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:57:44.337Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:44 smithi130 bash[27579]: cluster 2023-08-28T00:57:43.362878+0000 mgr.y (mgr.14901) 5374 : cluster [DBG] pgmap v5381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:45 smithi130 bash[51302]: debug 2023-08-28T00:57:45.048+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:45.052452+0000) 2023-08-28T00:57:45.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:45 smithi006 bash[33918]: audit 2023-08-28T00:57:44.844378+0000 mon.a (mon.0) 3734 : audit [DBG] from='client.? 172.21.15.6:0/3365302693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:45.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:45 smithi006 bash[27292]: audit 2023-08-28T00:57:44.844378+0000 mon.a (mon.0) 3734 : audit [DBG] from='client.? 172.21.15.6:0/3365302693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:45.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:45 smithi130 bash[27579]: audit 2023-08-28T00:57:44.844378+0000 mon.a (mon.0) 3734 : audit [DBG] from='client.? 172.21.15.6:0/3365302693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:46 smithi130 bash[51302]: debug 2023-08-28T00:57:46.048+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:46.052716+0000) 2023-08-28T00:57:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:46 smithi006 bash[33918]: cluster 2023-08-28T00:57:45.364023+0000 mgr.y (mgr.14901) 5375 : cluster [DBG] pgmap v5382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:46 smithi006 bash[27292]: cluster 2023-08-28T00:57:45.364023+0000 mgr.y (mgr.14901) 5375 : cluster [DBG] pgmap v5382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:46 smithi130 bash[27579]: cluster 2023-08-28T00:57:45.364023+0000 mgr.y (mgr.14901) 5375 : cluster [DBG] pgmap v5382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:47 smithi130 bash[51302]: debug 2023-08-28T00:57:47.048+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:47.052981+0000) 2023-08-28T00:57:47.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:47 smithi006 bash[33918]: audit 2023-08-28T00:57:47.256649+0000 mon.a (mon.0) 3735 : audit [DBG] from='client.? 172.21.15.6:0/2887294585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:47.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:47 smithi006 bash[27292]: audit 2023-08-28T00:57:47.256649+0000 mon.a (mon.0) 3735 : audit [DBG] from='client.? 172.21.15.6:0/2887294585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:47.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:47 smithi130 bash[27579]: audit 2023-08-28T00:57:47.256649+0000 mon.a (mon.0) 3735 : audit [DBG] from='client.? 172.21.15.6:0/2887294585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:48.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:48 smithi130 bash[51302]: debug 2023-08-28T00:57:48.052+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:48.053290+0000) 2023-08-28T00:57:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:48 smithi006 bash[27292]: cluster 2023-08-28T00:57:47.364811+0000 mgr.y (mgr.14901) 5376 : cluster [DBG] pgmap v5383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:48 smithi006 bash[33918]: cluster 2023-08-28T00:57:47.364811+0000 mgr.y (mgr.14901) 5376 : cluster [DBG] pgmap v5383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:48 smithi130 bash[27579]: cluster 2023-08-28T00:57:47.364811+0000 mgr.y (mgr.14901) 5376 : cluster [DBG] pgmap v5383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:49 smithi130 bash[51302]: debug 2023-08-28T00:57:49.048+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:49.053641+0000) 2023-08-28T00:57:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:50 smithi130 bash[51302]: debug 2023-08-28T00:57:50.048+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:50.053978+0000) 2023-08-28T00:57:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:50 smithi006 bash[33918]: cluster 2023-08-28T00:57:49.365673+0000 mgr.y (mgr.14901) 5377 : cluster [DBG] pgmap v5384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:50 smithi006 bash[33918]: audit 2023-08-28T00:57:49.670528+0000 mon.c (mon.2) 2081 : audit [DBG] from='client.? 172.21.15.6:0/1990645610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:50 smithi006 bash[27292]: cluster 2023-08-28T00:57:49.365673+0000 mgr.y (mgr.14901) 5377 : cluster [DBG] pgmap v5384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:50.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:50 smithi006 bash[27292]: audit 2023-08-28T00:57:49.670528+0000 mon.c (mon.2) 2081 : audit [DBG] from='client.? 172.21.15.6:0/1990645610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:50 smithi130 bash[27579]: cluster 2023-08-28T00:57:49.365673+0000 mgr.y (mgr.14901) 5377 : cluster [DBG] pgmap v5384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:50 smithi130 bash[27579]: audit 2023-08-28T00:57:49.670528+0000 mon.c (mon.2) 2081 : audit [DBG] from='client.? 172.21.15.6:0/1990645610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:57:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:57:50] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T00:57:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:57:50.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:57:50.882Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:51.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:51 smithi130 bash[51302]: debug 2023-08-28T00:57:51.048+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:51.054252+0000) 2023-08-28T00:57:52.360 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:52 smithi130 bash[51302]: debug 2023-08-28T00:57:52.052+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:52.054509+0000) 2023-08-28T00:57:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:52 smithi130 bash[27579]: cluster 2023-08-28T00:57:51.366409+0000 mgr.y (mgr.14901) 5378 : cluster [DBG] pgmap v5385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:52.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:52 smithi130 bash[27579]: audit 2023-08-28T00:57:52.087223+0000 mon.a (mon.0) 3736 : audit [DBG] from='client.? 172.21.15.6:0/670486104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:52 smithi006 bash[33918]: cluster 2023-08-28T00:57:51.366409+0000 mgr.y (mgr.14901) 5378 : cluster [DBG] pgmap v5385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:52 smithi006 bash[33918]: audit 2023-08-28T00:57:52.087223+0000 mon.a (mon.0) 3736 : audit [DBG] from='client.? 172.21.15.6:0/670486104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:57:52.454Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:57:52.456Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:52 smithi006 bash[27292]: cluster 2023-08-28T00:57:51.366409+0000 mgr.y (mgr.14901) 5378 : cluster [DBG] pgmap v5385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:52.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:52 smithi006 bash[27292]: audit 2023-08-28T00:57:52.087223+0000 mon.a (mon.0) 3736 : audit [DBG] from='client.? 172.21.15.6:0/670486104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:53.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:53 smithi130 bash[51302]: debug 2023-08-28T00:57:53.052+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:53.054814+0000) 2023-08-28T00:57:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:54 smithi130 bash[51302]: debug 2023-08-28T00:57:54.052+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:54.055112+0000) 2023-08-28T00:57:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:54 smithi006 bash[27292]: cluster 2023-08-28T00:57:53.367110+0000 mgr.y (mgr.14901) 5379 : cluster [DBG] pgmap v5386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:54.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:54 smithi006 bash[33918]: cluster 2023-08-28T00:57:53.367110+0000 mgr.y (mgr.14901) 5379 : cluster [DBG] pgmap v5386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:57:54.336Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:57:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:57:54.338Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:57:54.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:54 smithi130 bash[27579]: cluster 2023-08-28T00:57:53.367110+0000 mgr.y (mgr.14901) 5379 : cluster [DBG] pgmap v5386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:55.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:55 smithi130 bash[51302]: debug 2023-08-28T00:57:55.052+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:55.055411+0000) 2023-08-28T00:57:55.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:55 smithi006 bash[27292]: audit 2023-08-28T00:57:54.498018+0000 mon.a (mon.0) 3737 : audit [DBG] from='client.? 172.21.15.6:0/2311752555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:55.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:55 smithi006 bash[33918]: audit 2023-08-28T00:57:54.498018+0000 mon.a (mon.0) 3737 : audit [DBG] from='client.? 172.21.15.6:0/2311752555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:55.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:55 smithi130 bash[27579]: audit 2023-08-28T00:57:54.498018+0000 mon.a (mon.0) 3737 : audit [DBG] from='client.? 172.21.15.6:0/2311752555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:56.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:56 smithi130 bash[51302]: debug 2023-08-28T00:57:56.052+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:56.055687+0000) 2023-08-28T00:57:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:56 smithi006 bash[33918]: cluster 2023-08-28T00:57:55.368216+0000 mgr.y (mgr.14901) 5380 : cluster [DBG] pgmap v5387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:56 smithi006 bash[27292]: cluster 2023-08-28T00:57:55.368216+0000 mgr.y (mgr.14901) 5380 : cluster [DBG] pgmap v5387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:56 smithi130 bash[27579]: cluster 2023-08-28T00:57:55.368216+0000 mgr.y (mgr.14901) 5380 : cluster [DBG] pgmap v5387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:57.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:57 smithi130 bash[51302]: debug 2023-08-28T00:57:57.052+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:57.055959+0000) 2023-08-28T00:57:57.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:57 smithi130 bash[27579]: audit 2023-08-28T00:57:56.914543+0000 mon.c (mon.2) 2082 : audit [DBG] from='client.? 172.21.15.6:0/3103620809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:57.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:57 smithi006 bash[27292]: audit 2023-08-28T00:57:56.914543+0000 mon.c (mon.2) 2082 : audit [DBG] from='client.? 172.21.15.6:0/3103620809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:57.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:57 smithi006 bash[33918]: audit 2023-08-28T00:57:56.914543+0000 mon.c (mon.2) 2082 : audit [DBG] from='client.? 172.21.15.6:0/3103620809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:58.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:58 smithi130 bash[51302]: debug 2023-08-28T00:57:58.052+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:58.056259+0000) 2023-08-28T00:57:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:58 smithi130 bash[27579]: cluster 2023-08-28T00:57:57.368892+0000 mgr.y (mgr.14901) 5381 : cluster [DBG] pgmap v5388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:58 smithi006 bash[27292]: cluster 2023-08-28T00:57:57.368892+0000 mgr.y (mgr.14901) 5381 : cluster [DBG] pgmap v5388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:58 smithi006 bash[33918]: cluster 2023-08-28T00:57:57.368892+0000 mgr.y (mgr.14901) 5381 : cluster [DBG] pgmap v5388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:57:59.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:57:59 smithi130 bash[51302]: debug 2023-08-28T00:57:59.052+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:59.056580+0000) 2023-08-28T00:57:59.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:57:59 smithi130 bash[27579]: audit 2023-08-28T00:57:59.335408+0000 mon.a (mon.0) 3738 : audit [DBG] from='client.? 172.21.15.6:0/909908694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:59.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:57:59 smithi006 bash[27292]: audit 2023-08-28T00:57:59.335408+0000 mon.a (mon.0) 3738 : audit [DBG] from='client.? 172.21.15.6:0/909908694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:57:59 smithi006 bash[33918]: audit 2023-08-28T00:57:59.335408+0000 mon.a (mon.0) 3738 : audit [DBG] from='client.? 172.21.15.6:0/909908694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:00.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:00 smithi130 bash[51302]: debug 2023-08-28T00:58:00.052+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:00.056865+0000) 2023-08-28T00:58:00.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:00 smithi006 bash[27292]: cluster 2023-08-28T00:57:59.369928+0000 mgr.y (mgr.14901) 5382 : cluster [DBG] pgmap v5389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:00.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:00 smithi006 bash[33918]: cluster 2023-08-28T00:57:59.369928+0000 mgr.y (mgr.14901) 5382 : cluster [DBG] pgmap v5389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:00 smithi130 bash[27579]: cluster 2023-08-28T00:57:59.369928+0000 mgr.y (mgr.14901) 5382 : cluster [DBG] pgmap v5389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:58:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:58:00] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:58:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:58:00.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:58:00.883Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:01.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:01 smithi130 bash[51302]: debug 2023-08-28T00:58:01.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:01.057147+0000) 2023-08-28T00:58:02.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:02 smithi130 bash[51302]: debug 2023-08-28T00:58:02.052+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:02.057403+0000) 2023-08-28T00:58:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:02 smithi130 bash[27579]: cluster 2023-08-28T00:58:01.370557+0000 mgr.y (mgr.14901) 5383 : cluster [DBG] pgmap v5390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:02.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:02 smithi130 bash[27579]: audit 2023-08-28T00:58:01.744226+0000 mon.c (mon.2) 2083 : audit [DBG] from='client.? 172.21.15.6:0/1592931267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:02 smithi006 bash[27292]: cluster 2023-08-28T00:58:01.370557+0000 mgr.y (mgr.14901) 5383 : cluster [DBG] pgmap v5390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:02.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:02 smithi006 bash[27292]: audit 2023-08-28T00:58:01.744226+0000 mon.c (mon.2) 2083 : audit [DBG] from='client.? 172.21.15.6:0/1592931267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:58:02.454Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:58:02.456Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:02 smithi006 bash[33918]: cluster 2023-08-28T00:58:01.370557+0000 mgr.y (mgr.14901) 5383 : cluster [DBG] pgmap v5390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:02.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:02 smithi006 bash[33918]: audit 2023-08-28T00:58:01.744226+0000 mon.c (mon.2) 2083 : audit [DBG] from='client.? 172.21.15.6:0/1592931267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:03.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:03 smithi130 bash[51302]: debug 2023-08-28T00:58:03.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:03.057712+0000) 2023-08-28T00:58:04.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:04 smithi130 bash[51302]: debug 2023-08-28T00:58:04.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:04.057999+0000) 2023-08-28T00:58:04.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:04 smithi006 bash[27292]: cluster 2023-08-28T00:58:03.371201+0000 mgr.y (mgr.14901) 5384 : cluster [DBG] pgmap v5391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:04.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:04 smithi006 bash[27292]: audit 2023-08-28T00:58:04.153096+0000 mon.a (mon.0) 3739 : audit [DBG] from='client.? 172.21.15.6:0/4226352808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:58:04.337Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:58:04.339Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:04.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:04 smithi006 bash[33918]: cluster 2023-08-28T00:58:03.371201+0000 mgr.y (mgr.14901) 5384 : cluster [DBG] pgmap v5391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:04.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:04 smithi006 bash[33918]: audit 2023-08-28T00:58:04.153096+0000 mon.a (mon.0) 3739 : audit [DBG] from='client.? 172.21.15.6:0/4226352808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:04 smithi130 bash[27579]: cluster 2023-08-28T00:58:03.371201+0000 mgr.y (mgr.14901) 5384 : cluster [DBG] pgmap v5391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:04.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:04 smithi130 bash[27579]: audit 2023-08-28T00:58:04.153096+0000 mon.a (mon.0) 3739 : audit [DBG] from='client.? 172.21.15.6:0/4226352808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:05.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:05 smithi130 bash[51302]: debug 2023-08-28T00:58:05.052+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:05.058316+0000) 2023-08-28T00:58:06.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:06 smithi130 bash[51302]: debug 2023-08-28T00:58:06.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:06.058596+0000) 2023-08-28T00:58:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:06 smithi006 bash[33918]: cluster 2023-08-28T00:58:05.372307+0000 mgr.y (mgr.14901) 5385 : cluster [DBG] pgmap v5392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:06 smithi006 bash[27292]: cluster 2023-08-28T00:58:05.372307+0000 mgr.y (mgr.14901) 5385 : cluster [DBG] pgmap v5392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:06 smithi130 bash[27579]: cluster 2023-08-28T00:58:05.372307+0000 mgr.y (mgr.14901) 5385 : cluster [DBG] pgmap v5392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:07.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:07 smithi130 bash[51302]: debug 2023-08-28T00:58:07.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:07.058916+0000) 2023-08-28T00:58:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:07 smithi006 bash[33918]: audit 2023-08-28T00:58:06.576970+0000 mon.c (mon.2) 2084 : audit [DBG] from='client.? 172.21.15.6:0/3427587548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:07.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:07 smithi006 bash[33918]: audit 2023-08-28T00:58:06.883228+0000 mon.b (mon.1) 636 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:58:07.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:07 smithi006 bash[27292]: audit 2023-08-28T00:58:06.576970+0000 mon.c (mon.2) 2084 : audit [DBG] from='client.? 172.21.15.6:0/3427587548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:07.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:07 smithi006 bash[27292]: audit 2023-08-28T00:58:06.883228+0000 mon.b (mon.1) 636 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:58:07.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:07 smithi130 bash[27579]: audit 2023-08-28T00:58:06.576970+0000 mon.c (mon.2) 2084 : audit [DBG] from='client.? 172.21.15.6:0/3427587548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:07.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:07 smithi130 bash[27579]: audit 2023-08-28T00:58:06.883228+0000 mon.b (mon.1) 636 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:58:08.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:08 smithi130 bash[51302]: debug 2023-08-28T00:58:08.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:08.059226+0000) 2023-08-28T00:58:08.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:08 smithi006 bash[27292]: cluster 2023-08-28T00:58:07.372993+0000 mgr.y (mgr.14901) 5386 : cluster [DBG] pgmap v5393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:08.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:08 smithi006 bash[33918]: cluster 2023-08-28T00:58:07.372993+0000 mgr.y (mgr.14901) 5386 : cluster [DBG] pgmap v5393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:08 smithi130 bash[27579]: cluster 2023-08-28T00:58:07.372993+0000 mgr.y (mgr.14901) 5386 : cluster [DBG] pgmap v5393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:09.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:09 smithi130 bash[51302]: debug 2023-08-28T00:58:09.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:09.059540+0000) 2023-08-28T00:58:09.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:09 smithi006 bash[27292]: audit 2023-08-28T00:58:08.990172+0000 mon.c (mon.2) 2085 : audit [DBG] from='client.? 172.21.15.6:0/4293479069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:09.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:09 smithi006 bash[33918]: audit 2023-08-28T00:58:08.990172+0000 mon.c (mon.2) 2085 : audit [DBG] from='client.? 172.21.15.6:0/4293479069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:09.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:09 smithi130 bash[27579]: audit 2023-08-28T00:58:08.990172+0000 mon.c (mon.2) 2085 : audit [DBG] from='client.? 172.21.15.6:0/4293479069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:10.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:10 smithi130 bash[51302]: debug 2023-08-28T00:58:10.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:10.059863+0000) 2023-08-28T00:58:10.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:10 smithi006 bash[33918]: cluster 2023-08-28T00:58:09.374050+0000 mgr.y (mgr.14901) 5387 : cluster [DBG] pgmap v5394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:10.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:10 smithi006 bash[27292]: cluster 2023-08-28T00:58:09.374050+0000 mgr.y (mgr.14901) 5387 : cluster [DBG] pgmap v5394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:10.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:10 smithi130 bash[27579]: cluster 2023-08-28T00:58:09.374050+0000 mgr.y (mgr.14901) 5387 : cluster [DBG] pgmap v5394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:58:10.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:58:10.883Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:58:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:58:10] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:58:11.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:11 smithi130 bash[51302]: debug 2023-08-28T00:58:11.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:11.060090+0000) 2023-08-28T00:58:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:12 smithi130 bash[51302]: debug 2023-08-28T00:58:12.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:12.060350+0000) 2023-08-28T00:58:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:58:12.455Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:58:12.457Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:12 smithi006 bash[27292]: cluster 2023-08-28T00:58:11.374648+0000 mgr.y (mgr.14901) 5388 : cluster [DBG] pgmap v5395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:12 smithi006 bash[27292]: audit 2023-08-28T00:58:11.423837+0000 mon.a (mon.0) 3740 : audit [DBG] from='client.? 172.21.15.6:0/1270393490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:12 smithi006 bash[33918]: cluster 2023-08-28T00:58:11.374648+0000 mgr.y (mgr.14901) 5388 : cluster [DBG] pgmap v5395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:12.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:12 smithi006 bash[33918]: audit 2023-08-28T00:58:11.423837+0000 mon.a (mon.0) 3740 : audit [DBG] from='client.? 172.21.15.6:0/1270393490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:12 smithi130 bash[27579]: cluster 2023-08-28T00:58:11.374648+0000 mgr.y (mgr.14901) 5388 : cluster [DBG] pgmap v5395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:12 smithi130 bash[27579]: audit 2023-08-28T00:58:11.423837+0000 mon.a (mon.0) 3740 : audit [DBG] from='client.? 172.21.15.6:0/1270393490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:13.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:13 smithi130 bash[51302]: debug 2023-08-28T00:58:13.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:13.060572+0000) 2023-08-28T00:58:14.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:14 smithi130 bash[51302]: debug 2023-08-28T00:58:14.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:14.060864+0000) 2023-08-28T00:58:14.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:13 smithi130 bash[27579]: audit 2023-08-28T00:58:12.822059+0000 mon.a (mon.0) 3741 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:58:14.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:13 smithi130 bash[27579]: audit 2023-08-28T00:58:13.154331+0000 mon.a (mon.0) 3742 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:58:14.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:13 smithi130 bash[27579]: audit 2023-08-28T00:58:13.424962+0000 mon.b (mon.1) 637 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:58:14.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:13 smithi130 bash[27579]: audit 2023-08-28T00:58:13.427130+0000 mon.b (mon.1) 638 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:58:14.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:13 smithi130 bash[27579]: audit 2023-08-28T00:58:13.435795+0000 mon.a (mon.0) 3743 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:58:14.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:13 smithi006 bash[33918]: audit 2023-08-28T00:58:12.822059+0000 mon.a (mon.0) 3741 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:58:14.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:13 smithi006 bash[33918]: audit 2023-08-28T00:58:13.154331+0000 mon.a (mon.0) 3742 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:58:14.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:13 smithi006 bash[33918]: audit 2023-08-28T00:58:13.424962+0000 mon.b (mon.1) 637 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:58:14.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:13 smithi006 bash[33918]: audit 2023-08-28T00:58:13.427130+0000 mon.b (mon.1) 638 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:58:14.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:13 smithi006 bash[33918]: audit 2023-08-28T00:58:13.435795+0000 mon.a (mon.0) 3743 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:58:14.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:13 smithi006 bash[27292]: audit 2023-08-28T00:58:12.822059+0000 mon.a (mon.0) 3741 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:58:14.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:13 smithi006 bash[27292]: audit 2023-08-28T00:58:13.154331+0000 mon.a (mon.0) 3742 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:58:14.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:13 smithi006 bash[27292]: audit 2023-08-28T00:58:13.424962+0000 mon.b (mon.1) 637 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:58:14.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:13 smithi006 bash[27292]: audit 2023-08-28T00:58:13.427130+0000 mon.b (mon.1) 638 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:58:14.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:13 smithi006 bash[27292]: audit 2023-08-28T00:58:13.435795+0000 mon.a (mon.0) 3743 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:58:14.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:58:14.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:14.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:58:14.340Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:15.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:15 smithi130 bash[51302]: debug 2023-08-28T00:58:15.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:15.061194+0000) 2023-08-28T00:58:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:14 smithi130 bash[27579]: cluster 2023-08-28T00:58:13.375271+0000 mgr.y (mgr.14901) 5389 : cluster [DBG] pgmap v5396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:14 smithi130 bash[27579]: audit 2023-08-28T00:58:13.835272+0000 mon.c (mon.2) 2086 : audit [DBG] from='client.? 172.21.15.6:0/3023016039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:14 smithi006 bash[33918]: cluster 2023-08-28T00:58:13.375271+0000 mgr.y (mgr.14901) 5389 : cluster [DBG] pgmap v5396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:15.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:14 smithi006 bash[33918]: audit 2023-08-28T00:58:13.835272+0000 mon.c (mon.2) 2086 : audit [DBG] from='client.? 172.21.15.6:0/3023016039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:14 smithi006 bash[27292]: cluster 2023-08-28T00:58:13.375271+0000 mgr.y (mgr.14901) 5389 : cluster [DBG] pgmap v5396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:15.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:14 smithi006 bash[27292]: audit 2023-08-28T00:58:13.835272+0000 mon.c (mon.2) 2086 : audit [DBG] from='client.? 172.21.15.6:0/3023016039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:16.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:16 smithi130 bash[51302]: debug 2023-08-28T00:58:16.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:16.061462+0000) 2023-08-28T00:58:17.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:16 smithi130 bash[27579]: cluster 2023-08-28T00:58:15.376367+0000 mgr.y (mgr.14901) 5390 : cluster [DBG] pgmap v5397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:17.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:16 smithi130 bash[27579]: audit 2023-08-28T00:58:16.254341+0000 mon.c (mon.2) 2087 : audit [DBG] from='client.? 172.21.15.6:0/2021527803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:17.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:17 smithi130 bash[51302]: debug 2023-08-28T00:58:17.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:17.061785+0000) 2023-08-28T00:58:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:16 smithi006 bash[33918]: cluster 2023-08-28T00:58:15.376367+0000 mgr.y (mgr.14901) 5390 : cluster [DBG] pgmap v5397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:16 smithi006 bash[33918]: audit 2023-08-28T00:58:16.254341+0000 mon.c (mon.2) 2087 : audit [DBG] from='client.? 172.21.15.6:0/2021527803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:17.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:16 smithi006 bash[27292]: cluster 2023-08-28T00:58:15.376367+0000 mgr.y (mgr.14901) 5390 : cluster [DBG] pgmap v5397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:17.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:16 smithi006 bash[27292]: audit 2023-08-28T00:58:16.254341+0000 mon.c (mon.2) 2087 : audit [DBG] from='client.? 172.21.15.6:0/2021527803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:18.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:18 smithi130 bash[51302]: debug 2023-08-28T00:58:18.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:18.062122+0000) 2023-08-28T00:58:19.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:19 smithi130 bash[51302]: debug 2023-08-28T00:58:19.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:19.062414+0000) 2023-08-28T00:58:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:18 smithi130 bash[27579]: cluster 2023-08-28T00:58:17.376994+0000 mgr.y (mgr.14901) 5391 : cluster [DBG] pgmap v5398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:19.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:18 smithi130 bash[27579]: audit 2023-08-28T00:58:18.676065+0000 mon.c (mon.2) 2088 : audit [DBG] from='client.? 172.21.15.6:0/2216595822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:18 smithi006 bash[33918]: cluster 2023-08-28T00:58:17.376994+0000 mgr.y (mgr.14901) 5391 : cluster [DBG] pgmap v5398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:19.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:18 smithi006 bash[33918]: audit 2023-08-28T00:58:18.676065+0000 mon.c (mon.2) 2088 : audit [DBG] from='client.? 172.21.15.6:0/2216595822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:18 smithi006 bash[27292]: cluster 2023-08-28T00:58:17.376994+0000 mgr.y (mgr.14901) 5391 : cluster [DBG] pgmap v5398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:19.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:18 smithi006 bash[27292]: audit 2023-08-28T00:58:18.676065+0000 mon.c (mon.2) 2088 : audit [DBG] from='client.? 172.21.15.6:0/2216595822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:20.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:19 smithi130 bash[27579]: cluster 2023-08-28T00:58:19.378034+0000 mgr.y (mgr.14901) 5392 : cluster [DBG] pgmap v5399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:20.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:20 smithi130 bash[51302]: debug 2023-08-28T00:58:20.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:20.062700+0000) 2023-08-28T00:58:20.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:19 smithi006 bash[33918]: cluster 2023-08-28T00:58:19.378034+0000 mgr.y (mgr.14901) 5392 : cluster [DBG] pgmap v5399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:20.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:19 smithi006 bash[27292]: cluster 2023-08-28T00:58:19.378034+0000 mgr.y (mgr.14901) 5392 : cluster [DBG] pgmap v5399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:21.150 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:58:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:58:20] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T00:58:21.150 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:58:20.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:21.150 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:58:20.883Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:21.368 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:21 smithi130 bash[27579]: audit 2023-08-28T00:58:21.094554+0000 mon.a (mon.0) 3744 : audit [DBG] from='client.? 172.21.15.6:0/2761775103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:21.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:21 smithi130 bash[51302]: debug 2023-08-28T00:58:21.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:21.062987+0000) 2023-08-28T00:58:21.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:21 smithi006 bash[33918]: audit 2023-08-28T00:58:21.094554+0000 mon.a (mon.0) 3744 : audit [DBG] from='client.? 172.21.15.6:0/2761775103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:21.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:21 smithi006 bash[27292]: audit 2023-08-28T00:58:21.094554+0000 mon.a (mon.0) 3744 : audit [DBG] from='client.? 172.21.15.6:0/2761775103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:22.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:22 smithi130 bash[51302]: debug 2023-08-28T00:58:22.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:22.063274+0000) 2023-08-28T00:58:22.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:22 smithi006 bash[27292]: cluster 2023-08-28T00:58:21.378649+0000 mgr.y (mgr.14901) 5393 : cluster [DBG] pgmap v5400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:22.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:22 smithi006 bash[33918]: cluster 2023-08-28T00:58:21.378649+0000 mgr.y (mgr.14901) 5393 : cluster [DBG] pgmap v5400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:58:22.456Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:22.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:58:22.458Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:22 smithi130 bash[27579]: cluster 2023-08-28T00:58:21.378649+0000 mgr.y (mgr.14901) 5393 : cluster [DBG] pgmap v5400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:23.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:23 smithi130 bash[51302]: debug 2023-08-28T00:58:23.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:23.063557+0000) 2023-08-28T00:58:24.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:24 smithi130 bash[51302]: debug 2023-08-28T00:58:24.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:24.063829+0000) 2023-08-28T00:58:24.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:58:24.339Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:24.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:58:24.341Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:24.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:24 smithi006 bash[27292]: cluster 2023-08-28T00:58:23.379258+0000 mgr.y (mgr.14901) 5394 : cluster [DBG] pgmap v5401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:24.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:24 smithi006 bash[27292]: audit 2023-08-28T00:58:23.519244+0000 mon.a (mon.0) 3745 : audit [DBG] from='client.? 172.21.15.6:0/1142008306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:24.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:24 smithi006 bash[33918]: cluster 2023-08-28T00:58:23.379258+0000 mgr.y (mgr.14901) 5394 : cluster [DBG] pgmap v5401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:24.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:24 smithi006 bash[33918]: audit 2023-08-28T00:58:23.519244+0000 mon.a (mon.0) 3745 : audit [DBG] from='client.? 172.21.15.6:0/1142008306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:24 smithi130 bash[27579]: cluster 2023-08-28T00:58:23.379258+0000 mgr.y (mgr.14901) 5394 : cluster [DBG] pgmap v5401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:24.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:24 smithi130 bash[27579]: audit 2023-08-28T00:58:23.519244+0000 mon.a (mon.0) 3745 : audit [DBG] from='client.? 172.21.15.6:0/1142008306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:25.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:25 smithi130 bash[51302]: debug 2023-08-28T00:58:25.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:25.064084+0000) 2023-08-28T00:58:26.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:26 smithi130 bash[51302]: debug 2023-08-28T00:58:26.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:26.064348+0000) 2023-08-28T00:58:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:26 smithi006 bash[33918]: cluster 2023-08-28T00:58:25.380301+0000 mgr.y (mgr.14901) 5395 : cluster [DBG] pgmap v5402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:26 smithi006 bash[33918]: audit 2023-08-28T00:58:25.928728+0000 mon.c (mon.2) 2089 : audit [DBG] from='client.? 172.21.15.6:0/3530233522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:26 smithi006 bash[27292]: cluster 2023-08-28T00:58:25.380301+0000 mgr.y (mgr.14901) 5395 : cluster [DBG] pgmap v5402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:26 smithi006 bash[27292]: audit 2023-08-28T00:58:25.928728+0000 mon.c (mon.2) 2089 : audit [DBG] from='client.? 172.21.15.6:0/3530233522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:26 smithi130 bash[27579]: cluster 2023-08-28T00:58:25.380301+0000 mgr.y (mgr.14901) 5395 : cluster [DBG] pgmap v5402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:26 smithi130 bash[27579]: audit 2023-08-28T00:58:25.928728+0000 mon.c (mon.2) 2089 : audit [DBG] from='client.? 172.21.15.6:0/3530233522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:27.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:27 smithi130 bash[51302]: debug 2023-08-28T00:58:27.064+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:27.064611+0000) 2023-08-28T00:58:28.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:28 smithi130 bash[51302]: debug 2023-08-28T00:58:28.064+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:28.064883+0000) 2023-08-28T00:58:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:28 smithi006 bash[33918]: cluster 2023-08-28T00:58:27.380953+0000 mgr.y (mgr.14901) 5396 : cluster [DBG] pgmap v5403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:28.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:28 smithi006 bash[33918]: audit 2023-08-28T00:58:28.350241+0000 mon.c (mon.2) 2090 : audit [DBG] from='client.? 172.21.15.6:0/1069594404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:28.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:28 smithi006 bash[27292]: cluster 2023-08-28T00:58:27.380953+0000 mgr.y (mgr.14901) 5396 : cluster [DBG] pgmap v5403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:28.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:28 smithi006 bash[27292]: audit 2023-08-28T00:58:28.350241+0000 mon.c (mon.2) 2090 : audit [DBG] from='client.? 172.21.15.6:0/1069594404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:28 smithi130 bash[27579]: cluster 2023-08-28T00:58:27.380953+0000 mgr.y (mgr.14901) 5396 : cluster [DBG] pgmap v5403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:28.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:28 smithi130 bash[27579]: audit 2023-08-28T00:58:28.350241+0000 mon.c (mon.2) 2090 : audit [DBG] from='client.? 172.21.15.6:0/1069594404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:29.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:29 smithi130 bash[51302]: debug 2023-08-28T00:58:29.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:29.065207+0000) 2023-08-28T00:58:30.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:30 smithi130 bash[51302]: debug 2023-08-28T00:58:30.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:30.065512+0000) 2023-08-28T00:58:30.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:30 smithi006 bash[33918]: cluster 2023-08-28T00:58:29.381960+0000 mgr.y (mgr.14901) 5397 : cluster [DBG] pgmap v5404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:30.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:30 smithi006 bash[27292]: cluster 2023-08-28T00:58:29.381960+0000 mgr.y (mgr.14901) 5397 : cluster [DBG] pgmap v5404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:30.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:30 smithi130 bash[27579]: cluster 2023-08-28T00:58:29.381960+0000 mgr.y (mgr.14901) 5397 : cluster [DBG] pgmap v5404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:58:30.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:58:30.884Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:31.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:58:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:58:30] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:58:31.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:31 smithi130 bash[51302]: debug 2023-08-28T00:58:31.064+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:31.065823+0000) 2023-08-28T00:58:31.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:31 smithi006 bash[27292]: audit 2023-08-28T00:58:30.768204+0000 mon.a (mon.0) 3746 : audit [DBG] from='client.? 172.21.15.6:0/2768952865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:31.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:31 smithi006 bash[33918]: audit 2023-08-28T00:58:30.768204+0000 mon.a (mon.0) 3746 : audit [DBG] from='client.? 172.21.15.6:0/2768952865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:31.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:31 smithi130 bash[27579]: audit 2023-08-28T00:58:30.768204+0000 mon.a (mon.0) 3746 : audit [DBG] from='client.? 172.21.15.6:0/2768952865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:32.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:32 smithi130 bash[51302]: debug 2023-08-28T00:58:32.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:32.066110+0000) 2023-08-28T00:58:32.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:32 smithi006 bash[33918]: cluster 2023-08-28T00:58:31.382588+0000 mgr.y (mgr.14901) 5398 : cluster [DBG] pgmap v5405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:32.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:32 smithi006 bash[27292]: cluster 2023-08-28T00:58:31.382588+0000 mgr.y (mgr.14901) 5398 : cluster [DBG] pgmap v5405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:58:32.457Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:32.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:58:32.459Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:32 smithi130 bash[27579]: cluster 2023-08-28T00:58:31.382588+0000 mgr.y (mgr.14901) 5398 : cluster [DBG] pgmap v5405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:33.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:33 smithi130 bash[51302]: debug 2023-08-28T00:58:33.064+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:33.066392+0000) 2023-08-28T00:58:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:33 smithi130 bash[27579]: audit 2023-08-28T00:58:33.185845+0000 mon.c (mon.2) 2091 : audit [DBG] from='client.? 172.21.15.6:0/201380022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:33.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:33 smithi006 bash[27292]: audit 2023-08-28T00:58:33.185845+0000 mon.c (mon.2) 2091 : audit [DBG] from='client.? 172.21.15.6:0/201380022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:33.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:33 smithi006 bash[33918]: audit 2023-08-28T00:58:33.185845+0000 mon.c (mon.2) 2091 : audit [DBG] from='client.? 172.21.15.6:0/201380022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:34.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:34 smithi130 bash[51302]: debug 2023-08-28T00:58:34.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:34.066725+0000) 2023-08-28T00:58:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:34 smithi006 bash[27292]: cluster 2023-08-28T00:58:33.383226+0000 mgr.y (mgr.14901) 5399 : cluster [DBG] pgmap v5406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:34 smithi006 bash[33918]: cluster 2023-08-28T00:58:33.383226+0000 mgr.y (mgr.14901) 5399 : cluster [DBG] pgmap v5406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:34.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:58:34.339Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:58:34.341Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:34 smithi130 bash[27579]: cluster 2023-08-28T00:58:33.383226+0000 mgr.y (mgr.14901) 5399 : cluster [DBG] pgmap v5406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:35.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:35 smithi130 bash[51302]: debug 2023-08-28T00:58:35.064+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:35.067016+0000) 2023-08-28T00:58:36.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:36 smithi130 bash[51302]: debug 2023-08-28T00:58:36.064+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:36.067298+0000) 2023-08-28T00:58:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:36 smithi130 bash[27579]: cluster 2023-08-28T00:58:35.383956+0000 mgr.y (mgr.14901) 5400 : cluster [DBG] pgmap v5407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:36 smithi130 bash[27579]: audit 2023-08-28T00:58:35.610028+0000 mon.a (mon.0) 3747 : audit [DBG] from='client.? 172.21.15.6:0/1222808792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:36 smithi006 bash[27292]: cluster 2023-08-28T00:58:35.383956+0000 mgr.y (mgr.14901) 5400 : cluster [DBG] pgmap v5407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:36 smithi006 bash[27292]: audit 2023-08-28T00:58:35.610028+0000 mon.a (mon.0) 3747 : audit [DBG] from='client.? 172.21.15.6:0/1222808792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:36 smithi006 bash[33918]: cluster 2023-08-28T00:58:35.383956+0000 mgr.y (mgr.14901) 5400 : cluster [DBG] pgmap v5407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:36 smithi006 bash[33918]: audit 2023-08-28T00:58:35.610028+0000 mon.a (mon.0) 3747 : audit [DBG] from='client.? 172.21.15.6:0/1222808792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:37.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:37 smithi130 bash[51302]: debug 2023-08-28T00:58:37.064+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:37.067566+0000) 2023-08-28T00:58:38.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:38 smithi130 bash[51302]: debug 2023-08-28T00:58:38.064+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:38.067869+0000) 2023-08-28T00:58:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:38 smithi130 bash[27579]: cluster 2023-08-28T00:58:37.384567+0000 mgr.y (mgr.14901) 5401 : cluster [DBG] pgmap v5408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:38 smithi130 bash[27579]: audit 2023-08-28T00:58:38.027036+0000 mon.c (mon.2) 2092 : audit [DBG] from='client.? 172.21.15.6:0/1598466492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:38 smithi006 bash[27292]: cluster 2023-08-28T00:58:37.384567+0000 mgr.y (mgr.14901) 5401 : cluster [DBG] pgmap v5408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:38 smithi006 bash[27292]: audit 2023-08-28T00:58:38.027036+0000 mon.c (mon.2) 2092 : audit [DBG] from='client.? 172.21.15.6:0/1598466492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:38 smithi006 bash[33918]: cluster 2023-08-28T00:58:37.384567+0000 mgr.y (mgr.14901) 5401 : cluster [DBG] pgmap v5408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:38 smithi006 bash[33918]: audit 2023-08-28T00:58:38.027036+0000 mon.c (mon.2) 2092 : audit [DBG] from='client.? 172.21.15.6:0/1598466492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:39.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:39 smithi130 bash[51302]: debug 2023-08-28T00:58:39.064+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:39.068186+0000) 2023-08-28T00:58:40.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:40 smithi130 bash[51302]: debug 2023-08-28T00:58:40.063+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:40.068474+0000) 2023-08-28T00:58:40.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:40 smithi006 bash[33918]: cluster 2023-08-28T00:58:39.385753+0000 mgr.y (mgr.14901) 5402 : cluster [DBG] pgmap v5409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:40.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:40 smithi006 bash[33918]: audit 2023-08-28T00:58:40.451752+0000 mon.a (mon.0) 3748 : audit [DBG] from='client.? 172.21.15.6:0/3842032016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:40.756 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:40 smithi006 bash[27292]: cluster 2023-08-28T00:58:39.385753+0000 mgr.y (mgr.14901) 5402 : cluster [DBG] pgmap v5409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:40.756 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:40 smithi006 bash[27292]: audit 2023-08-28T00:58:40.451752+0000 mon.a (mon.0) 3748 : audit [DBG] from='client.? 172.21.15.6:0/3842032016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:40 smithi130 bash[27579]: cluster 2023-08-28T00:58:39.385753+0000 mgr.y (mgr.14901) 5402 : cluster [DBG] pgmap v5409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:40 smithi130 bash[27579]: audit 2023-08-28T00:58:40.451752+0000 mon.a (mon.0) 3748 : audit [DBG] from='client.? 172.21.15.6:0/3842032016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:58:40.883Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:58:40.885Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:41.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:58:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:58:40] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T00:58:41.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:41 smithi130 bash[51302]: debug 2023-08-28T00:58:41.063+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:41.068758+0000) 2023-08-28T00:58:42.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:42 smithi130 bash[51302]: debug 2023-08-28T00:58:42.067+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:42.069029+0000) 2023-08-28T00:58:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:42 smithi006 bash[27292]: cluster 2023-08-28T00:58:41.386444+0000 mgr.y (mgr.14901) 5403 : cluster [DBG] pgmap v5410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:42.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:42 smithi006 bash[33918]: cluster 2023-08-28T00:58:41.386444+0000 mgr.y (mgr.14901) 5403 : cluster [DBG] pgmap v5410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:58:42.457Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:42.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:58:42.459Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:42 smithi130 bash[27579]: cluster 2023-08-28T00:58:41.386444+0000 mgr.y (mgr.14901) 5403 : cluster [DBG] pgmap v5410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:43.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:43 smithi130 bash[51302]: debug 2023-08-28T00:58:43.067+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:43.069306+0000) 2023-08-28T00:58:43.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:43 smithi006 bash[27292]: audit 2023-08-28T00:58:42.867879+0000 mon.a (mon.0) 3749 : audit [DBG] from='client.? 172.21.15.6:0/4113716517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:43.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:43 smithi006 bash[33918]: audit 2023-08-28T00:58:42.867879+0000 mon.a (mon.0) 3749 : audit [DBG] from='client.? 172.21.15.6:0/4113716517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:43.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:43 smithi130 bash[27579]: audit 2023-08-28T00:58:42.867879+0000 mon.a (mon.0) 3749 : audit [DBG] from='client.? 172.21.15.6:0/4113716517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:44.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:44 smithi130 bash[51302]: debug 2023-08-28T00:58:44.067+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:44.069609+0000) 2023-08-28T00:58:44.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:44 smithi006 bash[27292]: cluster 2023-08-28T00:58:43.386994+0000 mgr.y (mgr.14901) 5404 : cluster [DBG] pgmap v5411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:44.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:44 smithi006 bash[33918]: cluster 2023-08-28T00:58:43.386994+0000 mgr.y (mgr.14901) 5404 : cluster [DBG] pgmap v5411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:44.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:58:44.340Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:58:44.342Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:44 smithi130 bash[27579]: cluster 2023-08-28T00:58:43.386994+0000 mgr.y (mgr.14901) 5404 : cluster [DBG] pgmap v5411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:45 smithi130 bash[51302]: debug 2023-08-28T00:58:45.063+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:45.069907+0000) 2023-08-28T00:58:45.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:45 smithi006 bash[33918]: audit 2023-08-28T00:58:45.288733+0000 mon.c (mon.2) 2093 : audit [DBG] from='client.? 172.21.15.6:0/514239647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:45.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:45 smithi006 bash[27292]: audit 2023-08-28T00:58:45.288733+0000 mon.c (mon.2) 2093 : audit [DBG] from='client.? 172.21.15.6:0/514239647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:45.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:45 smithi130 bash[27579]: audit 2023-08-28T00:58:45.288733+0000 mon.c (mon.2) 2093 : audit [DBG] from='client.? 172.21.15.6:0/514239647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:46.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:46 smithi130 bash[51302]: debug 2023-08-28T00:58:46.067+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:46.070292+0000) 2023-08-28T00:58:46.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:46 smithi006 bash[33918]: cluster 2023-08-28T00:58:45.388169+0000 mgr.y (mgr.14901) 5405 : cluster [DBG] pgmap v5412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:46.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:46 smithi006 bash[27292]: cluster 2023-08-28T00:58:45.388169+0000 mgr.y (mgr.14901) 5405 : cluster [DBG] pgmap v5412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:46 smithi130 bash[27579]: cluster 2023-08-28T00:58:45.388169+0000 mgr.y (mgr.14901) 5405 : cluster [DBG] pgmap v5412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:47.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:47 smithi130 bash[51302]: debug 2023-08-28T00:58:47.067+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:47.070611+0000) 2023-08-28T00:58:48.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:48 smithi130 bash[51302]: debug 2023-08-28T00:58:48.067+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:48.070986+0000) 2023-08-28T00:58:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:48 smithi006 bash[27292]: cluster 2023-08-28T00:58:47.388855+0000 mgr.y (mgr.14901) 5406 : cluster [DBG] pgmap v5413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:48.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:48 smithi006 bash[27292]: audit 2023-08-28T00:58:47.703477+0000 mon.c (mon.2) 2094 : audit [DBG] from='client.? 172.21.15.6:0/1436728770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:48 smithi006 bash[33918]: cluster 2023-08-28T00:58:47.388855+0000 mgr.y (mgr.14901) 5406 : cluster [DBG] pgmap v5413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:48.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:48 smithi006 bash[33918]: audit 2023-08-28T00:58:47.703477+0000 mon.c (mon.2) 2094 : audit [DBG] from='client.? 172.21.15.6:0/1436728770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:48 smithi130 bash[27579]: cluster 2023-08-28T00:58:47.388855+0000 mgr.y (mgr.14901) 5406 : cluster [DBG] pgmap v5413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:48 smithi130 bash[27579]: audit 2023-08-28T00:58:47.703477+0000 mon.c (mon.2) 2094 : audit [DBG] from='client.? 172.21.15.6:0/1436728770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:49.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:49 smithi130 bash[51302]: debug 2023-08-28T00:58:49.067+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:49.071326+0000) 2023-08-28T00:58:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:50 smithi130 bash[51302]: debug 2023-08-28T00:58:50.067+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:50.071706+0000) 2023-08-28T00:58:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:50 smithi006 bash[33918]: cluster 2023-08-28T00:58:49.389909+0000 mgr.y (mgr.14901) 5407 : cluster [DBG] pgmap v5414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:50.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:50 smithi006 bash[33918]: audit 2023-08-28T00:58:50.121841+0000 mon.c (mon.2) 2095 : audit [DBG] from='client.? 172.21.15.6:0/3668083798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:50 smithi006 bash[27292]: cluster 2023-08-28T00:58:49.389909+0000 mgr.y (mgr.14901) 5407 : cluster [DBG] pgmap v5414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:50.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:50 smithi006 bash[27292]: audit 2023-08-28T00:58:50.121841+0000 mon.c (mon.2) 2095 : audit [DBG] from='client.? 172.21.15.6:0/3668083798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:50 smithi130 bash[27579]: cluster 2023-08-28T00:58:49.389909+0000 mgr.y (mgr.14901) 5407 : cluster [DBG] pgmap v5414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:50.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:50 smithi130 bash[27579]: audit 2023-08-28T00:58:50.121841+0000 mon.c (mon.2) 2095 : audit [DBG] from='client.? 172.21.15.6:0/3668083798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:51.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:58:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:58:50] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T00:58:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:58:50.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:58:50.886Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:51.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:51 smithi130 bash[51302]: debug 2023-08-28T00:58:51.067+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:51.071973+0000) 2023-08-28T00:58:52.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:52 smithi130 bash[51302]: debug 2023-08-28T00:58:52.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:52.072285+0000) 2023-08-28T00:58:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:52 smithi006 bash[33918]: cluster 2023-08-28T00:58:51.390514+0000 mgr.y (mgr.14901) 5408 : cluster [DBG] pgmap v5415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:52.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:52 smithi006 bash[27292]: cluster 2023-08-28T00:58:51.390514+0000 mgr.y (mgr.14901) 5408 : cluster [DBG] pgmap v5415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:58:52.459Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:52.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:58:52.460Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:52 smithi130 bash[27579]: cluster 2023-08-28T00:58:51.390514+0000 mgr.y (mgr.14901) 5408 : cluster [DBG] pgmap v5415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:53.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:53 smithi130 bash[51302]: debug 2023-08-28T00:58:53.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:53.072618+0000) 2023-08-28T00:58:53.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:53 smithi006 bash[33918]: audit 2023-08-28T00:58:52.541156+0000 mon.c (mon.2) 2096 : audit [DBG] from='client.? 172.21.15.6:0/2624176137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:53.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:53 smithi006 bash[27292]: audit 2023-08-28T00:58:52.541156+0000 mon.c (mon.2) 2096 : audit [DBG] from='client.? 172.21.15.6:0/2624176137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:53.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:53 smithi130 bash[27579]: audit 2023-08-28T00:58:52.541156+0000 mon.c (mon.2) 2096 : audit [DBG] from='client.? 172.21.15.6:0/2624176137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:54 smithi130 bash[51302]: debug 2023-08-28T00:58:54.067+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:54.072953+0000) 2023-08-28T00:58:54.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:54 smithi006 bash[33918]: cluster 2023-08-28T00:58:53.391153+0000 mgr.y (mgr.14901) 5409 : cluster [DBG] pgmap v5416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:54.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:58:54.340Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:54.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:58:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:58:54.343Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:58:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:54 smithi006 bash[27292]: cluster 2023-08-28T00:58:53.391153+0000 mgr.y (mgr.14901) 5409 : cluster [DBG] pgmap v5416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:54 smithi130 bash[27579]: cluster 2023-08-28T00:58:53.391153+0000 mgr.y (mgr.14901) 5409 : cluster [DBG] pgmap v5416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:55.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:55 smithi130 bash[51302]: debug 2023-08-28T00:58:55.067+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:55.073312+0000) 2023-08-28T00:58:55.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:55 smithi006 bash[33918]: audit 2023-08-28T00:58:54.951000+0000 mon.c (mon.2) 2097 : audit [DBG] from='client.? 172.21.15.6:0/3098524478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:55.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:55 smithi006 bash[27292]: audit 2023-08-28T00:58:54.951000+0000 mon.c (mon.2) 2097 : audit [DBG] from='client.? 172.21.15.6:0/3098524478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:55.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:55 smithi130 bash[27579]: audit 2023-08-28T00:58:54.951000+0000 mon.c (mon.2) 2097 : audit [DBG] from='client.? 172.21.15.6:0/3098524478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:56.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:56 smithi130 bash[51302]: debug 2023-08-28T00:58:56.067+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:56.073653+0000) 2023-08-28T00:58:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:56 smithi006 bash[27292]: cluster 2023-08-28T00:58:55.392297+0000 mgr.y (mgr.14901) 5410 : cluster [DBG] pgmap v5417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:56.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:56 smithi006 bash[33918]: cluster 2023-08-28T00:58:55.392297+0000 mgr.y (mgr.14901) 5410 : cluster [DBG] pgmap v5417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:56.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:56 smithi130 bash[27579]: cluster 2023-08-28T00:58:55.392297+0000 mgr.y (mgr.14901) 5410 : cluster [DBG] pgmap v5417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:57.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:57 smithi130 bash[51302]: debug 2023-08-28T00:58:57.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:57.073970+0000) 2023-08-28T00:58:57.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:57 smithi006 bash[27292]: audit 2023-08-28T00:58:57.364722+0000 mon.a (mon.0) 3750 : audit [DBG] from='client.? 172.21.15.6:0/942790659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:57.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:57 smithi006 bash[33918]: audit 2023-08-28T00:58:57.364722+0000 mon.a (mon.0) 3750 : audit [DBG] from='client.? 172.21.15.6:0/942790659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:57.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:57 smithi130 bash[27579]: audit 2023-08-28T00:58:57.364722+0000 mon.a (mon.0) 3750 : audit [DBG] from='client.? 172.21.15.6:0/942790659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:58.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:58 smithi130 bash[51302]: debug 2023-08-28T00:58:58.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:58.074323+0000) 2023-08-28T00:58:58.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:58:58 smithi006 bash[27292]: cluster 2023-08-28T00:58:57.392926+0000 mgr.y (mgr.14901) 5411 : cluster [DBG] pgmap v5418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:58.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:58:58 smithi006 bash[33918]: cluster 2023-08-28T00:58:57.392926+0000 mgr.y (mgr.14901) 5411 : cluster [DBG] pgmap v5418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:58:58 smithi130 bash[27579]: cluster 2023-08-28T00:58:57.392926+0000 mgr.y (mgr.14901) 5411 : cluster [DBG] pgmap v5418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:58:59.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:58:59 smithi130 bash[51302]: debug 2023-08-28T00:58:59.067+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:59.074640+0000) 2023-08-28T00:59:00.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:00 smithi130 bash[51302]: debug 2023-08-28T00:59:00.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:00.075013+0000) 2023-08-28T00:59:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:00 smithi006 bash[33918]: cluster 2023-08-28T00:58:59.393946+0000 mgr.y (mgr.14901) 5412 : cluster [DBG] pgmap v5419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:00 smithi006 bash[33918]: audit 2023-08-28T00:58:59.782214+0000 mon.c (mon.2) 2098 : audit [DBG] from='client.? 172.21.15.6:0/1832887490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:00 smithi006 bash[27292]: cluster 2023-08-28T00:58:59.393946+0000 mgr.y (mgr.14901) 5412 : cluster [DBG] pgmap v5419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:00.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:00 smithi006 bash[27292]: audit 2023-08-28T00:58:59.782214+0000 mon.c (mon.2) 2098 : audit [DBG] from='client.? 172.21.15.6:0/1832887490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:00 smithi130 bash[27579]: cluster 2023-08-28T00:58:59.393946+0000 mgr.y (mgr.14901) 5412 : cluster [DBG] pgmap v5419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:00 smithi130 bash[27579]: audit 2023-08-28T00:58:59.782214+0000 mon.c (mon.2) 2098 : audit [DBG] from='client.? 172.21.15.6:0/1832887490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:01.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:59:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:59:00] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:59:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:00 smithi006 bash[56298]: level=error ts=2023-08-28T00:59:00.886Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:00 smithi006 bash[56298]: level=warn ts=2023-08-28T00:59:00.888Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:01.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:01 smithi130 bash[51302]: debug 2023-08-28T00:59:01.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:01.075341+0000) 2023-08-28T00:59:02.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:02 smithi130 bash[51302]: debug 2023-08-28T00:59:02.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:02.075699+0000) 2023-08-28T00:59:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:02 smithi006 bash[27292]: cluster 2023-08-28T00:59:01.394514+0000 mgr.y (mgr.14901) 5413 : cluster [DBG] pgmap v5420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:02 smithi006 bash[27292]: audit 2023-08-28T00:59:02.193179+0000 mon.a (mon.0) 3751 : audit [DBG] from='client.? 172.21.15.6:0/3394792303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:02 smithi006 bash[33918]: cluster 2023-08-28T00:59:01.394514+0000 mgr.y (mgr.14901) 5413 : cluster [DBG] pgmap v5420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:02 smithi006 bash[33918]: audit 2023-08-28T00:59:02.193179+0000 mon.a (mon.0) 3751 : audit [DBG] from='client.? 172.21.15.6:0/3394792303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:02 smithi006 bash[56298]: level=error ts=2023-08-28T00:59:02.460Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:02.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:02 smithi006 bash[56298]: level=warn ts=2023-08-28T00:59:02.462Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:02 smithi130 bash[27579]: cluster 2023-08-28T00:59:01.394514+0000 mgr.y (mgr.14901) 5413 : cluster [DBG] pgmap v5420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:02 smithi130 bash[27579]: audit 2023-08-28T00:59:02.193179+0000 mon.a (mon.0) 3751 : audit [DBG] from='client.? 172.21.15.6:0/3394792303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:03.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:03 smithi130 bash[51302]: debug 2023-08-28T00:59:03.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:03.076036+0000) 2023-08-28T00:59:04.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:04 smithi130 bash[51302]: debug 2023-08-28T00:59:04.075+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:04.076379+0000) 2023-08-28T00:59:04.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:04 smithi006 bash[33918]: cluster 2023-08-28T00:59:03.395149+0000 mgr.y (mgr.14901) 5414 : cluster [DBG] pgmap v5421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:04.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:04 smithi006 bash[27292]: cluster 2023-08-28T00:59:03.395149+0000 mgr.y (mgr.14901) 5414 : cluster [DBG] pgmap v5421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:04.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:04 smithi006 bash[56298]: level=error ts=2023-08-28T00:59:04.341Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:04.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:04 smithi006 bash[56298]: level=warn ts=2023-08-28T00:59:04.343Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:04 smithi130 bash[27579]: cluster 2023-08-28T00:59:03.395149+0000 mgr.y (mgr.14901) 5414 : cluster [DBG] pgmap v5421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:05.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:05 smithi130 bash[51302]: debug 2023-08-28T00:59:05.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:05.076728+0000) 2023-08-28T00:59:05.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:05 smithi006 bash[33918]: audit 2023-08-28T00:59:04.603217+0000 mon.a (mon.0) 3752 : audit [DBG] from='client.? 172.21.15.6:0/3607957994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:05.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:05 smithi006 bash[27292]: audit 2023-08-28T00:59:04.603217+0000 mon.a (mon.0) 3752 : audit [DBG] from='client.? 172.21.15.6:0/3607957994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:05.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:05 smithi130 bash[27579]: audit 2023-08-28T00:59:04.603217+0000 mon.a (mon.0) 3752 : audit [DBG] from='client.? 172.21.15.6:0/3607957994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:06.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:06 smithi130 bash[51302]: debug 2023-08-28T00:59:06.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:06.077079+0000) 2023-08-28T00:59:06.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:06 smithi006 bash[33918]: cluster 2023-08-28T00:59:05.396334+0000 mgr.y (mgr.14901) 5415 : cluster [DBG] pgmap v5422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:06.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:06 smithi006 bash[27292]: cluster 2023-08-28T00:59:05.396334+0000 mgr.y (mgr.14901) 5415 : cluster [DBG] pgmap v5422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:06 smithi130 bash[27579]: cluster 2023-08-28T00:59:05.396334+0000 mgr.y (mgr.14901) 5415 : cluster [DBG] pgmap v5422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:07.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:07 smithi130 bash[51302]: debug 2023-08-28T00:59:07.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:07.077431+0000) 2023-08-28T00:59:07.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:07 smithi130 bash[27579]: audit 2023-08-28T00:59:07.022470+0000 mon.c (mon.2) 2099 : audit [DBG] from='client.? 172.21.15.6:0/3280134507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:07.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:07 smithi006 bash[27292]: audit 2023-08-28T00:59:07.022470+0000 mon.c (mon.2) 2099 : audit [DBG] from='client.? 172.21.15.6:0/3280134507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:07.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:07 smithi006 bash[33918]: audit 2023-08-28T00:59:07.022470+0000 mon.c (mon.2) 2099 : audit [DBG] from='client.? 172.21.15.6:0/3280134507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:08.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:08 smithi130 bash[51302]: debug 2023-08-28T00:59:08.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:08.077758+0000) 2023-08-28T00:59:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:08 smithi130 bash[27579]: cluster 2023-08-28T00:59:07.397104+0000 mgr.y (mgr.14901) 5416 : cluster [DBG] pgmap v5423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:08.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:08 smithi006 bash[27292]: cluster 2023-08-28T00:59:07.397104+0000 mgr.y (mgr.14901) 5416 : cluster [DBG] pgmap v5423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:08 smithi006 bash[33918]: cluster 2023-08-28T00:59:07.397104+0000 mgr.y (mgr.14901) 5416 : cluster [DBG] pgmap v5423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:09.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:09 smithi130 bash[51302]: debug 2023-08-28T00:59:09.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:09.078092+0000) 2023-08-28T00:59:09.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:09 smithi130 bash[27579]: audit 2023-08-28T00:59:09.446086+0000 mon.c (mon.2) 2100 : audit [DBG] from='client.? 172.21.15.6:0/93466954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:09.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:09 smithi006 bash[27292]: audit 2023-08-28T00:59:09.446086+0000 mon.c (mon.2) 2100 : audit [DBG] from='client.? 172.21.15.6:0/93466954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:09.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:09 smithi006 bash[33918]: audit 2023-08-28T00:59:09.446086+0000 mon.c (mon.2) 2100 : audit [DBG] from='client.? 172.21.15.6:0/93466954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:10.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:10 smithi130 bash[51302]: debug 2023-08-28T00:59:10.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:10.078427+0000) 2023-08-28T00:59:10.754 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:10 smithi006 bash[27292]: cluster 2023-08-28T00:59:09.398139+0000 mgr.y (mgr.14901) 5417 : cluster [DBG] pgmap v5424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:10.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:10 smithi006 bash[33918]: cluster 2023-08-28T00:59:09.398139+0000 mgr.y (mgr.14901) 5417 : cluster [DBG] pgmap v5424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:10.870 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:10 smithi130 bash[27579]: cluster 2023-08-28T00:59:09.398139+0000 mgr.y (mgr.14901) 5417 : cluster [DBG] pgmap v5424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:11.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:59:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:59:10] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T00:59:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:10 smithi006 bash[56298]: level=error ts=2023-08-28T00:59:10.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:10 smithi006 bash[56298]: level=warn ts=2023-08-28T00:59:10.888Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:11.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:11 smithi130 bash[51302]: debug 2023-08-28T00:59:11.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:11.078769+0000) 2023-08-28T00:59:12.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:12 smithi130 bash[51302]: debug 2023-08-28T00:59:12.075+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:12.079144+0000) 2023-08-28T00:59:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:12 smithi006 bash[27292]: cluster 2023-08-28T00:59:11.398653+0000 mgr.y (mgr.14901) 5418 : cluster [DBG] pgmap v5425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:12 smithi006 bash[27292]: audit 2023-08-28T00:59:11.861946+0000 mon.a (mon.0) 3753 : audit [DBG] from='client.? 172.21.15.6:0/3361235280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:12.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:12 smithi006 bash[56298]: level=error ts=2023-08-28T00:59:12.460Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:12.719 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:12 smithi006 bash[56298]: level=warn ts=2023-08-28T00:59:12.462Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:12.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:12 smithi006 bash[33918]: cluster 2023-08-28T00:59:11.398653+0000 mgr.y (mgr.14901) 5418 : cluster [DBG] pgmap v5425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:12.719 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:12 smithi006 bash[33918]: audit 2023-08-28T00:59:11.861946+0000 mon.a (mon.0) 3753 : audit [DBG] from='client.? 172.21.15.6:0/3361235280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:12 smithi130 bash[27579]: cluster 2023-08-28T00:59:11.398653+0000 mgr.y (mgr.14901) 5418 : cluster [DBG] pgmap v5425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:12 smithi130 bash[27579]: audit 2023-08-28T00:59:11.861946+0000 mon.a (mon.0) 3753 : audit [DBG] from='client.? 172.21.15.6:0/3361235280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:13.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:13 smithi130 bash[51302]: debug 2023-08-28T00:59:13.075+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:13.079535+0000) 2023-08-28T00:59:13.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:13 smithi130 bash[27579]: audit 2023-08-28T00:59:13.449606+0000 mon.b (mon.1) 639 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:59:13.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:13 smithi006 bash[27292]: audit 2023-08-28T00:59:13.449606+0000 mon.b (mon.1) 639 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:59:13.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:13 smithi006 bash[33918]: audit 2023-08-28T00:59:13.449606+0000 mon.b (mon.1) 639 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:59:14.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:14 smithi130 bash[51302]: debug 2023-08-28T00:59:14.075+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:14.079886+0000) 2023-08-28T00:59:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:14 smithi006 bash[33918]: cluster 2023-08-28T00:59:13.399311+0000 mgr.y (mgr.14901) 5419 : cluster [DBG] pgmap v5426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:14 smithi006 bash[33918]: audit 2023-08-28T00:59:14.268266+0000 mon.c (mon.2) 2101 : audit [DBG] from='client.? 172.21.15.6:0/4252079310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:14.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:14 smithi006 bash[56298]: level=error ts=2023-08-28T00:59:14.342Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:14.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:14 smithi006 bash[56298]: level=warn ts=2023-08-28T00:59:14.344Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:14.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:14 smithi006 bash[27292]: cluster 2023-08-28T00:59:13.399311+0000 mgr.y (mgr.14901) 5419 : cluster [DBG] pgmap v5426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:14.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:14 smithi006 bash[27292]: audit 2023-08-28T00:59:14.268266+0000 mon.c (mon.2) 2101 : audit [DBG] from='client.? 172.21.15.6:0/4252079310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:14 smithi130 bash[27579]: cluster 2023-08-28T00:59:13.399311+0000 mgr.y (mgr.14901) 5419 : cluster [DBG] pgmap v5426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:14.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:14 smithi130 bash[27579]: audit 2023-08-28T00:59:14.268266+0000 mon.c (mon.2) 2101 : audit [DBG] from='client.? 172.21.15.6:0/4252079310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:15.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:15 smithi130 bash[51302]: debug 2023-08-28T00:59:15.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:15.080245+0000) 2023-08-28T00:59:16.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:16 smithi130 bash[51302]: debug 2023-08-28T00:59:16.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:16.080607+0000) 2023-08-28T00:59:16.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:16 smithi130 bash[27579]: cluster 2023-08-28T00:59:15.400499+0000 mgr.y (mgr.14901) 5420 : cluster [DBG] pgmap v5427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:16.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:16 smithi006 bash[27292]: cluster 2023-08-28T00:59:15.400499+0000 mgr.y (mgr.14901) 5420 : cluster [DBG] pgmap v5427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:16 smithi006 bash[33918]: cluster 2023-08-28T00:59:15.400499+0000 mgr.y (mgr.14901) 5420 : cluster [DBG] pgmap v5427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:17.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:17 smithi130 bash[51302]: debug 2023-08-28T00:59:17.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:17.080952+0000) 2023-08-28T00:59:17.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:17 smithi130 bash[27579]: audit 2023-08-28T00:59:16.678369+0000 mon.a (mon.0) 3754 : audit [DBG] from='client.? 172.21.15.6:0/2638608290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:17.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:17 smithi006 bash[33918]: audit 2023-08-28T00:59:16.678369+0000 mon.a (mon.0) 3754 : audit [DBG] from='client.? 172.21.15.6:0/2638608290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:17.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:17 smithi006 bash[27292]: audit 2023-08-28T00:59:16.678369+0000 mon.a (mon.0) 3754 : audit [DBG] from='client.? 172.21.15.6:0/2638608290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:18.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:18 smithi130 bash[51302]: debug 2023-08-28T00:59:18.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:18.081312+0000) 2023-08-28T00:59:18.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:18 smithi130 bash[27579]: cluster 2023-08-28T00:59:17.401220+0000 mgr.y (mgr.14901) 5421 : cluster [DBG] pgmap v5428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:18.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:18 smithi006 bash[27292]: cluster 2023-08-28T00:59:17.401220+0000 mgr.y (mgr.14901) 5421 : cluster [DBG] pgmap v5428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:18 smithi006 bash[33918]: cluster 2023-08-28T00:59:17.401220+0000 mgr.y (mgr.14901) 5421 : cluster [DBG] pgmap v5428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:19.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:19 smithi130 bash[51302]: debug 2023-08-28T00:59:19.075+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:19.081578+0000) 2023-08-28T00:59:19.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:19 smithi130 bash[27579]: audit 2023-08-28T00:59:19.157745+0000 mon.c (mon.2) 2102 : audit [DBG] from='client.? 172.21.15.6:0/1333857912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:19.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:19 smithi130 bash[27579]: audit 2023-08-28T00:59:19.287394+0000 mon.a (mon.0) 3755 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:59:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:19 smithi006 bash[33918]: audit 2023-08-28T00:59:19.157745+0000 mon.c (mon.2) 2102 : audit [DBG] from='client.? 172.21.15.6:0/1333857912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:19.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:19 smithi006 bash[33918]: audit 2023-08-28T00:59:19.287394+0000 mon.a (mon.0) 3755 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:59:19.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:19 smithi006 bash[27292]: audit 2023-08-28T00:59:19.157745+0000 mon.c (mon.2) 2102 : audit [DBG] from='client.? 172.21.15.6:0/1333857912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:19.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:19 smithi006 bash[27292]: audit 2023-08-28T00:59:19.287394+0000 mon.a (mon.0) 3755 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:59:20.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:20 smithi130 bash[51302]: debug 2023-08-28T00:59:20.075+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:20.081919+0000) 2023-08-28T00:59:21.083 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:20 smithi130 bash[27579]: cluster 2023-08-28T00:59:19.402335+0000 mgr.y (mgr.14901) 5422 : cluster [DBG] pgmap v5429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:21.083 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:20 smithi130 bash[27579]: audit 2023-08-28T00:59:19.795053+0000 mon.a (mon.0) 3756 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:59:21.083 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:20 smithi130 bash[27579]: audit 2023-08-28T00:59:20.091729+0000 mon.b (mon.1) 640 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:59:21.083 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:20 smithi130 bash[27579]: audit 2023-08-28T00:59:20.093873+0000 mon.b (mon.1) 641 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:59:21.083 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:20 smithi130 bash[27579]: audit 2023-08-28T00:59:20.102989+0000 mon.a (mon.0) 3757 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:59:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:20 smithi006 bash[27292]: cluster 2023-08-28T00:59:19.402335+0000 mgr.y (mgr.14901) 5422 : cluster [DBG] pgmap v5429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:20 smithi006 bash[27292]: audit 2023-08-28T00:59:19.795053+0000 mon.a (mon.0) 3756 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:59:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:20 smithi006 bash[27292]: audit 2023-08-28T00:59:20.091729+0000 mon.b (mon.1) 640 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:59:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:20 smithi006 bash[27292]: audit 2023-08-28T00:59:20.093873+0000 mon.b (mon.1) 641 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:59:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:20 smithi006 bash[27292]: audit 2023-08-28T00:59:20.102989+0000 mon.a (mon.0) 3757 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:59:21.217 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:59:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:59:20] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T00:59:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:20 smithi006 bash[56298]: level=error ts=2023-08-28T00:59:20.888Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:21.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:20 smithi006 bash[56298]: level=warn ts=2023-08-28T00:59:20.890Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:21.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:20 smithi006 bash[33918]: cluster 2023-08-28T00:59:19.402335+0000 mgr.y (mgr.14901) 5422 : cluster [DBG] pgmap v5429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:21.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:20 smithi006 bash[33918]: audit 2023-08-28T00:59:19.795053+0000 mon.a (mon.0) 3756 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:59:21.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:20 smithi006 bash[33918]: audit 2023-08-28T00:59:20.091729+0000 mon.b (mon.1) 640 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:59:21.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:20 smithi006 bash[33918]: audit 2023-08-28T00:59:20.093873+0000 mon.b (mon.1) 641 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:59:21.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:20 smithi006 bash[33918]: audit 2023-08-28T00:59:20.102989+0000 mon.a (mon.0) 3757 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T00:59:21.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:21 smithi130 bash[51302]: debug 2023-08-28T00:59:21.075+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:21.082216+0000) 2023-08-28T00:59:22.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:22 smithi130 bash[51302]: debug 2023-08-28T00:59:22.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:22.082547+0000) 2023-08-28T00:59:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:22 smithi006 bash[27292]: cluster 2023-08-28T00:59:21.402839+0000 mgr.y (mgr.14901) 5423 : cluster [DBG] pgmap v5430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:22 smithi006 bash[27292]: audit 2023-08-28T00:59:21.570505+0000 mon.c (mon.2) 2103 : audit [DBG] from='client.? 172.21.15.6:0/3174817527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:22 smithi006 bash[33918]: cluster 2023-08-28T00:59:21.402839+0000 mgr.y (mgr.14901) 5423 : cluster [DBG] pgmap v5430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:22 smithi006 bash[33918]: audit 2023-08-28T00:59:21.570505+0000 mon.c (mon.2) 2103 : audit [DBG] from='client.? 172.21.15.6:0/3174817527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:22 smithi006 bash[56298]: level=error ts=2023-08-28T00:59:22.461Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:22 smithi006 bash[56298]: level=warn ts=2023-08-28T00:59:22.464Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:22 smithi130 bash[27579]: cluster 2023-08-28T00:59:21.402839+0000 mgr.y (mgr.14901) 5423 : cluster [DBG] pgmap v5430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:22 smithi130 bash[27579]: audit 2023-08-28T00:59:21.570505+0000 mon.c (mon.2) 2103 : audit [DBG] from='client.? 172.21.15.6:0/3174817527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:23.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:23 smithi130 bash[51302]: debug 2023-08-28T00:59:23.075+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:23.082810+0000) 2023-08-28T00:59:24.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:24 smithi130 bash[51302]: debug 2023-08-28T00:59:24.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:24.083176+0000) 2023-08-28T00:59:24.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:24 smithi006 bash[27292]: cluster 2023-08-28T00:59:23.403575+0000 mgr.y (mgr.14901) 5424 : cluster [DBG] pgmap v5431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:24.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:24 smithi006 bash[27292]: audit 2023-08-28T00:59:23.986612+0000 mon.a (mon.0) 3758 : audit [DBG] from='client.? 172.21.15.6:0/3897846411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:24.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:24 smithi006 bash[33918]: cluster 2023-08-28T00:59:23.403575+0000 mgr.y (mgr.14901) 5424 : cluster [DBG] pgmap v5431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:24.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:24 smithi006 bash[33918]: audit 2023-08-28T00:59:23.986612+0000 mon.a (mon.0) 3758 : audit [DBG] from='client.? 172.21.15.6:0/3897846411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:24.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:24 smithi006 bash[56298]: level=error ts=2023-08-28T00:59:24.342Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:24 smithi006 bash[56298]: level=warn ts=2023-08-28T00:59:24.345Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:24 smithi130 bash[27579]: cluster 2023-08-28T00:59:23.403575+0000 mgr.y (mgr.14901) 5424 : cluster [DBG] pgmap v5431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:24.877 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:24 smithi130 bash[27579]: audit 2023-08-28T00:59:23.986612+0000 mon.a (mon.0) 3758 : audit [DBG] from='client.? 172.21.15.6:0/3897846411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:25.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:25 smithi130 bash[51302]: debug 2023-08-28T00:59:25.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:25.083508+0000) 2023-08-28T00:59:26.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:26 smithi130 bash[51302]: debug 2023-08-28T00:59:26.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:26.083831+0000) 2023-08-28T00:59:26.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:26 smithi130 bash[27579]: cluster 2023-08-28T00:59:25.404713+0000 mgr.y (mgr.14901) 5425 : cluster [DBG] pgmap v5432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:26 smithi130 bash[27579]: audit 2023-08-28T00:59:26.398984+0000 mon.c (mon.2) 2104 : audit [DBG] from='client.? 172.21.15.6:0/1944704145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:26.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:26 smithi006 bash[27292]: cluster 2023-08-28T00:59:25.404713+0000 mgr.y (mgr.14901) 5425 : cluster [DBG] pgmap v5432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:26 smithi006 bash[27292]: audit 2023-08-28T00:59:26.398984+0000 mon.c (mon.2) 2104 : audit [DBG] from='client.? 172.21.15.6:0/1944704145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:26 smithi006 bash[33918]: cluster 2023-08-28T00:59:25.404713+0000 mgr.y (mgr.14901) 5425 : cluster [DBG] pgmap v5432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:26 smithi006 bash[33918]: audit 2023-08-28T00:59:26.398984+0000 mon.c (mon.2) 2104 : audit [DBG] from='client.? 172.21.15.6:0/1944704145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:27.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:27 smithi130 bash[51302]: debug 2023-08-28T00:59:27.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:27.084114+0000) 2023-08-28T00:59:28.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:28 smithi130 bash[51302]: debug 2023-08-28T00:59:28.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:28.084437+0000) 2023-08-28T00:59:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:28 smithi130 bash[27579]: cluster 2023-08-28T00:59:27.405392+0000 mgr.y (mgr.14901) 5426 : cluster [DBG] pgmap v5433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:28 smithi006 bash[33918]: cluster 2023-08-28T00:59:27.405392+0000 mgr.y (mgr.14901) 5426 : cluster [DBG] pgmap v5433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:28.968 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:28 smithi006 bash[27292]: cluster 2023-08-28T00:59:27.405392+0000 mgr.y (mgr.14901) 5426 : cluster [DBG] pgmap v5433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:29.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:29 smithi130 bash[51302]: debug 2023-08-28T00:59:29.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:29.084767+0000) 2023-08-28T00:59:29.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:29 smithi130 bash[27579]: audit 2023-08-28T00:59:28.817572+0000 mon.c (mon.2) 2105 : audit [DBG] from='client.? 172.21.15.6:0/2654691508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:29.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:29 smithi006 bash[27292]: audit 2023-08-28T00:59:28.817572+0000 mon.c (mon.2) 2105 : audit [DBG] from='client.? 172.21.15.6:0/2654691508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:29.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:29 smithi006 bash[33918]: audit 2023-08-28T00:59:28.817572+0000 mon.c (mon.2) 2105 : audit [DBG] from='client.? 172.21.15.6:0/2654691508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:30.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:30 smithi130 bash[51302]: debug 2023-08-28T00:59:30.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:30.085135+0000) 2023-08-28T00:59:30.755 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:30 smithi006 bash[27292]: cluster 2023-08-28T00:59:29.406625+0000 mgr.y (mgr.14901) 5427 : cluster [DBG] pgmap v5434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:30.906 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:30 smithi006 bash[33918]: cluster 2023-08-28T00:59:29.406625+0000 mgr.y (mgr.14901) 5427 : cluster [DBG] pgmap v5434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:30.906 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:30 smithi130 bash[27579]: cluster 2023-08-28T00:59:29.406625+0000 mgr.y (mgr.14901) 5427 : cluster [DBG] pgmap v5434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:59:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:59:30] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:59:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:30 smithi006 bash[56298]: level=error ts=2023-08-28T00:59:30.888Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:30 smithi006 bash[56298]: level=warn ts=2023-08-28T00:59:30.890Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:31.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:31 smithi130 bash[51302]: debug 2023-08-28T00:59:31.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:31.085507+0000) 2023-08-28T00:59:31.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:31 smithi130 bash[27579]: audit 2023-08-28T00:59:31.235292+0000 mon.a (mon.0) 3759 : audit [DBG] from='client.? 172.21.15.6:0/2270502704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:33.160 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:31 smithi006 bash[27292]: audit 2023-08-28T00:59:31.235292+0000 mon.a (mon.0) 3759 : audit [DBG] from='client.? 172.21.15.6:0/2270502704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:33.160 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:32 smithi130 bash[51302]: debug 2023-08-28T00:59:32.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:32.085862+0000) 2023-08-28T00:59:33.161 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:31 smithi006 bash[33918]: audit 2023-08-28T00:59:31.235292+0000 mon.a (mon.0) 3759 : audit [DBG] from='client.? 172.21.15.6:0/2270502704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:33.314 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:32 smithi130 bash[27579]: cluster 2023-08-28T00:59:31.407337+0000 mgr.y (mgr.14901) 5428 : cluster [DBG] pgmap v5435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:33.315 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:32 smithi006 bash[27292]: cluster 2023-08-28T00:59:31.407337+0000 mgr.y (mgr.14901) 5428 : cluster [DBG] pgmap v5435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:33.315 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:32 smithi006 bash[33918]: cluster 2023-08-28T00:59:31.407337+0000 mgr.y (mgr.14901) 5428 : cluster [DBG] pgmap v5435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:33.315 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:32 smithi006 bash[56298]: level=error ts=2023-08-28T00:59:32.463Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:33.316 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:32 smithi006 bash[56298]: level=warn ts=2023-08-28T00:59:32.465Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:33.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:33 smithi130 bash[51302]: debug 2023-08-28T00:59:33.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:33.086208+0000) 2023-08-28T00:59:34.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:34 smithi130 bash[51302]: debug 2023-08-28T00:59:34.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:34.086564+0000) 2023-08-28T00:59:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:34 smithi006 bash[33918]: cluster 2023-08-28T00:59:33.407987+0000 mgr.y (mgr.14901) 5429 : cluster [DBG] pgmap v5436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:34.801 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:34 smithi006 bash[33918]: audit 2023-08-28T00:59:33.650881+0000 mon.a (mon.0) 3760 : audit [DBG] from='client.? 172.21.15.6:0/131385291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:34.801 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:34 smithi006 bash[56298]: level=error ts=2023-08-28T00:59:34.343Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:34.801 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:34 smithi006 bash[56298]: level=warn ts=2023-08-28T00:59:34.345Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:34.812 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:34 smithi006 bash[27292]: cluster 2023-08-28T00:59:33.407987+0000 mgr.y (mgr.14901) 5429 : cluster [DBG] pgmap v5436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:34.812 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:34 smithi006 bash[27292]: audit 2023-08-28T00:59:33.650881+0000 mon.a (mon.0) 3760 : audit [DBG] from='client.? 172.21.15.6:0/131385291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:34.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:34 smithi130 bash[27579]: cluster 2023-08-28T00:59:33.407987+0000 mgr.y (mgr.14901) 5429 : cluster [DBG] pgmap v5436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:34.976 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:34 smithi130 bash[27579]: audit 2023-08-28T00:59:33.650881+0000 mon.a (mon.0) 3760 : audit [DBG] from='client.? 172.21.15.6:0/131385291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:35.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:35 smithi130 bash[51302]: debug 2023-08-28T00:59:35.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:35.086899+0000) 2023-08-28T00:59:36.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:36 smithi130 bash[51302]: debug 2023-08-28T00:59:36.086+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:36.087284+0000) 2023-08-28T00:59:36.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:36 smithi130 bash[27579]: cluster 2023-08-28T00:59:35.409189+0000 mgr.y (mgr.14901) 5430 : cluster [DBG] pgmap v5437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:36.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:36 smithi130 bash[27579]: audit 2023-08-28T00:59:36.070038+0000 mon.a (mon.0) 3761 : audit [DBG] from='client.? 172.21.15.6:0/2874567190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:36 smithi006 bash[33918]: cluster 2023-08-28T00:59:35.409189+0000 mgr.y (mgr.14901) 5430 : cluster [DBG] pgmap v5437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:36 smithi006 bash[33918]: audit 2023-08-28T00:59:36.070038+0000 mon.a (mon.0) 3761 : audit [DBG] from='client.? 172.21.15.6:0/2874567190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:36 smithi006 bash[27292]: cluster 2023-08-28T00:59:35.409189+0000 mgr.y (mgr.14901) 5430 : cluster [DBG] pgmap v5437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:36 smithi006 bash[27292]: audit 2023-08-28T00:59:36.070038+0000 mon.a (mon.0) 3761 : audit [DBG] from='client.? 172.21.15.6:0/2874567190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:37.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:37 smithi130 bash[51302]: debug 2023-08-28T00:59:37.086+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:37.087662+0000) 2023-08-28T00:59:38.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:38 smithi130 bash[51302]: debug 2023-08-28T00:59:38.082+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:38.088016+0000) 2023-08-28T00:59:38.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:38 smithi130 bash[27579]: cluster 2023-08-28T00:59:37.409868+0000 mgr.y (mgr.14901) 5431 : cluster [DBG] pgmap v5438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:38.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:38 smithi130 bash[27579]: audit 2023-08-28T00:59:38.493312+0000 mon.a (mon.0) 3762 : audit [DBG] from='client.? 172.21.15.6:0/800987784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:38 smithi006 bash[33918]: cluster 2023-08-28T00:59:37.409868+0000 mgr.y (mgr.14901) 5431 : cluster [DBG] pgmap v5438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:38 smithi006 bash[33918]: audit 2023-08-28T00:59:38.493312+0000 mon.a (mon.0) 3762 : audit [DBG] from='client.? 172.21.15.6:0/800987784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:38 smithi006 bash[27292]: cluster 2023-08-28T00:59:37.409868+0000 mgr.y (mgr.14901) 5431 : cluster [DBG] pgmap v5438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:38 smithi006 bash[27292]: audit 2023-08-28T00:59:38.493312+0000 mon.a (mon.0) 3762 : audit [DBG] from='client.? 172.21.15.6:0/800987784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:39.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:39 smithi130 bash[51302]: debug 2023-08-28T00:59:39.082+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:39.088374+0000) 2023-08-28T00:59:40.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:40 smithi130 bash[51302]: debug 2023-08-28T00:59:40.082+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:40.088703+0000) 2023-08-28T00:59:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:40 smithi130 bash[27579]: cluster 2023-08-28T00:59:39.410986+0000 mgr.y (mgr.14901) 5432 : cluster [DBG] pgmap v5439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:40.891 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:59:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:59:40] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T00:59:40.892 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:40 smithi006 bash[33918]: cluster 2023-08-28T00:59:39.410986+0000 mgr.y (mgr.14901) 5432 : cluster [DBG] pgmap v5439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:40.892 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:40 smithi006 bash[27292]: cluster 2023-08-28T00:59:39.410986+0000 mgr.y (mgr.14901) 5432 : cluster [DBG] pgmap v5439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:40 smithi006 bash[56298]: level=error ts=2023-08-28T00:59:40.889Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:40 smithi006 bash[56298]: level=warn ts=2023-08-28T00:59:40.891Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:41.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:41 smithi130 bash[51302]: debug 2023-08-28T00:59:41.086+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:41.089060+0000) 2023-08-28T00:59:41.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:41 smithi130 bash[27579]: audit 2023-08-28T00:59:40.906732+0000 mon.c (mon.2) 2106 : audit [DBG] from='client.? 172.21.15.6:0/3008171789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:41.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:41 smithi006 bash[33918]: audit 2023-08-28T00:59:40.906732+0000 mon.c (mon.2) 2106 : audit [DBG] from='client.? 172.21.15.6:0/3008171789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:41.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:41 smithi006 bash[27292]: audit 2023-08-28T00:59:40.906732+0000 mon.c (mon.2) 2106 : audit [DBG] from='client.? 172.21.15.6:0/3008171789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:42.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:42 smithi130 bash[51302]: debug 2023-08-28T00:59:42.082+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:42.089328+0000) 2023-08-28T00:59:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:42 smithi006 bash[27292]: cluster 2023-08-28T00:59:41.411718+0000 mgr.y (mgr.14901) 5433 : cluster [DBG] pgmap v5440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:42.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:42 smithi006 bash[33918]: cluster 2023-08-28T00:59:41.411718+0000 mgr.y (mgr.14901) 5433 : cluster [DBG] pgmap v5440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:42 smithi006 bash[56298]: level=error ts=2023-08-28T00:59:42.463Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:42 smithi006 bash[56298]: level=warn ts=2023-08-28T00:59:42.465Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:42 smithi130 bash[27579]: cluster 2023-08-28T00:59:41.411718+0000 mgr.y (mgr.14901) 5433 : cluster [DBG] pgmap v5440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:43.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:43 smithi130 bash[51302]: debug 2023-08-28T00:59:43.086+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:43.089657+0000) 2023-08-28T00:59:43.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:43 smithi130 bash[27579]: audit 2023-08-28T00:59:43.324621+0000 mon.a (mon.0) 3763 : audit [DBG] from='client.? 172.21.15.6:0/3204990472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:43.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:43 smithi006 bash[27292]: audit 2023-08-28T00:59:43.324621+0000 mon.a (mon.0) 3763 : audit [DBG] from='client.? 172.21.15.6:0/3204990472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:43.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:43 smithi006 bash[33918]: audit 2023-08-28T00:59:43.324621+0000 mon.a (mon.0) 3763 : audit [DBG] from='client.? 172.21.15.6:0/3204990472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:44.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:44 smithi130 bash[51302]: debug 2023-08-28T00:59:44.086+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:44.089944+0000) 2023-08-28T00:59:44.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:44 smithi006 bash[27292]: cluster 2023-08-28T00:59:43.412352+0000 mgr.y (mgr.14901) 5434 : cluster [DBG] pgmap v5441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:44.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:44 smithi006 bash[33918]: cluster 2023-08-28T00:59:43.412352+0000 mgr.y (mgr.14901) 5434 : cluster [DBG] pgmap v5441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:44.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:44 smithi006 bash[56298]: level=error ts=2023-08-28T00:59:44.343Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:44.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:44 smithi006 bash[56298]: level=warn ts=2023-08-28T00:59:44.346Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:44 smithi130 bash[27579]: cluster 2023-08-28T00:59:43.412352+0000 mgr.y (mgr.14901) 5434 : cluster [DBG] pgmap v5441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:45.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:45 smithi130 bash[51302]: debug 2023-08-28T00:59:45.086+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:45.090279+0000) 2023-08-28T00:59:46.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:46 smithi130 bash[51302]: debug 2023-08-28T00:59:46.086+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:46.090575+0000) 2023-08-28T00:59:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:46 smithi130 bash[27579]: cluster 2023-08-28T00:59:45.413661+0000 mgr.y (mgr.14901) 5435 : cluster [DBG] pgmap v5442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:46 smithi130 bash[27579]: audit 2023-08-28T00:59:45.738360+0000 mon.a (mon.0) 3764 : audit [DBG] from='client.? 172.21.15.6:0/664452826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:46 smithi006 bash[33918]: cluster 2023-08-28T00:59:45.413661+0000 mgr.y (mgr.14901) 5435 : cluster [DBG] pgmap v5442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:46 smithi006 bash[33918]: audit 2023-08-28T00:59:45.738360+0000 mon.a (mon.0) 3764 : audit [DBG] from='client.? 172.21.15.6:0/664452826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:46 smithi006 bash[27292]: cluster 2023-08-28T00:59:45.413661+0000 mgr.y (mgr.14901) 5435 : cluster [DBG] pgmap v5442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:46 smithi006 bash[27292]: audit 2023-08-28T00:59:45.738360+0000 mon.a (mon.0) 3764 : audit [DBG] from='client.? 172.21.15.6:0/664452826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:47.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:47 smithi130 bash[51302]: debug 2023-08-28T00:59:47.090+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:47.090939+0000) 2023-08-28T00:59:48.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:48 smithi130 bash[51302]: debug 2023-08-28T00:59:48.090+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:48.091275+0000) 2023-08-28T00:59:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:48 smithi130 bash[27579]: cluster 2023-08-28T00:59:47.414493+0000 mgr.y (mgr.14901) 5436 : cluster [DBG] pgmap v5443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:48 smithi130 bash[27579]: audit 2023-08-28T00:59:48.157100+0000 mon.a (mon.0) 3765 : audit [DBG] from='client.? 172.21.15.6:0/1433349173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:48.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:48 smithi006 bash[33918]: cluster 2023-08-28T00:59:47.414493+0000 mgr.y (mgr.14901) 5436 : cluster [DBG] pgmap v5443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:48 smithi006 bash[33918]: audit 2023-08-28T00:59:48.157100+0000 mon.a (mon.0) 3765 : audit [DBG] from='client.? 172.21.15.6:0/1433349173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:48 smithi006 bash[27292]: cluster 2023-08-28T00:59:47.414493+0000 mgr.y (mgr.14901) 5436 : cluster [DBG] pgmap v5443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:48 smithi006 bash[27292]: audit 2023-08-28T00:59:48.157100+0000 mon.a (mon.0) 3765 : audit [DBG] from='client.? 172.21.15.6:0/1433349173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:49.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:49 smithi130 bash[51302]: debug 2023-08-28T00:59:49.090+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:49.091624+0000) 2023-08-28T00:59:50.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:50 smithi130 bash[51302]: debug 2023-08-28T00:59:50.090+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:50.091919+0000) 2023-08-28T00:59:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:50 smithi130 bash[27579]: cluster 2023-08-28T00:59:49.415664+0000 mgr.y (mgr.14901) 5437 : cluster [DBG] pgmap v5444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:50.892 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 00:59:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:00:59:50] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T00:59:50.892 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:50 smithi006 bash[27292]: cluster 2023-08-28T00:59:49.415664+0000 mgr.y (mgr.14901) 5437 : cluster [DBG] pgmap v5444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:50.892 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:50 smithi006 bash[33918]: cluster 2023-08-28T00:59:49.415664+0000 mgr.y (mgr.14901) 5437 : cluster [DBG] pgmap v5444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:50 smithi006 bash[56298]: level=error ts=2023-08-28T00:59:50.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:50 smithi006 bash[56298]: level=warn ts=2023-08-28T00:59:50.892Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:51.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:51 smithi130 bash[51302]: debug 2023-08-28T00:59:51.090+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:51.092198+0000) 2023-08-28T00:59:51.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:51 smithi130 bash[27579]: audit 2023-08-28T00:59:50.579595+0000 mon.c (mon.2) 2107 : audit [DBG] from='client.? 172.21.15.6:0/3860910169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:51.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:51 smithi006 bash[33918]: audit 2023-08-28T00:59:50.579595+0000 mon.c (mon.2) 2107 : audit [DBG] from='client.? 172.21.15.6:0/3860910169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:51.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:51 smithi006 bash[27292]: audit 2023-08-28T00:59:50.579595+0000 mon.c (mon.2) 2107 : audit [DBG] from='client.? 172.21.15.6:0/3860910169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:52.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:52 smithi130 bash[51302]: debug 2023-08-28T00:59:52.090+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:52.092465+0000) 2023-08-28T00:59:52.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:52 smithi006 bash[27292]: cluster 2023-08-28T00:59:51.416413+0000 mgr.y (mgr.14901) 5438 : cluster [DBG] pgmap v5445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:52 smithi006 bash[33918]: cluster 2023-08-28T00:59:51.416413+0000 mgr.y (mgr.14901) 5438 : cluster [DBG] pgmap v5445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:52 smithi006 bash[56298]: level=error ts=2023-08-28T00:59:52.464Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:52 smithi006 bash[56298]: level=warn ts=2023-08-28T00:59:52.465Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:52 smithi130 bash[27579]: cluster 2023-08-28T00:59:51.416413+0000 mgr.y (mgr.14901) 5438 : cluster [DBG] pgmap v5445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:53.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:53 smithi130 bash[51302]: debug 2023-08-28T00:59:53.090+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:53.092803+0000) 2023-08-28T00:59:53.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:53 smithi130 bash[27579]: audit 2023-08-28T00:59:52.994327+0000 mon.a (mon.0) 3766 : audit [DBG] from='client.? 172.21.15.6:0/2220872977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:53.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:53 smithi006 bash[33918]: audit 2023-08-28T00:59:52.994327+0000 mon.a (mon.0) 3766 : audit [DBG] from='client.? 172.21.15.6:0/2220872977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:53.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:53 smithi006 bash[27292]: audit 2023-08-28T00:59:52.994327+0000 mon.a (mon.0) 3766 : audit [DBG] from='client.? 172.21.15.6:0/2220872977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:54.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:54 smithi130 bash[51302]: debug 2023-08-28T00:59:54.090+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:54.093121+0000) 2023-08-28T00:59:54.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:54 smithi006 bash[27292]: cluster 2023-08-28T00:59:53.417146+0000 mgr.y (mgr.14901) 5439 : cluster [DBG] pgmap v5446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:54.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:54 smithi006 bash[33918]: cluster 2023-08-28T00:59:53.417146+0000 mgr.y (mgr.14901) 5439 : cluster [DBG] pgmap v5446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:54.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:54 smithi006 bash[56298]: level=error ts=2023-08-28T00:59:54.344Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:54.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 00:59:54 smithi006 bash[56298]: level=warn ts=2023-08-28T00:59:54.346Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T00:59:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:54 smithi130 bash[27579]: cluster 2023-08-28T00:59:53.417146+0000 mgr.y (mgr.14901) 5439 : cluster [DBG] pgmap v5446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:55.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:55 smithi130 bash[51302]: debug 2023-08-28T00:59:55.090+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:55.093443+0000) 2023-08-28T00:59:55.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:55 smithi130 bash[27579]: audit 2023-08-28T00:59:55.411624+0000 mon.a (mon.0) 3767 : audit [DBG] from='client.? 172.21.15.6:0/1404006703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:55.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:55 smithi006 bash[33918]: audit 2023-08-28T00:59:55.411624+0000 mon.a (mon.0) 3767 : audit [DBG] from='client.? 172.21.15.6:0/1404006703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:55.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:55 smithi006 bash[27292]: audit 2023-08-28T00:59:55.411624+0000 mon.a (mon.0) 3767 : audit [DBG] from='client.? 172.21.15.6:0/1404006703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:56.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:56 smithi130 bash[51302]: debug 2023-08-28T00:59:56.090+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:56.093784+0000) 2023-08-28T00:59:56.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:56 smithi130 bash[27579]: cluster 2023-08-28T00:59:55.418192+0000 mgr.y (mgr.14901) 5440 : cluster [DBG] pgmap v5447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:56 smithi006 bash[27292]: cluster 2023-08-28T00:59:55.418192+0000 mgr.y (mgr.14901) 5440 : cluster [DBG] pgmap v5447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:56 smithi006 bash[33918]: cluster 2023-08-28T00:59:55.418192+0000 mgr.y (mgr.14901) 5440 : cluster [DBG] pgmap v5447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:57.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:57 smithi130 bash[51302]: debug 2023-08-28T00:59:57.090+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:57.094125+0000) 2023-08-28T00:59:58.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:58 smithi130 bash[51302]: debug 2023-08-28T00:59:58.090+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:58.094435+0000) 2023-08-28T00:59:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:58 smithi130 bash[27579]: cluster 2023-08-28T00:59:57.418884+0000 mgr.y (mgr.14901) 5441 : cluster [DBG] pgmap v5448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 00:59:58 smithi130 bash[27579]: audit 2023-08-28T00:59:57.819980+0000 mon.a (mon.0) 3768 : audit [DBG] from='client.? 172.21.15.6:0/3271005183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:58.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:58 smithi006 bash[27292]: cluster 2023-08-28T00:59:57.418884+0000 mgr.y (mgr.14901) 5441 : cluster [DBG] pgmap v5448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 00:59:58 smithi006 bash[27292]: audit 2023-08-28T00:59:57.819980+0000 mon.a (mon.0) 3768 : audit [DBG] from='client.? 172.21.15.6:0/3271005183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:58 smithi006 bash[33918]: cluster 2023-08-28T00:59:57.418884+0000 mgr.y (mgr.14901) 5441 : cluster [DBG] pgmap v5448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T00:59:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 00:59:58 smithi006 bash[33918]: audit 2023-08-28T00:59:57.819980+0000 mon.a (mon.0) 3768 : audit [DBG] from='client.? 172.21.15.6:0/3271005183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:59.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 00:59:59 smithi130 bash[51302]: debug 2023-08-28T00:59:59.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:59.094807+0000) 2023-08-28T01:00:00.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:00 smithi130 bash[51302]: debug 2023-08-28T01:00:00.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:00.095115+0000) 2023-08-28T01:00:00.868 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 28 01:00:00 smithi130 bash[48907]: ts=2023-08-28T01:00:00.442Z caller=compact.go:519 level=info component=tsdb msg="write block" mint=1693173600138 maxt=1693180800000 ulid=01H8WTF6ZQNT4TK0BYBRXRPX2T duration=66.737907ms 2023-08-28T01:00:00.868 INFO:journalctl@ceph.prometheus.a.smithi130.stdout:Aug 28 01:00:00 smithi130 bash[48907]: ts=2023-08-28T01:00:00.446Z caller=head.go:827 level=info component=tsdb msg="Head GC completed" duration=3.902828ms 2023-08-28T01:00:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:00 smithi130 bash[27579]: cluster 2023-08-28T00:59:59.420024+0000 mgr.y (mgr.14901) 5442 : cluster [DBG] pgmap v5449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:00 smithi130 bash[27579]: cluster 2023-08-28T01:00:00.000150+0000 mon.a (mon.0) 3769 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:00:00.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:00 smithi130 bash[27579]: audit 2023-08-28T01:00:00.227159+0000 mon.c (mon.2) 2108 : audit [DBG] from='client.? 172.21.15.6:0/503427887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:00.893 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:00:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:00:00] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:00:00.893 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:00 smithi006 bash[27292]: cluster 2023-08-28T00:59:59.420024+0000 mgr.y (mgr.14901) 5442 : cluster [DBG] pgmap v5449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:00.893 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:00 smithi006 bash[27292]: cluster 2023-08-28T01:00:00.000150+0000 mon.a (mon.0) 3769 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:00:00.893 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:00 smithi006 bash[27292]: audit 2023-08-28T01:00:00.227159+0000 mon.c (mon.2) 2108 : audit [DBG] from='client.? 172.21.15.6:0/503427887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:00.894 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:00 smithi006 bash[33918]: cluster 2023-08-28T00:59:59.420024+0000 mgr.y (mgr.14901) 5442 : cluster [DBG] pgmap v5449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:00.894 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:00 smithi006 bash[33918]: cluster 2023-08-28T01:00:00.000150+0000 mon.a (mon.0) 3769 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:00:00.894 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:00 smithi006 bash[33918]: audit 2023-08-28T01:00:00.227159+0000 mon.c (mon.2) 2108 : audit [DBG] from='client.? 172.21.15.6:0/503427887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:01.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:00:00.891Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:00:00.893Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:01.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:01 smithi130 bash[51302]: debug 2023-08-28T01:00:01.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:01.095394+0000) 2023-08-28T01:00:02.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:02 smithi130 bash[51302]: debug 2023-08-28T01:00:02.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:02.095663+0000) 2023-08-28T01:00:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:02 smithi006 bash[27292]: cluster 2023-08-28T01:00:01.420692+0000 mgr.y (mgr.14901) 5443 : cluster [DBG] pgmap v5450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:02 smithi006 bash[33918]: cluster 2023-08-28T01:00:01.420692+0000 mgr.y (mgr.14901) 5443 : cluster [DBG] pgmap v5450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:00:02.465Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:02.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:00:02.467Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:02 smithi130 bash[27579]: cluster 2023-08-28T01:00:01.420692+0000 mgr.y (mgr.14901) 5443 : cluster [DBG] pgmap v5450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:03.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:03 smithi130 bash[51302]: debug 2023-08-28T01:00:03.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:03.095997+0000) 2023-08-28T01:00:03.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:03 smithi006 bash[27292]: audit 2023-08-28T01:00:02.645724+0000 mon.c (mon.2) 2109 : audit [DBG] from='client.? 172.21.15.6:0/1770450371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:03.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:03 smithi006 bash[33918]: audit 2023-08-28T01:00:02.645724+0000 mon.c (mon.2) 2109 : audit [DBG] from='client.? 172.21.15.6:0/1770450371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:03.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:03 smithi130 bash[27579]: audit 2023-08-28T01:00:02.645724+0000 mon.c (mon.2) 2109 : audit [DBG] from='client.? 172.21.15.6:0/1770450371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:04.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:04 smithi130 bash[51302]: debug 2023-08-28T01:00:04.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:04.096361+0000) 2023-08-28T01:00:04.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:04 smithi006 bash[27292]: cluster 2023-08-28T01:00:03.421314+0000 mgr.y (mgr.14901) 5444 : cluster [DBG] pgmap v5451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:04.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:04 smithi006 bash[33918]: cluster 2023-08-28T01:00:03.421314+0000 mgr.y (mgr.14901) 5444 : cluster [DBG] pgmap v5451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:04.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:00:04.345Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:04.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:00:04.347Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:04 smithi130 bash[27579]: cluster 2023-08-28T01:00:03.421314+0000 mgr.y (mgr.14901) 5444 : cluster [DBG] pgmap v5451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:05.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:05 smithi130 bash[51302]: debug 2023-08-28T01:00:05.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:05.096693+0000) 2023-08-28T01:00:05.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:05 smithi006 bash[33918]: audit 2023-08-28T01:00:05.061802+0000 mon.c (mon.2) 2110 : audit [DBG] from='client.? 172.21.15.6:0/961643021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:05.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:05 smithi006 bash[27292]: audit 2023-08-28T01:00:05.061802+0000 mon.c (mon.2) 2110 : audit [DBG] from='client.? 172.21.15.6:0/961643021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:05.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:05 smithi130 bash[27579]: audit 2023-08-28T01:00:05.061802+0000 mon.c (mon.2) 2110 : audit [DBG] from='client.? 172.21.15.6:0/961643021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:06.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:06 smithi130 bash[51302]: debug 2023-08-28T01:00:06.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:06.097023+0000) 2023-08-28T01:00:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:06 smithi130 bash[27579]: cluster 2023-08-28T01:00:05.422448+0000 mgr.y (mgr.14901) 5445 : cluster [DBG] pgmap v5452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:06.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:06 smithi006 bash[33918]: cluster 2023-08-28T01:00:05.422448+0000 mgr.y (mgr.14901) 5445 : cluster [DBG] pgmap v5452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:06 smithi006 bash[27292]: cluster 2023-08-28T01:00:05.422448+0000 mgr.y (mgr.14901) 5445 : cluster [DBG] pgmap v5452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:07.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:07 smithi130 bash[51302]: debug 2023-08-28T01:00:07.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:07.097342+0000) 2023-08-28T01:00:08.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:08 smithi130 bash[51302]: debug 2023-08-28T01:00:08.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:08.097683+0000) 2023-08-28T01:00:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:08 smithi130 bash[27579]: cluster 2023-08-28T01:00:07.423188+0000 mgr.y (mgr.14901) 5446 : cluster [DBG] pgmap v5453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:08 smithi130 bash[27579]: audit 2023-08-28T01:00:07.480439+0000 mon.c (mon.2) 2111 : audit [DBG] from='client.? 172.21.15.6:0/1738362237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:08.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:08 smithi006 bash[33918]: cluster 2023-08-28T01:00:07.423188+0000 mgr.y (mgr.14901) 5446 : cluster [DBG] pgmap v5453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:08 smithi006 bash[33918]: audit 2023-08-28T01:00:07.480439+0000 mon.c (mon.2) 2111 : audit [DBG] from='client.? 172.21.15.6:0/1738362237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:08 smithi006 bash[27292]: cluster 2023-08-28T01:00:07.423188+0000 mgr.y (mgr.14901) 5446 : cluster [DBG] pgmap v5453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:08 smithi006 bash[27292]: audit 2023-08-28T01:00:07.480439+0000 mon.c (mon.2) 2111 : audit [DBG] from='client.? 172.21.15.6:0/1738362237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:09.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:09 smithi130 bash[51302]: debug 2023-08-28T01:00:09.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:09.098021+0000) 2023-08-28T01:00:10.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:10 smithi130 bash[51302]: debug 2023-08-28T01:00:10.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:10.098353+0000) 2023-08-28T01:00:10.754 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:10 smithi006 bash[27292]: cluster 2023-08-28T01:00:09.424428+0000 mgr.y (mgr.14901) 5447 : cluster [DBG] pgmap v5454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:10.754 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:10 smithi006 bash[27292]: audit 2023-08-28T01:00:09.899755+0000 mon.c (mon.2) 2112 : audit [DBG] from='client.? 172.21.15.6:0/489432221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:10.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:10 smithi006 bash[33918]: cluster 2023-08-28T01:00:09.424428+0000 mgr.y (mgr.14901) 5447 : cluster [DBG] pgmap v5454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:10.755 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:10 smithi006 bash[33918]: audit 2023-08-28T01:00:09.899755+0000 mon.c (mon.2) 2112 : audit [DBG] from='client.? 172.21.15.6:0/489432221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:10 smithi130 bash[27579]: cluster 2023-08-28T01:00:09.424428+0000 mgr.y (mgr.14901) 5447 : cluster [DBG] pgmap v5454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:10 smithi130 bash[27579]: audit 2023-08-28T01:00:09.899755+0000 mon.c (mon.2) 2112 : audit [DBG] from='client.? 172.21.15.6:0/489432221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:00:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:00:10] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:00:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:00:10.891Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:00:10.893Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:11.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:11 smithi130 bash[51302]: debug 2023-08-28T01:00:11.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:11.098680+0000) 2023-08-28T01:00:12.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:12 smithi130 bash[51302]: debug 2023-08-28T01:00:12.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:12.098900+0000) 2023-08-28T01:00:12.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:12 smithi006 bash[33918]: cluster 2023-08-28T01:00:11.425166+0000 mgr.y (mgr.14901) 5448 : cluster [DBG] pgmap v5455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:12.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:12 smithi006 bash[33918]: audit 2023-08-28T01:00:12.314685+0000 mon.c (mon.2) 2113 : audit [DBG] from='client.? 172.21.15.6:0/3432429905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:00:12.465Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:00:12.467Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:12.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:12 smithi006 bash[27292]: cluster 2023-08-28T01:00:11.425166+0000 mgr.y (mgr.14901) 5448 : cluster [DBG] pgmap v5455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:12.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:12 smithi006 bash[27292]: audit 2023-08-28T01:00:12.314685+0000 mon.c (mon.2) 2113 : audit [DBG] from='client.? 172.21.15.6:0/3432429905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:12 smithi130 bash[27579]: cluster 2023-08-28T01:00:11.425166+0000 mgr.y (mgr.14901) 5448 : cluster [DBG] pgmap v5455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:12.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:12 smithi130 bash[27579]: audit 2023-08-28T01:00:12.314685+0000 mon.c (mon.2) 2113 : audit [DBG] from='client.? 172.21.15.6:0/3432429905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:13.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:13 smithi130 bash[51302]: debug 2023-08-28T01:00:13.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:13.099237+0000) 2023-08-28T01:00:14.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:14 smithi130 bash[51302]: debug 2023-08-28T01:00:14.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:14.099572+0000) 2023-08-28T01:00:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:00:14.346Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:14.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:00:14.347Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:14 smithi006 bash[33918]: cluster 2023-08-28T01:00:13.425806+0000 mgr.y (mgr.14901) 5449 : cluster [DBG] pgmap v5456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:14.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:14 smithi006 bash[27292]: cluster 2023-08-28T01:00:13.425806+0000 mgr.y (mgr.14901) 5449 : cluster [DBG] pgmap v5456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:14 smithi130 bash[27579]: cluster 2023-08-28T01:00:13.425806+0000 mgr.y (mgr.14901) 5449 : cluster [DBG] pgmap v5456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:15.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:15 smithi130 bash[51302]: debug 2023-08-28T01:00:15.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:15.099909+0000) 2023-08-28T01:00:15.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:15 smithi130 bash[27579]: audit 2023-08-28T01:00:14.728413+0000 mon.a (mon.0) 3770 : audit [DBG] from='client.? 172.21.15.6:0/3650926691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:15.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:15 smithi006 bash[27292]: audit 2023-08-28T01:00:14.728413+0000 mon.a (mon.0) 3770 : audit [DBG] from='client.? 172.21.15.6:0/3650926691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:15.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:15 smithi006 bash[33918]: audit 2023-08-28T01:00:14.728413+0000 mon.a (mon.0) 3770 : audit [DBG] from='client.? 172.21.15.6:0/3650926691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:16.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:16 smithi130 bash[51302]: debug 2023-08-28T01:00:16.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:16.100217+0000) 2023-08-28T01:00:16.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:16 smithi130 bash[27579]: cluster 2023-08-28T01:00:15.426967+0000 mgr.y (mgr.14901) 5450 : cluster [DBG] pgmap v5457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:16.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:16 smithi006 bash[27292]: cluster 2023-08-28T01:00:15.426967+0000 mgr.y (mgr.14901) 5450 : cluster [DBG] pgmap v5457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:16 smithi006 bash[33918]: cluster 2023-08-28T01:00:15.426967+0000 mgr.y (mgr.14901) 5450 : cluster [DBG] pgmap v5457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:17.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:17 smithi130 bash[51302]: debug 2023-08-28T01:00:17.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:17.100559+0000) 2023-08-28T01:00:17.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:17 smithi130 bash[27579]: audit 2023-08-28T01:00:17.141423+0000 mon.c (mon.2) 2114 : audit [DBG] from='client.? 172.21.15.6:0/3827537472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:17.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:17 smithi006 bash[27292]: audit 2023-08-28T01:00:17.141423+0000 mon.c (mon.2) 2114 : audit [DBG] from='client.? 172.21.15.6:0/3827537472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:17.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:17 smithi006 bash[33918]: audit 2023-08-28T01:00:17.141423+0000 mon.c (mon.2) 2114 : audit [DBG] from='client.? 172.21.15.6:0/3827537472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:18.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:18 smithi130 bash[51302]: debug 2023-08-28T01:00:18.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:18.100916+0000) 2023-08-28T01:00:18.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:18 smithi130 bash[27579]: cluster 2023-08-28T01:00:17.427766+0000 mgr.y (mgr.14901) 5451 : cluster [DBG] pgmap v5458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:18.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:18 smithi006 bash[33918]: cluster 2023-08-28T01:00:17.427766+0000 mgr.y (mgr.14901) 5451 : cluster [DBG] pgmap v5458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:18 smithi006 bash[27292]: cluster 2023-08-28T01:00:17.427766+0000 mgr.y (mgr.14901) 5451 : cluster [DBG] pgmap v5458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:19.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:19 smithi130 bash[51302]: debug 2023-08-28T01:00:19.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:19.101246+0000) 2023-08-28T01:00:20.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:20 smithi130 bash[51302]: debug 2023-08-28T01:00:20.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:20.101607+0000) 2023-08-28T01:00:20.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:20 smithi130 bash[27579]: cluster 2023-08-28T01:00:19.428544+0000 mgr.y (mgr.14901) 5452 : cluster [DBG] pgmap v5459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:20 smithi130 bash[27579]: audit 2023-08-28T01:00:19.554404+0000 mon.a (mon.0) 3771 : audit [DBG] from='client.? 172.21.15.6:0/566051280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:20.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:20 smithi130 bash[27579]: audit 2023-08-28T01:00:20.115400+0000 mon.b (mon.1) 642 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:00:20.894 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:00:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:00:20] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T01:00:20.895 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:20 smithi006 bash[33918]: cluster 2023-08-28T01:00:19.428544+0000 mgr.y (mgr.14901) 5452 : cluster [DBG] pgmap v5459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:20.895 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:20 smithi006 bash[33918]: audit 2023-08-28T01:00:19.554404+0000 mon.a (mon.0) 3771 : audit [DBG] from='client.? 172.21.15.6:0/566051280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:20.895 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:20 smithi006 bash[33918]: audit 2023-08-28T01:00:20.115400+0000 mon.b (mon.1) 642 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:00:20.895 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:20 smithi006 bash[27292]: cluster 2023-08-28T01:00:19.428544+0000 mgr.y (mgr.14901) 5452 : cluster [DBG] pgmap v5459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:20.895 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:20 smithi006 bash[27292]: audit 2023-08-28T01:00:19.554404+0000 mon.a (mon.0) 3771 : audit [DBG] from='client.? 172.21.15.6:0/566051280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:20.895 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:20 smithi006 bash[27292]: audit 2023-08-28T01:00:20.115400+0000 mon.b (mon.1) 642 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:00:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:00:20.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:00:20.894Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:21.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:21 smithi130 bash[51302]: debug 2023-08-28T01:00:21.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:21.101975+0000) 2023-08-28T01:00:22.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:22 smithi130 bash[51302]: debug 2023-08-28T01:00:22.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:22.102186+0000) 2023-08-28T01:00:22.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:00:22.466Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:00:22.468Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:22 smithi006 bash[27292]: cluster 2023-08-28T01:00:21.429341+0000 mgr.y (mgr.14901) 5453 : cluster [DBG] pgmap v5460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:22 smithi006 bash[27292]: audit 2023-08-28T01:00:21.983471+0000 mon.a (mon.0) 3772 : audit [DBG] from='client.? 172.21.15.6:0/3505701648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:22 smithi006 bash[33918]: cluster 2023-08-28T01:00:21.429341+0000 mgr.y (mgr.14901) 5453 : cluster [DBG] pgmap v5460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:22 smithi006 bash[33918]: audit 2023-08-28T01:00:21.983471+0000 mon.a (mon.0) 3772 : audit [DBG] from='client.? 172.21.15.6:0/3505701648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:22 smithi130 bash[27579]: cluster 2023-08-28T01:00:21.429341+0000 mgr.y (mgr.14901) 5453 : cluster [DBG] pgmap v5460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:22.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:22 smithi130 bash[27579]: audit 2023-08-28T01:00:21.983471+0000 mon.a (mon.0) 3772 : audit [DBG] from='client.? 172.21.15.6:0/3505701648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:23.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:23 smithi130 bash[51302]: debug 2023-08-28T01:00:23.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:23.102509+0000) 2023-08-28T01:00:24.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:24 smithi130 bash[51302]: debug 2023-08-28T01:00:24.102+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:24.102871+0000) 2023-08-28T01:00:24.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:24 smithi006 bash[27292]: cluster 2023-08-28T01:00:23.429990+0000 mgr.y (mgr.14901) 5454 : cluster [DBG] pgmap v5461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:24.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:24 smithi006 bash[27292]: audit 2023-08-28T01:00:24.404034+0000 mon.a (mon.0) 3773 : audit [DBG] from='client.? 172.21.15.6:0/1012527801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:24.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:24 smithi006 bash[33918]: cluster 2023-08-28T01:00:23.429990+0000 mgr.y (mgr.14901) 5454 : cluster [DBG] pgmap v5461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:24.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:24 smithi006 bash[33918]: audit 2023-08-28T01:00:24.404034+0000 mon.a (mon.0) 3773 : audit [DBG] from='client.? 172.21.15.6:0/1012527801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:24.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:00:24.347Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:24.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:00:24.349Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:24 smithi130 bash[27579]: cluster 2023-08-28T01:00:23.429990+0000 mgr.y (mgr.14901) 5454 : cluster [DBG] pgmap v5461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:24 smithi130 bash[27579]: audit 2023-08-28T01:00:24.404034+0000 mon.a (mon.0) 3773 : audit [DBG] from='client.? 172.21.15.6:0/1012527801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:25.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:25 smithi130 bash[51302]: debug 2023-08-28T01:00:25.102+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:25.103154+0000) 2023-08-28T01:00:26.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:26 smithi130 bash[51302]: debug 2023-08-28T01:00:26.102+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:26.103382+0000) 2023-08-28T01:00:26.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:26 smithi130 bash[27579]: cluster 2023-08-28T01:00:25.431063+0000 mgr.y (mgr.14901) 5455 : cluster [DBG] pgmap v5462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:26 smithi130 bash[27579]: audit 2023-08-28T01:00:26.077339+0000 mon.a (mon.0) 3774 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:00:26.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:26 smithi130 bash[27579]: audit 2023-08-28T01:00:26.437829+0000 mon.a (mon.0) 3775 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:00:26.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:26 smithi006 bash[33918]: cluster 2023-08-28T01:00:25.431063+0000 mgr.y (mgr.14901) 5455 : cluster [DBG] pgmap v5462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:26 smithi006 bash[33918]: audit 2023-08-28T01:00:26.077339+0000 mon.a (mon.0) 3774 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:00:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:26 smithi006 bash[33918]: audit 2023-08-28T01:00:26.437829+0000 mon.a (mon.0) 3775 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:00:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:26 smithi006 bash[27292]: cluster 2023-08-28T01:00:25.431063+0000 mgr.y (mgr.14901) 5455 : cluster [DBG] pgmap v5462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:26 smithi006 bash[27292]: audit 2023-08-28T01:00:26.077339+0000 mon.a (mon.0) 3774 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:00:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:26 smithi006 bash[27292]: audit 2023-08-28T01:00:26.437829+0000 mon.a (mon.0) 3775 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:00:27.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:27 smithi130 bash[51302]: debug 2023-08-28T01:00:27.102+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:27.103652+0000) 2023-08-28T01:00:27.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:27 smithi130 bash[27579]: audit 2023-08-28T01:00:26.735956+0000 mon.b (mon.1) 643 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:00:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:27 smithi130 bash[27579]: audit 2023-08-28T01:00:26.737904+0000 mon.b (mon.1) 644 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:00:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:27 smithi130 bash[27579]: audit 2023-08-28T01:00:26.745547+0000 mon.a (mon.0) 3776 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:00:27.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:27 smithi130 bash[27579]: audit 2023-08-28T01:00:26.813340+0000 mon.c (mon.2) 2115 : audit [DBG] from='client.? 172.21.15.6:0/1344646782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:27.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:27 smithi006 bash[27292]: audit 2023-08-28T01:00:26.735956+0000 mon.b (mon.1) 643 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:00:27.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:27 smithi006 bash[27292]: audit 2023-08-28T01:00:26.737904+0000 mon.b (mon.1) 644 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:00:27.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:27 smithi006 bash[27292]: audit 2023-08-28T01:00:26.745547+0000 mon.a (mon.0) 3776 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:00:27.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:27 smithi006 bash[27292]: audit 2023-08-28T01:00:26.813340+0000 mon.c (mon.2) 2115 : audit [DBG] from='client.? 172.21.15.6:0/1344646782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:27.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:27 smithi006 bash[33918]: audit 2023-08-28T01:00:26.735956+0000 mon.b (mon.1) 643 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:00:27.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:27 smithi006 bash[33918]: audit 2023-08-28T01:00:26.737904+0000 mon.b (mon.1) 644 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:00:27.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:27 smithi006 bash[33918]: audit 2023-08-28T01:00:26.745547+0000 mon.a (mon.0) 3776 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:00:27.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:27 smithi006 bash[33918]: audit 2023-08-28T01:00:26.813340+0000 mon.c (mon.2) 2115 : audit [DBG] from='client.? 172.21.15.6:0/1344646782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:28.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:28 smithi130 bash[51302]: debug 2023-08-28T01:00:28.102+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:28.103863+0000) 2023-08-28T01:00:29.105 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:28 smithi130 bash[27579]: cluster 2023-08-28T01:00:27.431728+0000 mgr.y (mgr.14901) 5456 : cluster [DBG] pgmap v5463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:29.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:28 smithi006 bash[27292]: cluster 2023-08-28T01:00:27.431728+0000 mgr.y (mgr.14901) 5456 : cluster [DBG] pgmap v5463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:29.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:28 smithi006 bash[33918]: cluster 2023-08-28T01:00:27.431728+0000 mgr.y (mgr.14901) 5456 : cluster [DBG] pgmap v5463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:29.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:29 smithi130 bash[51302]: debug 2023-08-28T01:00:29.102+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:29.104097+0000) 2023-08-28T01:00:30.105 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:29 smithi130 bash[27579]: audit 2023-08-28T01:00:29.230030+0000 mon.c (mon.2) 2116 : audit [DBG] from='client.? 172.21.15.6:0/2639129065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:30.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:29 smithi006 bash[27292]: audit 2023-08-28T01:00:29.230030+0000 mon.c (mon.2) 2116 : audit [DBG] from='client.? 172.21.15.6:0/2639129065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:30.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:29 smithi006 bash[33918]: audit 2023-08-28T01:00:29.230030+0000 mon.c (mon.2) 2116 : audit [DBG] from='client.? 172.21.15.6:0/2639129065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:30.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:30 smithi130 bash[51302]: debug 2023-08-28T01:00:30.102+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:30.104391+0000) 2023-08-28T01:00:31.080 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:30 smithi006 bash[27292]: cluster 2023-08-28T01:00:29.432891+0000 mgr.y (mgr.14901) 5457 : cluster [DBG] pgmap v5464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:31.080 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:00:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:00:30] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:00:31.080 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:30 smithi006 bash[33918]: cluster 2023-08-28T01:00:29.432891+0000 mgr.y (mgr.14901) 5457 : cluster [DBG] pgmap v5464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:31.081 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:00:30.893Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:31.105 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:30 smithi130 bash[27579]: cluster 2023-08-28T01:00:29.432891+0000 mgr.y (mgr.14901) 5457 : cluster [DBG] pgmap v5464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:31.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:31 smithi130 bash[51302]: debug 2023-08-28T01:00:31.102+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:31.104673+0000) 2023-08-28T01:00:31.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:00:30.895Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:32.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:32 smithi130 bash[51302]: debug 2023-08-28T01:00:32.101+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:32.104901+0000) 2023-08-28T01:00:32.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:32 smithi006 bash[33918]: cluster 2023-08-28T01:00:31.433665+0000 mgr.y (mgr.14901) 5458 : cluster [DBG] pgmap v5465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:32.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:32 smithi006 bash[33918]: audit 2023-08-28T01:00:31.665780+0000 mon.c (mon.2) 2117 : audit [DBG] from='client.? 172.21.15.6:0/4270198794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:00:32.467Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:00:32.469Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:32 smithi006 bash[27292]: cluster 2023-08-28T01:00:31.433665+0000 mgr.y (mgr.14901) 5458 : cluster [DBG] pgmap v5465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:32 smithi006 bash[27292]: audit 2023-08-28T01:00:31.665780+0000 mon.c (mon.2) 2117 : audit [DBG] from='client.? 172.21.15.6:0/4270198794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:32 smithi130 bash[27579]: cluster 2023-08-28T01:00:31.433665+0000 mgr.y (mgr.14901) 5458 : cluster [DBG] pgmap v5465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:32 smithi130 bash[27579]: audit 2023-08-28T01:00:31.665780+0000 mon.c (mon.2) 2117 : audit [DBG] from='client.? 172.21.15.6:0/4270198794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:33.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:33 smithi130 bash[51302]: debug 2023-08-28T01:00:33.101+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:33.105189+0000) 2023-08-28T01:00:34.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:34 smithi130 bash[51302]: debug 2023-08-28T01:00:34.101+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:34.105458+0000) 2023-08-28T01:00:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:34 smithi006 bash[27292]: cluster 2023-08-28T01:00:33.434360+0000 mgr.y (mgr.14901) 5459 : cluster [DBG] pgmap v5466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:34.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:34 smithi006 bash[27292]: audit 2023-08-28T01:00:34.070803+0000 mon.a (mon.0) 3777 : audit [DBG] from='client.? 172.21.15.6:0/872669267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:34 smithi006 bash[33918]: cluster 2023-08-28T01:00:33.434360+0000 mgr.y (mgr.14901) 5459 : cluster [DBG] pgmap v5466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:34 smithi006 bash[33918]: audit 2023-08-28T01:00:34.070803+0000 mon.a (mon.0) 3777 : audit [DBG] from='client.? 172.21.15.6:0/872669267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:34.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:00:34.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:34.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:00:34.350Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:34.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:34 smithi130 bash[27579]: cluster 2023-08-28T01:00:33.434360+0000 mgr.y (mgr.14901) 5459 : cluster [DBG] pgmap v5466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:34 smithi130 bash[27579]: audit 2023-08-28T01:00:34.070803+0000 mon.a (mon.0) 3777 : audit [DBG] from='client.? 172.21.15.6:0/872669267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:35.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:35 smithi130 bash[51302]: debug 2023-08-28T01:00:35.101+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:35.105750+0000) 2023-08-28T01:00:36.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:36 smithi130 bash[51302]: debug 2023-08-28T01:00:36.101+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:36.106021+0000) 2023-08-28T01:00:36.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:36 smithi130 bash[27579]: cluster 2023-08-28T01:00:35.435480+0000 mgr.y (mgr.14901) 5460 : cluster [DBG] pgmap v5467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:36.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:36 smithi130 bash[27579]: audit 2023-08-28T01:00:36.483212+0000 mon.a (mon.0) 3778 : audit [DBG] from='client.? 172.21.15.6:0/2425688665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:36.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:36 smithi006 bash[33918]: cluster 2023-08-28T01:00:35.435480+0000 mgr.y (mgr.14901) 5460 : cluster [DBG] pgmap v5467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:36.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:36 smithi006 bash[33918]: audit 2023-08-28T01:00:36.483212+0000 mon.a (mon.0) 3778 : audit [DBG] from='client.? 172.21.15.6:0/2425688665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:36 smithi006 bash[27292]: cluster 2023-08-28T01:00:35.435480+0000 mgr.y (mgr.14901) 5460 : cluster [DBG] pgmap v5467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:36 smithi006 bash[27292]: audit 2023-08-28T01:00:36.483212+0000 mon.a (mon.0) 3778 : audit [DBG] from='client.? 172.21.15.6:0/2425688665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:37.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:37 smithi130 bash[51302]: debug 2023-08-28T01:00:37.101+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:37.106293+0000) 2023-08-28T01:00:38.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:38 smithi130 bash[51302]: debug 2023-08-28T01:00:38.101+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:38.106607+0000) 2023-08-28T01:00:38.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:38 smithi130 bash[27579]: cluster 2023-08-28T01:00:37.436257+0000 mgr.y (mgr.14901) 5461 : cluster [DBG] pgmap v5468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:38.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:38 smithi006 bash[27292]: cluster 2023-08-28T01:00:37.436257+0000 mgr.y (mgr.14901) 5461 : cluster [DBG] pgmap v5468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:38 smithi006 bash[33918]: cluster 2023-08-28T01:00:37.436257+0000 mgr.y (mgr.14901) 5461 : cluster [DBG] pgmap v5468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:39.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:39 smithi130 bash[51302]: debug 2023-08-28T01:00:39.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:39.106968+0000) 2023-08-28T01:00:39.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:39 smithi130 bash[27579]: audit 2023-08-28T01:00:38.904720+0000 mon.a (mon.0) 3779 : audit [DBG] from='client.? 172.21.15.6:0/3297853408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:39.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:39 smithi006 bash[27292]: audit 2023-08-28T01:00:38.904720+0000 mon.a (mon.0) 3779 : audit [DBG] from='client.? 172.21.15.6:0/3297853408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:39 smithi006 bash[33918]: audit 2023-08-28T01:00:38.904720+0000 mon.a (mon.0) 3779 : audit [DBG] from='client.? 172.21.15.6:0/3297853408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:40.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:40 smithi130 bash[51302]: debug 2023-08-28T01:00:40.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:40.107233+0000) 2023-08-28T01:00:40.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:40 smithi130 bash[27579]: cluster 2023-08-28T01:00:39.437424+0000 mgr.y (mgr.14901) 5462 : cluster [DBG] pgmap v5469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:40.895 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:00:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:00:40] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:00:40.896 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:40 smithi006 bash[33918]: cluster 2023-08-28T01:00:39.437424+0000 mgr.y (mgr.14901) 5462 : cluster [DBG] pgmap v5469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:40.896 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:40 smithi006 bash[27292]: cluster 2023-08-28T01:00:39.437424+0000 mgr.y (mgr.14901) 5462 : cluster [DBG] pgmap v5469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:41.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:00:40.893Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:41.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:00:40.895Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:41.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:41 smithi130 bash[51302]: debug 2023-08-28T01:00:41.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:41.107541+0000) 2023-08-28T01:00:41.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:41 smithi130 bash[27579]: audit 2023-08-28T01:00:41.315381+0000 mon.a (mon.0) 3780 : audit [DBG] from='client.? 172.21.15.6:0/569454954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:41.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:41 smithi006 bash[33918]: audit 2023-08-28T01:00:41.315381+0000 mon.a (mon.0) 3780 : audit [DBG] from='client.? 172.21.15.6:0/569454954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:41.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:41 smithi006 bash[27292]: audit 2023-08-28T01:00:41.315381+0000 mon.a (mon.0) 3780 : audit [DBG] from='client.? 172.21.15.6:0/569454954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:42.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:42 smithi130 bash[51302]: debug 2023-08-28T01:00:42.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:42.107751+0000) 2023-08-28T01:00:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:42 smithi130 bash[27579]: cluster 2023-08-28T01:00:41.438048+0000 mgr.y (mgr.14901) 5463 : cluster [DBG] pgmap v5470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:42 smithi006 bash[27292]: cluster 2023-08-28T01:00:41.438048+0000 mgr.y (mgr.14901) 5463 : cluster [DBG] pgmap v5470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:42 smithi006 bash[33918]: cluster 2023-08-28T01:00:41.438048+0000 mgr.y (mgr.14901) 5463 : cluster [DBG] pgmap v5470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:42.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:00:42.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:42.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:00:42.470Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:43.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:43 smithi130 bash[51302]: debug 2023-08-28T01:00:43.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:43.108064+0000) 2023-08-28T01:00:44.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:44 smithi130 bash[51302]: debug 2023-08-28T01:00:44.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:44.108330+0000) 2023-08-28T01:00:44.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:44 smithi006 bash[27292]: cluster 2023-08-28T01:00:43.438641+0000 mgr.y (mgr.14901) 5464 : cluster [DBG] pgmap v5471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:44.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:44 smithi006 bash[27292]: audit 2023-08-28T01:00:43.751269+0000 mon.a (mon.0) 3781 : audit [DBG] from='client.? 172.21.15.6:0/3255033587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:44.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:44 smithi006 bash[33918]: cluster 2023-08-28T01:00:43.438641+0000 mgr.y (mgr.14901) 5464 : cluster [DBG] pgmap v5471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:44.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:44 smithi006 bash[33918]: audit 2023-08-28T01:00:43.751269+0000 mon.a (mon.0) 3781 : audit [DBG] from='client.? 172.21.15.6:0/3255033587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:44.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:00:44.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:00:44.351Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:44 smithi130 bash[27579]: cluster 2023-08-28T01:00:43.438641+0000 mgr.y (mgr.14901) 5464 : cluster [DBG] pgmap v5471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:44.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:44 smithi130 bash[27579]: audit 2023-08-28T01:00:43.751269+0000 mon.a (mon.0) 3781 : audit [DBG] from='client.? 172.21.15.6:0/3255033587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:45.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:45 smithi130 bash[51302]: debug 2023-08-28T01:00:45.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:45.108585+0000) 2023-08-28T01:00:46.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:46 smithi130 bash[51302]: debug 2023-08-28T01:00:46.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:46.108870+0000) 2023-08-28T01:00:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:46 smithi130 bash[27579]: cluster 2023-08-28T01:00:45.439753+0000 mgr.y (mgr.14901) 5465 : cluster [DBG] pgmap v5472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:46 smithi130 bash[27579]: audit 2023-08-28T01:00:46.170648+0000 mon.a (mon.0) 3782 : audit [DBG] from='client.? 172.21.15.6:0/2423387733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:46.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:46 smithi006 bash[27292]: cluster 2023-08-28T01:00:45.439753+0000 mgr.y (mgr.14901) 5465 : cluster [DBG] pgmap v5472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:46 smithi006 bash[27292]: audit 2023-08-28T01:00:46.170648+0000 mon.a (mon.0) 3782 : audit [DBG] from='client.? 172.21.15.6:0/2423387733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:46 smithi006 bash[33918]: cluster 2023-08-28T01:00:45.439753+0000 mgr.y (mgr.14901) 5465 : cluster [DBG] pgmap v5472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:46 smithi006 bash[33918]: audit 2023-08-28T01:00:46.170648+0000 mon.a (mon.0) 3782 : audit [DBG] from='client.? 172.21.15.6:0/2423387733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:47.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:47 smithi130 bash[51302]: debug 2023-08-28T01:00:47.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:47.109128+0000) 2023-08-28T01:00:48.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:48 smithi130 bash[51302]: debug 2023-08-28T01:00:48.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:48.109426+0000) 2023-08-28T01:00:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:48 smithi130 bash[27579]: cluster 2023-08-28T01:00:47.440394+0000 mgr.y (mgr.14901) 5466 : cluster [DBG] pgmap v5473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:48.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:48 smithi006 bash[33918]: cluster 2023-08-28T01:00:47.440394+0000 mgr.y (mgr.14901) 5466 : cluster [DBG] pgmap v5473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:48 smithi006 bash[27292]: cluster 2023-08-28T01:00:47.440394+0000 mgr.y (mgr.14901) 5466 : cluster [DBG] pgmap v5473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:49.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:49 smithi130 bash[51302]: debug 2023-08-28T01:00:49.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:49.109696+0000) 2023-08-28T01:00:49.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:49 smithi130 bash[27579]: audit 2023-08-28T01:00:48.585837+0000 mon.a (mon.0) 3783 : audit [DBG] from='client.? 172.21.15.6:0/4271270759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:49.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:49 smithi006 bash[33918]: audit 2023-08-28T01:00:48.585837+0000 mon.a (mon.0) 3783 : audit [DBG] from='client.? 172.21.15.6:0/4271270759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:49.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:49 smithi006 bash[27292]: audit 2023-08-28T01:00:48.585837+0000 mon.a (mon.0) 3783 : audit [DBG] from='client.? 172.21.15.6:0/4271270759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:50.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:50 smithi130 bash[51302]: debug 2023-08-28T01:00:50.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:50.109980+0000) 2023-08-28T01:00:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:50 smithi130 bash[27579]: cluster 2023-08-28T01:00:49.441635+0000 mgr.y (mgr.14901) 5467 : cluster [DBG] pgmap v5474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:50.896 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:50 smithi006 bash[33918]: cluster 2023-08-28T01:00:49.441635+0000 mgr.y (mgr.14901) 5467 : cluster [DBG] pgmap v5474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:50.897 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:00:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:00:50] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:00:50.897 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:50 smithi006 bash[27292]: cluster 2023-08-28T01:00:49.441635+0000 mgr.y (mgr.14901) 5467 : cluster [DBG] pgmap v5474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:00:50.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:00:50.896Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:51.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:51 smithi130 bash[51302]: debug 2023-08-28T01:00:51.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:51.110343+0000) 2023-08-28T01:00:51.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:51 smithi130 bash[27579]: audit 2023-08-28T01:00:51.003240+0000 mon.a (mon.0) 3784 : audit [DBG] from='client.? 172.21.15.6:0/2279837598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:51.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:51 smithi006 bash[27292]: audit 2023-08-28T01:00:51.003240+0000 mon.a (mon.0) 3784 : audit [DBG] from='client.? 172.21.15.6:0/2279837598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:51.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:51 smithi006 bash[33918]: audit 2023-08-28T01:00:51.003240+0000 mon.a (mon.0) 3784 : audit [DBG] from='client.? 172.21.15.6:0/2279837598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:52.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:52 smithi130 bash[51302]: debug 2023-08-28T01:00:52.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:52.110682+0000) 2023-08-28T01:00:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:52 smithi130 bash[27579]: cluster 2023-08-28T01:00:51.442286+0000 mgr.y (mgr.14901) 5468 : cluster [DBG] pgmap v5475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:52 smithi006 bash[27292]: cluster 2023-08-28T01:00:51.442286+0000 mgr.y (mgr.14901) 5468 : cluster [DBG] pgmap v5475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:52 smithi006 bash[33918]: cluster 2023-08-28T01:00:51.442286+0000 mgr.y (mgr.14901) 5468 : cluster [DBG] pgmap v5475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:52.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:00:52.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:52.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:00:52.470Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:53.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:53 smithi130 bash[51302]: debug 2023-08-28T01:00:53.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:53.111025+0000) 2023-08-28T01:00:53.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:53 smithi130 bash[27579]: audit 2023-08-28T01:00:53.422379+0000 mon.c (mon.2) 2118 : audit [DBG] from='client.? 172.21.15.6:0/1744715539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:53.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:53 smithi006 bash[27292]: audit 2023-08-28T01:00:53.422379+0000 mon.c (mon.2) 2118 : audit [DBG] from='client.? 172.21.15.6:0/1744715539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:53.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:53 smithi006 bash[33918]: audit 2023-08-28T01:00:53.422379+0000 mon.c (mon.2) 2118 : audit [DBG] from='client.? 172.21.15.6:0/1744715539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:54.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:54 smithi130 bash[51302]: debug 2023-08-28T01:00:54.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:54.111375+0000) 2023-08-28T01:00:54.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:54 smithi006 bash[33918]: cluster 2023-08-28T01:00:53.442958+0000 mgr.y (mgr.14901) 5469 : cluster [DBG] pgmap v5476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:54 smithi006 bash[27292]: cluster 2023-08-28T01:00:53.442958+0000 mgr.y (mgr.14901) 5469 : cluster [DBG] pgmap v5476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:54.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:00:54.349Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:54.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:00:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:00:54.351Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:00:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:54 smithi130 bash[27579]: cluster 2023-08-28T01:00:53.442958+0000 mgr.y (mgr.14901) 5469 : cluster [DBG] pgmap v5476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:55.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:55 smithi130 bash[51302]: debug 2023-08-28T01:00:55.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:55.111733+0000) 2023-08-28T01:00:56.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:56 smithi130 bash[51302]: debug 2023-08-28T01:00:56.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:56.112042+0000) 2023-08-28T01:00:56.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:56 smithi130 bash[27579]: cluster 2023-08-28T01:00:55.444154+0000 mgr.y (mgr.14901) 5470 : cluster [DBG] pgmap v5477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:56.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:56 smithi130 bash[27579]: audit 2023-08-28T01:00:55.838383+0000 mon.a (mon.0) 3785 : audit [DBG] from='client.? 172.21.15.6:0/3120941261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:56.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:56 smithi006 bash[33918]: cluster 2023-08-28T01:00:55.444154+0000 mgr.y (mgr.14901) 5470 : cluster [DBG] pgmap v5477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:56 smithi006 bash[33918]: audit 2023-08-28T01:00:55.838383+0000 mon.a (mon.0) 3785 : audit [DBG] from='client.? 172.21.15.6:0/3120941261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:56 smithi006 bash[27292]: cluster 2023-08-28T01:00:55.444154+0000 mgr.y (mgr.14901) 5470 : cluster [DBG] pgmap v5477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:56 smithi006 bash[27292]: audit 2023-08-28T01:00:55.838383+0000 mon.a (mon.0) 3785 : audit [DBG] from='client.? 172.21.15.6:0/3120941261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:57.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:57 smithi130 bash[51302]: debug 2023-08-28T01:00:57.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:57.112346+0000) 2023-08-28T01:00:58.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:58 smithi130 bash[51302]: debug 2023-08-28T01:00:58.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:58.112662+0000) 2023-08-28T01:00:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:58 smithi130 bash[27579]: cluster 2023-08-28T01:00:57.444972+0000 mgr.y (mgr.14901) 5471 : cluster [DBG] pgmap v5478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:58.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:00:58 smithi130 bash[27579]: audit 2023-08-28T01:00:58.255761+0000 mon.c (mon.2) 2119 : audit [DBG] from='client.? 172.21.15.6:0/1507543717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:58.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:58 smithi006 bash[27292]: cluster 2023-08-28T01:00:57.444972+0000 mgr.y (mgr.14901) 5471 : cluster [DBG] pgmap v5478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:58.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:00:58 smithi006 bash[27292]: audit 2023-08-28T01:00:58.255761+0000 mon.c (mon.2) 2119 : audit [DBG] from='client.? 172.21.15.6:0/1507543717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:58 smithi006 bash[33918]: cluster 2023-08-28T01:00:57.444972+0000 mgr.y (mgr.14901) 5471 : cluster [DBG] pgmap v5478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:00:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:00:58 smithi006 bash[33918]: audit 2023-08-28T01:00:58.255761+0000 mon.c (mon.2) 2119 : audit [DBG] from='client.? 172.21.15.6:0/1507543717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:59.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:00:59 smithi130 bash[51302]: debug 2023-08-28T01:00:59.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:59.112985+0000) 2023-08-28T01:01:00.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:00 smithi130 bash[51302]: debug 2023-08-28T01:01:00.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:00.113365+0000) 2023-08-28T01:01:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:00 smithi130 bash[27579]: cluster 2023-08-28T01:00:59.446109+0000 mgr.y (mgr.14901) 5472 : cluster [DBG] pgmap v5479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:00.897 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:00 smithi006 bash[33918]: cluster 2023-08-28T01:00:59.446109+0000 mgr.y (mgr.14901) 5472 : cluster [DBG] pgmap v5479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:00.898 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:00 smithi006 bash[27292]: cluster 2023-08-28T01:00:59.446109+0000 mgr.y (mgr.14901) 5472 : cluster [DBG] pgmap v5479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:00.898 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:01:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:01:00] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:01:01.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:01:00.895Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:01:00.897Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:01.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:01 smithi130 bash[51302]: debug 2023-08-28T01:01:01.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:01.113723+0000) 2023-08-28T01:01:01.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:01 smithi130 bash[27579]: audit 2023-08-28T01:01:00.683808+0000 mon.a (mon.0) 3786 : audit [DBG] from='client.? 172.21.15.6:0/2714498005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:01.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:01 smithi006 bash[27292]: audit 2023-08-28T01:01:00.683808+0000 mon.a (mon.0) 3786 : audit [DBG] from='client.? 172.21.15.6:0/2714498005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:01.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:01 smithi006 bash[33918]: audit 2023-08-28T01:01:00.683808+0000 mon.a (mon.0) 3786 : audit [DBG] from='client.? 172.21.15.6:0/2714498005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:02.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:02 smithi130 bash[51302]: debug 2023-08-28T01:01:02.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:02.113960+0000) 2023-08-28T01:01:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:02 smithi130 bash[27579]: cluster 2023-08-28T01:01:01.446784+0000 mgr.y (mgr.14901) 5473 : cluster [DBG] pgmap v5480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:02.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:01:02.469Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:02.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:01:02.471Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:02 smithi006 bash[27292]: cluster 2023-08-28T01:01:01.446784+0000 mgr.y (mgr.14901) 5473 : cluster [DBG] pgmap v5480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:02 smithi006 bash[33918]: cluster 2023-08-28T01:01:01.446784+0000 mgr.y (mgr.14901) 5473 : cluster [DBG] pgmap v5480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:03.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:03 smithi130 bash[51302]: debug 2023-08-28T01:01:03.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:03.114262+0000) 2023-08-28T01:01:03.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:03 smithi130 bash[27579]: audit 2023-08-28T01:01:03.106152+0000 mon.a (mon.0) 3787 : audit [DBG] from='client.? 172.21.15.6:0/551764214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:03.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:03 smithi006 bash[27292]: audit 2023-08-28T01:01:03.106152+0000 mon.a (mon.0) 3787 : audit [DBG] from='client.? 172.21.15.6:0/551764214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:03.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:03 smithi006 bash[33918]: audit 2023-08-28T01:01:03.106152+0000 mon.a (mon.0) 3787 : audit [DBG] from='client.? 172.21.15.6:0/551764214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:04.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:04 smithi130 bash[51302]: debug 2023-08-28T01:01:04.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:04.114493+0000) 2023-08-28T01:01:04.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:04 smithi006 bash[33918]: cluster 2023-08-28T01:01:03.447466+0000 mgr.y (mgr.14901) 5474 : cluster [DBG] pgmap v5481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:04.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:04 smithi006 bash[27292]: cluster 2023-08-28T01:01:03.447466+0000 mgr.y (mgr.14901) 5474 : cluster [DBG] pgmap v5481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:04.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:01:04.350Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:04.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:01:04.352Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:04 smithi130 bash[27579]: cluster 2023-08-28T01:01:03.447466+0000 mgr.y (mgr.14901) 5474 : cluster [DBG] pgmap v5481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:05.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:05 smithi130 bash[51302]: debug 2023-08-28T01:01:05.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:05.114787+0000) 2023-08-28T01:01:05.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:05 smithi130 bash[27579]: audit 2023-08-28T01:01:05.519952+0000 mon.c (mon.2) 2120 : audit [DBG] from='client.? 172.21.15.6:0/623586110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:05.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:05 smithi006 bash[27292]: audit 2023-08-28T01:01:05.519952+0000 mon.c (mon.2) 2120 : audit [DBG] from='client.? 172.21.15.6:0/623586110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:05.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:05 smithi006 bash[33918]: audit 2023-08-28T01:01:05.519952+0000 mon.c (mon.2) 2120 : audit [DBG] from='client.? 172.21.15.6:0/623586110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:06.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:06 smithi130 bash[51302]: debug 2023-08-28T01:01:06.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:06.115078+0000) 2023-08-28T01:01:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:06 smithi130 bash[27579]: cluster 2023-08-28T01:01:05.448578+0000 mgr.y (mgr.14901) 5475 : cluster [DBG] pgmap v5482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:06.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:06 smithi006 bash[33918]: cluster 2023-08-28T01:01:05.448578+0000 mgr.y (mgr.14901) 5475 : cluster [DBG] pgmap v5482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:06 smithi006 bash[27292]: cluster 2023-08-28T01:01:05.448578+0000 mgr.y (mgr.14901) 5475 : cluster [DBG] pgmap v5482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:07.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:07 smithi130 bash[51302]: debug 2023-08-28T01:01:07.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:07.115375+0000) 2023-08-28T01:01:08.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:08 smithi130 bash[51302]: debug 2023-08-28T01:01:08.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:08.115680+0000) 2023-08-28T01:01:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:08 smithi130 bash[27579]: cluster 2023-08-28T01:01:07.449268+0000 mgr.y (mgr.14901) 5476 : cluster [DBG] pgmap v5483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:08 smithi130 bash[27579]: audit 2023-08-28T01:01:07.940278+0000 mon.c (mon.2) 2121 : audit [DBG] from='client.? 172.21.15.6:0/681099445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:08.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:08 smithi006 bash[33918]: cluster 2023-08-28T01:01:07.449268+0000 mgr.y (mgr.14901) 5476 : cluster [DBG] pgmap v5483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:08 smithi006 bash[33918]: audit 2023-08-28T01:01:07.940278+0000 mon.c (mon.2) 2121 : audit [DBG] from='client.? 172.21.15.6:0/681099445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:08 smithi006 bash[27292]: cluster 2023-08-28T01:01:07.449268+0000 mgr.y (mgr.14901) 5476 : cluster [DBG] pgmap v5483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:08 smithi006 bash[27292]: audit 2023-08-28T01:01:07.940278+0000 mon.c (mon.2) 2121 : audit [DBG] from='client.? 172.21.15.6:0/681099445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:09.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:09 smithi130 bash[51302]: debug 2023-08-28T01:01:09.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:09.115954+0000) 2023-08-28T01:01:10.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:10 smithi130 bash[51302]: debug 2023-08-28T01:01:10.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:10.116258+0000) 2023-08-28T01:01:10.899 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:01:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:01:10] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:01:10.899 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:10 smithi006 bash[33918]: cluster 2023-08-28T01:01:09.450359+0000 mgr.y (mgr.14901) 5477 : cluster [DBG] pgmap v5484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:10.899 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:10 smithi006 bash[33918]: audit 2023-08-28T01:01:10.361732+0000 mon.c (mon.2) 2122 : audit [DBG] from='client.? 172.21.15.6:0/386109791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:10.899 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:10 smithi006 bash[27292]: cluster 2023-08-28T01:01:09.450359+0000 mgr.y (mgr.14901) 5477 : cluster [DBG] pgmap v5484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:10.899 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:10 smithi006 bash[27292]: audit 2023-08-28T01:01:10.361732+0000 mon.c (mon.2) 2122 : audit [DBG] from='client.? 172.21.15.6:0/386109791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:10 smithi130 bash[27579]: cluster 2023-08-28T01:01:09.450359+0000 mgr.y (mgr.14901) 5477 : cluster [DBG] pgmap v5484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:10 smithi130 bash[27579]: audit 2023-08-28T01:01:10.361732+0000 mon.c (mon.2) 2122 : audit [DBG] from='client.? 172.21.15.6:0/386109791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:11.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:01:10.896Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:01:10.898Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:11.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:11 smithi130 bash[51302]: debug 2023-08-28T01:01:11.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:11.116556+0000) 2023-08-28T01:01:12.427 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:12 smithi130 bash[51302]: debug 2023-08-28T01:01:12.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:12.116803+0000) 2023-08-28T01:01:12.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:01:12.470Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:12.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:01:12.472Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:12.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:12 smithi006 bash[33918]: cluster 2023-08-28T01:01:11.450972+0000 mgr.y (mgr.14901) 5478 : cluster [DBG] pgmap v5485: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:12.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:12 smithi006 bash[27292]: cluster 2023-08-28T01:01:11.450972+0000 mgr.y (mgr.14901) 5478 : cluster [DBG] pgmap v5485: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:12 smithi130 bash[27579]: cluster 2023-08-28T01:01:11.450972+0000 mgr.y (mgr.14901) 5478 : cluster [DBG] pgmap v5485: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:13.430 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:13 smithi130 bash[51302]: debug 2023-08-28T01:01:13.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:13.117096+0000) 2023-08-28T01:01:13.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:13 smithi006 bash[33918]: audit 2023-08-28T01:01:12.782721+0000 mon.c (mon.2) 2123 : audit [DBG] from='client.? 172.21.15.6:0/2325014919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:13.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:13 smithi006 bash[27292]: audit 2023-08-28T01:01:12.782721+0000 mon.c (mon.2) 2123 : audit [DBG] from='client.? 172.21.15.6:0/2325014919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:13.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:13 smithi130 bash[27579]: audit 2023-08-28T01:01:12.782721+0000 mon.c (mon.2) 2123 : audit [DBG] from='client.? 172.21.15.6:0/2325014919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:14.439 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:14 smithi130 bash[51302]: debug 2023-08-28T01:01:14.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:14.117399+0000) 2023-08-28T01:01:14.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:14 smithi006 bash[27292]: cluster 2023-08-28T01:01:13.451621+0000 mgr.y (mgr.14901) 5479 : cluster [DBG] pgmap v5486: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:14 smithi006 bash[33918]: cluster 2023-08-28T01:01:13.451621+0000 mgr.y (mgr.14901) 5479 : cluster [DBG] pgmap v5486: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:14.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:01:14.351Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:14.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:01:14.353Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:14 smithi130 bash[27579]: cluster 2023-08-28T01:01:13.451621+0000 mgr.y (mgr.14901) 5479 : cluster [DBG] pgmap v5486: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:15.441 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:15 smithi130 bash[51302]: debug 2023-08-28T01:01:15.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:15.117664+0000) 2023-08-28T01:01:15.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:15 smithi006 bash[27292]: audit 2023-08-28T01:01:15.185891+0000 mon.c (mon.2) 2124 : audit [DBG] from='client.? 172.21.15.6:0/978527811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:15.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:15 smithi006 bash[33918]: audit 2023-08-28T01:01:15.185891+0000 mon.c (mon.2) 2124 : audit [DBG] from='client.? 172.21.15.6:0/978527811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:15.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:15 smithi130 bash[27579]: audit 2023-08-28T01:01:15.185891+0000 mon.c (mon.2) 2124 : audit [DBG] from='client.? 172.21.15.6:0/978527811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:16.448 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:16 smithi130 bash[51302]: debug 2023-08-28T01:01:16.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:16.117960+0000) 2023-08-28T01:01:16.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:16 smithi006 bash[27292]: cluster 2023-08-28T01:01:15.452484+0000 mgr.y (mgr.14901) 5480 : cluster [DBG] pgmap v5487: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:16.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:16 smithi006 bash[33918]: cluster 2023-08-28T01:01:15.452484+0000 mgr.y (mgr.14901) 5480 : cluster [DBG] pgmap v5487: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:16.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:16 smithi130 bash[27579]: cluster 2023-08-28T01:01:15.452484+0000 mgr.y (mgr.14901) 5480 : cluster [DBG] pgmap v5487: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:17.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:17 smithi130 bash[51302]: debug 2023-08-28T01:01:17.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:17.118226+0000) 2023-08-28T01:01:18.515 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:18 smithi130 bash[51302]: debug 2023-08-28T01:01:18.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:18.118531+0000) 2023-08-28T01:01:18.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:18 smithi130 bash[27579]: cluster 2023-08-28T01:01:17.453140+0000 mgr.y (mgr.14901) 5481 : cluster [DBG] pgmap v5488: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:18.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:18 smithi130 bash[27579]: audit 2023-08-28T01:01:17.597358+0000 mon.a (mon.0) 3788 : audit [DBG] from='client.? 172.21.15.6:0/274790783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:18.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:18 smithi006 bash[27292]: cluster 2023-08-28T01:01:17.453140+0000 mgr.y (mgr.14901) 5481 : cluster [DBG] pgmap v5488: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:18.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:18 smithi006 bash[27292]: audit 2023-08-28T01:01:17.597358+0000 mon.a (mon.0) 3788 : audit [DBG] from='client.? 172.21.15.6:0/274790783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:18 smithi006 bash[33918]: cluster 2023-08-28T01:01:17.453140+0000 mgr.y (mgr.14901) 5481 : cluster [DBG] pgmap v5488: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:18 smithi006 bash[33918]: audit 2023-08-28T01:01:17.597358+0000 mon.a (mon.0) 3788 : audit [DBG] from='client.? 172.21.15.6:0/274790783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:19.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:19 smithi130 bash[51302]: debug 2023-08-28T01:01:19.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:19.118830+0000) 2023-08-28T01:01:20.520 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:20 smithi130 bash[51302]: debug 2023-08-28T01:01:20.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:20.119199+0000) 2023-08-28T01:01:20.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:20 smithi130 bash[27579]: cluster 2023-08-28T01:01:19.454162+0000 mgr.y (mgr.14901) 5482 : cluster [DBG] pgmap v5489: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:20.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:20 smithi130 bash[27579]: audit 2023-08-28T01:01:20.011972+0000 mon.c (mon.2) 2125 : audit [DBG] from='client.? 172.21.15.6:0/504188270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:20.899 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:01:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:01:20] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:01:20.900 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:20 smithi006 bash[27292]: cluster 2023-08-28T01:01:19.454162+0000 mgr.y (mgr.14901) 5482 : cluster [DBG] pgmap v5489: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:20.900 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:20 smithi006 bash[27292]: audit 2023-08-28T01:01:20.011972+0000 mon.c (mon.2) 2125 : audit [DBG] from='client.? 172.21.15.6:0/504188270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:20.900 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:20 smithi006 bash[33918]: cluster 2023-08-28T01:01:19.454162+0000 mgr.y (mgr.14901) 5482 : cluster [DBG] pgmap v5489: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:20.900 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:20 smithi006 bash[33918]: audit 2023-08-28T01:01:20.011972+0000 mon.c (mon.2) 2125 : audit [DBG] from='client.? 172.21.15.6:0/504188270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:21.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:01:20.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:21.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:01:20.899Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:21.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:21 smithi130 bash[51302]: debug 2023-08-28T01:01:21.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:21.119474+0000) 2023-08-28T01:01:22.430 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:22 smithi130 bash[51302]: debug 2023-08-28T01:01:22.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:22.119694+0000) 2023-08-28T01:01:22.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:22 smithi006 bash[33918]: cluster 2023-08-28T01:01:21.454810+0000 mgr.y (mgr.14901) 5483 : cluster [DBG] pgmap v5490: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:22.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:22 smithi006 bash[27292]: cluster 2023-08-28T01:01:21.454810+0000 mgr.y (mgr.14901) 5483 : cluster [DBG] pgmap v5490: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:01:22.471Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:01:22.474Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:22 smithi130 bash[27579]: cluster 2023-08-28T01:01:21.454810+0000 mgr.y (mgr.14901) 5483 : cluster [DBG] pgmap v5490: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:23.437 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:23 smithi130 bash[51302]: debug 2023-08-28T01:01:23.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:23.119965+0000) 2023-08-28T01:01:23.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:23 smithi006 bash[33918]: audit 2023-08-28T01:01:22.427243+0000 mon.a (mon.0) 3789 : audit [DBG] from='client.? 172.21.15.6:0/2982680011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:23.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:23 smithi006 bash[27292]: audit 2023-08-28T01:01:22.427243+0000 mon.a (mon.0) 3789 : audit [DBG] from='client.? 172.21.15.6:0/2982680011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:23.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:23 smithi130 bash[27579]: audit 2023-08-28T01:01:22.427243+0000 mon.a (mon.0) 3789 : audit [DBG] from='client.? 172.21.15.6:0/2982680011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:24.445 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:24 smithi130 bash[51302]: debug 2023-08-28T01:01:24.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:24.120250+0000) 2023-08-28T01:01:24.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:24 smithi006 bash[33918]: cluster 2023-08-28T01:01:23.455473+0000 mgr.y (mgr.14901) 5484 : cluster [DBG] pgmap v5491: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:24.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:24 smithi006 bash[27292]: cluster 2023-08-28T01:01:23.455473+0000 mgr.y (mgr.14901) 5484 : cluster [DBG] pgmap v5491: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:24.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:01:24.352Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:24.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:01:24.354Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:24 smithi130 bash[27579]: cluster 2023-08-28T01:01:23.455473+0000 mgr.y (mgr.14901) 5484 : cluster [DBG] pgmap v5491: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:25.448 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:25 smithi130 bash[51302]: debug 2023-08-28T01:01:25.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:25.120568+0000) 2023-08-28T01:01:25.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:25 smithi006 bash[33918]: audit 2023-08-28T01:01:24.840000+0000 mon.a (mon.0) 3790 : audit [DBG] from='client.? 172.21.15.6:0/3748997254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:25.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:25 smithi006 bash[27292]: audit 2023-08-28T01:01:24.840000+0000 mon.a (mon.0) 3790 : audit [DBG] from='client.? 172.21.15.6:0/3748997254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:25.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:25 smithi130 bash[27579]: audit 2023-08-28T01:01:24.840000+0000 mon.a (mon.0) 3790 : audit [DBG] from='client.? 172.21.15.6:0/3748997254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:26.459 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:26 smithi130 bash[51302]: debug 2023-08-28T01:01:26.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:26.120822+0000) 2023-08-28T01:01:26.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:26 smithi006 bash[33918]: cluster 2023-08-28T01:01:25.456517+0000 mgr.y (mgr.14901) 5485 : cluster [DBG] pgmap v5492: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:26 smithi006 bash[27292]: cluster 2023-08-28T01:01:25.456517+0000 mgr.y (mgr.14901) 5485 : cluster [DBG] pgmap v5492: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:26.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:26 smithi130 bash[27579]: cluster 2023-08-28T01:01:25.456517+0000 mgr.y (mgr.14901) 5485 : cluster [DBG] pgmap v5492: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:27.463 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:27 smithi130 bash[51302]: debug 2023-08-28T01:01:27.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:27.121142+0000) 2023-08-28T01:01:27.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:27 smithi006 bash[33918]: audit 2023-08-28T01:01:26.754573+0000 mon.b (mon.1) 645 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:01:27.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:27 smithi006 bash[33918]: audit 2023-08-28T01:01:27.254169+0000 mon.c (mon.2) 2126 : audit [DBG] from='client.? 172.21.15.6:0/2320718239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:27.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:27 smithi006 bash[27292]: audit 2023-08-28T01:01:26.754573+0000 mon.b (mon.1) 645 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:01:27.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:27 smithi006 bash[27292]: audit 2023-08-28T01:01:27.254169+0000 mon.c (mon.2) 2126 : audit [DBG] from='client.? 172.21.15.6:0/2320718239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:27.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:27 smithi130 bash[27579]: audit 2023-08-28T01:01:26.754573+0000 mon.b (mon.1) 645 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:01:27.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:27 smithi130 bash[27579]: audit 2023-08-28T01:01:27.254169+0000 mon.c (mon.2) 2126 : audit [DBG] from='client.? 172.21.15.6:0/2320718239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:28.517 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:28 smithi130 bash[51302]: debug 2023-08-28T01:01:28.116+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:28.121479+0000) 2023-08-28T01:01:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:28 smithi130 bash[27579]: cluster 2023-08-28T01:01:27.457231+0000 mgr.y (mgr.14901) 5486 : cluster [DBG] pgmap v5493: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:28.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:28 smithi006 bash[33918]: cluster 2023-08-28T01:01:27.457231+0000 mgr.y (mgr.14901) 5486 : cluster [DBG] pgmap v5493: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:28.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:28 smithi006 bash[27292]: cluster 2023-08-28T01:01:27.457231+0000 mgr.y (mgr.14901) 5486 : cluster [DBG] pgmap v5493: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:29.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:29 smithi130 bash[51302]: debug 2023-08-28T01:01:29.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:29.121756+0000) 2023-08-28T01:01:30.522 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:30 smithi130 bash[51302]: debug 2023-08-28T01:01:30.116+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:30.122120+0000) 2023-08-28T01:01:30.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:30 smithi130 bash[27579]: cluster 2023-08-28T01:01:29.458316+0000 mgr.y (mgr.14901) 5487 : cluster [DBG] pgmap v5494: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:30.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:30 smithi130 bash[27579]: audit 2023-08-28T01:01:29.683938+0000 mon.a (mon.0) 3791 : audit [DBG] from='client.? 172.21.15.6:0/1403167154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:30.900 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:01:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:01:30] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:01:30.900 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:30 smithi006 bash[33918]: cluster 2023-08-28T01:01:29.458316+0000 mgr.y (mgr.14901) 5487 : cluster [DBG] pgmap v5494: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:30.901 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:30 smithi006 bash[33918]: audit 2023-08-28T01:01:29.683938+0000 mon.a (mon.0) 3791 : audit [DBG] from='client.? 172.21.15.6:0/1403167154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:30.901 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:30 smithi006 bash[27292]: cluster 2023-08-28T01:01:29.458316+0000 mgr.y (mgr.14901) 5487 : cluster [DBG] pgmap v5494: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:30.901 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:30 smithi006 bash[27292]: audit 2023-08-28T01:01:29.683938+0000 mon.a (mon.0) 3791 : audit [DBG] from='client.? 172.21.15.6:0/1403167154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:31.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:01:30.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:30 smithi006 bash[56298]: level=warn ts=2023-08-28T01:01:30.900Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:31.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:31 smithi130 bash[51302]: debug 2023-08-28T01:01:31.116+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:31.122344+0000) 2023-08-28T01:01:32.434 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:32 smithi130 bash[51302]: debug 2023-08-28T01:01:32.116+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:32.122577+0000) 2023-08-28T01:01:32.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:32 smithi006 bash[33918]: cluster 2023-08-28T01:01:31.458987+0000 mgr.y (mgr.14901) 5488 : cluster [DBG] pgmap v5495: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:32.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:32 smithi006 bash[33918]: audit 2023-08-28T01:01:32.125872+0000 mon.a (mon.0) 3792 : audit [DBG] from='client.? 172.21.15.6:0/4273016368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:01:32.472Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:01:32.474Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:32 smithi006 bash[27292]: cluster 2023-08-28T01:01:31.458987+0000 mgr.y (mgr.14901) 5488 : cluster [DBG] pgmap v5495: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:32.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:32 smithi006 bash[27292]: audit 2023-08-28T01:01:32.125872+0000 mon.a (mon.0) 3792 : audit [DBG] from='client.? 172.21.15.6:0/4273016368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:32 smithi130 bash[27579]: cluster 2023-08-28T01:01:31.458987+0000 mgr.y (mgr.14901) 5488 : cluster [DBG] pgmap v5495: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:32.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:32 smithi130 bash[27579]: audit 2023-08-28T01:01:32.125872+0000 mon.a (mon.0) 3792 : audit [DBG] from='client.? 172.21.15.6:0/4273016368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:33.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:33 smithi130 bash[51302]: debug 2023-08-28T01:01:33.116+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:33.122807+0000) 2023-08-28T01:01:33.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:33 smithi006 bash[33918]: audit 2023-08-28T01:01:32.668335+0000 mon.a (mon.0) 3793 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:01:33.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:33 smithi006 bash[33918]: audit 2023-08-28T01:01:33.094242+0000 mon.a (mon.0) 3794 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:01:33.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:33 smithi006 bash[33918]: audit 2023-08-28T01:01:33.392557+0000 mon.b (mon.1) 646 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:01:33.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:33 smithi006 bash[33918]: audit 2023-08-28T01:01:33.394817+0000 mon.b (mon.1) 647 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:01:33.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:33 smithi006 bash[33918]: audit 2023-08-28T01:01:33.403335+0000 mon.a (mon.0) 3795 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:01:33.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:33 smithi006 bash[27292]: audit 2023-08-28T01:01:32.668335+0000 mon.a (mon.0) 3793 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:01:33.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:33 smithi006 bash[27292]: audit 2023-08-28T01:01:33.094242+0000 mon.a (mon.0) 3794 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:01:33.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:33 smithi006 bash[27292]: audit 2023-08-28T01:01:33.392557+0000 mon.b (mon.1) 646 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:01:33.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:33 smithi006 bash[27292]: audit 2023-08-28T01:01:33.394817+0000 mon.b (mon.1) 647 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:01:33.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:33 smithi006 bash[27292]: audit 2023-08-28T01:01:33.403335+0000 mon.a (mon.0) 3795 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:01:34.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:33 smithi130 bash[27579]: audit 2023-08-28T01:01:32.668335+0000 mon.a (mon.0) 3793 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:01:34.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:33 smithi130 bash[27579]: audit 2023-08-28T01:01:33.094242+0000 mon.a (mon.0) 3794 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:01:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:33 smithi130 bash[27579]: audit 2023-08-28T01:01:33.392557+0000 mon.b (mon.1) 646 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:01:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:33 smithi130 bash[27579]: audit 2023-08-28T01:01:33.394817+0000 mon.b (mon.1) 647 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:01:34.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:33 smithi130 bash[27579]: audit 2023-08-28T01:01:33.403335+0000 mon.a (mon.0) 3795 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:01:34.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:34 smithi130 bash[51302]: debug 2023-08-28T01:01:34.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:34.123033+0000) 2023-08-28T01:01:34.670 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:01:34.353Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:34.670 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:01:34.355Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:34.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:34 smithi006 bash[33918]: cluster 2023-08-28T01:01:33.459655+0000 mgr.y (mgr.14901) 5489 : cluster [DBG] pgmap v5496: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:34.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:34 smithi006 bash[33918]: audit 2023-08-28T01:01:34.550597+0000 mon.c (mon.2) 2127 : audit [DBG] from='client.? 172.21.15.6:0/2579936382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:34.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:34 smithi006 bash[27292]: cluster 2023-08-28T01:01:33.459655+0000 mgr.y (mgr.14901) 5489 : cluster [DBG] pgmap v5496: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:34.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:34 smithi006 bash[27292]: audit 2023-08-28T01:01:34.550597+0000 mon.c (mon.2) 2127 : audit [DBG] from='client.? 172.21.15.6:0/2579936382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:34 smithi130 bash[27579]: cluster 2023-08-28T01:01:33.459655+0000 mgr.y (mgr.14901) 5489 : cluster [DBG] pgmap v5496: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:34 smithi130 bash[27579]: audit 2023-08-28T01:01:34.550597+0000 mon.c (mon.2) 2127 : audit [DBG] from='client.? 172.21.15.6:0/2579936382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:35.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:35 smithi130 bash[51302]: debug 2023-08-28T01:01:35.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:35.123290+0000) 2023-08-28T01:01:36.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:36 smithi130 bash[51302]: debug 2023-08-28T01:01:36.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:36.123559+0000) 2023-08-28T01:01:36.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:36 smithi006 bash[27292]: cluster 2023-08-28T01:01:35.460786+0000 mgr.y (mgr.14901) 5490 : cluster [DBG] pgmap v5497: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:36.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:36 smithi006 bash[33918]: cluster 2023-08-28T01:01:35.460786+0000 mgr.y (mgr.14901) 5490 : cluster [DBG] pgmap v5497: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:36 smithi130 bash[27579]: cluster 2023-08-28T01:01:35.460786+0000 mgr.y (mgr.14901) 5490 : cluster [DBG] pgmap v5497: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:37.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:37 smithi130 bash[51302]: debug 2023-08-28T01:01:37.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:37.123775+0000) 2023-08-28T01:01:37.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:37 smithi006 bash[33918]: audit 2023-08-28T01:01:36.962994+0000 mon.c (mon.2) 2128 : audit [DBG] from='client.? 172.21.15.6:0/3875655936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:37.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:37 smithi006 bash[27292]: audit 2023-08-28T01:01:36.962994+0000 mon.c (mon.2) 2128 : audit [DBG] from='client.? 172.21.15.6:0/3875655936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:38.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:37 smithi130 bash[27579]: audit 2023-08-28T01:01:36.962994+0000 mon.c (mon.2) 2128 : audit [DBG] from='client.? 172.21.15.6:0/3875655936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:38.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:38 smithi130 bash[51302]: debug 2023-08-28T01:01:38.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:38.124029+0000) 2023-08-28T01:01:38.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:38 smithi006 bash[27292]: cluster 2023-08-28T01:01:37.461538+0000 mgr.y (mgr.14901) 5491 : cluster [DBG] pgmap v5498: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:38 smithi006 bash[33918]: cluster 2023-08-28T01:01:37.461538+0000 mgr.y (mgr.14901) 5491 : cluster [DBG] pgmap v5498: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:38 smithi130 bash[27579]: cluster 2023-08-28T01:01:37.461538+0000 mgr.y (mgr.14901) 5491 : cluster [DBG] pgmap v5498: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:39.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:39 smithi130 bash[51302]: debug 2023-08-28T01:01:39.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:39.124287+0000) 2023-08-28T01:01:39.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:39 smithi006 bash[27292]: audit 2023-08-28T01:01:39.377564+0000 mon.a (mon.0) 3796 : audit [DBG] from='client.? 172.21.15.6:0/1711153921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:39.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:39 smithi006 bash[33918]: audit 2023-08-28T01:01:39.377564+0000 mon.a (mon.0) 3796 : audit [DBG] from='client.? 172.21.15.6:0/1711153921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:40.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:39 smithi130 bash[27579]: audit 2023-08-28T01:01:39.377564+0000 mon.a (mon.0) 3796 : audit [DBG] from='client.? 172.21.15.6:0/1711153921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:40.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:40 smithi130 bash[51302]: debug 2023-08-28T01:01:40.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:40.124550+0000) 2023-08-28T01:01:40.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:01:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:01:40] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:01:40.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:40 smithi006 bash[33918]: cluster 2023-08-28T01:01:39.462665+0000 mgr.y (mgr.14901) 5492 : cluster [DBG] pgmap v5499: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:01:40.899Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:01:40.901Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:40.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:40 smithi006 bash[27292]: cluster 2023-08-28T01:01:39.462665+0000 mgr.y (mgr.14901) 5492 : cluster [DBG] pgmap v5499: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:40 smithi130 bash[27579]: cluster 2023-08-28T01:01:39.462665+0000 mgr.y (mgr.14901) 5492 : cluster [DBG] pgmap v5499: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:41.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:41 smithi130 bash[51302]: debug 2023-08-28T01:01:41.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:41.124813+0000) 2023-08-28T01:01:42.437 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:42 smithi130 bash[51302]: debug 2023-08-28T01:01:42.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:42.125069+0000) 2023-08-28T01:01:42.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:01:42.473Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:01:42.475Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:42.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:42 smithi006 bash[33918]: cluster 2023-08-28T01:01:41.463380+0000 mgr.y (mgr.14901) 5493 : cluster [DBG] pgmap v5500: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:42.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:42 smithi006 bash[33918]: audit 2023-08-28T01:01:41.787746+0000 mon.a (mon.0) 3797 : audit [DBG] from='client.? 172.21.15.6:0/998093973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:42 smithi006 bash[27292]: cluster 2023-08-28T01:01:41.463380+0000 mgr.y (mgr.14901) 5493 : cluster [DBG] pgmap v5500: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:42 smithi006 bash[27292]: audit 2023-08-28T01:01:41.787746+0000 mon.a (mon.0) 3797 : audit [DBG] from='client.? 172.21.15.6:0/998093973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:42 smithi130 bash[27579]: cluster 2023-08-28T01:01:41.463380+0000 mgr.y (mgr.14901) 5493 : cluster [DBG] pgmap v5500: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:42 smithi130 bash[27579]: audit 2023-08-28T01:01:41.787746+0000 mon.a (mon.0) 3797 : audit [DBG] from='client.? 172.21.15.6:0/998093973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:43.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:43 smithi130 bash[51302]: debug 2023-08-28T01:01:43.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:43.125342+0000) 2023-08-28T01:01:44.524 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:44 smithi130 bash[51302]: debug 2023-08-28T01:01:44.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:44.125621+0000) 2023-08-28T01:01:44.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:44 smithi006 bash[27292]: cluster 2023-08-28T01:01:43.464154+0000 mgr.y (mgr.14901) 5494 : cluster [DBG] pgmap v5501: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:44.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:44 smithi006 bash[27292]: audit 2023-08-28T01:01:44.196041+0000 mon.c (mon.2) 2129 : audit [DBG] from='client.? 172.21.15.6:0/2494113906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:44.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:44 smithi006 bash[33918]: cluster 2023-08-28T01:01:43.464154+0000 mgr.y (mgr.14901) 5494 : cluster [DBG] pgmap v5501: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:44.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:44 smithi006 bash[33918]: audit 2023-08-28T01:01:44.196041+0000 mon.c (mon.2) 2129 : audit [DBG] from='client.? 172.21.15.6:0/2494113906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:44.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:01:44.354Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:44.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:01:44.356Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:44 smithi130 bash[27579]: cluster 2023-08-28T01:01:43.464154+0000 mgr.y (mgr.14901) 5494 : cluster [DBG] pgmap v5501: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:44 smithi130 bash[27579]: audit 2023-08-28T01:01:44.196041+0000 mon.c (mon.2) 2129 : audit [DBG] from='client.? 172.21.15.6:0/2494113906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:45.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:45 smithi130 bash[51302]: debug 2023-08-28T01:01:45.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:45.125911+0000) 2023-08-28T01:01:46.530 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:46 smithi130 bash[51302]: debug 2023-08-28T01:01:46.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:46.126191+0000) 2023-08-28T01:01:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:46 smithi130 bash[27579]: cluster 2023-08-28T01:01:45.465363+0000 mgr.y (mgr.14901) 5495 : cluster [DBG] pgmap v5502: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:46 smithi006 bash[33918]: cluster 2023-08-28T01:01:45.465363+0000 mgr.y (mgr.14901) 5495 : cluster [DBG] pgmap v5502: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:46.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:46 smithi006 bash[27292]: cluster 2023-08-28T01:01:45.465363+0000 mgr.y (mgr.14901) 5495 : cluster [DBG] pgmap v5502: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:47.539 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:47 smithi130 bash[51302]: debug 2023-08-28T01:01:47.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:47.126475+0000) 2023-08-28T01:01:47.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:47 smithi130 bash[27579]: audit 2023-08-28T01:01:46.621561+0000 mon.c (mon.2) 2130 : audit [DBG] from='client.? 172.21.15.6:0/1463122481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:47.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:47 smithi006 bash[33918]: audit 2023-08-28T01:01:46.621561+0000 mon.c (mon.2) 2130 : audit [DBG] from='client.? 172.21.15.6:0/1463122481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:47.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:47 smithi006 bash[27292]: audit 2023-08-28T01:01:46.621561+0000 mon.c (mon.2) 2130 : audit [DBG] from='client.? 172.21.15.6:0/1463122481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:48.544 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:48 smithi130 bash[51302]: debug 2023-08-28T01:01:48.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:48.126808+0000) 2023-08-28T01:01:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:48 smithi130 bash[27579]: cluster 2023-08-28T01:01:47.466005+0000 mgr.y (mgr.14901) 5496 : cluster [DBG] pgmap v5503: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:48.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:48 smithi006 bash[33918]: cluster 2023-08-28T01:01:47.466005+0000 mgr.y (mgr.14901) 5496 : cluster [DBG] pgmap v5503: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:48 smithi006 bash[27292]: cluster 2023-08-28T01:01:47.466005+0000 mgr.y (mgr.14901) 5496 : cluster [DBG] pgmap v5503: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:49.546 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:49 smithi130 bash[51302]: debug 2023-08-28T01:01:49.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:49.127075+0000) 2023-08-28T01:01:49.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:49 smithi130 bash[27579]: audit 2023-08-28T01:01:49.034397+0000 mon.c (mon.2) 2131 : audit [DBG] from='client.? 172.21.15.6:0/3542538080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:49.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:49 smithi006 bash[27292]: audit 2023-08-28T01:01:49.034397+0000 mon.c (mon.2) 2131 : audit [DBG] from='client.? 172.21.15.6:0/3542538080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:49.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:49 smithi006 bash[33918]: audit 2023-08-28T01:01:49.034397+0000 mon.c (mon.2) 2131 : audit [DBG] from='client.? 172.21.15.6:0/3542538080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:50.556 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:50 smithi130 bash[51302]: debug 2023-08-28T01:01:50.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:50.127368+0000) 2023-08-28T01:01:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:50 smithi130 bash[27579]: cluster 2023-08-28T01:01:49.467102+0000 mgr.y (mgr.14901) 5497 : cluster [DBG] pgmap v5504: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:50.901 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:01:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:01:50] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:01:50.902 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:50 smithi006 bash[27292]: cluster 2023-08-28T01:01:49.467102+0000 mgr.y (mgr.14901) 5497 : cluster [DBG] pgmap v5504: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:50.902 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:50 smithi006 bash[33918]: cluster 2023-08-28T01:01:49.467102+0000 mgr.y (mgr.14901) 5497 : cluster [DBG] pgmap v5504: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:01:50.899Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:01:50.902Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:51.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:51 smithi130 bash[51302]: debug 2023-08-28T01:01:51.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:51.127655+0000) 2023-08-28T01:01:52.433 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:52 smithi130 bash[51302]: debug 2023-08-28T01:01:52.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:52.127986+0000) 2023-08-28T01:01:52.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:52 smithi006 bash[33918]: audit 2023-08-28T01:01:51.450057+0000 mon.c (mon.2) 2132 : audit [DBG] from='client.? 172.21.15.6:0/3147151466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:52.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:52 smithi006 bash[33918]: cluster 2023-08-28T01:01:51.467727+0000 mgr.y (mgr.14901) 5498 : cluster [DBG] pgmap v5505: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:52.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:52 smithi006 bash[27292]: audit 2023-08-28T01:01:51.450057+0000 mon.c (mon.2) 2132 : audit [DBG] from='client.? 172.21.15.6:0/3147151466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:52.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:52 smithi006 bash[27292]: cluster 2023-08-28T01:01:51.467727+0000 mgr.y (mgr.14901) 5498 : cluster [DBG] pgmap v5505: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:01:52.475Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:01:52.477Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:52 smithi130 bash[27579]: audit 2023-08-28T01:01:51.450057+0000 mon.c (mon.2) 2132 : audit [DBG] from='client.? 172.21.15.6:0/3147151466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:52 smithi130 bash[27579]: cluster 2023-08-28T01:01:51.467727+0000 mgr.y (mgr.14901) 5498 : cluster [DBG] pgmap v5505: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:53.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:53 smithi130 bash[51302]: debug 2023-08-28T01:01:53.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:53.128254+0000) 2023-08-28T01:01:54.530 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:54 smithi130 bash[51302]: debug 2023-08-28T01:01:54.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:54.128566+0000) 2023-08-28T01:01:54.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:54 smithi006 bash[33918]: cluster 2023-08-28T01:01:53.468391+0000 mgr.y (mgr.14901) 5499 : cluster [DBG] pgmap v5506: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:54.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:54 smithi006 bash[33918]: audit 2023-08-28T01:01:53.861502+0000 mon.c (mon.2) 2133 : audit [DBG] from='client.? 172.21.15.6:0/1688815397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:54 smithi006 bash[27292]: cluster 2023-08-28T01:01:53.468391+0000 mgr.y (mgr.14901) 5499 : cluster [DBG] pgmap v5506: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:54.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:54 smithi006 bash[27292]: audit 2023-08-28T01:01:53.861502+0000 mon.c (mon.2) 2133 : audit [DBG] from='client.? 172.21.15.6:0/1688815397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:54.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:01:54.355Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:54.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:01:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:01:54.356Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:01:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:54 smithi130 bash[27579]: cluster 2023-08-28T01:01:53.468391+0000 mgr.y (mgr.14901) 5499 : cluster [DBG] pgmap v5506: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:54 smithi130 bash[27579]: audit 2023-08-28T01:01:53.861502+0000 mon.c (mon.2) 2133 : audit [DBG] from='client.? 172.21.15.6:0/1688815397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:55.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:55 smithi130 bash[51302]: debug 2023-08-28T01:01:55.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:55.128842+0000) 2023-08-28T01:01:56.535 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:56 smithi130 bash[51302]: debug 2023-08-28T01:01:56.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:56.129114+0000) 2023-08-28T01:01:56.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:56 smithi130 bash[27579]: cluster 2023-08-28T01:01:55.469286+0000 mgr.y (mgr.14901) 5500 : cluster [DBG] pgmap v5507: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:56.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:56 smithi130 bash[27579]: audit 2023-08-28T01:01:56.271952+0000 mon.a (mon.0) 3798 : audit [DBG] from='client.? 172.21.15.6:0/4219582822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:56.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:56 smithi006 bash[33918]: cluster 2023-08-28T01:01:55.469286+0000 mgr.y (mgr.14901) 5500 : cluster [DBG] pgmap v5507: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:56.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:56 smithi006 bash[33918]: audit 2023-08-28T01:01:56.271952+0000 mon.a (mon.0) 3798 : audit [DBG] from='client.? 172.21.15.6:0/4219582822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:56 smithi006 bash[27292]: cluster 2023-08-28T01:01:55.469286+0000 mgr.y (mgr.14901) 5500 : cluster [DBG] pgmap v5507: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:56 smithi006 bash[27292]: audit 2023-08-28T01:01:56.271952+0000 mon.a (mon.0) 3798 : audit [DBG] from='client.? 172.21.15.6:0/4219582822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:57.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:57 smithi130 bash[51302]: debug 2023-08-28T01:01:57.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:57.129391+0000) 2023-08-28T01:01:58.540 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:58 smithi130 bash[51302]: debug 2023-08-28T01:01:58.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:58.129721+0000) 2023-08-28T01:01:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:58 smithi130 bash[27579]: cluster 2023-08-28T01:01:57.469892+0000 mgr.y (mgr.14901) 5501 : cluster [DBG] pgmap v5508: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:58.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:58 smithi006 bash[27292]: cluster 2023-08-28T01:01:57.469892+0000 mgr.y (mgr.14901) 5501 : cluster [DBG] pgmap v5508: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:58 smithi006 bash[33918]: cluster 2023-08-28T01:01:57.469892+0000 mgr.y (mgr.14901) 5501 : cluster [DBG] pgmap v5508: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:01:59.547 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:01:59 smithi130 bash[51302]: debug 2023-08-28T01:01:59.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:59.129968+0000) 2023-08-28T01:01:59.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:01:59 smithi130 bash[27579]: audit 2023-08-28T01:01:58.695407+0000 mon.a (mon.0) 3799 : audit [DBG] from='client.? 172.21.15.6:0/452057863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:59.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:01:59 smithi006 bash[27292]: audit 2023-08-28T01:01:58.695407+0000 mon.a (mon.0) 3799 : audit [DBG] from='client.? 172.21.15.6:0/452057863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:59.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:01:59 smithi006 bash[33918]: audit 2023-08-28T01:01:58.695407+0000 mon.a (mon.0) 3799 : audit [DBG] from='client.? 172.21.15.6:0/452057863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:00.550 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:00 smithi130 bash[51302]: debug 2023-08-28T01:02:00.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:00.130241+0000) 2023-08-28T01:02:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:00 smithi130 bash[27579]: cluster 2023-08-28T01:01:59.471158+0000 mgr.y (mgr.14901) 5502 : cluster [DBG] pgmap v5509: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:00.903 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:02:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:02:00] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:02:00.903 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:00 smithi006 bash[33918]: cluster 2023-08-28T01:01:59.471158+0000 mgr.y (mgr.14901) 5502 : cluster [DBG] pgmap v5509: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:00.903 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:00 smithi006 bash[27292]: cluster 2023-08-28T01:01:59.471158+0000 mgr.y (mgr.14901) 5502 : cluster [DBG] pgmap v5509: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:01.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:02:00.900Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:01.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:02:00.902Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:01.554 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:01 smithi130 bash[51302]: debug 2023-08-28T01:02:01.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:01.130544+0000) 2023-08-28T01:02:01.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:01 smithi130 bash[27579]: audit 2023-08-28T01:02:01.115004+0000 mon.c (mon.2) 2134 : audit [DBG] from='client.? 172.21.15.6:0/39713048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:01.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:01 smithi006 bash[27292]: audit 2023-08-28T01:02:01.115004+0000 mon.c (mon.2) 2134 : audit [DBG] from='client.? 172.21.15.6:0/39713048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:01.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:01 smithi006 bash[33918]: audit 2023-08-28T01:02:01.115004+0000 mon.c (mon.2) 2134 : audit [DBG] from='client.? 172.21.15.6:0/39713048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:02.557 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:02 smithi130 bash[51302]: debug 2023-08-28T01:02:02.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:02.130835+0000) 2023-08-28T01:02:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:02 smithi130 bash[27579]: cluster 2023-08-28T01:02:01.471860+0000 mgr.y (mgr.14901) 5503 : cluster [DBG] pgmap v5510: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:02 smithi006 bash[27292]: cluster 2023-08-28T01:02:01.471860+0000 mgr.y (mgr.14901) 5503 : cluster [DBG] pgmap v5510: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:02 smithi006 bash[33918]: cluster 2023-08-28T01:02:01.471860+0000 mgr.y (mgr.14901) 5503 : cluster [DBG] pgmap v5510: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:02.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:02:02.475Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:02.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:02:02.477Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:03.567 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:03 smithi130 bash[51302]: debug 2023-08-28T01:02:03.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:03.131172+0000) 2023-08-28T01:02:03.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:03 smithi130 bash[27579]: audit 2023-08-28T01:02:03.529085+0000 mon.c (mon.2) 2135 : audit [DBG] from='client.? 172.21.15.6:0/720246259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:03.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:03 smithi006 bash[27292]: audit 2023-08-28T01:02:03.529085+0000 mon.c (mon.2) 2135 : audit [DBG] from='client.? 172.21.15.6:0/720246259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:03.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:03 smithi006 bash[33918]: audit 2023-08-28T01:02:03.529085+0000 mon.c (mon.2) 2135 : audit [DBG] from='client.? 172.21.15.6:0/720246259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:04.570 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:04 smithi130 bash[51302]: debug 2023-08-28T01:02:04.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:04.131455+0000) 2023-08-28T01:02:04.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:04 smithi006 bash[33918]: cluster 2023-08-28T01:02:03.472497+0000 mgr.y (mgr.14901) 5504 : cluster [DBG] pgmap v5511: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:04.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:04 smithi006 bash[27292]: cluster 2023-08-28T01:02:03.472497+0000 mgr.y (mgr.14901) 5504 : cluster [DBG] pgmap v5511: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:04.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:02:04.355Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:04.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:02:04.359Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:04 smithi130 bash[27579]: cluster 2023-08-28T01:02:03.472497+0000 mgr.y (mgr.14901) 5504 : cluster [DBG] pgmap v5511: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:05.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:05 smithi130 bash[51302]: debug 2023-08-28T01:02:05.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:05.131756+0000) 2023-08-28T01:02:06.577 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:06 smithi130 bash[51302]: debug 2023-08-28T01:02:06.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:06.132021+0000) 2023-08-28T01:02:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:06 smithi130 bash[27579]: cluster 2023-08-28T01:02:05.473626+0000 mgr.y (mgr.14901) 5505 : cluster [DBG] pgmap v5512: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:06 smithi130 bash[27579]: audit 2023-08-28T01:02:05.942131+0000 mon.a (mon.0) 3800 : audit [DBG] from='client.? 172.21.15.6:0/559196545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:06.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:06 smithi006 bash[27292]: cluster 2023-08-28T01:02:05.473626+0000 mgr.y (mgr.14901) 5505 : cluster [DBG] pgmap v5512: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:06.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:06 smithi006 bash[27292]: audit 2023-08-28T01:02:05.942131+0000 mon.a (mon.0) 3800 : audit [DBG] from='client.? 172.21.15.6:0/559196545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:06 smithi006 bash[33918]: cluster 2023-08-28T01:02:05.473626+0000 mgr.y (mgr.14901) 5505 : cluster [DBG] pgmap v5512: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:06 smithi006 bash[33918]: audit 2023-08-28T01:02:05.942131+0000 mon.a (mon.0) 3800 : audit [DBG] from='client.? 172.21.15.6:0/559196545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:07.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:07 smithi130 bash[51302]: debug 2023-08-28T01:02:07.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:07.132297+0000) 2023-08-28T01:02:08.583 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:08 smithi130 bash[51302]: debug 2023-08-28T01:02:08.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:08.132585+0000) 2023-08-28T01:02:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:08 smithi130 bash[27579]: cluster 2023-08-28T01:02:07.474280+0000 mgr.y (mgr.14901) 5506 : cluster [DBG] pgmap v5513: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:08.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:08 smithi130 bash[27579]: audit 2023-08-28T01:02:08.357996+0000 mon.c (mon.2) 2136 : audit [DBG] from='client.? 172.21.15.6:0/2264177203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:08.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:08 smithi006 bash[27292]: cluster 2023-08-28T01:02:07.474280+0000 mgr.y (mgr.14901) 5506 : cluster [DBG] pgmap v5513: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:08.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:08 smithi006 bash[27292]: audit 2023-08-28T01:02:08.357996+0000 mon.c (mon.2) 2136 : audit [DBG] from='client.? 172.21.15.6:0/2264177203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:08 smithi006 bash[33918]: cluster 2023-08-28T01:02:07.474280+0000 mgr.y (mgr.14901) 5506 : cluster [DBG] pgmap v5513: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:08.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:08 smithi006 bash[33918]: audit 2023-08-28T01:02:08.357996+0000 mon.c (mon.2) 2136 : audit [DBG] from='client.? 172.21.15.6:0/2264177203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:09.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:09 smithi130 bash[51302]: debug 2023-08-28T01:02:09.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:09.132823+0000) 2023-08-28T01:02:10.588 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:10 smithi130 bash[51302]: debug 2023-08-28T01:02:10.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:10.133153+0000) 2023-08-28T01:02:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:10 smithi130 bash[27579]: cluster 2023-08-28T01:02:09.475338+0000 mgr.y (mgr.14901) 5507 : cluster [DBG] pgmap v5514: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:10.903 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:10 smithi006 bash[27292]: cluster 2023-08-28T01:02:09.475338+0000 mgr.y (mgr.14901) 5507 : cluster [DBG] pgmap v5514: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:10.904 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:02:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:02:10] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:02:10.904 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:10 smithi006 bash[33918]: cluster 2023-08-28T01:02:09.475338+0000 mgr.y (mgr.14901) 5507 : cluster [DBG] pgmap v5514: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:11.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:02:10.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:11.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:02:10.904Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:11.590 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:11 smithi130 bash[51302]: debug 2023-08-28T01:02:11.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:11.133439+0000) 2023-08-28T01:02:11.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:11 smithi130 bash[27579]: audit 2023-08-28T01:02:10.781675+0000 mon.c (mon.2) 2137 : audit [DBG] from='client.? 172.21.15.6:0/569629151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:11.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:11 smithi006 bash[27292]: audit 2023-08-28T01:02:10.781675+0000 mon.c (mon.2) 2137 : audit [DBG] from='client.? 172.21.15.6:0/569629151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:11.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:11 smithi006 bash[33918]: audit 2023-08-28T01:02:10.781675+0000 mon.c (mon.2) 2137 : audit [DBG] from='client.? 172.21.15.6:0/569629151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:12.593 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:12 smithi130 bash[51302]: debug 2023-08-28T01:02:12.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:12.133707+0000) 2023-08-28T01:02:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:12 smithi130 bash[27579]: cluster 2023-08-28T01:02:11.476125+0000 mgr.y (mgr.14901) 5508 : cluster [DBG] pgmap v5515: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:12 smithi006 bash[27292]: cluster 2023-08-28T01:02:11.476125+0000 mgr.y (mgr.14901) 5508 : cluster [DBG] pgmap v5515: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:12 smithi006 bash[33918]: cluster 2023-08-28T01:02:11.476125+0000 mgr.y (mgr.14901) 5508 : cluster [DBG] pgmap v5515: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:12.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:02:12.477Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:12.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:02:12.478Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:13.602 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:13 smithi130 bash[51302]: debug 2023-08-28T01:02:13.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:13.133995+0000) 2023-08-28T01:02:13.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:13 smithi130 bash[27579]: audit 2023-08-28T01:02:13.197061+0000 mon.c (mon.2) 2138 : audit [DBG] from='client.? 172.21.15.6:0/2198104228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:13.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:13 smithi006 bash[33918]: audit 2023-08-28T01:02:13.197061+0000 mon.c (mon.2) 2138 : audit [DBG] from='client.? 172.21.15.6:0/2198104228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:13.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:13 smithi006 bash[27292]: audit 2023-08-28T01:02:13.197061+0000 mon.c (mon.2) 2138 : audit [DBG] from='client.? 172.21.15.6:0/2198104228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:14.606 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:14 smithi130 bash[51302]: debug 2023-08-28T01:02:14.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:14.134263+0000) 2023-08-28T01:02:14.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:14 smithi006 bash[33918]: cluster 2023-08-28T01:02:13.476977+0000 mgr.y (mgr.14901) 5509 : cluster [DBG] pgmap v5516: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:14.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:14 smithi006 bash[27292]: cluster 2023-08-28T01:02:13.476977+0000 mgr.y (mgr.14901) 5509 : cluster [DBG] pgmap v5516: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:14.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:02:14.356Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:14.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:02:14.358Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:14 smithi130 bash[27579]: cluster 2023-08-28T01:02:13.476977+0000 mgr.y (mgr.14901) 5509 : cluster [DBG] pgmap v5516: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:15.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:15 smithi130 bash[51302]: debug 2023-08-28T01:02:15.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:15.134561+0000) 2023-08-28T01:02:16.612 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:16 smithi130 bash[51302]: debug 2023-08-28T01:02:16.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:16.134845+0000) 2023-08-28T01:02:16.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:16 smithi130 bash[27579]: cluster 2023-08-28T01:02:15.477993+0000 mgr.y (mgr.14901) 5510 : cluster [DBG] pgmap v5517: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:16.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:16 smithi130 bash[27579]: audit 2023-08-28T01:02:15.616215+0000 mon.a (mon.0) 3801 : audit [DBG] from='client.? 172.21.15.6:0/1665801029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:16.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:16 smithi006 bash[33918]: cluster 2023-08-28T01:02:15.477993+0000 mgr.y (mgr.14901) 5510 : cluster [DBG] pgmap v5517: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:16 smithi006 bash[33918]: audit 2023-08-28T01:02:15.616215+0000 mon.a (mon.0) 3801 : audit [DBG] from='client.? 172.21.15.6:0/1665801029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:16 smithi006 bash[27292]: cluster 2023-08-28T01:02:15.477993+0000 mgr.y (mgr.14901) 5510 : cluster [DBG] pgmap v5517: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:16.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:16 smithi006 bash[27292]: audit 2023-08-28T01:02:15.616215+0000 mon.a (mon.0) 3801 : audit [DBG] from='client.? 172.21.15.6:0/1665801029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:17.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:17 smithi130 bash[51302]: debug 2023-08-28T01:02:17.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:17.135168+0000) 2023-08-28T01:02:18.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:18 smithi130 bash[51302]: debug 2023-08-28T01:02:18.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:18.135439+0000) 2023-08-28T01:02:18.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:18 smithi130 bash[27579]: cluster 2023-08-28T01:02:17.478619+0000 mgr.y (mgr.14901) 5511 : cluster [DBG] pgmap v5518: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:18.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:18 smithi130 bash[27579]: audit 2023-08-28T01:02:18.033422+0000 mon.c (mon.2) 2139 : audit [DBG] from='client.? 172.21.15.6:0/115973982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:18.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:18 smithi006 bash[33918]: cluster 2023-08-28T01:02:17.478619+0000 mgr.y (mgr.14901) 5511 : cluster [DBG] pgmap v5518: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:18.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:18 smithi006 bash[33918]: audit 2023-08-28T01:02:18.033422+0000 mon.c (mon.2) 2139 : audit [DBG] from='client.? 172.21.15.6:0/115973982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:18 smithi006 bash[27292]: cluster 2023-08-28T01:02:17.478619+0000 mgr.y (mgr.14901) 5511 : cluster [DBG] pgmap v5518: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:18.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:18 smithi006 bash[27292]: audit 2023-08-28T01:02:18.033422+0000 mon.c (mon.2) 2139 : audit [DBG] from='client.? 172.21.15.6:0/115973982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:19.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:19 smithi130 bash[51302]: debug 2023-08-28T01:02:19.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:19.135718+0000) 2023-08-28T01:02:20.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:20 smithi130 bash[51302]: debug 2023-08-28T01:02:20.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:20.136024+0000) 2023-08-28T01:02:20.904 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:02:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:02:20] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T01:02:20.905 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:20 smithi006 bash[27292]: cluster 2023-08-28T01:02:19.479719+0000 mgr.y (mgr.14901) 5512 : cluster [DBG] pgmap v5519: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:20.905 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:20 smithi006 bash[27292]: audit 2023-08-28T01:02:20.457079+0000 mon.c (mon.2) 2140 : audit [DBG] from='client.? 172.21.15.6:0/2692616576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:20.905 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:20 smithi006 bash[33918]: cluster 2023-08-28T01:02:19.479719+0000 mgr.y (mgr.14901) 5512 : cluster [DBG] pgmap v5519: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:20.905 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:20 smithi006 bash[33918]: audit 2023-08-28T01:02:20.457079+0000 mon.c (mon.2) 2140 : audit [DBG] from='client.? 172.21.15.6:0/2692616576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:20 smithi130 bash[27579]: cluster 2023-08-28T01:02:19.479719+0000 mgr.y (mgr.14901) 5512 : cluster [DBG] pgmap v5519: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:20 smithi130 bash[27579]: audit 2023-08-28T01:02:20.457079+0000 mon.c (mon.2) 2140 : audit [DBG] from='client.? 172.21.15.6:0/2692616576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:21.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:02:20.902Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:21.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:02:20.904Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:21.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:21 smithi130 bash[51302]: debug 2023-08-28T01:02:21.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:21.136282+0000) 2023-08-28T01:02:22.445 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:22 smithi130 bash[51302]: debug 2023-08-28T01:02:22.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:22.136531+0000) 2023-08-28T01:02:22.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:22 smithi006 bash[27292]: cluster 2023-08-28T01:02:21.480378+0000 mgr.y (mgr.14901) 5513 : cluster [DBG] pgmap v5520: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:22.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:22 smithi006 bash[33918]: cluster 2023-08-28T01:02:21.480378+0000 mgr.y (mgr.14901) 5513 : cluster [DBG] pgmap v5520: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:22.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:02:22.477Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:22.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:02:22.479Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:22 smithi130 bash[27579]: cluster 2023-08-28T01:02:21.480378+0000 mgr.y (mgr.14901) 5513 : cluster [DBG] pgmap v5520: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:23.448 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:23 smithi130 bash[51302]: debug 2023-08-28T01:02:23.136+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:23.136860+0000) 2023-08-28T01:02:23.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:23 smithi006 bash[27292]: audit 2023-08-28T01:02:22.865235+0000 mon.c (mon.2) 2141 : audit [DBG] from='client.? 172.21.15.6:0/1957540284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:23.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:23 smithi006 bash[33918]: audit 2023-08-28T01:02:22.865235+0000 mon.c (mon.2) 2141 : audit [DBG] from='client.? 172.21.15.6:0/1957540284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:23.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:23 smithi130 bash[27579]: audit 2023-08-28T01:02:22.865235+0000 mon.c (mon.2) 2141 : audit [DBG] from='client.? 172.21.15.6:0/1957540284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:24.456 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:24 smithi130 bash[51302]: debug 2023-08-28T01:02:24.131+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:24.137199+0000) 2023-08-28T01:02:24.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:24 smithi006 bash[33918]: cluster 2023-08-28T01:02:23.481087+0000 mgr.y (mgr.14901) 5514 : cluster [DBG] pgmap v5521: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:24.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:24 smithi006 bash[27292]: cluster 2023-08-28T01:02:23.481087+0000 mgr.y (mgr.14901) 5514 : cluster [DBG] pgmap v5521: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:24.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:02:24.357Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:24.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:02:24.359Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:24 smithi130 bash[27579]: cluster 2023-08-28T01:02:23.481087+0000 mgr.y (mgr.14901) 5514 : cluster [DBG] pgmap v5521: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:25.459 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:25 smithi130 bash[51302]: debug 2023-08-28T01:02:25.135+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:25.137474+0000) 2023-08-28T01:02:25.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:25 smithi006 bash[33918]: audit 2023-08-28T01:02:25.282816+0000 mon.a (mon.0) 3802 : audit [DBG] from='client.? 172.21.15.6:0/711085308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:25.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:25 smithi006 bash[27292]: audit 2023-08-28T01:02:25.282816+0000 mon.a (mon.0) 3802 : audit [DBG] from='client.? 172.21.15.6:0/711085308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:25.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:25 smithi130 bash[27579]: audit 2023-08-28T01:02:25.282816+0000 mon.a (mon.0) 3802 : audit [DBG] from='client.? 172.21.15.6:0/711085308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:26.466 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:26 smithi130 bash[51302]: debug 2023-08-28T01:02:26.135+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:26.137796+0000) 2023-08-28T01:02:26.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:26 smithi006 bash[33918]: cluster 2023-08-28T01:02:25.482171+0000 mgr.y (mgr.14901) 5515 : cluster [DBG] pgmap v5522: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:26.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:26 smithi006 bash[27292]: cluster 2023-08-28T01:02:25.482171+0000 mgr.y (mgr.14901) 5515 : cluster [DBG] pgmap v5522: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:26.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:26 smithi130 bash[27579]: cluster 2023-08-28T01:02:25.482171+0000 mgr.y (mgr.14901) 5515 : cluster [DBG] pgmap v5522: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:27.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:27 smithi130 bash[51302]: debug 2023-08-28T01:02:27.131+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:27.138083+0000) 2023-08-28T01:02:28.542 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:28 smithi130 bash[51302]: debug 2023-08-28T01:02:28.131+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:28.138341+0000) 2023-08-28T01:02:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:28 smithi130 bash[27579]: cluster 2023-08-28T01:02:27.482769+0000 mgr.y (mgr.14901) 5516 : cluster [DBG] pgmap v5523: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:28 smithi130 bash[27579]: audit 2023-08-28T01:02:27.704310+0000 mon.a (mon.0) 3803 : audit [DBG] from='client.? 172.21.15.6:0/1936763543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:28.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:28 smithi006 bash[27292]: cluster 2023-08-28T01:02:27.482769+0000 mgr.y (mgr.14901) 5516 : cluster [DBG] pgmap v5523: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:28.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:28 smithi006 bash[27292]: audit 2023-08-28T01:02:27.704310+0000 mon.a (mon.0) 3803 : audit [DBG] from='client.? 172.21.15.6:0/1936763543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:28 smithi006 bash[33918]: cluster 2023-08-28T01:02:27.482769+0000 mgr.y (mgr.14901) 5516 : cluster [DBG] pgmap v5523: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:28 smithi006 bash[33918]: audit 2023-08-28T01:02:27.704310+0000 mon.a (mon.0) 3803 : audit [DBG] from='client.? 172.21.15.6:0/1936763543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:29.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:29 smithi130 bash[51302]: debug 2023-08-28T01:02:29.135+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:29.138600+0000) 2023-08-28T01:02:30.547 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:30 smithi130 bash[51302]: debug 2023-08-28T01:02:30.135+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:30.138937+0000) 2023-08-28T01:02:30.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:30 smithi130 bash[27579]: cluster 2023-08-28T01:02:29.483857+0000 mgr.y (mgr.14901) 5517 : cluster [DBG] pgmap v5524: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:30.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:30 smithi130 bash[27579]: audit 2023-08-28T01:02:30.121715+0000 mon.a (mon.0) 3804 : audit [DBG] from='client.? 172.21.15.6:0/646049963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:30.905 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:02:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:02:30] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:02:30.905 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:30 smithi006 bash[33918]: cluster 2023-08-28T01:02:29.483857+0000 mgr.y (mgr.14901) 5517 : cluster [DBG] pgmap v5524: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:30.906 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:30 smithi006 bash[33918]: audit 2023-08-28T01:02:30.121715+0000 mon.a (mon.0) 3804 : audit [DBG] from='client.? 172.21.15.6:0/646049963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:30.906 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:30 smithi006 bash[27292]: cluster 2023-08-28T01:02:29.483857+0000 mgr.y (mgr.14901) 5517 : cluster [DBG] pgmap v5524: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:30.906 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:30 smithi006 bash[27292]: audit 2023-08-28T01:02:30.121715+0000 mon.a (mon.0) 3804 : audit [DBG] from='client.? 172.21.15.6:0/646049963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:31.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:02:30.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:31.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:30 smithi006 bash[56298]: level=warn ts=2023-08-28T01:02:30.905Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:31.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:31 smithi130 bash[51302]: debug 2023-08-28T01:02:31.135+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:31.139164+0000) 2023-08-28T01:02:32.451 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:32 smithi130 bash[51302]: debug 2023-08-28T01:02:32.135+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:32.139459+0000) 2023-08-28T01:02:32.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:32 smithi006 bash[27292]: cluster 2023-08-28T01:02:31.484545+0000 mgr.y (mgr.14901) 5518 : cluster [DBG] pgmap v5525: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:32.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:32 smithi006 bash[33918]: cluster 2023-08-28T01:02:31.484545+0000 mgr.y (mgr.14901) 5518 : cluster [DBG] pgmap v5525: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:02:32.478Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:02:32.480Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:32 smithi130 bash[27579]: cluster 2023-08-28T01:02:31.484545+0000 mgr.y (mgr.14901) 5518 : cluster [DBG] pgmap v5525: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:33.456 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:33 smithi130 bash[51302]: debug 2023-08-28T01:02:33.135+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:33.139761+0000) 2023-08-28T01:02:33.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:33 smithi006 bash[33918]: audit 2023-08-28T01:02:32.542417+0000 mon.c (mon.2) 2142 : audit [DBG] from='client.? 172.21.15.6:0/2071592114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:33.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:33 smithi006 bash[33918]: audit 2023-08-28T01:02:33.414160+0000 mon.b (mon.1) 648 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:02:33.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:33 smithi006 bash[27292]: audit 2023-08-28T01:02:32.542417+0000 mon.c (mon.2) 2142 : audit [DBG] from='client.? 172.21.15.6:0/2071592114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:33.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:33 smithi006 bash[27292]: audit 2023-08-28T01:02:33.414160+0000 mon.b (mon.1) 648 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:02:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:33 smithi130 bash[27579]: audit 2023-08-28T01:02:32.542417+0000 mon.c (mon.2) 2142 : audit [DBG] from='client.? 172.21.15.6:0/2071592114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:33 smithi130 bash[27579]: audit 2023-08-28T01:02:33.414160+0000 mon.b (mon.1) 648 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:02:34.465 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:34 smithi130 bash[51302]: debug 2023-08-28T01:02:34.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:34.140088+0000) 2023-08-28T01:02:34.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:34 smithi006 bash[27292]: cluster 2023-08-28T01:02:33.485338+0000 mgr.y (mgr.14901) 5519 : cluster [DBG] pgmap v5526: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:34.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:34 smithi006 bash[33918]: cluster 2023-08-28T01:02:33.485338+0000 mgr.y (mgr.14901) 5519 : cluster [DBG] pgmap v5526: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:34.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:02:34.357Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:34.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:02:34.360Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:34.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:34 smithi130 bash[27579]: cluster 2023-08-28T01:02:33.485338+0000 mgr.y (mgr.14901) 5519 : cluster [DBG] pgmap v5526: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:35.467 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:35 smithi130 bash[51302]: debug 2023-08-28T01:02:35.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:35.140369+0000) 2023-08-28T01:02:35.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:35 smithi006 bash[33918]: audit 2023-08-28T01:02:34.964697+0000 mon.a (mon.0) 3805 : audit [DBG] from='client.? 172.21.15.6:0/3645804044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:35.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:35 smithi006 bash[27292]: audit 2023-08-28T01:02:34.964697+0000 mon.a (mon.0) 3805 : audit [DBG] from='client.? 172.21.15.6:0/3645804044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:35.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:35 smithi130 bash[27579]: audit 2023-08-28T01:02:34.964697+0000 mon.a (mon.0) 3805 : audit [DBG] from='client.? 172.21.15.6:0/3645804044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:36.477 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:36 smithi130 bash[51302]: debug 2023-08-28T01:02:36.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:36.140685+0000) 2023-08-28T01:02:36.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:36 smithi130 bash[27579]: cluster 2023-08-28T01:02:35.486512+0000 mgr.y (mgr.14901) 5520 : cluster [DBG] pgmap v5527: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:36.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:36 smithi006 bash[33918]: cluster 2023-08-28T01:02:35.486512+0000 mgr.y (mgr.14901) 5520 : cluster [DBG] pgmap v5527: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:36.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:36 smithi006 bash[27292]: cluster 2023-08-28T01:02:35.486512+0000 mgr.y (mgr.14901) 5520 : cluster [DBG] pgmap v5527: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:37.480 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:37 smithi130 bash[51302]: debug 2023-08-28T01:02:37.135+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:37.140954+0000) 2023-08-28T01:02:37.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:37 smithi130 bash[27579]: audit 2023-08-28T01:02:37.375329+0000 mon.a (mon.0) 3806 : audit [DBG] from='client.? 172.21.15.6:0/1458229272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:37.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:37 smithi006 bash[33918]: audit 2023-08-28T01:02:37.375329+0000 mon.a (mon.0) 3806 : audit [DBG] from='client.? 172.21.15.6:0/1458229272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:37.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:37 smithi006 bash[27292]: audit 2023-08-28T01:02:37.375329+0000 mon.a (mon.0) 3806 : audit [DBG] from='client.? 172.21.15.6:0/1458229272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:38.486 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:38 smithi130 bash[51302]: debug 2023-08-28T01:02:38.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:38.141319+0000) 2023-08-28T01:02:38.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:38 smithi130 bash[27579]: cluster 2023-08-28T01:02:37.487071+0000 mgr.y (mgr.14901) 5521 : cluster [DBG] pgmap v5528: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:38.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:38 smithi006 bash[27292]: cluster 2023-08-28T01:02:37.487071+0000 mgr.y (mgr.14901) 5521 : cluster [DBG] pgmap v5528: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:38 smithi006 bash[33918]: cluster 2023-08-28T01:02:37.487071+0000 mgr.y (mgr.14901) 5521 : cluster [DBG] pgmap v5528: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:39.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:39 smithi130 bash[51302]: debug 2023-08-28T01:02:39.135+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:39.141523+0000) 2023-08-28T01:02:40.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:40 smithi130 bash[51302]: debug 2023-08-28T01:02:40.135+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:40.141775+0000) 2023-08-28T01:02:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:40 smithi130 bash[27579]: audit 2023-08-28T01:02:39.358905+0000 mon.a (mon.0) 3807 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:02:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:40 smithi130 bash[27579]: cluster 2023-08-28T01:02:39.487675+0000 mgr.y (mgr.14901) 5522 : cluster [DBG] pgmap v5529: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:40 smithi130 bash[27579]: audit 2023-08-28T01:02:39.760141+0000 mon.a (mon.0) 3808 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:02:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:40 smithi130 bash[27579]: audit 2023-08-28T01:02:39.795864+0000 mon.a (mon.0) 3809 : audit [DBG] from='client.? 172.21.15.6:0/2862381695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:40 smithi130 bash[27579]: audit 2023-08-28T01:02:40.025271+0000 mon.b (mon.1) 649 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:02:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:40 smithi130 bash[27579]: audit 2023-08-28T01:02:40.027492+0000 mon.b (mon.1) 650 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:02:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:40 smithi130 bash[27579]: audit 2023-08-28T01:02:40.036121+0000 mon.a (mon.0) 3810 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:02:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:40 smithi006 bash[27292]: audit 2023-08-28T01:02:39.358905+0000 mon.a (mon.0) 3807 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:02:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:40 smithi006 bash[27292]: cluster 2023-08-28T01:02:39.487675+0000 mgr.y (mgr.14901) 5522 : cluster [DBG] pgmap v5529: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:40 smithi006 bash[27292]: audit 2023-08-28T01:02:39.760141+0000 mon.a (mon.0) 3808 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:02:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:40 smithi006 bash[27292]: audit 2023-08-28T01:02:39.795864+0000 mon.a (mon.0) 3809 : audit [DBG] from='client.? 172.21.15.6:0/2862381695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:40 smithi006 bash[27292]: audit 2023-08-28T01:02:40.025271+0000 mon.b (mon.1) 649 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:02:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:40 smithi006 bash[27292]: audit 2023-08-28T01:02:40.027492+0000 mon.b (mon.1) 650 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:02:40.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:40 smithi006 bash[27292]: audit 2023-08-28T01:02:40.036121+0000 mon.a (mon.0) 3810 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:02:40.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:40 smithi006 bash[33918]: audit 2023-08-28T01:02:39.358905+0000 mon.a (mon.0) 3807 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:02:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:40 smithi006 bash[33918]: cluster 2023-08-28T01:02:39.487675+0000 mgr.y (mgr.14901) 5522 : cluster [DBG] pgmap v5529: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:40 smithi006 bash[33918]: audit 2023-08-28T01:02:39.760141+0000 mon.a (mon.0) 3808 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:02:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:40 smithi006 bash[33918]: audit 2023-08-28T01:02:39.795864+0000 mon.a (mon.0) 3809 : audit [DBG] from='client.? 172.21.15.6:0/2862381695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:40 smithi006 bash[33918]: audit 2023-08-28T01:02:40.025271+0000 mon.b (mon.1) 649 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:02:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:40 smithi006 bash[33918]: audit 2023-08-28T01:02:40.027492+0000 mon.b (mon.1) 650 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:02:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:40 smithi006 bash[33918]: audit 2023-08-28T01:02:40.036121+0000 mon.a (mon.0) 3810 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:02:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:02:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:02:40] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:02:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:02:40.904Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:02:40.906Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:41.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:41 smithi130 bash[51302]: debug 2023-08-28T01:02:41.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:41.142040+0000) 2023-08-28T01:02:42.452 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:42 smithi130 bash[51302]: debug 2023-08-28T01:02:42.135+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:42.142319+0000) 2023-08-28T01:02:42.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:42 smithi006 bash[33918]: cluster 2023-08-28T01:02:41.488225+0000 mgr.y (mgr.14901) 5523 : cluster [DBG] pgmap v5530: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:42.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:42 smithi006 bash[33918]: audit 2023-08-28T01:02:42.214440+0000 mon.a (mon.0) 3811 : audit [DBG] from='client.? 172.21.15.6:0/2880527247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:02:42.479Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:02:42.481Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:42 smithi006 bash[27292]: cluster 2023-08-28T01:02:41.488225+0000 mgr.y (mgr.14901) 5523 : cluster [DBG] pgmap v5530: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:42.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:42 smithi006 bash[27292]: audit 2023-08-28T01:02:42.214440+0000 mon.a (mon.0) 3811 : audit [DBG] from='client.? 172.21.15.6:0/2880527247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:42 smithi130 bash[27579]: cluster 2023-08-28T01:02:41.488225+0000 mgr.y (mgr.14901) 5523 : cluster [DBG] pgmap v5530: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:42 smithi130 bash[27579]: audit 2023-08-28T01:02:42.214440+0000 mon.a (mon.0) 3811 : audit [DBG] from='client.? 172.21.15.6:0/2880527247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:43.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:43 smithi130 bash[51302]: debug 2023-08-28T01:02:43.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:43.142603+0000) 2023-08-28T01:02:44.549 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:44 smithi130 bash[51302]: debug 2023-08-28T01:02:44.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:44.142921+0000) 2023-08-28T01:02:44.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:44 smithi006 bash[33918]: cluster 2023-08-28T01:02:43.488927+0000 mgr.y (mgr.14901) 5524 : cluster [DBG] pgmap v5531: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:44.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:44 smithi006 bash[27292]: cluster 2023-08-28T01:02:43.488927+0000 mgr.y (mgr.14901) 5524 : cluster [DBG] pgmap v5531: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:44.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:02:44.358Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:44.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:02:44.360Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:44 smithi130 bash[27579]: cluster 2023-08-28T01:02:43.488927+0000 mgr.y (mgr.14901) 5524 : cluster [DBG] pgmap v5531: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:45.556 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:45 smithi130 bash[51302]: debug 2023-08-28T01:02:45.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:45.143199+0000) 2023-08-28T01:02:45.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:45 smithi130 bash[27579]: audit 2023-08-28T01:02:44.624144+0000 mon.a (mon.0) 3812 : audit [DBG] from='client.? 172.21.15.6:0/2348614115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:45.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:45 smithi006 bash[33918]: audit 2023-08-28T01:02:44.624144+0000 mon.a (mon.0) 3812 : audit [DBG] from='client.? 172.21.15.6:0/2348614115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:45.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:45 smithi006 bash[27292]: audit 2023-08-28T01:02:44.624144+0000 mon.a (mon.0) 3812 : audit [DBG] from='client.? 172.21.15.6:0/2348614115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:46.561 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:46 smithi130 bash[51302]: debug 2023-08-28T01:02:46.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:46.143505+0000) 2023-08-28T01:02:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:46 smithi130 bash[27579]: cluster 2023-08-28T01:02:45.489928+0000 mgr.y (mgr.14901) 5525 : cluster [DBG] pgmap v5532: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:46 smithi006 bash[33918]: cluster 2023-08-28T01:02:45.489928+0000 mgr.y (mgr.14901) 5525 : cluster [DBG] pgmap v5532: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:46.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:46 smithi006 bash[27292]: cluster 2023-08-28T01:02:45.489928+0000 mgr.y (mgr.14901) 5525 : cluster [DBG] pgmap v5532: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:47.569 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:47 smithi130 bash[51302]: debug 2023-08-28T01:02:47.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:47.143779+0000) 2023-08-28T01:02:47.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:47 smithi130 bash[27579]: audit 2023-08-28T01:02:47.038691+0000 mon.a (mon.0) 3813 : audit [DBG] from='client.? 172.21.15.6:0/2120251275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:47.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:47 smithi006 bash[33918]: audit 2023-08-28T01:02:47.038691+0000 mon.a (mon.0) 3813 : audit [DBG] from='client.? 172.21.15.6:0/2120251275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:47.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:47 smithi006 bash[27292]: audit 2023-08-28T01:02:47.038691+0000 mon.a (mon.0) 3813 : audit [DBG] from='client.? 172.21.15.6:0/2120251275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:48.574 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:48 smithi130 bash[51302]: debug 2023-08-28T01:02:48.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:48.144131+0000) 2023-08-28T01:02:48.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:48 smithi130 bash[27579]: cluster 2023-08-28T01:02:47.490592+0000 mgr.y (mgr.14901) 5526 : cluster [DBG] pgmap v5533: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:48.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:48 smithi006 bash[33918]: cluster 2023-08-28T01:02:47.490592+0000 mgr.y (mgr.14901) 5526 : cluster [DBG] pgmap v5533: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:48.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:48 smithi006 bash[27292]: cluster 2023-08-28T01:02:47.490592+0000 mgr.y (mgr.14901) 5526 : cluster [DBG] pgmap v5533: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:49.581 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:49 smithi130 bash[51302]: debug 2023-08-28T01:02:49.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:49.144459+0000) 2023-08-28T01:02:49.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:49 smithi130 bash[27579]: audit 2023-08-28T01:02:49.449784+0000 mon.c (mon.2) 2143 : audit [DBG] from='client.? 172.21.15.6:0/534433990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:49.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:49 smithi006 bash[27292]: audit 2023-08-28T01:02:49.449784+0000 mon.c (mon.2) 2143 : audit [DBG] from='client.? 172.21.15.6:0/534433990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:49.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:49 smithi006 bash[33918]: audit 2023-08-28T01:02:49.449784+0000 mon.c (mon.2) 2143 : audit [DBG] from='client.? 172.21.15.6:0/534433990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:50.584 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:50 smithi130 bash[51302]: debug 2023-08-28T01:02:50.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:50.144774+0000) 2023-08-28T01:02:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:50 smithi130 bash[27579]: cluster 2023-08-28T01:02:49.491718+0000 mgr.y (mgr.14901) 5527 : cluster [DBG] pgmap v5534: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:50.907 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:02:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:02:50] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:02:50.907 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:50 smithi006 bash[33918]: cluster 2023-08-28T01:02:49.491718+0000 mgr.y (mgr.14901) 5527 : cluster [DBG] pgmap v5534: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:50.907 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:50 smithi006 bash[27292]: cluster 2023-08-28T01:02:49.491718+0000 mgr.y (mgr.14901) 5527 : cluster [DBG] pgmap v5534: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:02:50.904Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:02:50.906Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:51.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:51 smithi130 bash[51302]: debug 2023-08-28T01:02:51.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:51.145074+0000) 2023-08-28T01:02:52.455 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:52 smithi130 bash[51302]: debug 2023-08-28T01:02:52.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:52.145430+0000) 2023-08-28T01:02:52.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:52 smithi006 bash[27292]: cluster 2023-08-28T01:02:51.492267+0000 mgr.y (mgr.14901) 5528 : cluster [DBG] pgmap v5535: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:52.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:52 smithi006 bash[27292]: audit 2023-08-28T01:02:51.869956+0000 mon.a (mon.0) 3814 : audit [DBG] from='client.? 172.21.15.6:0/1201294456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:02:52.479Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:52.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:02:52.482Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:52 smithi006 bash[33918]: cluster 2023-08-28T01:02:51.492267+0000 mgr.y (mgr.14901) 5528 : cluster [DBG] pgmap v5535: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:52 smithi006 bash[33918]: audit 2023-08-28T01:02:51.869956+0000 mon.a (mon.0) 3814 : audit [DBG] from='client.? 172.21.15.6:0/1201294456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:52 smithi130 bash[27579]: cluster 2023-08-28T01:02:51.492267+0000 mgr.y (mgr.14901) 5528 : cluster [DBG] pgmap v5535: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:52 smithi130 bash[27579]: audit 2023-08-28T01:02:51.869956+0000 mon.a (mon.0) 3814 : audit [DBG] from='client.? 172.21.15.6:0/1201294456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:53.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:53 smithi130 bash[51302]: debug 2023-08-28T01:02:53.143+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:53.145752+0000) 2023-08-28T01:02:54.553 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:54 smithi130 bash[51302]: debug 2023-08-28T01:02:54.143+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:54.146122+0000) 2023-08-28T01:02:54.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:54 smithi006 bash[27292]: cluster 2023-08-28T01:02:53.492954+0000 mgr.y (mgr.14901) 5529 : cluster [DBG] pgmap v5536: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:54.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:54 smithi006 bash[27292]: audit 2023-08-28T01:02:54.283576+0000 mon.c (mon.2) 2144 : audit [DBG] from='client.? 172.21.15.6:0/713801161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:54.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:02:54.359Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:54.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:02:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:02:54.361Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:02:54.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:54 smithi006 bash[33918]: cluster 2023-08-28T01:02:53.492954+0000 mgr.y (mgr.14901) 5529 : cluster [DBG] pgmap v5536: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:54.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:54 smithi006 bash[33918]: audit 2023-08-28T01:02:54.283576+0000 mon.c (mon.2) 2144 : audit [DBG] from='client.? 172.21.15.6:0/713801161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:54 smithi130 bash[27579]: cluster 2023-08-28T01:02:53.492954+0000 mgr.y (mgr.14901) 5529 : cluster [DBG] pgmap v5536: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:54 smithi130 bash[27579]: audit 2023-08-28T01:02:54.283576+0000 mon.c (mon.2) 2144 : audit [DBG] from='client.? 172.21.15.6:0/713801161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:55.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:55 smithi130 bash[51302]: debug 2023-08-28T01:02:55.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:55.146401+0000) 2023-08-28T01:02:56.557 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:56 smithi130 bash[51302]: debug 2023-08-28T01:02:56.143+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:56.146673+0000) 2023-08-28T01:02:56.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:56 smithi130 bash[27579]: cluster 2023-08-28T01:02:55.494123+0000 mgr.y (mgr.14901) 5530 : cluster [DBG] pgmap v5537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:56.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:56 smithi006 bash[27292]: cluster 2023-08-28T01:02:55.494123+0000 mgr.y (mgr.14901) 5530 : cluster [DBG] pgmap v5537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:56.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:56 smithi006 bash[33918]: cluster 2023-08-28T01:02:55.494123+0000 mgr.y (mgr.14901) 5530 : cluster [DBG] pgmap v5537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:57.567 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:57 smithi130 bash[51302]: debug 2023-08-28T01:02:57.143+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:57.146958+0000) 2023-08-28T01:02:57.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:57 smithi130 bash[27579]: audit 2023-08-28T01:02:56.699389+0000 mon.a (mon.0) 3815 : audit [DBG] from='client.? 172.21.15.6:0/2835624869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:57.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:57 smithi006 bash[33918]: audit 2023-08-28T01:02:56.699389+0000 mon.a (mon.0) 3815 : audit [DBG] from='client.? 172.21.15.6:0/2835624869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:57.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:57 smithi006 bash[27292]: audit 2023-08-28T01:02:56.699389+0000 mon.a (mon.0) 3815 : audit [DBG] from='client.? 172.21.15.6:0/2835624869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:58.570 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:58 smithi130 bash[51302]: debug 2023-08-28T01:02:58.143+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:58.147221+0000) 2023-08-28T01:02:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:58 smithi130 bash[27579]: cluster 2023-08-28T01:02:57.494926+0000 mgr.y (mgr.14901) 5531 : cluster [DBG] pgmap v5538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:58.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:58 smithi006 bash[27292]: cluster 2023-08-28T01:02:57.494926+0000 mgr.y (mgr.14901) 5531 : cluster [DBG] pgmap v5538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:58.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:58 smithi006 bash[33918]: cluster 2023-08-28T01:02:57.494926+0000 mgr.y (mgr.14901) 5531 : cluster [DBG] pgmap v5538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:02:59.577 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:02:59 smithi130 bash[51302]: debug 2023-08-28T01:02:59.143+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:59.147551+0000) 2023-08-28T01:02:59.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:02:59 smithi130 bash[27579]: audit 2023-08-28T01:02:59.121625+0000 mon.c (mon.2) 2145 : audit [DBG] from='client.? 172.21.15.6:0/1973477547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:59.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:02:59 smithi006 bash[33918]: audit 2023-08-28T01:02:59.121625+0000 mon.c (mon.2) 2145 : audit [DBG] from='client.? 172.21.15.6:0/1973477547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:59.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:02:59 smithi006 bash[27292]: audit 2023-08-28T01:02:59.121625+0000 mon.c (mon.2) 2145 : audit [DBG] from='client.? 172.21.15.6:0/1973477547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:00.580 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:00 smithi130 bash[51302]: debug 2023-08-28T01:03:00.143+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:00.147810+0000) 2023-08-28T01:03:00.869 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:00 smithi130 bash[27579]: cluster 2023-08-28T01:02:59.496101+0000 mgr.y (mgr.14901) 5532 : cluster [DBG] pgmap v5539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:00.908 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:03:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:03:00] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:03:00.908 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:00 smithi006 bash[33918]: cluster 2023-08-28T01:02:59.496101+0000 mgr.y (mgr.14901) 5532 : cluster [DBG] pgmap v5539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:00.908 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:00 smithi006 bash[27292]: cluster 2023-08-28T01:02:59.496101+0000 mgr.y (mgr.14901) 5532 : cluster [DBG] pgmap v5539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:01.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:03:00.906Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:01.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:03:00.908Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:01.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:01 smithi130 bash[51302]: debug 2023-08-28T01:03:01.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:01.148113+0000) 2023-08-28T01:03:02.459 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:02 smithi130 bash[51302]: debug 2023-08-28T01:03:02.143+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:02.148391+0000) 2023-08-28T01:03:02.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:03:02.479Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:02.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:03:02.481Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:02 smithi006 bash[27292]: cluster 2023-08-28T01:03:01.496664+0000 mgr.y (mgr.14901) 5533 : cluster [DBG] pgmap v5540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:02.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:02 smithi006 bash[27292]: audit 2023-08-28T01:03:01.530520+0000 mon.a (mon.0) 3816 : audit [DBG] from='client.? 172.21.15.6:0/3471120639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:02 smithi006 bash[33918]: cluster 2023-08-28T01:03:01.496664+0000 mgr.y (mgr.14901) 5533 : cluster [DBG] pgmap v5540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:02.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:02 smithi006 bash[33918]: audit 2023-08-28T01:03:01.530520+0000 mon.a (mon.0) 3816 : audit [DBG] from='client.? 172.21.15.6:0/3471120639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:02 smithi130 bash[27579]: cluster 2023-08-28T01:03:01.496664+0000 mgr.y (mgr.14901) 5533 : cluster [DBG] pgmap v5540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:02 smithi130 bash[27579]: audit 2023-08-28T01:03:01.530520+0000 mon.a (mon.0) 3816 : audit [DBG] from='client.? 172.21.15.6:0/3471120639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:03.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:03 smithi130 bash[51302]: debug 2023-08-28T01:03:03.143+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:03.148762+0000) 2023-08-28T01:03:04.557 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:04 smithi130 bash[51302]: debug 2023-08-28T01:03:04.143+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:04.149086+0000) 2023-08-28T01:03:04.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:04 smithi006 bash[27292]: cluster 2023-08-28T01:03:03.497375+0000 mgr.y (mgr.14901) 5534 : cluster [DBG] pgmap v5541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:04.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:04 smithi006 bash[27292]: audit 2023-08-28T01:03:03.942550+0000 mon.c (mon.2) 2146 : audit [DBG] from='client.? 172.21.15.6:0/40704006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:04.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:03:04.360Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:04.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:03:04.363Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:04.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:04 smithi006 bash[33918]: cluster 2023-08-28T01:03:03.497375+0000 mgr.y (mgr.14901) 5534 : cluster [DBG] pgmap v5541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:04.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:04 smithi006 bash[33918]: audit 2023-08-28T01:03:03.942550+0000 mon.c (mon.2) 2146 : audit [DBG] from='client.? 172.21.15.6:0/40704006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:04 smithi130 bash[27579]: cluster 2023-08-28T01:03:03.497375+0000 mgr.y (mgr.14901) 5534 : cluster [DBG] pgmap v5541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:04 smithi130 bash[27579]: audit 2023-08-28T01:03:03.942550+0000 mon.c (mon.2) 2146 : audit [DBG] from='client.? 172.21.15.6:0/40704006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:05.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:05 smithi130 bash[51302]: debug 2023-08-28T01:03:05.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:05.149416+0000) 2023-08-28T01:03:06.562 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:06 smithi130 bash[51302]: debug 2023-08-28T01:03:06.143+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:06.149718+0000) 2023-08-28T01:03:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:06 smithi130 bash[27579]: cluster 2023-08-28T01:03:05.498561+0000 mgr.y (mgr.14901) 5535 : cluster [DBG] pgmap v5542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:06 smithi130 bash[27579]: audit 2023-08-28T01:03:06.366516+0000 mon.c (mon.2) 2147 : audit [DBG] from='client.? 172.21.15.6:0/2907988223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:06.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:06 smithi006 bash[33918]: cluster 2023-08-28T01:03:05.498561+0000 mgr.y (mgr.14901) 5535 : cluster [DBG] pgmap v5542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:06.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:06 smithi006 bash[33918]: audit 2023-08-28T01:03:06.366516+0000 mon.c (mon.2) 2147 : audit [DBG] from='client.? 172.21.15.6:0/2907988223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:06 smithi006 bash[27292]: cluster 2023-08-28T01:03:05.498561+0000 mgr.y (mgr.14901) 5535 : cluster [DBG] pgmap v5542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:06.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:06 smithi006 bash[27292]: audit 2023-08-28T01:03:06.366516+0000 mon.c (mon.2) 2147 : audit [DBG] from='client.? 172.21.15.6:0/2907988223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:07.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:07 smithi130 bash[51302]: debug 2023-08-28T01:03:07.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:07.149986+0000) 2023-08-28T01:03:08.566 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:08 smithi130 bash[51302]: debug 2023-08-28T01:03:08.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:08.150291+0000) 2023-08-28T01:03:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:08 smithi130 bash[27579]: cluster 2023-08-28T01:03:07.499159+0000 mgr.y (mgr.14901) 5536 : cluster [DBG] pgmap v5543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:08.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:08 smithi006 bash[33918]: cluster 2023-08-28T01:03:07.499159+0000 mgr.y (mgr.14901) 5536 : cluster [DBG] pgmap v5543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:08.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:08 smithi006 bash[27292]: cluster 2023-08-28T01:03:07.499159+0000 mgr.y (mgr.14901) 5536 : cluster [DBG] pgmap v5543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:09.574 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:09 smithi130 bash[51302]: debug 2023-08-28T01:03:09.143+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:09.150655+0000) 2023-08-28T01:03:09.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:09 smithi130 bash[27579]: audit 2023-08-28T01:03:08.785777+0000 mon.a (mon.0) 3817 : audit [DBG] from='client.? 172.21.15.6:0/3301219760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:09.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:09 smithi006 bash[27292]: audit 2023-08-28T01:03:08.785777+0000 mon.a (mon.0) 3817 : audit [DBG] from='client.? 172.21.15.6:0/3301219760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:09.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:09 smithi006 bash[33918]: audit 2023-08-28T01:03:08.785777+0000 mon.a (mon.0) 3817 : audit [DBG] from='client.? 172.21.15.6:0/3301219760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:10.576 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:10 smithi130 bash[51302]: debug 2023-08-28T01:03:10.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:10.150937+0000) 2023-08-28T01:03:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:10 smithi130 bash[27579]: cluster 2023-08-28T01:03:09.500243+0000 mgr.y (mgr.14901) 5537 : cluster [DBG] pgmap v5544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:10.908 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:03:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:03:10] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:03:10.909 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:10 smithi006 bash[33918]: cluster 2023-08-28T01:03:09.500243+0000 mgr.y (mgr.14901) 5537 : cluster [DBG] pgmap v5544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:10.909 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:10 smithi006 bash[27292]: cluster 2023-08-28T01:03:09.500243+0000 mgr.y (mgr.14901) 5537 : cluster [DBG] pgmap v5544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:11.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:03:10.906Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:11.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:03:10.909Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:11.579 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:11 smithi130 bash[51302]: debug 2023-08-28T01:03:11.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:11.151243+0000) 2023-08-28T01:03:11.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:11 smithi130 bash[27579]: audit 2023-08-28T01:03:11.191779+0000 mon.c (mon.2) 2148 : audit [DBG] from='client.? 172.21.15.6:0/431629432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:11.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:11 smithi006 bash[27292]: audit 2023-08-28T01:03:11.191779+0000 mon.c (mon.2) 2148 : audit [DBG] from='client.? 172.21.15.6:0/431629432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:11.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:11 smithi006 bash[33918]: audit 2023-08-28T01:03:11.191779+0000 mon.c (mon.2) 2148 : audit [DBG] from='client.? 172.21.15.6:0/431629432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:12.582 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:12 smithi130 bash[51302]: debug 2023-08-28T01:03:12.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:12.151469+0000) 2023-08-28T01:03:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:12 smithi130 bash[27579]: cluster 2023-08-28T01:03:11.500983+0000 mgr.y (mgr.14901) 5538 : cluster [DBG] pgmap v5545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:12 smithi006 bash[27292]: cluster 2023-08-28T01:03:11.500983+0000 mgr.y (mgr.14901) 5538 : cluster [DBG] pgmap v5545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:12 smithi006 bash[33918]: cluster 2023-08-28T01:03:11.500983+0000 mgr.y (mgr.14901) 5538 : cluster [DBG] pgmap v5545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:12.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:03:12.480Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:12.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:03:12.482Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:13.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:13 smithi130 bash[51302]: debug 2023-08-28T01:03:13.151+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:13.151747+0000) 2023-08-28T01:03:14.590 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:14 smithi130 bash[51302]: debug 2023-08-28T01:03:14.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:14.152027+0000) 2023-08-28T01:03:14.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:14 smithi006 bash[27292]: cluster 2023-08-28T01:03:13.501588+0000 mgr.y (mgr.14901) 5539 : cluster [DBG] pgmap v5546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:14.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:14 smithi006 bash[27292]: audit 2023-08-28T01:03:13.605127+0000 mon.a (mon.0) 3818 : audit [DBG] from='client.? 172.21.15.6:0/2487263256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:14 smithi006 bash[33918]: cluster 2023-08-28T01:03:13.501588+0000 mgr.y (mgr.14901) 5539 : cluster [DBG] pgmap v5546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:14 smithi006 bash[33918]: audit 2023-08-28T01:03:13.605127+0000 mon.a (mon.0) 3818 : audit [DBG] from='client.? 172.21.15.6:0/2487263256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:14.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:03:14.361Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:14.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:03:14.364Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:14 smithi130 bash[27579]: cluster 2023-08-28T01:03:13.501588+0000 mgr.y (mgr.14901) 5539 : cluster [DBG] pgmap v5546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:14 smithi130 bash[27579]: audit 2023-08-28T01:03:13.605127+0000 mon.a (mon.0) 3818 : audit [DBG] from='client.? 172.21.15.6:0/2487263256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:15.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:15 smithi130 bash[51302]: debug 2023-08-28T01:03:15.151+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:15.152292+0000) 2023-08-28T01:03:16.592 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:16 smithi130 bash[51302]: debug 2023-08-28T01:03:16.151+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:16.152583+0000) 2023-08-28T01:03:16.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:16 smithi130 bash[27579]: cluster 2023-08-28T01:03:15.502671+0000 mgr.y (mgr.14901) 5540 : cluster [DBG] pgmap v5547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:16.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:16 smithi130 bash[27579]: audit 2023-08-28T01:03:16.031300+0000 mon.c (mon.2) 2149 : audit [DBG] from='client.? 172.21.15.6:0/3557328042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:16.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:16 smithi006 bash[27292]: cluster 2023-08-28T01:03:15.502671+0000 mgr.y (mgr.14901) 5540 : cluster [DBG] pgmap v5547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:16.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:16 smithi006 bash[27292]: audit 2023-08-28T01:03:16.031300+0000 mon.c (mon.2) 2149 : audit [DBG] from='client.? 172.21.15.6:0/3557328042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:16.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:16 smithi006 bash[33918]: cluster 2023-08-28T01:03:15.502671+0000 mgr.y (mgr.14901) 5540 : cluster [DBG] pgmap v5547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:16.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:16 smithi006 bash[33918]: audit 2023-08-28T01:03:16.031300+0000 mon.c (mon.2) 2149 : audit [DBG] from='client.? 172.21.15.6:0/3557328042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:17.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:17 smithi130 bash[51302]: debug 2023-08-28T01:03:17.151+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:17.152852+0000) 2023-08-28T01:03:18.598 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:18 smithi130 bash[51302]: debug 2023-08-28T01:03:18.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:18.153112+0000) 2023-08-28T01:03:18.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:18 smithi130 bash[27579]: cluster 2023-08-28T01:03:17.503336+0000 mgr.y (mgr.14901) 5541 : cluster [DBG] pgmap v5548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:18.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:18 smithi130 bash[27579]: audit 2023-08-28T01:03:18.452354+0000 mon.a (mon.0) 3819 : audit [DBG] from='client.? 172.21.15.6:0/4269101529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:18.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:18 smithi006 bash[27292]: cluster 2023-08-28T01:03:17.503336+0000 mgr.y (mgr.14901) 5541 : cluster [DBG] pgmap v5548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:18.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:18 smithi006 bash[27292]: audit 2023-08-28T01:03:18.452354+0000 mon.a (mon.0) 3819 : audit [DBG] from='client.? 172.21.15.6:0/4269101529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:18 smithi006 bash[33918]: cluster 2023-08-28T01:03:17.503336+0000 mgr.y (mgr.14901) 5541 : cluster [DBG] pgmap v5548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:18.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:18 smithi006 bash[33918]: audit 2023-08-28T01:03:18.452354+0000 mon.a (mon.0) 3819 : audit [DBG] from='client.? 172.21.15.6:0/4269101529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:19.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:19 smithi130 bash[51302]: debug 2023-08-28T01:03:19.150+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:19.153429+0000) 2023-08-28T01:03:20.605 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:20 smithi130 bash[51302]: debug 2023-08-28T01:03:20.150+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:20.153738+0000) 2023-08-28T01:03:20.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:20 smithi130 bash[27579]: cluster 2023-08-28T01:03:19.504438+0000 mgr.y (mgr.14901) 5542 : cluster [DBG] pgmap v5549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:20.909 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:03:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:03:20] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:03:20.909 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:20 smithi006 bash[27292]: cluster 2023-08-28T01:03:19.504438+0000 mgr.y (mgr.14901) 5542 : cluster [DBG] pgmap v5549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:20.910 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:20 smithi006 bash[33918]: cluster 2023-08-28T01:03:19.504438+0000 mgr.y (mgr.14901) 5542 : cluster [DBG] pgmap v5549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:21.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:03:20.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:21.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:03:20.909Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:21.609 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:21 smithi130 bash[51302]: debug 2023-08-28T01:03:21.146+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:21.154009+0000) 2023-08-28T01:03:21.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:21 smithi130 bash[27579]: audit 2023-08-28T01:03:20.864742+0000 mon.c (mon.2) 2150 : audit [DBG] from='client.? 172.21.15.6:0/4235203874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:21.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:21 smithi006 bash[27292]: audit 2023-08-28T01:03:20.864742+0000 mon.c (mon.2) 2150 : audit [DBG] from='client.? 172.21.15.6:0/4235203874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:21.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:21 smithi006 bash[33918]: audit 2023-08-28T01:03:20.864742+0000 mon.c (mon.2) 2150 : audit [DBG] from='client.? 172.21.15.6:0/4235203874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:22.612 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:22 smithi130 bash[51302]: debug 2023-08-28T01:03:22.146+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:22.154286+0000) 2023-08-28T01:03:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:22 smithi130 bash[27579]: cluster 2023-08-28T01:03:21.505150+0000 mgr.y (mgr.14901) 5543 : cluster [DBG] pgmap v5550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:22.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:22 smithi006 bash[33918]: cluster 2023-08-28T01:03:21.505150+0000 mgr.y (mgr.14901) 5543 : cluster [DBG] pgmap v5550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:22.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:22 smithi006 bash[27292]: cluster 2023-08-28T01:03:21.505150+0000 mgr.y (mgr.14901) 5543 : cluster [DBG] pgmap v5550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:22.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:03:22.482Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:22.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:03:22.484Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:23.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:23 smithi130 bash[51302]: debug 2023-08-28T01:03:23.150+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:23.154572+0000) 2023-08-28T01:03:23.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:23 smithi006 bash[27292]: audit 2023-08-28T01:03:23.277932+0000 mon.a (mon.0) 3820 : audit [DBG] from='client.? 172.21.15.6:0/584073321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:23.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:23 smithi006 bash[33918]: audit 2023-08-28T01:03:23.277932+0000 mon.a (mon.0) 3820 : audit [DBG] from='client.? 172.21.15.6:0/584073321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:24.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:23 smithi130 bash[27579]: audit 2023-08-28T01:03:23.277932+0000 mon.a (mon.0) 3820 : audit [DBG] from='client.? 172.21.15.6:0/584073321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:24.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:24 smithi130 bash[51302]: debug 2023-08-28T01:03:24.150+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:24.154876+0000) 2023-08-28T01:03:24.622 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:03:24.362Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:24.622 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:03:24.364Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:24.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:24 smithi006 bash[27292]: cluster 2023-08-28T01:03:23.505836+0000 mgr.y (mgr.14901) 5544 : cluster [DBG] pgmap v5551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:24.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:24 smithi006 bash[33918]: cluster 2023-08-28T01:03:23.505836+0000 mgr.y (mgr.14901) 5544 : cluster [DBG] pgmap v5551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:24 smithi130 bash[27579]: cluster 2023-08-28T01:03:23.505836+0000 mgr.y (mgr.14901) 5544 : cluster [DBG] pgmap v5551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:25.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:25 smithi130 bash[51302]: debug 2023-08-28T01:03:25.154+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:25.155143+0000) 2023-08-28T01:03:26.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:26 smithi130 bash[51302]: debug 2023-08-28T01:03:26.154+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:26.155471+0000) 2023-08-28T01:03:26.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:26 smithi006 bash[33918]: cluster 2023-08-28T01:03:25.506935+0000 mgr.y (mgr.14901) 5545 : cluster [DBG] pgmap v5552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:26.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:26 smithi006 bash[33918]: audit 2023-08-28T01:03:25.696980+0000 mon.a (mon.0) 3821 : audit [DBG] from='client.? 172.21.15.6:0/4101095742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:26 smithi006 bash[27292]: cluster 2023-08-28T01:03:25.506935+0000 mgr.y (mgr.14901) 5545 : cluster [DBG] pgmap v5552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:26 smithi006 bash[27292]: audit 2023-08-28T01:03:25.696980+0000 mon.a (mon.0) 3821 : audit [DBG] from='client.? 172.21.15.6:0/4101095742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:26 smithi130 bash[27579]: cluster 2023-08-28T01:03:25.506935+0000 mgr.y (mgr.14901) 5545 : cluster [DBG] pgmap v5552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:26 smithi130 bash[27579]: audit 2023-08-28T01:03:25.696980+0000 mon.a (mon.0) 3821 : audit [DBG] from='client.? 172.21.15.6:0/4101095742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:27.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:27 smithi130 bash[51302]: debug 2023-08-28T01:03:27.154+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:27.155770+0000) 2023-08-28T01:03:28.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:28 smithi130 bash[51302]: debug 2023-08-28T01:03:28.154+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:28.156035+0000) 2023-08-28T01:03:28.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:28 smithi006 bash[27292]: cluster 2023-08-28T01:03:27.507762+0000 mgr.y (mgr.14901) 5546 : cluster [DBG] pgmap v5553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:28.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:28 smithi006 bash[27292]: audit 2023-08-28T01:03:28.112041+0000 mon.a (mon.0) 3822 : audit [DBG] from='client.? 172.21.15.6:0/1044239997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:28 smithi006 bash[33918]: cluster 2023-08-28T01:03:27.507762+0000 mgr.y (mgr.14901) 5546 : cluster [DBG] pgmap v5553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:28 smithi006 bash[33918]: audit 2023-08-28T01:03:28.112041+0000 mon.a (mon.0) 3822 : audit [DBG] from='client.? 172.21.15.6:0/1044239997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:28 smithi130 bash[27579]: cluster 2023-08-28T01:03:27.507762+0000 mgr.y (mgr.14901) 5546 : cluster [DBG] pgmap v5553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:28 smithi130 bash[27579]: audit 2023-08-28T01:03:28.112041+0000 mon.a (mon.0) 3822 : audit [DBG] from='client.? 172.21.15.6:0/1044239997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:29.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:29 smithi130 bash[51302]: debug 2023-08-28T01:03:29.154+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:29.156329+0000) 2023-08-28T01:03:30.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:30 smithi130 bash[51302]: debug 2023-08-28T01:03:30.154+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:30.156640+0000) 2023-08-28T01:03:30.909 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:03:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:03:30] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T01:03:30.910 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:30 smithi006 bash[27292]: cluster 2023-08-28T01:03:29.508910+0000 mgr.y (mgr.14901) 5547 : cluster [DBG] pgmap v5554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:30.910 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:30 smithi006 bash[27292]: audit 2023-08-28T01:03:30.543109+0000 mon.a (mon.0) 3823 : audit [DBG] from='client.? 172.21.15.6:0/1568958857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:30.910 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:30 smithi006 bash[33918]: cluster 2023-08-28T01:03:29.508910+0000 mgr.y (mgr.14901) 5547 : cluster [DBG] pgmap v5554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:30.910 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:30 smithi006 bash[33918]: audit 2023-08-28T01:03:30.543109+0000 mon.a (mon.0) 3823 : audit [DBG] from='client.? 172.21.15.6:0/1568958857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:30 smithi130 bash[27579]: cluster 2023-08-28T01:03:29.508910+0000 mgr.y (mgr.14901) 5547 : cluster [DBG] pgmap v5554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:30 smithi130 bash[27579]: audit 2023-08-28T01:03:30.543109+0000 mon.a (mon.0) 3823 : audit [DBG] from='client.? 172.21.15.6:0/1568958857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:31.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:03:30.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:31.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:30 smithi006 bash[56298]: level=warn ts=2023-08-28T01:03:30.910Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:31.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:31 smithi130 bash[51302]: debug 2023-08-28T01:03:31.154+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:31.156953+0000) 2023-08-28T01:03:32.464 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:32 smithi130 bash[51302]: debug 2023-08-28T01:03:32.154+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:32.157274+0000) 2023-08-28T01:03:32.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:32 smithi006 bash[33918]: cluster 2023-08-28T01:03:31.509640+0000 mgr.y (mgr.14901) 5548 : cluster [DBG] pgmap v5555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:32.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:32 smithi006 bash[27292]: cluster 2023-08-28T01:03:31.509640+0000 mgr.y (mgr.14901) 5548 : cluster [DBG] pgmap v5555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:03:32.482Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:32.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:03:32.484Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:32 smithi130 bash[27579]: cluster 2023-08-28T01:03:31.509640+0000 mgr.y (mgr.14901) 5548 : cluster [DBG] pgmap v5555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:33.466 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:33 smithi130 bash[51302]: debug 2023-08-28T01:03:33.154+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:33.157576+0000) 2023-08-28T01:03:33.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:33 smithi006 bash[33918]: audit 2023-08-28T01:03:32.962008+0000 mon.a (mon.0) 3824 : audit [DBG] from='client.? 172.21.15.6:0/1732384620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:33.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:33 smithi006 bash[27292]: audit 2023-08-28T01:03:32.962008+0000 mon.a (mon.0) 3824 : audit [DBG] from='client.? 172.21.15.6:0/1732384620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:33 smithi130 bash[27579]: audit 2023-08-28T01:03:32.962008+0000 mon.a (mon.0) 3824 : audit [DBG] from='client.? 172.21.15.6:0/1732384620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:34.474 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:34 smithi130 bash[51302]: debug 2023-08-28T01:03:34.154+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:34.157847+0000) 2023-08-28T01:03:34.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:34 smithi006 bash[27292]: cluster 2023-08-28T01:03:33.510359+0000 mgr.y (mgr.14901) 5549 : cluster [DBG] pgmap v5556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:34.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:34 smithi006 bash[33918]: cluster 2023-08-28T01:03:33.510359+0000 mgr.y (mgr.14901) 5549 : cluster [DBG] pgmap v5556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:34.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:03:34.363Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:34.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:03:34.364Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:34.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:34 smithi130 bash[27579]: cluster 2023-08-28T01:03:33.510359+0000 mgr.y (mgr.14901) 5549 : cluster [DBG] pgmap v5556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:35.478 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:35 smithi130 bash[51302]: debug 2023-08-28T01:03:35.154+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:35.158130+0000) 2023-08-28T01:03:35.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:35 smithi130 bash[27579]: audit 2023-08-28T01:03:35.375440+0000 mon.a (mon.0) 3825 : audit [DBG] from='client.? 172.21.15.6:0/2663807461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:35.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:35 smithi006 bash[33918]: audit 2023-08-28T01:03:35.375440+0000 mon.a (mon.0) 3825 : audit [DBG] from='client.? 172.21.15.6:0/2663807461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:35.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:35 smithi006 bash[27292]: audit 2023-08-28T01:03:35.375440+0000 mon.a (mon.0) 3825 : audit [DBG] from='client.? 172.21.15.6:0/2663807461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:36.488 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:36 smithi130 bash[51302]: debug 2023-08-28T01:03:36.154+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:36.158374+0000) 2023-08-28T01:03:36.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:36 smithi130 bash[27579]: cluster 2023-08-28T01:03:35.511364+0000 mgr.y (mgr.14901) 5550 : cluster [DBG] pgmap v5557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:36.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:36 smithi006 bash[33918]: cluster 2023-08-28T01:03:35.511364+0000 mgr.y (mgr.14901) 5550 : cluster [DBG] pgmap v5557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:36.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:36 smithi006 bash[27292]: cluster 2023-08-28T01:03:35.511364+0000 mgr.y (mgr.14901) 5550 : cluster [DBG] pgmap v5557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:37.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:37 smithi130 bash[51302]: debug 2023-08-28T01:03:37.154+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:37.158635+0000) 2023-08-28T01:03:38.571 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:38 smithi130 bash[51302]: debug 2023-08-28T01:03:38.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:38.158952+0000) 2023-08-28T01:03:38.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:38 smithi130 bash[27579]: cluster 2023-08-28T01:03:37.511987+0000 mgr.y (mgr.14901) 5551 : cluster [DBG] pgmap v5558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:38.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:38 smithi130 bash[27579]: audit 2023-08-28T01:03:37.795685+0000 mon.c (mon.2) 2151 : audit [DBG] from='client.? 172.21.15.6:0/1397282944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:38.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:38 smithi006 bash[33918]: cluster 2023-08-28T01:03:37.511987+0000 mgr.y (mgr.14901) 5551 : cluster [DBG] pgmap v5558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:38.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:38 smithi006 bash[33918]: audit 2023-08-28T01:03:37.795685+0000 mon.c (mon.2) 2151 : audit [DBG] from='client.? 172.21.15.6:0/1397282944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:38 smithi006 bash[27292]: cluster 2023-08-28T01:03:37.511987+0000 mgr.y (mgr.14901) 5551 : cluster [DBG] pgmap v5558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:38 smithi006 bash[27292]: audit 2023-08-28T01:03:37.795685+0000 mon.c (mon.2) 2151 : audit [DBG] from='client.? 172.21.15.6:0/1397282944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:39.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:39 smithi130 bash[51302]: debug 2023-08-28T01:03:39.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:39.159231+0000) 2023-08-28T01:03:40.576 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:40 smithi130 bash[51302]: debug 2023-08-28T01:03:40.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:40.159513+0000) 2023-08-28T01:03:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:40 smithi130 bash[27579]: cluster 2023-08-28T01:03:39.513231+0000 mgr.y (mgr.14901) 5552 : cluster [DBG] pgmap v5559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:40 smithi130 bash[27579]: audit 2023-08-28T01:03:40.047802+0000 mon.b (mon.1) 651 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:03:40.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:40 smithi130 bash[27579]: audit 2023-08-28T01:03:40.207029+0000 mon.c (mon.2) 2152 : audit [DBG] from='client.? 172.21.15.6:0/2532606881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:40.910 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:03:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:03:40] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T01:03:40.910 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:40 smithi006 bash[27292]: cluster 2023-08-28T01:03:39.513231+0000 mgr.y (mgr.14901) 5552 : cluster [DBG] pgmap v5559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:40.910 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:40 smithi006 bash[27292]: audit 2023-08-28T01:03:40.047802+0000 mon.b (mon.1) 651 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:03:40.910 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:40 smithi006 bash[27292]: audit 2023-08-28T01:03:40.207029+0000 mon.c (mon.2) 2152 : audit [DBG] from='client.? 172.21.15.6:0/2532606881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:40.911 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:40 smithi006 bash[33918]: cluster 2023-08-28T01:03:39.513231+0000 mgr.y (mgr.14901) 5552 : cluster [DBG] pgmap v5559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:40.911 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:40 smithi006 bash[33918]: audit 2023-08-28T01:03:40.047802+0000 mon.b (mon.1) 651 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:03:40.911 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:40 smithi006 bash[33918]: audit 2023-08-28T01:03:40.207029+0000 mon.c (mon.2) 2152 : audit [DBG] from='client.? 172.21.15.6:0/2532606881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:41.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:03:40.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:41.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:03:40.910Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:41.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:41 smithi130 bash[51302]: debug 2023-08-28T01:03:41.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:41.159801+0000) 2023-08-28T01:03:42.467 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:42 smithi130 bash[51302]: debug 2023-08-28T01:03:42.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:42.160082+0000) 2023-08-28T01:03:42.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:42 smithi006 bash[27292]: cluster 2023-08-28T01:03:41.514062+0000 mgr.y (mgr.14901) 5553 : cluster [DBG] pgmap v5560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:42.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:42 smithi006 bash[33918]: cluster 2023-08-28T01:03:41.514062+0000 mgr.y (mgr.14901) 5553 : cluster [DBG] pgmap v5560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:03:42.483Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:42.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:03:42.485Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:42 smithi130 bash[27579]: cluster 2023-08-28T01:03:41.514062+0000 mgr.y (mgr.14901) 5553 : cluster [DBG] pgmap v5560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:43.470 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:43 smithi130 bash[51302]: debug 2023-08-28T01:03:43.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:43.160391+0000) 2023-08-28T01:03:43.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:43 smithi130 bash[27579]: audit 2023-08-28T01:03:42.636168+0000 mon.c (mon.2) 2153 : audit [DBG] from='client.? 172.21.15.6:0/1380533381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:43.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:43 smithi006 bash[27292]: audit 2023-08-28T01:03:42.636168+0000 mon.c (mon.2) 2153 : audit [DBG] from='client.? 172.21.15.6:0/1380533381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:43.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:43 smithi006 bash[33918]: audit 2023-08-28T01:03:42.636168+0000 mon.c (mon.2) 2153 : audit [DBG] from='client.? 172.21.15.6:0/1380533381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:44.477 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:44 smithi130 bash[51302]: debug 2023-08-28T01:03:44.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:44.160701+0000) 2023-08-28T01:03:44.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:44 smithi006 bash[27292]: cluster 2023-08-28T01:03:43.514797+0000 mgr.y (mgr.14901) 5554 : cluster [DBG] pgmap v5561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:44.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:44 smithi006 bash[33918]: cluster 2023-08-28T01:03:43.514797+0000 mgr.y (mgr.14901) 5554 : cluster [DBG] pgmap v5561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:44.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:03:44.363Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:44.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:03:44.366Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:44 smithi130 bash[27579]: cluster 2023-08-28T01:03:43.514797+0000 mgr.y (mgr.14901) 5554 : cluster [DBG] pgmap v5561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:45.482 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:45 smithi130 bash[51302]: debug 2023-08-28T01:03:45.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:45.160915+0000) 2023-08-28T01:03:45.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:45 smithi130 bash[27579]: audit 2023-08-28T01:03:45.052433+0000 mon.a (mon.0) 3826 : audit [DBG] from='client.? 172.21.15.6:0/1298754756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:45.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:45 smithi006 bash[27292]: audit 2023-08-28T01:03:45.052433+0000 mon.a (mon.0) 3826 : audit [DBG] from='client.? 172.21.15.6:0/1298754756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:45.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:45 smithi006 bash[33918]: audit 2023-08-28T01:03:45.052433+0000 mon.a (mon.0) 3826 : audit [DBG] from='client.? 172.21.15.6:0/1298754756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:46.488 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:46 smithi130 bash[51302]: debug 2023-08-28T01:03:46.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:46.161210+0000) 2023-08-28T01:03:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:46 smithi130 bash[27579]: cluster 2023-08-28T01:03:45.515783+0000 mgr.y (mgr.14901) 5555 : cluster [DBG] pgmap v5562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:46 smithi130 bash[27579]: audit 2023-08-28T01:03:45.997802+0000 mon.a (mon.0) 3827 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:03:46.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:46 smithi130 bash[27579]: audit 2023-08-28T01:03:46.342536+0000 mon.a (mon.0) 3828 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:03:46.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:46 smithi006 bash[27292]: cluster 2023-08-28T01:03:45.515783+0000 mgr.y (mgr.14901) 5555 : cluster [DBG] pgmap v5562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:46 smithi006 bash[27292]: audit 2023-08-28T01:03:45.997802+0000 mon.a (mon.0) 3827 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:03:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:46 smithi006 bash[27292]: audit 2023-08-28T01:03:46.342536+0000 mon.a (mon.0) 3828 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:03:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:46 smithi006 bash[33918]: cluster 2023-08-28T01:03:45.515783+0000 mgr.y (mgr.14901) 5555 : cluster [DBG] pgmap v5562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:46 smithi006 bash[33918]: audit 2023-08-28T01:03:45.997802+0000 mon.a (mon.0) 3827 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:03:46.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:46 smithi006 bash[33918]: audit 2023-08-28T01:03:46.342536+0000 mon.a (mon.0) 3828 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:03:47.492 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:47 smithi130 bash[51302]: debug 2023-08-28T01:03:47.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:47.161427+0000) 2023-08-28T01:03:47.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:47 smithi130 bash[27579]: audit 2023-08-28T01:03:46.636431+0000 mon.b (mon.1) 652 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:03:47.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:47 smithi130 bash[27579]: audit 2023-08-28T01:03:46.638519+0000 mon.b (mon.1) 653 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:03:47.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:47 smithi130 bash[27579]: audit 2023-08-28T01:03:46.646699+0000 mon.a (mon.0) 3829 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:03:47.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:47 smithi006 bash[27292]: audit 2023-08-28T01:03:46.636431+0000 mon.b (mon.1) 652 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:03:47.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:47 smithi006 bash[27292]: audit 2023-08-28T01:03:46.638519+0000 mon.b (mon.1) 653 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:03:47.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:47 smithi006 bash[27292]: audit 2023-08-28T01:03:46.646699+0000 mon.a (mon.0) 3829 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:03:47.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:47 smithi006 bash[33918]: audit 2023-08-28T01:03:46.636431+0000 mon.b (mon.1) 652 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:03:47.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:47 smithi006 bash[33918]: audit 2023-08-28T01:03:46.638519+0000 mon.b (mon.1) 653 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:03:47.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:47 smithi006 bash[33918]: audit 2023-08-28T01:03:46.646699+0000 mon.a (mon.0) 3829 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:03:48.494 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:48 smithi130 bash[51302]: debug 2023-08-28T01:03:48.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:48.161690+0000) 2023-08-28T01:03:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:48 smithi130 bash[27579]: audit 2023-08-28T01:03:47.501394+0000 mon.a (mon.0) 3830 : audit [DBG] from='client.? 172.21.15.6:0/791353996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:48 smithi130 bash[27579]: cluster 2023-08-28T01:03:47.516480+0000 mgr.y (mgr.14901) 5556 : cluster [DBG] pgmap v5563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:48.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:48 smithi006 bash[33918]: audit 2023-08-28T01:03:47.501394+0000 mon.a (mon.0) 3830 : audit [DBG] from='client.? 172.21.15.6:0/791353996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:48.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:48 smithi006 bash[33918]: cluster 2023-08-28T01:03:47.516480+0000 mgr.y (mgr.14901) 5556 : cluster [DBG] pgmap v5563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:48 smithi006 bash[27292]: audit 2023-08-28T01:03:47.501394+0000 mon.a (mon.0) 3830 : audit [DBG] from='client.? 172.21.15.6:0/791353996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:48 smithi006 bash[27292]: cluster 2023-08-28T01:03:47.516480+0000 mgr.y (mgr.14901) 5556 : cluster [DBG] pgmap v5563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:49.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:49 smithi130 bash[51302]: debug 2023-08-28T01:03:49.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:49.161938+0000) 2023-08-28T01:03:50.577 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:50 smithi130 bash[51302]: debug 2023-08-28T01:03:50.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:50.162203+0000) 2023-08-28T01:03:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:50 smithi130 bash[27579]: cluster 2023-08-28T01:03:49.517617+0000 mgr.y (mgr.14901) 5557 : cluster [DBG] pgmap v5564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:50 smithi130 bash[27579]: audit 2023-08-28T01:03:49.910321+0000 mon.a (mon.0) 3831 : audit [DBG] from='client.? 172.21.15.6:0/578346084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:50.911 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:03:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:03:50] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:03:50.911 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:50 smithi006 bash[27292]: cluster 2023-08-28T01:03:49.517617+0000 mgr.y (mgr.14901) 5557 : cluster [DBG] pgmap v5564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:50.911 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:50 smithi006 bash[27292]: audit 2023-08-28T01:03:49.910321+0000 mon.a (mon.0) 3831 : audit [DBG] from='client.? 172.21.15.6:0/578346084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:50.912 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:50 smithi006 bash[33918]: cluster 2023-08-28T01:03:49.517617+0000 mgr.y (mgr.14901) 5557 : cluster [DBG] pgmap v5564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:50.912 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:50 smithi006 bash[33918]: audit 2023-08-28T01:03:49.910321+0000 mon.a (mon.0) 3831 : audit [DBG] from='client.? 172.21.15.6:0/578346084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:03:50.909Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:03:50.911Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:51.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:51 smithi130 bash[51302]: debug 2023-08-28T01:03:51.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:51.162513+0000) 2023-08-28T01:03:52.469 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:52 smithi130 bash[51302]: debug 2023-08-28T01:03:52.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:52.162791+0000) 2023-08-28T01:03:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:52 smithi130 bash[27579]: cluster 2023-08-28T01:03:51.518289+0000 mgr.y (mgr.14901) 5558 : cluster [DBG] pgmap v5565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:52.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:52 smithi130 bash[27579]: audit 2023-08-28T01:03:52.323138+0000 mon.c (mon.2) 2154 : audit [DBG] from='client.? 172.21.15.6:0/3683928145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:52 smithi006 bash[33918]: cluster 2023-08-28T01:03:51.518289+0000 mgr.y (mgr.14901) 5558 : cluster [DBG] pgmap v5565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:52 smithi006 bash[33918]: audit 2023-08-28T01:03:52.323138+0000 mon.c (mon.2) 2154 : audit [DBG] from='client.? 172.21.15.6:0/3683928145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:52.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:03:52.484Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:52.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:03:52.486Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:52 smithi006 bash[27292]: cluster 2023-08-28T01:03:51.518289+0000 mgr.y (mgr.14901) 5558 : cluster [DBG] pgmap v5565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:52 smithi006 bash[27292]: audit 2023-08-28T01:03:52.323138+0000 mon.c (mon.2) 2154 : audit [DBG] from='client.? 172.21.15.6:0/3683928145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:53.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:53 smithi130 bash[51302]: debug 2023-08-28T01:03:53.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:53.163127+0000) 2023-08-28T01:03:54.579 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:54 smithi130 bash[51302]: debug 2023-08-28T01:03:54.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:54.163399+0000) 2023-08-28T01:03:54.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:54 smithi006 bash[27292]: cluster 2023-08-28T01:03:53.519100+0000 mgr.y (mgr.14901) 5559 : cluster [DBG] pgmap v5566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:54.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:54 smithi006 bash[33918]: cluster 2023-08-28T01:03:53.519100+0000 mgr.y (mgr.14901) 5559 : cluster [DBG] pgmap v5566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:54.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:03:54.365Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:54.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:03:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:03:54.366Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:03:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:54 smithi130 bash[27579]: cluster 2023-08-28T01:03:53.519100+0000 mgr.y (mgr.14901) 5559 : cluster [DBG] pgmap v5566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:55.586 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:55 smithi130 bash[51302]: debug 2023-08-28T01:03:55.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:55.163675+0000) 2023-08-28T01:03:55.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:55 smithi130 bash[27579]: audit 2023-08-28T01:03:54.734172+0000 mon.c (mon.2) 2155 : audit [DBG] from='client.? 172.21.15.6:0/2874566659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:55.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:55 smithi006 bash[27292]: audit 2023-08-28T01:03:54.734172+0000 mon.c (mon.2) 2155 : audit [DBG] from='client.? 172.21.15.6:0/2874566659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:55.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:55 smithi006 bash[33918]: audit 2023-08-28T01:03:54.734172+0000 mon.c (mon.2) 2155 : audit [DBG] from='client.? 172.21.15.6:0/2874566659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:56.590 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:56 smithi130 bash[51302]: debug 2023-08-28T01:03:56.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:56.163939+0000) 2023-08-28T01:03:56.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:56 smithi130 bash[27579]: cluster 2023-08-28T01:03:55.520270+0000 mgr.y (mgr.14901) 5560 : cluster [DBG] pgmap v5567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:56.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:56 smithi006 bash[33918]: cluster 2023-08-28T01:03:55.520270+0000 mgr.y (mgr.14901) 5560 : cluster [DBG] pgmap v5567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:56.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:56 smithi006 bash[27292]: cluster 2023-08-28T01:03:55.520270+0000 mgr.y (mgr.14901) 5560 : cluster [DBG] pgmap v5567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:57.601 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:57 smithi130 bash[51302]: debug 2023-08-28T01:03:57.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:57.164191+0000) 2023-08-28T01:03:57.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:57 smithi130 bash[27579]: audit 2023-08-28T01:03:57.151824+0000 mon.a (mon.0) 3832 : audit [DBG] from='client.? 172.21.15.6:0/2609778086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:57.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:57 smithi006 bash[27292]: audit 2023-08-28T01:03:57.151824+0000 mon.a (mon.0) 3832 : audit [DBG] from='client.? 172.21.15.6:0/2609778086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:57.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:57 smithi006 bash[33918]: audit 2023-08-28T01:03:57.151824+0000 mon.a (mon.0) 3832 : audit [DBG] from='client.? 172.21.15.6:0/2609778086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:58.605 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:58 smithi130 bash[51302]: debug 2023-08-28T01:03:58.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:58.164460+0000) 2023-08-28T01:03:58.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:58 smithi130 bash[27579]: cluster 2023-08-28T01:03:57.520994+0000 mgr.y (mgr.14901) 5561 : cluster [DBG] pgmap v5568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:58.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:58 smithi006 bash[27292]: cluster 2023-08-28T01:03:57.520994+0000 mgr.y (mgr.14901) 5561 : cluster [DBG] pgmap v5568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:58.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:58 smithi006 bash[33918]: cluster 2023-08-28T01:03:57.520994+0000 mgr.y (mgr.14901) 5561 : cluster [DBG] pgmap v5568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:03:59.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:03:59 smithi130 bash[51302]: debug 2023-08-28T01:03:59.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:59.164741+0000) 2023-08-28T01:03:59.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:03:59 smithi130 bash[27579]: audit 2023-08-28T01:03:59.563806+0000 mon.a (mon.0) 3833 : audit [DBG] from='client.? 172.21.15.6:0/1203575830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:59.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:03:59 smithi006 bash[27292]: audit 2023-08-28T01:03:59.563806+0000 mon.a (mon.0) 3833 : audit [DBG] from='client.? 172.21.15.6:0/1203575830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:59.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:03:59 smithi006 bash[33918]: audit 2023-08-28T01:03:59.563806+0000 mon.a (mon.0) 3833 : audit [DBG] from='client.? 172.21.15.6:0/1203575830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:00.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:00 smithi130 bash[51302]: debug 2023-08-28T01:04:00.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:00.165024+0000) 2023-08-28T01:04:00.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:00 smithi130 bash[27579]: cluster 2023-08-28T01:03:59.521975+0000 mgr.y (mgr.14901) 5562 : cluster [DBG] pgmap v5569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:00.912 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:04:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:04:00] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:04:00.912 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:00 smithi006 bash[27292]: cluster 2023-08-28T01:03:59.521975+0000 mgr.y (mgr.14901) 5562 : cluster [DBG] pgmap v5569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:00.912 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:00 smithi006 bash[33918]: cluster 2023-08-28T01:03:59.521975+0000 mgr.y (mgr.14901) 5562 : cluster [DBG] pgmap v5569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:01.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:04:00.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:01.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:04:00.912Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:01.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:01 smithi130 bash[51302]: debug 2023-08-28T01:04:01.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:01.165327+0000) 2023-08-28T01:04:02.475 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:02 smithi130 bash[51302]: debug 2023-08-28T01:04:02.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:02.165607+0000) 2023-08-28T01:04:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:02 smithi130 bash[27579]: cluster 2023-08-28T01:04:01.522653+0000 mgr.y (mgr.14901) 5563 : cluster [DBG] pgmap v5570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:02 smithi130 bash[27579]: audit 2023-08-28T01:04:01.972709+0000 mon.c (mon.2) 2156 : audit [DBG] from='client.? 172.21.15.6:0/3853290671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:02 smithi006 bash[27292]: cluster 2023-08-28T01:04:01.522653+0000 mgr.y (mgr.14901) 5563 : cluster [DBG] pgmap v5570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:02 smithi006 bash[27292]: audit 2023-08-28T01:04:01.972709+0000 mon.c (mon.2) 2156 : audit [DBG] from='client.? 172.21.15.6:0/3853290671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:02.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:04:02.484Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:02.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:04:02.486Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:02 smithi006 bash[33918]: cluster 2023-08-28T01:04:01.522653+0000 mgr.y (mgr.14901) 5563 : cluster [DBG] pgmap v5570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:02 smithi006 bash[33918]: audit 2023-08-28T01:04:01.972709+0000 mon.c (mon.2) 2156 : audit [DBG] from='client.? 172.21.15.6:0/3853290671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:03.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:03 smithi130 bash[51302]: debug 2023-08-28T01:04:03.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:03.165896+0000) 2023-08-28T01:04:04.583 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:04 smithi130 bash[51302]: debug 2023-08-28T01:04:04.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:04.166162+0000) 2023-08-28T01:04:04.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:04 smithi006 bash[33918]: cluster 2023-08-28T01:04:03.523327+0000 mgr.y (mgr.14901) 5564 : cluster [DBG] pgmap v5571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:04.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:04 smithi006 bash[33918]: audit 2023-08-28T01:04:04.390504+0000 mon.a (mon.0) 3834 : audit [DBG] from='client.? 172.21.15.6:0/1684769371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:04:04.365Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:04.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:04:04.367Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:04.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:04 smithi006 bash[27292]: cluster 2023-08-28T01:04:03.523327+0000 mgr.y (mgr.14901) 5564 : cluster [DBG] pgmap v5571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:04.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:04 smithi006 bash[27292]: audit 2023-08-28T01:04:04.390504+0000 mon.a (mon.0) 3834 : audit [DBG] from='client.? 172.21.15.6:0/1684769371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:04 smithi130 bash[27579]: cluster 2023-08-28T01:04:03.523327+0000 mgr.y (mgr.14901) 5564 : cluster [DBG] pgmap v5571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:04 smithi130 bash[27579]: audit 2023-08-28T01:04:04.390504+0000 mon.a (mon.0) 3834 : audit [DBG] from='client.? 172.21.15.6:0/1684769371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:05.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:05 smithi130 bash[51302]: debug 2023-08-28T01:04:05.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:05.166428+0000) 2023-08-28T01:04:06.588 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:06 smithi130 bash[51302]: debug 2023-08-28T01:04:06.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:06.166699+0000) 2023-08-28T01:04:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:06 smithi130 bash[27579]: cluster 2023-08-28T01:04:05.524449+0000 mgr.y (mgr.14901) 5565 : cluster [DBG] pgmap v5572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:06.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:06 smithi006 bash[27292]: cluster 2023-08-28T01:04:05.524449+0000 mgr.y (mgr.14901) 5565 : cluster [DBG] pgmap v5572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:06.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:06 smithi006 bash[33918]: cluster 2023-08-28T01:04:05.524449+0000 mgr.y (mgr.14901) 5565 : cluster [DBG] pgmap v5572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:07.595 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:07 smithi130 bash[51302]: debug 2023-08-28T01:04:07.166+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:07.166999+0000) 2023-08-28T01:04:07.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:07 smithi130 bash[27579]: audit 2023-08-28T01:04:06.811084+0000 mon.a (mon.0) 3835 : audit [DBG] from='client.? 172.21.15.6:0/510380065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:07.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:07 smithi006 bash[33918]: audit 2023-08-28T01:04:06.811084+0000 mon.a (mon.0) 3835 : audit [DBG] from='client.? 172.21.15.6:0/510380065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:07.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:07 smithi006 bash[27292]: audit 2023-08-28T01:04:06.811084+0000 mon.a (mon.0) 3835 : audit [DBG] from='client.? 172.21.15.6:0/510380065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:08.602 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:08 smithi130 bash[51302]: debug 2023-08-28T01:04:08.166+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:08.167249+0000) 2023-08-28T01:04:08.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:08 smithi130 bash[27579]: cluster 2023-08-28T01:04:07.525088+0000 mgr.y (mgr.14901) 5566 : cluster [DBG] pgmap v5573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:08.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:08 smithi006 bash[27292]: cluster 2023-08-28T01:04:07.525088+0000 mgr.y (mgr.14901) 5566 : cluster [DBG] pgmap v5573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:08.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:08 smithi006 bash[33918]: cluster 2023-08-28T01:04:07.525088+0000 mgr.y (mgr.14901) 5566 : cluster [DBG] pgmap v5573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:09.609 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:09 smithi130 bash[51302]: debug 2023-08-28T01:04:09.166+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:09.167475+0000) 2023-08-28T01:04:09.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:09 smithi130 bash[27579]: audit 2023-08-28T01:04:09.222472+0000 mon.a (mon.0) 3836 : audit [DBG] from='client.? 172.21.15.6:0/3326609881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:09.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:09 smithi006 bash[27292]: audit 2023-08-28T01:04:09.222472+0000 mon.a (mon.0) 3836 : audit [DBG] from='client.? 172.21.15.6:0/3326609881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:09.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:09 smithi006 bash[33918]: audit 2023-08-28T01:04:09.222472+0000 mon.a (mon.0) 3836 : audit [DBG] from='client.? 172.21.15.6:0/3326609881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:10.612 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:10 smithi130 bash[51302]: debug 2023-08-28T01:04:10.166+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:10.167796+0000) 2023-08-28T01:04:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:10 smithi130 bash[27579]: cluster 2023-08-28T01:04:09.526186+0000 mgr.y (mgr.14901) 5567 : cluster [DBG] pgmap v5574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:10.913 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:10 smithi006 bash[27292]: cluster 2023-08-28T01:04:09.526186+0000 mgr.y (mgr.14901) 5567 : cluster [DBG] pgmap v5574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:10.913 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:04:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:04:10] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:04:10.913 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:10 smithi006 bash[33918]: cluster 2023-08-28T01:04:09.526186+0000 mgr.y (mgr.14901) 5567 : cluster [DBG] pgmap v5574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:11.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:04:10.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:11.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:04:10.913Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:11.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:11 smithi130 bash[51302]: debug 2023-08-28T01:04:11.166+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:11.168073+0000) 2023-08-28T01:04:12.476 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:12 smithi130 bash[51302]: debug 2023-08-28T01:04:12.166+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:12.168343+0000) 2023-08-28T01:04:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:12 smithi130 bash[27579]: cluster 2023-08-28T01:04:11.526937+0000 mgr.y (mgr.14901) 5568 : cluster [DBG] pgmap v5575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:12 smithi130 bash[27579]: audit 2023-08-28T01:04:11.626274+0000 mon.a (mon.0) 3837 : audit [DBG] from='client.? 172.21.15.6:0/508989116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:12 smithi006 bash[27292]: cluster 2023-08-28T01:04:11.526937+0000 mgr.y (mgr.14901) 5568 : cluster [DBG] pgmap v5575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:12 smithi006 bash[27292]: audit 2023-08-28T01:04:11.626274+0000 mon.a (mon.0) 3837 : audit [DBG] from='client.? 172.21.15.6:0/508989116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:12.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:04:12.485Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:12.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:04:12.487Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:12 smithi006 bash[33918]: cluster 2023-08-28T01:04:11.526937+0000 mgr.y (mgr.14901) 5568 : cluster [DBG] pgmap v5575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:12 smithi006 bash[33918]: audit 2023-08-28T01:04:11.626274+0000 mon.a (mon.0) 3837 : audit [DBG] from='client.? 172.21.15.6:0/508989116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:13.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:13 smithi130 bash[51302]: debug 2023-08-28T01:04:13.166+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:13.168669+0000) 2023-08-28T01:04:14.587 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:14 smithi130 bash[51302]: debug 2023-08-28T01:04:14.166+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:14.168944+0000) 2023-08-28T01:04:14.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:14 smithi006 bash[27292]: cluster 2023-08-28T01:04:13.527647+0000 mgr.y (mgr.14901) 5569 : cluster [DBG] pgmap v5576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:14.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:14 smithi006 bash[27292]: audit 2023-08-28T01:04:14.041583+0000 mon.c (mon.2) 2157 : audit [DBG] from='client.? 172.21.15.6:0/3608423845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:14 smithi006 bash[33918]: cluster 2023-08-28T01:04:13.527647+0000 mgr.y (mgr.14901) 5569 : cluster [DBG] pgmap v5576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:14.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:14 smithi006 bash[33918]: audit 2023-08-28T01:04:14.041583+0000 mon.c (mon.2) 2157 : audit [DBG] from='client.? 172.21.15.6:0/3608423845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:14.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:04:14.367Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:14.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:04:14.368Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:14 smithi130 bash[27579]: cluster 2023-08-28T01:04:13.527647+0000 mgr.y (mgr.14901) 5569 : cluster [DBG] pgmap v5576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:14 smithi130 bash[27579]: audit 2023-08-28T01:04:14.041583+0000 mon.c (mon.2) 2157 : audit [DBG] from='client.? 172.21.15.6:0/3608423845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:15.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:15 smithi130 bash[51302]: debug 2023-08-28T01:04:15.165+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:15.169266+0000) 2023-08-28T01:04:16.592 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:16 smithi130 bash[51302]: debug 2023-08-28T01:04:16.165+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:16.169577+0000) 2023-08-28T01:04:16.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:16 smithi130 bash[27579]: cluster 2023-08-28T01:04:15.528774+0000 mgr.y (mgr.14901) 5570 : cluster [DBG] pgmap v5577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:16.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:16 smithi130 bash[27579]: audit 2023-08-28T01:04:16.454651+0000 mon.c (mon.2) 2158 : audit [DBG] from='client.? 172.21.15.6:0/2334634508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:16.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:16 smithi006 bash[27292]: cluster 2023-08-28T01:04:15.528774+0000 mgr.y (mgr.14901) 5570 : cluster [DBG] pgmap v5577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:16.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:16 smithi006 bash[27292]: audit 2023-08-28T01:04:16.454651+0000 mon.c (mon.2) 2158 : audit [DBG] from='client.? 172.21.15.6:0/2334634508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:16.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:16 smithi006 bash[33918]: cluster 2023-08-28T01:04:15.528774+0000 mgr.y (mgr.14901) 5570 : cluster [DBG] pgmap v5577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:16 smithi006 bash[33918]: audit 2023-08-28T01:04:16.454651+0000 mon.c (mon.2) 2158 : audit [DBG] from='client.? 172.21.15.6:0/2334634508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:17.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:17 smithi130 bash[51302]: debug 2023-08-28T01:04:17.165+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:17.169892+0000) 2023-08-28T01:04:18.596 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:18 smithi130 bash[51302]: debug 2023-08-28T01:04:18.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:18.170171+0000) 2023-08-28T01:04:18.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:18 smithi130 bash[27579]: cluster 2023-08-28T01:04:17.529408+0000 mgr.y (mgr.14901) 5571 : cluster [DBG] pgmap v5578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:18.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:18 smithi006 bash[33918]: cluster 2023-08-28T01:04:17.529408+0000 mgr.y (mgr.14901) 5571 : cluster [DBG] pgmap v5578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:18.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:18 smithi006 bash[27292]: cluster 2023-08-28T01:04:17.529408+0000 mgr.y (mgr.14901) 5571 : cluster [DBG] pgmap v5578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:19.599 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:19 smithi130 bash[51302]: debug 2023-08-28T01:04:19.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:19.170484+0000) 2023-08-28T01:04:19.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:19 smithi130 bash[27579]: audit 2023-08-28T01:04:18.873984+0000 mon.c (mon.2) 2159 : audit [DBG] from='client.? 172.21.15.6:0/406807322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:19.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:19 smithi006 bash[27292]: audit 2023-08-28T01:04:18.873984+0000 mon.c (mon.2) 2159 : audit [DBG] from='client.? 172.21.15.6:0/406807322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:19.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:19 smithi006 bash[33918]: audit 2023-08-28T01:04:18.873984+0000 mon.c (mon.2) 2159 : audit [DBG] from='client.? 172.21.15.6:0/406807322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:20.608 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:20 smithi130 bash[51302]: debug 2023-08-28T01:04:20.165+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:20.170766+0000) 2023-08-28T01:04:20.874 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:20 smithi130 bash[27579]: cluster 2023-08-28T01:04:19.530500+0000 mgr.y (mgr.14901) 5572 : cluster [DBG] pgmap v5579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:20.914 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:20 smithi006 bash[27292]: cluster 2023-08-28T01:04:19.530500+0000 mgr.y (mgr.14901) 5572 : cluster [DBG] pgmap v5579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:20.914 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:04:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:04:20] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:04:20.914 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:20 smithi006 bash[33918]: cluster 2023-08-28T01:04:19.530500+0000 mgr.y (mgr.14901) 5572 : cluster [DBG] pgmap v5579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:21.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:04:20.912Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:04:20.914Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:21.611 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:21 smithi130 bash[51302]: debug 2023-08-28T01:04:21.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:21.171043+0000) 2023-08-28T01:04:21.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:21 smithi130 bash[27579]: audit 2023-08-28T01:04:21.291572+0000 mon.c (mon.2) 2160 : audit [DBG] from='client.? 172.21.15.6:0/163648134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:21.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:21 smithi006 bash[27292]: audit 2023-08-28T01:04:21.291572+0000 mon.c (mon.2) 2160 : audit [DBG] from='client.? 172.21.15.6:0/163648134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:21.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:21 smithi006 bash[33918]: audit 2023-08-28T01:04:21.291572+0000 mon.c (mon.2) 2160 : audit [DBG] from='client.? 172.21.15.6:0/163648134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:22.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:22 smithi130 bash[51302]: debug 2023-08-28T01:04:22.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:22.171300+0000) 2023-08-28T01:04:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:22 smithi130 bash[27579]: cluster 2023-08-28T01:04:21.531111+0000 mgr.y (mgr.14901) 5573 : cluster [DBG] pgmap v5580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:22.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:22 smithi006 bash[27292]: cluster 2023-08-28T01:04:21.531111+0000 mgr.y (mgr.14901) 5573 : cluster [DBG] pgmap v5580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:22.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:22 smithi006 bash[33918]: cluster 2023-08-28T01:04:21.531111+0000 mgr.y (mgr.14901) 5573 : cluster [DBG] pgmap v5580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:22.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:04:22.486Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:22.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:04:22.487Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:23.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:23 smithi130 bash[51302]: debug 2023-08-28T01:04:23.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:23.171613+0000) 2023-08-28T01:04:24.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:24 smithi130 bash[51302]: debug 2023-08-28T01:04:24.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:24.171914+0000) 2023-08-28T01:04:24.620 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:04:24.367Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:24.620 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:04:24.369Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:24.621 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:24 smithi006 bash[27292]: cluster 2023-08-28T01:04:23.531743+0000 mgr.y (mgr.14901) 5574 : cluster [DBG] pgmap v5581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:24.621 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:24 smithi006 bash[27292]: audit 2023-08-28T01:04:23.704198+0000 mon.a (mon.0) 3838 : audit [DBG] from='client.? 172.21.15.6:0/955805479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:24.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:24 smithi006 bash[33918]: cluster 2023-08-28T01:04:23.531743+0000 mgr.y (mgr.14901) 5574 : cluster [DBG] pgmap v5581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:24.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:24 smithi006 bash[33918]: audit 2023-08-28T01:04:23.704198+0000 mon.a (mon.0) 3838 : audit [DBG] from='client.? 172.21.15.6:0/955805479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:24 smithi130 bash[27579]: cluster 2023-08-28T01:04:23.531743+0000 mgr.y (mgr.14901) 5574 : cluster [DBG] pgmap v5581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:24 smithi130 bash[27579]: audit 2023-08-28T01:04:23.704198+0000 mon.a (mon.0) 3838 : audit [DBG] from='client.? 172.21.15.6:0/955805479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:25.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:25 smithi130 bash[51302]: debug 2023-08-28T01:04:25.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:25.172182+0000) 2023-08-28T01:04:26.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:26 smithi130 bash[51302]: debug 2023-08-28T01:04:26.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:26.172454+0000) 2023-08-28T01:04:26.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:26 smithi006 bash[27292]: cluster 2023-08-28T01:04:25.532913+0000 mgr.y (mgr.14901) 5575 : cluster [DBG] pgmap v5582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:26.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:26 smithi006 bash[27292]: audit 2023-08-28T01:04:26.116625+0000 mon.c (mon.2) 2161 : audit [DBG] from='client.? 172.21.15.6:0/2066807303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:26.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:26 smithi006 bash[33918]: cluster 2023-08-28T01:04:25.532913+0000 mgr.y (mgr.14901) 5575 : cluster [DBG] pgmap v5582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:26 smithi006 bash[33918]: audit 2023-08-28T01:04:26.116625+0000 mon.c (mon.2) 2161 : audit [DBG] from='client.? 172.21.15.6:0/2066807303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:26 smithi130 bash[27579]: cluster 2023-08-28T01:04:25.532913+0000 mgr.y (mgr.14901) 5575 : cluster [DBG] pgmap v5582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:26 smithi130 bash[27579]: audit 2023-08-28T01:04:26.116625+0000 mon.c (mon.2) 2161 : audit [DBG] from='client.? 172.21.15.6:0/2066807303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:27.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:27 smithi130 bash[51302]: debug 2023-08-28T01:04:27.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:27.172719+0000) 2023-08-28T01:04:28.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:28 smithi130 bash[51302]: debug 2023-08-28T01:04:28.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:28.172928+0000) 2023-08-28T01:04:28.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:28 smithi006 bash[27292]: cluster 2023-08-28T01:04:27.533590+0000 mgr.y (mgr.14901) 5576 : cluster [DBG] pgmap v5583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:28.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:28 smithi006 bash[27292]: audit 2023-08-28T01:04:28.536263+0000 mon.c (mon.2) 2162 : audit [DBG] from='client.? 172.21.15.6:0/1934752009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:28.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:28 smithi006 bash[33918]: cluster 2023-08-28T01:04:27.533590+0000 mgr.y (mgr.14901) 5576 : cluster [DBG] pgmap v5583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:28.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:28 smithi006 bash[33918]: audit 2023-08-28T01:04:28.536263+0000 mon.c (mon.2) 2162 : audit [DBG] from='client.? 172.21.15.6:0/1934752009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:28 smithi130 bash[27579]: cluster 2023-08-28T01:04:27.533590+0000 mgr.y (mgr.14901) 5576 : cluster [DBG] pgmap v5583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:28 smithi130 bash[27579]: audit 2023-08-28T01:04:28.536263+0000 mon.c (mon.2) 2162 : audit [DBG] from='client.? 172.21.15.6:0/1934752009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:29.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:29 smithi130 bash[51302]: debug 2023-08-28T01:04:29.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:29.173208+0000) 2023-08-28T01:04:30.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:30 smithi130 bash[51302]: debug 2023-08-28T01:04:30.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:30.173582+0000) 2023-08-28T01:04:30.914 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:04:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:04:30] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:04:30.914 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:30 smithi006 bash[33918]: cluster 2023-08-28T01:04:29.534676+0000 mgr.y (mgr.14901) 5577 : cluster [DBG] pgmap v5584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:30.914 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:30 smithi006 bash[27292]: cluster 2023-08-28T01:04:29.534676+0000 mgr.y (mgr.14901) 5577 : cluster [DBG] pgmap v5584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:30 smithi130 bash[27579]: cluster 2023-08-28T01:04:29.534676+0000 mgr.y (mgr.14901) 5577 : cluster [DBG] pgmap v5584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:31.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:04:30.912Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:31.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:30 smithi006 bash[56298]: level=warn ts=2023-08-28T01:04:30.914Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:31.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:31 smithi130 bash[51302]: debug 2023-08-28T01:04:31.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:31.173953+0000) 2023-08-28T01:04:31.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:31 smithi006 bash[33918]: audit 2023-08-28T01:04:30.956167+0000 mon.a (mon.0) 3839 : audit [DBG] from='client.? 172.21.15.6:0/3761799307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:31.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:31 smithi006 bash[27292]: audit 2023-08-28T01:04:30.956167+0000 mon.a (mon.0) 3839 : audit [DBG] from='client.? 172.21.15.6:0/3761799307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:32.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:31 smithi130 bash[27579]: audit 2023-08-28T01:04:30.956167+0000 mon.a (mon.0) 3839 : audit [DBG] from='client.? 172.21.15.6:0/3761799307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:32.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:32 smithi130 bash[51302]: debug 2023-08-28T01:04:32.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:32.174270+0000) 2023-08-28T01:04:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:32 smithi006 bash[27292]: cluster 2023-08-28T01:04:31.535346+0000 mgr.y (mgr.14901) 5578 : cluster [DBG] pgmap v5585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:32 smithi006 bash[33918]: cluster 2023-08-28T01:04:31.535346+0000 mgr.y (mgr.14901) 5578 : cluster [DBG] pgmap v5585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:32.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:04:32.487Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:32.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:04:32.488Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:33.119 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:32 smithi130 bash[27579]: cluster 2023-08-28T01:04:31.535346+0000 mgr.y (mgr.14901) 5578 : cluster [DBG] pgmap v5585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:33.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:33 smithi130 bash[51302]: debug 2023-08-28T01:04:33.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:33.174548+0000) 2023-08-28T01:04:33.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:33 smithi006 bash[33918]: audit 2023-08-28T01:04:33.379459+0000 mon.c (mon.2) 2163 : audit [DBG] from='client.? 172.21.15.6:0/34011391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:33.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:33 smithi006 bash[27292]: audit 2023-08-28T01:04:33.379459+0000 mon.c (mon.2) 2163 : audit [DBG] from='client.? 172.21.15.6:0/34011391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:34.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:33 smithi130 bash[27579]: audit 2023-08-28T01:04:33.379459+0000 mon.c (mon.2) 2163 : audit [DBG] from='client.? 172.21.15.6:0/34011391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:34.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:34 smithi130 bash[51302]: debug 2023-08-28T01:04:34.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:34.174825+0000) 2023-08-28T01:04:34.653 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:04:34.368Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:34.653 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:04:34.370Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:34.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:34 smithi006 bash[27292]: cluster 2023-08-28T01:04:33.536144+0000 mgr.y (mgr.14901) 5579 : cluster [DBG] pgmap v5586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:34.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:34 smithi006 bash[33918]: cluster 2023-08-28T01:04:33.536144+0000 mgr.y (mgr.14901) 5579 : cluster [DBG] pgmap v5586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:34 smithi130 bash[27579]: cluster 2023-08-28T01:04:33.536144+0000 mgr.y (mgr.14901) 5579 : cluster [DBG] pgmap v5586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:35.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:35 smithi130 bash[51302]: debug 2023-08-28T01:04:35.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:35.175118+0000) 2023-08-28T01:04:36.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:36 smithi130 bash[51302]: debug 2023-08-28T01:04:36.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:36.175362+0000) 2023-08-28T01:04:36.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:36 smithi006 bash[33918]: cluster 2023-08-28T01:04:35.536965+0000 mgr.y (mgr.14901) 5580 : cluster [DBG] pgmap v5587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:36.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:36 smithi006 bash[33918]: audit 2023-08-28T01:04:35.798375+0000 mon.a (mon.0) 3840 : audit [DBG] from='client.? 172.21.15.6:0/1502160894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:36.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:36 smithi006 bash[27292]: cluster 2023-08-28T01:04:35.536965+0000 mgr.y (mgr.14901) 5580 : cluster [DBG] pgmap v5587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:36.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:36 smithi006 bash[27292]: audit 2023-08-28T01:04:35.798375+0000 mon.a (mon.0) 3840 : audit [DBG] from='client.? 172.21.15.6:0/1502160894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:36 smithi130 bash[27579]: cluster 2023-08-28T01:04:35.536965+0000 mgr.y (mgr.14901) 5580 : cluster [DBG] pgmap v5587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:36 smithi130 bash[27579]: audit 2023-08-28T01:04:35.798375+0000 mon.a (mon.0) 3840 : audit [DBG] from='client.? 172.21.15.6:0/1502160894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:37.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:37 smithi130 bash[51302]: debug 2023-08-28T01:04:37.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:37.175686+0000) 2023-08-28T01:04:38.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:38 smithi130 bash[51302]: debug 2023-08-28T01:04:38.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:38.176013+0000) 2023-08-28T01:04:38.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:38 smithi006 bash[27292]: cluster 2023-08-28T01:04:37.537595+0000 mgr.y (mgr.14901) 5581 : cluster [DBG] pgmap v5588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:38.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:38 smithi006 bash[27292]: audit 2023-08-28T01:04:38.220402+0000 mon.a (mon.0) 3841 : audit [DBG] from='client.? 172.21.15.6:0/532647700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:38.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:38 smithi006 bash[33918]: cluster 2023-08-28T01:04:37.537595+0000 mgr.y (mgr.14901) 5581 : cluster [DBG] pgmap v5588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:38.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:38 smithi006 bash[33918]: audit 2023-08-28T01:04:38.220402+0000 mon.a (mon.0) 3841 : audit [DBG] from='client.? 172.21.15.6:0/532647700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:38 smithi130 bash[27579]: cluster 2023-08-28T01:04:37.537595+0000 mgr.y (mgr.14901) 5581 : cluster [DBG] pgmap v5588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:38 smithi130 bash[27579]: audit 2023-08-28T01:04:38.220402+0000 mon.a (mon.0) 3841 : audit [DBG] from='client.? 172.21.15.6:0/532647700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:39.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:39 smithi130 bash[51302]: debug 2023-08-28T01:04:39.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:39.176306+0000) 2023-08-28T01:04:40.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:40 smithi130 bash[51302]: debug 2023-08-28T01:04:40.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:40.176591+0000) 2023-08-28T01:04:40.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:04:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:04:40] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:04:40.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:40 smithi006 bash[27292]: cluster 2023-08-28T01:04:39.538713+0000 mgr.y (mgr.14901) 5582 : cluster [DBG] pgmap v5589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:40.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:40 smithi006 bash[27292]: audit 2023-08-28T01:04:40.657719+0000 mon.c (mon.2) 2164 : audit [DBG] from='client.? 172.21.15.6:0/429335067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:40.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:40 smithi006 bash[33918]: cluster 2023-08-28T01:04:39.538713+0000 mgr.y (mgr.14901) 5582 : cluster [DBG] pgmap v5589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:40.968 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:40 smithi006 bash[33918]: audit 2023-08-28T01:04:40.657719+0000 mon.c (mon.2) 2164 : audit [DBG] from='client.? 172.21.15.6:0/429335067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:40.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:04:40.912Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:40.969 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:04:40.915Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:40 smithi130 bash[27579]: cluster 2023-08-28T01:04:39.538713+0000 mgr.y (mgr.14901) 5582 : cluster [DBG] pgmap v5589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:40 smithi130 bash[27579]: audit 2023-08-28T01:04:40.657719+0000 mon.c (mon.2) 2164 : audit [DBG] from='client.? 172.21.15.6:0/429335067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:41.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:41 smithi130 bash[51302]: debug 2023-08-28T01:04:41.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:41.176873+0000) 2023-08-28T01:04:42.503 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:42 smithi130 bash[51302]: debug 2023-08-28T01:04:42.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:42.177189+0000) 2023-08-28T01:04:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:42 smithi130 bash[27579]: cluster 2023-08-28T01:04:41.539162+0000 mgr.y (mgr.14901) 5583 : cluster [DBG] pgmap v5590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:42.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:42 smithi006 bash[33918]: cluster 2023-08-28T01:04:41.539162+0000 mgr.y (mgr.14901) 5583 : cluster [DBG] pgmap v5590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:04:42.487Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:04:42.489Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:42 smithi006 bash[27292]: cluster 2023-08-28T01:04:41.539162+0000 mgr.y (mgr.14901) 5583 : cluster [DBG] pgmap v5590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:43.506 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:43 smithi130 bash[51302]: debug 2023-08-28T01:04:43.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:43.177486+0000) 2023-08-28T01:04:43.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:43 smithi130 bash[27579]: audit 2023-08-28T01:04:43.069380+0000 mon.c (mon.2) 2165 : audit [DBG] from='client.? 172.21.15.6:0/2341074694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:43.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:43 smithi006 bash[33918]: audit 2023-08-28T01:04:43.069380+0000 mon.c (mon.2) 2165 : audit [DBG] from='client.? 172.21.15.6:0/2341074694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:43.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:43 smithi006 bash[27292]: audit 2023-08-28T01:04:43.069380+0000 mon.c (mon.2) 2165 : audit [DBG] from='client.? 172.21.15.6:0/2341074694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:44.515 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:44 smithi130 bash[51302]: debug 2023-08-28T01:04:44.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:44.177789+0000) 2023-08-28T01:04:44.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:44 smithi006 bash[33918]: cluster 2023-08-28T01:04:43.539799+0000 mgr.y (mgr.14901) 5584 : cluster [DBG] pgmap v5591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:44.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:44 smithi006 bash[27292]: cluster 2023-08-28T01:04:43.539799+0000 mgr.y (mgr.14901) 5584 : cluster [DBG] pgmap v5591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:44.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:04:44.369Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:44.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:04:44.371Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:44 smithi130 bash[27579]: cluster 2023-08-28T01:04:43.539799+0000 mgr.y (mgr.14901) 5584 : cluster [DBG] pgmap v5591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:45.518 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:45 smithi130 bash[51302]: debug 2023-08-28T01:04:45.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:45.178078+0000) 2023-08-28T01:04:45.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:45 smithi130 bash[27579]: audit 2023-08-28T01:04:45.484858+0000 mon.a (mon.0) 3842 : audit [DBG] from='client.? 172.21.15.6:0/2552354624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:45.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:45 smithi006 bash[27292]: audit 2023-08-28T01:04:45.484858+0000 mon.a (mon.0) 3842 : audit [DBG] from='client.? 172.21.15.6:0/2552354624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:45.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:45 smithi006 bash[33918]: audit 2023-08-28T01:04:45.484858+0000 mon.a (mon.0) 3842 : audit [DBG] from='client.? 172.21.15.6:0/2552354624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:46.528 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:46 smithi130 bash[51302]: debug 2023-08-28T01:04:46.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:46.178386+0000) 2023-08-28T01:04:46.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:46 smithi130 bash[27579]: cluster 2023-08-28T01:04:45.540952+0000 mgr.y (mgr.14901) 5585 : cluster [DBG] pgmap v5592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:46 smithi006 bash[33918]: cluster 2023-08-28T01:04:45.540952+0000 mgr.y (mgr.14901) 5585 : cluster [DBG] pgmap v5592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:46.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:46 smithi006 bash[27292]: cluster 2023-08-28T01:04:45.540952+0000 mgr.y (mgr.14901) 5585 : cluster [DBG] pgmap v5592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:47.532 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:47 smithi130 bash[51302]: debug 2023-08-28T01:04:47.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:47.178648+0000) 2023-08-28T01:04:47.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:47 smithi130 bash[27579]: audit 2023-08-28T01:04:46.657820+0000 mon.b (mon.1) 654 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:04:47.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:47 smithi006 bash[33918]: audit 2023-08-28T01:04:46.657820+0000 mon.b (mon.1) 654 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:04:47.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:47 smithi006 bash[27292]: audit 2023-08-28T01:04:46.657820+0000 mon.b (mon.1) 654 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:04:48.539 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:48 smithi130 bash[51302]: debug 2023-08-28T01:04:48.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:48.178990+0000) 2023-08-28T01:04:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:48 smithi130 bash[27579]: cluster 2023-08-28T01:04:47.541585+0000 mgr.y (mgr.14901) 5586 : cluster [DBG] pgmap v5593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:48.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:48 smithi130 bash[27579]: audit 2023-08-28T01:04:47.912743+0000 mon.b (mon.1) 655 : audit [DBG] from='client.? 172.21.15.6:0/673137330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:48.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:48 smithi006 bash[27292]: cluster 2023-08-28T01:04:47.541585+0000 mgr.y (mgr.14901) 5586 : cluster [DBG] pgmap v5593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:48.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:48 smithi006 bash[27292]: audit 2023-08-28T01:04:47.912743+0000 mon.b (mon.1) 655 : audit [DBG] from='client.? 172.21.15.6:0/673137330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:48.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:48 smithi006 bash[33918]: cluster 2023-08-28T01:04:47.541585+0000 mgr.y (mgr.14901) 5586 : cluster [DBG] pgmap v5593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:48.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:48 smithi006 bash[33918]: audit 2023-08-28T01:04:47.912743+0000 mon.b (mon.1) 655 : audit [DBG] from='client.? 172.21.15.6:0/673137330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:49.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:49 smithi130 bash[51302]: debug 2023-08-28T01:04:49.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:49.179303+0000) 2023-08-28T01:04:50.602 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:50 smithi130 bash[51302]: debug 2023-08-28T01:04:50.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:50.179545+0000) 2023-08-28T01:04:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:50 smithi130 bash[27579]: cluster 2023-08-28T01:04:49.542761+0000 mgr.y (mgr.14901) 5587 : cluster [DBG] pgmap v5594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:50.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:50 smithi130 bash[27579]: audit 2023-08-28T01:04:50.326014+0000 mon.a (mon.0) 3843 : audit [DBG] from='client.? 172.21.15.6:0/1388844991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:50.915 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:04:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:04:50] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:04:50.915 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:50 smithi006 bash[27292]: cluster 2023-08-28T01:04:49.542761+0000 mgr.y (mgr.14901) 5587 : cluster [DBG] pgmap v5594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:50.916 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:50 smithi006 bash[27292]: audit 2023-08-28T01:04:50.326014+0000 mon.a (mon.0) 3843 : audit [DBG] from='client.? 172.21.15.6:0/1388844991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:50.916 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:50 smithi006 bash[33918]: cluster 2023-08-28T01:04:49.542761+0000 mgr.y (mgr.14901) 5587 : cluster [DBG] pgmap v5594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:50.916 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:50 smithi006 bash[33918]: audit 2023-08-28T01:04:50.326014+0000 mon.a (mon.0) 3843 : audit [DBG] from='client.? 172.21.15.6:0/1388844991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:04:50.913Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:04:50.915Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:51.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:51 smithi130 bash[51302]: debug 2023-08-28T01:04:51.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:51.179862+0000) 2023-08-28T01:04:52.503 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:52 smithi130 bash[51302]: debug 2023-08-28T01:04:52.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:52.180079+0000) 2023-08-28T01:04:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:52 smithi130 bash[27579]: cluster 2023-08-28T01:04:51.543334+0000 mgr.y (mgr.14901) 5588 : cluster [DBG] pgmap v5595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:52 smithi006 bash[33918]: cluster 2023-08-28T01:04:51.543334+0000 mgr.y (mgr.14901) 5588 : cluster [DBG] pgmap v5595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:52 smithi006 bash[27292]: cluster 2023-08-28T01:04:51.543334+0000 mgr.y (mgr.14901) 5588 : cluster [DBG] pgmap v5595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:52.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:04:52.488Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:52.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:04:52.490Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:53.562 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:53 smithi130 bash[51302]: debug 2023-08-28T01:04:53.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:53.180371+0000) 2023-08-28T01:04:53.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:53 smithi130 bash[27579]: audit 2023-08-28T01:04:52.560625+0000 mon.a (mon.0) 3844 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:04:53.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:53 smithi130 bash[27579]: audit 2023-08-28T01:04:52.747118+0000 mon.a (mon.0) 3845 : audit [DBG] from='client.? 172.21.15.6:0/3613650241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:53.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:53 smithi130 bash[27579]: audit 2023-08-28T01:04:52.926333+0000 mon.a (mon.0) 3846 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:04:53.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:53 smithi130 bash[27579]: audit 2023-08-28T01:04:53.194271+0000 mon.b (mon.1) 656 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:04:53.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:53 smithi130 bash[27579]: audit 2023-08-28T01:04:53.196112+0000 mon.b (mon.1) 657 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:04:53.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:53 smithi130 bash[27579]: audit 2023-08-28T01:04:53.203594+0000 mon.a (mon.0) 3847 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:04:53.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:53 smithi006 bash[27292]: audit 2023-08-28T01:04:52.560625+0000 mon.a (mon.0) 3844 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:04:53.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:53 smithi006 bash[27292]: audit 2023-08-28T01:04:52.747118+0000 mon.a (mon.0) 3845 : audit [DBG] from='client.? 172.21.15.6:0/3613650241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:53.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:53 smithi006 bash[27292]: audit 2023-08-28T01:04:52.926333+0000 mon.a (mon.0) 3846 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:04:53.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:53 smithi006 bash[27292]: audit 2023-08-28T01:04:53.194271+0000 mon.b (mon.1) 656 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:04:53.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:53 smithi006 bash[27292]: audit 2023-08-28T01:04:53.196112+0000 mon.b (mon.1) 657 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:04:53.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:53 smithi006 bash[27292]: audit 2023-08-28T01:04:53.203594+0000 mon.a (mon.0) 3847 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:04:53.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:53 smithi006 bash[33918]: audit 2023-08-28T01:04:52.560625+0000 mon.a (mon.0) 3844 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:04:53.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:53 smithi006 bash[33918]: audit 2023-08-28T01:04:52.747118+0000 mon.a (mon.0) 3845 : audit [DBG] from='client.? 172.21.15.6:0/3613650241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:53.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:53 smithi006 bash[33918]: audit 2023-08-28T01:04:52.926333+0000 mon.a (mon.0) 3846 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:04:53.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:53 smithi006 bash[33918]: audit 2023-08-28T01:04:53.194271+0000 mon.b (mon.1) 656 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:04:53.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:53 smithi006 bash[33918]: audit 2023-08-28T01:04:53.196112+0000 mon.b (mon.1) 657 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:04:53.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:53 smithi006 bash[33918]: audit 2023-08-28T01:04:53.203594+0000 mon.a (mon.0) 3847 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:04:54.565 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:54 smithi130 bash[51302]: debug 2023-08-28T01:04:54.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:54.180719+0000) 2023-08-28T01:04:54.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:54 smithi006 bash[27292]: cluster 2023-08-28T01:04:53.543945+0000 mgr.y (mgr.14901) 5589 : cluster [DBG] pgmap v5596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:54.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:54 smithi006 bash[33918]: cluster 2023-08-28T01:04:53.543945+0000 mgr.y (mgr.14901) 5589 : cluster [DBG] pgmap v5596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:54.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:04:54.370Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:54.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:04:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:04:54.371Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:04:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:54 smithi130 bash[27579]: cluster 2023-08-28T01:04:53.543945+0000 mgr.y (mgr.14901) 5589 : cluster [DBG] pgmap v5596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:55.573 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:55 smithi130 bash[51302]: debug 2023-08-28T01:04:55.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:55.181051+0000) 2023-08-28T01:04:55.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:55 smithi130 bash[27579]: audit 2023-08-28T01:04:55.164922+0000 mon.a (mon.0) 3848 : audit [DBG] from='client.? 172.21.15.6:0/1253913752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:55.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:55 smithi006 bash[27292]: audit 2023-08-28T01:04:55.164922+0000 mon.a (mon.0) 3848 : audit [DBG] from='client.? 172.21.15.6:0/1253913752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:55.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:55 smithi006 bash[33918]: audit 2023-08-28T01:04:55.164922+0000 mon.a (mon.0) 3848 : audit [DBG] from='client.? 172.21.15.6:0/1253913752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:56.576 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:56 smithi130 bash[51302]: debug 2023-08-28T01:04:56.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:56.181382+0000) 2023-08-28T01:04:56.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:56 smithi130 bash[27579]: cluster 2023-08-28T01:04:55.545089+0000 mgr.y (mgr.14901) 5590 : cluster [DBG] pgmap v5597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:56.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:56 smithi006 bash[33918]: cluster 2023-08-28T01:04:55.545089+0000 mgr.y (mgr.14901) 5590 : cluster [DBG] pgmap v5597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:56.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:56 smithi006 bash[27292]: cluster 2023-08-28T01:04:55.545089+0000 mgr.y (mgr.14901) 5590 : cluster [DBG] pgmap v5597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:57.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:57 smithi130 bash[51302]: debug 2023-08-28T01:04:57.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:57.181699+0000) 2023-08-28T01:04:57.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:57 smithi006 bash[33918]: audit 2023-08-28T01:04:57.574718+0000 mon.a (mon.0) 3849 : audit [DBG] from='client.? 172.21.15.6:0/2924008249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:57.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:57 smithi006 bash[27292]: audit 2023-08-28T01:04:57.574718+0000 mon.a (mon.0) 3849 : audit [DBG] from='client.? 172.21.15.6:0/2924008249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:58.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:57 smithi130 bash[27579]: audit 2023-08-28T01:04:57.574718+0000 mon.a (mon.0) 3849 : audit [DBG] from='client.? 172.21.15.6:0/2924008249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:58.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:58 smithi130 bash[51302]: debug 2023-08-28T01:04:58.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:58.182044+0000) 2023-08-28T01:04:58.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:04:58 smithi006 bash[33918]: cluster 2023-08-28T01:04:57.545762+0000 mgr.y (mgr.14901) 5591 : cluster [DBG] pgmap v5598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:58.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:04:58 smithi006 bash[27292]: cluster 2023-08-28T01:04:57.545762+0000 mgr.y (mgr.14901) 5591 : cluster [DBG] pgmap v5598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:04:58 smithi130 bash[27579]: cluster 2023-08-28T01:04:57.545762+0000 mgr.y (mgr.14901) 5591 : cluster [DBG] pgmap v5598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:04:59.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:04:59 smithi130 bash[51302]: debug 2023-08-28T01:04:59.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:59.182374+0000) 2023-08-28T01:05:00.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:00 smithi130 bash[51302]: debug 2023-08-28T01:05:00.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:00.182721+0000) 2023-08-28T01:05:00.915 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:05:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:05:00] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:05:00.916 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:00 smithi006 bash[27292]: cluster 2023-08-28T01:04:59.546875+0000 mgr.y (mgr.14901) 5592 : cluster [DBG] pgmap v5599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:00.916 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:00 smithi006 bash[27292]: audit 2023-08-28T01:04:59.989655+0000 mon.c (mon.2) 2166 : audit [DBG] from='client.? 172.21.15.6:0/2474770287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:00.916 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:00 smithi006 bash[33918]: cluster 2023-08-28T01:04:59.546875+0000 mgr.y (mgr.14901) 5592 : cluster [DBG] pgmap v5599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:00.916 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:00 smithi006 bash[33918]: audit 2023-08-28T01:04:59.989655+0000 mon.c (mon.2) 2166 : audit [DBG] from='client.? 172.21.15.6:0/2474770287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:00 smithi130 bash[27579]: cluster 2023-08-28T01:04:59.546875+0000 mgr.y (mgr.14901) 5592 : cluster [DBG] pgmap v5599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:00 smithi130 bash[27579]: audit 2023-08-28T01:04:59.989655+0000 mon.c (mon.2) 2166 : audit [DBG] from='client.? 172.21.15.6:0/2474770287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:01.189 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:05:00.914Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:01.190 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:05:00.915Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:01.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:01 smithi130 bash[51302]: debug 2023-08-28T01:05:01.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:01.183042+0000) 2023-08-28T01:05:02.367 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:02 smithi130 bash[51302]: debug 2023-08-28T01:05:02.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:02.183337+0000) 2023-08-28T01:05:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:02 smithi130 bash[27579]: cluster 2023-08-28T01:05:01.547555+0000 mgr.y (mgr.14901) 5593 : cluster [DBG] pgmap v5600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:02 smithi130 bash[27579]: audit 2023-08-28T01:05:02.412481+0000 mon.a (mon.0) 3850 : audit [DBG] from='client.? 172.21.15.6:0/2679705544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:02 smithi006 bash[33918]: cluster 2023-08-28T01:05:01.547555+0000 mgr.y (mgr.14901) 5593 : cluster [DBG] pgmap v5600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:02 smithi006 bash[33918]: audit 2023-08-28T01:05:02.412481+0000 mon.a (mon.0) 3850 : audit [DBG] from='client.? 172.21.15.6:0/2679705544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:02.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:05:02.489Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:02.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:05:02.491Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:02 smithi006 bash[27292]: cluster 2023-08-28T01:05:01.547555+0000 mgr.y (mgr.14901) 5593 : cluster [DBG] pgmap v5600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:02 smithi006 bash[27292]: audit 2023-08-28T01:05:02.412481+0000 mon.a (mon.0) 3850 : audit [DBG] from='client.? 172.21.15.6:0/2679705544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:03.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:03 smithi130 bash[51302]: debug 2023-08-28T01:05:03.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:03.183689+0000) 2023-08-28T01:05:04.607 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:04 smithi130 bash[51302]: debug 2023-08-28T01:05:04.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:04.184021+0000) 2023-08-28T01:05:04.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:04 smithi006 bash[33918]: cluster 2023-08-28T01:05:03.548196+0000 mgr.y (mgr.14901) 5594 : cluster [DBG] pgmap v5601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:04.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:04 smithi006 bash[27292]: cluster 2023-08-28T01:05:03.548196+0000 mgr.y (mgr.14901) 5594 : cluster [DBG] pgmap v5601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:04.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:05:04.371Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:04.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:05:04.373Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:04 smithi130 bash[27579]: cluster 2023-08-28T01:05:03.548196+0000 mgr.y (mgr.14901) 5594 : cluster [DBG] pgmap v5601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:05.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:05 smithi130 bash[51302]: debug 2023-08-28T01:05:05.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:05.184381+0000) 2023-08-28T01:05:05.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:05 smithi006 bash[33918]: audit 2023-08-28T01:05:04.840973+0000 mon.c (mon.2) 2167 : audit [DBG] from='client.? 172.21.15.6:0/3907776143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:05.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:05 smithi006 bash[27292]: audit 2023-08-28T01:05:04.840973+0000 mon.c (mon.2) 2167 : audit [DBG] from='client.? 172.21.15.6:0/3907776143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:06.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:05 smithi130 bash[27579]: audit 2023-08-28T01:05:04.840973+0000 mon.c (mon.2) 2167 : audit [DBG] from='client.? 172.21.15.6:0/3907776143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:06.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:06 smithi130 bash[51302]: debug 2023-08-28T01:05:06.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:06.184695+0000) 2023-08-28T01:05:06.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:06 smithi006 bash[27292]: cluster 2023-08-28T01:05:05.549352+0000 mgr.y (mgr.14901) 5595 : cluster [DBG] pgmap v5602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:06.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:06 smithi006 bash[33918]: cluster 2023-08-28T01:05:05.549352+0000 mgr.y (mgr.14901) 5595 : cluster [DBG] pgmap v5602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:06 smithi130 bash[27579]: cluster 2023-08-28T01:05:05.549352+0000 mgr.y (mgr.14901) 5595 : cluster [DBG] pgmap v5602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:07.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:07 smithi130 bash[51302]: debug 2023-08-28T01:05:07.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:07.185034+0000) 2023-08-28T01:05:07.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:07 smithi006 bash[33918]: audit 2023-08-28T01:05:07.259925+0000 mon.c (mon.2) 2168 : audit [DBG] from='client.? 172.21.15.6:0/902600062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:07.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:07 smithi006 bash[27292]: audit 2023-08-28T01:05:07.259925+0000 mon.c (mon.2) 2168 : audit [DBG] from='client.? 172.21.15.6:0/902600062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:08.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:07 smithi130 bash[27579]: audit 2023-08-28T01:05:07.259925+0000 mon.c (mon.2) 2168 : audit [DBG] from='client.? 172.21.15.6:0/902600062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:08.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:08 smithi130 bash[51302]: debug 2023-08-28T01:05:08.185+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:08.185398+0000) 2023-08-28T01:05:08.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:08 smithi006 bash[33918]: cluster 2023-08-28T01:05:07.550009+0000 mgr.y (mgr.14901) 5596 : cluster [DBG] pgmap v5603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:08.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:08 smithi006 bash[27292]: cluster 2023-08-28T01:05:07.550009+0000 mgr.y (mgr.14901) 5596 : cluster [DBG] pgmap v5603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:08 smithi130 bash[27579]: cluster 2023-08-28T01:05:07.550009+0000 mgr.y (mgr.14901) 5596 : cluster [DBG] pgmap v5603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:09.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:09 smithi130 bash[51302]: debug 2023-08-28T01:05:09.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:09.185729+0000) 2023-08-28T01:05:10.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:10 smithi130 bash[51302]: debug 2023-08-28T01:05:10.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:10.186080+0000) 2023-08-28T01:05:10.916 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:05:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:05:10] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:05:10.916 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:10 smithi006 bash[33918]: cluster 2023-08-28T01:05:09.551101+0000 mgr.y (mgr.14901) 5597 : cluster [DBG] pgmap v5604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:10.917 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:10 smithi006 bash[33918]: audit 2023-08-28T01:05:09.667959+0000 mon.c (mon.2) 2169 : audit [DBG] from='client.? 172.21.15.6:0/2477778387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:10.917 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:10 smithi006 bash[27292]: cluster 2023-08-28T01:05:09.551101+0000 mgr.y (mgr.14901) 5597 : cluster [DBG] pgmap v5604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:10.917 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:10 smithi006 bash[27292]: audit 2023-08-28T01:05:09.667959+0000 mon.c (mon.2) 2169 : audit [DBG] from='client.? 172.21.15.6:0/2477778387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:10 smithi130 bash[27579]: cluster 2023-08-28T01:05:09.551101+0000 mgr.y (mgr.14901) 5597 : cluster [DBG] pgmap v5604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:10 smithi130 bash[27579]: audit 2023-08-28T01:05:09.667959+0000 mon.c (mon.2) 2169 : audit [DBG] from='client.? 172.21.15.6:0/2477778387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:11.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:05:10.914Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:11.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:05:10.916Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:11.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:11 smithi130 bash[51302]: debug 2023-08-28T01:05:11.185+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:11.186415+0000) 2023-08-28T01:05:12.504 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:12 smithi130 bash[51302]: debug 2023-08-28T01:05:12.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:12.186724+0000) 2023-08-28T01:05:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:12 smithi130 bash[27579]: cluster 2023-08-28T01:05:11.551781+0000 mgr.y (mgr.14901) 5598 : cluster [DBG] pgmap v5605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:12 smithi130 bash[27579]: audit 2023-08-28T01:05:12.077370+0000 mon.c (mon.2) 2170 : audit [DBG] from='client.? 172.21.15.6:0/1128303997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:12 smithi006 bash[27292]: cluster 2023-08-28T01:05:11.551781+0000 mgr.y (mgr.14901) 5598 : cluster [DBG] pgmap v5605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:12 smithi006 bash[27292]: audit 2023-08-28T01:05:12.077370+0000 mon.c (mon.2) 2170 : audit [DBG] from='client.? 172.21.15.6:0/1128303997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:12.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:05:12.490Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:12.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:05:12.491Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:12 smithi006 bash[33918]: cluster 2023-08-28T01:05:11.551781+0000 mgr.y (mgr.14901) 5598 : cluster [DBG] pgmap v5605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:12 smithi006 bash[33918]: audit 2023-08-28T01:05:12.077370+0000 mon.c (mon.2) 2170 : audit [DBG] from='client.? 172.21.15.6:0/1128303997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:13.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:13 smithi130 bash[51302]: debug 2023-08-28T01:05:13.184+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:13.187067+0000) 2023-08-28T01:05:14.613 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:14 smithi130 bash[51302]: debug 2023-08-28T01:05:14.184+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:14.187378+0000) 2023-08-28T01:05:14.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:14 smithi006 bash[33918]: cluster 2023-08-28T01:05:13.552443+0000 mgr.y (mgr.14901) 5599 : cluster [DBG] pgmap v5606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:14.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:14 smithi006 bash[33918]: audit 2023-08-28T01:05:14.490324+0000 mon.c (mon.2) 2171 : audit [DBG] from='client.? 172.21.15.6:0/4167075949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:14.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:14 smithi006 bash[27292]: cluster 2023-08-28T01:05:13.552443+0000 mgr.y (mgr.14901) 5599 : cluster [DBG] pgmap v5606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:14.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:14 smithi006 bash[27292]: audit 2023-08-28T01:05:14.490324+0000 mon.c (mon.2) 2171 : audit [DBG] from='client.? 172.21.15.6:0/4167075949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:14.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:05:14.371Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:14.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:05:14.373Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:14 smithi130 bash[27579]: cluster 2023-08-28T01:05:13.552443+0000 mgr.y (mgr.14901) 5599 : cluster [DBG] pgmap v5606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:14.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:14 smithi130 bash[27579]: audit 2023-08-28T01:05:14.490324+0000 mon.c (mon.2) 2171 : audit [DBG] from='client.? 172.21.15.6:0/4167075949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:15.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:15 smithi130 bash[51302]: debug 2023-08-28T01:05:15.184+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:15.187737+0000) 2023-08-28T01:05:16.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:16 smithi130 bash[51302]: debug 2023-08-28T01:05:16.184+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:16.188081+0000) 2023-08-28T01:05:16.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:16 smithi006 bash[33918]: cluster 2023-08-28T01:05:15.553582+0000 mgr.y (mgr.14901) 5600 : cluster [DBG] pgmap v5607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:16.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:16 smithi006 bash[27292]: cluster 2023-08-28T01:05:15.553582+0000 mgr.y (mgr.14901) 5600 : cluster [DBG] pgmap v5607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:16 smithi130 bash[27579]: cluster 2023-08-28T01:05:15.553582+0000 mgr.y (mgr.14901) 5600 : cluster [DBG] pgmap v5607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:17.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:17 smithi130 bash[51302]: debug 2023-08-28T01:05:17.184+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:17.188402+0000) 2023-08-28T01:05:17.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:17 smithi006 bash[33918]: audit 2023-08-28T01:05:16.904231+0000 mon.c (mon.2) 2172 : audit [DBG] from='client.? 172.21.15.6:0/399373698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:17.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:17 smithi006 bash[27292]: audit 2023-08-28T01:05:16.904231+0000 mon.c (mon.2) 2172 : audit [DBG] from='client.? 172.21.15.6:0/399373698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:18.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:17 smithi130 bash[27579]: audit 2023-08-28T01:05:16.904231+0000 mon.c (mon.2) 2172 : audit [DBG] from='client.? 172.21.15.6:0/399373698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:18.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:18 smithi130 bash[51302]: debug 2023-08-28T01:05:18.184+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:18.188723+0000) 2023-08-28T01:05:18.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:18 smithi006 bash[27292]: cluster 2023-08-28T01:05:17.554272+0000 mgr.y (mgr.14901) 5601 : cluster [DBG] pgmap v5608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:18.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:18 smithi006 bash[33918]: cluster 2023-08-28T01:05:17.554272+0000 mgr.y (mgr.14901) 5601 : cluster [DBG] pgmap v5608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:18 smithi130 bash[27579]: cluster 2023-08-28T01:05:17.554272+0000 mgr.y (mgr.14901) 5601 : cluster [DBG] pgmap v5608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:19.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:19 smithi130 bash[51302]: debug 2023-08-28T01:05:19.184+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:19.189071+0000) 2023-08-28T01:05:19.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:19 smithi006 bash[27292]: audit 2023-08-28T01:05:19.316666+0000 mon.c (mon.2) 2173 : audit [DBG] from='client.? 172.21.15.6:0/2223262164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:19.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:19 smithi006 bash[33918]: audit 2023-08-28T01:05:19.316666+0000 mon.c (mon.2) 2173 : audit [DBG] from='client.? 172.21.15.6:0/2223262164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:20.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:19 smithi130 bash[27579]: audit 2023-08-28T01:05:19.316666+0000 mon.c (mon.2) 2173 : audit [DBG] from='client.? 172.21.15.6:0/2223262164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:20.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:20 smithi130 bash[51302]: debug 2023-08-28T01:05:20.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:20.189454+0000) 2023-08-28T01:05:20.917 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:05:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:05:20] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:05:20.917 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:20 smithi006 bash[27292]: cluster 2023-08-28T01:05:19.555391+0000 mgr.y (mgr.14901) 5602 : cluster [DBG] pgmap v5609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:20.917 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:20 smithi006 bash[33918]: cluster 2023-08-28T01:05:19.555391+0000 mgr.y (mgr.14901) 5602 : cluster [DBG] pgmap v5609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:20 smithi130 bash[27579]: cluster 2023-08-28T01:05:19.555391+0000 mgr.y (mgr.14901) 5602 : cluster [DBG] pgmap v5609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:21.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:05:20.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:21.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:05:20.917Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:21.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:21 smithi130 bash[51302]: debug 2023-08-28T01:05:21.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:21.189811+0000) 2023-08-28T01:05:22.506 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:22 smithi130 bash[51302]: debug 2023-08-28T01:05:22.184+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:22.190084+0000) 2023-08-28T01:05:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:22 smithi130 bash[27579]: cluster 2023-08-28T01:05:21.556019+0000 mgr.y (mgr.14901) 5603 : cluster [DBG] pgmap v5610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:22 smithi130 bash[27579]: audit 2023-08-28T01:05:21.732357+0000 mon.a (mon.0) 3851 : audit [DBG] from='client.? 172.21.15.6:0/1433955204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:22.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:22 smithi006 bash[27292]: cluster 2023-08-28T01:05:21.556019+0000 mgr.y (mgr.14901) 5603 : cluster [DBG] pgmap v5610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:22.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:22 smithi006 bash[27292]: audit 2023-08-28T01:05:21.732357+0000 mon.a (mon.0) 3851 : audit [DBG] from='client.? 172.21.15.6:0/1433955204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:22 smithi006 bash[33918]: cluster 2023-08-28T01:05:21.556019+0000 mgr.y (mgr.14901) 5603 : cluster [DBG] pgmap v5610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:22 smithi006 bash[33918]: audit 2023-08-28T01:05:21.732357+0000 mon.a (mon.0) 3851 : audit [DBG] from='client.? 172.21.15.6:0/1433955204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:22.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:05:22.490Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:22.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:05:22.492Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:23.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:23 smithi130 bash[51302]: debug 2023-08-28T01:05:23.184+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:23.190413+0000) 2023-08-28T01:05:24.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:24 smithi130 bash[51302]: debug 2023-08-28T01:05:24.184+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:24.190756+0000) 2023-08-28T01:05:24.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:05:24.371Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:24.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:05:24.373Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:24.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:24 smithi006 bash[27292]: cluster 2023-08-28T01:05:23.556663+0000 mgr.y (mgr.14901) 5604 : cluster [DBG] pgmap v5611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:24.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:24 smithi006 bash[27292]: audit 2023-08-28T01:05:24.145350+0000 mon.c (mon.2) 2174 : audit [DBG] from='client.? 172.21.15.6:0/2518508565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:24.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:24 smithi006 bash[33918]: cluster 2023-08-28T01:05:23.556663+0000 mgr.y (mgr.14901) 5604 : cluster [DBG] pgmap v5611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:24.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:24 smithi006 bash[33918]: audit 2023-08-28T01:05:24.145350+0000 mon.c (mon.2) 2174 : audit [DBG] from='client.? 172.21.15.6:0/2518508565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:24 smithi130 bash[27579]: cluster 2023-08-28T01:05:23.556663+0000 mgr.y (mgr.14901) 5604 : cluster [DBG] pgmap v5611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:24.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:24 smithi130 bash[27579]: audit 2023-08-28T01:05:24.145350+0000 mon.c (mon.2) 2174 : audit [DBG] from='client.? 172.21.15.6:0/2518508565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:25.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:25 smithi130 bash[51302]: debug 2023-08-28T01:05:25.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:25.191071+0000) 2023-08-28T01:05:26.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:26 smithi130 bash[51302]: debug 2023-08-28T01:05:26.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:26.191350+0000) 2023-08-28T01:05:26.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:26 smithi006 bash[33918]: cluster 2023-08-28T01:05:25.557765+0000 mgr.y (mgr.14901) 5605 : cluster [DBG] pgmap v5612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:26.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:26 smithi006 bash[33918]: audit 2023-08-28T01:05:26.568142+0000 mon.a (mon.0) 3852 : audit [DBG] from='client.? 172.21.15.6:0/2703422642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:26 smithi006 bash[27292]: cluster 2023-08-28T01:05:25.557765+0000 mgr.y (mgr.14901) 5605 : cluster [DBG] pgmap v5612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:26.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:26 smithi006 bash[27292]: audit 2023-08-28T01:05:26.568142+0000 mon.a (mon.0) 3852 : audit [DBG] from='client.? 172.21.15.6:0/2703422642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:26 smithi130 bash[27579]: cluster 2023-08-28T01:05:25.557765+0000 mgr.y (mgr.14901) 5605 : cluster [DBG] pgmap v5612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:26 smithi130 bash[27579]: audit 2023-08-28T01:05:26.568142+0000 mon.a (mon.0) 3852 : audit [DBG] from='client.? 172.21.15.6:0/2703422642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:27.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:27 smithi130 bash[51302]: debug 2023-08-28T01:05:27.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:27.191617+0000) 2023-08-28T01:05:28.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:28 smithi130 bash[51302]: debug 2023-08-28T01:05:28.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:28.191961+0000) 2023-08-28T01:05:28.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:28 smithi006 bash[27292]: cluster 2023-08-28T01:05:27.558439+0000 mgr.y (mgr.14901) 5606 : cluster [DBG] pgmap v5613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:28.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:28 smithi006 bash[33918]: cluster 2023-08-28T01:05:27.558439+0000 mgr.y (mgr.14901) 5606 : cluster [DBG] pgmap v5613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:28 smithi130 bash[27579]: cluster 2023-08-28T01:05:27.558439+0000 mgr.y (mgr.14901) 5606 : cluster [DBG] pgmap v5613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:29.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:29 smithi130 bash[51302]: debug 2023-08-28T01:05:29.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:29.192331+0000) 2023-08-28T01:05:29.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:29 smithi006 bash[33918]: audit 2023-08-28T01:05:28.979368+0000 mon.a (mon.0) 3853 : audit [DBG] from='client.? 172.21.15.6:0/148762375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:29.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:29 smithi006 bash[27292]: audit 2023-08-28T01:05:28.979368+0000 mon.a (mon.0) 3853 : audit [DBG] from='client.? 172.21.15.6:0/148762375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:30.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:29 smithi130 bash[27579]: audit 2023-08-28T01:05:28.979368+0000 mon.a (mon.0) 3853 : audit [DBG] from='client.? 172.21.15.6:0/148762375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:30.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:30 smithi130 bash[51302]: debug 2023-08-28T01:05:30.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:30.192713+0000) 2023-08-28T01:05:30.917 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:05:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:05:30] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:05:30.918 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:30 smithi006 bash[33918]: cluster 2023-08-28T01:05:29.559668+0000 mgr.y (mgr.14901) 5607 : cluster [DBG] pgmap v5614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:30.918 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:30 smithi006 bash[27292]: cluster 2023-08-28T01:05:29.559668+0000 mgr.y (mgr.14901) 5607 : cluster [DBG] pgmap v5614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:30 smithi130 bash[27579]: cluster 2023-08-28T01:05:29.559668+0000 mgr.y (mgr.14901) 5607 : cluster [DBG] pgmap v5614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:31.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:05:30.916Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:31.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:05:30.918Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:31.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:31 smithi130 bash[51302]: debug 2023-08-28T01:05:31.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:31.193031+0000) 2023-08-28T01:05:31.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:31 smithi006 bash[27292]: audit 2023-08-28T01:05:31.388131+0000 mon.a (mon.0) 3854 : audit [DBG] from='client.? 172.21.15.6:0/2129566222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:31.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:31 smithi006 bash[33918]: audit 2023-08-28T01:05:31.388131+0000 mon.a (mon.0) 3854 : audit [DBG] from='client.? 172.21.15.6:0/2129566222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:32.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:31 smithi130 bash[27579]: audit 2023-08-28T01:05:31.388131+0000 mon.a (mon.0) 3854 : audit [DBG] from='client.? 172.21.15.6:0/2129566222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:32.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:32 smithi130 bash[51302]: debug 2023-08-28T01:05:32.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:32.193397+0000) 2023-08-28T01:05:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:32 smithi006 bash[33918]: cluster 2023-08-28T01:05:31.560377+0000 mgr.y (mgr.14901) 5608 : cluster [DBG] pgmap v5615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:05:32.491Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:05:32.492Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:32.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:32 smithi006 bash[27292]: cluster 2023-08-28T01:05:31.560377+0000 mgr.y (mgr.14901) 5608 : cluster [DBG] pgmap v5615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:33.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:32 smithi130 bash[27579]: cluster 2023-08-28T01:05:31.560377+0000 mgr.y (mgr.14901) 5608 : cluster [DBG] pgmap v5615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:33.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:33 smithi130 bash[51302]: debug 2023-08-28T01:05:33.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:33.193761+0000) 2023-08-28T01:05:34.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:34 smithi130 bash[51302]: debug 2023-08-28T01:05:34.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:34.194080+0000) 2023-08-28T01:05:34.648 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:05:34.372Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:34.648 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:05:34.373Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:34.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:34 smithi006 bash[33918]: cluster 2023-08-28T01:05:33.561059+0000 mgr.y (mgr.14901) 5609 : cluster [DBG] pgmap v5616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:34.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:34 smithi006 bash[33918]: audit 2023-08-28T01:05:33.801552+0000 mon.c (mon.2) 2175 : audit [DBG] from='client.? 172.21.15.6:0/2340396906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:34.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:34 smithi006 bash[27292]: cluster 2023-08-28T01:05:33.561059+0000 mgr.y (mgr.14901) 5609 : cluster [DBG] pgmap v5616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:34.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:34 smithi006 bash[27292]: audit 2023-08-28T01:05:33.801552+0000 mon.c (mon.2) 2175 : audit [DBG] from='client.? 172.21.15.6:0/2340396906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:34 smithi130 bash[27579]: cluster 2023-08-28T01:05:33.561059+0000 mgr.y (mgr.14901) 5609 : cluster [DBG] pgmap v5616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:34 smithi130 bash[27579]: audit 2023-08-28T01:05:33.801552+0000 mon.c (mon.2) 2175 : audit [DBG] from='client.? 172.21.15.6:0/2340396906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:35.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:35 smithi130 bash[51302]: debug 2023-08-28T01:05:35.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:35.194432+0000) 2023-08-28T01:05:36.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:36 smithi130 bash[51302]: debug 2023-08-28T01:05:36.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:36.194782+0000) 2023-08-28T01:05:36.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:36 smithi006 bash[33918]: cluster 2023-08-28T01:05:35.562144+0000 mgr.y (mgr.14901) 5610 : cluster [DBG] pgmap v5617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:36.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:36 smithi006 bash[33918]: audit 2023-08-28T01:05:36.212271+0000 mon.a (mon.0) 3855 : audit [DBG] from='client.? 172.21.15.6:0/1613434776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:36.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:36 smithi006 bash[27292]: cluster 2023-08-28T01:05:35.562144+0000 mgr.y (mgr.14901) 5610 : cluster [DBG] pgmap v5617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:36.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:36 smithi006 bash[27292]: audit 2023-08-28T01:05:36.212271+0000 mon.a (mon.0) 3855 : audit [DBG] from='client.? 172.21.15.6:0/1613434776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:36 smithi130 bash[27579]: cluster 2023-08-28T01:05:35.562144+0000 mgr.y (mgr.14901) 5610 : cluster [DBG] pgmap v5617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:36 smithi130 bash[27579]: audit 2023-08-28T01:05:36.212271+0000 mon.a (mon.0) 3855 : audit [DBG] from='client.? 172.21.15.6:0/1613434776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:37.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:37 smithi130 bash[51302]: debug 2023-08-28T01:05:37.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:37.195064+0000) 2023-08-28T01:05:38.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:38 smithi130 bash[51302]: debug 2023-08-28T01:05:38.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:38.195320+0000) 2023-08-28T01:05:38.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:38 smithi006 bash[33918]: cluster 2023-08-28T01:05:37.562802+0000 mgr.y (mgr.14901) 5611 : cluster [DBG] pgmap v5618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:38.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:38 smithi006 bash[33918]: audit 2023-08-28T01:05:38.624489+0000 mon.c (mon.2) 2176 : audit [DBG] from='client.? 172.21.15.6:0/1605179059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:38.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:38 smithi006 bash[27292]: cluster 2023-08-28T01:05:37.562802+0000 mgr.y (mgr.14901) 5611 : cluster [DBG] pgmap v5618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:38 smithi006 bash[27292]: audit 2023-08-28T01:05:38.624489+0000 mon.c (mon.2) 2176 : audit [DBG] from='client.? 172.21.15.6:0/1605179059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:38 smithi130 bash[27579]: cluster 2023-08-28T01:05:37.562802+0000 mgr.y (mgr.14901) 5611 : cluster [DBG] pgmap v5618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:38 smithi130 bash[27579]: audit 2023-08-28T01:05:38.624489+0000 mon.c (mon.2) 2176 : audit [DBG] from='client.? 172.21.15.6:0/1605179059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:39.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:39 smithi130 bash[51302]: debug 2023-08-28T01:05:39.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:39.195603+0000) 2023-08-28T01:05:40.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:40 smithi130 bash[51302]: debug 2023-08-28T01:05:40.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:40.195932+0000) 2023-08-28T01:05:40.918 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:05:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:05:40] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:05:40.919 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:40 smithi006 bash[33918]: cluster 2023-08-28T01:05:39.563874+0000 mgr.y (mgr.14901) 5612 : cluster [DBG] pgmap v5619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:40.919 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:40 smithi006 bash[27292]: cluster 2023-08-28T01:05:39.563874+0000 mgr.y (mgr.14901) 5612 : cluster [DBG] pgmap v5619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:40 smithi130 bash[27579]: cluster 2023-08-28T01:05:39.563874+0000 mgr.y (mgr.14901) 5612 : cluster [DBG] pgmap v5619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:41.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:05:40.916Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:41.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:05:40.918Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:41.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:41 smithi130 bash[51302]: debug 2023-08-28T01:05:41.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:41.196264+0000) 2023-08-28T01:05:41.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:41 smithi006 bash[27292]: audit 2023-08-28T01:05:41.045681+0000 mon.c (mon.2) 2177 : audit [DBG] from='client.? 172.21.15.6:0/2587913921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:41.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:41 smithi006 bash[33918]: audit 2023-08-28T01:05:41.045681+0000 mon.c (mon.2) 2177 : audit [DBG] from='client.? 172.21.15.6:0/2587913921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:42.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:41 smithi130 bash[27579]: audit 2023-08-28T01:05:41.045681+0000 mon.c (mon.2) 2177 : audit [DBG] from='client.? 172.21.15.6:0/2587913921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:42.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:42 smithi130 bash[51302]: debug 2023-08-28T01:05:42.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:42.196544+0000) 2023-08-28T01:05:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:42 smithi006 bash[27292]: cluster 2023-08-28T01:05:41.564518+0000 mgr.y (mgr.14901) 5613 : cluster [DBG] pgmap v5620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:05:42.492Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:42.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:05:42.494Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:42 smithi006 bash[33918]: cluster 2023-08-28T01:05:41.564518+0000 mgr.y (mgr.14901) 5613 : cluster [DBG] pgmap v5620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:43.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:42 smithi130 bash[27579]: cluster 2023-08-28T01:05:41.564518+0000 mgr.y (mgr.14901) 5613 : cluster [DBG] pgmap v5620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:43.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:43 smithi130 bash[51302]: debug 2023-08-28T01:05:43.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:43.196866+0000) 2023-08-28T01:05:43.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:43 smithi006 bash[27292]: audit 2023-08-28T01:05:43.460161+0000 mon.a (mon.0) 3856 : audit [DBG] from='client.? 172.21.15.6:0/2834243162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:43.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:43 smithi006 bash[33918]: audit 2023-08-28T01:05:43.460161+0000 mon.a (mon.0) 3856 : audit [DBG] from='client.? 172.21.15.6:0/2834243162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:44.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:43 smithi130 bash[27579]: audit 2023-08-28T01:05:43.460161+0000 mon.a (mon.0) 3856 : audit [DBG] from='client.? 172.21.15.6:0/2834243162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:44.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:44 smithi130 bash[51302]: debug 2023-08-28T01:05:44.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:44.197241+0000) 2023-08-28T01:05:44.685 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:05:44.373Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:44.685 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:05:44.375Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:44.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:44 smithi006 bash[33918]: cluster 2023-08-28T01:05:43.565169+0000 mgr.y (mgr.14901) 5614 : cluster [DBG] pgmap v5621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:44.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:44 smithi006 bash[27292]: cluster 2023-08-28T01:05:43.565169+0000 mgr.y (mgr.14901) 5614 : cluster [DBG] pgmap v5621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:44 smithi130 bash[27579]: cluster 2023-08-28T01:05:43.565169+0000 mgr.y (mgr.14901) 5614 : cluster [DBG] pgmap v5621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:45.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:45 smithi130 bash[51302]: debug 2023-08-28T01:05:45.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:45.197562+0000) 2023-08-28T01:05:46.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:46 smithi130 bash[51302]: debug 2023-08-28T01:05:46.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:46.197864+0000) 2023-08-28T01:05:46.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:46 smithi006 bash[27292]: cluster 2023-08-28T01:05:45.565923+0000 mgr.y (mgr.14901) 5615 : cluster [DBG] pgmap v5622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:46.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:46 smithi006 bash[27292]: audit 2023-08-28T01:05:45.878419+0000 mon.c (mon.2) 2178 : audit [DBG] from='client.? 172.21.15.6:0/3284374851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:46 smithi006 bash[33918]: cluster 2023-08-28T01:05:45.565923+0000 mgr.y (mgr.14901) 5615 : cluster [DBG] pgmap v5622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:46 smithi006 bash[33918]: audit 2023-08-28T01:05:45.878419+0000 mon.c (mon.2) 2178 : audit [DBG] from='client.? 172.21.15.6:0/3284374851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:46 smithi130 bash[27579]: cluster 2023-08-28T01:05:45.565923+0000 mgr.y (mgr.14901) 5615 : cluster [DBG] pgmap v5622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:46 smithi130 bash[27579]: audit 2023-08-28T01:05:45.878419+0000 mon.c (mon.2) 2178 : audit [DBG] from='client.? 172.21.15.6:0/3284374851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:47.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:47 smithi130 bash[51302]: debug 2023-08-28T01:05:47.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:47.198178+0000) 2023-08-28T01:05:48.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:48 smithi130 bash[51302]: debug 2023-08-28T01:05:48.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:48.198541+0000) 2023-08-28T01:05:48.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:48 smithi006 bash[33918]: cluster 2023-08-28T01:05:47.566513+0000 mgr.y (mgr.14901) 5616 : cluster [DBG] pgmap v5623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:48.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:48 smithi006 bash[33918]: audit 2023-08-28T01:05:48.292949+0000 mon.c (mon.2) 2179 : audit [DBG] from='client.? 172.21.15.6:0/3085041078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:48.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:48 smithi006 bash[27292]: cluster 2023-08-28T01:05:47.566513+0000 mgr.y (mgr.14901) 5616 : cluster [DBG] pgmap v5623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:48.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:48 smithi006 bash[27292]: audit 2023-08-28T01:05:48.292949+0000 mon.c (mon.2) 2179 : audit [DBG] from='client.? 172.21.15.6:0/3085041078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:49.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:48 smithi130 bash[27579]: cluster 2023-08-28T01:05:47.566513+0000 mgr.y (mgr.14901) 5616 : cluster [DBG] pgmap v5623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:49.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:48 smithi130 bash[27579]: audit 2023-08-28T01:05:48.292949+0000 mon.c (mon.2) 2179 : audit [DBG] from='client.? 172.21.15.6:0/3085041078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:49.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:49 smithi130 bash[51302]: debug 2023-08-28T01:05:49.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:49.198847+0000) 2023-08-28T01:05:50.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:50 smithi130 bash[51302]: debug 2023-08-28T01:05:50.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:50.199166+0000) 2023-08-28T01:05:50.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:05:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:05:50] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:05:50.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:50 smithi006 bash[33918]: cluster 2023-08-28T01:05:49.567618+0000 mgr.y (mgr.14901) 5617 : cluster [DBG] pgmap v5624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:50.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:50 smithi006 bash[27292]: cluster 2023-08-28T01:05:49.567618+0000 mgr.y (mgr.14901) 5617 : cluster [DBG] pgmap v5624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:05:50.917Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:05:50.919Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:50 smithi130 bash[27579]: cluster 2023-08-28T01:05:49.567618+0000 mgr.y (mgr.14901) 5617 : cluster [DBG] pgmap v5624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:51.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:51 smithi130 bash[51302]: debug 2023-08-28T01:05:51.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:51.199505+0000) 2023-08-28T01:05:51.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:51 smithi006 bash[27292]: audit 2023-08-28T01:05:50.721803+0000 mon.c (mon.2) 2180 : audit [DBG] from='client.? 172.21.15.6:0/244433575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:51.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:51 smithi006 bash[33918]: audit 2023-08-28T01:05:50.721803+0000 mon.c (mon.2) 2180 : audit [DBG] from='client.? 172.21.15.6:0/244433575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:52.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:51 smithi130 bash[27579]: audit 2023-08-28T01:05:50.721803+0000 mon.c (mon.2) 2180 : audit [DBG] from='client.? 172.21.15.6:0/244433575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:52.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:52 smithi130 bash[51302]: debug 2023-08-28T01:05:52.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:52.199812+0000) 2023-08-28T01:05:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:52 smithi006 bash[33918]: cluster 2023-08-28T01:05:51.568309+0000 mgr.y (mgr.14901) 5618 : cluster [DBG] pgmap v5625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:52 smithi006 bash[27292]: cluster 2023-08-28T01:05:51.568309+0000 mgr.y (mgr.14901) 5618 : cluster [DBG] pgmap v5625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:05:52.493Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:52.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:05:52.494Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:53.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:52 smithi130 bash[27579]: cluster 2023-08-28T01:05:51.568309+0000 mgr.y (mgr.14901) 5618 : cluster [DBG] pgmap v5625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:53.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:53 smithi130 bash[51302]: debug 2023-08-28T01:05:53.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:53.200146+0000) 2023-08-28T01:05:53.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:53 smithi006 bash[33918]: audit 2023-08-28T01:05:53.140993+0000 mon.c (mon.2) 2181 : audit [DBG] from='client.? 172.21.15.6:0/4136983015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:53.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:53 smithi006 bash[33918]: audit 2023-08-28T01:05:53.215516+0000 mon.b (mon.1) 658 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:05:53.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:53 smithi006 bash[27292]: audit 2023-08-28T01:05:53.140993+0000 mon.c (mon.2) 2181 : audit [DBG] from='client.? 172.21.15.6:0/4136983015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:53.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:53 smithi006 bash[27292]: audit 2023-08-28T01:05:53.215516+0000 mon.b (mon.1) 658 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:05:54.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:53 smithi130 bash[27579]: audit 2023-08-28T01:05:53.140993+0000 mon.c (mon.2) 2181 : audit [DBG] from='client.? 172.21.15.6:0/4136983015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:54.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:53 smithi130 bash[27579]: audit 2023-08-28T01:05:53.215516+0000 mon.b (mon.1) 658 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:05:54.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:54 smithi130 bash[51302]: debug 2023-08-28T01:05:54.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:54.200503+0000) 2023-08-28T01:05:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:05:54.374Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:05:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:05:54.375Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:05:54.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:54 smithi006 bash[27292]: cluster 2023-08-28T01:05:53.569098+0000 mgr.y (mgr.14901) 5619 : cluster [DBG] pgmap v5626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:54.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:54 smithi006 bash[33918]: cluster 2023-08-28T01:05:53.569098+0000 mgr.y (mgr.14901) 5619 : cluster [DBG] pgmap v5626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:54 smithi130 bash[27579]: cluster 2023-08-28T01:05:53.569098+0000 mgr.y (mgr.14901) 5619 : cluster [DBG] pgmap v5626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:55.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:55 smithi130 bash[51302]: debug 2023-08-28T01:05:55.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:55.200779+0000) 2023-08-28T01:05:56.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:55 smithi130 bash[27579]: audit 2023-08-28T01:05:55.556809+0000 mon.c (mon.2) 2182 : audit [DBG] from='client.? 172.21.15.6:0/3478634867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:56.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:55 smithi006 bash[33918]: audit 2023-08-28T01:05:55.556809+0000 mon.c (mon.2) 2182 : audit [DBG] from='client.? 172.21.15.6:0/3478634867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:56.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:55 smithi006 bash[27292]: audit 2023-08-28T01:05:55.556809+0000 mon.c (mon.2) 2182 : audit [DBG] from='client.? 172.21.15.6:0/3478634867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:56.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:56 smithi130 bash[51302]: debug 2023-08-28T01:05:56.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:56.201056+0000) 2023-08-28T01:05:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:56 smithi130 bash[27579]: cluster 2023-08-28T01:05:55.570139+0000 mgr.y (mgr.14901) 5620 : cluster [DBG] pgmap v5627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:57.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:56 smithi006 bash[33918]: cluster 2023-08-28T01:05:55.570139+0000 mgr.y (mgr.14901) 5620 : cluster [DBG] pgmap v5627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:57.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:56 smithi006 bash[27292]: cluster 2023-08-28T01:05:55.570139+0000 mgr.y (mgr.14901) 5620 : cluster [DBG] pgmap v5627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:57.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:57 smithi130 bash[51302]: debug 2023-08-28T01:05:57.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:57.201366+0000) 2023-08-28T01:05:58.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:58 smithi130 bash[51302]: debug 2023-08-28T01:05:58.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:58.201716+0000) 2023-08-28T01:05:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:58 smithi130 bash[27579]: cluster 2023-08-28T01:05:57.570859+0000 mgr.y (mgr.14901) 5621 : cluster [DBG] pgmap v5628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:05:58 smithi130 bash[27579]: audit 2023-08-28T01:05:57.967098+0000 mon.c (mon.2) 2183 : audit [DBG] from='client.? 172.21.15.6:0/1832274712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:59.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:58 smithi006 bash[27292]: cluster 2023-08-28T01:05:57.570859+0000 mgr.y (mgr.14901) 5621 : cluster [DBG] pgmap v5628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:59.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:05:58 smithi006 bash[27292]: audit 2023-08-28T01:05:57.967098+0000 mon.c (mon.2) 2183 : audit [DBG] from='client.? 172.21.15.6:0/1832274712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:59.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:58 smithi006 bash[33918]: cluster 2023-08-28T01:05:57.570859+0000 mgr.y (mgr.14901) 5621 : cluster [DBG] pgmap v5628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:05:59.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:05:58 smithi006 bash[33918]: audit 2023-08-28T01:05:57.967098+0000 mon.c (mon.2) 2183 : audit [DBG] from='client.? 172.21.15.6:0/1832274712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:59.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:05:59 smithi130 bash[51302]: debug 2023-08-28T01:05:59.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:59.202013+0000) 2023-08-28T01:06:00.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:00 smithi006 bash[33918]: audit 2023-08-28T01:05:59.135150+0000 mon.a (mon.0) 3857 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:06:00.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:00 smithi006 bash[33918]: audit 2023-08-28T01:05:59.540638+0000 mon.a (mon.0) 3858 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:06:00.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:00 smithi006 bash[33918]: cluster 2023-08-28T01:05:59.572001+0000 mgr.y (mgr.14901) 5622 : cluster [DBG] pgmap v5629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:00.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:00 smithi006 bash[33918]: audit 2023-08-28T01:05:59.811281+0000 mon.b (mon.1) 659 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:06:00.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:00 smithi006 bash[33918]: audit 2023-08-28T01:05:59.813463+0000 mon.b (mon.1) 660 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:06:00.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:00 smithi006 bash[33918]: audit 2023-08-28T01:05:59.821958+0000 mon.a (mon.0) 3859 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:06:00.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:00 smithi006 bash[27292]: audit 2023-08-28T01:05:59.135150+0000 mon.a (mon.0) 3857 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:06:00.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:00 smithi006 bash[27292]: audit 2023-08-28T01:05:59.540638+0000 mon.a (mon.0) 3858 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:06:00.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:00 smithi006 bash[27292]: cluster 2023-08-28T01:05:59.572001+0000 mgr.y (mgr.14901) 5622 : cluster [DBG] pgmap v5629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:00.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:00 smithi006 bash[27292]: audit 2023-08-28T01:05:59.811281+0000 mon.b (mon.1) 659 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:06:00.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:00 smithi006 bash[27292]: audit 2023-08-28T01:05:59.813463+0000 mon.b (mon.1) 660 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:06:00.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:00 smithi006 bash[27292]: audit 2023-08-28T01:05:59.821958+0000 mon.a (mon.0) 3859 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:06:00.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:00 smithi130 bash[51302]: debug 2023-08-28T01:06:00.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:00.202344+0000) 2023-08-28T01:06:00.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:00 smithi130 bash[27579]: audit 2023-08-28T01:05:59.135150+0000 mon.a (mon.0) 3857 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:06:00.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:00 smithi130 bash[27579]: audit 2023-08-28T01:05:59.540638+0000 mon.a (mon.0) 3858 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:06:00.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:00 smithi130 bash[27579]: cluster 2023-08-28T01:05:59.572001+0000 mgr.y (mgr.14901) 5622 : cluster [DBG] pgmap v5629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:00.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:00 smithi130 bash[27579]: audit 2023-08-28T01:05:59.811281+0000 mon.b (mon.1) 659 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:06:00.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:00 smithi130 bash[27579]: audit 2023-08-28T01:05:59.813463+0000 mon.b (mon.1) 660 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:06:00.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:00 smithi130 bash[27579]: audit 2023-08-28T01:05:59.821958+0000 mon.a (mon.0) 3859 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:06:01.139 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:06:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:06:00] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:06:01.139 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:06:00.918Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:01.139 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:06:00.921Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:01.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:01 smithi006 bash[33918]: audit 2023-08-28T01:06:00.397467+0000 mon.c (mon.2) 2184 : audit [DBG] from='client.? 172.21.15.6:0/4256342565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:01.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:01 smithi006 bash[27292]: audit 2023-08-28T01:06:00.397467+0000 mon.c (mon.2) 2184 : audit [DBG] from='client.? 172.21.15.6:0/4256342565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:01.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:01 smithi130 bash[27579]: audit 2023-08-28T01:06:00.397467+0000 mon.c (mon.2) 2184 : audit [DBG] from='client.? 172.21.15.6:0/4256342565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:01.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:01 smithi130 bash[51302]: debug 2023-08-28T01:06:01.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:01.202678+0000) 2023-08-28T01:06:02.518 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:02 smithi130 bash[51302]: debug 2023-08-28T01:06:02.200+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:02.203023+0000) 2023-08-28T01:06:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:02 smithi130 bash[27579]: cluster 2023-08-28T01:06:01.572661+0000 mgr.y (mgr.14901) 5623 : cluster [DBG] pgmap v5630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:02 smithi006 bash[27292]: cluster 2023-08-28T01:06:01.572661+0000 mgr.y (mgr.14901) 5623 : cluster [DBG] pgmap v5630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:02.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:06:02.493Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:02.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:06:02.496Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:02 smithi006 bash[33918]: cluster 2023-08-28T01:06:01.572661+0000 mgr.y (mgr.14901) 5623 : cluster [DBG] pgmap v5630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:03.521 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:03 smithi130 bash[51302]: debug 2023-08-28T01:06:03.200+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:03.203346+0000) 2023-08-28T01:06:03.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:03 smithi130 bash[27579]: audit 2023-08-28T01:06:02.810411+0000 mon.a (mon.0) 3860 : audit [DBG] from='client.? 172.21.15.6:0/1530508767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:03.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:03 smithi006 bash[33918]: audit 2023-08-28T01:06:02.810411+0000 mon.a (mon.0) 3860 : audit [DBG] from='client.? 172.21.15.6:0/1530508767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:03.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:03 smithi006 bash[27292]: audit 2023-08-28T01:06:02.810411+0000 mon.a (mon.0) 3860 : audit [DBG] from='client.? 172.21.15.6:0/1530508767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:04.530 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:04 smithi130 bash[51302]: debug 2023-08-28T01:06:04.200+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:04.203709+0000) 2023-08-28T01:06:04.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:04 smithi006 bash[27292]: cluster 2023-08-28T01:06:03.573335+0000 mgr.y (mgr.14901) 5624 : cluster [DBG] pgmap v5631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:04.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:04 smithi006 bash[33918]: cluster 2023-08-28T01:06:03.573335+0000 mgr.y (mgr.14901) 5624 : cluster [DBG] pgmap v5631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:04.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:06:04.374Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:04.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:06:04.376Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:04.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:04 smithi130 bash[27579]: cluster 2023-08-28T01:06:03.573335+0000 mgr.y (mgr.14901) 5624 : cluster [DBG] pgmap v5631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:05.531 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:05 smithi130 bash[51302]: debug 2023-08-28T01:06:05.200+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:05.204050+0000) 2023-08-28T01:06:05.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:05 smithi130 bash[27579]: audit 2023-08-28T01:06:05.226142+0000 mon.a (mon.0) 3861 : audit [DBG] from='client.? 172.21.15.6:0/846881624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:05.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:05 smithi006 bash[27292]: audit 2023-08-28T01:06:05.226142+0000 mon.a (mon.0) 3861 : audit [DBG] from='client.? 172.21.15.6:0/846881624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:05.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:05 smithi006 bash[33918]: audit 2023-08-28T01:06:05.226142+0000 mon.a (mon.0) 3861 : audit [DBG] from='client.? 172.21.15.6:0/846881624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:06.541 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:06 smithi130 bash[51302]: debug 2023-08-28T01:06:06.200+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:06.204383+0000) 2023-08-28T01:06:06.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:06 smithi130 bash[27579]: cluster 2023-08-28T01:06:05.574448+0000 mgr.y (mgr.14901) 5625 : cluster [DBG] pgmap v5632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:06.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:06 smithi006 bash[33918]: cluster 2023-08-28T01:06:05.574448+0000 mgr.y (mgr.14901) 5625 : cluster [DBG] pgmap v5632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:06.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:06 smithi006 bash[27292]: cluster 2023-08-28T01:06:05.574448+0000 mgr.y (mgr.14901) 5625 : cluster [DBG] pgmap v5632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:07.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:07 smithi130 bash[51302]: debug 2023-08-28T01:06:07.200+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:07.204703+0000) 2023-08-28T01:06:08.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:08 smithi130 bash[51302]: debug 2023-08-28T01:06:08.200+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:08.205032+0000) 2023-08-28T01:06:08.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:08 smithi006 bash[27292]: cluster 2023-08-28T01:06:07.575118+0000 mgr.y (mgr.14901) 5626 : cluster [DBG] pgmap v5633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:08.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:08 smithi006 bash[27292]: audit 2023-08-28T01:06:07.636670+0000 mon.a (mon.0) 3862 : audit [DBG] from='client.? 172.21.15.6:0/1683690614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:08.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:08 smithi006 bash[33918]: cluster 2023-08-28T01:06:07.575118+0000 mgr.y (mgr.14901) 5626 : cluster [DBG] pgmap v5633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:08.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:08 smithi006 bash[33918]: audit 2023-08-28T01:06:07.636670+0000 mon.a (mon.0) 3862 : audit [DBG] from='client.? 172.21.15.6:0/1683690614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:08 smithi130 bash[27579]: cluster 2023-08-28T01:06:07.575118+0000 mgr.y (mgr.14901) 5626 : cluster [DBG] pgmap v5633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:08 smithi130 bash[27579]: audit 2023-08-28T01:06:07.636670+0000 mon.a (mon.0) 3862 : audit [DBG] from='client.? 172.21.15.6:0/1683690614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:09.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:09 smithi130 bash[51302]: debug 2023-08-28T01:06:09.200+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:09.205310+0000) 2023-08-28T01:06:10.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:10 smithi130 bash[51302]: debug 2023-08-28T01:06:10.199+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:10.205643+0000) 2023-08-28T01:06:10.921 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:06:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:06:10] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:06:10.922 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:10 smithi006 bash[33918]: cluster 2023-08-28T01:06:09.576213+0000 mgr.y (mgr.14901) 5627 : cluster [DBG] pgmap v5634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:10.922 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:10 smithi006 bash[33918]: audit 2023-08-28T01:06:10.054556+0000 mon.a (mon.0) 3863 : audit [DBG] from='client.? 172.21.15.6:0/1387092252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:10.922 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:10 smithi006 bash[27292]: cluster 2023-08-28T01:06:09.576213+0000 mgr.y (mgr.14901) 5627 : cluster [DBG] pgmap v5634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:10.922 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:10 smithi006 bash[27292]: audit 2023-08-28T01:06:10.054556+0000 mon.a (mon.0) 3863 : audit [DBG] from='client.? 172.21.15.6:0/1387092252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:10 smithi130 bash[27579]: cluster 2023-08-28T01:06:09.576213+0000 mgr.y (mgr.14901) 5627 : cluster [DBG] pgmap v5634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:10 smithi130 bash[27579]: audit 2023-08-28T01:06:10.054556+0000 mon.a (mon.0) 3863 : audit [DBG] from='client.? 172.21.15.6:0/1387092252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:11.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:06:10.920Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:11.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:06:10.922Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:11.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:11 smithi130 bash[51302]: debug 2023-08-28T01:06:11.199+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:11.205964+0000) 2023-08-28T01:06:12.520 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:12 smithi130 bash[51302]: debug 2023-08-28T01:06:12.199+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:12.206265+0000) 2023-08-28T01:06:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:12 smithi130 bash[27579]: cluster 2023-08-28T01:06:11.576906+0000 mgr.y (mgr.14901) 5628 : cluster [DBG] pgmap v5635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:12 smithi130 bash[27579]: audit 2023-08-28T01:06:12.474295+0000 mon.c (mon.2) 2185 : audit [DBG] from='client.? 172.21.15.6:0/807217852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:12 smithi006 bash[33918]: cluster 2023-08-28T01:06:11.576906+0000 mgr.y (mgr.14901) 5628 : cluster [DBG] pgmap v5635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:12 smithi006 bash[33918]: audit 2023-08-28T01:06:12.474295+0000 mon.c (mon.2) 2185 : audit [DBG] from='client.? 172.21.15.6:0/807217852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:06:12.495Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:06:12.496Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:12 smithi006 bash[27292]: cluster 2023-08-28T01:06:11.576906+0000 mgr.y (mgr.14901) 5628 : cluster [DBG] pgmap v5635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:12.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:12 smithi006 bash[27292]: audit 2023-08-28T01:06:12.474295+0000 mon.c (mon.2) 2185 : audit [DBG] from='client.? 172.21.15.6:0/807217852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:13.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:13 smithi130 bash[51302]: debug 2023-08-28T01:06:13.203+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:13.206541+0000) 2023-08-28T01:06:14.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:14 smithi130 bash[51302]: debug 2023-08-28T01:06:14.203+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:14.206847+0000) 2023-08-28T01:06:14.636 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:06:14.374Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:14.636 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:06:14.376Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:14.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:14 smithi006 bash[27292]: cluster 2023-08-28T01:06:13.577677+0000 mgr.y (mgr.14901) 5629 : cluster [DBG] pgmap v5636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:14.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:14 smithi006 bash[33918]: cluster 2023-08-28T01:06:13.577677+0000 mgr.y (mgr.14901) 5629 : cluster [DBG] pgmap v5636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:14 smithi130 bash[27579]: cluster 2023-08-28T01:06:13.577677+0000 mgr.y (mgr.14901) 5629 : cluster [DBG] pgmap v5636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:15.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:15 smithi130 bash[51302]: debug 2023-08-28T01:06:15.203+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:15.207130+0000) 2023-08-28T01:06:15.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:15 smithi006 bash[27292]: audit 2023-08-28T01:06:14.915802+0000 mon.a (mon.0) 3864 : audit [DBG] from='client.? 172.21.15.6:0/2334435918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:15.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:15 smithi006 bash[33918]: audit 2023-08-28T01:06:14.915802+0000 mon.a (mon.0) 3864 : audit [DBG] from='client.? 172.21.15.6:0/2334435918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:16.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:15 smithi130 bash[27579]: audit 2023-08-28T01:06:14.915802+0000 mon.a (mon.0) 3864 : audit [DBG] from='client.? 172.21.15.6:0/2334435918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:16.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:16 smithi130 bash[51302]: debug 2023-08-28T01:06:16.203+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:16.207444+0000) 2023-08-28T01:06:16.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:16 smithi006 bash[27292]: cluster 2023-08-28T01:06:15.578691+0000 mgr.y (mgr.14901) 5630 : cluster [DBG] pgmap v5637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:16 smithi006 bash[33918]: cluster 2023-08-28T01:06:15.578691+0000 mgr.y (mgr.14901) 5630 : cluster [DBG] pgmap v5637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:16 smithi130 bash[27579]: cluster 2023-08-28T01:06:15.578691+0000 mgr.y (mgr.14901) 5630 : cluster [DBG] pgmap v5637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:17.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:17 smithi130 bash[51302]: debug 2023-08-28T01:06:17.203+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:17.207732+0000) 2023-08-28T01:06:17.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:17 smithi006 bash[33918]: audit 2023-08-28T01:06:17.331428+0000 mon.c (mon.2) 2186 : audit [DBG] from='client.? 172.21.15.6:0/1134737479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:17.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:17 smithi006 bash[27292]: audit 2023-08-28T01:06:17.331428+0000 mon.c (mon.2) 2186 : audit [DBG] from='client.? 172.21.15.6:0/1134737479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:18.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:17 smithi130 bash[27579]: audit 2023-08-28T01:06:17.331428+0000 mon.c (mon.2) 2186 : audit [DBG] from='client.? 172.21.15.6:0/1134737479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:18.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:18 smithi130 bash[51302]: debug 2023-08-28T01:06:18.203+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:18.208037+0000) 2023-08-28T01:06:18.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:18 smithi006 bash[33918]: cluster 2023-08-28T01:06:17.579310+0000 mgr.y (mgr.14901) 5631 : cluster [DBG] pgmap v5638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:18.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:18 smithi006 bash[27292]: cluster 2023-08-28T01:06:17.579310+0000 mgr.y (mgr.14901) 5631 : cluster [DBG] pgmap v5638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:18 smithi130 bash[27579]: cluster 2023-08-28T01:06:17.579310+0000 mgr.y (mgr.14901) 5631 : cluster [DBG] pgmap v5638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:19.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:19 smithi130 bash[51302]: debug 2023-08-28T01:06:19.203+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:19.208337+0000) 2023-08-28T01:06:20.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:20 smithi130 bash[51302]: debug 2023-08-28T01:06:20.207+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:20.208651+0000) 2023-08-28T01:06:20.922 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:06:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:06:20] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:06:20.922 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:20 smithi006 bash[33918]: cluster 2023-08-28T01:06:19.580390+0000 mgr.y (mgr.14901) 5632 : cluster [DBG] pgmap v5639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:20.923 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:20 smithi006 bash[33918]: audit 2023-08-28T01:06:19.747336+0000 mon.a (mon.0) 3865 : audit [DBG] from='client.? 172.21.15.6:0/1411257010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:20.923 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:20 smithi006 bash[27292]: cluster 2023-08-28T01:06:19.580390+0000 mgr.y (mgr.14901) 5632 : cluster [DBG] pgmap v5639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:20.923 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:20 smithi006 bash[27292]: audit 2023-08-28T01:06:19.747336+0000 mon.a (mon.0) 3865 : audit [DBG] from='client.? 172.21.15.6:0/1411257010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:20 smithi130 bash[27579]: cluster 2023-08-28T01:06:19.580390+0000 mgr.y (mgr.14901) 5632 : cluster [DBG] pgmap v5639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:20 smithi130 bash[27579]: audit 2023-08-28T01:06:19.747336+0000 mon.a (mon.0) 3865 : audit [DBG] from='client.? 172.21.15.6:0/1411257010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:21.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:06:20.920Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:21.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:06:20.922Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:21.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:21 smithi130 bash[51302]: debug 2023-08-28T01:06:21.203+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:21.208945+0000) 2023-08-28T01:06:22.530 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:22 smithi130 bash[51302]: debug 2023-08-28T01:06:22.203+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:22.209240+0000) 2023-08-28T01:06:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:22 smithi130 bash[27579]: cluster 2023-08-28T01:06:21.581047+0000 mgr.y (mgr.14901) 5633 : cluster [DBG] pgmap v5640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:22 smithi130 bash[27579]: audit 2023-08-28T01:06:22.158539+0000 mon.a (mon.0) 3866 : audit [DBG] from='client.? 172.21.15.6:0/1744008065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:22.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:06:22.495Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:22.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:06:22.497Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:22.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:22 smithi006 bash[27292]: cluster 2023-08-28T01:06:21.581047+0000 mgr.y (mgr.14901) 5633 : cluster [DBG] pgmap v5640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:22.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:22 smithi006 bash[27292]: audit 2023-08-28T01:06:22.158539+0000 mon.a (mon.0) 3866 : audit [DBG] from='client.? 172.21.15.6:0/1744008065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:22 smithi006 bash[33918]: cluster 2023-08-28T01:06:21.581047+0000 mgr.y (mgr.14901) 5633 : cluster [DBG] pgmap v5640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:22 smithi006 bash[33918]: audit 2023-08-28T01:06:22.158539+0000 mon.a (mon.0) 3866 : audit [DBG] from='client.? 172.21.15.6:0/1744008065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:23.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:23 smithi130 bash[51302]: debug 2023-08-28T01:06:23.207+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:23.209536+0000) 2023-08-28T01:06:24.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:24 smithi130 bash[51302]: debug 2023-08-28T01:06:24.207+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:24.209869+0000) 2023-08-28T01:06:24.640 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:06:24.375Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:24.640 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:06:24.377Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:24.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:24 smithi006 bash[33918]: cluster 2023-08-28T01:06:23.581620+0000 mgr.y (mgr.14901) 5634 : cluster [DBG] pgmap v5641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:24.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:24 smithi006 bash[33918]: audit 2023-08-28T01:06:24.580384+0000 mon.c (mon.2) 2187 : audit [DBG] from='client.? 172.21.15.6:0/4057967257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:24 smithi006 bash[27292]: cluster 2023-08-28T01:06:23.581620+0000 mgr.y (mgr.14901) 5634 : cluster [DBG] pgmap v5641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:24.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:24 smithi006 bash[27292]: audit 2023-08-28T01:06:24.580384+0000 mon.c (mon.2) 2187 : audit [DBG] from='client.? 172.21.15.6:0/4057967257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:24 smithi130 bash[27579]: cluster 2023-08-28T01:06:23.581620+0000 mgr.y (mgr.14901) 5634 : cluster [DBG] pgmap v5641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:25.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:24 smithi130 bash[27579]: audit 2023-08-28T01:06:24.580384+0000 mon.c (mon.2) 2187 : audit [DBG] from='client.? 172.21.15.6:0/4057967257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:25.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:25 smithi130 bash[51302]: debug 2023-08-28T01:06:25.203+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:25.210142+0000) 2023-08-28T01:06:26.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:26 smithi130 bash[51302]: debug 2023-08-28T01:06:26.207+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:26.210448+0000) 2023-08-28T01:06:26.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:26 smithi006 bash[27292]: cluster 2023-08-28T01:06:25.582714+0000 mgr.y (mgr.14901) 5635 : cluster [DBG] pgmap v5642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:26.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:26 smithi006 bash[33918]: cluster 2023-08-28T01:06:25.582714+0000 mgr.y (mgr.14901) 5635 : cluster [DBG] pgmap v5642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:26 smithi130 bash[27579]: cluster 2023-08-28T01:06:25.582714+0000 mgr.y (mgr.14901) 5635 : cluster [DBG] pgmap v5642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:27.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:27 smithi130 bash[51302]: debug 2023-08-28T01:06:27.207+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:27.210774+0000) 2023-08-28T01:06:27.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:27 smithi006 bash[27292]: audit 2023-08-28T01:06:26.998811+0000 mon.a (mon.0) 3867 : audit [DBG] from='client.? 172.21.15.6:0/4165672408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:27.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:27 smithi006 bash[33918]: audit 2023-08-28T01:06:26.998811+0000 mon.a (mon.0) 3867 : audit [DBG] from='client.? 172.21.15.6:0/4165672408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:28.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:27 smithi130 bash[27579]: audit 2023-08-28T01:06:26.998811+0000 mon.a (mon.0) 3867 : audit [DBG] from='client.? 172.21.15.6:0/4165672408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:28.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:28 smithi130 bash[51302]: debug 2023-08-28T01:06:28.207+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:28.211053+0000) 2023-08-28T01:06:28.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:28 smithi006 bash[27292]: cluster 2023-08-28T01:06:27.583356+0000 mgr.y (mgr.14901) 5636 : cluster [DBG] pgmap v5643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:28.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:28 smithi006 bash[33918]: cluster 2023-08-28T01:06:27.583356+0000 mgr.y (mgr.14901) 5636 : cluster [DBG] pgmap v5643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:28 smithi130 bash[27579]: cluster 2023-08-28T01:06:27.583356+0000 mgr.y (mgr.14901) 5636 : cluster [DBG] pgmap v5643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:29.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:29 smithi130 bash[51302]: debug 2023-08-28T01:06:29.207+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:29.211330+0000) 2023-08-28T01:06:29.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:29 smithi006 bash[33918]: audit 2023-08-28T01:06:29.421357+0000 mon.c (mon.2) 2188 : audit [DBG] from='client.? 172.21.15.6:0/2713304670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:29.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:29 smithi006 bash[27292]: audit 2023-08-28T01:06:29.421357+0000 mon.c (mon.2) 2188 : audit [DBG] from='client.? 172.21.15.6:0/2713304670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:30.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:29 smithi130 bash[27579]: audit 2023-08-28T01:06:29.421357+0000 mon.c (mon.2) 2188 : audit [DBG] from='client.? 172.21.15.6:0/2713304670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:30.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:30 smithi130 bash[51302]: debug 2023-08-28T01:06:30.207+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:30.211630+0000) 2023-08-28T01:06:30.924 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:30 smithi006 bash[27292]: cluster 2023-08-28T01:06:29.584473+0000 mgr.y (mgr.14901) 5637 : cluster [DBG] pgmap v5644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:30.924 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:06:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:06:30] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:06:30.924 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:30 smithi006 bash[33918]: cluster 2023-08-28T01:06:29.584473+0000 mgr.y (mgr.14901) 5637 : cluster [DBG] pgmap v5644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:30 smithi130 bash[27579]: cluster 2023-08-28T01:06:29.584473+0000 mgr.y (mgr.14901) 5637 : cluster [DBG] pgmap v5644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:31.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:06:30.922Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:31.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:30 smithi006 bash[56298]: level=warn ts=2023-08-28T01:06:30.924Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:31.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:31 smithi130 bash[51302]: debug 2023-08-28T01:06:31.207+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:31.211853+0000) 2023-08-28T01:06:32.533 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:32 smithi130 bash[51302]: debug 2023-08-28T01:06:32.207+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:32.212146+0000) 2023-08-28T01:06:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:32 smithi130 bash[27579]: cluster 2023-08-28T01:06:31.585101+0000 mgr.y (mgr.14901) 5638 : cluster [DBG] pgmap v5645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:32 smithi130 bash[27579]: audit 2023-08-28T01:06:31.839416+0000 mon.b (mon.1) 661 : audit [DBG] from='client.? 172.21.15.6:0/3488255406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:32 smithi006 bash[33918]: cluster 2023-08-28T01:06:31.585101+0000 mgr.y (mgr.14901) 5638 : cluster [DBG] pgmap v5645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:32 smithi006 bash[33918]: audit 2023-08-28T01:06:31.839416+0000 mon.b (mon.1) 661 : audit [DBG] from='client.? 172.21.15.6:0/3488255406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:32 smithi006 bash[27292]: cluster 2023-08-28T01:06:31.585101+0000 mgr.y (mgr.14901) 5638 : cluster [DBG] pgmap v5645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:32.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:32 smithi006 bash[27292]: audit 2023-08-28T01:06:31.839416+0000 mon.b (mon.1) 661 : audit [DBG] from='client.? 172.21.15.6:0/3488255406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:32.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:06:32.495Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:32.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:06:32.497Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:33.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:33 smithi130 bash[51302]: debug 2023-08-28T01:06:33.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:33.212467+0000) 2023-08-28T01:06:34.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:34 smithi130 bash[51302]: debug 2023-08-28T01:06:34.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:34.212761+0000) 2023-08-28T01:06:34.644 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:06:34.375Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:34.644 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:06:34.377Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:34.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:34 smithi006 bash[33918]: cluster 2023-08-28T01:06:33.585797+0000 mgr.y (mgr.14901) 5639 : cluster [DBG] pgmap v5646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:34.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:34 smithi006 bash[33918]: audit 2023-08-28T01:06:34.254016+0000 mon.a (mon.0) 3868 : audit [DBG] from='client.? 172.21.15.6:0/2080205849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:34.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:34 smithi006 bash[27292]: cluster 2023-08-28T01:06:33.585797+0000 mgr.y (mgr.14901) 5639 : cluster [DBG] pgmap v5646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:34.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:34 smithi006 bash[27292]: audit 2023-08-28T01:06:34.254016+0000 mon.a (mon.0) 3868 : audit [DBG] from='client.? 172.21.15.6:0/2080205849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:34 smithi130 bash[27579]: cluster 2023-08-28T01:06:33.585797+0000 mgr.y (mgr.14901) 5639 : cluster [DBG] pgmap v5646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:34 smithi130 bash[27579]: audit 2023-08-28T01:06:34.254016+0000 mon.a (mon.0) 3868 : audit [DBG] from='client.? 172.21.15.6:0/2080205849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:35.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:35 smithi130 bash[51302]: debug 2023-08-28T01:06:35.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:35.213073+0000) 2023-08-28T01:06:36.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:36 smithi130 bash[51302]: debug 2023-08-28T01:06:36.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:36.213383+0000) 2023-08-28T01:06:36.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:36 smithi006 bash[27292]: cluster 2023-08-28T01:06:35.587017+0000 mgr.y (mgr.14901) 5640 : cluster [DBG] pgmap v5647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:36.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:36 smithi006 bash[33918]: cluster 2023-08-28T01:06:35.587017+0000 mgr.y (mgr.14901) 5640 : cluster [DBG] pgmap v5647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:36 smithi130 bash[27579]: cluster 2023-08-28T01:06:35.587017+0000 mgr.y (mgr.14901) 5640 : cluster [DBG] pgmap v5647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:37.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:37 smithi130 bash[51302]: debug 2023-08-28T01:06:37.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:37.213655+0000) 2023-08-28T01:06:37.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:37 smithi006 bash[27292]: audit 2023-08-28T01:06:36.670190+0000 mon.c (mon.2) 2189 : audit [DBG] from='client.? 172.21.15.6:0/1704206715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:37.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:37 smithi006 bash[33918]: audit 2023-08-28T01:06:36.670190+0000 mon.c (mon.2) 2189 : audit [DBG] from='client.? 172.21.15.6:0/1704206715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:38.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:37 smithi130 bash[27579]: audit 2023-08-28T01:06:36.670190+0000 mon.c (mon.2) 2189 : audit [DBG] from='client.? 172.21.15.6:0/1704206715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:38.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:38 smithi130 bash[51302]: debug 2023-08-28T01:06:38.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:38.213929+0000) 2023-08-28T01:06:38.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:38 smithi006 bash[27292]: cluster 2023-08-28T01:06:37.587745+0000 mgr.y (mgr.14901) 5641 : cluster [DBG] pgmap v5648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:38.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:38 smithi006 bash[33918]: cluster 2023-08-28T01:06:37.587745+0000 mgr.y (mgr.14901) 5641 : cluster [DBG] pgmap v5648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:38 smithi130 bash[27579]: cluster 2023-08-28T01:06:37.587745+0000 mgr.y (mgr.14901) 5641 : cluster [DBG] pgmap v5648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:39.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:39 smithi130 bash[51302]: debug 2023-08-28T01:06:39.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:39.214227+0000) 2023-08-28T01:06:39.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:39 smithi006 bash[27292]: audit 2023-08-28T01:06:39.087378+0000 mon.c (mon.2) 2190 : audit [DBG] from='client.? 172.21.15.6:0/2887114002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:39.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:39 smithi006 bash[33918]: audit 2023-08-28T01:06:39.087378+0000 mon.c (mon.2) 2190 : audit [DBG] from='client.? 172.21.15.6:0/2887114002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:40.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:39 smithi130 bash[27579]: audit 2023-08-28T01:06:39.087378+0000 mon.c (mon.2) 2190 : audit [DBG] from='client.? 172.21.15.6:0/2887114002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:40.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:40 smithi130 bash[51302]: debug 2023-08-28T01:06:40.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:40.214569+0000) 2023-08-28T01:06:40.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:06:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:06:40] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:06:40.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:40 smithi006 bash[33918]: cluster 2023-08-28T01:06:39.588888+0000 mgr.y (mgr.14901) 5642 : cluster [DBG] pgmap v5649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:40.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:40 smithi006 bash[27292]: cluster 2023-08-28T01:06:39.588888+0000 mgr.y (mgr.14901) 5642 : cluster [DBG] pgmap v5649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:06:40.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:06:40.925Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:40 smithi130 bash[27579]: cluster 2023-08-28T01:06:39.588888+0000 mgr.y (mgr.14901) 5642 : cluster [DBG] pgmap v5649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:41.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:41 smithi130 bash[51302]: debug 2023-08-28T01:06:41.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:41.214884+0000) 2023-08-28T01:06:41.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:41 smithi006 bash[27292]: audit 2023-08-28T01:06:41.498138+0000 mon.a (mon.0) 3869 : audit [DBG] from='client.? 172.21.15.6:0/929589369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:41.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:41 smithi006 bash[33918]: audit 2023-08-28T01:06:41.498138+0000 mon.a (mon.0) 3869 : audit [DBG] from='client.? 172.21.15.6:0/929589369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:42.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:41 smithi130 bash[27579]: audit 2023-08-28T01:06:41.498138+0000 mon.a (mon.0) 3869 : audit [DBG] from='client.? 172.21.15.6:0/929589369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:42.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:42 smithi130 bash[51302]: debug 2023-08-28T01:06:42.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:42.215131+0000) 2023-08-28T01:06:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:42 smithi006 bash[27292]: cluster 2023-08-28T01:06:41.589468+0000 mgr.y (mgr.14901) 5643 : cluster [DBG] pgmap v5650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:42.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:42 smithi006 bash[33918]: cluster 2023-08-28T01:06:41.589468+0000 mgr.y (mgr.14901) 5643 : cluster [DBG] pgmap v5650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:06:42.496Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:42.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:06:42.498Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:43.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:42 smithi130 bash[27579]: cluster 2023-08-28T01:06:41.589468+0000 mgr.y (mgr.14901) 5643 : cluster [DBG] pgmap v5650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:43.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:43 smithi130 bash[51302]: debug 2023-08-28T01:06:43.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:43.215433+0000) 2023-08-28T01:06:44.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:44 smithi130 bash[51302]: debug 2023-08-28T01:06:44.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:44.215707+0000) 2023-08-28T01:06:44.687 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:06:44.376Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:44.687 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:06:44.378Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:44.688 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:44 smithi006 bash[27292]: cluster 2023-08-28T01:06:43.590125+0000 mgr.y (mgr.14901) 5644 : cluster [DBG] pgmap v5651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:44.688 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:44 smithi006 bash[27292]: audit 2023-08-28T01:06:43.925728+0000 mon.a (mon.0) 3870 : audit [DBG] from='client.? 172.21.15.6:0/2136566340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:44.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:44 smithi006 bash[33918]: cluster 2023-08-28T01:06:43.590125+0000 mgr.y (mgr.14901) 5644 : cluster [DBG] pgmap v5651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:44.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:44 smithi006 bash[33918]: audit 2023-08-28T01:06:43.925728+0000 mon.a (mon.0) 3870 : audit [DBG] from='client.? 172.21.15.6:0/2136566340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:44 smithi130 bash[27579]: cluster 2023-08-28T01:06:43.590125+0000 mgr.y (mgr.14901) 5644 : cluster [DBG] pgmap v5651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:44 smithi130 bash[27579]: audit 2023-08-28T01:06:43.925728+0000 mon.a (mon.0) 3870 : audit [DBG] from='client.? 172.21.15.6:0/2136566340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:45.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:45 smithi130 bash[51302]: debug 2023-08-28T01:06:45.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:45.216022+0000) 2023-08-28T01:06:46.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:46 smithi130 bash[51302]: debug 2023-08-28T01:06:46.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:46.216297+0000) 2023-08-28T01:06:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:46 smithi006 bash[33918]: cluster 2023-08-28T01:06:45.591278+0000 mgr.y (mgr.14901) 5645 : cluster [DBG] pgmap v5652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:46 smithi006 bash[33918]: audit 2023-08-28T01:06:46.341026+0000 mon.a (mon.0) 3871 : audit [DBG] from='client.? 172.21.15.6:0/2494133943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:46 smithi006 bash[27292]: cluster 2023-08-28T01:06:45.591278+0000 mgr.y (mgr.14901) 5645 : cluster [DBG] pgmap v5652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:46 smithi006 bash[27292]: audit 2023-08-28T01:06:46.341026+0000 mon.a (mon.0) 3871 : audit [DBG] from='client.? 172.21.15.6:0/2494133943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:46 smithi130 bash[27579]: cluster 2023-08-28T01:06:45.591278+0000 mgr.y (mgr.14901) 5645 : cluster [DBG] pgmap v5652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:46 smithi130 bash[27579]: audit 2023-08-28T01:06:46.341026+0000 mon.a (mon.0) 3871 : audit [DBG] from='client.? 172.21.15.6:0/2494133943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:47.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:47 smithi130 bash[51302]: debug 2023-08-28T01:06:47.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:47.216596+0000) 2023-08-28T01:06:48.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:48 smithi130 bash[51302]: debug 2023-08-28T01:06:48.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:48.216875+0000) 2023-08-28T01:06:48.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:48 smithi006 bash[33918]: cluster 2023-08-28T01:06:47.591942+0000 mgr.y (mgr.14901) 5646 : cluster [DBG] pgmap v5653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:48.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:48 smithi006 bash[27292]: cluster 2023-08-28T01:06:47.591942+0000 mgr.y (mgr.14901) 5646 : cluster [DBG] pgmap v5653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:49.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:48 smithi130 bash[27579]: cluster 2023-08-28T01:06:47.591942+0000 mgr.y (mgr.14901) 5646 : cluster [DBG] pgmap v5653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:49.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:49 smithi130 bash[51302]: debug 2023-08-28T01:06:49.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:49.217151+0000) 2023-08-28T01:06:49.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:49 smithi006 bash[33918]: audit 2023-08-28T01:06:48.753717+0000 mon.a (mon.0) 3872 : audit [DBG] from='client.? 172.21.15.6:0/3478701253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:49.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:49 smithi006 bash[27292]: audit 2023-08-28T01:06:48.753717+0000 mon.a (mon.0) 3872 : audit [DBG] from='client.? 172.21.15.6:0/3478701253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:50.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:49 smithi130 bash[27579]: audit 2023-08-28T01:06:48.753717+0000 mon.a (mon.0) 3872 : audit [DBG] from='client.? 172.21.15.6:0/3478701253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:50.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:50 smithi130 bash[51302]: debug 2023-08-28T01:06:50.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:50.217475+0000) 2023-08-28T01:06:50.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:50 smithi006 bash[33918]: cluster 2023-08-28T01:06:49.593060+0000 mgr.y (mgr.14901) 5647 : cluster [DBG] pgmap v5654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:50.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:06:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:06:50] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:06:50.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:50 smithi006 bash[27292]: cluster 2023-08-28T01:06:49.593060+0000 mgr.y (mgr.14901) 5647 : cluster [DBG] pgmap v5654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:06:50.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:06:50.925Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:51.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:50 smithi130 bash[27579]: cluster 2023-08-28T01:06:49.593060+0000 mgr.y (mgr.14901) 5647 : cluster [DBG] pgmap v5654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:51.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:51 smithi130 bash[51302]: debug 2023-08-28T01:06:51.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:51.217747+0000) 2023-08-28T01:06:51.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:51 smithi006 bash[27292]: audit 2023-08-28T01:06:51.164534+0000 mon.c (mon.2) 2191 : audit [DBG] from='client.? 172.21.15.6:0/1300537303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:51.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:51 smithi006 bash[33918]: audit 2023-08-28T01:06:51.164534+0000 mon.c (mon.2) 2191 : audit [DBG] from='client.? 172.21.15.6:0/1300537303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:52.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:51 smithi130 bash[27579]: audit 2023-08-28T01:06:51.164534+0000 mon.c (mon.2) 2191 : audit [DBG] from='client.? 172.21.15.6:0/1300537303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:52.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:52 smithi130 bash[51302]: debug 2023-08-28T01:06:52.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:52.218013+0000) 2023-08-28T01:06:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:52 smithi006 bash[27292]: cluster 2023-08-28T01:06:51.593747+0000 mgr.y (mgr.14901) 5648 : cluster [DBG] pgmap v5655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:52 smithi006 bash[33918]: cluster 2023-08-28T01:06:51.593747+0000 mgr.y (mgr.14901) 5648 : cluster [DBG] pgmap v5655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:06:52.496Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:06:52.498Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:53.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:52 smithi130 bash[27579]: cluster 2023-08-28T01:06:51.593747+0000 mgr.y (mgr.14901) 5648 : cluster [DBG] pgmap v5655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:53.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:53 smithi130 bash[51302]: debug 2023-08-28T01:06:53.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:53.218278+0000) 2023-08-28T01:06:54.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:53 smithi130 bash[27579]: audit 2023-08-28T01:06:53.580221+0000 mon.c (mon.2) 2192 : audit [DBG] from='client.? 172.21.15.6:0/1587087036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:54.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:53 smithi006 bash[27292]: audit 2023-08-28T01:06:53.580221+0000 mon.c (mon.2) 2192 : audit [DBG] from='client.? 172.21.15.6:0/1587087036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:54.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:53 smithi006 bash[33918]: audit 2023-08-28T01:06:53.580221+0000 mon.c (mon.2) 2192 : audit [DBG] from='client.? 172.21.15.6:0/1587087036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:54.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:54 smithi130 bash[51302]: debug 2023-08-28T01:06:54.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:54.218565+0000) 2023-08-28T01:06:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:06:54.377Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:06:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:06:54.379Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:06:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:54 smithi130 bash[27579]: cluster 2023-08-28T01:06:53.594213+0000 mgr.y (mgr.14901) 5649 : cluster [DBG] pgmap v5656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:55.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:54 smithi006 bash[33918]: cluster 2023-08-28T01:06:53.594213+0000 mgr.y (mgr.14901) 5649 : cluster [DBG] pgmap v5656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:55.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:54 smithi006 bash[27292]: cluster 2023-08-28T01:06:53.594213+0000 mgr.y (mgr.14901) 5649 : cluster [DBG] pgmap v5656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:55.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:55 smithi130 bash[51302]: debug 2023-08-28T01:06:55.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:55.218923+0000) 2023-08-28T01:06:56.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:56 smithi130 bash[51302]: debug 2023-08-28T01:06:56.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:56.219190+0000) 2023-08-28T01:06:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:56 smithi130 bash[27579]: cluster 2023-08-28T01:06:55.595329+0000 mgr.y (mgr.14901) 5650 : cluster [DBG] pgmap v5657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:57.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:56 smithi130 bash[27579]: audit 2023-08-28T01:06:55.989945+0000 mon.a (mon.0) 3873 : audit [DBG] from='client.? 172.21.15.6:0/4278561937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:57.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:56 smithi006 bash[33918]: cluster 2023-08-28T01:06:55.595329+0000 mgr.y (mgr.14901) 5650 : cluster [DBG] pgmap v5657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:57.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:56 smithi006 bash[33918]: audit 2023-08-28T01:06:55.989945+0000 mon.a (mon.0) 3873 : audit [DBG] from='client.? 172.21.15.6:0/4278561937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:57.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:56 smithi006 bash[27292]: cluster 2023-08-28T01:06:55.595329+0000 mgr.y (mgr.14901) 5650 : cluster [DBG] pgmap v5657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:57.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:56 smithi006 bash[27292]: audit 2023-08-28T01:06:55.989945+0000 mon.a (mon.0) 3873 : audit [DBG] from='client.? 172.21.15.6:0/4278561937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:57.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:57 smithi130 bash[51302]: debug 2023-08-28T01:06:57.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:57.219517+0000) 2023-08-28T01:06:58.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:58 smithi130 bash[51302]: debug 2023-08-28T01:06:58.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:58.219792+0000) 2023-08-28T01:06:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:58 smithi130 bash[27579]: cluster 2023-08-28T01:06:57.596007+0000 mgr.y (mgr.14901) 5651 : cluster [DBG] pgmap v5658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:06:58 smithi130 bash[27579]: audit 2023-08-28T01:06:58.407368+0000 mon.c (mon.2) 2193 : audit [DBG] from='client.? 172.21.15.6:0/2666347429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:59.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:58 smithi006 bash[33918]: cluster 2023-08-28T01:06:57.596007+0000 mgr.y (mgr.14901) 5651 : cluster [DBG] pgmap v5658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:59.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:06:58 smithi006 bash[33918]: audit 2023-08-28T01:06:58.407368+0000 mon.c (mon.2) 2193 : audit [DBG] from='client.? 172.21.15.6:0/2666347429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:59.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:58 smithi006 bash[27292]: cluster 2023-08-28T01:06:57.596007+0000 mgr.y (mgr.14901) 5651 : cluster [DBG] pgmap v5658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:06:59.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:06:58 smithi006 bash[27292]: audit 2023-08-28T01:06:58.407368+0000 mon.c (mon.2) 2193 : audit [DBG] from='client.? 172.21.15.6:0/2666347429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:59.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:06:59 smithi130 bash[51302]: debug 2023-08-28T01:06:59.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:59.220102+0000) 2023-08-28T01:07:00.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:00 smithi130 bash[51302]: debug 2023-08-28T01:07:00.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:00.220374+0000) 2023-08-28T01:07:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:00 smithi130 bash[27579]: cluster 2023-08-28T01:06:59.597276+0000 mgr.y (mgr.14901) 5652 : cluster [DBG] pgmap v5659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:00 smithi130 bash[27579]: audit 2023-08-28T01:06:59.834072+0000 mon.b (mon.1) 662 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:07:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:00 smithi130 bash[27579]: audit 2023-08-28T01:07:00.106506+0000 mon.a (mon.0) 3874 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:07:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:00 smithi130 bash[27579]: audit 2023-08-28T01:07:00.112363+0000 mon.a (mon.0) 3875 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:07:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:07:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:07:00] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:07:01.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:07:00.924Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:01.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:07:00.926Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:00 smithi006 bash[33918]: cluster 2023-08-28T01:06:59.597276+0000 mgr.y (mgr.14901) 5652 : cluster [DBG] pgmap v5659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:00 smithi006 bash[33918]: audit 2023-08-28T01:06:59.834072+0000 mon.b (mon.1) 662 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:07:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:00 smithi006 bash[33918]: audit 2023-08-28T01:07:00.106506+0000 mon.a (mon.0) 3874 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:07:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:00 smithi006 bash[33918]: audit 2023-08-28T01:07:00.112363+0000 mon.a (mon.0) 3875 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:07:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:00 smithi006 bash[27292]: cluster 2023-08-28T01:06:59.597276+0000 mgr.y (mgr.14901) 5652 : cluster [DBG] pgmap v5659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:00 smithi006 bash[27292]: audit 2023-08-28T01:06:59.834072+0000 mon.b (mon.1) 662 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:07:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:00 smithi006 bash[27292]: audit 2023-08-28T01:07:00.106506+0000 mon.a (mon.0) 3874 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:07:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:00 smithi006 bash[27292]: audit 2023-08-28T01:07:00.112363+0000 mon.a (mon.0) 3875 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:07:01.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:01 smithi130 bash[51302]: debug 2023-08-28T01:07:01.219+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:01.220643+0000) 2023-08-28T01:07:02.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:01 smithi130 bash[27579]: audit 2023-08-28T01:07:00.825415+0000 mon.a (mon.0) 3876 : audit [DBG] from='client.? 172.21.15.6:0/1116359067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:02.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:01 smithi006 bash[33918]: audit 2023-08-28T01:07:00.825415+0000 mon.a (mon.0) 3876 : audit [DBG] from='client.? 172.21.15.6:0/1116359067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:02.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:01 smithi006 bash[27292]: audit 2023-08-28T01:07:00.825415+0000 mon.a (mon.0) 3876 : audit [DBG] from='client.? 172.21.15.6:0/1116359067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:02.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:02 smithi130 bash[51302]: debug 2023-08-28T01:07:02.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:02.220909+0000) 2023-08-28T01:07:02.753 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:07:02.497Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:02.754 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:07:02.500Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:03.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:02 smithi130 bash[27579]: cluster 2023-08-28T01:07:01.598003+0000 mgr.y (mgr.14901) 5653 : cluster [DBG] pgmap v5660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:03.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:02 smithi006 bash[27292]: cluster 2023-08-28T01:07:01.598003+0000 mgr.y (mgr.14901) 5653 : cluster [DBG] pgmap v5660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:03.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:02 smithi006 bash[33918]: cluster 2023-08-28T01:07:01.598003+0000 mgr.y (mgr.14901) 5653 : cluster [DBG] pgmap v5660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:03.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:03 smithi130 bash[51302]: debug 2023-08-28T01:07:03.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:03.221187+0000) 2023-08-28T01:07:04.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:03 smithi130 bash[27579]: audit 2023-08-28T01:07:03.242720+0000 mon.c (mon.2) 2194 : audit [DBG] from='client.? 172.21.15.6:0/27818977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:04.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:03 smithi006 bash[33918]: audit 2023-08-28T01:07:03.242720+0000 mon.c (mon.2) 2194 : audit [DBG] from='client.? 172.21.15.6:0/27818977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:04.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:03 smithi006 bash[27292]: audit 2023-08-28T01:07:03.242720+0000 mon.c (mon.2) 2194 : audit [DBG] from='client.? 172.21.15.6:0/27818977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:04.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:04 smithi130 bash[51302]: debug 2023-08-28T01:07:04.219+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:04.221547+0000) 2023-08-28T01:07:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:07:04.378Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:07:04.380Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:04 smithi130 bash[27579]: cluster 2023-08-28T01:07:03.598681+0000 mgr.y (mgr.14901) 5654 : cluster [DBG] pgmap v5661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:05.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:04 smithi006 bash[27292]: cluster 2023-08-28T01:07:03.598681+0000 mgr.y (mgr.14901) 5654 : cluster [DBG] pgmap v5661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:05.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:04 smithi006 bash[33918]: cluster 2023-08-28T01:07:03.598681+0000 mgr.y (mgr.14901) 5654 : cluster [DBG] pgmap v5661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:05.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:05 smithi130 bash[51302]: debug 2023-08-28T01:07:05.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:05.221852+0000) 2023-08-28T01:07:06.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:05 smithi130 bash[27579]: audit 2023-08-28T01:07:05.669475+0000 mon.a (mon.0) 3877 : audit [DBG] from='client.? 172.21.15.6:0/2448506896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:06.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:05 smithi006 bash[27292]: audit 2023-08-28T01:07:05.669475+0000 mon.a (mon.0) 3877 : audit [DBG] from='client.? 172.21.15.6:0/2448506896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:06.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:05 smithi006 bash[33918]: audit 2023-08-28T01:07:05.669475+0000 mon.a (mon.0) 3877 : audit [DBG] from='client.? 172.21.15.6:0/2448506896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:06.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:06 smithi130 bash[51302]: debug 2023-08-28T01:07:06.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:06.222030+0000) 2023-08-28T01:07:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:06 smithi130 bash[27579]: cluster 2023-08-28T01:07:05.599456+0000 mgr.y (mgr.14901) 5655 : cluster [DBG] pgmap v5662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:06 smithi130 bash[27579]: audit 2023-08-28T01:07:06.004903+0000 mon.a (mon.0) 3878 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:07:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:06 smithi130 bash[27579]: audit 2023-08-28T01:07:06.406385+0000 mon.a (mon.0) 3879 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:07:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:06 smithi130 bash[27579]: audit 2023-08-28T01:07:06.701047+0000 mon.b (mon.1) 663 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:07:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:06 smithi130 bash[27579]: audit 2023-08-28T01:07:06.703321+0000 mon.b (mon.1) 664 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:07:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:06 smithi130 bash[27579]: audit 2023-08-28T01:07:06.712168+0000 mon.a (mon.0) 3880 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:07:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:06 smithi006 bash[27292]: cluster 2023-08-28T01:07:05.599456+0000 mgr.y (mgr.14901) 5655 : cluster [DBG] pgmap v5662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:06 smithi006 bash[27292]: audit 2023-08-28T01:07:06.004903+0000 mon.a (mon.0) 3878 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:07:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:06 smithi006 bash[27292]: audit 2023-08-28T01:07:06.406385+0000 mon.a (mon.0) 3879 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:07:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:06 smithi006 bash[27292]: audit 2023-08-28T01:07:06.701047+0000 mon.b (mon.1) 663 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:07:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:06 smithi006 bash[27292]: audit 2023-08-28T01:07:06.703321+0000 mon.b (mon.1) 664 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:07:07.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:06 smithi006 bash[27292]: audit 2023-08-28T01:07:06.712168+0000 mon.a (mon.0) 3880 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:07:07.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:06 smithi006 bash[33918]: cluster 2023-08-28T01:07:05.599456+0000 mgr.y (mgr.14901) 5655 : cluster [DBG] pgmap v5662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:07.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:06 smithi006 bash[33918]: audit 2023-08-28T01:07:06.004903+0000 mon.a (mon.0) 3878 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:07:07.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:06 smithi006 bash[33918]: audit 2023-08-28T01:07:06.406385+0000 mon.a (mon.0) 3879 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:07:07.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:06 smithi006 bash[33918]: audit 2023-08-28T01:07:06.701047+0000 mon.b (mon.1) 663 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:07:07.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:06 smithi006 bash[33918]: audit 2023-08-28T01:07:06.703321+0000 mon.b (mon.1) 664 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:07:07.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:06 smithi006 bash[33918]: audit 2023-08-28T01:07:06.712168+0000 mon.a (mon.0) 3880 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:07:07.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:07 smithi130 bash[51302]: debug 2023-08-28T01:07:07.218+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:07.222300+0000) 2023-08-28T01:07:08.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:08 smithi130 bash[51302]: debug 2023-08-28T01:07:08.214+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:08.222563+0000) 2023-08-28T01:07:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:08 smithi130 bash[27579]: cluster 2023-08-28T01:07:07.600092+0000 mgr.y (mgr.14901) 5656 : cluster [DBG] pgmap v5663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:08 smithi130 bash[27579]: audit 2023-08-28T01:07:08.087222+0000 mon.a (mon.0) 3881 : audit [DBG] from='client.? 172.21.15.6:0/2237146235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:09.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:08 smithi006 bash[27292]: cluster 2023-08-28T01:07:07.600092+0000 mgr.y (mgr.14901) 5656 : cluster [DBG] pgmap v5663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:09.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:08 smithi006 bash[27292]: audit 2023-08-28T01:07:08.087222+0000 mon.a (mon.0) 3881 : audit [DBG] from='client.? 172.21.15.6:0/2237146235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:09.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:08 smithi006 bash[33918]: cluster 2023-08-28T01:07:07.600092+0000 mgr.y (mgr.14901) 5656 : cluster [DBG] pgmap v5663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:09.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:08 smithi006 bash[33918]: audit 2023-08-28T01:07:08.087222+0000 mon.a (mon.0) 3881 : audit [DBG] from='client.? 172.21.15.6:0/2237146235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:09.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:09 smithi130 bash[51302]: debug 2023-08-28T01:07:09.218+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:09.222846+0000) 2023-08-28T01:07:10.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:10 smithi130 bash[51302]: debug 2023-08-28T01:07:10.218+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:10.223106+0000) 2023-08-28T01:07:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:10 smithi130 bash[27579]: cluster 2023-08-28T01:07:09.601232+0000 mgr.y (mgr.14901) 5657 : cluster [DBG] pgmap v5664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:10 smithi130 bash[27579]: audit 2023-08-28T01:07:10.525805+0000 mon.a (mon.0) 3882 : audit [DBG] from='client.? 172.21.15.6:0/3980734444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:07:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:07:10] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:07:11.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:10 smithi006 bash[27292]: cluster 2023-08-28T01:07:09.601232+0000 mgr.y (mgr.14901) 5657 : cluster [DBG] pgmap v5664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:11.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:10 smithi006 bash[27292]: audit 2023-08-28T01:07:10.525805+0000 mon.a (mon.0) 3882 : audit [DBG] from='client.? 172.21.15.6:0/3980734444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:10 smithi006 bash[33918]: cluster 2023-08-28T01:07:09.601232+0000 mgr.y (mgr.14901) 5657 : cluster [DBG] pgmap v5664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:10 smithi006 bash[33918]: audit 2023-08-28T01:07:10.525805+0000 mon.a (mon.0) 3882 : audit [DBG] from='client.? 172.21.15.6:0/3980734444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:07:10.925Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:07:10.927Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:11.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:11 smithi130 bash[51302]: debug 2023-08-28T01:07:11.218+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:11.223409+0000) 2023-08-28T01:07:12.542 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:12 smithi130 bash[51302]: debug 2023-08-28T01:07:12.218+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:12.223708+0000) 2023-08-28T01:07:12.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:12 smithi130 bash[27579]: cluster 2023-08-28T01:07:11.601817+0000 mgr.y (mgr.14901) 5658 : cluster [DBG] pgmap v5665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:12 smithi006 bash[33918]: cluster 2023-08-28T01:07:11.601817+0000 mgr.y (mgr.14901) 5658 : cluster [DBG] pgmap v5665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:12 smithi006 bash[27292]: cluster 2023-08-28T01:07:11.601817+0000 mgr.y (mgr.14901) 5658 : cluster [DBG] pgmap v5665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:07:12.499Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:12.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:07:12.500Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:13.544 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:13 smithi130 bash[51302]: debug 2023-08-28T01:07:13.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:13.224018+0000) 2023-08-28T01:07:13.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:13 smithi130 bash[27579]: audit 2023-08-28T01:07:12.945231+0000 mon.c (mon.2) 2195 : audit [DBG] from='client.? 172.21.15.6:0/3028735825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:13.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:13 smithi006 bash[27292]: audit 2023-08-28T01:07:12.945231+0000 mon.c (mon.2) 2195 : audit [DBG] from='client.? 172.21.15.6:0/3028735825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:13.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:13 smithi006 bash[33918]: audit 2023-08-28T01:07:12.945231+0000 mon.c (mon.2) 2195 : audit [DBG] from='client.? 172.21.15.6:0/3028735825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:14.547 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:14 smithi130 bash[51302]: debug 2023-08-28T01:07:14.218+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:14.224403+0000) 2023-08-28T01:07:14.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:14 smithi006 bash[27292]: cluster 2023-08-28T01:07:13.603165+0000 mgr.y (mgr.14901) 5659 : cluster [DBG] pgmap v5666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:14.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:14 smithi006 bash[33918]: cluster 2023-08-28T01:07:13.603165+0000 mgr.y (mgr.14901) 5659 : cluster [DBG] pgmap v5666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:14.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:07:14.378Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:14.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:07:14.380Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:14.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:14 smithi130 bash[27579]: cluster 2023-08-28T01:07:13.603165+0000 mgr.y (mgr.14901) 5659 : cluster [DBG] pgmap v5666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:15.559 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:15 smithi130 bash[51302]: debug 2023-08-28T01:07:15.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:15.224732+0000) 2023-08-28T01:07:15.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:15 smithi130 bash[27579]: audit 2023-08-28T01:07:15.359077+0000 mon.c (mon.2) 2196 : audit [DBG] from='client.? 172.21.15.6:0/2291165645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:15.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:15 smithi006 bash[27292]: audit 2023-08-28T01:07:15.359077+0000 mon.c (mon.2) 2196 : audit [DBG] from='client.? 172.21.15.6:0/2291165645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:15.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:15 smithi006 bash[33918]: audit 2023-08-28T01:07:15.359077+0000 mon.c (mon.2) 2196 : audit [DBG] from='client.? 172.21.15.6:0/2291165645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:16.569 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:16 smithi130 bash[51302]: debug 2023-08-28T01:07:16.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:16.225080+0000) 2023-08-28T01:07:16.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:16 smithi130 bash[27579]: cluster 2023-08-28T01:07:15.604313+0000 mgr.y (mgr.14901) 5660 : cluster [DBG] pgmap v5667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:16.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:16 smithi006 bash[27292]: cluster 2023-08-28T01:07:15.604313+0000 mgr.y (mgr.14901) 5660 : cluster [DBG] pgmap v5667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:16.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:16 smithi006 bash[33918]: cluster 2023-08-28T01:07:15.604313+0000 mgr.y (mgr.14901) 5660 : cluster [DBG] pgmap v5667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:17.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:17 smithi130 bash[51302]: debug 2023-08-28T01:07:17.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:17.225342+0000) 2023-08-28T01:07:18.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:18 smithi130 bash[51302]: debug 2023-08-28T01:07:18.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:18.225668+0000) 2023-08-28T01:07:18.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:18 smithi006 bash[33918]: cluster 2023-08-28T01:07:17.604984+0000 mgr.y (mgr.14901) 5661 : cluster [DBG] pgmap v5668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:18.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:18 smithi006 bash[33918]: audit 2023-08-28T01:07:17.773281+0000 mon.a (mon.0) 3883 : audit [DBG] from='client.? 172.21.15.6:0/3330988861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:18.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:18 smithi006 bash[27292]: cluster 2023-08-28T01:07:17.604984+0000 mgr.y (mgr.14901) 5661 : cluster [DBG] pgmap v5668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:18.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:18 smithi006 bash[27292]: audit 2023-08-28T01:07:17.773281+0000 mon.a (mon.0) 3883 : audit [DBG] from='client.? 172.21.15.6:0/3330988861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:18 smithi130 bash[27579]: cluster 2023-08-28T01:07:17.604984+0000 mgr.y (mgr.14901) 5661 : cluster [DBG] pgmap v5668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:18 smithi130 bash[27579]: audit 2023-08-28T01:07:17.773281+0000 mon.a (mon.0) 3883 : audit [DBG] from='client.? 172.21.15.6:0/3330988861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:19.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:19 smithi130 bash[51302]: debug 2023-08-28T01:07:19.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:19.225945+0000) 2023-08-28T01:07:20.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:20 smithi130 bash[51302]: debug 2023-08-28T01:07:20.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:20.226232+0000) 2023-08-28T01:07:20.928 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:07:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:07:20] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:07:20.928 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:20 smithi006 bash[33918]: cluster 2023-08-28T01:07:19.606018+0000 mgr.y (mgr.14901) 5662 : cluster [DBG] pgmap v5669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:20.928 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:20 smithi006 bash[33918]: audit 2023-08-28T01:07:20.187778+0000 mon.c (mon.2) 2197 : audit [DBG] from='client.? 172.21.15.6:0/790439959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:20.928 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:20 smithi006 bash[27292]: cluster 2023-08-28T01:07:19.606018+0000 mgr.y (mgr.14901) 5662 : cluster [DBG] pgmap v5669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:20.928 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:20 smithi006 bash[27292]: audit 2023-08-28T01:07:20.187778+0000 mon.c (mon.2) 2197 : audit [DBG] from='client.? 172.21.15.6:0/790439959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:20 smithi130 bash[27579]: cluster 2023-08-28T01:07:19.606018+0000 mgr.y (mgr.14901) 5662 : cluster [DBG] pgmap v5669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:20 smithi130 bash[27579]: audit 2023-08-28T01:07:20.187778+0000 mon.c (mon.2) 2197 : audit [DBG] from='client.? 172.21.15.6:0/790439959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:21.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:07:20.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:21.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:07:20.928Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:21.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:21 smithi130 bash[51302]: debug 2023-08-28T01:07:21.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:21.226516+0000) 2023-08-28T01:07:22.545 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:22 smithi130 bash[51302]: debug 2023-08-28T01:07:22.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:22.226814+0000) 2023-08-28T01:07:22.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:22 smithi130 bash[27579]: cluster 2023-08-28T01:07:21.606680+0000 mgr.y (mgr.14901) 5663 : cluster [DBG] pgmap v5670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:22.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:22 smithi006 bash[33918]: cluster 2023-08-28T01:07:21.606680+0000 mgr.y (mgr.14901) 5663 : cluster [DBG] pgmap v5670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:22.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:22 smithi006 bash[27292]: cluster 2023-08-28T01:07:21.606680+0000 mgr.y (mgr.14901) 5663 : cluster [DBG] pgmap v5670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:22.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:07:22.499Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:22.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:07:22.501Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:23.547 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:23 smithi130 bash[51302]: debug 2023-08-28T01:07:23.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:23.227143+0000) 2023-08-28T01:07:23.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:23 smithi130 bash[27579]: audit 2023-08-28T01:07:22.611824+0000 mon.c (mon.2) 2198 : audit [DBG] from='client.? 172.21.15.6:0/4235737274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:23.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:23 smithi006 bash[27292]: audit 2023-08-28T01:07:22.611824+0000 mon.c (mon.2) 2198 : audit [DBG] from='client.? 172.21.15.6:0/4235737274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:23.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:23 smithi006 bash[33918]: audit 2023-08-28T01:07:22.611824+0000 mon.c (mon.2) 2198 : audit [DBG] from='client.? 172.21.15.6:0/4235737274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:24.557 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:24 smithi130 bash[51302]: debug 2023-08-28T01:07:24.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:24.227470+0000) 2023-08-28T01:07:24.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:24 smithi006 bash[27292]: cluster 2023-08-28T01:07:23.607362+0000 mgr.y (mgr.14901) 5664 : cluster [DBG] pgmap v5671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:24.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:24 smithi006 bash[33918]: cluster 2023-08-28T01:07:23.607362+0000 mgr.y (mgr.14901) 5664 : cluster [DBG] pgmap v5671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:24.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:07:24.379Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:24.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:07:24.381Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:24.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:24 smithi130 bash[27579]: cluster 2023-08-28T01:07:23.607362+0000 mgr.y (mgr.14901) 5664 : cluster [DBG] pgmap v5671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:25.560 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:25 smithi130 bash[51302]: debug 2023-08-28T01:07:25.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:25.227744+0000) 2023-08-28T01:07:25.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:25 smithi130 bash[27579]: audit 2023-08-28T01:07:25.031982+0000 mon.c (mon.2) 2199 : audit [DBG] from='client.? 172.21.15.6:0/326693829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:25.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:25 smithi006 bash[27292]: audit 2023-08-28T01:07:25.031982+0000 mon.c (mon.2) 2199 : audit [DBG] from='client.? 172.21.15.6:0/326693829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:25.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:25 smithi006 bash[33918]: audit 2023-08-28T01:07:25.031982+0000 mon.c (mon.2) 2199 : audit [DBG] from='client.? 172.21.15.6:0/326693829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:26.567 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:26 smithi130 bash[51302]: debug 2023-08-28T01:07:26.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:26.228021+0000) 2023-08-28T01:07:26.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:26 smithi130 bash[27579]: cluster 2023-08-28T01:07:25.608427+0000 mgr.y (mgr.14901) 5665 : cluster [DBG] pgmap v5672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:26.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:26 smithi006 bash[33918]: cluster 2023-08-28T01:07:25.608427+0000 mgr.y (mgr.14901) 5665 : cluster [DBG] pgmap v5672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:26.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:26 smithi006 bash[27292]: cluster 2023-08-28T01:07:25.608427+0000 mgr.y (mgr.14901) 5665 : cluster [DBG] pgmap v5672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:27.571 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:27 smithi130 bash[51302]: debug 2023-08-28T01:07:27.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:27.228336+0000) 2023-08-28T01:07:27.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:27 smithi130 bash[27579]: audit 2023-08-28T01:07:27.447213+0000 mon.c (mon.2) 2200 : audit [DBG] from='client.? 172.21.15.6:0/417254103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:27.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:27 smithi006 bash[27292]: audit 2023-08-28T01:07:27.447213+0000 mon.c (mon.2) 2200 : audit [DBG] from='client.? 172.21.15.6:0/417254103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:27.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:27 smithi006 bash[33918]: audit 2023-08-28T01:07:27.447213+0000 mon.c (mon.2) 2200 : audit [DBG] from='client.? 172.21.15.6:0/417254103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:28.578 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:28 smithi130 bash[51302]: debug 2023-08-28T01:07:28.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:28.228613+0000) 2023-08-28T01:07:28.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:28 smithi130 bash[27579]: cluster 2023-08-28T01:07:27.609142+0000 mgr.y (mgr.14901) 5666 : cluster [DBG] pgmap v5673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:28.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:28 smithi006 bash[27292]: cluster 2023-08-28T01:07:27.609142+0000 mgr.y (mgr.14901) 5666 : cluster [DBG] pgmap v5673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:28.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:28 smithi006 bash[33918]: cluster 2023-08-28T01:07:27.609142+0000 mgr.y (mgr.14901) 5666 : cluster [DBG] pgmap v5673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:29.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:29 smithi130 bash[51302]: debug 2023-08-28T01:07:29.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:29.228882+0000) 2023-08-28T01:07:30.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:30 smithi130 bash[51302]: debug 2023-08-28T01:07:30.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:30.229158+0000) 2023-08-28T01:07:30.929 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:07:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:07:30] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:07:30.929 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:30 smithi006 bash[33918]: cluster 2023-08-28T01:07:29.610171+0000 mgr.y (mgr.14901) 5667 : cluster [DBG] pgmap v5674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:30.929 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:30 smithi006 bash[33918]: audit 2023-08-28T01:07:29.870948+0000 mon.a (mon.0) 3884 : audit [DBG] from='client.? 172.21.15.6:0/489505549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:30.930 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:30 smithi006 bash[27292]: cluster 2023-08-28T01:07:29.610171+0000 mgr.y (mgr.14901) 5667 : cluster [DBG] pgmap v5674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:30.930 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:30 smithi006 bash[27292]: audit 2023-08-28T01:07:29.870948+0000 mon.a (mon.0) 3884 : audit [DBG] from='client.? 172.21.15.6:0/489505549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:30 smithi130 bash[27579]: cluster 2023-08-28T01:07:29.610171+0000 mgr.y (mgr.14901) 5667 : cluster [DBG] pgmap v5674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:30 smithi130 bash[27579]: audit 2023-08-28T01:07:29.870948+0000 mon.a (mon.0) 3884 : audit [DBG] from='client.? 172.21.15.6:0/489505549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:31.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:07:30.927Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:31.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:30 smithi006 bash[56298]: level=warn ts=2023-08-28T01:07:30.930Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:31.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:31 smithi130 bash[51302]: debug 2023-08-28T01:07:31.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:31.229422+0000) 2023-08-28T01:07:32.546 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:32 smithi130 bash[51302]: debug 2023-08-28T01:07:32.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:32.229736+0000) 2023-08-28T01:07:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:32 smithi130 bash[27579]: cluster 2023-08-28T01:07:31.610843+0000 mgr.y (mgr.14901) 5668 : cluster [DBG] pgmap v5675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:32.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:32 smithi130 bash[27579]: audit 2023-08-28T01:07:32.281828+0000 mon.c (mon.2) 2201 : audit [DBG] from='client.? 172.21.15.6:0/174998139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:32 smithi006 bash[27292]: cluster 2023-08-28T01:07:31.610843+0000 mgr.y (mgr.14901) 5668 : cluster [DBG] pgmap v5675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:32 smithi006 bash[27292]: audit 2023-08-28T01:07:32.281828+0000 mon.c (mon.2) 2201 : audit [DBG] from='client.? 172.21.15.6:0/174998139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:07:32.500Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:07:32.502Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:32 smithi006 bash[33918]: cluster 2023-08-28T01:07:31.610843+0000 mgr.y (mgr.14901) 5668 : cluster [DBG] pgmap v5675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:32 smithi006 bash[33918]: audit 2023-08-28T01:07:32.281828+0000 mon.c (mon.2) 2201 : audit [DBG] from='client.? 172.21.15.6:0/174998139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:33.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:33 smithi130 bash[51302]: debug 2023-08-28T01:07:33.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:33.230036+0000) 2023-08-28T01:07:34.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:34 smithi130 bash[51302]: debug 2023-08-28T01:07:34.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:34.230370+0000) 2023-08-28T01:07:34.670 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:07:34.379Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:34.670 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:07:34.381Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:34.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:34 smithi006 bash[27292]: cluster 2023-08-28T01:07:33.611567+0000 mgr.y (mgr.14901) 5669 : cluster [DBG] pgmap v5676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:34.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:34 smithi006 bash[33918]: cluster 2023-08-28T01:07:33.611567+0000 mgr.y (mgr.14901) 5669 : cluster [DBG] pgmap v5676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:35.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:34 smithi130 bash[27579]: cluster 2023-08-28T01:07:33.611567+0000 mgr.y (mgr.14901) 5669 : cluster [DBG] pgmap v5676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:35.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:35 smithi130 bash[51302]: debug 2023-08-28T01:07:35.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:35.230659+0000) 2023-08-28T01:07:35.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:35 smithi006 bash[27292]: audit 2023-08-28T01:07:34.692792+0000 mon.a (mon.0) 3885 : audit [DBG] from='client.? 172.21.15.6:0/4237726828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:35.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:35 smithi006 bash[33918]: audit 2023-08-28T01:07:34.692792+0000 mon.a (mon.0) 3885 : audit [DBG] from='client.? 172.21.15.6:0/4237726828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:36.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:35 smithi130 bash[27579]: audit 2023-08-28T01:07:34.692792+0000 mon.a (mon.0) 3885 : audit [DBG] from='client.? 172.21.15.6:0/4237726828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:36.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:36 smithi130 bash[51302]: debug 2023-08-28T01:07:36.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:36.230978+0000) 2023-08-28T01:07:36.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:36 smithi006 bash[33918]: cluster 2023-08-28T01:07:35.612615+0000 mgr.y (mgr.14901) 5670 : cluster [DBG] pgmap v5677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:36.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:36 smithi006 bash[27292]: cluster 2023-08-28T01:07:35.612615+0000 mgr.y (mgr.14901) 5670 : cluster [DBG] pgmap v5677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:36 smithi130 bash[27579]: cluster 2023-08-28T01:07:35.612615+0000 mgr.y (mgr.14901) 5670 : cluster [DBG] pgmap v5677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:37.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:37 smithi130 bash[51302]: debug 2023-08-28T01:07:37.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:37.231264+0000) 2023-08-28T01:07:37.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:37 smithi006 bash[27292]: audit 2023-08-28T01:07:37.106408+0000 mon.c (mon.2) 2202 : audit [DBG] from='client.? 172.21.15.6:0/1901586597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:37.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:37 smithi006 bash[33918]: audit 2023-08-28T01:07:37.106408+0000 mon.c (mon.2) 2202 : audit [DBG] from='client.? 172.21.15.6:0/1901586597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:38.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:37 smithi130 bash[27579]: audit 2023-08-28T01:07:37.106408+0000 mon.c (mon.2) 2202 : audit [DBG] from='client.? 172.21.15.6:0/1901586597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:38.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:38 smithi130 bash[51302]: debug 2023-08-28T01:07:38.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:38.231531+0000) 2023-08-28T01:07:38.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:38 smithi006 bash[33918]: cluster 2023-08-28T01:07:37.613299+0000 mgr.y (mgr.14901) 5671 : cluster [DBG] pgmap v5678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:38.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:38 smithi006 bash[27292]: cluster 2023-08-28T01:07:37.613299+0000 mgr.y (mgr.14901) 5671 : cluster [DBG] pgmap v5678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:38 smithi130 bash[27579]: cluster 2023-08-28T01:07:37.613299+0000 mgr.y (mgr.14901) 5671 : cluster [DBG] pgmap v5678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:39.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:39 smithi130 bash[51302]: debug 2023-08-28T01:07:39.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:39.231809+0000) 2023-08-28T01:07:39.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:39 smithi006 bash[33918]: audit 2023-08-28T01:07:39.523045+0000 mon.a (mon.0) 3886 : audit [DBG] from='client.? 172.21.15.6:0/1518452221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:39.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:39 smithi006 bash[27292]: audit 2023-08-28T01:07:39.523045+0000 mon.a (mon.0) 3886 : audit [DBG] from='client.? 172.21.15.6:0/1518452221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:40.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:39 smithi130 bash[27579]: audit 2023-08-28T01:07:39.523045+0000 mon.a (mon.0) 3886 : audit [DBG] from='client.? 172.21.15.6:0/1518452221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:40.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:40 smithi130 bash[51302]: debug 2023-08-28T01:07:40.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:40.232102+0000) 2023-08-28T01:07:40.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:40 smithi006 bash[27292]: cluster 2023-08-28T01:07:39.614452+0000 mgr.y (mgr.14901) 5672 : cluster [DBG] pgmap v5679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:40.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:07:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:07:40] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:07:40.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:40 smithi006 bash[33918]: cluster 2023-08-28T01:07:39.614452+0000 mgr.y (mgr.14901) 5672 : cluster [DBG] pgmap v5679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:07:40.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:07:40.930Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:40 smithi130 bash[27579]: cluster 2023-08-28T01:07:39.614452+0000 mgr.y (mgr.14901) 5672 : cluster [DBG] pgmap v5679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:41.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:41 smithi130 bash[51302]: debug 2023-08-28T01:07:41.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:41.232476+0000) 2023-08-28T01:07:42.550 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:42 smithi130 bash[51302]: debug 2023-08-28T01:07:42.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:42.232785+0000) 2023-08-28T01:07:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:42 smithi130 bash[27579]: cluster 2023-08-28T01:07:41.615105+0000 mgr.y (mgr.14901) 5673 : cluster [DBG] pgmap v5680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:42 smithi130 bash[27579]: audit 2023-08-28T01:07:41.938734+0000 mon.c (mon.2) 2203 : audit [DBG] from='client.? 172.21.15.6:0/965205936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:42 smithi006 bash[27292]: cluster 2023-08-28T01:07:41.615105+0000 mgr.y (mgr.14901) 5673 : cluster [DBG] pgmap v5680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:42 smithi006 bash[27292]: audit 2023-08-28T01:07:41.938734+0000 mon.c (mon.2) 2203 : audit [DBG] from='client.? 172.21.15.6:0/965205936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:42 smithi006 bash[33918]: cluster 2023-08-28T01:07:41.615105+0000 mgr.y (mgr.14901) 5673 : cluster [DBG] pgmap v5680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:42 smithi006 bash[33918]: audit 2023-08-28T01:07:41.938734+0000 mon.c (mon.2) 2203 : audit [DBG] from='client.? 172.21.15.6:0/965205936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:42.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:07:42.501Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:42.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:07:42.503Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:43.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:43 smithi130 bash[51302]: debug 2023-08-28T01:07:43.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:43.233064+0000) 2023-08-28T01:07:44.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:44 smithi130 bash[51302]: debug 2023-08-28T01:07:44.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:44.233338+0000) 2023-08-28T01:07:44.674 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:07:44.380Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:44.674 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:07:44.382Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:44.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:44 smithi006 bash[33918]: cluster 2023-08-28T01:07:43.615736+0000 mgr.y (mgr.14901) 5674 : cluster [DBG] pgmap v5681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:44.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:44 smithi006 bash[33918]: audit 2023-08-28T01:07:44.352534+0000 mon.c (mon.2) 2204 : audit [DBG] from='client.? 172.21.15.6:0/3476132111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:44.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:44 smithi006 bash[27292]: cluster 2023-08-28T01:07:43.615736+0000 mgr.y (mgr.14901) 5674 : cluster [DBG] pgmap v5681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:44.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:44 smithi006 bash[27292]: audit 2023-08-28T01:07:44.352534+0000 mon.c (mon.2) 2204 : audit [DBG] from='client.? 172.21.15.6:0/3476132111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:44 smithi130 bash[27579]: cluster 2023-08-28T01:07:43.615736+0000 mgr.y (mgr.14901) 5674 : cluster [DBG] pgmap v5681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:45.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:44 smithi130 bash[27579]: audit 2023-08-28T01:07:44.352534+0000 mon.c (mon.2) 2204 : audit [DBG] from='client.? 172.21.15.6:0/3476132111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:45.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:45 smithi130 bash[51302]: debug 2023-08-28T01:07:45.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:45.233618+0000) 2023-08-28T01:07:46.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:46 smithi130 bash[51302]: debug 2023-08-28T01:07:46.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:46.233870+0000) 2023-08-28T01:07:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:46 smithi006 bash[33918]: cluster 2023-08-28T01:07:45.616886+0000 mgr.y (mgr.14901) 5675 : cluster [DBG] pgmap v5682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:46.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:46 smithi006 bash[27292]: cluster 2023-08-28T01:07:45.616886+0000 mgr.y (mgr.14901) 5675 : cluster [DBG] pgmap v5682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:46 smithi130 bash[27579]: cluster 2023-08-28T01:07:45.616886+0000 mgr.y (mgr.14901) 5675 : cluster [DBG] pgmap v5682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:47.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:47 smithi130 bash[51302]: debug 2023-08-28T01:07:47.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:47.234162+0000) 2023-08-28T01:07:47.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:47 smithi006 bash[27292]: audit 2023-08-28T01:07:46.769430+0000 mon.c (mon.2) 2205 : audit [DBG] from='client.? 172.21.15.6:0/2524890087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:47.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:47 smithi006 bash[33918]: audit 2023-08-28T01:07:46.769430+0000 mon.c (mon.2) 2205 : audit [DBG] from='client.? 172.21.15.6:0/2524890087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:48.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:47 smithi130 bash[27579]: audit 2023-08-28T01:07:46.769430+0000 mon.c (mon.2) 2205 : audit [DBG] from='client.? 172.21.15.6:0/2524890087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:48.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:48 smithi130 bash[51302]: debug 2023-08-28T01:07:48.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:48.234451+0000) 2023-08-28T01:07:48.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:48 smithi006 bash[27292]: cluster 2023-08-28T01:07:47.617470+0000 mgr.y (mgr.14901) 5676 : cluster [DBG] pgmap v5683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:48.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:48 smithi006 bash[33918]: cluster 2023-08-28T01:07:47.617470+0000 mgr.y (mgr.14901) 5676 : cluster [DBG] pgmap v5683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:49.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:48 smithi130 bash[27579]: cluster 2023-08-28T01:07:47.617470+0000 mgr.y (mgr.14901) 5676 : cluster [DBG] pgmap v5683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:49.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:49 smithi130 bash[51302]: debug 2023-08-28T01:07:49.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:49.234741+0000) 2023-08-28T01:07:49.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:49 smithi006 bash[27292]: audit 2023-08-28T01:07:49.188497+0000 mon.c (mon.2) 2206 : audit [DBG] from='client.? 172.21.15.6:0/4187756491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:49.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:49 smithi006 bash[33918]: audit 2023-08-28T01:07:49.188497+0000 mon.c (mon.2) 2206 : audit [DBG] from='client.? 172.21.15.6:0/4187756491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:50.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:49 smithi130 bash[27579]: audit 2023-08-28T01:07:49.188497+0000 mon.c (mon.2) 2206 : audit [DBG] from='client.? 172.21.15.6:0/4187756491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:50.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:50 smithi130 bash[51302]: debug 2023-08-28T01:07:50.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:50.235030+0000) 2023-08-28T01:07:50.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:07:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:07:50] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:07:50.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:50 smithi006 bash[33918]: cluster 2023-08-28T01:07:49.618608+0000 mgr.y (mgr.14901) 5677 : cluster [DBG] pgmap v5684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:50.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:50 smithi006 bash[27292]: cluster 2023-08-28T01:07:49.618608+0000 mgr.y (mgr.14901) 5677 : cluster [DBG] pgmap v5684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:07:50.929Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:50.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:07:50.931Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:50 smithi130 bash[27579]: cluster 2023-08-28T01:07:49.618608+0000 mgr.y (mgr.14901) 5677 : cluster [DBG] pgmap v5684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:51.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:51 smithi130 bash[51302]: debug 2023-08-28T01:07:51.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:51.235258+0000) 2023-08-28T01:07:52.551 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:52 smithi130 bash[51302]: debug 2023-08-28T01:07:52.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:52.235477+0000) 2023-08-28T01:07:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:52 smithi130 bash[27579]: audit 2023-08-28T01:07:51.598971+0000 mon.a (mon.0) 3887 : audit [DBG] from='client.? 172.21.15.6:0/3823259685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:52 smithi130 bash[27579]: cluster 2023-08-28T01:07:51.619211+0000 mgr.y (mgr.14901) 5678 : cluster [DBG] pgmap v5685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:52 smithi006 bash[27292]: audit 2023-08-28T01:07:51.598971+0000 mon.a (mon.0) 3887 : audit [DBG] from='client.? 172.21.15.6:0/3823259685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:52 smithi006 bash[27292]: cluster 2023-08-28T01:07:51.619211+0000 mgr.y (mgr.14901) 5678 : cluster [DBG] pgmap v5685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:52 smithi006 bash[33918]: audit 2023-08-28T01:07:51.598971+0000 mon.a (mon.0) 3887 : audit [DBG] from='client.? 172.21.15.6:0/3823259685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:52 smithi006 bash[33918]: cluster 2023-08-28T01:07:51.619211+0000 mgr.y (mgr.14901) 5678 : cluster [DBG] pgmap v5685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:52.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:07:52.501Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:52.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:07:52.503Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:53.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:53 smithi130 bash[51302]: debug 2023-08-28T01:07:53.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:53.235777+0000) 2023-08-28T01:07:54.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:54 smithi130 bash[51302]: debug 2023-08-28T01:07:54.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:54.236097+0000) 2023-08-28T01:07:54.680 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:07:54.381Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:54.680 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:07:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:07:54.383Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:07:54.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:54 smithi006 bash[27292]: cluster 2023-08-28T01:07:53.619836+0000 mgr.y (mgr.14901) 5679 : cluster [DBG] pgmap v5686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:54.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:54 smithi006 bash[27292]: audit 2023-08-28T01:07:54.015949+0000 mon.a (mon.0) 3888 : audit [DBG] from='client.? 172.21.15.6:0/2311935258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:54 smithi006 bash[33918]: cluster 2023-08-28T01:07:53.619836+0000 mgr.y (mgr.14901) 5679 : cluster [DBG] pgmap v5686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:54.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:54 smithi006 bash[33918]: audit 2023-08-28T01:07:54.015949+0000 mon.a (mon.0) 3888 : audit [DBG] from='client.? 172.21.15.6:0/2311935258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:55.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:54 smithi130 bash[27579]: cluster 2023-08-28T01:07:53.619836+0000 mgr.y (mgr.14901) 5679 : cluster [DBG] pgmap v5686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:54 smithi130 bash[27579]: audit 2023-08-28T01:07:54.015949+0000 mon.a (mon.0) 3888 : audit [DBG] from='client.? 172.21.15.6:0/2311935258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:55.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:55 smithi130 bash[51302]: debug 2023-08-28T01:07:55.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:55.236364+0000) 2023-08-28T01:07:56.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:56 smithi130 bash[51302]: debug 2023-08-28T01:07:56.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:56.236640+0000) 2023-08-28T01:07:56.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:56 smithi006 bash[33918]: cluster 2023-08-28T01:07:55.621007+0000 mgr.y (mgr.14901) 5680 : cluster [DBG] pgmap v5687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:56.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:56 smithi006 bash[33918]: audit 2023-08-28T01:07:56.428196+0000 mon.c (mon.2) 2207 : audit [DBG] from='client.? 172.21.15.6:0/2847651966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:56.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:56 smithi006 bash[27292]: cluster 2023-08-28T01:07:55.621007+0000 mgr.y (mgr.14901) 5680 : cluster [DBG] pgmap v5687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:56.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:56 smithi006 bash[27292]: audit 2023-08-28T01:07:56.428196+0000 mon.c (mon.2) 2207 : audit [DBG] from='client.? 172.21.15.6:0/2847651966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:56 smithi130 bash[27579]: cluster 2023-08-28T01:07:55.621007+0000 mgr.y (mgr.14901) 5680 : cluster [DBG] pgmap v5687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:57.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:56 smithi130 bash[27579]: audit 2023-08-28T01:07:56.428196+0000 mon.c (mon.2) 2207 : audit [DBG] from='client.? 172.21.15.6:0/2847651966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:57.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:57 smithi130 bash[51302]: debug 2023-08-28T01:07:57.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:57.236945+0000) 2023-08-28T01:07:58.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:58 smithi130 bash[51302]: debug 2023-08-28T01:07:58.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:58.237206+0000) 2023-08-28T01:07:59.319 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:58 smithi006 bash[33918]: cluster 2023-08-28T01:07:57.621643+0000 mgr.y (mgr.14901) 5681 : cluster [DBG] pgmap v5688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:59.319 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:58 smithi130 bash[27579]: cluster 2023-08-28T01:07:57.621643+0000 mgr.y (mgr.14901) 5681 : cluster [DBG] pgmap v5688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:59.319 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:58 smithi006 bash[27292]: cluster 2023-08-28T01:07:57.621643+0000 mgr.y (mgr.14901) 5681 : cluster [DBG] pgmap v5688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:07:59.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:07:59 smithi130 bash[51302]: debug 2023-08-28T01:07:59.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:59.237471+0000) 2023-08-28T01:07:59.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:07:59 smithi006 bash[27292]: audit 2023-08-28T01:07:58.843188+0000 mon.c (mon.2) 2208 : audit [DBG] from='client.? 172.21.15.6:0/974368771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:00.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:07:59 smithi006 bash[33918]: audit 2023-08-28T01:07:58.843188+0000 mon.c (mon.2) 2208 : audit [DBG] from='client.? 172.21.15.6:0/974368771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:00.217 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:07:59 smithi130 bash[27579]: audit 2023-08-28T01:07:58.843188+0000 mon.c (mon.2) 2208 : audit [DBG] from='client.? 172.21.15.6:0/974368771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:00.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:00 smithi130 bash[51302]: debug 2023-08-28T01:08:00.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:00.237802+0000) 2023-08-28T01:08:00.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:08:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:08:00] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T01:08:01.022 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:00 smithi006 bash[33918]: cluster 2023-08-28T01:07:59.622733+0000 mgr.y (mgr.14901) 5682 : cluster [DBG] pgmap v5689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:01.029 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:00 smithi006 bash[27292]: cluster 2023-08-28T01:07:59.622733+0000 mgr.y (mgr.14901) 5682 : cluster [DBG] pgmap v5689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:01.029 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:08:00.930Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:01.030 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:08:00.931Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:00 smithi130 bash[27579]: cluster 2023-08-28T01:07:59.622733+0000 mgr.y (mgr.14901) 5682 : cluster [DBG] pgmap v5689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:01.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:01 smithi130 bash[51302]: debug 2023-08-28T01:08:01.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:01.238117+0000) 2023-08-28T01:08:01.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:01 smithi006 bash[33918]: audit 2023-08-28T01:08:01.260267+0000 mon.c (mon.2) 2209 : audit [DBG] from='client.? 172.21.15.6:0/172126005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:01.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:01 smithi006 bash[27292]: audit 2023-08-28T01:08:01.260267+0000 mon.c (mon.2) 2209 : audit [DBG] from='client.? 172.21.15.6:0/172126005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:02.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:01 smithi130 bash[27579]: audit 2023-08-28T01:08:01.260267+0000 mon.c (mon.2) 2209 : audit [DBG] from='client.? 172.21.15.6:0/172126005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:02.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:02 smithi130 bash[51302]: debug 2023-08-28T01:08:02.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:02.238394+0000) 2023-08-28T01:08:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:02 smithi006 bash[33918]: cluster 2023-08-28T01:08:01.623399+0000 mgr.y (mgr.14901) 5683 : cluster [DBG] pgmap v5690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:02 smithi006 bash[27292]: cluster 2023-08-28T01:08:01.623399+0000 mgr.y (mgr.14901) 5683 : cluster [DBG] pgmap v5690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:02.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:08:02.502Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:02.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:08:02.504Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:02 smithi130 bash[27579]: cluster 2023-08-28T01:08:01.623399+0000 mgr.y (mgr.14901) 5683 : cluster [DBG] pgmap v5690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:03.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:03 smithi130 bash[51302]: debug 2023-08-28T01:08:03.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:03.238672+0000) 2023-08-28T01:08:04.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:03 smithi130 bash[27579]: audit 2023-08-28T01:08:03.668931+0000 mon.c (mon.2) 2210 : audit [DBG] from='client.? 172.21.15.6:0/1930148950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:04.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:03 smithi006 bash[27292]: audit 2023-08-28T01:08:03.668931+0000 mon.c (mon.2) 2210 : audit [DBG] from='client.? 172.21.15.6:0/1930148950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:04.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:03 smithi006 bash[33918]: audit 2023-08-28T01:08:03.668931+0000 mon.c (mon.2) 2210 : audit [DBG] from='client.? 172.21.15.6:0/1930148950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:04.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:04 smithi130 bash[51302]: debug 2023-08-28T01:08:04.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:04.238981+0000) 2023-08-28T01:08:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:08:04.382Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:08:04.384Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:05.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:04 smithi130 bash[27579]: cluster 2023-08-28T01:08:03.624027+0000 mgr.y (mgr.14901) 5684 : cluster [DBG] pgmap v5691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:05.386 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:04 smithi006 bash[27292]: cluster 2023-08-28T01:08:03.624027+0000 mgr.y (mgr.14901) 5684 : cluster [DBG] pgmap v5691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:05.386 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:04 smithi006 bash[33918]: cluster 2023-08-28T01:08:03.624027+0000 mgr.y (mgr.14901) 5684 : cluster [DBG] pgmap v5691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:05.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:05 smithi130 bash[51302]: debug 2023-08-28T01:08:05.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:05.239243+0000) 2023-08-28T01:08:06.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:06 smithi130 bash[51302]: debug 2023-08-28T01:08:06.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:06.239525+0000) 2023-08-28T01:08:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:06 smithi130 bash[27579]: cluster 2023-08-28T01:08:05.625181+0000 mgr.y (mgr.14901) 5685 : cluster [DBG] pgmap v5692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:06 smithi130 bash[27579]: audit 2023-08-28T01:08:06.082323+0000 mon.a (mon.0) 3889 : audit [DBG] from='client.? 172.21.15.6:0/45774228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:06 smithi130 bash[27579]: audit 2023-08-28T01:08:06.722273+0000 mon.b (mon.1) 665 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:08:07.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:06 smithi006 bash[33918]: cluster 2023-08-28T01:08:05.625181+0000 mgr.y (mgr.14901) 5685 : cluster [DBG] pgmap v5692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:07.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:06 smithi006 bash[33918]: audit 2023-08-28T01:08:06.082323+0000 mon.a (mon.0) 3889 : audit [DBG] from='client.? 172.21.15.6:0/45774228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:07.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:06 smithi006 bash[33918]: audit 2023-08-28T01:08:06.722273+0000 mon.b (mon.1) 665 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:08:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:06 smithi006 bash[27292]: cluster 2023-08-28T01:08:05.625181+0000 mgr.y (mgr.14901) 5685 : cluster [DBG] pgmap v5692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:06 smithi006 bash[27292]: audit 2023-08-28T01:08:06.082323+0000 mon.a (mon.0) 3889 : audit [DBG] from='client.? 172.21.15.6:0/45774228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:07.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:06 smithi006 bash[27292]: audit 2023-08-28T01:08:06.722273+0000 mon.b (mon.1) 665 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:08:07.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:07 smithi130 bash[51302]: debug 2023-08-28T01:08:07.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:07.239877+0000) 2023-08-28T01:08:08.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:08 smithi130 bash[51302]: debug 2023-08-28T01:08:08.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:08.240144+0000) 2023-08-28T01:08:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:08 smithi130 bash[27579]: cluster 2023-08-28T01:08:07.625722+0000 mgr.y (mgr.14901) 5686 : cluster [DBG] pgmap v5693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:09.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:08 smithi130 bash[27579]: audit 2023-08-28T01:08:08.490546+0000 mon.a (mon.0) 3890 : audit [DBG] from='client.? 172.21.15.6:0/87293983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:09.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:08 smithi006 bash[27292]: cluster 2023-08-28T01:08:07.625722+0000 mgr.y (mgr.14901) 5686 : cluster [DBG] pgmap v5693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:09.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:08 smithi006 bash[27292]: audit 2023-08-28T01:08:08.490546+0000 mon.a (mon.0) 3890 : audit [DBG] from='client.? 172.21.15.6:0/87293983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:09.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:08 smithi006 bash[33918]: cluster 2023-08-28T01:08:07.625722+0000 mgr.y (mgr.14901) 5686 : cluster [DBG] pgmap v5693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:09.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:08 smithi006 bash[33918]: audit 2023-08-28T01:08:08.490546+0000 mon.a (mon.0) 3890 : audit [DBG] from='client.? 172.21.15.6:0/87293983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:09.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:09 smithi130 bash[51302]: debug 2023-08-28T01:08:09.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:09.240432+0000) 2023-08-28T01:08:10.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:10 smithi130 bash[51302]: debug 2023-08-28T01:08:10.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:10.240740+0000) 2023-08-28T01:08:11.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:10 smithi130 bash[27579]: cluster 2023-08-28T01:08:09.626896+0000 mgr.y (mgr.14901) 5687 : cluster [DBG] pgmap v5694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:08:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:08:10] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T01:08:11.234 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:10 smithi006 bash[33918]: cluster 2023-08-28T01:08:09.626896+0000 mgr.y (mgr.14901) 5687 : cluster [DBG] pgmap v5694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:11.234 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:08:10.930Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:11.234 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:08:10.932Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:11.252 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:10 smithi006 bash[27292]: cluster 2023-08-28T01:08:09.626896+0000 mgr.y (mgr.14901) 5687 : cluster [DBG] pgmap v5694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:11.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:11 smithi130 bash[51302]: debug 2023-08-28T01:08:11.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:11.241014+0000) 2023-08-28T01:08:12.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:11 smithi130 bash[27579]: audit 2023-08-28T01:08:10.902686+0000 mon.c (mon.2) 2211 : audit [DBG] from='client.? 172.21.15.6:0/2890083725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:12.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:11 smithi006 bash[33918]: audit 2023-08-28T01:08:10.902686+0000 mon.c (mon.2) 2211 : audit [DBG] from='client.? 172.21.15.6:0/2890083725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:12.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:11 smithi006 bash[27292]: audit 2023-08-28T01:08:10.902686+0000 mon.c (mon.2) 2211 : audit [DBG] from='client.? 172.21.15.6:0/2890083725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:12.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:12 smithi130 bash[51302]: debug 2023-08-28T01:08:12.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:12.241217+0000) 2023-08-28T01:08:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:12 smithi006 bash[27292]: cluster 2023-08-28T01:08:11.627481+0000 mgr.y (mgr.14901) 5688 : cluster [DBG] pgmap v5695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:12 smithi006 bash[27292]: audit 2023-08-28T01:08:12.633686+0000 mon.a (mon.0) 3891 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:08:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:12 smithi006 bash[33918]: cluster 2023-08-28T01:08:11.627481+0000 mgr.y (mgr.14901) 5688 : cluster [DBG] pgmap v5695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:12 smithi006 bash[33918]: audit 2023-08-28T01:08:12.633686+0000 mon.a (mon.0) 3891 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:08:12.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:08:12.502Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:12.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:08:12.505Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:12 smithi130 bash[27579]: cluster 2023-08-28T01:08:11.627481+0000 mgr.y (mgr.14901) 5688 : cluster [DBG] pgmap v5695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:13.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:12 smithi130 bash[27579]: audit 2023-08-28T01:08:12.633686+0000 mon.a (mon.0) 3891 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:08:13.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:13 smithi130 bash[51302]: debug 2023-08-28T01:08:13.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:13.241378+0000) 2023-08-28T01:08:14.370 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:14 smithi130 bash[51302]: debug 2023-08-28T01:08:14.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:14.241613+0000) 2023-08-28T01:08:14.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:14 smithi130 bash[27579]: audit 2023-08-28T01:08:13.010256+0000 mon.a (mon.0) 3892 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:08:14.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:14 smithi130 bash[27579]: audit 2023-08-28T01:08:13.277908+0000 mon.b (mon.1) 666 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:08:14.370 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:14 smithi130 bash[27579]: audit 2023-08-28T01:08:13.279915+0000 mon.b (mon.1) 667 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:08:14.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:14 smithi130 bash[27579]: audit 2023-08-28T01:08:13.286742+0000 mon.a (mon.0) 3893 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:08:14.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:14 smithi130 bash[27579]: audit 2023-08-28T01:08:13.323100+0000 mon.c (mon.2) 2212 : audit [DBG] from='client.? 172.21.15.6:0/551873358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:14.371 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:14 smithi130 bash[27579]: cluster 2023-08-28T01:08:13.628027+0000 mgr.y (mgr.14901) 5689 : cluster [DBG] pgmap v5696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:14.385 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:14 smithi006 bash[33918]: audit 2023-08-28T01:08:13.010256+0000 mon.a (mon.0) 3892 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:08:14.385 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:14 smithi006 bash[33918]: audit 2023-08-28T01:08:13.277908+0000 mon.b (mon.1) 666 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:08:14.385 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:14 smithi006 bash[33918]: audit 2023-08-28T01:08:13.279915+0000 mon.b (mon.1) 667 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:08:14.385 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:14 smithi006 bash[33918]: audit 2023-08-28T01:08:13.286742+0000 mon.a (mon.0) 3893 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:08:14.385 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:14 smithi006 bash[33918]: audit 2023-08-28T01:08:13.323100+0000 mon.c (mon.2) 2212 : audit [DBG] from='client.? 172.21.15.6:0/551873358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:14.385 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:14 smithi006 bash[33918]: cluster 2023-08-28T01:08:13.628027+0000 mgr.y (mgr.14901) 5689 : cluster [DBG] pgmap v5696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:14.386 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:14 smithi006 bash[27292]: audit 2023-08-28T01:08:13.010256+0000 mon.a (mon.0) 3892 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:08:14.386 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:14 smithi006 bash[27292]: audit 2023-08-28T01:08:13.277908+0000 mon.b (mon.1) 666 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:08:14.386 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:14 smithi006 bash[27292]: audit 2023-08-28T01:08:13.279915+0000 mon.b (mon.1) 667 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:08:14.386 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:14 smithi006 bash[27292]: audit 2023-08-28T01:08:13.286742+0000 mon.a (mon.0) 3893 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:08:14.386 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:14 smithi006 bash[27292]: audit 2023-08-28T01:08:13.323100+0000 mon.c (mon.2) 2212 : audit [DBG] from='client.? 172.21.15.6:0/551873358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:14.386 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:14 smithi006 bash[27292]: cluster 2023-08-28T01:08:13.628027+0000 mgr.y (mgr.14901) 5689 : cluster [DBG] pgmap v5696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:08:14.383Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:08:14.385Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:15.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:15 smithi130 bash[51302]: debug 2023-08-28T01:08:15.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:15.241964+0000) 2023-08-28T01:08:16.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:16 smithi130 bash[51302]: debug 2023-08-28T01:08:16.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:16.242275+0000) 2023-08-28T01:08:16.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:16 smithi006 bash[27292]: cluster 2023-08-28T01:08:15.629075+0000 mgr.y (mgr.14901) 5690 : cluster [DBG] pgmap v5697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:16.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:16 smithi006 bash[27292]: audit 2023-08-28T01:08:15.733353+0000 mon.c (mon.2) 2213 : audit [DBG] from='client.? 172.21.15.6:0/455511571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:16 smithi006 bash[33918]: cluster 2023-08-28T01:08:15.629075+0000 mgr.y (mgr.14901) 5690 : cluster [DBG] pgmap v5697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:16.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:16 smithi006 bash[33918]: audit 2023-08-28T01:08:15.733353+0000 mon.c (mon.2) 2213 : audit [DBG] from='client.? 172.21.15.6:0/455511571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:16 smithi130 bash[27579]: cluster 2023-08-28T01:08:15.629075+0000 mgr.y (mgr.14901) 5690 : cluster [DBG] pgmap v5697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:16 smithi130 bash[27579]: audit 2023-08-28T01:08:15.733353+0000 mon.c (mon.2) 2213 : audit [DBG] from='client.? 172.21.15.6:0/455511571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:17.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:17 smithi130 bash[51302]: debug 2023-08-28T01:08:17.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:17.242576+0000) 2023-08-28T01:08:18.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:18 smithi130 bash[51302]: debug 2023-08-28T01:08:18.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:18.242910+0000) 2023-08-28T01:08:18.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:18 smithi006 bash[27292]: cluster 2023-08-28T01:08:17.629664+0000 mgr.y (mgr.14901) 5691 : cluster [DBG] pgmap v5698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:18.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:18 smithi006 bash[27292]: audit 2023-08-28T01:08:18.142099+0000 mon.c (mon.2) 2214 : audit [DBG] from='client.? 172.21.15.6:0/976830758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:18.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:18 smithi006 bash[33918]: cluster 2023-08-28T01:08:17.629664+0000 mgr.y (mgr.14901) 5691 : cluster [DBG] pgmap v5698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:18.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:18 smithi006 bash[33918]: audit 2023-08-28T01:08:18.142099+0000 mon.c (mon.2) 2214 : audit [DBG] from='client.? 172.21.15.6:0/976830758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:18 smithi130 bash[27579]: cluster 2023-08-28T01:08:17.629664+0000 mgr.y (mgr.14901) 5691 : cluster [DBG] pgmap v5698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:18 smithi130 bash[27579]: audit 2023-08-28T01:08:18.142099+0000 mon.c (mon.2) 2214 : audit [DBG] from='client.? 172.21.15.6:0/976830758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:19.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:19 smithi130 bash[51302]: debug 2023-08-28T01:08:19.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:19.243184+0000) 2023-08-28T01:08:20.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:20 smithi130 bash[51302]: debug 2023-08-28T01:08:20.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:20.243492+0000) 2023-08-28T01:08:20.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:08:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:08:20] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:08:20.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:08:20.931Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:20.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:08:20.932Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:20.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:20 smithi006 bash[27292]: cluster 2023-08-28T01:08:19.630738+0000 mgr.y (mgr.14901) 5692 : cluster [DBG] pgmap v5699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:20.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:20 smithi006 bash[27292]: audit 2023-08-28T01:08:20.564215+0000 mon.a (mon.0) 3894 : audit [DBG] from='client.? 172.21.15.6:0/328449341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:20.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:20 smithi006 bash[33918]: cluster 2023-08-28T01:08:19.630738+0000 mgr.y (mgr.14901) 5692 : cluster [DBG] pgmap v5699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:20.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:20 smithi006 bash[33918]: audit 2023-08-28T01:08:20.564215+0000 mon.a (mon.0) 3894 : audit [DBG] from='client.? 172.21.15.6:0/328449341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:21.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:20 smithi130 bash[27579]: cluster 2023-08-28T01:08:19.630738+0000 mgr.y (mgr.14901) 5692 : cluster [DBG] pgmap v5699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:20 smithi130 bash[27579]: audit 2023-08-28T01:08:20.564215+0000 mon.a (mon.0) 3894 : audit [DBG] from='client.? 172.21.15.6:0/328449341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:21.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:21 smithi130 bash[51302]: debug 2023-08-28T01:08:21.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:21.243771+0000) 2023-08-28T01:08:22.560 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:22 smithi130 bash[51302]: debug 2023-08-28T01:08:22.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:22.244032+0000) 2023-08-28T01:08:22.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:22 smithi130 bash[27579]: cluster 2023-08-28T01:08:21.631417+0000 mgr.y (mgr.14901) 5693 : cluster [DBG] pgmap v5700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:22.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:22 smithi006 bash[27292]: cluster 2023-08-28T01:08:21.631417+0000 mgr.y (mgr.14901) 5693 : cluster [DBG] pgmap v5700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:22.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:08:22.503Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:22.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:08:22.505Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:22 smithi006 bash[33918]: cluster 2023-08-28T01:08:21.631417+0000 mgr.y (mgr.14901) 5693 : cluster [DBG] pgmap v5700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:23.563 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:23 smithi130 bash[51302]: debug 2023-08-28T01:08:23.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:23.244337+0000) 2023-08-28T01:08:23.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:23 smithi130 bash[27579]: audit 2023-08-28T01:08:22.977704+0000 mon.c (mon.2) 2215 : audit [DBG] from='client.? 172.21.15.6:0/2642401674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:23.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:23 smithi006 bash[27292]: audit 2023-08-28T01:08:22.977704+0000 mon.c (mon.2) 2215 : audit [DBG] from='client.? 172.21.15.6:0/2642401674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:23.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:23 smithi006 bash[33918]: audit 2023-08-28T01:08:22.977704+0000 mon.c (mon.2) 2215 : audit [DBG] from='client.? 172.21.15.6:0/2642401674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:24.573 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:24 smithi130 bash[51302]: debug 2023-08-28T01:08:24.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:24.244652+0000) 2023-08-28T01:08:24.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:24 smithi006 bash[27292]: cluster 2023-08-28T01:08:23.632084+0000 mgr.y (mgr.14901) 5694 : cluster [DBG] pgmap v5701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:24.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:24 smithi006 bash[33918]: cluster 2023-08-28T01:08:23.632084+0000 mgr.y (mgr.14901) 5694 : cluster [DBG] pgmap v5701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:24.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:08:24.384Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:24.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:08:24.387Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:24.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:24 smithi130 bash[27579]: cluster 2023-08-28T01:08:23.632084+0000 mgr.y (mgr.14901) 5694 : cluster [DBG] pgmap v5701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:25.576 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:25 smithi130 bash[51302]: debug 2023-08-28T01:08:25.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:25.244976+0000) 2023-08-28T01:08:25.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:25 smithi130 bash[27579]: audit 2023-08-28T01:08:25.388261+0000 mon.c (mon.2) 2216 : audit [DBG] from='client.? 172.21.15.6:0/3740679447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:25.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:25 smithi006 bash[33918]: audit 2023-08-28T01:08:25.388261+0000 mon.c (mon.2) 2216 : audit [DBG] from='client.? 172.21.15.6:0/3740679447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:25.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:25 smithi006 bash[27292]: audit 2023-08-28T01:08:25.388261+0000 mon.c (mon.2) 2216 : audit [DBG] from='client.? 172.21.15.6:0/3740679447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:26.587 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:26 smithi130 bash[51302]: debug 2023-08-28T01:08:26.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:26.245183+0000) 2023-08-28T01:08:26.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:26 smithi130 bash[27579]: cluster 2023-08-28T01:08:25.633276+0000 mgr.y (mgr.14901) 5695 : cluster [DBG] pgmap v5702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:26.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:26 smithi006 bash[27292]: cluster 2023-08-28T01:08:25.633276+0000 mgr.y (mgr.14901) 5695 : cluster [DBG] pgmap v5702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:26.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:26 smithi006 bash[33918]: cluster 2023-08-28T01:08:25.633276+0000 mgr.y (mgr.14901) 5695 : cluster [DBG] pgmap v5702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:27.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:27 smithi130 bash[51302]: debug 2023-08-28T01:08:27.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:27.245446+0000) 2023-08-28T01:08:28.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:28 smithi130 bash[51302]: debug 2023-08-28T01:08:28.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:28.245713+0000) 2023-08-28T01:08:28.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:28 smithi006 bash[33918]: cluster 2023-08-28T01:08:27.633757+0000 mgr.y (mgr.14901) 5696 : cluster [DBG] pgmap v5703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:28.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:28 smithi006 bash[33918]: audit 2023-08-28T01:08:27.807212+0000 mon.c (mon.2) 2217 : audit [DBG] from='client.? 172.21.15.6:0/2798369381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:28.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:28 smithi006 bash[27292]: cluster 2023-08-28T01:08:27.633757+0000 mgr.y (mgr.14901) 5696 : cluster [DBG] pgmap v5703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:28.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:28 smithi006 bash[27292]: audit 2023-08-28T01:08:27.807212+0000 mon.c (mon.2) 2217 : audit [DBG] from='client.? 172.21.15.6:0/2798369381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:28 smithi130 bash[27579]: cluster 2023-08-28T01:08:27.633757+0000 mgr.y (mgr.14901) 5696 : cluster [DBG] pgmap v5703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:29.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:28 smithi130 bash[27579]: audit 2023-08-28T01:08:27.807212+0000 mon.c (mon.2) 2217 : audit [DBG] from='client.? 172.21.15.6:0/2798369381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:29.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:29 smithi130 bash[51302]: debug 2023-08-28T01:08:29.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:29.246004+0000) 2023-08-28T01:08:30.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:30 smithi130 bash[51302]: debug 2023-08-28T01:08:30.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:30.246342+0000) 2023-08-28T01:08:30.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:08:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:08:30] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:08:30.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:30 smithi006 bash[27292]: cluster 2023-08-28T01:08:29.634845+0000 mgr.y (mgr.14901) 5697 : cluster [DBG] pgmap v5704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:30.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:30 smithi006 bash[27292]: audit 2023-08-28T01:08:30.220979+0000 mon.a (mon.0) 3895 : audit [DBG] from='client.? 172.21.15.6:0/3887010703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:30.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:30 smithi006 bash[33918]: cluster 2023-08-28T01:08:29.634845+0000 mgr.y (mgr.14901) 5697 : cluster [DBG] pgmap v5704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:30.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:30 smithi006 bash[33918]: audit 2023-08-28T01:08:30.220979+0000 mon.a (mon.0) 3895 : audit [DBG] from='client.? 172.21.15.6:0/3887010703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:08:30.931Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:30.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:30 smithi006 bash[56298]: level=warn ts=2023-08-28T01:08:30.933Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:31.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:30 smithi130 bash[27579]: cluster 2023-08-28T01:08:29.634845+0000 mgr.y (mgr.14901) 5697 : cluster [DBG] pgmap v5704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:30 smithi130 bash[27579]: audit 2023-08-28T01:08:30.220979+0000 mon.a (mon.0) 3895 : audit [DBG] from='client.? 172.21.15.6:0/3887010703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:31.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:31 smithi130 bash[51302]: debug 2023-08-28T01:08:31.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:31.246660+0000) 2023-08-28T01:08:32.561 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:32 smithi130 bash[51302]: debug 2023-08-28T01:08:32.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:32.247005+0000) 2023-08-28T01:08:32.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:32 smithi130 bash[27579]: cluster 2023-08-28T01:08:31.635518+0000 mgr.y (mgr.14901) 5698 : cluster [DBG] pgmap v5705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:32 smithi006 bash[27292]: cluster 2023-08-28T01:08:31.635518+0000 mgr.y (mgr.14901) 5698 : cluster [DBG] pgmap v5705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:32 smithi006 bash[33918]: cluster 2023-08-28T01:08:31.635518+0000 mgr.y (mgr.14901) 5698 : cluster [DBG] pgmap v5705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:32.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:08:32.505Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:32.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:08:32.506Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:33.564 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:33 smithi130 bash[51302]: debug 2023-08-28T01:08:33.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:33.247238+0000) 2023-08-28T01:08:33.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:33 smithi130 bash[27579]: audit 2023-08-28T01:08:32.642835+0000 mon.c (mon.2) 2218 : audit [DBG] from='client.? 172.21.15.6:0/179928459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:33.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:33 smithi006 bash[33918]: audit 2023-08-28T01:08:32.642835+0000 mon.c (mon.2) 2218 : audit [DBG] from='client.? 172.21.15.6:0/179928459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:33.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:33 smithi006 bash[27292]: audit 2023-08-28T01:08:32.642835+0000 mon.c (mon.2) 2218 : audit [DBG] from='client.? 172.21.15.6:0/179928459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:34.574 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:34 smithi130 bash[51302]: debug 2023-08-28T01:08:34.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:34.247455+0000) 2023-08-28T01:08:34.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:34 smithi006 bash[27292]: cluster 2023-08-28T01:08:33.636148+0000 mgr.y (mgr.14901) 5699 : cluster [DBG] pgmap v5706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:34.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:34 smithi006 bash[33918]: cluster 2023-08-28T01:08:33.636148+0000 mgr.y (mgr.14901) 5699 : cluster [DBG] pgmap v5706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:34.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:08:34.385Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:34.717 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:08:34.388Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:34.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:34 smithi130 bash[27579]: cluster 2023-08-28T01:08:33.636148+0000 mgr.y (mgr.14901) 5699 : cluster [DBG] pgmap v5706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:35.577 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:35 smithi130 bash[51302]: debug 2023-08-28T01:08:35.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:35.247749+0000) 2023-08-28T01:08:35.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:35 smithi130 bash[27579]: audit 2023-08-28T01:08:35.058594+0000 mon.c (mon.2) 2219 : audit [DBG] from='client.? 172.21.15.6:0/2371301839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:35.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:35 smithi006 bash[27292]: audit 2023-08-28T01:08:35.058594+0000 mon.c (mon.2) 2219 : audit [DBG] from='client.? 172.21.15.6:0/2371301839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:35.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:35 smithi006 bash[33918]: audit 2023-08-28T01:08:35.058594+0000 mon.c (mon.2) 2219 : audit [DBG] from='client.? 172.21.15.6:0/2371301839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:36.584 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:36 smithi130 bash[51302]: debug 2023-08-28T01:08:36.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:36.248035+0000) 2023-08-28T01:08:36.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:36 smithi130 bash[27579]: cluster 2023-08-28T01:08:35.637327+0000 mgr.y (mgr.14901) 5700 : cluster [DBG] pgmap v5707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:36.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:36 smithi006 bash[33918]: cluster 2023-08-28T01:08:35.637327+0000 mgr.y (mgr.14901) 5700 : cluster [DBG] pgmap v5707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:36.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:36 smithi006 bash[27292]: cluster 2023-08-28T01:08:35.637327+0000 mgr.y (mgr.14901) 5700 : cluster [DBG] pgmap v5707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:37.587 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:37 smithi130 bash[51302]: debug 2023-08-28T01:08:37.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:37.248300+0000) 2023-08-28T01:08:37.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:37 smithi130 bash[27579]: audit 2023-08-28T01:08:37.469071+0000 mon.c (mon.2) 2220 : audit [DBG] from='client.? 172.21.15.6:0/4016238272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:37.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:37 smithi006 bash[33918]: audit 2023-08-28T01:08:37.469071+0000 mon.c (mon.2) 2220 : audit [DBG] from='client.? 172.21.15.6:0/4016238272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:37.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:37 smithi006 bash[27292]: audit 2023-08-28T01:08:37.469071+0000 mon.c (mon.2) 2220 : audit [DBG] from='client.? 172.21.15.6:0/4016238272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:38.596 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:38 smithi130 bash[51302]: debug 2023-08-28T01:08:38.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:38.248572+0000) 2023-08-28T01:08:38.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:38 smithi130 bash[27579]: cluster 2023-08-28T01:08:37.637931+0000 mgr.y (mgr.14901) 5701 : cluster [DBG] pgmap v5708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:38.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:38 smithi006 bash[27292]: cluster 2023-08-28T01:08:37.637931+0000 mgr.y (mgr.14901) 5701 : cluster [DBG] pgmap v5708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:38.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:38 smithi006 bash[33918]: cluster 2023-08-28T01:08:37.637931+0000 mgr.y (mgr.14901) 5701 : cluster [DBG] pgmap v5708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:39.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:39 smithi130 bash[51302]: debug 2023-08-28T01:08:39.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:39.248868+0000) 2023-08-28T01:08:40.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:40 smithi130 bash[51302]: debug 2023-08-28T01:08:40.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:40.249197+0000) 2023-08-28T01:08:40.966 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:08:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:08:40] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:08:40.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:40 smithi006 bash[27292]: cluster 2023-08-28T01:08:39.638935+0000 mgr.y (mgr.14901) 5702 : cluster [DBG] pgmap v5709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:40.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:40 smithi006 bash[27292]: audit 2023-08-28T01:08:39.884427+0000 mon.a (mon.0) 3896 : audit [DBG] from='client.? 172.21.15.6:0/340490525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:40.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:08:40.932Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:40.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:08:40.934Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:40.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:40 smithi006 bash[33918]: cluster 2023-08-28T01:08:39.638935+0000 mgr.y (mgr.14901) 5702 : cluster [DBG] pgmap v5709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:40.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:40 smithi006 bash[33918]: audit 2023-08-28T01:08:39.884427+0000 mon.a (mon.0) 3896 : audit [DBG] from='client.? 172.21.15.6:0/340490525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:41.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:40 smithi130 bash[27579]: cluster 2023-08-28T01:08:39.638935+0000 mgr.y (mgr.14901) 5702 : cluster [DBG] pgmap v5709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:40 smithi130 bash[27579]: audit 2023-08-28T01:08:39.884427+0000 mon.a (mon.0) 3896 : audit [DBG] from='client.? 172.21.15.6:0/340490525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:41.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:41 smithi130 bash[51302]: debug 2023-08-28T01:08:41.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:41.249498+0000) 2023-08-28T01:08:42.566 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:42 smithi130 bash[51302]: debug 2023-08-28T01:08:42.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:42.249789+0000) 2023-08-28T01:08:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:42 smithi130 bash[27579]: cluster 2023-08-28T01:08:41.639616+0000 mgr.y (mgr.14901) 5703 : cluster [DBG] pgmap v5710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:42.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:42 smithi130 bash[27579]: audit 2023-08-28T01:08:42.300037+0000 mon.a (mon.0) 3897 : audit [DBG] from='client.? 172.21.15.6:0/4194392664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:08:42.505Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:08:42.507Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:42 smithi006 bash[27292]: cluster 2023-08-28T01:08:41.639616+0000 mgr.y (mgr.14901) 5703 : cluster [DBG] pgmap v5710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:42 smithi006 bash[27292]: audit 2023-08-28T01:08:42.300037+0000 mon.a (mon.0) 3897 : audit [DBG] from='client.? 172.21.15.6:0/4194392664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:42 smithi006 bash[33918]: cluster 2023-08-28T01:08:41.639616+0000 mgr.y (mgr.14901) 5703 : cluster [DBG] pgmap v5710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:42 smithi006 bash[33918]: audit 2023-08-28T01:08:42.300037+0000 mon.a (mon.0) 3897 : audit [DBG] from='client.? 172.21.15.6:0/4194392664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:43.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:43 smithi130 bash[51302]: debug 2023-08-28T01:08:43.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:43.250090+0000) 2023-08-28T01:08:44.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:44 smithi130 bash[51302]: debug 2023-08-28T01:08:44.249+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:44.250405+0000) 2023-08-28T01:08:44.698 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:08:44.385Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:44.699 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:08:44.387Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:44.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:44 smithi006 bash[33918]: cluster 2023-08-28T01:08:43.640295+0000 mgr.y (mgr.14901) 5704 : cluster [DBG] pgmap v5711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:44.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:44 smithi006 bash[27292]: cluster 2023-08-28T01:08:43.640295+0000 mgr.y (mgr.14901) 5704 : cluster [DBG] pgmap v5711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:45.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:44 smithi130 bash[27579]: cluster 2023-08-28T01:08:43.640295+0000 mgr.y (mgr.14901) 5704 : cluster [DBG] pgmap v5711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:45.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:45 smithi130 bash[51302]: debug 2023-08-28T01:08:45.249+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:45.250690+0000) 2023-08-28T01:08:45.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:45 smithi006 bash[27292]: audit 2023-08-28T01:08:44.712461+0000 mon.c (mon.2) 2221 : audit [DBG] from='client.? 172.21.15.6:0/1566766865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:45.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:45 smithi006 bash[33918]: audit 2023-08-28T01:08:44.712461+0000 mon.c (mon.2) 2221 : audit [DBG] from='client.? 172.21.15.6:0/1566766865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:46.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:45 smithi130 bash[27579]: audit 2023-08-28T01:08:44.712461+0000 mon.c (mon.2) 2221 : audit [DBG] from='client.? 172.21.15.6:0/1566766865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:46.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:46 smithi130 bash[51302]: debug 2023-08-28T01:08:46.249+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:46.250994+0000) 2023-08-28T01:08:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:46 smithi006 bash[33918]: cluster 2023-08-28T01:08:45.641454+0000 mgr.y (mgr.14901) 5705 : cluster [DBG] pgmap v5712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:46.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:46 smithi006 bash[27292]: cluster 2023-08-28T01:08:45.641454+0000 mgr.y (mgr.14901) 5705 : cluster [DBG] pgmap v5712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:46 smithi130 bash[27579]: cluster 2023-08-28T01:08:45.641454+0000 mgr.y (mgr.14901) 5705 : cluster [DBG] pgmap v5712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:47.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:47 smithi130 bash[51302]: debug 2023-08-28T01:08:47.249+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:47.251265+0000) 2023-08-28T01:08:48.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:47 smithi130 bash[27579]: audit 2023-08-28T01:08:47.127178+0000 mon.c (mon.2) 2222 : audit [DBG] from='client.? 172.21.15.6:0/1101985724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:48.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:47 smithi006 bash[33918]: audit 2023-08-28T01:08:47.127178+0000 mon.c (mon.2) 2222 : audit [DBG] from='client.? 172.21.15.6:0/1101985724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:48.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:47 smithi006 bash[27292]: audit 2023-08-28T01:08:47.127178+0000 mon.c (mon.2) 2222 : audit [DBG] from='client.? 172.21.15.6:0/1101985724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:48.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:48 smithi130 bash[51302]: debug 2023-08-28T01:08:48.249+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:48.251589+0000) 2023-08-28T01:08:49.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:48 smithi130 bash[27579]: cluster 2023-08-28T01:08:47.642286+0000 mgr.y (mgr.14901) 5706 : cluster [DBG] pgmap v5713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:49.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:48 smithi006 bash[33918]: cluster 2023-08-28T01:08:47.642286+0000 mgr.y (mgr.14901) 5706 : cluster [DBG] pgmap v5713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:49.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:48 smithi006 bash[27292]: cluster 2023-08-28T01:08:47.642286+0000 mgr.y (mgr.14901) 5706 : cluster [DBG] pgmap v5713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:49.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:49 smithi130 bash[51302]: debug 2023-08-28T01:08:49.249+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:49.251849+0000) 2023-08-28T01:08:50.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:49 smithi130 bash[27579]: audit 2023-08-28T01:08:49.541982+0000 mon.c (mon.2) 2223 : audit [DBG] from='client.? 172.21.15.6:0/566600931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:50.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:49 smithi006 bash[33918]: audit 2023-08-28T01:08:49.541982+0000 mon.c (mon.2) 2223 : audit [DBG] from='client.? 172.21.15.6:0/566600931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:50.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:49 smithi006 bash[27292]: audit 2023-08-28T01:08:49.541982+0000 mon.c (mon.2) 2223 : audit [DBG] from='client.? 172.21.15.6:0/566600931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:50.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:50 smithi130 bash[51302]: debug 2023-08-28T01:08:50.249+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:50.252142+0000) 2023-08-28T01:08:51.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:50 smithi130 bash[27579]: cluster 2023-08-28T01:08:49.643448+0000 mgr.y (mgr.14901) 5707 : cluster [DBG] pgmap v5714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:51.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:50 smithi006 bash[27292]: cluster 2023-08-28T01:08:49.643448+0000 mgr.y (mgr.14901) 5707 : cluster [DBG] pgmap v5714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:08:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:08:50] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:08:51.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:50 smithi006 bash[33918]: cluster 2023-08-28T01:08:49.643448+0000 mgr.y (mgr.14901) 5707 : cluster [DBG] pgmap v5714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:08:50.933Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:08:50.935Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:51.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:51 smithi130 bash[51302]: debug 2023-08-28T01:08:51.249+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:51.252416+0000) 2023-08-28T01:08:52.563 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:52 smithi130 bash[51302]: debug 2023-08-28T01:08:52.249+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:52.252683+0000) 2023-08-28T01:08:52.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:52 smithi130 bash[27579]: cluster 2023-08-28T01:08:51.644223+0000 mgr.y (mgr.14901) 5708 : cluster [DBG] pgmap v5715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:52 smithi130 bash[27579]: audit 2023-08-28T01:08:51.953477+0000 mon.a (mon.0) 3898 : audit [DBG] from='client.? 172.21.15.6:0/393454372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:08:52.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:08:52.507Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:52 smithi006 bash[27292]: cluster 2023-08-28T01:08:51.644223+0000 mgr.y (mgr.14901) 5708 : cluster [DBG] pgmap v5715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:52 smithi006 bash[27292]: audit 2023-08-28T01:08:51.953477+0000 mon.a (mon.0) 3898 : audit [DBG] from='client.? 172.21.15.6:0/393454372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:52 smithi006 bash[33918]: cluster 2023-08-28T01:08:51.644223+0000 mgr.y (mgr.14901) 5708 : cluster [DBG] pgmap v5715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:52 smithi006 bash[33918]: audit 2023-08-28T01:08:51.953477+0000 mon.a (mon.0) 3898 : audit [DBG] from='client.? 172.21.15.6:0/393454372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:53.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:53 smithi130 bash[51302]: debug 2023-08-28T01:08:53.249+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:53.252960+0000) 2023-08-28T01:08:54.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:54 smithi130 bash[51302]: debug 2023-08-28T01:08:54.249+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:54.253243+0000) 2023-08-28T01:08:54.704 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:08:54.387Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:54.704 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:08:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:08:54.388Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:08:54.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:54 smithi006 bash[27292]: cluster 2023-08-28T01:08:53.644983+0000 mgr.y (mgr.14901) 5709 : cluster [DBG] pgmap v5716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:54.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:54 smithi006 bash[27292]: audit 2023-08-28T01:08:54.362629+0000 mon.a (mon.0) 3899 : audit [DBG] from='client.? 172.21.15.6:0/1726489133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:54.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:54 smithi006 bash[33918]: cluster 2023-08-28T01:08:53.644983+0000 mgr.y (mgr.14901) 5709 : cluster [DBG] pgmap v5716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:54.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:54 smithi006 bash[33918]: audit 2023-08-28T01:08:54.362629+0000 mon.a (mon.0) 3899 : audit [DBG] from='client.? 172.21.15.6:0/1726489133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:54 smithi130 bash[27579]: cluster 2023-08-28T01:08:53.644983+0000 mgr.y (mgr.14901) 5709 : cluster [DBG] pgmap v5716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:55.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:54 smithi130 bash[27579]: audit 2023-08-28T01:08:54.362629+0000 mon.a (mon.0) 3899 : audit [DBG] from='client.? 172.21.15.6:0/1726489133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:55.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:55 smithi130 bash[51302]: debug 2023-08-28T01:08:55.249+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:55.253514+0000) 2023-08-28T01:08:56.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:56 smithi130 bash[51302]: debug 2023-08-28T01:08:56.253+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:56.253777+0000) 2023-08-28T01:08:56.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:56 smithi006 bash[27292]: cluster 2023-08-28T01:08:55.646072+0000 mgr.y (mgr.14901) 5710 : cluster [DBG] pgmap v5717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:56.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:56 smithi006 bash[33918]: cluster 2023-08-28T01:08:55.646072+0000 mgr.y (mgr.14901) 5710 : cluster [DBG] pgmap v5717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:57.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:56 smithi130 bash[27579]: cluster 2023-08-28T01:08:55.646072+0000 mgr.y (mgr.14901) 5710 : cluster [DBG] pgmap v5717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:57.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:57 smithi130 bash[51302]: debug 2023-08-28T01:08:57.253+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:57.254113+0000) 2023-08-28T01:08:58.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:57 smithi130 bash[27579]: audit 2023-08-28T01:08:56.777830+0000 mon.c (mon.2) 2224 : audit [DBG] from='client.? 172.21.15.6:0/2860372107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:58.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:57 smithi006 bash[33918]: audit 2023-08-28T01:08:56.777830+0000 mon.c (mon.2) 2224 : audit [DBG] from='client.? 172.21.15.6:0/2860372107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:58.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:57 smithi006 bash[27292]: audit 2023-08-28T01:08:56.777830+0000 mon.c (mon.2) 2224 : audit [DBG] from='client.? 172.21.15.6:0/2860372107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:58.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:58 smithi130 bash[51302]: debug 2023-08-28T01:08:58.253+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:58.254430+0000) 2023-08-28T01:08:59.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:58 smithi130 bash[27579]: cluster 2023-08-28T01:08:57.646762+0000 mgr.y (mgr.14901) 5711 : cluster [DBG] pgmap v5718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:59.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:58 smithi006 bash[33918]: cluster 2023-08-28T01:08:57.646762+0000 mgr.y (mgr.14901) 5711 : cluster [DBG] pgmap v5718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:59.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:58 smithi006 bash[27292]: cluster 2023-08-28T01:08:57.646762+0000 mgr.y (mgr.14901) 5711 : cluster [DBG] pgmap v5718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:08:59.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:08:59 smithi130 bash[51302]: debug 2023-08-28T01:08:59.253+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:59.254668+0000) 2023-08-28T01:09:00.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:08:59 smithi130 bash[27579]: audit 2023-08-28T01:08:59.186714+0000 mon.a (mon.0) 3900 : audit [DBG] from='client.? 172.21.15.6:0/2454754859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:00.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:08:59 smithi006 bash[33918]: audit 2023-08-28T01:08:59.186714+0000 mon.a (mon.0) 3900 : audit [DBG] from='client.? 172.21.15.6:0/2454754859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:00.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:08:59 smithi006 bash[27292]: audit 2023-08-28T01:08:59.186714+0000 mon.a (mon.0) 3900 : audit [DBG] from='client.? 172.21.15.6:0/2454754859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:00.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:00 smithi130 bash[51302]: debug 2023-08-28T01:09:00.253+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:00.254972+0000) 2023-08-28T01:09:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:00 smithi130 bash[27579]: cluster 2023-08-28T01:08:59.647882+0000 mgr.y (mgr.14901) 5712 : cluster [DBG] pgmap v5719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:09:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:09:00] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:09:01.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:00 smithi006 bash[33918]: cluster 2023-08-28T01:08:59.647882+0000 mgr.y (mgr.14901) 5712 : cluster [DBG] pgmap v5719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:01.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:00 smithi006 bash[27292]: cluster 2023-08-28T01:08:59.647882+0000 mgr.y (mgr.14901) 5712 : cluster [DBG] pgmap v5719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:09:00.934Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:09:00.936Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:01.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:01 smithi130 bash[51302]: debug 2023-08-28T01:09:01.253+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:01.255244+0000) 2023-08-28T01:09:02.569 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:02 smithi130 bash[51302]: debug 2023-08-28T01:09:02.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:02.255466+0000) 2023-08-28T01:09:02.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:02 smithi130 bash[27579]: audit 2023-08-28T01:09:01.611121+0000 mon.c (mon.2) 2225 : audit [DBG] from='client.? 172.21.15.6:0/4014552809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:02 smithi130 bash[27579]: cluster 2023-08-28T01:09:01.648656+0000 mgr.y (mgr.14901) 5713 : cluster [DBG] pgmap v5720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:02 smithi006 bash[27292]: audit 2023-08-28T01:09:01.611121+0000 mon.c (mon.2) 2225 : audit [DBG] from='client.? 172.21.15.6:0/4014552809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:02 smithi006 bash[27292]: cluster 2023-08-28T01:09:01.648656+0000 mgr.y (mgr.14901) 5713 : cluster [DBG] pgmap v5720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:02.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:09:02.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:02.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:09:02.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:02 smithi006 bash[33918]: audit 2023-08-28T01:09:01.611121+0000 mon.c (mon.2) 2225 : audit [DBG] from='client.? 172.21.15.6:0/4014552809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:02 smithi006 bash[33918]: cluster 2023-08-28T01:09:01.648656+0000 mgr.y (mgr.14901) 5713 : cluster [DBG] pgmap v5720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:03.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:03 smithi130 bash[51302]: debug 2023-08-28T01:09:03.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:03.255696+0000) 2023-08-28T01:09:04.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:04 smithi130 bash[51302]: debug 2023-08-28T01:09:04.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:04.255973+0000) 2023-08-28T01:09:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:09:04.388Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:09:04.390Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:05.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:04 smithi130 bash[27579]: cluster 2023-08-28T01:09:03.649471+0000 mgr.y (mgr.14901) 5714 : cluster [DBG] pgmap v5721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:04 smithi130 bash[27579]: audit 2023-08-28T01:09:04.018073+0000 mon.a (mon.0) 3901 : audit [DBG] from='client.? 172.21.15.6:0/2365480923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:05.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:04 smithi006 bash[27292]: cluster 2023-08-28T01:09:03.649471+0000 mgr.y (mgr.14901) 5714 : cluster [DBG] pgmap v5721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:05.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:04 smithi006 bash[27292]: audit 2023-08-28T01:09:04.018073+0000 mon.a (mon.0) 3901 : audit [DBG] from='client.? 172.21.15.6:0/2365480923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:05.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:04 smithi006 bash[33918]: cluster 2023-08-28T01:09:03.649471+0000 mgr.y (mgr.14901) 5714 : cluster [DBG] pgmap v5721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:05.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:04 smithi006 bash[33918]: audit 2023-08-28T01:09:04.018073+0000 mon.a (mon.0) 3901 : audit [DBG] from='client.? 172.21.15.6:0/2365480923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:05.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:05 smithi130 bash[51302]: debug 2023-08-28T01:09:05.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:05.256255+0000) 2023-08-28T01:09:06.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:06 smithi130 bash[51302]: debug 2023-08-28T01:09:06.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:06.256527+0000) 2023-08-28T01:09:07.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:06 smithi130 bash[27579]: cluster 2023-08-28T01:09:05.650759+0000 mgr.y (mgr.14901) 5715 : cluster [DBG] pgmap v5722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:07.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:06 smithi130 bash[27579]: audit 2023-08-28T01:09:06.432337+0000 mon.c (mon.2) 2226 : audit [DBG] from='client.? 172.21.15.6:0/4024115064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:07.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:06 smithi006 bash[33918]: cluster 2023-08-28T01:09:05.650759+0000 mgr.y (mgr.14901) 5715 : cluster [DBG] pgmap v5722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:07.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:06 smithi006 bash[33918]: audit 2023-08-28T01:09:06.432337+0000 mon.c (mon.2) 2226 : audit [DBG] from='client.? 172.21.15.6:0/4024115064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:06 smithi006 bash[27292]: cluster 2023-08-28T01:09:05.650759+0000 mgr.y (mgr.14901) 5715 : cluster [DBG] pgmap v5722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:06 smithi006 bash[27292]: audit 2023-08-28T01:09:06.432337+0000 mon.c (mon.2) 2226 : audit [DBG] from='client.? 172.21.15.6:0/4024115064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:07.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:07 smithi130 bash[51302]: debug 2023-08-28T01:09:07.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:07.256772+0000) 2023-08-28T01:09:08.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:08 smithi130 bash[51302]: debug 2023-08-28T01:09:08.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:08.257025+0000) 2023-08-28T01:09:09.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:08 smithi130 bash[27579]: cluster 2023-08-28T01:09:07.651493+0000 mgr.y (mgr.14901) 5716 : cluster [DBG] pgmap v5723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:09.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:08 smithi006 bash[27292]: cluster 2023-08-28T01:09:07.651493+0000 mgr.y (mgr.14901) 5716 : cluster [DBG] pgmap v5723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:09.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:08 smithi006 bash[33918]: cluster 2023-08-28T01:09:07.651493+0000 mgr.y (mgr.14901) 5716 : cluster [DBG] pgmap v5723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:09.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:09 smithi130 bash[51302]: debug 2023-08-28T01:09:09.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:09.257347+0000) 2023-08-28T01:09:10.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:09 smithi130 bash[27579]: audit 2023-08-28T01:09:08.838564+0000 mon.c (mon.2) 2227 : audit [DBG] from='client.? 172.21.15.6:0/3956978673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:10.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:09 smithi006 bash[33918]: audit 2023-08-28T01:09:08.838564+0000 mon.c (mon.2) 2227 : audit [DBG] from='client.? 172.21.15.6:0/3956978673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:10.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:09 smithi006 bash[27292]: audit 2023-08-28T01:09:08.838564+0000 mon.c (mon.2) 2227 : audit [DBG] from='client.? 172.21.15.6:0/3956978673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:10.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:10 smithi130 bash[51302]: debug 2023-08-28T01:09:10.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:10.257632+0000) 2023-08-28T01:09:11.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:10 smithi130 bash[27579]: cluster 2023-08-28T01:09:09.652584+0000 mgr.y (mgr.14901) 5717 : cluster [DBG] pgmap v5724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:09:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:09:10] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:09:11.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:10 smithi006 bash[27292]: cluster 2023-08-28T01:09:09.652584+0000 mgr.y (mgr.14901) 5717 : cluster [DBG] pgmap v5724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:11.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:10 smithi006 bash[33918]: cluster 2023-08-28T01:09:09.652584+0000 mgr.y (mgr.14901) 5717 : cluster [DBG] pgmap v5724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:11.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:09:10.935Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:09:10.937Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:11.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:11 smithi130 bash[51302]: debug 2023-08-28T01:09:11.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:11.257891+0000) 2023-08-28T01:09:12.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:11 smithi130 bash[27579]: audit 2023-08-28T01:09:11.254620+0000 mon.c (mon.2) 2228 : audit [DBG] from='client.? 172.21.15.6:0/2386608297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:12.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:11 smithi006 bash[27292]: audit 2023-08-28T01:09:11.254620+0000 mon.c (mon.2) 2228 : audit [DBG] from='client.? 172.21.15.6:0/2386608297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:12.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:11 smithi006 bash[33918]: audit 2023-08-28T01:09:11.254620+0000 mon.c (mon.2) 2228 : audit [DBG] from='client.? 172.21.15.6:0/2386608297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:12.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:12 smithi130 bash[51302]: debug 2023-08-28T01:09:12.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:12.258165+0000) 2023-08-28T01:09:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:12 smithi006 bash[27292]: cluster 2023-08-28T01:09:11.653251+0000 mgr.y (mgr.14901) 5718 : cluster [DBG] pgmap v5725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:12 smithi006 bash[33918]: cluster 2023-08-28T01:09:11.653251+0000 mgr.y (mgr.14901) 5718 : cluster [DBG] pgmap v5725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:09:12.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:12.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:09:12.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:12 smithi130 bash[27579]: cluster 2023-08-28T01:09:11.653251+0000 mgr.y (mgr.14901) 5718 : cluster [DBG] pgmap v5725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:13.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:13 smithi130 bash[51302]: debug 2023-08-28T01:09:13.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:13.258464+0000) 2023-08-28T01:09:14.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:13 smithi130 bash[27579]: audit 2023-08-28T01:09:13.296725+0000 mon.b (mon.1) 668 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:09:14.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:13 smithi130 bash[27579]: audit 2023-08-28T01:09:13.677761+0000 mon.c (mon.2) 2229 : audit [DBG] from='client.? 172.21.15.6:0/1813810515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:14.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:13 smithi006 bash[27292]: audit 2023-08-28T01:09:13.296725+0000 mon.b (mon.1) 668 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:09:14.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:13 smithi006 bash[27292]: audit 2023-08-28T01:09:13.677761+0000 mon.c (mon.2) 2229 : audit [DBG] from='client.? 172.21.15.6:0/1813810515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:14.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:13 smithi006 bash[33918]: audit 2023-08-28T01:09:13.296725+0000 mon.b (mon.1) 668 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:09:14.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:13 smithi006 bash[33918]: audit 2023-08-28T01:09:13.677761+0000 mon.c (mon.2) 2229 : audit [DBG] from='client.? 172.21.15.6:0/1813810515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:14.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:14 smithi130 bash[51302]: debug 2023-08-28T01:09:14.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:14.258772+0000) 2023-08-28T01:09:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:09:14.388Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:09:14.390Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:15.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:14 smithi130 bash[27579]: cluster 2023-08-28T01:09:13.653867+0000 mgr.y (mgr.14901) 5719 : cluster [DBG] pgmap v5726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:15.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:14 smithi006 bash[33918]: cluster 2023-08-28T01:09:13.653867+0000 mgr.y (mgr.14901) 5719 : cluster [DBG] pgmap v5726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:15.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:14 smithi006 bash[27292]: cluster 2023-08-28T01:09:13.653867+0000 mgr.y (mgr.14901) 5719 : cluster [DBG] pgmap v5726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:15.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:15 smithi130 bash[51302]: debug 2023-08-28T01:09:15.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:15.259068+0000) 2023-08-28T01:09:16.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:16 smithi130 bash[51302]: debug 2023-08-28T01:09:16.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:16.259372+0000) 2023-08-28T01:09:17.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:16 smithi130 bash[27579]: cluster 2023-08-28T01:09:15.655014+0000 mgr.y (mgr.14901) 5720 : cluster [DBG] pgmap v5727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:16 smithi130 bash[27579]: audit 2023-08-28T01:09:16.097482+0000 mon.c (mon.2) 2230 : audit [DBG] from='client.? 172.21.15.6:0/2346920869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:17.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:16 smithi006 bash[33918]: cluster 2023-08-28T01:09:15.655014+0000 mgr.y (mgr.14901) 5720 : cluster [DBG] pgmap v5727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:17.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:16 smithi006 bash[33918]: audit 2023-08-28T01:09:16.097482+0000 mon.c (mon.2) 2230 : audit [DBG] from='client.? 172.21.15.6:0/2346920869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:17.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:16 smithi006 bash[27292]: cluster 2023-08-28T01:09:15.655014+0000 mgr.y (mgr.14901) 5720 : cluster [DBG] pgmap v5727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:17.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:16 smithi006 bash[27292]: audit 2023-08-28T01:09:16.097482+0000 mon.c (mon.2) 2230 : audit [DBG] from='client.? 172.21.15.6:0/2346920869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:17.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:17 smithi130 bash[51302]: debug 2023-08-28T01:09:17.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:17.259642+0000) 2023-08-28T01:09:18.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:18 smithi130 bash[51302]: debug 2023-08-28T01:09:18.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:18.260073+0000) 2023-08-28T01:09:19.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:18 smithi130 bash[27579]: cluster 2023-08-28T01:09:17.655640+0000 mgr.y (mgr.14901) 5721 : cluster [DBG] pgmap v5728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:18 smithi130 bash[27579]: audit 2023-08-28T01:09:18.522580+0000 mon.c (mon.2) 2231 : audit [DBG] from='client.? 172.21.15.6:0/1443306597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:19.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:18 smithi006 bash[27292]: cluster 2023-08-28T01:09:17.655640+0000 mgr.y (mgr.14901) 5721 : cluster [DBG] pgmap v5728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:19.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:18 smithi006 bash[27292]: audit 2023-08-28T01:09:18.522580+0000 mon.c (mon.2) 2231 : audit [DBG] from='client.? 172.21.15.6:0/1443306597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:19.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:18 smithi006 bash[33918]: cluster 2023-08-28T01:09:17.655640+0000 mgr.y (mgr.14901) 5721 : cluster [DBG] pgmap v5728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:19.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:18 smithi006 bash[33918]: audit 2023-08-28T01:09:18.522580+0000 mon.c (mon.2) 2231 : audit [DBG] from='client.? 172.21.15.6:0/1443306597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:19.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:19 smithi130 bash[51302]: debug 2023-08-28T01:09:19.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:19.260293+0000) 2023-08-28T01:09:20.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:20 smithi006 bash[33918]: audit 2023-08-28T01:09:19.170503+0000 mon.a (mon.0) 3902 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:09:20.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:20 smithi006 bash[33918]: audit 2023-08-28T01:09:19.613538+0000 mon.a (mon.0) 3903 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:09:20.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:20 smithi006 bash[33918]: cluster 2023-08-28T01:09:19.656763+0000 mgr.y (mgr.14901) 5722 : cluster [DBG] pgmap v5729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:20.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:20 smithi006 bash[33918]: audit 2023-08-28T01:09:19.880475+0000 mon.b (mon.1) 669 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:09:20.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:20 smithi006 bash[33918]: audit 2023-08-28T01:09:19.882897+0000 mon.b (mon.1) 670 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:09:20.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:20 smithi006 bash[33918]: audit 2023-08-28T01:09:19.893246+0000 mon.a (mon.0) 3904 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:09:20.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:20 smithi006 bash[27292]: audit 2023-08-28T01:09:19.170503+0000 mon.a (mon.0) 3902 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:09:20.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:20 smithi006 bash[27292]: audit 2023-08-28T01:09:19.613538+0000 mon.a (mon.0) 3903 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:09:20.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:20 smithi006 bash[27292]: cluster 2023-08-28T01:09:19.656763+0000 mgr.y (mgr.14901) 5722 : cluster [DBG] pgmap v5729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:20.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:20 smithi006 bash[27292]: audit 2023-08-28T01:09:19.880475+0000 mon.b (mon.1) 669 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:09:20.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:20 smithi006 bash[27292]: audit 2023-08-28T01:09:19.882897+0000 mon.b (mon.1) 670 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:09:20.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:20 smithi006 bash[27292]: audit 2023-08-28T01:09:19.893246+0000 mon.a (mon.0) 3904 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:09:20.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:20 smithi130 bash[51302]: debug 2023-08-28T01:09:20.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:20.260551+0000) 2023-08-28T01:09:20.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:20 smithi130 bash[27579]: audit 2023-08-28T01:09:19.170503+0000 mon.a (mon.0) 3902 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:09:20.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:20 smithi130 bash[27579]: audit 2023-08-28T01:09:19.613538+0000 mon.a (mon.0) 3903 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:09:20.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:20 smithi130 bash[27579]: cluster 2023-08-28T01:09:19.656763+0000 mgr.y (mgr.14901) 5722 : cluster [DBG] pgmap v5729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:20.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:20 smithi130 bash[27579]: audit 2023-08-28T01:09:19.880475+0000 mon.b (mon.1) 669 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:09:20.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:20 smithi130 bash[27579]: audit 2023-08-28T01:09:19.882897+0000 mon.b (mon.1) 670 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:09:20.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:20 smithi130 bash[27579]: audit 2023-08-28T01:09:19.893246+0000 mon.a (mon.0) 3904 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:09:21.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:09:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:09:20] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:09:21.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:09:20.935Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:21.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:09:20.937Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:21.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:21 smithi130 bash[51302]: debug 2023-08-28T01:09:21.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:21.260834+0000) 2023-08-28T01:09:21.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:21 smithi130 bash[27579]: audit 2023-08-28T01:09:20.946763+0000 mon.c (mon.2) 2232 : audit [DBG] from='client.? 172.21.15.6:0/2444305626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:21.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:21 smithi006 bash[33918]: audit 2023-08-28T01:09:20.946763+0000 mon.c (mon.2) 2232 : audit [DBG] from='client.? 172.21.15.6:0/2444305626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:21.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:21 smithi006 bash[27292]: audit 2023-08-28T01:09:20.946763+0000 mon.c (mon.2) 2232 : audit [DBG] from='client.? 172.21.15.6:0/2444305626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:22.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:22 smithi130 bash[51302]: debug 2023-08-28T01:09:22.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:22.261058+0000) 2023-08-28T01:09:22.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:22 smithi006 bash[27292]: cluster 2023-08-28T01:09:21.657368+0000 mgr.y (mgr.14901) 5723 : cluster [DBG] pgmap v5730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:22.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:22 smithi006 bash[33918]: cluster 2023-08-28T01:09:21.657368+0000 mgr.y (mgr.14901) 5723 : cluster [DBG] pgmap v5730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:22.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:09:22.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:22.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:09:22.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:23.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:22 smithi130 bash[27579]: cluster 2023-08-28T01:09:21.657368+0000 mgr.y (mgr.14901) 5723 : cluster [DBG] pgmap v5730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:23.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:23 smithi130 bash[51302]: debug 2023-08-28T01:09:23.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:23.261279+0000) 2023-08-28T01:09:23.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:23 smithi006 bash[27292]: audit 2023-08-28T01:09:23.366352+0000 mon.a (mon.0) 3905 : audit [DBG] from='client.? 172.21.15.6:0/769493405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:23.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:23 smithi006 bash[33918]: audit 2023-08-28T01:09:23.366352+0000 mon.a (mon.0) 3905 : audit [DBG] from='client.? 172.21.15.6:0/769493405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:24.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:23 smithi130 bash[27579]: audit 2023-08-28T01:09:23.366352+0000 mon.a (mon.0) 3905 : audit [DBG] from='client.? 172.21.15.6:0/769493405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:24.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:24 smithi130 bash[51302]: debug 2023-08-28T01:09:24.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:24.261559+0000) 2023-08-28T01:09:24.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:24 smithi006 bash[33918]: cluster 2023-08-28T01:09:23.657926+0000 mgr.y (mgr.14901) 5724 : cluster [DBG] pgmap v5731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:24.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:24 smithi006 bash[27292]: cluster 2023-08-28T01:09:23.657926+0000 mgr.y (mgr.14901) 5724 : cluster [DBG] pgmap v5731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:24.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:09:24.389Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:24.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:09:24.391Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:25.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:24 smithi130 bash[27579]: cluster 2023-08-28T01:09:23.657926+0000 mgr.y (mgr.14901) 5724 : cluster [DBG] pgmap v5731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:25.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:25 smithi130 bash[51302]: debug 2023-08-28T01:09:25.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:25.261892+0000) 2023-08-28T01:09:26.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:26 smithi130 bash[51302]: debug 2023-08-28T01:09:26.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:26.262163+0000) 2023-08-28T01:09:27.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:26 smithi130 bash[27579]: cluster 2023-08-28T01:09:25.659000+0000 mgr.y (mgr.14901) 5725 : cluster [DBG] pgmap v5732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:26 smithi130 bash[27579]: audit 2023-08-28T01:09:25.782676+0000 mon.a (mon.0) 3906 : audit [DBG] from='client.? 172.21.15.6:0/851522723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:27.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:26 smithi006 bash[27292]: cluster 2023-08-28T01:09:25.659000+0000 mgr.y (mgr.14901) 5725 : cluster [DBG] pgmap v5732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:27.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:26 smithi006 bash[27292]: audit 2023-08-28T01:09:25.782676+0000 mon.a (mon.0) 3906 : audit [DBG] from='client.? 172.21.15.6:0/851522723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:27.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:26 smithi006 bash[33918]: cluster 2023-08-28T01:09:25.659000+0000 mgr.y (mgr.14901) 5725 : cluster [DBG] pgmap v5732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:27.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:26 smithi006 bash[33918]: audit 2023-08-28T01:09:25.782676+0000 mon.a (mon.0) 3906 : audit [DBG] from='client.? 172.21.15.6:0/851522723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:27.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:27 smithi130 bash[51302]: debug 2023-08-28T01:09:27.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:27.262435+0000) 2023-08-28T01:09:28.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:28 smithi130 bash[51302]: debug 2023-08-28T01:09:28.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:28.262718+0000) 2023-08-28T01:09:29.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:28 smithi130 bash[27579]: cluster 2023-08-28T01:09:27.659675+0000 mgr.y (mgr.14901) 5726 : cluster [DBG] pgmap v5733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:28 smithi130 bash[27579]: audit 2023-08-28T01:09:28.195602+0000 mon.a (mon.0) 3907 : audit [DBG] from='client.? 172.21.15.6:0/1542915466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:29.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:28 smithi006 bash[33918]: cluster 2023-08-28T01:09:27.659675+0000 mgr.y (mgr.14901) 5726 : cluster [DBG] pgmap v5733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:29.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:28 smithi006 bash[33918]: audit 2023-08-28T01:09:28.195602+0000 mon.a (mon.0) 3907 : audit [DBG] from='client.? 172.21.15.6:0/1542915466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:29.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:28 smithi006 bash[27292]: cluster 2023-08-28T01:09:27.659675+0000 mgr.y (mgr.14901) 5726 : cluster [DBG] pgmap v5733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:29.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:28 smithi006 bash[27292]: audit 2023-08-28T01:09:28.195602+0000 mon.a (mon.0) 3907 : audit [DBG] from='client.? 172.21.15.6:0/1542915466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:29.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:29 smithi130 bash[51302]: debug 2023-08-28T01:09:29.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:29.262985+0000) 2023-08-28T01:09:30.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:30 smithi130 bash[51302]: debug 2023-08-28T01:09:30.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:30.263241+0000) 2023-08-28T01:09:31.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:30 smithi130 bash[27579]: cluster 2023-08-28T01:09:29.660807+0000 mgr.y (mgr.14901) 5727 : cluster [DBG] pgmap v5734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:31.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:30 smithi130 bash[27579]: audit 2023-08-28T01:09:30.616393+0000 mon.a (mon.0) 3908 : audit [DBG] from='client.? 172.21.15.6:0/2230823190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:09:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:09:30] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:09:31.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:09:30.936Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:31.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:30 smithi006 bash[56298]: level=warn ts=2023-08-28T01:09:30.937Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:31.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:30 smithi006 bash[33918]: cluster 2023-08-28T01:09:29.660807+0000 mgr.y (mgr.14901) 5727 : cluster [DBG] pgmap v5734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:31.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:30 smithi006 bash[33918]: audit 2023-08-28T01:09:30.616393+0000 mon.a (mon.0) 3908 : audit [DBG] from='client.? 172.21.15.6:0/2230823190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:31.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:30 smithi006 bash[27292]: cluster 2023-08-28T01:09:29.660807+0000 mgr.y (mgr.14901) 5727 : cluster [DBG] pgmap v5734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:31.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:30 smithi006 bash[27292]: audit 2023-08-28T01:09:30.616393+0000 mon.a (mon.0) 3908 : audit [DBG] from='client.? 172.21.15.6:0/2230823190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:31.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:31 smithi130 bash[51302]: debug 2023-08-28T01:09:31.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:31.263521+0000) 2023-08-28T01:09:32.578 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:32 smithi130 bash[51302]: debug 2023-08-28T01:09:32.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:32.263749+0000) 2023-08-28T01:09:32.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:32 smithi130 bash[27579]: cluster 2023-08-28T01:09:31.661441+0000 mgr.y (mgr.14901) 5728 : cluster [DBG] pgmap v5735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:32 smithi006 bash[33918]: cluster 2023-08-28T01:09:31.661441+0000 mgr.y (mgr.14901) 5728 : cluster [DBG] pgmap v5735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:32 smithi006 bash[27292]: cluster 2023-08-28T01:09:31.661441+0000 mgr.y (mgr.14901) 5728 : cluster [DBG] pgmap v5735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:09:32.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:09:32.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:33.581 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:33 smithi130 bash[51302]: debug 2023-08-28T01:09:33.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:33.264016+0000) 2023-08-28T01:09:33.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:33 smithi130 bash[27579]: audit 2023-08-28T01:09:33.032939+0000 mon.a (mon.0) 3909 : audit [DBG] from='client.? 172.21.15.6:0/3485803302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:33.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:33 smithi006 bash[33918]: audit 2023-08-28T01:09:33.032939+0000 mon.a (mon.0) 3909 : audit [DBG] from='client.? 172.21.15.6:0/3485803302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:33.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:33 smithi006 bash[27292]: audit 2023-08-28T01:09:33.032939+0000 mon.a (mon.0) 3909 : audit [DBG] from='client.? 172.21.15.6:0/3485803302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:34.592 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:34 smithi130 bash[51302]: debug 2023-08-28T01:09:34.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:34.264280+0000) 2023-08-28T01:09:34.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:34 smithi006 bash[27292]: cluster 2023-08-28T01:09:33.662152+0000 mgr.y (mgr.14901) 5729 : cluster [DBG] pgmap v5736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:34.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:34 smithi006 bash[33918]: cluster 2023-08-28T01:09:33.662152+0000 mgr.y (mgr.14901) 5729 : cluster [DBG] pgmap v5736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:34.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:09:34.390Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:34.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:09:34.392Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:34.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:34 smithi130 bash[27579]: cluster 2023-08-28T01:09:33.662152+0000 mgr.y (mgr.14901) 5729 : cluster [DBG] pgmap v5736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:35.595 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:35 smithi130 bash[51302]: debug 2023-08-28T01:09:35.264+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:35.264561+0000) 2023-08-28T01:09:35.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:35 smithi130 bash[27579]: audit 2023-08-28T01:09:35.452480+0000 mon.c (mon.2) 2233 : audit [DBG] from='client.? 172.21.15.6:0/213869864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:35.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:35 smithi006 bash[27292]: audit 2023-08-28T01:09:35.452480+0000 mon.c (mon.2) 2233 : audit [DBG] from='client.? 172.21.15.6:0/213869864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:35.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:35 smithi006 bash[33918]: audit 2023-08-28T01:09:35.452480+0000 mon.c (mon.2) 2233 : audit [DBG] from='client.? 172.21.15.6:0/213869864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:36.602 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:36 smithi130 bash[51302]: debug 2023-08-28T01:09:36.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:36.264838+0000) 2023-08-28T01:09:36.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:36 smithi130 bash[27579]: cluster 2023-08-28T01:09:35.663307+0000 mgr.y (mgr.14901) 5730 : cluster [DBG] pgmap v5737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:36.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:36 smithi006 bash[27292]: cluster 2023-08-28T01:09:35.663307+0000 mgr.y (mgr.14901) 5730 : cluster [DBG] pgmap v5737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:36.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:36 smithi006 bash[33918]: cluster 2023-08-28T01:09:35.663307+0000 mgr.y (mgr.14901) 5730 : cluster [DBG] pgmap v5737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:37.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:37 smithi130 bash[51302]: debug 2023-08-28T01:09:37.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:37.265054+0000) 2023-08-28T01:09:38.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:38 smithi130 bash[51302]: debug 2023-08-28T01:09:38.264+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:38.265342+0000) 2023-08-28T01:09:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:38 smithi130 bash[27579]: cluster 2023-08-28T01:09:37.663879+0000 mgr.y (mgr.14901) 5731 : cluster [DBG] pgmap v5738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:39.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:38 smithi130 bash[27579]: audit 2023-08-28T01:09:37.880492+0000 mon.c (mon.2) 2234 : audit [DBG] from='client.? 172.21.15.6:0/269677364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:39.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:38 smithi006 bash[27292]: cluster 2023-08-28T01:09:37.663879+0000 mgr.y (mgr.14901) 5731 : cluster [DBG] pgmap v5738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:39.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:38 smithi006 bash[27292]: audit 2023-08-28T01:09:37.880492+0000 mon.c (mon.2) 2234 : audit [DBG] from='client.? 172.21.15.6:0/269677364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:39.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:38 smithi006 bash[33918]: cluster 2023-08-28T01:09:37.663879+0000 mgr.y (mgr.14901) 5731 : cluster [DBG] pgmap v5738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:39.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:38 smithi006 bash[33918]: audit 2023-08-28T01:09:37.880492+0000 mon.c (mon.2) 2234 : audit [DBG] from='client.? 172.21.15.6:0/269677364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:39.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:39 smithi130 bash[51302]: debug 2023-08-28T01:09:39.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:39.265603+0000) 2023-08-28T01:09:40.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:40 smithi130 bash[51302]: debug 2023-08-28T01:09:40.264+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:40.265917+0000) 2023-08-28T01:09:41.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:40 smithi130 bash[27579]: cluster 2023-08-28T01:09:39.665009+0000 mgr.y (mgr.14901) 5732 : cluster [DBG] pgmap v5739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:40 smithi130 bash[27579]: audit 2023-08-28T01:09:40.288914+0000 mon.a (mon.0) 3910 : audit [DBG] from='client.? 172.21.15.6:0/2483292523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:41.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:40 smithi006 bash[27292]: cluster 2023-08-28T01:09:39.665009+0000 mgr.y (mgr.14901) 5732 : cluster [DBG] pgmap v5739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:41.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:40 smithi006 bash[27292]: audit 2023-08-28T01:09:40.288914+0000 mon.a (mon.0) 3910 : audit [DBG] from='client.? 172.21.15.6:0/2483292523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:09:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:09:40] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:09:41.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:40 smithi006 bash[33918]: cluster 2023-08-28T01:09:39.665009+0000 mgr.y (mgr.14901) 5732 : cluster [DBG] pgmap v5739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:41.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:40 smithi006 bash[33918]: audit 2023-08-28T01:09:40.288914+0000 mon.a (mon.0) 3910 : audit [DBG] from='client.? 172.21.15.6:0/2483292523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:09:40.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:09:40.939Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:41.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:41 smithi130 bash[51302]: debug 2023-08-28T01:09:41.264+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:41.266163+0000) 2023-08-28T01:09:42.581 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:42 smithi130 bash[51302]: debug 2023-08-28T01:09:42.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:42.266473+0000) 2023-08-28T01:09:42.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:42 smithi130 bash[27579]: cluster 2023-08-28T01:09:41.665689+0000 mgr.y (mgr.14901) 5733 : cluster [DBG] pgmap v5740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:42 smithi006 bash[27292]: cluster 2023-08-28T01:09:41.665689+0000 mgr.y (mgr.14901) 5733 : cluster [DBG] pgmap v5740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:42.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:42 smithi006 bash[33918]: cluster 2023-08-28T01:09:41.665689+0000 mgr.y (mgr.14901) 5733 : cluster [DBG] pgmap v5740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:09:42.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:09:42.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:43.587 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:43 smithi130 bash[51302]: debug 2023-08-28T01:09:43.264+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:43.266792+0000) 2023-08-28T01:09:43.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:43 smithi130 bash[27579]: audit 2023-08-28T01:09:42.705948+0000 mon.a (mon.0) 3911 : audit [DBG] from='client.? 172.21.15.6:0/3644194469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:43.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:43 smithi006 bash[27292]: audit 2023-08-28T01:09:42.705948+0000 mon.a (mon.0) 3911 : audit [DBG] from='client.? 172.21.15.6:0/3644194469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:43.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:43 smithi006 bash[33918]: audit 2023-08-28T01:09:42.705948+0000 mon.a (mon.0) 3911 : audit [DBG] from='client.? 172.21.15.6:0/3644194469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:44.595 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:44 smithi130 bash[51302]: debug 2023-08-28T01:09:44.264+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:44.267108+0000) 2023-08-28T01:09:44.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:44 smithi006 bash[27292]: cluster 2023-08-28T01:09:43.666330+0000 mgr.y (mgr.14901) 5734 : cluster [DBG] pgmap v5741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:44.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:44 smithi006 bash[33918]: cluster 2023-08-28T01:09:43.666330+0000 mgr.y (mgr.14901) 5734 : cluster [DBG] pgmap v5741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:44.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:09:44.391Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:44.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:09:44.394Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:44.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:44 smithi130 bash[27579]: cluster 2023-08-28T01:09:43.666330+0000 mgr.y (mgr.14901) 5734 : cluster [DBG] pgmap v5741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:45.598 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:45 smithi130 bash[51302]: debug 2023-08-28T01:09:45.264+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:45.267391+0000) 2023-08-28T01:09:45.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:45 smithi130 bash[27579]: audit 2023-08-28T01:09:45.116933+0000 mon.c (mon.2) 2235 : audit [DBG] from='client.? 172.21.15.6:0/1764128681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:45.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:45 smithi006 bash[33918]: audit 2023-08-28T01:09:45.116933+0000 mon.c (mon.2) 2235 : audit [DBG] from='client.? 172.21.15.6:0/1764128681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:45.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:45 smithi006 bash[27292]: audit 2023-08-28T01:09:45.116933+0000 mon.c (mon.2) 2235 : audit [DBG] from='client.? 172.21.15.6:0/1764128681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:46.607 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:46 smithi130 bash[51302]: debug 2023-08-28T01:09:46.264+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:46.267665+0000) 2023-08-28T01:09:46.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:46 smithi130 bash[27579]: cluster 2023-08-28T01:09:45.667444+0000 mgr.y (mgr.14901) 5735 : cluster [DBG] pgmap v5742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:46 smithi006 bash[33918]: cluster 2023-08-28T01:09:45.667444+0000 mgr.y (mgr.14901) 5735 : cluster [DBG] pgmap v5742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:46.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:46 smithi006 bash[27292]: cluster 2023-08-28T01:09:45.667444+0000 mgr.y (mgr.14901) 5735 : cluster [DBG] pgmap v5742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:47.610 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:47 smithi130 bash[51302]: debug 2023-08-28T01:09:47.264+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:47.267898+0000) 2023-08-28T01:09:47.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:47 smithi130 bash[27579]: audit 2023-08-28T01:09:47.529166+0000 mon.a (mon.0) 3912 : audit [DBG] from='client.? 172.21.15.6:0/3004064217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:47.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:47 smithi006 bash[33918]: audit 2023-08-28T01:09:47.529166+0000 mon.a (mon.0) 3912 : audit [DBG] from='client.? 172.21.15.6:0/3004064217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:47.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:47 smithi006 bash[27292]: audit 2023-08-28T01:09:47.529166+0000 mon.a (mon.0) 3912 : audit [DBG] from='client.? 172.21.15.6:0/3004064217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:48.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:48 smithi130 bash[51302]: debug 2023-08-28T01:09:48.264+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:48.268178+0000) 2023-08-28T01:09:48.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:48 smithi006 bash[27292]: cluster 2023-08-28T01:09:47.668134+0000 mgr.y (mgr.14901) 5736 : cluster [DBG] pgmap v5743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:48.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:48 smithi006 bash[33918]: cluster 2023-08-28T01:09:47.668134+0000 mgr.y (mgr.14901) 5736 : cluster [DBG] pgmap v5743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:49.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:48 smithi130 bash[27579]: cluster 2023-08-28T01:09:47.668134+0000 mgr.y (mgr.14901) 5736 : cluster [DBG] pgmap v5743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:49.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:49 smithi130 bash[51302]: debug 2023-08-28T01:09:49.268+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:49.268492+0000) 2023-08-28T01:09:50.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:50 smithi130 bash[51302]: debug 2023-08-28T01:09:50.268+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:50.268846+0000) 2023-08-28T01:09:51.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:50 smithi130 bash[27579]: cluster 2023-08-28T01:09:49.668799+0000 mgr.y (mgr.14901) 5737 : cluster [DBG] pgmap v5744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:50 smithi130 bash[27579]: audit 2023-08-28T01:09:49.945168+0000 mon.a (mon.0) 3913 : audit [DBG] from='client.? 172.21.15.6:0/3094954245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:09:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:09:50] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T01:09:51.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:50 smithi006 bash[27292]: cluster 2023-08-28T01:09:49.668799+0000 mgr.y (mgr.14901) 5737 : cluster [DBG] pgmap v5744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:51.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:50 smithi006 bash[27292]: audit 2023-08-28T01:09:49.945168+0000 mon.a (mon.0) 3913 : audit [DBG] from='client.? 172.21.15.6:0/3094954245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:09:50.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:09:50.939Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:50 smithi006 bash[33918]: cluster 2023-08-28T01:09:49.668799+0000 mgr.y (mgr.14901) 5737 : cluster [DBG] pgmap v5744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:50 smithi006 bash[33918]: audit 2023-08-28T01:09:49.945168+0000 mon.a (mon.0) 3913 : audit [DBG] from='client.? 172.21.15.6:0/3094954245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:51.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:51 smithi130 bash[51302]: debug 2023-08-28T01:09:51.264+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:51.269145+0000) 2023-08-28T01:09:52.583 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:52 smithi130 bash[51302]: debug 2023-08-28T01:09:52.264+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:52.269361+0000) 2023-08-28T01:09:52.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:52 smithi130 bash[27579]: cluster 2023-08-28T01:09:51.669510+0000 mgr.y (mgr.14901) 5738 : cluster [DBG] pgmap v5745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:52 smithi130 bash[27579]: audit 2023-08-28T01:09:52.352676+0000 mon.a (mon.0) 3914 : audit [DBG] from='client.? 172.21.15.6:0/4168786626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:09:52.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:09:52.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:52 smithi006 bash[33918]: cluster 2023-08-28T01:09:51.669510+0000 mgr.y (mgr.14901) 5738 : cluster [DBG] pgmap v5745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:52 smithi006 bash[33918]: audit 2023-08-28T01:09:52.352676+0000 mon.a (mon.0) 3914 : audit [DBG] from='client.? 172.21.15.6:0/4168786626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:52 smithi006 bash[27292]: cluster 2023-08-28T01:09:51.669510+0000 mgr.y (mgr.14901) 5738 : cluster [DBG] pgmap v5745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:52 smithi006 bash[27292]: audit 2023-08-28T01:09:52.352676+0000 mon.a (mon.0) 3914 : audit [DBG] from='client.? 172.21.15.6:0/4168786626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:53.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:53 smithi130 bash[51302]: debug 2023-08-28T01:09:53.268+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:53.269678+0000) 2023-08-28T01:09:54.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:54 smithi130 bash[51302]: debug 2023-08-28T01:09:54.264+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:54.269955+0000) 2023-08-28T01:09:54.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:09:54.392Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:54.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:09:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:09:54.393Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:09:55.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:54 smithi130 bash[27579]: cluster 2023-08-28T01:09:53.670136+0000 mgr.y (mgr.14901) 5739 : cluster [DBG] pgmap v5746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:55.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:54 smithi006 bash[33918]: cluster 2023-08-28T01:09:53.670136+0000 mgr.y (mgr.14901) 5739 : cluster [DBG] pgmap v5746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:55.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:54 smithi006 bash[27292]: cluster 2023-08-28T01:09:53.670136+0000 mgr.y (mgr.14901) 5739 : cluster [DBG] pgmap v5746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:55.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:55 smithi130 bash[51302]: debug 2023-08-28T01:09:55.264+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:55.270243+0000) 2023-08-28T01:09:56.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:55 smithi130 bash[27579]: audit 2023-08-28T01:09:54.766157+0000 mon.a (mon.0) 3915 : audit [DBG] from='client.? 172.21.15.6:0/3189201681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:56.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:55 smithi006 bash[27292]: audit 2023-08-28T01:09:54.766157+0000 mon.a (mon.0) 3915 : audit [DBG] from='client.? 172.21.15.6:0/3189201681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:56.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:55 smithi006 bash[33918]: audit 2023-08-28T01:09:54.766157+0000 mon.a (mon.0) 3915 : audit [DBG] from='client.? 172.21.15.6:0/3189201681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:56.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:56 smithi130 bash[51302]: debug 2023-08-28T01:09:56.268+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:56.270567+0000) 2023-08-28T01:09:57.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:56 smithi130 bash[27579]: cluster 2023-08-28T01:09:55.671290+0000 mgr.y (mgr.14901) 5740 : cluster [DBG] pgmap v5747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:57.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:56 smithi006 bash[33918]: cluster 2023-08-28T01:09:55.671290+0000 mgr.y (mgr.14901) 5740 : cluster [DBG] pgmap v5747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:57.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:56 smithi006 bash[27292]: cluster 2023-08-28T01:09:55.671290+0000 mgr.y (mgr.14901) 5740 : cluster [DBG] pgmap v5747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:57.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:57 smithi130 bash[51302]: debug 2023-08-28T01:09:57.268+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:57.270831+0000) 2023-08-28T01:09:58.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:57 smithi130 bash[27579]: audit 2023-08-28T01:09:57.188310+0000 mon.a (mon.0) 3916 : audit [DBG] from='client.? 172.21.15.6:0/3300477235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:58.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:57 smithi006 bash[33918]: audit 2023-08-28T01:09:57.188310+0000 mon.a (mon.0) 3916 : audit [DBG] from='client.? 172.21.15.6:0/3300477235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:58.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:57 smithi006 bash[27292]: audit 2023-08-28T01:09:57.188310+0000 mon.a (mon.0) 3916 : audit [DBG] from='client.? 172.21.15.6:0/3300477235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:58.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:58 smithi130 bash[51302]: debug 2023-08-28T01:09:58.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:58.271118+0000) 2023-08-28T01:09:59.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:58 smithi130 bash[27579]: cluster 2023-08-28T01:09:57.671933+0000 mgr.y (mgr.14901) 5741 : cluster [DBG] pgmap v5748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:59.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:58 smithi006 bash[27292]: cluster 2023-08-28T01:09:57.671933+0000 mgr.y (mgr.14901) 5741 : cluster [DBG] pgmap v5748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:59.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:58 smithi006 bash[33918]: cluster 2023-08-28T01:09:57.671933+0000 mgr.y (mgr.14901) 5741 : cluster [DBG] pgmap v5748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:09:59.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:09:59 smithi130 bash[51302]: debug 2023-08-28T01:09:59.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:59.271335+0000) 2023-08-28T01:10:00.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:09:59 smithi130 bash[27579]: audit 2023-08-28T01:09:59.616025+0000 mon.a (mon.0) 3917 : audit [DBG] from='client.? 172.21.15.6:0/392583828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:00.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:09:59 smithi006 bash[27292]: audit 2023-08-28T01:09:59.616025+0000 mon.a (mon.0) 3917 : audit [DBG] from='client.? 172.21.15.6:0/392583828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:00.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:09:59 smithi006 bash[33918]: audit 2023-08-28T01:09:59.616025+0000 mon.a (mon.0) 3917 : audit [DBG] from='client.? 172.21.15.6:0/392583828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:00.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:00 smithi130 bash[51302]: debug 2023-08-28T01:10:00.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:00.271652+0000) 2023-08-28T01:10:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:00 smithi130 bash[27579]: cluster 2023-08-28T01:09:59.673024+0000 mgr.y (mgr.14901) 5742 : cluster [DBG] pgmap v5749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:00 smithi130 bash[27579]: cluster 2023-08-28T01:10:00.000156+0000 mon.a (mon.0) 3918 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:10:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:10:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:10:00] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T01:10:01.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:00 smithi006 bash[33918]: cluster 2023-08-28T01:09:59.673024+0000 mgr.y (mgr.14901) 5742 : cluster [DBG] pgmap v5749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:01.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:00 smithi006 bash[33918]: cluster 2023-08-28T01:10:00.000156+0000 mon.a (mon.0) 3918 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:10:01.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:00 smithi006 bash[27292]: cluster 2023-08-28T01:09:59.673024+0000 mgr.y (mgr.14901) 5742 : cluster [DBG] pgmap v5749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:01.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:00 smithi006 bash[27292]: cluster 2023-08-28T01:10:00.000156+0000 mon.a (mon.0) 3918 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:10:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:10:00.939Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:10:00.941Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:01.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:01 smithi130 bash[51302]: debug 2023-08-28T01:10:01.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:01.271936+0000) 2023-08-28T01:10:02.585 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:02 smithi130 bash[51302]: debug 2023-08-28T01:10:02.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:02.272201+0000) 2023-08-28T01:10:02.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:02 smithi130 bash[27579]: cluster 2023-08-28T01:10:01.673668+0000 mgr.y (mgr.14901) 5743 : cluster [DBG] pgmap v5750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:02.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:02 smithi130 bash[27579]: audit 2023-08-28T01:10:02.035046+0000 mon.c (mon.2) 2236 : audit [DBG] from='client.? 172.21.15.6:0/2700214218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:02 smithi006 bash[33918]: cluster 2023-08-28T01:10:01.673668+0000 mgr.y (mgr.14901) 5743 : cluster [DBG] pgmap v5750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:02 smithi006 bash[33918]: audit 2023-08-28T01:10:02.035046+0000 mon.c (mon.2) 2236 : audit [DBG] from='client.? 172.21.15.6:0/2700214218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:02.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:10:02.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:02.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:10:02.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:02 smithi006 bash[27292]: cluster 2023-08-28T01:10:01.673668+0000 mgr.y (mgr.14901) 5743 : cluster [DBG] pgmap v5750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:02.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:02 smithi006 bash[27292]: audit 2023-08-28T01:10:02.035046+0000 mon.c (mon.2) 2236 : audit [DBG] from='client.? 172.21.15.6:0/2700214218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:03.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:03 smithi130 bash[51302]: debug 2023-08-28T01:10:03.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:03.272452+0000) 2023-08-28T01:10:04.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:04 smithi130 bash[51302]: debug 2023-08-28T01:10:04.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:04.272757+0000) 2023-08-28T01:10:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:10:04.393Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:10:04.395Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:05.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:04 smithi130 bash[27579]: cluster 2023-08-28T01:10:03.674288+0000 mgr.y (mgr.14901) 5744 : cluster [DBG] pgmap v5751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:04 smithi130 bash[27579]: audit 2023-08-28T01:10:04.448822+0000 mon.a (mon.0) 3919 : audit [DBG] from='client.? 172.21.15.6:0/1893201706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:05.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:04 smithi006 bash[27292]: cluster 2023-08-28T01:10:03.674288+0000 mgr.y (mgr.14901) 5744 : cluster [DBG] pgmap v5751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:05.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:04 smithi006 bash[27292]: audit 2023-08-28T01:10:04.448822+0000 mon.a (mon.0) 3919 : audit [DBG] from='client.? 172.21.15.6:0/1893201706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:05.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:04 smithi006 bash[33918]: cluster 2023-08-28T01:10:03.674288+0000 mgr.y (mgr.14901) 5744 : cluster [DBG] pgmap v5751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:05.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:04 smithi006 bash[33918]: audit 2023-08-28T01:10:04.448822+0000 mon.a (mon.0) 3919 : audit [DBG] from='client.? 172.21.15.6:0/1893201706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:05.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:05 smithi130 bash[51302]: debug 2023-08-28T01:10:05.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:05.273029+0000) 2023-08-28T01:10:06.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:06 smithi130 bash[51302]: debug 2023-08-28T01:10:06.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:06.273335+0000) 2023-08-28T01:10:07.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:06 smithi130 bash[27579]: cluster 2023-08-28T01:10:05.675368+0000 mgr.y (mgr.14901) 5745 : cluster [DBG] pgmap v5752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:06 smithi006 bash[27292]: cluster 2023-08-28T01:10:05.675368+0000 mgr.y (mgr.14901) 5745 : cluster [DBG] pgmap v5752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:07.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:06 smithi006 bash[33918]: cluster 2023-08-28T01:10:05.675368+0000 mgr.y (mgr.14901) 5745 : cluster [DBG] pgmap v5752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:07.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:07 smithi130 bash[51302]: debug 2023-08-28T01:10:07.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:07.273593+0000) 2023-08-28T01:10:08.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:07 smithi130 bash[27579]: audit 2023-08-28T01:10:06.866837+0000 mon.c (mon.2) 2237 : audit [DBG] from='client.? 172.21.15.6:0/543875572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:08.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:07 smithi006 bash[27292]: audit 2023-08-28T01:10:06.866837+0000 mon.c (mon.2) 2237 : audit [DBG] from='client.? 172.21.15.6:0/543875572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:08.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:07 smithi006 bash[33918]: audit 2023-08-28T01:10:06.866837+0000 mon.c (mon.2) 2237 : audit [DBG] from='client.? 172.21.15.6:0/543875572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:08.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:08 smithi130 bash[51302]: debug 2023-08-28T01:10:08.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:08.273866+0000) 2023-08-28T01:10:09.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:08 smithi130 bash[27579]: cluster 2023-08-28T01:10:07.676028+0000 mgr.y (mgr.14901) 5746 : cluster [DBG] pgmap v5753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:09.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:08 smithi006 bash[33918]: cluster 2023-08-28T01:10:07.676028+0000 mgr.y (mgr.14901) 5746 : cluster [DBG] pgmap v5753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:09.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:08 smithi006 bash[27292]: cluster 2023-08-28T01:10:07.676028+0000 mgr.y (mgr.14901) 5746 : cluster [DBG] pgmap v5753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:09.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:09 smithi130 bash[51302]: debug 2023-08-28T01:10:09.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:09.274171+0000) 2023-08-28T01:10:10.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:09 smithi130 bash[27579]: audit 2023-08-28T01:10:09.282738+0000 mon.a (mon.0) 3920 : audit [DBG] from='client.? 172.21.15.6:0/686420260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:10.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:09 smithi006 bash[27292]: audit 2023-08-28T01:10:09.282738+0000 mon.a (mon.0) 3920 : audit [DBG] from='client.? 172.21.15.6:0/686420260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:10.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:09 smithi006 bash[33918]: audit 2023-08-28T01:10:09.282738+0000 mon.a (mon.0) 3920 : audit [DBG] from='client.? 172.21.15.6:0/686420260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:10.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:10 smithi130 bash[51302]: debug 2023-08-28T01:10:10.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:10.274459+0000) 2023-08-28T01:10:11.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:10 smithi130 bash[27579]: cluster 2023-08-28T01:10:09.677126+0000 mgr.y (mgr.14901) 5747 : cluster [DBG] pgmap v5754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:10:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:10:10] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T01:10:11.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:10 smithi006 bash[27292]: cluster 2023-08-28T01:10:09.677126+0000 mgr.y (mgr.14901) 5747 : cluster [DBG] pgmap v5754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:11.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:10 smithi006 bash[33918]: cluster 2023-08-28T01:10:09.677126+0000 mgr.y (mgr.14901) 5747 : cluster [DBG] pgmap v5754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:10:10.939Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:10:10.942Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:11.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:11 smithi130 bash[51302]: debug 2023-08-28T01:10:11.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:11.274738+0000) 2023-08-28T01:10:12.587 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:12 smithi130 bash[51302]: debug 2023-08-28T01:10:12.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:12.274983+0000) 2023-08-28T01:10:12.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:12 smithi130 bash[27579]: cluster 2023-08-28T01:10:11.677789+0000 mgr.y (mgr.14901) 5748 : cluster [DBG] pgmap v5755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:12.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:12 smithi130 bash[27579]: audit 2023-08-28T01:10:11.694342+0000 mon.a (mon.0) 3921 : audit [DBG] from='client.? 172.21.15.6:0/361159690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:12 smithi006 bash[27292]: cluster 2023-08-28T01:10:11.677789+0000 mgr.y (mgr.14901) 5748 : cluster [DBG] pgmap v5755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:12 smithi006 bash[27292]: audit 2023-08-28T01:10:11.694342+0000 mon.a (mon.0) 3921 : audit [DBG] from='client.? 172.21.15.6:0/361159690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:12 smithi006 bash[33918]: cluster 2023-08-28T01:10:11.677789+0000 mgr.y (mgr.14901) 5748 : cluster [DBG] pgmap v5755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:12 smithi006 bash[33918]: audit 2023-08-28T01:10:11.694342+0000 mon.a (mon.0) 3921 : audit [DBG] from='client.? 172.21.15.6:0/361159690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:10:12.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:12.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:10:12.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:13.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:13 smithi130 bash[51302]: debug 2023-08-28T01:10:13.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:13.275272+0000) 2023-08-28T01:10:14.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:14 smithi130 bash[51302]: debug 2023-08-28T01:10:14.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:14.275544+0000) 2023-08-28T01:10:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:10:14.394Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:10:14.396Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:15.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:14 smithi130 bash[27579]: cluster 2023-08-28T01:10:13.678413+0000 mgr.y (mgr.14901) 5749 : cluster [DBG] pgmap v5756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:15.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:14 smithi130 bash[27579]: audit 2023-08-28T01:10:14.106001+0000 mon.a (mon.0) 3922 : audit [DBG] from='client.? 172.21.15.6:0/758421658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:15.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:14 smithi006 bash[33918]: cluster 2023-08-28T01:10:13.678413+0000 mgr.y (mgr.14901) 5749 : cluster [DBG] pgmap v5756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:15.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:14 smithi006 bash[33918]: audit 2023-08-28T01:10:14.106001+0000 mon.a (mon.0) 3922 : audit [DBG] from='client.? 172.21.15.6:0/758421658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:15.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:14 smithi006 bash[27292]: cluster 2023-08-28T01:10:13.678413+0000 mgr.y (mgr.14901) 5749 : cluster [DBG] pgmap v5756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:15.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:14 smithi006 bash[27292]: audit 2023-08-28T01:10:14.106001+0000 mon.a (mon.0) 3922 : audit [DBG] from='client.? 172.21.15.6:0/758421658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:15.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:15 smithi130 bash[51302]: debug 2023-08-28T01:10:15.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:15.275859+0000) 2023-08-28T01:10:16.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:16 smithi130 bash[51302]: debug 2023-08-28T01:10:16.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:16.276155+0000) 2023-08-28T01:10:17.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:16 smithi130 bash[27579]: cluster 2023-08-28T01:10:15.679601+0000 mgr.y (mgr.14901) 5750 : cluster [DBG] pgmap v5757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:17.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:16 smithi130 bash[27579]: audit 2023-08-28T01:10:16.527661+0000 mon.c (mon.2) 2238 : audit [DBG] from='client.? 172.21.15.6:0/3043852962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:17.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:16 smithi006 bash[33918]: cluster 2023-08-28T01:10:15.679601+0000 mgr.y (mgr.14901) 5750 : cluster [DBG] pgmap v5757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:17.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:16 smithi006 bash[33918]: audit 2023-08-28T01:10:16.527661+0000 mon.c (mon.2) 2238 : audit [DBG] from='client.? 172.21.15.6:0/3043852962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:17.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:16 smithi006 bash[27292]: cluster 2023-08-28T01:10:15.679601+0000 mgr.y (mgr.14901) 5750 : cluster [DBG] pgmap v5757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:17.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:16 smithi006 bash[27292]: audit 2023-08-28T01:10:16.527661+0000 mon.c (mon.2) 2238 : audit [DBG] from='client.? 172.21.15.6:0/3043852962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:17.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:17 smithi130 bash[51302]: debug 2023-08-28T01:10:17.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:17.276448+0000) 2023-08-28T01:10:18.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:18 smithi130 bash[51302]: debug 2023-08-28T01:10:18.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:18.276756+0000) 2023-08-28T01:10:19.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:18 smithi130 bash[27579]: cluster 2023-08-28T01:10:17.680229+0000 mgr.y (mgr.14901) 5751 : cluster [DBG] pgmap v5758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:19.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:18 smithi006 bash[27292]: cluster 2023-08-28T01:10:17.680229+0000 mgr.y (mgr.14901) 5751 : cluster [DBG] pgmap v5758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:19.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:18 smithi006 bash[33918]: cluster 2023-08-28T01:10:17.680229+0000 mgr.y (mgr.14901) 5751 : cluster [DBG] pgmap v5758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:19.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:19 smithi130 bash[51302]: debug 2023-08-28T01:10:19.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:19.277035+0000) 2023-08-28T01:10:20.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:19 smithi130 bash[27579]: audit 2023-08-28T01:10:18.946418+0000 mon.c (mon.2) 2239 : audit [DBG] from='client.? 172.21.15.6:0/3650377636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:20.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:19 smithi006 bash[27292]: audit 2023-08-28T01:10:18.946418+0000 mon.c (mon.2) 2239 : audit [DBG] from='client.? 172.21.15.6:0/3650377636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:20.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:19 smithi006 bash[33918]: audit 2023-08-28T01:10:18.946418+0000 mon.c (mon.2) 2239 : audit [DBG] from='client.? 172.21.15.6:0/3650377636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:20.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:20 smithi130 bash[51302]: debug 2023-08-28T01:10:20.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:20.277336+0000) 2023-08-28T01:10:21.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:20 smithi130 bash[27579]: cluster 2023-08-28T01:10:19.681315+0000 mgr.y (mgr.14901) 5752 : cluster [DBG] pgmap v5759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:21.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:20 smithi130 bash[27579]: audit 2023-08-28T01:10:19.904265+0000 mon.b (mon.1) 671 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:10:21.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:10:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:10:20] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:10:21.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:20 smithi006 bash[33918]: cluster 2023-08-28T01:10:19.681315+0000 mgr.y (mgr.14901) 5752 : cluster [DBG] pgmap v5759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:21.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:20 smithi006 bash[33918]: audit 2023-08-28T01:10:19.904265+0000 mon.b (mon.1) 671 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:10:21.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:10:20.940Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:21.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:10:20.942Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:20 smithi006 bash[27292]: cluster 2023-08-28T01:10:19.681315+0000 mgr.y (mgr.14901) 5752 : cluster [DBG] pgmap v5759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:20 smithi006 bash[27292]: audit 2023-08-28T01:10:19.904265+0000 mon.b (mon.1) 671 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:10:21.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:21 smithi130 bash[51302]: debug 2023-08-28T01:10:21.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:21.277628+0000) 2023-08-28T01:10:22.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:21 smithi130 bash[27579]: audit 2023-08-28T01:10:21.378459+0000 mon.a (mon.0) 3923 : audit [DBG] from='client.? 172.21.15.6:0/1582523245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:22.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:21 smithi006 bash[33918]: audit 2023-08-28T01:10:21.378459+0000 mon.a (mon.0) 3923 : audit [DBG] from='client.? 172.21.15.6:0/1582523245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:22.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:21 smithi006 bash[27292]: audit 2023-08-28T01:10:21.378459+0000 mon.a (mon.0) 3923 : audit [DBG] from='client.? 172.21.15.6:0/1582523245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:22.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:22 smithi130 bash[51302]: debug 2023-08-28T01:10:22.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:22.277914+0000) 2023-08-28T01:10:22.766 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:10:22.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:22.767 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:10:22.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:23.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:22 smithi130 bash[27579]: cluster 2023-08-28T01:10:21.681938+0000 mgr.y (mgr.14901) 5753 : cluster [DBG] pgmap v5760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:23.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:22 smithi006 bash[27292]: cluster 2023-08-28T01:10:21.681938+0000 mgr.y (mgr.14901) 5753 : cluster [DBG] pgmap v5760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:23.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:22 smithi006 bash[33918]: cluster 2023-08-28T01:10:21.681938+0000 mgr.y (mgr.14901) 5753 : cluster [DBG] pgmap v5760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:23.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:23 smithi130 bash[51302]: debug 2023-08-28T01:10:23.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:23.278170+0000) 2023-08-28T01:10:24.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:24 smithi130 bash[51302]: debug 2023-08-28T01:10:24.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:24.278396+0000) 2023-08-28T01:10:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:10:24.395Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:10:24.397Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:25.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:24 smithi130 bash[27579]: cluster 2023-08-28T01:10:23.682364+0000 mgr.y (mgr.14901) 5754 : cluster [DBG] pgmap v5761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:25.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:24 smithi130 bash[27579]: audit 2023-08-28T01:10:23.812469+0000 mon.c (mon.2) 2240 : audit [DBG] from='client.? 172.21.15.6:0/596699746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:25.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:24 smithi006 bash[33918]: cluster 2023-08-28T01:10:23.682364+0000 mgr.y (mgr.14901) 5754 : cluster [DBG] pgmap v5761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:25.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:24 smithi006 bash[33918]: audit 2023-08-28T01:10:23.812469+0000 mon.c (mon.2) 2240 : audit [DBG] from='client.? 172.21.15.6:0/596699746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:25.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:24 smithi006 bash[27292]: cluster 2023-08-28T01:10:23.682364+0000 mgr.y (mgr.14901) 5754 : cluster [DBG] pgmap v5761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:25.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:24 smithi006 bash[27292]: audit 2023-08-28T01:10:23.812469+0000 mon.c (mon.2) 2240 : audit [DBG] from='client.? 172.21.15.6:0/596699746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:25.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:25 smithi130 bash[51302]: debug 2023-08-28T01:10:25.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:25.278674+0000) 2023-08-28T01:10:26.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:26 smithi130 bash[51302]: debug 2023-08-28T01:10:26.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:26.278887+0000) 2023-08-28T01:10:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:26 smithi130 bash[27579]: cluster 2023-08-28T01:10:25.683136+0000 mgr.y (mgr.14901) 5755 : cluster [DBG] pgmap v5762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:26 smithi130 bash[27579]: audit 2023-08-28T01:10:25.804655+0000 mon.a (mon.0) 3924 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:10:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:26 smithi130 bash[27579]: audit 2023-08-28T01:10:26.211436+0000 mon.a (mon.0) 3925 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:10:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:26 smithi130 bash[27579]: audit 2023-08-28T01:10:26.237309+0000 mon.a (mon.0) 3926 : audit [DBG] from='client.? 172.21.15.6:0/1798877728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:26 smithi130 bash[27579]: audit 2023-08-28T01:10:26.508734+0000 mon.b (mon.1) 672 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:10:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:26 smithi130 bash[27579]: audit 2023-08-28T01:10:26.510877+0000 mon.b (mon.1) 673 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:10:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:26 smithi130 bash[27579]: audit 2023-08-28T01:10:26.517445+0000 mon.a (mon.0) 3927 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:10:27.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:26 smithi006 bash[27292]: cluster 2023-08-28T01:10:25.683136+0000 mgr.y (mgr.14901) 5755 : cluster [DBG] pgmap v5762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:27.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:26 smithi006 bash[27292]: audit 2023-08-28T01:10:25.804655+0000 mon.a (mon.0) 3924 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:10:27.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:26 smithi006 bash[27292]: audit 2023-08-28T01:10:26.211436+0000 mon.a (mon.0) 3925 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:10:27.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:26 smithi006 bash[27292]: audit 2023-08-28T01:10:26.237309+0000 mon.a (mon.0) 3926 : audit [DBG] from='client.? 172.21.15.6:0/1798877728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:26 smithi006 bash[27292]: audit 2023-08-28T01:10:26.508734+0000 mon.b (mon.1) 672 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:10:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:26 smithi006 bash[27292]: audit 2023-08-28T01:10:26.510877+0000 mon.b (mon.1) 673 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:10:27.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:26 smithi006 bash[27292]: audit 2023-08-28T01:10:26.517445+0000 mon.a (mon.0) 3927 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:10:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:26 smithi006 bash[33918]: cluster 2023-08-28T01:10:25.683136+0000 mgr.y (mgr.14901) 5755 : cluster [DBG] pgmap v5762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:26 smithi006 bash[33918]: audit 2023-08-28T01:10:25.804655+0000 mon.a (mon.0) 3924 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:10:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:26 smithi006 bash[33918]: audit 2023-08-28T01:10:26.211436+0000 mon.a (mon.0) 3925 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:10:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:26 smithi006 bash[33918]: audit 2023-08-28T01:10:26.237309+0000 mon.a (mon.0) 3926 : audit [DBG] from='client.? 172.21.15.6:0/1798877728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:27.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:26 smithi006 bash[33918]: audit 2023-08-28T01:10:26.508734+0000 mon.b (mon.1) 672 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:10:27.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:26 smithi006 bash[33918]: audit 2023-08-28T01:10:26.510877+0000 mon.b (mon.1) 673 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:10:27.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:26 smithi006 bash[33918]: audit 2023-08-28T01:10:26.517445+0000 mon.a (mon.0) 3927 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:10:27.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:27 smithi130 bash[51302]: debug 2023-08-28T01:10:27.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:27.279180+0000) 2023-08-28T01:10:28.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:28 smithi130 bash[51302]: debug 2023-08-28T01:10:28.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:28.279447+0000) 2023-08-28T01:10:29.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:28 smithi130 bash[27579]: cluster 2023-08-28T01:10:27.683754+0000 mgr.y (mgr.14901) 5756 : cluster [DBG] pgmap v5763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:29.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:28 smithi130 bash[27579]: audit 2023-08-28T01:10:28.659378+0000 mon.a (mon.0) 3928 : audit [DBG] from='client.? 172.21.15.6:0/2102126779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:29.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:28 smithi006 bash[33918]: cluster 2023-08-28T01:10:27.683754+0000 mgr.y (mgr.14901) 5756 : cluster [DBG] pgmap v5763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:29.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:28 smithi006 bash[33918]: audit 2023-08-28T01:10:28.659378+0000 mon.a (mon.0) 3928 : audit [DBG] from='client.? 172.21.15.6:0/2102126779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:29.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:28 smithi006 bash[27292]: cluster 2023-08-28T01:10:27.683754+0000 mgr.y (mgr.14901) 5756 : cluster [DBG] pgmap v5763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:29.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:28 smithi006 bash[27292]: audit 2023-08-28T01:10:28.659378+0000 mon.a (mon.0) 3928 : audit [DBG] from='client.? 172.21.15.6:0/2102126779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:29.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:29 smithi130 bash[51302]: debug 2023-08-28T01:10:29.279+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:29.279797+0000) 2023-08-28T01:10:30.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:30 smithi130 bash[51302]: debug 2023-08-28T01:10:30.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:30.280077+0000) 2023-08-28T01:10:31.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:30 smithi130 bash[27579]: cluster 2023-08-28T01:10:29.684899+0000 mgr.y (mgr.14901) 5757 : cluster [DBG] pgmap v5764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:31.129 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:10:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:10:30] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T01:10:31.130 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:30 smithi006 bash[33918]: cluster 2023-08-28T01:10:29.684899+0000 mgr.y (mgr.14901) 5757 : cluster [DBG] pgmap v5764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:31.130 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:10:30.941Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:31.130 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:30 smithi006 bash[27292]: cluster 2023-08-28T01:10:29.684899+0000 mgr.y (mgr.14901) 5757 : cluster [DBG] pgmap v5764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:31.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:10:30.943Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:31.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:31 smithi130 bash[51302]: debug 2023-08-28T01:10:31.279+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:31.280356+0000) 2023-08-28T01:10:32.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:31 smithi130 bash[27579]: audit 2023-08-28T01:10:31.078475+0000 mon.a (mon.0) 3929 : audit [DBG] from='client.? 172.21.15.6:0/2985029344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:32.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:31 smithi006 bash[33918]: audit 2023-08-28T01:10:31.078475+0000 mon.a (mon.0) 3929 : audit [DBG] from='client.? 172.21.15.6:0/2985029344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:32.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:31 smithi006 bash[27292]: audit 2023-08-28T01:10:31.078475+0000 mon.a (mon.0) 3929 : audit [DBG] from='client.? 172.21.15.6:0/2985029344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:32.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:32 smithi130 bash[51302]: debug 2023-08-28T01:10:32.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:32.280585+0000) 2023-08-28T01:10:32.823 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:10:32.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:32.824 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:10:32.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:32 smithi130 bash[27579]: cluster 2023-08-28T01:10:31.685513+0000 mgr.y (mgr.14901) 5758 : cluster [DBG] pgmap v5765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:33.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:32 smithi006 bash[33918]: cluster 2023-08-28T01:10:31.685513+0000 mgr.y (mgr.14901) 5758 : cluster [DBG] pgmap v5765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:33.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:32 smithi006 bash[27292]: cluster 2023-08-28T01:10:31.685513+0000 mgr.y (mgr.14901) 5758 : cluster [DBG] pgmap v5765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:33.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:33 smithi130 bash[51302]: debug 2023-08-28T01:10:33.279+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:33.280951+0000) 2023-08-28T01:10:34.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:33 smithi130 bash[27579]: audit 2023-08-28T01:10:33.493616+0000 mon.c (mon.2) 2241 : audit [DBG] from='client.? 172.21.15.6:0/3720047351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:34.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:33 smithi006 bash[33918]: audit 2023-08-28T01:10:33.493616+0000 mon.c (mon.2) 2241 : audit [DBG] from='client.? 172.21.15.6:0/3720047351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:34.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:33 smithi006 bash[27292]: audit 2023-08-28T01:10:33.493616+0000 mon.c (mon.2) 2241 : audit [DBG] from='client.? 172.21.15.6:0/3720047351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:34.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:34 smithi130 bash[51302]: debug 2023-08-28T01:10:34.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:34.281266+0000) 2023-08-28T01:10:34.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:10:34.395Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:34.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:10:34.397Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:35.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:34 smithi130 bash[27579]: cluster 2023-08-28T01:10:33.686207+0000 mgr.y (mgr.14901) 5759 : cluster [DBG] pgmap v5766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:35.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:34 smithi006 bash[27292]: cluster 2023-08-28T01:10:33.686207+0000 mgr.y (mgr.14901) 5759 : cluster [DBG] pgmap v5766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:35.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:34 smithi006 bash[33918]: cluster 2023-08-28T01:10:33.686207+0000 mgr.y (mgr.14901) 5759 : cluster [DBG] pgmap v5766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:35.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:35 smithi130 bash[51302]: debug 2023-08-28T01:10:35.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:35.281606+0000) 2023-08-28T01:10:36.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:36 smithi130 bash[51302]: debug 2023-08-28T01:10:36.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:36.281914+0000) 2023-08-28T01:10:37.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:36 smithi130 bash[27579]: cluster 2023-08-28T01:10:35.687351+0000 mgr.y (mgr.14901) 5760 : cluster [DBG] pgmap v5767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:37.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:36 smithi130 bash[27579]: audit 2023-08-28T01:10:35.902042+0000 mon.c (mon.2) 2242 : audit [DBG] from='client.? 172.21.15.6:0/3899023203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:37.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:36 smithi006 bash[27292]: cluster 2023-08-28T01:10:35.687351+0000 mgr.y (mgr.14901) 5760 : cluster [DBG] pgmap v5767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:37.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:36 smithi006 bash[27292]: audit 2023-08-28T01:10:35.902042+0000 mon.c (mon.2) 2242 : audit [DBG] from='client.? 172.21.15.6:0/3899023203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:37.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:36 smithi006 bash[33918]: cluster 2023-08-28T01:10:35.687351+0000 mgr.y (mgr.14901) 5760 : cluster [DBG] pgmap v5767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:37.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:36 smithi006 bash[33918]: audit 2023-08-28T01:10:35.902042+0000 mon.c (mon.2) 2242 : audit [DBG] from='client.? 172.21.15.6:0/3899023203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:37.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:37 smithi130 bash[51302]: debug 2023-08-28T01:10:37.279+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:37.282265+0000) 2023-08-28T01:10:38.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:38 smithi130 bash[51302]: debug 2023-08-28T01:10:38.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:38.282600+0000) 2023-08-28T01:10:39.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:38 smithi130 bash[27579]: cluster 2023-08-28T01:10:37.688006+0000 mgr.y (mgr.14901) 5761 : cluster [DBG] pgmap v5768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:39.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:38 smithi130 bash[27579]: audit 2023-08-28T01:10:38.308341+0000 mon.a (mon.0) 3930 : audit [DBG] from='client.? 172.21.15.6:0/1125978995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:39.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:38 smithi006 bash[27292]: cluster 2023-08-28T01:10:37.688006+0000 mgr.y (mgr.14901) 5761 : cluster [DBG] pgmap v5768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:39.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:38 smithi006 bash[27292]: audit 2023-08-28T01:10:38.308341+0000 mon.a (mon.0) 3930 : audit [DBG] from='client.? 172.21.15.6:0/1125978995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:39.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:38 smithi006 bash[33918]: cluster 2023-08-28T01:10:37.688006+0000 mgr.y (mgr.14901) 5761 : cluster [DBG] pgmap v5768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:39.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:38 smithi006 bash[33918]: audit 2023-08-28T01:10:38.308341+0000 mon.a (mon.0) 3930 : audit [DBG] from='client.? 172.21.15.6:0/1125978995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:39.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:39 smithi130 bash[51302]: debug 2023-08-28T01:10:39.279+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:39.282983+0000) 2023-08-28T01:10:40.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:40 smithi130 bash[51302]: debug 2023-08-28T01:10:40.279+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:40.283330+0000) 2023-08-28T01:10:41.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:40 smithi130 bash[27579]: cluster 2023-08-28T01:10:39.689098+0000 mgr.y (mgr.14901) 5762 : cluster [DBG] pgmap v5769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:41.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:40 smithi130 bash[27579]: audit 2023-08-28T01:10:40.731420+0000 mon.a (mon.0) 3931 : audit [DBG] from='client.? 172.21.15.6:0/1345757082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:10:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:10:40] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T01:10:41.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:10:40.942Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:41.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:10:40.944Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:40 smithi006 bash[27292]: cluster 2023-08-28T01:10:39.689098+0000 mgr.y (mgr.14901) 5762 : cluster [DBG] pgmap v5769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:40 smithi006 bash[27292]: audit 2023-08-28T01:10:40.731420+0000 mon.a (mon.0) 3931 : audit [DBG] from='client.? 172.21.15.6:0/1345757082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:41.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:40 smithi006 bash[33918]: cluster 2023-08-28T01:10:39.689098+0000 mgr.y (mgr.14901) 5762 : cluster [DBG] pgmap v5769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:41.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:40 smithi006 bash[33918]: audit 2023-08-28T01:10:40.731420+0000 mon.a (mon.0) 3931 : audit [DBG] from='client.? 172.21.15.6:0/1345757082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:41.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:41 smithi130 bash[51302]: debug 2023-08-28T01:10:41.283+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:41.283689+0000) 2023-08-28T01:10:42.611 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:42 smithi130 bash[51302]: debug 2023-08-28T01:10:42.279+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:42.284028+0000) 2023-08-28T01:10:42.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:42 smithi130 bash[27579]: cluster 2023-08-28T01:10:41.689774+0000 mgr.y (mgr.14901) 5763 : cluster [DBG] pgmap v5770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:42.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:42 smithi006 bash[33918]: cluster 2023-08-28T01:10:41.689774+0000 mgr.y (mgr.14901) 5763 : cluster [DBG] pgmap v5770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:42 smithi006 bash[27292]: cluster 2023-08-28T01:10:41.689774+0000 mgr.y (mgr.14901) 5763 : cluster [DBG] pgmap v5770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:10:42.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:10:42.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:43.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:43 smithi130 bash[51302]: debug 2023-08-28T01:10:43.279+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:43.284373+0000) 2023-08-28T01:10:43.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:43 smithi130 bash[27579]: audit 2023-08-28T01:10:43.153504+0000 mon.c (mon.2) 2243 : audit [DBG] from='client.? 172.21.15.6:0/4064633727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:43.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:43 smithi006 bash[27292]: audit 2023-08-28T01:10:43.153504+0000 mon.c (mon.2) 2243 : audit [DBG] from='client.? 172.21.15.6:0/4064633727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:43.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:43 smithi006 bash[33918]: audit 2023-08-28T01:10:43.153504+0000 mon.c (mon.2) 2243 : audit [DBG] from='client.? 172.21.15.6:0/4064633727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:44.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:44 smithi130 bash[51302]: debug 2023-08-28T01:10:44.283+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:44.284661+0000) 2023-08-28T01:10:44.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:44 smithi006 bash[27292]: cluster 2023-08-28T01:10:43.690428+0000 mgr.y (mgr.14901) 5764 : cluster [DBG] pgmap v5771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:44.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:44 smithi006 bash[33918]: cluster 2023-08-28T01:10:43.690428+0000 mgr.y (mgr.14901) 5764 : cluster [DBG] pgmap v5771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:44.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:10:44.396Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:44.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:10:44.398Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:45.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:44 smithi130 bash[27579]: cluster 2023-08-28T01:10:43.690428+0000 mgr.y (mgr.14901) 5764 : cluster [DBG] pgmap v5771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:45.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:45 smithi130 bash[51302]: debug 2023-08-28T01:10:45.283+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:45.284956+0000) 2023-08-28T01:10:45.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:45 smithi006 bash[27292]: audit 2023-08-28T01:10:45.567401+0000 mon.c (mon.2) 2244 : audit [DBG] from='client.? 172.21.15.6:0/2464890290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:45.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:45 smithi006 bash[33918]: audit 2023-08-28T01:10:45.567401+0000 mon.c (mon.2) 2244 : audit [DBG] from='client.? 172.21.15.6:0/2464890290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:46.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:45 smithi130 bash[27579]: audit 2023-08-28T01:10:45.567401+0000 mon.c (mon.2) 2244 : audit [DBG] from='client.? 172.21.15.6:0/2464890290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:46.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:46 smithi130 bash[51302]: debug 2023-08-28T01:10:46.283+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:46.285247+0000) 2023-08-28T01:10:46.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:46 smithi006 bash[27292]: cluster 2023-08-28T01:10:45.691631+0000 mgr.y (mgr.14901) 5765 : cluster [DBG] pgmap v5772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:46 smithi006 bash[33918]: cluster 2023-08-28T01:10:45.691631+0000 mgr.y (mgr.14901) 5765 : cluster [DBG] pgmap v5772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:46 smithi130 bash[27579]: cluster 2023-08-28T01:10:45.691631+0000 mgr.y (mgr.14901) 5765 : cluster [DBG] pgmap v5772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:47.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:47 smithi130 bash[51302]: debug 2023-08-28T01:10:47.283+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:47.285555+0000) 2023-08-28T01:10:48.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:48 smithi130 bash[51302]: debug 2023-08-28T01:10:48.279+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:48.285821+0000) 2023-08-28T01:10:49.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:48 smithi130 bash[27579]: cluster 2023-08-28T01:10:47.692250+0000 mgr.y (mgr.14901) 5766 : cluster [DBG] pgmap v5773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:49.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:48 smithi130 bash[27579]: audit 2023-08-28T01:10:47.986324+0000 mon.c (mon.2) 2245 : audit [DBG] from='client.? 172.21.15.6:0/3317515297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:49.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:48 smithi006 bash[33918]: cluster 2023-08-28T01:10:47.692250+0000 mgr.y (mgr.14901) 5766 : cluster [DBG] pgmap v5773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:49.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:48 smithi006 bash[33918]: audit 2023-08-28T01:10:47.986324+0000 mon.c (mon.2) 2245 : audit [DBG] from='client.? 172.21.15.6:0/3317515297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:49.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:48 smithi006 bash[27292]: cluster 2023-08-28T01:10:47.692250+0000 mgr.y (mgr.14901) 5766 : cluster [DBG] pgmap v5773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:49.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:48 smithi006 bash[27292]: audit 2023-08-28T01:10:47.986324+0000 mon.c (mon.2) 2245 : audit [DBG] from='client.? 172.21.15.6:0/3317515297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:49.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:49 smithi130 bash[51302]: debug 2023-08-28T01:10:49.283+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:49.286099+0000) 2023-08-28T01:10:50.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:50 smithi130 bash[51302]: debug 2023-08-28T01:10:50.283+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:50.286385+0000) 2023-08-28T01:10:51.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:50 smithi130 bash[27579]: cluster 2023-08-28T01:10:49.693423+0000 mgr.y (mgr.14901) 5767 : cluster [DBG] pgmap v5774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:51.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:50 smithi130 bash[27579]: audit 2023-08-28T01:10:50.418567+0000 mon.a (mon.0) 3932 : audit [DBG] from='client.? 172.21.15.6:0/2991554705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:51.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:50 smithi006 bash[27292]: cluster 2023-08-28T01:10:49.693423+0000 mgr.y (mgr.14901) 5767 : cluster [DBG] pgmap v5774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:51.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:50 smithi006 bash[27292]: audit 2023-08-28T01:10:50.418567+0000 mon.a (mon.0) 3932 : audit [DBG] from='client.? 172.21.15.6:0/2991554705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:10:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:10:50] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:10:51.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:50 smithi006 bash[33918]: cluster 2023-08-28T01:10:49.693423+0000 mgr.y (mgr.14901) 5767 : cluster [DBG] pgmap v5774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:50 smithi006 bash[33918]: audit 2023-08-28T01:10:50.418567+0000 mon.a (mon.0) 3932 : audit [DBG] from='client.? 172.21.15.6:0/2991554705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:10:50.943Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:10:50.945Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:51.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:51 smithi130 bash[51302]: debug 2023-08-28T01:10:51.279+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:51.286764+0000) 2023-08-28T01:10:52.610 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:52 smithi130 bash[51302]: debug 2023-08-28T01:10:52.283+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:52.286981+0000) 2023-08-28T01:10:52.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:52 smithi130 bash[27579]: cluster 2023-08-28T01:10:51.694095+0000 mgr.y (mgr.14901) 5768 : cluster [DBG] pgmap v5775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:52 smithi006 bash[27292]: cluster 2023-08-28T01:10:51.694095+0000 mgr.y (mgr.14901) 5768 : cluster [DBG] pgmap v5775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:52 smithi006 bash[33918]: cluster 2023-08-28T01:10:51.694095+0000 mgr.y (mgr.14901) 5768 : cluster [DBG] pgmap v5775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:10:52.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:10:52.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:53.613 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:53 smithi130 bash[51302]: debug 2023-08-28T01:10:53.283+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:53.287191+0000) 2023-08-28T01:10:53.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:53 smithi130 bash[27579]: audit 2023-08-28T01:10:52.846849+0000 mon.c (mon.2) 2246 : audit [DBG] from='client.? 172.21.15.6:0/2433847400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:53.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:53 smithi006 bash[27292]: audit 2023-08-28T01:10:52.846849+0000 mon.c (mon.2) 2246 : audit [DBG] from='client.? 172.21.15.6:0/2433847400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:53.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:53 smithi006 bash[33918]: audit 2023-08-28T01:10:52.846849+0000 mon.c (mon.2) 2246 : audit [DBG] from='client.? 172.21.15.6:0/2433847400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:54.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:54 smithi130 bash[51302]: debug 2023-08-28T01:10:54.287+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:54.287470+0000) 2023-08-28T01:10:54.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:54 smithi006 bash[33918]: cluster 2023-08-28T01:10:53.694747+0000 mgr.y (mgr.14901) 5769 : cluster [DBG] pgmap v5776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:54.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:54 smithi006 bash[27292]: cluster 2023-08-28T01:10:53.694747+0000 mgr.y (mgr.14901) 5769 : cluster [DBG] pgmap v5776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:54.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:10:54.397Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:54.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:10:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:10:54.399Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:10:55.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:54 smithi130 bash[27579]: cluster 2023-08-28T01:10:53.694747+0000 mgr.y (mgr.14901) 5769 : cluster [DBG] pgmap v5776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:55.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:55 smithi130 bash[51302]: debug 2023-08-28T01:10:55.286+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:55.287753+0000) 2023-08-28T01:10:55.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:55 smithi006 bash[27292]: audit 2023-08-28T01:10:55.268742+0000 mon.c (mon.2) 2247 : audit [DBG] from='client.? 172.21.15.6:0/800685700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:55.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:55 smithi006 bash[33918]: audit 2023-08-28T01:10:55.268742+0000 mon.c (mon.2) 2247 : audit [DBG] from='client.? 172.21.15.6:0/800685700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:56.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:55 smithi130 bash[27579]: audit 2023-08-28T01:10:55.268742+0000 mon.c (mon.2) 2247 : audit [DBG] from='client.? 172.21.15.6:0/800685700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:56.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:56 smithi130 bash[51302]: debug 2023-08-28T01:10:56.282+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:56.288048+0000) 2023-08-28T01:10:56.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:56 smithi006 bash[33918]: cluster 2023-08-28T01:10:55.695774+0000 mgr.y (mgr.14901) 5770 : cluster [DBG] pgmap v5777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:56.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:56 smithi006 bash[27292]: cluster 2023-08-28T01:10:55.695774+0000 mgr.y (mgr.14901) 5770 : cluster [DBG] pgmap v5777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:57.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:56 smithi130 bash[27579]: cluster 2023-08-28T01:10:55.695774+0000 mgr.y (mgr.14901) 5770 : cluster [DBG] pgmap v5777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:57.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:57 smithi130 bash[51302]: debug 2023-08-28T01:10:57.282+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:57.288313+0000) 2023-08-28T01:10:58.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:57 smithi130 bash[27579]: audit 2023-08-28T01:10:57.683056+0000 mon.c (mon.2) 2248 : audit [DBG] from='client.? 172.21.15.6:0/3451126267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:58.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:57 smithi006 bash[27292]: audit 2023-08-28T01:10:57.683056+0000 mon.c (mon.2) 2248 : audit [DBG] from='client.? 172.21.15.6:0/3451126267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:58.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:57 smithi006 bash[33918]: audit 2023-08-28T01:10:57.683056+0000 mon.c (mon.2) 2248 : audit [DBG] from='client.? 172.21.15.6:0/3451126267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:58.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:58 smithi130 bash[51302]: debug 2023-08-28T01:10:58.282+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:58.288616+0000) 2023-08-28T01:10:59.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:10:58 smithi130 bash[27579]: cluster 2023-08-28T01:10:57.696321+0000 mgr.y (mgr.14901) 5771 : cluster [DBG] pgmap v5778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:59.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:10:58 smithi006 bash[27292]: cluster 2023-08-28T01:10:57.696321+0000 mgr.y (mgr.14901) 5771 : cluster [DBG] pgmap v5778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:59.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:10:58 smithi006 bash[33918]: cluster 2023-08-28T01:10:57.696321+0000 mgr.y (mgr.14901) 5771 : cluster [DBG] pgmap v5778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:10:59.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:10:59 smithi130 bash[51302]: debug 2023-08-28T01:10:59.286+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:59.288892+0000) 2023-08-28T01:11:00.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:00 smithi130 bash[51302]: debug 2023-08-28T01:11:00.286+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:00.289185+0000) 2023-08-28T01:11:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:00 smithi130 bash[27579]: cluster 2023-08-28T01:10:59.697530+0000 mgr.y (mgr.14901) 5772 : cluster [DBG] pgmap v5779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:01.280 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:00 smithi130 bash[27579]: audit 2023-08-28T01:11:00.103785+0000 mon.c (mon.2) 2249 : audit [DBG] from='client.? 172.21.15.6:0/1394419396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:01.281 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:11:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:11:00] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:11:01.281 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:11:00.944Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:01.281 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:11:00.946Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:01.281 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:00 smithi006 bash[27292]: cluster 2023-08-28T01:10:59.697530+0000 mgr.y (mgr.14901) 5772 : cluster [DBG] pgmap v5779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:01.281 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:00 smithi006 bash[27292]: audit 2023-08-28T01:11:00.103785+0000 mon.c (mon.2) 2249 : audit [DBG] from='client.? 172.21.15.6:0/1394419396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:01.282 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:00 smithi006 bash[33918]: cluster 2023-08-28T01:10:59.697530+0000 mgr.y (mgr.14901) 5772 : cluster [DBG] pgmap v5779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:01.282 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:00 smithi006 bash[33918]: audit 2023-08-28T01:11:00.103785+0000 mon.c (mon.2) 2249 : audit [DBG] from='client.? 172.21.15.6:0/1394419396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:01.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:01 smithi130 bash[51302]: debug 2023-08-28T01:11:01.286+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:01.289600+0000) 2023-08-28T01:11:02.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:02 smithi130 bash[51302]: debug 2023-08-28T01:11:02.282+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:02.289979+0000) 2023-08-28T01:11:02.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:02 smithi130 bash[27579]: cluster 2023-08-28T01:11:01.698233+0000 mgr.y (mgr.14901) 5773 : cluster [DBG] pgmap v5780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:02.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:02 smithi130 bash[27579]: audit 2023-08-28T01:11:02.527910+0000 mon.a (mon.0) 3933 : audit [DBG] from='client.? 172.21.15.6:0/487982985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:02 smithi006 bash[27292]: cluster 2023-08-28T01:11:01.698233+0000 mgr.y (mgr.14901) 5773 : cluster [DBG] pgmap v5780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:02 smithi006 bash[27292]: audit 2023-08-28T01:11:02.527910+0000 mon.a (mon.0) 3933 : audit [DBG] from='client.? 172.21.15.6:0/487982985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:02 smithi006 bash[33918]: cluster 2023-08-28T01:11:01.698233+0000 mgr.y (mgr.14901) 5773 : cluster [DBG] pgmap v5780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:02 smithi006 bash[33918]: audit 2023-08-28T01:11:02.527910+0000 mon.a (mon.0) 3933 : audit [DBG] from='client.? 172.21.15.6:0/487982985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:02.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:11:02.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:02.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:11:02.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:03.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:03 smithi130 bash[51302]: debug 2023-08-28T01:11:03.282+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:03.290313+0000) 2023-08-28T01:11:04.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:04 smithi130 bash[51302]: debug 2023-08-28T01:11:04.286+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:04.290648+0000) 2023-08-28T01:11:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:11:04.398Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:11:04.400Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:05.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:04 smithi130 bash[27579]: cluster 2023-08-28T01:11:03.698863+0000 mgr.y (mgr.14901) 5774 : cluster [DBG] pgmap v5781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:05.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:04 smithi006 bash[27292]: cluster 2023-08-28T01:11:03.698863+0000 mgr.y (mgr.14901) 5774 : cluster [DBG] pgmap v5781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:05.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:04 smithi006 bash[33918]: cluster 2023-08-28T01:11:03.698863+0000 mgr.y (mgr.14901) 5774 : cluster [DBG] pgmap v5781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:05.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:05 smithi130 bash[51302]: debug 2023-08-28T01:11:05.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:05.290995+0000) 2023-08-28T01:11:06.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:05 smithi130 bash[27579]: audit 2023-08-28T01:11:04.944094+0000 mon.a (mon.0) 3934 : audit [DBG] from='client.? 172.21.15.6:0/1326950078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:06.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:05 smithi006 bash[27292]: audit 2023-08-28T01:11:04.944094+0000 mon.a (mon.0) 3934 : audit [DBG] from='client.? 172.21.15.6:0/1326950078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:06.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:05 smithi006 bash[33918]: audit 2023-08-28T01:11:04.944094+0000 mon.a (mon.0) 3934 : audit [DBG] from='client.? 172.21.15.6:0/1326950078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:06.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:06 smithi130 bash[51302]: debug 2023-08-28T01:11:06.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:06.291341+0000) 2023-08-28T01:11:07.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:06 smithi130 bash[27579]: cluster 2023-08-28T01:11:05.700012+0000 mgr.y (mgr.14901) 5775 : cluster [DBG] pgmap v5782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:07.433 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:06 smithi006 bash[27292]: cluster 2023-08-28T01:11:05.700012+0000 mgr.y (mgr.14901) 5775 : cluster [DBG] pgmap v5782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:07.435 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:06 smithi006 bash[33918]: cluster 2023-08-28T01:11:05.700012+0000 mgr.y (mgr.14901) 5775 : cluster [DBG] pgmap v5782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:07.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:07 smithi130 bash[51302]: debug 2023-08-28T01:11:07.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:07.291666+0000) 2023-08-28T01:11:08.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:07 smithi130 bash[27579]: audit 2023-08-28T01:11:07.361403+0000 mon.a (mon.0) 3935 : audit [DBG] from='client.? 172.21.15.6:0/3305889565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:08.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:07 smithi006 bash[33918]: audit 2023-08-28T01:11:07.361403+0000 mon.a (mon.0) 3935 : audit [DBG] from='client.? 172.21.15.6:0/3305889565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:08.237 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:07 smithi006 bash[27292]: audit 2023-08-28T01:11:07.361403+0000 mon.a (mon.0) 3935 : audit [DBG] from='client.? 172.21.15.6:0/3305889565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:08.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:08 smithi130 bash[51302]: debug 2023-08-28T01:11:08.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:08.291993+0000) 2023-08-28T01:11:09.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:08 smithi130 bash[27579]: cluster 2023-08-28T01:11:07.700693+0000 mgr.y (mgr.14901) 5776 : cluster [DBG] pgmap v5783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:09.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:08 smithi006 bash[33918]: cluster 2023-08-28T01:11:07.700693+0000 mgr.y (mgr.14901) 5776 : cluster [DBG] pgmap v5783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:08 smithi006 bash[27292]: cluster 2023-08-28T01:11:07.700693+0000 mgr.y (mgr.14901) 5776 : cluster [DBG] pgmap v5783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:09.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:09 smithi130 bash[51302]: debug 2023-08-28T01:11:09.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:09.292356+0000) 2023-08-28T01:11:10.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:10 smithi130 bash[51302]: debug 2023-08-28T01:11:10.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:10.292710+0000) 2023-08-28T01:11:11.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:10 smithi130 bash[27579]: cluster 2023-08-28T01:11:09.701873+0000 mgr.y (mgr.14901) 5777 : cluster [DBG] pgmap v5784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:11.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:10 smithi130 bash[27579]: audit 2023-08-28T01:11:09.785626+0000 mon.c (mon.2) 2250 : audit [DBG] from='client.? 172.21.15.6:0/4112812674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:11:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:11:10] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:11:11.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:10 smithi006 bash[33918]: cluster 2023-08-28T01:11:09.701873+0000 mgr.y (mgr.14901) 5777 : cluster [DBG] pgmap v5784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:10 smithi006 bash[33918]: audit 2023-08-28T01:11:09.785626+0000 mon.c (mon.2) 2250 : audit [DBG] from='client.? 172.21.15.6:0/4112812674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:11.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:10 smithi006 bash[27292]: cluster 2023-08-28T01:11:09.701873+0000 mgr.y (mgr.14901) 5777 : cluster [DBG] pgmap v5784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:11.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:10 smithi006 bash[27292]: audit 2023-08-28T01:11:09.785626+0000 mon.c (mon.2) 2250 : audit [DBG] from='client.? 172.21.15.6:0/4112812674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:11:10.944Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:11.218 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:11:10.946Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:11.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:11 smithi130 bash[51302]: debug 2023-08-28T01:11:11.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:11.293070+0000) 2023-08-28T01:11:12.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:12 smithi130 bash[51302]: debug 2023-08-28T01:11:12.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:12.293399+0000) 2023-08-28T01:11:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:12 smithi006 bash[27292]: cluster 2023-08-28T01:11:11.702592+0000 mgr.y (mgr.14901) 5778 : cluster [DBG] pgmap v5785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:12 smithi006 bash[27292]: audit 2023-08-28T01:11:12.203593+0000 mon.c (mon.2) 2251 : audit [DBG] from='client.? 172.21.15.6:0/3258367231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:12 smithi006 bash[33918]: cluster 2023-08-28T01:11:11.702592+0000 mgr.y (mgr.14901) 5778 : cluster [DBG] pgmap v5785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:12 smithi006 bash[33918]: audit 2023-08-28T01:11:12.203593+0000 mon.c (mon.2) 2251 : audit [DBG] from='client.? 172.21.15.6:0/3258367231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:11:12.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:12.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:11:12.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:12 smithi130 bash[27579]: cluster 2023-08-28T01:11:11.702592+0000 mgr.y (mgr.14901) 5778 : cluster [DBG] pgmap v5785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:12 smithi130 bash[27579]: audit 2023-08-28T01:11:12.203593+0000 mon.c (mon.2) 2251 : audit [DBG] from='client.? 172.21.15.6:0/3258367231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:13.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:13 smithi130 bash[51302]: debug 2023-08-28T01:11:13.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:13.293751+0000) 2023-08-28T01:11:14.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:14 smithi130 bash[51302]: debug 2023-08-28T01:11:14.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:14.294073+0000) 2023-08-28T01:11:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:11:14.399Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:11:14.401Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:14 smithi130 bash[27579]: cluster 2023-08-28T01:11:13.703260+0000 mgr.y (mgr.14901) 5779 : cluster [DBG] pgmap v5786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:14 smithi130 bash[27579]: audit 2023-08-28T01:11:14.623687+0000 mon.a (mon.0) 3936 : audit [DBG] from='client.? 172.21.15.6:0/455653270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:15.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:14 smithi006 bash[33918]: cluster 2023-08-28T01:11:13.703260+0000 mgr.y (mgr.14901) 5779 : cluster [DBG] pgmap v5786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:15.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:14 smithi006 bash[33918]: audit 2023-08-28T01:11:14.623687+0000 mon.a (mon.0) 3936 : audit [DBG] from='client.? 172.21.15.6:0/455653270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:15.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:14 smithi006 bash[27292]: cluster 2023-08-28T01:11:13.703260+0000 mgr.y (mgr.14901) 5779 : cluster [DBG] pgmap v5786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:15.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:14 smithi006 bash[27292]: audit 2023-08-28T01:11:14.623687+0000 mon.a (mon.0) 3936 : audit [DBG] from='client.? 172.21.15.6:0/455653270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:15.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:15 smithi130 bash[51302]: debug 2023-08-28T01:11:15.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:15.294414+0000) 2023-08-28T01:11:16.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:16 smithi130 bash[51302]: debug 2023-08-28T01:11:16.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:16.294769+0000) 2023-08-28T01:11:17.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:16 smithi130 bash[27579]: cluster 2023-08-28T01:11:15.704499+0000 mgr.y (mgr.14901) 5780 : cluster [DBG] pgmap v5787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:17.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:16 smithi006 bash[33918]: cluster 2023-08-28T01:11:15.704499+0000 mgr.y (mgr.14901) 5780 : cluster [DBG] pgmap v5787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:17.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:16 smithi006 bash[27292]: cluster 2023-08-28T01:11:15.704499+0000 mgr.y (mgr.14901) 5780 : cluster [DBG] pgmap v5787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:17.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:17 smithi130 bash[51302]: debug 2023-08-28T01:11:17.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:17.295092+0000) 2023-08-28T01:11:18.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:17 smithi130 bash[27579]: audit 2023-08-28T01:11:17.041951+0000 mon.a (mon.0) 3937 : audit [DBG] from='client.? 172.21.15.6:0/4230192333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:18.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:17 smithi006 bash[33918]: audit 2023-08-28T01:11:17.041951+0000 mon.a (mon.0) 3937 : audit [DBG] from='client.? 172.21.15.6:0/4230192333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:18.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:17 smithi006 bash[27292]: audit 2023-08-28T01:11:17.041951+0000 mon.a (mon.0) 3937 : audit [DBG] from='client.? 172.21.15.6:0/4230192333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:18.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:18 smithi130 bash[51302]: debug 2023-08-28T01:11:18.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:18.295447+0000) 2023-08-28T01:11:19.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:18 smithi130 bash[27579]: cluster 2023-08-28T01:11:17.705205+0000 mgr.y (mgr.14901) 5781 : cluster [DBG] pgmap v5788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:19.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:18 smithi006 bash[27292]: cluster 2023-08-28T01:11:17.705205+0000 mgr.y (mgr.14901) 5781 : cluster [DBG] pgmap v5788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:19.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:18 smithi006 bash[33918]: cluster 2023-08-28T01:11:17.705205+0000 mgr.y (mgr.14901) 5781 : cluster [DBG] pgmap v5788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:19.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:19 smithi130 bash[51302]: debug 2023-08-28T01:11:19.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:19.295783+0000) 2023-08-28T01:11:20.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:19 smithi130 bash[27579]: audit 2023-08-28T01:11:19.456707+0000 mon.a (mon.0) 3938 : audit [DBG] from='client.? 172.21.15.6:0/3845682622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:20.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:19 smithi006 bash[33918]: audit 2023-08-28T01:11:19.456707+0000 mon.a (mon.0) 3938 : audit [DBG] from='client.? 172.21.15.6:0/3845682622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:20.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:19 smithi006 bash[27292]: audit 2023-08-28T01:11:19.456707+0000 mon.a (mon.0) 3938 : audit [DBG] from='client.? 172.21.15.6:0/3845682622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:20.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:20 smithi130 bash[51302]: debug 2023-08-28T01:11:20.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:20.296146+0000) 2023-08-28T01:11:21.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:20 smithi130 bash[27579]: cluster 2023-08-28T01:11:19.706319+0000 mgr.y (mgr.14901) 5782 : cluster [DBG] pgmap v5789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:21.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:11:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:11:20] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:11:21.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:20 smithi006 bash[33918]: cluster 2023-08-28T01:11:19.706319+0000 mgr.y (mgr.14901) 5782 : cluster [DBG] pgmap v5789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:21.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:20 smithi006 bash[27292]: cluster 2023-08-28T01:11:19.706319+0000 mgr.y (mgr.14901) 5782 : cluster [DBG] pgmap v5789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:11:20.945Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:11:20.947Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:21.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:21 smithi130 bash[51302]: debug 2023-08-28T01:11:21.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:21.296520+0000) 2023-08-28T01:11:22.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:22 smithi130 bash[51302]: debug 2023-08-28T01:11:22.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:22.296873+0000) 2023-08-28T01:11:22.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:22 smithi006 bash[27292]: cluster 2023-08-28T01:11:21.706725+0000 mgr.y (mgr.14901) 5783 : cluster [DBG] pgmap v5790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:22.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:22 smithi006 bash[27292]: audit 2023-08-28T01:11:21.869331+0000 mon.c (mon.2) 2252 : audit [DBG] from='client.? 172.21.15.6:0/862024299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:22.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:22 smithi006 bash[33918]: cluster 2023-08-28T01:11:21.706725+0000 mgr.y (mgr.14901) 5783 : cluster [DBG] pgmap v5790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:22.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:22 smithi006 bash[33918]: audit 2023-08-28T01:11:21.869331+0000 mon.c (mon.2) 2252 : audit [DBG] from='client.? 172.21.15.6:0/862024299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:22.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:11:22.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:22.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:11:22.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:22 smithi130 bash[27579]: cluster 2023-08-28T01:11:21.706725+0000 mgr.y (mgr.14901) 5783 : cluster [DBG] pgmap v5790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:22 smithi130 bash[27579]: audit 2023-08-28T01:11:21.869331+0000 mon.c (mon.2) 2252 : audit [DBG] from='client.? 172.21.15.6:0/862024299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:23.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:23 smithi130 bash[51302]: debug 2023-08-28T01:11:23.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:23.297225+0000) 2023-08-28T01:11:24.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:24 smithi130 bash[51302]: debug 2023-08-28T01:11:24.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:24.297585+0000) 2023-08-28T01:11:24.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:11:24.400Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:24.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:11:24.402Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:25.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:24 smithi130 bash[27579]: cluster 2023-08-28T01:11:23.707346+0000 mgr.y (mgr.14901) 5784 : cluster [DBG] pgmap v5791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:25.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:24 smithi130 bash[27579]: audit 2023-08-28T01:11:24.289781+0000 mon.c (mon.2) 2253 : audit [DBG] from='client.? 172.21.15.6:0/1573183907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:25.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:24 smithi006 bash[33918]: cluster 2023-08-28T01:11:23.707346+0000 mgr.y (mgr.14901) 5784 : cluster [DBG] pgmap v5791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:25.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:24 smithi006 bash[33918]: audit 2023-08-28T01:11:24.289781+0000 mon.c (mon.2) 2253 : audit [DBG] from='client.? 172.21.15.6:0/1573183907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:25.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:24 smithi006 bash[27292]: cluster 2023-08-28T01:11:23.707346+0000 mgr.y (mgr.14901) 5784 : cluster [DBG] pgmap v5791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:25.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:24 smithi006 bash[27292]: audit 2023-08-28T01:11:24.289781+0000 mon.c (mon.2) 2253 : audit [DBG] from='client.? 172.21.15.6:0/1573183907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:25.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:25 smithi130 bash[51302]: debug 2023-08-28T01:11:25.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:25.297938+0000) 2023-08-28T01:11:26.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:26 smithi130 bash[51302]: debug 2023-08-28T01:11:26.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:26.298281+0000) 2023-08-28T01:11:27.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:26 smithi130 bash[27579]: cluster 2023-08-28T01:11:25.708489+0000 mgr.y (mgr.14901) 5785 : cluster [DBG] pgmap v5792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:27.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:26 smithi130 bash[27579]: audit 2023-08-28T01:11:26.528824+0000 mon.b (mon.1) 674 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:11:27.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:26 smithi130 bash[27579]: audit 2023-08-28T01:11:26.714222+0000 mon.a (mon.0) 3939 : audit [DBG] from='client.? 172.21.15.6:0/3675236704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:27.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:26 smithi006 bash[33918]: cluster 2023-08-28T01:11:25.708489+0000 mgr.y (mgr.14901) 5785 : cluster [DBG] pgmap v5792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:27.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:26 smithi006 bash[33918]: audit 2023-08-28T01:11:26.528824+0000 mon.b (mon.1) 674 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:11:27.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:26 smithi006 bash[33918]: audit 2023-08-28T01:11:26.714222+0000 mon.a (mon.0) 3939 : audit [DBG] from='client.? 172.21.15.6:0/3675236704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:27.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:26 smithi006 bash[27292]: cluster 2023-08-28T01:11:25.708489+0000 mgr.y (mgr.14901) 5785 : cluster [DBG] pgmap v5792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:27.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:26 smithi006 bash[27292]: audit 2023-08-28T01:11:26.528824+0000 mon.b (mon.1) 674 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:11:27.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:26 smithi006 bash[27292]: audit 2023-08-28T01:11:26.714222+0000 mon.a (mon.0) 3939 : audit [DBG] from='client.? 172.21.15.6:0/3675236704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:27.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:27 smithi130 bash[51302]: debug 2023-08-28T01:11:27.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:27.298601+0000) 2023-08-28T01:11:28.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:28 smithi130 bash[51302]: debug 2023-08-28T01:11:28.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:28.298982+0000) 2023-08-28T01:11:29.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:28 smithi130 bash[27579]: cluster 2023-08-28T01:11:27.709280+0000 mgr.y (mgr.14901) 5786 : cluster [DBG] pgmap v5793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:29.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:28 smithi006 bash[27292]: cluster 2023-08-28T01:11:27.709280+0000 mgr.y (mgr.14901) 5786 : cluster [DBG] pgmap v5793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:29.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:28 smithi006 bash[33918]: cluster 2023-08-28T01:11:27.709280+0000 mgr.y (mgr.14901) 5786 : cluster [DBG] pgmap v5793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:29.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:29 smithi130 bash[51302]: debug 2023-08-28T01:11:29.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:29.299251+0000) 2023-08-28T01:11:30.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:29 smithi130 bash[27579]: audit 2023-08-28T01:11:29.141876+0000 mon.c (mon.2) 2254 : audit [DBG] from='client.? 172.21.15.6:0/2201450718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:30.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:29 smithi006 bash[33918]: audit 2023-08-28T01:11:29.141876+0000 mon.c (mon.2) 2254 : audit [DBG] from='client.? 172.21.15.6:0/2201450718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:30.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:29 smithi006 bash[27292]: audit 2023-08-28T01:11:29.141876+0000 mon.c (mon.2) 2254 : audit [DBG] from='client.? 172.21.15.6:0/2201450718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:30.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:30 smithi130 bash[51302]: debug 2023-08-28T01:11:30.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:30.299602+0000) 2023-08-28T01:11:31.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:30 smithi130 bash[27579]: cluster 2023-08-28T01:11:29.710454+0000 mgr.y (mgr.14901) 5787 : cluster [DBG] pgmap v5794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:11:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:11:30] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:11:31.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:30 smithi006 bash[33918]: cluster 2023-08-28T01:11:29.710454+0000 mgr.y (mgr.14901) 5787 : cluster [DBG] pgmap v5794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:31.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:30 smithi006 bash[27292]: cluster 2023-08-28T01:11:29.710454+0000 mgr.y (mgr.14901) 5787 : cluster [DBG] pgmap v5794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:11:30.946Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:30 smithi006 bash[56298]: level=warn ts=2023-08-28T01:11:30.948Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:31.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:31 smithi130 bash[51302]: debug 2023-08-28T01:11:31.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:31.299879+0000) 2023-08-28T01:11:32.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:31 smithi130 bash[27579]: audit 2023-08-28T01:11:31.566563+0000 mon.a (mon.0) 3940 : audit [DBG] from='client.? 172.21.15.6:0/3261873318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:32.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:31 smithi006 bash[27292]: audit 2023-08-28T01:11:31.566563+0000 mon.a (mon.0) 3940 : audit [DBG] from='client.? 172.21.15.6:0/3261873318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:32.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:31 smithi006 bash[33918]: audit 2023-08-28T01:11:31.566563+0000 mon.a (mon.0) 3940 : audit [DBG] from='client.? 172.21.15.6:0/3261873318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:32.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:32 smithi130 bash[51302]: debug 2023-08-28T01:11:32.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:32.300214+0000) 2023-08-28T01:11:32.799 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:11:32.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:32.799 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:11:32.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:32 smithi130 bash[27579]: cluster 2023-08-28T01:11:31.710951+0000 mgr.y (mgr.14901) 5788 : cluster [DBG] pgmap v5795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:32 smithi130 bash[27579]: audit 2023-08-28T01:11:32.389512+0000 mon.a (mon.0) 3941 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:11:33.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:32 smithi006 bash[27292]: cluster 2023-08-28T01:11:31.710951+0000 mgr.y (mgr.14901) 5788 : cluster [DBG] pgmap v5795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:33.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:32 smithi006 bash[27292]: audit 2023-08-28T01:11:32.389512+0000 mon.a (mon.0) 3941 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:11:33.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:32 smithi006 bash[33918]: cluster 2023-08-28T01:11:31.710951+0000 mgr.y (mgr.14901) 5788 : cluster [DBG] pgmap v5795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:33.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:32 smithi006 bash[33918]: audit 2023-08-28T01:11:32.389512+0000 mon.a (mon.0) 3941 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:11:33.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:33 smithi130 bash[51302]: debug 2023-08-28T01:11:33.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:33.300516+0000) 2023-08-28T01:11:34.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:33 smithi006 bash[27292]: audit 2023-08-28T01:11:32.869420+0000 mon.a (mon.0) 3942 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:11:34.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:33 smithi006 bash[27292]: audit 2023-08-28T01:11:33.139287+0000 mon.b (mon.1) 675 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:11:34.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:33 smithi006 bash[27292]: audit 2023-08-28T01:11:33.141491+0000 mon.b (mon.1) 676 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:11:34.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:33 smithi006 bash[27292]: audit 2023-08-28T01:11:33.149500+0000 mon.a (mon.0) 3943 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:11:34.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:33 smithi006 bash[33918]: audit 2023-08-28T01:11:32.869420+0000 mon.a (mon.0) 3942 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:11:34.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:33 smithi006 bash[33918]: audit 2023-08-28T01:11:33.139287+0000 mon.b (mon.1) 675 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:11:34.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:33 smithi006 bash[33918]: audit 2023-08-28T01:11:33.141491+0000 mon.b (mon.1) 676 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:11:34.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:33 smithi006 bash[33918]: audit 2023-08-28T01:11:33.149500+0000 mon.a (mon.0) 3943 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:11:34.300 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:33 smithi130 bash[27579]: audit 2023-08-28T01:11:32.869420+0000 mon.a (mon.0) 3942 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:11:34.301 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:33 smithi130 bash[27579]: audit 2023-08-28T01:11:33.139287+0000 mon.b (mon.1) 675 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:11:34.301 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:33 smithi130 bash[27579]: audit 2023-08-28T01:11:33.141491+0000 mon.b (mon.1) 676 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:11:34.301 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:33 smithi130 bash[27579]: audit 2023-08-28T01:11:33.149500+0000 mon.a (mon.0) 3943 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:11:34.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:34 smithi130 bash[51302]: debug 2023-08-28T01:11:34.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:34.300845+0000) 2023-08-28T01:11:34.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:11:34.400Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:34.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:11:34.403Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:35.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:34 smithi006 bash[27292]: cluster 2023-08-28T01:11:33.711557+0000 mgr.y (mgr.14901) 5789 : cluster [DBG] pgmap v5796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:35.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:34 smithi006 bash[27292]: audit 2023-08-28T01:11:33.978550+0000 mon.a (mon.0) 3944 : audit [DBG] from='client.? 172.21.15.6:0/1826693004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:35.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:34 smithi006 bash[33918]: cluster 2023-08-28T01:11:33.711557+0000 mgr.y (mgr.14901) 5789 : cluster [DBG] pgmap v5796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:35.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:34 smithi006 bash[33918]: audit 2023-08-28T01:11:33.978550+0000 mon.a (mon.0) 3944 : audit [DBG] from='client.? 172.21.15.6:0/1826693004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:35.301 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:34 smithi130 bash[27579]: cluster 2023-08-28T01:11:33.711557+0000 mgr.y (mgr.14901) 5789 : cluster [DBG] pgmap v5796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:35.301 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:34 smithi130 bash[27579]: audit 2023-08-28T01:11:33.978550+0000 mon.a (mon.0) 3944 : audit [DBG] from='client.? 172.21.15.6:0/1826693004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:35.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:35 smithi130 bash[51302]: debug 2023-08-28T01:11:35.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:35.301211+0000) 2023-08-28T01:11:36.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:36 smithi130 bash[51302]: debug 2023-08-28T01:11:36.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:36.301544+0000) 2023-08-28T01:11:37.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:36 smithi006 bash[27292]: cluster 2023-08-28T01:11:35.712647+0000 mgr.y (mgr.14901) 5790 : cluster [DBG] pgmap v5797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:37.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:36 smithi006 bash[27292]: audit 2023-08-28T01:11:36.389531+0000 mon.a (mon.0) 3945 : audit [DBG] from='client.? 172.21.15.6:0/3994431668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:37.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:36 smithi006 bash[33918]: cluster 2023-08-28T01:11:35.712647+0000 mgr.y (mgr.14901) 5790 : cluster [DBG] pgmap v5797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:37.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:36 smithi006 bash[33918]: audit 2023-08-28T01:11:36.389531+0000 mon.a (mon.0) 3945 : audit [DBG] from='client.? 172.21.15.6:0/3994431668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:37.301 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:36 smithi130 bash[27579]: cluster 2023-08-28T01:11:35.712647+0000 mgr.y (mgr.14901) 5790 : cluster [DBG] pgmap v5797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:37.302 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:36 smithi130 bash[27579]: audit 2023-08-28T01:11:36.389531+0000 mon.a (mon.0) 3945 : audit [DBG] from='client.? 172.21.15.6:0/3994431668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:37.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:37 smithi130 bash[51302]: debug 2023-08-28T01:11:37.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:37.301883+0000) 2023-08-28T01:11:38.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:38 smithi130 bash[51302]: debug 2023-08-28T01:11:38.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:38.302200+0000) 2023-08-28T01:11:39.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:38 smithi006 bash[33918]: cluster 2023-08-28T01:11:37.713333+0000 mgr.y (mgr.14901) 5791 : cluster [DBG] pgmap v5798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:39.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:38 smithi006 bash[33918]: audit 2023-08-28T01:11:38.799358+0000 mon.a (mon.0) 3946 : audit [DBG] from='client.? 172.21.15.6:0/3201421741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:39.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:38 smithi006 bash[27292]: cluster 2023-08-28T01:11:37.713333+0000 mgr.y (mgr.14901) 5791 : cluster [DBG] pgmap v5798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:39.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:38 smithi006 bash[27292]: audit 2023-08-28T01:11:38.799358+0000 mon.a (mon.0) 3946 : audit [DBG] from='client.? 172.21.15.6:0/3201421741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:39.302 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:38 smithi130 bash[27579]: cluster 2023-08-28T01:11:37.713333+0000 mgr.y (mgr.14901) 5791 : cluster [DBG] pgmap v5798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:39.302 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:38 smithi130 bash[27579]: audit 2023-08-28T01:11:38.799358+0000 mon.a (mon.0) 3946 : audit [DBG] from='client.? 172.21.15.6:0/3201421741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:39.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:39 smithi130 bash[51302]: debug 2023-08-28T01:11:39.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:39.302531+0000) 2023-08-28T01:11:40.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:40 smithi130 bash[51302]: debug 2023-08-28T01:11:40.302+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:40.302928+0000) 2023-08-28T01:11:41.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:11:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:11:40] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:11:41.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:11:40.946Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:41.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:11:40.948Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:41.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:40 smithi006 bash[33918]: cluster 2023-08-28T01:11:39.714333+0000 mgr.y (mgr.14901) 5792 : cluster [DBG] pgmap v5799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:41.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:40 smithi006 bash[27292]: cluster 2023-08-28T01:11:39.714333+0000 mgr.y (mgr.14901) 5792 : cluster [DBG] pgmap v5799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:41.303 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:40 smithi130 bash[27579]: cluster 2023-08-28T01:11:39.714333+0000 mgr.y (mgr.14901) 5792 : cluster [DBG] pgmap v5799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:41.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:41 smithi130 bash[51302]: debug 2023-08-28T01:11:41.302+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:41.303196+0000) 2023-08-28T01:11:42.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:41 smithi006 bash[33918]: audit 2023-08-28T01:11:41.215693+0000 mon.a (mon.0) 3947 : audit [DBG] from='client.? 172.21.15.6:0/1530225091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:42.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:41 smithi006 bash[27292]: audit 2023-08-28T01:11:41.215693+0000 mon.a (mon.0) 3947 : audit [DBG] from='client.? 172.21.15.6:0/1530225091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:42.303 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:41 smithi130 bash[27579]: audit 2023-08-28T01:11:41.215693+0000 mon.a (mon.0) 3947 : audit [DBG] from='client.? 172.21.15.6:0/1530225091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:42.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:42 smithi130 bash[51302]: debug 2023-08-28T01:11:42.302+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:42.303499+0000) 2023-08-28T01:11:42.892 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:11:42.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:42.893 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:11:42.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:43.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:42 smithi006 bash[33918]: cluster 2023-08-28T01:11:41.714993+0000 mgr.y (mgr.14901) 5793 : cluster [DBG] pgmap v5800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:43.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:42 smithi006 bash[27292]: cluster 2023-08-28T01:11:41.714993+0000 mgr.y (mgr.14901) 5793 : cluster [DBG] pgmap v5800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:43.303 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:42 smithi130 bash[27579]: cluster 2023-08-28T01:11:41.714993+0000 mgr.y (mgr.14901) 5793 : cluster [DBG] pgmap v5800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:43.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:43 smithi130 bash[51302]: debug 2023-08-28T01:11:43.302+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:43.303881+0000) 2023-08-28T01:11:44.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:43 smithi006 bash[27292]: audit 2023-08-28T01:11:43.637223+0000 mon.c (mon.2) 2255 : audit [DBG] from='client.? 172.21.15.6:0/2633097501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:44.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:43 smithi006 bash[33918]: audit 2023-08-28T01:11:43.637223+0000 mon.c (mon.2) 2255 : audit [DBG] from='client.? 172.21.15.6:0/2633097501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:44.304 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:43 smithi130 bash[27579]: audit 2023-08-28T01:11:43.637223+0000 mon.c (mon.2) 2255 : audit [DBG] from='client.? 172.21.15.6:0/2633097501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:44.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:44 smithi130 bash[51302]: debug 2023-08-28T01:11:44.302+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:44.304199+0000) 2023-08-28T01:11:44.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:11:44.402Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:44.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:11:44.404Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:45.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:44 smithi006 bash[27292]: cluster 2023-08-28T01:11:43.715645+0000 mgr.y (mgr.14901) 5794 : cluster [DBG] pgmap v5801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:45.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:44 smithi006 bash[33918]: cluster 2023-08-28T01:11:43.715645+0000 mgr.y (mgr.14901) 5794 : cluster [DBG] pgmap v5801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:45.304 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:44 smithi130 bash[27579]: cluster 2023-08-28T01:11:43.715645+0000 mgr.y (mgr.14901) 5794 : cluster [DBG] pgmap v5801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:45.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:45 smithi130 bash[51302]: debug 2023-08-28T01:11:45.302+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:45.304530+0000) 2023-08-28T01:11:46.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:46 smithi130 bash[51302]: debug 2023-08-28T01:11:46.302+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:46.304842+0000) 2023-08-28T01:11:47.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:46 smithi006 bash[33918]: cluster 2023-08-28T01:11:45.716780+0000 mgr.y (mgr.14901) 5795 : cluster [DBG] pgmap v5802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:47.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:46 smithi006 bash[33918]: audit 2023-08-28T01:11:46.050935+0000 mon.c (mon.2) 2256 : audit [DBG] from='client.? 172.21.15.6:0/752379666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:47.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:46 smithi006 bash[27292]: cluster 2023-08-28T01:11:45.716780+0000 mgr.y (mgr.14901) 5795 : cluster [DBG] pgmap v5802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:47.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:46 smithi006 bash[27292]: audit 2023-08-28T01:11:46.050935+0000 mon.c (mon.2) 2256 : audit [DBG] from='client.? 172.21.15.6:0/752379666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:47.305 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:46 smithi130 bash[27579]: cluster 2023-08-28T01:11:45.716780+0000 mgr.y (mgr.14901) 5795 : cluster [DBG] pgmap v5802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:47.305 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:46 smithi130 bash[27579]: audit 2023-08-28T01:11:46.050935+0000 mon.c (mon.2) 2256 : audit [DBG] from='client.? 172.21.15.6:0/752379666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:47.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:47 smithi130 bash[51302]: debug 2023-08-28T01:11:47.302+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:47.305154+0000) 2023-08-28T01:11:48.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:48 smithi130 bash[51302]: debug 2023-08-28T01:11:48.302+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:48.305517+0000) 2023-08-28T01:11:49.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:48 smithi006 bash[27292]: cluster 2023-08-28T01:11:47.717438+0000 mgr.y (mgr.14901) 5796 : cluster [DBG] pgmap v5803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:49.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:48 smithi006 bash[27292]: audit 2023-08-28T01:11:48.465084+0000 mon.a (mon.0) 3948 : audit [DBG] from='client.? 172.21.15.6:0/2972360759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:49.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:48 smithi006 bash[33918]: cluster 2023-08-28T01:11:47.717438+0000 mgr.y (mgr.14901) 5796 : cluster [DBG] pgmap v5803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:49.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:48 smithi006 bash[33918]: audit 2023-08-28T01:11:48.465084+0000 mon.a (mon.0) 3948 : audit [DBG] from='client.? 172.21.15.6:0/2972360759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:49.305 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:48 smithi130 bash[27579]: cluster 2023-08-28T01:11:47.717438+0000 mgr.y (mgr.14901) 5796 : cluster [DBG] pgmap v5803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:49.306 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:48 smithi130 bash[27579]: audit 2023-08-28T01:11:48.465084+0000 mon.a (mon.0) 3948 : audit [DBG] from='client.? 172.21.15.6:0/2972360759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:49.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:49 smithi130 bash[51302]: debug 2023-08-28T01:11:49.302+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:49.305868+0000) 2023-08-28T01:11:50.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:50 smithi130 bash[51302]: debug 2023-08-28T01:11:50.306+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:50.306251+0000) 2023-08-28T01:11:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:11:50.948Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:11:50.949Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:50 smithi006 bash[27292]: cluster 2023-08-28T01:11:49.718448+0000 mgr.y (mgr.14901) 5797 : cluster [DBG] pgmap v5804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:50 smithi006 bash[27292]: audit 2023-08-28T01:11:50.889452+0000 mon.c (mon.2) 2257 : audit [DBG] from='client.? 172.21.15.6:0/4270311554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:51.218 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:11:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:11:50] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:11:51.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:50 smithi006 bash[33918]: cluster 2023-08-28T01:11:49.718448+0000 mgr.y (mgr.14901) 5797 : cluster [DBG] pgmap v5804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:51.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:50 smithi006 bash[33918]: audit 2023-08-28T01:11:50.889452+0000 mon.c (mon.2) 2257 : audit [DBG] from='client.? 172.21.15.6:0/4270311554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:51.306 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:50 smithi130 bash[27579]: cluster 2023-08-28T01:11:49.718448+0000 mgr.y (mgr.14901) 5797 : cluster [DBG] pgmap v5804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:51.306 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:50 smithi130 bash[27579]: audit 2023-08-28T01:11:50.889452+0000 mon.c (mon.2) 2257 : audit [DBG] from='client.? 172.21.15.6:0/4270311554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:51.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:51 smithi130 bash[51302]: debug 2023-08-28T01:11:51.302+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:51.306584+0000) 2023-08-28T01:11:52.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:52 smithi130 bash[51302]: debug 2023-08-28T01:11:52.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:52.306884+0000) 2023-08-28T01:11:52.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:52 smithi006 bash[33918]: cluster 2023-08-28T01:11:51.719116+0000 mgr.y (mgr.14901) 5798 : cluster [DBG] pgmap v5805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:52 smithi006 bash[27292]: cluster 2023-08-28T01:11:51.719116+0000 mgr.y (mgr.14901) 5798 : cluster [DBG] pgmap v5805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:11:52.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:11:52.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:52 smithi130 bash[27579]: cluster 2023-08-28T01:11:51.719116+0000 mgr.y (mgr.14901) 5798 : cluster [DBG] pgmap v5805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:53.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:53 smithi130 bash[51302]: debug 2023-08-28T01:11:53.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:53.307212+0000) 2023-08-28T01:11:53.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:53 smithi006 bash[33918]: audit 2023-08-28T01:11:53.311134+0000 mon.a (mon.0) 3949 : audit [DBG] from='client.? 172.21.15.6:0/2955922928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:53.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:53 smithi006 bash[27292]: audit 2023-08-28T01:11:53.311134+0000 mon.a (mon.0) 3949 : audit [DBG] from='client.? 172.21.15.6:0/2955922928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:54.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:53 smithi130 bash[27579]: audit 2023-08-28T01:11:53.311134+0000 mon.a (mon.0) 3949 : audit [DBG] from='client.? 172.21.15.6:0/2955922928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:54.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:54 smithi130 bash[51302]: debug 2023-08-28T01:11:54.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:54.307600+0000) 2023-08-28T01:11:54.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:54 smithi006 bash[33918]: cluster 2023-08-28T01:11:53.719695+0000 mgr.y (mgr.14901) 5799 : cluster [DBG] pgmap v5806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:54.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:11:54.403Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:54.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:11:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:11:54.405Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:11:54.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:54 smithi006 bash[27292]: cluster 2023-08-28T01:11:53.719695+0000 mgr.y (mgr.14901) 5799 : cluster [DBG] pgmap v5806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:55.120 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:54 smithi130 bash[27579]: cluster 2023-08-28T01:11:53.719695+0000 mgr.y (mgr.14901) 5799 : cluster [DBG] pgmap v5806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:55.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:55 smithi130 bash[51302]: debug 2023-08-28T01:11:55.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:55.307936+0000) 2023-08-28T01:11:56.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:55 smithi130 bash[27579]: audit 2023-08-28T01:11:55.727551+0000 mon.c (mon.2) 2258 : audit [DBG] from='client.? 172.21.15.6:0/752892440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:56.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:55 smithi006 bash[27292]: audit 2023-08-28T01:11:55.727551+0000 mon.c (mon.2) 2258 : audit [DBG] from='client.? 172.21.15.6:0/752892440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:56.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:55 smithi006 bash[33918]: audit 2023-08-28T01:11:55.727551+0000 mon.c (mon.2) 2258 : audit [DBG] from='client.? 172.21.15.6:0/752892440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:56.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:56 smithi130 bash[51302]: debug 2023-08-28T01:11:56.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:56.308269+0000) 2023-08-28T01:11:57.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:56 smithi130 bash[27579]: cluster 2023-08-28T01:11:55.720800+0000 mgr.y (mgr.14901) 5800 : cluster [DBG] pgmap v5807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:57.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:56 smithi006 bash[27292]: cluster 2023-08-28T01:11:55.720800+0000 mgr.y (mgr.14901) 5800 : cluster [DBG] pgmap v5807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:57.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:56 smithi006 bash[33918]: cluster 2023-08-28T01:11:55.720800+0000 mgr.y (mgr.14901) 5800 : cluster [DBG] pgmap v5807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:57.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:57 smithi130 bash[51302]: debug 2023-08-28T01:11:57.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:57.308587+0000) 2023-08-28T01:11:58.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:58 smithi130 bash[51302]: debug 2023-08-28T01:11:58.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:58.308930+0000) 2023-08-28T01:11:59.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:58 smithi130 bash[27579]: cluster 2023-08-28T01:11:57.721404+0000 mgr.y (mgr.14901) 5801 : cluster [DBG] pgmap v5808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:59.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:11:58 smithi130 bash[27579]: audit 2023-08-28T01:11:58.141802+0000 mon.a (mon.0) 3950 : audit [DBG] from='client.? 172.21.15.6:0/212607074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:59.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:58 smithi006 bash[33918]: cluster 2023-08-28T01:11:57.721404+0000 mgr.y (mgr.14901) 5801 : cluster [DBG] pgmap v5808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:59.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:11:58 smithi006 bash[33918]: audit 2023-08-28T01:11:58.141802+0000 mon.a (mon.0) 3950 : audit [DBG] from='client.? 172.21.15.6:0/212607074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:58 smithi006 bash[27292]: cluster 2023-08-28T01:11:57.721404+0000 mgr.y (mgr.14901) 5801 : cluster [DBG] pgmap v5808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:11:59.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:11:58 smithi006 bash[27292]: audit 2023-08-28T01:11:58.141802+0000 mon.a (mon.0) 3950 : audit [DBG] from='client.? 172.21.15.6:0/212607074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:59.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:11:59 smithi130 bash[51302]: debug 2023-08-28T01:11:59.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:59.309315+0000) 2023-08-28T01:12:00.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:00 smithi130 bash[51302]: debug 2023-08-28T01:12:00.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:00.309667+0000) 2023-08-28T01:12:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:00 smithi130 bash[27579]: cluster 2023-08-28T01:11:59.722473+0000 mgr.y (mgr.14901) 5802 : cluster [DBG] pgmap v5809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:00 smithi130 bash[27579]: audit 2023-08-28T01:12:00.558097+0000 mon.c (mon.2) 2259 : audit [DBG] from='client.? 172.21.15.6:0/2006283141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:12:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:12:00] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:12:01.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:00 smithi006 bash[27292]: cluster 2023-08-28T01:11:59.722473+0000 mgr.y (mgr.14901) 5802 : cluster [DBG] pgmap v5809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:01.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:00 smithi006 bash[27292]: audit 2023-08-28T01:12:00.558097+0000 mon.c (mon.2) 2259 : audit [DBG] from='client.? 172.21.15.6:0/2006283141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:01.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:00 smithi006 bash[33918]: cluster 2023-08-28T01:11:59.722473+0000 mgr.y (mgr.14901) 5802 : cluster [DBG] pgmap v5809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:01.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:00 smithi006 bash[33918]: audit 2023-08-28T01:12:00.558097+0000 mon.c (mon.2) 2259 : audit [DBG] from='client.? 172.21.15.6:0/2006283141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:12:00.949Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:12:00.950Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:01.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:01 smithi130 bash[51302]: debug 2023-08-28T01:12:01.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:01.310014+0000) 2023-08-28T01:12:02.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:02 smithi130 bash[51302]: debug 2023-08-28T01:12:02.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:02.310364+0000) 2023-08-28T01:12:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:02 smithi006 bash[27292]: cluster 2023-08-28T01:12:01.723179+0000 mgr.y (mgr.14901) 5803 : cluster [DBG] pgmap v5810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:02.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:02 smithi006 bash[33918]: cluster 2023-08-28T01:12:01.723179+0000 mgr.y (mgr.14901) 5803 : cluster [DBG] pgmap v5810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:02.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:12:02.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:02.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:12:02.523Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:02 smithi130 bash[27579]: cluster 2023-08-28T01:12:01.723179+0000 mgr.y (mgr.14901) 5803 : cluster [DBG] pgmap v5810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:03.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:03 smithi130 bash[51302]: debug 2023-08-28T01:12:03.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:03.310695+0000) 2023-08-28T01:12:03.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:03 smithi006 bash[33918]: audit 2023-08-28T01:12:02.969978+0000 mon.c (mon.2) 2260 : audit [DBG] from='client.? 172.21.15.6:0/1537925574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:03.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:03 smithi006 bash[27292]: audit 2023-08-28T01:12:02.969978+0000 mon.c (mon.2) 2260 : audit [DBG] from='client.? 172.21.15.6:0/1537925574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:04.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:03 smithi130 bash[27579]: audit 2023-08-28T01:12:02.969978+0000 mon.c (mon.2) 2260 : audit [DBG] from='client.? 172.21.15.6:0/1537925574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:04.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:04 smithi130 bash[51302]: debug 2023-08-28T01:12:04.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:04.311026+0000) 2023-08-28T01:12:04.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:04 smithi006 bash[27292]: cluster 2023-08-28T01:12:03.723876+0000 mgr.y (mgr.14901) 5804 : cluster [DBG] pgmap v5811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:04.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:04 smithi006 bash[33918]: cluster 2023-08-28T01:12:03.723876+0000 mgr.y (mgr.14901) 5804 : cluster [DBG] pgmap v5811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:12:04.404Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:12:04.406Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:05.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:04 smithi130 bash[27579]: cluster 2023-08-28T01:12:03.723876+0000 mgr.y (mgr.14901) 5804 : cluster [DBG] pgmap v5811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:05.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:05 smithi130 bash[51302]: debug 2023-08-28T01:12:05.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:05.311250+0000) 2023-08-28T01:12:05.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:05 smithi006 bash[27292]: audit 2023-08-28T01:12:05.381747+0000 mon.a (mon.0) 3951 : audit [DBG] from='client.? 172.21.15.6:0/1859842103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:05.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:05 smithi006 bash[33918]: audit 2023-08-28T01:12:05.381747+0000 mon.a (mon.0) 3951 : audit [DBG] from='client.? 172.21.15.6:0/1859842103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:06.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:05 smithi130 bash[27579]: audit 2023-08-28T01:12:05.381747+0000 mon.a (mon.0) 3951 : audit [DBG] from='client.? 172.21.15.6:0/1859842103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:06.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:06 smithi130 bash[51302]: debug 2023-08-28T01:12:06.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:06.311570+0000) 2023-08-28T01:12:06.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:06 smithi006 bash[27292]: cluster 2023-08-28T01:12:05.725081+0000 mgr.y (mgr.14901) 5805 : cluster [DBG] pgmap v5812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:06.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:06 smithi006 bash[33918]: cluster 2023-08-28T01:12:05.725081+0000 mgr.y (mgr.14901) 5805 : cluster [DBG] pgmap v5812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:07.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:06 smithi130 bash[27579]: cluster 2023-08-28T01:12:05.725081+0000 mgr.y (mgr.14901) 5805 : cluster [DBG] pgmap v5812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:07.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:07 smithi130 bash[51302]: debug 2023-08-28T01:12:07.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:07.311917+0000) 2023-08-28T01:12:08.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:08 smithi130 bash[51302]: debug 2023-08-28T01:12:08.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:08.312265+0000) 2023-08-28T01:12:09.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:08 smithi130 bash[27579]: cluster 2023-08-28T01:12:07.725712+0000 mgr.y (mgr.14901) 5806 : cluster [DBG] pgmap v5813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:09.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:08 smithi130 bash[27579]: audit 2023-08-28T01:12:07.796138+0000 mon.c (mon.2) 2261 : audit [DBG] from='client.? 172.21.15.6:0/3955636972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:09.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:08 smithi006 bash[27292]: cluster 2023-08-28T01:12:07.725712+0000 mgr.y (mgr.14901) 5806 : cluster [DBG] pgmap v5813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:09.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:08 smithi006 bash[27292]: audit 2023-08-28T01:12:07.796138+0000 mon.c (mon.2) 2261 : audit [DBG] from='client.? 172.21.15.6:0/3955636972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:09.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:08 smithi006 bash[33918]: cluster 2023-08-28T01:12:07.725712+0000 mgr.y (mgr.14901) 5806 : cluster [DBG] pgmap v5813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:09.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:08 smithi006 bash[33918]: audit 2023-08-28T01:12:07.796138+0000 mon.c (mon.2) 2261 : audit [DBG] from='client.? 172.21.15.6:0/3955636972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:09.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:09 smithi130 bash[51302]: debug 2023-08-28T01:12:09.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:09.312621+0000) 2023-08-28T01:12:10.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:10 smithi130 bash[51302]: debug 2023-08-28T01:12:10.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:10.313003+0000) 2023-08-28T01:12:11.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:10 smithi130 bash[27579]: cluster 2023-08-28T01:12:09.726828+0000 mgr.y (mgr.14901) 5807 : cluster [DBG] pgmap v5814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:11.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:10 smithi130 bash[27579]: audit 2023-08-28T01:12:10.203359+0000 mon.a (mon.0) 3952 : audit [DBG] from='client.? 172.21.15.6:0/2290864983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:12:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:12:10] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:12:11.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:10 smithi006 bash[27292]: cluster 2023-08-28T01:12:09.726828+0000 mgr.y (mgr.14901) 5807 : cluster [DBG] pgmap v5814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:11.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:10 smithi006 bash[27292]: audit 2023-08-28T01:12:10.203359+0000 mon.a (mon.0) 3952 : audit [DBG] from='client.? 172.21.15.6:0/2290864983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:11.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:12:10.949Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:12:10.951Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:10 smithi006 bash[33918]: cluster 2023-08-28T01:12:09.726828+0000 mgr.y (mgr.14901) 5807 : cluster [DBG] pgmap v5814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:10 smithi006 bash[33918]: audit 2023-08-28T01:12:10.203359+0000 mon.a (mon.0) 3952 : audit [DBG] from='client.? 172.21.15.6:0/2290864983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:11.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:11 smithi130 bash[51302]: debug 2023-08-28T01:12:11.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:11.313338+0000) 2023-08-28T01:12:12.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:12 smithi130 bash[51302]: debug 2023-08-28T01:12:12.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:12.313614+0000) 2023-08-28T01:12:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:12 smithi006 bash[33918]: cluster 2023-08-28T01:12:11.727504+0000 mgr.y (mgr.14901) 5808 : cluster [DBG] pgmap v5815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:12 smithi006 bash[33918]: audit 2023-08-28T01:12:12.609665+0000 mon.a (mon.0) 3953 : audit [DBG] from='client.? 172.21.15.6:0/976105385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:12 smithi006 bash[27292]: cluster 2023-08-28T01:12:11.727504+0000 mgr.y (mgr.14901) 5808 : cluster [DBG] pgmap v5815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:12 smithi006 bash[27292]: audit 2023-08-28T01:12:12.609665+0000 mon.a (mon.0) 3953 : audit [DBG] from='client.? 172.21.15.6:0/976105385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:12:12.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:12:12.523Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:12 smithi130 bash[27579]: cluster 2023-08-28T01:12:11.727504+0000 mgr.y (mgr.14901) 5808 : cluster [DBG] pgmap v5815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:12 smithi130 bash[27579]: audit 2023-08-28T01:12:12.609665+0000 mon.a (mon.0) 3953 : audit [DBG] from='client.? 172.21.15.6:0/976105385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:13.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:13 smithi130 bash[51302]: debug 2023-08-28T01:12:13.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:13.313930+0000) 2023-08-28T01:12:14.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:14 smithi130 bash[51302]: debug 2023-08-28T01:12:14.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:14.314349+0000) 2023-08-28T01:12:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:12:14.404Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:15.004 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:12:14.406Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:15.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:14 smithi130 bash[27579]: cluster 2023-08-28T01:12:13.728139+0000 mgr.y (mgr.14901) 5809 : cluster [DBG] pgmap v5816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:15.838 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:14 smithi006 bash[33918]: cluster 2023-08-28T01:12:13.728139+0000 mgr.y (mgr.14901) 5809 : cluster [DBG] pgmap v5816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:15.839 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:15 smithi130 bash[51302]: debug 2023-08-28T01:12:15.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:15.314682+0000) 2023-08-28T01:12:15.839 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:14 smithi006 bash[27292]: cluster 2023-08-28T01:12:13.728139+0000 mgr.y (mgr.14901) 5809 : cluster [DBG] pgmap v5816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:16.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:15 smithi130 bash[27579]: audit 2023-08-28T01:12:15.036707+0000 mon.c (mon.2) 2262 : audit [DBG] from='client.? 172.21.15.6:0/225148067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:17.019 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:15 smithi006 bash[33918]: audit 2023-08-28T01:12:15.036707+0000 mon.c (mon.2) 2262 : audit [DBG] from='client.? 172.21.15.6:0/225148067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:17.020 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:16 smithi130 bash[51302]: debug 2023-08-28T01:12:16.313+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:16.315050+0000) 2023-08-28T01:12:17.020 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:15 smithi006 bash[27292]: audit 2023-08-28T01:12:15.036707+0000 mon.c (mon.2) 2262 : audit [DBG] from='client.? 172.21.15.6:0/225148067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:17.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:16 smithi130 bash[27579]: cluster 2023-08-28T01:12:15.729468+0000 mgr.y (mgr.14901) 5810 : cluster [DBG] pgmap v5817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:17.276 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:16 smithi006 bash[27292]: cluster 2023-08-28T01:12:15.729468+0000 mgr.y (mgr.14901) 5810 : cluster [DBG] pgmap v5817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:17.276 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:16 smithi006 bash[33918]: cluster 2023-08-28T01:12:15.729468+0000 mgr.y (mgr.14901) 5810 : cluster [DBG] pgmap v5817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:17.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:17 smithi130 bash[51302]: debug 2023-08-28T01:12:17.313+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:17.315313+0000) 2023-08-28T01:12:18.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:17 smithi130 bash[27579]: audit 2023-08-28T01:12:17.458225+0000 mon.a (mon.0) 3954 : audit [DBG] from='client.? 172.21.15.6:0/4073301453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:18.288 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:17 smithi006 bash[33918]: audit 2023-08-28T01:12:17.458225+0000 mon.a (mon.0) 3954 : audit [DBG] from='client.? 172.21.15.6:0/4073301453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:18.288 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:17 smithi006 bash[27292]: audit 2023-08-28T01:12:17.458225+0000 mon.a (mon.0) 3954 : audit [DBG] from='client.? 172.21.15.6:0/4073301453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:18.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:18 smithi130 bash[51302]: debug 2023-08-28T01:12:18.313+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:18.315684+0000) 2023-08-28T01:12:19.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:18 smithi130 bash[27579]: cluster 2023-08-28T01:12:17.730200+0000 mgr.y (mgr.14901) 5811 : cluster [DBG] pgmap v5818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:19.311 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:18 smithi006 bash[27292]: cluster 2023-08-28T01:12:17.730200+0000 mgr.y (mgr.14901) 5811 : cluster [DBG] pgmap v5818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:19.312 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:18 smithi006 bash[33918]: cluster 2023-08-28T01:12:17.730200+0000 mgr.y (mgr.14901) 5811 : cluster [DBG] pgmap v5818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:19.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:19 smithi130 bash[51302]: debug 2023-08-28T01:12:19.313+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:19.316036+0000) 2023-08-28T01:12:20.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:20 smithi130 bash[51302]: debug 2023-08-28T01:12:20.313+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:20.316386+0000) 2023-08-28T01:12:21.274 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:12:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:12:20] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:12:21.274 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:20 smithi130 bash[27579]: cluster 2023-08-28T01:12:19.731287+0000 mgr.y (mgr.14901) 5812 : cluster [DBG] pgmap v5819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:21.274 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:20 smithi130 bash[27579]: audit 2023-08-28T01:12:19.868586+0000 mon.a (mon.0) 3955 : audit [DBG] from='client.? 172.21.15.6:0/1792822512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:21.274 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:20 smithi006 bash[27292]: cluster 2023-08-28T01:12:19.731287+0000 mgr.y (mgr.14901) 5812 : cluster [DBG] pgmap v5819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:21.274 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:20 smithi006 bash[27292]: audit 2023-08-28T01:12:19.868586+0000 mon.a (mon.0) 3955 : audit [DBG] from='client.? 172.21.15.6:0/1792822512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:21.275 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:12:20.950Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:21.275 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:12:20.951Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:21.275 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:20 smithi006 bash[33918]: cluster 2023-08-28T01:12:19.731287+0000 mgr.y (mgr.14901) 5812 : cluster [DBG] pgmap v5819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:21.275 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:20 smithi006 bash[33918]: audit 2023-08-28T01:12:19.868586+0000 mon.a (mon.0) 3955 : audit [DBG] from='client.? 172.21.15.6:0/1792822512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:21.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:21 smithi130 bash[51302]: debug 2023-08-28T01:12:21.313+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:21.316704+0000) 2023-08-28T01:12:22.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:22 smithi130 bash[51302]: debug 2023-08-28T01:12:22.313+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:22.316965+0000) 2023-08-28T01:12:23.003 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:22 smithi006 bash[33918]: cluster 2023-08-28T01:12:21.731951+0000 mgr.y (mgr.14901) 5813 : cluster [DBG] pgmap v5820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:23.003 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:22 smithi006 bash[33918]: audit 2023-08-28T01:12:22.282974+0000 mon.c (mon.2) 2263 : audit [DBG] from='client.? 172.21.15.6:0/4186373344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:23.003 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:12:22.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:23.004 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:12:22.524Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:23.004 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:22 smithi006 bash[27292]: cluster 2023-08-28T01:12:21.731951+0000 mgr.y (mgr.14901) 5813 : cluster [DBG] pgmap v5820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:23.004 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:22 smithi006 bash[27292]: audit 2023-08-28T01:12:22.282974+0000 mon.c (mon.2) 2263 : audit [DBG] from='client.? 172.21.15.6:0/4186373344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:23.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:22 smithi130 bash[27579]: cluster 2023-08-28T01:12:21.731951+0000 mgr.y (mgr.14901) 5813 : cluster [DBG] pgmap v5820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:23.165 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:22 smithi130 bash[27579]: audit 2023-08-28T01:12:22.282974+0000 mon.c (mon.2) 2263 : audit [DBG] from='client.? 172.21.15.6:0/4186373344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:23.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:23 smithi130 bash[51302]: debug 2023-08-28T01:12:23.313+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:23.317324+0000) 2023-08-28T01:12:24.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:24 smithi130 bash[51302]: debug 2023-08-28T01:12:24.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:24.317658+0000) 2023-08-28T01:12:25.120 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:12:24.405Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:25.121 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:12:24.407Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:25.121 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:24 smithi130 bash[27579]: cluster 2023-08-28T01:12:23.732741+0000 mgr.y (mgr.14901) 5814 : cluster [DBG] pgmap v5821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:25.270 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:24 smithi130 bash[27579]: audit 2023-08-28T01:12:24.696339+0000 mon.a (mon.0) 3956 : audit [DBG] from='client.? 172.21.15.6:0/811740833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:25.271 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:24 smithi006 bash[33918]: cluster 2023-08-28T01:12:23.732741+0000 mgr.y (mgr.14901) 5814 : cluster [DBG] pgmap v5821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:25.271 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:24 smithi006 bash[33918]: audit 2023-08-28T01:12:24.696339+0000 mon.a (mon.0) 3956 : audit [DBG] from='client.? 172.21.15.6:0/811740833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:25.271 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:24 smithi006 bash[27292]: cluster 2023-08-28T01:12:23.732741+0000 mgr.y (mgr.14901) 5814 : cluster [DBG] pgmap v5821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:25.272 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:24 smithi006 bash[27292]: audit 2023-08-28T01:12:24.696339+0000 mon.a (mon.0) 3956 : audit [DBG] from='client.? 172.21.15.6:0/811740833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:25.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:25 smithi130 bash[51302]: debug 2023-08-28T01:12:25.313+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:25.318049+0000) 2023-08-28T01:12:26.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:26 smithi130 bash[51302]: debug 2023-08-28T01:12:26.313+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:26.318400+0000) 2023-08-28T01:12:27.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:26 smithi130 bash[27579]: cluster 2023-08-28T01:12:25.734061+0000 mgr.y (mgr.14901) 5815 : cluster [DBG] pgmap v5822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:27.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:26 smithi006 bash[33918]: cluster 2023-08-28T01:12:25.734061+0000 mgr.y (mgr.14901) 5815 : cluster [DBG] pgmap v5822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:27.230 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:26 smithi006 bash[27292]: cluster 2023-08-28T01:12:25.734061+0000 mgr.y (mgr.14901) 5815 : cluster [DBG] pgmap v5822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:27.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:27 smithi130 bash[51302]: debug 2023-08-28T01:12:27.313+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:27.318719+0000) 2023-08-28T01:12:28.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:27 smithi130 bash[27579]: audit 2023-08-28T01:12:27.110751+0000 mon.c (mon.2) 2264 : audit [DBG] from='client.? 172.21.15.6:0/1766549622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:28.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:27 smithi006 bash[27292]: audit 2023-08-28T01:12:27.110751+0000 mon.c (mon.2) 2264 : audit [DBG] from='client.? 172.21.15.6:0/1766549622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:28.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:27 smithi006 bash[33918]: audit 2023-08-28T01:12:27.110751+0000 mon.c (mon.2) 2264 : audit [DBG] from='client.? 172.21.15.6:0/1766549622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:28.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:28 smithi130 bash[51302]: debug 2023-08-28T01:12:28.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:28.319069+0000) 2023-08-28T01:12:29.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:28 smithi130 bash[27579]: cluster 2023-08-28T01:12:27.734890+0000 mgr.y (mgr.14901) 5816 : cluster [DBG] pgmap v5823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:29.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:28 smithi006 bash[33918]: cluster 2023-08-28T01:12:27.734890+0000 mgr.y (mgr.14901) 5816 : cluster [DBG] pgmap v5823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:29.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:28 smithi006 bash[27292]: cluster 2023-08-28T01:12:27.734890+0000 mgr.y (mgr.14901) 5816 : cluster [DBG] pgmap v5823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:29.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:29 smithi130 bash[51302]: debug 2023-08-28T01:12:29.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:29.319427+0000) 2023-08-28T01:12:30.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:29 smithi130 bash[27579]: audit 2023-08-28T01:12:29.518785+0000 mon.c (mon.2) 2265 : audit [DBG] from='client.? 172.21.15.6:0/485957614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:30.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:29 smithi006 bash[33918]: audit 2023-08-28T01:12:29.518785+0000 mon.c (mon.2) 2265 : audit [DBG] from='client.? 172.21.15.6:0/485957614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:30.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:29 smithi006 bash[27292]: audit 2023-08-28T01:12:29.518785+0000 mon.c (mon.2) 2265 : audit [DBG] from='client.? 172.21.15.6:0/485957614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:30.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:30 smithi130 bash[51302]: debug 2023-08-28T01:12:30.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:30.319780+0000) 2023-08-28T01:12:31.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:30 smithi130 bash[27579]: cluster 2023-08-28T01:12:29.736022+0000 mgr.y (mgr.14901) 5817 : cluster [DBG] pgmap v5824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:12:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:12:30] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T01:12:31.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:30 smithi006 bash[33918]: cluster 2023-08-28T01:12:29.736022+0000 mgr.y (mgr.14901) 5817 : cluster [DBG] pgmap v5824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:31.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:30 smithi006 bash[27292]: cluster 2023-08-28T01:12:29.736022+0000 mgr.y (mgr.14901) 5817 : cluster [DBG] pgmap v5824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:12:30.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:30 smithi006 bash[56298]: level=warn ts=2023-08-28T01:12:30.952Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:31.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:31 smithi130 bash[51302]: debug 2023-08-28T01:12:31.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:31.320115+0000) 2023-08-28T01:12:32.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:32 smithi130 bash[51302]: debug 2023-08-28T01:12:32.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:32.320399+0000) 2023-08-28T01:12:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:12:32.523Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:12:32.524Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:32 smithi006 bash[27292]: cluster 2023-08-28T01:12:31.736780+0000 mgr.y (mgr.14901) 5818 : cluster [DBG] pgmap v5825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:32 smithi006 bash[27292]: audit 2023-08-28T01:12:31.939528+0000 mon.a (mon.0) 3957 : audit [DBG] from='client.? 172.21.15.6:0/772886119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:32 smithi006 bash[33918]: cluster 2023-08-28T01:12:31.736780+0000 mgr.y (mgr.14901) 5818 : cluster [DBG] pgmap v5825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:32 smithi006 bash[33918]: audit 2023-08-28T01:12:31.939528+0000 mon.a (mon.0) 3957 : audit [DBG] from='client.? 172.21.15.6:0/772886119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:32 smithi130 bash[27579]: cluster 2023-08-28T01:12:31.736780+0000 mgr.y (mgr.14901) 5818 : cluster [DBG] pgmap v5825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:32 smithi130 bash[27579]: audit 2023-08-28T01:12:31.939528+0000 mon.a (mon.0) 3957 : audit [DBG] from='client.? 172.21.15.6:0/772886119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:33.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:33 smithi130 bash[51302]: debug 2023-08-28T01:12:33.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:33.320686+0000) 2023-08-28T01:12:33.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:33 smithi006 bash[27292]: audit 2023-08-28T01:12:33.160237+0000 mon.b (mon.1) 677 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:12:33.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:33 smithi006 bash[33918]: audit 2023-08-28T01:12:33.160237+0000 mon.b (mon.1) 677 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:12:34.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:33 smithi130 bash[27579]: audit 2023-08-28T01:12:33.160237+0000 mon.b (mon.1) 677 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:12:34.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:34 smithi130 bash[51302]: debug 2023-08-28T01:12:34.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:34.320993+0000) 2023-08-28T01:12:34.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:12:34.405Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:34.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:12:34.407Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:34.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:34 smithi006 bash[33918]: cluster 2023-08-28T01:12:33.737465+0000 mgr.y (mgr.14901) 5819 : cluster [DBG] pgmap v5826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:34.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:34 smithi006 bash[33918]: audit 2023-08-28T01:12:34.358793+0000 mon.c (mon.2) 2266 : audit [DBG] from='client.? 172.21.15.6:0/187506892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:34.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:34 smithi006 bash[27292]: cluster 2023-08-28T01:12:33.737465+0000 mgr.y (mgr.14901) 5819 : cluster [DBG] pgmap v5826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:34.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:34 smithi006 bash[27292]: audit 2023-08-28T01:12:34.358793+0000 mon.c (mon.2) 2266 : audit [DBG] from='client.? 172.21.15.6:0/187506892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:35.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:34 smithi130 bash[27579]: cluster 2023-08-28T01:12:33.737465+0000 mgr.y (mgr.14901) 5819 : cluster [DBG] pgmap v5826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:35.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:34 smithi130 bash[27579]: audit 2023-08-28T01:12:34.358793+0000 mon.c (mon.2) 2266 : audit [DBG] from='client.? 172.21.15.6:0/187506892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:35.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:35 smithi130 bash[51302]: debug 2023-08-28T01:12:35.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:35.321329+0000) 2023-08-28T01:12:36.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:36 smithi130 bash[51302]: debug 2023-08-28T01:12:36.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:36.321697+0000) 2023-08-28T01:12:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:36 smithi130 bash[27579]: cluster 2023-08-28T01:12:35.738744+0000 mgr.y (mgr.14901) 5820 : cluster [DBG] pgmap v5827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:37.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:36 smithi130 bash[27579]: audit 2023-08-28T01:12:36.777466+0000 mon.c (mon.2) 2267 : audit [DBG] from='client.? 172.21.15.6:0/230817422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:37.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:36 smithi006 bash[33918]: cluster 2023-08-28T01:12:35.738744+0000 mgr.y (mgr.14901) 5820 : cluster [DBG] pgmap v5827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:37.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:36 smithi006 bash[33918]: audit 2023-08-28T01:12:36.777466+0000 mon.c (mon.2) 2267 : audit [DBG] from='client.? 172.21.15.6:0/230817422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:37.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:36 smithi006 bash[27292]: cluster 2023-08-28T01:12:35.738744+0000 mgr.y (mgr.14901) 5820 : cluster [DBG] pgmap v5827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:37.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:36 smithi006 bash[27292]: audit 2023-08-28T01:12:36.777466+0000 mon.c (mon.2) 2267 : audit [DBG] from='client.? 172.21.15.6:0/230817422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:37.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:37 smithi130 bash[51302]: debug 2023-08-28T01:12:37.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:37.322013+0000) 2023-08-28T01:12:38.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:38 smithi130 bash[51302]: debug 2023-08-28T01:12:38.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:38.322252+0000) 2023-08-28T01:12:39.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:38 smithi130 bash[27579]: cluster 2023-08-28T01:12:37.739436+0000 mgr.y (mgr.14901) 5821 : cluster [DBG] pgmap v5828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:39.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:38 smithi006 bash[33918]: cluster 2023-08-28T01:12:37.739436+0000 mgr.y (mgr.14901) 5821 : cluster [DBG] pgmap v5828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:39.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:38 smithi006 bash[27292]: cluster 2023-08-28T01:12:37.739436+0000 mgr.y (mgr.14901) 5821 : cluster [DBG] pgmap v5828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:39.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:39 smithi130 bash[51302]: debug 2023-08-28T01:12:39.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:39.322467+0000) 2023-08-28T01:12:40.322 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:40 smithi130 bash[27579]: audit 2023-08-28T01:12:39.039522+0000 mon.a (mon.0) 3958 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:12:40.322 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:40 smithi130 bash[27579]: audit 2023-08-28T01:12:39.227508+0000 mon.c (mon.2) 2268 : audit [DBG] from='client.? 172.21.15.6:0/2182697871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:40.323 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:40 smithi130 bash[27579]: audit 2023-08-28T01:12:39.537098+0000 mon.a (mon.0) 3959 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:12:40.323 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:40 smithi130 bash[27579]: cluster 2023-08-28T01:12:39.740560+0000 mgr.y (mgr.14901) 5822 : cluster [DBG] pgmap v5829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:40.323 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:40 smithi130 bash[27579]: audit 2023-08-28T01:12:39.832531+0000 mon.b (mon.1) 678 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:12:40.323 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:40 smithi130 bash[27579]: audit 2023-08-28T01:12:39.834849+0000 mon.b (mon.1) 679 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:12:40.323 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:40 smithi130 bash[27579]: audit 2023-08-28T01:12:39.844574+0000 mon.a (mon.0) 3960 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:12:40.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:40 smithi006 bash[27292]: audit 2023-08-28T01:12:39.039522+0000 mon.a (mon.0) 3958 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:12:40.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:40 smithi006 bash[27292]: audit 2023-08-28T01:12:39.227508+0000 mon.c (mon.2) 2268 : audit [DBG] from='client.? 172.21.15.6:0/2182697871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:40.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:40 smithi006 bash[27292]: audit 2023-08-28T01:12:39.537098+0000 mon.a (mon.0) 3959 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:12:40.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:40 smithi006 bash[27292]: cluster 2023-08-28T01:12:39.740560+0000 mgr.y (mgr.14901) 5822 : cluster [DBG] pgmap v5829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:40.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:40 smithi006 bash[27292]: audit 2023-08-28T01:12:39.832531+0000 mon.b (mon.1) 678 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:12:40.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:40 smithi006 bash[27292]: audit 2023-08-28T01:12:39.834849+0000 mon.b (mon.1) 679 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:12:40.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:40 smithi006 bash[27292]: audit 2023-08-28T01:12:39.844574+0000 mon.a (mon.0) 3960 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:12:40.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:40 smithi006 bash[33918]: audit 2023-08-28T01:12:39.039522+0000 mon.a (mon.0) 3958 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:12:40.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:40 smithi006 bash[33918]: audit 2023-08-28T01:12:39.227508+0000 mon.c (mon.2) 2268 : audit [DBG] from='client.? 172.21.15.6:0/2182697871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:40.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:40 smithi006 bash[33918]: audit 2023-08-28T01:12:39.537098+0000 mon.a (mon.0) 3959 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:12:40.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:40 smithi006 bash[33918]: cluster 2023-08-28T01:12:39.740560+0000 mgr.y (mgr.14901) 5822 : cluster [DBG] pgmap v5829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:40.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:40 smithi006 bash[33918]: audit 2023-08-28T01:12:39.832531+0000 mon.b (mon.1) 678 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:12:40.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:40 smithi006 bash[33918]: audit 2023-08-28T01:12:39.834849+0000 mon.b (mon.1) 679 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:12:40.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:40 smithi006 bash[33918]: audit 2023-08-28T01:12:39.844574+0000 mon.a (mon.0) 3960 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:12:40.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:40 smithi130 bash[51302]: debug 2023-08-28T01:12:40.317+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:40.322776+0000) 2023-08-28T01:12:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:12:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:12:40] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T01:12:41.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:12:40.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:41.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:12:40.953Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:41.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:41 smithi130 bash[51302]: debug 2023-08-28T01:12:41.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:41.323128+0000) 2023-08-28T01:12:42.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:42 smithi130 bash[51302]: debug 2023-08-28T01:12:42.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:42.323396+0000) 2023-08-28T01:12:42.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:42 smithi006 bash[33918]: audit 2023-08-28T01:12:41.646405+0000 mon.c (mon.2) 2269 : audit [DBG] from='client.? 172.21.15.6:0/235658863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:42.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:42 smithi006 bash[33918]: cluster 2023-08-28T01:12:41.741194+0000 mgr.y (mgr.14901) 5823 : cluster [DBG] pgmap v5830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:12:42.523Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:12:42.524Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:42 smithi006 bash[27292]: audit 2023-08-28T01:12:41.646405+0000 mon.c (mon.2) 2269 : audit [DBG] from='client.? 172.21.15.6:0/235658863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:42 smithi006 bash[27292]: cluster 2023-08-28T01:12:41.741194+0000 mgr.y (mgr.14901) 5823 : cluster [DBG] pgmap v5830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:42 smithi130 bash[27579]: audit 2023-08-28T01:12:41.646405+0000 mon.c (mon.2) 2269 : audit [DBG] from='client.? 172.21.15.6:0/235658863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:42 smithi130 bash[27579]: cluster 2023-08-28T01:12:41.741194+0000 mgr.y (mgr.14901) 5823 : cluster [DBG] pgmap v5830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:43.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:43 smithi130 bash[51302]: debug 2023-08-28T01:12:43.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:43.323741+0000) 2023-08-28T01:12:44.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:44 smithi130 bash[51302]: debug 2023-08-28T01:12:44.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:44.324066+0000) 2023-08-28T01:12:44.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:12:44.406Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:44.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:12:44.408Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:45.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:44 smithi130 bash[27579]: cluster 2023-08-28T01:12:43.741812+0000 mgr.y (mgr.14901) 5824 : cluster [DBG] pgmap v5831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:45.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:44 smithi130 bash[27579]: audit 2023-08-28T01:12:44.059531+0000 mon.c (mon.2) 2270 : audit [DBG] from='client.? 172.21.15.6:0/4035539139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:45.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:44 smithi006 bash[33918]: cluster 2023-08-28T01:12:43.741812+0000 mgr.y (mgr.14901) 5824 : cluster [DBG] pgmap v5831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:45.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:44 smithi006 bash[33918]: audit 2023-08-28T01:12:44.059531+0000 mon.c (mon.2) 2270 : audit [DBG] from='client.? 172.21.15.6:0/4035539139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:45.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:44 smithi006 bash[27292]: cluster 2023-08-28T01:12:43.741812+0000 mgr.y (mgr.14901) 5824 : cluster [DBG] pgmap v5831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:45.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:44 smithi006 bash[27292]: audit 2023-08-28T01:12:44.059531+0000 mon.c (mon.2) 2270 : audit [DBG] from='client.? 172.21.15.6:0/4035539139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:45.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:45 smithi130 bash[51302]: debug 2023-08-28T01:12:45.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:45.324399+0000) 2023-08-28T01:12:46.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:46 smithi130 bash[51302]: debug 2023-08-28T01:12:46.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:46.324758+0000) 2023-08-28T01:12:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:46 smithi130 bash[27579]: cluster 2023-08-28T01:12:45.742965+0000 mgr.y (mgr.14901) 5825 : cluster [DBG] pgmap v5832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:46 smithi130 bash[27579]: audit 2023-08-28T01:12:46.477915+0000 mon.a (mon.0) 3961 : audit [DBG] from='client.? 172.21.15.6:0/97171146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:47.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:46 smithi006 bash[27292]: cluster 2023-08-28T01:12:45.742965+0000 mgr.y (mgr.14901) 5825 : cluster [DBG] pgmap v5832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:47.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:46 smithi006 bash[27292]: audit 2023-08-28T01:12:46.477915+0000 mon.a (mon.0) 3961 : audit [DBG] from='client.? 172.21.15.6:0/97171146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:47.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:46 smithi006 bash[33918]: cluster 2023-08-28T01:12:45.742965+0000 mgr.y (mgr.14901) 5825 : cluster [DBG] pgmap v5832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:47.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:46 smithi006 bash[33918]: audit 2023-08-28T01:12:46.477915+0000 mon.a (mon.0) 3961 : audit [DBG] from='client.? 172.21.15.6:0/97171146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:47.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:47 smithi130 bash[51302]: debug 2023-08-28T01:12:47.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:47.325114+0000) 2023-08-28T01:12:48.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:48 smithi130 bash[51302]: debug 2023-08-28T01:12:48.321+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:48.325469+0000) 2023-08-28T01:12:49.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:48 smithi130 bash[27579]: cluster 2023-08-28T01:12:47.743604+0000 mgr.y (mgr.14901) 5826 : cluster [DBG] pgmap v5833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:49.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:48 smithi006 bash[33918]: cluster 2023-08-28T01:12:47.743604+0000 mgr.y (mgr.14901) 5826 : cluster [DBG] pgmap v5833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:49.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:48 smithi006 bash[27292]: cluster 2023-08-28T01:12:47.743604+0000 mgr.y (mgr.14901) 5826 : cluster [DBG] pgmap v5833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:49.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:49 smithi130 bash[51302]: debug 2023-08-28T01:12:49.320+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:49.325811+0000) 2023-08-28T01:12:50.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:49 smithi130 bash[27579]: audit 2023-08-28T01:12:48.898330+0000 mon.a (mon.0) 3962 : audit [DBG] from='client.? 172.21.15.6:0/2788129229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:50.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:49 smithi006 bash[27292]: audit 2023-08-28T01:12:48.898330+0000 mon.a (mon.0) 3962 : audit [DBG] from='client.? 172.21.15.6:0/2788129229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:50.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:49 smithi006 bash[33918]: audit 2023-08-28T01:12:48.898330+0000 mon.a (mon.0) 3962 : audit [DBG] from='client.? 172.21.15.6:0/2788129229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:50.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:50 smithi130 bash[51302]: debug 2023-08-28T01:12:50.320+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:50.326216+0000) 2023-08-28T01:12:51.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:50 smithi130 bash[27579]: cluster 2023-08-28T01:12:49.744869+0000 mgr.y (mgr.14901) 5827 : cluster [DBG] pgmap v5834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:51.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:50 smithi006 bash[27292]: cluster 2023-08-28T01:12:49.744869+0000 mgr.y (mgr.14901) 5827 : cluster [DBG] pgmap v5834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:12:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:12:50] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:12:51.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:50 smithi006 bash[33918]: cluster 2023-08-28T01:12:49.744869+0000 mgr.y (mgr.14901) 5827 : cluster [DBG] pgmap v5834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:12:50.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:12:50.954Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:51.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:51 smithi130 bash[51302]: debug 2023-08-28T01:12:51.320+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:51.326566+0000) 2023-08-28T01:12:52.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:51 smithi130 bash[27579]: audit 2023-08-28T01:12:51.314259+0000 mon.a (mon.0) 3963 : audit [DBG] from='client.? 172.21.15.6:0/1348820445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:52.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:51 smithi006 bash[33918]: audit 2023-08-28T01:12:51.314259+0000 mon.a (mon.0) 3963 : audit [DBG] from='client.? 172.21.15.6:0/1348820445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:52.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:51 smithi006 bash[27292]: audit 2023-08-28T01:12:51.314259+0000 mon.a (mon.0) 3963 : audit [DBG] from='client.? 172.21.15.6:0/1348820445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:52.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:52 smithi130 bash[51302]: debug 2023-08-28T01:12:52.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:52.326922+0000) 2023-08-28T01:12:52.831 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:12:52.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:52.831 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:12:52.526Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:52 smithi130 bash[27579]: cluster 2023-08-28T01:12:51.745686+0000 mgr.y (mgr.14901) 5828 : cluster [DBG] pgmap v5835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:53.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:52 smithi006 bash[27292]: cluster 2023-08-28T01:12:51.745686+0000 mgr.y (mgr.14901) 5828 : cluster [DBG] pgmap v5835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:53.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:52 smithi006 bash[33918]: cluster 2023-08-28T01:12:51.745686+0000 mgr.y (mgr.14901) 5828 : cluster [DBG] pgmap v5835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:53.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:53 smithi130 bash[51302]: debug 2023-08-28T01:12:53.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:53.327183+0000) 2023-08-28T01:12:54.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:53 smithi130 bash[27579]: audit 2023-08-28T01:12:53.729774+0000 mon.c (mon.2) 2271 : audit [DBG] from='client.? 172.21.15.6:0/2584541238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:54.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:53 smithi006 bash[27292]: audit 2023-08-28T01:12:53.729774+0000 mon.c (mon.2) 2271 : audit [DBG] from='client.? 172.21.15.6:0/2584541238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:54.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:53 smithi006 bash[33918]: audit 2023-08-28T01:12:53.729774+0000 mon.c (mon.2) 2271 : audit [DBG] from='client.? 172.21.15.6:0/2584541238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:54.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:54 smithi130 bash[51302]: debug 2023-08-28T01:12:54.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:54.327489+0000) 2023-08-28T01:12:54.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:12:54.406Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:54.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:12:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:12:54.408Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:12:55.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:54 smithi130 bash[27579]: cluster 2023-08-28T01:12:53.746288+0000 mgr.y (mgr.14901) 5829 : cluster [DBG] pgmap v5836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:55.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:54 smithi006 bash[33918]: cluster 2023-08-28T01:12:53.746288+0000 mgr.y (mgr.14901) 5829 : cluster [DBG] pgmap v5836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:55.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:54 smithi006 bash[27292]: cluster 2023-08-28T01:12:53.746288+0000 mgr.y (mgr.14901) 5829 : cluster [DBG] pgmap v5836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:55.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:55 smithi130 bash[51302]: debug 2023-08-28T01:12:55.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:55.327847+0000) 2023-08-28T01:12:56.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:56 smithi130 bash[51302]: debug 2023-08-28T01:12:56.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:56.328160+0000) 2023-08-28T01:12:57.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:56 smithi130 bash[27579]: cluster 2023-08-28T01:12:55.747400+0000 mgr.y (mgr.14901) 5830 : cluster [DBG] pgmap v5837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:57.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:56 smithi130 bash[27579]: audit 2023-08-28T01:12:56.139759+0000 mon.c (mon.2) 2272 : audit [DBG] from='client.? 172.21.15.6:0/4179148213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:57.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:56 smithi006 bash[33918]: cluster 2023-08-28T01:12:55.747400+0000 mgr.y (mgr.14901) 5830 : cluster [DBG] pgmap v5837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:57.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:56 smithi006 bash[33918]: audit 2023-08-28T01:12:56.139759+0000 mon.c (mon.2) 2272 : audit [DBG] from='client.? 172.21.15.6:0/4179148213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:57.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:56 smithi006 bash[27292]: cluster 2023-08-28T01:12:55.747400+0000 mgr.y (mgr.14901) 5830 : cluster [DBG] pgmap v5837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:57.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:56 smithi006 bash[27292]: audit 2023-08-28T01:12:56.139759+0000 mon.c (mon.2) 2272 : audit [DBG] from='client.? 172.21.15.6:0/4179148213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:57.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:57 smithi130 bash[51302]: debug 2023-08-28T01:12:57.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:57.328479+0000) 2023-08-28T01:12:58.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:58 smithi130 bash[51302]: debug 2023-08-28T01:12:58.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:58.328795+0000) 2023-08-28T01:12:59.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:58 smithi130 bash[27579]: cluster 2023-08-28T01:12:57.748020+0000 mgr.y (mgr.14901) 5831 : cluster [DBG] pgmap v5838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:59.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:12:58 smithi130 bash[27579]: audit 2023-08-28T01:12:58.558174+0000 mon.c (mon.2) 2273 : audit [DBG] from='client.? 172.21.15.6:0/3674920020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:59.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:58 smithi006 bash[33918]: cluster 2023-08-28T01:12:57.748020+0000 mgr.y (mgr.14901) 5831 : cluster [DBG] pgmap v5838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:59.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:12:58 smithi006 bash[33918]: audit 2023-08-28T01:12:58.558174+0000 mon.c (mon.2) 2273 : audit [DBG] from='client.? 172.21.15.6:0/3674920020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:59.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:58 smithi006 bash[27292]: cluster 2023-08-28T01:12:57.748020+0000 mgr.y (mgr.14901) 5831 : cluster [DBG] pgmap v5838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:12:59.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:12:58 smithi006 bash[27292]: audit 2023-08-28T01:12:58.558174+0000 mon.c (mon.2) 2273 : audit [DBG] from='client.? 172.21.15.6:0/3674920020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:59.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:12:59 smithi130 bash[51302]: debug 2023-08-28T01:12:59.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:59.329130+0000) 2023-08-28T01:13:00.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:00 smithi130 bash[51302]: debug 2023-08-28T01:13:00.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:00.329418+0000) 2023-08-28T01:13:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:00 smithi130 bash[27579]: cluster 2023-08-28T01:12:59.749218+0000 mgr.y (mgr.14901) 5832 : cluster [DBG] pgmap v5839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:01.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:00 smithi006 bash[27292]: cluster 2023-08-28T01:12:59.749218+0000 mgr.y (mgr.14901) 5832 : cluster [DBG] pgmap v5839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:13:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:13:00] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:13:01.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:00 smithi006 bash[33918]: cluster 2023-08-28T01:12:59.749218+0000 mgr.y (mgr.14901) 5832 : cluster [DBG] pgmap v5839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:13:00.953Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:13:00.955Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:01.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:01 smithi130 bash[51302]: debug 2023-08-28T01:13:01.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:01.329773+0000) 2023-08-28T01:13:02.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:01 smithi130 bash[27579]: audit 2023-08-28T01:13:00.979345+0000 mon.a (mon.0) 3964 : audit [DBG] from='client.? 172.21.15.6:0/1992064129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:02.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:01 smithi006 bash[27292]: audit 2023-08-28T01:13:00.979345+0000 mon.a (mon.0) 3964 : audit [DBG] from='client.? 172.21.15.6:0/1992064129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:02.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:01 smithi006 bash[33918]: audit 2023-08-28T01:13:00.979345+0000 mon.a (mon.0) 3964 : audit [DBG] from='client.? 172.21.15.6:0/1992064129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:02.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:02 smithi130 bash[51302]: debug 2023-08-28T01:13:02.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:02.330125+0000) 2023-08-28T01:13:02.864 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:13:02.525Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:02.864 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:13:02.527Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:02 smithi130 bash[27579]: cluster 2023-08-28T01:13:01.749891+0000 mgr.y (mgr.14901) 5833 : cluster [DBG] pgmap v5840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:03.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:02 smithi006 bash[27292]: cluster 2023-08-28T01:13:01.749891+0000 mgr.y (mgr.14901) 5833 : cluster [DBG] pgmap v5840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:03.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:02 smithi006 bash[33918]: cluster 2023-08-28T01:13:01.749891+0000 mgr.y (mgr.14901) 5833 : cluster [DBG] pgmap v5840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:03.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:03 smithi130 bash[51302]: debug 2023-08-28T01:13:03.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:03.330461+0000) 2023-08-28T01:13:04.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:03 smithi006 bash[27292]: audit 2023-08-28T01:13:03.404939+0000 mon.c (mon.2) 2274 : audit [DBG] from='client.? 172.21.15.6:0/3580030169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:04.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:03 smithi006 bash[33918]: audit 2023-08-28T01:13:03.404939+0000 mon.c (mon.2) 2274 : audit [DBG] from='client.? 172.21.15.6:0/3580030169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:04.330 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:03 smithi130 bash[27579]: audit 2023-08-28T01:13:03.404939+0000 mon.c (mon.2) 2274 : audit [DBG] from='client.? 172.21.15.6:0/3580030169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:04.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:04 smithi130 bash[51302]: debug 2023-08-28T01:13:04.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:04.330792+0000) 2023-08-28T01:13:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:13:04.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:13:04.409Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:05.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:04 smithi006 bash[33918]: cluster 2023-08-28T01:13:03.750574+0000 mgr.y (mgr.14901) 5834 : cluster [DBG] pgmap v5841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:05.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:04 smithi006 bash[27292]: cluster 2023-08-28T01:13:03.750574+0000 mgr.y (mgr.14901) 5834 : cluster [DBG] pgmap v5841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:05.330 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:04 smithi130 bash[27579]: cluster 2023-08-28T01:13:03.750574+0000 mgr.y (mgr.14901) 5834 : cluster [DBG] pgmap v5841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:05.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:05 smithi130 bash[51302]: debug 2023-08-28T01:13:05.328+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:05.331087+0000) 2023-08-28T01:13:06.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:05 smithi006 bash[27292]: audit 2023-08-28T01:13:05.822258+0000 mon.a (mon.0) 3965 : audit [DBG] from='client.? 172.21.15.6:0/763671581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:06.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:05 smithi006 bash[33918]: audit 2023-08-28T01:13:05.822258+0000 mon.a (mon.0) 3965 : audit [DBG] from='client.? 172.21.15.6:0/763671581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:06.331 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:05 smithi130 bash[27579]: audit 2023-08-28T01:13:05.822258+0000 mon.a (mon.0) 3965 : audit [DBG] from='client.? 172.21.15.6:0/763671581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:06.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:06 smithi130 bash[51302]: debug 2023-08-28T01:13:06.328+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:06.331409+0000) 2023-08-28T01:13:07.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:06 smithi006 bash[33918]: cluster 2023-08-28T01:13:05.751665+0000 mgr.y (mgr.14901) 5835 : cluster [DBG] pgmap v5842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:06 smithi006 bash[27292]: cluster 2023-08-28T01:13:05.751665+0000 mgr.y (mgr.14901) 5835 : cluster [DBG] pgmap v5842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:07.331 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:06 smithi130 bash[27579]: cluster 2023-08-28T01:13:05.751665+0000 mgr.y (mgr.14901) 5835 : cluster [DBG] pgmap v5842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:07.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:07 smithi130 bash[51302]: debug 2023-08-28T01:13:07.328+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:07.331722+0000) 2023-08-28T01:13:08.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:08 smithi130 bash[51302]: debug 2023-08-28T01:13:08.328+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:08.332049+0000) 2023-08-28T01:13:09.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:08 smithi006 bash[27292]: cluster 2023-08-28T01:13:07.752329+0000 mgr.y (mgr.14901) 5836 : cluster [DBG] pgmap v5843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:09.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:08 smithi006 bash[27292]: audit 2023-08-28T01:13:08.237368+0000 mon.c (mon.2) 2275 : audit [DBG] from='client.? 172.21.15.6:0/1536684190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:08 smithi006 bash[33918]: cluster 2023-08-28T01:13:07.752329+0000 mgr.y (mgr.14901) 5836 : cluster [DBG] pgmap v5843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:09.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:08 smithi006 bash[33918]: audit 2023-08-28T01:13:08.237368+0000 mon.c (mon.2) 2275 : audit [DBG] from='client.? 172.21.15.6:0/1536684190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:09.332 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:08 smithi130 bash[27579]: cluster 2023-08-28T01:13:07.752329+0000 mgr.y (mgr.14901) 5836 : cluster [DBG] pgmap v5843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:09.332 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:08 smithi130 bash[27579]: audit 2023-08-28T01:13:08.237368+0000 mon.c (mon.2) 2275 : audit [DBG] from='client.? 172.21.15.6:0/1536684190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:09.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:09 smithi130 bash[51302]: debug 2023-08-28T01:13:09.328+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:09.332379+0000) 2023-08-28T01:13:10.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:10 smithi130 bash[51302]: debug 2023-08-28T01:13:10.328+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:10.332722+0000) 2023-08-28T01:13:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:13:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:13:10] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:13:11.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:13:10.954Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:11.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:13:10.956Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:11.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:10 smithi006 bash[27292]: cluster 2023-08-28T01:13:09.753528+0000 mgr.y (mgr.14901) 5837 : cluster [DBG] pgmap v5844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:11.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:10 smithi006 bash[27292]: audit 2023-08-28T01:13:10.666944+0000 mon.c (mon.2) 2276 : audit [DBG] from='client.? 172.21.15.6:0/2555986418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:10 smithi006 bash[33918]: cluster 2023-08-28T01:13:09.753528+0000 mgr.y (mgr.14901) 5837 : cluster [DBG] pgmap v5844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:10 smithi006 bash[33918]: audit 2023-08-28T01:13:10.666944+0000 mon.c (mon.2) 2276 : audit [DBG] from='client.? 172.21.15.6:0/2555986418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:11.332 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:10 smithi130 bash[27579]: cluster 2023-08-28T01:13:09.753528+0000 mgr.y (mgr.14901) 5837 : cluster [DBG] pgmap v5844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:11.332 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:10 smithi130 bash[27579]: audit 2023-08-28T01:13:10.666944+0000 mon.c (mon.2) 2276 : audit [DBG] from='client.? 172.21.15.6:0/2555986418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:11.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:11 smithi130 bash[51302]: debug 2023-08-28T01:13:11.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:11.333067+0000) 2023-08-28T01:13:12.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:12 smithi130 bash[51302]: debug 2023-08-28T01:13:12.328+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:12.333402+0000) 2023-08-28T01:13:12.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:12 smithi006 bash[27292]: cluster 2023-08-28T01:13:11.754201+0000 mgr.y (mgr.14901) 5838 : cluster [DBG] pgmap v5845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:12 smithi006 bash[33918]: cluster 2023-08-28T01:13:11.754201+0000 mgr.y (mgr.14901) 5838 : cluster [DBG] pgmap v5845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:13:12.526Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:13:12.527Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:12 smithi130 bash[27579]: cluster 2023-08-28T01:13:11.754201+0000 mgr.y (mgr.14901) 5838 : cluster [DBG] pgmap v5845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:13.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:13 smithi130 bash[51302]: debug 2023-08-28T01:13:13.328+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:13.333737+0000) 2023-08-28T01:13:13.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:13 smithi006 bash[33918]: audit 2023-08-28T01:13:13.082679+0000 mon.c (mon.2) 2277 : audit [DBG] from='client.? 172.21.15.6:0/982685033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:13.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:13 smithi006 bash[27292]: audit 2023-08-28T01:13:13.082679+0000 mon.c (mon.2) 2277 : audit [DBG] from='client.? 172.21.15.6:0/982685033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:14.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:13 smithi130 bash[27579]: audit 2023-08-28T01:13:13.082679+0000 mon.c (mon.2) 2277 : audit [DBG] from='client.? 172.21.15.6:0/982685033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:14.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:14 smithi130 bash[51302]: debug 2023-08-28T01:13:14.328+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:14.334089+0000) 2023-08-28T01:13:14.659 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:13:14.407Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:14.659 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:13:14.410Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:14.659 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:14 smithi006 bash[27292]: cluster 2023-08-28T01:13:13.754875+0000 mgr.y (mgr.14901) 5839 : cluster [DBG] pgmap v5846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:14.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:14 smithi006 bash[33918]: cluster 2023-08-28T01:13:13.754875+0000 mgr.y (mgr.14901) 5839 : cluster [DBG] pgmap v5846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:15.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:14 smithi130 bash[27579]: cluster 2023-08-28T01:13:13.754875+0000 mgr.y (mgr.14901) 5839 : cluster [DBG] pgmap v5846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:15.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:15 smithi130 bash[51302]: debug 2023-08-28T01:13:15.328+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:15.334421+0000) 2023-08-28T01:13:15.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:15 smithi006 bash[27292]: audit 2023-08-28T01:13:15.501711+0000 mon.a (mon.0) 3966 : audit [DBG] from='client.? 172.21.15.6:0/1708318979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:15.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:15 smithi006 bash[33918]: audit 2023-08-28T01:13:15.501711+0000 mon.a (mon.0) 3966 : audit [DBG] from='client.? 172.21.15.6:0/1708318979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:16.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:15 smithi130 bash[27579]: audit 2023-08-28T01:13:15.501711+0000 mon.a (mon.0) 3966 : audit [DBG] from='client.? 172.21.15.6:0/1708318979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:16.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:16 smithi130 bash[51302]: debug 2023-08-28T01:13:16.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:16.334739+0000) 2023-08-28T01:13:16.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:16 smithi006 bash[27292]: cluster 2023-08-28T01:13:15.756018+0000 mgr.y (mgr.14901) 5840 : cluster [DBG] pgmap v5847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:16.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:16 smithi006 bash[33918]: cluster 2023-08-28T01:13:15.756018+0000 mgr.y (mgr.14901) 5840 : cluster [DBG] pgmap v5847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:17.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:16 smithi130 bash[27579]: cluster 2023-08-28T01:13:15.756018+0000 mgr.y (mgr.14901) 5840 : cluster [DBG] pgmap v5847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:17.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:17 smithi130 bash[51302]: debug 2023-08-28T01:13:17.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:17.335104+0000) 2023-08-28T01:13:18.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:18 smithi130 bash[51302]: debug 2023-08-28T01:13:18.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:18.335371+0000) 2023-08-28T01:13:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:18 smithi130 bash[27579]: cluster 2023-08-28T01:13:17.756649+0000 mgr.y (mgr.14901) 5841 : cluster [DBG] pgmap v5848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:19.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:18 smithi130 bash[27579]: audit 2023-08-28T01:13:17.915499+0000 mon.c (mon.2) 2278 : audit [DBG] from='client.? 172.21.15.6:0/1759880802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:19.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:18 smithi006 bash[33918]: cluster 2023-08-28T01:13:17.756649+0000 mgr.y (mgr.14901) 5841 : cluster [DBG] pgmap v5848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:19.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:18 smithi006 bash[33918]: audit 2023-08-28T01:13:17.915499+0000 mon.c (mon.2) 2278 : audit [DBG] from='client.? 172.21.15.6:0/1759880802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:19.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:18 smithi006 bash[27292]: cluster 2023-08-28T01:13:17.756649+0000 mgr.y (mgr.14901) 5841 : cluster [DBG] pgmap v5848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:19.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:18 smithi006 bash[27292]: audit 2023-08-28T01:13:17.915499+0000 mon.c (mon.2) 2278 : audit [DBG] from='client.? 172.21.15.6:0/1759880802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:19.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:19 smithi130 bash[51302]: debug 2023-08-28T01:13:19.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:19.335728+0000) 2023-08-28T01:13:20.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:20 smithi130 bash[51302]: debug 2023-08-28T01:13:20.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:20.336074+0000) 2023-08-28T01:13:21.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:20 smithi130 bash[27579]: cluster 2023-08-28T01:13:19.757834+0000 mgr.y (mgr.14901) 5842 : cluster [DBG] pgmap v5849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:21.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:20 smithi130 bash[27579]: audit 2023-08-28T01:13:20.339486+0000 mon.a (mon.0) 3967 : audit [DBG] from='client.? 172.21.15.6:0/2414682651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:21.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:13:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:13:20] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T01:13:21.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:20 smithi006 bash[33918]: cluster 2023-08-28T01:13:19.757834+0000 mgr.y (mgr.14901) 5842 : cluster [DBG] pgmap v5849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:21.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:20 smithi006 bash[33918]: audit 2023-08-28T01:13:20.339486+0000 mon.a (mon.0) 3967 : audit [DBG] from='client.? 172.21.15.6:0/2414682651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:21.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:13:20.955Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:13:20.956Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:20 smithi006 bash[27292]: cluster 2023-08-28T01:13:19.757834+0000 mgr.y (mgr.14901) 5842 : cluster [DBG] pgmap v5849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:21.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:20 smithi006 bash[27292]: audit 2023-08-28T01:13:20.339486+0000 mon.a (mon.0) 3967 : audit [DBG] from='client.? 172.21.15.6:0/2414682651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:21.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:21 smithi130 bash[51302]: debug 2023-08-28T01:13:21.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:21.336413+0000) 2023-08-28T01:13:22.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:22 smithi130 bash[51302]: debug 2023-08-28T01:13:22.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:22.336747+0000) 2023-08-28T01:13:22.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:22 smithi006 bash[33918]: cluster 2023-08-28T01:13:21.758509+0000 mgr.y (mgr.14901) 5843 : cluster [DBG] pgmap v5850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:22.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:22 smithi006 bash[27292]: cluster 2023-08-28T01:13:21.758509+0000 mgr.y (mgr.14901) 5843 : cluster [DBG] pgmap v5850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:22.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:13:22.527Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:22.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:13:22.528Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:23.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:22 smithi130 bash[27579]: cluster 2023-08-28T01:13:21.758509+0000 mgr.y (mgr.14901) 5843 : cluster [DBG] pgmap v5850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:23.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:23 smithi130 bash[51302]: debug 2023-08-28T01:13:23.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:23.337100+0000) 2023-08-28T01:13:23.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:23 smithi006 bash[33918]: audit 2023-08-28T01:13:22.753959+0000 mon.a (mon.0) 3968 : audit [DBG] from='client.? 172.21.15.6:0/3383953843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:23.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:23 smithi006 bash[27292]: audit 2023-08-28T01:13:22.753959+0000 mon.a (mon.0) 3968 : audit [DBG] from='client.? 172.21.15.6:0/3383953843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:24.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:23 smithi130 bash[27579]: audit 2023-08-28T01:13:22.753959+0000 mon.a (mon.0) 3968 : audit [DBG] from='client.? 172.21.15.6:0/3383953843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:24.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:24 smithi130 bash[51302]: debug 2023-08-28T01:13:24.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:24.337444+0000) 2023-08-28T01:13:24.663 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:13:24.408Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:24.663 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:13:24.410Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:24.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:24 smithi006 bash[33918]: cluster 2023-08-28T01:13:23.759296+0000 mgr.y (mgr.14901) 5844 : cluster [DBG] pgmap v5851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:24.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:24 smithi006 bash[27292]: cluster 2023-08-28T01:13:23.759296+0000 mgr.y (mgr.14901) 5844 : cluster [DBG] pgmap v5851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:25.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:24 smithi130 bash[27579]: cluster 2023-08-28T01:13:23.759296+0000 mgr.y (mgr.14901) 5844 : cluster [DBG] pgmap v5851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:25.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:25 smithi130 bash[51302]: debug 2023-08-28T01:13:25.336+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:25.337766+0000) 2023-08-28T01:13:25.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:25 smithi006 bash[27292]: audit 2023-08-28T01:13:25.174047+0000 mon.a (mon.0) 3969 : audit [DBG] from='client.? 172.21.15.6:0/2739332764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:25.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:25 smithi006 bash[33918]: audit 2023-08-28T01:13:25.174047+0000 mon.a (mon.0) 3969 : audit [DBG] from='client.? 172.21.15.6:0/2739332764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:26.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:25 smithi130 bash[27579]: audit 2023-08-28T01:13:25.174047+0000 mon.a (mon.0) 3969 : audit [DBG] from='client.? 172.21.15.6:0/2739332764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:26.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:26 smithi130 bash[51302]: debug 2023-08-28T01:13:26.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:26.338139+0000) 2023-08-28T01:13:26.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:26 smithi006 bash[33918]: cluster 2023-08-28T01:13:25.760475+0000 mgr.y (mgr.14901) 5845 : cluster [DBG] pgmap v5852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:26.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:26 smithi006 bash[27292]: cluster 2023-08-28T01:13:25.760475+0000 mgr.y (mgr.14901) 5845 : cluster [DBG] pgmap v5852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:27.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:26 smithi130 bash[27579]: cluster 2023-08-28T01:13:25.760475+0000 mgr.y (mgr.14901) 5845 : cluster [DBG] pgmap v5852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:27.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:27 smithi130 bash[51302]: debug 2023-08-28T01:13:27.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:27.338465+0000) 2023-08-28T01:13:27.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:27 smithi006 bash[33918]: audit 2023-08-28T01:13:27.590321+0000 mon.c (mon.2) 2279 : audit [DBG] from='client.? 172.21.15.6:0/2732218405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:27.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:27 smithi006 bash[27292]: audit 2023-08-28T01:13:27.590321+0000 mon.c (mon.2) 2279 : audit [DBG] from='client.? 172.21.15.6:0/2732218405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:28.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:27 smithi130 bash[27579]: audit 2023-08-28T01:13:27.590321+0000 mon.c (mon.2) 2279 : audit [DBG] from='client.? 172.21.15.6:0/2732218405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:28.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:28 smithi130 bash[51302]: debug 2023-08-28T01:13:28.336+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:28.338847+0000) 2023-08-28T01:13:28.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:28 smithi006 bash[33918]: cluster 2023-08-28T01:13:27.761173+0000 mgr.y (mgr.14901) 5846 : cluster [DBG] pgmap v5853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:28.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:28 smithi006 bash[27292]: cluster 2023-08-28T01:13:27.761173+0000 mgr.y (mgr.14901) 5846 : cluster [DBG] pgmap v5853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:29.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:28 smithi130 bash[27579]: cluster 2023-08-28T01:13:27.761173+0000 mgr.y (mgr.14901) 5846 : cluster [DBG] pgmap v5853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:29.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:29 smithi130 bash[51302]: debug 2023-08-28T01:13:29.336+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:29.339202+0000) 2023-08-28T01:13:30.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:30 smithi130 bash[51302]: debug 2023-08-28T01:13:30.336+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:30.339547+0000) 2023-08-28T01:13:31.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:30 smithi130 bash[27579]: cluster 2023-08-28T01:13:29.762076+0000 mgr.y (mgr.14901) 5847 : cluster [DBG] pgmap v5854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:31.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:30 smithi130 bash[27579]: audit 2023-08-28T01:13:30.010577+0000 mon.a (mon.0) 3970 : audit [DBG] from='client.? 172.21.15.6:0/4100515395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:13:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:13:30] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:13:31.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:30 smithi006 bash[33918]: cluster 2023-08-28T01:13:29.762076+0000 mgr.y (mgr.14901) 5847 : cluster [DBG] pgmap v5854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:31.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:30 smithi006 bash[33918]: audit 2023-08-28T01:13:30.010577+0000 mon.a (mon.0) 3970 : audit [DBG] from='client.? 172.21.15.6:0/4100515395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:31.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:13:30.956Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:30 smithi006 bash[56298]: level=warn ts=2023-08-28T01:13:30.957Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:31.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:30 smithi006 bash[27292]: cluster 2023-08-28T01:13:29.762076+0000 mgr.y (mgr.14901) 5847 : cluster [DBG] pgmap v5854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:31.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:30 smithi006 bash[27292]: audit 2023-08-28T01:13:30.010577+0000 mon.a (mon.0) 3970 : audit [DBG] from='client.? 172.21.15.6:0/4100515395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:31.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:31 smithi130 bash[51302]: debug 2023-08-28T01:13:31.336+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:31.339903+0000) 2023-08-28T01:13:32.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:32 smithi130 bash[51302]: debug 2023-08-28T01:13:32.336+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:32.340237+0000) 2023-08-28T01:13:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:32 smithi006 bash[27292]: cluster 2023-08-28T01:13:31.762736+0000 mgr.y (mgr.14901) 5848 : cluster [DBG] pgmap v5855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:32 smithi006 bash[27292]: audit 2023-08-28T01:13:32.429028+0000 mon.c (mon.2) 2280 : audit [DBG] from='client.? 172.21.15.6:0/3888686390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:13:32.527Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:13:32.529Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:32 smithi006 bash[33918]: cluster 2023-08-28T01:13:31.762736+0000 mgr.y (mgr.14901) 5848 : cluster [DBG] pgmap v5855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:32.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:32 smithi006 bash[33918]: audit 2023-08-28T01:13:32.429028+0000 mon.c (mon.2) 2280 : audit [DBG] from='client.? 172.21.15.6:0/3888686390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:32 smithi130 bash[27579]: cluster 2023-08-28T01:13:31.762736+0000 mgr.y (mgr.14901) 5848 : cluster [DBG] pgmap v5855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:33.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:32 smithi130 bash[27579]: audit 2023-08-28T01:13:32.429028+0000 mon.c (mon.2) 2280 : audit [DBG] from='client.? 172.21.15.6:0/3888686390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:33.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:33 smithi130 bash[51302]: debug 2023-08-28T01:13:33.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:33.340562+0000) 2023-08-28T01:13:34.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:34 smithi130 bash[51302]: debug 2023-08-28T01:13:34.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:34.340888+0000) 2023-08-28T01:13:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:13:34.409Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:34.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:13:34.411Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:35.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:34 smithi130 bash[27579]: cluster 2023-08-28T01:13:33.763420+0000 mgr.y (mgr.14901) 5849 : cluster [DBG] pgmap v5856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:35.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:34 smithi006 bash[33918]: cluster 2023-08-28T01:13:33.763420+0000 mgr.y (mgr.14901) 5849 : cluster [DBG] pgmap v5856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:35.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:34 smithi006 bash[27292]: cluster 2023-08-28T01:13:33.763420+0000 mgr.y (mgr.14901) 5849 : cluster [DBG] pgmap v5856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:35.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:35 smithi130 bash[51302]: debug 2023-08-28T01:13:35.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:35.341266+0000) 2023-08-28T01:13:36.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:35 smithi130 bash[27579]: audit 2023-08-28T01:13:34.843551+0000 mon.c (mon.2) 2281 : audit [DBG] from='client.? 172.21.15.6:0/3223284323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:36.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:35 smithi006 bash[33918]: audit 2023-08-28T01:13:34.843551+0000 mon.c (mon.2) 2281 : audit [DBG] from='client.? 172.21.15.6:0/3223284323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:36.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:35 smithi006 bash[27292]: audit 2023-08-28T01:13:34.843551+0000 mon.c (mon.2) 2281 : audit [DBG] from='client.? 172.21.15.6:0/3223284323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:36.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:36 smithi130 bash[51302]: debug 2023-08-28T01:13:36.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:36.341629+0000) 2023-08-28T01:13:37.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:36 smithi130 bash[27579]: cluster 2023-08-28T01:13:35.764549+0000 mgr.y (mgr.14901) 5850 : cluster [DBG] pgmap v5857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:37.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:36 smithi006 bash[33918]: cluster 2023-08-28T01:13:35.764549+0000 mgr.y (mgr.14901) 5850 : cluster [DBG] pgmap v5857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:37.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:36 smithi006 bash[27292]: cluster 2023-08-28T01:13:35.764549+0000 mgr.y (mgr.14901) 5850 : cluster [DBG] pgmap v5857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:37.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:37 smithi130 bash[51302]: debug 2023-08-28T01:13:37.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:37.341958+0000) 2023-08-28T01:13:38.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:37 smithi130 bash[27579]: audit 2023-08-28T01:13:37.265493+0000 mon.c (mon.2) 2282 : audit [DBG] from='client.? 172.21.15.6:0/245206014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:38.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:37 smithi006 bash[33918]: audit 2023-08-28T01:13:37.265493+0000 mon.c (mon.2) 2282 : audit [DBG] from='client.? 172.21.15.6:0/245206014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:38.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:37 smithi006 bash[27292]: audit 2023-08-28T01:13:37.265493+0000 mon.c (mon.2) 2282 : audit [DBG] from='client.? 172.21.15.6:0/245206014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:38.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:38 smithi130 bash[51302]: debug 2023-08-28T01:13:38.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:38.342297+0000) 2023-08-28T01:13:39.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:38 smithi130 bash[27579]: cluster 2023-08-28T01:13:37.765254+0000 mgr.y (mgr.14901) 5851 : cluster [DBG] pgmap v5858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:39.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:38 smithi006 bash[27292]: cluster 2023-08-28T01:13:37.765254+0000 mgr.y (mgr.14901) 5851 : cluster [DBG] pgmap v5858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:39.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:38 smithi006 bash[33918]: cluster 2023-08-28T01:13:37.765254+0000 mgr.y (mgr.14901) 5851 : cluster [DBG] pgmap v5858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:39.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:39 smithi130 bash[51302]: debug 2023-08-28T01:13:39.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:39.342631+0000) 2023-08-28T01:13:40.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:39 smithi130 bash[27579]: audit 2023-08-28T01:13:39.674924+0000 mon.a (mon.0) 3971 : audit [DBG] from='client.? 172.21.15.6:0/1970865207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:40.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:39 smithi006 bash[27292]: audit 2023-08-28T01:13:39.674924+0000 mon.a (mon.0) 3971 : audit [DBG] from='client.? 172.21.15.6:0/1970865207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:40.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:39 smithi006 bash[33918]: audit 2023-08-28T01:13:39.674924+0000 mon.a (mon.0) 3971 : audit [DBG] from='client.? 172.21.15.6:0/1970865207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:40.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:40 smithi130 bash[51302]: debug 2023-08-28T01:13:40.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:40.343004+0000) 2023-08-28T01:13:41.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:40 smithi130 bash[27579]: cluster 2023-08-28T01:13:39.766349+0000 mgr.y (mgr.14901) 5852 : cluster [DBG] pgmap v5859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:41.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:40 smithi130 bash[27579]: audit 2023-08-28T01:13:39.856232+0000 mon.b (mon.1) 680 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:13:41.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:13:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:13:40] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:13:41.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:40 smithi006 bash[27292]: cluster 2023-08-28T01:13:39.766349+0000 mgr.y (mgr.14901) 5852 : cluster [DBG] pgmap v5859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:41.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:40 smithi006 bash[27292]: audit 2023-08-28T01:13:39.856232+0000 mon.b (mon.1) 680 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:13:41.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:40 smithi006 bash[33918]: cluster 2023-08-28T01:13:39.766349+0000 mgr.y (mgr.14901) 5852 : cluster [DBG] pgmap v5859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:41.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:40 smithi006 bash[33918]: audit 2023-08-28T01:13:39.856232+0000 mon.b (mon.1) 680 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:13:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:13:40.957Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:13:40.959Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:41.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:41 smithi130 bash[51302]: debug 2023-08-28T01:13:41.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:41.343332+0000) 2023-08-28T01:13:42.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:42 smithi130 bash[51302]: debug 2023-08-28T01:13:42.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:42.343635+0000) 2023-08-28T01:13:42.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:42 smithi006 bash[33918]: cluster 2023-08-28T01:13:41.766976+0000 mgr.y (mgr.14901) 5853 : cluster [DBG] pgmap v5860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:42.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:42 smithi006 bash[33918]: audit 2023-08-28T01:13:42.088240+0000 mon.c (mon.2) 2283 : audit [DBG] from='client.? 172.21.15.6:0/711243191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:13:42.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:13:42.531Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:42 smithi006 bash[27292]: cluster 2023-08-28T01:13:41.766976+0000 mgr.y (mgr.14901) 5853 : cluster [DBG] pgmap v5860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:42.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:42 smithi006 bash[27292]: audit 2023-08-28T01:13:42.088240+0000 mon.c (mon.2) 2283 : audit [DBG] from='client.? 172.21.15.6:0/711243191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:42 smithi130 bash[27579]: cluster 2023-08-28T01:13:41.766976+0000 mgr.y (mgr.14901) 5853 : cluster [DBG] pgmap v5860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:42 smithi130 bash[27579]: audit 2023-08-28T01:13:42.088240+0000 mon.c (mon.2) 2283 : audit [DBG] from='client.? 172.21.15.6:0/711243191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:43.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:43 smithi130 bash[51302]: debug 2023-08-28T01:13:43.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:43.343964+0000) 2023-08-28T01:13:44.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:44 smithi130 bash[51302]: debug 2023-08-28T01:13:44.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:44.344280+0000) 2023-08-28T01:13:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:13:44.410Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:44.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:13:44.412Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:45.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:44 smithi130 bash[27579]: cluster 2023-08-28T01:13:43.767646+0000 mgr.y (mgr.14901) 5854 : cluster [DBG] pgmap v5861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:45.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:44 smithi130 bash[27579]: audit 2023-08-28T01:13:44.516590+0000 mon.c (mon.2) 2284 : audit [DBG] from='client.? 172.21.15.6:0/1522722901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:45.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:44 smithi006 bash[33918]: cluster 2023-08-28T01:13:43.767646+0000 mgr.y (mgr.14901) 5854 : cluster [DBG] pgmap v5861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:45.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:44 smithi006 bash[33918]: audit 2023-08-28T01:13:44.516590+0000 mon.c (mon.2) 2284 : audit [DBG] from='client.? 172.21.15.6:0/1522722901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:45.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:44 smithi006 bash[27292]: cluster 2023-08-28T01:13:43.767646+0000 mgr.y (mgr.14901) 5854 : cluster [DBG] pgmap v5861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:45.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:44 smithi006 bash[27292]: audit 2023-08-28T01:13:44.516590+0000 mon.c (mon.2) 2284 : audit [DBG] from='client.? 172.21.15.6:0/1522722901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:45.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:45 smithi130 bash[51302]: debug 2023-08-28T01:13:45.340+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:45.344515+0000) 2023-08-28T01:13:46.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:46 smithi130 bash[51302]: debug 2023-08-28T01:13:46.339+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:46.344793+0000) 2023-08-28T01:13:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:46 smithi130 bash[27579]: audit 2023-08-28T01:13:45.764006+0000 mon.a (mon.0) 3972 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:13:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:46 smithi130 bash[27579]: cluster 2023-08-28T01:13:45.768454+0000 mgr.y (mgr.14901) 5855 : cluster [DBG] pgmap v5862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:46 smithi130 bash[27579]: audit 2023-08-28T01:13:46.157116+0000 mon.a (mon.0) 3973 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:13:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:46 smithi130 bash[27579]: audit 2023-08-28T01:13:46.421372+0000 mon.b (mon.1) 681 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:13:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:46 smithi130 bash[27579]: audit 2023-08-28T01:13:46.423408+0000 mon.b (mon.1) 682 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:13:47.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:46 smithi130 bash[27579]: audit 2023-08-28T01:13:46.431327+0000 mon.a (mon.0) 3974 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:13:47.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:46 smithi006 bash[27292]: audit 2023-08-28T01:13:45.764006+0000 mon.a (mon.0) 3972 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:13:47.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:46 smithi006 bash[27292]: cluster 2023-08-28T01:13:45.768454+0000 mgr.y (mgr.14901) 5855 : cluster [DBG] pgmap v5862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:47.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:46 smithi006 bash[27292]: audit 2023-08-28T01:13:46.157116+0000 mon.a (mon.0) 3973 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:13:47.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:46 smithi006 bash[27292]: audit 2023-08-28T01:13:46.421372+0000 mon.b (mon.1) 681 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:13:47.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:46 smithi006 bash[27292]: audit 2023-08-28T01:13:46.423408+0000 mon.b (mon.1) 682 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:13:47.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:46 smithi006 bash[27292]: audit 2023-08-28T01:13:46.431327+0000 mon.a (mon.0) 3974 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:13:47.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:46 smithi006 bash[33918]: audit 2023-08-28T01:13:45.764006+0000 mon.a (mon.0) 3972 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:13:47.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:46 smithi006 bash[33918]: cluster 2023-08-28T01:13:45.768454+0000 mgr.y (mgr.14901) 5855 : cluster [DBG] pgmap v5862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:47.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:46 smithi006 bash[33918]: audit 2023-08-28T01:13:46.157116+0000 mon.a (mon.0) 3973 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:13:47.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:46 smithi006 bash[33918]: audit 2023-08-28T01:13:46.421372+0000 mon.b (mon.1) 681 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:13:47.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:46 smithi006 bash[33918]: audit 2023-08-28T01:13:46.423408+0000 mon.b (mon.1) 682 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:13:47.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:46 smithi006 bash[33918]: audit 2023-08-28T01:13:46.431327+0000 mon.a (mon.0) 3974 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:13:47.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:47 smithi130 bash[51302]: debug 2023-08-28T01:13:47.339+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:47.345110+0000) 2023-08-28T01:13:48.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:47 smithi130 bash[27579]: audit 2023-08-28T01:13:46.932479+0000 mon.a (mon.0) 3975 : audit [DBG] from='client.? 172.21.15.6:0/298382535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:48.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:47 smithi006 bash[27292]: audit 2023-08-28T01:13:46.932479+0000 mon.a (mon.0) 3975 : audit [DBG] from='client.? 172.21.15.6:0/298382535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:48.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:47 smithi006 bash[33918]: audit 2023-08-28T01:13:46.932479+0000 mon.a (mon.0) 3975 : audit [DBG] from='client.? 172.21.15.6:0/298382535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:48.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:48 smithi130 bash[51302]: debug 2023-08-28T01:13:48.343+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:48.345433+0000) 2023-08-28T01:13:49.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:48 smithi130 bash[27579]: cluster 2023-08-28T01:13:47.769073+0000 mgr.y (mgr.14901) 5856 : cluster [DBG] pgmap v5863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:49.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:48 smithi006 bash[27292]: cluster 2023-08-28T01:13:47.769073+0000 mgr.y (mgr.14901) 5856 : cluster [DBG] pgmap v5863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:49.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:48 smithi006 bash[33918]: cluster 2023-08-28T01:13:47.769073+0000 mgr.y (mgr.14901) 5856 : cluster [DBG] pgmap v5863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:49.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:49 smithi130 bash[51302]: debug 2023-08-28T01:13:49.339+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:49.345752+0000) 2023-08-28T01:13:50.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:49 smithi130 bash[27579]: audit 2023-08-28T01:13:49.346379+0000 mon.c (mon.2) 2285 : audit [DBG] from='client.? 172.21.15.6:0/3786459430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:50.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:49 smithi006 bash[33918]: audit 2023-08-28T01:13:49.346379+0000 mon.c (mon.2) 2285 : audit [DBG] from='client.? 172.21.15.6:0/3786459430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:50.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:49 smithi006 bash[27292]: audit 2023-08-28T01:13:49.346379+0000 mon.c (mon.2) 2285 : audit [DBG] from='client.? 172.21.15.6:0/3786459430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:50.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:50 smithi130 bash[51302]: debug 2023-08-28T01:13:50.339+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:50.346082+0000) 2023-08-28T01:13:51.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:50 smithi130 bash[27579]: cluster 2023-08-28T01:13:49.770195+0000 mgr.y (mgr.14901) 5857 : cluster [DBG] pgmap v5864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:51.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:50 smithi006 bash[27292]: cluster 2023-08-28T01:13:49.770195+0000 mgr.y (mgr.14901) 5857 : cluster [DBG] pgmap v5864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:51.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:50 smithi006 bash[33918]: cluster 2023-08-28T01:13:49.770195+0000 mgr.y (mgr.14901) 5857 : cluster [DBG] pgmap v5864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:13:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:13:50] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:13:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:13:50.957Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:13:50.959Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:51.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:51 smithi130 bash[51302]: debug 2023-08-28T01:13:51.339+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:51.346498+0000) 2023-08-28T01:13:52.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:52 smithi130 bash[51302]: debug 2023-08-28T01:13:52.339+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:52.346797+0000) 2023-08-28T01:13:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:13:52.530Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:13:52.531Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:52 smithi006 bash[27292]: audit 2023-08-28T01:13:51.763759+0000 mon.a (mon.0) 3976 : audit [DBG] from='client.? 172.21.15.6:0/1037760287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:52 smithi006 bash[27292]: cluster 2023-08-28T01:13:51.770817+0000 mgr.y (mgr.14901) 5858 : cluster [DBG] pgmap v5865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:52 smithi006 bash[33918]: audit 2023-08-28T01:13:51.763759+0000 mon.a (mon.0) 3976 : audit [DBG] from='client.? 172.21.15.6:0/1037760287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:52.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:52 smithi006 bash[33918]: cluster 2023-08-28T01:13:51.770817+0000 mgr.y (mgr.14901) 5858 : cluster [DBG] pgmap v5865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:52 smithi130 bash[27579]: audit 2023-08-28T01:13:51.763759+0000 mon.a (mon.0) 3976 : audit [DBG] from='client.? 172.21.15.6:0/1037760287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:52 smithi130 bash[27579]: cluster 2023-08-28T01:13:51.770817+0000 mgr.y (mgr.14901) 5858 : cluster [DBG] pgmap v5865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:53.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:53 smithi130 bash[51302]: debug 2023-08-28T01:13:53.343+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:53.347042+0000) 2023-08-28T01:13:54.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:54 smithi130 bash[51302]: debug 2023-08-28T01:13:54.343+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:54.347289+0000) 2023-08-28T01:13:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:13:54.410Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:54.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:13:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:13:54.412Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:13:55.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:54 smithi130 bash[27579]: cluster 2023-08-28T01:13:53.771467+0000 mgr.y (mgr.14901) 5859 : cluster [DBG] pgmap v5866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:55.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:54 smithi130 bash[27579]: audit 2023-08-28T01:13:54.181158+0000 mon.c (mon.2) 2286 : audit [DBG] from='client.? 172.21.15.6:0/4216643732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:55.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:54 smithi006 bash[33918]: cluster 2023-08-28T01:13:53.771467+0000 mgr.y (mgr.14901) 5859 : cluster [DBG] pgmap v5866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:55.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:54 smithi006 bash[33918]: audit 2023-08-28T01:13:54.181158+0000 mon.c (mon.2) 2286 : audit [DBG] from='client.? 172.21.15.6:0/4216643732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:55.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:54 smithi006 bash[27292]: cluster 2023-08-28T01:13:53.771467+0000 mgr.y (mgr.14901) 5859 : cluster [DBG] pgmap v5866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:55.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:54 smithi006 bash[27292]: audit 2023-08-28T01:13:54.181158+0000 mon.c (mon.2) 2286 : audit [DBG] from='client.? 172.21.15.6:0/4216643732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:55.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:55 smithi130 bash[51302]: debug 2023-08-28T01:13:55.343+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:55.347588+0000) 2023-08-28T01:13:56.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:56 smithi130 bash[51302]: debug 2023-08-28T01:13:56.343+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:56.347852+0000) 2023-08-28T01:13:57.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:56 smithi130 bash[27579]: cluster 2023-08-28T01:13:55.772579+0000 mgr.y (mgr.14901) 5860 : cluster [DBG] pgmap v5867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:57.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:56 smithi130 bash[27579]: audit 2023-08-28T01:13:56.591341+0000 mon.a (mon.0) 3977 : audit [DBG] from='client.? 172.21.15.6:0/1556108039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:57.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:56 smithi006 bash[33918]: cluster 2023-08-28T01:13:55.772579+0000 mgr.y (mgr.14901) 5860 : cluster [DBG] pgmap v5867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:57.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:56 smithi006 bash[33918]: audit 2023-08-28T01:13:56.591341+0000 mon.a (mon.0) 3977 : audit [DBG] from='client.? 172.21.15.6:0/1556108039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:57.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:56 smithi006 bash[27292]: cluster 2023-08-28T01:13:55.772579+0000 mgr.y (mgr.14901) 5860 : cluster [DBG] pgmap v5867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:57.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:56 smithi006 bash[27292]: audit 2023-08-28T01:13:56.591341+0000 mon.a (mon.0) 3977 : audit [DBG] from='client.? 172.21.15.6:0/1556108039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:57.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:57 smithi130 bash[51302]: debug 2023-08-28T01:13:57.343+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:57.348191+0000) 2023-08-28T01:13:58.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:58 smithi130 bash[51302]: debug 2023-08-28T01:13:58.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:58.348497+0000) 2023-08-28T01:13:59.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:58 smithi130 bash[27579]: cluster 2023-08-28T01:13:57.773244+0000 mgr.y (mgr.14901) 5861 : cluster [DBG] pgmap v5868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:59.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:58 smithi006 bash[27292]: cluster 2023-08-28T01:13:57.773244+0000 mgr.y (mgr.14901) 5861 : cluster [DBG] pgmap v5868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:59.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:58 smithi006 bash[33918]: cluster 2023-08-28T01:13:57.773244+0000 mgr.y (mgr.14901) 5861 : cluster [DBG] pgmap v5868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:13:59.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:13:59 smithi130 bash[51302]: debug 2023-08-28T01:13:59.343+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:59.348802+0000) 2023-08-28T01:14:00.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:13:59 smithi130 bash[27579]: audit 2023-08-28T01:13:59.005149+0000 mon.c (mon.2) 2287 : audit [DBG] from='client.? 172.21.15.6:0/984852488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:00.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:13:59 smithi006 bash[27292]: audit 2023-08-28T01:13:59.005149+0000 mon.c (mon.2) 2287 : audit [DBG] from='client.? 172.21.15.6:0/984852488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:00.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:13:59 smithi006 bash[33918]: audit 2023-08-28T01:13:59.005149+0000 mon.c (mon.2) 2287 : audit [DBG] from='client.? 172.21.15.6:0/984852488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:00.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:00 smithi130 bash[51302]: debug 2023-08-28T01:14:00.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:00.349092+0000) 2023-08-28T01:14:01.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:00 smithi130 bash[27579]: cluster 2023-08-28T01:13:59.774354+0000 mgr.y (mgr.14901) 5862 : cluster [DBG] pgmap v5869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:00 smithi006 bash[27292]: cluster 2023-08-28T01:13:59.774354+0000 mgr.y (mgr.14901) 5862 : cluster [DBG] pgmap v5869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:14:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:14:00] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:14:01.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:00 smithi006 bash[33918]: cluster 2023-08-28T01:13:59.774354+0000 mgr.y (mgr.14901) 5862 : cluster [DBG] pgmap v5869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:01.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:14:00.958Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:01.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:14:00.960Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:01.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:01 smithi130 bash[51302]: debug 2023-08-28T01:14:01.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:01.349447+0000) 2023-08-28T01:14:02.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:01 smithi130 bash[27579]: audit 2023-08-28T01:14:01.416103+0000 mon.c (mon.2) 2288 : audit [DBG] from='client.? 172.21.15.6:0/2463764653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:02.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:01 smithi006 bash[27292]: audit 2023-08-28T01:14:01.416103+0000 mon.c (mon.2) 2288 : audit [DBG] from='client.? 172.21.15.6:0/2463764653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:02.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:01 smithi006 bash[33918]: audit 2023-08-28T01:14:01.416103+0000 mon.c (mon.2) 2288 : audit [DBG] from='client.? 172.21.15.6:0/2463764653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:02.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:02 smithi130 bash[51302]: debug 2023-08-28T01:14:02.343+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:02.349741+0000) 2023-08-28T01:14:02.856 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:14:02.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:02.856 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:14:02.533Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:02 smithi130 bash[27579]: cluster 2023-08-28T01:14:01.775047+0000 mgr.y (mgr.14901) 5863 : cluster [DBG] pgmap v5870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:03.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:02 smithi006 bash[33918]: cluster 2023-08-28T01:14:01.775047+0000 mgr.y (mgr.14901) 5863 : cluster [DBG] pgmap v5870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:03.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:02 smithi006 bash[27292]: cluster 2023-08-28T01:14:01.775047+0000 mgr.y (mgr.14901) 5863 : cluster [DBG] pgmap v5870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:03.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:03 smithi130 bash[51302]: debug 2023-08-28T01:14:03.343+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:03.350008+0000) 2023-08-28T01:14:04.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:03 smithi130 bash[27579]: audit 2023-08-28T01:14:03.835444+0000 mon.c (mon.2) 2289 : audit [DBG] from='client.? 172.21.15.6:0/1152284740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:04.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:03 smithi006 bash[33918]: audit 2023-08-28T01:14:03.835444+0000 mon.c (mon.2) 2289 : audit [DBG] from='client.? 172.21.15.6:0/1152284740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:04.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:03 smithi006 bash[27292]: audit 2023-08-28T01:14:03.835444+0000 mon.c (mon.2) 2289 : audit [DBG] from='client.? 172.21.15.6:0/1152284740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:04.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:04 smithi130 bash[51302]: debug 2023-08-28T01:14:04.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:04.350331+0000) 2023-08-28T01:14:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:14:04.410Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:14:04.413Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:05.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:04 smithi006 bash[33918]: cluster 2023-08-28T01:14:03.775706+0000 mgr.y (mgr.14901) 5864 : cluster [DBG] pgmap v5871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:05.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:04 smithi006 bash[27292]: cluster 2023-08-28T01:14:03.775706+0000 mgr.y (mgr.14901) 5864 : cluster [DBG] pgmap v5871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:05.350 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:04 smithi130 bash[27579]: cluster 2023-08-28T01:14:03.775706+0000 mgr.y (mgr.14901) 5864 : cluster [DBG] pgmap v5871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:05.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:05 smithi130 bash[51302]: debug 2023-08-28T01:14:05.343+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:05.350603+0000) 2023-08-28T01:14:06.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:06 smithi130 bash[51302]: debug 2023-08-28T01:14:06.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:06.350898+0000) 2023-08-28T01:14:07.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:06 smithi006 bash[33918]: cluster 2023-08-28T01:14:05.776861+0000 mgr.y (mgr.14901) 5865 : cluster [DBG] pgmap v5872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:07.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:06 smithi006 bash[33918]: audit 2023-08-28T01:14:06.248408+0000 mon.c (mon.2) 2290 : audit [DBG] from='client.? 172.21.15.6:0/2718246919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:06 smithi006 bash[27292]: cluster 2023-08-28T01:14:05.776861+0000 mgr.y (mgr.14901) 5865 : cluster [DBG] pgmap v5872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:06 smithi006 bash[27292]: audit 2023-08-28T01:14:06.248408+0000 mon.c (mon.2) 2290 : audit [DBG] from='client.? 172.21.15.6:0/2718246919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:07.350 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:06 smithi130 bash[27579]: cluster 2023-08-28T01:14:05.776861+0000 mgr.y (mgr.14901) 5865 : cluster [DBG] pgmap v5872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:07.351 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:06 smithi130 bash[27579]: audit 2023-08-28T01:14:06.248408+0000 mon.c (mon.2) 2290 : audit [DBG] from='client.? 172.21.15.6:0/2718246919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:07.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:07 smithi130 bash[51302]: debug 2023-08-28T01:14:07.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:07.351192+0000) 2023-08-28T01:14:08.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:08 smithi130 bash[51302]: debug 2023-08-28T01:14:08.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:08.351457+0000) 2023-08-28T01:14:09.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:08 smithi006 bash[27292]: cluster 2023-08-28T01:14:07.777520+0000 mgr.y (mgr.14901) 5866 : cluster [DBG] pgmap v5873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:09.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:08 smithi006 bash[27292]: audit 2023-08-28T01:14:08.666300+0000 mon.c (mon.2) 2291 : audit [DBG] from='client.? 172.21.15.6:0/2539692933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:09.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:08 smithi006 bash[33918]: cluster 2023-08-28T01:14:07.777520+0000 mgr.y (mgr.14901) 5866 : cluster [DBG] pgmap v5873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:09.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:08 smithi006 bash[33918]: audit 2023-08-28T01:14:08.666300+0000 mon.c (mon.2) 2291 : audit [DBG] from='client.? 172.21.15.6:0/2539692933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:09.351 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:08 smithi130 bash[27579]: cluster 2023-08-28T01:14:07.777520+0000 mgr.y (mgr.14901) 5866 : cluster [DBG] pgmap v5873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:09.351 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:08 smithi130 bash[27579]: audit 2023-08-28T01:14:08.666300+0000 mon.c (mon.2) 2291 : audit [DBG] from='client.? 172.21.15.6:0/2539692933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:09.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:09 smithi130 bash[51302]: debug 2023-08-28T01:14:09.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:09.351793+0000) 2023-08-28T01:14:10.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:10 smithi130 bash[51302]: debug 2023-08-28T01:14:10.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:10.352086+0000) 2023-08-28T01:14:11.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:10 smithi006 bash[27292]: cluster 2023-08-28T01:14:09.778613+0000 mgr.y (mgr.14901) 5867 : cluster [DBG] pgmap v5874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:14:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:14:10] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:14:11.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:10 smithi006 bash[33918]: cluster 2023-08-28T01:14:09.778613+0000 mgr.y (mgr.14901) 5867 : cluster [DBG] pgmap v5874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:11.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:14:10.959Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:14:10.961Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:11.352 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:10 smithi130 bash[27579]: cluster 2023-08-28T01:14:09.778613+0000 mgr.y (mgr.14901) 5867 : cluster [DBG] pgmap v5874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:11.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:11 smithi130 bash[51302]: debug 2023-08-28T01:14:11.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:11.352345+0000) 2023-08-28T01:14:12.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:11 smithi006 bash[27292]: audit 2023-08-28T01:14:11.078715+0000 mon.a (mon.0) 3978 : audit [DBG] from='client.? 172.21.15.6:0/2186740957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:12.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:11 smithi006 bash[33918]: audit 2023-08-28T01:14:11.078715+0000 mon.a (mon.0) 3978 : audit [DBG] from='client.? 172.21.15.6:0/2186740957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:12.352 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:11 smithi130 bash[27579]: audit 2023-08-28T01:14:11.078715+0000 mon.a (mon.0) 3978 : audit [DBG] from='client.? 172.21.15.6:0/2186740957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:12.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:12 smithi130 bash[51302]: debug 2023-08-28T01:14:12.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:12.352614+0000) 2023-08-28T01:14:12.890 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:14:12.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:12.890 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:14:12.533Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:13.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:12 smithi006 bash[27292]: cluster 2023-08-28T01:14:11.779271+0000 mgr.y (mgr.14901) 5868 : cluster [DBG] pgmap v5875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:13.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:12 smithi006 bash[33918]: cluster 2023-08-28T01:14:11.779271+0000 mgr.y (mgr.14901) 5868 : cluster [DBG] pgmap v5875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:13.352 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:12 smithi130 bash[27579]: cluster 2023-08-28T01:14:11.779271+0000 mgr.y (mgr.14901) 5868 : cluster [DBG] pgmap v5875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:13.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:13 smithi130 bash[51302]: debug 2023-08-28T01:14:13.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:13.352924+0000) 2023-08-28T01:14:14.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:13 smithi006 bash[27292]: audit 2023-08-28T01:14:13.497433+0000 mon.c (mon.2) 2292 : audit [DBG] from='client.? 172.21.15.6:0/2726360299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:14.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:13 smithi006 bash[33918]: audit 2023-08-28T01:14:13.497433+0000 mon.c (mon.2) 2292 : audit [DBG] from='client.? 172.21.15.6:0/2726360299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:14.353 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:13 smithi130 bash[27579]: audit 2023-08-28T01:14:13.497433+0000 mon.c (mon.2) 2292 : audit [DBG] from='client.? 172.21.15.6:0/2726360299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:14.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:14 smithi130 bash[51302]: debug 2023-08-28T01:14:14.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:14.353246+0000) 2023-08-28T01:14:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:14:14.411Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:14:14.414Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:15.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:14 smithi006 bash[27292]: cluster 2023-08-28T01:14:13.779906+0000 mgr.y (mgr.14901) 5869 : cluster [DBG] pgmap v5876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:15.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:14 smithi006 bash[33918]: cluster 2023-08-28T01:14:13.779906+0000 mgr.y (mgr.14901) 5869 : cluster [DBG] pgmap v5876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:15.353 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:14 smithi130 bash[27579]: cluster 2023-08-28T01:14:13.779906+0000 mgr.y (mgr.14901) 5869 : cluster [DBG] pgmap v5876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:15.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:15 smithi130 bash[51302]: debug 2023-08-28T01:14:15.351+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:15.353577+0000) 2023-08-28T01:14:16.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:16 smithi130 bash[51302]: debug 2023-08-28T01:14:16.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:16.353910+0000) 2023-08-28T01:14:17.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:16 smithi006 bash[27292]: cluster 2023-08-28T01:14:15.780810+0000 mgr.y (mgr.14901) 5870 : cluster [DBG] pgmap v5877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:17.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:16 smithi006 bash[27292]: audit 2023-08-28T01:14:15.914266+0000 mon.a (mon.0) 3979 : audit [DBG] from='client.? 172.21.15.6:0/944149790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:17.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:16 smithi006 bash[33918]: cluster 2023-08-28T01:14:15.780810+0000 mgr.y (mgr.14901) 5870 : cluster [DBG] pgmap v5877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:17.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:16 smithi006 bash[33918]: audit 2023-08-28T01:14:15.914266+0000 mon.a (mon.0) 3979 : audit [DBG] from='client.? 172.21.15.6:0/944149790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:17.354 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:16 smithi130 bash[27579]: cluster 2023-08-28T01:14:15.780810+0000 mgr.y (mgr.14901) 5870 : cluster [DBG] pgmap v5877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:17.354 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:16 smithi130 bash[27579]: audit 2023-08-28T01:14:15.914266+0000 mon.a (mon.0) 3979 : audit [DBG] from='client.? 172.21.15.6:0/944149790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:17.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:17 smithi130 bash[51302]: debug 2023-08-28T01:14:17.347+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:17.354214+0000) 2023-08-28T01:14:18.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:18 smithi130 bash[51302]: debug 2023-08-28T01:14:18.351+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:18.354596+0000) 2023-08-28T01:14:19.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:18 smithi006 bash[33918]: cluster 2023-08-28T01:14:17.781351+0000 mgr.y (mgr.14901) 5871 : cluster [DBG] pgmap v5878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:19.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:18 smithi006 bash[33918]: audit 2023-08-28T01:14:18.322365+0000 mon.a (mon.0) 3980 : audit [DBG] from='client.? 172.21.15.6:0/3530709360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:19.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:18 smithi006 bash[27292]: cluster 2023-08-28T01:14:17.781351+0000 mgr.y (mgr.14901) 5871 : cluster [DBG] pgmap v5878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:19.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:18 smithi006 bash[27292]: audit 2023-08-28T01:14:18.322365+0000 mon.a (mon.0) 3980 : audit [DBG] from='client.? 172.21.15.6:0/3530709360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:19.354 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:18 smithi130 bash[27579]: cluster 2023-08-28T01:14:17.781351+0000 mgr.y (mgr.14901) 5871 : cluster [DBG] pgmap v5878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:19.354 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:18 smithi130 bash[27579]: audit 2023-08-28T01:14:18.322365+0000 mon.a (mon.0) 3980 : audit [DBG] from='client.? 172.21.15.6:0/3530709360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:19.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:19 smithi130 bash[51302]: debug 2023-08-28T01:14:19.351+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:19.354943+0000) 2023-08-28T01:14:20.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:20 smithi130 bash[51302]: debug 2023-08-28T01:14:20.351+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:20.356893+0000) 2023-08-28T01:14:21.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:14:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:14:20] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:14:21.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:20 smithi006 bash[27292]: cluster 2023-08-28T01:14:19.782394+0000 mgr.y (mgr.14901) 5872 : cluster [DBG] pgmap v5879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:21.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:20 smithi006 bash[27292]: audit 2023-08-28T01:14:20.744563+0000 mon.c (mon.2) 2293 : audit [DBG] from='client.? 172.21.15.6:0/3688977082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:21.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:20 smithi006 bash[33918]: cluster 2023-08-28T01:14:19.782394+0000 mgr.y (mgr.14901) 5872 : cluster [DBG] pgmap v5879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:21.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:20 smithi006 bash[33918]: audit 2023-08-28T01:14:20.744563+0000 mon.c (mon.2) 2293 : audit [DBG] from='client.? 172.21.15.6:0/3688977082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:14:20.959Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:14:20.961Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:21.356 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:20 smithi130 bash[27579]: cluster 2023-08-28T01:14:19.782394+0000 mgr.y (mgr.14901) 5872 : cluster [DBG] pgmap v5879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:21.356 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:20 smithi130 bash[27579]: audit 2023-08-28T01:14:20.744563+0000 mon.c (mon.2) 2293 : audit [DBG] from='client.? 172.21.15.6:0/3688977082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:21.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:21 smithi130 bash[51302]: debug 2023-08-28T01:14:21.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:21.357172+0000) 2023-08-28T01:14:22.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:22 smithi130 bash[51302]: debug 2023-08-28T01:14:22.351+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:22.357462+0000) 2023-08-28T01:14:22.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:22 smithi006 bash[33918]: cluster 2023-08-28T01:14:21.783052+0000 mgr.y (mgr.14901) 5873 : cluster [DBG] pgmap v5880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:22.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:22 smithi006 bash[27292]: cluster 2023-08-28T01:14:21.783052+0000 mgr.y (mgr.14901) 5873 : cluster [DBG] pgmap v5880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:22.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:14:22.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:22.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:14:22.535Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:23.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:22 smithi130 bash[27579]: cluster 2023-08-28T01:14:21.783052+0000 mgr.y (mgr.14901) 5873 : cluster [DBG] pgmap v5880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:23.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:23 smithi130 bash[51302]: debug 2023-08-28T01:14:23.351+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:23.357741+0000) 2023-08-28T01:14:23.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:23 smithi006 bash[33918]: audit 2023-08-28T01:14:23.161370+0000 mon.c (mon.2) 2294 : audit [DBG] from='client.? 172.21.15.6:0/2873339157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:23.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:23 smithi006 bash[27292]: audit 2023-08-28T01:14:23.161370+0000 mon.c (mon.2) 2294 : audit [DBG] from='client.? 172.21.15.6:0/2873339157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:24.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:23 smithi130 bash[27579]: audit 2023-08-28T01:14:23.161370+0000 mon.c (mon.2) 2294 : audit [DBG] from='client.? 172.21.15.6:0/2873339157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:24.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:24 smithi130 bash[51302]: debug 2023-08-28T01:14:24.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:24.358016+0000) 2023-08-28T01:14:24.676 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:14:24.412Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:24.676 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:14:24.414Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:24.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:24 smithi006 bash[33918]: cluster 2023-08-28T01:14:23.783713+0000 mgr.y (mgr.14901) 5874 : cluster [DBG] pgmap v5881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:24.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:24 smithi006 bash[27292]: cluster 2023-08-28T01:14:23.783713+0000 mgr.y (mgr.14901) 5874 : cluster [DBG] pgmap v5881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:25.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:24 smithi130 bash[27579]: cluster 2023-08-28T01:14:23.783713+0000 mgr.y (mgr.14901) 5874 : cluster [DBG] pgmap v5881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:25.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:25 smithi130 bash[51302]: debug 2023-08-28T01:14:25.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:25.358298+0000) 2023-08-28T01:14:25.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:25 smithi006 bash[33918]: audit 2023-08-28T01:14:25.571233+0000 mon.a (mon.0) 3981 : audit [DBG] from='client.? 172.21.15.6:0/2162096156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:25.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:25 smithi006 bash[27292]: audit 2023-08-28T01:14:25.571233+0000 mon.a (mon.0) 3981 : audit [DBG] from='client.? 172.21.15.6:0/2162096156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:26.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:25 smithi130 bash[27579]: audit 2023-08-28T01:14:25.571233+0000 mon.a (mon.0) 3981 : audit [DBG] from='client.? 172.21.15.6:0/2162096156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:26.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:26 smithi130 bash[51302]: debug 2023-08-28T01:14:26.351+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:26.358511+0000) 2023-08-28T01:14:26.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:26 smithi006 bash[33918]: cluster 2023-08-28T01:14:25.784887+0000 mgr.y (mgr.14901) 5875 : cluster [DBG] pgmap v5882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:26.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:26 smithi006 bash[27292]: cluster 2023-08-28T01:14:25.784887+0000 mgr.y (mgr.14901) 5875 : cluster [DBG] pgmap v5882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:27.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:26 smithi130 bash[27579]: cluster 2023-08-28T01:14:25.784887+0000 mgr.y (mgr.14901) 5875 : cluster [DBG] pgmap v5882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:27.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:27 smithi130 bash[51302]: debug 2023-08-28T01:14:27.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:27.358788+0000) 2023-08-28T01:14:28.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:28 smithi130 bash[51302]: debug 2023-08-28T01:14:28.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:28.359092+0000) 2023-08-28T01:14:29.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:28 smithi130 bash[27579]: cluster 2023-08-28T01:14:27.785714+0000 mgr.y (mgr.14901) 5876 : cluster [DBG] pgmap v5883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:29.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:28 smithi130 bash[27579]: audit 2023-08-28T01:14:27.989101+0000 mon.c (mon.2) 2295 : audit [DBG] from='client.? 172.21.15.6:0/3003170937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:29.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:28 smithi006 bash[33918]: cluster 2023-08-28T01:14:27.785714+0000 mgr.y (mgr.14901) 5876 : cluster [DBG] pgmap v5883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:29.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:28 smithi006 bash[33918]: audit 2023-08-28T01:14:27.989101+0000 mon.c (mon.2) 2295 : audit [DBG] from='client.? 172.21.15.6:0/3003170937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:29.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:28 smithi006 bash[27292]: cluster 2023-08-28T01:14:27.785714+0000 mgr.y (mgr.14901) 5876 : cluster [DBG] pgmap v5883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:29.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:28 smithi006 bash[27292]: audit 2023-08-28T01:14:27.989101+0000 mon.c (mon.2) 2295 : audit [DBG] from='client.? 172.21.15.6:0/3003170937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:29.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:29 smithi130 bash[51302]: debug 2023-08-28T01:14:29.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:29.359286+0000) 2023-08-28T01:14:30.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:30 smithi130 bash[51302]: debug 2023-08-28T01:14:30.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:30.359553+0000) 2023-08-28T01:14:31.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:30 smithi130 bash[27579]: cluster 2023-08-28T01:14:29.786876+0000 mgr.y (mgr.14901) 5877 : cluster [DBG] pgmap v5884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:31.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:30 smithi130 bash[27579]: audit 2023-08-28T01:14:30.402079+0000 mon.a (mon.0) 3982 : audit [DBG] from='client.? 172.21.15.6:0/4240221797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:31.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:30 smithi006 bash[27292]: cluster 2023-08-28T01:14:29.786876+0000 mgr.y (mgr.14901) 5877 : cluster [DBG] pgmap v5884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:31.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:30 smithi006 bash[27292]: audit 2023-08-28T01:14:30.402079+0000 mon.a (mon.0) 3982 : audit [DBG] from='client.? 172.21.15.6:0/4240221797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:14:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:14:30] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:14:31.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:30 smithi006 bash[33918]: cluster 2023-08-28T01:14:29.786876+0000 mgr.y (mgr.14901) 5877 : cluster [DBG] pgmap v5884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:31.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:30 smithi006 bash[33918]: audit 2023-08-28T01:14:30.402079+0000 mon.a (mon.0) 3982 : audit [DBG] from='client.? 172.21.15.6:0/4240221797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:14:30.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:30 smithi006 bash[56298]: level=warn ts=2023-08-28T01:14:30.962Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:31.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:31 smithi130 bash[51302]: debug 2023-08-28T01:14:31.359+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:31.359848+0000) 2023-08-28T01:14:32.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:32 smithi130 bash[51302]: debug 2023-08-28T01:14:32.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:32.360106+0000) 2023-08-28T01:14:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:32 smithi006 bash[27292]: cluster 2023-08-28T01:14:31.787576+0000 mgr.y (mgr.14901) 5878 : cluster [DBG] pgmap v5885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:32 smithi006 bash[33918]: cluster 2023-08-28T01:14:31.787576+0000 mgr.y (mgr.14901) 5878 : cluster [DBG] pgmap v5885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:32.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:14:32.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:32.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:14:32.535Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:32 smithi130 bash[27579]: cluster 2023-08-28T01:14:31.787576+0000 mgr.y (mgr.14901) 5878 : cluster [DBG] pgmap v5885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:33.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:33 smithi130 bash[51302]: debug 2023-08-28T01:14:33.359+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:33.360367+0000) 2023-08-28T01:14:33.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:33 smithi006 bash[27292]: audit 2023-08-28T01:14:32.809396+0000 mon.c (mon.2) 2296 : audit [DBG] from='client.? 172.21.15.6:0/2262120879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:33.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:33 smithi006 bash[33918]: audit 2023-08-28T01:14:32.809396+0000 mon.c (mon.2) 2296 : audit [DBG] from='client.? 172.21.15.6:0/2262120879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:34.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:33 smithi130 bash[27579]: audit 2023-08-28T01:14:32.809396+0000 mon.c (mon.2) 2296 : audit [DBG] from='client.? 172.21.15.6:0/2262120879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:34.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:34 smithi130 bash[51302]: debug 2023-08-28T01:14:34.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:34.360628+0000) 2023-08-28T01:14:34.678 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:14:34.412Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:34.679 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:14:34.414Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:34.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:34 smithi006 bash[27292]: cluster 2023-08-28T01:14:33.788271+0000 mgr.y (mgr.14901) 5879 : cluster [DBG] pgmap v5886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:34.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:34 smithi006 bash[33918]: cluster 2023-08-28T01:14:33.788271+0000 mgr.y (mgr.14901) 5879 : cluster [DBG] pgmap v5886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:35.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:34 smithi130 bash[27579]: cluster 2023-08-28T01:14:33.788271+0000 mgr.y (mgr.14901) 5879 : cluster [DBG] pgmap v5886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:35.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:35 smithi130 bash[51302]: debug 2023-08-28T01:14:35.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:35.360961+0000) 2023-08-28T01:14:35.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:35 smithi006 bash[33918]: audit 2023-08-28T01:14:35.223730+0000 mon.c (mon.2) 2297 : audit [DBG] from='client.? 172.21.15.6:0/4179157672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:35.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:35 smithi006 bash[27292]: audit 2023-08-28T01:14:35.223730+0000 mon.c (mon.2) 2297 : audit [DBG] from='client.? 172.21.15.6:0/4179157672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:36.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:35 smithi130 bash[27579]: audit 2023-08-28T01:14:35.223730+0000 mon.c (mon.2) 2297 : audit [DBG] from='client.? 172.21.15.6:0/4179157672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:36.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:36 smithi130 bash[51302]: debug 2023-08-28T01:14:36.359+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:36.361225+0000) 2023-08-28T01:14:36.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:36 smithi006 bash[33918]: cluster 2023-08-28T01:14:35.789419+0000 mgr.y (mgr.14901) 5880 : cluster [DBG] pgmap v5887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:36.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:36 smithi006 bash[27292]: cluster 2023-08-28T01:14:35.789419+0000 mgr.y (mgr.14901) 5880 : cluster [DBG] pgmap v5887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:37.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:36 smithi130 bash[27579]: cluster 2023-08-28T01:14:35.789419+0000 mgr.y (mgr.14901) 5880 : cluster [DBG] pgmap v5887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:37.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:37 smithi130 bash[51302]: debug 2023-08-28T01:14:37.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:37.361475+0000) 2023-08-28T01:14:37.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:37 smithi006 bash[33918]: audit 2023-08-28T01:14:37.632441+0000 mon.c (mon.2) 2298 : audit [DBG] from='client.? 172.21.15.6:0/4094501901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:37.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:37 smithi006 bash[27292]: audit 2023-08-28T01:14:37.632441+0000 mon.c (mon.2) 2298 : audit [DBG] from='client.? 172.21.15.6:0/4094501901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:38.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:37 smithi130 bash[27579]: audit 2023-08-28T01:14:37.632441+0000 mon.c (mon.2) 2298 : audit [DBG] from='client.? 172.21.15.6:0/4094501901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:38.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:38 smithi130 bash[51302]: debug 2023-08-28T01:14:38.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:38.361663+0000) 2023-08-28T01:14:38.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:38 smithi006 bash[27292]: cluster 2023-08-28T01:14:37.790045+0000 mgr.y (mgr.14901) 5881 : cluster [DBG] pgmap v5888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:38.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:38 smithi006 bash[33918]: cluster 2023-08-28T01:14:37.790045+0000 mgr.y (mgr.14901) 5881 : cluster [DBG] pgmap v5888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:39.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:38 smithi130 bash[27579]: cluster 2023-08-28T01:14:37.790045+0000 mgr.y (mgr.14901) 5881 : cluster [DBG] pgmap v5888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:39.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:39 smithi130 bash[51302]: debug 2023-08-28T01:14:39.359+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:39.361927+0000) 2023-08-28T01:14:40.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:40 smithi130 bash[51302]: debug 2023-08-28T01:14:40.359+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:40.362279+0000) 2023-08-28T01:14:41.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:40 smithi130 bash[27579]: cluster 2023-08-28T01:14:39.791069+0000 mgr.y (mgr.14901) 5882 : cluster [DBG] pgmap v5889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:41.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:40 smithi130 bash[27579]: audit 2023-08-28T01:14:40.044888+0000 mon.a (mon.0) 3983 : audit [DBG] from='client.? 172.21.15.6:0/3120316675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:14:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:14:40] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:14:41.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:40 smithi006 bash[27292]: cluster 2023-08-28T01:14:39.791069+0000 mgr.y (mgr.14901) 5882 : cluster [DBG] pgmap v5889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:41.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:40 smithi006 bash[27292]: audit 2023-08-28T01:14:40.044888+0000 mon.a (mon.0) 3983 : audit [DBG] from='client.? 172.21.15.6:0/3120316675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:41.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:40 smithi006 bash[33918]: cluster 2023-08-28T01:14:39.791069+0000 mgr.y (mgr.14901) 5882 : cluster [DBG] pgmap v5889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:41.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:40 smithi006 bash[33918]: audit 2023-08-28T01:14:40.044888+0000 mon.a (mon.0) 3983 : audit [DBG] from='client.? 172.21.15.6:0/3120316675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:14:40.961Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:14:40.963Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:41.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:41 smithi130 bash[51302]: debug 2023-08-28T01:14:41.355+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:41.362552+0000) 2023-08-28T01:14:42.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:42 smithi130 bash[51302]: debug 2023-08-28T01:14:42.358+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:42.362839+0000) 2023-08-28T01:14:42.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:42 smithi006 bash[27292]: cluster 2023-08-28T01:14:41.791761+0000 mgr.y (mgr.14901) 5883 : cluster [DBG] pgmap v5890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:42 smithi006 bash[27292]: audit 2023-08-28T01:14:42.462361+0000 mon.a (mon.0) 3984 : audit [DBG] from='client.? 172.21.15.6:0/1042051809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:42.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:42 smithi006 bash[33918]: cluster 2023-08-28T01:14:41.791761+0000 mgr.y (mgr.14901) 5883 : cluster [DBG] pgmap v5890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:42.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:42 smithi006 bash[33918]: audit 2023-08-28T01:14:42.462361+0000 mon.a (mon.0) 3984 : audit [DBG] from='client.? 172.21.15.6:0/1042051809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:14:42.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:14:42.536Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:42 smithi130 bash[27579]: cluster 2023-08-28T01:14:41.791761+0000 mgr.y (mgr.14901) 5883 : cluster [DBG] pgmap v5890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:42 smithi130 bash[27579]: audit 2023-08-28T01:14:42.462361+0000 mon.a (mon.0) 3984 : audit [DBG] from='client.? 172.21.15.6:0/1042051809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:43.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:43 smithi130 bash[51302]: debug 2023-08-28T01:14:43.358+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:43.363198+0000) 2023-08-28T01:14:44.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:44 smithi130 bash[51302]: debug 2023-08-28T01:14:44.358+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:44.363475+0000) 2023-08-28T01:14:44.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:14:44.413Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:44.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:14:44.415Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:45.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:44 smithi130 bash[27579]: cluster 2023-08-28T01:14:43.792434+0000 mgr.y (mgr.14901) 5884 : cluster [DBG] pgmap v5891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:45.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:44 smithi006 bash[33918]: cluster 2023-08-28T01:14:43.792434+0000 mgr.y (mgr.14901) 5884 : cluster [DBG] pgmap v5891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:45.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:44 smithi006 bash[27292]: cluster 2023-08-28T01:14:43.792434+0000 mgr.y (mgr.14901) 5884 : cluster [DBG] pgmap v5891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:45.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:45 smithi130 bash[51302]: debug 2023-08-28T01:14:45.358+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:45.363765+0000) 2023-08-28T01:14:46.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:45 smithi130 bash[27579]: audit 2023-08-28T01:14:44.881148+0000 mon.a (mon.0) 3985 : audit [DBG] from='client.? 172.21.15.6:0/737263294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:46.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:45 smithi006 bash[27292]: audit 2023-08-28T01:14:44.881148+0000 mon.a (mon.0) 3985 : audit [DBG] from='client.? 172.21.15.6:0/737263294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:46.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:45 smithi006 bash[33918]: audit 2023-08-28T01:14:44.881148+0000 mon.a (mon.0) 3985 : audit [DBG] from='client.? 172.21.15.6:0/737263294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:46.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:46 smithi130 bash[51302]: debug 2023-08-28T01:14:46.362+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:46.364066+0000) 2023-08-28T01:14:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:46 smithi130 bash[27579]: cluster 2023-08-28T01:14:45.793609+0000 mgr.y (mgr.14901) 5885 : cluster [DBG] pgmap v5892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:46 smithi130 bash[27579]: audit 2023-08-28T01:14:46.442252+0000 mon.b (mon.1) 683 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:14:47.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:46 smithi006 bash[27292]: cluster 2023-08-28T01:14:45.793609+0000 mgr.y (mgr.14901) 5885 : cluster [DBG] pgmap v5892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:47.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:46 smithi006 bash[27292]: audit 2023-08-28T01:14:46.442252+0000 mon.b (mon.1) 683 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:14:47.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:46 smithi006 bash[33918]: cluster 2023-08-28T01:14:45.793609+0000 mgr.y (mgr.14901) 5885 : cluster [DBG] pgmap v5892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:47.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:46 smithi006 bash[33918]: audit 2023-08-28T01:14:46.442252+0000 mon.b (mon.1) 683 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:14:47.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:47 smithi130 bash[51302]: debug 2023-08-28T01:14:47.358+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:47.364369+0000) 2023-08-28T01:14:48.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:47 smithi006 bash[33918]: audit 2023-08-28T01:14:47.297361+0000 mon.c (mon.2) 2299 : audit [DBG] from='client.? 172.21.15.6:0/3328315018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:48.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:47 smithi006 bash[27292]: audit 2023-08-28T01:14:47.297361+0000 mon.c (mon.2) 2299 : audit [DBG] from='client.? 172.21.15.6:0/3328315018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:48.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:47 smithi130 bash[27579]: audit 2023-08-28T01:14:47.297361+0000 mon.c (mon.2) 2299 : audit [DBG] from='client.? 172.21.15.6:0/3328315018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:48.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:48 smithi130 bash[51302]: debug 2023-08-28T01:14:48.362+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:48.364676+0000) 2023-08-28T01:14:49.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:48 smithi006 bash[33918]: cluster 2023-08-28T01:14:47.794322+0000 mgr.y (mgr.14901) 5886 : cluster [DBG] pgmap v5893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:49.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:48 smithi006 bash[27292]: cluster 2023-08-28T01:14:47.794322+0000 mgr.y (mgr.14901) 5886 : cluster [DBG] pgmap v5893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:49.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:48 smithi130 bash[27579]: cluster 2023-08-28T01:14:47.794322+0000 mgr.y (mgr.14901) 5886 : cluster [DBG] pgmap v5893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:49.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:49 smithi130 bash[51302]: debug 2023-08-28T01:14:49.362+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:49.364953+0000) 2023-08-28T01:14:50.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:49 smithi006 bash[33918]: audit 2023-08-28T01:14:49.709945+0000 mon.c (mon.2) 2300 : audit [DBG] from='client.? 172.21.15.6:0/1614500523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:50.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:49 smithi006 bash[27292]: audit 2023-08-28T01:14:49.709945+0000 mon.c (mon.2) 2300 : audit [DBG] from='client.? 172.21.15.6:0/1614500523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:50.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:49 smithi130 bash[27579]: audit 2023-08-28T01:14:49.709945+0000 mon.c (mon.2) 2300 : audit [DBG] from='client.? 172.21.15.6:0/1614500523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:50.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:50 smithi130 bash[51302]: debug 2023-08-28T01:14:50.362+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:50.365273+0000) 2023-08-28T01:14:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:14:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:14:50] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:14:51.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:50 smithi006 bash[33918]: cluster 2023-08-28T01:14:49.795476+0000 mgr.y (mgr.14901) 5887 : cluster [DBG] pgmap v5894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:51.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:50 smithi006 bash[27292]: cluster 2023-08-28T01:14:49.795476+0000 mgr.y (mgr.14901) 5887 : cluster [DBG] pgmap v5894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:14:50.961Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:14:50.963Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:51.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:50 smithi130 bash[27579]: cluster 2023-08-28T01:14:49.795476+0000 mgr.y (mgr.14901) 5887 : cluster [DBG] pgmap v5894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:51.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:51 smithi130 bash[51302]: debug 2023-08-28T01:14:51.362+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:51.365503+0000) 2023-08-28T01:14:52.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:52 smithi130 bash[51302]: debug 2023-08-28T01:14:52.362+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:52.365785+0000) 2023-08-28T01:14:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:14:52.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:14:52.537Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:52 smithi006 bash[33918]: cluster 2023-08-28T01:14:51.795958+0000 mgr.y (mgr.14901) 5888 : cluster [DBG] pgmap v5895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:52 smithi006 bash[33918]: audit 2023-08-28T01:14:52.153556+0000 mon.c (mon.2) 2301 : audit [DBG] from='client.? 172.21.15.6:0/1579511904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:52 smithi006 bash[33918]: audit 2023-08-28T01:14:52.291445+0000 mon.a (mon.0) 3986 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:14:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:52 smithi006 bash[27292]: cluster 2023-08-28T01:14:51.795958+0000 mgr.y (mgr.14901) 5888 : cluster [DBG] pgmap v5895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:52 smithi006 bash[27292]: audit 2023-08-28T01:14:52.153556+0000 mon.c (mon.2) 2301 : audit [DBG] from='client.? 172.21.15.6:0/1579511904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:52 smithi006 bash[27292]: audit 2023-08-28T01:14:52.291445+0000 mon.a (mon.0) 3986 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:14:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:52 smithi130 bash[27579]: cluster 2023-08-28T01:14:51.795958+0000 mgr.y (mgr.14901) 5888 : cluster [DBG] pgmap v5895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:52 smithi130 bash[27579]: audit 2023-08-28T01:14:52.153556+0000 mon.c (mon.2) 2301 : audit [DBG] from='client.? 172.21.15.6:0/1579511904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:53.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:52 smithi130 bash[27579]: audit 2023-08-28T01:14:52.291445+0000 mon.a (mon.0) 3986 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:14:53.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:53 smithi130 bash[51302]: debug 2023-08-28T01:14:53.362+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:53.366090+0000) 2023-08-28T01:14:54.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:53 smithi130 bash[27579]: audit 2023-08-28T01:14:52.724761+0000 mon.a (mon.0) 3987 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:14:54.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:53 smithi130 bash[27579]: audit 2023-08-28T01:14:52.994483+0000 mon.b (mon.1) 684 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:14:54.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:53 smithi130 bash[27579]: audit 2023-08-28T01:14:52.996867+0000 mon.b (mon.1) 685 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:14:54.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:53 smithi130 bash[27579]: audit 2023-08-28T01:14:53.006794+0000 mon.a (mon.0) 3988 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:14:54.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:53 smithi006 bash[33918]: audit 2023-08-28T01:14:52.724761+0000 mon.a (mon.0) 3987 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:14:54.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:53 smithi006 bash[33918]: audit 2023-08-28T01:14:52.994483+0000 mon.b (mon.1) 684 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:14:54.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:53 smithi006 bash[33918]: audit 2023-08-28T01:14:52.996867+0000 mon.b (mon.1) 685 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:14:54.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:53 smithi006 bash[33918]: audit 2023-08-28T01:14:53.006794+0000 mon.a (mon.0) 3988 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:14:54.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:53 smithi006 bash[27292]: audit 2023-08-28T01:14:52.724761+0000 mon.a (mon.0) 3987 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:14:54.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:53 smithi006 bash[27292]: audit 2023-08-28T01:14:52.994483+0000 mon.b (mon.1) 684 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:14:54.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:53 smithi006 bash[27292]: audit 2023-08-28T01:14:52.996867+0000 mon.b (mon.1) 685 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:14:54.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:53 smithi006 bash[27292]: audit 2023-08-28T01:14:53.006794+0000 mon.a (mon.0) 3988 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:14:54.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:54 smithi130 bash[51302]: debug 2023-08-28T01:14:54.362+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:54.366416+0000) 2023-08-28T01:14:54.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:14:54.414Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:54.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:14:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:14:54.415Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:14:55.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:54 smithi130 bash[27579]: cluster 2023-08-28T01:14:53.796623+0000 mgr.y (mgr.14901) 5889 : cluster [DBG] pgmap v5896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:55.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:54 smithi130 bash[27579]: audit 2023-08-28T01:14:54.579712+0000 mon.a (mon.0) 3989 : audit [DBG] from='client.? 172.21.15.6:0/1411408589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:55.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:54 smithi006 bash[33918]: cluster 2023-08-28T01:14:53.796623+0000 mgr.y (mgr.14901) 5889 : cluster [DBG] pgmap v5896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:55.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:54 smithi006 bash[33918]: audit 2023-08-28T01:14:54.579712+0000 mon.a (mon.0) 3989 : audit [DBG] from='client.? 172.21.15.6:0/1411408589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:55.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:54 smithi006 bash[27292]: cluster 2023-08-28T01:14:53.796623+0000 mgr.y (mgr.14901) 5889 : cluster [DBG] pgmap v5896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:55.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:54 smithi006 bash[27292]: audit 2023-08-28T01:14:54.579712+0000 mon.a (mon.0) 3989 : audit [DBG] from='client.? 172.21.15.6:0/1411408589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:55.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:55 smithi130 bash[51302]: debug 2023-08-28T01:14:55.362+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:55.366703+0000) 2023-08-28T01:14:56.857 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:56 smithi130 bash[51302]: debug 2023-08-28T01:14:56.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:56.367016+0000) 2023-08-28T01:14:57.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:56 smithi130 bash[27579]: cluster 2023-08-28T01:14:55.797796+0000 mgr.y (mgr.14901) 5890 : cluster [DBG] pgmap v5897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:57.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:56 smithi006 bash[27292]: cluster 2023-08-28T01:14:55.797796+0000 mgr.y (mgr.14901) 5890 : cluster [DBG] pgmap v5897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:57.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:56 smithi006 bash[33918]: cluster 2023-08-28T01:14:55.797796+0000 mgr.y (mgr.14901) 5890 : cluster [DBG] pgmap v5897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:57.864 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:57 smithi130 bash[51302]: debug 2023-08-28T01:14:57.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:57.367275+0000) 2023-08-28T01:14:58.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:57 smithi130 bash[27579]: audit 2023-08-28T01:14:56.991565+0000 mon.a (mon.0) 3990 : audit [DBG] from='client.? 172.21.15.6:0/2149865450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:58.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:57 smithi006 bash[33918]: audit 2023-08-28T01:14:56.991565+0000 mon.a (mon.0) 3990 : audit [DBG] from='client.? 172.21.15.6:0/2149865450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:58.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:57 smithi006 bash[27292]: audit 2023-08-28T01:14:56.991565+0000 mon.a (mon.0) 3990 : audit [DBG] from='client.? 172.21.15.6:0/2149865450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:58.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:58 smithi130 bash[51302]: debug 2023-08-28T01:14:58.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:58.367498+0000) 2023-08-28T01:14:59.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:58 smithi006 bash[27292]: cluster 2023-08-28T01:14:57.798464+0000 mgr.y (mgr.14901) 5891 : cluster [DBG] pgmap v5898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:59.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:58 smithi006 bash[33918]: cluster 2023-08-28T01:14:57.798464+0000 mgr.y (mgr.14901) 5891 : cluster [DBG] pgmap v5898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:59.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:58 smithi130 bash[27579]: cluster 2023-08-28T01:14:57.798464+0000 mgr.y (mgr.14901) 5891 : cluster [DBG] pgmap v5898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:14:59.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:14:59 smithi130 bash[51302]: debug 2023-08-28T01:14:59.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:59.367743+0000) 2023-08-28T01:15:00.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:14:59 smithi006 bash[33918]: audit 2023-08-28T01:14:59.417328+0000 mon.c (mon.2) 2302 : audit [DBG] from='client.? 172.21.15.6:0/3378556278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:00.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:14:59 smithi006 bash[27292]: audit 2023-08-28T01:14:59.417328+0000 mon.c (mon.2) 2302 : audit [DBG] from='client.? 172.21.15.6:0/3378556278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:00.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:14:59 smithi130 bash[27579]: audit 2023-08-28T01:14:59.417328+0000 mon.c (mon.2) 2302 : audit [DBG] from='client.? 172.21.15.6:0/3378556278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:00.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:00 smithi130 bash[51302]: debug 2023-08-28T01:15:00.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:00.367989+0000) 2023-08-28T01:15:01.200 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:00 smithi006 bash[33918]: cluster 2023-08-28T01:14:59.799793+0000 mgr.y (mgr.14901) 5892 : cluster [DBG] pgmap v5899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:01.200 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:00 smithi006 bash[27292]: cluster 2023-08-28T01:14:59.799793+0000 mgr.y (mgr.14901) 5892 : cluster [DBG] pgmap v5899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:01.201 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:15:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:15:00] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T01:15:01.201 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:15:00.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:01.201 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:15:00.965Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:01.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:00 smithi130 bash[27579]: cluster 2023-08-28T01:14:59.799793+0000 mgr.y (mgr.14901) 5892 : cluster [DBG] pgmap v5899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:01.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:01 smithi130 bash[51302]: debug 2023-08-28T01:15:01.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:01.368212+0000) 2023-08-28T01:15:02.680 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:02 smithi130 bash[51302]: debug 2023-08-28T01:15:02.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:02.368393+0000) 2023-08-28T01:15:02.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:02 smithi006 bash[27292]: cluster 2023-08-28T01:15:01.800380+0000 mgr.y (mgr.14901) 5893 : cluster [DBG] pgmap v5900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:02 smithi006 bash[27292]: audit 2023-08-28T01:15:01.841871+0000 mon.c (mon.2) 2303 : audit [DBG] from='client.? 172.21.15.6:0/721323238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:02 smithi006 bash[33918]: cluster 2023-08-28T01:15:01.800380+0000 mgr.y (mgr.14901) 5893 : cluster [DBG] pgmap v5900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:02 smithi006 bash[33918]: audit 2023-08-28T01:15:01.841871+0000 mon.c (mon.2) 2303 : audit [DBG] from='client.? 172.21.15.6:0/721323238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:02.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:15:02.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:02.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:15:02.537Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:02 smithi130 bash[27579]: cluster 2023-08-28T01:15:01.800380+0000 mgr.y (mgr.14901) 5893 : cluster [DBG] pgmap v5900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:02 smithi130 bash[27579]: audit 2023-08-28T01:15:01.841871+0000 mon.c (mon.2) 2303 : audit [DBG] from='client.? 172.21.15.6:0/721323238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:03.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:03 smithi130 bash[51302]: debug 2023-08-28T01:15:03.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:03.368635+0000) 2023-08-28T01:15:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:15:04.415Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:15:04.417Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:04.860 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:04 smithi130 bash[51302]: debug 2023-08-28T01:15:04.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:04.368913+0000) 2023-08-28T01:15:05.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:04 smithi130 bash[27579]: cluster 2023-08-28T01:15:03.801064+0000 mgr.y (mgr.14901) 5894 : cluster [DBG] pgmap v5901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:05.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:04 smithi130 bash[27579]: audit 2023-08-28T01:15:04.261660+0000 mon.a (mon.0) 3991 : audit [DBG] from='client.? 172.21.15.6:0/1019136998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:05.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:04 smithi006 bash[33918]: cluster 2023-08-28T01:15:03.801064+0000 mgr.y (mgr.14901) 5894 : cluster [DBG] pgmap v5901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:05.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:04 smithi006 bash[33918]: audit 2023-08-28T01:15:04.261660+0000 mon.a (mon.0) 3991 : audit [DBG] from='client.? 172.21.15.6:0/1019136998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:05.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:04 smithi006 bash[27292]: cluster 2023-08-28T01:15:03.801064+0000 mgr.y (mgr.14901) 5894 : cluster [DBG] pgmap v5901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:05.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:04 smithi006 bash[27292]: audit 2023-08-28T01:15:04.261660+0000 mon.a (mon.0) 3991 : audit [DBG] from='client.? 172.21.15.6:0/1019136998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:05.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:05 smithi130 bash[51302]: debug 2023-08-28T01:15:05.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:05.369234+0000) 2023-08-28T01:15:06.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:06 smithi130 bash[51302]: debug 2023-08-28T01:15:06.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:06.369652+0000) 2023-08-28T01:15:07.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:06 smithi130 bash[27579]: cluster 2023-08-28T01:15:05.802190+0000 mgr.y (mgr.14901) 5895 : cluster [DBG] pgmap v5902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:07.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:06 smithi130 bash[27579]: audit 2023-08-28T01:15:06.671735+0000 mon.c (mon.2) 2304 : audit [DBG] from='client.? 172.21.15.6:0/2867858827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:07.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:06 smithi006 bash[27292]: cluster 2023-08-28T01:15:05.802190+0000 mgr.y (mgr.14901) 5895 : cluster [DBG] pgmap v5902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:06 smithi006 bash[27292]: audit 2023-08-28T01:15:06.671735+0000 mon.c (mon.2) 2304 : audit [DBG] from='client.? 172.21.15.6:0/2867858827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:07.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:06 smithi006 bash[33918]: cluster 2023-08-28T01:15:05.802190+0000 mgr.y (mgr.14901) 5895 : cluster [DBG] pgmap v5902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:07.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:06 smithi006 bash[33918]: audit 2023-08-28T01:15:06.671735+0000 mon.c (mon.2) 2304 : audit [DBG] from='client.? 172.21.15.6:0/2867858827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:07.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:07 smithi130 bash[51302]: debug 2023-08-28T01:15:07.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:07.369915+0000) 2023-08-28T01:15:08.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:08 smithi130 bash[51302]: debug 2023-08-28T01:15:08.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:08.370189+0000) 2023-08-28T01:15:09.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:08 smithi006 bash[33918]: cluster 2023-08-28T01:15:07.802831+0000 mgr.y (mgr.14901) 5896 : cluster [DBG] pgmap v5903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:09.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:08 smithi006 bash[27292]: cluster 2023-08-28T01:15:07.802831+0000 mgr.y (mgr.14901) 5896 : cluster [DBG] pgmap v5903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:09.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:08 smithi130 bash[27579]: cluster 2023-08-28T01:15:07.802831+0000 mgr.y (mgr.14901) 5896 : cluster [DBG] pgmap v5903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:09.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:09 smithi130 bash[51302]: debug 2023-08-28T01:15:09.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:09.370451+0000) 2023-08-28T01:15:10.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:09 smithi006 bash[27292]: audit 2023-08-28T01:15:09.090209+0000 mon.c (mon.2) 2305 : audit [DBG] from='client.? 172.21.15.6:0/1502962022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:10.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:09 smithi006 bash[33918]: audit 2023-08-28T01:15:09.090209+0000 mon.c (mon.2) 2305 : audit [DBG] from='client.? 172.21.15.6:0/1502962022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:10.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:09 smithi130 bash[27579]: audit 2023-08-28T01:15:09.090209+0000 mon.c (mon.2) 2305 : audit [DBG] from='client.? 172.21.15.6:0/1502962022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:10.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:10 smithi130 bash[51302]: debug 2023-08-28T01:15:10.366+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:10.370724+0000) 2023-08-28T01:15:11.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:10 smithi006 bash[33918]: cluster 2023-08-28T01:15:09.803950+0000 mgr.y (mgr.14901) 5897 : cluster [DBG] pgmap v5904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:11.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:10 smithi006 bash[27292]: cluster 2023-08-28T01:15:09.803950+0000 mgr.y (mgr.14901) 5897 : cluster [DBG] pgmap v5904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:15:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:15:10] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T01:15:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:15:10.963Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:15:10.965Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:11.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:10 smithi130 bash[27579]: cluster 2023-08-28T01:15:09.803950+0000 mgr.y (mgr.14901) 5897 : cluster [DBG] pgmap v5904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:11.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:11 smithi130 bash[51302]: debug 2023-08-28T01:15:11.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:11.371000+0000) 2023-08-28T01:15:12.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:11 smithi006 bash[27292]: audit 2023-08-28T01:15:11.511094+0000 mon.c (mon.2) 2306 : audit [DBG] from='client.? 172.21.15.6:0/2177521428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:12.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:11 smithi006 bash[33918]: audit 2023-08-28T01:15:11.511094+0000 mon.c (mon.2) 2306 : audit [DBG] from='client.? 172.21.15.6:0/2177521428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:12.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:11 smithi130 bash[27579]: audit 2023-08-28T01:15:11.511094+0000 mon.c (mon.2) 2306 : audit [DBG] from='client.? 172.21.15.6:0/2177521428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:12.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:12 smithi130 bash[51302]: debug 2023-08-28T01:15:12.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:12.371241+0000) 2023-08-28T01:15:12.889 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:15:12.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:12.889 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:15:12.538Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:12 smithi006 bash[33918]: cluster 2023-08-28T01:15:11.804638+0000 mgr.y (mgr.14901) 5898 : cluster [DBG] pgmap v5905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:13.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:12 smithi006 bash[27292]: cluster 2023-08-28T01:15:11.804638+0000 mgr.y (mgr.14901) 5898 : cluster [DBG] pgmap v5905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:13.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:12 smithi130 bash[27579]: cluster 2023-08-28T01:15:11.804638+0000 mgr.y (mgr.14901) 5898 : cluster [DBG] pgmap v5905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:13.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:13 smithi130 bash[51302]: debug 2023-08-28T01:15:13.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:13.371492+0000) 2023-08-28T01:15:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:15:14.416Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:15:14.418Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:14.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:14 smithi130 bash[51302]: debug 2023-08-28T01:15:14.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:14.371747+0000) 2023-08-28T01:15:15.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:14 smithi006 bash[27292]: cluster 2023-08-28T01:15:13.805284+0000 mgr.y (mgr.14901) 5899 : cluster [DBG] pgmap v5906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:15.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:14 smithi006 bash[27292]: audit 2023-08-28T01:15:13.925667+0000 mon.c (mon.2) 2307 : audit [DBG] from='client.? 172.21.15.6:0/937918053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:15.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:14 smithi006 bash[33918]: cluster 2023-08-28T01:15:13.805284+0000 mgr.y (mgr.14901) 5899 : cluster [DBG] pgmap v5906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:15.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:14 smithi006 bash[33918]: audit 2023-08-28T01:15:13.925667+0000 mon.c (mon.2) 2307 : audit [DBG] from='client.? 172.21.15.6:0/937918053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:15.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:14 smithi130 bash[27579]: cluster 2023-08-28T01:15:13.805284+0000 mgr.y (mgr.14901) 5899 : cluster [DBG] pgmap v5906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:15.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:14 smithi130 bash[27579]: audit 2023-08-28T01:15:13.925667+0000 mon.c (mon.2) 2307 : audit [DBG] from='client.? 172.21.15.6:0/937918053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:15.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:15 smithi130 bash[51302]: debug 2023-08-28T01:15:15.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:15.372033+0000) 2023-08-28T01:15:16.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:16 smithi130 bash[51302]: debug 2023-08-28T01:15:16.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:16.372308+0000) 2023-08-28T01:15:17.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:16 smithi006 bash[27292]: cluster 2023-08-28T01:15:15.806353+0000 mgr.y (mgr.14901) 5900 : cluster [DBG] pgmap v5907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:17.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:16 smithi006 bash[27292]: audit 2023-08-28T01:15:16.332221+0000 mon.a (mon.0) 3992 : audit [DBG] from='client.? 172.21.15.6:0/2869837235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:17.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:16 smithi006 bash[33918]: cluster 2023-08-28T01:15:15.806353+0000 mgr.y (mgr.14901) 5900 : cluster [DBG] pgmap v5907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:17.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:16 smithi006 bash[33918]: audit 2023-08-28T01:15:16.332221+0000 mon.a (mon.0) 3992 : audit [DBG] from='client.? 172.21.15.6:0/2869837235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:17.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:16 smithi130 bash[27579]: cluster 2023-08-28T01:15:15.806353+0000 mgr.y (mgr.14901) 5900 : cluster [DBG] pgmap v5907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:17.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:16 smithi130 bash[27579]: audit 2023-08-28T01:15:16.332221+0000 mon.a (mon.0) 3992 : audit [DBG] from='client.? 172.21.15.6:0/2869837235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:17.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:17 smithi130 bash[51302]: debug 2023-08-28T01:15:17.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:17.372606+0000) 2023-08-28T01:15:18.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:18 smithi130 bash[51302]: debug 2023-08-28T01:15:18.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:18.372871+0000) 2023-08-28T01:15:19.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:18 smithi006 bash[27292]: cluster 2023-08-28T01:15:17.807010+0000 mgr.y (mgr.14901) 5901 : cluster [DBG] pgmap v5908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:19.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:18 smithi006 bash[27292]: audit 2023-08-28T01:15:18.755251+0000 mon.a (mon.0) 3993 : audit [DBG] from='client.? 172.21.15.6:0/3523481713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:19.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:18 smithi006 bash[33918]: cluster 2023-08-28T01:15:17.807010+0000 mgr.y (mgr.14901) 5901 : cluster [DBG] pgmap v5908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:19.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:18 smithi006 bash[33918]: audit 2023-08-28T01:15:18.755251+0000 mon.a (mon.0) 3993 : audit [DBG] from='client.? 172.21.15.6:0/3523481713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:19.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:18 smithi130 bash[27579]: cluster 2023-08-28T01:15:17.807010+0000 mgr.y (mgr.14901) 5901 : cluster [DBG] pgmap v5908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:19.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:18 smithi130 bash[27579]: audit 2023-08-28T01:15:18.755251+0000 mon.a (mon.0) 3993 : audit [DBG] from='client.? 172.21.15.6:0/3523481713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:19.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:19 smithi130 bash[51302]: debug 2023-08-28T01:15:19.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:19.373128+0000) 2023-08-28T01:15:20.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:20 smithi130 bash[51302]: debug 2023-08-28T01:15:20.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:20.373378+0000) 2023-08-28T01:15:21.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:20 smithi006 bash[27292]: cluster 2023-08-28T01:15:19.808144+0000 mgr.y (mgr.14901) 5902 : cluster [DBG] pgmap v5909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:21.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:15:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:15:20] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:15:21.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:20 smithi006 bash[33918]: cluster 2023-08-28T01:15:19.808144+0000 mgr.y (mgr.14901) 5902 : cluster [DBG] pgmap v5909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:21.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:15:20.963Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:15:20.965Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:21.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:20 smithi130 bash[27579]: cluster 2023-08-28T01:15:19.808144+0000 mgr.y (mgr.14901) 5902 : cluster [DBG] pgmap v5909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:21.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:21 smithi130 bash[51302]: debug 2023-08-28T01:15:21.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:21.373670+0000) 2023-08-28T01:15:22.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:21 smithi006 bash[33918]: audit 2023-08-28T01:15:21.176344+0000 mon.a (mon.0) 3994 : audit [DBG] from='client.? 172.21.15.6:0/1033075080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:22.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:21 smithi006 bash[27292]: audit 2023-08-28T01:15:21.176344+0000 mon.a (mon.0) 3994 : audit [DBG] from='client.? 172.21.15.6:0/1033075080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:22.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:21 smithi130 bash[27579]: audit 2023-08-28T01:15:21.176344+0000 mon.a (mon.0) 3994 : audit [DBG] from='client.? 172.21.15.6:0/1033075080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:22.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:22 smithi130 bash[51302]: debug 2023-08-28T01:15:22.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:22.373884+0000) 2023-08-28T01:15:22.923 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:15:22.536Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:22.924 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:15:22.538Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:23.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:22 smithi006 bash[33918]: cluster 2023-08-28T01:15:21.808812+0000 mgr.y (mgr.14901) 5903 : cluster [DBG] pgmap v5910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:23.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:22 smithi006 bash[27292]: cluster 2023-08-28T01:15:21.808812+0000 mgr.y (mgr.14901) 5903 : cluster [DBG] pgmap v5910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:23.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:22 smithi130 bash[27579]: cluster 2023-08-28T01:15:21.808812+0000 mgr.y (mgr.14901) 5903 : cluster [DBG] pgmap v5910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:23.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:23 smithi130 bash[51302]: debug 2023-08-28T01:15:23.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:23.374148+0000) 2023-08-28T01:15:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:23 smithi006 bash[27292]: audit 2023-08-28T01:15:23.592589+0000 mon.c (mon.2) 2308 : audit [DBG] from='client.? 172.21.15.6:0/1590443532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:24.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:23 smithi006 bash[33918]: audit 2023-08-28T01:15:23.592589+0000 mon.c (mon.2) 2308 : audit [DBG] from='client.? 172.21.15.6:0/1590443532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:24.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:23 smithi130 bash[27579]: audit 2023-08-28T01:15:23.592589+0000 mon.c (mon.2) 2308 : audit [DBG] from='client.? 172.21.15.6:0/1590443532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:15:24.416Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:24.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:15:24.418Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:24.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:24 smithi130 bash[51302]: debug 2023-08-28T01:15:24.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:24.374455+0000) 2023-08-28T01:15:25.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:24 smithi006 bash[33918]: cluster 2023-08-28T01:15:23.809491+0000 mgr.y (mgr.14901) 5904 : cluster [DBG] pgmap v5911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:25.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:24 smithi006 bash[27292]: cluster 2023-08-28T01:15:23.809491+0000 mgr.y (mgr.14901) 5904 : cluster [DBG] pgmap v5911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:25.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:24 smithi130 bash[27579]: cluster 2023-08-28T01:15:23.809491+0000 mgr.y (mgr.14901) 5904 : cluster [DBG] pgmap v5911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:25.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:25 smithi130 bash[51302]: debug 2023-08-28T01:15:25.370+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:25.374709+0000) 2023-08-28T01:15:26.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:26 smithi130 bash[51302]: debug 2023-08-28T01:15:26.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:26.374990+0000) 2023-08-28T01:15:27.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:26 smithi006 bash[27292]: cluster 2023-08-28T01:15:25.810596+0000 mgr.y (mgr.14901) 5905 : cluster [DBG] pgmap v5912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:27.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:26 smithi006 bash[27292]: audit 2023-08-28T01:15:26.013495+0000 mon.a (mon.0) 3995 : audit [DBG] from='client.? 172.21.15.6:0/3546857587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:27.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:26 smithi006 bash[33918]: cluster 2023-08-28T01:15:25.810596+0000 mgr.y (mgr.14901) 5905 : cluster [DBG] pgmap v5912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:27.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:26 smithi006 bash[33918]: audit 2023-08-28T01:15:26.013495+0000 mon.a (mon.0) 3995 : audit [DBG] from='client.? 172.21.15.6:0/3546857587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:27.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:26 smithi130 bash[27579]: cluster 2023-08-28T01:15:25.810596+0000 mgr.y (mgr.14901) 5905 : cluster [DBG] pgmap v5912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:27.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:26 smithi130 bash[27579]: audit 2023-08-28T01:15:26.013495+0000 mon.a (mon.0) 3995 : audit [DBG] from='client.? 172.21.15.6:0/3546857587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:27.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:27 smithi130 bash[51302]: debug 2023-08-28T01:15:27.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:27.375233+0000) 2023-08-28T01:15:28.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:28 smithi130 bash[51302]: debug 2023-08-28T01:15:28.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:28.375504+0000) 2023-08-28T01:15:29.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:28 smithi006 bash[33918]: cluster 2023-08-28T01:15:27.811252+0000 mgr.y (mgr.14901) 5906 : cluster [DBG] pgmap v5913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:29.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:28 smithi006 bash[33918]: audit 2023-08-28T01:15:28.426491+0000 mon.c (mon.2) 2309 : audit [DBG] from='client.? 172.21.15.6:0/3303382351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:29.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:28 smithi006 bash[27292]: cluster 2023-08-28T01:15:27.811252+0000 mgr.y (mgr.14901) 5906 : cluster [DBG] pgmap v5913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:29.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:28 smithi006 bash[27292]: audit 2023-08-28T01:15:28.426491+0000 mon.c (mon.2) 2309 : audit [DBG] from='client.? 172.21.15.6:0/3303382351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:29.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:28 smithi130 bash[27579]: cluster 2023-08-28T01:15:27.811252+0000 mgr.y (mgr.14901) 5906 : cluster [DBG] pgmap v5913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:29.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:28 smithi130 bash[27579]: audit 2023-08-28T01:15:28.426491+0000 mon.c (mon.2) 2309 : audit [DBG] from='client.? 172.21.15.6:0/3303382351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:29.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:29 smithi130 bash[51302]: debug 2023-08-28T01:15:29.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:29.375828+0000) 2023-08-28T01:15:30.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:30 smithi130 bash[51302]: debug 2023-08-28T01:15:30.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:30.376115+0000) 2023-08-28T01:15:31.165 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:15:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:15:30] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T01:15:31.166 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:30 smithi006 bash[27292]: cluster 2023-08-28T01:15:29.812418+0000 mgr.y (mgr.14901) 5907 : cluster [DBG] pgmap v5914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:31.166 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:30 smithi006 bash[27292]: audit 2023-08-28T01:15:30.845270+0000 mon.c (mon.2) 2310 : audit [DBG] from='client.? 172.21.15.6:0/485343439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:31.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:30 smithi130 bash[27579]: cluster 2023-08-28T01:15:29.812418+0000 mgr.y (mgr.14901) 5907 : cluster [DBG] pgmap v5914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:31.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:30 smithi130 bash[27579]: audit 2023-08-28T01:15:30.845270+0000 mon.c (mon.2) 2310 : audit [DBG] from='client.? 172.21.15.6:0/485343439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:15:30.963Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:31.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:15:30.965Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:31.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:31 smithi006 bash[33918]: cluster 2023-08-28T01:15:29.812418+0000 mgr.y (mgr.14901) 5907 : cluster [DBG] pgmap v5914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:31.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:31 smithi006 bash[33918]: audit 2023-08-28T01:15:30.845270+0000 mon.c (mon.2) 2310 : audit [DBG] from='client.? 172.21.15.6:0/485343439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:31.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:31 smithi130 bash[51302]: debug 2023-08-28T01:15:31.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:31.376556+0000) 2023-08-28T01:15:32.689 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:32 smithi130 bash[51302]: debug 2023-08-28T01:15:32.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:32.376835+0000) 2023-08-28T01:15:32.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:32 smithi006 bash[33918]: cluster 2023-08-28T01:15:31.813145+0000 mgr.y (mgr.14901) 5908 : cluster [DBG] pgmap v5915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:32 smithi006 bash[27292]: cluster 2023-08-28T01:15:31.813145+0000 mgr.y (mgr.14901) 5908 : cluster [DBG] pgmap v5915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:15:32.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:15:32.539Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:32 smithi130 bash[27579]: cluster 2023-08-28T01:15:31.813145+0000 mgr.y (mgr.14901) 5908 : cluster [DBG] pgmap v5915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:33.694 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:33 smithi130 bash[51302]: debug 2023-08-28T01:15:33.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:33.377127+0000) 2023-08-28T01:15:33.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:33 smithi006 bash[27292]: audit 2023-08-28T01:15:33.264969+0000 mon.c (mon.2) 2311 : audit [DBG] from='client.? 172.21.15.6:0/2666693685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:33.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:33 smithi006 bash[33918]: audit 2023-08-28T01:15:33.264969+0000 mon.c (mon.2) 2311 : audit [DBG] from='client.? 172.21.15.6:0/2666693685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:34.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:33 smithi130 bash[27579]: audit 2023-08-28T01:15:33.264969+0000 mon.c (mon.2) 2311 : audit [DBG] from='client.? 172.21.15.6:0/2666693685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:34.702 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:15:34.416Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:34.702 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:15:34.418Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:34.703 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:34 smithi130 bash[51302]: debug 2023-08-28T01:15:34.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:34.377405+0000) 2023-08-28T01:15:34.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:34 smithi006 bash[27292]: cluster 2023-08-28T01:15:33.813817+0000 mgr.y (mgr.14901) 5909 : cluster [DBG] pgmap v5916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:34.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:34 smithi006 bash[33918]: cluster 2023-08-28T01:15:33.813817+0000 mgr.y (mgr.14901) 5909 : cluster [DBG] pgmap v5916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:35.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:34 smithi130 bash[27579]: cluster 2023-08-28T01:15:33.813817+0000 mgr.y (mgr.14901) 5909 : cluster [DBG] pgmap v5916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:35.705 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:35 smithi130 bash[51302]: debug 2023-08-28T01:15:35.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:35.377711+0000) 2023-08-28T01:15:35.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:35 smithi006 bash[33918]: audit 2023-08-28T01:15:35.677077+0000 mon.a (mon.0) 3996 : audit [DBG] from='client.? 172.21.15.6:0/3593962585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:35.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:35 smithi006 bash[27292]: audit 2023-08-28T01:15:35.677077+0000 mon.a (mon.0) 3996 : audit [DBG] from='client.? 172.21.15.6:0/3593962585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:36.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:35 smithi130 bash[27579]: audit 2023-08-28T01:15:35.677077+0000 mon.a (mon.0) 3996 : audit [DBG] from='client.? 172.21.15.6:0/3593962585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:36.708 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:36 smithi130 bash[51302]: debug 2023-08-28T01:15:36.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:36.377983+0000) 2023-08-28T01:15:36.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:36 smithi006 bash[33918]: cluster 2023-08-28T01:15:35.814946+0000 mgr.y (mgr.14901) 5910 : cluster [DBG] pgmap v5917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:36.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:36 smithi006 bash[27292]: cluster 2023-08-28T01:15:35.814946+0000 mgr.y (mgr.14901) 5910 : cluster [DBG] pgmap v5917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:37.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:36 smithi130 bash[27579]: cluster 2023-08-28T01:15:35.814946+0000 mgr.y (mgr.14901) 5910 : cluster [DBG] pgmap v5917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:37.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:37 smithi130 bash[51302]: debug 2023-08-28T01:15:37.378+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:37.378308+0000) 2023-08-28T01:15:38.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:38 smithi130 bash[51302]: debug 2023-08-28T01:15:38.374+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:38.378551+0000) 2023-08-28T01:15:39.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:38 smithi006 bash[33918]: cluster 2023-08-28T01:15:37.815887+0000 mgr.y (mgr.14901) 5911 : cluster [DBG] pgmap v5918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:39.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:38 smithi006 bash[33918]: audit 2023-08-28T01:15:38.087665+0000 mon.c (mon.2) 2312 : audit [DBG] from='client.? 172.21.15.6:0/3863254359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:39.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:38 smithi006 bash[27292]: cluster 2023-08-28T01:15:37.815887+0000 mgr.y (mgr.14901) 5911 : cluster [DBG] pgmap v5918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:39.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:38 smithi006 bash[27292]: audit 2023-08-28T01:15:38.087665+0000 mon.c (mon.2) 2312 : audit [DBG] from='client.? 172.21.15.6:0/3863254359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:39.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:38 smithi130 bash[27579]: cluster 2023-08-28T01:15:37.815887+0000 mgr.y (mgr.14901) 5911 : cluster [DBG] pgmap v5918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:39.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:38 smithi130 bash[27579]: audit 2023-08-28T01:15:38.087665+0000 mon.c (mon.2) 2312 : audit [DBG] from='client.? 172.21.15.6:0/3863254359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:39.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:39 smithi130 bash[51302]: debug 2023-08-28T01:15:39.373+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:39.378819+0000) 2023-08-28T01:15:40.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:40 smithi130 bash[51302]: debug 2023-08-28T01:15:40.377+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:40.379058+0000) 2023-08-28T01:15:41.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:15:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:15:40] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T01:15:41.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:40 smithi006 bash[33918]: cluster 2023-08-28T01:15:39.817175+0000 mgr.y (mgr.14901) 5912 : cluster [DBG] pgmap v5919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:41.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:40 smithi006 bash[33918]: audit 2023-08-28T01:15:40.515863+0000 mon.a (mon.0) 3997 : audit [DBG] from='client.? 172.21.15.6:0/3428288402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:41.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:40 smithi006 bash[27292]: cluster 2023-08-28T01:15:39.817175+0000 mgr.y (mgr.14901) 5912 : cluster [DBG] pgmap v5919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:41.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:40 smithi006 bash[27292]: audit 2023-08-28T01:15:40.515863+0000 mon.a (mon.0) 3997 : audit [DBG] from='client.? 172.21.15.6:0/3428288402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:15:40.964Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:15:40.966Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:41.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:40 smithi130 bash[27579]: cluster 2023-08-28T01:15:39.817175+0000 mgr.y (mgr.14901) 5912 : cluster [DBG] pgmap v5919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:41.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:40 smithi130 bash[27579]: audit 2023-08-28T01:15:40.515863+0000 mon.a (mon.0) 3997 : audit [DBG] from='client.? 172.21.15.6:0/3428288402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:41.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:41 smithi130 bash[51302]: debug 2023-08-28T01:15:41.377+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:41.379286+0000) 2023-08-28T01:15:42.689 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:42 smithi130 bash[51302]: debug 2023-08-28T01:15:42.377+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:42.379569+0000) 2023-08-28T01:15:42.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:42 smithi006 bash[27292]: cluster 2023-08-28T01:15:41.817788+0000 mgr.y (mgr.14901) 5913 : cluster [DBG] pgmap v5920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:42.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:42 smithi006 bash[33918]: cluster 2023-08-28T01:15:41.817788+0000 mgr.y (mgr.14901) 5913 : cluster [DBG] pgmap v5920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:15:42.538Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:15:42.540Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:42 smithi130 bash[27579]: cluster 2023-08-28T01:15:41.817788+0000 mgr.y (mgr.14901) 5913 : cluster [DBG] pgmap v5920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:43.692 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:43 smithi130 bash[51302]: debug 2023-08-28T01:15:43.377+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:43.379848+0000) 2023-08-28T01:15:43.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:43 smithi006 bash[27292]: audit 2023-08-28T01:15:42.933051+0000 mon.c (mon.2) 2313 : audit [DBG] from='client.? 172.21.15.6:0/3481676121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:43.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:43 smithi006 bash[33918]: audit 2023-08-28T01:15:42.933051+0000 mon.c (mon.2) 2313 : audit [DBG] from='client.? 172.21.15.6:0/3481676121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:44.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:43 smithi130 bash[27579]: audit 2023-08-28T01:15:42.933051+0000 mon.c (mon.2) 2313 : audit [DBG] from='client.? 172.21.15.6:0/3481676121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:44.699 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:15:44.418Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:44.699 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:15:44.420Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:44.699 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:44 smithi130 bash[51302]: debug 2023-08-28T01:15:44.377+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:44.380155+0000) 2023-08-28T01:15:44.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:44 smithi006 bash[27292]: cluster 2023-08-28T01:15:43.818443+0000 mgr.y (mgr.14901) 5914 : cluster [DBG] pgmap v5921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:44.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:44 smithi006 bash[33918]: cluster 2023-08-28T01:15:43.818443+0000 mgr.y (mgr.14901) 5914 : cluster [DBG] pgmap v5921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:45.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:44 smithi130 bash[27579]: cluster 2023-08-28T01:15:43.818443+0000 mgr.y (mgr.14901) 5914 : cluster [DBG] pgmap v5921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:45.703 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:45 smithi130 bash[51302]: debug 2023-08-28T01:15:45.377+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:45.380436+0000) 2023-08-28T01:15:45.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:45 smithi006 bash[33918]: audit 2023-08-28T01:15:45.349067+0000 mon.c (mon.2) 2314 : audit [DBG] from='client.? 172.21.15.6:0/618971607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:46.089 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:45 smithi006 bash[27292]: audit 2023-08-28T01:15:45.349067+0000 mon.c (mon.2) 2314 : audit [DBG] from='client.? 172.21.15.6:0/618971607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:46.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:45 smithi130 bash[27579]: audit 2023-08-28T01:15:45.349067+0000 mon.c (mon.2) 2314 : audit [DBG] from='client.? 172.21.15.6:0/618971607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:46.709 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:46 smithi130 bash[51302]: debug 2023-08-28T01:15:46.377+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:46.380709+0000) 2023-08-28T01:15:46.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:46 smithi006 bash[33918]: cluster 2023-08-28T01:15:45.819664+0000 mgr.y (mgr.14901) 5915 : cluster [DBG] pgmap v5922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:46.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:46 smithi006 bash[27292]: cluster 2023-08-28T01:15:45.819664+0000 mgr.y (mgr.14901) 5915 : cluster [DBG] pgmap v5922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:46 smithi130 bash[27579]: cluster 2023-08-28T01:15:45.819664+0000 mgr.y (mgr.14901) 5915 : cluster [DBG] pgmap v5922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:47.814 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:47 smithi130 bash[51302]: debug 2023-08-28T01:15:47.377+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:47.380959+0000) 2023-08-28T01:15:48.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:47 smithi130 bash[27579]: audit 2023-08-28T01:15:47.759469+0000 mon.a (mon.0) 3998 : audit [DBG] from='client.? 172.21.15.6:0/3425811099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:48.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:47 smithi006 bash[27292]: audit 2023-08-28T01:15:47.759469+0000 mon.a (mon.0) 3998 : audit [DBG] from='client.? 172.21.15.6:0/3425811099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:48.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:47 smithi006 bash[33918]: audit 2023-08-28T01:15:47.759469+0000 mon.a (mon.0) 3998 : audit [DBG] from='client.? 172.21.15.6:0/3425811099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:48.822 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:48 smithi130 bash[51302]: debug 2023-08-28T01:15:48.377+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:48.381224+0000) 2023-08-28T01:15:49.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:48 smithi130 bash[27579]: cluster 2023-08-28T01:15:47.820176+0000 mgr.y (mgr.14901) 5916 : cluster [DBG] pgmap v5923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:49.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:48 smithi006 bash[33918]: cluster 2023-08-28T01:15:47.820176+0000 mgr.y (mgr.14901) 5916 : cluster [DBG] pgmap v5923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:49.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:48 smithi006 bash[27292]: cluster 2023-08-28T01:15:47.820176+0000 mgr.y (mgr.14901) 5916 : cluster [DBG] pgmap v5923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:49.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:49 smithi130 bash[51302]: debug 2023-08-28T01:15:49.377+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:49.381506+0000) 2023-08-28T01:15:50.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:50 smithi130 bash[51302]: debug 2023-08-28T01:15:50.377+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:50.381788+0000) 2023-08-28T01:15:51.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:15:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:15:50] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:15:51.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:50 smithi006 bash[33918]: cluster 2023-08-28T01:15:49.821364+0000 mgr.y (mgr.14901) 5917 : cluster [DBG] pgmap v5924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:51.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:50 smithi006 bash[33918]: audit 2023-08-28T01:15:50.175363+0000 mon.c (mon.2) 2315 : audit [DBG] from='client.? 172.21.15.6:0/2174371283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:15:50.965Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:15:50.967Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:50 smithi006 bash[27292]: cluster 2023-08-28T01:15:49.821364+0000 mgr.y (mgr.14901) 5917 : cluster [DBG] pgmap v5924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:51.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:50 smithi006 bash[27292]: audit 2023-08-28T01:15:50.175363+0000 mon.c (mon.2) 2315 : audit [DBG] from='client.? 172.21.15.6:0/2174371283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:51.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:50 smithi130 bash[27579]: cluster 2023-08-28T01:15:49.821364+0000 mgr.y (mgr.14901) 5917 : cluster [DBG] pgmap v5924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:51.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:50 smithi130 bash[27579]: audit 2023-08-28T01:15:50.175363+0000 mon.c (mon.2) 2315 : audit [DBG] from='client.? 172.21.15.6:0/2174371283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:51.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:51 smithi130 bash[51302]: debug 2023-08-28T01:15:51.377+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:51.382062+0000) 2023-08-28T01:15:52.693 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:52 smithi130 bash[51302]: debug 2023-08-28T01:15:52.377+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:52.382323+0000) 2023-08-28T01:15:52.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:52 smithi006 bash[27292]: cluster 2023-08-28T01:15:51.822036+0000 mgr.y (mgr.14901) 5918 : cluster [DBG] pgmap v5925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:52 smithi006 bash[27292]: audit 2023-08-28T01:15:52.581114+0000 mon.c (mon.2) 2316 : audit [DBG] from='client.? 172.21.15.6:0/1477131227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:52 smithi006 bash[33918]: cluster 2023-08-28T01:15:51.822036+0000 mgr.y (mgr.14901) 5918 : cluster [DBG] pgmap v5925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:52 smithi006 bash[33918]: audit 2023-08-28T01:15:52.581114+0000 mon.c (mon.2) 2316 : audit [DBG] from='client.? 172.21.15.6:0/1477131227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:15:52.538Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:15:52.540Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:52 smithi130 bash[27579]: cluster 2023-08-28T01:15:51.822036+0000 mgr.y (mgr.14901) 5918 : cluster [DBG] pgmap v5925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:52 smithi130 bash[27579]: audit 2023-08-28T01:15:52.581114+0000 mon.c (mon.2) 2316 : audit [DBG] from='client.? 172.21.15.6:0/1477131227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:53.695 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:53 smithi130 bash[51302]: debug 2023-08-28T01:15:53.377+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:53.382598+0000) 2023-08-28T01:15:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:53 smithi006 bash[33918]: audit 2023-08-28T01:15:53.017789+0000 mon.b (mon.1) 686 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:15:53.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:53 smithi006 bash[27292]: audit 2023-08-28T01:15:53.017789+0000 mon.b (mon.1) 686 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:15:54.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:53 smithi130 bash[27579]: audit 2023-08-28T01:15:53.017789+0000 mon.b (mon.1) 686 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:15:54.702 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:15:54.419Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:54.702 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:15:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:15:54.421Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:15:54.703 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:54 smithi006 bash[27292]: cluster 2023-08-28T01:15:53.822699+0000 mgr.y (mgr.14901) 5919 : cluster [DBG] pgmap v5926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:54.703 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:54 smithi130 bash[51302]: debug 2023-08-28T01:15:54.381+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:54.382903+0000) 2023-08-28T01:15:54.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:54 smithi006 bash[33918]: cluster 2023-08-28T01:15:53.822699+0000 mgr.y (mgr.14901) 5919 : cluster [DBG] pgmap v5926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:55.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:54 smithi130 bash[27579]: cluster 2023-08-28T01:15:53.822699+0000 mgr.y (mgr.14901) 5919 : cluster [DBG] pgmap v5926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:55.706 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:55 smithi130 bash[51302]: debug 2023-08-28T01:15:55.381+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:55.383183+0000) 2023-08-28T01:15:55.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:55 smithi006 bash[33918]: audit 2023-08-28T01:15:55.007327+0000 mon.c (mon.2) 2317 : audit [DBG] from='client.? 172.21.15.6:0/3768651001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:55.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:55 smithi006 bash[27292]: audit 2023-08-28T01:15:55.007327+0000 mon.c (mon.2) 2317 : audit [DBG] from='client.? 172.21.15.6:0/3768651001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:56.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:55 smithi130 bash[27579]: audit 2023-08-28T01:15:55.007327+0000 mon.c (mon.2) 2317 : audit [DBG] from='client.? 172.21.15.6:0/3768651001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:56.715 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:56 smithi130 bash[51302]: debug 2023-08-28T01:15:56.381+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:56.383412+0000) 2023-08-28T01:15:56.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:56 smithi006 bash[27292]: cluster 2023-08-28T01:15:55.823909+0000 mgr.y (mgr.14901) 5920 : cluster [DBG] pgmap v5927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:56.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:56 smithi006 bash[33918]: cluster 2023-08-28T01:15:55.823909+0000 mgr.y (mgr.14901) 5920 : cluster [DBG] pgmap v5927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:57.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:56 smithi130 bash[27579]: cluster 2023-08-28T01:15:55.823909+0000 mgr.y (mgr.14901) 5920 : cluster [DBG] pgmap v5927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:57.718 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:57 smithi130 bash[51302]: debug 2023-08-28T01:15:57.381+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:57.383639+0000) 2023-08-28T01:15:58.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:57 smithi130 bash[27579]: audit 2023-08-28T01:15:57.442566+0000 mon.a (mon.0) 3999 : audit [DBG] from='client.? 172.21.15.6:0/2678602119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:58.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:57 smithi006 bash[33918]: audit 2023-08-28T01:15:57.442566+0000 mon.a (mon.0) 3999 : audit [DBG] from='client.? 172.21.15.6:0/2678602119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:58.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:57 smithi006 bash[27292]: audit 2023-08-28T01:15:57.442566+0000 mon.a (mon.0) 3999 : audit [DBG] from='client.? 172.21.15.6:0/2678602119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:58.726 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:58 smithi130 bash[51302]: debug 2023-08-28T01:15:58.381+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:58.383898+0000) 2023-08-28T01:15:59.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:58 smithi130 bash[27579]: cluster 2023-08-28T01:15:57.824751+0000 mgr.y (mgr.14901) 5921 : cluster [DBG] pgmap v5928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:59.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:58 smithi006 bash[33918]: cluster 2023-08-28T01:15:57.824751+0000 mgr.y (mgr.14901) 5921 : cluster [DBG] pgmap v5928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:59.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:58 smithi006 bash[27292]: cluster 2023-08-28T01:15:57.824751+0000 mgr.y (mgr.14901) 5921 : cluster [DBG] pgmap v5928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:15:59.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:15:59 smithi130 bash[51302]: debug 2023-08-28T01:15:59.381+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:59.384147+0000) 2023-08-28T01:16:00.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:59 smithi006 bash[33918]: audit 2023-08-28T01:15:58.939254+0000 mon.a (mon.0) 4000 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:16:00.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:59 smithi006 bash[33918]: audit 2023-08-28T01:15:59.350163+0000 mon.a (mon.0) 4001 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:16:00.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:59 smithi006 bash[33918]: audit 2023-08-28T01:15:59.623406+0000 mon.b (mon.1) 687 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:16:00.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:59 smithi006 bash[33918]: audit 2023-08-28T01:15:59.625467+0000 mon.b (mon.1) 688 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:16:00.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:59 smithi006 bash[33918]: audit 2023-08-28T01:15:59.632402+0000 mon.a (mon.0) 4002 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:16:00.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:15:59 smithi006 bash[33918]: audit 2023-08-28T01:15:59.863667+0000 mon.c (mon.2) 2318 : audit [DBG] from='client.? 172.21.15.6:0/1318840020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:00.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:59 smithi006 bash[27292]: audit 2023-08-28T01:15:58.939254+0000 mon.a (mon.0) 4000 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:16:00.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:59 smithi006 bash[27292]: audit 2023-08-28T01:15:59.350163+0000 mon.a (mon.0) 4001 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:16:00.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:59 smithi006 bash[27292]: audit 2023-08-28T01:15:59.623406+0000 mon.b (mon.1) 687 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:16:00.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:59 smithi006 bash[27292]: audit 2023-08-28T01:15:59.625467+0000 mon.b (mon.1) 688 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:16:00.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:59 smithi006 bash[27292]: audit 2023-08-28T01:15:59.632402+0000 mon.a (mon.0) 4002 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:16:00.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:15:59 smithi006 bash[27292]: audit 2023-08-28T01:15:59.863667+0000 mon.c (mon.2) 2318 : audit [DBG] from='client.? 172.21.15.6:0/1318840020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:00.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:59 smithi130 bash[27579]: audit 2023-08-28T01:15:58.939254+0000 mon.a (mon.0) 4000 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:16:00.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:59 smithi130 bash[27579]: audit 2023-08-28T01:15:59.350163+0000 mon.a (mon.0) 4001 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:16:00.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:59 smithi130 bash[27579]: audit 2023-08-28T01:15:59.623406+0000 mon.b (mon.1) 687 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:16:00.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:59 smithi130 bash[27579]: audit 2023-08-28T01:15:59.625467+0000 mon.b (mon.1) 688 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:16:00.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:59 smithi130 bash[27579]: audit 2023-08-28T01:15:59.632402+0000 mon.a (mon.0) 4002 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:16:00.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:15:59 smithi130 bash[27579]: audit 2023-08-28T01:15:59.863667+0000 mon.c (mon.2) 2318 : audit [DBG] from='client.? 172.21.15.6:0/1318840020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:00.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:00 smithi130 bash[51302]: debug 2023-08-28T01:16:00.381+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:00.384428+0000) 2023-08-28T01:16:01.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:00 smithi006 bash[27292]: cluster 2023-08-28T01:15:59.825570+0000 mgr.y (mgr.14901) 5922 : cluster [DBG] pgmap v5929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:16:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:16:00] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:16:01.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:00 smithi006 bash[33918]: cluster 2023-08-28T01:15:59.825570+0000 mgr.y (mgr.14901) 5922 : cluster [DBG] pgmap v5929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:16:00.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:16:00.968Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:01.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:00 smithi130 bash[27579]: cluster 2023-08-28T01:15:59.825570+0000 mgr.y (mgr.14901) 5922 : cluster [DBG] pgmap v5929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:01.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:01 smithi130 bash[51302]: debug 2023-08-28T01:16:01.381+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:01.384694+0000) 2023-08-28T01:16:02.698 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:02 smithi130 bash[51302]: debug 2023-08-28T01:16:02.381+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:02.384973+0000) 2023-08-28T01:16:02.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:02 smithi006 bash[33918]: cluster 2023-08-28T01:16:01.826194+0000 mgr.y (mgr.14901) 5923 : cluster [DBG] pgmap v5930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:02 smithi006 bash[33918]: audit 2023-08-28T01:16:02.276408+0000 mon.c (mon.2) 2319 : audit [DBG] from='client.? 172.21.15.6:0/3242683626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:02.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:16:02.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:02.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:16:02.541Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:02 smithi006 bash[27292]: cluster 2023-08-28T01:16:01.826194+0000 mgr.y (mgr.14901) 5923 : cluster [DBG] pgmap v5930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:02 smithi006 bash[27292]: audit 2023-08-28T01:16:02.276408+0000 mon.c (mon.2) 2319 : audit [DBG] from='client.? 172.21.15.6:0/3242683626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:02 smithi130 bash[27579]: cluster 2023-08-28T01:16:01.826194+0000 mgr.y (mgr.14901) 5923 : cluster [DBG] pgmap v5930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:02 smithi130 bash[27579]: audit 2023-08-28T01:16:02.276408+0000 mon.c (mon.2) 2319 : audit [DBG] from='client.? 172.21.15.6:0/3242683626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:03.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:03 smithi130 bash[51302]: debug 2023-08-28T01:16:03.381+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:03.385236+0000) 2023-08-28T01:16:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:16:04.419Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:16:04.421Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:04.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:04 smithi130 bash[51302]: debug 2023-08-28T01:16:04.381+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:04.385598+0000) 2023-08-28T01:16:05.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:04 smithi006 bash[33918]: cluster 2023-08-28T01:16:03.826897+0000 mgr.y (mgr.14901) 5924 : cluster [DBG] pgmap v5931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:05.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:04 smithi006 bash[33918]: audit 2023-08-28T01:16:04.693420+0000 mon.c (mon.2) 2320 : audit [DBG] from='client.? 172.21.15.6:0/1051483573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:05.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:04 smithi006 bash[27292]: cluster 2023-08-28T01:16:03.826897+0000 mgr.y (mgr.14901) 5924 : cluster [DBG] pgmap v5931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:05.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:04 smithi006 bash[27292]: audit 2023-08-28T01:16:04.693420+0000 mon.c (mon.2) 2320 : audit [DBG] from='client.? 172.21.15.6:0/1051483573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:05.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:04 smithi130 bash[27579]: cluster 2023-08-28T01:16:03.826897+0000 mgr.y (mgr.14901) 5924 : cluster [DBG] pgmap v5931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:05.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:04 smithi130 bash[27579]: audit 2023-08-28T01:16:04.693420+0000 mon.c (mon.2) 2320 : audit [DBG] from='client.? 172.21.15.6:0/1051483573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:05.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:05 smithi130 bash[51302]: debug 2023-08-28T01:16:05.381+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:05.385859+0000) 2023-08-28T01:16:06.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:06 smithi130 bash[51302]: debug 2023-08-28T01:16:06.381+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:06.386111+0000) 2023-08-28T01:16:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:06 smithi006 bash[27292]: cluster 2023-08-28T01:16:05.828064+0000 mgr.y (mgr.14901) 5925 : cluster [DBG] pgmap v5932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:07.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:06 smithi006 bash[33918]: cluster 2023-08-28T01:16:05.828064+0000 mgr.y (mgr.14901) 5925 : cluster [DBG] pgmap v5932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:07.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:06 smithi130 bash[27579]: cluster 2023-08-28T01:16:05.828064+0000 mgr.y (mgr.14901) 5925 : cluster [DBG] pgmap v5932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:07.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:07 smithi130 bash[51302]: debug 2023-08-28T01:16:07.381+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:07.386355+0000) 2023-08-28T01:16:08.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:07 smithi006 bash[33918]: audit 2023-08-28T01:16:07.108291+0000 mon.a (mon.0) 4003 : audit [DBG] from='client.? 172.21.15.6:0/139867354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:08.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:07 smithi006 bash[27292]: audit 2023-08-28T01:16:07.108291+0000 mon.a (mon.0) 4003 : audit [DBG] from='client.? 172.21.15.6:0/139867354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:08.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:07 smithi130 bash[27579]: audit 2023-08-28T01:16:07.108291+0000 mon.a (mon.0) 4003 : audit [DBG] from='client.? 172.21.15.6:0/139867354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:08.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:08 smithi130 bash[51302]: debug 2023-08-28T01:16:08.381+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:08.386638+0000) 2023-08-28T01:16:09.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:08 smithi006 bash[33918]: cluster 2023-08-28T01:16:07.828710+0000 mgr.y (mgr.14901) 5926 : cluster [DBG] pgmap v5933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:09.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:08 smithi006 bash[27292]: cluster 2023-08-28T01:16:07.828710+0000 mgr.y (mgr.14901) 5926 : cluster [DBG] pgmap v5933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:09.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:08 smithi130 bash[27579]: cluster 2023-08-28T01:16:07.828710+0000 mgr.y (mgr.14901) 5926 : cluster [DBG] pgmap v5933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:09.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:09 smithi130 bash[51302]: debug 2023-08-28T01:16:09.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:09.386916+0000) 2023-08-28T01:16:10.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:09 smithi006 bash[33918]: audit 2023-08-28T01:16:09.525899+0000 mon.a (mon.0) 4004 : audit [DBG] from='client.? 172.21.15.6:0/686389602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:10.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:09 smithi006 bash[27292]: audit 2023-08-28T01:16:09.525899+0000 mon.a (mon.0) 4004 : audit [DBG] from='client.? 172.21.15.6:0/686389602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:10.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:09 smithi130 bash[27579]: audit 2023-08-28T01:16:09.525899+0000 mon.a (mon.0) 4004 : audit [DBG] from='client.? 172.21.15.6:0/686389602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:10.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:10 smithi130 bash[51302]: debug 2023-08-28T01:16:10.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:10.387163+0000) 2023-08-28T01:16:11.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:16:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:16:10] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:16:11.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:10 smithi006 bash[33918]: cluster 2023-08-28T01:16:09.829893+0000 mgr.y (mgr.14901) 5927 : cluster [DBG] pgmap v5934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:11.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:10 smithi006 bash[27292]: cluster 2023-08-28T01:16:09.829893+0000 mgr.y (mgr.14901) 5927 : cluster [DBG] pgmap v5934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:11.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:16:10.967Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:11.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:16:10.969Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:11.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:10 smithi130 bash[27579]: cluster 2023-08-28T01:16:09.829893+0000 mgr.y (mgr.14901) 5927 : cluster [DBG] pgmap v5934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:11.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:11 smithi130 bash[51302]: debug 2023-08-28T01:16:11.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:11.387380+0000) 2023-08-28T01:16:12.699 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:12 smithi130 bash[51302]: debug 2023-08-28T01:16:12.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:12.387599+0000) 2023-08-28T01:16:12.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:12 smithi006 bash[27292]: cluster 2023-08-28T01:16:11.830538+0000 mgr.y (mgr.14901) 5928 : cluster [DBG] pgmap v5935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:12 smithi006 bash[27292]: audit 2023-08-28T01:16:11.941758+0000 mon.a (mon.0) 4005 : audit [DBG] from='client.? 172.21.15.6:0/2557858284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:12 smithi006 bash[33918]: cluster 2023-08-28T01:16:11.830538+0000 mgr.y (mgr.14901) 5928 : cluster [DBG] pgmap v5935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:12 smithi006 bash[33918]: audit 2023-08-28T01:16:11.941758+0000 mon.a (mon.0) 4005 : audit [DBG] from='client.? 172.21.15.6:0/2557858284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:16:12.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:12.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:16:12.542Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:12 smithi130 bash[27579]: cluster 2023-08-28T01:16:11.830538+0000 mgr.y (mgr.14901) 5928 : cluster [DBG] pgmap v5935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:12 smithi130 bash[27579]: audit 2023-08-28T01:16:11.941758+0000 mon.a (mon.0) 4005 : audit [DBG] from='client.? 172.21.15.6:0/2557858284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:13.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:13 smithi130 bash[51302]: debug 2023-08-28T01:16:13.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:13.387870+0000) 2023-08-28T01:16:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:16:14.420Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:16:14.422Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:14.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:14 smithi130 bash[51302]: debug 2023-08-28T01:16:14.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:14.388151+0000) 2023-08-28T01:16:15.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:14 smithi006 bash[27292]: cluster 2023-08-28T01:16:13.831179+0000 mgr.y (mgr.14901) 5929 : cluster [DBG] pgmap v5936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:15.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:14 smithi006 bash[27292]: audit 2023-08-28T01:16:14.368105+0000 mon.c (mon.2) 2321 : audit [DBG] from='client.? 172.21.15.6:0/185800166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:15.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:14 smithi006 bash[33918]: cluster 2023-08-28T01:16:13.831179+0000 mgr.y (mgr.14901) 5929 : cluster [DBG] pgmap v5936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:15.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:14 smithi006 bash[33918]: audit 2023-08-28T01:16:14.368105+0000 mon.c (mon.2) 2321 : audit [DBG] from='client.? 172.21.15.6:0/185800166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:15.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:14 smithi130 bash[27579]: cluster 2023-08-28T01:16:13.831179+0000 mgr.y (mgr.14901) 5929 : cluster [DBG] pgmap v5936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:15.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:14 smithi130 bash[27579]: audit 2023-08-28T01:16:14.368105+0000 mon.c (mon.2) 2321 : audit [DBG] from='client.? 172.21.15.6:0/185800166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:15.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:15 smithi130 bash[51302]: debug 2023-08-28T01:16:15.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:15.388418+0000) 2023-08-28T01:16:16.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:16 smithi130 bash[51302]: debug 2023-08-28T01:16:16.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:16.388684+0000) 2023-08-28T01:16:17.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:16 smithi006 bash[33918]: cluster 2023-08-28T01:16:15.832300+0000 mgr.y (mgr.14901) 5930 : cluster [DBG] pgmap v5937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:17.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:16 smithi006 bash[33918]: audit 2023-08-28T01:16:16.779501+0000 mon.a (mon.0) 4006 : audit [DBG] from='client.? 172.21.15.6:0/3298793679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:17.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:16 smithi006 bash[27292]: cluster 2023-08-28T01:16:15.832300+0000 mgr.y (mgr.14901) 5930 : cluster [DBG] pgmap v5937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:17.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:16 smithi006 bash[27292]: audit 2023-08-28T01:16:16.779501+0000 mon.a (mon.0) 4006 : audit [DBG] from='client.? 172.21.15.6:0/3298793679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:17.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:16 smithi130 bash[27579]: cluster 2023-08-28T01:16:15.832300+0000 mgr.y (mgr.14901) 5930 : cluster [DBG] pgmap v5937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:17.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:16 smithi130 bash[27579]: audit 2023-08-28T01:16:16.779501+0000 mon.a (mon.0) 4006 : audit [DBG] from='client.? 172.21.15.6:0/3298793679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:17.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:17 smithi130 bash[51302]: debug 2023-08-28T01:16:17.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:17.388980+0000) 2023-08-28T01:16:18.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:18 smithi130 bash[51302]: debug 2023-08-28T01:16:18.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:18.389262+0000) 2023-08-28T01:16:19.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:18 smithi006 bash[33918]: cluster 2023-08-28T01:16:17.832966+0000 mgr.y (mgr.14901) 5931 : cluster [DBG] pgmap v5938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:19.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:18 smithi006 bash[27292]: cluster 2023-08-28T01:16:17.832966+0000 mgr.y (mgr.14901) 5931 : cluster [DBG] pgmap v5938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:19.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:18 smithi130 bash[27579]: cluster 2023-08-28T01:16:17.832966+0000 mgr.y (mgr.14901) 5931 : cluster [DBG] pgmap v5938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:19.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:19 smithi130 bash[51302]: debug 2023-08-28T01:16:19.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:19.389533+0000) 2023-08-28T01:16:20.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:19 smithi006 bash[33918]: audit 2023-08-28T01:16:19.192694+0000 mon.c (mon.2) 2322 : audit [DBG] from='client.? 172.21.15.6:0/3751458542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:20.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:19 smithi006 bash[27292]: audit 2023-08-28T01:16:19.192694+0000 mon.c (mon.2) 2322 : audit [DBG] from='client.? 172.21.15.6:0/3751458542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:20.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:19 smithi130 bash[27579]: audit 2023-08-28T01:16:19.192694+0000 mon.c (mon.2) 2322 : audit [DBG] from='client.? 172.21.15.6:0/3751458542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:20.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:20 smithi130 bash[51302]: debug 2023-08-28T01:16:20.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:20.389820+0000) 2023-08-28T01:16:21.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:16:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:16:20] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T01:16:21.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:20 smithi006 bash[33918]: cluster 2023-08-28T01:16:19.834040+0000 mgr.y (mgr.14901) 5932 : cluster [DBG] pgmap v5939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:21.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:20 smithi006 bash[27292]: cluster 2023-08-28T01:16:19.834040+0000 mgr.y (mgr.14901) 5932 : cluster [DBG] pgmap v5939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:21.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:16:20.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:21.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:16:20.970Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:21.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:20 smithi130 bash[27579]: cluster 2023-08-28T01:16:19.834040+0000 mgr.y (mgr.14901) 5932 : cluster [DBG] pgmap v5939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:21.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:21 smithi130 bash[51302]: debug 2023-08-28T01:16:21.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:21.390080+0000) 2023-08-28T01:16:22.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:21 smithi006 bash[27292]: audit 2023-08-28T01:16:21.605047+0000 mon.c (mon.2) 2323 : audit [DBG] from='client.? 172.21.15.6:0/2357902555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:22.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:21 smithi006 bash[33918]: audit 2023-08-28T01:16:21.605047+0000 mon.c (mon.2) 2323 : audit [DBG] from='client.? 172.21.15.6:0/2357902555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:22.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:21 smithi130 bash[27579]: audit 2023-08-28T01:16:21.605047+0000 mon.c (mon.2) 2323 : audit [DBG] from='client.? 172.21.15.6:0/2357902555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:22.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:22 smithi130 bash[51302]: debug 2023-08-28T01:16:22.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:22.390355+0000) 2023-08-28T01:16:22.924 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:16:22.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:22.924 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:16:22.542Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:23.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:22 smithi006 bash[33918]: cluster 2023-08-28T01:16:21.834865+0000 mgr.y (mgr.14901) 5933 : cluster [DBG] pgmap v5940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:23.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:22 smithi006 bash[27292]: cluster 2023-08-28T01:16:21.834865+0000 mgr.y (mgr.14901) 5933 : cluster [DBG] pgmap v5940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:23.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:22 smithi130 bash[27579]: cluster 2023-08-28T01:16:21.834865+0000 mgr.y (mgr.14901) 5933 : cluster [DBG] pgmap v5940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:23.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:23 smithi130 bash[51302]: debug 2023-08-28T01:16:23.385+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:23.390633+0000) 2023-08-28T01:16:24.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:16:24.421Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:24.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:16:24.423Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:24.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:24 smithi130 bash[51302]: debug 2023-08-28T01:16:24.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:24.390957+0000) 2023-08-28T01:16:25.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:24 smithi006 bash[27292]: cluster 2023-08-28T01:16:23.835497+0000 mgr.y (mgr.14901) 5934 : cluster [DBG] pgmap v5941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:25.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:24 smithi006 bash[27292]: audit 2023-08-28T01:16:24.015444+0000 mon.c (mon.2) 2324 : audit [DBG] from='client.? 172.21.15.6:0/1868494785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:25.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:24 smithi006 bash[33918]: cluster 2023-08-28T01:16:23.835497+0000 mgr.y (mgr.14901) 5934 : cluster [DBG] pgmap v5941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:25.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:24 smithi006 bash[33918]: audit 2023-08-28T01:16:24.015444+0000 mon.c (mon.2) 2324 : audit [DBG] from='client.? 172.21.15.6:0/1868494785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:25.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:24 smithi130 bash[27579]: cluster 2023-08-28T01:16:23.835497+0000 mgr.y (mgr.14901) 5934 : cluster [DBG] pgmap v5941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:25.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:24 smithi130 bash[27579]: audit 2023-08-28T01:16:24.015444+0000 mon.c (mon.2) 2324 : audit [DBG] from='client.? 172.21.15.6:0/1868494785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:25.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:25 smithi130 bash[51302]: debug 2023-08-28T01:16:25.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:25.391170+0000) 2023-08-28T01:16:26.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:26 smithi130 bash[51302]: debug 2023-08-28T01:16:26.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:26.391446+0000) 2023-08-28T01:16:27.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:26 smithi006 bash[27292]: cluster 2023-08-28T01:16:25.836624+0000 mgr.y (mgr.14901) 5935 : cluster [DBG] pgmap v5942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:27.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:26 smithi006 bash[27292]: audit 2023-08-28T01:16:26.424998+0000 mon.a (mon.0) 4007 : audit [DBG] from='client.? 172.21.15.6:0/1750773333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:27.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:26 smithi006 bash[33918]: cluster 2023-08-28T01:16:25.836624+0000 mgr.y (mgr.14901) 5935 : cluster [DBG] pgmap v5942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:27.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:26 smithi006 bash[33918]: audit 2023-08-28T01:16:26.424998+0000 mon.a (mon.0) 4007 : audit [DBG] from='client.? 172.21.15.6:0/1750773333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:27.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:26 smithi130 bash[27579]: cluster 2023-08-28T01:16:25.836624+0000 mgr.y (mgr.14901) 5935 : cluster [DBG] pgmap v5942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:27.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:26 smithi130 bash[27579]: audit 2023-08-28T01:16:26.424998+0000 mon.a (mon.0) 4007 : audit [DBG] from='client.? 172.21.15.6:0/1750773333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:27.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:27 smithi130 bash[51302]: debug 2023-08-28T01:16:27.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:27.391738+0000) 2023-08-28T01:16:28.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:28 smithi130 bash[51302]: debug 2023-08-28T01:16:28.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:28.392023+0000) 2023-08-28T01:16:29.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:28 smithi006 bash[33918]: cluster 2023-08-28T01:16:27.837288+0000 mgr.y (mgr.14901) 5936 : cluster [DBG] pgmap v5943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:29.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:28 smithi006 bash[33918]: audit 2023-08-28T01:16:28.837412+0000 mon.a (mon.0) 4008 : audit [DBG] from='client.? 172.21.15.6:0/2529441018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:29.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:28 smithi006 bash[27292]: cluster 2023-08-28T01:16:27.837288+0000 mgr.y (mgr.14901) 5936 : cluster [DBG] pgmap v5943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:29.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:28 smithi006 bash[27292]: audit 2023-08-28T01:16:28.837412+0000 mon.a (mon.0) 4008 : audit [DBG] from='client.? 172.21.15.6:0/2529441018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:29.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:28 smithi130 bash[27579]: cluster 2023-08-28T01:16:27.837288+0000 mgr.y (mgr.14901) 5936 : cluster [DBG] pgmap v5943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:29.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:28 smithi130 bash[27579]: audit 2023-08-28T01:16:28.837412+0000 mon.a (mon.0) 4008 : audit [DBG] from='client.? 172.21.15.6:0/2529441018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:29.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:29 smithi130 bash[51302]: debug 2023-08-28T01:16:29.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:29.392291+0000) 2023-08-28T01:16:30.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:30 smithi130 bash[51302]: debug 2023-08-28T01:16:30.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:30.392592+0000) 2023-08-28T01:16:31.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:30 smithi006 bash[27292]: cluster 2023-08-28T01:16:29.838381+0000 mgr.y (mgr.14901) 5937 : cluster [DBG] pgmap v5944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:16:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:16:30] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:16:31.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:30 smithi006 bash[33918]: cluster 2023-08-28T01:16:29.838381+0000 mgr.y (mgr.14901) 5937 : cluster [DBG] pgmap v5944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:31.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:16:30.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:30 smithi006 bash[56298]: level=warn ts=2023-08-28T01:16:30.970Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:31.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:30 smithi130 bash[27579]: cluster 2023-08-28T01:16:29.838381+0000 mgr.y (mgr.14901) 5937 : cluster [DBG] pgmap v5944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:31.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:31 smithi130 bash[51302]: debug 2023-08-28T01:16:31.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:31.392815+0000) 2023-08-28T01:16:32.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:31 smithi006 bash[33918]: audit 2023-08-28T01:16:31.250831+0000 mon.a (mon.0) 4009 : audit [DBG] from='client.? 172.21.15.6:0/1032425088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:32.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:31 smithi006 bash[27292]: audit 2023-08-28T01:16:31.250831+0000 mon.a (mon.0) 4009 : audit [DBG] from='client.? 172.21.15.6:0/1032425088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:32.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:31 smithi130 bash[27579]: audit 2023-08-28T01:16:31.250831+0000 mon.a (mon.0) 4009 : audit [DBG] from='client.? 172.21.15.6:0/1032425088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:32.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:32 smithi130 bash[51302]: debug 2023-08-28T01:16:32.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:32.393039+0000) 2023-08-28T01:16:32.950 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:16:32.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:32.951 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:16:32.542Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:33.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:32 smithi006 bash[33918]: cluster 2023-08-28T01:16:31.839154+0000 mgr.y (mgr.14901) 5938 : cluster [DBG] pgmap v5945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:33.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:32 smithi006 bash[27292]: cluster 2023-08-28T01:16:31.839154+0000 mgr.y (mgr.14901) 5938 : cluster [DBG] pgmap v5945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:33.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:32 smithi130 bash[27579]: cluster 2023-08-28T01:16:31.839154+0000 mgr.y (mgr.14901) 5938 : cluster [DBG] pgmap v5945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:33.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:33 smithi130 bash[51302]: debug 2023-08-28T01:16:33.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:33.393292+0000) 2023-08-28T01:16:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:33 smithi006 bash[33918]: audit 2023-08-28T01:16:33.662217+0000 mon.a (mon.0) 4010 : audit [DBG] from='client.? 172.21.15.6:0/473701965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:34.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:33 smithi006 bash[27292]: audit 2023-08-28T01:16:33.662217+0000 mon.a (mon.0) 4010 : audit [DBG] from='client.? 172.21.15.6:0/473701965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:34.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:33 smithi130 bash[27579]: audit 2023-08-28T01:16:33.662217+0000 mon.a (mon.0) 4010 : audit [DBG] from='client.? 172.21.15.6:0/473701965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:16:34.422Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:34.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:16:34.424Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:34.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:34 smithi130 bash[51302]: debug 2023-08-28T01:16:34.389+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:34.393570+0000) 2023-08-28T01:16:35.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:34 smithi006 bash[33918]: cluster 2023-08-28T01:16:33.839842+0000 mgr.y (mgr.14901) 5939 : cluster [DBG] pgmap v5946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:35.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:34 smithi006 bash[27292]: cluster 2023-08-28T01:16:33.839842+0000 mgr.y (mgr.14901) 5939 : cluster [DBG] pgmap v5946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:35.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:34 smithi130 bash[27579]: cluster 2023-08-28T01:16:33.839842+0000 mgr.y (mgr.14901) 5939 : cluster [DBG] pgmap v5946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:35.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:35 smithi130 bash[51302]: debug 2023-08-28T01:16:35.388+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:35.393865+0000) 2023-08-28T01:16:36.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:36 smithi130 bash[51302]: debug 2023-08-28T01:16:36.392+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:36.394119+0000) 2023-08-28T01:16:37.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:36 smithi130 bash[27579]: cluster 2023-08-28T01:16:35.840951+0000 mgr.y (mgr.14901) 5940 : cluster [DBG] pgmap v5947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:37.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:36 smithi130 bash[27579]: audit 2023-08-28T01:16:36.074619+0000 mon.a (mon.0) 4011 : audit [DBG] from='client.? 172.21.15.6:0/1285099382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:37.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:36 smithi006 bash[27292]: cluster 2023-08-28T01:16:35.840951+0000 mgr.y (mgr.14901) 5940 : cluster [DBG] pgmap v5947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:37.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:36 smithi006 bash[27292]: audit 2023-08-28T01:16:36.074619+0000 mon.a (mon.0) 4011 : audit [DBG] from='client.? 172.21.15.6:0/1285099382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:37.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:36 smithi006 bash[33918]: cluster 2023-08-28T01:16:35.840951+0000 mgr.y (mgr.14901) 5940 : cluster [DBG] pgmap v5947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:37.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:36 smithi006 bash[33918]: audit 2023-08-28T01:16:36.074619+0000 mon.a (mon.0) 4011 : audit [DBG] from='client.? 172.21.15.6:0/1285099382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:37.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:37 smithi130 bash[51302]: debug 2023-08-28T01:16:37.388+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:37.394434+0000) 2023-08-28T01:16:38.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:38 smithi130 bash[51302]: debug 2023-08-28T01:16:38.388+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:38.394746+0000) 2023-08-28T01:16:39.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:38 smithi130 bash[27579]: cluster 2023-08-28T01:16:37.841537+0000 mgr.y (mgr.14901) 5941 : cluster [DBG] pgmap v5948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:39.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:38 smithi130 bash[27579]: audit 2023-08-28T01:16:38.497275+0000 mon.c (mon.2) 2325 : audit [DBG] from='client.? 172.21.15.6:0/3852312066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:39.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:38 smithi006 bash[33918]: cluster 2023-08-28T01:16:37.841537+0000 mgr.y (mgr.14901) 5941 : cluster [DBG] pgmap v5948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:39.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:38 smithi006 bash[33918]: audit 2023-08-28T01:16:38.497275+0000 mon.c (mon.2) 2325 : audit [DBG] from='client.? 172.21.15.6:0/3852312066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:39.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:38 smithi006 bash[27292]: cluster 2023-08-28T01:16:37.841537+0000 mgr.y (mgr.14901) 5941 : cluster [DBG] pgmap v5948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:39.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:38 smithi006 bash[27292]: audit 2023-08-28T01:16:38.497275+0000 mon.c (mon.2) 2325 : audit [DBG] from='client.? 172.21.15.6:0/3852312066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:39.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:39 smithi130 bash[51302]: debug 2023-08-28T01:16:39.392+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:39.395057+0000) 2023-08-28T01:16:40.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:40 smithi130 bash[51302]: debug 2023-08-28T01:16:40.392+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:40.395261+0000) 2023-08-28T01:16:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:16:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:16:40] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:16:41.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:40 smithi006 bash[27292]: cluster 2023-08-28T01:16:39.842666+0000 mgr.y (mgr.14901) 5942 : cluster [DBG] pgmap v5949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:41.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:40 smithi006 bash[27292]: audit 2023-08-28T01:16:40.920427+0000 mon.a (mon.0) 4012 : audit [DBG] from='client.? 172.21.15.6:0/3472887742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:41.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:40 smithi006 bash[33918]: cluster 2023-08-28T01:16:39.842666+0000 mgr.y (mgr.14901) 5942 : cluster [DBG] pgmap v5949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:41.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:40 smithi006 bash[33918]: audit 2023-08-28T01:16:40.920427+0000 mon.a (mon.0) 4012 : audit [DBG] from='client.? 172.21.15.6:0/3472887742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:16:40.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:16:40.971Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:41.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:40 smithi130 bash[27579]: cluster 2023-08-28T01:16:39.842666+0000 mgr.y (mgr.14901) 5942 : cluster [DBG] pgmap v5949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:41.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:40 smithi130 bash[27579]: audit 2023-08-28T01:16:40.920427+0000 mon.a (mon.0) 4012 : audit [DBG] from='client.? 172.21.15.6:0/3472887742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:41.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:41 smithi130 bash[51302]: debug 2023-08-28T01:16:41.392+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:41.395476+0000) 2023-08-28T01:16:42.723 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:42 smithi130 bash[51302]: debug 2023-08-28T01:16:42.392+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:42.395690+0000) 2023-08-28T01:16:42.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:42 smithi006 bash[33918]: cluster 2023-08-28T01:16:41.843357+0000 mgr.y (mgr.14901) 5943 : cluster [DBG] pgmap v5950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:42 smithi006 bash[27292]: cluster 2023-08-28T01:16:41.843357+0000 mgr.y (mgr.14901) 5943 : cluster [DBG] pgmap v5950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:16:42.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:16:42.544Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:42 smithi130 bash[27579]: cluster 2023-08-28T01:16:41.843357+0000 mgr.y (mgr.14901) 5943 : cluster [DBG] pgmap v5950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:43.726 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:43 smithi130 bash[51302]: debug 2023-08-28T01:16:43.392+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:43.395995+0000) 2023-08-28T01:16:44.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:43 smithi130 bash[27579]: audit 2023-08-28T01:16:43.333814+0000 mon.c (mon.2) 2326 : audit [DBG] from='client.? 172.21.15.6:0/259115897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:44.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:43 smithi006 bash[33918]: audit 2023-08-28T01:16:43.333814+0000 mon.c (mon.2) 2326 : audit [DBG] from='client.? 172.21.15.6:0/259115897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:44.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:43 smithi006 bash[27292]: audit 2023-08-28T01:16:43.333814+0000 mon.c (mon.2) 2326 : audit [DBG] from='client.? 172.21.15.6:0/259115897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:16:44.423Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:44.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:16:44.425Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:44.733 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:44 smithi130 bash[51302]: debug 2023-08-28T01:16:44.392+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:44.396292+0000) 2023-08-28T01:16:45.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:44 smithi130 bash[27579]: cluster 2023-08-28T01:16:43.844072+0000 mgr.y (mgr.14901) 5944 : cluster [DBG] pgmap v5951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:45.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:44 smithi006 bash[27292]: cluster 2023-08-28T01:16:43.844072+0000 mgr.y (mgr.14901) 5944 : cluster [DBG] pgmap v5951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:45.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:44 smithi006 bash[33918]: cluster 2023-08-28T01:16:43.844072+0000 mgr.y (mgr.14901) 5944 : cluster [DBG] pgmap v5951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:45.806 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:45 smithi130 bash[51302]: debug 2023-08-28T01:16:45.392+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:45.396609+0000) 2023-08-28T01:16:46.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:45 smithi130 bash[27579]: audit 2023-08-28T01:16:45.751440+0000 mon.c (mon.2) 2327 : audit [DBG] from='client.? 172.21.15.6:0/2459456844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:46.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:45 smithi006 bash[27292]: audit 2023-08-28T01:16:45.751440+0000 mon.c (mon.2) 2327 : audit [DBG] from='client.? 172.21.15.6:0/2459456844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:46.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:45 smithi006 bash[33918]: audit 2023-08-28T01:16:45.751440+0000 mon.c (mon.2) 2327 : audit [DBG] from='client.? 172.21.15.6:0/2459456844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:46.814 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:46 smithi130 bash[51302]: debug 2023-08-28T01:16:46.392+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:46.396907+0000) 2023-08-28T01:16:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:46 smithi130 bash[27579]: cluster 2023-08-28T01:16:45.845212+0000 mgr.y (mgr.14901) 5945 : cluster [DBG] pgmap v5952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:47.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:46 smithi006 bash[27292]: cluster 2023-08-28T01:16:45.845212+0000 mgr.y (mgr.14901) 5945 : cluster [DBG] pgmap v5952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:47.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:46 smithi006 bash[33918]: cluster 2023-08-28T01:16:45.845212+0000 mgr.y (mgr.14901) 5945 : cluster [DBG] pgmap v5952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:47.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:47 smithi130 bash[51302]: debug 2023-08-28T01:16:47.392+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:47.397275+0000) 2023-08-28T01:16:48.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:48 smithi130 bash[51302]: debug 2023-08-28T01:16:48.396+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:48.397542+0000) 2023-08-28T01:16:49.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:48 smithi006 bash[27292]: cluster 2023-08-28T01:16:47.845861+0000 mgr.y (mgr.14901) 5946 : cluster [DBG] pgmap v5953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:49.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:48 smithi006 bash[27292]: audit 2023-08-28T01:16:48.162619+0000 mon.a (mon.0) 4013 : audit [DBG] from='client.? 172.21.15.6:0/2366970590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:49.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:48 smithi006 bash[33918]: cluster 2023-08-28T01:16:47.845861+0000 mgr.y (mgr.14901) 5946 : cluster [DBG] pgmap v5953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:49.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:48 smithi006 bash[33918]: audit 2023-08-28T01:16:48.162619+0000 mon.a (mon.0) 4013 : audit [DBG] from='client.? 172.21.15.6:0/2366970590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:49.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:48 smithi130 bash[27579]: cluster 2023-08-28T01:16:47.845861+0000 mgr.y (mgr.14901) 5946 : cluster [DBG] pgmap v5953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:49.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:48 smithi130 bash[27579]: audit 2023-08-28T01:16:48.162619+0000 mon.a (mon.0) 4013 : audit [DBG] from='client.? 172.21.15.6:0/2366970590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:49.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:49 smithi130 bash[51302]: debug 2023-08-28T01:16:49.392+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:49.397834+0000) 2023-08-28T01:16:50.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:50 smithi130 bash[51302]: debug 2023-08-28T01:16:50.392+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:50.398117+0000) 2023-08-28T01:16:51.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:16:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:16:50] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T01:16:51.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:50 smithi006 bash[27292]: cluster 2023-08-28T01:16:49.846966+0000 mgr.y (mgr.14901) 5947 : cluster [DBG] pgmap v5954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:51.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:50 smithi006 bash[27292]: audit 2023-08-28T01:16:50.588216+0000 mon.a (mon.0) 4014 : audit [DBG] from='client.? 172.21.15.6:0/290574772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:51.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:50 smithi006 bash[33918]: cluster 2023-08-28T01:16:49.846966+0000 mgr.y (mgr.14901) 5947 : cluster [DBG] pgmap v5954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:51.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:50 smithi006 bash[33918]: audit 2023-08-28T01:16:50.588216+0000 mon.a (mon.0) 4014 : audit [DBG] from='client.? 172.21.15.6:0/290574772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:16:50.971Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:16:50.972Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:51.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:50 smithi130 bash[27579]: cluster 2023-08-28T01:16:49.846966+0000 mgr.y (mgr.14901) 5947 : cluster [DBG] pgmap v5954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:51.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:50 smithi130 bash[27579]: audit 2023-08-28T01:16:50.588216+0000 mon.a (mon.0) 4014 : audit [DBG] from='client.? 172.21.15.6:0/290574772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:51.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:51 smithi130 bash[51302]: debug 2023-08-28T01:16:51.392+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:51.398387+0000) 2023-08-28T01:16:52.723 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:52 smithi130 bash[51302]: debug 2023-08-28T01:16:52.392+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:52.398652+0000) 2023-08-28T01:16:52.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:52 smithi006 bash[27292]: cluster 2023-08-28T01:16:51.847611+0000 mgr.y (mgr.14901) 5948 : cluster [DBG] pgmap v5955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:52 smithi006 bash[33918]: cluster 2023-08-28T01:16:51.847611+0000 mgr.y (mgr.14901) 5948 : cluster [DBG] pgmap v5955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:16:52.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:16:52.544Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:52 smithi130 bash[27579]: cluster 2023-08-28T01:16:51.847611+0000 mgr.y (mgr.14901) 5948 : cluster [DBG] pgmap v5955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:53.726 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:53 smithi130 bash[51302]: debug 2023-08-28T01:16:53.396+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:53.398971+0000) 2023-08-28T01:16:54.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:53 smithi130 bash[27579]: audit 2023-08-28T01:16:52.993915+0000 mon.a (mon.0) 4015 : audit [DBG] from='client.? 172.21.15.6:0/4261211701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:54.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:53 smithi006 bash[27292]: audit 2023-08-28T01:16:52.993915+0000 mon.a (mon.0) 4015 : audit [DBG] from='client.? 172.21.15.6:0/4261211701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:54.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:53 smithi006 bash[33918]: audit 2023-08-28T01:16:52.993915+0000 mon.a (mon.0) 4015 : audit [DBG] from='client.? 172.21.15.6:0/4261211701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:16:54.424Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:54.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:16:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:16:54.427Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:16:54.737 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:54 smithi130 bash[51302]: debug 2023-08-28T01:16:54.396+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:54.399435+0000) 2023-08-28T01:16:55.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:54 smithi130 bash[27579]: cluster 2023-08-28T01:16:53.848277+0000 mgr.y (mgr.14901) 5949 : cluster [DBG] pgmap v5956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:55.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:54 smithi006 bash[33918]: cluster 2023-08-28T01:16:53.848277+0000 mgr.y (mgr.14901) 5949 : cluster [DBG] pgmap v5956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:55.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:54 smithi006 bash[27292]: cluster 2023-08-28T01:16:53.848277+0000 mgr.y (mgr.14901) 5949 : cluster [DBG] pgmap v5956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:55.740 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:55 smithi130 bash[51302]: debug 2023-08-28T01:16:55.396+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:55.399921+0000) 2023-08-28T01:16:56.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:55 smithi130 bash[27579]: audit 2023-08-28T01:16:55.410299+0000 mon.a (mon.0) 4016 : audit [DBG] from='client.? 172.21.15.6:0/519588724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:56.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:55 smithi006 bash[27292]: audit 2023-08-28T01:16:55.410299+0000 mon.a (mon.0) 4016 : audit [DBG] from='client.? 172.21.15.6:0/519588724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:56.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:55 smithi006 bash[33918]: audit 2023-08-28T01:16:55.410299+0000 mon.a (mon.0) 4016 : audit [DBG] from='client.? 172.21.15.6:0/519588724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:56.747 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:56 smithi130 bash[51302]: debug 2023-08-28T01:16:56.396+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:56.400208+0000) 2023-08-28T01:16:57.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:56 smithi130 bash[27579]: cluster 2023-08-28T01:16:55.849411+0000 mgr.y (mgr.14901) 5950 : cluster [DBG] pgmap v5957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:57.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:56 smithi006 bash[27292]: cluster 2023-08-28T01:16:55.849411+0000 mgr.y (mgr.14901) 5950 : cluster [DBG] pgmap v5957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:57.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:56 smithi006 bash[33918]: cluster 2023-08-28T01:16:55.849411+0000 mgr.y (mgr.14901) 5950 : cluster [DBG] pgmap v5957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:57.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:57 smithi130 bash[51302]: debug 2023-08-28T01:16:57.396+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:57.400480+0000) 2023-08-28T01:16:58.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:57 smithi006 bash[33918]: audit 2023-08-28T01:16:57.819546+0000 mon.c (mon.2) 2328 : audit [DBG] from='client.? 172.21.15.6:0/3954432045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:58.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:57 smithi006 bash[27292]: audit 2023-08-28T01:16:57.819546+0000 mon.c (mon.2) 2328 : audit [DBG] from='client.? 172.21.15.6:0/3954432045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:58.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:57 smithi130 bash[27579]: audit 2023-08-28T01:16:57.819546+0000 mon.c (mon.2) 2328 : audit [DBG] from='client.? 172.21.15.6:0/3954432045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:58.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:58 smithi130 bash[51302]: debug 2023-08-28T01:16:58.396+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:58.400749+0000) 2023-08-28T01:16:59.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:58 smithi006 bash[27292]: cluster 2023-08-28T01:16:57.850042+0000 mgr.y (mgr.14901) 5951 : cluster [DBG] pgmap v5958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:59.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:58 smithi006 bash[33918]: cluster 2023-08-28T01:16:57.850042+0000 mgr.y (mgr.14901) 5951 : cluster [DBG] pgmap v5958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:59.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:58 smithi130 bash[27579]: cluster 2023-08-28T01:16:57.850042+0000 mgr.y (mgr.14901) 5951 : cluster [DBG] pgmap v5958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:16:59.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:16:59 smithi130 bash[51302]: debug 2023-08-28T01:16:59.396+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:59.401009+0000) 2023-08-28T01:17:00.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:16:59 smithi006 bash[27292]: audit 2023-08-28T01:16:59.643745+0000 mon.b (mon.1) 689 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:17:00.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:16:59 smithi006 bash[33918]: audit 2023-08-28T01:16:59.643745+0000 mon.b (mon.1) 689 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:17:00.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:16:59 smithi130 bash[27579]: audit 2023-08-28T01:16:59.643745+0000 mon.b (mon.1) 689 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:17:00.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:00 smithi130 bash[51302]: debug 2023-08-28T01:17:00.396+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:00.401295+0000) 2023-08-28T01:17:01.210 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:17:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:17:00] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:17:01.210 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:00 smithi006 bash[33918]: cluster 2023-08-28T01:16:59.850903+0000 mgr.y (mgr.14901) 5952 : cluster [DBG] pgmap v5959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:01.210 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:00 smithi006 bash[33918]: audit 2023-08-28T01:17:00.233875+0000 mon.c (mon.2) 2329 : audit [DBG] from='client.? 172.21.15.6:0/1955857528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:01.211 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:00 smithi006 bash[27292]: cluster 2023-08-28T01:16:59.850903+0000 mgr.y (mgr.14901) 5952 : cluster [DBG] pgmap v5959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:01.211 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:00 smithi006 bash[27292]: audit 2023-08-28T01:17:00.233875+0000 mon.c (mon.2) 2329 : audit [DBG] from='client.? 172.21.15.6:0/1955857528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:01.211 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:17:00.971Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:01.211 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:17:00.973Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:01.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:00 smithi130 bash[27579]: cluster 2023-08-28T01:16:59.850903+0000 mgr.y (mgr.14901) 5952 : cluster [DBG] pgmap v5959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:01.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:00 smithi130 bash[27579]: audit 2023-08-28T01:17:00.233875+0000 mon.c (mon.2) 2329 : audit [DBG] from='client.? 172.21.15.6:0/1955857528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:01.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:01 smithi130 bash[51302]: debug 2023-08-28T01:17:01.396+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:01.401602+0000) 2023-08-28T01:17:02.729 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:02 smithi130 bash[51302]: debug 2023-08-28T01:17:02.396+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:02.401826+0000) 2023-08-28T01:17:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:17:02.543Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:02.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:17:02.545Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:02 smithi006 bash[27292]: cluster 2023-08-28T01:17:01.851537+0000 mgr.y (mgr.14901) 5953 : cluster [DBG] pgmap v5960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:02 smithi006 bash[27292]: audit 2023-08-28T01:17:02.653606+0000 mon.a (mon.0) 4017 : audit [DBG] from='client.? 172.21.15.6:0/2371394544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:02 smithi006 bash[33918]: cluster 2023-08-28T01:17:01.851537+0000 mgr.y (mgr.14901) 5953 : cluster [DBG] pgmap v5960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:02 smithi006 bash[33918]: audit 2023-08-28T01:17:02.653606+0000 mon.a (mon.0) 4017 : audit [DBG] from='client.? 172.21.15.6:0/2371394544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:02 smithi130 bash[27579]: cluster 2023-08-28T01:17:01.851537+0000 mgr.y (mgr.14901) 5953 : cluster [DBG] pgmap v5960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:02 smithi130 bash[27579]: audit 2023-08-28T01:17:02.653606+0000 mon.a (mon.0) 4017 : audit [DBG] from='client.? 172.21.15.6:0/2371394544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:03.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:03 smithi130 bash[51302]: debug 2023-08-28T01:17:03.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:03.402093+0000) 2023-08-28T01:17:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:17:04.426Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:17:04.428Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:04.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:04 smithi130 bash[51302]: debug 2023-08-28T01:17:04.396+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:04.402390+0000) 2023-08-28T01:17:05.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:04 smithi006 bash[27292]: cluster 2023-08-28T01:17:03.852137+0000 mgr.y (mgr.14901) 5954 : cluster [DBG] pgmap v5961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:05.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:04 smithi006 bash[33918]: cluster 2023-08-28T01:17:03.852137+0000 mgr.y (mgr.14901) 5954 : cluster [DBG] pgmap v5961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:05.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:04 smithi130 bash[27579]: cluster 2023-08-28T01:17:03.852137+0000 mgr.y (mgr.14901) 5954 : cluster [DBG] pgmap v5961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:05.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:05 smithi130 bash[51302]: debug 2023-08-28T01:17:05.396+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:05.402603+0000) 2023-08-28T01:17:06.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:05 smithi006 bash[27292]: audit 2023-08-28T01:17:05.141744+0000 mon.a (mon.0) 4018 : audit [DBG] from='client.? 172.21.15.6:0/3656369275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:06.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:05 smithi006 bash[27292]: audit 2023-08-28T01:17:05.532527+0000 mon.a (mon.0) 4019 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:17:06.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:05 smithi006 bash[33918]: audit 2023-08-28T01:17:05.141744+0000 mon.a (mon.0) 4018 : audit [DBG] from='client.? 172.21.15.6:0/3656369275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:06.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:05 smithi006 bash[33918]: audit 2023-08-28T01:17:05.532527+0000 mon.a (mon.0) 4019 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:17:06.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:05 smithi130 bash[27579]: audit 2023-08-28T01:17:05.141744+0000 mon.a (mon.0) 4018 : audit [DBG] from='client.? 172.21.15.6:0/3656369275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:06.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:05 smithi130 bash[27579]: audit 2023-08-28T01:17:05.532527+0000 mon.a (mon.0) 4019 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:17:06.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:06 smithi130 bash[51302]: debug 2023-08-28T01:17:06.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:06.402876+0000) 2023-08-28T01:17:07.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:07 smithi130 bash[27579]: cluster 2023-08-28T01:17:05.853278+0000 mgr.y (mgr.14901) 5955 : cluster [DBG] pgmap v5962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:07.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:07 smithi130 bash[27579]: audit 2023-08-28T01:17:06.029188+0000 mon.a (mon.0) 4020 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:17:07.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:07 smithi130 bash[27579]: audit 2023-08-28T01:17:06.304822+0000 mon.b (mon.1) 690 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:17:07.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:07 smithi130 bash[27579]: audit 2023-08-28T01:17:06.307164+0000 mon.b (mon.1) 691 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:17:07.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:07 smithi130 bash[27579]: audit 2023-08-28T01:17:06.314856+0000 mon.a (mon.0) 4021 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:17:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:07 smithi006 bash[27292]: cluster 2023-08-28T01:17:05.853278+0000 mgr.y (mgr.14901) 5955 : cluster [DBG] pgmap v5962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:07.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:07 smithi006 bash[27292]: audit 2023-08-28T01:17:06.029188+0000 mon.a (mon.0) 4020 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:17:07.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:07 smithi006 bash[27292]: audit 2023-08-28T01:17:06.304822+0000 mon.b (mon.1) 690 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:17:07.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:07 smithi006 bash[27292]: audit 2023-08-28T01:17:06.307164+0000 mon.b (mon.1) 691 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:17:07.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:07 smithi006 bash[27292]: audit 2023-08-28T01:17:06.314856+0000 mon.a (mon.0) 4021 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:17:07.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:07 smithi006 bash[33918]: cluster 2023-08-28T01:17:05.853278+0000 mgr.y (mgr.14901) 5955 : cluster [DBG] pgmap v5962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:07.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:07 smithi006 bash[33918]: audit 2023-08-28T01:17:06.029188+0000 mon.a (mon.0) 4020 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:17:07.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:07 smithi006 bash[33918]: audit 2023-08-28T01:17:06.304822+0000 mon.b (mon.1) 690 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:17:07.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:07 smithi006 bash[33918]: audit 2023-08-28T01:17:06.307164+0000 mon.b (mon.1) 691 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:17:07.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:07 smithi006 bash[33918]: audit 2023-08-28T01:17:06.314856+0000 mon.a (mon.0) 4021 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:17:07.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:07 smithi130 bash[51302]: debug 2023-08-28T01:17:07.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:07.403201+0000) 2023-08-28T01:17:08.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:08 smithi130 bash[27579]: audit 2023-08-28T01:17:07.557367+0000 mon.c (mon.2) 2330 : audit [DBG] from='client.? 172.21.15.6:0/715580705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:08.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:08 smithi006 bash[33918]: audit 2023-08-28T01:17:07.557367+0000 mon.c (mon.2) 2330 : audit [DBG] from='client.? 172.21.15.6:0/715580705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:08.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:08 smithi006 bash[27292]: audit 2023-08-28T01:17:07.557367+0000 mon.c (mon.2) 2330 : audit [DBG] from='client.? 172.21.15.6:0/715580705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:08.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:08 smithi130 bash[51302]: debug 2023-08-28T01:17:08.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:08.403503+0000) 2023-08-28T01:17:09.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:09 smithi130 bash[27579]: cluster 2023-08-28T01:17:07.853930+0000 mgr.y (mgr.14901) 5956 : cluster [DBG] pgmap v5963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:09.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:09 smithi006 bash[33918]: cluster 2023-08-28T01:17:07.853930+0000 mgr.y (mgr.14901) 5956 : cluster [DBG] pgmap v5963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:09.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:09 smithi006 bash[27292]: cluster 2023-08-28T01:17:07.853930+0000 mgr.y (mgr.14901) 5956 : cluster [DBG] pgmap v5963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:09.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:09 smithi130 bash[51302]: debug 2023-08-28T01:17:09.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:09.403816+0000) 2023-08-28T01:17:10.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:10 smithi130 bash[27579]: audit 2023-08-28T01:17:09.977920+0000 mon.a (mon.0) 4022 : audit [DBG] from='client.? 172.21.15.6:0/2959194866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:10.503 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:10 smithi006 bash[33918]: audit 2023-08-28T01:17:09.977920+0000 mon.a (mon.0) 4022 : audit [DBG] from='client.? 172.21.15.6:0/2959194866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:10.503 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:10 smithi006 bash[27292]: audit 2023-08-28T01:17:09.977920+0000 mon.a (mon.0) 4022 : audit [DBG] from='client.? 172.21.15.6:0/2959194866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:10.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:10 smithi130 bash[51302]: debug 2023-08-28T01:17:10.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:10.404238+0000) 2023-08-28T01:17:11.049 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:17:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:17:10] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:17:11.049 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:17:10.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:11.049 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:17:10.974Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:11.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:11 smithi130 bash[27579]: cluster 2023-08-28T01:17:09.854954+0000 mgr.y (mgr.14901) 5957 : cluster [DBG] pgmap v5964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:11.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:11 smithi006 bash[27292]: cluster 2023-08-28T01:17:09.854954+0000 mgr.y (mgr.14901) 5957 : cluster [DBG] pgmap v5964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:11.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:11 smithi006 bash[33918]: cluster 2023-08-28T01:17:09.854954+0000 mgr.y (mgr.14901) 5957 : cluster [DBG] pgmap v5964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:11.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:11 smithi130 bash[51302]: debug 2023-08-28T01:17:11.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:11.404591+0000) 2023-08-28T01:17:12.728 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:12 smithi130 bash[51302]: debug 2023-08-28T01:17:12.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:12.404920+0000) 2023-08-28T01:17:12.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:12 smithi006 bash[27292]: cluster 2023-08-28T01:17:11.855636+0000 mgr.y (mgr.14901) 5958 : cluster [DBG] pgmap v5965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:12 smithi006 bash[27292]: audit 2023-08-28T01:17:12.386690+0000 mon.c (mon.2) 2331 : audit [DBG] from='client.? 172.21.15.6:0/4058622982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:17:12.544Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:17:12.546Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:12 smithi006 bash[33918]: cluster 2023-08-28T01:17:11.855636+0000 mgr.y (mgr.14901) 5958 : cluster [DBG] pgmap v5965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:12 smithi006 bash[33918]: audit 2023-08-28T01:17:12.386690+0000 mon.c (mon.2) 2331 : audit [DBG] from='client.? 172.21.15.6:0/4058622982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:12 smithi130 bash[27579]: cluster 2023-08-28T01:17:11.855636+0000 mgr.y (mgr.14901) 5958 : cluster [DBG] pgmap v5965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:12 smithi130 bash[27579]: audit 2023-08-28T01:17:12.386690+0000 mon.c (mon.2) 2331 : audit [DBG] from='client.? 172.21.15.6:0/4058622982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:13.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:13 smithi130 bash[51302]: debug 2023-08-28T01:17:13.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:13.405167+0000) 2023-08-28T01:17:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:17:14.427Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:17:14.429Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:14.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:14 smithi130 bash[51302]: debug 2023-08-28T01:17:14.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:14.405466+0000) 2023-08-28T01:17:15.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:14 smithi006 bash[33918]: cluster 2023-08-28T01:17:13.856209+0000 mgr.y (mgr.14901) 5959 : cluster [DBG] pgmap v5966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:15.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:14 smithi006 bash[33918]: audit 2023-08-28T01:17:14.809649+0000 mon.a (mon.0) 4023 : audit [DBG] from='client.? 172.21.15.6:0/2248001769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:15.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:14 smithi006 bash[27292]: cluster 2023-08-28T01:17:13.856209+0000 mgr.y (mgr.14901) 5959 : cluster [DBG] pgmap v5966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:15.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:14 smithi006 bash[27292]: audit 2023-08-28T01:17:14.809649+0000 mon.a (mon.0) 4023 : audit [DBG] from='client.? 172.21.15.6:0/2248001769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:15.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:14 smithi130 bash[27579]: cluster 2023-08-28T01:17:13.856209+0000 mgr.y (mgr.14901) 5959 : cluster [DBG] pgmap v5966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:15.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:14 smithi130 bash[27579]: audit 2023-08-28T01:17:14.809649+0000 mon.a (mon.0) 4023 : audit [DBG] from='client.? 172.21.15.6:0/2248001769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:15.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:15 smithi130 bash[51302]: debug 2023-08-28T01:17:15.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:15.405737+0000) 2023-08-28T01:17:16.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:16 smithi130 bash[51302]: debug 2023-08-28T01:17:16.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:16.406001+0000) 2023-08-28T01:17:17.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:16 smithi006 bash[27292]: cluster 2023-08-28T01:17:15.857366+0000 mgr.y (mgr.14901) 5960 : cluster [DBG] pgmap v5967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:17.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:16 smithi006 bash[33918]: cluster 2023-08-28T01:17:15.857366+0000 mgr.y (mgr.14901) 5960 : cluster [DBG] pgmap v5967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:17.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:16 smithi130 bash[27579]: cluster 2023-08-28T01:17:15.857366+0000 mgr.y (mgr.14901) 5960 : cluster [DBG] pgmap v5967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:17.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:17 smithi130 bash[51302]: debug 2023-08-28T01:17:17.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:17.406270+0000) 2023-08-28T01:17:18.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:17 smithi006 bash[33918]: audit 2023-08-28T01:17:17.221879+0000 mon.a (mon.0) 4024 : audit [DBG] from='client.? 172.21.15.6:0/651697992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:18.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:17 smithi006 bash[27292]: audit 2023-08-28T01:17:17.221879+0000 mon.a (mon.0) 4024 : audit [DBG] from='client.? 172.21.15.6:0/651697992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:18.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:17 smithi130 bash[27579]: audit 2023-08-28T01:17:17.221879+0000 mon.a (mon.0) 4024 : audit [DBG] from='client.? 172.21.15.6:0/651697992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:18.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:18 smithi130 bash[51302]: debug 2023-08-28T01:17:18.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:18.406483+0000) 2023-08-28T01:17:19.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:18 smithi006 bash[27292]: cluster 2023-08-28T01:17:17.858032+0000 mgr.y (mgr.14901) 5961 : cluster [DBG] pgmap v5968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:19.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:18 smithi006 bash[33918]: cluster 2023-08-28T01:17:17.858032+0000 mgr.y (mgr.14901) 5961 : cluster [DBG] pgmap v5968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:19.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:18 smithi130 bash[27579]: cluster 2023-08-28T01:17:17.858032+0000 mgr.y (mgr.14901) 5961 : cluster [DBG] pgmap v5968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:19.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:19 smithi130 bash[51302]: debug 2023-08-28T01:17:19.400+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:19.406765+0000) 2023-08-28T01:17:20.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:19 smithi006 bash[27292]: audit 2023-08-28T01:17:19.642384+0000 mon.c (mon.2) 2332 : audit [DBG] from='client.? 172.21.15.6:0/4279126441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:20.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:19 smithi006 bash[33918]: audit 2023-08-28T01:17:19.642384+0000 mon.c (mon.2) 2332 : audit [DBG] from='client.? 172.21.15.6:0/4279126441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:20.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:19 smithi130 bash[27579]: audit 2023-08-28T01:17:19.642384+0000 mon.c (mon.2) 2332 : audit [DBG] from='client.? 172.21.15.6:0/4279126441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:20.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:20 smithi130 bash[51302]: debug 2023-08-28T01:17:20.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:20.407051+0000) 2023-08-28T01:17:21.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:17:20.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:21.219 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:17:20.975Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:21.219 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:20 smithi006 bash[27292]: cluster 2023-08-28T01:17:19.859175+0000 mgr.y (mgr.14901) 5962 : cluster [DBG] pgmap v5969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:21.220 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:17:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:17:20] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:17:21.220 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:20 smithi006 bash[33918]: cluster 2023-08-28T01:17:19.859175+0000 mgr.y (mgr.14901) 5962 : cluster [DBG] pgmap v5969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:21.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:20 smithi130 bash[27579]: cluster 2023-08-28T01:17:19.859175+0000 mgr.y (mgr.14901) 5962 : cluster [DBG] pgmap v5969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:21.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:21 smithi130 bash[51302]: debug 2023-08-28T01:17:21.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:21.407297+0000) 2023-08-28T01:17:22.734 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:22 smithi130 bash[51302]: debug 2023-08-28T01:17:22.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:22.407548+0000) 2023-08-28T01:17:22.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:22 smithi006 bash[27292]: cluster 2023-08-28T01:17:21.859816+0000 mgr.y (mgr.14901) 5963 : cluster [DBG] pgmap v5970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:22.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:22 smithi006 bash[27292]: audit 2023-08-28T01:17:22.060360+0000 mon.a (mon.0) 4025 : audit [DBG] from='client.? 172.21.15.6:0/2110979239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:22.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:17:22.544Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:22.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:17:22.546Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:22 smithi006 bash[33918]: cluster 2023-08-28T01:17:21.859816+0000 mgr.y (mgr.14901) 5963 : cluster [DBG] pgmap v5970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:22.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:22 smithi006 bash[33918]: audit 2023-08-28T01:17:22.060360+0000 mon.a (mon.0) 4025 : audit [DBG] from='client.? 172.21.15.6:0/2110979239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:23.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:22 smithi130 bash[27579]: cluster 2023-08-28T01:17:21.859816+0000 mgr.y (mgr.14901) 5963 : cluster [DBG] pgmap v5970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:23.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:22 smithi130 bash[27579]: audit 2023-08-28T01:17:22.060360+0000 mon.a (mon.0) 4025 : audit [DBG] from='client.? 172.21.15.6:0/2110979239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:23.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:23 smithi130 bash[51302]: debug 2023-08-28T01:17:23.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:23.407824+0000) 2023-08-28T01:17:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:17:24.427Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:24.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:17:24.429Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:24.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:24 smithi130 bash[51302]: debug 2023-08-28T01:17:24.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:24.408129+0000) 2023-08-28T01:17:25.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:24 smithi006 bash[33918]: cluster 2023-08-28T01:17:23.860457+0000 mgr.y (mgr.14901) 5964 : cluster [DBG] pgmap v5971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:25.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:24 smithi006 bash[33918]: audit 2023-08-28T01:17:24.473836+0000 mon.a (mon.0) 4026 : audit [DBG] from='client.? 172.21.15.6:0/265984467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:25.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:24 smithi006 bash[27292]: cluster 2023-08-28T01:17:23.860457+0000 mgr.y (mgr.14901) 5964 : cluster [DBG] pgmap v5971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:25.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:24 smithi006 bash[27292]: audit 2023-08-28T01:17:24.473836+0000 mon.a (mon.0) 4026 : audit [DBG] from='client.? 172.21.15.6:0/265984467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:25.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:24 smithi130 bash[27579]: cluster 2023-08-28T01:17:23.860457+0000 mgr.y (mgr.14901) 5964 : cluster [DBG] pgmap v5971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:25.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:24 smithi130 bash[27579]: audit 2023-08-28T01:17:24.473836+0000 mon.a (mon.0) 4026 : audit [DBG] from='client.? 172.21.15.6:0/265984467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:25.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:25 smithi130 bash[51302]: debug 2023-08-28T01:17:25.408+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:25.408436+0000) 2023-08-28T01:17:26.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:26 smithi130 bash[51302]: debug 2023-08-28T01:17:26.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:26.408721+0000) 2023-08-28T01:17:27.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:26 smithi006 bash[33918]: cluster 2023-08-28T01:17:25.861740+0000 mgr.y (mgr.14901) 5965 : cluster [DBG] pgmap v5972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:27.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:26 smithi006 bash[33918]: audit 2023-08-28T01:17:26.895278+0000 mon.c (mon.2) 2333 : audit [DBG] from='client.? 172.21.15.6:0/3769006945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:27.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:26 smithi006 bash[27292]: cluster 2023-08-28T01:17:25.861740+0000 mgr.y (mgr.14901) 5965 : cluster [DBG] pgmap v5972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:27.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:26 smithi006 bash[27292]: audit 2023-08-28T01:17:26.895278+0000 mon.c (mon.2) 2333 : audit [DBG] from='client.? 172.21.15.6:0/3769006945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:27.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:26 smithi130 bash[27579]: cluster 2023-08-28T01:17:25.861740+0000 mgr.y (mgr.14901) 5965 : cluster [DBG] pgmap v5972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:27.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:26 smithi130 bash[27579]: audit 2023-08-28T01:17:26.895278+0000 mon.c (mon.2) 2333 : audit [DBG] from='client.? 172.21.15.6:0/3769006945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:27.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:27 smithi130 bash[51302]: debug 2023-08-28T01:17:27.408+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:27.408975+0000) 2023-08-28T01:17:28.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:28 smithi130 bash[51302]: debug 2023-08-28T01:17:28.408+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:28.409302+0000) 2023-08-28T01:17:29.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:28 smithi006 bash[27292]: cluster 2023-08-28T01:17:27.862555+0000 mgr.y (mgr.14901) 5966 : cluster [DBG] pgmap v5973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:29.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:28 smithi006 bash[33918]: cluster 2023-08-28T01:17:27.862555+0000 mgr.y (mgr.14901) 5966 : cluster [DBG] pgmap v5973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:29.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:28 smithi130 bash[27579]: cluster 2023-08-28T01:17:27.862555+0000 mgr.y (mgr.14901) 5966 : cluster [DBG] pgmap v5973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:29.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:29 smithi130 bash[51302]: debug 2023-08-28T01:17:29.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:29.409611+0000) 2023-08-28T01:17:30.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:29 smithi006 bash[33918]: audit 2023-08-28T01:17:29.309728+0000 mon.c (mon.2) 2334 : audit [DBG] from='client.? 172.21.15.6:0/2139635902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:30.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:29 smithi006 bash[27292]: audit 2023-08-28T01:17:29.309728+0000 mon.c (mon.2) 2334 : audit [DBG] from='client.? 172.21.15.6:0/2139635902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:30.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:29 smithi130 bash[27579]: audit 2023-08-28T01:17:29.309728+0000 mon.c (mon.2) 2334 : audit [DBG] from='client.? 172.21.15.6:0/2139635902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:30.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:30 smithi130 bash[51302]: debug 2023-08-28T01:17:30.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:30.409928+0000) 2023-08-28T01:17:31.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:30 smithi006 bash[27292]: cluster 2023-08-28T01:17:29.863827+0000 mgr.y (mgr.14901) 5967 : cluster [DBG] pgmap v5974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:17:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:17:30] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:17:31.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:30 smithi006 bash[33918]: cluster 2023-08-28T01:17:29.863827+0000 mgr.y (mgr.14901) 5967 : cluster [DBG] pgmap v5974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:31.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:17:30.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:31.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:30 smithi006 bash[56298]: level=warn ts=2023-08-28T01:17:30.976Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:31.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:30 smithi130 bash[27579]: cluster 2023-08-28T01:17:29.863827+0000 mgr.y (mgr.14901) 5967 : cluster [DBG] pgmap v5974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:31.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:31 smithi130 bash[51302]: debug 2023-08-28T01:17:31.404+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:31.410234+0000) 2023-08-28T01:17:32.733 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:32 smithi130 bash[51302]: debug 2023-08-28T01:17:32.403+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:32.410532+0000) 2023-08-28T01:17:32.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:32 smithi006 bash[33918]: audit 2023-08-28T01:17:31.728819+0000 mon.c (mon.2) 2335 : audit [DBG] from='client.? 172.21.15.6:0/655981155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:32 smithi006 bash[33918]: cluster 2023-08-28T01:17:31.864527+0000 mgr.y (mgr.14901) 5968 : cluster [DBG] pgmap v5975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:17:32.546Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:17:32.548Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:32 smithi006 bash[27292]: audit 2023-08-28T01:17:31.728819+0000 mon.c (mon.2) 2335 : audit [DBG] from='client.? 172.21.15.6:0/655981155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:32 smithi006 bash[27292]: cluster 2023-08-28T01:17:31.864527+0000 mgr.y (mgr.14901) 5968 : cluster [DBG] pgmap v5975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:32 smithi130 bash[27579]: audit 2023-08-28T01:17:31.728819+0000 mon.c (mon.2) 2335 : audit [DBG] from='client.? 172.21.15.6:0/655981155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:32 smithi130 bash[27579]: cluster 2023-08-28T01:17:31.864527+0000 mgr.y (mgr.14901) 5968 : cluster [DBG] pgmap v5975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:33.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:33 smithi130 bash[51302]: debug 2023-08-28T01:17:33.403+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:33.410798+0000) 2023-08-28T01:17:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:17:34.428Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:34.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:17:34.430Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:34.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:34 smithi130 bash[51302]: debug 2023-08-28T01:17:34.407+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:34.410951+0000) 2023-08-28T01:17:35.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:34 smithi006 bash[33918]: cluster 2023-08-28T01:17:33.865255+0000 mgr.y (mgr.14901) 5969 : cluster [DBG] pgmap v5976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:35.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:34 smithi006 bash[33918]: audit 2023-08-28T01:17:34.141331+0000 mon.a (mon.0) 4027 : audit [DBG] from='client.? 172.21.15.6:0/475112279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:35.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:34 smithi006 bash[27292]: cluster 2023-08-28T01:17:33.865255+0000 mgr.y (mgr.14901) 5969 : cluster [DBG] pgmap v5976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:35.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:34 smithi006 bash[27292]: audit 2023-08-28T01:17:34.141331+0000 mon.a (mon.0) 4027 : audit [DBG] from='client.? 172.21.15.6:0/475112279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:35.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:34 smithi130 bash[27579]: cluster 2023-08-28T01:17:33.865255+0000 mgr.y (mgr.14901) 5969 : cluster [DBG] pgmap v5976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:35.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:34 smithi130 bash[27579]: audit 2023-08-28T01:17:34.141331+0000 mon.a (mon.0) 4027 : audit [DBG] from='client.? 172.21.15.6:0/475112279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:35.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:35 smithi130 bash[51302]: debug 2023-08-28T01:17:35.407+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:35.411236+0000) 2023-08-28T01:17:36.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:36 smithi130 bash[51302]: debug 2023-08-28T01:17:36.407+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:36.411532+0000) 2023-08-28T01:17:37.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:36 smithi006 bash[27292]: cluster 2023-08-28T01:17:35.866366+0000 mgr.y (mgr.14901) 5970 : cluster [DBG] pgmap v5977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:37.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:36 smithi006 bash[27292]: audit 2023-08-28T01:17:36.563598+0000 mon.a (mon.0) 4028 : audit [DBG] from='client.? 172.21.15.6:0/4044010369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:37.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:36 smithi006 bash[33918]: cluster 2023-08-28T01:17:35.866366+0000 mgr.y (mgr.14901) 5970 : cluster [DBG] pgmap v5977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:37.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:36 smithi006 bash[33918]: audit 2023-08-28T01:17:36.563598+0000 mon.a (mon.0) 4028 : audit [DBG] from='client.? 172.21.15.6:0/4044010369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:37.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:36 smithi130 bash[27579]: cluster 2023-08-28T01:17:35.866366+0000 mgr.y (mgr.14901) 5970 : cluster [DBG] pgmap v5977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:37.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:36 smithi130 bash[27579]: audit 2023-08-28T01:17:36.563598+0000 mon.a (mon.0) 4028 : audit [DBG] from='client.? 172.21.15.6:0/4044010369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:37.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:37 smithi130 bash[51302]: debug 2023-08-28T01:17:37.407+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:37.411797+0000) 2023-08-28T01:17:38.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:38 smithi130 bash[51302]: debug 2023-08-28T01:17:38.407+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:38.412162+0000) 2023-08-28T01:17:39.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:38 smithi006 bash[27292]: cluster 2023-08-28T01:17:37.867012+0000 mgr.y (mgr.14901) 5971 : cluster [DBG] pgmap v5978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:39.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:38 smithi006 bash[33918]: cluster 2023-08-28T01:17:37.867012+0000 mgr.y (mgr.14901) 5971 : cluster [DBG] pgmap v5978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:39.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:38 smithi130 bash[27579]: cluster 2023-08-28T01:17:37.867012+0000 mgr.y (mgr.14901) 5971 : cluster [DBG] pgmap v5978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:39.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:39 smithi130 bash[51302]: debug 2023-08-28T01:17:39.407+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:39.412462+0000) 2023-08-28T01:17:40.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:39 smithi006 bash[27292]: audit 2023-08-28T01:17:38.986610+0000 mon.a (mon.0) 4029 : audit [DBG] from='client.? 172.21.15.6:0/1266988890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:40.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:39 smithi006 bash[33918]: audit 2023-08-28T01:17:38.986610+0000 mon.a (mon.0) 4029 : audit [DBG] from='client.? 172.21.15.6:0/1266988890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:40.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:39 smithi130 bash[27579]: audit 2023-08-28T01:17:38.986610+0000 mon.a (mon.0) 4029 : audit [DBG] from='client.? 172.21.15.6:0/1266988890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:40.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:40 smithi130 bash[51302]: debug 2023-08-28T01:17:40.407+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:40.412750+0000) 2023-08-28T01:17:41.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:17:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:17:40] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:17:41.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:40 smithi006 bash[33918]: cluster 2023-08-28T01:17:39.868111+0000 mgr.y (mgr.14901) 5972 : cluster [DBG] pgmap v5979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:41.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:40 smithi006 bash[27292]: cluster 2023-08-28T01:17:39.868111+0000 mgr.y (mgr.14901) 5972 : cluster [DBG] pgmap v5979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:41.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:17:40.975Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:17:40.977Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:41.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:40 smithi130 bash[27579]: cluster 2023-08-28T01:17:39.868111+0000 mgr.y (mgr.14901) 5972 : cluster [DBG] pgmap v5979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:41.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:41 smithi130 bash[51302]: debug 2023-08-28T01:17:41.407+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:41.413072+0000) 2023-08-28T01:17:42.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:41 smithi006 bash[33918]: audit 2023-08-28T01:17:41.395300+0000 mon.c (mon.2) 2336 : audit [DBG] from='client.? 172.21.15.6:0/3521949882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:42.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:41 smithi006 bash[27292]: audit 2023-08-28T01:17:41.395300+0000 mon.c (mon.2) 2336 : audit [DBG] from='client.? 172.21.15.6:0/3521949882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:42.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:41 smithi130 bash[27579]: audit 2023-08-28T01:17:41.395300+0000 mon.c (mon.2) 2336 : audit [DBG] from='client.? 172.21.15.6:0/3521949882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:42.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:42 smithi130 bash[51302]: debug 2023-08-28T01:17:42.407+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:42.413254+0000) 2023-08-28T01:17:42.949 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:17:42.547Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:42.950 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:17:42.549Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:43.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:42 smithi006 bash[27292]: cluster 2023-08-28T01:17:41.868757+0000 mgr.y (mgr.14901) 5973 : cluster [DBG] pgmap v5980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:43.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:42 smithi006 bash[33918]: cluster 2023-08-28T01:17:41.868757+0000 mgr.y (mgr.14901) 5973 : cluster [DBG] pgmap v5980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:43.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:42 smithi130 bash[27579]: cluster 2023-08-28T01:17:41.868757+0000 mgr.y (mgr.14901) 5973 : cluster [DBG] pgmap v5980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:43.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:43 smithi130 bash[51302]: debug 2023-08-28T01:17:43.407+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:43.413510+0000) 2023-08-28T01:17:44.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:43 smithi006 bash[33918]: audit 2023-08-28T01:17:43.808266+0000 mon.c (mon.2) 2337 : audit [DBG] from='client.? 172.21.15.6:0/3254640278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:44.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:43 smithi006 bash[27292]: audit 2023-08-28T01:17:43.808266+0000 mon.c (mon.2) 2337 : audit [DBG] from='client.? 172.21.15.6:0/3254640278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:44.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:43 smithi130 bash[27579]: audit 2023-08-28T01:17:43.808266+0000 mon.c (mon.2) 2337 : audit [DBG] from='client.? 172.21.15.6:0/3254640278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:17:44.429Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:44.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:17:44.431Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:44.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:44 smithi130 bash[51302]: debug 2023-08-28T01:17:44.407+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:44.413790+0000) 2023-08-28T01:17:45.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:44 smithi006 bash[33918]: cluster 2023-08-28T01:17:43.869397+0000 mgr.y (mgr.14901) 5974 : cluster [DBG] pgmap v5981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:45.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:44 smithi006 bash[27292]: cluster 2023-08-28T01:17:43.869397+0000 mgr.y (mgr.14901) 5974 : cluster [DBG] pgmap v5981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:45.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:44 smithi130 bash[27579]: cluster 2023-08-28T01:17:43.869397+0000 mgr.y (mgr.14901) 5974 : cluster [DBG] pgmap v5981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:45.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:45 smithi130 bash[51302]: debug 2023-08-28T01:17:45.411+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:45.414067+0000) 2023-08-28T01:17:46.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:46 smithi130 bash[51302]: debug 2023-08-28T01:17:46.407+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:46.414347+0000) 2023-08-28T01:17:47.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:46 smithi130 bash[27579]: cluster 2023-08-28T01:17:45.870218+0000 mgr.y (mgr.14901) 5975 : cluster [DBG] pgmap v5982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:47.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:46 smithi130 bash[27579]: audit 2023-08-28T01:17:46.217600+0000 mon.c (mon.2) 2338 : audit [DBG] from='client.? 172.21.15.6:0/804073589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:47.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:46 smithi006 bash[27292]: cluster 2023-08-28T01:17:45.870218+0000 mgr.y (mgr.14901) 5975 : cluster [DBG] pgmap v5982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:47.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:46 smithi006 bash[27292]: audit 2023-08-28T01:17:46.217600+0000 mon.c (mon.2) 2338 : audit [DBG] from='client.? 172.21.15.6:0/804073589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:47.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:46 smithi006 bash[33918]: cluster 2023-08-28T01:17:45.870218+0000 mgr.y (mgr.14901) 5975 : cluster [DBG] pgmap v5982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:47.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:46 smithi006 bash[33918]: audit 2023-08-28T01:17:46.217600+0000 mon.c (mon.2) 2338 : audit [DBG] from='client.? 172.21.15.6:0/804073589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:47.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:47 smithi130 bash[51302]: debug 2023-08-28T01:17:47.407+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:47.414631+0000) 2023-08-28T01:17:48.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:48 smithi130 bash[51302]: debug 2023-08-28T01:17:48.411+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:48.414942+0000) 2023-08-28T01:17:49.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:48 smithi130 bash[27579]: cluster 2023-08-28T01:17:47.870817+0000 mgr.y (mgr.14901) 5976 : cluster [DBG] pgmap v5983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:49.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:48 smithi130 bash[27579]: audit 2023-08-28T01:17:48.635074+0000 mon.a (mon.0) 4030 : audit [DBG] from='client.? 172.21.15.6:0/3387471651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:49.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:48 smithi006 bash[27292]: cluster 2023-08-28T01:17:47.870817+0000 mgr.y (mgr.14901) 5976 : cluster [DBG] pgmap v5983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:49.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:48 smithi006 bash[27292]: audit 2023-08-28T01:17:48.635074+0000 mon.a (mon.0) 4030 : audit [DBG] from='client.? 172.21.15.6:0/3387471651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:49.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:48 smithi006 bash[33918]: cluster 2023-08-28T01:17:47.870817+0000 mgr.y (mgr.14901) 5976 : cluster [DBG] pgmap v5983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:49.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:48 smithi006 bash[33918]: audit 2023-08-28T01:17:48.635074+0000 mon.a (mon.0) 4030 : audit [DBG] from='client.? 172.21.15.6:0/3387471651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:49.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:49 smithi130 bash[51302]: debug 2023-08-28T01:17:49.411+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:49.415229+0000) 2023-08-28T01:17:50.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:50 smithi130 bash[51302]: debug 2023-08-28T01:17:50.411+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:50.415527+0000) 2023-08-28T01:17:51.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:17:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:17:50] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:17:51.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:50 smithi006 bash[33918]: cluster 2023-08-28T01:17:49.871928+0000 mgr.y (mgr.14901) 5977 : cluster [DBG] pgmap v5984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:51.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:50 smithi006 bash[27292]: cluster 2023-08-28T01:17:49.871928+0000 mgr.y (mgr.14901) 5977 : cluster [DBG] pgmap v5984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:17:50.975Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:17:50.978Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:51.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:50 smithi130 bash[27579]: cluster 2023-08-28T01:17:49.871928+0000 mgr.y (mgr.14901) 5977 : cluster [DBG] pgmap v5984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:51.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:51 smithi130 bash[51302]: debug 2023-08-28T01:17:51.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:51.415810+0000) 2023-08-28T01:17:52.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:51 smithi130 bash[27579]: audit 2023-08-28T01:17:51.050496+0000 mon.a (mon.0) 4031 : audit [DBG] from='client.? 172.21.15.6:0/869668632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:52.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:51 smithi006 bash[33918]: audit 2023-08-28T01:17:51.050496+0000 mon.a (mon.0) 4031 : audit [DBG] from='client.? 172.21.15.6:0/869668632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:52.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:51 smithi006 bash[27292]: audit 2023-08-28T01:17:51.050496+0000 mon.a (mon.0) 4031 : audit [DBG] from='client.? 172.21.15.6:0/869668632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:52.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:52 smithi130 bash[51302]: debug 2023-08-28T01:17:52.411+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:52.416229+0000) 2023-08-28T01:17:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:17:52.547Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:17:52.549Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:53.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:52 smithi130 bash[27579]: cluster 2023-08-28T01:17:51.872602+0000 mgr.y (mgr.14901) 5978 : cluster [DBG] pgmap v5985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:53.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:52 smithi006 bash[33918]: cluster 2023-08-28T01:17:51.872602+0000 mgr.y (mgr.14901) 5978 : cluster [DBG] pgmap v5985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:53.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:52 smithi006 bash[27292]: cluster 2023-08-28T01:17:51.872602+0000 mgr.y (mgr.14901) 5978 : cluster [DBG] pgmap v5985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:53.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:53 smithi130 bash[51302]: debug 2023-08-28T01:17:53.411+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:53.416560+0000) 2023-08-28T01:17:54.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:53 smithi130 bash[27579]: audit 2023-08-28T01:17:53.464095+0000 mon.a (mon.0) 4032 : audit [DBG] from='client.? 172.21.15.6:0/3143157395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:54.432 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:53 smithi006 bash[27292]: audit 2023-08-28T01:17:53.464095+0000 mon.a (mon.0) 4032 : audit [DBG] from='client.? 172.21.15.6:0/3143157395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:54.432 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:53 smithi006 bash[33918]: audit 2023-08-28T01:17:53.464095+0000 mon.a (mon.0) 4032 : audit [DBG] from='client.? 172.21.15.6:0/3143157395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:17:54.430Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:54.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:17:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:17:54.432Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:17:54.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:54 smithi130 bash[51302]: debug 2023-08-28T01:17:54.411+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:54.416882+0000) 2023-08-28T01:17:55.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:54 smithi130 bash[27579]: cluster 2023-08-28T01:17:53.873269+0000 mgr.y (mgr.14901) 5979 : cluster [DBG] pgmap v5986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:55.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:54 smithi006 bash[33918]: cluster 2023-08-28T01:17:53.873269+0000 mgr.y (mgr.14901) 5979 : cluster [DBG] pgmap v5986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:55.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:54 smithi006 bash[27292]: cluster 2023-08-28T01:17:53.873269+0000 mgr.y (mgr.14901) 5979 : cluster [DBG] pgmap v5986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:55.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:55 smithi130 bash[51302]: debug 2023-08-28T01:17:55.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:55.417322+0000) 2023-08-28T01:17:56.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:56 smithi130 bash[27579]: audit 2023-08-28T01:17:55.875630+0000 mon.c (mon.2) 2339 : audit [DBG] from='client.? 172.21.15.6:0/916191568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:56.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:56 smithi006 bash[27292]: audit 2023-08-28T01:17:55.875630+0000 mon.c (mon.2) 2339 : audit [DBG] from='client.? 172.21.15.6:0/916191568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:56.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:56 smithi006 bash[33918]: audit 2023-08-28T01:17:55.875630+0000 mon.c (mon.2) 2339 : audit [DBG] from='client.? 172.21.15.6:0/916191568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:56.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:56 smithi130 bash[51302]: debug 2023-08-28T01:17:56.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:56.417661+0000) 2023-08-28T01:17:57.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:57 smithi130 bash[27579]: cluster 2023-08-28T01:17:55.874343+0000 mgr.y (mgr.14901) 5980 : cluster [DBG] pgmap v5987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:57.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:57 smithi006 bash[27292]: cluster 2023-08-28T01:17:55.874343+0000 mgr.y (mgr.14901) 5980 : cluster [DBG] pgmap v5987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:57.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:57 smithi006 bash[33918]: cluster 2023-08-28T01:17:55.874343+0000 mgr.y (mgr.14901) 5980 : cluster [DBG] pgmap v5987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:57.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:57 smithi130 bash[51302]: debug 2023-08-28T01:17:57.411+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:57.417948+0000) 2023-08-28T01:17:58.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:58 smithi130 bash[51302]: debug 2023-08-28T01:17:58.411+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:58.418228+0000) 2023-08-28T01:17:59.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:59 smithi130 bash[27579]: cluster 2023-08-28T01:17:57.875011+0000 mgr.y (mgr.14901) 5981 : cluster [DBG] pgmap v5988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:59.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:17:59 smithi130 bash[27579]: audit 2023-08-28T01:17:58.289032+0000 mon.a (mon.0) 4033 : audit [DBG] from='client.? 172.21.15.6:0/1586527295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:59.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:59 smithi006 bash[33918]: cluster 2023-08-28T01:17:57.875011+0000 mgr.y (mgr.14901) 5981 : cluster [DBG] pgmap v5988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:59.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:17:59 smithi006 bash[33918]: audit 2023-08-28T01:17:58.289032+0000 mon.a (mon.0) 4033 : audit [DBG] from='client.? 172.21.15.6:0/1586527295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:59.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:59 smithi006 bash[27292]: cluster 2023-08-28T01:17:57.875011+0000 mgr.y (mgr.14901) 5981 : cluster [DBG] pgmap v5988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:17:59.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:17:59 smithi006 bash[27292]: audit 2023-08-28T01:17:58.289032+0000 mon.a (mon.0) 4033 : audit [DBG] from='client.? 172.21.15.6:0/1586527295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:59.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:17:59 smithi130 bash[51302]: debug 2023-08-28T01:17:59.411+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:59.418449+0000) 2023-08-28T01:18:00.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:00 smithi130 bash[51302]: debug 2023-08-28T01:18:00.411+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:00.418749+0000) 2023-08-28T01:18:01.019 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:18:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:18:00] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:18:01.019 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:18:00.976Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:01.019 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:18:00.978Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:01.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:01 smithi130 bash[27579]: cluster 2023-08-28T01:17:59.876270+0000 mgr.y (mgr.14901) 5982 : cluster [DBG] pgmap v5989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:01.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:01 smithi130 bash[27579]: audit 2023-08-28T01:18:00.729510+0000 mon.c (mon.2) 2340 : audit [DBG] from='client.? 172.21.15.6:0/2617829332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:01.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:01 smithi006 bash[27292]: cluster 2023-08-28T01:17:59.876270+0000 mgr.y (mgr.14901) 5982 : cluster [DBG] pgmap v5989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:01.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:01 smithi006 bash[27292]: audit 2023-08-28T01:18:00.729510+0000 mon.c (mon.2) 2340 : audit [DBG] from='client.? 172.21.15.6:0/2617829332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:01.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:01 smithi006 bash[33918]: cluster 2023-08-28T01:17:59.876270+0000 mgr.y (mgr.14901) 5982 : cluster [DBG] pgmap v5989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:01.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:01 smithi006 bash[33918]: audit 2023-08-28T01:18:00.729510+0000 mon.c (mon.2) 2340 : audit [DBG] from='client.? 172.21.15.6:0/2617829332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:01.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:01 smithi130 bash[51302]: debug 2023-08-28T01:18:01.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:01.419053+0000) 2023-08-28T01:18:02.741 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:02 smithi130 bash[51302]: debug 2023-08-28T01:18:02.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:02.419310+0000) 2023-08-28T01:18:03.198 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:02 smithi006 bash[27292]: cluster 2023-08-28T01:18:01.877031+0000 mgr.y (mgr.14901) 5983 : cluster [DBG] pgmap v5990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:03.199 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:02 smithi130 bash[27579]: cluster 2023-08-28T01:18:01.877031+0000 mgr.y (mgr.14901) 5983 : cluster [DBG] pgmap v5990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:03.199 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:02 smithi006 bash[33918]: cluster 2023-08-28T01:18:01.877031+0000 mgr.y (mgr.14901) 5983 : cluster [DBG] pgmap v5990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:03.199 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:18:02.547Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:03.199 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:18:02.550Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:03.744 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:03 smithi130 bash[51302]: debug 2023-08-28T01:18:03.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:03.419641+0000) 2023-08-28T01:18:04.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:03 smithi130 bash[27579]: audit 2023-08-28T01:18:03.149171+0000 mon.a (mon.0) 4034 : audit [DBG] from='client.? 172.21.15.6:0/285377045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:04.281 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:03 smithi006 bash[33918]: audit 2023-08-28T01:18:03.149171+0000 mon.a (mon.0) 4034 : audit [DBG] from='client.? 172.21.15.6:0/285377045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:04.281 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:03 smithi006 bash[27292]: audit 2023-08-28T01:18:03.149171+0000 mon.a (mon.0) 4034 : audit [DBG] from='client.? 172.21.15.6:0/285377045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:18:04.431Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:18:04.433Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:04.752 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:04 smithi130 bash[51302]: debug 2023-08-28T01:18:04.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:04.419944+0000) 2023-08-28T01:18:05.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:04 smithi130 bash[27579]: cluster 2023-08-28T01:18:03.877686+0000 mgr.y (mgr.14901) 5984 : cluster [DBG] pgmap v5991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:05.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:04 smithi006 bash[27292]: cluster 2023-08-28T01:18:03.877686+0000 mgr.y (mgr.14901) 5984 : cluster [DBG] pgmap v5991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:05.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:04 smithi006 bash[33918]: cluster 2023-08-28T01:18:03.877686+0000 mgr.y (mgr.14901) 5984 : cluster [DBG] pgmap v5991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:05.754 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:05 smithi130 bash[51302]: debug 2023-08-28T01:18:05.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:05.420216+0000) 2023-08-28T01:18:06.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:05 smithi130 bash[27579]: audit 2023-08-28T01:18:05.565293+0000 mon.c (mon.2) 2341 : audit [DBG] from='client.? 172.21.15.6:0/1269335364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:06.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:05 smithi006 bash[33918]: audit 2023-08-28T01:18:05.565293+0000 mon.c (mon.2) 2341 : audit [DBG] from='client.? 172.21.15.6:0/1269335364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:06.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:05 smithi006 bash[27292]: audit 2023-08-28T01:18:05.565293+0000 mon.c (mon.2) 2341 : audit [DBG] from='client.? 172.21.15.6:0/1269335364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:06.761 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:06 smithi130 bash[51302]: debug 2023-08-28T01:18:06.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:06.420465+0000) 2023-08-28T01:18:07.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:06 smithi130 bash[27579]: cluster 2023-08-28T01:18:05.878842+0000 mgr.y (mgr.14901) 5985 : cluster [DBG] pgmap v5992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:07.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:06 smithi130 bash[27579]: audit 2023-08-28T01:18:06.326074+0000 mon.b (mon.1) 692 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:18:07.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:06 smithi130 bash[27579]: audit 2023-08-28T01:18:06.602740+0000 mon.a (mon.0) 4035 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:18:07.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:06 smithi130 bash[27579]: audit 2023-08-28T01:18:06.607478+0000 mon.a (mon.0) 4036 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:18:07.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:06 smithi006 bash[33918]: cluster 2023-08-28T01:18:05.878842+0000 mgr.y (mgr.14901) 5985 : cluster [DBG] pgmap v5992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:07.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:06 smithi006 bash[33918]: audit 2023-08-28T01:18:06.326074+0000 mon.b (mon.1) 692 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:18:07.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:06 smithi006 bash[33918]: audit 2023-08-28T01:18:06.602740+0000 mon.a (mon.0) 4035 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:18:07.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:06 smithi006 bash[33918]: audit 2023-08-28T01:18:06.607478+0000 mon.a (mon.0) 4036 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:18:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:06 smithi006 bash[27292]: cluster 2023-08-28T01:18:05.878842+0000 mgr.y (mgr.14901) 5985 : cluster [DBG] pgmap v5992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:06 smithi006 bash[27292]: audit 2023-08-28T01:18:06.326074+0000 mon.b (mon.1) 692 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:18:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:06 smithi006 bash[27292]: audit 2023-08-28T01:18:06.602740+0000 mon.a (mon.0) 4035 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:18:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:06 smithi006 bash[27292]: audit 2023-08-28T01:18:06.607478+0000 mon.a (mon.0) 4036 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:18:07.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:07 smithi130 bash[51302]: debug 2023-08-28T01:18:07.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:07.420699+0000) 2023-08-28T01:18:08.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:08 smithi130 bash[51302]: debug 2023-08-28T01:18:08.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:08.420978+0000) 2023-08-28T01:18:09.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:08 smithi006 bash[33918]: cluster 2023-08-28T01:18:07.879482+0000 mgr.y (mgr.14901) 5986 : cluster [DBG] pgmap v5993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:09.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:08 smithi006 bash[33918]: audit 2023-08-28T01:18:07.986672+0000 mon.c (mon.2) 2342 : audit [DBG] from='client.? 172.21.15.6:0/1609290186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:09.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:08 smithi006 bash[27292]: cluster 2023-08-28T01:18:07.879482+0000 mgr.y (mgr.14901) 5986 : cluster [DBG] pgmap v5993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:09.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:08 smithi006 bash[27292]: audit 2023-08-28T01:18:07.986672+0000 mon.c (mon.2) 2342 : audit [DBG] from='client.? 172.21.15.6:0/1609290186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:09.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:08 smithi130 bash[27579]: cluster 2023-08-28T01:18:07.879482+0000 mgr.y (mgr.14901) 5986 : cluster [DBG] pgmap v5993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:09.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:08 smithi130 bash[27579]: audit 2023-08-28T01:18:07.986672+0000 mon.c (mon.2) 2342 : audit [DBG] from='client.? 172.21.15.6:0/1609290186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:09.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:09 smithi130 bash[51302]: debug 2023-08-28T01:18:09.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:09.421264+0000) 2023-08-28T01:18:10.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:10 smithi130 bash[51302]: debug 2023-08-28T01:18:10.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:10.421547+0000) 2023-08-28T01:18:11.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:18:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:18:10] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:18:11.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:10 smithi006 bash[27292]: cluster 2023-08-28T01:18:09.880476+0000 mgr.y (mgr.14901) 5987 : cluster [DBG] pgmap v5994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:11.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:10 smithi006 bash[27292]: audit 2023-08-28T01:18:10.419126+0000 mon.a (mon.0) 4037 : audit [DBG] from='client.? 172.21.15.6:0/2554146139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:11.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:10 smithi006 bash[33918]: cluster 2023-08-28T01:18:09.880476+0000 mgr.y (mgr.14901) 5987 : cluster [DBG] pgmap v5994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:11.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:10 smithi006 bash[33918]: audit 2023-08-28T01:18:10.419126+0000 mon.a (mon.0) 4037 : audit [DBG] from='client.? 172.21.15.6:0/2554146139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:18:10.977Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:11.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:18:10.979Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:11.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:10 smithi130 bash[27579]: cluster 2023-08-28T01:18:09.880476+0000 mgr.y (mgr.14901) 5987 : cluster [DBG] pgmap v5994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:11.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:10 smithi130 bash[27579]: audit 2023-08-28T01:18:10.419126+0000 mon.a (mon.0) 4037 : audit [DBG] from='client.? 172.21.15.6:0/2554146139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:11.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:11 smithi130 bash[51302]: debug 2023-08-28T01:18:11.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:11.421797+0000) 2023-08-28T01:18:12.744 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:12 smithi130 bash[51302]: debug 2023-08-28T01:18:12.415+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:12.422010+0000) 2023-08-28T01:18:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:12 smithi006 bash[27292]: cluster 2023-08-28T01:18:11.880988+0000 mgr.y (mgr.14901) 5988 : cluster [DBG] pgmap v5995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:12 smithi006 bash[27292]: audit 2023-08-28T01:18:12.496006+0000 mon.a (mon.0) 4038 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:18:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:18:12.549Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:18:12.550Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:12 smithi006 bash[33918]: cluster 2023-08-28T01:18:11.880988+0000 mgr.y (mgr.14901) 5988 : cluster [DBG] pgmap v5995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:12.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:12 smithi006 bash[33918]: audit 2023-08-28T01:18:12.496006+0000 mon.a (mon.0) 4038 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:18:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:12 smithi130 bash[27579]: cluster 2023-08-28T01:18:11.880988+0000 mgr.y (mgr.14901) 5988 : cluster [DBG] pgmap v5995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:12 smithi130 bash[27579]: audit 2023-08-28T01:18:12.496006+0000 mon.a (mon.0) 4038 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:18:13.746 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:13 smithi130 bash[51302]: debug 2023-08-28T01:18:13.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:13.422281+0000) 2023-08-28T01:18:14.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:13 smithi130 bash[27579]: audit 2023-08-28T01:18:12.848169+0000 mon.a (mon.0) 4039 : audit [DBG] from='client.? 172.21.15.6:0/3155217984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:14.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:13 smithi130 bash[27579]: audit 2023-08-28T01:18:12.889914+0000 mon.a (mon.0) 4040 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:18:14.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:13 smithi130 bash[27579]: audit 2023-08-28T01:18:13.164316+0000 mon.b (mon.1) 693 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:18:14.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:13 smithi130 bash[27579]: audit 2023-08-28T01:18:13.166300+0000 mon.b (mon.1) 694 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:18:14.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:13 smithi130 bash[27579]: audit 2023-08-28T01:18:13.172362+0000 mon.a (mon.0) 4041 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:18:14.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:13 smithi006 bash[33918]: audit 2023-08-28T01:18:12.848169+0000 mon.a (mon.0) 4039 : audit [DBG] from='client.? 172.21.15.6:0/3155217984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:14.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:13 smithi006 bash[33918]: audit 2023-08-28T01:18:12.889914+0000 mon.a (mon.0) 4040 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:18:14.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:13 smithi006 bash[33918]: audit 2023-08-28T01:18:13.164316+0000 mon.b (mon.1) 693 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:18:14.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:13 smithi006 bash[33918]: audit 2023-08-28T01:18:13.166300+0000 mon.b (mon.1) 694 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:18:14.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:13 smithi006 bash[33918]: audit 2023-08-28T01:18:13.172362+0000 mon.a (mon.0) 4041 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:18:14.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:13 smithi006 bash[27292]: audit 2023-08-28T01:18:12.848169+0000 mon.a (mon.0) 4039 : audit [DBG] from='client.? 172.21.15.6:0/3155217984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:14.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:13 smithi006 bash[27292]: audit 2023-08-28T01:18:12.889914+0000 mon.a (mon.0) 4040 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:18:14.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:13 smithi006 bash[27292]: audit 2023-08-28T01:18:13.164316+0000 mon.b (mon.1) 693 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:18:14.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:13 smithi006 bash[27292]: audit 2023-08-28T01:18:13.166300+0000 mon.b (mon.1) 694 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:18:14.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:13 smithi006 bash[27292]: audit 2023-08-28T01:18:13.172362+0000 mon.a (mon.0) 4041 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:18:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:18:14.432Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:18:14.434Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:14.748 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:14 smithi130 bash[51302]: debug 2023-08-28T01:18:14.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:14.422644+0000) 2023-08-28T01:18:15.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:14 smithi130 bash[27579]: cluster 2023-08-28T01:18:13.881553+0000 mgr.y (mgr.14901) 5989 : cluster [DBG] pgmap v5996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:15.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:14 smithi006 bash[27292]: cluster 2023-08-28T01:18:13.881553+0000 mgr.y (mgr.14901) 5989 : cluster [DBG] pgmap v5996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:15.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:14 smithi006 bash[33918]: cluster 2023-08-28T01:18:13.881553+0000 mgr.y (mgr.14901) 5989 : cluster [DBG] pgmap v5996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:15.759 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:15 smithi130 bash[51302]: debug 2023-08-28T01:18:15.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:15.422938+0000) 2023-08-28T01:18:16.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:15 smithi130 bash[27579]: audit 2023-08-28T01:18:15.259541+0000 mon.c (mon.2) 2343 : audit [DBG] from='client.? 172.21.15.6:0/3878896054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:16.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:15 smithi006 bash[33918]: audit 2023-08-28T01:18:15.259541+0000 mon.c (mon.2) 2343 : audit [DBG] from='client.? 172.21.15.6:0/3878896054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:16.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:15 smithi006 bash[27292]: audit 2023-08-28T01:18:15.259541+0000 mon.c (mon.2) 2343 : audit [DBG] from='client.? 172.21.15.6:0/3878896054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:16.764 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:16 smithi130 bash[51302]: debug 2023-08-28T01:18:16.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:16.423213+0000) 2023-08-28T01:18:17.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:16 smithi130 bash[27579]: cluster 2023-08-28T01:18:15.882666+0000 mgr.y (mgr.14901) 5990 : cluster [DBG] pgmap v5997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:17.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:16 smithi006 bash[33918]: cluster 2023-08-28T01:18:15.882666+0000 mgr.y (mgr.14901) 5990 : cluster [DBG] pgmap v5997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:17.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:16 smithi006 bash[27292]: cluster 2023-08-28T01:18:15.882666+0000 mgr.y (mgr.14901) 5990 : cluster [DBG] pgmap v5997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:17.773 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:17 smithi130 bash[51302]: debug 2023-08-28T01:18:17.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:17.423523+0000) 2023-08-28T01:18:18.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:17 smithi130 bash[27579]: audit 2023-08-28T01:18:17.667308+0000 mon.a (mon.0) 4042 : audit [DBG] from='client.? 172.21.15.6:0/314092395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:18.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:17 smithi006 bash[33918]: audit 2023-08-28T01:18:17.667308+0000 mon.a (mon.0) 4042 : audit [DBG] from='client.? 172.21.15.6:0/314092395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:18.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:17 smithi006 bash[27292]: audit 2023-08-28T01:18:17.667308+0000 mon.a (mon.0) 4042 : audit [DBG] from='client.? 172.21.15.6:0/314092395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:18.782 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:18 smithi130 bash[51302]: debug 2023-08-28T01:18:18.423+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:18.423792+0000) 2023-08-28T01:18:19.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:18 smithi130 bash[27579]: cluster 2023-08-28T01:18:17.883309+0000 mgr.y (mgr.14901) 5991 : cluster [DBG] pgmap v5998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:19.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:18 smithi006 bash[27292]: cluster 2023-08-28T01:18:17.883309+0000 mgr.y (mgr.14901) 5991 : cluster [DBG] pgmap v5998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:19.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:18 smithi006 bash[33918]: cluster 2023-08-28T01:18:17.883309+0000 mgr.y (mgr.14901) 5991 : cluster [DBG] pgmap v5998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:19.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:19 smithi130 bash[51302]: debug 2023-08-28T01:18:19.423+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:19.424089+0000) 2023-08-28T01:18:20.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:20 smithi130 bash[51302]: debug 2023-08-28T01:18:20.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:20.424434+0000) 2023-08-28T01:18:21.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:20 smithi006 bash[27292]: cluster 2023-08-28T01:18:19.884328+0000 mgr.y (mgr.14901) 5992 : cluster [DBG] pgmap v5999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:21.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:20 smithi006 bash[27292]: audit 2023-08-28T01:18:20.089868+0000 mon.a (mon.0) 4043 : audit [DBG] from='client.? 172.21.15.6:0/2484065346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:21.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:18:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:18:20] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:18:21.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:20 smithi006 bash[33918]: cluster 2023-08-28T01:18:19.884328+0000 mgr.y (mgr.14901) 5992 : cluster [DBG] pgmap v5999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:21.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:20 smithi006 bash[33918]: audit 2023-08-28T01:18:20.089868+0000 mon.a (mon.0) 4043 : audit [DBG] from='client.? 172.21.15.6:0/2484065346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:18:20.978Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:21.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:18:20.980Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:21.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:20 smithi130 bash[27579]: cluster 2023-08-28T01:18:19.884328+0000 mgr.y (mgr.14901) 5992 : cluster [DBG] pgmap v5999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:21.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:20 smithi130 bash[27579]: audit 2023-08-28T01:18:20.089868+0000 mon.a (mon.0) 4043 : audit [DBG] from='client.? 172.21.15.6:0/2484065346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:21.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:21 smithi130 bash[51302]: debug 2023-08-28T01:18:21.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:21.424701+0000) 2023-08-28T01:18:22.745 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:22 smithi130 bash[51302]: debug 2023-08-28T01:18:22.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:22.424941+0000) 2023-08-28T01:18:22.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:22 smithi006 bash[33918]: cluster 2023-08-28T01:18:21.885053+0000 mgr.y (mgr.14901) 5993 : cluster [DBG] pgmap v6000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:22.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:22 smithi006 bash[33918]: audit 2023-08-28T01:18:22.509040+0000 mon.c (mon.2) 2344 : audit [DBG] from='client.? 172.21.15.6:0/2577846545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:22.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:18:22.550Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:22.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:18:22.552Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:22.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:22 smithi006 bash[27292]: cluster 2023-08-28T01:18:21.885053+0000 mgr.y (mgr.14901) 5993 : cluster [DBG] pgmap v6000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:22.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:22 smithi006 bash[27292]: audit 2023-08-28T01:18:22.509040+0000 mon.c (mon.2) 2344 : audit [DBG] from='client.? 172.21.15.6:0/2577846545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:23.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:22 smithi130 bash[27579]: cluster 2023-08-28T01:18:21.885053+0000 mgr.y (mgr.14901) 5993 : cluster [DBG] pgmap v6000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:23.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:22 smithi130 bash[27579]: audit 2023-08-28T01:18:22.509040+0000 mon.c (mon.2) 2344 : audit [DBG] from='client.? 172.21.15.6:0/2577846545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:23.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:23 smithi130 bash[51302]: debug 2023-08-28T01:18:23.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:23.425208+0000) 2023-08-28T01:18:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:18:24.432Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:18:24.435Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:24.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:24 smithi130 bash[51302]: debug 2023-08-28T01:18:24.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:24.425513+0000) 2023-08-28T01:18:25.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:24 smithi006 bash[27292]: cluster 2023-08-28T01:18:23.885689+0000 mgr.y (mgr.14901) 5994 : cluster [DBG] pgmap v6001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:25.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:24 smithi006 bash[27292]: audit 2023-08-28T01:18:24.929356+0000 mon.a (mon.0) 4044 : audit [DBG] from='client.? 172.21.15.6:0/2380281231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:25.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:24 smithi006 bash[33918]: cluster 2023-08-28T01:18:23.885689+0000 mgr.y (mgr.14901) 5994 : cluster [DBG] pgmap v6001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:25.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:24 smithi006 bash[33918]: audit 2023-08-28T01:18:24.929356+0000 mon.a (mon.0) 4044 : audit [DBG] from='client.? 172.21.15.6:0/2380281231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:25.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:24 smithi130 bash[27579]: cluster 2023-08-28T01:18:23.885689+0000 mgr.y (mgr.14901) 5994 : cluster [DBG] pgmap v6001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:25.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:24 smithi130 bash[27579]: audit 2023-08-28T01:18:24.929356+0000 mon.a (mon.0) 4044 : audit [DBG] from='client.? 172.21.15.6:0/2380281231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:25.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:25 smithi130 bash[51302]: debug 2023-08-28T01:18:25.423+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:25.425818+0000) 2023-08-28T01:18:26.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:26 smithi130 bash[51302]: debug 2023-08-28T01:18:26.423+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:26.426101+0000) 2023-08-28T01:18:27.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:26 smithi006 bash[33918]: cluster 2023-08-28T01:18:25.886877+0000 mgr.y (mgr.14901) 5995 : cluster [DBG] pgmap v6002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:27.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:26 smithi006 bash[27292]: cluster 2023-08-28T01:18:25.886877+0000 mgr.y (mgr.14901) 5995 : cluster [DBG] pgmap v6002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:27.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:26 smithi130 bash[27579]: cluster 2023-08-28T01:18:25.886877+0000 mgr.y (mgr.14901) 5995 : cluster [DBG] pgmap v6002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:27.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:27 smithi130 bash[51302]: debug 2023-08-28T01:18:27.419+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:27.426388+0000) 2023-08-28T01:18:28.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:27 smithi006 bash[27292]: audit 2023-08-28T01:18:27.357582+0000 mon.c (mon.2) 2345 : audit [DBG] from='client.? 172.21.15.6:0/3105415160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:28.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:27 smithi006 bash[33918]: audit 2023-08-28T01:18:27.357582+0000 mon.c (mon.2) 2345 : audit [DBG] from='client.? 172.21.15.6:0/3105415160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:28.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:27 smithi130 bash[27579]: audit 2023-08-28T01:18:27.357582+0000 mon.c (mon.2) 2345 : audit [DBG] from='client.? 172.21.15.6:0/3105415160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:28.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:28 smithi130 bash[51302]: debug 2023-08-28T01:18:28.422+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:28.426658+0000) 2023-08-28T01:18:29.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:28 smithi006 bash[33918]: cluster 2023-08-28T01:18:27.887517+0000 mgr.y (mgr.14901) 5996 : cluster [DBG] pgmap v6003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:29.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:28 smithi006 bash[27292]: cluster 2023-08-28T01:18:27.887517+0000 mgr.y (mgr.14901) 5996 : cluster [DBG] pgmap v6003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:29.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:28 smithi130 bash[27579]: cluster 2023-08-28T01:18:27.887517+0000 mgr.y (mgr.14901) 5996 : cluster [DBG] pgmap v6003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:29.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:29 smithi130 bash[51302]: debug 2023-08-28T01:18:29.422+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:29.426972+0000) 2023-08-28T01:18:30.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:29 smithi130 bash[27579]: audit 2023-08-28T01:18:29.772561+0000 mon.c (mon.2) 2346 : audit [DBG] from='client.? 172.21.15.6:0/3678036038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:30.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:29 smithi006 bash[33918]: audit 2023-08-28T01:18:29.772561+0000 mon.c (mon.2) 2346 : audit [DBG] from='client.? 172.21.15.6:0/3678036038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:30.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:29 smithi006 bash[27292]: audit 2023-08-28T01:18:29.772561+0000 mon.c (mon.2) 2346 : audit [DBG] from='client.? 172.21.15.6:0/3678036038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:30.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:30 smithi130 bash[51302]: debug 2023-08-28T01:18:30.422+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:30.427262+0000) 2023-08-28T01:18:31.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:18:30.979Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:31.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:30 smithi006 bash[56298]: level=warn ts=2023-08-28T01:18:30.981Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:18:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:18:30] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:18:31.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:30 smithi006 bash[33918]: cluster 2023-08-28T01:18:29.888903+0000 mgr.y (mgr.14901) 5997 : cluster [DBG] pgmap v6004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:31.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:30 smithi006 bash[27292]: cluster 2023-08-28T01:18:29.888903+0000 mgr.y (mgr.14901) 5997 : cluster [DBG] pgmap v6004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:31.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:30 smithi130 bash[27579]: cluster 2023-08-28T01:18:29.888903+0000 mgr.y (mgr.14901) 5997 : cluster [DBG] pgmap v6004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:31.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:31 smithi130 bash[51302]: debug 2023-08-28T01:18:31.422+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:31.427495+0000) 2023-08-28T01:18:32.749 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:32 smithi130 bash[51302]: debug 2023-08-28T01:18:32.422+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:32.427703+0000) 2023-08-28T01:18:32.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:32 smithi006 bash[33918]: cluster 2023-08-28T01:18:31.889620+0000 mgr.y (mgr.14901) 5998 : cluster [DBG] pgmap v6005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:32 smithi006 bash[33918]: audit 2023-08-28T01:18:32.191439+0000 mon.c (mon.2) 2347 : audit [DBG] from='client.? 172.21.15.6:0/78159437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:32 smithi006 bash[27292]: cluster 2023-08-28T01:18:31.889620+0000 mgr.y (mgr.14901) 5998 : cluster [DBG] pgmap v6005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:32 smithi006 bash[27292]: audit 2023-08-28T01:18:32.191439+0000 mon.c (mon.2) 2347 : audit [DBG] from='client.? 172.21.15.6:0/78159437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:18:32.550Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:18:32.552Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:32 smithi130 bash[27579]: cluster 2023-08-28T01:18:31.889620+0000 mgr.y (mgr.14901) 5998 : cluster [DBG] pgmap v6005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:32 smithi130 bash[27579]: audit 2023-08-28T01:18:32.191439+0000 mon.c (mon.2) 2347 : audit [DBG] from='client.? 172.21.15.6:0/78159437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:33.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:33 smithi130 bash[51302]: debug 2023-08-28T01:18:33.426+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:33.427947+0000) 2023-08-28T01:18:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:18:34.433Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:34.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:18:34.434Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:34.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:34 smithi130 bash[51302]: debug 2023-08-28T01:18:34.426+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:34.428276+0000) 2023-08-28T01:18:35.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:34 smithi006 bash[33918]: cluster 2023-08-28T01:18:33.890257+0000 mgr.y (mgr.14901) 5999 : cluster [DBG] pgmap v6006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:35.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:34 smithi006 bash[33918]: audit 2023-08-28T01:18:34.608645+0000 mon.a (mon.0) 4045 : audit [DBG] from='client.? 172.21.15.6:0/1912120478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:35.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:34 smithi006 bash[27292]: cluster 2023-08-28T01:18:33.890257+0000 mgr.y (mgr.14901) 5999 : cluster [DBG] pgmap v6006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:35.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:34 smithi006 bash[27292]: audit 2023-08-28T01:18:34.608645+0000 mon.a (mon.0) 4045 : audit [DBG] from='client.? 172.21.15.6:0/1912120478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:35.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:34 smithi130 bash[27579]: cluster 2023-08-28T01:18:33.890257+0000 mgr.y (mgr.14901) 5999 : cluster [DBG] pgmap v6006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:35.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:34 smithi130 bash[27579]: audit 2023-08-28T01:18:34.608645+0000 mon.a (mon.0) 4045 : audit [DBG] from='client.? 172.21.15.6:0/1912120478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:35.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:35 smithi130 bash[51302]: debug 2023-08-28T01:18:35.426+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:35.428556+0000) 2023-08-28T01:18:36.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:36 smithi130 bash[51302]: debug 2023-08-28T01:18:36.426+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:36.428838+0000) 2023-08-28T01:18:37.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:36 smithi006 bash[27292]: cluster 2023-08-28T01:18:35.891438+0000 mgr.y (mgr.14901) 6000 : cluster [DBG] pgmap v6007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:37.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:36 smithi006 bash[33918]: cluster 2023-08-28T01:18:35.891438+0000 mgr.y (mgr.14901) 6000 : cluster [DBG] pgmap v6007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:37.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:36 smithi130 bash[27579]: cluster 2023-08-28T01:18:35.891438+0000 mgr.y (mgr.14901) 6000 : cluster [DBG] pgmap v6007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:37.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:37 smithi130 bash[51302]: debug 2023-08-28T01:18:37.426+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:37.429104+0000) 2023-08-28T01:18:38.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:37 smithi006 bash[27292]: audit 2023-08-28T01:18:37.016687+0000 mon.c (mon.2) 2348 : audit [DBG] from='client.? 172.21.15.6:0/2970483944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:38.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:37 smithi006 bash[33918]: audit 2023-08-28T01:18:37.016687+0000 mon.c (mon.2) 2348 : audit [DBG] from='client.? 172.21.15.6:0/2970483944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:38.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:37 smithi130 bash[27579]: audit 2023-08-28T01:18:37.016687+0000 mon.c (mon.2) 2348 : audit [DBG] from='client.? 172.21.15.6:0/2970483944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:38.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:38 smithi130 bash[51302]: debug 2023-08-28T01:18:38.426+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:38.429319+0000) 2023-08-28T01:18:39.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:38 smithi130 bash[27579]: cluster 2023-08-28T01:18:37.892064+0000 mgr.y (mgr.14901) 6001 : cluster [DBG] pgmap v6008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:39.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:38 smithi006 bash[27292]: cluster 2023-08-28T01:18:37.892064+0000 mgr.y (mgr.14901) 6001 : cluster [DBG] pgmap v6008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:39.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:38 smithi006 bash[33918]: cluster 2023-08-28T01:18:37.892064+0000 mgr.y (mgr.14901) 6001 : cluster [DBG] pgmap v6008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:39.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:39 smithi130 bash[51302]: debug 2023-08-28T01:18:39.426+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:39.429555+0000) 2023-08-28T01:18:40.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:39 smithi130 bash[27579]: audit 2023-08-28T01:18:39.431174+0000 mon.c (mon.2) 2349 : audit [DBG] from='client.? 172.21.15.6:0/246893252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:39 smithi006 bash[33918]: audit 2023-08-28T01:18:39.431174+0000 mon.c (mon.2) 2349 : audit [DBG] from='client.? 172.21.15.6:0/246893252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:40.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:39 smithi006 bash[27292]: audit 2023-08-28T01:18:39.431174+0000 mon.c (mon.2) 2349 : audit [DBG] from='client.? 172.21.15.6:0/246893252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:40.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:40 smithi130 bash[51302]: debug 2023-08-28T01:18:40.426+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:40.429852+0000) 2023-08-28T01:18:41.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:40 smithi006 bash[27292]: cluster 2023-08-28T01:18:39.893228+0000 mgr.y (mgr.14901) 6002 : cluster [DBG] pgmap v6009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:41.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:18:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:18:40] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:18:41.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:40 smithi006 bash[33918]: cluster 2023-08-28T01:18:39.893228+0000 mgr.y (mgr.14901) 6002 : cluster [DBG] pgmap v6009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:41.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:18:40.979Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:18:40.981Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:41.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:40 smithi130 bash[27579]: cluster 2023-08-28T01:18:39.893228+0000 mgr.y (mgr.14901) 6002 : cluster [DBG] pgmap v6009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:41.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:41 smithi130 bash[51302]: debug 2023-08-28T01:18:41.426+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:41.430114+0000) 2023-08-28T01:18:42.751 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:42 smithi130 bash[51302]: debug 2023-08-28T01:18:42.426+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:42.430413+0000) 2023-08-28T01:18:42.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:42 smithi006 bash[33918]: audit 2023-08-28T01:18:41.848421+0000 mon.c (mon.2) 2350 : audit [DBG] from='client.? 172.21.15.6:0/779881264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:42.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:42 smithi006 bash[33918]: cluster 2023-08-28T01:18:41.893852+0000 mgr.y (mgr.14901) 6003 : cluster [DBG] pgmap v6010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:18:42.551Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:18:42.553Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:42 smithi006 bash[27292]: audit 2023-08-28T01:18:41.848421+0000 mon.c (mon.2) 2350 : audit [DBG] from='client.? 172.21.15.6:0/779881264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:42 smithi006 bash[27292]: cluster 2023-08-28T01:18:41.893852+0000 mgr.y (mgr.14901) 6003 : cluster [DBG] pgmap v6010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:42 smithi130 bash[27579]: audit 2023-08-28T01:18:41.848421+0000 mon.c (mon.2) 2350 : audit [DBG] from='client.? 172.21.15.6:0/779881264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:42 smithi130 bash[27579]: cluster 2023-08-28T01:18:41.893852+0000 mgr.y (mgr.14901) 6003 : cluster [DBG] pgmap v6010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:43.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:43 smithi130 bash[51302]: debug 2023-08-28T01:18:43.426+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:43.430685+0000) 2023-08-28T01:18:44.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:18:44.433Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:44.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:18:44.435Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:44.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:44 smithi130 bash[51302]: debug 2023-08-28T01:18:44.430+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:44.430995+0000) 2023-08-28T01:18:45.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:44 smithi006 bash[33918]: cluster 2023-08-28T01:18:43.894551+0000 mgr.y (mgr.14901) 6004 : cluster [DBG] pgmap v6011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:45.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:44 smithi006 bash[33918]: audit 2023-08-28T01:18:44.259641+0000 mon.a (mon.0) 4046 : audit [DBG] from='client.? 172.21.15.6:0/2717589947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:45.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:44 smithi006 bash[27292]: cluster 2023-08-28T01:18:43.894551+0000 mgr.y (mgr.14901) 6004 : cluster [DBG] pgmap v6011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:45.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:44 smithi006 bash[27292]: audit 2023-08-28T01:18:44.259641+0000 mon.a (mon.0) 4046 : audit [DBG] from='client.? 172.21.15.6:0/2717589947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:45.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:44 smithi130 bash[27579]: cluster 2023-08-28T01:18:43.894551+0000 mgr.y (mgr.14901) 6004 : cluster [DBG] pgmap v6011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:45.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:44 smithi130 bash[27579]: audit 2023-08-28T01:18:44.259641+0000 mon.a (mon.0) 4046 : audit [DBG] from='client.? 172.21.15.6:0/2717589947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:45.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:45 smithi130 bash[51302]: debug 2023-08-28T01:18:45.430+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:45.431212+0000) 2023-08-28T01:18:46.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:46 smithi130 bash[51302]: debug 2023-08-28T01:18:46.430+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:46.431468+0000) 2023-08-28T01:18:47.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:46 smithi006 bash[33918]: cluster 2023-08-28T01:18:45.895707+0000 mgr.y (mgr.14901) 6005 : cluster [DBG] pgmap v6012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:47.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:46 smithi006 bash[33918]: audit 2023-08-28T01:18:46.679048+0000 mon.a (mon.0) 4047 : audit [DBG] from='client.? 172.21.15.6:0/3645856361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:47.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:46 smithi006 bash[27292]: cluster 2023-08-28T01:18:45.895707+0000 mgr.y (mgr.14901) 6005 : cluster [DBG] pgmap v6012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:47.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:46 smithi006 bash[27292]: audit 2023-08-28T01:18:46.679048+0000 mon.a (mon.0) 4047 : audit [DBG] from='client.? 172.21.15.6:0/3645856361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:47.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:46 smithi130 bash[27579]: cluster 2023-08-28T01:18:45.895707+0000 mgr.y (mgr.14901) 6005 : cluster [DBG] pgmap v6012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:47.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:46 smithi130 bash[27579]: audit 2023-08-28T01:18:46.679048+0000 mon.a (mon.0) 4047 : audit [DBG] from='client.? 172.21.15.6:0/3645856361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:47.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:47 smithi130 bash[51302]: debug 2023-08-28T01:18:47.430+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:47.431834+0000) 2023-08-28T01:18:48.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:48 smithi130 bash[51302]: debug 2023-08-28T01:18:48.430+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:48.432163+0000) 2023-08-28T01:18:49.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:48 smithi006 bash[27292]: cluster 2023-08-28T01:18:47.896333+0000 mgr.y (mgr.14901) 6006 : cluster [DBG] pgmap v6013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:49.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:48 smithi006 bash[33918]: cluster 2023-08-28T01:18:47.896333+0000 mgr.y (mgr.14901) 6006 : cluster [DBG] pgmap v6013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:49.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:48 smithi130 bash[27579]: cluster 2023-08-28T01:18:47.896333+0000 mgr.y (mgr.14901) 6006 : cluster [DBG] pgmap v6013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:49.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:49 smithi130 bash[51302]: debug 2023-08-28T01:18:49.430+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:49.432500+0000) 2023-08-28T01:18:50.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:49 smithi130 bash[27579]: audit 2023-08-28T01:18:49.091191+0000 mon.a (mon.0) 4048 : audit [DBG] from='client.? 172.21.15.6:0/883663242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:50.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:49 smithi006 bash[33918]: audit 2023-08-28T01:18:49.091191+0000 mon.a (mon.0) 4048 : audit [DBG] from='client.? 172.21.15.6:0/883663242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:50.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:49 smithi006 bash[27292]: audit 2023-08-28T01:18:49.091191+0000 mon.a (mon.0) 4048 : audit [DBG] from='client.? 172.21.15.6:0/883663242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:50.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:50 smithi130 bash[51302]: debug 2023-08-28T01:18:50.430+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:50.432842+0000) 2023-08-28T01:18:51.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:18:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:18:50] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:18:51.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:50 smithi006 bash[33918]: cluster 2023-08-28T01:18:49.897482+0000 mgr.y (mgr.14901) 6007 : cluster [DBG] pgmap v6014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:51.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:50 smithi006 bash[27292]: cluster 2023-08-28T01:18:49.897482+0000 mgr.y (mgr.14901) 6007 : cluster [DBG] pgmap v6014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:18:50.980Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:51.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:18:50.982Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:51.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:50 smithi130 bash[27579]: cluster 2023-08-28T01:18:49.897482+0000 mgr.y (mgr.14901) 6007 : cluster [DBG] pgmap v6014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:51.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:51 smithi130 bash[51302]: debug 2023-08-28T01:18:51.430+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:51.433152+0000) 2023-08-28T01:18:52.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:51 smithi130 bash[27579]: audit 2023-08-28T01:18:51.504514+0000 mon.a (mon.0) 4049 : audit [DBG] from='client.? 172.21.15.6:0/2986668580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:52.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:51 smithi006 bash[27292]: audit 2023-08-28T01:18:51.504514+0000 mon.a (mon.0) 4049 : audit [DBG] from='client.? 172.21.15.6:0/2986668580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:52.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:51 smithi006 bash[33918]: audit 2023-08-28T01:18:51.504514+0000 mon.a (mon.0) 4049 : audit [DBG] from='client.? 172.21.15.6:0/2986668580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:52.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:52 smithi130 bash[51302]: debug 2023-08-28T01:18:52.430+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:52.433461+0000) 2023-08-28T01:18:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:18:52.551Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:18:52.553Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:53.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:52 smithi130 bash[27579]: cluster 2023-08-28T01:18:51.898154+0000 mgr.y (mgr.14901) 6008 : cluster [DBG] pgmap v6015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:53.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:52 smithi006 bash[27292]: cluster 2023-08-28T01:18:51.898154+0000 mgr.y (mgr.14901) 6008 : cluster [DBG] pgmap v6015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:53.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:52 smithi006 bash[33918]: cluster 2023-08-28T01:18:51.898154+0000 mgr.y (mgr.14901) 6008 : cluster [DBG] pgmap v6015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:53.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:53 smithi130 bash[51302]: debug 2023-08-28T01:18:53.430+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:53.433822+0000) 2023-08-28T01:18:54.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:53 smithi130 bash[27579]: audit 2023-08-28T01:18:53.919076+0000 mon.c (mon.2) 2351 : audit [DBG] from='client.? 172.21.15.6:0/3808333330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:54.435 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:53 smithi006 bash[33918]: audit 2023-08-28T01:18:53.919076+0000 mon.c (mon.2) 2351 : audit [DBG] from='client.? 172.21.15.6:0/3808333330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:54.435 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:53 smithi006 bash[27292]: audit 2023-08-28T01:18:53.919076+0000 mon.c (mon.2) 2351 : audit [DBG] from='client.? 172.21.15.6:0/3808333330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:18:54.433Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:54.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:18:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:18:54.435Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:18:54.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:54 smithi130 bash[51302]: debug 2023-08-28T01:18:54.430+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:54.434037+0000) 2023-08-28T01:18:55.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:54 smithi130 bash[27579]: cluster 2023-08-28T01:18:53.898620+0000 mgr.y (mgr.14901) 6009 : cluster [DBG] pgmap v6016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:55.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:54 smithi006 bash[33918]: cluster 2023-08-28T01:18:53.898620+0000 mgr.y (mgr.14901) 6009 : cluster [DBG] pgmap v6016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:55.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:54 smithi006 bash[27292]: cluster 2023-08-28T01:18:53.898620+0000 mgr.y (mgr.14901) 6009 : cluster [DBG] pgmap v6016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:55.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:55 smithi130 bash[51302]: debug 2023-08-28T01:18:55.430+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:55.434324+0000) 2023-08-28T01:18:56.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:56 smithi130 bash[51302]: debug 2023-08-28T01:18:56.430+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:56.434636+0000) 2023-08-28T01:18:57.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:56 smithi130 bash[27579]: cluster 2023-08-28T01:18:55.899727+0000 mgr.y (mgr.14901) 6010 : cluster [DBG] pgmap v6017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:57.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:56 smithi130 bash[27579]: audit 2023-08-28T01:18:56.333566+0000 mon.a (mon.0) 4050 : audit [DBG] from='client.? 172.21.15.6:0/2084190718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:57.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:56 smithi006 bash[33918]: cluster 2023-08-28T01:18:55.899727+0000 mgr.y (mgr.14901) 6010 : cluster [DBG] pgmap v6017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:57.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:56 smithi006 bash[33918]: audit 2023-08-28T01:18:56.333566+0000 mon.a (mon.0) 4050 : audit [DBG] from='client.? 172.21.15.6:0/2084190718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:57.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:56 smithi006 bash[27292]: cluster 2023-08-28T01:18:55.899727+0000 mgr.y (mgr.14901) 6010 : cluster [DBG] pgmap v6017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:57.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:56 smithi006 bash[27292]: audit 2023-08-28T01:18:56.333566+0000 mon.a (mon.0) 4050 : audit [DBG] from='client.? 172.21.15.6:0/2084190718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:57.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:57 smithi130 bash[51302]: debug 2023-08-28T01:18:57.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:57.434959+0000) 2023-08-28T01:18:58.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:58 smithi130 bash[51302]: debug 2023-08-28T01:18:58.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:58.435267+0000) 2023-08-28T01:18:59.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:59 smithi130 bash[27579]: cluster 2023-08-28T01:18:57.900357+0000 mgr.y (mgr.14901) 6011 : cluster [DBG] pgmap v6018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:59.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:18:59 smithi130 bash[27579]: audit 2023-08-28T01:18:58.749765+0000 mon.a (mon.0) 4051 : audit [DBG] from='client.? 172.21.15.6:0/3008536579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:59.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:59 smithi006 bash[33918]: cluster 2023-08-28T01:18:57.900357+0000 mgr.y (mgr.14901) 6011 : cluster [DBG] pgmap v6018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:59.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:18:59 smithi006 bash[33918]: audit 2023-08-28T01:18:58.749765+0000 mon.a (mon.0) 4051 : audit [DBG] from='client.? 172.21.15.6:0/3008536579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:59.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:58 smithi006 bash[27292]: cluster 2023-08-28T01:18:57.900357+0000 mgr.y (mgr.14901) 6011 : cluster [DBG] pgmap v6018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:18:59.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:18:58 smithi006 bash[27292]: audit 2023-08-28T01:18:58.749765+0000 mon.a (mon.0) 4051 : audit [DBG] from='client.? 172.21.15.6:0/3008536579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:59.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:18:59 smithi130 bash[51302]: debug 2023-08-28T01:18:59.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:59.435600+0000) 2023-08-28T01:19:00.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:00 smithi130 bash[51302]: debug 2023-08-28T01:19:00.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:00.435938+0000) 2023-08-28T01:19:01.004 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:19:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:19:00] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:19:01.004 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:19:00.980Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:01.004 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:19:00.983Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:01.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:01 smithi130 bash[27579]: cluster 2023-08-28T01:18:59.901583+0000 mgr.y (mgr.14901) 6012 : cluster [DBG] pgmap v6019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:01.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:01 smithi006 bash[33918]: cluster 2023-08-28T01:18:59.901583+0000 mgr.y (mgr.14901) 6012 : cluster [DBG] pgmap v6019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:01.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:01 smithi006 bash[27292]: cluster 2023-08-28T01:18:59.901583+0000 mgr.y (mgr.14901) 6012 : cluster [DBG] pgmap v6019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:01.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:01 smithi130 bash[51302]: debug 2023-08-28T01:19:01.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:01.436354+0000) 2023-08-28T01:19:02.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:02 smithi130 bash[27579]: audit 2023-08-28T01:19:01.178202+0000 mon.c (mon.2) 2352 : audit [DBG] from='client.? 172.21.15.6:0/126026270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:02.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:02 smithi006 bash[27292]: audit 2023-08-28T01:19:01.178202+0000 mon.c (mon.2) 2352 : audit [DBG] from='client.? 172.21.15.6:0/126026270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:02.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:02 smithi006 bash[33918]: audit 2023-08-28T01:19:01.178202+0000 mon.c (mon.2) 2352 : audit [DBG] from='client.? 172.21.15.6:0/126026270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:02.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:02 smithi130 bash[51302]: debug 2023-08-28T01:19:02.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:02.436595+0000) 2023-08-28T01:19:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:19:02.552Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:02.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:19:02.555Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:03.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:03 smithi130 bash[27579]: cluster 2023-08-28T01:19:01.902308+0000 mgr.y (mgr.14901) 6013 : cluster [DBG] pgmap v6020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:03.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:03 smithi006 bash[27292]: cluster 2023-08-28T01:19:01.902308+0000 mgr.y (mgr.14901) 6013 : cluster [DBG] pgmap v6020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:03.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:03 smithi006 bash[33918]: cluster 2023-08-28T01:19:01.902308+0000 mgr.y (mgr.14901) 6013 : cluster [DBG] pgmap v6020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:03.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:03 smithi130 bash[51302]: debug 2023-08-28T01:19:03.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:03.436887+0000) 2023-08-28T01:19:04.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:04 smithi130 bash[27579]: audit 2023-08-28T01:19:03.593748+0000 mon.c (mon.2) 2353 : audit [DBG] from='client.? 172.21.15.6:0/2592899920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:04.436 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:04 smithi006 bash[27292]: audit 2023-08-28T01:19:03.593748+0000 mon.c (mon.2) 2353 : audit [DBG] from='client.? 172.21.15.6:0/2592899920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:04.436 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:04 smithi006 bash[33918]: audit 2023-08-28T01:19:03.593748+0000 mon.c (mon.2) 2353 : audit [DBG] from='client.? 172.21.15.6:0/2592899920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:19:04.434Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:19:04.437Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:04.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:04 smithi130 bash[51302]: debug 2023-08-28T01:19:04.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:04.437217+0000) 2023-08-28T01:19:05.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:05 smithi130 bash[27579]: cluster 2023-08-28T01:19:03.902991+0000 mgr.y (mgr.14901) 6014 : cluster [DBG] pgmap v6021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:05.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:05 smithi006 bash[27292]: cluster 2023-08-28T01:19:03.902991+0000 mgr.y (mgr.14901) 6014 : cluster [DBG] pgmap v6021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:05.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:05 smithi006 bash[33918]: cluster 2023-08-28T01:19:03.902991+0000 mgr.y (mgr.14901) 6014 : cluster [DBG] pgmap v6021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:05.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:05 smithi130 bash[51302]: debug 2023-08-28T01:19:05.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:05.437501+0000) 2023-08-28T01:19:06.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:06 smithi130 bash[27579]: audit 2023-08-28T01:19:06.011781+0000 mon.c (mon.2) 2354 : audit [DBG] from='client.? 172.21.15.6:0/2604832198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:06.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:06 smithi006 bash[27292]: audit 2023-08-28T01:19:06.011781+0000 mon.c (mon.2) 2354 : audit [DBG] from='client.? 172.21.15.6:0/2604832198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:06.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:06 smithi006 bash[33918]: audit 2023-08-28T01:19:06.011781+0000 mon.c (mon.2) 2354 : audit [DBG] from='client.? 172.21.15.6:0/2604832198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:06.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:06 smithi130 bash[51302]: debug 2023-08-28T01:19:06.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:06.437774+0000) 2023-08-28T01:19:07.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:07 smithi130 bash[27579]: cluster 2023-08-28T01:19:05.904106+0000 mgr.y (mgr.14901) 6015 : cluster [DBG] pgmap v6022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:07 smithi006 bash[27292]: cluster 2023-08-28T01:19:05.904106+0000 mgr.y (mgr.14901) 6015 : cluster [DBG] pgmap v6022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:07.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:07 smithi006 bash[33918]: cluster 2023-08-28T01:19:05.904106+0000 mgr.y (mgr.14901) 6015 : cluster [DBG] pgmap v6022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:07.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:07 smithi130 bash[51302]: debug 2023-08-28T01:19:07.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:07.438068+0000) 2023-08-28T01:19:08.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:08 smithi130 bash[51302]: debug 2023-08-28T01:19:08.434+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:08.438347+0000) 2023-08-28T01:19:09.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:09 smithi130 bash[27579]: cluster 2023-08-28T01:19:07.904795+0000 mgr.y (mgr.14901) 6016 : cluster [DBG] pgmap v6023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:09.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:09 smithi130 bash[27579]: audit 2023-08-28T01:19:08.431227+0000 mon.c (mon.2) 2355 : audit [DBG] from='client.? 172.21.15.6:0/4030496628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:09.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:09 smithi006 bash[27292]: cluster 2023-08-28T01:19:07.904795+0000 mgr.y (mgr.14901) 6016 : cluster [DBG] pgmap v6023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:09.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:09 smithi006 bash[27292]: audit 2023-08-28T01:19:08.431227+0000 mon.c (mon.2) 2355 : audit [DBG] from='client.? 172.21.15.6:0/4030496628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:09.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:09 smithi006 bash[33918]: cluster 2023-08-28T01:19:07.904795+0000 mgr.y (mgr.14901) 6016 : cluster [DBG] pgmap v6023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:09.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:09 smithi006 bash[33918]: audit 2023-08-28T01:19:08.431227+0000 mon.c (mon.2) 2355 : audit [DBG] from='client.? 172.21.15.6:0/4030496628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:09.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:09 smithi130 bash[51302]: debug 2023-08-28T01:19:09.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:09.438623+0000) 2023-08-28T01:19:10.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:10 smithi130 bash[51302]: debug 2023-08-28T01:19:10.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:10.438867+0000) 2023-08-28T01:19:11.045 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:19:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:19:10] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:19:11.046 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:19:10.981Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:11.046 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:19:10.983Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:11.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:11 smithi130 bash[27579]: cluster 2023-08-28T01:19:09.906162+0000 mgr.y (mgr.14901) 6017 : cluster [DBG] pgmap v6024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:11.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:11 smithi130 bash[27579]: audit 2023-08-28T01:19:10.854176+0000 mon.c (mon.2) 2356 : audit [DBG] from='client.? 172.21.15.6:0/1395628685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:11.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:11 smithi006 bash[33918]: cluster 2023-08-28T01:19:09.906162+0000 mgr.y (mgr.14901) 6017 : cluster [DBG] pgmap v6024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:11.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:11 smithi006 bash[33918]: audit 2023-08-28T01:19:10.854176+0000 mon.c (mon.2) 2356 : audit [DBG] from='client.? 172.21.15.6:0/1395628685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:11.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:11 smithi006 bash[27292]: cluster 2023-08-28T01:19:09.906162+0000 mgr.y (mgr.14901) 6017 : cluster [DBG] pgmap v6024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:11.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:11 smithi006 bash[27292]: audit 2023-08-28T01:19:10.854176+0000 mon.c (mon.2) 2356 : audit [DBG] from='client.? 172.21.15.6:0/1395628685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:11.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:11 smithi130 bash[51302]: debug 2023-08-28T01:19:11.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:11.439139+0000) 2023-08-28T01:19:12.759 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:12 smithi130 bash[51302]: debug 2023-08-28T01:19:12.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:12.439355+0000) 2023-08-28T01:19:12.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:12 smithi006 bash[27292]: cluster 2023-08-28T01:19:11.906890+0000 mgr.y (mgr.14901) 6018 : cluster [DBG] pgmap v6025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:12 smithi006 bash[33918]: cluster 2023-08-28T01:19:11.906890+0000 mgr.y (mgr.14901) 6018 : cluster [DBG] pgmap v6025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:19:12.553Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:19:12.555Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:12 smithi130 bash[27579]: cluster 2023-08-28T01:19:11.906890+0000 mgr.y (mgr.14901) 6018 : cluster [DBG] pgmap v6025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:13.762 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:13 smithi130 bash[51302]: debug 2023-08-28T01:19:13.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:13.439594+0000) 2023-08-28T01:19:14.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:13 smithi130 bash[27579]: audit 2023-08-28T01:19:13.184567+0000 mon.b (mon.1) 695 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:19:14.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:13 smithi130 bash[27579]: audit 2023-08-28T01:19:13.280504+0000 mon.c (mon.2) 2357 : audit [DBG] from='client.? 172.21.15.6:0/895374400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:13 smithi006 bash[33918]: audit 2023-08-28T01:19:13.184567+0000 mon.b (mon.1) 695 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:19:14.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:13 smithi006 bash[33918]: audit 2023-08-28T01:19:13.280504+0000 mon.c (mon.2) 2357 : audit [DBG] from='client.? 172.21.15.6:0/895374400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:14.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:13 smithi006 bash[27292]: audit 2023-08-28T01:19:13.184567+0000 mon.b (mon.1) 695 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:19:14.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:13 smithi006 bash[27292]: audit 2023-08-28T01:19:13.280504+0000 mon.c (mon.2) 2357 : audit [DBG] from='client.? 172.21.15.6:0/895374400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:19:14.435Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:19:14.437Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:14.770 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:14 smithi130 bash[51302]: debug 2023-08-28T01:19:14.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:14.439815+0000) 2023-08-28T01:19:15.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:14 smithi130 bash[27579]: cluster 2023-08-28T01:19:13.907538+0000 mgr.y (mgr.14901) 6019 : cluster [DBG] pgmap v6026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:15.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:14 smithi006 bash[27292]: cluster 2023-08-28T01:19:13.907538+0000 mgr.y (mgr.14901) 6019 : cluster [DBG] pgmap v6026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:15.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:14 smithi006 bash[33918]: cluster 2023-08-28T01:19:13.907538+0000 mgr.y (mgr.14901) 6019 : cluster [DBG] pgmap v6026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:15.774 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:15 smithi130 bash[51302]: debug 2023-08-28T01:19:15.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:15.440026+0000) 2023-08-28T01:19:16.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:15 smithi130 bash[27579]: audit 2023-08-28T01:19:15.699321+0000 mon.a (mon.0) 4052 : audit [DBG] from='client.? 172.21.15.6:0/815792153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:16.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:15 smithi006 bash[27292]: audit 2023-08-28T01:19:15.699321+0000 mon.a (mon.0) 4052 : audit [DBG] from='client.? 172.21.15.6:0/815792153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:16.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:15 smithi006 bash[33918]: audit 2023-08-28T01:19:15.699321+0000 mon.a (mon.0) 4052 : audit [DBG] from='client.? 172.21.15.6:0/815792153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:16.781 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:16 smithi130 bash[51302]: debug 2023-08-28T01:19:16.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:16.440368+0000) 2023-08-28T01:19:17.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:16 smithi130 bash[27579]: cluster 2023-08-28T01:19:15.908674+0000 mgr.y (mgr.14901) 6020 : cluster [DBG] pgmap v6027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:17.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:16 smithi006 bash[27292]: cluster 2023-08-28T01:19:15.908674+0000 mgr.y (mgr.14901) 6020 : cluster [DBG] pgmap v6027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:17.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:16 smithi006 bash[33918]: cluster 2023-08-28T01:19:15.908674+0000 mgr.y (mgr.14901) 6020 : cluster [DBG] pgmap v6027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:17.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:17 smithi130 bash[51302]: debug 2023-08-28T01:19:17.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:17.440618+0000) 2023-08-28T01:19:18.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:18 smithi130 bash[51302]: debug 2023-08-28T01:19:18.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:18.440991+0000) 2023-08-28T01:19:19.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:18 smithi130 bash[27579]: cluster 2023-08-28T01:19:17.909328+0000 mgr.y (mgr.14901) 6021 : cluster [DBG] pgmap v6028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:19.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:18 smithi130 bash[27579]: audit 2023-08-28T01:19:18.113537+0000 mon.c (mon.2) 2358 : audit [DBG] from='client.? 172.21.15.6:0/3989001270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:19.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:18 smithi006 bash[27292]: cluster 2023-08-28T01:19:17.909328+0000 mgr.y (mgr.14901) 6021 : cluster [DBG] pgmap v6028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:19.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:18 smithi006 bash[27292]: audit 2023-08-28T01:19:18.113537+0000 mon.c (mon.2) 2358 : audit [DBG] from='client.? 172.21.15.6:0/3989001270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:19.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:18 smithi006 bash[33918]: cluster 2023-08-28T01:19:17.909328+0000 mgr.y (mgr.14901) 6021 : cluster [DBG] pgmap v6028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:19.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:18 smithi006 bash[33918]: audit 2023-08-28T01:19:18.113537+0000 mon.c (mon.2) 2358 : audit [DBG] from='client.? 172.21.15.6:0/3989001270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:19.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:19 smithi130 bash[51302]: debug 2023-08-28T01:19:19.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:19.441162+0000) 2023-08-28T01:19:20.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:20 smithi130 bash[27579]: audit 2023-08-28T01:19:19.054584+0000 mon.a (mon.0) 4053 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:19:20.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:20 smithi130 bash[27579]: audit 2023-08-28T01:19:19.504269+0000 mon.a (mon.0) 4054 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:19:20.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:20 smithi130 bash[27579]: audit 2023-08-28T01:19:19.806721+0000 mon.b (mon.1) 696 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:19:20.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:20 smithi130 bash[27579]: audit 2023-08-28T01:19:19.809134+0000 mon.b (mon.1) 697 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:19:20.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:20 smithi130 bash[27579]: audit 2023-08-28T01:19:19.817370+0000 mon.a (mon.0) 4055 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:19:20.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:20 smithi006 bash[33918]: audit 2023-08-28T01:19:19.054584+0000 mon.a (mon.0) 4053 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:19:20.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:20 smithi006 bash[33918]: audit 2023-08-28T01:19:19.504269+0000 mon.a (mon.0) 4054 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:19:20.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:20 smithi006 bash[33918]: audit 2023-08-28T01:19:19.806721+0000 mon.b (mon.1) 696 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:19:20.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:20 smithi006 bash[33918]: audit 2023-08-28T01:19:19.809134+0000 mon.b (mon.1) 697 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:19:20.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:20 smithi006 bash[33918]: audit 2023-08-28T01:19:19.817370+0000 mon.a (mon.0) 4055 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:19:20.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:20 smithi006 bash[27292]: audit 2023-08-28T01:19:19.054584+0000 mon.a (mon.0) 4053 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:19:20.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:20 smithi006 bash[27292]: audit 2023-08-28T01:19:19.504269+0000 mon.a (mon.0) 4054 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:19:20.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:20 smithi006 bash[27292]: audit 2023-08-28T01:19:19.806721+0000 mon.b (mon.1) 696 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:19:20.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:20 smithi006 bash[27292]: audit 2023-08-28T01:19:19.809134+0000 mon.b (mon.1) 697 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:19:20.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:20 smithi006 bash[27292]: audit 2023-08-28T01:19:19.817370+0000 mon.a (mon.0) 4055 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:19:20.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:20 smithi130 bash[51302]: debug 2023-08-28T01:19:20.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:20.441454+0000) 2023-08-28T01:19:21.058 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:19:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:19:20] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:19:21.059 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:19:20.982Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:21.059 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:19:20.984Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:21.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:21 smithi130 bash[27579]: cluster 2023-08-28T01:19:19.910474+0000 mgr.y (mgr.14901) 6022 : cluster [DBG] pgmap v6029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:21.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:21 smithi130 bash[27579]: audit 2023-08-28T01:19:20.542212+0000 mon.c (mon.2) 2359 : audit [DBG] from='client.? 172.21.15.6:0/1388579380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:21.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:21 smithi006 bash[33918]: cluster 2023-08-28T01:19:19.910474+0000 mgr.y (mgr.14901) 6022 : cluster [DBG] pgmap v6029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:21.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:21 smithi006 bash[33918]: audit 2023-08-28T01:19:20.542212+0000 mon.c (mon.2) 2359 : audit [DBG] from='client.? 172.21.15.6:0/1388579380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:21.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:21 smithi006 bash[27292]: cluster 2023-08-28T01:19:19.910474+0000 mgr.y (mgr.14901) 6022 : cluster [DBG] pgmap v6029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:21.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:21 smithi006 bash[27292]: audit 2023-08-28T01:19:20.542212+0000 mon.c (mon.2) 2359 : audit [DBG] from='client.? 172.21.15.6:0/1388579380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:21.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:21 smithi130 bash[51302]: debug 2023-08-28T01:19:21.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:21.441737+0000) 2023-08-28T01:19:22.759 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:22 smithi130 bash[51302]: debug 2023-08-28T01:19:22.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:22.441945+0000) 2023-08-28T01:19:22.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:22 smithi006 bash[27292]: cluster 2023-08-28T01:19:21.911131+0000 mgr.y (mgr.14901) 6023 : cluster [DBG] pgmap v6030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:22.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:22 smithi006 bash[33918]: cluster 2023-08-28T01:19:21.911131+0000 mgr.y (mgr.14901) 6023 : cluster [DBG] pgmap v6030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:22.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:19:22.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:22.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:19:22.556Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:23.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:22 smithi130 bash[27579]: cluster 2023-08-28T01:19:21.911131+0000 mgr.y (mgr.14901) 6023 : cluster [DBG] pgmap v6030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:23.766 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:23 smithi130 bash[51302]: debug 2023-08-28T01:19:23.442+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:23.442216+0000) 2023-08-28T01:19:24.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:23 smithi130 bash[27579]: audit 2023-08-28T01:19:22.961699+0000 mon.a (mon.0) 4056 : audit [DBG] from='client.? 172.21.15.6:0/4265753258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:24.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:23 smithi006 bash[33918]: audit 2023-08-28T01:19:22.961699+0000 mon.a (mon.0) 4056 : audit [DBG] from='client.? 172.21.15.6:0/4265753258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:24.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:23 smithi006 bash[27292]: audit 2023-08-28T01:19:22.961699+0000 mon.a (mon.0) 4056 : audit [DBG] from='client.? 172.21.15.6:0/4265753258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:19:24.435Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:24.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:19:24.437Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:24.776 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:24 smithi130 bash[51302]: debug 2023-08-28T01:19:24.438+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:24.442511+0000) 2023-08-28T01:19:25.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:24 smithi130 bash[27579]: cluster 2023-08-28T01:19:23.911831+0000 mgr.y (mgr.14901) 6024 : cluster [DBG] pgmap v6031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:25.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:24 smithi006 bash[27292]: cluster 2023-08-28T01:19:23.911831+0000 mgr.y (mgr.14901) 6024 : cluster [DBG] pgmap v6031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:25.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:24 smithi006 bash[33918]: cluster 2023-08-28T01:19:23.911831+0000 mgr.y (mgr.14901) 6024 : cluster [DBG] pgmap v6031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:25.779 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:25 smithi130 bash[51302]: debug 2023-08-28T01:19:25.441+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:25.442788+0000) 2023-08-28T01:19:26.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:25 smithi130 bash[27579]: audit 2023-08-28T01:19:25.375384+0000 mon.c (mon.2) 2360 : audit [DBG] from='client.? 172.21.15.6:0/1370842393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:26.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:25 smithi006 bash[33918]: audit 2023-08-28T01:19:25.375384+0000 mon.c (mon.2) 2360 : audit [DBG] from='client.? 172.21.15.6:0/1370842393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:26.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:25 smithi006 bash[27292]: audit 2023-08-28T01:19:25.375384+0000 mon.c (mon.2) 2360 : audit [DBG] from='client.? 172.21.15.6:0/1370842393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:26.791 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:26 smithi130 bash[51302]: debug 2023-08-28T01:19:26.441+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:26.443043+0000) 2023-08-28T01:19:27.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:26 smithi130 bash[27579]: cluster 2023-08-28T01:19:25.913017+0000 mgr.y (mgr.14901) 6025 : cluster [DBG] pgmap v6032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:27.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:26 smithi006 bash[33918]: cluster 2023-08-28T01:19:25.913017+0000 mgr.y (mgr.14901) 6025 : cluster [DBG] pgmap v6032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:27.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:26 smithi006 bash[27292]: cluster 2023-08-28T01:19:25.913017+0000 mgr.y (mgr.14901) 6025 : cluster [DBG] pgmap v6032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:27.793 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:27 smithi130 bash[51302]: debug 2023-08-28T01:19:27.441+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:27.443318+0000) 2023-08-28T01:19:28.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:27 smithi130 bash[27579]: audit 2023-08-28T01:19:27.787188+0000 mon.a (mon.0) 4057 : audit [DBG] from='client.? 172.21.15.6:0/2965045545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:28.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:27 smithi006 bash[33918]: audit 2023-08-28T01:19:27.787188+0000 mon.a (mon.0) 4057 : audit [DBG] from='client.? 172.21.15.6:0/2965045545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:28.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:27 smithi006 bash[27292]: audit 2023-08-28T01:19:27.787188+0000 mon.a (mon.0) 4057 : audit [DBG] from='client.? 172.21.15.6:0/2965045545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:28.803 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:28 smithi130 bash[51302]: debug 2023-08-28T01:19:28.441+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:28.443619+0000) 2023-08-28T01:19:29.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:28 smithi130 bash[27579]: cluster 2023-08-28T01:19:27.913709+0000 mgr.y (mgr.14901) 6026 : cluster [DBG] pgmap v6033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:29.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:28 smithi006 bash[33918]: cluster 2023-08-28T01:19:27.913709+0000 mgr.y (mgr.14901) 6026 : cluster [DBG] pgmap v6033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:29.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:28 smithi006 bash[27292]: cluster 2023-08-28T01:19:27.913709+0000 mgr.y (mgr.14901) 6026 : cluster [DBG] pgmap v6033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:29.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:29 smithi130 bash[51302]: debug 2023-08-28T01:19:29.441+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:29.443926+0000) 2023-08-28T01:19:30.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:30 smithi130 bash[51302]: debug 2023-08-28T01:19:30.441+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:30.444219+0000) 2023-08-28T01:19:31.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:30 smithi006 bash[27292]: cluster 2023-08-28T01:19:29.914741+0000 mgr.y (mgr.14901) 6027 : cluster [DBG] pgmap v6034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:31.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:30 smithi006 bash[27292]: audit 2023-08-28T01:19:30.206399+0000 mon.a (mon.0) 4058 : audit [DBG] from='client.? 172.21.15.6:0/2434380146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:31.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:19:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:19:30] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:19:31.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:30 smithi006 bash[33918]: cluster 2023-08-28T01:19:29.914741+0000 mgr.y (mgr.14901) 6027 : cluster [DBG] pgmap v6034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:31.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:30 smithi006 bash[33918]: audit 2023-08-28T01:19:30.206399+0000 mon.a (mon.0) 4058 : audit [DBG] from='client.? 172.21.15.6:0/2434380146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:31.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:19:30.983Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:31.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:30 smithi006 bash[56298]: level=warn ts=2023-08-28T01:19:30.985Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:31.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:30 smithi130 bash[27579]: cluster 2023-08-28T01:19:29.914741+0000 mgr.y (mgr.14901) 6027 : cluster [DBG] pgmap v6034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:31.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:30 smithi130 bash[27579]: audit 2023-08-28T01:19:30.206399+0000 mon.a (mon.0) 4058 : audit [DBG] from='client.? 172.21.15.6:0/2434380146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:31.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:31 smithi130 bash[51302]: debug 2023-08-28T01:19:31.441+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:31.444508+0000) 2023-08-28T01:19:32.763 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:32 smithi130 bash[51302]: debug 2023-08-28T01:19:32.441+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:32.444765+0000) 2023-08-28T01:19:32.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:32 smithi006 bash[33918]: cluster 2023-08-28T01:19:31.915435+0000 mgr.y (mgr.14901) 6028 : cluster [DBG] pgmap v6035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:32 smithi006 bash[33918]: audit 2023-08-28T01:19:32.615999+0000 mon.c (mon.2) 2361 : audit [DBG] from='client.? 172.21.15.6:0/3881873353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:32 smithi006 bash[27292]: cluster 2023-08-28T01:19:31.915435+0000 mgr.y (mgr.14901) 6028 : cluster [DBG] pgmap v6035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:32 smithi006 bash[27292]: audit 2023-08-28T01:19:32.615999+0000 mon.c (mon.2) 2361 : audit [DBG] from='client.? 172.21.15.6:0/3881873353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:19:32.555Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:19:32.557Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:32 smithi130 bash[27579]: cluster 2023-08-28T01:19:31.915435+0000 mgr.y (mgr.14901) 6028 : cluster [DBG] pgmap v6035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:32 smithi130 bash[27579]: audit 2023-08-28T01:19:32.615999+0000 mon.c (mon.2) 2361 : audit [DBG] from='client.? 172.21.15.6:0/3881873353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:33.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:33 smithi130 bash[51302]: debug 2023-08-28T01:19:33.441+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:33.445053+0000) 2023-08-28T01:19:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:19:34.436Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:34.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:19:34.438Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:34.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:34 smithi130 bash[51302]: debug 2023-08-28T01:19:34.441+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:34.445254+0000) 2023-08-28T01:19:35.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:34 smithi130 bash[27579]: cluster 2023-08-28T01:19:33.916099+0000 mgr.y (mgr.14901) 6029 : cluster [DBG] pgmap v6036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:35.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:34 smithi006 bash[27292]: cluster 2023-08-28T01:19:33.916099+0000 mgr.y (mgr.14901) 6029 : cluster [DBG] pgmap v6036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:35.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:34 smithi006 bash[33918]: cluster 2023-08-28T01:19:33.916099+0000 mgr.y (mgr.14901) 6029 : cluster [DBG] pgmap v6036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:35.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:35 smithi130 bash[51302]: debug 2023-08-28T01:19:35.441+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:35.445533+0000) 2023-08-28T01:19:36.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:35 smithi130 bash[27579]: audit 2023-08-28T01:19:35.040933+0000 mon.c (mon.2) 2362 : audit [DBG] from='client.? 172.21.15.6:0/4131546716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:36.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:35 smithi006 bash[27292]: audit 2023-08-28T01:19:35.040933+0000 mon.c (mon.2) 2362 : audit [DBG] from='client.? 172.21.15.6:0/4131546716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:36.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:35 smithi006 bash[33918]: audit 2023-08-28T01:19:35.040933+0000 mon.c (mon.2) 2362 : audit [DBG] from='client.? 172.21.15.6:0/4131546716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:36.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:36 smithi130 bash[51302]: debug 2023-08-28T01:19:36.441+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:36.445877+0000) 2023-08-28T01:19:37.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:36 smithi130 bash[27579]: cluster 2023-08-28T01:19:35.917413+0000 mgr.y (mgr.14901) 6030 : cluster [DBG] pgmap v6037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:37.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:36 smithi006 bash[27292]: cluster 2023-08-28T01:19:35.917413+0000 mgr.y (mgr.14901) 6030 : cluster [DBG] pgmap v6037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:37.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:36 smithi006 bash[33918]: cluster 2023-08-28T01:19:35.917413+0000 mgr.y (mgr.14901) 6030 : cluster [DBG] pgmap v6037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:37.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:37 smithi130 bash[51302]: debug 2023-08-28T01:19:37.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:37.446176+0000) 2023-08-28T01:19:38.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:37 smithi130 bash[27579]: audit 2023-08-28T01:19:37.448027+0000 mon.a (mon.0) 4059 : audit [DBG] from='client.? 172.21.15.6:0/1570656801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:38.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:37 smithi006 bash[27292]: audit 2023-08-28T01:19:37.448027+0000 mon.a (mon.0) 4059 : audit [DBG] from='client.? 172.21.15.6:0/1570656801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:38.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:37 smithi006 bash[33918]: audit 2023-08-28T01:19:37.448027+0000 mon.a (mon.0) 4059 : audit [DBG] from='client.? 172.21.15.6:0/1570656801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:38.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:38 smithi130 bash[51302]: debug 2023-08-28T01:19:38.441+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:38.446461+0000) 2023-08-28T01:19:39.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:38 smithi130 bash[27579]: cluster 2023-08-28T01:19:37.918135+0000 mgr.y (mgr.14901) 6031 : cluster [DBG] pgmap v6038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:39.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:38 smithi006 bash[27292]: cluster 2023-08-28T01:19:37.918135+0000 mgr.y (mgr.14901) 6031 : cluster [DBG] pgmap v6038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:39.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:38 smithi006 bash[33918]: cluster 2023-08-28T01:19:37.918135+0000 mgr.y (mgr.14901) 6031 : cluster [DBG] pgmap v6038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:39.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:39 smithi130 bash[51302]: debug 2023-08-28T01:19:39.441+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:39.446737+0000) 2023-08-28T01:19:40.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:40 smithi130 bash[27579]: audit 2023-08-28T01:19:39.858955+0000 mon.a (mon.0) 4060 : audit [DBG] from='client.? 172.21.15.6:0/2739922210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:40 smithi006 bash[33918]: audit 2023-08-28T01:19:39.858955+0000 mon.a (mon.0) 4060 : audit [DBG] from='client.? 172.21.15.6:0/2739922210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:40.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:39 smithi006 bash[27292]: audit 2023-08-28T01:19:39.858955+0000 mon.a (mon.0) 4060 : audit [DBG] from='client.? 172.21.15.6:0/2739922210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:40.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:40 smithi130 bash[51302]: debug 2023-08-28T01:19:40.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:40.447016+0000) 2023-08-28T01:19:41.010 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:19:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:19:40] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:19:41.011 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:19:40.984Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:41.011 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:19:40.986Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:41.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:41 smithi130 bash[27579]: cluster 2023-08-28T01:19:39.919373+0000 mgr.y (mgr.14901) 6032 : cluster [DBG] pgmap v6039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:41.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:41 smithi006 bash[27292]: cluster 2023-08-28T01:19:39.919373+0000 mgr.y (mgr.14901) 6032 : cluster [DBG] pgmap v6039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:41.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:41 smithi006 bash[33918]: cluster 2023-08-28T01:19:39.919373+0000 mgr.y (mgr.14901) 6032 : cluster [DBG] pgmap v6039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:41.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:41 smithi130 bash[51302]: debug 2023-08-28T01:19:41.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:41.447289+0000) 2023-08-28T01:19:42.766 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:42 smithi130 bash[51302]: debug 2023-08-28T01:19:42.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:42.447537+0000) 2023-08-28T01:19:42.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:42 smithi006 bash[27292]: cluster 2023-08-28T01:19:41.920072+0000 mgr.y (mgr.14901) 6033 : cluster [DBG] pgmap v6040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:42 smithi006 bash[27292]: audit 2023-08-28T01:19:42.279627+0000 mon.a (mon.0) 4061 : audit [DBG] from='client.? 172.21.15.6:0/4199145593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:19:42.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:19:42.558Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:42 smithi006 bash[33918]: cluster 2023-08-28T01:19:41.920072+0000 mgr.y (mgr.14901) 6033 : cluster [DBG] pgmap v6040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:42.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:42 smithi006 bash[33918]: audit 2023-08-28T01:19:42.279627+0000 mon.a (mon.0) 4061 : audit [DBG] from='client.? 172.21.15.6:0/4199145593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:42 smithi130 bash[27579]: cluster 2023-08-28T01:19:41.920072+0000 mgr.y (mgr.14901) 6033 : cluster [DBG] pgmap v6040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:42 smithi130 bash[27579]: audit 2023-08-28T01:19:42.279627+0000 mon.a (mon.0) 4061 : audit [DBG] from='client.? 172.21.15.6:0/4199145593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:43.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:43 smithi130 bash[51302]: debug 2023-08-28T01:19:43.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:43.447836+0000) 2023-08-28T01:19:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:19:44.437Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:44.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:19:44.438Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:44.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:44 smithi130 bash[51302]: debug 2023-08-28T01:19:44.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:44.448126+0000) 2023-08-28T01:19:45.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:44 smithi130 bash[27579]: cluster 2023-08-28T01:19:43.920667+0000 mgr.y (mgr.14901) 6034 : cluster [DBG] pgmap v6041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:45.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:44 smithi130 bash[27579]: audit 2023-08-28T01:19:44.699471+0000 mon.c (mon.2) 2363 : audit [DBG] from='client.? 172.21.15.6:0/4261969482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:45.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:44 smithi006 bash[27292]: cluster 2023-08-28T01:19:43.920667+0000 mgr.y (mgr.14901) 6034 : cluster [DBG] pgmap v6041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:45.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:44 smithi006 bash[27292]: audit 2023-08-28T01:19:44.699471+0000 mon.c (mon.2) 2363 : audit [DBG] from='client.? 172.21.15.6:0/4261969482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:45.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:44 smithi006 bash[33918]: cluster 2023-08-28T01:19:43.920667+0000 mgr.y (mgr.14901) 6034 : cluster [DBG] pgmap v6041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:45.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:44 smithi006 bash[33918]: audit 2023-08-28T01:19:44.699471+0000 mon.c (mon.2) 2363 : audit [DBG] from='client.? 172.21.15.6:0/4261969482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:45.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:45 smithi130 bash[51302]: debug 2023-08-28T01:19:45.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:45.448427+0000) 2023-08-28T01:19:46.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:46 smithi130 bash[51302]: debug 2023-08-28T01:19:46.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:46.448668+0000) 2023-08-28T01:19:47.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:46 smithi130 bash[27579]: cluster 2023-08-28T01:19:45.921846+0000 mgr.y (mgr.14901) 6035 : cluster [DBG] pgmap v6042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:47.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:46 smithi006 bash[33918]: cluster 2023-08-28T01:19:45.921846+0000 mgr.y (mgr.14901) 6035 : cluster [DBG] pgmap v6042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:47.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:46 smithi006 bash[27292]: cluster 2023-08-28T01:19:45.921846+0000 mgr.y (mgr.14901) 6035 : cluster [DBG] pgmap v6042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:47.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:47 smithi130 bash[51302]: debug 2023-08-28T01:19:47.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:47.448937+0000) 2023-08-28T01:19:48.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:47 smithi130 bash[27579]: audit 2023-08-28T01:19:47.110637+0000 mon.c (mon.2) 2364 : audit [DBG] from='client.? 172.21.15.6:0/3381721357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:48.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:47 smithi006 bash[33918]: audit 2023-08-28T01:19:47.110637+0000 mon.c (mon.2) 2364 : audit [DBG] from='client.? 172.21.15.6:0/3381721357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:48.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:47 smithi006 bash[27292]: audit 2023-08-28T01:19:47.110637+0000 mon.c (mon.2) 2364 : audit [DBG] from='client.? 172.21.15.6:0/3381721357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:48.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:48 smithi130 bash[51302]: debug 2023-08-28T01:19:48.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:48.449239+0000) 2023-08-28T01:19:49.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:49 smithi130 bash[27579]: cluster 2023-08-28T01:19:47.922516+0000 mgr.y (mgr.14901) 6036 : cluster [DBG] pgmap v6043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:49.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:48 smithi006 bash[27292]: cluster 2023-08-28T01:19:47.922516+0000 mgr.y (mgr.14901) 6036 : cluster [DBG] pgmap v6043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:49.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:48 smithi006 bash[33918]: cluster 2023-08-28T01:19:47.922516+0000 mgr.y (mgr.14901) 6036 : cluster [DBG] pgmap v6043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:49.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:49 smithi130 bash[51302]: debug 2023-08-28T01:19:49.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:49.449491+0000) 2023-08-28T01:19:50.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:50 smithi130 bash[27579]: audit 2023-08-28T01:19:49.523381+0000 mon.a (mon.0) 4062 : audit [DBG] from='client.? 172.21.15.6:0/41651258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:50.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:50 smithi006 bash[27292]: audit 2023-08-28T01:19:49.523381+0000 mon.a (mon.0) 4062 : audit [DBG] from='client.? 172.21.15.6:0/41651258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:50.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:50 smithi006 bash[33918]: audit 2023-08-28T01:19:49.523381+0000 mon.a (mon.0) 4062 : audit [DBG] from='client.? 172.21.15.6:0/41651258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:50.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:50 smithi130 bash[51302]: debug 2023-08-28T01:19:50.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:50.449754+0000) 2023-08-28T01:19:51.010 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:19:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:19:50] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:19:51.010 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:19:50.984Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:51.010 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:19:50.986Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:51.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:51 smithi130 bash[27579]: cluster 2023-08-28T01:19:49.923672+0000 mgr.y (mgr.14901) 6037 : cluster [DBG] pgmap v6044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:51.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:51 smithi006 bash[27292]: cluster 2023-08-28T01:19:49.923672+0000 mgr.y (mgr.14901) 6037 : cluster [DBG] pgmap v6044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:51.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:51 smithi006 bash[33918]: cluster 2023-08-28T01:19:49.923672+0000 mgr.y (mgr.14901) 6037 : cluster [DBG] pgmap v6044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:51.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:51 smithi130 bash[51302]: debug 2023-08-28T01:19:51.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:51.450010+0000) 2023-08-28T01:19:52.767 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:52 smithi130 bash[51302]: debug 2023-08-28T01:19:52.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:52.450290+0000) 2023-08-28T01:19:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:19:52.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:19:52.558Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:52 smithi006 bash[27292]: cluster 2023-08-28T01:19:51.924277+0000 mgr.y (mgr.14901) 6038 : cluster [DBG] pgmap v6045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:52 smithi006 bash[27292]: audit 2023-08-28T01:19:51.944968+0000 mon.c (mon.2) 2365 : audit [DBG] from='client.? 172.21.15.6:0/1757574756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:52 smithi006 bash[33918]: cluster 2023-08-28T01:19:51.924277+0000 mgr.y (mgr.14901) 6038 : cluster [DBG] pgmap v6045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:52 smithi006 bash[33918]: audit 2023-08-28T01:19:51.944968+0000 mon.c (mon.2) 2365 : audit [DBG] from='client.? 172.21.15.6:0/1757574756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:52 smithi130 bash[27579]: cluster 2023-08-28T01:19:51.924277+0000 mgr.y (mgr.14901) 6038 : cluster [DBG] pgmap v6045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:53.126 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:52 smithi130 bash[27579]: audit 2023-08-28T01:19:51.944968+0000 mon.c (mon.2) 2365 : audit [DBG] from='client.? 172.21.15.6:0/1757574756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:53.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:53 smithi130 bash[51302]: debug 2023-08-28T01:19:53.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:53.450512+0000) 2023-08-28T01:19:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:19:54.437Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:54.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:19:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:19:54.439Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:19:54.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:54 smithi130 bash[51302]: debug 2023-08-28T01:19:54.445+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:54.450721+0000) 2023-08-28T01:19:55.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:54 smithi130 bash[27579]: cluster 2023-08-28T01:19:53.924924+0000 mgr.y (mgr.14901) 6039 : cluster [DBG] pgmap v6046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:55.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:54 smithi130 bash[27579]: audit 2023-08-28T01:19:54.355656+0000 mon.c (mon.2) 2366 : audit [DBG] from='client.? 172.21.15.6:0/4227386738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:55.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:54 smithi006 bash[27292]: cluster 2023-08-28T01:19:53.924924+0000 mgr.y (mgr.14901) 6039 : cluster [DBG] pgmap v6046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:55.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:54 smithi006 bash[27292]: audit 2023-08-28T01:19:54.355656+0000 mon.c (mon.2) 2366 : audit [DBG] from='client.? 172.21.15.6:0/4227386738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:55.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:54 smithi006 bash[33918]: cluster 2023-08-28T01:19:53.924924+0000 mgr.y (mgr.14901) 6039 : cluster [DBG] pgmap v6046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:55.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:54 smithi006 bash[33918]: audit 2023-08-28T01:19:54.355656+0000 mon.c (mon.2) 2366 : audit [DBG] from='client.? 172.21.15.6:0/4227386738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:55.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:55 smithi130 bash[51302]: debug 2023-08-28T01:19:55.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:55.450953+0000) 2023-08-28T01:19:56.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:56 smithi130 bash[51302]: debug 2023-08-28T01:19:56.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:56.451168+0000) 2023-08-28T01:19:57.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:56 smithi130 bash[27579]: cluster 2023-08-28T01:19:55.926193+0000 mgr.y (mgr.14901) 6040 : cluster [DBG] pgmap v6047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:57.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:56 smithi130 bash[27579]: audit 2023-08-28T01:19:56.775864+0000 mon.c (mon.2) 2367 : audit [DBG] from='client.? 172.21.15.6:0/4218749729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:57.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:56 smithi006 bash[33918]: cluster 2023-08-28T01:19:55.926193+0000 mgr.y (mgr.14901) 6040 : cluster [DBG] pgmap v6047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:57.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:56 smithi006 bash[33918]: audit 2023-08-28T01:19:56.775864+0000 mon.c (mon.2) 2367 : audit [DBG] from='client.? 172.21.15.6:0/4218749729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:57.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:56 smithi006 bash[27292]: cluster 2023-08-28T01:19:55.926193+0000 mgr.y (mgr.14901) 6040 : cluster [DBG] pgmap v6047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:57.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:56 smithi006 bash[27292]: audit 2023-08-28T01:19:56.775864+0000 mon.c (mon.2) 2367 : audit [DBG] from='client.? 172.21.15.6:0/4218749729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:57.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:57 smithi130 bash[51302]: debug 2023-08-28T01:19:57.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:57.451387+0000) 2023-08-28T01:19:58.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:58 smithi130 bash[51302]: debug 2023-08-28T01:19:58.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:58.451593+0000) 2023-08-28T01:19:59.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:58 smithi130 bash[27579]: cluster 2023-08-28T01:19:57.926999+0000 mgr.y (mgr.14901) 6041 : cluster [DBG] pgmap v6048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:59.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:58 smithi006 bash[33918]: cluster 2023-08-28T01:19:57.926999+0000 mgr.y (mgr.14901) 6041 : cluster [DBG] pgmap v6048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:59.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:58 smithi006 bash[27292]: cluster 2023-08-28T01:19:57.926999+0000 mgr.y (mgr.14901) 6041 : cluster [DBG] pgmap v6048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:19:59.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:19:59 smithi130 bash[51302]: debug 2023-08-28T01:19:59.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:59.451826+0000) 2023-08-28T01:20:00.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:19:59 smithi130 bash[27579]: audit 2023-08-28T01:19:59.198241+0000 mon.a (mon.0) 4063 : audit [DBG] from='client.? 172.21.15.6:0/3333832919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:00.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:19:59 smithi006 bash[33918]: audit 2023-08-28T01:19:59.198241+0000 mon.a (mon.0) 4063 : audit [DBG] from='client.? 172.21.15.6:0/3333832919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:00.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:19:59 smithi006 bash[27292]: audit 2023-08-28T01:19:59.198241+0000 mon.a (mon.0) 4063 : audit [DBG] from='client.? 172.21.15.6:0/3333832919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:00.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:00 smithi130 bash[51302]: debug 2023-08-28T01:20:00.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:00.452109+0000) 2023-08-28T01:20:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:00 smithi006 bash[27292]: cluster 2023-08-28T01:19:59.928156+0000 mgr.y (mgr.14901) 6042 : cluster [DBG] pgmap v6049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:01.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:00 smithi006 bash[27292]: cluster 2023-08-28T01:20:00.000315+0000 mon.a (mon.0) 4064 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:20:01.216 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:20:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:20:00] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:20:01.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:00 smithi006 bash[33918]: cluster 2023-08-28T01:19:59.928156+0000 mgr.y (mgr.14901) 6042 : cluster [DBG] pgmap v6049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:01.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:00 smithi006 bash[33918]: cluster 2023-08-28T01:20:00.000315+0000 mon.a (mon.0) 4064 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:20:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:20:00.985Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:01.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:20:00.987Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:01.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:01 smithi130 bash[27579]: cluster 2023-08-28T01:19:59.928156+0000 mgr.y (mgr.14901) 6042 : cluster [DBG] pgmap v6049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:01.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:01 smithi130 bash[27579]: cluster 2023-08-28T01:20:00.000315+0000 mon.a (mon.0) 4064 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:20:01.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:01 smithi130 bash[51302]: debug 2023-08-28T01:20:01.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:01.452354+0000) 2023-08-28T01:20:02.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:02 smithi130 bash[27579]: audit 2023-08-28T01:20:01.607294+0000 mon.c (mon.2) 2368 : audit [DBG] from='client.? 172.21.15.6:0/2470395774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:02.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:02 smithi006 bash[33918]: audit 2023-08-28T01:20:01.607294+0000 mon.c (mon.2) 2368 : audit [DBG] from='client.? 172.21.15.6:0/2470395774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:02.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:02 smithi006 bash[27292]: audit 2023-08-28T01:20:01.607294+0000 mon.c (mon.2) 2368 : audit [DBG] from='client.? 172.21.15.6:0/2470395774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:02.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:02 smithi130 bash[51302]: debug 2023-08-28T01:20:02.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:02.452615+0000) 2023-08-28T01:20:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:20:02.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:20:02.559Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:03.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:03 smithi130 bash[27579]: cluster 2023-08-28T01:20:01.928885+0000 mgr.y (mgr.14901) 6043 : cluster [DBG] pgmap v6050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:03.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:03 smithi006 bash[33918]: cluster 2023-08-28T01:20:01.928885+0000 mgr.y (mgr.14901) 6043 : cluster [DBG] pgmap v6050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:03.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:03 smithi006 bash[27292]: cluster 2023-08-28T01:20:01.928885+0000 mgr.y (mgr.14901) 6043 : cluster [DBG] pgmap v6050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:03.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:03 smithi130 bash[51302]: debug 2023-08-28T01:20:03.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:03.452834+0000) 2023-08-28T01:20:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:20:04.438Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:20:04.441Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:04.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:04 smithi130 bash[51302]: debug 2023-08-28T01:20:04.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:04.453062+0000) 2023-08-28T01:20:05.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:05 smithi130 bash[27579]: cluster 2023-08-28T01:20:03.929477+0000 mgr.y (mgr.14901) 6044 : cluster [DBG] pgmap v6051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:05.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:05 smithi130 bash[27579]: audit 2023-08-28T01:20:04.022399+0000 mon.c (mon.2) 2369 : audit [DBG] from='client.? 172.21.15.6:0/179012136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:05.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:05 smithi006 bash[33918]: cluster 2023-08-28T01:20:03.929477+0000 mgr.y (mgr.14901) 6044 : cluster [DBG] pgmap v6051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:05.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:05 smithi006 bash[33918]: audit 2023-08-28T01:20:04.022399+0000 mon.c (mon.2) 2369 : audit [DBG] from='client.? 172.21.15.6:0/179012136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:05.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:05 smithi006 bash[27292]: cluster 2023-08-28T01:20:03.929477+0000 mgr.y (mgr.14901) 6044 : cluster [DBG] pgmap v6051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:05.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:05 smithi006 bash[27292]: audit 2023-08-28T01:20:04.022399+0000 mon.c (mon.2) 2369 : audit [DBG] from='client.? 172.21.15.6:0/179012136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:05.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:05 smithi130 bash[51302]: debug 2023-08-28T01:20:05.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:05.453308+0000) 2023-08-28T01:20:06.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:06 smithi130 bash[51302]: debug 2023-08-28T01:20:06.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:06.453515+0000) 2023-08-28T01:20:07.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:07 smithi130 bash[27579]: cluster 2023-08-28T01:20:05.930613+0000 mgr.y (mgr.14901) 6045 : cluster [DBG] pgmap v6052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:07.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:07 smithi130 bash[27579]: audit 2023-08-28T01:20:06.440134+0000 mon.a (mon.0) 4065 : audit [DBG] from='client.? 172.21.15.6:0/3020243122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:07 smithi006 bash[27292]: cluster 2023-08-28T01:20:05.930613+0000 mgr.y (mgr.14901) 6045 : cluster [DBG] pgmap v6052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:07 smithi006 bash[27292]: audit 2023-08-28T01:20:06.440134+0000 mon.a (mon.0) 4065 : audit [DBG] from='client.? 172.21.15.6:0/3020243122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:07.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:07 smithi006 bash[33918]: cluster 2023-08-28T01:20:05.930613+0000 mgr.y (mgr.14901) 6045 : cluster [DBG] pgmap v6052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:07.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:07 smithi006 bash[33918]: audit 2023-08-28T01:20:06.440134+0000 mon.a (mon.0) 4065 : audit [DBG] from='client.? 172.21.15.6:0/3020243122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:07.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:07 smithi130 bash[51302]: debug 2023-08-28T01:20:07.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:07.453731+0000) 2023-08-28T01:20:08.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:08 smithi130 bash[51302]: debug 2023-08-28T01:20:08.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:08.454004+0000) 2023-08-28T01:20:09.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:09 smithi130 bash[27579]: cluster 2023-08-28T01:20:07.931285+0000 mgr.y (mgr.14901) 6046 : cluster [DBG] pgmap v6053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:09.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:09 smithi130 bash[27579]: audit 2023-08-28T01:20:08.852754+0000 mon.c (mon.2) 2370 : audit [DBG] from='client.? 172.21.15.6:0/1350454029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:09.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:09 smithi006 bash[27292]: cluster 2023-08-28T01:20:07.931285+0000 mgr.y (mgr.14901) 6046 : cluster [DBG] pgmap v6053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:09.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:09 smithi006 bash[27292]: audit 2023-08-28T01:20:08.852754+0000 mon.c (mon.2) 2370 : audit [DBG] from='client.? 172.21.15.6:0/1350454029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:09.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:09 smithi006 bash[33918]: cluster 2023-08-28T01:20:07.931285+0000 mgr.y (mgr.14901) 6046 : cluster [DBG] pgmap v6053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:09.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:09 smithi006 bash[33918]: audit 2023-08-28T01:20:08.852754+0000 mon.c (mon.2) 2370 : audit [DBG] from='client.? 172.21.15.6:0/1350454029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:09.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:09 smithi130 bash[51302]: debug 2023-08-28T01:20:09.449+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:09.454312+0000) 2023-08-28T01:20:10.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:10 smithi130 bash[51302]: debug 2023-08-28T01:20:10.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:10.454599+0000) 2023-08-28T01:20:11.022 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:20:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:20:10] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:20:11.023 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:20:10.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:11.023 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:20:10.987Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:11.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:11 smithi130 bash[27579]: cluster 2023-08-28T01:20:09.932399+0000 mgr.y (mgr.14901) 6047 : cluster [DBG] pgmap v6054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:11.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:11 smithi006 bash[27292]: cluster 2023-08-28T01:20:09.932399+0000 mgr.y (mgr.14901) 6047 : cluster [DBG] pgmap v6054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:11.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:11 smithi006 bash[33918]: cluster 2023-08-28T01:20:09.932399+0000 mgr.y (mgr.14901) 6047 : cluster [DBG] pgmap v6054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:11.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:11 smithi130 bash[51302]: debug 2023-08-28T01:20:11.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:11.454923+0000) 2023-08-28T01:20:12.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:12 smithi130 bash[27579]: audit 2023-08-28T01:20:11.274920+0000 mon.a (mon.0) 4066 : audit [DBG] from='client.? 172.21.15.6:0/4162071541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:12.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:12 smithi006 bash[33918]: audit 2023-08-28T01:20:11.274920+0000 mon.a (mon.0) 4066 : audit [DBG] from='client.? 172.21.15.6:0/4162071541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:12.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:12 smithi006 bash[27292]: audit 2023-08-28T01:20:11.274920+0000 mon.a (mon.0) 4066 : audit [DBG] from='client.? 172.21.15.6:0/4162071541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:12.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:12 smithi130 bash[51302]: debug 2023-08-28T01:20:12.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:12.455148+0000) 2023-08-28T01:20:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:20:12.558Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:20:12.560Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:13.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:13 smithi130 bash[27579]: cluster 2023-08-28T01:20:11.933086+0000 mgr.y (mgr.14901) 6048 : cluster [DBG] pgmap v6055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:13.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:13 smithi006 bash[27292]: cluster 2023-08-28T01:20:11.933086+0000 mgr.y (mgr.14901) 6048 : cluster [DBG] pgmap v6055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:13.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:13 smithi006 bash[33918]: cluster 2023-08-28T01:20:11.933086+0000 mgr.y (mgr.14901) 6048 : cluster [DBG] pgmap v6055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:13.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:13 smithi130 bash[51302]: debug 2023-08-28T01:20:13.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:13.455398+0000) 2023-08-28T01:20:14.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:14 smithi130 bash[27579]: audit 2023-08-28T01:20:13.697424+0000 mon.c (mon.2) 2371 : audit [DBG] from='client.? 172.21.15.6:0/2453263120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:14.440 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:14 smithi006 bash[27292]: audit 2023-08-28T01:20:13.697424+0000 mon.c (mon.2) 2371 : audit [DBG] from='client.? 172.21.15.6:0/2453263120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:14.440 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:14 smithi006 bash[33918]: audit 2023-08-28T01:20:13.697424+0000 mon.c (mon.2) 2371 : audit [DBG] from='client.? 172.21.15.6:0/2453263120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:20:14.439Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:20:14.440Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:14.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:14 smithi130 bash[51302]: debug 2023-08-28T01:20:14.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:14.455629+0000) 2023-08-28T01:20:15.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:15 smithi130 bash[27579]: cluster 2023-08-28T01:20:13.933746+0000 mgr.y (mgr.14901) 6049 : cluster [DBG] pgmap v6056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:15.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:15 smithi006 bash[27292]: cluster 2023-08-28T01:20:13.933746+0000 mgr.y (mgr.14901) 6049 : cluster [DBG] pgmap v6056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:15.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:15 smithi006 bash[33918]: cluster 2023-08-28T01:20:13.933746+0000 mgr.y (mgr.14901) 6049 : cluster [DBG] pgmap v6056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:15.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:15 smithi130 bash[51302]: debug 2023-08-28T01:20:15.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:15.455849+0000) 2023-08-28T01:20:16.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:16 smithi130 bash[51302]: debug 2023-08-28T01:20:16.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:16.456153+0000) 2023-08-28T01:20:17.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:17 smithi130 bash[27579]: cluster 2023-08-28T01:20:15.934852+0000 mgr.y (mgr.14901) 6050 : cluster [DBG] pgmap v6057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:17.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:17 smithi130 bash[27579]: audit 2023-08-28T01:20:16.123055+0000 mon.a (mon.0) 4067 : audit [DBG] from='client.? 172.21.15.6:0/518142367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:17.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:17 smithi006 bash[27292]: cluster 2023-08-28T01:20:15.934852+0000 mgr.y (mgr.14901) 6050 : cluster [DBG] pgmap v6057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:17.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:17 smithi006 bash[27292]: audit 2023-08-28T01:20:16.123055+0000 mon.a (mon.0) 4067 : audit [DBG] from='client.? 172.21.15.6:0/518142367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:17.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:17 smithi006 bash[33918]: cluster 2023-08-28T01:20:15.934852+0000 mgr.y (mgr.14901) 6050 : cluster [DBG] pgmap v6057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:17.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:17 smithi006 bash[33918]: audit 2023-08-28T01:20:16.123055+0000 mon.a (mon.0) 4067 : audit [DBG] from='client.? 172.21.15.6:0/518142367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:17.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:17 smithi130 bash[51302]: debug 2023-08-28T01:20:17.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:17.456445+0000) 2023-08-28T01:20:18.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:18 smithi130 bash[51302]: debug 2023-08-28T01:20:18.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:18.456707+0000) 2023-08-28T01:20:19.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:19 smithi130 bash[27579]: cluster 2023-08-28T01:20:17.935513+0000 mgr.y (mgr.14901) 6051 : cluster [DBG] pgmap v6058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:19.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:19 smithi130 bash[27579]: audit 2023-08-28T01:20:18.536389+0000 mon.c (mon.2) 2372 : audit [DBG] from='client.? 172.21.15.6:0/683165777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:19.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:19 smithi006 bash[33918]: cluster 2023-08-28T01:20:17.935513+0000 mgr.y (mgr.14901) 6051 : cluster [DBG] pgmap v6058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:19.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:19 smithi006 bash[33918]: audit 2023-08-28T01:20:18.536389+0000 mon.c (mon.2) 2372 : audit [DBG] from='client.? 172.21.15.6:0/683165777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:19.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:19 smithi006 bash[27292]: cluster 2023-08-28T01:20:17.935513+0000 mgr.y (mgr.14901) 6051 : cluster [DBG] pgmap v6058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:19.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:19 smithi006 bash[27292]: audit 2023-08-28T01:20:18.536389+0000 mon.c (mon.2) 2372 : audit [DBG] from='client.? 172.21.15.6:0/683165777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:19.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:19 smithi130 bash[51302]: debug 2023-08-28T01:20:19.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:19.457012+0000) 2023-08-28T01:20:20.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:20 smithi130 bash[27579]: audit 2023-08-28T01:20:19.830467+0000 mon.b (mon.1) 698 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:20:20.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:20 smithi006 bash[33918]: audit 2023-08-28T01:20:19.830467+0000 mon.b (mon.1) 698 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:20:20.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:20 smithi006 bash[27292]: audit 2023-08-28T01:20:19.830467+0000 mon.b (mon.1) 698 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:20:20.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:20 smithi130 bash[51302]: debug 2023-08-28T01:20:20.453+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:20.457299+0000) 2023-08-28T01:20:21.061 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:20:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:20:20] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:20:21.061 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:20:20.987Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:21.061 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:20:20.989Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:21.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:21 smithi130 bash[27579]: cluster 2023-08-28T01:20:19.936633+0000 mgr.y (mgr.14901) 6052 : cluster [DBG] pgmap v6059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:21.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:21 smithi130 bash[27579]: audit 2023-08-28T01:20:20.951871+0000 mon.c (mon.2) 2373 : audit [DBG] from='client.? 172.21.15.6:0/246263087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:21.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:21 smithi006 bash[33918]: cluster 2023-08-28T01:20:19.936633+0000 mgr.y (mgr.14901) 6052 : cluster [DBG] pgmap v6059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:21.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:21 smithi006 bash[33918]: audit 2023-08-28T01:20:20.951871+0000 mon.c (mon.2) 2373 : audit [DBG] from='client.? 172.21.15.6:0/246263087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:21.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:21 smithi006 bash[27292]: cluster 2023-08-28T01:20:19.936633+0000 mgr.y (mgr.14901) 6052 : cluster [DBG] pgmap v6059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:21.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:21 smithi006 bash[27292]: audit 2023-08-28T01:20:20.951871+0000 mon.c (mon.2) 2373 : audit [DBG] from='client.? 172.21.15.6:0/246263087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:21.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:21 smithi130 bash[51302]: debug 2023-08-28T01:20:21.452+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:21.457559+0000) 2023-08-28T01:20:22.777 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:22 smithi130 bash[51302]: debug 2023-08-28T01:20:22.452+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:22.457834+0000) 2023-08-28T01:20:22.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:22 smithi006 bash[27292]: cluster 2023-08-28T01:20:21.937339+0000 mgr.y (mgr.14901) 6053 : cluster [DBG] pgmap v6060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:22.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:22 smithi006 bash[33918]: cluster 2023-08-28T01:20:21.937339+0000 mgr.y (mgr.14901) 6053 : cluster [DBG] pgmap v6060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:22.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:20:22.559Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:22.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:20:22.561Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:23.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:22 smithi130 bash[27579]: cluster 2023-08-28T01:20:21.937339+0000 mgr.y (mgr.14901) 6053 : cluster [DBG] pgmap v6060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:23.780 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:23 smithi130 bash[51302]: debug 2023-08-28T01:20:23.452+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:23.458114+0000) 2023-08-28T01:20:24.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:23 smithi130 bash[27579]: audit 2023-08-28T01:20:23.373901+0000 mon.c (mon.2) 2374 : audit [DBG] from='client.? 172.21.15.6:0/3548426651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:23 smithi006 bash[27292]: audit 2023-08-28T01:20:23.373901+0000 mon.c (mon.2) 2374 : audit [DBG] from='client.? 172.21.15.6:0/3548426651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:24.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:23 smithi006 bash[33918]: audit 2023-08-28T01:20:23.373901+0000 mon.c (mon.2) 2374 : audit [DBG] from='client.? 172.21.15.6:0/3548426651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:20:24.440Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:24.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:20:24.442Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:24.788 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:24 smithi130 bash[51302]: debug 2023-08-28T01:20:24.452+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:24.458398+0000) 2023-08-28T01:20:25.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:24 smithi130 bash[27579]: cluster 2023-08-28T01:20:23.938137+0000 mgr.y (mgr.14901) 6054 : cluster [DBG] pgmap v6061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:25.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:24 smithi006 bash[27292]: cluster 2023-08-28T01:20:23.938137+0000 mgr.y (mgr.14901) 6054 : cluster [DBG] pgmap v6061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:25.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:24 smithi006 bash[33918]: cluster 2023-08-28T01:20:23.938137+0000 mgr.y (mgr.14901) 6054 : cluster [DBG] pgmap v6061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:25.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:25 smithi130 bash[51302]: debug 2023-08-28T01:20:25.452+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:25.458621+0000) 2023-08-28T01:20:26.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:26 smithi130 bash[51302]: debug 2023-08-28T01:20:26.456+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:26.458925+0000) 2023-08-28T01:20:26.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:26 smithi130 bash[27579]: audit 2023-08-28T01:20:25.698030+0000 mon.a (mon.0) 4068 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:20:26.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:26 smithi130 bash[27579]: audit 2023-08-28T01:20:25.812212+0000 mon.a (mon.0) 4069 : audit [DBG] from='client.? 172.21.15.6:0/3569192346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:26.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:26 smithi130 bash[27579]: cluster 2023-08-28T01:20:25.939211+0000 mgr.y (mgr.14901) 6055 : cluster [DBG] pgmap v6062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:26.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:26 smithi130 bash[27579]: audit 2023-08-28T01:20:26.181799+0000 mon.a (mon.0) 4070 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:20:26.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:26 smithi130 bash[27579]: audit 2023-08-28T01:20:26.460576+0000 mon.b (mon.1) 699 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:20:26.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:26 smithi130 bash[27579]: audit 2023-08-28T01:20:26.462957+0000 mon.b (mon.1) 700 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:20:26.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:26 smithi130 bash[27579]: audit 2023-08-28T01:20:26.470004+0000 mon.a (mon.0) 4071 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:20:26.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:26 smithi006 bash[27292]: audit 2023-08-28T01:20:25.698030+0000 mon.a (mon.0) 4068 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:20:26.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:26 smithi006 bash[27292]: audit 2023-08-28T01:20:25.812212+0000 mon.a (mon.0) 4069 : audit [DBG] from='client.? 172.21.15.6:0/3569192346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:26.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:26 smithi006 bash[27292]: cluster 2023-08-28T01:20:25.939211+0000 mgr.y (mgr.14901) 6055 : cluster [DBG] pgmap v6062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:26.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:26 smithi006 bash[27292]: audit 2023-08-28T01:20:26.181799+0000 mon.a (mon.0) 4070 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:20:26.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:26 smithi006 bash[27292]: audit 2023-08-28T01:20:26.460576+0000 mon.b (mon.1) 699 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:20:26.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:26 smithi006 bash[27292]: audit 2023-08-28T01:20:26.462957+0000 mon.b (mon.1) 700 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:20:26.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:26 smithi006 bash[27292]: audit 2023-08-28T01:20:26.470004+0000 mon.a (mon.0) 4071 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:20:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:26 smithi006 bash[33918]: audit 2023-08-28T01:20:25.698030+0000 mon.a (mon.0) 4068 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:20:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:26 smithi006 bash[33918]: audit 2023-08-28T01:20:25.812212+0000 mon.a (mon.0) 4069 : audit [DBG] from='client.? 172.21.15.6:0/3569192346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:26 smithi006 bash[33918]: cluster 2023-08-28T01:20:25.939211+0000 mgr.y (mgr.14901) 6055 : cluster [DBG] pgmap v6062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:26 smithi006 bash[33918]: audit 2023-08-28T01:20:26.181799+0000 mon.a (mon.0) 4070 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:20:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:26 smithi006 bash[33918]: audit 2023-08-28T01:20:26.460576+0000 mon.b (mon.1) 699 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:20:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:26 smithi006 bash[33918]: audit 2023-08-28T01:20:26.462957+0000 mon.b (mon.1) 700 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:20:26.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:26 smithi006 bash[33918]: audit 2023-08-28T01:20:26.470004+0000 mon.a (mon.0) 4071 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:20:27.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:27 smithi130 bash[51302]: debug 2023-08-28T01:20:27.456+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:27.459297+0000) 2023-08-28T01:20:28.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:28 smithi130 bash[51302]: debug 2023-08-28T01:20:28.456+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:28.459550+0000) 2023-08-28T01:20:29.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:29 smithi130 bash[27579]: cluster 2023-08-28T01:20:27.939847+0000 mgr.y (mgr.14901) 6056 : cluster [DBG] pgmap v6063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:29.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:29 smithi130 bash[27579]: audit 2023-08-28T01:20:28.229728+0000 mon.a (mon.0) 4072 : audit [DBG] from='client.? 172.21.15.6:0/2131014168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:29.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:28 smithi006 bash[27292]: cluster 2023-08-28T01:20:27.939847+0000 mgr.y (mgr.14901) 6056 : cluster [DBG] pgmap v6063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:29.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:28 smithi006 bash[27292]: audit 2023-08-28T01:20:28.229728+0000 mon.a (mon.0) 4072 : audit [DBG] from='client.? 172.21.15.6:0/2131014168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:29.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:28 smithi006 bash[33918]: cluster 2023-08-28T01:20:27.939847+0000 mgr.y (mgr.14901) 6056 : cluster [DBG] pgmap v6063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:29.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:28 smithi006 bash[33918]: audit 2023-08-28T01:20:28.229728+0000 mon.a (mon.0) 4072 : audit [DBG] from='client.? 172.21.15.6:0/2131014168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:29.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:29 smithi130 bash[51302]: debug 2023-08-28T01:20:29.456+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:29.459899+0000) 2023-08-28T01:20:30.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:30 smithi130 bash[51302]: debug 2023-08-28T01:20:30.456+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:30.460303+0000) 2023-08-28T01:20:31.138 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:20:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:20:30] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:20:31.138 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:20:30.988Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:31.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:31 smithi130 bash[27579]: cluster 2023-08-28T01:20:29.940981+0000 mgr.y (mgr.14901) 6057 : cluster [DBG] pgmap v6064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:31.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:31 smithi130 bash[27579]: audit 2023-08-28T01:20:30.646555+0000 mon.a (mon.0) 4073 : audit [DBG] from='client.? 172.21.15.6:0/265290259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:20:30.989Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:31.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:31 smithi006 bash[27292]: cluster 2023-08-28T01:20:29.940981+0000 mgr.y (mgr.14901) 6057 : cluster [DBG] pgmap v6064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:31.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:31 smithi006 bash[27292]: audit 2023-08-28T01:20:30.646555+0000 mon.a (mon.0) 4073 : audit [DBG] from='client.? 172.21.15.6:0/265290259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:31.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:31 smithi006 bash[33918]: cluster 2023-08-28T01:20:29.940981+0000 mgr.y (mgr.14901) 6057 : cluster [DBG] pgmap v6064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:31.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:31 smithi006 bash[33918]: audit 2023-08-28T01:20:30.646555+0000 mon.a (mon.0) 4073 : audit [DBG] from='client.? 172.21.15.6:0/265290259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:31.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:31 smithi130 bash[51302]: debug 2023-08-28T01:20:31.456+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:31.460582+0000) 2023-08-28T01:20:32.782 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:32 smithi130 bash[51302]: debug 2023-08-28T01:20:32.456+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:32.460907+0000) 2023-08-28T01:20:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:20:32.560Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:20:32.562Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:32 smithi006 bash[27292]: cluster 2023-08-28T01:20:31.941646+0000 mgr.y (mgr.14901) 6058 : cluster [DBG] pgmap v6065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:32 smithi006 bash[33918]: cluster 2023-08-28T01:20:31.941646+0000 mgr.y (mgr.14901) 6058 : cluster [DBG] pgmap v6065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:32 smithi130 bash[27579]: cluster 2023-08-28T01:20:31.941646+0000 mgr.y (mgr.14901) 6058 : cluster [DBG] pgmap v6065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:33.786 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:33 smithi130 bash[51302]: debug 2023-08-28T01:20:33.456+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:33.461204+0000) 2023-08-28T01:20:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:33 smithi130 bash[27579]: audit 2023-08-28T01:20:33.056016+0000 mon.c (mon.2) 2375 : audit [DBG] from='client.? 172.21.15.6:0/2956888436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:34.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:33 smithi006 bash[33918]: audit 2023-08-28T01:20:33.056016+0000 mon.c (mon.2) 2375 : audit [DBG] from='client.? 172.21.15.6:0/2956888436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:34.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:33 smithi006 bash[27292]: audit 2023-08-28T01:20:33.056016+0000 mon.c (mon.2) 2375 : audit [DBG] from='client.? 172.21.15.6:0/2956888436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:20:34.441Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:34.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:20:34.443Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:34.795 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:34 smithi130 bash[51302]: debug 2023-08-28T01:20:34.456+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:34.461529+0000) 2023-08-28T01:20:35.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:34 smithi130 bash[27579]: cluster 2023-08-28T01:20:33.942321+0000 mgr.y (mgr.14901) 6059 : cluster [DBG] pgmap v6066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:35.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:34 smithi006 bash[27292]: cluster 2023-08-28T01:20:33.942321+0000 mgr.y (mgr.14901) 6059 : cluster [DBG] pgmap v6066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:35.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:34 smithi006 bash[33918]: cluster 2023-08-28T01:20:33.942321+0000 mgr.y (mgr.14901) 6059 : cluster [DBG] pgmap v6066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:35.796 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:35 smithi130 bash[51302]: debug 2023-08-28T01:20:35.456+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:35.461875+0000) 2023-08-28T01:20:36.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:35 smithi130 bash[27579]: audit 2023-08-28T01:20:35.474195+0000 mon.c (mon.2) 2376 : audit [DBG] from='client.? 172.21.15.6:0/1498420105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:36.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:35 smithi006 bash[27292]: audit 2023-08-28T01:20:35.474195+0000 mon.c (mon.2) 2376 : audit [DBG] from='client.? 172.21.15.6:0/1498420105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:36.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:35 smithi006 bash[33918]: audit 2023-08-28T01:20:35.474195+0000 mon.c (mon.2) 2376 : audit [DBG] from='client.? 172.21.15.6:0/1498420105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:36.805 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:36 smithi130 bash[51302]: debug 2023-08-28T01:20:36.456+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:36.462202+0000) 2023-08-28T01:20:37.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:36 smithi130 bash[27579]: cluster 2023-08-28T01:20:35.943466+0000 mgr.y (mgr.14901) 6060 : cluster [DBG] pgmap v6067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:37.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:36 smithi006 bash[27292]: cluster 2023-08-28T01:20:35.943466+0000 mgr.y (mgr.14901) 6060 : cluster [DBG] pgmap v6067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:37.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:36 smithi006 bash[33918]: cluster 2023-08-28T01:20:35.943466+0000 mgr.y (mgr.14901) 6060 : cluster [DBG] pgmap v6067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:37.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:37 smithi130 bash[51302]: debug 2023-08-28T01:20:37.456+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:37.462488+0000) 2023-08-28T01:20:38.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:37 smithi006 bash[33918]: audit 2023-08-28T01:20:37.884908+0000 mon.c (mon.2) 2377 : audit [DBG] from='client.? 172.21.15.6:0/3684600772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:38.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:37 smithi006 bash[27292]: audit 2023-08-28T01:20:37.884908+0000 mon.c (mon.2) 2377 : audit [DBG] from='client.? 172.21.15.6:0/3684600772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:38.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:37 smithi130 bash[27579]: audit 2023-08-28T01:20:37.884908+0000 mon.c (mon.2) 2377 : audit [DBG] from='client.? 172.21.15.6:0/3684600772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:38.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:38 smithi130 bash[51302]: debug 2023-08-28T01:20:38.460+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:38.462848+0000) 2023-08-28T01:20:39.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:38 smithi006 bash[33918]: cluster 2023-08-28T01:20:37.944043+0000 mgr.y (mgr.14901) 6061 : cluster [DBG] pgmap v6068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:39.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:38 smithi006 bash[27292]: cluster 2023-08-28T01:20:37.944043+0000 mgr.y (mgr.14901) 6061 : cluster [DBG] pgmap v6068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:39.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:38 smithi130 bash[27579]: cluster 2023-08-28T01:20:37.944043+0000 mgr.y (mgr.14901) 6061 : cluster [DBG] pgmap v6068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:39.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:39 smithi130 bash[51302]: debug 2023-08-28T01:20:39.460+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:39.463140+0000) 2023-08-28T01:20:40.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:40 smithi130 bash[51302]: debug 2023-08-28T01:20:40.460+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:40.463491+0000) 2023-08-28T01:20:41.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:20:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:20:40] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:20:41.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:40 smithi006 bash[27292]: cluster 2023-08-28T01:20:39.945216+0000 mgr.y (mgr.14901) 6062 : cluster [DBG] pgmap v6069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:41.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:40 smithi006 bash[27292]: audit 2023-08-28T01:20:40.301936+0000 mon.c (mon.2) 2378 : audit [DBG] from='client.? 172.21.15.6:0/468738858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:41.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:40 smithi006 bash[33918]: cluster 2023-08-28T01:20:39.945216+0000 mgr.y (mgr.14901) 6062 : cluster [DBG] pgmap v6069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:41.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:40 smithi006 bash[33918]: audit 2023-08-28T01:20:40.301936+0000 mon.c (mon.2) 2378 : audit [DBG] from='client.? 172.21.15.6:0/468738858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:20:40.988Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:41.217 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:20:40.990Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:41.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:40 smithi130 bash[27579]: cluster 2023-08-28T01:20:39.945216+0000 mgr.y (mgr.14901) 6062 : cluster [DBG] pgmap v6069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:41.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:40 smithi130 bash[27579]: audit 2023-08-28T01:20:40.301936+0000 mon.c (mon.2) 2378 : audit [DBG] from='client.? 172.21.15.6:0/468738858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:41.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:41 smithi130 bash[51302]: debug 2023-08-28T01:20:41.460+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:41.463828+0000) 2023-08-28T01:20:42.782 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:42 smithi130 bash[51302]: debug 2023-08-28T01:20:42.460+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:42.464274+0000) 2023-08-28T01:20:42.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:42 smithi006 bash[33918]: cluster 2023-08-28T01:20:41.946028+0000 mgr.y (mgr.14901) 6063 : cluster [DBG] pgmap v6070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:42.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:42 smithi006 bash[33918]: audit 2023-08-28T01:20:42.722262+0000 mon.c (mon.2) 2379 : audit [DBG] from='client.? 172.21.15.6:0/2241255329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:42 smithi006 bash[27292]: cluster 2023-08-28T01:20:41.946028+0000 mgr.y (mgr.14901) 6063 : cluster [DBG] pgmap v6070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:42.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:42 smithi006 bash[27292]: audit 2023-08-28T01:20:42.722262+0000 mon.c (mon.2) 2379 : audit [DBG] from='client.? 172.21.15.6:0/2241255329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:20:42.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:20:42.563Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:43.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:42 smithi130 bash[27579]: cluster 2023-08-28T01:20:41.946028+0000 mgr.y (mgr.14901) 6063 : cluster [DBG] pgmap v6070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:43.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:42 smithi130 bash[27579]: audit 2023-08-28T01:20:42.722262+0000 mon.c (mon.2) 2379 : audit [DBG] from='client.? 172.21.15.6:0/2241255329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:43.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:43 smithi130 bash[51302]: debug 2023-08-28T01:20:43.460+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:43.464621+0000) 2023-08-28T01:20:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:20:44.441Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:44.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:20:44.443Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:44.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:44 smithi130 bash[51302]: debug 2023-08-28T01:20:44.460+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:44.464923+0000) 2023-08-28T01:20:45.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:45 smithi130 bash[27579]: cluster 2023-08-28T01:20:43.946883+0000 mgr.y (mgr.14901) 6064 : cluster [DBG] pgmap v6071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:45.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:45 smithi006 bash[27292]: cluster 2023-08-28T01:20:43.946883+0000 mgr.y (mgr.14901) 6064 : cluster [DBG] pgmap v6071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:45.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:45 smithi006 bash[33918]: cluster 2023-08-28T01:20:43.946883+0000 mgr.y (mgr.14901) 6064 : cluster [DBG] pgmap v6071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:45.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:45 smithi130 bash[51302]: debug 2023-08-28T01:20:45.460+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:45.465202+0000) 2023-08-28T01:20:46.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:46 smithi130 bash[27579]: audit 2023-08-28T01:20:45.139659+0000 mon.a (mon.0) 4074 : audit [DBG] from='client.? 172.21.15.6:0/1829808623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:46.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:46 smithi006 bash[27292]: audit 2023-08-28T01:20:45.139659+0000 mon.a (mon.0) 4074 : audit [DBG] from='client.? 172.21.15.6:0/1829808623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:46.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:46 smithi006 bash[33918]: audit 2023-08-28T01:20:45.139659+0000 mon.a (mon.0) 4074 : audit [DBG] from='client.? 172.21.15.6:0/1829808623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:46.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:46 smithi130 bash[51302]: debug 2023-08-28T01:20:46.460+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:46.465511+0000) 2023-08-28T01:20:47.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:47 smithi130 bash[27579]: cluster 2023-08-28T01:20:45.948190+0000 mgr.y (mgr.14901) 6065 : cluster [DBG] pgmap v6072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:47.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:47 smithi006 bash[33918]: cluster 2023-08-28T01:20:45.948190+0000 mgr.y (mgr.14901) 6065 : cluster [DBG] pgmap v6072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:47.520 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:47 smithi006 bash[27292]: cluster 2023-08-28T01:20:45.948190+0000 mgr.y (mgr.14901) 6065 : cluster [DBG] pgmap v6072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:47.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:47 smithi130 bash[51302]: debug 2023-08-28T01:20:47.460+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:47.465785+0000) 2023-08-28T01:20:48.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:48 smithi130 bash[27579]: audit 2023-08-28T01:20:47.554465+0000 mon.c (mon.2) 2380 : audit [DBG] from='client.? 172.21.15.6:0/398044208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:48.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:48 smithi006 bash[33918]: audit 2023-08-28T01:20:47.554465+0000 mon.c (mon.2) 2380 : audit [DBG] from='client.? 172.21.15.6:0/398044208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:48.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:48 smithi006 bash[27292]: audit 2023-08-28T01:20:47.554465+0000 mon.c (mon.2) 2380 : audit [DBG] from='client.? 172.21.15.6:0/398044208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:48.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:48 smithi130 bash[51302]: debug 2023-08-28T01:20:48.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:48.466083+0000) 2023-08-28T01:20:49.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:49 smithi130 bash[27579]: cluster 2023-08-28T01:20:47.949071+0000 mgr.y (mgr.14901) 6066 : cluster [DBG] pgmap v6073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:49.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:49 smithi006 bash[27292]: cluster 2023-08-28T01:20:47.949071+0000 mgr.y (mgr.14901) 6066 : cluster [DBG] pgmap v6073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:49.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:49 smithi006 bash[33918]: cluster 2023-08-28T01:20:47.949071+0000 mgr.y (mgr.14901) 6066 : cluster [DBG] pgmap v6073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:49.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:49 smithi130 bash[51302]: debug 2023-08-28T01:20:49.460+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:49.466394+0000) 2023-08-28T01:20:50.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:50 smithi130 bash[27579]: audit 2023-08-28T01:20:49.967405+0000 mon.a (mon.0) 4075 : audit [DBG] from='client.? 172.21.15.6:0/4287500967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:50.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:50 smithi006 bash[33918]: audit 2023-08-28T01:20:49.967405+0000 mon.a (mon.0) 4075 : audit [DBG] from='client.? 172.21.15.6:0/4287500967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:50.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:50 smithi006 bash[27292]: audit 2023-08-28T01:20:49.967405+0000 mon.a (mon.0) 4075 : audit [DBG] from='client.? 172.21.15.6:0/4287500967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:50.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:50 smithi130 bash[51302]: debug 2023-08-28T01:20:50.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:50.466684+0000) 2023-08-28T01:20:51.036 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:20:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:20:50] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:20:51.037 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:20:50.989Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:51.037 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:20:50.990Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:51.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:51 smithi130 bash[27579]: cluster 2023-08-28T01:20:49.950285+0000 mgr.y (mgr.14901) 6067 : cluster [DBG] pgmap v6074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:51.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:51 smithi006 bash[27292]: cluster 2023-08-28T01:20:49.950285+0000 mgr.y (mgr.14901) 6067 : cluster [DBG] pgmap v6074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:51.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:51 smithi006 bash[33918]: cluster 2023-08-28T01:20:49.950285+0000 mgr.y (mgr.14901) 6067 : cluster [DBG] pgmap v6074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:51.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:51 smithi130 bash[51302]: debug 2023-08-28T01:20:51.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:51.467012+0000) 2023-08-28T01:20:52.786 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:52 smithi130 bash[51302]: debug 2023-08-28T01:20:52.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:52.467404+0000) 2023-08-28T01:20:52.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:52 smithi006 bash[33918]: cluster 2023-08-28T01:20:51.950979+0000 mgr.y (mgr.14901) 6068 : cluster [DBG] pgmap v6075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:52 smithi006 bash[33918]: audit 2023-08-28T01:20:52.379488+0000 mon.c (mon.2) 2381 : audit [DBG] from='client.? 172.21.15.6:0/283914840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:20:52.562Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:20:52.564Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:52 smithi006 bash[27292]: cluster 2023-08-28T01:20:51.950979+0000 mgr.y (mgr.14901) 6068 : cluster [DBG] pgmap v6075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:52 smithi006 bash[27292]: audit 2023-08-28T01:20:52.379488+0000 mon.c (mon.2) 2381 : audit [DBG] from='client.? 172.21.15.6:0/283914840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:52 smithi130 bash[27579]: cluster 2023-08-28T01:20:51.950979+0000 mgr.y (mgr.14901) 6068 : cluster [DBG] pgmap v6075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:52 smithi130 bash[27579]: audit 2023-08-28T01:20:52.379488+0000 mon.c (mon.2) 2381 : audit [DBG] from='client.? 172.21.15.6:0/283914840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:53.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:53 smithi130 bash[51302]: debug 2023-08-28T01:20:53.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:53.467720+0000) 2023-08-28T01:20:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:20:54.442Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:20:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:20:54.444Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:20:54.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:54 smithi130 bash[51302]: debug 2023-08-28T01:20:54.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:54.468021+0000) 2023-08-28T01:20:55.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:55 smithi130 bash[27579]: cluster 2023-08-28T01:20:53.951625+0000 mgr.y (mgr.14901) 6069 : cluster [DBG] pgmap v6076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:55.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:55 smithi130 bash[27579]: audit 2023-08-28T01:20:54.804431+0000 mon.a (mon.0) 4076 : audit [DBG] from='client.? 172.21.15.6:0/728210868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:55.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:55 smithi006 bash[33918]: cluster 2023-08-28T01:20:53.951625+0000 mgr.y (mgr.14901) 6069 : cluster [DBG] pgmap v6076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:55.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:55 smithi006 bash[33918]: audit 2023-08-28T01:20:54.804431+0000 mon.a (mon.0) 4076 : audit [DBG] from='client.? 172.21.15.6:0/728210868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:55.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:55 smithi006 bash[27292]: cluster 2023-08-28T01:20:53.951625+0000 mgr.y (mgr.14901) 6069 : cluster [DBG] pgmap v6076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:55.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:55 smithi006 bash[27292]: audit 2023-08-28T01:20:54.804431+0000 mon.a (mon.0) 4076 : audit [DBG] from='client.? 172.21.15.6:0/728210868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:55.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:55 smithi130 bash[51302]: debug 2023-08-28T01:20:55.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:55.468285+0000) 2023-08-28T01:20:56.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:56 smithi130 bash[51302]: debug 2023-08-28T01:20:56.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:56.468561+0000) 2023-08-28T01:20:57.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:57 smithi130 bash[27579]: cluster 2023-08-28T01:20:55.952808+0000 mgr.y (mgr.14901) 6070 : cluster [DBG] pgmap v6077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:57.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:57 smithi006 bash[27292]: cluster 2023-08-28T01:20:55.952808+0000 mgr.y (mgr.14901) 6070 : cluster [DBG] pgmap v6077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:57.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:57 smithi006 bash[33918]: cluster 2023-08-28T01:20:55.952808+0000 mgr.y (mgr.14901) 6070 : cluster [DBG] pgmap v6077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:57.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:57 smithi130 bash[51302]: debug 2023-08-28T01:20:57.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:57.468832+0000) 2023-08-28T01:20:58.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:58 smithi130 bash[27579]: audit 2023-08-28T01:20:57.219268+0000 mon.c (mon.2) 2382 : audit [DBG] from='client.? 172.21.15.6:0/2630661260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:58.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:58 smithi006 bash[27292]: audit 2023-08-28T01:20:57.219268+0000 mon.c (mon.2) 2382 : audit [DBG] from='client.? 172.21.15.6:0/2630661260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:58.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:58 smithi006 bash[33918]: audit 2023-08-28T01:20:57.219268+0000 mon.c (mon.2) 2382 : audit [DBG] from='client.? 172.21.15.6:0/2630661260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:58.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:58 smithi130 bash[51302]: debug 2023-08-28T01:20:58.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:58.469082+0000) 2023-08-28T01:20:59.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:20:59 smithi130 bash[27579]: cluster 2023-08-28T01:20:57.953443+0000 mgr.y (mgr.14901) 6071 : cluster [DBG] pgmap v6078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:59.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:20:59 smithi006 bash[33918]: cluster 2023-08-28T01:20:57.953443+0000 mgr.y (mgr.14901) 6071 : cluster [DBG] pgmap v6078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:59.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:20:59 smithi006 bash[27292]: cluster 2023-08-28T01:20:57.953443+0000 mgr.y (mgr.14901) 6071 : cluster [DBG] pgmap v6078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:20:59.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:20:59 smithi130 bash[51302]: debug 2023-08-28T01:20:59.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:59.469331+0000) 2023-08-28T01:21:00.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:00 smithi130 bash[27579]: audit 2023-08-28T01:20:59.637107+0000 mon.c (mon.2) 2383 : audit [DBG] from='client.? 172.21.15.6:0/273014098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:00.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:00 smithi006 bash[27292]: audit 2023-08-28T01:20:59.637107+0000 mon.c (mon.2) 2383 : audit [DBG] from='client.? 172.21.15.6:0/273014098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:00.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:00 smithi006 bash[33918]: audit 2023-08-28T01:20:59.637107+0000 mon.c (mon.2) 2383 : audit [DBG] from='client.? 172.21.15.6:0/273014098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:00.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:00 smithi130 bash[51302]: debug 2023-08-28T01:21:00.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:00.469645+0000) 2023-08-28T01:21:01.037 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:21:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:21:00] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:21:01.038 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:21:00.990Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:01.038 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:21:00.992Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:01.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:01 smithi130 bash[27579]: cluster 2023-08-28T01:20:59.954569+0000 mgr.y (mgr.14901) 6072 : cluster [DBG] pgmap v6079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:01.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:01 smithi006 bash[33918]: cluster 2023-08-28T01:20:59.954569+0000 mgr.y (mgr.14901) 6072 : cluster [DBG] pgmap v6079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:01.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:01 smithi006 bash[27292]: cluster 2023-08-28T01:20:59.954569+0000 mgr.y (mgr.14901) 6072 : cluster [DBG] pgmap v6079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:01.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:01 smithi130 bash[51302]: debug 2023-08-28T01:21:01.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:01.469904+0000) 2023-08-28T01:21:02.787 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:02 smithi130 bash[51302]: debug 2023-08-28T01:21:02.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:02.470168+0000) 2023-08-28T01:21:02.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:02 smithi006 bash[27292]: cluster 2023-08-28T01:21:01.955189+0000 mgr.y (mgr.14901) 6073 : cluster [DBG] pgmap v6080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:02 smithi006 bash[27292]: audit 2023-08-28T01:21:02.055734+0000 mon.a (mon.0) 4077 : audit [DBG] from='client.? 172.21.15.6:0/833175076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:02 smithi006 bash[33918]: cluster 2023-08-28T01:21:01.955189+0000 mgr.y (mgr.14901) 6073 : cluster [DBG] pgmap v6080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:02 smithi006 bash[33918]: audit 2023-08-28T01:21:02.055734+0000 mon.a (mon.0) 4077 : audit [DBG] from='client.? 172.21.15.6:0/833175076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:02.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:21:02.563Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:02.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:21:02.565Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:02 smithi130 bash[27579]: cluster 2023-08-28T01:21:01.955189+0000 mgr.y (mgr.14901) 6073 : cluster [DBG] pgmap v6080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:02 smithi130 bash[27579]: audit 2023-08-28T01:21:02.055734+0000 mon.a (mon.0) 4077 : audit [DBG] from='client.? 172.21.15.6:0/833175076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:03.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:03 smithi130 bash[51302]: debug 2023-08-28T01:21:03.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:03.470445+0000) 2023-08-28T01:21:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:21:04.443Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:21:04.445Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:04.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:04 smithi130 bash[51302]: debug 2023-08-28T01:21:04.464+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:04.470733+0000) 2023-08-28T01:21:05.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:05 smithi130 bash[27579]: cluster 2023-08-28T01:21:03.955841+0000 mgr.y (mgr.14901) 6074 : cluster [DBG] pgmap v6081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:05.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:05 smithi130 bash[27579]: audit 2023-08-28T01:21:04.466064+0000 mon.a (mon.0) 4078 : audit [DBG] from='client.? 172.21.15.6:0/2855691744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:05.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:05 smithi006 bash[27292]: cluster 2023-08-28T01:21:03.955841+0000 mgr.y (mgr.14901) 6074 : cluster [DBG] pgmap v6081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:05.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:05 smithi006 bash[27292]: audit 2023-08-28T01:21:04.466064+0000 mon.a (mon.0) 4078 : audit [DBG] from='client.? 172.21.15.6:0/2855691744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:05.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:05 smithi006 bash[33918]: cluster 2023-08-28T01:21:03.955841+0000 mgr.y (mgr.14901) 6074 : cluster [DBG] pgmap v6081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:05.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:05 smithi006 bash[33918]: audit 2023-08-28T01:21:04.466064+0000 mon.a (mon.0) 4078 : audit [DBG] from='client.? 172.21.15.6:0/2855691744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:05.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:05 smithi130 bash[51302]: debug 2023-08-28T01:21:05.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:05.471079+0000) 2023-08-28T01:21:06.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:06 smithi130 bash[51302]: debug 2023-08-28T01:21:06.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:06.471340+0000) 2023-08-28T01:21:07.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:07 smithi130 bash[27579]: cluster 2023-08-28T01:21:05.956984+0000 mgr.y (mgr.14901) 6075 : cluster [DBG] pgmap v6082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:07.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:07 smithi130 bash[27579]: audit 2023-08-28T01:21:06.878373+0000 mon.a (mon.0) 4079 : audit [DBG] from='client.? 172.21.15.6:0/753279973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:07 smithi006 bash[27292]: cluster 2023-08-28T01:21:05.956984+0000 mgr.y (mgr.14901) 6075 : cluster [DBG] pgmap v6082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:07 smithi006 bash[27292]: audit 2023-08-28T01:21:06.878373+0000 mon.a (mon.0) 4079 : audit [DBG] from='client.? 172.21.15.6:0/753279973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:07.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:07 smithi006 bash[33918]: cluster 2023-08-28T01:21:05.956984+0000 mgr.y (mgr.14901) 6075 : cluster [DBG] pgmap v6082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:07.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:07 smithi006 bash[33918]: audit 2023-08-28T01:21:06.878373+0000 mon.a (mon.0) 4079 : audit [DBG] from='client.? 172.21.15.6:0/753279973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:07.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:07 smithi130 bash[51302]: debug 2023-08-28T01:21:07.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:07.471648+0000) 2023-08-28T01:21:08.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:08 smithi130 bash[51302]: debug 2023-08-28T01:21:08.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:08.471928+0000) 2023-08-28T01:21:09.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:09 smithi130 bash[27579]: cluster 2023-08-28T01:21:07.957571+0000 mgr.y (mgr.14901) 6076 : cluster [DBG] pgmap v6083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:09.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:09 smithi006 bash[27292]: cluster 2023-08-28T01:21:07.957571+0000 mgr.y (mgr.14901) 6076 : cluster [DBG] pgmap v6083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:09.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:09 smithi006 bash[33918]: cluster 2023-08-28T01:21:07.957571+0000 mgr.y (mgr.14901) 6076 : cluster [DBG] pgmap v6083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:09.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:09 smithi130 bash[51302]: debug 2023-08-28T01:21:09.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:09.472229+0000) 2023-08-28T01:21:10.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:10 smithi130 bash[27579]: audit 2023-08-28T01:21:09.289140+0000 mon.a (mon.0) 4080 : audit [DBG] from='client.? 172.21.15.6:0/2932746421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:10.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:10 smithi006 bash[27292]: audit 2023-08-28T01:21:09.289140+0000 mon.a (mon.0) 4080 : audit [DBG] from='client.? 172.21.15.6:0/2932746421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:10.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:10 smithi006 bash[33918]: audit 2023-08-28T01:21:09.289140+0000 mon.a (mon.0) 4080 : audit [DBG] from='client.? 172.21.15.6:0/2932746421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:10.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:10 smithi130 bash[51302]: debug 2023-08-28T01:21:10.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:10.472538+0000) 2023-08-28T01:21:11.046 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:21:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:21:10] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:21:11.046 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:21:10.990Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:11.047 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:21:10.992Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:11.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:11 smithi130 bash[27579]: cluster 2023-08-28T01:21:09.958697+0000 mgr.y (mgr.14901) 6077 : cluster [DBG] pgmap v6084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:11.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:11 smithi006 bash[33918]: cluster 2023-08-28T01:21:09.958697+0000 mgr.y (mgr.14901) 6077 : cluster [DBG] pgmap v6084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:11.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:11 smithi006 bash[27292]: cluster 2023-08-28T01:21:09.958697+0000 mgr.y (mgr.14901) 6077 : cluster [DBG] pgmap v6084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:11.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:11 smithi130 bash[51302]: debug 2023-08-28T01:21:11.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:11.472809+0000) 2023-08-28T01:21:12.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:12 smithi130 bash[27579]: audit 2023-08-28T01:21:11.698453+0000 mon.a (mon.0) 4081 : audit [DBG] from='client.? 172.21.15.6:0/1542179879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:12.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:12 smithi006 bash[27292]: audit 2023-08-28T01:21:11.698453+0000 mon.a (mon.0) 4081 : audit [DBG] from='client.? 172.21.15.6:0/1542179879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:12.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:12 smithi006 bash[33918]: audit 2023-08-28T01:21:11.698453+0000 mon.a (mon.0) 4081 : audit [DBG] from='client.? 172.21.15.6:0/1542179879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:12.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:12 smithi130 bash[51302]: debug 2023-08-28T01:21:12.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:12.473062+0000) 2023-08-28T01:21:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:21:12.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:21:12.566Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:13.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:13 smithi130 bash[27579]: cluster 2023-08-28T01:21:11.959348+0000 mgr.y (mgr.14901) 6078 : cluster [DBG] pgmap v6085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:13.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:13 smithi006 bash[27292]: cluster 2023-08-28T01:21:11.959348+0000 mgr.y (mgr.14901) 6078 : cluster [DBG] pgmap v6085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:13.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:13 smithi006 bash[33918]: cluster 2023-08-28T01:21:11.959348+0000 mgr.y (mgr.14901) 6078 : cluster [DBG] pgmap v6085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:13.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:13 smithi130 bash[51302]: debug 2023-08-28T01:21:13.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:13.473335+0000) 2023-08-28T01:21:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:21:14.444Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:21:14.446Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:14.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:14 smithi130 bash[51302]: debug 2023-08-28T01:21:14.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:14.473632+0000) 2023-08-28T01:21:15.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:15 smithi130 bash[27579]: cluster 2023-08-28T01:21:13.959973+0000 mgr.y (mgr.14901) 6079 : cluster [DBG] pgmap v6086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:15.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:15 smithi130 bash[27579]: audit 2023-08-28T01:21:14.108212+0000 mon.a (mon.0) 4082 : audit [DBG] from='client.? 172.21.15.6:0/2340577535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:15.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:15 smithi006 bash[27292]: cluster 2023-08-28T01:21:13.959973+0000 mgr.y (mgr.14901) 6079 : cluster [DBG] pgmap v6086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:15.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:15 smithi006 bash[27292]: audit 2023-08-28T01:21:14.108212+0000 mon.a (mon.0) 4082 : audit [DBG] from='client.? 172.21.15.6:0/2340577535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:15.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:15 smithi006 bash[33918]: cluster 2023-08-28T01:21:13.959973+0000 mgr.y (mgr.14901) 6079 : cluster [DBG] pgmap v6086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:15.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:15 smithi006 bash[33918]: audit 2023-08-28T01:21:14.108212+0000 mon.a (mon.0) 4082 : audit [DBG] from='client.? 172.21.15.6:0/2340577535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:15.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:15 smithi130 bash[51302]: debug 2023-08-28T01:21:15.472+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:15.473933+0000) 2023-08-28T01:21:16.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:16 smithi130 bash[51302]: debug 2023-08-28T01:21:16.468+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:16.474211+0000) 2023-08-28T01:21:17.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:17 smithi130 bash[27579]: cluster 2023-08-28T01:21:15.961209+0000 mgr.y (mgr.14901) 6080 : cluster [DBG] pgmap v6087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:17.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:17 smithi130 bash[27579]: audit 2023-08-28T01:21:16.520006+0000 mon.c (mon.2) 2384 : audit [DBG] from='client.? 172.21.15.6:0/777531408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:17.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:17 smithi006 bash[27292]: cluster 2023-08-28T01:21:15.961209+0000 mgr.y (mgr.14901) 6080 : cluster [DBG] pgmap v6087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:17.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:17 smithi006 bash[27292]: audit 2023-08-28T01:21:16.520006+0000 mon.c (mon.2) 2384 : audit [DBG] from='client.? 172.21.15.6:0/777531408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:17.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:17 smithi006 bash[33918]: cluster 2023-08-28T01:21:15.961209+0000 mgr.y (mgr.14901) 6080 : cluster [DBG] pgmap v6087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:17.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:17 smithi006 bash[33918]: audit 2023-08-28T01:21:16.520006+0000 mon.c (mon.2) 2384 : audit [DBG] from='client.? 172.21.15.6:0/777531408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:17.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:17 smithi130 bash[51302]: debug 2023-08-28T01:21:17.467+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:17.474519+0000) 2023-08-28T01:21:18.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:18 smithi130 bash[51302]: debug 2023-08-28T01:21:18.467+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:18.474784+0000) 2023-08-28T01:21:19.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:19 smithi130 bash[27579]: cluster 2023-08-28T01:21:17.962008+0000 mgr.y (mgr.14901) 6081 : cluster [DBG] pgmap v6088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:19.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:19 smithi130 bash[27579]: audit 2023-08-28T01:21:18.934252+0000 mon.a (mon.0) 4083 : audit [DBG] from='client.? 172.21.15.6:0/1014077526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:19.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:19 smithi006 bash[33918]: cluster 2023-08-28T01:21:17.962008+0000 mgr.y (mgr.14901) 6081 : cluster [DBG] pgmap v6088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:19.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:19 smithi006 bash[33918]: audit 2023-08-28T01:21:18.934252+0000 mon.a (mon.0) 4083 : audit [DBG] from='client.? 172.21.15.6:0/1014077526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:19.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:19 smithi006 bash[27292]: cluster 2023-08-28T01:21:17.962008+0000 mgr.y (mgr.14901) 6081 : cluster [DBG] pgmap v6088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:19.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:19 smithi006 bash[27292]: audit 2023-08-28T01:21:18.934252+0000 mon.a (mon.0) 4083 : audit [DBG] from='client.? 172.21.15.6:0/1014077526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:19.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:19 smithi130 bash[51302]: debug 2023-08-28T01:21:19.471+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:19.475079+0000) 2023-08-28T01:21:20.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:20 smithi130 bash[51302]: debug 2023-08-28T01:21:20.471+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:20.475389+0000) 2023-08-28T01:21:21.072 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:21:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:21:20] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:21:21.072 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:21:20.991Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:21.072 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:21:20.993Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:21.072 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:21 smithi006 bash[27292]: cluster 2023-08-28T01:21:19.963283+0000 mgr.y (mgr.14901) 6082 : cluster [DBG] pgmap v6089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:21.073 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:21 smithi006 bash[33918]: cluster 2023-08-28T01:21:19.963283+0000 mgr.y (mgr.14901) 6082 : cluster [DBG] pgmap v6089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:21.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:21 smithi130 bash[27579]: cluster 2023-08-28T01:21:19.963283+0000 mgr.y (mgr.14901) 6082 : cluster [DBG] pgmap v6089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:21.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:21 smithi130 bash[51302]: debug 2023-08-28T01:21:21.471+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:21.475661+0000) 2023-08-28T01:21:22.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:22 smithi130 bash[27579]: audit 2023-08-28T01:21:21.345103+0000 mon.c (mon.2) 2385 : audit [DBG] from='client.? 172.21.15.6:0/1234628074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:22.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:22 smithi006 bash[27292]: audit 2023-08-28T01:21:21.345103+0000 mon.c (mon.2) 2385 : audit [DBG] from='client.? 172.21.15.6:0/1234628074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:22.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:22 smithi006 bash[33918]: audit 2023-08-28T01:21:21.345103+0000 mon.c (mon.2) 2385 : audit [DBG] from='client.? 172.21.15.6:0/1234628074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:22.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:22 smithi130 bash[51302]: debug 2023-08-28T01:21:22.471+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:22.475885+0000) 2023-08-28T01:21:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:21:22.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:21:22.566Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:23.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:23 smithi130 bash[27579]: cluster 2023-08-28T01:21:21.964119+0000 mgr.y (mgr.14901) 6083 : cluster [DBG] pgmap v6090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:23.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:23 smithi006 bash[27292]: cluster 2023-08-28T01:21:21.964119+0000 mgr.y (mgr.14901) 6083 : cluster [DBG] pgmap v6090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:23.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:23 smithi006 bash[33918]: cluster 2023-08-28T01:21:21.964119+0000 mgr.y (mgr.14901) 6083 : cluster [DBG] pgmap v6090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:23.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:23 smithi130 bash[51302]: debug 2023-08-28T01:21:23.471+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:23.476168+0000) 2023-08-28T01:21:24.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:24 smithi130 bash[27579]: audit 2023-08-28T01:21:23.760082+0000 mon.a (mon.0) 4084 : audit [DBG] from='client.? 172.21.15.6:0/958108882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:24.445 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:24 smithi006 bash[27292]: audit 2023-08-28T01:21:23.760082+0000 mon.a (mon.0) 4084 : audit [DBG] from='client.? 172.21.15.6:0/958108882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:24.446 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:24 smithi006 bash[33918]: audit 2023-08-28T01:21:23.760082+0000 mon.a (mon.0) 4084 : audit [DBG] from='client.? 172.21.15.6:0/958108882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:21:24.444Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:21:24.446Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:24.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:24 smithi130 bash[51302]: debug 2023-08-28T01:21:24.471+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:24.476438+0000) 2023-08-28T01:21:25.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:25 smithi130 bash[27579]: cluster 2023-08-28T01:21:23.965001+0000 mgr.y (mgr.14901) 6084 : cluster [DBG] pgmap v6091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:25.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:25 smithi006 bash[33918]: cluster 2023-08-28T01:21:23.965001+0000 mgr.y (mgr.14901) 6084 : cluster [DBG] pgmap v6091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:25.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:25 smithi006 bash[27292]: cluster 2023-08-28T01:21:23.965001+0000 mgr.y (mgr.14901) 6084 : cluster [DBG] pgmap v6091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:25.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:25 smithi130 bash[51302]: debug 2023-08-28T01:21:25.471+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:25.476697+0000) 2023-08-28T01:21:26.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:26 smithi130 bash[51302]: debug 2023-08-28T01:21:26.471+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:26.476990+0000) 2023-08-28T01:21:27.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:27 smithi130 bash[27579]: cluster 2023-08-28T01:21:25.966123+0000 mgr.y (mgr.14901) 6085 : cluster [DBG] pgmap v6092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:27.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:27 smithi130 bash[27579]: audit 2023-08-28T01:21:26.185657+0000 mon.a (mon.0) 4085 : audit [DBG] from='client.? 172.21.15.6:0/2239453530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:27.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:27 smithi130 bash[27579]: audit 2023-08-28T01:21:26.481313+0000 mon.b (mon.1) 701 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:21:27.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:27 smithi006 bash[27292]: cluster 2023-08-28T01:21:25.966123+0000 mgr.y (mgr.14901) 6085 : cluster [DBG] pgmap v6092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:27.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:27 smithi006 bash[27292]: audit 2023-08-28T01:21:26.185657+0000 mon.a (mon.0) 4085 : audit [DBG] from='client.? 172.21.15.6:0/2239453530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:27.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:27 smithi006 bash[27292]: audit 2023-08-28T01:21:26.481313+0000 mon.b (mon.1) 701 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:21:27.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:27 smithi006 bash[33918]: cluster 2023-08-28T01:21:25.966123+0000 mgr.y (mgr.14901) 6085 : cluster [DBG] pgmap v6092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:27.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:27 smithi006 bash[33918]: audit 2023-08-28T01:21:26.185657+0000 mon.a (mon.0) 4085 : audit [DBG] from='client.? 172.21.15.6:0/2239453530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:27.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:27 smithi006 bash[33918]: audit 2023-08-28T01:21:26.481313+0000 mon.b (mon.1) 701 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:21:27.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:27 smithi130 bash[51302]: debug 2023-08-28T01:21:27.471+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:27.477207+0000) 2023-08-28T01:21:28.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:28 smithi130 bash[51302]: debug 2023-08-28T01:21:28.471+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:28.477520+0000) 2023-08-28T01:21:29.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:29 smithi130 bash[27579]: cluster 2023-08-28T01:21:27.966919+0000 mgr.y (mgr.14901) 6086 : cluster [DBG] pgmap v6093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:29.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:29 smithi130 bash[27579]: audit 2023-08-28T01:21:28.594626+0000 mon.c (mon.2) 2386 : audit [DBG] from='client.? 172.21.15.6:0/560691034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:29.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:29 smithi006 bash[27292]: cluster 2023-08-28T01:21:27.966919+0000 mgr.y (mgr.14901) 6086 : cluster [DBG] pgmap v6093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:29.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:29 smithi006 bash[27292]: audit 2023-08-28T01:21:28.594626+0000 mon.c (mon.2) 2386 : audit [DBG] from='client.? 172.21.15.6:0/560691034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:29.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:29 smithi006 bash[33918]: cluster 2023-08-28T01:21:27.966919+0000 mgr.y (mgr.14901) 6086 : cluster [DBG] pgmap v6093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:29.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:29 smithi006 bash[33918]: audit 2023-08-28T01:21:28.594626+0000 mon.c (mon.2) 2386 : audit [DBG] from='client.? 172.21.15.6:0/560691034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:29.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:29 smithi130 bash[51302]: debug 2023-08-28T01:21:29.471+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:29.477803+0000) 2023-08-28T01:21:30.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:30 smithi006 bash[33918]: cluster 2023-08-28T01:21:29.968137+0000 mgr.y (mgr.14901) 6087 : cluster [DBG] pgmap v6094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:30.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:30 smithi006 bash[27292]: cluster 2023-08-28T01:21:29.968137+0000 mgr.y (mgr.14901) 6087 : cluster [DBG] pgmap v6094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:30.477 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:30 smithi130 bash[27579]: cluster 2023-08-28T01:21:29.968137+0000 mgr.y (mgr.14901) 6087 : cluster [DBG] pgmap v6094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:30.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:30 smithi130 bash[51302]: debug 2023-08-28T01:21:30.471+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:30.478127+0000) 2023-08-28T01:21:31.157 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:21:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:21:30] "GET /metrics HTTP/1.1" 200 193132 "" "Prometheus/2.33.4" 2023-08-28T01:21:31.157 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:21:30.991Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:31.157 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:30 smithi006 bash[56298]: level=warn ts=2023-08-28T01:21:30.993Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:31.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:31 smithi006 bash[33918]: audit 2023-08-28T01:21:31.017426+0000 mon.a (mon.0) 4086 : audit [DBG] from='client.? 172.21.15.6:0/1342363227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:31.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:31 smithi006 bash[27292]: audit 2023-08-28T01:21:31.017426+0000 mon.a (mon.0) 4086 : audit [DBG] from='client.? 172.21.15.6:0/1342363227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:31.477 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:31 smithi130 bash[27579]: audit 2023-08-28T01:21:31.017426+0000 mon.a (mon.0) 4086 : audit [DBG] from='client.? 172.21.15.6:0/1342363227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:31.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:31 smithi130 bash[51302]: debug 2023-08-28T01:21:31.471+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:31.478331+0000) 2023-08-28T01:21:32.794 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:32 smithi130 bash[51302]: debug 2023-08-28T01:21:32.471+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:32.478503+0000) 2023-08-28T01:21:32.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:32 smithi006 bash[33918]: cluster 2023-08-28T01:21:31.968753+0000 mgr.y (mgr.14901) 6088 : cluster [DBG] pgmap v6095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:32 smithi006 bash[33918]: audit 2023-08-28T01:21:32.360656+0000 mon.a (mon.0) 4087 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:21:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:32 smithi006 bash[33918]: audit 2023-08-28T01:21:32.752110+0000 mon.a (mon.0) 4088 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:21:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:21:32.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:21:32.567Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:32 smithi006 bash[27292]: cluster 2023-08-28T01:21:31.968753+0000 mgr.y (mgr.14901) 6088 : cluster [DBG] pgmap v6095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:32 smithi006 bash[27292]: audit 2023-08-28T01:21:32.360656+0000 mon.a (mon.0) 4087 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:21:32.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:32 smithi006 bash[27292]: audit 2023-08-28T01:21:32.752110+0000 mon.a (mon.0) 4088 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:21:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:32 smithi130 bash[27579]: cluster 2023-08-28T01:21:31.968753+0000 mgr.y (mgr.14901) 6088 : cluster [DBG] pgmap v6095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:32 smithi130 bash[27579]: audit 2023-08-28T01:21:32.360656+0000 mon.a (mon.0) 4087 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:21:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:32 smithi130 bash[27579]: audit 2023-08-28T01:21:32.752110+0000 mon.a (mon.0) 4088 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:21:33.798 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:33 smithi130 bash[51302]: debug 2023-08-28T01:21:33.471+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:33.478755+0000) 2023-08-28T01:21:34.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:33 smithi130 bash[27579]: audit 2023-08-28T01:21:33.428796+0000 mon.c (mon.2) 2387 : audit [DBG] from='client.? 172.21.15.6:0/3212580561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:33 smithi006 bash[27292]: audit 2023-08-28T01:21:33.428796+0000 mon.c (mon.2) 2387 : audit [DBG] from='client.? 172.21.15.6:0/3212580561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:34.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:33 smithi006 bash[33918]: audit 2023-08-28T01:21:33.428796+0000 mon.c (mon.2) 2387 : audit [DBG] from='client.? 172.21.15.6:0/3212580561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:21:34.445Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:34.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:21:34.447Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:34.804 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:34 smithi130 bash[51302]: debug 2023-08-28T01:21:34.475+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:34.479105+0000) 2023-08-28T01:21:35.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:34 smithi130 bash[27579]: cluster 2023-08-28T01:21:33.969438+0000 mgr.y (mgr.14901) 6089 : cluster [DBG] pgmap v6096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:35.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:34 smithi006 bash[33918]: cluster 2023-08-28T01:21:33.969438+0000 mgr.y (mgr.14901) 6089 : cluster [DBG] pgmap v6096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:35.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:34 smithi006 bash[27292]: cluster 2023-08-28T01:21:33.969438+0000 mgr.y (mgr.14901) 6089 : cluster [DBG] pgmap v6096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:35.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:35 smithi130 bash[51302]: debug 2023-08-28T01:21:35.475+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:35.479468+0000) 2023-08-28T01:21:36.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:35 smithi006 bash[33918]: audit 2023-08-28T01:21:35.855072+0000 mon.c (mon.2) 2388 : audit [DBG] from='client.? 172.21.15.6:0/1700826142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:36.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:35 smithi006 bash[27292]: audit 2023-08-28T01:21:35.855072+0000 mon.c (mon.2) 2388 : audit [DBG] from='client.? 172.21.15.6:0/1700826142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:36.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:35 smithi130 bash[27579]: audit 2023-08-28T01:21:35.855072+0000 mon.c (mon.2) 2388 : audit [DBG] from='client.? 172.21.15.6:0/1700826142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:36.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:36 smithi130 bash[51302]: debug 2023-08-28T01:21:36.475+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:36.479774+0000) 2023-08-28T01:21:37.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:36 smithi006 bash[33918]: cluster 2023-08-28T01:21:35.970549+0000 mgr.y (mgr.14901) 6090 : cluster [DBG] pgmap v6097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:37.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:36 smithi006 bash[27292]: cluster 2023-08-28T01:21:35.970549+0000 mgr.y (mgr.14901) 6090 : cluster [DBG] pgmap v6097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:37.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:36 smithi130 bash[27579]: cluster 2023-08-28T01:21:35.970549+0000 mgr.y (mgr.14901) 6090 : cluster [DBG] pgmap v6097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:37.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:37 smithi130 bash[51302]: debug 2023-08-28T01:21:37.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:37.480077+0000) 2023-08-28T01:21:38.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:38 smithi130 bash[51302]: debug 2023-08-28T01:21:38.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:38.480358+0000) 2023-08-28T01:21:39.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:39 smithi006 bash[33918]: cluster 2023-08-28T01:21:37.971350+0000 mgr.y (mgr.14901) 6091 : cluster [DBG] pgmap v6098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:39.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:39 smithi006 bash[33918]: audit 2023-08-28T01:21:38.258915+0000 mon.c (mon.2) 2389 : audit [DBG] from='client.? 172.21.15.6:0/2104315832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:39.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:39 smithi006 bash[27292]: cluster 2023-08-28T01:21:37.971350+0000 mgr.y (mgr.14901) 6091 : cluster [DBG] pgmap v6098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:39.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:39 smithi006 bash[27292]: audit 2023-08-28T01:21:38.258915+0000 mon.c (mon.2) 2389 : audit [DBG] from='client.? 172.21.15.6:0/2104315832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:39.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:39 smithi130 bash[27579]: cluster 2023-08-28T01:21:37.971350+0000 mgr.y (mgr.14901) 6091 : cluster [DBG] pgmap v6098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:39.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:39 smithi130 bash[27579]: audit 2023-08-28T01:21:38.258915+0000 mon.c (mon.2) 2389 : audit [DBG] from='client.? 172.21.15.6:0/2104315832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:39.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:39 smithi130 bash[51302]: debug 2023-08-28T01:21:39.475+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:39.480638+0000) 2023-08-28T01:21:40.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:40 smithi130 bash[51302]: debug 2023-08-28T01:21:40.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:40.480973+0000) 2023-08-28T01:21:41.034 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:21:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:21:40] "GET /metrics HTTP/1.1" 200 193132 "" "Prometheus/2.33.4" 2023-08-28T01:21:41.034 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:21:40.992Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:41.034 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:21:40.994Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:41.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:41 smithi130 bash[27579]: cluster 2023-08-28T01:21:39.972475+0000 mgr.y (mgr.14901) 6092 : cluster [DBG] pgmap v6099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:41.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:41 smithi130 bash[27579]: audit 2023-08-28T01:21:40.689900+0000 mon.a (mon.0) 4089 : audit [DBG] from='client.? 172.21.15.6:0/1024348566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:41.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:41 smithi006 bash[27292]: cluster 2023-08-28T01:21:39.972475+0000 mgr.y (mgr.14901) 6092 : cluster [DBG] pgmap v6099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:41.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:41 smithi006 bash[27292]: audit 2023-08-28T01:21:40.689900+0000 mon.a (mon.0) 4089 : audit [DBG] from='client.? 172.21.15.6:0/1024348566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:41.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:41 smithi006 bash[33918]: cluster 2023-08-28T01:21:39.972475+0000 mgr.y (mgr.14901) 6092 : cluster [DBG] pgmap v6099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:41.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:41 smithi006 bash[33918]: audit 2023-08-28T01:21:40.689900+0000 mon.a (mon.0) 4089 : audit [DBG] from='client.? 172.21.15.6:0/1024348566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:41.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:41 smithi130 bash[51302]: debug 2023-08-28T01:21:41.475+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:41.481212+0000) 2023-08-28T01:21:42.797 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:42 smithi130 bash[51302]: debug 2023-08-28T01:21:42.475+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:42.481431+0000) 2023-08-28T01:21:42.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:42 smithi006 bash[27292]: cluster 2023-08-28T01:21:41.973195+0000 mgr.y (mgr.14901) 6093 : cluster [DBG] pgmap v6100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:42.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:42 smithi006 bash[27292]: audit 2023-08-28T01:21:42.529917+0000 mon.a (mon.0) 4090 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:21:42.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:42 smithi006 bash[33918]: cluster 2023-08-28T01:21:41.973195+0000 mgr.y (mgr.14901) 6093 : cluster [DBG] pgmap v6100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:42.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:42 smithi006 bash[33918]: audit 2023-08-28T01:21:42.529917+0000 mon.a (mon.0) 4090 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:21:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:21:42.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:42.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:21:42.568Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:42 smithi130 bash[27579]: cluster 2023-08-28T01:21:41.973195+0000 mgr.y (mgr.14901) 6093 : cluster [DBG] pgmap v6100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:42 smithi130 bash[27579]: audit 2023-08-28T01:21:42.529917+0000 mon.a (mon.0) 4090 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:21:43.801 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:43 smithi130 bash[51302]: debug 2023-08-28T01:21:43.475+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:43.481720+0000) 2023-08-28T01:21:44.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:43 smithi130 bash[27579]: audit 2023-08-28T01:21:42.832690+0000 mon.b (mon.1) 702 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:21:44.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:43 smithi130 bash[27579]: audit 2023-08-28T01:21:42.834254+0000 mon.b (mon.1) 703 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:21:44.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:43 smithi130 bash[27579]: audit 2023-08-28T01:21:42.845668+0000 mon.a (mon.0) 4091 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:21:44.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:43 smithi130 bash[27579]: audit 2023-08-28T01:21:43.103398+0000 mon.a (mon.0) 4092 : audit [DBG] from='client.? 172.21.15.6:0/3771659443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:44.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:43 smithi006 bash[33918]: audit 2023-08-28T01:21:42.832690+0000 mon.b (mon.1) 702 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:21:44.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:43 smithi006 bash[33918]: audit 2023-08-28T01:21:42.834254+0000 mon.b (mon.1) 703 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:21:44.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:43 smithi006 bash[33918]: audit 2023-08-28T01:21:42.845668+0000 mon.a (mon.0) 4091 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:21:44.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:43 smithi006 bash[33918]: audit 2023-08-28T01:21:43.103398+0000 mon.a (mon.0) 4092 : audit [DBG] from='client.? 172.21.15.6:0/3771659443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:44.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:43 smithi006 bash[27292]: audit 2023-08-28T01:21:42.832690+0000 mon.b (mon.1) 702 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:21:44.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:43 smithi006 bash[27292]: audit 2023-08-28T01:21:42.834254+0000 mon.b (mon.1) 703 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:21:44.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:43 smithi006 bash[27292]: audit 2023-08-28T01:21:42.845668+0000 mon.a (mon.0) 4091 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:21:44.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:43 smithi006 bash[27292]: audit 2023-08-28T01:21:43.103398+0000 mon.a (mon.0) 4092 : audit [DBG] from='client.? 172.21.15.6:0/3771659443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:21:44.446Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:21:44.447Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:44.812 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:44 smithi130 bash[51302]: debug 2023-08-28T01:21:44.475+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:44.481997+0000) 2023-08-28T01:21:45.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:44 smithi130 bash[27579]: cluster 2023-08-28T01:21:43.973859+0000 mgr.y (mgr.14901) 6094 : cluster [DBG] pgmap v6101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:45.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:44 smithi006 bash[33918]: cluster 2023-08-28T01:21:43.973859+0000 mgr.y (mgr.14901) 6094 : cluster [DBG] pgmap v6101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:45.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:44 smithi006 bash[27292]: cluster 2023-08-28T01:21:43.973859+0000 mgr.y (mgr.14901) 6094 : cluster [DBG] pgmap v6101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:45.815 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:45 smithi130 bash[51302]: debug 2023-08-28T01:21:45.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:45.482308+0000) 2023-08-28T01:21:46.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:45 smithi130 bash[27579]: audit 2023-08-28T01:21:45.523555+0000 mon.c (mon.2) 2390 : audit [DBG] from='client.? 172.21.15.6:0/891529715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:46.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:45 smithi006 bash[27292]: audit 2023-08-28T01:21:45.523555+0000 mon.c (mon.2) 2390 : audit [DBG] from='client.? 172.21.15.6:0/891529715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:46.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:45 smithi006 bash[33918]: audit 2023-08-28T01:21:45.523555+0000 mon.c (mon.2) 2390 : audit [DBG] from='client.? 172.21.15.6:0/891529715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:46.830 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:46 smithi130 bash[51302]: debug 2023-08-28T01:21:46.475+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:46.482585+0000) 2023-08-28T01:21:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:46 smithi130 bash[27579]: cluster 2023-08-28T01:21:45.975029+0000 mgr.y (mgr.14901) 6095 : cluster [DBG] pgmap v6102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:47.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:46 smithi006 bash[27292]: cluster 2023-08-28T01:21:45.975029+0000 mgr.y (mgr.14901) 6095 : cluster [DBG] pgmap v6102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:47.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:46 smithi006 bash[33918]: cluster 2023-08-28T01:21:45.975029+0000 mgr.y (mgr.14901) 6095 : cluster [DBG] pgmap v6102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:47.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:47 smithi130 bash[51302]: debug 2023-08-28T01:21:47.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:47.482886+0000) 2023-08-28T01:21:48.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:47 smithi130 bash[27579]: audit 2023-08-28T01:21:47.937011+0000 mon.a (mon.0) 4093 : audit [DBG] from='client.? 172.21.15.6:0/2707743546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:48.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:47 smithi006 bash[27292]: audit 2023-08-28T01:21:47.937011+0000 mon.a (mon.0) 4093 : audit [DBG] from='client.? 172.21.15.6:0/2707743546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:48.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:47 smithi006 bash[33918]: audit 2023-08-28T01:21:47.937011+0000 mon.a (mon.0) 4093 : audit [DBG] from='client.? 172.21.15.6:0/2707743546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:48.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:48 smithi130 bash[51302]: debug 2023-08-28T01:21:48.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:48.483158+0000) 2023-08-28T01:21:49.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:49 smithi130 bash[27579]: cluster 2023-08-28T01:21:47.975669+0000 mgr.y (mgr.14901) 6096 : cluster [DBG] pgmap v6103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:49.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:49 smithi006 bash[33918]: cluster 2023-08-28T01:21:47.975669+0000 mgr.y (mgr.14901) 6096 : cluster [DBG] pgmap v6103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:49.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:49 smithi006 bash[27292]: cluster 2023-08-28T01:21:47.975669+0000 mgr.y (mgr.14901) 6096 : cluster [DBG] pgmap v6103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:49.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:49 smithi130 bash[51302]: debug 2023-08-28T01:21:49.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:49.483434+0000) 2023-08-28T01:21:50.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:50 smithi130 bash[51302]: debug 2023-08-28T01:21:50.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:50.483730+0000) 2023-08-28T01:21:51.041 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:21:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:21:50] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:21:51.041 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:21:50.992Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:51.041 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:50 smithi006 bash[56298]: level=warn ts=2023-08-28T01:21:50.995Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:51.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:51 smithi130 bash[27579]: cluster 2023-08-28T01:21:49.976647+0000 mgr.y (mgr.14901) 6097 : cluster [DBG] pgmap v6104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:51.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:51 smithi130 bash[27579]: audit 2023-08-28T01:21:50.357141+0000 mon.c (mon.2) 2391 : audit [DBG] from='client.? 172.21.15.6:0/2373686397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:51.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:51 smithi006 bash[27292]: cluster 2023-08-28T01:21:49.976647+0000 mgr.y (mgr.14901) 6097 : cluster [DBG] pgmap v6104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:51.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:51 smithi006 bash[27292]: audit 2023-08-28T01:21:50.357141+0000 mon.c (mon.2) 2391 : audit [DBG] from='client.? 172.21.15.6:0/2373686397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:51.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:51 smithi006 bash[33918]: cluster 2023-08-28T01:21:49.976647+0000 mgr.y (mgr.14901) 6097 : cluster [DBG] pgmap v6104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:51.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:51 smithi006 bash[33918]: audit 2023-08-28T01:21:50.357141+0000 mon.c (mon.2) 2391 : audit [DBG] from='client.? 172.21.15.6:0/2373686397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:51.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:51 smithi130 bash[51302]: debug 2023-08-28T01:21:51.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:51.484044+0000) 2023-08-28T01:21:52.799 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:52 smithi130 bash[51302]: debug 2023-08-28T01:21:52.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:52.484261+0000) 2023-08-28T01:21:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:52 smithi006 bash[33918]: cluster 2023-08-28T01:21:51.977331+0000 mgr.y (mgr.14901) 6098 : cluster [DBG] pgmap v6105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:52 smithi006 bash[33918]: audit 2023-08-28T01:21:52.771678+0000 mon.c (mon.2) 2392 : audit [DBG] from='client.? 172.21.15.6:0/2895528404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:21:52.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:52.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:21:52.569Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:52 smithi006 bash[27292]: cluster 2023-08-28T01:21:51.977331+0000 mgr.y (mgr.14901) 6098 : cluster [DBG] pgmap v6105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:52.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:52 smithi006 bash[27292]: audit 2023-08-28T01:21:52.771678+0000 mon.c (mon.2) 2392 : audit [DBG] from='client.? 172.21.15.6:0/2895528404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:53.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:52 smithi130 bash[27579]: cluster 2023-08-28T01:21:51.977331+0000 mgr.y (mgr.14901) 6098 : cluster [DBG] pgmap v6105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:52 smithi130 bash[27579]: audit 2023-08-28T01:21:52.771678+0000 mon.c (mon.2) 2392 : audit [DBG] from='client.? 172.21.15.6:0/2895528404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:53.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:53 smithi130 bash[51302]: debug 2023-08-28T01:21:53.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:53.484535+0000) 2023-08-28T01:21:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:21:54.447Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:21:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:21:54.449Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:21:54.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:54 smithi130 bash[51302]: debug 2023-08-28T01:21:54.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:54.484816+0000) 2023-08-28T01:21:55.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:55 smithi130 bash[27579]: cluster 2023-08-28T01:21:53.977927+0000 mgr.y (mgr.14901) 6099 : cluster [DBG] pgmap v6106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:55.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:55 smithi006 bash[27292]: cluster 2023-08-28T01:21:53.977927+0000 mgr.y (mgr.14901) 6099 : cluster [DBG] pgmap v6106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:55.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:55 smithi006 bash[33918]: cluster 2023-08-28T01:21:53.977927+0000 mgr.y (mgr.14901) 6099 : cluster [DBG] pgmap v6106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:55.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:55 smithi130 bash[51302]: debug 2023-08-28T01:21:55.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:55.485160+0000) 2023-08-28T01:21:56.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:56 smithi130 bash[27579]: audit 2023-08-28T01:21:55.182208+0000 mon.c (mon.2) 2393 : audit [DBG] from='client.? 172.21.15.6:0/3257528215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:56.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:56 smithi006 bash[27292]: audit 2023-08-28T01:21:55.182208+0000 mon.c (mon.2) 2393 : audit [DBG] from='client.? 172.21.15.6:0/3257528215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:56.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:56 smithi006 bash[33918]: audit 2023-08-28T01:21:55.182208+0000 mon.c (mon.2) 2393 : audit [DBG] from='client.? 172.21.15.6:0/3257528215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:56.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:56 smithi130 bash[51302]: debug 2023-08-28T01:21:56.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:56.485444+0000) 2023-08-28T01:21:57.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:57 smithi130 bash[27579]: cluster 2023-08-28T01:21:55.979024+0000 mgr.y (mgr.14901) 6100 : cluster [DBG] pgmap v6107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:57.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:57 smithi006 bash[27292]: cluster 2023-08-28T01:21:55.979024+0000 mgr.y (mgr.14901) 6100 : cluster [DBG] pgmap v6107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:57.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:57 smithi006 bash[33918]: cluster 2023-08-28T01:21:55.979024+0000 mgr.y (mgr.14901) 6100 : cluster [DBG] pgmap v6107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:57.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:57 smithi130 bash[51302]: debug 2023-08-28T01:21:57.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:57.485703+0000) 2023-08-28T01:21:58.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:58 smithi130 bash[27579]: audit 2023-08-28T01:21:57.589442+0000 mon.a (mon.0) 4094 : audit [DBG] from='client.? 172.21.15.6:0/4092743895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:58.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:58 smithi006 bash[27292]: audit 2023-08-28T01:21:57.589442+0000 mon.a (mon.0) 4094 : audit [DBG] from='client.? 172.21.15.6:0/4092743895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:58.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:58 smithi006 bash[33918]: audit 2023-08-28T01:21:57.589442+0000 mon.a (mon.0) 4094 : audit [DBG] from='client.? 172.21.15.6:0/4092743895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:58.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:58 smithi130 bash[51302]: debug 2023-08-28T01:21:58.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:58.485963+0000) 2023-08-28T01:21:59.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:21:59 smithi130 bash[27579]: cluster 2023-08-28T01:21:57.979653+0000 mgr.y (mgr.14901) 6101 : cluster [DBG] pgmap v6108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:59.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:21:59 smithi006 bash[33918]: cluster 2023-08-28T01:21:57.979653+0000 mgr.y (mgr.14901) 6101 : cluster [DBG] pgmap v6108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:59.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:21:59 smithi006 bash[27292]: cluster 2023-08-28T01:21:57.979653+0000 mgr.y (mgr.14901) 6101 : cluster [DBG] pgmap v6108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:21:59.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:21:59 smithi130 bash[51302]: debug 2023-08-28T01:21:59.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:59.486263+0000) 2023-08-28T01:22:00.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:00 smithi130 bash[27579]: audit 2023-08-28T01:22:00.001974+0000 mon.a (mon.0) 4095 : audit [DBG] from='client.? 172.21.15.6:0/1340006089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:00.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:00 smithi006 bash[27292]: audit 2023-08-28T01:22:00.001974+0000 mon.a (mon.0) 4095 : audit [DBG] from='client.? 172.21.15.6:0/1340006089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:00.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:00 smithi006 bash[33918]: audit 2023-08-28T01:22:00.001974+0000 mon.a (mon.0) 4095 : audit [DBG] from='client.? 172.21.15.6:0/1340006089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:00.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:00 smithi130 bash[51302]: debug 2023-08-28T01:22:00.479+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:00.486656+0000) 2023-08-28T01:22:01.071 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:22:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:22:00] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:22:01.071 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:22:00.994Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:01.071 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:00 smithi006 bash[56298]: level=warn ts=2023-08-28T01:22:00.995Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:01.071 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:01 smithi006 bash[27292]: cluster 2023-08-28T01:21:59.980773+0000 mgr.y (mgr.14901) 6102 : cluster [DBG] pgmap v6109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:01.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:01 smithi130 bash[27579]: cluster 2023-08-28T01:21:59.980773+0000 mgr.y (mgr.14901) 6102 : cluster [DBG] pgmap v6109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:01.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:01 smithi006 bash[33918]: cluster 2023-08-28T01:21:59.980773+0000 mgr.y (mgr.14901) 6102 : cluster [DBG] pgmap v6109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:01.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:01 smithi130 bash[51302]: debug 2023-08-28T01:22:01.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:01.486992+0000) 2023-08-28T01:22:02.804 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:02 smithi130 bash[51302]: debug 2023-08-28T01:22:02.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:02.487210+0000) 2023-08-28T01:22:02.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:02 smithi006 bash[27292]: cluster 2023-08-28T01:22:01.981439+0000 mgr.y (mgr.14901) 6103 : cluster [DBG] pgmap v6110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:02.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:02 smithi006 bash[27292]: audit 2023-08-28T01:22:02.413540+0000 mon.c (mon.2) 2394 : audit [DBG] from='client.? 172.21.15.6:0/1015590354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:02 smithi006 bash[33918]: cluster 2023-08-28T01:22:01.981439+0000 mgr.y (mgr.14901) 6103 : cluster [DBG] pgmap v6110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:02.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:02 smithi006 bash[33918]: audit 2023-08-28T01:22:02.413540+0000 mon.c (mon.2) 2394 : audit [DBG] from='client.? 172.21.15.6:0/1015590354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:02.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:22:02.567Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:02.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:22:02.569Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:03.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:02 smithi130 bash[27579]: cluster 2023-08-28T01:22:01.981439+0000 mgr.y (mgr.14901) 6103 : cluster [DBG] pgmap v6110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:03.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:02 smithi130 bash[27579]: audit 2023-08-28T01:22:02.413540+0000 mon.c (mon.2) 2394 : audit [DBG] from='client.? 172.21.15.6:0/1015590354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:03.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:03 smithi130 bash[51302]: debug 2023-08-28T01:22:03.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:03.487509+0000) 2023-08-28T01:22:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:22:04.448Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:22:04.450Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:04.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:04 smithi130 bash[51302]: debug 2023-08-28T01:22:04.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:04.487826+0000) 2023-08-28T01:22:05.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:05 smithi130 bash[27579]: cluster 2023-08-28T01:22:03.982186+0000 mgr.y (mgr.14901) 6104 : cluster [DBG] pgmap v6111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:05.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:05 smithi130 bash[27579]: audit 2023-08-28T01:22:04.832281+0000 mon.c (mon.2) 2395 : audit [DBG] from='client.? 172.21.15.6:0/399229229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:05.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:05 smithi006 bash[27292]: cluster 2023-08-28T01:22:03.982186+0000 mgr.y (mgr.14901) 6104 : cluster [DBG] pgmap v6111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:05.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:05 smithi006 bash[27292]: audit 2023-08-28T01:22:04.832281+0000 mon.c (mon.2) 2395 : audit [DBG] from='client.? 172.21.15.6:0/399229229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:05.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:05 smithi006 bash[33918]: cluster 2023-08-28T01:22:03.982186+0000 mgr.y (mgr.14901) 6104 : cluster [DBG] pgmap v6111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:05.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:05 smithi006 bash[33918]: audit 2023-08-28T01:22:04.832281+0000 mon.c (mon.2) 2395 : audit [DBG] from='client.? 172.21.15.6:0/399229229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:05.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:05 smithi130 bash[51302]: debug 2023-08-28T01:22:05.487+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:05.488155+0000) 2023-08-28T01:22:06.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:06 smithi130 bash[51302]: debug 2023-08-28T01:22:06.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:06.488434+0000) 2023-08-28T01:22:07.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:07 smithi130 bash[27579]: cluster 2023-08-28T01:22:05.983344+0000 mgr.y (mgr.14901) 6105 : cluster [DBG] pgmap v6112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:07.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:07 smithi006 bash[33918]: cluster 2023-08-28T01:22:05.983344+0000 mgr.y (mgr.14901) 6105 : cluster [DBG] pgmap v6112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:07.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:07 smithi006 bash[27292]: cluster 2023-08-28T01:22:05.983344+0000 mgr.y (mgr.14901) 6105 : cluster [DBG] pgmap v6112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:07.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:07 smithi130 bash[51302]: debug 2023-08-28T01:22:07.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:07.488693+0000) 2023-08-28T01:22:08.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:08 smithi130 bash[27579]: audit 2023-08-28T01:22:07.248975+0000 mon.a (mon.0) 4096 : audit [DBG] from='client.? 172.21.15.6:0/641805090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:08.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:08 smithi006 bash[33918]: audit 2023-08-28T01:22:07.248975+0000 mon.a (mon.0) 4096 : audit [DBG] from='client.? 172.21.15.6:0/641805090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:08.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:08 smithi006 bash[27292]: audit 2023-08-28T01:22:07.248975+0000 mon.a (mon.0) 4096 : audit [DBG] from='client.? 172.21.15.6:0/641805090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:08.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:08 smithi130 bash[51302]: debug 2023-08-28T01:22:08.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:08.488972+0000) 2023-08-28T01:22:09.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:09 smithi130 bash[27579]: cluster 2023-08-28T01:22:07.983979+0000 mgr.y (mgr.14901) 6106 : cluster [DBG] pgmap v6113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:09.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:09 smithi006 bash[27292]: cluster 2023-08-28T01:22:07.983979+0000 mgr.y (mgr.14901) 6106 : cluster [DBG] pgmap v6113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:09.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:09 smithi006 bash[33918]: cluster 2023-08-28T01:22:07.983979+0000 mgr.y (mgr.14901) 6106 : cluster [DBG] pgmap v6113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:09.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:09 smithi130 bash[51302]: debug 2023-08-28T01:22:09.487+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:09.489303+0000) 2023-08-28T01:22:10.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:10 smithi130 bash[27579]: audit 2023-08-28T01:22:09.664710+0000 mon.c (mon.2) 2396 : audit [DBG] from='client.? 172.21.15.6:0/4026185206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:10.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:10 smithi006 bash[33918]: audit 2023-08-28T01:22:09.664710+0000 mon.c (mon.2) 2396 : audit [DBG] from='client.? 172.21.15.6:0/4026185206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:10.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:10 smithi006 bash[27292]: audit 2023-08-28T01:22:09.664710+0000 mon.c (mon.2) 2396 : audit [DBG] from='client.? 172.21.15.6:0/4026185206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:10.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:10 smithi130 bash[51302]: debug 2023-08-28T01:22:10.487+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:10.489643+0000) 2023-08-28T01:22:11.070 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:22:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:22:10] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:22:11.070 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:10 smithi006 bash[56298]: level=error ts=2023-08-28T01:22:10.995Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:11.071 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:10 smithi006 bash[56298]: level=warn ts=2023-08-28T01:22:10.997Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:11.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:11 smithi130 bash[27579]: cluster 2023-08-28T01:22:09.985132+0000 mgr.y (mgr.14901) 6107 : cluster [DBG] pgmap v6114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:11.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:11 smithi006 bash[33918]: cluster 2023-08-28T01:22:09.985132+0000 mgr.y (mgr.14901) 6107 : cluster [DBG] pgmap v6114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:11.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:11 smithi006 bash[27292]: cluster 2023-08-28T01:22:09.985132+0000 mgr.y (mgr.14901) 6107 : cluster [DBG] pgmap v6114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:11.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:11 smithi130 bash[51302]: debug 2023-08-28T01:22:11.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:11.489973+0000) 2023-08-28T01:22:12.806 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:12 smithi130 bash[51302]: debug 2023-08-28T01:22:12.483+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:12.490295+0000) 2023-08-28T01:22:12.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:12 smithi006 bash[27292]: cluster 2023-08-28T01:22:11.985715+0000 mgr.y (mgr.14901) 6108 : cluster [DBG] pgmap v6115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:12.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:12 smithi006 bash[27292]: audit 2023-08-28T01:22:12.079054+0000 mon.a (mon.0) 4097 : audit [DBG] from='client.? 172.21.15.6:0/113428177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:12 smithi006 bash[33918]: cluster 2023-08-28T01:22:11.985715+0000 mgr.y (mgr.14901) 6108 : cluster [DBG] pgmap v6115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:12.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:12 smithi006 bash[33918]: audit 2023-08-28T01:22:12.079054+0000 mon.a (mon.0) 4097 : audit [DBG] from='client.? 172.21.15.6:0/113428177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:22:12.568Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:12.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:22:12.569Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:13.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:12 smithi130 bash[27579]: cluster 2023-08-28T01:22:11.985715+0000 mgr.y (mgr.14901) 6108 : cluster [DBG] pgmap v6115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:12 smithi130 bash[27579]: audit 2023-08-28T01:22:12.079054+0000 mon.a (mon.0) 4097 : audit [DBG] from='client.? 172.21.15.6:0/113428177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:13.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:13 smithi130 bash[51302]: debug 2023-08-28T01:22:13.487+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:13.490570+0000) 2023-08-28T01:22:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:22:14.449Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:22:14.450Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:14.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:14 smithi130 bash[51302]: debug 2023-08-28T01:22:14.482+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:14.490787+0000) 2023-08-28T01:22:15.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:15 smithi130 bash[27579]: cluster 2023-08-28T01:22:13.986320+0000 mgr.y (mgr.14901) 6109 : cluster [DBG] pgmap v6116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:15.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:15 smithi130 bash[27579]: audit 2023-08-28T01:22:14.494912+0000 mon.c (mon.2) 2397 : audit [DBG] from='client.? 172.21.15.6:0/1155095077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:15.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:15 smithi006 bash[27292]: cluster 2023-08-28T01:22:13.986320+0000 mgr.y (mgr.14901) 6109 : cluster [DBG] pgmap v6116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:15.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:15 smithi006 bash[27292]: audit 2023-08-28T01:22:14.494912+0000 mon.c (mon.2) 2397 : audit [DBG] from='client.? 172.21.15.6:0/1155095077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:15.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:15 smithi006 bash[33918]: cluster 2023-08-28T01:22:13.986320+0000 mgr.y (mgr.14901) 6109 : cluster [DBG] pgmap v6116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:15.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:15 smithi006 bash[33918]: audit 2023-08-28T01:22:14.494912+0000 mon.c (mon.2) 2397 : audit [DBG] from='client.? 172.21.15.6:0/1155095077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:15.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:15 smithi130 bash[51302]: debug 2023-08-28T01:22:15.486+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:15.491073+0000) 2023-08-28T01:22:16.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:16 smithi130 bash[51302]: debug 2023-08-28T01:22:16.486+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:16.491288+0000) 2023-08-28T01:22:17.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:17 smithi130 bash[27579]: cluster 2023-08-28T01:22:15.987429+0000 mgr.y (mgr.14901) 6110 : cluster [DBG] pgmap v6117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:17.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:17 smithi130 bash[27579]: audit 2023-08-28T01:22:16.919733+0000 mon.a (mon.0) 4098 : audit [DBG] from='client.? 172.21.15.6:0/121178629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:17.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:17 smithi006 bash[27292]: cluster 2023-08-28T01:22:15.987429+0000 mgr.y (mgr.14901) 6110 : cluster [DBG] pgmap v6117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:17.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:17 smithi006 bash[27292]: audit 2023-08-28T01:22:16.919733+0000 mon.a (mon.0) 4098 : audit [DBG] from='client.? 172.21.15.6:0/121178629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:17.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:17 smithi006 bash[33918]: cluster 2023-08-28T01:22:15.987429+0000 mgr.y (mgr.14901) 6110 : cluster [DBG] pgmap v6117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:17.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:17 smithi006 bash[33918]: audit 2023-08-28T01:22:16.919733+0000 mon.a (mon.0) 4098 : audit [DBG] from='client.? 172.21.15.6:0/121178629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:17.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:17 smithi130 bash[51302]: debug 2023-08-28T01:22:17.490+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:17.491608+0000) 2023-08-28T01:22:18.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:18 smithi130 bash[51302]: debug 2023-08-28T01:22:18.490+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:18.491931+0000) 2023-08-28T01:22:19.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:19 smithi130 bash[27579]: cluster 2023-08-28T01:22:17.988081+0000 mgr.y (mgr.14901) 6111 : cluster [DBG] pgmap v6118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:19.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:19 smithi006 bash[27292]: cluster 2023-08-28T01:22:17.988081+0000 mgr.y (mgr.14901) 6111 : cluster [DBG] pgmap v6118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:19.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:19 smithi006 bash[33918]: cluster 2023-08-28T01:22:17.988081+0000 mgr.y (mgr.14901) 6111 : cluster [DBG] pgmap v6118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:19.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:19 smithi130 bash[51302]: debug 2023-08-28T01:22:19.490+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:19.492221+0000) 2023-08-28T01:22:20.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:20 smithi130 bash[27579]: audit 2023-08-28T01:22:19.337570+0000 mon.a (mon.0) 4099 : audit [DBG] from='client.? 172.21.15.6:0/1314690414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:20.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:20 smithi006 bash[33918]: audit 2023-08-28T01:22:19.337570+0000 mon.a (mon.0) 4099 : audit [DBG] from='client.? 172.21.15.6:0/1314690414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:20.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:20 smithi006 bash[27292]: audit 2023-08-28T01:22:19.337570+0000 mon.a (mon.0) 4099 : audit [DBG] from='client.? 172.21.15.6:0/1314690414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:20.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:20 smithi130 bash[51302]: debug 2023-08-28T01:22:20.490+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:20.492549+0000) 2023-08-28T01:22:21.066 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:22:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:22:20] "GET /metrics HTTP/1.1" 200 193115 "" "Prometheus/2.33.4" 2023-08-28T01:22:21.066 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:20 smithi006 bash[56298]: level=error ts=2023-08-28T01:22:20.995Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:21.066 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:20 smithi006 bash[56298]: level=warn ts=2023-08-28T01:22:20.997Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:21.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:21 smithi130 bash[27579]: cluster 2023-08-28T01:22:19.989245+0000 mgr.y (mgr.14901) 6112 : cluster [DBG] pgmap v6119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:21.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:21 smithi006 bash[27292]: cluster 2023-08-28T01:22:19.989245+0000 mgr.y (mgr.14901) 6112 : cluster [DBG] pgmap v6119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:21.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:21 smithi006 bash[33918]: cluster 2023-08-28T01:22:19.989245+0000 mgr.y (mgr.14901) 6112 : cluster [DBG] pgmap v6119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:21.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:21 smithi130 bash[51302]: debug 2023-08-28T01:22:21.490+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:21.492819+0000) 2023-08-28T01:22:22.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:22 smithi130 bash[27579]: audit 2023-08-28T01:22:21.756178+0000 mon.a (mon.0) 4100 : audit [DBG] from='client.? 172.21.15.6:0/1937180123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:22.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:22 smithi006 bash[33918]: audit 2023-08-28T01:22:21.756178+0000 mon.a (mon.0) 4100 : audit [DBG] from='client.? 172.21.15.6:0/1937180123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:22.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:22 smithi006 bash[27292]: audit 2023-08-28T01:22:21.756178+0000 mon.a (mon.0) 4100 : audit [DBG] from='client.? 172.21.15.6:0/1937180123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:22.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:22 smithi130 bash[51302]: debug 2023-08-28T01:22:22.490+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:22.493043+0000) 2023-08-28T01:22:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:22:22.569Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:22:22.570Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:23.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:23 smithi130 bash[27579]: cluster 2023-08-28T01:22:21.989851+0000 mgr.y (mgr.14901) 6113 : cluster [DBG] pgmap v6120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:23.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:23 smithi006 bash[33918]: cluster 2023-08-28T01:22:21.989851+0000 mgr.y (mgr.14901) 6113 : cluster [DBG] pgmap v6120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:23.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:23 smithi006 bash[27292]: cluster 2023-08-28T01:22:21.989851+0000 mgr.y (mgr.14901) 6113 : cluster [DBG] pgmap v6120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:23.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:23 smithi130 bash[51302]: debug 2023-08-28T01:22:23.490+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:23.493314+0000) 2023-08-28T01:22:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:22:24.450Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:24.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:22:24.452Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:24.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:24 smithi130 bash[51302]: debug 2023-08-28T01:22:24.490+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:24.493591+0000) 2023-08-28T01:22:25.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:25 smithi130 bash[27579]: cluster 2023-08-28T01:22:23.990494+0000 mgr.y (mgr.14901) 6114 : cluster [DBG] pgmap v6121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:25.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:25 smithi130 bash[27579]: audit 2023-08-28T01:22:24.168425+0000 mon.c (mon.2) 2398 : audit [DBG] from='client.? 172.21.15.6:0/3856397551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:25.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:25 smithi006 bash[27292]: cluster 2023-08-28T01:22:23.990494+0000 mgr.y (mgr.14901) 6114 : cluster [DBG] pgmap v6121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:25.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:25 smithi006 bash[27292]: audit 2023-08-28T01:22:24.168425+0000 mon.c (mon.2) 2398 : audit [DBG] from='client.? 172.21.15.6:0/3856397551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:25.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:25 smithi006 bash[33918]: cluster 2023-08-28T01:22:23.990494+0000 mgr.y (mgr.14901) 6114 : cluster [DBG] pgmap v6121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:25.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:25 smithi006 bash[33918]: audit 2023-08-28T01:22:24.168425+0000 mon.c (mon.2) 2398 : audit [DBG] from='client.? 172.21.15.6:0/3856397551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:25.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:25 smithi130 bash[51302]: debug 2023-08-28T01:22:25.490+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:25.493863+0000) 2023-08-28T01:22:26.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:26 smithi130 bash[51302]: debug 2023-08-28T01:22:26.490+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:26.494126+0000) 2023-08-28T01:22:27.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:27 smithi130 bash[27579]: cluster 2023-08-28T01:22:25.991628+0000 mgr.y (mgr.14901) 6115 : cluster [DBG] pgmap v6122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:27.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:27 smithi130 bash[27579]: audit 2023-08-28T01:22:26.585143+0000 mon.a (mon.0) 4101 : audit [DBG] from='client.? 172.21.15.6:0/1520107411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:27.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:27 smithi006 bash[27292]: cluster 2023-08-28T01:22:25.991628+0000 mgr.y (mgr.14901) 6115 : cluster [DBG] pgmap v6122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:27.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:27 smithi006 bash[27292]: audit 2023-08-28T01:22:26.585143+0000 mon.a (mon.0) 4101 : audit [DBG] from='client.? 172.21.15.6:0/1520107411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:27.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:27 smithi006 bash[33918]: cluster 2023-08-28T01:22:25.991628+0000 mgr.y (mgr.14901) 6115 : cluster [DBG] pgmap v6122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:27.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:27 smithi006 bash[33918]: audit 2023-08-28T01:22:26.585143+0000 mon.a (mon.0) 4101 : audit [DBG] from='client.? 172.21.15.6:0/1520107411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:27.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:27 smithi130 bash[51302]: debug 2023-08-28T01:22:27.490+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:27.494398+0000) 2023-08-28T01:22:28.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:28 smithi130 bash[51302]: debug 2023-08-28T01:22:28.490+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:28.494666+0000) 2023-08-28T01:22:29.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:29 smithi130 bash[27579]: cluster 2023-08-28T01:22:27.992280+0000 mgr.y (mgr.14901) 6116 : cluster [DBG] pgmap v6123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:29.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:29 smithi130 bash[27579]: audit 2023-08-28T01:22:28.998292+0000 mon.a (mon.0) 4102 : audit [DBG] from='client.? 172.21.15.6:0/2185705486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:29.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:29 smithi006 bash[33918]: cluster 2023-08-28T01:22:27.992280+0000 mgr.y (mgr.14901) 6116 : cluster [DBG] pgmap v6123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:29.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:29 smithi006 bash[33918]: audit 2023-08-28T01:22:28.998292+0000 mon.a (mon.0) 4102 : audit [DBG] from='client.? 172.21.15.6:0/2185705486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:29.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:29 smithi006 bash[27292]: cluster 2023-08-28T01:22:27.992280+0000 mgr.y (mgr.14901) 6116 : cluster [DBG] pgmap v6123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:29.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:29 smithi006 bash[27292]: audit 2023-08-28T01:22:28.998292+0000 mon.a (mon.0) 4102 : audit [DBG] from='client.? 172.21.15.6:0/2185705486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:29.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:29 smithi130 bash[51302]: debug 2023-08-28T01:22:29.494+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:29.494967+0000) 2023-08-28T01:22:30.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:30 smithi130 bash[51302]: debug 2023-08-28T01:22:30.494+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:30.495204+0000) 2023-08-28T01:22:31.092 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:30 smithi006 bash[56298]: level=error ts=2023-08-28T01:22:30.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:31.093 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:30 smithi006 bash[56298]: level=warn ts=2023-08-28T01:22:30.998Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:31.093 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:22:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:22:30] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T01:22:31.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:31 smithi130 bash[27579]: cluster 2023-08-28T01:22:29.993420+0000 mgr.y (mgr.14901) 6117 : cluster [DBG] pgmap v6124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:31.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:31 smithi006 bash[27292]: cluster 2023-08-28T01:22:29.993420+0000 mgr.y (mgr.14901) 6117 : cluster [DBG] pgmap v6124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:31.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:31 smithi006 bash[33918]: cluster 2023-08-28T01:22:29.993420+0000 mgr.y (mgr.14901) 6117 : cluster [DBG] pgmap v6124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:31.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:31 smithi130 bash[51302]: debug 2023-08-28T01:22:31.494+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:31.495474+0000) 2023-08-28T01:22:32.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:32 smithi130 bash[27579]: audit 2023-08-28T01:22:31.416633+0000 mon.a (mon.0) 4103 : audit [DBG] from='client.? 172.21.15.6:0/4272582651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:32.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:32 smithi006 bash[27292]: audit 2023-08-28T01:22:31.416633+0000 mon.a (mon.0) 4103 : audit [DBG] from='client.? 172.21.15.6:0/4272582651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:32.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:32 smithi006 bash[33918]: audit 2023-08-28T01:22:31.416633+0000 mon.a (mon.0) 4103 : audit [DBG] from='client.? 172.21.15.6:0/4272582651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:32.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:32 smithi130 bash[51302]: debug 2023-08-28T01:22:32.494+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:32.495736+0000) 2023-08-28T01:22:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:22:32.570Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:22:32.572Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:33.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:33 smithi130 bash[27579]: cluster 2023-08-28T01:22:31.994128+0000 mgr.y (mgr.14901) 6118 : cluster [DBG] pgmap v6125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:33.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:33 smithi006 bash[33918]: cluster 2023-08-28T01:22:31.994128+0000 mgr.y (mgr.14901) 6118 : cluster [DBG] pgmap v6125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:33.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:33 smithi006 bash[27292]: cluster 2023-08-28T01:22:31.994128+0000 mgr.y (mgr.14901) 6118 : cluster [DBG] pgmap v6125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:33.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:33 smithi130 bash[51302]: debug 2023-08-28T01:22:33.494+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:33.496028+0000) 2023-08-28T01:22:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:34 smithi130 bash[27579]: audit 2023-08-28T01:22:33.834228+0000 mon.c (mon.2) 2399 : audit [DBG] from='client.? 172.21.15.6:0/2880491531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:34.452 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:34 smithi006 bash[27292]: audit 2023-08-28T01:22:33.834228+0000 mon.c (mon.2) 2399 : audit [DBG] from='client.? 172.21.15.6:0/2880491531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:34.452 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:34 smithi006 bash[33918]: audit 2023-08-28T01:22:33.834228+0000 mon.c (mon.2) 2399 : audit [DBG] from='client.? 172.21.15.6:0/2880491531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:22:34.451Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:34.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:22:34.453Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:34.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:34 smithi130 bash[51302]: debug 2023-08-28T01:22:34.494+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:34.496342+0000) 2023-08-28T01:22:35.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:35 smithi130 bash[27579]: cluster 2023-08-28T01:22:33.994789+0000 mgr.y (mgr.14901) 6119 : cluster [DBG] pgmap v6126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:35.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:35 smithi006 bash[27292]: cluster 2023-08-28T01:22:33.994789+0000 mgr.y (mgr.14901) 6119 : cluster [DBG] pgmap v6126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:35.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:35 smithi006 bash[33918]: cluster 2023-08-28T01:22:33.994789+0000 mgr.y (mgr.14901) 6119 : cluster [DBG] pgmap v6126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:35.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:35 smithi130 bash[51302]: debug 2023-08-28T01:22:35.494+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:35.496641+0000) 2023-08-28T01:22:36.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:36 smithi006 bash[33918]: cluster 2023-08-28T01:22:35.995748+0000 mgr.y (mgr.14901) 6120 : cluster [DBG] pgmap v6127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:36.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:36 smithi006 bash[27292]: cluster 2023-08-28T01:22:35.995748+0000 mgr.y (mgr.14901) 6120 : cluster [DBG] pgmap v6127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:36.495 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:36 smithi130 bash[27579]: cluster 2023-08-28T01:22:35.995748+0000 mgr.y (mgr.14901) 6120 : cluster [DBG] pgmap v6127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:36.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:36 smithi130 bash[51302]: debug 2023-08-28T01:22:36.494+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:36.496907+0000) 2023-08-28T01:22:37.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:37 smithi006 bash[33918]: audit 2023-08-28T01:22:36.253069+0000 mon.c (mon.2) 2400 : audit [DBG] from='client.? 172.21.15.6:0/1002295326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:37.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:37 smithi006 bash[27292]: audit 2023-08-28T01:22:36.253069+0000 mon.c (mon.2) 2400 : audit [DBG] from='client.? 172.21.15.6:0/1002295326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:37.496 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:37 smithi130 bash[27579]: audit 2023-08-28T01:22:36.253069+0000 mon.c (mon.2) 2400 : audit [DBG] from='client.? 172.21.15.6:0/1002295326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:37.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:37 smithi130 bash[51302]: debug 2023-08-28T01:22:37.494+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:37.497165+0000) 2023-08-28T01:22:38.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:38 smithi006 bash[27292]: cluster 2023-08-28T01:22:37.996354+0000 mgr.y (mgr.14901) 6121 : cluster [DBG] pgmap v6128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:38.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:38 smithi006 bash[33918]: cluster 2023-08-28T01:22:37.996354+0000 mgr.y (mgr.14901) 6121 : cluster [DBG] pgmap v6128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:38.496 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:38 smithi130 bash[27579]: cluster 2023-08-28T01:22:37.996354+0000 mgr.y (mgr.14901) 6121 : cluster [DBG] pgmap v6128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:38.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:38 smithi130 bash[51302]: debug 2023-08-28T01:22:38.494+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:38.497460+0000) 2023-08-28T01:22:39.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:39 smithi006 bash[27292]: audit 2023-08-28T01:22:38.683139+0000 mon.c (mon.2) 2401 : audit [DBG] from='client.? 172.21.15.6:0/3905037821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:39.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:39 smithi006 bash[33918]: audit 2023-08-28T01:22:38.683139+0000 mon.c (mon.2) 2401 : audit [DBG] from='client.? 172.21.15.6:0/3905037821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:39.496 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:39 smithi130 bash[27579]: audit 2023-08-28T01:22:38.683139+0000 mon.c (mon.2) 2401 : audit [DBG] from='client.? 172.21.15.6:0/3905037821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:39.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:39 smithi130 bash[51302]: debug 2023-08-28T01:22:39.494+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:39.497774+0000) 2023-08-28T01:22:40.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:40 smithi006 bash[27292]: cluster 2023-08-28T01:22:39.997498+0000 mgr.y (mgr.14901) 6122 : cluster [DBG] pgmap v6129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:40.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:40 smithi006 bash[33918]: cluster 2023-08-28T01:22:39.997498+0000 mgr.y (mgr.14901) 6122 : cluster [DBG] pgmap v6129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:40.497 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:40 smithi130 bash[27579]: cluster 2023-08-28T01:22:39.997498+0000 mgr.y (mgr.14901) 6122 : cluster [DBG] pgmap v6129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:40.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:40 smithi130 bash[51302]: debug 2023-08-28T01:22:40.494+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:40.498069+0000) 2023-08-28T01:22:41.194 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:22:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:22:40] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T01:22:41.194 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:40 smithi006 bash[56298]: level=error ts=2023-08-28T01:22:40.997Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:41.194 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:40 smithi006 bash[56298]: level=warn ts=2023-08-28T01:22:40.999Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:41.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:41 smithi006 bash[27292]: audit 2023-08-28T01:22:41.094560+0000 mon.c (mon.2) 2402 : audit [DBG] from='client.? 172.21.15.6:0/3804205557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:41.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:41 smithi006 bash[33918]: audit 2023-08-28T01:22:41.094560+0000 mon.c (mon.2) 2402 : audit [DBG] from='client.? 172.21.15.6:0/3804205557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:41.497 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:41 smithi130 bash[27579]: audit 2023-08-28T01:22:41.094560+0000 mon.c (mon.2) 2402 : audit [DBG] from='client.? 172.21.15.6:0/3804205557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:41.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:41 smithi130 bash[51302]: debug 2023-08-28T01:22:41.494+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:41.498378+0000) 2023-08-28T01:22:42.824 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:22:42.570Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:42.824 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:22:42.572Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:42.824 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:42 smithi006 bash[27292]: cluster 2023-08-28T01:22:41.998176+0000 mgr.y (mgr.14901) 6123 : cluster [DBG] pgmap v6130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:42.826 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:42 smithi130 bash[51302]: debug 2023-08-28T01:22:42.494+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:42.498705+0000) 2023-08-28T01:22:43.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:42 smithi130 bash[27579]: cluster 2023-08-28T01:22:41.998176+0000 mgr.y (mgr.14901) 6123 : cluster [DBG] pgmap v6130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:43.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:42 smithi006 bash[33918]: cluster 2023-08-28T01:22:41.998176+0000 mgr.y (mgr.14901) 6123 : cluster [DBG] pgmap v6130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:43.828 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:43 smithi130 bash[51302]: debug 2023-08-28T01:22:43.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:43.499042+0000) 2023-08-28T01:22:44.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:43 smithi130 bash[27579]: audit 2023-08-28T01:22:42.856404+0000 mon.b (mon.1) 704 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:22:44.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:43 smithi130 bash[27579]: audit 2023-08-28T01:22:43.502398+0000 mon.a (mon.0) 4104 : audit [DBG] from='client.? 172.21.15.6:0/3907235847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:44.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:43 smithi006 bash[27292]: audit 2023-08-28T01:22:42.856404+0000 mon.b (mon.1) 704 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:22:44.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:43 smithi006 bash[27292]: audit 2023-08-28T01:22:43.502398+0000 mon.a (mon.0) 4104 : audit [DBG] from='client.? 172.21.15.6:0/3907235847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:44.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:43 smithi006 bash[33918]: audit 2023-08-28T01:22:42.856404+0000 mon.b (mon.1) 704 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:22:44.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:43 smithi006 bash[33918]: audit 2023-08-28T01:22:43.502398+0000 mon.a (mon.0) 4104 : audit [DBG] from='client.? 172.21.15.6:0/3907235847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:44.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:22:44.451Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:44.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:22:44.453Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:44.835 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:44 smithi130 bash[51302]: debug 2023-08-28T01:22:44.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:44.499309+0000) 2023-08-28T01:22:45.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:44 smithi130 bash[27579]: cluster 2023-08-28T01:22:43.998890+0000 mgr.y (mgr.14901) 6124 : cluster [DBG] pgmap v6131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:45.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:44 smithi006 bash[33918]: cluster 2023-08-28T01:22:43.998890+0000 mgr.y (mgr.14901) 6124 : cluster [DBG] pgmap v6131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:45.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:44 smithi006 bash[27292]: cluster 2023-08-28T01:22:43.998890+0000 mgr.y (mgr.14901) 6124 : cluster [DBG] pgmap v6131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:45.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:45 smithi130 bash[51302]: debug 2023-08-28T01:22:45.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:45.499667+0000) 2023-08-28T01:22:46.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:45 smithi130 bash[27579]: audit 2023-08-28T01:22:45.917072+0000 mon.c (mon.2) 2403 : audit [DBG] from='client.? 172.21.15.6:0/4039970034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:46.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:45 smithi006 bash[27292]: audit 2023-08-28T01:22:45.917072+0000 mon.c (mon.2) 2403 : audit [DBG] from='client.? 172.21.15.6:0/4039970034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:46.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:45 smithi006 bash[33918]: audit 2023-08-28T01:22:45.917072+0000 mon.c (mon.2) 2403 : audit [DBG] from='client.? 172.21.15.6:0/4039970034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:46.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:46 smithi130 bash[51302]: debug 2023-08-28T01:22:46.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:46.500013+0000) 2023-08-28T01:22:47.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:46 smithi130 bash[27579]: cluster 2023-08-28T01:22:45.999985+0000 mgr.y (mgr.14901) 6125 : cluster [DBG] pgmap v6132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:47.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:46 smithi006 bash[27292]: cluster 2023-08-28T01:22:45.999985+0000 mgr.y (mgr.14901) 6125 : cluster [DBG] pgmap v6132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:47.537 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:46 smithi006 bash[33918]: cluster 2023-08-28T01:22:45.999985+0000 mgr.y (mgr.14901) 6125 : cluster [DBG] pgmap v6132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:47.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:47 smithi130 bash[51302]: debug 2023-08-28T01:22:47.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:47.500311+0000) 2023-08-28T01:22:48.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:48 smithi130 bash[51302]: debug 2023-08-28T01:22:48.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:48.500597+0000) 2023-08-28T01:22:49.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:49 smithi130 bash[27579]: cluster 2023-08-28T01:22:48.000694+0000 mgr.y (mgr.14901) 6126 : cluster [DBG] pgmap v6133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:49.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:49 smithi130 bash[27579]: audit 2023-08-28T01:22:48.349913+0000 mon.a (mon.0) 4105 : audit [DBG] from='client.? 172.21.15.6:0/1687727882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:49.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:49 smithi130 bash[27579]: audit 2023-08-28T01:22:48.735577+0000 mon.a (mon.0) 4106 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:22:49.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:49 smithi006 bash[27292]: cluster 2023-08-28T01:22:48.000694+0000 mgr.y (mgr.14901) 6126 : cluster [DBG] pgmap v6133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:49.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:49 smithi006 bash[27292]: audit 2023-08-28T01:22:48.349913+0000 mon.a (mon.0) 4105 : audit [DBG] from='client.? 172.21.15.6:0/1687727882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:49.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:49 smithi006 bash[27292]: audit 2023-08-28T01:22:48.735577+0000 mon.a (mon.0) 4106 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:22:49.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:49 smithi006 bash[33918]: cluster 2023-08-28T01:22:48.000694+0000 mgr.y (mgr.14901) 6126 : cluster [DBG] pgmap v6133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:49.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:49 smithi006 bash[33918]: audit 2023-08-28T01:22:48.349913+0000 mon.a (mon.0) 4105 : audit [DBG] from='client.? 172.21.15.6:0/1687727882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:49.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:49 smithi006 bash[33918]: audit 2023-08-28T01:22:48.735577+0000 mon.a (mon.0) 4106 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:22:49.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:49 smithi130 bash[51302]: debug 2023-08-28T01:22:49.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:49.500898+0000) 2023-08-28T01:22:50.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:50 smithi006 bash[27292]: audit 2023-08-28T01:22:49.198074+0000 mon.a (mon.0) 4107 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:22:50.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:50 smithi006 bash[27292]: audit 2023-08-28T01:22:49.469342+0000 mon.b (mon.1) 705 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:22:50.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:50 smithi006 bash[27292]: audit 2023-08-28T01:22:49.471558+0000 mon.b (mon.1) 706 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:22:50.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:50 smithi006 bash[27292]: audit 2023-08-28T01:22:49.480489+0000 mon.a (mon.0) 4108 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:22:50.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:50 smithi006 bash[27292]: cluster 2023-08-28T01:22:50.001843+0000 mgr.y (mgr.14901) 6127 : cluster [DBG] pgmap v6134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:50.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:50 smithi006 bash[33918]: audit 2023-08-28T01:22:49.198074+0000 mon.a (mon.0) 4107 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:22:50.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:50 smithi006 bash[33918]: audit 2023-08-28T01:22:49.469342+0000 mon.b (mon.1) 705 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:22:50.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:50 smithi006 bash[33918]: audit 2023-08-28T01:22:49.471558+0000 mon.b (mon.1) 706 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:22:50.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:50 smithi006 bash[33918]: audit 2023-08-28T01:22:49.480489+0000 mon.a (mon.0) 4108 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:22:50.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:50 smithi006 bash[33918]: cluster 2023-08-28T01:22:50.001843+0000 mgr.y (mgr.14901) 6127 : cluster [DBG] pgmap v6134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:50.500 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:50 smithi130 bash[27579]: audit 2023-08-28T01:22:49.198074+0000 mon.a (mon.0) 4107 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:22:50.500 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:50 smithi130 bash[27579]: audit 2023-08-28T01:22:49.469342+0000 mon.b (mon.1) 705 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:22:50.500 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:50 smithi130 bash[27579]: audit 2023-08-28T01:22:49.471558+0000 mon.b (mon.1) 706 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:22:50.500 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:50 smithi130 bash[27579]: audit 2023-08-28T01:22:49.480489+0000 mon.a (mon.0) 4108 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:22:50.501 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:50 smithi130 bash[27579]: cluster 2023-08-28T01:22:50.001843+0000 mgr.y (mgr.14901) 6127 : cluster [DBG] pgmap v6134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:50.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:50 smithi130 bash[51302]: debug 2023-08-28T01:22:50.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:50.501244+0000) 2023-08-28T01:22:51.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:22:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:22:50] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:22:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:50 smithi006 bash[56298]: level=error ts=2023-08-28T01:22:50.998Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:51.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:22:51.000Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:51.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:51 smithi130 bash[27579]: audit 2023-08-28T01:22:50.769281+0000 mon.c (mon.2) 2404 : audit [DBG] from='client.? 172.21.15.6:0/2487346126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:51.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:51 smithi130 bash[51302]: debug 2023-08-28T01:22:51.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:51.501500+0000) 2023-08-28T01:22:51.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:51 smithi006 bash[27292]: audit 2023-08-28T01:22:50.769281+0000 mon.c (mon.2) 2404 : audit [DBG] from='client.? 172.21.15.6:0/2487346126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:51.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:51 smithi006 bash[33918]: audit 2023-08-28T01:22:50.769281+0000 mon.c (mon.2) 2404 : audit [DBG] from='client.? 172.21.15.6:0/2487346126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:52.828 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:22:52.572Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:52.828 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:22:52.574Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:52.829 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:52 smithi130 bash[51302]: debug 2023-08-28T01:22:52.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:52.501762+0000) 2023-08-28T01:22:53.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:52 smithi130 bash[27579]: cluster 2023-08-28T01:22:52.002642+0000 mgr.y (mgr.14901) 6128 : cluster [DBG] pgmap v6135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:53.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:52 smithi006 bash[27292]: cluster 2023-08-28T01:22:52.002642+0000 mgr.y (mgr.14901) 6128 : cluster [DBG] pgmap v6135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:53.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:52 smithi006 bash[33918]: cluster 2023-08-28T01:22:52.002642+0000 mgr.y (mgr.14901) 6128 : cluster [DBG] pgmap v6135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:53.832 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:53 smithi130 bash[51302]: debug 2023-08-28T01:22:53.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:53.502036+0000) 2023-08-28T01:22:54.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:53 smithi130 bash[27579]: audit 2023-08-28T01:22:53.182542+0000 mon.a (mon.0) 4109 : audit [DBG] from='client.? 172.21.15.6:0/1801591087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:54.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:53 smithi006 bash[27292]: audit 2023-08-28T01:22:53.182542+0000 mon.a (mon.0) 4109 : audit [DBG] from='client.? 172.21.15.6:0/1801591087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:54.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:53 smithi006 bash[33918]: audit 2023-08-28T01:22:53.182542+0000 mon.a (mon.0) 4109 : audit [DBG] from='client.? 172.21.15.6:0/1801591087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:22:54.452Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:54.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:22:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:22:54.454Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:22:54.841 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:54 smithi130 bash[51302]: debug 2023-08-28T01:22:54.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:54.502377+0000) 2023-08-28T01:22:55.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:54 smithi130 bash[27579]: cluster 2023-08-28T01:22:54.003477+0000 mgr.y (mgr.14901) 6129 : cluster [DBG] pgmap v6136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:55.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:54 smithi006 bash[27292]: cluster 2023-08-28T01:22:54.003477+0000 mgr.y (mgr.14901) 6129 : cluster [DBG] pgmap v6136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:55.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:54 smithi006 bash[33918]: cluster 2023-08-28T01:22:54.003477+0000 mgr.y (mgr.14901) 6129 : cluster [DBG] pgmap v6136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:55.844 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:55 smithi130 bash[51302]: debug 2023-08-28T01:22:55.498+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:55.502780+0000) 2023-08-28T01:22:56.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:55 smithi130 bash[27579]: audit 2023-08-28T01:22:55.601973+0000 mon.c (mon.2) 2405 : audit [DBG] from='client.? 172.21.15.6:0/2312548881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:56.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:55 smithi006 bash[33918]: audit 2023-08-28T01:22:55.601973+0000 mon.c (mon.2) 2405 : audit [DBG] from='client.? 172.21.15.6:0/2312548881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:56.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:55 smithi006 bash[27292]: audit 2023-08-28T01:22:55.601973+0000 mon.c (mon.2) 2405 : audit [DBG] from='client.? 172.21.15.6:0/2312548881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:56.851 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:56 smithi130 bash[51302]: debug 2023-08-28T01:22:56.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:56.503049+0000) 2023-08-28T01:22:57.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:56 smithi130 bash[27579]: cluster 2023-08-28T01:22:56.004723+0000 mgr.y (mgr.14901) 6130 : cluster [DBG] pgmap v6137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:57.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:56 smithi006 bash[33918]: cluster 2023-08-28T01:22:56.004723+0000 mgr.y (mgr.14901) 6130 : cluster [DBG] pgmap v6137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:57.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:56 smithi006 bash[27292]: cluster 2023-08-28T01:22:56.004723+0000 mgr.y (mgr.14901) 6130 : cluster [DBG] pgmap v6137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:57.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:57 smithi130 bash[51302]: debug 2023-08-28T01:22:57.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:57.503520+0000) 2023-08-28T01:22:58.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:58 smithi130 bash[27579]: audit 2023-08-28T01:22:58.019730+0000 mon.a (mon.0) 4110 : audit [DBG] from='client.? 172.21.15.6:0/1023091548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:58.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:58 smithi006 bash[33918]: audit 2023-08-28T01:22:58.019730+0000 mon.a (mon.0) 4110 : audit [DBG] from='client.? 172.21.15.6:0/1023091548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:58.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:58 smithi006 bash[27292]: audit 2023-08-28T01:22:58.019730+0000 mon.a (mon.0) 4110 : audit [DBG] from='client.? 172.21.15.6:0/1023091548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:58.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:58 smithi130 bash[51302]: debug 2023-08-28T01:22:58.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:58.503867+0000) 2023-08-28T01:22:59.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:22:59 smithi130 bash[27579]: cluster 2023-08-28T01:22:58.005284+0000 mgr.y (mgr.14901) 6131 : cluster [DBG] pgmap v6138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:59.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:22:59 smithi006 bash[33918]: cluster 2023-08-28T01:22:58.005284+0000 mgr.y (mgr.14901) 6131 : cluster [DBG] pgmap v6138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:59.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:22:59 smithi006 bash[27292]: cluster 2023-08-28T01:22:58.005284+0000 mgr.y (mgr.14901) 6131 : cluster [DBG] pgmap v6138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:22:59.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:22:59 smithi130 bash[51302]: debug 2023-08-28T01:22:59.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:59.504205+0000) 2023-08-28T01:23:00.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:00 smithi130 bash[51302]: debug 2023-08-28T01:23:00.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:00.504550+0000) 2023-08-28T01:23:01.080 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:23:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:23:00] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T01:23:01.081 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:00 smithi006 bash[56298]: level=error ts=2023-08-28T01:23:00.999Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:01.081 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:23:01.000Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:01.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:01 smithi130 bash[27579]: cluster 2023-08-28T01:23:00.006339+0000 mgr.y (mgr.14901) 6132 : cluster [DBG] pgmap v6139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:01.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:01 smithi130 bash[27579]: audit 2023-08-28T01:23:00.441287+0000 mon.c (mon.2) 2406 : audit [DBG] from='client.? 172.21.15.6:0/3767878010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:01.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:01 smithi006 bash[27292]: cluster 2023-08-28T01:23:00.006339+0000 mgr.y (mgr.14901) 6132 : cluster [DBG] pgmap v6139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:01.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:01 smithi006 bash[27292]: audit 2023-08-28T01:23:00.441287+0000 mon.c (mon.2) 2406 : audit [DBG] from='client.? 172.21.15.6:0/3767878010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:01.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:01 smithi006 bash[33918]: cluster 2023-08-28T01:23:00.006339+0000 mgr.y (mgr.14901) 6132 : cluster [DBG] pgmap v6139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:01.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:01 smithi006 bash[33918]: audit 2023-08-28T01:23:00.441287+0000 mon.c (mon.2) 2406 : audit [DBG] from='client.? 172.21.15.6:0/3767878010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:01.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:01 smithi130 bash[51302]: debug 2023-08-28T01:23:01.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:01.504888+0000) 2023-08-28T01:23:02.841 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:23:02.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:02.841 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:23:02.574Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:02.841 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:02 smithi130 bash[51302]: debug 2023-08-28T01:23:02.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:02.505167+0000) 2023-08-28T01:23:03.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:02 smithi130 bash[27579]: cluster 2023-08-28T01:23:02.007033+0000 mgr.y (mgr.14901) 6133 : cluster [DBG] pgmap v6140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:03.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:02 smithi006 bash[33918]: cluster 2023-08-28T01:23:02.007033+0000 mgr.y (mgr.14901) 6133 : cluster [DBG] pgmap v6140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:03.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:02 smithi006 bash[27292]: cluster 2023-08-28T01:23:02.007033+0000 mgr.y (mgr.14901) 6133 : cluster [DBG] pgmap v6140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:03.848 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:03 smithi130 bash[51302]: debug 2023-08-28T01:23:03.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:03.505502+0000) 2023-08-28T01:23:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:03 smithi130 bash[27579]: audit 2023-08-28T01:23:02.857576+0000 mon.c (mon.2) 2407 : audit [DBG] from='client.? 172.21.15.6:0/4232928477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:04.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:03 smithi006 bash[27292]: audit 2023-08-28T01:23:02.857576+0000 mon.c (mon.2) 2407 : audit [DBG] from='client.? 172.21.15.6:0/4232928477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:04.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:03 smithi006 bash[33918]: audit 2023-08-28T01:23:02.857576+0000 mon.c (mon.2) 2407 : audit [DBG] from='client.? 172.21.15.6:0/4232928477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:23:04.452Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:23:04.454Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:04.856 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:04 smithi130 bash[51302]: debug 2023-08-28T01:23:04.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:04.505836+0000) 2023-08-28T01:23:05.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:04 smithi130 bash[27579]: cluster 2023-08-28T01:23:04.007725+0000 mgr.y (mgr.14901) 6134 : cluster [DBG] pgmap v6141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:05.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:04 smithi006 bash[27292]: cluster 2023-08-28T01:23:04.007725+0000 mgr.y (mgr.14901) 6134 : cluster [DBG] pgmap v6141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:05.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:04 smithi006 bash[33918]: cluster 2023-08-28T01:23:04.007725+0000 mgr.y (mgr.14901) 6134 : cluster [DBG] pgmap v6141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:05.859 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:05 smithi130 bash[51302]: debug 2023-08-28T01:23:05.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:05.506184+0000) 2023-08-28T01:23:06.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:05 smithi130 bash[27579]: audit 2023-08-28T01:23:05.265824+0000 mon.c (mon.2) 2408 : audit [DBG] from='client.? 172.21.15.6:0/1855048812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:06.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:05 smithi006 bash[27292]: audit 2023-08-28T01:23:05.265824+0000 mon.c (mon.2) 2408 : audit [DBG] from='client.? 172.21.15.6:0/1855048812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:06.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:05 smithi006 bash[33918]: audit 2023-08-28T01:23:05.265824+0000 mon.c (mon.2) 2408 : audit [DBG] from='client.? 172.21.15.6:0/1855048812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:06.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:06 smithi130 bash[51302]: debug 2023-08-28T01:23:06.502+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:06.506487+0000) 2023-08-28T01:23:07.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:06 smithi006 bash[27292]: cluster 2023-08-28T01:23:06.008895+0000 mgr.y (mgr.14901) 6135 : cluster [DBG] pgmap v6142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:07.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:06 smithi006 bash[33918]: cluster 2023-08-28T01:23:06.008895+0000 mgr.y (mgr.14901) 6135 : cluster [DBG] pgmap v6142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:07.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:06 smithi130 bash[27579]: cluster 2023-08-28T01:23:06.008895+0000 mgr.y (mgr.14901) 6135 : cluster [DBG] pgmap v6142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:07.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:07 smithi130 bash[51302]: debug 2023-08-28T01:23:07.506+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:07.506815+0000) 2023-08-28T01:23:08.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:07 smithi006 bash[33918]: audit 2023-08-28T01:23:07.675567+0000 mon.c (mon.2) 2409 : audit [DBG] from='client.? 172.21.15.6:0/1442283144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:08.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:07 smithi006 bash[27292]: audit 2023-08-28T01:23:07.675567+0000 mon.c (mon.2) 2409 : audit [DBG] from='client.? 172.21.15.6:0/1442283144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:08.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:07 smithi130 bash[27579]: audit 2023-08-28T01:23:07.675567+0000 mon.c (mon.2) 2409 : audit [DBG] from='client.? 172.21.15.6:0/1442283144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:08.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:08 smithi130 bash[51302]: debug 2023-08-28T01:23:08.506+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:08.507174+0000) 2023-08-28T01:23:09.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:08 smithi006 bash[33918]: cluster 2023-08-28T01:23:08.009704+0000 mgr.y (mgr.14901) 6136 : cluster [DBG] pgmap v6143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:09.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:08 smithi006 bash[27292]: cluster 2023-08-28T01:23:08.009704+0000 mgr.y (mgr.14901) 6136 : cluster [DBG] pgmap v6143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:09.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:08 smithi130 bash[27579]: cluster 2023-08-28T01:23:08.009704+0000 mgr.y (mgr.14901) 6136 : cluster [DBG] pgmap v6143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:09.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:09 smithi130 bash[51302]: debug 2023-08-28T01:23:09.506+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:09.507456+0000) 2023-08-28T01:23:10.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:10 smithi130 bash[51302]: debug 2023-08-28T01:23:10.505+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:10.507808+0000) 2023-08-28T01:23:11.003 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:23:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:23:10] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T01:23:11.003 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:23:10.999Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:11.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:11 smithi130 bash[27579]: cluster 2023-08-28T01:23:10.010830+0000 mgr.y (mgr.14901) 6137 : cluster [DBG] pgmap v6144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:11.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:11 smithi130 bash[27579]: audit 2023-08-28T01:23:10.093965+0000 mon.a (mon.0) 4111 : audit [DBG] from='client.? 172.21.15.6:0/3599852061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:11.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:11 smithi006 bash[33918]: cluster 2023-08-28T01:23:10.010830+0000 mgr.y (mgr.14901) 6137 : cluster [DBG] pgmap v6144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:11.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:11 smithi006 bash[33918]: audit 2023-08-28T01:23:10.093965+0000 mon.a (mon.0) 4111 : audit [DBG] from='client.? 172.21.15.6:0/3599852061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:11.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:23:11.002Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:11.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:11 smithi006 bash[27292]: cluster 2023-08-28T01:23:10.010830+0000 mgr.y (mgr.14901) 6137 : cluster [DBG] pgmap v6144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:11.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:11 smithi006 bash[27292]: audit 2023-08-28T01:23:10.093965+0000 mon.a (mon.0) 4111 : audit [DBG] from='client.? 172.21.15.6:0/3599852061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:11.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:11 smithi130 bash[51302]: debug 2023-08-28T01:23:11.505+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:11.508156+0000) 2023-08-28T01:23:12.844 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:23:12.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:12.844 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:23:12.575Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:12.845 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:12 smithi130 bash[51302]: debug 2023-08-28T01:23:12.505+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:12.508542+0000) 2023-08-28T01:23:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:12 smithi130 bash[27579]: cluster 2023-08-28T01:23:12.011473+0000 mgr.y (mgr.14901) 6138 : cluster [DBG] pgmap v6145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:12 smithi130 bash[27579]: audit 2023-08-28T01:23:12.509347+0000 mon.c (mon.2) 2410 : audit [DBG] from='client.? 172.21.15.6:0/4215515812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:13.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:12 smithi006 bash[27292]: cluster 2023-08-28T01:23:12.011473+0000 mgr.y (mgr.14901) 6138 : cluster [DBG] pgmap v6145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:13.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:12 smithi006 bash[27292]: audit 2023-08-28T01:23:12.509347+0000 mon.c (mon.2) 2410 : audit [DBG] from='client.? 172.21.15.6:0/4215515812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:13.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:12 smithi006 bash[33918]: cluster 2023-08-28T01:23:12.011473+0000 mgr.y (mgr.14901) 6138 : cluster [DBG] pgmap v6145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:13.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:12 smithi006 bash[33918]: audit 2023-08-28T01:23:12.509347+0000 mon.c (mon.2) 2410 : audit [DBG] from='client.? 172.21.15.6:0/4215515812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:13.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:13 smithi130 bash[51302]: debug 2023-08-28T01:23:13.505+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:13.508805+0000) 2023-08-28T01:23:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:23:14.453Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:23:14.455Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:14.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:14 smithi130 bash[51302]: debug 2023-08-28T01:23:14.505+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:14.509158+0000) 2023-08-28T01:23:15.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:15 smithi130 bash[27579]: cluster 2023-08-28T01:23:14.012137+0000 mgr.y (mgr.14901) 6139 : cluster [DBG] pgmap v6146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:15.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:15 smithi130 bash[27579]: audit 2023-08-28T01:23:14.921153+0000 mon.c (mon.2) 2411 : audit [DBG] from='client.? 172.21.15.6:0/2675921151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:15.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:15 smithi006 bash[33918]: cluster 2023-08-28T01:23:14.012137+0000 mgr.y (mgr.14901) 6139 : cluster [DBG] pgmap v6146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:15.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:15 smithi006 bash[33918]: audit 2023-08-28T01:23:14.921153+0000 mon.c (mon.2) 2411 : audit [DBG] from='client.? 172.21.15.6:0/2675921151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:15.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:15 smithi006 bash[27292]: cluster 2023-08-28T01:23:14.012137+0000 mgr.y (mgr.14901) 6139 : cluster [DBG] pgmap v6146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:15.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:15 smithi006 bash[27292]: audit 2023-08-28T01:23:14.921153+0000 mon.c (mon.2) 2411 : audit [DBG] from='client.? 172.21.15.6:0/2675921151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:15.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:15 smithi130 bash[51302]: debug 2023-08-28T01:23:15.505+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:15.509518+0000) 2023-08-28T01:23:16.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:16 smithi130 bash[51302]: debug 2023-08-28T01:23:16.505+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:16.509854+0000) 2023-08-28T01:23:17.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:17 smithi130 bash[27579]: cluster 2023-08-28T01:23:16.013275+0000 mgr.y (mgr.14901) 6140 : cluster [DBG] pgmap v6147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:17.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:17 smithi006 bash[27292]: cluster 2023-08-28T01:23:16.013275+0000 mgr.y (mgr.14901) 6140 : cluster [DBG] pgmap v6147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:17.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:17 smithi006 bash[33918]: cluster 2023-08-28T01:23:16.013275+0000 mgr.y (mgr.14901) 6140 : cluster [DBG] pgmap v6147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:17.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:17 smithi130 bash[51302]: debug 2023-08-28T01:23:17.505+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:17.510162+0000) 2023-08-28T01:23:18.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:18 smithi130 bash[27579]: audit 2023-08-28T01:23:17.339150+0000 mon.a (mon.0) 4112 : audit [DBG] from='client.? 172.21.15.6:0/2646316135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:18.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:18 smithi006 bash[27292]: audit 2023-08-28T01:23:17.339150+0000 mon.a (mon.0) 4112 : audit [DBG] from='client.? 172.21.15.6:0/2646316135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:18.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:18 smithi006 bash[33918]: audit 2023-08-28T01:23:17.339150+0000 mon.a (mon.0) 4112 : audit [DBG] from='client.? 172.21.15.6:0/2646316135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:18.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:18 smithi130 bash[51302]: debug 2023-08-28T01:23:18.505+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:18.510486+0000) 2023-08-28T01:23:19.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:19 smithi130 bash[27579]: cluster 2023-08-28T01:23:18.013925+0000 mgr.y (mgr.14901) 6141 : cluster [DBG] pgmap v6148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:19.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:19 smithi006 bash[27292]: cluster 2023-08-28T01:23:18.013925+0000 mgr.y (mgr.14901) 6141 : cluster [DBG] pgmap v6148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:19.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:19 smithi006 bash[33918]: cluster 2023-08-28T01:23:18.013925+0000 mgr.y (mgr.14901) 6141 : cluster [DBG] pgmap v6148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:19.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:19 smithi130 bash[51302]: debug 2023-08-28T01:23:19.505+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:19.510804+0000) 2023-08-28T01:23:20.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:20 smithi130 bash[27579]: audit 2023-08-28T01:23:19.759260+0000 mon.c (mon.2) 2412 : audit [DBG] from='client.? 172.21.15.6:0/844021126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:20.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:20 smithi006 bash[27292]: audit 2023-08-28T01:23:19.759260+0000 mon.c (mon.2) 2412 : audit [DBG] from='client.? 172.21.15.6:0/844021126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:20.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:20 smithi006 bash[33918]: audit 2023-08-28T01:23:19.759260+0000 mon.c (mon.2) 2412 : audit [DBG] from='client.? 172.21.15.6:0/844021126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:20.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:20 smithi130 bash[51302]: debug 2023-08-28T01:23:20.509+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:20.511170+0000) 2023-08-28T01:23:21.004 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:23:21.001Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:21.004 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:23:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:23:20] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:23:21.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:21 smithi130 bash[27579]: cluster 2023-08-28T01:23:20.015018+0000 mgr.y (mgr.14901) 6142 : cluster [DBG] pgmap v6149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:21.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:21 smithi006 bash[33918]: cluster 2023-08-28T01:23:20.015018+0000 mgr.y (mgr.14901) 6142 : cluster [DBG] pgmap v6149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:21.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:23:21.003Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:21.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:21 smithi006 bash[27292]: cluster 2023-08-28T01:23:20.015018+0000 mgr.y (mgr.14901) 6142 : cluster [DBG] pgmap v6149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:21.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:21 smithi130 bash[51302]: debug 2023-08-28T01:23:21.509+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:21.511491+0000) 2023-08-28T01:23:22.837 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:23:22.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:22.837 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:23:22.576Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:22.837 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:22 smithi130 bash[51302]: debug 2023-08-28T01:23:22.509+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:22.511772+0000) 2023-08-28T01:23:23.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:22 smithi130 bash[27579]: cluster 2023-08-28T01:23:22.015641+0000 mgr.y (mgr.14901) 6143 : cluster [DBG] pgmap v6150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:23.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:22 smithi130 bash[27579]: audit 2023-08-28T01:23:22.181628+0000 mon.c (mon.2) 2413 : audit [DBG] from='client.? 172.21.15.6:0/2995787474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:23.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:22 smithi006 bash[27292]: cluster 2023-08-28T01:23:22.015641+0000 mgr.y (mgr.14901) 6143 : cluster [DBG] pgmap v6150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:23.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:22 smithi006 bash[27292]: audit 2023-08-28T01:23:22.181628+0000 mon.c (mon.2) 2413 : audit [DBG] from='client.? 172.21.15.6:0/2995787474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:23.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:22 smithi006 bash[33918]: cluster 2023-08-28T01:23:22.015641+0000 mgr.y (mgr.14901) 6143 : cluster [DBG] pgmap v6150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:23.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:22 smithi006 bash[33918]: audit 2023-08-28T01:23:22.181628+0000 mon.c (mon.2) 2413 : audit [DBG] from='client.? 172.21.15.6:0/2995787474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:23.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:23 smithi130 bash[51302]: debug 2023-08-28T01:23:23.509+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:23.511999+0000) 2023-08-28T01:23:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:23:24.454Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:24.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:23:24.456Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:24.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:24 smithi130 bash[51302]: debug 2023-08-28T01:23:24.509+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:24.512302+0000) 2023-08-28T01:23:25.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:25 smithi130 bash[27579]: cluster 2023-08-28T01:23:24.016326+0000 mgr.y (mgr.14901) 6144 : cluster [DBG] pgmap v6151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:25.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:25 smithi130 bash[27579]: audit 2023-08-28T01:23:24.596795+0000 mon.c (mon.2) 2414 : audit [DBG] from='client.? 172.21.15.6:0/3713439903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:25.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:25 smithi006 bash[27292]: cluster 2023-08-28T01:23:24.016326+0000 mgr.y (mgr.14901) 6144 : cluster [DBG] pgmap v6151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:25.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:25 smithi006 bash[27292]: audit 2023-08-28T01:23:24.596795+0000 mon.c (mon.2) 2414 : audit [DBG] from='client.? 172.21.15.6:0/3713439903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:25.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:25 smithi006 bash[33918]: cluster 2023-08-28T01:23:24.016326+0000 mgr.y (mgr.14901) 6144 : cluster [DBG] pgmap v6151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:25.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:25 smithi006 bash[33918]: audit 2023-08-28T01:23:24.596795+0000 mon.c (mon.2) 2414 : audit [DBG] from='client.? 172.21.15.6:0/3713439903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:25.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:25 smithi130 bash[51302]: debug 2023-08-28T01:23:25.509+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:25.512581+0000) 2023-08-28T01:23:26.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:26 smithi130 bash[51302]: debug 2023-08-28T01:23:26.509+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:26.512935+0000) 2023-08-28T01:23:27.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:27 smithi130 bash[27579]: cluster 2023-08-28T01:23:26.017511+0000 mgr.y (mgr.14901) 6145 : cluster [DBG] pgmap v6152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:27.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:27 smithi130 bash[27579]: audit 2023-08-28T01:23:27.014227+0000 mon.c (mon.2) 2415 : audit [DBG] from='client.? 172.21.15.6:0/2638884647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:27.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:27 smithi006 bash[33918]: cluster 2023-08-28T01:23:26.017511+0000 mgr.y (mgr.14901) 6145 : cluster [DBG] pgmap v6152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:27.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:27 smithi006 bash[33918]: audit 2023-08-28T01:23:27.014227+0000 mon.c (mon.2) 2415 : audit [DBG] from='client.? 172.21.15.6:0/2638884647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:27.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:27 smithi006 bash[27292]: cluster 2023-08-28T01:23:26.017511+0000 mgr.y (mgr.14901) 6145 : cluster [DBG] pgmap v6152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:27.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:27 smithi006 bash[27292]: audit 2023-08-28T01:23:27.014227+0000 mon.c (mon.2) 2415 : audit [DBG] from='client.? 172.21.15.6:0/2638884647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:27.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:27 smithi130 bash[51302]: debug 2023-08-28T01:23:27.509+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:27.513247+0000) 2023-08-28T01:23:28.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:28 smithi130 bash[51302]: debug 2023-08-28T01:23:28.509+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:28.513584+0000) 2023-08-28T01:23:29.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:29 smithi130 bash[27579]: cluster 2023-08-28T01:23:28.018156+0000 mgr.y (mgr.14901) 6146 : cluster [DBG] pgmap v6153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:29.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:29 smithi006 bash[33918]: cluster 2023-08-28T01:23:28.018156+0000 mgr.y (mgr.14901) 6146 : cluster [DBG] pgmap v6153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:29.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:29 smithi006 bash[27292]: cluster 2023-08-28T01:23:28.018156+0000 mgr.y (mgr.14901) 6146 : cluster [DBG] pgmap v6153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:29.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:29 smithi130 bash[51302]: debug 2023-08-28T01:23:29.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:29.513949+0000) 2023-08-28T01:23:30.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:30 smithi130 bash[27579]: audit 2023-08-28T01:23:29.438503+0000 mon.c (mon.2) 2416 : audit [DBG] from='client.? 172.21.15.6:0/4003678640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:30.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:30 smithi006 bash[27292]: audit 2023-08-28T01:23:29.438503+0000 mon.c (mon.2) 2416 : audit [DBG] from='client.? 172.21.15.6:0/4003678640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:30.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:30 smithi006 bash[33918]: audit 2023-08-28T01:23:29.438503+0000 mon.c (mon.2) 2416 : audit [DBG] from='client.? 172.21.15.6:0/4003678640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:30.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:30 smithi130 bash[51302]: debug 2023-08-28T01:23:30.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:30.514304+0000) 2023-08-28T01:23:31.002 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:23:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:23:30] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:23:31.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:31 smithi130 bash[27579]: cluster 2023-08-28T01:23:30.019339+0000 mgr.y (mgr.14901) 6147 : cluster [DBG] pgmap v6154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:31.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:31 smithi006 bash[33918]: cluster 2023-08-28T01:23:30.019339+0000 mgr.y (mgr.14901) 6147 : cluster [DBG] pgmap v6154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:31.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:31 smithi006 bash[27292]: cluster 2023-08-28T01:23:30.019339+0000 mgr.y (mgr.14901) 6147 : cluster [DBG] pgmap v6154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:31.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:23:31.001Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:31.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:23:31.003Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:31.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:31 smithi130 bash[51302]: debug 2023-08-28T01:23:31.509+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:31.514564+0000) 2023-08-28T01:23:32.841 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:23:32.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:32.841 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:23:32.576Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:32.842 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:32 smithi130 bash[51302]: debug 2023-08-28T01:23:32.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:32.514923+0000) 2023-08-28T01:23:33.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:32 smithi130 bash[27579]: audit 2023-08-28T01:23:31.854351+0000 mon.c (mon.2) 2417 : audit [DBG] from='client.? 172.21.15.6:0/3330063697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:32 smithi130 bash[27579]: cluster 2023-08-28T01:23:32.020028+0000 mgr.y (mgr.14901) 6148 : cluster [DBG] pgmap v6155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:33.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:32 smithi006 bash[33918]: audit 2023-08-28T01:23:31.854351+0000 mon.c (mon.2) 2417 : audit [DBG] from='client.? 172.21.15.6:0/3330063697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:33.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:32 smithi006 bash[33918]: cluster 2023-08-28T01:23:32.020028+0000 mgr.y (mgr.14901) 6148 : cluster [DBG] pgmap v6155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:33.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:32 smithi006 bash[27292]: audit 2023-08-28T01:23:31.854351+0000 mon.c (mon.2) 2417 : audit [DBG] from='client.? 172.21.15.6:0/3330063697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:33.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:32 smithi006 bash[27292]: cluster 2023-08-28T01:23:32.020028+0000 mgr.y (mgr.14901) 6148 : cluster [DBG] pgmap v6155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:33.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:33 smithi130 bash[51302]: debug 2023-08-28T01:23:33.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:33.515135+0000) 2023-08-28T01:23:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:23:34.454Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:34.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:23:34.456Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:34.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:34 smithi130 bash[51302]: debug 2023-08-28T01:23:34.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:34.515451+0000) 2023-08-28T01:23:35.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:35 smithi130 bash[27579]: cluster 2023-08-28T01:23:34.020637+0000 mgr.y (mgr.14901) 6149 : cluster [DBG] pgmap v6156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:35.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:35 smithi130 bash[27579]: audit 2023-08-28T01:23:34.263711+0000 mon.a (mon.0) 4113 : audit [DBG] from='client.? 172.21.15.6:0/1332163164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:35.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:35 smithi006 bash[33918]: cluster 2023-08-28T01:23:34.020637+0000 mgr.y (mgr.14901) 6149 : cluster [DBG] pgmap v6156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:35.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:35 smithi006 bash[33918]: audit 2023-08-28T01:23:34.263711+0000 mon.a (mon.0) 4113 : audit [DBG] from='client.? 172.21.15.6:0/1332163164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:35.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:35 smithi006 bash[27292]: cluster 2023-08-28T01:23:34.020637+0000 mgr.y (mgr.14901) 6149 : cluster [DBG] pgmap v6156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:35.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:35 smithi006 bash[27292]: audit 2023-08-28T01:23:34.263711+0000 mon.a (mon.0) 4113 : audit [DBG] from='client.? 172.21.15.6:0/1332163164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:35.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:35 smithi130 bash[51302]: debug 2023-08-28T01:23:35.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:35.515832+0000) 2023-08-28T01:23:36.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:36 smithi130 bash[51302]: debug 2023-08-28T01:23:36.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:36.516188+0000) 2023-08-28T01:23:37.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:37 smithi130 bash[27579]: cluster 2023-08-28T01:23:36.021764+0000 mgr.y (mgr.14901) 6150 : cluster [DBG] pgmap v6157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:37.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:37 smithi130 bash[27579]: audit 2023-08-28T01:23:36.686401+0000 mon.a (mon.0) 4114 : audit [DBG] from='client.? 172.21.15.6:0/1435219011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:37.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:37 smithi006 bash[33918]: cluster 2023-08-28T01:23:36.021764+0000 mgr.y (mgr.14901) 6150 : cluster [DBG] pgmap v6157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:37.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:37 smithi006 bash[33918]: audit 2023-08-28T01:23:36.686401+0000 mon.a (mon.0) 4114 : audit [DBG] from='client.? 172.21.15.6:0/1435219011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:37.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:37 smithi006 bash[27292]: cluster 2023-08-28T01:23:36.021764+0000 mgr.y (mgr.14901) 6150 : cluster [DBG] pgmap v6157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:37.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:37 smithi006 bash[27292]: audit 2023-08-28T01:23:36.686401+0000 mon.a (mon.0) 4114 : audit [DBG] from='client.? 172.21.15.6:0/1435219011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:37.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:37 smithi130 bash[51302]: debug 2023-08-28T01:23:37.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:37.516578+0000) 2023-08-28T01:23:38.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:38 smithi130 bash[51302]: debug 2023-08-28T01:23:38.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:38.516932+0000) 2023-08-28T01:23:39.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:39 smithi130 bash[27579]: cluster 2023-08-28T01:23:38.022434+0000 mgr.y (mgr.14901) 6151 : cluster [DBG] pgmap v6158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:39.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:39 smithi006 bash[27292]: cluster 2023-08-28T01:23:38.022434+0000 mgr.y (mgr.14901) 6151 : cluster [DBG] pgmap v6158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:39.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:39 smithi006 bash[33918]: cluster 2023-08-28T01:23:38.022434+0000 mgr.y (mgr.14901) 6151 : cluster [DBG] pgmap v6158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:39.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:39 smithi130 bash[51302]: debug 2023-08-28T01:23:39.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:39.517276+0000) 2023-08-28T01:23:40.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:40 smithi130 bash[27579]: audit 2023-08-28T01:23:39.101788+0000 mon.a (mon.0) 4115 : audit [DBG] from='client.? 172.21.15.6:0/3125889200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:40 smithi006 bash[33918]: audit 2023-08-28T01:23:39.101788+0000 mon.a (mon.0) 4115 : audit [DBG] from='client.? 172.21.15.6:0/3125889200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:40.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:40 smithi006 bash[27292]: audit 2023-08-28T01:23:39.101788+0000 mon.a (mon.0) 4115 : audit [DBG] from='client.? 172.21.15.6:0/3125889200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:40.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:40 smithi130 bash[51302]: debug 2023-08-28T01:23:40.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:40.517699+0000) 2023-08-28T01:23:41.003 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:23:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:23:40] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:23:41.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:41 smithi130 bash[27579]: cluster 2023-08-28T01:23:40.023592+0000 mgr.y (mgr.14901) 6152 : cluster [DBG] pgmap v6159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:41.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:41 smithi006 bash[27292]: cluster 2023-08-28T01:23:40.023592+0000 mgr.y (mgr.14901) 6152 : cluster [DBG] pgmap v6159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:41.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:41 smithi006 bash[33918]: cluster 2023-08-28T01:23:40.023592+0000 mgr.y (mgr.14901) 6152 : cluster [DBG] pgmap v6159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:41.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:23:41.002Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:41.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:23:41.004Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:41.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:41 smithi130 bash[51302]: debug 2023-08-28T01:23:41.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:41.518061+0000) 2023-08-28T01:23:42.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:42 smithi130 bash[27579]: audit 2023-08-28T01:23:41.514781+0000 mon.a (mon.0) 4116 : audit [DBG] from='client.? 172.21.15.6:0/4237797022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:42.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:42 smithi006 bash[33918]: audit 2023-08-28T01:23:41.514781+0000 mon.a (mon.0) 4116 : audit [DBG] from='client.? 172.21.15.6:0/4237797022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:42.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:42 smithi006 bash[27292]: audit 2023-08-28T01:23:41.514781+0000 mon.a (mon.0) 4116 : audit [DBG] from='client.? 172.21.15.6:0/4237797022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:42.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:42 smithi130 bash[51302]: debug 2023-08-28T01:23:42.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:42.518389+0000) 2023-08-28T01:23:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:23:42.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:23:42.576Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:43.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:43 smithi130 bash[27579]: cluster 2023-08-28T01:23:42.024294+0000 mgr.y (mgr.14901) 6153 : cluster [DBG] pgmap v6160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:43.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:43 smithi006 bash[27292]: cluster 2023-08-28T01:23:42.024294+0000 mgr.y (mgr.14901) 6153 : cluster [DBG] pgmap v6160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:43.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:43 smithi006 bash[33918]: cluster 2023-08-28T01:23:42.024294+0000 mgr.y (mgr.14901) 6153 : cluster [DBG] pgmap v6160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:43.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:43 smithi130 bash[51302]: debug 2023-08-28T01:23:43.513+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:43.518644+0000) 2023-08-28T01:23:44.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:44 smithi130 bash[27579]: audit 2023-08-28T01:23:43.921953+0000 mon.a (mon.0) 4117 : audit [DBG] from='client.? 172.21.15.6:0/1456951272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:44.456 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:44 smithi006 bash[27292]: audit 2023-08-28T01:23:43.921953+0000 mon.a (mon.0) 4117 : audit [DBG] from='client.? 172.21.15.6:0/1456951272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:44.457 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:44 smithi006 bash[33918]: audit 2023-08-28T01:23:43.921953+0000 mon.a (mon.0) 4117 : audit [DBG] from='client.? 172.21.15.6:0/1456951272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:23:44.455Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:44.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:23:44.458Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:44.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:44 smithi130 bash[51302]: debug 2023-08-28T01:23:44.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:44.518968+0000) 2023-08-28T01:23:45.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:45 smithi130 bash[27579]: cluster 2023-08-28T01:23:44.025103+0000 mgr.y (mgr.14901) 6154 : cluster [DBG] pgmap v6161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:45.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:45 smithi006 bash[33918]: cluster 2023-08-28T01:23:44.025103+0000 mgr.y (mgr.14901) 6154 : cluster [DBG] pgmap v6161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:45.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:45 smithi006 bash[27292]: cluster 2023-08-28T01:23:44.025103+0000 mgr.y (mgr.14901) 6154 : cluster [DBG] pgmap v6161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:45.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:45 smithi130 bash[51302]: debug 2023-08-28T01:23:45.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:45.519236+0000) 2023-08-28T01:23:46.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:46 smithi130 bash[51302]: debug 2023-08-28T01:23:46.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:46.519504+0000) 2023-08-28T01:23:47.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:47 smithi006 bash[33918]: cluster 2023-08-28T01:23:46.026217+0000 mgr.y (mgr.14901) 6155 : cluster [DBG] pgmap v6162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:47.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:47 smithi006 bash[33918]: audit 2023-08-28T01:23:46.346438+0000 mon.a (mon.0) 4118 : audit [DBG] from='client.? 172.21.15.6:0/1087388220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:47.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:47 smithi006 bash[27292]: cluster 2023-08-28T01:23:46.026217+0000 mgr.y (mgr.14901) 6155 : cluster [DBG] pgmap v6162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:47.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:47 smithi006 bash[27292]: audit 2023-08-28T01:23:46.346438+0000 mon.a (mon.0) 4118 : audit [DBG] from='client.? 172.21.15.6:0/1087388220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:47.518 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:47 smithi130 bash[27579]: cluster 2023-08-28T01:23:46.026217+0000 mgr.y (mgr.14901) 6155 : cluster [DBG] pgmap v6162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:47.519 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:47 smithi130 bash[27579]: audit 2023-08-28T01:23:46.346438+0000 mon.a (mon.0) 4118 : audit [DBG] from='client.? 172.21.15.6:0/1087388220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:47.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:47 smithi130 bash[51302]: debug 2023-08-28T01:23:47.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:47.519821+0000) 2023-08-28T01:23:48.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:48 smithi130 bash[51302]: debug 2023-08-28T01:23:48.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:48.520156+0000) 2023-08-28T01:23:49.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:49 smithi006 bash[27292]: cluster 2023-08-28T01:23:48.026873+0000 mgr.y (mgr.14901) 6156 : cluster [DBG] pgmap v6163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:49.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:49 smithi006 bash[27292]: audit 2023-08-28T01:23:48.762750+0000 mon.a (mon.0) 4119 : audit [DBG] from='client.? 172.21.15.6:0/1455219307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:49.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:49 smithi006 bash[33918]: cluster 2023-08-28T01:23:48.026873+0000 mgr.y (mgr.14901) 6156 : cluster [DBG] pgmap v6163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:49.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:49 smithi006 bash[33918]: audit 2023-08-28T01:23:48.762750+0000 mon.a (mon.0) 4119 : audit [DBG] from='client.? 172.21.15.6:0/1455219307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:49.519 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:49 smithi130 bash[27579]: cluster 2023-08-28T01:23:48.026873+0000 mgr.y (mgr.14901) 6156 : cluster [DBG] pgmap v6163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:49.519 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:49 smithi130 bash[27579]: audit 2023-08-28T01:23:48.762750+0000 mon.a (mon.0) 4119 : audit [DBG] from='client.? 172.21.15.6:0/1455219307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:49.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:49 smithi130 bash[51302]: debug 2023-08-28T01:23:49.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:49.520496+0000) 2023-08-28T01:23:50.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:50 smithi006 bash[27292]: audit 2023-08-28T01:23:49.493818+0000 mon.b (mon.1) 707 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:23:50.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:50 smithi006 bash[33918]: audit 2023-08-28T01:23:49.493818+0000 mon.b (mon.1) 707 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:23:50.519 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:50 smithi130 bash[27579]: audit 2023-08-28T01:23:49.493818+0000 mon.b (mon.1) 707 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:23:50.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:50 smithi130 bash[51302]: debug 2023-08-28T01:23:50.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:50.520865+0000) 2023-08-28T01:23:51.003 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:23:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:23:50] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T01:23:51.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:51 smithi006 bash[27292]: cluster 2023-08-28T01:23:50.028109+0000 mgr.y (mgr.14901) 6157 : cluster [DBG] pgmap v6164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:51.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:51 smithi006 bash[33918]: cluster 2023-08-28T01:23:50.028109+0000 mgr.y (mgr.14901) 6157 : cluster [DBG] pgmap v6164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:51.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:23:51.002Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:51.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:23:51.006Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:51.520 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:51 smithi130 bash[27579]: cluster 2023-08-28T01:23:50.028109+0000 mgr.y (mgr.14901) 6157 : cluster [DBG] pgmap v6164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:51.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:51 smithi130 bash[51302]: debug 2023-08-28T01:23:51.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:51.521228+0000) 2023-08-28T01:23:52.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:52 smithi006 bash[27292]: audit 2023-08-28T01:23:51.187586+0000 mon.a (mon.0) 4120 : audit [DBG] from='client.? 172.21.15.6:0/1848500242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:52.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:52 smithi006 bash[27292]: cluster 2023-08-28T01:23:52.028825+0000 mgr.y (mgr.14901) 6158 : cluster [DBG] pgmap v6165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:52.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:52 smithi006 bash[33918]: audit 2023-08-28T01:23:51.187586+0000 mon.a (mon.0) 4120 : audit [DBG] from='client.? 172.21.15.6:0/1848500242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:52.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:52 smithi006 bash[33918]: cluster 2023-08-28T01:23:52.028825+0000 mgr.y (mgr.14901) 6158 : cluster [DBG] pgmap v6165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:52.520 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:52 smithi130 bash[27579]: audit 2023-08-28T01:23:51.187586+0000 mon.a (mon.0) 4120 : audit [DBG] from='client.? 172.21.15.6:0/1848500242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:52.520 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:52 smithi130 bash[27579]: cluster 2023-08-28T01:23:52.028825+0000 mgr.y (mgr.14901) 6158 : cluster [DBG] pgmap v6165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:52.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:52 smithi130 bash[51302]: debug 2023-08-28T01:23:52.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:52.521530+0000) 2023-08-28T01:23:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:23:52.575Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:23:52.577Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:53.842 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:53 smithi130 bash[51302]: debug 2023-08-28T01:23:53.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:53.521848+0000) 2023-08-28T01:23:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:53 smithi130 bash[27579]: audit 2023-08-28T01:23:53.602740+0000 mon.a (mon.0) 4121 : audit [DBG] from='client.? 172.21.15.6:0/1243887592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:54.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:53 smithi006 bash[33918]: audit 2023-08-28T01:23:53.602740+0000 mon.a (mon.0) 4121 : audit [DBG] from='client.? 172.21.15.6:0/1243887592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:54.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:53 smithi006 bash[27292]: audit 2023-08-28T01:23:53.602740+0000 mon.a (mon.0) 4121 : audit [DBG] from='client.? 172.21.15.6:0/1243887592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:23:54.456Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:23:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:23:54.458Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:23:54.850 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:54 smithi130 bash[51302]: debug 2023-08-28T01:23:54.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:54.522143+0000) 2023-08-28T01:23:55.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:54 smithi130 bash[27579]: cluster 2023-08-28T01:23:54.029559+0000 mgr.y (mgr.14901) 6159 : cluster [DBG] pgmap v6166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:55.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:54 smithi006 bash[27292]: cluster 2023-08-28T01:23:54.029559+0000 mgr.y (mgr.14901) 6159 : cluster [DBG] pgmap v6166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:55.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:54 smithi006 bash[33918]: cluster 2023-08-28T01:23:54.029559+0000 mgr.y (mgr.14901) 6159 : cluster [DBG] pgmap v6166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:55.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:55 smithi130 bash[51302]: debug 2023-08-28T01:23:55.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:55.522440+0000) 2023-08-28T01:23:56.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:56 smithi006 bash[33918]: audit 2023-08-28T01:23:55.440579+0000 mon.a (mon.0) 4122 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:23:56.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:56 smithi006 bash[33918]: audit 2023-08-28T01:23:55.821577+0000 mon.a (mon.0) 4123 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:23:56.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:56 smithi006 bash[33918]: audit 2023-08-28T01:23:56.008504+0000 mon.c (mon.2) 2418 : audit [DBG] from='client.? 172.21.15.6:0/1166941907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:56 smithi006 bash[33918]: cluster 2023-08-28T01:23:56.030247+0000 mgr.y (mgr.14901) 6160 : cluster [DBG] pgmap v6167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:56 smithi006 bash[33918]: audit 2023-08-28T01:23:56.092653+0000 mon.b (mon.1) 708 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:23:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:56 smithi006 bash[33918]: audit 2023-08-28T01:23:56.094640+0000 mon.b (mon.1) 709 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:23:56.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:56 smithi006 bash[33918]: audit 2023-08-28T01:23:56.101300+0000 mon.a (mon.0) 4124 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:23:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:56 smithi006 bash[27292]: audit 2023-08-28T01:23:55.440579+0000 mon.a (mon.0) 4122 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:23:56.717 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:56 smithi006 bash[27292]: audit 2023-08-28T01:23:55.821577+0000 mon.a (mon.0) 4123 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:23:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:56 smithi006 bash[27292]: audit 2023-08-28T01:23:56.008504+0000 mon.c (mon.2) 2418 : audit [DBG] from='client.? 172.21.15.6:0/1166941907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:56 smithi006 bash[27292]: cluster 2023-08-28T01:23:56.030247+0000 mgr.y (mgr.14901) 6160 : cluster [DBG] pgmap v6167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:56 smithi006 bash[27292]: audit 2023-08-28T01:23:56.092653+0000 mon.b (mon.1) 708 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:23:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:56 smithi006 bash[27292]: audit 2023-08-28T01:23:56.094640+0000 mon.b (mon.1) 709 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:23:56.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:56 smithi006 bash[27292]: audit 2023-08-28T01:23:56.101300+0000 mon.a (mon.0) 4124 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:23:56.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:56 smithi130 bash[51302]: debug 2023-08-28T01:23:56.517+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:56.522745+0000) 2023-08-28T01:23:56.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:56 smithi130 bash[27579]: audit 2023-08-28T01:23:55.440579+0000 mon.a (mon.0) 4122 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:23:56.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:56 smithi130 bash[27579]: audit 2023-08-28T01:23:55.821577+0000 mon.a (mon.0) 4123 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:23:56.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:56 smithi130 bash[27579]: audit 2023-08-28T01:23:56.008504+0000 mon.c (mon.2) 2418 : audit [DBG] from='client.? 172.21.15.6:0/1166941907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:56.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:56 smithi130 bash[27579]: cluster 2023-08-28T01:23:56.030247+0000 mgr.y (mgr.14901) 6160 : cluster [DBG] pgmap v6167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:56.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:56 smithi130 bash[27579]: audit 2023-08-28T01:23:56.092653+0000 mon.b (mon.1) 708 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:23:56.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:56 smithi130 bash[27579]: audit 2023-08-28T01:23:56.094640+0000 mon.b (mon.1) 709 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:23:56.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:56 smithi130 bash[27579]: audit 2023-08-28T01:23:56.101300+0000 mon.a (mon.0) 4124 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:23:57.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:57 smithi130 bash[51302]: debug 2023-08-28T01:23:57.521+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:57.523061+0000) 2023-08-28T01:23:58.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:58 smithi130 bash[51302]: debug 2023-08-28T01:23:58.521+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:58.523418+0000) 2023-08-28T01:23:59.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:59 smithi130 bash[27579]: cluster 2023-08-28T01:23:58.030861+0000 mgr.y (mgr.14901) 6161 : cluster [DBG] pgmap v6168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:59.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:23:59 smithi130 bash[27579]: audit 2023-08-28T01:23:58.436736+0000 mon.a (mon.0) 4125 : audit [DBG] from='client.? 172.21.15.6:0/2073072807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:59.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:59 smithi006 bash[27292]: cluster 2023-08-28T01:23:58.030861+0000 mgr.y (mgr.14901) 6161 : cluster [DBG] pgmap v6168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:59.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:23:59 smithi006 bash[27292]: audit 2023-08-28T01:23:58.436736+0000 mon.a (mon.0) 4125 : audit [DBG] from='client.? 172.21.15.6:0/2073072807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:59.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:59 smithi006 bash[33918]: cluster 2023-08-28T01:23:58.030861+0000 mgr.y (mgr.14901) 6161 : cluster [DBG] pgmap v6168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:23:59.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:23:59 smithi006 bash[33918]: audit 2023-08-28T01:23:58.436736+0000 mon.a (mon.0) 4125 : audit [DBG] from='client.? 172.21.15.6:0/2073072807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:59.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:23:59 smithi130 bash[51302]: debug 2023-08-28T01:23:59.521+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:59.523750+0000) 2023-08-28T01:24:00.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:00 smithi130 bash[51302]: debug 2023-08-28T01:24:00.521+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:00.524116+0000) 2023-08-28T01:24:01.005 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:24:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:24:00] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:24:01.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:01 smithi130 bash[27579]: cluster 2023-08-28T01:24:00.032032+0000 mgr.y (mgr.14901) 6162 : cluster [DBG] pgmap v6169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:01.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:01 smithi130 bash[27579]: audit 2023-08-28T01:24:00.854174+0000 mon.a (mon.0) 4126 : audit [DBG] from='client.? 172.21.15.6:0/2047660388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:01.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:01 smithi006 bash[27292]: cluster 2023-08-28T01:24:00.032032+0000 mgr.y (mgr.14901) 6162 : cluster [DBG] pgmap v6169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:01.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:01 smithi006 bash[27292]: audit 2023-08-28T01:24:00.854174+0000 mon.a (mon.0) 4126 : audit [DBG] from='client.? 172.21.15.6:0/2047660388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:01.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:24:01.003Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:01.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:24:01.006Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:01.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:01 smithi006 bash[33918]: cluster 2023-08-28T01:24:00.032032+0000 mgr.y (mgr.14901) 6162 : cluster [DBG] pgmap v6169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:01.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:01 smithi006 bash[33918]: audit 2023-08-28T01:24:00.854174+0000 mon.a (mon.0) 4126 : audit [DBG] from='client.? 172.21.15.6:0/2047660388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:01.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:01 smithi130 bash[51302]: debug 2023-08-28T01:24:01.521+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:01.524513+0000) 2023-08-28T01:24:02.848 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:24:02.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:02.848 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:24:02.578Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:02.849 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:02 smithi130 bash[51302]: debug 2023-08-28T01:24:02.521+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:02.524825+0000) 2023-08-28T01:24:03.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:02 smithi130 bash[27579]: cluster 2023-08-28T01:24:02.032692+0000 mgr.y (mgr.14901) 6163 : cluster [DBG] pgmap v6170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:03.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:02 smithi006 bash[27292]: cluster 2023-08-28T01:24:02.032692+0000 mgr.y (mgr.14901) 6163 : cluster [DBG] pgmap v6170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:03.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:02 smithi006 bash[33918]: cluster 2023-08-28T01:24:02.032692+0000 mgr.y (mgr.14901) 6163 : cluster [DBG] pgmap v6170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:03.852 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:03 smithi130 bash[51302]: debug 2023-08-28T01:24:03.521+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:03.525178+0000) 2023-08-28T01:24:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:03 smithi130 bash[27579]: audit 2023-08-28T01:24:03.272577+0000 mon.c (mon.2) 2419 : audit [DBG] from='client.? 172.21.15.6:0/140954614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:04.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:03 smithi006 bash[33918]: audit 2023-08-28T01:24:03.272577+0000 mon.c (mon.2) 2419 : audit [DBG] from='client.? 172.21.15.6:0/140954614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:04.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:03 smithi006 bash[27292]: audit 2023-08-28T01:24:03.272577+0000 mon.c (mon.2) 2419 : audit [DBG] from='client.? 172.21.15.6:0/140954614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:24:04.456Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:24:04.458Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:04.861 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:04 smithi130 bash[51302]: debug 2023-08-28T01:24:04.525+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:04.525560+0000) 2023-08-28T01:24:05.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:04 smithi130 bash[27579]: cluster 2023-08-28T01:24:04.033365+0000 mgr.y (mgr.14901) 6164 : cluster [DBG] pgmap v6171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:05.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:04 smithi006 bash[27292]: cluster 2023-08-28T01:24:04.033365+0000 mgr.y (mgr.14901) 6164 : cluster [DBG] pgmap v6171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:05.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:04 smithi006 bash[33918]: cluster 2023-08-28T01:24:04.033365+0000 mgr.y (mgr.14901) 6164 : cluster [DBG] pgmap v6171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:05.863 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:05 smithi130 bash[51302]: debug 2023-08-28T01:24:05.525+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:05.525898+0000) 2023-08-28T01:24:06.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:05 smithi130 bash[27579]: audit 2023-08-28T01:24:05.693602+0000 mon.c (mon.2) 2420 : audit [DBG] from='client.? 172.21.15.6:0/3127832084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:06.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:05 smithi006 bash[27292]: audit 2023-08-28T01:24:05.693602+0000 mon.c (mon.2) 2420 : audit [DBG] from='client.? 172.21.15.6:0/3127832084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:06.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:05 smithi006 bash[33918]: audit 2023-08-28T01:24:05.693602+0000 mon.c (mon.2) 2420 : audit [DBG] from='client.? 172.21.15.6:0/3127832084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:06.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:06 smithi130 bash[51302]: debug 2023-08-28T01:24:06.520+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:06.526209+0000) 2023-08-28T01:24:07.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:06 smithi006 bash[27292]: cluster 2023-08-28T01:24:06.034550+0000 mgr.y (mgr.14901) 6165 : cluster [DBG] pgmap v6172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:07.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:06 smithi006 bash[33918]: cluster 2023-08-28T01:24:06.034550+0000 mgr.y (mgr.14901) 6165 : cluster [DBG] pgmap v6172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:07.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:06 smithi130 bash[27579]: cluster 2023-08-28T01:24:06.034550+0000 mgr.y (mgr.14901) 6165 : cluster [DBG] pgmap v6172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:07.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:07 smithi130 bash[51302]: debug 2023-08-28T01:24:07.524+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:07.526562+0000) 2023-08-28T01:24:08.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:08 smithi130 bash[51302]: debug 2023-08-28T01:24:08.524+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:08.526981+0000) 2023-08-28T01:24:09.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:09 smithi130 bash[27579]: cluster 2023-08-28T01:24:08.035150+0000 mgr.y (mgr.14901) 6166 : cluster [DBG] pgmap v6173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:09.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:09 smithi130 bash[27579]: audit 2023-08-28T01:24:08.111351+0000 mon.c (mon.2) 2421 : audit [DBG] from='client.? 172.21.15.6:0/1134488009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:09.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:09 smithi006 bash[27292]: cluster 2023-08-28T01:24:08.035150+0000 mgr.y (mgr.14901) 6166 : cluster [DBG] pgmap v6173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:09.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:09 smithi006 bash[27292]: audit 2023-08-28T01:24:08.111351+0000 mon.c (mon.2) 2421 : audit [DBG] from='client.? 172.21.15.6:0/1134488009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:09.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:09 smithi006 bash[33918]: cluster 2023-08-28T01:24:08.035150+0000 mgr.y (mgr.14901) 6166 : cluster [DBG] pgmap v6173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:09.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:09 smithi006 bash[33918]: audit 2023-08-28T01:24:08.111351+0000 mon.c (mon.2) 2421 : audit [DBG] from='client.? 172.21.15.6:0/1134488009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:09.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:09 smithi130 bash[51302]: debug 2023-08-28T01:24:09.524+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:09.527318+0000) 2023-08-28T01:24:10.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:10 smithi130 bash[51302]: debug 2023-08-28T01:24:10.524+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:10.527686+0000) 2023-08-28T01:24:11.006 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:24:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:24:10] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:24:11.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:11 smithi130 bash[27579]: cluster 2023-08-28T01:24:10.036263+0000 mgr.y (mgr.14901) 6167 : cluster [DBG] pgmap v6174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:11.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:11 smithi130 bash[27579]: audit 2023-08-28T01:24:10.532075+0000 mon.a (mon.0) 4127 : audit [DBG] from='client.? 172.21.15.6:0/1797810840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:11.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:11 smithi006 bash[33918]: cluster 2023-08-28T01:24:10.036263+0000 mgr.y (mgr.14901) 6167 : cluster [DBG] pgmap v6174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:11.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:11 smithi006 bash[33918]: audit 2023-08-28T01:24:10.532075+0000 mon.a (mon.0) 4127 : audit [DBG] from='client.? 172.21.15.6:0/1797810840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:11.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:24:11.005Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:11.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:24:11.007Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:11.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:11 smithi006 bash[27292]: cluster 2023-08-28T01:24:10.036263+0000 mgr.y (mgr.14901) 6167 : cluster [DBG] pgmap v6174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:11.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:11 smithi006 bash[27292]: audit 2023-08-28T01:24:10.532075+0000 mon.a (mon.0) 4127 : audit [DBG] from='client.? 172.21.15.6:0/1797810840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:11.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:11 smithi130 bash[51302]: debug 2023-08-28T01:24:11.524+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:11.528017+0000) 2023-08-28T01:24:12.851 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:24:12.577Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:12.852 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:24:12.579Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:12.852 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:12 smithi130 bash[51302]: debug 2023-08-28T01:24:12.524+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:12.528364+0000) 2023-08-28T01:24:13.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:12 smithi130 bash[27579]: cluster 2023-08-28T01:24:12.036947+0000 mgr.y (mgr.14901) 6168 : cluster [DBG] pgmap v6175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:12 smithi006 bash[33918]: cluster 2023-08-28T01:24:12.036947+0000 mgr.y (mgr.14901) 6168 : cluster [DBG] pgmap v6175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:13.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:12 smithi006 bash[27292]: cluster 2023-08-28T01:24:12.036947+0000 mgr.y (mgr.14901) 6168 : cluster [DBG] pgmap v6175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:13.857 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:13 smithi130 bash[51302]: debug 2023-08-28T01:24:13.524+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:13.528700+0000) 2023-08-28T01:24:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:13 smithi130 bash[27579]: audit 2023-08-28T01:24:12.955152+0000 mon.a (mon.0) 4128 : audit [DBG] from='client.? 172.21.15.6:0/821297678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:13 smithi006 bash[27292]: audit 2023-08-28T01:24:12.955152+0000 mon.a (mon.0) 4128 : audit [DBG] from='client.? 172.21.15.6:0/821297678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:14.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:13 smithi006 bash[33918]: audit 2023-08-28T01:24:12.955152+0000 mon.a (mon.0) 4128 : audit [DBG] from='client.? 172.21.15.6:0/821297678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:24:14.457Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:24:14.459Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:14.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:14 smithi130 bash[51302]: debug 2023-08-28T01:24:14.524+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:14.529048+0000) 2023-08-28T01:24:15.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:14 smithi006 bash[27292]: cluster 2023-08-28T01:24:14.037579+0000 mgr.y (mgr.14901) 6169 : cluster [DBG] pgmap v6176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:15.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:14 smithi006 bash[33918]: cluster 2023-08-28T01:24:14.037579+0000 mgr.y (mgr.14901) 6169 : cluster [DBG] pgmap v6176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:15.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:14 smithi130 bash[27579]: cluster 2023-08-28T01:24:14.037579+0000 mgr.y (mgr.14901) 6169 : cluster [DBG] pgmap v6176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:15.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:15 smithi130 bash[51302]: debug 2023-08-28T01:24:15.524+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:15.529403+0000) 2023-08-28T01:24:16.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:15 smithi006 bash[27292]: audit 2023-08-28T01:24:15.371946+0000 mon.a (mon.0) 4129 : audit [DBG] from='client.? 172.21.15.6:0/1428728239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:16.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:15 smithi006 bash[33918]: audit 2023-08-28T01:24:15.371946+0000 mon.a (mon.0) 4129 : audit [DBG] from='client.? 172.21.15.6:0/1428728239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:16.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:15 smithi130 bash[27579]: audit 2023-08-28T01:24:15.371946+0000 mon.a (mon.0) 4129 : audit [DBG] from='client.? 172.21.15.6:0/1428728239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:16.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:16 smithi130 bash[51302]: debug 2023-08-28T01:24:16.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:16.529731+0000) 2023-08-28T01:24:17.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:16 smithi006 bash[33918]: cluster 2023-08-28T01:24:16.038844+0000 mgr.y (mgr.14901) 6170 : cluster [DBG] pgmap v6177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:17.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:16 smithi006 bash[27292]: cluster 2023-08-28T01:24:16.038844+0000 mgr.y (mgr.14901) 6170 : cluster [DBG] pgmap v6177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:17.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:16 smithi130 bash[27579]: cluster 2023-08-28T01:24:16.038844+0000 mgr.y (mgr.14901) 6170 : cluster [DBG] pgmap v6177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:17.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:17 smithi130 bash[51302]: debug 2023-08-28T01:24:17.524+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:17.530058+0000) 2023-08-28T01:24:18.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:17 smithi006 bash[27292]: audit 2023-08-28T01:24:17.784548+0000 mon.c (mon.2) 2422 : audit [DBG] from='client.? 172.21.15.6:0/739754010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:18.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:17 smithi006 bash[33918]: audit 2023-08-28T01:24:17.784548+0000 mon.c (mon.2) 2422 : audit [DBG] from='client.? 172.21.15.6:0/739754010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:18.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:17 smithi130 bash[27579]: audit 2023-08-28T01:24:17.784548+0000 mon.c (mon.2) 2422 : audit [DBG] from='client.? 172.21.15.6:0/739754010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:18.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:18 smithi130 bash[51302]: debug 2023-08-28T01:24:18.524+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:18.530402+0000) 2023-08-28T01:24:19.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:18 smithi006 bash[33918]: cluster 2023-08-28T01:24:18.039723+0000 mgr.y (mgr.14901) 6171 : cluster [DBG] pgmap v6178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:19.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:18 smithi006 bash[27292]: cluster 2023-08-28T01:24:18.039723+0000 mgr.y (mgr.14901) 6171 : cluster [DBG] pgmap v6178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:19.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:18 smithi130 bash[27579]: cluster 2023-08-28T01:24:18.039723+0000 mgr.y (mgr.14901) 6171 : cluster [DBG] pgmap v6178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:19.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:19 smithi130 bash[51302]: debug 2023-08-28T01:24:19.524+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:19.530764+0000) 2023-08-28T01:24:20.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:20 smithi130 bash[51302]: debug 2023-08-28T01:24:20.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:20.531173+0000) 2023-08-28T01:24:21.006 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:24:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:24:20] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T01:24:21.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:21 smithi130 bash[27579]: cluster 2023-08-28T01:24:20.040813+0000 mgr.y (mgr.14901) 6172 : cluster [DBG] pgmap v6179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:21.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:21 smithi130 bash[27579]: audit 2023-08-28T01:24:20.199622+0000 mon.c (mon.2) 2423 : audit [DBG] from='client.? 172.21.15.6:0/140865991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:21.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:21 smithi006 bash[27292]: cluster 2023-08-28T01:24:20.040813+0000 mgr.y (mgr.14901) 6172 : cluster [DBG] pgmap v6179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:21.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:21 smithi006 bash[27292]: audit 2023-08-28T01:24:20.199622+0000 mon.c (mon.2) 2423 : audit [DBG] from='client.? 172.21.15.6:0/140865991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:21.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:21 smithi006 bash[33918]: cluster 2023-08-28T01:24:20.040813+0000 mgr.y (mgr.14901) 6172 : cluster [DBG] pgmap v6179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:21.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:21 smithi006 bash[33918]: audit 2023-08-28T01:24:20.199622+0000 mon.c (mon.2) 2423 : audit [DBG] from='client.? 172.21.15.6:0/140865991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:21.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:24:21.005Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:21.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:24:21.007Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:21.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:21 smithi130 bash[51302]: debug 2023-08-28T01:24:21.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:21.531448+0000) 2023-08-28T01:24:22.852 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:24:22.578Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:22.852 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:24:22.579Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:22.853 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:22 smithi130 bash[51302]: debug 2023-08-28T01:24:22.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:22.531750+0000) 2023-08-28T01:24:23.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:22 smithi130 bash[27579]: cluster 2023-08-28T01:24:22.041633+0000 mgr.y (mgr.14901) 6173 : cluster [DBG] pgmap v6180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:23.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:22 smithi130 bash[27579]: audit 2023-08-28T01:24:22.623438+0000 mon.a (mon.0) 4130 : audit [DBG] from='client.? 172.21.15.6:0/1178326462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:23.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:22 smithi006 bash[33918]: cluster 2023-08-28T01:24:22.041633+0000 mgr.y (mgr.14901) 6173 : cluster [DBG] pgmap v6180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:23.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:22 smithi006 bash[33918]: audit 2023-08-28T01:24:22.623438+0000 mon.a (mon.0) 4130 : audit [DBG] from='client.? 172.21.15.6:0/1178326462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:23.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:22 smithi006 bash[27292]: cluster 2023-08-28T01:24:22.041633+0000 mgr.y (mgr.14901) 6173 : cluster [DBG] pgmap v6180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:23.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:22 smithi006 bash[27292]: audit 2023-08-28T01:24:22.623438+0000 mon.a (mon.0) 4130 : audit [DBG] from='client.? 172.21.15.6:0/1178326462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:23.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:23 smithi130 bash[51302]: debug 2023-08-28T01:24:23.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:23.532067+0000) 2023-08-28T01:24:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:24:24.458Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:24.718 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:24:24.460Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:24.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:24 smithi130 bash[51302]: debug 2023-08-28T01:24:24.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:24.532362+0000) 2023-08-28T01:24:25.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:25 smithi130 bash[27579]: cluster 2023-08-28T01:24:24.042446+0000 mgr.y (mgr.14901) 6174 : cluster [DBG] pgmap v6181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:25.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:25 smithi130 bash[27579]: audit 2023-08-28T01:24:25.038425+0000 mon.a (mon.0) 4131 : audit [DBG] from='client.? 172.21.15.6:0/2245407474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:25.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:25 smithi006 bash[27292]: cluster 2023-08-28T01:24:24.042446+0000 mgr.y (mgr.14901) 6174 : cluster [DBG] pgmap v6181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:25.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:25 smithi006 bash[27292]: audit 2023-08-28T01:24:25.038425+0000 mon.a (mon.0) 4131 : audit [DBG] from='client.? 172.21.15.6:0/2245407474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:25.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:25 smithi006 bash[33918]: cluster 2023-08-28T01:24:24.042446+0000 mgr.y (mgr.14901) 6174 : cluster [DBG] pgmap v6181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:25.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:25 smithi006 bash[33918]: audit 2023-08-28T01:24:25.038425+0000 mon.a (mon.0) 4131 : audit [DBG] from='client.? 172.21.15.6:0/2245407474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:25.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:25 smithi130 bash[51302]: debug 2023-08-28T01:24:25.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:25.532673+0000) 2023-08-28T01:24:26.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:26 smithi130 bash[51302]: debug 2023-08-28T01:24:26.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:26.532994+0000) 2023-08-28T01:24:27.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:27 smithi130 bash[27579]: cluster 2023-08-28T01:24:26.043664+0000 mgr.y (mgr.14901) 6175 : cluster [DBG] pgmap v6182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:27.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:27 smithi006 bash[27292]: cluster 2023-08-28T01:24:26.043664+0000 mgr.y (mgr.14901) 6175 : cluster [DBG] pgmap v6182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:27.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:27 smithi006 bash[33918]: cluster 2023-08-28T01:24:26.043664+0000 mgr.y (mgr.14901) 6175 : cluster [DBG] pgmap v6182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:27.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:27 smithi130 bash[51302]: debug 2023-08-28T01:24:27.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:27.533258+0000) 2023-08-28T01:24:28.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:28 smithi130 bash[27579]: audit 2023-08-28T01:24:27.455320+0000 mon.c (mon.2) 2424 : audit [DBG] from='client.? 172.21.15.6:0/3943157644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:28.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:28 smithi006 bash[27292]: audit 2023-08-28T01:24:27.455320+0000 mon.c (mon.2) 2424 : audit [DBG] from='client.? 172.21.15.6:0/3943157644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:28.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:28 smithi006 bash[33918]: audit 2023-08-28T01:24:27.455320+0000 mon.c (mon.2) 2424 : audit [DBG] from='client.? 172.21.15.6:0/3943157644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:28.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:28 smithi130 bash[51302]: debug 2023-08-28T01:24:28.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:28.533523+0000) 2023-08-28T01:24:29.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:29 smithi006 bash[27292]: cluster 2023-08-28T01:24:28.044291+0000 mgr.y (mgr.14901) 6176 : cluster [DBG] pgmap v6183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:29.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:29 smithi006 bash[33918]: cluster 2023-08-28T01:24:28.044291+0000 mgr.y (mgr.14901) 6176 : cluster [DBG] pgmap v6183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:29.532 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:29 smithi130 bash[27579]: cluster 2023-08-28T01:24:28.044291+0000 mgr.y (mgr.14901) 6176 : cluster [DBG] pgmap v6183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:29.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:29 smithi130 bash[51302]: debug 2023-08-28T01:24:29.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:29.533820+0000) 2023-08-28T01:24:30.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:30 smithi006 bash[27292]: audit 2023-08-28T01:24:29.857401+0000 mon.a (mon.0) 4132 : audit [DBG] from='client.? 172.21.15.6:0/86140908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:30.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:30 smithi006 bash[33918]: audit 2023-08-28T01:24:29.857401+0000 mon.a (mon.0) 4132 : audit [DBG] from='client.? 172.21.15.6:0/86140908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:30.533 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:30 smithi130 bash[27579]: audit 2023-08-28T01:24:29.857401+0000 mon.a (mon.0) 4132 : audit [DBG] from='client.? 172.21.15.6:0/86140908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:30.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:30 smithi130 bash[51302]: debug 2023-08-28T01:24:30.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:30.534173+0000) 2023-08-28T01:24:31.007 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:24:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:24:30] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:24:31.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:31 smithi006 bash[33918]: cluster 2023-08-28T01:24:30.045468+0000 mgr.y (mgr.14901) 6177 : cluster [DBG] pgmap v6184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:31.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:24:31.006Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:31.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:24:31.008Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:31.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:31 smithi006 bash[27292]: cluster 2023-08-28T01:24:30.045468+0000 mgr.y (mgr.14901) 6177 : cluster [DBG] pgmap v6184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:31.533 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:31 smithi130 bash[27579]: cluster 2023-08-28T01:24:30.045468+0000 mgr.y (mgr.14901) 6177 : cluster [DBG] pgmap v6184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:31.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:31 smithi130 bash[51302]: debug 2023-08-28T01:24:31.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:31.534464+0000) 2023-08-28T01:24:32.855 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:24:32.578Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:32.855 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:24:32.580Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:32.855 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:32 smithi130 bash[51302]: debug 2023-08-28T01:24:32.528+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:32.534763+0000) 2023-08-28T01:24:33.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:32 smithi130 bash[27579]: cluster 2023-08-28T01:24:32.046089+0000 mgr.y (mgr.14901) 6178 : cluster [DBG] pgmap v6185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:32 smithi130 bash[27579]: audit 2023-08-28T01:24:32.274783+0000 mon.c (mon.2) 2425 : audit [DBG] from='client.? 172.21.15.6:0/3962495643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:33.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:32 smithi006 bash[33918]: cluster 2023-08-28T01:24:32.046089+0000 mgr.y (mgr.14901) 6178 : cluster [DBG] pgmap v6185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:33.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:32 smithi006 bash[33918]: audit 2023-08-28T01:24:32.274783+0000 mon.c (mon.2) 2425 : audit [DBG] from='client.? 172.21.15.6:0/3962495643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:33.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:32 smithi006 bash[27292]: cluster 2023-08-28T01:24:32.046089+0000 mgr.y (mgr.14901) 6178 : cluster [DBG] pgmap v6185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:33.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:32 smithi006 bash[27292]: audit 2023-08-28T01:24:32.274783+0000 mon.c (mon.2) 2425 : audit [DBG] from='client.? 172.21.15.6:0/3962495643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:33.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:33 smithi130 bash[51302]: debug 2023-08-28T01:24:33.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:33.535047+0000) 2023-08-28T01:24:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:24:34.458Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:24:34.460Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:34.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:34 smithi130 bash[51302]: debug 2023-08-28T01:24:34.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:34.535281+0000) 2023-08-28T01:24:35.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:35 smithi130 bash[27579]: cluster 2023-08-28T01:24:34.046736+0000 mgr.y (mgr.14901) 6179 : cluster [DBG] pgmap v6186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:35.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:35 smithi130 bash[27579]: audit 2023-08-28T01:24:34.697398+0000 mon.c (mon.2) 2426 : audit [DBG] from='client.? 172.21.15.6:0/4269160635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:35.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:35 smithi006 bash[33918]: cluster 2023-08-28T01:24:34.046736+0000 mgr.y (mgr.14901) 6179 : cluster [DBG] pgmap v6186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:35.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:35 smithi006 bash[33918]: audit 2023-08-28T01:24:34.697398+0000 mon.c (mon.2) 2426 : audit [DBG] from='client.? 172.21.15.6:0/4269160635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:35.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:35 smithi006 bash[27292]: cluster 2023-08-28T01:24:34.046736+0000 mgr.y (mgr.14901) 6179 : cluster [DBG] pgmap v6186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:35.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:35 smithi006 bash[27292]: audit 2023-08-28T01:24:34.697398+0000 mon.c (mon.2) 2426 : audit [DBG] from='client.? 172.21.15.6:0/4269160635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:35.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:35 smithi130 bash[51302]: debug 2023-08-28T01:24:35.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:35.535523+0000) 2023-08-28T01:24:36.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:36 smithi130 bash[51302]: debug 2023-08-28T01:24:36.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:36.535817+0000) 2023-08-28T01:24:37.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:37 smithi130 bash[27579]: cluster 2023-08-28T01:24:36.047872+0000 mgr.y (mgr.14901) 6180 : cluster [DBG] pgmap v6187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:37.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:37 smithi006 bash[27292]: cluster 2023-08-28T01:24:36.047872+0000 mgr.y (mgr.14901) 6180 : cluster [DBG] pgmap v6187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:37.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:37 smithi006 bash[33918]: cluster 2023-08-28T01:24:36.047872+0000 mgr.y (mgr.14901) 6180 : cluster [DBG] pgmap v6187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:37.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:37 smithi130 bash[51302]: debug 2023-08-28T01:24:37.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:37.536098+0000) 2023-08-28T01:24:38.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:38 smithi006 bash[27292]: audit 2023-08-28T01:24:37.116733+0000 mon.c (mon.2) 2427 : audit [DBG] from='client.? 172.21.15.6:0/3327445029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:38.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:38 smithi006 bash[33918]: audit 2023-08-28T01:24:37.116733+0000 mon.c (mon.2) 2427 : audit [DBG] from='client.? 172.21.15.6:0/3327445029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:38.535 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:38 smithi130 bash[27579]: audit 2023-08-28T01:24:37.116733+0000 mon.c (mon.2) 2427 : audit [DBG] from='client.? 172.21.15.6:0/3327445029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:38.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:38 smithi130 bash[51302]: debug 2023-08-28T01:24:38.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:38.536389+0000) 2023-08-28T01:24:39.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:39 smithi006 bash[27292]: cluster 2023-08-28T01:24:38.048637+0000 mgr.y (mgr.14901) 6181 : cluster [DBG] pgmap v6188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:39.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:39 smithi006 bash[33918]: cluster 2023-08-28T01:24:38.048637+0000 mgr.y (mgr.14901) 6181 : cluster [DBG] pgmap v6188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:39.535 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:39 smithi130 bash[27579]: cluster 2023-08-28T01:24:38.048637+0000 mgr.y (mgr.14901) 6181 : cluster [DBG] pgmap v6188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:39.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:39 smithi130 bash[51302]: debug 2023-08-28T01:24:39.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:39.536674+0000) 2023-08-28T01:24:40.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:40 smithi006 bash[27292]: audit 2023-08-28T01:24:39.524120+0000 mon.c (mon.2) 2428 : audit [DBG] from='client.? 172.21.15.6:0/2605187169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:40.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:40 smithi006 bash[33918]: audit 2023-08-28T01:24:39.524120+0000 mon.c (mon.2) 2428 : audit [DBG] from='client.? 172.21.15.6:0/2605187169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:40.535 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:40 smithi130 bash[27579]: audit 2023-08-28T01:24:39.524120+0000 mon.c (mon.2) 2428 : audit [DBG] from='client.? 172.21.15.6:0/2605187169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:40.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:40 smithi130 bash[51302]: debug 2023-08-28T01:24:40.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:40.536977+0000) 2023-08-28T01:24:41.007 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:24:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:24:40] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:24:41.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:41 smithi006 bash[33918]: cluster 2023-08-28T01:24:40.049842+0000 mgr.y (mgr.14901) 6182 : cluster [DBG] pgmap v6189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:41.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:41 smithi006 bash[27292]: cluster 2023-08-28T01:24:40.049842+0000 mgr.y (mgr.14901) 6182 : cluster [DBG] pgmap v6189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:41.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:24:41.006Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:41.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:24:41.008Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:41.536 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:41 smithi130 bash[27579]: cluster 2023-08-28T01:24:40.049842+0000 mgr.y (mgr.14901) 6182 : cluster [DBG] pgmap v6189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:41.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:41 smithi130 bash[51302]: debug 2023-08-28T01:24:41.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:41.537289+0000) 2023-08-28T01:24:42.860 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:24:42.578Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:42.860 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:24:42.580Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:42.861 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:42 smithi130 bash[51302]: debug 2023-08-28T01:24:42.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:42.537601+0000) 2023-08-28T01:24:43.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:42 smithi130 bash[27579]: audit 2023-08-28T01:24:41.944616+0000 mon.c (mon.2) 2429 : audit [DBG] from='client.? 172.21.15.6:0/490484998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:42 smithi130 bash[27579]: cluster 2023-08-28T01:24:42.050518+0000 mgr.y (mgr.14901) 6183 : cluster [DBG] pgmap v6190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:43.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:42 smithi006 bash[33918]: audit 2023-08-28T01:24:41.944616+0000 mon.c (mon.2) 2429 : audit [DBG] from='client.? 172.21.15.6:0/490484998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:43.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:42 smithi006 bash[33918]: cluster 2023-08-28T01:24:42.050518+0000 mgr.y (mgr.14901) 6183 : cluster [DBG] pgmap v6190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:43.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:42 smithi006 bash[27292]: audit 2023-08-28T01:24:41.944616+0000 mon.c (mon.2) 2429 : audit [DBG] from='client.? 172.21.15.6:0/490484998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:43.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:42 smithi006 bash[27292]: cluster 2023-08-28T01:24:42.050518+0000 mgr.y (mgr.14901) 6183 : cluster [DBG] pgmap v6190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:43.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:43 smithi130 bash[51302]: debug 2023-08-28T01:24:43.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:43.537914+0000) 2023-08-28T01:24:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:24:44.459Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:24:44.461Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:44.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:44 smithi130 bash[51302]: debug 2023-08-28T01:24:44.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:44.538202+0000) 2023-08-28T01:24:45.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:45 smithi130 bash[27579]: cluster 2023-08-28T01:24:44.050992+0000 mgr.y (mgr.14901) 6184 : cluster [DBG] pgmap v6191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:45.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:45 smithi130 bash[27579]: audit 2023-08-28T01:24:44.362065+0000 mon.c (mon.2) 2430 : audit [DBG] from='client.? 172.21.15.6:0/2451693764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:45.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:45 smithi006 bash[33918]: cluster 2023-08-28T01:24:44.050992+0000 mgr.y (mgr.14901) 6184 : cluster [DBG] pgmap v6191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:45.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:45 smithi006 bash[33918]: audit 2023-08-28T01:24:44.362065+0000 mon.c (mon.2) 2430 : audit [DBG] from='client.? 172.21.15.6:0/2451693764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:45.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:45 smithi006 bash[27292]: cluster 2023-08-28T01:24:44.050992+0000 mgr.y (mgr.14901) 6184 : cluster [DBG] pgmap v6191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:45.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:45 smithi006 bash[27292]: audit 2023-08-28T01:24:44.362065+0000 mon.c (mon.2) 2430 : audit [DBG] from='client.? 172.21.15.6:0/2451693764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:45.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:45 smithi130 bash[51302]: debug 2023-08-28T01:24:45.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:45.538484+0000) 2023-08-28T01:24:46.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:46 smithi130 bash[51302]: debug 2023-08-28T01:24:46.532+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:46.538769+0000) 2023-08-28T01:24:47.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:47 smithi130 bash[27579]: cluster 2023-08-28T01:24:46.052072+0000 mgr.y (mgr.14901) 6185 : cluster [DBG] pgmap v6192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:47.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:47 smithi130 bash[27579]: audit 2023-08-28T01:24:46.774447+0000 mon.a (mon.0) 4133 : audit [DBG] from='client.? 172.21.15.6:0/3329733451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:47.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:47 smithi006 bash[33918]: cluster 2023-08-28T01:24:46.052072+0000 mgr.y (mgr.14901) 6185 : cluster [DBG] pgmap v6192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:47.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:47 smithi006 bash[33918]: audit 2023-08-28T01:24:46.774447+0000 mon.a (mon.0) 4133 : audit [DBG] from='client.? 172.21.15.6:0/3329733451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:47.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:47 smithi006 bash[27292]: cluster 2023-08-28T01:24:46.052072+0000 mgr.y (mgr.14901) 6185 : cluster [DBG] pgmap v6192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:47.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:47 smithi006 bash[27292]: audit 2023-08-28T01:24:46.774447+0000 mon.a (mon.0) 4133 : audit [DBG] from='client.? 172.21.15.6:0/3329733451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:47.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:47 smithi130 bash[51302]: debug 2023-08-28T01:24:47.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:47.539091+0000) 2023-08-28T01:24:48.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:48 smithi130 bash[51302]: debug 2023-08-28T01:24:48.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:48.539372+0000) 2023-08-28T01:24:49.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:49 smithi006 bash[33918]: cluster 2023-08-28T01:24:48.052688+0000 mgr.y (mgr.14901) 6186 : cluster [DBG] pgmap v6193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:49.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:49 smithi006 bash[27292]: cluster 2023-08-28T01:24:48.052688+0000 mgr.y (mgr.14901) 6186 : cluster [DBG] pgmap v6193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:49.538 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:49 smithi130 bash[27579]: cluster 2023-08-28T01:24:48.052688+0000 mgr.y (mgr.14901) 6186 : cluster [DBG] pgmap v6193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:49.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:49 smithi130 bash[51302]: debug 2023-08-28T01:24:49.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:49.539663+0000) 2023-08-28T01:24:50.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:50 smithi006 bash[33918]: audit 2023-08-28T01:24:49.188918+0000 mon.a (mon.0) 4134 : audit [DBG] from='client.? 172.21.15.6:0/1443037645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:50.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:50 smithi006 bash[27292]: audit 2023-08-28T01:24:49.188918+0000 mon.a (mon.0) 4134 : audit [DBG] from='client.? 172.21.15.6:0/1443037645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:50.539 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:50 smithi130 bash[27579]: audit 2023-08-28T01:24:49.188918+0000 mon.a (mon.0) 4134 : audit [DBG] from='client.? 172.21.15.6:0/1443037645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:50.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:50 smithi130 bash[51302]: debug 2023-08-28T01:24:50.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:50.539945+0000) 2023-08-28T01:24:51.008 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:24:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:24:50] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:24:51.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:51 smithi006 bash[27292]: cluster 2023-08-28T01:24:50.053843+0000 mgr.y (mgr.14901) 6187 : cluster [DBG] pgmap v6194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:51.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:51 smithi006 bash[33918]: cluster 2023-08-28T01:24:50.053843+0000 mgr.y (mgr.14901) 6187 : cluster [DBG] pgmap v6194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:51.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:24:51.007Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:51.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:24:51.009Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:51.539 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:51 smithi130 bash[27579]: cluster 2023-08-28T01:24:50.053843+0000 mgr.y (mgr.14901) 6187 : cluster [DBG] pgmap v6194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:51.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:51 smithi130 bash[51302]: debug 2023-08-28T01:24:51.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:51.540263+0000) 2023-08-28T01:24:52.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:52 smithi006 bash[33918]: audit 2023-08-28T01:24:51.602126+0000 mon.a (mon.0) 4135 : audit [DBG] from='client.? 172.21.15.6:0/1924387669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:52.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:52 smithi006 bash[27292]: audit 2023-08-28T01:24:51.602126+0000 mon.a (mon.0) 4135 : audit [DBG] from='client.? 172.21.15.6:0/1924387669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:52.539 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:52 smithi130 bash[27579]: audit 2023-08-28T01:24:51.602126+0000 mon.a (mon.0) 4135 : audit [DBG] from='client.? 172.21.15.6:0/1924387669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:52.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:52 smithi130 bash[51302]: debug 2023-08-28T01:24:52.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:52.540566+0000) 2023-08-28T01:24:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:24:52.579Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:24:52.582Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:53.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:53 smithi006 bash[33918]: cluster 2023-08-28T01:24:52.054535+0000 mgr.y (mgr.14901) 6188 : cluster [DBG] pgmap v6195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:53.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:53 smithi006 bash[27292]: cluster 2023-08-28T01:24:52.054535+0000 mgr.y (mgr.14901) 6188 : cluster [DBG] pgmap v6195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:53.539 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:53 smithi130 bash[27579]: cluster 2023-08-28T01:24:52.054535+0000 mgr.y (mgr.14901) 6188 : cluster [DBG] pgmap v6195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:53.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:53 smithi130 bash[51302]: debug 2023-08-28T01:24:53.540+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:53.540843+0000) 2023-08-28T01:24:54.461 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:54 smithi006 bash[27292]: audit 2023-08-28T01:24:54.018451+0000 mon.a (mon.0) 4136 : audit [DBG] from='client.? 172.21.15.6:0/3237174084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:54.461 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:54 smithi006 bash[33918]: audit 2023-08-28T01:24:54.018451+0000 mon.a (mon.0) 4136 : audit [DBG] from='client.? 172.21.15.6:0/3237174084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:54.540 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:54 smithi130 bash[27579]: audit 2023-08-28T01:24:54.018451+0000 mon.a (mon.0) 4136 : audit [DBG] from='client.? 172.21.15.6:0/3237174084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:24:54.459Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:24:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:24:54.461Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:24:54.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:54 smithi130 bash[51302]: debug 2023-08-28T01:24:54.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:54.541114+0000) 2023-08-28T01:24:55.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:55 smithi006 bash[27292]: cluster 2023-08-28T01:24:54.055171+0000 mgr.y (mgr.14901) 6189 : cluster [DBG] pgmap v6196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:55.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:55 smithi006 bash[33918]: cluster 2023-08-28T01:24:54.055171+0000 mgr.y (mgr.14901) 6189 : cluster [DBG] pgmap v6196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:55.540 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:55 smithi130 bash[27579]: cluster 2023-08-28T01:24:54.055171+0000 mgr.y (mgr.14901) 6189 : cluster [DBG] pgmap v6196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:55.540 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:55 smithi130 bash[51302]: debug 2023-08-28T01:24:55.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:55.541344+0000) 2023-08-28T01:24:56.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:56 smithi006 bash[33918]: cluster 2023-08-28T01:24:56.056280+0000 mgr.y (mgr.14901) 6190 : cluster [DBG] pgmap v6197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:56.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:56 smithi006 bash[33918]: audit 2023-08-28T01:24:56.114367+0000 mon.b (mon.1) 710 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:24:56.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:56 smithi006 bash[27292]: cluster 2023-08-28T01:24:56.056280+0000 mgr.y (mgr.14901) 6190 : cluster [DBG] pgmap v6197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:56.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:56 smithi006 bash[27292]: audit 2023-08-28T01:24:56.114367+0000 mon.b (mon.1) 710 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:24:56.540 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:56 smithi130 bash[27579]: cluster 2023-08-28T01:24:56.056280+0000 mgr.y (mgr.14901) 6190 : cluster [DBG] pgmap v6197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:56.540 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:56 smithi130 bash[27579]: audit 2023-08-28T01:24:56.114367+0000 mon.b (mon.1) 710 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:24:56.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:56 smithi130 bash[51302]: debug 2023-08-28T01:24:56.540+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:56.541684+0000) 2023-08-28T01:24:57.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:57 smithi006 bash[33918]: audit 2023-08-28T01:24:56.446583+0000 mon.c (mon.2) 2431 : audit [DBG] from='client.? 172.21.15.6:0/1487358047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:57.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:57 smithi006 bash[27292]: audit 2023-08-28T01:24:56.446583+0000 mon.c (mon.2) 2431 : audit [DBG] from='client.? 172.21.15.6:0/1487358047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:57.541 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:57 smithi130 bash[27579]: audit 2023-08-28T01:24:56.446583+0000 mon.c (mon.2) 2431 : audit [DBG] from='client.? 172.21.15.6:0/1487358047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:57.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:57 smithi130 bash[51302]: debug 2023-08-28T01:24:57.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:57.542053+0000) 2023-08-28T01:24:58.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:58 smithi006 bash[33918]: cluster 2023-08-28T01:24:58.057095+0000 mgr.y (mgr.14901) 6191 : cluster [DBG] pgmap v6198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:58.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:58 smithi006 bash[27292]: cluster 2023-08-28T01:24:58.057095+0000 mgr.y (mgr.14901) 6191 : cluster [DBG] pgmap v6198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:58.541 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:58 smithi130 bash[27579]: cluster 2023-08-28T01:24:58.057095+0000 mgr.y (mgr.14901) 6191 : cluster [DBG] pgmap v6198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:24:58.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:58 smithi130 bash[51302]: debug 2023-08-28T01:24:58.536+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:58.542387+0000) 2023-08-28T01:24:59.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:24:59 smithi006 bash[33918]: audit 2023-08-28T01:24:58.858739+0000 mon.a (mon.0) 4137 : audit [DBG] from='client.? 172.21.15.6:0/3202488291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:59.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:24:59 smithi006 bash[27292]: audit 2023-08-28T01:24:58.858739+0000 mon.a (mon.0) 4137 : audit [DBG] from='client.? 172.21.15.6:0/3202488291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:59.541 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:24:59 smithi130 bash[27579]: audit 2023-08-28T01:24:58.858739+0000 mon.a (mon.0) 4137 : audit [DBG] from='client.? 172.21.15.6:0/3202488291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:59.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:24:59 smithi130 bash[51302]: debug 2023-08-28T01:24:59.540+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:59.542726+0000) 2023-08-28T01:25:00.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:00 smithi006 bash[33918]: cluster 2023-08-28T01:25:00.057864+0000 mgr.y (mgr.14901) 6192 : cluster [DBG] pgmap v6199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:00.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:00 smithi006 bash[27292]: cluster 2023-08-28T01:25:00.057864+0000 mgr.y (mgr.14901) 6192 : cluster [DBG] pgmap v6199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:00.542 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:00 smithi130 bash[27579]: cluster 2023-08-28T01:25:00.057864+0000 mgr.y (mgr.14901) 6192 : cluster [DBG] pgmap v6199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:00.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:00 smithi130 bash[51302]: debug 2023-08-28T01:25:00.540+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:00.543029+0000) 2023-08-28T01:25:01.008 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:25:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:25:00] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:25:01.333 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:25:01.007Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:01.333 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:25:01.009Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:01.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:01 smithi130 bash[27579]: audit 2023-08-28T01:25:01.279699+0000 mon.c (mon.2) 2432 : audit [DBG] from='client.? 172.21.15.6:0/3344564078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:01.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:01 smithi130 bash[51302]: debug 2023-08-28T01:25:01.540+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:01.543278+0000) 2023-08-28T01:25:01.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:01 smithi006 bash[27292]: audit 2023-08-28T01:25:01.279699+0000 mon.c (mon.2) 2432 : audit [DBG] from='client.? 172.21.15.6:0/3344564078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:01.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:01 smithi006 bash[33918]: audit 2023-08-28T01:25:01.279699+0000 mon.c (mon.2) 2432 : audit [DBG] from='client.? 172.21.15.6:0/3344564078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:02.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:02 smithi130 bash[51302]: debug 2023-08-28T01:25:02.539+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:02.543595+0000) 2023-08-28T01:25:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:25:02.580Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:25:02.582Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:03.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:02 smithi130 bash[27579]: audit 2023-08-28T01:25:01.957963+0000 mon.a (mon.0) 4138 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:25:03.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:02 smithi130 bash[27579]: cluster 2023-08-28T01:25:02.058311+0000 mgr.y (mgr.14901) 6193 : cluster [DBG] pgmap v6200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:03.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:02 smithi130 bash[27579]: audit 2023-08-28T01:25:02.449644+0000 mon.a (mon.0) 4139 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:25:03.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:02 smithi006 bash[27292]: audit 2023-08-28T01:25:01.957963+0000 mon.a (mon.0) 4138 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:25:03.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:02 smithi006 bash[27292]: cluster 2023-08-28T01:25:02.058311+0000 mgr.y (mgr.14901) 6193 : cluster [DBG] pgmap v6200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:03.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:02 smithi006 bash[27292]: audit 2023-08-28T01:25:02.449644+0000 mon.a (mon.0) 4139 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:25:03.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:02 smithi006 bash[33918]: audit 2023-08-28T01:25:01.957963+0000 mon.a (mon.0) 4138 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:25:03.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:02 smithi006 bash[33918]: cluster 2023-08-28T01:25:02.058311+0000 mgr.y (mgr.14901) 6193 : cluster [DBG] pgmap v6200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:03.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:02 smithi006 bash[33918]: audit 2023-08-28T01:25:02.449644+0000 mon.a (mon.0) 4139 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:25:03.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:03 smithi130 bash[51302]: debug 2023-08-28T01:25:03.539+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:03.543903+0000) 2023-08-28T01:25:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:03 smithi130 bash[27579]: audit 2023-08-28T01:25:03.696775+0000 mon.a (mon.0) 4140 : audit [DBG] from='client.? 172.21.15.6:0/856489941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:04.461 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:03 smithi006 bash[27292]: audit 2023-08-28T01:25:03.696775+0000 mon.a (mon.0) 4140 : audit [DBG] from='client.? 172.21.15.6:0/856489941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:04.462 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:03 smithi006 bash[33918]: audit 2023-08-28T01:25:03.696775+0000 mon.a (mon.0) 4140 : audit [DBG] from='client.? 172.21.15.6:0/856489941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:25:04.460Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:25:04.462Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:04.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:04 smithi130 bash[51302]: debug 2023-08-28T01:25:04.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:04.544238+0000) 2023-08-28T01:25:05.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:04 smithi130 bash[27579]: cluster 2023-08-28T01:25:04.058963+0000 mgr.y (mgr.14901) 6194 : cluster [DBG] pgmap v6201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:05.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:04 smithi006 bash[27292]: cluster 2023-08-28T01:25:04.058963+0000 mgr.y (mgr.14901) 6194 : cluster [DBG] pgmap v6201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:05.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:04 smithi006 bash[33918]: cluster 2023-08-28T01:25:04.058963+0000 mgr.y (mgr.14901) 6194 : cluster [DBG] pgmap v6201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:05.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:05 smithi130 bash[51302]: debug 2023-08-28T01:25:05.539+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:05.544554+0000) 2023-08-28T01:25:06.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:06 smithi006 bash[33918]: cluster 2023-08-28T01:25:06.060122+0000 mgr.y (mgr.14901) 6195 : cluster [DBG] pgmap v6202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:06.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:06 smithi006 bash[33918]: audit 2023-08-28T01:25:06.109084+0000 mon.a (mon.0) 4141 : audit [DBG] from='client.? 172.21.15.6:0/1060870261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:06.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:06 smithi006 bash[27292]: cluster 2023-08-28T01:25:06.060122+0000 mgr.y (mgr.14901) 6195 : cluster [DBG] pgmap v6202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:06.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:06 smithi006 bash[27292]: audit 2023-08-28T01:25:06.109084+0000 mon.a (mon.0) 4141 : audit [DBG] from='client.? 172.21.15.6:0/1060870261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:06.544 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:06 smithi130 bash[27579]: cluster 2023-08-28T01:25:06.060122+0000 mgr.y (mgr.14901) 6195 : cluster [DBG] pgmap v6202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:06.544 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:06 smithi130 bash[27579]: audit 2023-08-28T01:25:06.109084+0000 mon.a (mon.0) 4141 : audit [DBG] from='client.? 172.21.15.6:0/1060870261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:06.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:06 smithi130 bash[51302]: debug 2023-08-28T01:25:06.539+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:06.544919+0000) 2023-08-28T01:25:08.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:07 smithi130 bash[51302]: debug 2023-08-28T01:25:07.539+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:07.545236+0000) 2023-08-28T01:25:08.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:08 smithi130 bash[51302]: debug 2023-08-28T01:25:08.539+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:08.545506+0000) 2023-08-28T01:25:09.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:09 smithi130 bash[27579]: cluster 2023-08-28T01:25:08.060802+0000 mgr.y (mgr.14901) 6196 : cluster [DBG] pgmap v6203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:09.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:09 smithi130 bash[27579]: audit 2023-08-28T01:25:08.520479+0000 mon.a (mon.0) 4142 : audit [DBG] from='client.? 172.21.15.6:0/4017517331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:09.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:09 smithi006 bash[33918]: cluster 2023-08-28T01:25:08.060802+0000 mgr.y (mgr.14901) 6196 : cluster [DBG] pgmap v6203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:09.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:09 smithi006 bash[33918]: audit 2023-08-28T01:25:08.520479+0000 mon.a (mon.0) 4142 : audit [DBG] from='client.? 172.21.15.6:0/4017517331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:09.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:09 smithi006 bash[27292]: cluster 2023-08-28T01:25:08.060802+0000 mgr.y (mgr.14901) 6196 : cluster [DBG] pgmap v6203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:09.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:09 smithi006 bash[27292]: audit 2023-08-28T01:25:08.520479+0000 mon.a (mon.0) 4142 : audit [DBG] from='client.? 172.21.15.6:0/4017517331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:09.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:09 smithi130 bash[51302]: debug 2023-08-28T01:25:09.539+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:09.545744+0000) 2023-08-28T01:25:10.806 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:10 smithi130 bash[51302]: debug 2023-08-28T01:25:10.539+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:10.546064+0000) 2023-08-28T01:25:11.009 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:25:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:25:10] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:25:11.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:11 smithi006 bash[27292]: cluster 2023-08-28T01:25:10.061981+0000 mgr.y (mgr.14901) 6197 : cluster [DBG] pgmap v6204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:11.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:11 smithi006 bash[27292]: audit 2023-08-28T01:25:10.934307+0000 mon.c (mon.2) 2433 : audit [DBG] from='client.? 172.21.15.6:0/2696473847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:11.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:11 smithi006 bash[33918]: cluster 2023-08-28T01:25:10.061981+0000 mgr.y (mgr.14901) 6197 : cluster [DBG] pgmap v6204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:11.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:11 smithi006 bash[33918]: audit 2023-08-28T01:25:10.934307+0000 mon.c (mon.2) 2433 : audit [DBG] from='client.? 172.21.15.6:0/2696473847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:11.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:25:11.008Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:11.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:25:11.010Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:11.545 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:11 smithi130 bash[27579]: cluster 2023-08-28T01:25:10.061981+0000 mgr.y (mgr.14901) 6197 : cluster [DBG] pgmap v6204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:11.545 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:11 smithi130 bash[27579]: audit 2023-08-28T01:25:10.934307+0000 mon.c (mon.2) 2433 : audit [DBG] from='client.? 172.21.15.6:0/2696473847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:11.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:11 smithi130 bash[51302]: debug 2023-08-28T01:25:11.539+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:11.546282+0000) 2023-08-28T01:25:12.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:12 smithi130 bash[51302]: debug 2023-08-28T01:25:12.539+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:12.546575+0000) 2023-08-28T01:25:12.869 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:25:12.581Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:12.869 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:25:12.583Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:12 smithi006 bash[33918]: cluster 2023-08-28T01:25:12.062669+0000 mgr.y (mgr.14901) 6198 : cluster [DBG] pgmap v6205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:12 smithi006 bash[33918]: audit 2023-08-28T01:25:12.622117+0000 mon.a (mon.0) 4143 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:25:13.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:12 smithi006 bash[27292]: cluster 2023-08-28T01:25:12.062669+0000 mgr.y (mgr.14901) 6198 : cluster [DBG] pgmap v6205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:13.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:12 smithi006 bash[27292]: audit 2023-08-28T01:25:12.622117+0000 mon.a (mon.0) 4143 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:25:13.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:12 smithi130 bash[27579]: cluster 2023-08-28T01:25:12.062669+0000 mgr.y (mgr.14901) 6198 : cluster [DBG] pgmap v6205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:13.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:12 smithi130 bash[27579]: audit 2023-08-28T01:25:12.622117+0000 mon.a (mon.0) 4143 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:25:13.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:13 smithi130 bash[51302]: debug 2023-08-28T01:25:13.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:13.546929+0000) 2023-08-28T01:25:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:13 smithi006 bash[27292]: audit 2023-08-28T01:25:12.884763+0000 mon.b (mon.1) 711 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:25:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:13 smithi006 bash[27292]: audit 2023-08-28T01:25:12.886477+0000 mon.b (mon.1) 712 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:25:14.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:13 smithi006 bash[27292]: audit 2023-08-28T01:25:12.893562+0000 mon.a (mon.0) 4144 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:25:14.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:13 smithi006 bash[27292]: audit 2023-08-28T01:25:13.350027+0000 mon.c (mon.2) 2434 : audit [DBG] from='client.? 172.21.15.6:0/3869785812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:14.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:13 smithi006 bash[33918]: audit 2023-08-28T01:25:12.884763+0000 mon.b (mon.1) 711 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:25:14.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:13 smithi006 bash[33918]: audit 2023-08-28T01:25:12.886477+0000 mon.b (mon.1) 712 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:25:14.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:13 smithi006 bash[33918]: audit 2023-08-28T01:25:12.893562+0000 mon.a (mon.0) 4144 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:25:14.217 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:13 smithi006 bash[33918]: audit 2023-08-28T01:25:13.350027+0000 mon.c (mon.2) 2434 : audit [DBG] from='client.? 172.21.15.6:0/3869785812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:14.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:13 smithi130 bash[27579]: audit 2023-08-28T01:25:12.884763+0000 mon.b (mon.1) 711 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:25:14.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:13 smithi130 bash[27579]: audit 2023-08-28T01:25:12.886477+0000 mon.b (mon.1) 712 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:25:14.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:13 smithi130 bash[27579]: audit 2023-08-28T01:25:12.893562+0000 mon.a (mon.0) 4144 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:25:14.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:13 smithi130 bash[27579]: audit 2023-08-28T01:25:13.350027+0000 mon.c (mon.2) 2434 : audit [DBG] from='client.? 172.21.15.6:0/3869785812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:25:14.461Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:25:14.463Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:14.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:14 smithi130 bash[51302]: debug 2023-08-28T01:25:14.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:14.547293+0000) 2023-08-28T01:25:15.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:14 smithi006 bash[33918]: cluster 2023-08-28T01:25:14.063340+0000 mgr.y (mgr.14901) 6199 : cluster [DBG] pgmap v6206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:15.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:14 smithi006 bash[27292]: cluster 2023-08-28T01:25:14.063340+0000 mgr.y (mgr.14901) 6199 : cluster [DBG] pgmap v6206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:15.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:14 smithi130 bash[27579]: cluster 2023-08-28T01:25:14.063340+0000 mgr.y (mgr.14901) 6199 : cluster [DBG] pgmap v6206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:15.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:15 smithi130 bash[51302]: debug 2023-08-28T01:25:15.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:15.547570+0000) 2023-08-28T01:25:16.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:15 smithi006 bash[33918]: audit 2023-08-28T01:25:15.771428+0000 mon.c (mon.2) 2435 : audit [DBG] from='client.? 172.21.15.6:0/1392607867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:16.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:15 smithi006 bash[27292]: audit 2023-08-28T01:25:15.771428+0000 mon.c (mon.2) 2435 : audit [DBG] from='client.? 172.21.15.6:0/1392607867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:16.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:15 smithi130 bash[27579]: audit 2023-08-28T01:25:15.771428+0000 mon.c (mon.2) 2435 : audit [DBG] from='client.? 172.21.15.6:0/1392607867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:16.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:16 smithi130 bash[51302]: debug 2023-08-28T01:25:16.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:16.547904+0000) 2023-08-28T01:25:17.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:16 smithi006 bash[27292]: cluster 2023-08-28T01:25:16.064492+0000 mgr.y (mgr.14901) 6200 : cluster [DBG] pgmap v6207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:17.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:16 smithi006 bash[33918]: cluster 2023-08-28T01:25:16.064492+0000 mgr.y (mgr.14901) 6200 : cluster [DBG] pgmap v6207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:17.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:16 smithi130 bash[27579]: cluster 2023-08-28T01:25:16.064492+0000 mgr.y (mgr.14901) 6200 : cluster [DBG] pgmap v6207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:17.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:17 smithi130 bash[51302]: debug 2023-08-28T01:25:17.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:17.548241+0000) 2023-08-28T01:25:18.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:18 smithi130 bash[51302]: debug 2023-08-28T01:25:18.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:18.548586+0000) 2023-08-28T01:25:19.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:19 smithi006 bash[27292]: cluster 2023-08-28T01:25:18.065182+0000 mgr.y (mgr.14901) 6201 : cluster [DBG] pgmap v6208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:19.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:19 smithi006 bash[27292]: audit 2023-08-28T01:25:18.184443+0000 mon.a (mon.0) 4145 : audit [DBG] from='client.? 172.21.15.6:0/1764767668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:19.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:19 smithi006 bash[33918]: cluster 2023-08-28T01:25:18.065182+0000 mgr.y (mgr.14901) 6201 : cluster [DBG] pgmap v6208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:19.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:19 smithi006 bash[33918]: audit 2023-08-28T01:25:18.184443+0000 mon.a (mon.0) 4145 : audit [DBG] from='client.? 172.21.15.6:0/1764767668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:19.547 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:19 smithi130 bash[27579]: cluster 2023-08-28T01:25:18.065182+0000 mgr.y (mgr.14901) 6201 : cluster [DBG] pgmap v6208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:19.548 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:19 smithi130 bash[27579]: audit 2023-08-28T01:25:18.184443+0000 mon.a (mon.0) 4145 : audit [DBG] from='client.? 172.21.15.6:0/1764767668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:19.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:19 smithi130 bash[51302]: debug 2023-08-28T01:25:19.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:19.548855+0000) 2023-08-28T01:25:20.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:20 smithi130 bash[51302]: debug 2023-08-28T01:25:20.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:20.549191+0000) 2023-08-28T01:25:21.010 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:25:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:25:20] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T01:25:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:25:21.009Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:21.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:25:21.010Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:21.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:21 smithi006 bash[27292]: cluster 2023-08-28T01:25:20.066338+0000 mgr.y (mgr.14901) 6202 : cluster [DBG] pgmap v6209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:21.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:21 smithi006 bash[27292]: audit 2023-08-28T01:25:20.600324+0000 mon.a (mon.0) 4146 : audit [DBG] from='client.? 172.21.15.6:0/3384744860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:21.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:21 smithi006 bash[33918]: cluster 2023-08-28T01:25:20.066338+0000 mgr.y (mgr.14901) 6202 : cluster [DBG] pgmap v6209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:21.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:21 smithi006 bash[33918]: audit 2023-08-28T01:25:20.600324+0000 mon.a (mon.0) 4146 : audit [DBG] from='client.? 172.21.15.6:0/3384744860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:21.548 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:21 smithi130 bash[27579]: cluster 2023-08-28T01:25:20.066338+0000 mgr.y (mgr.14901) 6202 : cluster [DBG] pgmap v6209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:21.548 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:21 smithi130 bash[27579]: audit 2023-08-28T01:25:20.600324+0000 mon.a (mon.0) 4146 : audit [DBG] from='client.? 172.21.15.6:0/3384744860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:21.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:21 smithi130 bash[51302]: debug 2023-08-28T01:25:21.547+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:21.549556+0000) 2023-08-28T01:25:22.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:22 smithi130 bash[51302]: debug 2023-08-28T01:25:22.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:22.549884+0000) 2023-08-28T01:25:22.873 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:25:22.582Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:22.873 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:25:22.584Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:22.874 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:22 smithi006 bash[33918]: cluster 2023-08-28T01:25:22.067017+0000 mgr.y (mgr.14901) 6203 : cluster [DBG] pgmap v6210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:23.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:22 smithi006 bash[27292]: cluster 2023-08-28T01:25:22.067017+0000 mgr.y (mgr.14901) 6203 : cluster [DBG] pgmap v6210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:23.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:22 smithi130 bash[27579]: cluster 2023-08-28T01:25:22.067017+0000 mgr.y (mgr.14901) 6203 : cluster [DBG] pgmap v6210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:23.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:23 smithi130 bash[51302]: debug 2023-08-28T01:25:23.547+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:23.550216+0000) 2023-08-28T01:25:24.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:23 smithi006 bash[33918]: audit 2023-08-28T01:25:23.014255+0000 mon.c (mon.2) 2436 : audit [DBG] from='client.? 172.21.15.6:0/3824336703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:24.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:23 smithi006 bash[27292]: audit 2023-08-28T01:25:23.014255+0000 mon.c (mon.2) 2436 : audit [DBG] from='client.? 172.21.15.6:0/3824336703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:24.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:23 smithi130 bash[27579]: audit 2023-08-28T01:25:23.014255+0000 mon.c (mon.2) 2436 : audit [DBG] from='client.? 172.21.15.6:0/3824336703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:25:24.462Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:25:24.464Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:24.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:24 smithi130 bash[51302]: debug 2023-08-28T01:25:24.543+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:24.550550+0000) 2023-08-28T01:25:25.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:24 smithi006 bash[27292]: cluster 2023-08-28T01:25:24.067670+0000 mgr.y (mgr.14901) 6204 : cluster [DBG] pgmap v6211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:25.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:24 smithi006 bash[33918]: cluster 2023-08-28T01:25:24.067670+0000 mgr.y (mgr.14901) 6204 : cluster [DBG] pgmap v6211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:25.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:24 smithi130 bash[27579]: cluster 2023-08-28T01:25:24.067670+0000 mgr.y (mgr.14901) 6204 : cluster [DBG] pgmap v6211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:25.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:25 smithi130 bash[51302]: debug 2023-08-28T01:25:25.547+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:25.550909+0000) 2023-08-28T01:25:26.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:25 smithi006 bash[33918]: audit 2023-08-28T01:25:25.421447+0000 mon.c (mon.2) 2437 : audit [DBG] from='client.? 172.21.15.6:0/1769321668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:26.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:25 smithi006 bash[27292]: audit 2023-08-28T01:25:25.421447+0000 mon.c (mon.2) 2437 : audit [DBG] from='client.? 172.21.15.6:0/1769321668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:26.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:25 smithi130 bash[27579]: audit 2023-08-28T01:25:25.421447+0000 mon.c (mon.2) 2437 : audit [DBG] from='client.? 172.21.15.6:0/1769321668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:26.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:26 smithi130 bash[51302]: debug 2023-08-28T01:25:26.547+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:26.551264+0000) 2023-08-28T01:25:27.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:26 smithi006 bash[33918]: cluster 2023-08-28T01:25:26.068813+0000 mgr.y (mgr.14901) 6205 : cluster [DBG] pgmap v6212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:27.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:26 smithi006 bash[27292]: cluster 2023-08-28T01:25:26.068813+0000 mgr.y (mgr.14901) 6205 : cluster [DBG] pgmap v6212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:27.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:26 smithi130 bash[27579]: cluster 2023-08-28T01:25:26.068813+0000 mgr.y (mgr.14901) 6205 : cluster [DBG] pgmap v6212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:27.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:27 smithi130 bash[51302]: debug 2023-08-28T01:25:27.547+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:27.551603+0000) 2023-08-28T01:25:28.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:27 smithi006 bash[33918]: audit 2023-08-28T01:25:27.829167+0000 mon.c (mon.2) 2438 : audit [DBG] from='client.? 172.21.15.6:0/2016692009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:28.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:27 smithi006 bash[27292]: audit 2023-08-28T01:25:27.829167+0000 mon.c (mon.2) 2438 : audit [DBG] from='client.? 172.21.15.6:0/2016692009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:28.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:27 smithi130 bash[27579]: audit 2023-08-28T01:25:27.829167+0000 mon.c (mon.2) 2438 : audit [DBG] from='client.? 172.21.15.6:0/2016692009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:28.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:28 smithi130 bash[51302]: debug 2023-08-28T01:25:28.547+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:28.551934+0000) 2023-08-28T01:25:29.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:28 smithi006 bash[27292]: cluster 2023-08-28T01:25:28.069508+0000 mgr.y (mgr.14901) 6206 : cluster [DBG] pgmap v6213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:29.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:28 smithi006 bash[33918]: cluster 2023-08-28T01:25:28.069508+0000 mgr.y (mgr.14901) 6206 : cluster [DBG] pgmap v6213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:29.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:28 smithi130 bash[27579]: cluster 2023-08-28T01:25:28.069508+0000 mgr.y (mgr.14901) 6206 : cluster [DBG] pgmap v6213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:29.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:29 smithi130 bash[51302]: debug 2023-08-28T01:25:29.547+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:29.552273+0000) 2023-08-28T01:25:30.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:30 smithi130 bash[51302]: debug 2023-08-28T01:25:30.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:30.552638+0000) 2023-08-28T01:25:31.010 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:25:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:25:30] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:25:31.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:31 smithi006 bash[27292]: cluster 2023-08-28T01:25:30.070607+0000 mgr.y (mgr.14901) 6207 : cluster [DBG] pgmap v6214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:31.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:31 smithi006 bash[27292]: audit 2023-08-28T01:25:30.247518+0000 mon.a (mon.0) 4147 : audit [DBG] from='client.? 172.21.15.6:0/1036070195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:31.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:31 smithi006 bash[33918]: cluster 2023-08-28T01:25:30.070607+0000 mgr.y (mgr.14901) 6207 : cluster [DBG] pgmap v6214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:31.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:31 smithi006 bash[33918]: audit 2023-08-28T01:25:30.247518+0000 mon.a (mon.0) 4147 : audit [DBG] from='client.? 172.21.15.6:0/1036070195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:31.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:25:31.009Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:31.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:25:31.011Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:31.552 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:31 smithi130 bash[27579]: cluster 2023-08-28T01:25:30.070607+0000 mgr.y (mgr.14901) 6207 : cluster [DBG] pgmap v6214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:31.552 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:31 smithi130 bash[27579]: audit 2023-08-28T01:25:30.247518+0000 mon.a (mon.0) 4147 : audit [DBG] from='client.? 172.21.15.6:0/1036070195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:31.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:31 smithi130 bash[51302]: debug 2023-08-28T01:25:31.547+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:31.553027+0000) 2023-08-28T01:25:32.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:32 smithi130 bash[51302]: debug 2023-08-28T01:25:32.547+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:32.553314+0000) 2023-08-28T01:25:32.874 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:25:32.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:32.875 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:25:32.586Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:33.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:32 smithi006 bash[33918]: cluster 2023-08-28T01:25:32.071277+0000 mgr.y (mgr.14901) 6208 : cluster [DBG] pgmap v6215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:33.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:32 smithi006 bash[33918]: audit 2023-08-28T01:25:32.661940+0000 mon.a (mon.0) 4148 : audit [DBG] from='client.? 172.21.15.6:0/886041534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:33.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:32 smithi006 bash[27292]: cluster 2023-08-28T01:25:32.071277+0000 mgr.y (mgr.14901) 6208 : cluster [DBG] pgmap v6215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:33.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:32 smithi006 bash[27292]: audit 2023-08-28T01:25:32.661940+0000 mon.a (mon.0) 4148 : audit [DBG] from='client.? 172.21.15.6:0/886041534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:33.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:32 smithi130 bash[27579]: cluster 2023-08-28T01:25:32.071277+0000 mgr.y (mgr.14901) 6208 : cluster [DBG] pgmap v6215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:33.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:32 smithi130 bash[27579]: audit 2023-08-28T01:25:32.661940+0000 mon.a (mon.0) 4148 : audit [DBG] from='client.? 172.21.15.6:0/886041534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:33.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:33 smithi130 bash[51302]: debug 2023-08-28T01:25:33.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:33.553644+0000) 2023-08-28T01:25:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:25:34.463Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:25:34.465Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:34.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:34 smithi130 bash[51302]: debug 2023-08-28T01:25:34.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:34.553996+0000) 2023-08-28T01:25:35.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:35 smithi006 bash[27292]: cluster 2023-08-28T01:25:34.071932+0000 mgr.y (mgr.14901) 6209 : cluster [DBG] pgmap v6216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:35.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:35 smithi006 bash[27292]: audit 2023-08-28T01:25:35.067188+0000 mon.c (mon.2) 2439 : audit [DBG] from='client.? 172.21.15.6:0/1276951749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:35.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:35 smithi006 bash[33918]: cluster 2023-08-28T01:25:34.071932+0000 mgr.y (mgr.14901) 6209 : cluster [DBG] pgmap v6216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:35.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:35 smithi006 bash[33918]: audit 2023-08-28T01:25:35.067188+0000 mon.c (mon.2) 2439 : audit [DBG] from='client.? 172.21.15.6:0/1276951749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:35.553 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:35 smithi130 bash[27579]: cluster 2023-08-28T01:25:34.071932+0000 mgr.y (mgr.14901) 6209 : cluster [DBG] pgmap v6216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:35.553 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:35 smithi130 bash[27579]: audit 2023-08-28T01:25:35.067188+0000 mon.c (mon.2) 2439 : audit [DBG] from='client.? 172.21.15.6:0/1276951749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:35.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:35 smithi130 bash[51302]: debug 2023-08-28T01:25:35.547+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:35.554308+0000) 2023-08-28T01:25:36.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:36 smithi130 bash[51302]: debug 2023-08-28T01:25:36.547+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:36.554690+0000) 2023-08-28T01:25:37.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:37 smithi006 bash[33918]: cluster 2023-08-28T01:25:36.073086+0000 mgr.y (mgr.14901) 6210 : cluster [DBG] pgmap v6217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:37.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:37 smithi006 bash[27292]: cluster 2023-08-28T01:25:36.073086+0000 mgr.y (mgr.14901) 6210 : cluster [DBG] pgmap v6217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:37.554 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:37 smithi130 bash[27579]: cluster 2023-08-28T01:25:36.073086+0000 mgr.y (mgr.14901) 6210 : cluster [DBG] pgmap v6217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:37.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:37 smithi130 bash[51302]: debug 2023-08-28T01:25:37.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:37.555039+0000) 2023-08-28T01:25:38.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:38 smithi006 bash[27292]: audit 2023-08-28T01:25:37.481489+0000 mon.c (mon.2) 2440 : audit [DBG] from='client.? 172.21.15.6:0/1234177095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:38.766 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:38 smithi006 bash[33918]: audit 2023-08-28T01:25:37.481489+0000 mon.c (mon.2) 2440 : audit [DBG] from='client.? 172.21.15.6:0/1234177095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:38.766 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:38 smithi130 bash[27579]: audit 2023-08-28T01:25:37.481489+0000 mon.c (mon.2) 2440 : audit [DBG] from='client.? 172.21.15.6:0/1234177095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:38.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:38 smithi130 bash[51302]: debug 2023-08-28T01:25:38.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:38.555372+0000) 2023-08-28T01:25:39.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:39 smithi006 bash[27292]: cluster 2023-08-28T01:25:38.073768+0000 mgr.y (mgr.14901) 6211 : cluster [DBG] pgmap v6218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:39.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:39 smithi006 bash[33918]: cluster 2023-08-28T01:25:38.073768+0000 mgr.y (mgr.14901) 6211 : cluster [DBG] pgmap v6218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:39.554 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:39 smithi130 bash[27579]: cluster 2023-08-28T01:25:38.073768+0000 mgr.y (mgr.14901) 6211 : cluster [DBG] pgmap v6218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:39.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:39 smithi130 bash[51302]: debug 2023-08-28T01:25:39.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:39.555711+0000) 2023-08-28T01:25:40.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:40 smithi006 bash[27292]: audit 2023-08-28T01:25:39.895841+0000 mon.a (mon.0) 4149 : audit [DBG] from='client.? 172.21.15.6:0/214738372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:40.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:40 smithi006 bash[33918]: audit 2023-08-28T01:25:39.895841+0000 mon.a (mon.0) 4149 : audit [DBG] from='client.? 172.21.15.6:0/214738372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:40.555 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:40 smithi130 bash[27579]: audit 2023-08-28T01:25:39.895841+0000 mon.a (mon.0) 4149 : audit [DBG] from='client.? 172.21.15.6:0/214738372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:40.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:40 smithi130 bash[51302]: debug 2023-08-28T01:25:40.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:40.556098+0000) 2023-08-28T01:25:41.011 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:25:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:25:40] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:25:41.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:41 smithi006 bash[27292]: cluster 2023-08-28T01:25:40.074943+0000 mgr.y (mgr.14901) 6212 : cluster [DBG] pgmap v6219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:41.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:25:41.010Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:41.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:25:41.012Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:41.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:41 smithi006 bash[33918]: cluster 2023-08-28T01:25:40.074943+0000 mgr.y (mgr.14901) 6212 : cluster [DBG] pgmap v6219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:41.555 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:41 smithi130 bash[27579]: cluster 2023-08-28T01:25:40.074943+0000 mgr.y (mgr.14901) 6212 : cluster [DBG] pgmap v6219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:41.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:41 smithi130 bash[51302]: debug 2023-08-28T01:25:41.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:41.556416+0000) 2023-08-28T01:25:42.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:42 smithi130 bash[51302]: debug 2023-08-28T01:25:42.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:42.556743+0000) 2023-08-28T01:25:42.879 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:25:42.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:42.879 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:25:42.586Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:43.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:42 smithi006 bash[33918]: cluster 2023-08-28T01:25:42.075452+0000 mgr.y (mgr.14901) 6213 : cluster [DBG] pgmap v6220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:43.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:42 smithi006 bash[33918]: audit 2023-08-28T01:25:42.312423+0000 mon.c (mon.2) 2441 : audit [DBG] from='client.? 172.21.15.6:0/4175250237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:43.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:42 smithi006 bash[27292]: cluster 2023-08-28T01:25:42.075452+0000 mgr.y (mgr.14901) 6213 : cluster [DBG] pgmap v6220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:43.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:42 smithi006 bash[27292]: audit 2023-08-28T01:25:42.312423+0000 mon.c (mon.2) 2441 : audit [DBG] from='client.? 172.21.15.6:0/4175250237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:43.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:42 smithi130 bash[27579]: cluster 2023-08-28T01:25:42.075452+0000 mgr.y (mgr.14901) 6213 : cluster [DBG] pgmap v6220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:43.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:42 smithi130 bash[27579]: audit 2023-08-28T01:25:42.312423+0000 mon.c (mon.2) 2441 : audit [DBG] from='client.? 172.21.15.6:0/4175250237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:43.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:43 smithi130 bash[51302]: debug 2023-08-28T01:25:43.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:43.557064+0000) 2023-08-28T01:25:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:25:44.464Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:25:44.465Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:44.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:44 smithi130 bash[51302]: debug 2023-08-28T01:25:44.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:44.557391+0000) 2023-08-28T01:25:45.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:45 smithi006 bash[33918]: cluster 2023-08-28T01:25:44.076061+0000 mgr.y (mgr.14901) 6214 : cluster [DBG] pgmap v6221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:45.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:45 smithi006 bash[33918]: audit 2023-08-28T01:25:44.733775+0000 mon.c (mon.2) 2442 : audit [DBG] from='client.? 172.21.15.6:0/2287103074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:45.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:45 smithi006 bash[27292]: cluster 2023-08-28T01:25:44.076061+0000 mgr.y (mgr.14901) 6214 : cluster [DBG] pgmap v6221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:45.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:45 smithi006 bash[27292]: audit 2023-08-28T01:25:44.733775+0000 mon.c (mon.2) 2442 : audit [DBG] from='client.? 172.21.15.6:0/2287103074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:45.556 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:45 smithi130 bash[27579]: cluster 2023-08-28T01:25:44.076061+0000 mgr.y (mgr.14901) 6214 : cluster [DBG] pgmap v6221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:45.556 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:45 smithi130 bash[27579]: audit 2023-08-28T01:25:44.733775+0000 mon.c (mon.2) 2442 : audit [DBG] from='client.? 172.21.15.6:0/2287103074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:45.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:45 smithi130 bash[51302]: debug 2023-08-28T01:25:45.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:45.557757+0000) 2023-08-28T01:25:46.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:46 smithi130 bash[51302]: debug 2023-08-28T01:25:46.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:46.558049+0000) 2023-08-28T01:25:47.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:47 smithi006 bash[27292]: cluster 2023-08-28T01:25:46.077228+0000 mgr.y (mgr.14901) 6215 : cluster [DBG] pgmap v6222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:47.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:47 smithi006 bash[33918]: cluster 2023-08-28T01:25:46.077228+0000 mgr.y (mgr.14901) 6215 : cluster [DBG] pgmap v6222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:47.557 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:47 smithi130 bash[27579]: cluster 2023-08-28T01:25:46.077228+0000 mgr.y (mgr.14901) 6215 : cluster [DBG] pgmap v6222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:47.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:47 smithi130 bash[51302]: debug 2023-08-28T01:25:47.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:47.558390+0000) 2023-08-28T01:25:48.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:48 smithi006 bash[27292]: audit 2023-08-28T01:25:47.157021+0000 mon.c (mon.2) 2443 : audit [DBG] from='client.? 172.21.15.6:0/44069346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:48.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:48 smithi006 bash[33918]: audit 2023-08-28T01:25:47.157021+0000 mon.c (mon.2) 2443 : audit [DBG] from='client.? 172.21.15.6:0/44069346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:48.557 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:48 smithi130 bash[27579]: audit 2023-08-28T01:25:47.157021+0000 mon.c (mon.2) 2443 : audit [DBG] from='client.? 172.21.15.6:0/44069346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:48.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:48 smithi130 bash[51302]: debug 2023-08-28T01:25:48.551+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:48.558674+0000) 2023-08-28T01:25:49.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:49 smithi006 bash[33918]: cluster 2023-08-28T01:25:48.077880+0000 mgr.y (mgr.14901) 6216 : cluster [DBG] pgmap v6223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:49.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:49 smithi006 bash[27292]: cluster 2023-08-28T01:25:48.077880+0000 mgr.y (mgr.14901) 6216 : cluster [DBG] pgmap v6223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:49.557 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:49 smithi130 bash[27579]: cluster 2023-08-28T01:25:48.077880+0000 mgr.y (mgr.14901) 6216 : cluster [DBG] pgmap v6223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:49.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:49 smithi130 bash[51302]: debug 2023-08-28T01:25:49.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:49.558999+0000) 2023-08-28T01:25:50.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:50 smithi006 bash[27292]: audit 2023-08-28T01:25:49.561472+0000 mon.c (mon.2) 2444 : audit [DBG] from='client.? 172.21.15.6:0/846846783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:50.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:50 smithi006 bash[27292]: cluster 2023-08-28T01:25:50.079020+0000 mgr.y (mgr.14901) 6217 : cluster [DBG] pgmap v6224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:50.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:50 smithi006 bash[33918]: audit 2023-08-28T01:25:49.561472+0000 mon.c (mon.2) 2444 : audit [DBG] from='client.? 172.21.15.6:0/846846783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:50.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:50 smithi006 bash[33918]: cluster 2023-08-28T01:25:50.079020+0000 mgr.y (mgr.14901) 6217 : cluster [DBG] pgmap v6224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:50.558 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:50 smithi130 bash[27579]: audit 2023-08-28T01:25:49.561472+0000 mon.c (mon.2) 2444 : audit [DBG] from='client.? 172.21.15.6:0/846846783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:50.558 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:50 smithi130 bash[27579]: cluster 2023-08-28T01:25:50.079020+0000 mgr.y (mgr.14901) 6217 : cluster [DBG] pgmap v6224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:50.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:50 smithi130 bash[51302]: debug 2023-08-28T01:25:50.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:50.559280+0000) 2023-08-28T01:25:51.012 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:25:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:25:50] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T01:25:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:25:51.011Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:25:51.013Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:51.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:51 smithi130 bash[51302]: debug 2023-08-28T01:25:51.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:51.559528+0000) 2023-08-28T01:25:52.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:52 smithi130 bash[51302]: debug 2023-08-28T01:25:52.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:52.559871+0000) 2023-08-28T01:25:52.883 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:25:52.585Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:52.883 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:25:52.587Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:53.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:52 smithi006 bash[33918]: audit 2023-08-28T01:25:51.971802+0000 mon.c (mon.2) 2445 : audit [DBG] from='client.? 172.21.15.6:0/4010360598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:53.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:52 smithi006 bash[33918]: cluster 2023-08-28T01:25:52.079680+0000 mgr.y (mgr.14901) 6218 : cluster [DBG] pgmap v6225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:53.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:52 smithi006 bash[27292]: audit 2023-08-28T01:25:51.971802+0000 mon.c (mon.2) 2445 : audit [DBG] from='client.? 172.21.15.6:0/4010360598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:53.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:52 smithi006 bash[27292]: cluster 2023-08-28T01:25:52.079680+0000 mgr.y (mgr.14901) 6218 : cluster [DBG] pgmap v6225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:53.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:52 smithi130 bash[27579]: audit 2023-08-28T01:25:51.971802+0000 mon.c (mon.2) 2445 : audit [DBG] from='client.? 172.21.15.6:0/4010360598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:53.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:52 smithi130 bash[27579]: cluster 2023-08-28T01:25:52.079680+0000 mgr.y (mgr.14901) 6218 : cluster [DBG] pgmap v6225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:53.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:53 smithi130 bash[51302]: debug 2023-08-28T01:25:53.559+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:53.560172+0000) 2023-08-28T01:25:54.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:54 smithi130 bash[51302]: debug 2023-08-28T01:25:54.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:54.560501+0000) 2023-08-28T01:25:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:25:54.465Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:25:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:25:54.467Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:25:55.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:55 smithi006 bash[33918]: cluster 2023-08-28T01:25:54.080307+0000 mgr.y (mgr.14901) 6219 : cluster [DBG] pgmap v6226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:55.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:55 smithi006 bash[33918]: audit 2023-08-28T01:25:54.382163+0000 mon.a (mon.0) 4150 : audit [DBG] from='client.? 172.21.15.6:0/1339551214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:55.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:55 smithi006 bash[27292]: cluster 2023-08-28T01:25:54.080307+0000 mgr.y (mgr.14901) 6219 : cluster [DBG] pgmap v6226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:55.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:55 smithi006 bash[27292]: audit 2023-08-28T01:25:54.382163+0000 mon.a (mon.0) 4150 : audit [DBG] from='client.? 172.21.15.6:0/1339551214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:55.559 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:55 smithi130 bash[27579]: cluster 2023-08-28T01:25:54.080307+0000 mgr.y (mgr.14901) 6219 : cluster [DBG] pgmap v6226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:55.559 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:55 smithi130 bash[27579]: audit 2023-08-28T01:25:54.382163+0000 mon.a (mon.0) 4150 : audit [DBG] from='client.? 172.21.15.6:0/1339551214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:55.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:55 smithi130 bash[51302]: debug 2023-08-28T01:25:55.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:55.560771+0000) 2023-08-28T01:25:56.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:56 smithi130 bash[51302]: debug 2023-08-28T01:25:56.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:56.561039+0000) 2023-08-28T01:25:57.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:57 smithi006 bash[27292]: cluster 2023-08-28T01:25:56.081583+0000 mgr.y (mgr.14901) 6220 : cluster [DBG] pgmap v6227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:57.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:57 smithi006 bash[27292]: audit 2023-08-28T01:25:56.798770+0000 mon.a (mon.0) 4151 : audit [DBG] from='client.? 172.21.15.6:0/1118955098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:57.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:57 smithi006 bash[33918]: cluster 2023-08-28T01:25:56.081583+0000 mgr.y (mgr.14901) 6220 : cluster [DBG] pgmap v6227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:57.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:57 smithi006 bash[33918]: audit 2023-08-28T01:25:56.798770+0000 mon.a (mon.0) 4151 : audit [DBG] from='client.? 172.21.15.6:0/1118955098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:57.560 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:57 smithi130 bash[27579]: cluster 2023-08-28T01:25:56.081583+0000 mgr.y (mgr.14901) 6220 : cluster [DBG] pgmap v6227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:57.560 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:57 smithi130 bash[27579]: audit 2023-08-28T01:25:56.798770+0000 mon.a (mon.0) 4151 : audit [DBG] from='client.? 172.21.15.6:0/1118955098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:57.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:57 smithi130 bash[51302]: debug 2023-08-28T01:25:57.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:57.561240+0000) 2023-08-28T01:25:58.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:58 smithi130 bash[51302]: debug 2023-08-28T01:25:58.555+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:58.561524+0000) 2023-08-28T01:25:59.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:25:59 smithi006 bash[27292]: cluster 2023-08-28T01:25:58.082206+0000 mgr.y (mgr.14901) 6221 : cluster [DBG] pgmap v6228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:59.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:25:59 smithi006 bash[33918]: cluster 2023-08-28T01:25:58.082206+0000 mgr.y (mgr.14901) 6221 : cluster [DBG] pgmap v6228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:59.560 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:25:59 smithi130 bash[27579]: cluster 2023-08-28T01:25:58.082206+0000 mgr.y (mgr.14901) 6221 : cluster [DBG] pgmap v6228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:25:59.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:25:59 smithi130 bash[51302]: debug 2023-08-28T01:25:59.554+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:59.561827+0000) 2023-08-28T01:26:00.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:00 smithi006 bash[27292]: audit 2023-08-28T01:25:59.209157+0000 mon.c (mon.2) 2446 : audit [DBG] from='client.? 172.21.15.6:0/3992462633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:00.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:00 smithi006 bash[33918]: audit 2023-08-28T01:25:59.209157+0000 mon.c (mon.2) 2446 : audit [DBG] from='client.? 172.21.15.6:0/3992462633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:00.561 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:00 smithi130 bash[27579]: audit 2023-08-28T01:25:59.209157+0000 mon.c (mon.2) 2446 : audit [DBG] from='client.? 172.21.15.6:0/3992462633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:00.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:00 smithi130 bash[51302]: debug 2023-08-28T01:26:00.558+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:00.562169+0000) 2023-08-28T01:26:01.013 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:26:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:26:00] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T01:26:01.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:01 smithi006 bash[33918]: cluster 2023-08-28T01:26:00.083292+0000 mgr.y (mgr.14901) 6222 : cluster [DBG] pgmap v6229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:01.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:01 smithi006 bash[27292]: cluster 2023-08-28T01:26:00.083292+0000 mgr.y (mgr.14901) 6222 : cluster [DBG] pgmap v6229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:01.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:26:01.012Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:01.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:26:01.014Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:01.561 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:01 smithi130 bash[27579]: cluster 2023-08-28T01:26:00.083292+0000 mgr.y (mgr.14901) 6222 : cluster [DBG] pgmap v6229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:01.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:01 smithi130 bash[51302]: debug 2023-08-28T01:26:01.558+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:01.562456+0000) 2023-08-28T01:26:02.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:02 smithi006 bash[33918]: audit 2023-08-28T01:26:01.626794+0000 mon.c (mon.2) 2447 : audit [DBG] from='client.? 172.21.15.6:0/2742876725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:02.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:02 smithi006 bash[33918]: cluster 2023-08-28T01:26:02.083953+0000 mgr.y (mgr.14901) 6223 : cluster [DBG] pgmap v6230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:02.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:02 smithi006 bash[27292]: audit 2023-08-28T01:26:01.626794+0000 mon.c (mon.2) 2447 : audit [DBG] from='client.? 172.21.15.6:0/2742876725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:02.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:02 smithi006 bash[27292]: cluster 2023-08-28T01:26:02.083953+0000 mgr.y (mgr.14901) 6223 : cluster [DBG] pgmap v6230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:02.561 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:02 smithi130 bash[27579]: audit 2023-08-28T01:26:01.626794+0000 mon.c (mon.2) 2447 : audit [DBG] from='client.? 172.21.15.6:0/2742876725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:02.562 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:02 smithi130 bash[27579]: cluster 2023-08-28T01:26:02.083953+0000 mgr.y (mgr.14901) 6223 : cluster [DBG] pgmap v6230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:02.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:02 smithi130 bash[51302]: debug 2023-08-28T01:26:02.554+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:02.562761+0000) 2023-08-28T01:26:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:26:02.586Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:26:02.588Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:03.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:03 smithi130 bash[51302]: debug 2023-08-28T01:26:03.558+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:03.563046+0000) 2023-08-28T01:26:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:04 smithi130 bash[27579]: audit 2023-08-28T01:26:04.040187+0000 mon.a (mon.0) 4152 : audit [DBG] from='client.? 172.21.15.6:0/166549843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:04 smithi006 bash[27292]: audit 2023-08-28T01:26:04.040187+0000 mon.a (mon.0) 4152 : audit [DBG] from='client.? 172.21.15.6:0/166549843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:04.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:04 smithi006 bash[33918]: audit 2023-08-28T01:26:04.040187+0000 mon.a (mon.0) 4152 : audit [DBG] from='client.? 172.21.15.6:0/166549843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:04.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:04 smithi130 bash[51302]: debug 2023-08-28T01:26:04.558+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:04.563277+0000) 2023-08-28T01:26:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:26:04.466Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:26:04.468Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:05.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:05 smithi006 bash[33918]: cluster 2023-08-28T01:26:04.084578+0000 mgr.y (mgr.14901) 6224 : cluster [DBG] pgmap v6231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:05.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:05 smithi006 bash[27292]: cluster 2023-08-28T01:26:04.084578+0000 mgr.y (mgr.14901) 6224 : cluster [DBG] pgmap v6231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:05.562 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:05 smithi130 bash[27579]: cluster 2023-08-28T01:26:04.084578+0000 mgr.y (mgr.14901) 6224 : cluster [DBG] pgmap v6231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:05.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:05 smithi130 bash[51302]: debug 2023-08-28T01:26:05.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:05.563582+0000) 2023-08-28T01:26:06.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:06 smithi130 bash[51302]: debug 2023-08-28T01:26:06.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:06.563879+0000) 2023-08-28T01:26:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:07 smithi006 bash[27292]: cluster 2023-08-28T01:26:06.085718+0000 mgr.y (mgr.14901) 6225 : cluster [DBG] pgmap v6232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:07 smithi006 bash[27292]: audit 2023-08-28T01:26:06.458321+0000 mon.a (mon.0) 4153 : audit [DBG] from='client.? 172.21.15.6:0/1662886755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:07.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:07 smithi006 bash[33918]: cluster 2023-08-28T01:26:06.085718+0000 mgr.y (mgr.14901) 6225 : cluster [DBG] pgmap v6232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:07.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:07 smithi006 bash[33918]: audit 2023-08-28T01:26:06.458321+0000 mon.a (mon.0) 4153 : audit [DBG] from='client.? 172.21.15.6:0/1662886755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:07.563 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:07 smithi130 bash[27579]: cluster 2023-08-28T01:26:06.085718+0000 mgr.y (mgr.14901) 6225 : cluster [DBG] pgmap v6232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:07.563 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:07 smithi130 bash[27579]: audit 2023-08-28T01:26:06.458321+0000 mon.a (mon.0) 4153 : audit [DBG] from='client.? 172.21.15.6:0/1662886755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:07.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:07 smithi130 bash[51302]: debug 2023-08-28T01:26:07.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:07.564163+0000) 2023-08-28T01:26:08.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:08 smithi006 bash[27292]: cluster 2023-08-28T01:26:08.086363+0000 mgr.y (mgr.14901) 6226 : cluster [DBG] pgmap v6233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:08.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:08 smithi006 bash[33918]: cluster 2023-08-28T01:26:08.086363+0000 mgr.y (mgr.14901) 6226 : cluster [DBG] pgmap v6233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:08.563 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:08 smithi130 bash[27579]: cluster 2023-08-28T01:26:08.086363+0000 mgr.y (mgr.14901) 6226 : cluster [DBG] pgmap v6233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:08.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:08 smithi130 bash[51302]: debug 2023-08-28T01:26:08.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:08.564447+0000) 2023-08-28T01:26:09.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:09 smithi006 bash[33918]: audit 2023-08-28T01:26:08.870735+0000 mon.a (mon.0) 4154 : audit [DBG] from='client.? 172.21.15.6:0/3739291583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:09.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:09 smithi006 bash[27292]: audit 2023-08-28T01:26:08.870735+0000 mon.a (mon.0) 4154 : audit [DBG] from='client.? 172.21.15.6:0/3739291583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:09.564 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:09 smithi130 bash[27579]: audit 2023-08-28T01:26:08.870735+0000 mon.a (mon.0) 4154 : audit [DBG] from='client.? 172.21.15.6:0/3739291583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:09.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:09 smithi130 bash[51302]: debug 2023-08-28T01:26:09.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:09.564749+0000) 2023-08-28T01:26:10.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:10 smithi006 bash[33918]: cluster 2023-08-28T01:26:10.087493+0000 mgr.y (mgr.14901) 6227 : cluster [DBG] pgmap v6234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:10.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:10 smithi006 bash[27292]: cluster 2023-08-28T01:26:10.087493+0000 mgr.y (mgr.14901) 6227 : cluster [DBG] pgmap v6234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:10.563 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:10 smithi130 bash[27579]: cluster 2023-08-28T01:26:10.087493+0000 mgr.y (mgr.14901) 6227 : cluster [DBG] pgmap v6234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:10.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:10 smithi130 bash[51302]: debug 2023-08-28T01:26:10.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:10.565029+0000) 2023-08-28T01:26:11.013 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:26:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:26:10] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T01:26:11.330 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:26:11.013Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:11.330 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:26:11.014Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:11.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:11 smithi130 bash[27579]: audit 2023-08-28T01:26:11.276393+0000 mon.a (mon.0) 4155 : audit [DBG] from='client.? 172.21.15.6:0/1649190397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:11.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:11 smithi130 bash[51302]: debug 2023-08-28T01:26:11.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:11.565246+0000) 2023-08-28T01:26:11.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:11 smithi006 bash[27292]: audit 2023-08-28T01:26:11.276393+0000 mon.a (mon.0) 4155 : audit [DBG] from='client.? 172.21.15.6:0/1649190397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:11.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:11 smithi006 bash[33918]: audit 2023-08-28T01:26:11.276393+0000 mon.a (mon.0) 4155 : audit [DBG] from='client.? 172.21.15.6:0/1649190397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:12.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:12 smithi130 bash[51302]: debug 2023-08-28T01:26:12.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:12.565535+0000) 2023-08-28T01:26:12.884 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:26:12.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:12.884 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:26:12.590Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:12 smithi006 bash[33918]: cluster 2023-08-28T01:26:12.088141+0000 mgr.y (mgr.14901) 6228 : cluster [DBG] pgmap v6235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:13.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:12 smithi006 bash[27292]: cluster 2023-08-28T01:26:12.088141+0000 mgr.y (mgr.14901) 6228 : cluster [DBG] pgmap v6235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:13.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:12 smithi130 bash[27579]: cluster 2023-08-28T01:26:12.088141+0000 mgr.y (mgr.14901) 6228 : cluster [DBG] pgmap v6235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:13.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:13 smithi130 bash[51302]: debug 2023-08-28T01:26:13.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:13.565821+0000) 2023-08-28T01:26:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:13 smithi006 bash[27292]: audit 2023-08-28T01:26:12.904137+0000 mon.b (mon.1) 713 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:26:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:13 smithi006 bash[27292]: audit 2023-08-28T01:26:13.687744+0000 mon.a (mon.0) 4156 : audit [DBG] from='client.? 172.21.15.6:0/522860873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:13 smithi006 bash[33918]: audit 2023-08-28T01:26:12.904137+0000 mon.b (mon.1) 713 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:26:14.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:13 smithi006 bash[33918]: audit 2023-08-28T01:26:13.687744+0000 mon.a (mon.0) 4156 : audit [DBG] from='client.? 172.21.15.6:0/522860873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:14.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:13 smithi130 bash[27579]: audit 2023-08-28T01:26:12.904137+0000 mon.b (mon.1) 713 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:26:14.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:13 smithi130 bash[27579]: audit 2023-08-28T01:26:13.687744+0000 mon.a (mon.0) 4156 : audit [DBG] from='client.? 172.21.15.6:0/522860873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:14.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:14 smithi130 bash[51302]: debug 2023-08-28T01:26:14.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:14.566114+0000) 2023-08-28T01:26:14.897 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:26:14.466Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:14.897 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:26:14.469Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:15.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:14 smithi006 bash[33918]: cluster 2023-08-28T01:26:14.088905+0000 mgr.y (mgr.14901) 6229 : cluster [DBG] pgmap v6236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:15.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:14 smithi006 bash[27292]: cluster 2023-08-28T01:26:14.088905+0000 mgr.y (mgr.14901) 6229 : cluster [DBG] pgmap v6236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:15.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:14 smithi130 bash[27579]: cluster 2023-08-28T01:26:14.088905+0000 mgr.y (mgr.14901) 6229 : cluster [DBG] pgmap v6236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:15.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:15 smithi130 bash[51302]: debug 2023-08-28T01:26:15.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:15.566424+0000) 2023-08-28T01:26:16.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:16 smithi006 bash[33918]: audit 2023-08-28T01:26:16.100223+0000 mon.c (mon.2) 2448 : audit [DBG] from='client.? 172.21.15.6:0/1008929068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:16.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:16 smithi006 bash[27292]: audit 2023-08-28T01:26:16.100223+0000 mon.c (mon.2) 2448 : audit [DBG] from='client.? 172.21.15.6:0/1008929068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:16.565 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:16 smithi130 bash[27579]: audit 2023-08-28T01:26:16.100223+0000 mon.c (mon.2) 2448 : audit [DBG] from='client.? 172.21.15.6:0/1008929068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:16.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:16 smithi130 bash[51302]: debug 2023-08-28T01:26:16.562+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:16.566706+0000) 2023-08-28T01:26:17.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:17 smithi006 bash[33918]: cluster 2023-08-28T01:26:16.089926+0000 mgr.y (mgr.14901) 6230 : cluster [DBG] pgmap v6237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:17.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:17 smithi006 bash[27292]: cluster 2023-08-28T01:26:16.089926+0000 mgr.y (mgr.14901) 6230 : cluster [DBG] pgmap v6237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:17.565 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:17 smithi130 bash[27579]: cluster 2023-08-28T01:26:16.089926+0000 mgr.y (mgr.14901) 6230 : cluster [DBG] pgmap v6237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:17.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:17 smithi130 bash[51302]: debug 2023-08-28T01:26:17.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:17.566997+0000) 2023-08-28T01:26:18.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:18 smithi006 bash[27292]: cluster 2023-08-28T01:26:18.090433+0000 mgr.y (mgr.14901) 6231 : cluster [DBG] pgmap v6238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:18.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:18 smithi006 bash[33918]: cluster 2023-08-28T01:26:18.090433+0000 mgr.y (mgr.14901) 6231 : cluster [DBG] pgmap v6238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:18.566 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:18 smithi130 bash[27579]: cluster 2023-08-28T01:26:18.090433+0000 mgr.y (mgr.14901) 6231 : cluster [DBG] pgmap v6238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:18.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:18 smithi130 bash[51302]: debug 2023-08-28T01:26:18.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:18.567262+0000) 2023-08-28T01:26:19.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:19 smithi006 bash[33918]: audit 2023-08-28T01:26:18.540457+0000 mon.a (mon.0) 4157 : audit [DBG] from='client.? 172.21.15.6:0/2480783061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:19.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:19 smithi006 bash[33918]: audit 2023-08-28T01:26:18.772549+0000 mon.a (mon.0) 4158 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:26:19.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:19 smithi006 bash[27292]: audit 2023-08-28T01:26:18.540457+0000 mon.a (mon.0) 4157 : audit [DBG] from='client.? 172.21.15.6:0/2480783061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:19.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:19 smithi006 bash[27292]: audit 2023-08-28T01:26:18.772549+0000 mon.a (mon.0) 4158 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:26:19.567 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:19 smithi130 bash[27579]: audit 2023-08-28T01:26:18.540457+0000 mon.a (mon.0) 4157 : audit [DBG] from='client.? 172.21.15.6:0/2480783061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:19.567 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:19 smithi130 bash[27579]: audit 2023-08-28T01:26:18.772549+0000 mon.a (mon.0) 4158 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:26:19.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:19 smithi130 bash[51302]: debug 2023-08-28T01:26:19.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:19.567538+0000) 2023-08-28T01:26:20.566 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:20 smithi130 bash[27579]: audit 2023-08-28T01:26:19.246886+0000 mon.a (mon.0) 4159 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:26:20.567 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:20 smithi130 bash[27579]: audit 2023-08-28T01:26:19.543124+0000 mon.b (mon.1) 714 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:26:20.567 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:20 smithi130 bash[27579]: audit 2023-08-28T01:26:19.545298+0000 mon.b (mon.1) 715 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:26:20.567 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:20 smithi130 bash[27579]: audit 2023-08-28T01:26:19.551912+0000 mon.a (mon.0) 4160 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:26:20.567 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:20 smithi130 bash[27579]: cluster 2023-08-28T01:26:20.091529+0000 mgr.y (mgr.14901) 6232 : cluster [DBG] pgmap v6239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:20.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:20 smithi006 bash[27292]: audit 2023-08-28T01:26:19.246886+0000 mon.a (mon.0) 4159 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:26:20.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:20 smithi006 bash[27292]: audit 2023-08-28T01:26:19.543124+0000 mon.b (mon.1) 714 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:26:20.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:20 smithi006 bash[27292]: audit 2023-08-28T01:26:19.545298+0000 mon.b (mon.1) 715 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:26:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:20 smithi006 bash[27292]: audit 2023-08-28T01:26:19.551912+0000 mon.a (mon.0) 4160 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:26:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:20 smithi006 bash[27292]: cluster 2023-08-28T01:26:20.091529+0000 mgr.y (mgr.14901) 6232 : cluster [DBG] pgmap v6239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:20.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:20 smithi006 bash[33918]: audit 2023-08-28T01:26:19.246886+0000 mon.a (mon.0) 4159 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:26:20.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:20 smithi006 bash[33918]: audit 2023-08-28T01:26:19.543124+0000 mon.b (mon.1) 714 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:26:20.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:20 smithi006 bash[33918]: audit 2023-08-28T01:26:19.545298+0000 mon.b (mon.1) 715 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:26:20.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:20 smithi006 bash[33918]: audit 2023-08-28T01:26:19.551912+0000 mon.a (mon.0) 4160 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:26:20.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:20 smithi006 bash[33918]: cluster 2023-08-28T01:26:20.091529+0000 mgr.y (mgr.14901) 6232 : cluster [DBG] pgmap v6239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:20.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:20 smithi130 bash[51302]: debug 2023-08-28T01:26:20.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:20.567862+0000) 2023-08-28T01:26:21.015 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:26:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:26:20] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:26:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:26:21.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:26:21.016Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:21.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:21 smithi130 bash[51302]: debug 2023-08-28T01:26:21.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:21.568125+0000) 2023-08-28T01:26:21.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:21 smithi130 bash[27579]: audit 2023-08-28T01:26:20.959265+0000 mon.c (mon.2) 2449 : audit [DBG] from='client.? 172.21.15.6:0/3078528474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:21.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:21 smithi006 bash[33918]: audit 2023-08-28T01:26:20.959265+0000 mon.c (mon.2) 2449 : audit [DBG] from='client.? 172.21.15.6:0/3078528474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:21.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:21 smithi006 bash[27292]: audit 2023-08-28T01:26:20.959265+0000 mon.c (mon.2) 2449 : audit [DBG] from='client.? 172.21.15.6:0/3078528474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:22.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:22 smithi130 bash[51302]: debug 2023-08-28T01:26:22.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:22.568418+0000) 2023-08-28T01:26:22.888 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:26:22.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:22.888 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:26:22.590Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:23.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:22 smithi006 bash[33918]: cluster 2023-08-28T01:26:22.092203+0000 mgr.y (mgr.14901) 6233 : cluster [DBG] pgmap v6240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:23.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:22 smithi006 bash[27292]: cluster 2023-08-28T01:26:22.092203+0000 mgr.y (mgr.14901) 6233 : cluster [DBG] pgmap v6240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:23.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:22 smithi130 bash[27579]: cluster 2023-08-28T01:26:22.092203+0000 mgr.y (mgr.14901) 6233 : cluster [DBG] pgmap v6240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:23.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:23 smithi130 bash[51302]: debug 2023-08-28T01:26:23.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:23.568792+0000) 2023-08-28T01:26:24.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:23 smithi006 bash[33918]: audit 2023-08-28T01:26:23.382361+0000 mon.a (mon.0) 4161 : audit [DBG] from='client.? 172.21.15.6:0/161868356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:23 smithi006 bash[27292]: audit 2023-08-28T01:26:23.382361+0000 mon.a (mon.0) 4161 : audit [DBG] from='client.? 172.21.15.6:0/161868356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:23 smithi130 bash[27579]: audit 2023-08-28T01:26:23.382361+0000 mon.a (mon.0) 4161 : audit [DBG] from='client.? 172.21.15.6:0/161868356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:24.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:24 smithi130 bash[51302]: debug 2023-08-28T01:26:24.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:24.569147+0000) 2023-08-28T01:26:24.899 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:26:24.467Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:24.899 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:26:24.469Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:24.900 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:24 smithi006 bash[27292]: cluster 2023-08-28T01:26:24.092965+0000 mgr.y (mgr.14901) 6234 : cluster [DBG] pgmap v6241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:25.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:24 smithi006 bash[33918]: cluster 2023-08-28T01:26:24.092965+0000 mgr.y (mgr.14901) 6234 : cluster [DBG] pgmap v6241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:25.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:24 smithi130 bash[27579]: cluster 2023-08-28T01:26:24.092965+0000 mgr.y (mgr.14901) 6234 : cluster [DBG] pgmap v6241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:25.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:25 smithi130 bash[51302]: debug 2023-08-28T01:26:25.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:25.569430+0000) 2023-08-28T01:26:26.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:25 smithi006 bash[27292]: audit 2023-08-28T01:26:25.797118+0000 mon.a (mon.0) 4162 : audit [DBG] from='client.? 172.21.15.6:0/3306979702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:26.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:25 smithi006 bash[33918]: audit 2023-08-28T01:26:25.797118+0000 mon.a (mon.0) 4162 : audit [DBG] from='client.? 172.21.15.6:0/3306979702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:26.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:25 smithi130 bash[27579]: audit 2023-08-28T01:26:25.797118+0000 mon.a (mon.0) 4162 : audit [DBG] from='client.? 172.21.15.6:0/3306979702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:26.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:26 smithi130 bash[51302]: debug 2023-08-28T01:26:26.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:26.569737+0000) 2023-08-28T01:26:27.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:26 smithi006 bash[27292]: cluster 2023-08-28T01:26:26.094118+0000 mgr.y (mgr.14901) 6235 : cluster [DBG] pgmap v6242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:27.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:26 smithi006 bash[33918]: cluster 2023-08-28T01:26:26.094118+0000 mgr.y (mgr.14901) 6235 : cluster [DBG] pgmap v6242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:27.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:26 smithi130 bash[27579]: cluster 2023-08-28T01:26:26.094118+0000 mgr.y (mgr.14901) 6235 : cluster [DBG] pgmap v6242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:27.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:27 smithi130 bash[51302]: debug 2023-08-28T01:26:27.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:27.569982+0000) 2023-08-28T01:26:28.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:28 smithi130 bash[51302]: debug 2023-08-28T01:26:28.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:28.570235+0000) 2023-08-28T01:26:29.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:29 smithi006 bash[27292]: cluster 2023-08-28T01:26:28.094612+0000 mgr.y (mgr.14901) 6236 : cluster [DBG] pgmap v6243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:29.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:29 smithi006 bash[27292]: audit 2023-08-28T01:26:28.206342+0000 mon.a (mon.0) 4163 : audit [DBG] from='client.? 172.21.15.6:0/3580026298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:29.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:29 smithi006 bash[33918]: cluster 2023-08-28T01:26:28.094612+0000 mgr.y (mgr.14901) 6236 : cluster [DBG] pgmap v6243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:29.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:29 smithi006 bash[33918]: audit 2023-08-28T01:26:28.206342+0000 mon.a (mon.0) 4163 : audit [DBG] from='client.? 172.21.15.6:0/3580026298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:29.569 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:29 smithi130 bash[27579]: cluster 2023-08-28T01:26:28.094612+0000 mgr.y (mgr.14901) 6236 : cluster [DBG] pgmap v6243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:29.569 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:29 smithi130 bash[27579]: audit 2023-08-28T01:26:28.206342+0000 mon.a (mon.0) 4163 : audit [DBG] from='client.? 172.21.15.6:0/3580026298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:29.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:29 smithi130 bash[51302]: debug 2023-08-28T01:26:29.566+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:29.570513+0000) 2023-08-28T01:26:30.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:30 smithi130 bash[51302]: debug 2023-08-28T01:26:30.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:30.570849+0000) 2023-08-28T01:26:31.015 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:26:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:26:30] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:26:31.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:31 smithi006 bash[27292]: cluster 2023-08-28T01:26:30.095726+0000 mgr.y (mgr.14901) 6237 : cluster [DBG] pgmap v6244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:31.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:31 smithi006 bash[27292]: audit 2023-08-28T01:26:30.625992+0000 mon.c (mon.2) 2450 : audit [DBG] from='client.? 172.21.15.6:0/3673126539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:31.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:31 smithi006 bash[33918]: cluster 2023-08-28T01:26:30.095726+0000 mgr.y (mgr.14901) 6237 : cluster [DBG] pgmap v6244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:31.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:31 smithi006 bash[33918]: audit 2023-08-28T01:26:30.625992+0000 mon.c (mon.2) 2450 : audit [DBG] from='client.? 172.21.15.6:0/3673126539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:31.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:26:31.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:31.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:26:31.017Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:31.570 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:31 smithi130 bash[27579]: cluster 2023-08-28T01:26:30.095726+0000 mgr.y (mgr.14901) 6237 : cluster [DBG] pgmap v6244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:31.570 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:31 smithi130 bash[27579]: audit 2023-08-28T01:26:30.625992+0000 mon.c (mon.2) 2450 : audit [DBG] from='client.? 172.21.15.6:0/3673126539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:31.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:31 smithi130 bash[51302]: debug 2023-08-28T01:26:31.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:31.571108+0000) 2023-08-28T01:26:32.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:32 smithi130 bash[51302]: debug 2023-08-28T01:26:32.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:32.571384+0000) 2023-08-28T01:26:32.890 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:26:32.589Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:32.890 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:26:32.591Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:32.891 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:32 smithi006 bash[33918]: cluster 2023-08-28T01:26:32.096390+0000 mgr.y (mgr.14901) 6238 : cluster [DBG] pgmap v6245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:33.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:32 smithi006 bash[27292]: cluster 2023-08-28T01:26:32.096390+0000 mgr.y (mgr.14901) 6238 : cluster [DBG] pgmap v6245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:33.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:32 smithi130 bash[27579]: cluster 2023-08-28T01:26:32.096390+0000 mgr.y (mgr.14901) 6238 : cluster [DBG] pgmap v6245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:33.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:33 smithi130 bash[51302]: debug 2023-08-28T01:26:33.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:33.571683+0000) 2023-08-28T01:26:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:33 smithi006 bash[27292]: audit 2023-08-28T01:26:33.046463+0000 mon.c (mon.2) 2451 : audit [DBG] from='client.? 172.21.15.6:0/1442973166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:33 smithi006 bash[33918]: audit 2023-08-28T01:26:33.046463+0000 mon.c (mon.2) 2451 : audit [DBG] from='client.? 172.21.15.6:0/1442973166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:33 smithi130 bash[27579]: audit 2023-08-28T01:26:33.046463+0000 mon.c (mon.2) 2451 : audit [DBG] from='client.? 172.21.15.6:0/1442973166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:34.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:34 smithi130 bash[51302]: debug 2023-08-28T01:26:34.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:34.571978+0000) 2023-08-28T01:26:34.903 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:26:34.468Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:34.903 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:26:34.470Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:35.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:34 smithi006 bash[27292]: cluster 2023-08-28T01:26:34.097137+0000 mgr.y (mgr.14901) 6239 : cluster [DBG] pgmap v6246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:35.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:34 smithi006 bash[33918]: cluster 2023-08-28T01:26:34.097137+0000 mgr.y (mgr.14901) 6239 : cluster [DBG] pgmap v6246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:35.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:34 smithi130 bash[27579]: cluster 2023-08-28T01:26:34.097137+0000 mgr.y (mgr.14901) 6239 : cluster [DBG] pgmap v6246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:35.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:35 smithi130 bash[51302]: debug 2023-08-28T01:26:35.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:35.572256+0000) 2023-08-28T01:26:36.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:35 smithi006 bash[27292]: audit 2023-08-28T01:26:35.461590+0000 mon.a (mon.0) 4164 : audit [DBG] from='client.? 172.21.15.6:0/32703111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:36.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:35 smithi006 bash[33918]: audit 2023-08-28T01:26:35.461590+0000 mon.a (mon.0) 4164 : audit [DBG] from='client.? 172.21.15.6:0/32703111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:36.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:35 smithi130 bash[27579]: audit 2023-08-28T01:26:35.461590+0000 mon.a (mon.0) 4164 : audit [DBG] from='client.? 172.21.15.6:0/32703111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:36.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:36 smithi130 bash[51302]: debug 2023-08-28T01:26:36.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:36.572546+0000) 2023-08-28T01:26:37.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:36 smithi006 bash[27292]: cluster 2023-08-28T01:26:36.098330+0000 mgr.y (mgr.14901) 6240 : cluster [DBG] pgmap v6247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:37.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:36 smithi006 bash[33918]: cluster 2023-08-28T01:26:36.098330+0000 mgr.y (mgr.14901) 6240 : cluster [DBG] pgmap v6247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:37.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:36 smithi130 bash[27579]: cluster 2023-08-28T01:26:36.098330+0000 mgr.y (mgr.14901) 6240 : cluster [DBG] pgmap v6247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:37.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:37 smithi130 bash[51302]: debug 2023-08-28T01:26:37.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:37.572827+0000) 2023-08-28T01:26:38.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:37 smithi006 bash[33918]: audit 2023-08-28T01:26:37.871487+0000 mon.a (mon.0) 4165 : audit [DBG] from='client.? 172.21.15.6:0/2311039732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:38.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:37 smithi006 bash[27292]: audit 2023-08-28T01:26:37.871487+0000 mon.a (mon.0) 4165 : audit [DBG] from='client.? 172.21.15.6:0/2311039732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:38.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:37 smithi130 bash[27579]: audit 2023-08-28T01:26:37.871487+0000 mon.a (mon.0) 4165 : audit [DBG] from='client.? 172.21.15.6:0/2311039732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:38.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:38 smithi130 bash[51302]: debug 2023-08-28T01:26:38.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:38.573105+0000) 2023-08-28T01:26:39.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:38 smithi006 bash[27292]: cluster 2023-08-28T01:26:38.099158+0000 mgr.y (mgr.14901) 6241 : cluster [DBG] pgmap v6248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:39.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:38 smithi006 bash[33918]: cluster 2023-08-28T01:26:38.099158+0000 mgr.y (mgr.14901) 6241 : cluster [DBG] pgmap v6248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:39.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:38 smithi130 bash[27579]: cluster 2023-08-28T01:26:38.099158+0000 mgr.y (mgr.14901) 6241 : cluster [DBG] pgmap v6248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:39.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:39 smithi130 bash[51302]: debug 2023-08-28T01:26:39.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:39.573395+0000) 2023-08-28T01:26:40.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:40 smithi130 bash[51302]: debug 2023-08-28T01:26:40.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:40.573673+0000) 2023-08-28T01:26:41.016 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:26:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:26:40] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:26:41.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:41 smithi006 bash[27292]: cluster 2023-08-28T01:26:40.099944+0000 mgr.y (mgr.14901) 6242 : cluster [DBG] pgmap v6249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:41.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:41 smithi006 bash[27292]: audit 2023-08-28T01:26:40.284775+0000 mon.c (mon.2) 2452 : audit [DBG] from='client.? 172.21.15.6:0/3061407818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:41.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:41 smithi006 bash[33918]: cluster 2023-08-28T01:26:40.099944+0000 mgr.y (mgr.14901) 6242 : cluster [DBG] pgmap v6249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:41.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:41 smithi006 bash[33918]: audit 2023-08-28T01:26:40.284775+0000 mon.c (mon.2) 2452 : audit [DBG] from='client.? 172.21.15.6:0/3061407818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:41.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:26:41.015Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:41.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:26:41.017Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:41.572 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:41 smithi130 bash[27579]: cluster 2023-08-28T01:26:40.099944+0000 mgr.y (mgr.14901) 6242 : cluster [DBG] pgmap v6249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:41.573 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:41 smithi130 bash[27579]: audit 2023-08-28T01:26:40.284775+0000 mon.c (mon.2) 2452 : audit [DBG] from='client.? 172.21.15.6:0/3061407818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:41.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:41 smithi130 bash[51302]: debug 2023-08-28T01:26:41.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:41.573947+0000) 2023-08-28T01:26:42.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:42 smithi130 bash[51302]: debug 2023-08-28T01:26:42.570+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:42.574158+0000) 2023-08-28T01:26:42.892 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:26:42.589Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:42.892 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:26:42.591Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:43.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:42 smithi006 bash[33918]: cluster 2023-08-28T01:26:42.100578+0000 mgr.y (mgr.14901) 6243 : cluster [DBG] pgmap v6250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:43.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:42 smithi006 bash[33918]: audit 2023-08-28T01:26:42.698249+0000 mon.c (mon.2) 2453 : audit [DBG] from='client.? 172.21.15.6:0/2920107139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:43.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:42 smithi006 bash[27292]: cluster 2023-08-28T01:26:42.100578+0000 mgr.y (mgr.14901) 6243 : cluster [DBG] pgmap v6250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:43.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:42 smithi006 bash[27292]: audit 2023-08-28T01:26:42.698249+0000 mon.c (mon.2) 2453 : audit [DBG] from='client.? 172.21.15.6:0/2920107139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:43.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:42 smithi130 bash[27579]: cluster 2023-08-28T01:26:42.100578+0000 mgr.y (mgr.14901) 6243 : cluster [DBG] pgmap v6250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:43.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:42 smithi130 bash[27579]: audit 2023-08-28T01:26:42.698249+0000 mon.c (mon.2) 2453 : audit [DBG] from='client.? 172.21.15.6:0/2920107139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:43.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:43 smithi130 bash[51302]: debug 2023-08-28T01:26:43.574+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:43.574428+0000) 2023-08-28T01:26:44.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:44 smithi130 bash[51302]: debug 2023-08-28T01:26:44.574+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:44.574754+0000) 2023-08-28T01:26:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:26:44.469Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:26:44.471Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:45.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:45 smithi006 bash[27292]: cluster 2023-08-28T01:26:44.101330+0000 mgr.y (mgr.14901) 6244 : cluster [DBG] pgmap v6251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:45.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:45 smithi006 bash[27292]: audit 2023-08-28T01:26:45.119583+0000 mon.a (mon.0) 4166 : audit [DBG] from='client.? 172.21.15.6:0/190802098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:45.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:45 smithi006 bash[33918]: cluster 2023-08-28T01:26:44.101330+0000 mgr.y (mgr.14901) 6244 : cluster [DBG] pgmap v6251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:45.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:45 smithi006 bash[33918]: audit 2023-08-28T01:26:45.119583+0000 mon.a (mon.0) 4166 : audit [DBG] from='client.? 172.21.15.6:0/190802098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:45.573 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:45 smithi130 bash[27579]: cluster 2023-08-28T01:26:44.101330+0000 mgr.y (mgr.14901) 6244 : cluster [DBG] pgmap v6251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:45.574 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:45 smithi130 bash[27579]: audit 2023-08-28T01:26:45.119583+0000 mon.a (mon.0) 4166 : audit [DBG] from='client.? 172.21.15.6:0/190802098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:45.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:45 smithi130 bash[51302]: debug 2023-08-28T01:26:45.574+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:45.575052+0000) 2023-08-28T01:26:46.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:46 smithi006 bash[27292]: cluster 2023-08-28T01:26:46.102336+0000 mgr.y (mgr.14901) 6245 : cluster [DBG] pgmap v6252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:46.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:46 smithi006 bash[33918]: cluster 2023-08-28T01:26:46.102336+0000 mgr.y (mgr.14901) 6245 : cluster [DBG] pgmap v6252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:46.574 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:46 smithi130 bash[27579]: cluster 2023-08-28T01:26:46.102336+0000 mgr.y (mgr.14901) 6245 : cluster [DBG] pgmap v6252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:46.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:46 smithi130 bash[51302]: debug 2023-08-28T01:26:46.574+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:46.575277+0000) 2023-08-28T01:26:47.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:47 smithi130 bash[51302]: debug 2023-08-28T01:26:47.574+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:47.575577+0000) 2023-08-28T01:26:48.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:47 smithi006 bash[27292]: audit 2023-08-28T01:26:47.534752+0000 mon.c (mon.2) 2454 : audit [DBG] from='client.? 172.21.15.6:0/4140991134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:48.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:47 smithi006 bash[33918]: audit 2023-08-28T01:26:47.534752+0000 mon.c (mon.2) 2454 : audit [DBG] from='client.? 172.21.15.6:0/4140991134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:48.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:47 smithi130 bash[27579]: audit 2023-08-28T01:26:47.534752+0000 mon.c (mon.2) 2454 : audit [DBG] from='client.? 172.21.15.6:0/4140991134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:48.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:48 smithi130 bash[51302]: debug 2023-08-28T01:26:48.574+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:48.575806+0000) 2023-08-28T01:26:49.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:48 smithi006 bash[33918]: cluster 2023-08-28T01:26:48.103005+0000 mgr.y (mgr.14901) 6246 : cluster [DBG] pgmap v6253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:49.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:48 smithi006 bash[27292]: cluster 2023-08-28T01:26:48.103005+0000 mgr.y (mgr.14901) 6246 : cluster [DBG] pgmap v6253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:49.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:48 smithi130 bash[27579]: cluster 2023-08-28T01:26:48.103005+0000 mgr.y (mgr.14901) 6246 : cluster [DBG] pgmap v6253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:49.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:49 smithi130 bash[51302]: debug 2023-08-28T01:26:49.574+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:49.576074+0000) 2023-08-28T01:26:50.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:50 smithi130 bash[27579]: audit 2023-08-28T01:26:49.956525+0000 mon.a (mon.0) 4167 : audit [DBG] from='client.? 172.21.15.6:0/1073824109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:50.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:50 smithi006 bash[33918]: audit 2023-08-28T01:26:49.956525+0000 mon.a (mon.0) 4167 : audit [DBG] from='client.? 172.21.15.6:0/1073824109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:50.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:50 smithi006 bash[27292]: audit 2023-08-28T01:26:49.956525+0000 mon.a (mon.0) 4167 : audit [DBG] from='client.? 172.21.15.6:0/1073824109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:50.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:50 smithi130 bash[51302]: debug 2023-08-28T01:26:50.574+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:50.576406+0000) 2023-08-28T01:26:51.017 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:26:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:26:50] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:26:51.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:51 smithi130 bash[27579]: cluster 2023-08-28T01:26:50.104153+0000 mgr.y (mgr.14901) 6247 : cluster [DBG] pgmap v6254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:51.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:51 smithi006 bash[27292]: cluster 2023-08-28T01:26:50.104153+0000 mgr.y (mgr.14901) 6247 : cluster [DBG] pgmap v6254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:51.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:51 smithi006 bash[33918]: cluster 2023-08-28T01:26:50.104153+0000 mgr.y (mgr.14901) 6247 : cluster [DBG] pgmap v6254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:51.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:26:51.016Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:51.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:26:51.019Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:51.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:51 smithi130 bash[51302]: debug 2023-08-28T01:26:51.574+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:51.576678+0000) 2023-08-28T01:26:52.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:52 smithi130 bash[51302]: debug 2023-08-28T01:26:52.574+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:52.576965+0000) 2023-08-28T01:26:52.895 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:26:52.590Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:52.895 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:26:52.592Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:53.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:52 smithi006 bash[27292]: cluster 2023-08-28T01:26:52.104719+0000 mgr.y (mgr.14901) 6248 : cluster [DBG] pgmap v6255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:53.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:52 smithi006 bash[27292]: audit 2023-08-28T01:26:52.374528+0000 mon.b (mon.1) 716 : audit [DBG] from='client.? 172.21.15.6:0/365623249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:53.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:52 smithi006 bash[33918]: cluster 2023-08-28T01:26:52.104719+0000 mgr.y (mgr.14901) 6248 : cluster [DBG] pgmap v6255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:53.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:52 smithi006 bash[33918]: audit 2023-08-28T01:26:52.374528+0000 mon.b (mon.1) 716 : audit [DBG] from='client.? 172.21.15.6:0/365623249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:53.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:52 smithi130 bash[27579]: cluster 2023-08-28T01:26:52.104719+0000 mgr.y (mgr.14901) 6248 : cluster [DBG] pgmap v6255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:53.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:52 smithi130 bash[27579]: audit 2023-08-28T01:26:52.374528+0000 mon.b (mon.1) 716 : audit [DBG] from='client.? 172.21.15.6:0/365623249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:53.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:53 smithi130 bash[51302]: debug 2023-08-28T01:26:53.574+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:53.577238+0000) 2023-08-28T01:26:54.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:54 smithi130 bash[51302]: debug 2023-08-28T01:26:54.574+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:54.577525+0000) 2023-08-28T01:26:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:26:54.469Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:26:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:26:54.471Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:26:55.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:55 smithi006 bash[27292]: cluster 2023-08-28T01:26:54.105467+0000 mgr.y (mgr.14901) 6249 : cluster [DBG] pgmap v6256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:55.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:55 smithi006 bash[27292]: audit 2023-08-28T01:26:54.792711+0000 mon.a (mon.0) 4168 : audit [DBG] from='client.? 172.21.15.6:0/1781645414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:55.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:55 smithi006 bash[33918]: cluster 2023-08-28T01:26:54.105467+0000 mgr.y (mgr.14901) 6249 : cluster [DBG] pgmap v6256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:55.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:55 smithi006 bash[33918]: audit 2023-08-28T01:26:54.792711+0000 mon.a (mon.0) 4168 : audit [DBG] from='client.? 172.21.15.6:0/1781645414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:55.576 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:55 smithi130 bash[27579]: cluster 2023-08-28T01:26:54.105467+0000 mgr.y (mgr.14901) 6249 : cluster [DBG] pgmap v6256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:55.577 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:55 smithi130 bash[27579]: audit 2023-08-28T01:26:54.792711+0000 mon.a (mon.0) 4168 : audit [DBG] from='client.? 172.21.15.6:0/1781645414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:55.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:55 smithi130 bash[51302]: debug 2023-08-28T01:26:55.573+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:55.577858+0000) 2023-08-28T01:26:56.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:56 smithi006 bash[27292]: cluster 2023-08-28T01:26:56.106484+0000 mgr.y (mgr.14901) 6250 : cluster [DBG] pgmap v6257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:56.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:56 smithi006 bash[33918]: cluster 2023-08-28T01:26:56.106484+0000 mgr.y (mgr.14901) 6250 : cluster [DBG] pgmap v6257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:56.577 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:56 smithi130 bash[27579]: cluster 2023-08-28T01:26:56.106484+0000 mgr.y (mgr.14901) 6250 : cluster [DBG] pgmap v6257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:56.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:56 smithi130 bash[51302]: debug 2023-08-28T01:26:56.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:56.578179+0000) 2023-08-28T01:26:57.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:57 smithi130 bash[51302]: debug 2023-08-28T01:26:57.573+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:57.578622+0000) 2023-08-28T01:26:58.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:57 smithi006 bash[33918]: audit 2023-08-28T01:26:57.204715+0000 mon.c (mon.2) 2455 : audit [DBG] from='client.? 172.21.15.6:0/2563942610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:58.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:57 smithi006 bash[27292]: audit 2023-08-28T01:26:57.204715+0000 mon.c (mon.2) 2455 : audit [DBG] from='client.? 172.21.15.6:0/2563942610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:58.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:57 smithi130 bash[27579]: audit 2023-08-28T01:26:57.204715+0000 mon.c (mon.2) 2455 : audit [DBG] from='client.? 172.21.15.6:0/2563942610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:58.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:58 smithi130 bash[51302]: debug 2023-08-28T01:26:58.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:58.578929+0000) 2023-08-28T01:26:59.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:58 smithi006 bash[33918]: cluster 2023-08-28T01:26:58.107169+0000 mgr.y (mgr.14901) 6251 : cluster [DBG] pgmap v6258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:59.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:58 smithi006 bash[27292]: cluster 2023-08-28T01:26:58.107169+0000 mgr.y (mgr.14901) 6251 : cluster [DBG] pgmap v6258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:59.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:58 smithi130 bash[27579]: cluster 2023-08-28T01:26:58.107169+0000 mgr.y (mgr.14901) 6251 : cluster [DBG] pgmap v6258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:26:59.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:26:59 smithi130 bash[51302]: debug 2023-08-28T01:26:59.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:59.579195+0000) 2023-08-28T01:27:00.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:26:59 smithi006 bash[33918]: audit 2023-08-28T01:26:59.625348+0000 mon.a (mon.0) 4169 : audit [DBG] from='client.? 172.21.15.6:0/1885798906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:00.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:26:59 smithi006 bash[27292]: audit 2023-08-28T01:26:59.625348+0000 mon.a (mon.0) 4169 : audit [DBG] from='client.? 172.21.15.6:0/1885798906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:00.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:26:59 smithi130 bash[27579]: audit 2023-08-28T01:26:59.625348+0000 mon.a (mon.0) 4169 : audit [DBG] from='client.? 172.21.15.6:0/1885798906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:00.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:00 smithi130 bash[51302]: debug 2023-08-28T01:27:00.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:00.579462+0000) 2023-08-28T01:27:01.018 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:27:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:27:00] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:27:01.018 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:00 smithi006 bash[27292]: cluster 2023-08-28T01:27:00.107991+0000 mgr.y (mgr.14901) 6252 : cluster [DBG] pgmap v6259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:01.018 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:00 smithi006 bash[33918]: cluster 2023-08-28T01:27:00.107991+0000 mgr.y (mgr.14901) 6252 : cluster [DBG] pgmap v6259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:01.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:00 smithi130 bash[27579]: cluster 2023-08-28T01:27:00.107991+0000 mgr.y (mgr.14901) 6252 : cluster [DBG] pgmap v6259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:27:01.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:27:01.019Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:01.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:01 smithi130 bash[51302]: debug 2023-08-28T01:27:01.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:01.579771+0000) 2023-08-28T01:27:02.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:02 smithi130 bash[51302]: debug 2023-08-28T01:27:02.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:02.579997+0000) 2023-08-28T01:27:02.899 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:27:02.590Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:02.899 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:27:02.592Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:03.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:02 smithi006 bash[27292]: audit 2023-08-28T01:27:02.038344+0000 mon.c (mon.2) 2456 : audit [DBG] from='client.? 172.21.15.6:0/2782849674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:03.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:02 smithi006 bash[27292]: cluster 2023-08-28T01:27:02.108550+0000 mgr.y (mgr.14901) 6253 : cluster [DBG] pgmap v6260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:03.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:02 smithi006 bash[33918]: audit 2023-08-28T01:27:02.038344+0000 mon.c (mon.2) 2456 : audit [DBG] from='client.? 172.21.15.6:0/2782849674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:03.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:02 smithi006 bash[33918]: cluster 2023-08-28T01:27:02.108550+0000 mgr.y (mgr.14901) 6253 : cluster [DBG] pgmap v6260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:03.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:02 smithi130 bash[27579]: audit 2023-08-28T01:27:02.038344+0000 mon.c (mon.2) 2456 : audit [DBG] from='client.? 172.21.15.6:0/2782849674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:03.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:02 smithi130 bash[27579]: cluster 2023-08-28T01:27:02.108550+0000 mgr.y (mgr.14901) 6253 : cluster [DBG] pgmap v6260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:03.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:03 smithi130 bash[51302]: debug 2023-08-28T01:27:03.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:03.580208+0000) 2023-08-28T01:27:04.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:04 smithi130 bash[51302]: debug 2023-08-28T01:27:04.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:04.580491+0000) 2023-08-28T01:27:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:27:04.470Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:27:04.472Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:05.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:05 smithi006 bash[33918]: cluster 2023-08-28T01:27:04.109309+0000 mgr.y (mgr.14901) 6254 : cluster [DBG] pgmap v6261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:05.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:05 smithi006 bash[33918]: audit 2023-08-28T01:27:04.449089+0000 mon.a (mon.0) 4170 : audit [DBG] from='client.? 172.21.15.6:0/3123766405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:05.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:05 smithi006 bash[27292]: cluster 2023-08-28T01:27:04.109309+0000 mgr.y (mgr.14901) 6254 : cluster [DBG] pgmap v6261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:05.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:05 smithi006 bash[27292]: audit 2023-08-28T01:27:04.449089+0000 mon.a (mon.0) 4170 : audit [DBG] from='client.? 172.21.15.6:0/3123766405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:05.579 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:05 smithi130 bash[27579]: cluster 2023-08-28T01:27:04.109309+0000 mgr.y (mgr.14901) 6254 : cluster [DBG] pgmap v6261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:05.579 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:05 smithi130 bash[27579]: audit 2023-08-28T01:27:04.449089+0000 mon.a (mon.0) 4170 : audit [DBG] from='client.? 172.21.15.6:0/3123766405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:05.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:05 smithi130 bash[51302]: debug 2023-08-28T01:27:05.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:05.580775+0000) 2023-08-28T01:27:06.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:06 smithi006 bash[33918]: cluster 2023-08-28T01:27:06.110283+0000 mgr.y (mgr.14901) 6255 : cluster [DBG] pgmap v6262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:06.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:06 smithi006 bash[27292]: cluster 2023-08-28T01:27:06.110283+0000 mgr.y (mgr.14901) 6255 : cluster [DBG] pgmap v6262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:06.580 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:06 smithi130 bash[27579]: cluster 2023-08-28T01:27:06.110283+0000 mgr.y (mgr.14901) 6255 : cluster [DBG] pgmap v6262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:06.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:06 smithi130 bash[51302]: debug 2023-08-28T01:27:06.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:06.581090+0000) 2023-08-28T01:27:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:07 smithi006 bash[27292]: audit 2023-08-28T01:27:06.860901+0000 mon.c (mon.2) 2457 : audit [DBG] from='client.? 172.21.15.6:0/1861353837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:07.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:07 smithi006 bash[33918]: audit 2023-08-28T01:27:06.860901+0000 mon.c (mon.2) 2457 : audit [DBG] from='client.? 172.21.15.6:0/1861353837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:07.580 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:07 smithi130 bash[27579]: audit 2023-08-28T01:27:06.860901+0000 mon.c (mon.2) 2457 : audit [DBG] from='client.? 172.21.15.6:0/1861353837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:07.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:07 smithi130 bash[51302]: debug 2023-08-28T01:27:07.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:07.581376+0000) 2023-08-28T01:27:08.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:08 smithi006 bash[27292]: cluster 2023-08-28T01:27:08.110926+0000 mgr.y (mgr.14901) 6256 : cluster [DBG] pgmap v6263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:08.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:08 smithi006 bash[33918]: cluster 2023-08-28T01:27:08.110926+0000 mgr.y (mgr.14901) 6256 : cluster [DBG] pgmap v6263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:08.580 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:08 smithi130 bash[27579]: cluster 2023-08-28T01:27:08.110926+0000 mgr.y (mgr.14901) 6256 : cluster [DBG] pgmap v6263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:08.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:08 smithi130 bash[51302]: debug 2023-08-28T01:27:08.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:08.581641+0000) 2023-08-28T01:27:09.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:09 smithi130 bash[27579]: audit 2023-08-28T01:27:09.281246+0000 mon.a (mon.0) 4171 : audit [DBG] from='client.? 172.21.15.6:0/594480286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:09.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:09 smithi130 bash[51302]: debug 2023-08-28T01:27:09.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:09.581947+0000) 2023-08-28T01:27:09.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:09 smithi006 bash[33918]: audit 2023-08-28T01:27:09.281246+0000 mon.a (mon.0) 4171 : audit [DBG] from='client.? 172.21.15.6:0/594480286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:09.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:09 smithi006 bash[27292]: audit 2023-08-28T01:27:09.281246+0000 mon.a (mon.0) 4171 : audit [DBG] from='client.? 172.21.15.6:0/594480286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:10 smithi130 bash[27579]: cluster 2023-08-28T01:27:10.112056+0000 mgr.y (mgr.14901) 6257 : cluster [DBG] pgmap v6264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:10.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:10 smithi130 bash[51302]: debug 2023-08-28T01:27:10.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:10.582234+0000) 2023-08-28T01:27:10.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:10 smithi006 bash[33918]: cluster 2023-08-28T01:27:10.112056+0000 mgr.y (mgr.14901) 6257 : cluster [DBG] pgmap v6264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:10 smithi006 bash[27292]: cluster 2023-08-28T01:27:10.112056+0000 mgr.y (mgr.14901) 6257 : cluster [DBG] pgmap v6264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:11.018 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:27:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:27:10] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:27:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:27:11.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:27:11.019Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:11.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:11 smithi130 bash[27579]: audit 2023-08-28T01:27:11.705976+0000 mon.a (mon.0) 4172 : audit [DBG] from='client.? 172.21.15.6:0/1078657111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:11.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:11 smithi130 bash[51302]: debug 2023-08-28T01:27:11.577+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:11.582501+0000) 2023-08-28T01:27:12.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:11 smithi006 bash[33918]: audit 2023-08-28T01:27:11.705976+0000 mon.a (mon.0) 4172 : audit [DBG] from='client.? 172.21.15.6:0/1078657111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:12.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:11 smithi006 bash[27292]: audit 2023-08-28T01:27:11.705976+0000 mon.a (mon.0) 4172 : audit [DBG] from='client.? 172.21.15.6:0/1078657111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:12.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:12 smithi130 bash[51302]: debug 2023-08-28T01:27:12.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:12.582806+0000) 2023-08-28T01:27:12.899 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:27:12.591Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:12.899 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:27:12.593Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:13.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:12 smithi006 bash[27292]: cluster 2023-08-28T01:27:12.112710+0000 mgr.y (mgr.14901) 6258 : cluster [DBG] pgmap v6265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:12 smithi006 bash[33918]: cluster 2023-08-28T01:27:12.112710+0000 mgr.y (mgr.14901) 6258 : cluster [DBG] pgmap v6265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:13.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:12 smithi130 bash[27579]: cluster 2023-08-28T01:27:12.112710+0000 mgr.y (mgr.14901) 6258 : cluster [DBG] pgmap v6265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:13.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:13 smithi130 bash[51302]: debug 2023-08-28T01:27:13.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:13.583121+0000) 2023-08-28T01:27:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:14 smithi006 bash[33918]: cluster 2023-08-28T01:27:14.113435+0000 mgr.y (mgr.14901) 6259 : cluster [DBG] pgmap v6266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:14 smithi006 bash[33918]: audit 2023-08-28T01:27:14.119894+0000 mon.a (mon.0) 4173 : audit [DBG] from='client.? 172.21.15.6:0/281199981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:14.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:14 smithi006 bash[27292]: cluster 2023-08-28T01:27:14.113435+0000 mgr.y (mgr.14901) 6259 : cluster [DBG] pgmap v6266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:14.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:14 smithi006 bash[27292]: audit 2023-08-28T01:27:14.119894+0000 mon.a (mon.0) 4173 : audit [DBG] from='client.? 172.21.15.6:0/281199981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:14.582 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:14 smithi130 bash[27579]: cluster 2023-08-28T01:27:14.113435+0000 mgr.y (mgr.14901) 6259 : cluster [DBG] pgmap v6266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:14.582 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:14 smithi130 bash[27579]: audit 2023-08-28T01:27:14.119894+0000 mon.a (mon.0) 4173 : audit [DBG] from='client.? 172.21.15.6:0/281199981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:14.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:14 smithi130 bash[51302]: debug 2023-08-28T01:27:14.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:14.583404+0000) 2023-08-28T01:27:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:27:14.471Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:14.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:27:14.473Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:15.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:15 smithi130 bash[51302]: debug 2023-08-28T01:27:15.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:15.583668+0000) 2023-08-28T01:27:16.582 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:16 smithi130 bash[27579]: cluster 2023-08-28T01:27:16.114517+0000 mgr.y (mgr.14901) 6260 : cluster [DBG] pgmap v6267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:16 smithi006 bash[27292]: cluster 2023-08-28T01:27:16.114517+0000 mgr.y (mgr.14901) 6260 : cluster [DBG] pgmap v6267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:16 smithi006 bash[33918]: cluster 2023-08-28T01:27:16.114517+0000 mgr.y (mgr.14901) 6260 : cluster [DBG] pgmap v6267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:16.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:16 smithi130 bash[51302]: debug 2023-08-28T01:27:16.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:16.583975+0000) 2023-08-28T01:27:17.583 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:17 smithi130 bash[27579]: audit 2023-08-28T01:27:16.536264+0000 mon.a (mon.0) 4174 : audit [DBG] from='client.? 172.21.15.6:0/3485863088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:17.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:17 smithi006 bash[33918]: audit 2023-08-28T01:27:16.536264+0000 mon.a (mon.0) 4174 : audit [DBG] from='client.? 172.21.15.6:0/3485863088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:17.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:17 smithi006 bash[27292]: audit 2023-08-28T01:27:16.536264+0000 mon.a (mon.0) 4174 : audit [DBG] from='client.? 172.21.15.6:0/3485863088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:17.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:17 smithi130 bash[51302]: debug 2023-08-28T01:27:17.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:17.584288+0000) 2023-08-28T01:27:18.583 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:18 smithi130 bash[27579]: cluster 2023-08-28T01:27:18.115332+0000 mgr.y (mgr.14901) 6261 : cluster [DBG] pgmap v6268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:18.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:18 smithi006 bash[33918]: cluster 2023-08-28T01:27:18.115332+0000 mgr.y (mgr.14901) 6261 : cluster [DBG] pgmap v6268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:18.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:18 smithi006 bash[27292]: cluster 2023-08-28T01:27:18.115332+0000 mgr.y (mgr.14901) 6261 : cluster [DBG] pgmap v6268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:18.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:18 smithi130 bash[51302]: debug 2023-08-28T01:27:18.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:18.584596+0000) 2023-08-28T01:27:19.583 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:19 smithi130 bash[27579]: audit 2023-08-28T01:27:18.951064+0000 mon.c (mon.2) 2458 : audit [DBG] from='client.? 172.21.15.6:0/3957665518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:19.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:19 smithi006 bash[33918]: audit 2023-08-28T01:27:18.951064+0000 mon.c (mon.2) 2458 : audit [DBG] from='client.? 172.21.15.6:0/3957665518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:19.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:19 smithi006 bash[27292]: audit 2023-08-28T01:27:18.951064+0000 mon.c (mon.2) 2458 : audit [DBG] from='client.? 172.21.15.6:0/3957665518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:19.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:19 smithi130 bash[51302]: debug 2023-08-28T01:27:19.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:19.584895+0000) 2023-08-28T01:27:20.584 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:20 smithi130 bash[27579]: audit 2023-08-28T01:27:19.563332+0000 mon.b (mon.1) 717 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:27:20.584 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:20 smithi130 bash[27579]: cluster 2023-08-28T01:27:20.116656+0000 mgr.y (mgr.14901) 6262 : cluster [DBG] pgmap v6269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:20 smithi006 bash[33918]: audit 2023-08-28T01:27:19.563332+0000 mon.b (mon.1) 717 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:27:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:20 smithi006 bash[33918]: cluster 2023-08-28T01:27:20.116656+0000 mgr.y (mgr.14901) 6262 : cluster [DBG] pgmap v6269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:20 smithi006 bash[27292]: audit 2023-08-28T01:27:19.563332+0000 mon.b (mon.1) 717 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:27:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:20 smithi006 bash[27292]: cluster 2023-08-28T01:27:20.116656+0000 mgr.y (mgr.14901) 6262 : cluster [DBG] pgmap v6269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:20.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:20 smithi130 bash[51302]: debug 2023-08-28T01:27:20.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:20.585231+0000) 2023-08-28T01:27:21.019 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:27:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:27:20] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:27:21.419 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:27:21.018Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:21.419 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:27:21.020Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:21.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:21 smithi006 bash[27292]: audit 2023-08-28T01:27:21.365114+0000 mon.a (mon.0) 4175 : audit [DBG] from='client.? 172.21.15.6:0/3586119533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:21.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:21 smithi006 bash[33918]: audit 2023-08-28T01:27:21.365114+0000 mon.a (mon.0) 4175 : audit [DBG] from='client.? 172.21.15.6:0/3586119533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:21.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:21 smithi130 bash[27579]: audit 2023-08-28T01:27:21.365114+0000 mon.a (mon.0) 4175 : audit [DBG] from='client.? 172.21.15.6:0/3586119533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:21.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:21 smithi130 bash[51302]: debug 2023-08-28T01:27:21.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:21.585562+0000) 2023-08-28T01:27:22.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:22 smithi130 bash[51302]: debug 2023-08-28T01:27:22.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:22.585900+0000) 2023-08-28T01:27:22.902 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:27:22.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:22.903 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:27:22.594Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:23.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:22 smithi006 bash[27292]: cluster 2023-08-28T01:27:22.117452+0000 mgr.y (mgr.14901) 6263 : cluster [DBG] pgmap v6270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:23.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:22 smithi006 bash[33918]: cluster 2023-08-28T01:27:22.117452+0000 mgr.y (mgr.14901) 6263 : cluster [DBG] pgmap v6270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:23.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:22 smithi130 bash[27579]: cluster 2023-08-28T01:27:22.117452+0000 mgr.y (mgr.14901) 6263 : cluster [DBG] pgmap v6270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:23.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:23 smithi130 bash[51302]: debug 2023-08-28T01:27:23.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:23.586262+0000) 2023-08-28T01:27:24.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:23 smithi006 bash[33918]: audit 2023-08-28T01:27:23.781845+0000 mon.c (mon.2) 2459 : audit [DBG] from='client.? 172.21.15.6:0/4294455071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:23 smithi006 bash[27292]: audit 2023-08-28T01:27:23.781845+0000 mon.c (mon.2) 2459 : audit [DBG] from='client.? 172.21.15.6:0/4294455071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:23 smithi130 bash[27579]: audit 2023-08-28T01:27:23.781845+0000 mon.c (mon.2) 2459 : audit [DBG] from='client.? 172.21.15.6:0/4294455071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:24.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:24 smithi130 bash[51302]: debug 2023-08-28T01:27:24.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:24.586546+0000) 2023-08-28T01:27:24.912 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:27:24.471Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:24.912 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:27:24.474Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:24.915 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:24 smithi006 bash[33918]: cluster 2023-08-28T01:27:24.118338+0000 mgr.y (mgr.14901) 6264 : cluster [DBG] pgmap v6271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:25.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:24 smithi006 bash[27292]: cluster 2023-08-28T01:27:24.118338+0000 mgr.y (mgr.14901) 6264 : cluster [DBG] pgmap v6271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:25.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:24 smithi130 bash[27579]: cluster 2023-08-28T01:27:24.118338+0000 mgr.y (mgr.14901) 6264 : cluster [DBG] pgmap v6271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:25.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:25 smithi130 bash[51302]: debug 2023-08-28T01:27:25.581+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:25.586864+0000) 2023-08-28T01:27:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:26 smithi006 bash[27292]: audit 2023-08-28T01:27:25.452001+0000 mon.a (mon.0) 4176 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:27:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:26 smithi006 bash[27292]: audit 2023-08-28T01:27:25.982264+0000 mon.a (mon.0) 4177 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:27:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:26 smithi006 bash[27292]: cluster 2023-08-28T01:27:26.118950+0000 mgr.y (mgr.14901) 6265 : cluster [DBG] pgmap v6272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:26.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:26 smithi006 bash[27292]: audit 2023-08-28T01:27:26.202698+0000 mon.a (mon.0) 4178 : audit [DBG] from='client.? 172.21.15.6:0/2372900245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:26.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:26 smithi006 bash[27292]: audit 2023-08-28T01:27:26.280607+0000 mon.b (mon.1) 718 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:27:26.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:26 smithi006 bash[27292]: audit 2023-08-28T01:27:26.283112+0000 mon.b (mon.1) 719 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:27:26.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:26 smithi006 bash[27292]: audit 2023-08-28T01:27:26.290332+0000 mon.a (mon.0) 4179 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:27:26.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:26 smithi006 bash[33918]: audit 2023-08-28T01:27:25.452001+0000 mon.a (mon.0) 4176 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:27:26.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:26 smithi006 bash[33918]: audit 2023-08-28T01:27:25.982264+0000 mon.a (mon.0) 4177 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:27:26.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:26 smithi006 bash[33918]: cluster 2023-08-28T01:27:26.118950+0000 mgr.y (mgr.14901) 6265 : cluster [DBG] pgmap v6272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:26.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:26 smithi006 bash[33918]: audit 2023-08-28T01:27:26.202698+0000 mon.a (mon.0) 4178 : audit [DBG] from='client.? 172.21.15.6:0/2372900245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:26.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:26 smithi006 bash[33918]: audit 2023-08-28T01:27:26.280607+0000 mon.b (mon.1) 718 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:27:26.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:26 smithi006 bash[33918]: audit 2023-08-28T01:27:26.283112+0000 mon.b (mon.1) 719 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:27:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:26 smithi006 bash[33918]: audit 2023-08-28T01:27:26.290332+0000 mon.a (mon.0) 4179 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:27:26.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:26 smithi130 bash[51302]: debug 2023-08-28T01:27:26.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:26.587176+0000) 2023-08-28T01:27:26.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:26 smithi130 bash[27579]: audit 2023-08-28T01:27:25.452001+0000 mon.a (mon.0) 4176 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:27:26.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:26 smithi130 bash[27579]: audit 2023-08-28T01:27:25.982264+0000 mon.a (mon.0) 4177 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:27:26.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:26 smithi130 bash[27579]: cluster 2023-08-28T01:27:26.118950+0000 mgr.y (mgr.14901) 6265 : cluster [DBG] pgmap v6272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:26.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:26 smithi130 bash[27579]: audit 2023-08-28T01:27:26.202698+0000 mon.a (mon.0) 4178 : audit [DBG] from='client.? 172.21.15.6:0/2372900245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:26.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:26 smithi130 bash[27579]: audit 2023-08-28T01:27:26.280607+0000 mon.b (mon.1) 718 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:27:26.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:26 smithi130 bash[27579]: audit 2023-08-28T01:27:26.283112+0000 mon.b (mon.1) 719 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:27:26.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:26 smithi130 bash[27579]: audit 2023-08-28T01:27:26.290332+0000 mon.a (mon.0) 4179 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:27:27.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:27 smithi130 bash[51302]: debug 2023-08-28T01:27:27.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:27.587519+0000) 2023-08-28T01:27:28.586 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:28 smithi130 bash[27579]: cluster 2023-08-28T01:27:28.119561+0000 mgr.y (mgr.14901) 6266 : cluster [DBG] pgmap v6273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:28 smithi006 bash[33918]: cluster 2023-08-28T01:27:28.119561+0000 mgr.y (mgr.14901) 6266 : cluster [DBG] pgmap v6273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:28 smithi006 bash[27292]: cluster 2023-08-28T01:27:28.119561+0000 mgr.y (mgr.14901) 6266 : cluster [DBG] pgmap v6273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:28.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:28 smithi130 bash[51302]: debug 2023-08-28T01:27:28.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:28.587865+0000) 2023-08-28T01:27:29.587 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:29 smithi130 bash[27579]: audit 2023-08-28T01:27:28.615411+0000 mon.a (mon.0) 4180 : audit [DBG] from='client.? 172.21.15.6:0/1319141808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:29.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:29 smithi006 bash[33918]: audit 2023-08-28T01:27:28.615411+0000 mon.a (mon.0) 4180 : audit [DBG] from='client.? 172.21.15.6:0/1319141808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:29.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:29 smithi006 bash[27292]: audit 2023-08-28T01:27:28.615411+0000 mon.a (mon.0) 4180 : audit [DBG] from='client.? 172.21.15.6:0/1319141808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:29.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:29 smithi130 bash[51302]: debug 2023-08-28T01:27:29.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:29.588190+0000) 2023-08-28T01:27:30.587 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:30 smithi130 bash[27579]: cluster 2023-08-28T01:27:30.120698+0000 mgr.y (mgr.14901) 6267 : cluster [DBG] pgmap v6274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:30 smithi006 bash[27292]: cluster 2023-08-28T01:27:30.120698+0000 mgr.y (mgr.14901) 6267 : cluster [DBG] pgmap v6274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:30 smithi006 bash[33918]: cluster 2023-08-28T01:27:30.120698+0000 mgr.y (mgr.14901) 6267 : cluster [DBG] pgmap v6274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:30.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:30 smithi130 bash[51302]: debug 2023-08-28T01:27:30.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:30.588545+0000) 2023-08-28T01:27:31.020 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:27:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:27:30] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T01:27:31.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:31 smithi006 bash[33918]: audit 2023-08-28T01:27:31.033405+0000 mon.c (mon.2) 2460 : audit [DBG] from='client.? 172.21.15.6:0/411259352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:31.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:31 smithi006 bash[27292]: audit 2023-08-28T01:27:31.033405+0000 mon.c (mon.2) 2460 : audit [DBG] from='client.? 172.21.15.6:0/411259352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:31.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:27:31.019Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:31.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:27:31.021Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:31.587 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:31 smithi130 bash[27579]: audit 2023-08-28T01:27:31.033405+0000 mon.c (mon.2) 2460 : audit [DBG] from='client.? 172.21.15.6:0/411259352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:31.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:31 smithi130 bash[51302]: debug 2023-08-28T01:27:31.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:31.588838+0000) 2023-08-28T01:27:32.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:32 smithi130 bash[51302]: debug 2023-08-28T01:27:32.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:32.589158+0000) 2023-08-28T01:27:32.904 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:27:32.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:32.905 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:27:32.594Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:33.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:32 smithi006 bash[27292]: cluster 2023-08-28T01:27:32.121344+0000 mgr.y (mgr.14901) 6268 : cluster [DBG] pgmap v6275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:33.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:32 smithi006 bash[33918]: cluster 2023-08-28T01:27:32.121344+0000 mgr.y (mgr.14901) 6268 : cluster [DBG] pgmap v6275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:33.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:32 smithi130 bash[27579]: cluster 2023-08-28T01:27:32.121344+0000 mgr.y (mgr.14901) 6268 : cluster [DBG] pgmap v6275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:33.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:33 smithi130 bash[51302]: debug 2023-08-28T01:27:33.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:33.589511+0000) 2023-08-28T01:27:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:33 smithi006 bash[27292]: audit 2023-08-28T01:27:33.446170+0000 mon.a (mon.0) 4181 : audit [DBG] from='client.? 172.21.15.6:0/1125211231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:33 smithi006 bash[33918]: audit 2023-08-28T01:27:33.446170+0000 mon.a (mon.0) 4181 : audit [DBG] from='client.? 172.21.15.6:0/1125211231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:33 smithi130 bash[27579]: audit 2023-08-28T01:27:33.446170+0000 mon.a (mon.0) 4181 : audit [DBG] from='client.? 172.21.15.6:0/1125211231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:34.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:34 smithi130 bash[51302]: debug 2023-08-28T01:27:34.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:34.589847+0000) 2023-08-28T01:27:34.910 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:27:34.472Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:34.910 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:27:34.474Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:35.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:34 smithi006 bash[27292]: cluster 2023-08-28T01:27:34.122296+0000 mgr.y (mgr.14901) 6269 : cluster [DBG] pgmap v6276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:35.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:34 smithi006 bash[33918]: cluster 2023-08-28T01:27:34.122296+0000 mgr.y (mgr.14901) 6269 : cluster [DBG] pgmap v6276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:35.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:34 smithi130 bash[27579]: cluster 2023-08-28T01:27:34.122296+0000 mgr.y (mgr.14901) 6269 : cluster [DBG] pgmap v6276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:35.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:35 smithi130 bash[51302]: debug 2023-08-28T01:27:35.585+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:35.590112+0000) 2023-08-28T01:27:36.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:35 smithi006 bash[33918]: audit 2023-08-28T01:27:35.865801+0000 mon.a (mon.0) 4182 : audit [DBG] from='client.? 172.21.15.6:0/3618650437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:36.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:35 smithi006 bash[27292]: audit 2023-08-28T01:27:35.865801+0000 mon.a (mon.0) 4182 : audit [DBG] from='client.? 172.21.15.6:0/3618650437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:36.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:35 smithi130 bash[27579]: audit 2023-08-28T01:27:35.865801+0000 mon.a (mon.0) 4182 : audit [DBG] from='client.? 172.21.15.6:0/3618650437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:36.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:36 smithi130 bash[51302]: debug 2023-08-28T01:27:36.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:36.590439+0000) 2023-08-28T01:27:37.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:36 smithi006 bash[27292]: cluster 2023-08-28T01:27:36.123117+0000 mgr.y (mgr.14901) 6270 : cluster [DBG] pgmap v6277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:37.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:36 smithi006 bash[33918]: cluster 2023-08-28T01:27:36.123117+0000 mgr.y (mgr.14901) 6270 : cluster [DBG] pgmap v6277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:37.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:36 smithi130 bash[27579]: cluster 2023-08-28T01:27:36.123117+0000 mgr.y (mgr.14901) 6270 : cluster [DBG] pgmap v6277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:37.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:37 smithi130 bash[51302]: debug 2023-08-28T01:27:37.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:37.590791+0000) 2023-08-28T01:27:38.590 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:38 smithi130 bash[27579]: cluster 2023-08-28T01:27:38.123746+0000 mgr.y (mgr.14901) 6271 : cluster [DBG] pgmap v6278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:38 smithi006 bash[27292]: cluster 2023-08-28T01:27:38.123746+0000 mgr.y (mgr.14901) 6271 : cluster [DBG] pgmap v6278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:38.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:38 smithi006 bash[33918]: cluster 2023-08-28T01:27:38.123746+0000 mgr.y (mgr.14901) 6271 : cluster [DBG] pgmap v6278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:38.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:38 smithi130 bash[51302]: debug 2023-08-28T01:27:38.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:38.591170+0000) 2023-08-28T01:27:39.590 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:39 smithi130 bash[27579]: audit 2023-08-28T01:27:38.278674+0000 mon.a (mon.0) 4183 : audit [DBG] from='client.? 172.21.15.6:0/2244783460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:39.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:39 smithi006 bash[27292]: audit 2023-08-28T01:27:38.278674+0000 mon.a (mon.0) 4183 : audit [DBG] from='client.? 172.21.15.6:0/2244783460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:39.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:39 smithi006 bash[33918]: audit 2023-08-28T01:27:38.278674+0000 mon.a (mon.0) 4183 : audit [DBG] from='client.? 172.21.15.6:0/2244783460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:39.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:39 smithi130 bash[51302]: debug 2023-08-28T01:27:39.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:39.591511+0000) 2023-08-28T01:27:40.590 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:40 smithi130 bash[27579]: cluster 2023-08-28T01:27:40.125036+0000 mgr.y (mgr.14901) 6272 : cluster [DBG] pgmap v6279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:40 smithi006 bash[27292]: cluster 2023-08-28T01:27:40.125036+0000 mgr.y (mgr.14901) 6272 : cluster [DBG] pgmap v6279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:40 smithi006 bash[33918]: cluster 2023-08-28T01:27:40.125036+0000 mgr.y (mgr.14901) 6272 : cluster [DBG] pgmap v6279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:40.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:40 smithi130 bash[51302]: debug 2023-08-28T01:27:40.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:40.591878+0000) 2023-08-28T01:27:41.021 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:27:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:27:40] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T01:27:41.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:41 smithi006 bash[33918]: audit 2023-08-28T01:27:40.690583+0000 mon.a (mon.0) 4184 : audit [DBG] from='client.? 172.21.15.6:0/2979624669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:27:41.020Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:41.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:27:41.022Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:41.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:41 smithi006 bash[27292]: audit 2023-08-28T01:27:40.690583+0000 mon.a (mon.0) 4184 : audit [DBG] from='client.? 172.21.15.6:0/2979624669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:41.591 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:41 smithi130 bash[27579]: audit 2023-08-28T01:27:40.690583+0000 mon.a (mon.0) 4184 : audit [DBG] from='client.? 172.21.15.6:0/2979624669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:41.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:41 smithi130 bash[51302]: debug 2023-08-28T01:27:41.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:41.592244+0000) 2023-08-28T01:27:42.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:42 smithi130 bash[51302]: debug 2023-08-28T01:27:42.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:42.592509+0000) 2023-08-28T01:27:42.908 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:27:42.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:42.908 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:27:42.595Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:43.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:42 smithi006 bash[33918]: cluster 2023-08-28T01:27:42.125695+0000 mgr.y (mgr.14901) 6273 : cluster [DBG] pgmap v6280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:43.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:42 smithi006 bash[27292]: cluster 2023-08-28T01:27:42.125695+0000 mgr.y (mgr.14901) 6273 : cluster [DBG] pgmap v6280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:43.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:42 smithi130 bash[27579]: cluster 2023-08-28T01:27:42.125695+0000 mgr.y (mgr.14901) 6273 : cluster [DBG] pgmap v6280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:43.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:43 smithi130 bash[51302]: debug 2023-08-28T01:27:43.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:43.592825+0000) 2023-08-28T01:27:44.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:43 smithi006 bash[33918]: audit 2023-08-28T01:27:43.111643+0000 mon.a (mon.0) 4185 : audit [DBG] from='client.? 172.21.15.6:0/3053708461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:44.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:43 smithi006 bash[27292]: audit 2023-08-28T01:27:43.111643+0000 mon.a (mon.0) 4185 : audit [DBG] from='client.? 172.21.15.6:0/3053708461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:44.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:43 smithi130 bash[27579]: audit 2023-08-28T01:27:43.111643+0000 mon.a (mon.0) 4185 : audit [DBG] from='client.? 172.21.15.6:0/3053708461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:44.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:44 smithi130 bash[51302]: debug 2023-08-28T01:27:44.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:44.593137+0000) 2023-08-28T01:27:44.914 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:27:44.472Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:44.914 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:27:44.475Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:45.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:44 smithi006 bash[33918]: cluster 2023-08-28T01:27:44.126574+0000 mgr.y (mgr.14901) 6274 : cluster [DBG] pgmap v6281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:45.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:44 smithi006 bash[27292]: cluster 2023-08-28T01:27:44.126574+0000 mgr.y (mgr.14901) 6274 : cluster [DBG] pgmap v6281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:45.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:44 smithi130 bash[27579]: cluster 2023-08-28T01:27:44.126574+0000 mgr.y (mgr.14901) 6274 : cluster [DBG] pgmap v6281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:45.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:45 smithi130 bash[51302]: debug 2023-08-28T01:27:45.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:45.593467+0000) 2023-08-28T01:27:46.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:45 smithi006 bash[33918]: audit 2023-08-28T01:27:45.524445+0000 mon.c (mon.2) 2461 : audit [DBG] from='client.? 172.21.15.6:0/1048916296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:46.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:45 smithi006 bash[27292]: audit 2023-08-28T01:27:45.524445+0000 mon.c (mon.2) 2461 : audit [DBG] from='client.? 172.21.15.6:0/1048916296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:46.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:45 smithi130 bash[27579]: audit 2023-08-28T01:27:45.524445+0000 mon.c (mon.2) 2461 : audit [DBG] from='client.? 172.21.15.6:0/1048916296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:46.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:46 smithi130 bash[51302]: debug 2023-08-28T01:27:46.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:46.593811+0000) 2023-08-28T01:27:47.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:46 smithi006 bash[33918]: cluster 2023-08-28T01:27:46.127483+0000 mgr.y (mgr.14901) 6275 : cluster [DBG] pgmap v6282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:47.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:46 smithi006 bash[27292]: cluster 2023-08-28T01:27:46.127483+0000 mgr.y (mgr.14901) 6275 : cluster [DBG] pgmap v6282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:47.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:46 smithi130 bash[27579]: cluster 2023-08-28T01:27:46.127483+0000 mgr.y (mgr.14901) 6275 : cluster [DBG] pgmap v6282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:47.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:47 smithi130 bash[51302]: debug 2023-08-28T01:27:47.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:47.594189+0000) 2023-08-28T01:27:48.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:47 smithi130 bash[27579]: audit 2023-08-28T01:27:47.936421+0000 mon.c (mon.2) 2462 : audit [DBG] from='client.? 172.21.15.6:0/2885443328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:48.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:47 smithi006 bash[27292]: audit 2023-08-28T01:27:47.936421+0000 mon.c (mon.2) 2462 : audit [DBG] from='client.? 172.21.15.6:0/2885443328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:48.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:47 smithi006 bash[33918]: audit 2023-08-28T01:27:47.936421+0000 mon.c (mon.2) 2462 : audit [DBG] from='client.? 172.21.15.6:0/2885443328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:48.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:48 smithi130 bash[51302]: debug 2023-08-28T01:27:48.589+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:48.594517+0000) 2023-08-28T01:27:49.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:49 smithi130 bash[27579]: cluster 2023-08-28T01:27:48.128173+0000 mgr.y (mgr.14901) 6276 : cluster [DBG] pgmap v6283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:49.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:48 smithi006 bash[27292]: cluster 2023-08-28T01:27:48.128173+0000 mgr.y (mgr.14901) 6276 : cluster [DBG] pgmap v6283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:49.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:48 smithi006 bash[33918]: cluster 2023-08-28T01:27:48.128173+0000 mgr.y (mgr.14901) 6276 : cluster [DBG] pgmap v6283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:49.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:49 smithi130 bash[51302]: debug 2023-08-28T01:27:49.593+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:49.594895+0000) 2023-08-28T01:27:50.594 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:50 smithi130 bash[27579]: cluster 2023-08-28T01:27:50.129308+0000 mgr.y (mgr.14901) 6277 : cluster [DBG] pgmap v6284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:50.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:50 smithi006 bash[33918]: cluster 2023-08-28T01:27:50.129308+0000 mgr.y (mgr.14901) 6277 : cluster [DBG] pgmap v6284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:50 smithi006 bash[27292]: cluster 2023-08-28T01:27:50.129308+0000 mgr.y (mgr.14901) 6277 : cluster [DBG] pgmap v6284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:50.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:50 smithi130 bash[51302]: debug 2023-08-28T01:27:50.593+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:50.595273+0000) 2023-08-28T01:27:51.022 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:27:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:27:50] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:27:51.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:51 smithi006 bash[27292]: audit 2023-08-28T01:27:50.357445+0000 mon.c (mon.2) 2463 : audit [DBG] from='client.? 172.21.15.6:0/1714941297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:51.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:51 smithi006 bash[33918]: audit 2023-08-28T01:27:50.357445+0000 mon.c (mon.2) 2463 : audit [DBG] from='client.? 172.21.15.6:0/1714941297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:51.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:27:51.021Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:51.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:27:51.023Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:51.594 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:51 smithi130 bash[27579]: audit 2023-08-28T01:27:50.357445+0000 mon.c (mon.2) 2463 : audit [DBG] from='client.? 172.21.15.6:0/1714941297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:51.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:51 smithi130 bash[51302]: debug 2023-08-28T01:27:51.592+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:51.595574+0000) 2023-08-28T01:27:52.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:52 smithi130 bash[51302]: debug 2023-08-28T01:27:52.592+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:52.595885+0000) 2023-08-28T01:27:52.910 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:27:52.594Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:52.910 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:27:52.596Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:53.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:52 smithi006 bash[33918]: cluster 2023-08-28T01:27:52.129958+0000 mgr.y (mgr.14901) 6278 : cluster [DBG] pgmap v6285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:53.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:52 smithi006 bash[33918]: audit 2023-08-28T01:27:52.768794+0000 mon.c (mon.2) 2464 : audit [DBG] from='client.? 172.21.15.6:0/2157509469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:53.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:52 smithi006 bash[27292]: cluster 2023-08-28T01:27:52.129958+0000 mgr.y (mgr.14901) 6278 : cluster [DBG] pgmap v6285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:53.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:52 smithi006 bash[27292]: audit 2023-08-28T01:27:52.768794+0000 mon.c (mon.2) 2464 : audit [DBG] from='client.? 172.21.15.6:0/2157509469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:53.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:52 smithi130 bash[27579]: cluster 2023-08-28T01:27:52.129958+0000 mgr.y (mgr.14901) 6278 : cluster [DBG] pgmap v6285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:53.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:52 smithi130 bash[27579]: audit 2023-08-28T01:27:52.768794+0000 mon.c (mon.2) 2464 : audit [DBG] from='client.? 172.21.15.6:0/2157509469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:53.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:53 smithi130 bash[51302]: debug 2023-08-28T01:27:53.592+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:53.596197+0000) 2023-08-28T01:27:54.476 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:27:54.473Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:54.477 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:54 smithi006 bash[27292]: cluster 2023-08-28T01:27:54.130959+0000 mgr.y (mgr.14901) 6279 : cluster [DBG] pgmap v6286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:54.477 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:54 smithi006 bash[33918]: cluster 2023-08-28T01:27:54.130959+0000 mgr.y (mgr.14901) 6279 : cluster [DBG] pgmap v6286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:54.595 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:54 smithi130 bash[27579]: cluster 2023-08-28T01:27:54.130959+0000 mgr.y (mgr.14901) 6279 : cluster [DBG] pgmap v6286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:54.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:54 smithi130 bash[51302]: debug 2023-08-28T01:27:54.592+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:54.596558+0000) 2023-08-28T01:27:55.595 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:55 smithi130 bash[27579]: audit 2023-08-28T01:27:55.189318+0000 mon.a (mon.0) 4186 : audit [DBG] from='client.? 172.21.15.6:0/2078367454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:55.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:55 smithi006 bash[27292]: audit 2023-08-28T01:27:55.189318+0000 mon.a (mon.0) 4186 : audit [DBG] from='client.? 172.21.15.6:0/2078367454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:55.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:27:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:27:54.476Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:27:55.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:55 smithi006 bash[33918]: audit 2023-08-28T01:27:55.189318+0000 mon.a (mon.0) 4186 : audit [DBG] from='client.? 172.21.15.6:0/2078367454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:55.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:55 smithi130 bash[51302]: debug 2023-08-28T01:27:55.592+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:55.596860+0000) 2023-08-28T01:27:56.596 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:56 smithi130 bash[27579]: cluster 2023-08-28T01:27:56.131773+0000 mgr.y (mgr.14901) 6280 : cluster [DBG] pgmap v6287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:56.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:56 smithi006 bash[27292]: cluster 2023-08-28T01:27:56.131773+0000 mgr.y (mgr.14901) 6280 : cluster [DBG] pgmap v6287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:56.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:56 smithi006 bash[33918]: cluster 2023-08-28T01:27:56.131773+0000 mgr.y (mgr.14901) 6280 : cluster [DBG] pgmap v6287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:56.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:56 smithi130 bash[51302]: debug 2023-08-28T01:27:56.592+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:56.597177+0000) 2023-08-28T01:27:57.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:57 smithi130 bash[51302]: debug 2023-08-28T01:27:57.592+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:57.597480+0000) 2023-08-28T01:27:58.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:57 smithi006 bash[33918]: audit 2023-08-28T01:27:57.600362+0000 mon.a (mon.0) 4187 : audit [DBG] from='client.? 172.21.15.6:0/2759443435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:58.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:57 smithi006 bash[27292]: audit 2023-08-28T01:27:57.600362+0000 mon.a (mon.0) 4187 : audit [DBG] from='client.? 172.21.15.6:0/2759443435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:58.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:57 smithi130 bash[27579]: audit 2023-08-28T01:27:57.600362+0000 mon.a (mon.0) 4187 : audit [DBG] from='client.? 172.21.15.6:0/2759443435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:58.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:58 smithi130 bash[51302]: debug 2023-08-28T01:27:58.592+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:58.597855+0000) 2023-08-28T01:27:59.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:27:58 smithi006 bash[33918]: cluster 2023-08-28T01:27:58.132452+0000 mgr.y (mgr.14901) 6281 : cluster [DBG] pgmap v6288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:59.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:27:58 smithi006 bash[27292]: cluster 2023-08-28T01:27:58.132452+0000 mgr.y (mgr.14901) 6281 : cluster [DBG] pgmap v6288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:59.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:27:58 smithi130 bash[27579]: cluster 2023-08-28T01:27:58.132452+0000 mgr.y (mgr.14901) 6281 : cluster [DBG] pgmap v6288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:27:59.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:27:59 smithi130 bash[51302]: debug 2023-08-28T01:27:59.592+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:59.598207+0000) 2023-08-28T01:28:00.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:00 smithi130 bash[27579]: audit 2023-08-28T01:28:00.010718+0000 mon.c (mon.2) 2465 : audit [DBG] from='client.? 172.21.15.6:0/402878012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:00.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:00 smithi006 bash[33918]: audit 2023-08-28T01:28:00.010718+0000 mon.c (mon.2) 2465 : audit [DBG] from='client.? 172.21.15.6:0/402878012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:00.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:00 smithi006 bash[27292]: audit 2023-08-28T01:28:00.010718+0000 mon.c (mon.2) 2465 : audit [DBG] from='client.? 172.21.15.6:0/402878012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:00.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:00 smithi130 bash[51302]: debug 2023-08-28T01:28:00.592+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:00.598568+0000) 2023-08-28T01:28:01.023 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:28:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:28:00] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:28:01.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:01 smithi130 bash[27579]: cluster 2023-08-28T01:28:00.133580+0000 mgr.y (mgr.14901) 6282 : cluster [DBG] pgmap v6289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:01.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:01 smithi006 bash[33918]: cluster 2023-08-28T01:28:00.133580+0000 mgr.y (mgr.14901) 6282 : cluster [DBG] pgmap v6289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:01.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:01 smithi006 bash[27292]: cluster 2023-08-28T01:28:00.133580+0000 mgr.y (mgr.14901) 6282 : cluster [DBG] pgmap v6289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:01.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:28:01.022Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:01.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:28:01.025Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:01.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:01 smithi130 bash[51302]: debug 2023-08-28T01:28:01.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:01.598950+0000) 2023-08-28T01:28:02.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:02 smithi130 bash[51302]: debug 2023-08-28T01:28:02.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:02.599250+0000) 2023-08-28T01:28:02.927 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:28:02.595Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:02.927 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:28:02.597Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:03.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:02 smithi006 bash[27292]: cluster 2023-08-28T01:28:02.134211+0000 mgr.y (mgr.14901) 6283 : cluster [DBG] pgmap v6290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:03.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:02 smithi006 bash[27292]: audit 2023-08-28T01:28:02.423688+0000 mon.c (mon.2) 2466 : audit [DBG] from='client.? 172.21.15.6:0/13604599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:03.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:02 smithi006 bash[33918]: cluster 2023-08-28T01:28:02.134211+0000 mgr.y (mgr.14901) 6283 : cluster [DBG] pgmap v6290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:03.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:02 smithi006 bash[33918]: audit 2023-08-28T01:28:02.423688+0000 mon.c (mon.2) 2466 : audit [DBG] from='client.? 172.21.15.6:0/13604599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:03.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:02 smithi130 bash[27579]: cluster 2023-08-28T01:28:02.134211+0000 mgr.y (mgr.14901) 6283 : cluster [DBG] pgmap v6290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:03.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:02 smithi130 bash[27579]: audit 2023-08-28T01:28:02.423688+0000 mon.c (mon.2) 2466 : audit [DBG] from='client.? 172.21.15.6:0/13604599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:03.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:03 smithi130 bash[51302]: debug 2023-08-28T01:28:03.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:03.599516+0000) 2023-08-28T01:28:04.598 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:04 smithi130 bash[27579]: cluster 2023-08-28T01:28:04.135222+0000 mgr.y (mgr.14901) 6284 : cluster [DBG] pgmap v6291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:04.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:04 smithi006 bash[27292]: cluster 2023-08-28T01:28:04.135222+0000 mgr.y (mgr.14901) 6284 : cluster [DBG] pgmap v6291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:04.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:04 smithi006 bash[33918]: cluster 2023-08-28T01:28:04.135222+0000 mgr.y (mgr.14901) 6284 : cluster [DBG] pgmap v6291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:28:04.475Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:28:04.476Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:04.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:04 smithi130 bash[51302]: debug 2023-08-28T01:28:04.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:04.599830+0000) 2023-08-28T01:28:05.599 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:05 smithi130 bash[27579]: audit 2023-08-28T01:28:04.843529+0000 mon.c (mon.2) 2467 : audit [DBG] from='client.? 172.21.15.6:0/3230986518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:05.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:05 smithi006 bash[33918]: audit 2023-08-28T01:28:04.843529+0000 mon.c (mon.2) 2467 : audit [DBG] from='client.? 172.21.15.6:0/3230986518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:05.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:05 smithi006 bash[27292]: audit 2023-08-28T01:28:04.843529+0000 mon.c (mon.2) 2467 : audit [DBG] from='client.? 172.21.15.6:0/3230986518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:05.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:05 smithi130 bash[51302]: debug 2023-08-28T01:28:05.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:05.600170+0000) 2023-08-28T01:28:06.599 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:06 smithi130 bash[27579]: cluster 2023-08-28T01:28:06.135878+0000 mgr.y (mgr.14901) 6285 : cluster [DBG] pgmap v6292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:06 smithi006 bash[33918]: cluster 2023-08-28T01:28:06.135878+0000 mgr.y (mgr.14901) 6285 : cluster [DBG] pgmap v6292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:06 smithi006 bash[27292]: cluster 2023-08-28T01:28:06.135878+0000 mgr.y (mgr.14901) 6285 : cluster [DBG] pgmap v6292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:06.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:06 smithi130 bash[51302]: debug 2023-08-28T01:28:06.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:06.600442+0000) 2023-08-28T01:28:07.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:07 smithi130 bash[51302]: debug 2023-08-28T01:28:07.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:07.600788+0000) 2023-08-28T01:28:08.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:07 smithi006 bash[33918]: audit 2023-08-28T01:28:07.267258+0000 mon.c (mon.2) 2468 : audit [DBG] from='client.? 172.21.15.6:0/1811948061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:08.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:07 smithi006 bash[27292]: audit 2023-08-28T01:28:07.267258+0000 mon.c (mon.2) 2468 : audit [DBG] from='client.? 172.21.15.6:0/1811948061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:08.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:07 smithi130 bash[27579]: audit 2023-08-28T01:28:07.267258+0000 mon.c (mon.2) 2468 : audit [DBG] from='client.? 172.21.15.6:0/1811948061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:08.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:08 smithi130 bash[51302]: debug 2023-08-28T01:28:08.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:08.601130+0000) 2023-08-28T01:28:09.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:08 smithi006 bash[33918]: cluster 2023-08-28T01:28:08.136551+0000 mgr.y (mgr.14901) 6286 : cluster [DBG] pgmap v6293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:09.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:08 smithi006 bash[27292]: cluster 2023-08-28T01:28:08.136551+0000 mgr.y (mgr.14901) 6286 : cluster [DBG] pgmap v6293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:09.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:08 smithi130 bash[27579]: cluster 2023-08-28T01:28:08.136551+0000 mgr.y (mgr.14901) 6286 : cluster [DBG] pgmap v6293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:09.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:09 smithi130 bash[51302]: debug 2023-08-28T01:28:09.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:09.601430+0000) 2023-08-28T01:28:10.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:09 smithi006 bash[27292]: audit 2023-08-28T01:28:09.686441+0000 mon.c (mon.2) 2469 : audit [DBG] from='client.? 172.21.15.6:0/423175931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:10.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:09 smithi006 bash[33918]: audit 2023-08-28T01:28:09.686441+0000 mon.c (mon.2) 2469 : audit [DBG] from='client.? 172.21.15.6:0/423175931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:10.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:09 smithi130 bash[27579]: audit 2023-08-28T01:28:09.686441+0000 mon.c (mon.2) 2469 : audit [DBG] from='client.? 172.21.15.6:0/423175931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:10.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:10 smithi130 bash[51302]: debug 2023-08-28T01:28:10.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:10.601709+0000) 2023-08-28T01:28:11.024 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:28:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:28:10] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:28:11.024 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:10 smithi006 bash[33918]: cluster 2023-08-28T01:28:10.137856+0000 mgr.y (mgr.14901) 6287 : cluster [DBG] pgmap v6294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:11.025 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:10 smithi006 bash[27292]: cluster 2023-08-28T01:28:10.137856+0000 mgr.y (mgr.14901) 6287 : cluster [DBG] pgmap v6294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:11.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:10 smithi130 bash[27579]: cluster 2023-08-28T01:28:10.137856+0000 mgr.y (mgr.14901) 6287 : cluster [DBG] pgmap v6294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:28:11.023Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:28:11.025Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:11.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:11 smithi130 bash[51302]: debug 2023-08-28T01:28:11.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:11.601975+0000) 2023-08-28T01:28:12.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:12 smithi130 bash[51302]: debug 2023-08-28T01:28:12.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:12.602370+0000) 2023-08-28T01:28:12.932 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:28:12.595Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:12.932 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:28:12.597Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:13.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:12 smithi006 bash[27292]: audit 2023-08-28T01:28:12.100404+0000 mon.a (mon.0) 4188 : audit [DBG] from='client.? 172.21.15.6:0/2458618354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:13.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:12 smithi006 bash[27292]: cluster 2023-08-28T01:28:12.138496+0000 mgr.y (mgr.14901) 6288 : cluster [DBG] pgmap v6295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:13.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:12 smithi006 bash[33918]: audit 2023-08-28T01:28:12.100404+0000 mon.a (mon.0) 4188 : audit [DBG] from='client.? 172.21.15.6:0/2458618354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:13.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:12 smithi006 bash[33918]: cluster 2023-08-28T01:28:12.138496+0000 mgr.y (mgr.14901) 6288 : cluster [DBG] pgmap v6295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:13.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:12 smithi130 bash[27579]: audit 2023-08-28T01:28:12.100404+0000 mon.a (mon.0) 4188 : audit [DBG] from='client.? 172.21.15.6:0/2458618354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:13.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:12 smithi130 bash[27579]: cluster 2023-08-28T01:28:12.138496+0000 mgr.y (mgr.14901) 6288 : cluster [DBG] pgmap v6295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:13.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:13 smithi130 bash[51302]: debug 2023-08-28T01:28:13.596+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:13.602686+0000) 2023-08-28T01:28:14.601 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:14 smithi130 bash[27579]: cluster 2023-08-28T01:28:14.139574+0000 mgr.y (mgr.14901) 6289 : cluster [DBG] pgmap v6296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:14.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:14 smithi006 bash[27292]: cluster 2023-08-28T01:28:14.139574+0000 mgr.y (mgr.14901) 6289 : cluster [DBG] pgmap v6296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:14.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:14 smithi006 bash[33918]: cluster 2023-08-28T01:28:14.139574+0000 mgr.y (mgr.14901) 6289 : cluster [DBG] pgmap v6296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:28:14.475Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:28:14.477Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:14.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:14 smithi130 bash[51302]: debug 2023-08-28T01:28:14.600+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:14.602950+0000) 2023-08-28T01:28:15.603 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:15 smithi130 bash[27579]: audit 2023-08-28T01:28:14.514538+0000 mon.c (mon.2) 2470 : audit [DBG] from='client.? 172.21.15.6:0/3642952431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:15.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:15 smithi006 bash[27292]: audit 2023-08-28T01:28:14.514538+0000 mon.c (mon.2) 2470 : audit [DBG] from='client.? 172.21.15.6:0/3642952431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:15.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:15 smithi006 bash[33918]: audit 2023-08-28T01:28:14.514538+0000 mon.c (mon.2) 2470 : audit [DBG] from='client.? 172.21.15.6:0/3642952431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:15.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:15 smithi130 bash[51302]: debug 2023-08-28T01:28:15.600+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:15.603219+0000) 2023-08-28T01:28:16.602 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:16 smithi130 bash[27579]: cluster 2023-08-28T01:28:16.140366+0000 mgr.y (mgr.14901) 6290 : cluster [DBG] pgmap v6297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:16 smithi006 bash[33918]: cluster 2023-08-28T01:28:16.140366+0000 mgr.y (mgr.14901) 6290 : cluster [DBG] pgmap v6297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:16 smithi006 bash[27292]: cluster 2023-08-28T01:28:16.140366+0000 mgr.y (mgr.14901) 6290 : cluster [DBG] pgmap v6297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:16.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:16 smithi130 bash[51302]: debug 2023-08-28T01:28:16.600+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:16.603464+0000) 2023-08-28T01:28:17.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:17 smithi130 bash[51302]: debug 2023-08-28T01:28:17.600+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:17.603740+0000) 2023-08-28T01:28:18.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:17 smithi006 bash[27292]: audit 2023-08-28T01:28:16.929126+0000 mon.a (mon.0) 4189 : audit [DBG] from='client.? 172.21.15.6:0/4031853456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:18.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:17 smithi006 bash[33918]: audit 2023-08-28T01:28:16.929126+0000 mon.a (mon.0) 4189 : audit [DBG] from='client.? 172.21.15.6:0/4031853456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:18.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:17 smithi130 bash[27579]: audit 2023-08-28T01:28:16.929126+0000 mon.a (mon.0) 4189 : audit [DBG] from='client.? 172.21.15.6:0/4031853456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:18.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:18 smithi130 bash[51302]: debug 2023-08-28T01:28:18.600+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:18.604153+0000) 2023-08-28T01:28:19.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:18 smithi006 bash[33918]: cluster 2023-08-28T01:28:18.141137+0000 mgr.y (mgr.14901) 6291 : cluster [DBG] pgmap v6298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:19.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:18 smithi006 bash[27292]: cluster 2023-08-28T01:28:18.141137+0000 mgr.y (mgr.14901) 6291 : cluster [DBG] pgmap v6298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:19.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:18 smithi130 bash[27579]: cluster 2023-08-28T01:28:18.141137+0000 mgr.y (mgr.14901) 6291 : cluster [DBG] pgmap v6298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:19.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:19 smithi130 bash[51302]: debug 2023-08-28T01:28:19.600+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:19.604455+0000) 2023-08-28T01:28:20.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:19 smithi006 bash[33918]: audit 2023-08-28T01:28:19.344365+0000 mon.a (mon.0) 4190 : audit [DBG] from='client.? 172.21.15.6:0/563656614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:20.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:19 smithi006 bash[27292]: audit 2023-08-28T01:28:19.344365+0000 mon.a (mon.0) 4190 : audit [DBG] from='client.? 172.21.15.6:0/563656614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:20.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:19 smithi130 bash[27579]: audit 2023-08-28T01:28:19.344365+0000 mon.a (mon.0) 4190 : audit [DBG] from='client.? 172.21.15.6:0/563656614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:20.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:20 smithi130 bash[51302]: debug 2023-08-28T01:28:20.600+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:20.604752+0000) 2023-08-28T01:28:21.025 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:28:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:28:20] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:28:21.025 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:20 smithi006 bash[33918]: cluster 2023-08-28T01:28:20.142249+0000 mgr.y (mgr.14901) 6292 : cluster [DBG] pgmap v6299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:21.025 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:20 smithi006 bash[27292]: cluster 2023-08-28T01:28:20.142249+0000 mgr.y (mgr.14901) 6292 : cluster [DBG] pgmap v6299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:21.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:20 smithi130 bash[27579]: cluster 2023-08-28T01:28:20.142249+0000 mgr.y (mgr.14901) 6292 : cluster [DBG] pgmap v6299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:28:21.024Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:28:21.026Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:21.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:21 smithi130 bash[51302]: debug 2023-08-28T01:28:21.600+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:21.605059+0000) 2023-08-28T01:28:22.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:21 smithi006 bash[27292]: audit 2023-08-28T01:28:21.753964+0000 mon.a (mon.0) 4191 : audit [DBG] from='client.? 172.21.15.6:0/3348358835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:22.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:21 smithi006 bash[33918]: audit 2023-08-28T01:28:21.753964+0000 mon.a (mon.0) 4191 : audit [DBG] from='client.? 172.21.15.6:0/3348358835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:22.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:21 smithi130 bash[27579]: audit 2023-08-28T01:28:21.753964+0000 mon.a (mon.0) 4191 : audit [DBG] from='client.? 172.21.15.6:0/3348358835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:22.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:22 smithi130 bash[51302]: debug 2023-08-28T01:28:22.600+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:22.605347+0000) 2023-08-28T01:28:22.963 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:28:22.596Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:22.963 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:28:22.598Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:23.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:22 smithi006 bash[33918]: cluster 2023-08-28T01:28:22.142899+0000 mgr.y (mgr.14901) 6293 : cluster [DBG] pgmap v6300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:23.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:22 smithi006 bash[27292]: cluster 2023-08-28T01:28:22.142899+0000 mgr.y (mgr.14901) 6293 : cluster [DBG] pgmap v6300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:23.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:22 smithi130 bash[27579]: cluster 2023-08-28T01:28:22.142899+0000 mgr.y (mgr.14901) 6293 : cluster [DBG] pgmap v6300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:23.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:23 smithi130 bash[51302]: debug 2023-08-28T01:28:23.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:23.605632+0000) 2023-08-28T01:28:24.476 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:24 smithi006 bash[27292]: cluster 2023-08-28T01:28:24.143571+0000 mgr.y (mgr.14901) 6294 : cluster [DBG] pgmap v6301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:24.477 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:24 smithi006 bash[27292]: audit 2023-08-28T01:28:24.164318+0000 mon.c (mon.2) 2471 : audit [DBG] from='client.? 172.21.15.6:0/936448276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:24.477 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:24 smithi006 bash[33918]: cluster 2023-08-28T01:28:24.143571+0000 mgr.y (mgr.14901) 6294 : cluster [DBG] pgmap v6301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:24.477 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:24 smithi006 bash[33918]: audit 2023-08-28T01:28:24.164318+0000 mon.c (mon.2) 2471 : audit [DBG] from='client.? 172.21.15.6:0/936448276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:24.604 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:24 smithi130 bash[27579]: cluster 2023-08-28T01:28:24.143571+0000 mgr.y (mgr.14901) 6294 : cluster [DBG] pgmap v6301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:24.604 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:24 smithi130 bash[27579]: audit 2023-08-28T01:28:24.164318+0000 mon.c (mon.2) 2471 : audit [DBG] from='client.? 172.21.15.6:0/936448276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:24.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:24 smithi130 bash[51302]: debug 2023-08-28T01:28:24.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:24.605937+0000) 2023-08-28T01:28:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:28:24.475Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:28:24.478Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:25.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:25 smithi130 bash[51302]: debug 2023-08-28T01:28:25.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:25.606216+0000) 2023-08-28T01:28:26.606 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:26 smithi130 bash[27579]: cluster 2023-08-28T01:28:26.144313+0000 mgr.y (mgr.14901) 6295 : cluster [DBG] pgmap v6302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:26 smithi006 bash[27292]: cluster 2023-08-28T01:28:26.144313+0000 mgr.y (mgr.14901) 6295 : cluster [DBG] pgmap v6302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:26 smithi006 bash[33918]: cluster 2023-08-28T01:28:26.144313+0000 mgr.y (mgr.14901) 6295 : cluster [DBG] pgmap v6302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:26.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:26 smithi130 bash[51302]: debug 2023-08-28T01:28:26.600+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:26.606505+0000) 2023-08-28T01:28:27.605 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:27 smithi130 bash[27579]: audit 2023-08-28T01:28:26.302605+0000 mon.b (mon.1) 720 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:28:27.606 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:27 smithi130 bash[27579]: audit 2023-08-28T01:28:26.564365+0000 mon.a (mon.0) 4192 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:28:27.606 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:27 smithi130 bash[27579]: audit 2023-08-28T01:28:26.574572+0000 mon.a (mon.0) 4193 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:28:27.606 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:27 smithi130 bash[27579]: audit 2023-08-28T01:28:26.583915+0000 mon.a (mon.0) 4194 : audit [DBG] from='client.? 172.21.15.6:0/1391053528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:27.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:27 smithi006 bash[33918]: audit 2023-08-28T01:28:26.302605+0000 mon.b (mon.1) 720 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:28:27.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:27 smithi006 bash[33918]: audit 2023-08-28T01:28:26.564365+0000 mon.a (mon.0) 4192 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:28:27.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:27 smithi006 bash[33918]: audit 2023-08-28T01:28:26.574572+0000 mon.a (mon.0) 4193 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:28:27.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:27 smithi006 bash[33918]: audit 2023-08-28T01:28:26.583915+0000 mon.a (mon.0) 4194 : audit [DBG] from='client.? 172.21.15.6:0/1391053528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:27.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:27 smithi006 bash[27292]: audit 2023-08-28T01:28:26.302605+0000 mon.b (mon.1) 720 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:28:27.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:27 smithi006 bash[27292]: audit 2023-08-28T01:28:26.564365+0000 mon.a (mon.0) 4192 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:28:27.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:27 smithi006 bash[27292]: audit 2023-08-28T01:28:26.574572+0000 mon.a (mon.0) 4193 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:28:27.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:27 smithi006 bash[27292]: audit 2023-08-28T01:28:26.583915+0000 mon.a (mon.0) 4194 : audit [DBG] from='client.? 172.21.15.6:0/1391053528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:27.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:27 smithi130 bash[51302]: debug 2023-08-28T01:28:27.600+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:27.606812+0000) 2023-08-28T01:28:28.606 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:28 smithi130 bash[27579]: cluster 2023-08-28T01:28:28.145180+0000 mgr.y (mgr.14901) 6296 : cluster [DBG] pgmap v6303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:28 smithi006 bash[27292]: cluster 2023-08-28T01:28:28.145180+0000 mgr.y (mgr.14901) 6296 : cluster [DBG] pgmap v6303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:28 smithi006 bash[33918]: cluster 2023-08-28T01:28:28.145180+0000 mgr.y (mgr.14901) 6296 : cluster [DBG] pgmap v6303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:28.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:28 smithi130 bash[51302]: debug 2023-08-28T01:28:28.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:28.607179+0000) 2023-08-28T01:28:29.606 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:29 smithi130 bash[27579]: audit 2023-08-28T01:28:29.001170+0000 mon.a (mon.0) 4195 : audit [DBG] from='client.? 172.21.15.6:0/1781609109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:29.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:29 smithi006 bash[27292]: audit 2023-08-28T01:28:29.001170+0000 mon.a (mon.0) 4195 : audit [DBG] from='client.? 172.21.15.6:0/1781609109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:29.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:29 smithi006 bash[33918]: audit 2023-08-28T01:28:29.001170+0000 mon.a (mon.0) 4195 : audit [DBG] from='client.? 172.21.15.6:0/1781609109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:29.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:29 smithi130 bash[51302]: debug 2023-08-28T01:28:29.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:29.607554+0000) 2023-08-28T01:28:30.606 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:30 smithi130 bash[27579]: cluster 2023-08-28T01:28:30.146503+0000 mgr.y (mgr.14901) 6297 : cluster [DBG] pgmap v6304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:30 smithi006 bash[33918]: cluster 2023-08-28T01:28:30.146503+0000 mgr.y (mgr.14901) 6297 : cluster [DBG] pgmap v6304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:30 smithi006 bash[27292]: cluster 2023-08-28T01:28:30.146503+0000 mgr.y (mgr.14901) 6297 : cluster [DBG] pgmap v6304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:30.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:30 smithi130 bash[51302]: debug 2023-08-28T01:28:30.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:30.607903+0000) 2023-08-28T01:28:31.025 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:28:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:28:30] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:28:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:28:31.024Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:28:31.026Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:31.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:31 smithi130 bash[27579]: audit 2023-08-28T01:28:31.414121+0000 mon.c (mon.2) 2472 : audit [DBG] from='client.? 172.21.15.6:0/1347194386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:31.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:31 smithi130 bash[51302]: debug 2023-08-28T01:28:31.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:31.608241+0000) 2023-08-28T01:28:31.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:31 smithi006 bash[27292]: audit 2023-08-28T01:28:31.414121+0000 mon.c (mon.2) 2472 : audit [DBG] from='client.? 172.21.15.6:0/1347194386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:31.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:31 smithi006 bash[33918]: audit 2023-08-28T01:28:31.414121+0000 mon.c (mon.2) 2472 : audit [DBG] from='client.? 172.21.15.6:0/1347194386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:32.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:32 smithi130 bash[51302]: debug 2023-08-28T01:28:32.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:32.608472+0000) 2023-08-28T01:28:32.937 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:28:32.596Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:32.938 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:28:32.598Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:33.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:32 smithi006 bash[33918]: cluster 2023-08-28T01:28:32.147082+0000 mgr.y (mgr.14901) 6298 : cluster [DBG] pgmap v6305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:33.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:32 smithi006 bash[33918]: audit 2023-08-28T01:28:32.422014+0000 mon.a (mon.0) 4196 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:28:33.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:32 smithi006 bash[33918]: audit 2023-08-28T01:28:32.861467+0000 mon.a (mon.0) 4197 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:28:33.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:32 smithi006 bash[27292]: cluster 2023-08-28T01:28:32.147082+0000 mgr.y (mgr.14901) 6298 : cluster [DBG] pgmap v6305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:33.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:32 smithi006 bash[27292]: audit 2023-08-28T01:28:32.422014+0000 mon.a (mon.0) 4196 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:28:33.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:32 smithi006 bash[27292]: audit 2023-08-28T01:28:32.861467+0000 mon.a (mon.0) 4197 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:28:33.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:32 smithi130 bash[27579]: cluster 2023-08-28T01:28:32.147082+0000 mgr.y (mgr.14901) 6298 : cluster [DBG] pgmap v6305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:33.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:32 smithi130 bash[27579]: audit 2023-08-28T01:28:32.422014+0000 mon.a (mon.0) 4196 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:28:33.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:32 smithi130 bash[27579]: audit 2023-08-28T01:28:32.861467+0000 mon.a (mon.0) 4197 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:28:33.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:33 smithi130 bash[51302]: debug 2023-08-28T01:28:33.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:33.608764+0000) 2023-08-28T01:28:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:33 smithi006 bash[33918]: audit 2023-08-28T01:28:33.129868+0000 mon.b (mon.1) 721 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:28:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:33 smithi006 bash[33918]: audit 2023-08-28T01:28:33.131866+0000 mon.b (mon.1) 722 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:28:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:33 smithi006 bash[33918]: audit 2023-08-28T01:28:33.142253+0000 mon.a (mon.0) 4198 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:28:34.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:33 smithi006 bash[33918]: audit 2023-08-28T01:28:33.829277+0000 mon.c (mon.2) 2473 : audit [DBG] from='client.? 172.21.15.6:0/2700117578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:34.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:33 smithi006 bash[27292]: audit 2023-08-28T01:28:33.129868+0000 mon.b (mon.1) 721 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:28:34.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:33 smithi006 bash[27292]: audit 2023-08-28T01:28:33.131866+0000 mon.b (mon.1) 722 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:28:34.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:33 smithi006 bash[27292]: audit 2023-08-28T01:28:33.142253+0000 mon.a (mon.0) 4198 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:28:34.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:33 smithi006 bash[27292]: audit 2023-08-28T01:28:33.829277+0000 mon.c (mon.2) 2473 : audit [DBG] from='client.? 172.21.15.6:0/2700117578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:33 smithi130 bash[27579]: audit 2023-08-28T01:28:33.129868+0000 mon.b (mon.1) 721 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:28:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:33 smithi130 bash[27579]: audit 2023-08-28T01:28:33.131866+0000 mon.b (mon.1) 722 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:28:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:33 smithi130 bash[27579]: audit 2023-08-28T01:28:33.142253+0000 mon.a (mon.0) 4198 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:28:34.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:33 smithi130 bash[27579]: audit 2023-08-28T01:28:33.829277+0000 mon.c (mon.2) 2473 : audit [DBG] from='client.? 172.21.15.6:0/2700117578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:34.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:34 smithi130 bash[51302]: debug 2023-08-28T01:28:34.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:34.609100+0000) 2023-08-28T01:28:34.951 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:28:34.476Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:34.951 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:28:34.478Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:35.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:34 smithi006 bash[27292]: cluster 2023-08-28T01:28:34.148119+0000 mgr.y (mgr.14901) 6299 : cluster [DBG] pgmap v6306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:35.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:34 smithi006 bash[33918]: cluster 2023-08-28T01:28:34.148119+0000 mgr.y (mgr.14901) 6299 : cluster [DBG] pgmap v6306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:35.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:34 smithi130 bash[27579]: cluster 2023-08-28T01:28:34.148119+0000 mgr.y (mgr.14901) 6299 : cluster [DBG] pgmap v6306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:35.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:35 smithi130 bash[51302]: debug 2023-08-28T01:28:35.608+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:35.609445+0000) 2023-08-28T01:28:36.608 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:36 smithi130 bash[27579]: cluster 2023-08-28T01:28:36.148739+0000 mgr.y (mgr.14901) 6300 : cluster [DBG] pgmap v6307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:36.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:36 smithi006 bash[27292]: cluster 2023-08-28T01:28:36.148739+0000 mgr.y (mgr.14901) 6300 : cluster [DBG] pgmap v6307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:36.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:36 smithi006 bash[33918]: cluster 2023-08-28T01:28:36.148739+0000 mgr.y (mgr.14901) 6300 : cluster [DBG] pgmap v6307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:36.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:36 smithi130 bash[51302]: debug 2023-08-28T01:28:36.608+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:36.609786+0000) 2023-08-28T01:28:37.608 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:37 smithi130 bash[27579]: audit 2023-08-28T01:28:36.246657+0000 mon.a (mon.0) 4199 : audit [DBG] from='client.? 172.21.15.6:0/2611474063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:37.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:37 smithi006 bash[27292]: audit 2023-08-28T01:28:36.246657+0000 mon.a (mon.0) 4199 : audit [DBG] from='client.? 172.21.15.6:0/2611474063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:37.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:37 smithi006 bash[33918]: audit 2023-08-28T01:28:36.246657+0000 mon.a (mon.0) 4199 : audit [DBG] from='client.? 172.21.15.6:0/2611474063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:37.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:37 smithi130 bash[51302]: debug 2023-08-28T01:28:37.608+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:37.610110+0000) 2023-08-28T01:28:38.609 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:38 smithi130 bash[27579]: cluster 2023-08-28T01:28:38.149380+0000 mgr.y (mgr.14901) 6301 : cluster [DBG] pgmap v6308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:38.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:38 smithi006 bash[33918]: cluster 2023-08-28T01:28:38.149380+0000 mgr.y (mgr.14901) 6301 : cluster [DBG] pgmap v6308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:38 smithi006 bash[27292]: cluster 2023-08-28T01:28:38.149380+0000 mgr.y (mgr.14901) 6301 : cluster [DBG] pgmap v6308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:38.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:38 smithi130 bash[51302]: debug 2023-08-28T01:28:38.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:38.610455+0000) 2023-08-28T01:28:39.609 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:39 smithi130 bash[27579]: audit 2023-08-28T01:28:38.667899+0000 mon.c (mon.2) 2474 : audit [DBG] from='client.? 172.21.15.6:0/1706132211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:39.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:39 smithi006 bash[27292]: audit 2023-08-28T01:28:38.667899+0000 mon.c (mon.2) 2474 : audit [DBG] from='client.? 172.21.15.6:0/1706132211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:39.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:39 smithi006 bash[33918]: audit 2023-08-28T01:28:38.667899+0000 mon.c (mon.2) 2474 : audit [DBG] from='client.? 172.21.15.6:0/1706132211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:39.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:39 smithi130 bash[51302]: debug 2023-08-28T01:28:39.604+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:39.610788+0000) 2023-08-28T01:28:40.609 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:40 smithi130 bash[27579]: cluster 2023-08-28T01:28:40.150516+0000 mgr.y (mgr.14901) 6302 : cluster [DBG] pgmap v6309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:40 smithi006 bash[33918]: cluster 2023-08-28T01:28:40.150516+0000 mgr.y (mgr.14901) 6302 : cluster [DBG] pgmap v6309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:40 smithi006 bash[27292]: cluster 2023-08-28T01:28:40.150516+0000 mgr.y (mgr.14901) 6302 : cluster [DBG] pgmap v6309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:40.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:40 smithi130 bash[51302]: debug 2023-08-28T01:28:40.608+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:40.611145+0000) 2023-08-28T01:28:41.026 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:28:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:28:40] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:28:41.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:41 smithi006 bash[27292]: audit 2023-08-28T01:28:41.081712+0000 mon.c (mon.2) 2475 : audit [DBG] from='client.? 172.21.15.6:0/293612429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:41.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:41 smithi006 bash[33918]: audit 2023-08-28T01:28:41.081712+0000 mon.c (mon.2) 2475 : audit [DBG] from='client.? 172.21.15.6:0/293612429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:41.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:28:41.025Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:41.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:28:41.027Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:41.610 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:41 smithi130 bash[27579]: audit 2023-08-28T01:28:41.081712+0000 mon.c (mon.2) 2475 : audit [DBG] from='client.? 172.21.15.6:0/293612429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:41.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:41 smithi130 bash[51302]: debug 2023-08-28T01:28:41.608+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:41.611431+0000) 2023-08-28T01:28:42.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:42 smithi130 bash[51302]: debug 2023-08-28T01:28:42.608+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:42.611707+0000) 2023-08-28T01:28:42.945 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:28:42.597Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:42.945 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:28:42.599Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:43.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:42 smithi006 bash[27292]: cluster 2023-08-28T01:28:42.151171+0000 mgr.y (mgr.14901) 6303 : cluster [DBG] pgmap v6310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:43.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:42 smithi006 bash[33918]: cluster 2023-08-28T01:28:42.151171+0000 mgr.y (mgr.14901) 6303 : cluster [DBG] pgmap v6310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:43.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:42 smithi130 bash[27579]: cluster 2023-08-28T01:28:42.151171+0000 mgr.y (mgr.14901) 6303 : cluster [DBG] pgmap v6310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:43.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:43 smithi130 bash[51302]: debug 2023-08-28T01:28:43.608+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:43.612032+0000) 2023-08-28T01:28:44.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:43 smithi006 bash[27292]: audit 2023-08-28T01:28:43.506515+0000 mon.a (mon.0) 4200 : audit [DBG] from='client.? 172.21.15.6:0/1415492127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:44.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:43 smithi006 bash[33918]: audit 2023-08-28T01:28:43.506515+0000 mon.a (mon.0) 4200 : audit [DBG] from='client.? 172.21.15.6:0/1415492127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:44.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:43 smithi130 bash[27579]: audit 2023-08-28T01:28:43.506515+0000 mon.a (mon.0) 4200 : audit [DBG] from='client.? 172.21.15.6:0/1415492127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:44.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:44 smithi130 bash[51302]: debug 2023-08-28T01:28:44.612+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:44.612367+0000) 2023-08-28T01:28:44.958 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:28:44.477Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:44.958 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:28:44.479Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:45.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:44 smithi006 bash[33918]: cluster 2023-08-28T01:28:44.152212+0000 mgr.y (mgr.14901) 6304 : cluster [DBG] pgmap v6311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:45.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:44 smithi006 bash[27292]: cluster 2023-08-28T01:28:44.152212+0000 mgr.y (mgr.14901) 6304 : cluster [DBG] pgmap v6311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:45.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:44 smithi130 bash[27579]: cluster 2023-08-28T01:28:44.152212+0000 mgr.y (mgr.14901) 6304 : cluster [DBG] pgmap v6311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:45.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:45 smithi130 bash[51302]: debug 2023-08-28T01:28:45.608+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:45.612713+0000) 2023-08-28T01:28:46.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:45 smithi006 bash[27292]: audit 2023-08-28T01:28:45.925615+0000 mon.a (mon.0) 4201 : audit [DBG] from='client.? 172.21.15.6:0/3353876440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:46.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:45 smithi006 bash[33918]: audit 2023-08-28T01:28:45.925615+0000 mon.a (mon.0) 4201 : audit [DBG] from='client.? 172.21.15.6:0/3353876440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:46.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:45 smithi130 bash[27579]: audit 2023-08-28T01:28:45.925615+0000 mon.a (mon.0) 4201 : audit [DBG] from='client.? 172.21.15.6:0/3353876440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:46.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:46 smithi130 bash[51302]: debug 2023-08-28T01:28:46.608+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:46.613037+0000) 2023-08-28T01:28:47.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:46 smithi130 bash[27579]: cluster 2023-08-28T01:28:46.152861+0000 mgr.y (mgr.14901) 6305 : cluster [DBG] pgmap v6312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:47.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:46 smithi006 bash[27292]: cluster 2023-08-28T01:28:46.152861+0000 mgr.y (mgr.14901) 6305 : cluster [DBG] pgmap v6312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:47.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:46 smithi006 bash[33918]: cluster 2023-08-28T01:28:46.152861+0000 mgr.y (mgr.14901) 6305 : cluster [DBG] pgmap v6312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:47.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:47 smithi130 bash[51302]: debug 2023-08-28T01:28:47.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:47.613377+0000) 2023-08-28T01:28:48.612 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:48 smithi130 bash[27579]: cluster 2023-08-28T01:28:48.153508+0000 mgr.y (mgr.14901) 6306 : cluster [DBG] pgmap v6313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:48 smithi006 bash[27292]: cluster 2023-08-28T01:28:48.153508+0000 mgr.y (mgr.14901) 6306 : cluster [DBG] pgmap v6313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:48.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:48 smithi006 bash[33918]: cluster 2023-08-28T01:28:48.153508+0000 mgr.y (mgr.14901) 6306 : cluster [DBG] pgmap v6313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:48.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:48 smithi130 bash[51302]: debug 2023-08-28T01:28:48.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:48.613704+0000) 2023-08-28T01:28:49.612 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:49 smithi130 bash[27579]: audit 2023-08-28T01:28:48.330878+0000 mon.c (mon.2) 2476 : audit [DBG] from='client.? 172.21.15.6:0/3697078721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:49.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:49 smithi006 bash[27292]: audit 2023-08-28T01:28:48.330878+0000 mon.c (mon.2) 2476 : audit [DBG] from='client.? 172.21.15.6:0/3697078721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:49.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:49 smithi006 bash[33918]: audit 2023-08-28T01:28:48.330878+0000 mon.c (mon.2) 2476 : audit [DBG] from='client.? 172.21.15.6:0/3697078721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:49.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:49 smithi130 bash[51302]: debug 2023-08-28T01:28:49.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:49.614040+0000) 2023-08-28T01:28:50.613 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:50 smithi130 bash[27579]: cluster 2023-08-28T01:28:50.154648+0000 mgr.y (mgr.14901) 6307 : cluster [DBG] pgmap v6314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:50.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:50 smithi006 bash[33918]: cluster 2023-08-28T01:28:50.154648+0000 mgr.y (mgr.14901) 6307 : cluster [DBG] pgmap v6314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:50 smithi006 bash[27292]: cluster 2023-08-28T01:28:50.154648+0000 mgr.y (mgr.14901) 6307 : cluster [DBG] pgmap v6314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:50.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:50 smithi130 bash[51302]: debug 2023-08-28T01:28:50.607+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:50.614424+0000) 2023-08-28T01:28:51.027 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:28:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:28:50] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:28:51.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:51 smithi006 bash[33918]: audit 2023-08-28T01:28:50.752721+0000 mon.a (mon.0) 4202 : audit [DBG] from='client.? 172.21.15.6:0/3307258240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:51.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:51 smithi006 bash[27292]: audit 2023-08-28T01:28:50.752721+0000 mon.a (mon.0) 4202 : audit [DBG] from='client.? 172.21.15.6:0/3307258240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:51.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:28:51.026Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:51.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:28:51.028Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:51.613 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:51 smithi130 bash[27579]: audit 2023-08-28T01:28:50.752721+0000 mon.a (mon.0) 4202 : audit [DBG] from='client.? 172.21.15.6:0/3307258240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:51.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:51 smithi130 bash[51302]: debug 2023-08-28T01:28:51.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:51.614731+0000) 2023-08-28T01:28:52.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:52 smithi130 bash[51302]: debug 2023-08-28T01:28:52.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:52.615058+0000) 2023-08-28T01:28:52.940 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:28:52.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:52.940 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:28:52.600Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:53.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:52 smithi006 bash[27292]: cluster 2023-08-28T01:28:52.155303+0000 mgr.y (mgr.14901) 6308 : cluster [DBG] pgmap v6315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:53.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:52 smithi006 bash[33918]: cluster 2023-08-28T01:28:52.155303+0000 mgr.y (mgr.14901) 6308 : cluster [DBG] pgmap v6315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:53.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:52 smithi130 bash[27579]: cluster 2023-08-28T01:28:52.155303+0000 mgr.y (mgr.14901) 6308 : cluster [DBG] pgmap v6315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:53.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:53 smithi130 bash[51302]: debug 2023-08-28T01:28:53.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:53.615270+0000) 2023-08-28T01:28:54.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:53 smithi006 bash[27292]: audit 2023-08-28T01:28:53.163395+0000 mon.c (mon.2) 2477 : audit [DBG] from='client.? 172.21.15.6:0/2884039467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:54.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:53 smithi006 bash[33918]: audit 2023-08-28T01:28:53.163395+0000 mon.c (mon.2) 2477 : audit [DBG] from='client.? 172.21.15.6:0/2884039467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:53 smithi130 bash[27579]: audit 2023-08-28T01:28:53.163395+0000 mon.c (mon.2) 2477 : audit [DBG] from='client.? 172.21.15.6:0/2884039467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:54.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:54 smithi130 bash[51302]: debug 2023-08-28T01:28:54.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:54.615566+0000) 2023-08-28T01:28:54.952 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:28:54.477Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:54.952 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:28:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:28:54.479Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:28:55.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:54 smithi006 bash[27292]: cluster 2023-08-28T01:28:54.156266+0000 mgr.y (mgr.14901) 6309 : cluster [DBG] pgmap v6316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:55.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:54 smithi006 bash[33918]: cluster 2023-08-28T01:28:54.156266+0000 mgr.y (mgr.14901) 6309 : cluster [DBG] pgmap v6316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:55.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:54 smithi130 bash[27579]: cluster 2023-08-28T01:28:54.156266+0000 mgr.y (mgr.14901) 6309 : cluster [DBG] pgmap v6316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:55.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:55 smithi130 bash[51302]: debug 2023-08-28T01:28:55.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:55.615806+0000) 2023-08-28T01:28:56.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:55 smithi006 bash[33918]: audit 2023-08-28T01:28:55.586595+0000 mon.a (mon.0) 4203 : audit [DBG] from='client.? 172.21.15.6:0/2256418334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:56.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:55 smithi006 bash[27292]: audit 2023-08-28T01:28:55.586595+0000 mon.a (mon.0) 4203 : audit [DBG] from='client.? 172.21.15.6:0/2256418334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:56.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:55 smithi130 bash[27579]: audit 2023-08-28T01:28:55.586595+0000 mon.a (mon.0) 4203 : audit [DBG] from='client.? 172.21.15.6:0/2256418334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:56.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:56 smithi130 bash[51302]: debug 2023-08-28T01:28:56.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:56.616130+0000) 2023-08-28T01:28:57.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:56 smithi006 bash[27292]: cluster 2023-08-28T01:28:56.157005+0000 mgr.y (mgr.14901) 6310 : cluster [DBG] pgmap v6317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:57.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:56 smithi006 bash[33918]: cluster 2023-08-28T01:28:56.157005+0000 mgr.y (mgr.14901) 6310 : cluster [DBG] pgmap v6317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:57.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:56 smithi130 bash[27579]: cluster 2023-08-28T01:28:56.157005+0000 mgr.y (mgr.14901) 6310 : cluster [DBG] pgmap v6317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:57.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:57 smithi130 bash[51302]: debug 2023-08-28T01:28:57.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:57.616407+0000) 2023-08-28T01:28:58.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:58 smithi130 bash[27579]: audit 2023-08-28T01:28:57.994071+0000 mon.a (mon.0) 4204 : audit [DBG] from='client.? 172.21.15.6:0/2452316021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:58.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:58 smithi006 bash[33918]: audit 2023-08-28T01:28:57.994071+0000 mon.a (mon.0) 4204 : audit [DBG] from='client.? 172.21.15.6:0/2452316021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:58.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:58 smithi006 bash[27292]: audit 2023-08-28T01:28:57.994071+0000 mon.a (mon.0) 4204 : audit [DBG] from='client.? 172.21.15.6:0/2452316021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:58.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:58 smithi130 bash[51302]: debug 2023-08-28T01:28:58.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:58.616678+0000) 2023-08-28T01:28:59.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:28:59 smithi130 bash[27579]: cluster 2023-08-28T01:28:58.157709+0000 mgr.y (mgr.14901) 6311 : cluster [DBG] pgmap v6318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:59.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:28:59 smithi006 bash[27292]: cluster 2023-08-28T01:28:58.157709+0000 mgr.y (mgr.14901) 6311 : cluster [DBG] pgmap v6318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:28:59.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:28:59 smithi006 bash[33918]: cluster 2023-08-28T01:28:58.157709+0000 mgr.y (mgr.14901) 6311 : cluster [DBG] pgmap v6318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:28:59 smithi130 bash[51302]: debug 2023-08-28T01:28:59.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:59.616949+0000) 2023-08-28T01:29:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:00 smithi130 bash[27579]: cluster 2023-08-28T01:29:00.158484+0000 mgr.y (mgr.14901) 6312 : cluster [DBG] pgmap v6319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:00 smithi006 bash[33918]: cluster 2023-08-28T01:29:00.158484+0000 mgr.y (mgr.14901) 6312 : cluster [DBG] pgmap v6319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:00 smithi006 bash[27292]: cluster 2023-08-28T01:29:00.158484+0000 mgr.y (mgr.14901) 6312 : cluster [DBG] pgmap v6319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:01.027 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:29:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:29:00] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:29:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:00 smithi130 bash[51302]: debug 2023-08-28T01:29:00.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:00.617295+0000) 2023-08-28T01:29:01.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:01 smithi006 bash[33918]: audit 2023-08-28T01:29:00.416059+0000 mon.c (mon.2) 2478 : audit [DBG] from='client.? 172.21.15.6:0/2058212842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:29:01.027Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:29:01.029Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:01.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:01 smithi006 bash[27292]: audit 2023-08-28T01:29:00.416059+0000 mon.c (mon.2) 2478 : audit [DBG] from='client.? 172.21.15.6:0/2058212842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:01.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:01 smithi130 bash[27579]: audit 2023-08-28T01:29:00.416059+0000 mon.c (mon.2) 2478 : audit [DBG] from='client.? 172.21.15.6:0/2058212842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:02.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:01 smithi130 bash[51302]: debug 2023-08-28T01:29:01.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:01.617673+0000) 2023-08-28T01:29:02.947 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:29:02.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:02.947 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:29:02.600Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:02.947 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:02 smithi130 bash[51302]: debug 2023-08-28T01:29:02.611+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:02.617966+0000) 2023-08-28T01:29:03.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:02 smithi006 bash[33918]: cluster 2023-08-28T01:29:02.159113+0000 mgr.y (mgr.14901) 6313 : cluster [DBG] pgmap v6320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:03.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:02 smithi006 bash[33918]: audit 2023-08-28T01:29:02.838216+0000 mon.a (mon.0) 4205 : audit [DBG] from='client.? 172.21.15.6:0/2047284111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:03.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:02 smithi006 bash[27292]: cluster 2023-08-28T01:29:02.159113+0000 mgr.y (mgr.14901) 6313 : cluster [DBG] pgmap v6320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:03.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:02 smithi006 bash[27292]: audit 2023-08-28T01:29:02.838216+0000 mon.a (mon.0) 4205 : audit [DBG] from='client.? 172.21.15.6:0/2047284111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:03.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:02 smithi130 bash[27579]: cluster 2023-08-28T01:29:02.159113+0000 mgr.y (mgr.14901) 6313 : cluster [DBG] pgmap v6320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:03.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:02 smithi130 bash[27579]: audit 2023-08-28T01:29:02.838216+0000 mon.a (mon.0) 4205 : audit [DBG] from='client.? 172.21.15.6:0/2047284111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:04.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:03 smithi130 bash[51302]: debug 2023-08-28T01:29:03.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:03.618283+0000) 2023-08-28T01:29:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:04 smithi130 bash[27579]: cluster 2023-08-28T01:29:04.160266+0000 mgr.y (mgr.14901) 6314 : cluster [DBG] pgmap v6321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:04.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:04 smithi006 bash[27292]: cluster 2023-08-28T01:29:04.160266+0000 mgr.y (mgr.14901) 6314 : cluster [DBG] pgmap v6321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:04.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:04 smithi006 bash[33918]: cluster 2023-08-28T01:29:04.160266+0000 mgr.y (mgr.14901) 6314 : cluster [DBG] pgmap v6321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:29:04.478Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:29:04.480Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:05.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:04 smithi130 bash[51302]: debug 2023-08-28T01:29:04.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:04.618706+0000) 2023-08-28T01:29:05.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:05 smithi130 bash[27579]: audit 2023-08-28T01:29:05.253974+0000 mon.c (mon.2) 2479 : audit [DBG] from='client.? 172.21.15.6:0/2768193348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:05.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:05 smithi006 bash[27292]: audit 2023-08-28T01:29:05.253974+0000 mon.c (mon.2) 2479 : audit [DBG] from='client.? 172.21.15.6:0/2768193348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:05.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:05 smithi006 bash[33918]: audit 2023-08-28T01:29:05.253974+0000 mon.c (mon.2) 2479 : audit [DBG] from='client.? 172.21.15.6:0/2768193348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:06.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:05 smithi130 bash[51302]: debug 2023-08-28T01:29:05.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:05.619010+0000) 2023-08-28T01:29:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:06 smithi130 bash[27579]: cluster 2023-08-28T01:29:06.161032+0000 mgr.y (mgr.14901) 6315 : cluster [DBG] pgmap v6322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:06 smithi006 bash[27292]: cluster 2023-08-28T01:29:06.161032+0000 mgr.y (mgr.14901) 6315 : cluster [DBG] pgmap v6322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:06 smithi006 bash[33918]: cluster 2023-08-28T01:29:06.161032+0000 mgr.y (mgr.14901) 6315 : cluster [DBG] pgmap v6322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:07.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:06 smithi130 bash[51302]: debug 2023-08-28T01:29:06.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:06.619251+0000) 2023-08-28T01:29:07.939 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:07 smithi130 bash[51302]: debug 2023-08-28T01:29:07.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:07.619539+0000) 2023-08-28T01:29:08.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:07 smithi006 bash[27292]: audit 2023-08-28T01:29:07.666953+0000 mon.a (mon.0) 4206 : audit [DBG] from='client.? 172.21.15.6:0/2175452176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:08.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:07 smithi006 bash[33918]: audit 2023-08-28T01:29:07.666953+0000 mon.a (mon.0) 4206 : audit [DBG] from='client.? 172.21.15.6:0/2175452176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:08.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:07 smithi130 bash[27579]: audit 2023-08-28T01:29:07.666953+0000 mon.a (mon.0) 4206 : audit [DBG] from='client.? 172.21.15.6:0/2175452176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:08.952 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:08 smithi130 bash[51302]: debug 2023-08-28T01:29:08.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:08.619814+0000) 2023-08-28T01:29:09.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:08 smithi006 bash[33918]: cluster 2023-08-28T01:29:08.161683+0000 mgr.y (mgr.14901) 6316 : cluster [DBG] pgmap v6323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:09.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:08 smithi006 bash[27292]: cluster 2023-08-28T01:29:08.161683+0000 mgr.y (mgr.14901) 6316 : cluster [DBG] pgmap v6323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:09.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:08 smithi130 bash[27579]: cluster 2023-08-28T01:29:08.161683+0000 mgr.y (mgr.14901) 6316 : cluster [DBG] pgmap v6323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:10.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:09 smithi130 bash[51302]: debug 2023-08-28T01:29:09.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:09.620268+0000) 2023-08-28T01:29:10.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:10 smithi006 bash[33918]: audit 2023-08-28T01:29:10.072602+0000 mon.a (mon.0) 4207 : audit [DBG] from='client.? 172.21.15.6:0/2711658325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:10.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:10 smithi006 bash[27292]: audit 2023-08-28T01:29:10.072602+0000 mon.a (mon.0) 4207 : audit [DBG] from='client.? 172.21.15.6:0/2711658325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:10 smithi130 bash[27579]: audit 2023-08-28T01:29:10.072602+0000 mon.a (mon.0) 4207 : audit [DBG] from='client.? 172.21.15.6:0/2711658325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:11.028 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:29:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:29:10] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:29:11.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:10 smithi130 bash[51302]: debug 2023-08-28T01:29:10.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:10.620623+0000) 2023-08-28T01:29:11.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:11 smithi006 bash[27292]: cluster 2023-08-28T01:29:10.162768+0000 mgr.y (mgr.14901) 6317 : cluster [DBG] pgmap v6324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:11.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:11 smithi006 bash[33918]: cluster 2023-08-28T01:29:10.162768+0000 mgr.y (mgr.14901) 6317 : cluster [DBG] pgmap v6324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:11.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:29:11.027Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:11.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:29:11.030Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:11.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:11 smithi130 bash[27579]: cluster 2023-08-28T01:29:10.162768+0000 mgr.y (mgr.14901) 6317 : cluster [DBG] pgmap v6324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:12.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:11 smithi130 bash[51302]: debug 2023-08-28T01:29:11.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:11.620926+0000) 2023-08-28T01:29:12.946 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:29:12.599Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:12.947 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:29:12.601Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:12.947 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:12 smithi130 bash[51302]: debug 2023-08-28T01:29:12.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:12.621242+0000) 2023-08-28T01:29:13.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:12 smithi006 bash[27292]: cluster 2023-08-28T01:29:12.163422+0000 mgr.y (mgr.14901) 6318 : cluster [DBG] pgmap v6325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:13.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:12 smithi006 bash[27292]: audit 2023-08-28T01:29:12.487848+0000 mon.c (mon.2) 2480 : audit [DBG] from='client.? 172.21.15.6:0/3472737494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:12 smithi006 bash[33918]: cluster 2023-08-28T01:29:12.163422+0000 mgr.y (mgr.14901) 6318 : cluster [DBG] pgmap v6325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:13.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:12 smithi006 bash[33918]: audit 2023-08-28T01:29:12.487848+0000 mon.c (mon.2) 2480 : audit [DBG] from='client.? 172.21.15.6:0/3472737494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:13.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:12 smithi130 bash[27579]: cluster 2023-08-28T01:29:12.163422+0000 mgr.y (mgr.14901) 6318 : cluster [DBG] pgmap v6325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:13.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:12 smithi130 bash[27579]: audit 2023-08-28T01:29:12.487848+0000 mon.c (mon.2) 2480 : audit [DBG] from='client.? 172.21.15.6:0/3472737494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:14.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:13 smithi130 bash[51302]: debug 2023-08-28T01:29:13.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:13.621572+0000) 2023-08-28T01:29:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:14 smithi130 bash[27579]: cluster 2023-08-28T01:29:14.164611+0000 mgr.y (mgr.14901) 6319 : cluster [DBG] pgmap v6326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:14.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:14 smithi006 bash[27292]: cluster 2023-08-28T01:29:14.164611+0000 mgr.y (mgr.14901) 6319 : cluster [DBG] pgmap v6326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:14.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:14 smithi006 bash[33918]: cluster 2023-08-28T01:29:14.164611+0000 mgr.y (mgr.14901) 6319 : cluster [DBG] pgmap v6326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:29:14.478Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:14.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:29:14.480Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:14 smithi130 bash[51302]: debug 2023-08-28T01:29:14.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:14.621887+0000) 2023-08-28T01:29:15.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:15 smithi130 bash[27579]: audit 2023-08-28T01:29:14.899513+0000 mon.c (mon.2) 2481 : audit [DBG] from='client.? 172.21.15.6:0/1888638206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:15.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:15 smithi006 bash[33918]: audit 2023-08-28T01:29:14.899513+0000 mon.c (mon.2) 2481 : audit [DBG] from='client.? 172.21.15.6:0/1888638206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:15.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:15 smithi006 bash[27292]: audit 2023-08-28T01:29:14.899513+0000 mon.c (mon.2) 2481 : audit [DBG] from='client.? 172.21.15.6:0/1888638206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:16.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:15 smithi130 bash[51302]: debug 2023-08-28T01:29:15.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:15.622186+0000) 2023-08-28T01:29:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:16 smithi130 bash[27579]: cluster 2023-08-28T01:29:16.165419+0000 mgr.y (mgr.14901) 6320 : cluster [DBG] pgmap v6327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:16 smithi006 bash[33918]: cluster 2023-08-28T01:29:16.165419+0000 mgr.y (mgr.14901) 6320 : cluster [DBG] pgmap v6327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:16 smithi006 bash[27292]: cluster 2023-08-28T01:29:16.165419+0000 mgr.y (mgr.14901) 6320 : cluster [DBG] pgmap v6327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:17.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:16 smithi130 bash[51302]: debug 2023-08-28T01:29:16.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:16.622492+0000) 2023-08-28T01:29:17.941 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:17 smithi130 bash[51302]: debug 2023-08-28T01:29:17.615+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:17.622701+0000) 2023-08-28T01:29:18.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:17 smithi006 bash[33918]: audit 2023-08-28T01:29:17.315059+0000 mon.a (mon.0) 4208 : audit [DBG] from='client.? 172.21.15.6:0/3389922319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:18.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:17 smithi006 bash[27292]: audit 2023-08-28T01:29:17.315059+0000 mon.a (mon.0) 4208 : audit [DBG] from='client.? 172.21.15.6:0/3389922319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:18.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:17 smithi130 bash[27579]: audit 2023-08-28T01:29:17.315059+0000 mon.a (mon.0) 4208 : audit [DBG] from='client.? 172.21.15.6:0/3389922319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:18.950 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:18 smithi130 bash[51302]: debug 2023-08-28T01:29:18.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:18.622977+0000) 2023-08-28T01:29:19.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:18 smithi006 bash[33918]: cluster 2023-08-28T01:29:18.166157+0000 mgr.y (mgr.14901) 6321 : cluster [DBG] pgmap v6328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:19.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:18 smithi006 bash[27292]: cluster 2023-08-28T01:29:18.166157+0000 mgr.y (mgr.14901) 6321 : cluster [DBG] pgmap v6328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:19.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:18 smithi130 bash[27579]: cluster 2023-08-28T01:29:18.166157+0000 mgr.y (mgr.14901) 6321 : cluster [DBG] pgmap v6328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:19.953 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:19 smithi130 bash[51302]: debug 2023-08-28T01:29:19.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:19.623257+0000) 2023-08-28T01:29:20.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:19 smithi006 bash[27292]: audit 2023-08-28T01:29:19.735022+0000 mon.a (mon.0) 4209 : audit [DBG] from='client.? 172.21.15.6:0/1363227022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:20.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:19 smithi006 bash[33918]: audit 2023-08-28T01:29:19.735022+0000 mon.a (mon.0) 4209 : audit [DBG] from='client.? 172.21.15.6:0/1363227022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:20.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:19 smithi130 bash[27579]: audit 2023-08-28T01:29:19.735022+0000 mon.a (mon.0) 4209 : audit [DBG] from='client.? 172.21.15.6:0/1363227022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:20.962 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:20 smithi130 bash[51302]: debug 2023-08-28T01:29:20.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:20.623538+0000) 2023-08-28T01:29:21.029 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:20 smithi006 bash[27292]: cluster 2023-08-28T01:29:20.167224+0000 mgr.y (mgr.14901) 6322 : cluster [DBG] pgmap v6329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:21.030 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:29:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:29:20] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:29:21.030 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:20 smithi006 bash[33918]: cluster 2023-08-28T01:29:20.167224+0000 mgr.y (mgr.14901) 6322 : cluster [DBG] pgmap v6329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:21.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:20 smithi130 bash[27579]: cluster 2023-08-28T01:29:20.167224+0000 mgr.y (mgr.14901) 6322 : cluster [DBG] pgmap v6329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:29:21.029Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:29:21.031Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:22.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:21 smithi130 bash[51302]: debug 2023-08-28T01:29:21.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:21.623830+0000) 2023-08-28T01:29:22.942 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:29:22.599Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:22.942 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:29:22.601Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:22.943 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:22 smithi130 bash[51302]: debug 2023-08-28T01:29:22.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:22.624052+0000) 2023-08-28T01:29:23.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:22 smithi006 bash[33918]: audit 2023-08-28T01:29:22.147930+0000 mon.a (mon.0) 4210 : audit [DBG] from='client.? 172.21.15.6:0/327996162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:23.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:22 smithi006 bash[33918]: cluster 2023-08-28T01:29:22.167835+0000 mgr.y (mgr.14901) 6323 : cluster [DBG] pgmap v6330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:23.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:22 smithi006 bash[27292]: audit 2023-08-28T01:29:22.147930+0000 mon.a (mon.0) 4210 : audit [DBG] from='client.? 172.21.15.6:0/327996162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:23.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:22 smithi006 bash[27292]: cluster 2023-08-28T01:29:22.167835+0000 mgr.y (mgr.14901) 6323 : cluster [DBG] pgmap v6330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:23.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:22 smithi130 bash[27579]: audit 2023-08-28T01:29:22.147930+0000 mon.a (mon.0) 4210 : audit [DBG] from='client.? 172.21.15.6:0/327996162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:23.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:22 smithi130 bash[27579]: cluster 2023-08-28T01:29:22.167835+0000 mgr.y (mgr.14901) 6323 : cluster [DBG] pgmap v6330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:24.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:23 smithi130 bash[51302]: debug 2023-08-28T01:29:23.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:23.624358+0000) 2023-08-28T01:29:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:24 smithi130 bash[27579]: cluster 2023-08-28T01:29:24.168964+0000 mgr.y (mgr.14901) 6324 : cluster [DBG] pgmap v6331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:24.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:24 smithi006 bash[27292]: cluster 2023-08-28T01:29:24.168964+0000 mgr.y (mgr.14901) 6324 : cluster [DBG] pgmap v6331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:24.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:24 smithi006 bash[33918]: cluster 2023-08-28T01:29:24.168964+0000 mgr.y (mgr.14901) 6324 : cluster [DBG] pgmap v6331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:24.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:29:24.479Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:24.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:29:24.481Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:25.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:24 smithi130 bash[51302]: debug 2023-08-28T01:29:24.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:24.624621+0000) 2023-08-28T01:29:25.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:25 smithi130 bash[27579]: audit 2023-08-28T01:29:24.577761+0000 mon.c (mon.2) 2482 : audit [DBG] from='client.? 172.21.15.6:0/2170711750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:25.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:25 smithi006 bash[33918]: audit 2023-08-28T01:29:24.577761+0000 mon.c (mon.2) 2482 : audit [DBG] from='client.? 172.21.15.6:0/2170711750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:25.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:25 smithi006 bash[27292]: audit 2023-08-28T01:29:24.577761+0000 mon.c (mon.2) 2482 : audit [DBG] from='client.? 172.21.15.6:0/2170711750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:26.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:25 smithi130 bash[51302]: debug 2023-08-28T01:29:25.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:25.624899+0000) 2023-08-28T01:29:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:26 smithi130 bash[27579]: cluster 2023-08-28T01:29:26.169618+0000 mgr.y (mgr.14901) 6325 : cluster [DBG] pgmap v6332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:26 smithi006 bash[27292]: cluster 2023-08-28T01:29:26.169618+0000 mgr.y (mgr.14901) 6325 : cluster [DBG] pgmap v6332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:26 smithi006 bash[33918]: cluster 2023-08-28T01:29:26.169618+0000 mgr.y (mgr.14901) 6325 : cluster [DBG] pgmap v6332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:27.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:26 smithi130 bash[51302]: debug 2023-08-28T01:29:26.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:26.625195+0000) 2023-08-28T01:29:27.944 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:27 smithi130 bash[51302]: debug 2023-08-28T01:29:27.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:27.625451+0000) 2023-08-28T01:29:28.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:27 smithi006 bash[27292]: audit 2023-08-28T01:29:26.996181+0000 mon.a (mon.0) 4211 : audit [DBG] from='client.? 172.21.15.6:0/1322517151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:28.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:27 smithi006 bash[33918]: audit 2023-08-28T01:29:26.996181+0000 mon.a (mon.0) 4211 : audit [DBG] from='client.? 172.21.15.6:0/1322517151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:28.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:27 smithi130 bash[27579]: audit 2023-08-28T01:29:26.996181+0000 mon.a (mon.0) 4211 : audit [DBG] from='client.? 172.21.15.6:0/1322517151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:28.953 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:28 smithi130 bash[51302]: debug 2023-08-28T01:29:28.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:28.625765+0000) 2023-08-28T01:29:29.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:28 smithi006 bash[27292]: cluster 2023-08-28T01:29:28.170396+0000 mgr.y (mgr.14901) 6326 : cluster [DBG] pgmap v6333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:29.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:28 smithi006 bash[33918]: cluster 2023-08-28T01:29:28.170396+0000 mgr.y (mgr.14901) 6326 : cluster [DBG] pgmap v6333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:29.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:28 smithi130 bash[27579]: cluster 2023-08-28T01:29:28.170396+0000 mgr.y (mgr.14901) 6326 : cluster [DBG] pgmap v6333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:29.958 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:29 smithi130 bash[51302]: debug 2023-08-28T01:29:29.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:29.626038+0000) 2023-08-28T01:29:30.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:29 smithi006 bash[27292]: audit 2023-08-28T01:29:29.413672+0000 mon.a (mon.0) 4212 : audit [DBG] from='client.? 172.21.15.6:0/721341941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:30.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:29 smithi006 bash[33918]: audit 2023-08-28T01:29:29.413672+0000 mon.a (mon.0) 4212 : audit [DBG] from='client.? 172.21.15.6:0/721341941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:30.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:29 smithi130 bash[27579]: audit 2023-08-28T01:29:29.413672+0000 mon.a (mon.0) 4212 : audit [DBG] from='client.? 172.21.15.6:0/721341941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:30.967 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:30 smithi130 bash[51302]: debug 2023-08-28T01:29:30.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:30.626316+0000) 2023-08-28T01:29:31.031 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:29:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:29:30] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T01:29:31.031 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:30 smithi006 bash[33918]: cluster 2023-08-28T01:29:30.171618+0000 mgr.y (mgr.14901) 6327 : cluster [DBG] pgmap v6334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:31.032 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:30 smithi006 bash[27292]: cluster 2023-08-28T01:29:30.171618+0000 mgr.y (mgr.14901) 6327 : cluster [DBG] pgmap v6334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:31.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:30 smithi130 bash[27579]: cluster 2023-08-28T01:29:30.171618+0000 mgr.y (mgr.14901) 6327 : cluster [DBG] pgmap v6334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:29:31.030Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:29:31.032Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:31.968 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:31 smithi130 bash[51302]: debug 2023-08-28T01:29:31.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:31.626551+0000) 2023-08-28T01:29:32.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:31 smithi130 bash[27579]: audit 2023-08-28T01:29:31.825828+0000 mon.c (mon.2) 2483 : audit [DBG] from='client.? 172.21.15.6:0/96947656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:32.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:31 smithi006 bash[27292]: audit 2023-08-28T01:29:31.825828+0000 mon.c (mon.2) 2483 : audit [DBG] from='client.? 172.21.15.6:0/96947656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:32.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:31 smithi006 bash[33918]: audit 2023-08-28T01:29:31.825828+0000 mon.c (mon.2) 2483 : audit [DBG] from='client.? 172.21.15.6:0/96947656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:29:32.600Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:29:32.602Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:32.979 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:32 smithi130 bash[51302]: debug 2023-08-28T01:29:32.619+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:32.626815+0000) 2023-08-28T01:29:33.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:32 smithi130 bash[27579]: cluster 2023-08-28T01:29:32.172301+0000 mgr.y (mgr.14901) 6328 : cluster [DBG] pgmap v6335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:33.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:32 smithi006 bash[33918]: cluster 2023-08-28T01:29:32.172301+0000 mgr.y (mgr.14901) 6328 : cluster [DBG] pgmap v6335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:33.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:32 smithi006 bash[27292]: cluster 2023-08-28T01:29:32.172301+0000 mgr.y (mgr.14901) 6328 : cluster [DBG] pgmap v6335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:33.987 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:33 smithi130 bash[51302]: debug 2023-08-28T01:29:33.623+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:33.627140+0000) 2023-08-28T01:29:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:33 smithi130 bash[27579]: audit 2023-08-28T01:29:33.153745+0000 mon.b (mon.1) 723 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:29:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:33 smithi006 bash[33918]: audit 2023-08-28T01:29:33.153745+0000 mon.b (mon.1) 723 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:29:34.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:33 smithi006 bash[27292]: audit 2023-08-28T01:29:33.153745+0000 mon.b (mon.1) 723 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:29:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:29:34.480Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:29:34.483Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:34.995 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:34 smithi130 bash[51302]: debug 2023-08-28T01:29:34.623+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:34.627377+0000) 2023-08-28T01:29:35.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:34 smithi130 bash[27579]: cluster 2023-08-28T01:29:34.173531+0000 mgr.y (mgr.14901) 6329 : cluster [DBG] pgmap v6336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:35.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:34 smithi130 bash[27579]: audit 2023-08-28T01:29:34.242571+0000 mon.a (mon.0) 4213 : audit [DBG] from='client.? 172.21.15.6:0/52579503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:35.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:34 smithi006 bash[27292]: cluster 2023-08-28T01:29:34.173531+0000 mgr.y (mgr.14901) 6329 : cluster [DBG] pgmap v6336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:35.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:34 smithi006 bash[27292]: audit 2023-08-28T01:29:34.242571+0000 mon.a (mon.0) 4213 : audit [DBG] from='client.? 172.21.15.6:0/52579503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:35.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:34 smithi006 bash[33918]: cluster 2023-08-28T01:29:34.173531+0000 mgr.y (mgr.14901) 6329 : cluster [DBG] pgmap v6336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:35.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:34 smithi006 bash[33918]: audit 2023-08-28T01:29:34.242571+0000 mon.a (mon.0) 4213 : audit [DBG] from='client.? 172.21.15.6:0/52579503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:36.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:35 smithi130 bash[51302]: debug 2023-08-28T01:29:35.623+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:35.627625+0000) 2023-08-28T01:29:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:36 smithi130 bash[27579]: cluster 2023-08-28T01:29:36.174311+0000 mgr.y (mgr.14901) 6330 : cluster [DBG] pgmap v6337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:36.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:36 smithi006 bash[27292]: cluster 2023-08-28T01:29:36.174311+0000 mgr.y (mgr.14901) 6330 : cluster [DBG] pgmap v6337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:36.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:36 smithi006 bash[33918]: cluster 2023-08-28T01:29:36.174311+0000 mgr.y (mgr.14901) 6330 : cluster [DBG] pgmap v6337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:37.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:36 smithi130 bash[51302]: debug 2023-08-28T01:29:36.623+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:36.627869+0000) 2023-08-28T01:29:37.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:37 smithi130 bash[27579]: audit 2023-08-28T01:29:36.662152+0000 mon.a (mon.0) 4214 : audit [DBG] from='client.? 172.21.15.6:0/3158996351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:37.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:37 smithi006 bash[33918]: audit 2023-08-28T01:29:36.662152+0000 mon.a (mon.0) 4214 : audit [DBG] from='client.? 172.21.15.6:0/3158996351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:37.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:37 smithi006 bash[27292]: audit 2023-08-28T01:29:36.662152+0000 mon.a (mon.0) 4214 : audit [DBG] from='client.? 172.21.15.6:0/3158996351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:37 smithi130 bash[51302]: debug 2023-08-28T01:29:37.623+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:37.628144+0000) 2023-08-28T01:29:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:38 smithi130 bash[27579]: cluster 2023-08-28T01:29:38.174979+0000 mgr.y (mgr.14901) 6331 : cluster [DBG] pgmap v6338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:38 smithi006 bash[27292]: cluster 2023-08-28T01:29:38.174979+0000 mgr.y (mgr.14901) 6331 : cluster [DBG] pgmap v6338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:38.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:38 smithi006 bash[33918]: cluster 2023-08-28T01:29:38.174979+0000 mgr.y (mgr.14901) 6331 : cluster [DBG] pgmap v6338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:39.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:38 smithi130 bash[51302]: debug 2023-08-28T01:29:38.623+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:38.628369+0000) 2023-08-28T01:29:40.021 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:39 smithi130 bash[51302]: debug 2023-08-28T01:29:39.623+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:39.628580+0000) 2023-08-28T01:29:40.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:40 smithi130 bash[27579]: audit 2023-08-28T01:29:39.020747+0000 mon.a (mon.0) 4215 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:29:40.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:40 smithi130 bash[27579]: audit 2023-08-28T01:29:39.114731+0000 mon.c (mon.2) 2484 : audit [DBG] from='client.? 172.21.15.6:0/264872117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:40.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:40 smithi130 bash[27579]: audit 2023-08-28T01:29:39.489023+0000 mon.a (mon.0) 4216 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:29:40.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:40 smithi130 bash[27579]: audit 2023-08-28T01:29:39.785039+0000 mon.b (mon.1) 724 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:29:40.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:40 smithi130 bash[27579]: audit 2023-08-28T01:29:39.787559+0000 mon.b (mon.1) 725 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:29:40.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:40 smithi130 bash[27579]: audit 2023-08-28T01:29:39.795980+0000 mon.a (mon.0) 4217 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:29:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:40 smithi006 bash[33918]: audit 2023-08-28T01:29:39.020747+0000 mon.a (mon.0) 4215 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:29:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:40 smithi006 bash[33918]: audit 2023-08-28T01:29:39.114731+0000 mon.c (mon.2) 2484 : audit [DBG] from='client.? 172.21.15.6:0/264872117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:40 smithi006 bash[33918]: audit 2023-08-28T01:29:39.489023+0000 mon.a (mon.0) 4216 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:29:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:40 smithi006 bash[33918]: audit 2023-08-28T01:29:39.785039+0000 mon.b (mon.1) 724 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:29:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:40 smithi006 bash[33918]: audit 2023-08-28T01:29:39.787559+0000 mon.b (mon.1) 725 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:29:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:40 smithi006 bash[33918]: audit 2023-08-28T01:29:39.795980+0000 mon.a (mon.0) 4217 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:29:40.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:40 smithi006 bash[27292]: audit 2023-08-28T01:29:39.020747+0000 mon.a (mon.0) 4215 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:29:40.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:40 smithi006 bash[27292]: audit 2023-08-28T01:29:39.114731+0000 mon.c (mon.2) 2484 : audit [DBG] from='client.? 172.21.15.6:0/264872117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:40.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:40 smithi006 bash[27292]: audit 2023-08-28T01:29:39.489023+0000 mon.a (mon.0) 4216 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:29:40.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:40 smithi006 bash[27292]: audit 2023-08-28T01:29:39.785039+0000 mon.b (mon.1) 724 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:29:40.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:40 smithi006 bash[27292]: audit 2023-08-28T01:29:39.787559+0000 mon.b (mon.1) 725 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:29:40.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:40 smithi006 bash[27292]: audit 2023-08-28T01:29:39.795980+0000 mon.a (mon.0) 4217 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:29:41.027 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:29:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:29:40] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T01:29:41.028 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:40 smithi130 bash[51302]: debug 2023-08-28T01:29:40.623+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:40.628883+0000) 2023-08-28T01:29:41.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:41 smithi130 bash[27579]: cluster 2023-08-28T01:29:40.176415+0000 mgr.y (mgr.14901) 6332 : cluster [DBG] pgmap v6339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:41.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:41 smithi006 bash[33918]: cluster 2023-08-28T01:29:40.176415+0000 mgr.y (mgr.14901) 6332 : cluster [DBG] pgmap v6339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:41.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:41 smithi006 bash[27292]: cluster 2023-08-28T01:29:40.176415+0000 mgr.y (mgr.14901) 6332 : cluster [DBG] pgmap v6339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:29:41.031Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:29:41.032Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:42.030 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:41 smithi130 bash[51302]: debug 2023-08-28T01:29:41.623+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:41.629185+0000) 2023-08-28T01:29:42.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:42 smithi130 bash[27579]: audit 2023-08-28T01:29:41.528258+0000 mon.c (mon.2) 2485 : audit [DBG] from='client.? 172.21.15.6:0/2210974006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:42.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:42 smithi006 bash[27292]: audit 2023-08-28T01:29:41.528258+0000 mon.c (mon.2) 2485 : audit [DBG] from='client.? 172.21.15.6:0/2210974006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:42.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:42 smithi006 bash[33918]: audit 2023-08-28T01:29:41.528258+0000 mon.c (mon.2) 2485 : audit [DBG] from='client.? 172.21.15.6:0/2210974006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:42.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:29:42.601Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:42.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:29:42.602Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:43.040 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:42 smithi130 bash[51302]: debug 2023-08-28T01:29:42.623+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:42.629433+0000) 2023-08-28T01:29:43.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:43 smithi130 bash[27579]: cluster 2023-08-28T01:29:42.177156+0000 mgr.y (mgr.14901) 6333 : cluster [DBG] pgmap v6340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:43.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:43 smithi006 bash[33918]: cluster 2023-08-28T01:29:42.177156+0000 mgr.y (mgr.14901) 6333 : cluster [DBG] pgmap v6340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:43.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:43 smithi006 bash[27292]: cluster 2023-08-28T01:29:42.177156+0000 mgr.y (mgr.14901) 6333 : cluster [DBG] pgmap v6340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:44.045 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:43 smithi130 bash[51302]: debug 2023-08-28T01:29:43.622+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:43.629714+0000) 2023-08-28T01:29:44.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:44 smithi130 bash[27579]: audit 2023-08-28T01:29:43.943604+0000 mon.c (mon.2) 2486 : audit [DBG] from='client.? 172.21.15.6:0/403489733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:44.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:44 smithi006 bash[27292]: audit 2023-08-28T01:29:43.943604+0000 mon.c (mon.2) 2486 : audit [DBG] from='client.? 172.21.15.6:0/403489733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:44 smithi006 bash[33918]: audit 2023-08-28T01:29:43.943604+0000 mon.c (mon.2) 2486 : audit [DBG] from='client.? 172.21.15.6:0/403489733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:29:44.481Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:29:44.483Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:45.052 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:44 smithi130 bash[51302]: debug 2023-08-28T01:29:44.622+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:44.629986+0000) 2023-08-28T01:29:45.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:45 smithi130 bash[27579]: cluster 2023-08-28T01:29:44.178164+0000 mgr.y (mgr.14901) 6334 : cluster [DBG] pgmap v6341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:45.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:45 smithi006 bash[27292]: cluster 2023-08-28T01:29:44.178164+0000 mgr.y (mgr.14901) 6334 : cluster [DBG] pgmap v6341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:45.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:45 smithi006 bash[33918]: cluster 2023-08-28T01:29:44.178164+0000 mgr.y (mgr.14901) 6334 : cluster [DBG] pgmap v6341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:45 smithi130 bash[51302]: debug 2023-08-28T01:29:45.622+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:45.630256+0000) 2023-08-28T01:29:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:46 smithi130 bash[27579]: cluster 2023-08-28T01:29:46.178792+0000 mgr.y (mgr.14901) 6335 : cluster [DBG] pgmap v6342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:46.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:46 smithi006 bash[27292]: cluster 2023-08-28T01:29:46.178792+0000 mgr.y (mgr.14901) 6335 : cluster [DBG] pgmap v6342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:46.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:46 smithi006 bash[33918]: cluster 2023-08-28T01:29:46.178792+0000 mgr.y (mgr.14901) 6335 : cluster [DBG] pgmap v6342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:46 smithi130 bash[51302]: debug 2023-08-28T01:29:46.622+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:46.630555+0000) 2023-08-28T01:29:47.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:47 smithi130 bash[27579]: audit 2023-08-28T01:29:46.361965+0000 mon.c (mon.2) 2487 : audit [DBG] from='client.? 172.21.15.6:0/2851953682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:47.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:47 smithi006 bash[33918]: audit 2023-08-28T01:29:46.361965+0000 mon.c (mon.2) 2487 : audit [DBG] from='client.? 172.21.15.6:0/2851953682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:47.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:47 smithi006 bash[27292]: audit 2023-08-28T01:29:46.361965+0000 mon.c (mon.2) 2487 : audit [DBG] from='client.? 172.21.15.6:0/2851953682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:47 smithi130 bash[51302]: debug 2023-08-28T01:29:47.622+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:47.630818+0000) 2023-08-28T01:29:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:48 smithi130 bash[27579]: cluster 2023-08-28T01:29:48.179451+0000 mgr.y (mgr.14901) 6336 : cluster [DBG] pgmap v6343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:48 smithi006 bash[27292]: cluster 2023-08-28T01:29:48.179451+0000 mgr.y (mgr.14901) 6336 : cluster [DBG] pgmap v6343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:48.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:48 smithi006 bash[33918]: cluster 2023-08-28T01:29:48.179451+0000 mgr.y (mgr.14901) 6336 : cluster [DBG] pgmap v6343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:49.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:48 smithi130 bash[51302]: debug 2023-08-28T01:29:48.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:48.631080+0000) 2023-08-28T01:29:49.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:49 smithi130 bash[27579]: audit 2023-08-28T01:29:48.783249+0000 mon.c (mon.2) 2488 : audit [DBG] from='client.? 172.21.15.6:0/2647500162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:49.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:49 smithi006 bash[33918]: audit 2023-08-28T01:29:48.783249+0000 mon.c (mon.2) 2488 : audit [DBG] from='client.? 172.21.15.6:0/2647500162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:49.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:49 smithi006 bash[27292]: audit 2023-08-28T01:29:48.783249+0000 mon.c (mon.2) 2488 : audit [DBG] from='client.? 172.21.15.6:0/2647500162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:50.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:49 smithi130 bash[51302]: debug 2023-08-28T01:29:49.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:49.631295+0000) 2023-08-28T01:29:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:50 smithi130 bash[27579]: cluster 2023-08-28T01:29:50.180598+0000 mgr.y (mgr.14901) 6337 : cluster [DBG] pgmap v6344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:50.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:50 smithi006 bash[33918]: cluster 2023-08-28T01:29:50.180598+0000 mgr.y (mgr.14901) 6337 : cluster [DBG] pgmap v6344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:50 smithi006 bash[27292]: cluster 2023-08-28T01:29:50.180598+0000 mgr.y (mgr.14901) 6337 : cluster [DBG] pgmap v6344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:51.032 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:29:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:29:50] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:29:51.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:50 smithi130 bash[51302]: debug 2023-08-28T01:29:50.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:50.631523+0000) 2023-08-28T01:29:51.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:51 smithi006 bash[27292]: audit 2023-08-28T01:29:51.203444+0000 mon.a (mon.0) 4218 : audit [DBG] from='client.? 172.21.15.6:0/3825384827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:51.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:51 smithi006 bash[33918]: audit 2023-08-28T01:29:51.203444+0000 mon.a (mon.0) 4218 : audit [DBG] from='client.? 172.21.15.6:0/3825384827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:51.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:29:51.031Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:51.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:29:51.033Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:51.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:51 smithi130 bash[27579]: audit 2023-08-28T01:29:51.203444+0000 mon.a (mon.0) 4218 : audit [DBG] from='client.? 172.21.15.6:0/3825384827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:52.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:51 smithi130 bash[51302]: debug 2023-08-28T01:29:51.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:51.631800+0000) 2023-08-28T01:29:52.956 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:29:52.601Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:52.956 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:29:52.603Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:52.956 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:52 smithi130 bash[51302]: debug 2023-08-28T01:29:52.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:52.632053+0000) 2023-08-28T01:29:53.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:52 smithi006 bash[33918]: cluster 2023-08-28T01:29:52.181265+0000 mgr.y (mgr.14901) 6338 : cluster [DBG] pgmap v6345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:53.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:52 smithi006 bash[27292]: cluster 2023-08-28T01:29:52.181265+0000 mgr.y (mgr.14901) 6338 : cluster [DBG] pgmap v6345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:53.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:52 smithi130 bash[27579]: cluster 2023-08-28T01:29:52.181265+0000 mgr.y (mgr.14901) 6338 : cluster [DBG] pgmap v6345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:53.961 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:53 smithi130 bash[51302]: debug 2023-08-28T01:29:53.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:53.632334+0000) 2023-08-28T01:29:54.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:53 smithi006 bash[33918]: audit 2023-08-28T01:29:53.613652+0000 mon.a (mon.0) 4219 : audit [DBG] from='client.? 172.21.15.6:0/2972359826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:54.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:53 smithi006 bash[27292]: audit 2023-08-28T01:29:53.613652+0000 mon.a (mon.0) 4219 : audit [DBG] from='client.? 172.21.15.6:0/2972359826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:53 smithi130 bash[27579]: audit 2023-08-28T01:29:53.613652+0000 mon.a (mon.0) 4219 : audit [DBG] from='client.? 172.21.15.6:0/2972359826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:29:54.482Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:29:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:29:54.484Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:29:54.968 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:54 smithi130 bash[51302]: debug 2023-08-28T01:29:54.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:54.632623+0000) 2023-08-28T01:29:55.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:54 smithi130 bash[27579]: cluster 2023-08-28T01:29:54.182332+0000 mgr.y (mgr.14901) 6339 : cluster [DBG] pgmap v6346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:55.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:54 smithi006 bash[27292]: cluster 2023-08-28T01:29:54.182332+0000 mgr.y (mgr.14901) 6339 : cluster [DBG] pgmap v6346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:55.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:54 smithi006 bash[33918]: cluster 2023-08-28T01:29:54.182332+0000 mgr.y (mgr.14901) 6339 : cluster [DBG] pgmap v6346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:56.076 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:55 smithi130 bash[51302]: debug 2023-08-28T01:29:55.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:55.632953+0000) 2023-08-28T01:29:56.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:56 smithi130 bash[27579]: audit 2023-08-28T01:29:56.021458+0000 mon.a (mon.0) 4220 : audit [DBG] from='client.? 172.21.15.6:0/4189712218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:56.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:56 smithi006 bash[33918]: audit 2023-08-28T01:29:56.021458+0000 mon.a (mon.0) 4220 : audit [DBG] from='client.? 172.21.15.6:0/4189712218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:56.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:56 smithi006 bash[27292]: audit 2023-08-28T01:29:56.021458+0000 mon.a (mon.0) 4220 : audit [DBG] from='client.? 172.21.15.6:0/4189712218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:57.084 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:56 smithi130 bash[51302]: debug 2023-08-28T01:29:56.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:56.633307+0000) 2023-08-28T01:29:57.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:57 smithi130 bash[27579]: cluster 2023-08-28T01:29:56.183035+0000 mgr.y (mgr.14901) 6340 : cluster [DBG] pgmap v6347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:57.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:57 smithi006 bash[33918]: cluster 2023-08-28T01:29:56.183035+0000 mgr.y (mgr.14901) 6340 : cluster [DBG] pgmap v6347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:57.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:57 smithi006 bash[27292]: cluster 2023-08-28T01:29:56.183035+0000 mgr.y (mgr.14901) 6340 : cluster [DBG] pgmap v6347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:58.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:57 smithi130 bash[51302]: debug 2023-08-28T01:29:57.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:57.633583+0000) 2023-08-28T01:29:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:58 smithi130 bash[27579]: cluster 2023-08-28T01:29:58.183646+0000 mgr.y (mgr.14901) 6341 : cluster [DBG] pgmap v6348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:58.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:58 smithi006 bash[33918]: cluster 2023-08-28T01:29:58.183646+0000 mgr.y (mgr.14901) 6341 : cluster [DBG] pgmap v6348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:58.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:58 smithi006 bash[27292]: cluster 2023-08-28T01:29:58.183646+0000 mgr.y (mgr.14901) 6341 : cluster [DBG] pgmap v6348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:29:59.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:58 smithi130 bash[51302]: debug 2023-08-28T01:29:58.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:58.633810+0000) 2023-08-28T01:29:59.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:29:59 smithi130 bash[27579]: audit 2023-08-28T01:29:58.433229+0000 mon.c (mon.2) 2489 : audit [DBG] from='client.? 172.21.15.6:0/163120233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:59.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:29:59 smithi006 bash[27292]: audit 2023-08-28T01:29:58.433229+0000 mon.c (mon.2) 2489 : audit [DBG] from='client.? 172.21.15.6:0/163120233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:59.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:29:59 smithi006 bash[33918]: audit 2023-08-28T01:29:58.433229+0000 mon.c (mon.2) 2489 : audit [DBG] from='client.? 172.21.15.6:0/163120233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:29:59 smithi130 bash[51302]: debug 2023-08-28T01:29:59.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:59.634086+0000) 2023-08-28T01:30:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:00 smithi130 bash[27579]: cluster 2023-08-28T01:30:00.000158+0000 mon.a (mon.0) 4221 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:30:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:00 smithi130 bash[27579]: cluster 2023-08-28T01:30:00.184876+0000 mgr.y (mgr.14901) 6342 : cluster [DBG] pgmap v6349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:00 smithi006 bash[27292]: cluster 2023-08-28T01:30:00.000158+0000 mon.a (mon.0) 4221 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:30:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:00 smithi006 bash[27292]: cluster 2023-08-28T01:30:00.184876+0000 mgr.y (mgr.14901) 6342 : cluster [DBG] pgmap v6349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:00 smithi006 bash[33918]: cluster 2023-08-28T01:30:00.000158+0000 mon.a (mon.0) 4221 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:30:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:00 smithi006 bash[33918]: cluster 2023-08-28T01:30:00.184876+0000 mgr.y (mgr.14901) 6342 : cluster [DBG] pgmap v6349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:01.033 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:30:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:30:00] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:30:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:00 smithi130 bash[51302]: debug 2023-08-28T01:30:00.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:00.634373+0000) 2023-08-28T01:30:01.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:01 smithi006 bash[33918]: audit 2023-08-28T01:30:00.856433+0000 mon.a (mon.0) 4222 : audit [DBG] from='client.? 172.21.15.6:0/486102657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:01.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:01 smithi006 bash[27292]: audit 2023-08-28T01:30:00.856433+0000 mon.a (mon.0) 4222 : audit [DBG] from='client.? 172.21.15.6:0/486102657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:30:01.032Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:01.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:30:01.034Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:01.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:01 smithi130 bash[27579]: audit 2023-08-28T01:30:00.856433+0000 mon.a (mon.0) 4222 : audit [DBG] from='client.? 172.21.15.6:0/486102657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:02.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:01 smithi130 bash[51302]: debug 2023-08-28T01:30:01.630+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:01.634598+0000) 2023-08-28T01:30:02.957 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:30:02.602Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:02.957 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:30:02.604Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:02.958 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:02 smithi130 bash[51302]: debug 2023-08-28T01:30:02.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:02.634952+0000) 2023-08-28T01:30:02.958 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:02 smithi006 bash[27292]: cluster 2023-08-28T01:30:02.185502+0000 mgr.y (mgr.14901) 6343 : cluster [DBG] pgmap v6350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:03.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:02 smithi006 bash[33918]: cluster 2023-08-28T01:30:02.185502+0000 mgr.y (mgr.14901) 6343 : cluster [DBG] pgmap v6350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:03.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:02 smithi130 bash[27579]: cluster 2023-08-28T01:30:02.185502+0000 mgr.y (mgr.14901) 6343 : cluster [DBG] pgmap v6350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:03.961 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:03 smithi130 bash[51302]: debug 2023-08-28T01:30:03.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:03.635169+0000) 2023-08-28T01:30:04.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:03 smithi006 bash[27292]: audit 2023-08-28T01:30:03.269352+0000 mon.a (mon.0) 4223 : audit [DBG] from='client.? 172.21.15.6:0/2431515097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:04.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:03 smithi006 bash[33918]: audit 2023-08-28T01:30:03.269352+0000 mon.a (mon.0) 4223 : audit [DBG] from='client.? 172.21.15.6:0/2431515097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:03 smithi130 bash[27579]: audit 2023-08-28T01:30:03.269352+0000 mon.a (mon.0) 4223 : audit [DBG] from='client.? 172.21.15.6:0/2431515097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:30:04.483Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:30:04.485Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:04.969 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:04 smithi130 bash[51302]: debug 2023-08-28T01:30:04.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:04.635390+0000) 2023-08-28T01:30:05.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:04 smithi130 bash[27579]: cluster 2023-08-28T01:30:04.186610+0000 mgr.y (mgr.14901) 6344 : cluster [DBG] pgmap v6351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:05.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:04 smithi006 bash[33918]: cluster 2023-08-28T01:30:04.186610+0000 mgr.y (mgr.14901) 6344 : cluster [DBG] pgmap v6351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:05.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:04 smithi006 bash[27292]: cluster 2023-08-28T01:30:04.186610+0000 mgr.y (mgr.14901) 6344 : cluster [DBG] pgmap v6351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:05.973 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:05 smithi130 bash[51302]: debug 2023-08-28T01:30:05.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:05.635682+0000) 2023-08-28T01:30:06.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:05 smithi130 bash[27579]: audit 2023-08-28T01:30:05.686564+0000 mon.a (mon.0) 4224 : audit [DBG] from='client.? 172.21.15.6:0/1087153980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:06.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:05 smithi006 bash[33918]: audit 2023-08-28T01:30:05.686564+0000 mon.a (mon.0) 4224 : audit [DBG] from='client.? 172.21.15.6:0/1087153980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:06.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:05 smithi006 bash[27292]: audit 2023-08-28T01:30:05.686564+0000 mon.a (mon.0) 4224 : audit [DBG] from='client.? 172.21.15.6:0/1087153980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:06.976 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:06 smithi130 bash[51302]: debug 2023-08-28T01:30:06.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:06.635973+0000) 2023-08-28T01:30:07.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:06 smithi130 bash[27579]: cluster 2023-08-28T01:30:06.187240+0000 mgr.y (mgr.14901) 6345 : cluster [DBG] pgmap v6352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:07.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:06 smithi006 bash[33918]: cluster 2023-08-28T01:30:06.187240+0000 mgr.y (mgr.14901) 6345 : cluster [DBG] pgmap v6352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:06 smithi006 bash[27292]: cluster 2023-08-28T01:30:06.187240+0000 mgr.y (mgr.14901) 6345 : cluster [DBG] pgmap v6352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:08.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:07 smithi130 bash[51302]: debug 2023-08-28T01:30:07.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:07.636269+0000) 2023-08-28T01:30:08.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:08 smithi006 bash[33918]: audit 2023-08-28T01:30:08.098187+0000 mon.a (mon.0) 4225 : audit [DBG] from='client.? 172.21.15.6:0/1706687124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:08.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:08 smithi006 bash[27292]: audit 2023-08-28T01:30:08.098187+0000 mon.a (mon.0) 4225 : audit [DBG] from='client.? 172.21.15.6:0/1706687124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:08 smithi130 bash[27579]: audit 2023-08-28T01:30:08.098187+0000 mon.a (mon.0) 4225 : audit [DBG] from='client.? 172.21.15.6:0/1706687124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:09.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:08 smithi130 bash[51302]: debug 2023-08-28T01:30:08.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:08.636560+0000) 2023-08-28T01:30:09.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:09 smithi006 bash[27292]: cluster 2023-08-28T01:30:08.187904+0000 mgr.y (mgr.14901) 6346 : cluster [DBG] pgmap v6353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:09.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:09 smithi006 bash[33918]: cluster 2023-08-28T01:30:08.187904+0000 mgr.y (mgr.14901) 6346 : cluster [DBG] pgmap v6353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:09.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:09 smithi130 bash[27579]: cluster 2023-08-28T01:30:08.187904+0000 mgr.y (mgr.14901) 6346 : cluster [DBG] pgmap v6353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:10.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:09 smithi130 bash[51302]: debug 2023-08-28T01:30:09.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:09.636817+0000) 2023-08-28T01:30:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:10 smithi130 bash[27579]: cluster 2023-08-28T01:30:10.188997+0000 mgr.y (mgr.14901) 6347 : cluster [DBG] pgmap v6354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:10 smithi006 bash[27292]: cluster 2023-08-28T01:30:10.188997+0000 mgr.y (mgr.14901) 6347 : cluster [DBG] pgmap v6354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:10.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:10 smithi006 bash[33918]: cluster 2023-08-28T01:30:10.188997+0000 mgr.y (mgr.14901) 6347 : cluster [DBG] pgmap v6354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:11.034 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:30:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:30:10] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:30:11.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:10 smithi130 bash[51302]: debug 2023-08-28T01:30:10.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:10.637109+0000) 2023-08-28T01:30:11.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:11 smithi006 bash[33918]: audit 2023-08-28T01:30:10.530977+0000 mon.a (mon.0) 4226 : audit [DBG] from='client.? 172.21.15.6:0/3791403818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:11.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:11 smithi006 bash[27292]: audit 2023-08-28T01:30:10.530977+0000 mon.a (mon.0) 4226 : audit [DBG] from='client.? 172.21.15.6:0/3791403818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:30:11.033Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:11.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:30:11.035Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:11.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:11 smithi130 bash[27579]: audit 2023-08-28T01:30:10.530977+0000 mon.a (mon.0) 4226 : audit [DBG] from='client.? 172.21.15.6:0/3791403818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:12.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:11 smithi130 bash[51302]: debug 2023-08-28T01:30:11.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:11.637380+0000) 2023-08-28T01:30:12.961 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:30:12.603Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:12.962 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:30:12.605Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:12.962 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:12 smithi130 bash[51302]: debug 2023-08-28T01:30:12.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:12.637680+0000) 2023-08-28T01:30:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:12 smithi006 bash[33918]: cluster 2023-08-28T01:30:12.189572+0000 mgr.y (mgr.14901) 6348 : cluster [DBG] pgmap v6355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:13.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:12 smithi006 bash[27292]: cluster 2023-08-28T01:30:12.189572+0000 mgr.y (mgr.14901) 6348 : cluster [DBG] pgmap v6355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:13.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:12 smithi130 bash[27579]: cluster 2023-08-28T01:30:12.189572+0000 mgr.y (mgr.14901) 6348 : cluster [DBG] pgmap v6355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:13.965 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:13 smithi130 bash[51302]: debug 2023-08-28T01:30:13.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:13.637977+0000) 2023-08-28T01:30:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:13 smithi006 bash[27292]: audit 2023-08-28T01:30:12.958641+0000 mon.c (mon.2) 2490 : audit [DBG] from='client.? 172.21.15.6:0/560520413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:13 smithi006 bash[33918]: audit 2023-08-28T01:30:12.958641+0000 mon.c (mon.2) 2490 : audit [DBG] from='client.? 172.21.15.6:0/560520413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:14.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:13 smithi130 bash[27579]: audit 2023-08-28T01:30:12.958641+0000 mon.c (mon.2) 2490 : audit [DBG] from='client.? 172.21.15.6:0/560520413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:30:14.484Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:30:14.486Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:14.973 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:14 smithi130 bash[51302]: debug 2023-08-28T01:30:14.634+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:14.638243+0000) 2023-08-28T01:30:15.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:14 smithi130 bash[27579]: cluster 2023-08-28T01:30:14.190595+0000 mgr.y (mgr.14901) 6349 : cluster [DBG] pgmap v6356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:15.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:14 smithi006 bash[27292]: cluster 2023-08-28T01:30:14.190595+0000 mgr.y (mgr.14901) 6349 : cluster [DBG] pgmap v6356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:15.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:14 smithi006 bash[33918]: cluster 2023-08-28T01:30:14.190595+0000 mgr.y (mgr.14901) 6349 : cluster [DBG] pgmap v6356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:15.976 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:15 smithi130 bash[51302]: debug 2023-08-28T01:30:15.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:15.638617+0000) 2023-08-28T01:30:16.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:15 smithi130 bash[27579]: audit 2023-08-28T01:30:15.376541+0000 mon.c (mon.2) 2491 : audit [DBG] from='client.? 172.21.15.6:0/281668073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:16.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:15 smithi006 bash[27292]: audit 2023-08-28T01:30:15.376541+0000 mon.c (mon.2) 2491 : audit [DBG] from='client.? 172.21.15.6:0/281668073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:16.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:15 smithi006 bash[33918]: audit 2023-08-28T01:30:15.376541+0000 mon.c (mon.2) 2491 : audit [DBG] from='client.? 172.21.15.6:0/281668073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:16.983 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:16 smithi130 bash[51302]: debug 2023-08-28T01:30:16.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:16.638971+0000) 2023-08-28T01:30:17.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:16 smithi130 bash[27579]: cluster 2023-08-28T01:30:16.191307+0000 mgr.y (mgr.14901) 6350 : cluster [DBG] pgmap v6357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:17.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:16 smithi006 bash[27292]: cluster 2023-08-28T01:30:16.191307+0000 mgr.y (mgr.14901) 6350 : cluster [DBG] pgmap v6357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:17.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:16 smithi006 bash[33918]: cluster 2023-08-28T01:30:16.191307+0000 mgr.y (mgr.14901) 6350 : cluster [DBG] pgmap v6357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:17.986 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:17 smithi130 bash[51302]: debug 2023-08-28T01:30:17.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:17.639199+0000) 2023-08-28T01:30:18.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:17 smithi130 bash[27579]: audit 2023-08-28T01:30:17.793334+0000 mon.a (mon.0) 4227 : audit [DBG] from='client.? 172.21.15.6:0/2572177507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:18.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:17 smithi006 bash[27292]: audit 2023-08-28T01:30:17.793334+0000 mon.a (mon.0) 4227 : audit [DBG] from='client.? 172.21.15.6:0/2572177507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:18.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:17 smithi006 bash[33918]: audit 2023-08-28T01:30:17.793334+0000 mon.a (mon.0) 4227 : audit [DBG] from='client.? 172.21.15.6:0/2572177507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:18.994 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:18 smithi130 bash[51302]: debug 2023-08-28T01:30:18.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:18.639466+0000) 2023-08-28T01:30:19.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:18 smithi130 bash[27579]: cluster 2023-08-28T01:30:18.192005+0000 mgr.y (mgr.14901) 6351 : cluster [DBG] pgmap v6358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:19.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:18 smithi006 bash[27292]: cluster 2023-08-28T01:30:18.192005+0000 mgr.y (mgr.14901) 6351 : cluster [DBG] pgmap v6358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:19.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:18 smithi006 bash[33918]: cluster 2023-08-28T01:30:18.192005+0000 mgr.y (mgr.14901) 6351 : cluster [DBG] pgmap v6358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:19 smithi130 bash[51302]: debug 2023-08-28T01:30:19.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:19.639762+0000) 2023-08-28T01:30:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:20 smithi130 bash[27579]: cluster 2023-08-28T01:30:20.192751+0000 mgr.y (mgr.14901) 6352 : cluster [DBG] pgmap v6359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:20 smithi130 bash[27579]: audit 2023-08-28T01:30:20.205408+0000 mon.a (mon.0) 4228 : audit [DBG] from='client.? 172.21.15.6:0/67049419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:20 smithi006 bash[33918]: cluster 2023-08-28T01:30:20.192751+0000 mgr.y (mgr.14901) 6352 : cluster [DBG] pgmap v6359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:20 smithi006 bash[33918]: audit 2023-08-28T01:30:20.205408+0000 mon.a (mon.0) 4228 : audit [DBG] from='client.? 172.21.15.6:0/67049419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:20.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:20 smithi006 bash[27292]: cluster 2023-08-28T01:30:20.192751+0000 mgr.y (mgr.14901) 6352 : cluster [DBG] pgmap v6359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:20 smithi006 bash[27292]: audit 2023-08-28T01:30:20.205408+0000 mon.a (mon.0) 4228 : audit [DBG] from='client.? 172.21.15.6:0/67049419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:21.034 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:30:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:30:20] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:30:21.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:20 smithi130 bash[51302]: debug 2023-08-28T01:30:20.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:20.640048+0000) 2023-08-28T01:30:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:30:21.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:30:21.036Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:22.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:21 smithi130 bash[51302]: debug 2023-08-28T01:30:21.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:21.640306+0000) 2023-08-28T01:30:22.962 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:30:22.604Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:22.963 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:30:22.606Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:22.963 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:22 smithi130 bash[51302]: debug 2023-08-28T01:30:22.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:22.640591+0000) 2023-08-28T01:30:23.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:22 smithi006 bash[33918]: cluster 2023-08-28T01:30:22.193351+0000 mgr.y (mgr.14901) 6353 : cluster [DBG] pgmap v6360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:23.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:22 smithi006 bash[33918]: audit 2023-08-28T01:30:22.627458+0000 mon.a (mon.0) 4229 : audit [DBG] from='client.? 172.21.15.6:0/2536713922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:23.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:22 smithi006 bash[27292]: cluster 2023-08-28T01:30:22.193351+0000 mgr.y (mgr.14901) 6353 : cluster [DBG] pgmap v6360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:23.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:22 smithi006 bash[27292]: audit 2023-08-28T01:30:22.627458+0000 mon.a (mon.0) 4229 : audit [DBG] from='client.? 172.21.15.6:0/2536713922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:23.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:22 smithi130 bash[27579]: cluster 2023-08-28T01:30:22.193351+0000 mgr.y (mgr.14901) 6353 : cluster [DBG] pgmap v6360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:23.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:22 smithi130 bash[27579]: audit 2023-08-28T01:30:22.627458+0000 mon.a (mon.0) 4229 : audit [DBG] from='client.? 172.21.15.6:0/2536713922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:24.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:23 smithi130 bash[51302]: debug 2023-08-28T01:30:23.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:23.640939+0000) 2023-08-28T01:30:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:24 smithi130 bash[27579]: cluster 2023-08-28T01:30:24.194461+0000 mgr.y (mgr.14901) 6354 : cluster [DBG] pgmap v6361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:24.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:24 smithi006 bash[27292]: cluster 2023-08-28T01:30:24.194461+0000 mgr.y (mgr.14901) 6354 : cluster [DBG] pgmap v6361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:30:24.485Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:30:24.487Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:24.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:24 smithi006 bash[33918]: cluster 2023-08-28T01:30:24.194461+0000 mgr.y (mgr.14901) 6354 : cluster [DBG] pgmap v6361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:25.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:24 smithi130 bash[51302]: debug 2023-08-28T01:30:24.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:24.641238+0000) 2023-08-28T01:30:25.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:25 smithi130 bash[27579]: audit 2023-08-28T01:30:25.046005+0000 mon.a (mon.0) 4230 : audit [DBG] from='client.? 172.21.15.6:0/3349153873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:25.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:25 smithi006 bash[27292]: audit 2023-08-28T01:30:25.046005+0000 mon.a (mon.0) 4230 : audit [DBG] from='client.? 172.21.15.6:0/3349153873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:25.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:25 smithi006 bash[33918]: audit 2023-08-28T01:30:25.046005+0000 mon.a (mon.0) 4230 : audit [DBG] from='client.? 172.21.15.6:0/3349153873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:26.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:25 smithi130 bash[51302]: debug 2023-08-28T01:30:25.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:25.641505+0000) 2023-08-28T01:30:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:26 smithi130 bash[27579]: cluster 2023-08-28T01:30:26.195119+0000 mgr.y (mgr.14901) 6355 : cluster [DBG] pgmap v6362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:26 smithi006 bash[33918]: cluster 2023-08-28T01:30:26.195119+0000 mgr.y (mgr.14901) 6355 : cluster [DBG] pgmap v6362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:26 smithi006 bash[27292]: cluster 2023-08-28T01:30:26.195119+0000 mgr.y (mgr.14901) 6355 : cluster [DBG] pgmap v6362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:27.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:26 smithi130 bash[51302]: debug 2023-08-28T01:30:26.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:26.641821+0000) 2023-08-28T01:30:27.961 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:27 smithi130 bash[51302]: debug 2023-08-28T01:30:27.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:27.642077+0000) 2023-08-28T01:30:28.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:27 smithi006 bash[33918]: audit 2023-08-28T01:30:27.464038+0000 mon.a (mon.0) 4231 : audit [DBG] from='client.? 172.21.15.6:0/3762632407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:28.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:27 smithi006 bash[27292]: audit 2023-08-28T01:30:27.464038+0000 mon.a (mon.0) 4231 : audit [DBG] from='client.? 172.21.15.6:0/3762632407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:28.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:27 smithi130 bash[27579]: audit 2023-08-28T01:30:27.464038+0000 mon.a (mon.0) 4231 : audit [DBG] from='client.? 172.21.15.6:0/3762632407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:28.972 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:28 smithi130 bash[51302]: debug 2023-08-28T01:30:28.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:28.642353+0000) 2023-08-28T01:30:29.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:28 smithi130 bash[27579]: cluster 2023-08-28T01:30:28.195755+0000 mgr.y (mgr.14901) 6356 : cluster [DBG] pgmap v6363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:29.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:28 smithi006 bash[33918]: cluster 2023-08-28T01:30:28.195755+0000 mgr.y (mgr.14901) 6356 : cluster [DBG] pgmap v6363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:29.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:28 smithi006 bash[27292]: cluster 2023-08-28T01:30:28.195755+0000 mgr.y (mgr.14901) 6356 : cluster [DBG] pgmap v6363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:29.977 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:29 smithi130 bash[51302]: debug 2023-08-28T01:30:29.638+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:29.642621+0000) 2023-08-28T01:30:30.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:29 smithi130 bash[27579]: audit 2023-08-28T01:30:29.872020+0000 mon.c (mon.2) 2492 : audit [DBG] from='client.? 172.21.15.6:0/1955303247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:30.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:29 smithi006 bash[33918]: audit 2023-08-28T01:30:29.872020+0000 mon.c (mon.2) 2492 : audit [DBG] from='client.? 172.21.15.6:0/1955303247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:30.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:29 smithi006 bash[27292]: audit 2023-08-28T01:30:29.872020+0000 mon.c (mon.2) 2492 : audit [DBG] from='client.? 172.21.15.6:0/1955303247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:30.985 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:30 smithi130 bash[51302]: debug 2023-08-28T01:30:30.642+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:30.642974+0000) 2023-08-28T01:30:31.035 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:30:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:30:30] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:30:31.035 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:30 smithi006 bash[33918]: cluster 2023-08-28T01:30:30.196905+0000 mgr.y (mgr.14901) 6357 : cluster [DBG] pgmap v6364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:31.035 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:30 smithi006 bash[27292]: cluster 2023-08-28T01:30:30.196905+0000 mgr.y (mgr.14901) 6357 : cluster [DBG] pgmap v6364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:31.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:30 smithi130 bash[27579]: cluster 2023-08-28T01:30:30.196905+0000 mgr.y (mgr.14901) 6357 : cluster [DBG] pgmap v6364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:30:31.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:30:31.036Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:32.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:31 smithi130 bash[51302]: debug 2023-08-28T01:30:31.642+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:31.643217+0000) 2023-08-28T01:30:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:30:32.604Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:30:32.606Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:32.967 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:32 smithi130 bash[51302]: debug 2023-08-28T01:30:32.642+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:32.643481+0000) 2023-08-28T01:30:33.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:32 smithi130 bash[27579]: cluster 2023-08-28T01:30:32.197523+0000 mgr.y (mgr.14901) 6358 : cluster [DBG] pgmap v6365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:33.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:32 smithi130 bash[27579]: audit 2023-08-28T01:30:32.282494+0000 mon.a (mon.0) 4232 : audit [DBG] from='client.? 172.21.15.6:0/3395588692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:33.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:32 smithi006 bash[27292]: cluster 2023-08-28T01:30:32.197523+0000 mgr.y (mgr.14901) 6358 : cluster [DBG] pgmap v6365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:33.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:32 smithi006 bash[27292]: audit 2023-08-28T01:30:32.282494+0000 mon.a (mon.0) 4232 : audit [DBG] from='client.? 172.21.15.6:0/3395588692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:33.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:32 smithi006 bash[33918]: cluster 2023-08-28T01:30:32.197523+0000 mgr.y (mgr.14901) 6358 : cluster [DBG] pgmap v6365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:33.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:32 smithi006 bash[33918]: audit 2023-08-28T01:30:32.282494+0000 mon.a (mon.0) 4232 : audit [DBG] from='client.? 172.21.15.6:0/3395588692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:34.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:33 smithi130 bash[51302]: debug 2023-08-28T01:30:33.642+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:33.643755+0000) 2023-08-28T01:30:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:34 smithi130 bash[27579]: cluster 2023-08-28T01:30:34.198596+0000 mgr.y (mgr.14901) 6359 : cluster [DBG] pgmap v6366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:34.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:34 smithi006 bash[33918]: cluster 2023-08-28T01:30:34.198596+0000 mgr.y (mgr.14901) 6359 : cluster [DBG] pgmap v6366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:34.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:34 smithi006 bash[27292]: cluster 2023-08-28T01:30:34.198596+0000 mgr.y (mgr.14901) 6359 : cluster [DBG] pgmap v6366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:30:34.486Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:30:34.488Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:35.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:34 smithi130 bash[51302]: debug 2023-08-28T01:30:34.642+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:34.644021+0000) 2023-08-28T01:30:35.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:35 smithi130 bash[27579]: audit 2023-08-28T01:30:34.702367+0000 mon.c (mon.2) 2493 : audit [DBG] from='client.? 172.21.15.6:0/3395854051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:35.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:35 smithi006 bash[27292]: audit 2023-08-28T01:30:34.702367+0000 mon.c (mon.2) 2493 : audit [DBG] from='client.? 172.21.15.6:0/3395854051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:35.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:35 smithi006 bash[33918]: audit 2023-08-28T01:30:34.702367+0000 mon.c (mon.2) 2493 : audit [DBG] from='client.? 172.21.15.6:0/3395854051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:36.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:35 smithi130 bash[51302]: debug 2023-08-28T01:30:35.642+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:35.644296+0000) 2023-08-28T01:30:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:36 smithi130 bash[27579]: cluster 2023-08-28T01:30:36.199267+0000 mgr.y (mgr.14901) 6360 : cluster [DBG] pgmap v6367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:36.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:36 smithi006 bash[33918]: cluster 2023-08-28T01:30:36.199267+0000 mgr.y (mgr.14901) 6360 : cluster [DBG] pgmap v6367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:36.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:36 smithi006 bash[27292]: cluster 2023-08-28T01:30:36.199267+0000 mgr.y (mgr.14901) 6360 : cluster [DBG] pgmap v6367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:37.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:36 smithi130 bash[51302]: debug 2023-08-28T01:30:36.642+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:36.644591+0000) 2023-08-28T01:30:37.961 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:37 smithi130 bash[51302]: debug 2023-08-28T01:30:37.642+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:37.644867+0000) 2023-08-28T01:30:38.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:37 smithi006 bash[27292]: audit 2023-08-28T01:30:37.119890+0000 mon.a (mon.0) 4233 : audit [DBG] from='client.? 172.21.15.6:0/52268428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:38.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:37 smithi006 bash[33918]: audit 2023-08-28T01:30:37.119890+0000 mon.a (mon.0) 4233 : audit [DBG] from='client.? 172.21.15.6:0/52268428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:38.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:37 smithi130 bash[27579]: audit 2023-08-28T01:30:37.119890+0000 mon.a (mon.0) 4233 : audit [DBG] from='client.? 172.21.15.6:0/52268428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:38.964 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:38 smithi130 bash[51302]: debug 2023-08-28T01:30:38.642+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:38.645231+0000) 2023-08-28T01:30:39.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:38 smithi006 bash[27292]: cluster 2023-08-28T01:30:38.199971+0000 mgr.y (mgr.14901) 6361 : cluster [DBG] pgmap v6368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:39.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:38 smithi006 bash[33918]: cluster 2023-08-28T01:30:38.199971+0000 mgr.y (mgr.14901) 6361 : cluster [DBG] pgmap v6368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:39.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:38 smithi130 bash[27579]: cluster 2023-08-28T01:30:38.199971+0000 mgr.y (mgr.14901) 6361 : cluster [DBG] pgmap v6368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:39.974 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:39 smithi130 bash[51302]: debug 2023-08-28T01:30:39.641+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:39.645661+0000) 2023-08-28T01:30:40.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:39 smithi130 bash[27579]: audit 2023-08-28T01:30:39.542949+0000 mon.c (mon.2) 2494 : audit [DBG] from='client.? 172.21.15.6:0/2132951977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:40.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:39 smithi130 bash[27579]: audit 2023-08-28T01:30:39.807371+0000 mon.b (mon.1) 726 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:30:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:39 smithi006 bash[33918]: audit 2023-08-28T01:30:39.542949+0000 mon.c (mon.2) 2494 : audit [DBG] from='client.? 172.21.15.6:0/2132951977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:39 smithi006 bash[33918]: audit 2023-08-28T01:30:39.807371+0000 mon.b (mon.1) 726 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:30:40.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:39 smithi006 bash[27292]: audit 2023-08-28T01:30:39.542949+0000 mon.c (mon.2) 2494 : audit [DBG] from='client.? 172.21.15.6:0/2132951977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:40.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:39 smithi006 bash[27292]: audit 2023-08-28T01:30:39.807371+0000 mon.b (mon.1) 726 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:30:40.982 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:40 smithi130 bash[51302]: debug 2023-08-28T01:30:40.641+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:40.645987+0000) 2023-08-28T01:30:41.035 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:30:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:30:40] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:30:41.036 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:40 smithi006 bash[33918]: cluster 2023-08-28T01:30:40.201165+0000 mgr.y (mgr.14901) 6362 : cluster [DBG] pgmap v6369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:41.036 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:40 smithi006 bash[27292]: cluster 2023-08-28T01:30:40.201165+0000 mgr.y (mgr.14901) 6362 : cluster [DBG] pgmap v6369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:41.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:40 smithi130 bash[27579]: cluster 2023-08-28T01:30:40.201165+0000 mgr.y (mgr.14901) 6362 : cluster [DBG] pgmap v6369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:30:41.035Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:30:41.037Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:41 smithi130 bash[51302]: debug 2023-08-28T01:30:41.641+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:41.646371+0000) 2023-08-28T01:30:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:30:42.604Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:30:42.606Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:42.968 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:42 smithi130 bash[51302]: debug 2023-08-28T01:30:42.641+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:42.646793+0000) 2023-08-28T01:30:43.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:42 smithi130 bash[27579]: audit 2023-08-28T01:30:41.962546+0000 mon.c (mon.2) 2495 : audit [DBG] from='client.? 172.21.15.6:0/4163722244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:43.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:42 smithi130 bash[27579]: cluster 2023-08-28T01:30:42.201807+0000 mgr.y (mgr.14901) 6363 : cluster [DBG] pgmap v6370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:43.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:42 smithi006 bash[27292]: audit 2023-08-28T01:30:41.962546+0000 mon.c (mon.2) 2495 : audit [DBG] from='client.? 172.21.15.6:0/4163722244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:43.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:42 smithi006 bash[27292]: cluster 2023-08-28T01:30:42.201807+0000 mgr.y (mgr.14901) 6363 : cluster [DBG] pgmap v6370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:43.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:42 smithi006 bash[33918]: audit 2023-08-28T01:30:41.962546+0000 mon.c (mon.2) 2495 : audit [DBG] from='client.? 172.21.15.6:0/4163722244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:43.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:42 smithi006 bash[33918]: cluster 2023-08-28T01:30:42.201807+0000 mgr.y (mgr.14901) 6363 : cluster [DBG] pgmap v6370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:44.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:43 smithi130 bash[51302]: debug 2023-08-28T01:30:43.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:43.647027+0000) 2023-08-28T01:30:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:44 smithi130 bash[27579]: cluster 2023-08-28T01:30:44.202592+0000 mgr.y (mgr.14901) 6364 : cluster [DBG] pgmap v6371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:44.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:44 smithi006 bash[33918]: cluster 2023-08-28T01:30:44.202592+0000 mgr.y (mgr.14901) 6364 : cluster [DBG] pgmap v6371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:44.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:44 smithi006 bash[27292]: cluster 2023-08-28T01:30:44.202592+0000 mgr.y (mgr.14901) 6364 : cluster [DBG] pgmap v6371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:30:44.487Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:30:44.489Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:45.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:44 smithi130 bash[51302]: debug 2023-08-28T01:30:44.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:44.647230+0000) 2023-08-28T01:30:45.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:45 smithi130 bash[27579]: audit 2023-08-28T01:30:44.389892+0000 mon.a (mon.0) 4234 : audit [DBG] from='client.? 172.21.15.6:0/2648338672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:45.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:45 smithi006 bash[33918]: audit 2023-08-28T01:30:44.389892+0000 mon.a (mon.0) 4234 : audit [DBG] from='client.? 172.21.15.6:0/2648338672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:45.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:45 smithi006 bash[27292]: audit 2023-08-28T01:30:44.389892+0000 mon.a (mon.0) 4234 : audit [DBG] from='client.? 172.21.15.6:0/2648338672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:45 smithi130 bash[51302]: debug 2023-08-28T01:30:45.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:45.647441+0000) 2023-08-28T01:30:46.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:46 smithi006 bash[27292]: audit 2023-08-28T01:30:45.663500+0000 mon.a (mon.0) 4235 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:30:46.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:46 smithi006 bash[27292]: audit 2023-08-28T01:30:46.171545+0000 mon.a (mon.0) 4236 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:30:46.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:46 smithi006 bash[27292]: cluster 2023-08-28T01:30:46.203026+0000 mgr.y (mgr.14901) 6365 : cluster [DBG] pgmap v6372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:46.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:46 smithi006 bash[27292]: audit 2023-08-28T01:30:46.468554+0000 mon.b (mon.1) 727 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:30:46.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:46 smithi006 bash[27292]: audit 2023-08-28T01:30:46.470362+0000 mon.b (mon.1) 728 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:30:46.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:46 smithi006 bash[27292]: audit 2023-08-28T01:30:46.475229+0000 mon.a (mon.0) 4237 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:30:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:46 smithi006 bash[33918]: audit 2023-08-28T01:30:45.663500+0000 mon.a (mon.0) 4235 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:30:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:46 smithi006 bash[33918]: audit 2023-08-28T01:30:46.171545+0000 mon.a (mon.0) 4236 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:30:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:46 smithi006 bash[33918]: cluster 2023-08-28T01:30:46.203026+0000 mgr.y (mgr.14901) 6365 : cluster [DBG] pgmap v6372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:46 smithi006 bash[33918]: audit 2023-08-28T01:30:46.468554+0000 mon.b (mon.1) 727 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:30:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:46 smithi006 bash[33918]: audit 2023-08-28T01:30:46.470362+0000 mon.b (mon.1) 728 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:30:46.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:46 smithi006 bash[33918]: audit 2023-08-28T01:30:46.475229+0000 mon.a (mon.0) 4237 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:30:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:46 smithi130 bash[51302]: debug 2023-08-28T01:30:46.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:46.647704+0000) 2023-08-28T01:30:47.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:46 smithi130 bash[27579]: audit 2023-08-28T01:30:45.663500+0000 mon.a (mon.0) 4235 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:30:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:46 smithi130 bash[27579]: audit 2023-08-28T01:30:46.171545+0000 mon.a (mon.0) 4236 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:30:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:46 smithi130 bash[27579]: cluster 2023-08-28T01:30:46.203026+0000 mgr.y (mgr.14901) 6365 : cluster [DBG] pgmap v6372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:46 smithi130 bash[27579]: audit 2023-08-28T01:30:46.468554+0000 mon.b (mon.1) 727 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:30:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:46 smithi130 bash[27579]: audit 2023-08-28T01:30:46.470362+0000 mon.b (mon.1) 728 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:30:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:46 smithi130 bash[27579]: audit 2023-08-28T01:30:46.475229+0000 mon.a (mon.0) 4237 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:30:47.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:47 smithi006 bash[27292]: audit 2023-08-28T01:30:46.807510+0000 mon.a (mon.0) 4238 : audit [DBG] from='client.? 172.21.15.6:0/880923818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:47.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:47 smithi006 bash[33918]: audit 2023-08-28T01:30:46.807510+0000 mon.a (mon.0) 4238 : audit [DBG] from='client.? 172.21.15.6:0/880923818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:48.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:47 smithi130 bash[27579]: audit 2023-08-28T01:30:46.807510+0000 mon.a (mon.0) 4238 : audit [DBG] from='client.? 172.21.15.6:0/880923818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:47 smithi130 bash[51302]: debug 2023-08-28T01:30:47.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:47.647935+0000) 2023-08-28T01:30:48.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:48 smithi006 bash[27292]: cluster 2023-08-28T01:30:48.203633+0000 mgr.y (mgr.14901) 6366 : cluster [DBG] pgmap v6373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:48.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:48 smithi006 bash[33918]: cluster 2023-08-28T01:30:48.203633+0000 mgr.y (mgr.14901) 6366 : cluster [DBG] pgmap v6373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:49.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:48 smithi130 bash[27579]: cluster 2023-08-28T01:30:48.203633+0000 mgr.y (mgr.14901) 6366 : cluster [DBG] pgmap v6373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:49.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:48 smithi130 bash[51302]: debug 2023-08-28T01:30:48.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:48.648253+0000) 2023-08-28T01:30:49.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:49 smithi006 bash[27292]: audit 2023-08-28T01:30:49.225857+0000 mon.c (mon.2) 2496 : audit [DBG] from='client.? 172.21.15.6:0/3922042987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:49.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:49 smithi006 bash[33918]: audit 2023-08-28T01:30:49.225857+0000 mon.c (mon.2) 2496 : audit [DBG] from='client.? 172.21.15.6:0/3922042987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:50.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:49 smithi130 bash[27579]: audit 2023-08-28T01:30:49.225857+0000 mon.c (mon.2) 2496 : audit [DBG] from='client.? 172.21.15.6:0/3922042987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:50.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:49 smithi130 bash[51302]: debug 2023-08-28T01:30:49.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:49.648546+0000) 2023-08-28T01:30:51.036 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:30:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:30:50] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T01:30:51.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:50 smithi130 bash[51302]: debug 2023-08-28T01:30:50.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:50.648826+0000) 2023-08-28T01:30:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:30:51.035Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:30:51.037Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:51.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:51 smithi006 bash[33918]: cluster 2023-08-28T01:30:50.204700+0000 mgr.y (mgr.14901) 6367 : cluster [DBG] pgmap v6374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:51.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:51 smithi006 bash[33918]: audit 2023-08-28T01:30:51.644714+0000 mon.a (mon.0) 4239 : audit [DBG] from='client.? 172.21.15.6:0/3022242577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:51.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:51 smithi006 bash[27292]: cluster 2023-08-28T01:30:50.204700+0000 mgr.y (mgr.14901) 6367 : cluster [DBG] pgmap v6374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:51.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:51 smithi006 bash[27292]: audit 2023-08-28T01:30:51.644714+0000 mon.a (mon.0) 4239 : audit [DBG] from='client.? 172.21.15.6:0/3022242577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:52.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:51 smithi130 bash[51302]: debug 2023-08-28T01:30:51.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:51.649084+0000) 2023-08-28T01:30:52.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:51 smithi130 bash[27579]: cluster 2023-08-28T01:30:50.204700+0000 mgr.y (mgr.14901) 6367 : cluster [DBG] pgmap v6374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:52.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:51 smithi130 bash[27579]: audit 2023-08-28T01:30:51.644714+0000 mon.a (mon.0) 4239 : audit [DBG] from='client.? 172.21.15.6:0/3022242577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:30:52.605Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:30:52.607Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:53.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:52 smithi130 bash[51302]: debug 2023-08-28T01:30:52.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:52.649312+0000) 2023-08-28T01:30:53.969 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:53 smithi130 bash[51302]: debug 2023-08-28T01:30:53.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:53.649608+0000) 2023-08-28T01:30:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:53 smithi130 bash[27579]: cluster 2023-08-28T01:30:52.205360+0000 mgr.y (mgr.14901) 6368 : cluster [DBG] pgmap v6375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:53 smithi006 bash[33918]: cluster 2023-08-28T01:30:52.205360+0000 mgr.y (mgr.14901) 6368 : cluster [DBG] pgmap v6375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:53 smithi006 bash[27292]: cluster 2023-08-28T01:30:52.205360+0000 mgr.y (mgr.14901) 6368 : cluster [DBG] pgmap v6375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:30:54.487Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:30:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:30:54.489Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:30:54.978 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:54 smithi130 bash[51302]: debug 2023-08-28T01:30:54.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:54.649890+0000) 2023-08-28T01:30:55.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:54 smithi130 bash[27579]: audit 2023-08-28T01:30:54.059421+0000 mon.c (mon.2) 2497 : audit [DBG] from='client.? 172.21.15.6:0/3638179829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:55.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:54 smithi006 bash[27292]: audit 2023-08-28T01:30:54.059421+0000 mon.c (mon.2) 2497 : audit [DBG] from='client.? 172.21.15.6:0/3638179829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:55.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:54 smithi006 bash[33918]: audit 2023-08-28T01:30:54.059421+0000 mon.c (mon.2) 2497 : audit [DBG] from='client.? 172.21.15.6:0/3638179829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:55.981 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:55 smithi130 bash[51302]: debug 2023-08-28T01:30:55.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:55.650140+0000) 2023-08-28T01:30:56.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:55 smithi130 bash[27579]: cluster 2023-08-28T01:30:54.206464+0000 mgr.y (mgr.14901) 6369 : cluster [DBG] pgmap v6376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:56.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:55 smithi006 bash[33918]: cluster 2023-08-28T01:30:54.206464+0000 mgr.y (mgr.14901) 6369 : cluster [DBG] pgmap v6376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:56.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:55 smithi006 bash[27292]: cluster 2023-08-28T01:30:54.206464+0000 mgr.y (mgr.14901) 6369 : cluster [DBG] pgmap v6376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:56.990 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:56 smithi130 bash[51302]: debug 2023-08-28T01:30:56.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:56.650437+0000) 2023-08-28T01:30:57.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:56 smithi130 bash[27579]: audit 2023-08-28T01:30:56.474105+0000 mon.c (mon.2) 2498 : audit [DBG] from='client.? 172.21.15.6:0/2576140776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:57.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:56 smithi006 bash[33918]: audit 2023-08-28T01:30:56.474105+0000 mon.c (mon.2) 2498 : audit [DBG] from='client.? 172.21.15.6:0/2576140776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:57.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:56 smithi006 bash[27292]: audit 2023-08-28T01:30:56.474105+0000 mon.c (mon.2) 2498 : audit [DBG] from='client.? 172.21.15.6:0/2576140776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:57.992 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:57 smithi130 bash[51302]: debug 2023-08-28T01:30:57.645+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:57.650743+0000) 2023-08-28T01:30:58.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:57 smithi130 bash[27579]: cluster 2023-08-28T01:30:56.206954+0000 mgr.y (mgr.14901) 6370 : cluster [DBG] pgmap v6377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:58.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:57 smithi006 bash[33918]: cluster 2023-08-28T01:30:56.206954+0000 mgr.y (mgr.14901) 6370 : cluster [DBG] pgmap v6377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:58.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:57 smithi006 bash[27292]: cluster 2023-08-28T01:30:56.206954+0000 mgr.y (mgr.14901) 6370 : cluster [DBG] pgmap v6377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:30:59.000 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:58 smithi130 bash[51302]: debug 2023-08-28T01:30:58.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:58.651008+0000) 2023-08-28T01:30:59.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:30:59 smithi130 bash[27579]: audit 2023-08-28T01:30:58.891831+0000 mon.a (mon.0) 4240 : audit [DBG] from='client.? 172.21.15.6:0/2005108906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:59.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:30:58 smithi006 bash[33918]: audit 2023-08-28T01:30:58.891831+0000 mon.a (mon.0) 4240 : audit [DBG] from='client.? 172.21.15.6:0/2005108906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:59.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:30:58 smithi006 bash[27292]: audit 2023-08-28T01:30:58.891831+0000 mon.a (mon.0) 4240 : audit [DBG] from='client.? 172.21.15.6:0/2005108906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:00.003 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:30:59 smithi130 bash[51302]: debug 2023-08-28T01:30:59.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:59.651230+0000) 2023-08-28T01:31:00.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:00 smithi130 bash[27579]: cluster 2023-08-28T01:30:58.207614+0000 mgr.y (mgr.14901) 6371 : cluster [DBG] pgmap v6378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:00.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:00 smithi006 bash[27292]: cluster 2023-08-28T01:30:58.207614+0000 mgr.y (mgr.14901) 6371 : cluster [DBG] pgmap v6378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:00.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:00 smithi006 bash[33918]: cluster 2023-08-28T01:30:58.207614+0000 mgr.y (mgr.14901) 6371 : cluster [DBG] pgmap v6378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:01.037 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:31:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:31:00] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:31:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:00 smithi130 bash[51302]: debug 2023-08-28T01:31:00.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:00.651473+0000) 2023-08-28T01:31:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:31:01.036Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:31:01.038Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:02.007 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:01 smithi130 bash[51302]: debug 2023-08-28T01:31:01.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:01.651786+0000) 2023-08-28T01:31:02.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:02 smithi130 bash[27579]: cluster 2023-08-28T01:31:00.208735+0000 mgr.y (mgr.14901) 6372 : cluster [DBG] pgmap v6379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:02.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:02 smithi130 bash[27579]: audit 2023-08-28T01:31:01.302678+0000 mon.a (mon.0) 4241 : audit [DBG] from='client.? 172.21.15.6:0/1345149341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:02.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:02 smithi006 bash[27292]: cluster 2023-08-28T01:31:00.208735+0000 mgr.y (mgr.14901) 6372 : cluster [DBG] pgmap v6379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:02.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:02 smithi006 bash[27292]: audit 2023-08-28T01:31:01.302678+0000 mon.a (mon.0) 4241 : audit [DBG] from='client.? 172.21.15.6:0/1345149341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:02.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:02 smithi006 bash[33918]: cluster 2023-08-28T01:31:00.208735+0000 mgr.y (mgr.14901) 6372 : cluster [DBG] pgmap v6379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:02.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:02 smithi006 bash[33918]: audit 2023-08-28T01:31:01.302678+0000 mon.a (mon.0) 4241 : audit [DBG] from='client.? 172.21.15.6:0/1345149341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:31:02.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:31:02.608Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:03.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:02 smithi130 bash[51302]: debug 2023-08-28T01:31:02.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:02.652077+0000) 2023-08-28T01:31:04.012 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:03 smithi130 bash[51302]: debug 2023-08-28T01:31:03.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:03.652300+0000) 2023-08-28T01:31:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:04 smithi130 bash[27579]: cluster 2023-08-28T01:31:02.209362+0000 mgr.y (mgr.14901) 6373 : cluster [DBG] pgmap v6380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:04 smithi130 bash[27579]: audit 2023-08-28T01:31:03.713203+0000 mon.a (mon.0) 4242 : audit [DBG] from='client.? 172.21.15.6:0/701187034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:04.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:04 smithi006 bash[33918]: cluster 2023-08-28T01:31:02.209362+0000 mgr.y (mgr.14901) 6373 : cluster [DBG] pgmap v6380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:04.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:04 smithi006 bash[33918]: audit 2023-08-28T01:31:03.713203+0000 mon.a (mon.0) 4242 : audit [DBG] from='client.? 172.21.15.6:0/701187034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:04 smithi006 bash[27292]: cluster 2023-08-28T01:31:02.209362+0000 mgr.y (mgr.14901) 6373 : cluster [DBG] pgmap v6380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:04 smithi006 bash[27292]: audit 2023-08-28T01:31:03.713203+0000 mon.a (mon.0) 4242 : audit [DBG] from='client.? 172.21.15.6:0/701187034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:31:04.488Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:31:04.490Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:05.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:04 smithi130 bash[51302]: debug 2023-08-28T01:31:04.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:04.652559+0000) 2023-08-28T01:31:06.017 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:05 smithi130 bash[51302]: debug 2023-08-28T01:31:05.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:05.652865+0000) 2023-08-28T01:31:06.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:06 smithi130 bash[27579]: cluster 2023-08-28T01:31:04.210433+0000 mgr.y (mgr.14901) 6374 : cluster [DBG] pgmap v6381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:06.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:06 smithi006 bash[27292]: cluster 2023-08-28T01:31:04.210433+0000 mgr.y (mgr.14901) 6374 : cluster [DBG] pgmap v6381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:06.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:06 smithi006 bash[33918]: cluster 2023-08-28T01:31:04.210433+0000 mgr.y (mgr.14901) 6374 : cluster [DBG] pgmap v6381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:07.027 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:06 smithi130 bash[51302]: debug 2023-08-28T01:31:06.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:06.653180+0000) 2023-08-28T01:31:07.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:07 smithi130 bash[27579]: audit 2023-08-28T01:31:06.138385+0000 mon.a (mon.0) 4243 : audit [DBG] from='client.? 172.21.15.6:0/566913168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:07 smithi006 bash[27292]: audit 2023-08-28T01:31:06.138385+0000 mon.a (mon.0) 4243 : audit [DBG] from='client.? 172.21.15.6:0/566913168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:07.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:07 smithi006 bash[33918]: audit 2023-08-28T01:31:06.138385+0000 mon.a (mon.0) 4243 : audit [DBG] from='client.? 172.21.15.6:0/566913168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:08.030 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:07 smithi130 bash[51302]: debug 2023-08-28T01:31:07.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:07.653478+0000) 2023-08-28T01:31:08.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:08 smithi130 bash[27579]: cluster 2023-08-28T01:31:06.211139+0000 mgr.y (mgr.14901) 6375 : cluster [DBG] pgmap v6382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:08.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:08 smithi006 bash[33918]: cluster 2023-08-28T01:31:06.211139+0000 mgr.y (mgr.14901) 6375 : cluster [DBG] pgmap v6382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:08.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:08 smithi006 bash[27292]: cluster 2023-08-28T01:31:06.211139+0000 mgr.y (mgr.14901) 6375 : cluster [DBG] pgmap v6382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:09.038 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:08 smithi130 bash[51302]: debug 2023-08-28T01:31:08.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:08.653790+0000) 2023-08-28T01:31:09.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:09 smithi130 bash[27579]: audit 2023-08-28T01:31:08.550765+0000 mon.c (mon.2) 2499 : audit [DBG] from='client.? 172.21.15.6:0/3851496427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:09.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:09 smithi006 bash[27292]: audit 2023-08-28T01:31:08.550765+0000 mon.c (mon.2) 2499 : audit [DBG] from='client.? 172.21.15.6:0/3851496427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:09.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:09 smithi006 bash[33918]: audit 2023-08-28T01:31:08.550765+0000 mon.c (mon.2) 2499 : audit [DBG] from='client.? 172.21.15.6:0/3851496427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:10.041 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:09 smithi130 bash[51302]: debug 2023-08-28T01:31:09.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:09.654074+0000) 2023-08-28T01:31:10.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:10 smithi130 bash[27579]: cluster 2023-08-28T01:31:08.211778+0000 mgr.y (mgr.14901) 6376 : cluster [DBG] pgmap v6383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:10.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:10 smithi006 bash[27292]: cluster 2023-08-28T01:31:08.211778+0000 mgr.y (mgr.14901) 6376 : cluster [DBG] pgmap v6383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:10.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:10 smithi006 bash[33918]: cluster 2023-08-28T01:31:08.211778+0000 mgr.y (mgr.14901) 6376 : cluster [DBG] pgmap v6383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:11.037 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:31:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:31:10] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:31:11.048 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:10 smithi130 bash[51302]: debug 2023-08-28T01:31:10.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:10.654352+0000) 2023-08-28T01:31:11.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:11 smithi130 bash[27579]: audit 2023-08-28T01:31:10.968619+0000 mon.c (mon.2) 2500 : audit [DBG] from='client.? 172.21.15.6:0/1871307792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:11.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:11 smithi006 bash[27292]: audit 2023-08-28T01:31:10.968619+0000 mon.c (mon.2) 2500 : audit [DBG] from='client.? 172.21.15.6:0/1871307792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:11.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:11 smithi006 bash[33918]: audit 2023-08-28T01:31:10.968619+0000 mon.c (mon.2) 2500 : audit [DBG] from='client.? 172.21.15.6:0/1871307792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:31:11.037Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:11.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:31:11.039Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:12.053 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:11 smithi130 bash[51302]: debug 2023-08-28T01:31:11.649+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:11.654629+0000) 2023-08-28T01:31:12.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:12 smithi130 bash[27579]: cluster 2023-08-28T01:31:10.212941+0000 mgr.y (mgr.14901) 6377 : cluster [DBG] pgmap v6384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:12.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:12 smithi006 bash[27292]: cluster 2023-08-28T01:31:10.212941+0000 mgr.y (mgr.14901) 6377 : cluster [DBG] pgmap v6384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:12.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:12 smithi006 bash[33918]: cluster 2023-08-28T01:31:10.212941+0000 mgr.y (mgr.14901) 6377 : cluster [DBG] pgmap v6384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:31:12.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:31:12.608Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:13.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:12 smithi130 bash[51302]: debug 2023-08-28T01:31:12.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:12.654927+0000) 2023-08-28T01:31:14.058 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:13 smithi130 bash[51302]: debug 2023-08-28T01:31:13.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:13.655208+0000) 2023-08-28T01:31:14.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:14 smithi130 bash[27579]: cluster 2023-08-28T01:31:12.213532+0000 mgr.y (mgr.14901) 6378 : cluster [DBG] pgmap v6385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:14.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:14 smithi130 bash[27579]: audit 2023-08-28T01:31:13.384560+0000 mon.a (mon.0) 4244 : audit [DBG] from='client.? 172.21.15.6:0/639540086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:14 smithi006 bash[33918]: cluster 2023-08-28T01:31:12.213532+0000 mgr.y (mgr.14901) 6378 : cluster [DBG] pgmap v6385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:14 smithi006 bash[33918]: audit 2023-08-28T01:31:13.384560+0000 mon.a (mon.0) 4244 : audit [DBG] from='client.? 172.21.15.6:0/639540086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:14 smithi006 bash[27292]: cluster 2023-08-28T01:31:12.213532+0000 mgr.y (mgr.14901) 6378 : cluster [DBG] pgmap v6385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:14 smithi006 bash[27292]: audit 2023-08-28T01:31:13.384560+0000 mon.a (mon.0) 4244 : audit [DBG] from='client.? 172.21.15.6:0/639540086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:31:14.489Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:31:14.491Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:14 smithi130 bash[51302]: debug 2023-08-28T01:31:14.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:14.655540+0000) 2023-08-28T01:31:16.063 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:15 smithi130 bash[51302]: debug 2023-08-28T01:31:15.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:15.655849+0000) 2023-08-28T01:31:16.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:16 smithi130 bash[27579]: cluster 2023-08-28T01:31:14.214726+0000 mgr.y (mgr.14901) 6379 : cluster [DBG] pgmap v6386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:16.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:16 smithi130 bash[27579]: audit 2023-08-28T01:31:15.803988+0000 mon.c (mon.2) 2501 : audit [DBG] from='client.? 172.21.15.6:0/2167203843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:16.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:16 smithi006 bash[33918]: cluster 2023-08-28T01:31:14.214726+0000 mgr.y (mgr.14901) 6379 : cluster [DBG] pgmap v6386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:16.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:16 smithi006 bash[33918]: audit 2023-08-28T01:31:15.803988+0000 mon.c (mon.2) 2501 : audit [DBG] from='client.? 172.21.15.6:0/2167203843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:16.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:16 smithi006 bash[27292]: cluster 2023-08-28T01:31:14.214726+0000 mgr.y (mgr.14901) 6379 : cluster [DBG] pgmap v6386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:16.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:16 smithi006 bash[27292]: audit 2023-08-28T01:31:15.803988+0000 mon.c (mon.2) 2501 : audit [DBG] from='client.? 172.21.15.6:0/2167203843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:17.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:16 smithi130 bash[51302]: debug 2023-08-28T01:31:16.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:16.656128+0000) 2023-08-28T01:31:18.070 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:17 smithi130 bash[51302]: debug 2023-08-28T01:31:17.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:17.656435+0000) 2023-08-28T01:31:18.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:18 smithi130 bash[27579]: cluster 2023-08-28T01:31:16.215371+0000 mgr.y (mgr.14901) 6380 : cluster [DBG] pgmap v6387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:18.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:18 smithi006 bash[27292]: cluster 2023-08-28T01:31:16.215371+0000 mgr.y (mgr.14901) 6380 : cluster [DBG] pgmap v6387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:18.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:18 smithi006 bash[33918]: cluster 2023-08-28T01:31:16.215371+0000 mgr.y (mgr.14901) 6380 : cluster [DBG] pgmap v6387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:19.083 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:18 smithi130 bash[51302]: debug 2023-08-28T01:31:18.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:18.656713+0000) 2023-08-28T01:31:19.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:19 smithi130 bash[27579]: audit 2023-08-28T01:31:18.214482+0000 mon.a (mon.0) 4245 : audit [DBG] from='client.? 172.21.15.6:0/213992714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:19.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:19 smithi006 bash[27292]: audit 2023-08-28T01:31:18.214482+0000 mon.a (mon.0) 4245 : audit [DBG] from='client.? 172.21.15.6:0/213992714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:19.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:19 smithi006 bash[33918]: audit 2023-08-28T01:31:18.214482+0000 mon.a (mon.0) 4245 : audit [DBG] from='client.? 172.21.15.6:0/213992714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:20.086 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:19 smithi130 bash[51302]: debug 2023-08-28T01:31:19.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:19.656911+0000) 2023-08-28T01:31:20.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:20 smithi130 bash[27579]: cluster 2023-08-28T01:31:18.216125+0000 mgr.y (mgr.14901) 6381 : cluster [DBG] pgmap v6388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:20.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:20 smithi006 bash[27292]: cluster 2023-08-28T01:31:18.216125+0000 mgr.y (mgr.14901) 6381 : cluster [DBG] pgmap v6388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:20.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:20 smithi006 bash[33918]: cluster 2023-08-28T01:31:18.216125+0000 mgr.y (mgr.14901) 6381 : cluster [DBG] pgmap v6388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:21.038 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:31:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:31:20] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T01:31:21.096 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:20 smithi130 bash[51302]: debug 2023-08-28T01:31:20.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:20.657241+0000) 2023-08-28T01:31:21.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:21 smithi130 bash[27579]: audit 2023-08-28T01:31:20.633868+0000 mon.c (mon.2) 2502 : audit [DBG] from='client.? 172.21.15.6:0/2418152710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:21.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:21 smithi006 bash[27292]: audit 2023-08-28T01:31:20.633868+0000 mon.c (mon.2) 2502 : audit [DBG] from='client.? 172.21.15.6:0/2418152710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:21.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:21 smithi006 bash[33918]: audit 2023-08-28T01:31:20.633868+0000 mon.c (mon.2) 2502 : audit [DBG] from='client.? 172.21.15.6:0/2418152710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:21.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:31:21.037Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:21.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:31:21.039Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:22.108 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:21 smithi130 bash[51302]: debug 2023-08-28T01:31:21.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:21.657553+0000) 2023-08-28T01:31:22.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:22 smithi130 bash[27579]: cluster 2023-08-28T01:31:20.217326+0000 mgr.y (mgr.14901) 6382 : cluster [DBG] pgmap v6389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:22.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:22 smithi006 bash[33918]: cluster 2023-08-28T01:31:20.217326+0000 mgr.y (mgr.14901) 6382 : cluster [DBG] pgmap v6389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:22.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:22 smithi006 bash[27292]: cluster 2023-08-28T01:31:20.217326+0000 mgr.y (mgr.14901) 6382 : cluster [DBG] pgmap v6389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:31:22.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:31:22.609Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:23.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:22 smithi130 bash[51302]: debug 2023-08-28T01:31:22.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:22.657835+0000) 2023-08-28T01:31:23.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:23 smithi006 bash[33918]: audit 2023-08-28T01:31:23.043893+0000 mon.c (mon.2) 2503 : audit [DBG] from='client.? 172.21.15.6:0/1966494160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:23.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:23 smithi006 bash[27292]: audit 2023-08-28T01:31:23.043893+0000 mon.c (mon.2) 2503 : audit [DBG] from='client.? 172.21.15.6:0/1966494160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:23.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:23 smithi130 bash[27579]: audit 2023-08-28T01:31:23.043893+0000 mon.c (mon.2) 2503 : audit [DBG] from='client.? 172.21.15.6:0/1966494160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:24.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:23 smithi130 bash[51302]: debug 2023-08-28T01:31:23.653+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:23.658109+0000) 2023-08-28T01:31:24.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:24 smithi006 bash[33918]: cluster 2023-08-28T01:31:22.218072+0000 mgr.y (mgr.14901) 6383 : cluster [DBG] pgmap v6390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:24 smithi006 bash[27292]: cluster 2023-08-28T01:31:22.218072+0000 mgr.y (mgr.14901) 6383 : cluster [DBG] pgmap v6390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:24 smithi130 bash[27579]: cluster 2023-08-28T01:31:22.218072+0000 mgr.y (mgr.14901) 6383 : cluster [DBG] pgmap v6390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:31:24.490Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:31:24.492Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:25.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:24 smithi130 bash[51302]: debug 2023-08-28T01:31:24.657+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:24.658436+0000) 2023-08-28T01:31:26.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:25 smithi130 bash[51302]: debug 2023-08-28T01:31:25.657+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:25.658709+0000) 2023-08-28T01:31:26.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:26 smithi006 bash[27292]: cluster 2023-08-28T01:31:24.219321+0000 mgr.y (mgr.14901) 6384 : cluster [DBG] pgmap v6391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:26.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:26 smithi006 bash[27292]: audit 2023-08-28T01:31:25.460706+0000 mon.a (mon.0) 4246 : audit [DBG] from='client.? 172.21.15.6:0/2281258000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:26.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:26 smithi006 bash[33918]: cluster 2023-08-28T01:31:24.219321+0000 mgr.y (mgr.14901) 6384 : cluster [DBG] pgmap v6391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:26.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:26 smithi006 bash[33918]: audit 2023-08-28T01:31:25.460706+0000 mon.a (mon.0) 4246 : audit [DBG] from='client.? 172.21.15.6:0/2281258000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:26 smithi130 bash[27579]: cluster 2023-08-28T01:31:24.219321+0000 mgr.y (mgr.14901) 6384 : cluster [DBG] pgmap v6391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:26 smithi130 bash[27579]: audit 2023-08-28T01:31:25.460706+0000 mon.a (mon.0) 4246 : audit [DBG] from='client.? 172.21.15.6:0/2281258000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:27.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:26 smithi130 bash[51302]: debug 2023-08-28T01:31:26.657+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:26.659048+0000) 2023-08-28T01:31:28.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:27 smithi130 bash[51302]: debug 2023-08-28T01:31:27.657+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:27.659247+0000) 2023-08-28T01:31:28.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:28 smithi006 bash[33918]: cluster 2023-08-28T01:31:26.220087+0000 mgr.y (mgr.14901) 6385 : cluster [DBG] pgmap v6392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:28.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:28 smithi006 bash[33918]: audit 2023-08-28T01:31:27.870438+0000 mon.a (mon.0) 4247 : audit [DBG] from='client.? 172.21.15.6:0/2810387638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:28.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:28 smithi006 bash[27292]: cluster 2023-08-28T01:31:26.220087+0000 mgr.y (mgr.14901) 6385 : cluster [DBG] pgmap v6392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:28.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:28 smithi006 bash[27292]: audit 2023-08-28T01:31:27.870438+0000 mon.a (mon.0) 4247 : audit [DBG] from='client.? 172.21.15.6:0/2810387638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:28 smithi130 bash[27579]: cluster 2023-08-28T01:31:26.220087+0000 mgr.y (mgr.14901) 6385 : cluster [DBG] pgmap v6392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:28 smithi130 bash[27579]: audit 2023-08-28T01:31:27.870438+0000 mon.a (mon.0) 4247 : audit [DBG] from='client.? 172.21.15.6:0/2810387638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:28 smithi130 bash[51302]: debug 2023-08-28T01:31:28.657+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:28.659445+0000) 2023-08-28T01:31:30.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:29 smithi130 bash[51302]: debug 2023-08-28T01:31:29.657+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:29.659735+0000) 2023-08-28T01:31:30.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:30 smithi006 bash[27292]: cluster 2023-08-28T01:31:28.220751+0000 mgr.y (mgr.14901) 6386 : cluster [DBG] pgmap v6393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:30.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:30 smithi006 bash[33918]: cluster 2023-08-28T01:31:28.220751+0000 mgr.y (mgr.14901) 6386 : cluster [DBG] pgmap v6393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:30 smithi130 bash[27579]: cluster 2023-08-28T01:31:28.220751+0000 mgr.y (mgr.14901) 6386 : cluster [DBG] pgmap v6393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:31.038 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:31:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:31:30] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:31:31.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:30 smithi130 bash[51302]: debug 2023-08-28T01:31:30.657+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:30.660029+0000) 2023-08-28T01:31:31.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:31 smithi006 bash[27292]: audit 2023-08-28T01:31:30.285374+0000 mon.c (mon.2) 2504 : audit [DBG] from='client.? 172.21.15.6:0/633253065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:31.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:31 smithi006 bash[33918]: audit 2023-08-28T01:31:30.285374+0000 mon.c (mon.2) 2504 : audit [DBG] from='client.? 172.21.15.6:0/633253065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:31:31.038Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:31:31.040Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:31.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:31 smithi130 bash[27579]: audit 2023-08-28T01:31:30.285374+0000 mon.c (mon.2) 2504 : audit [DBG] from='client.? 172.21.15.6:0/633253065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:32.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:31 smithi130 bash[51302]: debug 2023-08-28T01:31:31.657+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:31.660320+0000) 2023-08-28T01:31:32.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:32 smithi006 bash[27292]: cluster 2023-08-28T01:31:30.221850+0000 mgr.y (mgr.14901) 6387 : cluster [DBG] pgmap v6394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:32.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:32 smithi006 bash[33918]: cluster 2023-08-28T01:31:30.221850+0000 mgr.y (mgr.14901) 6387 : cluster [DBG] pgmap v6394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:32 smithi130 bash[27579]: cluster 2023-08-28T01:31:30.221850+0000 mgr.y (mgr.14901) 6387 : cluster [DBG] pgmap v6394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:31:32.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:31:32.610Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:33.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:32 smithi130 bash[51302]: debug 2023-08-28T01:31:32.657+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:32.660594+0000) 2023-08-28T01:31:33.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:33 smithi006 bash[33918]: audit 2023-08-28T01:31:32.703047+0000 mon.a (mon.0) 4248 : audit [DBG] from='client.? 172.21.15.6:0/2122032619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:33.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:33 smithi006 bash[27292]: audit 2023-08-28T01:31:32.703047+0000 mon.a (mon.0) 4248 : audit [DBG] from='client.? 172.21.15.6:0/2122032619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:33.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:33 smithi130 bash[27579]: audit 2023-08-28T01:31:32.703047+0000 mon.a (mon.0) 4248 : audit [DBG] from='client.? 172.21.15.6:0/2122032619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:34.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:33 smithi130 bash[51302]: debug 2023-08-28T01:31:33.657+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:33.660921+0000) 2023-08-28T01:31:34.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:34 smithi006 bash[27292]: cluster 2023-08-28T01:31:32.222608+0000 mgr.y (mgr.14901) 6388 : cluster [DBG] pgmap v6395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:34 smithi006 bash[33918]: cluster 2023-08-28T01:31:32.222608+0000 mgr.y (mgr.14901) 6388 : cluster [DBG] pgmap v6395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:34 smithi130 bash[27579]: cluster 2023-08-28T01:31:32.222608+0000 mgr.y (mgr.14901) 6388 : cluster [DBG] pgmap v6395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:31:34.490Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:31:34.493Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:35.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:34 smithi130 bash[51302]: debug 2023-08-28T01:31:34.657+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:34.661219+0000) 2023-08-28T01:31:35.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:35 smithi006 bash[33918]: audit 2023-08-28T01:31:35.121738+0000 mon.a (mon.0) 4249 : audit [DBG] from='client.? 172.21.15.6:0/1848442107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:35.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:35 smithi006 bash[27292]: audit 2023-08-28T01:31:35.121738+0000 mon.a (mon.0) 4249 : audit [DBG] from='client.? 172.21.15.6:0/1848442107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:35.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:35 smithi130 bash[27579]: audit 2023-08-28T01:31:35.121738+0000 mon.a (mon.0) 4249 : audit [DBG] from='client.? 172.21.15.6:0/1848442107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:36.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:35 smithi130 bash[51302]: debug 2023-08-28T01:31:35.656+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:35.661504+0000) 2023-08-28T01:31:36.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:36 smithi006 bash[33918]: cluster 2023-08-28T01:31:34.223752+0000 mgr.y (mgr.14901) 6389 : cluster [DBG] pgmap v6396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:36.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:36 smithi006 bash[27292]: cluster 2023-08-28T01:31:34.223752+0000 mgr.y (mgr.14901) 6389 : cluster [DBG] pgmap v6396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:36 smithi130 bash[27579]: cluster 2023-08-28T01:31:34.223752+0000 mgr.y (mgr.14901) 6389 : cluster [DBG] pgmap v6396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:37.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:36 smithi130 bash[51302]: debug 2023-08-28T01:31:36.656+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:36.661843+0000) 2023-08-28T01:31:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:37 smithi130 bash[51302]: debug 2023-08-28T01:31:37.656+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:37.662116+0000) 2023-08-28T01:31:38.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:38 smithi006 bash[27292]: cluster 2023-08-28T01:31:36.224439+0000 mgr.y (mgr.14901) 6390 : cluster [DBG] pgmap v6397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:38.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:38 smithi006 bash[27292]: audit 2023-08-28T01:31:37.536719+0000 mon.c (mon.2) 2505 : audit [DBG] from='client.? 172.21.15.6:0/1253444705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:38.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:38 smithi006 bash[33918]: cluster 2023-08-28T01:31:36.224439+0000 mgr.y (mgr.14901) 6390 : cluster [DBG] pgmap v6397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:38.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:38 smithi006 bash[33918]: audit 2023-08-28T01:31:37.536719+0000 mon.c (mon.2) 2505 : audit [DBG] from='client.? 172.21.15.6:0/1253444705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:38 smithi130 bash[27579]: cluster 2023-08-28T01:31:36.224439+0000 mgr.y (mgr.14901) 6390 : cluster [DBG] pgmap v6397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:38 smithi130 bash[27579]: audit 2023-08-28T01:31:37.536719+0000 mon.c (mon.2) 2505 : audit [DBG] from='client.? 172.21.15.6:0/1253444705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:39.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:38 smithi130 bash[51302]: debug 2023-08-28T01:31:38.656+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:38.662421+0000) 2023-08-28T01:31:40.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:39 smithi130 bash[51302]: debug 2023-08-28T01:31:39.656+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:39.662742+0000) 2023-08-28T01:31:40.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:40 smithi006 bash[27292]: cluster 2023-08-28T01:31:38.225174+0000 mgr.y (mgr.14901) 6391 : cluster [DBG] pgmap v6398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:40.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:40 smithi006 bash[27292]: audit 2023-08-28T01:31:39.949845+0000 mon.a (mon.0) 4250 : audit [DBG] from='client.? 172.21.15.6:0/2777276979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:40 smithi006 bash[33918]: cluster 2023-08-28T01:31:38.225174+0000 mgr.y (mgr.14901) 6391 : cluster [DBG] pgmap v6398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:40 smithi006 bash[33918]: audit 2023-08-28T01:31:39.949845+0000 mon.a (mon.0) 4250 : audit [DBG] from='client.? 172.21.15.6:0/2777276979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:40 smithi130 bash[27579]: cluster 2023-08-28T01:31:38.225174+0000 mgr.y (mgr.14901) 6391 : cluster [DBG] pgmap v6398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:40 smithi130 bash[27579]: audit 2023-08-28T01:31:39.949845+0000 mon.a (mon.0) 4250 : audit [DBG] from='client.? 172.21.15.6:0/2777276979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:41.039 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:31:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:31:40] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:31:41.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:40 smithi130 bash[51302]: debug 2023-08-28T01:31:40.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:40.663104+0000) 2023-08-28T01:31:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:31:41.039Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:31:41.041Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:41 smithi130 bash[51302]: debug 2023-08-28T01:31:41.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:41.663451+0000) 2023-08-28T01:31:42.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:42 smithi006 bash[33918]: cluster 2023-08-28T01:31:40.226249+0000 mgr.y (mgr.14901) 6392 : cluster [DBG] pgmap v6399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:42.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:42 smithi006 bash[27292]: cluster 2023-08-28T01:31:40.226249+0000 mgr.y (mgr.14901) 6392 : cluster [DBG] pgmap v6399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:42 smithi130 bash[27579]: cluster 2023-08-28T01:31:40.226249+0000 mgr.y (mgr.14901) 6392 : cluster [DBG] pgmap v6399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:31:42.609Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:31:42.611Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:43.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:42 smithi130 bash[51302]: debug 2023-08-28T01:31:42.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:42.663825+0000) 2023-08-28T01:31:43.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:43 smithi006 bash[33918]: audit 2023-08-28T01:31:42.365313+0000 mon.a (mon.0) 4251 : audit [DBG] from='client.? 172.21.15.6:0/3094773620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:43.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:43 smithi006 bash[27292]: audit 2023-08-28T01:31:42.365313+0000 mon.a (mon.0) 4251 : audit [DBG] from='client.? 172.21.15.6:0/3094773620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:43.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:43 smithi130 bash[27579]: audit 2023-08-28T01:31:42.365313+0000 mon.a (mon.0) 4251 : audit [DBG] from='client.? 172.21.15.6:0/3094773620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:44.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:43 smithi130 bash[51302]: debug 2023-08-28T01:31:43.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:43.664202+0000) 2023-08-28T01:31:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:44 smithi006 bash[33918]: cluster 2023-08-28T01:31:42.226856+0000 mgr.y (mgr.14901) 6393 : cluster [DBG] pgmap v6400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:44.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:44 smithi006 bash[27292]: cluster 2023-08-28T01:31:42.226856+0000 mgr.y (mgr.14901) 6393 : cluster [DBG] pgmap v6400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:44 smithi130 bash[27579]: cluster 2023-08-28T01:31:42.226856+0000 mgr.y (mgr.14901) 6393 : cluster [DBG] pgmap v6400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:31:44.491Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:31:44.493Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:45.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:44 smithi130 bash[51302]: debug 2023-08-28T01:31:44.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:44.664552+0000) 2023-08-28T01:31:45.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:45 smithi006 bash[27292]: audit 2023-08-28T01:31:44.777677+0000 mon.a (mon.0) 4252 : audit [DBG] from='client.? 172.21.15.6:0/942980386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:45.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:45 smithi006 bash[33918]: audit 2023-08-28T01:31:44.777677+0000 mon.a (mon.0) 4252 : audit [DBG] from='client.? 172.21.15.6:0/942980386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:45.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:45 smithi130 bash[27579]: audit 2023-08-28T01:31:44.777677+0000 mon.a (mon.0) 4252 : audit [DBG] from='client.? 172.21.15.6:0/942980386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:45 smithi130 bash[51302]: debug 2023-08-28T01:31:45.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:45.664941+0000) 2023-08-28T01:31:46.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:46 smithi006 bash[27292]: cluster 2023-08-28T01:31:44.227966+0000 mgr.y (mgr.14901) 6394 : cluster [DBG] pgmap v6401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:46.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:46 smithi006 bash[33918]: cluster 2023-08-28T01:31:44.227966+0000 mgr.y (mgr.14901) 6394 : cluster [DBG] pgmap v6401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:46 smithi130 bash[27579]: cluster 2023-08-28T01:31:44.227966+0000 mgr.y (mgr.14901) 6394 : cluster [DBG] pgmap v6401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:46 smithi130 bash[51302]: debug 2023-08-28T01:31:46.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:46.665239+0000) 2023-08-28T01:31:47.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:47 smithi006 bash[33918]: audit 2023-08-28T01:31:46.485149+0000 mon.b (mon.1) 729 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:31:47.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:47 smithi006 bash[33918]: audit 2023-08-28T01:31:47.191820+0000 mon.c (mon.2) 2506 : audit [DBG] from='client.? 172.21.15.6:0/3462578664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:47.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:47 smithi006 bash[27292]: audit 2023-08-28T01:31:46.485149+0000 mon.b (mon.1) 729 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:31:47.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:47 smithi006 bash[27292]: audit 2023-08-28T01:31:47.191820+0000 mon.c (mon.2) 2506 : audit [DBG] from='client.? 172.21.15.6:0/3462578664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:47.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:47 smithi130 bash[27579]: audit 2023-08-28T01:31:46.485149+0000 mon.b (mon.1) 729 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:31:47.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:47 smithi130 bash[27579]: audit 2023-08-28T01:31:47.191820+0000 mon.c (mon.2) 2506 : audit [DBG] from='client.? 172.21.15.6:0/3462578664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:47 smithi130 bash[51302]: debug 2023-08-28T01:31:47.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:47.665570+0000) 2023-08-28T01:31:48.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:48 smithi006 bash[33918]: cluster 2023-08-28T01:31:46.228625+0000 mgr.y (mgr.14901) 6395 : cluster [DBG] pgmap v6402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:48.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:48 smithi006 bash[27292]: cluster 2023-08-28T01:31:46.228625+0000 mgr.y (mgr.14901) 6395 : cluster [DBG] pgmap v6402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:48 smithi130 bash[27579]: cluster 2023-08-28T01:31:46.228625+0000 mgr.y (mgr.14901) 6395 : cluster [DBG] pgmap v6402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:49.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:48 smithi130 bash[51302]: debug 2023-08-28T01:31:48.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:48.665966+0000) 2023-08-28T01:31:50.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:49 smithi130 bash[51302]: debug 2023-08-28T01:31:49.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:49.666303+0000) 2023-08-28T01:31:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:50 smithi130 bash[27579]: cluster 2023-08-28T01:31:48.229230+0000 mgr.y (mgr.14901) 6396 : cluster [DBG] pgmap v6403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:50 smithi130 bash[27579]: audit 2023-08-28T01:31:49.613176+0000 mon.a (mon.0) 4253 : audit [DBG] from='client.? 172.21.15.6:0/442267038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:50.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:50 smithi006 bash[33918]: cluster 2023-08-28T01:31:48.229230+0000 mgr.y (mgr.14901) 6396 : cluster [DBG] pgmap v6403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:50.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:50 smithi006 bash[33918]: audit 2023-08-28T01:31:49.613176+0000 mon.a (mon.0) 4253 : audit [DBG] from='client.? 172.21.15.6:0/442267038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:50 smithi006 bash[27292]: cluster 2023-08-28T01:31:48.229230+0000 mgr.y (mgr.14901) 6396 : cluster [DBG] pgmap v6403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:50 smithi006 bash[27292]: audit 2023-08-28T01:31:49.613176+0000 mon.a (mon.0) 4253 : audit [DBG] from='client.? 172.21.15.6:0/442267038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:51.041 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:31:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:31:50] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T01:31:51.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:50 smithi130 bash[51302]: debug 2023-08-28T01:31:50.660+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:50.666645+0000) 2023-08-28T01:31:51.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:51 smithi006 bash[33918]: cluster 2023-08-28T01:31:50.230277+0000 mgr.y (mgr.14901) 6397 : cluster [DBG] pgmap v6404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:51.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:51 smithi006 bash[27292]: cluster 2023-08-28T01:31:50.230277+0000 mgr.y (mgr.14901) 6397 : cluster [DBG] pgmap v6404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:31:51.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:31:51.042Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:51.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:51 smithi130 bash[27579]: cluster 2023-08-28T01:31:50.230277+0000 mgr.y (mgr.14901) 6397 : cluster [DBG] pgmap v6404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:52.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:51 smithi130 bash[51302]: debug 2023-08-28T01:31:51.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:51.666930+0000) 2023-08-28T01:31:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:31:52.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:31:52.612Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:52.991 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:52 smithi130 bash[51302]: debug 2023-08-28T01:31:52.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:52.667134+0000) 2023-08-28T01:31:53.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:52 smithi130 bash[27579]: audit 2023-08-28T01:31:52.037483+0000 mon.a (mon.0) 4254 : audit [DBG] from='client.? 172.21.15.6:0/2746553456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:53.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:52 smithi130 bash[27579]: audit 2023-08-28T01:31:52.366215+0000 mon.a (mon.0) 4255 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:31:53.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:52 smithi130 bash[27579]: audit 2023-08-28T01:31:52.799748+0000 mon.a (mon.0) 4256 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:31:53.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:52 smithi006 bash[33918]: audit 2023-08-28T01:31:52.037483+0000 mon.a (mon.0) 4254 : audit [DBG] from='client.? 172.21.15.6:0/2746553456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:53.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:52 smithi006 bash[33918]: audit 2023-08-28T01:31:52.366215+0000 mon.a (mon.0) 4255 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:31:53.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:52 smithi006 bash[33918]: audit 2023-08-28T01:31:52.799748+0000 mon.a (mon.0) 4256 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:31:53.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:52 smithi006 bash[27292]: audit 2023-08-28T01:31:52.037483+0000 mon.a (mon.0) 4254 : audit [DBG] from='client.? 172.21.15.6:0/2746553456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:53.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:52 smithi006 bash[27292]: audit 2023-08-28T01:31:52.366215+0000 mon.a (mon.0) 4255 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:31:53.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:52 smithi006 bash[27292]: audit 2023-08-28T01:31:52.799748+0000 mon.a (mon.0) 4256 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:31:53.997 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:53 smithi130 bash[51302]: debug 2023-08-28T01:31:53.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:53.667433+0000) 2023-08-28T01:31:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:53 smithi130 bash[27579]: cluster 2023-08-28T01:31:52.230894+0000 mgr.y (mgr.14901) 6398 : cluster [DBG] pgmap v6405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:53 smithi130 bash[27579]: audit 2023-08-28T01:31:53.070989+0000 mon.b (mon.1) 730 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:31:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:53 smithi130 bash[27579]: audit 2023-08-28T01:31:53.073344+0000 mon.b (mon.1) 731 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:31:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:54 smithi130 bash[27579]: audit 2023-08-28T01:31:53.083162+0000 mon.a (mon.0) 4257 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:31:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:53 smithi006 bash[27292]: cluster 2023-08-28T01:31:52.230894+0000 mgr.y (mgr.14901) 6398 : cluster [DBG] pgmap v6405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:53 smithi006 bash[27292]: audit 2023-08-28T01:31:53.070989+0000 mon.b (mon.1) 730 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:31:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:53 smithi006 bash[27292]: audit 2023-08-28T01:31:53.073344+0000 mon.b (mon.1) 731 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:31:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:53 smithi006 bash[27292]: audit 2023-08-28T01:31:53.083162+0000 mon.a (mon.0) 4257 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:31:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:53 smithi006 bash[33918]: cluster 2023-08-28T01:31:52.230894+0000 mgr.y (mgr.14901) 6398 : cluster [DBG] pgmap v6405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:53 smithi006 bash[33918]: audit 2023-08-28T01:31:53.070989+0000 mon.b (mon.1) 730 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:31:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:53 smithi006 bash[33918]: audit 2023-08-28T01:31:53.073344+0000 mon.b (mon.1) 731 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:31:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:53 smithi006 bash[33918]: audit 2023-08-28T01:31:53.083162+0000 mon.a (mon.0) 4257 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:31:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:31:54.492Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:31:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:31:54.493Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:31:55.007 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:54 smithi130 bash[51302]: debug 2023-08-28T01:31:54.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:54.667774+0000) 2023-08-28T01:31:55.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:55 smithi130 bash[27579]: audit 2023-08-28T01:31:54.459594+0000 mon.a (mon.0) 4258 : audit [DBG] from='client.? 172.21.15.6:0/2372151372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:55.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:55 smithi006 bash[33918]: audit 2023-08-28T01:31:54.459594+0000 mon.a (mon.0) 4258 : audit [DBG] from='client.? 172.21.15.6:0/2372151372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:55.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:55 smithi006 bash[27292]: audit 2023-08-28T01:31:54.459594+0000 mon.a (mon.0) 4258 : audit [DBG] from='client.? 172.21.15.6:0/2372151372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:56.009 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:55 smithi130 bash[51302]: debug 2023-08-28T01:31:55.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:55.668088+0000) 2023-08-28T01:31:56.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:56 smithi130 bash[27579]: cluster 2023-08-28T01:31:54.231997+0000 mgr.y (mgr.14901) 6399 : cluster [DBG] pgmap v6406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:56.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:56 smithi006 bash[27292]: cluster 2023-08-28T01:31:54.231997+0000 mgr.y (mgr.14901) 6399 : cluster [DBG] pgmap v6406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:56.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:56 smithi006 bash[33918]: cluster 2023-08-28T01:31:54.231997+0000 mgr.y (mgr.14901) 6399 : cluster [DBG] pgmap v6406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:57.015 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:56 smithi130 bash[51302]: debug 2023-08-28T01:31:56.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:56.668420+0000) 2023-08-28T01:31:57.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:57 smithi130 bash[27579]: audit 2023-08-28T01:31:56.874339+0000 mon.c (mon.2) 2507 : audit [DBG] from='client.? 172.21.15.6:0/1667816313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:57.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:57 smithi006 bash[27292]: audit 2023-08-28T01:31:56.874339+0000 mon.c (mon.2) 2507 : audit [DBG] from='client.? 172.21.15.6:0/1667816313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:57.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:57 smithi006 bash[33918]: audit 2023-08-28T01:31:56.874339+0000 mon.c (mon.2) 2507 : audit [DBG] from='client.? 172.21.15.6:0/1667816313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:58.018 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:57 smithi130 bash[51302]: debug 2023-08-28T01:31:57.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:57.668690+0000) 2023-08-28T01:31:58.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:31:58 smithi130 bash[27579]: cluster 2023-08-28T01:31:56.232641+0000 mgr.y (mgr.14901) 6400 : cluster [DBG] pgmap v6407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:58.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:31:58 smithi006 bash[33918]: cluster 2023-08-28T01:31:56.232641+0000 mgr.y (mgr.14901) 6400 : cluster [DBG] pgmap v6407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:58.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:31:58 smithi006 bash[27292]: cluster 2023-08-28T01:31:56.232641+0000 mgr.y (mgr.14901) 6400 : cluster [DBG] pgmap v6407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:31:59.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:58 smithi130 bash[51302]: debug 2023-08-28T01:31:58.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:58.668996+0000) 2023-08-28T01:32:00.023 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:31:59 smithi130 bash[51302]: debug 2023-08-28T01:31:59.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:59.669293+0000) 2023-08-28T01:32:00.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:00 smithi130 bash[27579]: cluster 2023-08-28T01:31:58.233312+0000 mgr.y (mgr.14901) 6401 : cluster [DBG] pgmap v6408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:00.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:00 smithi130 bash[27579]: audit 2023-08-28T01:31:59.305090+0000 mon.c (mon.2) 2508 : audit [DBG] from='client.? 172.21.15.6:0/3182053215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:00.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:00 smithi006 bash[33918]: cluster 2023-08-28T01:31:58.233312+0000 mgr.y (mgr.14901) 6401 : cluster [DBG] pgmap v6408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:00.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:00 smithi006 bash[33918]: audit 2023-08-28T01:31:59.305090+0000 mon.c (mon.2) 2508 : audit [DBG] from='client.? 172.21.15.6:0/3182053215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:00.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:00 smithi006 bash[27292]: cluster 2023-08-28T01:31:58.233312+0000 mgr.y (mgr.14901) 6401 : cluster [DBG] pgmap v6408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:00.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:00 smithi006 bash[27292]: audit 2023-08-28T01:31:59.305090+0000 mon.c (mon.2) 2508 : audit [DBG] from='client.? 172.21.15.6:0/3182053215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:01.041 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:32:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:32:00] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:32:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:00 smithi130 bash[51302]: debug 2023-08-28T01:32:00.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:00.669587+0000) 2023-08-28T01:32:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:32:01.041Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:32:01.043Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:02.029 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:01 smithi130 bash[51302]: debug 2023-08-28T01:32:01.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:01.669868+0000) 2023-08-28T01:32:02.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:02 smithi130 bash[27579]: cluster 2023-08-28T01:32:00.234394+0000 mgr.y (mgr.14901) 6402 : cluster [DBG] pgmap v6409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:02.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:02 smithi130 bash[27579]: audit 2023-08-28T01:32:01.707558+0000 mon.a (mon.0) 4259 : audit [DBG] from='client.? 172.21.15.6:0/1290804721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:02.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:02 smithi006 bash[27292]: cluster 2023-08-28T01:32:00.234394+0000 mgr.y (mgr.14901) 6402 : cluster [DBG] pgmap v6409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:02.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:02 smithi006 bash[27292]: audit 2023-08-28T01:32:01.707558+0000 mon.a (mon.0) 4259 : audit [DBG] from='client.? 172.21.15.6:0/1290804721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:02.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:02 smithi006 bash[33918]: cluster 2023-08-28T01:32:00.234394+0000 mgr.y (mgr.14901) 6402 : cluster [DBG] pgmap v6409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:02.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:02 smithi006 bash[33918]: audit 2023-08-28T01:32:01.707558+0000 mon.a (mon.0) 4259 : audit [DBG] from='client.? 172.21.15.6:0/1290804721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:32:02.611Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:32:02.612Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:03.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:02 smithi130 bash[51302]: debug 2023-08-28T01:32:02.664+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:02.670188+0000) 2023-08-28T01:32:04.032 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:03 smithi130 bash[51302]: debug 2023-08-28T01:32:03.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:03.670461+0000) 2023-08-28T01:32:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:04 smithi130 bash[27579]: cluster 2023-08-28T01:32:02.234982+0000 mgr.y (mgr.14901) 6403 : cluster [DBG] pgmap v6410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:04.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:04 smithi006 bash[33918]: cluster 2023-08-28T01:32:02.234982+0000 mgr.y (mgr.14901) 6403 : cluster [DBG] pgmap v6410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:04 smithi006 bash[27292]: cluster 2023-08-28T01:32:02.234982+0000 mgr.y (mgr.14901) 6403 : cluster [DBG] pgmap v6410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:32:04.492Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:32:04.495Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:05.042 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:04 smithi130 bash[51302]: debug 2023-08-28T01:32:04.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:04.670741+0000) 2023-08-28T01:32:05.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:05 smithi130 bash[27579]: audit 2023-08-28T01:32:04.133873+0000 mon.a (mon.0) 4260 : audit [DBG] from='client.? 172.21.15.6:0/4076930224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:05.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:05 smithi006 bash[33918]: audit 2023-08-28T01:32:04.133873+0000 mon.a (mon.0) 4260 : audit [DBG] from='client.? 172.21.15.6:0/4076930224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:05.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:05 smithi006 bash[27292]: audit 2023-08-28T01:32:04.133873+0000 mon.a (mon.0) 4260 : audit [DBG] from='client.? 172.21.15.6:0/4076930224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:06.045 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:05 smithi130 bash[51302]: debug 2023-08-28T01:32:05.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:05.671035+0000) 2023-08-28T01:32:06.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:06 smithi130 bash[27579]: cluster 2023-08-28T01:32:04.236079+0000 mgr.y (mgr.14901) 6404 : cluster [DBG] pgmap v6411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:06.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:06 smithi006 bash[27292]: cluster 2023-08-28T01:32:04.236079+0000 mgr.y (mgr.14901) 6404 : cluster [DBG] pgmap v6411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:06.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:06 smithi006 bash[33918]: cluster 2023-08-28T01:32:04.236079+0000 mgr.y (mgr.14901) 6404 : cluster [DBG] pgmap v6411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:07.056 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:06 smithi130 bash[51302]: debug 2023-08-28T01:32:06.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:06.671273+0000) 2023-08-28T01:32:07.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:07 smithi130 bash[27579]: audit 2023-08-28T01:32:06.547284+0000 mon.c (mon.2) 2509 : audit [DBG] from='client.? 172.21.15.6:0/123444037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:07.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:07 smithi006 bash[33918]: audit 2023-08-28T01:32:06.547284+0000 mon.c (mon.2) 2509 : audit [DBG] from='client.? 172.21.15.6:0/123444037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:07 smithi006 bash[27292]: audit 2023-08-28T01:32:06.547284+0000 mon.c (mon.2) 2509 : audit [DBG] from='client.? 172.21.15.6:0/123444037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:08.059 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:07 smithi130 bash[51302]: debug 2023-08-28T01:32:07.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:07.671551+0000) 2023-08-28T01:32:08.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:08 smithi130 bash[27579]: cluster 2023-08-28T01:32:06.236710+0000 mgr.y (mgr.14901) 6405 : cluster [DBG] pgmap v6412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:08.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:08 smithi006 bash[27292]: cluster 2023-08-28T01:32:06.236710+0000 mgr.y (mgr.14901) 6405 : cluster [DBG] pgmap v6412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:08.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:08 smithi006 bash[33918]: cluster 2023-08-28T01:32:06.236710+0000 mgr.y (mgr.14901) 6405 : cluster [DBG] pgmap v6412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:09.066 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:08 smithi130 bash[51302]: debug 2023-08-28T01:32:08.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:08.671834+0000) 2023-08-28T01:32:09.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:09 smithi130 bash[27579]: audit 2023-08-28T01:32:08.961283+0000 mon.c (mon.2) 2510 : audit [DBG] from='client.? 172.21.15.6:0/4250887415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:09.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:09 smithi006 bash[27292]: audit 2023-08-28T01:32:08.961283+0000 mon.c (mon.2) 2510 : audit [DBG] from='client.? 172.21.15.6:0/4250887415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:09.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:09 smithi006 bash[33918]: audit 2023-08-28T01:32:08.961283+0000 mon.c (mon.2) 2510 : audit [DBG] from='client.? 172.21.15.6:0/4250887415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:10.070 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:09 smithi130 bash[51302]: debug 2023-08-28T01:32:09.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:09.672107+0000) 2023-08-28T01:32:10.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:10 smithi130 bash[27579]: cluster 2023-08-28T01:32:08.237356+0000 mgr.y (mgr.14901) 6406 : cluster [DBG] pgmap v6413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:10.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:10 smithi006 bash[27292]: cluster 2023-08-28T01:32:08.237356+0000 mgr.y (mgr.14901) 6406 : cluster [DBG] pgmap v6413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:10.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:10 smithi006 bash[33918]: cluster 2023-08-28T01:32:08.237356+0000 mgr.y (mgr.14901) 6406 : cluster [DBG] pgmap v6413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:11.042 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:32:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:32:10] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:32:11.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:10 smithi130 bash[51302]: debug 2023-08-28T01:32:10.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:10.672435+0000) 2023-08-28T01:32:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:32:11.042Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:32:11.044Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:12.074 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:11 smithi130 bash[51302]: debug 2023-08-28T01:32:11.672+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:11.672715+0000) 2023-08-28T01:32:12.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:12 smithi130 bash[27579]: cluster 2023-08-28T01:32:10.238525+0000 mgr.y (mgr.14901) 6407 : cluster [DBG] pgmap v6414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:12.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:12 smithi130 bash[27579]: audit 2023-08-28T01:32:11.382008+0000 mon.a (mon.0) 4261 : audit [DBG] from='client.? 172.21.15.6:0/2154522882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:12.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:12 smithi006 bash[33918]: cluster 2023-08-28T01:32:10.238525+0000 mgr.y (mgr.14901) 6407 : cluster [DBG] pgmap v6414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:12.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:12 smithi006 bash[33918]: audit 2023-08-28T01:32:11.382008+0000 mon.a (mon.0) 4261 : audit [DBG] from='client.? 172.21.15.6:0/2154522882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:12.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:12 smithi006 bash[27292]: cluster 2023-08-28T01:32:10.238525+0000 mgr.y (mgr.14901) 6407 : cluster [DBG] pgmap v6414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:12.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:12 smithi006 bash[27292]: audit 2023-08-28T01:32:11.382008+0000 mon.a (mon.0) 4261 : audit [DBG] from='client.? 172.21.15.6:0/2154522882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:32:12.611Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:32:12.613Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:13.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:12 smithi130 bash[51302]: debug 2023-08-28T01:32:12.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:12.673117+0000) 2023-08-28T01:32:14.079 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:13 smithi130 bash[51302]: debug 2023-08-28T01:32:13.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:13.673406+0000) 2023-08-28T01:32:14.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:14 smithi130 bash[27579]: cluster 2023-08-28T01:32:12.239196+0000 mgr.y (mgr.14901) 6408 : cluster [DBG] pgmap v6415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:14.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:14 smithi130 bash[27579]: audit 2023-08-28T01:32:13.801804+0000 mon.c (mon.2) 2511 : audit [DBG] from='client.? 172.21.15.6:0/3315082190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:14 smithi006 bash[27292]: cluster 2023-08-28T01:32:12.239196+0000 mgr.y (mgr.14901) 6408 : cluster [DBG] pgmap v6415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:14 smithi006 bash[27292]: audit 2023-08-28T01:32:13.801804+0000 mon.c (mon.2) 2511 : audit [DBG] from='client.? 172.21.15.6:0/3315082190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:14 smithi006 bash[33918]: cluster 2023-08-28T01:32:12.239196+0000 mgr.y (mgr.14901) 6408 : cluster [DBG] pgmap v6415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:14 smithi006 bash[33918]: audit 2023-08-28T01:32:13.801804+0000 mon.c (mon.2) 2511 : audit [DBG] from='client.? 172.21.15.6:0/3315082190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:32:14.493Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:32:14.496Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:15.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:14 smithi130 bash[51302]: debug 2023-08-28T01:32:14.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:14.673667+0000) 2023-08-28T01:32:16.084 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:15 smithi130 bash[51302]: debug 2023-08-28T01:32:15.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:15.673945+0000) 2023-08-28T01:32:16.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:16 smithi130 bash[27579]: cluster 2023-08-28T01:32:14.240401+0000 mgr.y (mgr.14901) 6409 : cluster [DBG] pgmap v6416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:16.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:16 smithi006 bash[33918]: cluster 2023-08-28T01:32:14.240401+0000 mgr.y (mgr.14901) 6409 : cluster [DBG] pgmap v6416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:16.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:16 smithi006 bash[27292]: cluster 2023-08-28T01:32:14.240401+0000 mgr.y (mgr.14901) 6409 : cluster [DBG] pgmap v6416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:17.094 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:16 smithi130 bash[51302]: debug 2023-08-28T01:32:16.672+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:16.674218+0000) 2023-08-28T01:32:17.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:17 smithi130 bash[27579]: audit 2023-08-28T01:32:16.234266+0000 mon.c (mon.2) 2512 : audit [DBG] from='client.? 172.21.15.6:0/4269796896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:17.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:17 smithi006 bash[27292]: audit 2023-08-28T01:32:16.234266+0000 mon.c (mon.2) 2512 : audit [DBG] from='client.? 172.21.15.6:0/4269796896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:17.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:17 smithi006 bash[33918]: audit 2023-08-28T01:32:16.234266+0000 mon.c (mon.2) 2512 : audit [DBG] from='client.? 172.21.15.6:0/4269796896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:18.097 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:17 smithi130 bash[51302]: debug 2023-08-28T01:32:17.668+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:17.674540+0000) 2023-08-28T01:32:18.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:18 smithi130 bash[27579]: cluster 2023-08-28T01:32:16.241060+0000 mgr.y (mgr.14901) 6410 : cluster [DBG] pgmap v6417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:18.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:18 smithi006 bash[27292]: cluster 2023-08-28T01:32:16.241060+0000 mgr.y (mgr.14901) 6410 : cluster [DBG] pgmap v6417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:18.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:18 smithi006 bash[33918]: cluster 2023-08-28T01:32:16.241060+0000 mgr.y (mgr.14901) 6410 : cluster [DBG] pgmap v6417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:19.108 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:18 smithi130 bash[51302]: debug 2023-08-28T01:32:18.672+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:18.674843+0000) 2023-08-28T01:32:19.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:19 smithi130 bash[27579]: audit 2023-08-28T01:32:18.651145+0000 mon.c (mon.2) 2513 : audit [DBG] from='client.? 172.21.15.6:0/2432460597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:19.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:19 smithi006 bash[27292]: audit 2023-08-28T01:32:18.651145+0000 mon.c (mon.2) 2513 : audit [DBG] from='client.? 172.21.15.6:0/2432460597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:19.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:19 smithi006 bash[33918]: audit 2023-08-28T01:32:18.651145+0000 mon.c (mon.2) 2513 : audit [DBG] from='client.? 172.21.15.6:0/2432460597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:20.112 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:19 smithi130 bash[51302]: debug 2023-08-28T01:32:19.672+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:19.675081+0000) 2023-08-28T01:32:20.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:20 smithi130 bash[27579]: cluster 2023-08-28T01:32:18.241703+0000 mgr.y (mgr.14901) 6411 : cluster [DBG] pgmap v6418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:20.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:20 smithi006 bash[33918]: cluster 2023-08-28T01:32:18.241703+0000 mgr.y (mgr.14901) 6411 : cluster [DBG] pgmap v6418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:20.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:20 smithi006 bash[27292]: cluster 2023-08-28T01:32:18.241703+0000 mgr.y (mgr.14901) 6411 : cluster [DBG] pgmap v6418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:21.043 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:32:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:32:20] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:32:21.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:20 smithi130 bash[51302]: debug 2023-08-28T01:32:20.672+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:20.675339+0000) 2023-08-28T01:32:21.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:21 smithi006 bash[27292]: audit 2023-08-28T01:32:21.065993+0000 mon.c (mon.2) 2514 : audit [DBG] from='client.? 172.21.15.6:0/233402619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:21.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:21 smithi006 bash[33918]: audit 2023-08-28T01:32:21.065993+0000 mon.c (mon.2) 2514 : audit [DBG] from='client.? 172.21.15.6:0/233402619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:32:21.043Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:32:21.045Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:21.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:21 smithi130 bash[27579]: audit 2023-08-28T01:32:21.065993+0000 mon.c (mon.2) 2514 : audit [DBG] from='client.? 172.21.15.6:0/233402619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:22.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:21 smithi130 bash[51302]: debug 2023-08-28T01:32:21.672+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:21.675672+0000) 2023-08-28T01:32:22.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:22 smithi006 bash[33918]: cluster 2023-08-28T01:32:20.242862+0000 mgr.y (mgr.14901) 6412 : cluster [DBG] pgmap v6419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:22.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:22 smithi006 bash[27292]: cluster 2023-08-28T01:32:20.242862+0000 mgr.y (mgr.14901) 6412 : cluster [DBG] pgmap v6419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:22 smithi130 bash[27579]: cluster 2023-08-28T01:32:20.242862+0000 mgr.y (mgr.14901) 6412 : cluster [DBG] pgmap v6419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:32:22.612Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:32:22.614Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:23.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:22 smithi130 bash[51302]: debug 2023-08-28T01:32:22.672+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:22.675964+0000) 2023-08-28T01:32:24.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:23 smithi130 bash[51302]: debug 2023-08-28T01:32:23.672+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:23.676276+0000) 2023-08-28T01:32:24.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:24 smithi006 bash[33918]: cluster 2023-08-28T01:32:22.243519+0000 mgr.y (mgr.14901) 6413 : cluster [DBG] pgmap v6420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:24.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:24 smithi006 bash[33918]: audit 2023-08-28T01:32:23.491236+0000 mon.a (mon.0) 4262 : audit [DBG] from='client.? 172.21.15.6:0/3465215653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:24 smithi006 bash[27292]: cluster 2023-08-28T01:32:22.243519+0000 mgr.y (mgr.14901) 6413 : cluster [DBG] pgmap v6420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:24 smithi006 bash[27292]: audit 2023-08-28T01:32:23.491236+0000 mon.a (mon.0) 4262 : audit [DBG] from='client.? 172.21.15.6:0/3465215653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:24 smithi130 bash[27579]: cluster 2023-08-28T01:32:22.243519+0000 mgr.y (mgr.14901) 6413 : cluster [DBG] pgmap v6420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:24 smithi130 bash[27579]: audit 2023-08-28T01:32:23.491236+0000 mon.a (mon.0) 4262 : audit [DBG] from='client.? 172.21.15.6:0/3465215653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:32:24.494Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:32:24.497Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:25.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:24 smithi130 bash[51302]: debug 2023-08-28T01:32:24.676+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:24.676572+0000) 2023-08-28T01:32:26.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:25 smithi130 bash[51302]: debug 2023-08-28T01:32:25.672+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:25.676871+0000) 2023-08-28T01:32:26.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:26 smithi006 bash[27292]: cluster 2023-08-28T01:32:24.244651+0000 mgr.y (mgr.14901) 6414 : cluster [DBG] pgmap v6421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:26.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:26 smithi006 bash[27292]: audit 2023-08-28T01:32:25.913137+0000 mon.c (mon.2) 2515 : audit [DBG] from='client.? 172.21.15.6:0/1166138493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:26.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:26 smithi006 bash[33918]: cluster 2023-08-28T01:32:24.244651+0000 mgr.y (mgr.14901) 6414 : cluster [DBG] pgmap v6421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:26.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:26 smithi006 bash[33918]: audit 2023-08-28T01:32:25.913137+0000 mon.c (mon.2) 2515 : audit [DBG] from='client.? 172.21.15.6:0/1166138493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:26 smithi130 bash[27579]: cluster 2023-08-28T01:32:24.244651+0000 mgr.y (mgr.14901) 6414 : cluster [DBG] pgmap v6421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:26 smithi130 bash[27579]: audit 2023-08-28T01:32:25.913137+0000 mon.c (mon.2) 2515 : audit [DBG] from='client.? 172.21.15.6:0/1166138493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:27.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:26 smithi130 bash[51302]: debug 2023-08-28T01:32:26.672+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:26.677118+0000) 2023-08-28T01:32:28.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:27 smithi130 bash[51302]: debug 2023-08-28T01:32:27.672+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:27.677382+0000) 2023-08-28T01:32:28.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:28 smithi006 bash[27292]: cluster 2023-08-28T01:32:26.245317+0000 mgr.y (mgr.14901) 6415 : cluster [DBG] pgmap v6422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:28.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:28 smithi006 bash[33918]: cluster 2023-08-28T01:32:26.245317+0000 mgr.y (mgr.14901) 6415 : cluster [DBG] pgmap v6422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:28 smithi130 bash[27579]: cluster 2023-08-28T01:32:26.245317+0000 mgr.y (mgr.14901) 6415 : cluster [DBG] pgmap v6422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:28 smithi130 bash[51302]: debug 2023-08-28T01:32:28.676+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:28.677649+0000) 2023-08-28T01:32:29.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:29 smithi006 bash[27292]: audit 2023-08-28T01:32:28.327003+0000 mon.a (mon.0) 4263 : audit [DBG] from='client.? 172.21.15.6:0/285684800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:29.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:29 smithi006 bash[33918]: audit 2023-08-28T01:32:28.327003+0000 mon.a (mon.0) 4263 : audit [DBG] from='client.? 172.21.15.6:0/285684800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:29.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:29 smithi130 bash[27579]: audit 2023-08-28T01:32:28.327003+0000 mon.a (mon.0) 4263 : audit [DBG] from='client.? 172.21.15.6:0/285684800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:30.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:29 smithi130 bash[51302]: debug 2023-08-28T01:32:29.676+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:29.677921+0000) 2023-08-28T01:32:30.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:30 smithi006 bash[33918]: cluster 2023-08-28T01:32:28.245900+0000 mgr.y (mgr.14901) 6416 : cluster [DBG] pgmap v6423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:30.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:30 smithi006 bash[27292]: cluster 2023-08-28T01:32:28.245900+0000 mgr.y (mgr.14901) 6416 : cluster [DBG] pgmap v6423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:30 smithi130 bash[27579]: cluster 2023-08-28T01:32:28.245900+0000 mgr.y (mgr.14901) 6416 : cluster [DBG] pgmap v6423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:31.044 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:32:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:32:30] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:32:31.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:30 smithi130 bash[51302]: debug 2023-08-28T01:32:30.672+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:30.678232+0000) 2023-08-28T01:32:31.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:31 smithi006 bash[27292]: audit 2023-08-28T01:32:30.741802+0000 mon.a (mon.0) 4264 : audit [DBG] from='client.? 172.21.15.6:0/497558020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:31.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:31 smithi006 bash[33918]: audit 2023-08-28T01:32:30.741802+0000 mon.a (mon.0) 4264 : audit [DBG] from='client.? 172.21.15.6:0/497558020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:32:31.043Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:31.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:32:31.045Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:31.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:31 smithi130 bash[27579]: audit 2023-08-28T01:32:30.741802+0000 mon.a (mon.0) 4264 : audit [DBG] from='client.? 172.21.15.6:0/497558020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:32.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:31 smithi130 bash[51302]: debug 2023-08-28T01:32:31.671+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:31.678553+0000) 2023-08-28T01:32:32.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:32 smithi006 bash[27292]: cluster 2023-08-28T01:32:30.247017+0000 mgr.y (mgr.14901) 6417 : cluster [DBG] pgmap v6424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:32.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:32 smithi006 bash[33918]: cluster 2023-08-28T01:32:30.247017+0000 mgr.y (mgr.14901) 6417 : cluster [DBG] pgmap v6424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:32 smithi130 bash[27579]: cluster 2023-08-28T01:32:30.247017+0000 mgr.y (mgr.14901) 6417 : cluster [DBG] pgmap v6424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:32:32.613Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:32:32.615Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:33.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:32 smithi130 bash[51302]: debug 2023-08-28T01:32:32.671+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:32.678833+0000) 2023-08-28T01:32:33.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:33 smithi006 bash[33918]: audit 2023-08-28T01:32:33.157174+0000 mon.a (mon.0) 4265 : audit [DBG] from='client.? 172.21.15.6:0/2357739253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:33.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:33 smithi006 bash[27292]: audit 2023-08-28T01:32:33.157174+0000 mon.a (mon.0) 4265 : audit [DBG] from='client.? 172.21.15.6:0/2357739253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:33.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:33 smithi130 bash[27579]: audit 2023-08-28T01:32:33.157174+0000 mon.a (mon.0) 4265 : audit [DBG] from='client.? 172.21.15.6:0/2357739253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:34.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:33 smithi130 bash[51302]: debug 2023-08-28T01:32:33.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:33.679124+0000) 2023-08-28T01:32:34.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:34 smithi006 bash[27292]: cluster 2023-08-28T01:32:32.247607+0000 mgr.y (mgr.14901) 6418 : cluster [DBG] pgmap v6425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:34 smithi006 bash[33918]: cluster 2023-08-28T01:32:32.247607+0000 mgr.y (mgr.14901) 6418 : cluster [DBG] pgmap v6425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:34 smithi130 bash[27579]: cluster 2023-08-28T01:32:32.247607+0000 mgr.y (mgr.14901) 6418 : cluster [DBG] pgmap v6425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:32:34.495Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:32:34.497Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:35.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:34 smithi130 bash[51302]: debug 2023-08-28T01:32:34.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:34.679422+0000) 2023-08-28T01:32:36.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:35 smithi130 bash[51302]: debug 2023-08-28T01:32:35.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:35.679718+0000) 2023-08-28T01:32:36.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:36 smithi006 bash[27292]: cluster 2023-08-28T01:32:34.248665+0000 mgr.y (mgr.14901) 6419 : cluster [DBG] pgmap v6426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:36.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:36 smithi006 bash[27292]: audit 2023-08-28T01:32:35.569960+0000 mon.a (mon.0) 4266 : audit [DBG] from='client.? 172.21.15.6:0/119795377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:36.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:36 smithi006 bash[33918]: cluster 2023-08-28T01:32:34.248665+0000 mgr.y (mgr.14901) 6419 : cluster [DBG] pgmap v6426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:36.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:36 smithi006 bash[33918]: audit 2023-08-28T01:32:35.569960+0000 mon.a (mon.0) 4266 : audit [DBG] from='client.? 172.21.15.6:0/119795377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:36 smithi130 bash[27579]: cluster 2023-08-28T01:32:34.248665+0000 mgr.y (mgr.14901) 6419 : cluster [DBG] pgmap v6426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:36 smithi130 bash[27579]: audit 2023-08-28T01:32:35.569960+0000 mon.a (mon.0) 4266 : audit [DBG] from='client.? 172.21.15.6:0/119795377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:37.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:36 smithi130 bash[51302]: debug 2023-08-28T01:32:36.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:36.680014+0000) 2023-08-28T01:32:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:37 smithi130 bash[51302]: debug 2023-08-28T01:32:37.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:37.680313+0000) 2023-08-28T01:32:38.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:38 smithi006 bash[27292]: cluster 2023-08-28T01:32:36.249441+0000 mgr.y (mgr.14901) 6420 : cluster [DBG] pgmap v6427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:38.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:38 smithi006 bash[27292]: audit 2023-08-28T01:32:37.984969+0000 mon.c (mon.2) 2516 : audit [DBG] from='client.? 172.21.15.6:0/3932584798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:38.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:38 smithi006 bash[33918]: cluster 2023-08-28T01:32:36.249441+0000 mgr.y (mgr.14901) 6420 : cluster [DBG] pgmap v6427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:38.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:38 smithi006 bash[33918]: audit 2023-08-28T01:32:37.984969+0000 mon.c (mon.2) 2516 : audit [DBG] from='client.? 172.21.15.6:0/3932584798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:38 smithi130 bash[27579]: cluster 2023-08-28T01:32:36.249441+0000 mgr.y (mgr.14901) 6420 : cluster [DBG] pgmap v6427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:38 smithi130 bash[27579]: audit 2023-08-28T01:32:37.984969+0000 mon.c (mon.2) 2516 : audit [DBG] from='client.? 172.21.15.6:0/3932584798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:39.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:38 smithi130 bash[51302]: debug 2023-08-28T01:32:38.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:38.680582+0000) 2023-08-28T01:32:40.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:39 smithi130 bash[51302]: debug 2023-08-28T01:32:39.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:39.680850+0000) 2023-08-28T01:32:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:40 smithi006 bash[33918]: cluster 2023-08-28T01:32:38.250147+0000 mgr.y (mgr.14901) 6421 : cluster [DBG] pgmap v6428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:40.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:40 smithi006 bash[27292]: cluster 2023-08-28T01:32:38.250147+0000 mgr.y (mgr.14901) 6421 : cluster [DBG] pgmap v6428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:40 smithi130 bash[27579]: cluster 2023-08-28T01:32:38.250147+0000 mgr.y (mgr.14901) 6421 : cluster [DBG] pgmap v6428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:41.044 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:32:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:32:40] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:32:41.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:40 smithi130 bash[51302]: debug 2023-08-28T01:32:40.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:40.681181+0000) 2023-08-28T01:32:41.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:41 smithi006 bash[27292]: audit 2023-08-28T01:32:40.404694+0000 mon.a (mon.0) 4267 : audit [DBG] from='client.? 172.21.15.6:0/4118067524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:41.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:41 smithi006 bash[33918]: audit 2023-08-28T01:32:40.404694+0000 mon.a (mon.0) 4267 : audit [DBG] from='client.? 172.21.15.6:0/4118067524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:32:41.044Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:32:41.046Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:41.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:41 smithi130 bash[27579]: audit 2023-08-28T01:32:40.404694+0000 mon.a (mon.0) 4267 : audit [DBG] from='client.? 172.21.15.6:0/4118067524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:41 smithi130 bash[51302]: debug 2023-08-28T01:32:41.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:41.681501+0000) 2023-08-28T01:32:42.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:42 smithi006 bash[27292]: cluster 2023-08-28T01:32:40.251016+0000 mgr.y (mgr.14901) 6422 : cluster [DBG] pgmap v6429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:42.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:42 smithi006 bash[33918]: cluster 2023-08-28T01:32:40.251016+0000 mgr.y (mgr.14901) 6422 : cluster [DBG] pgmap v6429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:42 smithi130 bash[27579]: cluster 2023-08-28T01:32:40.251016+0000 mgr.y (mgr.14901) 6422 : cluster [DBG] pgmap v6429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:32:42.614Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:32:42.615Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:43.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:42 smithi130 bash[51302]: debug 2023-08-28T01:32:42.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:42.681806+0000) 2023-08-28T01:32:43.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:43 smithi006 bash[27292]: audit 2023-08-28T01:32:42.822048+0000 mon.a (mon.0) 4268 : audit [DBG] from='client.? 172.21.15.6:0/352357732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:43.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:43 smithi006 bash[33918]: audit 2023-08-28T01:32:42.822048+0000 mon.a (mon.0) 4268 : audit [DBG] from='client.? 172.21.15.6:0/352357732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:43.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:43 smithi130 bash[27579]: audit 2023-08-28T01:32:42.822048+0000 mon.a (mon.0) 4268 : audit [DBG] from='client.? 172.21.15.6:0/352357732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:44.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:43 smithi130 bash[51302]: debug 2023-08-28T01:32:43.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:43.682079+0000) 2023-08-28T01:32:44.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:44 smithi006 bash[27292]: cluster 2023-08-28T01:32:42.251617+0000 mgr.y (mgr.14901) 6423 : cluster [DBG] pgmap v6430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:44 smithi006 bash[33918]: cluster 2023-08-28T01:32:42.251617+0000 mgr.y (mgr.14901) 6423 : cluster [DBG] pgmap v6430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:44 smithi130 bash[27579]: cluster 2023-08-28T01:32:42.251617+0000 mgr.y (mgr.14901) 6423 : cluster [DBG] pgmap v6430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:32:44.495Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:32:44.498Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:45.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:44 smithi130 bash[51302]: debug 2023-08-28T01:32:44.675+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:44.682363+0000) 2023-08-28T01:32:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:45 smithi130 bash[51302]: debug 2023-08-28T01:32:45.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:45.682673+0000) 2023-08-28T01:32:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:46 smithi130 bash[27579]: cluster 2023-08-28T01:32:44.252719+0000 mgr.y (mgr.14901) 6424 : cluster [DBG] pgmap v6431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:46 smithi130 bash[27579]: audit 2023-08-28T01:32:45.245725+0000 mon.b (mon.1) 732 : audit [DBG] from='client.? 172.21.15.6:0/1721764488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:46.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:46 smithi006 bash[27292]: cluster 2023-08-28T01:32:44.252719+0000 mgr.y (mgr.14901) 6424 : cluster [DBG] pgmap v6431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:46.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:46 smithi006 bash[27292]: audit 2023-08-28T01:32:45.245725+0000 mon.b (mon.1) 732 : audit [DBG] from='client.? 172.21.15.6:0/1721764488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:46.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:46 smithi006 bash[33918]: cluster 2023-08-28T01:32:44.252719+0000 mgr.y (mgr.14901) 6424 : cluster [DBG] pgmap v6431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:46.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:46 smithi006 bash[33918]: audit 2023-08-28T01:32:45.245725+0000 mon.b (mon.1) 732 : audit [DBG] from='client.? 172.21.15.6:0/1721764488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:47.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:46 smithi130 bash[51302]: debug 2023-08-28T01:32:46.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:46.682984+0000) 2023-08-28T01:32:48.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:47 smithi130 bash[51302]: debug 2023-08-28T01:32:47.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:47.683217+0000) 2023-08-28T01:32:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:48 smithi130 bash[27579]: cluster 2023-08-28T01:32:46.253374+0000 mgr.y (mgr.14901) 6425 : cluster [DBG] pgmap v6432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:48 smithi130 bash[27579]: audit 2023-08-28T01:32:47.666653+0000 mon.c (mon.2) 2517 : audit [DBG] from='client.? 172.21.15.6:0/2007324724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:48.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:48 smithi006 bash[33918]: cluster 2023-08-28T01:32:46.253374+0000 mgr.y (mgr.14901) 6425 : cluster [DBG] pgmap v6432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:48.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:48 smithi006 bash[33918]: audit 2023-08-28T01:32:47.666653+0000 mon.c (mon.2) 2517 : audit [DBG] from='client.? 172.21.15.6:0/2007324724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:48 smithi006 bash[27292]: cluster 2023-08-28T01:32:46.253374+0000 mgr.y (mgr.14901) 6425 : cluster [DBG] pgmap v6432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:48 smithi006 bash[27292]: audit 2023-08-28T01:32:47.666653+0000 mon.c (mon.2) 2517 : audit [DBG] from='client.? 172.21.15.6:0/2007324724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:49.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:48 smithi130 bash[51302]: debug 2023-08-28T01:32:48.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:48.683499+0000) 2023-08-28T01:32:50.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:49 smithi130 bash[51302]: debug 2023-08-28T01:32:49.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:49.683773+0000) 2023-08-28T01:32:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:50 smithi130 bash[27579]: cluster 2023-08-28T01:32:48.254035+0000 mgr.y (mgr.14901) 6426 : cluster [DBG] pgmap v6433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:50 smithi130 bash[27579]: audit 2023-08-28T01:32:50.081121+0000 mon.a (mon.0) 4269 : audit [DBG] from='client.? 172.21.15.6:0/4121446868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:50 smithi006 bash[27292]: cluster 2023-08-28T01:32:48.254035+0000 mgr.y (mgr.14901) 6426 : cluster [DBG] pgmap v6433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:50 smithi006 bash[27292]: audit 2023-08-28T01:32:50.081121+0000 mon.a (mon.0) 4269 : audit [DBG] from='client.? 172.21.15.6:0/4121446868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:50.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:50 smithi006 bash[33918]: cluster 2023-08-28T01:32:48.254035+0000 mgr.y (mgr.14901) 6426 : cluster [DBG] pgmap v6433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:50.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:50 smithi006 bash[33918]: audit 2023-08-28T01:32:50.081121+0000 mon.a (mon.0) 4269 : audit [DBG] from='client.? 172.21.15.6:0/4121446868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:51.045 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:32:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:32:50] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T01:32:51.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:50 smithi130 bash[51302]: debug 2023-08-28T01:32:50.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:50.684114+0000) 2023-08-28T01:32:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:32:51.045Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:32:51.047Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:52.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:51 smithi130 bash[51302]: debug 2023-08-28T01:32:51.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:51.684403+0000) 2023-08-28T01:32:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:52 smithi130 bash[27579]: cluster 2023-08-28T01:32:50.255109+0000 mgr.y (mgr.14901) 6427 : cluster [DBG] pgmap v6434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:52.616 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:52 smithi006 bash[27292]: cluster 2023-08-28T01:32:50.255109+0000 mgr.y (mgr.14901) 6427 : cluster [DBG] pgmap v6434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:52.616 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:52 smithi006 bash[33918]: cluster 2023-08-28T01:32:50.255109+0000 mgr.y (mgr.14901) 6427 : cluster [DBG] pgmap v6434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:32:52.615Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:32:52.617Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:53.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:52 smithi130 bash[51302]: debug 2023-08-28T01:32:52.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:52.684653+0000) 2023-08-28T01:32:53.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:53 smithi130 bash[27579]: cluster 2023-08-28T01:32:52.255564+0000 mgr.y (mgr.14901) 6428 : cluster [DBG] pgmap v6435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:53.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:53 smithi130 bash[27579]: audit 2023-08-28T01:32:52.495403+0000 mon.a (mon.0) 4270 : audit [DBG] from='client.? 172.21.15.6:0/1493189014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:53.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:53 smithi130 bash[27579]: audit 2023-08-28T01:32:53.094572+0000 mon.b (mon.1) 733 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:32:53.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:53 smithi006 bash[27292]: cluster 2023-08-28T01:32:52.255564+0000 mgr.y (mgr.14901) 6428 : cluster [DBG] pgmap v6435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:53.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:53 smithi006 bash[27292]: audit 2023-08-28T01:32:52.495403+0000 mon.a (mon.0) 4270 : audit [DBG] from='client.? 172.21.15.6:0/1493189014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:53.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:53 smithi006 bash[27292]: audit 2023-08-28T01:32:53.094572+0000 mon.b (mon.1) 733 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:32:53.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:53 smithi006 bash[33918]: cluster 2023-08-28T01:32:52.255564+0000 mgr.y (mgr.14901) 6428 : cluster [DBG] pgmap v6435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:53.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:53 smithi006 bash[33918]: audit 2023-08-28T01:32:52.495403+0000 mon.a (mon.0) 4270 : audit [DBG] from='client.? 172.21.15.6:0/1493189014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:53.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:53 smithi006 bash[33918]: audit 2023-08-28T01:32:53.094572+0000 mon.b (mon.1) 733 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:32:54.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:53 smithi130 bash[51302]: debug 2023-08-28T01:32:53.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:53.684932+0000) 2023-08-28T01:32:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:32:54.496Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:32:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:32:54.498Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:32:55.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:54 smithi130 bash[51302]: debug 2023-08-28T01:32:54.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:54.685198+0000) 2023-08-28T01:32:55.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:55 smithi130 bash[27579]: cluster 2023-08-28T01:32:54.256591+0000 mgr.y (mgr.14901) 6429 : cluster [DBG] pgmap v6436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:55.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:55 smithi130 bash[27579]: audit 2023-08-28T01:32:54.903321+0000 mon.a (mon.0) 4271 : audit [DBG] from='client.? 172.21.15.6:0/3549428334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:55.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:55 smithi006 bash[27292]: cluster 2023-08-28T01:32:54.256591+0000 mgr.y (mgr.14901) 6429 : cluster [DBG] pgmap v6436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:55.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:55 smithi006 bash[27292]: audit 2023-08-28T01:32:54.903321+0000 mon.a (mon.0) 4271 : audit [DBG] from='client.? 172.21.15.6:0/3549428334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:55.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:55 smithi006 bash[33918]: cluster 2023-08-28T01:32:54.256591+0000 mgr.y (mgr.14901) 6429 : cluster [DBG] pgmap v6436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:55.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:55 smithi006 bash[33918]: audit 2023-08-28T01:32:54.903321+0000 mon.a (mon.0) 4271 : audit [DBG] from='client.? 172.21.15.6:0/3549428334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:56.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:55 smithi130 bash[51302]: debug 2023-08-28T01:32:55.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:55.685465+0000) 2023-08-28T01:32:57.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:56 smithi130 bash[51302]: debug 2023-08-28T01:32:56.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:56.685750+0000) 2023-08-28T01:32:58.002 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:57 smithi130 bash[51302]: debug 2023-08-28T01:32:57.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:57.686036+0000) 2023-08-28T01:32:58.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:58 smithi130 bash[27579]: cluster 2023-08-28T01:32:56.257384+0000 mgr.y (mgr.14901) 6430 : cluster [DBG] pgmap v6437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:58.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:32:58 smithi130 bash[27579]: audit 2023-08-28T01:32:57.322979+0000 mon.c (mon.2) 2518 : audit [DBG] from='client.? 172.21.15.6:0/3876070168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:58.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:58 smithi006 bash[27292]: cluster 2023-08-28T01:32:56.257384+0000 mgr.y (mgr.14901) 6430 : cluster [DBG] pgmap v6437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:58.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:32:58 smithi006 bash[27292]: audit 2023-08-28T01:32:57.322979+0000 mon.c (mon.2) 2518 : audit [DBG] from='client.? 172.21.15.6:0/3876070168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:58.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:58 smithi006 bash[33918]: cluster 2023-08-28T01:32:56.257384+0000 mgr.y (mgr.14901) 6430 : cluster [DBG] pgmap v6437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:32:58.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:32:58 smithi006 bash[33918]: audit 2023-08-28T01:32:57.322979+0000 mon.c (mon.2) 2518 : audit [DBG] from='client.? 172.21.15.6:0/3876070168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:59.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:58 smithi130 bash[51302]: debug 2023-08-28T01:32:58.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:58.686314+0000) 2023-08-28T01:33:00.038 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:32:59 smithi130 bash[51302]: debug 2023-08-28T01:32:59.679+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:59.686627+0000) 2023-08-28T01:33:00.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:00 smithi130 bash[27579]: cluster 2023-08-28T01:32:58.258045+0000 mgr.y (mgr.14901) 6431 : cluster [DBG] pgmap v6438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:00.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:00 smithi130 bash[27579]: audit 2023-08-28T01:32:59.037567+0000 mon.a (mon.0) 4272 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:33:00.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:00 smithi130 bash[27579]: audit 2023-08-28T01:32:59.459731+0000 mon.a (mon.0) 4273 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:33:00.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:00 smithi130 bash[27579]: audit 2023-08-28T01:32:59.728485+0000 mon.a (mon.0) 4274 : audit [DBG] from='client.? 172.21.15.6:0/4133960636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:00.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:00 smithi130 bash[27579]: audit 2023-08-28T01:32:59.758079+0000 mon.b (mon.1) 734 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:33:00.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:00 smithi130 bash[27579]: audit 2023-08-28T01:32:59.760282+0000 mon.b (mon.1) 735 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:33:00.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:00 smithi130 bash[27579]: audit 2023-08-28T01:32:59.767116+0000 mon.a (mon.0) 4275 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:33:00.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:00 smithi006 bash[33918]: cluster 2023-08-28T01:32:58.258045+0000 mgr.y (mgr.14901) 6431 : cluster [DBG] pgmap v6438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:00.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:00 smithi006 bash[33918]: audit 2023-08-28T01:32:59.037567+0000 mon.a (mon.0) 4272 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:33:00.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:00 smithi006 bash[33918]: audit 2023-08-28T01:32:59.459731+0000 mon.a (mon.0) 4273 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:33:00.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:00 smithi006 bash[33918]: audit 2023-08-28T01:32:59.728485+0000 mon.a (mon.0) 4274 : audit [DBG] from='client.? 172.21.15.6:0/4133960636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:00.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:00 smithi006 bash[33918]: audit 2023-08-28T01:32:59.758079+0000 mon.b (mon.1) 734 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:33:00.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:00 smithi006 bash[33918]: audit 2023-08-28T01:32:59.760282+0000 mon.b (mon.1) 735 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:33:00.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:00 smithi006 bash[33918]: audit 2023-08-28T01:32:59.767116+0000 mon.a (mon.0) 4275 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:33:00.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:00 smithi006 bash[27292]: cluster 2023-08-28T01:32:58.258045+0000 mgr.y (mgr.14901) 6431 : cluster [DBG] pgmap v6438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:00.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:00 smithi006 bash[27292]: audit 2023-08-28T01:32:59.037567+0000 mon.a (mon.0) 4272 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:33:00.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:00 smithi006 bash[27292]: audit 2023-08-28T01:32:59.459731+0000 mon.a (mon.0) 4273 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:33:00.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:00 smithi006 bash[27292]: audit 2023-08-28T01:32:59.728485+0000 mon.a (mon.0) 4274 : audit [DBG] from='client.? 172.21.15.6:0/4133960636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:00.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:00 smithi006 bash[27292]: audit 2023-08-28T01:32:59.758079+0000 mon.b (mon.1) 734 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:33:00.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:00 smithi006 bash[27292]: audit 2023-08-28T01:32:59.760282+0000 mon.b (mon.1) 735 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:33:00.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:00 smithi006 bash[27292]: audit 2023-08-28T01:32:59.767116+0000 mon.a (mon.0) 4275 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:33:01.046 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:33:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:33:00] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T01:33:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:00 smithi130 bash[51302]: debug 2023-08-28T01:33:00.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:00.686931+0000) 2023-08-28T01:33:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:33:01.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:33:01.047Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:02.044 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:01 smithi130 bash[51302]: debug 2023-08-28T01:33:01.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:01.687220+0000) 2023-08-28T01:33:02.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:02 smithi130 bash[27579]: cluster 2023-08-28T01:33:00.259192+0000 mgr.y (mgr.14901) 6432 : cluster [DBG] pgmap v6439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:02.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:02 smithi006 bash[33918]: cluster 2023-08-28T01:33:00.259192+0000 mgr.y (mgr.14901) 6432 : cluster [DBG] pgmap v6439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:02.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:02 smithi006 bash[27292]: cluster 2023-08-28T01:33:00.259192+0000 mgr.y (mgr.14901) 6432 : cluster [DBG] pgmap v6439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:33:02.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:33:02.618Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:03.056 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:02 smithi130 bash[51302]: debug 2023-08-28T01:33:02.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:02.687517+0000) 2023-08-28T01:33:03.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:03 smithi130 bash[27579]: audit 2023-08-28T01:33:02.143672+0000 mon.c (mon.2) 2519 : audit [DBG] from='client.? 172.21.15.6:0/3328666502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:03.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:03 smithi006 bash[33918]: audit 2023-08-28T01:33:02.143672+0000 mon.c (mon.2) 2519 : audit [DBG] from='client.? 172.21.15.6:0/3328666502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:03.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:03 smithi006 bash[27292]: audit 2023-08-28T01:33:02.143672+0000 mon.c (mon.2) 2519 : audit [DBG] from='client.? 172.21.15.6:0/3328666502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:04.059 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:03 smithi130 bash[51302]: debug 2023-08-28T01:33:03.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:03.687782+0000) 2023-08-28T01:33:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:04 smithi130 bash[27579]: cluster 2023-08-28T01:33:02.259873+0000 mgr.y (mgr.14901) 6433 : cluster [DBG] pgmap v6440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:04.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:04 smithi006 bash[33918]: cluster 2023-08-28T01:33:02.259873+0000 mgr.y (mgr.14901) 6433 : cluster [DBG] pgmap v6440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:04 smithi006 bash[27292]: cluster 2023-08-28T01:33:02.259873+0000 mgr.y (mgr.14901) 6433 : cluster [DBG] pgmap v6440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:33:04.497Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:33:04.499Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:05.068 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:04 smithi130 bash[51302]: debug 2023-08-28T01:33:04.687+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:04.688090+0000) 2023-08-28T01:33:05.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:05 smithi130 bash[27579]: audit 2023-08-28T01:33:04.557932+0000 mon.a (mon.0) 4276 : audit [DBG] from='client.? 172.21.15.6:0/1288240721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:05.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:05 smithi006 bash[27292]: audit 2023-08-28T01:33:04.557932+0000 mon.a (mon.0) 4276 : audit [DBG] from='client.? 172.21.15.6:0/1288240721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:05.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:05 smithi006 bash[33918]: audit 2023-08-28T01:33:04.557932+0000 mon.a (mon.0) 4276 : audit [DBG] from='client.? 172.21.15.6:0/1288240721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:06.073 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:05 smithi130 bash[51302]: debug 2023-08-28T01:33:05.687+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:05.688371+0000) 2023-08-28T01:33:06.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:06 smithi130 bash[27579]: cluster 2023-08-28T01:33:04.260988+0000 mgr.y (mgr.14901) 6434 : cluster [DBG] pgmap v6441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:06.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:06 smithi006 bash[27292]: cluster 2023-08-28T01:33:04.260988+0000 mgr.y (mgr.14901) 6434 : cluster [DBG] pgmap v6441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:06.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:06 smithi006 bash[33918]: cluster 2023-08-28T01:33:04.260988+0000 mgr.y (mgr.14901) 6434 : cluster [DBG] pgmap v6441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:07.083 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:06 smithi130 bash[51302]: debug 2023-08-28T01:33:06.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:06.688636+0000) 2023-08-28T01:33:07.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:07 smithi130 bash[27579]: audit 2023-08-28T01:33:06.972597+0000 mon.a (mon.0) 4277 : audit [DBG] from='client.? 172.21.15.6:0/176462099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:07 smithi006 bash[27292]: audit 2023-08-28T01:33:06.972597+0000 mon.a (mon.0) 4277 : audit [DBG] from='client.? 172.21.15.6:0/176462099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:07.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:07 smithi006 bash[33918]: audit 2023-08-28T01:33:06.972597+0000 mon.a (mon.0) 4277 : audit [DBG] from='client.? 172.21.15.6:0/176462099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:08.086 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:07 smithi130 bash[51302]: debug 2023-08-28T01:33:07.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:07.688920+0000) 2023-08-28T01:33:08.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:08 smithi130 bash[27579]: cluster 2023-08-28T01:33:06.261681+0000 mgr.y (mgr.14901) 6435 : cluster [DBG] pgmap v6442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:08.767 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:08 smithi006 bash[33918]: cluster 2023-08-28T01:33:06.261681+0000 mgr.y (mgr.14901) 6435 : cluster [DBG] pgmap v6442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:08.768 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:08 smithi006 bash[27292]: cluster 2023-08-28T01:33:06.261681+0000 mgr.y (mgr.14901) 6435 : cluster [DBG] pgmap v6442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:09.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:08 smithi130 bash[51302]: debug 2023-08-28T01:33:08.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:08.689221+0000) 2023-08-28T01:33:10.091 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:09 smithi130 bash[51302]: debug 2023-08-28T01:33:09.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:09.689502+0000) 2023-08-28T01:33:10.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:10 smithi130 bash[27579]: cluster 2023-08-28T01:33:08.262293+0000 mgr.y (mgr.14901) 6436 : cluster [DBG] pgmap v6443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:10.434 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:10 smithi130 bash[27579]: audit 2023-08-28T01:33:09.396510+0000 mon.a (mon.0) 4278 : audit [DBG] from='client.? 172.21.15.6:0/83397042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:10.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:10 smithi006 bash[33918]: cluster 2023-08-28T01:33:08.262293+0000 mgr.y (mgr.14901) 6436 : cluster [DBG] pgmap v6443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:10.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:10 smithi006 bash[33918]: audit 2023-08-28T01:33:09.396510+0000 mon.a (mon.0) 4278 : audit [DBG] from='client.? 172.21.15.6:0/83397042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:10.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:10 smithi006 bash[27292]: cluster 2023-08-28T01:33:08.262293+0000 mgr.y (mgr.14901) 6436 : cluster [DBG] pgmap v6443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:10.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:10 smithi006 bash[27292]: audit 2023-08-28T01:33:09.396510+0000 mon.a (mon.0) 4278 : audit [DBG] from='client.? 172.21.15.6:0/83397042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:11.047 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:33:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:33:10] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T01:33:11.162 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:10 smithi130 bash[51302]: debug 2023-08-28T01:33:10.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:10.689782+0000) 2023-08-28T01:33:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:33:11.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:33:11.048Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:12.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:11 smithi130 bash[51302]: debug 2023-08-28T01:33:11.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:11.690051+0000) 2023-08-28T01:33:12.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:12 smithi006 bash[33918]: cluster 2023-08-28T01:33:10.263451+0000 mgr.y (mgr.14901) 6437 : cluster [DBG] pgmap v6444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:12.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:12 smithi006 bash[33918]: audit 2023-08-28T01:33:11.810762+0000 mon.c (mon.2) 2520 : audit [DBG] from='client.? 172.21.15.6:0/1181846681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:12.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:12 smithi006 bash[27292]: cluster 2023-08-28T01:33:10.263451+0000 mgr.y (mgr.14901) 6437 : cluster [DBG] pgmap v6444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:12.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:12 smithi006 bash[27292]: audit 2023-08-28T01:33:11.810762+0000 mon.c (mon.2) 2520 : audit [DBG] from='client.? 172.21.15.6:0/1181846681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:12 smithi130 bash[27579]: cluster 2023-08-28T01:33:10.263451+0000 mgr.y (mgr.14901) 6437 : cluster [DBG] pgmap v6444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:12 smithi130 bash[27579]: audit 2023-08-28T01:33:11.810762+0000 mon.c (mon.2) 2520 : audit [DBG] from='client.? 172.21.15.6:0/1181846681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:33:12.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:33:12.619Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:13.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:12 smithi130 bash[51302]: debug 2023-08-28T01:33:12.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:12.690324+0000) 2023-08-28T01:33:14.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:13 smithi130 bash[51302]: debug 2023-08-28T01:33:13.683+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:13.690664+0000) 2023-08-28T01:33:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:14 smithi006 bash[33918]: cluster 2023-08-28T01:33:12.264060+0000 mgr.y (mgr.14901) 6438 : cluster [DBG] pgmap v6445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:14 smithi006 bash[27292]: cluster 2023-08-28T01:33:12.264060+0000 mgr.y (mgr.14901) 6438 : cluster [DBG] pgmap v6445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:14 smithi130 bash[27579]: cluster 2023-08-28T01:33:12.264060+0000 mgr.y (mgr.14901) 6438 : cluster [DBG] pgmap v6445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:33:14.498Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:33:14.500Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:14 smithi130 bash[51302]: debug 2023-08-28T01:33:14.687+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:14.690994+0000) 2023-08-28T01:33:15.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:15 smithi006 bash[27292]: audit 2023-08-28T01:33:14.240605+0000 mon.a (mon.0) 4279 : audit [DBG] from='client.? 172.21.15.6:0/21866811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:15.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:15 smithi006 bash[33918]: audit 2023-08-28T01:33:14.240605+0000 mon.a (mon.0) 4279 : audit [DBG] from='client.? 172.21.15.6:0/21866811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:15.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:15 smithi130 bash[27579]: audit 2023-08-28T01:33:14.240605+0000 mon.a (mon.0) 4279 : audit [DBG] from='client.? 172.21.15.6:0/21866811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:16.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:15 smithi130 bash[51302]: debug 2023-08-28T01:33:15.687+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:15.691289+0000) 2023-08-28T01:33:16.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:16 smithi006 bash[27292]: cluster 2023-08-28T01:33:14.265197+0000 mgr.y (mgr.14901) 6439 : cluster [DBG] pgmap v6446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:16.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:16 smithi006 bash[33918]: cluster 2023-08-28T01:33:14.265197+0000 mgr.y (mgr.14901) 6439 : cluster [DBG] pgmap v6446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:16 smithi130 bash[27579]: cluster 2023-08-28T01:33:14.265197+0000 mgr.y (mgr.14901) 6439 : cluster [DBG] pgmap v6446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:17.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:16 smithi130 bash[51302]: debug 2023-08-28T01:33:16.687+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:16.691506+0000) 2023-08-28T01:33:17.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:17 smithi006 bash[27292]: audit 2023-08-28T01:33:16.640832+0000 mon.c (mon.2) 2521 : audit [DBG] from='client.? 172.21.15.6:0/2689117342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:17.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:17 smithi006 bash[33918]: audit 2023-08-28T01:33:16.640832+0000 mon.c (mon.2) 2521 : audit [DBG] from='client.? 172.21.15.6:0/2689117342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:17.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:17 smithi130 bash[27579]: audit 2023-08-28T01:33:16.640832+0000 mon.c (mon.2) 2521 : audit [DBG] from='client.? 172.21.15.6:0/2689117342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:18.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:17 smithi130 bash[51302]: debug 2023-08-28T01:33:17.691+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:17.691784+0000) 2023-08-28T01:33:18.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:18 smithi006 bash[27292]: cluster 2023-08-28T01:33:16.265813+0000 mgr.y (mgr.14901) 6440 : cluster [DBG] pgmap v6447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:18.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:18 smithi006 bash[33918]: cluster 2023-08-28T01:33:16.265813+0000 mgr.y (mgr.14901) 6440 : cluster [DBG] pgmap v6447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:18 smithi130 bash[27579]: cluster 2023-08-28T01:33:16.265813+0000 mgr.y (mgr.14901) 6440 : cluster [DBG] pgmap v6447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:19.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:18 smithi130 bash[51302]: debug 2023-08-28T01:33:18.687+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:18.692095+0000) 2023-08-28T01:33:19.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:19 smithi006 bash[33918]: audit 2023-08-28T01:33:19.051929+0000 mon.c (mon.2) 2522 : audit [DBG] from='client.? 172.21.15.6:0/828091035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:19.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:19 smithi006 bash[27292]: audit 2023-08-28T01:33:19.051929+0000 mon.c (mon.2) 2522 : audit [DBG] from='client.? 172.21.15.6:0/828091035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:19.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:19 smithi130 bash[27579]: audit 2023-08-28T01:33:19.051929+0000 mon.c (mon.2) 2522 : audit [DBG] from='client.? 172.21.15.6:0/828091035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:19 smithi130 bash[51302]: debug 2023-08-28T01:33:19.687+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:19.692388+0000) 2023-08-28T01:33:20.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:20 smithi006 bash[33918]: cluster 2023-08-28T01:33:18.266487+0000 mgr.y (mgr.14901) 6441 : cluster [DBG] pgmap v6448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:20.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:20 smithi006 bash[27292]: cluster 2023-08-28T01:33:18.266487+0000 mgr.y (mgr.14901) 6441 : cluster [DBG] pgmap v6448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:20 smithi130 bash[27579]: cluster 2023-08-28T01:33:18.266487+0000 mgr.y (mgr.14901) 6441 : cluster [DBG] pgmap v6448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:21.048 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:33:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:33:20] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:33:21.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:20 smithi130 bash[51302]: debug 2023-08-28T01:33:20.691+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:20.692673+0000) 2023-08-28T01:33:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:33:21.047Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:33:21.049Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:22.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:21 smithi130 bash[51302]: debug 2023-08-28T01:33:21.691+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:21.692952+0000) 2023-08-28T01:33:22.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:22 smithi006 bash[27292]: cluster 2023-08-28T01:33:20.267656+0000 mgr.y (mgr.14901) 6442 : cluster [DBG] pgmap v6449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:22.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:22 smithi006 bash[27292]: audit 2023-08-28T01:33:21.459132+0000 mon.c (mon.2) 2523 : audit [DBG] from='client.? 172.21.15.6:0/216993370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:22.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:22 smithi006 bash[33918]: cluster 2023-08-28T01:33:20.267656+0000 mgr.y (mgr.14901) 6442 : cluster [DBG] pgmap v6449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:22.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:22 smithi006 bash[33918]: audit 2023-08-28T01:33:21.459132+0000 mon.c (mon.2) 2523 : audit [DBG] from='client.? 172.21.15.6:0/216993370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:22 smithi130 bash[27579]: cluster 2023-08-28T01:33:20.267656+0000 mgr.y (mgr.14901) 6442 : cluster [DBG] pgmap v6449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:22 smithi130 bash[27579]: audit 2023-08-28T01:33:21.459132+0000 mon.c (mon.2) 2523 : audit [DBG] from='client.? 172.21.15.6:0/216993370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:33:22.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:33:22.620Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:23.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:22 smithi130 bash[51302]: debug 2023-08-28T01:33:22.687+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:22.693238+0000) 2023-08-28T01:33:24.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:23 smithi130 bash[51302]: debug 2023-08-28T01:33:23.691+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:23.693510+0000) 2023-08-28T01:33:24.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:24 smithi006 bash[33918]: cluster 2023-08-28T01:33:22.268344+0000 mgr.y (mgr.14901) 6443 : cluster [DBG] pgmap v6450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:24.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:24 smithi006 bash[33918]: audit 2023-08-28T01:33:23.880535+0000 mon.a (mon.0) 4280 : audit [DBG] from='client.? 172.21.15.6:0/2758883634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:24 smithi006 bash[27292]: cluster 2023-08-28T01:33:22.268344+0000 mgr.y (mgr.14901) 6443 : cluster [DBG] pgmap v6450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:24 smithi006 bash[27292]: audit 2023-08-28T01:33:23.880535+0000 mon.a (mon.0) 4280 : audit [DBG] from='client.? 172.21.15.6:0/2758883634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:24 smithi130 bash[27579]: cluster 2023-08-28T01:33:22.268344+0000 mgr.y (mgr.14901) 6443 : cluster [DBG] pgmap v6450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:24 smithi130 bash[27579]: audit 2023-08-28T01:33:23.880535+0000 mon.a (mon.0) 4280 : audit [DBG] from='client.? 172.21.15.6:0/2758883634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:33:24.499Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:33:24.501Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:25.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:24 smithi130 bash[51302]: debug 2023-08-28T01:33:24.687+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:24.693777+0000) 2023-08-28T01:33:26.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:25 smithi130 bash[51302]: debug 2023-08-28T01:33:25.691+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:25.694076+0000) 2023-08-28T01:33:26.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:26 smithi006 bash[33918]: cluster 2023-08-28T01:33:24.269541+0000 mgr.y (mgr.14901) 6444 : cluster [DBG] pgmap v6451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:26.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:26 smithi006 bash[27292]: cluster 2023-08-28T01:33:24.269541+0000 mgr.y (mgr.14901) 6444 : cluster [DBG] pgmap v6451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:26 smithi130 bash[27579]: cluster 2023-08-28T01:33:24.269541+0000 mgr.y (mgr.14901) 6444 : cluster [DBG] pgmap v6451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:27.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:26 smithi130 bash[51302]: debug 2023-08-28T01:33:26.687+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:26.694390+0000) 2023-08-28T01:33:27.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:27 smithi006 bash[27292]: audit 2023-08-28T01:33:26.290924+0000 mon.c (mon.2) 2524 : audit [DBG] from='client.? 172.21.15.6:0/2144771472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:27.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:27 smithi006 bash[33918]: audit 2023-08-28T01:33:26.290924+0000 mon.c (mon.2) 2524 : audit [DBG] from='client.? 172.21.15.6:0/2144771472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:27.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:27 smithi130 bash[27579]: audit 2023-08-28T01:33:26.290924+0000 mon.c (mon.2) 2524 : audit [DBG] from='client.? 172.21.15.6:0/2144771472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:28.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:27 smithi130 bash[51302]: debug 2023-08-28T01:33:27.686+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:27.694744+0000) 2023-08-28T01:33:28.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:28 smithi006 bash[27292]: cluster 2023-08-28T01:33:26.269984+0000 mgr.y (mgr.14901) 6445 : cluster [DBG] pgmap v6452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:28.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:28 smithi006 bash[33918]: cluster 2023-08-28T01:33:26.269984+0000 mgr.y (mgr.14901) 6445 : cluster [DBG] pgmap v6452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:28 smithi130 bash[27579]: cluster 2023-08-28T01:33:26.269984+0000 mgr.y (mgr.14901) 6445 : cluster [DBG] pgmap v6452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:28 smithi130 bash[51302]: debug 2023-08-28T01:33:28.690+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:28.695092+0000) 2023-08-28T01:33:29.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:29 smithi006 bash[27292]: audit 2023-08-28T01:33:28.712203+0000 mon.c (mon.2) 2525 : audit [DBG] from='client.? 172.21.15.6:0/1520210776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:29.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:29 smithi006 bash[33918]: audit 2023-08-28T01:33:28.712203+0000 mon.c (mon.2) 2525 : audit [DBG] from='client.? 172.21.15.6:0/1520210776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:29.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:29 smithi130 bash[27579]: audit 2023-08-28T01:33:28.712203+0000 mon.c (mon.2) 2525 : audit [DBG] from='client.? 172.21.15.6:0/1520210776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:30.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:29 smithi130 bash[51302]: debug 2023-08-28T01:33:29.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:29.695460+0000) 2023-08-28T01:33:30.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:30 smithi006 bash[27292]: cluster 2023-08-28T01:33:28.270736+0000 mgr.y (mgr.14901) 6446 : cluster [DBG] pgmap v6453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:30.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:30 smithi006 bash[33918]: cluster 2023-08-28T01:33:28.270736+0000 mgr.y (mgr.14901) 6446 : cluster [DBG] pgmap v6453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:30 smithi130 bash[27579]: cluster 2023-08-28T01:33:28.270736+0000 mgr.y (mgr.14901) 6446 : cluster [DBG] pgmap v6453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:31.049 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:33:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:33:30] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:33:31.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:30 smithi130 bash[51302]: debug 2023-08-28T01:33:30.690+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:30.695809+0000) 2023-08-28T01:33:31.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:31 smithi006 bash[27292]: audit 2023-08-28T01:33:31.136338+0000 mon.c (mon.2) 2526 : audit [DBG] from='client.? 172.21.15.6:0/3192827596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:31.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:31 smithi006 bash[33918]: audit 2023-08-28T01:33:31.136338+0000 mon.c (mon.2) 2526 : audit [DBG] from='client.? 172.21.15.6:0/3192827596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:33:31.048Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:33:31.050Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:31.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:31 smithi130 bash[27579]: audit 2023-08-28T01:33:31.136338+0000 mon.c (mon.2) 2526 : audit [DBG] from='client.? 172.21.15.6:0/3192827596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:32.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:31 smithi130 bash[51302]: debug 2023-08-28T01:33:31.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:31.696131+0000) 2023-08-28T01:33:32.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:32 smithi006 bash[27292]: cluster 2023-08-28T01:33:30.271880+0000 mgr.y (mgr.14901) 6447 : cluster [DBG] pgmap v6454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:32.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:32 smithi006 bash[33918]: cluster 2023-08-28T01:33:30.271880+0000 mgr.y (mgr.14901) 6447 : cluster [DBG] pgmap v6454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:32 smithi130 bash[27579]: cluster 2023-08-28T01:33:30.271880+0000 mgr.y (mgr.14901) 6447 : cluster [DBG] pgmap v6454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:33:32.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:33:32.620Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:33.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:32 smithi130 bash[51302]: debug 2023-08-28T01:33:32.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:32.696468+0000) 2023-08-28T01:33:34.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:33 smithi130 bash[51302]: debug 2023-08-28T01:33:33.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:33.696831+0000) 2023-08-28T01:33:34.500 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:34 smithi006 bash[27292]: cluster 2023-08-28T01:33:32.272527+0000 mgr.y (mgr.14901) 6448 : cluster [DBG] pgmap v6455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:34.500 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:34 smithi006 bash[27292]: audit 2023-08-28T01:33:33.548252+0000 mon.a (mon.0) 4281 : audit [DBG] from='client.? 172.21.15.6:0/1524607653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:34.501 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:34 smithi006 bash[33918]: cluster 2023-08-28T01:33:32.272527+0000 mgr.y (mgr.14901) 6448 : cluster [DBG] pgmap v6455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:34.501 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:34 smithi006 bash[33918]: audit 2023-08-28T01:33:33.548252+0000 mon.a (mon.0) 4281 : audit [DBG] from='client.? 172.21.15.6:0/1524607653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:34 smithi130 bash[27579]: cluster 2023-08-28T01:33:32.272527+0000 mgr.y (mgr.14901) 6448 : cluster [DBG] pgmap v6455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:34 smithi130 bash[27579]: audit 2023-08-28T01:33:33.548252+0000 mon.a (mon.0) 4281 : audit [DBG] from='client.? 172.21.15.6:0/1524607653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:33:34.499Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:33:34.502Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:35.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:34 smithi130 bash[51302]: debug 2023-08-28T01:33:34.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:34.697163+0000) 2023-08-28T01:33:36.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:35 smithi130 bash[51302]: debug 2023-08-28T01:33:35.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:35.697429+0000) 2023-08-28T01:33:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:36 smithi130 bash[27579]: cluster 2023-08-28T01:33:34.273684+0000 mgr.y (mgr.14901) 6449 : cluster [DBG] pgmap v6456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:36 smithi130 bash[27579]: audit 2023-08-28T01:33:35.961088+0000 mon.c (mon.2) 2527 : audit [DBG] from='client.? 172.21.15.6:0/1138886619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:36.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:36 smithi006 bash[33918]: cluster 2023-08-28T01:33:34.273684+0000 mgr.y (mgr.14901) 6449 : cluster [DBG] pgmap v6456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:36.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:36 smithi006 bash[33918]: audit 2023-08-28T01:33:35.961088+0000 mon.c (mon.2) 2527 : audit [DBG] from='client.? 172.21.15.6:0/1138886619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:36.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:36 smithi006 bash[27292]: cluster 2023-08-28T01:33:34.273684+0000 mgr.y (mgr.14901) 6449 : cluster [DBG] pgmap v6456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:36.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:36 smithi006 bash[27292]: audit 2023-08-28T01:33:35.961088+0000 mon.c (mon.2) 2527 : audit [DBG] from='client.? 172.21.15.6:0/1138886619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:37.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:36 smithi130 bash[51302]: debug 2023-08-28T01:33:36.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:36.697781+0000) 2023-08-28T01:33:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:37 smithi130 bash[51302]: debug 2023-08-28T01:33:37.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:37.698124+0000) 2023-08-28T01:33:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:38 smithi130 bash[27579]: cluster 2023-08-28T01:33:36.274322+0000 mgr.y (mgr.14901) 6450 : cluster [DBG] pgmap v6457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:38.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:38 smithi006 bash[33918]: cluster 2023-08-28T01:33:36.274322+0000 mgr.y (mgr.14901) 6450 : cluster [DBG] pgmap v6457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:38 smithi006 bash[27292]: cluster 2023-08-28T01:33:36.274322+0000 mgr.y (mgr.14901) 6450 : cluster [DBG] pgmap v6457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:39.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:38 smithi130 bash[51302]: debug 2023-08-28T01:33:38.694+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:38.698490+0000) 2023-08-28T01:33:39.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:39 smithi130 bash[27579]: audit 2023-08-28T01:33:38.382997+0000 mon.c (mon.2) 2528 : audit [DBG] from='client.? 172.21.15.6:0/2509478609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:39.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:39 smithi006 bash[27292]: audit 2023-08-28T01:33:38.382997+0000 mon.c (mon.2) 2528 : audit [DBG] from='client.? 172.21.15.6:0/2509478609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:39.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:39 smithi006 bash[33918]: audit 2023-08-28T01:33:38.382997+0000 mon.c (mon.2) 2528 : audit [DBG] from='client.? 172.21.15.6:0/2509478609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:40.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:39 smithi130 bash[51302]: debug 2023-08-28T01:33:39.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:39.698811+0000) 2023-08-28T01:33:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:40 smithi130 bash[27579]: cluster 2023-08-28T01:33:38.274945+0000 mgr.y (mgr.14901) 6451 : cluster [DBG] pgmap v6458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:40 smithi006 bash[33918]: cluster 2023-08-28T01:33:38.274945+0000 mgr.y (mgr.14901) 6451 : cluster [DBG] pgmap v6458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:40 smithi006 bash[27292]: cluster 2023-08-28T01:33:38.274945+0000 mgr.y (mgr.14901) 6451 : cluster [DBG] pgmap v6458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:41.049 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:33:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:33:40] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:33:41.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:40 smithi130 bash[51302]: debug 2023-08-28T01:33:40.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:40.699241+0000) 2023-08-28T01:33:41.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:41 smithi006 bash[33918]: audit 2023-08-28T01:33:40.803459+0000 mon.c (mon.2) 2529 : audit [DBG] from='client.? 172.21.15.6:0/2781238699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:41.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:41 smithi006 bash[27292]: audit 2023-08-28T01:33:40.803459+0000 mon.c (mon.2) 2529 : audit [DBG] from='client.? 172.21.15.6:0/2781238699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:33:41.049Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:33:41.051Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:41.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:41 smithi130 bash[27579]: audit 2023-08-28T01:33:40.803459+0000 mon.c (mon.2) 2529 : audit [DBG] from='client.? 172.21.15.6:0/2781238699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:41 smithi130 bash[51302]: debug 2023-08-28T01:33:41.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:41.699579+0000) 2023-08-28T01:33:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:42 smithi130 bash[27579]: cluster 2023-08-28T01:33:40.276064+0000 mgr.y (mgr.14901) 6452 : cluster [DBG] pgmap v6459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:42.620 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:42 smithi006 bash[27292]: cluster 2023-08-28T01:33:40.276064+0000 mgr.y (mgr.14901) 6452 : cluster [DBG] pgmap v6459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:42.620 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:42 smithi006 bash[33918]: cluster 2023-08-28T01:33:40.276064+0000 mgr.y (mgr.14901) 6452 : cluster [DBG] pgmap v6459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:33:42.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:33:42.621Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:43.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:42 smithi130 bash[51302]: debug 2023-08-28T01:33:42.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:42.699916+0000) 2023-08-28T01:33:43.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:43 smithi130 bash[27579]: cluster 2023-08-28T01:33:42.276739+0000 mgr.y (mgr.14901) 6453 : cluster [DBG] pgmap v6460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:43.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:43 smithi130 bash[27579]: audit 2023-08-28T01:33:43.227748+0000 mon.c (mon.2) 2530 : audit [DBG] from='client.? 172.21.15.6:0/1474096807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:43.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:43 smithi006 bash[33918]: cluster 2023-08-28T01:33:42.276739+0000 mgr.y (mgr.14901) 6453 : cluster [DBG] pgmap v6460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:43.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:43 smithi006 bash[33918]: audit 2023-08-28T01:33:43.227748+0000 mon.c (mon.2) 2530 : audit [DBG] from='client.? 172.21.15.6:0/1474096807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:43.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:43 smithi006 bash[27292]: cluster 2023-08-28T01:33:42.276739+0000 mgr.y (mgr.14901) 6453 : cluster [DBG] pgmap v6460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:43.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:43 smithi006 bash[27292]: audit 2023-08-28T01:33:43.227748+0000 mon.c (mon.2) 2530 : audit [DBG] from='client.? 172.21.15.6:0/1474096807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:44.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:43 smithi130 bash[51302]: debug 2023-08-28T01:33:43.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:43.700266+0000) 2023-08-28T01:33:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:33:44.501Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:33:44.503Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:45.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:44 smithi130 bash[51302]: debug 2023-08-28T01:33:44.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:44.700636+0000) 2023-08-28T01:33:45.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:45 smithi130 bash[27579]: cluster 2023-08-28T01:33:44.277722+0000 mgr.y (mgr.14901) 6454 : cluster [DBG] pgmap v6461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:45.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:45 smithi006 bash[33918]: cluster 2023-08-28T01:33:44.277722+0000 mgr.y (mgr.14901) 6454 : cluster [DBG] pgmap v6461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:45.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:45 smithi006 bash[27292]: cluster 2023-08-28T01:33:44.277722+0000 mgr.y (mgr.14901) 6454 : cluster [DBG] pgmap v6461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:45 smithi130 bash[51302]: debug 2023-08-28T01:33:45.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:45.700979+0000) 2023-08-28T01:33:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:46 smithi130 bash[27579]: audit 2023-08-28T01:33:45.649331+0000 mon.c (mon.2) 2531 : audit [DBG] from='client.? 172.21.15.6:0/1816473267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:46.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:46 smithi006 bash[33918]: audit 2023-08-28T01:33:45.649331+0000 mon.c (mon.2) 2531 : audit [DBG] from='client.? 172.21.15.6:0/1816473267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:46.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:46 smithi006 bash[27292]: audit 2023-08-28T01:33:45.649331+0000 mon.c (mon.2) 2531 : audit [DBG] from='client.? 172.21.15.6:0/1816473267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:46 smithi130 bash[51302]: debug 2023-08-28T01:33:46.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:46.701307+0000) 2023-08-28T01:33:48.029 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:47 smithi130 bash[51302]: debug 2023-08-28T01:33:47.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:47.701648+0000) 2023-08-28T01:33:48.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:48 smithi130 bash[27579]: cluster 2023-08-28T01:33:46.278371+0000 mgr.y (mgr.14901) 6455 : cluster [DBG] pgmap v6462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:48.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:48 smithi006 bash[33918]: cluster 2023-08-28T01:33:46.278371+0000 mgr.y (mgr.14901) 6455 : cluster [DBG] pgmap v6462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:48.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:48 smithi006 bash[27292]: cluster 2023-08-28T01:33:46.278371+0000 mgr.y (mgr.14901) 6455 : cluster [DBG] pgmap v6462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:49.037 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:48 smithi130 bash[51302]: debug 2023-08-28T01:33:48.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:48.701971+0000) 2023-08-28T01:33:49.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:49 smithi130 bash[27579]: audit 2023-08-28T01:33:48.064411+0000 mon.c (mon.2) 2532 : audit [DBG] from='client.? 172.21.15.6:0/2102195047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:49.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:49 smithi006 bash[27292]: audit 2023-08-28T01:33:48.064411+0000 mon.c (mon.2) 2532 : audit [DBG] from='client.? 172.21.15.6:0/2102195047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:49.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:49 smithi006 bash[33918]: audit 2023-08-28T01:33:48.064411+0000 mon.c (mon.2) 2532 : audit [DBG] from='client.? 172.21.15.6:0/2102195047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:50.040 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:49 smithi130 bash[51302]: debug 2023-08-28T01:33:49.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:49.702294+0000) 2023-08-28T01:33:50.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:50 smithi130 bash[27579]: cluster 2023-08-28T01:33:48.278969+0000 mgr.y (mgr.14901) 6456 : cluster [DBG] pgmap v6463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:50.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:50 smithi006 bash[33918]: cluster 2023-08-28T01:33:48.278969+0000 mgr.y (mgr.14901) 6456 : cluster [DBG] pgmap v6463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:50.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:50 smithi006 bash[27292]: cluster 2023-08-28T01:33:48.278969+0000 mgr.y (mgr.14901) 6456 : cluster [DBG] pgmap v6463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:51.047 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:33:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:33:50] "GET /metrics HTTP/1.1" 200 193132 "" "Prometheus/2.33.4" 2023-08-28T01:33:51.048 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:50 smithi130 bash[51302]: debug 2023-08-28T01:33:50.698+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:50.702637+0000) 2023-08-28T01:33:51.048 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:51 smithi006 bash[27292]: audit 2023-08-28T01:33:50.496923+0000 mon.c (mon.2) 2533 : audit [DBG] from='client.? 172.21.15.6:0/37833651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:51.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:51 smithi130 bash[27579]: audit 2023-08-28T01:33:50.496923+0000 mon.c (mon.2) 2533 : audit [DBG] from='client.? 172.21.15.6:0/37833651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:51.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:51 smithi006 bash[33918]: audit 2023-08-28T01:33:50.496923+0000 mon.c (mon.2) 2533 : audit [DBG] from='client.? 172.21.15.6:0/37833651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:33:51.050Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:33:51.052Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:52.050 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:51 smithi130 bash[51302]: debug 2023-08-28T01:33:51.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:51.703015+0000) 2023-08-28T01:33:52.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:52 smithi130 bash[27579]: cluster 2023-08-28T01:33:50.279731+0000 mgr.y (mgr.14901) 6457 : cluster [DBG] pgmap v6464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:52.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:52 smithi006 bash[33918]: cluster 2023-08-28T01:33:50.279731+0000 mgr.y (mgr.14901) 6457 : cluster [DBG] pgmap v6464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:52.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:52 smithi006 bash[27292]: cluster 2023-08-28T01:33:50.279731+0000 mgr.y (mgr.14901) 6457 : cluster [DBG] pgmap v6464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:33:52.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:33:52.622Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:53.053 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:52 smithi130 bash[51302]: debug 2023-08-28T01:33:52.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:52.703277+0000) 2023-08-28T01:33:53.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:53 smithi130 bash[27579]: audit 2023-08-28T01:33:52.917338+0000 mon.c (mon.2) 2534 : audit [DBG] from='client.? 172.21.15.6:0/4285496431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:53.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:53 smithi006 bash[33918]: audit 2023-08-28T01:33:52.917338+0000 mon.c (mon.2) 2534 : audit [DBG] from='client.? 172.21.15.6:0/4285496431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:53.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:53 smithi006 bash[27292]: audit 2023-08-28T01:33:52.917338+0000 mon.c (mon.2) 2534 : audit [DBG] from='client.? 172.21.15.6:0/4285496431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:54.061 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:53 smithi130 bash[51302]: debug 2023-08-28T01:33:53.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:53.703549+0000) 2023-08-28T01:33:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:54 smithi130 bash[27579]: cluster 2023-08-28T01:33:52.280335+0000 mgr.y (mgr.14901) 6458 : cluster [DBG] pgmap v6465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:54 smithi006 bash[27292]: cluster 2023-08-28T01:33:52.280335+0000 mgr.y (mgr.14901) 6458 : cluster [DBG] pgmap v6465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:54 smithi006 bash[33918]: cluster 2023-08-28T01:33:52.280335+0000 mgr.y (mgr.14901) 6458 : cluster [DBG] pgmap v6465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:33:54.501Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:33:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:33:54.503Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:33:55.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:54 smithi130 bash[51302]: debug 2023-08-28T01:33:54.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:54.703915+0000) 2023-08-28T01:33:56.065 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:55 smithi130 bash[51302]: debug 2023-08-28T01:33:55.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:55.704224+0000) 2023-08-28T01:33:56.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:56 smithi130 bash[27579]: cluster 2023-08-28T01:33:54.281438+0000 mgr.y (mgr.14901) 6459 : cluster [DBG] pgmap v6466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:56.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:56 smithi130 bash[27579]: audit 2023-08-28T01:33:55.336151+0000 mon.a (mon.0) 4282 : audit [DBG] from='client.? 172.21.15.6:0/2214470762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:56.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:56 smithi006 bash[27292]: cluster 2023-08-28T01:33:54.281438+0000 mgr.y (mgr.14901) 6459 : cluster [DBG] pgmap v6466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:56.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:56 smithi006 bash[27292]: audit 2023-08-28T01:33:55.336151+0000 mon.a (mon.0) 4282 : audit [DBG] from='client.? 172.21.15.6:0/2214470762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:56.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:56 smithi006 bash[33918]: cluster 2023-08-28T01:33:54.281438+0000 mgr.y (mgr.14901) 6459 : cluster [DBG] pgmap v6466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:56.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:56 smithi006 bash[33918]: audit 2023-08-28T01:33:55.336151+0000 mon.a (mon.0) 4282 : audit [DBG] from='client.? 172.21.15.6:0/2214470762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:57.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:56 smithi130 bash[51302]: debug 2023-08-28T01:33:56.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:56.704596+0000) 2023-08-28T01:33:58.072 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:57 smithi130 bash[51302]: debug 2023-08-28T01:33:57.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:57.704929+0000) 2023-08-28T01:33:58.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:58 smithi130 bash[27579]: cluster 2023-08-28T01:33:56.282140+0000 mgr.y (mgr.14901) 6460 : cluster [DBG] pgmap v6467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:58.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:33:58 smithi130 bash[27579]: audit 2023-08-28T01:33:57.750629+0000 mon.c (mon.2) 2535 : audit [DBG] from='client.? 172.21.15.6:0/898980408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:58.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:58 smithi006 bash[27292]: cluster 2023-08-28T01:33:56.282140+0000 mgr.y (mgr.14901) 6460 : cluster [DBG] pgmap v6467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:58.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:33:58 smithi006 bash[27292]: audit 2023-08-28T01:33:57.750629+0000 mon.c (mon.2) 2535 : audit [DBG] from='client.? 172.21.15.6:0/898980408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:58.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:58 smithi006 bash[33918]: cluster 2023-08-28T01:33:56.282140+0000 mgr.y (mgr.14901) 6460 : cluster [DBG] pgmap v6467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:33:58.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:33:58 smithi006 bash[33918]: audit 2023-08-28T01:33:57.750629+0000 mon.c (mon.2) 2535 : audit [DBG] from='client.? 172.21.15.6:0/898980408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:59.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:58 smithi130 bash[51302]: debug 2023-08-28T01:33:58.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:58.705234+0000) 2023-08-28T01:34:00.077 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:33:59 smithi130 bash[51302]: debug 2023-08-28T01:33:59.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:59.705577+0000) 2023-08-28T01:34:00.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:00 smithi130 bash[27579]: cluster 2023-08-28T01:33:58.282792+0000 mgr.y (mgr.14901) 6461 : cluster [DBG] pgmap v6468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:00.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:00 smithi130 bash[27579]: audit 2023-08-28T01:33:59.778528+0000 mon.b (mon.1) 736 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:34:00.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:00 smithi006 bash[33918]: cluster 2023-08-28T01:33:58.282792+0000 mgr.y (mgr.14901) 6461 : cluster [DBG] pgmap v6468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:00.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:00 smithi006 bash[33918]: audit 2023-08-28T01:33:59.778528+0000 mon.b (mon.1) 736 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:34:00.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:00 smithi006 bash[27292]: cluster 2023-08-28T01:33:58.282792+0000 mgr.y (mgr.14901) 6461 : cluster [DBG] pgmap v6468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:00.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:00 smithi006 bash[27292]: audit 2023-08-28T01:33:59.778528+0000 mon.b (mon.1) 736 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:34:01.051 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:34:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:34:00] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T01:34:01.086 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:00 smithi130 bash[51302]: debug 2023-08-28T01:34:00.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:00.705962+0000) 2023-08-28T01:34:01.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:01 smithi130 bash[27579]: audit 2023-08-28T01:34:00.185027+0000 mon.a (mon.0) 4283 : audit [DBG] from='client.? 172.21.15.6:0/2319334772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:01.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:01 smithi006 bash[27292]: audit 2023-08-28T01:34:00.185027+0000 mon.a (mon.0) 4283 : audit [DBG] from='client.? 172.21.15.6:0/2319334772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:01.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:01 smithi006 bash[33918]: audit 2023-08-28T01:34:00.185027+0000 mon.a (mon.0) 4283 : audit [DBG] from='client.? 172.21.15.6:0/2319334772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:34:01.051Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:01.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:34:01.053Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:02.091 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:01 smithi130 bash[51302]: debug 2023-08-28T01:34:01.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:01.706295+0000) 2023-08-28T01:34:02.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:02 smithi130 bash[27579]: cluster 2023-08-28T01:34:00.283944+0000 mgr.y (mgr.14901) 6462 : cluster [DBG] pgmap v6469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:02.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:02 smithi006 bash[27292]: cluster 2023-08-28T01:34:00.283944+0000 mgr.y (mgr.14901) 6462 : cluster [DBG] pgmap v6469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:02.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:02 smithi006 bash[33918]: cluster 2023-08-28T01:34:00.283944+0000 mgr.y (mgr.14901) 6462 : cluster [DBG] pgmap v6469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:34:02.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:34:02.623Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:03.098 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:02 smithi130 bash[51302]: debug 2023-08-28T01:34:02.702+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:02.706564+0000) 2023-08-28T01:34:03.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:03 smithi130 bash[27579]: audit 2023-08-28T01:34:02.596961+0000 mon.c (mon.2) 2536 : audit [DBG] from='client.? 172.21.15.6:0/1386544466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:03.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:03 smithi006 bash[27292]: audit 2023-08-28T01:34:02.596961+0000 mon.c (mon.2) 2536 : audit [DBG] from='client.? 172.21.15.6:0/1386544466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:03.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:03 smithi006 bash[33918]: audit 2023-08-28T01:34:02.596961+0000 mon.c (mon.2) 2536 : audit [DBG] from='client.? 172.21.15.6:0/1386544466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:04.101 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:03 smithi130 bash[51302]: debug 2023-08-28T01:34:03.706+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:03.706934+0000) 2023-08-28T01:34:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:04 smithi130 bash[27579]: cluster 2023-08-28T01:34:02.284702+0000 mgr.y (mgr.14901) 6463 : cluster [DBG] pgmap v6470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:04 smithi006 bash[27292]: cluster 2023-08-28T01:34:02.284702+0000 mgr.y (mgr.14901) 6463 : cluster [DBG] pgmap v6470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:04.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:04 smithi006 bash[33918]: cluster 2023-08-28T01:34:02.284702+0000 mgr.y (mgr.14901) 6463 : cluster [DBG] pgmap v6470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:34:04.503Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:34:04.505Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:05.108 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:04 smithi130 bash[51302]: debug 2023-08-28T01:34:04.706+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:04.707212+0000) 2023-08-28T01:34:05.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:05 smithi130 bash[27579]: audit 2023-08-28T01:34:05.021910+0000 mon.c (mon.2) 2537 : audit [DBG] from='client.? 172.21.15.6:0/1525482526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:05.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:05 smithi006 bash[27292]: audit 2023-08-28T01:34:05.021910+0000 mon.c (mon.2) 2537 : audit [DBG] from='client.? 172.21.15.6:0/1525482526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:05.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:05 smithi006 bash[33918]: audit 2023-08-28T01:34:05.021910+0000 mon.c (mon.2) 2537 : audit [DBG] from='client.? 172.21.15.6:0/1525482526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:06.111 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:05 smithi130 bash[51302]: debug 2023-08-28T01:34:05.706+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:05.707478+0000) 2023-08-28T01:34:06.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:06 smithi130 bash[27579]: cluster 2023-08-28T01:34:04.285914+0000 mgr.y (mgr.14901) 6464 : cluster [DBG] pgmap v6471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:06.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:06 smithi130 bash[27579]: audit 2023-08-28T01:34:05.666015+0000 mon.a (mon.0) 4284 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:34:06.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:06 smithi130 bash[27579]: audit 2023-08-28T01:34:06.088944+0000 mon.a (mon.0) 4285 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:34:06.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:06 smithi006 bash[27292]: cluster 2023-08-28T01:34:04.285914+0000 mgr.y (mgr.14901) 6464 : cluster [DBG] pgmap v6471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:06.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:06 smithi006 bash[27292]: audit 2023-08-28T01:34:05.666015+0000 mon.a (mon.0) 4284 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:34:06.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:06 smithi006 bash[27292]: audit 2023-08-28T01:34:06.088944+0000 mon.a (mon.0) 4285 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:34:06.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:06 smithi006 bash[33918]: cluster 2023-08-28T01:34:04.285914+0000 mgr.y (mgr.14901) 6464 : cluster [DBG] pgmap v6471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:06.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:06 smithi006 bash[33918]: audit 2023-08-28T01:34:05.666015+0000 mon.a (mon.0) 4284 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:34:06.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:06 smithi006 bash[33918]: audit 2023-08-28T01:34:06.088944+0000 mon.a (mon.0) 4285 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:34:07.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:06 smithi130 bash[51302]: debug 2023-08-28T01:34:06.706+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:06.707800+0000) 2023-08-28T01:34:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:07 smithi006 bash[27292]: audit 2023-08-28T01:34:06.383042+0000 mon.b (mon.1) 737 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:34:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:07 smithi006 bash[27292]: audit 2023-08-28T01:34:06.385238+0000 mon.b (mon.1) 738 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:34:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:07 smithi006 bash[27292]: audit 2023-08-28T01:34:06.392210+0000 mon.a (mon.0) 4286 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:34:07.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:07 smithi006 bash[33918]: audit 2023-08-28T01:34:06.383042+0000 mon.b (mon.1) 737 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:34:07.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:07 smithi006 bash[33918]: audit 2023-08-28T01:34:06.385238+0000 mon.b (mon.1) 738 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:34:07.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:07 smithi006 bash[33918]: audit 2023-08-28T01:34:06.392210+0000 mon.a (mon.0) 4286 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:34:07.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:07 smithi130 bash[27579]: audit 2023-08-28T01:34:06.383042+0000 mon.b (mon.1) 737 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:34:07.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:07 smithi130 bash[27579]: audit 2023-08-28T01:34:06.385238+0000 mon.b (mon.1) 738 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:34:07.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:07 smithi130 bash[27579]: audit 2023-08-28T01:34:06.392210+0000 mon.a (mon.0) 4286 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:34:08.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:07 smithi130 bash[51302]: debug 2023-08-28T01:34:07.706+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:07.708141+0000) 2023-08-28T01:34:08.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:08 smithi006 bash[33918]: cluster 2023-08-28T01:34:06.286628+0000 mgr.y (mgr.14901) 6465 : cluster [DBG] pgmap v6472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:08.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:08 smithi006 bash[33918]: audit 2023-08-28T01:34:07.431607+0000 mon.a (mon.0) 4287 : audit [DBG] from='client.? 172.21.15.6:0/283102987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:08.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:08 smithi006 bash[27292]: cluster 2023-08-28T01:34:06.286628+0000 mgr.y (mgr.14901) 6465 : cluster [DBG] pgmap v6472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:08.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:08 smithi006 bash[27292]: audit 2023-08-28T01:34:07.431607+0000 mon.a (mon.0) 4287 : audit [DBG] from='client.? 172.21.15.6:0/283102987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:08 smithi130 bash[27579]: cluster 2023-08-28T01:34:06.286628+0000 mgr.y (mgr.14901) 6465 : cluster [DBG] pgmap v6472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:08 smithi130 bash[27579]: audit 2023-08-28T01:34:07.431607+0000 mon.a (mon.0) 4287 : audit [DBG] from='client.? 172.21.15.6:0/283102987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:09.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:08 smithi130 bash[51302]: debug 2023-08-28T01:34:08.706+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:08.708466+0000) 2023-08-28T01:34:10.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:09 smithi130 bash[51302]: debug 2023-08-28T01:34:09.706+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:09.708793+0000) 2023-08-28T01:34:10.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:10 smithi006 bash[33918]: cluster 2023-08-28T01:34:08.287349+0000 mgr.y (mgr.14901) 6466 : cluster [DBG] pgmap v6473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:10.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:10 smithi006 bash[33918]: audit 2023-08-28T01:34:09.842506+0000 mon.a (mon.0) 4288 : audit [DBG] from='client.? 172.21.15.6:0/2045015659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:10.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:10 smithi006 bash[27292]: cluster 2023-08-28T01:34:08.287349+0000 mgr.y (mgr.14901) 6466 : cluster [DBG] pgmap v6473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:10.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:10 smithi006 bash[27292]: audit 2023-08-28T01:34:09.842506+0000 mon.a (mon.0) 4288 : audit [DBG] from='client.? 172.21.15.6:0/2045015659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:10 smithi130 bash[27579]: cluster 2023-08-28T01:34:08.287349+0000 mgr.y (mgr.14901) 6466 : cluster [DBG] pgmap v6473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:10 smithi130 bash[27579]: audit 2023-08-28T01:34:09.842506+0000 mon.a (mon.0) 4288 : audit [DBG] from='client.? 172.21.15.6:0/2045015659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:11.052 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:34:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:34:10] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T01:34:11.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:10 smithi130 bash[51302]: debug 2023-08-28T01:34:10.706+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:10.709167+0000) 2023-08-28T01:34:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:34:11.051Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:34:11.053Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:12.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:11 smithi130 bash[51302]: debug 2023-08-28T01:34:11.706+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:11.709516+0000) 2023-08-28T01:34:12.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:12 smithi006 bash[33918]: cluster 2023-08-28T01:34:10.288637+0000 mgr.y (mgr.14901) 6467 : cluster [DBG] pgmap v6474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:12.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:12 smithi006 bash[27292]: cluster 2023-08-28T01:34:10.288637+0000 mgr.y (mgr.14901) 6467 : cluster [DBG] pgmap v6474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:12 smithi130 bash[27579]: cluster 2023-08-28T01:34:10.288637+0000 mgr.y (mgr.14901) 6467 : cluster [DBG] pgmap v6474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:34:12.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:34:12.624Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:13.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:12 smithi130 bash[51302]: debug 2023-08-28T01:34:12.706+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:12.709852+0000) 2023-08-28T01:34:13.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:13 smithi006 bash[33918]: audit 2023-08-28T01:34:12.262806+0000 mon.a (mon.0) 4289 : audit [DBG] from='client.? 172.21.15.6:0/3296076613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:13.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:13 smithi006 bash[27292]: audit 2023-08-28T01:34:12.262806+0000 mon.a (mon.0) 4289 : audit [DBG] from='client.? 172.21.15.6:0/3296076613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:13.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:13 smithi130 bash[27579]: audit 2023-08-28T01:34:12.262806+0000 mon.a (mon.0) 4289 : audit [DBG] from='client.? 172.21.15.6:0/3296076613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:14.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:13 smithi130 bash[51302]: debug 2023-08-28T01:34:13.706+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:13.710175+0000) 2023-08-28T01:34:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:14 smithi006 bash[27292]: cluster 2023-08-28T01:34:12.289474+0000 mgr.y (mgr.14901) 6468 : cluster [DBG] pgmap v6475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:14 smithi006 bash[33918]: cluster 2023-08-28T01:34:12.289474+0000 mgr.y (mgr.14901) 6468 : cluster [DBG] pgmap v6475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:14 smithi130 bash[27579]: cluster 2023-08-28T01:34:12.289474+0000 mgr.y (mgr.14901) 6468 : cluster [DBG] pgmap v6475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:34:14.504Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:34:14.505Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:14 smithi130 bash[51302]: debug 2023-08-28T01:34:14.706+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:14.710532+0000) 2023-08-28T01:34:15.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:15 smithi006 bash[27292]: audit 2023-08-28T01:34:14.686567+0000 mon.a (mon.0) 4290 : audit [DBG] from='client.? 172.21.15.6:0/3401486594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:15.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:15 smithi006 bash[33918]: audit 2023-08-28T01:34:14.686567+0000 mon.a (mon.0) 4290 : audit [DBG] from='client.? 172.21.15.6:0/3401486594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:15.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:15 smithi130 bash[27579]: audit 2023-08-28T01:34:14.686567+0000 mon.a (mon.0) 4290 : audit [DBG] from='client.? 172.21.15.6:0/3401486594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:16.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:15 smithi130 bash[51302]: debug 2023-08-28T01:34:15.710+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:15.710900+0000) 2023-08-28T01:34:16.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:16 smithi006 bash[27292]: cluster 2023-08-28T01:34:14.290622+0000 mgr.y (mgr.14901) 6469 : cluster [DBG] pgmap v6476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:16.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:16 smithi006 bash[33918]: cluster 2023-08-28T01:34:14.290622+0000 mgr.y (mgr.14901) 6469 : cluster [DBG] pgmap v6476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:16 smithi130 bash[27579]: cluster 2023-08-28T01:34:14.290622+0000 mgr.y (mgr.14901) 6469 : cluster [DBG] pgmap v6476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:17.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:16 smithi130 bash[51302]: debug 2023-08-28T01:34:16.710+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:16.711163+0000) 2023-08-28T01:34:18.036 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:17 smithi130 bash[51302]: debug 2023-08-28T01:34:17.710+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:17.711496+0000) 2023-08-28T01:34:18.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:18 smithi130 bash[27579]: cluster 2023-08-28T01:34:16.291435+0000 mgr.y (mgr.14901) 6470 : cluster [DBG] pgmap v6477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:18.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:18 smithi130 bash[27579]: audit 2023-08-28T01:34:17.104409+0000 mon.c (mon.2) 2538 : audit [DBG] from='client.? 172.21.15.6:0/3695386024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:18.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:18 smithi006 bash[27292]: cluster 2023-08-28T01:34:16.291435+0000 mgr.y (mgr.14901) 6470 : cluster [DBG] pgmap v6477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:18.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:18 smithi006 bash[27292]: audit 2023-08-28T01:34:17.104409+0000 mon.c (mon.2) 2538 : audit [DBG] from='client.? 172.21.15.6:0/3695386024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:18.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:18 smithi006 bash[33918]: cluster 2023-08-28T01:34:16.291435+0000 mgr.y (mgr.14901) 6470 : cluster [DBG] pgmap v6477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:18.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:18 smithi006 bash[33918]: audit 2023-08-28T01:34:17.104409+0000 mon.c (mon.2) 2538 : audit [DBG] from='client.? 172.21.15.6:0/3695386024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:19.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:18 smithi130 bash[51302]: debug 2023-08-28T01:34:18.710+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:18.711833+0000) 2023-08-28T01:34:20.041 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:19 smithi130 bash[51302]: debug 2023-08-28T01:34:19.710+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:19.712157+0000) 2023-08-28T01:34:20.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:20 smithi130 bash[27579]: cluster 2023-08-28T01:34:18.292268+0000 mgr.y (mgr.14901) 6471 : cluster [DBG] pgmap v6478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:20.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:20 smithi130 bash[27579]: audit 2023-08-28T01:34:19.518112+0000 mon.a (mon.0) 4291 : audit [DBG] from='client.? 172.21.15.6:0/2920179938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:20.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:20 smithi006 bash[33918]: cluster 2023-08-28T01:34:18.292268+0000 mgr.y (mgr.14901) 6471 : cluster [DBG] pgmap v6478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:20.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:20 smithi006 bash[33918]: audit 2023-08-28T01:34:19.518112+0000 mon.a (mon.0) 4291 : audit [DBG] from='client.? 172.21.15.6:0/2920179938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:20.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:20 smithi006 bash[27292]: cluster 2023-08-28T01:34:18.292268+0000 mgr.y (mgr.14901) 6471 : cluster [DBG] pgmap v6478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:20.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:20 smithi006 bash[27292]: audit 2023-08-28T01:34:19.518112+0000 mon.a (mon.0) 4291 : audit [DBG] from='client.? 172.21.15.6:0/2920179938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:21.052 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:34:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:34:20] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:34:21.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:20 smithi130 bash[51302]: debug 2023-08-28T01:34:20.710+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:20.712514+0000) 2023-08-28T01:34:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:34:21.051Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:34:21.053Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:22.049 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:21 smithi130 bash[51302]: debug 2023-08-28T01:34:21.710+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:21.712841+0000) 2023-08-28T01:34:22.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:22 smithi130 bash[27579]: cluster 2023-08-28T01:34:20.293456+0000 mgr.y (mgr.14901) 6472 : cluster [DBG] pgmap v6479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:22.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:22 smithi130 bash[27579]: audit 2023-08-28T01:34:21.926963+0000 mon.a (mon.0) 4292 : audit [DBG] from='client.? 172.21.15.6:0/428137054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:22.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:22 smithi006 bash[27292]: cluster 2023-08-28T01:34:20.293456+0000 mgr.y (mgr.14901) 6472 : cluster [DBG] pgmap v6479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:22.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:22 smithi006 bash[27292]: audit 2023-08-28T01:34:21.926963+0000 mon.a (mon.0) 4292 : audit [DBG] from='client.? 172.21.15.6:0/428137054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:22.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:22 smithi006 bash[33918]: cluster 2023-08-28T01:34:20.293456+0000 mgr.y (mgr.14901) 6472 : cluster [DBG] pgmap v6479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:22.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:22 smithi006 bash[33918]: audit 2023-08-28T01:34:21.926963+0000 mon.a (mon.0) 4292 : audit [DBG] from='client.? 172.21.15.6:0/428137054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:34:22.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:34:22.624Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:23.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:22 smithi130 bash[51302]: debug 2023-08-28T01:34:22.710+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:22.713165+0000) 2023-08-28T01:34:24.056 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:23 smithi130 bash[51302]: debug 2023-08-28T01:34:23.709+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:23.713499+0000) 2023-08-28T01:34:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:24 smithi130 bash[27579]: cluster 2023-08-28T01:34:22.294086+0000 mgr.y (mgr.14901) 6473 : cluster [DBG] pgmap v6480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:24 smithi006 bash[27292]: cluster 2023-08-28T01:34:22.294086+0000 mgr.y (mgr.14901) 6473 : cluster [DBG] pgmap v6480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:24.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:24 smithi006 bash[33918]: cluster 2023-08-28T01:34:22.294086+0000 mgr.y (mgr.14901) 6473 : cluster [DBG] pgmap v6480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:34:24.504Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:34:24.507Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:25.063 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:24 smithi130 bash[51302]: debug 2023-08-28T01:34:24.709+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:24.713823+0000) 2023-08-28T01:34:25.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:25 smithi130 bash[27579]: audit 2023-08-28T01:34:24.340819+0000 mon.c (mon.2) 2539 : audit [DBG] from='client.? 172.21.15.6:0/62984246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:25.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:25 smithi006 bash[33918]: audit 2023-08-28T01:34:24.340819+0000 mon.c (mon.2) 2539 : audit [DBG] from='client.? 172.21.15.6:0/62984246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:25.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:25 smithi006 bash[27292]: audit 2023-08-28T01:34:24.340819+0000 mon.c (mon.2) 2539 : audit [DBG] from='client.? 172.21.15.6:0/62984246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:26.066 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:25 smithi130 bash[51302]: debug 2023-08-28T01:34:25.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:25.714152+0000) 2023-08-28T01:34:26.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:26 smithi130 bash[27579]: cluster 2023-08-28T01:34:24.295152+0000 mgr.y (mgr.14901) 6474 : cluster [DBG] pgmap v6481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:26.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:26 smithi006 bash[27292]: cluster 2023-08-28T01:34:24.295152+0000 mgr.y (mgr.14901) 6474 : cluster [DBG] pgmap v6481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:26.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:26 smithi006 bash[33918]: cluster 2023-08-28T01:34:24.295152+0000 mgr.y (mgr.14901) 6474 : cluster [DBG] pgmap v6481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:27.075 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:26 smithi130 bash[51302]: debug 2023-08-28T01:34:26.709+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:26.714478+0000) 2023-08-28T01:34:27.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:27 smithi130 bash[27579]: audit 2023-08-28T01:34:26.758598+0000 mon.a (mon.0) 4293 : audit [DBG] from='client.? 172.21.15.6:0/2307641939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:27.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:27 smithi006 bash[27292]: audit 2023-08-28T01:34:26.758598+0000 mon.a (mon.0) 4293 : audit [DBG] from='client.? 172.21.15.6:0/2307641939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:27.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:27 smithi006 bash[33918]: audit 2023-08-28T01:34:26.758598+0000 mon.a (mon.0) 4293 : audit [DBG] from='client.? 172.21.15.6:0/2307641939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:28.078 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:27 smithi130 bash[51302]: debug 2023-08-28T01:34:27.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:27.714790+0000) 2023-08-28T01:34:28.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:28 smithi130 bash[27579]: cluster 2023-08-28T01:34:26.295837+0000 mgr.y (mgr.14901) 6475 : cluster [DBG] pgmap v6482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:28.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:28 smithi006 bash[33918]: cluster 2023-08-28T01:34:26.295837+0000 mgr.y (mgr.14901) 6475 : cluster [DBG] pgmap v6482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:28.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:28 smithi006 bash[27292]: cluster 2023-08-28T01:34:26.295837+0000 mgr.y (mgr.14901) 6475 : cluster [DBG] pgmap v6482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:28 smithi130 bash[51302]: debug 2023-08-28T01:34:28.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:28.715122+0000) 2023-08-28T01:34:30.085 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:29 smithi130 bash[51302]: debug 2023-08-28T01:34:29.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:29.715454+0000) 2023-08-28T01:34:30.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:30 smithi130 bash[27579]: cluster 2023-08-28T01:34:28.296487+0000 mgr.y (mgr.14901) 6476 : cluster [DBG] pgmap v6483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:30.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:30 smithi130 bash[27579]: audit 2023-08-28T01:34:29.189625+0000 mon.a (mon.0) 4294 : audit [DBG] from='client.? 172.21.15.6:0/3344361424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:30.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:30 smithi006 bash[27292]: cluster 2023-08-28T01:34:28.296487+0000 mgr.y (mgr.14901) 6476 : cluster [DBG] pgmap v6483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:30.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:30 smithi006 bash[27292]: audit 2023-08-28T01:34:29.189625+0000 mon.a (mon.0) 4294 : audit [DBG] from='client.? 172.21.15.6:0/3344361424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:30.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:30 smithi006 bash[33918]: cluster 2023-08-28T01:34:28.296487+0000 mgr.y (mgr.14901) 6476 : cluster [DBG] pgmap v6483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:30.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:30 smithi006 bash[33918]: audit 2023-08-28T01:34:29.189625+0000 mon.a (mon.0) 4294 : audit [DBG] from='client.? 172.21.15.6:0/3344361424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:31.053 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:34:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:34:30] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T01:34:31.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:30 smithi130 bash[51302]: debug 2023-08-28T01:34:30.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:30.715806+0000) 2023-08-28T01:34:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:34:31.053Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:34:31.054Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:32.094 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:31 smithi130 bash[51302]: debug 2023-08-28T01:34:31.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:31.716173+0000) 2023-08-28T01:34:32.095 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:32 smithi130 bash[27579]: cluster 2023-08-28T01:34:30.297427+0000 mgr.y (mgr.14901) 6477 : cluster [DBG] pgmap v6484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:32.095 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:32 smithi130 bash[27579]: audit 2023-08-28T01:34:31.596441+0000 mon.c (mon.2) 2540 : audit [DBG] from='client.? 172.21.15.6:0/98225954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:32.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:32 smithi006 bash[33918]: cluster 2023-08-28T01:34:30.297427+0000 mgr.y (mgr.14901) 6477 : cluster [DBG] pgmap v6484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:32.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:32 smithi006 bash[33918]: audit 2023-08-28T01:34:31.596441+0000 mon.c (mon.2) 2540 : audit [DBG] from='client.? 172.21.15.6:0/98225954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:32.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:32 smithi006 bash[27292]: cluster 2023-08-28T01:34:30.297427+0000 mgr.y (mgr.14901) 6477 : cluster [DBG] pgmap v6484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:32.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:32 smithi006 bash[27292]: audit 2023-08-28T01:34:31.596441+0000 mon.c (mon.2) 2540 : audit [DBG] from='client.? 172.21.15.6:0/98225954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:34:32.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:34:32.624Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:33.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:32 smithi130 bash[51302]: debug 2023-08-28T01:34:32.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:32.716504+0000) 2023-08-28T01:34:34.096 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:33 smithi130 bash[51302]: debug 2023-08-28T01:34:33.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:33.717067+0000) 2023-08-28T01:34:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:34 smithi130 bash[27579]: cluster 2023-08-28T01:34:32.298053+0000 mgr.y (mgr.14901) 6478 : cluster [DBG] pgmap v6485: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:34 smithi130 bash[27579]: audit 2023-08-28T01:34:34.017393+0000 mon.a (mon.0) 4295 : audit [DBG] from='client.? 172.21.15.6:0/288647329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:34.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:34 smithi006 bash[27292]: cluster 2023-08-28T01:34:32.298053+0000 mgr.y (mgr.14901) 6478 : cluster [DBG] pgmap v6485: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:34.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:34 smithi006 bash[27292]: audit 2023-08-28T01:34:34.017393+0000 mon.a (mon.0) 4295 : audit [DBG] from='client.? 172.21.15.6:0/288647329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:34 smithi006 bash[33918]: cluster 2023-08-28T01:34:32.298053+0000 mgr.y (mgr.14901) 6478 : cluster [DBG] pgmap v6485: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:34 smithi006 bash[33918]: audit 2023-08-28T01:34:34.017393+0000 mon.a (mon.0) 4295 : audit [DBG] from='client.? 172.21.15.6:0/288647329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:34:34.505Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:34:34.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:35.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:34 smithi130 bash[51302]: debug 2023-08-28T01:34:34.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:34.717357+0000) 2023-08-28T01:34:36.101 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:35 smithi130 bash[51302]: debug 2023-08-28T01:34:35.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:35.717703+0000) 2023-08-28T01:34:36.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:36 smithi130 bash[27579]: cluster 2023-08-28T01:34:34.299200+0000 mgr.y (mgr.14901) 6479 : cluster [DBG] pgmap v6486: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:36.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:36 smithi006 bash[33918]: cluster 2023-08-28T01:34:34.299200+0000 mgr.y (mgr.14901) 6479 : cluster [DBG] pgmap v6486: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:36.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:36 smithi006 bash[27292]: cluster 2023-08-28T01:34:34.299200+0000 mgr.y (mgr.14901) 6479 : cluster [DBG] pgmap v6486: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:37.110 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:36 smithi130 bash[51302]: debug 2023-08-28T01:34:36.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:36.718050+0000) 2023-08-28T01:34:37.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:37 smithi130 bash[27579]: audit 2023-08-28T01:34:36.434388+0000 mon.c (mon.2) 2541 : audit [DBG] from='client.? 172.21.15.6:0/2864939181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:37.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:37 smithi006 bash[33918]: audit 2023-08-28T01:34:36.434388+0000 mon.c (mon.2) 2541 : audit [DBG] from='client.? 172.21.15.6:0/2864939181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:37.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:37 smithi006 bash[27292]: audit 2023-08-28T01:34:36.434388+0000 mon.c (mon.2) 2541 : audit [DBG] from='client.? 172.21.15.6:0/2864939181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:38.113 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:37 smithi130 bash[51302]: debug 2023-08-28T01:34:37.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:37.718353+0000) 2023-08-28T01:34:38.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:38 smithi130 bash[27579]: cluster 2023-08-28T01:34:36.299850+0000 mgr.y (mgr.14901) 6480 : cluster [DBG] pgmap v6487: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:38.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:38 smithi006 bash[27292]: cluster 2023-08-28T01:34:36.299850+0000 mgr.y (mgr.14901) 6480 : cluster [DBG] pgmap v6487: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:38.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:38 smithi006 bash[33918]: cluster 2023-08-28T01:34:36.299850+0000 mgr.y (mgr.14901) 6480 : cluster [DBG] pgmap v6487: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:39.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:38 smithi130 bash[51302]: debug 2023-08-28T01:34:38.713+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:38.718697+0000) 2023-08-28T01:34:39.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:39 smithi006 bash[27292]: audit 2023-08-28T01:34:38.848390+0000 mon.a (mon.0) 4296 : audit [DBG] from='client.? 172.21.15.6:0/2053801792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:39.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:39 smithi006 bash[33918]: audit 2023-08-28T01:34:38.848390+0000 mon.a (mon.0) 4296 : audit [DBG] from='client.? 172.21.15.6:0/2053801792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:39.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:39 smithi130 bash[27579]: audit 2023-08-28T01:34:38.848390+0000 mon.a (mon.0) 4296 : audit [DBG] from='client.? 172.21.15.6:0/2053801792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:40.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:39 smithi130 bash[51302]: debug 2023-08-28T01:34:39.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:39.719053+0000) 2023-08-28T01:34:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:40 smithi006 bash[33918]: cluster 2023-08-28T01:34:38.300520+0000 mgr.y (mgr.14901) 6481 : cluster [DBG] pgmap v6488: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:40.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:40 smithi006 bash[27292]: cluster 2023-08-28T01:34:38.300520+0000 mgr.y (mgr.14901) 6481 : cluster [DBG] pgmap v6488: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:40 smithi130 bash[27579]: cluster 2023-08-28T01:34:38.300520+0000 mgr.y (mgr.14901) 6481 : cluster [DBG] pgmap v6488: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:41.054 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:34:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:34:40] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T01:34:41.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:40 smithi130 bash[51302]: debug 2023-08-28T01:34:40.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:40.719443+0000) 2023-08-28T01:34:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:34:41.053Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:34:41.055Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:41 smithi130 bash[51302]: debug 2023-08-28T01:34:41.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:41.719777+0000) 2023-08-28T01:34:42.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:42 smithi006 bash[33918]: cluster 2023-08-28T01:34:40.301705+0000 mgr.y (mgr.14901) 6482 : cluster [DBG] pgmap v6489: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:42.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:42 smithi006 bash[33918]: audit 2023-08-28T01:34:41.267848+0000 mon.a (mon.0) 4297 : audit [DBG] from='client.? 172.21.15.6:0/698886929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:42.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:42 smithi006 bash[27292]: cluster 2023-08-28T01:34:40.301705+0000 mgr.y (mgr.14901) 6482 : cluster [DBG] pgmap v6489: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:42.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:42 smithi006 bash[27292]: audit 2023-08-28T01:34:41.267848+0000 mon.a (mon.0) 4297 : audit [DBG] from='client.? 172.21.15.6:0/698886929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:42 smithi130 bash[27579]: cluster 2023-08-28T01:34:40.301705+0000 mgr.y (mgr.14901) 6482 : cluster [DBG] pgmap v6489: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:42 smithi130 bash[27579]: audit 2023-08-28T01:34:41.267848+0000 mon.a (mon.0) 4297 : audit [DBG] from='client.? 172.21.15.6:0/698886929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:34:42.623Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:34:42.625Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:43.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:42 smithi130 bash[51302]: debug 2023-08-28T01:34:42.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:42.720110+0000) 2023-08-28T01:34:44.027 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:43 smithi130 bash[51302]: debug 2023-08-28T01:34:43.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:43.720471+0000) 2023-08-28T01:34:44.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:44 smithi130 bash[27579]: cluster 2023-08-28T01:34:42.302363+0000 mgr.y (mgr.14901) 6483 : cluster [DBG] pgmap v6490: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:44.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:44 smithi130 bash[27579]: audit 2023-08-28T01:34:43.685258+0000 mon.c (mon.2) 2542 : audit [DBG] from='client.? 172.21.15.6:0/818623179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:44.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:44 smithi006 bash[27292]: cluster 2023-08-28T01:34:42.302363+0000 mgr.y (mgr.14901) 6483 : cluster [DBG] pgmap v6490: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:44.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:44 smithi006 bash[27292]: audit 2023-08-28T01:34:43.685258+0000 mon.c (mon.2) 2542 : audit [DBG] from='client.? 172.21.15.6:0/818623179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:44 smithi006 bash[33918]: cluster 2023-08-28T01:34:42.302363+0000 mgr.y (mgr.14901) 6483 : cluster [DBG] pgmap v6490: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:44 smithi006 bash[33918]: audit 2023-08-28T01:34:43.685258+0000 mon.c (mon.2) 2542 : audit [DBG] from='client.? 172.21.15.6:0/818623179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:34:44.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:34:44.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:45.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:44 smithi130 bash[51302]: debug 2023-08-28T01:34:44.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:44.720808+0000) 2023-08-28T01:34:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:45 smithi130 bash[51302]: debug 2023-08-28T01:34:45.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:45.721128+0000) 2023-08-28T01:34:46.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:46 smithi006 bash[33918]: cluster 2023-08-28T01:34:44.303466+0000 mgr.y (mgr.14901) 6484 : cluster [DBG] pgmap v6491: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:46.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:46 smithi006 bash[33918]: audit 2023-08-28T01:34:46.096202+0000 mon.c (mon.2) 2543 : audit [DBG] from='client.? 172.21.15.6:0/846805254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:46.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:46 smithi006 bash[27292]: cluster 2023-08-28T01:34:44.303466+0000 mgr.y (mgr.14901) 6484 : cluster [DBG] pgmap v6491: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:46.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:46 smithi006 bash[27292]: audit 2023-08-28T01:34:46.096202+0000 mon.c (mon.2) 2543 : audit [DBG] from='client.? 172.21.15.6:0/846805254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:46 smithi130 bash[27579]: cluster 2023-08-28T01:34:44.303466+0000 mgr.y (mgr.14901) 6484 : cluster [DBG] pgmap v6491: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:46 smithi130 bash[27579]: audit 2023-08-28T01:34:46.096202+0000 mon.c (mon.2) 2543 : audit [DBG] from='client.? 172.21.15.6:0/846805254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:46 smithi130 bash[51302]: debug 2023-08-28T01:34:46.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:46.721486+0000) 2023-08-28T01:34:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:47 smithi130 bash[51302]: debug 2023-08-28T01:34:47.721+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:47.721822+0000) 2023-08-28T01:34:48.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:48 smithi006 bash[33918]: cluster 2023-08-28T01:34:46.304259+0000 mgr.y (mgr.14901) 6485 : cluster [DBG] pgmap v6492: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:48.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:48 smithi006 bash[27292]: cluster 2023-08-28T01:34:46.304259+0000 mgr.y (mgr.14901) 6485 : cluster [DBG] pgmap v6492: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:48 smithi130 bash[27579]: cluster 2023-08-28T01:34:46.304259+0000 mgr.y (mgr.14901) 6485 : cluster [DBG] pgmap v6492: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:49.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:48 smithi130 bash[51302]: debug 2023-08-28T01:34:48.721+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:48.722143+0000) 2023-08-28T01:34:49.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:49 smithi006 bash[33918]: audit 2023-08-28T01:34:48.508363+0000 mon.a (mon.0) 4298 : audit [DBG] from='client.? 172.21.15.6:0/1014140866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:49.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:49 smithi006 bash[27292]: audit 2023-08-28T01:34:48.508363+0000 mon.a (mon.0) 4298 : audit [DBG] from='client.? 172.21.15.6:0/1014140866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:49.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:49 smithi130 bash[27579]: audit 2023-08-28T01:34:48.508363+0000 mon.a (mon.0) 4298 : audit [DBG] from='client.? 172.21.15.6:0/1014140866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:50.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:49 smithi130 bash[51302]: debug 2023-08-28T01:34:49.717+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:49.722473+0000) 2023-08-28T01:34:50.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:50 smithi006 bash[33918]: cluster 2023-08-28T01:34:48.304774+0000 mgr.y (mgr.14901) 6486 : cluster [DBG] pgmap v6493: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:50.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:50 smithi006 bash[27292]: cluster 2023-08-28T01:34:48.304774+0000 mgr.y (mgr.14901) 6486 : cluster [DBG] pgmap v6493: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:50 smithi130 bash[27579]: cluster 2023-08-28T01:34:48.304774+0000 mgr.y (mgr.14901) 6486 : cluster [DBG] pgmap v6493: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:51.055 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:34:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:34:50] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:34:51.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:50 smithi130 bash[51302]: debug 2023-08-28T01:34:50.721+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:50.722846+0000) 2023-08-28T01:34:51.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:51 smithi006 bash[27292]: audit 2023-08-28T01:34:50.930047+0000 mon.c (mon.2) 2544 : audit [DBG] from='client.? 172.21.15.6:0/2007447619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:51.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:51 smithi006 bash[33918]: audit 2023-08-28T01:34:50.930047+0000 mon.c (mon.2) 2544 : audit [DBG] from='client.? 172.21.15.6:0/2007447619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:34:51.054Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:51.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:34:51.056Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:51.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:51 smithi130 bash[27579]: audit 2023-08-28T01:34:50.930047+0000 mon.c (mon.2) 2544 : audit [DBG] from='client.? 172.21.15.6:0/2007447619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:52.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:51 smithi130 bash[51302]: debug 2023-08-28T01:34:51.721+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:51.723231+0000) 2023-08-28T01:34:52.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:52 smithi006 bash[27292]: cluster 2023-08-28T01:34:50.305845+0000 mgr.y (mgr.14901) 6487 : cluster [DBG] pgmap v6494: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:52.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:52 smithi006 bash[33918]: cluster 2023-08-28T01:34:50.305845+0000 mgr.y (mgr.14901) 6487 : cluster [DBG] pgmap v6494: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:52 smithi130 bash[27579]: cluster 2023-08-28T01:34:50.305845+0000 mgr.y (mgr.14901) 6487 : cluster [DBG] pgmap v6494: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:34:52.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:34:52.625Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:53.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:52 smithi130 bash[51302]: debug 2023-08-28T01:34:52.721+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:52.723567+0000) 2023-08-28T01:34:54.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:53 smithi130 bash[51302]: debug 2023-08-28T01:34:53.721+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:53.723907+0000) 2023-08-28T01:34:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:54 smithi006 bash[27292]: cluster 2023-08-28T01:34:52.306502+0000 mgr.y (mgr.14901) 6488 : cluster [DBG] pgmap v6495: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:54 smithi006 bash[27292]: audit 2023-08-28T01:34:53.343732+0000 mon.a (mon.0) 4299 : audit [DBG] from='client.? 172.21.15.6:0/3288682174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:54 smithi006 bash[33918]: cluster 2023-08-28T01:34:52.306502+0000 mgr.y (mgr.14901) 6488 : cluster [DBG] pgmap v6495: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:54 smithi006 bash[33918]: audit 2023-08-28T01:34:53.343732+0000 mon.a (mon.0) 4299 : audit [DBG] from='client.? 172.21.15.6:0/3288682174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:54 smithi130 bash[27579]: cluster 2023-08-28T01:34:52.306502+0000 mgr.y (mgr.14901) 6488 : cluster [DBG] pgmap v6495: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:54 smithi130 bash[27579]: audit 2023-08-28T01:34:53.343732+0000 mon.a (mon.0) 4299 : audit [DBG] from='client.? 172.21.15.6:0/3288682174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:34:54.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:34:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:34:54.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:34:55.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:54 smithi130 bash[51302]: debug 2023-08-28T01:34:54.721+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:54.724236+0000) 2023-08-28T01:34:56.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:55 smithi130 bash[51302]: debug 2023-08-28T01:34:55.721+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:55.724506+0000) 2023-08-28T01:34:56.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:56 smithi006 bash[27292]: cluster 2023-08-28T01:34:54.307623+0000 mgr.y (mgr.14901) 6489 : cluster [DBG] pgmap v6496: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:56.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:56 smithi006 bash[27292]: audit 2023-08-28T01:34:55.762813+0000 mon.a (mon.0) 4300 : audit [DBG] from='client.? 172.21.15.6:0/626426424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:56.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:56 smithi006 bash[33918]: cluster 2023-08-28T01:34:54.307623+0000 mgr.y (mgr.14901) 6489 : cluster [DBG] pgmap v6496: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:56.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:56 smithi006 bash[33918]: audit 2023-08-28T01:34:55.762813+0000 mon.a (mon.0) 4300 : audit [DBG] from='client.? 172.21.15.6:0/626426424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:56 smithi130 bash[27579]: cluster 2023-08-28T01:34:54.307623+0000 mgr.y (mgr.14901) 6489 : cluster [DBG] pgmap v6496: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:56 smithi130 bash[27579]: audit 2023-08-28T01:34:55.762813+0000 mon.a (mon.0) 4300 : audit [DBG] from='client.? 172.21.15.6:0/626426424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:57.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:56 smithi130 bash[51302]: debug 2023-08-28T01:34:56.721+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:56.724844+0000) 2023-08-28T01:34:58.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:57 smithi130 bash[51302]: debug 2023-08-28T01:34:57.721+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:57.725179+0000) 2023-08-28T01:34:58.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:58 smithi006 bash[33918]: cluster 2023-08-28T01:34:56.308301+0000 mgr.y (mgr.14901) 6490 : cluster [DBG] pgmap v6497: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:58.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:58 smithi006 bash[27292]: cluster 2023-08-28T01:34:56.308301+0000 mgr.y (mgr.14901) 6490 : cluster [DBG] pgmap v6497: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:58 smithi130 bash[27579]: cluster 2023-08-28T01:34:56.308301+0000 mgr.y (mgr.14901) 6490 : cluster [DBG] pgmap v6497: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:34:59.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:58 smithi130 bash[51302]: debug 2023-08-28T01:34:58.721+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:58.725527+0000) 2023-08-28T01:34:59.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:34:59 smithi006 bash[27292]: audit 2023-08-28T01:34:58.176990+0000 mon.c (mon.2) 2545 : audit [DBG] from='client.? 172.21.15.6:0/128177773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:59.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:34:59 smithi006 bash[33918]: audit 2023-08-28T01:34:58.176990+0000 mon.c (mon.2) 2545 : audit [DBG] from='client.? 172.21.15.6:0/128177773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:59.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:34:59 smithi130 bash[27579]: audit 2023-08-28T01:34:58.176990+0000 mon.c (mon.2) 2545 : audit [DBG] from='client.? 172.21.15.6:0/128177773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:34:59 smithi130 bash[51302]: debug 2023-08-28T01:34:59.721+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:59.725878+0000) 2023-08-28T01:35:00.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:00 smithi006 bash[33918]: cluster 2023-08-28T01:34:58.308971+0000 mgr.y (mgr.14901) 6491 : cluster [DBG] pgmap v6498: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:00.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:00 smithi006 bash[27292]: cluster 2023-08-28T01:34:58.308971+0000 mgr.y (mgr.14901) 6491 : cluster [DBG] pgmap v6498: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:00 smithi130 bash[27579]: cluster 2023-08-28T01:34:58.308971+0000 mgr.y (mgr.14901) 6491 : cluster [DBG] pgmap v6498: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:01.007 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:00 smithi130 bash[51302]: debug 2023-08-28T01:35:00.725+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:00.726228+0000) 2023-08-28T01:35:01.055 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:35:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:35:00] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:35:01.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:01 smithi130 bash[27579]: audit 2023-08-28T01:35:00.595078+0000 mon.a (mon.0) 4301 : audit [DBG] from='client.? 172.21.15.6:0/193108442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:01.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:01 smithi006 bash[27292]: audit 2023-08-28T01:35:00.595078+0000 mon.a (mon.0) 4301 : audit [DBG] from='client.? 172.21.15.6:0/193108442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:35:01.055Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:35:01.056Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:01.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:01 smithi006 bash[33918]: audit 2023-08-28T01:35:00.595078+0000 mon.a (mon.0) 4301 : audit [DBG] from='client.? 172.21.15.6:0/193108442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:02.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:01 smithi130 bash[51302]: debug 2023-08-28T01:35:01.721+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:01.726547+0000) 2023-08-28T01:35:02.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:02 smithi006 bash[33918]: cluster 2023-08-28T01:35:00.309958+0000 mgr.y (mgr.14901) 6492 : cluster [DBG] pgmap v6499: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:02.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:02 smithi006 bash[27292]: cluster 2023-08-28T01:35:00.309958+0000 mgr.y (mgr.14901) 6492 : cluster [DBG] pgmap v6499: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:02 smithi130 bash[27579]: cluster 2023-08-28T01:35:00.309958+0000 mgr.y (mgr.14901) 6492 : cluster [DBG] pgmap v6499: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:35:02.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:35:02.626Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:03.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:02 smithi130 bash[51302]: debug 2023-08-28T01:35:02.725+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:02.726895+0000) 2023-08-28T01:35:03.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:03 smithi006 bash[33918]: audit 2023-08-28T01:35:03.018253+0000 mon.a (mon.0) 4302 : audit [DBG] from='client.? 172.21.15.6:0/1547447507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:03.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:03 smithi006 bash[27292]: audit 2023-08-28T01:35:03.018253+0000 mon.a (mon.0) 4302 : audit [DBG] from='client.? 172.21.15.6:0/1547447507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:03.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:03 smithi130 bash[27579]: audit 2023-08-28T01:35:03.018253+0000 mon.a (mon.0) 4302 : audit [DBG] from='client.? 172.21.15.6:0/1547447507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:04.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:03 smithi130 bash[51302]: debug 2023-08-28T01:35:03.725+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:03.727172+0000) 2023-08-28T01:35:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:04 smithi006 bash[27292]: cluster 2023-08-28T01:35:02.310616+0000 mgr.y (mgr.14901) 6493 : cluster [DBG] pgmap v6500: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:04.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:04 smithi006 bash[33918]: cluster 2023-08-28T01:35:02.310616+0000 mgr.y (mgr.14901) 6493 : cluster [DBG] pgmap v6500: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:04 smithi130 bash[27579]: cluster 2023-08-28T01:35:02.310616+0000 mgr.y (mgr.14901) 6493 : cluster [DBG] pgmap v6500: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:04.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:35:04.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:04.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:35:04.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:05.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:04 smithi130 bash[51302]: debug 2023-08-28T01:35:04.725+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:04.727481+0000) 2023-08-28T01:35:06.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:05 smithi130 bash[51302]: debug 2023-08-28T01:35:05.725+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:05.727806+0000) 2023-08-28T01:35:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:06 smithi130 bash[27579]: cluster 2023-08-28T01:35:04.311752+0000 mgr.y (mgr.14901) 6494 : cluster [DBG] pgmap v6501: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:06 smithi130 bash[27579]: audit 2023-08-28T01:35:05.432949+0000 mon.c (mon.2) 2546 : audit [DBG] from='client.? 172.21.15.6:0/3391742226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:06 smithi006 bash[33918]: cluster 2023-08-28T01:35:04.311752+0000 mgr.y (mgr.14901) 6494 : cluster [DBG] pgmap v6501: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:06 smithi006 bash[33918]: audit 2023-08-28T01:35:05.432949+0000 mon.c (mon.2) 2546 : audit [DBG] from='client.? 172.21.15.6:0/3391742226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:06 smithi006 bash[27292]: cluster 2023-08-28T01:35:04.311752+0000 mgr.y (mgr.14901) 6494 : cluster [DBG] pgmap v6501: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:06 smithi006 bash[27292]: audit 2023-08-28T01:35:05.432949+0000 mon.c (mon.2) 2546 : audit [DBG] from='client.? 172.21.15.6:0/3391742226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:07.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:06 smithi130 bash[51302]: debug 2023-08-28T01:35:06.725+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:06.728163+0000) 2023-08-28T01:35:07.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:07 smithi130 bash[27579]: audit 2023-08-28T01:35:06.403777+0000 mon.b (mon.1) 739 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:35:07.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:07 smithi006 bash[33918]: audit 2023-08-28T01:35:06.403777+0000 mon.b (mon.1) 739 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:35:07.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:07 smithi006 bash[27292]: audit 2023-08-28T01:35:06.403777+0000 mon.b (mon.1) 739 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:35:08.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:07 smithi130 bash[51302]: debug 2023-08-28T01:35:07.725+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:07.728487+0000) 2023-08-28T01:35:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:08 smithi130 bash[27579]: cluster 2023-08-28T01:35:06.312408+0000 mgr.y (mgr.14901) 6495 : cluster [DBG] pgmap v6502: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:08 smithi130 bash[27579]: audit 2023-08-28T01:35:07.860560+0000 mon.c (mon.2) 2547 : audit [DBG] from='client.? 172.21.15.6:0/3288410575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:08 smithi006 bash[27292]: cluster 2023-08-28T01:35:06.312408+0000 mgr.y (mgr.14901) 6495 : cluster [DBG] pgmap v6502: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:08 smithi006 bash[27292]: audit 2023-08-28T01:35:07.860560+0000 mon.c (mon.2) 2547 : audit [DBG] from='client.? 172.21.15.6:0/3288410575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:08.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:08 smithi006 bash[33918]: cluster 2023-08-28T01:35:06.312408+0000 mgr.y (mgr.14901) 6495 : cluster [DBG] pgmap v6502: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:08.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:08 smithi006 bash[33918]: audit 2023-08-28T01:35:07.860560+0000 mon.c (mon.2) 2547 : audit [DBG] from='client.? 172.21.15.6:0/3288410575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:09.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:08 smithi130 bash[51302]: debug 2023-08-28T01:35:08.725+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:08.728827+0000) 2023-08-28T01:35:10.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:09 smithi130 bash[51302]: debug 2023-08-28T01:35:09.725+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:09.729163+0000) 2023-08-28T01:35:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:10 smithi130 bash[27579]: cluster 2023-08-28T01:35:08.313124+0000 mgr.y (mgr.14901) 6496 : cluster [DBG] pgmap v6503: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:10 smithi006 bash[27292]: cluster 2023-08-28T01:35:08.313124+0000 mgr.y (mgr.14901) 6496 : cluster [DBG] pgmap v6503: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:10.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:10 smithi006 bash[33918]: cluster 2023-08-28T01:35:08.313124+0000 mgr.y (mgr.14901) 6496 : cluster [DBG] pgmap v6503: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:11.056 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:35:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:35:10] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:35:11.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:10 smithi130 bash[51302]: debug 2023-08-28T01:35:10.725+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:10.729466+0000) 2023-08-28T01:35:11.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:11 smithi006 bash[27292]: audit 2023-08-28T01:35:10.277288+0000 mon.a (mon.0) 4303 : audit [DBG] from='client.? 172.21.15.6:0/2965692299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:11.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:11 smithi006 bash[33918]: audit 2023-08-28T01:35:10.277288+0000 mon.a (mon.0) 4303 : audit [DBG] from='client.? 172.21.15.6:0/2965692299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:35:11.055Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:35:11.058Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:11.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:11 smithi130 bash[27579]: audit 2023-08-28T01:35:10.277288+0000 mon.a (mon.0) 4303 : audit [DBG] from='client.? 172.21.15.6:0/2965692299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:12.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:11 smithi130 bash[51302]: debug 2023-08-28T01:35:11.725+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:11.729713+0000) 2023-08-28T01:35:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:12 smithi130 bash[27579]: cluster 2023-08-28T01:35:10.314320+0000 mgr.y (mgr.14901) 6497 : cluster [DBG] pgmap v6504: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:12.626 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:12 smithi006 bash[33918]: cluster 2023-08-28T01:35:10.314320+0000 mgr.y (mgr.14901) 6497 : cluster [DBG] pgmap v6504: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:12.626 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:12 smithi006 bash[27292]: cluster 2023-08-28T01:35:10.314320+0000 mgr.y (mgr.14901) 6497 : cluster [DBG] pgmap v6504: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:35:12.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:35:12.628Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:13.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:12 smithi130 bash[51302]: debug 2023-08-28T01:35:12.725+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:12.729963+0000) 2023-08-28T01:35:13.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:13 smithi130 bash[27579]: audit 2023-08-28T01:35:12.286544+0000 mon.a (mon.0) 4304 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:35:13.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:13 smithi130 bash[27579]: cluster 2023-08-28T01:35:12.314973+0000 mgr.y (mgr.14901) 6498 : cluster [DBG] pgmap v6505: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:13.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:13 smithi130 bash[27579]: audit 2023-08-28T01:35:12.681625+0000 mon.a (mon.0) 4305 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:35:13.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:13 smithi130 bash[27579]: audit 2023-08-28T01:35:12.693746+0000 mon.c (mon.2) 2548 : audit [DBG] from='client.? 172.21.15.6:0/1143179639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:13.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:13 smithi130 bash[27579]: audit 2023-08-28T01:35:12.950204+0000 mon.b (mon.1) 740 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:35:13.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:13 smithi130 bash[27579]: audit 2023-08-28T01:35:12.952314+0000 mon.b (mon.1) 741 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:35:13.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:13 smithi130 bash[27579]: audit 2023-08-28T01:35:12.960574+0000 mon.a (mon.0) 4306 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:35:13.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:13 smithi006 bash[27292]: audit 2023-08-28T01:35:12.286544+0000 mon.a (mon.0) 4304 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:35:13.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:13 smithi006 bash[27292]: cluster 2023-08-28T01:35:12.314973+0000 mgr.y (mgr.14901) 6498 : cluster [DBG] pgmap v6505: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:13.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:13 smithi006 bash[27292]: audit 2023-08-28T01:35:12.681625+0000 mon.a (mon.0) 4305 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:35:13.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:13 smithi006 bash[27292]: audit 2023-08-28T01:35:12.693746+0000 mon.c (mon.2) 2548 : audit [DBG] from='client.? 172.21.15.6:0/1143179639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:13.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:13 smithi006 bash[27292]: audit 2023-08-28T01:35:12.950204+0000 mon.b (mon.1) 740 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:35:13.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:13 smithi006 bash[27292]: audit 2023-08-28T01:35:12.952314+0000 mon.b (mon.1) 741 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:35:13.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:13 smithi006 bash[27292]: audit 2023-08-28T01:35:12.960574+0000 mon.a (mon.0) 4306 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:35:13.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:13 smithi006 bash[33918]: audit 2023-08-28T01:35:12.286544+0000 mon.a (mon.0) 4304 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:35:13.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:13 smithi006 bash[33918]: cluster 2023-08-28T01:35:12.314973+0000 mgr.y (mgr.14901) 6498 : cluster [DBG] pgmap v6505: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:13.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:13 smithi006 bash[33918]: audit 2023-08-28T01:35:12.681625+0000 mon.a (mon.0) 4305 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:35:13.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:13 smithi006 bash[33918]: audit 2023-08-28T01:35:12.693746+0000 mon.c (mon.2) 2548 : audit [DBG] from='client.? 172.21.15.6:0/1143179639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:13.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:13 smithi006 bash[33918]: audit 2023-08-28T01:35:12.950204+0000 mon.b (mon.1) 740 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:35:13.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:13 smithi006 bash[33918]: audit 2023-08-28T01:35:12.952314+0000 mon.b (mon.1) 741 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:35:13.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:13 smithi006 bash[33918]: audit 2023-08-28T01:35:12.960574+0000 mon.a (mon.0) 4306 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:35:14.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:13 smithi130 bash[51302]: debug 2023-08-28T01:35:13.725+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:13.730278+0000) 2023-08-28T01:35:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:35:14.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:35:14.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:14 smithi130 bash[51302]: debug 2023-08-28T01:35:14.725+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:14.730574+0000) 2023-08-28T01:35:15.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:15 smithi006 bash[27292]: cluster 2023-08-28T01:35:14.316035+0000 mgr.y (mgr.14901) 6499 : cluster [DBG] pgmap v6506: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:15.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:15 smithi006 bash[27292]: audit 2023-08-28T01:35:15.112894+0000 mon.a (mon.0) 4307 : audit [DBG] from='client.? 172.21.15.6:0/2289397699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:15.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:15 smithi006 bash[33918]: cluster 2023-08-28T01:35:14.316035+0000 mgr.y (mgr.14901) 6499 : cluster [DBG] pgmap v6506: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:15.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:15 smithi006 bash[33918]: audit 2023-08-28T01:35:15.112894+0000 mon.a (mon.0) 4307 : audit [DBG] from='client.? 172.21.15.6:0/2289397699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:15.729 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:15 smithi130 bash[27579]: cluster 2023-08-28T01:35:14.316035+0000 mgr.y (mgr.14901) 6499 : cluster [DBG] pgmap v6506: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:15.729 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:15 smithi130 bash[27579]: audit 2023-08-28T01:35:15.112894+0000 mon.a (mon.0) 4307 : audit [DBG] from='client.? 172.21.15.6:0/2289397699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:16.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:15 smithi130 bash[51302]: debug 2023-08-28T01:35:15.729+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:15.730839+0000) 2023-08-28T01:35:17.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:16 smithi130 bash[51302]: debug 2023-08-28T01:35:16.729+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:16.731203+0000) 2023-08-28T01:35:18.054 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:17 smithi130 bash[51302]: debug 2023-08-28T01:35:17.729+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:17.731526+0000) 2023-08-28T01:35:18.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:18 smithi130 bash[27579]: cluster 2023-08-28T01:35:16.316699+0000 mgr.y (mgr.14901) 6500 : cluster [DBG] pgmap v6507: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:18.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:18 smithi130 bash[27579]: audit 2023-08-28T01:35:17.526117+0000 mon.c (mon.2) 2549 : audit [DBG] from='client.? 172.21.15.6:0/3381886752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:18.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:18 smithi006 bash[33918]: cluster 2023-08-28T01:35:16.316699+0000 mgr.y (mgr.14901) 6500 : cluster [DBG] pgmap v6507: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:18.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:18 smithi006 bash[33918]: audit 2023-08-28T01:35:17.526117+0000 mon.c (mon.2) 2549 : audit [DBG] from='client.? 172.21.15.6:0/3381886752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:18.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:18 smithi006 bash[27292]: cluster 2023-08-28T01:35:16.316699+0000 mgr.y (mgr.14901) 6500 : cluster [DBG] pgmap v6507: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:18.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:18 smithi006 bash[27292]: audit 2023-08-28T01:35:17.526117+0000 mon.c (mon.2) 2549 : audit [DBG] from='client.? 172.21.15.6:0/3381886752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:19.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:18 smithi130 bash[51302]: debug 2023-08-28T01:35:18.729+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:18.731836+0000) 2023-08-28T01:35:20.058 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:19 smithi130 bash[51302]: debug 2023-08-28T01:35:19.728+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:19.732107+0000) 2023-08-28T01:35:20.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:20 smithi130 bash[27579]: cluster 2023-08-28T01:35:18.317373+0000 mgr.y (mgr.14901) 6501 : cluster [DBG] pgmap v6508: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:20.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:20 smithi130 bash[27579]: audit 2023-08-28T01:35:19.939411+0000 mon.c (mon.2) 2550 : audit [DBG] from='client.? 172.21.15.6:0/3188551560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:20.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:20 smithi006 bash[27292]: cluster 2023-08-28T01:35:18.317373+0000 mgr.y (mgr.14901) 6501 : cluster [DBG] pgmap v6508: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:20.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:20 smithi006 bash[27292]: audit 2023-08-28T01:35:19.939411+0000 mon.c (mon.2) 2550 : audit [DBG] from='client.? 172.21.15.6:0/3188551560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:20.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:20 smithi006 bash[33918]: cluster 2023-08-28T01:35:18.317373+0000 mgr.y (mgr.14901) 6501 : cluster [DBG] pgmap v6508: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:20.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:20 smithi006 bash[33918]: audit 2023-08-28T01:35:19.939411+0000 mon.c (mon.2) 2550 : audit [DBG] from='client.? 172.21.15.6:0/3188551560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:21.056 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:35:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:35:20] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:35:21.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:20 smithi130 bash[51302]: debug 2023-08-28T01:35:20.728+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:20.732398+0000) 2023-08-28T01:35:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:35:21.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:35:21.058Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:22.062 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:21 smithi130 bash[51302]: debug 2023-08-28T01:35:21.728+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:21.732667+0000) 2023-08-28T01:35:22.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:22 smithi130 bash[27579]: cluster 2023-08-28T01:35:20.318495+0000 mgr.y (mgr.14901) 6502 : cluster [DBG] pgmap v6509: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:22.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:22 smithi006 bash[27292]: cluster 2023-08-28T01:35:20.318495+0000 mgr.y (mgr.14901) 6502 : cluster [DBG] pgmap v6509: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:22.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:22 smithi006 bash[33918]: cluster 2023-08-28T01:35:20.318495+0000 mgr.y (mgr.14901) 6502 : cluster [DBG] pgmap v6509: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:35:22.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:35:22.628Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:23.072 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:22 smithi130 bash[51302]: debug 2023-08-28T01:35:22.728+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:22.732936+0000) 2023-08-28T01:35:23.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:23 smithi130 bash[27579]: audit 2023-08-28T01:35:22.350361+0000 mon.c (mon.2) 2551 : audit [DBG] from='client.? 172.21.15.6:0/1432254401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:23.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:23 smithi006 bash[27292]: audit 2023-08-28T01:35:22.350361+0000 mon.c (mon.2) 2551 : audit [DBG] from='client.? 172.21.15.6:0/1432254401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:23.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:23 smithi006 bash[33918]: audit 2023-08-28T01:35:22.350361+0000 mon.c (mon.2) 2551 : audit [DBG] from='client.? 172.21.15.6:0/1432254401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:24.076 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:23 smithi130 bash[51302]: debug 2023-08-28T01:35:23.728+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:23.733216+0000) 2023-08-28T01:35:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:24 smithi130 bash[27579]: cluster 2023-08-28T01:35:22.319120+0000 mgr.y (mgr.14901) 6503 : cluster [DBG] pgmap v6510: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:24.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:24 smithi006 bash[27292]: cluster 2023-08-28T01:35:22.319120+0000 mgr.y (mgr.14901) 6503 : cluster [DBG] pgmap v6510: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:24.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:24 smithi006 bash[33918]: cluster 2023-08-28T01:35:22.319120+0000 mgr.y (mgr.14901) 6503 : cluster [DBG] pgmap v6510: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:35:24.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:35:24.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:25.084 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:24 smithi130 bash[51302]: debug 2023-08-28T01:35:24.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:24.733488+0000) 2023-08-28T01:35:25.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:25 smithi130 bash[27579]: audit 2023-08-28T01:35:24.756248+0000 mon.c (mon.2) 2552 : audit [DBG] from='client.? 172.21.15.6:0/2145914248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:25.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:25 smithi006 bash[33918]: audit 2023-08-28T01:35:24.756248+0000 mon.c (mon.2) 2552 : audit [DBG] from='client.? 172.21.15.6:0/2145914248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:25.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:25 smithi006 bash[27292]: audit 2023-08-28T01:35:24.756248+0000 mon.c (mon.2) 2552 : audit [DBG] from='client.? 172.21.15.6:0/2145914248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:26.087 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:25 smithi130 bash[51302]: debug 2023-08-28T01:35:25.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:25.733763+0000) 2023-08-28T01:35:26.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:26 smithi130 bash[27579]: cluster 2023-08-28T01:35:24.320224+0000 mgr.y (mgr.14901) 6504 : cluster [DBG] pgmap v6511: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:26.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:26 smithi006 bash[27292]: cluster 2023-08-28T01:35:24.320224+0000 mgr.y (mgr.14901) 6504 : cluster [DBG] pgmap v6511: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:26.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:26 smithi006 bash[33918]: cluster 2023-08-28T01:35:24.320224+0000 mgr.y (mgr.14901) 6504 : cluster [DBG] pgmap v6511: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:27.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:26 smithi130 bash[51302]: debug 2023-08-28T01:35:26.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:26.734080+0000) 2023-08-28T01:35:28.092 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:27 smithi130 bash[51302]: debug 2023-08-28T01:35:27.728+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:27.734376+0000) 2023-08-28T01:35:28.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:28 smithi130 bash[27579]: cluster 2023-08-28T01:35:26.320884+0000 mgr.y (mgr.14901) 6505 : cluster [DBG] pgmap v6512: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:28.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:28 smithi130 bash[27579]: audit 2023-08-28T01:35:27.167929+0000 mon.c (mon.2) 2553 : audit [DBG] from='client.? 172.21.15.6:0/2545867661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:28.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:28 smithi006 bash[27292]: cluster 2023-08-28T01:35:26.320884+0000 mgr.y (mgr.14901) 6505 : cluster [DBG] pgmap v6512: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:28.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:28 smithi006 bash[27292]: audit 2023-08-28T01:35:27.167929+0000 mon.c (mon.2) 2553 : audit [DBG] from='client.? 172.21.15.6:0/2545867661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:28.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:28 smithi006 bash[33918]: cluster 2023-08-28T01:35:26.320884+0000 mgr.y (mgr.14901) 6505 : cluster [DBG] pgmap v6512: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:28.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:28 smithi006 bash[33918]: audit 2023-08-28T01:35:27.167929+0000 mon.c (mon.2) 2553 : audit [DBG] from='client.? 172.21.15.6:0/2545867661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:28 smithi130 bash[51302]: debug 2023-08-28T01:35:28.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:28.734678+0000) 2023-08-28T01:35:30.097 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:29 smithi130 bash[51302]: debug 2023-08-28T01:35:29.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:29.735027+0000) 2023-08-28T01:35:30.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:30 smithi130 bash[27579]: cluster 2023-08-28T01:35:28.321487+0000 mgr.y (mgr.14901) 6506 : cluster [DBG] pgmap v6513: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:30.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:30 smithi130 bash[27579]: audit 2023-08-28T01:35:29.584984+0000 mon.c (mon.2) 2554 : audit [DBG] from='client.? 172.21.15.6:0/2046672857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:30.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:30 smithi006 bash[33918]: cluster 2023-08-28T01:35:28.321487+0000 mgr.y (mgr.14901) 6506 : cluster [DBG] pgmap v6513: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:30.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:30 smithi006 bash[33918]: audit 2023-08-28T01:35:29.584984+0000 mon.c (mon.2) 2554 : audit [DBG] from='client.? 172.21.15.6:0/2046672857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:30.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:30 smithi006 bash[27292]: cluster 2023-08-28T01:35:28.321487+0000 mgr.y (mgr.14901) 6506 : cluster [DBG] pgmap v6513: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:30.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:30 smithi006 bash[27292]: audit 2023-08-28T01:35:29.584984+0000 mon.c (mon.2) 2554 : audit [DBG] from='client.? 172.21.15.6:0/2046672857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:31.057 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:35:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:35:30] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:35:31.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:30 smithi130 bash[51302]: debug 2023-08-28T01:35:30.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:30.735377+0000) 2023-08-28T01:35:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:35:31.057Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:35:31.059Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:32.105 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:31 smithi130 bash[51302]: debug 2023-08-28T01:35:31.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:31.735631+0000) 2023-08-28T01:35:32.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:32 smithi130 bash[27579]: cluster 2023-08-28T01:35:30.322571+0000 mgr.y (mgr.14901) 6507 : cluster [DBG] pgmap v6514: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:32.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:32 smithi130 bash[27579]: audit 2023-08-28T01:35:31.996436+0000 mon.c (mon.2) 2555 : audit [DBG] from='client.? 172.21.15.6:0/2409352530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:32.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:32 smithi006 bash[33918]: cluster 2023-08-28T01:35:30.322571+0000 mgr.y (mgr.14901) 6507 : cluster [DBG] pgmap v6514: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:32.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:32 smithi006 bash[33918]: audit 2023-08-28T01:35:31.996436+0000 mon.c (mon.2) 2555 : audit [DBG] from='client.? 172.21.15.6:0/2409352530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:32.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:32 smithi006 bash[27292]: cluster 2023-08-28T01:35:30.322571+0000 mgr.y (mgr.14901) 6507 : cluster [DBG] pgmap v6514: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:32.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:32 smithi006 bash[27292]: audit 2023-08-28T01:35:31.996436+0000 mon.c (mon.2) 2555 : audit [DBG] from='client.? 172.21.15.6:0/2409352530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:35:32.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:35:32.629Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:33.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:32 smithi130 bash[51302]: debug 2023-08-28T01:35:32.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:32.735972+0000) 2023-08-28T01:35:34.110 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:33 smithi130 bash[51302]: debug 2023-08-28T01:35:33.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:33.736306+0000) 2023-08-28T01:35:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:34 smithi130 bash[27579]: cluster 2023-08-28T01:35:32.323210+0000 mgr.y (mgr.14901) 6508 : cluster [DBG] pgmap v6515: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:34.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:34 smithi006 bash[27292]: cluster 2023-08-28T01:35:32.323210+0000 mgr.y (mgr.14901) 6508 : cluster [DBG] pgmap v6515: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:34 smithi006 bash[33918]: cluster 2023-08-28T01:35:32.323210+0000 mgr.y (mgr.14901) 6508 : cluster [DBG] pgmap v6515: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:35:34.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:35:34.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:35.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:34 smithi130 bash[51302]: debug 2023-08-28T01:35:34.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:34.736665+0000) 2023-08-28T01:35:35.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:35 smithi006 bash[27292]: audit 2023-08-28T01:35:34.410278+0000 mon.c (mon.2) 2556 : audit [DBG] from='client.? 172.21.15.6:0/2363258357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:35.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:35 smithi006 bash[33918]: audit 2023-08-28T01:35:34.410278+0000 mon.c (mon.2) 2556 : audit [DBG] from='client.? 172.21.15.6:0/2363258357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:35.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:35 smithi130 bash[27579]: audit 2023-08-28T01:35:34.410278+0000 mon.c (mon.2) 2556 : audit [DBG] from='client.? 172.21.15.6:0/2363258357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:36.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:35 smithi130 bash[51302]: debug 2023-08-28T01:35:35.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:35.736996+0000) 2023-08-28T01:35:36.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:36 smithi006 bash[33918]: cluster 2023-08-28T01:35:34.324247+0000 mgr.y (mgr.14901) 6509 : cluster [DBG] pgmap v6516: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:36.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:36 smithi006 bash[27292]: cluster 2023-08-28T01:35:34.324247+0000 mgr.y (mgr.14901) 6509 : cluster [DBG] pgmap v6516: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:36 smithi130 bash[27579]: cluster 2023-08-28T01:35:34.324247+0000 mgr.y (mgr.14901) 6509 : cluster [DBG] pgmap v6516: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:37.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:36 smithi130 bash[51302]: debug 2023-08-28T01:35:36.736+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:36.737371+0000) 2023-08-28T01:35:37.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:37 smithi006 bash[33918]: audit 2023-08-28T01:35:36.820961+0000 mon.c (mon.2) 2557 : audit [DBG] from='client.? 172.21.15.6:0/3746796119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:37.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:37 smithi006 bash[27292]: audit 2023-08-28T01:35:36.820961+0000 mon.c (mon.2) 2557 : audit [DBG] from='client.? 172.21.15.6:0/3746796119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:37.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:37 smithi130 bash[27579]: audit 2023-08-28T01:35:36.820961+0000 mon.c (mon.2) 2557 : audit [DBG] from='client.? 172.21.15.6:0/3746796119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:37 smithi130 bash[51302]: debug 2023-08-28T01:35:37.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:37.737715+0000) 2023-08-28T01:35:38.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:38 smithi006 bash[27292]: cluster 2023-08-28T01:35:36.325064+0000 mgr.y (mgr.14901) 6510 : cluster [DBG] pgmap v6517: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:38.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:38 smithi006 bash[33918]: cluster 2023-08-28T01:35:36.325064+0000 mgr.y (mgr.14901) 6510 : cluster [DBG] pgmap v6517: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:38 smithi130 bash[27579]: cluster 2023-08-28T01:35:36.325064+0000 mgr.y (mgr.14901) 6510 : cluster [DBG] pgmap v6517: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:39.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:38 smithi130 bash[51302]: debug 2023-08-28T01:35:38.732+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:38.738071+0000) 2023-08-28T01:35:40.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:39 smithi130 bash[51302]: debug 2023-08-28T01:35:39.736+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:39.738449+0000) 2023-08-28T01:35:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:40 smithi006 bash[33918]: cluster 2023-08-28T01:35:38.325898+0000 mgr.y (mgr.14901) 6511 : cluster [DBG] pgmap v6518: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:40 smithi006 bash[33918]: audit 2023-08-28T01:35:39.241135+0000 mon.a (mon.0) 4308 : audit [DBG] from='client.? 172.21.15.6:0/2088645725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:40.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:40 smithi006 bash[27292]: cluster 2023-08-28T01:35:38.325898+0000 mgr.y (mgr.14901) 6511 : cluster [DBG] pgmap v6518: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:40.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:40 smithi006 bash[27292]: audit 2023-08-28T01:35:39.241135+0000 mon.a (mon.0) 4308 : audit [DBG] from='client.? 172.21.15.6:0/2088645725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:40 smithi130 bash[27579]: cluster 2023-08-28T01:35:38.325898+0000 mgr.y (mgr.14901) 6511 : cluster [DBG] pgmap v6518: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:40 smithi130 bash[27579]: audit 2023-08-28T01:35:39.241135+0000 mon.a (mon.0) 4308 : audit [DBG] from='client.? 172.21.15.6:0/2088645725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:41.058 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:35:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:35:40] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:35:41.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:40 smithi130 bash[51302]: debug 2023-08-28T01:35:40.736+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:40.738819+0000) 2023-08-28T01:35:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:35:41.057Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:35:41.059Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:41 smithi130 bash[51302]: debug 2023-08-28T01:35:41.736+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:41.739167+0000) 2023-08-28T01:35:42.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:42 smithi006 bash[33918]: cluster 2023-08-28T01:35:40.327202+0000 mgr.y (mgr.14901) 6512 : cluster [DBG] pgmap v6519: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:42.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:42 smithi006 bash[33918]: audit 2023-08-28T01:35:41.655365+0000 mon.c (mon.2) 2558 : audit [DBG] from='client.? 172.21.15.6:0/3835270049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:42.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:42 smithi006 bash[27292]: cluster 2023-08-28T01:35:40.327202+0000 mgr.y (mgr.14901) 6512 : cluster [DBG] pgmap v6519: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:42.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:42 smithi006 bash[27292]: audit 2023-08-28T01:35:41.655365+0000 mon.c (mon.2) 2558 : audit [DBG] from='client.? 172.21.15.6:0/3835270049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:42 smithi130 bash[27579]: cluster 2023-08-28T01:35:40.327202+0000 mgr.y (mgr.14901) 6512 : cluster [DBG] pgmap v6519: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:42 smithi130 bash[27579]: audit 2023-08-28T01:35:41.655365+0000 mon.c (mon.2) 2558 : audit [DBG] from='client.? 172.21.15.6:0/3835270049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:35:42.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:35:42.629Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:43.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:42 smithi130 bash[51302]: debug 2023-08-28T01:35:42.736+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:42.739464+0000) 2023-08-28T01:35:44.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:43 smithi130 bash[51302]: debug 2023-08-28T01:35:43.736+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:43.739801+0000) 2023-08-28T01:35:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:44 smithi006 bash[33918]: cluster 2023-08-28T01:35:42.327875+0000 mgr.y (mgr.14901) 6513 : cluster [DBG] pgmap v6520: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:44 smithi006 bash[33918]: audit 2023-08-28T01:35:44.068174+0000 mon.a (mon.0) 4309 : audit [DBG] from='client.? 172.21.15.6:0/810782626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:44.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:44 smithi006 bash[27292]: cluster 2023-08-28T01:35:42.327875+0000 mgr.y (mgr.14901) 6513 : cluster [DBG] pgmap v6520: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:44.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:44 smithi006 bash[27292]: audit 2023-08-28T01:35:44.068174+0000 mon.a (mon.0) 4309 : audit [DBG] from='client.? 172.21.15.6:0/810782626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:44 smithi130 bash[27579]: cluster 2023-08-28T01:35:42.327875+0000 mgr.y (mgr.14901) 6513 : cluster [DBG] pgmap v6520: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:44 smithi130 bash[27579]: audit 2023-08-28T01:35:44.068174+0000 mon.a (mon.0) 4309 : audit [DBG] from='client.? 172.21.15.6:0/810782626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:35:44.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:35:44.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:45.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:44 smithi130 bash[51302]: debug 2023-08-28T01:35:44.736+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:44.740140+0000) 2023-08-28T01:35:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:45 smithi130 bash[51302]: debug 2023-08-28T01:35:45.736+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:45.740475+0000) 2023-08-28T01:35:46.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:46 smithi006 bash[33918]: cluster 2023-08-28T01:35:44.329007+0000 mgr.y (mgr.14901) 6514 : cluster [DBG] pgmap v6521: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:46.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:46 smithi006 bash[27292]: cluster 2023-08-28T01:35:44.329007+0000 mgr.y (mgr.14901) 6514 : cluster [DBG] pgmap v6521: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:46 smithi130 bash[27579]: cluster 2023-08-28T01:35:44.329007+0000 mgr.y (mgr.14901) 6514 : cluster [DBG] pgmap v6521: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:46 smithi130 bash[51302]: debug 2023-08-28T01:35:46.736+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:46.740817+0000) 2023-08-28T01:35:47.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:47 smithi006 bash[27292]: audit 2023-08-28T01:35:46.477874+0000 mon.a (mon.0) 4310 : audit [DBG] from='client.? 172.21.15.6:0/1116164541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:47.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:47 smithi006 bash[33918]: audit 2023-08-28T01:35:46.477874+0000 mon.a (mon.0) 4310 : audit [DBG] from='client.? 172.21.15.6:0/1116164541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:47.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:47 smithi130 bash[27579]: audit 2023-08-28T01:35:46.477874+0000 mon.a (mon.0) 4310 : audit [DBG] from='client.? 172.21.15.6:0/1116164541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:47 smithi130 bash[51302]: debug 2023-08-28T01:35:47.736+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:47.741122+0000) 2023-08-28T01:35:48.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:48 smithi006 bash[27292]: cluster 2023-08-28T01:35:46.329645+0000 mgr.y (mgr.14901) 6515 : cluster [DBG] pgmap v6522: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:48.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:48 smithi006 bash[33918]: cluster 2023-08-28T01:35:46.329645+0000 mgr.y (mgr.14901) 6515 : cluster [DBG] pgmap v6522: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:48 smithi130 bash[27579]: cluster 2023-08-28T01:35:46.329645+0000 mgr.y (mgr.14901) 6515 : cluster [DBG] pgmap v6522: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:49.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:48 smithi130 bash[51302]: debug 2023-08-28T01:35:48.740+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:48.741456+0000) 2023-08-28T01:35:49.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:49 smithi006 bash[27292]: audit 2023-08-28T01:35:48.891658+0000 mon.a (mon.0) 4311 : audit [DBG] from='client.? 172.21.15.6:0/462199850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:49.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:49 smithi006 bash[33918]: audit 2023-08-28T01:35:48.891658+0000 mon.a (mon.0) 4311 : audit [DBG] from='client.? 172.21.15.6:0/462199850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:49.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:49 smithi130 bash[27579]: audit 2023-08-28T01:35:48.891658+0000 mon.a (mon.0) 4311 : audit [DBG] from='client.? 172.21.15.6:0/462199850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:50.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:49 smithi130 bash[51302]: debug 2023-08-28T01:35:49.736+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:49.741762+0000) 2023-08-28T01:35:50.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:50 smithi006 bash[27292]: cluster 2023-08-28T01:35:48.330315+0000 mgr.y (mgr.14901) 6516 : cluster [DBG] pgmap v6523: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:50.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:50 smithi006 bash[33918]: cluster 2023-08-28T01:35:48.330315+0000 mgr.y (mgr.14901) 6516 : cluster [DBG] pgmap v6523: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:50 smithi130 bash[27579]: cluster 2023-08-28T01:35:48.330315+0000 mgr.y (mgr.14901) 6516 : cluster [DBG] pgmap v6523: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:51.058 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:35:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:35:50] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:35:51.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:50 smithi130 bash[51302]: debug 2023-08-28T01:35:50.736+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:50.742112+0000) 2023-08-28T01:35:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:35:51.058Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:35:51.060Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:52.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:51 smithi130 bash[51302]: debug 2023-08-28T01:35:51.740+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:51.742470+0000) 2023-08-28T01:35:52.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:52 smithi006 bash[33918]: cluster 2023-08-28T01:35:50.331563+0000 mgr.y (mgr.14901) 6517 : cluster [DBG] pgmap v6524: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:52.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:52 smithi006 bash[33918]: audit 2023-08-28T01:35:51.308984+0000 mon.c (mon.2) 2559 : audit [DBG] from='client.? 172.21.15.6:0/389109751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:52.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:52 smithi006 bash[27292]: cluster 2023-08-28T01:35:50.331563+0000 mgr.y (mgr.14901) 6517 : cluster [DBG] pgmap v6524: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:52.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:52 smithi006 bash[27292]: audit 2023-08-28T01:35:51.308984+0000 mon.c (mon.2) 2559 : audit [DBG] from='client.? 172.21.15.6:0/389109751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:52 smithi130 bash[27579]: cluster 2023-08-28T01:35:50.331563+0000 mgr.y (mgr.14901) 6517 : cluster [DBG] pgmap v6524: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:52 smithi130 bash[27579]: audit 2023-08-28T01:35:51.308984+0000 mon.c (mon.2) 2559 : audit [DBG] from='client.? 172.21.15.6:0/389109751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:35:52.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:35:52.631Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:53.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:52 smithi130 bash[51302]: debug 2023-08-28T01:35:52.740+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:52.742803+0000) 2023-08-28T01:35:54.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:53 smithi130 bash[51302]: debug 2023-08-28T01:35:53.740+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:53.743219+0000) 2023-08-28T01:35:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:54 smithi006 bash[33918]: cluster 2023-08-28T01:35:52.332180+0000 mgr.y (mgr.14901) 6518 : cluster [DBG] pgmap v6525: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:54 smithi006 bash[33918]: audit 2023-08-28T01:35:53.722431+0000 mon.c (mon.2) 2560 : audit [DBG] from='client.? 172.21.15.6:0/2928082835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:54 smithi006 bash[27292]: cluster 2023-08-28T01:35:52.332180+0000 mgr.y (mgr.14901) 6518 : cluster [DBG] pgmap v6525: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:54 smithi006 bash[27292]: audit 2023-08-28T01:35:53.722431+0000 mon.c (mon.2) 2560 : audit [DBG] from='client.? 172.21.15.6:0/2928082835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:54 smithi130 bash[27579]: cluster 2023-08-28T01:35:52.332180+0000 mgr.y (mgr.14901) 6518 : cluster [DBG] pgmap v6525: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:54 smithi130 bash[27579]: audit 2023-08-28T01:35:53.722431+0000 mon.c (mon.2) 2560 : audit [DBG] from='client.? 172.21.15.6:0/2928082835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:35:54.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:35:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:35:54.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:35:55.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:54 smithi130 bash[51302]: debug 2023-08-28T01:35:54.740+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:54.743536+0000) 2023-08-28T01:35:56.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:55 smithi130 bash[51302]: debug 2023-08-28T01:35:55.740+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:55.743878+0000) 2023-08-28T01:35:56.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:56 smithi006 bash[33918]: cluster 2023-08-28T01:35:54.333356+0000 mgr.y (mgr.14901) 6519 : cluster [DBG] pgmap v6526: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:56.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:56 smithi006 bash[33918]: audit 2023-08-28T01:35:56.137389+0000 mon.c (mon.2) 2561 : audit [DBG] from='client.? 172.21.15.6:0/914419048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:56.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:56 smithi006 bash[27292]: cluster 2023-08-28T01:35:54.333356+0000 mgr.y (mgr.14901) 6519 : cluster [DBG] pgmap v6526: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:56.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:56 smithi006 bash[27292]: audit 2023-08-28T01:35:56.137389+0000 mon.c (mon.2) 2561 : audit [DBG] from='client.? 172.21.15.6:0/914419048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:56 smithi130 bash[27579]: cluster 2023-08-28T01:35:54.333356+0000 mgr.y (mgr.14901) 6519 : cluster [DBG] pgmap v6526: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:56 smithi130 bash[27579]: audit 2023-08-28T01:35:56.137389+0000 mon.c (mon.2) 2561 : audit [DBG] from='client.? 172.21.15.6:0/914419048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:57.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:56 smithi130 bash[51302]: debug 2023-08-28T01:35:56.740+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:56.744176+0000) 2023-08-28T01:35:58.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:57 smithi130 bash[51302]: debug 2023-08-28T01:35:57.740+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:57.744478+0000) 2023-08-28T01:35:58.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:58 smithi006 bash[27292]: cluster 2023-08-28T01:35:56.334037+0000 mgr.y (mgr.14901) 6520 : cluster [DBG] pgmap v6527: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:58.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:58 smithi006 bash[33918]: cluster 2023-08-28T01:35:56.334037+0000 mgr.y (mgr.14901) 6520 : cluster [DBG] pgmap v6527: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:58 smithi130 bash[27579]: cluster 2023-08-28T01:35:56.334037+0000 mgr.y (mgr.14901) 6520 : cluster [DBG] pgmap v6527: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:35:59.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:58 smithi130 bash[51302]: debug 2023-08-28T01:35:58.744+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:58.744786+0000) 2023-08-28T01:35:59.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:35:59 smithi006 bash[27292]: audit 2023-08-28T01:35:58.559705+0000 mon.a (mon.0) 4312 : audit [DBG] from='client.? 172.21.15.6:0/4104911342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:59.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:35:59 smithi006 bash[33918]: audit 2023-08-28T01:35:58.559705+0000 mon.a (mon.0) 4312 : audit [DBG] from='client.? 172.21.15.6:0/4104911342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:59.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:35:59 smithi130 bash[27579]: audit 2023-08-28T01:35:58.559705+0000 mon.a (mon.0) 4312 : audit [DBG] from='client.? 172.21.15.6:0/4104911342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:35:59 smithi130 bash[51302]: debug 2023-08-28T01:35:59.744+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:59.745104+0000) 2023-08-28T01:36:00.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:00 smithi006 bash[27292]: cluster 2023-08-28T01:35:58.334709+0000 mgr.y (mgr.14901) 6521 : cluster [DBG] pgmap v6528: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:00.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:00 smithi006 bash[33918]: cluster 2023-08-28T01:35:58.334709+0000 mgr.y (mgr.14901) 6521 : cluster [DBG] pgmap v6528: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:00 smithi130 bash[27579]: cluster 2023-08-28T01:35:58.334709+0000 mgr.y (mgr.14901) 6521 : cluster [DBG] pgmap v6528: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:01.059 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:36:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:36:00] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:36:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:00 smithi130 bash[51302]: debug 2023-08-28T01:36:00.740+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:00.745450+0000) 2023-08-28T01:36:01.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:01 smithi006 bash[33918]: audit 2023-08-28T01:36:00.979772+0000 mon.c (mon.2) 2562 : audit [DBG] from='client.? 172.21.15.6:0/4118024133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:01.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:01 smithi006 bash[27292]: audit 2023-08-28T01:36:00.979772+0000 mon.c (mon.2) 2562 : audit [DBG] from='client.? 172.21.15.6:0/4118024133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:36:01.059Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:36:01.061Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:01.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:01 smithi130 bash[27579]: audit 2023-08-28T01:36:00.979772+0000 mon.c (mon.2) 2562 : audit [DBG] from='client.? 172.21.15.6:0/4118024133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:02.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:01 smithi130 bash[51302]: debug 2023-08-28T01:36:01.744+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:01.745752+0000) 2023-08-28T01:36:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:02 smithi130 bash[27579]: cluster 2023-08-28T01:36:00.335912+0000 mgr.y (mgr.14901) 6522 : cluster [DBG] pgmap v6529: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:02.630 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:02 smithi006 bash[33918]: cluster 2023-08-28T01:36:00.335912+0000 mgr.y (mgr.14901) 6522 : cluster [DBG] pgmap v6529: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:02.630 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:02 smithi006 bash[27292]: cluster 2023-08-28T01:36:00.335912+0000 mgr.y (mgr.14901) 6522 : cluster [DBG] pgmap v6529: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:36:02.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:36:02.632Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:03.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:02 smithi130 bash[51302]: debug 2023-08-28T01:36:02.740+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:02.746015+0000) 2023-08-28T01:36:04.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:03 smithi130 bash[51302]: debug 2023-08-28T01:36:03.740+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:03.746286+0000) 2023-08-28T01:36:04.514 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:04 smithi006 bash[33918]: cluster 2023-08-28T01:36:02.336562+0000 mgr.y (mgr.14901) 6523 : cluster [DBG] pgmap v6530: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:04.514 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:04 smithi006 bash[33918]: audit 2023-08-28T01:36:03.400229+0000 mon.a (mon.0) 4313 : audit [DBG] from='client.? 172.21.15.6:0/2107405852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:04.514 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:04 smithi006 bash[27292]: cluster 2023-08-28T01:36:02.336562+0000 mgr.y (mgr.14901) 6523 : cluster [DBG] pgmap v6530: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:04.515 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:04 smithi006 bash[27292]: audit 2023-08-28T01:36:03.400229+0000 mon.a (mon.0) 4313 : audit [DBG] from='client.? 172.21.15.6:0/2107405852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:04 smithi130 bash[27579]: cluster 2023-08-28T01:36:02.336562+0000 mgr.y (mgr.14901) 6523 : cluster [DBG] pgmap v6530: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:04 smithi130 bash[27579]: audit 2023-08-28T01:36:03.400229+0000 mon.a (mon.0) 4313 : audit [DBG] from='client.? 172.21.15.6:0/2107405852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:36:04.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:36:04.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:05.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:04 smithi130 bash[51302]: debug 2023-08-28T01:36:04.740+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:04.746648+0000) 2023-08-28T01:36:06.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:05 smithi130 bash[51302]: debug 2023-08-28T01:36:05.744+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:05.746948+0000) 2023-08-28T01:36:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:06 smithi130 bash[27579]: cluster 2023-08-28T01:36:04.337705+0000 mgr.y (mgr.14901) 6524 : cluster [DBG] pgmap v6531: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:06 smithi130 bash[27579]: audit 2023-08-28T01:36:05.812131+0000 mon.c (mon.2) 2563 : audit [DBG] from='client.? 172.21.15.6:0/830938629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:06 smithi006 bash[33918]: cluster 2023-08-28T01:36:04.337705+0000 mgr.y (mgr.14901) 6524 : cluster [DBG] pgmap v6531: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:06 smithi006 bash[33918]: audit 2023-08-28T01:36:05.812131+0000 mon.c (mon.2) 2563 : audit [DBG] from='client.? 172.21.15.6:0/830938629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:06 smithi006 bash[27292]: cluster 2023-08-28T01:36:04.337705+0000 mgr.y (mgr.14901) 6524 : cluster [DBG] pgmap v6531: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:06 smithi006 bash[27292]: audit 2023-08-28T01:36:05.812131+0000 mon.c (mon.2) 2563 : audit [DBG] from='client.? 172.21.15.6:0/830938629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:07.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:06 smithi130 bash[51302]: debug 2023-08-28T01:36:06.744+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:06.747218+0000) 2023-08-28T01:36:08.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:07 smithi130 bash[51302]: debug 2023-08-28T01:36:07.744+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:07.747503+0000) 2023-08-28T01:36:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:08 smithi130 bash[27579]: cluster 2023-08-28T01:36:06.338365+0000 mgr.y (mgr.14901) 6525 : cluster [DBG] pgmap v6532: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:08 smithi130 bash[27579]: audit 2023-08-28T01:36:08.221160+0000 mon.a (mon.0) 4314 : audit [DBG] from='client.? 172.21.15.6:0/2780376239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:08 smithi006 bash[27292]: cluster 2023-08-28T01:36:06.338365+0000 mgr.y (mgr.14901) 6525 : cluster [DBG] pgmap v6532: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:08 smithi006 bash[27292]: audit 2023-08-28T01:36:08.221160+0000 mon.a (mon.0) 4314 : audit [DBG] from='client.? 172.21.15.6:0/2780376239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:08.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:08 smithi006 bash[33918]: cluster 2023-08-28T01:36:06.338365+0000 mgr.y (mgr.14901) 6525 : cluster [DBG] pgmap v6532: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:08.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:08 smithi006 bash[33918]: audit 2023-08-28T01:36:08.221160+0000 mon.a (mon.0) 4314 : audit [DBG] from='client.? 172.21.15.6:0/2780376239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:09.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:08 smithi130 bash[51302]: debug 2023-08-28T01:36:08.744+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:08.747783+0000) 2023-08-28T01:36:10.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:09 smithi130 bash[51302]: debug 2023-08-28T01:36:09.744+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:09.748067+0000) 2023-08-28T01:36:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:10 smithi130 bash[27579]: cluster 2023-08-28T01:36:08.339132+0000 mgr.y (mgr.14901) 6526 : cluster [DBG] pgmap v6533: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:10 smithi006 bash[27292]: cluster 2023-08-28T01:36:08.339132+0000 mgr.y (mgr.14901) 6526 : cluster [DBG] pgmap v6533: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:10.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:10 smithi006 bash[33918]: cluster 2023-08-28T01:36:08.339132+0000 mgr.y (mgr.14901) 6526 : cluster [DBG] pgmap v6533: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:11.060 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:36:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:36:10] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:36:11.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:10 smithi130 bash[51302]: debug 2023-08-28T01:36:10.748+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:10.748373+0000) 2023-08-28T01:36:11.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:11 smithi006 bash[33918]: audit 2023-08-28T01:36:10.639683+0000 mon.a (mon.0) 4315 : audit [DBG] from='client.? 172.21.15.6:0/516692983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:11.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:11 smithi006 bash[27292]: audit 2023-08-28T01:36:10.639683+0000 mon.a (mon.0) 4315 : audit [DBG] from='client.? 172.21.15.6:0/516692983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:36:11.060Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:36:11.062Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:11.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:11 smithi130 bash[27579]: audit 2023-08-28T01:36:10.639683+0000 mon.a (mon.0) 4315 : audit [DBG] from='client.? 172.21.15.6:0/516692983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:12.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:11 smithi130 bash[51302]: debug 2023-08-28T01:36:11.748+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:11.748654+0000) 2023-08-28T01:36:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:12 smithi130 bash[27579]: cluster 2023-08-28T01:36:10.339803+0000 mgr.y (mgr.14901) 6527 : cluster [DBG] pgmap v6534: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:12.630 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:12 smithi006 bash[27292]: cluster 2023-08-28T01:36:10.339803+0000 mgr.y (mgr.14901) 6527 : cluster [DBG] pgmap v6534: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:12.631 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:12 smithi006 bash[33918]: cluster 2023-08-28T01:36:10.339803+0000 mgr.y (mgr.14901) 6527 : cluster [DBG] pgmap v6534: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:36:12.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:36:12.632Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:13.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:12 smithi130 bash[51302]: debug 2023-08-28T01:36:12.748+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:12.748933+0000) 2023-08-28T01:36:13.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:13 smithi130 bash[27579]: audit 2023-08-28T01:36:12.971199+0000 mon.b (mon.1) 742 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:36:13.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:13 smithi130 bash[27579]: audit 2023-08-28T01:36:13.059777+0000 mon.a (mon.0) 4316 : audit [DBG] from='client.? 172.21.15.6:0/1183405739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:13.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:13 smithi006 bash[27292]: audit 2023-08-28T01:36:12.971199+0000 mon.b (mon.1) 742 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:36:13.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:13 smithi006 bash[27292]: audit 2023-08-28T01:36:13.059777+0000 mon.a (mon.0) 4316 : audit [DBG] from='client.? 172.21.15.6:0/1183405739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:13.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:13 smithi006 bash[33918]: audit 2023-08-28T01:36:12.971199+0000 mon.b (mon.1) 742 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:36:13.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:13 smithi006 bash[33918]: audit 2023-08-28T01:36:13.059777+0000 mon.a (mon.0) 4316 : audit [DBG] from='client.? 172.21.15.6:0/1183405739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:14.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:13 smithi130 bash[51302]: debug 2023-08-28T01:36:13.748+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:13.749350+0000) 2023-08-28T01:36:14.517 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:14 smithi006 bash[33918]: cluster 2023-08-28T01:36:12.340428+0000 mgr.y (mgr.14901) 6528 : cluster [DBG] pgmap v6535: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:14.518 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:36:14.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:14.518 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:14 smithi006 bash[27292]: cluster 2023-08-28T01:36:12.340428+0000 mgr.y (mgr.14901) 6528 : cluster [DBG] pgmap v6535: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:14 smithi130 bash[27579]: cluster 2023-08-28T01:36:12.340428+0000 mgr.y (mgr.14901) 6528 : cluster [DBG] pgmap v6535: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:36:14.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:14 smithi130 bash[51302]: debug 2023-08-28T01:36:14.744+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:14.749617+0000) 2023-08-28T01:36:16.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:15 smithi130 bash[51302]: debug 2023-08-28T01:36:15.743+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:15.749887+0000) 2023-08-28T01:36:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:16 smithi130 bash[27579]: cluster 2023-08-28T01:36:14.341577+0000 mgr.y (mgr.14901) 6529 : cluster [DBG] pgmap v6536: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:16 smithi130 bash[27579]: audit 2023-08-28T01:36:15.480080+0000 mon.a (mon.0) 4317 : audit [DBG] from='client.? 172.21.15.6:0/2436728714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:16 smithi006 bash[33918]: cluster 2023-08-28T01:36:14.341577+0000 mgr.y (mgr.14901) 6529 : cluster [DBG] pgmap v6536: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:16 smithi006 bash[33918]: audit 2023-08-28T01:36:15.480080+0000 mon.a (mon.0) 4317 : audit [DBG] from='client.? 172.21.15.6:0/2436728714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:16 smithi006 bash[27292]: cluster 2023-08-28T01:36:14.341577+0000 mgr.y (mgr.14901) 6529 : cluster [DBG] pgmap v6536: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:16 smithi006 bash[27292]: audit 2023-08-28T01:36:15.480080+0000 mon.a (mon.0) 4317 : audit [DBG] from='client.? 172.21.15.6:0/2436728714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:17.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:16 smithi130 bash[51302]: debug 2023-08-28T01:36:16.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:16.750160+0000) 2023-08-28T01:36:18.069 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:17 smithi130 bash[51302]: debug 2023-08-28T01:36:17.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:17.750419+0000) 2023-08-28T01:36:18.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:18 smithi130 bash[27579]: cluster 2023-08-28T01:36:16.342181+0000 mgr.y (mgr.14901) 6530 : cluster [DBG] pgmap v6537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:18.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:18 smithi130 bash[27579]: audit 2023-08-28T01:36:17.897162+0000 mon.a (mon.0) 4318 : audit [DBG] from='client.? 172.21.15.6:0/379916620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:18.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:18 smithi006 bash[27292]: cluster 2023-08-28T01:36:16.342181+0000 mgr.y (mgr.14901) 6530 : cluster [DBG] pgmap v6537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:18.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:18 smithi006 bash[27292]: audit 2023-08-28T01:36:17.897162+0000 mon.a (mon.0) 4318 : audit [DBG] from='client.? 172.21.15.6:0/379916620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:18.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:18 smithi006 bash[33918]: cluster 2023-08-28T01:36:16.342181+0000 mgr.y (mgr.14901) 6530 : cluster [DBG] pgmap v6537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:18.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:18 smithi006 bash[33918]: audit 2023-08-28T01:36:17.897162+0000 mon.a (mon.0) 4318 : audit [DBG] from='client.? 172.21.15.6:0/379916620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:19.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:18 smithi130 bash[51302]: debug 2023-08-28T01:36:18.743+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:18.750708+0000) 2023-08-28T01:36:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:19 smithi130 bash[51302]: debug 2023-08-28T01:36:19.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:19.750983+0000) 2023-08-28T01:36:20.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:19 smithi130 bash[27579]: cluster 2023-08-28T01:36:18.342766+0000 mgr.y (mgr.14901) 6531 : cluster [DBG] pgmap v6538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:20.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:19 smithi130 bash[27579]: audit 2023-08-28T01:36:18.842698+0000 mon.a (mon.0) 4319 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:36:20.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:19 smithi130 bash[27579]: audit 2023-08-28T01:36:19.274637+0000 mon.a (mon.0) 4320 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:36:20.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:19 smithi130 bash[27579]: audit 2023-08-28T01:36:19.543864+0000 mon.b (mon.1) 743 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:36:20.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:19 smithi130 bash[27579]: audit 2023-08-28T01:36:19.546580+0000 mon.b (mon.1) 744 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:36:20.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:19 smithi130 bash[27579]: audit 2023-08-28T01:36:19.555813+0000 mon.a (mon.0) 4321 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:36:20.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:19 smithi006 bash[27292]: cluster 2023-08-28T01:36:18.342766+0000 mgr.y (mgr.14901) 6531 : cluster [DBG] pgmap v6538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:20.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:19 smithi006 bash[27292]: audit 2023-08-28T01:36:18.842698+0000 mon.a (mon.0) 4319 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:36:20.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:19 smithi006 bash[27292]: audit 2023-08-28T01:36:19.274637+0000 mon.a (mon.0) 4320 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:36:20.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:19 smithi006 bash[27292]: audit 2023-08-28T01:36:19.543864+0000 mon.b (mon.1) 743 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:36:20.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:19 smithi006 bash[27292]: audit 2023-08-28T01:36:19.546580+0000 mon.b (mon.1) 744 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:36:20.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:19 smithi006 bash[27292]: audit 2023-08-28T01:36:19.555813+0000 mon.a (mon.0) 4321 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:36:20.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:19 smithi006 bash[33918]: cluster 2023-08-28T01:36:18.342766+0000 mgr.y (mgr.14901) 6531 : cluster [DBG] pgmap v6538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:20.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:19 smithi006 bash[33918]: audit 2023-08-28T01:36:18.842698+0000 mon.a (mon.0) 4319 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:36:20.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:19 smithi006 bash[33918]: audit 2023-08-28T01:36:19.274637+0000 mon.a (mon.0) 4320 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:36:20.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:19 smithi006 bash[33918]: audit 2023-08-28T01:36:19.543864+0000 mon.b (mon.1) 743 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:36:20.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:19 smithi006 bash[33918]: audit 2023-08-28T01:36:19.546580+0000 mon.b (mon.1) 744 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:36:20.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:19 smithi006 bash[33918]: audit 2023-08-28T01:36:19.555813+0000 mon.a (mon.0) 4321 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:36:21.061 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:20 smithi006 bash[33918]: audit 2023-08-28T01:36:20.312571+0000 mon.c (mon.2) 2564 : audit [DBG] from='client.? 172.21.15.6:0/3617485175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:21.061 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:20 smithi006 bash[27292]: audit 2023-08-28T01:36:20.312571+0000 mon.c (mon.2) 2564 : audit [DBG] from='client.? 172.21.15.6:0/3617485175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:21.061 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:36:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:36:20] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:36:21.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:20 smithi130 bash[27579]: audit 2023-08-28T01:36:20.312571+0000 mon.c (mon.2) 2564 : audit [DBG] from='client.? 172.21.15.6:0/3617485175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:21.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:20 smithi130 bash[51302]: debug 2023-08-28T01:36:20.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:20.751330+0000) 2023-08-28T01:36:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:36:21.060Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:36:21.062Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:22.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:21 smithi130 bash[27579]: cluster 2023-08-28T01:36:20.343370+0000 mgr.y (mgr.14901) 6532 : cluster [DBG] pgmap v6539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:22.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:21 smithi130 bash[51302]: debug 2023-08-28T01:36:21.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:21.751667+0000) 2023-08-28T01:36:22.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:21 smithi006 bash[33918]: cluster 2023-08-28T01:36:20.343370+0000 mgr.y (mgr.14901) 6532 : cluster [DBG] pgmap v6539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:22.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:21 smithi006 bash[27292]: cluster 2023-08-28T01:36:20.343370+0000 mgr.y (mgr.14901) 6532 : cluster [DBG] pgmap v6539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:36:22.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:36:22.632Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:23.079 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:22 smithi130 bash[51302]: debug 2023-08-28T01:36:22.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:22.751953+0000) 2023-08-28T01:36:23.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:23 smithi130 bash[27579]: audit 2023-08-28T01:36:22.745365+0000 mon.a (mon.0) 4322 : audit [DBG] from='client.? 172.21.15.6:0/689704928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:23.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:23 smithi006 bash[33918]: audit 2023-08-28T01:36:22.745365+0000 mon.a (mon.0) 4322 : audit [DBG] from='client.? 172.21.15.6:0/689704928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:23.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:23 smithi006 bash[27292]: audit 2023-08-28T01:36:22.745365+0000 mon.a (mon.0) 4322 : audit [DBG] from='client.? 172.21.15.6:0/689704928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:24.082 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:23 smithi130 bash[51302]: debug 2023-08-28T01:36:23.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:23.752233+0000) 2023-08-28T01:36:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:24 smithi130 bash[27579]: cluster 2023-08-28T01:36:22.343971+0000 mgr.y (mgr.14901) 6533 : cluster [DBG] pgmap v6540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:24.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:24 smithi006 bash[33918]: cluster 2023-08-28T01:36:22.343971+0000 mgr.y (mgr.14901) 6533 : cluster [DBG] pgmap v6540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:24 smithi006 bash[27292]: cluster 2023-08-28T01:36:22.343971+0000 mgr.y (mgr.14901) 6533 : cluster [DBG] pgmap v6540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:36:24.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:36:24.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:25.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:24 smithi130 bash[51302]: debug 2023-08-28T01:36:24.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:24.752506+0000) 2023-08-28T01:36:26.087 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:25 smithi130 bash[51302]: debug 2023-08-28T01:36:25.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:25.752780+0000) 2023-08-28T01:36:26.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:26 smithi130 bash[27579]: cluster 2023-08-28T01:36:24.345109+0000 mgr.y (mgr.14901) 6534 : cluster [DBG] pgmap v6541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:26.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:26 smithi130 bash[27579]: audit 2023-08-28T01:36:25.162059+0000 mon.c (mon.2) 2565 : audit [DBG] from='client.? 172.21.15.6:0/712629129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:26.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:26 smithi006 bash[27292]: cluster 2023-08-28T01:36:24.345109+0000 mgr.y (mgr.14901) 6534 : cluster [DBG] pgmap v6541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:26.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:26 smithi006 bash[27292]: audit 2023-08-28T01:36:25.162059+0000 mon.c (mon.2) 2565 : audit [DBG] from='client.? 172.21.15.6:0/712629129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:26.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:26 smithi006 bash[33918]: cluster 2023-08-28T01:36:24.345109+0000 mgr.y (mgr.14901) 6534 : cluster [DBG] pgmap v6541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:26.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:26 smithi006 bash[33918]: audit 2023-08-28T01:36:25.162059+0000 mon.c (mon.2) 2565 : audit [DBG] from='client.? 172.21.15.6:0/712629129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:27.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:26 smithi130 bash[51302]: debug 2023-08-28T01:36:26.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:26.753053+0000) 2023-08-28T01:36:28.091 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:27 smithi130 bash[51302]: debug 2023-08-28T01:36:27.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:27.753317+0000) 2023-08-28T01:36:28.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:28 smithi130 bash[27579]: cluster 2023-08-28T01:36:26.345774+0000 mgr.y (mgr.14901) 6535 : cluster [DBG] pgmap v6542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:28.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:28 smithi130 bash[27579]: audit 2023-08-28T01:36:27.573727+0000 mon.c (mon.2) 2566 : audit [DBG] from='client.? 172.21.15.6:0/2979733664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:28.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:28 smithi006 bash[27292]: cluster 2023-08-28T01:36:26.345774+0000 mgr.y (mgr.14901) 6535 : cluster [DBG] pgmap v6542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:28.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:28 smithi006 bash[27292]: audit 2023-08-28T01:36:27.573727+0000 mon.c (mon.2) 2566 : audit [DBG] from='client.? 172.21.15.6:0/2979733664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:28.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:28 smithi006 bash[33918]: cluster 2023-08-28T01:36:26.345774+0000 mgr.y (mgr.14901) 6535 : cluster [DBG] pgmap v6542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:28.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:28 smithi006 bash[33918]: audit 2023-08-28T01:36:27.573727+0000 mon.c (mon.2) 2566 : audit [DBG] from='client.? 172.21.15.6:0/2979733664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:28 smithi130 bash[51302]: debug 2023-08-28T01:36:28.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:28.753635+0000) 2023-08-28T01:36:30.098 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:29 smithi130 bash[51302]: debug 2023-08-28T01:36:29.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:29.753951+0000) 2023-08-28T01:36:30.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:30 smithi130 bash[27579]: cluster 2023-08-28T01:36:28.346430+0000 mgr.y (mgr.14901) 6536 : cluster [DBG] pgmap v6543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:30.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:30 smithi130 bash[27579]: audit 2023-08-28T01:36:29.991632+0000 mon.c (mon.2) 2567 : audit [DBG] from='client.? 172.21.15.6:0/2269634979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:30.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:30 smithi006 bash[33918]: cluster 2023-08-28T01:36:28.346430+0000 mgr.y (mgr.14901) 6536 : cluster [DBG] pgmap v6543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:30.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:30 smithi006 bash[33918]: audit 2023-08-28T01:36:29.991632+0000 mon.c (mon.2) 2567 : audit [DBG] from='client.? 172.21.15.6:0/2269634979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:30.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:30 smithi006 bash[27292]: cluster 2023-08-28T01:36:28.346430+0000 mgr.y (mgr.14901) 6536 : cluster [DBG] pgmap v6543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:30.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:30 smithi006 bash[27292]: audit 2023-08-28T01:36:29.991632+0000 mon.c (mon.2) 2567 : audit [DBG] from='client.? 172.21.15.6:0/2269634979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:31.061 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:36:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:36:30] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:36:31.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:30 smithi130 bash[51302]: debug 2023-08-28T01:36:30.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:30.754194+0000) 2023-08-28T01:36:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:36:31.061Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:36:31.063Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:32.103 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:31 smithi130 bash[51302]: debug 2023-08-28T01:36:31.747+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:31.754524+0000) 2023-08-28T01:36:32.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:32 smithi130 bash[27579]: cluster 2023-08-28T01:36:30.347173+0000 mgr.y (mgr.14901) 6537 : cluster [DBG] pgmap v6544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:32.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:32 smithi006 bash[33918]: cluster 2023-08-28T01:36:30.347173+0000 mgr.y (mgr.14901) 6537 : cluster [DBG] pgmap v6544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:32.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:32 smithi006 bash[27292]: cluster 2023-08-28T01:36:30.347173+0000 mgr.y (mgr.14901) 6537 : cluster [DBG] pgmap v6544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:36:32.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:32.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:36:32.633Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:33.113 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:32 smithi130 bash[51302]: debug 2023-08-28T01:36:32.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:32.754799+0000) 2023-08-28T01:36:33.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:33 smithi130 bash[27579]: audit 2023-08-28T01:36:32.408501+0000 mon.c (mon.2) 2568 : audit [DBG] from='client.? 172.21.15.6:0/463508315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:33.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:33 smithi006 bash[27292]: audit 2023-08-28T01:36:32.408501+0000 mon.c (mon.2) 2568 : audit [DBG] from='client.? 172.21.15.6:0/463508315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:33.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:33 smithi006 bash[33918]: audit 2023-08-28T01:36:32.408501+0000 mon.c (mon.2) 2568 : audit [DBG] from='client.? 172.21.15.6:0/463508315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:34.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:33 smithi130 bash[51302]: debug 2023-08-28T01:36:33.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:33.755088+0000) 2023-08-28T01:36:34.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:34 smithi006 bash[27292]: cluster 2023-08-28T01:36:32.347815+0000 mgr.y (mgr.14901) 6538 : cluster [DBG] pgmap v6545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:34 smithi006 bash[33918]: cluster 2023-08-28T01:36:32.347815+0000 mgr.y (mgr.14901) 6538 : cluster [DBG] pgmap v6545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:34 smithi130 bash[27579]: cluster 2023-08-28T01:36:32.347815+0000 mgr.y (mgr.14901) 6538 : cluster [DBG] pgmap v6545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:36:34.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:36:34.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:35.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:34 smithi130 bash[51302]: debug 2023-08-28T01:36:34.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:34.755364+0000) 2023-08-28T01:36:35.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:35 smithi006 bash[27292]: audit 2023-08-28T01:36:34.831641+0000 mon.c (mon.2) 2569 : audit [DBG] from='client.? 172.21.15.6:0/2258153444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:35.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:35 smithi006 bash[33918]: audit 2023-08-28T01:36:34.831641+0000 mon.c (mon.2) 2569 : audit [DBG] from='client.? 172.21.15.6:0/2258153444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:35.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:35 smithi130 bash[27579]: audit 2023-08-28T01:36:34.831641+0000 mon.c (mon.2) 2569 : audit [DBG] from='client.? 172.21.15.6:0/2258153444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:36.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:35 smithi130 bash[51302]: debug 2023-08-28T01:36:35.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:35.755650+0000) 2023-08-28T01:36:36.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:36 smithi006 bash[33918]: cluster 2023-08-28T01:36:34.348944+0000 mgr.y (mgr.14901) 6539 : cluster [DBG] pgmap v6546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:36.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:36 smithi006 bash[27292]: cluster 2023-08-28T01:36:34.348944+0000 mgr.y (mgr.14901) 6539 : cluster [DBG] pgmap v6546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:36 smithi130 bash[27579]: cluster 2023-08-28T01:36:34.348944+0000 mgr.y (mgr.14901) 6539 : cluster [DBG] pgmap v6546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:37.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:36 smithi130 bash[51302]: debug 2023-08-28T01:36:36.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:36.755893+0000) 2023-08-28T01:36:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:37 smithi130 bash[51302]: debug 2023-08-28T01:36:37.755+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:37.756147+0000) 2023-08-28T01:36:38.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:38 smithi006 bash[33918]: cluster 2023-08-28T01:36:36.349564+0000 mgr.y (mgr.14901) 6540 : cluster [DBG] pgmap v6547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:38.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:38 smithi006 bash[33918]: audit 2023-08-28T01:36:37.241074+0000 mon.c (mon.2) 2570 : audit [DBG] from='client.? 172.21.15.6:0/2080651918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:38.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:38 smithi006 bash[27292]: cluster 2023-08-28T01:36:36.349564+0000 mgr.y (mgr.14901) 6540 : cluster [DBG] pgmap v6547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:38.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:38 smithi006 bash[27292]: audit 2023-08-28T01:36:37.241074+0000 mon.c (mon.2) 2570 : audit [DBG] from='client.? 172.21.15.6:0/2080651918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:38 smithi130 bash[27579]: cluster 2023-08-28T01:36:36.349564+0000 mgr.y (mgr.14901) 6540 : cluster [DBG] pgmap v6547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:38 smithi130 bash[27579]: audit 2023-08-28T01:36:37.241074+0000 mon.c (mon.2) 2570 : audit [DBG] from='client.? 172.21.15.6:0/2080651918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:39.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:38 smithi130 bash[51302]: debug 2023-08-28T01:36:38.755+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:38.756423+0000) 2023-08-28T01:36:40.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:39 smithi130 bash[51302]: debug 2023-08-28T01:36:39.755+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:39.756701+0000) 2023-08-28T01:36:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:40 smithi006 bash[33918]: cluster 2023-08-28T01:36:38.350327+0000 mgr.y (mgr.14901) 6541 : cluster [DBG] pgmap v6548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:40 smithi006 bash[33918]: audit 2023-08-28T01:36:39.653947+0000 mon.a (mon.0) 4323 : audit [DBG] from='client.? 172.21.15.6:0/192855564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:40.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:40 smithi006 bash[27292]: cluster 2023-08-28T01:36:38.350327+0000 mgr.y (mgr.14901) 6541 : cluster [DBG] pgmap v6548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:40.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:40 smithi006 bash[27292]: audit 2023-08-28T01:36:39.653947+0000 mon.a (mon.0) 4323 : audit [DBG] from='client.? 172.21.15.6:0/192855564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:40 smithi130 bash[27579]: cluster 2023-08-28T01:36:38.350327+0000 mgr.y (mgr.14901) 6541 : cluster [DBG] pgmap v6548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:40 smithi130 bash[27579]: audit 2023-08-28T01:36:39.653947+0000 mon.a (mon.0) 4323 : audit [DBG] from='client.? 172.21.15.6:0/192855564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:41.062 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:36:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:36:40] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:36:41.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:40 smithi130 bash[51302]: debug 2023-08-28T01:36:40.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:40.757038+0000) 2023-08-28T01:36:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:36:41.061Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:36:41.063Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:41 smithi130 bash[51302]: debug 2023-08-28T01:36:41.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:41.757317+0000) 2023-08-28T01:36:42.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:42 smithi006 bash[27292]: cluster 2023-08-28T01:36:40.351380+0000 mgr.y (mgr.14901) 6542 : cluster [DBG] pgmap v6549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:42.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:42 smithi006 bash[27292]: audit 2023-08-28T01:36:42.071146+0000 mon.c (mon.2) 2571 : audit [DBG] from='client.? 172.21.15.6:0/2446353190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:42.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:42 smithi006 bash[33918]: cluster 2023-08-28T01:36:40.351380+0000 mgr.y (mgr.14901) 6542 : cluster [DBG] pgmap v6549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:42.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:42 smithi006 bash[33918]: audit 2023-08-28T01:36:42.071146+0000 mon.c (mon.2) 2571 : audit [DBG] from='client.? 172.21.15.6:0/2446353190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:42 smithi130 bash[27579]: cluster 2023-08-28T01:36:40.351380+0000 mgr.y (mgr.14901) 6542 : cluster [DBG] pgmap v6549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:42 smithi130 bash[27579]: audit 2023-08-28T01:36:42.071146+0000 mon.c (mon.2) 2571 : audit [DBG] from='client.? 172.21.15.6:0/2446353190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:36:42.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:36:42.633Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:43.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:42 smithi130 bash[51302]: debug 2023-08-28T01:36:42.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:42.757560+0000) 2023-08-28T01:36:44.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:43 smithi130 bash[51302]: debug 2023-08-28T01:36:43.755+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:43.757844+0000) 2023-08-28T01:36:44.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:44 smithi006 bash[27292]: cluster 2023-08-28T01:36:42.352059+0000 mgr.y (mgr.14901) 6543 : cluster [DBG] pgmap v6550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:44 smithi006 bash[33918]: cluster 2023-08-28T01:36:42.352059+0000 mgr.y (mgr.14901) 6543 : cluster [DBG] pgmap v6550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:44 smithi130 bash[27579]: cluster 2023-08-28T01:36:42.352059+0000 mgr.y (mgr.14901) 6543 : cluster [DBG] pgmap v6550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:36:44.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:36:44.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:45.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:44 smithi130 bash[51302]: debug 2023-08-28T01:36:44.755+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:44.758166+0000) 2023-08-28T01:36:45.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:45 smithi006 bash[33918]: audit 2023-08-28T01:36:44.486718+0000 mon.c (mon.2) 2572 : audit [DBG] from='client.? 172.21.15.6:0/3925868106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:45.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:45 smithi006 bash[27292]: audit 2023-08-28T01:36:44.486718+0000 mon.c (mon.2) 2572 : audit [DBG] from='client.? 172.21.15.6:0/3925868106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:45.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:45 smithi130 bash[27579]: audit 2023-08-28T01:36:44.486718+0000 mon.c (mon.2) 2572 : audit [DBG] from='client.? 172.21.15.6:0/3925868106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:45 smithi130 bash[51302]: debug 2023-08-28T01:36:45.755+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:45.758426+0000) 2023-08-28T01:36:46.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:46 smithi006 bash[33918]: cluster 2023-08-28T01:36:44.353258+0000 mgr.y (mgr.14901) 6544 : cluster [DBG] pgmap v6551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:46.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:46 smithi006 bash[27292]: cluster 2023-08-28T01:36:44.353258+0000 mgr.y (mgr.14901) 6544 : cluster [DBG] pgmap v6551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:46 smithi130 bash[27579]: cluster 2023-08-28T01:36:44.353258+0000 mgr.y (mgr.14901) 6544 : cluster [DBG] pgmap v6551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:46 smithi130 bash[51302]: debug 2023-08-28T01:36:46.751+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:46.758703+0000) 2023-08-28T01:36:47.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:47 smithi006 bash[27292]: audit 2023-08-28T01:36:46.899834+0000 mon.a (mon.0) 4324 : audit [DBG] from='client.? 172.21.15.6:0/1447630614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:47.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:47 smithi006 bash[33918]: audit 2023-08-28T01:36:46.899834+0000 mon.a (mon.0) 4324 : audit [DBG] from='client.? 172.21.15.6:0/1447630614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:47.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:47 smithi130 bash[27579]: audit 2023-08-28T01:36:46.899834+0000 mon.a (mon.0) 4324 : audit [DBG] from='client.? 172.21.15.6:0/1447630614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:47 smithi130 bash[51302]: debug 2023-08-28T01:36:47.755+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:47.759016+0000) 2023-08-28T01:36:48.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:48 smithi006 bash[27292]: cluster 2023-08-28T01:36:46.354117+0000 mgr.y (mgr.14901) 6545 : cluster [DBG] pgmap v6552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:48.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:48 smithi006 bash[33918]: cluster 2023-08-28T01:36:46.354117+0000 mgr.y (mgr.14901) 6545 : cluster [DBG] pgmap v6552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:48 smithi130 bash[27579]: cluster 2023-08-28T01:36:46.354117+0000 mgr.y (mgr.14901) 6545 : cluster [DBG] pgmap v6552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:49.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:48 smithi130 bash[51302]: debug 2023-08-28T01:36:48.755+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:48.759342+0000) 2023-08-28T01:36:50.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:49 smithi130 bash[51302]: debug 2023-08-28T01:36:49.759+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:49.759617+0000) 2023-08-28T01:36:50.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:50 smithi006 bash[27292]: cluster 2023-08-28T01:36:48.354938+0000 mgr.y (mgr.14901) 6546 : cluster [DBG] pgmap v6553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:50.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:50 smithi006 bash[27292]: audit 2023-08-28T01:36:49.317458+0000 mon.c (mon.2) 2573 : audit [DBG] from='client.? 172.21.15.6:0/2428121602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:50.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:50 smithi006 bash[33918]: cluster 2023-08-28T01:36:48.354938+0000 mgr.y (mgr.14901) 6546 : cluster [DBG] pgmap v6553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:50.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:50 smithi006 bash[33918]: audit 2023-08-28T01:36:49.317458+0000 mon.c (mon.2) 2573 : audit [DBG] from='client.? 172.21.15.6:0/2428121602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:50 smithi130 bash[27579]: cluster 2023-08-28T01:36:48.354938+0000 mgr.y (mgr.14901) 6546 : cluster [DBG] pgmap v6553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:50 smithi130 bash[27579]: audit 2023-08-28T01:36:49.317458+0000 mon.c (mon.2) 2573 : audit [DBG] from='client.? 172.21.15.6:0/2428121602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:51.062 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:36:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:36:50] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:36:51.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:50 smithi130 bash[51302]: debug 2023-08-28T01:36:50.755+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:50.759913+0000) 2023-08-28T01:36:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:36:51.062Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:36:51.064Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:52.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:51 smithi130 bash[51302]: debug 2023-08-28T01:36:51.759+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:51.760222+0000) 2023-08-28T01:36:52.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:52 smithi006 bash[27292]: cluster 2023-08-28T01:36:50.356013+0000 mgr.y (mgr.14901) 6547 : cluster [DBG] pgmap v6554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:52.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:52 smithi006 bash[27292]: audit 2023-08-28T01:36:51.744732+0000 mon.c (mon.2) 2574 : audit [DBG] from='client.? 172.21.15.6:0/18077995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:52.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:52 smithi006 bash[33918]: cluster 2023-08-28T01:36:50.356013+0000 mgr.y (mgr.14901) 6547 : cluster [DBG] pgmap v6554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:52.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:52 smithi006 bash[33918]: audit 2023-08-28T01:36:51.744732+0000 mon.c (mon.2) 2574 : audit [DBG] from='client.? 172.21.15.6:0/18077995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:52 smithi130 bash[27579]: cluster 2023-08-28T01:36:50.356013+0000 mgr.y (mgr.14901) 6547 : cluster [DBG] pgmap v6554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:52 smithi130 bash[27579]: audit 2023-08-28T01:36:51.744732+0000 mon.c (mon.2) 2574 : audit [DBG] from='client.? 172.21.15.6:0/18077995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:36:52.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:36:52.634Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:53.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:52 smithi130 bash[51302]: debug 2023-08-28T01:36:52.755+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:52.760483+0000) 2023-08-28T01:36:54.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:53 smithi130 bash[51302]: debug 2023-08-28T01:36:53.759+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:53.760760+0000) 2023-08-28T01:36:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:54 smithi006 bash[33918]: cluster 2023-08-28T01:36:52.356646+0000 mgr.y (mgr.14901) 6548 : cluster [DBG] pgmap v6555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:54 smithi006 bash[33918]: audit 2023-08-28T01:36:54.168015+0000 mon.a (mon.0) 4325 : audit [DBG] from='client.? 172.21.15.6:0/12848622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:54 smithi006 bash[27292]: cluster 2023-08-28T01:36:52.356646+0000 mgr.y (mgr.14901) 6548 : cluster [DBG] pgmap v6555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:54 smithi006 bash[27292]: audit 2023-08-28T01:36:54.168015+0000 mon.a (mon.0) 4325 : audit [DBG] from='client.? 172.21.15.6:0/12848622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:54 smithi130 bash[27579]: cluster 2023-08-28T01:36:52.356646+0000 mgr.y (mgr.14901) 6548 : cluster [DBG] pgmap v6555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:54 smithi130 bash[27579]: audit 2023-08-28T01:36:54.168015+0000 mon.a (mon.0) 4325 : audit [DBG] from='client.? 172.21.15.6:0/12848622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:36:54.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:36:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:36:54.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:36:55.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:54 smithi130 bash[51302]: debug 2023-08-28T01:36:54.755+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:54.761065+0000) 2023-08-28T01:36:56.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:55 smithi130 bash[51302]: debug 2023-08-28T01:36:55.755+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:55.761361+0000) 2023-08-28T01:36:56.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:56 smithi006 bash[33918]: cluster 2023-08-28T01:36:54.357778+0000 mgr.y (mgr.14901) 6549 : cluster [DBG] pgmap v6556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:56.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:56 smithi006 bash[27292]: cluster 2023-08-28T01:36:54.357778+0000 mgr.y (mgr.14901) 6549 : cluster [DBG] pgmap v6556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:56 smithi130 bash[27579]: cluster 2023-08-28T01:36:54.357778+0000 mgr.y (mgr.14901) 6549 : cluster [DBG] pgmap v6556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:57.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:56 smithi130 bash[51302]: debug 2023-08-28T01:36:56.755+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:56.761652+0000) 2023-08-28T01:36:57.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:57 smithi006 bash[27292]: audit 2023-08-28T01:36:56.580581+0000 mon.a (mon.0) 4326 : audit [DBG] from='client.? 172.21.15.6:0/1302958417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:57.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:57 smithi006 bash[33918]: audit 2023-08-28T01:36:56.580581+0000 mon.a (mon.0) 4326 : audit [DBG] from='client.? 172.21.15.6:0/1302958417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:57.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:57 smithi130 bash[27579]: audit 2023-08-28T01:36:56.580581+0000 mon.a (mon.0) 4326 : audit [DBG] from='client.? 172.21.15.6:0/1302958417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:58.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:57 smithi130 bash[51302]: debug 2023-08-28T01:36:57.759+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:57.761958+0000) 2023-08-28T01:36:58.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:58 smithi006 bash[27292]: cluster 2023-08-28T01:36:56.358410+0000 mgr.y (mgr.14901) 6550 : cluster [DBG] pgmap v6557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:58.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:58 smithi006 bash[33918]: cluster 2023-08-28T01:36:56.358410+0000 mgr.y (mgr.14901) 6550 : cluster [DBG] pgmap v6557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:58 smithi130 bash[27579]: cluster 2023-08-28T01:36:56.358410+0000 mgr.y (mgr.14901) 6550 : cluster [DBG] pgmap v6557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:36:59.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:58 smithi130 bash[51302]: debug 2023-08-28T01:36:58.759+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:58.762236+0000) 2023-08-28T01:36:59.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:36:59 smithi006 bash[33918]: audit 2023-08-28T01:36:58.992268+0000 mon.c (mon.2) 2575 : audit [DBG] from='client.? 172.21.15.6:0/1900387062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:59.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:36:59 smithi006 bash[27292]: audit 2023-08-28T01:36:58.992268+0000 mon.c (mon.2) 2575 : audit [DBG] from='client.? 172.21.15.6:0/1900387062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:59.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:36:59 smithi130 bash[27579]: audit 2023-08-28T01:36:58.992268+0000 mon.c (mon.2) 2575 : audit [DBG] from='client.? 172.21.15.6:0/1900387062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:36:59 smithi130 bash[51302]: debug 2023-08-28T01:36:59.755+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:59.762539+0000) 2023-08-28T01:37:00.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:00 smithi006 bash[33918]: cluster 2023-08-28T01:36:58.359091+0000 mgr.y (mgr.14901) 6551 : cluster [DBG] pgmap v6558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:00.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:00 smithi006 bash[27292]: cluster 2023-08-28T01:36:58.359091+0000 mgr.y (mgr.14901) 6551 : cluster [DBG] pgmap v6558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:00 smithi130 bash[27579]: cluster 2023-08-28T01:36:58.359091+0000 mgr.y (mgr.14901) 6551 : cluster [DBG] pgmap v6558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:01.063 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:37:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:37:00] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:37:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:00 smithi130 bash[51302]: debug 2023-08-28T01:37:00.759+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:00.762842+0000) 2023-08-28T01:37:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:37:01.063Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:37:01.065Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:02.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:01 smithi130 bash[51302]: debug 2023-08-28T01:37:01.759+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:01.763156+0000) 2023-08-28T01:37:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:02 smithi130 bash[27579]: cluster 2023-08-28T01:37:00.360231+0000 mgr.y (mgr.14901) 6552 : cluster [DBG] pgmap v6559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:02 smithi130 bash[27579]: audit 2023-08-28T01:37:01.421223+0000 mon.c (mon.2) 2576 : audit [DBG] from='client.? 172.21.15.6:0/1906487511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:02.633 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:02 smithi006 bash[27292]: cluster 2023-08-28T01:37:00.360231+0000 mgr.y (mgr.14901) 6552 : cluster [DBG] pgmap v6559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:02.633 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:02 smithi006 bash[27292]: audit 2023-08-28T01:37:01.421223+0000 mon.c (mon.2) 2576 : audit [DBG] from='client.? 172.21.15.6:0/1906487511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:02.633 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:02 smithi006 bash[33918]: cluster 2023-08-28T01:37:00.360231+0000 mgr.y (mgr.14901) 6552 : cluster [DBG] pgmap v6559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:02.633 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:02 smithi006 bash[33918]: audit 2023-08-28T01:37:01.421223+0000 mon.c (mon.2) 2576 : audit [DBG] from='client.? 172.21.15.6:0/1906487511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:37:02.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:37:02.634Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:03.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:02 smithi130 bash[51302]: debug 2023-08-28T01:37:02.759+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:02.763385+0000) 2023-08-28T01:37:04.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:03 smithi130 bash[51302]: debug 2023-08-28T01:37:03.759+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:03.763603+0000) 2023-08-28T01:37:04.520 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:04 smithi006 bash[33918]: cluster 2023-08-28T01:37:02.360940+0000 mgr.y (mgr.14901) 6553 : cluster [DBG] pgmap v6560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:04.520 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:04 smithi006 bash[33918]: audit 2023-08-28T01:37:03.838677+0000 mon.c (mon.2) 2577 : audit [DBG] from='client.? 172.21.15.6:0/3797477924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:04.520 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:04 smithi006 bash[27292]: cluster 2023-08-28T01:37:02.360940+0000 mgr.y (mgr.14901) 6553 : cluster [DBG] pgmap v6560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:04.521 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:04 smithi006 bash[27292]: audit 2023-08-28T01:37:03.838677+0000 mon.c (mon.2) 2577 : audit [DBG] from='client.? 172.21.15.6:0/3797477924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:04 smithi130 bash[27579]: cluster 2023-08-28T01:37:02.360940+0000 mgr.y (mgr.14901) 6553 : cluster [DBG] pgmap v6560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:04 smithi130 bash[27579]: audit 2023-08-28T01:37:03.838677+0000 mon.c (mon.2) 2577 : audit [DBG] from='client.? 172.21.15.6:0/3797477924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:37:04.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:37:04.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:05.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:04 smithi130 bash[51302]: debug 2023-08-28T01:37:04.759+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:04.763880+0000) 2023-08-28T01:37:06.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:05 smithi130 bash[51302]: debug 2023-08-28T01:37:05.759+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:05.764191+0000) 2023-08-28T01:37:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:06 smithi130 bash[27579]: cluster 2023-08-28T01:37:04.361945+0000 mgr.y (mgr.14901) 6554 : cluster [DBG] pgmap v6561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:06.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:06 smithi006 bash[27292]: cluster 2023-08-28T01:37:04.361945+0000 mgr.y (mgr.14901) 6554 : cluster [DBG] pgmap v6561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:06 smithi006 bash[33918]: cluster 2023-08-28T01:37:04.361945+0000 mgr.y (mgr.14901) 6554 : cluster [DBG] pgmap v6561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:07.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:06 smithi130 bash[51302]: debug 2023-08-28T01:37:06.763+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:06.764487+0000) 2023-08-28T01:37:07.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:07 smithi130 bash[27579]: audit 2023-08-28T01:37:06.252256+0000 mon.c (mon.2) 2578 : audit [DBG] from='client.? 172.21.15.6:0/3800054366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:07.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:07 smithi006 bash[27292]: audit 2023-08-28T01:37:06.252256+0000 mon.c (mon.2) 2578 : audit [DBG] from='client.? 172.21.15.6:0/3800054366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:07.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:07 smithi006 bash[33918]: audit 2023-08-28T01:37:06.252256+0000 mon.c (mon.2) 2578 : audit [DBG] from='client.? 172.21.15.6:0/3800054366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:08.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:07 smithi130 bash[51302]: debug 2023-08-28T01:37:07.763+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:07.764761+0000) 2023-08-28T01:37:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:08 smithi130 bash[27579]: cluster 2023-08-28T01:37:06.362608+0000 mgr.y (mgr.14901) 6555 : cluster [DBG] pgmap v6562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:08 smithi006 bash[33918]: cluster 2023-08-28T01:37:06.362608+0000 mgr.y (mgr.14901) 6555 : cluster [DBG] pgmap v6562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:08 smithi006 bash[27292]: cluster 2023-08-28T01:37:06.362608+0000 mgr.y (mgr.14901) 6555 : cluster [DBG] pgmap v6562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:09.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:08 smithi130 bash[51302]: debug 2023-08-28T01:37:08.763+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:08.765044+0000) 2023-08-28T01:37:09.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:09 smithi130 bash[27579]: audit 2023-08-28T01:37:08.672400+0000 mon.a (mon.0) 4327 : audit [DBG] from='client.? 172.21.15.6:0/3882333258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:09.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:09 smithi006 bash[27292]: audit 2023-08-28T01:37:08.672400+0000 mon.a (mon.0) 4327 : audit [DBG] from='client.? 172.21.15.6:0/3882333258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:09.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:09 smithi006 bash[33918]: audit 2023-08-28T01:37:08.672400+0000 mon.a (mon.0) 4327 : audit [DBG] from='client.? 172.21.15.6:0/3882333258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:10.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:09 smithi130 bash[51302]: debug 2023-08-28T01:37:09.763+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:09.765318+0000) 2023-08-28T01:37:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:10 smithi130 bash[27579]: cluster 2023-08-28T01:37:08.363261+0000 mgr.y (mgr.14901) 6556 : cluster [DBG] pgmap v6563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:10.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:10 smithi006 bash[27292]: cluster 2023-08-28T01:37:08.363261+0000 mgr.y (mgr.14901) 6556 : cluster [DBG] pgmap v6563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:10.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:10 smithi006 bash[33918]: cluster 2023-08-28T01:37:08.363261+0000 mgr.y (mgr.14901) 6556 : cluster [DBG] pgmap v6563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:11.064 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:37:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:37:10] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:37:11.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:10 smithi130 bash[51302]: debug 2023-08-28T01:37:10.763+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:10.765654+0000) 2023-08-28T01:37:11.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:11 smithi006 bash[27292]: audit 2023-08-28T01:37:11.084042+0000 mon.c (mon.2) 2579 : audit [DBG] from='client.? 172.21.15.6:0/1738880024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:11.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:11 smithi006 bash[33918]: audit 2023-08-28T01:37:11.084042+0000 mon.c (mon.2) 2579 : audit [DBG] from='client.? 172.21.15.6:0/1738880024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:11.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:37:11.063Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:11.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:37:11.065Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:11.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:11 smithi130 bash[27579]: audit 2023-08-28T01:37:11.084042+0000 mon.c (mon.2) 2579 : audit [DBG] from='client.? 172.21.15.6:0/1738880024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:12.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:11 smithi130 bash[51302]: debug 2023-08-28T01:37:11.758+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:11.765957+0000) 2023-08-28T01:37:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:12 smithi130 bash[27579]: cluster 2023-08-28T01:37:10.364227+0000 mgr.y (mgr.14901) 6557 : cluster [DBG] pgmap v6564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:12.633 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:12 smithi006 bash[27292]: cluster 2023-08-28T01:37:10.364227+0000 mgr.y (mgr.14901) 6557 : cluster [DBG] pgmap v6564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:12.633 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:12 smithi006 bash[33918]: cluster 2023-08-28T01:37:10.364227+0000 mgr.y (mgr.14901) 6557 : cluster [DBG] pgmap v6564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:37:12.633Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:37:12.634Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:13.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:12 smithi130 bash[51302]: debug 2023-08-28T01:37:12.758+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:12.766234+0000) 2023-08-28T01:37:14.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:13 smithi130 bash[51302]: debug 2023-08-28T01:37:13.762+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:13.766544+0000) 2023-08-28T01:37:14.521 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:14 smithi006 bash[33918]: cluster 2023-08-28T01:37:12.364995+0000 mgr.y (mgr.14901) 6558 : cluster [DBG] pgmap v6565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:14.521 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:14 smithi006 bash[33918]: audit 2023-08-28T01:37:13.503595+0000 mon.a (mon.0) 4328 : audit [DBG] from='client.? 172.21.15.6:0/3674046943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:14.521 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:14 smithi006 bash[27292]: cluster 2023-08-28T01:37:12.364995+0000 mgr.y (mgr.14901) 6558 : cluster [DBG] pgmap v6565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:14.521 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:14 smithi006 bash[27292]: audit 2023-08-28T01:37:13.503595+0000 mon.a (mon.0) 4328 : audit [DBG] from='client.? 172.21.15.6:0/3674046943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:14 smithi130 bash[27579]: cluster 2023-08-28T01:37:12.364995+0000 mgr.y (mgr.14901) 6558 : cluster [DBG] pgmap v6565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:14 smithi130 bash[27579]: audit 2023-08-28T01:37:13.503595+0000 mon.a (mon.0) 4328 : audit [DBG] from='client.? 172.21.15.6:0/3674046943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:37:14.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:37:14.523Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:14 smithi130 bash[51302]: debug 2023-08-28T01:37:14.762+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:14.766874+0000) 2023-08-28T01:37:16.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:15 smithi130 bash[51302]: debug 2023-08-28T01:37:15.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:15.767069+0000) 2023-08-28T01:37:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:16 smithi130 bash[27579]: cluster 2023-08-28T01:37:14.366132+0000 mgr.y (mgr.14901) 6559 : cluster [DBG] pgmap v6566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:16 smithi130 bash[27579]: audit 2023-08-28T01:37:15.914218+0000 mon.a (mon.0) 4329 : audit [DBG] from='client.? 172.21.15.6:0/3429327764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:16 smithi006 bash[27292]: cluster 2023-08-28T01:37:14.366132+0000 mgr.y (mgr.14901) 6559 : cluster [DBG] pgmap v6566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:16 smithi006 bash[27292]: audit 2023-08-28T01:37:15.914218+0000 mon.a (mon.0) 4329 : audit [DBG] from='client.? 172.21.15.6:0/3429327764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:16 smithi006 bash[33918]: cluster 2023-08-28T01:37:14.366132+0000 mgr.y (mgr.14901) 6559 : cluster [DBG] pgmap v6566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:16 smithi006 bash[33918]: audit 2023-08-28T01:37:15.914218+0000 mon.a (mon.0) 4329 : audit [DBG] from='client.? 172.21.15.6:0/3429327764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:17.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:16 smithi130 bash[51302]: debug 2023-08-28T01:37:16.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:16.767283+0000) 2023-08-28T01:37:18.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:17 smithi130 bash[51302]: debug 2023-08-28T01:37:17.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:17.767501+0000) 2023-08-28T01:37:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:18 smithi130 bash[27579]: cluster 2023-08-28T01:37:16.366907+0000 mgr.y (mgr.14901) 6560 : cluster [DBG] pgmap v6567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:18.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:18 smithi006 bash[27292]: cluster 2023-08-28T01:37:16.366907+0000 mgr.y (mgr.14901) 6560 : cluster [DBG] pgmap v6567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:18.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:18 smithi006 bash[33918]: cluster 2023-08-28T01:37:16.366907+0000 mgr.y (mgr.14901) 6560 : cluster [DBG] pgmap v6567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:19.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:18 smithi130 bash[51302]: debug 2023-08-28T01:37:18.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:18.767789+0000) 2023-08-28T01:37:19.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:19 smithi130 bash[27579]: audit 2023-08-28T01:37:18.335204+0000 mon.c (mon.2) 2580 : audit [DBG] from='client.? 172.21.15.6:0/3232128360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:19.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:19 smithi006 bash[27292]: audit 2023-08-28T01:37:18.335204+0000 mon.c (mon.2) 2580 : audit [DBG] from='client.? 172.21.15.6:0/3232128360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:19.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:19 smithi006 bash[33918]: audit 2023-08-28T01:37:18.335204+0000 mon.c (mon.2) 2580 : audit [DBG] from='client.? 172.21.15.6:0/3232128360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:19 smithi130 bash[51302]: debug 2023-08-28T01:37:19.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:19.768025+0000) 2023-08-28T01:37:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:20 smithi130 bash[27579]: cluster 2023-08-28T01:37:18.367630+0000 mgr.y (mgr.14901) 6561 : cluster [DBG] pgmap v6568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:20 smithi130 bash[27579]: audit 2023-08-28T01:37:19.567896+0000 mon.b (mon.1) 745 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:37:20.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:20 smithi006 bash[27292]: cluster 2023-08-28T01:37:18.367630+0000 mgr.y (mgr.14901) 6561 : cluster [DBG] pgmap v6568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:20.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:20 smithi006 bash[27292]: audit 2023-08-28T01:37:19.567896+0000 mon.b (mon.1) 745 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:37:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:20 smithi006 bash[33918]: cluster 2023-08-28T01:37:18.367630+0000 mgr.y (mgr.14901) 6561 : cluster [DBG] pgmap v6568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:20 smithi006 bash[33918]: audit 2023-08-28T01:37:19.567896+0000 mon.b (mon.1) 745 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:37:21.065 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:37:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:37:20] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:37:21.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:20 smithi130 bash[51302]: debug 2023-08-28T01:37:20.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:20.768322+0000) 2023-08-28T01:37:21.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:21 smithi006 bash[27292]: cluster 2023-08-28T01:37:20.368309+0000 mgr.y (mgr.14901) 6562 : cluster [DBG] pgmap v6569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:21.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:21 smithi006 bash[27292]: audit 2023-08-28T01:37:20.744695+0000 mon.a (mon.0) 4330 : audit [DBG] from='client.? 172.21.15.6:0/1870430712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:37:21.065Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:37:21.066Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:21.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:21 smithi006 bash[33918]: cluster 2023-08-28T01:37:20.368309+0000 mgr.y (mgr.14901) 6562 : cluster [DBG] pgmap v6569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:21.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:21 smithi006 bash[33918]: audit 2023-08-28T01:37:20.744695+0000 mon.a (mon.0) 4330 : audit [DBG] from='client.? 172.21.15.6:0/1870430712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:21.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:21 smithi130 bash[27579]: cluster 2023-08-28T01:37:20.368309+0000 mgr.y (mgr.14901) 6562 : cluster [DBG] pgmap v6569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:21.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:21 smithi130 bash[27579]: audit 2023-08-28T01:37:20.744695+0000 mon.a (mon.0) 4330 : audit [DBG] from='client.? 172.21.15.6:0/1870430712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:22.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:21 smithi130 bash[51302]: debug 2023-08-28T01:37:21.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:21.768679+0000) 2023-08-28T01:37:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:37:22.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:37:22.636Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:23.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:22 smithi130 bash[51302]: debug 2023-08-28T01:37:22.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:22.768967+0000) 2023-08-28T01:37:24.089 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:23 smithi130 bash[51302]: debug 2023-08-28T01:37:23.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:23.769228+0000) 2023-08-28T01:37:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:24 smithi130 bash[27579]: cluster 2023-08-28T01:37:22.368923+0000 mgr.y (mgr.14901) 6563 : cluster [DBG] pgmap v6570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:24 smithi130 bash[27579]: audit 2023-08-28T01:37:23.169315+0000 mon.c (mon.2) 2581 : audit [DBG] from='client.? 172.21.15.6:0/4187499701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:24.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:24 smithi006 bash[33918]: cluster 2023-08-28T01:37:22.368923+0000 mgr.y (mgr.14901) 6563 : cluster [DBG] pgmap v6570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:24.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:24 smithi006 bash[33918]: audit 2023-08-28T01:37:23.169315+0000 mon.c (mon.2) 2581 : audit [DBG] from='client.? 172.21.15.6:0/4187499701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:24 smithi006 bash[27292]: cluster 2023-08-28T01:37:22.368923+0000 mgr.y (mgr.14901) 6563 : cluster [DBG] pgmap v6570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:24 smithi006 bash[27292]: audit 2023-08-28T01:37:23.169315+0000 mon.c (mon.2) 2581 : audit [DBG] from='client.? 172.21.15.6:0/4187499701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:37:24.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:37:24.523Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:25.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:24 smithi130 bash[51302]: debug 2023-08-28T01:37:24.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:24.769450+0000) 2023-08-28T01:37:26.095 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:25 smithi130 bash[51302]: debug 2023-08-28T01:37:25.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:25.769616+0000) 2023-08-28T01:37:26.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:26 smithi130 bash[27579]: cluster 2023-08-28T01:37:24.369961+0000 mgr.y (mgr.14901) 6564 : cluster [DBG] pgmap v6571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:26.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:26 smithi130 bash[27579]: audit 2023-08-28T01:37:25.445011+0000 mon.a (mon.0) 4331 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:37:26.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:26 smithi130 bash[27579]: audit 2023-08-28T01:37:25.616404+0000 mon.c (mon.2) 2582 : audit [DBG] from='client.? 172.21.15.6:0/210530483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:26.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:26 smithi130 bash[27579]: audit 2023-08-28T01:37:25.889845+0000 mon.a (mon.0) 4332 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:37:26.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:26 smithi006 bash[33918]: cluster 2023-08-28T01:37:24.369961+0000 mgr.y (mgr.14901) 6564 : cluster [DBG] pgmap v6571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:26.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:26 smithi006 bash[33918]: audit 2023-08-28T01:37:25.445011+0000 mon.a (mon.0) 4331 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:37:26.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:26 smithi006 bash[33918]: audit 2023-08-28T01:37:25.616404+0000 mon.c (mon.2) 2582 : audit [DBG] from='client.? 172.21.15.6:0/210530483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:26.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:26 smithi006 bash[33918]: audit 2023-08-28T01:37:25.889845+0000 mon.a (mon.0) 4332 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:37:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:26 smithi006 bash[27292]: cluster 2023-08-28T01:37:24.369961+0000 mgr.y (mgr.14901) 6564 : cluster [DBG] pgmap v6571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:26 smithi006 bash[27292]: audit 2023-08-28T01:37:25.445011+0000 mon.a (mon.0) 4331 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:37:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:26 smithi006 bash[27292]: audit 2023-08-28T01:37:25.616404+0000 mon.c (mon.2) 2582 : audit [DBG] from='client.? 172.21.15.6:0/210530483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:26.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:26 smithi006 bash[27292]: audit 2023-08-28T01:37:25.889845+0000 mon.a (mon.0) 4332 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:37:27.101 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:26 smithi130 bash[51302]: debug 2023-08-28T01:37:26.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:26.769917+0000) 2023-08-28T01:37:27.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:27 smithi130 bash[27579]: audit 2023-08-28T01:37:26.187825+0000 mon.b (mon.1) 746 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:37:27.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:27 smithi130 bash[27579]: audit 2023-08-28T01:37:26.189959+0000 mon.b (mon.1) 747 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:37:27.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:27 smithi130 bash[27579]: audit 2023-08-28T01:37:26.200325+0000 mon.a (mon.0) 4333 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:37:27.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:27 smithi006 bash[33918]: audit 2023-08-28T01:37:26.187825+0000 mon.b (mon.1) 746 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:37:27.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:27 smithi006 bash[33918]: audit 2023-08-28T01:37:26.189959+0000 mon.b (mon.1) 747 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:37:27.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:27 smithi006 bash[33918]: audit 2023-08-28T01:37:26.200325+0000 mon.a (mon.0) 4333 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:37:27.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:27 smithi006 bash[27292]: audit 2023-08-28T01:37:26.187825+0000 mon.b (mon.1) 746 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:37:27.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:27 smithi006 bash[27292]: audit 2023-08-28T01:37:26.189959+0000 mon.b (mon.1) 747 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:37:27.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:27 smithi006 bash[27292]: audit 2023-08-28T01:37:26.200325+0000 mon.a (mon.0) 4333 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:37:28.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:27 smithi130 bash[51302]: debug 2023-08-28T01:37:27.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:27.770183+0000) 2023-08-28T01:37:28.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:28 smithi006 bash[27292]: cluster 2023-08-28T01:37:26.370672+0000 mgr.y (mgr.14901) 6565 : cluster [DBG] pgmap v6572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:28.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:28 smithi006 bash[27292]: audit 2023-08-28T01:37:28.033861+0000 mon.a (mon.0) 4334 : audit [DBG] from='client.? 172.21.15.6:0/549189276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:28.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:28 smithi006 bash[33918]: cluster 2023-08-28T01:37:26.370672+0000 mgr.y (mgr.14901) 6565 : cluster [DBG] pgmap v6572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:28.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:28 smithi006 bash[33918]: audit 2023-08-28T01:37:28.033861+0000 mon.a (mon.0) 4334 : audit [DBG] from='client.? 172.21.15.6:0/549189276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:28 smithi130 bash[27579]: cluster 2023-08-28T01:37:26.370672+0000 mgr.y (mgr.14901) 6565 : cluster [DBG] pgmap v6572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:28 smithi130 bash[27579]: audit 2023-08-28T01:37:28.033861+0000 mon.a (mon.0) 4334 : audit [DBG] from='client.? 172.21.15.6:0/549189276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:28 smithi130 bash[51302]: debug 2023-08-28T01:37:28.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:28.770479+0000) 2023-08-28T01:37:30.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:29 smithi130 bash[51302]: debug 2023-08-28T01:37:29.766+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:29.770802+0000) 2023-08-28T01:37:30.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:30 smithi006 bash[33918]: cluster 2023-08-28T01:37:28.371383+0000 mgr.y (mgr.14901) 6566 : cluster [DBG] pgmap v6573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:30.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:30 smithi006 bash[27292]: cluster 2023-08-28T01:37:28.371383+0000 mgr.y (mgr.14901) 6566 : cluster [DBG] pgmap v6573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:30 smithi130 bash[27579]: cluster 2023-08-28T01:37:28.371383+0000 mgr.y (mgr.14901) 6566 : cluster [DBG] pgmap v6573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:31.066 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:37:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:37:30] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:37:31.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:30 smithi130 bash[51302]: debug 2023-08-28T01:37:30.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:30.771114+0000) 2023-08-28T01:37:31.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:31 smithi006 bash[27292]: audit 2023-08-28T01:37:30.460997+0000 mon.a (mon.0) 4335 : audit [DBG] from='client.? 172.21.15.6:0/398365818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:31.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:31 smithi006 bash[33918]: audit 2023-08-28T01:37:30.460997+0000 mon.a (mon.0) 4335 : audit [DBG] from='client.? 172.21.15.6:0/398365818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:37:31.066Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:37:31.068Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:31.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:31 smithi130 bash[27579]: audit 2023-08-28T01:37:30.460997+0000 mon.a (mon.0) 4335 : audit [DBG] from='client.? 172.21.15.6:0/398365818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:32.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:31 smithi130 bash[51302]: debug 2023-08-28T01:37:31.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:31.771410+0000) 2023-08-28T01:37:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:32 smithi130 bash[27579]: cluster 2023-08-28T01:37:30.372422+0000 mgr.y (mgr.14901) 6567 : cluster [DBG] pgmap v6574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:32.634 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:32 smithi006 bash[27292]: cluster 2023-08-28T01:37:30.372422+0000 mgr.y (mgr.14901) 6567 : cluster [DBG] pgmap v6574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:32.635 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:32 smithi006 bash[33918]: cluster 2023-08-28T01:37:30.372422+0000 mgr.y (mgr.14901) 6567 : cluster [DBG] pgmap v6574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:37:32.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:37:32.636Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:33.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:32 smithi130 bash[51302]: debug 2023-08-28T01:37:32.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:32.771756+0000) 2023-08-28T01:37:33.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:33 smithi130 bash[27579]: audit 2023-08-28T01:37:32.873484+0000 mon.a (mon.0) 4336 : audit [DBG] from='client.? 172.21.15.6:0/601846319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:33.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:33 smithi006 bash[27292]: audit 2023-08-28T01:37:32.873484+0000 mon.a (mon.0) 4336 : audit [DBG] from='client.? 172.21.15.6:0/601846319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:33.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:33 smithi006 bash[33918]: audit 2023-08-28T01:37:32.873484+0000 mon.a (mon.0) 4336 : audit [DBG] from='client.? 172.21.15.6:0/601846319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:34.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:33 smithi130 bash[51302]: debug 2023-08-28T01:37:33.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:33.772059+0000) 2023-08-28T01:37:34.523 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:34 smithi006 bash[27292]: cluster 2023-08-28T01:37:32.373107+0000 mgr.y (mgr.14901) 6568 : cluster [DBG] pgmap v6575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:34.523 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:34 smithi006 bash[33918]: cluster 2023-08-28T01:37:32.373107+0000 mgr.y (mgr.14901) 6568 : cluster [DBG] pgmap v6575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:34 smithi130 bash[27579]: cluster 2023-08-28T01:37:32.373107+0000 mgr.y (mgr.14901) 6568 : cluster [DBG] pgmap v6575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:37:34.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:37:34.524Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:35.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:34 smithi130 bash[51302]: debug 2023-08-28T01:37:34.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:34.772379+0000) 2023-08-28T01:37:36.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:35 smithi130 bash[51302]: debug 2023-08-28T01:37:35.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:35.772678+0000) 2023-08-28T01:37:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:36 smithi130 bash[27579]: cluster 2023-08-28T01:37:34.374214+0000 mgr.y (mgr.14901) 6569 : cluster [DBG] pgmap v6576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:36 smithi130 bash[27579]: audit 2023-08-28T01:37:35.285861+0000 mon.a (mon.0) 4337 : audit [DBG] from='client.? 172.21.15.6:0/1379552683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:36.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:36 smithi006 bash[27292]: cluster 2023-08-28T01:37:34.374214+0000 mgr.y (mgr.14901) 6569 : cluster [DBG] pgmap v6576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:36.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:36 smithi006 bash[27292]: audit 2023-08-28T01:37:35.285861+0000 mon.a (mon.0) 4337 : audit [DBG] from='client.? 172.21.15.6:0/1379552683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:36.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:36 smithi006 bash[33918]: cluster 2023-08-28T01:37:34.374214+0000 mgr.y (mgr.14901) 6569 : cluster [DBG] pgmap v6576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:36.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:36 smithi006 bash[33918]: audit 2023-08-28T01:37:35.285861+0000 mon.a (mon.0) 4337 : audit [DBG] from='client.? 172.21.15.6:0/1379552683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:37.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:36 smithi130 bash[51302]: debug 2023-08-28T01:37:36.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:36.773129+0000) 2023-08-28T01:37:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:37 smithi130 bash[51302]: debug 2023-08-28T01:37:37.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:37.773450+0000) 2023-08-28T01:37:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:38 smithi130 bash[27579]: cluster 2023-08-28T01:37:36.374870+0000 mgr.y (mgr.14901) 6570 : cluster [DBG] pgmap v6577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:38 smithi130 bash[27579]: audit 2023-08-28T01:37:37.696565+0000 mon.c (mon.2) 2583 : audit [DBG] from='client.? 172.21.15.6:0/369060960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:38 smithi006 bash[27292]: cluster 2023-08-28T01:37:36.374870+0000 mgr.y (mgr.14901) 6570 : cluster [DBG] pgmap v6577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:38 smithi006 bash[27292]: audit 2023-08-28T01:37:37.696565+0000 mon.c (mon.2) 2583 : audit [DBG] from='client.? 172.21.15.6:0/369060960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:38.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:38 smithi006 bash[33918]: cluster 2023-08-28T01:37:36.374870+0000 mgr.y (mgr.14901) 6570 : cluster [DBG] pgmap v6577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:38.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:38 smithi006 bash[33918]: audit 2023-08-28T01:37:37.696565+0000 mon.c (mon.2) 2583 : audit [DBG] from='client.? 172.21.15.6:0/369060960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:39.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:38 smithi130 bash[51302]: debug 2023-08-28T01:37:38.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:38.773755+0000) 2023-08-28T01:37:40.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:39 smithi130 bash[51302]: debug 2023-08-28T01:37:39.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:39.774031+0000) 2023-08-28T01:37:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:40 smithi130 bash[27579]: cluster 2023-08-28T01:37:38.375535+0000 mgr.y (mgr.14901) 6571 : cluster [DBG] pgmap v6578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:40 smithi130 bash[27579]: audit 2023-08-28T01:37:40.113150+0000 mon.a (mon.0) 4338 : audit [DBG] from='client.? 172.21.15.6:0/989163681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:40 smithi006 bash[27292]: cluster 2023-08-28T01:37:38.375535+0000 mgr.y (mgr.14901) 6571 : cluster [DBG] pgmap v6578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:40 smithi006 bash[27292]: audit 2023-08-28T01:37:40.113150+0000 mon.a (mon.0) 4338 : audit [DBG] from='client.? 172.21.15.6:0/989163681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:40 smithi006 bash[33918]: cluster 2023-08-28T01:37:38.375535+0000 mgr.y (mgr.14901) 6571 : cluster [DBG] pgmap v6578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:40 smithi006 bash[33918]: audit 2023-08-28T01:37:40.113150+0000 mon.a (mon.0) 4338 : audit [DBG] from='client.? 172.21.15.6:0/989163681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:41.067 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:37:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:37:40] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:37:41.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:40 smithi130 bash[51302]: debug 2023-08-28T01:37:40.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:40.774320+0000) 2023-08-28T01:37:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:37:41.067Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:37:41.069Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:41 smithi130 bash[51302]: debug 2023-08-28T01:37:41.770+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:41.774561+0000) 2023-08-28T01:37:42.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:42 smithi130 bash[27579]: cluster 2023-08-28T01:37:40.376637+0000 mgr.y (mgr.14901) 6572 : cluster [DBG] pgmap v6579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:42.635 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:42 smithi006 bash[33918]: cluster 2023-08-28T01:37:40.376637+0000 mgr.y (mgr.14901) 6572 : cluster [DBG] pgmap v6579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:42.635 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:42 smithi006 bash[27292]: cluster 2023-08-28T01:37:40.376637+0000 mgr.y (mgr.14901) 6572 : cluster [DBG] pgmap v6579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:37:42.635Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:37:42.636Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:43.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:42 smithi130 bash[51302]: debug 2023-08-28T01:37:42.774+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:42.774890+0000) 2023-08-28T01:37:43.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:43 smithi130 bash[27579]: audit 2023-08-28T01:37:42.520473+0000 mon.c (mon.2) 2584 : audit [DBG] from='client.? 172.21.15.6:0/1375288178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:43.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:43 smithi006 bash[33918]: audit 2023-08-28T01:37:42.520473+0000 mon.c (mon.2) 2584 : audit [DBG] from='client.? 172.21.15.6:0/1375288178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:43.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:43 smithi006 bash[27292]: audit 2023-08-28T01:37:42.520473+0000 mon.c (mon.2) 2584 : audit [DBG] from='client.? 172.21.15.6:0/1375288178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:44.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:43 smithi130 bash[51302]: debug 2023-08-28T01:37:43.774+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:43.775180+0000) 2023-08-28T01:37:44.523 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:44 smithi006 bash[27292]: cluster 2023-08-28T01:37:42.377290+0000 mgr.y (mgr.14901) 6573 : cluster [DBG] pgmap v6580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:44.523 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:44 smithi006 bash[33918]: cluster 2023-08-28T01:37:42.377290+0000 mgr.y (mgr.14901) 6573 : cluster [DBG] pgmap v6580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:44 smithi130 bash[27579]: cluster 2023-08-28T01:37:42.377290+0000 mgr.y (mgr.14901) 6573 : cluster [DBG] pgmap v6580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:37:44.523Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:37:44.525Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:45.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:44 smithi130 bash[51302]: debug 2023-08-28T01:37:44.774+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:44.775440+0000) 2023-08-28T01:37:45.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:45 smithi130 bash[27579]: audit 2023-08-28T01:37:44.930008+0000 mon.c (mon.2) 2585 : audit [DBG] from='client.? 172.21.15.6:0/421380543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:45.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:45 smithi006 bash[27292]: audit 2023-08-28T01:37:44.930008+0000 mon.c (mon.2) 2585 : audit [DBG] from='client.? 172.21.15.6:0/421380543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:45.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:45 smithi006 bash[33918]: audit 2023-08-28T01:37:44.930008+0000 mon.c (mon.2) 2585 : audit [DBG] from='client.? 172.21.15.6:0/421380543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:45 smithi130 bash[51302]: debug 2023-08-28T01:37:45.774+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:45.775769+0000) 2023-08-28T01:37:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:46 smithi130 bash[27579]: cluster 2023-08-28T01:37:44.378400+0000 mgr.y (mgr.14901) 6574 : cluster [DBG] pgmap v6581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:46.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:46 smithi006 bash[33918]: cluster 2023-08-28T01:37:44.378400+0000 mgr.y (mgr.14901) 6574 : cluster [DBG] pgmap v6581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:46.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:46 smithi006 bash[27292]: cluster 2023-08-28T01:37:44.378400+0000 mgr.y (mgr.14901) 6574 : cluster [DBG] pgmap v6581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:46 smithi130 bash[51302]: debug 2023-08-28T01:37:46.774+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:46.776111+0000) 2023-08-28T01:37:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:47 smithi130 bash[51302]: debug 2023-08-28T01:37:47.774+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:47.776393+0000) 2023-08-28T01:37:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:48 smithi130 bash[27579]: cluster 2023-08-28T01:37:46.379077+0000 mgr.y (mgr.14901) 6575 : cluster [DBG] pgmap v6582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:48 smithi130 bash[27579]: audit 2023-08-28T01:37:47.343999+0000 mon.a (mon.0) 4339 : audit [DBG] from='client.? 172.21.15.6:0/4095772178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:48 smithi006 bash[27292]: cluster 2023-08-28T01:37:46.379077+0000 mgr.y (mgr.14901) 6575 : cluster [DBG] pgmap v6582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:48 smithi006 bash[27292]: audit 2023-08-28T01:37:47.343999+0000 mon.a (mon.0) 4339 : audit [DBG] from='client.? 172.21.15.6:0/4095772178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:48.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:48 smithi006 bash[33918]: cluster 2023-08-28T01:37:46.379077+0000 mgr.y (mgr.14901) 6575 : cluster [DBG] pgmap v6582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:48.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:48 smithi006 bash[33918]: audit 2023-08-28T01:37:47.343999+0000 mon.a (mon.0) 4339 : audit [DBG] from='client.? 172.21.15.6:0/4095772178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:49.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:48 smithi130 bash[51302]: debug 2023-08-28T01:37:48.774+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:48.776691+0000) 2023-08-28T01:37:50.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:49 smithi130 bash[51302]: debug 2023-08-28T01:37:49.774+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:49.776966+0000) 2023-08-28T01:37:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:50 smithi130 bash[27579]: cluster 2023-08-28T01:37:48.379744+0000 mgr.y (mgr.14901) 6576 : cluster [DBG] pgmap v6583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:50 smithi130 bash[27579]: audit 2023-08-28T01:37:49.751204+0000 mon.c (mon.2) 2586 : audit [DBG] from='client.? 172.21.15.6:0/215263736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:50.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:50 smithi006 bash[27292]: cluster 2023-08-28T01:37:48.379744+0000 mgr.y (mgr.14901) 6576 : cluster [DBG] pgmap v6583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:50 smithi006 bash[27292]: audit 2023-08-28T01:37:49.751204+0000 mon.c (mon.2) 2586 : audit [DBG] from='client.? 172.21.15.6:0/215263736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:50.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:50 smithi006 bash[33918]: cluster 2023-08-28T01:37:48.379744+0000 mgr.y (mgr.14901) 6576 : cluster [DBG] pgmap v6583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:50.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:50 smithi006 bash[33918]: audit 2023-08-28T01:37:49.751204+0000 mon.c (mon.2) 2586 : audit [DBG] from='client.? 172.21.15.6:0/215263736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:51.068 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:37:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:37:50] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:37:51.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:50 smithi130 bash[51302]: debug 2023-08-28T01:37:50.774+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:50.777255+0000) 2023-08-28T01:37:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:37:51.068Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:37:51.069Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:52.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:51 smithi130 bash[51302]: debug 2023-08-28T01:37:51.774+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:51.777525+0000) 2023-08-28T01:37:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:52 smithi130 bash[27579]: cluster 2023-08-28T01:37:50.380876+0000 mgr.y (mgr.14901) 6577 : cluster [DBG] pgmap v6584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:52 smithi130 bash[27579]: audit 2023-08-28T01:37:52.178873+0000 mon.c (mon.2) 2587 : audit [DBG] from='client.? 172.21.15.6:0/3243965440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:52.636 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:52 smithi006 bash[33918]: cluster 2023-08-28T01:37:50.380876+0000 mgr.y (mgr.14901) 6577 : cluster [DBG] pgmap v6584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:52.636 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:52 smithi006 bash[33918]: audit 2023-08-28T01:37:52.178873+0000 mon.c (mon.2) 2587 : audit [DBG] from='client.? 172.21.15.6:0/3243965440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:52.637 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:52 smithi006 bash[27292]: cluster 2023-08-28T01:37:50.380876+0000 mgr.y (mgr.14901) 6577 : cluster [DBG] pgmap v6584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:52.637 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:52 smithi006 bash[27292]: audit 2023-08-28T01:37:52.178873+0000 mon.c (mon.2) 2587 : audit [DBG] from='client.? 172.21.15.6:0/3243965440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:37:52.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:37:52.638Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:53.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:52 smithi130 bash[51302]: debug 2023-08-28T01:37:52.774+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:52.777802+0000) 2023-08-28T01:37:53.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:53 smithi130 bash[27579]: cluster 2023-08-28T01:37:52.381553+0000 mgr.y (mgr.14901) 6578 : cluster [DBG] pgmap v6585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:53.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:53 smithi006 bash[27292]: cluster 2023-08-28T01:37:52.381553+0000 mgr.y (mgr.14901) 6578 : cluster [DBG] pgmap v6585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:53.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:53 smithi006 bash[33918]: cluster 2023-08-28T01:37:52.381553+0000 mgr.y (mgr.14901) 6578 : cluster [DBG] pgmap v6585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:54.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:53 smithi130 bash[51302]: debug 2023-08-28T01:37:53.774+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:53.778059+0000) 2023-08-28T01:37:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:37:54.523Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:37:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:37:54.525Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:37:55.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:54 smithi130 bash[51302]: debug 2023-08-28T01:37:54.774+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:54.778351+0000) 2023-08-28T01:37:55.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:55 smithi006 bash[33918]: cluster 2023-08-28T01:37:54.382673+0000 mgr.y (mgr.14901) 6579 : cluster [DBG] pgmap v6586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:55.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:55 smithi006 bash[33918]: audit 2023-08-28T01:37:54.599376+0000 mon.a (mon.0) 4340 : audit [DBG] from='client.? 172.21.15.6:0/2903359544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:55.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:55 smithi006 bash[27292]: cluster 2023-08-28T01:37:54.382673+0000 mgr.y (mgr.14901) 6579 : cluster [DBG] pgmap v6586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:55.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:55 smithi006 bash[27292]: audit 2023-08-28T01:37:54.599376+0000 mon.a (mon.0) 4340 : audit [DBG] from='client.? 172.21.15.6:0/2903359544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:55.777 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:55 smithi130 bash[27579]: cluster 2023-08-28T01:37:54.382673+0000 mgr.y (mgr.14901) 6579 : cluster [DBG] pgmap v6586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:55.777 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:55 smithi130 bash[27579]: audit 2023-08-28T01:37:54.599376+0000 mon.a (mon.0) 4340 : audit [DBG] from='client.? 172.21.15.6:0/2903359544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:56.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:55 smithi130 bash[51302]: debug 2023-08-28T01:37:55.778+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:55.778632+0000) 2023-08-28T01:37:57.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:56 smithi130 bash[51302]: debug 2023-08-28T01:37:56.778+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:56.778975+0000) 2023-08-28T01:37:57.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:57 smithi006 bash[27292]: cluster 2023-08-28T01:37:56.383305+0000 mgr.y (mgr.14901) 6580 : cluster [DBG] pgmap v6587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:57.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:57 smithi006 bash[27292]: audit 2023-08-28T01:37:57.007737+0000 mon.a (mon.0) 4341 : audit [DBG] from='client.? 172.21.15.6:0/2938042717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:57.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:57 smithi006 bash[33918]: cluster 2023-08-28T01:37:56.383305+0000 mgr.y (mgr.14901) 6580 : cluster [DBG] pgmap v6587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:57.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:57 smithi006 bash[33918]: audit 2023-08-28T01:37:57.007737+0000 mon.a (mon.0) 4341 : audit [DBG] from='client.? 172.21.15.6:0/2938042717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:57.777 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:57 smithi130 bash[27579]: cluster 2023-08-28T01:37:56.383305+0000 mgr.y (mgr.14901) 6580 : cluster [DBG] pgmap v6587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:57.777 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:57 smithi130 bash[27579]: audit 2023-08-28T01:37:57.007737+0000 mon.a (mon.0) 4341 : audit [DBG] from='client.? 172.21.15.6:0/2938042717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:58.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:57 smithi130 bash[51302]: debug 2023-08-28T01:37:57.778+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:57.779187+0000) 2023-08-28T01:37:59.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:58 smithi130 bash[51302]: debug 2023-08-28T01:37:58.778+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:58.779460+0000) 2023-08-28T01:37:59.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:59 smithi006 bash[33918]: cluster 2023-08-28T01:37:58.383950+0000 mgr.y (mgr.14901) 6581 : cluster [DBG] pgmap v6588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:59.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:37:59 smithi006 bash[33918]: audit 2023-08-28T01:37:59.413149+0000 mon.c (mon.2) 2588 : audit [DBG] from='client.? 172.21.15.6:0/311576608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:59.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:59 smithi006 bash[27292]: cluster 2023-08-28T01:37:58.383950+0000 mgr.y (mgr.14901) 6581 : cluster [DBG] pgmap v6588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:59.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:37:59 smithi006 bash[27292]: audit 2023-08-28T01:37:59.413149+0000 mon.c (mon.2) 2588 : audit [DBG] from='client.? 172.21.15.6:0/311576608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:59.778 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:59 smithi130 bash[27579]: cluster 2023-08-28T01:37:58.383950+0000 mgr.y (mgr.14901) 6581 : cluster [DBG] pgmap v6588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:37:59.778 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:37:59 smithi130 bash[27579]: audit 2023-08-28T01:37:59.413149+0000 mon.c (mon.2) 2588 : audit [DBG] from='client.? 172.21.15.6:0/311576608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:37:59 smithi130 bash[51302]: debug 2023-08-28T01:37:59.778+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:59.779759+0000) 2023-08-28T01:38:01.069 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:38:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:38:00] "GET /metrics HTTP/1.1" 200 193131 "" "Prometheus/2.33.4" 2023-08-28T01:38:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:00 smithi130 bash[51302]: debug 2023-08-28T01:38:00.778+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:00.780081+0000) 2023-08-28T01:38:01.453 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:38:01.069Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:01.453 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:38:01.071Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:01.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:01 smithi006 bash[33918]: cluster 2023-08-28T01:38:00.385185+0000 mgr.y (mgr.14901) 6582 : cluster [DBG] pgmap v6589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:01.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:01 smithi006 bash[27292]: cluster 2023-08-28T01:38:00.385185+0000 mgr.y (mgr.14901) 6582 : cluster [DBG] pgmap v6589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:01.778 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:01 smithi130 bash[27579]: cluster 2023-08-28T01:38:00.385185+0000 mgr.y (mgr.14901) 6582 : cluster [DBG] pgmap v6589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:02.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:01 smithi130 bash[51302]: debug 2023-08-28T01:38:01.778+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:01.780376+0000) 2023-08-28T01:38:02.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:02 smithi006 bash[27292]: audit 2023-08-28T01:38:01.817023+0000 mon.c (mon.2) 2589 : audit [DBG] from='client.? 172.21.15.6:0/1177969604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:02.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:02 smithi006 bash[33918]: audit 2023-08-28T01:38:01.817023+0000 mon.c (mon.2) 2589 : audit [DBG] from='client.? 172.21.15.6:0/1177969604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:02.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:38:02.637Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:02.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:38:02.639Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:02.779 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:02 smithi130 bash[27579]: audit 2023-08-28T01:38:01.817023+0000 mon.c (mon.2) 2589 : audit [DBG] from='client.? 172.21.15.6:0/1177969604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:03.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:02 smithi130 bash[51302]: debug 2023-08-28T01:38:02.778+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:02.780639+0000) 2023-08-28T01:38:04.098 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:03 smithi130 bash[51302]: debug 2023-08-28T01:38:03.778+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:03.780913+0000) 2023-08-28T01:38:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:04 smithi130 bash[27579]: cluster 2023-08-28T01:38:02.385824+0000 mgr.y (mgr.14901) 6583 : cluster [DBG] pgmap v6590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:04.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:04 smithi006 bash[27292]: cluster 2023-08-28T01:38:02.385824+0000 mgr.y (mgr.14901) 6583 : cluster [DBG] pgmap v6590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:04.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:04 smithi006 bash[33918]: cluster 2023-08-28T01:38:02.385824+0000 mgr.y (mgr.14901) 6583 : cluster [DBG] pgmap v6590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:38:04.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:38:04.526Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:05.106 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:04 smithi130 bash[51302]: debug 2023-08-28T01:38:04.778+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:04.781209+0000) 2023-08-28T01:38:05.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:05 smithi130 bash[27579]: audit 2023-08-28T01:38:04.232571+0000 mon.c (mon.2) 2590 : audit [DBG] from='client.? 172.21.15.6:0/218316349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:05.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:05 smithi006 bash[33918]: audit 2023-08-28T01:38:04.232571+0000 mon.c (mon.2) 2590 : audit [DBG] from='client.? 172.21.15.6:0/218316349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:05.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:05 smithi006 bash[27292]: audit 2023-08-28T01:38:04.232571+0000 mon.c (mon.2) 2590 : audit [DBG] from='client.? 172.21.15.6:0/218316349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:06.109 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:05 smithi130 bash[51302]: debug 2023-08-28T01:38:05.778+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:05.781464+0000) 2023-08-28T01:38:06.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:06 smithi130 bash[27579]: cluster 2023-08-28T01:38:04.386950+0000 mgr.y (mgr.14901) 6584 : cluster [DBG] pgmap v6591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:06.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:06 smithi006 bash[27292]: cluster 2023-08-28T01:38:04.386950+0000 mgr.y (mgr.14901) 6584 : cluster [DBG] pgmap v6591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:06.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:06 smithi006 bash[33918]: cluster 2023-08-28T01:38:04.386950+0000 mgr.y (mgr.14901) 6584 : cluster [DBG] pgmap v6591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:07.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:06 smithi130 bash[51302]: debug 2023-08-28T01:38:06.778+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:06.781731+0000) 2023-08-28T01:38:07.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:07 smithi006 bash[33918]: audit 2023-08-28T01:38:06.656672+0000 mon.c (mon.2) 2591 : audit [DBG] from='client.? 172.21.15.6:0/62267697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:07 smithi006 bash[27292]: audit 2023-08-28T01:38:06.656672+0000 mon.c (mon.2) 2591 : audit [DBG] from='client.? 172.21.15.6:0/62267697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:07.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:07 smithi130 bash[27579]: audit 2023-08-28T01:38:06.656672+0000 mon.c (mon.2) 2591 : audit [DBG] from='client.? 172.21.15.6:0/62267697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:08.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:07 smithi130 bash[51302]: debug 2023-08-28T01:38:07.777+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:07.782019+0000) 2023-08-28T01:38:08.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:08 smithi006 bash[27292]: cluster 2023-08-28T01:38:06.387339+0000 mgr.y (mgr.14901) 6585 : cluster [DBG] pgmap v6592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:08.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:08 smithi006 bash[33918]: cluster 2023-08-28T01:38:06.387339+0000 mgr.y (mgr.14901) 6585 : cluster [DBG] pgmap v6592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:08 smithi130 bash[27579]: cluster 2023-08-28T01:38:06.387339+0000 mgr.y (mgr.14901) 6585 : cluster [DBG] pgmap v6592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:09.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:08 smithi130 bash[51302]: debug 2023-08-28T01:38:08.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:08.782301+0000) 2023-08-28T01:38:09.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:09 smithi006 bash[33918]: audit 2023-08-28T01:38:09.080084+0000 mon.a (mon.0) 4342 : audit [DBG] from='client.? 172.21.15.6:0/1188230912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:09.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:09 smithi006 bash[27292]: audit 2023-08-28T01:38:09.080084+0000 mon.a (mon.0) 4342 : audit [DBG] from='client.? 172.21.15.6:0/1188230912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:09.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:09 smithi130 bash[27579]: audit 2023-08-28T01:38:09.080084+0000 mon.a (mon.0) 4342 : audit [DBG] from='client.? 172.21.15.6:0/1188230912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:10.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:09 smithi130 bash[51302]: debug 2023-08-28T01:38:09.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:09.782583+0000) 2023-08-28T01:38:10.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:10 smithi006 bash[27292]: cluster 2023-08-28T01:38:08.387961+0000 mgr.y (mgr.14901) 6586 : cluster [DBG] pgmap v6593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:10.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:10 smithi006 bash[33918]: cluster 2023-08-28T01:38:08.387961+0000 mgr.y (mgr.14901) 6586 : cluster [DBG] pgmap v6593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:10 smithi130 bash[27579]: cluster 2023-08-28T01:38:08.387961+0000 mgr.y (mgr.14901) 6586 : cluster [DBG] pgmap v6593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:11.070 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:38:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:38:10] "GET /metrics HTTP/1.1" 200 193131 "" "Prometheus/2.33.4" 2023-08-28T01:38:11.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:10 smithi130 bash[51302]: debug 2023-08-28T01:38:10.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:10.782907+0000) 2023-08-28T01:38:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:38:11.070Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:38:11.072Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:12.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:11 smithi130 bash[51302]: debug 2023-08-28T01:38:11.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:11.783180+0000) 2023-08-28T01:38:12.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:12 smithi006 bash[27292]: cluster 2023-08-28T01:38:10.389098+0000 mgr.y (mgr.14901) 6587 : cluster [DBG] pgmap v6594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:12.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:12 smithi006 bash[27292]: audit 2023-08-28T01:38:11.501492+0000 mon.c (mon.2) 2592 : audit [DBG] from='client.? 172.21.15.6:0/79940204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:12.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:12 smithi006 bash[33918]: cluster 2023-08-28T01:38:10.389098+0000 mgr.y (mgr.14901) 6587 : cluster [DBG] pgmap v6594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:12.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:12 smithi006 bash[33918]: audit 2023-08-28T01:38:11.501492+0000 mon.c (mon.2) 2592 : audit [DBG] from='client.? 172.21.15.6:0/79940204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:12 smithi130 bash[27579]: cluster 2023-08-28T01:38:10.389098+0000 mgr.y (mgr.14901) 6587 : cluster [DBG] pgmap v6594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:12 smithi130 bash[27579]: audit 2023-08-28T01:38:11.501492+0000 mon.c (mon.2) 2592 : audit [DBG] from='client.? 172.21.15.6:0/79940204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:38:12.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:38:12.640Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:13.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:12 smithi130 bash[51302]: debug 2023-08-28T01:38:12.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:12.783472+0000) 2023-08-28T01:38:14.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:13 smithi130 bash[51302]: debug 2023-08-28T01:38:13.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:13.783729+0000) 2023-08-28T01:38:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:14 smithi006 bash[27292]: cluster 2023-08-28T01:38:12.389716+0000 mgr.y (mgr.14901) 6588 : cluster [DBG] pgmap v6595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:14 smithi006 bash[27292]: audit 2023-08-28T01:38:13.926785+0000 mon.c (mon.2) 2593 : audit [DBG] from='client.? 172.21.15.6:0/2208021481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:14 smithi006 bash[33918]: cluster 2023-08-28T01:38:12.389716+0000 mgr.y (mgr.14901) 6588 : cluster [DBG] pgmap v6595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:14 smithi006 bash[33918]: audit 2023-08-28T01:38:13.926785+0000 mon.c (mon.2) 2593 : audit [DBG] from='client.? 172.21.15.6:0/2208021481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:14 smithi130 bash[27579]: cluster 2023-08-28T01:38:12.389716+0000 mgr.y (mgr.14901) 6588 : cluster [DBG] pgmap v6595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:14 smithi130 bash[27579]: audit 2023-08-28T01:38:13.926785+0000 mon.c (mon.2) 2593 : audit [DBG] from='client.? 172.21.15.6:0/2208021481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:38:14.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:38:14.526Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:14 smithi130 bash[51302]: debug 2023-08-28T01:38:14.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:14.784061+0000) 2023-08-28T01:38:16.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:15 smithi130 bash[51302]: debug 2023-08-28T01:38:15.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:15.784355+0000) 2023-08-28T01:38:16.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:16 smithi006 bash[27292]: cluster 2023-08-28T01:38:14.390804+0000 mgr.y (mgr.14901) 6589 : cluster [DBG] pgmap v6596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:16.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:16 smithi006 bash[33918]: cluster 2023-08-28T01:38:14.390804+0000 mgr.y (mgr.14901) 6589 : cluster [DBG] pgmap v6596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:16 smithi130 bash[27579]: cluster 2023-08-28T01:38:14.390804+0000 mgr.y (mgr.14901) 6589 : cluster [DBG] pgmap v6596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:17.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:16 smithi130 bash[51302]: debug 2023-08-28T01:38:16.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:16.784660+0000) 2023-08-28T01:38:17.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:17 smithi006 bash[27292]: audit 2023-08-28T01:38:16.341465+0000 mon.a (mon.0) 4343 : audit [DBG] from='client.? 172.21.15.6:0/3731488919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:17.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:17 smithi006 bash[33918]: audit 2023-08-28T01:38:16.341465+0000 mon.a (mon.0) 4343 : audit [DBG] from='client.? 172.21.15.6:0/3731488919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:17.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:17 smithi130 bash[27579]: audit 2023-08-28T01:38:16.341465+0000 mon.a (mon.0) 4343 : audit [DBG] from='client.? 172.21.15.6:0/3731488919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:18.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:17 smithi130 bash[51302]: debug 2023-08-28T01:38:17.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:17.784933+0000) 2023-08-28T01:38:18.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:18 smithi006 bash[27292]: cluster 2023-08-28T01:38:16.391432+0000 mgr.y (mgr.14901) 6590 : cluster [DBG] pgmap v6597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:18.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:18 smithi006 bash[33918]: cluster 2023-08-28T01:38:16.391432+0000 mgr.y (mgr.14901) 6590 : cluster [DBG] pgmap v6597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:18 smithi130 bash[27579]: cluster 2023-08-28T01:38:16.391432+0000 mgr.y (mgr.14901) 6590 : cluster [DBG] pgmap v6597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:19.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:18 smithi130 bash[51302]: debug 2023-08-28T01:38:18.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:18.785243+0000) 2023-08-28T01:38:19.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:19 smithi006 bash[33918]: audit 2023-08-28T01:38:18.758685+0000 mon.a (mon.0) 4344 : audit [DBG] from='client.? 172.21.15.6:0/669452643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:19.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:19 smithi006 bash[27292]: audit 2023-08-28T01:38:18.758685+0000 mon.a (mon.0) 4344 : audit [DBG] from='client.? 172.21.15.6:0/669452643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:19.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:19 smithi130 bash[27579]: audit 2023-08-28T01:38:18.758685+0000 mon.a (mon.0) 4344 : audit [DBG] from='client.? 172.21.15.6:0/669452643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:19 smithi130 bash[51302]: debug 2023-08-28T01:38:19.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:19.785513+0000) 2023-08-28T01:38:20.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:20 smithi006 bash[27292]: cluster 2023-08-28T01:38:18.392121+0000 mgr.y (mgr.14901) 6591 : cluster [DBG] pgmap v6598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:20.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:20 smithi006 bash[33918]: cluster 2023-08-28T01:38:18.392121+0000 mgr.y (mgr.14901) 6591 : cluster [DBG] pgmap v6598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:20 smithi130 bash[27579]: cluster 2023-08-28T01:38:18.392121+0000 mgr.y (mgr.14901) 6591 : cluster [DBG] pgmap v6598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:21.072 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:38:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:38:20] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:38:21.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:20 smithi130 bash[51302]: debug 2023-08-28T01:38:20.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:20.785835+0000) 2023-08-28T01:38:21.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:21 smithi006 bash[33918]: audit 2023-08-28T01:38:21.175482+0000 mon.c (mon.2) 2594 : audit [DBG] from='client.? 172.21.15.6:0/1778686647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:38:21.071Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:38:21.073Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:21.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:21 smithi006 bash[27292]: audit 2023-08-28T01:38:21.175482+0000 mon.c (mon.2) 2594 : audit [DBG] from='client.? 172.21.15.6:0/1778686647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:21.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:21 smithi130 bash[27579]: audit 2023-08-28T01:38:21.175482+0000 mon.c (mon.2) 2594 : audit [DBG] from='client.? 172.21.15.6:0/1778686647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:22.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:21 smithi130 bash[51302]: debug 2023-08-28T01:38:21.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:21.786089+0000) 2023-08-28T01:38:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:22 smithi130 bash[27579]: cluster 2023-08-28T01:38:20.393275+0000 mgr.y (mgr.14901) 6592 : cluster [DBG] pgmap v6599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:22.640 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:22 smithi006 bash[27292]: cluster 2023-08-28T01:38:20.393275+0000 mgr.y (mgr.14901) 6592 : cluster [DBG] pgmap v6599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:22.640 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:22 smithi006 bash[33918]: cluster 2023-08-28T01:38:20.393275+0000 mgr.y (mgr.14901) 6592 : cluster [DBG] pgmap v6599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:38:22.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:38:22.641Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:23.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:22 smithi130 bash[51302]: debug 2023-08-28T01:38:22.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:22.786361+0000) 2023-08-28T01:38:24.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:23 smithi130 bash[51302]: debug 2023-08-28T01:38:23.781+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:23.786654+0000) 2023-08-28T01:38:24.526 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:24 smithi006 bash[33918]: cluster 2023-08-28T01:38:22.393932+0000 mgr.y (mgr.14901) 6593 : cluster [DBG] pgmap v6600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:24.526 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:24 smithi006 bash[33918]: audit 2023-08-28T01:38:23.588199+0000 mon.a (mon.0) 4345 : audit [DBG] from='client.? 172.21.15.6:0/2950876508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:24.526 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:24 smithi006 bash[27292]: cluster 2023-08-28T01:38:22.393932+0000 mgr.y (mgr.14901) 6593 : cluster [DBG] pgmap v6600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:24.526 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:24 smithi006 bash[27292]: audit 2023-08-28T01:38:23.588199+0000 mon.a (mon.0) 4345 : audit [DBG] from='client.? 172.21.15.6:0/2950876508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:24 smithi130 bash[27579]: cluster 2023-08-28T01:38:22.393932+0000 mgr.y (mgr.14901) 6593 : cluster [DBG] pgmap v6600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:24 smithi130 bash[27579]: audit 2023-08-28T01:38:23.588199+0000 mon.a (mon.0) 4345 : audit [DBG] from='client.? 172.21.15.6:0/2950876508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:38:24.525Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:38:24.527Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:25.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:24 smithi130 bash[51302]: debug 2023-08-28T01:38:24.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:24.786973+0000) 2023-08-28T01:38:26.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:25 smithi130 bash[51302]: debug 2023-08-28T01:38:25.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:25.787242+0000) 2023-08-28T01:38:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:26 smithi130 bash[27579]: cluster 2023-08-28T01:38:24.395095+0000 mgr.y (mgr.14901) 6594 : cluster [DBG] pgmap v6601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:26 smithi130 bash[27579]: audit 2023-08-28T01:38:26.002250+0000 mon.c (mon.2) 2595 : audit [DBG] from='client.? 172.21.15.6:0/228508329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:26 smithi130 bash[27579]: audit 2023-08-28T01:38:26.211777+0000 mon.b (mon.1) 748 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:38:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:26 smithi006 bash[33918]: cluster 2023-08-28T01:38:24.395095+0000 mgr.y (mgr.14901) 6594 : cluster [DBG] pgmap v6601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:26 smithi006 bash[33918]: audit 2023-08-28T01:38:26.002250+0000 mon.c (mon.2) 2595 : audit [DBG] from='client.? 172.21.15.6:0/228508329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:26 smithi006 bash[33918]: audit 2023-08-28T01:38:26.211777+0000 mon.b (mon.1) 748 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:38:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:26 smithi006 bash[27292]: cluster 2023-08-28T01:38:24.395095+0000 mgr.y (mgr.14901) 6594 : cluster [DBG] pgmap v6601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:26 smithi006 bash[27292]: audit 2023-08-28T01:38:26.002250+0000 mon.c (mon.2) 2595 : audit [DBG] from='client.? 172.21.15.6:0/228508329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:26 smithi006 bash[27292]: audit 2023-08-28T01:38:26.211777+0000 mon.b (mon.1) 748 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:38:27.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:26 smithi130 bash[51302]: debug 2023-08-28T01:38:26.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:26.787526+0000) 2023-08-28T01:38:28.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:27 smithi130 bash[51302]: debug 2023-08-28T01:38:27.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:27.787805+0000) 2023-08-28T01:38:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:28 smithi130 bash[27579]: cluster 2023-08-28T01:38:26.395703+0000 mgr.y (mgr.14901) 6595 : cluster [DBG] pgmap v6602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:28.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:28 smithi006 bash[33918]: cluster 2023-08-28T01:38:26.395703+0000 mgr.y (mgr.14901) 6595 : cluster [DBG] pgmap v6602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:28 smithi006 bash[27292]: cluster 2023-08-28T01:38:26.395703+0000 mgr.y (mgr.14901) 6595 : cluster [DBG] pgmap v6602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:28 smithi130 bash[51302]: debug 2023-08-28T01:38:28.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:28.788112+0000) 2023-08-28T01:38:29.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:29 smithi130 bash[27579]: audit 2023-08-28T01:38:28.415289+0000 mon.c (mon.2) 2596 : audit [DBG] from='client.? 172.21.15.6:0/2352611908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:29.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:29 smithi006 bash[33918]: audit 2023-08-28T01:38:28.415289+0000 mon.c (mon.2) 2596 : audit [DBG] from='client.? 172.21.15.6:0/2352611908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:29.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:29 smithi006 bash[27292]: audit 2023-08-28T01:38:28.415289+0000 mon.c (mon.2) 2596 : audit [DBG] from='client.? 172.21.15.6:0/2352611908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:30.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:29 smithi130 bash[51302]: debug 2023-08-28T01:38:29.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:29.788342+0000) 2023-08-28T01:38:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:30 smithi130 bash[27579]: cluster 2023-08-28T01:38:28.396319+0000 mgr.y (mgr.14901) 6596 : cluster [DBG] pgmap v6603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:30.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:30 smithi006 bash[27292]: cluster 2023-08-28T01:38:28.396319+0000 mgr.y (mgr.14901) 6596 : cluster [DBG] pgmap v6603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:30 smithi006 bash[33918]: cluster 2023-08-28T01:38:28.396319+0000 mgr.y (mgr.14901) 6596 : cluster [DBG] pgmap v6603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:31.072 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:38:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:38:30] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:38:31.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:30 smithi130 bash[51302]: debug 2023-08-28T01:38:30.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:30.788661+0000) 2023-08-28T01:38:31.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:31 smithi006 bash[33918]: audit 2023-08-28T01:38:30.841560+0000 mon.a (mon.0) 4346 : audit [DBG] from='client.? 172.21.15.6:0/3292758603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:31.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:31 smithi006 bash[27292]: audit 2023-08-28T01:38:30.841560+0000 mon.a (mon.0) 4346 : audit [DBG] from='client.? 172.21.15.6:0/3292758603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:38:31.072Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:38:31.074Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:31.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:31 smithi130 bash[27579]: audit 2023-08-28T01:38:30.841560+0000 mon.a (mon.0) 4346 : audit [DBG] from='client.? 172.21.15.6:0/3292758603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:32.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:31 smithi130 bash[51302]: debug 2023-08-28T01:38:31.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:31.788950+0000) 2023-08-28T01:38:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:32 smithi130 bash[27579]: cluster 2023-08-28T01:38:30.397483+0000 mgr.y (mgr.14901) 6597 : cluster [DBG] pgmap v6604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:32 smithi130 bash[27579]: audit 2023-08-28T01:38:32.136331+0000 mon.a (mon.0) 4347 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:38:32.641 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:32 smithi006 bash[33918]: cluster 2023-08-28T01:38:30.397483+0000 mgr.y (mgr.14901) 6597 : cluster [DBG] pgmap v6604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:32.641 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:32 smithi006 bash[33918]: audit 2023-08-28T01:38:32.136331+0000 mon.a (mon.0) 4347 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:38:32.641 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:32 smithi006 bash[27292]: cluster 2023-08-28T01:38:30.397483+0000 mgr.y (mgr.14901) 6597 : cluster [DBG] pgmap v6604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:32.641 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:32 smithi006 bash[27292]: audit 2023-08-28T01:38:32.136331+0000 mon.a (mon.0) 4347 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:38:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:38:32.640Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:38:32.642Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:33.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:32 smithi130 bash[51302]: debug 2023-08-28T01:38:32.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:32.789220+0000) 2023-08-28T01:38:33.788 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:33 smithi130 bash[27579]: cluster 2023-08-28T01:38:32.398073+0000 mgr.y (mgr.14901) 6598 : cluster [DBG] pgmap v6605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:33.788 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:33 smithi130 bash[27579]: audit 2023-08-28T01:38:32.532730+0000 mon.a (mon.0) 4348 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:38:33.788 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:33 smithi130 bash[27579]: audit 2023-08-28T01:38:32.798966+0000 mon.b (mon.1) 749 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:38:33.788 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:33 smithi130 bash[27579]: audit 2023-08-28T01:38:32.801134+0000 mon.b (mon.1) 750 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:38:33.788 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:33 smithi130 bash[27579]: audit 2023-08-28T01:38:32.811720+0000 mon.a (mon.0) 4349 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:38:33.789 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:33 smithi130 bash[27579]: audit 2023-08-28T01:38:33.259500+0000 mon.a (mon.0) 4350 : audit [DBG] from='client.? 172.21.15.6:0/2240260730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:33.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:33 smithi006 bash[27292]: cluster 2023-08-28T01:38:32.398073+0000 mgr.y (mgr.14901) 6598 : cluster [DBG] pgmap v6605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:33.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:33 smithi006 bash[27292]: audit 2023-08-28T01:38:32.532730+0000 mon.a (mon.0) 4348 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:38:33.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:33 smithi006 bash[27292]: audit 2023-08-28T01:38:32.798966+0000 mon.b (mon.1) 749 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:38:33.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:33 smithi006 bash[27292]: audit 2023-08-28T01:38:32.801134+0000 mon.b (mon.1) 750 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:38:33.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:33 smithi006 bash[27292]: audit 2023-08-28T01:38:32.811720+0000 mon.a (mon.0) 4349 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:38:33.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:33 smithi006 bash[27292]: audit 2023-08-28T01:38:33.259500+0000 mon.a (mon.0) 4350 : audit [DBG] from='client.? 172.21.15.6:0/2240260730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:33.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:33 smithi006 bash[33918]: cluster 2023-08-28T01:38:32.398073+0000 mgr.y (mgr.14901) 6598 : cluster [DBG] pgmap v6605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:33.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:33 smithi006 bash[33918]: audit 2023-08-28T01:38:32.532730+0000 mon.a (mon.0) 4348 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:38:33.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:33 smithi006 bash[33918]: audit 2023-08-28T01:38:32.798966+0000 mon.b (mon.1) 749 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:38:33.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:33 smithi006 bash[33918]: audit 2023-08-28T01:38:32.801134+0000 mon.b (mon.1) 750 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:38:33.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:33 smithi006 bash[33918]: audit 2023-08-28T01:38:32.811720+0000 mon.a (mon.0) 4349 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:38:33.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:33 smithi006 bash[33918]: audit 2023-08-28T01:38:33.259500+0000 mon.a (mon.0) 4350 : audit [DBG] from='client.? 172.21.15.6:0/2240260730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:34.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:33 smithi130 bash[51302]: debug 2023-08-28T01:38:33.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:33.789527+0000) 2023-08-28T01:38:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:38:34.526Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:38:34.528Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:35.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:34 smithi130 bash[51302]: debug 2023-08-28T01:38:34.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:34.789859+0000) 2023-08-28T01:38:35.788 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:35 smithi130 bash[27579]: cluster 2023-08-28T01:38:34.399186+0000 mgr.y (mgr.14901) 6599 : cluster [DBG] pgmap v6606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:35.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:35 smithi006 bash[27292]: cluster 2023-08-28T01:38:34.399186+0000 mgr.y (mgr.14901) 6599 : cluster [DBG] pgmap v6606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:35.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:35 smithi006 bash[33918]: cluster 2023-08-28T01:38:34.399186+0000 mgr.y (mgr.14901) 6599 : cluster [DBG] pgmap v6606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:36.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:35 smithi130 bash[51302]: debug 2023-08-28T01:38:35.789+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:35.790166+0000) 2023-08-28T01:38:36.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:36 smithi130 bash[27579]: audit 2023-08-28T01:38:35.678114+0000 mon.c (mon.2) 2597 : audit [DBG] from='client.? 172.21.15.6:0/3192378373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:36.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:36 smithi130 bash[51302]: debug 2023-08-28T01:38:36.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:36.790450+0000) 2023-08-28T01:38:36.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:36 smithi006 bash[33918]: audit 2023-08-28T01:38:35.678114+0000 mon.c (mon.2) 2597 : audit [DBG] from='client.? 172.21.15.6:0/3192378373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:36.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:36 smithi006 bash[27292]: audit 2023-08-28T01:38:35.678114+0000 mon.c (mon.2) 2597 : audit [DBG] from='client.? 172.21.15.6:0/3192378373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:38.104 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:37 smithi130 bash[51302]: debug 2023-08-28T01:38:37.785+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:37.790716+0000) 2023-08-28T01:38:38.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:38 smithi130 bash[27579]: cluster 2023-08-28T01:38:36.399892+0000 mgr.y (mgr.14901) 6600 : cluster [DBG] pgmap v6607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:38.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:38 smithi130 bash[27579]: audit 2023-08-28T01:38:38.099635+0000 mon.a (mon.0) 4351 : audit [DBG] from='client.? 172.21.15.6:0/3734078283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:38.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:38 smithi006 bash[33918]: cluster 2023-08-28T01:38:36.399892+0000 mgr.y (mgr.14901) 6600 : cluster [DBG] pgmap v6607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:38.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:38 smithi006 bash[33918]: audit 2023-08-28T01:38:38.099635+0000 mon.a (mon.0) 4351 : audit [DBG] from='client.? 172.21.15.6:0/3734078283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:38.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:38 smithi006 bash[27292]: cluster 2023-08-28T01:38:36.399892+0000 mgr.y (mgr.14901) 6600 : cluster [DBG] pgmap v6607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:38.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:38 smithi006 bash[27292]: audit 2023-08-28T01:38:38.099635+0000 mon.a (mon.0) 4351 : audit [DBG] from='client.? 172.21.15.6:0/3734078283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:39.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:38 smithi130 bash[51302]: debug 2023-08-28T01:38:38.789+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:38.791017+0000) 2023-08-28T01:38:40.111 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:39 smithi130 bash[51302]: debug 2023-08-28T01:38:39.789+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:39.791263+0000) 2023-08-28T01:38:40.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:40 smithi130 bash[27579]: cluster 2023-08-28T01:38:38.400703+0000 mgr.y (mgr.14901) 6601 : cluster [DBG] pgmap v6608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:40.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:40 smithi006 bash[27292]: cluster 2023-08-28T01:38:38.400703+0000 mgr.y (mgr.14901) 6601 : cluster [DBG] pgmap v6608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:40 smithi006 bash[33918]: cluster 2023-08-28T01:38:38.400703+0000 mgr.y (mgr.14901) 6601 : cluster [DBG] pgmap v6608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:41.073 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:38:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:38:40] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:38:41.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:40 smithi130 bash[51302]: debug 2023-08-28T01:38:40.789+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:40.791557+0000) 2023-08-28T01:38:41.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:41 smithi006 bash[27292]: audit 2023-08-28T01:38:40.525418+0000 mon.c (mon.2) 2598 : audit [DBG] from='client.? 172.21.15.6:0/2939028874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:38:41.073Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:38:41.075Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:41.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:41 smithi006 bash[33918]: audit 2023-08-28T01:38:40.525418+0000 mon.c (mon.2) 2598 : audit [DBG] from='client.? 172.21.15.6:0/2939028874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:41.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:41 smithi130 bash[27579]: audit 2023-08-28T01:38:40.525418+0000 mon.c (mon.2) 2598 : audit [DBG] from='client.? 172.21.15.6:0/2939028874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:41 smithi130 bash[51302]: debug 2023-08-28T01:38:41.789+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:41.791875+0000) 2023-08-28T01:38:42.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:42 smithi006 bash[33918]: cluster 2023-08-28T01:38:40.401816+0000 mgr.y (mgr.14901) 6602 : cluster [DBG] pgmap v6609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:42.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:42 smithi006 bash[27292]: cluster 2023-08-28T01:38:40.401816+0000 mgr.y (mgr.14901) 6602 : cluster [DBG] pgmap v6609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:42 smithi130 bash[27579]: cluster 2023-08-28T01:38:40.401816+0000 mgr.y (mgr.14901) 6602 : cluster [DBG] pgmap v6609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:38:42.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:38:42.642Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:43.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:42 smithi130 bash[51302]: debug 2023-08-28T01:38:42.789+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:42.792142+0000) 2023-08-28T01:38:43.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:43 smithi006 bash[33918]: audit 2023-08-28T01:38:42.942064+0000 mon.a (mon.0) 4352 : audit [DBG] from='client.? 172.21.15.6:0/1097120799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:43.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:43 smithi006 bash[27292]: audit 2023-08-28T01:38:42.942064+0000 mon.a (mon.0) 4352 : audit [DBG] from='client.? 172.21.15.6:0/1097120799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:43.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:43 smithi130 bash[27579]: audit 2023-08-28T01:38:42.942064+0000 mon.a (mon.0) 4352 : audit [DBG] from='client.? 172.21.15.6:0/1097120799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:44.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:43 smithi130 bash[51302]: debug 2023-08-28T01:38:43.789+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:43.792409+0000) 2023-08-28T01:38:44.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:44 smithi006 bash[33918]: cluster 2023-08-28T01:38:42.402435+0000 mgr.y (mgr.14901) 6603 : cluster [DBG] pgmap v6610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:44.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:44 smithi006 bash[27292]: cluster 2023-08-28T01:38:42.402435+0000 mgr.y (mgr.14901) 6603 : cluster [DBG] pgmap v6610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:44 smithi130 bash[27579]: cluster 2023-08-28T01:38:42.402435+0000 mgr.y (mgr.14901) 6603 : cluster [DBG] pgmap v6610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:38:44.527Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:38:44.530Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:45.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:44 smithi130 bash[51302]: debug 2023-08-28T01:38:44.789+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:44.792699+0000) 2023-08-28T01:38:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:45 smithi130 bash[51302]: debug 2023-08-28T01:38:45.789+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:45.792913+0000) 2023-08-28T01:38:46.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:46 smithi006 bash[27292]: cluster 2023-08-28T01:38:44.403660+0000 mgr.y (mgr.14901) 6604 : cluster [DBG] pgmap v6611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:46.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:46 smithi006 bash[27292]: audit 2023-08-28T01:38:45.357218+0000 mon.c (mon.2) 2599 : audit [DBG] from='client.? 172.21.15.6:0/2804664145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:46.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:46 smithi006 bash[33918]: cluster 2023-08-28T01:38:44.403660+0000 mgr.y (mgr.14901) 6604 : cluster [DBG] pgmap v6611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:46.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:46 smithi006 bash[33918]: audit 2023-08-28T01:38:45.357218+0000 mon.c (mon.2) 2599 : audit [DBG] from='client.? 172.21.15.6:0/2804664145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:46 smithi130 bash[27579]: cluster 2023-08-28T01:38:44.403660+0000 mgr.y (mgr.14901) 6604 : cluster [DBG] pgmap v6611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:46 smithi130 bash[27579]: audit 2023-08-28T01:38:45.357218+0000 mon.c (mon.2) 2599 : audit [DBG] from='client.? 172.21.15.6:0/2804664145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:46 smithi130 bash[51302]: debug 2023-08-28T01:38:46.789+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:46.793181+0000) 2023-08-28T01:38:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:47 smithi130 bash[51302]: debug 2023-08-28T01:38:47.793+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:47.793474+0000) 2023-08-28T01:38:48.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:48 smithi006 bash[27292]: cluster 2023-08-28T01:38:46.404370+0000 mgr.y (mgr.14901) 6605 : cluster [DBG] pgmap v6612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:48.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:48 smithi006 bash[27292]: audit 2023-08-28T01:38:47.775995+0000 mon.a (mon.0) 4353 : audit [DBG] from='client.? 172.21.15.6:0/3996064071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:48.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:48 smithi006 bash[33918]: cluster 2023-08-28T01:38:46.404370+0000 mgr.y (mgr.14901) 6605 : cluster [DBG] pgmap v6612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:48.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:48 smithi006 bash[33918]: audit 2023-08-28T01:38:47.775995+0000 mon.a (mon.0) 4353 : audit [DBG] from='client.? 172.21.15.6:0/3996064071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:48 smithi130 bash[27579]: cluster 2023-08-28T01:38:46.404370+0000 mgr.y (mgr.14901) 6605 : cluster [DBG] pgmap v6612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:48 smithi130 bash[27579]: audit 2023-08-28T01:38:47.775995+0000 mon.a (mon.0) 4353 : audit [DBG] from='client.? 172.21.15.6:0/3996064071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:49.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:48 smithi130 bash[51302]: debug 2023-08-28T01:38:48.793+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:48.793763+0000) 2023-08-28T01:38:50.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:49 smithi130 bash[51302]: debug 2023-08-28T01:38:49.789+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:49.794015+0000) 2023-08-28T01:38:50.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:50 smithi006 bash[27292]: cluster 2023-08-28T01:38:48.405076+0000 mgr.y (mgr.14901) 6606 : cluster [DBG] pgmap v6613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:50.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:50 smithi006 bash[33918]: cluster 2023-08-28T01:38:48.405076+0000 mgr.y (mgr.14901) 6606 : cluster [DBG] pgmap v6613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:50 smithi130 bash[27579]: cluster 2023-08-28T01:38:48.405076+0000 mgr.y (mgr.14901) 6606 : cluster [DBG] pgmap v6613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:51.074 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:38:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:38:50] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:38:51.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:50 smithi130 bash[51302]: debug 2023-08-28T01:38:50.789+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:50.794341+0000) 2023-08-28T01:38:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:38:51.073Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:38:51.075Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:51.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:51 smithi006 bash[27292]: audit 2023-08-28T01:38:50.197027+0000 mon.c (mon.2) 2600 : audit [DBG] from='client.? 172.21.15.6:0/2363158289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:51.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:51 smithi006 bash[33918]: audit 2023-08-28T01:38:50.197027+0000 mon.c (mon.2) 2600 : audit [DBG] from='client.? 172.21.15.6:0/2363158289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:51.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:51 smithi130 bash[27579]: audit 2023-08-28T01:38:50.197027+0000 mon.c (mon.2) 2600 : audit [DBG] from='client.? 172.21.15.6:0/2363158289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:52.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:51 smithi130 bash[51302]: debug 2023-08-28T01:38:51.789+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:51.794577+0000) 2023-08-28T01:38:52.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:52 smithi006 bash[27292]: cluster 2023-08-28T01:38:50.406228+0000 mgr.y (mgr.14901) 6607 : cluster [DBG] pgmap v6614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:52.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:52 smithi006 bash[33918]: cluster 2023-08-28T01:38:50.406228+0000 mgr.y (mgr.14901) 6607 : cluster [DBG] pgmap v6614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:52 smithi130 bash[27579]: cluster 2023-08-28T01:38:50.406228+0000 mgr.y (mgr.14901) 6607 : cluster [DBG] pgmap v6614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:38:52.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:52.987 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:38:52.643Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:53.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:52 smithi130 bash[51302]: debug 2023-08-28T01:38:52.793+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:52.794896+0000) 2023-08-28T01:38:53.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:53 smithi006 bash[27292]: audit 2023-08-28T01:38:52.613005+0000 mon.a (mon.0) 4354 : audit [DBG] from='client.? 172.21.15.6:0/1426257705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:53.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:53 smithi006 bash[33918]: audit 2023-08-28T01:38:52.613005+0000 mon.a (mon.0) 4354 : audit [DBG] from='client.? 172.21.15.6:0/1426257705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:53.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:53 smithi130 bash[27579]: audit 2023-08-28T01:38:52.613005+0000 mon.a (mon.0) 4354 : audit [DBG] from='client.? 172.21.15.6:0/1426257705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:54.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:53 smithi130 bash[51302]: debug 2023-08-28T01:38:53.793+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:53.795144+0000) 2023-08-28T01:38:54.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:54 smithi006 bash[33918]: cluster 2023-08-28T01:38:52.406861+0000 mgr.y (mgr.14901) 6608 : cluster [DBG] pgmap v6615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:54 smithi006 bash[27292]: cluster 2023-08-28T01:38:52.406861+0000 mgr.y (mgr.14901) 6608 : cluster [DBG] pgmap v6615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:54 smithi130 bash[27579]: cluster 2023-08-28T01:38:52.406861+0000 mgr.y (mgr.14901) 6608 : cluster [DBG] pgmap v6615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:38:54.528Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:38:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:38:54.530Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:38:55.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:54 smithi130 bash[51302]: debug 2023-08-28T01:38:54.793+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:54.795365+0000) 2023-08-28T01:38:55.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:55 smithi006 bash[27292]: audit 2023-08-28T01:38:55.028438+0000 mon.a (mon.0) 4355 : audit [DBG] from='client.? 172.21.15.6:0/2820563517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:55.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:55 smithi006 bash[33918]: audit 2023-08-28T01:38:55.028438+0000 mon.a (mon.0) 4355 : audit [DBG] from='client.? 172.21.15.6:0/2820563517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:55.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:55 smithi130 bash[27579]: audit 2023-08-28T01:38:55.028438+0000 mon.a (mon.0) 4355 : audit [DBG] from='client.? 172.21.15.6:0/2820563517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:56.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:55 smithi130 bash[51302]: debug 2023-08-28T01:38:55.793+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:55.795589+0000) 2023-08-28T01:38:56.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:56 smithi006 bash[27292]: cluster 2023-08-28T01:38:54.407989+0000 mgr.y (mgr.14901) 6609 : cluster [DBG] pgmap v6616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:56.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:56 smithi006 bash[33918]: cluster 2023-08-28T01:38:54.407989+0000 mgr.y (mgr.14901) 6609 : cluster [DBG] pgmap v6616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:56 smithi130 bash[27579]: cluster 2023-08-28T01:38:54.407989+0000 mgr.y (mgr.14901) 6609 : cluster [DBG] pgmap v6616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:57.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:56 smithi130 bash[51302]: debug 2023-08-28T01:38:56.793+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:56.795890+0000) 2023-08-28T01:38:58.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:57 smithi130 bash[51302]: debug 2023-08-28T01:38:57.793+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:57.796160+0000) 2023-08-28T01:38:58.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:58 smithi006 bash[33918]: cluster 2023-08-28T01:38:56.408663+0000 mgr.y (mgr.14901) 6610 : cluster [DBG] pgmap v6617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:58.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:38:58 smithi006 bash[33918]: audit 2023-08-28T01:38:57.442283+0000 mon.c (mon.2) 2601 : audit [DBG] from='client.? 172.21.15.6:0/1007217097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:58.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:58 smithi006 bash[27292]: cluster 2023-08-28T01:38:56.408663+0000 mgr.y (mgr.14901) 6610 : cluster [DBG] pgmap v6617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:58.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:38:58 smithi006 bash[27292]: audit 2023-08-28T01:38:57.442283+0000 mon.c (mon.2) 2601 : audit [DBG] from='client.? 172.21.15.6:0/1007217097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:58 smithi130 bash[27579]: cluster 2023-08-28T01:38:56.408663+0000 mgr.y (mgr.14901) 6610 : cluster [DBG] pgmap v6617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:38:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:38:58 smithi130 bash[27579]: audit 2023-08-28T01:38:57.442283+0000 mon.c (mon.2) 2601 : audit [DBG] from='client.? 172.21.15.6:0/1007217097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:59.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:58 smithi130 bash[51302]: debug 2023-08-28T01:38:58.793+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:58.796451+0000) 2023-08-28T01:39:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:38:59 smithi130 bash[51302]: debug 2023-08-28T01:38:59.793+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:59.796742+0000) 2023-08-28T01:39:00.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:00 smithi006 bash[27292]: cluster 2023-08-28T01:38:58.409259+0000 mgr.y (mgr.14901) 6611 : cluster [DBG] pgmap v6618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:00.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:00 smithi006 bash[27292]: audit 2023-08-28T01:38:59.856195+0000 mon.a (mon.0) 4356 : audit [DBG] from='client.? 172.21.15.6:0/3854930883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:00.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:00 smithi006 bash[33918]: cluster 2023-08-28T01:38:58.409259+0000 mgr.y (mgr.14901) 6611 : cluster [DBG] pgmap v6618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:00.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:00 smithi006 bash[33918]: audit 2023-08-28T01:38:59.856195+0000 mon.a (mon.0) 4356 : audit [DBG] from='client.? 172.21.15.6:0/3854930883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:00 smithi130 bash[27579]: cluster 2023-08-28T01:38:58.409259+0000 mgr.y (mgr.14901) 6611 : cluster [DBG] pgmap v6618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:00 smithi130 bash[27579]: audit 2023-08-28T01:38:59.856195+0000 mon.a (mon.0) 4356 : audit [DBG] from='client.? 172.21.15.6:0/3854930883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:01.075 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:39:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:39:00] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:39:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:00 smithi130 bash[51302]: debug 2023-08-28T01:39:00.793+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:00.797053+0000) 2023-08-28T01:39:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:39:01.075Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:39:01.076Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:02.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:01 smithi130 bash[51302]: debug 2023-08-28T01:39:01.793+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:01.797278+0000) 2023-08-28T01:39:02.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:02 smithi006 bash[27292]: cluster 2023-08-28T01:39:00.410327+0000 mgr.y (mgr.14901) 6612 : cluster [DBG] pgmap v6619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:02.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:02 smithi006 bash[33918]: cluster 2023-08-28T01:39:00.410327+0000 mgr.y (mgr.14901) 6612 : cluster [DBG] pgmap v6619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:02 smithi130 bash[27579]: cluster 2023-08-28T01:39:00.410327+0000 mgr.y (mgr.14901) 6612 : cluster [DBG] pgmap v6619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:39:02.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:39:02.644Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:03.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:02 smithi130 bash[51302]: debug 2023-08-28T01:39:02.792+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:02.797574+0000) 2023-08-28T01:39:03.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:03 smithi006 bash[27292]: audit 2023-08-28T01:39:02.274298+0000 mon.c (mon.2) 2602 : audit [DBG] from='client.? 172.21.15.6:0/878294915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:03.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:03 smithi006 bash[33918]: audit 2023-08-28T01:39:02.274298+0000 mon.c (mon.2) 2602 : audit [DBG] from='client.? 172.21.15.6:0/878294915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:03.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:03 smithi130 bash[27579]: audit 2023-08-28T01:39:02.274298+0000 mon.c (mon.2) 2602 : audit [DBG] from='client.? 172.21.15.6:0/878294915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:04.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:03 smithi130 bash[51302]: debug 2023-08-28T01:39:03.792+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:03.797847+0000) 2023-08-28T01:39:04.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:04 smithi006 bash[27292]: cluster 2023-08-28T01:39:02.410994+0000 mgr.y (mgr.14901) 6613 : cluster [DBG] pgmap v6620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:04.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:04 smithi006 bash[33918]: cluster 2023-08-28T01:39:02.410994+0000 mgr.y (mgr.14901) 6613 : cluster [DBG] pgmap v6620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:04 smithi130 bash[27579]: cluster 2023-08-28T01:39:02.410994+0000 mgr.y (mgr.14901) 6613 : cluster [DBG] pgmap v6620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:39:04.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:39:04.531Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:05.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:04 smithi130 bash[51302]: debug 2023-08-28T01:39:04.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:04.798113+0000) 2023-08-28T01:39:05.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:05 smithi130 bash[27579]: audit 2023-08-28T01:39:04.693736+0000 mon.a (mon.0) 4357 : audit [DBG] from='client.? 172.21.15.6:0/908745251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:05.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:05 smithi006 bash[27292]: audit 2023-08-28T01:39:04.693736+0000 mon.a (mon.0) 4357 : audit [DBG] from='client.? 172.21.15.6:0/908745251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:05.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:05 smithi006 bash[33918]: audit 2023-08-28T01:39:04.693736+0000 mon.a (mon.0) 4357 : audit [DBG] from='client.? 172.21.15.6:0/908745251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:06.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:05 smithi130 bash[51302]: debug 2023-08-28T01:39:05.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:05.798466+0000) 2023-08-28T01:39:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:06 smithi130 bash[27579]: cluster 2023-08-28T01:39:04.412123+0000 mgr.y (mgr.14901) 6614 : cluster [DBG] pgmap v6621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:06 smithi006 bash[27292]: cluster 2023-08-28T01:39:04.412123+0000 mgr.y (mgr.14901) 6614 : cluster [DBG] pgmap v6621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:06 smithi006 bash[33918]: cluster 2023-08-28T01:39:04.412123+0000 mgr.y (mgr.14901) 6614 : cluster [DBG] pgmap v6621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:07.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:06 smithi130 bash[51302]: debug 2023-08-28T01:39:06.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:06.798729+0000) 2023-08-28T01:39:07.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:07 smithi130 bash[27579]: audit 2023-08-28T01:39:07.120926+0000 mon.a (mon.0) 4358 : audit [DBG] from='client.? 172.21.15.6:0/261412677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:07.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:07 smithi006 bash[33918]: audit 2023-08-28T01:39:07.120926+0000 mon.a (mon.0) 4358 : audit [DBG] from='client.? 172.21.15.6:0/261412677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:07.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:07 smithi006 bash[27292]: audit 2023-08-28T01:39:07.120926+0000 mon.a (mon.0) 4358 : audit [DBG] from='client.? 172.21.15.6:0/261412677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:08.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:07 smithi130 bash[51302]: debug 2023-08-28T01:39:07.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:07.798978+0000) 2023-08-28T01:39:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:08 smithi130 bash[27579]: cluster 2023-08-28T01:39:06.412795+0000 mgr.y (mgr.14901) 6615 : cluster [DBG] pgmap v6622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:08 smithi006 bash[27292]: cluster 2023-08-28T01:39:06.412795+0000 mgr.y (mgr.14901) 6615 : cluster [DBG] pgmap v6622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:08.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:08 smithi006 bash[33918]: cluster 2023-08-28T01:39:06.412795+0000 mgr.y (mgr.14901) 6615 : cluster [DBG] pgmap v6622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:09.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:08 smithi130 bash[51302]: debug 2023-08-28T01:39:08.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:08.799230+0000) 2023-08-28T01:39:10.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:09 smithi130 bash[51302]: debug 2023-08-28T01:39:09.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:09.799477+0000) 2023-08-28T01:39:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:10 smithi130 bash[27579]: cluster 2023-08-28T01:39:08.413520+0000 mgr.y (mgr.14901) 6616 : cluster [DBG] pgmap v6623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:10 smithi130 bash[27579]: audit 2023-08-28T01:39:09.543928+0000 mon.a (mon.0) 4359 : audit [DBG] from='client.? 172.21.15.6:0/1300770737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:10.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:10 smithi006 bash[27292]: cluster 2023-08-28T01:39:08.413520+0000 mgr.y (mgr.14901) 6616 : cluster [DBG] pgmap v6623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:10 smithi006 bash[27292]: audit 2023-08-28T01:39:09.543928+0000 mon.a (mon.0) 4359 : audit [DBG] from='client.? 172.21.15.6:0/1300770737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:10.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:10 smithi006 bash[33918]: cluster 2023-08-28T01:39:08.413520+0000 mgr.y (mgr.14901) 6616 : cluster [DBG] pgmap v6623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:10.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:10 smithi006 bash[33918]: audit 2023-08-28T01:39:09.543928+0000 mon.a (mon.0) 4359 : audit [DBG] from='client.? 172.21.15.6:0/1300770737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:11.076 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:39:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:39:10] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:39:11.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:10 smithi130 bash[51302]: debug 2023-08-28T01:39:10.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:10.799711+0000) 2023-08-28T01:39:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:39:11.076Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:39:11.078Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:12.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:11 smithi130 bash[51302]: debug 2023-08-28T01:39:11.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:11.800032+0000) 2023-08-28T01:39:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:12 smithi130 bash[27579]: cluster 2023-08-28T01:39:10.414626+0000 mgr.y (mgr.14901) 6617 : cluster [DBG] pgmap v6624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:12 smithi130 bash[27579]: audit 2023-08-28T01:39:11.961895+0000 mon.c (mon.2) 2603 : audit [DBG] from='client.? 172.21.15.6:0/1551030772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:12.643 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:12 smithi006 bash[33918]: cluster 2023-08-28T01:39:10.414626+0000 mgr.y (mgr.14901) 6617 : cluster [DBG] pgmap v6624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:12.644 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:12 smithi006 bash[33918]: audit 2023-08-28T01:39:11.961895+0000 mon.c (mon.2) 2603 : audit [DBG] from='client.? 172.21.15.6:0/1551030772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:12.644 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:12 smithi006 bash[27292]: cluster 2023-08-28T01:39:10.414626+0000 mgr.y (mgr.14901) 6617 : cluster [DBG] pgmap v6624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:12.644 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:12 smithi006 bash[27292]: audit 2023-08-28T01:39:11.961895+0000 mon.c (mon.2) 2603 : audit [DBG] from='client.? 172.21.15.6:0/1551030772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:39:12.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:39:12.645Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:13.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:12 smithi130 bash[51302]: debug 2023-08-28T01:39:12.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:12.800349+0000) 2023-08-28T01:39:14.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:13 smithi130 bash[51302]: debug 2023-08-28T01:39:13.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:13.800673+0000) 2023-08-28T01:39:14.530 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:14 smithi006 bash[27292]: cluster 2023-08-28T01:39:12.415259+0000 mgr.y (mgr.14901) 6618 : cluster [DBG] pgmap v6625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:14.531 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:14 smithi006 bash[33918]: cluster 2023-08-28T01:39:12.415259+0000 mgr.y (mgr.14901) 6618 : cluster [DBG] pgmap v6625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:14 smithi130 bash[27579]: cluster 2023-08-28T01:39:12.415259+0000 mgr.y (mgr.14901) 6618 : cluster [DBG] pgmap v6625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:39:14.530Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:39:14.532Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:14 smithi130 bash[51302]: debug 2023-08-28T01:39:14.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:14.800994+0000) 2023-08-28T01:39:15.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:15 smithi130 bash[27579]: audit 2023-08-28T01:39:14.380705+0000 mon.a (mon.0) 4360 : audit [DBG] from='client.? 172.21.15.6:0/3447248361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:15.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:15 smithi006 bash[27292]: audit 2023-08-28T01:39:14.380705+0000 mon.a (mon.0) 4360 : audit [DBG] from='client.? 172.21.15.6:0/3447248361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:15.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:15 smithi006 bash[33918]: audit 2023-08-28T01:39:14.380705+0000 mon.a (mon.0) 4360 : audit [DBG] from='client.? 172.21.15.6:0/3447248361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:16.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:15 smithi130 bash[51302]: debug 2023-08-28T01:39:15.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:15.801264+0000) 2023-08-28T01:39:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:16 smithi130 bash[27579]: cluster 2023-08-28T01:39:14.416375+0000 mgr.y (mgr.14901) 6619 : cluster [DBG] pgmap v6626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:16.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:16 smithi006 bash[33918]: cluster 2023-08-28T01:39:14.416375+0000 mgr.y (mgr.14901) 6619 : cluster [DBG] pgmap v6626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:16 smithi006 bash[27292]: cluster 2023-08-28T01:39:14.416375+0000 mgr.y (mgr.14901) 6619 : cluster [DBG] pgmap v6626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:17.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:16 smithi130 bash[51302]: debug 2023-08-28T01:39:16.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:16.801543+0000) 2023-08-28T01:39:17.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:17 smithi130 bash[27579]: audit 2023-08-28T01:39:16.794498+0000 mon.a (mon.0) 4361 : audit [DBG] from='client.? 172.21.15.6:0/2919240638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:17.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:17 smithi006 bash[33918]: audit 2023-08-28T01:39:16.794498+0000 mon.a (mon.0) 4361 : audit [DBG] from='client.? 172.21.15.6:0/2919240638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:17.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:17 smithi006 bash[27292]: audit 2023-08-28T01:39:16.794498+0000 mon.a (mon.0) 4361 : audit [DBG] from='client.? 172.21.15.6:0/2919240638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:18.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:17 smithi130 bash[51302]: debug 2023-08-28T01:39:17.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:17.801849+0000) 2023-08-28T01:39:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:18 smithi130 bash[27579]: cluster 2023-08-28T01:39:16.417034+0000 mgr.y (mgr.14901) 6620 : cluster [DBG] pgmap v6627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:18 smithi006 bash[27292]: cluster 2023-08-28T01:39:16.417034+0000 mgr.y (mgr.14901) 6620 : cluster [DBG] pgmap v6627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:18.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:18 smithi006 bash[33918]: cluster 2023-08-28T01:39:16.417034+0000 mgr.y (mgr.14901) 6620 : cluster [DBG] pgmap v6627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:19.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:18 smithi130 bash[51302]: debug 2023-08-28T01:39:18.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:18.802122+0000) 2023-08-28T01:39:19.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:19 smithi130 bash[27579]: audit 2023-08-28T01:39:19.214025+0000 mon.a (mon.0) 4362 : audit [DBG] from='client.? 172.21.15.6:0/968273039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:19.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:19 smithi006 bash[27292]: audit 2023-08-28T01:39:19.214025+0000 mon.a (mon.0) 4362 : audit [DBG] from='client.? 172.21.15.6:0/968273039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:19.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:19 smithi006 bash[33918]: audit 2023-08-28T01:39:19.214025+0000 mon.a (mon.0) 4362 : audit [DBG] from='client.? 172.21.15.6:0/968273039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:19 smithi130 bash[51302]: debug 2023-08-28T01:39:19.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:19.802397+0000) 2023-08-28T01:39:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:20 smithi130 bash[27579]: cluster 2023-08-28T01:39:18.417819+0000 mgr.y (mgr.14901) 6621 : cluster [DBG] pgmap v6628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:20.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:20 smithi006 bash[33918]: cluster 2023-08-28T01:39:18.417819+0000 mgr.y (mgr.14901) 6621 : cluster [DBG] pgmap v6628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:20.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:20 smithi006 bash[27292]: cluster 2023-08-28T01:39:18.417819+0000 mgr.y (mgr.14901) 6621 : cluster [DBG] pgmap v6628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:21.076 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:39:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:39:20] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:39:21.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:20 smithi130 bash[51302]: debug 2023-08-28T01:39:20.796+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:20.802734+0000) 2023-08-28T01:39:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:39:21.076Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:39:21.078Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:22.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:21 smithi130 bash[51302]: debug 2023-08-28T01:39:21.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:21.803044+0000) 2023-08-28T01:39:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:22 smithi130 bash[27579]: cluster 2023-08-28T01:39:20.418953+0000 mgr.y (mgr.14901) 6622 : cluster [DBG] pgmap v6629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:22 smithi130 bash[27579]: audit 2023-08-28T01:39:21.624792+0000 mon.c (mon.2) 2604 : audit [DBG] from='client.? 172.21.15.6:0/3063790471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:22.645 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:22 smithi006 bash[27292]: cluster 2023-08-28T01:39:20.418953+0000 mgr.y (mgr.14901) 6622 : cluster [DBG] pgmap v6629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:22.645 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:22 smithi006 bash[27292]: audit 2023-08-28T01:39:21.624792+0000 mon.c (mon.2) 2604 : audit [DBG] from='client.? 172.21.15.6:0/3063790471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:22.645 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:22 smithi006 bash[33918]: cluster 2023-08-28T01:39:20.418953+0000 mgr.y (mgr.14901) 6622 : cluster [DBG] pgmap v6629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:22.645 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:22 smithi006 bash[33918]: audit 2023-08-28T01:39:21.624792+0000 mon.c (mon.2) 2604 : audit [DBG] from='client.? 172.21.15.6:0/3063790471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:39:22.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:39:22.646Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:23.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:22 smithi130 bash[51302]: debug 2023-08-28T01:39:22.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:22.803298+0000) 2023-08-28T01:39:24.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:23 smithi130 bash[51302]: debug 2023-08-28T01:39:23.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:23.803582+0000) 2023-08-28T01:39:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:24 smithi130 bash[27579]: cluster 2023-08-28T01:39:22.419758+0000 mgr.y (mgr.14901) 6623 : cluster [DBG] pgmap v6630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:24.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:24 smithi130 bash[27579]: audit 2023-08-28T01:39:24.037198+0000 mon.c (mon.2) 2605 : audit [DBG] from='client.? 172.21.15.6:0/2410865825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:24.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:24 smithi006 bash[33918]: cluster 2023-08-28T01:39:22.419758+0000 mgr.y (mgr.14901) 6623 : cluster [DBG] pgmap v6630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:24.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:24 smithi006 bash[33918]: audit 2023-08-28T01:39:24.037198+0000 mon.c (mon.2) 2605 : audit [DBG] from='client.? 172.21.15.6:0/2410865825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:39:24.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:24.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:39:24.533Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:24.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:24 smithi006 bash[27292]: cluster 2023-08-28T01:39:22.419758+0000 mgr.y (mgr.14901) 6623 : cluster [DBG] pgmap v6630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:24.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:24 smithi006 bash[27292]: audit 2023-08-28T01:39:24.037198+0000 mon.c (mon.2) 2605 : audit [DBG] from='client.? 172.21.15.6:0/2410865825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:25.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:24 smithi130 bash[51302]: debug 2023-08-28T01:39:24.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:24.803865+0000) 2023-08-28T01:39:26.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:25 smithi130 bash[51302]: debug 2023-08-28T01:39:25.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:25.804130+0000) 2023-08-28T01:39:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:26 smithi130 bash[27579]: cluster 2023-08-28T01:39:24.420943+0000 mgr.y (mgr.14901) 6624 : cluster [DBG] pgmap v6631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:26.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:26 smithi006 bash[27292]: cluster 2023-08-28T01:39:24.420943+0000 mgr.y (mgr.14901) 6624 : cluster [DBG] pgmap v6631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:26 smithi006 bash[33918]: cluster 2023-08-28T01:39:24.420943+0000 mgr.y (mgr.14901) 6624 : cluster [DBG] pgmap v6631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:27.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:26 smithi130 bash[51302]: debug 2023-08-28T01:39:26.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:26.804490+0000) 2023-08-28T01:39:27.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:27 smithi130 bash[27579]: audit 2023-08-28T01:39:26.450930+0000 mon.a (mon.0) 4363 : audit [DBG] from='client.? 172.21.15.6:0/2554810031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:27.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:27 smithi006 bash[33918]: audit 2023-08-28T01:39:26.450930+0000 mon.a (mon.0) 4363 : audit [DBG] from='client.? 172.21.15.6:0/2554810031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:27.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:27 smithi006 bash[27292]: audit 2023-08-28T01:39:26.450930+0000 mon.a (mon.0) 4363 : audit [DBG] from='client.? 172.21.15.6:0/2554810031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:28.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:27 smithi130 bash[51302]: debug 2023-08-28T01:39:27.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:27.804800+0000) 2023-08-28T01:39:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:28 smithi130 bash[27579]: cluster 2023-08-28T01:39:26.421518+0000 mgr.y (mgr.14901) 6625 : cluster [DBG] pgmap v6632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:28.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:28 smithi006 bash[33918]: cluster 2023-08-28T01:39:26.421518+0000 mgr.y (mgr.14901) 6625 : cluster [DBG] pgmap v6632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:28 smithi006 bash[27292]: cluster 2023-08-28T01:39:26.421518+0000 mgr.y (mgr.14901) 6625 : cluster [DBG] pgmap v6632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:28 smithi130 bash[51302]: debug 2023-08-28T01:39:28.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:28.805104+0000) 2023-08-28T01:39:29.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:29 smithi130 bash[27579]: audit 2023-08-28T01:39:28.869391+0000 mon.c (mon.2) 2606 : audit [DBG] from='client.? 172.21.15.6:0/3533220595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:29.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:29 smithi006 bash[33918]: audit 2023-08-28T01:39:28.869391+0000 mon.c (mon.2) 2606 : audit [DBG] from='client.? 172.21.15.6:0/3533220595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:29.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:29 smithi006 bash[27292]: audit 2023-08-28T01:39:28.869391+0000 mon.c (mon.2) 2606 : audit [DBG] from='client.? 172.21.15.6:0/3533220595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:30.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:29 smithi130 bash[51302]: debug 2023-08-28T01:39:29.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:29.805411+0000) 2023-08-28T01:39:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:30 smithi130 bash[27579]: cluster 2023-08-28T01:39:28.422211+0000 mgr.y (mgr.14901) 6626 : cluster [DBG] pgmap v6633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:30 smithi006 bash[33918]: cluster 2023-08-28T01:39:28.422211+0000 mgr.y (mgr.14901) 6626 : cluster [DBG] pgmap v6633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:30 smithi006 bash[27292]: cluster 2023-08-28T01:39:28.422211+0000 mgr.y (mgr.14901) 6626 : cluster [DBG] pgmap v6633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:31.077 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:39:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:39:30] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:39:31.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:30 smithi130 bash[51302]: debug 2023-08-28T01:39:30.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:30.805738+0000) 2023-08-28T01:39:31.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:31 smithi006 bash[27292]: cluster 2023-08-28T01:39:30.423382+0000 mgr.y (mgr.14901) 6627 : cluster [DBG] pgmap v6634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:31.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:31 smithi006 bash[27292]: audit 2023-08-28T01:39:31.289419+0000 mon.c (mon.2) 2607 : audit [DBG] from='client.? 172.21.15.6:0/1238833428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:31.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:31 smithi006 bash[33918]: cluster 2023-08-28T01:39:30.423382+0000 mgr.y (mgr.14901) 6627 : cluster [DBG] pgmap v6634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:31.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:31 smithi006 bash[33918]: audit 2023-08-28T01:39:31.289419+0000 mon.c (mon.2) 2607 : audit [DBG] from='client.? 172.21.15.6:0/1238833428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:39:31.077Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:39:31.079Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:31.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:31 smithi130 bash[27579]: cluster 2023-08-28T01:39:30.423382+0000 mgr.y (mgr.14901) 6627 : cluster [DBG] pgmap v6634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:31.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:31 smithi130 bash[27579]: audit 2023-08-28T01:39:31.289419+0000 mon.c (mon.2) 2607 : audit [DBG] from='client.? 172.21.15.6:0/1238833428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:32.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:31 smithi130 bash[51302]: debug 2023-08-28T01:39:31.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:31.806062+0000) 2023-08-28T01:39:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:39:32.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:39:32.647Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:33.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:32 smithi130 bash[51302]: debug 2023-08-28T01:39:32.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:32.806436+0000) 2023-08-28T01:39:33.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:33 smithi006 bash[33918]: audit 2023-08-28T01:39:32.824789+0000 mon.b (mon.1) 751 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:39:33.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:33 smithi006 bash[33918]: audit 2023-08-28T01:39:33.101058+0000 mon.a (mon.0) 4364 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:39:33.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:33 smithi006 bash[33918]: audit 2023-08-28T01:39:33.107173+0000 mon.a (mon.0) 4365 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:39:33.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:33 smithi006 bash[27292]: audit 2023-08-28T01:39:32.824789+0000 mon.b (mon.1) 751 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:39:33.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:33 smithi006 bash[27292]: audit 2023-08-28T01:39:33.101058+0000 mon.a (mon.0) 4364 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:39:33.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:33 smithi006 bash[27292]: audit 2023-08-28T01:39:33.107173+0000 mon.a (mon.0) 4365 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:39:33.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:33 smithi130 bash[27579]: audit 2023-08-28T01:39:32.824789+0000 mon.b (mon.1) 751 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:39:33.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:33 smithi130 bash[27579]: audit 2023-08-28T01:39:33.101058+0000 mon.a (mon.0) 4364 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:39:33.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:33 smithi130 bash[27579]: audit 2023-08-28T01:39:33.107173+0000 mon.a (mon.0) 4365 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:39:34.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:33 smithi130 bash[51302]: debug 2023-08-28T01:39:33.800+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:33.806798+0000) 2023-08-28T01:39:34.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:34 smithi006 bash[33918]: cluster 2023-08-28T01:39:32.424047+0000 mgr.y (mgr.14901) 6628 : cluster [DBG] pgmap v6635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:34 smithi006 bash[33918]: audit 2023-08-28T01:39:33.703074+0000 mon.a (mon.0) 4366 : audit [DBG] from='client.? 172.21.15.6:0/1302330174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:34.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:34 smithi006 bash[27292]: cluster 2023-08-28T01:39:32.424047+0000 mgr.y (mgr.14901) 6628 : cluster [DBG] pgmap v6635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:34.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:34 smithi006 bash[27292]: audit 2023-08-28T01:39:33.703074+0000 mon.a (mon.0) 4366 : audit [DBG] from='client.? 172.21.15.6:0/1302330174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:34 smithi130 bash[27579]: cluster 2023-08-28T01:39:32.424047+0000 mgr.y (mgr.14901) 6628 : cluster [DBG] pgmap v6635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:34 smithi130 bash[27579]: audit 2023-08-28T01:39:33.703074+0000 mon.a (mon.0) 4366 : audit [DBG] from='client.? 172.21.15.6:0/1302330174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:39:34.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:39:34.534Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:35.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:34 smithi130 bash[51302]: debug 2023-08-28T01:39:34.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:34.807118+0000) 2023-08-28T01:39:36.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:35 smithi130 bash[51302]: debug 2023-08-28T01:39:35.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:35.807374+0000) 2023-08-28T01:39:36.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:36 smithi006 bash[27292]: cluster 2023-08-28T01:39:34.425263+0000 mgr.y (mgr.14901) 6629 : cluster [DBG] pgmap v6636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:36.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:36 smithi006 bash[27292]: audit 2023-08-28T01:39:36.130255+0000 mon.c (mon.2) 2608 : audit [DBG] from='client.? 172.21.15.6:0/1866614991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:36.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:36 smithi006 bash[33918]: cluster 2023-08-28T01:39:34.425263+0000 mgr.y (mgr.14901) 6629 : cluster [DBG] pgmap v6636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:36.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:36 smithi006 bash[33918]: audit 2023-08-28T01:39:36.130255+0000 mon.c (mon.2) 2608 : audit [DBG] from='client.? 172.21.15.6:0/1866614991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:36 smithi130 bash[27579]: cluster 2023-08-28T01:39:34.425263+0000 mgr.y (mgr.14901) 6629 : cluster [DBG] pgmap v6636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:36 smithi130 bash[27579]: audit 2023-08-28T01:39:36.130255+0000 mon.c (mon.2) 2608 : audit [DBG] from='client.? 172.21.15.6:0/1866614991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:37.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:36 smithi130 bash[51302]: debug 2023-08-28T01:39:36.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:36.807631+0000) 2023-08-28T01:39:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:37 smithi130 bash[51302]: debug 2023-08-28T01:39:37.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:37.807961+0000) 2023-08-28T01:39:38.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:38 smithi006 bash[27292]: cluster 2023-08-28T01:39:36.425936+0000 mgr.y (mgr.14901) 6630 : cluster [DBG] pgmap v6637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:38.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:38 smithi006 bash[33918]: cluster 2023-08-28T01:39:36.425936+0000 mgr.y (mgr.14901) 6630 : cluster [DBG] pgmap v6637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:38 smithi130 bash[27579]: cluster 2023-08-28T01:39:36.425936+0000 mgr.y (mgr.14901) 6630 : cluster [DBG] pgmap v6637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:39.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:38 smithi130 bash[51302]: debug 2023-08-28T01:39:38.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:38.808262+0000) 2023-08-28T01:39:39.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:39 smithi006 bash[27292]: audit 2023-08-28T01:39:38.550823+0000 mon.a (mon.0) 4367 : audit [DBG] from='client.? 172.21.15.6:0/2110833030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:39.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:39 smithi006 bash[27292]: audit 2023-08-28T01:39:39.001877+0000 mon.a (mon.0) 4368 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:39:39.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:39 smithi006 bash[33918]: audit 2023-08-28T01:39:38.550823+0000 mon.a (mon.0) 4367 : audit [DBG] from='client.? 172.21.15.6:0/2110833030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:39.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:39 smithi006 bash[33918]: audit 2023-08-28T01:39:39.001877+0000 mon.a (mon.0) 4368 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:39:39.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:39 smithi130 bash[27579]: audit 2023-08-28T01:39:38.550823+0000 mon.a (mon.0) 4367 : audit [DBG] from='client.? 172.21.15.6:0/2110833030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:39.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:39 smithi130 bash[27579]: audit 2023-08-28T01:39:39.001877+0000 mon.a (mon.0) 4368 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:39:40.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:39 smithi130 bash[51302]: debug 2023-08-28T01:39:39.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:39.808614+0000) 2023-08-28T01:39:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:40 smithi006 bash[33918]: cluster 2023-08-28T01:39:38.426512+0000 mgr.y (mgr.14901) 6631 : cluster [DBG] pgmap v6638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:40 smithi006 bash[33918]: audit 2023-08-28T01:39:39.420992+0000 mon.a (mon.0) 4369 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:39:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:40 smithi006 bash[33918]: audit 2023-08-28T01:39:39.695570+0000 mon.b (mon.1) 752 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:39:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:40 smithi006 bash[33918]: audit 2023-08-28T01:39:39.697535+0000 mon.b (mon.1) 753 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:39:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:40 smithi006 bash[33918]: audit 2023-08-28T01:39:39.705240+0000 mon.a (mon.0) 4370 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:39:40.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:40 smithi006 bash[27292]: cluster 2023-08-28T01:39:38.426512+0000 mgr.y (mgr.14901) 6631 : cluster [DBG] pgmap v6638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:40.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:40 smithi006 bash[27292]: audit 2023-08-28T01:39:39.420992+0000 mon.a (mon.0) 4369 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:39:40.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:40 smithi006 bash[27292]: audit 2023-08-28T01:39:39.695570+0000 mon.b (mon.1) 752 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:39:40.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:40 smithi006 bash[27292]: audit 2023-08-28T01:39:39.697535+0000 mon.b (mon.1) 753 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:39:40.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:40 smithi006 bash[27292]: audit 2023-08-28T01:39:39.705240+0000 mon.a (mon.0) 4370 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:39:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:40 smithi130 bash[27579]: cluster 2023-08-28T01:39:38.426512+0000 mgr.y (mgr.14901) 6631 : cluster [DBG] pgmap v6638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:40 smithi130 bash[27579]: audit 2023-08-28T01:39:39.420992+0000 mon.a (mon.0) 4369 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:39:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:40 smithi130 bash[27579]: audit 2023-08-28T01:39:39.695570+0000 mon.b (mon.1) 752 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:39:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:40 smithi130 bash[27579]: audit 2023-08-28T01:39:39.697535+0000 mon.b (mon.1) 753 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:39:40.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:40 smithi130 bash[27579]: audit 2023-08-28T01:39:39.705240+0000 mon.a (mon.0) 4370 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:39:41.077 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:39:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:39:40] "GET /metrics HTTP/1.1" 200 193129 "" "Prometheus/2.33.4" 2023-08-28T01:39:41.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:40 smithi130 bash[51302]: debug 2023-08-28T01:39:40.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:40.808904+0000) 2023-08-28T01:39:41.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:41 smithi006 bash[33918]: audit 2023-08-28T01:39:40.962625+0000 mon.a (mon.0) 4371 : audit [DBG] from='client.? 172.21.15.6:0/1710518633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:41.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:41 smithi006 bash[27292]: audit 2023-08-28T01:39:40.962625+0000 mon.a (mon.0) 4371 : audit [DBG] from='client.? 172.21.15.6:0/1710518633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:39:41.077Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:39:41.079Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:41.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:41 smithi130 bash[27579]: audit 2023-08-28T01:39:40.962625+0000 mon.a (mon.0) 4371 : audit [DBG] from='client.? 172.21.15.6:0/1710518633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:41 smithi130 bash[51302]: debug 2023-08-28T01:39:41.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:41.809203+0000) 2023-08-28T01:39:42.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:42 smithi006 bash[33918]: cluster 2023-08-28T01:39:40.427781+0000 mgr.y (mgr.14901) 6632 : cluster [DBG] pgmap v6639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:42.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:42 smithi006 bash[27292]: cluster 2023-08-28T01:39:40.427781+0000 mgr.y (mgr.14901) 6632 : cluster [DBG] pgmap v6639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:42 smithi130 bash[27579]: cluster 2023-08-28T01:39:40.427781+0000 mgr.y (mgr.14901) 6632 : cluster [DBG] pgmap v6639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:39:42.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:39:42.648Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:43.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:42 smithi130 bash[51302]: debug 2023-08-28T01:39:42.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:42.809414+0000) 2023-08-28T01:39:44.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:43 smithi130 bash[51302]: debug 2023-08-28T01:39:43.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:43.809673+0000) 2023-08-28T01:39:44.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:44 smithi006 bash[27292]: cluster 2023-08-28T01:39:42.428460+0000 mgr.y (mgr.14901) 6633 : cluster [DBG] pgmap v6640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:44.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:44 smithi006 bash[27292]: audit 2023-08-28T01:39:43.376943+0000 mon.c (mon.2) 2609 : audit [DBG] from='client.? 172.21.15.6:0/249270700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:44 smithi006 bash[33918]: cluster 2023-08-28T01:39:42.428460+0000 mgr.y (mgr.14901) 6633 : cluster [DBG] pgmap v6640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:44 smithi006 bash[33918]: audit 2023-08-28T01:39:43.376943+0000 mon.c (mon.2) 2609 : audit [DBG] from='client.? 172.21.15.6:0/249270700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:44 smithi130 bash[27579]: cluster 2023-08-28T01:39:42.428460+0000 mgr.y (mgr.14901) 6633 : cluster [DBG] pgmap v6640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:44 smithi130 bash[27579]: audit 2023-08-28T01:39:43.376943+0000 mon.c (mon.2) 2609 : audit [DBG] from='client.? 172.21.15.6:0/249270700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:39:44.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:39:44.534Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:45.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:44 smithi130 bash[51302]: debug 2023-08-28T01:39:44.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:44.809955+0000) 2023-08-28T01:39:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:45 smithi130 bash[51302]: debug 2023-08-28T01:39:45.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:45.810261+0000) 2023-08-28T01:39:46.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:46 smithi006 bash[27292]: cluster 2023-08-28T01:39:44.429521+0000 mgr.y (mgr.14901) 6634 : cluster [DBG] pgmap v6641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:46.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:46 smithi006 bash[27292]: audit 2023-08-28T01:39:45.798367+0000 mon.a (mon.0) 4372 : audit [DBG] from='client.? 172.21.15.6:0/228196239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:46.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:46 smithi006 bash[33918]: cluster 2023-08-28T01:39:44.429521+0000 mgr.y (mgr.14901) 6634 : cluster [DBG] pgmap v6641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:46.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:46 smithi006 bash[33918]: audit 2023-08-28T01:39:45.798367+0000 mon.a (mon.0) 4372 : audit [DBG] from='client.? 172.21.15.6:0/228196239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:46 smithi130 bash[27579]: cluster 2023-08-28T01:39:44.429521+0000 mgr.y (mgr.14901) 6634 : cluster [DBG] pgmap v6641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:46 smithi130 bash[27579]: audit 2023-08-28T01:39:45.798367+0000 mon.a (mon.0) 4372 : audit [DBG] from='client.? 172.21.15.6:0/228196239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:46 smithi130 bash[51302]: debug 2023-08-28T01:39:46.804+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:46.810594+0000) 2023-08-28T01:39:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:47 smithi130 bash[51302]: debug 2023-08-28T01:39:47.808+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:47.810904+0000) 2023-08-28T01:39:48.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:48 smithi006 bash[27292]: cluster 2023-08-28T01:39:46.430209+0000 mgr.y (mgr.14901) 6635 : cluster [DBG] pgmap v6642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:48.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:48 smithi006 bash[33918]: cluster 2023-08-28T01:39:46.430209+0000 mgr.y (mgr.14901) 6635 : cluster [DBG] pgmap v6642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:48 smithi130 bash[27579]: cluster 2023-08-28T01:39:46.430209+0000 mgr.y (mgr.14901) 6635 : cluster [DBG] pgmap v6642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:49.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:48 smithi130 bash[51302]: debug 2023-08-28T01:39:48.808+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:48.811117+0000) 2023-08-28T01:39:49.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:49 smithi006 bash[27292]: audit 2023-08-28T01:39:48.225619+0000 mon.c (mon.2) 2610 : audit [DBG] from='client.? 172.21.15.6:0/1503103639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:49.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:49 smithi006 bash[33918]: audit 2023-08-28T01:39:48.225619+0000 mon.c (mon.2) 2610 : audit [DBG] from='client.? 172.21.15.6:0/1503103639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:49.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:49 smithi130 bash[27579]: audit 2023-08-28T01:39:48.225619+0000 mon.c (mon.2) 2610 : audit [DBG] from='client.? 172.21.15.6:0/1503103639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:50.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:49 smithi130 bash[51302]: debug 2023-08-28T01:39:49.808+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:49.811338+0000) 2023-08-28T01:39:50.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:50 smithi006 bash[27292]: cluster 2023-08-28T01:39:48.430901+0000 mgr.y (mgr.14901) 6636 : cluster [DBG] pgmap v6643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:50.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:50 smithi006 bash[33918]: cluster 2023-08-28T01:39:48.430901+0000 mgr.y (mgr.14901) 6636 : cluster [DBG] pgmap v6643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:50 smithi130 bash[27579]: cluster 2023-08-28T01:39:48.430901+0000 mgr.y (mgr.14901) 6636 : cluster [DBG] pgmap v6643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:51.078 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:39:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:39:50] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T01:39:51.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:50 smithi130 bash[51302]: debug 2023-08-28T01:39:50.808+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:50.811577+0000) 2023-08-28T01:39:51.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:51 smithi006 bash[33918]: audit 2023-08-28T01:39:50.644360+0000 mon.a (mon.0) 4373 : audit [DBG] from='client.? 172.21.15.6:0/117121560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:51.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:51 smithi006 bash[27292]: audit 2023-08-28T01:39:50.644360+0000 mon.a (mon.0) 4373 : audit [DBG] from='client.? 172.21.15.6:0/117121560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:39:51.078Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:39:51.080Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:51.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:51 smithi130 bash[27579]: audit 2023-08-28T01:39:50.644360+0000 mon.a (mon.0) 4373 : audit [DBG] from='client.? 172.21.15.6:0/117121560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:52.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:51 smithi130 bash[51302]: debug 2023-08-28T01:39:51.808+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:51.811867+0000) 2023-08-28T01:39:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:52 smithi130 bash[27579]: cluster 2023-08-28T01:39:50.431947+0000 mgr.y (mgr.14901) 6637 : cluster [DBG] pgmap v6644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:52.647 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:52 smithi006 bash[27292]: cluster 2023-08-28T01:39:50.431947+0000 mgr.y (mgr.14901) 6637 : cluster [DBG] pgmap v6644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:52.648 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:52 smithi006 bash[33918]: cluster 2023-08-28T01:39:50.431947+0000 mgr.y (mgr.14901) 6637 : cluster [DBG] pgmap v6644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:39:52.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:39:52.649Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:53.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:52 smithi130 bash[51302]: debug 2023-08-28T01:39:52.808+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:52.812159+0000) 2023-08-28T01:39:53.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:53 smithi130 bash[27579]: audit 2023-08-28T01:39:53.053861+0000 mon.a (mon.0) 4374 : audit [DBG] from='client.? 172.21.15.6:0/2577144131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:53.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:53 smithi006 bash[33918]: audit 2023-08-28T01:39:53.053861+0000 mon.a (mon.0) 4374 : audit [DBG] from='client.? 172.21.15.6:0/2577144131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:53.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:53 smithi006 bash[27292]: audit 2023-08-28T01:39:53.053861+0000 mon.a (mon.0) 4374 : audit [DBG] from='client.? 172.21.15.6:0/2577144131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:54.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:53 smithi130 bash[51302]: debug 2023-08-28T01:39:53.812+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:53.812444+0000) 2023-08-28T01:39:54.533 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:54 smithi006 bash[33918]: cluster 2023-08-28T01:39:52.432728+0000 mgr.y (mgr.14901) 6638 : cluster [DBG] pgmap v6645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:54.533 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:54 smithi006 bash[27292]: cluster 2023-08-28T01:39:52.432728+0000 mgr.y (mgr.14901) 6638 : cluster [DBG] pgmap v6645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:54 smithi130 bash[27579]: cluster 2023-08-28T01:39:52.432728+0000 mgr.y (mgr.14901) 6638 : cluster [DBG] pgmap v6645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:39:54.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:39:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:39:54.535Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:39:55.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:54 smithi130 bash[51302]: debug 2023-08-28T01:39:54.808+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:54.812697+0000) 2023-08-28T01:39:56.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:55 smithi130 bash[51302]: debug 2023-08-28T01:39:55.808+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:55.812983+0000) 2023-08-28T01:39:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:56 smithi130 bash[27579]: cluster 2023-08-28T01:39:54.433847+0000 mgr.y (mgr.14901) 6639 : cluster [DBG] pgmap v6646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:56 smithi130 bash[27579]: audit 2023-08-28T01:39:55.475640+0000 mon.a (mon.0) 4375 : audit [DBG] from='client.? 172.21.15.6:0/1804467941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:56.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:56 smithi006 bash[27292]: cluster 2023-08-28T01:39:54.433847+0000 mgr.y (mgr.14901) 6639 : cluster [DBG] pgmap v6646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:56.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:56 smithi006 bash[27292]: audit 2023-08-28T01:39:55.475640+0000 mon.a (mon.0) 4375 : audit [DBG] from='client.? 172.21.15.6:0/1804467941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:56.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:56 smithi006 bash[33918]: cluster 2023-08-28T01:39:54.433847+0000 mgr.y (mgr.14901) 6639 : cluster [DBG] pgmap v6646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:56.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:56 smithi006 bash[33918]: audit 2023-08-28T01:39:55.475640+0000 mon.a (mon.0) 4375 : audit [DBG] from='client.? 172.21.15.6:0/1804467941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:57.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:56 smithi130 bash[51302]: debug 2023-08-28T01:39:56.808+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:56.813271+0000) 2023-08-28T01:39:58.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:57 smithi130 bash[51302]: debug 2023-08-28T01:39:57.808+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:57.813530+0000) 2023-08-28T01:39:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:58 smithi130 bash[27579]: cluster 2023-08-28T01:39:56.434489+0000 mgr.y (mgr.14901) 6640 : cluster [DBG] pgmap v6647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:39:58 smithi130 bash[27579]: audit 2023-08-28T01:39:57.891387+0000 mon.c (mon.2) 2611 : audit [DBG] from='client.? 172.21.15.6:0/1870501415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:58.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:58 smithi006 bash[33918]: cluster 2023-08-28T01:39:56.434489+0000 mgr.y (mgr.14901) 6640 : cluster [DBG] pgmap v6647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:58.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:39:58 smithi006 bash[33918]: audit 2023-08-28T01:39:57.891387+0000 mon.c (mon.2) 2611 : audit [DBG] from='client.? 172.21.15.6:0/1870501415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:58.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:58 smithi006 bash[27292]: cluster 2023-08-28T01:39:56.434489+0000 mgr.y (mgr.14901) 6640 : cluster [DBG] pgmap v6647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:39:58.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:39:58 smithi006 bash[27292]: audit 2023-08-28T01:39:57.891387+0000 mon.c (mon.2) 2611 : audit [DBG] from='client.? 172.21.15.6:0/1870501415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:59.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:58 smithi130 bash[51302]: debug 2023-08-28T01:39:58.807+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:58.813894+0000) 2023-08-28T01:40:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:39:59 smithi130 bash[51302]: debug 2023-08-28T01:39:59.807+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:59.814228+0000) 2023-08-28T01:40:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:00 smithi130 bash[27579]: cluster 2023-08-28T01:39:58.435142+0000 mgr.y (mgr.14901) 6641 : cluster [DBG] pgmap v6648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:00.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:00 smithi130 bash[27579]: cluster 2023-08-28T01:40:00.000166+0000 mon.a (mon.0) 4376 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:40:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:00 smithi006 bash[27292]: cluster 2023-08-28T01:39:58.435142+0000 mgr.y (mgr.14901) 6641 : cluster [DBG] pgmap v6648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:00 smithi006 bash[27292]: cluster 2023-08-28T01:40:00.000166+0000 mon.a (mon.0) 4376 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:40:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:00 smithi006 bash[33918]: cluster 2023-08-28T01:39:58.435142+0000 mgr.y (mgr.14901) 6641 : cluster [DBG] pgmap v6648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:00 smithi006 bash[33918]: cluster 2023-08-28T01:40:00.000166+0000 mon.a (mon.0) 4376 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:40:01.079 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:40:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:40:00] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T01:40:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:00 smithi130 bash[51302]: debug 2023-08-28T01:40:00.807+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:00.814560+0000) 2023-08-28T01:40:01.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:01 smithi006 bash[27292]: audit 2023-08-28T01:40:00.306815+0000 mon.a (mon.0) 4377 : audit [DBG] from='client.? 172.21.15.6:0/377458947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:01.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:01 smithi006 bash[33918]: audit 2023-08-28T01:40:00.306815+0000 mon.a (mon.0) 4377 : audit [DBG] from='client.? 172.21.15.6:0/377458947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:40:01.079Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:40:01.082Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:01.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:01 smithi130 bash[27579]: audit 2023-08-28T01:40:00.306815+0000 mon.a (mon.0) 4377 : audit [DBG] from='client.? 172.21.15.6:0/377458947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:02.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:01 smithi130 bash[51302]: debug 2023-08-28T01:40:01.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:01.814891+0000) 2023-08-28T01:40:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:02 smithi130 bash[27579]: cluster 2023-08-28T01:40:00.436268+0000 mgr.y (mgr.14901) 6642 : cluster [DBG] pgmap v6649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:02.648 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:02 smithi006 bash[33918]: cluster 2023-08-28T01:40:00.436268+0000 mgr.y (mgr.14901) 6642 : cluster [DBG] pgmap v6649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:02.648 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:02 smithi006 bash[27292]: cluster 2023-08-28T01:40:00.436268+0000 mgr.y (mgr.14901) 6642 : cluster [DBG] pgmap v6649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:40:02.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:40:02.650Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:03.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:02 smithi130 bash[51302]: debug 2023-08-28T01:40:02.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:02.815253+0000) 2023-08-28T01:40:03.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:03 smithi130 bash[27579]: audit 2023-08-28T01:40:02.721525+0000 mon.a (mon.0) 4378 : audit [DBG] from='client.? 172.21.15.6:0/1810960276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:03.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:03 smithi006 bash[33918]: audit 2023-08-28T01:40:02.721525+0000 mon.a (mon.0) 4378 : audit [DBG] from='client.? 172.21.15.6:0/1810960276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:03.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:03 smithi006 bash[27292]: audit 2023-08-28T01:40:02.721525+0000 mon.a (mon.0) 4378 : audit [DBG] from='client.? 172.21.15.6:0/1810960276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:04.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:03 smithi130 bash[51302]: debug 2023-08-28T01:40:03.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:03.815483+0000) 2023-08-28T01:40:04.534 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:04 smithi006 bash[27292]: cluster 2023-08-28T01:40:02.436968+0000 mgr.y (mgr.14901) 6643 : cluster [DBG] pgmap v6650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:04.534 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:04 smithi006 bash[33918]: cluster 2023-08-28T01:40:02.436968+0000 mgr.y (mgr.14901) 6643 : cluster [DBG] pgmap v6650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:04 smithi130 bash[27579]: cluster 2023-08-28T01:40:02.436968+0000 mgr.y (mgr.14901) 6643 : cluster [DBG] pgmap v6650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:04.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:40:04.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:04.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:40:04.535Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:05.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:04 smithi130 bash[51302]: debug 2023-08-28T01:40:04.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:04.815751+0000) 2023-08-28T01:40:05.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:05 smithi130 bash[27579]: audit 2023-08-28T01:40:05.139217+0000 mon.c (mon.2) 2612 : audit [DBG] from='client.? 172.21.15.6:0/153474895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:05.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:05 smithi006 bash[27292]: audit 2023-08-28T01:40:05.139217+0000 mon.c (mon.2) 2612 : audit [DBG] from='client.? 172.21.15.6:0/153474895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:05.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:05 smithi006 bash[33918]: audit 2023-08-28T01:40:05.139217+0000 mon.c (mon.2) 2612 : audit [DBG] from='client.? 172.21.15.6:0/153474895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:06.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:05 smithi130 bash[51302]: debug 2023-08-28T01:40:05.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:05.816011+0000) 2023-08-28T01:40:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:06 smithi130 bash[27579]: cluster 2023-08-28T01:40:04.438100+0000 mgr.y (mgr.14901) 6644 : cluster [DBG] pgmap v6651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:06.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:06 smithi006 bash[27292]: cluster 2023-08-28T01:40:04.438100+0000 mgr.y (mgr.14901) 6644 : cluster [DBG] pgmap v6651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:06 smithi006 bash[33918]: cluster 2023-08-28T01:40:04.438100+0000 mgr.y (mgr.14901) 6644 : cluster [DBG] pgmap v6651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:07.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:06 smithi130 bash[51302]: debug 2023-08-28T01:40:06.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:06.816271+0000) 2023-08-28T01:40:08.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:07 smithi130 bash[51302]: debug 2023-08-28T01:40:07.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:07.816531+0000) 2023-08-28T01:40:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:08 smithi130 bash[27579]: cluster 2023-08-28T01:40:06.438793+0000 mgr.y (mgr.14901) 6645 : cluster [DBG] pgmap v6652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:08 smithi130 bash[27579]: audit 2023-08-28T01:40:07.550015+0000 mon.c (mon.2) 2613 : audit [DBG] from='client.? 172.21.15.6:0/752109930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:08.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:08 smithi006 bash[27292]: cluster 2023-08-28T01:40:06.438793+0000 mgr.y (mgr.14901) 6645 : cluster [DBG] pgmap v6652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:08 smithi006 bash[27292]: audit 2023-08-28T01:40:07.550015+0000 mon.c (mon.2) 2613 : audit [DBG] from='client.? 172.21.15.6:0/752109930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:08.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:08 smithi006 bash[33918]: cluster 2023-08-28T01:40:06.438793+0000 mgr.y (mgr.14901) 6645 : cluster [DBG] pgmap v6652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:08.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:08 smithi006 bash[33918]: audit 2023-08-28T01:40:07.550015+0000 mon.c (mon.2) 2613 : audit [DBG] from='client.? 172.21.15.6:0/752109930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:09.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:08 smithi130 bash[51302]: debug 2023-08-28T01:40:08.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:08.816841+0000) 2023-08-28T01:40:10.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:09 smithi130 bash[51302]: debug 2023-08-28T01:40:09.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:09.817135+0000) 2023-08-28T01:40:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:10 smithi130 bash[27579]: cluster 2023-08-28T01:40:08.439465+0000 mgr.y (mgr.14901) 6646 : cluster [DBG] pgmap v6653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:10 smithi130 bash[27579]: audit 2023-08-28T01:40:09.964931+0000 mon.c (mon.2) 2614 : audit [DBG] from='client.? 172.21.15.6:0/4034332285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:10 smithi006 bash[27292]: cluster 2023-08-28T01:40:08.439465+0000 mgr.y (mgr.14901) 6646 : cluster [DBG] pgmap v6653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:10 smithi006 bash[27292]: audit 2023-08-28T01:40:09.964931+0000 mon.c (mon.2) 2614 : audit [DBG] from='client.? 172.21.15.6:0/4034332285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:10.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:10 smithi006 bash[33918]: cluster 2023-08-28T01:40:08.439465+0000 mgr.y (mgr.14901) 6646 : cluster [DBG] pgmap v6653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:10.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:10 smithi006 bash[33918]: audit 2023-08-28T01:40:09.964931+0000 mon.c (mon.2) 2614 : audit [DBG] from='client.? 172.21.15.6:0/4034332285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:11.080 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:40:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:40:10] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T01:40:11.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:10 smithi130 bash[51302]: debug 2023-08-28T01:40:10.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:10.817419+0000) 2023-08-28T01:40:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:40:11.080Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:40:11.082Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:12.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:11 smithi130 bash[51302]: debug 2023-08-28T01:40:11.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:11.817716+0000) 2023-08-28T01:40:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:12 smithi130 bash[27579]: cluster 2023-08-28T01:40:10.440628+0000 mgr.y (mgr.14901) 6647 : cluster [DBG] pgmap v6654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:12.649 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:12 smithi006 bash[27292]: cluster 2023-08-28T01:40:10.440628+0000 mgr.y (mgr.14901) 6647 : cluster [DBG] pgmap v6654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:12.649 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:12 smithi006 bash[33918]: cluster 2023-08-28T01:40:10.440628+0000 mgr.y (mgr.14901) 6647 : cluster [DBG] pgmap v6654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:40:12.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:40:12.650Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:13.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:12 smithi130 bash[51302]: debug 2023-08-28T01:40:12.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:12.818001+0000) 2023-08-28T01:40:13.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:13 smithi130 bash[27579]: audit 2023-08-28T01:40:12.374347+0000 mon.c (mon.2) 2615 : audit [DBG] from='client.? 172.21.15.6:0/3956505121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:13.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:13 smithi006 bash[27292]: audit 2023-08-28T01:40:12.374347+0000 mon.c (mon.2) 2615 : audit [DBG] from='client.? 172.21.15.6:0/3956505121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:13.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:13 smithi006 bash[33918]: audit 2023-08-28T01:40:12.374347+0000 mon.c (mon.2) 2615 : audit [DBG] from='client.? 172.21.15.6:0/3956505121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:14.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:13 smithi130 bash[51302]: debug 2023-08-28T01:40:13.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:13.818286+0000) 2023-08-28T01:40:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:14 smithi130 bash[27579]: cluster 2023-08-28T01:40:12.441277+0000 mgr.y (mgr.14901) 6648 : cluster [DBG] pgmap v6655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:14.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:14 smithi006 bash[33918]: cluster 2023-08-28T01:40:12.441277+0000 mgr.y (mgr.14901) 6648 : cluster [DBG] pgmap v6655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:14.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:14 smithi006 bash[27292]: cluster 2023-08-28T01:40:12.441277+0000 mgr.y (mgr.14901) 6648 : cluster [DBG] pgmap v6655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:40:14.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:40:14.537Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:14 smithi130 bash[51302]: debug 2023-08-28T01:40:14.811+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:14.818595+0000) 2023-08-28T01:40:15.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:15 smithi130 bash[27579]: audit 2023-08-28T01:40:14.800559+0000 mon.a (mon.0) 4379 : audit [DBG] from='client.? 172.21.15.6:0/1406047792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:15.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:15 smithi006 bash[27292]: audit 2023-08-28T01:40:14.800559+0000 mon.a (mon.0) 4379 : audit [DBG] from='client.? 172.21.15.6:0/1406047792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:15.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:15 smithi006 bash[33918]: audit 2023-08-28T01:40:14.800559+0000 mon.a (mon.0) 4379 : audit [DBG] from='client.? 172.21.15.6:0/1406047792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:16.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:15 smithi130 bash[51302]: debug 2023-08-28T01:40:15.815+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:15.818913+0000) 2023-08-28T01:40:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:16 smithi130 bash[27579]: cluster 2023-08-28T01:40:14.442354+0000 mgr.y (mgr.14901) 6649 : cluster [DBG] pgmap v6656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:16.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:16 smithi006 bash[27292]: cluster 2023-08-28T01:40:14.442354+0000 mgr.y (mgr.14901) 6649 : cluster [DBG] pgmap v6656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:16 smithi006 bash[33918]: cluster 2023-08-28T01:40:14.442354+0000 mgr.y (mgr.14901) 6649 : cluster [DBG] pgmap v6656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:17.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:16 smithi130 bash[51302]: debug 2023-08-28T01:40:16.815+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:16.819206+0000) 2023-08-28T01:40:18.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:17 smithi130 bash[51302]: debug 2023-08-28T01:40:17.815+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:17.819464+0000) 2023-08-28T01:40:18.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:18 smithi006 bash[27292]: cluster 2023-08-28T01:40:16.443027+0000 mgr.y (mgr.14901) 6650 : cluster [DBG] pgmap v6657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:18.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:18 smithi006 bash[27292]: audit 2023-08-28T01:40:17.210798+0000 mon.a (mon.0) 4380 : audit [DBG] from='client.? 172.21.15.6:0/1854470308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:18.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:18 smithi006 bash[33918]: cluster 2023-08-28T01:40:16.443027+0000 mgr.y (mgr.14901) 6650 : cluster [DBG] pgmap v6657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:18.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:18 smithi006 bash[33918]: audit 2023-08-28T01:40:17.210798+0000 mon.a (mon.0) 4380 : audit [DBG] from='client.? 172.21.15.6:0/1854470308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:18 smithi130 bash[27579]: cluster 2023-08-28T01:40:16.443027+0000 mgr.y (mgr.14901) 6650 : cluster [DBG] pgmap v6657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:18 smithi130 bash[27579]: audit 2023-08-28T01:40:17.210798+0000 mon.a (mon.0) 4380 : audit [DBG] from='client.? 172.21.15.6:0/1854470308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:19.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:18 smithi130 bash[51302]: debug 2023-08-28T01:40:18.815+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:18.819824+0000) 2023-08-28T01:40:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:19 smithi130 bash[51302]: debug 2023-08-28T01:40:19.815+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:19.820123+0000) 2023-08-28T01:40:20.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:20 smithi006 bash[27292]: cluster 2023-08-28T01:40:18.443649+0000 mgr.y (mgr.14901) 6651 : cluster [DBG] pgmap v6658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:20.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:20 smithi006 bash[27292]: audit 2023-08-28T01:40:19.632823+0000 mon.c (mon.2) 2616 : audit [DBG] from='client.? 172.21.15.6:0/1478682835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:20.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:20 smithi006 bash[33918]: cluster 2023-08-28T01:40:18.443649+0000 mgr.y (mgr.14901) 6651 : cluster [DBG] pgmap v6658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:20.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:20 smithi006 bash[33918]: audit 2023-08-28T01:40:19.632823+0000 mon.c (mon.2) 2616 : audit [DBG] from='client.? 172.21.15.6:0/1478682835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:20 smithi130 bash[27579]: cluster 2023-08-28T01:40:18.443649+0000 mgr.y (mgr.14901) 6651 : cluster [DBG] pgmap v6658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:20 smithi130 bash[27579]: audit 2023-08-28T01:40:19.632823+0000 mon.c (mon.2) 2616 : audit [DBG] from='client.? 172.21.15.6:0/1478682835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:21.081 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:40:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:40:20] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:40:21.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:20 smithi130 bash[51302]: debug 2023-08-28T01:40:20.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:20.820501+0000) 2023-08-28T01:40:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:40:21.080Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:40:21.082Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:22.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:21 smithi130 bash[51302]: debug 2023-08-28T01:40:21.815+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:21.820831+0000) 2023-08-28T01:40:22.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:22 smithi006 bash[27292]: cluster 2023-08-28T01:40:20.444818+0000 mgr.y (mgr.14901) 6652 : cluster [DBG] pgmap v6659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:22.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:22 smithi006 bash[27292]: audit 2023-08-28T01:40:22.047691+0000 mon.c (mon.2) 2617 : audit [DBG] from='client.? 172.21.15.6:0/3288021562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:22.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:22 smithi006 bash[33918]: cluster 2023-08-28T01:40:20.444818+0000 mgr.y (mgr.14901) 6652 : cluster [DBG] pgmap v6659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:22.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:22 smithi006 bash[33918]: audit 2023-08-28T01:40:22.047691+0000 mon.c (mon.2) 2617 : audit [DBG] from='client.? 172.21.15.6:0/3288021562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:22 smithi130 bash[27579]: cluster 2023-08-28T01:40:20.444818+0000 mgr.y (mgr.14901) 6652 : cluster [DBG] pgmap v6659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:22 smithi130 bash[27579]: audit 2023-08-28T01:40:22.047691+0000 mon.c (mon.2) 2617 : audit [DBG] from='client.? 172.21.15.6:0/3288021562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:40:22.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:40:22.651Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:23.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:22 smithi130 bash[51302]: debug 2023-08-28T01:40:22.815+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:22.821140+0000) 2023-08-28T01:40:24.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:23 smithi130 bash[51302]: debug 2023-08-28T01:40:23.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:23.821567+0000) 2023-08-28T01:40:24.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:24 smithi006 bash[33918]: cluster 2023-08-28T01:40:22.445794+0000 mgr.y (mgr.14901) 6653 : cluster [DBG] pgmap v6660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:24 smithi006 bash[27292]: cluster 2023-08-28T01:40:22.445794+0000 mgr.y (mgr.14901) 6653 : cluster [DBG] pgmap v6660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:24 smithi130 bash[27579]: cluster 2023-08-28T01:40:22.445794+0000 mgr.y (mgr.14901) 6653 : cluster [DBG] pgmap v6660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:40:24.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:40:24.537Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:25.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:24 smithi130 bash[51302]: debug 2023-08-28T01:40:24.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:24.821869+0000) 2023-08-28T01:40:25.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:25 smithi006 bash[27292]: audit 2023-08-28T01:40:24.475466+0000 mon.c (mon.2) 2618 : audit [DBG] from='client.? 172.21.15.6:0/2468751141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:25.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:25 smithi006 bash[33918]: audit 2023-08-28T01:40:24.475466+0000 mon.c (mon.2) 2618 : audit [DBG] from='client.? 172.21.15.6:0/2468751141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:25.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:25 smithi130 bash[27579]: audit 2023-08-28T01:40:24.475466+0000 mon.c (mon.2) 2618 : audit [DBG] from='client.? 172.21.15.6:0/2468751141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:26.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:25 smithi130 bash[51302]: debug 2023-08-28T01:40:25.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:25.822137+0000) 2023-08-28T01:40:26.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:26 smithi006 bash[33918]: cluster 2023-08-28T01:40:24.446849+0000 mgr.y (mgr.14901) 6654 : cluster [DBG] pgmap v6661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:26.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:26 smithi006 bash[27292]: cluster 2023-08-28T01:40:24.446849+0000 mgr.y (mgr.14901) 6654 : cluster [DBG] pgmap v6661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:26 smithi130 bash[27579]: cluster 2023-08-28T01:40:24.446849+0000 mgr.y (mgr.14901) 6654 : cluster [DBG] pgmap v6661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:27.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:26 smithi130 bash[51302]: debug 2023-08-28T01:40:26.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:26.822434+0000) 2023-08-28T01:40:27.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:27 smithi006 bash[33918]: audit 2023-08-28T01:40:26.886760+0000 mon.a (mon.0) 4381 : audit [DBG] from='client.? 172.21.15.6:0/3491810194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:27.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:27 smithi006 bash[27292]: audit 2023-08-28T01:40:26.886760+0000 mon.a (mon.0) 4381 : audit [DBG] from='client.? 172.21.15.6:0/3491810194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:27.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:27 smithi130 bash[27579]: audit 2023-08-28T01:40:26.886760+0000 mon.a (mon.0) 4381 : audit [DBG] from='client.? 172.21.15.6:0/3491810194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:28.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:27 smithi130 bash[51302]: debug 2023-08-28T01:40:27.815+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:27.822703+0000) 2023-08-28T01:40:28.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:28 smithi006 bash[33918]: cluster 2023-08-28T01:40:26.447481+0000 mgr.y (mgr.14901) 6655 : cluster [DBG] pgmap v6662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:28.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:28 smithi006 bash[27292]: cluster 2023-08-28T01:40:26.447481+0000 mgr.y (mgr.14901) 6655 : cluster [DBG] pgmap v6662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:28 smithi130 bash[27579]: cluster 2023-08-28T01:40:26.447481+0000 mgr.y (mgr.14901) 6655 : cluster [DBG] pgmap v6662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:28 smithi130 bash[51302]: debug 2023-08-28T01:40:28.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:28.822981+0000) 2023-08-28T01:40:30.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:29 smithi130 bash[51302]: debug 2023-08-28T01:40:29.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:29.823256+0000) 2023-08-28T01:40:30.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:30 smithi006 bash[33918]: cluster 2023-08-28T01:40:28.448130+0000 mgr.y (mgr.14901) 6656 : cluster [DBG] pgmap v6663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:30.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:30 smithi006 bash[33918]: audit 2023-08-28T01:40:29.298010+0000 mon.a (mon.0) 4382 : audit [DBG] from='client.? 172.21.15.6:0/911983655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:30.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:30 smithi006 bash[27292]: cluster 2023-08-28T01:40:28.448130+0000 mgr.y (mgr.14901) 6656 : cluster [DBG] pgmap v6663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:30.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:30 smithi006 bash[27292]: audit 2023-08-28T01:40:29.298010+0000 mon.a (mon.0) 4382 : audit [DBG] from='client.? 172.21.15.6:0/911983655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:30 smithi130 bash[27579]: cluster 2023-08-28T01:40:28.448130+0000 mgr.y (mgr.14901) 6656 : cluster [DBG] pgmap v6663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:30 smithi130 bash[27579]: audit 2023-08-28T01:40:29.298010+0000 mon.a (mon.0) 4382 : audit [DBG] from='client.? 172.21.15.6:0/911983655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:31.082 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:40:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:40:30] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:40:31.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:30 smithi130 bash[51302]: debug 2023-08-28T01:40:30.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:30.823631+0000) 2023-08-28T01:40:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:40:31.082Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:40:31.084Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:32.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:31 smithi130 bash[51302]: debug 2023-08-28T01:40:31.823+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:31.823931+0000) 2023-08-28T01:40:32.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:32 smithi006 bash[33918]: cluster 2023-08-28T01:40:30.449291+0000 mgr.y (mgr.14901) 6657 : cluster [DBG] pgmap v6664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:32.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:32 smithi006 bash[33918]: audit 2023-08-28T01:40:31.712576+0000 mon.c (mon.2) 2619 : audit [DBG] from='client.? 172.21.15.6:0/4172035611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:32.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:32 smithi006 bash[27292]: cluster 2023-08-28T01:40:30.449291+0000 mgr.y (mgr.14901) 6657 : cluster [DBG] pgmap v6664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:32.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:32 smithi006 bash[27292]: audit 2023-08-28T01:40:31.712576+0000 mon.c (mon.2) 2619 : audit [DBG] from='client.? 172.21.15.6:0/4172035611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:32 smithi130 bash[27579]: cluster 2023-08-28T01:40:30.449291+0000 mgr.y (mgr.14901) 6657 : cluster [DBG] pgmap v6664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:32 smithi130 bash[27579]: audit 2023-08-28T01:40:31.712576+0000 mon.c (mon.2) 2619 : audit [DBG] from='client.? 172.21.15.6:0/4172035611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:40:32.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:40:32.652Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:33.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:32 smithi130 bash[51302]: debug 2023-08-28T01:40:32.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:32.824195+0000) 2023-08-28T01:40:34.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:33 smithi130 bash[51302]: debug 2023-08-28T01:40:33.823+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:33.824474+0000) 2023-08-28T01:40:34.206 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:34 smithi006 bash[27292]: cluster 2023-08-28T01:40:32.449944+0000 mgr.y (mgr.14901) 6658 : cluster [DBG] pgmap v6665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:34.206 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:34 smithi006 bash[27292]: audit 2023-08-28T01:40:34.127683+0000 mon.c (mon.2) 2620 : audit [DBG] from='client.? 172.21.15.6:0/353020477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:34.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:34 smithi006 bash[33918]: cluster 2023-08-28T01:40:32.449944+0000 mgr.y (mgr.14901) 6658 : cluster [DBG] pgmap v6665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:34 smithi006 bash[33918]: audit 2023-08-28T01:40:34.127683+0000 mon.c (mon.2) 2620 : audit [DBG] from='client.? 172.21.15.6:0/353020477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:34 smithi130 bash[27579]: cluster 2023-08-28T01:40:32.449944+0000 mgr.y (mgr.14901) 6658 : cluster [DBG] pgmap v6665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:34 smithi130 bash[27579]: audit 2023-08-28T01:40:34.127683+0000 mon.c (mon.2) 2620 : audit [DBG] from='client.? 172.21.15.6:0/353020477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:40:34.536Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:40:34.538Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:35.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:34 smithi130 bash[51302]: debug 2023-08-28T01:40:34.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:34.824754+0000) 2023-08-28T01:40:36.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:35 smithi130 bash[51302]: debug 2023-08-28T01:40:35.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:35.825020+0000) 2023-08-28T01:40:36.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:36 smithi006 bash[33918]: cluster 2023-08-28T01:40:34.451042+0000 mgr.y (mgr.14901) 6659 : cluster [DBG] pgmap v6666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:36.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:36 smithi006 bash[27292]: cluster 2023-08-28T01:40:34.451042+0000 mgr.y (mgr.14901) 6659 : cluster [DBG] pgmap v6666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:36 smithi130 bash[27579]: cluster 2023-08-28T01:40:34.451042+0000 mgr.y (mgr.14901) 6659 : cluster [DBG] pgmap v6666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:37.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:36 smithi130 bash[51302]: debug 2023-08-28T01:40:36.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:36.825281+0000) 2023-08-28T01:40:37.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:37 smithi006 bash[33918]: audit 2023-08-28T01:40:36.540243+0000 mon.a (mon.0) 4383 : audit [DBG] from='client.? 172.21.15.6:0/1755846793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:37.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:37 smithi006 bash[27292]: audit 2023-08-28T01:40:36.540243+0000 mon.a (mon.0) 4383 : audit [DBG] from='client.? 172.21.15.6:0/1755846793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:37.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:37 smithi130 bash[27579]: audit 2023-08-28T01:40:36.540243+0000 mon.a (mon.0) 4383 : audit [DBG] from='client.? 172.21.15.6:0/1755846793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:37 smithi130 bash[51302]: debug 2023-08-28T01:40:37.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:37.825561+0000) 2023-08-28T01:40:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:38 smithi130 bash[27579]: cluster 2023-08-28T01:40:36.451672+0000 mgr.y (mgr.14901) 6660 : cluster [DBG] pgmap v6667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:38 smithi006 bash[33918]: cluster 2023-08-28T01:40:36.451672+0000 mgr.y (mgr.14901) 6660 : cluster [DBG] pgmap v6667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:38 smithi006 bash[27292]: cluster 2023-08-28T01:40:36.451672+0000 mgr.y (mgr.14901) 6660 : cluster [DBG] pgmap v6667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:39.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:38 smithi130 bash[51302]: debug 2023-08-28T01:40:38.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:38.825835+0000) 2023-08-28T01:40:39.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:39 smithi130 bash[27579]: audit 2023-08-28T01:40:38.965249+0000 mon.c (mon.2) 2621 : audit [DBG] from='client.? 172.21.15.6:0/2851859370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:39.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:39 smithi006 bash[33918]: audit 2023-08-28T01:40:38.965249+0000 mon.c (mon.2) 2621 : audit [DBG] from='client.? 172.21.15.6:0/2851859370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:39.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:39 smithi006 bash[27292]: audit 2023-08-28T01:40:38.965249+0000 mon.c (mon.2) 2621 : audit [DBG] from='client.? 172.21.15.6:0/2851859370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:40.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:39 smithi130 bash[51302]: debug 2023-08-28T01:40:39.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:39.826031+0000) 2023-08-28T01:40:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:40 smithi130 bash[27579]: cluster 2023-08-28T01:40:38.452335+0000 mgr.y (mgr.14901) 6661 : cluster [DBG] pgmap v6668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:40 smithi130 bash[27579]: audit 2023-08-28T01:40:39.716471+0000 mon.b (mon.1) 754 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:40:40.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:40 smithi006 bash[27292]: cluster 2023-08-28T01:40:38.452335+0000 mgr.y (mgr.14901) 6661 : cluster [DBG] pgmap v6668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:40 smithi006 bash[27292]: audit 2023-08-28T01:40:39.716471+0000 mon.b (mon.1) 754 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:40:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:40 smithi006 bash[33918]: cluster 2023-08-28T01:40:38.452335+0000 mgr.y (mgr.14901) 6661 : cluster [DBG] pgmap v6668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:40 smithi006 bash[33918]: audit 2023-08-28T01:40:39.716471+0000 mon.b (mon.1) 754 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:40:41.082 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:40:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:40:40] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:40:41.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:40 smithi130 bash[51302]: debug 2023-08-28T01:40:40.819+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:40.826305+0000) 2023-08-28T01:40:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:40:41.082Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:40:41.084Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:41 smithi130 bash[51302]: debug 2023-08-28T01:40:41.823+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:41.826616+0000) 2023-08-28T01:40:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:42 smithi130 bash[27579]: cluster 2023-08-28T01:40:40.453472+0000 mgr.y (mgr.14901) 6662 : cluster [DBG] pgmap v6669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:42 smithi130 bash[27579]: audit 2023-08-28T01:40:41.378288+0000 mon.a (mon.0) 4384 : audit [DBG] from='client.? 172.21.15.6:0/3734758846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:42.652 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:42 smithi006 bash[33918]: cluster 2023-08-28T01:40:40.453472+0000 mgr.y (mgr.14901) 6662 : cluster [DBG] pgmap v6669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:42.652 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:42 smithi006 bash[33918]: audit 2023-08-28T01:40:41.378288+0000 mon.a (mon.0) 4384 : audit [DBG] from='client.? 172.21.15.6:0/3734758846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:42.653 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:42 smithi006 bash[27292]: cluster 2023-08-28T01:40:40.453472+0000 mgr.y (mgr.14901) 6662 : cluster [DBG] pgmap v6669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:42.653 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:42 smithi006 bash[27292]: audit 2023-08-28T01:40:41.378288+0000 mon.a (mon.0) 4384 : audit [DBG] from='client.? 172.21.15.6:0/3734758846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:40:42.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:40:42.654Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:43.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:42 smithi130 bash[51302]: debug 2023-08-28T01:40:42.823+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:42.826908+0000) 2023-08-28T01:40:44.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:43 smithi130 bash[51302]: debug 2023-08-28T01:40:43.823+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:43.827216+0000) 2023-08-28T01:40:44.537 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:44 smithi006 bash[33918]: cluster 2023-08-28T01:40:42.454112+0000 mgr.y (mgr.14901) 6663 : cluster [DBG] pgmap v6670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:44.537 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:44 smithi006 bash[33918]: audit 2023-08-28T01:40:43.794299+0000 mon.a (mon.0) 4385 : audit [DBG] from='client.? 172.21.15.6:0/2188795566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:44.537 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:44 smithi006 bash[27292]: cluster 2023-08-28T01:40:42.454112+0000 mgr.y (mgr.14901) 6663 : cluster [DBG] pgmap v6670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:44.538 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:44 smithi006 bash[27292]: audit 2023-08-28T01:40:43.794299+0000 mon.a (mon.0) 4385 : audit [DBG] from='client.? 172.21.15.6:0/2188795566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:44 smithi130 bash[27579]: cluster 2023-08-28T01:40:42.454112+0000 mgr.y (mgr.14901) 6663 : cluster [DBG] pgmap v6670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:44 smithi130 bash[27579]: audit 2023-08-28T01:40:43.794299+0000 mon.a (mon.0) 4385 : audit [DBG] from='client.? 172.21.15.6:0/2188795566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:40:44.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:40:44.539Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:45.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:44 smithi130 bash[51302]: debug 2023-08-28T01:40:44.823+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:44.827389+0000) 2023-08-28T01:40:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:45 smithi130 bash[51302]: debug 2023-08-28T01:40:45.823+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:45.827588+0000) 2023-08-28T01:40:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:46 smithi130 bash[27579]: cluster 2023-08-28T01:40:44.455239+0000 mgr.y (mgr.14901) 6664 : cluster [DBG] pgmap v6671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:46 smithi130 bash[27579]: audit 2023-08-28T01:40:45.619665+0000 mon.a (mon.0) 4386 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:40:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:46 smithi130 bash[27579]: audit 2023-08-28T01:40:46.103973+0000 mon.a (mon.0) 4387 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:40:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:46 smithi130 bash[27579]: audit 2023-08-28T01:40:46.206916+0000 mon.c (mon.2) 2622 : audit [DBG] from='client.? 172.21.15.6:0/3291829396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:46.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:46 smithi006 bash[27292]: cluster 2023-08-28T01:40:44.455239+0000 mgr.y (mgr.14901) 6664 : cluster [DBG] pgmap v6671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:46.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:46 smithi006 bash[27292]: audit 2023-08-28T01:40:45.619665+0000 mon.a (mon.0) 4386 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:40:46.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:46 smithi006 bash[27292]: audit 2023-08-28T01:40:46.103973+0000 mon.a (mon.0) 4387 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:40:46.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:46 smithi006 bash[27292]: audit 2023-08-28T01:40:46.206916+0000 mon.c (mon.2) 2622 : audit [DBG] from='client.? 172.21.15.6:0/3291829396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:46.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:46 smithi006 bash[33918]: cluster 2023-08-28T01:40:44.455239+0000 mgr.y (mgr.14901) 6664 : cluster [DBG] pgmap v6671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:46.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:46 smithi006 bash[33918]: audit 2023-08-28T01:40:45.619665+0000 mon.a (mon.0) 4386 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:40:46.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:46 smithi006 bash[33918]: audit 2023-08-28T01:40:46.103973+0000 mon.a (mon.0) 4387 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:40:46.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:46 smithi006 bash[33918]: audit 2023-08-28T01:40:46.206916+0000 mon.c (mon.2) 2622 : audit [DBG] from='client.? 172.21.15.6:0/3291829396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:46 smithi130 bash[51302]: debug 2023-08-28T01:40:46.823+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:46.827856+0000) 2023-08-28T01:40:47.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:47 smithi130 bash[27579]: audit 2023-08-28T01:40:46.400785+0000 mon.b (mon.1) 755 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:40:47.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:47 smithi130 bash[27579]: audit 2023-08-28T01:40:46.403004+0000 mon.b (mon.1) 756 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:40:47.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:47 smithi130 bash[27579]: audit 2023-08-28T01:40:46.412909+0000 mon.a (mon.0) 4388 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:40:47.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:47 smithi006 bash[27292]: audit 2023-08-28T01:40:46.400785+0000 mon.b (mon.1) 755 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:40:47.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:47 smithi006 bash[27292]: audit 2023-08-28T01:40:46.403004+0000 mon.b (mon.1) 756 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:40:47.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:47 smithi006 bash[27292]: audit 2023-08-28T01:40:46.412909+0000 mon.a (mon.0) 4388 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:40:47.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:47 smithi006 bash[33918]: audit 2023-08-28T01:40:46.400785+0000 mon.b (mon.1) 755 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:40:47.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:47 smithi006 bash[33918]: audit 2023-08-28T01:40:46.403004+0000 mon.b (mon.1) 756 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:40:47.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:47 smithi006 bash[33918]: audit 2023-08-28T01:40:46.412909+0000 mon.a (mon.0) 4388 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:40:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:47 smithi130 bash[51302]: debug 2023-08-28T01:40:47.827+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:47.828121+0000) 2023-08-28T01:40:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:48 smithi130 bash[27579]: cluster 2023-08-28T01:40:46.455903+0000 mgr.y (mgr.14901) 6665 : cluster [DBG] pgmap v6672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:48.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:48 smithi006 bash[33918]: cluster 2023-08-28T01:40:46.455903+0000 mgr.y (mgr.14901) 6665 : cluster [DBG] pgmap v6672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:48 smithi006 bash[27292]: cluster 2023-08-28T01:40:46.455903+0000 mgr.y (mgr.14901) 6665 : cluster [DBG] pgmap v6672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:49.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:48 smithi130 bash[51302]: debug 2023-08-28T01:40:48.827+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:48.828382+0000) 2023-08-28T01:40:49.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:49 smithi130 bash[27579]: audit 2023-08-28T01:40:48.628554+0000 mon.c (mon.2) 2623 : audit [DBG] from='client.? 172.21.15.6:0/2892019396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:49.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:49 smithi006 bash[33918]: audit 2023-08-28T01:40:48.628554+0000 mon.c (mon.2) 2623 : audit [DBG] from='client.? 172.21.15.6:0/2892019396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:49.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:49 smithi006 bash[27292]: audit 2023-08-28T01:40:48.628554+0000 mon.c (mon.2) 2623 : audit [DBG] from='client.? 172.21.15.6:0/2892019396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:50.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:49 smithi130 bash[51302]: debug 2023-08-28T01:40:49.827+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:49.828697+0000) 2023-08-28T01:40:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:50 smithi130 bash[27579]: cluster 2023-08-28T01:40:48.456534+0000 mgr.y (mgr.14901) 6666 : cluster [DBG] pgmap v6673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:50.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:50 smithi006 bash[27292]: cluster 2023-08-28T01:40:48.456534+0000 mgr.y (mgr.14901) 6666 : cluster [DBG] pgmap v6673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:50.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:50 smithi006 bash[33918]: cluster 2023-08-28T01:40:48.456534+0000 mgr.y (mgr.14901) 6666 : cluster [DBG] pgmap v6673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:51.083 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:40:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:40:50] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:40:51.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:50 smithi130 bash[51302]: debug 2023-08-28T01:40:50.827+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:50.828991+0000) 2023-08-28T01:40:51.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:51 smithi006 bash[27292]: audit 2023-08-28T01:40:51.038407+0000 mon.c (mon.2) 2624 : audit [DBG] from='client.? 172.21.15.6:0/385945749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:51.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:51 smithi006 bash[33918]: audit 2023-08-28T01:40:51.038407+0000 mon.c (mon.2) 2624 : audit [DBG] from='client.? 172.21.15.6:0/385945749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:40:51.083Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:40:51.085Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:51.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:51 smithi130 bash[27579]: audit 2023-08-28T01:40:51.038407+0000 mon.c (mon.2) 2624 : audit [DBG] from='client.? 172.21.15.6:0/385945749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:52.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:51 smithi130 bash[51302]: debug 2023-08-28T01:40:51.827+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:51.829264+0000) 2023-08-28T01:40:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:52 smithi130 bash[27579]: cluster 2023-08-28T01:40:50.457675+0000 mgr.y (mgr.14901) 6667 : cluster [DBG] pgmap v6674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:52.653 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:52 smithi006 bash[27292]: cluster 2023-08-28T01:40:50.457675+0000 mgr.y (mgr.14901) 6667 : cluster [DBG] pgmap v6674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:52.654 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:52 smithi006 bash[33918]: cluster 2023-08-28T01:40:50.457675+0000 mgr.y (mgr.14901) 6667 : cluster [DBG] pgmap v6674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:40:52.653Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:40:52.655Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:53.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:52 smithi130 bash[51302]: debug 2023-08-28T01:40:52.827+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:52.829525+0000) 2023-08-28T01:40:54.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:53 smithi130 bash[51302]: debug 2023-08-28T01:40:53.823+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:53.829819+0000) 2023-08-28T01:40:54.539 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:54 smithi006 bash[33918]: cluster 2023-08-28T01:40:52.458346+0000 mgr.y (mgr.14901) 6668 : cluster [DBG] pgmap v6675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:54.539 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:54 smithi006 bash[33918]: audit 2023-08-28T01:40:53.466601+0000 mon.a (mon.0) 4389 : audit [DBG] from='client.? 172.21.15.6:0/1703735805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:54.539 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:54 smithi006 bash[27292]: cluster 2023-08-28T01:40:52.458346+0000 mgr.y (mgr.14901) 6668 : cluster [DBG] pgmap v6675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:54.539 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:54 smithi006 bash[27292]: audit 2023-08-28T01:40:53.466601+0000 mon.a (mon.0) 4389 : audit [DBG] from='client.? 172.21.15.6:0/1703735805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:54 smithi130 bash[27579]: cluster 2023-08-28T01:40:52.458346+0000 mgr.y (mgr.14901) 6668 : cluster [DBG] pgmap v6675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:54 smithi130 bash[27579]: audit 2023-08-28T01:40:53.466601+0000 mon.a (mon.0) 4389 : audit [DBG] from='client.? 172.21.15.6:0/1703735805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:40:54.538Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:40:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:40:54.540Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:40:55.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:54 smithi130 bash[51302]: debug 2023-08-28T01:40:54.822+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:54.830119+0000) 2023-08-28T01:40:56.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:55 smithi130 bash[51302]: debug 2023-08-28T01:40:55.826+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:55.830391+0000) 2023-08-28T01:40:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:56 smithi130 bash[27579]: cluster 2023-08-28T01:40:54.459453+0000 mgr.y (mgr.14901) 6669 : cluster [DBG] pgmap v6676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:56 smithi130 bash[27579]: audit 2023-08-28T01:40:55.885103+0000 mon.c (mon.2) 2625 : audit [DBG] from='client.? 172.21.15.6:0/1906369458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:56.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:56 smithi006 bash[27292]: cluster 2023-08-28T01:40:54.459453+0000 mgr.y (mgr.14901) 6669 : cluster [DBG] pgmap v6676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:56.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:56 smithi006 bash[27292]: audit 2023-08-28T01:40:55.885103+0000 mon.c (mon.2) 2625 : audit [DBG] from='client.? 172.21.15.6:0/1906369458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:56.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:56 smithi006 bash[33918]: cluster 2023-08-28T01:40:54.459453+0000 mgr.y (mgr.14901) 6669 : cluster [DBG] pgmap v6676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:56.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:56 smithi006 bash[33918]: audit 2023-08-28T01:40:55.885103+0000 mon.c (mon.2) 2625 : audit [DBG] from='client.? 172.21.15.6:0/1906369458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:57.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:56 smithi130 bash[51302]: debug 2023-08-28T01:40:56.822+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:56.830711+0000) 2023-08-28T01:40:58.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:57 smithi130 bash[51302]: debug 2023-08-28T01:40:57.826+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:57.830970+0000) 2023-08-28T01:40:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:58 smithi130 bash[27579]: cluster 2023-08-28T01:40:56.460117+0000 mgr.y (mgr.14901) 6670 : cluster [DBG] pgmap v6677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:58.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:58 smithi006 bash[27292]: cluster 2023-08-28T01:40:56.460117+0000 mgr.y (mgr.14901) 6670 : cluster [DBG] pgmap v6677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:58.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:58 smithi006 bash[33918]: cluster 2023-08-28T01:40:56.460117+0000 mgr.y (mgr.14901) 6670 : cluster [DBG] pgmap v6677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:40:59.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:58 smithi130 bash[51302]: debug 2023-08-28T01:40:58.826+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:58.831226+0000) 2023-08-28T01:40:59.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:40:59 smithi130 bash[27579]: audit 2023-08-28T01:40:58.308722+0000 mon.c (mon.2) 2626 : audit [DBG] from='client.? 172.21.15.6:0/2925324711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:59.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:40:59 smithi006 bash[33918]: audit 2023-08-28T01:40:58.308722+0000 mon.c (mon.2) 2626 : audit [DBG] from='client.? 172.21.15.6:0/2925324711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:59.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:40:59 smithi006 bash[27292]: audit 2023-08-28T01:40:58.308722+0000 mon.c (mon.2) 2626 : audit [DBG] from='client.? 172.21.15.6:0/2925324711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:40:59 smithi130 bash[51302]: debug 2023-08-28T01:40:59.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:59.831519+0000) 2023-08-28T01:41:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:00 smithi130 bash[27579]: cluster 2023-08-28T01:40:58.460791+0000 mgr.y (mgr.14901) 6671 : cluster [DBG] pgmap v6678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:00 smithi006 bash[27292]: cluster 2023-08-28T01:40:58.460791+0000 mgr.y (mgr.14901) 6671 : cluster [DBG] pgmap v6678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:00 smithi006 bash[33918]: cluster 2023-08-28T01:40:58.460791+0000 mgr.y (mgr.14901) 6671 : cluster [DBG] pgmap v6678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:01.084 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:41:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:41:00] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:41:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:00 smithi130 bash[51302]: debug 2023-08-28T01:41:00.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:00.831847+0000) 2023-08-28T01:41:01.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:01 smithi006 bash[27292]: audit 2023-08-28T01:41:00.721283+0000 mon.c (mon.2) 2627 : audit [DBG] from='client.? 172.21.15.6:0/2146054870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:01.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:01 smithi006 bash[33918]: audit 2023-08-28T01:41:00.721283+0000 mon.c (mon.2) 2627 : audit [DBG] from='client.? 172.21.15.6:0/2146054870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:41:01.083Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:41:01.085Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:01.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:01 smithi130 bash[27579]: audit 2023-08-28T01:41:00.721283+0000 mon.c (mon.2) 2627 : audit [DBG] from='client.? 172.21.15.6:0/2146054870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:02.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:01 smithi130 bash[51302]: debug 2023-08-28T01:41:01.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:01.832134+0000) 2023-08-28T01:41:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:02 smithi130 bash[27579]: cluster 2023-08-28T01:41:00.461849+0000 mgr.y (mgr.14901) 6672 : cluster [DBG] pgmap v6679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:02.654 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:02 smithi006 bash[33918]: cluster 2023-08-28T01:41:00.461849+0000 mgr.y (mgr.14901) 6672 : cluster [DBG] pgmap v6679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:02.655 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:02 smithi006 bash[27292]: cluster 2023-08-28T01:41:00.461849+0000 mgr.y (mgr.14901) 6672 : cluster [DBG] pgmap v6679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:41:02.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:41:02.656Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:03.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:02 smithi130 bash[51302]: debug 2023-08-28T01:41:02.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:02.832411+0000) 2023-08-28T01:41:03.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:03 smithi130 bash[27579]: audit 2023-08-28T01:41:03.140133+0000 mon.a (mon.0) 4390 : audit [DBG] from='client.? 172.21.15.6:0/2555224387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:03.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:03 smithi006 bash[27292]: audit 2023-08-28T01:41:03.140133+0000 mon.a (mon.0) 4390 : audit [DBG] from='client.? 172.21.15.6:0/2555224387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:03.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:03 smithi006 bash[33918]: audit 2023-08-28T01:41:03.140133+0000 mon.a (mon.0) 4390 : audit [DBG] from='client.? 172.21.15.6:0/2555224387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:04.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:03 smithi130 bash[51302]: debug 2023-08-28T01:41:03.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:03.832697+0000) 2023-08-28T01:41:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:04 smithi130 bash[27579]: cluster 2023-08-28T01:41:02.462587+0000 mgr.y (mgr.14901) 6673 : cluster [DBG] pgmap v6680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:04.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:04 smithi006 bash[27292]: cluster 2023-08-28T01:41:02.462587+0000 mgr.y (mgr.14901) 6673 : cluster [DBG] pgmap v6680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:04.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:04 smithi006 bash[33918]: cluster 2023-08-28T01:41:02.462587+0000 mgr.y (mgr.14901) 6673 : cluster [DBG] pgmap v6680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:41:04.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:41:04.541Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:05.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:04 smithi130 bash[51302]: debug 2023-08-28T01:41:04.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:04.833004+0000) 2023-08-28T01:41:06.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:05 smithi130 bash[51302]: debug 2023-08-28T01:41:05.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:05.833282+0000) 2023-08-28T01:41:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:06 smithi130 bash[27579]: cluster 2023-08-28T01:41:04.463800+0000 mgr.y (mgr.14901) 6674 : cluster [DBG] pgmap v6681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:06 smithi130 bash[27579]: audit 2023-08-28T01:41:05.555825+0000 mon.c (mon.2) 2628 : audit [DBG] from='client.? 172.21.15.6:0/1383208405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:06.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:06 smithi006 bash[27292]: cluster 2023-08-28T01:41:04.463800+0000 mgr.y (mgr.14901) 6674 : cluster [DBG] pgmap v6681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:06 smithi006 bash[27292]: audit 2023-08-28T01:41:05.555825+0000 mon.c (mon.2) 2628 : audit [DBG] from='client.? 172.21.15.6:0/1383208405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:06 smithi006 bash[33918]: cluster 2023-08-28T01:41:04.463800+0000 mgr.y (mgr.14901) 6674 : cluster [DBG] pgmap v6681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:06 smithi006 bash[33918]: audit 2023-08-28T01:41:05.555825+0000 mon.c (mon.2) 2628 : audit [DBG] from='client.? 172.21.15.6:0/1383208405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:07.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:06 smithi130 bash[51302]: debug 2023-08-28T01:41:06.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:06.833553+0000) 2023-08-28T01:41:08.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:07 smithi130 bash[51302]: debug 2023-08-28T01:41:07.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:07.833824+0000) 2023-08-28T01:41:08.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:08 smithi006 bash[33918]: cluster 2023-08-28T01:41:06.464609+0000 mgr.y (mgr.14901) 6675 : cluster [DBG] pgmap v6682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:08.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:08 smithi006 bash[33918]: audit 2023-08-28T01:41:07.975467+0000 mon.a (mon.0) 4391 : audit [DBG] from='client.? 172.21.15.6:0/821445068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:08.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:08 smithi006 bash[27292]: cluster 2023-08-28T01:41:06.464609+0000 mgr.y (mgr.14901) 6675 : cluster [DBG] pgmap v6682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:08.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:08 smithi006 bash[27292]: audit 2023-08-28T01:41:07.975467+0000 mon.a (mon.0) 4391 : audit [DBG] from='client.? 172.21.15.6:0/821445068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:08 smithi130 bash[27579]: cluster 2023-08-28T01:41:06.464609+0000 mgr.y (mgr.14901) 6675 : cluster [DBG] pgmap v6682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:08 smithi130 bash[27579]: audit 2023-08-28T01:41:07.975467+0000 mon.a (mon.0) 4391 : audit [DBG] from='client.? 172.21.15.6:0/821445068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:09.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:08 smithi130 bash[51302]: debug 2023-08-28T01:41:08.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:08.834101+0000) 2023-08-28T01:41:10.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:09 smithi130 bash[51302]: debug 2023-08-28T01:41:09.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:09.834417+0000) 2023-08-28T01:41:10.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:10 smithi006 bash[27292]: cluster 2023-08-28T01:41:08.465456+0000 mgr.y (mgr.14901) 6676 : cluster [DBG] pgmap v6683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:10.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:10 smithi006 bash[33918]: cluster 2023-08-28T01:41:08.465456+0000 mgr.y (mgr.14901) 6676 : cluster [DBG] pgmap v6683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:10 smithi130 bash[27579]: cluster 2023-08-28T01:41:08.465456+0000 mgr.y (mgr.14901) 6676 : cluster [DBG] pgmap v6683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:11.085 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:41:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:41:10] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:41:11.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:10 smithi130 bash[51302]: debug 2023-08-28T01:41:10.830+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:10.834655+0000) 2023-08-28T01:41:11.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:11 smithi006 bash[27292]: audit 2023-08-28T01:41:10.405076+0000 mon.c (mon.2) 2629 : audit [DBG] from='client.? 172.21.15.6:0/2072741086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:11.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:11 smithi006 bash[33918]: audit 2023-08-28T01:41:10.405076+0000 mon.c (mon.2) 2629 : audit [DBG] from='client.? 172.21.15.6:0/2072741086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:41:11.085Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:41:11.087Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:11.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:11 smithi130 bash[27579]: audit 2023-08-28T01:41:10.405076+0000 mon.c (mon.2) 2629 : audit [DBG] from='client.? 172.21.15.6:0/2072741086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:12.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:11 smithi130 bash[51302]: debug 2023-08-28T01:41:11.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:11.834944+0000) 2023-08-28T01:41:12.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:12 smithi006 bash[27292]: cluster 2023-08-28T01:41:10.466421+0000 mgr.y (mgr.14901) 6677 : cluster [DBG] pgmap v6684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:12.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:12 smithi006 bash[33918]: cluster 2023-08-28T01:41:10.466421+0000 mgr.y (mgr.14901) 6677 : cluster [DBG] pgmap v6684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:12 smithi130 bash[27579]: cluster 2023-08-28T01:41:10.466421+0000 mgr.y (mgr.14901) 6677 : cluster [DBG] pgmap v6684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:41:12.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:41:12.657Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:13.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:12 smithi130 bash[51302]: debug 2023-08-28T01:41:12.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:12.835241+0000) 2023-08-28T01:41:13.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:13 smithi006 bash[27292]: audit 2023-08-28T01:41:12.816627+0000 mon.c (mon.2) 2630 : audit [DBG] from='client.? 172.21.15.6:0/765959472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:13.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:13 smithi006 bash[33918]: audit 2023-08-28T01:41:12.816627+0000 mon.c (mon.2) 2630 : audit [DBG] from='client.? 172.21.15.6:0/765959472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:13.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:13 smithi130 bash[27579]: audit 2023-08-28T01:41:12.816627+0000 mon.c (mon.2) 2630 : audit [DBG] from='client.? 172.21.15.6:0/765959472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:14.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:13 smithi130 bash[51302]: debug 2023-08-28T01:41:13.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:13.835515+0000) 2023-08-28T01:41:14.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:14 smithi006 bash[33918]: cluster 2023-08-28T01:41:12.467032+0000 mgr.y (mgr.14901) 6678 : cluster [DBG] pgmap v6685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:14 smithi006 bash[27292]: cluster 2023-08-28T01:41:12.467032+0000 mgr.y (mgr.14901) 6678 : cluster [DBG] pgmap v6685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:14 smithi130 bash[27579]: cluster 2023-08-28T01:41:12.467032+0000 mgr.y (mgr.14901) 6678 : cluster [DBG] pgmap v6685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:41:14.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:41:14.542Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:14 smithi130 bash[51302]: debug 2023-08-28T01:41:14.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:14.835805+0000) 2023-08-28T01:41:16.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:15 smithi130 bash[51302]: debug 2023-08-28T01:41:15.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:15.836100+0000) 2023-08-28T01:41:16.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:16 smithi006 bash[27292]: cluster 2023-08-28T01:41:14.468065+0000 mgr.y (mgr.14901) 6679 : cluster [DBG] pgmap v6686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:16.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:16 smithi006 bash[27292]: audit 2023-08-28T01:41:15.224453+0000 mon.c (mon.2) 2631 : audit [DBG] from='client.? 172.21.15.6:0/2824988425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:16.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:16 smithi006 bash[33918]: cluster 2023-08-28T01:41:14.468065+0000 mgr.y (mgr.14901) 6679 : cluster [DBG] pgmap v6686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:16.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:16 smithi006 bash[33918]: audit 2023-08-28T01:41:15.224453+0000 mon.c (mon.2) 2631 : audit [DBG] from='client.? 172.21.15.6:0/2824988425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:16 smithi130 bash[27579]: cluster 2023-08-28T01:41:14.468065+0000 mgr.y (mgr.14901) 6679 : cluster [DBG] pgmap v6686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:16 smithi130 bash[27579]: audit 2023-08-28T01:41:15.224453+0000 mon.c (mon.2) 2631 : audit [DBG] from='client.? 172.21.15.6:0/2824988425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:17.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:16 smithi130 bash[51302]: debug 2023-08-28T01:41:16.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:16.836403+0000) 2023-08-28T01:41:18.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:17 smithi130 bash[51302]: debug 2023-08-28T01:41:17.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:17.836710+0000) 2023-08-28T01:41:18.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:18 smithi006 bash[27292]: cluster 2023-08-28T01:41:16.468736+0000 mgr.y (mgr.14901) 6680 : cluster [DBG] pgmap v6687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:18.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:18 smithi006 bash[27292]: audit 2023-08-28T01:41:17.636205+0000 mon.a (mon.0) 4392 : audit [DBG] from='client.? 172.21.15.6:0/115989384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:18.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:18 smithi006 bash[33918]: cluster 2023-08-28T01:41:16.468736+0000 mgr.y (mgr.14901) 6680 : cluster [DBG] pgmap v6687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:18.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:18 smithi006 bash[33918]: audit 2023-08-28T01:41:17.636205+0000 mon.a (mon.0) 4392 : audit [DBG] from='client.? 172.21.15.6:0/115989384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:18 smithi130 bash[27579]: cluster 2023-08-28T01:41:16.468736+0000 mgr.y (mgr.14901) 6680 : cluster [DBG] pgmap v6687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:18 smithi130 bash[27579]: audit 2023-08-28T01:41:17.636205+0000 mon.a (mon.0) 4392 : audit [DBG] from='client.? 172.21.15.6:0/115989384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:19.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:18 smithi130 bash[51302]: debug 2023-08-28T01:41:18.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:18.837015+0000) 2023-08-28T01:41:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:19 smithi130 bash[51302]: debug 2023-08-28T01:41:19.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:19.837361+0000) 2023-08-28T01:41:20.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:20 smithi006 bash[27292]: cluster 2023-08-28T01:41:18.469342+0000 mgr.y (mgr.14901) 6681 : cluster [DBG] pgmap v6688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:20.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:20 smithi006 bash[27292]: audit 2023-08-28T01:41:20.056478+0000 mon.c (mon.2) 2632 : audit [DBG] from='client.? 172.21.15.6:0/1344861217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:20.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:20 smithi006 bash[33918]: cluster 2023-08-28T01:41:18.469342+0000 mgr.y (mgr.14901) 6681 : cluster [DBG] pgmap v6688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:20.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:20 smithi006 bash[33918]: audit 2023-08-28T01:41:20.056478+0000 mon.c (mon.2) 2632 : audit [DBG] from='client.? 172.21.15.6:0/1344861217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:20 smithi130 bash[27579]: cluster 2023-08-28T01:41:18.469342+0000 mgr.y (mgr.14901) 6681 : cluster [DBG] pgmap v6688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:20 smithi130 bash[27579]: audit 2023-08-28T01:41:20.056478+0000 mon.c (mon.2) 2632 : audit [DBG] from='client.? 172.21.15.6:0/1344861217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:21.086 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:41:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:41:20] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:41:21.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:20 smithi130 bash[51302]: debug 2023-08-28T01:41:20.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:20.837683+0000) 2023-08-28T01:41:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:41:21.086Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:41:21.088Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:22.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:21 smithi130 bash[51302]: debug 2023-08-28T01:41:21.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:21.837992+0000) 2023-08-28T01:41:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:22 smithi130 bash[27579]: cluster 2023-08-28T01:41:20.470449+0000 mgr.y (mgr.14901) 6682 : cluster [DBG] pgmap v6689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:22.656 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:22 smithi006 bash[27292]: cluster 2023-08-28T01:41:20.470449+0000 mgr.y (mgr.14901) 6682 : cluster [DBG] pgmap v6689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:22.656 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:22 smithi006 bash[33918]: cluster 2023-08-28T01:41:20.470449+0000 mgr.y (mgr.14901) 6682 : cluster [DBG] pgmap v6689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:41:22.656Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:41:22.658Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:23.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:22 smithi130 bash[51302]: debug 2023-08-28T01:41:22.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:22.838261+0000) 2023-08-28T01:41:23.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:23 smithi130 bash[27579]: audit 2023-08-28T01:41:22.478370+0000 mon.c (mon.2) 2633 : audit [DBG] from='client.? 172.21.15.6:0/629070627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:23.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:23 smithi006 bash[27292]: audit 2023-08-28T01:41:22.478370+0000 mon.c (mon.2) 2633 : audit [DBG] from='client.? 172.21.15.6:0/629070627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:23.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:23 smithi006 bash[33918]: audit 2023-08-28T01:41:22.478370+0000 mon.c (mon.2) 2633 : audit [DBG] from='client.? 172.21.15.6:0/629070627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:24.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:23 smithi130 bash[51302]: debug 2023-08-28T01:41:23.834+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:23.838533+0000) 2023-08-28T01:41:24.541 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:24 smithi006 bash[27292]: cluster 2023-08-28T01:41:22.471016+0000 mgr.y (mgr.14901) 6683 : cluster [DBG] pgmap v6690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:24.541 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:24 smithi006 bash[33918]: cluster 2023-08-28T01:41:22.471016+0000 mgr.y (mgr.14901) 6683 : cluster [DBG] pgmap v6690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:24 smithi130 bash[27579]: cluster 2023-08-28T01:41:22.471016+0000 mgr.y (mgr.14901) 6683 : cluster [DBG] pgmap v6690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:41:24.540Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:41:24.543Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:25.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:24 smithi130 bash[51302]: debug 2023-08-28T01:41:24.838+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:24.838835+0000) 2023-08-28T01:41:25.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:25 smithi130 bash[27579]: audit 2023-08-28T01:41:24.886048+0000 mon.a (mon.0) 4393 : audit [DBG] from='client.? 172.21.15.6:0/17007071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:25.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:25 smithi006 bash[33918]: audit 2023-08-28T01:41:24.886048+0000 mon.a (mon.0) 4393 : audit [DBG] from='client.? 172.21.15.6:0/17007071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:25.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:25 smithi006 bash[27292]: audit 2023-08-28T01:41:24.886048+0000 mon.a (mon.0) 4393 : audit [DBG] from='client.? 172.21.15.6:0/17007071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:26.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:25 smithi130 bash[51302]: debug 2023-08-28T01:41:25.838+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:25.839100+0000) 2023-08-28T01:41:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:26 smithi130 bash[27579]: cluster 2023-08-28T01:41:24.472077+0000 mgr.y (mgr.14901) 6684 : cluster [DBG] pgmap v6691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:26.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:26 smithi006 bash[33918]: cluster 2023-08-28T01:41:24.472077+0000 mgr.y (mgr.14901) 6684 : cluster [DBG] pgmap v6691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:26 smithi006 bash[27292]: cluster 2023-08-28T01:41:24.472077+0000 mgr.y (mgr.14901) 6684 : cluster [DBG] pgmap v6691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:27.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:26 smithi130 bash[51302]: debug 2023-08-28T01:41:26.838+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:26.839328+0000) 2023-08-28T01:41:28.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:27 smithi130 bash[51302]: debug 2023-08-28T01:41:27.838+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:27.839596+0000) 2023-08-28T01:41:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:28 smithi130 bash[27579]: cluster 2023-08-28T01:41:26.472739+0000 mgr.y (mgr.14901) 6685 : cluster [DBG] pgmap v6692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:28 smithi130 bash[27579]: audit 2023-08-28T01:41:27.310029+0000 mon.c (mon.2) 2634 : audit [DBG] from='client.? 172.21.15.6:0/1586875100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:28.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:28 smithi006 bash[33918]: cluster 2023-08-28T01:41:26.472739+0000 mgr.y (mgr.14901) 6685 : cluster [DBG] pgmap v6692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:28 smithi006 bash[33918]: audit 2023-08-28T01:41:27.310029+0000 mon.c (mon.2) 2634 : audit [DBG] from='client.? 172.21.15.6:0/1586875100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:28 smithi006 bash[27292]: cluster 2023-08-28T01:41:26.472739+0000 mgr.y (mgr.14901) 6685 : cluster [DBG] pgmap v6692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:28 smithi006 bash[27292]: audit 2023-08-28T01:41:27.310029+0000 mon.c (mon.2) 2634 : audit [DBG] from='client.? 172.21.15.6:0/1586875100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:28 smithi130 bash[51302]: debug 2023-08-28T01:41:28.838+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:28.839918+0000) 2023-08-28T01:41:30.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:29 smithi130 bash[51302]: debug 2023-08-28T01:41:29.838+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:29.840254+0000) 2023-08-28T01:41:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:30 smithi130 bash[27579]: cluster 2023-08-28T01:41:28.473371+0000 mgr.y (mgr.14901) 6686 : cluster [DBG] pgmap v6693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:30 smithi130 bash[27579]: audit 2023-08-28T01:41:29.730495+0000 mon.c (mon.2) 2635 : audit [DBG] from='client.? 172.21.15.6:0/915660621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:30.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:30 smithi006 bash[27292]: cluster 2023-08-28T01:41:28.473371+0000 mgr.y (mgr.14901) 6686 : cluster [DBG] pgmap v6693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:30 smithi006 bash[27292]: audit 2023-08-28T01:41:29.730495+0000 mon.c (mon.2) 2635 : audit [DBG] from='client.? 172.21.15.6:0/915660621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:30 smithi006 bash[33918]: cluster 2023-08-28T01:41:28.473371+0000 mgr.y (mgr.14901) 6686 : cluster [DBG] pgmap v6693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:30 smithi006 bash[33918]: audit 2023-08-28T01:41:29.730495+0000 mon.c (mon.2) 2635 : audit [DBG] from='client.? 172.21.15.6:0/915660621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:31.087 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:41:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:41:30] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:41:31.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:30 smithi130 bash[51302]: debug 2023-08-28T01:41:30.838+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:30.840604+0000) 2023-08-28T01:41:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:41:31.087Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:41:31.089Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:32.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:31 smithi130 bash[51302]: debug 2023-08-28T01:41:31.838+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:31.840870+0000) 2023-08-28T01:41:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:32 smithi130 bash[27579]: cluster 2023-08-28T01:41:30.474437+0000 mgr.y (mgr.14901) 6687 : cluster [DBG] pgmap v6694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:32 smithi130 bash[27579]: audit 2023-08-28T01:41:32.140037+0000 mon.c (mon.2) 2636 : audit [DBG] from='client.? 172.21.15.6:0/854243765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:32.656 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:32 smithi006 bash[27292]: cluster 2023-08-28T01:41:30.474437+0000 mgr.y (mgr.14901) 6687 : cluster [DBG] pgmap v6694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:32.656 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:32 smithi006 bash[27292]: audit 2023-08-28T01:41:32.140037+0000 mon.c (mon.2) 2636 : audit [DBG] from='client.? 172.21.15.6:0/854243765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:32.657 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:32 smithi006 bash[33918]: cluster 2023-08-28T01:41:30.474437+0000 mgr.y (mgr.14901) 6687 : cluster [DBG] pgmap v6694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:32.657 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:32 smithi006 bash[33918]: audit 2023-08-28T01:41:32.140037+0000 mon.c (mon.2) 2636 : audit [DBG] from='client.? 172.21.15.6:0/854243765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:41:32.656Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:41:32.658Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:33.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:32 smithi130 bash[51302]: debug 2023-08-28T01:41:32.838+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:32.841149+0000) 2023-08-28T01:41:34.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:33 smithi130 bash[51302]: debug 2023-08-28T01:41:33.838+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:33.841432+0000) 2023-08-28T01:41:34.543 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:34 smithi006 bash[27292]: cluster 2023-08-28T01:41:32.475099+0000 mgr.y (mgr.14901) 6688 : cluster [DBG] pgmap v6695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:34.543 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:34 smithi006 bash[33918]: cluster 2023-08-28T01:41:32.475099+0000 mgr.y (mgr.14901) 6688 : cluster [DBG] pgmap v6695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:34.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:34 smithi130 bash[27579]: cluster 2023-08-28T01:41:32.475099+0000 mgr.y (mgr.14901) 6688 : cluster [DBG] pgmap v6695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:41:34.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:41:34.543Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:35.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:34 smithi130 bash[51302]: debug 2023-08-28T01:41:34.838+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:34.841736+0000) 2023-08-28T01:41:35.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:35 smithi130 bash[27579]: audit 2023-08-28T01:41:34.563401+0000 mon.a (mon.0) 4394 : audit [DBG] from='client.? 172.21.15.6:0/781638616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:35.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:35 smithi006 bash[27292]: audit 2023-08-28T01:41:34.563401+0000 mon.a (mon.0) 4394 : audit [DBG] from='client.? 172.21.15.6:0/781638616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:35.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:35 smithi006 bash[33918]: audit 2023-08-28T01:41:34.563401+0000 mon.a (mon.0) 4394 : audit [DBG] from='client.? 172.21.15.6:0/781638616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:36.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:35 smithi130 bash[51302]: debug 2023-08-28T01:41:35.838+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:35.842033+0000) 2023-08-28T01:41:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:36 smithi130 bash[27579]: cluster 2023-08-28T01:41:34.476306+0000 mgr.y (mgr.14901) 6689 : cluster [DBG] pgmap v6696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:36.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:36 smithi006 bash[27292]: cluster 2023-08-28T01:41:34.476306+0000 mgr.y (mgr.14901) 6689 : cluster [DBG] pgmap v6696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:36.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:36 smithi006 bash[33918]: cluster 2023-08-28T01:41:34.476306+0000 mgr.y (mgr.14901) 6689 : cluster [DBG] pgmap v6696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:37.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:36 smithi130 bash[51302]: debug 2023-08-28T01:41:36.838+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:36.842324+0000) 2023-08-28T01:41:37.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:37 smithi130 bash[27579]: audit 2023-08-28T01:41:36.979489+0000 mon.c (mon.2) 2637 : audit [DBG] from='client.? 172.21.15.6:0/1967748362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:37.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:37 smithi006 bash[27292]: audit 2023-08-28T01:41:36.979489+0000 mon.c (mon.2) 2637 : audit [DBG] from='client.? 172.21.15.6:0/1967748362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:37.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:37 smithi006 bash[33918]: audit 2023-08-28T01:41:36.979489+0000 mon.c (mon.2) 2637 : audit [DBG] from='client.? 172.21.15.6:0/1967748362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:37 smithi130 bash[51302]: debug 2023-08-28T01:41:37.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:37.842659+0000) 2023-08-28T01:41:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:38 smithi130 bash[27579]: cluster 2023-08-28T01:41:36.477089+0000 mgr.y (mgr.14901) 6690 : cluster [DBG] pgmap v6697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:38 smithi006 bash[33918]: cluster 2023-08-28T01:41:36.477089+0000 mgr.y (mgr.14901) 6690 : cluster [DBG] pgmap v6697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:38 smithi006 bash[27292]: cluster 2023-08-28T01:41:36.477089+0000 mgr.y (mgr.14901) 6690 : cluster [DBG] pgmap v6697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:39.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:38 smithi130 bash[51302]: debug 2023-08-28T01:41:38.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:38.842990+0000) 2023-08-28T01:41:40.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:39 smithi130 bash[51302]: debug 2023-08-28T01:41:39.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:39.843204+0000) 2023-08-28T01:41:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:40 smithi130 bash[27579]: cluster 2023-08-28T01:41:38.477709+0000 mgr.y (mgr.14901) 6691 : cluster [DBG] pgmap v6698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:40 smithi130 bash[27579]: audit 2023-08-28T01:41:39.397843+0000 mon.c (mon.2) 2638 : audit [DBG] from='client.? 172.21.15.6:0/1387793886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:40.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:40 smithi006 bash[33918]: cluster 2023-08-28T01:41:38.477709+0000 mgr.y (mgr.14901) 6691 : cluster [DBG] pgmap v6698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:40 smithi006 bash[33918]: audit 2023-08-28T01:41:39.397843+0000 mon.c (mon.2) 2638 : audit [DBG] from='client.? 172.21.15.6:0/1387793886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:40 smithi006 bash[27292]: cluster 2023-08-28T01:41:38.477709+0000 mgr.y (mgr.14901) 6691 : cluster [DBG] pgmap v6698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:40 smithi006 bash[27292]: audit 2023-08-28T01:41:39.397843+0000 mon.c (mon.2) 2638 : audit [DBG] from='client.? 172.21.15.6:0/1387793886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:41.088 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:41:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:41:40] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:41:41.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:40 smithi130 bash[51302]: debug 2023-08-28T01:41:40.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:40.843525+0000) 2023-08-28T01:41:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:41:41.088Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:41:41.090Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:41 smithi130 bash[51302]: debug 2023-08-28T01:41:41.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:41.843799+0000) 2023-08-28T01:41:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:42 smithi130 bash[27579]: cluster 2023-08-28T01:41:40.478861+0000 mgr.y (mgr.14901) 6692 : cluster [DBG] pgmap v6699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:42 smithi130 bash[27579]: audit 2023-08-28T01:41:41.821139+0000 mon.a (mon.0) 4395 : audit [DBG] from='client.? 172.21.15.6:0/1009040136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:42.657 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:42 smithi006 bash[33918]: cluster 2023-08-28T01:41:40.478861+0000 mgr.y (mgr.14901) 6692 : cluster [DBG] pgmap v6699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:42.658 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:42 smithi006 bash[33918]: audit 2023-08-28T01:41:41.821139+0000 mon.a (mon.0) 4395 : audit [DBG] from='client.? 172.21.15.6:0/1009040136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:42.658 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:42 smithi006 bash[27292]: cluster 2023-08-28T01:41:40.478861+0000 mgr.y (mgr.14901) 6692 : cluster [DBG] pgmap v6699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:42.658 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:42 smithi006 bash[27292]: audit 2023-08-28T01:41:41.821139+0000 mon.a (mon.0) 4395 : audit [DBG] from='client.? 172.21.15.6:0/1009040136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:41:42.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:41:42.659Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:43.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:42 smithi130 bash[51302]: debug 2023-08-28T01:41:42.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:42.844015+0000) 2023-08-28T01:41:44.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:43 smithi130 bash[51302]: debug 2023-08-28T01:41:43.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:43.844269+0000) 2023-08-28T01:41:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:44 smithi130 bash[27579]: cluster 2023-08-28T01:41:42.479483+0000 mgr.y (mgr.14901) 6693 : cluster [DBG] pgmap v6700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:44 smithi130 bash[27579]: audit 2023-08-28T01:41:44.240020+0000 mon.a (mon.0) 4396 : audit [DBG] from='client.? 172.21.15.6:0/2762119118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:44.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:44 smithi006 bash[27292]: cluster 2023-08-28T01:41:42.479483+0000 mgr.y (mgr.14901) 6693 : cluster [DBG] pgmap v6700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:44.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:44 smithi006 bash[27292]: audit 2023-08-28T01:41:44.240020+0000 mon.a (mon.0) 4396 : audit [DBG] from='client.? 172.21.15.6:0/2762119118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:44.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:44 smithi006 bash[33918]: cluster 2023-08-28T01:41:42.479483+0000 mgr.y (mgr.14901) 6693 : cluster [DBG] pgmap v6700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:44.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:44 smithi006 bash[33918]: audit 2023-08-28T01:41:44.240020+0000 mon.a (mon.0) 4396 : audit [DBG] from='client.? 172.21.15.6:0/2762119118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:41:44.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:41:44.544Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:45.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:44 smithi130 bash[51302]: debug 2023-08-28T01:41:44.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:44.844580+0000) 2023-08-28T01:41:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:45 smithi130 bash[51302]: debug 2023-08-28T01:41:45.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:45.844851+0000) 2023-08-28T01:41:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:46 smithi130 bash[27579]: cluster 2023-08-28T01:41:44.480558+0000 mgr.y (mgr.14901) 6694 : cluster [DBG] pgmap v6701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:46.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:46 smithi006 bash[27292]: cluster 2023-08-28T01:41:44.480558+0000 mgr.y (mgr.14901) 6694 : cluster [DBG] pgmap v6701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:46.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:46 smithi006 bash[33918]: cluster 2023-08-28T01:41:44.480558+0000 mgr.y (mgr.14901) 6694 : cluster [DBG] pgmap v6701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:46 smithi130 bash[51302]: debug 2023-08-28T01:41:46.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:46.845124+0000) 2023-08-28T01:41:47.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:47 smithi130 bash[27579]: audit 2023-08-28T01:41:46.425871+0000 mon.b (mon.1) 757 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:41:47.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:47 smithi130 bash[27579]: audit 2023-08-28T01:41:46.669997+0000 mon.a (mon.0) 4397 : audit [DBG] from='client.? 172.21.15.6:0/1706430230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:47.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:47 smithi006 bash[27292]: audit 2023-08-28T01:41:46.425871+0000 mon.b (mon.1) 757 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:41:47.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:47 smithi006 bash[27292]: audit 2023-08-28T01:41:46.669997+0000 mon.a (mon.0) 4397 : audit [DBG] from='client.? 172.21.15.6:0/1706430230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:47.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:47 smithi006 bash[33918]: audit 2023-08-28T01:41:46.425871+0000 mon.b (mon.1) 757 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:41:47.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:47 smithi006 bash[33918]: audit 2023-08-28T01:41:46.669997+0000 mon.a (mon.0) 4397 : audit [DBG] from='client.? 172.21.15.6:0/1706430230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:47 smithi130 bash[51302]: debug 2023-08-28T01:41:47.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:47.845420+0000) 2023-08-28T01:41:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:48 smithi130 bash[27579]: cluster 2023-08-28T01:41:46.481143+0000 mgr.y (mgr.14901) 6695 : cluster [DBG] pgmap v6702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:48.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:48 smithi006 bash[27292]: cluster 2023-08-28T01:41:46.481143+0000 mgr.y (mgr.14901) 6695 : cluster [DBG] pgmap v6702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:48.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:48 smithi006 bash[33918]: cluster 2023-08-28T01:41:46.481143+0000 mgr.y (mgr.14901) 6695 : cluster [DBG] pgmap v6702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:49.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:48 smithi130 bash[51302]: debug 2023-08-28T01:41:48.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:48.845696+0000) 2023-08-28T01:41:49.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:49 smithi130 bash[27579]: audit 2023-08-28T01:41:49.090263+0000 mon.a (mon.0) 4398 : audit [DBG] from='client.? 172.21.15.6:0/1940150694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:49.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:49 smithi006 bash[27292]: audit 2023-08-28T01:41:49.090263+0000 mon.a (mon.0) 4398 : audit [DBG] from='client.? 172.21.15.6:0/1940150694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:49.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:49 smithi006 bash[33918]: audit 2023-08-28T01:41:49.090263+0000 mon.a (mon.0) 4398 : audit [DBG] from='client.? 172.21.15.6:0/1940150694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:50.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:49 smithi130 bash[51302]: debug 2023-08-28T01:41:49.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:49.846004+0000) 2023-08-28T01:41:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:50 smithi130 bash[27579]: cluster 2023-08-28T01:41:48.481738+0000 mgr.y (mgr.14901) 6696 : cluster [DBG] pgmap v6703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:50.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:50 smithi006 bash[27292]: cluster 2023-08-28T01:41:48.481738+0000 mgr.y (mgr.14901) 6696 : cluster [DBG] pgmap v6703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:50.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:50 smithi006 bash[33918]: cluster 2023-08-28T01:41:48.481738+0000 mgr.y (mgr.14901) 6696 : cluster [DBG] pgmap v6703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:51.089 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:41:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:41:50] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:41:51.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:50 smithi130 bash[51302]: debug 2023-08-28T01:41:50.841+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:50.846230+0000) 2023-08-28T01:41:51.390 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:41:51.089Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:51.390 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:41:51.091Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:51.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:51 smithi006 bash[27292]: cluster 2023-08-28T01:41:50.482724+0000 mgr.y (mgr.14901) 6697 : cluster [DBG] pgmap v6704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:51.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:51 smithi006 bash[33918]: cluster 2023-08-28T01:41:50.482724+0000 mgr.y (mgr.14901) 6697 : cluster [DBG] pgmap v6704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:51.845 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:51 smithi130 bash[27579]: cluster 2023-08-28T01:41:50.482724+0000 mgr.y (mgr.14901) 6697 : cluster [DBG] pgmap v6704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:52.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:51 smithi130 bash[51302]: debug 2023-08-28T01:41:51.841+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:51.846477+0000) 2023-08-28T01:41:52.658 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:52 smithi006 bash[33918]: audit 2023-08-28T01:41:51.523951+0000 mon.c (mon.2) 2639 : audit [DBG] from='client.? 172.21.15.6:0/1097013350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:52.658 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:52 smithi006 bash[33918]: audit 2023-08-28T01:41:52.310787+0000 mon.a (mon.0) 4399 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:41:52.659 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:52 smithi006 bash[27292]: audit 2023-08-28T01:41:51.523951+0000 mon.c (mon.2) 2639 : audit [DBG] from='client.? 172.21.15.6:0/1097013350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:52.659 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:52 smithi006 bash[27292]: audit 2023-08-28T01:41:52.310787+0000 mon.a (mon.0) 4399 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:41:52.845 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:52 smithi130 bash[27579]: audit 2023-08-28T01:41:51.523951+0000 mon.c (mon.2) 2639 : audit [DBG] from='client.? 172.21.15.6:0/1097013350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:52.845 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:52 smithi130 bash[27579]: audit 2023-08-28T01:41:52.310787+0000 mon.a (mon.0) 4399 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:41:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:41:52.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:41:52.660Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:53.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:52 smithi130 bash[51302]: debug 2023-08-28T01:41:52.841+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:52.846749+0000) 2023-08-28T01:41:53.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:53 smithi006 bash[27292]: cluster 2023-08-28T01:41:52.483225+0000 mgr.y (mgr.14901) 6698 : cluster [DBG] pgmap v6705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:53.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:53 smithi006 bash[27292]: audit 2023-08-28T01:41:52.684334+0000 mon.a (mon.0) 4400 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:41:53.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:53 smithi006 bash[27292]: audit 2023-08-28T01:41:52.957375+0000 mon.b (mon.1) 758 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:41:53.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:53 smithi006 bash[27292]: audit 2023-08-28T01:41:52.959602+0000 mon.b (mon.1) 759 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:41:53.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:53 smithi006 bash[27292]: audit 2023-08-28T01:41:52.967706+0000 mon.a (mon.0) 4401 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:41:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:53 smithi006 bash[33918]: cluster 2023-08-28T01:41:52.483225+0000 mgr.y (mgr.14901) 6698 : cluster [DBG] pgmap v6705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:53 smithi006 bash[33918]: audit 2023-08-28T01:41:52.684334+0000 mon.a (mon.0) 4400 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:41:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:53 smithi006 bash[33918]: audit 2023-08-28T01:41:52.957375+0000 mon.b (mon.1) 758 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:41:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:53 smithi006 bash[33918]: audit 2023-08-28T01:41:52.959602+0000 mon.b (mon.1) 759 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:41:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:53 smithi006 bash[33918]: audit 2023-08-28T01:41:52.967706+0000 mon.a (mon.0) 4401 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:41:54.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:53 smithi130 bash[51302]: debug 2023-08-28T01:41:53.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:53.847064+0000) 2023-08-28T01:41:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:53 smithi130 bash[27579]: cluster 2023-08-28T01:41:52.483225+0000 mgr.y (mgr.14901) 6698 : cluster [DBG] pgmap v6705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:53 smithi130 bash[27579]: audit 2023-08-28T01:41:52.684334+0000 mon.a (mon.0) 4400 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:41:54.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:53 smithi130 bash[27579]: audit 2023-08-28T01:41:52.957375+0000 mon.b (mon.1) 758 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:41:54.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:53 smithi130 bash[27579]: audit 2023-08-28T01:41:52.959602+0000 mon.b (mon.1) 759 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:41:54.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:53 smithi130 bash[27579]: audit 2023-08-28T01:41:52.967706+0000 mon.a (mon.0) 4401 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:41:54.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:54 smithi006 bash[27292]: audit 2023-08-28T01:41:53.940201+0000 mon.c (mon.2) 2640 : audit [DBG] from='client.? 172.21.15.6:0/498711562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:54.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:54 smithi006 bash[33918]: audit 2023-08-28T01:41:53.940201+0000 mon.c (mon.2) 2640 : audit [DBG] from='client.? 172.21.15.6:0/498711562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:41:54.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:41:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:41:54.544Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:41:55.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:54 smithi130 bash[27579]: audit 2023-08-28T01:41:53.940201+0000 mon.c (mon.2) 2640 : audit [DBG] from='client.? 172.21.15.6:0/498711562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:55.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:54 smithi130 bash[51302]: debug 2023-08-28T01:41:54.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:54.847338+0000) 2023-08-28T01:41:55.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:55 smithi006 bash[33918]: cluster 2023-08-28T01:41:54.484246+0000 mgr.y (mgr.14901) 6699 : cluster [DBG] pgmap v6706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:55.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:55 smithi006 bash[27292]: cluster 2023-08-28T01:41:54.484246+0000 mgr.y (mgr.14901) 6699 : cluster [DBG] pgmap v6706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:56.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:55 smithi130 bash[27579]: cluster 2023-08-28T01:41:54.484246+0000 mgr.y (mgr.14901) 6699 : cluster [DBG] pgmap v6706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:56.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:55 smithi130 bash[51302]: debug 2023-08-28T01:41:55.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:55.847610+0000) 2023-08-28T01:41:56.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:56 smithi006 bash[33918]: audit 2023-08-28T01:41:56.360896+0000 mon.a (mon.0) 4402 : audit [DBG] from='client.? 172.21.15.6:0/728030453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:56.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:56 smithi006 bash[27292]: audit 2023-08-28T01:41:56.360896+0000 mon.a (mon.0) 4402 : audit [DBG] from='client.? 172.21.15.6:0/728030453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:57.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:56 smithi130 bash[51302]: debug 2023-08-28T01:41:56.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:56.847880+0000) 2023-08-28T01:41:57.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:56 smithi130 bash[27579]: audit 2023-08-28T01:41:56.360896+0000 mon.a (mon.0) 4402 : audit [DBG] from='client.? 172.21.15.6:0/728030453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:58.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:57 smithi130 bash[51302]: debug 2023-08-28T01:41:57.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:57.848173+0000) 2023-08-28T01:41:58.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:58 smithi006 bash[33918]: cluster 2023-08-28T01:41:56.484899+0000 mgr.y (mgr.14901) 6700 : cluster [DBG] pgmap v6707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:58.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:58 smithi006 bash[27292]: cluster 2023-08-28T01:41:56.484899+0000 mgr.y (mgr.14901) 6700 : cluster [DBG] pgmap v6707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:58 smithi130 bash[27579]: cluster 2023-08-28T01:41:56.484899+0000 mgr.y (mgr.14901) 6700 : cluster [DBG] pgmap v6707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:41:59.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:58 smithi130 bash[51302]: debug 2023-08-28T01:41:58.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:58.848521+0000) 2023-08-28T01:41:59.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:41:59 smithi006 bash[33918]: audit 2023-08-28T01:41:58.773881+0000 mon.c (mon.2) 2641 : audit [DBG] from='client.? 172.21.15.6:0/2866673044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:59.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:41:59 smithi006 bash[27292]: audit 2023-08-28T01:41:58.773881+0000 mon.c (mon.2) 2641 : audit [DBG] from='client.? 172.21.15.6:0/2866673044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:59.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:41:59 smithi130 bash[27579]: audit 2023-08-28T01:41:58.773881+0000 mon.c (mon.2) 2641 : audit [DBG] from='client.? 172.21.15.6:0/2866673044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:41:59 smithi130 bash[51302]: debug 2023-08-28T01:41:59.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:59.848822+0000) 2023-08-28T01:42:00.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:00 smithi006 bash[27292]: cluster 2023-08-28T01:41:58.485375+0000 mgr.y (mgr.14901) 6701 : cluster [DBG] pgmap v6708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:00.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:00 smithi006 bash[33918]: cluster 2023-08-28T01:41:58.485375+0000 mgr.y (mgr.14901) 6701 : cluster [DBG] pgmap v6708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:00 smithi130 bash[27579]: cluster 2023-08-28T01:41:58.485375+0000 mgr.y (mgr.14901) 6701 : cluster [DBG] pgmap v6708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:01.089 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:42:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:42:00] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:42:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:00 smithi130 bash[51302]: debug 2023-08-28T01:42:00.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:00.849111+0000) 2023-08-28T01:42:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:42:01.089Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:42:01.091Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:02.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:01 smithi130 bash[51302]: debug 2023-08-28T01:42:01.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:01.849414+0000) 2023-08-28T01:42:02.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:02 smithi006 bash[27292]: cluster 2023-08-28T01:42:00.486380+0000 mgr.y (mgr.14901) 6702 : cluster [DBG] pgmap v6709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:02.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:02 smithi006 bash[27292]: audit 2023-08-28T01:42:01.192318+0000 mon.a (mon.0) 4403 : audit [DBG] from='client.? 172.21.15.6:0/1449077205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:02.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:02 smithi006 bash[33918]: cluster 2023-08-28T01:42:00.486380+0000 mgr.y (mgr.14901) 6702 : cluster [DBG] pgmap v6709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:02.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:02 smithi006 bash[33918]: audit 2023-08-28T01:42:01.192318+0000 mon.a (mon.0) 4403 : audit [DBG] from='client.? 172.21.15.6:0/1449077205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:02 smithi130 bash[27579]: cluster 2023-08-28T01:42:00.486380+0000 mgr.y (mgr.14901) 6702 : cluster [DBG] pgmap v6709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:02 smithi130 bash[27579]: audit 2023-08-28T01:42:01.192318+0000 mon.a (mon.0) 4403 : audit [DBG] from='client.? 172.21.15.6:0/1449077205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:42:02.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:42:02.660Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:03.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:02 smithi130 bash[51302]: debug 2023-08-28T01:42:02.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:02.849686+0000) 2023-08-28T01:42:04.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:03 smithi130 bash[51302]: debug 2023-08-28T01:42:03.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:03.849952+0000) 2023-08-28T01:42:04.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:04 smithi006 bash[27292]: cluster 2023-08-28T01:42:02.486928+0000 mgr.y (mgr.14901) 6703 : cluster [DBG] pgmap v6710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:04.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:04 smithi006 bash[27292]: audit 2023-08-28T01:42:03.606353+0000 mon.a (mon.0) 4404 : audit [DBG] from='client.? 172.21.15.6:0/2155853056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:04.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:04 smithi006 bash[33918]: cluster 2023-08-28T01:42:02.486928+0000 mgr.y (mgr.14901) 6703 : cluster [DBG] pgmap v6710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:04.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:04 smithi006 bash[33918]: audit 2023-08-28T01:42:03.606353+0000 mon.a (mon.0) 4404 : audit [DBG] from='client.? 172.21.15.6:0/2155853056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:04 smithi130 bash[27579]: cluster 2023-08-28T01:42:02.486928+0000 mgr.y (mgr.14901) 6703 : cluster [DBG] pgmap v6710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:04 smithi130 bash[27579]: audit 2023-08-28T01:42:03.606353+0000 mon.a (mon.0) 4404 : audit [DBG] from='client.? 172.21.15.6:0/2155853056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:42:04.543Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:42:04.545Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:05.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:04 smithi130 bash[51302]: debug 2023-08-28T01:42:04.845+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:04.850228+0000) 2023-08-28T01:42:06.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:05 smithi130 bash[51302]: debug 2023-08-28T01:42:05.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:05.850540+0000) 2023-08-28T01:42:06.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:06 smithi006 bash[27292]: cluster 2023-08-28T01:42:04.488031+0000 mgr.y (mgr.14901) 6704 : cluster [DBG] pgmap v6711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:06.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:06 smithi006 bash[27292]: audit 2023-08-28T01:42:06.020037+0000 mon.a (mon.0) 4405 : audit [DBG] from='client.? 172.21.15.6:0/534585276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:06.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:06 smithi006 bash[33918]: cluster 2023-08-28T01:42:04.488031+0000 mgr.y (mgr.14901) 6704 : cluster [DBG] pgmap v6711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:06.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:06 smithi006 bash[33918]: audit 2023-08-28T01:42:06.020037+0000 mon.a (mon.0) 4405 : audit [DBG] from='client.? 172.21.15.6:0/534585276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:06 smithi130 bash[27579]: cluster 2023-08-28T01:42:04.488031+0000 mgr.y (mgr.14901) 6704 : cluster [DBG] pgmap v6711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:06 smithi130 bash[27579]: audit 2023-08-28T01:42:06.020037+0000 mon.a (mon.0) 4405 : audit [DBG] from='client.? 172.21.15.6:0/534585276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:07.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:06 smithi130 bash[51302]: debug 2023-08-28T01:42:06.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:06.850823+0000) 2023-08-28T01:42:08.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:07 smithi130 bash[51302]: debug 2023-08-28T01:42:07.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:07.851062+0000) 2023-08-28T01:42:08.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:08 smithi006 bash[33918]: cluster 2023-08-28T01:42:06.488784+0000 mgr.y (mgr.14901) 6705 : cluster [DBG] pgmap v6712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:08.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:08 smithi006 bash[27292]: cluster 2023-08-28T01:42:06.488784+0000 mgr.y (mgr.14901) 6705 : cluster [DBG] pgmap v6712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:08 smithi130 bash[27579]: cluster 2023-08-28T01:42:06.488784+0000 mgr.y (mgr.14901) 6705 : cluster [DBG] pgmap v6712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:09.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:08 smithi130 bash[51302]: debug 2023-08-28T01:42:08.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:08.851387+0000) 2023-08-28T01:42:09.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:09 smithi130 bash[27579]: audit 2023-08-28T01:42:08.442810+0000 mon.a (mon.0) 4406 : audit [DBG] from='client.? 172.21.15.6:0/1992391812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:09.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:09 smithi006 bash[27292]: audit 2023-08-28T01:42:08.442810+0000 mon.a (mon.0) 4406 : audit [DBG] from='client.? 172.21.15.6:0/1992391812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:09.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:09 smithi006 bash[33918]: audit 2023-08-28T01:42:08.442810+0000 mon.a (mon.0) 4406 : audit [DBG] from='client.? 172.21.15.6:0/1992391812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:10.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:09 smithi130 bash[51302]: debug 2023-08-28T01:42:09.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:09.851668+0000) 2023-08-28T01:42:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:10 smithi130 bash[27579]: cluster 2023-08-28T01:42:08.489418+0000 mgr.y (mgr.14901) 6706 : cluster [DBG] pgmap v6713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:10 smithi006 bash[33918]: cluster 2023-08-28T01:42:08.489418+0000 mgr.y (mgr.14901) 6706 : cluster [DBG] pgmap v6713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:10 smithi006 bash[27292]: cluster 2023-08-28T01:42:08.489418+0000 mgr.y (mgr.14901) 6706 : cluster [DBG] pgmap v6713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:11.090 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:42:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:42:10] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:42:11.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:10 smithi130 bash[51302]: debug 2023-08-28T01:42:10.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:10.851953+0000) 2023-08-28T01:42:11.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:11 smithi006 bash[27292]: audit 2023-08-28T01:42:10.856052+0000 mon.c (mon.2) 2642 : audit [DBG] from='client.? 172.21.15.6:0/1838316259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:11.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:11 smithi006 bash[33918]: audit 2023-08-28T01:42:10.856052+0000 mon.c (mon.2) 2642 : audit [DBG] from='client.? 172.21.15.6:0/1838316259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:42:11.090Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:42:11.092Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:11.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:11 smithi130 bash[27579]: audit 2023-08-28T01:42:10.856052+0000 mon.c (mon.2) 2642 : audit [DBG] from='client.? 172.21.15.6:0/1838316259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:12.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:11 smithi130 bash[51302]: debug 2023-08-28T01:42:11.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:11.852209+0000) 2023-08-28T01:42:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:12 smithi130 bash[27579]: cluster 2023-08-28T01:42:10.490478+0000 mgr.y (mgr.14901) 6707 : cluster [DBG] pgmap v6714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:12.660 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:12 smithi006 bash[33918]: cluster 2023-08-28T01:42:10.490478+0000 mgr.y (mgr.14901) 6707 : cluster [DBG] pgmap v6714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:12.660 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:12 smithi006 bash[27292]: cluster 2023-08-28T01:42:10.490478+0000 mgr.y (mgr.14901) 6707 : cluster [DBG] pgmap v6714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:42:12.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:42:12.662Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:13.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:12 smithi130 bash[51302]: debug 2023-08-28T01:42:12.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:12.852434+0000) 2023-08-28T01:42:14.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:13 smithi130 bash[51302]: debug 2023-08-28T01:42:13.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:13.852720+0000) 2023-08-28T01:42:14.544 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:14 smithi006 bash[33918]: cluster 2023-08-28T01:42:12.491136+0000 mgr.y (mgr.14901) 6708 : cluster [DBG] pgmap v6715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:14.544 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:14 smithi006 bash[33918]: audit 2023-08-28T01:42:13.273903+0000 mon.c (mon.2) 2643 : audit [DBG] from='client.? 172.21.15.6:0/1365270819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:14.545 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:14 smithi006 bash[27292]: cluster 2023-08-28T01:42:12.491136+0000 mgr.y (mgr.14901) 6708 : cluster [DBG] pgmap v6715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:14.545 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:14 smithi006 bash[27292]: audit 2023-08-28T01:42:13.273903+0000 mon.c (mon.2) 2643 : audit [DBG] from='client.? 172.21.15.6:0/1365270819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:14 smithi130 bash[27579]: cluster 2023-08-28T01:42:12.491136+0000 mgr.y (mgr.14901) 6708 : cluster [DBG] pgmap v6715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:14 smithi130 bash[27579]: audit 2023-08-28T01:42:13.273903+0000 mon.c (mon.2) 2643 : audit [DBG] from='client.? 172.21.15.6:0/1365270819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:42:14.544Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:42:14.546Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:14 smithi130 bash[51302]: debug 2023-08-28T01:42:14.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:14.853044+0000) 2023-08-28T01:42:16.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:15 smithi130 bash[51302]: debug 2023-08-28T01:42:15.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:15.853326+0000) 2023-08-28T01:42:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:16 smithi130 bash[27579]: cluster 2023-08-28T01:42:14.492238+0000 mgr.y (mgr.14901) 6709 : cluster [DBG] pgmap v6716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:16 smithi130 bash[27579]: audit 2023-08-28T01:42:15.687859+0000 mon.c (mon.2) 2644 : audit [DBG] from='client.? 172.21.15.6:0/4052444807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:16.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:16 smithi006 bash[33918]: cluster 2023-08-28T01:42:14.492238+0000 mgr.y (mgr.14901) 6709 : cluster [DBG] pgmap v6716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:16 smithi006 bash[33918]: audit 2023-08-28T01:42:15.687859+0000 mon.c (mon.2) 2644 : audit [DBG] from='client.? 172.21.15.6:0/4052444807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:16 smithi006 bash[27292]: cluster 2023-08-28T01:42:14.492238+0000 mgr.y (mgr.14901) 6709 : cluster [DBG] pgmap v6716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:16 smithi006 bash[27292]: audit 2023-08-28T01:42:15.687859+0000 mon.c (mon.2) 2644 : audit [DBG] from='client.? 172.21.15.6:0/4052444807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:17.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:16 smithi130 bash[51302]: debug 2023-08-28T01:42:16.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:16.853599+0000) 2023-08-28T01:42:18.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:17 smithi130 bash[51302]: debug 2023-08-28T01:42:17.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:17.853900+0000) 2023-08-28T01:42:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:18 smithi130 bash[27579]: cluster 2023-08-28T01:42:16.492962+0000 mgr.y (mgr.14901) 6710 : cluster [DBG] pgmap v6717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:18 smithi130 bash[27579]: audit 2023-08-28T01:42:18.100928+0000 mon.c (mon.2) 2645 : audit [DBG] from='client.? 172.21.15.6:0/1177569041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:18.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:18 smithi006 bash[33918]: cluster 2023-08-28T01:42:16.492962+0000 mgr.y (mgr.14901) 6710 : cluster [DBG] pgmap v6717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:18.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:18 smithi006 bash[33918]: audit 2023-08-28T01:42:18.100928+0000 mon.c (mon.2) 2645 : audit [DBG] from='client.? 172.21.15.6:0/1177569041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:18.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:18 smithi006 bash[27292]: cluster 2023-08-28T01:42:16.492962+0000 mgr.y (mgr.14901) 6710 : cluster [DBG] pgmap v6717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:18.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:18 smithi006 bash[27292]: audit 2023-08-28T01:42:18.100928+0000 mon.c (mon.2) 2645 : audit [DBG] from='client.? 172.21.15.6:0/1177569041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:19.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:18 smithi130 bash[51302]: debug 2023-08-28T01:42:18.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:18.854235+0000) 2023-08-28T01:42:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:19 smithi130 bash[51302]: debug 2023-08-28T01:42:19.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:19.854498+0000) 2023-08-28T01:42:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:20 smithi130 bash[27579]: cluster 2023-08-28T01:42:18.493574+0000 mgr.y (mgr.14901) 6711 : cluster [DBG] pgmap v6718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:20.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:20 smithi006 bash[27292]: cluster 2023-08-28T01:42:18.493574+0000 mgr.y (mgr.14901) 6711 : cluster [DBG] pgmap v6718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:20 smithi006 bash[33918]: cluster 2023-08-28T01:42:18.493574+0000 mgr.y (mgr.14901) 6711 : cluster [DBG] pgmap v6718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:21.091 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:42:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:42:20] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:42:21.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:20 smithi130 bash[51302]: debug 2023-08-28T01:42:20.849+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:20.854783+0000) 2023-08-28T01:42:21.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:21 smithi006 bash[33918]: audit 2023-08-28T01:42:20.528781+0000 mon.c (mon.2) 2646 : audit [DBG] from='client.? 172.21.15.6:0/3402891294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:21.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:21 smithi006 bash[27292]: audit 2023-08-28T01:42:20.528781+0000 mon.c (mon.2) 2646 : audit [DBG] from='client.? 172.21.15.6:0/3402891294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:42:21.091Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:42:21.093Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:21.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:21 smithi130 bash[27579]: audit 2023-08-28T01:42:20.528781+0000 mon.c (mon.2) 2646 : audit [DBG] from='client.? 172.21.15.6:0/3402891294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:22.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:21 smithi130 bash[51302]: debug 2023-08-28T01:42:21.853+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:21.855067+0000) 2023-08-28T01:42:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:22 smithi130 bash[27579]: cluster 2023-08-28T01:42:20.494885+0000 mgr.y (mgr.14901) 6712 : cluster [DBG] pgmap v6719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:22.661 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:22 smithi006 bash[33918]: cluster 2023-08-28T01:42:20.494885+0000 mgr.y (mgr.14901) 6712 : cluster [DBG] pgmap v6719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:22.661 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:22 smithi006 bash[27292]: cluster 2023-08-28T01:42:20.494885+0000 mgr.y (mgr.14901) 6712 : cluster [DBG] pgmap v6719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:42:22.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:42:22.662Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:23.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:22 smithi130 bash[51302]: debug 2023-08-28T01:42:22.853+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:22.855295+0000) 2023-08-28T01:42:23.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:23 smithi130 bash[27579]: audit 2023-08-28T01:42:22.941293+0000 mon.a (mon.0) 4407 : audit [DBG] from='client.? 172.21.15.6:0/2099061216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:23.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:23 smithi006 bash[27292]: audit 2023-08-28T01:42:22.941293+0000 mon.a (mon.0) 4407 : audit [DBG] from='client.? 172.21.15.6:0/2099061216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:23.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:23 smithi006 bash[33918]: audit 2023-08-28T01:42:22.941293+0000 mon.a (mon.0) 4407 : audit [DBG] from='client.? 172.21.15.6:0/2099061216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:24.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:23 smithi130 bash[51302]: debug 2023-08-28T01:42:23.853+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:23.855590+0000) 2023-08-28T01:42:24.545 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:24 smithi006 bash[27292]: cluster 2023-08-28T01:42:22.495385+0000 mgr.y (mgr.14901) 6713 : cluster [DBG] pgmap v6720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:24.545 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:24 smithi006 bash[33918]: cluster 2023-08-28T01:42:22.495385+0000 mgr.y (mgr.14901) 6713 : cluster [DBG] pgmap v6720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:24 smithi130 bash[27579]: cluster 2023-08-28T01:42:22.495385+0000 mgr.y (mgr.14901) 6713 : cluster [DBG] pgmap v6720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:42:24.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:42:24.547Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:25.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:24 smithi130 bash[51302]: debug 2023-08-28T01:42:24.853+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:24.855899+0000) 2023-08-28T01:42:26.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:25 smithi130 bash[51302]: debug 2023-08-28T01:42:25.853+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:25.856177+0000) 2023-08-28T01:42:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:26 smithi130 bash[27579]: cluster 2023-08-28T01:42:24.496516+0000 mgr.y (mgr.14901) 6714 : cluster [DBG] pgmap v6721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:26 smithi130 bash[27579]: audit 2023-08-28T01:42:25.356060+0000 mon.a (mon.0) 4408 : audit [DBG] from='client.? 172.21.15.6:0/991585452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:26.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:26 smithi006 bash[27292]: cluster 2023-08-28T01:42:24.496516+0000 mgr.y (mgr.14901) 6714 : cluster [DBG] pgmap v6721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:26 smithi006 bash[27292]: audit 2023-08-28T01:42:25.356060+0000 mon.a (mon.0) 4408 : audit [DBG] from='client.? 172.21.15.6:0/991585452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:26 smithi006 bash[33918]: cluster 2023-08-28T01:42:24.496516+0000 mgr.y (mgr.14901) 6714 : cluster [DBG] pgmap v6721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:26 smithi006 bash[33918]: audit 2023-08-28T01:42:25.356060+0000 mon.a (mon.0) 4408 : audit [DBG] from='client.? 172.21.15.6:0/991585452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:27.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:26 smithi130 bash[51302]: debug 2023-08-28T01:42:26.853+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:26.856411+0000) 2023-08-28T01:42:28.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:27 smithi130 bash[51302]: debug 2023-08-28T01:42:27.853+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:27.856720+0000) 2023-08-28T01:42:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:28 smithi130 bash[27579]: cluster 2023-08-28T01:42:26.497172+0000 mgr.y (mgr.14901) 6715 : cluster [DBG] pgmap v6722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:28 smithi130 bash[27579]: audit 2023-08-28T01:42:27.769401+0000 mon.a (mon.0) 4409 : audit [DBG] from='client.? 172.21.15.6:0/1341629880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:28.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:28 smithi006 bash[27292]: cluster 2023-08-28T01:42:26.497172+0000 mgr.y (mgr.14901) 6715 : cluster [DBG] pgmap v6722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:28 smithi006 bash[27292]: audit 2023-08-28T01:42:27.769401+0000 mon.a (mon.0) 4409 : audit [DBG] from='client.? 172.21.15.6:0/1341629880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:28 smithi006 bash[33918]: cluster 2023-08-28T01:42:26.497172+0000 mgr.y (mgr.14901) 6715 : cluster [DBG] pgmap v6722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:28 smithi006 bash[33918]: audit 2023-08-28T01:42:27.769401+0000 mon.a (mon.0) 4409 : audit [DBG] from='client.? 172.21.15.6:0/1341629880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:28 smithi130 bash[51302]: debug 2023-08-28T01:42:28.853+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:28.857065+0000) 2023-08-28T01:42:30.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:29 smithi130 bash[51302]: debug 2023-08-28T01:42:29.853+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:29.857370+0000) 2023-08-28T01:42:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:30 smithi130 bash[27579]: cluster 2023-08-28T01:42:28.497765+0000 mgr.y (mgr.14901) 6716 : cluster [DBG] pgmap v6723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:30 smithi130 bash[27579]: audit 2023-08-28T01:42:30.180395+0000 mon.c (mon.2) 2647 : audit [DBG] from='client.? 172.21.15.6:0/1697368883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:30 smithi006 bash[33918]: cluster 2023-08-28T01:42:28.497765+0000 mgr.y (mgr.14901) 6716 : cluster [DBG] pgmap v6723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:30 smithi006 bash[33918]: audit 2023-08-28T01:42:30.180395+0000 mon.c (mon.2) 2647 : audit [DBG] from='client.? 172.21.15.6:0/1697368883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:30 smithi006 bash[27292]: cluster 2023-08-28T01:42:28.497765+0000 mgr.y (mgr.14901) 6716 : cluster [DBG] pgmap v6723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:30 smithi006 bash[27292]: audit 2023-08-28T01:42:30.180395+0000 mon.c (mon.2) 2647 : audit [DBG] from='client.? 172.21.15.6:0/1697368883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:31.092 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:42:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:42:30] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:42:31.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:30 smithi130 bash[51302]: debug 2023-08-28T01:42:30.853+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:30.857715+0000) 2023-08-28T01:42:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:42:31.092Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:42:31.094Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:32.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:31 smithi130 bash[51302]: debug 2023-08-28T01:42:31.857+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:31.858043+0000) 2023-08-28T01:42:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:32 smithi130 bash[27579]: cluster 2023-08-28T01:42:30.498612+0000 mgr.y (mgr.14901) 6717 : cluster [DBG] pgmap v6724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:32.661 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:32 smithi006 bash[33918]: cluster 2023-08-28T01:42:30.498612+0000 mgr.y (mgr.14901) 6717 : cluster [DBG] pgmap v6724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:32.662 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:32 smithi006 bash[27292]: cluster 2023-08-28T01:42:30.498612+0000 mgr.y (mgr.14901) 6717 : cluster [DBG] pgmap v6724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:42:32.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:42:32.664Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:33.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:32 smithi130 bash[51302]: debug 2023-08-28T01:42:32.853+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:32.858359+0000) 2023-08-28T01:42:33.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:33 smithi130 bash[27579]: audit 2023-08-28T01:42:32.597248+0000 mon.c (mon.2) 2648 : audit [DBG] from='client.? 172.21.15.6:0/898067725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:33.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:33 smithi006 bash[33918]: audit 2023-08-28T01:42:32.597248+0000 mon.c (mon.2) 2648 : audit [DBG] from='client.? 172.21.15.6:0/898067725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:33.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:33 smithi006 bash[27292]: audit 2023-08-28T01:42:32.597248+0000 mon.c (mon.2) 2648 : audit [DBG] from='client.? 172.21.15.6:0/898067725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:34.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:33 smithi130 bash[51302]: debug 2023-08-28T01:42:33.857+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:33.858654+0000) 2023-08-28T01:42:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:34 smithi130 bash[27579]: cluster 2023-08-28T01:42:32.499228+0000 mgr.y (mgr.14901) 6718 : cluster [DBG] pgmap v6725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:34.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:34 smithi006 bash[27292]: cluster 2023-08-28T01:42:32.499228+0000 mgr.y (mgr.14901) 6718 : cluster [DBG] pgmap v6725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:34.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:34 smithi006 bash[33918]: cluster 2023-08-28T01:42:32.499228+0000 mgr.y (mgr.14901) 6718 : cluster [DBG] pgmap v6725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:42:34.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:42:34.547Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:35.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:34 smithi130 bash[51302]: debug 2023-08-28T01:42:34.857+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:34.858949+0000) 2023-08-28T01:42:35.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:35 smithi130 bash[27579]: audit 2023-08-28T01:42:35.012710+0000 mon.a (mon.0) 4410 : audit [DBG] from='client.? 172.21.15.6:0/2834743280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:35.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:35 smithi006 bash[27292]: audit 2023-08-28T01:42:35.012710+0000 mon.a (mon.0) 4410 : audit [DBG] from='client.? 172.21.15.6:0/2834743280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:35.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:35 smithi006 bash[33918]: audit 2023-08-28T01:42:35.012710+0000 mon.a (mon.0) 4410 : audit [DBG] from='client.? 172.21.15.6:0/2834743280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:36.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:35 smithi130 bash[51302]: debug 2023-08-28T01:42:35.857+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:35.859224+0000) 2023-08-28T01:42:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:36 smithi130 bash[27579]: cluster 2023-08-28T01:42:34.500362+0000 mgr.y (mgr.14901) 6719 : cluster [DBG] pgmap v6726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:36.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:36 smithi006 bash[27292]: cluster 2023-08-28T01:42:34.500362+0000 mgr.y (mgr.14901) 6719 : cluster [DBG] pgmap v6726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:36.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:36 smithi006 bash[33918]: cluster 2023-08-28T01:42:34.500362+0000 mgr.y (mgr.14901) 6719 : cluster [DBG] pgmap v6726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:37.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:36 smithi130 bash[51302]: debug 2023-08-28T01:42:36.857+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:36.859524+0000) 2023-08-28T01:42:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:37 smithi130 bash[51302]: debug 2023-08-28T01:42:37.857+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:37.859797+0000) 2023-08-28T01:42:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:38 smithi130 bash[27579]: cluster 2023-08-28T01:42:36.501030+0000 mgr.y (mgr.14901) 6720 : cluster [DBG] pgmap v6727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:38 smithi130 bash[27579]: audit 2023-08-28T01:42:37.425109+0000 mon.c (mon.2) 2649 : audit [DBG] from='client.? 172.21.15.6:0/3717967945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:38 smithi006 bash[33918]: cluster 2023-08-28T01:42:36.501030+0000 mgr.y (mgr.14901) 6720 : cluster [DBG] pgmap v6727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:38 smithi006 bash[33918]: audit 2023-08-28T01:42:37.425109+0000 mon.c (mon.2) 2649 : audit [DBG] from='client.? 172.21.15.6:0/3717967945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:38 smithi006 bash[27292]: cluster 2023-08-28T01:42:36.501030+0000 mgr.y (mgr.14901) 6720 : cluster [DBG] pgmap v6727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:38 smithi006 bash[27292]: audit 2023-08-28T01:42:37.425109+0000 mon.c (mon.2) 2649 : audit [DBG] from='client.? 172.21.15.6:0/3717967945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:39.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:38 smithi130 bash[51302]: debug 2023-08-28T01:42:38.857+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:38.860154+0000) 2023-08-28T01:42:40.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:39 smithi130 bash[51302]: debug 2023-08-28T01:42:39.857+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:39.860463+0000) 2023-08-28T01:42:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:40 smithi130 bash[27579]: cluster 2023-08-28T01:42:38.501559+0000 mgr.y (mgr.14901) 6721 : cluster [DBG] pgmap v6728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:40 smithi130 bash[27579]: audit 2023-08-28T01:42:39.841257+0000 mon.a (mon.0) 4411 : audit [DBG] from='client.? 172.21.15.6:0/3636906778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:40.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:40 smithi006 bash[27292]: cluster 2023-08-28T01:42:38.501559+0000 mgr.y (mgr.14901) 6721 : cluster [DBG] pgmap v6728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:40 smithi006 bash[27292]: audit 2023-08-28T01:42:39.841257+0000 mon.a (mon.0) 4411 : audit [DBG] from='client.? 172.21.15.6:0/3636906778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:40 smithi006 bash[33918]: cluster 2023-08-28T01:42:38.501559+0000 mgr.y (mgr.14901) 6721 : cluster [DBG] pgmap v6728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:40 smithi006 bash[33918]: audit 2023-08-28T01:42:39.841257+0000 mon.a (mon.0) 4411 : audit [DBG] from='client.? 172.21.15.6:0/3636906778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:41.093 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:42:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:42:40] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:42:41.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:40 smithi130 bash[51302]: debug 2023-08-28T01:42:40.857+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:40.860782+0000) 2023-08-28T01:42:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:42:41.093Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:42:41.095Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:41 smithi130 bash[51302]: debug 2023-08-28T01:42:41.857+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:41.861061+0000) 2023-08-28T01:42:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:42 smithi130 bash[27579]: cluster 2023-08-28T01:42:40.502657+0000 mgr.y (mgr.14901) 6722 : cluster [DBG] pgmap v6729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:42 smithi130 bash[27579]: audit 2023-08-28T01:42:42.253180+0000 mon.c (mon.2) 2650 : audit [DBG] from='client.? 172.21.15.6:0/3463464122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:42.663 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:42 smithi006 bash[27292]: cluster 2023-08-28T01:42:40.502657+0000 mgr.y (mgr.14901) 6722 : cluster [DBG] pgmap v6729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:42.663 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:42 smithi006 bash[27292]: audit 2023-08-28T01:42:42.253180+0000 mon.c (mon.2) 2650 : audit [DBG] from='client.? 172.21.15.6:0/3463464122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:42.663 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:42 smithi006 bash[33918]: cluster 2023-08-28T01:42:40.502657+0000 mgr.y (mgr.14901) 6722 : cluster [DBG] pgmap v6729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:42.663 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:42 smithi006 bash[33918]: audit 2023-08-28T01:42:42.253180+0000 mon.c (mon.2) 2650 : audit [DBG] from='client.? 172.21.15.6:0/3463464122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:42:42.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:42:42.664Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:43.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:42 smithi130 bash[51302]: debug 2023-08-28T01:42:42.857+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:42.861295+0000) 2023-08-28T01:42:43.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:43 smithi130 bash[27579]: cluster 2023-08-28T01:42:42.503329+0000 mgr.y (mgr.14901) 6723 : cluster [DBG] pgmap v6730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:43.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:43 smithi006 bash[27292]: cluster 2023-08-28T01:42:42.503329+0000 mgr.y (mgr.14901) 6723 : cluster [DBG] pgmap v6730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:43.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:43 smithi006 bash[33918]: cluster 2023-08-28T01:42:42.503329+0000 mgr.y (mgr.14901) 6723 : cluster [DBG] pgmap v6730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:44.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:43 smithi130 bash[51302]: debug 2023-08-28T01:42:43.861+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:43.861649+0000) 2023-08-28T01:42:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:42:44.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:42:44.547Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:45.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:44 smithi130 bash[51302]: debug 2023-08-28T01:42:44.861+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:44.861996+0000) 2023-08-28T01:42:45.860 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:45 smithi130 bash[27579]: cluster 2023-08-28T01:42:44.504412+0000 mgr.y (mgr.14901) 6724 : cluster [DBG] pgmap v6731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:45.861 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:45 smithi130 bash[27579]: audit 2023-08-28T01:42:44.665765+0000 mon.c (mon.2) 2651 : audit [DBG] from='client.? 172.21.15.6:0/2163451584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:45.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:45 smithi006 bash[27292]: cluster 2023-08-28T01:42:44.504412+0000 mgr.y (mgr.14901) 6724 : cluster [DBG] pgmap v6731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:45.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:45 smithi006 bash[27292]: audit 2023-08-28T01:42:44.665765+0000 mon.c (mon.2) 2651 : audit [DBG] from='client.? 172.21.15.6:0/2163451584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:45.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:45 smithi006 bash[33918]: cluster 2023-08-28T01:42:44.504412+0000 mgr.y (mgr.14901) 6724 : cluster [DBG] pgmap v6731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:45.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:45 smithi006 bash[33918]: audit 2023-08-28T01:42:44.665765+0000 mon.c (mon.2) 2651 : audit [DBG] from='client.? 172.21.15.6:0/2163451584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:45 smithi130 bash[51302]: debug 2023-08-28T01:42:45.861+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:45.862448+0000) 2023-08-28T01:42:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:46 smithi130 bash[51302]: debug 2023-08-28T01:42:46.856+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:46.862738+0000) 2023-08-28T01:42:47.861 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:47 smithi130 bash[27579]: cluster 2023-08-28T01:42:46.504984+0000 mgr.y (mgr.14901) 6725 : cluster [DBG] pgmap v6732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:47.861 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:47 smithi130 bash[27579]: audit 2023-08-28T01:42:47.077205+0000 mon.c (mon.2) 2652 : audit [DBG] from='client.? 172.21.15.6:0/398751382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:47.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:47 smithi006 bash[33918]: cluster 2023-08-28T01:42:46.504984+0000 mgr.y (mgr.14901) 6725 : cluster [DBG] pgmap v6732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:47.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:47 smithi006 bash[33918]: audit 2023-08-28T01:42:47.077205+0000 mon.c (mon.2) 2652 : audit [DBG] from='client.? 172.21.15.6:0/398751382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:47.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:47 smithi006 bash[27292]: cluster 2023-08-28T01:42:46.504984+0000 mgr.y (mgr.14901) 6725 : cluster [DBG] pgmap v6732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:47.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:47 smithi006 bash[27292]: audit 2023-08-28T01:42:47.077205+0000 mon.c (mon.2) 2652 : audit [DBG] from='client.? 172.21.15.6:0/398751382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:47 smithi130 bash[51302]: debug 2023-08-28T01:42:47.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:47.863050+0000) 2023-08-28T01:42:49.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:48 smithi130 bash[51302]: debug 2023-08-28T01:42:48.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:48.863284+0000) 2023-08-28T01:42:49.862 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:49 smithi130 bash[27579]: cluster 2023-08-28T01:42:48.505579+0000 mgr.y (mgr.14901) 6726 : cluster [DBG] pgmap v6733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:49.862 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:49 smithi130 bash[27579]: audit 2023-08-28T01:42:49.495909+0000 mon.c (mon.2) 2653 : audit [DBG] from='client.? 172.21.15.6:0/2664683333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:49.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:49 smithi006 bash[33918]: cluster 2023-08-28T01:42:48.505579+0000 mgr.y (mgr.14901) 6726 : cluster [DBG] pgmap v6733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:49.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:49 smithi006 bash[33918]: audit 2023-08-28T01:42:49.495909+0000 mon.c (mon.2) 2653 : audit [DBG] from='client.? 172.21.15.6:0/2664683333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:49.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:49 smithi006 bash[27292]: cluster 2023-08-28T01:42:48.505579+0000 mgr.y (mgr.14901) 6726 : cluster [DBG] pgmap v6733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:49.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:49 smithi006 bash[27292]: audit 2023-08-28T01:42:49.495909+0000 mon.c (mon.2) 2653 : audit [DBG] from='client.? 172.21.15.6:0/2664683333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:50.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:49 smithi130 bash[51302]: debug 2023-08-28T01:42:49.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:49.863541+0000) 2023-08-28T01:42:51.094 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:42:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:42:50] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:42:51.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:50 smithi130 bash[51302]: debug 2023-08-28T01:42:50.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:50.863881+0000) 2023-08-28T01:42:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:42:51.094Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:42:51.096Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:51.862 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:51 smithi130 bash[27579]: cluster 2023-08-28T01:42:50.506708+0000 mgr.y (mgr.14901) 6727 : cluster [DBG] pgmap v6734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:51.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:51 smithi006 bash[33918]: cluster 2023-08-28T01:42:50.506708+0000 mgr.y (mgr.14901) 6727 : cluster [DBG] pgmap v6734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:51.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:51 smithi006 bash[27292]: cluster 2023-08-28T01:42:50.506708+0000 mgr.y (mgr.14901) 6727 : cluster [DBG] pgmap v6734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:52.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:51 smithi130 bash[51302]: debug 2023-08-28T01:42:51.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:51.864156+0000) 2023-08-28T01:42:52.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:52 smithi130 bash[27579]: audit 2023-08-28T01:42:51.910715+0000 mon.c (mon.2) 2654 : audit [DBG] from='client.? 172.21.15.6:0/2876479038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:52 smithi006 bash[27292]: audit 2023-08-28T01:42:51.910715+0000 mon.c (mon.2) 2654 : audit [DBG] from='client.? 172.21.15.6:0/2876479038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:52.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:52 smithi006 bash[33918]: audit 2023-08-28T01:42:51.910715+0000 mon.c (mon.2) 2654 : audit [DBG] from='client.? 172.21.15.6:0/2876479038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:52.968 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:42:52.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:52.970 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:42:52.665Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:53.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:52 smithi130 bash[51302]: debug 2023-08-28T01:42:52.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:52.864428+0000) 2023-08-28T01:42:53.863 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:53 smithi130 bash[27579]: cluster 2023-08-28T01:42:52.507373+0000 mgr.y (mgr.14901) 6728 : cluster [DBG] pgmap v6735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:53.863 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:53 smithi130 bash[27579]: audit 2023-08-28T01:42:52.977563+0000 mon.b (mon.1) 760 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:42:53.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:53 smithi006 bash[27292]: cluster 2023-08-28T01:42:52.507373+0000 mgr.y (mgr.14901) 6728 : cluster [DBG] pgmap v6735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:53.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:53 smithi006 bash[27292]: audit 2023-08-28T01:42:52.977563+0000 mon.b (mon.1) 760 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:42:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:53 smithi006 bash[33918]: cluster 2023-08-28T01:42:52.507373+0000 mgr.y (mgr.14901) 6728 : cluster [DBG] pgmap v6735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:53 smithi006 bash[33918]: audit 2023-08-28T01:42:52.977563+0000 mon.b (mon.1) 760 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:42:54.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:53 smithi130 bash[51302]: debug 2023-08-28T01:42:53.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:53.864697+0000) 2023-08-28T01:42:54.863 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:54 smithi130 bash[27579]: audit 2023-08-28T01:42:54.335417+0000 mon.c (mon.2) 2655 : audit [DBG] from='client.? 172.21.15.6:0/1144579394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:54.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:54 smithi006 bash[33918]: audit 2023-08-28T01:42:54.335417+0000 mon.c (mon.2) 2655 : audit [DBG] from='client.? 172.21.15.6:0/1144579394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:54.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:54 smithi006 bash[27292]: audit 2023-08-28T01:42:54.335417+0000 mon.c (mon.2) 2655 : audit [DBG] from='client.? 172.21.15.6:0/1144579394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:42:54.546Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:42:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:42:54.548Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:42:55.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:54 smithi130 bash[51302]: debug 2023-08-28T01:42:54.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:54.865001+0000) 2023-08-28T01:42:55.863 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:55 smithi130 bash[27579]: cluster 2023-08-28T01:42:54.508320+0000 mgr.y (mgr.14901) 6729 : cluster [DBG] pgmap v6736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:55.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:55 smithi006 bash[27292]: cluster 2023-08-28T01:42:54.508320+0000 mgr.y (mgr.14901) 6729 : cluster [DBG] pgmap v6736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:55.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:55 smithi006 bash[33918]: cluster 2023-08-28T01:42:54.508320+0000 mgr.y (mgr.14901) 6729 : cluster [DBG] pgmap v6736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:56.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:55 smithi130 bash[51302]: debug 2023-08-28T01:42:55.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:55.865230+0000) 2023-08-28T01:42:57.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:56 smithi130 bash[51302]: debug 2023-08-28T01:42:56.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:56.865513+0000) 2023-08-28T01:42:57.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:57 smithi130 bash[27579]: cluster 2023-08-28T01:42:56.508953+0000 mgr.y (mgr.14901) 6730 : cluster [DBG] pgmap v6737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:57.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:57 smithi130 bash[27579]: audit 2023-08-28T01:42:56.747161+0000 mon.a (mon.0) 4412 : audit [DBG] from='client.? 172.21.15.6:0/2490460091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:57.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:57 smithi006 bash[33918]: cluster 2023-08-28T01:42:56.508953+0000 mgr.y (mgr.14901) 6730 : cluster [DBG] pgmap v6737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:57.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:57 smithi006 bash[33918]: audit 2023-08-28T01:42:56.747161+0000 mon.a (mon.0) 4412 : audit [DBG] from='client.? 172.21.15.6:0/2490460091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:57.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:57 smithi006 bash[27292]: cluster 2023-08-28T01:42:56.508953+0000 mgr.y (mgr.14901) 6730 : cluster [DBG] pgmap v6737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:42:57.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:57 smithi006 bash[27292]: audit 2023-08-28T01:42:56.747161+0000 mon.a (mon.0) 4412 : audit [DBG] from='client.? 172.21.15.6:0/2490460091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:58.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:57 smithi130 bash[51302]: debug 2023-08-28T01:42:57.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:57.865723+0000) 2023-08-28T01:42:59.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:58 smithi130 bash[51302]: debug 2023-08-28T01:42:58.860+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:58.865993+0000) 2023-08-28T01:43:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:42:59 smithi130 bash[51302]: debug 2023-08-28T01:42:59.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:59.866309+0000) 2023-08-28T01:43:00.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:59 smithi130 bash[27579]: cluster 2023-08-28T01:42:58.509573+0000 mgr.y (mgr.14901) 6731 : cluster [DBG] pgmap v6738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:00.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:59 smithi130 bash[27579]: audit 2023-08-28T01:42:58.901697+0000 mon.a (mon.0) 4413 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:43:00.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:59 smithi130 bash[27579]: audit 2023-08-28T01:42:59.171928+0000 mon.c (mon.2) 2656 : audit [DBG] from='client.? 172.21.15.6:0/1945183812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:00.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:59 smithi130 bash[27579]: audit 2023-08-28T01:42:59.378987+0000 mon.a (mon.0) 4414 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:43:00.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:59 smithi130 bash[27579]: audit 2023-08-28T01:42:59.679939+0000 mon.b (mon.1) 761 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:43:00.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:59 smithi130 bash[27579]: audit 2023-08-28T01:42:59.682549+0000 mon.b (mon.1) 762 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:43:00.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:42:59 smithi130 bash[27579]: audit 2023-08-28T01:42:59.691311+0000 mon.a (mon.0) 4415 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:43:00.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:59 smithi006 bash[27292]: cluster 2023-08-28T01:42:58.509573+0000 mgr.y (mgr.14901) 6731 : cluster [DBG] pgmap v6738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:00.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:59 smithi006 bash[27292]: audit 2023-08-28T01:42:58.901697+0000 mon.a (mon.0) 4413 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:43:00.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:59 smithi006 bash[27292]: audit 2023-08-28T01:42:59.171928+0000 mon.c (mon.2) 2656 : audit [DBG] from='client.? 172.21.15.6:0/1945183812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:00.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:59 smithi006 bash[27292]: audit 2023-08-28T01:42:59.378987+0000 mon.a (mon.0) 4414 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:43:00.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:59 smithi006 bash[27292]: audit 2023-08-28T01:42:59.679939+0000 mon.b (mon.1) 761 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:43:00.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:59 smithi006 bash[27292]: audit 2023-08-28T01:42:59.682549+0000 mon.b (mon.1) 762 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:43:00.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:42:59 smithi006 bash[27292]: audit 2023-08-28T01:42:59.691311+0000 mon.a (mon.0) 4415 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:43:00.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:59 smithi006 bash[33918]: cluster 2023-08-28T01:42:58.509573+0000 mgr.y (mgr.14901) 6731 : cluster [DBG] pgmap v6738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:00.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:59 smithi006 bash[33918]: audit 2023-08-28T01:42:58.901697+0000 mon.a (mon.0) 4413 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:43:00.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:59 smithi006 bash[33918]: audit 2023-08-28T01:42:59.171928+0000 mon.c (mon.2) 2656 : audit [DBG] from='client.? 172.21.15.6:0/1945183812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:00.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:59 smithi006 bash[33918]: audit 2023-08-28T01:42:59.378987+0000 mon.a (mon.0) 4414 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:43:00.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:59 smithi006 bash[33918]: audit 2023-08-28T01:42:59.679939+0000 mon.b (mon.1) 761 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:43:00.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:59 smithi006 bash[33918]: audit 2023-08-28T01:42:59.682549+0000 mon.b (mon.1) 762 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:43:00.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:42:59 smithi006 bash[33918]: audit 2023-08-28T01:42:59.691311+0000 mon.a (mon.0) 4415 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:43:01.095 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:43:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:43:00] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T01:43:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:00 smithi130 bash[51302]: debug 2023-08-28T01:43:00.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:00.866611+0000) 2023-08-28T01:43:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:43:01.095Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:43:01.097Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:02.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:01 smithi006 bash[27292]: cluster 2023-08-28T01:43:00.510801+0000 mgr.y (mgr.14901) 6732 : cluster [DBG] pgmap v6739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:02.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:01 smithi006 bash[27292]: audit 2023-08-28T01:43:01.590825+0000 mon.a (mon.0) 4416 : audit [DBG] from='client.? 172.21.15.6:0/2399964216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:02.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:01 smithi006 bash[33918]: cluster 2023-08-28T01:43:00.510801+0000 mgr.y (mgr.14901) 6732 : cluster [DBG] pgmap v6739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:02.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:01 smithi006 bash[33918]: audit 2023-08-28T01:43:01.590825+0000 mon.a (mon.0) 4416 : audit [DBG] from='client.? 172.21.15.6:0/2399964216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:02.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:01 smithi130 bash[27579]: cluster 2023-08-28T01:43:00.510801+0000 mgr.y (mgr.14901) 6732 : cluster [DBG] pgmap v6739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:02.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:01 smithi130 bash[27579]: audit 2023-08-28T01:43:01.590825+0000 mon.a (mon.0) 4416 : audit [DBG] from='client.? 172.21.15.6:0/2399964216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:02.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:01 smithi130 bash[51302]: debug 2023-08-28T01:43:01.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:01.866908+0000) 2023-08-28T01:43:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:43:02.664Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:43:02.666Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:03.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:02 smithi130 bash[51302]: debug 2023-08-28T01:43:02.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:02.867216+0000) 2023-08-28T01:43:04.194 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:03 smithi130 bash[51302]: debug 2023-08-28T01:43:03.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:03.867435+0000) 2023-08-28T01:43:04.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:04 smithi006 bash[33918]: cluster 2023-08-28T01:43:02.511502+0000 mgr.y (mgr.14901) 6733 : cluster [DBG] pgmap v6740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:04.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:04 smithi006 bash[33918]: audit 2023-08-28T01:43:04.010091+0000 mon.a (mon.0) 4417 : audit [DBG] from='client.? 172.21.15.6:0/3437144073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:04.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:04 smithi006 bash[27292]: cluster 2023-08-28T01:43:02.511502+0000 mgr.y (mgr.14901) 6733 : cluster [DBG] pgmap v6740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:04.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:04 smithi006 bash[27292]: audit 2023-08-28T01:43:04.010091+0000 mon.a (mon.0) 4417 : audit [DBG] from='client.? 172.21.15.6:0/3437144073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:04 smithi130 bash[27579]: cluster 2023-08-28T01:43:02.511502+0000 mgr.y (mgr.14901) 6733 : cluster [DBG] pgmap v6740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:04 smithi130 bash[27579]: audit 2023-08-28T01:43:04.010091+0000 mon.a (mon.0) 4417 : audit [DBG] from='client.? 172.21.15.6:0/3437144073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:43:04.547Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:43:04.549Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:05.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:04 smithi130 bash[51302]: debug 2023-08-28T01:43:04.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:04.867703+0000) 2023-08-28T01:43:06.198 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:05 smithi130 bash[51302]: debug 2023-08-28T01:43:05.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:05.867980+0000) 2023-08-28T01:43:06.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:06 smithi006 bash[27292]: cluster 2023-08-28T01:43:04.512584+0000 mgr.y (mgr.14901) 6734 : cluster [DBG] pgmap v6741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:06.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:06 smithi006 bash[33918]: cluster 2023-08-28T01:43:04.512584+0000 mgr.y (mgr.14901) 6734 : cluster [DBG] pgmap v6741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:06 smithi130 bash[27579]: cluster 2023-08-28T01:43:04.512584+0000 mgr.y (mgr.14901) 6734 : cluster [DBG] pgmap v6741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:07.211 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:06 smithi130 bash[51302]: debug 2023-08-28T01:43:06.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:06.868251+0000) 2023-08-28T01:43:07.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:07 smithi006 bash[33918]: audit 2023-08-28T01:43:06.425378+0000 mon.a (mon.0) 4418 : audit [DBG] from='client.? 172.21.15.6:0/36411071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:07 smithi006 bash[27292]: audit 2023-08-28T01:43:06.425378+0000 mon.a (mon.0) 4418 : audit [DBG] from='client.? 172.21.15.6:0/36411071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:07.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:07 smithi130 bash[27579]: audit 2023-08-28T01:43:06.425378+0000 mon.a (mon.0) 4418 : audit [DBG] from='client.? 172.21.15.6:0/36411071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:08.213 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:07 smithi130 bash[51302]: debug 2023-08-28T01:43:07.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:07.868544+0000) 2023-08-28T01:43:08.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:08 smithi006 bash[27292]: cluster 2023-08-28T01:43:06.513269+0000 mgr.y (mgr.14901) 6735 : cluster [DBG] pgmap v6742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:08.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:08 smithi006 bash[33918]: cluster 2023-08-28T01:43:06.513269+0000 mgr.y (mgr.14901) 6735 : cluster [DBG] pgmap v6742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:08 smithi130 bash[27579]: cluster 2023-08-28T01:43:06.513269+0000 mgr.y (mgr.14901) 6735 : cluster [DBG] pgmap v6742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:09.221 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:08 smithi130 bash[51302]: debug 2023-08-28T01:43:08.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:08.868819+0000) 2023-08-28T01:43:09.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:09 smithi130 bash[27579]: audit 2023-08-28T01:43:08.841595+0000 mon.a (mon.0) 4419 : audit [DBG] from='client.? 172.21.15.6:0/3996850750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:09.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:09 smithi006 bash[27292]: audit 2023-08-28T01:43:08.841595+0000 mon.a (mon.0) 4419 : audit [DBG] from='client.? 172.21.15.6:0/3996850750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:09.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:09 smithi006 bash[33918]: audit 2023-08-28T01:43:08.841595+0000 mon.a (mon.0) 4419 : audit [DBG] from='client.? 172.21.15.6:0/3996850750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:10.224 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:09 smithi130 bash[51302]: debug 2023-08-28T01:43:09.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:09.869104+0000) 2023-08-28T01:43:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:10 smithi130 bash[27579]: cluster 2023-08-28T01:43:08.513872+0000 mgr.y (mgr.14901) 6736 : cluster [DBG] pgmap v6743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:10 smithi006 bash[33918]: cluster 2023-08-28T01:43:08.513872+0000 mgr.y (mgr.14901) 6736 : cluster [DBG] pgmap v6743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:10 smithi006 bash[27292]: cluster 2023-08-28T01:43:08.513872+0000 mgr.y (mgr.14901) 6736 : cluster [DBG] pgmap v6743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:11.096 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:43:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:43:10] "GET /metrics HTTP/1.1" 200 193130 "" "Prometheus/2.33.4" 2023-08-28T01:43:11.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:10 smithi130 bash[51302]: debug 2023-08-28T01:43:10.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:10.869409+0000) 2023-08-28T01:43:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:43:11.095Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:43:11.097Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:12.228 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:11 smithi130 bash[51302]: debug 2023-08-28T01:43:11.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:11.869692+0000) 2023-08-28T01:43:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:12 smithi130 bash[27579]: cluster 2023-08-28T01:43:10.514986+0000 mgr.y (mgr.14901) 6737 : cluster [DBG] pgmap v6744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:12 smithi130 bash[27579]: audit 2023-08-28T01:43:11.261209+0000 mon.a (mon.0) 4420 : audit [DBG] from='client.? 172.21.15.6:0/1475552135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:12.666 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:12 smithi006 bash[33918]: cluster 2023-08-28T01:43:10.514986+0000 mgr.y (mgr.14901) 6737 : cluster [DBG] pgmap v6744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:12.666 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:12 smithi006 bash[33918]: audit 2023-08-28T01:43:11.261209+0000 mon.a (mon.0) 4420 : audit [DBG] from='client.? 172.21.15.6:0/1475552135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:12.666 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:12 smithi006 bash[27292]: cluster 2023-08-28T01:43:10.514986+0000 mgr.y (mgr.14901) 6737 : cluster [DBG] pgmap v6744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:12.666 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:12 smithi006 bash[27292]: audit 2023-08-28T01:43:11.261209+0000 mon.a (mon.0) 4420 : audit [DBG] from='client.? 172.21.15.6:0/1475552135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:43:12.665Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:43:12.668Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:13.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:12 smithi130 bash[51302]: debug 2023-08-28T01:43:12.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:12.869971+0000) 2023-08-28T01:43:14.233 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:13 smithi130 bash[51302]: debug 2023-08-28T01:43:13.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:13.870251+0000) 2023-08-28T01:43:14.549 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:14 smithi006 bash[33918]: cluster 2023-08-28T01:43:12.515647+0000 mgr.y (mgr.14901) 6738 : cluster [DBG] pgmap v6745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:14.549 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:14 smithi006 bash[33918]: audit 2023-08-28T01:43:13.671558+0000 mon.c (mon.2) 2657 : audit [DBG] from='client.? 172.21.15.6:0/1169574346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:14.549 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:14 smithi006 bash[27292]: cluster 2023-08-28T01:43:12.515647+0000 mgr.y (mgr.14901) 6738 : cluster [DBG] pgmap v6745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:14.549 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:14 smithi006 bash[27292]: audit 2023-08-28T01:43:13.671558+0000 mon.c (mon.2) 2657 : audit [DBG] from='client.? 172.21.15.6:0/1169574346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:14 smithi130 bash[27579]: cluster 2023-08-28T01:43:12.515647+0000 mgr.y (mgr.14901) 6738 : cluster [DBG] pgmap v6745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:14 smithi130 bash[27579]: audit 2023-08-28T01:43:13.671558+0000 mon.c (mon.2) 2657 : audit [DBG] from='client.? 172.21.15.6:0/1169574346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:43:14.549Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:43:14.551Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:15.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:14 smithi130 bash[51302]: debug 2023-08-28T01:43:14.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:14.870521+0000) 2023-08-28T01:43:16.237 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:15 smithi130 bash[51302]: debug 2023-08-28T01:43:15.864+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:15.870794+0000) 2023-08-28T01:43:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:16 smithi130 bash[27579]: cluster 2023-08-28T01:43:14.516769+0000 mgr.y (mgr.14901) 6739 : cluster [DBG] pgmap v6746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:16 smithi130 bash[27579]: audit 2023-08-28T01:43:16.084955+0000 mon.c (mon.2) 2658 : audit [DBG] from='client.? 172.21.15.6:0/4264915914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:16.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:16 smithi006 bash[27292]: cluster 2023-08-28T01:43:14.516769+0000 mgr.y (mgr.14901) 6739 : cluster [DBG] pgmap v6746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:16 smithi006 bash[27292]: audit 2023-08-28T01:43:16.084955+0000 mon.c (mon.2) 2658 : audit [DBG] from='client.? 172.21.15.6:0/4264915914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:16 smithi006 bash[33918]: cluster 2023-08-28T01:43:14.516769+0000 mgr.y (mgr.14901) 6739 : cluster [DBG] pgmap v6746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:16 smithi006 bash[33918]: audit 2023-08-28T01:43:16.084955+0000 mon.c (mon.2) 2658 : audit [DBG] from='client.? 172.21.15.6:0/4264915914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:17.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:16 smithi130 bash[51302]: debug 2023-08-28T01:43:16.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:16.871065+0000) 2023-08-28T01:43:18.241 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:17 smithi130 bash[51302]: debug 2023-08-28T01:43:17.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:17.871337+0000) 2023-08-28T01:43:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:18 smithi130 bash[27579]: cluster 2023-08-28T01:43:16.517499+0000 mgr.y (mgr.14901) 6740 : cluster [DBG] pgmap v6747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:18 smithi006 bash[27292]: cluster 2023-08-28T01:43:16.517499+0000 mgr.y (mgr.14901) 6740 : cluster [DBG] pgmap v6747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:18.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:18 smithi006 bash[33918]: cluster 2023-08-28T01:43:16.517499+0000 mgr.y (mgr.14901) 6740 : cluster [DBG] pgmap v6747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:19.249 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:18 smithi130 bash[51302]: debug 2023-08-28T01:43:18.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:18.871622+0000) 2023-08-28T01:43:19.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:19 smithi130 bash[27579]: audit 2023-08-28T01:43:18.499277+0000 mon.c (mon.2) 2659 : audit [DBG] from='client.? 172.21.15.6:0/1641947372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:19.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:19 smithi006 bash[33918]: audit 2023-08-28T01:43:18.499277+0000 mon.c (mon.2) 2659 : audit [DBG] from='client.? 172.21.15.6:0/1641947372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:19.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:19 smithi006 bash[27292]: audit 2023-08-28T01:43:18.499277+0000 mon.c (mon.2) 2659 : audit [DBG] from='client.? 172.21.15.6:0/1641947372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:20.252 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:19 smithi130 bash[51302]: debug 2023-08-28T01:43:19.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:19.871932+0000) 2023-08-28T01:43:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:20 smithi130 bash[27579]: cluster 2023-08-28T01:43:18.518163+0000 mgr.y (mgr.14901) 6741 : cluster [DBG] pgmap v6748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:20.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:20 smithi006 bash[33918]: cluster 2023-08-28T01:43:18.518163+0000 mgr.y (mgr.14901) 6741 : cluster [DBG] pgmap v6748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:20.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:20 smithi006 bash[27292]: cluster 2023-08-28T01:43:18.518163+0000 mgr.y (mgr.14901) 6741 : cluster [DBG] pgmap v6748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:21.097 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:43:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:43:20] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:43:21.260 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:20 smithi130 bash[51302]: debug 2023-08-28T01:43:20.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:20.872250+0000) 2023-08-28T01:43:21.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:21 smithi006 bash[33918]: audit 2023-08-28T01:43:20.916749+0000 mon.c (mon.2) 2660 : audit [DBG] from='client.? 172.21.15.6:0/3400779796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:21.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:21 smithi006 bash[27292]: audit 2023-08-28T01:43:20.916749+0000 mon.c (mon.2) 2660 : audit [DBG] from='client.? 172.21.15.6:0/3400779796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:43:21.096Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:43:21.098Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:21.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:21 smithi130 bash[27579]: audit 2023-08-28T01:43:20.916749+0000 mon.c (mon.2) 2660 : audit [DBG] from='client.? 172.21.15.6:0/3400779796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:22.263 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:21 smithi130 bash[51302]: debug 2023-08-28T01:43:21.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:21.872549+0000) 2023-08-28T01:43:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:22 smithi130 bash[27579]: cluster 2023-08-28T01:43:20.519191+0000 mgr.y (mgr.14901) 6742 : cluster [DBG] pgmap v6749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:22.666 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:22 smithi006 bash[33918]: cluster 2023-08-28T01:43:20.519191+0000 mgr.y (mgr.14901) 6742 : cluster [DBG] pgmap v6749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:22.666 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:22 smithi006 bash[27292]: cluster 2023-08-28T01:43:20.519191+0000 mgr.y (mgr.14901) 6742 : cluster [DBG] pgmap v6749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:43:22.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:43:22.668Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:23.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:22 smithi130 bash[51302]: debug 2023-08-28T01:43:22.872+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:22.872823+0000) 2023-08-28T01:43:24.267 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:23 smithi130 bash[51302]: debug 2023-08-28T01:43:23.872+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:23.873143+0000) 2023-08-28T01:43:24.550 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:24 smithi006 bash[33918]: cluster 2023-08-28T01:43:22.519859+0000 mgr.y (mgr.14901) 6743 : cluster [DBG] pgmap v6750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:24.550 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:24 smithi006 bash[33918]: audit 2023-08-28T01:43:23.335127+0000 mon.a (mon.0) 4421 : audit [DBG] from='client.? 172.21.15.6:0/803810007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:24.550 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:24 smithi006 bash[27292]: cluster 2023-08-28T01:43:22.519859+0000 mgr.y (mgr.14901) 6743 : cluster [DBG] pgmap v6750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:24.550 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:24 smithi006 bash[27292]: audit 2023-08-28T01:43:23.335127+0000 mon.a (mon.0) 4421 : audit [DBG] from='client.? 172.21.15.6:0/803810007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:24 smithi130 bash[27579]: cluster 2023-08-28T01:43:22.519859+0000 mgr.y (mgr.14901) 6743 : cluster [DBG] pgmap v6750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:24 smithi130 bash[27579]: audit 2023-08-28T01:43:23.335127+0000 mon.a (mon.0) 4421 : audit [DBG] from='client.? 172.21.15.6:0/803810007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:43:24.549Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:43:24.552Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:25.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:24 smithi130 bash[51302]: debug 2023-08-28T01:43:24.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:24.873415+0000) 2023-08-28T01:43:26.272 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:25 smithi130 bash[51302]: debug 2023-08-28T01:43:25.872+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:25.873734+0000) 2023-08-28T01:43:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:26 smithi130 bash[27579]: cluster 2023-08-28T01:43:24.521067+0000 mgr.y (mgr.14901) 6744 : cluster [DBG] pgmap v6751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:26 smithi130 bash[27579]: audit 2023-08-28T01:43:25.750282+0000 mon.c (mon.2) 2661 : audit [DBG] from='client.? 172.21.15.6:0/796916432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:26.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:26 smithi006 bash[27292]: cluster 2023-08-28T01:43:24.521067+0000 mgr.y (mgr.14901) 6744 : cluster [DBG] pgmap v6751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:26 smithi006 bash[27292]: audit 2023-08-28T01:43:25.750282+0000 mon.c (mon.2) 2661 : audit [DBG] from='client.? 172.21.15.6:0/796916432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:26 smithi006 bash[33918]: cluster 2023-08-28T01:43:24.521067+0000 mgr.y (mgr.14901) 6744 : cluster [DBG] pgmap v6751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:26 smithi006 bash[33918]: audit 2023-08-28T01:43:25.750282+0000 mon.c (mon.2) 2661 : audit [DBG] from='client.? 172.21.15.6:0/796916432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:27.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:26 smithi130 bash[51302]: debug 2023-08-28T01:43:26.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:26.874013+0000) 2023-08-28T01:43:28.280 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:27 smithi130 bash[51302]: debug 2023-08-28T01:43:27.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:27.874386+0000) 2023-08-28T01:43:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:28 smithi130 bash[27579]: cluster 2023-08-28T01:43:26.521648+0000 mgr.y (mgr.14901) 6745 : cluster [DBG] pgmap v6752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:28 smithi130 bash[27579]: audit 2023-08-28T01:43:28.164680+0000 mon.a (mon.0) 4422 : audit [DBG] from='client.? 172.21.15.6:0/3409008817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:28.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:28 smithi006 bash[27292]: cluster 2023-08-28T01:43:26.521648+0000 mgr.y (mgr.14901) 6745 : cluster [DBG] pgmap v6752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:28.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:28 smithi006 bash[27292]: audit 2023-08-28T01:43:28.164680+0000 mon.a (mon.0) 4422 : audit [DBG] from='client.? 172.21.15.6:0/3409008817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:28 smithi006 bash[33918]: cluster 2023-08-28T01:43:26.521648+0000 mgr.y (mgr.14901) 6745 : cluster [DBG] pgmap v6752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:28 smithi006 bash[33918]: audit 2023-08-28T01:43:28.164680+0000 mon.a (mon.0) 4422 : audit [DBG] from='client.? 172.21.15.6:0/3409008817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:29.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:28 smithi130 bash[51302]: debug 2023-08-28T01:43:28.868+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:28.874661+0000) 2023-08-28T01:43:30.284 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:29 smithi130 bash[51302]: debug 2023-08-28T01:43:29.872+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:29.874958+0000) 2023-08-28T01:43:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:30 smithi130 bash[27579]: cluster 2023-08-28T01:43:28.522290+0000 mgr.y (mgr.14901) 6746 : cluster [DBG] pgmap v6753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:30 smithi006 bash[33918]: cluster 2023-08-28T01:43:28.522290+0000 mgr.y (mgr.14901) 6746 : cluster [DBG] pgmap v6753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:30 smithi006 bash[27292]: cluster 2023-08-28T01:43:28.522290+0000 mgr.y (mgr.14901) 6746 : cluster [DBG] pgmap v6753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:31.098 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:43:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:43:30] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:43:31.293 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:30 smithi130 bash[51302]: debug 2023-08-28T01:43:30.872+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:30.875285+0000) 2023-08-28T01:43:31.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:31 smithi006 bash[27292]: audit 2023-08-28T01:43:30.582107+0000 mon.a (mon.0) 4423 : audit [DBG] from='client.? 172.21.15.6:0/1942004389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:31.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:31 smithi006 bash[33918]: audit 2023-08-28T01:43:30.582107+0000 mon.a (mon.0) 4423 : audit [DBG] from='client.? 172.21.15.6:0/1942004389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:43:31.098Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:43:31.099Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:31.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:31 smithi130 bash[27579]: audit 2023-08-28T01:43:30.582107+0000 mon.a (mon.0) 4423 : audit [DBG] from='client.? 172.21.15.6:0/1942004389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:32.297 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:31 smithi130 bash[51302]: debug 2023-08-28T01:43:31.872+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:31.875590+0000) 2023-08-28T01:43:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:32 smithi130 bash[27579]: cluster 2023-08-28T01:43:30.523347+0000 mgr.y (mgr.14901) 6747 : cluster [DBG] pgmap v6754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:32.667 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:32 smithi006 bash[27292]: cluster 2023-08-28T01:43:30.523347+0000 mgr.y (mgr.14901) 6747 : cluster [DBG] pgmap v6754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:32.667 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:32 smithi006 bash[33918]: cluster 2023-08-28T01:43:30.523347+0000 mgr.y (mgr.14901) 6747 : cluster [DBG] pgmap v6754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:43:32.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:43:32.669Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:33.306 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:32 smithi130 bash[51302]: debug 2023-08-28T01:43:32.872+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:32.875876+0000) 2023-08-28T01:43:33.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:33 smithi130 bash[27579]: audit 2023-08-28T01:43:32.999149+0000 mon.a (mon.0) 4424 : audit [DBG] from='client.? 172.21.15.6:0/1856414697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:33.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:33 smithi006 bash[33918]: audit 2023-08-28T01:43:32.999149+0000 mon.a (mon.0) 4424 : audit [DBG] from='client.? 172.21.15.6:0/1856414697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:33.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:33 smithi006 bash[27292]: audit 2023-08-28T01:43:32.999149+0000 mon.a (mon.0) 4424 : audit [DBG] from='client.? 172.21.15.6:0/1856414697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:34.309 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:33 smithi130 bash[51302]: debug 2023-08-28T01:43:33.872+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:33.876182+0000) 2023-08-28T01:43:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:34 smithi130 bash[27579]: cluster 2023-08-28T01:43:32.524019+0000 mgr.y (mgr.14901) 6748 : cluster [DBG] pgmap v6755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:34.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:34 smithi006 bash[27292]: cluster 2023-08-28T01:43:32.524019+0000 mgr.y (mgr.14901) 6748 : cluster [DBG] pgmap v6755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:34.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:34 smithi006 bash[33918]: cluster 2023-08-28T01:43:32.524019+0000 mgr.y (mgr.14901) 6748 : cluster [DBG] pgmap v6755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:43:34.550Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:43:34.552Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:35.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:34 smithi130 bash[51302]: debug 2023-08-28T01:43:34.876+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:34.876455+0000) 2023-08-28T01:43:36.313 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:35 smithi130 bash[51302]: debug 2023-08-28T01:43:35.872+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:35.876779+0000) 2023-08-28T01:43:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:36 smithi130 bash[27579]: cluster 2023-08-28T01:43:34.524962+0000 mgr.y (mgr.14901) 6749 : cluster [DBG] pgmap v6756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:36 smithi130 bash[27579]: audit 2023-08-28T01:43:35.420918+0000 mon.a (mon.0) 4425 : audit [DBG] from='client.? 172.21.15.6:0/366765005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:36.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:36 smithi006 bash[27292]: cluster 2023-08-28T01:43:34.524962+0000 mgr.y (mgr.14901) 6749 : cluster [DBG] pgmap v6756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:36.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:36 smithi006 bash[27292]: audit 2023-08-28T01:43:35.420918+0000 mon.a (mon.0) 4425 : audit [DBG] from='client.? 172.21.15.6:0/366765005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:36.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:36 smithi006 bash[33918]: cluster 2023-08-28T01:43:34.524962+0000 mgr.y (mgr.14901) 6749 : cluster [DBG] pgmap v6756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:36.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:36 smithi006 bash[33918]: audit 2023-08-28T01:43:35.420918+0000 mon.a (mon.0) 4425 : audit [DBG] from='client.? 172.21.15.6:0/366765005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:37.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:36 smithi130 bash[51302]: debug 2023-08-28T01:43:36.872+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:36.877036+0000) 2023-08-28T01:43:38.318 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:37 smithi130 bash[51302]: debug 2023-08-28T01:43:37.876+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:37.877347+0000) 2023-08-28T01:43:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:38 smithi130 bash[27579]: cluster 2023-08-28T01:43:36.525607+0000 mgr.y (mgr.14901) 6750 : cluster [DBG] pgmap v6757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:38 smithi130 bash[27579]: audit 2023-08-28T01:43:37.831250+0000 mon.a (mon.0) 4426 : audit [DBG] from='client.? 172.21.15.6:0/1163056915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:38.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:38 smithi006 bash[27292]: cluster 2023-08-28T01:43:36.525607+0000 mgr.y (mgr.14901) 6750 : cluster [DBG] pgmap v6757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:38.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:38 smithi006 bash[27292]: audit 2023-08-28T01:43:37.831250+0000 mon.a (mon.0) 4426 : audit [DBG] from='client.? 172.21.15.6:0/1163056915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:38.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:38 smithi006 bash[33918]: cluster 2023-08-28T01:43:36.525607+0000 mgr.y (mgr.14901) 6750 : cluster [DBG] pgmap v6757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:38.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:38 smithi006 bash[33918]: audit 2023-08-28T01:43:37.831250+0000 mon.a (mon.0) 4426 : audit [DBG] from='client.? 172.21.15.6:0/1163056915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:39.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:38 smithi130 bash[51302]: debug 2023-08-28T01:43:38.876+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:38.877659+0000) 2023-08-28T01:43:40.323 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:39 smithi130 bash[51302]: debug 2023-08-28T01:43:39.876+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:39.877938+0000) 2023-08-28T01:43:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:40 smithi130 bash[27579]: cluster 2023-08-28T01:43:38.526261+0000 mgr.y (mgr.14901) 6751 : cluster [DBG] pgmap v6758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:40 smithi130 bash[27579]: audit 2023-08-28T01:43:40.251510+0000 mon.c (mon.2) 2662 : audit [DBG] from='client.? 172.21.15.6:0/3365908532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:40.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:40 smithi006 bash[27292]: cluster 2023-08-28T01:43:38.526261+0000 mgr.y (mgr.14901) 6751 : cluster [DBG] pgmap v6758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:40.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:40 smithi006 bash[27292]: audit 2023-08-28T01:43:40.251510+0000 mon.c (mon.2) 2662 : audit [DBG] from='client.? 172.21.15.6:0/3365908532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:40 smithi006 bash[33918]: cluster 2023-08-28T01:43:38.526261+0000 mgr.y (mgr.14901) 6751 : cluster [DBG] pgmap v6758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:40 smithi006 bash[33918]: audit 2023-08-28T01:43:40.251510+0000 mon.c (mon.2) 2662 : audit [DBG] from='client.? 172.21.15.6:0/3365908532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:41.098 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:43:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:43:40] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:43:41.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:40 smithi130 bash[51302]: debug 2023-08-28T01:43:40.876+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:40.878286+0000) 2023-08-28T01:43:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:43:41.098Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:43:41.100Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:42.327 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:41 smithi130 bash[51302]: debug 2023-08-28T01:43:41.872+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:41.878564+0000) 2023-08-28T01:43:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:42 smithi130 bash[27579]: cluster 2023-08-28T01:43:40.527443+0000 mgr.y (mgr.14901) 6752 : cluster [DBG] pgmap v6759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:42.668 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:42 smithi006 bash[27292]: cluster 2023-08-28T01:43:40.527443+0000 mgr.y (mgr.14901) 6752 : cluster [DBG] pgmap v6759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:42.669 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:42 smithi006 bash[33918]: cluster 2023-08-28T01:43:40.527443+0000 mgr.y (mgr.14901) 6752 : cluster [DBG] pgmap v6759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:43:42.668Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:43:42.670Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:43.335 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:42 smithi130 bash[51302]: debug 2023-08-28T01:43:42.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:42.878882+0000) 2023-08-28T01:43:43.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:43 smithi130 bash[27579]: audit 2023-08-28T01:43:42.680624+0000 mon.c (mon.2) 2663 : audit [DBG] from='client.? 172.21.15.6:0/3486695377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:43.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:43 smithi006 bash[27292]: audit 2023-08-28T01:43:42.680624+0000 mon.c (mon.2) 2663 : audit [DBG] from='client.? 172.21.15.6:0/3486695377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:43.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:43 smithi006 bash[33918]: audit 2023-08-28T01:43:42.680624+0000 mon.c (mon.2) 2663 : audit [DBG] from='client.? 172.21.15.6:0/3486695377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:44.338 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:43 smithi130 bash[51302]: debug 2023-08-28T01:43:43.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:43.879151+0000) 2023-08-28T01:43:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:44 smithi130 bash[27579]: cluster 2023-08-28T01:43:42.528092+0000 mgr.y (mgr.14901) 6753 : cluster [DBG] pgmap v6760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:44.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:44 smithi006 bash[33918]: cluster 2023-08-28T01:43:42.528092+0000 mgr.y (mgr.14901) 6753 : cluster [DBG] pgmap v6760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:44.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:44 smithi006 bash[27292]: cluster 2023-08-28T01:43:42.528092+0000 mgr.y (mgr.14901) 6753 : cluster [DBG] pgmap v6760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:43:44.551Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:43:44.553Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:45.346 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:44 smithi130 bash[51302]: debug 2023-08-28T01:43:44.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:44.879424+0000) 2023-08-28T01:43:45.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:45 smithi130 bash[27579]: audit 2023-08-28T01:43:45.095958+0000 mon.c (mon.2) 2664 : audit [DBG] from='client.? 172.21.15.6:0/1222618708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:45.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:45 smithi006 bash[33918]: audit 2023-08-28T01:43:45.095958+0000 mon.c (mon.2) 2664 : audit [DBG] from='client.? 172.21.15.6:0/1222618708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:45.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:45 smithi006 bash[27292]: audit 2023-08-28T01:43:45.095958+0000 mon.c (mon.2) 2664 : audit [DBG] from='client.? 172.21.15.6:0/1222618708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:46.348 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:45 smithi130 bash[51302]: debug 2023-08-28T01:43:45.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:45.879720+0000) 2023-08-28T01:43:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:46 smithi130 bash[27579]: cluster 2023-08-28T01:43:44.529301+0000 mgr.y (mgr.14901) 6754 : cluster [DBG] pgmap v6761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:46.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:46 smithi006 bash[33918]: cluster 2023-08-28T01:43:44.529301+0000 mgr.y (mgr.14901) 6754 : cluster [DBG] pgmap v6761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:46.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:46 smithi006 bash[27292]: cluster 2023-08-28T01:43:44.529301+0000 mgr.y (mgr.14901) 6754 : cluster [DBG] pgmap v6761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:47.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:46 smithi130 bash[51302]: debug 2023-08-28T01:43:46.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:46.880027+0000) 2023-08-28T01:43:48.355 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:47 smithi130 bash[51302]: debug 2023-08-28T01:43:47.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:47.880356+0000) 2023-08-28T01:43:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:48 smithi130 bash[27579]: cluster 2023-08-28T01:43:46.529835+0000 mgr.y (mgr.14901) 6755 : cluster [DBG] pgmap v6762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:48 smithi130 bash[27579]: audit 2023-08-28T01:43:47.511899+0000 mon.c (mon.2) 2665 : audit [DBG] from='client.? 172.21.15.6:0/1800207689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:48.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:48 smithi006 bash[33918]: cluster 2023-08-28T01:43:46.529835+0000 mgr.y (mgr.14901) 6755 : cluster [DBG] pgmap v6762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:48.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:48 smithi006 bash[33918]: audit 2023-08-28T01:43:47.511899+0000 mon.c (mon.2) 2665 : audit [DBG] from='client.? 172.21.15.6:0/1800207689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:48 smithi006 bash[27292]: cluster 2023-08-28T01:43:46.529835+0000 mgr.y (mgr.14901) 6755 : cluster [DBG] pgmap v6762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:48 smithi006 bash[27292]: audit 2023-08-28T01:43:47.511899+0000 mon.c (mon.2) 2665 : audit [DBG] from='client.? 172.21.15.6:0/1800207689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:49.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:48 smithi130 bash[51302]: debug 2023-08-28T01:43:48.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:48.880638+0000) 2023-08-28T01:43:50.360 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:49 smithi130 bash[51302]: debug 2023-08-28T01:43:49.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:49.880942+0000) 2023-08-28T01:43:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:50 smithi130 bash[27579]: cluster 2023-08-28T01:43:48.530460+0000 mgr.y (mgr.14901) 6756 : cluster [DBG] pgmap v6763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:50 smithi130 bash[27579]: audit 2023-08-28T01:43:49.936244+0000 mon.a (mon.0) 4427 : audit [DBG] from='client.? 172.21.15.6:0/1816106870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:50.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:50 smithi006 bash[33918]: cluster 2023-08-28T01:43:48.530460+0000 mgr.y (mgr.14901) 6756 : cluster [DBG] pgmap v6763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:50.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:50 smithi006 bash[33918]: audit 2023-08-28T01:43:49.936244+0000 mon.a (mon.0) 4427 : audit [DBG] from='client.? 172.21.15.6:0/1816106870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:50 smithi006 bash[27292]: cluster 2023-08-28T01:43:48.530460+0000 mgr.y (mgr.14901) 6756 : cluster [DBG] pgmap v6763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:50 smithi006 bash[27292]: audit 2023-08-28T01:43:49.936244+0000 mon.a (mon.0) 4427 : audit [DBG] from='client.? 172.21.15.6:0/1816106870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:51.099 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:43:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:43:50] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T01:43:51.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:50 smithi130 bash[51302]: debug 2023-08-28T01:43:50.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:50.881234+0000) 2023-08-28T01:43:51.414 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:43:51.099Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:51.414 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:43:51.101Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:51.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:51 smithi006 bash[33918]: cluster 2023-08-28T01:43:50.531514+0000 mgr.y (mgr.14901) 6757 : cluster [DBG] pgmap v6764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:51.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:51 smithi006 bash[27292]: cluster 2023-08-28T01:43:50.531514+0000 mgr.y (mgr.14901) 6757 : cluster [DBG] pgmap v6764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:51.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:51 smithi130 bash[27579]: cluster 2023-08-28T01:43:50.531514+0000 mgr.y (mgr.14901) 6757 : cluster [DBG] pgmap v6764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:52.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:51 smithi130 bash[51302]: debug 2023-08-28T01:43:51.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:51.881502+0000) 2023-08-28T01:43:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:43:52.669Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:43:52.671Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:53.210 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:52 smithi130 bash[51302]: debug 2023-08-28T01:43:52.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:52.881772+0000) 2023-08-28T01:43:53.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:53 smithi006 bash[33918]: audit 2023-08-28T01:43:52.353322+0000 mon.c (mon.2) 2666 : audit [DBG] from='client.? 172.21.15.6:0/719830273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:53.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:53 smithi006 bash[27292]: audit 2023-08-28T01:43:52.353322+0000 mon.c (mon.2) 2666 : audit [DBG] from='client.? 172.21.15.6:0/719830273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:53.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:53 smithi130 bash[27579]: audit 2023-08-28T01:43:52.353322+0000 mon.c (mon.2) 2666 : audit [DBG] from='client.? 172.21.15.6:0/719830273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:54.213 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:53 smithi130 bash[51302]: debug 2023-08-28T01:43:53.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:53.882040+0000) 2023-08-28T01:43:54.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:54 smithi006 bash[27292]: cluster 2023-08-28T01:43:52.532197+0000 mgr.y (mgr.14901) 6758 : cluster [DBG] pgmap v6765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:54 smithi006 bash[33918]: cluster 2023-08-28T01:43:52.532197+0000 mgr.y (mgr.14901) 6758 : cluster [DBG] pgmap v6765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:54 smithi130 bash[27579]: cluster 2023-08-28T01:43:52.532197+0000 mgr.y (mgr.14901) 6758 : cluster [DBG] pgmap v6765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:43:54.552Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:43:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:43:54.554Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:43:55.222 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:54 smithi130 bash[51302]: debug 2023-08-28T01:43:54.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:54.882304+0000) 2023-08-28T01:43:55.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:55 smithi130 bash[27579]: audit 2023-08-28T01:43:54.770885+0000 mon.c (mon.2) 2667 : audit [DBG] from='client.? 172.21.15.6:0/3146387464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:55.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:55 smithi006 bash[27292]: audit 2023-08-28T01:43:54.770885+0000 mon.c (mon.2) 2667 : audit [DBG] from='client.? 172.21.15.6:0/3146387464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:55.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:55 smithi006 bash[33918]: audit 2023-08-28T01:43:54.770885+0000 mon.c (mon.2) 2667 : audit [DBG] from='client.? 172.21.15.6:0/3146387464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:56.225 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:55 smithi130 bash[51302]: debug 2023-08-28T01:43:55.875+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:55.882571+0000) 2023-08-28T01:43:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:56 smithi130 bash[27579]: cluster 2023-08-28T01:43:54.533280+0000 mgr.y (mgr.14901) 6759 : cluster [DBG] pgmap v6766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:56.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:56 smithi006 bash[33918]: cluster 2023-08-28T01:43:54.533280+0000 mgr.y (mgr.14901) 6759 : cluster [DBG] pgmap v6766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:56.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:56 smithi006 bash[27292]: cluster 2023-08-28T01:43:54.533280+0000 mgr.y (mgr.14901) 6759 : cluster [DBG] pgmap v6766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:57.232 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:56 smithi130 bash[51302]: debug 2023-08-28T01:43:56.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:56.882891+0000) 2023-08-28T01:43:57.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:57 smithi130 bash[27579]: audit 2023-08-28T01:43:57.177877+0000 mon.c (mon.2) 2668 : audit [DBG] from='client.? 172.21.15.6:0/1551516318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:57.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:57 smithi006 bash[27292]: audit 2023-08-28T01:43:57.177877+0000 mon.c (mon.2) 2668 : audit [DBG] from='client.? 172.21.15.6:0/1551516318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:57.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:57 smithi006 bash[33918]: audit 2023-08-28T01:43:57.177877+0000 mon.c (mon.2) 2668 : audit [DBG] from='client.? 172.21.15.6:0/1551516318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:58.235 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:57 smithi130 bash[51302]: debug 2023-08-28T01:43:57.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:57.883164+0000) 2023-08-28T01:43:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:43:58 smithi130 bash[27579]: cluster 2023-08-28T01:43:56.533964+0000 mgr.y (mgr.14901) 6760 : cluster [DBG] pgmap v6767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:58.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:43:58 smithi006 bash[33918]: cluster 2023-08-28T01:43:56.533964+0000 mgr.y (mgr.14901) 6760 : cluster [DBG] pgmap v6767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:58.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:43:58 smithi006 bash[27292]: cluster 2023-08-28T01:43:56.533964+0000 mgr.y (mgr.14901) 6760 : cluster [DBG] pgmap v6767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:43:59.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:58 smithi130 bash[51302]: debug 2023-08-28T01:43:58.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:58.883444+0000) 2023-08-28T01:44:00.241 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:43:59 smithi130 bash[51302]: debug 2023-08-28T01:43:59.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:59.883697+0000) 2023-08-28T01:44:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:00 smithi130 bash[27579]: cluster 2023-08-28T01:43:58.534594+0000 mgr.y (mgr.14901) 6761 : cluster [DBG] pgmap v6768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:00 smithi130 bash[27579]: audit 2023-08-28T01:43:59.594318+0000 mon.c (mon.2) 2669 : audit [DBG] from='client.? 172.21.15.6:0/78104217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:00 smithi130 bash[27579]: audit 2023-08-28T01:43:59.704142+0000 mon.b (mon.1) 763 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:44:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:00 smithi006 bash[33918]: cluster 2023-08-28T01:43:58.534594+0000 mgr.y (mgr.14901) 6761 : cluster [DBG] pgmap v6768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:00 smithi006 bash[33918]: audit 2023-08-28T01:43:59.594318+0000 mon.c (mon.2) 2669 : audit [DBG] from='client.? 172.21.15.6:0/78104217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:00 smithi006 bash[33918]: audit 2023-08-28T01:43:59.704142+0000 mon.b (mon.1) 763 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:44:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:00 smithi006 bash[27292]: cluster 2023-08-28T01:43:58.534594+0000 mgr.y (mgr.14901) 6761 : cluster [DBG] pgmap v6768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:00 smithi006 bash[27292]: audit 2023-08-28T01:43:59.594318+0000 mon.c (mon.2) 2669 : audit [DBG] from='client.? 172.21.15.6:0/78104217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:00 smithi006 bash[27292]: audit 2023-08-28T01:43:59.704142+0000 mon.b (mon.1) 763 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:44:01.100 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:44:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:44:00] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:44:01.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:00 smithi130 bash[51302]: debug 2023-08-28T01:44:00.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:00.884030+0000) 2023-08-28T01:44:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:44:01.100Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:44:01.102Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:02.245 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:01 smithi130 bash[51302]: debug 2023-08-28T01:44:01.883+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:01.884327+0000) 2023-08-28T01:44:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:02 smithi130 bash[27579]: cluster 2023-08-28T01:44:00.535685+0000 mgr.y (mgr.14901) 6762 : cluster [DBG] pgmap v6769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:02 smithi130 bash[27579]: audit 2023-08-28T01:44:02.014670+0000 mon.a (mon.0) 4428 : audit [DBG] from='client.? 172.21.15.6:0/1262483081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:02.671 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:02 smithi006 bash[27292]: cluster 2023-08-28T01:44:00.535685+0000 mgr.y (mgr.14901) 6762 : cluster [DBG] pgmap v6769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:02.671 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:02 smithi006 bash[27292]: audit 2023-08-28T01:44:02.014670+0000 mon.a (mon.0) 4428 : audit [DBG] from='client.? 172.21.15.6:0/1262483081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:02.671 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:02 smithi006 bash[33918]: cluster 2023-08-28T01:44:00.535685+0000 mgr.y (mgr.14901) 6762 : cluster [DBG] pgmap v6769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:02.671 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:02 smithi006 bash[33918]: audit 2023-08-28T01:44:02.014670+0000 mon.a (mon.0) 4428 : audit [DBG] from='client.? 172.21.15.6:0/1262483081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:44:02.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:44:02.673Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:03.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:02 smithi130 bash[51302]: debug 2023-08-28T01:44:02.883+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:02.884627+0000) 2023-08-28T01:44:04.250 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:03 smithi130 bash[51302]: debug 2023-08-28T01:44:03.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:03.884899+0000) 2023-08-28T01:44:04.553 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:04 smithi006 bash[33918]: cluster 2023-08-28T01:44:02.536315+0000 mgr.y (mgr.14901) 6763 : cluster [DBG] pgmap v6770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:04.553 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:04 smithi006 bash[27292]: cluster 2023-08-28T01:44:02.536315+0000 mgr.y (mgr.14901) 6763 : cluster [DBG] pgmap v6770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:04 smithi130 bash[27579]: cluster 2023-08-28T01:44:02.536315+0000 mgr.y (mgr.14901) 6763 : cluster [DBG] pgmap v6770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:44:04.553Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:44:04.555Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:05.256 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:04 smithi130 bash[51302]: debug 2023-08-28T01:44:04.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:04.885135+0000) 2023-08-28T01:44:05.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:05 smithi130 bash[27579]: audit 2023-08-28T01:44:04.426408+0000 mon.a (mon.0) 4429 : audit [DBG] from='client.? 172.21.15.6:0/1673596861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:05.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:05 smithi006 bash[27292]: audit 2023-08-28T01:44:04.426408+0000 mon.a (mon.0) 4429 : audit [DBG] from='client.? 172.21.15.6:0/1673596861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:05.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:05 smithi006 bash[33918]: audit 2023-08-28T01:44:04.426408+0000 mon.a (mon.0) 4429 : audit [DBG] from='client.? 172.21.15.6:0/1673596861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:06.260 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:05 smithi130 bash[51302]: debug 2023-08-28T01:44:05.879+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:05.885403+0000) 2023-08-28T01:44:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:06 smithi130 bash[27579]: cluster 2023-08-28T01:44:04.537473+0000 mgr.y (mgr.14901) 6764 : cluster [DBG] pgmap v6771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:06 smithi130 bash[27579]: audit 2023-08-28T01:44:05.635211+0000 mon.a (mon.0) 4430 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:44:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:06 smithi130 bash[27579]: audit 2023-08-28T01:44:06.079800+0000 mon.a (mon.0) 4431 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:44:06.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:06 smithi006 bash[27292]: cluster 2023-08-28T01:44:04.537473+0000 mgr.y (mgr.14901) 6764 : cluster [DBG] pgmap v6771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:06.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:06 smithi006 bash[27292]: audit 2023-08-28T01:44:05.635211+0000 mon.a (mon.0) 4430 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:44:06.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:06 smithi006 bash[27292]: audit 2023-08-28T01:44:06.079800+0000 mon.a (mon.0) 4431 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:44:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:06 smithi006 bash[33918]: cluster 2023-08-28T01:44:04.537473+0000 mgr.y (mgr.14901) 6764 : cluster [DBG] pgmap v6771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:06 smithi006 bash[33918]: audit 2023-08-28T01:44:05.635211+0000 mon.a (mon.0) 4430 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:44:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:06 smithi006 bash[33918]: audit 2023-08-28T01:44:06.079800+0000 mon.a (mon.0) 4431 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:44:07.262 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:06 smithi130 bash[51302]: debug 2023-08-28T01:44:06.883+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:06.885725+0000) 2023-08-28T01:44:07.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:07 smithi130 bash[27579]: audit 2023-08-28T01:44:06.376151+0000 mon.b (mon.1) 764 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:44:07.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:07 smithi130 bash[27579]: audit 2023-08-28T01:44:06.378364+0000 mon.b (mon.1) 765 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:44:07.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:07 smithi130 bash[27579]: audit 2023-08-28T01:44:06.386598+0000 mon.a (mon.0) 4432 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:44:07.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:07 smithi130 bash[27579]: audit 2023-08-28T01:44:06.835892+0000 mon.c (mon.2) 2670 : audit [DBG] from='client.? 172.21.15.6:0/667377110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:07.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:07 smithi006 bash[33918]: audit 2023-08-28T01:44:06.376151+0000 mon.b (mon.1) 764 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:44:07.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:07 smithi006 bash[33918]: audit 2023-08-28T01:44:06.378364+0000 mon.b (mon.1) 765 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:44:07.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:07 smithi006 bash[33918]: audit 2023-08-28T01:44:06.386598+0000 mon.a (mon.0) 4432 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:44:07.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:07 smithi006 bash[33918]: audit 2023-08-28T01:44:06.835892+0000 mon.c (mon.2) 2670 : audit [DBG] from='client.? 172.21.15.6:0/667377110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:07.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:07 smithi006 bash[27292]: audit 2023-08-28T01:44:06.376151+0000 mon.b (mon.1) 764 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:44:07.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:07 smithi006 bash[27292]: audit 2023-08-28T01:44:06.378364+0000 mon.b (mon.1) 765 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:44:07.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:07 smithi006 bash[27292]: audit 2023-08-28T01:44:06.386598+0000 mon.a (mon.0) 4432 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:44:07.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:07 smithi006 bash[27292]: audit 2023-08-28T01:44:06.835892+0000 mon.c (mon.2) 2670 : audit [DBG] from='client.? 172.21.15.6:0/667377110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:08.265 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:07 smithi130 bash[51302]: debug 2023-08-28T01:44:07.883+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:07.886003+0000) 2023-08-28T01:44:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:08 smithi130 bash[27579]: cluster 2023-08-28T01:44:06.537894+0000 mgr.y (mgr.14901) 6765 : cluster [DBG] pgmap v6772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:08 smithi006 bash[33918]: cluster 2023-08-28T01:44:06.537894+0000 mgr.y (mgr.14901) 6765 : cluster [DBG] pgmap v6772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:08 smithi006 bash[27292]: cluster 2023-08-28T01:44:06.537894+0000 mgr.y (mgr.14901) 6765 : cluster [DBG] pgmap v6772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:09.273 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:08 smithi130 bash[51302]: debug 2023-08-28T01:44:08.883+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:08.886280+0000) 2023-08-28T01:44:09.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:09 smithi130 bash[27579]: audit 2023-08-28T01:44:09.249230+0000 mon.c (mon.2) 2671 : audit [DBG] from='client.? 172.21.15.6:0/1276615714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:09.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:09 smithi006 bash[33918]: audit 2023-08-28T01:44:09.249230+0000 mon.c (mon.2) 2671 : audit [DBG] from='client.? 172.21.15.6:0/1276615714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:09.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:09 smithi006 bash[27292]: audit 2023-08-28T01:44:09.249230+0000 mon.c (mon.2) 2671 : audit [DBG] from='client.? 172.21.15.6:0/1276615714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:10.274 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:09 smithi130 bash[51302]: debug 2023-08-28T01:44:09.883+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:09.886566+0000) 2023-08-28T01:44:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:10 smithi130 bash[27579]: cluster 2023-08-28T01:44:08.538504+0000 mgr.y (mgr.14901) 6766 : cluster [DBG] pgmap v6773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:10 smithi006 bash[33918]: cluster 2023-08-28T01:44:08.538504+0000 mgr.y (mgr.14901) 6766 : cluster [DBG] pgmap v6773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:10.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:10 smithi006 bash[27292]: cluster 2023-08-28T01:44:08.538504+0000 mgr.y (mgr.14901) 6766 : cluster [DBG] pgmap v6773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:11.100 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:44:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:44:10] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:44:11.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:10 smithi130 bash[51302]: debug 2023-08-28T01:44:10.883+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:10.886919+0000) 2023-08-28T01:44:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:44:11.100Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:44:11.102Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:12.281 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:11 smithi130 bash[51302]: debug 2023-08-28T01:44:11.883+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:11.887205+0000) 2023-08-28T01:44:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:12 smithi130 bash[27579]: cluster 2023-08-28T01:44:10.539693+0000 mgr.y (mgr.14901) 6767 : cluster [DBG] pgmap v6774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:12 smithi130 bash[27579]: audit 2023-08-28T01:44:11.657658+0000 mon.c (mon.2) 2672 : audit [DBG] from='client.? 172.21.15.6:0/3996058001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:12.671 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:12 smithi006 bash[33918]: cluster 2023-08-28T01:44:10.539693+0000 mgr.y (mgr.14901) 6767 : cluster [DBG] pgmap v6774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:12.671 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:12 smithi006 bash[33918]: audit 2023-08-28T01:44:11.657658+0000 mon.c (mon.2) 2672 : audit [DBG] from='client.? 172.21.15.6:0/3996058001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:12.671 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:12 smithi006 bash[27292]: cluster 2023-08-28T01:44:10.539693+0000 mgr.y (mgr.14901) 6767 : cluster [DBG] pgmap v6774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:12.671 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:12 smithi006 bash[27292]: audit 2023-08-28T01:44:11.657658+0000 mon.c (mon.2) 2672 : audit [DBG] from='client.? 172.21.15.6:0/3996058001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:44:12.671Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:44:12.673Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:13.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:12 smithi130 bash[51302]: debug 2023-08-28T01:44:12.883+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:12.887482+0000) 2023-08-28T01:44:14.285 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:13 smithi130 bash[51302]: debug 2023-08-28T01:44:13.887+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:13.887758+0000) 2023-08-28T01:44:14.554 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:14 smithi006 bash[33918]: cluster 2023-08-28T01:44:12.540374+0000 mgr.y (mgr.14901) 6768 : cluster [DBG] pgmap v6775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:14.554 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:14 smithi006 bash[33918]: audit 2023-08-28T01:44:14.062761+0000 mon.c (mon.2) 2673 : audit [DBG] from='client.? 172.21.15.6:0/812179225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:14.555 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:14 smithi006 bash[27292]: cluster 2023-08-28T01:44:12.540374+0000 mgr.y (mgr.14901) 6768 : cluster [DBG] pgmap v6775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:14.555 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:14 smithi006 bash[27292]: audit 2023-08-28T01:44:14.062761+0000 mon.c (mon.2) 2673 : audit [DBG] from='client.? 172.21.15.6:0/812179225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:14.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:14 smithi130 bash[27579]: cluster 2023-08-28T01:44:12.540374+0000 mgr.y (mgr.14901) 6768 : cluster [DBG] pgmap v6775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:14 smithi130 bash[27579]: audit 2023-08-28T01:44:14.062761+0000 mon.c (mon.2) 2673 : audit [DBG] from='client.? 172.21.15.6:0/812179225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:44:14.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:44:14.556Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:15.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:14 smithi130 bash[51302]: debug 2023-08-28T01:44:14.883+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:14.888068+0000) 2023-08-28T01:44:16.291 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:15 smithi130 bash[51302]: debug 2023-08-28T01:44:15.887+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:15.888352+0000) 2023-08-28T01:44:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:16 smithi130 bash[27579]: cluster 2023-08-28T01:44:14.541586+0000 mgr.y (mgr.14901) 6769 : cluster [DBG] pgmap v6776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:16.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:16 smithi006 bash[27292]: cluster 2023-08-28T01:44:14.541586+0000 mgr.y (mgr.14901) 6769 : cluster [DBG] pgmap v6776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:16 smithi006 bash[33918]: cluster 2023-08-28T01:44:14.541586+0000 mgr.y (mgr.14901) 6769 : cluster [DBG] pgmap v6776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:17.301 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:16 smithi130 bash[51302]: debug 2023-08-28T01:44:16.887+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:16.888616+0000) 2023-08-28T01:44:17.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:17 smithi130 bash[27579]: audit 2023-08-28T01:44:16.476480+0000 mon.a (mon.0) 4433 : audit [DBG] from='client.? 172.21.15.6:0/3447213307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:17.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:17 smithi006 bash[27292]: audit 2023-08-28T01:44:16.476480+0000 mon.a (mon.0) 4433 : audit [DBG] from='client.? 172.21.15.6:0/3447213307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:17.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:17 smithi006 bash[33918]: audit 2023-08-28T01:44:16.476480+0000 mon.a (mon.0) 4433 : audit [DBG] from='client.? 172.21.15.6:0/3447213307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:18.304 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:17 smithi130 bash[51302]: debug 2023-08-28T01:44:17.887+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:17.888938+0000) 2023-08-28T01:44:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:18 smithi130 bash[27579]: cluster 2023-08-28T01:44:16.542292+0000 mgr.y (mgr.14901) 6770 : cluster [DBG] pgmap v6777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:18 smithi006 bash[27292]: cluster 2023-08-28T01:44:16.542292+0000 mgr.y (mgr.14901) 6770 : cluster [DBG] pgmap v6777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:18.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:18 smithi006 bash[33918]: cluster 2023-08-28T01:44:16.542292+0000 mgr.y (mgr.14901) 6770 : cluster [DBG] pgmap v6777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:19.315 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:18 smithi130 bash[51302]: debug 2023-08-28T01:44:18.887+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:18.889210+0000) 2023-08-28T01:44:19.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:19 smithi130 bash[27579]: audit 2023-08-28T01:44:18.890930+0000 mon.c (mon.2) 2674 : audit [DBG] from='client.? 172.21.15.6:0/2278229209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:19.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:19 smithi006 bash[33918]: audit 2023-08-28T01:44:18.890930+0000 mon.c (mon.2) 2674 : audit [DBG] from='client.? 172.21.15.6:0/2278229209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:19.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:19 smithi006 bash[27292]: audit 2023-08-28T01:44:18.890930+0000 mon.c (mon.2) 2674 : audit [DBG] from='client.? 172.21.15.6:0/2278229209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:20.318 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:19 smithi130 bash[51302]: debug 2023-08-28T01:44:19.887+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:19.889532+0000) 2023-08-28T01:44:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:20 smithi130 bash[27579]: cluster 2023-08-28T01:44:18.542959+0000 mgr.y (mgr.14901) 6771 : cluster [DBG] pgmap v6778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:20.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:20 smithi006 bash[33918]: cluster 2023-08-28T01:44:18.542959+0000 mgr.y (mgr.14901) 6771 : cluster [DBG] pgmap v6778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:20.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:20 smithi006 bash[27292]: cluster 2023-08-28T01:44:18.542959+0000 mgr.y (mgr.14901) 6771 : cluster [DBG] pgmap v6778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:21.100 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:44:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:44:20] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:44:21.325 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:20 smithi130 bash[51302]: debug 2023-08-28T01:44:20.887+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:20.889826+0000) 2023-08-28T01:44:21.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:21 smithi006 bash[33918]: audit 2023-08-28T01:44:21.314395+0000 mon.a (mon.0) 4434 : audit [DBG] from='client.? 172.21.15.6:0/1430672730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:21.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:21 smithi006 bash[27292]: audit 2023-08-28T01:44:21.314395+0000 mon.a (mon.0) 4434 : audit [DBG] from='client.? 172.21.15.6:0/1430672730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:21.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:44:21.100Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:21.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:44:21.102Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:21.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:21 smithi130 bash[27579]: audit 2023-08-28T01:44:21.314395+0000 mon.a (mon.0) 4434 : audit [DBG] from='client.? 172.21.15.6:0/1430672730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:22.328 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:21 smithi130 bash[51302]: debug 2023-08-28T01:44:21.883+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:21.890111+0000) 2023-08-28T01:44:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:22 smithi130 bash[27579]: cluster 2023-08-28T01:44:20.544170+0000 mgr.y (mgr.14901) 6772 : cluster [DBG] pgmap v6779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:22.671 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:22 smithi006 bash[33918]: cluster 2023-08-28T01:44:20.544170+0000 mgr.y (mgr.14901) 6772 : cluster [DBG] pgmap v6779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:22.672 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:22 smithi006 bash[27292]: cluster 2023-08-28T01:44:20.544170+0000 mgr.y (mgr.14901) 6772 : cluster [DBG] pgmap v6779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:44:22.671Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:44:22.673Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:23.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:22 smithi130 bash[51302]: debug 2023-08-28T01:44:22.887+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:22.890423+0000) 2023-08-28T01:44:24.334 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:23 smithi130 bash[51302]: debug 2023-08-28T01:44:23.887+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:23.890701+0000) 2023-08-28T01:44:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:24 smithi130 bash[27579]: cluster 2023-08-28T01:44:22.544902+0000 mgr.y (mgr.14901) 6773 : cluster [DBG] pgmap v6780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:24 smithi130 bash[27579]: audit 2023-08-28T01:44:23.733583+0000 mon.a (mon.0) 4435 : audit [DBG] from='client.? 172.21.15.6:0/3447143406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:24.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:24 smithi006 bash[33918]: cluster 2023-08-28T01:44:22.544902+0000 mgr.y (mgr.14901) 6773 : cluster [DBG] pgmap v6780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:24.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:24 smithi006 bash[33918]: audit 2023-08-28T01:44:23.733583+0000 mon.a (mon.0) 4435 : audit [DBG] from='client.? 172.21.15.6:0/3447143406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:44:24.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:44:24.557Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:24.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:24 smithi006 bash[27292]: cluster 2023-08-28T01:44:22.544902+0000 mgr.y (mgr.14901) 6773 : cluster [DBG] pgmap v6780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:24.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:24 smithi006 bash[27292]: audit 2023-08-28T01:44:23.733583+0000 mon.a (mon.0) 4435 : audit [DBG] from='client.? 172.21.15.6:0/3447143406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:25.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:24 smithi130 bash[51302]: debug 2023-08-28T01:44:24.887+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:24.890985+0000) 2023-08-28T01:44:26.339 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:25 smithi130 bash[51302]: debug 2023-08-28T01:44:25.887+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:25.891253+0000) 2023-08-28T01:44:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:26 smithi130 bash[27579]: cluster 2023-08-28T01:44:24.546005+0000 mgr.y (mgr.14901) 6774 : cluster [DBG] pgmap v6781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:26 smithi130 bash[27579]: audit 2023-08-28T01:44:26.157732+0000 mon.c (mon.2) 2675 : audit [DBG] from='client.? 172.21.15.6:0/484404056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:26.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:26 smithi006 bash[33918]: cluster 2023-08-28T01:44:24.546005+0000 mgr.y (mgr.14901) 6774 : cluster [DBG] pgmap v6781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:26 smithi006 bash[33918]: audit 2023-08-28T01:44:26.157732+0000 mon.c (mon.2) 2675 : audit [DBG] from='client.? 172.21.15.6:0/484404056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:26 smithi006 bash[27292]: cluster 2023-08-28T01:44:24.546005+0000 mgr.y (mgr.14901) 6774 : cluster [DBG] pgmap v6781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:26 smithi006 bash[27292]: audit 2023-08-28T01:44:26.157732+0000 mon.c (mon.2) 2675 : audit [DBG] from='client.? 172.21.15.6:0/484404056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:27.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:26 smithi130 bash[51302]: debug 2023-08-28T01:44:26.887+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:26.891476+0000) 2023-08-28T01:44:28.343 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:27 smithi130 bash[51302]: debug 2023-08-28T01:44:27.891+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:27.891753+0000) 2023-08-28T01:44:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:28 smithi130 bash[27579]: cluster 2023-08-28T01:44:26.546695+0000 mgr.y (mgr.14901) 6775 : cluster [DBG] pgmap v6782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:28.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:28 smithi006 bash[27292]: cluster 2023-08-28T01:44:26.546695+0000 mgr.y (mgr.14901) 6775 : cluster [DBG] pgmap v6782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:28 smithi006 bash[33918]: cluster 2023-08-28T01:44:26.546695+0000 mgr.y (mgr.14901) 6775 : cluster [DBG] pgmap v6782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:29.353 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:28 smithi130 bash[51302]: debug 2023-08-28T01:44:28.891+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:28.892070+0000) 2023-08-28T01:44:29.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:29 smithi130 bash[27579]: audit 2023-08-28T01:44:28.579221+0000 mon.a (mon.0) 4436 : audit [DBG] from='client.? 172.21.15.6:0/1832358490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:29.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:29 smithi006 bash[27292]: audit 2023-08-28T01:44:28.579221+0000 mon.a (mon.0) 4436 : audit [DBG] from='client.? 172.21.15.6:0/1832358490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:29.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:29 smithi006 bash[33918]: audit 2023-08-28T01:44:28.579221+0000 mon.a (mon.0) 4436 : audit [DBG] from='client.? 172.21.15.6:0/1832358490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:30.354 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:29 smithi130 bash[51302]: debug 2023-08-28T01:44:29.891+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:29.892354+0000) 2023-08-28T01:44:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:30 smithi130 bash[27579]: cluster 2023-08-28T01:44:28.547283+0000 mgr.y (mgr.14901) 6776 : cluster [DBG] pgmap v6783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:30.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:30 smithi006 bash[27292]: cluster 2023-08-28T01:44:28.547283+0000 mgr.y (mgr.14901) 6776 : cluster [DBG] pgmap v6783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:30 smithi006 bash[33918]: cluster 2023-08-28T01:44:28.547283+0000 mgr.y (mgr.14901) 6776 : cluster [DBG] pgmap v6783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:31.102 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:44:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:44:30] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:44:31.362 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:44:31.102Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:31.436 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:44:31.104Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:31.436 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:30 smithi130 bash[51302]: debug 2023-08-28T01:44:30.891+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:30.892645+0000) 2023-08-28T01:44:31.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:31 smithi130 bash[27579]: audit 2023-08-28T01:44:30.995841+0000 mon.a (mon.0) 4437 : audit [DBG] from='client.? 172.21.15.6:0/3251658787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:31.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:31 smithi006 bash[27292]: audit 2023-08-28T01:44:30.995841+0000 mon.a (mon.0) 4437 : audit [DBG] from='client.? 172.21.15.6:0/3251658787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:31.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:31 smithi006 bash[33918]: audit 2023-08-28T01:44:30.995841+0000 mon.a (mon.0) 4437 : audit [DBG] from='client.? 172.21.15.6:0/3251658787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:32.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:31 smithi130 bash[51302]: debug 2023-08-28T01:44:31.891+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:31.892933+0000) 2023-08-28T01:44:32.672 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:32 smithi006 bash[33918]: cluster 2023-08-28T01:44:30.548765+0000 mgr.y (mgr.14901) 6777 : cluster [DBG] pgmap v6784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:32.672 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:32 smithi006 bash[27292]: cluster 2023-08-28T01:44:30.548765+0000 mgr.y (mgr.14901) 6777 : cluster [DBG] pgmap v6784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:32 smithi130 bash[27579]: cluster 2023-08-28T01:44:30.548765+0000 mgr.y (mgr.14901) 6777 : cluster [DBG] pgmap v6784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:44:32.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:44:32.674Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:33.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:32 smithi130 bash[51302]: debug 2023-08-28T01:44:32.887+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:32.893201+0000) 2023-08-28T01:44:34.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:33 smithi130 bash[51302]: debug 2023-08-28T01:44:33.891+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:33.893477+0000) 2023-08-28T01:44:34.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:34 smithi006 bash[27292]: cluster 2023-08-28T01:44:32.549591+0000 mgr.y (mgr.14901) 6778 : cluster [DBG] pgmap v6785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:34.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:34 smithi006 bash[27292]: audit 2023-08-28T01:44:33.412071+0000 mon.c (mon.2) 2676 : audit [DBG] from='client.? 172.21.15.6:0/2328474107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:44:34.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:44:34.558Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:34.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:34 smithi006 bash[33918]: cluster 2023-08-28T01:44:32.549591+0000 mgr.y (mgr.14901) 6778 : cluster [DBG] pgmap v6785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:34.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:34 smithi006 bash[33918]: audit 2023-08-28T01:44:33.412071+0000 mon.c (mon.2) 2676 : audit [DBG] from='client.? 172.21.15.6:0/2328474107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:34.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:34 smithi130 bash[27579]: cluster 2023-08-28T01:44:32.549591+0000 mgr.y (mgr.14901) 6778 : cluster [DBG] pgmap v6785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:34.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:34 smithi130 bash[27579]: audit 2023-08-28T01:44:33.412071+0000 mon.c (mon.2) 2676 : audit [DBG] from='client.? 172.21.15.6:0/2328474107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:35.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:34 smithi130 bash[51302]: debug 2023-08-28T01:44:34.887+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:34.893742+0000) 2023-08-28T01:44:35.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:35 smithi006 bash[27292]: cluster 2023-08-28T01:44:34.550729+0000 mgr.y (mgr.14901) 6779 : cluster [DBG] pgmap v6786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:35.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:35 smithi006 bash[33918]: cluster 2023-08-28T01:44:34.550729+0000 mgr.y (mgr.14901) 6779 : cluster [DBG] pgmap v6786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:35.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:35 smithi130 bash[27579]: cluster 2023-08-28T01:44:34.550729+0000 mgr.y (mgr.14901) 6779 : cluster [DBG] pgmap v6786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:36.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:35 smithi130 bash[51302]: debug 2023-08-28T01:44:35.887+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:35.894005+0000) 2023-08-28T01:44:36.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:36 smithi006 bash[33918]: audit 2023-08-28T01:44:35.828620+0000 mon.a (mon.0) 4438 : audit [DBG] from='client.? 172.21.15.6:0/2805089463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:36.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:36 smithi006 bash[27292]: audit 2023-08-28T01:44:35.828620+0000 mon.a (mon.0) 4438 : audit [DBG] from='client.? 172.21.15.6:0/2805089463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:36.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:36 smithi130 bash[27579]: audit 2023-08-28T01:44:35.828620+0000 mon.a (mon.0) 4438 : audit [DBG] from='client.? 172.21.15.6:0/2805089463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:37.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:36 smithi130 bash[51302]: debug 2023-08-28T01:44:36.887+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:36.894287+0000) 2023-08-28T01:44:38.215 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:37 smithi130 bash[51302]: debug 2023-08-28T01:44:37.890+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:37.894550+0000) 2023-08-28T01:44:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:38 smithi130 bash[27579]: cluster 2023-08-28T01:44:36.551471+0000 mgr.y (mgr.14901) 6780 : cluster [DBG] pgmap v6787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:38 smithi006 bash[33918]: cluster 2023-08-28T01:44:36.551471+0000 mgr.y (mgr.14901) 6780 : cluster [DBG] pgmap v6787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:38 smithi006 bash[27292]: cluster 2023-08-28T01:44:36.551471+0000 mgr.y (mgr.14901) 6780 : cluster [DBG] pgmap v6787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:39.228 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:38 smithi130 bash[51302]: debug 2023-08-28T01:44:38.890+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:38.894874+0000) 2023-08-28T01:44:39.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:39 smithi130 bash[27579]: audit 2023-08-28T01:44:38.242040+0000 mon.a (mon.0) 4439 : audit [DBG] from='client.? 172.21.15.6:0/2103490036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:39.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:39 smithi006 bash[33918]: audit 2023-08-28T01:44:38.242040+0000 mon.a (mon.0) 4439 : audit [DBG] from='client.? 172.21.15.6:0/2103490036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:39.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:39 smithi006 bash[27292]: audit 2023-08-28T01:44:38.242040+0000 mon.a (mon.0) 4439 : audit [DBG] from='client.? 172.21.15.6:0/2103490036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:40.231 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:39 smithi130 bash[51302]: debug 2023-08-28T01:44:39.890+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:39.895124+0000) 2023-08-28T01:44:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:40 smithi130 bash[27579]: cluster 2023-08-28T01:44:38.552047+0000 mgr.y (mgr.14901) 6781 : cluster [DBG] pgmap v6788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:40.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:40 smithi006 bash[27292]: cluster 2023-08-28T01:44:38.552047+0000 mgr.y (mgr.14901) 6781 : cluster [DBG] pgmap v6788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:40 smithi006 bash[33918]: cluster 2023-08-28T01:44:38.552047+0000 mgr.y (mgr.14901) 6781 : cluster [DBG] pgmap v6788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:41.103 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:44:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:44:40] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:44:41.240 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:40 smithi130 bash[51302]: debug 2023-08-28T01:44:40.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:40.895452+0000) 2023-08-28T01:44:41.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:41 smithi006 bash[33918]: audit 2023-08-28T01:44:40.661437+0000 mon.a (mon.0) 4440 : audit [DBG] from='client.? 172.21.15.6:0/3828783169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:41.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:41 smithi006 bash[27292]: audit 2023-08-28T01:44:40.661437+0000 mon.a (mon.0) 4440 : audit [DBG] from='client.? 172.21.15.6:0/3828783169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:44:41.103Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:44:41.105Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:41.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:41 smithi130 bash[27579]: audit 2023-08-28T01:44:40.661437+0000 mon.a (mon.0) 4440 : audit [DBG] from='client.? 172.21.15.6:0/3828783169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:42.244 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:41 smithi130 bash[51302]: debug 2023-08-28T01:44:41.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:41.895737+0000) 2023-08-28T01:44:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:42 smithi130 bash[27579]: cluster 2023-08-28T01:44:40.553195+0000 mgr.y (mgr.14901) 6782 : cluster [DBG] pgmap v6789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:42.673 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:42 smithi006 bash[33918]: cluster 2023-08-28T01:44:40.553195+0000 mgr.y (mgr.14901) 6782 : cluster [DBG] pgmap v6789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:42.673 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:42 smithi006 bash[27292]: cluster 2023-08-28T01:44:40.553195+0000 mgr.y (mgr.14901) 6782 : cluster [DBG] pgmap v6789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:44:42.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:44:42.675Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:43.252 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:42 smithi130 bash[51302]: debug 2023-08-28T01:44:42.890+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:42.896023+0000) 2023-08-28T01:44:43.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:43 smithi130 bash[27579]: audit 2023-08-28T01:44:43.079597+0000 mon.c (mon.2) 2677 : audit [DBG] from='client.? 172.21.15.6:0/2210593350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:43.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:43 smithi006 bash[27292]: audit 2023-08-28T01:44:43.079597+0000 mon.c (mon.2) 2677 : audit [DBG] from='client.? 172.21.15.6:0/2210593350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:43.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:43 smithi006 bash[33918]: audit 2023-08-28T01:44:43.079597+0000 mon.c (mon.2) 2677 : audit [DBG] from='client.? 172.21.15.6:0/2210593350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:44.255 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:43 smithi130 bash[51302]: debug 2023-08-28T01:44:43.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:43.896340+0000) 2023-08-28T01:44:44.557 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:44 smithi006 bash[27292]: cluster 2023-08-28T01:44:42.553914+0000 mgr.y (mgr.14901) 6783 : cluster [DBG] pgmap v6790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:44.557 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:44 smithi006 bash[33918]: cluster 2023-08-28T01:44:42.553914+0000 mgr.y (mgr.14901) 6783 : cluster [DBG] pgmap v6790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:44 smithi130 bash[27579]: cluster 2023-08-28T01:44:42.553914+0000 mgr.y (mgr.14901) 6783 : cluster [DBG] pgmap v6790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:44:44.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:44:44.559Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:45.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:44 smithi130 bash[51302]: debug 2023-08-28T01:44:44.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:44.896607+0000) 2023-08-28T01:44:46.260 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:45 smithi130 bash[51302]: debug 2023-08-28T01:44:45.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:45.896821+0000) 2023-08-28T01:44:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:46 smithi130 bash[27579]: cluster 2023-08-28T01:44:44.555076+0000 mgr.y (mgr.14901) 6784 : cluster [DBG] pgmap v6791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:46 smithi130 bash[27579]: audit 2023-08-28T01:44:45.499068+0000 mon.a (mon.0) 4441 : audit [DBG] from='client.? 172.21.15.6:0/3450837551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:46.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:46 smithi006 bash[33918]: cluster 2023-08-28T01:44:44.555076+0000 mgr.y (mgr.14901) 6784 : cluster [DBG] pgmap v6791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:46.731 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:46 smithi006 bash[33918]: audit 2023-08-28T01:44:45.499068+0000 mon.a (mon.0) 4441 : audit [DBG] from='client.? 172.21.15.6:0/3450837551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:46.732 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:46 smithi006 bash[27292]: cluster 2023-08-28T01:44:44.555076+0000 mgr.y (mgr.14901) 6784 : cluster [DBG] pgmap v6791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:46.732 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:46 smithi006 bash[27292]: audit 2023-08-28T01:44:45.499068+0000 mon.a (mon.0) 4441 : audit [DBG] from='client.? 172.21.15.6:0/3450837551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:47.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:46 smithi130 bash[51302]: debug 2023-08-28T01:44:46.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:46.897131+0000) 2023-08-28T01:44:48.265 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:47 smithi130 bash[51302]: debug 2023-08-28T01:44:47.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:47.897475+0000) 2023-08-28T01:44:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:48 smithi130 bash[27579]: cluster 2023-08-28T01:44:46.555698+0000 mgr.y (mgr.14901) 6785 : cluster [DBG] pgmap v6792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:48 smithi130 bash[27579]: audit 2023-08-28T01:44:47.902635+0000 mon.a (mon.0) 4442 : audit [DBG] from='client.? 172.21.15.6:0/951779360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:48.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:48 smithi006 bash[27292]: cluster 2023-08-28T01:44:46.555698+0000 mgr.y (mgr.14901) 6785 : cluster [DBG] pgmap v6792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:48.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:48 smithi006 bash[27292]: audit 2023-08-28T01:44:47.902635+0000 mon.a (mon.0) 4442 : audit [DBG] from='client.? 172.21.15.6:0/951779360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:48.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:48 smithi006 bash[33918]: cluster 2023-08-28T01:44:46.555698+0000 mgr.y (mgr.14901) 6785 : cluster [DBG] pgmap v6792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:48.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:48 smithi006 bash[33918]: audit 2023-08-28T01:44:47.902635+0000 mon.a (mon.0) 4442 : audit [DBG] from='client.? 172.21.15.6:0/951779360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:49.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:48 smithi130 bash[51302]: debug 2023-08-28T01:44:48.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:48.897785+0000) 2023-08-28T01:44:50.269 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:49 smithi130 bash[51302]: debug 2023-08-28T01:44:49.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:49.898082+0000) 2023-08-28T01:44:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:50 smithi130 bash[27579]: cluster 2023-08-28T01:44:48.556333+0000 mgr.y (mgr.14901) 6786 : cluster [DBG] pgmap v6793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:50.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:50 smithi006 bash[33918]: cluster 2023-08-28T01:44:48.556333+0000 mgr.y (mgr.14901) 6786 : cluster [DBG] pgmap v6793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:50.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:50 smithi006 bash[27292]: cluster 2023-08-28T01:44:48.556333+0000 mgr.y (mgr.14901) 6786 : cluster [DBG] pgmap v6793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:51.103 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:44:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:44:50] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:44:51.277 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:50 smithi130 bash[51302]: debug 2023-08-28T01:44:50.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:50.898449+0000) 2023-08-28T01:44:51.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:51 smithi006 bash[33918]: audit 2023-08-28T01:44:50.314745+0000 mon.a (mon.0) 4443 : audit [DBG] from='client.? 172.21.15.6:0/2479218062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:51.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:51 smithi006 bash[27292]: audit 2023-08-28T01:44:50.314745+0000 mon.a (mon.0) 4443 : audit [DBG] from='client.? 172.21.15.6:0/2479218062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:44:51.103Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:44:51.104Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:51.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:51 smithi130 bash[27579]: audit 2023-08-28T01:44:50.314745+0000 mon.a (mon.0) 4443 : audit [DBG] from='client.? 172.21.15.6:0/2479218062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:52.279 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:51 smithi130 bash[51302]: debug 2023-08-28T01:44:51.894+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:51.898753+0000) 2023-08-28T01:44:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:52 smithi130 bash[27579]: cluster 2023-08-28T01:44:50.557639+0000 mgr.y (mgr.14901) 6787 : cluster [DBG] pgmap v6794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:52.674 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:52 smithi006 bash[27292]: cluster 2023-08-28T01:44:50.557639+0000 mgr.y (mgr.14901) 6787 : cluster [DBG] pgmap v6794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:52.674 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:52 smithi006 bash[33918]: cluster 2023-08-28T01:44:50.557639+0000 mgr.y (mgr.14901) 6787 : cluster [DBG] pgmap v6794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:44:52.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:44:52.676Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:53.290 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:52 smithi130 bash[51302]: debug 2023-08-28T01:44:52.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:52.899071+0000) 2023-08-28T01:44:53.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:53 smithi130 bash[27579]: audit 2023-08-28T01:44:52.735331+0000 mon.a (mon.0) 4444 : audit [DBG] from='client.? 172.21.15.6:0/1794464034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:53.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:53 smithi006 bash[27292]: audit 2023-08-28T01:44:52.735331+0000 mon.a (mon.0) 4444 : audit [DBG] from='client.? 172.21.15.6:0/1794464034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:53.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:53 smithi006 bash[33918]: audit 2023-08-28T01:44:52.735331+0000 mon.a (mon.0) 4444 : audit [DBG] from='client.? 172.21.15.6:0/1794464034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:54.292 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:53 smithi130 bash[51302]: debug 2023-08-28T01:44:53.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:53.899336+0000) 2023-08-28T01:44:54.558 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:54 smithi006 bash[33918]: cluster 2023-08-28T01:44:52.558334+0000 mgr.y (mgr.14901) 6788 : cluster [DBG] pgmap v6795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:54.558 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:54 smithi006 bash[27292]: cluster 2023-08-28T01:44:52.558334+0000 mgr.y (mgr.14901) 6788 : cluster [DBG] pgmap v6795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:54 smithi130 bash[27579]: cluster 2023-08-28T01:44:52.558334+0000 mgr.y (mgr.14901) 6788 : cluster [DBG] pgmap v6795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:44:54.558Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:44:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:44:54.560Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:44:55.302 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:54 smithi130 bash[51302]: debug 2023-08-28T01:44:54.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:54.899636+0000) 2023-08-28T01:44:55.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:55 smithi130 bash[27579]: audit 2023-08-28T01:44:55.154277+0000 mon.a (mon.0) 4445 : audit [DBG] from='client.? 172.21.15.6:0/2244496272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:55.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:55 smithi006 bash[27292]: audit 2023-08-28T01:44:55.154277+0000 mon.a (mon.0) 4445 : audit [DBG] from='client.? 172.21.15.6:0/2244496272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:55.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:55 smithi006 bash[33918]: audit 2023-08-28T01:44:55.154277+0000 mon.a (mon.0) 4445 : audit [DBG] from='client.? 172.21.15.6:0/2244496272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:56.304 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:55 smithi130 bash[51302]: debug 2023-08-28T01:44:55.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:55.899853+0000) 2023-08-28T01:44:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:56 smithi130 bash[27579]: cluster 2023-08-28T01:44:54.559315+0000 mgr.y (mgr.14901) 6789 : cluster [DBG] pgmap v6796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:56.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:56 smithi006 bash[33918]: cluster 2023-08-28T01:44:54.559315+0000 mgr.y (mgr.14901) 6789 : cluster [DBG] pgmap v6796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:56.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:56 smithi006 bash[27292]: cluster 2023-08-28T01:44:54.559315+0000 mgr.y (mgr.14901) 6789 : cluster [DBG] pgmap v6796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:57.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:56 smithi130 bash[51302]: debug 2023-08-28T01:44:56.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:56.900133+0000) 2023-08-28T01:44:58.319 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:57 smithi130 bash[51302]: debug 2023-08-28T01:44:57.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:57.900415+0000) 2023-08-28T01:44:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:58 smithi130 bash[27579]: cluster 2023-08-28T01:44:56.560006+0000 mgr.y (mgr.14901) 6790 : cluster [DBG] pgmap v6797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:44:58 smithi130 bash[27579]: audit 2023-08-28T01:44:57.570679+0000 mon.a (mon.0) 4446 : audit [DBG] from='client.? 172.21.15.6:0/2629541611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:58.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:58 smithi006 bash[33918]: cluster 2023-08-28T01:44:56.560006+0000 mgr.y (mgr.14901) 6790 : cluster [DBG] pgmap v6797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:58.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:44:58 smithi006 bash[33918]: audit 2023-08-28T01:44:57.570679+0000 mon.a (mon.0) 4446 : audit [DBG] from='client.? 172.21.15.6:0/2629541611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:58.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:58 smithi006 bash[27292]: cluster 2023-08-28T01:44:56.560006+0000 mgr.y (mgr.14901) 6790 : cluster [DBG] pgmap v6797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:44:58.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:44:58 smithi006 bash[27292]: audit 2023-08-28T01:44:57.570679+0000 mon.a (mon.0) 4446 : audit [DBG] from='client.? 172.21.15.6:0/2629541611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:59.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:58 smithi130 bash[51302]: debug 2023-08-28T01:44:58.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:58.900688+0000) 2023-08-28T01:45:00.325 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:44:59 smithi130 bash[51302]: debug 2023-08-28T01:44:59.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:59.900960+0000) 2023-08-28T01:45:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:00 smithi130 bash[27579]: cluster 2023-08-28T01:44:58.560692+0000 mgr.y (mgr.14901) 6791 : cluster [DBG] pgmap v6798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:00 smithi130 bash[27579]: audit 2023-08-28T01:44:59.985930+0000 mon.a (mon.0) 4447 : audit [DBG] from='client.? 172.21.15.6:0/1182653965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:00 smithi006 bash[27292]: cluster 2023-08-28T01:44:58.560692+0000 mgr.y (mgr.14901) 6791 : cluster [DBG] pgmap v6798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:00 smithi006 bash[27292]: audit 2023-08-28T01:44:59.985930+0000 mon.a (mon.0) 4447 : audit [DBG] from='client.? 172.21.15.6:0/1182653965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:00 smithi006 bash[33918]: cluster 2023-08-28T01:44:58.560692+0000 mgr.y (mgr.14901) 6791 : cluster [DBG] pgmap v6798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:00 smithi006 bash[33918]: audit 2023-08-28T01:44:59.985930+0000 mon.a (mon.0) 4447 : audit [DBG] from='client.? 172.21.15.6:0/1182653965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:01.104 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:45:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:45:00] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:45:01.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:00 smithi130 bash[51302]: debug 2023-08-28T01:45:00.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:00.901251+0000) 2023-08-28T01:45:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:45:01.104Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:45:01.106Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:02.343 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:01 smithi130 bash[51302]: debug 2023-08-28T01:45:01.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:01.901527+0000) 2023-08-28T01:45:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:02 smithi130 bash[27579]: cluster 2023-08-28T01:45:00.561915+0000 mgr.y (mgr.14901) 6792 : cluster [DBG] pgmap v6799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:02.675 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:02 smithi006 bash[27292]: cluster 2023-08-28T01:45:00.561915+0000 mgr.y (mgr.14901) 6792 : cluster [DBG] pgmap v6799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:02.675 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:02 smithi006 bash[33918]: cluster 2023-08-28T01:45:00.561915+0000 mgr.y (mgr.14901) 6792 : cluster [DBG] pgmap v6799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:45:02.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:45:02.677Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:03.352 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:02 smithi130 bash[51302]: debug 2023-08-28T01:45:02.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:02.901835+0000) 2023-08-28T01:45:03.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:03 smithi130 bash[27579]: audit 2023-08-28T01:45:02.415973+0000 mon.a (mon.0) 4448 : audit [DBG] from='client.? 172.21.15.6:0/1475545407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:03.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:03 smithi006 bash[27292]: audit 2023-08-28T01:45:02.415973+0000 mon.a (mon.0) 4448 : audit [DBG] from='client.? 172.21.15.6:0/1475545407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:03.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:03 smithi006 bash[33918]: audit 2023-08-28T01:45:02.415973+0000 mon.a (mon.0) 4448 : audit [DBG] from='client.? 172.21.15.6:0/1475545407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:04.356 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:03 smithi130 bash[51302]: debug 2023-08-28T01:45:03.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:03.902103+0000) 2023-08-28T01:45:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:04 smithi130 bash[27579]: cluster 2023-08-28T01:45:02.562532+0000 mgr.y (mgr.14901) 6793 : cluster [DBG] pgmap v6800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:04.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:04 smithi006 bash[27292]: cluster 2023-08-28T01:45:02.562532+0000 mgr.y (mgr.14901) 6793 : cluster [DBG] pgmap v6800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:45:04.558Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:45:04.560Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:04.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:04 smithi006 bash[33918]: cluster 2023-08-28T01:45:02.562532+0000 mgr.y (mgr.14901) 6793 : cluster [DBG] pgmap v6800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:05.363 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:04 smithi130 bash[51302]: debug 2023-08-28T01:45:04.898+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:04.902435+0000) 2023-08-28T01:45:05.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:05 smithi130 bash[27579]: audit 2023-08-28T01:45:04.832424+0000 mon.a (mon.0) 4449 : audit [DBG] from='client.? 172.21.15.6:0/236459613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:05.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:05 smithi006 bash[33918]: audit 2023-08-28T01:45:04.832424+0000 mon.a (mon.0) 4449 : audit [DBG] from='client.? 172.21.15.6:0/236459613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:05.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:05 smithi006 bash[27292]: audit 2023-08-28T01:45:04.832424+0000 mon.a (mon.0) 4449 : audit [DBG] from='client.? 172.21.15.6:0/236459613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:06.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:05 smithi130 bash[51302]: debug 2023-08-28T01:45:05.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:05.902759+0000) 2023-08-28T01:45:06.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:06 smithi006 bash[33918]: cluster 2023-08-28T01:45:04.563529+0000 mgr.y (mgr.14901) 6794 : cluster [DBG] pgmap v6801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:06 smithi006 bash[27292]: cluster 2023-08-28T01:45:04.563529+0000 mgr.y (mgr.14901) 6794 : cluster [DBG] pgmap v6801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:06.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:06 smithi130 bash[27579]: cluster 2023-08-28T01:45:04.563529+0000 mgr.y (mgr.14901) 6794 : cluster [DBG] pgmap v6801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:07.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:06 smithi130 bash[51302]: debug 2023-08-28T01:45:06.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:06.903089+0000) 2023-08-28T01:45:07.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:07 smithi006 bash[33918]: audit 2023-08-28T01:45:06.399867+0000 mon.b (mon.1) 766 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:45:07.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:07 smithi006 bash[33918]: audit 2023-08-28T01:45:07.246689+0000 mon.a (mon.0) 4450 : audit [DBG] from='client.? 172.21.15.6:0/3068680553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:07.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:07 smithi006 bash[27292]: audit 2023-08-28T01:45:06.399867+0000 mon.b (mon.1) 766 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:45:07.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:07 smithi006 bash[27292]: audit 2023-08-28T01:45:07.246689+0000 mon.a (mon.0) 4450 : audit [DBG] from='client.? 172.21.15.6:0/3068680553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:07.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:07 smithi130 bash[27579]: audit 2023-08-28T01:45:06.399867+0000 mon.b (mon.1) 766 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:45:07.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:07 smithi130 bash[27579]: audit 2023-08-28T01:45:07.246689+0000 mon.a (mon.0) 4450 : audit [DBG] from='client.? 172.21.15.6:0/3068680553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:08.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:07 smithi130 bash[51302]: debug 2023-08-28T01:45:07.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:07.903348+0000) 2023-08-28T01:45:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:08 smithi006 bash[33918]: cluster 2023-08-28T01:45:06.564156+0000 mgr.y (mgr.14901) 6795 : cluster [DBG] pgmap v6802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:08 smithi006 bash[27292]: cluster 2023-08-28T01:45:06.564156+0000 mgr.y (mgr.14901) 6795 : cluster [DBG] pgmap v6802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:08 smithi130 bash[27579]: cluster 2023-08-28T01:45:06.564156+0000 mgr.y (mgr.14901) 6795 : cluster [DBG] pgmap v6802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:09.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:08 smithi130 bash[51302]: debug 2023-08-28T01:45:08.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:08.903580+0000) 2023-08-28T01:45:09.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:09 smithi006 bash[33918]: cluster 2023-08-28T01:45:08.564792+0000 mgr.y (mgr.14901) 6796 : cluster [DBG] pgmap v6803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:09.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:09 smithi006 bash[27292]: cluster 2023-08-28T01:45:08.564792+0000 mgr.y (mgr.14901) 6796 : cluster [DBG] pgmap v6803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:09.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:09 smithi130 bash[27579]: cluster 2023-08-28T01:45:08.564792+0000 mgr.y (mgr.14901) 6796 : cluster [DBG] pgmap v6803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:10.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:09 smithi130 bash[51302]: debug 2023-08-28T01:45:09.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:09.903796+0000) 2023-08-28T01:45:10.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:10 smithi006 bash[27292]: audit 2023-08-28T01:45:09.665377+0000 mon.a (mon.0) 4451 : audit [DBG] from='client.? 172.21.15.6:0/625192894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:10.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:10 smithi006 bash[33918]: audit 2023-08-28T01:45:09.665377+0000 mon.a (mon.0) 4451 : audit [DBG] from='client.? 172.21.15.6:0/625192894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:10.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:10 smithi130 bash[27579]: audit 2023-08-28T01:45:09.665377+0000 mon.a (mon.0) 4451 : audit [DBG] from='client.? 172.21.15.6:0/625192894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:11.105 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:45:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:45:10] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:45:11.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:10 smithi130 bash[51302]: debug 2023-08-28T01:45:10.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:10.904105+0000) 2023-08-28T01:45:11.399 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:45:11.105Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:11.399 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:45:11.107Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:11.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:11 smithi006 bash[27292]: cluster 2023-08-28T01:45:10.565721+0000 mgr.y (mgr.14901) 6797 : cluster [DBG] pgmap v6804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:11.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:11 smithi006 bash[33918]: cluster 2023-08-28T01:45:10.565721+0000 mgr.y (mgr.14901) 6797 : cluster [DBG] pgmap v6804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:11.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:11 smithi130 bash[27579]: cluster 2023-08-28T01:45:10.565721+0000 mgr.y (mgr.14901) 6797 : cluster [DBG] pgmap v6804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:12.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:11 smithi130 bash[51302]: debug 2023-08-28T01:45:11.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:11.904352+0000) 2023-08-28T01:45:12.676 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:12 smithi006 bash[27292]: audit 2023-08-28T01:45:12.101639+0000 mon.c (mon.2) 2678 : audit [DBG] from='client.? 172.21.15.6:0/3951280439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:12.676 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:12 smithi006 bash[27292]: audit 2023-08-28T01:45:12.236515+0000 mon.a (mon.0) 4452 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:45:12.676 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:12 smithi006 bash[33918]: audit 2023-08-28T01:45:12.101639+0000 mon.c (mon.2) 2678 : audit [DBG] from='client.? 172.21.15.6:0/3951280439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:12.676 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:12 smithi006 bash[33918]: audit 2023-08-28T01:45:12.236515+0000 mon.a (mon.0) 4452 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:45:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:12 smithi130 bash[27579]: audit 2023-08-28T01:45:12.101639+0000 mon.c (mon.2) 2678 : audit [DBG] from='client.? 172.21.15.6:0/3951280439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:12 smithi130 bash[27579]: audit 2023-08-28T01:45:12.236515+0000 mon.a (mon.0) 4452 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:45:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:45:12.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:45:12.678Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:13.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:12 smithi130 bash[51302]: debug 2023-08-28T01:45:12.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:12.904597+0000) 2023-08-28T01:45:14.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:13 smithi130 bash[51302]: debug 2023-08-28T01:45:13.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:13.904866+0000) 2023-08-28T01:45:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:13 smithi130 bash[27579]: cluster 2023-08-28T01:45:12.566360+0000 mgr.y (mgr.14901) 6798 : cluster [DBG] pgmap v6805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:13 smithi130 bash[27579]: audit 2023-08-28T01:45:12.744938+0000 mon.a (mon.0) 4453 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:45:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:13 smithi130 bash[27579]: audit 2023-08-28T01:45:13.015285+0000 mon.b (mon.1) 767 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:45:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:13 smithi130 bash[27579]: audit 2023-08-28T01:45:13.017534+0000 mon.b (mon.1) 768 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:45:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:13 smithi130 bash[27579]: audit 2023-08-28T01:45:13.025382+0000 mon.a (mon.0) 4454 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:45:14.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:13 smithi006 bash[33918]: cluster 2023-08-28T01:45:12.566360+0000 mgr.y (mgr.14901) 6798 : cluster [DBG] pgmap v6805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:14.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:13 smithi006 bash[33918]: audit 2023-08-28T01:45:12.744938+0000 mon.a (mon.0) 4453 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:45:14.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:13 smithi006 bash[33918]: audit 2023-08-28T01:45:13.015285+0000 mon.b (mon.1) 767 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:45:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:13 smithi006 bash[33918]: audit 2023-08-28T01:45:13.017534+0000 mon.b (mon.1) 768 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:45:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:13 smithi006 bash[33918]: audit 2023-08-28T01:45:13.025382+0000 mon.a (mon.0) 4454 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:45:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:13 smithi006 bash[27292]: cluster 2023-08-28T01:45:12.566360+0000 mgr.y (mgr.14901) 6798 : cluster [DBG] pgmap v6805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:13 smithi006 bash[27292]: audit 2023-08-28T01:45:12.744938+0000 mon.a (mon.0) 4453 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:45:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:13 smithi006 bash[27292]: audit 2023-08-28T01:45:13.015285+0000 mon.b (mon.1) 767 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:45:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:13 smithi006 bash[27292]: audit 2023-08-28T01:45:13.017534+0000 mon.b (mon.1) 768 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:45:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:13 smithi006 bash[27292]: audit 2023-08-28T01:45:13.025382+0000 mon.a (mon.0) 4454 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:45:14.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:14 smithi006 bash[33918]: audit 2023-08-28T01:45:14.514565+0000 mon.a (mon.0) 4455 : audit [DBG] from='client.? 172.21.15.6:0/661745539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:14.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:14 smithi006 bash[27292]: audit 2023-08-28T01:45:14.514565+0000 mon.a (mon.0) 4455 : audit [DBG] from='client.? 172.21.15.6:0/661745539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:45:14.559Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:45:14.561Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:15.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:14 smithi130 bash[27579]: audit 2023-08-28T01:45:14.514565+0000 mon.a (mon.0) 4455 : audit [DBG] from='client.? 172.21.15.6:0/661745539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:14 smithi130 bash[51302]: debug 2023-08-28T01:45:14.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:14.905191+0000) 2023-08-28T01:45:16.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:15 smithi130 bash[27579]: cluster 2023-08-28T01:45:14.567404+0000 mgr.y (mgr.14901) 6799 : cluster [DBG] pgmap v6806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:16.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:15 smithi130 bash[51302]: debug 2023-08-28T01:45:15.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:15.905446+0000) 2023-08-28T01:45:16.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:15 smithi006 bash[27292]: cluster 2023-08-28T01:45:14.567404+0000 mgr.y (mgr.14901) 6799 : cluster [DBG] pgmap v6806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:16.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:15 smithi006 bash[33918]: cluster 2023-08-28T01:45:14.567404+0000 mgr.y (mgr.14901) 6799 : cluster [DBG] pgmap v6806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:17.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:16 smithi130 bash[51302]: debug 2023-08-28T01:45:16.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:16.905698+0000) 2023-08-28T01:45:18.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:17 smithi130 bash[51302]: debug 2023-08-28T01:45:17.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:17.905975+0000) 2023-08-28T01:45:18.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:17 smithi130 bash[27579]: cluster 2023-08-28T01:45:16.568060+0000 mgr.y (mgr.14901) 6800 : cluster [DBG] pgmap v6807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:18.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:17 smithi130 bash[27579]: audit 2023-08-28T01:45:16.922511+0000 mon.a (mon.0) 4456 : audit [DBG] from='client.? 172.21.15.6:0/3108284163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:18.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:17 smithi006 bash[27292]: cluster 2023-08-28T01:45:16.568060+0000 mgr.y (mgr.14901) 6800 : cluster [DBG] pgmap v6807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:18.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:17 smithi006 bash[27292]: audit 2023-08-28T01:45:16.922511+0000 mon.a (mon.0) 4456 : audit [DBG] from='client.? 172.21.15.6:0/3108284163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:18.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:17 smithi006 bash[33918]: cluster 2023-08-28T01:45:16.568060+0000 mgr.y (mgr.14901) 6800 : cluster [DBG] pgmap v6807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:18.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:17 smithi006 bash[33918]: audit 2023-08-28T01:45:16.922511+0000 mon.a (mon.0) 4456 : audit [DBG] from='client.? 172.21.15.6:0/3108284163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:19.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:18 smithi130 bash[51302]: debug 2023-08-28T01:45:18.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:18.906288+0000) 2023-08-28T01:45:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:19 smithi130 bash[51302]: debug 2023-08-28T01:45:19.902+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:19.906565+0000) 2023-08-28T01:45:20.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:19 smithi130 bash[27579]: cluster 2023-08-28T01:45:18.568752+0000 mgr.y (mgr.14901) 6801 : cluster [DBG] pgmap v6808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:20.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:19 smithi130 bash[27579]: audit 2023-08-28T01:45:19.337738+0000 mon.a (mon.0) 4457 : audit [DBG] from='client.? 172.21.15.6:0/633432361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:20.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:19 smithi006 bash[27292]: cluster 2023-08-28T01:45:18.568752+0000 mgr.y (mgr.14901) 6801 : cluster [DBG] pgmap v6808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:20.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:19 smithi006 bash[27292]: audit 2023-08-28T01:45:19.337738+0000 mon.a (mon.0) 4457 : audit [DBG] from='client.? 172.21.15.6:0/633432361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:20.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:19 smithi006 bash[33918]: cluster 2023-08-28T01:45:18.568752+0000 mgr.y (mgr.14901) 6801 : cluster [DBG] pgmap v6808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:20.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:19 smithi006 bash[33918]: audit 2023-08-28T01:45:19.337738+0000 mon.a (mon.0) 4457 : audit [DBG] from='client.? 172.21.15.6:0/633432361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:21.105 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:45:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:45:20] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:45:21.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:20 smithi130 bash[51302]: debug 2023-08-28T01:45:20.906+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:20.906894+0000) 2023-08-28T01:45:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:45:21.105Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:45:21.108Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:22.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:21 smithi130 bash[27579]: cluster 2023-08-28T01:45:20.569829+0000 mgr.y (mgr.14901) 6802 : cluster [DBG] pgmap v6809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:22.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:21 smithi130 bash[27579]: audit 2023-08-28T01:45:21.749232+0000 mon.a (mon.0) 4458 : audit [DBG] from='client.? 172.21.15.6:0/426554768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:22.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:21 smithi130 bash[51302]: debug 2023-08-28T01:45:21.906+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:21.907224+0000) 2023-08-28T01:45:22.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:21 smithi006 bash[33918]: cluster 2023-08-28T01:45:20.569829+0000 mgr.y (mgr.14901) 6802 : cluster [DBG] pgmap v6809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:22.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:21 smithi006 bash[33918]: audit 2023-08-28T01:45:21.749232+0000 mon.a (mon.0) 4458 : audit [DBG] from='client.? 172.21.15.6:0/426554768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:22.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:21 smithi006 bash[27292]: cluster 2023-08-28T01:45:20.569829+0000 mgr.y (mgr.14901) 6802 : cluster [DBG] pgmap v6809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:22.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:21 smithi006 bash[27292]: audit 2023-08-28T01:45:21.749232+0000 mon.a (mon.0) 4458 : audit [DBG] from='client.? 172.21.15.6:0/426554768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:45:22.676Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:45:22.678Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:23.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:22 smithi130 bash[51302]: debug 2023-08-28T01:45:22.906+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:22.907499+0000) 2023-08-28T01:45:24.251 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:23 smithi130 bash[51302]: debug 2023-08-28T01:45:23.906+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:23.907829+0000) 2023-08-28T01:45:24.627 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:24 smithi006 bash[33918]: cluster 2023-08-28T01:45:22.570441+0000 mgr.y (mgr.14901) 6803 : cluster [DBG] pgmap v6810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:24.627 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:24 smithi006 bash[33918]: audit 2023-08-28T01:45:24.168309+0000 mon.c (mon.2) 2679 : audit [DBG] from='client.? 172.21.15.6:0/2124518892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:24.627 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:24 smithi130 bash[27579]: cluster 2023-08-28T01:45:22.570441+0000 mgr.y (mgr.14901) 6803 : cluster [DBG] pgmap v6810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:24.627 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:24 smithi130 bash[27579]: audit 2023-08-28T01:45:24.168309+0000 mon.c (mon.2) 2679 : audit [DBG] from='client.? 172.21.15.6:0/2124518892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:24.628 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:24 smithi006 bash[27292]: cluster 2023-08-28T01:45:22.570441+0000 mgr.y (mgr.14901) 6803 : cluster [DBG] pgmap v6810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:24.628 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:24 smithi006 bash[27292]: audit 2023-08-28T01:45:24.168309+0000 mon.c (mon.2) 2679 : audit [DBG] from='client.? 172.21.15.6:0/2124518892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:45:24.560Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:25.124 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:45:24.562Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:25.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:24 smithi130 bash[51302]: debug 2023-08-28T01:45:24.906+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:24.908162+0000) 2023-08-28T01:45:26.256 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:25 smithi130 bash[51302]: debug 2023-08-28T01:45:25.906+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:25.908460+0000) 2023-08-28T01:45:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:26 smithi130 bash[27579]: cluster 2023-08-28T01:45:24.571506+0000 mgr.y (mgr.14901) 6804 : cluster [DBG] pgmap v6811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:26.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:26 smithi006 bash[27292]: cluster 2023-08-28T01:45:24.571506+0000 mgr.y (mgr.14901) 6804 : cluster [DBG] pgmap v6811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:26 smithi006 bash[33918]: cluster 2023-08-28T01:45:24.571506+0000 mgr.y (mgr.14901) 6804 : cluster [DBG] pgmap v6811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:27.266 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:26 smithi130 bash[51302]: debug 2023-08-28T01:45:26.906+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:26.908766+0000) 2023-08-28T01:45:27.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:27 smithi130 bash[27579]: audit 2023-08-28T01:45:26.590655+0000 mon.a (mon.0) 4459 : audit [DBG] from='client.? 172.21.15.6:0/2554500286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:27.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:27 smithi006 bash[27292]: audit 2023-08-28T01:45:26.590655+0000 mon.a (mon.0) 4459 : audit [DBG] from='client.? 172.21.15.6:0/2554500286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:27.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:27 smithi006 bash[33918]: audit 2023-08-28T01:45:26.590655+0000 mon.a (mon.0) 4459 : audit [DBG] from='client.? 172.21.15.6:0/2554500286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:28.269 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:27 smithi130 bash[51302]: debug 2023-08-28T01:45:27.906+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:27.909041+0000) 2023-08-28T01:45:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:28 smithi130 bash[27579]: cluster 2023-08-28T01:45:26.572108+0000 mgr.y (mgr.14901) 6805 : cluster [DBG] pgmap v6812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:28.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:28 smithi006 bash[33918]: cluster 2023-08-28T01:45:26.572108+0000 mgr.y (mgr.14901) 6805 : cluster [DBG] pgmap v6812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:28.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:28 smithi006 bash[27292]: cluster 2023-08-28T01:45:26.572108+0000 mgr.y (mgr.14901) 6805 : cluster [DBG] pgmap v6812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:29.278 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:28 smithi130 bash[51302]: debug 2023-08-28T01:45:28.906+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:28.909367+0000) 2023-08-28T01:45:29.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:29 smithi130 bash[27579]: audit 2023-08-28T01:45:29.010905+0000 mon.c (mon.2) 2680 : audit [DBG] from='client.? 172.21.15.6:0/1800690612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:29.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:29 smithi006 bash[27292]: audit 2023-08-28T01:45:29.010905+0000 mon.c (mon.2) 2680 : audit [DBG] from='client.? 172.21.15.6:0/1800690612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:29.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:29 smithi006 bash[33918]: audit 2023-08-28T01:45:29.010905+0000 mon.c (mon.2) 2680 : audit [DBG] from='client.? 172.21.15.6:0/1800690612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:30.282 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:29 smithi130 bash[51302]: debug 2023-08-28T01:45:29.906+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:29.909702+0000) 2023-08-28T01:45:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:30 smithi130 bash[27579]: cluster 2023-08-28T01:45:28.572757+0000 mgr.y (mgr.14901) 6806 : cluster [DBG] pgmap v6813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:30 smithi006 bash[33918]: cluster 2023-08-28T01:45:28.572757+0000 mgr.y (mgr.14901) 6806 : cluster [DBG] pgmap v6813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:30.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:30 smithi006 bash[27292]: cluster 2023-08-28T01:45:28.572757+0000 mgr.y (mgr.14901) 6806 : cluster [DBG] pgmap v6813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:31.107 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:45:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:45:30] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:45:31.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:30 smithi130 bash[51302]: debug 2023-08-28T01:45:30.906+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:30.910011+0000) 2023-08-28T01:45:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:45:31.106Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:45:31.109Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:32.335 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:31 smithi130 bash[51302]: debug 2023-08-28T01:45:31.910+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:31.910276+0000) 2023-08-28T01:45:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:32 smithi130 bash[27579]: cluster 2023-08-28T01:45:30.573827+0000 mgr.y (mgr.14901) 6807 : cluster [DBG] pgmap v6814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:32 smithi130 bash[27579]: audit 2023-08-28T01:45:31.425903+0000 mon.a (mon.0) 4460 : audit [DBG] from='client.? 172.21.15.6:0/3819082048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:32.677 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:32 smithi006 bash[33918]: cluster 2023-08-28T01:45:30.573827+0000 mgr.y (mgr.14901) 6807 : cluster [DBG] pgmap v6814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:32.677 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:32 smithi006 bash[33918]: audit 2023-08-28T01:45:31.425903+0000 mon.a (mon.0) 4460 : audit [DBG] from='client.? 172.21.15.6:0/3819082048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:32.677 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:32 smithi006 bash[27292]: cluster 2023-08-28T01:45:30.573827+0000 mgr.y (mgr.14901) 6807 : cluster [DBG] pgmap v6814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:32.677 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:32 smithi006 bash[27292]: audit 2023-08-28T01:45:31.425903+0000 mon.a (mon.0) 4460 : audit [DBG] from='client.? 172.21.15.6:0/3819082048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:45:32.676Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:45:32.679Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:33.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:32 smithi130 bash[51302]: debug 2023-08-28T01:45:32.906+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:32.910591+0000) 2023-08-28T01:45:34.341 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:33 smithi130 bash[51302]: debug 2023-08-28T01:45:33.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:33.910874+0000) 2023-08-28T01:45:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:34 smithi130 bash[27579]: cluster 2023-08-28T01:45:32.574459+0000 mgr.y (mgr.14901) 6808 : cluster [DBG] pgmap v6815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:34 smithi130 bash[27579]: audit 2023-08-28T01:45:33.846485+0000 mon.a (mon.0) 4461 : audit [DBG] from='client.? 172.21.15.6:0/1866362875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:34.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:34 smithi006 bash[27292]: cluster 2023-08-28T01:45:32.574459+0000 mgr.y (mgr.14901) 6808 : cluster [DBG] pgmap v6815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:34.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:34 smithi006 bash[27292]: audit 2023-08-28T01:45:33.846485+0000 mon.a (mon.0) 4461 : audit [DBG] from='client.? 172.21.15.6:0/1866362875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:45:34.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:45:34.563Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:34.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:34 smithi006 bash[33918]: cluster 2023-08-28T01:45:32.574459+0000 mgr.y (mgr.14901) 6808 : cluster [DBG] pgmap v6815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:34.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:34 smithi006 bash[33918]: audit 2023-08-28T01:45:33.846485+0000 mon.a (mon.0) 4461 : audit [DBG] from='client.? 172.21.15.6:0/1866362875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:35.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:34 smithi130 bash[51302]: debug 2023-08-28T01:45:34.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:34.911158+0000) 2023-08-28T01:45:36.346 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:35 smithi130 bash[51302]: debug 2023-08-28T01:45:35.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:35.911385+0000) 2023-08-28T01:45:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:36 smithi130 bash[27579]: cluster 2023-08-28T01:45:34.575512+0000 mgr.y (mgr.14901) 6809 : cluster [DBG] pgmap v6816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:36 smithi130 bash[27579]: audit 2023-08-28T01:45:36.264074+0000 mon.a (mon.0) 4462 : audit [DBG] from='client.? 172.21.15.6:0/852875449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:36.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:36 smithi006 bash[33918]: cluster 2023-08-28T01:45:34.575512+0000 mgr.y (mgr.14901) 6809 : cluster [DBG] pgmap v6816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:36.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:36 smithi006 bash[33918]: audit 2023-08-28T01:45:36.264074+0000 mon.a (mon.0) 4462 : audit [DBG] from='client.? 172.21.15.6:0/852875449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:36.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:36 smithi006 bash[27292]: cluster 2023-08-28T01:45:34.575512+0000 mgr.y (mgr.14901) 6809 : cluster [DBG] pgmap v6816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:36.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:36 smithi006 bash[27292]: audit 2023-08-28T01:45:36.264074+0000 mon.a (mon.0) 4462 : audit [DBG] from='client.? 172.21.15.6:0/852875449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:37.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:36 smithi130 bash[51302]: debug 2023-08-28T01:45:36.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:36.911658+0000) 2023-08-28T01:45:38.353 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:37 smithi130 bash[51302]: debug 2023-08-28T01:45:37.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:37.911930+0000) 2023-08-28T01:45:38.354 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:38 smithi130 bash[27579]: cluster 2023-08-28T01:45:36.576130+0000 mgr.y (mgr.14901) 6810 : cluster [DBG] pgmap v6817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:38 smithi006 bash[33918]: cluster 2023-08-28T01:45:36.576130+0000 mgr.y (mgr.14901) 6810 : cluster [DBG] pgmap v6817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:38 smithi006 bash[27292]: cluster 2023-08-28T01:45:36.576130+0000 mgr.y (mgr.14901) 6810 : cluster [DBG] pgmap v6817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:39.359 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:38 smithi130 bash[51302]: debug 2023-08-28T01:45:38.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:38.912201+0000) 2023-08-28T01:45:39.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:39 smithi130 bash[27579]: audit 2023-08-28T01:45:38.674825+0000 mon.a (mon.0) 4463 : audit [DBG] from='client.? 172.21.15.6:0/3879331144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:39.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:39 smithi006 bash[27292]: audit 2023-08-28T01:45:38.674825+0000 mon.a (mon.0) 4463 : audit [DBG] from='client.? 172.21.15.6:0/3879331144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:39.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:39 smithi006 bash[33918]: audit 2023-08-28T01:45:38.674825+0000 mon.a (mon.0) 4463 : audit [DBG] from='client.? 172.21.15.6:0/3879331144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:40.363 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:39 smithi130 bash[51302]: debug 2023-08-28T01:45:39.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:39.912509+0000) 2023-08-28T01:45:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:40 smithi130 bash[27579]: cluster 2023-08-28T01:45:38.576742+0000 mgr.y (mgr.14901) 6811 : cluster [DBG] pgmap v6818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:40.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:40 smithi006 bash[27292]: cluster 2023-08-28T01:45:38.576742+0000 mgr.y (mgr.14901) 6811 : cluster [DBG] pgmap v6818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:40.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:40 smithi006 bash[33918]: cluster 2023-08-28T01:45:38.576742+0000 mgr.y (mgr.14901) 6811 : cluster [DBG] pgmap v6818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:41.108 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:45:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:45:40] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:45:41.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:40 smithi130 bash[51302]: debug 2023-08-28T01:45:40.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:40.912803+0000) 2023-08-28T01:45:41.370 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:45:41.108Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:41.370 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:45:41.109Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:41.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:41 smithi006 bash[27292]: audit 2023-08-28T01:45:41.094385+0000 mon.a (mon.0) 4464 : audit [DBG] from='client.? 172.21.15.6:0/3618087502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:41.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:41 smithi006 bash[33918]: audit 2023-08-28T01:45:41.094385+0000 mon.a (mon.0) 4464 : audit [DBG] from='client.? 172.21.15.6:0/3618087502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:41.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:41 smithi130 bash[27579]: audit 2023-08-28T01:45:41.094385+0000 mon.a (mon.0) 4464 : audit [DBG] from='client.? 172.21.15.6:0/3618087502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:42.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:41 smithi130 bash[51302]: debug 2023-08-28T01:45:41.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:41.913146+0000) 2023-08-28T01:45:42.677 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:42 smithi006 bash[33918]: cluster 2023-08-28T01:45:40.577876+0000 mgr.y (mgr.14901) 6812 : cluster [DBG] pgmap v6819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:42.678 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:42 smithi006 bash[27292]: cluster 2023-08-28T01:45:40.577876+0000 mgr.y (mgr.14901) 6812 : cluster [DBG] pgmap v6819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:42.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:42 smithi130 bash[27579]: cluster 2023-08-28T01:45:40.577876+0000 mgr.y (mgr.14901) 6812 : cluster [DBG] pgmap v6819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:45:42.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:45:42.679Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:43.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:42 smithi130 bash[51302]: debug 2023-08-28T01:45:42.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:42.913443+0000) 2023-08-28T01:45:44.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:43 smithi130 bash[51302]: debug 2023-08-28T01:45:43.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:43.913726+0000) 2023-08-28T01:45:44.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:45:44.562Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:44.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:45:44.564Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:44.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:44 smithi006 bash[27292]: cluster 2023-08-28T01:45:42.578504+0000 mgr.y (mgr.14901) 6813 : cluster [DBG] pgmap v6820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:44.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:44 smithi006 bash[27292]: audit 2023-08-28T01:45:43.509428+0000 mon.c (mon.2) 2681 : audit [DBG] from='client.? 172.21.15.6:0/651174426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:44.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:44 smithi006 bash[33918]: cluster 2023-08-28T01:45:42.578504+0000 mgr.y (mgr.14901) 6813 : cluster [DBG] pgmap v6820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:44.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:44 smithi006 bash[33918]: audit 2023-08-28T01:45:43.509428+0000 mon.c (mon.2) 2681 : audit [DBG] from='client.? 172.21.15.6:0/651174426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:44.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:44 smithi130 bash[27579]: cluster 2023-08-28T01:45:42.578504+0000 mgr.y (mgr.14901) 6813 : cluster [DBG] pgmap v6820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:44.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:44 smithi130 bash[27579]: audit 2023-08-28T01:45:43.509428+0000 mon.c (mon.2) 2681 : audit [DBG] from='client.? 172.21.15.6:0/651174426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:45.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:44 smithi130 bash[51302]: debug 2023-08-28T01:45:44.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:44.913999+0000) 2023-08-28T01:45:46.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:45 smithi130 bash[51302]: debug 2023-08-28T01:45:45.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:45.914276+0000) 2023-08-28T01:45:46.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:46 smithi006 bash[27292]: cluster 2023-08-28T01:45:44.579610+0000 mgr.y (mgr.14901) 6814 : cluster [DBG] pgmap v6821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:46.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:46 smithi006 bash[27292]: audit 2023-08-28T01:45:45.932263+0000 mon.a (mon.0) 4465 : audit [DBG] from='client.? 172.21.15.6:0/4016298355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:46.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:46 smithi006 bash[33918]: cluster 2023-08-28T01:45:44.579610+0000 mgr.y (mgr.14901) 6814 : cluster [DBG] pgmap v6821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:46.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:46 smithi006 bash[33918]: audit 2023-08-28T01:45:45.932263+0000 mon.a (mon.0) 4465 : audit [DBG] from='client.? 172.21.15.6:0/4016298355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:46.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:46 smithi130 bash[27579]: cluster 2023-08-28T01:45:44.579610+0000 mgr.y (mgr.14901) 6814 : cluster [DBG] pgmap v6821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:46.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:46 smithi130 bash[27579]: audit 2023-08-28T01:45:45.932263+0000 mon.a (mon.0) 4465 : audit [DBG] from='client.? 172.21.15.6:0/4016298355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:47.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:46 smithi130 bash[51302]: debug 2023-08-28T01:45:46.909+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:46.914565+0000) 2023-08-28T01:45:48.252 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:47 smithi130 bash[51302]: debug 2023-08-28T01:45:47.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:47.914844+0000) 2023-08-28T01:45:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:48 smithi130 bash[27579]: cluster 2023-08-28T01:45:46.580257+0000 mgr.y (mgr.14901) 6815 : cluster [DBG] pgmap v6822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:48.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:48 smithi006 bash[27292]: cluster 2023-08-28T01:45:46.580257+0000 mgr.y (mgr.14901) 6815 : cluster [DBG] pgmap v6822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:48.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:48 smithi006 bash[33918]: cluster 2023-08-28T01:45:46.580257+0000 mgr.y (mgr.14901) 6815 : cluster [DBG] pgmap v6822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:49.265 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:48 smithi130 bash[51302]: debug 2023-08-28T01:45:48.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:48.915205+0000) 2023-08-28T01:45:49.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:49 smithi130 bash[27579]: audit 2023-08-28T01:45:48.349866+0000 mon.a (mon.0) 4466 : audit [DBG] from='client.? 172.21.15.6:0/1401589949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:49.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:49 smithi006 bash[33918]: audit 2023-08-28T01:45:48.349866+0000 mon.a (mon.0) 4466 : audit [DBG] from='client.? 172.21.15.6:0/1401589949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:49.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:49 smithi006 bash[27292]: audit 2023-08-28T01:45:48.349866+0000 mon.a (mon.0) 4466 : audit [DBG] from='client.? 172.21.15.6:0/1401589949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:50.268 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:49 smithi130 bash[51302]: debug 2023-08-28T01:45:49.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:49.915516+0000) 2023-08-28T01:45:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:50 smithi130 bash[27579]: cluster 2023-08-28T01:45:48.580918+0000 mgr.y (mgr.14901) 6816 : cluster [DBG] pgmap v6823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:50.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:50 smithi006 bash[33918]: cluster 2023-08-28T01:45:48.580918+0000 mgr.y (mgr.14901) 6816 : cluster [DBG] pgmap v6823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:50.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:50 smithi006 bash[27292]: cluster 2023-08-28T01:45:48.580918+0000 mgr.y (mgr.14901) 6816 : cluster [DBG] pgmap v6823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:51.108 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:45:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:45:50] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:45:51.276 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:50 smithi130 bash[51302]: debug 2023-08-28T01:45:50.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:50.915817+0000) 2023-08-28T01:45:51.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:51 smithi006 bash[27292]: audit 2023-08-28T01:45:50.777167+0000 mon.c (mon.2) 2682 : audit [DBG] from='client.? 172.21.15.6:0/1126786530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:51.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:51 smithi006 bash[33918]: audit 2023-08-28T01:45:50.777167+0000 mon.c (mon.2) 2682 : audit [DBG] from='client.? 172.21.15.6:0/1126786530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:45:51.108Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:45:51.111Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:51.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:51 smithi130 bash[27579]: audit 2023-08-28T01:45:50.777167+0000 mon.c (mon.2) 2682 : audit [DBG] from='client.? 172.21.15.6:0/1126786530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:52.279 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:51 smithi130 bash[51302]: debug 2023-08-28T01:45:51.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:51.916170+0000) 2023-08-28T01:45:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:52 smithi130 bash[27579]: cluster 2023-08-28T01:45:50.581829+0000 mgr.y (mgr.14901) 6817 : cluster [DBG] pgmap v6824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:52.678 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:52 smithi006 bash[33918]: cluster 2023-08-28T01:45:50.581829+0000 mgr.y (mgr.14901) 6817 : cluster [DBG] pgmap v6824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:52.678 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:52 smithi006 bash[27292]: cluster 2023-08-28T01:45:50.581829+0000 mgr.y (mgr.14901) 6817 : cluster [DBG] pgmap v6824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:45:52.678Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:45:52.680Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:53.288 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:52 smithi130 bash[51302]: debug 2023-08-28T01:45:52.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:52.916453+0000) 2023-08-28T01:45:53.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:53 smithi130 bash[27579]: audit 2023-08-28T01:45:53.191087+0000 mon.c (mon.2) 2683 : audit [DBG] from='client.? 172.21.15.6:0/4256154022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:53.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:53 smithi006 bash[27292]: audit 2023-08-28T01:45:53.191087+0000 mon.c (mon.2) 2683 : audit [DBG] from='client.? 172.21.15.6:0/4256154022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:53.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:53 smithi006 bash[33918]: audit 2023-08-28T01:45:53.191087+0000 mon.c (mon.2) 2683 : audit [DBG] from='client.? 172.21.15.6:0/4256154022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:54.291 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:53 smithi130 bash[51302]: debug 2023-08-28T01:45:53.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:53.916734+0000) 2023-08-28T01:45:54.562 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:54 smithi006 bash[33918]: cluster 2023-08-28T01:45:52.582430+0000 mgr.y (mgr.14901) 6818 : cluster [DBG] pgmap v6825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:54.562 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:54 smithi006 bash[27292]: cluster 2023-08-28T01:45:52.582430+0000 mgr.y (mgr.14901) 6818 : cluster [DBG] pgmap v6825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:54 smithi130 bash[27579]: cluster 2023-08-28T01:45:52.582430+0000 mgr.y (mgr.14901) 6818 : cluster [DBG] pgmap v6825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:45:54.562Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:45:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:45:54.565Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:45:55.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:54 smithi130 bash[51302]: debug 2023-08-28T01:45:54.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:54.917047+0000) 2023-08-28T01:45:56.296 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:55 smithi130 bash[51302]: debug 2023-08-28T01:45:55.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:55.917335+0000) 2023-08-28T01:45:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:56 smithi130 bash[27579]: cluster 2023-08-28T01:45:54.583525+0000 mgr.y (mgr.14901) 6819 : cluster [DBG] pgmap v6826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:56 smithi130 bash[27579]: audit 2023-08-28T01:45:55.599936+0000 mon.a (mon.0) 4467 : audit [DBG] from='client.? 172.21.15.6:0/1953833149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:56.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:56 smithi006 bash[33918]: cluster 2023-08-28T01:45:54.583525+0000 mgr.y (mgr.14901) 6819 : cluster [DBG] pgmap v6826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:56.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:56 smithi006 bash[33918]: audit 2023-08-28T01:45:55.599936+0000 mon.a (mon.0) 4467 : audit [DBG] from='client.? 172.21.15.6:0/1953833149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:56.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:56 smithi006 bash[27292]: cluster 2023-08-28T01:45:54.583525+0000 mgr.y (mgr.14901) 6819 : cluster [DBG] pgmap v6826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:56.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:56 smithi006 bash[27292]: audit 2023-08-28T01:45:55.599936+0000 mon.a (mon.0) 4467 : audit [DBG] from='client.? 172.21.15.6:0/1953833149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:57.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:56 smithi130 bash[51302]: debug 2023-08-28T01:45:56.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:56.917659+0000) 2023-08-28T01:45:58.300 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:57 smithi130 bash[51302]: debug 2023-08-28T01:45:57.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:57.917917+0000) 2023-08-28T01:45:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:58 smithi130 bash[27579]: cluster 2023-08-28T01:45:56.584137+0000 mgr.y (mgr.14901) 6820 : cluster [DBG] pgmap v6827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:45:58 smithi130 bash[27579]: audit 2023-08-28T01:45:58.015969+0000 mon.c (mon.2) 2684 : audit [DBG] from='client.? 172.21.15.6:0/1485786635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:58.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:58 smithi006 bash[27292]: cluster 2023-08-28T01:45:56.584137+0000 mgr.y (mgr.14901) 6820 : cluster [DBG] pgmap v6827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:58.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:45:58 smithi006 bash[27292]: audit 2023-08-28T01:45:58.015969+0000 mon.c (mon.2) 2684 : audit [DBG] from='client.? 172.21.15.6:0/1485786635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:58.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:58 smithi006 bash[33918]: cluster 2023-08-28T01:45:56.584137+0000 mgr.y (mgr.14901) 6820 : cluster [DBG] pgmap v6827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:45:58.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:45:58 smithi006 bash[33918]: audit 2023-08-28T01:45:58.015969+0000 mon.c (mon.2) 2684 : audit [DBG] from='client.? 172.21.15.6:0/1485786635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:59.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:58 smithi130 bash[51302]: debug 2023-08-28T01:45:58.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:58.918241+0000) 2023-08-28T01:46:00.305 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:45:59 smithi130 bash[51302]: debug 2023-08-28T01:45:59.913+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:59.918504+0000) 2023-08-28T01:46:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:00 smithi130 bash[27579]: cluster 2023-08-28T01:45:58.584747+0000 mgr.y (mgr.14901) 6821 : cluster [DBG] pgmap v6828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:00 smithi006 bash[27292]: cluster 2023-08-28T01:45:58.584747+0000 mgr.y (mgr.14901) 6821 : cluster [DBG] pgmap v6828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:00 smithi006 bash[33918]: cluster 2023-08-28T01:45:58.584747+0000 mgr.y (mgr.14901) 6821 : cluster [DBG] pgmap v6828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:01.110 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:46:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:46:00] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:46:01.313 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:00 smithi130 bash[51302]: debug 2023-08-28T01:46:00.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:00.918838+0000) 2023-08-28T01:46:01.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:01 smithi006 bash[27292]: audit 2023-08-28T01:46:00.439070+0000 mon.c (mon.2) 2685 : audit [DBG] from='client.? 172.21.15.6:0/1303785396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:01.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:01 smithi006 bash[33918]: audit 2023-08-28T01:46:00.439070+0000 mon.c (mon.2) 2685 : audit [DBG] from='client.? 172.21.15.6:0/1303785396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:46:01.110Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:46:01.112Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:01.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:01 smithi130 bash[27579]: audit 2023-08-28T01:46:00.439070+0000 mon.c (mon.2) 2685 : audit [DBG] from='client.? 172.21.15.6:0/1303785396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:02.317 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:01 smithi130 bash[51302]: debug 2023-08-28T01:46:01.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:01.919159+0000) 2023-08-28T01:46:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:02 smithi130 bash[27579]: cluster 2023-08-28T01:46:00.585873+0000 mgr.y (mgr.14901) 6822 : cluster [DBG] pgmap v6829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:02.679 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:02 smithi006 bash[27292]: cluster 2023-08-28T01:46:00.585873+0000 mgr.y (mgr.14901) 6822 : cluster [DBG] pgmap v6829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:02.679 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:02 smithi006 bash[33918]: cluster 2023-08-28T01:46:00.585873+0000 mgr.y (mgr.14901) 6822 : cluster [DBG] pgmap v6829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:46:02.679Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:46:02.680Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:03.324 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:02 smithi130 bash[51302]: debug 2023-08-28T01:46:02.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:02.919384+0000) 2023-08-28T01:46:03.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:03 smithi130 bash[27579]: audit 2023-08-28T01:46:02.860998+0000 mon.c (mon.2) 2686 : audit [DBG] from='client.? 172.21.15.6:0/3890661943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:03.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:03 smithi006 bash[33918]: audit 2023-08-28T01:46:02.860998+0000 mon.c (mon.2) 2686 : audit [DBG] from='client.? 172.21.15.6:0/3890661943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:03.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:03 smithi006 bash[27292]: audit 2023-08-28T01:46:02.860998+0000 mon.c (mon.2) 2686 : audit [DBG] from='client.? 172.21.15.6:0/3890661943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:04.329 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:03 smithi130 bash[51302]: debug 2023-08-28T01:46:03.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:03.919672+0000) 2023-08-28T01:46:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:04 smithi130 bash[27579]: cluster 2023-08-28T01:46:02.586350+0000 mgr.y (mgr.14901) 6823 : cluster [DBG] pgmap v6830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:04.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:04 smithi006 bash[33918]: cluster 2023-08-28T01:46:02.586350+0000 mgr.y (mgr.14901) 6823 : cluster [DBG] pgmap v6830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:04.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:04 smithi006 bash[27292]: cluster 2023-08-28T01:46:02.586350+0000 mgr.y (mgr.14901) 6823 : cluster [DBG] pgmap v6830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:46:04.563Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:46:04.566Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:05.336 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:04 smithi130 bash[51302]: debug 2023-08-28T01:46:04.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:04.919940+0000) 2023-08-28T01:46:05.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:05 smithi130 bash[27579]: audit 2023-08-28T01:46:05.296862+0000 mon.a (mon.0) 4468 : audit [DBG] from='client.? 172.21.15.6:0/1782946114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:05.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:05 smithi006 bash[27292]: audit 2023-08-28T01:46:05.296862+0000 mon.a (mon.0) 4468 : audit [DBG] from='client.? 172.21.15.6:0/1782946114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:05.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:05 smithi006 bash[33918]: audit 2023-08-28T01:46:05.296862+0000 mon.a (mon.0) 4468 : audit [DBG] from='client.? 172.21.15.6:0/1782946114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:06.339 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:05 smithi130 bash[51302]: debug 2023-08-28T01:46:05.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:05.920252+0000) 2023-08-28T01:46:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:06 smithi130 bash[27579]: cluster 2023-08-28T01:46:04.587584+0000 mgr.y (mgr.14901) 6824 : cluster [DBG] pgmap v6831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:06.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:06 smithi006 bash[33918]: cluster 2023-08-28T01:46:04.587584+0000 mgr.y (mgr.14901) 6824 : cluster [DBG] pgmap v6831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:06 smithi006 bash[27292]: cluster 2023-08-28T01:46:04.587584+0000 mgr.y (mgr.14901) 6824 : cluster [DBG] pgmap v6831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:07.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:06 smithi130 bash[51302]: debug 2023-08-28T01:46:06.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:06.920526+0000) 2023-08-28T01:46:08.346 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:07 smithi130 bash[51302]: debug 2023-08-28T01:46:07.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:07.920880+0000) 2023-08-28T01:46:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:08 smithi130 bash[27579]: cluster 2023-08-28T01:46:06.588378+0000 mgr.y (mgr.14901) 6825 : cluster [DBG] pgmap v6832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:08 smithi130 bash[27579]: audit 2023-08-28T01:46:07.713440+0000 mon.a (mon.0) 4469 : audit [DBG] from='client.? 172.21.15.6:0/2114938716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:08.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:08 smithi006 bash[27292]: cluster 2023-08-28T01:46:06.588378+0000 mgr.y (mgr.14901) 6825 : cluster [DBG] pgmap v6832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:08 smithi006 bash[27292]: audit 2023-08-28T01:46:07.713440+0000 mon.a (mon.0) 4469 : audit [DBG] from='client.? 172.21.15.6:0/2114938716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:08.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:08 smithi006 bash[33918]: cluster 2023-08-28T01:46:06.588378+0000 mgr.y (mgr.14901) 6825 : cluster [DBG] pgmap v6832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:08.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:08 smithi006 bash[33918]: audit 2023-08-28T01:46:07.713440+0000 mon.a (mon.0) 4469 : audit [DBG] from='client.? 172.21.15.6:0/2114938716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:09.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:08 smithi130 bash[51302]: debug 2023-08-28T01:46:08.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:08.921140+0000) 2023-08-28T01:46:10.350 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:09 smithi130 bash[51302]: debug 2023-08-28T01:46:09.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:09.921411+0000) 2023-08-28T01:46:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:10 smithi130 bash[27579]: cluster 2023-08-28T01:46:08.589108+0000 mgr.y (mgr.14901) 6826 : cluster [DBG] pgmap v6833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:10 smithi130 bash[27579]: audit 2023-08-28T01:46:10.140062+0000 mon.c (mon.2) 2687 : audit [DBG] from='client.? 172.21.15.6:0/393297032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:10 smithi006 bash[33918]: cluster 2023-08-28T01:46:08.589108+0000 mgr.y (mgr.14901) 6826 : cluster [DBG] pgmap v6833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:10 smithi006 bash[33918]: audit 2023-08-28T01:46:10.140062+0000 mon.c (mon.2) 2687 : audit [DBG] from='client.? 172.21.15.6:0/393297032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:10.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:10 smithi006 bash[27292]: cluster 2023-08-28T01:46:08.589108+0000 mgr.y (mgr.14901) 6826 : cluster [DBG] pgmap v6833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:10.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:10 smithi006 bash[27292]: audit 2023-08-28T01:46:10.140062+0000 mon.c (mon.2) 2687 : audit [DBG] from='client.? 172.21.15.6:0/393297032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:11.111 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:46:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:46:10] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:46:11.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:10 smithi130 bash[51302]: debug 2023-08-28T01:46:10.921+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:10.921716+0000) 2023-08-28T01:46:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:46:11.111Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:46:11.112Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:12.356 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:11 smithi130 bash[51302]: debug 2023-08-28T01:46:11.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:11.922020+0000) 2023-08-28T01:46:12.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:12 smithi130 bash[27579]: cluster 2023-08-28T01:46:10.590014+0000 mgr.y (mgr.14901) 6827 : cluster [DBG] pgmap v6834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:12.680 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:12 smithi006 bash[33918]: cluster 2023-08-28T01:46:10.590014+0000 mgr.y (mgr.14901) 6827 : cluster [DBG] pgmap v6834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:12.680 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:12 smithi006 bash[27292]: cluster 2023-08-28T01:46:10.590014+0000 mgr.y (mgr.14901) 6827 : cluster [DBG] pgmap v6834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:46:12.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:46:12.682Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:13.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:12 smithi130 bash[51302]: debug 2023-08-28T01:46:12.921+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:12.922350+0000) 2023-08-28T01:46:13.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:13 smithi006 bash[27292]: audit 2023-08-28T01:46:12.557250+0000 mon.c (mon.2) 2688 : audit [DBG] from='client.? 172.21.15.6:0/1618587549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:13.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:13 smithi006 bash[27292]: audit 2023-08-28T01:46:13.036711+0000 mon.b (mon.1) 769 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:46:13.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:13 smithi006 bash[33918]: audit 2023-08-28T01:46:12.557250+0000 mon.c (mon.2) 2688 : audit [DBG] from='client.? 172.21.15.6:0/1618587549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:13.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:13 smithi006 bash[33918]: audit 2023-08-28T01:46:13.036711+0000 mon.b (mon.1) 769 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:46:13.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:13 smithi130 bash[27579]: audit 2023-08-28T01:46:12.557250+0000 mon.c (mon.2) 2688 : audit [DBG] from='client.? 172.21.15.6:0/1618587549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:13.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:13 smithi130 bash[27579]: audit 2023-08-28T01:46:13.036711+0000 mon.b (mon.1) 769 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:46:14.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:13 smithi130 bash[51302]: debug 2023-08-28T01:46:13.917+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:13.922650+0000) 2023-08-28T01:46:14.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:14 smithi006 bash[33918]: cluster 2023-08-28T01:46:12.590644+0000 mgr.y (mgr.14901) 6828 : cluster [DBG] pgmap v6835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:14.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:14 smithi006 bash[27292]: cluster 2023-08-28T01:46:12.590644+0000 mgr.y (mgr.14901) 6828 : cluster [DBG] pgmap v6835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:46:14.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:46:14.566Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:14.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:14 smithi130 bash[27579]: cluster 2023-08-28T01:46:12.590644+0000 mgr.y (mgr.14901) 6828 : cluster [DBG] pgmap v6835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:15.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:14 smithi130 bash[51302]: debug 2023-08-28T01:46:14.921+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:14.922981+0000) 2023-08-28T01:46:15.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:15 smithi006 bash[27292]: audit 2023-08-28T01:46:14.976128+0000 mon.c (mon.2) 2689 : audit [DBG] from='client.? 172.21.15.6:0/3151291108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:15.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:15 smithi006 bash[33918]: audit 2023-08-28T01:46:14.976128+0000 mon.c (mon.2) 2689 : audit [DBG] from='client.? 172.21.15.6:0/3151291108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:15.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:15 smithi130 bash[27579]: audit 2023-08-28T01:46:14.976128+0000 mon.c (mon.2) 2689 : audit [DBG] from='client.? 172.21.15.6:0/3151291108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:16.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:15 smithi130 bash[51302]: debug 2023-08-28T01:46:15.921+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:15.923226+0000) 2023-08-28T01:46:16.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:16 smithi006 bash[33918]: cluster 2023-08-28T01:46:14.591649+0000 mgr.y (mgr.14901) 6829 : cluster [DBG] pgmap v6836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:16.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:16 smithi006 bash[27292]: cluster 2023-08-28T01:46:14.591649+0000 mgr.y (mgr.14901) 6829 : cluster [DBG] pgmap v6836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:16.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:16 smithi130 bash[27579]: cluster 2023-08-28T01:46:14.591649+0000 mgr.y (mgr.14901) 6829 : cluster [DBG] pgmap v6836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:17.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:16 smithi130 bash[51302]: debug 2023-08-28T01:46:16.921+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:16.923521+0000) 2023-08-28T01:46:18.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:17 smithi130 bash[51302]: debug 2023-08-28T01:46:17.921+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:17.923744+0000) 2023-08-28T01:46:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:18 smithi006 bash[27292]: cluster 2023-08-28T01:46:16.592312+0000 mgr.y (mgr.14901) 6830 : cluster [DBG] pgmap v6837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:18 smithi006 bash[27292]: audit 2023-08-28T01:46:17.396118+0000 mon.c (mon.2) 2690 : audit [DBG] from='client.? 172.21.15.6:0/892584097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:18.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:18 smithi006 bash[33918]: cluster 2023-08-28T01:46:16.592312+0000 mgr.y (mgr.14901) 6830 : cluster [DBG] pgmap v6837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:18.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:18 smithi006 bash[33918]: audit 2023-08-28T01:46:17.396118+0000 mon.c (mon.2) 2690 : audit [DBG] from='client.? 172.21.15.6:0/892584097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:18.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:18 smithi130 bash[27579]: cluster 2023-08-28T01:46:16.592312+0000 mgr.y (mgr.14901) 6830 : cluster [DBG] pgmap v6837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:18.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:18 smithi130 bash[27579]: audit 2023-08-28T01:46:17.396118+0000 mon.c (mon.2) 2690 : audit [DBG] from='client.? 172.21.15.6:0/892584097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:19.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:18 smithi130 bash[51302]: debug 2023-08-28T01:46:18.921+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:18.923948+0000) 2023-08-28T01:46:20.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:19 smithi006 bash[27292]: cluster 2023-08-28T01:46:18.592815+0000 mgr.y (mgr.14901) 6831 : cluster [DBG] pgmap v6838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:20.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:19 smithi006 bash[27292]: audit 2023-08-28T01:46:18.953641+0000 mon.a (mon.0) 4470 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:46:20.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:19 smithi006 bash[27292]: audit 2023-08-28T01:46:19.365336+0000 mon.a (mon.0) 4471 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:46:20.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:19 smithi006 bash[27292]: audit 2023-08-28T01:46:19.633942+0000 mon.b (mon.1) 770 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:46:20.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:19 smithi006 bash[27292]: audit 2023-08-28T01:46:19.635719+0000 mon.b (mon.1) 771 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:46:20.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:19 smithi006 bash[27292]: audit 2023-08-28T01:46:19.642951+0000 mon.a (mon.0) 4472 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:46:20.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:19 smithi006 bash[27292]: audit 2023-08-28T01:46:19.812215+0000 mon.a (mon.0) 4473 : audit [DBG] from='client.? 172.21.15.6:0/2322230540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:20.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:19 smithi006 bash[33918]: cluster 2023-08-28T01:46:18.592815+0000 mgr.y (mgr.14901) 6831 : cluster [DBG] pgmap v6838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:20.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:19 smithi006 bash[33918]: audit 2023-08-28T01:46:18.953641+0000 mon.a (mon.0) 4470 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:46:20.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:19 smithi006 bash[33918]: audit 2023-08-28T01:46:19.365336+0000 mon.a (mon.0) 4471 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:46:20.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:19 smithi006 bash[33918]: audit 2023-08-28T01:46:19.633942+0000 mon.b (mon.1) 770 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:46:20.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:19 smithi006 bash[33918]: audit 2023-08-28T01:46:19.635719+0000 mon.b (mon.1) 771 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:46:20.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:19 smithi006 bash[33918]: audit 2023-08-28T01:46:19.642951+0000 mon.a (mon.0) 4472 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:46:20.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:19 smithi006 bash[33918]: audit 2023-08-28T01:46:19.812215+0000 mon.a (mon.0) 4473 : audit [DBG] from='client.? 172.21.15.6:0/2322230540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:20.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:19 smithi130 bash[51302]: debug 2023-08-28T01:46:19.921+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:19.924216+0000) 2023-08-28T01:46:20.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:19 smithi130 bash[27579]: cluster 2023-08-28T01:46:18.592815+0000 mgr.y (mgr.14901) 6831 : cluster [DBG] pgmap v6838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:20.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:19 smithi130 bash[27579]: audit 2023-08-28T01:46:18.953641+0000 mon.a (mon.0) 4470 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:46:20.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:19 smithi130 bash[27579]: audit 2023-08-28T01:46:19.365336+0000 mon.a (mon.0) 4471 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:46:20.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:19 smithi130 bash[27579]: audit 2023-08-28T01:46:19.633942+0000 mon.b (mon.1) 770 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:46:20.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:19 smithi130 bash[27579]: audit 2023-08-28T01:46:19.635719+0000 mon.b (mon.1) 771 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:46:20.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:19 smithi130 bash[27579]: audit 2023-08-28T01:46:19.642951+0000 mon.a (mon.0) 4472 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:46:20.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:19 smithi130 bash[27579]: audit 2023-08-28T01:46:19.812215+0000 mon.a (mon.0) 4473 : audit [DBG] from='client.? 172.21.15.6:0/2322230540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:21.112 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:46:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:46:20] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:46:21.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:20 smithi130 bash[51302]: debug 2023-08-28T01:46:20.921+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:20.924507+0000) 2023-08-28T01:46:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:46:21.112Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:46:21.114Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:22.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:21 smithi006 bash[27292]: cluster 2023-08-28T01:46:20.593937+0000 mgr.y (mgr.14901) 6832 : cluster [DBG] pgmap v6839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:22.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:21 smithi006 bash[33918]: cluster 2023-08-28T01:46:20.593937+0000 mgr.y (mgr.14901) 6832 : cluster [DBG] pgmap v6839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:22.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:21 smithi130 bash[27579]: cluster 2023-08-28T01:46:20.593937+0000 mgr.y (mgr.14901) 6832 : cluster [DBG] pgmap v6839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:22.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:21 smithi130 bash[51302]: debug 2023-08-28T01:46:21.921+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:21.924771+0000) 2023-08-28T01:46:22.958 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:46:22.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:22.958 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:46:22.682Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:23.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:22 smithi006 bash[33918]: audit 2023-08-28T01:46:22.226098+0000 mon.a (mon.0) 4474 : audit [DBG] from='client.? 172.21.15.6:0/3847134903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:23.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:22 smithi006 bash[27292]: audit 2023-08-28T01:46:22.226098+0000 mon.a (mon.0) 4474 : audit [DBG] from='client.? 172.21.15.6:0/3847134903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:23.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:22 smithi130 bash[27579]: audit 2023-08-28T01:46:22.226098+0000 mon.a (mon.0) 4474 : audit [DBG] from='client.? 172.21.15.6:0/3847134903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:23.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:22 smithi130 bash[51302]: debug 2023-08-28T01:46:22.921+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:22.925067+0000) 2023-08-28T01:46:24.264 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:23 smithi130 bash[51302]: debug 2023-08-28T01:46:23.925+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:23.925347+0000) 2023-08-28T01:46:24.565 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:24 smithi006 bash[27292]: cluster 2023-08-28T01:46:22.594630+0000 mgr.y (mgr.14901) 6833 : cluster [DBG] pgmap v6840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:24.566 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:24 smithi006 bash[33918]: cluster 2023-08-28T01:46:22.594630+0000 mgr.y (mgr.14901) 6833 : cluster [DBG] pgmap v6840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:24.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:24 smithi130 bash[27579]: cluster 2023-08-28T01:46:22.594630+0000 mgr.y (mgr.14901) 6833 : cluster [DBG] pgmap v6840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:46:24.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:46:24.568Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:25.272 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:24 smithi130 bash[51302]: debug 2023-08-28T01:46:24.925+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:24.925647+0000) 2023-08-28T01:46:25.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:25 smithi130 bash[27579]: audit 2023-08-28T01:46:24.647623+0000 mon.a (mon.0) 4475 : audit [DBG] from='client.? 172.21.15.6:0/1213662290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:25.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:25 smithi006 bash[33918]: audit 2023-08-28T01:46:24.647623+0000 mon.a (mon.0) 4475 : audit [DBG] from='client.? 172.21.15.6:0/1213662290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:25.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:25 smithi006 bash[27292]: audit 2023-08-28T01:46:24.647623+0000 mon.a (mon.0) 4475 : audit [DBG] from='client.? 172.21.15.6:0/1213662290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:26.275 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:25 smithi130 bash[51302]: debug 2023-08-28T01:46:25.921+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:25.925918+0000) 2023-08-28T01:46:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:26 smithi130 bash[27579]: cluster 2023-08-28T01:46:24.595790+0000 mgr.y (mgr.14901) 6834 : cluster [DBG] pgmap v6841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:26.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:26 smithi006 bash[27292]: cluster 2023-08-28T01:46:24.595790+0000 mgr.y (mgr.14901) 6834 : cluster [DBG] pgmap v6841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:26.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:26 smithi006 bash[33918]: cluster 2023-08-28T01:46:24.595790+0000 mgr.y (mgr.14901) 6834 : cluster [DBG] pgmap v6841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:27.280 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:26 smithi130 bash[51302]: debug 2023-08-28T01:46:26.921+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:26.926244+0000) 2023-08-28T01:46:27.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:27 smithi130 bash[27579]: audit 2023-08-28T01:46:27.056052+0000 mon.c (mon.2) 2691 : audit [DBG] from='client.? 172.21.15.6:0/133522854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:27.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:27 smithi006 bash[27292]: audit 2023-08-28T01:46:27.056052+0000 mon.c (mon.2) 2691 : audit [DBG] from='client.? 172.21.15.6:0/133522854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:27.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:27 smithi006 bash[33918]: audit 2023-08-28T01:46:27.056052+0000 mon.c (mon.2) 2691 : audit [DBG] from='client.? 172.21.15.6:0/133522854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:28.290 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:27 smithi130 bash[51302]: debug 2023-08-28T01:46:27.925+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:27.926518+0000) 2023-08-28T01:46:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:28 smithi130 bash[27579]: cluster 2023-08-28T01:46:26.596525+0000 mgr.y (mgr.14901) 6835 : cluster [DBG] pgmap v6842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:28.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:28 smithi006 bash[27292]: cluster 2023-08-28T01:46:26.596525+0000 mgr.y (mgr.14901) 6835 : cluster [DBG] pgmap v6842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:28.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:28 smithi006 bash[33918]: cluster 2023-08-28T01:46:26.596525+0000 mgr.y (mgr.14901) 6835 : cluster [DBG] pgmap v6842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:29.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:28 smithi130 bash[51302]: debug 2023-08-28T01:46:28.920+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:28.926784+0000) 2023-08-28T01:46:30.294 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:29 smithi130 bash[51302]: debug 2023-08-28T01:46:29.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:29.927065+0000) 2023-08-28T01:46:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:30 smithi130 bash[27579]: cluster 2023-08-28T01:46:28.597206+0000 mgr.y (mgr.14901) 6836 : cluster [DBG] pgmap v6843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:30 smithi130 bash[27579]: audit 2023-08-28T01:46:29.482477+0000 mon.a (mon.0) 4476 : audit [DBG] from='client.? 172.21.15.6:0/1797516226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:30 smithi006 bash[33918]: cluster 2023-08-28T01:46:28.597206+0000 mgr.y (mgr.14901) 6836 : cluster [DBG] pgmap v6843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:30 smithi006 bash[33918]: audit 2023-08-28T01:46:29.482477+0000 mon.a (mon.0) 4476 : audit [DBG] from='client.? 172.21.15.6:0/1797516226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:30 smithi006 bash[27292]: cluster 2023-08-28T01:46:28.597206+0000 mgr.y (mgr.14901) 6836 : cluster [DBG] pgmap v6843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:30 smithi006 bash[27292]: audit 2023-08-28T01:46:29.482477+0000 mon.a (mon.0) 4476 : audit [DBG] from='client.? 172.21.15.6:0/1797516226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:31.113 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:46:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:46:30] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:46:31.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:30 smithi130 bash[51302]: debug 2023-08-28T01:46:30.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:30.927380+0000) 2023-08-28T01:46:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:46:31.113Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:46:31.115Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:32.300 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:31 smithi130 bash[51302]: debug 2023-08-28T01:46:31.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:31.927603+0000) 2023-08-28T01:46:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:32 smithi130 bash[27579]: cluster 2023-08-28T01:46:30.598454+0000 mgr.y (mgr.14901) 6837 : cluster [DBG] pgmap v6844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:32 smithi130 bash[27579]: audit 2023-08-28T01:46:31.904307+0000 mon.c (mon.2) 2692 : audit [DBG] from='client.? 172.21.15.6:0/113514917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:32.681 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:32 smithi006 bash[33918]: cluster 2023-08-28T01:46:30.598454+0000 mgr.y (mgr.14901) 6837 : cluster [DBG] pgmap v6844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:32.681 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:32 smithi006 bash[33918]: audit 2023-08-28T01:46:31.904307+0000 mon.c (mon.2) 2692 : audit [DBG] from='client.? 172.21.15.6:0/113514917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:32.681 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:32 smithi006 bash[27292]: cluster 2023-08-28T01:46:30.598454+0000 mgr.y (mgr.14901) 6837 : cluster [DBG] pgmap v6844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:32.681 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:32 smithi006 bash[27292]: audit 2023-08-28T01:46:31.904307+0000 mon.c (mon.2) 2692 : audit [DBG] from='client.? 172.21.15.6:0/113514917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:46:32.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:46:32.683Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:33.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:32 smithi130 bash[51302]: debug 2023-08-28T01:46:32.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:32.927893+0000) 2023-08-28T01:46:34.306 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:33 smithi130 bash[51302]: debug 2023-08-28T01:46:33.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:33.928164+0000) 2023-08-28T01:46:34.566 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:34 smithi006 bash[27292]: cluster 2023-08-28T01:46:32.599228+0000 mgr.y (mgr.14901) 6838 : cluster [DBG] pgmap v6845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:34.566 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:34 smithi006 bash[33918]: cluster 2023-08-28T01:46:32.599228+0000 mgr.y (mgr.14901) 6838 : cluster [DBG] pgmap v6845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:34 smithi130 bash[27579]: cluster 2023-08-28T01:46:32.599228+0000 mgr.y (mgr.14901) 6838 : cluster [DBG] pgmap v6845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:46:34.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:46:34.568Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:35.309 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:34 smithi130 bash[51302]: debug 2023-08-28T01:46:34.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:34.928512+0000) 2023-08-28T01:46:35.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:35 smithi130 bash[27579]: audit 2023-08-28T01:46:34.318136+0000 mon.c (mon.2) 2693 : audit [DBG] from='client.? 172.21.15.6:0/3049986792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:35.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:35 smithi006 bash[27292]: audit 2023-08-28T01:46:34.318136+0000 mon.c (mon.2) 2693 : audit [DBG] from='client.? 172.21.15.6:0/3049986792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:35.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:35 smithi006 bash[33918]: audit 2023-08-28T01:46:34.318136+0000 mon.c (mon.2) 2693 : audit [DBG] from='client.? 172.21.15.6:0/3049986792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:36.318 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:35 smithi130 bash[51302]: debug 2023-08-28T01:46:35.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:35.928817+0000) 2023-08-28T01:46:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:36 smithi130 bash[27579]: cluster 2023-08-28T01:46:34.600353+0000 mgr.y (mgr.14901) 6839 : cluster [DBG] pgmap v6846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:36.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:36 smithi006 bash[33918]: cluster 2023-08-28T01:46:34.600353+0000 mgr.y (mgr.14901) 6839 : cluster [DBG] pgmap v6846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:36.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:36 smithi006 bash[27292]: cluster 2023-08-28T01:46:34.600353+0000 mgr.y (mgr.14901) 6839 : cluster [DBG] pgmap v6846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:37.327 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:36 smithi130 bash[51302]: debug 2023-08-28T01:46:36.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:36.929083+0000) 2023-08-28T01:46:37.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:37 smithi130 bash[27579]: audit 2023-08-28T01:46:36.729507+0000 mon.a (mon.0) 4477 : audit [DBG] from='client.? 172.21.15.6:0/2596553719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:37.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:37 smithi006 bash[27292]: audit 2023-08-28T01:46:36.729507+0000 mon.a (mon.0) 4477 : audit [DBG] from='client.? 172.21.15.6:0/2596553719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:37.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:37 smithi006 bash[33918]: audit 2023-08-28T01:46:36.729507+0000 mon.a (mon.0) 4477 : audit [DBG] from='client.? 172.21.15.6:0/2596553719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:38.330 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:37 smithi130 bash[51302]: debug 2023-08-28T01:46:37.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:37.929354+0000) 2023-08-28T01:46:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:38 smithi130 bash[27579]: cluster 2023-08-28T01:46:36.601002+0000 mgr.y (mgr.14901) 6840 : cluster [DBG] pgmap v6847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:38.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:38 smithi006 bash[27292]: cluster 2023-08-28T01:46:36.601002+0000 mgr.y (mgr.14901) 6840 : cluster [DBG] pgmap v6847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:38 smithi006 bash[33918]: cluster 2023-08-28T01:46:36.601002+0000 mgr.y (mgr.14901) 6840 : cluster [DBG] pgmap v6847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:39.337 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:38 smithi130 bash[51302]: debug 2023-08-28T01:46:38.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:38.929646+0000) 2023-08-28T01:46:39.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:39 smithi130 bash[27579]: audit 2023-08-28T01:46:39.145661+0000 mon.a (mon.0) 4478 : audit [DBG] from='client.? 172.21.15.6:0/1624632378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:39.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:39 smithi006 bash[33918]: audit 2023-08-28T01:46:39.145661+0000 mon.a (mon.0) 4478 : audit [DBG] from='client.? 172.21.15.6:0/1624632378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:39.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:39 smithi006 bash[27292]: audit 2023-08-28T01:46:39.145661+0000 mon.a (mon.0) 4478 : audit [DBG] from='client.? 172.21.15.6:0/1624632378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:40.340 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:39 smithi130 bash[51302]: debug 2023-08-28T01:46:39.928+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:39.929944+0000) 2023-08-28T01:46:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:40 smithi130 bash[27579]: cluster 2023-08-28T01:46:38.601605+0000 mgr.y (mgr.14901) 6841 : cluster [DBG] pgmap v6848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:40.953 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:40 smithi006 bash[33918]: cluster 2023-08-28T01:46:38.601605+0000 mgr.y (mgr.14901) 6841 : cluster [DBG] pgmap v6848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:40.953 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:40 smithi006 bash[27292]: cluster 2023-08-28T01:46:38.601605+0000 mgr.y (mgr.14901) 6841 : cluster [DBG] pgmap v6848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:41.114 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:46:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:46:40] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:46:41.379 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:40 smithi130 bash[51302]: debug 2023-08-28T01:46:40.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:40.930255+0000) 2023-08-28T01:46:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:46:41.113Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:41.769 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:46:41.116Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:42.344 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:41 smithi130 bash[51302]: debug 2023-08-28T01:46:41.924+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:41.930544+0000) 2023-08-28T01:46:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:42 smithi130 bash[27579]: cluster 2023-08-28T01:46:40.602756+0000 mgr.y (mgr.14901) 6842 : cluster [DBG] pgmap v6849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:42 smithi130 bash[27579]: audit 2023-08-28T01:46:41.563183+0000 mon.a (mon.0) 4479 : audit [DBG] from='client.? 172.21.15.6:0/1999251731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:42.682 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:42 smithi006 bash[33918]: cluster 2023-08-28T01:46:40.602756+0000 mgr.y (mgr.14901) 6842 : cluster [DBG] pgmap v6849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:42.682 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:42 smithi006 bash[33918]: audit 2023-08-28T01:46:41.563183+0000 mon.a (mon.0) 4479 : audit [DBG] from='client.? 172.21.15.6:0/1999251731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:42.682 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:42 smithi006 bash[27292]: cluster 2023-08-28T01:46:40.602756+0000 mgr.y (mgr.14901) 6842 : cluster [DBG] pgmap v6849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:42.682 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:42 smithi006 bash[27292]: audit 2023-08-28T01:46:41.563183+0000 mon.a (mon.0) 4479 : audit [DBG] from='client.? 172.21.15.6:0/1999251731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:46:42.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:46:42.684Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:43.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:42 smithi130 bash[51302]: debug 2023-08-28T01:46:42.928+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:42.930836+0000) 2023-08-28T01:46:44.350 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:43 smithi130 bash[51302]: debug 2023-08-28T01:46:43.928+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:43.931183+0000) 2023-08-28T01:46:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:44 smithi130 bash[27579]: cluster 2023-08-28T01:46:42.603552+0000 mgr.y (mgr.14901) 6843 : cluster [DBG] pgmap v6850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:44 smithi130 bash[27579]: audit 2023-08-28T01:46:43.974990+0000 mon.a (mon.0) 4480 : audit [DBG] from='client.? 172.21.15.6:0/1645126240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:44.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:44 smithi006 bash[27292]: cluster 2023-08-28T01:46:42.603552+0000 mgr.y (mgr.14901) 6843 : cluster [DBG] pgmap v6850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:44.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:44 smithi006 bash[27292]: audit 2023-08-28T01:46:43.974990+0000 mon.a (mon.0) 4480 : audit [DBG] from='client.? 172.21.15.6:0/1645126240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:44.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:44 smithi006 bash[33918]: cluster 2023-08-28T01:46:42.603552+0000 mgr.y (mgr.14901) 6843 : cluster [DBG] pgmap v6850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:44.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:44 smithi006 bash[33918]: audit 2023-08-28T01:46:43.974990+0000 mon.a (mon.0) 4480 : audit [DBG] from='client.? 172.21.15.6:0/1645126240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:46:44.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:46:44.568Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:45.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:44 smithi130 bash[51302]: debug 2023-08-28T01:46:44.928+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:44.931466+0000) 2023-08-28T01:46:46.354 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:45 smithi130 bash[51302]: debug 2023-08-28T01:46:45.928+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:45.931764+0000) 2023-08-28T01:46:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:46 smithi130 bash[27579]: cluster 2023-08-28T01:46:44.604706+0000 mgr.y (mgr.14901) 6844 : cluster [DBG] pgmap v6851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:46.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:46 smithi006 bash[27292]: cluster 2023-08-28T01:46:44.604706+0000 mgr.y (mgr.14901) 6844 : cluster [DBG] pgmap v6851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:46.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:46 smithi006 bash[33918]: cluster 2023-08-28T01:46:44.604706+0000 mgr.y (mgr.14901) 6844 : cluster [DBG] pgmap v6851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:47.357 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:46 smithi130 bash[51302]: debug 2023-08-28T01:46:46.928+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:46.932105+0000) 2023-08-28T01:46:47.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:47 smithi130 bash[27579]: audit 2023-08-28T01:46:46.394520+0000 mon.c (mon.2) 2694 : audit [DBG] from='client.? 172.21.15.6:0/747206101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:47.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:47 smithi006 bash[27292]: audit 2023-08-28T01:46:46.394520+0000 mon.c (mon.2) 2694 : audit [DBG] from='client.? 172.21.15.6:0/747206101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:47.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:47 smithi006 bash[33918]: audit 2023-08-28T01:46:46.394520+0000 mon.c (mon.2) 2694 : audit [DBG] from='client.? 172.21.15.6:0/747206101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:48.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:47 smithi130 bash[51302]: debug 2023-08-28T01:46:47.928+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:47.932407+0000) 2023-08-28T01:46:48.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:48 smithi006 bash[27292]: cluster 2023-08-28T01:46:46.605369+0000 mgr.y (mgr.14901) 6845 : cluster [DBG] pgmap v6852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:48.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:48 smithi006 bash[33918]: cluster 2023-08-28T01:46:46.605369+0000 mgr.y (mgr.14901) 6845 : cluster [DBG] pgmap v6852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:48.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:48 smithi130 bash[27579]: cluster 2023-08-28T01:46:46.605369+0000 mgr.y (mgr.14901) 6845 : cluster [DBG] pgmap v6852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:49.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:48 smithi130 bash[51302]: debug 2023-08-28T01:46:48.928+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:48.932731+0000) 2023-08-28T01:46:49.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:49 smithi006 bash[33918]: audit 2023-08-28T01:46:48.811549+0000 mon.a (mon.0) 4481 : audit [DBG] from='client.? 172.21.15.6:0/1595683911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:49.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:49 smithi006 bash[27292]: audit 2023-08-28T01:46:48.811549+0000 mon.a (mon.0) 4481 : audit [DBG] from='client.? 172.21.15.6:0/1595683911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:49.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:49 smithi130 bash[27579]: audit 2023-08-28T01:46:48.811549+0000 mon.a (mon.0) 4481 : audit [DBG] from='client.? 172.21.15.6:0/1595683911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:50.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:49 smithi130 bash[51302]: debug 2023-08-28T01:46:49.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:49.933090+0000) 2023-08-28T01:46:50.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:50 smithi006 bash[33918]: cluster 2023-08-28T01:46:48.605934+0000 mgr.y (mgr.14901) 6846 : cluster [DBG] pgmap v6853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:50 smithi006 bash[27292]: cluster 2023-08-28T01:46:48.605934+0000 mgr.y (mgr.14901) 6846 : cluster [DBG] pgmap v6853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:50.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:50 smithi130 bash[27579]: cluster 2023-08-28T01:46:48.605934+0000 mgr.y (mgr.14901) 6846 : cluster [DBG] pgmap v6853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:51.115 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:46:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:46:50] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T01:46:51.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:50 smithi130 bash[51302]: debug 2023-08-28T01:46:50.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:50.933405+0000) 2023-08-28T01:46:51.386 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:46:51.115Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:51.386 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:46:51.117Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:51.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:51 smithi006 bash[33918]: audit 2023-08-28T01:46:51.227298+0000 mon.c (mon.2) 2695 : audit [DBG] from='client.? 172.21.15.6:0/3842957284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:51.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:51 smithi006 bash[27292]: audit 2023-08-28T01:46:51.227298+0000 mon.c (mon.2) 2695 : audit [DBG] from='client.? 172.21.15.6:0/3842957284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:51.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:51 smithi130 bash[27579]: audit 2023-08-28T01:46:51.227298+0000 mon.c (mon.2) 2695 : audit [DBG] from='client.? 172.21.15.6:0/3842957284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:52.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:51 smithi130 bash[51302]: debug 2023-08-28T01:46:51.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:51.933682+0000) 2023-08-28T01:46:52.682 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:52 smithi006 bash[27292]: cluster 2023-08-28T01:46:50.607020+0000 mgr.y (mgr.14901) 6847 : cluster [DBG] pgmap v6854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:52.682 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:52 smithi006 bash[33918]: cluster 2023-08-28T01:46:50.607020+0000 mgr.y (mgr.14901) 6847 : cluster [DBG] pgmap v6854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:52.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:52 smithi130 bash[27579]: cluster 2023-08-28T01:46:50.607020+0000 mgr.y (mgr.14901) 6847 : cluster [DBG] pgmap v6854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:46:52.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:46:52.684Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:53.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:52 smithi130 bash[51302]: debug 2023-08-28T01:46:52.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:52.933984+0000) 2023-08-28T01:46:54.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:53 smithi130 bash[51302]: debug 2023-08-28T01:46:53.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:53.934254+0000) 2023-08-28T01:46:54.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:54 smithi006 bash[33918]: cluster 2023-08-28T01:46:52.607673+0000 mgr.y (mgr.14901) 6848 : cluster [DBG] pgmap v6855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:54.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:54 smithi006 bash[33918]: audit 2023-08-28T01:46:53.639676+0000 mon.a (mon.0) 4482 : audit [DBG] from='client.? 172.21.15.6:0/854518239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:54.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:54 smithi006 bash[27292]: cluster 2023-08-28T01:46:52.607673+0000 mgr.y (mgr.14901) 6848 : cluster [DBG] pgmap v6855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:54.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:54 smithi006 bash[27292]: audit 2023-08-28T01:46:53.639676+0000 mon.a (mon.0) 4482 : audit [DBG] from='client.? 172.21.15.6:0/854518239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:46:54.567Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:46:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:46:54.569Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:46:54.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:54 smithi130 bash[27579]: cluster 2023-08-28T01:46:52.607673+0000 mgr.y (mgr.14901) 6848 : cluster [DBG] pgmap v6855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:54.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:54 smithi130 bash[27579]: audit 2023-08-28T01:46:53.639676+0000 mon.a (mon.0) 4482 : audit [DBG] from='client.? 172.21.15.6:0/854518239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:55.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:54 smithi130 bash[51302]: debug 2023-08-28T01:46:54.928+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:54.934565+0000) 2023-08-28T01:46:56.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:55 smithi130 bash[51302]: debug 2023-08-28T01:46:55.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:55.934887+0000) 2023-08-28T01:46:56.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:56 smithi006 bash[27292]: cluster 2023-08-28T01:46:54.608876+0000 mgr.y (mgr.14901) 6849 : cluster [DBG] pgmap v6856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:56.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:56 smithi006 bash[27292]: audit 2023-08-28T01:46:56.053414+0000 mon.a (mon.0) 4483 : audit [DBG] from='client.? 172.21.15.6:0/1284819763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:56.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:56 smithi006 bash[33918]: cluster 2023-08-28T01:46:54.608876+0000 mgr.y (mgr.14901) 6849 : cluster [DBG] pgmap v6856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:56.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:56 smithi006 bash[33918]: audit 2023-08-28T01:46:56.053414+0000 mon.a (mon.0) 4483 : audit [DBG] from='client.? 172.21.15.6:0/1284819763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:56 smithi130 bash[27579]: cluster 2023-08-28T01:46:54.608876+0000 mgr.y (mgr.14901) 6849 : cluster [DBG] pgmap v6856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:56 smithi130 bash[27579]: audit 2023-08-28T01:46:56.053414+0000 mon.a (mon.0) 4483 : audit [DBG] from='client.? 172.21.15.6:0/1284819763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:57.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:56 smithi130 bash[51302]: debug 2023-08-28T01:46:56.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:56.935163+0000) 2023-08-28T01:46:58.271 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:57 smithi130 bash[51302]: debug 2023-08-28T01:46:57.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:57.935373+0000) 2023-08-28T01:46:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:58 smithi130 bash[27579]: cluster 2023-08-28T01:46:56.609701+0000 mgr.y (mgr.14901) 6850 : cluster [DBG] pgmap v6857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:58.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:58 smithi006 bash[33918]: cluster 2023-08-28T01:46:56.609701+0000 mgr.y (mgr.14901) 6850 : cluster [DBG] pgmap v6857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:58.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:58 smithi006 bash[27292]: cluster 2023-08-28T01:46:56.609701+0000 mgr.y (mgr.14901) 6850 : cluster [DBG] pgmap v6857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:46:59.280 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:58 smithi130 bash[51302]: debug 2023-08-28T01:46:58.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:58.935642+0000) 2023-08-28T01:46:59.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:46:59 smithi130 bash[27579]: audit 2023-08-28T01:46:58.469878+0000 mon.a (mon.0) 4484 : audit [DBG] from='client.? 172.21.15.6:0/2170785713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:59.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:46:59 smithi006 bash[33918]: audit 2023-08-28T01:46:58.469878+0000 mon.a (mon.0) 4484 : audit [DBG] from='client.? 172.21.15.6:0/2170785713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:59.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:46:59 smithi006 bash[27292]: audit 2023-08-28T01:46:58.469878+0000 mon.a (mon.0) 4484 : audit [DBG] from='client.? 172.21.15.6:0/2170785713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:00.283 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:46:59 smithi130 bash[51302]: debug 2023-08-28T01:46:59.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:59.935920+0000) 2023-08-28T01:47:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:00 smithi130 bash[27579]: cluster 2023-08-28T01:46:58.610356+0000 mgr.y (mgr.14901) 6851 : cluster [DBG] pgmap v6858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:00 smithi006 bash[33918]: cluster 2023-08-28T01:46:58.610356+0000 mgr.y (mgr.14901) 6851 : cluster [DBG] pgmap v6858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:00 smithi006 bash[27292]: cluster 2023-08-28T01:46:58.610356+0000 mgr.y (mgr.14901) 6851 : cluster [DBG] pgmap v6858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:01.115 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:47:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:47:00] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T01:47:01.292 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:00 smithi130 bash[51302]: debug 2023-08-28T01:47:00.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:00.936201+0000) 2023-08-28T01:47:01.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:01 smithi006 bash[27292]: audit 2023-08-28T01:47:00.890342+0000 mon.a (mon.0) 4485 : audit [DBG] from='client.? 172.21.15.6:0/1785084483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:01.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:01 smithi006 bash[33918]: audit 2023-08-28T01:47:00.890342+0000 mon.a (mon.0) 4485 : audit [DBG] from='client.? 172.21.15.6:0/1785084483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:47:01.115Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:47:01.117Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:01.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:01 smithi130 bash[27579]: audit 2023-08-28T01:47:00.890342+0000 mon.a (mon.0) 4485 : audit [DBG] from='client.? 172.21.15.6:0/1785084483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:02.296 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:01 smithi130 bash[51302]: debug 2023-08-28T01:47:01.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:01.936478+0000) 2023-08-28T01:47:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:02 smithi130 bash[27579]: cluster 2023-08-28T01:47:00.611066+0000 mgr.y (mgr.14901) 6852 : cluster [DBG] pgmap v6859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:02.683 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:02 smithi006 bash[33918]: cluster 2023-08-28T01:47:00.611066+0000 mgr.y (mgr.14901) 6852 : cluster [DBG] pgmap v6859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:02.683 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:02 smithi006 bash[27292]: cluster 2023-08-28T01:47:00.611066+0000 mgr.y (mgr.14901) 6852 : cluster [DBG] pgmap v6859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:47:02.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:47:02.685Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:03.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:02 smithi130 bash[51302]: debug 2023-08-28T01:47:02.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:02.936781+0000) 2023-08-28T01:47:04.300 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:03 smithi130 bash[51302]: debug 2023-08-28T01:47:03.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:03.937072+0000) 2023-08-28T01:47:04.568 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:04 smithi006 bash[33918]: cluster 2023-08-28T01:47:02.611681+0000 mgr.y (mgr.14901) 6853 : cluster [DBG] pgmap v6860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:04.568 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:04 smithi006 bash[33918]: audit 2023-08-28T01:47:03.305151+0000 mon.c (mon.2) 2696 : audit [DBG] from='client.? 172.21.15.6:0/2879644563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:04.568 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:04 smithi006 bash[27292]: cluster 2023-08-28T01:47:02.611681+0000 mgr.y (mgr.14901) 6853 : cluster [DBG] pgmap v6860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:04.568 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:04 smithi006 bash[27292]: audit 2023-08-28T01:47:03.305151+0000 mon.c (mon.2) 2696 : audit [DBG] from='client.? 172.21.15.6:0/2879644563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:04 smithi130 bash[27579]: cluster 2023-08-28T01:47:02.611681+0000 mgr.y (mgr.14901) 6853 : cluster [DBG] pgmap v6860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:04 smithi130 bash[27579]: audit 2023-08-28T01:47:03.305151+0000 mon.c (mon.2) 2696 : audit [DBG] from='client.? 172.21.15.6:0/2879644563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:47:04.567Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:05.008 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:47:04.570Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:05.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:04 smithi130 bash[51302]: debug 2023-08-28T01:47:04.936+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:04.937381+0000) 2023-08-28T01:47:06.304 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:05 smithi130 bash[51302]: debug 2023-08-28T01:47:05.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:05.937640+0000) 2023-08-28T01:47:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:06 smithi130 bash[27579]: cluster 2023-08-28T01:47:04.612828+0000 mgr.y (mgr.14901) 6854 : cluster [DBG] pgmap v6861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:06 smithi130 bash[27579]: audit 2023-08-28T01:47:05.722320+0000 mon.a (mon.0) 4486 : audit [DBG] from='client.? 172.21.15.6:0/3984611272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:06.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:06 smithi006 bash[27292]: cluster 2023-08-28T01:47:04.612828+0000 mgr.y (mgr.14901) 6854 : cluster [DBG] pgmap v6861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:06.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:06 smithi006 bash[27292]: audit 2023-08-28T01:47:05.722320+0000 mon.a (mon.0) 4486 : audit [DBG] from='client.? 172.21.15.6:0/3984611272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:06 smithi006 bash[33918]: cluster 2023-08-28T01:47:04.612828+0000 mgr.y (mgr.14901) 6854 : cluster [DBG] pgmap v6861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:06 smithi006 bash[33918]: audit 2023-08-28T01:47:05.722320+0000 mon.a (mon.0) 4486 : audit [DBG] from='client.? 172.21.15.6:0/3984611272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:07.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:06 smithi130 bash[51302]: debug 2023-08-28T01:47:06.936+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:06.937937+0000) 2023-08-28T01:47:08.309 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:07 smithi130 bash[51302]: debug 2023-08-28T01:47:07.936+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:07.938200+0000) 2023-08-28T01:47:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:08 smithi130 bash[27579]: cluster 2023-08-28T01:47:06.613485+0000 mgr.y (mgr.14901) 6855 : cluster [DBG] pgmap v6862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:08 smithi130 bash[27579]: audit 2023-08-28T01:47:08.138050+0000 mon.a (mon.0) 4487 : audit [DBG] from='client.? 172.21.15.6:0/1185565660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:08 smithi006 bash[33918]: cluster 2023-08-28T01:47:06.613485+0000 mgr.y (mgr.14901) 6855 : cluster [DBG] pgmap v6862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:08 smithi006 bash[33918]: audit 2023-08-28T01:47:08.138050+0000 mon.a (mon.0) 4487 : audit [DBG] from='client.? 172.21.15.6:0/1185565660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:08.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:08 smithi006 bash[27292]: cluster 2023-08-28T01:47:06.613485+0000 mgr.y (mgr.14901) 6855 : cluster [DBG] pgmap v6862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:08 smithi006 bash[27292]: audit 2023-08-28T01:47:08.138050+0000 mon.a (mon.0) 4487 : audit [DBG] from='client.? 172.21.15.6:0/1185565660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:09.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:08 smithi130 bash[51302]: debug 2023-08-28T01:47:08.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:08.938492+0000) 2023-08-28T01:47:10.314 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:09 smithi130 bash[51302]: debug 2023-08-28T01:47:09.932+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:09.938721+0000) 2023-08-28T01:47:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:10 smithi130 bash[27579]: cluster 2023-08-28T01:47:08.614189+0000 mgr.y (mgr.14901) 6856 : cluster [DBG] pgmap v6863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:10 smithi006 bash[33918]: cluster 2023-08-28T01:47:08.614189+0000 mgr.y (mgr.14901) 6856 : cluster [DBG] pgmap v6863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:10.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:10 smithi006 bash[27292]: cluster 2023-08-28T01:47:08.614189+0000 mgr.y (mgr.14901) 6856 : cluster [DBG] pgmap v6863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:11.115 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:47:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:47:10] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T01:47:11.322 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:10 smithi130 bash[51302]: debug 2023-08-28T01:47:10.936+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:10.939010+0000) 2023-08-28T01:47:11.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:11 smithi006 bash[27292]: audit 2023-08-28T01:47:10.565632+0000 mon.c (mon.2) 2697 : audit [DBG] from='client.? 172.21.15.6:0/2727277552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:11.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:11 smithi006 bash[33918]: audit 2023-08-28T01:47:10.565632+0000 mon.c (mon.2) 2697 : audit [DBG] from='client.? 172.21.15.6:0/2727277552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:47:11.116Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:47:11.117Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:11.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:11 smithi130 bash[27579]: audit 2023-08-28T01:47:10.565632+0000 mon.c (mon.2) 2697 : audit [DBG] from='client.? 172.21.15.6:0/2727277552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:12.325 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:11 smithi130 bash[51302]: debug 2023-08-28T01:47:11.936+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:11.939305+0000) 2023-08-28T01:47:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:12 smithi130 bash[27579]: cluster 2023-08-28T01:47:10.615350+0000 mgr.y (mgr.14901) 6857 : cluster [DBG] pgmap v6864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:12.684 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:12 smithi006 bash[27292]: cluster 2023-08-28T01:47:10.615350+0000 mgr.y (mgr.14901) 6857 : cluster [DBG] pgmap v6864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:12.684 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:12 smithi006 bash[33918]: cluster 2023-08-28T01:47:10.615350+0000 mgr.y (mgr.14901) 6857 : cluster [DBG] pgmap v6864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:47:12.684Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:47:12.686Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:13.334 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:12 smithi130 bash[51302]: debug 2023-08-28T01:47:12.936+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:12.939577+0000) 2023-08-28T01:47:13.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:13 smithi130 bash[27579]: audit 2023-08-28T01:47:12.980803+0000 mon.c (mon.2) 2698 : audit [DBG] from='client.? 172.21.15.6:0/183620568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:13.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:13 smithi006 bash[27292]: audit 2023-08-28T01:47:12.980803+0000 mon.c (mon.2) 2698 : audit [DBG] from='client.? 172.21.15.6:0/183620568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:13.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:13 smithi006 bash[33918]: audit 2023-08-28T01:47:12.980803+0000 mon.c (mon.2) 2698 : audit [DBG] from='client.? 172.21.15.6:0/183620568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:14.336 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:13 smithi130 bash[51302]: debug 2023-08-28T01:47:13.936+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:13.939868+0000) 2023-08-28T01:47:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:14 smithi130 bash[27579]: cluster 2023-08-28T01:47:12.616155+0000 mgr.y (mgr.14901) 6858 : cluster [DBG] pgmap v6865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:14.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:14 smithi006 bash[33918]: cluster 2023-08-28T01:47:12.616155+0000 mgr.y (mgr.14901) 6858 : cluster [DBG] pgmap v6865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:14.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:14 smithi006 bash[27292]: cluster 2023-08-28T01:47:12.616155+0000 mgr.y (mgr.14901) 6858 : cluster [DBG] pgmap v6865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:47:14.568Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:47:14.570Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:15.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:14 smithi130 bash[51302]: debug 2023-08-28T01:47:14.936+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:14.940148+0000) 2023-08-28T01:47:16.340 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:15 smithi130 bash[51302]: debug 2023-08-28T01:47:15.940+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:15.940416+0000) 2023-08-28T01:47:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:16 smithi130 bash[27579]: cluster 2023-08-28T01:47:14.617481+0000 mgr.y (mgr.14901) 6859 : cluster [DBG] pgmap v6866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:16 smithi130 bash[27579]: audit 2023-08-28T01:47:15.395445+0000 mon.c (mon.2) 2699 : audit [DBG] from='client.? 172.21.15.6:0/2611765826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:16.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:16 smithi006 bash[33918]: cluster 2023-08-28T01:47:14.617481+0000 mgr.y (mgr.14901) 6859 : cluster [DBG] pgmap v6866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:16 smithi006 bash[33918]: audit 2023-08-28T01:47:15.395445+0000 mon.c (mon.2) 2699 : audit [DBG] from='client.? 172.21.15.6:0/2611765826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:16 smithi006 bash[27292]: cluster 2023-08-28T01:47:14.617481+0000 mgr.y (mgr.14901) 6859 : cluster [DBG] pgmap v6866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:16 smithi006 bash[27292]: audit 2023-08-28T01:47:15.395445+0000 mon.c (mon.2) 2699 : audit [DBG] from='client.? 172.21.15.6:0/2611765826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:17.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:16 smithi130 bash[51302]: debug 2023-08-28T01:47:16.936+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:16.940673+0000) 2023-08-28T01:47:18.345 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:17 smithi130 bash[51302]: debug 2023-08-28T01:47:17.936+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:17.940958+0000) 2023-08-28T01:47:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:18 smithi130 bash[27579]: cluster 2023-08-28T01:47:16.618316+0000 mgr.y (mgr.14901) 6860 : cluster [DBG] pgmap v6867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:18 smithi130 bash[27579]: audit 2023-08-28T01:47:17.806900+0000 mon.a (mon.0) 4488 : audit [DBG] from='client.? 172.21.15.6:0/1327458385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:18.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:18 smithi006 bash[33918]: cluster 2023-08-28T01:47:16.618316+0000 mgr.y (mgr.14901) 6860 : cluster [DBG] pgmap v6867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:18.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:18 smithi006 bash[33918]: audit 2023-08-28T01:47:17.806900+0000 mon.a (mon.0) 4488 : audit [DBG] from='client.? 172.21.15.6:0/1327458385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:18 smithi006 bash[27292]: cluster 2023-08-28T01:47:16.618316+0000 mgr.y (mgr.14901) 6860 : cluster [DBG] pgmap v6867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:18.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:18 smithi006 bash[27292]: audit 2023-08-28T01:47:17.806900+0000 mon.a (mon.0) 4488 : audit [DBG] from='client.? 172.21.15.6:0/1327458385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:19.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:18 smithi130 bash[51302]: debug 2023-08-28T01:47:18.940+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:18.941259+0000) 2023-08-28T01:47:20.350 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:19 smithi130 bash[51302]: debug 2023-08-28T01:47:19.940+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:19.941540+0000) 2023-08-28T01:47:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:20 smithi130 bash[27579]: cluster 2023-08-28T01:47:18.619095+0000 mgr.y (mgr.14901) 6861 : cluster [DBG] pgmap v6868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:20 smithi130 bash[27579]: audit 2023-08-28T01:47:19.653326+0000 mon.b (mon.1) 772 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:47:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:20 smithi130 bash[27579]: audit 2023-08-28T01:47:20.220300+0000 mon.c (mon.2) 2700 : audit [DBG] from='client.? 172.21.15.6:0/3125403041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:20.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:20 smithi006 bash[27292]: cluster 2023-08-28T01:47:18.619095+0000 mgr.y (mgr.14901) 6861 : cluster [DBG] pgmap v6868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:20.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:20 smithi006 bash[27292]: audit 2023-08-28T01:47:19.653326+0000 mon.b (mon.1) 772 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:47:20.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:20 smithi006 bash[27292]: audit 2023-08-28T01:47:20.220300+0000 mon.c (mon.2) 2700 : audit [DBG] from='client.? 172.21.15.6:0/3125403041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:20 smithi006 bash[33918]: cluster 2023-08-28T01:47:18.619095+0000 mgr.y (mgr.14901) 6861 : cluster [DBG] pgmap v6868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:20 smithi006 bash[33918]: audit 2023-08-28T01:47:19.653326+0000 mon.b (mon.1) 772 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:47:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:20 smithi006 bash[33918]: audit 2023-08-28T01:47:20.220300+0000 mon.c (mon.2) 2700 : audit [DBG] from='client.? 172.21.15.6:0/3125403041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:21.117 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:47:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:47:20] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:47:21.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:20 smithi130 bash[51302]: debug 2023-08-28T01:47:20.936+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:20.941831+0000) 2023-08-28T01:47:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:47:21.117Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:47:21.119Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:22.356 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:21 smithi130 bash[51302]: debug 2023-08-28T01:47:21.936+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:21.942129+0000) 2023-08-28T01:47:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:22 smithi130 bash[27579]: cluster 2023-08-28T01:47:20.620253+0000 mgr.y (mgr.14901) 6862 : cluster [DBG] pgmap v6869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:22.685 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:22 smithi006 bash[27292]: cluster 2023-08-28T01:47:20.620253+0000 mgr.y (mgr.14901) 6862 : cluster [DBG] pgmap v6869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:22.685 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:22 smithi006 bash[33918]: cluster 2023-08-28T01:47:20.620253+0000 mgr.y (mgr.14901) 6862 : cluster [DBG] pgmap v6869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:47:22.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:47:22.687Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:23.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:22 smithi130 bash[51302]: debug 2023-08-28T01:47:22.936+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:22.942355+0000) 2023-08-28T01:47:23.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:23 smithi006 bash[27292]: audit 2023-08-28T01:47:22.652530+0000 mon.c (mon.2) 2701 : audit [DBG] from='client.? 172.21.15.6:0/3715792389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:23.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:23 smithi006 bash[33918]: audit 2023-08-28T01:47:22.652530+0000 mon.c (mon.2) 2701 : audit [DBG] from='client.? 172.21.15.6:0/3715792389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:23.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:23 smithi130 bash[27579]: audit 2023-08-28T01:47:22.652530+0000 mon.c (mon.2) 2701 : audit [DBG] from='client.? 172.21.15.6:0/3715792389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:24.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:23 smithi130 bash[51302]: debug 2023-08-28T01:47:23.936+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:23.942599+0000) 2023-08-28T01:47:24.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:47:24.569Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:24.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:47:24.571Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:24.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:24 smithi006 bash[27292]: cluster 2023-08-28T01:47:22.620894+0000 mgr.y (mgr.14901) 6863 : cluster [DBG] pgmap v6870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:24.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:24 smithi006 bash[33918]: cluster 2023-08-28T01:47:22.620894+0000 mgr.y (mgr.14901) 6863 : cluster [DBG] pgmap v6870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:24.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:24 smithi130 bash[27579]: cluster 2023-08-28T01:47:22.620894+0000 mgr.y (mgr.14901) 6863 : cluster [DBG] pgmap v6870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:25.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:24 smithi130 bash[51302]: debug 2023-08-28T01:47:24.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:24.942899+0000) 2023-08-28T01:47:25.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:25 smithi006 bash[27292]: audit 2023-08-28T01:47:25.075569+0000 mon.a (mon.0) 4489 : audit [DBG] from='client.? 172.21.15.6:0/811859500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:25.821 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:25 smithi006 bash[33918]: audit 2023-08-28T01:47:25.075569+0000 mon.a (mon.0) 4489 : audit [DBG] from='client.? 172.21.15.6:0/811859500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:25.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:25 smithi130 bash[27579]: audit 2023-08-28T01:47:25.075569+0000 mon.a (mon.0) 4489 : audit [DBG] from='client.? 172.21.15.6:0/811859500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:26.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:25 smithi130 bash[51302]: debug 2023-08-28T01:47:25.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:25.943127+0000) 2023-08-28T01:47:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:26 smithi006 bash[27292]: cluster 2023-08-28T01:47:24.621869+0000 mgr.y (mgr.14901) 6864 : cluster [DBG] pgmap v6871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:26.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:26 smithi006 bash[27292]: audit 2023-08-28T01:47:25.569029+0000 mon.a (mon.0) 4490 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:47:26.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:26 smithi006 bash[27292]: audit 2023-08-28T01:47:26.023160+0000 mon.a (mon.0) 4491 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:47:26.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:26 smithi006 bash[27292]: audit 2023-08-28T01:47:26.319836+0000 mon.b (mon.1) 773 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:47:26.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:26 smithi006 bash[27292]: audit 2023-08-28T01:47:26.322181+0000 mon.b (mon.1) 774 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:47:26.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:26 smithi006 bash[27292]: audit 2023-08-28T01:47:26.330250+0000 mon.a (mon.0) 4492 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:47:26.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:26 smithi006 bash[33918]: cluster 2023-08-28T01:47:24.621869+0000 mgr.y (mgr.14901) 6864 : cluster [DBG] pgmap v6871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:26 smithi006 bash[33918]: audit 2023-08-28T01:47:25.569029+0000 mon.a (mon.0) 4490 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:47:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:26 smithi006 bash[33918]: audit 2023-08-28T01:47:26.023160+0000 mon.a (mon.0) 4491 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:47:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:26 smithi006 bash[33918]: audit 2023-08-28T01:47:26.319836+0000 mon.b (mon.1) 773 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:47:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:26 smithi006 bash[33918]: audit 2023-08-28T01:47:26.322181+0000 mon.b (mon.1) 774 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:47:26.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:26 smithi006 bash[33918]: audit 2023-08-28T01:47:26.330250+0000 mon.a (mon.0) 4492 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:47:26.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:26 smithi130 bash[27579]: cluster 2023-08-28T01:47:24.621869+0000 mgr.y (mgr.14901) 6864 : cluster [DBG] pgmap v6871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:26.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:26 smithi130 bash[27579]: audit 2023-08-28T01:47:25.569029+0000 mon.a (mon.0) 4490 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:47:26.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:26 smithi130 bash[27579]: audit 2023-08-28T01:47:26.023160+0000 mon.a (mon.0) 4491 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:47:26.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:26 smithi130 bash[27579]: audit 2023-08-28T01:47:26.319836+0000 mon.b (mon.1) 773 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:47:26.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:26 smithi130 bash[27579]: audit 2023-08-28T01:47:26.322181+0000 mon.b (mon.1) 774 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:47:26.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:26 smithi130 bash[27579]: audit 2023-08-28T01:47:26.330250+0000 mon.a (mon.0) 4492 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:47:27.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:26 smithi130 bash[51302]: debug 2023-08-28T01:47:26.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:26.943389+0000) 2023-08-28T01:47:28.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:27 smithi130 bash[51302]: debug 2023-08-28T01:47:27.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:27.943609+0000) 2023-08-28T01:47:28.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:28 smithi006 bash[27292]: cluster 2023-08-28T01:47:26.622544+0000 mgr.y (mgr.14901) 6865 : cluster [DBG] pgmap v6872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:28.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:28 smithi006 bash[27292]: audit 2023-08-28T01:47:27.484285+0000 mon.a (mon.0) 4493 : audit [DBG] from='client.? 172.21.15.6:0/546719338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:28.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:28 smithi006 bash[33918]: cluster 2023-08-28T01:47:26.622544+0000 mgr.y (mgr.14901) 6865 : cluster [DBG] pgmap v6872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:28.720 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:28 smithi006 bash[33918]: audit 2023-08-28T01:47:27.484285+0000 mon.a (mon.0) 4493 : audit [DBG] from='client.? 172.21.15.6:0/546719338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:28.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:28 smithi130 bash[27579]: cluster 2023-08-28T01:47:26.622544+0000 mgr.y (mgr.14901) 6865 : cluster [DBG] pgmap v6872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:28.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:28 smithi130 bash[27579]: audit 2023-08-28T01:47:27.484285+0000 mon.a (mon.0) 4493 : audit [DBG] from='client.? 172.21.15.6:0/546719338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:29.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:28 smithi130 bash[51302]: debug 2023-08-28T01:47:28.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:28.943885+0000) 2023-08-28T01:47:30.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:29 smithi130 bash[51302]: debug 2023-08-28T01:47:29.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:29.944161+0000) 2023-08-28T01:47:30.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:30 smithi006 bash[27292]: cluster 2023-08-28T01:47:28.623322+0000 mgr.y (mgr.14901) 6866 : cluster [DBG] pgmap v6873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:30.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:30 smithi006 bash[27292]: audit 2023-08-28T01:47:29.903601+0000 mon.a (mon.0) 4494 : audit [DBG] from='client.? 172.21.15.6:0/3480197138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:30 smithi006 bash[33918]: cluster 2023-08-28T01:47:28.623322+0000 mgr.y (mgr.14901) 6866 : cluster [DBG] pgmap v6873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:30 smithi006 bash[33918]: audit 2023-08-28T01:47:29.903601+0000 mon.a (mon.0) 4494 : audit [DBG] from='client.? 172.21.15.6:0/3480197138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:30.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:30 smithi130 bash[27579]: cluster 2023-08-28T01:47:28.623322+0000 mgr.y (mgr.14901) 6866 : cluster [DBG] pgmap v6873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:30 smithi130 bash[27579]: audit 2023-08-28T01:47:29.903601+0000 mon.a (mon.0) 4494 : audit [DBG] from='client.? 172.21.15.6:0/3480197138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:31.117 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:47:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:47:30] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T01:47:31.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:30 smithi130 bash[51302]: debug 2023-08-28T01:47:30.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:30.944452+0000) 2023-08-28T01:47:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:47:31.118Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:47:31.119Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:32.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:31 smithi130 bash[51302]: debug 2023-08-28T01:47:31.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:31.944736+0000) 2023-08-28T01:47:32.686 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:32 smithi006 bash[33918]: cluster 2023-08-28T01:47:30.624594+0000 mgr.y (mgr.14901) 6867 : cluster [DBG] pgmap v6874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:32.686 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:32 smithi006 bash[33918]: audit 2023-08-28T01:47:32.318530+0000 mon.a (mon.0) 4495 : audit [DBG] from='client.? 172.21.15.6:0/2007849684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:32.687 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:32 smithi006 bash[27292]: cluster 2023-08-28T01:47:30.624594+0000 mgr.y (mgr.14901) 6867 : cluster [DBG] pgmap v6874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:32.687 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:32 smithi006 bash[27292]: audit 2023-08-28T01:47:32.318530+0000 mon.a (mon.0) 4495 : audit [DBG] from='client.? 172.21.15.6:0/2007849684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:32 smithi130 bash[27579]: cluster 2023-08-28T01:47:30.624594+0000 mgr.y (mgr.14901) 6867 : cluster [DBG] pgmap v6874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:32 smithi130 bash[27579]: audit 2023-08-28T01:47:32.318530+0000 mon.a (mon.0) 4495 : audit [DBG] from='client.? 172.21.15.6:0/2007849684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:47:32.686Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:47:32.688Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:33.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:32 smithi130 bash[51302]: debug 2023-08-28T01:47:32.943+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:32.945068+0000) 2023-08-28T01:47:34.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:33 smithi130 bash[51302]: debug 2023-08-28T01:47:33.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:33.945365+0000) 2023-08-28T01:47:34.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:34 smithi006 bash[27292]: cluster 2023-08-28T01:47:32.625376+0000 mgr.y (mgr.14901) 6868 : cluster [DBG] pgmap v6875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:34.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:34 smithi006 bash[33918]: cluster 2023-08-28T01:47:32.625376+0000 mgr.y (mgr.14901) 6868 : cluster [DBG] pgmap v6875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:47:34.569Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:47:34.571Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:34.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:34 smithi130 bash[27579]: cluster 2023-08-28T01:47:32.625376+0000 mgr.y (mgr.14901) 6868 : cluster [DBG] pgmap v6875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:35.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:34 smithi130 bash[51302]: debug 2023-08-28T01:47:34.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:34.945643+0000) 2023-08-28T01:47:35.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:35 smithi006 bash[27292]: audit 2023-08-28T01:47:34.743431+0000 mon.a (mon.0) 4496 : audit [DBG] from='client.? 172.21.15.6:0/685039288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:35.735 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:35 smithi006 bash[33918]: audit 2023-08-28T01:47:34.743431+0000 mon.a (mon.0) 4496 : audit [DBG] from='client.? 172.21.15.6:0/685039288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:35.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:35 smithi130 bash[27579]: audit 2023-08-28T01:47:34.743431+0000 mon.a (mon.0) 4496 : audit [DBG] from='client.? 172.21.15.6:0/685039288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:36.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:35 smithi130 bash[51302]: debug 2023-08-28T01:47:35.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:35.945918+0000) 2023-08-28T01:47:36.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:36 smithi006 bash[27292]: cluster 2023-08-28T01:47:34.626494+0000 mgr.y (mgr.14901) 6869 : cluster [DBG] pgmap v6876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:36.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:36 smithi006 bash[33918]: cluster 2023-08-28T01:47:34.626494+0000 mgr.y (mgr.14901) 6869 : cluster [DBG] pgmap v6876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:36.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:36 smithi130 bash[27579]: cluster 2023-08-28T01:47:34.626494+0000 mgr.y (mgr.14901) 6869 : cluster [DBG] pgmap v6876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:37.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:36 smithi130 bash[51302]: debug 2023-08-28T01:47:36.939+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:36.946226+0000) 2023-08-28T01:47:37.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:37 smithi006 bash[33918]: cluster 2023-08-28T01:47:36.627179+0000 mgr.y (mgr.14901) 6870 : cluster [DBG] pgmap v6877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:37.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:37 smithi006 bash[33918]: audit 2023-08-28T01:47:37.170151+0000 mon.a (mon.0) 4497 : audit [DBG] from='client.? 172.21.15.6:0/609129795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:37.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:37 smithi006 bash[27292]: cluster 2023-08-28T01:47:36.627179+0000 mgr.y (mgr.14901) 6870 : cluster [DBG] pgmap v6877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:37.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:37 smithi006 bash[27292]: audit 2023-08-28T01:47:37.170151+0000 mon.a (mon.0) 4497 : audit [DBG] from='client.? 172.21.15.6:0/609129795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:37.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:37 smithi130 bash[27579]: cluster 2023-08-28T01:47:36.627179+0000 mgr.y (mgr.14901) 6870 : cluster [DBG] pgmap v6877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:37.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:37 smithi130 bash[27579]: audit 2023-08-28T01:47:37.170151+0000 mon.a (mon.0) 4497 : audit [DBG] from='client.? 172.21.15.6:0/609129795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:38.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:37 smithi130 bash[51302]: debug 2023-08-28T01:47:37.943+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:37.946499+0000) 2023-08-28T01:47:39.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:38 smithi130 bash[51302]: debug 2023-08-28T01:47:38.943+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:38.946767+0000) 2023-08-28T01:47:39.945 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:39 smithi130 bash[27579]: cluster 2023-08-28T01:47:38.627854+0000 mgr.y (mgr.14901) 6871 : cluster [DBG] pgmap v6878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:39.946 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:39 smithi130 bash[27579]: audit 2023-08-28T01:47:39.585378+0000 mon.a (mon.0) 4498 : audit [DBG] from='client.? 172.21.15.6:0/2246454498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:39.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:39 smithi006 bash[33918]: cluster 2023-08-28T01:47:38.627854+0000 mgr.y (mgr.14901) 6871 : cluster [DBG] pgmap v6878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:39.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:39 smithi006 bash[33918]: audit 2023-08-28T01:47:39.585378+0000 mon.a (mon.0) 4498 : audit [DBG] from='client.? 172.21.15.6:0/2246454498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:39.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:39 smithi006 bash[27292]: cluster 2023-08-28T01:47:38.627854+0000 mgr.y (mgr.14901) 6871 : cluster [DBG] pgmap v6878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:39.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:39 smithi006 bash[27292]: audit 2023-08-28T01:47:39.585378+0000 mon.a (mon.0) 4498 : audit [DBG] from='client.? 172.21.15.6:0/2246454498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:40.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:39 smithi130 bash[51302]: debug 2023-08-28T01:47:39.943+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:39.947068+0000) 2023-08-28T01:47:41.119 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:47:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:47:40] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T01:47:41.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:40 smithi130 bash[51302]: debug 2023-08-28T01:47:40.943+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:40.947305+0000) 2023-08-28T01:47:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:47:41.119Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:47:41.120Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:41.945 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:41 smithi130 bash[27579]: cluster 2023-08-28T01:47:40.629049+0000 mgr.y (mgr.14901) 6872 : cluster [DBG] pgmap v6879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:41.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:41 smithi006 bash[33918]: cluster 2023-08-28T01:47:40.629049+0000 mgr.y (mgr.14901) 6872 : cluster [DBG] pgmap v6879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:41.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:41 smithi006 bash[27292]: cluster 2023-08-28T01:47:40.629049+0000 mgr.y (mgr.14901) 6872 : cluster [DBG] pgmap v6879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:42.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:41 smithi130 bash[51302]: debug 2023-08-28T01:47:41.943+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:41.947588+0000) 2023-08-28T01:47:42.946 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:42 smithi130 bash[27579]: audit 2023-08-28T01:47:42.001101+0000 mon.a (mon.0) 4499 : audit [DBG] from='client.? 172.21.15.6:0/4249801991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:42.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:42 smithi006 bash[33918]: audit 2023-08-28T01:47:42.001101+0000 mon.a (mon.0) 4499 : audit [DBG] from='client.? 172.21.15.6:0/4249801991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:42.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:42 smithi006 bash[27292]: audit 2023-08-28T01:47:42.001101+0000 mon.a (mon.0) 4499 : audit [DBG] from='client.? 172.21.15.6:0/4249801991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:47:42.687Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:47:42.689Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:43.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:42 smithi130 bash[51302]: debug 2023-08-28T01:47:42.943+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:42.947882+0000) 2023-08-28T01:47:44.285 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:43 smithi130 bash[51302]: debug 2023-08-28T01:47:43.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:43.948159+0000) 2023-08-28T01:47:44.570 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:44 smithi006 bash[27292]: cluster 2023-08-28T01:47:42.629647+0000 mgr.y (mgr.14901) 6873 : cluster [DBG] pgmap v6880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:44.571 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:44 smithi006 bash[33918]: cluster 2023-08-28T01:47:42.629647+0000 mgr.y (mgr.14901) 6873 : cluster [DBG] pgmap v6880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:44.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:44 smithi130 bash[27579]: cluster 2023-08-28T01:47:42.629647+0000 mgr.y (mgr.14901) 6873 : cluster [DBG] pgmap v6880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:47:44.570Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:47:44.573Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:45.296 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:44 smithi130 bash[51302]: debug 2023-08-28T01:47:44.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:44.948443+0000) 2023-08-28T01:47:45.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:45 smithi130 bash[27579]: audit 2023-08-28T01:47:44.416580+0000 mon.c (mon.2) 2702 : audit [DBG] from='client.? 172.21.15.6:0/3802088814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:45.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:45 smithi006 bash[27292]: audit 2023-08-28T01:47:44.416580+0000 mon.c (mon.2) 2702 : audit [DBG] from='client.? 172.21.15.6:0/3802088814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:45.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:45 smithi006 bash[33918]: audit 2023-08-28T01:47:44.416580+0000 mon.c (mon.2) 2702 : audit [DBG] from='client.? 172.21.15.6:0/3802088814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:46.298 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:45 smithi130 bash[51302]: debug 2023-08-28T01:47:45.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:45.948718+0000) 2023-08-28T01:47:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:46 smithi130 bash[27579]: cluster 2023-08-28T01:47:44.630827+0000 mgr.y (mgr.14901) 6874 : cluster [DBG] pgmap v6881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:46.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:46 smithi006 bash[27292]: cluster 2023-08-28T01:47:44.630827+0000 mgr.y (mgr.14901) 6874 : cluster [DBG] pgmap v6881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:46.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:46 smithi006 bash[33918]: cluster 2023-08-28T01:47:44.630827+0000 mgr.y (mgr.14901) 6874 : cluster [DBG] pgmap v6881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:47.305 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:46 smithi130 bash[51302]: debug 2023-08-28T01:47:46.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:46.948987+0000) 2023-08-28T01:47:47.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:47 smithi130 bash[27579]: audit 2023-08-28T01:47:46.830355+0000 mon.a (mon.0) 4500 : audit [DBG] from='client.? 172.21.15.6:0/1246225689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:47.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:47 smithi006 bash[33918]: audit 2023-08-28T01:47:46.830355+0000 mon.a (mon.0) 4500 : audit [DBG] from='client.? 172.21.15.6:0/1246225689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:47.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:47 smithi006 bash[27292]: audit 2023-08-28T01:47:46.830355+0000 mon.a (mon.0) 4500 : audit [DBG] from='client.? 172.21.15.6:0/1246225689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:48.308 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:47 smithi130 bash[51302]: debug 2023-08-28T01:47:47.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:47.949303+0000) 2023-08-28T01:47:48.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:48 smithi130 bash[27579]: cluster 2023-08-28T01:47:46.631576+0000 mgr.y (mgr.14901) 6875 : cluster [DBG] pgmap v6882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:48.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:48 smithi006 bash[27292]: cluster 2023-08-28T01:47:46.631576+0000 mgr.y (mgr.14901) 6875 : cluster [DBG] pgmap v6882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:48.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:48 smithi006 bash[33918]: cluster 2023-08-28T01:47:46.631576+0000 mgr.y (mgr.14901) 6875 : cluster [DBG] pgmap v6882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:49.315 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:48 smithi130 bash[51302]: debug 2023-08-28T01:47:48.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:48.949609+0000) 2023-08-28T01:47:49.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:49 smithi130 bash[27579]: audit 2023-08-28T01:47:49.239496+0000 mon.a (mon.0) 4501 : audit [DBG] from='client.? 172.21.15.6:0/3300459337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:49.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:49 smithi006 bash[27292]: audit 2023-08-28T01:47:49.239496+0000 mon.a (mon.0) 4501 : audit [DBG] from='client.? 172.21.15.6:0/3300459337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:49.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:49 smithi006 bash[33918]: audit 2023-08-28T01:47:49.239496+0000 mon.a (mon.0) 4501 : audit [DBG] from='client.? 172.21.15.6:0/3300459337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:50.318 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:49 smithi130 bash[51302]: debug 2023-08-28T01:47:49.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:49.949931+0000) 2023-08-28T01:47:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:50 smithi130 bash[27579]: cluster 2023-08-28T01:47:48.632273+0000 mgr.y (mgr.14901) 6876 : cluster [DBG] pgmap v6883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:50.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:50 smithi006 bash[33918]: cluster 2023-08-28T01:47:48.632273+0000 mgr.y (mgr.14901) 6876 : cluster [DBG] pgmap v6883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:50 smithi006 bash[27292]: cluster 2023-08-28T01:47:48.632273+0000 mgr.y (mgr.14901) 6876 : cluster [DBG] pgmap v6883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:51.120 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:47:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:47:50] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:47:51.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:50 smithi130 bash[51302]: debug 2023-08-28T01:47:50.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:50.950224+0000) 2023-08-28T01:47:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:47:51.120Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:47:51.122Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:52.324 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:51 smithi130 bash[51302]: debug 2023-08-28T01:47:51.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:51.950531+0000) 2023-08-28T01:47:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:52 smithi130 bash[27579]: cluster 2023-08-28T01:47:50.633477+0000 mgr.y (mgr.14901) 6877 : cluster [DBG] pgmap v6884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:52 smithi130 bash[27579]: audit 2023-08-28T01:47:51.652946+0000 mon.c (mon.2) 2703 : audit [DBG] from='client.? 172.21.15.6:0/986183928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:52.688 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:52 smithi006 bash[33918]: cluster 2023-08-28T01:47:50.633477+0000 mgr.y (mgr.14901) 6877 : cluster [DBG] pgmap v6884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:52.688 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:52 smithi006 bash[33918]: audit 2023-08-28T01:47:51.652946+0000 mon.c (mon.2) 2703 : audit [DBG] from='client.? 172.21.15.6:0/986183928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:52.689 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:52 smithi006 bash[27292]: cluster 2023-08-28T01:47:50.633477+0000 mgr.y (mgr.14901) 6877 : cluster [DBG] pgmap v6884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:52.689 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:52 smithi006 bash[27292]: audit 2023-08-28T01:47:51.652946+0000 mon.c (mon.2) 2703 : audit [DBG] from='client.? 172.21.15.6:0/986183928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:47:52.688Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:47:52.690Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:53.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:52 smithi130 bash[51302]: debug 2023-08-28T01:47:52.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:52.950812+0000) 2023-08-28T01:47:54.329 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:53 smithi130 bash[51302]: debug 2023-08-28T01:47:53.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:53.951079+0000) 2023-08-28T01:47:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:54 smithi130 bash[27579]: cluster 2023-08-28T01:47:52.634179+0000 mgr.y (mgr.14901) 6878 : cluster [DBG] pgmap v6885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:54 smithi130 bash[27579]: audit 2023-08-28T01:47:54.063453+0000 mon.c (mon.2) 2704 : audit [DBG] from='client.? 172.21.15.6:0/831467697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:54.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:54 smithi006 bash[33918]: cluster 2023-08-28T01:47:52.634179+0000 mgr.y (mgr.14901) 6878 : cluster [DBG] pgmap v6885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:54.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:54 smithi006 bash[33918]: audit 2023-08-28T01:47:54.063453+0000 mon.c (mon.2) 2704 : audit [DBG] from='client.? 172.21.15.6:0/831467697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:47:54.571Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:47:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:47:54.573Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:47:54.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:54 smithi006 bash[27292]: cluster 2023-08-28T01:47:52.634179+0000 mgr.y (mgr.14901) 6878 : cluster [DBG] pgmap v6885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:54.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:54 smithi006 bash[27292]: audit 2023-08-28T01:47:54.063453+0000 mon.c (mon.2) 2704 : audit [DBG] from='client.? 172.21.15.6:0/831467697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:55.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:54 smithi130 bash[51302]: debug 2023-08-28T01:47:54.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:54.951409+0000) 2023-08-28T01:47:56.333 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:55 smithi130 bash[51302]: debug 2023-08-28T01:47:55.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:55.951668+0000) 2023-08-28T01:47:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:56 smithi130 bash[27579]: cluster 2023-08-28T01:47:54.635334+0000 mgr.y (mgr.14901) 6879 : cluster [DBG] pgmap v6886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:56.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:56 smithi006 bash[27292]: cluster 2023-08-28T01:47:54.635334+0000 mgr.y (mgr.14901) 6879 : cluster [DBG] pgmap v6886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:56.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:56 smithi006 bash[33918]: cluster 2023-08-28T01:47:54.635334+0000 mgr.y (mgr.14901) 6879 : cluster [DBG] pgmap v6886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:57.340 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:56 smithi130 bash[51302]: debug 2023-08-28T01:47:56.951+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:56.952009+0000) 2023-08-28T01:47:57.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:57 smithi130 bash[27579]: audit 2023-08-28T01:47:56.490848+0000 mon.c (mon.2) 2705 : audit [DBG] from='client.? 172.21.15.6:0/3083686156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:57.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:57 smithi006 bash[27292]: audit 2023-08-28T01:47:56.490848+0000 mon.c (mon.2) 2705 : audit [DBG] from='client.? 172.21.15.6:0/3083686156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:57.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:57 smithi006 bash[33918]: audit 2023-08-28T01:47:56.490848+0000 mon.c (mon.2) 2705 : audit [DBG] from='client.? 172.21.15.6:0/3083686156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:58.342 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:57 smithi130 bash[51302]: debug 2023-08-28T01:47:57.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:57.952260+0000) 2023-08-28T01:47:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:58 smithi130 bash[27579]: cluster 2023-08-28T01:47:56.636040+0000 mgr.y (mgr.14901) 6880 : cluster [DBG] pgmap v6887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:58.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:58 smithi006 bash[33918]: cluster 2023-08-28T01:47:56.636040+0000 mgr.y (mgr.14901) 6880 : cluster [DBG] pgmap v6887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:58.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:58 smithi006 bash[27292]: cluster 2023-08-28T01:47:56.636040+0000 mgr.y (mgr.14901) 6880 : cluster [DBG] pgmap v6887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:47:59.351 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:58 smithi130 bash[51302]: debug 2023-08-28T01:47:58.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:58.952540+0000) 2023-08-28T01:47:59.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:47:59 smithi130 bash[27579]: audit 2023-08-28T01:47:58.914961+0000 mon.a (mon.0) 4502 : audit [DBG] from='client.? 172.21.15.6:0/3391495720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:59.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:47:59 smithi006 bash[33918]: audit 2023-08-28T01:47:58.914961+0000 mon.a (mon.0) 4502 : audit [DBG] from='client.? 172.21.15.6:0/3391495720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:59.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:47:59 smithi006 bash[27292]: audit 2023-08-28T01:47:58.914961+0000 mon.a (mon.0) 4502 : audit [DBG] from='client.? 172.21.15.6:0/3391495720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:00.353 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:47:59 smithi130 bash[51302]: debug 2023-08-28T01:47:59.951+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:59.952827+0000) 2023-08-28T01:48:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:00 smithi130 bash[27579]: cluster 2023-08-28T01:47:58.636889+0000 mgr.y (mgr.14901) 6881 : cluster [DBG] pgmap v6888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:00 smithi006 bash[33918]: cluster 2023-08-28T01:47:58.636889+0000 mgr.y (mgr.14901) 6881 : cluster [DBG] pgmap v6888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:00 smithi006 bash[27292]: cluster 2023-08-28T01:47:58.636889+0000 mgr.y (mgr.14901) 6881 : cluster [DBG] pgmap v6888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:01.121 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:48:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:48:00] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T01:48:01.361 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:00 smithi130 bash[51302]: debug 2023-08-28T01:48:00.951+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:00.953244+0000) 2023-08-28T01:48:01.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:01 smithi006 bash[27292]: audit 2023-08-28T01:48:01.323133+0000 mon.a (mon.0) 4503 : audit [DBG] from='client.? 172.21.15.6:0/116115876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:01.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:01 smithi006 bash[33918]: audit 2023-08-28T01:48:01.323133+0000 mon.a (mon.0) 4503 : audit [DBG] from='client.? 172.21.15.6:0/116115876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:48:01.120Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:48:01.123Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:01.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:01 smithi130 bash[27579]: audit 2023-08-28T01:48:01.323133+0000 mon.a (mon.0) 4503 : audit [DBG] from='client.? 172.21.15.6:0/116115876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:02.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:01 smithi130 bash[51302]: debug 2023-08-28T01:48:01.951+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:01.953530+0000) 2023-08-28T01:48:02.689 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:02 smithi006 bash[27292]: cluster 2023-08-28T01:48:00.638012+0000 mgr.y (mgr.14901) 6882 : cluster [DBG] pgmap v6889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:02.689 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:02 smithi006 bash[33918]: cluster 2023-08-28T01:48:00.638012+0000 mgr.y (mgr.14901) 6882 : cluster [DBG] pgmap v6889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:02 smithi130 bash[27579]: cluster 2023-08-28T01:48:00.638012+0000 mgr.y (mgr.14901) 6882 : cluster [DBG] pgmap v6889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:48:02.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:48:02.690Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:03.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:02 smithi130 bash[51302]: debug 2023-08-28T01:48:02.947+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:02.953871+0000) 2023-08-28T01:48:03.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:03 smithi006 bash[33918]: cluster 2023-08-28T01:48:02.638661+0000 mgr.y (mgr.14901) 6883 : cluster [DBG] pgmap v6890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:03.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:03 smithi006 bash[27292]: cluster 2023-08-28T01:48:02.638661+0000 mgr.y (mgr.14901) 6883 : cluster [DBG] pgmap v6890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:03.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:03 smithi130 bash[27579]: cluster 2023-08-28T01:48:02.638661+0000 mgr.y (mgr.14901) 6883 : cluster [DBG] pgmap v6890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:04.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:03 smithi130 bash[51302]: debug 2023-08-28T01:48:03.951+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:03.954198+0000) 2023-08-28T01:48:04.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:04 smithi006 bash[27292]: audit 2023-08-28T01:48:03.740131+0000 mon.c (mon.2) 2706 : audit [DBG] from='client.? 172.21.15.6:0/3611946099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:04.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:04 smithi006 bash[33918]: audit 2023-08-28T01:48:03.740131+0000 mon.c (mon.2) 2706 : audit [DBG] from='client.? 172.21.15.6:0/3611946099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:48:04.572Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:48:04.574Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:04.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:04 smithi130 bash[27579]: audit 2023-08-28T01:48:03.740131+0000 mon.c (mon.2) 2706 : audit [DBG] from='client.? 172.21.15.6:0/3611946099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:05.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:04 smithi130 bash[51302]: debug 2023-08-28T01:48:04.951+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:04.954558+0000) 2023-08-28T01:48:05.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:05 smithi130 bash[27579]: cluster 2023-08-28T01:48:04.639850+0000 mgr.y (mgr.14901) 6884 : cluster [DBG] pgmap v6891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:05.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:05 smithi006 bash[33918]: cluster 2023-08-28T01:48:04.639850+0000 mgr.y (mgr.14901) 6884 : cluster [DBG] pgmap v6891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:05.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:05 smithi006 bash[27292]: cluster 2023-08-28T01:48:04.639850+0000 mgr.y (mgr.14901) 6884 : cluster [DBG] pgmap v6891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:06.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:05 smithi130 bash[51302]: debug 2023-08-28T01:48:05.951+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:05.954836+0000) 2023-08-28T01:48:06.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:06 smithi130 bash[27579]: audit 2023-08-28T01:48:06.155376+0000 mon.c (mon.2) 2707 : audit [DBG] from='client.? 172.21.15.6:0/3692700887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:06.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:06 smithi006 bash[33918]: audit 2023-08-28T01:48:06.155376+0000 mon.c (mon.2) 2707 : audit [DBG] from='client.? 172.21.15.6:0/3692700887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:06.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:06 smithi006 bash[27292]: audit 2023-08-28T01:48:06.155376+0000 mon.c (mon.2) 2707 : audit [DBG] from='client.? 172.21.15.6:0/3692700887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:07.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:06 smithi130 bash[51302]: debug 2023-08-28T01:48:06.951+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:06.955075+0000) 2023-08-28T01:48:08.294 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:07 smithi130 bash[51302]: debug 2023-08-28T01:48:07.955+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:07.955382+0000) 2023-08-28T01:48:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:08 smithi130 bash[27579]: cluster 2023-08-28T01:48:06.640697+0000 mgr.y (mgr.14901) 6885 : cluster [DBG] pgmap v6892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:08.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:08 smithi006 bash[27292]: cluster 2023-08-28T01:48:06.640697+0000 mgr.y (mgr.14901) 6885 : cluster [DBG] pgmap v6892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:08 smithi006 bash[33918]: cluster 2023-08-28T01:48:06.640697+0000 mgr.y (mgr.14901) 6885 : cluster [DBG] pgmap v6892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:09.304 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:08 smithi130 bash[51302]: debug 2023-08-28T01:48:08.955+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:08.955723+0000) 2023-08-28T01:48:09.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:09 smithi130 bash[27579]: audit 2023-08-28T01:48:08.570659+0000 mon.c (mon.2) 2708 : audit [DBG] from='client.? 172.21.15.6:0/3139899508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:09.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:09 smithi006 bash[33918]: audit 2023-08-28T01:48:08.570659+0000 mon.c (mon.2) 2708 : audit [DBG] from='client.? 172.21.15.6:0/3139899508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:09.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:09 smithi006 bash[27292]: audit 2023-08-28T01:48:08.570659+0000 mon.c (mon.2) 2708 : audit [DBG] from='client.? 172.21.15.6:0/3139899508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:10.307 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:09 smithi130 bash[51302]: debug 2023-08-28T01:48:09.955+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:09.956062+0000) 2023-08-28T01:48:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:10 smithi130 bash[27579]: cluster 2023-08-28T01:48:08.641336+0000 mgr.y (mgr.14901) 6886 : cluster [DBG] pgmap v6893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:10.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:10 smithi006 bash[27292]: cluster 2023-08-28T01:48:08.641336+0000 mgr.y (mgr.14901) 6886 : cluster [DBG] pgmap v6893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:10 smithi006 bash[33918]: cluster 2023-08-28T01:48:08.641336+0000 mgr.y (mgr.14901) 6886 : cluster [DBG] pgmap v6893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:11.122 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:48:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:48:10] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T01:48:11.316 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:10 smithi130 bash[51302]: debug 2023-08-28T01:48:10.951+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:10.956394+0000) 2023-08-28T01:48:11.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:11 smithi006 bash[27292]: audit 2023-08-28T01:48:10.987103+0000 mon.c (mon.2) 2709 : audit [DBG] from='client.? 172.21.15.6:0/3285222053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:11.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:11 smithi006 bash[33918]: audit 2023-08-28T01:48:10.987103+0000 mon.c (mon.2) 2709 : audit [DBG] from='client.? 172.21.15.6:0/3285222053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:48:11.121Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:48:11.124Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:11.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:11 smithi130 bash[27579]: audit 2023-08-28T01:48:10.987103+0000 mon.c (mon.2) 2709 : audit [DBG] from='client.? 172.21.15.6:0/3285222053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:12.317 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:11 smithi130 bash[51302]: debug 2023-08-28T01:48:11.951+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:11.956661+0000) 2023-08-28T01:48:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:12 smithi130 bash[27579]: cluster 2023-08-28T01:48:10.642411+0000 mgr.y (mgr.14901) 6887 : cluster [DBG] pgmap v6894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:12.690 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:12 smithi006 bash[33918]: cluster 2023-08-28T01:48:10.642411+0000 mgr.y (mgr.14901) 6887 : cluster [DBG] pgmap v6894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:12.690 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:12 smithi006 bash[27292]: cluster 2023-08-28T01:48:10.642411+0000 mgr.y (mgr.14901) 6887 : cluster [DBG] pgmap v6894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:48:12.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:48:12.692Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:13.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:12 smithi130 bash[51302]: debug 2023-08-28T01:48:12.951+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:12.956937+0000) 2023-08-28T01:48:14.326 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:13 smithi130 bash[51302]: debug 2023-08-28T01:48:13.955+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:13.957227+0000) 2023-08-28T01:48:14.575 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:48:14.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:14.575 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:14 smithi006 bash[27292]: cluster 2023-08-28T01:48:12.643066+0000 mgr.y (mgr.14901) 6888 : cluster [DBG] pgmap v6895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:14.575 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:14 smithi006 bash[27292]: audit 2023-08-28T01:48:13.400518+0000 mon.c (mon.2) 2710 : audit [DBG] from='client.? 172.21.15.6:0/2194178197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:14.575 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:14 smithi006 bash[33918]: cluster 2023-08-28T01:48:12.643066+0000 mgr.y (mgr.14901) 6888 : cluster [DBG] pgmap v6895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:14.575 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:14 smithi006 bash[33918]: audit 2023-08-28T01:48:13.400518+0000 mon.c (mon.2) 2710 : audit [DBG] from='client.? 172.21.15.6:0/2194178197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:14 smithi130 bash[27579]: cluster 2023-08-28T01:48:12.643066+0000 mgr.y (mgr.14901) 6888 : cluster [DBG] pgmap v6895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:14 smithi130 bash[27579]: audit 2023-08-28T01:48:13.400518+0000 mon.c (mon.2) 2710 : audit [DBG] from='client.? 172.21.15.6:0/2194178197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:48:14.575Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:15.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:14 smithi130 bash[51302]: debug 2023-08-28T01:48:14.955+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:14.957537+0000) 2023-08-28T01:48:16.330 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:15 smithi130 bash[51302]: debug 2023-08-28T01:48:15.955+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:15.957907+0000) 2023-08-28T01:48:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:16 smithi130 bash[27579]: cluster 2023-08-28T01:48:14.644148+0000 mgr.y (mgr.14901) 6889 : cluster [DBG] pgmap v6896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:16 smithi130 bash[27579]: audit 2023-08-28T01:48:15.820633+0000 mon.a (mon.0) 4504 : audit [DBG] from='client.? 172.21.15.6:0/3955243810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:16.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:16 smithi006 bash[27292]: cluster 2023-08-28T01:48:14.644148+0000 mgr.y (mgr.14901) 6889 : cluster [DBG] pgmap v6896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:16.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:16 smithi006 bash[27292]: audit 2023-08-28T01:48:15.820633+0000 mon.a (mon.0) 4504 : audit [DBG] from='client.? 172.21.15.6:0/3955243810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:16 smithi006 bash[33918]: cluster 2023-08-28T01:48:14.644148+0000 mgr.y (mgr.14901) 6889 : cluster [DBG] pgmap v6896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:16 smithi006 bash[33918]: audit 2023-08-28T01:48:15.820633+0000 mon.a (mon.0) 4504 : audit [DBG] from='client.? 172.21.15.6:0/3955243810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:17.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:16 smithi130 bash[51302]: debug 2023-08-28T01:48:16.955+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:16.958232+0000) 2023-08-28T01:48:18.335 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:17 smithi130 bash[51302]: debug 2023-08-28T01:48:17.955+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:17.958546+0000) 2023-08-28T01:48:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:18 smithi130 bash[27579]: cluster 2023-08-28T01:48:16.644812+0000 mgr.y (mgr.14901) 6890 : cluster [DBG] pgmap v6897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:18 smithi130 bash[27579]: audit 2023-08-28T01:48:18.240696+0000 mon.a (mon.0) 4505 : audit [DBG] from='client.? 172.21.15.6:0/2410147372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:18.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:18 smithi006 bash[33918]: cluster 2023-08-28T01:48:16.644812+0000 mgr.y (mgr.14901) 6890 : cluster [DBG] pgmap v6897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:18.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:18 smithi006 bash[33918]: audit 2023-08-28T01:48:18.240696+0000 mon.a (mon.0) 4505 : audit [DBG] from='client.? 172.21.15.6:0/2410147372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:18 smithi006 bash[27292]: cluster 2023-08-28T01:48:16.644812+0000 mgr.y (mgr.14901) 6890 : cluster [DBG] pgmap v6897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:18.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:18 smithi006 bash[27292]: audit 2023-08-28T01:48:18.240696+0000 mon.a (mon.0) 4505 : audit [DBG] from='client.? 172.21.15.6:0/2410147372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:19.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:18 smithi130 bash[51302]: debug 2023-08-28T01:48:18.951+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:18.958832+0000) 2023-08-28T01:48:20.339 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:19 smithi130 bash[51302]: debug 2023-08-28T01:48:19.954+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:19.959058+0000) 2023-08-28T01:48:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:20 smithi130 bash[27579]: cluster 2023-08-28T01:48:18.645454+0000 mgr.y (mgr.14901) 6891 : cluster [DBG] pgmap v6898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:20.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:20 smithi006 bash[27292]: cluster 2023-08-28T01:48:18.645454+0000 mgr.y (mgr.14901) 6891 : cluster [DBG] pgmap v6898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:20.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:20 smithi006 bash[33918]: cluster 2023-08-28T01:48:18.645454+0000 mgr.y (mgr.14901) 6891 : cluster [DBG] pgmap v6898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:21.123 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:48:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:48:20] "GET /metrics HTTP/1.1" 200 193128 "" "Prometheus/2.33.4" 2023-08-28T01:48:21.347 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:20 smithi130 bash[51302]: debug 2023-08-28T01:48:20.954+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:20.959350+0000) 2023-08-28T01:48:21.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:21 smithi006 bash[27292]: audit 2023-08-28T01:48:20.664666+0000 mon.a (mon.0) 4506 : audit [DBG] from='client.? 172.21.15.6:0/1050269039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:21.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:21 smithi006 bash[33918]: audit 2023-08-28T01:48:20.664666+0000 mon.a (mon.0) 4506 : audit [DBG] from='client.? 172.21.15.6:0/1050269039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:48:21.123Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:48:21.124Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:21.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:21 smithi130 bash[27579]: audit 2023-08-28T01:48:20.664666+0000 mon.a (mon.0) 4506 : audit [DBG] from='client.? 172.21.15.6:0/1050269039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:22.356 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:21 smithi130 bash[51302]: debug 2023-08-28T01:48:21.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:21.959650+0000) 2023-08-28T01:48:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:22 smithi130 bash[27579]: cluster 2023-08-28T01:48:20.646523+0000 mgr.y (mgr.14901) 6892 : cluster [DBG] pgmap v6899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:22.690 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:22 smithi006 bash[27292]: cluster 2023-08-28T01:48:20.646523+0000 mgr.y (mgr.14901) 6892 : cluster [DBG] pgmap v6899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:22.690 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:22 smithi006 bash[33918]: cluster 2023-08-28T01:48:20.646523+0000 mgr.y (mgr.14901) 6892 : cluster [DBG] pgmap v6899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:48:22.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:48:22.692Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:23.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:22 smithi130 bash[51302]: debug 2023-08-28T01:48:22.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:22.959947+0000) 2023-08-28T01:48:23.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:23 smithi006 bash[27292]: audit 2023-08-28T01:48:23.085556+0000 mon.a (mon.0) 4507 : audit [DBG] from='client.? 172.21.15.6:0/3131938430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:23.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:23 smithi006 bash[33918]: audit 2023-08-28T01:48:23.085556+0000 mon.a (mon.0) 4507 : audit [DBG] from='client.? 172.21.15.6:0/3131938430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:23.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:23 smithi130 bash[27579]: audit 2023-08-28T01:48:23.085556+0000 mon.a (mon.0) 4507 : audit [DBG] from='client.? 172.21.15.6:0/3131938430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:24.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:23 smithi130 bash[51302]: debug 2023-08-28T01:48:23.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:23.960237+0000) 2023-08-28T01:48:24.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:24 smithi006 bash[33918]: cluster 2023-08-28T01:48:22.647207+0000 mgr.y (mgr.14901) 6893 : cluster [DBG] pgmap v6900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:24.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:24 smithi006 bash[27292]: cluster 2023-08-28T01:48:22.647207+0000 mgr.y (mgr.14901) 6893 : cluster [DBG] pgmap v6900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:48:24.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:48:24.576Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:24.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:24 smithi130 bash[27579]: cluster 2023-08-28T01:48:22.647207+0000 mgr.y (mgr.14901) 6893 : cluster [DBG] pgmap v6900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:25.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:24 smithi130 bash[51302]: debug 2023-08-28T01:48:24.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:24.960541+0000) 2023-08-28T01:48:26.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:25 smithi130 bash[51302]: debug 2023-08-28T01:48:25.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:25.960818+0000) 2023-08-28T01:48:26.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:26 smithi006 bash[27292]: cluster 2023-08-28T01:48:24.648362+0000 mgr.y (mgr.14901) 6894 : cluster [DBG] pgmap v6901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:26.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:26 smithi006 bash[27292]: audit 2023-08-28T01:48:25.500149+0000 mon.a (mon.0) 4508 : audit [DBG] from='client.? 172.21.15.6:0/2565333499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:26.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:26 smithi006 bash[27292]: audit 2023-08-28T01:48:26.343039+0000 mon.b (mon.1) 775 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:48:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:26 smithi006 bash[33918]: cluster 2023-08-28T01:48:24.648362+0000 mgr.y (mgr.14901) 6894 : cluster [DBG] pgmap v6901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:26 smithi006 bash[33918]: audit 2023-08-28T01:48:25.500149+0000 mon.a (mon.0) 4508 : audit [DBG] from='client.? 172.21.15.6:0/2565333499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:26 smithi006 bash[33918]: audit 2023-08-28T01:48:26.343039+0000 mon.b (mon.1) 775 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:48:26.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:26 smithi130 bash[27579]: cluster 2023-08-28T01:48:24.648362+0000 mgr.y (mgr.14901) 6894 : cluster [DBG] pgmap v6901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:26.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:26 smithi130 bash[27579]: audit 2023-08-28T01:48:25.500149+0000 mon.a (mon.0) 4508 : audit [DBG] from='client.? 172.21.15.6:0/2565333499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:26.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:26 smithi130 bash[27579]: audit 2023-08-28T01:48:26.343039+0000 mon.b (mon.1) 775 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:48:27.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:26 smithi130 bash[51302]: debug 2023-08-28T01:48:26.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:26.961092+0000) 2023-08-28T01:48:28.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:27 smithi130 bash[51302]: debug 2023-08-28T01:48:27.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:27.961380+0000) 2023-08-28T01:48:28.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:28 smithi006 bash[27292]: cluster 2023-08-28T01:48:26.649052+0000 mgr.y (mgr.14901) 6895 : cluster [DBG] pgmap v6902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:28.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:28 smithi006 bash[27292]: audit 2023-08-28T01:48:27.918270+0000 mon.c (mon.2) 2711 : audit [DBG] from='client.? 172.21.15.6:0/678161116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:28 smithi006 bash[33918]: cluster 2023-08-28T01:48:26.649052+0000 mgr.y (mgr.14901) 6895 : cluster [DBG] pgmap v6902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:28 smithi006 bash[33918]: audit 2023-08-28T01:48:27.918270+0000 mon.c (mon.2) 2711 : audit [DBG] from='client.? 172.21.15.6:0/678161116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:28.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:28 smithi130 bash[27579]: cluster 2023-08-28T01:48:26.649052+0000 mgr.y (mgr.14901) 6895 : cluster [DBG] pgmap v6902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:28.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:28 smithi130 bash[27579]: audit 2023-08-28T01:48:27.918270+0000 mon.c (mon.2) 2711 : audit [DBG] from='client.? 172.21.15.6:0/678161116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:29.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:28 smithi130 bash[51302]: debug 2023-08-28T01:48:28.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:28.961677+0000) 2023-08-28T01:48:30.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:29 smithi130 bash[51302]: debug 2023-08-28T01:48:29.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:29.961955+0000) 2023-08-28T01:48:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:30 smithi006 bash[33918]: cluster 2023-08-28T01:48:28.649656+0000 mgr.y (mgr.14901) 6896 : cluster [DBG] pgmap v6903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:30 smithi006 bash[33918]: audit 2023-08-28T01:48:30.342211+0000 mon.c (mon.2) 2712 : audit [DBG] from='client.? 172.21.15.6:0/1962336410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:30 smithi006 bash[27292]: cluster 2023-08-28T01:48:28.649656+0000 mgr.y (mgr.14901) 6896 : cluster [DBG] pgmap v6903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:30 smithi006 bash[27292]: audit 2023-08-28T01:48:30.342211+0000 mon.c (mon.2) 2712 : audit [DBG] from='client.? 172.21.15.6:0/1962336410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:30 smithi130 bash[27579]: cluster 2023-08-28T01:48:28.649656+0000 mgr.y (mgr.14901) 6896 : cluster [DBG] pgmap v6903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:30 smithi130 bash[27579]: audit 2023-08-28T01:48:30.342211+0000 mon.c (mon.2) 2712 : audit [DBG] from='client.? 172.21.15.6:0/1962336410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:31.123 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:48:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:48:30] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:48:31.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:30 smithi130 bash[51302]: debug 2023-08-28T01:48:30.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:30.962235+0000) 2023-08-28T01:48:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:48:31.124Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:48:31.126Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:32.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:31 smithi130 bash[51302]: debug 2023-08-28T01:48:31.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:31.962471+0000) 2023-08-28T01:48:32.690 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:32 smithi006 bash[27292]: cluster 2023-08-28T01:48:30.650527+0000 mgr.y (mgr.14901) 6897 : cluster [DBG] pgmap v6904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:32.690 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:32 smithi006 bash[27292]: audit 2023-08-28T01:48:32.209789+0000 mon.a (mon.0) 4509 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:48:32.690 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:32 smithi006 bash[33918]: cluster 2023-08-28T01:48:30.650527+0000 mgr.y (mgr.14901) 6897 : cluster [DBG] pgmap v6904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:32.691 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:32 smithi006 bash[33918]: audit 2023-08-28T01:48:32.209789+0000 mon.a (mon.0) 4509 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:48:32.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:32 smithi130 bash[27579]: cluster 2023-08-28T01:48:30.650527+0000 mgr.y (mgr.14901) 6897 : cluster [DBG] pgmap v6904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:32 smithi130 bash[27579]: audit 2023-08-28T01:48:32.209789+0000 mon.a (mon.0) 4509 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:48:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:48:32.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:48:32.692Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:33.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:32 smithi130 bash[51302]: debug 2023-08-28T01:48:32.958+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:32.962743+0000) 2023-08-28T01:48:33.961 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:33 smithi130 bash[27579]: audit 2023-08-28T01:48:32.623680+0000 mon.a (mon.0) 4510 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:48:33.961 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:33 smithi130 bash[27579]: cluster 2023-08-28T01:48:32.650954+0000 mgr.y (mgr.14901) 6898 : cluster [DBG] pgmap v6905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:33.961 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:33 smithi130 bash[27579]: audit 2023-08-28T01:48:32.745118+0000 mon.a (mon.0) 4511 : audit [DBG] from='client.? 172.21.15.6:0/146336589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:33.961 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:33 smithi130 bash[27579]: audit 2023-08-28T01:48:32.898849+0000 mon.b (mon.1) 776 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:48:33.961 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:33 smithi130 bash[27579]: audit 2023-08-28T01:48:32.901066+0000 mon.b (mon.1) 777 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:48:33.962 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:33 smithi130 bash[27579]: audit 2023-08-28T01:48:32.908230+0000 mon.a (mon.0) 4512 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:48:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:33 smithi006 bash[27292]: audit 2023-08-28T01:48:32.623680+0000 mon.a (mon.0) 4510 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:48:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:33 smithi006 bash[27292]: cluster 2023-08-28T01:48:32.650954+0000 mgr.y (mgr.14901) 6898 : cluster [DBG] pgmap v6905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:33 smithi006 bash[27292]: audit 2023-08-28T01:48:32.745118+0000 mon.a (mon.0) 4511 : audit [DBG] from='client.? 172.21.15.6:0/146336589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:33 smithi006 bash[27292]: audit 2023-08-28T01:48:32.898849+0000 mon.b (mon.1) 776 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:48:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:33 smithi006 bash[27292]: audit 2023-08-28T01:48:32.901066+0000 mon.b (mon.1) 777 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:48:33.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:33 smithi006 bash[27292]: audit 2023-08-28T01:48:32.908230+0000 mon.a (mon.0) 4512 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:48:33.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:33 smithi006 bash[33918]: audit 2023-08-28T01:48:32.623680+0000 mon.a (mon.0) 4510 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:48:33.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:33 smithi006 bash[33918]: cluster 2023-08-28T01:48:32.650954+0000 mgr.y (mgr.14901) 6898 : cluster [DBG] pgmap v6905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:33.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:33 smithi006 bash[33918]: audit 2023-08-28T01:48:32.745118+0000 mon.a (mon.0) 4511 : audit [DBG] from='client.? 172.21.15.6:0/146336589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:33.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:33 smithi006 bash[33918]: audit 2023-08-28T01:48:32.898849+0000 mon.b (mon.1) 776 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:48:33.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:33 smithi006 bash[33918]: audit 2023-08-28T01:48:32.901066+0000 mon.b (mon.1) 777 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:48:33.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:33 smithi006 bash[33918]: audit 2023-08-28T01:48:32.908230+0000 mon.a (mon.0) 4512 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:48:34.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:33 smithi130 bash[51302]: debug 2023-08-28T01:48:33.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:33.963009+0000) 2023-08-28T01:48:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:48:34.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:48:34.577Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:35.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:34 smithi130 bash[51302]: debug 2023-08-28T01:48:34.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:34.963277+0000) 2023-08-28T01:48:35.961 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:35 smithi130 bash[27579]: cluster 2023-08-28T01:48:34.652046+0000 mgr.y (mgr.14901) 6899 : cluster [DBG] pgmap v6906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:35.962 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:35 smithi130 bash[27579]: audit 2023-08-28T01:48:35.157038+0000 mon.c (mon.2) 2713 : audit [DBG] from='client.? 172.21.15.6:0/1687365528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:35.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:35 smithi006 bash[27292]: cluster 2023-08-28T01:48:34.652046+0000 mgr.y (mgr.14901) 6899 : cluster [DBG] pgmap v6906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:35.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:35 smithi006 bash[27292]: audit 2023-08-28T01:48:35.157038+0000 mon.c (mon.2) 2713 : audit [DBG] from='client.? 172.21.15.6:0/1687365528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:35.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:35 smithi006 bash[33918]: cluster 2023-08-28T01:48:34.652046+0000 mgr.y (mgr.14901) 6899 : cluster [DBG] pgmap v6906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:35.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:35 smithi006 bash[33918]: audit 2023-08-28T01:48:35.157038+0000 mon.c (mon.2) 2713 : audit [DBG] from='client.? 172.21.15.6:0/1687365528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:36.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:35 smithi130 bash[51302]: debug 2023-08-28T01:48:35.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:35.963555+0000) 2023-08-28T01:48:37.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:36 smithi130 bash[51302]: debug 2023-08-28T01:48:36.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:36.963831+0000) 2023-08-28T01:48:38.302 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:37 smithi130 bash[51302]: debug 2023-08-28T01:48:37.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:37.964097+0000) 2023-08-28T01:48:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:38 smithi130 bash[27579]: cluster 2023-08-28T01:48:36.652899+0000 mgr.y (mgr.14901) 6900 : cluster [DBG] pgmap v6907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:38 smithi130 bash[27579]: audit 2023-08-28T01:48:37.575889+0000 mon.a (mon.0) 4513 : audit [DBG] from='client.? 172.21.15.6:0/4057639596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:38 smithi006 bash[33918]: cluster 2023-08-28T01:48:36.652899+0000 mgr.y (mgr.14901) 6900 : cluster [DBG] pgmap v6907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:38 smithi006 bash[33918]: audit 2023-08-28T01:48:37.575889+0000 mon.a (mon.0) 4513 : audit [DBG] from='client.? 172.21.15.6:0/4057639596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:38 smithi006 bash[27292]: cluster 2023-08-28T01:48:36.652899+0000 mgr.y (mgr.14901) 6900 : cluster [DBG] pgmap v6907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:38 smithi006 bash[27292]: audit 2023-08-28T01:48:37.575889+0000 mon.a (mon.0) 4513 : audit [DBG] from='client.? 172.21.15.6:0/4057639596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:39.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:38 smithi130 bash[51302]: debug 2023-08-28T01:48:38.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:38.964398+0000) 2023-08-28T01:48:40.308 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:39 smithi130 bash[51302]: debug 2023-08-28T01:48:39.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:39.964700+0000) 2023-08-28T01:48:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:40 smithi130 bash[27579]: cluster 2023-08-28T01:48:38.653454+0000 mgr.y (mgr.14901) 6901 : cluster [DBG] pgmap v6908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:40 smithi130 bash[27579]: audit 2023-08-28T01:48:39.985642+0000 mon.a (mon.0) 4514 : audit [DBG] from='client.? 172.21.15.6:0/4011139108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:40.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:40 smithi006 bash[33918]: cluster 2023-08-28T01:48:38.653454+0000 mgr.y (mgr.14901) 6901 : cluster [DBG] pgmap v6908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:40.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:40 smithi006 bash[33918]: audit 2023-08-28T01:48:39.985642+0000 mon.a (mon.0) 4514 : audit [DBG] from='client.? 172.21.15.6:0/4011139108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:40 smithi006 bash[27292]: cluster 2023-08-28T01:48:38.653454+0000 mgr.y (mgr.14901) 6901 : cluster [DBG] pgmap v6908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:40 smithi006 bash[27292]: audit 2023-08-28T01:48:39.985642+0000 mon.a (mon.0) 4514 : audit [DBG] from='client.? 172.21.15.6:0/4011139108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:41.124 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:48:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:48:40] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:48:41.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:40 smithi130 bash[51302]: debug 2023-08-28T01:48:40.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:40.964978+0000) 2023-08-28T01:48:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:48:41.125Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:48:41.127Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:42.313 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:41 smithi130 bash[51302]: debug 2023-08-28T01:48:41.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:41.965334+0000) 2023-08-28T01:48:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:42 smithi130 bash[27579]: cluster 2023-08-28T01:48:40.654588+0000 mgr.y (mgr.14901) 6902 : cluster [DBG] pgmap v6909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:42.691 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:42 smithi006 bash[27292]: cluster 2023-08-28T01:48:40.654588+0000 mgr.y (mgr.14901) 6902 : cluster [DBG] pgmap v6909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:42.691 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:42 smithi006 bash[33918]: cluster 2023-08-28T01:48:40.654588+0000 mgr.y (mgr.14901) 6902 : cluster [DBG] pgmap v6909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:48:42.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:48:42.693Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:43.320 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:42 smithi130 bash[51302]: debug 2023-08-28T01:48:42.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:42.965625+0000) 2023-08-28T01:48:43.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:43 smithi130 bash[27579]: audit 2023-08-28T01:48:42.401547+0000 mon.a (mon.0) 4515 : audit [DBG] from='client.? 172.21.15.6:0/2640799336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:43.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:43 smithi006 bash[33918]: audit 2023-08-28T01:48:42.401547+0000 mon.a (mon.0) 4515 : audit [DBG] from='client.? 172.21.15.6:0/2640799336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:43.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:43 smithi006 bash[27292]: audit 2023-08-28T01:48:42.401547+0000 mon.a (mon.0) 4515 : audit [DBG] from='client.? 172.21.15.6:0/2640799336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:44.324 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:43 smithi130 bash[51302]: debug 2023-08-28T01:48:43.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:43.965899+0000) 2023-08-28T01:48:44.575 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:44 smithi006 bash[33918]: cluster 2023-08-28T01:48:42.655330+0000 mgr.y (mgr.14901) 6903 : cluster [DBG] pgmap v6910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:44.575 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:44 smithi006 bash[27292]: cluster 2023-08-28T01:48:42.655330+0000 mgr.y (mgr.14901) 6903 : cluster [DBG] pgmap v6910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:44 smithi130 bash[27579]: cluster 2023-08-28T01:48:42.655330+0000 mgr.y (mgr.14901) 6903 : cluster [DBG] pgmap v6910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:48:44.575Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:48:44.579Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:45.335 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:44 smithi130 bash[51302]: debug 2023-08-28T01:48:44.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:44.966208+0000) 2023-08-28T01:48:45.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:45 smithi130 bash[27579]: audit 2023-08-28T01:48:44.810034+0000 mon.a (mon.0) 4516 : audit [DBG] from='client.? 172.21.15.6:0/228646138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:45.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:45 smithi006 bash[33918]: audit 2023-08-28T01:48:44.810034+0000 mon.a (mon.0) 4516 : audit [DBG] from='client.? 172.21.15.6:0/228646138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:45.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:45 smithi006 bash[27292]: audit 2023-08-28T01:48:44.810034+0000 mon.a (mon.0) 4516 : audit [DBG] from='client.? 172.21.15.6:0/228646138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:46.338 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:45 smithi130 bash[51302]: debug 2023-08-28T01:48:45.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:45.966486+0000) 2023-08-28T01:48:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:46 smithi130 bash[27579]: cluster 2023-08-28T01:48:44.656410+0000 mgr.y (mgr.14901) 6904 : cluster [DBG] pgmap v6911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:46.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:46 smithi006 bash[27292]: cluster 2023-08-28T01:48:44.656410+0000 mgr.y (mgr.14901) 6904 : cluster [DBG] pgmap v6911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:46.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:46 smithi006 bash[33918]: cluster 2023-08-28T01:48:44.656410+0000 mgr.y (mgr.14901) 6904 : cluster [DBG] pgmap v6911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:47.345 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:46 smithi130 bash[51302]: debug 2023-08-28T01:48:46.962+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:46.966793+0000) 2023-08-28T01:48:47.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:47 smithi130 bash[27579]: audit 2023-08-28T01:48:47.223583+0000 mon.a (mon.0) 4517 : audit [DBG] from='client.? 172.21.15.6:0/290353524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:47.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:47 smithi006 bash[33918]: audit 2023-08-28T01:48:47.223583+0000 mon.a (mon.0) 4517 : audit [DBG] from='client.? 172.21.15.6:0/290353524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:47.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:47 smithi006 bash[27292]: audit 2023-08-28T01:48:47.223583+0000 mon.a (mon.0) 4517 : audit [DBG] from='client.? 172.21.15.6:0/290353524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:48.348 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:47 smithi130 bash[51302]: debug 2023-08-28T01:48:47.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:47.967075+0000) 2023-08-28T01:48:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:48 smithi130 bash[27579]: cluster 2023-08-28T01:48:46.657169+0000 mgr.y (mgr.14901) 6905 : cluster [DBG] pgmap v6912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:48.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:48 smithi006 bash[33918]: cluster 2023-08-28T01:48:46.657169+0000 mgr.y (mgr.14901) 6905 : cluster [DBG] pgmap v6912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:48.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:48 smithi006 bash[27292]: cluster 2023-08-28T01:48:46.657169+0000 mgr.y (mgr.14901) 6905 : cluster [DBG] pgmap v6912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:49.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:48 smithi130 bash[51302]: debug 2023-08-28T01:48:48.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:48.967379+0000) 2023-08-28T01:48:50.352 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:49 smithi130 bash[51302]: debug 2023-08-28T01:48:49.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:49.967686+0000) 2023-08-28T01:48:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:50 smithi130 bash[27579]: cluster 2023-08-28T01:48:48.658000+0000 mgr.y (mgr.14901) 6906 : cluster [DBG] pgmap v6913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:50 smithi130 bash[27579]: audit 2023-08-28T01:48:49.646153+0000 mon.c (mon.2) 2714 : audit [DBG] from='client.? 172.21.15.6:0/615775427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:50.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:50 smithi006 bash[33918]: cluster 2023-08-28T01:48:48.658000+0000 mgr.y (mgr.14901) 6906 : cluster [DBG] pgmap v6913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:50.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:50 smithi006 bash[33918]: audit 2023-08-28T01:48:49.646153+0000 mon.c (mon.2) 2714 : audit [DBG] from='client.? 172.21.15.6:0/615775427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:50 smithi006 bash[27292]: cluster 2023-08-28T01:48:48.658000+0000 mgr.y (mgr.14901) 6906 : cluster [DBG] pgmap v6913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:50 smithi006 bash[27292]: audit 2023-08-28T01:48:49.646153+0000 mon.c (mon.2) 2714 : audit [DBG] from='client.? 172.21.15.6:0/615775427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:51.126 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:48:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:48:50] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-28T01:48:51.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:50 smithi130 bash[51302]: debug 2023-08-28T01:48:50.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:50.968010+0000) 2023-08-28T01:48:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:48:51.126Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:48:51.128Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:52.356 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:51 smithi130 bash[51302]: debug 2023-08-28T01:48:51.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:51.968302+0000) 2023-08-28T01:48:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:52 smithi130 bash[27579]: cluster 2023-08-28T01:48:50.659184+0000 mgr.y (mgr.14901) 6907 : cluster [DBG] pgmap v6914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:52 smithi130 bash[27579]: audit 2023-08-28T01:48:52.060644+0000 mon.c (mon.2) 2715 : audit [DBG] from='client.? 172.21.15.6:0/3515520296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:52.691 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:52 smithi006 bash[33918]: cluster 2023-08-28T01:48:50.659184+0000 mgr.y (mgr.14901) 6907 : cluster [DBG] pgmap v6914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:52.691 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:52 smithi006 bash[33918]: audit 2023-08-28T01:48:52.060644+0000 mon.c (mon.2) 2715 : audit [DBG] from='client.? 172.21.15.6:0/3515520296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:52.691 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:52 smithi006 bash[27292]: cluster 2023-08-28T01:48:50.659184+0000 mgr.y (mgr.14901) 6907 : cluster [DBG] pgmap v6914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:52.691 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:52 smithi006 bash[27292]: audit 2023-08-28T01:48:52.060644+0000 mon.c (mon.2) 2715 : audit [DBG] from='client.? 172.21.15.6:0/3515520296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:48:52.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:48:52.693Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:53.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:52 smithi130 bash[51302]: debug 2023-08-28T01:48:52.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:52.968668+0000) 2023-08-28T01:48:54.362 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:53 smithi130 bash[51302]: debug 2023-08-28T01:48:53.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:53.968961+0000) 2023-08-28T01:48:54.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:54 smithi130 bash[27579]: cluster 2023-08-28T01:48:52.659991+0000 mgr.y (mgr.14901) 6908 : cluster [DBG] pgmap v6915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:54.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:54 smithi006 bash[27292]: cluster 2023-08-28T01:48:52.659991+0000 mgr.y (mgr.14901) 6908 : cluster [DBG] pgmap v6915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:54.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:54 smithi006 bash[33918]: cluster 2023-08-28T01:48:52.659991+0000 mgr.y (mgr.14901) 6908 : cluster [DBG] pgmap v6915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:48:54.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:48:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:48:54.578Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:48:55.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:54 smithi130 bash[51302]: debug 2023-08-28T01:48:54.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:54.969225+0000) 2023-08-28T01:48:55.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:55 smithi006 bash[33918]: audit 2023-08-28T01:48:54.475713+0000 mon.c (mon.2) 2716 : audit [DBG] from='client.? 172.21.15.6:0/2744953760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:55.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:55 smithi006 bash[27292]: audit 2023-08-28T01:48:54.475713+0000 mon.c (mon.2) 2716 : audit [DBG] from='client.? 172.21.15.6:0/2744953760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:55.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:55 smithi130 bash[27579]: audit 2023-08-28T01:48:54.475713+0000 mon.c (mon.2) 2716 : audit [DBG] from='client.? 172.21.15.6:0/2744953760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:56.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:55 smithi130 bash[51302]: debug 2023-08-28T01:48:55.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:55.969527+0000) 2023-08-28T01:48:56.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:56 smithi006 bash[27292]: cluster 2023-08-28T01:48:54.661053+0000 mgr.y (mgr.14901) 6909 : cluster [DBG] pgmap v6916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:56.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:56 smithi006 bash[33918]: cluster 2023-08-28T01:48:54.661053+0000 mgr.y (mgr.14901) 6909 : cluster [DBG] pgmap v6916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:56 smithi130 bash[27579]: cluster 2023-08-28T01:48:54.661053+0000 mgr.y (mgr.14901) 6909 : cluster [DBG] pgmap v6916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:57.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:56 smithi130 bash[51302]: debug 2023-08-28T01:48:56.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:56.969827+0000) 2023-08-28T01:48:57.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:57 smithi006 bash[27292]: audit 2023-08-28T01:48:56.885875+0000 mon.a (mon.0) 4518 : audit [DBG] from='client.? 172.21.15.6:0/3433757174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:57.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:57 smithi006 bash[33918]: audit 2023-08-28T01:48:56.885875+0000 mon.a (mon.0) 4518 : audit [DBG] from='client.? 172.21.15.6:0/3433757174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:57.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:57 smithi130 bash[27579]: audit 2023-08-28T01:48:56.885875+0000 mon.a (mon.0) 4518 : audit [DBG] from='client.? 172.21.15.6:0/3433757174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:58.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:57 smithi130 bash[51302]: debug 2023-08-28T01:48:57.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:57.970114+0000) 2023-08-28T01:48:58.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:58 smithi006 bash[27292]: cluster 2023-08-28T01:48:56.661629+0000 mgr.y (mgr.14901) 6910 : cluster [DBG] pgmap v6917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:58.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:58 smithi006 bash[33918]: cluster 2023-08-28T01:48:56.661629+0000 mgr.y (mgr.14901) 6910 : cluster [DBG] pgmap v6917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:58.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:58 smithi130 bash[27579]: cluster 2023-08-28T01:48:56.661629+0000 mgr.y (mgr.14901) 6910 : cluster [DBG] pgmap v6917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:48:59.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:58 smithi130 bash[51302]: debug 2023-08-28T01:48:58.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:58.970402+0000) 2023-08-28T01:48:59.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:48:59 smithi006 bash[33918]: audit 2023-08-28T01:48:59.301792+0000 mon.a (mon.0) 4519 : audit [DBG] from='client.? 172.21.15.6:0/3482711400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:59.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:48:59 smithi006 bash[27292]: audit 2023-08-28T01:48:59.301792+0000 mon.a (mon.0) 4519 : audit [DBG] from='client.? 172.21.15.6:0/3482711400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:59.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:48:59 smithi130 bash[27579]: audit 2023-08-28T01:48:59.301792+0000 mon.a (mon.0) 4519 : audit [DBG] from='client.? 172.21.15.6:0/3482711400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:00.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:48:59 smithi130 bash[51302]: debug 2023-08-28T01:48:59.966+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:59.970692+0000) 2023-08-28T01:49:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:00 smithi006 bash[33918]: cluster 2023-08-28T01:48:58.664442+0000 mgr.y (mgr.14901) 6911 : cluster [DBG] pgmap v6918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:00 smithi006 bash[27292]: cluster 2023-08-28T01:48:58.664442+0000 mgr.y (mgr.14901) 6911 : cluster [DBG] pgmap v6918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:00 smithi130 bash[27579]: cluster 2023-08-28T01:48:58.664442+0000 mgr.y (mgr.14901) 6911 : cluster [DBG] pgmap v6918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:01.126 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:49:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:49:00] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:49:01.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:00 smithi130 bash[51302]: debug 2023-08-28T01:49:00.970+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:00.971009+0000) 2023-08-28T01:49:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:49:01.126Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:49:01.129Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:02.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:01 smithi130 bash[51302]: debug 2023-08-28T01:49:01.970+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:01.971252+0000) 2023-08-28T01:49:02.692 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:02 smithi006 bash[33918]: cluster 2023-08-28T01:49:00.665606+0000 mgr.y (mgr.14901) 6912 : cluster [DBG] pgmap v6919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:02.692 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:02 smithi006 bash[33918]: audit 2023-08-28T01:49:01.715958+0000 mon.c (mon.2) 2717 : audit [DBG] from='client.? 172.21.15.6:0/918166633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:02.692 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:02 smithi006 bash[27292]: cluster 2023-08-28T01:49:00.665606+0000 mgr.y (mgr.14901) 6912 : cluster [DBG] pgmap v6919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:02.693 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:02 smithi006 bash[27292]: audit 2023-08-28T01:49:01.715958+0000 mon.c (mon.2) 2717 : audit [DBG] from='client.? 172.21.15.6:0/918166633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:02 smithi130 bash[27579]: cluster 2023-08-28T01:49:00.665606+0000 mgr.y (mgr.14901) 6912 : cluster [DBG] pgmap v6919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:02 smithi130 bash[27579]: audit 2023-08-28T01:49:01.715958+0000 mon.c (mon.2) 2717 : audit [DBG] from='client.? 172.21.15.6:0/918166633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:49:02.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:49:02.694Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:03.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:02 smithi130 bash[51302]: debug 2023-08-28T01:49:02.970+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:02.971507+0000) 2023-08-28T01:49:04.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:03 smithi130 bash[51302]: debug 2023-08-28T01:49:03.970+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:03.971737+0000) 2023-08-28T01:49:04.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:04 smithi006 bash[27292]: cluster 2023-08-28T01:49:02.666266+0000 mgr.y (mgr.14901) 6913 : cluster [DBG] pgmap v6920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:04.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:04 smithi006 bash[27292]: audit 2023-08-28T01:49:04.131705+0000 mon.a (mon.0) 4520 : audit [DBG] from='client.? 172.21.15.6:0/2651000793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:04.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:04 smithi006 bash[33918]: cluster 2023-08-28T01:49:02.666266+0000 mgr.y (mgr.14901) 6913 : cluster [DBG] pgmap v6920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:04.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:04 smithi006 bash[33918]: audit 2023-08-28T01:49:04.131705+0000 mon.a (mon.0) 4520 : audit [DBG] from='client.? 172.21.15.6:0/2651000793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:49:04.577Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:49:04.579Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:04.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:04 smithi130 bash[27579]: cluster 2023-08-28T01:49:02.666266+0000 mgr.y (mgr.14901) 6913 : cluster [DBG] pgmap v6920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:04.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:04 smithi130 bash[27579]: audit 2023-08-28T01:49:04.131705+0000 mon.a (mon.0) 4520 : audit [DBG] from='client.? 172.21.15.6:0/2651000793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:05.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:04 smithi130 bash[51302]: debug 2023-08-28T01:49:04.970+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:04.972021+0000) 2023-08-28T01:49:06.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:05 smithi130 bash[51302]: debug 2023-08-28T01:49:05.970+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:05.972296+0000) 2023-08-28T01:49:06.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:06 smithi006 bash[33918]: cluster 2023-08-28T01:49:04.667417+0000 mgr.y (mgr.14901) 6914 : cluster [DBG] pgmap v6921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:06.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:06 smithi006 bash[27292]: cluster 2023-08-28T01:49:04.667417+0000 mgr.y (mgr.14901) 6914 : cluster [DBG] pgmap v6921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:06.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:06 smithi130 bash[27579]: cluster 2023-08-28T01:49:04.667417+0000 mgr.y (mgr.14901) 6914 : cluster [DBG] pgmap v6921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:07.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:06 smithi130 bash[51302]: debug 2023-08-28T01:49:06.970+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:06.972569+0000) 2023-08-28T01:49:07.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:07 smithi006 bash[27292]: audit 2023-08-28T01:49:06.546943+0000 mon.a (mon.0) 4521 : audit [DBG] from='client.? 172.21.15.6:0/3139594080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:07.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:07 smithi006 bash[33918]: audit 2023-08-28T01:49:06.546943+0000 mon.a (mon.0) 4521 : audit [DBG] from='client.? 172.21.15.6:0/3139594080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:07.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:07 smithi130 bash[27579]: audit 2023-08-28T01:49:06.546943+0000 mon.a (mon.0) 4521 : audit [DBG] from='client.? 172.21.15.6:0/3139594080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:08.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:07 smithi130 bash[51302]: debug 2023-08-28T01:49:07.970+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:07.972840+0000) 2023-08-28T01:49:08.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:08 smithi006 bash[27292]: cluster 2023-08-28T01:49:06.668087+0000 mgr.y (mgr.14901) 6915 : cluster [DBG] pgmap v6922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:08.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:08 smithi006 bash[33918]: cluster 2023-08-28T01:49:06.668087+0000 mgr.y (mgr.14901) 6915 : cluster [DBG] pgmap v6922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:08 smithi130 bash[27579]: cluster 2023-08-28T01:49:06.668087+0000 mgr.y (mgr.14901) 6915 : cluster [DBG] pgmap v6922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:09.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:08 smithi130 bash[51302]: debug 2023-08-28T01:49:08.970+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:08.973100+0000) 2023-08-28T01:49:09.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:09 smithi006 bash[33918]: audit 2023-08-28T01:49:08.955931+0000 mon.a (mon.0) 4522 : audit [DBG] from='client.? 172.21.15.6:0/3118765509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:09.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:09 smithi006 bash[27292]: audit 2023-08-28T01:49:08.955931+0000 mon.a (mon.0) 4522 : audit [DBG] from='client.? 172.21.15.6:0/3118765509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:09.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:09 smithi130 bash[27579]: audit 2023-08-28T01:49:08.955931+0000 mon.a (mon.0) 4522 : audit [DBG] from='client.? 172.21.15.6:0/3118765509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:10.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:09 smithi130 bash[51302]: debug 2023-08-28T01:49:09.970+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:09.973364+0000) 2023-08-28T01:49:10.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:10 smithi006 bash[27292]: cluster 2023-08-28T01:49:08.668775+0000 mgr.y (mgr.14901) 6916 : cluster [DBG] pgmap v6923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:10 smithi006 bash[33918]: cluster 2023-08-28T01:49:08.668775+0000 mgr.y (mgr.14901) 6916 : cluster [DBG] pgmap v6923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:10.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:10 smithi130 bash[27579]: cluster 2023-08-28T01:49:08.668775+0000 mgr.y (mgr.14901) 6916 : cluster [DBG] pgmap v6923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:11.127 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:49:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:49:10] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:49:11.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:10 smithi130 bash[51302]: debug 2023-08-28T01:49:10.970+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:10.973642+0000) 2023-08-28T01:49:11.436 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:49:11.127Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:11.436 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:49:11.129Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:11.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:11 smithi006 bash[33918]: cluster 2023-08-28T01:49:10.669813+0000 mgr.y (mgr.14901) 6917 : cluster [DBG] pgmap v6924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:11.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:11 smithi006 bash[33918]: audit 2023-08-28T01:49:11.377437+0000 mon.c (mon.2) 2718 : audit [DBG] from='client.? 172.21.15.6:0/3969082904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:11.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:11 smithi006 bash[27292]: cluster 2023-08-28T01:49:10.669813+0000 mgr.y (mgr.14901) 6917 : cluster [DBG] pgmap v6924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:11.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:11 smithi006 bash[27292]: audit 2023-08-28T01:49:11.377437+0000 mon.c (mon.2) 2718 : audit [DBG] from='client.? 172.21.15.6:0/3969082904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:11.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:11 smithi130 bash[27579]: cluster 2023-08-28T01:49:10.669813+0000 mgr.y (mgr.14901) 6917 : cluster [DBG] pgmap v6924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:11.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:11 smithi130 bash[27579]: audit 2023-08-28T01:49:11.377437+0000 mon.c (mon.2) 2718 : audit [DBG] from='client.? 172.21.15.6:0/3969082904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:12.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:11 smithi130 bash[51302]: debug 2023-08-28T01:49:11.970+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:11.973901+0000) 2023-08-28T01:49:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:49:12.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:49:12.695Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:13.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:12 smithi130 bash[51302]: debug 2023-08-28T01:49:12.970+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:12.974165+0000) 2023-08-28T01:49:14.315 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:13 smithi130 bash[51302]: debug 2023-08-28T01:49:13.970+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:13.974419+0000) 2023-08-28T01:49:14.578 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:14 smithi006 bash[33918]: cluster 2023-08-28T01:49:12.670416+0000 mgr.y (mgr.14901) 6918 : cluster [DBG] pgmap v6925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:14.578 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:14 smithi006 bash[33918]: audit 2023-08-28T01:49:13.788082+0000 mon.c (mon.2) 2719 : audit [DBG] from='client.? 172.21.15.6:0/3333924272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:14.579 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:14 smithi006 bash[27292]: cluster 2023-08-28T01:49:12.670416+0000 mgr.y (mgr.14901) 6918 : cluster [DBG] pgmap v6925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:14.579 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:14 smithi006 bash[27292]: audit 2023-08-28T01:49:13.788082+0000 mon.c (mon.2) 2719 : audit [DBG] from='client.? 172.21.15.6:0/3333924272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:14 smithi130 bash[27579]: cluster 2023-08-28T01:49:12.670416+0000 mgr.y (mgr.14901) 6918 : cluster [DBG] pgmap v6925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:14 smithi130 bash[27579]: audit 2023-08-28T01:49:13.788082+0000 mon.c (mon.2) 2719 : audit [DBG] from='client.? 172.21.15.6:0/3333924272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:49:14.578Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:49:14.580Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:15.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:14 smithi130 bash[51302]: debug 2023-08-28T01:49:14.970+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:14.974697+0000) 2023-08-28T01:49:16.319 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:15 smithi130 bash[51302]: debug 2023-08-28T01:49:15.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:15.974938+0000) 2023-08-28T01:49:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:16 smithi130 bash[27579]: cluster 2023-08-28T01:49:14.671637+0000 mgr.y (mgr.14901) 6919 : cluster [DBG] pgmap v6926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:16 smithi130 bash[27579]: audit 2023-08-28T01:49:16.208634+0000 mon.a (mon.0) 4523 : audit [DBG] from='client.? 172.21.15.6:0/3087713014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:16.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:16 smithi006 bash[33918]: cluster 2023-08-28T01:49:14.671637+0000 mgr.y (mgr.14901) 6919 : cluster [DBG] pgmap v6926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:16 smithi006 bash[33918]: audit 2023-08-28T01:49:16.208634+0000 mon.a (mon.0) 4523 : audit [DBG] from='client.? 172.21.15.6:0/3087713014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:16 smithi006 bash[27292]: cluster 2023-08-28T01:49:14.671637+0000 mgr.y (mgr.14901) 6919 : cluster [DBG] pgmap v6926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:16 smithi006 bash[27292]: audit 2023-08-28T01:49:16.208634+0000 mon.a (mon.0) 4523 : audit [DBG] from='client.? 172.21.15.6:0/3087713014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:17.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:16 smithi130 bash[51302]: debug 2023-08-28T01:49:16.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:16.975195+0000) 2023-08-28T01:49:18.324 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:17 smithi130 bash[51302]: debug 2023-08-28T01:49:17.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:17.975500+0000) 2023-08-28T01:49:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:18 smithi130 bash[27579]: cluster 2023-08-28T01:49:16.672474+0000 mgr.y (mgr.14901) 6920 : cluster [DBG] pgmap v6927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:18 smithi006 bash[27292]: cluster 2023-08-28T01:49:16.672474+0000 mgr.y (mgr.14901) 6920 : cluster [DBG] pgmap v6927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:18.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:18 smithi006 bash[33918]: cluster 2023-08-28T01:49:16.672474+0000 mgr.y (mgr.14901) 6920 : cluster [DBG] pgmap v6927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:19.332 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:18 smithi130 bash[51302]: debug 2023-08-28T01:49:18.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:18.975802+0000) 2023-08-28T01:49:19.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:19 smithi130 bash[27579]: audit 2023-08-28T01:49:18.623384+0000 mon.c (mon.2) 2720 : audit [DBG] from='client.? 172.21.15.6:0/2518342832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:19.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:19 smithi006 bash[27292]: audit 2023-08-28T01:49:18.623384+0000 mon.c (mon.2) 2720 : audit [DBG] from='client.? 172.21.15.6:0/2518342832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:19.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:19 smithi006 bash[33918]: audit 2023-08-28T01:49:18.623384+0000 mon.c (mon.2) 2720 : audit [DBG] from='client.? 172.21.15.6:0/2518342832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:20.336 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:19 smithi130 bash[51302]: debug 2023-08-28T01:49:19.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:19.976103+0000) 2023-08-28T01:49:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:20 smithi130 bash[27579]: cluster 2023-08-28T01:49:18.673197+0000 mgr.y (mgr.14901) 6921 : cluster [DBG] pgmap v6928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:20.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:20 smithi006 bash[27292]: cluster 2023-08-28T01:49:18.673197+0000 mgr.y (mgr.14901) 6921 : cluster [DBG] pgmap v6928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:20.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:20 smithi006 bash[33918]: cluster 2023-08-28T01:49:18.673197+0000 mgr.y (mgr.14901) 6921 : cluster [DBG] pgmap v6928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:21.127 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:49:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:49:20] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:49:21.345 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:20 smithi130 bash[51302]: debug 2023-08-28T01:49:20.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:20.976451+0000) 2023-08-28T01:49:21.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:21 smithi006 bash[27292]: audit 2023-08-28T01:49:21.035797+0000 mon.a (mon.0) 4524 : audit [DBG] from='client.? 172.21.15.6:0/3155571732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:21.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:21 smithi006 bash[33918]: audit 2023-08-28T01:49:21.035797+0000 mon.a (mon.0) 4524 : audit [DBG] from='client.? 172.21.15.6:0/3155571732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:49:21.127Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:21.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:49:21.129Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:21.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:21 smithi130 bash[27579]: audit 2023-08-28T01:49:21.035797+0000 mon.a (mon.0) 4524 : audit [DBG] from='client.? 172.21.15.6:0/3155571732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:22.349 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:21 smithi130 bash[51302]: debug 2023-08-28T01:49:21.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:21.976741+0000) 2023-08-28T01:49:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:22 smithi130 bash[27579]: cluster 2023-08-28T01:49:20.674343+0000 mgr.y (mgr.14901) 6922 : cluster [DBG] pgmap v6929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:22.693 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:22 smithi006 bash[33918]: cluster 2023-08-28T01:49:20.674343+0000 mgr.y (mgr.14901) 6922 : cluster [DBG] pgmap v6929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:22.693 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:22 smithi006 bash[27292]: cluster 2023-08-28T01:49:20.674343+0000 mgr.y (mgr.14901) 6922 : cluster [DBG] pgmap v6929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:49:22.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:49:22.695Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:23.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:22 smithi130 bash[51302]: debug 2023-08-28T01:49:22.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:22.977018+0000) 2023-08-28T01:49:24.353 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:23 smithi130 bash[51302]: debug 2023-08-28T01:49:23.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:23.977301+0000) 2023-08-28T01:49:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:24 smithi130 bash[27579]: cluster 2023-08-28T01:49:22.675014+0000 mgr.y (mgr.14901) 6923 : cluster [DBG] pgmap v6930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:24 smithi130 bash[27579]: audit 2023-08-28T01:49:23.456015+0000 mon.c (mon.2) 2721 : audit [DBG] from='client.? 172.21.15.6:0/2835317217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:24.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:24 smithi006 bash[27292]: cluster 2023-08-28T01:49:22.675014+0000 mgr.y (mgr.14901) 6923 : cluster [DBG] pgmap v6930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:24.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:24 smithi006 bash[27292]: audit 2023-08-28T01:49:23.456015+0000 mon.c (mon.2) 2721 : audit [DBG] from='client.? 172.21.15.6:0/2835317217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:24.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:24 smithi006 bash[33918]: cluster 2023-08-28T01:49:22.675014+0000 mgr.y (mgr.14901) 6923 : cluster [DBG] pgmap v6930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:24.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:24 smithi006 bash[33918]: audit 2023-08-28T01:49:23.456015+0000 mon.c (mon.2) 2721 : audit [DBG] from='client.? 172.21.15.6:0/2835317217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:49:24.578Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:49:24.581Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:25.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:24 smithi130 bash[51302]: debug 2023-08-28T01:49:24.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:24.977575+0000) 2023-08-28T01:49:26.360 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:25 smithi130 bash[51302]: debug 2023-08-28T01:49:25.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:25.977830+0000) 2023-08-28T01:49:26.361 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:26 smithi130 bash[27579]: cluster 2023-08-28T01:49:24.676282+0000 mgr.y (mgr.14901) 6924 : cluster [DBG] pgmap v6931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:26.361 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:26 smithi130 bash[27579]: audit 2023-08-28T01:49:25.877785+0000 mon.a (mon.0) 4525 : audit [DBG] from='client.? 172.21.15.6:0/3799406236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:26.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:26 smithi006 bash[27292]: cluster 2023-08-28T01:49:24.676282+0000 mgr.y (mgr.14901) 6924 : cluster [DBG] pgmap v6931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:26.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:26 smithi006 bash[27292]: audit 2023-08-28T01:49:25.877785+0000 mon.a (mon.0) 4525 : audit [DBG] from='client.? 172.21.15.6:0/3799406236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:26 smithi006 bash[33918]: cluster 2023-08-28T01:49:24.676282+0000 mgr.y (mgr.14901) 6924 : cluster [DBG] pgmap v6931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:26 smithi006 bash[33918]: audit 2023-08-28T01:49:25.877785+0000 mon.a (mon.0) 4525 : audit [DBG] from='client.? 172.21.15.6:0/3799406236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:27.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:26 smithi130 bash[51302]: debug 2023-08-28T01:49:26.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:26.978097+0000) 2023-08-28T01:49:28.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:27 smithi130 bash[51302]: debug 2023-08-28T01:49:27.973+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:27.978402+0000) 2023-08-28T01:49:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:28 smithi130 bash[27579]: cluster 2023-08-28T01:49:26.677069+0000 mgr.y (mgr.14901) 6925 : cluster [DBG] pgmap v6932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:28 smithi130 bash[27579]: audit 2023-08-28T01:49:28.284872+0000 mon.a (mon.0) 4526 : audit [DBG] from='client.? 172.21.15.6:0/2551464256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:28.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:28 smithi006 bash[33918]: cluster 2023-08-28T01:49:26.677069+0000 mgr.y (mgr.14901) 6925 : cluster [DBG] pgmap v6932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:28 smithi006 bash[33918]: audit 2023-08-28T01:49:28.284872+0000 mon.a (mon.0) 4526 : audit [DBG] from='client.? 172.21.15.6:0/2551464256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:28 smithi006 bash[27292]: cluster 2023-08-28T01:49:26.677069+0000 mgr.y (mgr.14901) 6925 : cluster [DBG] pgmap v6932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:28 smithi006 bash[27292]: audit 2023-08-28T01:49:28.284872+0000 mon.a (mon.0) 4526 : audit [DBG] from='client.? 172.21.15.6:0/2551464256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:29.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:28 smithi130 bash[51302]: debug 2023-08-28T01:49:28.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:28.978740+0000) 2023-08-28T01:49:30.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:29 smithi130 bash[51302]: debug 2023-08-28T01:49:29.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:29.979062+0000) 2023-08-28T01:49:30.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:30 smithi006 bash[27292]: cluster 2023-08-28T01:49:28.677786+0000 mgr.y (mgr.14901) 6926 : cluster [DBG] pgmap v6933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:30 smithi006 bash[33918]: cluster 2023-08-28T01:49:28.677786+0000 mgr.y (mgr.14901) 6926 : cluster [DBG] pgmap v6933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:30 smithi130 bash[27579]: cluster 2023-08-28T01:49:28.677786+0000 mgr.y (mgr.14901) 6926 : cluster [DBG] pgmap v6933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:31.129 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:49:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:49:30] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:49:31.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:30 smithi130 bash[51302]: debug 2023-08-28T01:49:30.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:30.979368+0000) 2023-08-28T01:49:31.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:31 smithi006 bash[33918]: audit 2023-08-28T01:49:30.707449+0000 mon.c (mon.2) 2722 : audit [DBG] from='client.? 172.21.15.6:0/4281416620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:31.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:31 smithi006 bash[27292]: audit 2023-08-28T01:49:30.707449+0000 mon.c (mon.2) 2722 : audit [DBG] from='client.? 172.21.15.6:0/4281416620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:49:31.128Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:49:31.131Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:31.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:31 smithi130 bash[27579]: audit 2023-08-28T01:49:30.707449+0000 mon.c (mon.2) 2722 : audit [DBG] from='client.? 172.21.15.6:0/4281416620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:32.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:31 smithi130 bash[51302]: debug 2023-08-28T01:49:31.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:31.979670+0000) 2023-08-28T01:49:32.693 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:32 smithi006 bash[33918]: cluster 2023-08-28T01:49:30.679061+0000 mgr.y (mgr.14901) 6927 : cluster [DBG] pgmap v6934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:32.694 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:32 smithi006 bash[27292]: cluster 2023-08-28T01:49:30.679061+0000 mgr.y (mgr.14901) 6927 : cluster [DBG] pgmap v6934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:32 smithi130 bash[27579]: cluster 2023-08-28T01:49:30.679061+0000 mgr.y (mgr.14901) 6927 : cluster [DBG] pgmap v6934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:49:32.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:49:32.695Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:33.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:32 smithi130 bash[51302]: debug 2023-08-28T01:49:32.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:32.979948+0000) 2023-08-28T01:49:33.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:33 smithi006 bash[27292]: audit 2023-08-28T01:49:32.922071+0000 mon.b (mon.1) 778 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:49:33.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:33 smithi006 bash[27292]: audit 2023-08-28T01:49:33.134155+0000 mon.a (mon.0) 4527 : audit [DBG] from='client.? 172.21.15.6:0/966031669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:33.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:33 smithi006 bash[33918]: audit 2023-08-28T01:49:32.922071+0000 mon.b (mon.1) 778 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:49:33.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:33 smithi006 bash[33918]: audit 2023-08-28T01:49:33.134155+0000 mon.a (mon.0) 4527 : audit [DBG] from='client.? 172.21.15.6:0/966031669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:33.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:33 smithi130 bash[27579]: audit 2023-08-28T01:49:32.922071+0000 mon.b (mon.1) 778 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:49:33.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:33 smithi130 bash[27579]: audit 2023-08-28T01:49:33.134155+0000 mon.a (mon.0) 4527 : audit [DBG] from='client.? 172.21.15.6:0/966031669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:34.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:33 smithi130 bash[51302]: debug 2023-08-28T01:49:33.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:33.980200+0000) 2023-08-28T01:49:34.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:34 smithi006 bash[33918]: cluster 2023-08-28T01:49:32.679858+0000 mgr.y (mgr.14901) 6928 : cluster [DBG] pgmap v6935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:34.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:34 smithi006 bash[27292]: cluster 2023-08-28T01:49:32.679858+0000 mgr.y (mgr.14901) 6928 : cluster [DBG] pgmap v6935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:49:34.579Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:49:34.581Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:34.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:34 smithi130 bash[27579]: cluster 2023-08-28T01:49:32.679858+0000 mgr.y (mgr.14901) 6928 : cluster [DBG] pgmap v6935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:35.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:34 smithi130 bash[51302]: debug 2023-08-28T01:49:34.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:34.980479+0000) 2023-08-28T01:49:36.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:35 smithi130 bash[51302]: debug 2023-08-28T01:49:35.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:35.980785+0000) 2023-08-28T01:49:36.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:36 smithi006 bash[33918]: cluster 2023-08-28T01:49:34.681049+0000 mgr.y (mgr.14901) 6929 : cluster [DBG] pgmap v6936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:36.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:36 smithi006 bash[33918]: audit 2023-08-28T01:49:35.556863+0000 mon.a (mon.0) 4528 : audit [DBG] from='client.? 172.21.15.6:0/2461106553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:36.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:36 smithi006 bash[27292]: cluster 2023-08-28T01:49:34.681049+0000 mgr.y (mgr.14901) 6929 : cluster [DBG] pgmap v6936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:36.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:36 smithi006 bash[27292]: audit 2023-08-28T01:49:35.556863+0000 mon.a (mon.0) 4528 : audit [DBG] from='client.? 172.21.15.6:0/2461106553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:36.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:36 smithi130 bash[27579]: cluster 2023-08-28T01:49:34.681049+0000 mgr.y (mgr.14901) 6929 : cluster [DBG] pgmap v6936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:36.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:36 smithi130 bash[27579]: audit 2023-08-28T01:49:35.556863+0000 mon.a (mon.0) 4528 : audit [DBG] from='client.? 172.21.15.6:0/2461106553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:37.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:36 smithi130 bash[51302]: debug 2023-08-28T01:49:36.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:36.981054+0000) 2023-08-28T01:49:38.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:37 smithi130 bash[51302]: debug 2023-08-28T01:49:37.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:37.981316+0000) 2023-08-28T01:49:38.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:38 smithi006 bash[27292]: cluster 2023-08-28T01:49:36.681681+0000 mgr.y (mgr.14901) 6930 : cluster [DBG] pgmap v6937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:38.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:38 smithi006 bash[27292]: audit 2023-08-28T01:49:37.982046+0000 mon.c (mon.2) 2723 : audit [DBG] from='client.? 172.21.15.6:0/3271033702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:38 smithi006 bash[33918]: cluster 2023-08-28T01:49:36.681681+0000 mgr.y (mgr.14901) 6930 : cluster [DBG] pgmap v6937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:38 smithi006 bash[33918]: audit 2023-08-28T01:49:37.982046+0000 mon.c (mon.2) 2723 : audit [DBG] from='client.? 172.21.15.6:0/3271033702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:38.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:38 smithi130 bash[27579]: cluster 2023-08-28T01:49:36.681681+0000 mgr.y (mgr.14901) 6930 : cluster [DBG] pgmap v6937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:38.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:38 smithi130 bash[27579]: audit 2023-08-28T01:49:37.982046+0000 mon.c (mon.2) 2723 : audit [DBG] from='client.? 172.21.15.6:0/3271033702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:39.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:38 smithi130 bash[51302]: debug 2023-08-28T01:49:38.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:38.981557+0000) 2023-08-28T01:49:40.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:39 smithi130 bash[51302]: debug 2023-08-28T01:49:39.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:39.981815+0000) 2023-08-28T01:49:40.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:39 smithi130 bash[27579]: cluster 2023-08-28T01:49:38.682270+0000 mgr.y (mgr.14901) 6931 : cluster [DBG] pgmap v6938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:40.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:39 smithi130 bash[27579]: audit 2023-08-28T01:49:38.820891+0000 mon.a (mon.0) 4529 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:49:40.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:39 smithi130 bash[27579]: audit 2023-08-28T01:49:39.274115+0000 mon.a (mon.0) 4530 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:49:40.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:39 smithi130 bash[27579]: audit 2023-08-28T01:49:39.544420+0000 mon.b (mon.1) 779 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:49:40.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:39 smithi130 bash[27579]: audit 2023-08-28T01:49:39.546725+0000 mon.b (mon.1) 780 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:49:40.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:39 smithi130 bash[27579]: audit 2023-08-28T01:49:39.554770+0000 mon.a (mon.0) 4531 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:49:40.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:39 smithi006 bash[33918]: cluster 2023-08-28T01:49:38.682270+0000 mgr.y (mgr.14901) 6931 : cluster [DBG] pgmap v6938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:40.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:39 smithi006 bash[33918]: audit 2023-08-28T01:49:38.820891+0000 mon.a (mon.0) 4529 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:49:40.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:39 smithi006 bash[33918]: audit 2023-08-28T01:49:39.274115+0000 mon.a (mon.0) 4530 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:49:40.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:39 smithi006 bash[33918]: audit 2023-08-28T01:49:39.544420+0000 mon.b (mon.1) 779 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:49:40.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:39 smithi006 bash[33918]: audit 2023-08-28T01:49:39.546725+0000 mon.b (mon.1) 780 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:49:40.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:39 smithi006 bash[33918]: audit 2023-08-28T01:49:39.554770+0000 mon.a (mon.0) 4531 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:49:40.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:39 smithi006 bash[27292]: cluster 2023-08-28T01:49:38.682270+0000 mgr.y (mgr.14901) 6931 : cluster [DBG] pgmap v6938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:40.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:39 smithi006 bash[27292]: audit 2023-08-28T01:49:38.820891+0000 mon.a (mon.0) 4529 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:49:40.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:39 smithi006 bash[27292]: audit 2023-08-28T01:49:39.274115+0000 mon.a (mon.0) 4530 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:49:40.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:39 smithi006 bash[27292]: audit 2023-08-28T01:49:39.544420+0000 mon.b (mon.1) 779 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:49:40.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:39 smithi006 bash[27292]: audit 2023-08-28T01:49:39.546725+0000 mon.b (mon.1) 780 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:49:40.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:39 smithi006 bash[27292]: audit 2023-08-28T01:49:39.554770+0000 mon.a (mon.0) 4531 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:49:41.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:40 smithi130 bash[27579]: audit 2023-08-28T01:49:40.405984+0000 mon.c (mon.2) 2724 : audit [DBG] from='client.? 172.21.15.6:0/1219945076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:41.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:40 smithi130 bash[51302]: debug 2023-08-28T01:49:40.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:40.982129+0000) 2023-08-28T01:49:41.129 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:49:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:49:40] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:49:41.130 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:40 smithi006 bash[33918]: audit 2023-08-28T01:49:40.405984+0000 mon.c (mon.2) 2724 : audit [DBG] from='client.? 172.21.15.6:0/1219945076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:41.130 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:40 smithi006 bash[27292]: audit 2023-08-28T01:49:40.405984+0000 mon.c (mon.2) 2724 : audit [DBG] from='client.? 172.21.15.6:0/1219945076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:49:41.129Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:49:41.132Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:42.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:41 smithi130 bash[27579]: cluster 2023-08-28T01:49:40.683373+0000 mgr.y (mgr.14901) 6932 : cluster [DBG] pgmap v6939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:41 smithi130 bash[51302]: debug 2023-08-28T01:49:41.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:41.982425+0000) 2023-08-28T01:49:42.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:41 smithi006 bash[27292]: cluster 2023-08-28T01:49:40.683373+0000 mgr.y (mgr.14901) 6932 : cluster [DBG] pgmap v6939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:42.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:41 smithi006 bash[33918]: cluster 2023-08-28T01:49:40.683373+0000 mgr.y (mgr.14901) 6932 : cluster [DBG] pgmap v6939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:49:42.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:49:42.696Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:43.324 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:42 smithi130 bash[51302]: debug 2023-08-28T01:49:42.977+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:42.982729+0000) 2023-08-28T01:49:43.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:43 smithi130 bash[27579]: audit 2023-08-28T01:49:42.822653+0000 mon.c (mon.2) 2725 : audit [DBG] from='client.? 172.21.15.6:0/1695774727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:43.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:43 smithi006 bash[27292]: audit 2023-08-28T01:49:42.822653+0000 mon.c (mon.2) 2725 : audit [DBG] from='client.? 172.21.15.6:0/1695774727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:43.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:43 smithi006 bash[33918]: audit 2023-08-28T01:49:42.822653+0000 mon.c (mon.2) 2725 : audit [DBG] from='client.? 172.21.15.6:0/1695774727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:44.327 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:43 smithi130 bash[51302]: debug 2023-08-28T01:49:43.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:43.983008+0000) 2023-08-28T01:49:44.580 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:44 smithi006 bash[27292]: cluster 2023-08-28T01:49:42.683982+0000 mgr.y (mgr.14901) 6933 : cluster [DBG] pgmap v6940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:44.580 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:44 smithi006 bash[33918]: cluster 2023-08-28T01:49:42.683982+0000 mgr.y (mgr.14901) 6933 : cluster [DBG] pgmap v6940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:44.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:44 smithi130 bash[27579]: cluster 2023-08-28T01:49:42.683982+0000 mgr.y (mgr.14901) 6933 : cluster [DBG] pgmap v6940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:49:44.580Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:44.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:49:44.582Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:45.335 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:44 smithi130 bash[51302]: debug 2023-08-28T01:49:44.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:44.983225+0000) 2023-08-28T01:49:45.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:45 smithi130 bash[27579]: audit 2023-08-28T01:49:45.237668+0000 mon.a (mon.0) 4532 : audit [DBG] from='client.? 172.21.15.6:0/3707569017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:45.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:45 smithi006 bash[33918]: audit 2023-08-28T01:49:45.237668+0000 mon.a (mon.0) 4532 : audit [DBG] from='client.? 172.21.15.6:0/3707569017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:45.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:45 smithi006 bash[27292]: audit 2023-08-28T01:49:45.237668+0000 mon.a (mon.0) 4532 : audit [DBG] from='client.? 172.21.15.6:0/3707569017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:46.338 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:45 smithi130 bash[51302]: debug 2023-08-28T01:49:45.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:45.983507+0000) 2023-08-28T01:49:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:46 smithi130 bash[27579]: cluster 2023-08-28T01:49:44.685093+0000 mgr.y (mgr.14901) 6934 : cluster [DBG] pgmap v6941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:46.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:46 smithi006 bash[33918]: cluster 2023-08-28T01:49:44.685093+0000 mgr.y (mgr.14901) 6934 : cluster [DBG] pgmap v6941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:46.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:46 smithi006 bash[27292]: cluster 2023-08-28T01:49:44.685093+0000 mgr.y (mgr.14901) 6934 : cluster [DBG] pgmap v6941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:47.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:46 smithi130 bash[51302]: debug 2023-08-28T01:49:46.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:46.983798+0000) 2023-08-28T01:49:48.343 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:47 smithi130 bash[51302]: debug 2023-08-28T01:49:47.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:47.984042+0000) 2023-08-28T01:49:48.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:48 smithi130 bash[27579]: cluster 2023-08-28T01:49:46.685699+0000 mgr.y (mgr.14901) 6935 : cluster [DBG] pgmap v6942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:48 smithi130 bash[27579]: audit 2023-08-28T01:49:47.654741+0000 mon.a (mon.0) 4533 : audit [DBG] from='client.? 172.21.15.6:0/3444162726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:48.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:48 smithi006 bash[33918]: cluster 2023-08-28T01:49:46.685699+0000 mgr.y (mgr.14901) 6935 : cluster [DBG] pgmap v6942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:48.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:48 smithi006 bash[33918]: audit 2023-08-28T01:49:47.654741+0000 mon.a (mon.0) 4533 : audit [DBG] from='client.? 172.21.15.6:0/3444162726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:48 smithi006 bash[27292]: cluster 2023-08-28T01:49:46.685699+0000 mgr.y (mgr.14901) 6935 : cluster [DBG] pgmap v6942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:48 smithi006 bash[27292]: audit 2023-08-28T01:49:47.654741+0000 mon.a (mon.0) 4533 : audit [DBG] from='client.? 172.21.15.6:0/3444162726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:49.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:48 smithi130 bash[51302]: debug 2023-08-28T01:49:48.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:48.984311+0000) 2023-08-28T01:49:50.348 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:49 smithi130 bash[51302]: debug 2023-08-28T01:49:49.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:49.984521+0000) 2023-08-28T01:49:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:50 smithi130 bash[27579]: cluster 2023-08-28T01:49:48.686352+0000 mgr.y (mgr.14901) 6936 : cluster [DBG] pgmap v6943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:50 smithi130 bash[27579]: audit 2023-08-28T01:49:50.083521+0000 mon.a (mon.0) 4534 : audit [DBG] from='client.? 172.21.15.6:0/3499176606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:50.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:50 smithi006 bash[33918]: cluster 2023-08-28T01:49:48.686352+0000 mgr.y (mgr.14901) 6936 : cluster [DBG] pgmap v6943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:50.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:50 smithi006 bash[33918]: audit 2023-08-28T01:49:50.083521+0000 mon.a (mon.0) 4534 : audit [DBG] from='client.? 172.21.15.6:0/3499176606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:50.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:50 smithi006 bash[27292]: cluster 2023-08-28T01:49:48.686352+0000 mgr.y (mgr.14901) 6936 : cluster [DBG] pgmap v6943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:50 smithi006 bash[27292]: audit 2023-08-28T01:49:50.083521+0000 mon.a (mon.0) 4534 : audit [DBG] from='client.? 172.21.15.6:0/3499176606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:51.130 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:49:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:49:50] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:49:51.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:50 smithi130 bash[51302]: debug 2023-08-28T01:49:50.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:50.984751+0000) 2023-08-28T01:49:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:49:51.130Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:49:51.132Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:52.353 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:51 smithi130 bash[51302]: debug 2023-08-28T01:49:51.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:51.984964+0000) 2023-08-28T01:49:52.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:52 smithi130 bash[27579]: cluster 2023-08-28T01:49:50.687433+0000 mgr.y (mgr.14901) 6937 : cluster [DBG] pgmap v6944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:52.694 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:52 smithi006 bash[33918]: cluster 2023-08-28T01:49:50.687433+0000 mgr.y (mgr.14901) 6937 : cluster [DBG] pgmap v6944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:52.694 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:52 smithi006 bash[27292]: cluster 2023-08-28T01:49:50.687433+0000 mgr.y (mgr.14901) 6937 : cluster [DBG] pgmap v6944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:49:52.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:49:52.696Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:53.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:52 smithi130 bash[51302]: debug 2023-08-28T01:49:52.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:52.985268+0000) 2023-08-28T01:49:53.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:53 smithi130 bash[27579]: audit 2023-08-28T01:49:52.499041+0000 mon.a (mon.0) 4535 : audit [DBG] from='client.? 172.21.15.6:0/2677229745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:53.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:53 smithi006 bash[33918]: audit 2023-08-28T01:49:52.499041+0000 mon.a (mon.0) 4535 : audit [DBG] from='client.? 172.21.15.6:0/2677229745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:53.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:53 smithi006 bash[27292]: audit 2023-08-28T01:49:52.499041+0000 mon.a (mon.0) 4535 : audit [DBG] from='client.? 172.21.15.6:0/2677229745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:54.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:53 smithi130 bash[51302]: debug 2023-08-28T01:49:53.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:53.985539+0000) 2023-08-28T01:49:54.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:54 smithi006 bash[27292]: cluster 2023-08-28T01:49:52.688109+0000 mgr.y (mgr.14901) 6938 : cluster [DBG] pgmap v6945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:54.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:49:54.581Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:49:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:49:54.583Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:49:54.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:54 smithi006 bash[33918]: cluster 2023-08-28T01:49:52.688109+0000 mgr.y (mgr.14901) 6938 : cluster [DBG] pgmap v6945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:54.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:54 smithi130 bash[27579]: cluster 2023-08-28T01:49:52.688109+0000 mgr.y (mgr.14901) 6938 : cluster [DBG] pgmap v6945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:55.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:54 smithi130 bash[51302]: debug 2023-08-28T01:49:54.985+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:54.985817+0000) 2023-08-28T01:49:55.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:55 smithi006 bash[27292]: audit 2023-08-28T01:49:54.908957+0000 mon.c (mon.2) 2726 : audit [DBG] from='client.? 172.21.15.6:0/3041760040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:55.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:55 smithi006 bash[33918]: audit 2023-08-28T01:49:54.908957+0000 mon.c (mon.2) 2726 : audit [DBG] from='client.? 172.21.15.6:0/3041760040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:55.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:55 smithi130 bash[27579]: audit 2023-08-28T01:49:54.908957+0000 mon.c (mon.2) 2726 : audit [DBG] from='client.? 172.21.15.6:0/3041760040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:56.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:55 smithi130 bash[51302]: debug 2023-08-28T01:49:55.985+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:55.986151+0000) 2023-08-28T01:49:56.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:56 smithi006 bash[27292]: cluster 2023-08-28T01:49:54.688851+0000 mgr.y (mgr.14901) 6939 : cluster [DBG] pgmap v6946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:56.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:56 smithi006 bash[33918]: cluster 2023-08-28T01:49:54.688851+0000 mgr.y (mgr.14901) 6939 : cluster [DBG] pgmap v6946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:56 smithi130 bash[27579]: cluster 2023-08-28T01:49:54.688851+0000 mgr.y (mgr.14901) 6939 : cluster [DBG] pgmap v6946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:57.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:56 smithi130 bash[51302]: debug 2023-08-28T01:49:56.985+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:56.986434+0000) 2023-08-28T01:49:58.322 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:57 smithi130 bash[51302]: debug 2023-08-28T01:49:57.981+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:57.986712+0000) 2023-08-28T01:49:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:58 smithi130 bash[27579]: cluster 2023-08-28T01:49:56.689531+0000 mgr.y (mgr.14901) 6940 : cluster [DBG] pgmap v6947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:49:58 smithi130 bash[27579]: audit 2023-08-28T01:49:57.322917+0000 mon.a (mon.0) 4536 : audit [DBG] from='client.? 172.21.15.6:0/67005795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:58.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:58 smithi006 bash[27292]: cluster 2023-08-28T01:49:56.689531+0000 mgr.y (mgr.14901) 6940 : cluster [DBG] pgmap v6947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:58.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:49:58 smithi006 bash[27292]: audit 2023-08-28T01:49:57.322917+0000 mon.a (mon.0) 4536 : audit [DBG] from='client.? 172.21.15.6:0/67005795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:58.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:58 smithi006 bash[33918]: cluster 2023-08-28T01:49:56.689531+0000 mgr.y (mgr.14901) 6940 : cluster [DBG] pgmap v6947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:49:58.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:49:58 smithi006 bash[33918]: audit 2023-08-28T01:49:57.322917+0000 mon.a (mon.0) 4536 : audit [DBG] from='client.? 172.21.15.6:0/67005795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:59.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:58 smithi130 bash[51302]: debug 2023-08-28T01:49:58.985+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:58.986967+0000) 2023-08-28T01:50:00.329 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:49:59 smithi130 bash[51302]: debug 2023-08-28T01:49:59.985+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:59.987191+0000) 2023-08-28T01:50:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:00 smithi130 bash[27579]: cluster 2023-08-28T01:49:58.690016+0000 mgr.y (mgr.14901) 6941 : cluster [DBG] pgmap v6948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:00 smithi130 bash[27579]: audit 2023-08-28T01:49:59.735274+0000 mon.c (mon.2) 2727 : audit [DBG] from='client.? 172.21.15.6:0/3021429712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:00 smithi130 bash[27579]: cluster 2023-08-28T01:50:00.000151+0000 mon.a (mon.0) 4537 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:50:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:00 smithi006 bash[33918]: cluster 2023-08-28T01:49:58.690016+0000 mgr.y (mgr.14901) 6941 : cluster [DBG] pgmap v6948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:00 smithi006 bash[33918]: audit 2023-08-28T01:49:59.735274+0000 mon.c (mon.2) 2727 : audit [DBG] from='client.? 172.21.15.6:0/3021429712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:00 smithi006 bash[33918]: cluster 2023-08-28T01:50:00.000151+0000 mon.a (mon.0) 4537 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:50:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:00 smithi006 bash[27292]: cluster 2023-08-28T01:49:58.690016+0000 mgr.y (mgr.14901) 6941 : cluster [DBG] pgmap v6948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:00 smithi006 bash[27292]: audit 2023-08-28T01:49:59.735274+0000 mon.c (mon.2) 2727 : audit [DBG] from='client.? 172.21.15.6:0/3021429712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:00 smithi006 bash[27292]: cluster 2023-08-28T01:50:00.000151+0000 mon.a (mon.0) 4537 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:50:01.131 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:50:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:50:00] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:50:01.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:00 smithi130 bash[51302]: debug 2023-08-28T01:50:00.985+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:00.987422+0000) 2023-08-28T01:50:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:50:01.132Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:50:01.133Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:02.334 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:01 smithi130 bash[51302]: debug 2023-08-28T01:50:01.985+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:01.987689+0000) 2023-08-28T01:50:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:02 smithi130 bash[27579]: cluster 2023-08-28T01:50:00.691126+0000 mgr.y (mgr.14901) 6942 : cluster [DBG] pgmap v6949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:02 smithi130 bash[27579]: audit 2023-08-28T01:50:02.151923+0000 mon.c (mon.2) 2728 : audit [DBG] from='client.? 172.21.15.6:0/3335152622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:02.695 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:02 smithi006 bash[27292]: cluster 2023-08-28T01:50:00.691126+0000 mgr.y (mgr.14901) 6942 : cluster [DBG] pgmap v6949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:02.695 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:02 smithi006 bash[27292]: audit 2023-08-28T01:50:02.151923+0000 mon.c (mon.2) 2728 : audit [DBG] from='client.? 172.21.15.6:0/3335152622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:02.696 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:02 smithi006 bash[33918]: cluster 2023-08-28T01:50:00.691126+0000 mgr.y (mgr.14901) 6942 : cluster [DBG] pgmap v6949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:02.696 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:02 smithi006 bash[33918]: audit 2023-08-28T01:50:02.151923+0000 mon.c (mon.2) 2728 : audit [DBG] from='client.? 172.21.15.6:0/3335152622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:50:02.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:50:02.697Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:03.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:02 smithi130 bash[51302]: debug 2023-08-28T01:50:02.985+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:02.988025+0000) 2023-08-28T01:50:04.337 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:03 smithi130 bash[51302]: debug 2023-08-28T01:50:03.985+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:03.988328+0000) 2023-08-28T01:50:04.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:04 smithi130 bash[27579]: cluster 2023-08-28T01:50:02.691852+0000 mgr.y (mgr.14901) 6943 : cluster [DBG] pgmap v6950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:04.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:04 smithi006 bash[33918]: cluster 2023-08-28T01:50:02.691852+0000 mgr.y (mgr.14901) 6943 : cluster [DBG] pgmap v6950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:04.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:04 smithi006 bash[27292]: cluster 2023-08-28T01:50:02.691852+0000 mgr.y (mgr.14901) 6943 : cluster [DBG] pgmap v6950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:50:04.582Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:50:04.584Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:05.345 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:04 smithi130 bash[51302]: debug 2023-08-28T01:50:04.985+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:04.988667+0000) 2023-08-28T01:50:05.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:05 smithi130 bash[27579]: audit 2023-08-28T01:50:04.573434+0000 mon.a (mon.0) 4538 : audit [DBG] from='client.? 172.21.15.6:0/3864520154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:05.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:05 smithi006 bash[33918]: audit 2023-08-28T01:50:04.573434+0000 mon.a (mon.0) 4538 : audit [DBG] from='client.? 172.21.15.6:0/3864520154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:05.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:05 smithi006 bash[27292]: audit 2023-08-28T01:50:04.573434+0000 mon.a (mon.0) 4538 : audit [DBG] from='client.? 172.21.15.6:0/3864520154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:06.348 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:05 smithi130 bash[51302]: debug 2023-08-28T01:50:05.985+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:05.988964+0000) 2023-08-28T01:50:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:06 smithi130 bash[27579]: cluster 2023-08-28T01:50:04.692983+0000 mgr.y (mgr.14901) 6944 : cluster [DBG] pgmap v6951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:06.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:06 smithi006 bash[33918]: cluster 2023-08-28T01:50:04.692983+0000 mgr.y (mgr.14901) 6944 : cluster [DBG] pgmap v6951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:06.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:06 smithi006 bash[27292]: cluster 2023-08-28T01:50:04.692983+0000 mgr.y (mgr.14901) 6944 : cluster [DBG] pgmap v6951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:07.355 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:06 smithi130 bash[51302]: debug 2023-08-28T01:50:06.989+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:06.989272+0000) 2023-08-28T01:50:07.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:07 smithi130 bash[27579]: audit 2023-08-28T01:50:06.985199+0000 mon.a (mon.0) 4539 : audit [DBG] from='client.? 172.21.15.6:0/2530979680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:07.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:07 smithi006 bash[33918]: audit 2023-08-28T01:50:06.985199+0000 mon.a (mon.0) 4539 : audit [DBG] from='client.? 172.21.15.6:0/2530979680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:07.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:07 smithi006 bash[27292]: audit 2023-08-28T01:50:06.985199+0000 mon.a (mon.0) 4539 : audit [DBG] from='client.? 172.21.15.6:0/2530979680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:08.358 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:07 smithi130 bash[51302]: debug 2023-08-28T01:50:07.985+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:07.989533+0000) 2023-08-28T01:50:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:08 smithi130 bash[27579]: cluster 2023-08-28T01:50:06.693557+0000 mgr.y (mgr.14901) 6945 : cluster [DBG] pgmap v6952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:08 smithi006 bash[33918]: cluster 2023-08-28T01:50:06.693557+0000 mgr.y (mgr.14901) 6945 : cluster [DBG] pgmap v6952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:08.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:08 smithi006 bash[27292]: cluster 2023-08-28T01:50:06.693557+0000 mgr.y (mgr.14901) 6945 : cluster [DBG] pgmap v6952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:09.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:08 smithi130 bash[51302]: debug 2023-08-28T01:50:08.989+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:08.989802+0000) 2023-08-28T01:50:10.363 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:09 smithi130 bash[51302]: debug 2023-08-28T01:50:09.989+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:09.990112+0000) 2023-08-28T01:50:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:10 smithi130 bash[27579]: cluster 2023-08-28T01:50:08.694318+0000 mgr.y (mgr.14901) 6946 : cluster [DBG] pgmap v6953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:10 smithi130 bash[27579]: audit 2023-08-28T01:50:09.398388+0000 mon.a (mon.0) 4540 : audit [DBG] from='client.? 172.21.15.6:0/2657002294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:10.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:10 smithi006 bash[33918]: cluster 2023-08-28T01:50:08.694318+0000 mgr.y (mgr.14901) 6946 : cluster [DBG] pgmap v6953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:10.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:10 smithi006 bash[33918]: audit 2023-08-28T01:50:09.398388+0000 mon.a (mon.0) 4540 : audit [DBG] from='client.? 172.21.15.6:0/2657002294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:10 smithi006 bash[27292]: cluster 2023-08-28T01:50:08.694318+0000 mgr.y (mgr.14901) 6946 : cluster [DBG] pgmap v6953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:10 smithi006 bash[27292]: audit 2023-08-28T01:50:09.398388+0000 mon.a (mon.0) 4540 : audit [DBG] from='client.? 172.21.15.6:0/2657002294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:11.132 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:50:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:50:10] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:50:11.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:10 smithi130 bash[51302]: debug 2023-08-28T01:50:10.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:10.990420+0000) 2023-08-28T01:50:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:50:11.132Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:50:11.134Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:12.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:11 smithi130 bash[51302]: debug 2023-08-28T01:50:11.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:11.990695+0000) 2023-08-28T01:50:12.696 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:12 smithi006 bash[27292]: cluster 2023-08-28T01:50:10.695484+0000 mgr.y (mgr.14901) 6947 : cluster [DBG] pgmap v6954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:12.696 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:12 smithi006 bash[27292]: audit 2023-08-28T01:50:11.815296+0000 mon.a (mon.0) 4541 : audit [DBG] from='client.? 172.21.15.6:0/3277149720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:12.696 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:12 smithi006 bash[33918]: cluster 2023-08-28T01:50:10.695484+0000 mgr.y (mgr.14901) 6947 : cluster [DBG] pgmap v6954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:12.697 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:12 smithi006 bash[33918]: audit 2023-08-28T01:50:11.815296+0000 mon.a (mon.0) 4541 : audit [DBG] from='client.? 172.21.15.6:0/3277149720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:12.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:12 smithi130 bash[27579]: cluster 2023-08-28T01:50:10.695484+0000 mgr.y (mgr.14901) 6947 : cluster [DBG] pgmap v6954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:12 smithi130 bash[27579]: audit 2023-08-28T01:50:11.815296+0000 mon.a (mon.0) 4541 : audit [DBG] from='client.? 172.21.15.6:0/3277149720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:50:12.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:50:12.698Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:13.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:12 smithi130 bash[51302]: debug 2023-08-28T01:50:12.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:12.991002+0000) 2023-08-28T01:50:14.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:13 smithi130 bash[51302]: debug 2023-08-28T01:50:13.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:13.991308+0000) 2023-08-28T01:50:14.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:14 smithi006 bash[27292]: cluster 2023-08-28T01:50:12.696167+0000 mgr.y (mgr.14901) 6948 : cluster [DBG] pgmap v6955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:14.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:14 smithi006 bash[27292]: audit 2023-08-28T01:50:14.233992+0000 mon.c (mon.2) 2729 : audit [DBG] from='client.? 172.21.15.6:0/1058975017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:50:14.583Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:50:14.585Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:14.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:14 smithi006 bash[33918]: cluster 2023-08-28T01:50:12.696167+0000 mgr.y (mgr.14901) 6948 : cluster [DBG] pgmap v6955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:14.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:14 smithi006 bash[33918]: audit 2023-08-28T01:50:14.233992+0000 mon.c (mon.2) 2729 : audit [DBG] from='client.? 172.21.15.6:0/1058975017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:14.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:14 smithi130 bash[27579]: cluster 2023-08-28T01:50:12.696167+0000 mgr.y (mgr.14901) 6948 : cluster [DBG] pgmap v6955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:14.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:14 smithi130 bash[27579]: audit 2023-08-28T01:50:14.233992+0000 mon.c (mon.2) 2729 : audit [DBG] from='client.? 172.21.15.6:0/1058975017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:15.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:14 smithi130 bash[51302]: debug 2023-08-28T01:50:14.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:14.991662+0000) 2023-08-28T01:50:16.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:15 smithi130 bash[51302]: debug 2023-08-28T01:50:15.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:15.991963+0000) 2023-08-28T01:50:16.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:16 smithi006 bash[27292]: cluster 2023-08-28T01:50:14.697348+0000 mgr.y (mgr.14901) 6949 : cluster [DBG] pgmap v6956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:16.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:16 smithi006 bash[33918]: cluster 2023-08-28T01:50:14.697348+0000 mgr.y (mgr.14901) 6949 : cluster [DBG] pgmap v6956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:16.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:16 smithi130 bash[27579]: cluster 2023-08-28T01:50:14.697348+0000 mgr.y (mgr.14901) 6949 : cluster [DBG] pgmap v6956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:17.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:16 smithi130 bash[51302]: debug 2023-08-28T01:50:16.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:16.992290+0000) 2023-08-28T01:50:17.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:17 smithi006 bash[33918]: audit 2023-08-28T01:50:16.656632+0000 mon.b (mon.1) 781 : audit [DBG] from='client.? 172.21.15.6:0/461627229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:17.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:17 smithi006 bash[27292]: audit 2023-08-28T01:50:16.656632+0000 mon.b (mon.1) 781 : audit [DBG] from='client.? 172.21.15.6:0/461627229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:17.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:17 smithi130 bash[27579]: audit 2023-08-28T01:50:16.656632+0000 mon.b (mon.1) 781 : audit [DBG] from='client.? 172.21.15.6:0/461627229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:18.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:17 smithi130 bash[51302]: debug 2023-08-28T01:50:17.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:17.992603+0000) 2023-08-28T01:50:18.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:18 smithi006 bash[33918]: cluster 2023-08-28T01:50:16.697956+0000 mgr.y (mgr.14901) 6950 : cluster [DBG] pgmap v6957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:18 smithi006 bash[27292]: cluster 2023-08-28T01:50:16.697956+0000 mgr.y (mgr.14901) 6950 : cluster [DBG] pgmap v6957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:18.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:18 smithi130 bash[27579]: cluster 2023-08-28T01:50:16.697956+0000 mgr.y (mgr.14901) 6950 : cluster [DBG] pgmap v6957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:19.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:18 smithi130 bash[51302]: debug 2023-08-28T01:50:18.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:18.992893+0000) 2023-08-28T01:50:19.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:19 smithi006 bash[27292]: audit 2023-08-28T01:50:19.069857+0000 mon.a (mon.0) 4542 : audit [DBG] from='client.? 172.21.15.6:0/2556857486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:19.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:19 smithi006 bash[33918]: audit 2023-08-28T01:50:19.069857+0000 mon.a (mon.0) 4542 : audit [DBG] from='client.? 172.21.15.6:0/2556857486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:19.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:19 smithi130 bash[27579]: audit 2023-08-28T01:50:19.069857+0000 mon.a (mon.0) 4542 : audit [DBG] from='client.? 172.21.15.6:0/2556857486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:20.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:19 smithi130 bash[51302]: debug 2023-08-28T01:50:19.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:19.993154+0000) 2023-08-28T01:50:20.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:20 smithi006 bash[33918]: cluster 2023-08-28T01:50:18.698559+0000 mgr.y (mgr.14901) 6951 : cluster [DBG] pgmap v6958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:20.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:20 smithi006 bash[27292]: cluster 2023-08-28T01:50:18.698559+0000 mgr.y (mgr.14901) 6951 : cluster [DBG] pgmap v6958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:20 smithi130 bash[27579]: cluster 2023-08-28T01:50:18.698559+0000 mgr.y (mgr.14901) 6951 : cluster [DBG] pgmap v6958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:21.133 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:50:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:50:20] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T01:50:21.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:20 smithi130 bash[51302]: debug 2023-08-28T01:50:20.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:20.993430+0000) 2023-08-28T01:50:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:50:21.133Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:50:21.136Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:22.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:21 smithi130 bash[51302]: debug 2023-08-28T01:50:21.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:21.993706+0000) 2023-08-28T01:50:22.696 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:22 smithi006 bash[33918]: cluster 2023-08-28T01:50:20.699561+0000 mgr.y (mgr.14901) 6952 : cluster [DBG] pgmap v6959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:22.697 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:22 smithi006 bash[33918]: audit 2023-08-28T01:50:21.489183+0000 mon.c (mon.2) 2730 : audit [DBG] from='client.? 172.21.15.6:0/3273066142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:22.697 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:22 smithi006 bash[27292]: cluster 2023-08-28T01:50:20.699561+0000 mgr.y (mgr.14901) 6952 : cluster [DBG] pgmap v6959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:22.697 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:22 smithi006 bash[27292]: audit 2023-08-28T01:50:21.489183+0000 mon.c (mon.2) 2730 : audit [DBG] from='client.? 172.21.15.6:0/3273066142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:22 smithi130 bash[27579]: cluster 2023-08-28T01:50:20.699561+0000 mgr.y (mgr.14901) 6952 : cluster [DBG] pgmap v6959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:22 smithi130 bash[27579]: audit 2023-08-28T01:50:21.489183+0000 mon.c (mon.2) 2730 : audit [DBG] from='client.? 172.21.15.6:0/3273066142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:50:22.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:50:22.698Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:23.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:22 smithi130 bash[51302]: debug 2023-08-28T01:50:22.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:22.994012+0000) 2023-08-28T01:50:24.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:23 smithi130 bash[51302]: debug 2023-08-28T01:50:23.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:23.994301+0000) 2023-08-28T01:50:24.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:24 smithi006 bash[27292]: cluster 2023-08-28T01:50:22.700180+0000 mgr.y (mgr.14901) 6953 : cluster [DBG] pgmap v6960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:24.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:24 smithi006 bash[27292]: audit 2023-08-28T01:50:23.910758+0000 mon.a (mon.0) 4543 : audit [DBG] from='client.? 172.21.15.6:0/3541024444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:24.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:24 smithi006 bash[33918]: cluster 2023-08-28T01:50:22.700180+0000 mgr.y (mgr.14901) 6953 : cluster [DBG] pgmap v6960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:24.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:24 smithi006 bash[33918]: audit 2023-08-28T01:50:23.910758+0000 mon.a (mon.0) 4543 : audit [DBG] from='client.? 172.21.15.6:0/3541024444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:50:24.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:50:24.586Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:24.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:24 smithi130 bash[27579]: cluster 2023-08-28T01:50:22.700180+0000 mgr.y (mgr.14901) 6953 : cluster [DBG] pgmap v6960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:24.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:24 smithi130 bash[27579]: audit 2023-08-28T01:50:23.910758+0000 mon.a (mon.0) 4543 : audit [DBG] from='client.? 172.21.15.6:0/3541024444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:25.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:24 smithi130 bash[51302]: debug 2023-08-28T01:50:24.988+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:24.994594+0000) 2023-08-28T01:50:26.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:25 smithi130 bash[51302]: debug 2023-08-28T01:50:25.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:25.994975+0000) 2023-08-28T01:50:26.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:26 smithi006 bash[27292]: cluster 2023-08-28T01:50:24.701337+0000 mgr.y (mgr.14901) 6954 : cluster [DBG] pgmap v6961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:26.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:26 smithi006 bash[27292]: audit 2023-08-28T01:50:26.325882+0000 mon.a (mon.0) 4544 : audit [DBG] from='client.? 172.21.15.6:0/2395932998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:26 smithi006 bash[33918]: cluster 2023-08-28T01:50:24.701337+0000 mgr.y (mgr.14901) 6954 : cluster [DBG] pgmap v6961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:26 smithi006 bash[33918]: audit 2023-08-28T01:50:26.325882+0000 mon.a (mon.0) 4544 : audit [DBG] from='client.? 172.21.15.6:0/2395932998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:26.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:26 smithi130 bash[27579]: cluster 2023-08-28T01:50:24.701337+0000 mgr.y (mgr.14901) 6954 : cluster [DBG] pgmap v6961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:26.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:26 smithi130 bash[27579]: audit 2023-08-28T01:50:26.325882+0000 mon.a (mon.0) 4544 : audit [DBG] from='client.? 172.21.15.6:0/2395932998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:27.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:26 smithi130 bash[51302]: debug 2023-08-28T01:50:26.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:26.995248+0000) 2023-08-28T01:50:28.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:27 smithi130 bash[51302]: debug 2023-08-28T01:50:27.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:27.995456+0000) 2023-08-28T01:50:28.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:28 smithi006 bash[27292]: cluster 2023-08-28T01:50:26.701990+0000 mgr.y (mgr.14901) 6955 : cluster [DBG] pgmap v6962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:28 smithi006 bash[33918]: cluster 2023-08-28T01:50:26.701990+0000 mgr.y (mgr.14901) 6955 : cluster [DBG] pgmap v6962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:28.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:28 smithi130 bash[27579]: cluster 2023-08-28T01:50:26.701990+0000 mgr.y (mgr.14901) 6955 : cluster [DBG] pgmap v6962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:29.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:28 smithi130 bash[51302]: debug 2023-08-28T01:50:28.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:28.995766+0000) 2023-08-28T01:50:29.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:29 smithi006 bash[27292]: audit 2023-08-28T01:50:28.742543+0000 mon.a (mon.0) 4545 : audit [DBG] from='client.? 172.21.15.6:0/1441124439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:29.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:29 smithi006 bash[33918]: audit 2023-08-28T01:50:28.742543+0000 mon.a (mon.0) 4545 : audit [DBG] from='client.? 172.21.15.6:0/1441124439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:29.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:29 smithi130 bash[27579]: audit 2023-08-28T01:50:28.742543+0000 mon.a (mon.0) 4545 : audit [DBG] from='client.? 172.21.15.6:0/1441124439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:30.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:29 smithi130 bash[51302]: debug 2023-08-28T01:50:29.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:29.996067+0000) 2023-08-28T01:50:30.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:30 smithi006 bash[27292]: cluster 2023-08-28T01:50:28.702682+0000 mgr.y (mgr.14901) 6956 : cluster [DBG] pgmap v6963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:30 smithi006 bash[33918]: cluster 2023-08-28T01:50:28.702682+0000 mgr.y (mgr.14901) 6956 : cluster [DBG] pgmap v6963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:30.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:30 smithi130 bash[27579]: cluster 2023-08-28T01:50:28.702682+0000 mgr.y (mgr.14901) 6956 : cluster [DBG] pgmap v6963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:31.134 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:50:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:50:30] "GET /metrics HTTP/1.1" 200 193115 "" "Prometheus/2.33.4" 2023-08-28T01:50:31.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:30 smithi130 bash[51302]: debug 2023-08-28T01:50:30.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:30.996358+0000) 2023-08-28T01:50:31.444 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:50:31.134Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:31.445 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:50:31.136Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:31.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:31 smithi006 bash[27292]: audit 2023-08-28T01:50:31.170364+0000 mon.a (mon.0) 4546 : audit [DBG] from='client.? 172.21.15.6:0/3312859855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:31.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:31 smithi006 bash[33918]: audit 2023-08-28T01:50:31.170364+0000 mon.a (mon.0) 4546 : audit [DBG] from='client.? 172.21.15.6:0/3312859855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:31.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:31 smithi130 bash[27579]: audit 2023-08-28T01:50:31.170364+0000 mon.a (mon.0) 4546 : audit [DBG] from='client.? 172.21.15.6:0/3312859855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:32.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:31 smithi130 bash[51302]: debug 2023-08-28T01:50:31.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:31.996654+0000) 2023-08-28T01:50:32.697 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:32 smithi006 bash[33918]: cluster 2023-08-28T01:50:30.703793+0000 mgr.y (mgr.14901) 6957 : cluster [DBG] pgmap v6964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:32.697 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:32 smithi006 bash[27292]: cluster 2023-08-28T01:50:30.703793+0000 mgr.y (mgr.14901) 6957 : cluster [DBG] pgmap v6964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:32 smithi130 bash[27579]: cluster 2023-08-28T01:50:30.703793+0000 mgr.y (mgr.14901) 6957 : cluster [DBG] pgmap v6964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:50:32.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:50:32.699Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:33.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:32 smithi130 bash[51302]: debug 2023-08-28T01:50:32.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:32.996933+0000) 2023-08-28T01:50:33.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:33 smithi006 bash[27292]: cluster 2023-08-28T01:50:32.704437+0000 mgr.y (mgr.14901) 6958 : cluster [DBG] pgmap v6965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:33.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:33 smithi006 bash[33918]: cluster 2023-08-28T01:50:32.704437+0000 mgr.y (mgr.14901) 6958 : cluster [DBG] pgmap v6965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:33.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:33 smithi130 bash[27579]: cluster 2023-08-28T01:50:32.704437+0000 mgr.y (mgr.14901) 6958 : cluster [DBG] pgmap v6965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:34.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:33 smithi130 bash[51302]: debug 2023-08-28T01:50:33.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:33.997213+0000) 2023-08-28T01:50:34.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:34 smithi006 bash[33918]: audit 2023-08-28T01:50:33.591544+0000 mon.c (mon.2) 2731 : audit [DBG] from='client.? 172.21.15.6:0/274118093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:34.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:34 smithi006 bash[27292]: audit 2023-08-28T01:50:33.591544+0000 mon.c (mon.2) 2731 : audit [DBG] from='client.? 172.21.15.6:0/274118093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:50:34.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:50:34.586Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:34.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:34 smithi130 bash[27579]: audit 2023-08-28T01:50:33.591544+0000 mon.c (mon.2) 2731 : audit [DBG] from='client.? 172.21.15.6:0/274118093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:35.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:34 smithi130 bash[51302]: debug 2023-08-28T01:50:34.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:34.997504+0000) 2023-08-28T01:50:35.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:35 smithi130 bash[27579]: cluster 2023-08-28T01:50:34.705618+0000 mgr.y (mgr.14901) 6959 : cluster [DBG] pgmap v6966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:35.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:35 smithi006 bash[33918]: cluster 2023-08-28T01:50:34.705618+0000 mgr.y (mgr.14901) 6959 : cluster [DBG] pgmap v6966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:35.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:35 smithi006 bash[27292]: cluster 2023-08-28T01:50:34.705618+0000 mgr.y (mgr.14901) 6959 : cluster [DBG] pgmap v6966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:36.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:35 smithi130 bash[51302]: debug 2023-08-28T01:50:35.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:35.997781+0000) 2023-08-28T01:50:36.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:36 smithi130 bash[27579]: audit 2023-08-28T01:50:35.999856+0000 mon.c (mon.2) 2732 : audit [DBG] from='client.? 172.21.15.6:0/3125906656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:36.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:36 smithi006 bash[27292]: audit 2023-08-28T01:50:35.999856+0000 mon.c (mon.2) 2732 : audit [DBG] from='client.? 172.21.15.6:0/3125906656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:36.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:36 smithi006 bash[33918]: audit 2023-08-28T01:50:35.999856+0000 mon.c (mon.2) 2732 : audit [DBG] from='client.? 172.21.15.6:0/3125906656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:37.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:36 smithi130 bash[51302]: debug 2023-08-28T01:50:36.992+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:36.998042+0000) 2023-08-28T01:50:38.332 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:37 smithi130 bash[51302]: debug 2023-08-28T01:50:37.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:37.998318+0000) 2023-08-28T01:50:38.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:38 smithi130 bash[27579]: cluster 2023-08-28T01:50:36.706379+0000 mgr.y (mgr.14901) 6960 : cluster [DBG] pgmap v6967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:38.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:38 smithi006 bash[27292]: cluster 2023-08-28T01:50:36.706379+0000 mgr.y (mgr.14901) 6960 : cluster [DBG] pgmap v6967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:38 smithi006 bash[33918]: cluster 2023-08-28T01:50:36.706379+0000 mgr.y (mgr.14901) 6960 : cluster [DBG] pgmap v6967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:39.341 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:38 smithi130 bash[51302]: debug 2023-08-28T01:50:38.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:38.998609+0000) 2023-08-28T01:50:39.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:39 smithi130 bash[27579]: audit 2023-08-28T01:50:38.411036+0000 mon.a (mon.0) 4547 : audit [DBG] from='client.? 172.21.15.6:0/129704314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:39.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:39 smithi006 bash[33918]: audit 2023-08-28T01:50:38.411036+0000 mon.a (mon.0) 4547 : audit [DBG] from='client.? 172.21.15.6:0/129704314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:39.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:39 smithi006 bash[27292]: audit 2023-08-28T01:50:38.411036+0000 mon.a (mon.0) 4547 : audit [DBG] from='client.? 172.21.15.6:0/129704314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:40.343 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:39 smithi130 bash[51302]: debug 2023-08-28T01:50:39.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:39.998841+0000) 2023-08-28T01:50:40.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:40 smithi130 bash[27579]: cluster 2023-08-28T01:50:38.707149+0000 mgr.y (mgr.14901) 6961 : cluster [DBG] pgmap v6968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:40.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:40 smithi130 bash[27579]: audit 2023-08-28T01:50:39.566517+0000 mon.b (mon.1) 782 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:50:40.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:40 smithi130 bash[27579]: audit 2023-08-28T01:50:39.841766+0000 mon.a (mon.0) 4548 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:50:40.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:40 smithi130 bash[27579]: audit 2023-08-28T01:50:39.850948+0000 mon.a (mon.0) 4549 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:50:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:40 smithi006 bash[27292]: cluster 2023-08-28T01:50:38.707149+0000 mgr.y (mgr.14901) 6961 : cluster [DBG] pgmap v6968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:40 smithi006 bash[27292]: audit 2023-08-28T01:50:39.566517+0000 mon.b (mon.1) 782 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:50:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:40 smithi006 bash[27292]: audit 2023-08-28T01:50:39.841766+0000 mon.a (mon.0) 4548 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:50:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:40 smithi006 bash[27292]: audit 2023-08-28T01:50:39.850948+0000 mon.a (mon.0) 4549 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:50:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:40 smithi006 bash[33918]: cluster 2023-08-28T01:50:38.707149+0000 mgr.y (mgr.14901) 6961 : cluster [DBG] pgmap v6968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:40 smithi006 bash[33918]: audit 2023-08-28T01:50:39.566517+0000 mon.b (mon.1) 782 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:50:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:40 smithi006 bash[33918]: audit 2023-08-28T01:50:39.841766+0000 mon.a (mon.0) 4548 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:50:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:40 smithi006 bash[33918]: audit 2023-08-28T01:50:39.850948+0000 mon.a (mon.0) 4549 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:50:41.135 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:50:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:50:40] "GET /metrics HTTP/1.1" 200 193115 "" "Prometheus/2.33.4" 2023-08-28T01:50:41.350 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:40 smithi130 bash[51302]: debug 2023-08-28T01:50:40.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:40.999102+0000) 2023-08-28T01:50:41.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:41 smithi006 bash[27292]: audit 2023-08-28T01:50:40.832382+0000 mon.a (mon.0) 4550 : audit [DBG] from='client.? 172.21.15.6:0/3222054241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:41.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:41 smithi006 bash[33918]: audit 2023-08-28T01:50:40.832382+0000 mon.a (mon.0) 4550 : audit [DBG] from='client.? 172.21.15.6:0/3222054241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:50:41.135Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:50:41.137Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:41.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:41 smithi130 bash[27579]: audit 2023-08-28T01:50:40.832382+0000 mon.a (mon.0) 4550 : audit [DBG] from='client.? 172.21.15.6:0/3222054241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:42.355 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:41 smithi130 bash[51302]: debug 2023-08-28T01:50:41.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:41.999322+0000) 2023-08-28T01:50:42.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:42 smithi130 bash[27579]: cluster 2023-08-28T01:50:40.708271+0000 mgr.y (mgr.14901) 6962 : cluster [DBG] pgmap v6969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:42.698 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:42 smithi006 bash[33918]: cluster 2023-08-28T01:50:40.708271+0000 mgr.y (mgr.14901) 6962 : cluster [DBG] pgmap v6969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:42.698 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:42 smithi006 bash[27292]: cluster 2023-08-28T01:50:40.708271+0000 mgr.y (mgr.14901) 6962 : cluster [DBG] pgmap v6969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:50:42.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:50:42.700Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:43.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:42 smithi130 bash[51302]: debug 2023-08-28T01:50:42.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:42.999616+0000) 2023-08-28T01:50:43.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:43 smithi006 bash[33918]: audit 2023-08-28T01:50:43.247594+0000 mon.a (mon.0) 4551 : audit [DBG] from='client.? 172.21.15.6:0/1868869793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:43.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:43 smithi006 bash[27292]: audit 2023-08-28T01:50:43.247594+0000 mon.a (mon.0) 4551 : audit [DBG] from='client.? 172.21.15.6:0/1868869793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:43.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:43 smithi130 bash[27579]: audit 2023-08-28T01:50:43.247594+0000 mon.a (mon.0) 4551 : audit [DBG] from='client.? 172.21.15.6:0/1868869793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:44.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:44 smithi130 bash[51302]: debug 2023-08-28T01:50:43.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:43.999886+0000) 2023-08-28T01:50:44.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:44 smithi006 bash[33918]: cluster 2023-08-28T01:50:42.708956+0000 mgr.y (mgr.14901) 6963 : cluster [DBG] pgmap v6970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:44.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:50:44.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:44.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:50:44.586Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:44.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:44 smithi006 bash[27292]: cluster 2023-08-28T01:50:42.708956+0000 mgr.y (mgr.14901) 6963 : cluster [DBG] pgmap v6970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:44.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:44 smithi130 bash[27579]: cluster 2023-08-28T01:50:42.708956+0000 mgr.y (mgr.14901) 6963 : cluster [DBG] pgmap v6970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:45.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:45 smithi130 bash[51302]: debug 2023-08-28T01:50:44.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:45.000191+0000) 2023-08-28T01:50:46.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:46 smithi130 bash[51302]: debug 2023-08-28T01:50:45.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:46.000497+0000) 2023-08-28T01:50:46.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:46 smithi006 bash[33918]: cluster 2023-08-28T01:50:44.710020+0000 mgr.y (mgr.14901) 6964 : cluster [DBG] pgmap v6971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:46.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:46 smithi006 bash[33918]: audit 2023-08-28T01:50:45.727314+0000 mon.c (mon.2) 2733 : audit [DBG] from='client.? 172.21.15.6:0/1650274171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:46.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:46 smithi006 bash[33918]: audit 2023-08-28T01:50:45.741076+0000 mon.a (mon.0) 4552 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:50:46.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:46 smithi006 bash[33918]: audit 2023-08-28T01:50:46.172510+0000 mon.a (mon.0) 4553 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:50:46.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:46 smithi006 bash[27292]: cluster 2023-08-28T01:50:44.710020+0000 mgr.y (mgr.14901) 6964 : cluster [DBG] pgmap v6971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:46.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:46 smithi006 bash[27292]: audit 2023-08-28T01:50:45.727314+0000 mon.c (mon.2) 2733 : audit [DBG] from='client.? 172.21.15.6:0/1650274171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:46.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:46 smithi006 bash[27292]: audit 2023-08-28T01:50:45.741076+0000 mon.a (mon.0) 4552 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:50:46.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:46 smithi006 bash[27292]: audit 2023-08-28T01:50:46.172510+0000 mon.a (mon.0) 4553 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:50:46.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:46 smithi130 bash[27579]: cluster 2023-08-28T01:50:44.710020+0000 mgr.y (mgr.14901) 6964 : cluster [DBG] pgmap v6971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:46.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:46 smithi130 bash[27579]: audit 2023-08-28T01:50:45.727314+0000 mon.c (mon.2) 2733 : audit [DBG] from='client.? 172.21.15.6:0/1650274171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:46.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:46 smithi130 bash[27579]: audit 2023-08-28T01:50:45.741076+0000 mon.a (mon.0) 4552 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:50:46.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:46 smithi130 bash[27579]: audit 2023-08-28T01:50:46.172510+0000 mon.a (mon.0) 4553 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:50:47.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:47 smithi130 bash[51302]: debug 2023-08-28T01:50:46.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:47.000826+0000) 2023-08-28T01:50:47.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:47 smithi006 bash[33918]: audit 2023-08-28T01:50:46.442632+0000 mon.b (mon.1) 783 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:50:47.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:47 smithi006 bash[33918]: audit 2023-08-28T01:50:46.444674+0000 mon.b (mon.1) 784 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:50:47.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:47 smithi006 bash[33918]: audit 2023-08-28T01:50:46.451864+0000 mon.a (mon.0) 4554 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:50:47.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:47 smithi006 bash[27292]: audit 2023-08-28T01:50:46.442632+0000 mon.b (mon.1) 783 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:50:47.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:47 smithi006 bash[27292]: audit 2023-08-28T01:50:46.444674+0000 mon.b (mon.1) 784 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:50:47.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:47 smithi006 bash[27292]: audit 2023-08-28T01:50:46.451864+0000 mon.a (mon.0) 4554 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:50:47.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:47 smithi130 bash[27579]: audit 2023-08-28T01:50:46.442632+0000 mon.b (mon.1) 783 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:50:47.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:47 smithi130 bash[27579]: audit 2023-08-28T01:50:46.444674+0000 mon.b (mon.1) 784 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:50:47.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:47 smithi130 bash[27579]: audit 2023-08-28T01:50:46.451864+0000 mon.a (mon.0) 4554 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:50:48.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:48 smithi130 bash[51302]: debug 2023-08-28T01:50:47.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:48.001181+0000) 2023-08-28T01:50:48.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:48 smithi006 bash[27292]: cluster 2023-08-28T01:50:46.710700+0000 mgr.y (mgr.14901) 6965 : cluster [DBG] pgmap v6972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:48.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:48 smithi006 bash[27292]: audit 2023-08-28T01:50:48.139120+0000 mon.a (mon.0) 4555 : audit [DBG] from='client.? 172.21.15.6:0/122640137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:48.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:48 smithi006 bash[33918]: cluster 2023-08-28T01:50:46.710700+0000 mgr.y (mgr.14901) 6965 : cluster [DBG] pgmap v6972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:48.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:48 smithi006 bash[33918]: audit 2023-08-28T01:50:48.139120+0000 mon.a (mon.0) 4555 : audit [DBG] from='client.? 172.21.15.6:0/122640137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:48.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:48 smithi130 bash[27579]: cluster 2023-08-28T01:50:46.710700+0000 mgr.y (mgr.14901) 6965 : cluster [DBG] pgmap v6972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:48.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:48 smithi130 bash[27579]: audit 2023-08-28T01:50:48.139120+0000 mon.a (mon.0) 4555 : audit [DBG] from='client.? 172.21.15.6:0/122640137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:49.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:49 smithi130 bash[51302]: debug 2023-08-28T01:50:48.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:49.001524+0000) 2023-08-28T01:50:50.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:50 smithi130 bash[51302]: debug 2023-08-28T01:50:49.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:50.001838+0000) 2023-08-28T01:50:50.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:50 smithi006 bash[27292]: cluster 2023-08-28T01:50:48.711372+0000 mgr.y (mgr.14901) 6966 : cluster [DBG] pgmap v6973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:50.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:50 smithi006 bash[33918]: cluster 2023-08-28T01:50:48.711372+0000 mgr.y (mgr.14901) 6966 : cluster [DBG] pgmap v6973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:50.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:50 smithi130 bash[27579]: cluster 2023-08-28T01:50:48.711372+0000 mgr.y (mgr.14901) 6966 : cluster [DBG] pgmap v6973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:51.135 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:50:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:50:50] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T01:50:51.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:51 smithi130 bash[51302]: debug 2023-08-28T01:50:50.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:51.002182+0000) 2023-08-28T01:50:51.390 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:50:51.135Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:51.390 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:50:51.139Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:51.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:51 smithi006 bash[33918]: audit 2023-08-28T01:50:50.562816+0000 mon.a (mon.0) 4556 : audit [DBG] from='client.? 172.21.15.6:0/2850541445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:51.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:51 smithi006 bash[27292]: audit 2023-08-28T01:50:50.562816+0000 mon.a (mon.0) 4556 : audit [DBG] from='client.? 172.21.15.6:0/2850541445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:51.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:51 smithi130 bash[27579]: audit 2023-08-28T01:50:50.562816+0000 mon.a (mon.0) 4556 : audit [DBG] from='client.? 172.21.15.6:0/2850541445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:52.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:52 smithi130 bash[51302]: debug 2023-08-28T01:50:51.996+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:52.002541+0000) 2023-08-28T01:50:52.698 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:52 smithi006 bash[33918]: cluster 2023-08-28T01:50:50.712525+0000 mgr.y (mgr.14901) 6967 : cluster [DBG] pgmap v6974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:52.698 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:52 smithi006 bash[27292]: cluster 2023-08-28T01:50:50.712525+0000 mgr.y (mgr.14901) 6967 : cluster [DBG] pgmap v6974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:52.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:52 smithi130 bash[27579]: cluster 2023-08-28T01:50:50.712525+0000 mgr.y (mgr.14901) 6967 : cluster [DBG] pgmap v6974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:50:52.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:50:52.700Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:53.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:53 smithi130 bash[51302]: debug 2023-08-28T01:50:53.000+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:53.002905+0000) 2023-08-28T01:50:53.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:53 smithi006 bash[27292]: audit 2023-08-28T01:50:52.983062+0000 mon.c (mon.2) 2734 : audit [DBG] from='client.? 172.21.15.6:0/4172819244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:53.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:53 smithi006 bash[33918]: audit 2023-08-28T01:50:52.983062+0000 mon.c (mon.2) 2734 : audit [DBG] from='client.? 172.21.15.6:0/4172819244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:53.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:53 smithi130 bash[27579]: audit 2023-08-28T01:50:52.983062+0000 mon.c (mon.2) 2734 : audit [DBG] from='client.? 172.21.15.6:0/4172819244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:54.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:54 smithi130 bash[51302]: debug 2023-08-28T01:50:54.000+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:54.003255+0000) 2023-08-28T01:50:54.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:54 smithi006 bash[33918]: cluster 2023-08-28T01:50:52.713077+0000 mgr.y (mgr.14901) 6968 : cluster [DBG] pgmap v6975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:50:54.585Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:50:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:50:54.587Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:50:54.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:54 smithi006 bash[27292]: cluster 2023-08-28T01:50:52.713077+0000 mgr.y (mgr.14901) 6968 : cluster [DBG] pgmap v6975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:54.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:54 smithi130 bash[27579]: cluster 2023-08-28T01:50:52.713077+0000 mgr.y (mgr.14901) 6968 : cluster [DBG] pgmap v6975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:55.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:55 smithi130 bash[51302]: debug 2023-08-28T01:50:55.000+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:55.003600+0000) 2023-08-28T01:50:55.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:55 smithi006 bash[33918]: audit 2023-08-28T01:50:55.399377+0000 mon.a (mon.0) 4557 : audit [DBG] from='client.? 172.21.15.6:0/701735793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:55.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:55 smithi006 bash[27292]: audit 2023-08-28T01:50:55.399377+0000 mon.a (mon.0) 4557 : audit [DBG] from='client.? 172.21.15.6:0/701735793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:55.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:55 smithi130 bash[27579]: audit 2023-08-28T01:50:55.399377+0000 mon.a (mon.0) 4557 : audit [DBG] from='client.? 172.21.15.6:0/701735793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:56.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:56 smithi130 bash[51302]: debug 2023-08-28T01:50:56.000+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:56.003958+0000) 2023-08-28T01:50:56.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:56 smithi006 bash[33918]: cluster 2023-08-28T01:50:54.714164+0000 mgr.y (mgr.14901) 6969 : cluster [DBG] pgmap v6976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:56.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:56 smithi006 bash[27292]: cluster 2023-08-28T01:50:54.714164+0000 mgr.y (mgr.14901) 6969 : cluster [DBG] pgmap v6976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:56 smithi130 bash[27579]: cluster 2023-08-28T01:50:54.714164+0000 mgr.y (mgr.14901) 6969 : cluster [DBG] pgmap v6976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:57.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:57 smithi130 bash[51302]: debug 2023-08-28T01:50:57.000+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:57.004274+0000) 2023-08-28T01:50:58.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:58 smithi130 bash[51302]: debug 2023-08-28T01:50:58.004+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:58.004590+0000) 2023-08-28T01:50:58.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:58 smithi006 bash[27292]: cluster 2023-08-28T01:50:56.714947+0000 mgr.y (mgr.14901) 6970 : cluster [DBG] pgmap v6977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:58.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:50:58 smithi006 bash[27292]: audit 2023-08-28T01:50:57.807979+0000 mon.c (mon.2) 2735 : audit [DBG] from='client.? 172.21.15.6:0/1326448366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:58.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:58 smithi006 bash[33918]: cluster 2023-08-28T01:50:56.714947+0000 mgr.y (mgr.14901) 6970 : cluster [DBG] pgmap v6977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:58.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:50:58 smithi006 bash[33918]: audit 2023-08-28T01:50:57.807979+0000 mon.c (mon.2) 2735 : audit [DBG] from='client.? 172.21.15.6:0/1326448366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:58.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:58 smithi130 bash[27579]: cluster 2023-08-28T01:50:56.714947+0000 mgr.y (mgr.14901) 6970 : cluster [DBG] pgmap v6977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:50:58.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:50:58 smithi130 bash[27579]: audit 2023-08-28T01:50:57.807979+0000 mon.c (mon.2) 2735 : audit [DBG] from='client.? 172.21.15.6:0/1326448366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:59.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:50:59 smithi130 bash[51302]: debug 2023-08-28T01:50:59.004+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:59.004925+0000) 2023-08-28T01:51:00.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:00 smithi130 bash[51302]: debug 2023-08-28T01:51:00.000+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:00.005287+0000) 2023-08-28T01:51:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:00 smithi006 bash[27292]: cluster 2023-08-28T01:50:58.715691+0000 mgr.y (mgr.14901) 6971 : cluster [DBG] pgmap v6978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:00 smithi006 bash[27292]: audit 2023-08-28T01:51:00.225814+0000 mon.c (mon.2) 2736 : audit [DBG] from='client.? 172.21.15.6:0/3625128433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:00 smithi006 bash[33918]: cluster 2023-08-28T01:50:58.715691+0000 mgr.y (mgr.14901) 6971 : cluster [DBG] pgmap v6978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:00 smithi006 bash[33918]: audit 2023-08-28T01:51:00.225814+0000 mon.c (mon.2) 2736 : audit [DBG] from='client.? 172.21.15.6:0/3625128433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:00 smithi130 bash[27579]: cluster 2023-08-28T01:50:58.715691+0000 mgr.y (mgr.14901) 6971 : cluster [DBG] pgmap v6978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:00 smithi130 bash[27579]: audit 2023-08-28T01:51:00.225814+0000 mon.c (mon.2) 2736 : audit [DBG] from='client.? 172.21.15.6:0/3625128433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:01.135 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:51:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:51:00] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:51:01.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:01 smithi130 bash[51302]: debug 2023-08-28T01:51:01.000+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:01.005661+0000) 2023-08-28T01:51:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:51:01.135Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:51:01.138Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:02.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:02 smithi130 bash[51302]: debug 2023-08-28T01:51:02.004+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:02.006006+0000) 2023-08-28T01:51:02.699 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:02 smithi006 bash[27292]: cluster 2023-08-28T01:51:00.716744+0000 mgr.y (mgr.14901) 6972 : cluster [DBG] pgmap v6979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:02.700 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:02 smithi006 bash[33918]: cluster 2023-08-28T01:51:00.716744+0000 mgr.y (mgr.14901) 6972 : cluster [DBG] pgmap v6979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:02.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:02 smithi130 bash[27579]: cluster 2023-08-28T01:51:00.716744+0000 mgr.y (mgr.14901) 6972 : cluster [DBG] pgmap v6979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:51:02.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:51:02.702Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:03.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:03 smithi130 bash[51302]: debug 2023-08-28T01:51:03.004+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:03.006365+0000) 2023-08-28T01:51:03.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:03 smithi006 bash[27292]: audit 2023-08-28T01:51:02.644284+0000 mon.c (mon.2) 2737 : audit [DBG] from='client.? 172.21.15.6:0/1135895864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:03.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:03 smithi006 bash[33918]: audit 2023-08-28T01:51:02.644284+0000 mon.c (mon.2) 2737 : audit [DBG] from='client.? 172.21.15.6:0/1135895864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:03.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:03 smithi130 bash[27579]: audit 2023-08-28T01:51:02.644284+0000 mon.c (mon.2) 2737 : audit [DBG] from='client.? 172.21.15.6:0/1135895864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:04.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:04 smithi130 bash[51302]: debug 2023-08-28T01:51:04.004+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:04.006729+0000) 2023-08-28T01:51:04.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:04 smithi006 bash[27292]: cluster 2023-08-28T01:51:02.717423+0000 mgr.y (mgr.14901) 6973 : cluster [DBG] pgmap v6980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:04.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:04 smithi006 bash[33918]: cluster 2023-08-28T01:51:02.717423+0000 mgr.y (mgr.14901) 6973 : cluster [DBG] pgmap v6980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:51:04.586Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:04.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:51:04.588Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:04.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:04 smithi130 bash[27579]: cluster 2023-08-28T01:51:02.717423+0000 mgr.y (mgr.14901) 6973 : cluster [DBG] pgmap v6980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:05.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:05 smithi130 bash[51302]: debug 2023-08-28T01:51:05.004+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:05.007083+0000) 2023-08-28T01:51:05.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:05 smithi006 bash[27292]: cluster 2023-08-28T01:51:04.718590+0000 mgr.y (mgr.14901) 6974 : cluster [DBG] pgmap v6981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:05.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:05 smithi006 bash[27292]: audit 2023-08-28T01:51:05.056083+0000 mon.a (mon.0) 4558 : audit [DBG] from='client.? 172.21.15.6:0/1624682304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:05.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:05 smithi006 bash[33918]: cluster 2023-08-28T01:51:04.718590+0000 mgr.y (mgr.14901) 6974 : cluster [DBG] pgmap v6981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:05.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:05 smithi006 bash[33918]: audit 2023-08-28T01:51:05.056083+0000 mon.a (mon.0) 4558 : audit [DBG] from='client.? 172.21.15.6:0/1624682304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:05.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:05 smithi130 bash[27579]: cluster 2023-08-28T01:51:04.718590+0000 mgr.y (mgr.14901) 6974 : cluster [DBG] pgmap v6981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:05.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:05 smithi130 bash[27579]: audit 2023-08-28T01:51:05.056083+0000 mon.a (mon.0) 4558 : audit [DBG] from='client.? 172.21.15.6:0/1624682304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:06.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:06 smithi130 bash[51302]: debug 2023-08-28T01:51:06.004+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:06.007445+0000) 2023-08-28T01:51:07.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:07 smithi130 bash[51302]: debug 2023-08-28T01:51:07.003+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:07.007715+0000) 2023-08-28T01:51:08.355 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:08 smithi130 bash[51302]: debug 2023-08-28T01:51:08.003+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:08.008053+0000) 2023-08-28T01:51:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:08 smithi130 bash[27579]: cluster 2023-08-28T01:51:06.719279+0000 mgr.y (mgr.14901) 6975 : cluster [DBG] pgmap v6982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:08 smithi130 bash[27579]: audit 2023-08-28T01:51:07.474010+0000 mon.a (mon.0) 4559 : audit [DBG] from='client.? 172.21.15.6:0/3361650458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:08 smithi006 bash[33918]: cluster 2023-08-28T01:51:06.719279+0000 mgr.y (mgr.14901) 6975 : cluster [DBG] pgmap v6982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:08 smithi006 bash[33918]: audit 2023-08-28T01:51:07.474010+0000 mon.a (mon.0) 4559 : audit [DBG] from='client.? 172.21.15.6:0/3361650458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:08.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:08 smithi006 bash[27292]: cluster 2023-08-28T01:51:06.719279+0000 mgr.y (mgr.14901) 6975 : cluster [DBG] pgmap v6982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:08 smithi006 bash[27292]: audit 2023-08-28T01:51:07.474010+0000 mon.a (mon.0) 4559 : audit [DBG] from='client.? 172.21.15.6:0/3361650458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:09.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:09 smithi130 bash[51302]: debug 2023-08-28T01:51:09.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:09.008387+0000) 2023-08-28T01:51:10.361 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:10 smithi130 bash[51302]: debug 2023-08-28T01:51:10.003+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:10.008720+0000) 2023-08-28T01:51:10.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:10 smithi130 bash[27579]: cluster 2023-08-28T01:51:08.719936+0000 mgr.y (mgr.14901) 6976 : cluster [DBG] pgmap v6983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:10 smithi130 bash[27579]: audit 2023-08-28T01:51:09.895811+0000 mon.c (mon.2) 2738 : audit [DBG] from='client.? 172.21.15.6:0/479925837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:10 smithi006 bash[33918]: cluster 2023-08-28T01:51:08.719936+0000 mgr.y (mgr.14901) 6976 : cluster [DBG] pgmap v6983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:10 smithi006 bash[33918]: audit 2023-08-28T01:51:09.895811+0000 mon.c (mon.2) 2738 : audit [DBG] from='client.? 172.21.15.6:0/479925837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:10 smithi006 bash[27292]: cluster 2023-08-28T01:51:08.719936+0000 mgr.y (mgr.14901) 6976 : cluster [DBG] pgmap v6983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:10 smithi006 bash[27292]: audit 2023-08-28T01:51:09.895811+0000 mon.c (mon.2) 2738 : audit [DBG] from='client.? 172.21.15.6:0/479925837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:11.136 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:51:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:51:10] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:51:11.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:11 smithi130 bash[51302]: debug 2023-08-28T01:51:11.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:11.009066+0000) 2023-08-28T01:51:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:51:11.137Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:51:11.139Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:12.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:12 smithi130 bash[51302]: debug 2023-08-28T01:51:12.003+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:12.009405+0000) 2023-08-28T01:51:12.700 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:12 smithi006 bash[27292]: cluster 2023-08-28T01:51:10.721019+0000 mgr.y (mgr.14901) 6977 : cluster [DBG] pgmap v6984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:12.700 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:12 smithi006 bash[27292]: audit 2023-08-28T01:51:12.318058+0000 mon.a (mon.0) 4560 : audit [DBG] from='client.? 172.21.15.6:0/3342581059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:12.701 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:12 smithi006 bash[33918]: cluster 2023-08-28T01:51:10.721019+0000 mgr.y (mgr.14901) 6977 : cluster [DBG] pgmap v6984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:12.701 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:12 smithi006 bash[33918]: audit 2023-08-28T01:51:12.318058+0000 mon.a (mon.0) 4560 : audit [DBG] from='client.? 172.21.15.6:0/3342581059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:12 smithi130 bash[27579]: cluster 2023-08-28T01:51:10.721019+0000 mgr.y (mgr.14901) 6977 : cluster [DBG] pgmap v6984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:12 smithi130 bash[27579]: audit 2023-08-28T01:51:12.318058+0000 mon.a (mon.0) 4560 : audit [DBG] from='client.? 172.21.15.6:0/3342581059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:51:12.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:51:12.702Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:13.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:13 smithi130 bash[51302]: debug 2023-08-28T01:51:13.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:13.009739+0000) 2023-08-28T01:51:14.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:14 smithi130 bash[51302]: debug 2023-08-28T01:51:14.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:14.010104+0000) 2023-08-28T01:51:14.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:14 smithi006 bash[27292]: cluster 2023-08-28T01:51:12.721686+0000 mgr.y (mgr.14901) 6978 : cluster [DBG] pgmap v6985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:14.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:14 smithi006 bash[33918]: cluster 2023-08-28T01:51:12.721686+0000 mgr.y (mgr.14901) 6978 : cluster [DBG] pgmap v6985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:51:14.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:51:14.589Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:14.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:14 smithi130 bash[27579]: cluster 2023-08-28T01:51:12.721686+0000 mgr.y (mgr.14901) 6978 : cluster [DBG] pgmap v6985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:15.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:15 smithi130 bash[51302]: debug 2023-08-28T01:51:15.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:15.010437+0000) 2023-08-28T01:51:15.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:15 smithi006 bash[33918]: audit 2023-08-28T01:51:14.727992+0000 mon.a (mon.0) 4561 : audit [DBG] from='client.? 172.21.15.6:0/2659936686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:15.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:15 smithi006 bash[27292]: audit 2023-08-28T01:51:14.727992+0000 mon.a (mon.0) 4561 : audit [DBG] from='client.? 172.21.15.6:0/2659936686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:15.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:15 smithi130 bash[27579]: audit 2023-08-28T01:51:14.727992+0000 mon.a (mon.0) 4561 : audit [DBG] from='client.? 172.21.15.6:0/2659936686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:16.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:16 smithi130 bash[51302]: debug 2023-08-28T01:51:16.003+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:16.010764+0000) 2023-08-28T01:51:16.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:16 smithi006 bash[33918]: cluster 2023-08-28T01:51:14.722435+0000 mgr.y (mgr.14901) 6979 : cluster [DBG] pgmap v6986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:16.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:16 smithi006 bash[27292]: cluster 2023-08-28T01:51:14.722435+0000 mgr.y (mgr.14901) 6979 : cluster [DBG] pgmap v6986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:16.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:16 smithi130 bash[27579]: cluster 2023-08-28T01:51:14.722435+0000 mgr.y (mgr.14901) 6979 : cluster [DBG] pgmap v6986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:17.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:17 smithi130 bash[51302]: debug 2023-08-28T01:51:17.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:17.011106+0000) 2023-08-28T01:51:17.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:17 smithi006 bash[33918]: audit 2023-08-28T01:51:17.144733+0000 mon.a (mon.0) 4562 : audit [DBG] from='client.? 172.21.15.6:0/1350514853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:17.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:17 smithi006 bash[27292]: audit 2023-08-28T01:51:17.144733+0000 mon.a (mon.0) 4562 : audit [DBG] from='client.? 172.21.15.6:0/1350514853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:17.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:17 smithi130 bash[27579]: audit 2023-08-28T01:51:17.144733+0000 mon.a (mon.0) 4562 : audit [DBG] from='client.? 172.21.15.6:0/1350514853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:18.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:18 smithi130 bash[51302]: debug 2023-08-28T01:51:18.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:18.011443+0000) 2023-08-28T01:51:18.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:18 smithi006 bash[33918]: cluster 2023-08-28T01:51:16.723040+0000 mgr.y (mgr.14901) 6980 : cluster [DBG] pgmap v6987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:18 smithi006 bash[27292]: cluster 2023-08-28T01:51:16.723040+0000 mgr.y (mgr.14901) 6980 : cluster [DBG] pgmap v6987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:18.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:18 smithi130 bash[27579]: cluster 2023-08-28T01:51:16.723040+0000 mgr.y (mgr.14901) 6980 : cluster [DBG] pgmap v6987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:19.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:19 smithi130 bash[51302]: debug 2023-08-28T01:51:19.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:19.011792+0000) 2023-08-28T01:51:20.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:20 smithi130 bash[51302]: debug 2023-08-28T01:51:20.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:20.012115+0000) 2023-08-28T01:51:20.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:20 smithi006 bash[27292]: cluster 2023-08-28T01:51:18.723771+0000 mgr.y (mgr.14901) 6981 : cluster [DBG] pgmap v6988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:20.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:20 smithi006 bash[27292]: audit 2023-08-28T01:51:19.559824+0000 mon.a (mon.0) 4563 : audit [DBG] from='client.? 172.21.15.6:0/698279295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:20 smithi006 bash[33918]: cluster 2023-08-28T01:51:18.723771+0000 mgr.y (mgr.14901) 6981 : cluster [DBG] pgmap v6988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:20 smithi006 bash[33918]: audit 2023-08-28T01:51:19.559824+0000 mon.a (mon.0) 4563 : audit [DBG] from='client.? 172.21.15.6:0/698279295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:20.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:20 smithi130 bash[27579]: cluster 2023-08-28T01:51:18.723771+0000 mgr.y (mgr.14901) 6981 : cluster [DBG] pgmap v6988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:20 smithi130 bash[27579]: audit 2023-08-28T01:51:19.559824+0000 mon.a (mon.0) 4563 : audit [DBG] from='client.? 172.21.15.6:0/698279295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:21.138 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:51:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:51:20] "GET /metrics HTTP/1.1" 200 193115 "" "Prometheus/2.33.4" 2023-08-28T01:51:21.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:21 smithi130 bash[51302]: debug 2023-08-28T01:51:21.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:21.012470+0000) 2023-08-28T01:51:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:51:21.138Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:51:21.140Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:22.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:22 smithi130 bash[51302]: debug 2023-08-28T01:51:22.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:22.012809+0000) 2023-08-28T01:51:22.700 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:22 smithi006 bash[27292]: cluster 2023-08-28T01:51:20.724882+0000 mgr.y (mgr.14901) 6982 : cluster [DBG] pgmap v6989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:22.701 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:22 smithi006 bash[27292]: audit 2023-08-28T01:51:21.971128+0000 mon.c (mon.2) 2739 : audit [DBG] from='client.? 172.21.15.6:0/3520422350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:22.701 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:22 smithi006 bash[33918]: cluster 2023-08-28T01:51:20.724882+0000 mgr.y (mgr.14901) 6982 : cluster [DBG] pgmap v6989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:22.701 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:22 smithi006 bash[33918]: audit 2023-08-28T01:51:21.971128+0000 mon.c (mon.2) 2739 : audit [DBG] from='client.? 172.21.15.6:0/3520422350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:22.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:22 smithi130 bash[27579]: cluster 2023-08-28T01:51:20.724882+0000 mgr.y (mgr.14901) 6982 : cluster [DBG] pgmap v6989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:22 smithi130 bash[27579]: audit 2023-08-28T01:51:21.971128+0000 mon.c (mon.2) 2739 : audit [DBG] from='client.? 172.21.15.6:0/3520422350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:51:22.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:51:22.702Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:23.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:23 smithi130 bash[51302]: debug 2023-08-28T01:51:23.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:23.013142+0000) 2023-08-28T01:51:24.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:24 smithi130 bash[51302]: debug 2023-08-28T01:51:24.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:24.013487+0000) 2023-08-28T01:51:24.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:24 smithi006 bash[27292]: cluster 2023-08-28T01:51:22.725493+0000 mgr.y (mgr.14901) 6983 : cluster [DBG] pgmap v6990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:24.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:24 smithi006 bash[27292]: audit 2023-08-28T01:51:24.379372+0000 mon.a (mon.0) 4564 : audit [DBG] from='client.? 172.21.15.6:0/3625428548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:24.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:24 smithi006 bash[33918]: cluster 2023-08-28T01:51:22.725493+0000 mgr.y (mgr.14901) 6983 : cluster [DBG] pgmap v6990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:24.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:24 smithi006 bash[33918]: audit 2023-08-28T01:51:24.379372+0000 mon.a (mon.0) 4564 : audit [DBG] from='client.? 172.21.15.6:0/3625428548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:51:24.587Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:51:24.589Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:24.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:24 smithi130 bash[27579]: cluster 2023-08-28T01:51:22.725493+0000 mgr.y (mgr.14901) 6983 : cluster [DBG] pgmap v6990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:24.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:24 smithi130 bash[27579]: audit 2023-08-28T01:51:24.379372+0000 mon.a (mon.0) 4564 : audit [DBG] from='client.? 172.21.15.6:0/3625428548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:25.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:25 smithi130 bash[51302]: debug 2023-08-28T01:51:25.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:25.013856+0000) 2023-08-28T01:51:26.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:26 smithi130 bash[51302]: debug 2023-08-28T01:51:26.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:26.014198+0000) 2023-08-28T01:51:26.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:26 smithi006 bash[33918]: cluster 2023-08-28T01:51:24.726836+0000 mgr.y (mgr.14901) 6984 : cluster [DBG] pgmap v6991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:26.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:26 smithi006 bash[27292]: cluster 2023-08-28T01:51:24.726836+0000 mgr.y (mgr.14901) 6984 : cluster [DBG] pgmap v6991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:26.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:26 smithi130 bash[27579]: cluster 2023-08-28T01:51:24.726836+0000 mgr.y (mgr.14901) 6984 : cluster [DBG] pgmap v6991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:27.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:27 smithi130 bash[51302]: debug 2023-08-28T01:51:27.007+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:27.014555+0000) 2023-08-28T01:51:27.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:27 smithi006 bash[27292]: audit 2023-08-28T01:51:26.797158+0000 mon.c (mon.2) 2740 : audit [DBG] from='client.? 172.21.15.6:0/2901239608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:27.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:27 smithi006 bash[33918]: audit 2023-08-28T01:51:26.797158+0000 mon.c (mon.2) 2740 : audit [DBG] from='client.? 172.21.15.6:0/2901239608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:27.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:27 smithi130 bash[27579]: audit 2023-08-28T01:51:26.797158+0000 mon.c (mon.2) 2740 : audit [DBG] from='client.? 172.21.15.6:0/2901239608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:28.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:28 smithi130 bash[51302]: debug 2023-08-28T01:51:28.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:28.014897+0000) 2023-08-28T01:51:28.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:28 smithi006 bash[27292]: cluster 2023-08-28T01:51:26.727415+0000 mgr.y (mgr.14901) 6985 : cluster [DBG] pgmap v6992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:28.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:28 smithi006 bash[33918]: cluster 2023-08-28T01:51:26.727415+0000 mgr.y (mgr.14901) 6985 : cluster [DBG] pgmap v6992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:28.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:28 smithi130 bash[27579]: cluster 2023-08-28T01:51:26.727415+0000 mgr.y (mgr.14901) 6985 : cluster [DBG] pgmap v6992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:29.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:29 smithi130 bash[51302]: debug 2023-08-28T01:51:29.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:29.015247+0000) 2023-08-28T01:51:29.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:29 smithi006 bash[27292]: audit 2023-08-28T01:51:29.208810+0000 mon.a (mon.0) 4565 : audit [DBG] from='client.? 172.21.15.6:0/3691247651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:29.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:29 smithi006 bash[33918]: audit 2023-08-28T01:51:29.208810+0000 mon.a (mon.0) 4565 : audit [DBG] from='client.? 172.21.15.6:0/3691247651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:29.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:29 smithi130 bash[27579]: audit 2023-08-28T01:51:29.208810+0000 mon.a (mon.0) 4565 : audit [DBG] from='client.? 172.21.15.6:0/3691247651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:30.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:30 smithi130 bash[51302]: debug 2023-08-28T01:51:30.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:30.015616+0000) 2023-08-28T01:51:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:30 smithi006 bash[33918]: cluster 2023-08-28T01:51:28.728056+0000 mgr.y (mgr.14901) 6986 : cluster [DBG] pgmap v6993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:30.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:30 smithi006 bash[27292]: cluster 2023-08-28T01:51:28.728056+0000 mgr.y (mgr.14901) 6986 : cluster [DBG] pgmap v6993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:30 smithi130 bash[27579]: cluster 2023-08-28T01:51:28.728056+0000 mgr.y (mgr.14901) 6986 : cluster [DBG] pgmap v6993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:31.139 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:51:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:51:30] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T01:51:31.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:31 smithi130 bash[51302]: debug 2023-08-28T01:51:31.015+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:31.015967+0000) 2023-08-28T01:51:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:51:31.139Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:51:31.141Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:32.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:32 smithi130 bash[51302]: debug 2023-08-28T01:51:32.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:32.016317+0000) 2023-08-28T01:51:32.701 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:32 smithi006 bash[33918]: cluster 2023-08-28T01:51:30.729344+0000 mgr.y (mgr.14901) 6987 : cluster [DBG] pgmap v6994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:32.702 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:32 smithi006 bash[33918]: audit 2023-08-28T01:51:31.632786+0000 mon.c (mon.2) 2741 : audit [DBG] from='client.? 172.21.15.6:0/3393681326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:32.702 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:32 smithi006 bash[27292]: cluster 2023-08-28T01:51:30.729344+0000 mgr.y (mgr.14901) 6987 : cluster [DBG] pgmap v6994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:32.702 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:32 smithi006 bash[27292]: audit 2023-08-28T01:51:31.632786+0000 mon.c (mon.2) 2741 : audit [DBG] from='client.? 172.21.15.6:0/3393681326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:32.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:32 smithi130 bash[27579]: cluster 2023-08-28T01:51:30.729344+0000 mgr.y (mgr.14901) 6987 : cluster [DBG] pgmap v6994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:32 smithi130 bash[27579]: audit 2023-08-28T01:51:31.632786+0000 mon.c (mon.2) 2741 : audit [DBG] from='client.? 172.21.15.6:0/3393681326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:51:32.702Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:51:32.704Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:33.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:33 smithi130 bash[51302]: debug 2023-08-28T01:51:33.015+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:33.016646+0000) 2023-08-28T01:51:34.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:34 smithi130 bash[51302]: debug 2023-08-28T01:51:34.015+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:34.016987+0000) 2023-08-28T01:51:34.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:34 smithi006 bash[33918]: cluster 2023-08-28T01:51:32.730018+0000 mgr.y (mgr.14901) 6988 : cluster [DBG] pgmap v6995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:34.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:34 smithi006 bash[33918]: audit 2023-08-28T01:51:34.050331+0000 mon.a (mon.0) 4566 : audit [DBG] from='client.? 172.21.15.6:0/2764234865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:51:34.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:51:34.590Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:34.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:34 smithi006 bash[27292]: cluster 2023-08-28T01:51:32.730018+0000 mgr.y (mgr.14901) 6988 : cluster [DBG] pgmap v6995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:34.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:34 smithi006 bash[27292]: audit 2023-08-28T01:51:34.050331+0000 mon.a (mon.0) 4566 : audit [DBG] from='client.? 172.21.15.6:0/2764234865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:34.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:34 smithi130 bash[27579]: cluster 2023-08-28T01:51:32.730018+0000 mgr.y (mgr.14901) 6988 : cluster [DBG] pgmap v6995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:34.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:34 smithi130 bash[27579]: audit 2023-08-28T01:51:34.050331+0000 mon.a (mon.0) 4566 : audit [DBG] from='client.? 172.21.15.6:0/2764234865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:35.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:35 smithi130 bash[51302]: debug 2023-08-28T01:51:35.015+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:35.017342+0000) 2023-08-28T01:51:35.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:35 smithi130 bash[27579]: cluster 2023-08-28T01:51:34.731250+0000 mgr.y (mgr.14901) 6989 : cluster [DBG] pgmap v6996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:35.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:35 smithi006 bash[27292]: cluster 2023-08-28T01:51:34.731250+0000 mgr.y (mgr.14901) 6989 : cluster [DBG] pgmap v6996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:35.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:35 smithi006 bash[33918]: cluster 2023-08-28T01:51:34.731250+0000 mgr.y (mgr.14901) 6989 : cluster [DBG] pgmap v6996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:36.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:36 smithi130 bash[51302]: debug 2023-08-28T01:51:36.015+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:36.017713+0000) 2023-08-28T01:51:36.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:36 smithi130 bash[27579]: audit 2023-08-28T01:51:36.468452+0000 mon.c (mon.2) 2742 : audit [DBG] from='client.? 172.21.15.6:0/110831704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:36.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:36 smithi006 bash[33918]: audit 2023-08-28T01:51:36.468452+0000 mon.c (mon.2) 2742 : audit [DBG] from='client.? 172.21.15.6:0/110831704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:36.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:36 smithi006 bash[27292]: audit 2023-08-28T01:51:36.468452+0000 mon.c (mon.2) 2742 : audit [DBG] from='client.? 172.21.15.6:0/110831704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:37.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:37 smithi130 bash[51302]: debug 2023-08-28T01:51:37.011+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:37.018084+0000) 2023-08-28T01:51:38.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:38 smithi130 bash[51302]: debug 2023-08-28T01:51:38.015+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:38.018429+0000) 2023-08-28T01:51:38.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:38 smithi130 bash[27579]: cluster 2023-08-28T01:51:36.731980+0000 mgr.y (mgr.14901) 6990 : cluster [DBG] pgmap v6997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:38.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:38 smithi006 bash[27292]: cluster 2023-08-28T01:51:36.731980+0000 mgr.y (mgr.14901) 6990 : cluster [DBG] pgmap v6997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:38 smithi006 bash[33918]: cluster 2023-08-28T01:51:36.731980+0000 mgr.y (mgr.14901) 6990 : cluster [DBG] pgmap v6997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:39.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:39 smithi130 bash[51302]: debug 2023-08-28T01:51:39.015+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:39.018759+0000) 2023-08-28T01:51:39.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:39 smithi006 bash[27292]: audit 2023-08-28T01:51:38.884942+0000 mon.c (mon.2) 2743 : audit [DBG] from='client.? 172.21.15.6:0/1279684763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:39.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:39 smithi006 bash[33918]: audit 2023-08-28T01:51:38.884942+0000 mon.c (mon.2) 2743 : audit [DBG] from='client.? 172.21.15.6:0/1279684763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:39.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:39 smithi130 bash[27579]: audit 2023-08-28T01:51:38.884942+0000 mon.c (mon.2) 2743 : audit [DBG] from='client.? 172.21.15.6:0/1279684763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:40.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:40 smithi130 bash[51302]: debug 2023-08-28T01:51:40.015+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:40.019083+0000) 2023-08-28T01:51:40.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:40 smithi006 bash[27292]: cluster 2023-08-28T01:51:38.732625+0000 mgr.y (mgr.14901) 6991 : cluster [DBG] pgmap v6998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:40 smithi006 bash[33918]: cluster 2023-08-28T01:51:38.732625+0000 mgr.y (mgr.14901) 6991 : cluster [DBG] pgmap v6998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:40.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:40 smithi130 bash[27579]: cluster 2023-08-28T01:51:38.732625+0000 mgr.y (mgr.14901) 6991 : cluster [DBG] pgmap v6998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:41.140 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:51:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:51:40] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T01:51:41.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:41 smithi130 bash[51302]: debug 2023-08-28T01:51:41.015+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:41.019372+0000) 2023-08-28T01:51:41.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:41 smithi006 bash[27292]: audit 2023-08-28T01:51:41.305490+0000 mon.c (mon.2) 2744 : audit [DBG] from='client.? 172.21.15.6:0/2171374374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:41.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:41 smithi006 bash[33918]: audit 2023-08-28T01:51:41.305490+0000 mon.c (mon.2) 2744 : audit [DBG] from='client.? 172.21.15.6:0/2171374374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:51:41.140Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:51:41.142Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:41.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:41 smithi130 bash[27579]: audit 2023-08-28T01:51:41.305490+0000 mon.c (mon.2) 2744 : audit [DBG] from='client.? 172.21.15.6:0/2171374374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:42.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:42 smithi130 bash[51302]: debug 2023-08-28T01:51:42.015+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:42.019649+0000) 2023-08-28T01:51:42.703 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:42 smithi006 bash[27292]: cluster 2023-08-28T01:51:40.733748+0000 mgr.y (mgr.14901) 6992 : cluster [DBG] pgmap v6999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:42.703 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:42 smithi006 bash[33918]: cluster 2023-08-28T01:51:40.733748+0000 mgr.y (mgr.14901) 6992 : cluster [DBG] pgmap v6999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:42.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:42 smithi130 bash[27579]: cluster 2023-08-28T01:51:40.733748+0000 mgr.y (mgr.14901) 6992 : cluster [DBG] pgmap v6999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:51:42.702Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:51:42.705Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:43.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:43 smithi130 bash[51302]: debug 2023-08-28T01:51:43.019+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:43.020014+0000) 2023-08-28T01:51:44.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:44 smithi130 bash[51302]: debug 2023-08-28T01:51:44.019+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:44.020347+0000) 2023-08-28T01:51:44.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:44 smithi006 bash[33918]: cluster 2023-08-28T01:51:42.734425+0000 mgr.y (mgr.14901) 6993 : cluster [DBG] pgmap v7000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:44.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:44 smithi006 bash[33918]: audit 2023-08-28T01:51:43.760176+0000 mon.c (mon.2) 2745 : audit [DBG] from='client.? 172.21.15.6:0/3735922922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:51:44.589Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:51:44.591Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:44.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:44 smithi006 bash[27292]: cluster 2023-08-28T01:51:42.734425+0000 mgr.y (mgr.14901) 6993 : cluster [DBG] pgmap v7000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:44.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:44 smithi006 bash[27292]: audit 2023-08-28T01:51:43.760176+0000 mon.c (mon.2) 2745 : audit [DBG] from='client.? 172.21.15.6:0/3735922922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:44.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:44 smithi130 bash[27579]: cluster 2023-08-28T01:51:42.734425+0000 mgr.y (mgr.14901) 6993 : cluster [DBG] pgmap v7000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:44.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:44 smithi130 bash[27579]: audit 2023-08-28T01:51:43.760176+0000 mon.c (mon.2) 2745 : audit [DBG] from='client.? 172.21.15.6:0/3735922922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:45.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:45 smithi130 bash[51302]: debug 2023-08-28T01:51:45.019+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:45.020703+0000) 2023-08-28T01:51:46.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:46 smithi130 bash[51302]: debug 2023-08-28T01:51:46.015+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:46.021052+0000) 2023-08-28T01:51:46.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:46 smithi006 bash[27292]: cluster 2023-08-28T01:51:44.735566+0000 mgr.y (mgr.14901) 6994 : cluster [DBG] pgmap v7001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:46.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:46 smithi006 bash[27292]: audit 2023-08-28T01:51:46.172275+0000 mon.c (mon.2) 2746 : audit [DBG] from='client.? 172.21.15.6:0/1506425845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:46.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:46 smithi006 bash[33918]: cluster 2023-08-28T01:51:44.735566+0000 mgr.y (mgr.14901) 6994 : cluster [DBG] pgmap v7001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:46.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:46 smithi006 bash[33918]: audit 2023-08-28T01:51:46.172275+0000 mon.c (mon.2) 2746 : audit [DBG] from='client.? 172.21.15.6:0/1506425845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:46.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:46 smithi130 bash[27579]: cluster 2023-08-28T01:51:44.735566+0000 mgr.y (mgr.14901) 6994 : cluster [DBG] pgmap v7001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:46.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:46 smithi130 bash[27579]: audit 2023-08-28T01:51:46.172275+0000 mon.c (mon.2) 2746 : audit [DBG] from='client.? 172.21.15.6:0/1506425845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:47.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:47 smithi130 bash[51302]: debug 2023-08-28T01:51:47.019+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:47.021416+0000) 2023-08-28T01:51:47.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:47 smithi006 bash[27292]: audit 2023-08-28T01:51:46.465083+0000 mon.b (mon.1) 785 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:51:47.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:47 smithi006 bash[33918]: audit 2023-08-28T01:51:46.465083+0000 mon.b (mon.1) 785 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:51:47.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:47 smithi130 bash[27579]: audit 2023-08-28T01:51:46.465083+0000 mon.b (mon.1) 785 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:51:48.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:48 smithi130 bash[51302]: debug 2023-08-28T01:51:48.019+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:48.021749+0000) 2023-08-28T01:51:48.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:48 smithi006 bash[27292]: cluster 2023-08-28T01:51:46.736202+0000 mgr.y (mgr.14901) 6995 : cluster [DBG] pgmap v7002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:48.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:48 smithi006 bash[33918]: cluster 2023-08-28T01:51:46.736202+0000 mgr.y (mgr.14901) 6995 : cluster [DBG] pgmap v7002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:48.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:48 smithi130 bash[27579]: cluster 2023-08-28T01:51:46.736202+0000 mgr.y (mgr.14901) 6995 : cluster [DBG] pgmap v7002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:49.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:49 smithi130 bash[51302]: debug 2023-08-28T01:51:49.015+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:49.022074+0000) 2023-08-28T01:51:49.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:49 smithi006 bash[27292]: audit 2023-08-28T01:51:48.592219+0000 mon.a (mon.0) 4567 : audit [DBG] from='client.? 172.21.15.6:0/2460929690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:49.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:49 smithi006 bash[33918]: audit 2023-08-28T01:51:48.592219+0000 mon.a (mon.0) 4567 : audit [DBG] from='client.? 172.21.15.6:0/2460929690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:49.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:49 smithi130 bash[27579]: audit 2023-08-28T01:51:48.592219+0000 mon.a (mon.0) 4567 : audit [DBG] from='client.? 172.21.15.6:0/2460929690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:50.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:50 smithi130 bash[51302]: debug 2023-08-28T01:51:50.015+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:50.022433+0000) 2023-08-28T01:51:50.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:50 smithi006 bash[27292]: cluster 2023-08-28T01:51:48.736721+0000 mgr.y (mgr.14901) 6996 : cluster [DBG] pgmap v7003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:50.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:50 smithi006 bash[33918]: cluster 2023-08-28T01:51:48.736721+0000 mgr.y (mgr.14901) 6996 : cluster [DBG] pgmap v7003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:50.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:50 smithi130 bash[27579]: cluster 2023-08-28T01:51:48.736721+0000 mgr.y (mgr.14901) 6996 : cluster [DBG] pgmap v7003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:51.140 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:51:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:51:50] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:51:51.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:51 smithi130 bash[51302]: debug 2023-08-28T01:51:51.019+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:51.022803+0000) 2023-08-28T01:51:51.426 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:51:51.141Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:51.426 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:51:51.143Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:51.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:51 smithi006 bash[33918]: audit 2023-08-28T01:51:51.021138+0000 mon.c (mon.2) 2747 : audit [DBG] from='client.? 172.21.15.6:0/102187463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:51.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:51 smithi006 bash[27292]: audit 2023-08-28T01:51:51.021138+0000 mon.c (mon.2) 2747 : audit [DBG] from='client.? 172.21.15.6:0/102187463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:51.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:51 smithi130 bash[27579]: audit 2023-08-28T01:51:51.021138+0000 mon.c (mon.2) 2747 : audit [DBG] from='client.? 172.21.15.6:0/102187463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:52.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:52 smithi130 bash[51302]: debug 2023-08-28T01:51:52.019+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:52.023092+0000) 2023-08-28T01:51:52.703 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:52 smithi006 bash[33918]: cluster 2023-08-28T01:51:50.737759+0000 mgr.y (mgr.14901) 6997 : cluster [DBG] pgmap v7004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:52.704 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:52 smithi006 bash[33918]: audit 2023-08-28T01:51:52.373575+0000 mon.a (mon.0) 4568 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:51:52.704 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:52 smithi006 bash[27292]: cluster 2023-08-28T01:51:50.737759+0000 mgr.y (mgr.14901) 6997 : cluster [DBG] pgmap v7004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:52.704 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:52 smithi006 bash[27292]: audit 2023-08-28T01:51:52.373575+0000 mon.a (mon.0) 4568 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:51:52.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:52 smithi130 bash[27579]: cluster 2023-08-28T01:51:50.737759+0000 mgr.y (mgr.14901) 6997 : cluster [DBG] pgmap v7004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:52.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:52 smithi130 bash[27579]: audit 2023-08-28T01:51:52.373575+0000 mon.a (mon.0) 4568 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:51:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:51:52.703Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:51:52.706Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:53.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:53 smithi130 bash[51302]: debug 2023-08-28T01:51:53.019+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:53.023289+0000) 2023-08-28T01:51:54.021 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:53 smithi130 bash[27579]: cluster 2023-08-28T01:51:52.738422+0000 mgr.y (mgr.14901) 6998 : cluster [DBG] pgmap v7005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:54.022 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:53 smithi130 bash[27579]: audit 2023-08-28T01:51:52.751166+0000 mon.a (mon.0) 4569 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:51:54.022 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:53 smithi130 bash[27579]: audit 2023-08-28T01:51:53.435279+0000 mon.a (mon.0) 4570 : audit [DBG] from='client.? 172.21.15.6:0/3077516845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:54.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:53 smithi006 bash[27292]: cluster 2023-08-28T01:51:52.738422+0000 mgr.y (mgr.14901) 6998 : cluster [DBG] pgmap v7005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:54.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:53 smithi006 bash[27292]: audit 2023-08-28T01:51:52.751166+0000 mon.a (mon.0) 4569 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:51:54.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:53 smithi006 bash[27292]: audit 2023-08-28T01:51:53.435279+0000 mon.a (mon.0) 4570 : audit [DBG] from='client.? 172.21.15.6:0/3077516845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:54.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:53 smithi006 bash[33918]: cluster 2023-08-28T01:51:52.738422+0000 mgr.y (mgr.14901) 6998 : cluster [DBG] pgmap v7005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:54.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:53 smithi006 bash[33918]: audit 2023-08-28T01:51:52.751166+0000 mon.a (mon.0) 4569 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:51:54.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:53 smithi006 bash[33918]: audit 2023-08-28T01:51:53.435279+0000 mon.a (mon.0) 4570 : audit [DBG] from='client.? 172.21.15.6:0/3077516845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:54.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:54 smithi130 bash[51302]: debug 2023-08-28T01:51:54.019+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:54.023621+0000) 2023-08-28T01:51:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:51:54.589Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:51:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:51:54.591Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:51:55.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:55 smithi130 bash[51302]: debug 2023-08-28T01:51:55.023+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:55.023947+0000) 2023-08-28T01:51:56.022 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:55 smithi130 bash[27579]: cluster 2023-08-28T01:51:54.739561+0000 mgr.y (mgr.14901) 6999 : cluster [DBG] pgmap v7006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:56.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:55 smithi006 bash[33918]: cluster 2023-08-28T01:51:54.739561+0000 mgr.y (mgr.14901) 6999 : cluster [DBG] pgmap v7006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:56.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:55 smithi006 bash[27292]: cluster 2023-08-28T01:51:54.739561+0000 mgr.y (mgr.14901) 6999 : cluster [DBG] pgmap v7006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:56.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:56 smithi130 bash[51302]: debug 2023-08-28T01:51:56.019+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:56.024251+0000) 2023-08-28T01:51:57.022 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:56 smithi130 bash[27579]: audit 2023-08-28T01:51:55.862062+0000 mon.a (mon.0) 4571 : audit [DBG] from='client.? 172.21.15.6:0/3921570008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:57.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:56 smithi006 bash[33918]: audit 2023-08-28T01:51:55.862062+0000 mon.a (mon.0) 4571 : audit [DBG] from='client.? 172.21.15.6:0/3921570008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:57.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:56 smithi006 bash[27292]: audit 2023-08-28T01:51:55.862062+0000 mon.a (mon.0) 4571 : audit [DBG] from='client.? 172.21.15.6:0/3921570008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:57.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:57 smithi130 bash[51302]: debug 2023-08-28T01:51:57.019+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:57.024612+0000) 2023-08-28T01:51:58.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:58 smithi130 bash[51302]: debug 2023-08-28T01:51:58.019+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:58.024952+0000) 2023-08-28T01:51:58.669 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:58 smithi006 bash[27292]: cluster 2023-08-28T01:51:56.740152+0000 mgr.y (mgr.14901) 7000 : cluster [DBG] pgmap v7007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:58.669 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:51:58 smithi006 bash[27292]: audit 2023-08-28T01:51:58.271522+0000 mon.c (mon.2) 2748 : audit [DBG] from='client.? 172.21.15.6:0/3206083487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:58.670 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:58 smithi006 bash[33918]: cluster 2023-08-28T01:51:56.740152+0000 mgr.y (mgr.14901) 7000 : cluster [DBG] pgmap v7007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:58.670 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:51:58 smithi006 bash[33918]: audit 2023-08-28T01:51:58.271522+0000 mon.c (mon.2) 2748 : audit [DBG] from='client.? 172.21.15.6:0/3206083487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:58.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:58 smithi130 bash[27579]: cluster 2023-08-28T01:51:56.740152+0000 mgr.y (mgr.14901) 7000 : cluster [DBG] pgmap v7007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:51:58.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:51:58 smithi130 bash[27579]: audit 2023-08-28T01:51:58.271522+0000 mon.c (mon.2) 2748 : audit [DBG] from='client.? 172.21.15.6:0/3206083487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:59.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:51:59 smithi130 bash[51302]: debug 2023-08-28T01:51:59.019+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:59.025255+0000) 2023-08-28T01:52:00.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:00 smithi130 bash[51302]: debug 2023-08-28T01:52:00.019+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:00.025593+0000) 2023-08-28T01:52:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:00 smithi006 bash[27292]: cluster 2023-08-28T01:51:58.740785+0000 mgr.y (mgr.14901) 7001 : cluster [DBG] pgmap v7008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:00 smithi006 bash[33918]: cluster 2023-08-28T01:51:58.740785+0000 mgr.y (mgr.14901) 7001 : cluster [DBG] pgmap v7008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:00 smithi130 bash[27579]: cluster 2023-08-28T01:51:58.740785+0000 mgr.y (mgr.14901) 7001 : cluster [DBG] pgmap v7008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:01.141 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:52:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:52:00] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:52:01.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:01 smithi130 bash[51302]: debug 2023-08-28T01:52:01.019+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:01.025959+0000) 2023-08-28T01:52:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:52:01.141Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:52:01.143Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:01.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:01 smithi006 bash[27292]: audit 2023-08-28T01:52:00.693346+0000 mon.a (mon.0) 4572 : audit [DBG] from='client.? 172.21.15.6:0/2627875737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:01.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:01 smithi006 bash[33918]: audit 2023-08-28T01:52:00.693346+0000 mon.a (mon.0) 4572 : audit [DBG] from='client.? 172.21.15.6:0/2627875737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:01.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:01 smithi130 bash[27579]: audit 2023-08-28T01:52:00.693346+0000 mon.a (mon.0) 4572 : audit [DBG] from='client.? 172.21.15.6:0/2627875737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:02.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:02 smithi130 bash[51302]: debug 2023-08-28T01:52:02.018+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:02.026307+0000) 2023-08-28T01:52:02.704 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:02 smithi006 bash[33918]: cluster 2023-08-28T01:52:00.741652+0000 mgr.y (mgr.14901) 7002 : cluster [DBG] pgmap v7009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:02.705 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:02 smithi006 bash[27292]: cluster 2023-08-28T01:52:00.741652+0000 mgr.y (mgr.14901) 7002 : cluster [DBG] pgmap v7009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:02 smithi130 bash[27579]: cluster 2023-08-28T01:52:00.741652+0000 mgr.y (mgr.14901) 7002 : cluster [DBG] pgmap v7009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:52:02.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:52:02.707Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:03.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:03 smithi130 bash[51302]: debug 2023-08-28T01:52:03.022+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:03.026624+0000) 2023-08-28T01:52:04.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:04 smithi130 bash[51302]: debug 2023-08-28T01:52:04.022+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:04.026951+0000) 2023-08-28T01:52:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:03 smithi130 bash[27579]: cluster 2023-08-28T01:52:02.742099+0000 mgr.y (mgr.14901) 7003 : cluster [DBG] pgmap v7010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:03 smithi130 bash[27579]: audit 2023-08-28T01:52:02.830800+0000 mon.a (mon.0) 4573 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:52:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:03 smithi130 bash[27579]: audit 2023-08-28T01:52:03.100732+0000 mon.a (mon.0) 4574 : audit [DBG] from='client.? 172.21.15.6:0/1349293148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:03 smithi130 bash[27579]: audit 2023-08-28T01:52:03.126382+0000 mon.b (mon.1) 786 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:52:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:03 smithi130 bash[27579]: audit 2023-08-28T01:52:03.128334+0000 mon.b (mon.1) 787 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:52:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:03 smithi130 bash[27579]: audit 2023-08-28T01:52:03.135302+0000 mon.a (mon.0) 4575 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:52:04.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:03 smithi006 bash[33918]: cluster 2023-08-28T01:52:02.742099+0000 mgr.y (mgr.14901) 7003 : cluster [DBG] pgmap v7010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:04.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:03 smithi006 bash[33918]: audit 2023-08-28T01:52:02.830800+0000 mon.a (mon.0) 4573 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:52:04.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:03 smithi006 bash[33918]: audit 2023-08-28T01:52:03.100732+0000 mon.a (mon.0) 4574 : audit [DBG] from='client.? 172.21.15.6:0/1349293148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:04.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:03 smithi006 bash[33918]: audit 2023-08-28T01:52:03.126382+0000 mon.b (mon.1) 786 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:52:04.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:03 smithi006 bash[33918]: audit 2023-08-28T01:52:03.128334+0000 mon.b (mon.1) 787 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:52:04.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:03 smithi006 bash[33918]: audit 2023-08-28T01:52:03.135302+0000 mon.a (mon.0) 4575 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:52:04.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:03 smithi006 bash[27292]: cluster 2023-08-28T01:52:02.742099+0000 mgr.y (mgr.14901) 7003 : cluster [DBG] pgmap v7010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:04.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:03 smithi006 bash[27292]: audit 2023-08-28T01:52:02.830800+0000 mon.a (mon.0) 4573 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:52:04.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:03 smithi006 bash[27292]: audit 2023-08-28T01:52:03.100732+0000 mon.a (mon.0) 4574 : audit [DBG] from='client.? 172.21.15.6:0/1349293148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:04.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:03 smithi006 bash[27292]: audit 2023-08-28T01:52:03.126382+0000 mon.b (mon.1) 786 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:52:04.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:03 smithi006 bash[27292]: audit 2023-08-28T01:52:03.128334+0000 mon.b (mon.1) 787 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:52:04.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:03 smithi006 bash[27292]: audit 2023-08-28T01:52:03.135302+0000 mon.a (mon.0) 4575 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:52:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:52:04.590Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:52:04.592Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:05.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:05 smithi130 bash[51302]: debug 2023-08-28T01:52:05.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:05.027296+0000) 2023-08-28T01:52:06.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:05 smithi130 bash[27579]: cluster 2023-08-28T01:52:04.743199+0000 mgr.y (mgr.14901) 7004 : cluster [DBG] pgmap v7011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:06.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:05 smithi130 bash[27579]: audit 2023-08-28T01:52:05.517033+0000 mon.c (mon.2) 2749 : audit [DBG] from='client.? 172.21.15.6:0/159900697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:06.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:06 smithi130 bash[51302]: debug 2023-08-28T01:52:06.022+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:06.027633+0000) 2023-08-28T01:52:06.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:05 smithi006 bash[33918]: cluster 2023-08-28T01:52:04.743199+0000 mgr.y (mgr.14901) 7004 : cluster [DBG] pgmap v7011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:06.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:05 smithi006 bash[33918]: audit 2023-08-28T01:52:05.517033+0000 mon.c (mon.2) 2749 : audit [DBG] from='client.? 172.21.15.6:0/159900697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:06.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:05 smithi006 bash[27292]: cluster 2023-08-28T01:52:04.743199+0000 mgr.y (mgr.14901) 7004 : cluster [DBG] pgmap v7011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:06.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:05 smithi006 bash[27292]: audit 2023-08-28T01:52:05.517033+0000 mon.c (mon.2) 2749 : audit [DBG] from='client.? 172.21.15.6:0/159900697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:07.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:07 smithi130 bash[51302]: debug 2023-08-28T01:52:07.022+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:07.027945+0000) 2023-08-28T01:52:08.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:08 smithi130 bash[51302]: debug 2023-08-28T01:52:08.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:08.028260+0000) 2023-08-28T01:52:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:08 smithi006 bash[33918]: cluster 2023-08-28T01:52:06.743862+0000 mgr.y (mgr.14901) 7005 : cluster [DBG] pgmap v7012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:08 smithi006 bash[33918]: audit 2023-08-28T01:52:07.934926+0000 mon.c (mon.2) 2750 : audit [DBG] from='client.? 172.21.15.6:0/1661967216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:08.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:08 smithi006 bash[27292]: cluster 2023-08-28T01:52:06.743862+0000 mgr.y (mgr.14901) 7005 : cluster [DBG] pgmap v7012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:08 smithi006 bash[27292]: audit 2023-08-28T01:52:07.934926+0000 mon.c (mon.2) 2750 : audit [DBG] from='client.? 172.21.15.6:0/1661967216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:08 smithi130 bash[27579]: cluster 2023-08-28T01:52:06.743862+0000 mgr.y (mgr.14901) 7005 : cluster [DBG] pgmap v7012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:08 smithi130 bash[27579]: audit 2023-08-28T01:52:07.934926+0000 mon.c (mon.2) 2750 : audit [DBG] from='client.? 172.21.15.6:0/1661967216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:09.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:09 smithi130 bash[51302]: debug 2023-08-28T01:52:09.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:09.028614+0000) 2023-08-28T01:52:10.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:10 smithi130 bash[51302]: debug 2023-08-28T01:52:10.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:10.028944+0000) 2023-08-28T01:52:10.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:10 smithi006 bash[33918]: cluster 2023-08-28T01:52:08.744530+0000 mgr.y (mgr.14901) 7006 : cluster [DBG] pgmap v7013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:10.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:10 smithi006 bash[33918]: audit 2023-08-28T01:52:10.344401+0000 mon.a (mon.0) 4576 : audit [DBG] from='client.? 172.21.15.6:0/986995509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:10 smithi006 bash[27292]: cluster 2023-08-28T01:52:08.744530+0000 mgr.y (mgr.14901) 7006 : cluster [DBG] pgmap v7013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:10.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:10 smithi006 bash[27292]: audit 2023-08-28T01:52:10.344401+0000 mon.a (mon.0) 4576 : audit [DBG] from='client.? 172.21.15.6:0/986995509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:10 smithi130 bash[27579]: cluster 2023-08-28T01:52:08.744530+0000 mgr.y (mgr.14901) 7006 : cluster [DBG] pgmap v7013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:10.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:10 smithi130 bash[27579]: audit 2023-08-28T01:52:10.344401+0000 mon.a (mon.0) 4576 : audit [DBG] from='client.? 172.21.15.6:0/986995509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:11.142 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:52:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:52:10] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T01:52:11.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:11 smithi130 bash[51302]: debug 2023-08-28T01:52:11.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:11.029235+0000) 2023-08-28T01:52:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:52:11.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:52:11.144Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:12.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:12 smithi130 bash[51302]: debug 2023-08-28T01:52:12.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:12.029538+0000) 2023-08-28T01:52:12.705 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:12 smithi006 bash[27292]: cluster 2023-08-28T01:52:10.745712+0000 mgr.y (mgr.14901) 7007 : cluster [DBG] pgmap v7014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:12.705 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:12 smithi006 bash[33918]: cluster 2023-08-28T01:52:10.745712+0000 mgr.y (mgr.14901) 7007 : cluster [DBG] pgmap v7014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:12 smithi130 bash[27579]: cluster 2023-08-28T01:52:10.745712+0000 mgr.y (mgr.14901) 7007 : cluster [DBG] pgmap v7014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:52:12.705Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:52:12.707Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:13.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:13 smithi130 bash[51302]: debug 2023-08-28T01:52:13.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:13.029867+0000) 2023-08-28T01:52:13.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:13 smithi006 bash[33918]: audit 2023-08-28T01:52:12.764776+0000 mon.c (mon.2) 2751 : audit [DBG] from='client.? 172.21.15.6:0/3997300245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:13.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:13 smithi006 bash[27292]: audit 2023-08-28T01:52:12.764776+0000 mon.c (mon.2) 2751 : audit [DBG] from='client.? 172.21.15.6:0/3997300245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:13.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:13 smithi130 bash[27579]: audit 2023-08-28T01:52:12.764776+0000 mon.c (mon.2) 2751 : audit [DBG] from='client.? 172.21.15.6:0/3997300245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:14.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:14 smithi130 bash[51302]: debug 2023-08-28T01:52:14.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:14.030193+0000) 2023-08-28T01:52:14.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:14 smithi006 bash[27292]: cluster 2023-08-28T01:52:12.746326+0000 mgr.y (mgr.14901) 7008 : cluster [DBG] pgmap v7015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:14.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:14 smithi006 bash[33918]: cluster 2023-08-28T01:52:12.746326+0000 mgr.y (mgr.14901) 7008 : cluster [DBG] pgmap v7015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:52:14.591Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:52:14.593Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:14.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:14 smithi130 bash[27579]: cluster 2023-08-28T01:52:12.746326+0000 mgr.y (mgr.14901) 7008 : cluster [DBG] pgmap v7015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:15.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:15 smithi130 bash[51302]: debug 2023-08-28T01:52:15.026+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:15.030528+0000) 2023-08-28T01:52:15.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:15 smithi006 bash[33918]: audit 2023-08-28T01:52:15.170368+0000 mon.c (mon.2) 2752 : audit [DBG] from='client.? 172.21.15.6:0/1395581066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:15.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:15 smithi006 bash[27292]: audit 2023-08-28T01:52:15.170368+0000 mon.c (mon.2) 2752 : audit [DBG] from='client.? 172.21.15.6:0/1395581066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:15.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:15 smithi130 bash[27579]: audit 2023-08-28T01:52:15.170368+0000 mon.c (mon.2) 2752 : audit [DBG] from='client.? 172.21.15.6:0/1395581066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:16.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:16 smithi130 bash[51302]: debug 2023-08-28T01:52:16.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:16.030923+0000) 2023-08-28T01:52:16.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:16 smithi006 bash[27292]: cluster 2023-08-28T01:52:14.747449+0000 mgr.y (mgr.14901) 7009 : cluster [DBG] pgmap v7016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:16.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:16 smithi006 bash[33918]: cluster 2023-08-28T01:52:14.747449+0000 mgr.y (mgr.14901) 7009 : cluster [DBG] pgmap v7016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:16.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:16 smithi130 bash[27579]: cluster 2023-08-28T01:52:14.747449+0000 mgr.y (mgr.14901) 7009 : cluster [DBG] pgmap v7016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:17.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:17 smithi130 bash[51302]: debug 2023-08-28T01:52:17.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:17.031202+0000) 2023-08-28T01:52:18.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:18 smithi130 bash[51302]: debug 2023-08-28T01:52:18.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:18.031560+0000) 2023-08-28T01:52:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:18 smithi006 bash[27292]: cluster 2023-08-28T01:52:16.748123+0000 mgr.y (mgr.14901) 7010 : cluster [DBG] pgmap v7017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:18 smithi006 bash[27292]: audit 2023-08-28T01:52:17.582718+0000 mon.a (mon.0) 4577 : audit [DBG] from='client.? 172.21.15.6:0/2462167791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:18.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:18 smithi006 bash[33918]: cluster 2023-08-28T01:52:16.748123+0000 mgr.y (mgr.14901) 7010 : cluster [DBG] pgmap v7017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:18.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:18 smithi006 bash[33918]: audit 2023-08-28T01:52:17.582718+0000 mon.a (mon.0) 4577 : audit [DBG] from='client.? 172.21.15.6:0/2462167791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:18.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:18 smithi130 bash[27579]: cluster 2023-08-28T01:52:16.748123+0000 mgr.y (mgr.14901) 7010 : cluster [DBG] pgmap v7017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:18.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:18 smithi130 bash[27579]: audit 2023-08-28T01:52:17.582718+0000 mon.a (mon.0) 4577 : audit [DBG] from='client.? 172.21.15.6:0/2462167791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:19.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:19 smithi130 bash[51302]: debug 2023-08-28T01:52:19.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:19.031912+0000) 2023-08-28T01:52:20.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:20 smithi130 bash[51302]: debug 2023-08-28T01:52:20.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:20.032233+0000) 2023-08-28T01:52:20.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:20 smithi006 bash[33918]: cluster 2023-08-28T01:52:18.748767+0000 mgr.y (mgr.14901) 7011 : cluster [DBG] pgmap v7018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:20.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:20 smithi006 bash[33918]: audit 2023-08-28T01:52:19.991320+0000 mon.a (mon.0) 4578 : audit [DBG] from='client.? 172.21.15.6:0/3459761242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:20.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:20 smithi006 bash[27292]: cluster 2023-08-28T01:52:18.748767+0000 mgr.y (mgr.14901) 7011 : cluster [DBG] pgmap v7018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:20.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:20 smithi006 bash[27292]: audit 2023-08-28T01:52:19.991320+0000 mon.a (mon.0) 4578 : audit [DBG] from='client.? 172.21.15.6:0/3459761242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:20 smithi130 bash[27579]: cluster 2023-08-28T01:52:18.748767+0000 mgr.y (mgr.14901) 7011 : cluster [DBG] pgmap v7018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:20 smithi130 bash[27579]: audit 2023-08-28T01:52:19.991320+0000 mon.a (mon.0) 4578 : audit [DBG] from='client.? 172.21.15.6:0/3459761242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:21.142 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:52:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:52:20] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T01:52:21.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:21 smithi130 bash[51302]: debug 2023-08-28T01:52:21.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:21.032596+0000) 2023-08-28T01:52:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:52:21.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:52:21.144Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:22.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:22 smithi130 bash[51302]: debug 2023-08-28T01:52:22.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:22.032936+0000) 2023-08-28T01:52:22.706 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:22 smithi006 bash[27292]: cluster 2023-08-28T01:52:20.749884+0000 mgr.y (mgr.14901) 7012 : cluster [DBG] pgmap v7019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:22.706 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:22 smithi006 bash[27292]: audit 2023-08-28T01:52:22.411980+0000 mon.a (mon.0) 4579 : audit [DBG] from='client.? 172.21.15.6:0/2912561350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:22.707 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:22 smithi006 bash[33918]: cluster 2023-08-28T01:52:20.749884+0000 mgr.y (mgr.14901) 7012 : cluster [DBG] pgmap v7019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:22.707 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:22 smithi006 bash[33918]: audit 2023-08-28T01:52:22.411980+0000 mon.a (mon.0) 4579 : audit [DBG] from='client.? 172.21.15.6:0/2912561350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:22.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:22 smithi130 bash[27579]: cluster 2023-08-28T01:52:20.749884+0000 mgr.y (mgr.14901) 7012 : cluster [DBG] pgmap v7019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:22 smithi130 bash[27579]: audit 2023-08-28T01:52:22.411980+0000 mon.a (mon.0) 4579 : audit [DBG] from='client.? 172.21.15.6:0/2912561350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:52:22.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:52:22.708Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:23.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:23 smithi130 bash[51302]: debug 2023-08-28T01:52:23.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:23.033265+0000) 2023-08-28T01:52:24.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:24 smithi130 bash[51302]: debug 2023-08-28T01:52:24.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:24.033621+0000) 2023-08-28T01:52:24.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:52:24.591Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:24.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:52:24.594Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:24.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:24 smithi006 bash[33918]: cluster 2023-08-28T01:52:22.750492+0000 mgr.y (mgr.14901) 7013 : cluster [DBG] pgmap v7020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:24.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:24 smithi006 bash[27292]: cluster 2023-08-28T01:52:22.750492+0000 mgr.y (mgr.14901) 7013 : cluster [DBG] pgmap v7020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:24.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:24 smithi130 bash[27579]: cluster 2023-08-28T01:52:22.750492+0000 mgr.y (mgr.14901) 7013 : cluster [DBG] pgmap v7020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:25.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:25 smithi130 bash[51302]: debug 2023-08-28T01:52:25.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:25.033988+0000) 2023-08-28T01:52:25.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:25 smithi006 bash[33918]: audit 2023-08-28T01:52:24.835436+0000 mon.a (mon.0) 4580 : audit [DBG] from='client.? 172.21.15.6:0/417820552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:25.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:25 smithi006 bash[27292]: audit 2023-08-28T01:52:24.835436+0000 mon.a (mon.0) 4580 : audit [DBG] from='client.? 172.21.15.6:0/417820552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:25.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:25 smithi130 bash[27579]: audit 2023-08-28T01:52:24.835436+0000 mon.a (mon.0) 4580 : audit [DBG] from='client.? 172.21.15.6:0/417820552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:26.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:26 smithi130 bash[51302]: debug 2023-08-28T01:52:26.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:26.034341+0000) 2023-08-28T01:52:26.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:26 smithi006 bash[27292]: cluster 2023-08-28T01:52:24.751218+0000 mgr.y (mgr.14901) 7014 : cluster [DBG] pgmap v7021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:26.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:26 smithi006 bash[33918]: cluster 2023-08-28T01:52:24.751218+0000 mgr.y (mgr.14901) 7014 : cluster [DBG] pgmap v7021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:26.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:26 smithi130 bash[27579]: cluster 2023-08-28T01:52:24.751218+0000 mgr.y (mgr.14901) 7014 : cluster [DBG] pgmap v7021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:27.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:27 smithi130 bash[51302]: debug 2023-08-28T01:52:27.030+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:27.034653+0000) 2023-08-28T01:52:27.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:27 smithi006 bash[33918]: audit 2023-08-28T01:52:27.248766+0000 mon.c (mon.2) 2753 : audit [DBG] from='client.? 172.21.15.6:0/3430211058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:27.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:27 smithi006 bash[27292]: audit 2023-08-28T01:52:27.248766+0000 mon.c (mon.2) 2753 : audit [DBG] from='client.? 172.21.15.6:0/3430211058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:27.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:27 smithi130 bash[27579]: audit 2023-08-28T01:52:27.248766+0000 mon.c (mon.2) 2753 : audit [DBG] from='client.? 172.21.15.6:0/3430211058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:28.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:28 smithi130 bash[51302]: debug 2023-08-28T01:52:28.034+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:28.034987+0000) 2023-08-28T01:52:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:28 smithi006 bash[27292]: cluster 2023-08-28T01:52:26.751856+0000 mgr.y (mgr.14901) 7015 : cluster [DBG] pgmap v7022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:28 smithi006 bash[33918]: cluster 2023-08-28T01:52:26.751856+0000 mgr.y (mgr.14901) 7015 : cluster [DBG] pgmap v7022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:28.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:28 smithi130 bash[27579]: cluster 2023-08-28T01:52:26.751856+0000 mgr.y (mgr.14901) 7015 : cluster [DBG] pgmap v7022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:29.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:29 smithi130 bash[51302]: debug 2023-08-28T01:52:29.034+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:29.035324+0000) 2023-08-28T01:52:30.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:30 smithi130 bash[51302]: debug 2023-08-28T01:52:30.034+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:30.035591+0000) 2023-08-28T01:52:30.752 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:30 smithi006 bash[33918]: cluster 2023-08-28T01:52:28.752504+0000 mgr.y (mgr.14901) 7016 : cluster [DBG] pgmap v7023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:30.752 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:30 smithi006 bash[33918]: audit 2023-08-28T01:52:29.664186+0000 mon.c (mon.2) 2754 : audit [DBG] from='client.? 172.21.15.6:0/2422382243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:30.753 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:30 smithi006 bash[27292]: cluster 2023-08-28T01:52:28.752504+0000 mgr.y (mgr.14901) 7016 : cluster [DBG] pgmap v7023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:30.753 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:30 smithi006 bash[27292]: audit 2023-08-28T01:52:29.664186+0000 mon.c (mon.2) 2754 : audit [DBG] from='client.? 172.21.15.6:0/2422382243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:30.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:30 smithi130 bash[27579]: cluster 2023-08-28T01:52:28.752504+0000 mgr.y (mgr.14901) 7016 : cluster [DBG] pgmap v7023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:30 smithi130 bash[27579]: audit 2023-08-28T01:52:29.664186+0000 mon.c (mon.2) 2754 : audit [DBG] from='client.? 172.21.15.6:0/2422382243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:31.143 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:52:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:52:30] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:52:31.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:31 smithi130 bash[51302]: debug 2023-08-28T01:52:31.034+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:31.035946+0000) 2023-08-28T01:52:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:52:31.143Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:52:31.145Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:31.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:31 smithi130 bash[27579]: cluster 2023-08-28T01:52:30.753886+0000 mgr.y (mgr.14901) 7017 : cluster [DBG] pgmap v7024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:31.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:31 smithi006 bash[27292]: cluster 2023-08-28T01:52:30.753886+0000 mgr.y (mgr.14901) 7017 : cluster [DBG] pgmap v7024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:31.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:31 smithi006 bash[33918]: cluster 2023-08-28T01:52:30.753886+0000 mgr.y (mgr.14901) 7017 : cluster [DBG] pgmap v7024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:32.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:32 smithi130 bash[51302]: debug 2023-08-28T01:52:32.034+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:32.036217+0000) 2023-08-28T01:52:32.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:32 smithi130 bash[27579]: audit 2023-08-28T01:52:32.075998+0000 mon.c (mon.2) 2755 : audit [DBG] from='client.? 172.21.15.6:0/3105280712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:32.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:32 smithi006 bash[27292]: audit 2023-08-28T01:52:32.075998+0000 mon.c (mon.2) 2755 : audit [DBG] from='client.? 172.21.15.6:0/3105280712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:32.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:32 smithi006 bash[33918]: audit 2023-08-28T01:52:32.075998+0000 mon.c (mon.2) 2755 : audit [DBG] from='client.? 172.21.15.6:0/3105280712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:52:32.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:52:32.709Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:33.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:33 smithi130 bash[51302]: debug 2023-08-28T01:52:33.034+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:33.036504+0000) 2023-08-28T01:52:33.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:33 smithi130 bash[27579]: cluster 2023-08-28T01:52:32.754597+0000 mgr.y (mgr.14901) 7018 : cluster [DBG] pgmap v7025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:33 smithi006 bash[27292]: cluster 2023-08-28T01:52:32.754597+0000 mgr.y (mgr.14901) 7018 : cluster [DBG] pgmap v7025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:33.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:33 smithi006 bash[33918]: cluster 2023-08-28T01:52:32.754597+0000 mgr.y (mgr.14901) 7018 : cluster [DBG] pgmap v7025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:34.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:34 smithi130 bash[51302]: debug 2023-08-28T01:52:34.034+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:34.036777+0000) 2023-08-28T01:52:34.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:34 smithi130 bash[27579]: audit 2023-08-28T01:52:34.486821+0000 mon.a (mon.0) 4581 : audit [DBG] from='client.? 172.21.15.6:0/3820672264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:34.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:34 smithi006 bash[33918]: audit 2023-08-28T01:52:34.486821+0000 mon.a (mon.0) 4581 : audit [DBG] from='client.? 172.21.15.6:0/3820672264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:34.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:34 smithi006 bash[27292]: audit 2023-08-28T01:52:34.486821+0000 mon.a (mon.0) 4581 : audit [DBG] from='client.? 172.21.15.6:0/3820672264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:52:34.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:52:34.595Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:35.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:35 smithi130 bash[51302]: debug 2023-08-28T01:52:35.034+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:35.037087+0000) 2023-08-28T01:52:35.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:35 smithi130 bash[27579]: cluster 2023-08-28T01:52:34.755743+0000 mgr.y (mgr.14901) 7019 : cluster [DBG] pgmap v7026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:35.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:35 smithi006 bash[27292]: cluster 2023-08-28T01:52:34.755743+0000 mgr.y (mgr.14901) 7019 : cluster [DBG] pgmap v7026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:35.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:35 smithi006 bash[33918]: cluster 2023-08-28T01:52:34.755743+0000 mgr.y (mgr.14901) 7019 : cluster [DBG] pgmap v7026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:36.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:36 smithi130 bash[51302]: debug 2023-08-28T01:52:36.034+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:36.037418+0000) 2023-08-28T01:52:37.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:37 smithi130 bash[51302]: debug 2023-08-28T01:52:37.034+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:37.037699+0000) 2023-08-28T01:52:38.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:37 smithi130 bash[27579]: cluster 2023-08-28T01:52:36.756373+0000 mgr.y (mgr.14901) 7020 : cluster [DBG] pgmap v7027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:38.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:37 smithi130 bash[27579]: audit 2023-08-28T01:52:36.898724+0000 mon.c (mon.2) 2756 : audit [DBG] from='client.? 172.21.15.6:0/605143893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:38 smithi130 bash[51302]: debug 2023-08-28T01:52:38.034+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:38.037962+0000) 2023-08-28T01:52:38.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:37 smithi006 bash[27292]: cluster 2023-08-28T01:52:36.756373+0000 mgr.y (mgr.14901) 7020 : cluster [DBG] pgmap v7027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:38.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:37 smithi006 bash[27292]: audit 2023-08-28T01:52:36.898724+0000 mon.c (mon.2) 2756 : audit [DBG] from='client.? 172.21.15.6:0/605143893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:38.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:37 smithi006 bash[33918]: cluster 2023-08-28T01:52:36.756373+0000 mgr.y (mgr.14901) 7020 : cluster [DBG] pgmap v7027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:38.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:37 smithi006 bash[33918]: audit 2023-08-28T01:52:36.898724+0000 mon.c (mon.2) 2756 : audit [DBG] from='client.? 172.21.15.6:0/605143893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:39.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:39 smithi130 bash[51302]: debug 2023-08-28T01:52:39.034+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:39.038224+0000) 2023-08-28T01:52:40.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:40 smithi130 bash[51302]: debug 2023-08-28T01:52:40.034+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:40.038495+0000) 2023-08-28T01:52:40.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:39 smithi130 bash[27579]: cluster 2023-08-28T01:52:38.757058+0000 mgr.y (mgr.14901) 7021 : cluster [DBG] pgmap v7028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:40.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:39 smithi130 bash[27579]: audit 2023-08-28T01:52:39.316309+0000 mon.a (mon.0) 4582 : audit [DBG] from='client.? 172.21.15.6:0/3238128147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:40.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:39 smithi006 bash[33918]: cluster 2023-08-28T01:52:38.757058+0000 mgr.y (mgr.14901) 7021 : cluster [DBG] pgmap v7028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:40.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:39 smithi006 bash[33918]: audit 2023-08-28T01:52:39.316309+0000 mon.a (mon.0) 4582 : audit [DBG] from='client.? 172.21.15.6:0/3238128147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:40.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:39 smithi006 bash[27292]: cluster 2023-08-28T01:52:38.757058+0000 mgr.y (mgr.14901) 7021 : cluster [DBG] pgmap v7028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:40.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:39 smithi006 bash[27292]: audit 2023-08-28T01:52:39.316309+0000 mon.a (mon.0) 4582 : audit [DBG] from='client.? 172.21.15.6:0/3238128147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:41.147 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:52:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:52:40] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:52:41.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:41 smithi130 bash[51302]: debug 2023-08-28T01:52:41.038+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:41.038819+0000) 2023-08-28T01:52:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:52:41.144Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:52:41.146Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:42.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:41 smithi130 bash[27579]: cluster 2023-08-28T01:52:40.757995+0000 mgr.y (mgr.14901) 7022 : cluster [DBG] pgmap v7029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:42.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:41 smithi130 bash[27579]: audit 2023-08-28T01:52:41.735511+0000 mon.a (mon.0) 4583 : audit [DBG] from='client.? 172.21.15.6:0/1444740371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:42 smithi130 bash[51302]: debug 2023-08-28T01:52:42.038+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:42.039109+0000) 2023-08-28T01:52:42.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:41 smithi006 bash[27292]: cluster 2023-08-28T01:52:40.757995+0000 mgr.y (mgr.14901) 7022 : cluster [DBG] pgmap v7029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:42.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:41 smithi006 bash[27292]: audit 2023-08-28T01:52:41.735511+0000 mon.a (mon.0) 4583 : audit [DBG] from='client.? 172.21.15.6:0/1444740371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:42.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:41 smithi006 bash[33918]: cluster 2023-08-28T01:52:40.757995+0000 mgr.y (mgr.14901) 7022 : cluster [DBG] pgmap v7029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:42.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:41 smithi006 bash[33918]: audit 2023-08-28T01:52:41.735511+0000 mon.a (mon.0) 4583 : audit [DBG] from='client.? 172.21.15.6:0/1444740371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:52:42.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:52:42.710Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:43.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:43 smithi130 bash[51302]: debug 2023-08-28T01:52:43.038+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:43.039326+0000) 2023-08-28T01:52:44.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:44 smithi130 bash[51302]: debug 2023-08-28T01:52:44.038+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:44.039598+0000) 2023-08-28T01:52:44.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:44 smithi006 bash[27292]: cluster 2023-08-28T01:52:42.758636+0000 mgr.y (mgr.14901) 7023 : cluster [DBG] pgmap v7030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:44.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:44 smithi006 bash[27292]: audit 2023-08-28T01:52:44.150273+0000 mon.c (mon.2) 2757 : audit [DBG] from='client.? 172.21.15.6:0/1251382712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:52:44.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:52:44.595Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:44.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:44 smithi006 bash[33918]: cluster 2023-08-28T01:52:42.758636+0000 mgr.y (mgr.14901) 7023 : cluster [DBG] pgmap v7030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:44.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:44 smithi006 bash[33918]: audit 2023-08-28T01:52:44.150273+0000 mon.c (mon.2) 2757 : audit [DBG] from='client.? 172.21.15.6:0/1251382712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:44.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:44 smithi130 bash[27579]: cluster 2023-08-28T01:52:42.758636+0000 mgr.y (mgr.14901) 7023 : cluster [DBG] pgmap v7030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:44.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:44 smithi130 bash[27579]: audit 2023-08-28T01:52:44.150273+0000 mon.c (mon.2) 2757 : audit [DBG] from='client.? 172.21.15.6:0/1251382712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:45.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:45 smithi130 bash[51302]: debug 2023-08-28T01:52:45.038+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:45.039880+0000) 2023-08-28T01:52:46.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:46 smithi130 bash[51302]: debug 2023-08-28T01:52:46.038+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:46.040138+0000) 2023-08-28T01:52:46.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:46 smithi006 bash[33918]: cluster 2023-08-28T01:52:44.759748+0000 mgr.y (mgr.14901) 7024 : cluster [DBG] pgmap v7031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:46.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:46 smithi006 bash[27292]: cluster 2023-08-28T01:52:44.759748+0000 mgr.y (mgr.14901) 7024 : cluster [DBG] pgmap v7031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:46.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:46 smithi130 bash[27579]: cluster 2023-08-28T01:52:44.759748+0000 mgr.y (mgr.14901) 7024 : cluster [DBG] pgmap v7031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:47.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:47 smithi130 bash[51302]: debug 2023-08-28T01:52:47.038+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:47.040425+0000) 2023-08-28T01:52:47.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:47 smithi006 bash[33918]: audit 2023-08-28T01:52:46.568928+0000 mon.a (mon.0) 4584 : audit [DBG] from='client.? 172.21.15.6:0/3992798800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:47.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:47 smithi006 bash[27292]: audit 2023-08-28T01:52:46.568928+0000 mon.a (mon.0) 4584 : audit [DBG] from='client.? 172.21.15.6:0/3992798800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:47.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:47 smithi130 bash[27579]: audit 2023-08-28T01:52:46.568928+0000 mon.a (mon.0) 4584 : audit [DBG] from='client.? 172.21.15.6:0/3992798800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:48.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:48 smithi130 bash[51302]: debug 2023-08-28T01:52:48.038+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:48.040707+0000) 2023-08-28T01:52:48.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:48 smithi006 bash[27292]: cluster 2023-08-28T01:52:46.760459+0000 mgr.y (mgr.14901) 7025 : cluster [DBG] pgmap v7032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:48.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:48 smithi006 bash[33918]: cluster 2023-08-28T01:52:46.760459+0000 mgr.y (mgr.14901) 7025 : cluster [DBG] pgmap v7032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:48.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:48 smithi130 bash[27579]: cluster 2023-08-28T01:52:46.760459+0000 mgr.y (mgr.14901) 7025 : cluster [DBG] pgmap v7032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:49.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:49 smithi130 bash[51302]: debug 2023-08-28T01:52:49.038+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:49.041003+0000) 2023-08-28T01:52:49.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:49 smithi006 bash[33918]: audit 2023-08-28T01:52:48.983247+0000 mon.a (mon.0) 4585 : audit [DBG] from='client.? 172.21.15.6:0/1009927370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:49.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:49 smithi006 bash[27292]: audit 2023-08-28T01:52:48.983247+0000 mon.a (mon.0) 4585 : audit [DBG] from='client.? 172.21.15.6:0/1009927370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:49.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:49 smithi130 bash[27579]: audit 2023-08-28T01:52:48.983247+0000 mon.a (mon.0) 4585 : audit [DBG] from='client.? 172.21.15.6:0/1009927370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:50.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:50 smithi130 bash[51302]: debug 2023-08-28T01:52:50.038+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:50.041281+0000) 2023-08-28T01:52:50.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:50 smithi006 bash[33918]: cluster 2023-08-28T01:52:48.761274+0000 mgr.y (mgr.14901) 7026 : cluster [DBG] pgmap v7033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:50.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:50 smithi006 bash[27292]: cluster 2023-08-28T01:52:48.761274+0000 mgr.y (mgr.14901) 7026 : cluster [DBG] pgmap v7033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:50.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:50 smithi130 bash[27579]: cluster 2023-08-28T01:52:48.761274+0000 mgr.y (mgr.14901) 7026 : cluster [DBG] pgmap v7033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:51.144 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:52:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:52:50] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:52:51.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:51 smithi130 bash[51302]: debug 2023-08-28T01:52:51.038+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:51.041582+0000) 2023-08-28T01:52:51.420 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:52:51.144Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:51.421 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:52:51.146Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:51.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:51 smithi006 bash[27292]: audit 2023-08-28T01:52:51.401036+0000 mon.c (mon.2) 2758 : audit [DBG] from='client.? 172.21.15.6:0/2396973208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:51.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:51 smithi006 bash[33918]: audit 2023-08-28T01:52:51.401036+0000 mon.c (mon.2) 2758 : audit [DBG] from='client.? 172.21.15.6:0/2396973208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:51.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:51 smithi130 bash[27579]: audit 2023-08-28T01:52:51.401036+0000 mon.c (mon.2) 2758 : audit [DBG] from='client.? 172.21.15.6:0/2396973208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:52.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:52 smithi130 bash[51302]: debug 2023-08-28T01:52:52.038+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:52.041859+0000) 2023-08-28T01:52:52.708 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:52 smithi006 bash[27292]: cluster 2023-08-28T01:52:50.762328+0000 mgr.y (mgr.14901) 7027 : cluster [DBG] pgmap v7034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:52.709 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:52 smithi006 bash[33918]: cluster 2023-08-28T01:52:50.762328+0000 mgr.y (mgr.14901) 7027 : cluster [DBG] pgmap v7034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:52.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:52 smithi130 bash[27579]: cluster 2023-08-28T01:52:50.762328+0000 mgr.y (mgr.14901) 7027 : cluster [DBG] pgmap v7034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:52:52.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:52:52.711Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:53.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:53 smithi130 bash[51302]: debug 2023-08-28T01:52:53.038+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:53.042141+0000) 2023-08-28T01:52:54.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:54 smithi130 bash[51302]: debug 2023-08-28T01:52:54.038+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:54.042448+0000) 2023-08-28T01:52:54.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:54 smithi006 bash[33918]: cluster 2023-08-28T01:52:52.763028+0000 mgr.y (mgr.14901) 7028 : cluster [DBG] pgmap v7035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:54.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:54 smithi006 bash[33918]: audit 2023-08-28T01:52:53.813696+0000 mon.a (mon.0) 4586 : audit [DBG] from='client.? 172.21.15.6:0/130531656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:54.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:52:54.594Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:52:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:52:54.596Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:52:54.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:54 smithi006 bash[27292]: cluster 2023-08-28T01:52:52.763028+0000 mgr.y (mgr.14901) 7028 : cluster [DBG] pgmap v7035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:54.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:54 smithi006 bash[27292]: audit 2023-08-28T01:52:53.813696+0000 mon.a (mon.0) 4586 : audit [DBG] from='client.? 172.21.15.6:0/130531656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:54.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:54 smithi130 bash[27579]: cluster 2023-08-28T01:52:52.763028+0000 mgr.y (mgr.14901) 7028 : cluster [DBG] pgmap v7035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:54.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:54 smithi130 bash[27579]: audit 2023-08-28T01:52:53.813696+0000 mon.a (mon.0) 4586 : audit [DBG] from='client.? 172.21.15.6:0/130531656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:55.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:55 smithi130 bash[51302]: debug 2023-08-28T01:52:55.042+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:55.042725+0000) 2023-08-28T01:52:56.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:56 smithi130 bash[51302]: debug 2023-08-28T01:52:56.042+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:56.043017+0000) 2023-08-28T01:52:56.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:56 smithi006 bash[33918]: cluster 2023-08-28T01:52:54.764179+0000 mgr.y (mgr.14901) 7029 : cluster [DBG] pgmap v7036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:56.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:56 smithi006 bash[33918]: audit 2023-08-28T01:52:56.225866+0000 mon.a (mon.0) 4587 : audit [DBG] from='client.? 172.21.15.6:0/343247385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:56.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:56 smithi006 bash[27292]: cluster 2023-08-28T01:52:54.764179+0000 mgr.y (mgr.14901) 7029 : cluster [DBG] pgmap v7036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:56.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:56 smithi006 bash[27292]: audit 2023-08-28T01:52:56.225866+0000 mon.a (mon.0) 4587 : audit [DBG] from='client.? 172.21.15.6:0/343247385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:56 smithi130 bash[27579]: cluster 2023-08-28T01:52:54.764179+0000 mgr.y (mgr.14901) 7029 : cluster [DBG] pgmap v7036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:56 smithi130 bash[27579]: audit 2023-08-28T01:52:56.225866+0000 mon.a (mon.0) 4587 : audit [DBG] from='client.? 172.21.15.6:0/343247385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:57.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:57 smithi130 bash[51302]: debug 2023-08-28T01:52:57.042+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:57.043290+0000) 2023-08-28T01:52:58.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:58 smithi130 bash[51302]: debug 2023-08-28T01:52:58.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:58.043623+0000) 2023-08-28T01:52:58.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:58 smithi006 bash[27292]: cluster 2023-08-28T01:52:56.764863+0000 mgr.y (mgr.14901) 7030 : cluster [DBG] pgmap v7037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:58.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:58 smithi006 bash[33918]: cluster 2023-08-28T01:52:56.764863+0000 mgr.y (mgr.14901) 7030 : cluster [DBG] pgmap v7037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:58.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:58 smithi130 bash[27579]: cluster 2023-08-28T01:52:56.764863+0000 mgr.y (mgr.14901) 7030 : cluster [DBG] pgmap v7037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:52:59.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:52:59 smithi130 bash[51302]: debug 2023-08-28T01:52:59.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:59.043919+0000) 2023-08-28T01:52:59.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:52:59 smithi006 bash[33918]: audit 2023-08-28T01:52:58.639049+0000 mon.c (mon.2) 2759 : audit [DBG] from='client.? 172.21.15.6:0/2911382846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:59.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:52:59 smithi006 bash[27292]: audit 2023-08-28T01:52:58.639049+0000 mon.c (mon.2) 2759 : audit [DBG] from='client.? 172.21.15.6:0/2911382846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:59.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:52:59 smithi130 bash[27579]: audit 2023-08-28T01:52:58.639049+0000 mon.c (mon.2) 2759 : audit [DBG] from='client.? 172.21.15.6:0/2911382846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:00.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:00 smithi130 bash[51302]: debug 2023-08-28T01:53:00.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:00.044229+0000) 2023-08-28T01:53:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:00 smithi006 bash[27292]: cluster 2023-08-28T01:52:58.765434+0000 mgr.y (mgr.14901) 7031 : cluster [DBG] pgmap v7038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:00 smithi006 bash[33918]: cluster 2023-08-28T01:52:58.765434+0000 mgr.y (mgr.14901) 7031 : cluster [DBG] pgmap v7038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:00.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:00 smithi130 bash[27579]: cluster 2023-08-28T01:52:58.765434+0000 mgr.y (mgr.14901) 7031 : cluster [DBG] pgmap v7038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:01.144 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:53:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:53:00] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:53:01.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:01 smithi130 bash[51302]: debug 2023-08-28T01:53:01.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:01.044540+0000) 2023-08-28T01:53:01.458 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:53:01.145Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:01.458 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:53:01.147Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:01.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:01 smithi006 bash[33918]: audit 2023-08-28T01:53:01.055746+0000 mon.c (mon.2) 2760 : audit [DBG] from='client.? 172.21.15.6:0/2918142550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:01.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:01 smithi006 bash[27292]: audit 2023-08-28T01:53:01.055746+0000 mon.c (mon.2) 2760 : audit [DBG] from='client.? 172.21.15.6:0/2918142550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:01.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:01 smithi130 bash[27579]: audit 2023-08-28T01:53:01.055746+0000 mon.c (mon.2) 2760 : audit [DBG] from='client.? 172.21.15.6:0/2918142550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:02.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:02 smithi130 bash[51302]: debug 2023-08-28T01:53:02.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:02.044800+0000) 2023-08-28T01:53:02.711 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:02 smithi006 bash[33918]: cluster 2023-08-28T01:53:00.766068+0000 mgr.y (mgr.14901) 7032 : cluster [DBG] pgmap v7039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:02.712 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:53:02.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:02.712 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:02 smithi006 bash[27292]: cluster 2023-08-28T01:53:00.766068+0000 mgr.y (mgr.14901) 7032 : cluster [DBG] pgmap v7039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:02.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:02 smithi130 bash[27579]: cluster 2023-08-28T01:53:00.766068+0000 mgr.y (mgr.14901) 7032 : cluster [DBG] pgmap v7039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:53:02.711Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:03.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:03 smithi130 bash[51302]: debug 2023-08-28T01:53:03.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:03.045073+0000) 2023-08-28T01:53:03.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:03 smithi130 bash[27579]: audit 2023-08-28T01:53:03.147747+0000 mon.b (mon.1) 788 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:53:03.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:03 smithi006 bash[27292]: audit 2023-08-28T01:53:03.147747+0000 mon.b (mon.1) 788 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:53:03.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:03 smithi006 bash[33918]: audit 2023-08-28T01:53:03.147747+0000 mon.b (mon.1) 788 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:53:04.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:04 smithi130 bash[51302]: debug 2023-08-28T01:53:04.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:04.045427+0000) 2023-08-28T01:53:04.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:04 smithi130 bash[27579]: cluster 2023-08-28T01:53:02.766659+0000 mgr.y (mgr.14901) 7033 : cluster [DBG] pgmap v7040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:04.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:04 smithi130 bash[27579]: audit 2023-08-28T01:53:03.474154+0000 mon.c (mon.2) 2761 : audit [DBG] from='client.? 172.21.15.6:0/1196096918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:04.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:04 smithi006 bash[33918]: cluster 2023-08-28T01:53:02.766659+0000 mgr.y (mgr.14901) 7033 : cluster [DBG] pgmap v7040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:04.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:04 smithi006 bash[33918]: audit 2023-08-28T01:53:03.474154+0000 mon.c (mon.2) 2761 : audit [DBG] from='client.? 172.21.15.6:0/1196096918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:53:04.595Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:53:04.597Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:04.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:04 smithi006 bash[27292]: cluster 2023-08-28T01:53:02.766659+0000 mgr.y (mgr.14901) 7033 : cluster [DBG] pgmap v7040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:04.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:04 smithi006 bash[27292]: audit 2023-08-28T01:53:03.474154+0000 mon.c (mon.2) 2761 : audit [DBG] from='client.? 172.21.15.6:0/1196096918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:05.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:05 smithi130 bash[51302]: debug 2023-08-28T01:53:05.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:05.045742+0000) 2023-08-28T01:53:05.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:05 smithi130 bash[27579]: cluster 2023-08-28T01:53:04.767800+0000 mgr.y (mgr.14901) 7034 : cluster [DBG] pgmap v7041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:05.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:05 smithi006 bash[27292]: cluster 2023-08-28T01:53:04.767800+0000 mgr.y (mgr.14901) 7034 : cluster [DBG] pgmap v7041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:05.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:05 smithi006 bash[33918]: cluster 2023-08-28T01:53:04.767800+0000 mgr.y (mgr.14901) 7034 : cluster [DBG] pgmap v7041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:06.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:06 smithi130 bash[51302]: debug 2023-08-28T01:53:06.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:06.046105+0000) 2023-08-28T01:53:06.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:06 smithi130 bash[27579]: audit 2023-08-28T01:53:05.888445+0000 mon.c (mon.2) 2762 : audit [DBG] from='client.? 172.21.15.6:0/3545731041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:06.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:06 smithi006 bash[27292]: audit 2023-08-28T01:53:05.888445+0000 mon.c (mon.2) 2762 : audit [DBG] from='client.? 172.21.15.6:0/3545731041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:06.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:06 smithi006 bash[33918]: audit 2023-08-28T01:53:05.888445+0000 mon.c (mon.2) 2762 : audit [DBG] from='client.? 172.21.15.6:0/3545731041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:07.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:07 smithi130 bash[51302]: debug 2023-08-28T01:53:07.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:07.046442+0000) 2023-08-28T01:53:08.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:08 smithi130 bash[51302]: debug 2023-08-28T01:53:08.041+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:08.046667+0000) 2023-08-28T01:53:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:08 smithi006 bash[33918]: cluster 2023-08-28T01:53:06.768511+0000 mgr.y (mgr.14901) 7035 : cluster [DBG] pgmap v7042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:08 smithi006 bash[33918]: audit 2023-08-28T01:53:08.311743+0000 mon.a (mon.0) 4588 : audit [DBG] from='client.? 172.21.15.6:0/3682773314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:08.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:08 smithi006 bash[27292]: cluster 2023-08-28T01:53:06.768511+0000 mgr.y (mgr.14901) 7035 : cluster [DBG] pgmap v7042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:08 smithi006 bash[27292]: audit 2023-08-28T01:53:08.311743+0000 mon.a (mon.0) 4588 : audit [DBG] from='client.? 172.21.15.6:0/3682773314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:08.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:08 smithi130 bash[27579]: cluster 2023-08-28T01:53:06.768511+0000 mgr.y (mgr.14901) 7035 : cluster [DBG] pgmap v7042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:08 smithi130 bash[27579]: audit 2023-08-28T01:53:08.311743+0000 mon.a (mon.0) 4588 : audit [DBG] from='client.? 172.21.15.6:0/3682773314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:09.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:09 smithi130 bash[51302]: debug 2023-08-28T01:53:09.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:09.046843+0000) 2023-08-28T01:53:10.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:10 smithi130 bash[51302]: debug 2023-08-28T01:53:10.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:10.047068+0000) 2023-08-28T01:53:10.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:10 smithi130 bash[27579]: cluster 2023-08-28T01:53:08.769175+0000 mgr.y (mgr.14901) 7036 : cluster [DBG] pgmap v7043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:10.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:10 smithi130 bash[27579]: audit 2023-08-28T01:53:09.055724+0000 mon.a (mon.0) 4589 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:53:10.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:10 smithi130 bash[27579]: audit 2023-08-28T01:53:09.502386+0000 mon.a (mon.0) 4590 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:53:10.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:10 smithi130 bash[27579]: audit 2023-08-28T01:53:09.804194+0000 mon.b (mon.1) 789 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:53:10.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:10 smithi130 bash[27579]: audit 2023-08-28T01:53:09.806398+0000 mon.b (mon.1) 790 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:53:10.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:10 smithi130 bash[27579]: audit 2023-08-28T01:53:09.814819+0000 mon.a (mon.0) 4591 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:53:10.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:10 smithi006 bash[27292]: cluster 2023-08-28T01:53:08.769175+0000 mgr.y (mgr.14901) 7036 : cluster [DBG] pgmap v7043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:10.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:10 smithi006 bash[27292]: audit 2023-08-28T01:53:09.055724+0000 mon.a (mon.0) 4589 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:53:10.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:10 smithi006 bash[27292]: audit 2023-08-28T01:53:09.502386+0000 mon.a (mon.0) 4590 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:53:10.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:10 smithi006 bash[27292]: audit 2023-08-28T01:53:09.804194+0000 mon.b (mon.1) 789 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:53:10.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:10 smithi006 bash[27292]: audit 2023-08-28T01:53:09.806398+0000 mon.b (mon.1) 790 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:53:10.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:10 smithi006 bash[27292]: audit 2023-08-28T01:53:09.814819+0000 mon.a (mon.0) 4591 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:53:10.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:10 smithi006 bash[33918]: cluster 2023-08-28T01:53:08.769175+0000 mgr.y (mgr.14901) 7036 : cluster [DBG] pgmap v7043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:10.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:10 smithi006 bash[33918]: audit 2023-08-28T01:53:09.055724+0000 mon.a (mon.0) 4589 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:53:10.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:10 smithi006 bash[33918]: audit 2023-08-28T01:53:09.502386+0000 mon.a (mon.0) 4590 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:53:10.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:10 smithi006 bash[33918]: audit 2023-08-28T01:53:09.804194+0000 mon.b (mon.1) 789 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:53:10.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:10 smithi006 bash[33918]: audit 2023-08-28T01:53:09.806398+0000 mon.b (mon.1) 790 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:53:10.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:10 smithi006 bash[33918]: audit 2023-08-28T01:53:09.814819+0000 mon.a (mon.0) 4591 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:53:11.065 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:53:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:53:10] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:53:11.067 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:11 smithi006 bash[27292]: audit 2023-08-28T01:53:10.746319+0000 mon.c (mon.2) 2763 : audit [DBG] from='client.? 172.21.15.6:0/2521101932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:11.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:11 smithi130 bash[27579]: audit 2023-08-28T01:53:10.746319+0000 mon.c (mon.2) 2763 : audit [DBG] from='client.? 172.21.15.6:0/2521101932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:11.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:11 smithi130 bash[51302]: debug 2023-08-28T01:53:11.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:11.047361+0000) 2023-08-28T01:53:11.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:11 smithi006 bash[33918]: audit 2023-08-28T01:53:10.746319+0000 mon.c (mon.2) 2763 : audit [DBG] from='client.? 172.21.15.6:0/2521101932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:53:11.146Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:11.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:53:11.148Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:12.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:12 smithi130 bash[27579]: cluster 2023-08-28T01:53:10.770234+0000 mgr.y (mgr.14901) 7037 : cluster [DBG] pgmap v7044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:12.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:12 smithi130 bash[51302]: debug 2023-08-28T01:53:12.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:12.047645+0000) 2023-08-28T01:53:12.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:12 smithi006 bash[33918]: cluster 2023-08-28T01:53:10.770234+0000 mgr.y (mgr.14901) 7037 : cluster [DBG] pgmap v7044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:12.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:12 smithi006 bash[27292]: cluster 2023-08-28T01:53:10.770234+0000 mgr.y (mgr.14901) 7037 : cluster [DBG] pgmap v7044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:53:12.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:53:12.712Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:13.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:13 smithi130 bash[51302]: debug 2023-08-28T01:53:13.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:13.047926+0000) 2023-08-28T01:53:13.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:13 smithi006 bash[33918]: audit 2023-08-28T01:53:13.163060+0000 mon.a (mon.0) 4592 : audit [DBG] from='client.? 172.21.15.6:0/4194900661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:13.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:13 smithi006 bash[27292]: audit 2023-08-28T01:53:13.163060+0000 mon.a (mon.0) 4592 : audit [DBG] from='client.? 172.21.15.6:0/4194900661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:13.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:13 smithi130 bash[27579]: audit 2023-08-28T01:53:13.163060+0000 mon.a (mon.0) 4592 : audit [DBG] from='client.? 172.21.15.6:0/4194900661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:14.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:14 smithi130 bash[51302]: debug 2023-08-28T01:53:14.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:14.048232+0000) 2023-08-28T01:53:14.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:14 smithi006 bash[33918]: cluster 2023-08-28T01:53:12.770909+0000 mgr.y (mgr.14901) 7038 : cluster [DBG] pgmap v7045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:14.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:14 smithi006 bash[27292]: cluster 2023-08-28T01:53:12.770909+0000 mgr.y (mgr.14901) 7038 : cluster [DBG] pgmap v7045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:53:14.596Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:53:14.598Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:14.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:14 smithi130 bash[27579]: cluster 2023-08-28T01:53:12.770909+0000 mgr.y (mgr.14901) 7038 : cluster [DBG] pgmap v7045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:15.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:15 smithi130 bash[51302]: debug 2023-08-28T01:53:15.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:15.048506+0000) 2023-08-28T01:53:16.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:16 smithi130 bash[51302]: debug 2023-08-28T01:53:16.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:16.048782+0000) 2023-08-28T01:53:16.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:16 smithi006 bash[27292]: cluster 2023-08-28T01:53:14.772032+0000 mgr.y (mgr.14901) 7039 : cluster [DBG] pgmap v7046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:16.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:16 smithi006 bash[27292]: audit 2023-08-28T01:53:15.576487+0000 mon.c (mon.2) 2764 : audit [DBG] from='client.? 172.21.15.6:0/1473201345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:16.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:16 smithi006 bash[33918]: cluster 2023-08-28T01:53:14.772032+0000 mgr.y (mgr.14901) 7039 : cluster [DBG] pgmap v7046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:16 smithi006 bash[33918]: audit 2023-08-28T01:53:15.576487+0000 mon.c (mon.2) 2764 : audit [DBG] from='client.? 172.21.15.6:0/1473201345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:16.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:16 smithi130 bash[27579]: cluster 2023-08-28T01:53:14.772032+0000 mgr.y (mgr.14901) 7039 : cluster [DBG] pgmap v7046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:16.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:16 smithi130 bash[27579]: audit 2023-08-28T01:53:15.576487+0000 mon.c (mon.2) 2764 : audit [DBG] from='client.? 172.21.15.6:0/1473201345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:17.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:17 smithi130 bash[51302]: debug 2023-08-28T01:53:17.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:17.049059+0000) 2023-08-28T01:53:18.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:18 smithi130 bash[51302]: debug 2023-08-28T01:53:18.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:18.049357+0000) 2023-08-28T01:53:18.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:18 smithi006 bash[33918]: cluster 2023-08-28T01:53:16.772669+0000 mgr.y (mgr.14901) 7040 : cluster [DBG] pgmap v7047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:18.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:18 smithi006 bash[33918]: audit 2023-08-28T01:53:17.995227+0000 mon.c (mon.2) 2765 : audit [DBG] from='client.? 172.21.15.6:0/2709689586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:18 smithi006 bash[27292]: cluster 2023-08-28T01:53:16.772669+0000 mgr.y (mgr.14901) 7040 : cluster [DBG] pgmap v7047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:18.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:18 smithi006 bash[27292]: audit 2023-08-28T01:53:17.995227+0000 mon.c (mon.2) 2765 : audit [DBG] from='client.? 172.21.15.6:0/2709689586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:18.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:18 smithi130 bash[27579]: cluster 2023-08-28T01:53:16.772669+0000 mgr.y (mgr.14901) 7040 : cluster [DBG] pgmap v7047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:18.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:18 smithi130 bash[27579]: audit 2023-08-28T01:53:17.995227+0000 mon.c (mon.2) 2765 : audit [DBG] from='client.? 172.21.15.6:0/2709689586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:19.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:19 smithi130 bash[51302]: debug 2023-08-28T01:53:19.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:19.049639+0000) 2023-08-28T01:53:20.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:20 smithi130 bash[51302]: debug 2023-08-28T01:53:20.049+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:20.049937+0000) 2023-08-28T01:53:20.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:20 smithi006 bash[33918]: cluster 2023-08-28T01:53:18.773326+0000 mgr.y (mgr.14901) 7041 : cluster [DBG] pgmap v7048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:20.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:20 smithi006 bash[27292]: cluster 2023-08-28T01:53:18.773326+0000 mgr.y (mgr.14901) 7041 : cluster [DBG] pgmap v7048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:20.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:20 smithi130 bash[27579]: cluster 2023-08-28T01:53:18.773326+0000 mgr.y (mgr.14901) 7041 : cluster [DBG] pgmap v7048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:21.146 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:53:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:53:20] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T01:53:21.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:21 smithi130 bash[51302]: debug 2023-08-28T01:53:21.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:21.050248+0000) 2023-08-28T01:53:21.417 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:53:21.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:21.417 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:53:21.149Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:21.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:21 smithi006 bash[27292]: audit 2023-08-28T01:53:20.418664+0000 mon.c (mon.2) 2766 : audit [DBG] from='client.? 172.21.15.6:0/1923635542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:21.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:21 smithi006 bash[33918]: audit 2023-08-28T01:53:20.418664+0000 mon.c (mon.2) 2766 : audit [DBG] from='client.? 172.21.15.6:0/1923635542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:21.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:21 smithi130 bash[27579]: audit 2023-08-28T01:53:20.418664+0000 mon.c (mon.2) 2766 : audit [DBG] from='client.? 172.21.15.6:0/1923635542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:22.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:22 smithi130 bash[51302]: debug 2023-08-28T01:53:22.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:22.050502+0000) 2023-08-28T01:53:22.711 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:22 smithi006 bash[27292]: cluster 2023-08-28T01:53:20.774385+0000 mgr.y (mgr.14901) 7042 : cluster [DBG] pgmap v7049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:22.711 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:22 smithi006 bash[33918]: cluster 2023-08-28T01:53:20.774385+0000 mgr.y (mgr.14901) 7042 : cluster [DBG] pgmap v7049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:22 smithi130 bash[27579]: cluster 2023-08-28T01:53:20.774385+0000 mgr.y (mgr.14901) 7042 : cluster [DBG] pgmap v7049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:53:22.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:53:22.713Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:23.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:23 smithi130 bash[51302]: debug 2023-08-28T01:53:23.045+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:23.050779+0000) 2023-08-28T01:53:23.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:23 smithi006 bash[27292]: audit 2023-08-28T01:53:22.839016+0000 mon.c (mon.2) 2767 : audit [DBG] from='client.? 172.21.15.6:0/2592896426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:23.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:23 smithi006 bash[33918]: audit 2023-08-28T01:53:22.839016+0000 mon.c (mon.2) 2767 : audit [DBG] from='client.? 172.21.15.6:0/2592896426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:23.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:23 smithi130 bash[27579]: audit 2023-08-28T01:53:22.839016+0000 mon.c (mon.2) 2767 : audit [DBG] from='client.? 172.21.15.6:0/2592896426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:24.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:24 smithi130 bash[51302]: debug 2023-08-28T01:53:24.049+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:24.051070+0000) 2023-08-28T01:53:24.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:24 smithi006 bash[33918]: cluster 2023-08-28T01:53:22.774968+0000 mgr.y (mgr.14901) 7043 : cluster [DBG] pgmap v7050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:24.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:24 smithi006 bash[27292]: cluster 2023-08-28T01:53:22.774968+0000 mgr.y (mgr.14901) 7043 : cluster [DBG] pgmap v7050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:53:24.597Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:53:24.599Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:24.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:24 smithi130 bash[27579]: cluster 2023-08-28T01:53:22.774968+0000 mgr.y (mgr.14901) 7043 : cluster [DBG] pgmap v7050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:25.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:25 smithi130 bash[51302]: debug 2023-08-28T01:53:25.049+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:25.051400+0000) 2023-08-28T01:53:25.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:25 smithi006 bash[27292]: audit 2023-08-28T01:53:25.258587+0000 mon.c (mon.2) 2768 : audit [DBG] from='client.? 172.21.15.6:0/3932287412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:25.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:25 smithi006 bash[33918]: audit 2023-08-28T01:53:25.258587+0000 mon.c (mon.2) 2768 : audit [DBG] from='client.? 172.21.15.6:0/3932287412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:25.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:25 smithi130 bash[27579]: audit 2023-08-28T01:53:25.258587+0000 mon.c (mon.2) 2768 : audit [DBG] from='client.? 172.21.15.6:0/3932287412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:26.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:26 smithi130 bash[51302]: debug 2023-08-28T01:53:26.049+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:26.051675+0000) 2023-08-28T01:53:26.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:26 smithi006 bash[27292]: cluster 2023-08-28T01:53:24.776105+0000 mgr.y (mgr.14901) 7044 : cluster [DBG] pgmap v7051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:26 smithi006 bash[33918]: cluster 2023-08-28T01:53:24.776105+0000 mgr.y (mgr.14901) 7044 : cluster [DBG] pgmap v7051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:26.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:26 smithi130 bash[27579]: cluster 2023-08-28T01:53:24.776105+0000 mgr.y (mgr.14901) 7044 : cluster [DBG] pgmap v7051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:27.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:27 smithi130 bash[51302]: debug 2023-08-28T01:53:27.049+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:27.051928+0000) 2023-08-28T01:53:28.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:28 smithi130 bash[51302]: debug 2023-08-28T01:53:28.049+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:28.052137+0000) 2023-08-28T01:53:28.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:28 smithi006 bash[33918]: cluster 2023-08-28T01:53:26.776775+0000 mgr.y (mgr.14901) 7045 : cluster [DBG] pgmap v7052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:28.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:28 smithi006 bash[33918]: audit 2023-08-28T01:53:27.676477+0000 mon.a (mon.0) 4593 : audit [DBG] from='client.? 172.21.15.6:0/3831875197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:28 smithi006 bash[27292]: cluster 2023-08-28T01:53:26.776775+0000 mgr.y (mgr.14901) 7045 : cluster [DBG] pgmap v7052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:28 smithi006 bash[27292]: audit 2023-08-28T01:53:27.676477+0000 mon.a (mon.0) 4593 : audit [DBG] from='client.? 172.21.15.6:0/3831875197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:28.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:28 smithi130 bash[27579]: cluster 2023-08-28T01:53:26.776775+0000 mgr.y (mgr.14901) 7045 : cluster [DBG] pgmap v7052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:28.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:28 smithi130 bash[27579]: audit 2023-08-28T01:53:27.676477+0000 mon.a (mon.0) 4593 : audit [DBG] from='client.? 172.21.15.6:0/3831875197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:29.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:29 smithi130 bash[51302]: debug 2023-08-28T01:53:29.049+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:29.052482+0000) 2023-08-28T01:53:30.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:30 smithi130 bash[51302]: debug 2023-08-28T01:53:30.049+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:30.052757+0000) 2023-08-28T01:53:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:30 smithi006 bash[33918]: cluster 2023-08-28T01:53:28.777415+0000 mgr.y (mgr.14901) 7046 : cluster [DBG] pgmap v7053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:30 smithi006 bash[33918]: audit 2023-08-28T01:53:30.095077+0000 mon.c (mon.2) 2769 : audit [DBG] from='client.? 172.21.15.6:0/160250467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:30.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:30 smithi006 bash[27292]: cluster 2023-08-28T01:53:28.777415+0000 mgr.y (mgr.14901) 7046 : cluster [DBG] pgmap v7053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:30.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:30 smithi006 bash[27292]: audit 2023-08-28T01:53:30.095077+0000 mon.c (mon.2) 2769 : audit [DBG] from='client.? 172.21.15.6:0/160250467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:30.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:30 smithi130 bash[27579]: cluster 2023-08-28T01:53:28.777415+0000 mgr.y (mgr.14901) 7046 : cluster [DBG] pgmap v7053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:30 smithi130 bash[27579]: audit 2023-08-28T01:53:30.095077+0000 mon.c (mon.2) 2769 : audit [DBG] from='client.? 172.21.15.6:0/160250467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:31.147 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:53:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:53:30] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:53:31.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:31 smithi130 bash[51302]: debug 2023-08-28T01:53:31.049+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:31.053081+0000) 2023-08-28T01:53:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:53:31.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:53:31.149Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:32.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:32 smithi130 bash[51302]: debug 2023-08-28T01:53:32.049+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:32.053303+0000) 2023-08-28T01:53:32.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:32 smithi130 bash[27579]: cluster 2023-08-28T01:53:30.778478+0000 mgr.y (mgr.14901) 7047 : cluster [DBG] pgmap v7054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:32.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:32 smithi006 bash[33918]: cluster 2023-08-28T01:53:30.778478+0000 mgr.y (mgr.14901) 7047 : cluster [DBG] pgmap v7054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:32.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:32 smithi006 bash[27292]: cluster 2023-08-28T01:53:30.778478+0000 mgr.y (mgr.14901) 7047 : cluster [DBG] pgmap v7054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:53:32.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:53:32.714Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:33.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:33 smithi130 bash[51302]: debug 2023-08-28T01:53:33.053+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:33.053579+0000) 2023-08-28T01:53:33.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:33 smithi130 bash[27579]: audit 2023-08-28T01:53:32.508863+0000 mon.a (mon.0) 4594 : audit [DBG] from='client.? 172.21.15.6:0/1626809118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:33 smithi006 bash[27292]: audit 2023-08-28T01:53:32.508863+0000 mon.a (mon.0) 4594 : audit [DBG] from='client.? 172.21.15.6:0/1626809118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:33.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:33 smithi006 bash[33918]: audit 2023-08-28T01:53:32.508863+0000 mon.a (mon.0) 4594 : audit [DBG] from='client.? 172.21.15.6:0/1626809118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:34.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:34 smithi130 bash[51302]: debug 2023-08-28T01:53:34.049+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:34.053786+0000) 2023-08-28T01:53:34.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:34 smithi130 bash[27579]: cluster 2023-08-28T01:53:32.779130+0000 mgr.y (mgr.14901) 7048 : cluster [DBG] pgmap v7055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:34.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:34 smithi006 bash[27292]: cluster 2023-08-28T01:53:32.779130+0000 mgr.y (mgr.14901) 7048 : cluster [DBG] pgmap v7055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:34.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:34 smithi006 bash[33918]: cluster 2023-08-28T01:53:32.779130+0000 mgr.y (mgr.14901) 7048 : cluster [DBG] pgmap v7055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:53:34.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:53:34.600Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:35.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:35 smithi130 bash[51302]: debug 2023-08-28T01:53:35.053+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:35.054088+0000) 2023-08-28T01:53:35.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:35 smithi130 bash[27579]: cluster 2023-08-28T01:53:34.780198+0000 mgr.y (mgr.14901) 7049 : cluster [DBG] pgmap v7056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:35.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:35 smithi130 bash[27579]: audit 2023-08-28T01:53:34.918688+0000 mon.c (mon.2) 2770 : audit [DBG] from='client.? 172.21.15.6:0/348682630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:35.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:35 smithi006 bash[27292]: cluster 2023-08-28T01:53:34.780198+0000 mgr.y (mgr.14901) 7049 : cluster [DBG] pgmap v7056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:35.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:35 smithi006 bash[27292]: audit 2023-08-28T01:53:34.918688+0000 mon.c (mon.2) 2770 : audit [DBG] from='client.? 172.21.15.6:0/348682630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:35.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:35 smithi006 bash[33918]: cluster 2023-08-28T01:53:34.780198+0000 mgr.y (mgr.14901) 7049 : cluster [DBG] pgmap v7056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:35.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:35 smithi006 bash[33918]: audit 2023-08-28T01:53:34.918688+0000 mon.c (mon.2) 2770 : audit [DBG] from='client.? 172.21.15.6:0/348682630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:36.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:36 smithi130 bash[51302]: debug 2023-08-28T01:53:36.053+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:36.054409+0000) 2023-08-28T01:53:37.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:37 smithi130 bash[51302]: debug 2023-08-28T01:53:37.053+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:37.054705+0000) 2023-08-28T01:53:38.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:37 smithi130 bash[27579]: cluster 2023-08-28T01:53:36.780887+0000 mgr.y (mgr.14901) 7050 : cluster [DBG] pgmap v7057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:38.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:37 smithi130 bash[27579]: audit 2023-08-28T01:53:37.330349+0000 mon.a (mon.0) 4595 : audit [DBG] from='client.? 172.21.15.6:0/793998116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:38 smithi130 bash[51302]: debug 2023-08-28T01:53:38.053+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:38.054980+0000) 2023-08-28T01:53:38.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:37 smithi006 bash[33918]: cluster 2023-08-28T01:53:36.780887+0000 mgr.y (mgr.14901) 7050 : cluster [DBG] pgmap v7057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:38.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:37 smithi006 bash[33918]: audit 2023-08-28T01:53:37.330349+0000 mon.a (mon.0) 4595 : audit [DBG] from='client.? 172.21.15.6:0/793998116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:38.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:37 smithi006 bash[27292]: cluster 2023-08-28T01:53:36.780887+0000 mgr.y (mgr.14901) 7050 : cluster [DBG] pgmap v7057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:38.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:37 smithi006 bash[27292]: audit 2023-08-28T01:53:37.330349+0000 mon.a (mon.0) 4595 : audit [DBG] from='client.? 172.21.15.6:0/793998116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:39.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:39 smithi130 bash[51302]: debug 2023-08-28T01:53:39.053+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:39.055283+0000) 2023-08-28T01:53:40.114 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:40 smithi130 bash[51302]: debug 2023-08-28T01:53:40.053+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:40.055572+0000) 2023-08-28T01:53:40.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:39 smithi130 bash[27579]: cluster 2023-08-28T01:53:38.781471+0000 mgr.y (mgr.14901) 7051 : cluster [DBG] pgmap v7058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:40.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:39 smithi130 bash[27579]: audit 2023-08-28T01:53:39.750229+0000 mon.a (mon.0) 4596 : audit [DBG] from='client.? 172.21.15.6:0/1499210148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:40.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:39 smithi006 bash[33918]: cluster 2023-08-28T01:53:38.781471+0000 mgr.y (mgr.14901) 7051 : cluster [DBG] pgmap v7058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:40.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:39 smithi006 bash[33918]: audit 2023-08-28T01:53:39.750229+0000 mon.a (mon.0) 4596 : audit [DBG] from='client.? 172.21.15.6:0/1499210148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:39 smithi006 bash[27292]: cluster 2023-08-28T01:53:38.781471+0000 mgr.y (mgr.14901) 7051 : cluster [DBG] pgmap v7058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:40.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:39 smithi006 bash[27292]: audit 2023-08-28T01:53:39.750229+0000 mon.a (mon.0) 4596 : audit [DBG] from='client.? 172.21.15.6:0/1499210148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:41.148 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:53:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:53:40] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:53:41.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:41 smithi130 bash[51302]: debug 2023-08-28T01:53:41.053+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:41.055951+0000) 2023-08-28T01:53:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:53:41.148Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:53:41.150Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:42.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:41 smithi130 bash[27579]: cluster 2023-08-28T01:53:40.782571+0000 mgr.y (mgr.14901) 7052 : cluster [DBG] pgmap v7059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:42 smithi130 bash[51302]: debug 2023-08-28T01:53:42.053+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:42.056241+0000) 2023-08-28T01:53:42.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:41 smithi006 bash[27292]: cluster 2023-08-28T01:53:40.782571+0000 mgr.y (mgr.14901) 7052 : cluster [DBG] pgmap v7059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:42.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:41 smithi006 bash[33918]: cluster 2023-08-28T01:53:40.782571+0000 mgr.y (mgr.14901) 7052 : cluster [DBG] pgmap v7059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:42.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:42 smithi006 bash[33918]: audit 2023-08-28T01:53:42.160432+0000 mon.c (mon.2) 2771 : audit [DBG] from='client.? 172.21.15.6:0/508518502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:42.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:42 smithi006 bash[27292]: audit 2023-08-28T01:53:42.160432+0000 mon.c (mon.2) 2771 : audit [DBG] from='client.? 172.21.15.6:0/508518502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:53:42.713Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:53:42.715Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:43.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:42 smithi130 bash[27579]: audit 2023-08-28T01:53:42.160432+0000 mon.c (mon.2) 2771 : audit [DBG] from='client.? 172.21.15.6:0/508518502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:43.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:43 smithi130 bash[51302]: debug 2023-08-28T01:53:43.053+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:43.056466+0000) 2023-08-28T01:53:44.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:44 smithi130 bash[51302]: debug 2023-08-28T01:53:44.053+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:44.056752+0000) 2023-08-28T01:53:44.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:44 smithi006 bash[27292]: cluster 2023-08-28T01:53:42.783219+0000 mgr.y (mgr.14901) 7053 : cluster [DBG] pgmap v7060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:44.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:44 smithi006 bash[33918]: cluster 2023-08-28T01:53:42.783219+0000 mgr.y (mgr.14901) 7053 : cluster [DBG] pgmap v7060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:53:44.598Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:44.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:53:44.600Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:44.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:44 smithi130 bash[27579]: cluster 2023-08-28T01:53:42.783219+0000 mgr.y (mgr.14901) 7053 : cluster [DBG] pgmap v7060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:45.368 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:45 smithi130 bash[51302]: debug 2023-08-28T01:53:45.053+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:45.057017+0000) 2023-08-28T01:53:45.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:45 smithi006 bash[27292]: audit 2023-08-28T01:53:44.585421+0000 mon.c (mon.2) 2772 : audit [DBG] from='client.? 172.21.15.6:0/3652185481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:45.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:45 smithi006 bash[33918]: audit 2023-08-28T01:53:44.585421+0000 mon.c (mon.2) 2772 : audit [DBG] from='client.? 172.21.15.6:0/3652185481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:45.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:45 smithi130 bash[27579]: audit 2023-08-28T01:53:44.585421+0000 mon.c (mon.2) 2772 : audit [DBG] from='client.? 172.21.15.6:0/3652185481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:46.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:46 smithi130 bash[51302]: debug 2023-08-28T01:53:46.053+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:46.057320+0000) 2023-08-28T01:53:46.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:46 smithi006 bash[27292]: cluster 2023-08-28T01:53:44.784366+0000 mgr.y (mgr.14901) 7054 : cluster [DBG] pgmap v7061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:46.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:46 smithi006 bash[33918]: cluster 2023-08-28T01:53:44.784366+0000 mgr.y (mgr.14901) 7054 : cluster [DBG] pgmap v7061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:46.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:46 smithi130 bash[27579]: cluster 2023-08-28T01:53:44.784366+0000 mgr.y (mgr.14901) 7054 : cluster [DBG] pgmap v7061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:47.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:47 smithi130 bash[51302]: debug 2023-08-28T01:53:47.057+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:47.057624+0000) 2023-08-28T01:53:47.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:47 smithi006 bash[27292]: audit 2023-08-28T01:53:47.008572+0000 mon.a (mon.0) 4597 : audit [DBG] from='client.? 172.21.15.6:0/3089962999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:47.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:47 smithi006 bash[33918]: audit 2023-08-28T01:53:47.008572+0000 mon.a (mon.0) 4597 : audit [DBG] from='client.? 172.21.15.6:0/3089962999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:47.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:47 smithi130 bash[27579]: audit 2023-08-28T01:53:47.008572+0000 mon.a (mon.0) 4597 : audit [DBG] from='client.? 172.21.15.6:0/3089962999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:48.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:48 smithi130 bash[51302]: debug 2023-08-28T01:53:48.057+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:48.057905+0000) 2023-08-28T01:53:48.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:48 smithi006 bash[27292]: cluster 2023-08-28T01:53:46.785016+0000 mgr.y (mgr.14901) 7055 : cluster [DBG] pgmap v7062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:48.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:48 smithi006 bash[33918]: cluster 2023-08-28T01:53:46.785016+0000 mgr.y (mgr.14901) 7055 : cluster [DBG] pgmap v7062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:48.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:48 smithi130 bash[27579]: cluster 2023-08-28T01:53:46.785016+0000 mgr.y (mgr.14901) 7055 : cluster [DBG] pgmap v7062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:49.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:49 smithi130 bash[51302]: debug 2023-08-28T01:53:49.053+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:49.058193+0000) 2023-08-28T01:53:49.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:49 smithi006 bash[27292]: audit 2023-08-28T01:53:49.419443+0000 mon.c (mon.2) 2773 : audit [DBG] from='client.? 172.21.15.6:0/1138394406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:49.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:49 smithi006 bash[33918]: audit 2023-08-28T01:53:49.419443+0000 mon.c (mon.2) 2773 : audit [DBG] from='client.? 172.21.15.6:0/1138394406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:49.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:49 smithi130 bash[27579]: audit 2023-08-28T01:53:49.419443+0000 mon.c (mon.2) 2773 : audit [DBG] from='client.? 172.21.15.6:0/1138394406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:50.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:50 smithi130 bash[51302]: debug 2023-08-28T01:53:50.053+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:50.058480+0000) 2023-08-28T01:53:50.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:50 smithi006 bash[27292]: cluster 2023-08-28T01:53:48.785656+0000 mgr.y (mgr.14901) 7056 : cluster [DBG] pgmap v7063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:50.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:50 smithi006 bash[33918]: cluster 2023-08-28T01:53:48.785656+0000 mgr.y (mgr.14901) 7056 : cluster [DBG] pgmap v7063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:50.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:50 smithi130 bash[27579]: cluster 2023-08-28T01:53:48.785656+0000 mgr.y (mgr.14901) 7056 : cluster [DBG] pgmap v7063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:51.148 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:53:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:53:50] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:53:51.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:51 smithi130 bash[51302]: debug 2023-08-28T01:53:51.057+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:51.058778+0000) 2023-08-28T01:53:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:53:51.148Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:53:51.150Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:52.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:52 smithi130 bash[51302]: debug 2023-08-28T01:53:52.057+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:52.059067+0000) 2023-08-28T01:53:52.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:52 smithi006 bash[27292]: cluster 2023-08-28T01:53:50.786778+0000 mgr.y (mgr.14901) 7057 : cluster [DBG] pgmap v7064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:52.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:52 smithi006 bash[27292]: audit 2023-08-28T01:53:51.836043+0000 mon.c (mon.2) 2774 : audit [DBG] from='client.? 172.21.15.6:0/1136944409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:52.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:52 smithi006 bash[33918]: cluster 2023-08-28T01:53:50.786778+0000 mgr.y (mgr.14901) 7057 : cluster [DBG] pgmap v7064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:52.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:52 smithi006 bash[33918]: audit 2023-08-28T01:53:51.836043+0000 mon.c (mon.2) 2774 : audit [DBG] from='client.? 172.21.15.6:0/1136944409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:52.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:52 smithi130 bash[27579]: cluster 2023-08-28T01:53:50.786778+0000 mgr.y (mgr.14901) 7057 : cluster [DBG] pgmap v7064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:52.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:52 smithi130 bash[27579]: audit 2023-08-28T01:53:51.836043+0000 mon.c (mon.2) 2774 : audit [DBG] from='client.? 172.21.15.6:0/1136944409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:53:52.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:53:52.716Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:53.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:53 smithi130 bash[51302]: debug 2023-08-28T01:53:53.057+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:53.059371+0000) 2023-08-28T01:53:54.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:54 smithi130 bash[51302]: debug 2023-08-28T01:53:54.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:54.059609+0000) 2023-08-28T01:53:54.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:54 smithi006 bash[27292]: cluster 2023-08-28T01:53:52.787526+0000 mgr.y (mgr.14901) 7058 : cluster [DBG] pgmap v7065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:54.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:54 smithi006 bash[27292]: audit 2023-08-28T01:53:54.249214+0000 mon.c (mon.2) 2775 : audit [DBG] from='client.? 172.21.15.6:0/1856126868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:54.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:54 smithi006 bash[33918]: cluster 2023-08-28T01:53:52.787526+0000 mgr.y (mgr.14901) 7058 : cluster [DBG] pgmap v7065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:54.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:54 smithi006 bash[33918]: audit 2023-08-28T01:53:54.249214+0000 mon.c (mon.2) 2775 : audit [DBG] from='client.? 172.21.15.6:0/1856126868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:53:54.599Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:53:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:53:54.602Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:53:54.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:54 smithi130 bash[27579]: cluster 2023-08-28T01:53:52.787526+0000 mgr.y (mgr.14901) 7058 : cluster [DBG] pgmap v7065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:54.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:54 smithi130 bash[27579]: audit 2023-08-28T01:53:54.249214+0000 mon.c (mon.2) 2775 : audit [DBG] from='client.? 172.21.15.6:0/1856126868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:55.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:55 smithi130 bash[51302]: debug 2023-08-28T01:53:55.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:55.059893+0000) 2023-08-28T01:53:56.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:56 smithi130 bash[51302]: debug 2023-08-28T01:53:56.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:56.060161+0000) 2023-08-28T01:53:56.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:56 smithi006 bash[33918]: cluster 2023-08-28T01:53:54.788675+0000 mgr.y (mgr.14901) 7059 : cluster [DBG] pgmap v7066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:56.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:56 smithi006 bash[27292]: cluster 2023-08-28T01:53:54.788675+0000 mgr.y (mgr.14901) 7059 : cluster [DBG] pgmap v7066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:56 smithi130 bash[27579]: cluster 2023-08-28T01:53:54.788675+0000 mgr.y (mgr.14901) 7059 : cluster [DBG] pgmap v7066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:57.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:57 smithi130 bash[51302]: debug 2023-08-28T01:53:57.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:57.060506+0000) 2023-08-28T01:53:57.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:57 smithi006 bash[33918]: audit 2023-08-28T01:53:56.660310+0000 mon.c (mon.2) 2776 : audit [DBG] from='client.? 172.21.15.6:0/95261386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:57.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:57 smithi006 bash[27292]: audit 2023-08-28T01:53:56.660310+0000 mon.c (mon.2) 2776 : audit [DBG] from='client.? 172.21.15.6:0/95261386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:57.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:57 smithi130 bash[27579]: audit 2023-08-28T01:53:56.660310+0000 mon.c (mon.2) 2776 : audit [DBG] from='client.? 172.21.15.6:0/95261386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:58.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:58 smithi130 bash[51302]: debug 2023-08-28T01:53:58.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:58.060781+0000) 2023-08-28T01:53:58.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:58 smithi006 bash[27292]: cluster 2023-08-28T01:53:56.789355+0000 mgr.y (mgr.14901) 7060 : cluster [DBG] pgmap v7067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:58.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:58 smithi006 bash[33918]: cluster 2023-08-28T01:53:56.789355+0000 mgr.y (mgr.14901) 7060 : cluster [DBG] pgmap v7067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:58.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:58 smithi130 bash[27579]: cluster 2023-08-28T01:53:56.789355+0000 mgr.y (mgr.14901) 7060 : cluster [DBG] pgmap v7067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:53:59.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:53:59 smithi130 bash[51302]: debug 2023-08-28T01:53:59.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:59.060982+0000) 2023-08-28T01:53:59.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:53:59 smithi006 bash[33918]: audit 2023-08-28T01:53:59.080621+0000 mon.c (mon.2) 2777 : audit [DBG] from='client.? 172.21.15.6:0/2475619521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:59.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:53:59 smithi006 bash[27292]: audit 2023-08-28T01:53:59.080621+0000 mon.c (mon.2) 2777 : audit [DBG] from='client.? 172.21.15.6:0/2475619521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:59.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:53:59 smithi130 bash[27579]: audit 2023-08-28T01:53:59.080621+0000 mon.c (mon.2) 2777 : audit [DBG] from='client.? 172.21.15.6:0/2475619521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:00.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:00 smithi130 bash[51302]: debug 2023-08-28T01:54:00.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:00.061244+0000) 2023-08-28T01:54:00.752 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:00 smithi006 bash[27292]: cluster 2023-08-28T01:53:58.789979+0000 mgr.y (mgr.14901) 7061 : cluster [DBG] pgmap v7068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:00.752 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:00 smithi006 bash[33918]: cluster 2023-08-28T01:53:58.789979+0000 mgr.y (mgr.14901) 7061 : cluster [DBG] pgmap v7068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:00 smithi130 bash[27579]: cluster 2023-08-28T01:53:58.789979+0000 mgr.y (mgr.14901) 7061 : cluster [DBG] pgmap v7068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:01.149 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:54:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:54:00] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:54:01.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:01 smithi130 bash[51302]: debug 2023-08-28T01:54:01.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:01.061545+0000) 2023-08-28T01:54:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:54:01.149Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:54:01.151Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:02.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:02 smithi130 bash[51302]: debug 2023-08-28T01:54:02.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:02.061817+0000) 2023-08-28T01:54:02.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:02 smithi130 bash[27579]: cluster 2023-08-28T01:54:00.791139+0000 mgr.y (mgr.14901) 7062 : cluster [DBG] pgmap v7069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:02 smithi130 bash[27579]: audit 2023-08-28T01:54:01.493985+0000 mon.a (mon.0) 4598 : audit [DBG] from='client.? 172.21.15.6:0/3207107175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:02.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:02 smithi006 bash[27292]: cluster 2023-08-28T01:54:00.791139+0000 mgr.y (mgr.14901) 7062 : cluster [DBG] pgmap v7069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:02.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:02 smithi006 bash[27292]: audit 2023-08-28T01:54:01.493985+0000 mon.a (mon.0) 4598 : audit [DBG] from='client.? 172.21.15.6:0/3207107175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:54:02.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:54:02.717Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:02.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:02 smithi006 bash[33918]: cluster 2023-08-28T01:54:00.791139+0000 mgr.y (mgr.14901) 7062 : cluster [DBG] pgmap v7069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:02.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:02 smithi006 bash[33918]: audit 2023-08-28T01:54:01.493985+0000 mon.a (mon.0) 4598 : audit [DBG] from='client.? 172.21.15.6:0/3207107175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:03.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:03 smithi130 bash[51302]: debug 2023-08-28T01:54:03.056+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:03.062064+0000) 2023-08-28T01:54:04.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:04 smithi130 bash[51302]: debug 2023-08-28T01:54:04.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:04.062351+0000) 2023-08-28T01:54:04.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:04 smithi130 bash[27579]: cluster 2023-08-28T01:54:02.791896+0000 mgr.y (mgr.14901) 7063 : cluster [DBG] pgmap v7070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:04.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:04 smithi130 bash[27579]: audit 2023-08-28T01:54:03.905833+0000 mon.c (mon.2) 2778 : audit [DBG] from='client.? 172.21.15.6:0/339750160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:04.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:04 smithi006 bash[33918]: cluster 2023-08-28T01:54:02.791896+0000 mgr.y (mgr.14901) 7063 : cluster [DBG] pgmap v7070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:04.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:04 smithi006 bash[33918]: audit 2023-08-28T01:54:03.905833+0000 mon.c (mon.2) 2778 : audit [DBG] from='client.? 172.21.15.6:0/339750160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:54:04.600Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:54:04.602Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:04.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:04 smithi006 bash[27292]: cluster 2023-08-28T01:54:02.791896+0000 mgr.y (mgr.14901) 7063 : cluster [DBG] pgmap v7070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:04.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:04 smithi006 bash[27292]: audit 2023-08-28T01:54:03.905833+0000 mon.c (mon.2) 2778 : audit [DBG] from='client.? 172.21.15.6:0/339750160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:05.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:05 smithi130 bash[51302]: debug 2023-08-28T01:54:05.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:05.062630+0000) 2023-08-28T01:54:05.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:05 smithi130 bash[27579]: cluster 2023-08-28T01:54:04.793072+0000 mgr.y (mgr.14901) 7064 : cluster [DBG] pgmap v7071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:05.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:05 smithi006 bash[33918]: cluster 2023-08-28T01:54:04.793072+0000 mgr.y (mgr.14901) 7064 : cluster [DBG] pgmap v7071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:05.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:05 smithi006 bash[27292]: cluster 2023-08-28T01:54:04.793072+0000 mgr.y (mgr.14901) 7064 : cluster [DBG] pgmap v7071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:06.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:06 smithi130 bash[51302]: debug 2023-08-28T01:54:06.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:06.062958+0000) 2023-08-28T01:54:06.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:06 smithi130 bash[27579]: audit 2023-08-28T01:54:06.314552+0000 mon.c (mon.2) 2779 : audit [DBG] from='client.? 172.21.15.6:0/3814845003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:06.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:06 smithi006 bash[27292]: audit 2023-08-28T01:54:06.314552+0000 mon.c (mon.2) 2779 : audit [DBG] from='client.? 172.21.15.6:0/3814845003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:06.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:06 smithi006 bash[33918]: audit 2023-08-28T01:54:06.314552+0000 mon.c (mon.2) 2779 : audit [DBG] from='client.? 172.21.15.6:0/3814845003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:07.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:07 smithi130 bash[51302]: debug 2023-08-28T01:54:07.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:07.063237+0000) 2023-08-28T01:54:08.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:08 smithi130 bash[51302]: debug 2023-08-28T01:54:08.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:08.063455+0000) 2023-08-28T01:54:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:08 smithi006 bash[33918]: cluster 2023-08-28T01:54:06.793734+0000 mgr.y (mgr.14901) 7065 : cluster [DBG] pgmap v7072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:08.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:08 smithi006 bash[27292]: cluster 2023-08-28T01:54:06.793734+0000 mgr.y (mgr.14901) 7065 : cluster [DBG] pgmap v7072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:08.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:08 smithi130 bash[27579]: cluster 2023-08-28T01:54:06.793734+0000 mgr.y (mgr.14901) 7065 : cluster [DBG] pgmap v7072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:09.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:09 smithi130 bash[51302]: debug 2023-08-28T01:54:09.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:09.063732+0000) 2023-08-28T01:54:09.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:09 smithi006 bash[33918]: audit 2023-08-28T01:54:08.735875+0000 mon.a (mon.0) 4599 : audit [DBG] from='client.? 172.21.15.6:0/1875987982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:09.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:09 smithi006 bash[27292]: audit 2023-08-28T01:54:08.735875+0000 mon.a (mon.0) 4599 : audit [DBG] from='client.? 172.21.15.6:0/1875987982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:09.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:09 smithi130 bash[27579]: audit 2023-08-28T01:54:08.735875+0000 mon.a (mon.0) 4599 : audit [DBG] from='client.? 172.21.15.6:0/1875987982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:10.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:10 smithi130 bash[51302]: debug 2023-08-28T01:54:10.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:10.063969+0000) 2023-08-28T01:54:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:10 smithi006 bash[33918]: cluster 2023-08-28T01:54:08.794349+0000 mgr.y (mgr.14901) 7066 : cluster [DBG] pgmap v7073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:10 smithi006 bash[33918]: audit 2023-08-28T01:54:09.825826+0000 mon.b (mon.1) 791 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:54:10.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:10 smithi006 bash[27292]: cluster 2023-08-28T01:54:08.794349+0000 mgr.y (mgr.14901) 7066 : cluster [DBG] pgmap v7073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:10 smithi006 bash[27292]: audit 2023-08-28T01:54:09.825826+0000 mon.b (mon.1) 791 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:54:10.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:10 smithi130 bash[27579]: cluster 2023-08-28T01:54:08.794349+0000 mgr.y (mgr.14901) 7066 : cluster [DBG] pgmap v7073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:10.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:10 smithi130 bash[27579]: audit 2023-08-28T01:54:09.825826+0000 mon.b (mon.1) 791 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:54:11.150 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:54:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:54:10] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:54:11.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:11 smithi130 bash[51302]: debug 2023-08-28T01:54:11.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:11.064175+0000) 2023-08-28T01:54:11.425 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:54:11.151Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:11.425 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:54:11.153Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:11.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:11 smithi006 bash[33918]: audit 2023-08-28T01:54:11.154948+0000 mon.a (mon.0) 4600 : audit [DBG] from='client.? 172.21.15.6:0/2480716950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:11.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:11 smithi006 bash[27292]: audit 2023-08-28T01:54:11.154948+0000 mon.a (mon.0) 4600 : audit [DBG] from='client.? 172.21.15.6:0/2480716950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:11.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:11 smithi130 bash[27579]: audit 2023-08-28T01:54:11.154948+0000 mon.a (mon.0) 4600 : audit [DBG] from='client.? 172.21.15.6:0/2480716950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:12.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:12 smithi130 bash[51302]: debug 2023-08-28T01:54:12.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:12.064493+0000) 2023-08-28T01:54:12.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:12 smithi006 bash[27292]: cluster 2023-08-28T01:54:10.795447+0000 mgr.y (mgr.14901) 7067 : cluster [DBG] pgmap v7074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:12.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:12 smithi006 bash[33918]: cluster 2023-08-28T01:54:10.795447+0000 mgr.y (mgr.14901) 7067 : cluster [DBG] pgmap v7074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:12 smithi130 bash[27579]: cluster 2023-08-28T01:54:10.795447+0000 mgr.y (mgr.14901) 7067 : cluster [DBG] pgmap v7074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:54:12.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:54:12.718Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:13.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:13 smithi130 bash[51302]: debug 2023-08-28T01:54:13.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:13.064805+0000) 2023-08-28T01:54:14.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:14 smithi130 bash[51302]: debug 2023-08-28T01:54:14.064+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:14.065170+0000) 2023-08-28T01:54:14.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:14 smithi006 bash[33918]: cluster 2023-08-28T01:54:12.796148+0000 mgr.y (mgr.14901) 7068 : cluster [DBG] pgmap v7075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:14.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:14 smithi006 bash[33918]: audit 2023-08-28T01:54:13.571810+0000 mon.c (mon.2) 2780 : audit [DBG] from='client.? 172.21.15.6:0/238759309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:54:14.600Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:14.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:54:14.603Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:14.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:14 smithi006 bash[27292]: cluster 2023-08-28T01:54:12.796148+0000 mgr.y (mgr.14901) 7068 : cluster [DBG] pgmap v7075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:14.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:14 smithi006 bash[27292]: audit 2023-08-28T01:54:13.571810+0000 mon.c (mon.2) 2780 : audit [DBG] from='client.? 172.21.15.6:0/238759309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:14.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:14 smithi130 bash[27579]: cluster 2023-08-28T01:54:12.796148+0000 mgr.y (mgr.14901) 7068 : cluster [DBG] pgmap v7075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:14.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:14 smithi130 bash[27579]: audit 2023-08-28T01:54:13.571810+0000 mon.c (mon.2) 2780 : audit [DBG] from='client.? 172.21.15.6:0/238759309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:15.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:15 smithi130 bash[51302]: debug 2023-08-28T01:54:15.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:15.065491+0000) 2023-08-28T01:54:16.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:16 smithi130 bash[51302]: debug 2023-08-28T01:54:16.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:16.065743+0000) 2023-08-28T01:54:16.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:16 smithi006 bash[33918]: cluster 2023-08-28T01:54:14.797291+0000 mgr.y (mgr.14901) 7069 : cluster [DBG] pgmap v7076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:16.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:16 smithi006 bash[33918]: audit 2023-08-28T01:54:15.741554+0000 mon.a (mon.0) 4601 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:54:16.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:16 smithi006 bash[33918]: audit 2023-08-28T01:54:15.997500+0000 mon.c (mon.2) 2781 : audit [DBG] from='client.? 172.21.15.6:0/1623163974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:16 smithi006 bash[33918]: audit 2023-08-28T01:54:16.153098+0000 mon.a (mon.0) 4602 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:54:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:16 smithi006 bash[27292]: cluster 2023-08-28T01:54:14.797291+0000 mgr.y (mgr.14901) 7069 : cluster [DBG] pgmap v7076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:16 smithi006 bash[27292]: audit 2023-08-28T01:54:15.741554+0000 mon.a (mon.0) 4601 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:54:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:16 smithi006 bash[27292]: audit 2023-08-28T01:54:15.997500+0000 mon.c (mon.2) 2781 : audit [DBG] from='client.? 172.21.15.6:0/1623163974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:16 smithi006 bash[27292]: audit 2023-08-28T01:54:16.153098+0000 mon.a (mon.0) 4602 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:54:16.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:16 smithi130 bash[27579]: cluster 2023-08-28T01:54:14.797291+0000 mgr.y (mgr.14901) 7069 : cluster [DBG] pgmap v7076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:16.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:16 smithi130 bash[27579]: audit 2023-08-28T01:54:15.741554+0000 mon.a (mon.0) 4601 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:54:16.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:16 smithi130 bash[27579]: audit 2023-08-28T01:54:15.997500+0000 mon.c (mon.2) 2781 : audit [DBG] from='client.? 172.21.15.6:0/1623163974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:16.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:16 smithi130 bash[27579]: audit 2023-08-28T01:54:16.153098+0000 mon.a (mon.0) 4602 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:54:17.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:17 smithi130 bash[51302]: debug 2023-08-28T01:54:17.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:17.066026+0000) 2023-08-28T01:54:17.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:17 smithi006 bash[33918]: audit 2023-08-28T01:54:16.449674+0000 mon.b (mon.1) 792 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:54:17.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:17 smithi006 bash[33918]: audit 2023-08-28T01:54:16.451204+0000 mon.b (mon.1) 793 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:54:17.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:17 smithi006 bash[33918]: audit 2023-08-28T01:54:16.457929+0000 mon.a (mon.0) 4603 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:54:17.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:17 smithi006 bash[27292]: audit 2023-08-28T01:54:16.449674+0000 mon.b (mon.1) 792 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:54:17.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:17 smithi006 bash[27292]: audit 2023-08-28T01:54:16.451204+0000 mon.b (mon.1) 793 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:54:17.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:17 smithi006 bash[27292]: audit 2023-08-28T01:54:16.457929+0000 mon.a (mon.0) 4603 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:54:17.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:17 smithi130 bash[27579]: audit 2023-08-28T01:54:16.449674+0000 mon.b (mon.1) 792 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:54:17.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:17 smithi130 bash[27579]: audit 2023-08-28T01:54:16.451204+0000 mon.b (mon.1) 793 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:54:17.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:17 smithi130 bash[27579]: audit 2023-08-28T01:54:16.457929+0000 mon.a (mon.0) 4603 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:54:18.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:18 smithi130 bash[51302]: debug 2023-08-28T01:54:18.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:18.066342+0000) 2023-08-28T01:54:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:18 smithi006 bash[27292]: cluster 2023-08-28T01:54:16.798007+0000 mgr.y (mgr.14901) 7070 : cluster [DBG] pgmap v7077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:18 smithi006 bash[27292]: audit 2023-08-28T01:54:18.411118+0000 mon.a (mon.0) 4604 : audit [DBG] from='client.? 172.21.15.6:0/211709771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:18.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:18 smithi006 bash[33918]: cluster 2023-08-28T01:54:16.798007+0000 mgr.y (mgr.14901) 7070 : cluster [DBG] pgmap v7077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:18.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:18 smithi006 bash[33918]: audit 2023-08-28T01:54:18.411118+0000 mon.a (mon.0) 4604 : audit [DBG] from='client.? 172.21.15.6:0/211709771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:18.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:18 smithi130 bash[27579]: cluster 2023-08-28T01:54:16.798007+0000 mgr.y (mgr.14901) 7070 : cluster [DBG] pgmap v7077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:18.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:18 smithi130 bash[27579]: audit 2023-08-28T01:54:18.411118+0000 mon.a (mon.0) 4604 : audit [DBG] from='client.? 172.21.15.6:0/211709771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:19.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:19 smithi130 bash[51302]: debug 2023-08-28T01:54:19.060+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:19.066629+0000) 2023-08-28T01:54:20.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:20 smithi130 bash[51302]: debug 2023-08-28T01:54:20.064+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:20.066918+0000) 2023-08-28T01:54:20.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:20 smithi006 bash[27292]: cluster 2023-08-28T01:54:18.798671+0000 mgr.y (mgr.14901) 7071 : cluster [DBG] pgmap v7078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:20.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:20 smithi006 bash[33918]: cluster 2023-08-28T01:54:18.798671+0000 mgr.y (mgr.14901) 7071 : cluster [DBG] pgmap v7078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:20.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:20 smithi130 bash[27579]: cluster 2023-08-28T01:54:18.798671+0000 mgr.y (mgr.14901) 7071 : cluster [DBG] pgmap v7078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:21.151 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:54:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:54:20] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:54:21.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:21 smithi130 bash[51302]: debug 2023-08-28T01:54:21.064+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:21.067236+0000) 2023-08-28T01:54:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:54:21.152Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:54:21.154Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:21.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:21 smithi130 bash[27579]: audit 2023-08-28T01:54:20.831493+0000 mon.c (mon.2) 2782 : audit [DBG] from='client.? 172.21.15.6:0/2352394109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:21.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:21 smithi006 bash[33918]: audit 2023-08-28T01:54:20.831493+0000 mon.c (mon.2) 2782 : audit [DBG] from='client.? 172.21.15.6:0/2352394109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:21.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:21 smithi006 bash[27292]: audit 2023-08-28T01:54:20.831493+0000 mon.c (mon.2) 2782 : audit [DBG] from='client.? 172.21.15.6:0/2352394109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:22.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:22 smithi130 bash[51302]: debug 2023-08-28T01:54:22.064+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:22.067543+0000) 2023-08-28T01:54:22.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:22 smithi130 bash[27579]: cluster 2023-08-28T01:54:20.799775+0000 mgr.y (mgr.14901) 7072 : cluster [DBG] pgmap v7079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:22.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:22 smithi006 bash[27292]: cluster 2023-08-28T01:54:20.799775+0000 mgr.y (mgr.14901) 7072 : cluster [DBG] pgmap v7079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:54:22.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:54:22.718Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:22.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:22 smithi006 bash[33918]: cluster 2023-08-28T01:54:20.799775+0000 mgr.y (mgr.14901) 7072 : cluster [DBG] pgmap v7079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:23.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:23 smithi130 bash[51302]: debug 2023-08-28T01:54:23.064+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:23.067802+0000) 2023-08-28T01:54:23.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:23 smithi130 bash[27579]: audit 2023-08-28T01:54:23.244049+0000 mon.c (mon.2) 2783 : audit [DBG] from='client.? 172.21.15.6:0/1896104599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:23.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:23 smithi006 bash[33918]: audit 2023-08-28T01:54:23.244049+0000 mon.c (mon.2) 2783 : audit [DBG] from='client.? 172.21.15.6:0/1896104599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:23.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:23 smithi006 bash[27292]: audit 2023-08-28T01:54:23.244049+0000 mon.c (mon.2) 2783 : audit [DBG] from='client.? 172.21.15.6:0/1896104599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:24.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:24 smithi130 bash[51302]: debug 2023-08-28T01:54:24.064+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:24.068108+0000) 2023-08-28T01:54:24.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:24 smithi130 bash[27579]: cluster 2023-08-28T01:54:22.800436+0000 mgr.y (mgr.14901) 7073 : cluster [DBG] pgmap v7080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:24.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:24 smithi006 bash[27292]: cluster 2023-08-28T01:54:22.800436+0000 mgr.y (mgr.14901) 7073 : cluster [DBG] pgmap v7080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:24.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:24 smithi006 bash[33918]: cluster 2023-08-28T01:54:22.800436+0000 mgr.y (mgr.14901) 7073 : cluster [DBG] pgmap v7080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:54:24.602Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:54:24.604Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:25.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:25 smithi130 bash[51302]: debug 2023-08-28T01:54:25.064+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:25.068414+0000) 2023-08-28T01:54:25.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:25 smithi130 bash[27579]: cluster 2023-08-28T01:54:24.801539+0000 mgr.y (mgr.14901) 7074 : cluster [DBG] pgmap v7081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:25.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:25 smithi006 bash[27292]: cluster 2023-08-28T01:54:24.801539+0000 mgr.y (mgr.14901) 7074 : cluster [DBG] pgmap v7081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:25.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:25 smithi006 bash[33918]: cluster 2023-08-28T01:54:24.801539+0000 mgr.y (mgr.14901) 7074 : cluster [DBG] pgmap v7081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:26.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:26 smithi130 bash[51302]: debug 2023-08-28T01:54:26.068+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:26.068748+0000) 2023-08-28T01:54:26.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:26 smithi130 bash[27579]: audit 2023-08-28T01:54:25.665113+0000 mon.a (mon.0) 4605 : audit [DBG] from='client.? 172.21.15.6:0/264255596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:26.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:26 smithi006 bash[33918]: audit 2023-08-28T01:54:25.665113+0000 mon.a (mon.0) 4605 : audit [DBG] from='client.? 172.21.15.6:0/264255596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:27.109 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:26 smithi006 bash[27292]: audit 2023-08-28T01:54:25.665113+0000 mon.a (mon.0) 4605 : audit [DBG] from='client.? 172.21.15.6:0/264255596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:27.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:27 smithi130 bash[51302]: debug 2023-08-28T01:54:27.068+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:27.069044+0000) 2023-08-28T01:54:28.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:28 smithi130 bash[51302]: debug 2023-08-28T01:54:28.068+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:28.069308+0000) 2023-08-28T01:54:28.879 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:28 smithi006 bash[27292]: cluster 2023-08-28T01:54:26.802191+0000 mgr.y (mgr.14901) 7075 : cluster [DBG] pgmap v7082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:28.879 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:28 smithi006 bash[27292]: audit 2023-08-28T01:54:28.082963+0000 mon.c (mon.2) 2784 : audit [DBG] from='client.? 172.21.15.6:0/491706556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:28.880 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:28 smithi130 bash[27579]: cluster 2023-08-28T01:54:26.802191+0000 mgr.y (mgr.14901) 7075 : cluster [DBG] pgmap v7082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:28.880 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:28 smithi130 bash[27579]: audit 2023-08-28T01:54:28.082963+0000 mon.c (mon.2) 2784 : audit [DBG] from='client.? 172.21.15.6:0/491706556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:28.880 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:28 smithi006 bash[33918]: cluster 2023-08-28T01:54:26.802191+0000 mgr.y (mgr.14901) 7075 : cluster [DBG] pgmap v7082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:28.880 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:28 smithi006 bash[33918]: audit 2023-08-28T01:54:28.082963+0000 mon.c (mon.2) 2784 : audit [DBG] from='client.? 172.21.15.6:0/491706556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:29.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:29 smithi130 bash[51302]: debug 2023-08-28T01:54:29.068+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:29.069625+0000) 2023-08-28T01:54:30.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:30 smithi130 bash[51302]: debug 2023-08-28T01:54:30.068+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:30.069900+0000) 2023-08-28T01:54:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:30 smithi006 bash[33918]: cluster 2023-08-28T01:54:28.802884+0000 mgr.y (mgr.14901) 7076 : cluster [DBG] pgmap v7083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:30.761 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:30 smithi006 bash[27292]: cluster 2023-08-28T01:54:28.802884+0000 mgr.y (mgr.14901) 7076 : cluster [DBG] pgmap v7083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:30.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:30 smithi130 bash[27579]: cluster 2023-08-28T01:54:28.802884+0000 mgr.y (mgr.14901) 7076 : cluster [DBG] pgmap v7083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:31.153 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:54:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:54:30] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:54:31.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:31 smithi130 bash[51302]: debug 2023-08-28T01:54:31.064+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:31.070210+0000) 2023-08-28T01:54:31.423 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:54:31.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:31.424 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:54:31.155Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:31.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:31 smithi006 bash[27292]: audit 2023-08-28T01:54:30.507736+0000 mon.a (mon.0) 4606 : audit [DBG] from='client.? 172.21.15.6:0/1902654481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:31.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:31 smithi006 bash[33918]: audit 2023-08-28T01:54:30.507736+0000 mon.a (mon.0) 4606 : audit [DBG] from='client.? 172.21.15.6:0/1902654481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:31.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:31 smithi130 bash[27579]: audit 2023-08-28T01:54:30.507736+0000 mon.a (mon.0) 4606 : audit [DBG] from='client.? 172.21.15.6:0/1902654481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:32.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:32 smithi130 bash[51302]: debug 2023-08-28T01:54:32.064+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:32.070482+0000) 2023-08-28T01:54:32.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:32 smithi006 bash[33918]: cluster 2023-08-28T01:54:30.804005+0000 mgr.y (mgr.14901) 7077 : cluster [DBG] pgmap v7084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:32.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:32 smithi006 bash[27292]: cluster 2023-08-28T01:54:30.804005+0000 mgr.y (mgr.14901) 7077 : cluster [DBG] pgmap v7084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:32.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:32 smithi130 bash[27579]: cluster 2023-08-28T01:54:30.804005+0000 mgr.y (mgr.14901) 7077 : cluster [DBG] pgmap v7084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:54:32.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:54:32.719Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:33.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:33 smithi130 bash[51302]: debug 2023-08-28T01:54:33.068+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:33.070762+0000) 2023-08-28T01:54:33.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:33 smithi006 bash[27292]: audit 2023-08-28T01:54:32.928902+0000 mon.a (mon.0) 4607 : audit [DBG] from='client.? 172.21.15.6:0/2303976102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:33.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:33 smithi006 bash[33918]: audit 2023-08-28T01:54:32.928902+0000 mon.a (mon.0) 4607 : audit [DBG] from='client.? 172.21.15.6:0/2303976102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:33.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:33 smithi130 bash[27579]: audit 2023-08-28T01:54:32.928902+0000 mon.a (mon.0) 4607 : audit [DBG] from='client.? 172.21.15.6:0/2303976102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:34.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:34 smithi130 bash[51302]: debug 2023-08-28T01:54:34.068+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:34.071060+0000) 2023-08-28T01:54:34.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:34 smithi006 bash[33918]: cluster 2023-08-28T01:54:32.804654+0000 mgr.y (mgr.14901) 7078 : cluster [DBG] pgmap v7085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:34.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:34 smithi006 bash[27292]: cluster 2023-08-28T01:54:32.804654+0000 mgr.y (mgr.14901) 7078 : cluster [DBG] pgmap v7085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:54:34.602Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:34.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:54:34.604Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:34.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:34 smithi130 bash[27579]: cluster 2023-08-28T01:54:32.804654+0000 mgr.y (mgr.14901) 7078 : cluster [DBG] pgmap v7085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:35.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:35 smithi130 bash[51302]: debug 2023-08-28T01:54:35.068+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:35.071331+0000) 2023-08-28T01:54:35.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:35 smithi006 bash[33918]: audit 2023-08-28T01:54:35.340989+0000 mon.a (mon.0) 4608 : audit [DBG] from='client.? 172.21.15.6:0/4263843420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:35.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:35 smithi006 bash[27292]: audit 2023-08-28T01:54:35.340989+0000 mon.a (mon.0) 4608 : audit [DBG] from='client.? 172.21.15.6:0/4263843420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:35.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:35 smithi130 bash[27579]: audit 2023-08-28T01:54:35.340989+0000 mon.a (mon.0) 4608 : audit [DBG] from='client.? 172.21.15.6:0/4263843420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:36.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:36 smithi130 bash[51302]: debug 2023-08-28T01:54:36.068+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:36.071636+0000) 2023-08-28T01:54:36.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:36 smithi006 bash[33918]: cluster 2023-08-28T01:54:34.805828+0000 mgr.y (mgr.14901) 7079 : cluster [DBG] pgmap v7086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:36.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:36 smithi006 bash[27292]: cluster 2023-08-28T01:54:34.805828+0000 mgr.y (mgr.14901) 7079 : cluster [DBG] pgmap v7086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:36.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:36 smithi130 bash[27579]: cluster 2023-08-28T01:54:34.805828+0000 mgr.y (mgr.14901) 7079 : cluster [DBG] pgmap v7086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:37.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:37 smithi130 bash[51302]: debug 2023-08-28T01:54:37.068+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:37.071893+0000) 2023-08-28T01:54:38.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:38 smithi130 bash[51302]: debug 2023-08-28T01:54:38.068+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:38.072176+0000) 2023-08-28T01:54:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:38 smithi006 bash[33918]: cluster 2023-08-28T01:54:36.806507+0000 mgr.y (mgr.14901) 7080 : cluster [DBG] pgmap v7087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:38 smithi006 bash[33918]: audit 2023-08-28T01:54:37.760370+0000 mon.a (mon.0) 4609 : audit [DBG] from='client.? 172.21.15.6:0/3654690583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:38 smithi006 bash[27292]: cluster 2023-08-28T01:54:36.806507+0000 mgr.y (mgr.14901) 7080 : cluster [DBG] pgmap v7087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:38 smithi006 bash[27292]: audit 2023-08-28T01:54:37.760370+0000 mon.a (mon.0) 4609 : audit [DBG] from='client.? 172.21.15.6:0/3654690583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:38.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:38 smithi130 bash[27579]: cluster 2023-08-28T01:54:36.806507+0000 mgr.y (mgr.14901) 7080 : cluster [DBG] pgmap v7087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:38.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:38 smithi130 bash[27579]: audit 2023-08-28T01:54:37.760370+0000 mon.a (mon.0) 4609 : audit [DBG] from='client.? 172.21.15.6:0/3654690583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:39.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:39 smithi130 bash[51302]: debug 2023-08-28T01:54:39.068+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:39.072418+0000) 2023-08-28T01:54:40.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:40 smithi130 bash[51302]: debug 2023-08-28T01:54:40.072+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:40.072697+0000) 2023-08-28T01:54:40.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:40 smithi006 bash[33918]: cluster 2023-08-28T01:54:38.807192+0000 mgr.y (mgr.14901) 7081 : cluster [DBG] pgmap v7088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:40.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:40 smithi006 bash[33918]: audit 2023-08-28T01:54:40.170808+0000 mon.a (mon.0) 4610 : audit [DBG] from='client.? 172.21.15.6:0/3107218504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:40.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:40 smithi006 bash[27292]: cluster 2023-08-28T01:54:38.807192+0000 mgr.y (mgr.14901) 7081 : cluster [DBG] pgmap v7088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:40 smithi006 bash[27292]: audit 2023-08-28T01:54:40.170808+0000 mon.a (mon.0) 4610 : audit [DBG] from='client.? 172.21.15.6:0/3107218504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:40.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:40 smithi130 bash[27579]: cluster 2023-08-28T01:54:38.807192+0000 mgr.y (mgr.14901) 7081 : cluster [DBG] pgmap v7088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:40.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:40 smithi130 bash[27579]: audit 2023-08-28T01:54:40.170808+0000 mon.a (mon.0) 4610 : audit [DBG] from='client.? 172.21.15.6:0/3107218504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:41.153 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:54:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:54:40] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:54:41.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:41 smithi130 bash[51302]: debug 2023-08-28T01:54:41.068+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:41.072985+0000) 2023-08-28T01:54:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:54:41.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:54:41.155Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:42.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:42 smithi130 bash[51302]: debug 2023-08-28T01:54:42.068+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:42.073316+0000) 2023-08-28T01:54:42.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:42 smithi006 bash[33918]: cluster 2023-08-28T01:54:40.808417+0000 mgr.y (mgr.14901) 7082 : cluster [DBG] pgmap v7089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:42.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:42 smithi006 bash[27292]: cluster 2023-08-28T01:54:40.808417+0000 mgr.y (mgr.14901) 7082 : cluster [DBG] pgmap v7089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:42.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:42 smithi130 bash[27579]: cluster 2023-08-28T01:54:40.808417+0000 mgr.y (mgr.14901) 7082 : cluster [DBG] pgmap v7089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:54:42.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:54:42.719Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:43.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:43 smithi130 bash[51302]: debug 2023-08-28T01:54:43.068+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:43.073597+0000) 2023-08-28T01:54:43.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:43 smithi130 bash[27579]: audit 2023-08-28T01:54:42.585784+0000 mon.c (mon.2) 2785 : audit [DBG] from='client.? 172.21.15.6:0/2717486411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:43.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:43 smithi006 bash[27292]: audit 2023-08-28T01:54:42.585784+0000 mon.c (mon.2) 2785 : audit [DBG] from='client.? 172.21.15.6:0/2717486411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:43.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:43 smithi006 bash[33918]: audit 2023-08-28T01:54:42.585784+0000 mon.c (mon.2) 2785 : audit [DBG] from='client.? 172.21.15.6:0/2717486411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:44.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:44 smithi130 bash[51302]: debug 2023-08-28T01:54:44.068+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:44.073861+0000) 2023-08-28T01:54:44.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:44 smithi130 bash[27579]: cluster 2023-08-28T01:54:42.809157+0000 mgr.y (mgr.14901) 7083 : cluster [DBG] pgmap v7090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:44.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:44 smithi006 bash[33918]: cluster 2023-08-28T01:54:42.809157+0000 mgr.y (mgr.14901) 7083 : cluster [DBG] pgmap v7090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:54:44.603Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:54:44.605Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:44.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:44 smithi006 bash[27292]: cluster 2023-08-28T01:54:42.809157+0000 mgr.y (mgr.14901) 7083 : cluster [DBG] pgmap v7090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:45.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:45 smithi130 bash[51302]: debug 2023-08-28T01:54:45.068+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:45.074160+0000) 2023-08-28T01:54:45.872 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:45 smithi130 bash[27579]: audit 2023-08-28T01:54:45.004602+0000 mon.c (mon.2) 2786 : audit [DBG] from='client.? 172.21.15.6:0/3213846378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:45.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:45 smithi006 bash[27292]: audit 2023-08-28T01:54:45.004602+0000 mon.c (mon.2) 2786 : audit [DBG] from='client.? 172.21.15.6:0/3213846378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:45.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:45 smithi006 bash[33918]: audit 2023-08-28T01:54:45.004602+0000 mon.c (mon.2) 2786 : audit [DBG] from='client.? 172.21.15.6:0/3213846378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:46.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:46 smithi130 bash[51302]: debug 2023-08-28T01:54:46.072+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:46.074495+0000) 2023-08-28T01:54:46.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:46 smithi130 bash[27579]: cluster 2023-08-28T01:54:44.810103+0000 mgr.y (mgr.14901) 7084 : cluster [DBG] pgmap v7091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:46.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:46 smithi006 bash[33918]: cluster 2023-08-28T01:54:44.810103+0000 mgr.y (mgr.14901) 7084 : cluster [DBG] pgmap v7091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:46.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:46 smithi006 bash[27292]: cluster 2023-08-28T01:54:44.810103+0000 mgr.y (mgr.14901) 7084 : cluster [DBG] pgmap v7091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:47.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:47 smithi130 bash[51302]: debug 2023-08-28T01:54:47.068+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:47.074804+0000) 2023-08-28T01:54:48.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:48 smithi130 bash[51302]: debug 2023-08-28T01:54:48.072+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:48.075141+0000) 2023-08-28T01:54:48.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:48 smithi006 bash[33918]: cluster 2023-08-28T01:54:46.810761+0000 mgr.y (mgr.14901) 7085 : cluster [DBG] pgmap v7092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:48.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:48 smithi006 bash[33918]: audit 2023-08-28T01:54:47.431814+0000 mon.c (mon.2) 2787 : audit [DBG] from='client.? 172.21.15.6:0/585758531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:48.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:48 smithi006 bash[27292]: cluster 2023-08-28T01:54:46.810761+0000 mgr.y (mgr.14901) 7085 : cluster [DBG] pgmap v7092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:48 smithi006 bash[27292]: audit 2023-08-28T01:54:47.431814+0000 mon.c (mon.2) 2787 : audit [DBG] from='client.? 172.21.15.6:0/585758531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:48.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:48 smithi130 bash[27579]: cluster 2023-08-28T01:54:46.810761+0000 mgr.y (mgr.14901) 7085 : cluster [DBG] pgmap v7092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:48.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:48 smithi130 bash[27579]: audit 2023-08-28T01:54:47.431814+0000 mon.c (mon.2) 2787 : audit [DBG] from='client.? 172.21.15.6:0/585758531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:49.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:49 smithi130 bash[51302]: debug 2023-08-28T01:54:49.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:49.075469+0000) 2023-08-28T01:54:50.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:50 smithi130 bash[51302]: debug 2023-08-28T01:54:50.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:50.075777+0000) 2023-08-28T01:54:50.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:50 smithi006 bash[27292]: cluster 2023-08-28T01:54:48.811528+0000 mgr.y (mgr.14901) 7086 : cluster [DBG] pgmap v7093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:50.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:50 smithi006 bash[27292]: audit 2023-08-28T01:54:49.854191+0000 mon.a (mon.0) 4611 : audit [DBG] from='client.? 172.21.15.6:0/1687288308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:50.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:50 smithi006 bash[33918]: cluster 2023-08-28T01:54:48.811528+0000 mgr.y (mgr.14901) 7086 : cluster [DBG] pgmap v7093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:50.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:50 smithi006 bash[33918]: audit 2023-08-28T01:54:49.854191+0000 mon.a (mon.0) 4611 : audit [DBG] from='client.? 172.21.15.6:0/1687288308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:50.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:50 smithi130 bash[27579]: cluster 2023-08-28T01:54:48.811528+0000 mgr.y (mgr.14901) 7086 : cluster [DBG] pgmap v7093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:50.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:50 smithi130 bash[27579]: audit 2023-08-28T01:54:49.854191+0000 mon.a (mon.0) 4611 : audit [DBG] from='client.? 172.21.15.6:0/1687288308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:51.154 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:54:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:54:50] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:54:51.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:51 smithi130 bash[51302]: debug 2023-08-28T01:54:51.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:51.076111+0000) 2023-08-28T01:54:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:54:51.155Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:54:51.157Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:52.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:52 smithi130 bash[51302]: debug 2023-08-28T01:54:52.075+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:52.076549+0000) 2023-08-28T01:54:52.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:52 smithi006 bash[27292]: cluster 2023-08-28T01:54:50.812764+0000 mgr.y (mgr.14901) 7087 : cluster [DBG] pgmap v7094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:52.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:52 smithi006 bash[27292]: audit 2023-08-28T01:54:52.266101+0000 mon.c (mon.2) 2788 : audit [DBG] from='client.? 172.21.15.6:0/730094497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:52.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:52 smithi006 bash[33918]: cluster 2023-08-28T01:54:50.812764+0000 mgr.y (mgr.14901) 7087 : cluster [DBG] pgmap v7094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:52.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:52 smithi006 bash[33918]: audit 2023-08-28T01:54:52.266101+0000 mon.c (mon.2) 2788 : audit [DBG] from='client.? 172.21.15.6:0/730094497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:52.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:52 smithi130 bash[27579]: cluster 2023-08-28T01:54:50.812764+0000 mgr.y (mgr.14901) 7087 : cluster [DBG] pgmap v7094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:52.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:52 smithi130 bash[27579]: audit 2023-08-28T01:54:52.266101+0000 mon.c (mon.2) 2788 : audit [DBG] from='client.? 172.21.15.6:0/730094497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:54:52.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:54:52.720Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:53.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:53 smithi130 bash[51302]: debug 2023-08-28T01:54:53.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:53.076826+0000) 2023-08-28T01:54:54.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:54 smithi130 bash[51302]: debug 2023-08-28T01:54:54.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:54.077129+0000) 2023-08-28T01:54:54.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:54 smithi006 bash[33918]: cluster 2023-08-28T01:54:52.813564+0000 mgr.y (mgr.14901) 7088 : cluster [DBG] pgmap v7095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:54.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:54 smithi006 bash[27292]: cluster 2023-08-28T01:54:52.813564+0000 mgr.y (mgr.14901) 7088 : cluster [DBG] pgmap v7095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:54:54.603Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:54.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:54:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:54:54.605Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:54:54.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:54 smithi130 bash[27579]: cluster 2023-08-28T01:54:52.813564+0000 mgr.y (mgr.14901) 7088 : cluster [DBG] pgmap v7095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:55.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:55 smithi130 bash[51302]: debug 2023-08-28T01:54:55.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:55.077406+0000) 2023-08-28T01:54:55.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:55 smithi006 bash[27292]: audit 2023-08-28T01:54:54.685957+0000 mon.a (mon.0) 4612 : audit [DBG] from='client.? 172.21.15.6:0/1770125209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:55.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:55 smithi006 bash[33918]: audit 2023-08-28T01:54:54.685957+0000 mon.a (mon.0) 4612 : audit [DBG] from='client.? 172.21.15.6:0/1770125209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:55.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:55 smithi130 bash[27579]: audit 2023-08-28T01:54:54.685957+0000 mon.a (mon.0) 4612 : audit [DBG] from='client.? 172.21.15.6:0/1770125209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:56.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:56 smithi130 bash[51302]: debug 2023-08-28T01:54:56.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:56.077689+0000) 2023-08-28T01:54:56.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:56 smithi006 bash[27292]: cluster 2023-08-28T01:54:54.814728+0000 mgr.y (mgr.14901) 7089 : cluster [DBG] pgmap v7096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:56.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:56 smithi006 bash[33918]: cluster 2023-08-28T01:54:54.814728+0000 mgr.y (mgr.14901) 7089 : cluster [DBG] pgmap v7096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:56.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:56 smithi130 bash[27579]: cluster 2023-08-28T01:54:54.814728+0000 mgr.y (mgr.14901) 7089 : cluster [DBG] pgmap v7096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:57.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:57 smithi130 bash[51302]: debug 2023-08-28T01:54:57.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:57.077957+0000) 2023-08-28T01:54:57.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:57 smithi006 bash[33918]: audit 2023-08-28T01:54:57.098848+0000 mon.a (mon.0) 4613 : audit [DBG] from='client.? 172.21.15.6:0/3807481896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:57.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:57 smithi006 bash[27292]: audit 2023-08-28T01:54:57.098848+0000 mon.a (mon.0) 4613 : audit [DBG] from='client.? 172.21.15.6:0/3807481896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:57.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:57 smithi130 bash[27579]: audit 2023-08-28T01:54:57.098848+0000 mon.a (mon.0) 4613 : audit [DBG] from='client.? 172.21.15.6:0/3807481896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:58.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:58 smithi130 bash[51302]: debug 2023-08-28T01:54:58.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:58.078217+0000) 2023-08-28T01:54:58.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:54:58 smithi006 bash[27292]: cluster 2023-08-28T01:54:56.815342+0000 mgr.y (mgr.14901) 7090 : cluster [DBG] pgmap v7097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:58.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:54:58 smithi006 bash[33918]: cluster 2023-08-28T01:54:56.815342+0000 mgr.y (mgr.14901) 7090 : cluster [DBG] pgmap v7097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:58.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:54:58 smithi130 bash[27579]: cluster 2023-08-28T01:54:56.815342+0000 mgr.y (mgr.14901) 7090 : cluster [DBG] pgmap v7097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:54:59.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:54:59 smithi130 bash[51302]: debug 2023-08-28T01:54:59.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:59.078471+0000) 2023-08-28T01:55:00.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:00 smithi130 bash[51302]: debug 2023-08-28T01:55:00.071+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:00.078690+0000) 2023-08-28T01:55:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:00 smithi006 bash[27292]: cluster 2023-08-28T01:54:58.816003+0000 mgr.y (mgr.14901) 7091 : cluster [DBG] pgmap v7098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:00 smithi006 bash[27292]: audit 2023-08-28T01:54:59.513373+0000 mon.c (mon.2) 2789 : audit [DBG] from='client.? 172.21.15.6:0/1928663198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:00 smithi006 bash[33918]: cluster 2023-08-28T01:54:58.816003+0000 mgr.y (mgr.14901) 7091 : cluster [DBG] pgmap v7098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:00 smithi006 bash[33918]: audit 2023-08-28T01:54:59.513373+0000 mon.c (mon.2) 2789 : audit [DBG] from='client.? 172.21.15.6:0/1928663198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:00.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:00 smithi130 bash[27579]: cluster 2023-08-28T01:54:58.816003+0000 mgr.y (mgr.14901) 7091 : cluster [DBG] pgmap v7098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:00 smithi130 bash[27579]: audit 2023-08-28T01:54:59.513373+0000 mon.c (mon.2) 2789 : audit [DBG] from='client.? 172.21.15.6:0/1928663198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:01.155 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:55:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:55:00] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:55:01.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:01 smithi130 bash[51302]: debug 2023-08-28T01:55:01.075+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:01.079021+0000) 2023-08-28T01:55:01.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:55:01.156Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:01.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:55:01.158Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:02.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:02 smithi130 bash[51302]: debug 2023-08-28T01:55:02.075+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:02.079318+0000) 2023-08-28T01:55:02.720 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:02 smithi006 bash[27292]: cluster 2023-08-28T01:55:00.817150+0000 mgr.y (mgr.14901) 7092 : cluster [DBG] pgmap v7099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:02.721 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:02 smithi006 bash[27292]: audit 2023-08-28T01:55:01.922402+0000 mon.a (mon.0) 4614 : audit [DBG] from='client.? 172.21.15.6:0/2291662407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:02.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:02 smithi006 bash[33918]: cluster 2023-08-28T01:55:00.817150+0000 mgr.y (mgr.14901) 7092 : cluster [DBG] pgmap v7099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:02.721 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:02 smithi006 bash[33918]: audit 2023-08-28T01:55:01.922402+0000 mon.a (mon.0) 4614 : audit [DBG] from='client.? 172.21.15.6:0/2291662407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:02.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:55:02.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:02.721 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:55:02.721Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:02 smithi130 bash[27579]: cluster 2023-08-28T01:55:00.817150+0000 mgr.y (mgr.14901) 7092 : cluster [DBG] pgmap v7099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:02 smithi130 bash[27579]: audit 2023-08-28T01:55:01.922402+0000 mon.a (mon.0) 4614 : audit [DBG] from='client.? 172.21.15.6:0/2291662407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:03.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:03 smithi130 bash[51302]: debug 2023-08-28T01:55:03.075+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:03.079594+0000) 2023-08-28T01:55:04.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:04 smithi130 bash[51302]: debug 2023-08-28T01:55:04.075+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:04.079877+0000) 2023-08-28T01:55:04.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:04 smithi130 bash[27579]: cluster 2023-08-28T01:55:02.817777+0000 mgr.y (mgr.14901) 7093 : cluster [DBG] pgmap v7100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:04.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:04 smithi130 bash[27579]: audit 2023-08-28T01:55:04.333082+0000 mon.a (mon.0) 4615 : audit [DBG] from='client.? 172.21.15.6:0/1435139738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:04.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:04 smithi006 bash[33918]: cluster 2023-08-28T01:55:02.817777+0000 mgr.y (mgr.14901) 7093 : cluster [DBG] pgmap v7100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:04.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:04 smithi006 bash[33918]: audit 2023-08-28T01:55:04.333082+0000 mon.a (mon.0) 4615 : audit [DBG] from='client.? 172.21.15.6:0/1435139738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:55:04.604Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:55:04.606Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:04.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:04 smithi006 bash[27292]: cluster 2023-08-28T01:55:02.817777+0000 mgr.y (mgr.14901) 7093 : cluster [DBG] pgmap v7100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:04.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:04 smithi006 bash[27292]: audit 2023-08-28T01:55:04.333082+0000 mon.a (mon.0) 4615 : audit [DBG] from='client.? 172.21.15.6:0/1435139738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:05.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:05 smithi130 bash[51302]: debug 2023-08-28T01:55:05.075+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:05.080164+0000) 2023-08-28T01:55:06.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:06 smithi130 bash[51302]: debug 2023-08-28T01:55:06.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:06.080440+0000) 2023-08-28T01:55:06.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:06 smithi130 bash[27579]: cluster 2023-08-28T01:55:04.818890+0000 mgr.y (mgr.14901) 7094 : cluster [DBG] pgmap v7101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:06.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:06 smithi006 bash[27292]: cluster 2023-08-28T01:55:04.818890+0000 mgr.y (mgr.14901) 7094 : cluster [DBG] pgmap v7101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:06.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:06 smithi006 bash[33918]: cluster 2023-08-28T01:55:04.818890+0000 mgr.y (mgr.14901) 7094 : cluster [DBG] pgmap v7101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:07.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:07 smithi130 bash[51302]: debug 2023-08-28T01:55:07.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:07.080737+0000) 2023-08-28T01:55:07.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:07 smithi130 bash[27579]: audit 2023-08-28T01:55:06.747675+0000 mon.c (mon.2) 2790 : audit [DBG] from='client.? 172.21.15.6:0/3252844638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:07.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:07 smithi006 bash[33918]: audit 2023-08-28T01:55:06.747675+0000 mon.c (mon.2) 2790 : audit [DBG] from='client.? 172.21.15.6:0/3252844638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:07.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:07 smithi006 bash[27292]: audit 2023-08-28T01:55:06.747675+0000 mon.c (mon.2) 2790 : audit [DBG] from='client.? 172.21.15.6:0/3252844638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:08.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:08 smithi130 bash[51302]: debug 2023-08-28T01:55:08.075+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:08.081053+0000) 2023-08-28T01:55:08.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:08 smithi130 bash[27579]: cluster 2023-08-28T01:55:06.819607+0000 mgr.y (mgr.14901) 7095 : cluster [DBG] pgmap v7102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:08.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:08 smithi006 bash[27292]: cluster 2023-08-28T01:55:06.819607+0000 mgr.y (mgr.14901) 7095 : cluster [DBG] pgmap v7102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:08.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:08 smithi006 bash[33918]: cluster 2023-08-28T01:55:06.819607+0000 mgr.y (mgr.14901) 7095 : cluster [DBG] pgmap v7102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:09.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:09 smithi130 bash[51302]: debug 2023-08-28T01:55:09.075+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:09.081363+0000) 2023-08-28T01:55:09.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:09 smithi130 bash[27579]: audit 2023-08-28T01:55:09.168235+0000 mon.c (mon.2) 2791 : audit [DBG] from='client.? 172.21.15.6:0/1504635405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:09.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:09 smithi006 bash[33918]: audit 2023-08-28T01:55:09.168235+0000 mon.c (mon.2) 2791 : audit [DBG] from='client.? 172.21.15.6:0/1504635405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:09.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:09 smithi006 bash[27292]: audit 2023-08-28T01:55:09.168235+0000 mon.c (mon.2) 2791 : audit [DBG] from='client.? 172.21.15.6:0/1504635405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:10.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:10 smithi130 bash[51302]: debug 2023-08-28T01:55:10.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:10.081648+0000) 2023-08-28T01:55:10.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:10 smithi130 bash[27579]: cluster 2023-08-28T01:55:08.820272+0000 mgr.y (mgr.14901) 7096 : cluster [DBG] pgmap v7103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:10.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:55:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:55:10] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T01:55:10.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:10 smithi006 bash[27292]: cluster 2023-08-28T01:55:08.820272+0000 mgr.y (mgr.14901) 7096 : cluster [DBG] pgmap v7103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:10.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:10 smithi006 bash[33918]: cluster 2023-08-28T01:55:08.820272+0000 mgr.y (mgr.14901) 7096 : cluster [DBG] pgmap v7103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:11.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:11 smithi130 bash[51302]: debug 2023-08-28T01:55:11.075+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:11.081958+0000) 2023-08-28T01:55:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:55:11.156Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:55:11.158Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:11.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:11 smithi130 bash[27579]: cluster 2023-08-28T01:55:10.821432+0000 mgr.y (mgr.14901) 7097 : cluster [DBG] pgmap v7104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:11.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:11 smithi006 bash[33918]: cluster 2023-08-28T01:55:10.821432+0000 mgr.y (mgr.14901) 7097 : cluster [DBG] pgmap v7104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:11.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:11 smithi006 bash[27292]: cluster 2023-08-28T01:55:10.821432+0000 mgr.y (mgr.14901) 7097 : cluster [DBG] pgmap v7104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:12.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:12 smithi130 bash[51302]: debug 2023-08-28T01:55:12.075+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:12.082241+0000) 2023-08-28T01:55:12.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:12 smithi130 bash[27579]: audit 2023-08-28T01:55:11.580611+0000 mon.a (mon.0) 4616 : audit [DBG] from='client.? 172.21.15.6:0/3345913965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:12.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:12 smithi006 bash[33918]: audit 2023-08-28T01:55:11.580611+0000 mon.a (mon.0) 4616 : audit [DBG] from='client.? 172.21.15.6:0/3345913965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:12.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:12 smithi006 bash[27292]: audit 2023-08-28T01:55:11.580611+0000 mon.a (mon.0) 4616 : audit [DBG] from='client.? 172.21.15.6:0/3345913965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:55:12.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:55:12.722Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:13.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:13 smithi130 bash[51302]: debug 2023-08-28T01:55:13.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:13.082544+0000) 2023-08-28T01:55:13.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:13 smithi130 bash[27579]: cluster 2023-08-28T01:55:12.822080+0000 mgr.y (mgr.14901) 7098 : cluster [DBG] pgmap v7105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:13.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:13 smithi006 bash[27292]: cluster 2023-08-28T01:55:12.822080+0000 mgr.y (mgr.14901) 7098 : cluster [DBG] pgmap v7105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:13.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:13 smithi006 bash[33918]: cluster 2023-08-28T01:55:12.822080+0000 mgr.y (mgr.14901) 7098 : cluster [DBG] pgmap v7105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:14.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:14 smithi130 bash[51302]: debug 2023-08-28T01:55:14.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:14.082827+0000) 2023-08-28T01:55:14.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:14 smithi130 bash[27579]: audit 2023-08-28T01:55:13.993969+0000 mon.c (mon.2) 2792 : audit [DBG] from='client.? 172.21.15.6:0/1945566457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:14.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:14 smithi006 bash[27292]: audit 2023-08-28T01:55:13.993969+0000 mon.c (mon.2) 2792 : audit [DBG] from='client.? 172.21.15.6:0/1945566457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:14.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:14 smithi006 bash[33918]: audit 2023-08-28T01:55:13.993969+0000 mon.c (mon.2) 2792 : audit [DBG] from='client.? 172.21.15.6:0/1945566457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:55:14.605Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:55:14.607Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:15.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:15 smithi130 bash[51302]: debug 2023-08-28T01:55:15.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:15.083195+0000) 2023-08-28T01:55:15.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:15 smithi130 bash[27579]: cluster 2023-08-28T01:55:14.823240+0000 mgr.y (mgr.14901) 7099 : cluster [DBG] pgmap v7106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:15.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:15 smithi006 bash[27292]: cluster 2023-08-28T01:55:14.823240+0000 mgr.y (mgr.14901) 7099 : cluster [DBG] pgmap v7106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:15.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:15 smithi006 bash[33918]: cluster 2023-08-28T01:55:14.823240+0000 mgr.y (mgr.14901) 7099 : cluster [DBG] pgmap v7106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:16.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:16 smithi130 bash[51302]: debug 2023-08-28T01:55:16.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:16.083471+0000) 2023-08-28T01:55:16.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:16 smithi130 bash[27579]: audit 2023-08-28T01:55:16.403097+0000 mon.a (mon.0) 4617 : audit [DBG] from='client.? 172.21.15.6:0/3460372921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:16.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:16 smithi130 bash[27579]: audit 2023-08-28T01:55:16.469575+0000 mon.b (mon.1) 794 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:55:16.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:16 smithi006 bash[27292]: audit 2023-08-28T01:55:16.403097+0000 mon.a (mon.0) 4617 : audit [DBG] from='client.? 172.21.15.6:0/3460372921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:16.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:16 smithi006 bash[27292]: audit 2023-08-28T01:55:16.469575+0000 mon.b (mon.1) 794 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:55:16.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:16 smithi006 bash[33918]: audit 2023-08-28T01:55:16.403097+0000 mon.a (mon.0) 4617 : audit [DBG] from='client.? 172.21.15.6:0/3460372921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:16.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:16 smithi006 bash[33918]: audit 2023-08-28T01:55:16.469575+0000 mon.b (mon.1) 794 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:55:17.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:17 smithi130 bash[51302]: debug 2023-08-28T01:55:17.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:17.083754+0000) 2023-08-28T01:55:18.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:18 smithi130 bash[51302]: debug 2023-08-28T01:55:18.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:18.084038+0000) 2023-08-28T01:55:18.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:18 smithi006 bash[33918]: cluster 2023-08-28T01:55:16.823894+0000 mgr.y (mgr.14901) 7100 : cluster [DBG] pgmap v7107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:18 smithi006 bash[27292]: cluster 2023-08-28T01:55:16.823894+0000 mgr.y (mgr.14901) 7100 : cluster [DBG] pgmap v7107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:18.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:18 smithi130 bash[27579]: cluster 2023-08-28T01:55:16.823894+0000 mgr.y (mgr.14901) 7100 : cluster [DBG] pgmap v7107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:19.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:19 smithi130 bash[51302]: debug 2023-08-28T01:55:19.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:19.084350+0000) 2023-08-28T01:55:19.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:19 smithi006 bash[27292]: audit 2023-08-28T01:55:18.829407+0000 mon.c (mon.2) 2793 : audit [DBG] from='client.? 172.21.15.6:0/1802499030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:19.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:19 smithi006 bash[33918]: audit 2023-08-28T01:55:18.829407+0000 mon.c (mon.2) 2793 : audit [DBG] from='client.? 172.21.15.6:0/1802499030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:19.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:19 smithi130 bash[27579]: audit 2023-08-28T01:55:18.829407+0000 mon.c (mon.2) 2793 : audit [DBG] from='client.? 172.21.15.6:0/1802499030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:20.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:20 smithi130 bash[51302]: debug 2023-08-28T01:55:20.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:20.084649+0000) 2023-08-28T01:55:20.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:20 smithi006 bash[27292]: cluster 2023-08-28T01:55:18.824361+0000 mgr.y (mgr.14901) 7101 : cluster [DBG] pgmap v7108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:20.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:20 smithi006 bash[33918]: cluster 2023-08-28T01:55:18.824361+0000 mgr.y (mgr.14901) 7101 : cluster [DBG] pgmap v7108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:20.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:20 smithi130 bash[27579]: cluster 2023-08-28T01:55:18.824361+0000 mgr.y (mgr.14901) 7101 : cluster [DBG] pgmap v7108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:21.157 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:55:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:55:20] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T01:55:21.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:21 smithi130 bash[51302]: debug 2023-08-28T01:55:21.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:21.084961+0000) 2023-08-28T01:55:21.443 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:55:21.157Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:21.443 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:55:21.159Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:21.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:21 smithi006 bash[33918]: audit 2023-08-28T01:55:21.248918+0000 mon.c (mon.2) 2794 : audit [DBG] from='client.? 172.21.15.6:0/1388273467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:21.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:21 smithi006 bash[27292]: audit 2023-08-28T01:55:21.248918+0000 mon.c (mon.2) 2794 : audit [DBG] from='client.? 172.21.15.6:0/1388273467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:21.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:21 smithi130 bash[27579]: audit 2023-08-28T01:55:21.248918+0000 mon.c (mon.2) 2794 : audit [DBG] from='client.? 172.21.15.6:0/1388273467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:22.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:22 smithi130 bash[51302]: debug 2023-08-28T01:55:22.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:22.085208+0000) 2023-08-28T01:55:22.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:22 smithi006 bash[27292]: cluster 2023-08-28T01:55:20.825472+0000 mgr.y (mgr.14901) 7102 : cluster [DBG] pgmap v7109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:22.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:22 smithi006 bash[27292]: audit 2023-08-28T01:55:22.335297+0000 mon.a (mon.0) 4618 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:55:22.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:22 smithi006 bash[33918]: cluster 2023-08-28T01:55:20.825472+0000 mgr.y (mgr.14901) 7102 : cluster [DBG] pgmap v7109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:22.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:22 smithi006 bash[33918]: audit 2023-08-28T01:55:22.335297+0000 mon.a (mon.0) 4618 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:55:22.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:22 smithi130 bash[27579]: cluster 2023-08-28T01:55:20.825472+0000 mgr.y (mgr.14901) 7102 : cluster [DBG] pgmap v7109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:22 smithi130 bash[27579]: audit 2023-08-28T01:55:22.335297+0000 mon.a (mon.0) 4618 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:55:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:55:22.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:55:22.723Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:23.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:23 smithi130 bash[51302]: debug 2023-08-28T01:55:23.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:23.085496+0000) 2023-08-28T01:55:24.083 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:23 smithi130 bash[27579]: audit 2023-08-28T01:55:22.739876+0000 mon.a (mon.0) 4619 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:55:24.084 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:23 smithi130 bash[27579]: cluster 2023-08-28T01:55:22.826235+0000 mgr.y (mgr.14901) 7103 : cluster [DBG] pgmap v7110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:24.084 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:23 smithi130 bash[27579]: audit 2023-08-28T01:55:23.662525+0000 mon.c (mon.2) 2795 : audit [DBG] from='client.? 172.21.15.6:0/1068322476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:24.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:23 smithi006 bash[33918]: audit 2023-08-28T01:55:22.739876+0000 mon.a (mon.0) 4619 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:55:24.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:23 smithi006 bash[33918]: cluster 2023-08-28T01:55:22.826235+0000 mgr.y (mgr.14901) 7103 : cluster [DBG] pgmap v7110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:24.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:23 smithi006 bash[33918]: audit 2023-08-28T01:55:23.662525+0000 mon.c (mon.2) 2795 : audit [DBG] from='client.? 172.21.15.6:0/1068322476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:24.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:23 smithi006 bash[27292]: audit 2023-08-28T01:55:22.739876+0000 mon.a (mon.0) 4619 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:55:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:23 smithi006 bash[27292]: cluster 2023-08-28T01:55:22.826235+0000 mgr.y (mgr.14901) 7103 : cluster [DBG] pgmap v7110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:23 smithi006 bash[27292]: audit 2023-08-28T01:55:23.662525+0000 mon.c (mon.2) 2795 : audit [DBG] from='client.? 172.21.15.6:0/1068322476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:24.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:24 smithi130 bash[51302]: debug 2023-08-28T01:55:24.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:24.085773+0000) 2023-08-28T01:55:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:55:24.605Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:55:24.608Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:25.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:25 smithi130 bash[51302]: debug 2023-08-28T01:55:25.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:25.086039+0000) 2023-08-28T01:55:26.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:25 smithi006 bash[33918]: cluster 2023-08-28T01:55:24.827417+0000 mgr.y (mgr.14901) 7104 : cluster [DBG] pgmap v7111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:26.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:25 smithi006 bash[27292]: cluster 2023-08-28T01:55:24.827417+0000 mgr.y (mgr.14901) 7104 : cluster [DBG] pgmap v7111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:26.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:25 smithi130 bash[27579]: cluster 2023-08-28T01:55:24.827417+0000 mgr.y (mgr.14901) 7104 : cluster [DBG] pgmap v7111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:26.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:26 smithi130 bash[51302]: debug 2023-08-28T01:55:26.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:26.086273+0000) 2023-08-28T01:55:27.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:26 smithi006 bash[27292]: audit 2023-08-28T01:55:26.083581+0000 mon.c (mon.2) 2796 : audit [DBG] from='client.? 172.21.15.6:0/2448310570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:27.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:26 smithi006 bash[33918]: audit 2023-08-28T01:55:26.083581+0000 mon.c (mon.2) 2796 : audit [DBG] from='client.? 172.21.15.6:0/2448310570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:27.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:26 smithi130 bash[27579]: audit 2023-08-28T01:55:26.083581+0000 mon.c (mon.2) 2796 : audit [DBG] from='client.? 172.21.15.6:0/2448310570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:27.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:27 smithi130 bash[51302]: debug 2023-08-28T01:55:27.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:27.086522+0000) 2023-08-28T01:55:28.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:28 smithi130 bash[51302]: debug 2023-08-28T01:55:28.079+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:28.086791+0000) 2023-08-28T01:55:28.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:28 smithi006 bash[33918]: cluster 2023-08-28T01:55:26.828216+0000 mgr.y (mgr.14901) 7105 : cluster [DBG] pgmap v7112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:28.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:28 smithi006 bash[27292]: cluster 2023-08-28T01:55:26.828216+0000 mgr.y (mgr.14901) 7105 : cluster [DBG] pgmap v7112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:28.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:28 smithi130 bash[27579]: cluster 2023-08-28T01:55:26.828216+0000 mgr.y (mgr.14901) 7105 : cluster [DBG] pgmap v7112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:29.312 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:29 smithi130 bash[51302]: debug 2023-08-28T01:55:29.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:29.087044+0000) 2023-08-28T01:55:29.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:29 smithi130 bash[27579]: audit 2023-08-28T01:55:28.501006+0000 mon.a (mon.0) 4620 : audit [DBG] from='client.? 172.21.15.6:0/1501256651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:29.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:29 smithi006 bash[27292]: audit 2023-08-28T01:55:28.501006+0000 mon.a (mon.0) 4620 : audit [DBG] from='client.? 172.21.15.6:0/1501256651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:29.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:29 smithi006 bash[33918]: audit 2023-08-28T01:55:28.501006+0000 mon.a (mon.0) 4620 : audit [DBG] from='client.? 172.21.15.6:0/1501256651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:30.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:30 smithi130 bash[51302]: debug 2023-08-28T01:55:30.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:30.087283+0000) 2023-08-28T01:55:30.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:30 smithi006 bash[27292]: cluster 2023-08-28T01:55:28.828914+0000 mgr.y (mgr.14901) 7106 : cluster [DBG] pgmap v7113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:30 smithi006 bash[33918]: cluster 2023-08-28T01:55:28.828914+0000 mgr.y (mgr.14901) 7106 : cluster [DBG] pgmap v7113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:30.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:30 smithi130 bash[27579]: cluster 2023-08-28T01:55:28.828914+0000 mgr.y (mgr.14901) 7106 : cluster [DBG] pgmap v7113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:31.157 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:55:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:55:30] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:55:31.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:31 smithi130 bash[51302]: debug 2023-08-28T01:55:31.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:31.087532+0000) 2023-08-28T01:55:31.446 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:55:31.157Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:31.446 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:55:31.160Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:31.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:31 smithi006 bash[27292]: audit 2023-08-28T01:55:30.910337+0000 mon.a (mon.0) 4621 : audit [DBG] from='client.? 172.21.15.6:0/3104293668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:31.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:31 smithi006 bash[33918]: audit 2023-08-28T01:55:30.910337+0000 mon.a (mon.0) 4621 : audit [DBG] from='client.? 172.21.15.6:0/3104293668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:31.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:31 smithi130 bash[27579]: audit 2023-08-28T01:55:30.910337+0000 mon.a (mon.0) 4621 : audit [DBG] from='client.? 172.21.15.6:0/3104293668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:32.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:32 smithi130 bash[51302]: debug 2023-08-28T01:55:32.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:32.087769+0000) 2023-08-28T01:55:32.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:32 smithi130 bash[27579]: cluster 2023-08-28T01:55:30.829987+0000 mgr.y (mgr.14901) 7107 : cluster [DBG] pgmap v7114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:32.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:32 smithi006 bash[27292]: cluster 2023-08-28T01:55:30.829987+0000 mgr.y (mgr.14901) 7107 : cluster [DBG] pgmap v7114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:32.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:32 smithi006 bash[33918]: cluster 2023-08-28T01:55:30.829987+0000 mgr.y (mgr.14901) 7107 : cluster [DBG] pgmap v7114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:55:32.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:55:32.723Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:33.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:33 smithi130 bash[51302]: debug 2023-08-28T01:55:33.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:33.088075+0000) 2023-08-28T01:55:34.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:34 smithi130 bash[51302]: debug 2023-08-28T01:55:34.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:34.088338+0000) 2023-08-28T01:55:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:34 smithi130 bash[27579]: cluster 2023-08-28T01:55:32.830649+0000 mgr.y (mgr.14901) 7108 : cluster [DBG] pgmap v7115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:34 smithi130 bash[27579]: audit 2023-08-28T01:55:33.070454+0000 mon.a (mon.0) 4622 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:55:34.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:34 smithi130 bash[27579]: audit 2023-08-28T01:55:33.326326+0000 mon.a (mon.0) 4623 : audit [DBG] from='client.? 172.21.15.6:0/3336886317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:34.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:34 smithi130 bash[27579]: audit 2023-08-28T01:55:33.365543+0000 mon.b (mon.1) 795 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:55:34.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:34 smithi130 bash[27579]: audit 2023-08-28T01:55:33.367465+0000 mon.b (mon.1) 796 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:55:34.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:34 smithi130 bash[27579]: audit 2023-08-28T01:55:33.375363+0000 mon.a (mon.0) 4624 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:55:34.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:34 smithi006 bash[27292]: cluster 2023-08-28T01:55:32.830649+0000 mgr.y (mgr.14901) 7108 : cluster [DBG] pgmap v7115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:34.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:34 smithi006 bash[27292]: audit 2023-08-28T01:55:33.070454+0000 mon.a (mon.0) 4622 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:55:34.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:34 smithi006 bash[27292]: audit 2023-08-28T01:55:33.326326+0000 mon.a (mon.0) 4623 : audit [DBG] from='client.? 172.21.15.6:0/3336886317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:34.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:34 smithi006 bash[27292]: audit 2023-08-28T01:55:33.365543+0000 mon.b (mon.1) 795 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:55:34.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:34 smithi006 bash[27292]: audit 2023-08-28T01:55:33.367465+0000 mon.b (mon.1) 796 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:55:34.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:34 smithi006 bash[27292]: audit 2023-08-28T01:55:33.375363+0000 mon.a (mon.0) 4624 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:55:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:34 smithi006 bash[33918]: cluster 2023-08-28T01:55:32.830649+0000 mgr.y (mgr.14901) 7108 : cluster [DBG] pgmap v7115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:34 smithi006 bash[33918]: audit 2023-08-28T01:55:33.070454+0000 mon.a (mon.0) 4622 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:55:34.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:34 smithi006 bash[33918]: audit 2023-08-28T01:55:33.326326+0000 mon.a (mon.0) 4623 : audit [DBG] from='client.? 172.21.15.6:0/3336886317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:34.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:34 smithi006 bash[33918]: audit 2023-08-28T01:55:33.365543+0000 mon.b (mon.1) 795 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:55:34.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:34 smithi006 bash[33918]: audit 2023-08-28T01:55:33.367465+0000 mon.b (mon.1) 796 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:55:34.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:34 smithi006 bash[33918]: audit 2023-08-28T01:55:33.375363+0000 mon.a (mon.0) 4624 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:55:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:55:34.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:55:34.609Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:35.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:35 smithi130 bash[51302]: debug 2023-08-28T01:55:35.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:35.088612+0000) 2023-08-28T01:55:36.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:36 smithi130 bash[27579]: cluster 2023-08-28T01:55:34.832026+0000 mgr.y (mgr.14901) 7109 : cluster [DBG] pgmap v7116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:36.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:36 smithi130 bash[27579]: audit 2023-08-28T01:55:35.744921+0000 mon.c (mon.2) 2797 : audit [DBG] from='client.? 172.21.15.6:0/1818026550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:36.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:36 smithi130 bash[51302]: debug 2023-08-28T01:55:36.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:36.088941+0000) 2023-08-28T01:55:36.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:36 smithi006 bash[27292]: cluster 2023-08-28T01:55:34.832026+0000 mgr.y (mgr.14901) 7109 : cluster [DBG] pgmap v7116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:36.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:36 smithi006 bash[27292]: audit 2023-08-28T01:55:35.744921+0000 mon.c (mon.2) 2797 : audit [DBG] from='client.? 172.21.15.6:0/1818026550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:36.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:36 smithi006 bash[33918]: cluster 2023-08-28T01:55:34.832026+0000 mgr.y (mgr.14901) 7109 : cluster [DBG] pgmap v7116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:36.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:36 smithi006 bash[33918]: audit 2023-08-28T01:55:35.744921+0000 mon.c (mon.2) 2797 : audit [DBG] from='client.? 172.21.15.6:0/1818026550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:37.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:37 smithi130 bash[51302]: debug 2023-08-28T01:55:37.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:37.089198+0000) 2023-08-28T01:55:38.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:38 smithi130 bash[51302]: debug 2023-08-28T01:55:38.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:38.089512+0000) 2023-08-28T01:55:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:38 smithi006 bash[33918]: cluster 2023-08-28T01:55:36.832901+0000 mgr.y (mgr.14901) 7110 : cluster [DBG] pgmap v7117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:38 smithi006 bash[33918]: audit 2023-08-28T01:55:38.159565+0000 mon.a (mon.0) 4625 : audit [DBG] from='client.? 172.21.15.6:0/4262082748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:38.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:38 smithi006 bash[27292]: cluster 2023-08-28T01:55:36.832901+0000 mgr.y (mgr.14901) 7110 : cluster [DBG] pgmap v7117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:38 smithi006 bash[27292]: audit 2023-08-28T01:55:38.159565+0000 mon.a (mon.0) 4625 : audit [DBG] from='client.? 172.21.15.6:0/4262082748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:38.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:38 smithi130 bash[27579]: cluster 2023-08-28T01:55:36.832901+0000 mgr.y (mgr.14901) 7110 : cluster [DBG] pgmap v7117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:38.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:38 smithi130 bash[27579]: audit 2023-08-28T01:55:38.159565+0000 mon.a (mon.0) 4625 : audit [DBG] from='client.? 172.21.15.6:0/4262082748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:39.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:39 smithi130 bash[51302]: debug 2023-08-28T01:55:39.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:39.089795+0000) 2023-08-28T01:55:40.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:40 smithi130 bash[51302]: debug 2023-08-28T01:55:40.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:40.090081+0000) 2023-08-28T01:55:40.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:40 smithi006 bash[27292]: cluster 2023-08-28T01:55:38.833540+0000 mgr.y (mgr.14901) 7111 : cluster [DBG] pgmap v7118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:40.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:40 smithi006 bash[33918]: cluster 2023-08-28T01:55:38.833540+0000 mgr.y (mgr.14901) 7111 : cluster [DBG] pgmap v7118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:40.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:40 smithi130 bash[27579]: cluster 2023-08-28T01:55:38.833540+0000 mgr.y (mgr.14901) 7111 : cluster [DBG] pgmap v7118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:41.158 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:55:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:55:40] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:55:41.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:41 smithi130 bash[51302]: debug 2023-08-28T01:55:41.087+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:41.090399+0000) 2023-08-28T01:55:41.440 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:55:41.158Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:41.440 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:55:41.160Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:41.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:41 smithi006 bash[33918]: audit 2023-08-28T01:55:40.587236+0000 mon.c (mon.2) 2798 : audit [DBG] from='client.? 172.21.15.6:0/2091232639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:41.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:41 smithi006 bash[27292]: audit 2023-08-28T01:55:40.587236+0000 mon.c (mon.2) 2798 : audit [DBG] from='client.? 172.21.15.6:0/2091232639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:41.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:41 smithi130 bash[27579]: audit 2023-08-28T01:55:40.587236+0000 mon.c (mon.2) 2798 : audit [DBG] from='client.? 172.21.15.6:0/2091232639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:42.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:42 smithi130 bash[51302]: debug 2023-08-28T01:55:42.083+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:42.090694+0000) 2023-08-28T01:55:42.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:42 smithi006 bash[27292]: cluster 2023-08-28T01:55:40.834654+0000 mgr.y (mgr.14901) 7112 : cluster [DBG] pgmap v7119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:42.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:42 smithi006 bash[33918]: cluster 2023-08-28T01:55:40.834654+0000 mgr.y (mgr.14901) 7112 : cluster [DBG] pgmap v7119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:42.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:42 smithi130 bash[27579]: cluster 2023-08-28T01:55:40.834654+0000 mgr.y (mgr.14901) 7112 : cluster [DBG] pgmap v7119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:55:42.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:55:42.725Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:43.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:43 smithi130 bash[51302]: debug 2023-08-28T01:55:43.087+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:43.090995+0000) 2023-08-28T01:55:43.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:43 smithi006 bash[27292]: audit 2023-08-28T01:55:42.992206+0000 mon.c (mon.2) 2799 : audit [DBG] from='client.? 172.21.15.6:0/1692214435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:43.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:43 smithi006 bash[33918]: audit 2023-08-28T01:55:42.992206+0000 mon.c (mon.2) 2799 : audit [DBG] from='client.? 172.21.15.6:0/1692214435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:43.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:43 smithi130 bash[27579]: audit 2023-08-28T01:55:42.992206+0000 mon.c (mon.2) 2799 : audit [DBG] from='client.? 172.21.15.6:0/1692214435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:44.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:44 smithi130 bash[51302]: debug 2023-08-28T01:55:44.087+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:44.091244+0000) 2023-08-28T01:55:44.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:44 smithi006 bash[27292]: cluster 2023-08-28T01:55:42.835110+0000 mgr.y (mgr.14901) 7113 : cluster [DBG] pgmap v7120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:44.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:44 smithi006 bash[33918]: cluster 2023-08-28T01:55:42.835110+0000 mgr.y (mgr.14901) 7113 : cluster [DBG] pgmap v7120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:44.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:55:44.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:44.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:55:44.610Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:44.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:44 smithi130 bash[27579]: cluster 2023-08-28T01:55:42.835110+0000 mgr.y (mgr.14901) 7113 : cluster [DBG] pgmap v7120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:45.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:45 smithi130 bash[51302]: debug 2023-08-28T01:55:45.090+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:45.091503+0000) 2023-08-28T01:55:45.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:45 smithi130 bash[27579]: audit 2023-08-28T01:55:45.410935+0000 mon.c (mon.2) 2800 : audit [DBG] from='client.? 172.21.15.6:0/886419127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:45.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:45 smithi006 bash[33918]: audit 2023-08-28T01:55:45.410935+0000 mon.c (mon.2) 2800 : audit [DBG] from='client.? 172.21.15.6:0/886419127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:45.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:45 smithi006 bash[27292]: audit 2023-08-28T01:55:45.410935+0000 mon.c (mon.2) 2800 : audit [DBG] from='client.? 172.21.15.6:0/886419127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:46.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:46 smithi130 bash[51302]: debug 2023-08-28T01:55:46.090+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:46.091782+0000) 2023-08-28T01:55:46.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:46 smithi130 bash[27579]: cluster 2023-08-28T01:55:44.836179+0000 mgr.y (mgr.14901) 7114 : cluster [DBG] pgmap v7121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:46.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:46 smithi006 bash[27292]: cluster 2023-08-28T01:55:44.836179+0000 mgr.y (mgr.14901) 7114 : cluster [DBG] pgmap v7121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:46.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:46 smithi006 bash[33918]: cluster 2023-08-28T01:55:44.836179+0000 mgr.y (mgr.14901) 7114 : cluster [DBG] pgmap v7121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:47.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:47 smithi130 bash[51302]: debug 2023-08-28T01:55:47.086+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:47.092059+0000) 2023-08-28T01:55:48.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:48 smithi130 bash[51302]: debug 2023-08-28T01:55:48.090+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:48.092337+0000) 2023-08-28T01:55:48.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:48 smithi130 bash[27579]: cluster 2023-08-28T01:55:46.836868+0000 mgr.y (mgr.14901) 7115 : cluster [DBG] pgmap v7122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:48.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:48 smithi130 bash[27579]: audit 2023-08-28T01:55:47.831174+0000 mon.a (mon.0) 4626 : audit [DBG] from='client.? 172.21.15.6:0/940676772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:48.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:48 smithi006 bash[33918]: cluster 2023-08-28T01:55:46.836868+0000 mgr.y (mgr.14901) 7115 : cluster [DBG] pgmap v7122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:48.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:48 smithi006 bash[33918]: audit 2023-08-28T01:55:47.831174+0000 mon.a (mon.0) 4626 : audit [DBG] from='client.? 172.21.15.6:0/940676772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:48.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:48 smithi006 bash[27292]: cluster 2023-08-28T01:55:46.836868+0000 mgr.y (mgr.14901) 7115 : cluster [DBG] pgmap v7122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:48.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:48 smithi006 bash[27292]: audit 2023-08-28T01:55:47.831174+0000 mon.a (mon.0) 4626 : audit [DBG] from='client.? 172.21.15.6:0/940676772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:49.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:49 smithi130 bash[51302]: debug 2023-08-28T01:55:49.090+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:49.092616+0000) 2023-08-28T01:55:50.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:50 smithi130 bash[51302]: debug 2023-08-28T01:55:50.086+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:50.092888+0000) 2023-08-28T01:55:50.751 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:50 smithi006 bash[33918]: cluster 2023-08-28T01:55:48.837498+0000 mgr.y (mgr.14901) 7116 : cluster [DBG] pgmap v7123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:50.751 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:50 smithi006 bash[33918]: audit 2023-08-28T01:55:50.240012+0000 mon.c (mon.2) 2801 : audit [DBG] from='client.? 172.21.15.6:0/140333880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:50.751 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:50 smithi006 bash[27292]: cluster 2023-08-28T01:55:48.837498+0000 mgr.y (mgr.14901) 7116 : cluster [DBG] pgmap v7123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:50.751 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:50 smithi006 bash[27292]: audit 2023-08-28T01:55:50.240012+0000 mon.c (mon.2) 2801 : audit [DBG] from='client.? 172.21.15.6:0/140333880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:50.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:50 smithi130 bash[27579]: cluster 2023-08-28T01:55:48.837498+0000 mgr.y (mgr.14901) 7116 : cluster [DBG] pgmap v7123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:50.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:50 smithi130 bash[27579]: audit 2023-08-28T01:55:50.240012+0000 mon.c (mon.2) 2801 : audit [DBG] from='client.? 172.21.15.6:0/140333880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:51.158 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:55:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:55:50] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:55:51.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:51 smithi130 bash[51302]: debug 2023-08-28T01:55:51.090+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:51.093233+0000) 2023-08-28T01:55:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:55:51.159Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:55:51.161Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:52.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:52 smithi130 bash[51302]: debug 2023-08-28T01:55:52.086+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:52.093540+0000) 2023-08-28T01:55:52.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:52 smithi130 bash[27579]: cluster 2023-08-28T01:55:50.838609+0000 mgr.y (mgr.14901) 7117 : cluster [DBG] pgmap v7124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:52.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:52 smithi006 bash[33918]: cluster 2023-08-28T01:55:50.838609+0000 mgr.y (mgr.14901) 7117 : cluster [DBG] pgmap v7124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:52.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:52 smithi006 bash[27292]: cluster 2023-08-28T01:55:50.838609+0000 mgr.y (mgr.14901) 7117 : cluster [DBG] pgmap v7124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:55:52.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:55:52.726Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:53.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:53 smithi130 bash[51302]: debug 2023-08-28T01:55:53.090+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:53.093847+0000) 2023-08-28T01:55:53.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:53 smithi130 bash[27579]: audit 2023-08-28T01:55:52.660471+0000 mon.a (mon.0) 4627 : audit [DBG] from='client.? 172.21.15.6:0/1564116400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:53.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:53 smithi006 bash[33918]: audit 2023-08-28T01:55:52.660471+0000 mon.a (mon.0) 4627 : audit [DBG] from='client.? 172.21.15.6:0/1564116400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:53.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:53 smithi006 bash[27292]: audit 2023-08-28T01:55:52.660471+0000 mon.a (mon.0) 4627 : audit [DBG] from='client.? 172.21.15.6:0/1564116400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:54.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:54 smithi130 bash[51302]: debug 2023-08-28T01:55:54.090+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:54.094132+0000) 2023-08-28T01:55:54.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:54 smithi130 bash[27579]: cluster 2023-08-28T01:55:52.839262+0000 mgr.y (mgr.14901) 7118 : cluster [DBG] pgmap v7125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:54.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:54 smithi006 bash[27292]: cluster 2023-08-28T01:55:52.839262+0000 mgr.y (mgr.14901) 7118 : cluster [DBG] pgmap v7125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:55:54.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:55:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:55:54.610Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:55:54.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:54 smithi006 bash[33918]: cluster 2023-08-28T01:55:52.839262+0000 mgr.y (mgr.14901) 7118 : cluster [DBG] pgmap v7125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:55.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:55 smithi130 bash[51302]: debug 2023-08-28T01:55:55.090+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:55.094424+0000) 2023-08-28T01:55:55.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:55 smithi130 bash[27579]: audit 2023-08-28T01:55:55.078764+0000 mon.a (mon.0) 4628 : audit [DBG] from='client.? 172.21.15.6:0/3917293612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:55.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:55 smithi006 bash[27292]: audit 2023-08-28T01:55:55.078764+0000 mon.a (mon.0) 4628 : audit [DBG] from='client.? 172.21.15.6:0/3917293612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:55.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:55 smithi006 bash[33918]: audit 2023-08-28T01:55:55.078764+0000 mon.a (mon.0) 4628 : audit [DBG] from='client.? 172.21.15.6:0/3917293612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:56.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:56 smithi130 bash[51302]: debug 2023-08-28T01:55:56.090+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:56.094754+0000) 2023-08-28T01:55:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:56 smithi130 bash[27579]: cluster 2023-08-28T01:55:54.840367+0000 mgr.y (mgr.14901) 7119 : cluster [DBG] pgmap v7126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:56.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:56 smithi006 bash[27292]: cluster 2023-08-28T01:55:54.840367+0000 mgr.y (mgr.14901) 7119 : cluster [DBG] pgmap v7126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:56.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:56 smithi006 bash[33918]: cluster 2023-08-28T01:55:54.840367+0000 mgr.y (mgr.14901) 7119 : cluster [DBG] pgmap v7126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:57.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:57 smithi130 bash[51302]: debug 2023-08-28T01:55:57.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:57.095032+0000) 2023-08-28T01:55:58.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:58 smithi130 bash[51302]: debug 2023-08-28T01:55:58.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:58.095314+0000) 2023-08-28T01:55:58.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:58 smithi006 bash[33918]: cluster 2023-08-28T01:55:56.841059+0000 mgr.y (mgr.14901) 7120 : cluster [DBG] pgmap v7127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:58.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:55:58 smithi006 bash[33918]: audit 2023-08-28T01:55:57.497901+0000 mon.c (mon.2) 2802 : audit [DBG] from='client.? 172.21.15.6:0/881369484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:58.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:58 smithi006 bash[27292]: cluster 2023-08-28T01:55:56.841059+0000 mgr.y (mgr.14901) 7120 : cluster [DBG] pgmap v7127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:58.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:55:58 smithi006 bash[27292]: audit 2023-08-28T01:55:57.497901+0000 mon.c (mon.2) 2802 : audit [DBG] from='client.? 172.21.15.6:0/881369484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:58.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:58 smithi130 bash[27579]: cluster 2023-08-28T01:55:56.841059+0000 mgr.y (mgr.14901) 7120 : cluster [DBG] pgmap v7127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:55:58.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:55:58 smithi130 bash[27579]: audit 2023-08-28T01:55:57.497901+0000 mon.c (mon.2) 2802 : audit [DBG] from='client.? 172.21.15.6:0/881369484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:59.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:55:59 smithi130 bash[51302]: debug 2023-08-28T01:55:59.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:59.095592+0000) 2023-08-28T01:56:00.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:00 smithi130 bash[51302]: debug 2023-08-28T01:56:00.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:00.095867+0000) 2023-08-28T01:56:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:00 smithi006 bash[33918]: cluster 2023-08-28T01:55:58.841711+0000 mgr.y (mgr.14901) 7121 : cluster [DBG] pgmap v7128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:00 smithi006 bash[33918]: audit 2023-08-28T01:55:59.915774+0000 mon.c (mon.2) 2803 : audit [DBG] from='client.? 172.21.15.6:0/483183959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:00 smithi006 bash[27292]: cluster 2023-08-28T01:55:58.841711+0000 mgr.y (mgr.14901) 7121 : cluster [DBG] pgmap v7128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:00 smithi006 bash[27292]: audit 2023-08-28T01:55:59.915774+0000 mon.c (mon.2) 2803 : audit [DBG] from='client.? 172.21.15.6:0/483183959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:00.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:00 smithi130 bash[27579]: cluster 2023-08-28T01:55:58.841711+0000 mgr.y (mgr.14901) 7121 : cluster [DBG] pgmap v7128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:00 smithi130 bash[27579]: audit 2023-08-28T01:55:59.915774+0000 mon.c (mon.2) 2803 : audit [DBG] from='client.? 172.21.15.6:0/483183959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:01.159 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:56:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:56:00] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T01:56:01.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:01 smithi130 bash[51302]: debug 2023-08-28T01:56:01.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:01.096195+0000) 2023-08-28T01:56:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:56:01.159Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:56:01.162Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:02.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:02 smithi130 bash[51302]: debug 2023-08-28T01:56:02.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:02.096396+0000) 2023-08-28T01:56:02.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:02 smithi006 bash[33918]: cluster 2023-08-28T01:56:00.842869+0000 mgr.y (mgr.14901) 7122 : cluster [DBG] pgmap v7129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:02.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:02 smithi006 bash[33918]: audit 2023-08-28T01:56:02.324463+0000 mon.c (mon.2) 2804 : audit [DBG] from='client.? 172.21.15.6:0/3335671508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:02.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:02 smithi006 bash[27292]: cluster 2023-08-28T01:56:00.842869+0000 mgr.y (mgr.14901) 7122 : cluster [DBG] pgmap v7129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:02.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:02 smithi006 bash[27292]: audit 2023-08-28T01:56:02.324463+0000 mon.c (mon.2) 2804 : audit [DBG] from='client.? 172.21.15.6:0/3335671508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:02.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:02 smithi130 bash[27579]: cluster 2023-08-28T01:56:00.842869+0000 mgr.y (mgr.14901) 7122 : cluster [DBG] pgmap v7129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:02 smithi130 bash[27579]: audit 2023-08-28T01:56:02.324463+0000 mon.c (mon.2) 2804 : audit [DBG] from='client.? 172.21.15.6:0/3335671508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:56:02.725Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:56:02.727Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:03.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:03 smithi130 bash[51302]: debug 2023-08-28T01:56:03.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:03.096684+0000) 2023-08-28T01:56:04.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:04 smithi130 bash[51302]: debug 2023-08-28T01:56:04.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:04.097014+0000) 2023-08-28T01:56:04.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:04 smithi006 bash[27292]: cluster 2023-08-28T01:56:02.843520+0000 mgr.y (mgr.14901) 7123 : cluster [DBG] pgmap v7130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:04.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:04 smithi006 bash[33918]: cluster 2023-08-28T01:56:02.843520+0000 mgr.y (mgr.14901) 7123 : cluster [DBG] pgmap v7130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:56:04.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:04.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:56:04.611Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:04.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:04 smithi130 bash[27579]: cluster 2023-08-28T01:56:02.843520+0000 mgr.y (mgr.14901) 7123 : cluster [DBG] pgmap v7130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:05.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:05 smithi130 bash[51302]: debug 2023-08-28T01:56:05.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:05.097303+0000) 2023-08-28T01:56:05.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:05 smithi130 bash[27579]: audit 2023-08-28T01:56:04.735755+0000 mon.a (mon.0) 4629 : audit [DBG] from='client.? 172.21.15.6:0/2388887986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:05.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:05 smithi006 bash[27292]: audit 2023-08-28T01:56:04.735755+0000 mon.a (mon.0) 4629 : audit [DBG] from='client.? 172.21.15.6:0/2388887986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:05.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:05 smithi006 bash[33918]: audit 2023-08-28T01:56:04.735755+0000 mon.a (mon.0) 4629 : audit [DBG] from='client.? 172.21.15.6:0/2388887986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:06.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:06 smithi130 bash[51302]: debug 2023-08-28T01:56:06.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:06.097605+0000) 2023-08-28T01:56:06.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:06 smithi130 bash[27579]: cluster 2023-08-28T01:56:04.844778+0000 mgr.y (mgr.14901) 7124 : cluster [DBG] pgmap v7131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:06.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:06 smithi006 bash[33918]: cluster 2023-08-28T01:56:04.844778+0000 mgr.y (mgr.14901) 7124 : cluster [DBG] pgmap v7131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:06.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:06 smithi006 bash[27292]: cluster 2023-08-28T01:56:04.844778+0000 mgr.y (mgr.14901) 7124 : cluster [DBG] pgmap v7131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:07.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:07 smithi130 bash[51302]: debug 2023-08-28T01:56:07.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:07.097882+0000) 2023-08-28T01:56:07.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:07 smithi130 bash[27579]: audit 2023-08-28T01:56:07.153080+0000 mon.c (mon.2) 2805 : audit [DBG] from='client.? 172.21.15.6:0/1097593926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:07.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:07 smithi006 bash[33918]: audit 2023-08-28T01:56:07.153080+0000 mon.c (mon.2) 2805 : audit [DBG] from='client.? 172.21.15.6:0/1097593926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:07.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:07 smithi006 bash[27292]: audit 2023-08-28T01:56:07.153080+0000 mon.c (mon.2) 2805 : audit [DBG] from='client.? 172.21.15.6:0/1097593926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:08.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:08 smithi130 bash[51302]: debug 2023-08-28T01:56:08.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:08.098161+0000) 2023-08-28T01:56:08.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:08 smithi130 bash[27579]: cluster 2023-08-28T01:56:06.845475+0000 mgr.y (mgr.14901) 7125 : cluster [DBG] pgmap v7132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:08.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:08 smithi006 bash[33918]: cluster 2023-08-28T01:56:06.845475+0000 mgr.y (mgr.14901) 7125 : cluster [DBG] pgmap v7132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:08.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:08 smithi006 bash[27292]: cluster 2023-08-28T01:56:06.845475+0000 mgr.y (mgr.14901) 7125 : cluster [DBG] pgmap v7132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:09.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:09 smithi130 bash[51302]: debug 2023-08-28T01:56:09.094+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:09.098437+0000) 2023-08-28T01:56:10.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:10 smithi130 bash[51302]: debug 2023-08-28T01:56:10.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:10.098722+0000) 2023-08-28T01:56:10.752 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:10 smithi006 bash[27292]: cluster 2023-08-28T01:56:08.846171+0000 mgr.y (mgr.14901) 7126 : cluster [DBG] pgmap v7133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:10.752 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:10 smithi006 bash[27292]: audit 2023-08-28T01:56:09.568109+0000 mon.a (mon.0) 4630 : audit [DBG] from='client.? 172.21.15.6:0/2789038995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:10.752 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:10 smithi006 bash[33918]: cluster 2023-08-28T01:56:08.846171+0000 mgr.y (mgr.14901) 7126 : cluster [DBG] pgmap v7133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:10.753 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:10 smithi006 bash[33918]: audit 2023-08-28T01:56:09.568109+0000 mon.a (mon.0) 4630 : audit [DBG] from='client.? 172.21.15.6:0/2789038995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:10.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:10 smithi130 bash[27579]: cluster 2023-08-28T01:56:08.846171+0000 mgr.y (mgr.14901) 7126 : cluster [DBG] pgmap v7133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:10.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:10 smithi130 bash[27579]: audit 2023-08-28T01:56:09.568109+0000 mon.a (mon.0) 4630 : audit [DBG] from='client.? 172.21.15.6:0/2789038995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:11.160 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:56:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:56:10] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T01:56:11.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:11 smithi130 bash[51302]: debug 2023-08-28T01:56:11.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:11.099044+0000) 2023-08-28T01:56:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:56:11.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:56:11.162Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:12.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:12 smithi130 bash[51302]: debug 2023-08-28T01:56:12.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:12.099223+0000) 2023-08-28T01:56:12.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:12 smithi130 bash[27579]: cluster 2023-08-28T01:56:10.847388+0000 mgr.y (mgr.14901) 7127 : cluster [DBG] pgmap v7134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:12 smithi130 bash[27579]: audit 2023-08-28T01:56:11.983980+0000 mon.a (mon.0) 4631 : audit [DBG] from='client.? 172.21.15.6:0/2021743570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:12.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:12 smithi006 bash[33918]: cluster 2023-08-28T01:56:10.847388+0000 mgr.y (mgr.14901) 7127 : cluster [DBG] pgmap v7134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:12.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:12 smithi006 bash[33918]: audit 2023-08-28T01:56:11.983980+0000 mon.a (mon.0) 4631 : audit [DBG] from='client.? 172.21.15.6:0/2021743570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:56:12.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:56:12.728Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:12.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:12 smithi006 bash[27292]: cluster 2023-08-28T01:56:10.847388+0000 mgr.y (mgr.14901) 7127 : cluster [DBG] pgmap v7134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:12.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:12 smithi006 bash[27292]: audit 2023-08-28T01:56:11.983980+0000 mon.a (mon.0) 4631 : audit [DBG] from='client.? 172.21.15.6:0/2021743570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:13.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:13 smithi130 bash[51302]: debug 2023-08-28T01:56:13.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:13.099477+0000) 2023-08-28T01:56:14.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:14 smithi130 bash[51302]: debug 2023-08-28T01:56:14.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:14.099751+0000) 2023-08-28T01:56:14.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:14 smithi130 bash[27579]: cluster 2023-08-28T01:56:12.848082+0000 mgr.y (mgr.14901) 7128 : cluster [DBG] pgmap v7135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:14.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:14 smithi130 bash[27579]: audit 2023-08-28T01:56:14.390509+0000 mon.a (mon.0) 4632 : audit [DBG] from='client.? 172.21.15.6:0/105673239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:14.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:14 smithi006 bash[27292]: cluster 2023-08-28T01:56:12.848082+0000 mgr.y (mgr.14901) 7128 : cluster [DBG] pgmap v7135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:14.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:14 smithi006 bash[27292]: audit 2023-08-28T01:56:14.390509+0000 mon.a (mon.0) 4632 : audit [DBG] from='client.? 172.21.15.6:0/105673239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:14.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:14 smithi006 bash[33918]: cluster 2023-08-28T01:56:12.848082+0000 mgr.y (mgr.14901) 7128 : cluster [DBG] pgmap v7135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:14.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:14 smithi006 bash[33918]: audit 2023-08-28T01:56:14.390509+0000 mon.a (mon.0) 4632 : audit [DBG] from='client.? 172.21.15.6:0/105673239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:56:14.609Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:56:14.611Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:15.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:15 smithi130 bash[51302]: debug 2023-08-28T01:56:15.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:15.100044+0000) 2023-08-28T01:56:16.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:16 smithi130 bash[51302]: debug 2023-08-28T01:56:16.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:16.100309+0000) 2023-08-28T01:56:16.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:16 smithi130 bash[27579]: cluster 2023-08-28T01:56:14.849245+0000 mgr.y (mgr.14901) 7129 : cluster [DBG] pgmap v7136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:16.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:16 smithi006 bash[33918]: cluster 2023-08-28T01:56:14.849245+0000 mgr.y (mgr.14901) 7129 : cluster [DBG] pgmap v7136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:16.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:16 smithi006 bash[27292]: cluster 2023-08-28T01:56:14.849245+0000 mgr.y (mgr.14901) 7129 : cluster [DBG] pgmap v7136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:17.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:17 smithi130 bash[51302]: debug 2023-08-28T01:56:17.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:17.100584+0000) 2023-08-28T01:56:17.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:17 smithi130 bash[27579]: audit 2023-08-28T01:56:16.812325+0000 mon.a (mon.0) 4633 : audit [DBG] from='client.? 172.21.15.6:0/126665727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:17.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:17 smithi006 bash[27292]: audit 2023-08-28T01:56:16.812325+0000 mon.a (mon.0) 4633 : audit [DBG] from='client.? 172.21.15.6:0/126665727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:17.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:17 smithi006 bash[33918]: audit 2023-08-28T01:56:16.812325+0000 mon.a (mon.0) 4633 : audit [DBG] from='client.? 172.21.15.6:0/126665727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:18.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:18 smithi130 bash[51302]: debug 2023-08-28T01:56:18.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:18.100929+0000) 2023-08-28T01:56:18.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:18 smithi130 bash[27579]: cluster 2023-08-28T01:56:16.849868+0000 mgr.y (mgr.14901) 7130 : cluster [DBG] pgmap v7137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:18.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:18 smithi006 bash[27292]: cluster 2023-08-28T01:56:16.849868+0000 mgr.y (mgr.14901) 7130 : cluster [DBG] pgmap v7137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:18.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:18 smithi006 bash[33918]: cluster 2023-08-28T01:56:16.849868+0000 mgr.y (mgr.14901) 7130 : cluster [DBG] pgmap v7137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:19.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:19 smithi130 bash[51302]: debug 2023-08-28T01:56:19.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:19.101211+0000) 2023-08-28T01:56:19.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:19 smithi130 bash[27579]: cluster 2023-08-28T01:56:18.850598+0000 mgr.y (mgr.14901) 7131 : cluster [DBG] pgmap v7138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:19.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:19 smithi130 bash[27579]: audit 2023-08-28T01:56:19.231648+0000 mon.a (mon.0) 4634 : audit [DBG] from='client.? 172.21.15.6:0/3080250620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:19.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:19 smithi006 bash[33918]: cluster 2023-08-28T01:56:18.850598+0000 mgr.y (mgr.14901) 7131 : cluster [DBG] pgmap v7138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:19.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:19 smithi006 bash[33918]: audit 2023-08-28T01:56:19.231648+0000 mon.a (mon.0) 4634 : audit [DBG] from='client.? 172.21.15.6:0/3080250620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:19.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:19 smithi006 bash[27292]: cluster 2023-08-28T01:56:18.850598+0000 mgr.y (mgr.14901) 7131 : cluster [DBG] pgmap v7138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:19.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:19 smithi006 bash[27292]: audit 2023-08-28T01:56:19.231648+0000 mon.a (mon.0) 4634 : audit [DBG] from='client.? 172.21.15.6:0/3080250620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:20.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:20 smithi130 bash[51302]: debug 2023-08-28T01:56:20.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:20.101489+0000) 2023-08-28T01:56:21.161 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:56:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:56:20] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T01:56:21.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:21 smithi130 bash[51302]: debug 2023-08-28T01:56:21.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:21.101796+0000) 2023-08-28T01:56:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:56:21.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:56:21.163Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:22.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:21 smithi006 bash[27292]: cluster 2023-08-28T01:56:20.851742+0000 mgr.y (mgr.14901) 7132 : cluster [DBG] pgmap v7139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:22.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:21 smithi006 bash[27292]: audit 2023-08-28T01:56:21.647952+0000 mon.c (mon.2) 2806 : audit [DBG] from='client.? 172.21.15.6:0/2078830233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:22.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:21 smithi006 bash[33918]: cluster 2023-08-28T01:56:20.851742+0000 mgr.y (mgr.14901) 7132 : cluster [DBG] pgmap v7139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:22.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:21 smithi006 bash[33918]: audit 2023-08-28T01:56:21.647952+0000 mon.c (mon.2) 2806 : audit [DBG] from='client.? 172.21.15.6:0/2078830233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:22.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:22 smithi130 bash[51302]: debug 2023-08-28T01:56:22.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:22.101982+0000) 2023-08-28T01:56:22.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:21 smithi130 bash[27579]: cluster 2023-08-28T01:56:20.851742+0000 mgr.y (mgr.14901) 7132 : cluster [DBG] pgmap v7139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:22.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:21 smithi130 bash[27579]: audit 2023-08-28T01:56:21.647952+0000 mon.c (mon.2) 2806 : audit [DBG] from='client.? 172.21.15.6:0/2078830233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:56:22.727Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:56:22.729Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:23.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:23 smithi130 bash[51302]: debug 2023-08-28T01:56:23.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:23.102225+0000) 2023-08-28T01:56:24.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:24 smithi130 bash[51302]: debug 2023-08-28T01:56:24.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:24.102543+0000) 2023-08-28T01:56:24.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:24 smithi006 bash[33918]: cluster 2023-08-28T01:56:22.852418+0000 mgr.y (mgr.14901) 7133 : cluster [DBG] pgmap v7140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:24.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:24 smithi006 bash[33918]: audit 2023-08-28T01:56:24.063884+0000 mon.a (mon.0) 4635 : audit [DBG] from='client.? 172.21.15.6:0/691915629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:56:24.609Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:24.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:56:24.611Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:24.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:24 smithi006 bash[27292]: cluster 2023-08-28T01:56:22.852418+0000 mgr.y (mgr.14901) 7133 : cluster [DBG] pgmap v7140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:24.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:24 smithi006 bash[27292]: audit 2023-08-28T01:56:24.063884+0000 mon.a (mon.0) 4635 : audit [DBG] from='client.? 172.21.15.6:0/691915629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:24.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:24 smithi130 bash[27579]: cluster 2023-08-28T01:56:22.852418+0000 mgr.y (mgr.14901) 7133 : cluster [DBG] pgmap v7140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:24.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:24 smithi130 bash[27579]: audit 2023-08-28T01:56:24.063884+0000 mon.a (mon.0) 4635 : audit [DBG] from='client.? 172.21.15.6:0/691915629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:25.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:25 smithi130 bash[51302]: debug 2023-08-28T01:56:25.098+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:25.102809+0000) 2023-08-28T01:56:26.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:26 smithi130 bash[51302]: debug 2023-08-28T01:56:26.102+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:26.103120+0000) 2023-08-28T01:56:26.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:26 smithi006 bash[27292]: cluster 2023-08-28T01:56:24.853602+0000 mgr.y (mgr.14901) 7134 : cluster [DBG] pgmap v7141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:26.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:26 smithi006 bash[33918]: cluster 2023-08-28T01:56:24.853602+0000 mgr.y (mgr.14901) 7134 : cluster [DBG] pgmap v7141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:26.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:26 smithi130 bash[27579]: cluster 2023-08-28T01:56:24.853602+0000 mgr.y (mgr.14901) 7134 : cluster [DBG] pgmap v7141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:27.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:27 smithi130 bash[51302]: debug 2023-08-28T01:56:27.102+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:27.103389+0000) 2023-08-28T01:56:27.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:27 smithi130 bash[27579]: audit 2023-08-28T01:56:26.481476+0000 mon.c (mon.2) 2807 : audit [DBG] from='client.? 172.21.15.6:0/1674203228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:27.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:27 smithi006 bash[33918]: audit 2023-08-28T01:56:26.481476+0000 mon.c (mon.2) 2807 : audit [DBG] from='client.? 172.21.15.6:0/1674203228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:27.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:27 smithi006 bash[27292]: audit 2023-08-28T01:56:26.481476+0000 mon.c (mon.2) 2807 : audit [DBG] from='client.? 172.21.15.6:0/1674203228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:28.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:28 smithi130 bash[51302]: debug 2023-08-28T01:56:28.102+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:28.103689+0000) 2023-08-28T01:56:28.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:28 smithi130 bash[27579]: cluster 2023-08-28T01:56:26.854259+0000 mgr.y (mgr.14901) 7135 : cluster [DBG] pgmap v7142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:28.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:28 smithi006 bash[33918]: cluster 2023-08-28T01:56:26.854259+0000 mgr.y (mgr.14901) 7135 : cluster [DBG] pgmap v7142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:28.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:28 smithi006 bash[27292]: cluster 2023-08-28T01:56:26.854259+0000 mgr.y (mgr.14901) 7135 : cluster [DBG] pgmap v7142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:29.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:29 smithi130 bash[51302]: debug 2023-08-28T01:56:29.102+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:29.104018+0000) 2023-08-28T01:56:29.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:29 smithi130 bash[27579]: audit 2023-08-28T01:56:28.896673+0000 mon.a (mon.0) 4636 : audit [DBG] from='client.? 172.21.15.6:0/2097680611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:29.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:29 smithi006 bash[33918]: audit 2023-08-28T01:56:28.896673+0000 mon.a (mon.0) 4636 : audit [DBG] from='client.? 172.21.15.6:0/2097680611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:29.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:29 smithi006 bash[27292]: audit 2023-08-28T01:56:28.896673+0000 mon.a (mon.0) 4636 : audit [DBG] from='client.? 172.21.15.6:0/2097680611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:30.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:30 smithi130 bash[51302]: debug 2023-08-28T01:56:30.102+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:30.104299+0000) 2023-08-28T01:56:30.751 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:30 smithi006 bash[27292]: cluster 2023-08-28T01:56:28.854723+0000 mgr.y (mgr.14901) 7136 : cluster [DBG] pgmap v7143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:30.751 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:30 smithi006 bash[33918]: cluster 2023-08-28T01:56:28.854723+0000 mgr.y (mgr.14901) 7136 : cluster [DBG] pgmap v7143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:30.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:30 smithi130 bash[27579]: cluster 2023-08-28T01:56:28.854723+0000 mgr.y (mgr.14901) 7136 : cluster [DBG] pgmap v7143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:31.161 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:56:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:56:30] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:56:31.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:31 smithi130 bash[51302]: debug 2023-08-28T01:56:31.102+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:31.104587+0000) 2023-08-28T01:56:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:56:31.162Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:56:31.164Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:31.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:31 smithi130 bash[27579]: audit 2023-08-28T01:56:31.307408+0000 mon.a (mon.0) 4637 : audit [DBG] from='client.? 172.21.15.6:0/476103723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:31.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:31 smithi006 bash[27292]: audit 2023-08-28T01:56:31.307408+0000 mon.a (mon.0) 4637 : audit [DBG] from='client.? 172.21.15.6:0/476103723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:31.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:31 smithi006 bash[33918]: audit 2023-08-28T01:56:31.307408+0000 mon.a (mon.0) 4637 : audit [DBG] from='client.? 172.21.15.6:0/476103723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:32.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:32 smithi130 bash[51302]: debug 2023-08-28T01:56:32.102+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:32.104863+0000) 2023-08-28T01:56:32.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:32 smithi130 bash[27579]: cluster 2023-08-28T01:56:30.855658+0000 mgr.y (mgr.14901) 7137 : cluster [DBG] pgmap v7144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:32.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:32 smithi006 bash[27292]: cluster 2023-08-28T01:56:30.855658+0000 mgr.y (mgr.14901) 7137 : cluster [DBG] pgmap v7144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:32.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:32 smithi006 bash[33918]: cluster 2023-08-28T01:56:30.855658+0000 mgr.y (mgr.14901) 7137 : cluster [DBG] pgmap v7144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:56:32.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:56:32.730Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:33.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:33 smithi130 bash[51302]: debug 2023-08-28T01:56:33.102+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:33.105111+0000) 2023-08-28T01:56:33.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:33 smithi130 bash[27579]: audit 2023-08-28T01:56:33.386222+0000 mon.b (mon.1) 797 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:56:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:33 smithi006 bash[27292]: audit 2023-08-28T01:56:33.386222+0000 mon.b (mon.1) 797 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:56:33.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:33 smithi006 bash[33918]: audit 2023-08-28T01:56:33.386222+0000 mon.b (mon.1) 797 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:56:34.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:34 smithi130 bash[51302]: debug 2023-08-28T01:56:34.102+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:34.105393+0000) 2023-08-28T01:56:34.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:34 smithi130 bash[27579]: cluster 2023-08-28T01:56:32.856296+0000 mgr.y (mgr.14901) 7138 : cluster [DBG] pgmap v7145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:34.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:34 smithi130 bash[27579]: audit 2023-08-28T01:56:33.728137+0000 mon.a (mon.0) 4638 : audit [DBG] from='client.? 172.21.15.6:0/351607844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:34.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:34 smithi006 bash[33918]: cluster 2023-08-28T01:56:32.856296+0000 mgr.y (mgr.14901) 7138 : cluster [DBG] pgmap v7145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:34.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:34 smithi006 bash[33918]: audit 2023-08-28T01:56:33.728137+0000 mon.a (mon.0) 4638 : audit [DBG] from='client.? 172.21.15.6:0/351607844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:56:34.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:56:34.612Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:34.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:34 smithi006 bash[27292]: cluster 2023-08-28T01:56:32.856296+0000 mgr.y (mgr.14901) 7138 : cluster [DBG] pgmap v7145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:34.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:34 smithi006 bash[27292]: audit 2023-08-28T01:56:33.728137+0000 mon.a (mon.0) 4638 : audit [DBG] from='client.? 172.21.15.6:0/351607844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:35.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:35 smithi130 bash[51302]: debug 2023-08-28T01:56:35.102+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:35.105673+0000) 2023-08-28T01:56:36.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:36 smithi130 bash[51302]: debug 2023-08-28T01:56:36.102+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:36.105950+0000) 2023-08-28T01:56:36.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:36 smithi130 bash[27579]: cluster 2023-08-28T01:56:34.857601+0000 mgr.y (mgr.14901) 7139 : cluster [DBG] pgmap v7146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:36.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:36 smithi130 bash[27579]: audit 2023-08-28T01:56:36.142288+0000 mon.c (mon.2) 2808 : audit [DBG] from='client.? 172.21.15.6:0/4293147811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:36.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:36 smithi006 bash[27292]: cluster 2023-08-28T01:56:34.857601+0000 mgr.y (mgr.14901) 7139 : cluster [DBG] pgmap v7146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:36.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:36 smithi006 bash[27292]: audit 2023-08-28T01:56:36.142288+0000 mon.c (mon.2) 2808 : audit [DBG] from='client.? 172.21.15.6:0/4293147811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:36.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:36 smithi006 bash[33918]: cluster 2023-08-28T01:56:34.857601+0000 mgr.y (mgr.14901) 7139 : cluster [DBG] pgmap v7146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:36.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:36 smithi006 bash[33918]: audit 2023-08-28T01:56:36.142288+0000 mon.c (mon.2) 2808 : audit [DBG] from='client.? 172.21.15.6:0/4293147811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:37.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:37 smithi130 bash[51302]: debug 2023-08-28T01:56:37.102+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:37.106232+0000) 2023-08-28T01:56:38.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:38 smithi130 bash[51302]: debug 2023-08-28T01:56:38.102+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:38.106566+0000) 2023-08-28T01:56:38.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:38 smithi130 bash[27579]: cluster 2023-08-28T01:56:36.858451+0000 mgr.y (mgr.14901) 7140 : cluster [DBG] pgmap v7147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:38.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:38 smithi006 bash[27292]: cluster 2023-08-28T01:56:36.858451+0000 mgr.y (mgr.14901) 7140 : cluster [DBG] pgmap v7147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:38.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:38 smithi006 bash[33918]: cluster 2023-08-28T01:56:36.858451+0000 mgr.y (mgr.14901) 7140 : cluster [DBG] pgmap v7147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:39.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:39 smithi130 bash[51302]: debug 2023-08-28T01:56:39.106+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:39.106848+0000) 2023-08-28T01:56:39.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:39 smithi130 bash[27579]: audit 2023-08-28T01:56:38.579495+0000 mon.a (mon.0) 4639 : audit [DBG] from='client.? 172.21.15.6:0/1030823976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:39.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:39 smithi130 bash[27579]: cluster 2023-08-28T01:56:38.859106+0000 mgr.y (mgr.14901) 7141 : cluster [DBG] pgmap v7148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:39.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:39 smithi130 bash[27579]: audit 2023-08-28T01:56:39.270345+0000 mon.a (mon.0) 4640 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:56:39.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:39 smithi006 bash[33918]: audit 2023-08-28T01:56:38.579495+0000 mon.a (mon.0) 4639 : audit [DBG] from='client.? 172.21.15.6:0/1030823976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:39.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:39 smithi006 bash[33918]: cluster 2023-08-28T01:56:38.859106+0000 mgr.y (mgr.14901) 7141 : cluster [DBG] pgmap v7148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:39.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:39 smithi006 bash[33918]: audit 2023-08-28T01:56:39.270345+0000 mon.a (mon.0) 4640 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:56:39.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:39 smithi006 bash[27292]: audit 2023-08-28T01:56:38.579495+0000 mon.a (mon.0) 4639 : audit [DBG] from='client.? 172.21.15.6:0/1030823976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:39.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:39 smithi006 bash[27292]: cluster 2023-08-28T01:56:38.859106+0000 mgr.y (mgr.14901) 7141 : cluster [DBG] pgmap v7148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:39.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:39 smithi006 bash[27292]: audit 2023-08-28T01:56:39.270345+0000 mon.a (mon.0) 4640 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:56:40.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:40 smithi130 bash[51302]: debug 2023-08-28T01:56:40.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:40.107148+0000) 2023-08-28T01:56:41.105 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:40 smithi130 bash[27579]: audit 2023-08-28T01:56:39.720603+0000 mon.a (mon.0) 4641 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:56:41.105 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:40 smithi130 bash[27579]: audit 2023-08-28T01:56:39.996401+0000 mon.b (mon.1) 798 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:56:41.105 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:40 smithi130 bash[27579]: audit 2023-08-28T01:56:39.998942+0000 mon.b (mon.1) 799 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:56:41.105 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:40 smithi130 bash[27579]: audit 2023-08-28T01:56:40.006873+0000 mon.a (mon.0) 4642 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:56:41.162 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:56:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:56:40] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:56:41.162 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:40 smithi006 bash[27292]: audit 2023-08-28T01:56:39.720603+0000 mon.a (mon.0) 4641 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:56:41.163 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:40 smithi006 bash[27292]: audit 2023-08-28T01:56:39.996401+0000 mon.b (mon.1) 798 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:56:41.163 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:40 smithi006 bash[27292]: audit 2023-08-28T01:56:39.998942+0000 mon.b (mon.1) 799 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:56:41.163 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:40 smithi006 bash[27292]: audit 2023-08-28T01:56:40.006873+0000 mon.a (mon.0) 4642 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:56:41.163 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:40 smithi006 bash[33918]: audit 2023-08-28T01:56:39.720603+0000 mon.a (mon.0) 4641 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:56:41.163 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:40 smithi006 bash[33918]: audit 2023-08-28T01:56:39.996401+0000 mon.b (mon.1) 798 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:56:41.163 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:40 smithi006 bash[33918]: audit 2023-08-28T01:56:39.998942+0000 mon.b (mon.1) 799 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:56:41.163 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:40 smithi006 bash[33918]: audit 2023-08-28T01:56:40.006873+0000 mon.a (mon.0) 4642 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:56:41.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:41 smithi130 bash[51302]: debug 2023-08-28T01:56:41.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:41.107431+0000) 2023-08-28T01:56:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:56:41.162Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:56:41.164Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:42.105 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:41 smithi130 bash[27579]: cluster 2023-08-28T01:56:40.860157+0000 mgr.y (mgr.14901) 7142 : cluster [DBG] pgmap v7149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:42.105 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:41 smithi130 bash[27579]: audit 2023-08-28T01:56:40.993888+0000 mon.a (mon.0) 4643 : audit [DBG] from='client.? 172.21.15.6:0/3466170963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:42.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:41 smithi006 bash[33918]: cluster 2023-08-28T01:56:40.860157+0000 mgr.y (mgr.14901) 7142 : cluster [DBG] pgmap v7149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:42.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:41 smithi006 bash[33918]: audit 2023-08-28T01:56:40.993888+0000 mon.a (mon.0) 4643 : audit [DBG] from='client.? 172.21.15.6:0/3466170963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:42.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:41 smithi006 bash[27292]: cluster 2023-08-28T01:56:40.860157+0000 mgr.y (mgr.14901) 7142 : cluster [DBG] pgmap v7149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:42.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:41 smithi006 bash[27292]: audit 2023-08-28T01:56:40.993888+0000 mon.a (mon.0) 4643 : audit [DBG] from='client.? 172.21.15.6:0/3466170963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:42.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:42 smithi130 bash[51302]: debug 2023-08-28T01:56:42.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:42.107663+0000) 2023-08-28T01:56:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:56:42.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:56:42.730Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:43.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:43 smithi130 bash[51302]: debug 2023-08-28T01:56:43.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:43.107944+0000) 2023-08-28T01:56:43.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:43 smithi130 bash[27579]: audit 2023-08-28T01:56:43.407227+0000 mon.c (mon.2) 2809 : audit [DBG] from='client.? 172.21.15.6:0/387244652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:43.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:43 smithi006 bash[27292]: audit 2023-08-28T01:56:43.407227+0000 mon.c (mon.2) 2809 : audit [DBG] from='client.? 172.21.15.6:0/387244652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:43.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:43 smithi006 bash[33918]: audit 2023-08-28T01:56:43.407227+0000 mon.c (mon.2) 2809 : audit [DBG] from='client.? 172.21.15.6:0/387244652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:44.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:44 smithi130 bash[51302]: debug 2023-08-28T01:56:44.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:44.108232+0000) 2023-08-28T01:56:44.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:44 smithi130 bash[27579]: cluster 2023-08-28T01:56:42.860830+0000 mgr.y (mgr.14901) 7143 : cluster [DBG] pgmap v7150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:44.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:44 smithi006 bash[33918]: cluster 2023-08-28T01:56:42.860830+0000 mgr.y (mgr.14901) 7143 : cluster [DBG] pgmap v7150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:44.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:44 smithi006 bash[27292]: cluster 2023-08-28T01:56:42.860830+0000 mgr.y (mgr.14901) 7143 : cluster [DBG] pgmap v7150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:56:44.611Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:56:44.613Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:45.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:45 smithi130 bash[51302]: debug 2023-08-28T01:56:45.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:45.108518+0000) 2023-08-28T01:56:46.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:46 smithi130 bash[51302]: debug 2023-08-28T01:56:46.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:46.108818+0000) 2023-08-28T01:56:46.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:46 smithi130 bash[27579]: cluster 2023-08-28T01:56:44.861891+0000 mgr.y (mgr.14901) 7144 : cluster [DBG] pgmap v7151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:46.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:46 smithi130 bash[27579]: audit 2023-08-28T01:56:45.823464+0000 mon.a (mon.0) 4644 : audit [DBG] from='client.? 172.21.15.6:0/4153064604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:46.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:46 smithi006 bash[27292]: cluster 2023-08-28T01:56:44.861891+0000 mgr.y (mgr.14901) 7144 : cluster [DBG] pgmap v7151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:46.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:46 smithi006 bash[27292]: audit 2023-08-28T01:56:45.823464+0000 mon.a (mon.0) 4644 : audit [DBG] from='client.? 172.21.15.6:0/4153064604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:46.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:46 smithi006 bash[33918]: cluster 2023-08-28T01:56:44.861891+0000 mgr.y (mgr.14901) 7144 : cluster [DBG] pgmap v7151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:46.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:46 smithi006 bash[33918]: audit 2023-08-28T01:56:45.823464+0000 mon.a (mon.0) 4644 : audit [DBG] from='client.? 172.21.15.6:0/4153064604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:47.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:47 smithi130 bash[51302]: debug 2023-08-28T01:56:47.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:47.109094+0000) 2023-08-28T01:56:48.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:48 smithi130 bash[51302]: debug 2023-08-28T01:56:48.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:48.109378+0000) 2023-08-28T01:56:48.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:48 smithi130 bash[27579]: cluster 2023-08-28T01:56:46.862554+0000 mgr.y (mgr.14901) 7145 : cluster [DBG] pgmap v7152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:48.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:48 smithi130 bash[27579]: audit 2023-08-28T01:56:48.237869+0000 mon.c (mon.2) 2810 : audit [DBG] from='client.? 172.21.15.6:0/3055264807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:48.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:48 smithi006 bash[27292]: cluster 2023-08-28T01:56:46.862554+0000 mgr.y (mgr.14901) 7145 : cluster [DBG] pgmap v7152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:48.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:48 smithi006 bash[27292]: audit 2023-08-28T01:56:48.237869+0000 mon.c (mon.2) 2810 : audit [DBG] from='client.? 172.21.15.6:0/3055264807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:48.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:48 smithi006 bash[33918]: cluster 2023-08-28T01:56:46.862554+0000 mgr.y (mgr.14901) 7145 : cluster [DBG] pgmap v7152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:48.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:48 smithi006 bash[33918]: audit 2023-08-28T01:56:48.237869+0000 mon.c (mon.2) 2810 : audit [DBG] from='client.? 172.21.15.6:0/3055264807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:49.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:49 smithi130 bash[51302]: debug 2023-08-28T01:56:49.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:49.109684+0000) 2023-08-28T01:56:50.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:50 smithi130 bash[51302]: debug 2023-08-28T01:56:50.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:50.109968+0000) 2023-08-28T01:56:50.751 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:50 smithi006 bash[27292]: cluster 2023-08-28T01:56:48.863366+0000 mgr.y (mgr.14901) 7146 : cluster [DBG] pgmap v7153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:50.751 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:50 smithi006 bash[33918]: cluster 2023-08-28T01:56:48.863366+0000 mgr.y (mgr.14901) 7146 : cluster [DBG] pgmap v7153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:50.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:50 smithi130 bash[27579]: cluster 2023-08-28T01:56:48.863366+0000 mgr.y (mgr.14901) 7146 : cluster [DBG] pgmap v7153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:51.163 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:56:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:56:50] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T01:56:51.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:51 smithi130 bash[51302]: debug 2023-08-28T01:56:51.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:51.110250+0000) 2023-08-28T01:56:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:56:51.163Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:51.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:56:51.165Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:51.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:51 smithi130 bash[27579]: audit 2023-08-28T01:56:50.655146+0000 mon.a (mon.0) 4645 : audit [DBG] from='client.? 172.21.15.6:0/784421643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:51.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:51 smithi006 bash[33918]: audit 2023-08-28T01:56:50.655146+0000 mon.a (mon.0) 4645 : audit [DBG] from='client.? 172.21.15.6:0/784421643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:51.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:51 smithi006 bash[27292]: audit 2023-08-28T01:56:50.655146+0000 mon.a (mon.0) 4645 : audit [DBG] from='client.? 172.21.15.6:0/784421643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:52.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:52 smithi130 bash[51302]: debug 2023-08-28T01:56:52.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:52.110507+0000) 2023-08-28T01:56:52.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:52 smithi130 bash[27579]: cluster 2023-08-28T01:56:50.864557+0000 mgr.y (mgr.14901) 7147 : cluster [DBG] pgmap v7154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:56:52.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:56:52.731Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:52.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:52 smithi006 bash[27292]: cluster 2023-08-28T01:56:50.864557+0000 mgr.y (mgr.14901) 7147 : cluster [DBG] pgmap v7154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:52.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:52 smithi006 bash[33918]: cluster 2023-08-28T01:56:50.864557+0000 mgr.y (mgr.14901) 7147 : cluster [DBG] pgmap v7154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:53.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:53 smithi130 bash[51302]: debug 2023-08-28T01:56:53.105+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:53.110781+0000) 2023-08-28T01:56:53.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:53 smithi130 bash[27579]: audit 2023-08-28T01:56:53.067979+0000 mon.a (mon.0) 4646 : audit [DBG] from='client.? 172.21.15.6:0/952860973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:53.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:53 smithi006 bash[27292]: audit 2023-08-28T01:56:53.067979+0000 mon.a (mon.0) 4646 : audit [DBG] from='client.? 172.21.15.6:0/952860973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:53.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:53 smithi006 bash[33918]: audit 2023-08-28T01:56:53.067979+0000 mon.a (mon.0) 4646 : audit [DBG] from='client.? 172.21.15.6:0/952860973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:54.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:54 smithi130 bash[51302]: debug 2023-08-28T01:56:54.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:54.111093+0000) 2023-08-28T01:56:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:54 smithi130 bash[27579]: cluster 2023-08-28T01:56:52.865334+0000 mgr.y (mgr.14901) 7148 : cluster [DBG] pgmap v7155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:54.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:54 smithi006 bash[27292]: cluster 2023-08-28T01:56:52.865334+0000 mgr.y (mgr.14901) 7148 : cluster [DBG] pgmap v7155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:54.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:54 smithi006 bash[33918]: cluster 2023-08-28T01:56:52.865334+0000 mgr.y (mgr.14901) 7148 : cluster [DBG] pgmap v7155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:56:54.612Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:56:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:56:54.614Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:56:55.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:55 smithi130 bash[51302]: debug 2023-08-28T01:56:55.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:55.111329+0000) 2023-08-28T01:56:55.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:55 smithi130 bash[27579]: cluster 2023-08-28T01:56:54.866604+0000 mgr.y (mgr.14901) 7149 : cluster [DBG] pgmap v7156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:55.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:55 smithi130 bash[27579]: audit 2023-08-28T01:56:55.477380+0000 mon.a (mon.0) 4647 : audit [DBG] from='client.? 172.21.15.6:0/4107830863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:55.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:55 smithi006 bash[27292]: cluster 2023-08-28T01:56:54.866604+0000 mgr.y (mgr.14901) 7149 : cluster [DBG] pgmap v7156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:55.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:55 smithi006 bash[27292]: audit 2023-08-28T01:56:55.477380+0000 mon.a (mon.0) 4647 : audit [DBG] from='client.? 172.21.15.6:0/4107830863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:55.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:55 smithi006 bash[33918]: cluster 2023-08-28T01:56:54.866604+0000 mgr.y (mgr.14901) 7149 : cluster [DBG] pgmap v7156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:55.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:55 smithi006 bash[33918]: audit 2023-08-28T01:56:55.477380+0000 mon.a (mon.0) 4647 : audit [DBG] from='client.? 172.21.15.6:0/4107830863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:56.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:56 smithi130 bash[51302]: debug 2023-08-28T01:56:56.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:56.111625+0000) 2023-08-28T01:56:57.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:57 smithi130 bash[51302]: debug 2023-08-28T01:56:57.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:57.111905+0000) 2023-08-28T01:56:58.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:58 smithi130 bash[51302]: debug 2023-08-28T01:56:58.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:58.112169+0000) 2023-08-28T01:56:58.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:58 smithi130 bash[27579]: cluster 2023-08-28T01:56:56.867444+0000 mgr.y (mgr.14901) 7150 : cluster [DBG] pgmap v7157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:58.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:56:58 smithi130 bash[27579]: audit 2023-08-28T01:56:57.891938+0000 mon.c (mon.2) 2811 : audit [DBG] from='client.? 172.21.15.6:0/1717915466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:58.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:58 smithi006 bash[27292]: cluster 2023-08-28T01:56:56.867444+0000 mgr.y (mgr.14901) 7150 : cluster [DBG] pgmap v7157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:58.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:56:58 smithi006 bash[27292]: audit 2023-08-28T01:56:57.891938+0000 mon.c (mon.2) 2811 : audit [DBG] from='client.? 172.21.15.6:0/1717915466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:58.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:58 smithi006 bash[33918]: cluster 2023-08-28T01:56:56.867444+0000 mgr.y (mgr.14901) 7150 : cluster [DBG] pgmap v7157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:56:58.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:56:58 smithi006 bash[33918]: audit 2023-08-28T01:56:57.891938+0000 mon.c (mon.2) 2811 : audit [DBG] from='client.? 172.21.15.6:0/1717915466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:59.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:56:59 smithi130 bash[51302]: debug 2023-08-28T01:56:59.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:59.112450+0000) 2023-08-28T01:57:00.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:00 smithi130 bash[51302]: debug 2023-08-28T01:57:00.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:00.112723+0000) 2023-08-28T01:57:00.751 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:00 smithi006 bash[33918]: cluster 2023-08-28T01:56:58.868132+0000 mgr.y (mgr.14901) 7151 : cluster [DBG] pgmap v7158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:00.752 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:00 smithi006 bash[33918]: audit 2023-08-28T01:57:00.348897+0000 mon.a (mon.0) 4648 : audit [DBG] from='client.? 172.21.15.6:0/2102242397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:00.752 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:00 smithi006 bash[27292]: cluster 2023-08-28T01:56:58.868132+0000 mgr.y (mgr.14901) 7151 : cluster [DBG] pgmap v7158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:00.752 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:00 smithi006 bash[27292]: audit 2023-08-28T01:57:00.348897+0000 mon.a (mon.0) 4648 : audit [DBG] from='client.? 172.21.15.6:0/2102242397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:00.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:00 smithi130 bash[27579]: cluster 2023-08-28T01:56:58.868132+0000 mgr.y (mgr.14901) 7151 : cluster [DBG] pgmap v7158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:00 smithi130 bash[27579]: audit 2023-08-28T01:57:00.348897+0000 mon.a (mon.0) 4648 : audit [DBG] from='client.? 172.21.15.6:0/2102242397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:01.164 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:57:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:57:00] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:57:01.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:01 smithi130 bash[51302]: debug 2023-08-28T01:57:01.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:01.113046+0000) 2023-08-28T01:57:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:57:01.164Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:57:01.166Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:02.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:02 smithi130 bash[51302]: debug 2023-08-28T01:57:02.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:02.113338+0000) 2023-08-28T01:57:02.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:02 smithi130 bash[27579]: cluster 2023-08-28T01:57:00.869187+0000 mgr.y (mgr.14901) 7152 : cluster [DBG] pgmap v7159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:02.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:02 smithi006 bash[33918]: cluster 2023-08-28T01:57:00.869187+0000 mgr.y (mgr.14901) 7152 : cluster [DBG] pgmap v7159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:02.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:02 smithi006 bash[27292]: cluster 2023-08-28T01:57:00.869187+0000 mgr.y (mgr.14901) 7152 : cluster [DBG] pgmap v7159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:57:02.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:57:02.732Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:03.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:03 smithi130 bash[51302]: debug 2023-08-28T01:57:03.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:03.113616+0000) 2023-08-28T01:57:03.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:03 smithi130 bash[27579]: audit 2023-08-28T01:57:02.765090+0000 mon.a (mon.0) 4649 : audit [DBG] from='client.? 172.21.15.6:0/548274793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:03.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:03 smithi006 bash[33918]: audit 2023-08-28T01:57:02.765090+0000 mon.a (mon.0) 4649 : audit [DBG] from='client.? 172.21.15.6:0/548274793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:03.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:03 smithi006 bash[27292]: audit 2023-08-28T01:57:02.765090+0000 mon.a (mon.0) 4649 : audit [DBG] from='client.? 172.21.15.6:0/548274793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:04.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:04 smithi130 bash[51302]: debug 2023-08-28T01:57:04.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:04.113969+0000) 2023-08-28T01:57:04.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:04 smithi130 bash[27579]: cluster 2023-08-28T01:57:02.869896+0000 mgr.y (mgr.14901) 7153 : cluster [DBG] pgmap v7160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:04.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:04 smithi006 bash[27292]: cluster 2023-08-28T01:57:02.869896+0000 mgr.y (mgr.14901) 7153 : cluster [DBG] pgmap v7160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:04.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:04 smithi006 bash[33918]: cluster 2023-08-28T01:57:02.869896+0000 mgr.y (mgr.14901) 7153 : cluster [DBG] pgmap v7160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:57:04.612Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:57:04.614Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:05.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:05 smithi130 bash[51302]: debug 2023-08-28T01:57:05.109+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:05.114254+0000) 2023-08-28T01:57:05.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:05 smithi130 bash[27579]: audit 2023-08-28T01:57:05.175983+0000 mon.a (mon.0) 4650 : audit [DBG] from='client.? 172.21.15.6:0/529333303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:05.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:05 smithi006 bash[27292]: audit 2023-08-28T01:57:05.175983+0000 mon.a (mon.0) 4650 : audit [DBG] from='client.? 172.21.15.6:0/529333303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:05.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:05 smithi006 bash[33918]: audit 2023-08-28T01:57:05.175983+0000 mon.a (mon.0) 4650 : audit [DBG] from='client.? 172.21.15.6:0/529333303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:06.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:06 smithi130 bash[51302]: debug 2023-08-28T01:57:06.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:06.114560+0000) 2023-08-28T01:57:06.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:06 smithi130 bash[27579]: cluster 2023-08-28T01:57:04.871027+0000 mgr.y (mgr.14901) 7154 : cluster [DBG] pgmap v7161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:06.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:06 smithi006 bash[27292]: cluster 2023-08-28T01:57:04.871027+0000 mgr.y (mgr.14901) 7154 : cluster [DBG] pgmap v7161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:06.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:06 smithi006 bash[33918]: cluster 2023-08-28T01:57:04.871027+0000 mgr.y (mgr.14901) 7154 : cluster [DBG] pgmap v7161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:07.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:07 smithi130 bash[51302]: debug 2023-08-28T01:57:07.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:07.114877+0000) 2023-08-28T01:57:08.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:08 smithi130 bash[51302]: debug 2023-08-28T01:57:08.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:08.115196+0000) 2023-08-28T01:57:08.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:08 smithi130 bash[27579]: cluster 2023-08-28T01:57:06.871688+0000 mgr.y (mgr.14901) 7155 : cluster [DBG] pgmap v7162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:08 smithi130 bash[27579]: audit 2023-08-28T01:57:07.595200+0000 mon.c (mon.2) 2812 : audit [DBG] from='client.? 172.21.15.6:0/1869424861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:08.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:08 smithi006 bash[33918]: cluster 2023-08-28T01:57:06.871688+0000 mgr.y (mgr.14901) 7155 : cluster [DBG] pgmap v7162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:08.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:08 smithi006 bash[33918]: audit 2023-08-28T01:57:07.595200+0000 mon.c (mon.2) 2812 : audit [DBG] from='client.? 172.21.15.6:0/1869424861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:08.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:08 smithi006 bash[27292]: cluster 2023-08-28T01:57:06.871688+0000 mgr.y (mgr.14901) 7155 : cluster [DBG] pgmap v7162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:08.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:08 smithi006 bash[27292]: audit 2023-08-28T01:57:07.595200+0000 mon.c (mon.2) 2812 : audit [DBG] from='client.? 172.21.15.6:0/1869424861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:09.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:09 smithi130 bash[51302]: debug 2023-08-28T01:57:09.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:09.115504+0000) 2023-08-28T01:57:09.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:09 smithi130 bash[27579]: cluster 2023-08-28T01:57:08.872488+0000 mgr.y (mgr.14901) 7156 : cluster [DBG] pgmap v7163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:09.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:09 smithi006 bash[27292]: cluster 2023-08-28T01:57:08.872488+0000 mgr.y (mgr.14901) 7156 : cluster [DBG] pgmap v7163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:09.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:09 smithi006 bash[33918]: cluster 2023-08-28T01:57:08.872488+0000 mgr.y (mgr.14901) 7156 : cluster [DBG] pgmap v7163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:10.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:10 smithi130 bash[51302]: debug 2023-08-28T01:57:10.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:10.115808+0000) 2023-08-28T01:57:10.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:10 smithi130 bash[27579]: audit 2023-08-28T01:57:10.009280+0000 mon.a (mon.0) 4651 : audit [DBG] from='client.? 172.21.15.6:0/1781292218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:10.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:57:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:57:10] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:57:10.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:10 smithi006 bash[33918]: audit 2023-08-28T01:57:10.009280+0000 mon.a (mon.0) 4651 : audit [DBG] from='client.? 172.21.15.6:0/1781292218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:10.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:10 smithi006 bash[27292]: audit 2023-08-28T01:57:10.009280+0000 mon.a (mon.0) 4651 : audit [DBG] from='client.? 172.21.15.6:0/1781292218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:11.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:11 smithi130 bash[51302]: debug 2023-08-28T01:57:11.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:11.116080+0000) 2023-08-28T01:57:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:57:11.166Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:57:11.168Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:11.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:11 smithi130 bash[27579]: cluster 2023-08-28T01:57:10.873706+0000 mgr.y (mgr.14901) 7157 : cluster [DBG] pgmap v7164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:11.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:11 smithi006 bash[33918]: cluster 2023-08-28T01:57:10.873706+0000 mgr.y (mgr.14901) 7157 : cluster [DBG] pgmap v7164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:11.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:11 smithi006 bash[27292]: cluster 2023-08-28T01:57:10.873706+0000 mgr.y (mgr.14901) 7157 : cluster [DBG] pgmap v7164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:12.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:12 smithi130 bash[51302]: debug 2023-08-28T01:57:12.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:12.116294+0000) 2023-08-28T01:57:12.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:12 smithi130 bash[27579]: audit 2023-08-28T01:57:12.413987+0000 mon.c (mon.2) 2813 : audit [DBG] from='client.? 172.21.15.6:0/3541362494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:12.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:12 smithi006 bash[27292]: audit 2023-08-28T01:57:12.413987+0000 mon.c (mon.2) 2813 : audit [DBG] from='client.? 172.21.15.6:0/3541362494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:12.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:12 smithi006 bash[33918]: audit 2023-08-28T01:57:12.413987+0000 mon.c (mon.2) 2813 : audit [DBG] from='client.? 172.21.15.6:0/3541362494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:57:12.732Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:57:12.733Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:13.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:13 smithi130 bash[51302]: debug 2023-08-28T01:57:13.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:13.116548+0000) 2023-08-28T01:57:13.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:13 smithi130 bash[27579]: cluster 2023-08-28T01:57:12.874352+0000 mgr.y (mgr.14901) 7158 : cluster [DBG] pgmap v7165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:13.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:13 smithi006 bash[27292]: cluster 2023-08-28T01:57:12.874352+0000 mgr.y (mgr.14901) 7158 : cluster [DBG] pgmap v7165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:13.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:13 smithi006 bash[33918]: cluster 2023-08-28T01:57:12.874352+0000 mgr.y (mgr.14901) 7158 : cluster [DBG] pgmap v7165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:14.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:14 smithi130 bash[51302]: debug 2023-08-28T01:57:14.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:14.116816+0000) 2023-08-28T01:57:14.882 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:57:14.613Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:14.882 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:57:14.615Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:15.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:14 smithi006 bash[33918]: audit 2023-08-28T01:57:14.829942+0000 mon.a (mon.0) 4652 : audit [DBG] from='client.? 172.21.15.6:0/3622605664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:15.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:14 smithi006 bash[27292]: audit 2023-08-28T01:57:14.829942+0000 mon.a (mon.0) 4652 : audit [DBG] from='client.? 172.21.15.6:0/3622605664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:15.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:15 smithi130 bash[51302]: debug 2023-08-28T01:57:15.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:15.117103+0000) 2023-08-28T01:57:15.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:14 smithi130 bash[27579]: audit 2023-08-28T01:57:14.829942+0000 mon.a (mon.0) 4652 : audit [DBG] from='client.? 172.21.15.6:0/3622605664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:16.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:15 smithi006 bash[27292]: cluster 2023-08-28T01:57:14.875487+0000 mgr.y (mgr.14901) 7159 : cluster [DBG] pgmap v7166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:16.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:15 smithi006 bash[33918]: cluster 2023-08-28T01:57:14.875487+0000 mgr.y (mgr.14901) 7159 : cluster [DBG] pgmap v7166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:16.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:15 smithi130 bash[27579]: cluster 2023-08-28T01:57:14.875487+0000 mgr.y (mgr.14901) 7159 : cluster [DBG] pgmap v7166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:16.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:16 smithi130 bash[51302]: debug 2023-08-28T01:57:16.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:16.117373+0000) 2023-08-28T01:57:17.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:17 smithi130 bash[51302]: debug 2023-08-28T01:57:17.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:17.117648+0000) 2023-08-28T01:57:18.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:17 smithi006 bash[27292]: cluster 2023-08-28T01:57:16.876145+0000 mgr.y (mgr.14901) 7160 : cluster [DBG] pgmap v7167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:18.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:17 smithi006 bash[27292]: audit 2023-08-28T01:57:17.242723+0000 mon.c (mon.2) 2814 : audit [DBG] from='client.? 172.21.15.6:0/2192235331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:18.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:17 smithi006 bash[33918]: cluster 2023-08-28T01:57:16.876145+0000 mgr.y (mgr.14901) 7160 : cluster [DBG] pgmap v7167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:18.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:17 smithi006 bash[33918]: audit 2023-08-28T01:57:17.242723+0000 mon.c (mon.2) 2814 : audit [DBG] from='client.? 172.21.15.6:0/2192235331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:18.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:17 smithi130 bash[27579]: cluster 2023-08-28T01:57:16.876145+0000 mgr.y (mgr.14901) 7160 : cluster [DBG] pgmap v7167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:18.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:17 smithi130 bash[27579]: audit 2023-08-28T01:57:17.242723+0000 mon.c (mon.2) 2814 : audit [DBG] from='client.? 172.21.15.6:0/2192235331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:18.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:18 smithi130 bash[51302]: debug 2023-08-28T01:57:18.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:18.117959+0000) 2023-08-28T01:57:19.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:19 smithi130 bash[51302]: debug 2023-08-28T01:57:19.113+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:19.118272+0000) 2023-08-28T01:57:20.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:19 smithi006 bash[27292]: cluster 2023-08-28T01:57:18.876829+0000 mgr.y (mgr.14901) 7161 : cluster [DBG] pgmap v7168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:20.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:19 smithi006 bash[27292]: audit 2023-08-28T01:57:19.657742+0000 mon.a (mon.0) 4653 : audit [DBG] from='client.? 172.21.15.6:0/1163708579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:20.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:19 smithi006 bash[33918]: cluster 2023-08-28T01:57:18.876829+0000 mgr.y (mgr.14901) 7161 : cluster [DBG] pgmap v7168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:20.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:19 smithi006 bash[33918]: audit 2023-08-28T01:57:19.657742+0000 mon.a (mon.0) 4653 : audit [DBG] from='client.? 172.21.15.6:0/1163708579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:20.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:19 smithi130 bash[27579]: cluster 2023-08-28T01:57:18.876829+0000 mgr.y (mgr.14901) 7161 : cluster [DBG] pgmap v7168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:20.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:19 smithi130 bash[27579]: audit 2023-08-28T01:57:19.657742+0000 mon.a (mon.0) 4653 : audit [DBG] from='client.? 172.21.15.6:0/1163708579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:20.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:20 smithi130 bash[51302]: debug 2023-08-28T01:57:20.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:20.118547+0000) 2023-08-28T01:57:21.166 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:57:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:57:20] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T01:57:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:57:21.166Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:57:21.168Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:21.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:21 smithi130 bash[51302]: debug 2023-08-28T01:57:21.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:21.118876+0000) 2023-08-28T01:57:22.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:21 smithi006 bash[27292]: cluster 2023-08-28T01:57:20.877932+0000 mgr.y (mgr.14901) 7162 : cluster [DBG] pgmap v7169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:22.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:21 smithi006 bash[33918]: cluster 2023-08-28T01:57:20.877932+0000 mgr.y (mgr.14901) 7162 : cluster [DBG] pgmap v7169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:22.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:21 smithi130 bash[27579]: cluster 2023-08-28T01:57:20.877932+0000 mgr.y (mgr.14901) 7162 : cluster [DBG] pgmap v7169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:22.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:22 smithi130 bash[51302]: debug 2023-08-28T01:57:22.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:22.119098+0000) 2023-08-28T01:57:23.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:22 smithi006 bash[33918]: audit 2023-08-28T01:57:22.076904+0000 mon.c (mon.2) 2815 : audit [DBG] from='client.? 172.21.15.6:0/585235304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:23.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:22 smithi006 bash[27292]: audit 2023-08-28T01:57:22.076904+0000 mon.c (mon.2) 2815 : audit [DBG] from='client.? 172.21.15.6:0/585235304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:57:22.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:57:22.735Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:23.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:22 smithi130 bash[27579]: audit 2023-08-28T01:57:22.076904+0000 mon.c (mon.2) 2815 : audit [DBG] from='client.? 172.21.15.6:0/585235304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:23.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:23 smithi130 bash[51302]: debug 2023-08-28T01:57:23.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:23.119418+0000) 2023-08-28T01:57:24.477 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:24 smithi130 bash[51302]: debug 2023-08-28T01:57:24.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:24.119727+0000) 2023-08-28T01:57:24.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:24 smithi130 bash[27579]: cluster 2023-08-28T01:57:22.878590+0000 mgr.y (mgr.14901) 7163 : cluster [DBG] pgmap v7170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:24.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:24 smithi006 bash[27292]: cluster 2023-08-28T01:57:22.878590+0000 mgr.y (mgr.14901) 7163 : cluster [DBG] pgmap v7170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:24.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:24 smithi006 bash[33918]: cluster 2023-08-28T01:57:22.878590+0000 mgr.y (mgr.14901) 7163 : cluster [DBG] pgmap v7170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:57:24.614Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:57:24.616Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:25.479 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:25 smithi130 bash[51302]: debug 2023-08-28T01:57:25.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:25.120056+0000) 2023-08-28T01:57:25.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:25 smithi130 bash[27579]: audit 2023-08-28T01:57:24.486921+0000 mon.a (mon.0) 4654 : audit [DBG] from='client.? 172.21.15.6:0/2416229838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:25.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:25 smithi006 bash[27292]: audit 2023-08-28T01:57:24.486921+0000 mon.a (mon.0) 4654 : audit [DBG] from='client.? 172.21.15.6:0/2416229838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:25.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:25 smithi006 bash[33918]: audit 2023-08-28T01:57:24.486921+0000 mon.a (mon.0) 4654 : audit [DBG] from='client.? 172.21.15.6:0/2416229838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:26.489 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:26 smithi130 bash[51302]: debug 2023-08-28T01:57:26.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:26.120319+0000) 2023-08-28T01:57:26.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:26 smithi130 bash[27579]: cluster 2023-08-28T01:57:24.879684+0000 mgr.y (mgr.14901) 7164 : cluster [DBG] pgmap v7171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:26.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:26 smithi006 bash[33918]: cluster 2023-08-28T01:57:24.879684+0000 mgr.y (mgr.14901) 7164 : cluster [DBG] pgmap v7171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:26.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:26 smithi006 bash[27292]: cluster 2023-08-28T01:57:24.879684+0000 mgr.y (mgr.14901) 7164 : cluster [DBG] pgmap v7171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:27.499 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:27 smithi130 bash[51302]: debug 2023-08-28T01:57:27.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:27.120637+0000) 2023-08-28T01:57:27.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:27 smithi130 bash[27579]: audit 2023-08-28T01:57:26.904693+0000 mon.c (mon.2) 2816 : audit [DBG] from='client.? 172.21.15.6:0/1161835001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:27.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:27 smithi006 bash[27292]: audit 2023-08-28T01:57:26.904693+0000 mon.c (mon.2) 2816 : audit [DBG] from='client.? 172.21.15.6:0/1161835001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:27.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:27 smithi006 bash[33918]: audit 2023-08-28T01:57:26.904693+0000 mon.c (mon.2) 2816 : audit [DBG] from='client.? 172.21.15.6:0/1161835001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:28.502 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:28 smithi130 bash[51302]: debug 2023-08-28T01:57:28.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:28.120990+0000) 2023-08-28T01:57:28.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:28 smithi130 bash[27579]: cluster 2023-08-28T01:57:26.880309+0000 mgr.y (mgr.14901) 7165 : cluster [DBG] pgmap v7172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:28.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:28 smithi006 bash[27292]: cluster 2023-08-28T01:57:26.880309+0000 mgr.y (mgr.14901) 7165 : cluster [DBG] pgmap v7172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:28.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:28 smithi006 bash[33918]: cluster 2023-08-28T01:57:26.880309+0000 mgr.y (mgr.14901) 7165 : cluster [DBG] pgmap v7172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:29.510 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:29 smithi130 bash[51302]: debug 2023-08-28T01:57:29.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:29.121290+0000) 2023-08-28T01:57:29.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:29 smithi130 bash[27579]: audit 2023-08-28T01:57:29.326803+0000 mon.a (mon.0) 4655 : audit [DBG] from='client.? 172.21.15.6:0/1420337633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:29.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:29 smithi006 bash[33918]: audit 2023-08-28T01:57:29.326803+0000 mon.a (mon.0) 4655 : audit [DBG] from='client.? 172.21.15.6:0/1420337633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:29.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:29 smithi006 bash[27292]: audit 2023-08-28T01:57:29.326803+0000 mon.a (mon.0) 4655 : audit [DBG] from='client.? 172.21.15.6:0/1420337633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:30.517 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:30 smithi130 bash[51302]: debug 2023-08-28T01:57:30.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:30.121616+0000) 2023-08-28T01:57:30.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:30 smithi130 bash[27579]: cluster 2023-08-28T01:57:28.880978+0000 mgr.y (mgr.14901) 7166 : cluster [DBG] pgmap v7173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:30.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:57:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:57:30] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:57:30.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:30 smithi006 bash[33918]: cluster 2023-08-28T01:57:28.880978+0000 mgr.y (mgr.14901) 7166 : cluster [DBG] pgmap v7173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:30.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:30 smithi006 bash[27292]: cluster 2023-08-28T01:57:28.880978+0000 mgr.y (mgr.14901) 7166 : cluster [DBG] pgmap v7173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:57:31.167Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:57:31.170Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:31.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:31 smithi130 bash[51302]: debug 2023-08-28T01:57:31.121+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:31.121963+0000) 2023-08-28T01:57:32.522 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:32 smithi130 bash[51302]: debug 2023-08-28T01:57:32.117+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:32.122223+0000) 2023-08-28T01:57:32.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:32 smithi130 bash[27579]: cluster 2023-08-28T01:57:30.882012+0000 mgr.y (mgr.14901) 7167 : cluster [DBG] pgmap v7174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:32 smithi130 bash[27579]: audit 2023-08-28T01:57:31.744937+0000 mon.c (mon.2) 2817 : audit [DBG] from='client.? 172.21.15.6:0/3621447255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:32.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:32 smithi006 bash[27292]: cluster 2023-08-28T01:57:30.882012+0000 mgr.y (mgr.14901) 7167 : cluster [DBG] pgmap v7174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:32.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:32 smithi006 bash[27292]: audit 2023-08-28T01:57:31.744937+0000 mon.c (mon.2) 2817 : audit [DBG] from='client.? 172.21.15.6:0/3621447255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:32 smithi006 bash[33918]: cluster 2023-08-28T01:57:30.882012+0000 mgr.y (mgr.14901) 7167 : cluster [DBG] pgmap v7174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:32.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:32 smithi006 bash[33918]: audit 2023-08-28T01:57:31.744937+0000 mon.c (mon.2) 2817 : audit [DBG] from='client.? 172.21.15.6:0/3621447255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:32.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:57:32.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:32.967 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:57:32.736Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:33.529 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:33 smithi130 bash[51302]: debug 2023-08-28T01:57:33.121+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:33.122535+0000) 2023-08-28T01:57:33.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:33 smithi130 bash[27579]: cluster 2023-08-28T01:57:32.882646+0000 mgr.y (mgr.14901) 7168 : cluster [DBG] pgmap v7175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:33.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:33 smithi006 bash[33918]: cluster 2023-08-28T01:57:32.882646+0000 mgr.y (mgr.14901) 7168 : cluster [DBG] pgmap v7175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:33 smithi006 bash[27292]: cluster 2023-08-28T01:57:32.882646+0000 mgr.y (mgr.14901) 7168 : cluster [DBG] pgmap v7175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:34.533 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:34 smithi130 bash[51302]: debug 2023-08-28T01:57:34.121+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:34.122885+0000) 2023-08-28T01:57:34.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:34 smithi130 bash[27579]: audit 2023-08-28T01:57:34.159307+0000 mon.a (mon.0) 4656 : audit [DBG] from='client.? 172.21.15.6:0/664237461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:34.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:34 smithi006 bash[33918]: audit 2023-08-28T01:57:34.159307+0000 mon.a (mon.0) 4656 : audit [DBG] from='client.? 172.21.15.6:0/664237461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:34.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:34 smithi006 bash[27292]: audit 2023-08-28T01:57:34.159307+0000 mon.a (mon.0) 4656 : audit [DBG] from='client.? 172.21.15.6:0/664237461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:57:34.614Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:57:34.616Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:35.543 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:35 smithi130 bash[51302]: debug 2023-08-28T01:57:35.121+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:35.123154+0000) 2023-08-28T01:57:35.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:35 smithi130 bash[27579]: cluster 2023-08-28T01:57:34.883714+0000 mgr.y (mgr.14901) 7169 : cluster [DBG] pgmap v7176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:35.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:35 smithi006 bash[27292]: cluster 2023-08-28T01:57:34.883714+0000 mgr.y (mgr.14901) 7169 : cluster [DBG] pgmap v7176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:35.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:35 smithi006 bash[33918]: cluster 2023-08-28T01:57:34.883714+0000 mgr.y (mgr.14901) 7169 : cluster [DBG] pgmap v7176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:36.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:36 smithi130 bash[51302]: debug 2023-08-28T01:57:36.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:36.123541+0000) 2023-08-28T01:57:36.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:36 smithi006 bash[33918]: audit 2023-08-28T01:57:36.573524+0000 mon.c (mon.2) 2818 : audit [DBG] from='client.? 172.21.15.6:0/651640595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:36.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:36 smithi006 bash[27292]: audit 2023-08-28T01:57:36.573524+0000 mon.c (mon.2) 2818 : audit [DBG] from='client.? 172.21.15.6:0/651640595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:37.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:36 smithi130 bash[27579]: audit 2023-08-28T01:57:36.573524+0000 mon.c (mon.2) 2818 : audit [DBG] from='client.? 172.21.15.6:0/651640595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:37.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:37 smithi130 bash[51302]: debug 2023-08-28T01:57:37.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:37.123867+0000) 2023-08-28T01:57:38.478 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:38 smithi130 bash[51302]: debug 2023-08-28T01:57:38.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:38.124199+0000) 2023-08-28T01:57:38.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:38 smithi130 bash[27579]: cluster 2023-08-28T01:57:36.884341+0000 mgr.y (mgr.14901) 7170 : cluster [DBG] pgmap v7177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:38.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:38 smithi006 bash[27292]: cluster 2023-08-28T01:57:36.884341+0000 mgr.y (mgr.14901) 7170 : cluster [DBG] pgmap v7177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:38.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:38 smithi006 bash[33918]: cluster 2023-08-28T01:57:36.884341+0000 mgr.y (mgr.14901) 7170 : cluster [DBG] pgmap v7177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:39.486 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:39 smithi130 bash[51302]: debug 2023-08-28T01:57:39.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:39.124509+0000) 2023-08-28T01:57:39.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:39 smithi130 bash[27579]: audit 2023-08-28T01:57:38.993143+0000 mon.c (mon.2) 2819 : audit [DBG] from='client.? 172.21.15.6:0/1652544039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:39.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:39 smithi006 bash[33918]: audit 2023-08-28T01:57:38.993143+0000 mon.c (mon.2) 2819 : audit [DBG] from='client.? 172.21.15.6:0/1652544039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:39.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:39 smithi006 bash[27292]: audit 2023-08-28T01:57:38.993143+0000 mon.c (mon.2) 2819 : audit [DBG] from='client.? 172.21.15.6:0/1652544039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:40.490 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:40 smithi130 bash[51302]: debug 2023-08-28T01:57:40.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:40.124779+0000) 2023-08-28T01:57:40.751 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:40 smithi006 bash[33918]: cluster 2023-08-28T01:57:38.884960+0000 mgr.y (mgr.14901) 7171 : cluster [DBG] pgmap v7178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:40.752 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:40 smithi006 bash[33918]: audit 2023-08-28T01:57:40.017592+0000 mon.b (mon.1) 800 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:57:40.752 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:40 smithi006 bash[27292]: cluster 2023-08-28T01:57:38.884960+0000 mgr.y (mgr.14901) 7171 : cluster [DBG] pgmap v7178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:40.752 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:40 smithi006 bash[27292]: audit 2023-08-28T01:57:40.017592+0000 mon.b (mon.1) 800 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:57:40.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:40 smithi130 bash[27579]: cluster 2023-08-28T01:57:38.884960+0000 mgr.y (mgr.14901) 7171 : cluster [DBG] pgmap v7178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:40.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:40 smithi130 bash[27579]: audit 2023-08-28T01:57:40.017592+0000 mon.b (mon.1) 800 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:57:41.168 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:57:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:57:40] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:57:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:57:41.168Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:57:41.170Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:41.497 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:41 smithi130 bash[51302]: debug 2023-08-28T01:57:41.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:41.125057+0000) 2023-08-28T01:57:41.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:41 smithi130 bash[27579]: audit 2023-08-28T01:57:41.410604+0000 mon.c (mon.2) 2820 : audit [DBG] from='client.? 172.21.15.6:0/3911560624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:41.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:41 smithi006 bash[33918]: audit 2023-08-28T01:57:41.410604+0000 mon.c (mon.2) 2820 : audit [DBG] from='client.? 172.21.15.6:0/3911560624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:41.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:41 smithi006 bash[27292]: audit 2023-08-28T01:57:41.410604+0000 mon.c (mon.2) 2820 : audit [DBG] from='client.? 172.21.15.6:0/3911560624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:42.501 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:42 smithi130 bash[51302]: debug 2023-08-28T01:57:42.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:42.125311+0000) 2023-08-28T01:57:42.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:42 smithi130 bash[27579]: cluster 2023-08-28T01:57:40.885999+0000 mgr.y (mgr.14901) 7172 : cluster [DBG] pgmap v7179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:42.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:42 smithi006 bash[33918]: cluster 2023-08-28T01:57:40.885999+0000 mgr.y (mgr.14901) 7172 : cluster [DBG] pgmap v7179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:42.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:42 smithi006 bash[27292]: cluster 2023-08-28T01:57:40.885999+0000 mgr.y (mgr.14901) 7172 : cluster [DBG] pgmap v7179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:57:42.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:57:42.737Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:43.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:43 smithi130 bash[51302]: debug 2023-08-28T01:57:43.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:43.125580+0000) 2023-08-28T01:57:44.507 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:44 smithi130 bash[51302]: debug 2023-08-28T01:57:44.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:44.125847+0000) 2023-08-28T01:57:44.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:44 smithi130 bash[27579]: cluster 2023-08-28T01:57:42.886662+0000 mgr.y (mgr.14901) 7173 : cluster [DBG] pgmap v7180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:44.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:44 smithi130 bash[27579]: audit 2023-08-28T01:57:43.824558+0000 mon.a (mon.0) 4657 : audit [DBG] from='client.? 172.21.15.6:0/333200830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:57:44.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:57:44.618Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:44.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:44 smithi006 bash[27292]: cluster 2023-08-28T01:57:42.886662+0000 mgr.y (mgr.14901) 7173 : cluster [DBG] pgmap v7180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:44.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:44 smithi006 bash[27292]: audit 2023-08-28T01:57:43.824558+0000 mon.a (mon.0) 4657 : audit [DBG] from='client.? 172.21.15.6:0/333200830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:44.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:44 smithi006 bash[33918]: cluster 2023-08-28T01:57:42.886662+0000 mgr.y (mgr.14901) 7173 : cluster [DBG] pgmap v7180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:44.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:44 smithi006 bash[33918]: audit 2023-08-28T01:57:43.824558+0000 mon.a (mon.0) 4657 : audit [DBG] from='client.? 172.21.15.6:0/333200830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:45.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:45 smithi130 bash[51302]: debug 2023-08-28T01:57:45.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:45.126066+0000) 2023-08-28T01:57:46.513 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:46 smithi130 bash[51302]: debug 2023-08-28T01:57:46.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:46.126328+0000) 2023-08-28T01:57:46.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:46 smithi130 bash[27579]: cluster 2023-08-28T01:57:44.887665+0000 mgr.y (mgr.14901) 7174 : cluster [DBG] pgmap v7181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:46.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:46 smithi130 bash[27579]: audit 2023-08-28T01:57:45.891482+0000 mon.a (mon.0) 4658 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:57:46.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:46 smithi130 bash[27579]: audit 2023-08-28T01:57:46.264201+0000 mon.a (mon.0) 4659 : audit [DBG] from='client.? 172.21.15.6:0/4226913756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:46.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:46 smithi130 bash[27579]: audit 2023-08-28T01:57:46.367555+0000 mon.a (mon.0) 4660 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:57:46.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:46 smithi006 bash[27292]: cluster 2023-08-28T01:57:44.887665+0000 mgr.y (mgr.14901) 7174 : cluster [DBG] pgmap v7181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:46.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:46 smithi006 bash[27292]: audit 2023-08-28T01:57:45.891482+0000 mon.a (mon.0) 4658 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:57:46.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:46 smithi006 bash[27292]: audit 2023-08-28T01:57:46.264201+0000 mon.a (mon.0) 4659 : audit [DBG] from='client.? 172.21.15.6:0/4226913756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:46.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:46 smithi006 bash[27292]: audit 2023-08-28T01:57:46.367555+0000 mon.a (mon.0) 4660 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:57:46.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:46 smithi006 bash[33918]: cluster 2023-08-28T01:57:44.887665+0000 mgr.y (mgr.14901) 7174 : cluster [DBG] pgmap v7181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:46.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:46 smithi006 bash[33918]: audit 2023-08-28T01:57:45.891482+0000 mon.a (mon.0) 4658 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:57:46.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:46 smithi006 bash[33918]: audit 2023-08-28T01:57:46.264201+0000 mon.a (mon.0) 4659 : audit [DBG] from='client.? 172.21.15.6:0/4226913756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:46.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:46 smithi006 bash[33918]: audit 2023-08-28T01:57:46.367555+0000 mon.a (mon.0) 4660 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:57:47.517 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:47 smithi130 bash[51302]: debug 2023-08-28T01:57:47.120+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:47.126640+0000) 2023-08-28T01:57:47.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:47 smithi130 bash[27579]: audit 2023-08-28T01:57:46.637758+0000 mon.b (mon.1) 801 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:57:47.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:47 smithi130 bash[27579]: audit 2023-08-28T01:57:46.639884+0000 mon.b (mon.1) 802 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:57:47.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:47 smithi130 bash[27579]: audit 2023-08-28T01:57:46.651656+0000 mon.a (mon.0) 4661 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:57:47.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:47 smithi006 bash[33918]: audit 2023-08-28T01:57:46.637758+0000 mon.b (mon.1) 801 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:57:47.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:47 smithi006 bash[33918]: audit 2023-08-28T01:57:46.639884+0000 mon.b (mon.1) 802 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:57:47.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:47 smithi006 bash[33918]: audit 2023-08-28T01:57:46.651656+0000 mon.a (mon.0) 4661 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:57:47.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:47 smithi006 bash[27292]: audit 2023-08-28T01:57:46.637758+0000 mon.b (mon.1) 801 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:57:47.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:47 smithi006 bash[27292]: audit 2023-08-28T01:57:46.639884+0000 mon.b (mon.1) 802 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:57:47.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:47 smithi006 bash[27292]: audit 2023-08-28T01:57:46.651656+0000 mon.a (mon.0) 4661 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:57:48.519 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:48 smithi130 bash[51302]: debug 2023-08-28T01:57:48.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:48.126928+0000) 2023-08-28T01:57:48.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:48 smithi130 bash[27579]: cluster 2023-08-28T01:57:46.888317+0000 mgr.y (mgr.14901) 7175 : cluster [DBG] pgmap v7182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:48.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:48 smithi006 bash[27292]: cluster 2023-08-28T01:57:46.888317+0000 mgr.y (mgr.14901) 7175 : cluster [DBG] pgmap v7182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:48.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:48 smithi006 bash[33918]: cluster 2023-08-28T01:57:46.888317+0000 mgr.y (mgr.14901) 7175 : cluster [DBG] pgmap v7182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:49.531 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:49 smithi130 bash[51302]: debug 2023-08-28T01:57:49.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:49.127166+0000) 2023-08-28T01:57:49.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:49 smithi130 bash[27579]: audit 2023-08-28T01:57:48.681388+0000 mon.a (mon.0) 4662 : audit [DBG] from='client.? 172.21.15.6:0/4092986424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:49.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:49 smithi006 bash[27292]: audit 2023-08-28T01:57:48.681388+0000 mon.a (mon.0) 4662 : audit [DBG] from='client.? 172.21.15.6:0/4092986424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:49.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:49 smithi006 bash[33918]: audit 2023-08-28T01:57:48.681388+0000 mon.a (mon.0) 4662 : audit [DBG] from='client.? 172.21.15.6:0/4092986424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:50.533 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:50 smithi130 bash[51302]: debug 2023-08-28T01:57:50.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:50.127479+0000) 2023-08-28T01:57:50.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:50 smithi130 bash[27579]: cluster 2023-08-28T01:57:48.888909+0000 mgr.y (mgr.14901) 7176 : cluster [DBG] pgmap v7183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:50.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:57:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:57:50] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T01:57:50.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:50 smithi006 bash[33918]: cluster 2023-08-28T01:57:48.888909+0000 mgr.y (mgr.14901) 7176 : cluster [DBG] pgmap v7183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:50.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:50 smithi006 bash[27292]: cluster 2023-08-28T01:57:48.888909+0000 mgr.y (mgr.14901) 7176 : cluster [DBG] pgmap v7183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:57:51.169Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:57:51.171Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:51.543 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:51 smithi130 bash[51302]: debug 2023-08-28T01:57:51.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:51.127818+0000) 2023-08-28T01:57:51.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:51 smithi130 bash[27579]: cluster 2023-08-28T01:57:50.889868+0000 mgr.y (mgr.14901) 7177 : cluster [DBG] pgmap v7184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:51.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:51 smithi130 bash[27579]: audit 2023-08-28T01:57:51.095921+0000 mon.c (mon.2) 2821 : audit [DBG] from='client.? 172.21.15.6:0/3870770723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:51.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:51 smithi006 bash[27292]: cluster 2023-08-28T01:57:50.889868+0000 mgr.y (mgr.14901) 7177 : cluster [DBG] pgmap v7184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:51.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:51 smithi006 bash[27292]: audit 2023-08-28T01:57:51.095921+0000 mon.c (mon.2) 2821 : audit [DBG] from='client.? 172.21.15.6:0/3870770723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:51.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:51 smithi006 bash[33918]: cluster 2023-08-28T01:57:50.889868+0000 mgr.y (mgr.14901) 7177 : cluster [DBG] pgmap v7184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:51.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:51 smithi006 bash[33918]: audit 2023-08-28T01:57:51.095921+0000 mon.c (mon.2) 2821 : audit [DBG] from='client.? 172.21.15.6:0/3870770723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:52.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:52 smithi130 bash[51302]: debug 2023-08-28T01:57:52.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:52.128024+0000) 2023-08-28T01:57:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:57:52.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:57:52.738Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:53.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:53 smithi130 bash[51302]: debug 2023-08-28T01:57:53.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:53.128361+0000) 2023-08-28T01:57:54.484 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:54 smithi130 bash[51302]: debug 2023-08-28T01:57:54.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:54.128690+0000) 2023-08-28T01:57:54.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:54 smithi130 bash[27579]: cluster 2023-08-28T01:57:52.890628+0000 mgr.y (mgr.14901) 7178 : cluster [DBG] pgmap v7185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:54.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:54 smithi130 bash[27579]: audit 2023-08-28T01:57:53.510338+0000 mon.a (mon.0) 4663 : audit [DBG] from='client.? 172.21.15.6:0/510722772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:54.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:54 smithi006 bash[27292]: cluster 2023-08-28T01:57:52.890628+0000 mgr.y (mgr.14901) 7178 : cluster [DBG] pgmap v7185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:54.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:54 smithi006 bash[27292]: audit 2023-08-28T01:57:53.510338+0000 mon.a (mon.0) 4663 : audit [DBG] from='client.? 172.21.15.6:0/510722772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:57:54.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:57:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:57:54.619Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:57:54.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:54 smithi006 bash[33918]: cluster 2023-08-28T01:57:52.890628+0000 mgr.y (mgr.14901) 7178 : cluster [DBG] pgmap v7185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:54.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:54 smithi006 bash[33918]: audit 2023-08-28T01:57:53.510338+0000 mon.a (mon.0) 4663 : audit [DBG] from='client.? 172.21.15.6:0/510722772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:55.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:55 smithi130 bash[51302]: debug 2023-08-28T01:57:55.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:55.129013+0000) 2023-08-28T01:57:56.490 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:56 smithi130 bash[51302]: debug 2023-08-28T01:57:56.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:56.129351+0000) 2023-08-28T01:57:56.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:56 smithi130 bash[27579]: cluster 2023-08-28T01:57:54.891816+0000 mgr.y (mgr.14901) 7179 : cluster [DBG] pgmap v7186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:56 smithi130 bash[27579]: audit 2023-08-28T01:57:55.917736+0000 mon.a (mon.0) 4664 : audit [DBG] from='client.? 172.21.15.6:0/3910523270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:56.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:56 smithi006 bash[27292]: cluster 2023-08-28T01:57:54.891816+0000 mgr.y (mgr.14901) 7179 : cluster [DBG] pgmap v7186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:56.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:56 smithi006 bash[27292]: audit 2023-08-28T01:57:55.917736+0000 mon.a (mon.0) 4664 : audit [DBG] from='client.? 172.21.15.6:0/3910523270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:56.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:56 smithi006 bash[33918]: cluster 2023-08-28T01:57:54.891816+0000 mgr.y (mgr.14901) 7179 : cluster [DBG] pgmap v7186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:56.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:56 smithi006 bash[33918]: audit 2023-08-28T01:57:55.917736+0000 mon.a (mon.0) 4664 : audit [DBG] from='client.? 172.21.15.6:0/3910523270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:57.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:57 smithi130 bash[51302]: debug 2023-08-28T01:57:57.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:57.129734+0000) 2023-08-28T01:57:58.494 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:58 smithi130 bash[51302]: debug 2023-08-28T01:57:58.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:58.130051+0000) 2023-08-28T01:57:58.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:58 smithi130 bash[27579]: cluster 2023-08-28T01:57:56.892502+0000 mgr.y (mgr.14901) 7180 : cluster [DBG] pgmap v7187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:58.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:57:58 smithi130 bash[27579]: audit 2023-08-28T01:57:58.336257+0000 mon.c (mon.2) 2822 : audit [DBG] from='client.? 172.21.15.6:0/925005267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:58.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:58 smithi006 bash[27292]: cluster 2023-08-28T01:57:56.892502+0000 mgr.y (mgr.14901) 7180 : cluster [DBG] pgmap v7187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:58.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:57:58 smithi006 bash[27292]: audit 2023-08-28T01:57:58.336257+0000 mon.c (mon.2) 2822 : audit [DBG] from='client.? 172.21.15.6:0/925005267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:58.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:58 smithi006 bash[33918]: cluster 2023-08-28T01:57:56.892502+0000 mgr.y (mgr.14901) 7180 : cluster [DBG] pgmap v7187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:57:58.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:57:58 smithi006 bash[33918]: audit 2023-08-28T01:57:58.336257+0000 mon.c (mon.2) 2822 : audit [DBG] from='client.? 172.21.15.6:0/925005267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:59.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:57:59 smithi130 bash[51302]: debug 2023-08-28T01:57:59.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:59.130385+0000) 2023-08-28T01:58:00.502 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:00 smithi130 bash[51302]: debug 2023-08-28T01:58:00.124+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:00.130752+0000) 2023-08-28T01:58:00.751 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:00 smithi006 bash[33918]: cluster 2023-08-28T01:57:58.893179+0000 mgr.y (mgr.14901) 7181 : cluster [DBG] pgmap v7188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:00.751 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:00 smithi006 bash[27292]: cluster 2023-08-28T01:57:58.893179+0000 mgr.y (mgr.14901) 7181 : cluster [DBG] pgmap v7188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:00.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:00 smithi130 bash[27579]: cluster 2023-08-28T01:57:58.893179+0000 mgr.y (mgr.14901) 7181 : cluster [DBG] pgmap v7188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:01.169 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:58:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:58:00] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:58:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:58:01.169Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:58:01.171Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:01.510 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:01 smithi130 bash[51302]: debug 2023-08-28T01:58:01.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:01.131091+0000) 2023-08-28T01:58:01.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:01 smithi130 bash[27579]: audit 2023-08-28T01:58:00.767232+0000 mon.a (mon.0) 4665 : audit [DBG] from='client.? 172.21.15.6:0/3082685718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:01.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:01 smithi006 bash[27292]: audit 2023-08-28T01:58:00.767232+0000 mon.a (mon.0) 4665 : audit [DBG] from='client.? 172.21.15.6:0/3082685718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:01.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:01 smithi006 bash[33918]: audit 2023-08-28T01:58:00.767232+0000 mon.a (mon.0) 4665 : audit [DBG] from='client.? 172.21.15.6:0/3082685718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:02.512 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:02 smithi130 bash[51302]: debug 2023-08-28T01:58:02.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:02.131376+0000) 2023-08-28T01:58:02.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:02 smithi130 bash[27579]: cluster 2023-08-28T01:58:00.894263+0000 mgr.y (mgr.14901) 7182 : cluster [DBG] pgmap v7189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:02.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:02 smithi006 bash[33918]: cluster 2023-08-28T01:58:00.894263+0000 mgr.y (mgr.14901) 7182 : cluster [DBG] pgmap v7189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:02.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:02 smithi006 bash[27292]: cluster 2023-08-28T01:58:00.894263+0000 mgr.y (mgr.14901) 7182 : cluster [DBG] pgmap v7189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:58:02.737Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:58:02.739Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:03.520 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:03 smithi130 bash[51302]: debug 2023-08-28T01:58:03.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:03.131655+0000) 2023-08-28T01:58:03.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:03 smithi130 bash[27579]: audit 2023-08-28T01:58:03.181494+0000 mon.c (mon.2) 2823 : audit [DBG] from='client.? 172.21.15.6:0/1924587550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:03.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:03 smithi006 bash[27292]: audit 2023-08-28T01:58:03.181494+0000 mon.c (mon.2) 2823 : audit [DBG] from='client.? 172.21.15.6:0/1924587550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:03.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:03 smithi006 bash[33918]: audit 2023-08-28T01:58:03.181494+0000 mon.c (mon.2) 2823 : audit [DBG] from='client.? 172.21.15.6:0/1924587550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:04.523 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:04 smithi130 bash[51302]: debug 2023-08-28T01:58:04.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:04.131964+0000) 2023-08-28T01:58:04.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:04 smithi130 bash[27579]: cluster 2023-08-28T01:58:02.894862+0000 mgr.y (mgr.14901) 7183 : cluster [DBG] pgmap v7190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:04.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:04 smithi006 bash[27292]: cluster 2023-08-28T01:58:02.894862+0000 mgr.y (mgr.14901) 7183 : cluster [DBG] pgmap v7190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:04.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:04 smithi006 bash[33918]: cluster 2023-08-28T01:58:02.894862+0000 mgr.y (mgr.14901) 7183 : cluster [DBG] pgmap v7190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:58:04.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:58:04.620Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:05.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:05 smithi130 bash[51302]: debug 2023-08-28T01:58:05.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:05.132215+0000) 2023-08-28T01:58:06.531 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:06 smithi130 bash[51302]: debug 2023-08-28T01:58:06.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:06.132507+0000) 2023-08-28T01:58:06.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:06 smithi130 bash[27579]: cluster 2023-08-28T01:58:04.896010+0000 mgr.y (mgr.14901) 7184 : cluster [DBG] pgmap v7191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:06.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:06 smithi130 bash[27579]: audit 2023-08-28T01:58:05.592824+0000 mon.c (mon.2) 2824 : audit [DBG] from='client.? 172.21.15.6:0/3661207250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:06.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:06 smithi006 bash[27292]: cluster 2023-08-28T01:58:04.896010+0000 mgr.y (mgr.14901) 7184 : cluster [DBG] pgmap v7191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:06.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:06 smithi006 bash[27292]: audit 2023-08-28T01:58:05.592824+0000 mon.c (mon.2) 2824 : audit [DBG] from='client.? 172.21.15.6:0/3661207250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:06.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:06 smithi006 bash[33918]: cluster 2023-08-28T01:58:04.896010+0000 mgr.y (mgr.14901) 7184 : cluster [DBG] pgmap v7191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:06.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:06 smithi006 bash[33918]: audit 2023-08-28T01:58:05.592824+0000 mon.c (mon.2) 2824 : audit [DBG] from='client.? 172.21.15.6:0/3661207250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:07.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:07 smithi130 bash[51302]: debug 2023-08-28T01:58:07.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:07.132803+0000) 2023-08-28T01:58:08.486 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:08 smithi130 bash[51302]: debug 2023-08-28T01:58:08.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:08.133081+0000) 2023-08-28T01:58:08.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:08 smithi130 bash[27579]: cluster 2023-08-28T01:58:06.896682+0000 mgr.y (mgr.14901) 7185 : cluster [DBG] pgmap v7192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:08 smithi130 bash[27579]: audit 2023-08-28T01:58:08.002662+0000 mon.a (mon.0) 4666 : audit [DBG] from='client.? 172.21.15.6:0/2194444529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:08.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:08 smithi006 bash[33918]: cluster 2023-08-28T01:58:06.896682+0000 mgr.y (mgr.14901) 7185 : cluster [DBG] pgmap v7192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:08.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:08 smithi006 bash[33918]: audit 2023-08-28T01:58:08.002662+0000 mon.a (mon.0) 4666 : audit [DBG] from='client.? 172.21.15.6:0/2194444529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:08.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:08 smithi006 bash[27292]: cluster 2023-08-28T01:58:06.896682+0000 mgr.y (mgr.14901) 7185 : cluster [DBG] pgmap v7192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:08.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:08 smithi006 bash[27292]: audit 2023-08-28T01:58:08.002662+0000 mon.a (mon.0) 4666 : audit [DBG] from='client.? 172.21.15.6:0/2194444529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:09.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:09 smithi130 bash[51302]: debug 2023-08-28T01:58:09.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:09.133394+0000) 2023-08-28T01:58:10.491 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:10 smithi130 bash[51302]: debug 2023-08-28T01:58:10.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:10.133779+0000) 2023-08-28T01:58:10.752 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:10 smithi006 bash[27292]: cluster 2023-08-28T01:58:08.897422+0000 mgr.y (mgr.14901) 7186 : cluster [DBG] pgmap v7193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:10.752 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:10 smithi006 bash[27292]: audit 2023-08-28T01:58:10.427745+0000 mon.a (mon.0) 4667 : audit [DBG] from='client.? 172.21.15.6:0/3009293831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:10.752 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:10 smithi006 bash[33918]: cluster 2023-08-28T01:58:08.897422+0000 mgr.y (mgr.14901) 7186 : cluster [DBG] pgmap v7193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:10.752 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:10 smithi006 bash[33918]: audit 2023-08-28T01:58:10.427745+0000 mon.a (mon.0) 4667 : audit [DBG] from='client.? 172.21.15.6:0/3009293831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:10.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:10 smithi130 bash[27579]: cluster 2023-08-28T01:58:08.897422+0000 mgr.y (mgr.14901) 7186 : cluster [DBG] pgmap v7193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:10.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:10 smithi130 bash[27579]: audit 2023-08-28T01:58:10.427745+0000 mon.a (mon.0) 4667 : audit [DBG] from='client.? 172.21.15.6:0/3009293831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:11.170 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:58:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:58:10] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:58:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:58:11.170Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:58:11.172Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:11.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:11 smithi130 bash[51302]: debug 2023-08-28T01:58:11.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:11.134242+0000) 2023-08-28T01:58:12.495 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:12 smithi130 bash[51302]: debug 2023-08-28T01:58:12.128+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:12.134516+0000) 2023-08-28T01:58:12.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:12 smithi130 bash[27579]: cluster 2023-08-28T01:58:10.898538+0000 mgr.y (mgr.14901) 7187 : cluster [DBG] pgmap v7194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:12.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:12 smithi006 bash[27292]: cluster 2023-08-28T01:58:10.898538+0000 mgr.y (mgr.14901) 7187 : cluster [DBG] pgmap v7194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:12.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:12 smithi006 bash[33918]: cluster 2023-08-28T01:58:10.898538+0000 mgr.y (mgr.14901) 7187 : cluster [DBG] pgmap v7194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:58:12.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:58:12.741Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:13.498 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:13 smithi130 bash[51302]: debug 2023-08-28T01:58:13.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:13.134820+0000) 2023-08-28T01:58:13.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:13 smithi130 bash[27579]: audit 2023-08-28T01:58:12.848865+0000 mon.c (mon.2) 2825 : audit [DBG] from='client.? 172.21.15.6:0/2026892789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:13.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:13 smithi006 bash[33918]: audit 2023-08-28T01:58:12.848865+0000 mon.c (mon.2) 2825 : audit [DBG] from='client.? 172.21.15.6:0/2026892789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:13.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:13 smithi006 bash[27292]: audit 2023-08-28T01:58:12.848865+0000 mon.c (mon.2) 2825 : audit [DBG] from='client.? 172.21.15.6:0/2026892789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:14.508 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:14 smithi130 bash[51302]: debug 2023-08-28T01:58:14.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:14.135112+0000) 2023-08-28T01:58:14.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:14 smithi130 bash[27579]: cluster 2023-08-28T01:58:12.899193+0000 mgr.y (mgr.14901) 7188 : cluster [DBG] pgmap v7195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:14.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:14 smithi006 bash[27292]: cluster 2023-08-28T01:58:12.899193+0000 mgr.y (mgr.14901) 7188 : cluster [DBG] pgmap v7195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:14.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:14 smithi006 bash[33918]: cluster 2023-08-28T01:58:12.899193+0000 mgr.y (mgr.14901) 7188 : cluster [DBG] pgmap v7195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:58:14.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:58:14.620Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:15.520 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:15 smithi130 bash[51302]: debug 2023-08-28T01:58:15.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:15.135410+0000) 2023-08-28T01:58:15.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:15 smithi130 bash[27579]: audit 2023-08-28T01:58:15.266831+0000 mon.a (mon.0) 4668 : audit [DBG] from='client.? 172.21.15.6:0/3739871864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:15.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:15 smithi006 bash[33918]: audit 2023-08-28T01:58:15.266831+0000 mon.a (mon.0) 4668 : audit [DBG] from='client.? 172.21.15.6:0/3739871864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:15.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:15 smithi006 bash[27292]: audit 2023-08-28T01:58:15.266831+0000 mon.a (mon.0) 4668 : audit [DBG] from='client.? 172.21.15.6:0/3739871864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:16.522 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:16 smithi130 bash[51302]: debug 2023-08-28T01:58:16.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:16.135699+0000) 2023-08-28T01:58:16.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:16 smithi130 bash[27579]: cluster 2023-08-28T01:58:14.900341+0000 mgr.y (mgr.14901) 7189 : cluster [DBG] pgmap v7196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:16.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:16 smithi006 bash[27292]: cluster 2023-08-28T01:58:14.900341+0000 mgr.y (mgr.14901) 7189 : cluster [DBG] pgmap v7196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:16.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:16 smithi006 bash[33918]: cluster 2023-08-28T01:58:14.900341+0000 mgr.y (mgr.14901) 7189 : cluster [DBG] pgmap v7196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:17.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:17 smithi130 bash[51302]: debug 2023-08-28T01:58:17.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:17.135980+0000) 2023-08-28T01:58:18.529 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:18 smithi130 bash[51302]: debug 2023-08-28T01:58:18.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:18.136313+0000) 2023-08-28T01:58:18.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:18 smithi130 bash[27579]: cluster 2023-08-28T01:58:16.901041+0000 mgr.y (mgr.14901) 7190 : cluster [DBG] pgmap v7197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:18.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:18 smithi130 bash[27579]: audit 2023-08-28T01:58:17.680508+0000 mon.a (mon.0) 4669 : audit [DBG] from='client.? 172.21.15.6:0/2230346865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:18.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:18 smithi006 bash[33918]: cluster 2023-08-28T01:58:16.901041+0000 mgr.y (mgr.14901) 7190 : cluster [DBG] pgmap v7197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:18.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:18 smithi006 bash[33918]: audit 2023-08-28T01:58:17.680508+0000 mon.a (mon.0) 4669 : audit [DBG] from='client.? 172.21.15.6:0/2230346865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:18.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:18 smithi006 bash[27292]: cluster 2023-08-28T01:58:16.901041+0000 mgr.y (mgr.14901) 7190 : cluster [DBG] pgmap v7197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:18.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:18 smithi006 bash[27292]: audit 2023-08-28T01:58:17.680508+0000 mon.a (mon.0) 4669 : audit [DBG] from='client.? 172.21.15.6:0/2230346865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:19.582 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:19 smithi130 bash[51302]: debug 2023-08-28T01:58:19.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:19.136749+0000) 2023-08-28T01:58:19.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:19 smithi130 bash[27579]: cluster 2023-08-28T01:58:18.901707+0000 mgr.y (mgr.14901) 7191 : cluster [DBG] pgmap v7198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:19.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:19 smithi006 bash[27292]: cluster 2023-08-28T01:58:18.901707+0000 mgr.y (mgr.14901) 7191 : cluster [DBG] pgmap v7198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:19.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:19 smithi006 bash[33918]: cluster 2023-08-28T01:58:18.901707+0000 mgr.y (mgr.14901) 7191 : cluster [DBG] pgmap v7198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:20.585 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:20 smithi130 bash[51302]: debug 2023-08-28T01:58:20.136+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:20.137024+0000) 2023-08-28T01:58:20.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:20 smithi130 bash[27579]: audit 2023-08-28T01:58:20.099352+0000 mon.a (mon.0) 4670 : audit [DBG] from='client.? 172.21.15.6:0/3582243872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:20.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:58:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:58:20] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T01:58:20.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:20 smithi006 bash[33918]: audit 2023-08-28T01:58:20.099352+0000 mon.a (mon.0) 4670 : audit [DBG] from='client.? 172.21.15.6:0/3582243872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:20.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:20 smithi006 bash[27292]: audit 2023-08-28T01:58:20.099352+0000 mon.a (mon.0) 4670 : audit [DBG] from='client.? 172.21.15.6:0/3582243872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:58:21.171Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:58:21.174Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:21.593 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:21 smithi130 bash[51302]: debug 2023-08-28T01:58:21.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:21.137317+0000) 2023-08-28T01:58:21.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:21 smithi130 bash[27579]: cluster 2023-08-28T01:58:20.902815+0000 mgr.y (mgr.14901) 7192 : cluster [DBG] pgmap v7199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:21.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:21 smithi006 bash[33918]: cluster 2023-08-28T01:58:20.902815+0000 mgr.y (mgr.14901) 7192 : cluster [DBG] pgmap v7199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:21.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:21 smithi006 bash[27292]: cluster 2023-08-28T01:58:20.902815+0000 mgr.y (mgr.14901) 7192 : cluster [DBG] pgmap v7199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:22.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:22 smithi130 bash[51302]: debug 2023-08-28T01:58:22.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:22.137536+0000) 2023-08-28T01:58:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:58:22.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:58:22.740Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:23.496 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:23 smithi130 bash[51302]: debug 2023-08-28T01:58:23.136+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:23.137803+0000) 2023-08-28T01:58:23.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:23 smithi130 bash[27579]: audit 2023-08-28T01:58:22.519679+0000 mon.c (mon.2) 2826 : audit [DBG] from='client.? 172.21.15.6:0/2843465907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:23.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:23 smithi006 bash[33918]: audit 2023-08-28T01:58:22.519679+0000 mon.c (mon.2) 2826 : audit [DBG] from='client.? 172.21.15.6:0/2843465907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:23.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:23 smithi006 bash[27292]: audit 2023-08-28T01:58:22.519679+0000 mon.c (mon.2) 2826 : audit [DBG] from='client.? 172.21.15.6:0/2843465907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:24.498 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:24 smithi130 bash[51302]: debug 2023-08-28T01:58:24.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:24.138091+0000) 2023-08-28T01:58:24.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:24 smithi130 bash[27579]: cluster 2023-08-28T01:58:22.903610+0000 mgr.y (mgr.14901) 7193 : cluster [DBG] pgmap v7200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:24.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:24 smithi006 bash[27292]: cluster 2023-08-28T01:58:22.903610+0000 mgr.y (mgr.14901) 7193 : cluster [DBG] pgmap v7200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:24.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:24 smithi006 bash[33918]: cluster 2023-08-28T01:58:22.903610+0000 mgr.y (mgr.14901) 7193 : cluster [DBG] pgmap v7200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:58:24.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:58:24.620Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:25.501 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:25 smithi130 bash[51302]: debug 2023-08-28T01:58:25.132+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:25.138368+0000) 2023-08-28T01:58:25.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:25 smithi130 bash[27579]: audit 2023-08-28T01:58:24.939368+0000 mon.c (mon.2) 2827 : audit [DBG] from='client.? 172.21.15.6:0/1850352415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:25.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:25 smithi006 bash[33918]: audit 2023-08-28T01:58:24.939368+0000 mon.c (mon.2) 2827 : audit [DBG] from='client.? 172.21.15.6:0/1850352415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:25.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:25 smithi006 bash[27292]: audit 2023-08-28T01:58:24.939368+0000 mon.c (mon.2) 2827 : audit [DBG] from='client.? 172.21.15.6:0/1850352415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:26.512 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:26 smithi130 bash[51302]: debug 2023-08-28T01:58:26.136+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:26.138635+0000) 2023-08-28T01:58:26.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:26 smithi130 bash[27579]: cluster 2023-08-28T01:58:24.904698+0000 mgr.y (mgr.14901) 7194 : cluster [DBG] pgmap v7201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:26.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:26 smithi006 bash[33918]: cluster 2023-08-28T01:58:24.904698+0000 mgr.y (mgr.14901) 7194 : cluster [DBG] pgmap v7201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:26.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:26 smithi006 bash[27292]: cluster 2023-08-28T01:58:24.904698+0000 mgr.y (mgr.14901) 7194 : cluster [DBG] pgmap v7201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:27.520 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:27 smithi130 bash[51302]: debug 2023-08-28T01:58:27.136+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:27.138932+0000) 2023-08-28T01:58:27.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:27 smithi130 bash[27579]: audit 2023-08-28T01:58:27.353217+0000 mon.c (mon.2) 2828 : audit [DBG] from='client.? 172.21.15.6:0/2786175639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:27.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:27 smithi006 bash[33918]: audit 2023-08-28T01:58:27.353217+0000 mon.c (mon.2) 2828 : audit [DBG] from='client.? 172.21.15.6:0/2786175639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:27.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:27 smithi006 bash[27292]: audit 2023-08-28T01:58:27.353217+0000 mon.c (mon.2) 2828 : audit [DBG] from='client.? 172.21.15.6:0/2786175639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:28.525 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:28 smithi130 bash[51302]: debug 2023-08-28T01:58:28.136+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:28.139222+0000) 2023-08-28T01:58:28.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:28 smithi130 bash[27579]: cluster 2023-08-28T01:58:26.905383+0000 mgr.y (mgr.14901) 7195 : cluster [DBG] pgmap v7202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:28.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:28 smithi006 bash[27292]: cluster 2023-08-28T01:58:26.905383+0000 mgr.y (mgr.14901) 7195 : cluster [DBG] pgmap v7202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:28.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:28 smithi006 bash[33918]: cluster 2023-08-28T01:58:26.905383+0000 mgr.y (mgr.14901) 7195 : cluster [DBG] pgmap v7202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:29.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:29 smithi130 bash[51302]: debug 2023-08-28T01:58:29.136+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:29.139471+0000) 2023-08-28T01:58:30.533 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:30 smithi130 bash[51302]: debug 2023-08-28T01:58:30.136+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:30.139791+0000) 2023-08-28T01:58:30.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:30 smithi130 bash[27579]: cluster 2023-08-28T01:58:28.906013+0000 mgr.y (mgr.14901) 7196 : cluster [DBG] pgmap v7203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:30 smithi130 bash[27579]: audit 2023-08-28T01:58:29.774161+0000 mon.a (mon.0) 4671 : audit [DBG] from='client.? 172.21.15.6:0/748211424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:30.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:58:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:58:30] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:58:30.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:30 smithi006 bash[27292]: cluster 2023-08-28T01:58:28.906013+0000 mgr.y (mgr.14901) 7196 : cluster [DBG] pgmap v7203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:30.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:30 smithi006 bash[27292]: audit 2023-08-28T01:58:29.774161+0000 mon.a (mon.0) 4671 : audit [DBG] from='client.? 172.21.15.6:0/748211424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:30.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:30 smithi006 bash[33918]: cluster 2023-08-28T01:58:28.906013+0000 mgr.y (mgr.14901) 7196 : cluster [DBG] pgmap v7203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:30.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:30 smithi006 bash[33918]: audit 2023-08-28T01:58:29.774161+0000 mon.a (mon.0) 4671 : audit [DBG] from='client.? 172.21.15.6:0/748211424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:58:31.173Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:58:31.175Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:31.584 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:31 smithi130 bash[51302]: debug 2023-08-28T01:58:31.135+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:31.140106+0000) 2023-08-28T01:58:31.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:31 smithi130 bash[27579]: cluster 2023-08-28T01:58:30.907212+0000 mgr.y (mgr.14901) 7197 : cluster [DBG] pgmap v7204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:31.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:31 smithi006 bash[33918]: cluster 2023-08-28T01:58:30.907212+0000 mgr.y (mgr.14901) 7197 : cluster [DBG] pgmap v7204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:31.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:31 smithi006 bash[27292]: cluster 2023-08-28T01:58:30.907212+0000 mgr.y (mgr.14901) 7197 : cluster [DBG] pgmap v7204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:32.589 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:32 smithi130 bash[51302]: debug 2023-08-28T01:58:32.135+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:32.140378+0000) 2023-08-28T01:58:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:32 smithi130 bash[27579]: audit 2023-08-28T01:58:32.190992+0000 mon.a (mon.0) 4672 : audit [DBG] from='client.? 172.21.15.6:0/3233087331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:32.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:32 smithi006 bash[27292]: audit 2023-08-28T01:58:32.190992+0000 mon.a (mon.0) 4672 : audit [DBG] from='client.? 172.21.15.6:0/3233087331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:32.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:32 smithi006 bash[33918]: audit 2023-08-28T01:58:32.190992+0000 mon.a (mon.0) 4672 : audit [DBG] from='client.? 172.21.15.6:0/3233087331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:58:32.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:58:32.741Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:33.598 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:33 smithi130 bash[51302]: debug 2023-08-28T01:58:33.135+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:33.140683+0000) 2023-08-28T01:58:33.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:33 smithi130 bash[27579]: cluster 2023-08-28T01:58:32.908062+0000 mgr.y (mgr.14901) 7198 : cluster [DBG] pgmap v7205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:33 smithi006 bash[27292]: cluster 2023-08-28T01:58:32.908062+0000 mgr.y (mgr.14901) 7198 : cluster [DBG] pgmap v7205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:33.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:33 smithi006 bash[33918]: cluster 2023-08-28T01:58:32.908062+0000 mgr.y (mgr.14901) 7198 : cluster [DBG] pgmap v7205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:34.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:34 smithi130 bash[51302]: debug 2023-08-28T01:58:34.135+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:34.140985+0000) 2023-08-28T01:58:34.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:34 smithi006 bash[27292]: audit 2023-08-28T01:58:34.604989+0000 mon.c (mon.2) 2829 : audit [DBG] from='client.? 172.21.15.6:0/339459014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:34.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:34 smithi006 bash[33918]: audit 2023-08-28T01:58:34.604989+0000 mon.c (mon.2) 2829 : audit [DBG] from='client.? 172.21.15.6:0/339459014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:58:34.619Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:58:34.621Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:35.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:34 smithi130 bash[27579]: audit 2023-08-28T01:58:34.604989+0000 mon.c (mon.2) 2829 : audit [DBG] from='client.? 172.21.15.6:0/339459014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:35.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:35 smithi130 bash[51302]: debug 2023-08-28T01:58:35.135+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:35.141242+0000) 2023-08-28T01:58:35.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:35 smithi006 bash[33918]: cluster 2023-08-28T01:58:34.909254+0000 mgr.y (mgr.14901) 7199 : cluster [DBG] pgmap v7206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:35.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:35 smithi006 bash[27292]: cluster 2023-08-28T01:58:34.909254+0000 mgr.y (mgr.14901) 7199 : cluster [DBG] pgmap v7206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:36.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:35 smithi130 bash[27579]: cluster 2023-08-28T01:58:34.909254+0000 mgr.y (mgr.14901) 7199 : cluster [DBG] pgmap v7206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:36.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:36 smithi130 bash[51302]: debug 2023-08-28T01:58:36.135+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:36.141515+0000) 2023-08-28T01:58:37.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:37 smithi130 bash[51302]: debug 2023-08-28T01:58:37.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:37.141817+0000) 2023-08-28T01:58:38.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:37 smithi130 bash[27579]: cluster 2023-08-28T01:58:36.909881+0000 mgr.y (mgr.14901) 7200 : cluster [DBG] pgmap v7207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:38.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:37 smithi130 bash[27579]: audit 2023-08-28T01:58:37.022832+0000 mon.c (mon.2) 2830 : audit [DBG] from='client.? 172.21.15.6:0/3843566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:38.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:38 smithi130 bash[51302]: debug 2023-08-28T01:58:38.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:38.142094+0000) 2023-08-28T01:58:38.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:37 smithi006 bash[27292]: cluster 2023-08-28T01:58:36.909881+0000 mgr.y (mgr.14901) 7200 : cluster [DBG] pgmap v7207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:38.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:37 smithi006 bash[27292]: audit 2023-08-28T01:58:37.022832+0000 mon.c (mon.2) 2830 : audit [DBG] from='client.? 172.21.15.6:0/3843566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:38.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:37 smithi006 bash[33918]: cluster 2023-08-28T01:58:36.909881+0000 mgr.y (mgr.14901) 7200 : cluster [DBG] pgmap v7207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:38.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:37 smithi006 bash[33918]: audit 2023-08-28T01:58:37.022832+0000 mon.c (mon.2) 2830 : audit [DBG] from='client.? 172.21.15.6:0/3843566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:39.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:39 smithi130 bash[51302]: debug 2023-08-28T01:58:39.135+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:39.142348+0000) 2023-08-28T01:58:40.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:40 smithi130 bash[51302]: debug 2023-08-28T01:58:40.135+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:40.142606+0000) 2023-08-28T01:58:40.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:39 smithi130 bash[27579]: cluster 2023-08-28T01:58:38.910548+0000 mgr.y (mgr.14901) 7201 : cluster [DBG] pgmap v7208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:40.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:39 smithi130 bash[27579]: audit 2023-08-28T01:58:39.439533+0000 mon.a (mon.0) 4673 : audit [DBG] from='client.? 172.21.15.6:0/3388258586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:40.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:39 smithi006 bash[33918]: cluster 2023-08-28T01:58:38.910548+0000 mgr.y (mgr.14901) 7201 : cluster [DBG] pgmap v7208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:40.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:39 smithi006 bash[33918]: audit 2023-08-28T01:58:39.439533+0000 mon.a (mon.0) 4673 : audit [DBG] from='client.? 172.21.15.6:0/3388258586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:40.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:39 smithi006 bash[27292]: cluster 2023-08-28T01:58:38.910548+0000 mgr.y (mgr.14901) 7201 : cluster [DBG] pgmap v7208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:40.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:39 smithi006 bash[27292]: audit 2023-08-28T01:58:39.439533+0000 mon.a (mon.0) 4673 : audit [DBG] from='client.? 172.21.15.6:0/3388258586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:41.174 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:58:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:58:40] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:58:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:58:41.174Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:58:41.176Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:41.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:41 smithi130 bash[51302]: debug 2023-08-28T01:58:41.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:41.142949+0000) 2023-08-28T01:58:42.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:41 smithi130 bash[27579]: cluster 2023-08-28T01:58:40.911668+0000 mgr.y (mgr.14901) 7202 : cluster [DBG] pgmap v7209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:42.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:41 smithi130 bash[27579]: audit 2023-08-28T01:58:41.852721+0000 mon.a (mon.0) 4674 : audit [DBG] from='client.? 172.21.15.6:0/848247910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:42.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:42 smithi130 bash[51302]: debug 2023-08-28T01:58:42.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:42.143201+0000) 2023-08-28T01:58:42.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:41 smithi006 bash[27292]: cluster 2023-08-28T01:58:40.911668+0000 mgr.y (mgr.14901) 7202 : cluster [DBG] pgmap v7209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:42.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:41 smithi006 bash[27292]: audit 2023-08-28T01:58:41.852721+0000 mon.a (mon.0) 4674 : audit [DBG] from='client.? 172.21.15.6:0/848247910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:42.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:41 smithi006 bash[33918]: cluster 2023-08-28T01:58:40.911668+0000 mgr.y (mgr.14901) 7202 : cluster [DBG] pgmap v7209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:42.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:41 smithi006 bash[33918]: audit 2023-08-28T01:58:41.852721+0000 mon.a (mon.0) 4674 : audit [DBG] from='client.? 172.21.15.6:0/848247910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:58:42.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:58:42.741Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:43.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:43 smithi130 bash[51302]: debug 2023-08-28T01:58:43.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:43.143413+0000) 2023-08-28T01:58:44.499 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:44 smithi130 bash[51302]: debug 2023-08-28T01:58:44.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:44.143771+0000) 2023-08-28T01:58:44.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:44 smithi130 bash[27579]: cluster 2023-08-28T01:58:42.912286+0000 mgr.y (mgr.14901) 7203 : cluster [DBG] pgmap v7210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:44.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:44 smithi130 bash[27579]: audit 2023-08-28T01:58:44.278312+0000 mon.a (mon.0) 4675 : audit [DBG] from='client.? 172.21.15.6:0/2555279157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:44.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:44 smithi006 bash[27292]: cluster 2023-08-28T01:58:42.912286+0000 mgr.y (mgr.14901) 7203 : cluster [DBG] pgmap v7210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:44.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:44 smithi006 bash[27292]: audit 2023-08-28T01:58:44.278312+0000 mon.a (mon.0) 4675 : audit [DBG] from='client.? 172.21.15.6:0/2555279157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:58:44.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:58:44.622Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:44.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:44 smithi006 bash[33918]: cluster 2023-08-28T01:58:42.912286+0000 mgr.y (mgr.14901) 7203 : cluster [DBG] pgmap v7210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:44.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:44 smithi006 bash[33918]: audit 2023-08-28T01:58:44.278312+0000 mon.a (mon.0) 4675 : audit [DBG] from='client.? 172.21.15.6:0/2555279157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:45.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:45 smithi130 bash[51302]: debug 2023-08-28T01:58:45.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:45.144046+0000) 2023-08-28T01:58:46.505 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:46 smithi130 bash[51302]: debug 2023-08-28T01:58:46.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:46.144372+0000) 2023-08-28T01:58:46.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:46 smithi130 bash[27579]: cluster 2023-08-28T01:58:44.913391+0000 mgr.y (mgr.14901) 7204 : cluster [DBG] pgmap v7211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:46.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:46 smithi006 bash[33918]: cluster 2023-08-28T01:58:44.913391+0000 mgr.y (mgr.14901) 7204 : cluster [DBG] pgmap v7211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:46.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:46 smithi006 bash[27292]: cluster 2023-08-28T01:58:44.913391+0000 mgr.y (mgr.14901) 7204 : cluster [DBG] pgmap v7211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:47.514 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:47 smithi130 bash[51302]: debug 2023-08-28T01:58:47.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:47.144634+0000) 2023-08-28T01:58:47.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:47 smithi130 bash[27579]: audit 2023-08-28T01:58:46.664628+0000 mon.b (mon.1) 803 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:58:47.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:47 smithi130 bash[27579]: audit 2023-08-28T01:58:46.698516+0000 mon.a (mon.0) 4676 : audit [DBG] from='client.? 172.21.15.6:0/804356167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:47.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:47 smithi006 bash[33918]: audit 2023-08-28T01:58:46.664628+0000 mon.b (mon.1) 803 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:58:47.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:47 smithi006 bash[33918]: audit 2023-08-28T01:58:46.698516+0000 mon.a (mon.0) 4676 : audit [DBG] from='client.? 172.21.15.6:0/804356167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:47.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:47 smithi006 bash[27292]: audit 2023-08-28T01:58:46.664628+0000 mon.b (mon.1) 803 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:58:47.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:47 smithi006 bash[27292]: audit 2023-08-28T01:58:46.698516+0000 mon.a (mon.0) 4676 : audit [DBG] from='client.? 172.21.15.6:0/804356167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:48.516 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:48 smithi130 bash[51302]: debug 2023-08-28T01:58:48.143+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:48.144947+0000) 2023-08-28T01:58:48.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:48 smithi130 bash[27579]: cluster 2023-08-28T01:58:46.914003+0000 mgr.y (mgr.14901) 7205 : cluster [DBG] pgmap v7212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:48.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:48 smithi006 bash[27292]: cluster 2023-08-28T01:58:46.914003+0000 mgr.y (mgr.14901) 7205 : cluster [DBG] pgmap v7212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:48.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:48 smithi006 bash[33918]: cluster 2023-08-28T01:58:46.914003+0000 mgr.y (mgr.14901) 7205 : cluster [DBG] pgmap v7212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:49.525 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:49 smithi130 bash[51302]: debug 2023-08-28T01:58:49.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:49.145274+0000) 2023-08-28T01:58:49.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:49 smithi130 bash[27579]: audit 2023-08-28T01:58:49.111653+0000 mon.a (mon.0) 4677 : audit [DBG] from='client.? 172.21.15.6:0/3358326565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:49.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:49 smithi006 bash[27292]: audit 2023-08-28T01:58:49.111653+0000 mon.a (mon.0) 4677 : audit [DBG] from='client.? 172.21.15.6:0/3358326565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:49.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:49 smithi006 bash[33918]: audit 2023-08-28T01:58:49.111653+0000 mon.a (mon.0) 4677 : audit [DBG] from='client.? 172.21.15.6:0/3358326565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:50.527 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:50 smithi130 bash[51302]: debug 2023-08-28T01:58:50.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:50.145539+0000) 2023-08-28T01:58:50.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:50 smithi130 bash[27579]: cluster 2023-08-28T01:58:48.914555+0000 mgr.y (mgr.14901) 7206 : cluster [DBG] pgmap v7213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:50.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:58:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:58:50] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T01:58:50.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:50 smithi006 bash[27292]: cluster 2023-08-28T01:58:48.914555+0000 mgr.y (mgr.14901) 7206 : cluster [DBG] pgmap v7213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:50.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:50 smithi006 bash[33918]: cluster 2023-08-28T01:58:48.914555+0000 mgr.y (mgr.14901) 7206 : cluster [DBG] pgmap v7213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:58:51.175Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:58:51.177Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:51.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:51 smithi130 bash[51302]: debug 2023-08-28T01:58:51.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:51.145829+0000) 2023-08-28T01:58:52.532 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:52 smithi130 bash[51302]: debug 2023-08-28T01:58:52.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:52.146043+0000) 2023-08-28T01:58:52.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:52 smithi130 bash[27579]: cluster 2023-08-28T01:58:50.915572+0000 mgr.y (mgr.14901) 7207 : cluster [DBG] pgmap v7214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:52.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:52 smithi130 bash[27579]: audit 2023-08-28T01:58:51.551723+0000 mon.a (mon.0) 4678 : audit [DBG] from='client.? 172.21.15.6:0/2108124558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:58:52.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:58:52.741Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:52.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:52 smithi006 bash[27292]: cluster 2023-08-28T01:58:50.915572+0000 mgr.y (mgr.14901) 7207 : cluster [DBG] pgmap v7214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:52.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:52 smithi006 bash[27292]: audit 2023-08-28T01:58:51.551723+0000 mon.a (mon.0) 4678 : audit [DBG] from='client.? 172.21.15.6:0/2108124558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:52.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:52 smithi006 bash[33918]: cluster 2023-08-28T01:58:50.915572+0000 mgr.y (mgr.14901) 7207 : cluster [DBG] pgmap v7214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:52.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:52 smithi006 bash[33918]: audit 2023-08-28T01:58:51.551723+0000 mon.a (mon.0) 4678 : audit [DBG] from='client.? 172.21.15.6:0/2108124558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:53.561 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:53 smithi130 bash[51302]: debug 2023-08-28T01:58:53.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:53.146299+0000) 2023-08-28T01:58:53.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:53 smithi130 bash[27579]: audit 2023-08-28T01:58:52.557193+0000 mon.a (mon.0) 4679 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:58:53.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:53 smithi130 bash[27579]: cluster 2023-08-28T01:58:52.916214+0000 mgr.y (mgr.14901) 7208 : cluster [DBG] pgmap v7215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:53.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:53 smithi130 bash[27579]: audit 2023-08-28T01:58:52.978995+0000 mon.a (mon.0) 4680 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:58:53.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:53 smithi130 bash[27579]: audit 2023-08-28T01:58:53.263963+0000 mon.b (mon.1) 804 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:58:53.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:53 smithi130 bash[27579]: audit 2023-08-28T01:58:53.266252+0000 mon.b (mon.1) 805 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:58:53.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:53 smithi130 bash[27579]: audit 2023-08-28T01:58:53.273796+0000 mon.a (mon.0) 4681 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:58:53.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:53 smithi006 bash[27292]: audit 2023-08-28T01:58:52.557193+0000 mon.a (mon.0) 4679 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:58:53.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:53 smithi006 bash[27292]: cluster 2023-08-28T01:58:52.916214+0000 mgr.y (mgr.14901) 7208 : cluster [DBG] pgmap v7215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:53.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:53 smithi006 bash[27292]: audit 2023-08-28T01:58:52.978995+0000 mon.a (mon.0) 4680 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:58:53.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:53 smithi006 bash[27292]: audit 2023-08-28T01:58:53.263963+0000 mon.b (mon.1) 804 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:58:53.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:53 smithi006 bash[27292]: audit 2023-08-28T01:58:53.266252+0000 mon.b (mon.1) 805 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:58:53.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:53 smithi006 bash[27292]: audit 2023-08-28T01:58:53.273796+0000 mon.a (mon.0) 4681 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:58:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:53 smithi006 bash[33918]: audit 2023-08-28T01:58:52.557193+0000 mon.a (mon.0) 4679 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:58:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:53 smithi006 bash[33918]: cluster 2023-08-28T01:58:52.916214+0000 mgr.y (mgr.14901) 7208 : cluster [DBG] pgmap v7215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:53 smithi006 bash[33918]: audit 2023-08-28T01:58:52.978995+0000 mon.a (mon.0) 4680 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:58:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:53 smithi006 bash[33918]: audit 2023-08-28T01:58:53.263963+0000 mon.b (mon.1) 804 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:58:53.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:53 smithi006 bash[33918]: audit 2023-08-28T01:58:53.266252+0000 mon.b (mon.1) 805 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:58:53.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:53 smithi006 bash[33918]: audit 2023-08-28T01:58:53.273796+0000 mon.a (mon.0) 4681 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T01:58:54.563 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:54 smithi130 bash[51302]: debug 2023-08-28T01:58:54.139+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:54.146602+0000) 2023-08-28T01:58:54.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:54 smithi130 bash[27579]: audit 2023-08-28T01:58:53.973056+0000 mon.c (mon.2) 2831 : audit [DBG] from='client.? 172.21.15.6:0/1740742351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:54.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:54 smithi006 bash[33918]: audit 2023-08-28T01:58:53.973056+0000 mon.c (mon.2) 2831 : audit [DBG] from='client.? 172.21.15.6:0/1740742351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:54.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:54 smithi006 bash[27292]: audit 2023-08-28T01:58:53.973056+0000 mon.c (mon.2) 2831 : audit [DBG] from='client.? 172.21.15.6:0/1740742351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:58:54.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:58:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:58:54.623Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:58:55.572 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:55 smithi130 bash[51302]: debug 2023-08-28T01:58:55.143+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:55.146966+0000) 2023-08-28T01:58:55.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:55 smithi130 bash[27579]: cluster 2023-08-28T01:58:54.917363+0000 mgr.y (mgr.14901) 7209 : cluster [DBG] pgmap v7216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:55.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:55 smithi006 bash[33918]: cluster 2023-08-28T01:58:54.917363+0000 mgr.y (mgr.14901) 7209 : cluster [DBG] pgmap v7216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:55.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:55 smithi006 bash[27292]: cluster 2023-08-28T01:58:54.917363+0000 mgr.y (mgr.14901) 7209 : cluster [DBG] pgmap v7216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:56.576 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:56 smithi130 bash[51302]: debug 2023-08-28T01:58:56.143+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:56.147184+0000) 2023-08-28T01:58:56.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:56 smithi130 bash[27579]: audit 2023-08-28T01:58:56.382204+0000 mon.c (mon.2) 2832 : audit [DBG] from='client.? 172.21.15.6:0/283318439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:56.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:56 smithi006 bash[27292]: audit 2023-08-28T01:58:56.382204+0000 mon.c (mon.2) 2832 : audit [DBG] from='client.? 172.21.15.6:0/283318439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:56.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:56 smithi006 bash[33918]: audit 2023-08-28T01:58:56.382204+0000 mon.c (mon.2) 2832 : audit [DBG] from='client.? 172.21.15.6:0/283318439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:57.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:57 smithi130 bash[51302]: debug 2023-08-28T01:58:57.143+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:57.147474+0000) 2023-08-28T01:58:58.498 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:58 smithi130 bash[51302]: debug 2023-08-28T01:58:58.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:58.147749+0000) 2023-08-28T01:58:58.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:58 smithi130 bash[27579]: cluster 2023-08-28T01:58:56.918028+0000 mgr.y (mgr.14901) 7210 : cluster [DBG] pgmap v7217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:58.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:58 smithi006 bash[33918]: cluster 2023-08-28T01:58:56.918028+0000 mgr.y (mgr.14901) 7210 : cluster [DBG] pgmap v7217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:58.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:58 smithi006 bash[27292]: cluster 2023-08-28T01:58:56.918028+0000 mgr.y (mgr.14901) 7210 : cluster [DBG] pgmap v7217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:58:59.501 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:58:59 smithi130 bash[51302]: debug 2023-08-28T01:58:59.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:59.148063+0000) 2023-08-28T01:58:59.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:58:59 smithi130 bash[27579]: audit 2023-08-28T01:58:58.809371+0000 mon.a (mon.0) 4682 : audit [DBG] from='client.? 172.21.15.6:0/3311768765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:59.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:58:59 smithi006 bash[33918]: audit 2023-08-28T01:58:58.809371+0000 mon.a (mon.0) 4682 : audit [DBG] from='client.? 172.21.15.6:0/3311768765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:59.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:58:59 smithi006 bash[27292]: audit 2023-08-28T01:58:58.809371+0000 mon.a (mon.0) 4682 : audit [DBG] from='client.? 172.21.15.6:0/3311768765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:00.510 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:00 smithi130 bash[51302]: debug 2023-08-28T01:59:00.143+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:00.148344+0000) 2023-08-28T01:59:00.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:00 smithi130 bash[27579]: cluster 2023-08-28T01:58:58.918713+0000 mgr.y (mgr.14901) 7211 : cluster [DBG] pgmap v7218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:00.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:59:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:59:00] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T01:59:00.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:00 smithi006 bash[27292]: cluster 2023-08-28T01:58:58.918713+0000 mgr.y (mgr.14901) 7211 : cluster [DBG] pgmap v7218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:00.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:00 smithi006 bash[33918]: cluster 2023-08-28T01:58:58.918713+0000 mgr.y (mgr.14901) 7211 : cluster [DBG] pgmap v7218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:01 smithi006 bash[56298]: level=error ts=2023-08-28T01:59:01.176Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:01 smithi006 bash[56298]: level=warn ts=2023-08-28T01:59:01.177Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:01.520 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:01 smithi130 bash[51302]: debug 2023-08-28T01:59:01.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:01.148635+0000) 2023-08-28T01:59:01.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:01 smithi130 bash[27579]: audit 2023-08-28T01:59:01.229356+0000 mon.c (mon.2) 2833 : audit [DBG] from='client.? 172.21.15.6:0/3703000424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:01.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:01 smithi006 bash[27292]: audit 2023-08-28T01:59:01.229356+0000 mon.c (mon.2) 2833 : audit [DBG] from='client.? 172.21.15.6:0/3703000424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:01.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:01 smithi006 bash[33918]: audit 2023-08-28T01:59:01.229356+0000 mon.c (mon.2) 2833 : audit [DBG] from='client.? 172.21.15.6:0/3703000424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:02.522 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:02 smithi130 bash[51302]: debug 2023-08-28T01:59:02.143+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:02.149015+0000) 2023-08-28T01:59:02.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:02 smithi130 bash[27579]: cluster 2023-08-28T01:59:00.919513+0000 mgr.y (mgr.14901) 7212 : cluster [DBG] pgmap v7219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:02.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:02 smithi006 bash[33918]: cluster 2023-08-28T01:59:00.919513+0000 mgr.y (mgr.14901) 7212 : cluster [DBG] pgmap v7219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:02.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:02 smithi006 bash[27292]: cluster 2023-08-28T01:59:00.919513+0000 mgr.y (mgr.14901) 7212 : cluster [DBG] pgmap v7219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:02 smithi006 bash[56298]: level=error ts=2023-08-28T01:59:02.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:02 smithi006 bash[56298]: level=warn ts=2023-08-28T01:59:02.742Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:03.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:03 smithi130 bash[51302]: debug 2023-08-28T01:59:03.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:03.149303+0000) 2023-08-28T01:59:04.527 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:04 smithi130 bash[51302]: debug 2023-08-28T01:59:04.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:04.149605+0000) 2023-08-28T01:59:04.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:04 smithi130 bash[27579]: cluster 2023-08-28T01:59:02.920143+0000 mgr.y (mgr.14901) 7213 : cluster [DBG] pgmap v7220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:04.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:04 smithi130 bash[27579]: audit 2023-08-28T01:59:03.644220+0000 mon.a (mon.0) 4683 : audit [DBG] from='client.? 172.21.15.6:0/2261583625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:04.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:04 smithi006 bash[33918]: cluster 2023-08-28T01:59:02.920143+0000 mgr.y (mgr.14901) 7213 : cluster [DBG] pgmap v7220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:04.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:04 smithi006 bash[33918]: audit 2023-08-28T01:59:03.644220+0000 mon.a (mon.0) 4683 : audit [DBG] from='client.? 172.21.15.6:0/2261583625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:04 smithi006 bash[56298]: level=error ts=2023-08-28T01:59:04.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:04 smithi006 bash[56298]: level=warn ts=2023-08-28T01:59:04.623Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:04.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:04 smithi006 bash[27292]: cluster 2023-08-28T01:59:02.920143+0000 mgr.y (mgr.14901) 7213 : cluster [DBG] pgmap v7220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:04.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:04 smithi006 bash[27292]: audit 2023-08-28T01:59:03.644220+0000 mon.a (mon.0) 4683 : audit [DBG] from='client.? 172.21.15.6:0/2261583625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:05.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:05 smithi130 bash[51302]: debug 2023-08-28T01:59:05.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:05.149918+0000) 2023-08-28T01:59:06.532 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:06 smithi130 bash[51302]: debug 2023-08-28T01:59:06.143+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:06.150215+0000) 2023-08-28T01:59:06.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:06 smithi130 bash[27579]: cluster 2023-08-28T01:59:04.921366+0000 mgr.y (mgr.14901) 7214 : cluster [DBG] pgmap v7221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:06.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:06 smithi130 bash[27579]: audit 2023-08-28T01:59:06.057201+0000 mon.a (mon.0) 4684 : audit [DBG] from='client.? 172.21.15.6:0/2291256067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:06.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:06 smithi006 bash[33918]: cluster 2023-08-28T01:59:04.921366+0000 mgr.y (mgr.14901) 7214 : cluster [DBG] pgmap v7221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:06.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:06 smithi006 bash[33918]: audit 2023-08-28T01:59:06.057201+0000 mon.a (mon.0) 4684 : audit [DBG] from='client.? 172.21.15.6:0/2291256067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:06.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:06 smithi006 bash[27292]: cluster 2023-08-28T01:59:04.921366+0000 mgr.y (mgr.14901) 7214 : cluster [DBG] pgmap v7221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:06.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:06 smithi006 bash[27292]: audit 2023-08-28T01:59:06.057201+0000 mon.a (mon.0) 4684 : audit [DBG] from='client.? 172.21.15.6:0/2291256067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:07.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:07 smithi130 bash[51302]: debug 2023-08-28T01:59:07.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:07.150467+0000) 2023-08-28T01:59:08.536 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:08 smithi130 bash[51302]: debug 2023-08-28T01:59:08.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:08.150733+0000) 2023-08-28T01:59:08.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:08 smithi130 bash[27579]: cluster 2023-08-28T01:59:06.922059+0000 mgr.y (mgr.14901) 7215 : cluster [DBG] pgmap v7222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:08.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:08 smithi130 bash[27579]: audit 2023-08-28T01:59:08.471893+0000 mon.c (mon.2) 2834 : audit [DBG] from='client.? 172.21.15.6:0/1086342805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:08.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:08 smithi006 bash[27292]: cluster 2023-08-28T01:59:06.922059+0000 mgr.y (mgr.14901) 7215 : cluster [DBG] pgmap v7222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:08.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:08 smithi006 bash[27292]: audit 2023-08-28T01:59:08.471893+0000 mon.c (mon.2) 2834 : audit [DBG] from='client.? 172.21.15.6:0/1086342805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:08.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:08 smithi006 bash[33918]: cluster 2023-08-28T01:59:06.922059+0000 mgr.y (mgr.14901) 7215 : cluster [DBG] pgmap v7222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:08.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:08 smithi006 bash[33918]: audit 2023-08-28T01:59:08.471893+0000 mon.c (mon.2) 2834 : audit [DBG] from='client.? 172.21.15.6:0/1086342805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:09.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:09 smithi130 bash[51302]: debug 2023-08-28T01:59:09.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:09.151010+0000) 2023-08-28T01:59:10.539 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:10 smithi130 bash[51302]: debug 2023-08-28T01:59:10.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:10.151231+0000) 2023-08-28T01:59:10.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:10 smithi130 bash[27579]: cluster 2023-08-28T01:59:08.922756+0000 mgr.y (mgr.14901) 7216 : cluster [DBG] pgmap v7223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:10.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:10 smithi006 bash[27292]: cluster 2023-08-28T01:59:08.922756+0000 mgr.y (mgr.14901) 7216 : cluster [DBG] pgmap v7223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:10.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:59:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:59:10] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T01:59:10.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:10 smithi006 bash[33918]: cluster 2023-08-28T01:59:08.922756+0000 mgr.y (mgr.14901) 7216 : cluster [DBG] pgmap v7223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:11 smithi006 bash[56298]: level=error ts=2023-08-28T01:59:11.176Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:11 smithi006 bash[56298]: level=warn ts=2023-08-28T01:59:11.178Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:11.549 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:11 smithi130 bash[51302]: debug 2023-08-28T01:59:11.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:11.151479+0000) 2023-08-28T01:59:11.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:11 smithi130 bash[27579]: audit 2023-08-28T01:59:10.887627+0000 mon.a (mon.0) 4685 : audit [DBG] from='client.? 172.21.15.6:0/2032840682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:11.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:11 smithi130 bash[27579]: cluster 2023-08-28T01:59:10.923929+0000 mgr.y (mgr.14901) 7217 : cluster [DBG] pgmap v7224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:11.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:11 smithi006 bash[27292]: audit 2023-08-28T01:59:10.887627+0000 mon.a (mon.0) 4685 : audit [DBG] from='client.? 172.21.15.6:0/2032840682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:11.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:11 smithi006 bash[27292]: cluster 2023-08-28T01:59:10.923929+0000 mgr.y (mgr.14901) 7217 : cluster [DBG] pgmap v7224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:11.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:11 smithi006 bash[33918]: audit 2023-08-28T01:59:10.887627+0000 mon.a (mon.0) 4685 : audit [DBG] from='client.? 172.21.15.6:0/2032840682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:11.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:11 smithi006 bash[33918]: cluster 2023-08-28T01:59:10.923929+0000 mgr.y (mgr.14901) 7217 : cluster [DBG] pgmap v7224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:12.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:12 smithi130 bash[51302]: debug 2023-08-28T01:59:12.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:12.151740+0000) 2023-08-28T01:59:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:12 smithi006 bash[56298]: level=error ts=2023-08-28T01:59:12.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:12 smithi006 bash[56298]: level=warn ts=2023-08-28T01:59:12.744Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:13.506 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:13 smithi130 bash[51302]: debug 2023-08-28T01:59:13.151+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:13.152011+0000) 2023-08-28T01:59:13.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:13 smithi130 bash[27579]: audit 2023-08-28T01:59:13.308802+0000 mon.c (mon.2) 2835 : audit [DBG] from='client.? 172.21.15.6:0/3662040979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:13.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:13 smithi006 bash[33918]: audit 2023-08-28T01:59:13.308802+0000 mon.c (mon.2) 2835 : audit [DBG] from='client.? 172.21.15.6:0/3662040979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:13.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:13 smithi006 bash[27292]: audit 2023-08-28T01:59:13.308802+0000 mon.c (mon.2) 2835 : audit [DBG] from='client.? 172.21.15.6:0/3662040979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:14.509 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:14 smithi130 bash[51302]: debug 2023-08-28T01:59:14.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:14.152234+0000) 2023-08-28T01:59:14.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:14 smithi130 bash[27579]: cluster 2023-08-28T01:59:12.924634+0000 mgr.y (mgr.14901) 7218 : cluster [DBG] pgmap v7225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:14.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:14 smithi006 bash[27292]: cluster 2023-08-28T01:59:12.924634+0000 mgr.y (mgr.14901) 7218 : cluster [DBG] pgmap v7225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:14.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:14 smithi006 bash[33918]: cluster 2023-08-28T01:59:12.924634+0000 mgr.y (mgr.14901) 7218 : cluster [DBG] pgmap v7225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:14 smithi006 bash[56298]: level=error ts=2023-08-28T01:59:14.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:14 smithi006 bash[56298]: level=warn ts=2023-08-28T01:59:14.624Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:15.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:15 smithi130 bash[51302]: debug 2023-08-28T01:59:15.151+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:15.152525+0000) 2023-08-28T01:59:16.514 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:16 smithi130 bash[51302]: debug 2023-08-28T01:59:16.151+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:16.152795+0000) 2023-08-28T01:59:16.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:16 smithi130 bash[27579]: cluster 2023-08-28T01:59:14.925757+0000 mgr.y (mgr.14901) 7219 : cluster [DBG] pgmap v7226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:16.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:16 smithi130 bash[27579]: audit 2023-08-28T01:59:15.728085+0000 mon.a (mon.0) 4686 : audit [DBG] from='client.? 172.21.15.6:0/705648645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:16.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:16 smithi006 bash[27292]: cluster 2023-08-28T01:59:14.925757+0000 mgr.y (mgr.14901) 7219 : cluster [DBG] pgmap v7226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:16.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:16 smithi006 bash[27292]: audit 2023-08-28T01:59:15.728085+0000 mon.a (mon.0) 4686 : audit [DBG] from='client.? 172.21.15.6:0/705648645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:16.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:16 smithi006 bash[33918]: cluster 2023-08-28T01:59:14.925757+0000 mgr.y (mgr.14901) 7219 : cluster [DBG] pgmap v7226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:16.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:16 smithi006 bash[33918]: audit 2023-08-28T01:59:15.728085+0000 mon.a (mon.0) 4686 : audit [DBG] from='client.? 172.21.15.6:0/705648645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:17.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:17 smithi130 bash[51302]: debug 2023-08-28T01:59:17.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:17.153114+0000) 2023-08-28T01:59:18.520 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:18 smithi130 bash[51302]: debug 2023-08-28T01:59:18.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:18.153386+0000) 2023-08-28T01:59:18.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:18 smithi130 bash[27579]: cluster 2023-08-28T01:59:16.926448+0000 mgr.y (mgr.14901) 7220 : cluster [DBG] pgmap v7227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:18.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:18 smithi130 bash[27579]: audit 2023-08-28T01:59:18.142698+0000 mon.c (mon.2) 2836 : audit [DBG] from='client.? 172.21.15.6:0/766557784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:18.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:18 smithi006 bash[33918]: cluster 2023-08-28T01:59:16.926448+0000 mgr.y (mgr.14901) 7220 : cluster [DBG] pgmap v7227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:18.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:18 smithi006 bash[33918]: audit 2023-08-28T01:59:18.142698+0000 mon.c (mon.2) 2836 : audit [DBG] from='client.? 172.21.15.6:0/766557784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:18.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:18 smithi006 bash[27292]: cluster 2023-08-28T01:59:16.926448+0000 mgr.y (mgr.14901) 7220 : cluster [DBG] pgmap v7227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:18.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:18 smithi006 bash[27292]: audit 2023-08-28T01:59:18.142698+0000 mon.c (mon.2) 2836 : audit [DBG] from='client.? 172.21.15.6:0/766557784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:19.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:19 smithi130 bash[51302]: debug 2023-08-28T01:59:19.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:19.153650+0000) 2023-08-28T01:59:20.525 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:20 smithi130 bash[51302]: debug 2023-08-28T01:59:20.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:20.153903+0000) 2023-08-28T01:59:20.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:20 smithi130 bash[27579]: cluster 2023-08-28T01:59:18.927166+0000 mgr.y (mgr.14901) 7221 : cluster [DBG] pgmap v7228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:20.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:20 smithi006 bash[27292]: cluster 2023-08-28T01:59:18.927166+0000 mgr.y (mgr.14901) 7221 : cluster [DBG] pgmap v7228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:20.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:20 smithi006 bash[33918]: cluster 2023-08-28T01:59:18.927166+0000 mgr.y (mgr.14901) 7221 : cluster [DBG] pgmap v7228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:20.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:59:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:59:20] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T01:59:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:21 smithi006 bash[56298]: level=error ts=2023-08-28T01:59:21.177Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:21 smithi006 bash[56298]: level=warn ts=2023-08-28T01:59:21.179Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:21.527 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:21 smithi130 bash[51302]: debug 2023-08-28T01:59:21.151+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:21.154188+0000) 2023-08-28T01:59:21.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:21 smithi130 bash[27579]: audit 2023-08-28T01:59:20.565415+0000 mon.c (mon.2) 2837 : audit [DBG] from='client.? 172.21.15.6:0/798253986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:21.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:21 smithi006 bash[27292]: audit 2023-08-28T01:59:20.565415+0000 mon.c (mon.2) 2837 : audit [DBG] from='client.? 172.21.15.6:0/798253986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:21.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:21 smithi006 bash[33918]: audit 2023-08-28T01:59:20.565415+0000 mon.c (mon.2) 2837 : audit [DBG] from='client.? 172.21.15.6:0/798253986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:22.539 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:22 smithi130 bash[51302]: debug 2023-08-28T01:59:22.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:22.154477+0000) 2023-08-28T01:59:22.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:22 smithi130 bash[27579]: cluster 2023-08-28T01:59:20.928317+0000 mgr.y (mgr.14901) 7222 : cluster [DBG] pgmap v7229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:22.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:22 smithi006 bash[27292]: cluster 2023-08-28T01:59:20.928317+0000 mgr.y (mgr.14901) 7222 : cluster [DBG] pgmap v7229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:22.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:22 smithi006 bash[33918]: cluster 2023-08-28T01:59:20.928317+0000 mgr.y (mgr.14901) 7222 : cluster [DBG] pgmap v7229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:22 smithi006 bash[56298]: level=error ts=2023-08-28T01:59:22.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:22 smithi006 bash[56298]: level=warn ts=2023-08-28T01:59:22.745Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:23.550 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:23 smithi130 bash[51302]: debug 2023-08-28T01:59:23.147+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:23.154755+0000) 2023-08-28T01:59:23.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:23 smithi130 bash[27579]: audit 2023-08-28T01:59:22.990772+0000 mon.c (mon.2) 2838 : audit [DBG] from='client.? 172.21.15.6:0/1937932531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:23.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:23 smithi006 bash[27292]: audit 2023-08-28T01:59:22.990772+0000 mon.c (mon.2) 2838 : audit [DBG] from='client.? 172.21.15.6:0/1937932531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:23.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:23 smithi006 bash[33918]: audit 2023-08-28T01:59:22.990772+0000 mon.c (mon.2) 2838 : audit [DBG] from='client.? 172.21.15.6:0/1937932531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:24.553 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:24 smithi130 bash[51302]: debug 2023-08-28T01:59:24.151+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:24.155035+0000) 2023-08-28T01:59:24.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:24 smithi130 bash[27579]: cluster 2023-08-28T01:59:22.929035+0000 mgr.y (mgr.14901) 7223 : cluster [DBG] pgmap v7230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:24.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:24 smithi006 bash[27292]: cluster 2023-08-28T01:59:22.929035+0000 mgr.y (mgr.14901) 7223 : cluster [DBG] pgmap v7230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:24.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:24 smithi006 bash[33918]: cluster 2023-08-28T01:59:22.929035+0000 mgr.y (mgr.14901) 7223 : cluster [DBG] pgmap v7230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:24 smithi006 bash[56298]: level=error ts=2023-08-28T01:59:24.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:24 smithi006 bash[56298]: level=warn ts=2023-08-28T01:59:24.624Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:25.563 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:25 smithi130 bash[51302]: debug 2023-08-28T01:59:25.151+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:25.155254+0000) 2023-08-28T01:59:25.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:25 smithi130 bash[27579]: cluster 2023-08-28T01:59:24.930172+0000 mgr.y (mgr.14901) 7224 : cluster [DBG] pgmap v7231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:25.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:25 smithi130 bash[27579]: audit 2023-08-28T01:59:25.405026+0000 mon.c (mon.2) 2839 : audit [DBG] from='client.? 172.21.15.6:0/832723906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:25.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:25 smithi006 bash[27292]: cluster 2023-08-28T01:59:24.930172+0000 mgr.y (mgr.14901) 7224 : cluster [DBG] pgmap v7231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:25.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:25 smithi006 bash[27292]: audit 2023-08-28T01:59:25.405026+0000 mon.c (mon.2) 2839 : audit [DBG] from='client.? 172.21.15.6:0/832723906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:25.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:25 smithi006 bash[33918]: cluster 2023-08-28T01:59:24.930172+0000 mgr.y (mgr.14901) 7224 : cluster [DBG] pgmap v7231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:25.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:25 smithi006 bash[33918]: audit 2023-08-28T01:59:25.405026+0000 mon.c (mon.2) 2839 : audit [DBG] from='client.? 172.21.15.6:0/832723906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:26.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:26 smithi130 bash[51302]: debug 2023-08-28T01:59:26.151+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:26.155464+0000) 2023-08-28T01:59:27.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:27 smithi130 bash[51302]: debug 2023-08-28T01:59:27.150+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:27.155687+0000) 2023-08-28T01:59:28.505 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:28 smithi130 bash[51302]: debug 2023-08-28T01:59:28.154+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:28.155988+0000) 2023-08-28T01:59:28.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:28 smithi130 bash[27579]: cluster 2023-08-28T01:59:26.930859+0000 mgr.y (mgr.14901) 7225 : cluster [DBG] pgmap v7232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:28.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:28 smithi130 bash[27579]: audit 2023-08-28T01:59:27.829356+0000 mon.c (mon.2) 2840 : audit [DBG] from='client.? 172.21.15.6:0/2910338298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:28.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:28 smithi006 bash[33918]: cluster 2023-08-28T01:59:26.930859+0000 mgr.y (mgr.14901) 7225 : cluster [DBG] pgmap v7232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:28.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:28 smithi006 bash[33918]: audit 2023-08-28T01:59:27.829356+0000 mon.c (mon.2) 2840 : audit [DBG] from='client.? 172.21.15.6:0/2910338298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:28.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:28 smithi006 bash[27292]: cluster 2023-08-28T01:59:26.930859+0000 mgr.y (mgr.14901) 7225 : cluster [DBG] pgmap v7232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:28.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:28 smithi006 bash[27292]: audit 2023-08-28T01:59:27.829356+0000 mon.c (mon.2) 2840 : audit [DBG] from='client.? 172.21.15.6:0/2910338298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:29.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:29 smithi130 bash[51302]: debug 2023-08-28T01:59:29.154+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:29.156262+0000) 2023-08-28T01:59:30.511 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:30 smithi130 bash[51302]: debug 2023-08-28T01:59:30.150+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:30.156553+0000) 2023-08-28T01:59:30.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:30 smithi130 bash[27579]: cluster 2023-08-28T01:59:28.931535+0000 mgr.y (mgr.14901) 7226 : cluster [DBG] pgmap v7233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:30 smithi130 bash[27579]: audit 2023-08-28T01:59:30.249976+0000 mon.c (mon.2) 2841 : audit [DBG] from='client.? 172.21.15.6:0/672143091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:30.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:59:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:59:30] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:59:30.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:30 smithi006 bash[33918]: cluster 2023-08-28T01:59:28.931535+0000 mgr.y (mgr.14901) 7226 : cluster [DBG] pgmap v7233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:30.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:30 smithi006 bash[33918]: audit 2023-08-28T01:59:30.249976+0000 mon.c (mon.2) 2841 : audit [DBG] from='client.? 172.21.15.6:0/672143091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:30.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:30 smithi006 bash[27292]: cluster 2023-08-28T01:59:28.931535+0000 mgr.y (mgr.14901) 7226 : cluster [DBG] pgmap v7233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:30.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:30 smithi006 bash[27292]: audit 2023-08-28T01:59:30.249976+0000 mon.c (mon.2) 2841 : audit [DBG] from='client.? 172.21.15.6:0/672143091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:31 smithi006 bash[56298]: level=error ts=2023-08-28T01:59:31.178Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:31 smithi006 bash[56298]: level=warn ts=2023-08-28T01:59:31.180Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:31.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:31 smithi130 bash[51302]: debug 2023-08-28T01:59:31.154+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:31.156878+0000) 2023-08-28T01:59:32.514 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:32 smithi130 bash[51302]: debug 2023-08-28T01:59:32.150+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:32.157113+0000) 2023-08-28T01:59:32.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:32 smithi130 bash[27579]: cluster 2023-08-28T01:59:30.932719+0000 mgr.y (mgr.14901) 7227 : cluster [DBG] pgmap v7234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:32.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:32 smithi006 bash[27292]: cluster 2023-08-28T01:59:30.932719+0000 mgr.y (mgr.14901) 7227 : cluster [DBG] pgmap v7234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:32.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:32 smithi006 bash[33918]: cluster 2023-08-28T01:59:30.932719+0000 mgr.y (mgr.14901) 7227 : cluster [DBG] pgmap v7234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:32 smithi006 bash[56298]: level=error ts=2023-08-28T01:59:32.744Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:32 smithi006 bash[56298]: level=warn ts=2023-08-28T01:59:32.746Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:33.523 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:33 smithi130 bash[51302]: debug 2023-08-28T01:59:33.154+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:33.157417+0000) 2023-08-28T01:59:33.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:33 smithi130 bash[27579]: audit 2023-08-28T01:59:32.668134+0000 mon.a (mon.0) 4687 : audit [DBG] from='client.? 172.21.15.6:0/2815462216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:33 smithi006 bash[27292]: audit 2023-08-28T01:59:32.668134+0000 mon.a (mon.0) 4687 : audit [DBG] from='client.? 172.21.15.6:0/2815462216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:33.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:33 smithi006 bash[33918]: audit 2023-08-28T01:59:32.668134+0000 mon.a (mon.0) 4687 : audit [DBG] from='client.? 172.21.15.6:0/2815462216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:34.526 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:34 smithi130 bash[51302]: debug 2023-08-28T01:59:34.154+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:34.157724+0000) 2023-08-28T01:59:34.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:34 smithi130 bash[27579]: cluster 2023-08-28T01:59:32.933588+0000 mgr.y (mgr.14901) 7228 : cluster [DBG] pgmap v7235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:34.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:34 smithi006 bash[33918]: cluster 2023-08-28T01:59:32.933588+0000 mgr.y (mgr.14901) 7228 : cluster [DBG] pgmap v7235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:34.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:34 smithi006 bash[27292]: cluster 2023-08-28T01:59:32.933588+0000 mgr.y (mgr.14901) 7228 : cluster [DBG] pgmap v7235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:34 smithi006 bash[56298]: level=error ts=2023-08-28T01:59:34.623Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:34 smithi006 bash[56298]: level=warn ts=2023-08-28T01:59:34.624Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:35.535 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:35 smithi130 bash[51302]: debug 2023-08-28T01:59:35.154+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:35.158043+0000) 2023-08-28T01:59:35.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:35 smithi130 bash[27579]: audit 2023-08-28T01:59:35.083853+0000 mon.a (mon.0) 4688 : audit [DBG] from='client.? 172.21.15.6:0/4207714414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:35.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:35 smithi006 bash[33918]: audit 2023-08-28T01:59:35.083853+0000 mon.a (mon.0) 4688 : audit [DBG] from='client.? 172.21.15.6:0/4207714414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:35.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:35 smithi006 bash[27292]: audit 2023-08-28T01:59:35.083853+0000 mon.a (mon.0) 4688 : audit [DBG] from='client.? 172.21.15.6:0/4207714414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:36.537 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:36 smithi130 bash[51302]: debug 2023-08-28T01:59:36.154+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:36.158295+0000) 2023-08-28T01:59:36.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:36 smithi130 bash[27579]: cluster 2023-08-28T01:59:34.934859+0000 mgr.y (mgr.14901) 7229 : cluster [DBG] pgmap v7236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:36.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:36 smithi006 bash[27292]: cluster 2023-08-28T01:59:34.934859+0000 mgr.y (mgr.14901) 7229 : cluster [DBG] pgmap v7236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:36.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:36 smithi006 bash[33918]: cluster 2023-08-28T01:59:34.934859+0000 mgr.y (mgr.14901) 7229 : cluster [DBG] pgmap v7236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:37.550 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:37 smithi130 bash[51302]: debug 2023-08-28T01:59:37.154+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:37.158616+0000) 2023-08-28T01:59:37.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:37 smithi130 bash[27579]: audit 2023-08-28T01:59:37.498383+0000 mon.a (mon.0) 4689 : audit [DBG] from='client.? 172.21.15.6:0/2791886398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:37.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:37 smithi006 bash[33918]: audit 2023-08-28T01:59:37.498383+0000 mon.a (mon.0) 4689 : audit [DBG] from='client.? 172.21.15.6:0/2791886398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:37.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:37 smithi006 bash[27292]: audit 2023-08-28T01:59:37.498383+0000 mon.a (mon.0) 4689 : audit [DBG] from='client.? 172.21.15.6:0/2791886398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:38.549 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:38 smithi130 bash[51302]: debug 2023-08-28T01:59:38.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:38.158938+0000) 2023-08-28T01:59:38.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:38 smithi130 bash[27579]: cluster 2023-08-28T01:59:36.935655+0000 mgr.y (mgr.14901) 7230 : cluster [DBG] pgmap v7237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:38.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:38 smithi006 bash[27292]: cluster 2023-08-28T01:59:36.935655+0000 mgr.y (mgr.14901) 7230 : cluster [DBG] pgmap v7237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:38.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:38 smithi006 bash[33918]: cluster 2023-08-28T01:59:36.935655+0000 mgr.y (mgr.14901) 7230 : cluster [DBG] pgmap v7237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:39.558 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:39 smithi130 bash[51302]: debug 2023-08-28T01:59:39.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:39.159225+0000) 2023-08-28T01:59:39.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:39 smithi130 bash[27579]: cluster 2023-08-28T01:59:38.936357+0000 mgr.y (mgr.14901) 7231 : cluster [DBG] pgmap v7238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:39.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:39 smithi006 bash[27292]: cluster 2023-08-28T01:59:38.936357+0000 mgr.y (mgr.14901) 7231 : cluster [DBG] pgmap v7238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:39.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:39 smithi006 bash[33918]: cluster 2023-08-28T01:59:38.936357+0000 mgr.y (mgr.14901) 7231 : cluster [DBG] pgmap v7238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:40.561 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:40 smithi130 bash[51302]: debug 2023-08-28T01:59:40.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:40.159519+0000) 2023-08-28T01:59:40.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:40 smithi130 bash[27579]: audit 2023-08-28T01:59:39.915131+0000 mon.c (mon.2) 2842 : audit [DBG] from='client.? 172.21.15.6:0/2946937271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:40.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:59:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:59:40] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T01:59:40.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:40 smithi006 bash[27292]: audit 2023-08-28T01:59:39.915131+0000 mon.c (mon.2) 2842 : audit [DBG] from='client.? 172.21.15.6:0/2946937271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:40.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:40 smithi006 bash[33918]: audit 2023-08-28T01:59:39.915131+0000 mon.c (mon.2) 2842 : audit [DBG] from='client.? 172.21.15.6:0/2946937271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:41 smithi006 bash[56298]: level=error ts=2023-08-28T01:59:41.179Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:41 smithi006 bash[56298]: level=warn ts=2023-08-28T01:59:41.181Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:41.573 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:41 smithi130 bash[51302]: debug 2023-08-28T01:59:41.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:41.159815+0000) 2023-08-28T01:59:41.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:41 smithi130 bash[27579]: cluster 2023-08-28T01:59:40.937546+0000 mgr.y (mgr.14901) 7232 : cluster [DBG] pgmap v7239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:41.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:41 smithi006 bash[33918]: cluster 2023-08-28T01:59:40.937546+0000 mgr.y (mgr.14901) 7232 : cluster [DBG] pgmap v7239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:41.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:41 smithi006 bash[27292]: cluster 2023-08-28T01:59:40.937546+0000 mgr.y (mgr.14901) 7232 : cluster [DBG] pgmap v7239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:42.576 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:42 smithi130 bash[51302]: debug 2023-08-28T01:59:42.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:42.160051+0000) 2023-08-28T01:59:42.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:42 smithi130 bash[27579]: audit 2023-08-28T01:59:42.328238+0000 mon.a (mon.0) 4690 : audit [DBG] from='client.? 172.21.15.6:0/1750324474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:42.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:42 smithi006 bash[27292]: audit 2023-08-28T01:59:42.328238+0000 mon.a (mon.0) 4690 : audit [DBG] from='client.? 172.21.15.6:0/1750324474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:42.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:42 smithi006 bash[33918]: audit 2023-08-28T01:59:42.328238+0000 mon.a (mon.0) 4690 : audit [DBG] from='client.? 172.21.15.6:0/1750324474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:42 smithi006 bash[56298]: level=error ts=2023-08-28T01:59:42.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:42 smithi006 bash[56298]: level=warn ts=2023-08-28T01:59:42.746Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:43.585 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:43 smithi130 bash[51302]: debug 2023-08-28T01:59:43.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:43.160363+0000) 2023-08-28T01:59:43.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:43 smithi130 bash[27579]: cluster 2023-08-28T01:59:42.938233+0000 mgr.y (mgr.14901) 7233 : cluster [DBG] pgmap v7240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:43.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:43 smithi006 bash[27292]: cluster 2023-08-28T01:59:42.938233+0000 mgr.y (mgr.14901) 7233 : cluster [DBG] pgmap v7240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:43.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:43 smithi006 bash[33918]: cluster 2023-08-28T01:59:42.938233+0000 mgr.y (mgr.14901) 7233 : cluster [DBG] pgmap v7240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:44.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:44 smithi130 bash[51302]: debug 2023-08-28T01:59:44.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:44.160672+0000) 2023-08-28T01:59:44.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:44 smithi006 bash[27292]: audit 2023-08-28T01:59:44.739903+0000 mon.c (mon.2) 2843 : audit [DBG] from='client.? 172.21.15.6:0/1680637389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:44.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:44 smithi006 bash[33918]: audit 2023-08-28T01:59:44.739903+0000 mon.c (mon.2) 2843 : audit [DBG] from='client.? 172.21.15.6:0/1680637389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:44 smithi006 bash[56298]: level=error ts=2023-08-28T01:59:44.623Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:44 smithi006 bash[56298]: level=warn ts=2023-08-28T01:59:44.625Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:45.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:44 smithi130 bash[27579]: audit 2023-08-28T01:59:44.739903+0000 mon.c (mon.2) 2843 : audit [DBG] from='client.? 172.21.15.6:0/1680637389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:45.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:45 smithi130 bash[51302]: debug 2023-08-28T01:59:45.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:45.160955+0000) 2023-08-28T01:59:46.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:45 smithi130 bash[27579]: cluster 2023-08-28T01:59:44.939543+0000 mgr.y (mgr.14901) 7234 : cluster [DBG] pgmap v7241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:46.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:45 smithi006 bash[33918]: cluster 2023-08-28T01:59:44.939543+0000 mgr.y (mgr.14901) 7234 : cluster [DBG] pgmap v7241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:46.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:45 smithi006 bash[27292]: cluster 2023-08-28T01:59:44.939543+0000 mgr.y (mgr.14901) 7234 : cluster [DBG] pgmap v7241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:46.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:46 smithi130 bash[51302]: debug 2023-08-28T01:59:46.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:46.161217+0000) 2023-08-28T01:59:47.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:47 smithi130 bash[51302]: debug 2023-08-28T01:59:47.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:47.161423+0000) 2023-08-28T01:59:48.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:48 smithi130 bash[51302]: debug 2023-08-28T01:59:48.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:48.161720+0000) 2023-08-28T01:59:48.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:48 smithi130 bash[27579]: cluster 2023-08-28T01:59:46.940364+0000 mgr.y (mgr.14901) 7235 : cluster [DBG] pgmap v7242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:48.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:48 smithi130 bash[27579]: audit 2023-08-28T01:59:47.161801+0000 mon.c (mon.2) 2844 : audit [DBG] from='client.? 172.21.15.6:0/773498769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:48.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:47 smithi006 bash[33918]: cluster 2023-08-28T01:59:46.940364+0000 mgr.y (mgr.14901) 7235 : cluster [DBG] pgmap v7242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:48.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:47 smithi006 bash[33918]: audit 2023-08-28T01:59:47.161801+0000 mon.c (mon.2) 2844 : audit [DBG] from='client.? 172.21.15.6:0/773498769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:48.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:47 smithi006 bash[27292]: cluster 2023-08-28T01:59:46.940364+0000 mgr.y (mgr.14901) 7235 : cluster [DBG] pgmap v7242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:48.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:47 smithi006 bash[27292]: audit 2023-08-28T01:59:47.161801+0000 mon.c (mon.2) 2844 : audit [DBG] from='client.? 172.21.15.6:0/773498769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:49.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:49 smithi130 bash[51302]: debug 2023-08-28T01:59:49.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:49.162077+0000) 2023-08-28T01:59:50.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:50 smithi130 bash[27579]: cluster 2023-08-28T01:59:48.941060+0000 mgr.y (mgr.14901) 7236 : cluster [DBG] pgmap v7243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:50.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:50 smithi130 bash[27579]: audit 2023-08-28T01:59:49.581286+0000 mon.a (mon.0) 4691 : audit [DBG] from='client.? 172.21.15.6:0/1290761270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:50.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:50 smithi130 bash[51302]: debug 2023-08-28T01:59:50.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:50.162349+0000) 2023-08-28T01:59:50.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:50 smithi006 bash[33918]: cluster 2023-08-28T01:59:48.941060+0000 mgr.y (mgr.14901) 7236 : cluster [DBG] pgmap v7243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:50.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:50 smithi006 bash[33918]: audit 2023-08-28T01:59:49.581286+0000 mon.a (mon.0) 4691 : audit [DBG] from='client.? 172.21.15.6:0/1290761270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:50.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:50 smithi006 bash[27292]: cluster 2023-08-28T01:59:48.941060+0000 mgr.y (mgr.14901) 7236 : cluster [DBG] pgmap v7243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:50.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:50 smithi006 bash[27292]: audit 2023-08-28T01:59:49.581286+0000 mon.a (mon.0) 4691 : audit [DBG] from='client.? 172.21.15.6:0/1290761270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:51.179 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 01:59:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:01:59:50] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T01:59:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:51 smithi006 bash[56298]: level=error ts=2023-08-28T01:59:51.180Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:51.466 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:51 smithi006 bash[56298]: level=warn ts=2023-08-28T01:59:51.182Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:51.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:51 smithi130 bash[51302]: debug 2023-08-28T01:59:51.158+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:51.162648+0000) 2023-08-28T01:59:52.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:52 smithi130 bash[51302]: debug 2023-08-28T01:59:52.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:52.163014+0000) 2023-08-28T01:59:52.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:52 smithi130 bash[27579]: cluster 2023-08-28T01:59:50.942184+0000 mgr.y (mgr.14901) 7237 : cluster [DBG] pgmap v7244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:52.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:52 smithi130 bash[27579]: audit 2023-08-28T01:59:51.992546+0000 mon.a (mon.0) 4692 : audit [DBG] from='client.? 172.21.15.6:0/2262724330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:52.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:52 smithi006 bash[33918]: cluster 2023-08-28T01:59:50.942184+0000 mgr.y (mgr.14901) 7237 : cluster [DBG] pgmap v7244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:52.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:52 smithi006 bash[33918]: audit 2023-08-28T01:59:51.992546+0000 mon.a (mon.0) 4692 : audit [DBG] from='client.? 172.21.15.6:0/2262724330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:52.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:52 smithi006 bash[27292]: cluster 2023-08-28T01:59:50.942184+0000 mgr.y (mgr.14901) 7237 : cluster [DBG] pgmap v7244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:52.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:52 smithi006 bash[27292]: audit 2023-08-28T01:59:51.992546+0000 mon.a (mon.0) 4692 : audit [DBG] from='client.? 172.21.15.6:0/2262724330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:52 smithi006 bash[56298]: level=error ts=2023-08-28T01:59:52.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:52 smithi006 bash[56298]: level=warn ts=2023-08-28T01:59:52.746Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:53.516 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:53 smithi130 bash[51302]: debug 2023-08-28T01:59:53.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:53.163297+0000) 2023-08-28T01:59:53.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:53 smithi130 bash[27579]: audit 2023-08-28T01:59:53.286049+0000 mon.b (mon.1) 806 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:59:53.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:53 smithi006 bash[33918]: audit 2023-08-28T01:59:53.286049+0000 mon.b (mon.1) 806 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:59:53.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:53 smithi006 bash[27292]: audit 2023-08-28T01:59:53.286049+0000 mon.b (mon.1) 806 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:59:54.518 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:54 smithi130 bash[51302]: debug 2023-08-28T01:59:54.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:54.163523+0000) 2023-08-28T01:59:54.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:54 smithi130 bash[27579]: cluster 2023-08-28T01:59:52.942853+0000 mgr.y (mgr.14901) 7238 : cluster [DBG] pgmap v7245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:54.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:54 smithi130 bash[27579]: audit 2023-08-28T01:59:54.412063+0000 mon.c (mon.2) 2845 : audit [DBG] from='client.? 172.21.15.6:0/2690774664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:54.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:54 smithi006 bash[33918]: cluster 2023-08-28T01:59:52.942853+0000 mgr.y (mgr.14901) 7238 : cluster [DBG] pgmap v7245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:54.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:54 smithi006 bash[33918]: audit 2023-08-28T01:59:54.412063+0000 mon.c (mon.2) 2845 : audit [DBG] from='client.? 172.21.15.6:0/2690774664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:54 smithi006 bash[56298]: level=error ts=2023-08-28T01:59:54.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 01:59:54 smithi006 bash[56298]: level=warn ts=2023-08-28T01:59:54.626Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T01:59:54.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:54 smithi006 bash[27292]: cluster 2023-08-28T01:59:52.942853+0000 mgr.y (mgr.14901) 7238 : cluster [DBG] pgmap v7245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:54.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:54 smithi006 bash[27292]: audit 2023-08-28T01:59:54.412063+0000 mon.c (mon.2) 2845 : audit [DBG] from='client.? 172.21.15.6:0/2690774664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:55.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:55 smithi130 bash[51302]: debug 2023-08-28T01:59:55.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:55.163803+0000) 2023-08-28T01:59:56.525 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:56 smithi130 bash[51302]: debug 2023-08-28T01:59:56.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:56.164113+0000) 2023-08-28T01:59:56.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:56 smithi130 bash[27579]: cluster 2023-08-28T01:59:54.943993+0000 mgr.y (mgr.14901) 7239 : cluster [DBG] pgmap v7246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:56.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:56 smithi006 bash[33918]: cluster 2023-08-28T01:59:54.943993+0000 mgr.y (mgr.14901) 7239 : cluster [DBG] pgmap v7246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:56.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:56 smithi006 bash[27292]: cluster 2023-08-28T01:59:54.943993+0000 mgr.y (mgr.14901) 7239 : cluster [DBG] pgmap v7246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:57.529 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:57 smithi130 bash[51302]: debug 2023-08-28T01:59:57.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:57.164408+0000) 2023-08-28T01:59:57.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:57 smithi130 bash[27579]: audit 2023-08-28T01:59:56.838949+0000 mon.c (mon.2) 2846 : audit [DBG] from='client.? 172.21.15.6:0/2533860064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:57.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:57 smithi006 bash[27292]: audit 2023-08-28T01:59:56.838949+0000 mon.c (mon.2) 2846 : audit [DBG] from='client.? 172.21.15.6:0/2533860064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:57.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:57 smithi006 bash[33918]: audit 2023-08-28T01:59:56.838949+0000 mon.c (mon.2) 2846 : audit [DBG] from='client.? 172.21.15.6:0/2533860064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:58.535 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:58 smithi130 bash[51302]: debug 2023-08-28T01:59:58.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:58.164653+0000) 2023-08-28T01:59:58.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 01:59:58 smithi130 bash[27579]: cluster 2023-08-28T01:59:56.944737+0000 mgr.y (mgr.14901) 7240 : cluster [DBG] pgmap v7247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:58.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 01:59:58 smithi006 bash[27292]: cluster 2023-08-28T01:59:56.944737+0000 mgr.y (mgr.14901) 7240 : cluster [DBG] pgmap v7247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:58.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 01:59:58 smithi006 bash[33918]: cluster 2023-08-28T01:59:56.944737+0000 mgr.y (mgr.14901) 7240 : cluster [DBG] pgmap v7247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T01:59:59.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 01:59:59 smithi130 bash[51302]: debug 2023-08-28T01:59:59.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:59.164890+0000) 2023-08-28T02:00:00.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:00 smithi130 bash[51302]: debug 2023-08-28T02:00:00.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:00.165136+0000) 2023-08-28T02:00:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:00 smithi130 bash[27579]: cluster 2023-08-28T01:59:58.945216+0000 mgr.y (mgr.14901) 7241 : cluster [DBG] pgmap v7248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:00 smithi130 bash[27579]: audit 2023-08-28T01:59:59.227233+0000 mon.a (mon.0) 4693 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:00:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:00 smithi130 bash[27579]: audit 2023-08-28T01:59:59.296081+0000 mon.a (mon.0) 4694 : audit [DBG] from='client.? 172.21.15.6:0/332450664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:00 smithi130 bash[27579]: audit 2023-08-28T01:59:59.604928+0000 mon.a (mon.0) 4695 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:00:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:00 smithi130 bash[27579]: audit 2023-08-28T01:59:59.895227+0000 mon.b (mon.1) 807 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:00:00.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:00 smithi130 bash[27579]: audit 2023-08-28T01:59:59.897619+0000 mon.b (mon.1) 808 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:00:00.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:00 smithi130 bash[27579]: audit 2023-08-28T01:59:59.908013+0000 mon.a (mon.0) 4696 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:00:00.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:00 smithi130 bash[27579]: cluster 2023-08-28T02:00:00.000133+0000 mon.a (mon.0) 4697 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:00:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:00 smithi006 bash[27292]: cluster 2023-08-28T01:59:58.945216+0000 mgr.y (mgr.14901) 7241 : cluster [DBG] pgmap v7248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:00 smithi006 bash[27292]: audit 2023-08-28T01:59:59.227233+0000 mon.a (mon.0) 4693 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:00:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:00 smithi006 bash[27292]: audit 2023-08-28T01:59:59.296081+0000 mon.a (mon.0) 4694 : audit [DBG] from='client.? 172.21.15.6:0/332450664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:00 smithi006 bash[27292]: audit 2023-08-28T01:59:59.604928+0000 mon.a (mon.0) 4695 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:00:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:00 smithi006 bash[27292]: audit 2023-08-28T01:59:59.895227+0000 mon.b (mon.1) 807 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:00:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:00 smithi006 bash[27292]: audit 2023-08-28T01:59:59.897619+0000 mon.b (mon.1) 808 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:00:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:00 smithi006 bash[27292]: audit 2023-08-28T01:59:59.908013+0000 mon.a (mon.0) 4696 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:00:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:00 smithi006 bash[27292]: cluster 2023-08-28T02:00:00.000133+0000 mon.a (mon.0) 4697 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:00:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:00 smithi006 bash[33918]: cluster 2023-08-28T01:59:58.945216+0000 mgr.y (mgr.14901) 7241 : cluster [DBG] pgmap v7248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:00 smithi006 bash[33918]: audit 2023-08-28T01:59:59.227233+0000 mon.a (mon.0) 4693 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:00:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:00 smithi006 bash[33918]: audit 2023-08-28T01:59:59.296081+0000 mon.a (mon.0) 4694 : audit [DBG] from='client.? 172.21.15.6:0/332450664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:00 smithi006 bash[33918]: audit 2023-08-28T01:59:59.604928+0000 mon.a (mon.0) 4695 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:00:00.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:00 smithi006 bash[33918]: audit 2023-08-28T01:59:59.895227+0000 mon.b (mon.1) 807 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:00:00.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:00 smithi006 bash[33918]: audit 2023-08-28T01:59:59.897619+0000 mon.b (mon.1) 808 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:00:00.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:00 smithi006 bash[33918]: audit 2023-08-28T01:59:59.908013+0000 mon.a (mon.0) 4696 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:00:00.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:00 smithi006 bash[33918]: cluster 2023-08-28T02:00:00.000133+0000 mon.a (mon.0) 4697 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:00:01.180 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:00:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:00:00] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:00:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:00:01.180Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:00:01.183Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:01.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:01 smithi130 bash[51302]: debug 2023-08-28T02:00:01.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:01.165482+0000) 2023-08-28T02:00:02.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:02 smithi130 bash[51302]: debug 2023-08-28T02:00:02.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:02.165768+0000) 2023-08-28T02:00:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:02 smithi130 bash[27579]: cluster 2023-08-28T02:00:00.946435+0000 mgr.y (mgr.14901) 7242 : cluster [DBG] pgmap v7249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:02 smithi130 bash[27579]: audit 2023-08-28T02:00:01.712706+0000 mon.c (mon.2) 2847 : audit [DBG] from='client.? 172.21.15.6:0/2276447680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:02.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:02 smithi006 bash[27292]: cluster 2023-08-28T02:00:00.946435+0000 mgr.y (mgr.14901) 7242 : cluster [DBG] pgmap v7249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:02.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:02 smithi006 bash[27292]: audit 2023-08-28T02:00:01.712706+0000 mon.c (mon.2) 2847 : audit [DBG] from='client.? 172.21.15.6:0/2276447680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:02.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:02 smithi006 bash[33918]: cluster 2023-08-28T02:00:00.946435+0000 mgr.y (mgr.14901) 7242 : cluster [DBG] pgmap v7249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:02.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:02 smithi006 bash[33918]: audit 2023-08-28T02:00:01.712706+0000 mon.c (mon.2) 2847 : audit [DBG] from='client.? 172.21.15.6:0/2276447680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:00:02.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:00:02.747Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:03.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:03 smithi130 bash[51302]: debug 2023-08-28T02:00:03.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:03.166045+0000) 2023-08-28T02:00:04.520 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:04 smithi130 bash[51302]: debug 2023-08-28T02:00:04.162+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:04.166323+0000) 2023-08-28T02:00:04.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:04 smithi130 bash[27579]: cluster 2023-08-28T02:00:02.947100+0000 mgr.y (mgr.14901) 7243 : cluster [DBG] pgmap v7250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:04.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:04 smithi130 bash[27579]: audit 2023-08-28T02:00:04.130507+0000 mon.a (mon.0) 4698 : audit [DBG] from='client.? 172.21.15.6:0/164261355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:00:04.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:00:04.627Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:04.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:04 smithi006 bash[27292]: cluster 2023-08-28T02:00:02.947100+0000 mgr.y (mgr.14901) 7243 : cluster [DBG] pgmap v7250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:04.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:04 smithi006 bash[27292]: audit 2023-08-28T02:00:04.130507+0000 mon.a (mon.0) 4698 : audit [DBG] from='client.? 172.21.15.6:0/164261355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:04.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:04 smithi006 bash[33918]: cluster 2023-08-28T02:00:02.947100+0000 mgr.y (mgr.14901) 7243 : cluster [DBG] pgmap v7250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:04.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:04 smithi006 bash[33918]: audit 2023-08-28T02:00:04.130507+0000 mon.a (mon.0) 4698 : audit [DBG] from='client.? 172.21.15.6:0/164261355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:05.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:05 smithi130 bash[51302]: debug 2023-08-28T02:00:05.166+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:05.166604+0000) 2023-08-28T02:00:06.525 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:06 smithi130 bash[51302]: debug 2023-08-28T02:00:06.166+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:06.166906+0000) 2023-08-28T02:00:06.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:06 smithi130 bash[27579]: cluster 2023-08-28T02:00:04.948210+0000 mgr.y (mgr.14901) 7244 : cluster [DBG] pgmap v7251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:06.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:06 smithi006 bash[33918]: cluster 2023-08-28T02:00:04.948210+0000 mgr.y (mgr.14901) 7244 : cluster [DBG] pgmap v7251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:06.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:06 smithi006 bash[27292]: cluster 2023-08-28T02:00:04.948210+0000 mgr.y (mgr.14901) 7244 : cluster [DBG] pgmap v7251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:07.530 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:07 smithi130 bash[51302]: debug 2023-08-28T02:00:07.166+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:07.167144+0000) 2023-08-28T02:00:07.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:07 smithi130 bash[27579]: audit 2023-08-28T02:00:06.552791+0000 mon.a (mon.0) 4699 : audit [DBG] from='client.? 172.21.15.6:0/1507197164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:07.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:07 smithi006 bash[33918]: audit 2023-08-28T02:00:06.552791+0000 mon.a (mon.0) 4699 : audit [DBG] from='client.? 172.21.15.6:0/1507197164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:07.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:07 smithi006 bash[27292]: audit 2023-08-28T02:00:06.552791+0000 mon.a (mon.0) 4699 : audit [DBG] from='client.? 172.21.15.6:0/1507197164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:08.535 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:08 smithi130 bash[51302]: debug 2023-08-28T02:00:08.166+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:08.167379+0000) 2023-08-28T02:00:08.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:08 smithi130 bash[27579]: cluster 2023-08-28T02:00:06.949014+0000 mgr.y (mgr.14901) 7245 : cluster [DBG] pgmap v7252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:08.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:08 smithi006 bash[33918]: cluster 2023-08-28T02:00:06.949014+0000 mgr.y (mgr.14901) 7245 : cluster [DBG] pgmap v7252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:08.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:08 smithi006 bash[27292]: cluster 2023-08-28T02:00:06.949014+0000 mgr.y (mgr.14901) 7245 : cluster [DBG] pgmap v7252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:09.537 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:09 smithi130 bash[51302]: debug 2023-08-28T02:00:09.166+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:09.167692+0000) 2023-08-28T02:00:09.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:09 smithi130 bash[27579]: audit 2023-08-28T02:00:08.960636+0000 mon.c (mon.2) 2848 : audit [DBG] from='client.? 172.21.15.6:0/1577713472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:09.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:09 smithi006 bash[27292]: audit 2023-08-28T02:00:08.960636+0000 mon.c (mon.2) 2848 : audit [DBG] from='client.? 172.21.15.6:0/1577713472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:09.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:09 smithi006 bash[33918]: audit 2023-08-28T02:00:08.960636+0000 mon.c (mon.2) 2848 : audit [DBG] from='client.? 172.21.15.6:0/1577713472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:10.546 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:10 smithi130 bash[51302]: debug 2023-08-28T02:00:10.166+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:10.167966+0000) 2023-08-28T02:00:10.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:10 smithi130 bash[27579]: cluster 2023-08-28T02:00:08.949845+0000 mgr.y (mgr.14901) 7246 : cluster [DBG] pgmap v7253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:10.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:10 smithi006 bash[27292]: cluster 2023-08-28T02:00:08.949845+0000 mgr.y (mgr.14901) 7246 : cluster [DBG] pgmap v7253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:10.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:00:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:00:10] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:00:10.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:10 smithi006 bash[33918]: cluster 2023-08-28T02:00:08.949845+0000 mgr.y (mgr.14901) 7246 : cluster [DBG] pgmap v7253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:00:11.181Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:00:11.183Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:11.548 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:11 smithi130 bash[51302]: debug 2023-08-28T02:00:11.166+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:11.168282+0000) 2023-08-28T02:00:11.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:11 smithi130 bash[27579]: audit 2023-08-28T02:00:11.384279+0000 mon.c (mon.2) 2849 : audit [DBG] from='client.? 172.21.15.6:0/1441599413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:11.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:11 smithi006 bash[27292]: audit 2023-08-28T02:00:11.384279+0000 mon.c (mon.2) 2849 : audit [DBG] from='client.? 172.21.15.6:0/1441599413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:11.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:11 smithi006 bash[33918]: audit 2023-08-28T02:00:11.384279+0000 mon.c (mon.2) 2849 : audit [DBG] from='client.? 172.21.15.6:0/1441599413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:12.552 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:12 smithi130 bash[51302]: debug 2023-08-28T02:00:12.166+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:12.168558+0000) 2023-08-28T02:00:12.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:12 smithi130 bash[27579]: cluster 2023-08-28T02:00:10.950971+0000 mgr.y (mgr.14901) 7247 : cluster [DBG] pgmap v7254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:12.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:12 smithi006 bash[27292]: cluster 2023-08-28T02:00:10.950971+0000 mgr.y (mgr.14901) 7247 : cluster [DBG] pgmap v7254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:12.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:12 smithi006 bash[33918]: cluster 2023-08-28T02:00:10.950971+0000 mgr.y (mgr.14901) 7247 : cluster [DBG] pgmap v7254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:00:12.746Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:00:12.748Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:13.558 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:13 smithi130 bash[51302]: debug 2023-08-28T02:00:13.166+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:13.168873+0000) 2023-08-28T02:00:13.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:13 smithi130 bash[27579]: cluster 2023-08-28T02:00:12.951625+0000 mgr.y (mgr.14901) 7248 : cluster [DBG] pgmap v7255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:13.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:13 smithi006 bash[33918]: cluster 2023-08-28T02:00:12.951625+0000 mgr.y (mgr.14901) 7248 : cluster [DBG] pgmap v7255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:13.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:13 smithi006 bash[27292]: cluster 2023-08-28T02:00:12.951625+0000 mgr.y (mgr.14901) 7248 : cluster [DBG] pgmap v7255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:14.560 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:14 smithi130 bash[51302]: debug 2023-08-28T02:00:14.166+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:14.169166+0000) 2023-08-28T02:00:14.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:14 smithi130 bash[27579]: audit 2023-08-28T02:00:13.836507+0000 mon.a (mon.0) 4700 : audit [DBG] from='client.? 172.21.15.6:0/2137696801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:14.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:14 smithi006 bash[27292]: audit 2023-08-28T02:00:13.836507+0000 mon.a (mon.0) 4700 : audit [DBG] from='client.? 172.21.15.6:0/2137696801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:14.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:14 smithi006 bash[33918]: audit 2023-08-28T02:00:13.836507+0000 mon.a (mon.0) 4700 : audit [DBG] from='client.? 172.21.15.6:0/2137696801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:00:14.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:00:14.627Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:15.569 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:15 smithi130 bash[51302]: debug 2023-08-28T02:00:15.166+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:15.169393+0000) 2023-08-28T02:00:15.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:15 smithi130 bash[27579]: cluster 2023-08-28T02:00:14.952749+0000 mgr.y (mgr.14901) 7249 : cluster [DBG] pgmap v7256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:15.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:15 smithi006 bash[33918]: cluster 2023-08-28T02:00:14.952749+0000 mgr.y (mgr.14901) 7249 : cluster [DBG] pgmap v7256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:15.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:15 smithi006 bash[27292]: cluster 2023-08-28T02:00:14.952749+0000 mgr.y (mgr.14901) 7249 : cluster [DBG] pgmap v7256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:16.571 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:16 smithi130 bash[51302]: debug 2023-08-28T02:00:16.166+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:16.169669+0000) 2023-08-28T02:00:16.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:16 smithi130 bash[27579]: audit 2023-08-28T02:00:16.245952+0000 mon.a (mon.0) 4701 : audit [DBG] from='client.? 172.21.15.6:0/4080820332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:16.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:16 smithi006 bash[27292]: audit 2023-08-28T02:00:16.245952+0000 mon.a (mon.0) 4701 : audit [DBG] from='client.? 172.21.15.6:0/4080820332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:16.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:16 smithi006 bash[33918]: audit 2023-08-28T02:00:16.245952+0000 mon.a (mon.0) 4701 : audit [DBG] from='client.? 172.21.15.6:0/4080820332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:17.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:17 smithi130 bash[51302]: debug 2023-08-28T02:00:17.166+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:17.169987+0000) 2023-08-28T02:00:18.519 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:18 smithi130 bash[51302]: debug 2023-08-28T02:00:18.170+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:18.170265+0000) 2023-08-28T02:00:18.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:18 smithi130 bash[27579]: cluster 2023-08-28T02:00:16.953400+0000 mgr.y (mgr.14901) 7250 : cluster [DBG] pgmap v7257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:18.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:18 smithi006 bash[33918]: cluster 2023-08-28T02:00:16.953400+0000 mgr.y (mgr.14901) 7250 : cluster [DBG] pgmap v7257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:18.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:18 smithi006 bash[27292]: cluster 2023-08-28T02:00:16.953400+0000 mgr.y (mgr.14901) 7250 : cluster [DBG] pgmap v7257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:19.530 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:19 smithi130 bash[51302]: debug 2023-08-28T02:00:19.170+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:19.170633+0000) 2023-08-28T02:00:19.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:19 smithi130 bash[27579]: audit 2023-08-28T02:00:18.664827+0000 mon.a (mon.0) 4702 : audit [DBG] from='client.? 172.21.15.6:0/874994330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:19.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:19 smithi006 bash[27292]: audit 2023-08-28T02:00:18.664827+0000 mon.a (mon.0) 4702 : audit [DBG] from='client.? 172.21.15.6:0/874994330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:19.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:19 smithi006 bash[33918]: audit 2023-08-28T02:00:18.664827+0000 mon.a (mon.0) 4702 : audit [DBG] from='client.? 172.21.15.6:0/874994330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:20.535 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:20 smithi130 bash[51302]: debug 2023-08-28T02:00:20.170+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:20.170923+0000) 2023-08-28T02:00:20.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:20 smithi130 bash[27579]: cluster 2023-08-28T02:00:18.954130+0000 mgr.y (mgr.14901) 7251 : cluster [DBG] pgmap v7258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:20.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:20 smithi006 bash[27292]: cluster 2023-08-28T02:00:18.954130+0000 mgr.y (mgr.14901) 7251 : cluster [DBG] pgmap v7258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:20.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:20 smithi006 bash[33918]: cluster 2023-08-28T02:00:18.954130+0000 mgr.y (mgr.14901) 7251 : cluster [DBG] pgmap v7258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:20.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:00:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:00:20] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:00:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:00:21.182Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:00:21.184Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:21.544 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:21 smithi130 bash[51302]: debug 2023-08-28T02:00:21.170+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:21.171173+0000) 2023-08-28T02:00:21.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:21 smithi130 bash[27579]: audit 2023-08-28T02:00:21.081855+0000 mon.a (mon.0) 4703 : audit [DBG] from='client.? 172.21.15.6:0/3856612220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:21.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:21 smithi006 bash[27292]: audit 2023-08-28T02:00:21.081855+0000 mon.a (mon.0) 4703 : audit [DBG] from='client.? 172.21.15.6:0/3856612220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:21.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:21 smithi006 bash[33918]: audit 2023-08-28T02:00:21.081855+0000 mon.a (mon.0) 4703 : audit [DBG] from='client.? 172.21.15.6:0/3856612220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:22.544 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:22 smithi130 bash[51302]: debug 2023-08-28T02:00:22.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:22.171478+0000) 2023-08-28T02:00:22.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:22 smithi130 bash[27579]: cluster 2023-08-28T02:00:20.955228+0000 mgr.y (mgr.14901) 7252 : cluster [DBG] pgmap v7259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:22.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:22 smithi006 bash[27292]: cluster 2023-08-28T02:00:20.955228+0000 mgr.y (mgr.14901) 7252 : cluster [DBG] pgmap v7259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:22.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:22 smithi006 bash[33918]: cluster 2023-08-28T02:00:20.955228+0000 mgr.y (mgr.14901) 7252 : cluster [DBG] pgmap v7259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:00:22.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:00:22.749Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:23.554 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:23 smithi130 bash[51302]: debug 2023-08-28T02:00:23.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:23.171800+0000) 2023-08-28T02:00:23.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:23 smithi130 bash[27579]: audit 2023-08-28T02:00:23.497195+0000 mon.a (mon.0) 4704 : audit [DBG] from='client.? 172.21.15.6:0/2763041868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:23.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:23 smithi006 bash[27292]: audit 2023-08-28T02:00:23.497195+0000 mon.a (mon.0) 4704 : audit [DBG] from='client.? 172.21.15.6:0/2763041868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:23.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:23 smithi006 bash[33918]: audit 2023-08-28T02:00:23.497195+0000 mon.a (mon.0) 4704 : audit [DBG] from='client.? 172.21.15.6:0/2763041868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:24.557 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:24 smithi130 bash[51302]: debug 2023-08-28T02:00:24.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:24.172106+0000) 2023-08-28T02:00:24.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:24 smithi130 bash[27579]: cluster 2023-08-28T02:00:22.955901+0000 mgr.y (mgr.14901) 7253 : cluster [DBG] pgmap v7260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:24.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:24 smithi006 bash[33918]: cluster 2023-08-28T02:00:22.955901+0000 mgr.y (mgr.14901) 7253 : cluster [DBG] pgmap v7260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:00:24.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:00:24.628Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:24.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:24 smithi006 bash[27292]: cluster 2023-08-28T02:00:22.955901+0000 mgr.y (mgr.14901) 7253 : cluster [DBG] pgmap v7260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:25.564 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:25 smithi130 bash[51302]: debug 2023-08-28T02:00:25.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:25.172376+0000) 2023-08-28T02:00:25.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:25 smithi130 bash[27579]: cluster 2023-08-28T02:00:24.957072+0000 mgr.y (mgr.14901) 7254 : cluster [DBG] pgmap v7261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:25.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:25 smithi006 bash[27292]: cluster 2023-08-28T02:00:24.957072+0000 mgr.y (mgr.14901) 7254 : cluster [DBG] pgmap v7261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:25.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:25 smithi006 bash[33918]: cluster 2023-08-28T02:00:24.957072+0000 mgr.y (mgr.14901) 7254 : cluster [DBG] pgmap v7261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:26.567 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:26 smithi130 bash[51302]: debug 2023-08-28T02:00:26.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:26.172649+0000) 2023-08-28T02:00:26.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:26 smithi130 bash[27579]: audit 2023-08-28T02:00:25.907594+0000 mon.c (mon.2) 2850 : audit [DBG] from='client.? 172.21.15.6:0/3464430666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:26.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:26 smithi006 bash[27292]: audit 2023-08-28T02:00:25.907594+0000 mon.c (mon.2) 2850 : audit [DBG] from='client.? 172.21.15.6:0/3464430666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:26.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:26 smithi006 bash[33918]: audit 2023-08-28T02:00:25.907594+0000 mon.c (mon.2) 2850 : audit [DBG] from='client.? 172.21.15.6:0/3464430666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:27.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:27 smithi130 bash[51302]: debug 2023-08-28T02:00:27.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:27.172982+0000) 2023-08-28T02:00:28.522 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:28 smithi130 bash[51302]: debug 2023-08-28T02:00:28.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:28.173256+0000) 2023-08-28T02:00:28.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:28 smithi130 bash[27579]: cluster 2023-08-28T02:00:26.957773+0000 mgr.y (mgr.14901) 7255 : cluster [DBG] pgmap v7262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:28.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:28 smithi130 bash[27579]: audit 2023-08-28T02:00:28.323159+0000 mon.a (mon.0) 4705 : audit [DBG] from='client.? 172.21.15.6:0/3621135863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:28.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:28 smithi006 bash[27292]: cluster 2023-08-28T02:00:26.957773+0000 mgr.y (mgr.14901) 7255 : cluster [DBG] pgmap v7262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:28.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:28 smithi006 bash[27292]: audit 2023-08-28T02:00:28.323159+0000 mon.a (mon.0) 4705 : audit [DBG] from='client.? 172.21.15.6:0/3621135863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:28.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:28 smithi006 bash[33918]: cluster 2023-08-28T02:00:26.957773+0000 mgr.y (mgr.14901) 7255 : cluster [DBG] pgmap v7262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:28.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:28 smithi006 bash[33918]: audit 2023-08-28T02:00:28.323159+0000 mon.a (mon.0) 4705 : audit [DBG] from='client.? 172.21.15.6:0/3621135863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:29.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:29 smithi130 bash[51302]: debug 2023-08-28T02:00:29.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:29.173563+0000) 2023-08-28T02:00:30.543 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:30 smithi130 bash[51302]: debug 2023-08-28T02:00:30.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:30.173841+0000) 2023-08-28T02:00:30.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:30 smithi130 bash[27579]: cluster 2023-08-28T02:00:28.958470+0000 mgr.y (mgr.14901) 7256 : cluster [DBG] pgmap v7263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:30.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:30 smithi006 bash[27292]: cluster 2023-08-28T02:00:28.958470+0000 mgr.y (mgr.14901) 7256 : cluster [DBG] pgmap v7263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:30.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:00:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:00:30] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T02:00:30.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:30 smithi006 bash[33918]: cluster 2023-08-28T02:00:28.958470+0000 mgr.y (mgr.14901) 7256 : cluster [DBG] pgmap v7263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:00:31.183Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:00:31.185Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:31.530 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:31 smithi130 bash[51302]: debug 2023-08-28T02:00:31.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:31.174139+0000) 2023-08-28T02:00:31.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:31 smithi130 bash[27579]: audit 2023-08-28T02:00:30.745797+0000 mon.c (mon.2) 2851 : audit [DBG] from='client.? 172.21.15.6:0/1812097464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:31.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:31 smithi006 bash[27292]: audit 2023-08-28T02:00:30.745797+0000 mon.c (mon.2) 2851 : audit [DBG] from='client.? 172.21.15.6:0/1812097464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:31.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:31 smithi006 bash[33918]: audit 2023-08-28T02:00:30.745797+0000 mon.c (mon.2) 2851 : audit [DBG] from='client.? 172.21.15.6:0/1812097464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:32.538 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:32 smithi130 bash[51302]: debug 2023-08-28T02:00:32.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:32.174381+0000) 2023-08-28T02:00:32.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:32 smithi130 bash[27579]: cluster 2023-08-28T02:00:30.959622+0000 mgr.y (mgr.14901) 7257 : cluster [DBG] pgmap v7264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:32.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:32 smithi006 bash[27292]: cluster 2023-08-28T02:00:30.959622+0000 mgr.y (mgr.14901) 7257 : cluster [DBG] pgmap v7264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:32.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:32 smithi006 bash[33918]: cluster 2023-08-28T02:00:30.959622+0000 mgr.y (mgr.14901) 7257 : cluster [DBG] pgmap v7264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:00:32.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:00:32.749Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:33.545 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:33 smithi130 bash[51302]: debug 2023-08-28T02:00:33.169+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:33.174654+0000) 2023-08-28T02:00:33.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:33 smithi130 bash[27579]: audit 2023-08-28T02:00:33.161101+0000 mon.a (mon.0) 4706 : audit [DBG] from='client.? 172.21.15.6:0/2703931526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:33 smithi006 bash[27292]: audit 2023-08-28T02:00:33.161101+0000 mon.a (mon.0) 4706 : audit [DBG] from='client.? 172.21.15.6:0/2703931526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:33.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:33 smithi006 bash[33918]: audit 2023-08-28T02:00:33.161101+0000 mon.a (mon.0) 4706 : audit [DBG] from='client.? 172.21.15.6:0/2703931526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:34.548 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:34 smithi130 bash[51302]: debug 2023-08-28T02:00:34.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:34.174977+0000) 2023-08-28T02:00:34.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:34 smithi130 bash[27579]: cluster 2023-08-28T02:00:32.960403+0000 mgr.y (mgr.14901) 7258 : cluster [DBG] pgmap v7265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:34.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:34 smithi006 bash[33918]: cluster 2023-08-28T02:00:32.960403+0000 mgr.y (mgr.14901) 7258 : cluster [DBG] pgmap v7265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:34.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:34 smithi006 bash[27292]: cluster 2023-08-28T02:00:32.960403+0000 mgr.y (mgr.14901) 7258 : cluster [DBG] pgmap v7265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:00:34.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:00:34.629Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:35.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:35 smithi130 bash[51302]: debug 2023-08-28T02:00:35.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:35.175191+0000) 2023-08-28T02:00:36.552 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:36 smithi130 bash[51302]: debug 2023-08-28T02:00:36.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:36.175505+0000) 2023-08-28T02:00:36.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:36 smithi130 bash[27579]: cluster 2023-08-28T02:00:34.961734+0000 mgr.y (mgr.14901) 7259 : cluster [DBG] pgmap v7266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:36.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:36 smithi130 bash[27579]: audit 2023-08-28T02:00:35.574825+0000 mon.c (mon.2) 2852 : audit [DBG] from='client.? 172.21.15.6:0/3882960465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:36.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:36 smithi006 bash[27292]: cluster 2023-08-28T02:00:34.961734+0000 mgr.y (mgr.14901) 7259 : cluster [DBG] pgmap v7266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:36.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:36 smithi006 bash[27292]: audit 2023-08-28T02:00:35.574825+0000 mon.c (mon.2) 2852 : audit [DBG] from='client.? 172.21.15.6:0/3882960465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:36.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:36 smithi006 bash[33918]: cluster 2023-08-28T02:00:34.961734+0000 mgr.y (mgr.14901) 7259 : cluster [DBG] pgmap v7266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:36.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:36 smithi006 bash[33918]: audit 2023-08-28T02:00:35.574825+0000 mon.c (mon.2) 2852 : audit [DBG] from='client.? 172.21.15.6:0/3882960465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:37.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:37 smithi130 bash[51302]: debug 2023-08-28T02:00:37.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:37.175780+0000) 2023-08-28T02:00:38.557 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:38 smithi130 bash[51302]: debug 2023-08-28T02:00:38.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:38.176079+0000) 2023-08-28T02:00:38.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:38 smithi130 bash[27579]: cluster 2023-08-28T02:00:36.962449+0000 mgr.y (mgr.14901) 7260 : cluster [DBG] pgmap v7267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:38.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:38 smithi130 bash[27579]: audit 2023-08-28T02:00:37.995664+0000 mon.a (mon.0) 4707 : audit [DBG] from='client.? 172.21.15.6:0/3744881180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:38.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:38 smithi006 bash[33918]: cluster 2023-08-28T02:00:36.962449+0000 mgr.y (mgr.14901) 7260 : cluster [DBG] pgmap v7267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:38.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:38 smithi006 bash[33918]: audit 2023-08-28T02:00:37.995664+0000 mon.a (mon.0) 4707 : audit [DBG] from='client.? 172.21.15.6:0/3744881180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:38.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:38 smithi006 bash[27292]: cluster 2023-08-28T02:00:36.962449+0000 mgr.y (mgr.14901) 7260 : cluster [DBG] pgmap v7267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:38.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:38 smithi006 bash[27292]: audit 2023-08-28T02:00:37.995664+0000 mon.a (mon.0) 4707 : audit [DBG] from='client.? 172.21.15.6:0/3744881180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:39.564 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:39 smithi130 bash[51302]: debug 2023-08-28T02:00:39.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:39.176342+0000) 2023-08-28T02:00:39.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:39 smithi130 bash[27579]: cluster 2023-08-28T02:00:38.963094+0000 mgr.y (mgr.14901) 7261 : cluster [DBG] pgmap v7268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:39.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:39 smithi006 bash[27292]: cluster 2023-08-28T02:00:38.963094+0000 mgr.y (mgr.14901) 7261 : cluster [DBG] pgmap v7268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:39.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:39 smithi006 bash[33918]: cluster 2023-08-28T02:00:38.963094+0000 mgr.y (mgr.14901) 7261 : cluster [DBG] pgmap v7268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:40.566 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:40 smithi130 bash[51302]: debug 2023-08-28T02:00:40.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:40.176655+0000) 2023-08-28T02:00:40.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:40 smithi130 bash[27579]: audit 2023-08-28T02:00:40.419061+0000 mon.a (mon.0) 4708 : audit [DBG] from='client.? 172.21.15.6:0/1939680803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:40.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:40 smithi006 bash[33918]: audit 2023-08-28T02:00:40.419061+0000 mon.a (mon.0) 4708 : audit [DBG] from='client.? 172.21.15.6:0/1939680803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:40.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:40 smithi006 bash[27292]: audit 2023-08-28T02:00:40.419061+0000 mon.a (mon.0) 4708 : audit [DBG] from='client.? 172.21.15.6:0/1939680803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:40.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:00:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:00:40] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T02:00:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:00:41.184Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:00:41.186Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:41.575 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:41 smithi130 bash[51302]: debug 2023-08-28T02:00:41.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:41.176973+0000) 2023-08-28T02:00:41.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:41 smithi130 bash[27579]: cluster 2023-08-28T02:00:40.964249+0000 mgr.y (mgr.14901) 7262 : cluster [DBG] pgmap v7269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:41.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:41 smithi006 bash[27292]: cluster 2023-08-28T02:00:40.964249+0000 mgr.y (mgr.14901) 7262 : cluster [DBG] pgmap v7269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:41.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:41 smithi006 bash[33918]: cluster 2023-08-28T02:00:40.964249+0000 mgr.y (mgr.14901) 7262 : cluster [DBG] pgmap v7269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:42.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:42 smithi130 bash[51302]: debug 2023-08-28T02:00:42.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:42.177213+0000) 2023-08-28T02:00:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:00:42.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:00:42.750Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:43.532 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:43 smithi130 bash[51302]: debug 2023-08-28T02:00:43.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:43.177474+0000) 2023-08-28T02:00:43.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:43 smithi130 bash[27579]: audit 2023-08-28T02:00:42.834110+0000 mon.a (mon.0) 4709 : audit [DBG] from='client.? 172.21.15.6:0/3281568067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:43.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:43 smithi006 bash[33918]: audit 2023-08-28T02:00:42.834110+0000 mon.a (mon.0) 4709 : audit [DBG] from='client.? 172.21.15.6:0/3281568067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:43.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:43 smithi006 bash[27292]: audit 2023-08-28T02:00:42.834110+0000 mon.a (mon.0) 4709 : audit [DBG] from='client.? 172.21.15.6:0/3281568067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:44.534 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:44 smithi130 bash[51302]: debug 2023-08-28T02:00:44.173+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:44.177743+0000) 2023-08-28T02:00:44.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:44 smithi130 bash[27579]: cluster 2023-08-28T02:00:42.964960+0000 mgr.y (mgr.14901) 7263 : cluster [DBG] pgmap v7270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:00:44.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:00:44.630Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:44.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:44 smithi006 bash[27292]: cluster 2023-08-28T02:00:42.964960+0000 mgr.y (mgr.14901) 7263 : cluster [DBG] pgmap v7270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:44.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:44 smithi006 bash[33918]: cluster 2023-08-28T02:00:42.964960+0000 mgr.y (mgr.14901) 7263 : cluster [DBG] pgmap v7270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:45.542 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:45 smithi130 bash[51302]: debug 2023-08-28T02:00:45.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:45.178041+0000) 2023-08-28T02:00:45.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:45 smithi130 bash[27579]: audit 2023-08-28T02:00:45.255566+0000 mon.c (mon.2) 2853 : audit [DBG] from='client.? 172.21.15.6:0/3291692432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:45.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:45 smithi006 bash[33918]: audit 2023-08-28T02:00:45.255566+0000 mon.c (mon.2) 2853 : audit [DBG] from='client.? 172.21.15.6:0/3291692432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:45.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:45 smithi006 bash[27292]: audit 2023-08-28T02:00:45.255566+0000 mon.c (mon.2) 2853 : audit [DBG] from='client.? 172.21.15.6:0/3291692432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:46.544 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:46 smithi130 bash[51302]: debug 2023-08-28T02:00:46.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:46.178319+0000) 2023-08-28T02:00:46.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:46 smithi130 bash[27579]: cluster 2023-08-28T02:00:44.965701+0000 mgr.y (mgr.14901) 7264 : cluster [DBG] pgmap v7271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:46.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:46 smithi006 bash[33918]: cluster 2023-08-28T02:00:44.965701+0000 mgr.y (mgr.14901) 7264 : cluster [DBG] pgmap v7271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:46.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:46 smithi006 bash[27292]: cluster 2023-08-28T02:00:44.965701+0000 mgr.y (mgr.14901) 7264 : cluster [DBG] pgmap v7271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:47.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:47 smithi130 bash[51302]: debug 2023-08-28T02:00:47.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:47.178596+0000) 2023-08-28T02:00:48.551 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:48 smithi130 bash[51302]: debug 2023-08-28T02:00:48.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:48.178908+0000) 2023-08-28T02:00:48.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:48 smithi130 bash[27579]: cluster 2023-08-28T02:00:46.966331+0000 mgr.y (mgr.14901) 7265 : cluster [DBG] pgmap v7272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:48.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:48 smithi130 bash[27579]: audit 2023-08-28T02:00:47.669436+0000 mon.a (mon.0) 4710 : audit [DBG] from='client.? 172.21.15.6:0/2650323808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:48.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:48 smithi006 bash[33918]: cluster 2023-08-28T02:00:46.966331+0000 mgr.y (mgr.14901) 7265 : cluster [DBG] pgmap v7272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:48.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:48 smithi006 bash[33918]: audit 2023-08-28T02:00:47.669436+0000 mon.a (mon.0) 4710 : audit [DBG] from='client.? 172.21.15.6:0/2650323808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:48.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:48 smithi006 bash[27292]: cluster 2023-08-28T02:00:46.966331+0000 mgr.y (mgr.14901) 7265 : cluster [DBG] pgmap v7272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:48.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:48 smithi006 bash[27292]: audit 2023-08-28T02:00:47.669436+0000 mon.a (mon.0) 4710 : audit [DBG] from='client.? 172.21.15.6:0/2650323808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:49.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:49 smithi130 bash[51302]: debug 2023-08-28T02:00:49.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:49.179208+0000) 2023-08-28T02:00:50.557 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:50 smithi130 bash[51302]: debug 2023-08-28T02:00:50.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:50.179503+0000) 2023-08-28T02:00:50.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:50 smithi130 bash[27579]: cluster 2023-08-28T02:00:48.967024+0000 mgr.y (mgr.14901) 7266 : cluster [DBG] pgmap v7273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:50.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:50 smithi130 bash[27579]: audit 2023-08-28T02:00:50.082279+0000 mon.a (mon.0) 4711 : audit [DBG] from='client.? 172.21.15.6:0/1646028073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:50.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:00:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:00:50] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:00:50.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:50 smithi006 bash[27292]: cluster 2023-08-28T02:00:48.967024+0000 mgr.y (mgr.14901) 7266 : cluster [DBG] pgmap v7273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:50.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:50 smithi006 bash[27292]: audit 2023-08-28T02:00:50.082279+0000 mon.a (mon.0) 4711 : audit [DBG] from='client.? 172.21.15.6:0/1646028073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:50.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:50 smithi006 bash[33918]: cluster 2023-08-28T02:00:48.967024+0000 mgr.y (mgr.14901) 7266 : cluster [DBG] pgmap v7273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:50.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:50 smithi006 bash[33918]: audit 2023-08-28T02:00:50.082279+0000 mon.a (mon.0) 4711 : audit [DBG] from='client.? 172.21.15.6:0/1646028073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:00:51.185Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:00:51.187Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:51.612 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:51 smithi130 bash[51302]: debug 2023-08-28T02:00:51.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:51.179778+0000) 2023-08-28T02:00:51.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:51 smithi130 bash[27579]: cluster 2023-08-28T02:00:50.968132+0000 mgr.y (mgr.14901) 7267 : cluster [DBG] pgmap v7274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:51.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:51 smithi006 bash[27292]: cluster 2023-08-28T02:00:50.968132+0000 mgr.y (mgr.14901) 7267 : cluster [DBG] pgmap v7274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:51.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:51 smithi006 bash[33918]: cluster 2023-08-28T02:00:50.968132+0000 mgr.y (mgr.14901) 7267 : cluster [DBG] pgmap v7274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:52.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:52 smithi130 bash[51302]: debug 2023-08-28T02:00:52.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:52.180002+0000) 2023-08-28T02:00:52.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:52 smithi006 bash[33918]: audit 2023-08-28T02:00:52.501061+0000 mon.a (mon.0) 4712 : audit [DBG] from='client.? 172.21.15.6:0/773434988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:52.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:52 smithi006 bash[27292]: audit 2023-08-28T02:00:52.501061+0000 mon.a (mon.0) 4712 : audit [DBG] from='client.? 172.21.15.6:0/773434988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:00:52.749Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:00:52.751Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:53.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:52 smithi130 bash[27579]: audit 2023-08-28T02:00:52.501061+0000 mon.a (mon.0) 4712 : audit [DBG] from='client.? 172.21.15.6:0/773434988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:53.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:53 smithi130 bash[51302]: debug 2023-08-28T02:00:53.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:53.180272+0000) 2023-08-28T02:00:53.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:53 smithi006 bash[33918]: cluster 2023-08-28T02:00:52.968796+0000 mgr.y (mgr.14901) 7268 : cluster [DBG] pgmap v7275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:53.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:53 smithi006 bash[27292]: cluster 2023-08-28T02:00:52.968796+0000 mgr.y (mgr.14901) 7268 : cluster [DBG] pgmap v7275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:54.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:53 smithi130 bash[27579]: cluster 2023-08-28T02:00:52.968796+0000 mgr.y (mgr.14901) 7268 : cluster [DBG] pgmap v7275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:54.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:54 smithi130 bash[51302]: debug 2023-08-28T02:00:54.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:54.180576+0000) 2023-08-28T02:00:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:00:54.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:00:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:00:54.630Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:00:55.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:54 smithi130 bash[27579]: audit 2023-08-28T02:00:54.925069+0000 mon.c (mon.2) 2854 : audit [DBG] from='client.? 172.21.15.6:0/1785511359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:55.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:55 smithi130 bash[51302]: debug 2023-08-28T02:00:55.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:55.180863+0000) 2023-08-28T02:00:55.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:54 smithi006 bash[27292]: audit 2023-08-28T02:00:54.925069+0000 mon.c (mon.2) 2854 : audit [DBG] from='client.? 172.21.15.6:0/1785511359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:55.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:54 smithi006 bash[33918]: audit 2023-08-28T02:00:54.925069+0000 mon.c (mon.2) 2854 : audit [DBG] from='client.? 172.21.15.6:0/1785511359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:56.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:56 smithi130 bash[27579]: cluster 2023-08-28T02:00:54.969882+0000 mgr.y (mgr.14901) 7269 : cluster [DBG] pgmap v7276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:56.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:56 smithi130 bash[51302]: debug 2023-08-28T02:00:56.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:56.181136+0000) 2023-08-28T02:00:56.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:56 smithi006 bash[27292]: cluster 2023-08-28T02:00:54.969882+0000 mgr.y (mgr.14901) 7269 : cluster [DBG] pgmap v7276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:56.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:56 smithi006 bash[33918]: cluster 2023-08-28T02:00:54.969882+0000 mgr.y (mgr.14901) 7269 : cluster [DBG] pgmap v7276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:57.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:57 smithi130 bash[51302]: debug 2023-08-28T02:00:57.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:57.181425+0000) 2023-08-28T02:00:58.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:58 smithi130 bash[51302]: debug 2023-08-28T02:00:58.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:58.181702+0000) 2023-08-28T02:00:58.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:58 smithi130 bash[27579]: cluster 2023-08-28T02:00:56.970548+0000 mgr.y (mgr.14901) 7270 : cluster [DBG] pgmap v7277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:58.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:00:58 smithi130 bash[27579]: audit 2023-08-28T02:00:57.339798+0000 mon.a (mon.0) 4713 : audit [DBG] from='client.? 172.21.15.6:0/3098715856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:58.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:58 smithi006 bash[27292]: cluster 2023-08-28T02:00:56.970548+0000 mgr.y (mgr.14901) 7270 : cluster [DBG] pgmap v7277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:58.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:00:58 smithi006 bash[27292]: audit 2023-08-28T02:00:57.339798+0000 mon.a (mon.0) 4713 : audit [DBG] from='client.? 172.21.15.6:0/3098715856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:58.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:58 smithi006 bash[33918]: cluster 2023-08-28T02:00:56.970548+0000 mgr.y (mgr.14901) 7270 : cluster [DBG] pgmap v7277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:00:58.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:00:58 smithi006 bash[33918]: audit 2023-08-28T02:00:57.339798+0000 mon.a (mon.0) 4713 : audit [DBG] from='client.? 172.21.15.6:0/3098715856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:59.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:00:59 smithi130 bash[51302]: debug 2023-08-28T02:00:59.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:59.181979+0000) 2023-08-28T02:01:00.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:00 smithi130 bash[27579]: cluster 2023-08-28T02:00:58.971280+0000 mgr.y (mgr.14901) 7271 : cluster [DBG] pgmap v7278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:00.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:00 smithi130 bash[27579]: audit 2023-08-28T02:00:59.767773+0000 mon.c (mon.2) 2855 : audit [DBG] from='client.? 172.21.15.6:0/3020645316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:00.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:00 smithi130 bash[27579]: audit 2023-08-28T02:00:59.919798+0000 mon.b (mon.1) 809 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:01:00.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:00 smithi130 bash[51302]: debug 2023-08-28T02:01:00.177+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:00.182233+0000) 2023-08-28T02:01:00.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:00 smithi006 bash[27292]: cluster 2023-08-28T02:00:58.971280+0000 mgr.y (mgr.14901) 7271 : cluster [DBG] pgmap v7278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:00.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:00 smithi006 bash[27292]: audit 2023-08-28T02:00:59.767773+0000 mon.c (mon.2) 2855 : audit [DBG] from='client.? 172.21.15.6:0/3020645316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:00.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:00 smithi006 bash[27292]: audit 2023-08-28T02:00:59.919798+0000 mon.b (mon.1) 809 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:01:00.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:00 smithi006 bash[33918]: cluster 2023-08-28T02:00:58.971280+0000 mgr.y (mgr.14901) 7271 : cluster [DBG] pgmap v7278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:00.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:00 smithi006 bash[33918]: audit 2023-08-28T02:00:59.767773+0000 mon.c (mon.2) 2855 : audit [DBG] from='client.? 172.21.15.6:0/3020645316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:00.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:00 smithi006 bash[33918]: audit 2023-08-28T02:00:59.919798+0000 mon.b (mon.1) 809 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:01:01.185 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:01:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:01:00] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T02:01:01.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:01 smithi006 bash[33918]: audit 2023-08-28T02:01:00.196792+0000 mon.a (mon.0) 4714 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:01:01.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:01 smithi006 bash[33918]: audit 2023-08-28T02:01:00.201997+0000 mon.a (mon.0) 4715 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:01:01.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:01 smithi006 bash[27292]: audit 2023-08-28T02:01:00.196792+0000 mon.a (mon.0) 4714 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:01:01.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:01 smithi006 bash[27292]: audit 2023-08-28T02:01:00.201997+0000 mon.a (mon.0) 4715 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:01:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:01:01.186Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:01:01.188Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:01.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:01 smithi130 bash[27579]: audit 2023-08-28T02:01:00.196792+0000 mon.a (mon.0) 4714 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:01:01.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:01 smithi130 bash[27579]: audit 2023-08-28T02:01:00.201997+0000 mon.a (mon.0) 4715 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:01:01.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:01 smithi130 bash[51302]: debug 2023-08-28T02:01:01.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:01.182528+0000) 2023-08-28T02:01:02.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:02 smithi006 bash[27292]: cluster 2023-08-28T02:01:00.972359+0000 mgr.y (mgr.14901) 7272 : cluster [DBG] pgmap v7279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:02.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:02 smithi006 bash[27292]: audit 2023-08-28T02:01:02.182740+0000 mon.a (mon.0) 4716 : audit [DBG] from='client.? 172.21.15.6:0/503865372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:02.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:02 smithi006 bash[33918]: cluster 2023-08-28T02:01:00.972359+0000 mgr.y (mgr.14901) 7272 : cluster [DBG] pgmap v7279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:02.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:02 smithi006 bash[33918]: audit 2023-08-28T02:01:02.182740+0000 mon.a (mon.0) 4716 : audit [DBG] from='client.? 172.21.15.6:0/503865372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:02.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:02 smithi130 bash[27579]: cluster 2023-08-28T02:01:00.972359+0000 mgr.y (mgr.14901) 7272 : cluster [DBG] pgmap v7279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:02 smithi130 bash[27579]: audit 2023-08-28T02:01:02.182740+0000 mon.a (mon.0) 4716 : audit [DBG] from='client.? 172.21.15.6:0/503865372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:02.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:02 smithi130 bash[51302]: debug 2023-08-28T02:01:02.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:02.182820+0000) 2023-08-28T02:01:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:01:02.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:01:02.752Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:03.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:03 smithi130 bash[51302]: debug 2023-08-28T02:01:03.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:03.183119+0000) 2023-08-28T02:01:04.533 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:04 smithi130 bash[51302]: debug 2023-08-28T02:01:04.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:04.183439+0000) 2023-08-28T02:01:04.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:04 smithi130 bash[27579]: cluster 2023-08-28T02:01:02.972868+0000 mgr.y (mgr.14901) 7273 : cluster [DBG] pgmap v7280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:04.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:04 smithi006 bash[33918]: cluster 2023-08-28T02:01:02.972868+0000 mgr.y (mgr.14901) 7273 : cluster [DBG] pgmap v7280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:04.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:04 smithi006 bash[27292]: cluster 2023-08-28T02:01:02.972868+0000 mgr.y (mgr.14901) 7273 : cluster [DBG] pgmap v7280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:01:04.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:01:04.630Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:05.540 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:05 smithi130 bash[51302]: debug 2023-08-28T02:01:05.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:05.183698+0000) 2023-08-28T02:01:05.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:05 smithi130 bash[27579]: audit 2023-08-28T02:01:04.601139+0000 mon.a (mon.0) 4717 : audit [DBG] from='client.? 172.21.15.6:0/3227906000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:05.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:05 smithi006 bash[27292]: audit 2023-08-28T02:01:04.601139+0000 mon.a (mon.0) 4717 : audit [DBG] from='client.? 172.21.15.6:0/3227906000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:05.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:05 smithi006 bash[33918]: audit 2023-08-28T02:01:04.601139+0000 mon.a (mon.0) 4717 : audit [DBG] from='client.? 172.21.15.6:0/3227906000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:06.543 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:06 smithi130 bash[51302]: debug 2023-08-28T02:01:06.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:06.183939+0000) 2023-08-28T02:01:06.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:06 smithi130 bash[27579]: cluster 2023-08-28T02:01:04.973926+0000 mgr.y (mgr.14901) 7274 : cluster [DBG] pgmap v7281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:06.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:06 smithi130 bash[27579]: audit 2023-08-28T02:01:06.102146+0000 mon.a (mon.0) 4718 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:01:06.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:06 smithi006 bash[33918]: cluster 2023-08-28T02:01:04.973926+0000 mgr.y (mgr.14901) 7274 : cluster [DBG] pgmap v7281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:06.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:06 smithi006 bash[33918]: audit 2023-08-28T02:01:06.102146+0000 mon.a (mon.0) 4718 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:01:06.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:06 smithi006 bash[27292]: cluster 2023-08-28T02:01:04.973926+0000 mgr.y (mgr.14901) 7274 : cluster [DBG] pgmap v7281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:06.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:06 smithi006 bash[27292]: audit 2023-08-28T02:01:06.102146+0000 mon.a (mon.0) 4718 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:01:07.606 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:07 smithi130 bash[51302]: debug 2023-08-28T02:01:07.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:07.184254+0000) 2023-08-28T02:01:07.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:07 smithi130 bash[27579]: audit 2023-08-28T02:01:06.604109+0000 mon.a (mon.0) 4719 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:01:07.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:07 smithi130 bash[27579]: audit 2023-08-28T02:01:06.901374+0000 mon.b (mon.1) 810 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:01:07.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:07 smithi130 bash[27579]: audit 2023-08-28T02:01:06.903161+0000 mon.b (mon.1) 811 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:01:07.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:07 smithi130 bash[27579]: audit 2023-08-28T02:01:06.912175+0000 mon.a (mon.0) 4720 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:01:07.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:07 smithi130 bash[27579]: cluster 2023-08-28T02:01:06.974704+0000 mgr.y (mgr.14901) 7275 : cluster [DBG] pgmap v7282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:07.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:07 smithi130 bash[27579]: audit 2023-08-28T02:01:07.009905+0000 mon.a (mon.0) 4721 : audit [DBG] from='client.? 172.21.15.6:0/1750247865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:07.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:07 smithi006 bash[27292]: audit 2023-08-28T02:01:06.604109+0000 mon.a (mon.0) 4719 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:01:07.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:07 smithi006 bash[27292]: audit 2023-08-28T02:01:06.901374+0000 mon.b (mon.1) 810 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:01:07.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:07 smithi006 bash[27292]: audit 2023-08-28T02:01:06.903161+0000 mon.b (mon.1) 811 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:01:07.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:07 smithi006 bash[27292]: audit 2023-08-28T02:01:06.912175+0000 mon.a (mon.0) 4720 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:01:07.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:07 smithi006 bash[27292]: cluster 2023-08-28T02:01:06.974704+0000 mgr.y (mgr.14901) 7275 : cluster [DBG] pgmap v7282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:07.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:07 smithi006 bash[27292]: audit 2023-08-28T02:01:07.009905+0000 mon.a (mon.0) 4721 : audit [DBG] from='client.? 172.21.15.6:0/1750247865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:07.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:07 smithi006 bash[33918]: audit 2023-08-28T02:01:06.604109+0000 mon.a (mon.0) 4719 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:01:07.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:07 smithi006 bash[33918]: audit 2023-08-28T02:01:06.901374+0000 mon.b (mon.1) 810 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:01:07.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:07 smithi006 bash[33918]: audit 2023-08-28T02:01:06.903161+0000 mon.b (mon.1) 811 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:01:07.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:07 smithi006 bash[33918]: audit 2023-08-28T02:01:06.912175+0000 mon.a (mon.0) 4720 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:01:07.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:07 smithi006 bash[33918]: cluster 2023-08-28T02:01:06.974704+0000 mgr.y (mgr.14901) 7275 : cluster [DBG] pgmap v7282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:07.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:07 smithi006 bash[33918]: audit 2023-08-28T02:01:07.009905+0000 mon.a (mon.0) 4721 : audit [DBG] from='client.? 172.21.15.6:0/1750247865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:08.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:08 smithi130 bash[51302]: debug 2023-08-28T02:01:08.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:08.184526+0000) 2023-08-28T02:01:09.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:09 smithi130 bash[51302]: debug 2023-08-28T02:01:09.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:09.184891+0000) 2023-08-28T02:01:10.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:10 smithi130 bash[27579]: cluster 2023-08-28T02:01:08.975387+0000 mgr.y (mgr.14901) 7276 : cluster [DBG] pgmap v7283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:10.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:10 smithi130 bash[27579]: audit 2023-08-28T02:01:09.419848+0000 mon.c (mon.2) 2856 : audit [DBG] from='client.? 172.21.15.6:0/2683059542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:10.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:10 smithi130 bash[51302]: debug 2023-08-28T02:01:10.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:10.185147+0000) 2023-08-28T02:01:10.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:10 smithi006 bash[27292]: cluster 2023-08-28T02:01:08.975387+0000 mgr.y (mgr.14901) 7276 : cluster [DBG] pgmap v7283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:10.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:10 smithi006 bash[27292]: audit 2023-08-28T02:01:09.419848+0000 mon.c (mon.2) 2856 : audit [DBG] from='client.? 172.21.15.6:0/2683059542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:10.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:10 smithi006 bash[33918]: cluster 2023-08-28T02:01:08.975387+0000 mgr.y (mgr.14901) 7276 : cluster [DBG] pgmap v7283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:10.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:10 smithi006 bash[33918]: audit 2023-08-28T02:01:09.419848+0000 mon.c (mon.2) 2856 : audit [DBG] from='client.? 172.21.15.6:0/2683059542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:11.187 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:01:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:01:10] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T02:01:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:01:11.187Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:01:11.189Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:11.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:11 smithi130 bash[51302]: debug 2023-08-28T02:01:11.185+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:11.185499+0000) 2023-08-28T02:01:12.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:12 smithi130 bash[51302]: debug 2023-08-28T02:01:12.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:12.185773+0000) 2023-08-28T02:01:12.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:12 smithi130 bash[27579]: cluster 2023-08-28T02:01:10.976514+0000 mgr.y (mgr.14901) 7277 : cluster [DBG] pgmap v7284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:12.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:12 smithi130 bash[27579]: audit 2023-08-28T02:01:11.829717+0000 mon.a (mon.0) 4722 : audit [DBG] from='client.? 172.21.15.6:0/3704542424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:12.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:12 smithi006 bash[27292]: cluster 2023-08-28T02:01:10.976514+0000 mgr.y (mgr.14901) 7277 : cluster [DBG] pgmap v7284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:12.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:12 smithi006 bash[27292]: audit 2023-08-28T02:01:11.829717+0000 mon.a (mon.0) 4722 : audit [DBG] from='client.? 172.21.15.6:0/3704542424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:12.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:12 smithi006 bash[33918]: cluster 2023-08-28T02:01:10.976514+0000 mgr.y (mgr.14901) 7277 : cluster [DBG] pgmap v7284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:12.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:12 smithi006 bash[33918]: audit 2023-08-28T02:01:11.829717+0000 mon.a (mon.0) 4722 : audit [DBG] from='client.? 172.21.15.6:0/3704542424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:01:12.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:01:12.752Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:13.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:13 smithi130 bash[51302]: debug 2023-08-28T02:01:13.185+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:13.186077+0000) 2023-08-28T02:01:14.536 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:14 smithi130 bash[51302]: debug 2023-08-28T02:01:14.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:14.186373+0000) 2023-08-28T02:01:14.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:14 smithi130 bash[27579]: cluster 2023-08-28T02:01:12.977340+0000 mgr.y (mgr.14901) 7278 : cluster [DBG] pgmap v7285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:14.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:14 smithi130 bash[27579]: audit 2023-08-28T02:01:14.248149+0000 mon.a (mon.0) 4723 : audit [DBG] from='client.? 172.21.15.6:0/1524563690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:14.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:14 smithi006 bash[33918]: cluster 2023-08-28T02:01:12.977340+0000 mgr.y (mgr.14901) 7278 : cluster [DBG] pgmap v7285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:14.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:14 smithi006 bash[33918]: audit 2023-08-28T02:01:14.248149+0000 mon.a (mon.0) 4723 : audit [DBG] from='client.? 172.21.15.6:0/1524563690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:01:14.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:01:14.631Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:14.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:14 smithi006 bash[27292]: cluster 2023-08-28T02:01:12.977340+0000 mgr.y (mgr.14901) 7278 : cluster [DBG] pgmap v7285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:14.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:14 smithi006 bash[27292]: audit 2023-08-28T02:01:14.248149+0000 mon.a (mon.0) 4723 : audit [DBG] from='client.? 172.21.15.6:0/1524563690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:15.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:15 smithi130 bash[51302]: debug 2023-08-28T02:01:15.181+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:15.186644+0000) 2023-08-28T02:01:16.541 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:16 smithi130 bash[51302]: debug 2023-08-28T02:01:16.185+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:16.186999+0000) 2023-08-28T02:01:16.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:16 smithi130 bash[27579]: cluster 2023-08-28T02:01:14.978439+0000 mgr.y (mgr.14901) 7279 : cluster [DBG] pgmap v7286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:16.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:16 smithi006 bash[27292]: cluster 2023-08-28T02:01:14.978439+0000 mgr.y (mgr.14901) 7279 : cluster [DBG] pgmap v7286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:16.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:16 smithi006 bash[33918]: cluster 2023-08-28T02:01:14.978439+0000 mgr.y (mgr.14901) 7279 : cluster [DBG] pgmap v7286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:17.549 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:17 smithi130 bash[51302]: debug 2023-08-28T02:01:17.185+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:17.187281+0000) 2023-08-28T02:01:17.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:17 smithi130 bash[27579]: audit 2023-08-28T02:01:16.674259+0000 mon.a (mon.0) 4724 : audit [DBG] from='client.? 172.21.15.6:0/898398474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:17.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:17 smithi006 bash[27292]: audit 2023-08-28T02:01:16.674259+0000 mon.a (mon.0) 4724 : audit [DBG] from='client.? 172.21.15.6:0/898398474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:17.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:17 smithi006 bash[33918]: audit 2023-08-28T02:01:16.674259+0000 mon.a (mon.0) 4724 : audit [DBG] from='client.? 172.21.15.6:0/898398474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:18.550 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:18 smithi130 bash[51302]: debug 2023-08-28T02:01:18.184+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:18.187496+0000) 2023-08-28T02:01:18.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:18 smithi130 bash[27579]: cluster 2023-08-28T02:01:16.979070+0000 mgr.y (mgr.14901) 7280 : cluster [DBG] pgmap v7287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:18.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:18 smithi006 bash[27292]: cluster 2023-08-28T02:01:16.979070+0000 mgr.y (mgr.14901) 7280 : cluster [DBG] pgmap v7287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:18.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:18 smithi006 bash[33918]: cluster 2023-08-28T02:01:16.979070+0000 mgr.y (mgr.14901) 7280 : cluster [DBG] pgmap v7287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:19.561 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:19 smithi130 bash[51302]: debug 2023-08-28T02:01:19.184+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:19.187774+0000) 2023-08-28T02:01:19.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:19 smithi130 bash[27579]: audit 2023-08-28T02:01:19.089111+0000 mon.a (mon.0) 4725 : audit [DBG] from='client.? 172.21.15.6:0/3915255018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:19.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:19 smithi006 bash[27292]: audit 2023-08-28T02:01:19.089111+0000 mon.a (mon.0) 4725 : audit [DBG] from='client.? 172.21.15.6:0/3915255018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:19.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:19 smithi006 bash[33918]: audit 2023-08-28T02:01:19.089111+0000 mon.a (mon.0) 4725 : audit [DBG] from='client.? 172.21.15.6:0/3915255018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:20.563 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:20 smithi130 bash[51302]: debug 2023-08-28T02:01:20.184+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:20.188076+0000) 2023-08-28T02:01:20.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:20 smithi130 bash[27579]: cluster 2023-08-28T02:01:18.979696+0000 mgr.y (mgr.14901) 7281 : cluster [DBG] pgmap v7288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:20.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:01:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:01:20] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T02:01:20.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:20 smithi006 bash[33918]: cluster 2023-08-28T02:01:18.979696+0000 mgr.y (mgr.14901) 7281 : cluster [DBG] pgmap v7288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:20.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:20 smithi006 bash[27292]: cluster 2023-08-28T02:01:18.979696+0000 mgr.y (mgr.14901) 7281 : cluster [DBG] pgmap v7288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:01:21.188Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:01:21.190Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:21.570 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:21 smithi130 bash[51302]: debug 2023-08-28T02:01:21.184+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:21.188361+0000) 2023-08-28T02:01:21.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:21 smithi130 bash[27579]: cluster 2023-08-28T02:01:20.980856+0000 mgr.y (mgr.14901) 7282 : cluster [DBG] pgmap v7289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:21.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:21 smithi130 bash[27579]: audit 2023-08-28T02:01:21.513651+0000 mon.a (mon.0) 4726 : audit [DBG] from='client.? 172.21.15.6:0/2089918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:21.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:21 smithi006 bash[27292]: cluster 2023-08-28T02:01:20.980856+0000 mgr.y (mgr.14901) 7282 : cluster [DBG] pgmap v7289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:21.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:21 smithi006 bash[27292]: audit 2023-08-28T02:01:21.513651+0000 mon.a (mon.0) 4726 : audit [DBG] from='client.? 172.21.15.6:0/2089918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:21.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:21 smithi006 bash[33918]: cluster 2023-08-28T02:01:20.980856+0000 mgr.y (mgr.14901) 7282 : cluster [DBG] pgmap v7289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:21.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:21 smithi006 bash[33918]: audit 2023-08-28T02:01:21.513651+0000 mon.a (mon.0) 4726 : audit [DBG] from='client.? 172.21.15.6:0/2089918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:22.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:22 smithi130 bash[51302]: debug 2023-08-28T02:01:22.184+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:22.188602+0000) 2023-08-28T02:01:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:01:22.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:01:22.753Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:23.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:23 smithi130 bash[51302]: debug 2023-08-28T02:01:23.184+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:23.188868+0000) 2023-08-28T02:01:24.539 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:24 smithi130 bash[51302]: debug 2023-08-28T02:01:24.184+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:24.189128+0000) 2023-08-28T02:01:24.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:24 smithi130 bash[27579]: cluster 2023-08-28T02:01:22.981526+0000 mgr.y (mgr.14901) 7283 : cluster [DBG] pgmap v7290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:24.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:24 smithi130 bash[27579]: audit 2023-08-28T02:01:23.933758+0000 mon.a (mon.0) 4727 : audit [DBG] from='client.? 172.21.15.6:0/3708255354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:24.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:24 smithi006 bash[33918]: cluster 2023-08-28T02:01:22.981526+0000 mgr.y (mgr.14901) 7283 : cluster [DBG] pgmap v7290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:24.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:24 smithi006 bash[33918]: audit 2023-08-28T02:01:23.933758+0000 mon.a (mon.0) 4727 : audit [DBG] from='client.? 172.21.15.6:0/3708255354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:01:24.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:01:24.633Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:24.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:24 smithi006 bash[27292]: cluster 2023-08-28T02:01:22.981526+0000 mgr.y (mgr.14901) 7283 : cluster [DBG] pgmap v7290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:24.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:24 smithi006 bash[27292]: audit 2023-08-28T02:01:23.933758+0000 mon.a (mon.0) 4727 : audit [DBG] from='client.? 172.21.15.6:0/3708255354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:25.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:25 smithi130 bash[51302]: debug 2023-08-28T02:01:25.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:25.189404+0000) 2023-08-28T02:01:26.543 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:26 smithi130 bash[51302]: debug 2023-08-28T02:01:26.184+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:26.189710+0000) 2023-08-28T02:01:26.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:26 smithi130 bash[27579]: cluster 2023-08-28T02:01:24.982796+0000 mgr.y (mgr.14901) 7284 : cluster [DBG] pgmap v7291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:26.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:26 smithi130 bash[27579]: audit 2023-08-28T02:01:26.350432+0000 mon.c (mon.2) 2857 : audit [DBG] from='client.? 172.21.15.6:0/3841024744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:26.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:26 smithi006 bash[27292]: cluster 2023-08-28T02:01:24.982796+0000 mgr.y (mgr.14901) 7284 : cluster [DBG] pgmap v7291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:26.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:26 smithi006 bash[27292]: audit 2023-08-28T02:01:26.350432+0000 mon.c (mon.2) 2857 : audit [DBG] from='client.? 172.21.15.6:0/3841024744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:26.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:26 smithi006 bash[33918]: cluster 2023-08-28T02:01:24.982796+0000 mgr.y (mgr.14901) 7284 : cluster [DBG] pgmap v7291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:26.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:26 smithi006 bash[33918]: audit 2023-08-28T02:01:26.350432+0000 mon.c (mon.2) 2857 : audit [DBG] from='client.? 172.21.15.6:0/3841024744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:27.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:27 smithi130 bash[51302]: debug 2023-08-28T02:01:27.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:27.189989+0000) 2023-08-28T02:01:28.547 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:28 smithi130 bash[51302]: debug 2023-08-28T02:01:28.184+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:28.190240+0000) 2023-08-28T02:01:28.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:28 smithi130 bash[27579]: cluster 2023-08-28T02:01:26.983562+0000 mgr.y (mgr.14901) 7285 : cluster [DBG] pgmap v7292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:28.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:28 smithi006 bash[27292]: cluster 2023-08-28T02:01:26.983562+0000 mgr.y (mgr.14901) 7285 : cluster [DBG] pgmap v7292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:28.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:28 smithi006 bash[33918]: cluster 2023-08-28T02:01:26.983562+0000 mgr.y (mgr.14901) 7285 : cluster [DBG] pgmap v7292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:29.557 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:29 smithi130 bash[51302]: debug 2023-08-28T02:01:29.184+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:29.190534+0000) 2023-08-28T02:01:29.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:29 smithi130 bash[27579]: audit 2023-08-28T02:01:28.768052+0000 mon.c (mon.2) 2858 : audit [DBG] from='client.? 172.21.15.6:0/1448635367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:29.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:29 smithi006 bash[33918]: audit 2023-08-28T02:01:28.768052+0000 mon.c (mon.2) 2858 : audit [DBG] from='client.? 172.21.15.6:0/1448635367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:29.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:29 smithi006 bash[27292]: audit 2023-08-28T02:01:28.768052+0000 mon.c (mon.2) 2858 : audit [DBG] from='client.? 172.21.15.6:0/1448635367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:30.560 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:30 smithi130 bash[51302]: debug 2023-08-28T02:01:30.184+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:30.190800+0000) 2023-08-28T02:01:30.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:30 smithi130 bash[27579]: cluster 2023-08-28T02:01:28.984222+0000 mgr.y (mgr.14901) 7286 : cluster [DBG] pgmap v7293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:30.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:01:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:01:30] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:01:30.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:30 smithi006 bash[33918]: cluster 2023-08-28T02:01:28.984222+0000 mgr.y (mgr.14901) 7286 : cluster [DBG] pgmap v7293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:30.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:30 smithi006 bash[27292]: cluster 2023-08-28T02:01:28.984222+0000 mgr.y (mgr.14901) 7286 : cluster [DBG] pgmap v7293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:01:31.189Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:01:31.190Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:31.569 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:31 smithi130 bash[51302]: debug 2023-08-28T02:01:31.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:31.191051+0000) 2023-08-28T02:01:31.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:31 smithi130 bash[27579]: audit 2023-08-28T02:01:31.185445+0000 mon.c (mon.2) 2859 : audit [DBG] from='client.? 172.21.15.6:0/384834223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:31.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:31 smithi006 bash[27292]: audit 2023-08-28T02:01:31.185445+0000 mon.c (mon.2) 2859 : audit [DBG] from='client.? 172.21.15.6:0/384834223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:31.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:31 smithi006 bash[33918]: audit 2023-08-28T02:01:31.185445+0000 mon.c (mon.2) 2859 : audit [DBG] from='client.? 172.21.15.6:0/384834223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:32.574 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:32 smithi130 bash[51302]: debug 2023-08-28T02:01:32.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:32.191303+0000) 2023-08-28T02:01:32.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:32 smithi130 bash[27579]: cluster 2023-08-28T02:01:30.985443+0000 mgr.y (mgr.14901) 7287 : cluster [DBG] pgmap v7294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:32.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:32 smithi006 bash[27292]: cluster 2023-08-28T02:01:30.985443+0000 mgr.y (mgr.14901) 7287 : cluster [DBG] pgmap v7294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:32.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:32 smithi006 bash[33918]: cluster 2023-08-28T02:01:30.985443+0000 mgr.y (mgr.14901) 7287 : cluster [DBG] pgmap v7294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:01:32.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:01:32.753Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:33.576 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:33 smithi130 bash[51302]: debug 2023-08-28T02:01:33.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:33.191585+0000) 2023-08-28T02:01:33.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:33 smithi130 bash[27579]: cluster 2023-08-28T02:01:32.986125+0000 mgr.y (mgr.14901) 7288 : cluster [DBG] pgmap v7295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:33.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:33 smithi006 bash[33918]: cluster 2023-08-28T02:01:32.986125+0000 mgr.y (mgr.14901) 7288 : cluster [DBG] pgmap v7295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:33 smithi006 bash[27292]: cluster 2023-08-28T02:01:32.986125+0000 mgr.y (mgr.14901) 7288 : cluster [DBG] pgmap v7295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:34.585 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:34 smithi130 bash[51302]: debug 2023-08-28T02:01:34.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:34.191860+0000) 2023-08-28T02:01:34.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:34 smithi130 bash[27579]: audit 2023-08-28T02:01:33.596342+0000 mon.a (mon.0) 4728 : audit [DBG] from='client.? 172.21.15.6:0/1962909541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:34.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:34 smithi006 bash[33918]: audit 2023-08-28T02:01:33.596342+0000 mon.a (mon.0) 4728 : audit [DBG] from='client.? 172.21.15.6:0/1962909541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:34.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:34 smithi006 bash[27292]: audit 2023-08-28T02:01:33.596342+0000 mon.a (mon.0) 4728 : audit [DBG] from='client.? 172.21.15.6:0/1962909541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:01:34.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:01:34.633Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:35.592 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:35 smithi130 bash[51302]: debug 2023-08-28T02:01:35.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:35.192124+0000) 2023-08-28T02:01:35.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:35 smithi130 bash[27579]: cluster 2023-08-28T02:01:34.987230+0000 mgr.y (mgr.14901) 7289 : cluster [DBG] pgmap v7296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:35.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:35 smithi006 bash[33918]: cluster 2023-08-28T02:01:34.987230+0000 mgr.y (mgr.14901) 7289 : cluster [DBG] pgmap v7296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:35.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:35 smithi006 bash[27292]: cluster 2023-08-28T02:01:34.987230+0000 mgr.y (mgr.14901) 7289 : cluster [DBG] pgmap v7296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:36.595 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:36 smithi130 bash[51302]: debug 2023-08-28T02:01:36.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:36.192391+0000) 2023-08-28T02:01:36.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:36 smithi130 bash[27579]: audit 2023-08-28T02:01:36.009192+0000 mon.c (mon.2) 2860 : audit [DBG] from='client.? 172.21.15.6:0/2599207037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:36.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:36 smithi006 bash[33918]: audit 2023-08-28T02:01:36.009192+0000 mon.c (mon.2) 2860 : audit [DBG] from='client.? 172.21.15.6:0/2599207037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:36.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:36 smithi006 bash[27292]: audit 2023-08-28T02:01:36.009192+0000 mon.c (mon.2) 2860 : audit [DBG] from='client.? 172.21.15.6:0/2599207037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:37.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:37 smithi130 bash[51302]: debug 2023-08-28T02:01:37.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:37.192693+0000) 2023-08-28T02:01:38.541 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:38 smithi130 bash[51302]: debug 2023-08-28T02:01:38.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:38.192954+0000) 2023-08-28T02:01:38.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:38 smithi130 bash[27579]: cluster 2023-08-28T02:01:36.987877+0000 mgr.y (mgr.14901) 7290 : cluster [DBG] pgmap v7297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:38.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:38 smithi130 bash[27579]: audit 2023-08-28T02:01:38.429252+0000 mon.c (mon.2) 2861 : audit [DBG] from='client.? 172.21.15.6:0/3360437532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:38.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:38 smithi006 bash[27292]: cluster 2023-08-28T02:01:36.987877+0000 mgr.y (mgr.14901) 7290 : cluster [DBG] pgmap v7297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:38.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:38 smithi006 bash[27292]: audit 2023-08-28T02:01:38.429252+0000 mon.c (mon.2) 2861 : audit [DBG] from='client.? 172.21.15.6:0/3360437532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:38.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:38 smithi006 bash[33918]: cluster 2023-08-28T02:01:36.987877+0000 mgr.y (mgr.14901) 7290 : cluster [DBG] pgmap v7297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:38.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:38 smithi006 bash[33918]: audit 2023-08-28T02:01:38.429252+0000 mon.c (mon.2) 2861 : audit [DBG] from='client.? 172.21.15.6:0/3360437532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:39.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:39 smithi130 bash[51302]: debug 2023-08-28T02:01:39.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:39.193262+0000) 2023-08-28T02:01:40.546 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:40 smithi130 bash[51302]: debug 2023-08-28T02:01:40.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:40.193564+0000) 2023-08-28T02:01:40.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:40 smithi130 bash[27579]: cluster 2023-08-28T02:01:38.988461+0000 mgr.y (mgr.14901) 7291 : cluster [DBG] pgmap v7298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:40.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:01:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:01:40] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:01:40.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:40 smithi006 bash[33918]: cluster 2023-08-28T02:01:38.988461+0000 mgr.y (mgr.14901) 7291 : cluster [DBG] pgmap v7298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:40.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:40 smithi006 bash[27292]: cluster 2023-08-28T02:01:38.988461+0000 mgr.y (mgr.14901) 7291 : cluster [DBG] pgmap v7298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:01:41.189Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:01:41.191Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:41.555 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:41 smithi130 bash[51302]: debug 2023-08-28T02:01:41.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:41.193861+0000) 2023-08-28T02:01:41.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:41 smithi130 bash[27579]: audit 2023-08-28T02:01:40.843741+0000 mon.c (mon.2) 2862 : audit [DBG] from='client.? 172.21.15.6:0/3101168701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:41.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:41 smithi006 bash[27292]: audit 2023-08-28T02:01:40.843741+0000 mon.c (mon.2) 2862 : audit [DBG] from='client.? 172.21.15.6:0/3101168701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:41.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:41 smithi006 bash[33918]: audit 2023-08-28T02:01:40.843741+0000 mon.c (mon.2) 2862 : audit [DBG] from='client.? 172.21.15.6:0/3101168701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:42.557 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:42 smithi130 bash[51302]: debug 2023-08-28T02:01:42.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:42.194103+0000) 2023-08-28T02:01:42.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:42 smithi130 bash[27579]: cluster 2023-08-28T02:01:40.989687+0000 mgr.y (mgr.14901) 7292 : cluster [DBG] pgmap v7299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:42.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:42 smithi006 bash[33918]: cluster 2023-08-28T02:01:40.989687+0000 mgr.y (mgr.14901) 7292 : cluster [DBG] pgmap v7299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:42.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:42 smithi006 bash[27292]: cluster 2023-08-28T02:01:40.989687+0000 mgr.y (mgr.14901) 7292 : cluster [DBG] pgmap v7299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:01:42.752Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:42.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:01:42.753Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:43.566 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:43 smithi130 bash[51302]: debug 2023-08-28T02:01:43.188+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:43.194434+0000) 2023-08-28T02:01:43.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:43 smithi130 bash[27579]: audit 2023-08-28T02:01:43.265168+0000 mon.c (mon.2) 2863 : audit [DBG] from='client.? 172.21.15.6:0/3672778359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:43.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:43 smithi006 bash[27292]: audit 2023-08-28T02:01:43.265168+0000 mon.c (mon.2) 2863 : audit [DBG] from='client.? 172.21.15.6:0/3672778359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:43.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:43 smithi006 bash[33918]: audit 2023-08-28T02:01:43.265168+0000 mon.c (mon.2) 2863 : audit [DBG] from='client.? 172.21.15.6:0/3672778359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:44.567 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:44 smithi130 bash[51302]: debug 2023-08-28T02:01:44.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:44.194754+0000) 2023-08-28T02:01:44.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:44 smithi130 bash[27579]: cluster 2023-08-28T02:01:42.990215+0000 mgr.y (mgr.14901) 7293 : cluster [DBG] pgmap v7300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:44.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:44 smithi006 bash[27292]: cluster 2023-08-28T02:01:42.990215+0000 mgr.y (mgr.14901) 7293 : cluster [DBG] pgmap v7300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:44.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:44 smithi006 bash[33918]: cluster 2023-08-28T02:01:42.990215+0000 mgr.y (mgr.14901) 7293 : cluster [DBG] pgmap v7300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:01:44.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:01:44.634Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:45.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:45 smithi130 bash[51302]: debug 2023-08-28T02:01:45.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:45.195035+0000) 2023-08-28T02:01:45.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:45 smithi006 bash[27292]: cluster 2023-08-28T02:01:44.991415+0000 mgr.y (mgr.14901) 7294 : cluster [DBG] pgmap v7301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:45.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:45 smithi006 bash[33918]: cluster 2023-08-28T02:01:44.991415+0000 mgr.y (mgr.14901) 7294 : cluster [DBG] pgmap v7301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:46.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:45 smithi130 bash[27579]: cluster 2023-08-28T02:01:44.991415+0000 mgr.y (mgr.14901) 7294 : cluster [DBG] pgmap v7301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:46.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:46 smithi130 bash[51302]: debug 2023-08-28T02:01:46.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:46.195288+0000) 2023-08-28T02:01:46.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:46 smithi006 bash[33918]: audit 2023-08-28T02:01:45.684764+0000 mon.a (mon.0) 4729 : audit [DBG] from='client.? 172.21.15.6:0/3234586878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:46.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:46 smithi006 bash[27292]: audit 2023-08-28T02:01:45.684764+0000 mon.a (mon.0) 4729 : audit [DBG] from='client.? 172.21.15.6:0/3234586878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:47.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:46 smithi130 bash[27579]: audit 2023-08-28T02:01:45.684764+0000 mon.a (mon.0) 4729 : audit [DBG] from='client.? 172.21.15.6:0/3234586878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:47.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:47 smithi130 bash[51302]: debug 2023-08-28T02:01:47.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:47.195592+0000) 2023-08-28T02:01:48.543 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:48 smithi130 bash[51302]: debug 2023-08-28T02:01:48.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:48.195868+0000) 2023-08-28T02:01:48.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:48 smithi130 bash[27579]: cluster 2023-08-28T02:01:46.992185+0000 mgr.y (mgr.14901) 7295 : cluster [DBG] pgmap v7302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:48.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:48 smithi130 bash[27579]: audit 2023-08-28T02:01:48.098901+0000 mon.a (mon.0) 4730 : audit [DBG] from='client.? 172.21.15.6:0/3772534194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:48.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:48 smithi006 bash[27292]: cluster 2023-08-28T02:01:46.992185+0000 mgr.y (mgr.14901) 7295 : cluster [DBG] pgmap v7302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:48.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:48 smithi006 bash[27292]: audit 2023-08-28T02:01:48.098901+0000 mon.a (mon.0) 4730 : audit [DBG] from='client.? 172.21.15.6:0/3772534194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:48.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:48 smithi006 bash[33918]: cluster 2023-08-28T02:01:46.992185+0000 mgr.y (mgr.14901) 7295 : cluster [DBG] pgmap v7302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:48.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:48 smithi006 bash[33918]: audit 2023-08-28T02:01:48.098901+0000 mon.a (mon.0) 4730 : audit [DBG] from='client.? 172.21.15.6:0/3772534194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:49.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:49 smithi130 bash[51302]: debug 2023-08-28T02:01:49.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:49.196202+0000) 2023-08-28T02:01:50.547 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:50 smithi130 bash[51302]: debug 2023-08-28T02:01:50.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:50.196485+0000) 2023-08-28T02:01:50.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:50 smithi130 bash[27579]: cluster 2023-08-28T02:01:48.992991+0000 mgr.y (mgr.14901) 7296 : cluster [DBG] pgmap v7303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:50.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:50 smithi130 bash[27579]: audit 2023-08-28T02:01:50.522339+0000 mon.a (mon.0) 4731 : audit [DBG] from='client.? 172.21.15.6:0/2572150300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:50.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:01:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:01:50] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:01:50.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:50 smithi006 bash[33918]: cluster 2023-08-28T02:01:48.992991+0000 mgr.y (mgr.14901) 7296 : cluster [DBG] pgmap v7303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:50.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:50 smithi006 bash[33918]: audit 2023-08-28T02:01:50.522339+0000 mon.a (mon.0) 4731 : audit [DBG] from='client.? 172.21.15.6:0/2572150300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:50.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:50 smithi006 bash[27292]: cluster 2023-08-28T02:01:48.992991+0000 mgr.y (mgr.14901) 7296 : cluster [DBG] pgmap v7303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:50.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:50 smithi006 bash[27292]: audit 2023-08-28T02:01:50.522339+0000 mon.a (mon.0) 4731 : audit [DBG] from='client.? 172.21.15.6:0/2572150300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:01:51.190Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:01:51.192Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:51.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:51 smithi130 bash[51302]: debug 2023-08-28T02:01:51.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:51.196784+0000) 2023-08-28T02:01:52.551 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:52 smithi130 bash[51302]: debug 2023-08-28T02:01:52.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:52.197042+0000) 2023-08-28T02:01:52.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:52 smithi130 bash[27579]: cluster 2023-08-28T02:01:50.994148+0000 mgr.y (mgr.14901) 7297 : cluster [DBG] pgmap v7304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:52.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:52 smithi006 bash[33918]: cluster 2023-08-28T02:01:50.994148+0000 mgr.y (mgr.14901) 7297 : cluster [DBG] pgmap v7304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:52.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:52 smithi006 bash[27292]: cluster 2023-08-28T02:01:50.994148+0000 mgr.y (mgr.14901) 7297 : cluster [DBG] pgmap v7304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:01:52.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:01:52.755Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:53.563 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:53 smithi130 bash[51302]: debug 2023-08-28T02:01:53.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:53.197334+0000) 2023-08-28T02:01:53.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:53 smithi130 bash[27579]: audit 2023-08-28T02:01:52.936699+0000 mon.a (mon.0) 4732 : audit [DBG] from='client.? 172.21.15.6:0/2567081478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:53.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:53 smithi006 bash[33918]: audit 2023-08-28T02:01:52.936699+0000 mon.a (mon.0) 4732 : audit [DBG] from='client.? 172.21.15.6:0/2567081478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:53.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:53 smithi006 bash[27292]: audit 2023-08-28T02:01:52.936699+0000 mon.a (mon.0) 4732 : audit [DBG] from='client.? 172.21.15.6:0/2567081478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:54.566 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:54 smithi130 bash[51302]: debug 2023-08-28T02:01:54.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:54.197643+0000) 2023-08-28T02:01:54.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:54 smithi130 bash[27579]: cluster 2023-08-28T02:01:52.994773+0000 mgr.y (mgr.14901) 7298 : cluster [DBG] pgmap v7305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:54.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:54 smithi006 bash[27292]: cluster 2023-08-28T02:01:52.994773+0000 mgr.y (mgr.14901) 7298 : cluster [DBG] pgmap v7305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:54.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:54 smithi006 bash[33918]: cluster 2023-08-28T02:01:52.994773+0000 mgr.y (mgr.14901) 7298 : cluster [DBG] pgmap v7305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:01:54.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:01:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:01:54.634Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:01:55.569 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:55 smithi130 bash[51302]: debug 2023-08-28T02:01:55.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:55.197924+0000) 2023-08-28T02:01:55.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:55 smithi130 bash[27579]: audit 2023-08-28T02:01:55.350406+0000 mon.c (mon.2) 2864 : audit [DBG] from='client.? 172.21.15.6:0/2066609306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:55.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:55 smithi006 bash[27292]: audit 2023-08-28T02:01:55.350406+0000 mon.c (mon.2) 2864 : audit [DBG] from='client.? 172.21.15.6:0/2066609306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:55.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:55 smithi006 bash[33918]: audit 2023-08-28T02:01:55.350406+0000 mon.c (mon.2) 2864 : audit [DBG] from='client.? 172.21.15.6:0/2066609306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:56.572 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:56 smithi130 bash[51302]: debug 2023-08-28T02:01:56.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:56.198186+0000) 2023-08-28T02:01:56.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:56 smithi130 bash[27579]: cluster 2023-08-28T02:01:54.995991+0000 mgr.y (mgr.14901) 7299 : cluster [DBG] pgmap v7306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:56.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:56 smithi006 bash[33918]: cluster 2023-08-28T02:01:54.995991+0000 mgr.y (mgr.14901) 7299 : cluster [DBG] pgmap v7306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:56.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:56 smithi006 bash[27292]: cluster 2023-08-28T02:01:54.995991+0000 mgr.y (mgr.14901) 7299 : cluster [DBG] pgmap v7306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:57.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:57 smithi130 bash[51302]: debug 2023-08-28T02:01:57.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:57.198435+0000) 2023-08-28T02:01:58.546 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:58 smithi130 bash[51302]: debug 2023-08-28T02:01:58.192+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:58.198697+0000) 2023-08-28T02:01:58.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:58 smithi130 bash[27579]: cluster 2023-08-28T02:01:56.996716+0000 mgr.y (mgr.14901) 7300 : cluster [DBG] pgmap v7307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:58.878 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:01:58 smithi130 bash[27579]: audit 2023-08-28T02:01:57.769652+0000 mon.a (mon.0) 4733 : audit [DBG] from='client.? 172.21.15.6:0/587826722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:58.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:58 smithi006 bash[33918]: cluster 2023-08-28T02:01:56.996716+0000 mgr.y (mgr.14901) 7300 : cluster [DBG] pgmap v7307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:58.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:01:58 smithi006 bash[33918]: audit 2023-08-28T02:01:57.769652+0000 mon.a (mon.0) 4733 : audit [DBG] from='client.? 172.21.15.6:0/587826722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:58.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:58 smithi006 bash[27292]: cluster 2023-08-28T02:01:56.996716+0000 mgr.y (mgr.14901) 7300 : cluster [DBG] pgmap v7307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:01:58.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:01:58 smithi006 bash[27292]: audit 2023-08-28T02:01:57.769652+0000 mon.a (mon.0) 4733 : audit [DBG] from='client.? 172.21.15.6:0/587826722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:59.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:01:59 smithi130 bash[51302]: debug 2023-08-28T02:01:59.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:59.198973+0000) 2023-08-28T02:02:00.551 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:00 smithi130 bash[51302]: debug 2023-08-28T02:02:00.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:00.199195+0000) 2023-08-28T02:02:00.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:00 smithi130 bash[27579]: cluster 2023-08-28T02:01:58.997366+0000 mgr.y (mgr.14901) 7301 : cluster [DBG] pgmap v7308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:00.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:00 smithi130 bash[27579]: audit 2023-08-28T02:02:00.180633+0000 mon.a (mon.0) 4734 : audit [DBG] from='client.? 172.21.15.6:0/642540613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:00.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:02:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:02:00] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T02:02:00.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:00 smithi006 bash[33918]: cluster 2023-08-28T02:01:58.997366+0000 mgr.y (mgr.14901) 7301 : cluster [DBG] pgmap v7308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:00.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:00 smithi006 bash[33918]: audit 2023-08-28T02:02:00.180633+0000 mon.a (mon.0) 4734 : audit [DBG] from='client.? 172.21.15.6:0/642540613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:00.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:00 smithi006 bash[27292]: cluster 2023-08-28T02:01:58.997366+0000 mgr.y (mgr.14901) 7301 : cluster [DBG] pgmap v7308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:00.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:00 smithi006 bash[27292]: audit 2023-08-28T02:02:00.180633+0000 mon.a (mon.0) 4734 : audit [DBG] from='client.? 172.21.15.6:0/642540613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:02:01.190Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:02:01.193Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:01.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:01 smithi130 bash[51302]: debug 2023-08-28T02:02:01.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:01.199511+0000) 2023-08-28T02:02:02.556 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:02 smithi130 bash[51302]: debug 2023-08-28T02:02:02.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:02.199743+0000) 2023-08-28T02:02:02.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:02 smithi130 bash[27579]: cluster 2023-08-28T02:02:00.998472+0000 mgr.y (mgr.14901) 7302 : cluster [DBG] pgmap v7309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:02.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:02 smithi006 bash[27292]: cluster 2023-08-28T02:02:00.998472+0000 mgr.y (mgr.14901) 7302 : cluster [DBG] pgmap v7309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:02.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:02 smithi006 bash[33918]: cluster 2023-08-28T02:02:00.998472+0000 mgr.y (mgr.14901) 7302 : cluster [DBG] pgmap v7309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:02:02.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:02:02.756Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:03.560 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:03 smithi130 bash[51302]: debug 2023-08-28T02:02:03.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:03.200047+0000) 2023-08-28T02:02:03.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:03 smithi130 bash[27579]: audit 2023-08-28T02:02:02.595566+0000 mon.a (mon.0) 4735 : audit [DBG] from='client.? 172.21.15.6:0/1647092359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:03.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:03 smithi006 bash[27292]: audit 2023-08-28T02:02:02.595566+0000 mon.a (mon.0) 4735 : audit [DBG] from='client.? 172.21.15.6:0/1647092359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:03.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:03 smithi006 bash[33918]: audit 2023-08-28T02:02:02.595566+0000 mon.a (mon.0) 4735 : audit [DBG] from='client.? 172.21.15.6:0/1647092359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:04.568 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:04 smithi130 bash[51302]: debug 2023-08-28T02:02:04.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:04.200321+0000) 2023-08-28T02:02:04.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:04 smithi130 bash[27579]: cluster 2023-08-28T02:02:02.999156+0000 mgr.y (mgr.14901) 7303 : cluster [DBG] pgmap v7310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:02:04.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:02:04.634Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:04.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:04 smithi006 bash[33918]: cluster 2023-08-28T02:02:02.999156+0000 mgr.y (mgr.14901) 7303 : cluster [DBG] pgmap v7310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:04.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:04 smithi006 bash[27292]: cluster 2023-08-28T02:02:02.999156+0000 mgr.y (mgr.14901) 7303 : cluster [DBG] pgmap v7310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:05.576 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:05 smithi130 bash[51302]: debug 2023-08-28T02:02:05.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:05.200601+0000) 2023-08-28T02:02:05.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:05 smithi130 bash[27579]: audit 2023-08-28T02:02:05.013019+0000 mon.c (mon.2) 2865 : audit [DBG] from='client.? 172.21.15.6:0/4091192537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:05.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:05 smithi006 bash[33918]: audit 2023-08-28T02:02:05.013019+0000 mon.c (mon.2) 2865 : audit [DBG] from='client.? 172.21.15.6:0/4091192537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:05.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:05 smithi006 bash[27292]: audit 2023-08-28T02:02:05.013019+0000 mon.c (mon.2) 2865 : audit [DBG] from='client.? 172.21.15.6:0/4091192537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:06.580 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:06 smithi130 bash[51302]: debug 2023-08-28T02:02:06.200+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:06.200877+0000) 2023-08-28T02:02:06.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:06 smithi130 bash[27579]: cluster 2023-08-28T02:02:05.000355+0000 mgr.y (mgr.14901) 7304 : cluster [DBG] pgmap v7311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:06.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:06 smithi006 bash[27292]: cluster 2023-08-28T02:02:05.000355+0000 mgr.y (mgr.14901) 7304 : cluster [DBG] pgmap v7311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:06.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:06 smithi006 bash[33918]: cluster 2023-08-28T02:02:05.000355+0000 mgr.y (mgr.14901) 7304 : cluster [DBG] pgmap v7311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:07.588 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:07 smithi130 bash[51302]: debug 2023-08-28T02:02:07.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:07.201168+0000) 2023-08-28T02:02:07.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:07 smithi130 bash[27579]: audit 2023-08-28T02:02:06.923019+0000 mon.b (mon.1) 812 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:02:07.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:07 smithi130 bash[27579]: cluster 2023-08-28T02:02:07.001116+0000 mgr.y (mgr.14901) 7305 : cluster [DBG] pgmap v7312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:07.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:07 smithi130 bash[27579]: audit 2023-08-28T02:02:07.426814+0000 mon.a (mon.0) 4736 : audit [DBG] from='client.? 172.21.15.6:0/3838868374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:07.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:07 smithi006 bash[27292]: audit 2023-08-28T02:02:06.923019+0000 mon.b (mon.1) 812 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:02:07.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:07 smithi006 bash[27292]: cluster 2023-08-28T02:02:07.001116+0000 mgr.y (mgr.14901) 7305 : cluster [DBG] pgmap v7312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:07.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:07 smithi006 bash[27292]: audit 2023-08-28T02:02:07.426814+0000 mon.a (mon.0) 4736 : audit [DBG] from='client.? 172.21.15.6:0/3838868374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:07.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:07 smithi006 bash[33918]: audit 2023-08-28T02:02:06.923019+0000 mon.b (mon.1) 812 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:02:07.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:07 smithi006 bash[33918]: cluster 2023-08-28T02:02:07.001116+0000 mgr.y (mgr.14901) 7305 : cluster [DBG] pgmap v7312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:07.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:07 smithi006 bash[33918]: audit 2023-08-28T02:02:07.426814+0000 mon.a (mon.0) 4736 : audit [DBG] from='client.? 172.21.15.6:0/3838868374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:08.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:08 smithi130 bash[51302]: debug 2023-08-28T02:02:08.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:08.201473+0000) 2023-08-28T02:02:09.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:09 smithi130 bash[51302]: debug 2023-08-28T02:02:09.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:09.201745+0000) 2023-08-28T02:02:10.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:10 smithi130 bash[51302]: debug 2023-08-28T02:02:10.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:10.201946+0000) 2023-08-28T02:02:10.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:10 smithi130 bash[27579]: cluster 2023-08-28T02:02:09.001886+0000 mgr.y (mgr.14901) 7306 : cluster [DBG] pgmap v7313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:10.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:10 smithi130 bash[27579]: audit 2023-08-28T02:02:09.851882+0000 mon.c (mon.2) 2866 : audit [DBG] from='client.? 172.21.15.6:0/766821702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:10.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:10 smithi006 bash[27292]: cluster 2023-08-28T02:02:09.001886+0000 mgr.y (mgr.14901) 7306 : cluster [DBG] pgmap v7313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:10.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:10 smithi006 bash[27292]: audit 2023-08-28T02:02:09.851882+0000 mon.c (mon.2) 2866 : audit [DBG] from='client.? 172.21.15.6:0/766821702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:10.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:10 smithi006 bash[33918]: cluster 2023-08-28T02:02:09.001886+0000 mgr.y (mgr.14901) 7306 : cluster [DBG] pgmap v7313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:10.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:10 smithi006 bash[33918]: audit 2023-08-28T02:02:09.851882+0000 mon.c (mon.2) 2866 : audit [DBG] from='client.? 172.21.15.6:0/766821702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:11.191 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:02:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:02:10] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T02:02:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:02:11.191Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:02:11.193Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:11.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:11 smithi130 bash[51302]: debug 2023-08-28T02:02:11.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:11.202260+0000) 2023-08-28T02:02:12.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:12 smithi130 bash[27579]: cluster 2023-08-28T02:02:11.003135+0000 mgr.y (mgr.14901) 7307 : cluster [DBG] pgmap v7314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:12.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:12 smithi130 bash[51302]: debug 2023-08-28T02:02:12.196+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:12.202530+0000) 2023-08-28T02:02:12.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:12 smithi006 bash[27292]: cluster 2023-08-28T02:02:11.003135+0000 mgr.y (mgr.14901) 7307 : cluster [DBG] pgmap v7314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:12.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:12 smithi006 bash[33918]: cluster 2023-08-28T02:02:11.003135+0000 mgr.y (mgr.14901) 7307 : cluster [DBG] pgmap v7314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:13.065 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:02:12.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:13.065 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:02:12.756Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:13.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:13 smithi130 bash[51302]: debug 2023-08-28T02:02:13.195+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:13.202806+0000) 2023-08-28T02:02:13.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:13 smithi130 bash[27579]: audit 2023-08-28T02:02:12.336120+0000 mon.a (mon.0) 4737 : audit [DBG] from='client.? 172.21.15.6:0/2025913043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:13.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:13 smithi130 bash[27579]: audit 2023-08-28T02:02:12.799290+0000 mon.a (mon.0) 4738 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:02:13.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:13 smithi006 bash[27292]: audit 2023-08-28T02:02:12.336120+0000 mon.a (mon.0) 4737 : audit [DBG] from='client.? 172.21.15.6:0/2025913043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:13.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:13 smithi006 bash[27292]: audit 2023-08-28T02:02:12.799290+0000 mon.a (mon.0) 4738 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:02:13.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:13 smithi006 bash[33918]: audit 2023-08-28T02:02:12.336120+0000 mon.a (mon.0) 4737 : audit [DBG] from='client.? 172.21.15.6:0/2025913043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:13.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:13 smithi006 bash[33918]: audit 2023-08-28T02:02:12.799290+0000 mon.a (mon.0) 4738 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:02:14.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:14 smithi130 bash[51302]: debug 2023-08-28T02:02:14.199+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:14.203086+0000) 2023-08-28T02:02:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:14 smithi130 bash[27579]: cluster 2023-08-28T02:02:13.003726+0000 mgr.y (mgr.14901) 7308 : cluster [DBG] pgmap v7315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:14 smithi130 bash[27579]: audit 2023-08-28T02:02:13.270501+0000 mon.a (mon.0) 4739 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:02:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:14 smithi130 bash[27579]: audit 2023-08-28T02:02:13.573783+0000 mon.b (mon.1) 813 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:02:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:14 smithi130 bash[27579]: audit 2023-08-28T02:02:13.576125+0000 mon.b (mon.1) 814 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:02:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:14 smithi130 bash[27579]: audit 2023-08-28T02:02:13.584730+0000 mon.a (mon.0) 4740 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:02:14.632 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:14 smithi006 bash[33918]: cluster 2023-08-28T02:02:13.003726+0000 mgr.y (mgr.14901) 7308 : cluster [DBG] pgmap v7315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:14.633 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:14 smithi006 bash[33918]: audit 2023-08-28T02:02:13.270501+0000 mon.a (mon.0) 4739 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:02:14.633 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:14 smithi006 bash[33918]: audit 2023-08-28T02:02:13.573783+0000 mon.b (mon.1) 813 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:02:14.633 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:14 smithi006 bash[33918]: audit 2023-08-28T02:02:13.576125+0000 mon.b (mon.1) 814 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:02:14.633 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:14 smithi006 bash[33918]: audit 2023-08-28T02:02:13.584730+0000 mon.a (mon.0) 4740 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:02:14.633 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:14 smithi006 bash[27292]: cluster 2023-08-28T02:02:13.003726+0000 mgr.y (mgr.14901) 7308 : cluster [DBG] pgmap v7315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:14.633 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:14 smithi006 bash[27292]: audit 2023-08-28T02:02:13.270501+0000 mon.a (mon.0) 4739 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:02:14.633 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:14 smithi006 bash[27292]: audit 2023-08-28T02:02:13.573783+0000 mon.b (mon.1) 813 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:02:14.633 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:14 smithi006 bash[27292]: audit 2023-08-28T02:02:13.576125+0000 mon.b (mon.1) 814 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:02:14.633 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:14 smithi006 bash[27292]: audit 2023-08-28T02:02:13.584730+0000 mon.a (mon.0) 4740 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:02:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:02:14.633Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:02:14.635Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:15.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:15 smithi130 bash[27579]: audit 2023-08-28T02:02:14.754106+0000 mon.a (mon.0) 4741 : audit [DBG] from='client.? 172.21.15.6:0/1593385376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:15.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:15 smithi130 bash[51302]: debug 2023-08-28T02:02:15.199+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:15.203351+0000) 2023-08-28T02:02:15.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:15 smithi006 bash[33918]: audit 2023-08-28T02:02:14.754106+0000 mon.a (mon.0) 4741 : audit [DBG] from='client.? 172.21.15.6:0/1593385376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:15.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:15 smithi006 bash[27292]: audit 2023-08-28T02:02:14.754106+0000 mon.a (mon.0) 4741 : audit [DBG] from='client.? 172.21.15.6:0/1593385376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:16.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:16 smithi130 bash[27579]: cluster 2023-08-28T02:02:15.004917+0000 mgr.y (mgr.14901) 7309 : cluster [DBG] pgmap v7316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:16.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:16 smithi130 bash[51302]: debug 2023-08-28T02:02:16.199+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:16.203633+0000) 2023-08-28T02:02:16.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:16 smithi006 bash[27292]: cluster 2023-08-28T02:02:15.004917+0000 mgr.y (mgr.14901) 7309 : cluster [DBG] pgmap v7316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:16.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:16 smithi006 bash[33918]: cluster 2023-08-28T02:02:15.004917+0000 mgr.y (mgr.14901) 7309 : cluster [DBG] pgmap v7316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:17.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:17 smithi130 bash[27579]: audit 2023-08-28T02:02:17.166611+0000 mon.c (mon.2) 2867 : audit [DBG] from='client.? 172.21.15.6:0/392945656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:17.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:17 smithi130 bash[51302]: debug 2023-08-28T02:02:17.199+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:17.203906+0000) 2023-08-28T02:02:17.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:17 smithi006 bash[33918]: audit 2023-08-28T02:02:17.166611+0000 mon.c (mon.2) 2867 : audit [DBG] from='client.? 172.21.15.6:0/392945656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:17.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:17 smithi006 bash[27292]: audit 2023-08-28T02:02:17.166611+0000 mon.c (mon.2) 2867 : audit [DBG] from='client.? 172.21.15.6:0/392945656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:18.553 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:18 smithi130 bash[51302]: debug 2023-08-28T02:02:18.199+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:18.204227+0000) 2023-08-28T02:02:18.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:18 smithi130 bash[27579]: cluster 2023-08-28T02:02:17.005519+0000 mgr.y (mgr.14901) 7310 : cluster [DBG] pgmap v7317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:18.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:18 smithi006 bash[33918]: cluster 2023-08-28T02:02:17.005519+0000 mgr.y (mgr.14901) 7310 : cluster [DBG] pgmap v7317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:18.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:18 smithi006 bash[27292]: cluster 2023-08-28T02:02:17.005519+0000 mgr.y (mgr.14901) 7310 : cluster [DBG] pgmap v7317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:19.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:19 smithi130 bash[51302]: debug 2023-08-28T02:02:19.199+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:19.204488+0000) 2023-08-28T02:02:20.557 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:20 smithi130 bash[51302]: debug 2023-08-28T02:02:20.199+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:20.204791+0000) 2023-08-28T02:02:20.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:20 smithi130 bash[27579]: cluster 2023-08-28T02:02:19.006306+0000 mgr.y (mgr.14901) 7311 : cluster [DBG] pgmap v7318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:20.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:20 smithi130 bash[27579]: audit 2023-08-28T02:02:19.582169+0000 mon.c (mon.2) 2868 : audit [DBG] from='client.? 172.21.15.6:0/1070747486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:20.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:02:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:02:20] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:02:20.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:20 smithi006 bash[27292]: cluster 2023-08-28T02:02:19.006306+0000 mgr.y (mgr.14901) 7311 : cluster [DBG] pgmap v7318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:20.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:20 smithi006 bash[27292]: audit 2023-08-28T02:02:19.582169+0000 mon.c (mon.2) 2868 : audit [DBG] from='client.? 172.21.15.6:0/1070747486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:20.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:20 smithi006 bash[33918]: cluster 2023-08-28T02:02:19.006306+0000 mgr.y (mgr.14901) 7311 : cluster [DBG] pgmap v7318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:20.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:20 smithi006 bash[33918]: audit 2023-08-28T02:02:19.582169+0000 mon.c (mon.2) 2868 : audit [DBG] from='client.? 172.21.15.6:0/1070747486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:02:21.191Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:02:21.193Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:21.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:21 smithi130 bash[51302]: debug 2023-08-28T02:02:21.199+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:21.205110+0000) 2023-08-28T02:02:22.563 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:22 smithi130 bash[51302]: debug 2023-08-28T02:02:22.199+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:22.205412+0000) 2023-08-28T02:02:22.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:22 smithi130 bash[27579]: cluster 2023-08-28T02:02:21.007592+0000 mgr.y (mgr.14901) 7312 : cluster [DBG] pgmap v7319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:22 smithi130 bash[27579]: audit 2023-08-28T02:02:21.985790+0000 mon.c (mon.2) 2869 : audit [DBG] from='client.? 172.21.15.6:0/1634660738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:22.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:22 smithi006 bash[27292]: cluster 2023-08-28T02:02:21.007592+0000 mgr.y (mgr.14901) 7312 : cluster [DBG] pgmap v7319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:22.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:22 smithi006 bash[27292]: audit 2023-08-28T02:02:21.985790+0000 mon.c (mon.2) 2869 : audit [DBG] from='client.? 172.21.15.6:0/1634660738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:02:22.755Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:02:22.757Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:22.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:22 smithi006 bash[33918]: cluster 2023-08-28T02:02:21.007592+0000 mgr.y (mgr.14901) 7312 : cluster [DBG] pgmap v7319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:22.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:22 smithi006 bash[33918]: audit 2023-08-28T02:02:21.985790+0000 mon.c (mon.2) 2869 : audit [DBG] from='client.? 172.21.15.6:0/1634660738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:23.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:23 smithi130 bash[51302]: debug 2023-08-28T02:02:23.199+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:23.205670+0000) 2023-08-28T02:02:24.575 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:24 smithi130 bash[51302]: debug 2023-08-28T02:02:24.199+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:24.205944+0000) 2023-08-28T02:02:24.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:24 smithi130 bash[27579]: cluster 2023-08-28T02:02:23.008418+0000 mgr.y (mgr.14901) 7313 : cluster [DBG] pgmap v7320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:24.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:24 smithi130 bash[27579]: audit 2023-08-28T02:02:24.400981+0000 mon.c (mon.2) 2870 : audit [DBG] from='client.? 172.21.15.6:0/1633103988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:24.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:24 smithi006 bash[33918]: cluster 2023-08-28T02:02:23.008418+0000 mgr.y (mgr.14901) 7313 : cluster [DBG] pgmap v7320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:24.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:24 smithi006 bash[33918]: audit 2023-08-28T02:02:24.400981+0000 mon.c (mon.2) 2870 : audit [DBG] from='client.? 172.21.15.6:0/1633103988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:02:24.633Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:02:24.636Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:24.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:24 smithi006 bash[27292]: cluster 2023-08-28T02:02:23.008418+0000 mgr.y (mgr.14901) 7313 : cluster [DBG] pgmap v7320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:24.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:24 smithi006 bash[27292]: audit 2023-08-28T02:02:24.400981+0000 mon.c (mon.2) 2870 : audit [DBG] from='client.? 172.21.15.6:0/1633103988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:25.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:25 smithi130 bash[51302]: debug 2023-08-28T02:02:25.203+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:25.206226+0000) 2023-08-28T02:02:25.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:25 smithi006 bash[33918]: cluster 2023-08-28T02:02:25.009655+0000 mgr.y (mgr.14901) 7314 : cluster [DBG] pgmap v7321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:25.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:25 smithi006 bash[27292]: cluster 2023-08-28T02:02:25.009655+0000 mgr.y (mgr.14901) 7314 : cluster [DBG] pgmap v7321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:26.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:25 smithi130 bash[27579]: cluster 2023-08-28T02:02:25.009655+0000 mgr.y (mgr.14901) 7314 : cluster [DBG] pgmap v7321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:26.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:26 smithi130 bash[51302]: debug 2023-08-28T02:02:26.199+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:26.206555+0000) 2023-08-28T02:02:27.204 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:26 smithi130 bash[27579]: audit 2023-08-28T02:02:26.821655+0000 mon.c (mon.2) 2871 : audit [DBG] from='client.? 172.21.15.6:0/1264000587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:27.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:26 smithi006 bash[27292]: audit 2023-08-28T02:02:26.821655+0000 mon.c (mon.2) 2871 : audit [DBG] from='client.? 172.21.15.6:0/1264000587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:27.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:26 smithi006 bash[33918]: audit 2023-08-28T02:02:26.821655+0000 mon.c (mon.2) 2871 : audit [DBG] from='client.? 172.21.15.6:0/1264000587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:27.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:27 smithi130 bash[51302]: debug 2023-08-28T02:02:27.203+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:27.206832+0000) 2023-08-28T02:02:28.575 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:28 smithi130 bash[51302]: debug 2023-08-28T02:02:28.203+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:28.207147+0000) 2023-08-28T02:02:28.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:28 smithi130 bash[27579]: cluster 2023-08-28T02:02:27.010370+0000 mgr.y (mgr.14901) 7315 : cluster [DBG] pgmap v7322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:28.963 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:28 smithi006 bash[33918]: cluster 2023-08-28T02:02:27.010370+0000 mgr.y (mgr.14901) 7315 : cluster [DBG] pgmap v7322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:28.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:28 smithi006 bash[27292]: cluster 2023-08-28T02:02:27.010370+0000 mgr.y (mgr.14901) 7315 : cluster [DBG] pgmap v7322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:29.583 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:29 smithi130 bash[51302]: debug 2023-08-28T02:02:29.203+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:29.207478+0000) 2023-08-28T02:02:29.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:29 smithi130 bash[27579]: audit 2023-08-28T02:02:29.235415+0000 mon.a (mon.0) 4742 : audit [DBG] from='client.? 172.21.15.6:0/1277388256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:29.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:29 smithi006 bash[27292]: audit 2023-08-28T02:02:29.235415+0000 mon.a (mon.0) 4742 : audit [DBG] from='client.? 172.21.15.6:0/1277388256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:29.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:29 smithi006 bash[33918]: audit 2023-08-28T02:02:29.235415+0000 mon.a (mon.0) 4742 : audit [DBG] from='client.? 172.21.15.6:0/1277388256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:30.585 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:30 smithi130 bash[51302]: debug 2023-08-28T02:02:30.203+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:30.207754+0000) 2023-08-28T02:02:30.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:30 smithi130 bash[27579]: cluster 2023-08-28T02:02:29.011019+0000 mgr.y (mgr.14901) 7316 : cluster [DBG] pgmap v7323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:30.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:02:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:02:30] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T02:02:30.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:30 smithi006 bash[27292]: cluster 2023-08-28T02:02:29.011019+0000 mgr.y (mgr.14901) 7316 : cluster [DBG] pgmap v7323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:30.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:30 smithi006 bash[33918]: cluster 2023-08-28T02:02:29.011019+0000 mgr.y (mgr.14901) 7316 : cluster [DBG] pgmap v7323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:02:31.192Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:02:31.194Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:31.593 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:31 smithi130 bash[51302]: debug 2023-08-28T02:02:31.207+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:31.208139+0000) 2023-08-28T02:02:31.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:31 smithi130 bash[27579]: cluster 2023-08-28T02:02:31.012135+0000 mgr.y (mgr.14901) 7317 : cluster [DBG] pgmap v7324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:31.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:31 smithi006 bash[27292]: cluster 2023-08-28T02:02:31.012135+0000 mgr.y (mgr.14901) 7317 : cluster [DBG] pgmap v7324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:31.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:31 smithi006 bash[33918]: cluster 2023-08-28T02:02:31.012135+0000 mgr.y (mgr.14901) 7317 : cluster [DBG] pgmap v7324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:32.595 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:32 smithi130 bash[51302]: debug 2023-08-28T02:02:32.203+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:32.208420+0000) 2023-08-28T02:02:32.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:32 smithi130 bash[27579]: audit 2023-08-28T02:02:31.651231+0000 mon.a (mon.0) 4743 : audit [DBG] from='client.? 172.21.15.6:0/800508626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:32.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:32 smithi006 bash[33918]: audit 2023-08-28T02:02:31.651231+0000 mon.a (mon.0) 4743 : audit [DBG] from='client.? 172.21.15.6:0/800508626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:32.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:32 smithi006 bash[27292]: audit 2023-08-28T02:02:31.651231+0000 mon.a (mon.0) 4743 : audit [DBG] from='client.? 172.21.15.6:0/800508626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:02:32.755Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:02:32.757Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:33.604 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:33 smithi130 bash[51302]: debug 2023-08-28T02:02:33.203+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:33.208696+0000) 2023-08-28T02:02:33.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:33 smithi130 bash[27579]: cluster 2023-08-28T02:02:33.012808+0000 mgr.y (mgr.14901) 7318 : cluster [DBG] pgmap v7325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:33 smithi006 bash[27292]: cluster 2023-08-28T02:02:33.012808+0000 mgr.y (mgr.14901) 7318 : cluster [DBG] pgmap v7325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:33.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:33 smithi006 bash[33918]: cluster 2023-08-28T02:02:33.012808+0000 mgr.y (mgr.14901) 7318 : cluster [DBG] pgmap v7325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:34.605 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:34 smithi130 bash[51302]: debug 2023-08-28T02:02:34.203+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:34.208977+0000) 2023-08-28T02:02:34.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:34 smithi130 bash[27579]: audit 2023-08-28T02:02:34.069789+0000 mon.a (mon.0) 4744 : audit [DBG] from='client.? 172.21.15.6:0/4129880525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:34.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:34 smithi006 bash[27292]: audit 2023-08-28T02:02:34.069789+0000 mon.a (mon.0) 4744 : audit [DBG] from='client.? 172.21.15.6:0/4129880525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:34.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:34 smithi006 bash[33918]: audit 2023-08-28T02:02:34.069789+0000 mon.a (mon.0) 4744 : audit [DBG] from='client.? 172.21.15.6:0/4129880525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:02:34.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:02:34.637Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:35.613 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:35 smithi130 bash[51302]: debug 2023-08-28T02:02:35.207+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:35.209257+0000) 2023-08-28T02:02:35.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:35 smithi130 bash[27579]: cluster 2023-08-28T02:02:35.013988+0000 mgr.y (mgr.14901) 7319 : cluster [DBG] pgmap v7326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:35.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:35 smithi006 bash[27292]: cluster 2023-08-28T02:02:35.013988+0000 mgr.y (mgr.14901) 7319 : cluster [DBG] pgmap v7326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:35.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:35 smithi006 bash[33918]: cluster 2023-08-28T02:02:35.013988+0000 mgr.y (mgr.14901) 7319 : cluster [DBG] pgmap v7326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:36.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:36 smithi130 bash[51302]: debug 2023-08-28T02:02:36.203+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:36.209550+0000) 2023-08-28T02:02:36.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:36 smithi006 bash[27292]: audit 2023-08-28T02:02:36.483758+0000 mon.c (mon.2) 2872 : audit [DBG] from='client.? 172.21.15.6:0/3850415110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:36.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:36 smithi006 bash[33918]: audit 2023-08-28T02:02:36.483758+0000 mon.c (mon.2) 2872 : audit [DBG] from='client.? 172.21.15.6:0/3850415110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:37.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:36 smithi130 bash[27579]: audit 2023-08-28T02:02:36.483758+0000 mon.c (mon.2) 2872 : audit [DBG] from='client.? 172.21.15.6:0/3850415110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:37.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:37 smithi130 bash[51302]: debug 2023-08-28T02:02:37.207+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:37.209839+0000) 2023-08-28T02:02:38.570 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:38 smithi130 bash[51302]: debug 2023-08-28T02:02:38.207+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:38.210137+0000) 2023-08-28T02:02:38.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:38 smithi130 bash[27579]: cluster 2023-08-28T02:02:37.014658+0000 mgr.y (mgr.14901) 7320 : cluster [DBG] pgmap v7327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:38.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:38 smithi006 bash[27292]: cluster 2023-08-28T02:02:37.014658+0000 mgr.y (mgr.14901) 7320 : cluster [DBG] pgmap v7327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:38.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:38 smithi006 bash[33918]: cluster 2023-08-28T02:02:37.014658+0000 mgr.y (mgr.14901) 7320 : cluster [DBG] pgmap v7327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:39.578 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:39 smithi130 bash[51302]: debug 2023-08-28T02:02:39.203+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:39.210396+0000) 2023-08-28T02:02:39.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:39 smithi130 bash[27579]: audit 2023-08-28T02:02:38.902638+0000 mon.c (mon.2) 2873 : audit [DBG] from='client.? 172.21.15.6:0/2264810185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:39.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:39 smithi006 bash[33918]: audit 2023-08-28T02:02:38.902638+0000 mon.c (mon.2) 2873 : audit [DBG] from='client.? 172.21.15.6:0/2264810185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:39.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:39 smithi006 bash[27292]: audit 2023-08-28T02:02:38.902638+0000 mon.c (mon.2) 2873 : audit [DBG] from='client.? 172.21.15.6:0/2264810185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:40.582 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:40 smithi130 bash[51302]: debug 2023-08-28T02:02:40.203+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:40.210681+0000) 2023-08-28T02:02:40.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:40 smithi130 bash[27579]: cluster 2023-08-28T02:02:39.015320+0000 mgr.y (mgr.14901) 7321 : cluster [DBG] pgmap v7328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:40.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:02:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:02:40] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T02:02:40.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:40 smithi006 bash[33918]: cluster 2023-08-28T02:02:39.015320+0000 mgr.y (mgr.14901) 7321 : cluster [DBG] pgmap v7328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:40.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:40 smithi006 bash[27292]: cluster 2023-08-28T02:02:39.015320+0000 mgr.y (mgr.14901) 7321 : cluster [DBG] pgmap v7328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:02:41.193Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:02:41.195Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:41.590 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:41 smithi130 bash[51302]: debug 2023-08-28T02:02:41.207+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:41.210980+0000) 2023-08-28T02:02:41.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:41 smithi130 bash[27579]: cluster 2023-08-28T02:02:41.016387+0000 mgr.y (mgr.14901) 7322 : cluster [DBG] pgmap v7329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:41.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:41 smithi130 bash[27579]: audit 2023-08-28T02:02:41.323296+0000 mon.c (mon.2) 2874 : audit [DBG] from='client.? 172.21.15.6:0/963408688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:41.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:41 smithi006 bash[27292]: cluster 2023-08-28T02:02:41.016387+0000 mgr.y (mgr.14901) 7322 : cluster [DBG] pgmap v7329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:41.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:41 smithi006 bash[27292]: audit 2023-08-28T02:02:41.323296+0000 mon.c (mon.2) 2874 : audit [DBG] from='client.? 172.21.15.6:0/963408688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:41.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:41 smithi006 bash[33918]: cluster 2023-08-28T02:02:41.016387+0000 mgr.y (mgr.14901) 7322 : cluster [DBG] pgmap v7329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:41.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:41 smithi006 bash[33918]: audit 2023-08-28T02:02:41.323296+0000 mon.c (mon.2) 2874 : audit [DBG] from='client.? 172.21.15.6:0/963408688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:42.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:42 smithi130 bash[51302]: debug 2023-08-28T02:02:42.207+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:42.211257+0000) 2023-08-28T02:02:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:02:42.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:02:42.758Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:43.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:43 smithi130 bash[51302]: debug 2023-08-28T02:02:43.207+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:43.211485+0000) 2023-08-28T02:02:44.574 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:44 smithi130 bash[51302]: debug 2023-08-28T02:02:44.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:44.211761+0000) 2023-08-28T02:02:44.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:44 smithi130 bash[27579]: cluster 2023-08-28T02:02:43.017083+0000 mgr.y (mgr.14901) 7323 : cluster [DBG] pgmap v7330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:44.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:44 smithi130 bash[27579]: audit 2023-08-28T02:02:43.737192+0000 mon.c (mon.2) 2875 : audit [DBG] from='client.? 172.21.15.6:0/259546778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:44.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:44 smithi006 bash[27292]: cluster 2023-08-28T02:02:43.017083+0000 mgr.y (mgr.14901) 7323 : cluster [DBG] pgmap v7330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:44.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:44 smithi006 bash[27292]: audit 2023-08-28T02:02:43.737192+0000 mon.c (mon.2) 2875 : audit [DBG] from='client.? 172.21.15.6:0/259546778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:44.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:44 smithi006 bash[33918]: cluster 2023-08-28T02:02:43.017083+0000 mgr.y (mgr.14901) 7323 : cluster [DBG] pgmap v7330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:44.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:44 smithi006 bash[33918]: audit 2023-08-28T02:02:43.737192+0000 mon.c (mon.2) 2875 : audit [DBG] from='client.? 172.21.15.6:0/259546778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:02:44.635Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:02:44.637Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:45.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:45 smithi130 bash[51302]: debug 2023-08-28T02:02:45.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:45.212047+0000) 2023-08-28T02:02:46.577 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:46 smithi130 bash[51302]: debug 2023-08-28T02:02:46.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:46.212354+0000) 2023-08-28T02:02:46.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:46 smithi130 bash[27579]: cluster 2023-08-28T02:02:45.018257+0000 mgr.y (mgr.14901) 7324 : cluster [DBG] pgmap v7331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:46.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:46 smithi130 bash[27579]: audit 2023-08-28T02:02:46.145862+0000 mon.a (mon.0) 4745 : audit [DBG] from='client.? 172.21.15.6:0/579562416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:46.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:46 smithi006 bash[27292]: cluster 2023-08-28T02:02:45.018257+0000 mgr.y (mgr.14901) 7324 : cluster [DBG] pgmap v7331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:46.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:46 smithi006 bash[27292]: audit 2023-08-28T02:02:46.145862+0000 mon.a (mon.0) 4745 : audit [DBG] from='client.? 172.21.15.6:0/579562416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:46.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:46 smithi006 bash[33918]: cluster 2023-08-28T02:02:45.018257+0000 mgr.y (mgr.14901) 7324 : cluster [DBG] pgmap v7331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:46.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:46 smithi006 bash[33918]: audit 2023-08-28T02:02:46.145862+0000 mon.a (mon.0) 4745 : audit [DBG] from='client.? 172.21.15.6:0/579562416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:47.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:47 smithi130 bash[51302]: debug 2023-08-28T02:02:47.207+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:47.212649+0000) 2023-08-28T02:02:48.572 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:48 smithi130 bash[51302]: debug 2023-08-28T02:02:48.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:48.212924+0000) 2023-08-28T02:02:48.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:48 smithi130 bash[27579]: cluster 2023-08-28T02:02:47.018919+0000 mgr.y (mgr.14901) 7325 : cluster [DBG] pgmap v7332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:48.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:48 smithi130 bash[27579]: audit 2023-08-28T02:02:48.559470+0000 mon.a (mon.0) 4746 : audit [DBG] from='client.? 172.21.15.6:0/2817148339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:48.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:48 smithi006 bash[33918]: cluster 2023-08-28T02:02:47.018919+0000 mgr.y (mgr.14901) 7325 : cluster [DBG] pgmap v7332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:48.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:48 smithi006 bash[33918]: audit 2023-08-28T02:02:48.559470+0000 mon.a (mon.0) 4746 : audit [DBG] from='client.? 172.21.15.6:0/2817148339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:48.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:48 smithi006 bash[27292]: cluster 2023-08-28T02:02:47.018919+0000 mgr.y (mgr.14901) 7325 : cluster [DBG] pgmap v7332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:48.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:48 smithi006 bash[27292]: audit 2023-08-28T02:02:48.559470+0000 mon.a (mon.0) 4746 : audit [DBG] from='client.? 172.21.15.6:0/2817148339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:49.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:49 smithi130 bash[51302]: debug 2023-08-28T02:02:49.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:49.213200+0000) 2023-08-28T02:02:50.577 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:50 smithi130 bash[51302]: debug 2023-08-28T02:02:50.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:50.213482+0000) 2023-08-28T02:02:50.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:50 smithi130 bash[27579]: cluster 2023-08-28T02:02:49.019606+0000 mgr.y (mgr.14901) 7326 : cluster [DBG] pgmap v7333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:50.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:50 smithi006 bash[27292]: cluster 2023-08-28T02:02:49.019606+0000 mgr.y (mgr.14901) 7326 : cluster [DBG] pgmap v7333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:50.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:02:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:02:50] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:02:50.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:50 smithi006 bash[33918]: cluster 2023-08-28T02:02:49.019606+0000 mgr.y (mgr.14901) 7326 : cluster [DBG] pgmap v7333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:02:51.194Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:02:51.195Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:51.587 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:51 smithi130 bash[51302]: debug 2023-08-28T02:02:51.207+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:51.213818+0000) 2023-08-28T02:02:51.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:51 smithi130 bash[27579]: audit 2023-08-28T02:02:50.966872+0000 mon.a (mon.0) 4747 : audit [DBG] from='client.? 172.21.15.6:0/3285143612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:51.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:51 smithi006 bash[27292]: audit 2023-08-28T02:02:50.966872+0000 mon.a (mon.0) 4747 : audit [DBG] from='client.? 172.21.15.6:0/3285143612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:51.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:51 smithi006 bash[33918]: audit 2023-08-28T02:02:50.966872+0000 mon.a (mon.0) 4747 : audit [DBG] from='client.? 172.21.15.6:0/3285143612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:52.589 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:52 smithi130 bash[51302]: debug 2023-08-28T02:02:52.207+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:52.214118+0000) 2023-08-28T02:02:52.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:52 smithi130 bash[27579]: cluster 2023-08-28T02:02:51.020728+0000 mgr.y (mgr.14901) 7327 : cluster [DBG] pgmap v7334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:52.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:52 smithi006 bash[33918]: cluster 2023-08-28T02:02:51.020728+0000 mgr.y (mgr.14901) 7327 : cluster [DBG] pgmap v7334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:52.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:52 smithi006 bash[27292]: cluster 2023-08-28T02:02:51.020728+0000 mgr.y (mgr.14901) 7327 : cluster [DBG] pgmap v7334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:52.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:02:52.757Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:02:52.759Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:53.598 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:53 smithi130 bash[51302]: debug 2023-08-28T02:02:53.207+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:53.214394+0000) 2023-08-28T02:02:53.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:53 smithi130 bash[27579]: cluster 2023-08-28T02:02:53.021210+0000 mgr.y (mgr.14901) 7328 : cluster [DBG] pgmap v7335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:53.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:53 smithi130 bash[27579]: audit 2023-08-28T02:02:53.383821+0000 mon.c (mon.2) 2876 : audit [DBG] from='client.? 172.21.15.6:0/197475464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:53.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:53 smithi006 bash[33918]: cluster 2023-08-28T02:02:53.021210+0000 mgr.y (mgr.14901) 7328 : cluster [DBG] pgmap v7335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:53.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:53 smithi006 bash[33918]: audit 2023-08-28T02:02:53.383821+0000 mon.c (mon.2) 2876 : audit [DBG] from='client.? 172.21.15.6:0/197475464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:53.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:53 smithi006 bash[27292]: cluster 2023-08-28T02:02:53.021210+0000 mgr.y (mgr.14901) 7328 : cluster [DBG] pgmap v7335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:53.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:53 smithi006 bash[27292]: audit 2023-08-28T02:02:53.383821+0000 mon.c (mon.2) 2876 : audit [DBG] from='client.? 172.21.15.6:0/197475464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:54.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:54 smithi130 bash[51302]: debug 2023-08-28T02:02:54.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:54.214680+0000) 2023-08-28T02:02:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:02:54.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:02:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:02:54.638Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:02:55.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:55 smithi130 bash[51302]: debug 2023-08-28T02:02:55.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:55.214975+0000) 2023-08-28T02:02:56.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:56 smithi130 bash[27579]: cluster 2023-08-28T02:02:55.022336+0000 mgr.y (mgr.14901) 7329 : cluster [DBG] pgmap v7336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:56.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:56 smithi130 bash[27579]: audit 2023-08-28T02:02:55.808530+0000 mon.c (mon.2) 2877 : audit [DBG] from='client.? 172.21.15.6:0/941833721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:56.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:56 smithi130 bash[51302]: debug 2023-08-28T02:02:56.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:56.215289+0000) 2023-08-28T02:02:56.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:56 smithi006 bash[33918]: cluster 2023-08-28T02:02:55.022336+0000 mgr.y (mgr.14901) 7329 : cluster [DBG] pgmap v7336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:56.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:56 smithi006 bash[33918]: audit 2023-08-28T02:02:55.808530+0000 mon.c (mon.2) 2877 : audit [DBG] from='client.? 172.21.15.6:0/941833721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:56.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:56 smithi006 bash[27292]: cluster 2023-08-28T02:02:55.022336+0000 mgr.y (mgr.14901) 7329 : cluster [DBG] pgmap v7336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:56.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:56 smithi006 bash[27292]: audit 2023-08-28T02:02:55.808530+0000 mon.c (mon.2) 2877 : audit [DBG] from='client.? 172.21.15.6:0/941833721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:57.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:57 smithi130 bash[51302]: debug 2023-08-28T02:02:57.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:57.215570+0000) 2023-08-28T02:02:58.575 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:58 smithi130 bash[51302]: debug 2023-08-28T02:02:58.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:58.215823+0000) 2023-08-28T02:02:58.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:58 smithi130 bash[27579]: cluster 2023-08-28T02:02:57.023068+0000 mgr.y (mgr.14901) 7330 : cluster [DBG] pgmap v7337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:58.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:02:58 smithi130 bash[27579]: audit 2023-08-28T02:02:58.221279+0000 mon.a (mon.0) 4748 : audit [DBG] from='client.? 172.21.15.6:0/3584357549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:58.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:58 smithi006 bash[33918]: cluster 2023-08-28T02:02:57.023068+0000 mgr.y (mgr.14901) 7330 : cluster [DBG] pgmap v7337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:58.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:02:58 smithi006 bash[33918]: audit 2023-08-28T02:02:58.221279+0000 mon.a (mon.0) 4748 : audit [DBG] from='client.? 172.21.15.6:0/3584357549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:58.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:58 smithi006 bash[27292]: cluster 2023-08-28T02:02:57.023068+0000 mgr.y (mgr.14901) 7330 : cluster [DBG] pgmap v7337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:02:58.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:02:58 smithi006 bash[27292]: audit 2023-08-28T02:02:58.221279+0000 mon.a (mon.0) 4748 : audit [DBG] from='client.? 172.21.15.6:0/3584357549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:59.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:02:59 smithi130 bash[51302]: debug 2023-08-28T02:02:59.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:59.216127+0000) 2023-08-28T02:03:00.580 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:00 smithi130 bash[51302]: debug 2023-08-28T02:03:00.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:00.216382+0000) 2023-08-28T02:03:00.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:00 smithi130 bash[27579]: cluster 2023-08-28T02:02:59.023731+0000 mgr.y (mgr.14901) 7331 : cluster [DBG] pgmap v7338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:00.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:03:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:03:00] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:03:00.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:00 smithi006 bash[27292]: cluster 2023-08-28T02:02:59.023731+0000 mgr.y (mgr.14901) 7331 : cluster [DBG] pgmap v7338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:00.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:00 smithi006 bash[33918]: cluster 2023-08-28T02:02:59.023731+0000 mgr.y (mgr.14901) 7331 : cluster [DBG] pgmap v7338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:03:01.194Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:03:01.196Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:01.590 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:01 smithi130 bash[51302]: debug 2023-08-28T02:03:01.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:01.216672+0000) 2023-08-28T02:03:01.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:01 smithi130 bash[27579]: audit 2023-08-28T02:03:00.644381+0000 mon.c (mon.2) 2878 : audit [DBG] from='client.? 172.21.15.6:0/3912714780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:01.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:01 smithi006 bash[27292]: audit 2023-08-28T02:03:00.644381+0000 mon.c (mon.2) 2878 : audit [DBG] from='client.? 172.21.15.6:0/3912714780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:01.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:01 smithi006 bash[33918]: audit 2023-08-28T02:03:00.644381+0000 mon.c (mon.2) 2878 : audit [DBG] from='client.? 172.21.15.6:0/3912714780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:02.595 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:02 smithi130 bash[51302]: debug 2023-08-28T02:03:02.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:02.216998+0000) 2023-08-28T02:03:02.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:02 smithi130 bash[27579]: cluster 2023-08-28T02:03:01.024915+0000 mgr.y (mgr.14901) 7332 : cluster [DBG] pgmap v7339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:02.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:02 smithi006 bash[33918]: cluster 2023-08-28T02:03:01.024915+0000 mgr.y (mgr.14901) 7332 : cluster [DBG] pgmap v7339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:02.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:02 smithi006 bash[27292]: cluster 2023-08-28T02:03:01.024915+0000 mgr.y (mgr.14901) 7332 : cluster [DBG] pgmap v7339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:03:02.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:03:02.760Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:03.605 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:03 smithi130 bash[51302]: debug 2023-08-28T02:03:03.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:03.217301+0000) 2023-08-28T02:03:03.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:03 smithi130 bash[27579]: cluster 2023-08-28T02:03:03.025546+0000 mgr.y (mgr.14901) 7333 : cluster [DBG] pgmap v7340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:03.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:03 smithi130 bash[27579]: audit 2023-08-28T02:03:03.060094+0000 mon.a (mon.0) 4749 : audit [DBG] from='client.? 172.21.15.6:0/2688774234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:03.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:03 smithi006 bash[27292]: cluster 2023-08-28T02:03:03.025546+0000 mgr.y (mgr.14901) 7333 : cluster [DBG] pgmap v7340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:03.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:03 smithi006 bash[27292]: audit 2023-08-28T02:03:03.060094+0000 mon.a (mon.0) 4749 : audit [DBG] from='client.? 172.21.15.6:0/2688774234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:03.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:03 smithi006 bash[33918]: cluster 2023-08-28T02:03:03.025546+0000 mgr.y (mgr.14901) 7333 : cluster [DBG] pgmap v7340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:03.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:03 smithi006 bash[33918]: audit 2023-08-28T02:03:03.060094+0000 mon.a (mon.0) 4749 : audit [DBG] from='client.? 172.21.15.6:0/2688774234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:04.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:04 smithi130 bash[51302]: debug 2023-08-28T02:03:04.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:04.217583+0000) 2023-08-28T02:03:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:03:04.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:03:04.638Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:05.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:05 smithi130 bash[51302]: debug 2023-08-28T02:03:05.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:05.217836+0000) 2023-08-28T02:03:06.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:06 smithi130 bash[51302]: debug 2023-08-28T02:03:06.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:06.218153+0000) 2023-08-28T02:03:06.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:06 smithi130 bash[27579]: cluster 2023-08-28T02:03:05.026665+0000 mgr.y (mgr.14901) 7334 : cluster [DBG] pgmap v7341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:06.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:06 smithi130 bash[27579]: audit 2023-08-28T02:03:05.471696+0000 mon.c (mon.2) 2879 : audit [DBG] from='client.? 172.21.15.6:0/2493307215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:06.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:06 smithi006 bash[27292]: cluster 2023-08-28T02:03:05.026665+0000 mgr.y (mgr.14901) 7334 : cluster [DBG] pgmap v7341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:06.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:06 smithi006 bash[27292]: audit 2023-08-28T02:03:05.471696+0000 mon.c (mon.2) 2879 : audit [DBG] from='client.? 172.21.15.6:0/2493307215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:06.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:06 smithi006 bash[33918]: cluster 2023-08-28T02:03:05.026665+0000 mgr.y (mgr.14901) 7334 : cluster [DBG] pgmap v7341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:06.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:06 smithi006 bash[33918]: audit 2023-08-28T02:03:05.471696+0000 mon.c (mon.2) 2879 : audit [DBG] from='client.? 172.21.15.6:0/2493307215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:07.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:07 smithi130 bash[51302]: debug 2023-08-28T02:03:07.215+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:07.218501+0000) 2023-08-28T02:03:08.579 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:08 smithi130 bash[51302]: debug 2023-08-28T02:03:08.211+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:08.218776+0000) 2023-08-28T02:03:08.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:08 smithi130 bash[27579]: cluster 2023-08-28T02:03:07.027367+0000 mgr.y (mgr.14901) 7335 : cluster [DBG] pgmap v7342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:08 smithi130 bash[27579]: audit 2023-08-28T02:03:07.882022+0000 mon.a (mon.0) 4750 : audit [DBG] from='client.? 172.21.15.6:0/2552712429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:08.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:08 smithi006 bash[27292]: cluster 2023-08-28T02:03:07.027367+0000 mgr.y (mgr.14901) 7335 : cluster [DBG] pgmap v7342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:08.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:08 smithi006 bash[27292]: audit 2023-08-28T02:03:07.882022+0000 mon.a (mon.0) 4750 : audit [DBG] from='client.? 172.21.15.6:0/2552712429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:08.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:08 smithi006 bash[33918]: cluster 2023-08-28T02:03:07.027367+0000 mgr.y (mgr.14901) 7335 : cluster [DBG] pgmap v7342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:08.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:08 smithi006 bash[33918]: audit 2023-08-28T02:03:07.882022+0000 mon.a (mon.0) 4750 : audit [DBG] from='client.? 172.21.15.6:0/2552712429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:09.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:09 smithi130 bash[51302]: debug 2023-08-28T02:03:09.214+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:09.219057+0000) 2023-08-28T02:03:10.583 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:10 smithi130 bash[51302]: debug 2023-08-28T02:03:10.214+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:10.219301+0000) 2023-08-28T02:03:10.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:10 smithi130 bash[27579]: cluster 2023-08-28T02:03:09.027979+0000 mgr.y (mgr.14901) 7336 : cluster [DBG] pgmap v7343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:10.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:10 smithi130 bash[27579]: audit 2023-08-28T02:03:10.290939+0000 mon.c (mon.2) 2880 : audit [DBG] from='client.? 172.21.15.6:0/1652666744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:10.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:03:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:03:10] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:03:10.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:10 smithi006 bash[27292]: cluster 2023-08-28T02:03:09.027979+0000 mgr.y (mgr.14901) 7336 : cluster [DBG] pgmap v7343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:10.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:10 smithi006 bash[27292]: audit 2023-08-28T02:03:10.290939+0000 mon.c (mon.2) 2880 : audit [DBG] from='client.? 172.21.15.6:0/1652666744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:10.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:10 smithi006 bash[33918]: cluster 2023-08-28T02:03:09.027979+0000 mgr.y (mgr.14901) 7336 : cluster [DBG] pgmap v7343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:10.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:10 smithi006 bash[33918]: audit 2023-08-28T02:03:10.290939+0000 mon.c (mon.2) 2880 : audit [DBG] from='client.? 172.21.15.6:0/1652666744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:03:11.195Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:03:11.198Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:11.595 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:11 smithi130 bash[51302]: debug 2023-08-28T02:03:11.218+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:11.219623+0000) 2023-08-28T02:03:11.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:11 smithi130 bash[27579]: cluster 2023-08-28T02:03:11.029151+0000 mgr.y (mgr.14901) 7337 : cluster [DBG] pgmap v7344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:11.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:11 smithi006 bash[27292]: cluster 2023-08-28T02:03:11.029151+0000 mgr.y (mgr.14901) 7337 : cluster [DBG] pgmap v7344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:11.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:11 smithi006 bash[33918]: cluster 2023-08-28T02:03:11.029151+0000 mgr.y (mgr.14901) 7337 : cluster [DBG] pgmap v7344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:12.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:12 smithi130 bash[51302]: debug 2023-08-28T02:03:12.214+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:12.219948+0000) 2023-08-28T02:03:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:03:12.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:03:12.761Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:13.583 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:13 smithi130 bash[51302]: debug 2023-08-28T02:03:13.218+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:13.220226+0000) 2023-08-28T02:03:13.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:13 smithi130 bash[27579]: audit 2023-08-28T02:03:12.702467+0000 mon.c (mon.2) 2881 : audit [DBG] from='client.? 172.21.15.6:0/508666996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:13.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:13 smithi006 bash[33918]: audit 2023-08-28T02:03:12.702467+0000 mon.c (mon.2) 2881 : audit [DBG] from='client.? 172.21.15.6:0/508666996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:13.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:13 smithi006 bash[27292]: audit 2023-08-28T02:03:12.702467+0000 mon.c (mon.2) 2881 : audit [DBG] from='client.? 172.21.15.6:0/508666996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:14.587 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:14 smithi130 bash[51302]: debug 2023-08-28T02:03:14.218+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:14.220534+0000) 2023-08-28T02:03:14.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:14 smithi130 bash[27579]: cluster 2023-08-28T02:03:13.029841+0000 mgr.y (mgr.14901) 7338 : cluster [DBG] pgmap v7345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:14.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:14 smithi130 bash[27579]: audit 2023-08-28T02:03:13.595826+0000 mon.b (mon.1) 815 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:03:14.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:14 smithi006 bash[27292]: cluster 2023-08-28T02:03:13.029841+0000 mgr.y (mgr.14901) 7338 : cluster [DBG] pgmap v7345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:14.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:14 smithi006 bash[27292]: audit 2023-08-28T02:03:13.595826+0000 mon.b (mon.1) 815 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:03:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:03:14.637Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:03:14.640Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:14.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:14 smithi006 bash[33918]: cluster 2023-08-28T02:03:13.029841+0000 mgr.y (mgr.14901) 7338 : cluster [DBG] pgmap v7345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:14.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:14 smithi006 bash[33918]: audit 2023-08-28T02:03:13.595826+0000 mon.b (mon.1) 815 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:03:15.595 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:15 smithi130 bash[51302]: debug 2023-08-28T02:03:15.218+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:15.220814+0000) 2023-08-28T02:03:15.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:15 smithi130 bash[27579]: cluster 2023-08-28T02:03:15.030936+0000 mgr.y (mgr.14901) 7339 : cluster [DBG] pgmap v7346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:15.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:15 smithi130 bash[27579]: audit 2023-08-28T02:03:15.116970+0000 mon.a (mon.0) 4751 : audit [DBG] from='client.? 172.21.15.6:0/231605536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:15.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:15 smithi006 bash[33918]: cluster 2023-08-28T02:03:15.030936+0000 mgr.y (mgr.14901) 7339 : cluster [DBG] pgmap v7346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:15.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:15 smithi006 bash[33918]: audit 2023-08-28T02:03:15.116970+0000 mon.a (mon.0) 4751 : audit [DBG] from='client.? 172.21.15.6:0/231605536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:15.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:15 smithi006 bash[27292]: cluster 2023-08-28T02:03:15.030936+0000 mgr.y (mgr.14901) 7339 : cluster [DBG] pgmap v7346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:15.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:15 smithi006 bash[27292]: audit 2023-08-28T02:03:15.116970+0000 mon.a (mon.0) 4751 : audit [DBG] from='client.? 172.21.15.6:0/231605536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:16.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:16 smithi130 bash[51302]: debug 2023-08-28T02:03:16.218+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:16.221124+0000) 2023-08-28T02:03:17.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:17 smithi130 bash[51302]: debug 2023-08-28T02:03:17.218+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:17.221409+0000) 2023-08-28T02:03:18.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:18 smithi130 bash[51302]: debug 2023-08-28T02:03:18.218+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:18.221717+0000) 2023-08-28T02:03:18.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:18 smithi130 bash[27579]: cluster 2023-08-28T02:03:17.031742+0000 mgr.y (mgr.14901) 7340 : cluster [DBG] pgmap v7347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:18.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:18 smithi130 bash[27579]: audit 2023-08-28T02:03:17.528411+0000 mon.a (mon.0) 4752 : audit [DBG] from='client.? 172.21.15.6:0/2218085764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:18.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:18 smithi006 bash[33918]: cluster 2023-08-28T02:03:17.031742+0000 mgr.y (mgr.14901) 7340 : cluster [DBG] pgmap v7347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:18.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:18 smithi006 bash[33918]: audit 2023-08-28T02:03:17.528411+0000 mon.a (mon.0) 4752 : audit [DBG] from='client.? 172.21.15.6:0/2218085764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:18.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:18 smithi006 bash[27292]: cluster 2023-08-28T02:03:17.031742+0000 mgr.y (mgr.14901) 7340 : cluster [DBG] pgmap v7347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:18.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:18 smithi006 bash[27292]: audit 2023-08-28T02:03:17.528411+0000 mon.a (mon.0) 4752 : audit [DBG] from='client.? 172.21.15.6:0/2218085764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:19.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:19 smithi130 bash[51302]: debug 2023-08-28T02:03:19.218+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:19.221994+0000) 2023-08-28T02:03:20.555 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:20 smithi130 bash[51302]: debug 2023-08-28T02:03:20.218+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:20.222280+0000) 2023-08-28T02:03:20.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:20 smithi130 bash[27579]: cluster 2023-08-28T02:03:19.032410+0000 mgr.y (mgr.14901) 7341 : cluster [DBG] pgmap v7348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:20 smithi130 bash[27579]: audit 2023-08-28T02:03:19.555245+0000 mon.a (mon.0) 4753 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:03:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:20 smithi130 bash[27579]: audit 2023-08-28T02:03:19.878166+0000 mon.a (mon.0) 4754 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:03:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:20 smithi130 bash[27579]: audit 2023-08-28T02:03:19.974703+0000 mon.a (mon.0) 4755 : audit [DBG] from='client.? 172.21.15.6:0/2412311612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:20 smithi130 bash[27579]: audit 2023-08-28T02:03:20.151044+0000 mon.b (mon.1) 816 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:03:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:20 smithi130 bash[27579]: audit 2023-08-28T02:03:20.153230+0000 mon.b (mon.1) 817 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:03:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:20 smithi130 bash[27579]: audit 2023-08-28T02:03:20.160321+0000 mon.a (mon.0) 4756 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:03:20.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:03:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:03:20] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:03:20.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:20 smithi006 bash[27292]: cluster 2023-08-28T02:03:19.032410+0000 mgr.y (mgr.14901) 7341 : cluster [DBG] pgmap v7348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:20.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:20 smithi006 bash[27292]: audit 2023-08-28T02:03:19.555245+0000 mon.a (mon.0) 4753 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:03:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:20 smithi006 bash[27292]: audit 2023-08-28T02:03:19.878166+0000 mon.a (mon.0) 4754 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:03:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:20 smithi006 bash[27292]: audit 2023-08-28T02:03:19.974703+0000 mon.a (mon.0) 4755 : audit [DBG] from='client.? 172.21.15.6:0/2412311612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:20 smithi006 bash[27292]: audit 2023-08-28T02:03:20.151044+0000 mon.b (mon.1) 816 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:03:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:20 smithi006 bash[27292]: audit 2023-08-28T02:03:20.153230+0000 mon.b (mon.1) 817 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:03:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:20 smithi006 bash[27292]: audit 2023-08-28T02:03:20.160321+0000 mon.a (mon.0) 4756 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:03:20.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:20 smithi006 bash[33918]: cluster 2023-08-28T02:03:19.032410+0000 mgr.y (mgr.14901) 7341 : cluster [DBG] pgmap v7348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:20.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:20 smithi006 bash[33918]: audit 2023-08-28T02:03:19.555245+0000 mon.a (mon.0) 4753 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:03:20.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:20 smithi006 bash[33918]: audit 2023-08-28T02:03:19.878166+0000 mon.a (mon.0) 4754 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:03:20.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:20 smithi006 bash[33918]: audit 2023-08-28T02:03:19.974703+0000 mon.a (mon.0) 4755 : audit [DBG] from='client.? 172.21.15.6:0/2412311612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:20.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:20 smithi006 bash[33918]: audit 2023-08-28T02:03:20.151044+0000 mon.b (mon.1) 816 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:03:20.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:20 smithi006 bash[33918]: audit 2023-08-28T02:03:20.153230+0000 mon.b (mon.1) 817 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:03:20.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:20 smithi006 bash[33918]: audit 2023-08-28T02:03:20.160321+0000 mon.a (mon.0) 4756 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:03:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:03:21.196Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:03:21.198Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:21.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:21 smithi130 bash[51302]: debug 2023-08-28T02:03:21.218+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:21.222564+0000) 2023-08-28T02:03:22.559 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:22 smithi130 bash[51302]: debug 2023-08-28T02:03:22.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:22.222887+0000) 2023-08-28T02:03:22.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:22 smithi130 bash[27579]: cluster 2023-08-28T02:03:21.033569+0000 mgr.y (mgr.14901) 7342 : cluster [DBG] pgmap v7349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:22.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:22 smithi130 bash[27579]: audit 2023-08-28T02:03:22.388971+0000 mon.c (mon.2) 2882 : audit [DBG] from='client.? 172.21.15.6:0/3931775126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:03:22.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:03:22.762Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:22.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:22 smithi006 bash[27292]: cluster 2023-08-28T02:03:21.033569+0000 mgr.y (mgr.14901) 7342 : cluster [DBG] pgmap v7349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:22.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:22 smithi006 bash[27292]: audit 2023-08-28T02:03:22.388971+0000 mon.c (mon.2) 2882 : audit [DBG] from='client.? 172.21.15.6:0/3931775126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:22.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:22 smithi006 bash[33918]: cluster 2023-08-28T02:03:21.033569+0000 mgr.y (mgr.14901) 7342 : cluster [DBG] pgmap v7349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:22.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:22 smithi006 bash[33918]: audit 2023-08-28T02:03:22.388971+0000 mon.c (mon.2) 2882 : audit [DBG] from='client.? 172.21.15.6:0/3931775126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:23.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:23 smithi130 bash[51302]: debug 2023-08-28T02:03:23.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:23.223149+0000) 2023-08-28T02:03:24.583 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:24 smithi130 bash[51302]: debug 2023-08-28T02:03:24.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:24.223462+0000) 2023-08-28T02:03:24.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:24 smithi130 bash[27579]: cluster 2023-08-28T02:03:23.034217+0000 mgr.y (mgr.14901) 7343 : cluster [DBG] pgmap v7350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:24.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:24 smithi006 bash[27292]: cluster 2023-08-28T02:03:23.034217+0000 mgr.y (mgr.14901) 7343 : cluster [DBG] pgmap v7350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:24.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:24 smithi006 bash[33918]: cluster 2023-08-28T02:03:23.034217+0000 mgr.y (mgr.14901) 7343 : cluster [DBG] pgmap v7350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:03:24.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:03:24.640Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:25.590 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:25 smithi130 bash[51302]: debug 2023-08-28T02:03:25.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:25.223742+0000) 2023-08-28T02:03:25.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:25 smithi130 bash[27579]: audit 2023-08-28T02:03:24.804698+0000 mon.a (mon.0) 4757 : audit [DBG] from='client.? 172.21.15.6:0/1384887186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:25.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:25 smithi006 bash[33918]: audit 2023-08-28T02:03:24.804698+0000 mon.a (mon.0) 4757 : audit [DBG] from='client.? 172.21.15.6:0/1384887186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:25.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:25 smithi006 bash[27292]: audit 2023-08-28T02:03:24.804698+0000 mon.a (mon.0) 4757 : audit [DBG] from='client.? 172.21.15.6:0/1384887186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:26.594 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:26 smithi130 bash[51302]: debug 2023-08-28T02:03:26.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:26.224046+0000) 2023-08-28T02:03:26.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:26 smithi130 bash[27579]: cluster 2023-08-28T02:03:25.035365+0000 mgr.y (mgr.14901) 7344 : cluster [DBG] pgmap v7351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:26.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:26 smithi006 bash[27292]: cluster 2023-08-28T02:03:25.035365+0000 mgr.y (mgr.14901) 7344 : cluster [DBG] pgmap v7351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:26.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:26 smithi006 bash[33918]: cluster 2023-08-28T02:03:25.035365+0000 mgr.y (mgr.14901) 7344 : cluster [DBG] pgmap v7351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:27.602 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:27 smithi130 bash[51302]: debug 2023-08-28T02:03:27.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:27.224323+0000) 2023-08-28T02:03:27.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:27 smithi130 bash[27579]: cluster 2023-08-28T02:03:27.036009+0000 mgr.y (mgr.14901) 7345 : cluster [DBG] pgmap v7352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:27.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:27 smithi130 bash[27579]: audit 2023-08-28T02:03:27.219283+0000 mon.a (mon.0) 4758 : audit [DBG] from='client.? 172.21.15.6:0/4072559540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:27.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:27 smithi006 bash[27292]: cluster 2023-08-28T02:03:27.036009+0000 mgr.y (mgr.14901) 7345 : cluster [DBG] pgmap v7352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:27.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:27 smithi006 bash[27292]: audit 2023-08-28T02:03:27.219283+0000 mon.a (mon.0) 4758 : audit [DBG] from='client.? 172.21.15.6:0/4072559540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:27.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:27 smithi006 bash[33918]: cluster 2023-08-28T02:03:27.036009+0000 mgr.y (mgr.14901) 7345 : cluster [DBG] pgmap v7352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:27.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:27 smithi006 bash[33918]: audit 2023-08-28T02:03:27.219283+0000 mon.a (mon.0) 4758 : audit [DBG] from='client.? 172.21.15.6:0/4072559540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:28.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:28 smithi130 bash[51302]: debug 2023-08-28T02:03:28.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:28.224619+0000) 2023-08-28T02:03:29.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:29 smithi130 bash[51302]: debug 2023-08-28T02:03:29.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:29.224912+0000) 2023-08-28T02:03:30.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:30 smithi130 bash[27579]: cluster 2023-08-28T02:03:29.036667+0000 mgr.y (mgr.14901) 7346 : cluster [DBG] pgmap v7353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:30.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:30 smithi130 bash[27579]: audit 2023-08-28T02:03:29.644557+0000 mon.b (mon.1) 818 : audit [DBG] from='client.? 172.21.15.6:0/424185083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:30.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:30 smithi130 bash[51302]: debug 2023-08-28T02:03:30.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:30.225196+0000) 2023-08-28T02:03:30.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:30 smithi006 bash[33918]: cluster 2023-08-28T02:03:29.036667+0000 mgr.y (mgr.14901) 7346 : cluster [DBG] pgmap v7353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:30.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:30 smithi006 bash[33918]: audit 2023-08-28T02:03:29.644557+0000 mon.b (mon.1) 818 : audit [DBG] from='client.? 172.21.15.6:0/424185083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:30.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:30 smithi006 bash[27292]: cluster 2023-08-28T02:03:29.036667+0000 mgr.y (mgr.14901) 7346 : cluster [DBG] pgmap v7353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:30.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:30 smithi006 bash[27292]: audit 2023-08-28T02:03:29.644557+0000 mon.b (mon.1) 818 : audit [DBG] from='client.? 172.21.15.6:0/424185083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:31.197 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:03:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:03:30] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:03:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:03:31.197Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:03:31.199Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:31.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:31 smithi130 bash[51302]: debug 2023-08-28T02:03:31.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:31.225494+0000) 2023-08-28T02:03:32.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:32 smithi130 bash[27579]: cluster 2023-08-28T02:03:31.037831+0000 mgr.y (mgr.14901) 7347 : cluster [DBG] pgmap v7354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:32.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:32 smithi130 bash[27579]: audit 2023-08-28T02:03:32.050146+0000 mon.a (mon.0) 4759 : audit [DBG] from='client.? 172.21.15.6:0/2367660586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:32.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:32 smithi130 bash[51302]: debug 2023-08-28T02:03:32.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:32.225794+0000) 2023-08-28T02:03:32.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:32 smithi006 bash[27292]: cluster 2023-08-28T02:03:31.037831+0000 mgr.y (mgr.14901) 7347 : cluster [DBG] pgmap v7354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:32.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:32 smithi006 bash[27292]: audit 2023-08-28T02:03:32.050146+0000 mon.a (mon.0) 4759 : audit [DBG] from='client.? 172.21.15.6:0/2367660586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:32.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:32 smithi006 bash[33918]: cluster 2023-08-28T02:03:31.037831+0000 mgr.y (mgr.14901) 7347 : cluster [DBG] pgmap v7354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:32.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:32 smithi006 bash[33918]: audit 2023-08-28T02:03:32.050146+0000 mon.a (mon.0) 4759 : audit [DBG] from='client.? 172.21.15.6:0/2367660586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:03:32.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:03:32.763Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:33.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:33 smithi130 bash[51302]: debug 2023-08-28T02:03:33.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:33.226070+0000) 2023-08-28T02:03:34.586 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:34 smithi130 bash[51302]: debug 2023-08-28T02:03:34.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:34.226328+0000) 2023-08-28T02:03:34.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:34 smithi130 bash[27579]: cluster 2023-08-28T02:03:33.038610+0000 mgr.y (mgr.14901) 7348 : cluster [DBG] pgmap v7355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:34.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:34 smithi130 bash[27579]: audit 2023-08-28T02:03:34.464254+0000 mon.c (mon.2) 2883 : audit [DBG] from='client.? 172.21.15.6:0/2889748382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:34.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:34 smithi006 bash[27292]: cluster 2023-08-28T02:03:33.038610+0000 mgr.y (mgr.14901) 7348 : cluster [DBG] pgmap v7355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:34.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:34 smithi006 bash[27292]: audit 2023-08-28T02:03:34.464254+0000 mon.c (mon.2) 2883 : audit [DBG] from='client.? 172.21.15.6:0/2889748382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:34.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:34 smithi006 bash[33918]: cluster 2023-08-28T02:03:33.038610+0000 mgr.y (mgr.14901) 7348 : cluster [DBG] pgmap v7355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:34.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:34 smithi006 bash[33918]: audit 2023-08-28T02:03:34.464254+0000 mon.c (mon.2) 2883 : audit [DBG] from='client.? 172.21.15.6:0/2889748382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:03:34.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:03:34.640Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:35.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:35 smithi130 bash[51302]: debug 2023-08-28T02:03:35.222+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:35.226605+0000) 2023-08-28T02:03:36.590 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:36 smithi130 bash[51302]: debug 2023-08-28T02:03:36.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:36.226915+0000) 2023-08-28T02:03:36.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:36 smithi130 bash[27579]: cluster 2023-08-28T02:03:35.039797+0000 mgr.y (mgr.14901) 7349 : cluster [DBG] pgmap v7356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:36.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:36 smithi006 bash[33918]: cluster 2023-08-28T02:03:35.039797+0000 mgr.y (mgr.14901) 7349 : cluster [DBG] pgmap v7356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:36.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:36 smithi006 bash[27292]: cluster 2023-08-28T02:03:35.039797+0000 mgr.y (mgr.14901) 7349 : cluster [DBG] pgmap v7356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:37.597 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:37 smithi130 bash[51302]: debug 2023-08-28T02:03:37.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:37.227202+0000) 2023-08-28T02:03:37.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:37 smithi130 bash[27579]: audit 2023-08-28T02:03:36.872109+0000 mon.c (mon.2) 2884 : audit [DBG] from='client.? 172.21.15.6:0/277387955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:37.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:37 smithi130 bash[27579]: cluster 2023-08-28T02:03:37.040465+0000 mgr.y (mgr.14901) 7350 : cluster [DBG] pgmap v7357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:37.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:37 smithi006 bash[27292]: audit 2023-08-28T02:03:36.872109+0000 mon.c (mon.2) 2884 : audit [DBG] from='client.? 172.21.15.6:0/277387955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:37.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:37 smithi006 bash[27292]: cluster 2023-08-28T02:03:37.040465+0000 mgr.y (mgr.14901) 7350 : cluster [DBG] pgmap v7357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:37.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:37 smithi006 bash[33918]: audit 2023-08-28T02:03:36.872109+0000 mon.c (mon.2) 2884 : audit [DBG] from='client.? 172.21.15.6:0/277387955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:37.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:37 smithi006 bash[33918]: cluster 2023-08-28T02:03:37.040465+0000 mgr.y (mgr.14901) 7350 : cluster [DBG] pgmap v7357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:38.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:38 smithi130 bash[51302]: debug 2023-08-28T02:03:38.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:38.227490+0000) 2023-08-28T02:03:39.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:39 smithi130 bash[51302]: debug 2023-08-28T02:03:39.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:39.227770+0000) 2023-08-28T02:03:40.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:40 smithi130 bash[27579]: cluster 2023-08-28T02:03:39.041092+0000 mgr.y (mgr.14901) 7351 : cluster [DBG] pgmap v7358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:40.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:40 smithi130 bash[27579]: audit 2023-08-28T02:03:39.295058+0000 mon.a (mon.0) 4760 : audit [DBG] from='client.? 172.21.15.6:0/1185552462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:40.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:40 smithi130 bash[51302]: debug 2023-08-28T02:03:40.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:40.228047+0000) 2023-08-28T02:03:40.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:40 smithi006 bash[27292]: cluster 2023-08-28T02:03:39.041092+0000 mgr.y (mgr.14901) 7351 : cluster [DBG] pgmap v7358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:40.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:40 smithi006 bash[27292]: audit 2023-08-28T02:03:39.295058+0000 mon.a (mon.0) 4760 : audit [DBG] from='client.? 172.21.15.6:0/1185552462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:40.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:40 smithi006 bash[33918]: cluster 2023-08-28T02:03:39.041092+0000 mgr.y (mgr.14901) 7351 : cluster [DBG] pgmap v7358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:40.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:40 smithi006 bash[33918]: audit 2023-08-28T02:03:39.295058+0000 mon.a (mon.0) 4760 : audit [DBG] from='client.? 172.21.15.6:0/1185552462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:41.197 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:03:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:03:40] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:03:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:03:41.198Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:03:41.200Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:41.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:41 smithi130 bash[51302]: debug 2023-08-28T02:03:41.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:41.228372+0000) 2023-08-28T02:03:42.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:42 smithi130 bash[27579]: cluster 2023-08-28T02:03:41.042190+0000 mgr.y (mgr.14901) 7352 : cluster [DBG] pgmap v7359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:42.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:42 smithi130 bash[27579]: audit 2023-08-28T02:03:41.702421+0000 mon.c (mon.2) 2885 : audit [DBG] from='client.? 172.21.15.6:0/3876167148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:42.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:42 smithi130 bash[51302]: debug 2023-08-28T02:03:42.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:42.228646+0000) 2023-08-28T02:03:42.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:42 smithi006 bash[33918]: cluster 2023-08-28T02:03:41.042190+0000 mgr.y (mgr.14901) 7352 : cluster [DBG] pgmap v7359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:42.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:42 smithi006 bash[33918]: audit 2023-08-28T02:03:41.702421+0000 mon.c (mon.2) 2885 : audit [DBG] from='client.? 172.21.15.6:0/3876167148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:42.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:42 smithi006 bash[27292]: cluster 2023-08-28T02:03:41.042190+0000 mgr.y (mgr.14901) 7352 : cluster [DBG] pgmap v7359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:42.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:42 smithi006 bash[27292]: audit 2023-08-28T02:03:41.702421+0000 mon.c (mon.2) 2885 : audit [DBG] from='client.? 172.21.15.6:0/3876167148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:03:42.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:03:42.763Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:43.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:43 smithi130 bash[51302]: debug 2023-08-28T02:03:43.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:43.228919+0000) 2023-08-28T02:03:44.588 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:44 smithi130 bash[51302]: debug 2023-08-28T02:03:44.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:44.229201+0000) 2023-08-28T02:03:44.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:44 smithi130 bash[27579]: cluster 2023-08-28T02:03:43.042844+0000 mgr.y (mgr.14901) 7353 : cluster [DBG] pgmap v7360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:44.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:44 smithi130 bash[27579]: audit 2023-08-28T02:03:44.116772+0000 mon.a (mon.0) 4761 : audit [DBG] from='client.? 172.21.15.6:0/1180675583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:44.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:44 smithi006 bash[27292]: cluster 2023-08-28T02:03:43.042844+0000 mgr.y (mgr.14901) 7353 : cluster [DBG] pgmap v7360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:44.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:44 smithi006 bash[27292]: audit 2023-08-28T02:03:44.116772+0000 mon.a (mon.0) 4761 : audit [DBG] from='client.? 172.21.15.6:0/1180675583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:44.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:44 smithi006 bash[33918]: cluster 2023-08-28T02:03:43.042844+0000 mgr.y (mgr.14901) 7353 : cluster [DBG] pgmap v7360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:44.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:44 smithi006 bash[33918]: audit 2023-08-28T02:03:44.116772+0000 mon.a (mon.0) 4761 : audit [DBG] from='client.? 172.21.15.6:0/1180675583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:03:44.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:03:44.640Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:45.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:45 smithi130 bash[51302]: debug 2023-08-28T02:03:45.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:45.229507+0000) 2023-08-28T02:03:45.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:45 smithi006 bash[33918]: cluster 2023-08-28T02:03:45.043966+0000 mgr.y (mgr.14901) 7354 : cluster [DBG] pgmap v7361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:45.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:45 smithi006 bash[27292]: cluster 2023-08-28T02:03:45.043966+0000 mgr.y (mgr.14901) 7354 : cluster [DBG] pgmap v7361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:46.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:45 smithi130 bash[27579]: cluster 2023-08-28T02:03:45.043966+0000 mgr.y (mgr.14901) 7354 : cluster [DBG] pgmap v7361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:46.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:46 smithi130 bash[51302]: debug 2023-08-28T02:03:46.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:46.229813+0000) 2023-08-28T02:03:46.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:46 smithi006 bash[33918]: audit 2023-08-28T02:03:46.535362+0000 mon.c (mon.2) 2886 : audit [DBG] from='client.? 172.21.15.6:0/1658580044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:46.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:46 smithi006 bash[27292]: audit 2023-08-28T02:03:46.535362+0000 mon.c (mon.2) 2886 : audit [DBG] from='client.? 172.21.15.6:0/1658580044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:47.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:46 smithi130 bash[27579]: audit 2023-08-28T02:03:46.535362+0000 mon.c (mon.2) 2886 : audit [DBG] from='client.? 172.21.15.6:0/1658580044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:47.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:47 smithi130 bash[51302]: debug 2023-08-28T02:03:47.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:47.230103+0000) 2023-08-28T02:03:48.587 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:48 smithi130 bash[51302]: debug 2023-08-28T02:03:48.226+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:48.230408+0000) 2023-08-28T02:03:48.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:48 smithi130 bash[27579]: cluster 2023-08-28T02:03:47.044644+0000 mgr.y (mgr.14901) 7355 : cluster [DBG] pgmap v7362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:48.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:48 smithi006 bash[27292]: cluster 2023-08-28T02:03:47.044644+0000 mgr.y (mgr.14901) 7355 : cluster [DBG] pgmap v7362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:48.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:48 smithi006 bash[33918]: cluster 2023-08-28T02:03:47.044644+0000 mgr.y (mgr.14901) 7355 : cluster [DBG] pgmap v7362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:49.599 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:49 smithi130 bash[51302]: debug 2023-08-28T02:03:49.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:49.230723+0000) 2023-08-28T02:03:49.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:49 smithi130 bash[27579]: audit 2023-08-28T02:03:48.946523+0000 mon.c (mon.2) 2887 : audit [DBG] from='client.? 172.21.15.6:0/1320778694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:49.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:49 smithi006 bash[27292]: audit 2023-08-28T02:03:48.946523+0000 mon.c (mon.2) 2887 : audit [DBG] from='client.? 172.21.15.6:0/1320778694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:49.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:49 smithi006 bash[33918]: audit 2023-08-28T02:03:48.946523+0000 mon.c (mon.2) 2887 : audit [DBG] from='client.? 172.21.15.6:0/1320778694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:50.602 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:50 smithi130 bash[51302]: debug 2023-08-28T02:03:50.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:50.230985+0000) 2023-08-28T02:03:50.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:50 smithi130 bash[27579]: cluster 2023-08-28T02:03:49.045389+0000 mgr.y (mgr.14901) 7356 : cluster [DBG] pgmap v7363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:50.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:03:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:03:50] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:03:50.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:50 smithi006 bash[27292]: cluster 2023-08-28T02:03:49.045389+0000 mgr.y (mgr.14901) 7356 : cluster [DBG] pgmap v7363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:50.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:50 smithi006 bash[33918]: cluster 2023-08-28T02:03:49.045389+0000 mgr.y (mgr.14901) 7356 : cluster [DBG] pgmap v7363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:03:51.198Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:03:51.200Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:51.604 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:51 smithi130 bash[51302]: debug 2023-08-28T02:03:51.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:51.231261+0000) 2023-08-28T02:03:51.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:51 smithi130 bash[27579]: cluster 2023-08-28T02:03:51.046609+0000 mgr.y (mgr.14901) 7357 : cluster [DBG] pgmap v7364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:51.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:51 smithi130 bash[27579]: audit 2023-08-28T02:03:51.351417+0000 mon.a (mon.0) 4762 : audit [DBG] from='client.? 172.21.15.6:0/3234039851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:51.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:51 smithi006 bash[27292]: cluster 2023-08-28T02:03:51.046609+0000 mgr.y (mgr.14901) 7357 : cluster [DBG] pgmap v7364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:51.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:51 smithi006 bash[27292]: audit 2023-08-28T02:03:51.351417+0000 mon.a (mon.0) 4762 : audit [DBG] from='client.? 172.21.15.6:0/3234039851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:51.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:51 smithi006 bash[33918]: cluster 2023-08-28T02:03:51.046609+0000 mgr.y (mgr.14901) 7357 : cluster [DBG] pgmap v7364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:51.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:51 smithi006 bash[33918]: audit 2023-08-28T02:03:51.351417+0000 mon.a (mon.0) 4762 : audit [DBG] from='client.? 172.21.15.6:0/3234039851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:52.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:52 smithi130 bash[51302]: debug 2023-08-28T02:03:52.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:52.231519+0000) 2023-08-28T02:03:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:03:52.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:03:52.764Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:53.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:53 smithi130 bash[51302]: debug 2023-08-28T02:03:53.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:53.231799+0000) 2023-08-28T02:03:54.591 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:54 smithi130 bash[51302]: debug 2023-08-28T02:03:54.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:54.232142+0000) 2023-08-28T02:03:54.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:54 smithi130 bash[27579]: cluster 2023-08-28T02:03:53.047248+0000 mgr.y (mgr.14901) 7358 : cluster [DBG] pgmap v7365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:54.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:54 smithi130 bash[27579]: audit 2023-08-28T02:03:53.766347+0000 mon.c (mon.2) 2888 : audit [DBG] from='client.? 172.21.15.6:0/3850114888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:54.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:54 smithi006 bash[33918]: cluster 2023-08-28T02:03:53.047248+0000 mgr.y (mgr.14901) 7358 : cluster [DBG] pgmap v7365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:54.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:54 smithi006 bash[33918]: audit 2023-08-28T02:03:53.766347+0000 mon.c (mon.2) 2888 : audit [DBG] from='client.? 172.21.15.6:0/3850114888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:03:54.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:03:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:03:54.641Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:03:54.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:54 smithi006 bash[27292]: cluster 2023-08-28T02:03:53.047248+0000 mgr.y (mgr.14901) 7358 : cluster [DBG] pgmap v7365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:54.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:54 smithi006 bash[27292]: audit 2023-08-28T02:03:53.766347+0000 mon.c (mon.2) 2888 : audit [DBG] from='client.? 172.21.15.6:0/3850114888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:55.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:55 smithi130 bash[51302]: debug 2023-08-28T02:03:55.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:55.232473+0000) 2023-08-28T02:03:55.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:55 smithi006 bash[27292]: cluster 2023-08-28T02:03:55.048361+0000 mgr.y (mgr.14901) 7359 : cluster [DBG] pgmap v7366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:55.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:55 smithi006 bash[33918]: cluster 2023-08-28T02:03:55.048361+0000 mgr.y (mgr.14901) 7359 : cluster [DBG] pgmap v7366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:56.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:55 smithi130 bash[27579]: cluster 2023-08-28T02:03:55.048361+0000 mgr.y (mgr.14901) 7359 : cluster [DBG] pgmap v7366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:56.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:56 smithi130 bash[51302]: debug 2023-08-28T02:03:56.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:56.232739+0000) 2023-08-28T02:03:56.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:56 smithi006 bash[27292]: audit 2023-08-28T02:03:56.184323+0000 mon.c (mon.2) 2889 : audit [DBG] from='client.? 172.21.15.6:0/2478411264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:56.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:56 smithi006 bash[33918]: audit 2023-08-28T02:03:56.184323+0000 mon.c (mon.2) 2889 : audit [DBG] from='client.? 172.21.15.6:0/2478411264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:57.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:56 smithi130 bash[27579]: audit 2023-08-28T02:03:56.184323+0000 mon.c (mon.2) 2889 : audit [DBG] from='client.? 172.21.15.6:0/2478411264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:57.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:57 smithi130 bash[51302]: debug 2023-08-28T02:03:57.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:57.233044+0000) 2023-08-28T02:03:58.600 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:58 smithi130 bash[51302]: debug 2023-08-28T02:03:58.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:58.233399+0000) 2023-08-28T02:03:58.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:58 smithi130 bash[27579]: cluster 2023-08-28T02:03:57.049044+0000 mgr.y (mgr.14901) 7360 : cluster [DBG] pgmap v7367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:58.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:58 smithi006 bash[27292]: cluster 2023-08-28T02:03:57.049044+0000 mgr.y (mgr.14901) 7360 : cluster [DBG] pgmap v7367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:58.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:58 smithi006 bash[33918]: cluster 2023-08-28T02:03:57.049044+0000 mgr.y (mgr.14901) 7360 : cluster [DBG] pgmap v7367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:59.606 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:03:59 smithi130 bash[51302]: debug 2023-08-28T02:03:59.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:59.233708+0000) 2023-08-28T02:03:59.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:59 smithi130 bash[27579]: audit 2023-08-28T02:03:58.606704+0000 mon.a (mon.0) 4763 : audit [DBG] from='client.? 172.21.15.6:0/1090933195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:59.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:03:59 smithi130 bash[27579]: cluster 2023-08-28T02:03:59.049795+0000 mgr.y (mgr.14901) 7361 : cluster [DBG] pgmap v7368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:59.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:59 smithi006 bash[27292]: audit 2023-08-28T02:03:58.606704+0000 mon.a (mon.0) 4763 : audit [DBG] from='client.? 172.21.15.6:0/1090933195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:59.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:03:59 smithi006 bash[27292]: cluster 2023-08-28T02:03:59.049795+0000 mgr.y (mgr.14901) 7361 : cluster [DBG] pgmap v7368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:03:59.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:59 smithi006 bash[33918]: audit 2023-08-28T02:03:58.606704+0000 mon.a (mon.0) 4763 : audit [DBG] from='client.? 172.21.15.6:0/1090933195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:59.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:03:59 smithi006 bash[33918]: cluster 2023-08-28T02:03:59.049795+0000 mgr.y (mgr.14901) 7361 : cluster [DBG] pgmap v7368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:00.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:00 smithi130 bash[51302]: debug 2023-08-28T02:04:00.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:00.234006+0000) 2023-08-28T02:04:01.070 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:04:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:04:00] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:04:01.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:01 smithi130 bash[27579]: audit 2023-08-28T02:04:01.017980+0000 mon.a (mon.0) 4764 : audit [DBG] from='client.? 172.21.15.6:0/3692801142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:01.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:01 smithi130 bash[51302]: debug 2023-08-28T02:04:01.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:01.234346+0000) 2023-08-28T02:04:01.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:01 smithi006 bash[27292]: audit 2023-08-28T02:04:01.017980+0000 mon.a (mon.0) 4764 : audit [DBG] from='client.? 172.21.15.6:0/3692801142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:01.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:01 smithi006 bash[33918]: audit 2023-08-28T02:04:01.017980+0000 mon.a (mon.0) 4764 : audit [DBG] from='client.? 172.21.15.6:0/3692801142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:04:01.199Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:01.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:04:01.201Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:02.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:02 smithi130 bash[27579]: cluster 2023-08-28T02:04:01.050976+0000 mgr.y (mgr.14901) 7362 : cluster [DBG] pgmap v7369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:02.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:02 smithi130 bash[51302]: debug 2023-08-28T02:04:02.230+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:02.234675+0000) 2023-08-28T02:04:02.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:02 smithi006 bash[33918]: cluster 2023-08-28T02:04:01.050976+0000 mgr.y (mgr.14901) 7362 : cluster [DBG] pgmap v7369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:02.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:02 smithi006 bash[27292]: cluster 2023-08-28T02:04:01.050976+0000 mgr.y (mgr.14901) 7362 : cluster [DBG] pgmap v7369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:04:02.763Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:04:02.765Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:03.599 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:03 smithi130 bash[51302]: debug 2023-08-28T02:04:03.234+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:03.234975+0000) 2023-08-28T02:04:03.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:03 smithi130 bash[27579]: audit 2023-08-28T02:04:03.431956+0000 mon.c (mon.2) 2890 : audit [DBG] from='client.? 172.21.15.6:0/297071998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:03.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:03 smithi006 bash[27292]: audit 2023-08-28T02:04:03.431956+0000 mon.c (mon.2) 2890 : audit [DBG] from='client.? 172.21.15.6:0/297071998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:03.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:03 smithi006 bash[33918]: audit 2023-08-28T02:04:03.431956+0000 mon.c (mon.2) 2890 : audit [DBG] from='client.? 172.21.15.6:0/297071998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:04.602 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:04 smithi130 bash[51302]: debug 2023-08-28T02:04:04.233+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:04.235290+0000) 2023-08-28T02:04:04.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:04 smithi130 bash[27579]: cluster 2023-08-28T02:04:03.051791+0000 mgr.y (mgr.14901) 7363 : cluster [DBG] pgmap v7370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:04.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:04 smithi006 bash[27292]: cluster 2023-08-28T02:04:03.051791+0000 mgr.y (mgr.14901) 7363 : cluster [DBG] pgmap v7370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:04.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:04 smithi006 bash[33918]: cluster 2023-08-28T02:04:03.051791+0000 mgr.y (mgr.14901) 7363 : cluster [DBG] pgmap v7370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:04:04.640Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:04:04.642Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:05.604 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:05 smithi130 bash[51302]: debug 2023-08-28T02:04:05.233+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:05.235569+0000) 2023-08-28T02:04:05.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:05 smithi130 bash[27579]: cluster 2023-08-28T02:04:05.053001+0000 mgr.y (mgr.14901) 7364 : cluster [DBG] pgmap v7371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:05.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:05 smithi006 bash[33918]: cluster 2023-08-28T02:04:05.053001+0000 mgr.y (mgr.14901) 7364 : cluster [DBG] pgmap v7371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:05.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:05 smithi006 bash[27292]: cluster 2023-08-28T02:04:05.053001+0000 mgr.y (mgr.14901) 7364 : cluster [DBG] pgmap v7371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:06.612 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:06 smithi130 bash[51302]: debug 2023-08-28T02:04:06.233+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:06.235846+0000) 2023-08-28T02:04:06.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:06 smithi130 bash[27579]: audit 2023-08-28T02:04:05.860769+0000 mon.c (mon.2) 2891 : audit [DBG] from='client.? 172.21.15.6:0/2544461264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:06.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:06 smithi006 bash[33918]: audit 2023-08-28T02:04:05.860769+0000 mon.c (mon.2) 2891 : audit [DBG] from='client.? 172.21.15.6:0/2544461264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:06.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:06 smithi006 bash[27292]: audit 2023-08-28T02:04:05.860769+0000 mon.c (mon.2) 2891 : audit [DBG] from='client.? 172.21.15.6:0/2544461264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:07.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:07 smithi130 bash[51302]: debug 2023-08-28T02:04:07.233+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:07.236159+0000) 2023-08-28T02:04:08.594 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:08 smithi130 bash[51302]: debug 2023-08-28T02:04:08.233+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:08.236429+0000) 2023-08-28T02:04:08.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:08 smithi130 bash[27579]: cluster 2023-08-28T02:04:07.053819+0000 mgr.y (mgr.14901) 7365 : cluster [DBG] pgmap v7372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:08.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:08 smithi130 bash[27579]: audit 2023-08-28T02:04:08.277075+0000 mon.c (mon.2) 2892 : audit [DBG] from='client.? 172.21.15.6:0/273107077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:08.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:08 smithi006 bash[33918]: cluster 2023-08-28T02:04:07.053819+0000 mgr.y (mgr.14901) 7365 : cluster [DBG] pgmap v7372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:08.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:08 smithi006 bash[33918]: audit 2023-08-28T02:04:08.277075+0000 mon.c (mon.2) 2892 : audit [DBG] from='client.? 172.21.15.6:0/273107077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:08.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:08 smithi006 bash[27292]: cluster 2023-08-28T02:04:07.053819+0000 mgr.y (mgr.14901) 7365 : cluster [DBG] pgmap v7372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:08.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:08 smithi006 bash[27292]: audit 2023-08-28T02:04:08.277075+0000 mon.c (mon.2) 2892 : audit [DBG] from='client.? 172.21.15.6:0/273107077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:09.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:09 smithi130 bash[51302]: debug 2023-08-28T02:04:09.233+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:09.236734+0000) 2023-08-28T02:04:09.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:09 smithi006 bash[27292]: cluster 2023-08-28T02:04:09.054599+0000 mgr.y (mgr.14901) 7366 : cluster [DBG] pgmap v7373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:09.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:09 smithi006 bash[33918]: cluster 2023-08-28T02:04:09.054599+0000 mgr.y (mgr.14901) 7366 : cluster [DBG] pgmap v7373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:10.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:09 smithi130 bash[27579]: cluster 2023-08-28T02:04:09.054599+0000 mgr.y (mgr.14901) 7366 : cluster [DBG] pgmap v7373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:10.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:10 smithi130 bash[51302]: debug 2023-08-28T02:04:10.233+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:10.237017+0000) 2023-08-28T02:04:11.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:10 smithi130 bash[27579]: audit 2023-08-28T02:04:10.700747+0000 mon.c (mon.2) 2893 : audit [DBG] from='client.? 172.21.15.6:0/2984631001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:11.199 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:04:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:04:10] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:04:11.200 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:10 smithi006 bash[27292]: audit 2023-08-28T02:04:10.700747+0000 mon.c (mon.2) 2893 : audit [DBG] from='client.? 172.21.15.6:0/2984631001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:11.200 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:10 smithi006 bash[33918]: audit 2023-08-28T02:04:10.700747+0000 mon.c (mon.2) 2893 : audit [DBG] from='client.? 172.21.15.6:0/2984631001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:04:11.200Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:04:11.202Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:11.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:11 smithi130 bash[51302]: debug 2023-08-28T02:04:11.233+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:11.237346+0000) 2023-08-28T02:04:12.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:11 smithi130 bash[27579]: cluster 2023-08-28T02:04:11.055821+0000 mgr.y (mgr.14901) 7367 : cluster [DBG] pgmap v7374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:12.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:11 smithi006 bash[33918]: cluster 2023-08-28T02:04:11.055821+0000 mgr.y (mgr.14901) 7367 : cluster [DBG] pgmap v7374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:12.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:11 smithi006 bash[27292]: cluster 2023-08-28T02:04:11.055821+0000 mgr.y (mgr.14901) 7367 : cluster [DBG] pgmap v7374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:12.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:12 smithi130 bash[51302]: debug 2023-08-28T02:04:12.233+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:12.237647+0000) 2023-08-28T02:04:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:04:12.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:04:12.766Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:13.601 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:13 smithi130 bash[51302]: debug 2023-08-28T02:04:13.233+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:13.237933+0000) 2023-08-28T02:04:13.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:13 smithi130 bash[27579]: audit 2023-08-28T02:04:13.108511+0000 mon.c (mon.2) 2894 : audit [DBG] from='client.? 172.21.15.6:0/2172797247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:13.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:13 smithi006 bash[27292]: audit 2023-08-28T02:04:13.108511+0000 mon.c (mon.2) 2894 : audit [DBG] from='client.? 172.21.15.6:0/2172797247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:13.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:13 smithi006 bash[33918]: audit 2023-08-28T02:04:13.108511+0000 mon.c (mon.2) 2894 : audit [DBG] from='client.? 172.21.15.6:0/2172797247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:14.605 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:14 smithi130 bash[51302]: debug 2023-08-28T02:04:14.233+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:14.238204+0000) 2023-08-28T02:04:14.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:14 smithi130 bash[27579]: cluster 2023-08-28T02:04:13.056595+0000 mgr.y (mgr.14901) 7368 : cluster [DBG] pgmap v7375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:14.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:14 smithi006 bash[27292]: cluster 2023-08-28T02:04:13.056595+0000 mgr.y (mgr.14901) 7368 : cluster [DBG] pgmap v7375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:14.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:14 smithi006 bash[33918]: cluster 2023-08-28T02:04:13.056595+0000 mgr.y (mgr.14901) 7368 : cluster [DBG] pgmap v7375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:04:14.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:04:14.644Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:15.607 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:15 smithi130 bash[51302]: debug 2023-08-28T02:04:15.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:15.238475+0000) 2023-08-28T02:04:15.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:15 smithi130 bash[27579]: cluster 2023-08-28T02:04:15.057794+0000 mgr.y (mgr.14901) 7369 : cluster [DBG] pgmap v7376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:15.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:15 smithi130 bash[27579]: audit 2023-08-28T02:04:15.522989+0000 mon.c (mon.2) 2895 : audit [DBG] from='client.? 172.21.15.6:0/1869718015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:15.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:15 smithi006 bash[33918]: cluster 2023-08-28T02:04:15.057794+0000 mgr.y (mgr.14901) 7369 : cluster [DBG] pgmap v7376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:15.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:15 smithi006 bash[33918]: audit 2023-08-28T02:04:15.522989+0000 mon.c (mon.2) 2895 : audit [DBG] from='client.? 172.21.15.6:0/1869718015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:15.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:15 smithi006 bash[27292]: cluster 2023-08-28T02:04:15.057794+0000 mgr.y (mgr.14901) 7369 : cluster [DBG] pgmap v7376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:15.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:15 smithi006 bash[27292]: audit 2023-08-28T02:04:15.522989+0000 mon.c (mon.2) 2895 : audit [DBG] from='client.? 172.21.15.6:0/1869718015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:16.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:16 smithi130 bash[51302]: debug 2023-08-28T02:04:16.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:16.238755+0000) 2023-08-28T02:04:17.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:17 smithi130 bash[51302]: debug 2023-08-28T02:04:17.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:17.239051+0000) 2023-08-28T02:04:18.597 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:18 smithi130 bash[51302]: debug 2023-08-28T02:04:18.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:18.239324+0000) 2023-08-28T02:04:18.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:18 smithi130 bash[27579]: cluster 2023-08-28T02:04:17.058478+0000 mgr.y (mgr.14901) 7370 : cluster [DBG] pgmap v7377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:18.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:18 smithi130 bash[27579]: audit 2023-08-28T02:04:17.929433+0000 mon.a (mon.0) 4765 : audit [DBG] from='client.? 172.21.15.6:0/851260707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:18.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:18 smithi006 bash[27292]: cluster 2023-08-28T02:04:17.058478+0000 mgr.y (mgr.14901) 7370 : cluster [DBG] pgmap v7377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:18.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:18 smithi006 bash[27292]: audit 2023-08-28T02:04:17.929433+0000 mon.a (mon.0) 4765 : audit [DBG] from='client.? 172.21.15.6:0/851260707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:18.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:18 smithi006 bash[33918]: cluster 2023-08-28T02:04:17.058478+0000 mgr.y (mgr.14901) 7370 : cluster [DBG] pgmap v7377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:18.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:18 smithi006 bash[33918]: audit 2023-08-28T02:04:17.929433+0000 mon.a (mon.0) 4765 : audit [DBG] from='client.? 172.21.15.6:0/851260707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:19.607 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:19 smithi130 bash[51302]: debug 2023-08-28T02:04:19.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:19.239608+0000) 2023-08-28T02:04:19.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:19 smithi130 bash[27579]: cluster 2023-08-28T02:04:19.059252+0000 mgr.y (mgr.14901) 7371 : cluster [DBG] pgmap v7378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:19.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:19 smithi006 bash[27292]: cluster 2023-08-28T02:04:19.059252+0000 mgr.y (mgr.14901) 7371 : cluster [DBG] pgmap v7378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:19.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:19 smithi006 bash[33918]: cluster 2023-08-28T02:04:19.059252+0000 mgr.y (mgr.14901) 7371 : cluster [DBG] pgmap v7378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:20.612 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:20 smithi130 bash[51302]: debug 2023-08-28T02:04:20.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:20.239878+0000) 2023-08-28T02:04:20.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:20 smithi130 bash[27579]: audit 2023-08-28T02:04:20.170605+0000 mon.b (mon.1) 819 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:04:20.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:20 smithi130 bash[27579]: audit 2023-08-28T02:04:20.347205+0000 mon.c (mon.2) 2896 : audit [DBG] from='client.? 172.21.15.6:0/3007060918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:20.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:04:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:04:20] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-28T02:04:20.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:20 smithi006 bash[33918]: audit 2023-08-28T02:04:20.170605+0000 mon.b (mon.1) 819 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:04:20.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:20 smithi006 bash[33918]: audit 2023-08-28T02:04:20.347205+0000 mon.c (mon.2) 2896 : audit [DBG] from='client.? 172.21.15.6:0/3007060918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:20.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:20 smithi006 bash[27292]: audit 2023-08-28T02:04:20.170605+0000 mon.b (mon.1) 819 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:04:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:20 smithi006 bash[27292]: audit 2023-08-28T02:04:20.347205+0000 mon.c (mon.2) 2896 : audit [DBG] from='client.? 172.21.15.6:0/3007060918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:04:21.200Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:04:21.202Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:21.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:21 smithi130 bash[51302]: debug 2023-08-28T02:04:21.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:21.240200+0000) 2023-08-28T02:04:21.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:21 smithi006 bash[27292]: cluster 2023-08-28T02:04:21.060345+0000 mgr.y (mgr.14901) 7372 : cluster [DBG] pgmap v7379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:21.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:21 smithi006 bash[33918]: cluster 2023-08-28T02:04:21.060345+0000 mgr.y (mgr.14901) 7372 : cluster [DBG] pgmap v7379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:22.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:21 smithi130 bash[27579]: cluster 2023-08-28T02:04:21.060345+0000 mgr.y (mgr.14901) 7372 : cluster [DBG] pgmap v7379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:22.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:22 smithi130 bash[51302]: debug 2023-08-28T02:04:22.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:22.240520+0000) 2023-08-28T02:04:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:04:22.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:04:22.766Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:23.602 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:23 smithi130 bash[51302]: debug 2023-08-28T02:04:23.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:23.240809+0000) 2023-08-28T02:04:23.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:23 smithi130 bash[27579]: audit 2023-08-28T02:04:22.770076+0000 mon.a (mon.0) 4766 : audit [DBG] from='client.? 172.21.15.6:0/345031262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:23.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:23 smithi006 bash[27292]: audit 2023-08-28T02:04:22.770076+0000 mon.a (mon.0) 4766 : audit [DBG] from='client.? 172.21.15.6:0/345031262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:23.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:23 smithi006 bash[33918]: audit 2023-08-28T02:04:22.770076+0000 mon.a (mon.0) 4766 : audit [DBG] from='client.? 172.21.15.6:0/345031262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:24.604 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:24 smithi130 bash[51302]: debug 2023-08-28T02:04:24.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:24.241089+0000) 2023-08-28T02:04:24.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:24 smithi130 bash[27579]: cluster 2023-08-28T02:04:23.061075+0000 mgr.y (mgr.14901) 7373 : cluster [DBG] pgmap v7380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:24.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:24 smithi006 bash[33918]: cluster 2023-08-28T02:04:23.061075+0000 mgr.y (mgr.14901) 7373 : cluster [DBG] pgmap v7380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:24.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:24 smithi006 bash[27292]: cluster 2023-08-28T02:04:23.061075+0000 mgr.y (mgr.14901) 7373 : cluster [DBG] pgmap v7380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:04:24.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:04:24.645Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:25.606 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:25 smithi130 bash[51302]: debug 2023-08-28T02:04:25.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:25.241297+0000) 2023-08-28T02:04:25.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:25 smithi130 bash[27579]: cluster 2023-08-28T02:04:25.061826+0000 mgr.y (mgr.14901) 7374 : cluster [DBG] pgmap v7381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:25.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:25 smithi130 bash[27579]: audit 2023-08-28T02:04:25.194515+0000 mon.a (mon.0) 4767 : audit [DBG] from='client.? 172.21.15.6:0/372419297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:25.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:25 smithi006 bash[27292]: cluster 2023-08-28T02:04:25.061826+0000 mgr.y (mgr.14901) 7374 : cluster [DBG] pgmap v7381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:25.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:25 smithi006 bash[27292]: audit 2023-08-28T02:04:25.194515+0000 mon.a (mon.0) 4767 : audit [DBG] from='client.? 172.21.15.6:0/372419297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:25.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:25 smithi006 bash[33918]: cluster 2023-08-28T02:04:25.061826+0000 mgr.y (mgr.14901) 7374 : cluster [DBG] pgmap v7381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:25.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:25 smithi006 bash[33918]: audit 2023-08-28T02:04:25.194515+0000 mon.a (mon.0) 4767 : audit [DBG] from='client.? 172.21.15.6:0/372419297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:26.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:26 smithi130 bash[51302]: debug 2023-08-28T02:04:26.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:26.241510+0000) 2023-08-28T02:04:27.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:27 smithi130 bash[51302]: debug 2023-08-28T02:04:27.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:27.241759+0000) 2023-08-28T02:04:27.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:27 smithi130 bash[27579]: audit 2023-08-28T02:04:26.076781+0000 mon.a (mon.0) 4768 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:04:27.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:27 smithi130 bash[27579]: audit 2023-08-28T02:04:26.515052+0000 mon.a (mon.0) 4769 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:04:27.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:27 smithi130 bash[27579]: audit 2023-08-28T02:04:26.782765+0000 mon.b (mon.1) 820 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:04:27.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:27 smithi130 bash[27579]: audit 2023-08-28T02:04:26.784949+0000 mon.b (mon.1) 821 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:04:27.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:27 smithi130 bash[27579]: audit 2023-08-28T02:04:26.793455+0000 mon.a (mon.0) 4770 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:04:27.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:27 smithi006 bash[27292]: audit 2023-08-28T02:04:26.076781+0000 mon.a (mon.0) 4768 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:04:27.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:27 smithi006 bash[27292]: audit 2023-08-28T02:04:26.515052+0000 mon.a (mon.0) 4769 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:04:27.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:27 smithi006 bash[27292]: audit 2023-08-28T02:04:26.782765+0000 mon.b (mon.1) 820 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:04:27.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:27 smithi006 bash[27292]: audit 2023-08-28T02:04:26.784949+0000 mon.b (mon.1) 821 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:04:27.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:27 smithi006 bash[27292]: audit 2023-08-28T02:04:26.793455+0000 mon.a (mon.0) 4770 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:04:27.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:27 smithi006 bash[33918]: audit 2023-08-28T02:04:26.076781+0000 mon.a (mon.0) 4768 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:04:27.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:27 smithi006 bash[33918]: audit 2023-08-28T02:04:26.515052+0000 mon.a (mon.0) 4769 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:04:27.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:27 smithi006 bash[33918]: audit 2023-08-28T02:04:26.782765+0000 mon.b (mon.1) 820 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:04:27.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:27 smithi006 bash[33918]: audit 2023-08-28T02:04:26.784949+0000 mon.b (mon.1) 821 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:04:27.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:27 smithi006 bash[33918]: audit 2023-08-28T02:04:26.793455+0000 mon.a (mon.0) 4770 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:04:28.597 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:28 smithi130 bash[51302]: debug 2023-08-28T02:04:28.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:28.242050+0000) 2023-08-28T02:04:28.864 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:28 smithi130 bash[27579]: cluster 2023-08-28T02:04:27.062432+0000 mgr.y (mgr.14901) 7375 : cluster [DBG] pgmap v7382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:28.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:28 smithi130 bash[27579]: audit 2023-08-28T02:04:27.600316+0000 mon.a (mon.0) 4771 : audit [DBG] from='client.? 172.21.15.6:0/519711006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:28.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:28 smithi006 bash[33918]: cluster 2023-08-28T02:04:27.062432+0000 mgr.y (mgr.14901) 7375 : cluster [DBG] pgmap v7382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:28.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:28 smithi006 bash[33918]: audit 2023-08-28T02:04:27.600316+0000 mon.a (mon.0) 4771 : audit [DBG] from='client.? 172.21.15.6:0/519711006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:28.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:28 smithi006 bash[27292]: cluster 2023-08-28T02:04:27.062432+0000 mgr.y (mgr.14901) 7375 : cluster [DBG] pgmap v7382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:28.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:28 smithi006 bash[27292]: audit 2023-08-28T02:04:27.600316+0000 mon.a (mon.0) 4771 : audit [DBG] from='client.? 172.21.15.6:0/519711006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:29.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:29 smithi130 bash[51302]: debug 2023-08-28T02:04:29.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:29.242322+0000) 2023-08-28T02:04:29.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:29 smithi006 bash[27292]: cluster 2023-08-28T02:04:29.063197+0000 mgr.y (mgr.14901) 7376 : cluster [DBG] pgmap v7383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:29.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:29 smithi006 bash[33918]: cluster 2023-08-28T02:04:29.063197+0000 mgr.y (mgr.14901) 7376 : cluster [DBG] pgmap v7383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:30.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:29 smithi130 bash[27579]: cluster 2023-08-28T02:04:29.063197+0000 mgr.y (mgr.14901) 7376 : cluster [DBG] pgmap v7383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:30.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:30 smithi130 bash[51302]: debug 2023-08-28T02:04:30.237+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:30.242606+0000) 2023-08-28T02:04:30.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:04:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:04:30] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:04:30.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:30 smithi006 bash[33918]: audit 2023-08-28T02:04:30.016810+0000 mon.c (mon.2) 2897 : audit [DBG] from='client.? 172.21.15.6:0/1996207853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:30.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:30 smithi006 bash[27292]: audit 2023-08-28T02:04:30.016810+0000 mon.c (mon.2) 2897 : audit [DBG] from='client.? 172.21.15.6:0/1996207853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:31.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:30 smithi130 bash[27579]: audit 2023-08-28T02:04:30.016810+0000 mon.c (mon.2) 2897 : audit [DBG] from='client.? 172.21.15.6:0/1996207853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:04:31.201Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:04:31.203Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:31.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:31 smithi130 bash[51302]: debug 2023-08-28T02:04:31.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:31.242940+0000) 2023-08-28T02:04:31.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:31 smithi006 bash[27292]: cluster 2023-08-28T02:04:31.064194+0000 mgr.y (mgr.14901) 7377 : cluster [DBG] pgmap v7384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:31.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:31 smithi006 bash[33918]: cluster 2023-08-28T02:04:31.064194+0000 mgr.y (mgr.14901) 7377 : cluster [DBG] pgmap v7384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:32.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:31 smithi130 bash[27579]: cluster 2023-08-28T02:04:31.064194+0000 mgr.y (mgr.14901) 7377 : cluster [DBG] pgmap v7384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:32.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:32 smithi130 bash[51302]: debug 2023-08-28T02:04:32.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:32.243207+0000) 2023-08-28T02:04:32.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:32 smithi006 bash[27292]: audit 2023-08-28T02:04:32.425853+0000 mon.c (mon.2) 2898 : audit [DBG] from='client.? 172.21.15.6:0/3815576083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:32.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:32 smithi006 bash[33918]: audit 2023-08-28T02:04:32.425853+0000 mon.c (mon.2) 2898 : audit [DBG] from='client.? 172.21.15.6:0/3815576083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:04:32.766Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:04:32.768Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:33.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:32 smithi130 bash[27579]: audit 2023-08-28T02:04:32.425853+0000 mon.c (mon.2) 2898 : audit [DBG] from='client.? 172.21.15.6:0/3815576083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:33.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:33 smithi130 bash[51302]: debug 2023-08-28T02:04:33.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:33.243470+0000) 2023-08-28T02:04:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:33 smithi006 bash[27292]: cluster 2023-08-28T02:04:33.064905+0000 mgr.y (mgr.14901) 7378 : cluster [DBG] pgmap v7385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:33.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:33 smithi006 bash[33918]: cluster 2023-08-28T02:04:33.064905+0000 mgr.y (mgr.14901) 7378 : cluster [DBG] pgmap v7385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:34.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:33 smithi130 bash[27579]: cluster 2023-08-28T02:04:33.064905+0000 mgr.y (mgr.14901) 7378 : cluster [DBG] pgmap v7385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:34.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:34 smithi130 bash[51302]: debug 2023-08-28T02:04:34.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:34.243792+0000) 2023-08-28T02:04:34.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:34 smithi006 bash[33918]: audit 2023-08-28T02:04:34.837213+0000 mon.c (mon.2) 2899 : audit [DBG] from='client.? 172.21.15.6:0/1990861043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:34.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:34 smithi006 bash[27292]: audit 2023-08-28T02:04:34.837213+0000 mon.c (mon.2) 2899 : audit [DBG] from='client.? 172.21.15.6:0/1990861043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:04:34.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:04:34.645Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:35.241 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:34 smithi130 bash[27579]: audit 2023-08-28T02:04:34.837213+0000 mon.c (mon.2) 2899 : audit [DBG] from='client.? 172.21.15.6:0/1990861043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:35.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:35 smithi130 bash[51302]: debug 2023-08-28T02:04:35.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:35.244067+0000) 2023-08-28T02:04:36.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:35 smithi006 bash[27292]: cluster 2023-08-28T02:04:35.065942+0000 mgr.y (mgr.14901) 7379 : cluster [DBG] pgmap v7386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:36.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:35 smithi006 bash[33918]: cluster 2023-08-28T02:04:35.065942+0000 mgr.y (mgr.14901) 7379 : cluster [DBG] pgmap v7386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:36.242 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:35 smithi130 bash[27579]: cluster 2023-08-28T02:04:35.065942+0000 mgr.y (mgr.14901) 7379 : cluster [DBG] pgmap v7386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:36.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:36 smithi130 bash[51302]: debug 2023-08-28T02:04:36.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:36.244343+0000) 2023-08-28T02:04:37.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:37 smithi130 bash[51302]: debug 2023-08-28T02:04:37.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:37.244624+0000) 2023-08-28T02:04:38.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:38 smithi006 bash[33918]: cluster 2023-08-28T02:04:37.066400+0000 mgr.y (mgr.14901) 7380 : cluster [DBG] pgmap v7387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:38.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:38 smithi006 bash[33918]: audit 2023-08-28T02:04:37.253729+0000 mon.c (mon.2) 2900 : audit [DBG] from='client.? 172.21.15.6:0/3243236977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:38.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:38 smithi006 bash[27292]: cluster 2023-08-28T02:04:37.066400+0000 mgr.y (mgr.14901) 7380 : cluster [DBG] pgmap v7387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:38.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:38 smithi006 bash[27292]: audit 2023-08-28T02:04:37.253729+0000 mon.c (mon.2) 2900 : audit [DBG] from='client.? 172.21.15.6:0/3243236977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:38.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:38 smithi130 bash[51302]: debug 2023-08-28T02:04:38.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:38.244961+0000) 2023-08-28T02:04:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:38 smithi130 bash[27579]: cluster 2023-08-28T02:04:37.066400+0000 mgr.y (mgr.14901) 7380 : cluster [DBG] pgmap v7387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:38 smithi130 bash[27579]: audit 2023-08-28T02:04:37.253729+0000 mon.c (mon.2) 2900 : audit [DBG] from='client.? 172.21.15.6:0/3243236977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:39.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:39 smithi130 bash[51302]: debug 2023-08-28T02:04:39.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:39.245258+0000) 2023-08-28T02:04:40.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:40 smithi006 bash[27292]: cluster 2023-08-28T02:04:39.067169+0000 mgr.y (mgr.14901) 7381 : cluster [DBG] pgmap v7388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:40.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:40 smithi006 bash[27292]: audit 2023-08-28T02:04:39.666415+0000 mon.a (mon.0) 4772 : audit [DBG] from='client.? 172.21.15.6:0/3716869621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:40 smithi006 bash[33918]: cluster 2023-08-28T02:04:39.067169+0000 mgr.y (mgr.14901) 7381 : cluster [DBG] pgmap v7388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:40 smithi006 bash[33918]: audit 2023-08-28T02:04:39.666415+0000 mon.a (mon.0) 4772 : audit [DBG] from='client.? 172.21.15.6:0/3716869621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:40.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:40 smithi130 bash[51302]: debug 2023-08-28T02:04:40.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:40.245529+0000) 2023-08-28T02:04:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:40 smithi130 bash[27579]: cluster 2023-08-28T02:04:39.067169+0000 mgr.y (mgr.14901) 7381 : cluster [DBG] pgmap v7388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:40 smithi130 bash[27579]: audit 2023-08-28T02:04:39.666415+0000 mon.a (mon.0) 4772 : audit [DBG] from='client.? 172.21.15.6:0/3716869621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:41.202 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:04:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:04:40] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:04:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:04:41.202Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:04:41.204Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:41.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:41 smithi130 bash[51302]: debug 2023-08-28T02:04:41.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:41.245813+0000) 2023-08-28T02:04:42.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:42 smithi006 bash[33918]: cluster 2023-08-28T02:04:41.068189+0000 mgr.y (mgr.14901) 7382 : cluster [DBG] pgmap v7389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:42.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:42 smithi006 bash[33918]: audit 2023-08-28T02:04:42.084502+0000 mon.c (mon.2) 2901 : audit [DBG] from='client.? 172.21.15.6:0/1750222331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:42.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:42 smithi006 bash[27292]: cluster 2023-08-28T02:04:41.068189+0000 mgr.y (mgr.14901) 7382 : cluster [DBG] pgmap v7389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:42.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:42 smithi006 bash[27292]: audit 2023-08-28T02:04:42.084502+0000 mon.c (mon.2) 2901 : audit [DBG] from='client.? 172.21.15.6:0/1750222331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:42.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:42 smithi130 bash[27579]: cluster 2023-08-28T02:04:41.068189+0000 mgr.y (mgr.14901) 7382 : cluster [DBG] pgmap v7389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:42.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:42 smithi130 bash[27579]: audit 2023-08-28T02:04:42.084502+0000 mon.c (mon.2) 2901 : audit [DBG] from='client.? 172.21.15.6:0/1750222331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:42.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:42 smithi130 bash[51302]: debug 2023-08-28T02:04:42.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:42.246163+0000) 2023-08-28T02:04:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:04:42.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:04:42.768Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:43.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:43 smithi130 bash[51302]: debug 2023-08-28T02:04:43.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:43.246438+0000) 2023-08-28T02:04:43.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:43 smithi006 bash[27292]: cluster 2023-08-28T02:04:43.068919+0000 mgr.y (mgr.14901) 7383 : cluster [DBG] pgmap v7390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:43.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:43 smithi006 bash[33918]: cluster 2023-08-28T02:04:43.068919+0000 mgr.y (mgr.14901) 7383 : cluster [DBG] pgmap v7390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:44.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:43 smithi130 bash[27579]: cluster 2023-08-28T02:04:43.068919+0000 mgr.y (mgr.14901) 7383 : cluster [DBG] pgmap v7390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:44.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:44 smithi130 bash[51302]: debug 2023-08-28T02:04:44.241+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:44.246722+0000) 2023-08-28T02:04:44.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:44 smithi006 bash[33918]: audit 2023-08-28T02:04:44.501353+0000 mon.c (mon.2) 2902 : audit [DBG] from='client.? 172.21.15.6:0/3841594387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:44.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:44 smithi006 bash[27292]: audit 2023-08-28T02:04:44.501353+0000 mon.c (mon.2) 2902 : audit [DBG] from='client.? 172.21.15.6:0/3841594387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:04:44.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:04:44.646Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:45.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:44 smithi130 bash[27579]: audit 2023-08-28T02:04:44.501353+0000 mon.c (mon.2) 2902 : audit [DBG] from='client.? 172.21.15.6:0/3841594387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:45.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:45 smithi130 bash[51302]: debug 2023-08-28T02:04:45.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:45.246998+0000) 2023-08-28T02:04:45.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:45 smithi006 bash[27292]: cluster 2023-08-28T02:04:45.069986+0000 mgr.y (mgr.14901) 7384 : cluster [DBG] pgmap v7391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:45.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:45 smithi006 bash[33918]: cluster 2023-08-28T02:04:45.069986+0000 mgr.y (mgr.14901) 7384 : cluster [DBG] pgmap v7391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:46.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:45 smithi130 bash[27579]: cluster 2023-08-28T02:04:45.069986+0000 mgr.y (mgr.14901) 7384 : cluster [DBG] pgmap v7391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:46.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:46 smithi130 bash[51302]: debug 2023-08-28T02:04:46.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:46.247271+0000) 2023-08-28T02:04:47.245 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:46 smithi130 bash[27579]: audit 2023-08-28T02:04:46.918736+0000 mon.a (mon.0) 4773 : audit [DBG] from='client.? 172.21.15.6:0/421351029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:47.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:46 smithi006 bash[33918]: audit 2023-08-28T02:04:46.918736+0000 mon.a (mon.0) 4773 : audit [DBG] from='client.? 172.21.15.6:0/421351029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:47.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:46 smithi006 bash[27292]: audit 2023-08-28T02:04:46.918736+0000 mon.a (mon.0) 4773 : audit [DBG] from='client.? 172.21.15.6:0/421351029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:47.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:47 smithi130 bash[51302]: debug 2023-08-28T02:04:47.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:47.247572+0000) 2023-08-28T02:04:48.245 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:47 smithi130 bash[27579]: cluster 2023-08-28T02:04:47.070722+0000 mgr.y (mgr.14901) 7385 : cluster [DBG] pgmap v7392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:48.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:47 smithi006 bash[33918]: cluster 2023-08-28T02:04:47.070722+0000 mgr.y (mgr.14901) 7385 : cluster [DBG] pgmap v7392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:48.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:47 smithi006 bash[27292]: cluster 2023-08-28T02:04:47.070722+0000 mgr.y (mgr.14901) 7385 : cluster [DBG] pgmap v7392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:48.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:48 smithi130 bash[51302]: debug 2023-08-28T02:04:48.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:48.247857+0000) 2023-08-28T02:04:49.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:49 smithi130 bash[51302]: debug 2023-08-28T02:04:49.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:49.248186+0000) 2023-08-28T02:04:50.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:50 smithi006 bash[27292]: cluster 2023-08-28T02:04:49.071393+0000 mgr.y (mgr.14901) 7386 : cluster [DBG] pgmap v7393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:50.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:50 smithi006 bash[27292]: audit 2023-08-28T02:04:49.330132+0000 mon.a (mon.0) 4774 : audit [DBG] from='client.? 172.21.15.6:0/1507969834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:50.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:50 smithi006 bash[33918]: cluster 2023-08-28T02:04:49.071393+0000 mgr.y (mgr.14901) 7386 : cluster [DBG] pgmap v7393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:50.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:50 smithi006 bash[33918]: audit 2023-08-28T02:04:49.330132+0000 mon.a (mon.0) 4774 : audit [DBG] from='client.? 172.21.15.6:0/1507969834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:50.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:50 smithi130 bash[27579]: cluster 2023-08-28T02:04:49.071393+0000 mgr.y (mgr.14901) 7386 : cluster [DBG] pgmap v7393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:50 smithi130 bash[27579]: audit 2023-08-28T02:04:49.330132+0000 mon.a (mon.0) 4774 : audit [DBG] from='client.? 172.21.15.6:0/1507969834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:50.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:50 smithi130 bash[51302]: debug 2023-08-28T02:04:50.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:50.248489+0000) 2023-08-28T02:04:51.202 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:04:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:04:50] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:04:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:04:51.203Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:04:51.205Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:51.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:51 smithi130 bash[51302]: debug 2023-08-28T02:04:51.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:51.248781+0000) 2023-08-28T02:04:52.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:52 smithi006 bash[33918]: cluster 2023-08-28T02:04:51.072448+0000 mgr.y (mgr.14901) 7387 : cluster [DBG] pgmap v7394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:52.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:52 smithi006 bash[33918]: audit 2023-08-28T02:04:51.747260+0000 mon.c (mon.2) 2903 : audit [DBG] from='client.? 172.21.15.6:0/2512891720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:52.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:52 smithi006 bash[27292]: cluster 2023-08-28T02:04:51.072448+0000 mgr.y (mgr.14901) 7387 : cluster [DBG] pgmap v7394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:52.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:52 smithi006 bash[27292]: audit 2023-08-28T02:04:51.747260+0000 mon.c (mon.2) 2903 : audit [DBG] from='client.? 172.21.15.6:0/2512891720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:52.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:52 smithi130 bash[27579]: cluster 2023-08-28T02:04:51.072448+0000 mgr.y (mgr.14901) 7387 : cluster [DBG] pgmap v7394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:52.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:52 smithi130 bash[27579]: audit 2023-08-28T02:04:51.747260+0000 mon.c (mon.2) 2903 : audit [DBG] from='client.? 172.21.15.6:0/2512891720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:52.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:52 smithi130 bash[51302]: debug 2023-08-28T02:04:52.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:52.249097+0000) 2023-08-28T02:04:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:04:52.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:04:52.769Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:53.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:53 smithi130 bash[51302]: debug 2023-08-28T02:04:53.249+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:53.249354+0000) 2023-08-28T02:04:53.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:53 smithi006 bash[33918]: cluster 2023-08-28T02:04:53.073180+0000 mgr.y (mgr.14901) 7388 : cluster [DBG] pgmap v7395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:53.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:53 smithi006 bash[27292]: cluster 2023-08-28T02:04:53.073180+0000 mgr.y (mgr.14901) 7388 : cluster [DBG] pgmap v7395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:54.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:53 smithi130 bash[27579]: cluster 2023-08-28T02:04:53.073180+0000 mgr.y (mgr.14901) 7388 : cluster [DBG] pgmap v7395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:54.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:54 smithi130 bash[51302]: debug 2023-08-28T02:04:54.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:54.249614+0000) 2023-08-28T02:04:54.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:54 smithi006 bash[27292]: audit 2023-08-28T02:04:54.162778+0000 mon.c (mon.2) 2904 : audit [DBG] from='client.? 172.21.15.6:0/2922703442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:54.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:54 smithi006 bash[33918]: audit 2023-08-28T02:04:54.162778+0000 mon.c (mon.2) 2904 : audit [DBG] from='client.? 172.21.15.6:0/2922703442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:04:54.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:04:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:04:54.647Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:04:55.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:54 smithi130 bash[27579]: audit 2023-08-28T02:04:54.162778+0000 mon.c (mon.2) 2904 : audit [DBG] from='client.? 172.21.15.6:0/2922703442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:55.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:55 smithi130 bash[51302]: debug 2023-08-28T02:04:55.249+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:55.249892+0000) 2023-08-28T02:04:55.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:55 smithi006 bash[27292]: cluster 2023-08-28T02:04:55.074292+0000 mgr.y (mgr.14901) 7389 : cluster [DBG] pgmap v7396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:55.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:55 smithi006 bash[33918]: cluster 2023-08-28T02:04:55.074292+0000 mgr.y (mgr.14901) 7389 : cluster [DBG] pgmap v7396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:56.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:55 smithi130 bash[27579]: cluster 2023-08-28T02:04:55.074292+0000 mgr.y (mgr.14901) 7389 : cluster [DBG] pgmap v7396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:56.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:56 smithi130 bash[51302]: debug 2023-08-28T02:04:56.245+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:56.250183+0000) 2023-08-28T02:04:56.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:56 smithi006 bash[27292]: audit 2023-08-28T02:04:56.574507+0000 mon.c (mon.2) 2905 : audit [DBG] from='client.? 172.21.15.6:0/3966316783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:56.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:56 smithi006 bash[33918]: audit 2023-08-28T02:04:56.574507+0000 mon.c (mon.2) 2905 : audit [DBG] from='client.? 172.21.15.6:0/3966316783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:57.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:56 smithi130 bash[27579]: audit 2023-08-28T02:04:56.574507+0000 mon.c (mon.2) 2905 : audit [DBG] from='client.? 172.21.15.6:0/3966316783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:57.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:57 smithi130 bash[51302]: debug 2023-08-28T02:04:57.249+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:57.250458+0000) 2023-08-28T02:04:57.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:57 smithi006 bash[27292]: cluster 2023-08-28T02:04:57.074978+0000 mgr.y (mgr.14901) 7390 : cluster [DBG] pgmap v7397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:57.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:57 smithi006 bash[33918]: cluster 2023-08-28T02:04:57.074978+0000 mgr.y (mgr.14901) 7390 : cluster [DBG] pgmap v7397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:58.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:57 smithi130 bash[27579]: cluster 2023-08-28T02:04:57.074978+0000 mgr.y (mgr.14901) 7390 : cluster [DBG] pgmap v7397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:04:58.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:58 smithi130 bash[51302]: debug 2023-08-28T02:04:58.249+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:58.250733+0000) 2023-08-28T02:04:59.364 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:04:59 smithi130 bash[27579]: audit 2023-08-28T02:04:58.994698+0000 mon.a (mon.0) 4775 : audit [DBG] from='client.? 172.21.15.6:0/3517324694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:59.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:04:59 smithi130 bash[51302]: debug 2023-08-28T02:04:59.249+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:59.251028+0000) 2023-08-28T02:04:59.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:04:59 smithi006 bash[27292]: audit 2023-08-28T02:04:58.994698+0000 mon.a (mon.0) 4775 : audit [DBG] from='client.? 172.21.15.6:0/3517324694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:59.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:04:59 smithi006 bash[33918]: audit 2023-08-28T02:04:58.994698+0000 mon.a (mon.0) 4775 : audit [DBG] from='client.? 172.21.15.6:0/3517324694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:00.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:00 smithi130 bash[51302]: debug 2023-08-28T02:05:00.248+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:00.251240+0000) 2023-08-28T02:05:00.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:00 smithi130 bash[27579]: cluster 2023-08-28T02:04:59.075771+0000 mgr.y (mgr.14901) 7391 : cluster [DBG] pgmap v7398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:00.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:00 smithi006 bash[33918]: cluster 2023-08-28T02:04:59.075771+0000 mgr.y (mgr.14901) 7391 : cluster [DBG] pgmap v7398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:00.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:00 smithi006 bash[27292]: cluster 2023-08-28T02:04:59.075771+0000 mgr.y (mgr.14901) 7391 : cluster [DBG] pgmap v7398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:01.203 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:05:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:05:00] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:05:01.364 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:01 smithi130 bash[51302]: debug 2023-08-28T02:05:01.248+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:01.251495+0000) 2023-08-28T02:05:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:05:01.203Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:05:01.205Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:02.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:02 smithi006 bash[33918]: cluster 2023-08-28T02:05:01.076757+0000 mgr.y (mgr.14901) 7392 : cluster [DBG] pgmap v7399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:02.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:02 smithi006 bash[33918]: audit 2023-08-28T02:05:01.407981+0000 mon.c (mon.2) 2906 : audit [DBG] from='client.? 172.21.15.6:0/3129737272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:02.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:02 smithi006 bash[27292]: cluster 2023-08-28T02:05:01.076757+0000 mgr.y (mgr.14901) 7392 : cluster [DBG] pgmap v7399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:02.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:02 smithi006 bash[27292]: audit 2023-08-28T02:05:01.407981+0000 mon.c (mon.2) 2906 : audit [DBG] from='client.? 172.21.15.6:0/3129737272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:02.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:02 smithi130 bash[27579]: cluster 2023-08-28T02:05:01.076757+0000 mgr.y (mgr.14901) 7392 : cluster [DBG] pgmap v7399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:02.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:02 smithi130 bash[27579]: audit 2023-08-28T02:05:01.407981+0000 mon.c (mon.2) 2906 : audit [DBG] from='client.? 172.21.15.6:0/3129737272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:02.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:02 smithi130 bash[51302]: debug 2023-08-28T02:05:02.248+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:02.251732+0000) 2023-08-28T02:05:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:05:02.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:05:02.770Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:03.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:03 smithi130 bash[51302]: debug 2023-08-28T02:05:03.248+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:03.252029+0000) 2023-08-28T02:05:03.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:03 smithi006 bash[33918]: cluster 2023-08-28T02:05:03.077435+0000 mgr.y (mgr.14901) 7393 : cluster [DBG] pgmap v7400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:03.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:03 smithi006 bash[27292]: cluster 2023-08-28T02:05:03.077435+0000 mgr.y (mgr.14901) 7393 : cluster [DBG] pgmap v7400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:04.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:03 smithi130 bash[27579]: cluster 2023-08-28T02:05:03.077435+0000 mgr.y (mgr.14901) 7393 : cluster [DBG] pgmap v7400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:04.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:04 smithi130 bash[51302]: debug 2023-08-28T02:05:04.248+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:04.252314+0000) 2023-08-28T02:05:04.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:04 smithi006 bash[33918]: audit 2023-08-28T02:05:03.827736+0000 mon.c (mon.2) 2907 : audit [DBG] from='client.? 172.21.15.6:0/1254981719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:04.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:04 smithi006 bash[27292]: audit 2023-08-28T02:05:03.827736+0000 mon.c (mon.2) 2907 : audit [DBG] from='client.? 172.21.15.6:0/1254981719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:05:04.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:05:04.647Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:05.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:04 smithi130 bash[27579]: audit 2023-08-28T02:05:03.827736+0000 mon.c (mon.2) 2907 : audit [DBG] from='client.? 172.21.15.6:0/1254981719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:05.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:05 smithi130 bash[51302]: debug 2023-08-28T02:05:05.248+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:05.252579+0000) 2023-08-28T02:05:05.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:05 smithi006 bash[33918]: cluster 2023-08-28T02:05:05.078576+0000 mgr.y (mgr.14901) 7394 : cluster [DBG] pgmap v7401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:05.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:05 smithi006 bash[27292]: cluster 2023-08-28T02:05:05.078576+0000 mgr.y (mgr.14901) 7394 : cluster [DBG] pgmap v7401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:06.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:05 smithi130 bash[27579]: cluster 2023-08-28T02:05:05.078576+0000 mgr.y (mgr.14901) 7394 : cluster [DBG] pgmap v7401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:06.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:06 smithi130 bash[51302]: debug 2023-08-28T02:05:06.248+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:06.252854+0000) 2023-08-28T02:05:06.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:06 smithi006 bash[27292]: audit 2023-08-28T02:05:06.236214+0000 mon.c (mon.2) 2908 : audit [DBG] from='client.? 172.21.15.6:0/2609199201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:06.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:06 smithi006 bash[33918]: audit 2023-08-28T02:05:06.236214+0000 mon.c (mon.2) 2908 : audit [DBG] from='client.? 172.21.15.6:0/2609199201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:07.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:06 smithi130 bash[27579]: audit 2023-08-28T02:05:06.236214+0000 mon.c (mon.2) 2908 : audit [DBG] from='client.? 172.21.15.6:0/2609199201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:07.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:07 smithi130 bash[51302]: debug 2023-08-28T02:05:07.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:07.253128+0000) 2023-08-28T02:05:07.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:07 smithi006 bash[33918]: cluster 2023-08-28T02:05:07.079235+0000 mgr.y (mgr.14901) 7395 : cluster [DBG] pgmap v7402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:07.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:07 smithi006 bash[27292]: cluster 2023-08-28T02:05:07.079235+0000 mgr.y (mgr.14901) 7395 : cluster [DBG] pgmap v7402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:08.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:07 smithi130 bash[27579]: cluster 2023-08-28T02:05:07.079235+0000 mgr.y (mgr.14901) 7395 : cluster [DBG] pgmap v7402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:08.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:08 smithi130 bash[51302]: debug 2023-08-28T02:05:08.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:08.253417+0000) 2023-08-28T02:05:08.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:08 smithi006 bash[33918]: audit 2023-08-28T02:05:08.655915+0000 mon.a (mon.0) 4776 : audit [DBG] from='client.? 172.21.15.6:0/2098279533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:08.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:08 smithi006 bash[27292]: audit 2023-08-28T02:05:08.655915+0000 mon.a (mon.0) 4776 : audit [DBG] from='client.? 172.21.15.6:0/2098279533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:09.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:08 smithi130 bash[27579]: audit 2023-08-28T02:05:08.655915+0000 mon.a (mon.0) 4776 : audit [DBG] from='client.? 172.21.15.6:0/2098279533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:09.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:09 smithi130 bash[51302]: debug 2023-08-28T02:05:09.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:09.253697+0000) 2023-08-28T02:05:10.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:09 smithi130 bash[27579]: cluster 2023-08-28T02:05:09.080186+0000 mgr.y (mgr.14901) 7396 : cluster [DBG] pgmap v7403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:10.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:09 smithi006 bash[27292]: cluster 2023-08-28T02:05:09.080186+0000 mgr.y (mgr.14901) 7396 : cluster [DBG] pgmap v7403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:10.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:09 smithi006 bash[33918]: cluster 2023-08-28T02:05:09.080186+0000 mgr.y (mgr.14901) 7396 : cluster [DBG] pgmap v7403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:10.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:10 smithi130 bash[51302]: debug 2023-08-28T02:05:10.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:10.253977+0000) 2023-08-28T02:05:11.127 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:05:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:05:10] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:05:11.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:11 smithi006 bash[33918]: audit 2023-08-28T02:05:11.074773+0000 mon.c (mon.2) 2909 : audit [DBG] from='client.? 172.21.15.6:0/307837976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:11.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:11 smithi006 bash[27292]: audit 2023-08-28T02:05:11.074773+0000 mon.c (mon.2) 2909 : audit [DBG] from='client.? 172.21.15.6:0/307837976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:05:11.204Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:11.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:05:11.206Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:11.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:11 smithi130 bash[27579]: audit 2023-08-28T02:05:11.074773+0000 mon.c (mon.2) 2909 : audit [DBG] from='client.? 172.21.15.6:0/307837976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:11.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:11 smithi130 bash[51302]: debug 2023-08-28T02:05:11.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:11.254297+0000) 2023-08-28T02:05:12.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:12 smithi006 bash[27292]: cluster 2023-08-28T02:05:11.080900+0000 mgr.y (mgr.14901) 7397 : cluster [DBG] pgmap v7404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:12.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:12 smithi006 bash[33918]: cluster 2023-08-28T02:05:11.080900+0000 mgr.y (mgr.14901) 7397 : cluster [DBG] pgmap v7404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:12.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:12 smithi130 bash[27579]: cluster 2023-08-28T02:05:11.080900+0000 mgr.y (mgr.14901) 7397 : cluster [DBG] pgmap v7404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:12.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:12 smithi130 bash[51302]: debug 2023-08-28T02:05:12.248+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:12.254570+0000) 2023-08-28T02:05:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:05:12.770Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:05:12.771Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:13.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:13 smithi130 bash[51302]: debug 2023-08-28T02:05:13.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:13.254873+0000) 2023-08-28T02:05:13.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:13 smithi006 bash[27292]: cluster 2023-08-28T02:05:13.081556+0000 mgr.y (mgr.14901) 7398 : cluster [DBG] pgmap v7405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:13.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:13 smithi006 bash[27292]: audit 2023-08-28T02:05:13.491061+0000 mon.c (mon.2) 2910 : audit [DBG] from='client.? 172.21.15.6:0/3425708722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:13.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:13 smithi006 bash[33918]: cluster 2023-08-28T02:05:13.081556+0000 mgr.y (mgr.14901) 7398 : cluster [DBG] pgmap v7405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:13.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:13 smithi006 bash[33918]: audit 2023-08-28T02:05:13.491061+0000 mon.c (mon.2) 2910 : audit [DBG] from='client.? 172.21.15.6:0/3425708722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:14.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:13 smithi130 bash[27579]: cluster 2023-08-28T02:05:13.081556+0000 mgr.y (mgr.14901) 7398 : cluster [DBG] pgmap v7405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:14.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:13 smithi130 bash[27579]: audit 2023-08-28T02:05:13.491061+0000 mon.c (mon.2) 2910 : audit [DBG] from='client.? 172.21.15.6:0/3425708722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:14.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:14 smithi130 bash[51302]: debug 2023-08-28T02:05:14.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:14.255181+0000) 2023-08-28T02:05:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:05:14.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:05:14.648Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:15.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:15 smithi130 bash[51302]: debug 2023-08-28T02:05:15.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:15.255488+0000) 2023-08-28T02:05:16.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:16 smithi006 bash[33918]: cluster 2023-08-28T02:05:15.082688+0000 mgr.y (mgr.14901) 7399 : cluster [DBG] pgmap v7406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:16.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:16 smithi006 bash[33918]: audit 2023-08-28T02:05:15.913240+0000 mon.c (mon.2) 2911 : audit [DBG] from='client.? 172.21.15.6:0/1273308867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:16.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:16 smithi006 bash[27292]: cluster 2023-08-28T02:05:15.082688+0000 mgr.y (mgr.14901) 7399 : cluster [DBG] pgmap v7406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:16.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:16 smithi006 bash[27292]: audit 2023-08-28T02:05:15.913240+0000 mon.c (mon.2) 2911 : audit [DBG] from='client.? 172.21.15.6:0/1273308867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:16.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:16 smithi130 bash[51302]: debug 2023-08-28T02:05:16.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:16.255776+0000) 2023-08-28T02:05:16.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:16 smithi130 bash[27579]: cluster 2023-08-28T02:05:15.082688+0000 mgr.y (mgr.14901) 7399 : cluster [DBG] pgmap v7406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:16.619 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:16 smithi130 bash[27579]: audit 2023-08-28T02:05:15.913240+0000 mon.c (mon.2) 2911 : audit [DBG] from='client.? 172.21.15.6:0/1273308867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:17.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:17 smithi130 bash[51302]: debug 2023-08-28T02:05:17.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:17.256070+0000) 2023-08-28T02:05:18.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:18 smithi006 bash[33918]: cluster 2023-08-28T02:05:17.083363+0000 mgr.y (mgr.14901) 7400 : cluster [DBG] pgmap v7407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:18.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:18 smithi006 bash[27292]: cluster 2023-08-28T02:05:17.083363+0000 mgr.y (mgr.14901) 7400 : cluster [DBG] pgmap v7407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:18.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:18 smithi130 bash[27579]: cluster 2023-08-28T02:05:17.083363+0000 mgr.y (mgr.14901) 7400 : cluster [DBG] pgmap v7407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:18.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:18 smithi130 bash[51302]: debug 2023-08-28T02:05:18.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:18.256355+0000) 2023-08-28T02:05:19.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:19 smithi006 bash[27292]: audit 2023-08-28T02:05:18.339035+0000 mon.a (mon.0) 4777 : audit [DBG] from='client.? 172.21.15.6:0/3962307439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:19.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:19 smithi006 bash[33918]: audit 2023-08-28T02:05:18.339035+0000 mon.a (mon.0) 4777 : audit [DBG] from='client.? 172.21.15.6:0/3962307439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:19.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:19 smithi130 bash[27579]: audit 2023-08-28T02:05:18.339035+0000 mon.a (mon.0) 4777 : audit [DBG] from='client.? 172.21.15.6:0/3962307439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:19.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:19 smithi130 bash[51302]: debug 2023-08-28T02:05:19.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:19.256635+0000) 2023-08-28T02:05:20.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:20 smithi006 bash[27292]: cluster 2023-08-28T02:05:19.084526+0000 mgr.y (mgr.14901) 7401 : cluster [DBG] pgmap v7408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:20.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:20 smithi006 bash[33918]: cluster 2023-08-28T02:05:19.084526+0000 mgr.y (mgr.14901) 7401 : cluster [DBG] pgmap v7408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:20.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:20 smithi130 bash[27579]: cluster 2023-08-28T02:05:19.084526+0000 mgr.y (mgr.14901) 7401 : cluster [DBG] pgmap v7408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:20.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:20 smithi130 bash[51302]: debug 2023-08-28T02:05:20.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:20.256914+0000) 2023-08-28T02:05:21.171 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:05:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:05:20] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:05:21.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:21 smithi006 bash[33918]: audit 2023-08-28T02:05:20.759487+0000 mon.c (mon.2) 2912 : audit [DBG] from='client.? 172.21.15.6:0/3461560138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:21.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:21 smithi006 bash[27292]: audit 2023-08-28T02:05:20.759487+0000 mon.c (mon.2) 2912 : audit [DBG] from='client.? 172.21.15.6:0/3461560138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:21.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:05:21.205Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:21.467 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:05:21.207Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:21.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:21 smithi130 bash[27579]: audit 2023-08-28T02:05:20.759487+0000 mon.c (mon.2) 2912 : audit [DBG] from='client.? 172.21.15.6:0/3461560138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:21.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:21 smithi130 bash[51302]: debug 2023-08-28T02:05:21.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:21.257212+0000) 2023-08-28T02:05:22.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:22 smithi006 bash[33918]: cluster 2023-08-28T02:05:21.085372+0000 mgr.y (mgr.14901) 7402 : cluster [DBG] pgmap v7409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:22.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:22 smithi006 bash[27292]: cluster 2023-08-28T02:05:21.085372+0000 mgr.y (mgr.14901) 7402 : cluster [DBG] pgmap v7409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:22.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:22 smithi130 bash[51302]: debug 2023-08-28T02:05:22.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:22.257505+0000) 2023-08-28T02:05:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:22 smithi130 bash[27579]: cluster 2023-08-28T02:05:21.085372+0000 mgr.y (mgr.14901) 7402 : cluster [DBG] pgmap v7409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:05:22.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:05:22.772Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:23.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:23 smithi130 bash[51302]: debug 2023-08-28T02:05:23.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:23.257788+0000) 2023-08-28T02:05:23.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:23 smithi006 bash[33918]: cluster 2023-08-28T02:05:23.086049+0000 mgr.y (mgr.14901) 7403 : cluster [DBG] pgmap v7410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:23.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:23 smithi006 bash[33918]: audit 2023-08-28T02:05:23.171043+0000 mon.c (mon.2) 2913 : audit [DBG] from='client.? 172.21.15.6:0/468432271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:23.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:23 smithi006 bash[27292]: cluster 2023-08-28T02:05:23.086049+0000 mgr.y (mgr.14901) 7403 : cluster [DBG] pgmap v7410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:23.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:23 smithi006 bash[27292]: audit 2023-08-28T02:05:23.171043+0000 mon.c (mon.2) 2913 : audit [DBG] from='client.? 172.21.15.6:0/468432271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:24.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:23 smithi130 bash[27579]: cluster 2023-08-28T02:05:23.086049+0000 mgr.y (mgr.14901) 7403 : cluster [DBG] pgmap v7410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:24.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:23 smithi130 bash[27579]: audit 2023-08-28T02:05:23.171043+0000 mon.c (mon.2) 2913 : audit [DBG] from='client.? 172.21.15.6:0/468432271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:24.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:24 smithi130 bash[51302]: debug 2023-08-28T02:05:24.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:24.258109+0000) 2023-08-28T02:05:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:05:24.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:05:24.649Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:25.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:25 smithi130 bash[51302]: debug 2023-08-28T02:05:25.252+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:25.258386+0000) 2023-08-28T02:05:26.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:26 smithi006 bash[27292]: cluster 2023-08-28T02:05:25.087148+0000 mgr.y (mgr.14901) 7404 : cluster [DBG] pgmap v7411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:26.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:26 smithi006 bash[27292]: audit 2023-08-28T02:05:25.589358+0000 mon.c (mon.2) 2914 : audit [DBG] from='client.? 172.21.15.6:0/2686493915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:26.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:26 smithi006 bash[33918]: cluster 2023-08-28T02:05:25.087148+0000 mgr.y (mgr.14901) 7404 : cluster [DBG] pgmap v7411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:26.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:26 smithi006 bash[33918]: audit 2023-08-28T02:05:25.589358+0000 mon.c (mon.2) 2914 : audit [DBG] from='client.? 172.21.15.6:0/2686493915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:26.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:26 smithi130 bash[51302]: debug 2023-08-28T02:05:26.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:26.258669+0000) 2023-08-28T02:05:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:26 smithi130 bash[27579]: cluster 2023-08-28T02:05:25.087148+0000 mgr.y (mgr.14901) 7404 : cluster [DBG] pgmap v7411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:26 smithi130 bash[27579]: audit 2023-08-28T02:05:25.589358+0000 mon.c (mon.2) 2914 : audit [DBG] from='client.? 172.21.15.6:0/2686493915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:27.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:27 smithi006 bash[27292]: audit 2023-08-28T02:05:26.805709+0000 mon.b (mon.1) 822 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:05:27.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:27 smithi006 bash[33918]: audit 2023-08-28T02:05:26.805709+0000 mon.b (mon.1) 822 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:05:27.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:27 smithi130 bash[27579]: audit 2023-08-28T02:05:26.805709+0000 mon.b (mon.1) 822 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:05:27.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:27 smithi130 bash[51302]: debug 2023-08-28T02:05:27.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:27.258982+0000) 2023-08-28T02:05:28.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:28 smithi006 bash[33918]: cluster 2023-08-28T02:05:27.087813+0000 mgr.y (mgr.14901) 7405 : cluster [DBG] pgmap v7412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:28.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:28 smithi006 bash[33918]: audit 2023-08-28T02:05:28.017275+0000 mon.c (mon.2) 2915 : audit [DBG] from='client.? 172.21.15.6:0/4252489352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:28.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:28 smithi006 bash[27292]: cluster 2023-08-28T02:05:27.087813+0000 mgr.y (mgr.14901) 7405 : cluster [DBG] pgmap v7412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:28.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:28 smithi006 bash[27292]: audit 2023-08-28T02:05:28.017275+0000 mon.c (mon.2) 2915 : audit [DBG] from='client.? 172.21.15.6:0/4252489352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:28.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:28 smithi130 bash[51302]: debug 2023-08-28T02:05:28.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:28.259228+0000) 2023-08-28T02:05:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:28 smithi130 bash[27579]: cluster 2023-08-28T02:05:27.087813+0000 mgr.y (mgr.14901) 7405 : cluster [DBG] pgmap v7412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:28 smithi130 bash[27579]: audit 2023-08-28T02:05:28.017275+0000 mon.c (mon.2) 2915 : audit [DBG] from='client.? 172.21.15.6:0/4252489352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:29.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:29 smithi130 bash[51302]: debug 2023-08-28T02:05:29.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:29.259514+0000) 2023-08-28T02:05:30.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:30 smithi006 bash[33918]: cluster 2023-08-28T02:05:29.088808+0000 mgr.y (mgr.14901) 7406 : cluster [DBG] pgmap v7413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:30.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:30 smithi006 bash[27292]: cluster 2023-08-28T02:05:29.088808+0000 mgr.y (mgr.14901) 7406 : cluster [DBG] pgmap v7413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:30.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:30 smithi130 bash[27579]: cluster 2023-08-28T02:05:29.088808+0000 mgr.y (mgr.14901) 7406 : cluster [DBG] pgmap v7413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:30.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:30 smithi130 bash[51302]: debug 2023-08-28T02:05:30.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:30.259832+0000) 2023-08-28T02:05:31.170 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:05:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:05:30] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:05:31.457 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:31 smithi006 bash[33918]: audit 2023-08-28T02:05:30.445812+0000 mon.a (mon.0) 4778 : audit [DBG] from='client.? 172.21.15.6:0/3750616670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:31.458 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:05:31.205Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:31.458 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:31 smithi006 bash[27292]: audit 2023-08-28T02:05:30.445812+0000 mon.a (mon.0) 4778 : audit [DBG] from='client.? 172.21.15.6:0/3750616670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:31.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:31 smithi130 bash[27579]: audit 2023-08-28T02:05:30.445812+0000 mon.a (mon.0) 4778 : audit [DBG] from='client.? 172.21.15.6:0/3750616670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:31.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:31 smithi130 bash[51302]: debug 2023-08-28T02:05:31.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:31.260081+0000) 2023-08-28T02:05:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:05:31.207Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:32.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:32 smithi006 bash[27292]: cluster 2023-08-28T02:05:31.089611+0000 mgr.y (mgr.14901) 7407 : cluster [DBG] pgmap v7414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:32.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:32 smithi006 bash[33918]: cluster 2023-08-28T02:05:31.089611+0000 mgr.y (mgr.14901) 7407 : cluster [DBG] pgmap v7414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:32.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:32 smithi130 bash[27579]: cluster 2023-08-28T02:05:31.089611+0000 mgr.y (mgr.14901) 7407 : cluster [DBG] pgmap v7414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:32.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:32 smithi130 bash[51302]: debug 2023-08-28T02:05:32.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:32.260357+0000) 2023-08-28T02:05:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:05:32.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:05:32.773Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:33.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:33 smithi130 bash[51302]: debug 2023-08-28T02:05:33.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:33.260596+0000) 2023-08-28T02:05:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:33 smithi006 bash[27292]: audit 2023-08-28T02:05:32.711100+0000 mon.a (mon.0) 4779 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:05:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:33 smithi006 bash[27292]: audit 2023-08-28T02:05:32.899656+0000 mon.c (mon.2) 2916 : audit [DBG] from='client.? 172.21.15.6:0/2299682151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:33 smithi006 bash[27292]: audit 2023-08-28T02:05:33.087776+0000 mon.a (mon.0) 4780 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:05:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:33 smithi006 bash[27292]: cluster 2023-08-28T02:05:33.090142+0000 mgr.y (mgr.14901) 7408 : cluster [DBG] pgmap v7415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:33 smithi006 bash[27292]: audit 2023-08-28T02:05:33.380420+0000 mon.b (mon.1) 823 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:05:33.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:33 smithi006 bash[27292]: audit 2023-08-28T02:05:33.382747+0000 mon.b (mon.1) 824 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:05:33.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:33 smithi006 bash[27292]: audit 2023-08-28T02:05:33.392432+0000 mon.a (mon.0) 4781 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:05:33.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:33 smithi006 bash[33918]: audit 2023-08-28T02:05:32.711100+0000 mon.a (mon.0) 4779 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:05:33.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:33 smithi006 bash[33918]: audit 2023-08-28T02:05:32.899656+0000 mon.c (mon.2) 2916 : audit [DBG] from='client.? 172.21.15.6:0/2299682151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:33.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:33 smithi006 bash[33918]: audit 2023-08-28T02:05:33.087776+0000 mon.a (mon.0) 4780 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:05:33.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:33 smithi006 bash[33918]: cluster 2023-08-28T02:05:33.090142+0000 mgr.y (mgr.14901) 7408 : cluster [DBG] pgmap v7415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:33.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:33 smithi006 bash[33918]: audit 2023-08-28T02:05:33.380420+0000 mon.b (mon.1) 823 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:05:33.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:33 smithi006 bash[33918]: audit 2023-08-28T02:05:33.382747+0000 mon.b (mon.1) 824 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:05:33.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:33 smithi006 bash[33918]: audit 2023-08-28T02:05:33.392432+0000 mon.a (mon.0) 4781 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:05:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:33 smithi130 bash[27579]: audit 2023-08-28T02:05:32.711100+0000 mon.a (mon.0) 4779 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:05:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:33 smithi130 bash[27579]: audit 2023-08-28T02:05:32.899656+0000 mon.c (mon.2) 2916 : audit [DBG] from='client.? 172.21.15.6:0/2299682151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:33 smithi130 bash[27579]: audit 2023-08-28T02:05:33.087776+0000 mon.a (mon.0) 4780 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:05:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:33 smithi130 bash[27579]: cluster 2023-08-28T02:05:33.090142+0000 mgr.y (mgr.14901) 7408 : cluster [DBG] pgmap v7415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:34.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:33 smithi130 bash[27579]: audit 2023-08-28T02:05:33.380420+0000 mon.b (mon.1) 823 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:05:34.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:33 smithi130 bash[27579]: audit 2023-08-28T02:05:33.382747+0000 mon.b (mon.1) 824 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:05:34.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:33 smithi130 bash[27579]: audit 2023-08-28T02:05:33.392432+0000 mon.a (mon.0) 4781 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:05:34.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:34 smithi130 bash[51302]: debug 2023-08-28T02:05:34.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:34.260926+0000) 2023-08-28T02:05:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:05:34.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:05:34.650Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:35.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:35 smithi130 bash[51302]: debug 2023-08-28T02:05:35.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:35.261203+0000) 2023-08-28T02:05:36.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:36 smithi006 bash[33918]: cluster 2023-08-28T02:05:35.091199+0000 mgr.y (mgr.14901) 7409 : cluster [DBG] pgmap v7416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:36.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:36 smithi006 bash[33918]: audit 2023-08-28T02:05:35.321301+0000 mon.a (mon.0) 4782 : audit [DBG] from='client.? 172.21.15.6:0/2532135867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:36.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:36 smithi006 bash[27292]: cluster 2023-08-28T02:05:35.091199+0000 mgr.y (mgr.14901) 7409 : cluster [DBG] pgmap v7416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:36.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:36 smithi006 bash[27292]: audit 2023-08-28T02:05:35.321301+0000 mon.a (mon.0) 4782 : audit [DBG] from='client.? 172.21.15.6:0/2532135867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:36.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:36 smithi130 bash[27579]: cluster 2023-08-28T02:05:35.091199+0000 mgr.y (mgr.14901) 7409 : cluster [DBG] pgmap v7416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:36 smithi130 bash[27579]: audit 2023-08-28T02:05:35.321301+0000 mon.a (mon.0) 4782 : audit [DBG] from='client.? 172.21.15.6:0/2532135867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:36.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:36 smithi130 bash[51302]: debug 2023-08-28T02:05:36.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:36.261543+0000) 2023-08-28T02:05:37.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:37 smithi130 bash[51302]: debug 2023-08-28T02:05:37.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:37.261851+0000) 2023-08-28T02:05:38.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:38 smithi006 bash[27292]: cluster 2023-08-28T02:05:37.091869+0000 mgr.y (mgr.14901) 7410 : cluster [DBG] pgmap v7417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:38.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:38 smithi006 bash[27292]: audit 2023-08-28T02:05:37.745773+0000 mon.c (mon.2) 2917 : audit [DBG] from='client.? 172.21.15.6:0/2566707736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:38.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:38 smithi006 bash[33918]: cluster 2023-08-28T02:05:37.091869+0000 mgr.y (mgr.14901) 7410 : cluster [DBG] pgmap v7417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:38.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:38 smithi006 bash[33918]: audit 2023-08-28T02:05:37.745773+0000 mon.c (mon.2) 2917 : audit [DBG] from='client.? 172.21.15.6:0/2566707736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:38.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:38 smithi130 bash[51302]: debug 2023-08-28T02:05:38.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:38.262165+0000) 2023-08-28T02:05:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:38 smithi130 bash[27579]: cluster 2023-08-28T02:05:37.091869+0000 mgr.y (mgr.14901) 7410 : cluster [DBG] pgmap v7417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:38 smithi130 bash[27579]: audit 2023-08-28T02:05:37.745773+0000 mon.c (mon.2) 2917 : audit [DBG] from='client.? 172.21.15.6:0/2566707736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:39.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:39 smithi130 bash[51302]: debug 2023-08-28T02:05:39.256+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:39.262480+0000) 2023-08-28T02:05:40.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:40 smithi006 bash[27292]: cluster 2023-08-28T02:05:39.092936+0000 mgr.y (mgr.14901) 7411 : cluster [DBG] pgmap v7418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:40.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:40 smithi006 bash[33918]: cluster 2023-08-28T02:05:39.092936+0000 mgr.y (mgr.14901) 7411 : cluster [DBG] pgmap v7418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:40.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:40 smithi130 bash[27579]: cluster 2023-08-28T02:05:39.092936+0000 mgr.y (mgr.14901) 7411 : cluster [DBG] pgmap v7418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:40.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:40 smithi130 bash[51302]: debug 2023-08-28T02:05:40.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:40.262760+0000) 2023-08-28T02:05:41.159 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:05:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:05:40] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:05:41.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:41 smithi006 bash[33918]: audit 2023-08-28T02:05:40.162078+0000 mon.a (mon.0) 4783 : audit [DBG] from='client.? 172.21.15.6:0/3009007211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:41.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:41 smithi006 bash[27292]: audit 2023-08-28T02:05:40.162078+0000 mon.a (mon.0) 4783 : audit [DBG] from='client.? 172.21.15.6:0/3009007211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:05:41.206Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:05:41.207Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:41.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:41 smithi130 bash[27579]: audit 2023-08-28T02:05:40.162078+0000 mon.a (mon.0) 4783 : audit [DBG] from='client.? 172.21.15.6:0/3009007211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:41.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:41 smithi130 bash[51302]: debug 2023-08-28T02:05:41.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:41.263098+0000) 2023-08-28T02:05:42.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:42 smithi006 bash[33918]: cluster 2023-08-28T02:05:41.093388+0000 mgr.y (mgr.14901) 7412 : cluster [DBG] pgmap v7419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:42.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:42 smithi006 bash[27292]: cluster 2023-08-28T02:05:41.093388+0000 mgr.y (mgr.14901) 7412 : cluster [DBG] pgmap v7419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:42.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:42 smithi130 bash[27579]: cluster 2023-08-28T02:05:41.093388+0000 mgr.y (mgr.14901) 7412 : cluster [DBG] pgmap v7419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:42.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:42 smithi130 bash[51302]: debug 2023-08-28T02:05:42.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:42.263323+0000) 2023-08-28T02:05:43.170 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:05:42.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:43.170 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:05:42.773Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:43.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:43 smithi006 bash[33918]: audit 2023-08-28T02:05:42.574786+0000 mon.a (mon.0) 4784 : audit [DBG] from='client.? 172.21.15.6:0/2900772828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:43.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:43 smithi006 bash[27292]: audit 2023-08-28T02:05:42.574786+0000 mon.a (mon.0) 4784 : audit [DBG] from='client.? 172.21.15.6:0/2900772828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:43.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:43 smithi130 bash[27579]: audit 2023-08-28T02:05:42.574786+0000 mon.a (mon.0) 4784 : audit [DBG] from='client.? 172.21.15.6:0/2900772828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:43.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:43 smithi130 bash[51302]: debug 2023-08-28T02:05:43.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:43.263539+0000) 2023-08-28T02:05:44.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:44 smithi006 bash[27292]: cluster 2023-08-28T02:05:43.094007+0000 mgr.y (mgr.14901) 7413 : cluster [DBG] pgmap v7420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:44.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:44 smithi006 bash[33918]: cluster 2023-08-28T02:05:43.094007+0000 mgr.y (mgr.14901) 7413 : cluster [DBG] pgmap v7420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:44.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:44 smithi130 bash[27579]: cluster 2023-08-28T02:05:43.094007+0000 mgr.y (mgr.14901) 7413 : cluster [DBG] pgmap v7420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:44.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:44 smithi130 bash[51302]: debug 2023-08-28T02:05:44.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:44.263815+0000) 2023-08-28T02:05:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:05:44.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:45.148 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:05:44.650Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:45.182 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:45 smithi130 bash[27579]: audit 2023-08-28T02:05:44.986742+0000 mon.a (mon.0) 4785 : audit [DBG] from='client.? 172.21.15.6:0/1265317644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:45.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:45 smithi006 bash[27292]: audit 2023-08-28T02:05:44.986742+0000 mon.a (mon.0) 4785 : audit [DBG] from='client.? 172.21.15.6:0/1265317644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:45.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:45 smithi006 bash[33918]: audit 2023-08-28T02:05:44.986742+0000 mon.a (mon.0) 4785 : audit [DBG] from='client.? 172.21.15.6:0/1265317644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:45.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:45 smithi130 bash[51302]: debug 2023-08-28T02:05:45.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:45.264137+0000) 2023-08-28T02:05:46.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:46 smithi006 bash[27292]: cluster 2023-08-28T02:05:45.095112+0000 mgr.y (mgr.14901) 7414 : cluster [DBG] pgmap v7421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:46.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:46 smithi006 bash[33918]: cluster 2023-08-28T02:05:45.095112+0000 mgr.y (mgr.14901) 7414 : cluster [DBG] pgmap v7421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:46.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:46 smithi130 bash[27579]: cluster 2023-08-28T02:05:45.095112+0000 mgr.y (mgr.14901) 7414 : cluster [DBG] pgmap v7421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:46.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:46 smithi130 bash[51302]: debug 2023-08-28T02:05:46.264+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:46.264413+0000) 2023-08-28T02:05:47.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:47 smithi130 bash[51302]: debug 2023-08-28T02:05:47.264+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:47.264687+0000) 2023-08-28T02:05:48.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:48 smithi006 bash[27292]: cluster 2023-08-28T02:05:47.095881+0000 mgr.y (mgr.14901) 7415 : cluster [DBG] pgmap v7422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:48.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:48 smithi006 bash[27292]: audit 2023-08-28T02:05:47.401908+0000 mon.c (mon.2) 2918 : audit [DBG] from='client.? 172.21.15.6:0/4233905432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:48.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:48 smithi006 bash[33918]: cluster 2023-08-28T02:05:47.095881+0000 mgr.y (mgr.14901) 7415 : cluster [DBG] pgmap v7422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:48.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:48 smithi006 bash[33918]: audit 2023-08-28T02:05:47.401908+0000 mon.c (mon.2) 2918 : audit [DBG] from='client.? 172.21.15.6:0/4233905432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:48.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:48 smithi130 bash[51302]: debug 2023-08-28T02:05:48.264+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:48.264967+0000) 2023-08-28T02:05:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:48 smithi130 bash[27579]: cluster 2023-08-28T02:05:47.095881+0000 mgr.y (mgr.14901) 7415 : cluster [DBG] pgmap v7422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:48 smithi130 bash[27579]: audit 2023-08-28T02:05:47.401908+0000 mon.c (mon.2) 2918 : audit [DBG] from='client.? 172.21.15.6:0/4233905432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:49.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:49 smithi130 bash[51302]: debug 2023-08-28T02:05:49.264+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:49.265261+0000) 2023-08-28T02:05:50.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:50 smithi006 bash[33918]: cluster 2023-08-28T02:05:49.097225+0000 mgr.y (mgr.14901) 7416 : cluster [DBG] pgmap v7423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:50.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:50 smithi006 bash[33918]: audit 2023-08-28T02:05:49.821443+0000 mon.a (mon.0) 4786 : audit [DBG] from='client.? 172.21.15.6:0/3731393844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:50.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:50 smithi006 bash[27292]: cluster 2023-08-28T02:05:49.097225+0000 mgr.y (mgr.14901) 7416 : cluster [DBG] pgmap v7423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:50.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:50 smithi006 bash[27292]: audit 2023-08-28T02:05:49.821443+0000 mon.a (mon.0) 4786 : audit [DBG] from='client.? 172.21.15.6:0/3731393844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:50 smithi130 bash[27579]: cluster 2023-08-28T02:05:49.097225+0000 mgr.y (mgr.14901) 7416 : cluster [DBG] pgmap v7423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:50 smithi130 bash[27579]: audit 2023-08-28T02:05:49.821443+0000 mon.a (mon.0) 4786 : audit [DBG] from='client.? 172.21.15.6:0/3731393844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:50.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:50 smithi130 bash[51302]: debug 2023-08-28T02:05:50.264+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:50.265629+0000) 2023-08-28T02:05:51.206 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:05:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:05:50] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-28T02:05:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:05:51.207Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:51.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:05:51.208Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:51.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:51 smithi130 bash[51302]: debug 2023-08-28T02:05:51.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:51.265909+0000) 2023-08-28T02:05:52.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:52 smithi006 bash[33918]: cluster 2023-08-28T02:05:51.097973+0000 mgr.y (mgr.14901) 7417 : cluster [DBG] pgmap v7424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:52.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:52 smithi006 bash[27292]: cluster 2023-08-28T02:05:51.097973+0000 mgr.y (mgr.14901) 7417 : cluster [DBG] pgmap v7424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:52.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:52 smithi130 bash[51302]: debug 2023-08-28T02:05:52.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:52.266211+0000) 2023-08-28T02:05:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:52 smithi130 bash[27579]: cluster 2023-08-28T02:05:51.097973+0000 mgr.y (mgr.14901) 7417 : cluster [DBG] pgmap v7424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:53.200 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:05:52.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": context deadline exceeded" 2023-08-28T02:05:53.201 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:05:52.773Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:53.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:53 smithi006 bash[27292]: audit 2023-08-28T02:05:52.238009+0000 mon.a (mon.0) 4787 : audit [DBG] from='client.? 172.21.15.6:0/3516379874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:53.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:53 smithi006 bash[33918]: audit 2023-08-28T02:05:52.238009+0000 mon.a (mon.0) 4787 : audit [DBG] from='client.? 172.21.15.6:0/3516379874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:53.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:53 smithi130 bash[27579]: audit 2023-08-28T02:05:52.238009+0000 mon.a (mon.0) 4787 : audit [DBG] from='client.? 172.21.15.6:0/3516379874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:53.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:53 smithi130 bash[51302]: debug 2023-08-28T02:05:53.260+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:53.266574+0000) 2023-08-28T02:05:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:54 smithi006 bash[33918]: cluster 2023-08-28T02:05:53.098783+0000 mgr.y (mgr.14901) 7418 : cluster [DBG] pgmap v7425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:54 smithi006 bash[27292]: cluster 2023-08-28T02:05:53.098783+0000 mgr.y (mgr.14901) 7418 : cluster [DBG] pgmap v7425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:54.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:54 smithi130 bash[27579]: cluster 2023-08-28T02:05:53.098783+0000 mgr.y (mgr.14901) 7418 : cluster [DBG] pgmap v7425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:54.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:54 smithi130 bash[51302]: debug 2023-08-28T02:05:54.264+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:54.266847+0000) 2023-08-28T02:05:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:05:54.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:05:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:05:54.650Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:05:55.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:55 smithi006 bash[33918]: audit 2023-08-28T02:05:54.651558+0000 mon.c (mon.2) 2919 : audit [DBG] from='client.? 172.21.15.6:0/954369804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:55.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:55 smithi006 bash[27292]: audit 2023-08-28T02:05:54.651558+0000 mon.c (mon.2) 2919 : audit [DBG] from='client.? 172.21.15.6:0/954369804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:55.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:55 smithi130 bash[27579]: audit 2023-08-28T02:05:54.651558+0000 mon.c (mon.2) 2919 : audit [DBG] from='client.? 172.21.15.6:0/954369804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:55.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:55 smithi130 bash[51302]: debug 2023-08-28T02:05:55.264+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:55.267155+0000) 2023-08-28T02:05:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:56 smithi130 bash[27579]: cluster 2023-08-28T02:05:55.099936+0000 mgr.y (mgr.14901) 7419 : cluster [DBG] pgmap v7426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:56.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:56 smithi130 bash[51302]: debug 2023-08-28T02:05:56.263+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:56.267484+0000) 2023-08-28T02:05:56.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:56 smithi006 bash[33918]: cluster 2023-08-28T02:05:55.099936+0000 mgr.y (mgr.14901) 7419 : cluster [DBG] pgmap v7426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:56.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:56 smithi006 bash[27292]: cluster 2023-08-28T02:05:55.099936+0000 mgr.y (mgr.14901) 7419 : cluster [DBG] pgmap v7426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:57.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:57 smithi130 bash[27579]: audit 2023-08-28T02:05:57.055843+0000 mon.a (mon.0) 4788 : audit [DBG] from='client.? 172.21.15.6:0/875559128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:57.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:57 smithi130 bash[51302]: debug 2023-08-28T02:05:57.263+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:57.267818+0000) 2023-08-28T02:05:57.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:57 smithi006 bash[33918]: audit 2023-08-28T02:05:57.055843+0000 mon.a (mon.0) 4788 : audit [DBG] from='client.? 172.21.15.6:0/875559128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:57.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:57 smithi006 bash[27292]: audit 2023-08-28T02:05:57.055843+0000 mon.a (mon.0) 4788 : audit [DBG] from='client.? 172.21.15.6:0/875559128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:58.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:05:58 smithi130 bash[27579]: cluster 2023-08-28T02:05:57.100685+0000 mgr.y (mgr.14901) 7420 : cluster [DBG] pgmap v7427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:58.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:58 smithi130 bash[51302]: debug 2023-08-28T02:05:58.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:58.268084+0000) 2023-08-28T02:05:58.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:05:58 smithi006 bash[33918]: cluster 2023-08-28T02:05:57.100685+0000 mgr.y (mgr.14901) 7420 : cluster [DBG] pgmap v7427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:58.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:05:58 smithi006 bash[27292]: cluster 2023-08-28T02:05:57.100685+0000 mgr.y (mgr.14901) 7420 : cluster [DBG] pgmap v7427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:05:59.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:05:59 smithi130 bash[51302]: debug 2023-08-28T02:05:59.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:59.268376+0000) 2023-08-28T02:06:00.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:00 smithi130 bash[27579]: cluster 2023-08-28T02:05:59.101704+0000 mgr.y (mgr.14901) 7421 : cluster [DBG] pgmap v7428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:00 smithi130 bash[27579]: audit 2023-08-28T02:05:59.475485+0000 mon.a (mon.0) 4789 : audit [DBG] from='client.? 172.21.15.6:0/1277008406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:00.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:00 smithi130 bash[51302]: debug 2023-08-28T02:06:00.263+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:00.268659+0000) 2023-08-28T02:06:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:00 smithi006 bash[33918]: cluster 2023-08-28T02:05:59.101704+0000 mgr.y (mgr.14901) 7421 : cluster [DBG] pgmap v7428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:00 smithi006 bash[33918]: audit 2023-08-28T02:05:59.475485+0000 mon.a (mon.0) 4789 : audit [DBG] from='client.? 172.21.15.6:0/1277008406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:00 smithi006 bash[27292]: cluster 2023-08-28T02:05:59.101704+0000 mgr.y (mgr.14901) 7421 : cluster [DBG] pgmap v7428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:00 smithi006 bash[27292]: audit 2023-08-28T02:05:59.475485+0000 mon.a (mon.0) 4789 : audit [DBG] from='client.? 172.21.15.6:0/1277008406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:01.207 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:06:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:06:00] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:06:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:06:01.207Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:01.570 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:06:01.209Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:01.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:01 smithi130 bash[51302]: debug 2023-08-28T02:06:01.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:01.268950+0000) 2023-08-28T02:06:02.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:02 smithi130 bash[51302]: debug 2023-08-28T02:06:02.263+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:02.269227+0000) 2023-08-28T02:06:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:02 smithi130 bash[27579]: cluster 2023-08-28T02:06:01.102382+0000 mgr.y (mgr.14901) 7422 : cluster [DBG] pgmap v7429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:02 smithi130 bash[27579]: audit 2023-08-28T02:06:01.881288+0000 mon.a (mon.0) 4790 : audit [DBG] from='client.? 172.21.15.6:0/2325415101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:02.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:02 smithi006 bash[27292]: cluster 2023-08-28T02:06:01.102382+0000 mgr.y (mgr.14901) 7422 : cluster [DBG] pgmap v7429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:02.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:02 smithi006 bash[27292]: audit 2023-08-28T02:06:01.881288+0000 mon.a (mon.0) 4790 : audit [DBG] from='client.? 172.21.15.6:0/2325415101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:02.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:02 smithi006 bash[33918]: cluster 2023-08-28T02:06:01.102382+0000 mgr.y (mgr.14901) 7422 : cluster [DBG] pgmap v7429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:02.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:02 smithi006 bash[33918]: audit 2023-08-28T02:06:01.881288+0000 mon.a (mon.0) 4790 : audit [DBG] from='client.? 172.21.15.6:0/2325415101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:06:02.773Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:06:02.775Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:03.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:03 smithi130 bash[51302]: debug 2023-08-28T02:06:03.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:03.269527+0000) 2023-08-28T02:06:03.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:03 smithi006 bash[27292]: cluster 2023-08-28T02:06:03.103040+0000 mgr.y (mgr.14901) 7423 : cluster [DBG] pgmap v7430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:03.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:03 smithi006 bash[33918]: cluster 2023-08-28T02:06:03.103040+0000 mgr.y (mgr.14901) 7423 : cluster [DBG] pgmap v7430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:04.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:03 smithi130 bash[27579]: cluster 2023-08-28T02:06:03.103040+0000 mgr.y (mgr.14901) 7423 : cluster [DBG] pgmap v7430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:04.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:04 smithi130 bash[51302]: debug 2023-08-28T02:06:04.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:04.269868+0000) 2023-08-28T02:06:04.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:04 smithi006 bash[27292]: audit 2023-08-28T02:06:04.306564+0000 mon.a (mon.0) 4791 : audit [DBG] from='client.? 172.21.15.6:0/3127494608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:04.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:04 smithi006 bash[33918]: audit 2023-08-28T02:06:04.306564+0000 mon.a (mon.0) 4791 : audit [DBG] from='client.? 172.21.15.6:0/3127494608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:06:04.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:06:04.652Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:05.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:04 smithi130 bash[27579]: audit 2023-08-28T02:06:04.306564+0000 mon.a (mon.0) 4791 : audit [DBG] from='client.? 172.21.15.6:0/3127494608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:05.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:05 smithi130 bash[51302]: debug 2023-08-28T02:06:05.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:05.270157+0000) 2023-08-28T02:06:05.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:05 smithi006 bash[33918]: cluster 2023-08-28T02:06:05.104111+0000 mgr.y (mgr.14901) 7424 : cluster [DBG] pgmap v7431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:05.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:05 smithi006 bash[27292]: cluster 2023-08-28T02:06:05.104111+0000 mgr.y (mgr.14901) 7424 : cluster [DBG] pgmap v7431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:06.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:05 smithi130 bash[27579]: cluster 2023-08-28T02:06:05.104111+0000 mgr.y (mgr.14901) 7424 : cluster [DBG] pgmap v7431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:06.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:06 smithi130 bash[51302]: debug 2023-08-28T02:06:06.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:06.270464+0000) 2023-08-28T02:06:07.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:06 smithi130 bash[27579]: audit 2023-08-28T02:06:06.727339+0000 mon.a (mon.0) 4792 : audit [DBG] from='client.? 172.21.15.6:0/523170490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:07.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:06 smithi006 bash[33918]: audit 2023-08-28T02:06:06.727339+0000 mon.a (mon.0) 4792 : audit [DBG] from='client.? 172.21.15.6:0/523170490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:07.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:06 smithi006 bash[27292]: audit 2023-08-28T02:06:06.727339+0000 mon.a (mon.0) 4792 : audit [DBG] from='client.? 172.21.15.6:0/523170490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:07.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:07 smithi130 bash[51302]: debug 2023-08-28T02:06:07.263+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:07.270774+0000) 2023-08-28T02:06:08.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:07 smithi130 bash[27579]: cluster 2023-08-28T02:06:07.104739+0000 mgr.y (mgr.14901) 7425 : cluster [DBG] pgmap v7432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:08.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:07 smithi006 bash[33918]: cluster 2023-08-28T02:06:07.104739+0000 mgr.y (mgr.14901) 7425 : cluster [DBG] pgmap v7432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:08.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:07 smithi006 bash[27292]: cluster 2023-08-28T02:06:07.104739+0000 mgr.y (mgr.14901) 7425 : cluster [DBG] pgmap v7432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:08.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:08 smithi130 bash[51302]: debug 2023-08-28T02:06:08.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:08.271137+0000) 2023-08-28T02:06:09.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:09 smithi006 bash[27292]: audit 2023-08-28T02:06:09.148778+0000 mon.c (mon.2) 2920 : audit [DBG] from='client.? 172.21.15.6:0/1203276690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:09.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:09 smithi006 bash[33918]: audit 2023-08-28T02:06:09.148778+0000 mon.c (mon.2) 2920 : audit [DBG] from='client.? 172.21.15.6:0/1203276690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:09.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:09 smithi130 bash[27579]: audit 2023-08-28T02:06:09.148778+0000 mon.c (mon.2) 2920 : audit [DBG] from='client.? 172.21.15.6:0/1203276690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:09.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:09 smithi130 bash[51302]: debug 2023-08-28T02:06:09.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:09.271458+0000) 2023-08-28T02:06:10.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:10 smithi006 bash[27292]: cluster 2023-08-28T02:06:09.105696+0000 mgr.y (mgr.14901) 7426 : cluster [DBG] pgmap v7433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:10.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:10 smithi006 bash[33918]: cluster 2023-08-28T02:06:09.105696+0000 mgr.y (mgr.14901) 7426 : cluster [DBG] pgmap v7433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:10.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:10 smithi130 bash[27579]: cluster 2023-08-28T02:06:09.105696+0000 mgr.y (mgr.14901) 7426 : cluster [DBG] pgmap v7433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:10.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:10 smithi130 bash[51302]: debug 2023-08-28T02:06:10.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:10.271724+0000) 2023-08-28T02:06:11.207 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:06:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:06:10] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:06:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:06:11.208Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:06:11.210Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:11.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:11 smithi130 bash[51302]: debug 2023-08-28T02:06:11.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:11.272157+0000) 2023-08-28T02:06:12.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:12 smithi006 bash[27292]: cluster 2023-08-28T02:06:11.106325+0000 mgr.y (mgr.14901) 7427 : cluster [DBG] pgmap v7434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:12.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:12 smithi006 bash[27292]: audit 2023-08-28T02:06:11.573290+0000 mon.a (mon.0) 4793 : audit [DBG] from='client.? 172.21.15.6:0/1344999640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:12.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:12 smithi006 bash[33918]: cluster 2023-08-28T02:06:11.106325+0000 mgr.y (mgr.14901) 7427 : cluster [DBG] pgmap v7434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:12.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:12 smithi006 bash[33918]: audit 2023-08-28T02:06:11.573290+0000 mon.a (mon.0) 4793 : audit [DBG] from='client.? 172.21.15.6:0/1344999640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:12.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:12 smithi130 bash[27579]: cluster 2023-08-28T02:06:11.106325+0000 mgr.y (mgr.14901) 7427 : cluster [DBG] pgmap v7434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:12 smithi130 bash[27579]: audit 2023-08-28T02:06:11.573290+0000 mon.a (mon.0) 4793 : audit [DBG] from='client.? 172.21.15.6:0/1344999640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:12.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:12 smithi130 bash[51302]: debug 2023-08-28T02:06:12.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:12.272395+0000) 2023-08-28T02:06:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:06:12.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:06:12.776Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:13.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:13 smithi130 bash[51302]: debug 2023-08-28T02:06:13.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:13.272655+0000) 2023-08-28T02:06:13.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:13 smithi006 bash[33918]: cluster 2023-08-28T02:06:13.107013+0000 mgr.y (mgr.14901) 7428 : cluster [DBG] pgmap v7435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:13.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:13 smithi006 bash[27292]: cluster 2023-08-28T02:06:13.107013+0000 mgr.y (mgr.14901) 7428 : cluster [DBG] pgmap v7435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:14.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:13 smithi130 bash[27579]: cluster 2023-08-28T02:06:13.107013+0000 mgr.y (mgr.14901) 7428 : cluster [DBG] pgmap v7435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:14.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:14 smithi130 bash[51302]: debug 2023-08-28T02:06:14.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:14.272957+0000) 2023-08-28T02:06:14.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:14 smithi006 bash[33918]: audit 2023-08-28T02:06:13.979510+0000 mon.c (mon.2) 2921 : audit [DBG] from='client.? 172.21.15.6:0/1361658212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:14.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:14 smithi006 bash[27292]: audit 2023-08-28T02:06:13.979510+0000 mon.c (mon.2) 2921 : audit [DBG] from='client.? 172.21.15.6:0/1361658212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:06:14.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:06:14.652Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:15.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:14 smithi130 bash[27579]: audit 2023-08-28T02:06:13.979510+0000 mon.c (mon.2) 2921 : audit [DBG] from='client.? 172.21.15.6:0/1361658212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:15.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:15 smithi130 bash[51302]: debug 2023-08-28T02:06:15.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:15.273244+0000) 2023-08-28T02:06:15.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:15 smithi006 bash[27292]: cluster 2023-08-28T02:06:15.108183+0000 mgr.y (mgr.14901) 7429 : cluster [DBG] pgmap v7436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:15.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:15 smithi006 bash[33918]: cluster 2023-08-28T02:06:15.108183+0000 mgr.y (mgr.14901) 7429 : cluster [DBG] pgmap v7436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:16.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:15 smithi130 bash[27579]: cluster 2023-08-28T02:06:15.108183+0000 mgr.y (mgr.14901) 7429 : cluster [DBG] pgmap v7436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:16.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:16 smithi130 bash[51302]: debug 2023-08-28T02:06:16.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:16.273610+0000) 2023-08-28T02:06:16.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:16 smithi006 bash[27292]: audit 2023-08-28T02:06:16.398620+0000 mon.c (mon.2) 2922 : audit [DBG] from='client.? 172.21.15.6:0/3934127100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:16.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:16 smithi006 bash[33918]: audit 2023-08-28T02:06:16.398620+0000 mon.c (mon.2) 2922 : audit [DBG] from='client.? 172.21.15.6:0/3934127100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:17.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:16 smithi130 bash[27579]: audit 2023-08-28T02:06:16.398620+0000 mon.c (mon.2) 2922 : audit [DBG] from='client.? 172.21.15.6:0/3934127100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:17.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:17 smithi130 bash[51302]: debug 2023-08-28T02:06:17.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:17.273878+0000) 2023-08-28T02:06:17.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:17 smithi006 bash[27292]: cluster 2023-08-28T02:06:17.108876+0000 mgr.y (mgr.14901) 7430 : cluster [DBG] pgmap v7437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:17.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:17 smithi006 bash[33918]: cluster 2023-08-28T02:06:17.108876+0000 mgr.y (mgr.14901) 7430 : cluster [DBG] pgmap v7437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:18.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:17 smithi130 bash[27579]: cluster 2023-08-28T02:06:17.108876+0000 mgr.y (mgr.14901) 7430 : cluster [DBG] pgmap v7437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:18.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:18 smithi130 bash[51302]: debug 2023-08-28T02:06:18.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:18.274155+0000) 2023-08-28T02:06:19.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:18 smithi006 bash[33918]: audit 2023-08-28T02:06:18.813880+0000 mon.a (mon.0) 4794 : audit [DBG] from='client.? 172.21.15.6:0/303394729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:19.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:18 smithi006 bash[27292]: audit 2023-08-28T02:06:18.813880+0000 mon.a (mon.0) 4794 : audit [DBG] from='client.? 172.21.15.6:0/303394729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:19.272 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:18 smithi130 bash[27579]: audit 2023-08-28T02:06:18.813880+0000 mon.a (mon.0) 4794 : audit [DBG] from='client.? 172.21.15.6:0/303394729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:19.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:19 smithi130 bash[51302]: debug 2023-08-28T02:06:19.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:19.274424+0000) 2023-08-28T02:06:20.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:19 smithi006 bash[27292]: cluster 2023-08-28T02:06:19.109867+0000 mgr.y (mgr.14901) 7431 : cluster [DBG] pgmap v7438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:20.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:19 smithi006 bash[33918]: cluster 2023-08-28T02:06:19.109867+0000 mgr.y (mgr.14901) 7431 : cluster [DBG] pgmap v7438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:20.272 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:19 smithi130 bash[27579]: cluster 2023-08-28T02:06:19.109867+0000 mgr.y (mgr.14901) 7431 : cluster [DBG] pgmap v7438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:20.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:20 smithi130 bash[51302]: debug 2023-08-28T02:06:20.267+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:20.274692+0000) 2023-08-28T02:06:21.208 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:06:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:06:20] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T02:06:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:06:21.208Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:06:21.210Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:21.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:21 smithi130 bash[51302]: debug 2023-08-28T02:06:21.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:21.274994+0000) 2023-08-28T02:06:22.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:22 smithi006 bash[33918]: cluster 2023-08-28T02:06:21.110491+0000 mgr.y (mgr.14901) 7432 : cluster [DBG] pgmap v7439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:22.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:22 smithi006 bash[33918]: audit 2023-08-28T02:06:21.236112+0000 mon.c (mon.2) 2923 : audit [DBG] from='client.? 172.21.15.6:0/1268039909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:22.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:22 smithi006 bash[27292]: cluster 2023-08-28T02:06:21.110491+0000 mgr.y (mgr.14901) 7432 : cluster [DBG] pgmap v7439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:22.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:22 smithi006 bash[27292]: audit 2023-08-28T02:06:21.236112+0000 mon.c (mon.2) 2923 : audit [DBG] from='client.? 172.21.15.6:0/1268039909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:22.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:22 smithi130 bash[51302]: debug 2023-08-28T02:06:22.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:22.275224+0000) 2023-08-28T02:06:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:22 smithi130 bash[27579]: cluster 2023-08-28T02:06:21.110491+0000 mgr.y (mgr.14901) 7432 : cluster [DBG] pgmap v7439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:22 smithi130 bash[27579]: audit 2023-08-28T02:06:21.236112+0000 mon.c (mon.2) 2923 : audit [DBG] from='client.? 172.21.15.6:0/1268039909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:06:22.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:06:22.776Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:23.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:23 smithi130 bash[51302]: debug 2023-08-28T02:06:23.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:23.275467+0000) 2023-08-28T02:06:23.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:23 smithi006 bash[27292]: cluster 2023-08-28T02:06:23.111151+0000 mgr.y (mgr.14901) 7433 : cluster [DBG] pgmap v7440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:23.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:23 smithi006 bash[33918]: cluster 2023-08-28T02:06:23.111151+0000 mgr.y (mgr.14901) 7433 : cluster [DBG] pgmap v7440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:24.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:23 smithi130 bash[27579]: cluster 2023-08-28T02:06:23.111151+0000 mgr.y (mgr.14901) 7433 : cluster [DBG] pgmap v7440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:24.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:24 smithi130 bash[51302]: debug 2023-08-28T02:06:24.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:24.275782+0000) 2023-08-28T02:06:24.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:24 smithi006 bash[27292]: audit 2023-08-28T02:06:23.653627+0000 mon.a (mon.0) 4795 : audit [DBG] from='client.? 172.21.15.6:0/2975243469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:24.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:24 smithi006 bash[33918]: audit 2023-08-28T02:06:23.653627+0000 mon.a (mon.0) 4795 : audit [DBG] from='client.? 172.21.15.6:0/2975243469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:06:24.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:06:24.652Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:25.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:24 smithi130 bash[27579]: audit 2023-08-28T02:06:23.653627+0000 mon.a (mon.0) 4795 : audit [DBG] from='client.? 172.21.15.6:0/2975243469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:25.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:25 smithi130 bash[51302]: debug 2023-08-28T02:06:25.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:25.276067+0000) 2023-08-28T02:06:25.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:25 smithi006 bash[27292]: cluster 2023-08-28T02:06:25.112295+0000 mgr.y (mgr.14901) 7434 : cluster [DBG] pgmap v7441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:25.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:25 smithi006 bash[33918]: cluster 2023-08-28T02:06:25.112295+0000 mgr.y (mgr.14901) 7434 : cluster [DBG] pgmap v7441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:26.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:25 smithi130 bash[27579]: cluster 2023-08-28T02:06:25.112295+0000 mgr.y (mgr.14901) 7434 : cluster [DBG] pgmap v7441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:26.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:26 smithi130 bash[51302]: debug 2023-08-28T02:06:26.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:26.276397+0000) 2023-08-28T02:06:26.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:26 smithi006 bash[27292]: audit 2023-08-28T02:06:26.070739+0000 mon.a (mon.0) 4796 : audit [DBG] from='client.? 172.21.15.6:0/1999672775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:26.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:26 smithi006 bash[33918]: audit 2023-08-28T02:06:26.070739+0000 mon.a (mon.0) 4796 : audit [DBG] from='client.? 172.21.15.6:0/1999672775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:27.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:26 smithi130 bash[27579]: audit 2023-08-28T02:06:26.070739+0000 mon.a (mon.0) 4796 : audit [DBG] from='client.? 172.21.15.6:0/1999672775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:27.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:27 smithi130 bash[51302]: debug 2023-08-28T02:06:27.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:27.276674+0000) 2023-08-28T02:06:27.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:27 smithi006 bash[27292]: cluster 2023-08-28T02:06:27.112998+0000 mgr.y (mgr.14901) 7435 : cluster [DBG] pgmap v7442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:27.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:27 smithi006 bash[33918]: cluster 2023-08-28T02:06:27.112998+0000 mgr.y (mgr.14901) 7435 : cluster [DBG] pgmap v7442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:28.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:27 smithi130 bash[27579]: cluster 2023-08-28T02:06:27.112998+0000 mgr.y (mgr.14901) 7435 : cluster [DBG] pgmap v7442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:28.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:28 smithi130 bash[51302]: debug 2023-08-28T02:06:28.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:28.276967+0000) 2023-08-28T02:06:28.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:28 smithi006 bash[27292]: audit 2023-08-28T02:06:28.485437+0000 mon.a (mon.0) 4797 : audit [DBG] from='client.? 172.21.15.6:0/1298139209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:28.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:28 smithi006 bash[33918]: audit 2023-08-28T02:06:28.485437+0000 mon.a (mon.0) 4797 : audit [DBG] from='client.? 172.21.15.6:0/1298139209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:29.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:28 smithi130 bash[27579]: audit 2023-08-28T02:06:28.485437+0000 mon.a (mon.0) 4797 : audit [DBG] from='client.? 172.21.15.6:0/1298139209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:29.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:29 smithi130 bash[51302]: debug 2023-08-28T02:06:29.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:29.277247+0000) 2023-08-28T02:06:29.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:29 smithi006 bash[27292]: cluster 2023-08-28T02:06:29.113978+0000 mgr.y (mgr.14901) 7436 : cluster [DBG] pgmap v7443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:29.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:29 smithi006 bash[33918]: cluster 2023-08-28T02:06:29.113978+0000 mgr.y (mgr.14901) 7436 : cluster [DBG] pgmap v7443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:30.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:29 smithi130 bash[27579]: cluster 2023-08-28T02:06:29.113978+0000 mgr.y (mgr.14901) 7436 : cluster [DBG] pgmap v7443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:30.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:30 smithi130 bash[51302]: debug 2023-08-28T02:06:30.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:30.277556+0000) 2023-08-28T02:06:31.208 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:06:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:06:30] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:06:31.209 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:30 smithi006 bash[33918]: audit 2023-08-28T02:06:30.901121+0000 mon.a (mon.0) 4798 : audit [DBG] from='client.? 172.21.15.6:0/2720616855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:31.209 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:30 smithi006 bash[27292]: audit 2023-08-28T02:06:30.901121+0000 mon.a (mon.0) 4798 : audit [DBG] from='client.? 172.21.15.6:0/2720616855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:31.276 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:30 smithi130 bash[27579]: audit 2023-08-28T02:06:30.901121+0000 mon.a (mon.0) 4798 : audit [DBG] from='client.? 172.21.15.6:0/2720616855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:06:31.209Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:06:31.211Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:31.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:31 smithi130 bash[51302]: debug 2023-08-28T02:06:31.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:31.277872+0000) 2023-08-28T02:06:32.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:31 smithi006 bash[33918]: cluster 2023-08-28T02:06:31.114655+0000 mgr.y (mgr.14901) 7437 : cluster [DBG] pgmap v7444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:32.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:31 smithi006 bash[27292]: cluster 2023-08-28T02:06:31.114655+0000 mgr.y (mgr.14901) 7437 : cluster [DBG] pgmap v7444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:32.276 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:31 smithi130 bash[27579]: cluster 2023-08-28T02:06:31.114655+0000 mgr.y (mgr.14901) 7437 : cluster [DBG] pgmap v7444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:32.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:32 smithi130 bash[51302]: debug 2023-08-28T02:06:32.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:32.278099+0000) 2023-08-28T02:06:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:06:32.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:06:32.776Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:33.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:33 smithi130 bash[51302]: debug 2023-08-28T02:06:33.271+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:33.278384+0000) 2023-08-28T02:06:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:33 smithi006 bash[27292]: cluster 2023-08-28T02:06:33.115291+0000 mgr.y (mgr.14901) 7438 : cluster [DBG] pgmap v7445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:33 smithi006 bash[27292]: audit 2023-08-28T02:06:33.309982+0000 mon.a (mon.0) 4799 : audit [DBG] from='client.? 172.21.15.6:0/63065338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:33 smithi006 bash[27292]: audit 2023-08-28T02:06:33.404180+0000 mon.b (mon.1) 825 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:33.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:33 smithi006 bash[33918]: cluster 2023-08-28T02:06:33.115291+0000 mgr.y (mgr.14901) 7438 : cluster [DBG] pgmap v7445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:33.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:33 smithi006 bash[33918]: audit 2023-08-28T02:06:33.309982+0000 mon.a (mon.0) 4799 : audit [DBG] from='client.? 172.21.15.6:0/63065338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:33.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:33 smithi006 bash[33918]: audit 2023-08-28T02:06:33.404180+0000 mon.b (mon.1) 825 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:33 smithi130 bash[27579]: cluster 2023-08-28T02:06:33.115291+0000 mgr.y (mgr.14901) 7438 : cluster [DBG] pgmap v7445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:33 smithi130 bash[27579]: audit 2023-08-28T02:06:33.309982+0000 mon.a (mon.0) 4799 : audit [DBG] from='client.? 172.21.15.6:0/63065338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:33 smithi130 bash[27579]: audit 2023-08-28T02:06:33.404180+0000 mon.b (mon.1) 825 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:34.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:34 smithi130 bash[51302]: debug 2023-08-28T02:06:34.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:34.278712+0000) 2023-08-28T02:06:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:06:34.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:06:34.653Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:35.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:35 smithi130 bash[51302]: debug 2023-08-28T02:06:35.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:35.279020+0000) 2023-08-28T02:06:36.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:36 smithi006 bash[27292]: cluster 2023-08-28T02:06:35.116418+0000 mgr.y (mgr.14901) 7439 : cluster [DBG] pgmap v7446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:36.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:36 smithi006 bash[27292]: audit 2023-08-28T02:06:35.736436+0000 mon.c (mon.2) 2924 : audit [DBG] from='client.? 172.21.15.6:0/2788325788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:36.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:36 smithi006 bash[33918]: cluster 2023-08-28T02:06:35.116418+0000 mgr.y (mgr.14901) 7439 : cluster [DBG] pgmap v7446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:36.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:36 smithi006 bash[33918]: audit 2023-08-28T02:06:35.736436+0000 mon.c (mon.2) 2924 : audit [DBG] from='client.? 172.21.15.6:0/2788325788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:36.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:36 smithi130 bash[27579]: cluster 2023-08-28T02:06:35.116418+0000 mgr.y (mgr.14901) 7439 : cluster [DBG] pgmap v7446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:36 smithi130 bash[27579]: audit 2023-08-28T02:06:35.736436+0000 mon.c (mon.2) 2924 : audit [DBG] from='client.? 172.21.15.6:0/2788325788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:36.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:36 smithi130 bash[51302]: debug 2023-08-28T02:06:36.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:36.279249+0000) 2023-08-28T02:06:37.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:37 smithi130 bash[51302]: debug 2023-08-28T02:06:37.279+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:37.279525+0000) 2023-08-28T02:06:38.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:38 smithi006 bash[27292]: cluster 2023-08-28T02:06:37.117134+0000 mgr.y (mgr.14901) 7440 : cluster [DBG] pgmap v7447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:38.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:38 smithi006 bash[27292]: audit 2023-08-28T02:06:38.155723+0000 mon.a (mon.0) 4800 : audit [DBG] from='client.? 172.21.15.6:0/3264802638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:38.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:38 smithi006 bash[33918]: cluster 2023-08-28T02:06:37.117134+0000 mgr.y (mgr.14901) 7440 : cluster [DBG] pgmap v7447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:38.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:38 smithi006 bash[33918]: audit 2023-08-28T02:06:38.155723+0000 mon.a (mon.0) 4800 : audit [DBG] from='client.? 172.21.15.6:0/3264802638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:38.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:38 smithi130 bash[27579]: cluster 2023-08-28T02:06:37.117134+0000 mgr.y (mgr.14901) 7440 : cluster [DBG] pgmap v7447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:38.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:38 smithi130 bash[27579]: audit 2023-08-28T02:06:38.155723+0000 mon.a (mon.0) 4800 : audit [DBG] from='client.? 172.21.15.6:0/3264802638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:38.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:38 smithi130 bash[51302]: debug 2023-08-28T02:06:38.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:38.279719+0000) 2023-08-28T02:06:39.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:39 smithi130 bash[51302]: debug 2023-08-28T02:06:39.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:39.279913+0000) 2023-08-28T02:06:40.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:40 smithi130 bash[51302]: debug 2023-08-28T02:06:40.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:40.280186+0000) 2023-08-28T02:06:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:40 smithi130 bash[27579]: cluster 2023-08-28T02:06:39.118172+0000 mgr.y (mgr.14901) 7441 : cluster [DBG] pgmap v7448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:40 smithi130 bash[27579]: audit 2023-08-28T02:06:39.261597+0000 mon.a (mon.0) 4801 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:06:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:40 smithi130 bash[27579]: audit 2023-08-28T02:06:39.765753+0000 mon.a (mon.0) 4802 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:06:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:40 smithi130 bash[27579]: audit 2023-08-28T02:06:40.061061+0000 mon.b (mon.1) 826 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:40 smithi130 bash[27579]: audit 2023-08-28T02:06:40.063337+0000 mon.b (mon.1) 827 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:40.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:40 smithi130 bash[27579]: audit 2023-08-28T02:06:40.071832+0000 mon.a (mon.0) 4803 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:06:40.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:40 smithi006 bash[27292]: cluster 2023-08-28T02:06:39.118172+0000 mgr.y (mgr.14901) 7441 : cluster [DBG] pgmap v7448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:40.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:40 smithi006 bash[27292]: audit 2023-08-28T02:06:39.261597+0000 mon.a (mon.0) 4801 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:06:40.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:40 smithi006 bash[27292]: audit 2023-08-28T02:06:39.765753+0000 mon.a (mon.0) 4802 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:06:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:40 smithi006 bash[27292]: audit 2023-08-28T02:06:40.061061+0000 mon.b (mon.1) 826 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:40 smithi006 bash[27292]: audit 2023-08-28T02:06:40.063337+0000 mon.b (mon.1) 827 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:40 smithi006 bash[27292]: audit 2023-08-28T02:06:40.071832+0000 mon.a (mon.0) 4803 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:06:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:40 smithi006 bash[33918]: cluster 2023-08-28T02:06:39.118172+0000 mgr.y (mgr.14901) 7441 : cluster [DBG] pgmap v7448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:40 smithi006 bash[33918]: audit 2023-08-28T02:06:39.261597+0000 mon.a (mon.0) 4801 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:06:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:40 smithi006 bash[33918]: audit 2023-08-28T02:06:39.765753+0000 mon.a (mon.0) 4802 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:06:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:40 smithi006 bash[33918]: audit 2023-08-28T02:06:40.061061+0000 mon.b (mon.1) 826 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:40 smithi006 bash[33918]: audit 2023-08-28T02:06:40.063337+0000 mon.b (mon.1) 827 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:40 smithi006 bash[33918]: audit 2023-08-28T02:06:40.071832+0000 mon.a (mon.0) 4803 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:06:41.209 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:06:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:06:40] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:06:41.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:41 smithi006 bash[27292]: audit 2023-08-28T02:06:40.589993+0000 mon.a (mon.0) 4804 : audit [DBG] from='client.? 172.21.15.6:0/2457220379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:06:41.209Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:06:41.211Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:41.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:41 smithi006 bash[33918]: audit 2023-08-28T02:06:40.589993+0000 mon.a (mon.0) 4804 : audit [DBG] from='client.? 172.21.15.6:0/2457220379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:41.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:41 smithi130 bash[27579]: audit 2023-08-28T02:06:40.589993+0000 mon.a (mon.0) 4804 : audit [DBG] from='client.? 172.21.15.6:0/2457220379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:41.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:41 smithi130 bash[51302]: debug 2023-08-28T02:06:41.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:41.280454+0000) 2023-08-28T02:06:42.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:42 smithi130 bash[27579]: cluster 2023-08-28T02:06:41.118857+0000 mgr.y (mgr.14901) 7442 : cluster [DBG] pgmap v7449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:42.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:42 smithi130 bash[51302]: debug 2023-08-28T02:06:42.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:42.280727+0000) 2023-08-28T02:06:42.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:42 smithi006 bash[27292]: cluster 2023-08-28T02:06:41.118857+0000 mgr.y (mgr.14901) 7442 : cluster [DBG] pgmap v7449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:42.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:42 smithi006 bash[33918]: cluster 2023-08-28T02:06:41.118857+0000 mgr.y (mgr.14901) 7442 : cluster [DBG] pgmap v7449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:06:42.776Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:06:42.778Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:43.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:43 smithi130 bash[51302]: debug 2023-08-28T02:06:43.279+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:43.281065+0000) 2023-08-28T02:06:43.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:43 smithi006 bash[27292]: audit 2023-08-28T02:06:43.002980+0000 mon.a (mon.0) 4805 : audit [DBG] from='client.? 172.21.15.6:0/383412955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:43.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:43 smithi006 bash[27292]: cluster 2023-08-28T02:06:43.119567+0000 mgr.y (mgr.14901) 7443 : cluster [DBG] pgmap v7450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:43.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:43 smithi006 bash[33918]: audit 2023-08-28T02:06:43.002980+0000 mon.a (mon.0) 4805 : audit [DBG] from='client.? 172.21.15.6:0/383412955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:43.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:43 smithi006 bash[33918]: cluster 2023-08-28T02:06:43.119567+0000 mgr.y (mgr.14901) 7443 : cluster [DBG] pgmap v7450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:44.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:43 smithi130 bash[27579]: audit 2023-08-28T02:06:43.002980+0000 mon.a (mon.0) 4805 : audit [DBG] from='client.? 172.21.15.6:0/383412955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:44.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:43 smithi130 bash[27579]: cluster 2023-08-28T02:06:43.119567+0000 mgr.y (mgr.14901) 7443 : cluster [DBG] pgmap v7450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:44.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:44 smithi130 bash[51302]: debug 2023-08-28T02:06:44.275+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:44.281342+0000) 2023-08-28T02:06:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:06:44.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:06:44.654Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:45.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:45 smithi130 bash[51302]: debug 2023-08-28T02:06:45.279+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:45.281711+0000) 2023-08-28T02:06:46.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:46 smithi006 bash[27292]: cluster 2023-08-28T02:06:45.120651+0000 mgr.y (mgr.14901) 7444 : cluster [DBG] pgmap v7451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:46.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:46 smithi006 bash[27292]: audit 2023-08-28T02:06:45.410977+0000 mon.c (mon.2) 2925 : audit [DBG] from='client.? 172.21.15.6:0/3096548821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:46.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:46 smithi006 bash[33918]: cluster 2023-08-28T02:06:45.120651+0000 mgr.y (mgr.14901) 7444 : cluster [DBG] pgmap v7451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:46.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:46 smithi006 bash[33918]: audit 2023-08-28T02:06:45.410977+0000 mon.c (mon.2) 2925 : audit [DBG] from='client.? 172.21.15.6:0/3096548821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:46.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:46 smithi130 bash[27579]: cluster 2023-08-28T02:06:45.120651+0000 mgr.y (mgr.14901) 7444 : cluster [DBG] pgmap v7451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:46 smithi130 bash[27579]: audit 2023-08-28T02:06:45.410977+0000 mon.c (mon.2) 2925 : audit [DBG] from='client.? 172.21.15.6:0/3096548821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:46.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:46 smithi130 bash[51302]: debug 2023-08-28T02:06:46.279+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:46.282023+0000) 2023-08-28T02:06:47.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:47 smithi130 bash[51302]: debug 2023-08-28T02:06:47.279+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:47.282290+0000) 2023-08-28T02:06:48.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:48 smithi006 bash[27292]: cluster 2023-08-28T02:06:47.121363+0000 mgr.y (mgr.14901) 7445 : cluster [DBG] pgmap v7452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:48.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:48 smithi006 bash[27292]: audit 2023-08-28T02:06:47.820582+0000 mon.c (mon.2) 2926 : audit [DBG] from='client.? 172.21.15.6:0/1638881068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:48.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:48 smithi006 bash[33918]: cluster 2023-08-28T02:06:47.121363+0000 mgr.y (mgr.14901) 7445 : cluster [DBG] pgmap v7452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:48.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:48 smithi006 bash[33918]: audit 2023-08-28T02:06:47.820582+0000 mon.c (mon.2) 2926 : audit [DBG] from='client.? 172.21.15.6:0/1638881068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:48.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:48 smithi130 bash[51302]: debug 2023-08-28T02:06:48.279+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:48.282569+0000) 2023-08-28T02:06:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:48 smithi130 bash[27579]: cluster 2023-08-28T02:06:47.121363+0000 mgr.y (mgr.14901) 7445 : cluster [DBG] pgmap v7452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:48 smithi130 bash[27579]: audit 2023-08-28T02:06:47.820582+0000 mon.c (mon.2) 2926 : audit [DBG] from='client.? 172.21.15.6:0/1638881068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:49.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:49 smithi130 bash[51302]: debug 2023-08-28T02:06:49.279+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:49.282932+0000) 2023-08-28T02:06:50.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:50 smithi006 bash[33918]: cluster 2023-08-28T02:06:49.122510+0000 mgr.y (mgr.14901) 7446 : cluster [DBG] pgmap v7453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:50.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:50 smithi006 bash[27292]: cluster 2023-08-28T02:06:49.122510+0000 mgr.y (mgr.14901) 7446 : cluster [DBG] pgmap v7453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:50.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:50 smithi130 bash[27579]: cluster 2023-08-28T02:06:49.122510+0000 mgr.y (mgr.14901) 7446 : cluster [DBG] pgmap v7453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:50.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:50 smithi130 bash[51302]: debug 2023-08-28T02:06:50.279+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:50.283211+0000) 2023-08-28T02:06:51.195 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:06:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:06:50] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:06:52.347 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:51 smithi130 bash[51302]: debug 2023-08-28T02:06:51.282+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:51.283529+0000) 2023-08-28T02:06:52.347 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:51 smithi006 bash[27292]: audit 2023-08-28T02:06:50.232995+0000 mon.a (mon.0) 4806 : audit [DBG] from='client.? 172.21.15.6:0/3482988010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:52.347 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:51 smithi130 bash[27579]: audit 2023-08-28T02:06:50.232995+0000 mon.a (mon.0) 4806 : audit [DBG] from='client.? 172.21.15.6:0/3482988010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:52.348 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:51 smithi006 bash[33918]: audit 2023-08-28T02:06:50.232995+0000 mon.a (mon.0) 4806 : audit [DBG] from='client.? 172.21.15.6:0/3482988010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:52.348 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:06:51.210Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:52.348 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:06:51.212Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:52.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:52 smithi006 bash[33918]: cluster 2023-08-28T02:06:51.123227+0000 mgr.y (mgr.14901) 7447 : cluster [DBG] pgmap v7454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:52.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:52 smithi006 bash[27292]: cluster 2023-08-28T02:06:51.123227+0000 mgr.y (mgr.14901) 7447 : cluster [DBG] pgmap v7454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:52.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:52 smithi130 bash[27579]: cluster 2023-08-28T02:06:51.123227+0000 mgr.y (mgr.14901) 7447 : cluster [DBG] pgmap v7454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:52.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:52 smithi130 bash[51302]: debug 2023-08-28T02:06:52.278+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:52.283801+0000) 2023-08-28T02:06:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:06:52.776Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:06:52.778Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:53.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:53 smithi130 bash[51302]: debug 2023-08-28T02:06:53.282+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:53.284076+0000) 2023-08-28T02:06:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:53 smithi006 bash[33918]: audit 2023-08-28T02:06:52.641700+0000 mon.c (mon.2) 2927 : audit [DBG] from='client.? 172.21.15.6:0/3649105493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:53 smithi006 bash[33918]: cluster 2023-08-28T02:06:53.123894+0000 mgr.y (mgr.14901) 7448 : cluster [DBG] pgmap v7455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:53.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:53 smithi006 bash[27292]: audit 2023-08-28T02:06:52.641700+0000 mon.c (mon.2) 2927 : audit [DBG] from='client.? 172.21.15.6:0/3649105493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:53.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:53 smithi006 bash[27292]: cluster 2023-08-28T02:06:53.123894+0000 mgr.y (mgr.14901) 7448 : cluster [DBG] pgmap v7455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:54.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:53 smithi130 bash[27579]: audit 2023-08-28T02:06:52.641700+0000 mon.c (mon.2) 2927 : audit [DBG] from='client.? 172.21.15.6:0/3649105493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:53 smithi130 bash[27579]: cluster 2023-08-28T02:06:53.123894+0000 mgr.y (mgr.14901) 7448 : cluster [DBG] pgmap v7455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:54.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:54 smithi130 bash[51302]: debug 2023-08-28T02:06:54.282+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:54.284353+0000) 2023-08-28T02:06:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:06:54.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:06:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:06:54.655Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:06:55.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:55 smithi006 bash[27292]: audit 2023-08-28T02:06:55.065535+0000 mon.a (mon.0) 4807 : audit [DBG] from='client.? 172.21.15.6:0/185387989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:55.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:55 smithi006 bash[33918]: audit 2023-08-28T02:06:55.065535+0000 mon.a (mon.0) 4807 : audit [DBG] from='client.? 172.21.15.6:0/185387989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:55.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:55 smithi130 bash[27579]: audit 2023-08-28T02:06:55.065535+0000 mon.a (mon.0) 4807 : audit [DBG] from='client.? 172.21.15.6:0/185387989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:55.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:55 smithi130 bash[51302]: debug 2023-08-28T02:06:55.282+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:55.284655+0000) 2023-08-28T02:06:56.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:56 smithi006 bash[33918]: cluster 2023-08-28T02:06:55.124931+0000 mgr.y (mgr.14901) 7449 : cluster [DBG] pgmap v7456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:56.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:56 smithi006 bash[27292]: cluster 2023-08-28T02:06:55.124931+0000 mgr.y (mgr.14901) 7449 : cluster [DBG] pgmap v7456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:56.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:56 smithi130 bash[27579]: cluster 2023-08-28T02:06:55.124931+0000 mgr.y (mgr.14901) 7449 : cluster [DBG] pgmap v7456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:56.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:56 smithi130 bash[51302]: debug 2023-08-28T02:06:56.282+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:56.284980+0000) 2023-08-28T02:06:57.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:57 smithi130 bash[51302]: debug 2023-08-28T02:06:57.282+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:57.285304+0000) 2023-08-28T02:06:58.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:58 smithi006 bash[27292]: cluster 2023-08-28T02:06:57.125545+0000 mgr.y (mgr.14901) 7450 : cluster [DBG] pgmap v7457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:58.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:06:58 smithi006 bash[27292]: audit 2023-08-28T02:06:57.479901+0000 mon.c (mon.2) 2928 : audit [DBG] from='client.? 172.21.15.6:0/2792190913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:58.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:58 smithi006 bash[33918]: cluster 2023-08-28T02:06:57.125545+0000 mgr.y (mgr.14901) 7450 : cluster [DBG] pgmap v7457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:58.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:06:58 smithi006 bash[33918]: audit 2023-08-28T02:06:57.479901+0000 mon.c (mon.2) 2928 : audit [DBG] from='client.? 172.21.15.6:0/2792190913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:58.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:58 smithi130 bash[27579]: cluster 2023-08-28T02:06:57.125545+0000 mgr.y (mgr.14901) 7450 : cluster [DBG] pgmap v7457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:06:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:06:58 smithi130 bash[27579]: audit 2023-08-28T02:06:57.479901+0000 mon.c (mon.2) 2928 : audit [DBG] from='client.? 172.21.15.6:0/2792190913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:58.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:58 smithi130 bash[51302]: debug 2023-08-28T02:06:58.282+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:58.285601+0000) 2023-08-28T02:06:59.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:06:59 smithi130 bash[51302]: debug 2023-08-28T02:06:59.282+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:59.285881+0000) 2023-08-28T02:07:00.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:00 smithi006 bash[27292]: cluster 2023-08-28T02:06:59.126596+0000 mgr.y (mgr.14901) 7451 : cluster [DBG] pgmap v7458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:00.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:00 smithi006 bash[27292]: audit 2023-08-28T02:06:59.897700+0000 mon.c (mon.2) 2929 : audit [DBG] from='client.? 172.21.15.6:0/677295945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:00.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:00 smithi006 bash[33918]: cluster 2023-08-28T02:06:59.126596+0000 mgr.y (mgr.14901) 7451 : cluster [DBG] pgmap v7458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:00.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:00 smithi006 bash[33918]: audit 2023-08-28T02:06:59.897700+0000 mon.c (mon.2) 2929 : audit [DBG] from='client.? 172.21.15.6:0/677295945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:00.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:00 smithi130 bash[27579]: cluster 2023-08-28T02:06:59.126596+0000 mgr.y (mgr.14901) 7451 : cluster [DBG] pgmap v7458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:00 smithi130 bash[27579]: audit 2023-08-28T02:06:59.897700+0000 mon.c (mon.2) 2929 : audit [DBG] from='client.? 172.21.15.6:0/677295945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:00.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:00 smithi130 bash[51302]: debug 2023-08-28T02:07:00.282+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:00.286172+0000) 2023-08-28T02:07:01.210 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:07:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:07:00] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:07:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:07:01.210Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:01.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:07:01.212Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:01.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:01 smithi130 bash[51302]: debug 2023-08-28T02:07:01.282+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:01.286453+0000) 2023-08-28T02:07:02.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:02 smithi006 bash[27292]: cluster 2023-08-28T02:07:01.127205+0000 mgr.y (mgr.14901) 7452 : cluster [DBG] pgmap v7459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:02.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:02 smithi006 bash[33918]: cluster 2023-08-28T02:07:01.127205+0000 mgr.y (mgr.14901) 7452 : cluster [DBG] pgmap v7459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:02.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:02 smithi130 bash[27579]: cluster 2023-08-28T02:07:01.127205+0000 mgr.y (mgr.14901) 7452 : cluster [DBG] pgmap v7459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:02.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:02 smithi130 bash[51302]: debug 2023-08-28T02:07:02.282+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:02.286697+0000) 2023-08-28T02:07:03.144 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:07:02.777Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:03.145 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:07:02.779Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:03.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:03 smithi006 bash[27292]: audit 2023-08-28T02:07:02.319848+0000 mon.a (mon.0) 4808 : audit [DBG] from='client.? 172.21.15.6:0/2037336503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:03.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:03 smithi006 bash[33918]: audit 2023-08-28T02:07:02.319848+0000 mon.a (mon.0) 4808 : audit [DBG] from='client.? 172.21.15.6:0/2037336503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:03.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:03 smithi130 bash[27579]: audit 2023-08-28T02:07:02.319848+0000 mon.a (mon.0) 4808 : audit [DBG] from='client.? 172.21.15.6:0/2037336503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:03.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:03 smithi130 bash[51302]: debug 2023-08-28T02:07:03.286+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:03.286986+0000) 2023-08-28T02:07:04.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:04 smithi006 bash[27292]: cluster 2023-08-28T02:07:03.127835+0000 mgr.y (mgr.14901) 7453 : cluster [DBG] pgmap v7460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:04.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:04 smithi006 bash[33918]: cluster 2023-08-28T02:07:03.127835+0000 mgr.y (mgr.14901) 7453 : cluster [DBG] pgmap v7460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:04.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:04 smithi130 bash[27579]: cluster 2023-08-28T02:07:03.127835+0000 mgr.y (mgr.14901) 7453 : cluster [DBG] pgmap v7460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:04.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:04 smithi130 bash[51302]: debug 2023-08-28T02:07:04.286+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:04.287211+0000) 2023-08-28T02:07:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:07:04.653Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:07:04.655Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:05.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:05 smithi006 bash[27292]: audit 2023-08-28T02:07:04.728245+0000 mon.c (mon.2) 2930 : audit [DBG] from='client.? 172.21.15.6:0/3686304048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:05.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:05 smithi006 bash[33918]: audit 2023-08-28T02:07:04.728245+0000 mon.c (mon.2) 2930 : audit [DBG] from='client.? 172.21.15.6:0/3686304048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:05.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:05 smithi130 bash[27579]: audit 2023-08-28T02:07:04.728245+0000 mon.c (mon.2) 2930 : audit [DBG] from='client.? 172.21.15.6:0/3686304048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:05.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:05 smithi130 bash[51302]: debug 2023-08-28T02:07:05.286+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:05.287477+0000) 2023-08-28T02:07:06.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:06 smithi006 bash[33918]: cluster 2023-08-28T02:07:05.129008+0000 mgr.y (mgr.14901) 7454 : cluster [DBG] pgmap v7461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:06.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:06 smithi006 bash[27292]: cluster 2023-08-28T02:07:05.129008+0000 mgr.y (mgr.14901) 7454 : cluster [DBG] pgmap v7461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:06.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:06 smithi130 bash[27579]: cluster 2023-08-28T02:07:05.129008+0000 mgr.y (mgr.14901) 7454 : cluster [DBG] pgmap v7461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:06.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:06 smithi130 bash[51302]: debug 2023-08-28T02:07:06.286+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:06.287803+0000) 2023-08-28T02:07:07.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:07 smithi006 bash[27292]: audit 2023-08-28T02:07:07.141806+0000 mon.c (mon.2) 2931 : audit [DBG] from='client.? 172.21.15.6:0/1595456467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:07.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:07 smithi006 bash[33918]: audit 2023-08-28T02:07:07.141806+0000 mon.c (mon.2) 2931 : audit [DBG] from='client.? 172.21.15.6:0/1595456467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:07.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:07 smithi130 bash[27579]: audit 2023-08-28T02:07:07.141806+0000 mon.c (mon.2) 2931 : audit [DBG] from='client.? 172.21.15.6:0/1595456467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:07.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:07 smithi130 bash[51302]: debug 2023-08-28T02:07:07.286+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:07.288107+0000) 2023-08-28T02:07:08.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:08 smithi006 bash[33918]: cluster 2023-08-28T02:07:07.129629+0000 mgr.y (mgr.14901) 7455 : cluster [DBG] pgmap v7462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:08.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:08 smithi006 bash[27292]: cluster 2023-08-28T02:07:07.129629+0000 mgr.y (mgr.14901) 7455 : cluster [DBG] pgmap v7462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:08.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:08 smithi130 bash[27579]: cluster 2023-08-28T02:07:07.129629+0000 mgr.y (mgr.14901) 7455 : cluster [DBG] pgmap v7462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:08.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:08 smithi130 bash[51302]: debug 2023-08-28T02:07:08.286+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:08.288383+0000) 2023-08-28T02:07:09.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:09 smithi130 bash[51302]: debug 2023-08-28T02:07:09.286+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:09.288657+0000) 2023-08-28T02:07:10.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:10 smithi006 bash[33918]: cluster 2023-08-28T02:07:09.130680+0000 mgr.y (mgr.14901) 7456 : cluster [DBG] pgmap v7463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:10.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:10 smithi006 bash[33918]: audit 2023-08-28T02:07:09.550835+0000 mon.a (mon.0) 4809 : audit [DBG] from='client.? 172.21.15.6:0/409644253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:10.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:10 smithi006 bash[27292]: cluster 2023-08-28T02:07:09.130680+0000 mgr.y (mgr.14901) 7456 : cluster [DBG] pgmap v7463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:10.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:10 smithi006 bash[27292]: audit 2023-08-28T02:07:09.550835+0000 mon.a (mon.0) 4809 : audit [DBG] from='client.? 172.21.15.6:0/409644253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:10.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:10 smithi130 bash[51302]: debug 2023-08-28T02:07:10.286+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:10.288956+0000) 2023-08-28T02:07:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:10 smithi130 bash[27579]: cluster 2023-08-28T02:07:09.130680+0000 mgr.y (mgr.14901) 7456 : cluster [DBG] pgmap v7463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:10 smithi130 bash[27579]: audit 2023-08-28T02:07:09.550835+0000 mon.a (mon.0) 4809 : audit [DBG] from='client.? 172.21.15.6:0/409644253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:11.211 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:07:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:07:10] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:07:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:07:11.211Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:11.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:07:11.213Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:11.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:11 smithi130 bash[51302]: debug 2023-08-28T02:07:11.286+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:11.289304+0000) 2023-08-28T02:07:12.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:12 smithi006 bash[33918]: cluster 2023-08-28T02:07:11.131336+0000 mgr.y (mgr.14901) 7457 : cluster [DBG] pgmap v7464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:12.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:12 smithi006 bash[33918]: audit 2023-08-28T02:07:11.964927+0000 mon.c (mon.2) 2932 : audit [DBG] from='client.? 172.21.15.6:0/1153133851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:12.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:12 smithi006 bash[27292]: cluster 2023-08-28T02:07:11.131336+0000 mgr.y (mgr.14901) 7457 : cluster [DBG] pgmap v7464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:12.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:12 smithi006 bash[27292]: audit 2023-08-28T02:07:11.964927+0000 mon.c (mon.2) 2932 : audit [DBG] from='client.? 172.21.15.6:0/1153133851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:12 smithi130 bash[27579]: cluster 2023-08-28T02:07:11.131336+0000 mgr.y (mgr.14901) 7457 : cluster [DBG] pgmap v7464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:12 smithi130 bash[27579]: audit 2023-08-28T02:07:11.964927+0000 mon.c (mon.2) 2932 : audit [DBG] from='client.? 172.21.15.6:0/1153133851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:12.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:12 smithi130 bash[51302]: debug 2023-08-28T02:07:12.286+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:12.289521+0000) 2023-08-28T02:07:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:07:12.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:07:12.780Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:13.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:13 smithi130 bash[51302]: debug 2023-08-28T02:07:13.286+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:13.289805+0000) 2023-08-28T02:07:13.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:13 smithi006 bash[27292]: cluster 2023-08-28T02:07:13.131999+0000 mgr.y (mgr.14901) 7458 : cluster [DBG] pgmap v7465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:13.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:13 smithi006 bash[33918]: cluster 2023-08-28T02:07:13.131999+0000 mgr.y (mgr.14901) 7458 : cluster [DBG] pgmap v7465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:14.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:13 smithi130 bash[27579]: cluster 2023-08-28T02:07:13.131999+0000 mgr.y (mgr.14901) 7458 : cluster [DBG] pgmap v7465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:14.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:14 smithi130 bash[51302]: debug 2023-08-28T02:07:14.286+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:14.290075+0000) 2023-08-28T02:07:14.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:14 smithi006 bash[27292]: audit 2023-08-28T02:07:14.375617+0000 mon.a (mon.0) 4810 : audit [DBG] from='client.? 172.21.15.6:0/3129773620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:14.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:14 smithi006 bash[33918]: audit 2023-08-28T02:07:14.375617+0000 mon.a (mon.0) 4810 : audit [DBG] from='client.? 172.21.15.6:0/3129773620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:07:14.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:07:14.657Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:15.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:14 smithi130 bash[27579]: audit 2023-08-28T02:07:14.375617+0000 mon.a (mon.0) 4810 : audit [DBG] from='client.? 172.21.15.6:0/3129773620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:15.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:15 smithi130 bash[51302]: debug 2023-08-28T02:07:15.286+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:15.290348+0000) 2023-08-28T02:07:15.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:15 smithi006 bash[33918]: cluster 2023-08-28T02:07:15.133177+0000 mgr.y (mgr.14901) 7459 : cluster [DBG] pgmap v7466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:15.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:15 smithi006 bash[27292]: cluster 2023-08-28T02:07:15.133177+0000 mgr.y (mgr.14901) 7459 : cluster [DBG] pgmap v7466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:16.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:15 smithi130 bash[27579]: cluster 2023-08-28T02:07:15.133177+0000 mgr.y (mgr.14901) 7459 : cluster [DBG] pgmap v7466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:16.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:16 smithi130 bash[51302]: debug 2023-08-28T02:07:16.286+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:16.290627+0000) 2023-08-28T02:07:17.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:16 smithi130 bash[27579]: audit 2023-08-28T02:07:16.779843+0000 mon.a (mon.0) 4811 : audit [DBG] from='client.? 172.21.15.6:0/93885309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:17.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:16 smithi006 bash[27292]: audit 2023-08-28T02:07:16.779843+0000 mon.a (mon.0) 4811 : audit [DBG] from='client.? 172.21.15.6:0/93885309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:17.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:16 smithi006 bash[33918]: audit 2023-08-28T02:07:16.779843+0000 mon.a (mon.0) 4811 : audit [DBG] from='client.? 172.21.15.6:0/93885309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:17.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:17 smithi130 bash[51302]: debug 2023-08-28T02:07:17.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:17.290923+0000) 2023-08-28T02:07:18.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:17 smithi130 bash[27579]: cluster 2023-08-28T02:07:17.133972+0000 mgr.y (mgr.14901) 7460 : cluster [DBG] pgmap v7467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:18.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:17 smithi006 bash[33918]: cluster 2023-08-28T02:07:17.133972+0000 mgr.y (mgr.14901) 7460 : cluster [DBG] pgmap v7467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:18.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:17 smithi006 bash[27292]: cluster 2023-08-28T02:07:17.133972+0000 mgr.y (mgr.14901) 7460 : cluster [DBG] pgmap v7467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:18.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:18 smithi130 bash[51302]: debug 2023-08-28T02:07:18.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:18.291189+0000) 2023-08-28T02:07:19.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:19 smithi130 bash[51302]: debug 2023-08-28T02:07:19.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:19.291515+0000) 2023-08-28T02:07:20.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:20 smithi006 bash[33918]: cluster 2023-08-28T02:07:19.134700+0000 mgr.y (mgr.14901) 7461 : cluster [DBG] pgmap v7468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:20.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:20 smithi006 bash[33918]: audit 2023-08-28T02:07:19.206502+0000 mon.a (mon.0) 4812 : audit [DBG] from='client.? 172.21.15.6:0/2837220184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:20.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:20 smithi006 bash[27292]: cluster 2023-08-28T02:07:19.134700+0000 mgr.y (mgr.14901) 7461 : cluster [DBG] pgmap v7468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:20.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:20 smithi006 bash[27292]: audit 2023-08-28T02:07:19.206502+0000 mon.a (mon.0) 4812 : audit [DBG] from='client.? 172.21.15.6:0/2837220184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:20.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:20 smithi130 bash[51302]: debug 2023-08-28T02:07:20.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:20.291826+0000) 2023-08-28T02:07:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:20 smithi130 bash[27579]: cluster 2023-08-28T02:07:19.134700+0000 mgr.y (mgr.14901) 7461 : cluster [DBG] pgmap v7468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:20 smithi130 bash[27579]: audit 2023-08-28T02:07:19.206502+0000 mon.a (mon.0) 4812 : audit [DBG] from='client.? 172.21.15.6:0/2837220184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:21.212 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:07:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:07:20] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T02:07:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:07:21.212Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:21.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:07:21.214Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:21.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:21 smithi130 bash[51302]: debug 2023-08-28T02:07:21.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:21.292096+0000) 2023-08-28T02:07:22.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:22 smithi006 bash[33918]: cluster 2023-08-28T02:07:21.135322+0000 mgr.y (mgr.14901) 7462 : cluster [DBG] pgmap v7469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:22.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:22 smithi006 bash[33918]: audit 2023-08-28T02:07:21.620448+0000 mon.a (mon.0) 4813 : audit [DBG] from='client.? 172.21.15.6:0/2153562365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:22.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:22 smithi006 bash[27292]: cluster 2023-08-28T02:07:21.135322+0000 mgr.y (mgr.14901) 7462 : cluster [DBG] pgmap v7469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:22.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:22 smithi006 bash[27292]: audit 2023-08-28T02:07:21.620448+0000 mon.a (mon.0) 4813 : audit [DBG] from='client.? 172.21.15.6:0/2153562365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:22 smithi130 bash[27579]: cluster 2023-08-28T02:07:21.135322+0000 mgr.y (mgr.14901) 7462 : cluster [DBG] pgmap v7469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:22 smithi130 bash[27579]: audit 2023-08-28T02:07:21.620448+0000 mon.a (mon.0) 4813 : audit [DBG] from='client.? 172.21.15.6:0/2153562365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:22.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:22 smithi130 bash[51302]: debug 2023-08-28T02:07:22.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:22.292443+0000) 2023-08-28T02:07:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:07:22.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:07:22.780Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:23.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:23 smithi130 bash[51302]: debug 2023-08-28T02:07:23.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:23.292717+0000) 2023-08-28T02:07:23.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:23 smithi006 bash[27292]: cluster 2023-08-28T02:07:23.136030+0000 mgr.y (mgr.14901) 7463 : cluster [DBG] pgmap v7470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:23.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:23 smithi006 bash[33918]: cluster 2023-08-28T02:07:23.136030+0000 mgr.y (mgr.14901) 7463 : cluster [DBG] pgmap v7470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:24.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:23 smithi130 bash[27579]: cluster 2023-08-28T02:07:23.136030+0000 mgr.y (mgr.14901) 7463 : cluster [DBG] pgmap v7470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:24.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:24 smithi130 bash[51302]: debug 2023-08-28T02:07:24.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:24.292996+0000) 2023-08-28T02:07:24.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:24 smithi006 bash[27292]: audit 2023-08-28T02:07:24.034695+0000 mon.c (mon.2) 2933 : audit [DBG] from='client.? 172.21.15.6:0/1633873784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:24.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:24 smithi006 bash[33918]: audit 2023-08-28T02:07:24.034695+0000 mon.c (mon.2) 2933 : audit [DBG] from='client.? 172.21.15.6:0/1633873784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:07:24.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:07:24.657Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:25.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:24 smithi130 bash[27579]: audit 2023-08-28T02:07:24.034695+0000 mon.c (mon.2) 2933 : audit [DBG] from='client.? 172.21.15.6:0/1633873784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:25.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:25 smithi130 bash[51302]: debug 2023-08-28T02:07:25.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:25.293295+0000) 2023-08-28T02:07:25.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:25 smithi006 bash[27292]: cluster 2023-08-28T02:07:25.137216+0000 mgr.y (mgr.14901) 7464 : cluster [DBG] pgmap v7471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:25.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:25 smithi006 bash[33918]: cluster 2023-08-28T02:07:25.137216+0000 mgr.y (mgr.14901) 7464 : cluster [DBG] pgmap v7471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:26.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:25 smithi130 bash[27579]: cluster 2023-08-28T02:07:25.137216+0000 mgr.y (mgr.14901) 7464 : cluster [DBG] pgmap v7471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:26.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:26 smithi130 bash[51302]: debug 2023-08-28T02:07:26.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:26.293565+0000) 2023-08-28T02:07:26.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:26 smithi006 bash[33918]: audit 2023-08-28T02:07:26.448141+0000 mon.a (mon.0) 4814 : audit [DBG] from='client.? 172.21.15.6:0/1250206155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:26.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:26 smithi006 bash[27292]: audit 2023-08-28T02:07:26.448141+0000 mon.a (mon.0) 4814 : audit [DBG] from='client.? 172.21.15.6:0/1250206155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:27.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:26 smithi130 bash[27579]: audit 2023-08-28T02:07:26.448141+0000 mon.a (mon.0) 4814 : audit [DBG] from='client.? 172.21.15.6:0/1250206155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:27.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:27 smithi130 bash[51302]: debug 2023-08-28T02:07:27.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:27.293814+0000) 2023-08-28T02:07:27.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:27 smithi006 bash[33918]: cluster 2023-08-28T02:07:27.137860+0000 mgr.y (mgr.14901) 7465 : cluster [DBG] pgmap v7472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:27.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:27 smithi006 bash[27292]: cluster 2023-08-28T02:07:27.137860+0000 mgr.y (mgr.14901) 7465 : cluster [DBG] pgmap v7472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:28.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:27 smithi130 bash[27579]: cluster 2023-08-28T02:07:27.137860+0000 mgr.y (mgr.14901) 7465 : cluster [DBG] pgmap v7472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:28.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:28 smithi130 bash[51302]: debug 2023-08-28T02:07:28.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:28.294122+0000) 2023-08-28T02:07:29.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:28 smithi006 bash[33918]: audit 2023-08-28T02:07:28.856247+0000 mon.a (mon.0) 4815 : audit [DBG] from='client.? 172.21.15.6:0/399161360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:29.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:28 smithi006 bash[27292]: audit 2023-08-28T02:07:28.856247+0000 mon.a (mon.0) 4815 : audit [DBG] from='client.? 172.21.15.6:0/399161360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:29.292 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:28 smithi130 bash[27579]: audit 2023-08-28T02:07:28.856247+0000 mon.a (mon.0) 4815 : audit [DBG] from='client.? 172.21.15.6:0/399161360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:29.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:29 smithi130 bash[51302]: debug 2023-08-28T02:07:29.290+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:29.294394+0000) 2023-08-28T02:07:30.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:29 smithi006 bash[27292]: cluster 2023-08-28T02:07:29.138975+0000 mgr.y (mgr.14901) 7466 : cluster [DBG] pgmap v7473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:30.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:29 smithi006 bash[33918]: cluster 2023-08-28T02:07:29.138975+0000 mgr.y (mgr.14901) 7466 : cluster [DBG] pgmap v7473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:30.292 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:29 smithi130 bash[27579]: cluster 2023-08-28T02:07:29.138975+0000 mgr.y (mgr.14901) 7466 : cluster [DBG] pgmap v7473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:30.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:30 smithi130 bash[51302]: debug 2023-08-28T02:07:30.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:30.294678+0000) 2023-08-28T02:07:31.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:07:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:07:30] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T02:07:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:07:31.213Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:31.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:07:31.215Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:31.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:31 smithi130 bash[51302]: debug 2023-08-28T02:07:31.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:31.295004+0000) 2023-08-28T02:07:32.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:32 smithi006 bash[27292]: cluster 2023-08-28T02:07:31.139589+0000 mgr.y (mgr.14901) 7467 : cluster [DBG] pgmap v7474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:32.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:32 smithi006 bash[27292]: audit 2023-08-28T02:07:31.278017+0000 mon.c (mon.2) 2934 : audit [DBG] from='client.? 172.21.15.6:0/1306984347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:32.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:32 smithi006 bash[33918]: cluster 2023-08-28T02:07:31.139589+0000 mgr.y (mgr.14901) 7467 : cluster [DBG] pgmap v7474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:32.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:32 smithi006 bash[33918]: audit 2023-08-28T02:07:31.278017+0000 mon.c (mon.2) 2934 : audit [DBG] from='client.? 172.21.15.6:0/1306984347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:32 smithi130 bash[27579]: cluster 2023-08-28T02:07:31.139589+0000 mgr.y (mgr.14901) 7467 : cluster [DBG] pgmap v7474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:32 smithi130 bash[27579]: audit 2023-08-28T02:07:31.278017+0000 mon.c (mon.2) 2934 : audit [DBG] from='client.? 172.21.15.6:0/1306984347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:32.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:32 smithi130 bash[51302]: debug 2023-08-28T02:07:32.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:32.295255+0000) 2023-08-28T02:07:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:07:32.780Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:07:32.781Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:33.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:33 smithi130 bash[51302]: debug 2023-08-28T02:07:33.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:33.295527+0000) 2023-08-28T02:07:33.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:33 smithi006 bash[33918]: cluster 2023-08-28T02:07:33.140251+0000 mgr.y (mgr.14901) 7468 : cluster [DBG] pgmap v7475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:33 smithi006 bash[27292]: cluster 2023-08-28T02:07:33.140251+0000 mgr.y (mgr.14901) 7468 : cluster [DBG] pgmap v7475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:34.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:33 smithi130 bash[27579]: cluster 2023-08-28T02:07:33.140251+0000 mgr.y (mgr.14901) 7468 : cluster [DBG] pgmap v7475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:34.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:34 smithi130 bash[51302]: debug 2023-08-28T02:07:34.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:34.295842+0000) 2023-08-28T02:07:34.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:34 smithi006 bash[33918]: audit 2023-08-28T02:07:33.696504+0000 mon.a (mon.0) 4816 : audit [DBG] from='client.? 172.21.15.6:0/2080017795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:34.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:34 smithi006 bash[27292]: audit 2023-08-28T02:07:33.696504+0000 mon.a (mon.0) 4816 : audit [DBG] from='client.? 172.21.15.6:0/2080017795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:07:34.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:07:34.657Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:35.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:34 smithi130 bash[27579]: audit 2023-08-28T02:07:33.696504+0000 mon.a (mon.0) 4816 : audit [DBG] from='client.? 172.21.15.6:0/2080017795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:35.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:35 smithi130 bash[51302]: debug 2023-08-28T02:07:35.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:35.296149+0000) 2023-08-28T02:07:35.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:35 smithi006 bash[27292]: cluster 2023-08-28T02:07:35.141479+0000 mgr.y (mgr.14901) 7469 : cluster [DBG] pgmap v7476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:35.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:35 smithi006 bash[33918]: cluster 2023-08-28T02:07:35.141479+0000 mgr.y (mgr.14901) 7469 : cluster [DBG] pgmap v7476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:36.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:35 smithi130 bash[27579]: cluster 2023-08-28T02:07:35.141479+0000 mgr.y (mgr.14901) 7469 : cluster [DBG] pgmap v7476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:36.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:36 smithi130 bash[51302]: debug 2023-08-28T02:07:36.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:36.296454+0000) 2023-08-28T02:07:36.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:36 smithi006 bash[27292]: audit 2023-08-28T02:07:36.116959+0000 mon.a (mon.0) 4817 : audit [DBG] from='client.? 172.21.15.6:0/548856538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:36.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:36 smithi006 bash[33918]: audit 2023-08-28T02:07:36.116959+0000 mon.a (mon.0) 4817 : audit [DBG] from='client.? 172.21.15.6:0/548856538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:37.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:36 smithi130 bash[27579]: audit 2023-08-28T02:07:36.116959+0000 mon.a (mon.0) 4817 : audit [DBG] from='client.? 172.21.15.6:0/548856538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:37.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:37 smithi130 bash[51302]: debug 2023-08-28T02:07:37.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:37.296726+0000) 2023-08-28T02:07:37.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:37 smithi006 bash[33918]: cluster 2023-08-28T02:07:37.142124+0000 mgr.y (mgr.14901) 7470 : cluster [DBG] pgmap v7477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:37.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:37 smithi006 bash[27292]: cluster 2023-08-28T02:07:37.142124+0000 mgr.y (mgr.14901) 7470 : cluster [DBG] pgmap v7477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:38.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:37 smithi130 bash[27579]: cluster 2023-08-28T02:07:37.142124+0000 mgr.y (mgr.14901) 7470 : cluster [DBG] pgmap v7477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:38.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:38 smithi130 bash[51302]: debug 2023-08-28T02:07:38.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:38.296985+0000) 2023-08-28T02:07:38.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:38 smithi006 bash[33918]: audit 2023-08-28T02:07:38.533469+0000 mon.c (mon.2) 2935 : audit [DBG] from='client.? 172.21.15.6:0/1440726099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:38.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:38 smithi006 bash[27292]: audit 2023-08-28T02:07:38.533469+0000 mon.c (mon.2) 2935 : audit [DBG] from='client.? 172.21.15.6:0/1440726099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:39.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:38 smithi130 bash[27579]: audit 2023-08-28T02:07:38.533469+0000 mon.c (mon.2) 2935 : audit [DBG] from='client.? 172.21.15.6:0/1440726099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:39.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:39 smithi130 bash[51302]: debug 2023-08-28T02:07:39.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:39.297260+0000) 2023-08-28T02:07:39.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:39 smithi006 bash[27292]: cluster 2023-08-28T02:07:39.143215+0000 mgr.y (mgr.14901) 7471 : cluster [DBG] pgmap v7478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:39.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:39 smithi006 bash[33918]: cluster 2023-08-28T02:07:39.143215+0000 mgr.y (mgr.14901) 7471 : cluster [DBG] pgmap v7478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:40.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:39 smithi130 bash[27579]: cluster 2023-08-28T02:07:39.143215+0000 mgr.y (mgr.14901) 7471 : cluster [DBG] pgmap v7478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:40.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:40 smithi130 bash[51302]: debug 2023-08-28T02:07:40.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:40.297473+0000) 2023-08-28T02:07:40.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:40 smithi006 bash[33918]: audit 2023-08-28T02:07:40.084079+0000 mon.b (mon.1) 828 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:40.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:40 smithi006 bash[27292]: audit 2023-08-28T02:07:40.084079+0000 mon.b (mon.1) 828 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:40.965 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:07:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:07:40] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T02:07:41.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:40 smithi130 bash[27579]: audit 2023-08-28T02:07:40.084079+0000 mon.b (mon.1) 828 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:07:41.214Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:41.464 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:07:41.216Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:41.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:41 smithi130 bash[51302]: debug 2023-08-28T02:07:41.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:41.297800+0000) 2023-08-28T02:07:41.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:41 smithi006 bash[27292]: audit 2023-08-28T02:07:40.958376+0000 mon.c (mon.2) 2936 : audit [DBG] from='client.? 172.21.15.6:0/1356613375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:41.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:41 smithi006 bash[27292]: cluster 2023-08-28T02:07:41.143916+0000 mgr.y (mgr.14901) 7472 : cluster [DBG] pgmap v7479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:41.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:41 smithi006 bash[33918]: audit 2023-08-28T02:07:40.958376+0000 mon.c (mon.2) 2936 : audit [DBG] from='client.? 172.21.15.6:0/1356613375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:41.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:41 smithi006 bash[33918]: cluster 2023-08-28T02:07:41.143916+0000 mgr.y (mgr.14901) 7472 : cluster [DBG] pgmap v7479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:42.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:41 smithi130 bash[27579]: audit 2023-08-28T02:07:40.958376+0000 mon.c (mon.2) 2936 : audit [DBG] from='client.? 172.21.15.6:0/1356613375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:42.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:41 smithi130 bash[27579]: cluster 2023-08-28T02:07:41.143916+0000 mgr.y (mgr.14901) 7472 : cluster [DBG] pgmap v7479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:42.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:42 smithi130 bash[51302]: debug 2023-08-28T02:07:42.294+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:42.298055+0000) 2023-08-28T02:07:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:07:42.780Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:07:42.782Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:43.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:43 smithi130 bash[51302]: debug 2023-08-28T02:07:43.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:43.298358+0000) 2023-08-28T02:07:43.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:43 smithi006 bash[33918]: cluster 2023-08-28T02:07:43.144669+0000 mgr.y (mgr.14901) 7473 : cluster [DBG] pgmap v7480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:43.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:43 smithi006 bash[33918]: audit 2023-08-28T02:07:43.373745+0000 mon.c (mon.2) 2937 : audit [DBG] from='client.? 172.21.15.6:0/4037078901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:43.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:43 smithi006 bash[27292]: cluster 2023-08-28T02:07:43.144669+0000 mgr.y (mgr.14901) 7473 : cluster [DBG] pgmap v7480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:43.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:43 smithi006 bash[27292]: audit 2023-08-28T02:07:43.373745+0000 mon.c (mon.2) 2937 : audit [DBG] from='client.? 172.21.15.6:0/4037078901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:44.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:43 smithi130 bash[27579]: cluster 2023-08-28T02:07:43.144669+0000 mgr.y (mgr.14901) 7473 : cluster [DBG] pgmap v7480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:44.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:43 smithi130 bash[27579]: audit 2023-08-28T02:07:43.373745+0000 mon.c (mon.2) 2937 : audit [DBG] from='client.? 172.21.15.6:0/4037078901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:44.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:44 smithi130 bash[51302]: debug 2023-08-28T02:07:44.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:44.298644+0000) 2023-08-28T02:07:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:07:44.656Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:07:44.658Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:45.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:45 smithi130 bash[51302]: debug 2023-08-28T02:07:45.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:45.298883+0000) 2023-08-28T02:07:46.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:46 smithi006 bash[33918]: cluster 2023-08-28T02:07:45.145850+0000 mgr.y (mgr.14901) 7474 : cluster [DBG] pgmap v7481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:46.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:46 smithi006 bash[33918]: audit 2023-08-28T02:07:45.817833+0000 mon.c (mon.2) 2938 : audit [DBG] from='client.? 172.21.15.6:0/13983967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:46.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:46 smithi006 bash[33918]: audit 2023-08-28T02:07:45.946877+0000 mon.a (mon.0) 4818 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:07:46.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:46 smithi006 bash[27292]: cluster 2023-08-28T02:07:45.145850+0000 mgr.y (mgr.14901) 7474 : cluster [DBG] pgmap v7481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:46.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:46 smithi006 bash[27292]: audit 2023-08-28T02:07:45.817833+0000 mon.c (mon.2) 2938 : audit [DBG] from='client.? 172.21.15.6:0/13983967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:46.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:46 smithi006 bash[27292]: audit 2023-08-28T02:07:45.946877+0000 mon.a (mon.0) 4818 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:07:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:46 smithi130 bash[27579]: cluster 2023-08-28T02:07:45.145850+0000 mgr.y (mgr.14901) 7474 : cluster [DBG] pgmap v7481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:46 smithi130 bash[27579]: audit 2023-08-28T02:07:45.817833+0000 mon.c (mon.2) 2938 : audit [DBG] from='client.? 172.21.15.6:0/13983967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:46 smithi130 bash[27579]: audit 2023-08-28T02:07:45.946877+0000 mon.a (mon.0) 4818 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:07:46.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:46 smithi130 bash[51302]: debug 2023-08-28T02:07:46.298+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:46.299162+0000) 2023-08-28T02:07:47.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:47 smithi130 bash[51302]: debug 2023-08-28T02:07:47.297+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:47.299454+0000) 2023-08-28T02:07:47.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:47 smithi130 bash[27579]: audit 2023-08-28T02:07:46.431868+0000 mon.a (mon.0) 4819 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:07:47.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:47 smithi130 bash[27579]: audit 2023-08-28T02:07:46.727988+0000 mon.b (mon.1) 829 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:47.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:47 smithi130 bash[27579]: audit 2023-08-28T02:07:46.729986+0000 mon.b (mon.1) 830 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:07:47.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:47 smithi130 bash[27579]: audit 2023-08-28T02:07:46.736941+0000 mon.a (mon.0) 4820 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:07:47.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:47 smithi006 bash[33918]: audit 2023-08-28T02:07:46.431868+0000 mon.a (mon.0) 4819 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:07:47.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:47 smithi006 bash[33918]: audit 2023-08-28T02:07:46.727988+0000 mon.b (mon.1) 829 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:47.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:47 smithi006 bash[33918]: audit 2023-08-28T02:07:46.729986+0000 mon.b (mon.1) 830 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:07:47.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:47 smithi006 bash[33918]: audit 2023-08-28T02:07:46.736941+0000 mon.a (mon.0) 4820 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:07:47.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:47 smithi006 bash[27292]: audit 2023-08-28T02:07:46.431868+0000 mon.a (mon.0) 4819 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:07:47.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:47 smithi006 bash[27292]: audit 2023-08-28T02:07:46.727988+0000 mon.b (mon.1) 829 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:47.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:47 smithi006 bash[27292]: audit 2023-08-28T02:07:46.729986+0000 mon.b (mon.1) 830 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:07:47.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:47 smithi006 bash[27292]: audit 2023-08-28T02:07:46.736941+0000 mon.a (mon.0) 4820 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:07:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:48 smithi130 bash[27579]: cluster 2023-08-28T02:07:47.146495+0000 mgr.y (mgr.14901) 7475 : cluster [DBG] pgmap v7482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:48 smithi130 bash[27579]: audit 2023-08-28T02:07:48.227363+0000 mon.a (mon.0) 4821 : audit [DBG] from='client.? 172.21.15.6:0/657186355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:48.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:48 smithi130 bash[51302]: debug 2023-08-28T02:07:48.297+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:48.299722+0000) 2023-08-28T02:07:48.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:48 smithi006 bash[33918]: cluster 2023-08-28T02:07:47.146495+0000 mgr.y (mgr.14901) 7475 : cluster [DBG] pgmap v7482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:48.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:48 smithi006 bash[33918]: audit 2023-08-28T02:07:48.227363+0000 mon.a (mon.0) 4821 : audit [DBG] from='client.? 172.21.15.6:0/657186355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:48.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:48 smithi006 bash[27292]: cluster 2023-08-28T02:07:47.146495+0000 mgr.y (mgr.14901) 7475 : cluster [DBG] pgmap v7482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:48 smithi006 bash[27292]: audit 2023-08-28T02:07:48.227363+0000 mon.a (mon.0) 4821 : audit [DBG] from='client.? 172.21.15.6:0/657186355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:49.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:49 smithi130 bash[51302]: debug 2023-08-28T02:07:49.297+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:49.300021+0000) 2023-08-28T02:07:50.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:50 smithi130 bash[27579]: cluster 2023-08-28T02:07:49.147556+0000 mgr.y (mgr.14901) 7476 : cluster [DBG] pgmap v7483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:50.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:50 smithi130 bash[51302]: debug 2023-08-28T02:07:50.297+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:50.300342+0000) 2023-08-28T02:07:50.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:50 smithi006 bash[33918]: cluster 2023-08-28T02:07:49.147556+0000 mgr.y (mgr.14901) 7476 : cluster [DBG] pgmap v7483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:50.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:50 smithi006 bash[27292]: cluster 2023-08-28T02:07:49.147556+0000 mgr.y (mgr.14901) 7476 : cluster [DBG] pgmap v7483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:07:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:07:50] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:07:51.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:51 smithi130 bash[27579]: audit 2023-08-28T02:07:50.654793+0000 mon.a (mon.0) 4822 : audit [DBG] from='client.? 172.21.15.6:0/864646158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:51.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:51 smithi130 bash[51302]: debug 2023-08-28T02:07:51.297+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:51.300639+0000) 2023-08-28T02:07:51.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:51 smithi006 bash[27292]: audit 2023-08-28T02:07:50.654793+0000 mon.a (mon.0) 4822 : audit [DBG] from='client.? 172.21.15.6:0/864646158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:51.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:51 smithi006 bash[33918]: audit 2023-08-28T02:07:50.654793+0000 mon.a (mon.0) 4822 : audit [DBG] from='client.? 172.21.15.6:0/864646158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:07:51.215Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:07:51.217Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:52 smithi130 bash[27579]: cluster 2023-08-28T02:07:51.148102+0000 mgr.y (mgr.14901) 7477 : cluster [DBG] pgmap v7484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:52.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:52 smithi130 bash[51302]: debug 2023-08-28T02:07:52.297+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:52.300931+0000) 2023-08-28T02:07:52.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:52 smithi006 bash[27292]: cluster 2023-08-28T02:07:51.148102+0000 mgr.y (mgr.14901) 7477 : cluster [DBG] pgmap v7484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:52.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:52 smithi006 bash[33918]: cluster 2023-08-28T02:07:51.148102+0000 mgr.y (mgr.14901) 7477 : cluster [DBG] pgmap v7484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:07:52.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:07:52.783Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:53.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:53 smithi130 bash[51302]: debug 2023-08-28T02:07:53.297+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:53.301203+0000) 2023-08-28T02:07:53.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:53 smithi006 bash[27292]: audit 2023-08-28T02:07:53.073146+0000 mon.a (mon.0) 4823 : audit [DBG] from='client.? 172.21.15.6:0/1592205136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:53.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:53 smithi006 bash[27292]: cluster 2023-08-28T02:07:53.148760+0000 mgr.y (mgr.14901) 7478 : cluster [DBG] pgmap v7485: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:53.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:53 smithi006 bash[33918]: audit 2023-08-28T02:07:53.073146+0000 mon.a (mon.0) 4823 : audit [DBG] from='client.? 172.21.15.6:0/1592205136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:53 smithi006 bash[33918]: cluster 2023-08-28T02:07:53.148760+0000 mgr.y (mgr.14901) 7478 : cluster [DBG] pgmap v7485: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:54.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:53 smithi130 bash[27579]: audit 2023-08-28T02:07:53.073146+0000 mon.a (mon.0) 4823 : audit [DBG] from='client.? 172.21.15.6:0/1592205136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:53 smithi130 bash[27579]: cluster 2023-08-28T02:07:53.148760+0000 mgr.y (mgr.14901) 7478 : cluster [DBG] pgmap v7485: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:54.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:54 smithi130 bash[51302]: debug 2023-08-28T02:07:54.297+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:54.301495+0000) 2023-08-28T02:07:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:07:54.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:07:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:07:54.658Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:07:55.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:55 smithi130 bash[51302]: debug 2023-08-28T02:07:55.297+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:55.301822+0000) 2023-08-28T02:07:56.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:56 smithi006 bash[33918]: cluster 2023-08-28T02:07:55.149910+0000 mgr.y (mgr.14901) 7479 : cluster [DBG] pgmap v7486: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:56.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:56 smithi006 bash[33918]: audit 2023-08-28T02:07:55.488530+0000 mon.a (mon.0) 4824 : audit [DBG] from='client.? 172.21.15.6:0/1253014716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:56.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:56 smithi006 bash[27292]: cluster 2023-08-28T02:07:55.149910+0000 mgr.y (mgr.14901) 7479 : cluster [DBG] pgmap v7486: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:56.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:56 smithi006 bash[27292]: audit 2023-08-28T02:07:55.488530+0000 mon.a (mon.0) 4824 : audit [DBG] from='client.? 172.21.15.6:0/1253014716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:56 smithi130 bash[27579]: cluster 2023-08-28T02:07:55.149910+0000 mgr.y (mgr.14901) 7479 : cluster [DBG] pgmap v7486: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:56 smithi130 bash[27579]: audit 2023-08-28T02:07:55.488530+0000 mon.a (mon.0) 4824 : audit [DBG] from='client.? 172.21.15.6:0/1253014716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:56.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:56 smithi130 bash[51302]: debug 2023-08-28T02:07:56.297+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:56.302100+0000) 2023-08-28T02:07:57.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:57 smithi130 bash[51302]: debug 2023-08-28T02:07:57.301+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:57.302392+0000) 2023-08-28T02:07:58.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:58 smithi006 bash[33918]: cluster 2023-08-28T02:07:57.150576+0000 mgr.y (mgr.14901) 7480 : cluster [DBG] pgmap v7487: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:58.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:07:58 smithi006 bash[33918]: audit 2023-08-28T02:07:57.903681+0000 mon.c (mon.2) 2939 : audit [DBG] from='client.? 172.21.15.6:0/904597085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:58.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:58 smithi006 bash[27292]: cluster 2023-08-28T02:07:57.150576+0000 mgr.y (mgr.14901) 7480 : cluster [DBG] pgmap v7487: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:58.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:07:58 smithi006 bash[27292]: audit 2023-08-28T02:07:57.903681+0000 mon.c (mon.2) 2939 : audit [DBG] from='client.? 172.21.15.6:0/904597085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:58 smithi130 bash[27579]: cluster 2023-08-28T02:07:57.150576+0000 mgr.y (mgr.14901) 7480 : cluster [DBG] pgmap v7487: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:07:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:07:58 smithi130 bash[27579]: audit 2023-08-28T02:07:57.903681+0000 mon.c (mon.2) 2939 : audit [DBG] from='client.? 172.21.15.6:0/904597085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:58.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:58 smithi130 bash[51302]: debug 2023-08-28T02:07:58.297+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:58.302695+0000) 2023-08-28T02:07:59.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:07:59 smithi130 bash[51302]: debug 2023-08-28T02:07:59.301+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:59.302975+0000) 2023-08-28T02:08:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:00 smithi130 bash[27579]: cluster 2023-08-28T02:07:59.151660+0000 mgr.y (mgr.14901) 7481 : cluster [DBG] pgmap v7488: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:00.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:00 smithi130 bash[51302]: debug 2023-08-28T02:08:00.301+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:00.303235+0000) 2023-08-28T02:08:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:00 smithi006 bash[33918]: cluster 2023-08-28T02:07:59.151660+0000 mgr.y (mgr.14901) 7481 : cluster [DBG] pgmap v7488: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:00 smithi006 bash[27292]: cluster 2023-08-28T02:07:59.151660+0000 mgr.y (mgr.14901) 7481 : cluster [DBG] pgmap v7488: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:08:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:08:00] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:08:01.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:01 smithi130 bash[27579]: audit 2023-08-28T02:08:00.317834+0000 mon.c (mon.2) 2940 : audit [DBG] from='client.? 172.21.15.6:0/3887927678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:01.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:01 smithi130 bash[51302]: debug 2023-08-28T02:08:01.301+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:01.303543+0000) 2023-08-28T02:08:01.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:01 smithi006 bash[27292]: audit 2023-08-28T02:08:00.317834+0000 mon.c (mon.2) 2940 : audit [DBG] from='client.? 172.21.15.6:0/3887927678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:01.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:01 smithi006 bash[33918]: audit 2023-08-28T02:08:00.317834+0000 mon.c (mon.2) 2940 : audit [DBG] from='client.? 172.21.15.6:0/3887927678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:08:01.215Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:08:01.217Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:02 smithi130 bash[27579]: cluster 2023-08-28T02:08:01.152310+0000 mgr.y (mgr.14901) 7482 : cluster [DBG] pgmap v7489: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:02.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:02 smithi130 bash[51302]: debug 2023-08-28T02:08:02.301+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:02.303820+0000) 2023-08-28T02:08:02.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:02 smithi006 bash[27292]: cluster 2023-08-28T02:08:01.152310+0000 mgr.y (mgr.14901) 7482 : cluster [DBG] pgmap v7489: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:02.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:02 smithi006 bash[33918]: cluster 2023-08-28T02:08:01.152310+0000 mgr.y (mgr.14901) 7482 : cluster [DBG] pgmap v7489: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:08:02.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:08:02.784Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:03.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:03 smithi130 bash[51302]: debug 2023-08-28T02:08:03.301+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:03.304105+0000) 2023-08-28T02:08:03.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:03 smithi006 bash[33918]: audit 2023-08-28T02:08:02.747492+0000 mon.c (mon.2) 2941 : audit [DBG] from='client.? 172.21.15.6:0/922578485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:03.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:03 smithi006 bash[33918]: cluster 2023-08-28T02:08:03.153004+0000 mgr.y (mgr.14901) 7483 : cluster [DBG] pgmap v7490: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:03.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:03 smithi006 bash[27292]: audit 2023-08-28T02:08:02.747492+0000 mon.c (mon.2) 2941 : audit [DBG] from='client.? 172.21.15.6:0/922578485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:03.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:03 smithi006 bash[27292]: cluster 2023-08-28T02:08:03.153004+0000 mgr.y (mgr.14901) 7483 : cluster [DBG] pgmap v7490: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:03 smithi130 bash[27579]: audit 2023-08-28T02:08:02.747492+0000 mon.c (mon.2) 2941 : audit [DBG] from='client.? 172.21.15.6:0/922578485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:03 smithi130 bash[27579]: cluster 2023-08-28T02:08:03.153004+0000 mgr.y (mgr.14901) 7483 : cluster [DBG] pgmap v7490: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:04.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:04 smithi130 bash[51302]: debug 2023-08-28T02:08:04.301+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:04.304418+0000) 2023-08-28T02:08:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:08:04.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:08:04.660Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:05.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:05 smithi130 bash[27579]: audit 2023-08-28T02:08:05.167363+0000 mon.c (mon.2) 2942 : audit [DBG] from='client.? 172.21.15.6:0/2788873705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:05.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:05 smithi130 bash[51302]: debug 2023-08-28T02:08:05.301+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:05.304639+0000) 2023-08-28T02:08:05.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:05 smithi006 bash[33918]: audit 2023-08-28T02:08:05.167363+0000 mon.c (mon.2) 2942 : audit [DBG] from='client.? 172.21.15.6:0/2788873705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:05.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:05 smithi006 bash[27292]: audit 2023-08-28T02:08:05.167363+0000 mon.c (mon.2) 2942 : audit [DBG] from='client.? 172.21.15.6:0/2788873705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:06 smithi130 bash[27579]: cluster 2023-08-28T02:08:05.154032+0000 mgr.y (mgr.14901) 7484 : cluster [DBG] pgmap v7491: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:06.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:06 smithi130 bash[51302]: debug 2023-08-28T02:08:06.301+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:06.304930+0000) 2023-08-28T02:08:06.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:06 smithi006 bash[33918]: cluster 2023-08-28T02:08:05.154032+0000 mgr.y (mgr.14901) 7484 : cluster [DBG] pgmap v7491: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:06.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:06 smithi006 bash[27292]: cluster 2023-08-28T02:08:05.154032+0000 mgr.y (mgr.14901) 7484 : cluster [DBG] pgmap v7491: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:07.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:07 smithi130 bash[51302]: debug 2023-08-28T02:08:07.301+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:07.305220+0000) 2023-08-28T02:08:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:08 smithi130 bash[27579]: cluster 2023-08-28T02:08:07.154699+0000 mgr.y (mgr.14901) 7485 : cluster [DBG] pgmap v7492: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:08 smithi130 bash[27579]: audit 2023-08-28T02:08:07.578335+0000 mon.a (mon.0) 4825 : audit [DBG] from='client.? 172.21.15.6:0/599201522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:08.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:08 smithi130 bash[51302]: debug 2023-08-28T02:08:08.301+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:08.305475+0000) 2023-08-28T02:08:08.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:08 smithi006 bash[27292]: cluster 2023-08-28T02:08:07.154699+0000 mgr.y (mgr.14901) 7485 : cluster [DBG] pgmap v7492: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:08.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:08 smithi006 bash[27292]: audit 2023-08-28T02:08:07.578335+0000 mon.a (mon.0) 4825 : audit [DBG] from='client.? 172.21.15.6:0/599201522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:08 smithi006 bash[33918]: cluster 2023-08-28T02:08:07.154699+0000 mgr.y (mgr.14901) 7485 : cluster [DBG] pgmap v7492: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:08.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:08 smithi006 bash[33918]: audit 2023-08-28T02:08:07.578335+0000 mon.a (mon.0) 4825 : audit [DBG] from='client.? 172.21.15.6:0/599201522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:09.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:09 smithi130 bash[51302]: debug 2023-08-28T02:08:09.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:09.305784+0000) 2023-08-28T02:08:10.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:10 smithi130 bash[51302]: debug 2023-08-28T02:08:10.301+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:10.306116+0000) 2023-08-28T02:08:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:10 smithi130 bash[27579]: cluster 2023-08-28T02:08:09.155804+0000 mgr.y (mgr.14901) 7486 : cluster [DBG] pgmap v7493: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:10 smithi130 bash[27579]: audit 2023-08-28T02:08:09.988307+0000 mon.a (mon.0) 4826 : audit [DBG] from='client.? 172.21.15.6:0/4280852181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:10 smithi006 bash[33918]: cluster 2023-08-28T02:08:09.155804+0000 mgr.y (mgr.14901) 7486 : cluster [DBG] pgmap v7493: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:10 smithi006 bash[33918]: audit 2023-08-28T02:08:09.988307+0000 mon.a (mon.0) 4826 : audit [DBG] from='client.? 172.21.15.6:0/4280852181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:10 smithi006 bash[27292]: cluster 2023-08-28T02:08:09.155804+0000 mgr.y (mgr.14901) 7486 : cluster [DBG] pgmap v7493: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:10 smithi006 bash[27292]: audit 2023-08-28T02:08:09.988307+0000 mon.a (mon.0) 4826 : audit [DBG] from='client.? 172.21.15.6:0/4280852181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:11.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:08:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:08:10] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:08:11.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:11 smithi130 bash[51302]: debug 2023-08-28T02:08:11.301+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:11.306422+0000) 2023-08-28T02:08:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:08:11.216Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:08:11.218Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:12 smithi130 bash[27579]: cluster 2023-08-28T02:08:11.156473+0000 mgr.y (mgr.14901) 7487 : cluster [DBG] pgmap v7494: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:12.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:12 smithi130 bash[51302]: debug 2023-08-28T02:08:12.301+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:12.306781+0000) 2023-08-28T02:08:12.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:12 smithi006 bash[27292]: cluster 2023-08-28T02:08:11.156473+0000 mgr.y (mgr.14901) 7487 : cluster [DBG] pgmap v7494: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:12.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:12 smithi006 bash[33918]: cluster 2023-08-28T02:08:11.156473+0000 mgr.y (mgr.14901) 7487 : cluster [DBG] pgmap v7494: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:08:12.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:08:12.786Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:13.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:13 smithi130 bash[51302]: debug 2023-08-28T02:08:13.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:13.307061+0000) 2023-08-28T02:08:13.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:13 smithi130 bash[27579]: audit 2023-08-28T02:08:12.397003+0000 mon.c (mon.2) 2943 : audit [DBG] from='client.? 172.21.15.6:0/3054143041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:13.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:13 smithi006 bash[27292]: audit 2023-08-28T02:08:12.397003+0000 mon.c (mon.2) 2943 : audit [DBG] from='client.? 172.21.15.6:0/3054143041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:13.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:13 smithi006 bash[33918]: audit 2023-08-28T02:08:12.397003+0000 mon.c (mon.2) 2943 : audit [DBG] from='client.? 172.21.15.6:0/3054143041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:14.614 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:14 smithi130 bash[27579]: cluster 2023-08-28T02:08:13.157178+0000 mgr.y (mgr.14901) 7488 : cluster [DBG] pgmap v7495: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:14.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:14 smithi130 bash[51302]: debug 2023-08-28T02:08:14.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:14.307337+0000) 2023-08-28T02:08:14.659 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:14 smithi006 bash[27292]: cluster 2023-08-28T02:08:13.157178+0000 mgr.y (mgr.14901) 7488 : cluster [DBG] pgmap v7495: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:14.659 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:14 smithi006 bash[33918]: cluster 2023-08-28T02:08:13.157178+0000 mgr.y (mgr.14901) 7488 : cluster [DBG] pgmap v7495: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:08:14.659Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:08:14.660Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:15.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:15 smithi130 bash[27579]: audit 2023-08-28T02:08:14.815764+0000 mon.c (mon.2) 2944 : audit [DBG] from='client.? 172.21.15.6:0/994963272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:15.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:15 smithi130 bash[51302]: debug 2023-08-28T02:08:15.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:15.307618+0000) 2023-08-28T02:08:15.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:15 smithi006 bash[33918]: audit 2023-08-28T02:08:14.815764+0000 mon.c (mon.2) 2944 : audit [DBG] from='client.? 172.21.15.6:0/994963272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:15.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:15 smithi006 bash[27292]: audit 2023-08-28T02:08:14.815764+0000 mon.c (mon.2) 2944 : audit [DBG] from='client.? 172.21.15.6:0/994963272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:16 smithi130 bash[27579]: cluster 2023-08-28T02:08:15.158231+0000 mgr.y (mgr.14901) 7489 : cluster [DBG] pgmap v7496: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:16.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:16 smithi130 bash[51302]: debug 2023-08-28T02:08:16.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:16.307928+0000) 2023-08-28T02:08:16.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:16 smithi006 bash[27292]: cluster 2023-08-28T02:08:15.158231+0000 mgr.y (mgr.14901) 7489 : cluster [DBG] pgmap v7496: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:16.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:16 smithi006 bash[33918]: cluster 2023-08-28T02:08:15.158231+0000 mgr.y (mgr.14901) 7489 : cluster [DBG] pgmap v7496: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:17.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:17 smithi130 bash[27579]: audit 2023-08-28T02:08:17.230446+0000 mon.c (mon.2) 2945 : audit [DBG] from='client.? 172.21.15.6:0/262995131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:17.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:17 smithi130 bash[51302]: debug 2023-08-28T02:08:17.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:17.308238+0000) 2023-08-28T02:08:17.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:17 smithi006 bash[27292]: audit 2023-08-28T02:08:17.230446+0000 mon.c (mon.2) 2945 : audit [DBG] from='client.? 172.21.15.6:0/262995131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:17.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:17 smithi006 bash[33918]: audit 2023-08-28T02:08:17.230446+0000 mon.c (mon.2) 2945 : audit [DBG] from='client.? 172.21.15.6:0/262995131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:18 smithi130 bash[27579]: cluster 2023-08-28T02:08:17.158848+0000 mgr.y (mgr.14901) 7490 : cluster [DBG] pgmap v7497: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:18.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:18 smithi130 bash[51302]: debug 2023-08-28T02:08:18.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:18.308529+0000) 2023-08-28T02:08:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:18 smithi006 bash[27292]: cluster 2023-08-28T02:08:17.158848+0000 mgr.y (mgr.14901) 7490 : cluster [DBG] pgmap v7497: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:18.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:18 smithi006 bash[33918]: cluster 2023-08-28T02:08:17.158848+0000 mgr.y (mgr.14901) 7490 : cluster [DBG] pgmap v7497: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:19.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:19 smithi130 bash[51302]: debug 2023-08-28T02:08:19.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:19.308886+0000) 2023-08-28T02:08:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:20 smithi130 bash[27579]: cluster 2023-08-28T02:08:19.160014+0000 mgr.y (mgr.14901) 7491 : cluster [DBG] pgmap v7498: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:20.795 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:20 smithi130 bash[27579]: audit 2023-08-28T02:08:19.641135+0000 mon.a (mon.0) 4827 : audit [DBG] from='client.? 172.21.15.6:0/1141506595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:20.795 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:20 smithi130 bash[51302]: debug 2023-08-28T02:08:20.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:20.309179+0000) 2023-08-28T02:08:20.796 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:20 smithi006 bash[33918]: cluster 2023-08-28T02:08:19.160014+0000 mgr.y (mgr.14901) 7491 : cluster [DBG] pgmap v7498: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:20.796 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:20 smithi006 bash[33918]: audit 2023-08-28T02:08:19.641135+0000 mon.a (mon.0) 4827 : audit [DBG] from='client.? 172.21.15.6:0/1141506595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:20.796 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:20 smithi006 bash[27292]: cluster 2023-08-28T02:08:19.160014+0000 mgr.y (mgr.14901) 7491 : cluster [DBG] pgmap v7498: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:20.796 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:20 smithi006 bash[27292]: audit 2023-08-28T02:08:19.641135+0000 mon.a (mon.0) 4827 : audit [DBG] from='client.? 172.21.15.6:0/1141506595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:08:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:08:20] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T02:08:21.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:21 smithi130 bash[51302]: debug 2023-08-28T02:08:21.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:21.309407+0000) 2023-08-28T02:08:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:08:21.217Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:08:21.219Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:22.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:22 smithi130 bash[51302]: debug 2023-08-28T02:08:22.305+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:22.309685+0000) 2023-08-28T02:08:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:22 smithi130 bash[27579]: cluster 2023-08-28T02:08:21.160693+0000 mgr.y (mgr.14901) 7492 : cluster [DBG] pgmap v7499: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:22 smithi130 bash[27579]: audit 2023-08-28T02:08:22.063185+0000 mon.a (mon.0) 4828 : audit [DBG] from='client.? 172.21.15.6:0/2769759408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:22.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:22 smithi006 bash[27292]: cluster 2023-08-28T02:08:21.160693+0000 mgr.y (mgr.14901) 7492 : cluster [DBG] pgmap v7499: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:22.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:22 smithi006 bash[27292]: audit 2023-08-28T02:08:22.063185+0000 mon.a (mon.0) 4828 : audit [DBG] from='client.? 172.21.15.6:0/2769759408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:22.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:22 smithi006 bash[33918]: cluster 2023-08-28T02:08:21.160693+0000 mgr.y (mgr.14901) 7492 : cluster [DBG] pgmap v7499: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:22.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:22 smithi006 bash[33918]: audit 2023-08-28T02:08:22.063185+0000 mon.a (mon.0) 4828 : audit [DBG] from='client.? 172.21.15.6:0/2769759408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:08:22.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:08:22.785Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:23.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:23 smithi130 bash[51302]: debug 2023-08-28T02:08:23.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:23.309989+0000) 2023-08-28T02:08:23.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:23 smithi006 bash[33918]: cluster 2023-08-28T02:08:23.161393+0000 mgr.y (mgr.14901) 7493 : cluster [DBG] pgmap v7500: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:23.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:23 smithi006 bash[27292]: cluster 2023-08-28T02:08:23.161393+0000 mgr.y (mgr.14901) 7493 : cluster [DBG] pgmap v7500: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:24.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:23 smithi130 bash[27579]: cluster 2023-08-28T02:08:23.161393+0000 mgr.y (mgr.14901) 7493 : cluster [DBG] pgmap v7500: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:24.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:24 smithi130 bash[51302]: debug 2023-08-28T02:08:24.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:24.310271+0000) 2023-08-28T02:08:24.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:24 smithi006 bash[27292]: audit 2023-08-28T02:08:24.478275+0000 mon.c (mon.2) 2946 : audit [DBG] from='client.? 172.21.15.6:0/185902689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:24.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:24 smithi006 bash[33918]: audit 2023-08-28T02:08:24.478275+0000 mon.c (mon.2) 2946 : audit [DBG] from='client.? 172.21.15.6:0/185902689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:08:24.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:08:24.662Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:25.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:24 smithi130 bash[27579]: audit 2023-08-28T02:08:24.478275+0000 mon.c (mon.2) 2946 : audit [DBG] from='client.? 172.21.15.6:0/185902689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:25.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:25 smithi130 bash[51302]: debug 2023-08-28T02:08:25.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:25.310540+0000) 2023-08-28T02:08:25.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:25 smithi006 bash[27292]: cluster 2023-08-28T02:08:25.162516+0000 mgr.y (mgr.14901) 7494 : cluster [DBG] pgmap v7501: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:25.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:25 smithi006 bash[33918]: cluster 2023-08-28T02:08:25.162516+0000 mgr.y (mgr.14901) 7494 : cluster [DBG] pgmap v7501: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:26.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:25 smithi130 bash[27579]: cluster 2023-08-28T02:08:25.162516+0000 mgr.y (mgr.14901) 7494 : cluster [DBG] pgmap v7501: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:26.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:26 smithi130 bash[51302]: debug 2023-08-28T02:08:26.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:26.310845+0000) 2023-08-28T02:08:27.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:26 smithi006 bash[27292]: audit 2023-08-28T02:08:26.904292+0000 mon.c (mon.2) 2947 : audit [DBG] from='client.? 172.21.15.6:0/34470109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:27.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:26 smithi006 bash[33918]: audit 2023-08-28T02:08:26.904292+0000 mon.c (mon.2) 2947 : audit [DBG] from='client.? 172.21.15.6:0/34470109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:27.309 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:26 smithi130 bash[27579]: audit 2023-08-28T02:08:26.904292+0000 mon.c (mon.2) 2947 : audit [DBG] from='client.? 172.21.15.6:0/34470109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:27.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:27 smithi130 bash[51302]: debug 2023-08-28T02:08:27.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:27.311097+0000) 2023-08-28T02:08:28.309 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:27 smithi130 bash[27579]: cluster 2023-08-28T02:08:27.163214+0000 mgr.y (mgr.14901) 7495 : cluster [DBG] pgmap v7502: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:28.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:27 smithi006 bash[27292]: cluster 2023-08-28T02:08:27.163214+0000 mgr.y (mgr.14901) 7495 : cluster [DBG] pgmap v7502: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:28.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:27 smithi006 bash[33918]: cluster 2023-08-28T02:08:27.163214+0000 mgr.y (mgr.14901) 7495 : cluster [DBG] pgmap v7502: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:28.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:28 smithi130 bash[51302]: debug 2023-08-28T02:08:28.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:28.311312+0000) 2023-08-28T02:08:29.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:29 smithi130 bash[51302]: debug 2023-08-28T02:08:29.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:29.311614+0000) 2023-08-28T02:08:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:30 smithi130 bash[27579]: cluster 2023-08-28T02:08:29.164285+0000 mgr.y (mgr.14901) 7496 : cluster [DBG] pgmap v7503: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:30 smithi130 bash[27579]: audit 2023-08-28T02:08:29.328256+0000 mon.c (mon.2) 2948 : audit [DBG] from='client.? 172.21.15.6:0/3718611462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:30.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:30 smithi130 bash[51302]: debug 2023-08-28T02:08:30.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:30.311904+0000) 2023-08-28T02:08:30.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:30 smithi006 bash[27292]: cluster 2023-08-28T02:08:29.164285+0000 mgr.y (mgr.14901) 7496 : cluster [DBG] pgmap v7503: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:30.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:30 smithi006 bash[27292]: audit 2023-08-28T02:08:29.328256+0000 mon.c (mon.2) 2948 : audit [DBG] from='client.? 172.21.15.6:0/3718611462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:30 smithi006 bash[33918]: cluster 2023-08-28T02:08:29.164285+0000 mgr.y (mgr.14901) 7496 : cluster [DBG] pgmap v7503: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:30 smithi006 bash[33918]: audit 2023-08-28T02:08:29.328256+0000 mon.c (mon.2) 2948 : audit [DBG] from='client.? 172.21.15.6:0/3718611462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:31.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:08:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:08:30] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:08:31.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:31 smithi130 bash[51302]: debug 2023-08-28T02:08:31.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:31.312218+0000) 2023-08-28T02:08:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:08:31.218Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:08:31.219Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:32.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:32 smithi130 bash[51302]: debug 2023-08-28T02:08:32.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:32.312543+0000) 2023-08-28T02:08:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:32 smithi130 bash[27579]: cluster 2023-08-28T02:08:31.164981+0000 mgr.y (mgr.14901) 7497 : cluster [DBG] pgmap v7504: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:32 smithi130 bash[27579]: audit 2023-08-28T02:08:31.748156+0000 mon.a (mon.0) 4829 : audit [DBG] from='client.? 172.21.15.6:0/3401061910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:32.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:32 smithi006 bash[33918]: cluster 2023-08-28T02:08:31.164981+0000 mgr.y (mgr.14901) 7497 : cluster [DBG] pgmap v7504: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:32.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:32 smithi006 bash[33918]: audit 2023-08-28T02:08:31.748156+0000 mon.a (mon.0) 4829 : audit [DBG] from='client.? 172.21.15.6:0/3401061910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:32.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:32 smithi006 bash[27292]: cluster 2023-08-28T02:08:31.164981+0000 mgr.y (mgr.14901) 7497 : cluster [DBG] pgmap v7504: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:32.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:32 smithi006 bash[27292]: audit 2023-08-28T02:08:31.748156+0000 mon.a (mon.0) 4829 : audit [DBG] from='client.? 172.21.15.6:0/3401061910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:08:32.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:08:32.786Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:33.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:33 smithi130 bash[51302]: debug 2023-08-28T02:08:33.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:33.312864+0000) 2023-08-28T02:08:33.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:33 smithi006 bash[33918]: cluster 2023-08-28T02:08:33.165603+0000 mgr.y (mgr.14901) 7498 : cluster [DBG] pgmap v7505: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:33 smithi006 bash[27292]: cluster 2023-08-28T02:08:33.165603+0000 mgr.y (mgr.14901) 7498 : cluster [DBG] pgmap v7505: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:34.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:33 smithi130 bash[27579]: cluster 2023-08-28T02:08:33.165603+0000 mgr.y (mgr.14901) 7498 : cluster [DBG] pgmap v7505: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:34.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:34 smithi130 bash[51302]: debug 2023-08-28T02:08:34.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:34.313227+0000) 2023-08-28T02:08:34.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:34 smithi006 bash[27292]: audit 2023-08-28T02:08:34.167908+0000 mon.a (mon.0) 4830 : audit [DBG] from='client.? 172.21.15.6:0/4184070536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:34.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:34 smithi006 bash[33918]: audit 2023-08-28T02:08:34.167908+0000 mon.a (mon.0) 4830 : audit [DBG] from='client.? 172.21.15.6:0/4184070536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:08:34.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:34.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:08:34.662Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:35.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:34 smithi130 bash[27579]: audit 2023-08-28T02:08:34.167908+0000 mon.a (mon.0) 4830 : audit [DBG] from='client.? 172.21.15.6:0/4184070536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:35.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:35 smithi130 bash[51302]: debug 2023-08-28T02:08:35.313+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:35.313542+0000) 2023-08-28T02:08:35.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:35 smithi006 bash[27292]: cluster 2023-08-28T02:08:35.166686+0000 mgr.y (mgr.14901) 7499 : cluster [DBG] pgmap v7506: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:35.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:35 smithi006 bash[33918]: cluster 2023-08-28T02:08:35.166686+0000 mgr.y (mgr.14901) 7499 : cluster [DBG] pgmap v7506: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:36.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:35 smithi130 bash[27579]: cluster 2023-08-28T02:08:35.166686+0000 mgr.y (mgr.14901) 7499 : cluster [DBG] pgmap v7506: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:36.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:36 smithi130 bash[51302]: debug 2023-08-28T02:08:36.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:36.313863+0000) 2023-08-28T02:08:36.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:36 smithi006 bash[27292]: audit 2023-08-28T02:08:36.584613+0000 mon.a (mon.0) 4831 : audit [DBG] from='client.? 172.21.15.6:0/818294777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:36.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:36 smithi006 bash[33918]: audit 2023-08-28T02:08:36.584613+0000 mon.a (mon.0) 4831 : audit [DBG] from='client.? 172.21.15.6:0/818294777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:37.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:36 smithi130 bash[27579]: audit 2023-08-28T02:08:36.584613+0000 mon.a (mon.0) 4831 : audit [DBG] from='client.? 172.21.15.6:0/818294777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:37.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:37 smithi130 bash[51302]: debug 2023-08-28T02:08:37.309+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:37.314222+0000) 2023-08-28T02:08:37.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:37 smithi006 bash[27292]: cluster 2023-08-28T02:08:37.167344+0000 mgr.y (mgr.14901) 7500 : cluster [DBG] pgmap v7507: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:37.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:37 smithi006 bash[33918]: cluster 2023-08-28T02:08:37.167344+0000 mgr.y (mgr.14901) 7500 : cluster [DBG] pgmap v7507: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:38.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:37 smithi130 bash[27579]: cluster 2023-08-28T02:08:37.167344+0000 mgr.y (mgr.14901) 7500 : cluster [DBG] pgmap v7507: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:38.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:38 smithi130 bash[51302]: debug 2023-08-28T02:08:38.313+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:38.314578+0000) 2023-08-28T02:08:39.313 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:39 smithi130 bash[27579]: audit 2023-08-28T02:08:38.999931+0000 mon.c (mon.2) 2949 : audit [DBG] from='client.? 172.21.15.6:0/2998035958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:39.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:39 smithi006 bash[33918]: audit 2023-08-28T02:08:38.999931+0000 mon.c (mon.2) 2949 : audit [DBG] from='client.? 172.21.15.6:0/2998035958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:39.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:39 smithi006 bash[27292]: audit 2023-08-28T02:08:38.999931+0000 mon.c (mon.2) 2949 : audit [DBG] from='client.? 172.21.15.6:0/2998035958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:39.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:39 smithi130 bash[51302]: debug 2023-08-28T02:08:39.313+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:39.314930+0000) 2023-08-28T02:08:40.313 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:40 smithi130 bash[27579]: cluster 2023-08-28T02:08:39.168572+0000 mgr.y (mgr.14901) 7501 : cluster [DBG] pgmap v7508: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:40.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:40 smithi006 bash[27292]: cluster 2023-08-28T02:08:39.168572+0000 mgr.y (mgr.14901) 7501 : cluster [DBG] pgmap v7508: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:40 smithi006 bash[33918]: cluster 2023-08-28T02:08:39.168572+0000 mgr.y (mgr.14901) 7501 : cluster [DBG] pgmap v7508: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:40.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:40 smithi130 bash[51302]: debug 2023-08-28T02:08:40.313+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:40.315211+0000) 2023-08-28T02:08:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:08:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:08:40] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:08:41.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:41 smithi130 bash[51302]: debug 2023-08-28T02:08:41.313+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:41.315499+0000) 2023-08-28T02:08:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:08:41.218Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:08:41.219Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:42.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:42 smithi130 bash[51302]: debug 2023-08-28T02:08:42.312+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:42.315801+0000) 2023-08-28T02:08:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:42 smithi130 bash[27579]: cluster 2023-08-28T02:08:41.169301+0000 mgr.y (mgr.14901) 7502 : cluster [DBG] pgmap v7509: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:42 smithi130 bash[27579]: audit 2023-08-28T02:08:41.415310+0000 mon.a (mon.0) 4832 : audit [DBG] from='client.? 172.21.15.6:0/2186208552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:42.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:42 smithi006 bash[33918]: cluster 2023-08-28T02:08:41.169301+0000 mgr.y (mgr.14901) 7502 : cluster [DBG] pgmap v7509: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:42.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:42 smithi006 bash[33918]: audit 2023-08-28T02:08:41.415310+0000 mon.a (mon.0) 4832 : audit [DBG] from='client.? 172.21.15.6:0/2186208552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:42.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:42 smithi006 bash[27292]: cluster 2023-08-28T02:08:41.169301+0000 mgr.y (mgr.14901) 7502 : cluster [DBG] pgmap v7509: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:42.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:42 smithi006 bash[27292]: audit 2023-08-28T02:08:41.415310+0000 mon.a (mon.0) 4832 : audit [DBG] from='client.? 172.21.15.6:0/2186208552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:08:42.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:08:42.788Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:43.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:43 smithi130 bash[51302]: debug 2023-08-28T02:08:43.312+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:43.316106+0000) 2023-08-28T02:08:43.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:43 smithi006 bash[27292]: cluster 2023-08-28T02:08:43.169965+0000 mgr.y (mgr.14901) 7503 : cluster [DBG] pgmap v7510: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:43.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:43 smithi006 bash[33918]: cluster 2023-08-28T02:08:43.169965+0000 mgr.y (mgr.14901) 7503 : cluster [DBG] pgmap v7510: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:44.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:43 smithi130 bash[27579]: cluster 2023-08-28T02:08:43.169965+0000 mgr.y (mgr.14901) 7503 : cluster [DBG] pgmap v7510: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:44.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:44 smithi130 bash[51302]: debug 2023-08-28T02:08:44.312+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:44.316520+0000) 2023-08-28T02:08:44.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:44 smithi006 bash[27292]: audit 2023-08-28T02:08:43.833434+0000 mon.c (mon.2) 2950 : audit [DBG] from='client.? 172.21.15.6:0/2710576182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:44.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:44 smithi006 bash[33918]: audit 2023-08-28T02:08:43.833434+0000 mon.c (mon.2) 2950 : audit [DBG] from='client.? 172.21.15.6:0/2710576182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:08:44.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:08:44.663Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:45.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:44 smithi130 bash[27579]: audit 2023-08-28T02:08:43.833434+0000 mon.c (mon.2) 2950 : audit [DBG] from='client.? 172.21.15.6:0/2710576182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:45.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:45 smithi130 bash[51302]: debug 2023-08-28T02:08:45.312+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:45.316884+0000) 2023-08-28T02:08:45.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:45 smithi006 bash[27292]: cluster 2023-08-28T02:08:45.171107+0000 mgr.y (mgr.14901) 7504 : cluster [DBG] pgmap v7511: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:45.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:45 smithi006 bash[33918]: cluster 2023-08-28T02:08:45.171107+0000 mgr.y (mgr.14901) 7504 : cluster [DBG] pgmap v7511: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:46.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:45 smithi130 bash[27579]: cluster 2023-08-28T02:08:45.171107+0000 mgr.y (mgr.14901) 7504 : cluster [DBG] pgmap v7511: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:46.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:46 smithi130 bash[51302]: debug 2023-08-28T02:08:46.312+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:46.317206+0000) 2023-08-28T02:08:46.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:46 smithi006 bash[27292]: audit 2023-08-28T02:08:46.254840+0000 mon.c (mon.2) 2951 : audit [DBG] from='client.? 172.21.15.6:0/4028328606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:46.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:46 smithi006 bash[33918]: audit 2023-08-28T02:08:46.254840+0000 mon.c (mon.2) 2951 : audit [DBG] from='client.? 172.21.15.6:0/4028328606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:47.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:46 smithi130 bash[27579]: audit 2023-08-28T02:08:46.254840+0000 mon.c (mon.2) 2951 : audit [DBG] from='client.? 172.21.15.6:0/4028328606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:47.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:47 smithi130 bash[51302]: debug 2023-08-28T02:08:47.316+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:47.317541+0000) 2023-08-28T02:08:47.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:47 smithi006 bash[27292]: audit 2023-08-28T02:08:46.750273+0000 mon.b (mon.1) 831 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:08:47.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:47 smithi006 bash[27292]: cluster 2023-08-28T02:08:47.171856+0000 mgr.y (mgr.14901) 7505 : cluster [DBG] pgmap v7512: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:47.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:47 smithi006 bash[33918]: audit 2023-08-28T02:08:46.750273+0000 mon.b (mon.1) 831 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:08:47.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:47 smithi006 bash[33918]: cluster 2023-08-28T02:08:47.171856+0000 mgr.y (mgr.14901) 7505 : cluster [DBG] pgmap v7512: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:48.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:47 smithi130 bash[27579]: audit 2023-08-28T02:08:46.750273+0000 mon.b (mon.1) 831 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:08:48.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:47 smithi130 bash[27579]: cluster 2023-08-28T02:08:47.171856+0000 mgr.y (mgr.14901) 7505 : cluster [DBG] pgmap v7512: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:48.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:48 smithi130 bash[51302]: debug 2023-08-28T02:08:48.316+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:48.317909+0000) 2023-08-28T02:08:49.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:48 smithi130 bash[27579]: audit 2023-08-28T02:08:48.664794+0000 mon.a (mon.0) 4833 : audit [DBG] from='client.? 172.21.15.6:0/1033185832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:49.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:48 smithi006 bash[27292]: audit 2023-08-28T02:08:48.664794+0000 mon.a (mon.0) 4833 : audit [DBG] from='client.? 172.21.15.6:0/1033185832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:49.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:48 smithi006 bash[33918]: audit 2023-08-28T02:08:48.664794+0000 mon.a (mon.0) 4833 : audit [DBG] from='client.? 172.21.15.6:0/1033185832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:49.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:49 smithi130 bash[51302]: debug 2023-08-28T02:08:49.312+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:49.318247+0000) 2023-08-28T02:08:50.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:49 smithi130 bash[27579]: cluster 2023-08-28T02:08:49.173013+0000 mgr.y (mgr.14901) 7506 : cluster [DBG] pgmap v7513: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:50.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:49 smithi006 bash[33918]: cluster 2023-08-28T02:08:49.173013+0000 mgr.y (mgr.14901) 7506 : cluster [DBG] pgmap v7513: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:50.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:49 smithi006 bash[27292]: cluster 2023-08-28T02:08:49.173013+0000 mgr.y (mgr.14901) 7506 : cluster [DBG] pgmap v7513: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:50.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:50 smithi130 bash[51302]: debug 2023-08-28T02:08:50.312+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:50.318597+0000) 2023-08-28T02:08:51.143 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:08:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:08:50] "GET /metrics HTTP/1.1" 200 193113 "" "Prometheus/2.33.4" 2023-08-28T02:08:51.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:51 smithi006 bash[33918]: audit 2023-08-28T02:08:51.091017+0000 mon.a (mon.0) 4834 : audit [DBG] from='client.? 172.21.15.6:0/1953448920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:51.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:51 smithi006 bash[27292]: audit 2023-08-28T02:08:51.091017+0000 mon.a (mon.0) 4834 : audit [DBG] from='client.? 172.21.15.6:0/1953448920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:08:51.218Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:51.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:08:51.220Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:51.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:51 smithi130 bash[27579]: audit 2023-08-28T02:08:51.091017+0000 mon.a (mon.0) 4834 : audit [DBG] from='client.? 172.21.15.6:0/1953448920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:51.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:51 smithi130 bash[51302]: debug 2023-08-28T02:08:51.316+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:51.318977+0000) 2023-08-28T02:08:52.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:52 smithi006 bash[27292]: cluster 2023-08-28T02:08:51.173586+0000 mgr.y (mgr.14901) 7507 : cluster [DBG] pgmap v7514: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:52.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:52 smithi006 bash[33918]: cluster 2023-08-28T02:08:51.173586+0000 mgr.y (mgr.14901) 7507 : cluster [DBG] pgmap v7514: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:52.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:52 smithi130 bash[51302]: debug 2023-08-28T02:08:52.316+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:52.319238+0000) 2023-08-28T02:08:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:52 smithi130 bash[27579]: cluster 2023-08-28T02:08:51.173586+0000 mgr.y (mgr.14901) 7507 : cluster [DBG] pgmap v7514: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:08:52.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:08:52.788Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:53.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:53 smithi130 bash[51302]: debug 2023-08-28T02:08:53.316+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:53.319481+0000) 2023-08-28T02:08:53.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:53 smithi006 bash[27292]: audit 2023-08-28T02:08:52.678379+0000 mon.a (mon.0) 4835 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:08:53.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:53 smithi006 bash[27292]: audit 2023-08-28T02:08:53.068282+0000 mon.a (mon.0) 4836 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:08:53.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:53 smithi006 bash[27292]: cluster 2023-08-28T02:08:53.174229+0000 mgr.y (mgr.14901) 7508 : cluster [DBG] pgmap v7515: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:53.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:53 smithi006 bash[27292]: audit 2023-08-28T02:08:53.366046+0000 mon.b (mon.1) 832 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:53.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:53 smithi006 bash[27292]: audit 2023-08-28T02:08:53.367799+0000 mon.b (mon.1) 833 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:08:53.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:53 smithi006 bash[27292]: audit 2023-08-28T02:08:53.375234+0000 mon.a (mon.0) 4837 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:08:53.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:53 smithi006 bash[27292]: audit 2023-08-28T02:08:53.501746+0000 mon.a (mon.0) 4838 : audit [DBG] from='client.? 172.21.15.6:0/2885187276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:53 smithi006 bash[33918]: audit 2023-08-28T02:08:52.678379+0000 mon.a (mon.0) 4835 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:08:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:53 smithi006 bash[33918]: audit 2023-08-28T02:08:53.068282+0000 mon.a (mon.0) 4836 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:08:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:53 smithi006 bash[33918]: cluster 2023-08-28T02:08:53.174229+0000 mgr.y (mgr.14901) 7508 : cluster [DBG] pgmap v7515: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:53 smithi006 bash[33918]: audit 2023-08-28T02:08:53.366046+0000 mon.b (mon.1) 832 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:53.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:53 smithi006 bash[33918]: audit 2023-08-28T02:08:53.367799+0000 mon.b (mon.1) 833 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:08:53.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:53 smithi006 bash[33918]: audit 2023-08-28T02:08:53.375234+0000 mon.a (mon.0) 4837 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:08:53.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:53 smithi006 bash[33918]: audit 2023-08-28T02:08:53.501746+0000 mon.a (mon.0) 4838 : audit [DBG] from='client.? 172.21.15.6:0/2885187276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:53 smithi130 bash[27579]: audit 2023-08-28T02:08:52.678379+0000 mon.a (mon.0) 4835 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:08:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:53 smithi130 bash[27579]: audit 2023-08-28T02:08:53.068282+0000 mon.a (mon.0) 4836 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:08:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:53 smithi130 bash[27579]: cluster 2023-08-28T02:08:53.174229+0000 mgr.y (mgr.14901) 7508 : cluster [DBG] pgmap v7515: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:53 smithi130 bash[27579]: audit 2023-08-28T02:08:53.366046+0000 mon.b (mon.1) 832 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:53 smithi130 bash[27579]: audit 2023-08-28T02:08:53.367799+0000 mon.b (mon.1) 833 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:08:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:53 smithi130 bash[27579]: audit 2023-08-28T02:08:53.375234+0000 mon.a (mon.0) 4837 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:08:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:53 smithi130 bash[27579]: audit 2023-08-28T02:08:53.501746+0000 mon.a (mon.0) 4838 : audit [DBG] from='client.? 172.21.15.6:0/2885187276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:54.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:54 smithi130 bash[51302]: debug 2023-08-28T02:08:54.316+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:54.319700+0000) 2023-08-28T02:08:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:08:54.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:08:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:08:54.664Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:08:55.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:55 smithi130 bash[51302]: debug 2023-08-28T02:08:55.316+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:55.320012+0000) 2023-08-28T02:08:56.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:56 smithi130 bash[51302]: debug 2023-08-28T02:08:56.316+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:56.320285+0000) 2023-08-28T02:08:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:56 smithi130 bash[27579]: cluster 2023-08-28T02:08:55.175351+0000 mgr.y (mgr.14901) 7509 : cluster [DBG] pgmap v7516: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:56 smithi130 bash[27579]: audit 2023-08-28T02:08:55.919516+0000 mon.c (mon.2) 2952 : audit [DBG] from='client.? 172.21.15.6:0/394842719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:56.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:56 smithi006 bash[27292]: cluster 2023-08-28T02:08:55.175351+0000 mgr.y (mgr.14901) 7509 : cluster [DBG] pgmap v7516: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:56.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:56 smithi006 bash[27292]: audit 2023-08-28T02:08:55.919516+0000 mon.c (mon.2) 2952 : audit [DBG] from='client.? 172.21.15.6:0/394842719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:56.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:56 smithi006 bash[33918]: cluster 2023-08-28T02:08:55.175351+0000 mgr.y (mgr.14901) 7509 : cluster [DBG] pgmap v7516: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:56.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:56 smithi006 bash[33918]: audit 2023-08-28T02:08:55.919516+0000 mon.c (mon.2) 2952 : audit [DBG] from='client.? 172.21.15.6:0/394842719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:57.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:57 smithi130 bash[51302]: debug 2023-08-28T02:08:57.316+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:57.320565+0000) 2023-08-28T02:08:58.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:58 smithi130 bash[51302]: debug 2023-08-28T02:08:58.316+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:58.320843+0000) 2023-08-28T02:08:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:58 smithi130 bash[27579]: cluster 2023-08-28T02:08:57.176027+0000 mgr.y (mgr.14901) 7510 : cluster [DBG] pgmap v7517: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:58.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:58 smithi006 bash[33918]: cluster 2023-08-28T02:08:57.176027+0000 mgr.y (mgr.14901) 7510 : cluster [DBG] pgmap v7517: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:58.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:58 smithi006 bash[27292]: cluster 2023-08-28T02:08:57.176027+0000 mgr.y (mgr.14901) 7510 : cluster [DBG] pgmap v7517: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:08:59.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:08:59 smithi130 bash[27579]: audit 2023-08-28T02:08:58.332871+0000 mon.c (mon.2) 2953 : audit [DBG] from='client.? 172.21.15.6:0/178500057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:59.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:08:59 smithi130 bash[51302]: debug 2023-08-28T02:08:59.320+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:59.321127+0000) 2023-08-28T02:08:59.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:08:59 smithi006 bash[27292]: audit 2023-08-28T02:08:58.332871+0000 mon.c (mon.2) 2953 : audit [DBG] from='client.? 172.21.15.6:0/178500057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:59.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:08:59 smithi006 bash[33918]: audit 2023-08-28T02:08:58.332871+0000 mon.c (mon.2) 2953 : audit [DBG] from='client.? 172.21.15.6:0/178500057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:00 smithi130 bash[27579]: cluster 2023-08-28T02:08:59.177121+0000 mgr.y (mgr.14901) 7511 : cluster [DBG] pgmap v7518: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:00.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:00 smithi130 bash[51302]: debug 2023-08-28T02:09:00.320+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:00.321449+0000) 2023-08-28T02:09:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:00 smithi006 bash[33918]: cluster 2023-08-28T02:08:59.177121+0000 mgr.y (mgr.14901) 7511 : cluster [DBG] pgmap v7518: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:00 smithi006 bash[27292]: cluster 2023-08-28T02:08:59.177121+0000 mgr.y (mgr.14901) 7511 : cluster [DBG] pgmap v7518: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:09:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:09:00] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T02:09:01.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:01 smithi130 bash[27579]: audit 2023-08-28T02:09:00.750777+0000 mon.c (mon.2) 2954 : audit [DBG] from='client.? 172.21.15.6:0/647223559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:01.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:01 smithi130 bash[51302]: debug 2023-08-28T02:09:01.316+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:01.321779+0000) 2023-08-28T02:09:01.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:01 smithi006 bash[27292]: audit 2023-08-28T02:09:00.750777+0000 mon.c (mon.2) 2954 : audit [DBG] from='client.? 172.21.15.6:0/647223559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:01.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:01 smithi006 bash[33918]: audit 2023-08-28T02:09:00.750777+0000 mon.c (mon.2) 2954 : audit [DBG] from='client.? 172.21.15.6:0/647223559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:01.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:09:01.219Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:01.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:09:01.220Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:02 smithi130 bash[27579]: cluster 2023-08-28T02:09:01.177736+0000 mgr.y (mgr.14901) 7512 : cluster [DBG] pgmap v7519: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:02.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:02 smithi130 bash[51302]: debug 2023-08-28T02:09:02.316+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:02.322016+0000) 2023-08-28T02:09:02.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:02 smithi006 bash[33918]: cluster 2023-08-28T02:09:01.177736+0000 mgr.y (mgr.14901) 7512 : cluster [DBG] pgmap v7519: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:02.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:02 smithi006 bash[27292]: cluster 2023-08-28T02:09:01.177736+0000 mgr.y (mgr.14901) 7512 : cluster [DBG] pgmap v7519: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:09:02.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:09:02.788Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:03.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:03 smithi130 bash[51302]: debug 2023-08-28T02:09:03.316+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:03.322305+0000) 2023-08-28T02:09:03.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:03 smithi006 bash[27292]: audit 2023-08-28T02:09:03.168380+0000 mon.c (mon.2) 2955 : audit [DBG] from='client.? 172.21.15.6:0/1377586758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:03.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:03 smithi006 bash[27292]: cluster 2023-08-28T02:09:03.178213+0000 mgr.y (mgr.14901) 7513 : cluster [DBG] pgmap v7520: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:03.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:03 smithi006 bash[33918]: audit 2023-08-28T02:09:03.168380+0000 mon.c (mon.2) 2955 : audit [DBG] from='client.? 172.21.15.6:0/1377586758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:03.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:03 smithi006 bash[33918]: cluster 2023-08-28T02:09:03.178213+0000 mgr.y (mgr.14901) 7513 : cluster [DBG] pgmap v7520: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:03 smithi130 bash[27579]: audit 2023-08-28T02:09:03.168380+0000 mon.c (mon.2) 2955 : audit [DBG] from='client.? 172.21.15.6:0/1377586758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:03 smithi130 bash[27579]: cluster 2023-08-28T02:09:03.178213+0000 mgr.y (mgr.14901) 7513 : cluster [DBG] pgmap v7520: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:04.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:04 smithi130 bash[51302]: debug 2023-08-28T02:09:04.320+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:04.322594+0000) 2023-08-28T02:09:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:09:04.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:09:04.664Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:05.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:05 smithi130 bash[51302]: debug 2023-08-28T02:09:05.320+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:05.322898+0000) 2023-08-28T02:09:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:06 smithi130 bash[27579]: cluster 2023-08-28T02:09:05.179250+0000 mgr.y (mgr.14901) 7514 : cluster [DBG] pgmap v7521: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:06 smithi130 bash[27579]: audit 2023-08-28T02:09:05.587090+0000 mon.c (mon.2) 2956 : audit [DBG] from='client.? 172.21.15.6:0/3858152677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:06.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:06 smithi130 bash[51302]: debug 2023-08-28T02:09:06.320+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:06.323129+0000) 2023-08-28T02:09:06.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:06 smithi006 bash[33918]: cluster 2023-08-28T02:09:05.179250+0000 mgr.y (mgr.14901) 7514 : cluster [DBG] pgmap v7521: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:06.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:06 smithi006 bash[33918]: audit 2023-08-28T02:09:05.587090+0000 mon.c (mon.2) 2956 : audit [DBG] from='client.? 172.21.15.6:0/3858152677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:06 smithi006 bash[27292]: cluster 2023-08-28T02:09:05.179250+0000 mgr.y (mgr.14901) 7514 : cluster [DBG] pgmap v7521: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:06 smithi006 bash[27292]: audit 2023-08-28T02:09:05.587090+0000 mon.c (mon.2) 2956 : audit [DBG] from='client.? 172.21.15.6:0/3858152677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:07.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:07 smithi130 bash[51302]: debug 2023-08-28T02:09:07.320+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:07.323398+0000) 2023-08-28T02:09:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:08 smithi130 bash[27579]: cluster 2023-08-28T02:09:07.179911+0000 mgr.y (mgr.14901) 7515 : cluster [DBG] pgmap v7522: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:08 smithi130 bash[27579]: audit 2023-08-28T02:09:07.997092+0000 mon.c (mon.2) 2957 : audit [DBG] from='client.? 172.21.15.6:0/2950086948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:08.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:08 smithi130 bash[51302]: debug 2023-08-28T02:09:08.320+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:08.323660+0000) 2023-08-28T02:09:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:08 smithi006 bash[33918]: cluster 2023-08-28T02:09:07.179911+0000 mgr.y (mgr.14901) 7515 : cluster [DBG] pgmap v7522: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:08 smithi006 bash[33918]: audit 2023-08-28T02:09:07.997092+0000 mon.c (mon.2) 2957 : audit [DBG] from='client.? 172.21.15.6:0/2950086948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:08 smithi006 bash[27292]: cluster 2023-08-28T02:09:07.179911+0000 mgr.y (mgr.14901) 7515 : cluster [DBG] pgmap v7522: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:08 smithi006 bash[27292]: audit 2023-08-28T02:09:07.997092+0000 mon.c (mon.2) 2957 : audit [DBG] from='client.? 172.21.15.6:0/2950086948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:09.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:09 smithi130 bash[51302]: debug 2023-08-28T02:09:09.320+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:09.323945+0000) 2023-08-28T02:09:10.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:10 smithi130 bash[51302]: debug 2023-08-28T02:09:10.320+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:10.324308+0000) 2023-08-28T02:09:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:10 smithi130 bash[27579]: cluster 2023-08-28T02:09:09.180925+0000 mgr.y (mgr.14901) 7516 : cluster [DBG] pgmap v7523: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:10.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:10 smithi006 bash[27292]: cluster 2023-08-28T02:09:09.180925+0000 mgr.y (mgr.14901) 7516 : cluster [DBG] pgmap v7523: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:10 smithi006 bash[33918]: cluster 2023-08-28T02:09:09.180925+0000 mgr.y (mgr.14901) 7516 : cluster [DBG] pgmap v7523: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:11.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:09:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:09:10] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T02:09:11.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:11 smithi130 bash[27579]: audit 2023-08-28T02:09:10.427838+0000 mon.c (mon.2) 2958 : audit [DBG] from='client.? 172.21.15.6:0/3010802852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:11.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:11 smithi130 bash[51302]: debug 2023-08-28T02:09:11.320+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:11.324616+0000) 2023-08-28T02:09:11.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:11 smithi006 bash[27292]: audit 2023-08-28T02:09:10.427838+0000 mon.c (mon.2) 2958 : audit [DBG] from='client.? 172.21.15.6:0/3010802852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:09:11.219Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:09:11.221Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:11.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:11 smithi006 bash[33918]: audit 2023-08-28T02:09:10.427838+0000 mon.c (mon.2) 2958 : audit [DBG] from='client.? 172.21.15.6:0/3010802852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:12.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:12 smithi130 bash[51302]: debug 2023-08-28T02:09:12.320+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:12.324976+0000) 2023-08-28T02:09:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:12 smithi130 bash[27579]: cluster 2023-08-28T02:09:11.181510+0000 mgr.y (mgr.14901) 7517 : cluster [DBG] pgmap v7524: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:12.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:12 smithi006 bash[27292]: cluster 2023-08-28T02:09:11.181510+0000 mgr.y (mgr.14901) 7517 : cluster [DBG] pgmap v7524: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:12.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:12 smithi006 bash[33918]: cluster 2023-08-28T02:09:11.181510+0000 mgr.y (mgr.14901) 7517 : cluster [DBG] pgmap v7524: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:09:12.788Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:09:12.789Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:13.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:13 smithi130 bash[51302]: debug 2023-08-28T02:09:13.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:13.325260+0000) 2023-08-28T02:09:13.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:13 smithi006 bash[27292]: audit 2023-08-28T02:09:12.848323+0000 mon.c (mon.2) 2959 : audit [DBG] from='client.? 172.21.15.6:0/4186253201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:13.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:13 smithi006 bash[27292]: cluster 2023-08-28T02:09:13.182153+0000 mgr.y (mgr.14901) 7518 : cluster [DBG] pgmap v7525: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:13.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:13 smithi006 bash[33918]: audit 2023-08-28T02:09:12.848323+0000 mon.c (mon.2) 2959 : audit [DBG] from='client.? 172.21.15.6:0/4186253201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:13.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:13 smithi006 bash[33918]: cluster 2023-08-28T02:09:13.182153+0000 mgr.y (mgr.14901) 7518 : cluster [DBG] pgmap v7525: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:13 smithi130 bash[27579]: audit 2023-08-28T02:09:12.848323+0000 mon.c (mon.2) 2959 : audit [DBG] from='client.? 172.21.15.6:0/4186253201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:13 smithi130 bash[27579]: cluster 2023-08-28T02:09:13.182153+0000 mgr.y (mgr.14901) 7518 : cluster [DBG] pgmap v7525: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:14.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:14 smithi130 bash[51302]: debug 2023-08-28T02:09:14.320+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:14.325628+0000) 2023-08-28T02:09:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:09:14.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:09:14.664Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:15.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:15 smithi130 bash[51302]: debug 2023-08-28T02:09:15.320+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:15.325935+0000) 2023-08-28T02:09:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:16 smithi130 bash[27579]: cluster 2023-08-28T02:09:15.183258+0000 mgr.y (mgr.14901) 7519 : cluster [DBG] pgmap v7526: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:16 smithi130 bash[27579]: audit 2023-08-28T02:09:15.266099+0000 mon.a (mon.0) 4839 : audit [DBG] from='client.? 172.21.15.6:0/3447996028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:16.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:16 smithi130 bash[51302]: debug 2023-08-28T02:09:16.320+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:16.326200+0000) 2023-08-28T02:09:16.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:16 smithi006 bash[33918]: cluster 2023-08-28T02:09:15.183258+0000 mgr.y (mgr.14901) 7519 : cluster [DBG] pgmap v7526: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:16.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:16 smithi006 bash[33918]: audit 2023-08-28T02:09:15.266099+0000 mon.a (mon.0) 4839 : audit [DBG] from='client.? 172.21.15.6:0/3447996028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:16 smithi006 bash[27292]: cluster 2023-08-28T02:09:15.183258+0000 mgr.y (mgr.14901) 7519 : cluster [DBG] pgmap v7526: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:16 smithi006 bash[27292]: audit 2023-08-28T02:09:15.266099+0000 mon.a (mon.0) 4839 : audit [DBG] from='client.? 172.21.15.6:0/3447996028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:17.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:17 smithi130 bash[51302]: debug 2023-08-28T02:09:17.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:17.326470+0000) 2023-08-28T02:09:18.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:18 smithi130 bash[51302]: debug 2023-08-28T02:09:18.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:18.326748+0000) 2023-08-28T02:09:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:18 smithi130 bash[27579]: cluster 2023-08-28T02:09:17.183916+0000 mgr.y (mgr.14901) 7520 : cluster [DBG] pgmap v7527: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:18 smithi130 bash[27579]: audit 2023-08-28T02:09:17.679608+0000 mon.c (mon.2) 2960 : audit [DBG] from='client.? 172.21.15.6:0/3016930358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:18 smithi006 bash[27292]: cluster 2023-08-28T02:09:17.183916+0000 mgr.y (mgr.14901) 7520 : cluster [DBG] pgmap v7527: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:18 smithi006 bash[27292]: audit 2023-08-28T02:09:17.679608+0000 mon.c (mon.2) 2960 : audit [DBG] from='client.? 172.21.15.6:0/3016930358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:18.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:18 smithi006 bash[33918]: cluster 2023-08-28T02:09:17.183916+0000 mgr.y (mgr.14901) 7520 : cluster [DBG] pgmap v7527: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:18.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:18 smithi006 bash[33918]: audit 2023-08-28T02:09:17.679608+0000 mon.c (mon.2) 2960 : audit [DBG] from='client.? 172.21.15.6:0/3016930358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:19.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:19 smithi130 bash[51302]: debug 2023-08-28T02:09:19.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:19.327050+0000) 2023-08-28T02:09:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:20 smithi130 bash[27579]: cluster 2023-08-28T02:09:19.184852+0000 mgr.y (mgr.14901) 7521 : cluster [DBG] pgmap v7528: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:20 smithi130 bash[27579]: audit 2023-08-28T02:09:20.092073+0000 mon.a (mon.0) 4840 : audit [DBG] from='client.? 172.21.15.6:0/501925108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:20.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:20 smithi130 bash[51302]: debug 2023-08-28T02:09:20.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:20.327330+0000) 2023-08-28T02:09:20.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:20 smithi006 bash[27292]: cluster 2023-08-28T02:09:19.184852+0000 mgr.y (mgr.14901) 7521 : cluster [DBG] pgmap v7528: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:20.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:20 smithi006 bash[27292]: audit 2023-08-28T02:09:20.092073+0000 mon.a (mon.0) 4840 : audit [DBG] from='client.? 172.21.15.6:0/501925108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:20 smithi006 bash[33918]: cluster 2023-08-28T02:09:19.184852+0000 mgr.y (mgr.14901) 7521 : cluster [DBG] pgmap v7528: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:20 smithi006 bash[33918]: audit 2023-08-28T02:09:20.092073+0000 mon.a (mon.0) 4840 : audit [DBG] from='client.? 172.21.15.6:0/501925108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:09:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:09:20] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T02:09:21.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:21 smithi130 bash[51302]: debug 2023-08-28T02:09:21.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:21.327557+0000) 2023-08-28T02:09:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:09:21.220Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:09:21.223Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:22 smithi130 bash[27579]: cluster 2023-08-28T02:09:21.185486+0000 mgr.y (mgr.14901) 7522 : cluster [DBG] pgmap v7529: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:22.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:22 smithi130 bash[51302]: debug 2023-08-28T02:09:22.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:22.327829+0000) 2023-08-28T02:09:22.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:22 smithi006 bash[33918]: cluster 2023-08-28T02:09:21.185486+0000 mgr.y (mgr.14901) 7522 : cluster [DBG] pgmap v7529: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:22.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:22 smithi006 bash[27292]: cluster 2023-08-28T02:09:21.185486+0000 mgr.y (mgr.14901) 7522 : cluster [DBG] pgmap v7529: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:09:22.788Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:09:22.791Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:23.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:23 smithi130 bash[27579]: audit 2023-08-28T02:09:22.507749+0000 mon.a (mon.0) 4841 : audit [DBG] from='client.? 172.21.15.6:0/2682129767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:23.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:23 smithi130 bash[51302]: debug 2023-08-28T02:09:23.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:23.328111+0000) 2023-08-28T02:09:23.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:23 smithi006 bash[27292]: audit 2023-08-28T02:09:22.507749+0000 mon.a (mon.0) 4841 : audit [DBG] from='client.? 172.21.15.6:0/2682129767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:23.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:23 smithi006 bash[33918]: audit 2023-08-28T02:09:22.507749+0000 mon.a (mon.0) 4841 : audit [DBG] from='client.? 172.21.15.6:0/2682129767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:24 smithi130 bash[27579]: cluster 2023-08-28T02:09:23.186135+0000 mgr.y (mgr.14901) 7523 : cluster [DBG] pgmap v7530: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:24.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:24 smithi130 bash[51302]: debug 2023-08-28T02:09:24.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:24.328390+0000) 2023-08-28T02:09:24.665 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:24 smithi006 bash[33918]: cluster 2023-08-28T02:09:23.186135+0000 mgr.y (mgr.14901) 7523 : cluster [DBG] pgmap v7530: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:24.665 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:24 smithi006 bash[27292]: cluster 2023-08-28T02:09:23.186135+0000 mgr.y (mgr.14901) 7523 : cluster [DBG] pgmap v7530: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:09:24.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:09:24.666Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:25.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:25 smithi130 bash[27579]: audit 2023-08-28T02:09:24.918245+0000 mon.c (mon.2) 2961 : audit [DBG] from='client.? 172.21.15.6:0/4153796850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:25.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:25 smithi130 bash[51302]: debug 2023-08-28T02:09:25.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:25.328688+0000) 2023-08-28T02:09:25.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:25 smithi006 bash[27292]: audit 2023-08-28T02:09:24.918245+0000 mon.c (mon.2) 2961 : audit [DBG] from='client.? 172.21.15.6:0/4153796850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:25.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:25 smithi006 bash[33918]: audit 2023-08-28T02:09:24.918245+0000 mon.c (mon.2) 2961 : audit [DBG] from='client.? 172.21.15.6:0/4153796850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:26 smithi130 bash[27579]: cluster 2023-08-28T02:09:25.187270+0000 mgr.y (mgr.14901) 7524 : cluster [DBG] pgmap v7531: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:26.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:26 smithi130 bash[51302]: debug 2023-08-28T02:09:26.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:26.328998+0000) 2023-08-28T02:09:26.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:26 smithi006 bash[27292]: cluster 2023-08-28T02:09:25.187270+0000 mgr.y (mgr.14901) 7524 : cluster [DBG] pgmap v7531: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:26.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:26 smithi006 bash[33918]: cluster 2023-08-28T02:09:25.187270+0000 mgr.y (mgr.14901) 7524 : cluster [DBG] pgmap v7531: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:27.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:27 smithi130 bash[51302]: debug 2023-08-28T02:09:27.328+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:27.329280+0000) 2023-08-28T02:09:28.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:28 smithi130 bash[51302]: debug 2023-08-28T02:09:28.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:28.329628+0000) 2023-08-28T02:09:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:28 smithi130 bash[27579]: cluster 2023-08-28T02:09:27.188030+0000 mgr.y (mgr.14901) 7525 : cluster [DBG] pgmap v7532: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:28 smithi130 bash[27579]: audit 2023-08-28T02:09:27.338497+0000 mon.a (mon.0) 4842 : audit [DBG] from='client.? 172.21.15.6:0/737068957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:28.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:28 smithi006 bash[33918]: cluster 2023-08-28T02:09:27.188030+0000 mgr.y (mgr.14901) 7525 : cluster [DBG] pgmap v7532: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:28 smithi006 bash[33918]: audit 2023-08-28T02:09:27.338497+0000 mon.a (mon.0) 4842 : audit [DBG] from='client.? 172.21.15.6:0/737068957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:28 smithi006 bash[27292]: cluster 2023-08-28T02:09:27.188030+0000 mgr.y (mgr.14901) 7525 : cluster [DBG] pgmap v7532: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:28 smithi006 bash[27292]: audit 2023-08-28T02:09:27.338497+0000 mon.a (mon.0) 4842 : audit [DBG] from='client.? 172.21.15.6:0/737068957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:29.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:29 smithi130 bash[51302]: debug 2023-08-28T02:09:29.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:29.329930+0000) 2023-08-28T02:09:30.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:30 smithi130 bash[51302]: debug 2023-08-28T02:09:30.328+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:30.330182+0000) 2023-08-28T02:09:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:30 smithi130 bash[27579]: cluster 2023-08-28T02:09:29.189310+0000 mgr.y (mgr.14901) 7526 : cluster [DBG] pgmap v7533: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:30 smithi130 bash[27579]: audit 2023-08-28T02:09:29.752530+0000 mon.a (mon.0) 4843 : audit [DBG] from='client.? 172.21.15.6:0/4228229966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:30.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:30 smithi006 bash[27292]: cluster 2023-08-28T02:09:29.189310+0000 mgr.y (mgr.14901) 7526 : cluster [DBG] pgmap v7533: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:30.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:30 smithi006 bash[27292]: audit 2023-08-28T02:09:29.752530+0000 mon.a (mon.0) 4843 : audit [DBG] from='client.? 172.21.15.6:0/4228229966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:30 smithi006 bash[33918]: cluster 2023-08-28T02:09:29.189310+0000 mgr.y (mgr.14901) 7526 : cluster [DBG] pgmap v7533: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:30 smithi006 bash[33918]: audit 2023-08-28T02:09:29.752530+0000 mon.a (mon.0) 4843 : audit [DBG] from='client.? 172.21.15.6:0/4228229966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:31.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:09:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:09:30] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T02:09:31.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:31 smithi130 bash[51302]: debug 2023-08-28T02:09:31.324+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:31.330525+0000) 2023-08-28T02:09:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:09:31.221Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:09:31.223Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:32.617 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:32 smithi130 bash[51302]: debug 2023-08-28T02:09:32.328+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:32.330840+0000) 2023-08-28T02:09:32.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:32 smithi130 bash[27579]: cluster 2023-08-28T02:09:31.189952+0000 mgr.y (mgr.14901) 7527 : cluster [DBG] pgmap v7534: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:32.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:32 smithi130 bash[27579]: audit 2023-08-28T02:09:32.170794+0000 mon.a (mon.0) 4844 : audit [DBG] from='client.? 172.21.15.6:0/1384182338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:32.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:32 smithi006 bash[27292]: cluster 2023-08-28T02:09:31.189952+0000 mgr.y (mgr.14901) 7527 : cluster [DBG] pgmap v7534: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:32.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:32 smithi006 bash[27292]: audit 2023-08-28T02:09:32.170794+0000 mon.a (mon.0) 4844 : audit [DBG] from='client.? 172.21.15.6:0/1384182338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:32.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:32 smithi006 bash[33918]: cluster 2023-08-28T02:09:31.189952+0000 mgr.y (mgr.14901) 7527 : cluster [DBG] pgmap v7534: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:32.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:32 smithi006 bash[33918]: audit 2023-08-28T02:09:32.170794+0000 mon.a (mon.0) 4844 : audit [DBG] from='client.? 172.21.15.6:0/1384182338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:09:32.789Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:09:32.791Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:33.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:33 smithi130 bash[51302]: debug 2023-08-28T02:09:33.328+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:33.331052+0000) 2023-08-28T02:09:33.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:33 smithi006 bash[33918]: cluster 2023-08-28T02:09:33.190606+0000 mgr.y (mgr.14901) 7528 : cluster [DBG] pgmap v7535: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:33.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:33 smithi006 bash[27292]: cluster 2023-08-28T02:09:33.190606+0000 mgr.y (mgr.14901) 7528 : cluster [DBG] pgmap v7535: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:33 smithi130 bash[27579]: cluster 2023-08-28T02:09:33.190606+0000 mgr.y (mgr.14901) 7528 : cluster [DBG] pgmap v7535: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:34.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:34 smithi130 bash[51302]: debug 2023-08-28T02:09:34.328+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:34.331274+0000) 2023-08-28T02:09:34.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:34 smithi006 bash[33918]: audit 2023-08-28T02:09:34.590030+0000 mon.a (mon.0) 4845 : audit [DBG] from='client.? 172.21.15.6:0/1727508421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:34.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:34 smithi006 bash[27292]: audit 2023-08-28T02:09:34.590030+0000 mon.a (mon.0) 4845 : audit [DBG] from='client.? 172.21.15.6:0/1727508421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:09:34.664Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:09:34.666Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:35.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:34 smithi130 bash[27579]: audit 2023-08-28T02:09:34.590030+0000 mon.a (mon.0) 4845 : audit [DBG] from='client.? 172.21.15.6:0/1727508421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:35.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:35 smithi130 bash[51302]: debug 2023-08-28T02:09:35.328+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:35.331544+0000) 2023-08-28T02:09:35.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:35 smithi006 bash[33918]: cluster 2023-08-28T02:09:35.191770+0000 mgr.y (mgr.14901) 7529 : cluster [DBG] pgmap v7536: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:35.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:35 smithi006 bash[27292]: cluster 2023-08-28T02:09:35.191770+0000 mgr.y (mgr.14901) 7529 : cluster [DBG] pgmap v7536: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:36.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:35 smithi130 bash[27579]: cluster 2023-08-28T02:09:35.191770+0000 mgr.y (mgr.14901) 7529 : cluster [DBG] pgmap v7536: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:36.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:36 smithi130 bash[51302]: debug 2023-08-28T02:09:36.328+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:36.331836+0000) 2023-08-28T02:09:37.330 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:37 smithi130 bash[27579]: audit 2023-08-28T02:09:37.003285+0000 mon.a (mon.0) 4846 : audit [DBG] from='client.? 172.21.15.6:0/2584124830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:37.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:37 smithi006 bash[27292]: audit 2023-08-28T02:09:37.003285+0000 mon.a (mon.0) 4846 : audit [DBG] from='client.? 172.21.15.6:0/2584124830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:37.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:37 smithi006 bash[33918]: audit 2023-08-28T02:09:37.003285+0000 mon.a (mon.0) 4846 : audit [DBG] from='client.? 172.21.15.6:0/2584124830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:37.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:37 smithi130 bash[51302]: debug 2023-08-28T02:09:37.332+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:37.332145+0000) 2023-08-28T02:09:38.330 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:38 smithi130 bash[27579]: cluster 2023-08-28T02:09:37.192442+0000 mgr.y (mgr.14901) 7530 : cluster [DBG] pgmap v7537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:38.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:38 smithi006 bash[27292]: cluster 2023-08-28T02:09:37.192442+0000 mgr.y (mgr.14901) 7530 : cluster [DBG] pgmap v7537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:38.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:38 smithi006 bash[33918]: cluster 2023-08-28T02:09:37.192442+0000 mgr.y (mgr.14901) 7530 : cluster [DBG] pgmap v7537: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:38.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:38 smithi130 bash[51302]: debug 2023-08-28T02:09:38.331+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:38.332416+0000) 2023-08-28T02:09:39.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:39 smithi130 bash[51302]: debug 2023-08-28T02:09:39.331+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:39.332776+0000) 2023-08-28T02:09:40.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:40 smithi130 bash[51302]: debug 2023-08-28T02:09:40.331+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:40.333058+0000) 2023-08-28T02:09:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:40 smithi130 bash[27579]: cluster 2023-08-28T02:09:39.193643+0000 mgr.y (mgr.14901) 7531 : cluster [DBG] pgmap v7538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:40 smithi130 bash[27579]: audit 2023-08-28T02:09:39.419317+0000 mon.a (mon.0) 4847 : audit [DBG] from='client.? 172.21.15.6:0/2074223302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:40.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:40 smithi006 bash[33918]: cluster 2023-08-28T02:09:39.193643+0000 mgr.y (mgr.14901) 7531 : cluster [DBG] pgmap v7538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:40.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:40 smithi006 bash[33918]: audit 2023-08-28T02:09:39.419317+0000 mon.a (mon.0) 4847 : audit [DBG] from='client.? 172.21.15.6:0/2074223302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:40 smithi006 bash[27292]: cluster 2023-08-28T02:09:39.193643+0000 mgr.y (mgr.14901) 7531 : cluster [DBG] pgmap v7538: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:40 smithi006 bash[27292]: audit 2023-08-28T02:09:39.419317+0000 mon.a (mon.0) 4847 : audit [DBG] from='client.? 172.21.15.6:0/2074223302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:09:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:09:40] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T02:09:41.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:41 smithi130 bash[51302]: debug 2023-08-28T02:09:41.327+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:41.333381+0000) 2023-08-28T02:09:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:09:41.222Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:09:41.224Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:42 smithi130 bash[27579]: cluster 2023-08-28T02:09:41.194233+0000 mgr.y (mgr.14901) 7532 : cluster [DBG] pgmap v7539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:42 smithi130 bash[27579]: audit 2023-08-28T02:09:41.835532+0000 mon.c (mon.2) 2962 : audit [DBG] from='client.? 172.21.15.6:0/194276895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:42.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:42 smithi130 bash[51302]: debug 2023-08-28T02:09:42.327+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:42.333700+0000) 2023-08-28T02:09:42.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:42 smithi006 bash[33918]: cluster 2023-08-28T02:09:41.194233+0000 mgr.y (mgr.14901) 7532 : cluster [DBG] pgmap v7539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:42.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:42 smithi006 bash[33918]: audit 2023-08-28T02:09:41.835532+0000 mon.c (mon.2) 2962 : audit [DBG] from='client.? 172.21.15.6:0/194276895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:42.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:42 smithi006 bash[27292]: cluster 2023-08-28T02:09:41.194233+0000 mgr.y (mgr.14901) 7532 : cluster [DBG] pgmap v7539: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:42.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:42 smithi006 bash[27292]: audit 2023-08-28T02:09:41.835532+0000 mon.c (mon.2) 2962 : audit [DBG] from='client.? 172.21.15.6:0/194276895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:43.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:09:42.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:43.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:09:42.793Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:43.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:43 smithi130 bash[51302]: debug 2023-08-28T02:09:43.331+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:43.333975+0000) 2023-08-28T02:09:43.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:43 smithi006 bash[27292]: cluster 2023-08-28T02:09:43.194983+0000 mgr.y (mgr.14901) 7533 : cluster [DBG] pgmap v7540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:43.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:43 smithi006 bash[33918]: cluster 2023-08-28T02:09:43.194983+0000 mgr.y (mgr.14901) 7533 : cluster [DBG] pgmap v7540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:44.114 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:43 smithi130 bash[27579]: cluster 2023-08-28T02:09:43.194983+0000 mgr.y (mgr.14901) 7533 : cluster [DBG] pgmap v7540: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:44.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:44 smithi130 bash[51302]: debug 2023-08-28T02:09:44.331+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:44.334247+0000) 2023-08-28T02:09:44.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:44 smithi006 bash[27292]: audit 2023-08-28T02:09:44.244309+0000 mon.a (mon.0) 4848 : audit [DBG] from='client.? 172.21.15.6:0/599327149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:44.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:44 smithi006 bash[33918]: audit 2023-08-28T02:09:44.244309+0000 mon.a (mon.0) 4848 : audit [DBG] from='client.? 172.21.15.6:0/599327149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:09:44.665Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:44.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:09:44.667Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:45.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:44 smithi130 bash[27579]: audit 2023-08-28T02:09:44.244309+0000 mon.a (mon.0) 4848 : audit [DBG] from='client.? 172.21.15.6:0/599327149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:45.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:45 smithi130 bash[51302]: debug 2023-08-28T02:09:45.331+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:45.334511+0000) 2023-08-28T02:09:45.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:45 smithi006 bash[27292]: cluster 2023-08-28T02:09:45.196232+0000 mgr.y (mgr.14901) 7534 : cluster [DBG] pgmap v7541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:45.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:45 smithi006 bash[33918]: cluster 2023-08-28T02:09:45.196232+0000 mgr.y (mgr.14901) 7534 : cluster [DBG] pgmap v7541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:46.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:45 smithi130 bash[27579]: cluster 2023-08-28T02:09:45.196232+0000 mgr.y (mgr.14901) 7534 : cluster [DBG] pgmap v7541: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:46.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:46 smithi130 bash[51302]: debug 2023-08-28T02:09:46.327+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:46.334790+0000) 2023-08-28T02:09:47.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:46 smithi130 bash[27579]: audit 2023-08-28T02:09:46.670935+0000 mon.a (mon.0) 4849 : audit [DBG] from='client.? 172.21.15.6:0/4037584693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:47.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:46 smithi006 bash[27292]: audit 2023-08-28T02:09:46.670935+0000 mon.a (mon.0) 4849 : audit [DBG] from='client.? 172.21.15.6:0/4037584693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:47.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:46 smithi006 bash[33918]: audit 2023-08-28T02:09:46.670935+0000 mon.a (mon.0) 4849 : audit [DBG] from='client.? 172.21.15.6:0/4037584693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:47.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:47 smithi130 bash[51302]: debug 2023-08-28T02:09:47.331+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:47.335046+0000) 2023-08-28T02:09:48.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:47 smithi130 bash[27579]: cluster 2023-08-28T02:09:47.197015+0000 mgr.y (mgr.14901) 7535 : cluster [DBG] pgmap v7542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:48.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:47 smithi006 bash[27292]: cluster 2023-08-28T02:09:47.197015+0000 mgr.y (mgr.14901) 7535 : cluster [DBG] pgmap v7542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:48.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:47 smithi006 bash[33918]: cluster 2023-08-28T02:09:47.197015+0000 mgr.y (mgr.14901) 7535 : cluster [DBG] pgmap v7542: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:48.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:48 smithi130 bash[51302]: debug 2023-08-28T02:09:48.331+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:48.335268+0000) 2023-08-28T02:09:49.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:49 smithi006 bash[33918]: audit 2023-08-28T02:09:49.084104+0000 mon.c (mon.2) 2963 : audit [DBG] from='client.? 172.21.15.6:0/3181675440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:49.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:49 smithi006 bash[27292]: audit 2023-08-28T02:09:49.084104+0000 mon.c (mon.2) 2963 : audit [DBG] from='client.? 172.21.15.6:0/3181675440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:49.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:49 smithi130 bash[27579]: audit 2023-08-28T02:09:49.084104+0000 mon.c (mon.2) 2963 : audit [DBG] from='client.? 172.21.15.6:0/3181675440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:49.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:49 smithi130 bash[51302]: debug 2023-08-28T02:09:49.331+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:49.335486+0000) 2023-08-28T02:09:50.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:50 smithi006 bash[27292]: cluster 2023-08-28T02:09:49.198226+0000 mgr.y (mgr.14901) 7536 : cluster [DBG] pgmap v7543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:50.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:50 smithi006 bash[33918]: cluster 2023-08-28T02:09:49.198226+0000 mgr.y (mgr.14901) 7536 : cluster [DBG] pgmap v7543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:50 smithi130 bash[27579]: cluster 2023-08-28T02:09:49.198226+0000 mgr.y (mgr.14901) 7536 : cluster [DBG] pgmap v7543: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:50.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:50 smithi130 bash[51302]: debug 2023-08-28T02:09:50.331+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:50.335760+0000) 2023-08-28T02:09:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:09:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:09:50] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:09:51.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:51 smithi130 bash[51302]: debug 2023-08-28T02:09:51.331+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:51.336054+0000) 2023-08-28T02:09:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:09:51.222Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:09:51.224Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:52 smithi130 bash[27579]: cluster 2023-08-28T02:09:51.198890+0000 mgr.y (mgr.14901) 7537 : cluster [DBG] pgmap v7544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:52 smithi130 bash[27579]: audit 2023-08-28T02:09:51.505197+0000 mon.a (mon.0) 4850 : audit [DBG] from='client.? 172.21.15.6:0/2404441335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:52.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:52 smithi130 bash[51302]: debug 2023-08-28T02:09:52.331+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:52.336352+0000) 2023-08-28T02:09:52.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:52 smithi006 bash[27292]: cluster 2023-08-28T02:09:51.198890+0000 mgr.y (mgr.14901) 7537 : cluster [DBG] pgmap v7544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:52.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:52 smithi006 bash[27292]: audit 2023-08-28T02:09:51.505197+0000 mon.a (mon.0) 4850 : audit [DBG] from='client.? 172.21.15.6:0/2404441335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:52.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:52 smithi006 bash[33918]: cluster 2023-08-28T02:09:51.198890+0000 mgr.y (mgr.14901) 7537 : cluster [DBG] pgmap v7544: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:52.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:52 smithi006 bash[33918]: audit 2023-08-28T02:09:51.505197+0000 mon.a (mon.0) 4850 : audit [DBG] from='client.? 172.21.15.6:0/2404441335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:09:52.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:09:52.793Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:53.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:53 smithi130 bash[51302]: debug 2023-08-28T02:09:53.331+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:53.336614+0000) 2023-08-28T02:09:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:53 smithi006 bash[33918]: cluster 2023-08-28T02:09:53.199702+0000 mgr.y (mgr.14901) 7538 : cluster [DBG] pgmap v7545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:53 smithi006 bash[33918]: audit 2023-08-28T02:09:53.387196+0000 mon.b (mon.1) 834 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:53.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:53 smithi006 bash[27292]: cluster 2023-08-28T02:09:53.199702+0000 mgr.y (mgr.14901) 7538 : cluster [DBG] pgmap v7545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:53.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:53 smithi006 bash[27292]: audit 2023-08-28T02:09:53.387196+0000 mon.b (mon.1) 834 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:53 smithi130 bash[27579]: cluster 2023-08-28T02:09:53.199702+0000 mgr.y (mgr.14901) 7538 : cluster [DBG] pgmap v7545: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:54.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:53 smithi130 bash[27579]: audit 2023-08-28T02:09:53.387196+0000 mon.b (mon.1) 834 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:54.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:54 smithi130 bash[51302]: debug 2023-08-28T02:09:54.331+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:54.336929+0000) 2023-08-28T02:09:54.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:54 smithi006 bash[33918]: audit 2023-08-28T02:09:53.923355+0000 mon.c (mon.2) 2964 : audit [DBG] from='client.? 172.21.15.6:0/932773086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:54.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:54 smithi006 bash[27292]: audit 2023-08-28T02:09:53.923355+0000 mon.c (mon.2) 2964 : audit [DBG] from='client.? 172.21.15.6:0/932773086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:09:54.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:09:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:09:54.670Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:09:55.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:54 smithi130 bash[27579]: audit 2023-08-28T02:09:53.923355+0000 mon.c (mon.2) 2964 : audit [DBG] from='client.? 172.21.15.6:0/932773086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:55.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:55 smithi130 bash[51302]: debug 2023-08-28T02:09:55.331+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:55.337230+0000) 2023-08-28T02:09:56.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:55 smithi130 bash[27579]: cluster 2023-08-28T02:09:55.200882+0000 mgr.y (mgr.14901) 7539 : cluster [DBG] pgmap v7546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:56.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:55 smithi006 bash[33918]: cluster 2023-08-28T02:09:55.200882+0000 mgr.y (mgr.14901) 7539 : cluster [DBG] pgmap v7546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:56.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:55 smithi006 bash[27292]: cluster 2023-08-28T02:09:55.200882+0000 mgr.y (mgr.14901) 7539 : cluster [DBG] pgmap v7546: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:56.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:56 smithi130 bash[51302]: debug 2023-08-28T02:09:56.331+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:56.337498+0000) 2023-08-28T02:09:57.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:56 smithi130 bash[27579]: audit 2023-08-28T02:09:56.337266+0000 mon.c (mon.2) 2965 : audit [DBG] from='client.? 172.21.15.6:0/3451809025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:57.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:56 smithi006 bash[33918]: audit 2023-08-28T02:09:56.337266+0000 mon.c (mon.2) 2965 : audit [DBG] from='client.? 172.21.15.6:0/3451809025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:57.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:56 smithi006 bash[27292]: audit 2023-08-28T02:09:56.337266+0000 mon.c (mon.2) 2965 : audit [DBG] from='client.? 172.21.15.6:0/3451809025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:57.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:57 smithi130 bash[51302]: debug 2023-08-28T02:09:57.331+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:57.337765+0000) 2023-08-28T02:09:58.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:57 smithi130 bash[27579]: cluster 2023-08-28T02:09:57.201486+0000 mgr.y (mgr.14901) 7540 : cluster [DBG] pgmap v7547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:58.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:57 smithi006 bash[27292]: cluster 2023-08-28T02:09:57.201486+0000 mgr.y (mgr.14901) 7540 : cluster [DBG] pgmap v7547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:58.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:57 smithi006 bash[33918]: cluster 2023-08-28T02:09:57.201486+0000 mgr.y (mgr.14901) 7540 : cluster [DBG] pgmap v7547: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:09:58.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:58 smithi130 bash[51302]: debug 2023-08-28T02:09:58.331+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:58.338080+0000) 2023-08-28T02:09:59.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:09:58 smithi130 bash[27579]: audit 2023-08-28T02:09:58.767368+0000 mon.a (mon.0) 4851 : audit [DBG] from='client.? 172.21.15.6:0/4279037964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:59.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:09:58 smithi006 bash[33918]: audit 2023-08-28T02:09:58.767368+0000 mon.a (mon.0) 4851 : audit [DBG] from='client.? 172.21.15.6:0/4279037964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:59.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:09:58 smithi006 bash[27292]: audit 2023-08-28T02:09:58.767368+0000 mon.a (mon.0) 4851 : audit [DBG] from='client.? 172.21.15.6:0/4279037964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:59.614 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:09:59 smithi130 bash[51302]: debug 2023-08-28T02:09:59.331+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:59.338306+0000) 2023-08-28T02:10:00.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:00 smithi130 bash[51302]: debug 2023-08-28T02:10:00.331+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:00.338563+0000) 2023-08-28T02:10:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:00 smithi130 bash[27579]: cluster 2023-08-28T02:09:59.202475+0000 mgr.y (mgr.14901) 7541 : cluster [DBG] pgmap v7548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:00 smithi130 bash[27579]: audit 2023-08-28T02:09:59.247201+0000 mon.a (mon.0) 4852 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:10:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:00 smithi130 bash[27579]: audit 2023-08-28T02:09:59.683253+0000 mon.a (mon.0) 4853 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:10:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:00 smithi130 bash[27579]: audit 2023-08-28T02:09:59.956448+0000 mon.b (mon.1) 835 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:10:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:00 smithi130 bash[27579]: audit 2023-08-28T02:09:59.958755+0000 mon.b (mon.1) 836 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:10:00.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:00 smithi130 bash[27579]: audit 2023-08-28T02:09:59.968145+0000 mon.a (mon.0) 4854 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:10:00.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:00 smithi130 bash[27579]: cluster 2023-08-28T02:10:00.000154+0000 mon.a (mon.0) 4855 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:10:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:00 smithi006 bash[27292]: cluster 2023-08-28T02:09:59.202475+0000 mgr.y (mgr.14901) 7541 : cluster [DBG] pgmap v7548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:00 smithi006 bash[27292]: audit 2023-08-28T02:09:59.247201+0000 mon.a (mon.0) 4852 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:10:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:00 smithi006 bash[27292]: audit 2023-08-28T02:09:59.683253+0000 mon.a (mon.0) 4853 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:10:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:00 smithi006 bash[27292]: audit 2023-08-28T02:09:59.956448+0000 mon.b (mon.1) 835 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:10:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:00 smithi006 bash[27292]: audit 2023-08-28T02:09:59.958755+0000 mon.b (mon.1) 836 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:10:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:00 smithi006 bash[27292]: audit 2023-08-28T02:09:59.968145+0000 mon.a (mon.0) 4854 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:10:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:00 smithi006 bash[27292]: cluster 2023-08-28T02:10:00.000154+0000 mon.a (mon.0) 4855 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:10:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:00 smithi006 bash[33918]: cluster 2023-08-28T02:09:59.202475+0000 mgr.y (mgr.14901) 7541 : cluster [DBG] pgmap v7548: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:00 smithi006 bash[33918]: audit 2023-08-28T02:09:59.247201+0000 mon.a (mon.0) 4852 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:10:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:00 smithi006 bash[33918]: audit 2023-08-28T02:09:59.683253+0000 mon.a (mon.0) 4853 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:10:00.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:00 smithi006 bash[33918]: audit 2023-08-28T02:09:59.956448+0000 mon.b (mon.1) 835 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:10:00.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:00 smithi006 bash[33918]: audit 2023-08-28T02:09:59.958755+0000 mon.b (mon.1) 836 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:10:00.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:00 smithi006 bash[33918]: audit 2023-08-28T02:09:59.968145+0000 mon.a (mon.0) 4854 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:10:00.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:00 smithi006 bash[33918]: cluster 2023-08-28T02:10:00.000154+0000 mon.a (mon.0) 4855 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:10:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:10:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:10:00] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:10:01.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:01 smithi130 bash[27579]: audit 2023-08-28T02:10:01.181447+0000 mon.c (mon.2) 2966 : audit [DBG] from='client.? 172.21.15.6:0/3390391323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:01.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:01 smithi130 bash[51302]: debug 2023-08-28T02:10:01.335+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:01.338906+0000) 2023-08-28T02:10:01.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:01 smithi006 bash[27292]: audit 2023-08-28T02:10:01.181447+0000 mon.c (mon.2) 2966 : audit [DBG] from='client.? 172.21.15.6:0/3390391323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:01.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:01 smithi006 bash[33918]: audit 2023-08-28T02:10:01.181447+0000 mon.c (mon.2) 2966 : audit [DBG] from='client.? 172.21.15.6:0/3390391323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:10:01.223Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:10:01.225Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:02 smithi130 bash[27579]: cluster 2023-08-28T02:10:01.203129+0000 mgr.y (mgr.14901) 7542 : cluster [DBG] pgmap v7549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:02.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:02 smithi130 bash[51302]: debug 2023-08-28T02:10:02.335+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:02.339183+0000) 2023-08-28T02:10:02.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:02 smithi006 bash[33918]: cluster 2023-08-28T02:10:01.203129+0000 mgr.y (mgr.14901) 7542 : cluster [DBG] pgmap v7549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:02.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:02 smithi006 bash[27292]: cluster 2023-08-28T02:10:01.203129+0000 mgr.y (mgr.14901) 7542 : cluster [DBG] pgmap v7549: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:10:02.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:10:02.793Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:03.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:03 smithi130 bash[51302]: debug 2023-08-28T02:10:03.335+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:03.339408+0000) 2023-08-28T02:10:03.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:03 smithi006 bash[33918]: cluster 2023-08-28T02:10:03.203800+0000 mgr.y (mgr.14901) 7543 : cluster [DBG] pgmap v7550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:03.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:03 smithi006 bash[33918]: audit 2023-08-28T02:10:03.596569+0000 mon.a (mon.0) 4856 : audit [DBG] from='client.? 172.21.15.6:0/2369944946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:03.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:03 smithi006 bash[27292]: cluster 2023-08-28T02:10:03.203800+0000 mgr.y (mgr.14901) 7543 : cluster [DBG] pgmap v7550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:03.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:03 smithi006 bash[27292]: audit 2023-08-28T02:10:03.596569+0000 mon.a (mon.0) 4856 : audit [DBG] from='client.? 172.21.15.6:0/2369944946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:03 smithi130 bash[27579]: cluster 2023-08-28T02:10:03.203800+0000 mgr.y (mgr.14901) 7543 : cluster [DBG] pgmap v7550: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:03 smithi130 bash[27579]: audit 2023-08-28T02:10:03.596569+0000 mon.a (mon.0) 4856 : audit [DBG] from='client.? 172.21.15.6:0/2369944946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:04.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:04 smithi130 bash[51302]: debug 2023-08-28T02:10:04.335+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:04.339654+0000) 2023-08-28T02:10:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:10:04.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:10:04.669Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:05.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:05 smithi130 bash[51302]: debug 2023-08-28T02:10:05.335+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:05.339916+0000) 2023-08-28T02:10:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:06 smithi130 bash[27579]: cluster 2023-08-28T02:10:05.204965+0000 mgr.y (mgr.14901) 7544 : cluster [DBG] pgmap v7551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:06 smithi130 bash[27579]: audit 2023-08-28T02:10:06.014815+0000 mon.c (mon.2) 2967 : audit [DBG] from='client.? 172.21.15.6:0/711835153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:06.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:06 smithi130 bash[51302]: debug 2023-08-28T02:10:06.335+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:06.340193+0000) 2023-08-28T02:10:06.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:06 smithi006 bash[27292]: cluster 2023-08-28T02:10:05.204965+0000 mgr.y (mgr.14901) 7544 : cluster [DBG] pgmap v7551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:06 smithi006 bash[27292]: audit 2023-08-28T02:10:06.014815+0000 mon.c (mon.2) 2967 : audit [DBG] from='client.? 172.21.15.6:0/711835153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:06 smithi006 bash[33918]: cluster 2023-08-28T02:10:05.204965+0000 mgr.y (mgr.14901) 7544 : cluster [DBG] pgmap v7551: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:06 smithi006 bash[33918]: audit 2023-08-28T02:10:06.014815+0000 mon.c (mon.2) 2967 : audit [DBG] from='client.? 172.21.15.6:0/711835153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:07.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:07 smithi130 bash[51302]: debug 2023-08-28T02:10:07.335+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:07.340466+0000) 2023-08-28T02:10:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:08 smithi130 bash[27579]: cluster 2023-08-28T02:10:07.205636+0000 mgr.y (mgr.14901) 7545 : cluster [DBG] pgmap v7552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:08.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:08 smithi130 bash[51302]: debug 2023-08-28T02:10:08.335+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:08.340754+0000) 2023-08-28T02:10:08.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:08 smithi006 bash[27292]: cluster 2023-08-28T02:10:07.205636+0000 mgr.y (mgr.14901) 7545 : cluster [DBG] pgmap v7552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:08 smithi006 bash[33918]: cluster 2023-08-28T02:10:07.205636+0000 mgr.y (mgr.14901) 7545 : cluster [DBG] pgmap v7552: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:09.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:09 smithi130 bash[27579]: audit 2023-08-28T02:10:08.427946+0000 mon.a (mon.0) 4857 : audit [DBG] from='client.? 172.21.15.6:0/2366426082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:09.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:09 smithi130 bash[51302]: debug 2023-08-28T02:10:09.339+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:09.341032+0000) 2023-08-28T02:10:09.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:09 smithi006 bash[27292]: audit 2023-08-28T02:10:08.427946+0000 mon.a (mon.0) 4857 : audit [DBG] from='client.? 172.21.15.6:0/2366426082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:09.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:09 smithi006 bash[33918]: audit 2023-08-28T02:10:08.427946+0000 mon.a (mon.0) 4857 : audit [DBG] from='client.? 172.21.15.6:0/2366426082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:10 smithi130 bash[27579]: cluster 2023-08-28T02:10:09.206823+0000 mgr.y (mgr.14901) 7546 : cluster [DBG] pgmap v7553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:10.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:10 smithi130 bash[51302]: debug 2023-08-28T02:10:10.335+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:10.341292+0000) 2023-08-28T02:10:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:10 smithi006 bash[33918]: cluster 2023-08-28T02:10:09.206823+0000 mgr.y (mgr.14901) 7546 : cluster [DBG] pgmap v7553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:10.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:10 smithi006 bash[27292]: cluster 2023-08-28T02:10:09.206823+0000 mgr.y (mgr.14901) 7546 : cluster [DBG] pgmap v7553: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:11.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:10:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:10:10] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:10:11.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:11 smithi130 bash[27579]: audit 2023-08-28T02:10:10.847856+0000 mon.a (mon.0) 4858 : audit [DBG] from='client.? 172.21.15.6:0/1307349263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:11.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:11 smithi130 bash[51302]: debug 2023-08-28T02:10:11.335+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:11.341582+0000) 2023-08-28T02:10:11.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:11 smithi006 bash[27292]: audit 2023-08-28T02:10:10.847856+0000 mon.a (mon.0) 4858 : audit [DBG] from='client.? 172.21.15.6:0/1307349263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:11.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:11 smithi006 bash[33918]: audit 2023-08-28T02:10:10.847856+0000 mon.a (mon.0) 4858 : audit [DBG] from='client.? 172.21.15.6:0/1307349263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:10:11.224Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:10:11.226Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:12 smithi130 bash[27579]: cluster 2023-08-28T02:10:11.207528+0000 mgr.y (mgr.14901) 7547 : cluster [DBG] pgmap v7554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:12.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:12 smithi130 bash[51302]: debug 2023-08-28T02:10:12.335+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:12.341827+0000) 2023-08-28T02:10:12.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:12 smithi006 bash[27292]: cluster 2023-08-28T02:10:11.207528+0000 mgr.y (mgr.14901) 7547 : cluster [DBG] pgmap v7554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:12.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:12 smithi006 bash[33918]: cluster 2023-08-28T02:10:11.207528+0000 mgr.y (mgr.14901) 7547 : cluster [DBG] pgmap v7554: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:10:12.793Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:10:12.795Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:13.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:13 smithi130 bash[51302]: debug 2023-08-28T02:10:13.339+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:13.342105+0000) 2023-08-28T02:10:13.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:13 smithi006 bash[33918]: cluster 2023-08-28T02:10:13.208151+0000 mgr.y (mgr.14901) 7548 : cluster [DBG] pgmap v7555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:13.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:13 smithi006 bash[33918]: audit 2023-08-28T02:10:13.270564+0000 mon.c (mon.2) 2968 : audit [DBG] from='client.? 172.21.15.6:0/3093359312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:13.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:13 smithi006 bash[27292]: cluster 2023-08-28T02:10:13.208151+0000 mgr.y (mgr.14901) 7548 : cluster [DBG] pgmap v7555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:13.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:13 smithi006 bash[27292]: audit 2023-08-28T02:10:13.270564+0000 mon.c (mon.2) 2968 : audit [DBG] from='client.? 172.21.15.6:0/3093359312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:13 smithi130 bash[27579]: cluster 2023-08-28T02:10:13.208151+0000 mgr.y (mgr.14901) 7548 : cluster [DBG] pgmap v7555: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:13 smithi130 bash[27579]: audit 2023-08-28T02:10:13.270564+0000 mon.c (mon.2) 2968 : audit [DBG] from='client.? 172.21.15.6:0/3093359312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:14.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:14 smithi130 bash[51302]: debug 2023-08-28T02:10:14.339+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:14.342423+0000) 2023-08-28T02:10:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:10:14.668Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:10:14.670Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:15.745 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:15 smithi130 bash[51302]: debug 2023-08-28T02:10:15.335+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:15.342686+0000) 2023-08-28T02:10:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:16 smithi130 bash[27579]: cluster 2023-08-28T02:10:15.209400+0000 mgr.y (mgr.14901) 7549 : cluster [DBG] pgmap v7556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:16 smithi130 bash[27579]: audit 2023-08-28T02:10:15.686539+0000 mon.c (mon.2) 2969 : audit [DBG] from='client.? 172.21.15.6:0/1047664169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:16.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:16 smithi130 bash[51302]: debug 2023-08-28T02:10:16.339+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:16.342987+0000) 2023-08-28T02:10:16.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:16 smithi006 bash[27292]: cluster 2023-08-28T02:10:15.209400+0000 mgr.y (mgr.14901) 7549 : cluster [DBG] pgmap v7556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:16.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:16 smithi006 bash[27292]: audit 2023-08-28T02:10:15.686539+0000 mon.c (mon.2) 2969 : audit [DBG] from='client.? 172.21.15.6:0/1047664169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:16 smithi006 bash[33918]: cluster 2023-08-28T02:10:15.209400+0000 mgr.y (mgr.14901) 7549 : cluster [DBG] pgmap v7556: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:16 smithi006 bash[33918]: audit 2023-08-28T02:10:15.686539+0000 mon.c (mon.2) 2969 : audit [DBG] from='client.? 172.21.15.6:0/1047664169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:17.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:17 smithi130 bash[51302]: debug 2023-08-28T02:10:17.339+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:17.343219+0000) 2023-08-28T02:10:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:18 smithi130 bash[27579]: cluster 2023-08-28T02:10:17.210132+0000 mgr.y (mgr.14901) 7550 : cluster [DBG] pgmap v7557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:18 smithi130 bash[27579]: audit 2023-08-28T02:10:18.107504+0000 mon.c (mon.2) 2970 : audit [DBG] from='client.? 172.21.15.6:0/2920590208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:18.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:18 smithi130 bash[51302]: debug 2023-08-28T02:10:18.339+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:18.343435+0000) 2023-08-28T02:10:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:18 smithi006 bash[27292]: cluster 2023-08-28T02:10:17.210132+0000 mgr.y (mgr.14901) 7550 : cluster [DBG] pgmap v7557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:18 smithi006 bash[27292]: audit 2023-08-28T02:10:18.107504+0000 mon.c (mon.2) 2970 : audit [DBG] from='client.? 172.21.15.6:0/2920590208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:18.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:18 smithi006 bash[33918]: cluster 2023-08-28T02:10:17.210132+0000 mgr.y (mgr.14901) 7550 : cluster [DBG] pgmap v7557: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:18.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:18 smithi006 bash[33918]: audit 2023-08-28T02:10:18.107504+0000 mon.c (mon.2) 2970 : audit [DBG] from='client.? 172.21.15.6:0/2920590208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:19.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:19 smithi130 bash[51302]: debug 2023-08-28T02:10:19.343+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:19.343769+0000) 2023-08-28T02:10:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:20 smithi130 bash[27579]: cluster 2023-08-28T02:10:19.211134+0000 mgr.y (mgr.14901) 7551 : cluster [DBG] pgmap v7558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:20.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:20 smithi130 bash[51302]: debug 2023-08-28T02:10:20.339+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:20.344032+0000) 2023-08-28T02:10:20.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:20 smithi006 bash[27292]: cluster 2023-08-28T02:10:19.211134+0000 mgr.y (mgr.14901) 7551 : cluster [DBG] pgmap v7558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:20 smithi006 bash[33918]: cluster 2023-08-28T02:10:19.211134+0000 mgr.y (mgr.14901) 7551 : cluster [DBG] pgmap v7558: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:10:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:10:20] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:10:21.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:21 smithi130 bash[27579]: audit 2023-08-28T02:10:20.534675+0000 mon.c (mon.2) 2971 : audit [DBG] from='client.? 172.21.15.6:0/644953899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:21.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:21 smithi130 bash[51302]: debug 2023-08-28T02:10:21.339+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:21.344333+0000) 2023-08-28T02:10:21.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:21 smithi006 bash[33918]: audit 2023-08-28T02:10:20.534675+0000 mon.c (mon.2) 2971 : audit [DBG] from='client.? 172.21.15.6:0/644953899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:10:21.225Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:10:21.227Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:21.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:21 smithi006 bash[27292]: audit 2023-08-28T02:10:20.534675+0000 mon.c (mon.2) 2971 : audit [DBG] from='client.? 172.21.15.6:0/644953899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:22 smithi130 bash[27579]: cluster 2023-08-28T02:10:21.211810+0000 mgr.y (mgr.14901) 7552 : cluster [DBG] pgmap v7559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:22.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:22 smithi130 bash[51302]: debug 2023-08-28T02:10:22.339+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:22.344563+0000) 2023-08-28T02:10:22.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:22 smithi006 bash[27292]: cluster 2023-08-28T02:10:21.211810+0000 mgr.y (mgr.14901) 7552 : cluster [DBG] pgmap v7559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:22.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:22 smithi006 bash[33918]: cluster 2023-08-28T02:10:21.211810+0000 mgr.y (mgr.14901) 7552 : cluster [DBG] pgmap v7559: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:10:22.793Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:10:22.795Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:23.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:23 smithi130 bash[51302]: debug 2023-08-28T02:10:23.343+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:23.344833+0000) 2023-08-28T02:10:23.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:23 smithi006 bash[33918]: audit 2023-08-28T02:10:22.959401+0000 mon.a (mon.0) 4859 : audit [DBG] from='client.? 172.21.15.6:0/2578226840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:23.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:23 smithi006 bash[33918]: cluster 2023-08-28T02:10:23.212509+0000 mgr.y (mgr.14901) 7553 : cluster [DBG] pgmap v7560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:23.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:23 smithi006 bash[27292]: audit 2023-08-28T02:10:22.959401+0000 mon.a (mon.0) 4859 : audit [DBG] from='client.? 172.21.15.6:0/2578226840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:23.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:23 smithi006 bash[27292]: cluster 2023-08-28T02:10:23.212509+0000 mgr.y (mgr.14901) 7553 : cluster [DBG] pgmap v7560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:24.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:23 smithi130 bash[27579]: audit 2023-08-28T02:10:22.959401+0000 mon.a (mon.0) 4859 : audit [DBG] from='client.? 172.21.15.6:0/2578226840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:24.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:23 smithi130 bash[27579]: cluster 2023-08-28T02:10:23.212509+0000 mgr.y (mgr.14901) 7553 : cluster [DBG] pgmap v7560: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:24.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:24 smithi130 bash[51302]: debug 2023-08-28T02:10:24.343+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:24.345117+0000) 2023-08-28T02:10:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:10:24.668Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:10:24.670Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:25.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:25 smithi130 bash[51302]: debug 2023-08-28T02:10:25.339+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:25.345396+0000) 2023-08-28T02:10:26.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:26 smithi130 bash[51302]: debug 2023-08-28T02:10:26.339+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:26.345662+0000) 2023-08-28T02:10:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:26 smithi130 bash[27579]: cluster 2023-08-28T02:10:25.213641+0000 mgr.y (mgr.14901) 7554 : cluster [DBG] pgmap v7561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:26 smithi130 bash[27579]: audit 2023-08-28T02:10:25.375131+0000 mon.a (mon.0) 4860 : audit [DBG] from='client.? 172.21.15.6:0/3222858796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:26.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:26 smithi006 bash[27292]: cluster 2023-08-28T02:10:25.213641+0000 mgr.y (mgr.14901) 7554 : cluster [DBG] pgmap v7561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:26.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:26 smithi006 bash[27292]: audit 2023-08-28T02:10:25.375131+0000 mon.a (mon.0) 4860 : audit [DBG] from='client.? 172.21.15.6:0/3222858796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:26 smithi006 bash[33918]: cluster 2023-08-28T02:10:25.213641+0000 mgr.y (mgr.14901) 7554 : cluster [DBG] pgmap v7561: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:26 smithi006 bash[33918]: audit 2023-08-28T02:10:25.375131+0000 mon.a (mon.0) 4860 : audit [DBG] from='client.? 172.21.15.6:0/3222858796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:27.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:27 smithi130 bash[51302]: debug 2023-08-28T02:10:27.339+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:27.345953+0000) 2023-08-28T02:10:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:28 smithi130 bash[27579]: cluster 2023-08-28T02:10:27.214320+0000 mgr.y (mgr.14901) 7555 : cluster [DBG] pgmap v7562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:28 smithi130 bash[27579]: audit 2023-08-28T02:10:27.791917+0000 mon.a (mon.0) 4861 : audit [DBG] from='client.? 172.21.15.6:0/1343306818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:28.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:28 smithi130 bash[51302]: debug 2023-08-28T02:10:28.339+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:28.346250+0000) 2023-08-28T02:10:28.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:28 smithi006 bash[33918]: cluster 2023-08-28T02:10:27.214320+0000 mgr.y (mgr.14901) 7555 : cluster [DBG] pgmap v7562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:28.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:28 smithi006 bash[33918]: audit 2023-08-28T02:10:27.791917+0000 mon.a (mon.0) 4861 : audit [DBG] from='client.? 172.21.15.6:0/1343306818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:28 smithi006 bash[27292]: cluster 2023-08-28T02:10:27.214320+0000 mgr.y (mgr.14901) 7555 : cluster [DBG] pgmap v7562: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:28 smithi006 bash[27292]: audit 2023-08-28T02:10:27.791917+0000 mon.a (mon.0) 4861 : audit [DBG] from='client.? 172.21.15.6:0/1343306818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:29.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:29 smithi130 bash[51302]: debug 2023-08-28T02:10:29.339+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:29.346478+0000) 2023-08-28T02:10:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:30 smithi130 bash[27579]: cluster 2023-08-28T02:10:29.215419+0000 mgr.y (mgr.14901) 7556 : cluster [DBG] pgmap v7563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:30 smithi130 bash[27579]: audit 2023-08-28T02:10:30.210636+0000 mon.c (mon.2) 2972 : audit [DBG] from='client.? 172.21.15.6:0/89832599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:30.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:30 smithi130 bash[51302]: debug 2023-08-28T02:10:30.339+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:30.346768+0000) 2023-08-28T02:10:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:30 smithi006 bash[33918]: cluster 2023-08-28T02:10:29.215419+0000 mgr.y (mgr.14901) 7556 : cluster [DBG] pgmap v7563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:30 smithi006 bash[33918]: audit 2023-08-28T02:10:30.210636+0000 mon.c (mon.2) 2972 : audit [DBG] from='client.? 172.21.15.6:0/89832599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:30 smithi006 bash[27292]: cluster 2023-08-28T02:10:29.215419+0000 mgr.y (mgr.14901) 7556 : cluster [DBG] pgmap v7563: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:30 smithi006 bash[27292]: audit 2023-08-28T02:10:30.210636+0000 mon.c (mon.2) 2972 : audit [DBG] from='client.? 172.21.15.6:0/89832599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:31.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:10:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:10:30] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T02:10:31.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:31 smithi130 bash[51302]: debug 2023-08-28T02:10:31.343+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:31.347065+0000) 2023-08-28T02:10:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:10:31.225Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:10:31.227Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:32 smithi130 bash[27579]: cluster 2023-08-28T02:10:31.216033+0000 mgr.y (mgr.14901) 7557 : cluster [DBG] pgmap v7564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:32.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:32 smithi130 bash[51302]: debug 2023-08-28T02:10:32.343+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:32.347318+0000) 2023-08-28T02:10:32.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:32 smithi006 bash[27292]: cluster 2023-08-28T02:10:31.216033+0000 mgr.y (mgr.14901) 7557 : cluster [DBG] pgmap v7564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:32.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:32 smithi006 bash[33918]: cluster 2023-08-28T02:10:31.216033+0000 mgr.y (mgr.14901) 7557 : cluster [DBG] pgmap v7564: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:10:32.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:10:32.795Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:33.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:33 smithi130 bash[51302]: debug 2023-08-28T02:10:33.346+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:33.347669+0000) 2023-08-28T02:10:33.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:33 smithi130 bash[27579]: audit 2023-08-28T02:10:32.629677+0000 mon.a (mon.0) 4862 : audit [DBG] from='client.? 172.21.15.6:0/3106778840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:33.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:33 smithi006 bash[33918]: audit 2023-08-28T02:10:32.629677+0000 mon.a (mon.0) 4862 : audit [DBG] from='client.? 172.21.15.6:0/3106778840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:33.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:33 smithi006 bash[27292]: audit 2023-08-28T02:10:32.629677+0000 mon.a (mon.0) 4862 : audit [DBG] from='client.? 172.21.15.6:0/3106778840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:34.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:34 smithi130 bash[51302]: debug 2023-08-28T02:10:34.346+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:34.348016+0000) 2023-08-28T02:10:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:34 smithi130 bash[27579]: cluster 2023-08-28T02:10:33.216678+0000 mgr.y (mgr.14901) 7558 : cluster [DBG] pgmap v7565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:34.669 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:34 smithi006 bash[27292]: cluster 2023-08-28T02:10:33.216678+0000 mgr.y (mgr.14901) 7558 : cluster [DBG] pgmap v7565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:34.670 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:34 smithi006 bash[33918]: cluster 2023-08-28T02:10:33.216678+0000 mgr.y (mgr.14901) 7558 : cluster [DBG] pgmap v7565: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:10:34.669Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:10:34.671Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:35.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:35 smithi130 bash[27579]: audit 2023-08-28T02:10:35.042583+0000 mon.c (mon.2) 2973 : audit [DBG] from='client.? 172.21.15.6:0/79625385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:35.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:35 smithi130 bash[51302]: debug 2023-08-28T02:10:35.342+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:35.348284+0000) 2023-08-28T02:10:35.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:35 smithi006 bash[33918]: audit 2023-08-28T02:10:35.042583+0000 mon.c (mon.2) 2973 : audit [DBG] from='client.? 172.21.15.6:0/79625385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:35.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:35 smithi006 bash[27292]: audit 2023-08-28T02:10:35.042583+0000 mon.c (mon.2) 2973 : audit [DBG] from='client.? 172.21.15.6:0/79625385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:36 smithi130 bash[27579]: cluster 2023-08-28T02:10:35.217937+0000 mgr.y (mgr.14901) 7559 : cluster [DBG] pgmap v7566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:36.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:36 smithi130 bash[51302]: debug 2023-08-28T02:10:36.342+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:36.348621+0000) 2023-08-28T02:10:36.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:36 smithi006 bash[33918]: cluster 2023-08-28T02:10:35.217937+0000 mgr.y (mgr.14901) 7559 : cluster [DBG] pgmap v7566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:36.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:36 smithi006 bash[27292]: cluster 2023-08-28T02:10:35.217937+0000 mgr.y (mgr.14901) 7559 : cluster [DBG] pgmap v7566: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:37.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:37 smithi130 bash[51302]: debug 2023-08-28T02:10:37.346+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:37.348904+0000) 2023-08-28T02:10:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:38 smithi130 bash[27579]: cluster 2023-08-28T02:10:37.218438+0000 mgr.y (mgr.14901) 7560 : cluster [DBG] pgmap v7567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:38 smithi130 bash[27579]: audit 2023-08-28T02:10:37.465503+0000 mon.c (mon.2) 2974 : audit [DBG] from='client.? 172.21.15.6:0/657579644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:38.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:38 smithi130 bash[51302]: debug 2023-08-28T02:10:38.342+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:38.349157+0000) 2023-08-28T02:10:38.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:38 smithi006 bash[27292]: cluster 2023-08-28T02:10:37.218438+0000 mgr.y (mgr.14901) 7560 : cluster [DBG] pgmap v7567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:38.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:38 smithi006 bash[27292]: audit 2023-08-28T02:10:37.465503+0000 mon.c (mon.2) 2974 : audit [DBG] from='client.? 172.21.15.6:0/657579644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:38.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:38 smithi006 bash[33918]: cluster 2023-08-28T02:10:37.218438+0000 mgr.y (mgr.14901) 7560 : cluster [DBG] pgmap v7567: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:38.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:38 smithi006 bash[33918]: audit 2023-08-28T02:10:37.465503+0000 mon.c (mon.2) 2974 : audit [DBG] from='client.? 172.21.15.6:0/657579644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:39.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:39 smithi130 bash[51302]: debug 2023-08-28T02:10:39.346+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:39.349431+0000) 2023-08-28T02:10:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:40 smithi130 bash[27579]: cluster 2023-08-28T02:10:39.219443+0000 mgr.y (mgr.14901) 7561 : cluster [DBG] pgmap v7568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:40 smithi130 bash[27579]: audit 2023-08-28T02:10:39.881828+0000 mon.c (mon.2) 2975 : audit [DBG] from='client.? 172.21.15.6:0/956883664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:40.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:40 smithi130 bash[51302]: debug 2023-08-28T02:10:40.346+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:40.349700+0000) 2023-08-28T02:10:40.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:40 smithi006 bash[27292]: cluster 2023-08-28T02:10:39.219443+0000 mgr.y (mgr.14901) 7561 : cluster [DBG] pgmap v7568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:40.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:40 smithi006 bash[27292]: audit 2023-08-28T02:10:39.881828+0000 mon.c (mon.2) 2975 : audit [DBG] from='client.? 172.21.15.6:0/956883664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:40 smithi006 bash[33918]: cluster 2023-08-28T02:10:39.219443+0000 mgr.y (mgr.14901) 7561 : cluster [DBG] pgmap v7568: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:40 smithi006 bash[33918]: audit 2023-08-28T02:10:39.881828+0000 mon.c (mon.2) 2975 : audit [DBG] from='client.? 172.21.15.6:0/956883664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:10:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:10:40] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T02:10:41.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:41 smithi130 bash[51302]: debug 2023-08-28T02:10:41.346+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:41.350008+0000) 2023-08-28T02:10:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:10:41.226Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:10:41.227Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:42 smithi130 bash[27579]: cluster 2023-08-28T02:10:41.220034+0000 mgr.y (mgr.14901) 7562 : cluster [DBG] pgmap v7569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:42 smithi130 bash[27579]: audit 2023-08-28T02:10:42.297155+0000 mon.a (mon.0) 4863 : audit [DBG] from='client.? 172.21.15.6:0/1270533819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:42.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:42 smithi130 bash[51302]: debug 2023-08-28T02:10:42.346+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:42.350391+0000) 2023-08-28T02:10:42.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:42 smithi006 bash[33918]: cluster 2023-08-28T02:10:41.220034+0000 mgr.y (mgr.14901) 7562 : cluster [DBG] pgmap v7569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:42.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:42 smithi006 bash[33918]: audit 2023-08-28T02:10:42.297155+0000 mon.a (mon.0) 4863 : audit [DBG] from='client.? 172.21.15.6:0/1270533819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:42.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:42 smithi006 bash[27292]: cluster 2023-08-28T02:10:41.220034+0000 mgr.y (mgr.14901) 7562 : cluster [DBG] pgmap v7569: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:42.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:42 smithi006 bash[27292]: audit 2023-08-28T02:10:42.297155+0000 mon.a (mon.0) 4863 : audit [DBG] from='client.? 172.21.15.6:0/1270533819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:10:42.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:10:42.796Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:43.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:43 smithi130 bash[51302]: debug 2023-08-28T02:10:43.346+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:43.350688+0000) 2023-08-28T02:10:44.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:43 smithi130 bash[27579]: cluster 2023-08-28T02:10:43.220702+0000 mgr.y (mgr.14901) 7563 : cluster [DBG] pgmap v7570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:44.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:43 smithi006 bash[27292]: cluster 2023-08-28T02:10:43.220702+0000 mgr.y (mgr.14901) 7563 : cluster [DBG] pgmap v7570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:44.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:43 smithi006 bash[33918]: cluster 2023-08-28T02:10:43.220702+0000 mgr.y (mgr.14901) 7563 : cluster [DBG] pgmap v7570: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:44.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:44 smithi130 bash[51302]: debug 2023-08-28T02:10:44.350+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:44.350992+0000) 2023-08-28T02:10:44.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:44 smithi006 bash[33918]: audit 2023-08-28T02:10:44.709160+0000 mon.a (mon.0) 4864 : audit [DBG] from='client.? 172.21.15.6:0/1660726662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:44.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:44 smithi006 bash[27292]: audit 2023-08-28T02:10:44.709160+0000 mon.a (mon.0) 4864 : audit [DBG] from='client.? 172.21.15.6:0/1660726662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:10:44.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:10:44.673Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:45.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:44 smithi130 bash[27579]: audit 2023-08-28T02:10:44.709160+0000 mon.a (mon.0) 4864 : audit [DBG] from='client.? 172.21.15.6:0/1660726662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:45.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:45 smithi130 bash[51302]: debug 2023-08-28T02:10:45.350+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:45.351348+0000) 2023-08-28T02:10:46.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:45 smithi130 bash[27579]: cluster 2023-08-28T02:10:45.221801+0000 mgr.y (mgr.14901) 7564 : cluster [DBG] pgmap v7571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:46.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:45 smithi006 bash[33918]: cluster 2023-08-28T02:10:45.221801+0000 mgr.y (mgr.14901) 7564 : cluster [DBG] pgmap v7571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:46.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:45 smithi006 bash[27292]: cluster 2023-08-28T02:10:45.221801+0000 mgr.y (mgr.14901) 7564 : cluster [DBG] pgmap v7571: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:46.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:46 smithi130 bash[51302]: debug 2023-08-28T02:10:46.350+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:46.351635+0000) 2023-08-28T02:10:47.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:47 smithi006 bash[33918]: audit 2023-08-28T02:10:47.122183+0000 mon.a (mon.0) 4865 : audit [DBG] from='client.? 172.21.15.6:0/2078029786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:47.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:47 smithi006 bash[27292]: audit 2023-08-28T02:10:47.122183+0000 mon.a (mon.0) 4865 : audit [DBG] from='client.? 172.21.15.6:0/2078029786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:47.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:47 smithi130 bash[27579]: audit 2023-08-28T02:10:47.122183+0000 mon.a (mon.0) 4865 : audit [DBG] from='client.? 172.21.15.6:0/2078029786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:47.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:47 smithi130 bash[51302]: debug 2023-08-28T02:10:47.350+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:47.351912+0000) 2023-08-28T02:10:48.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:48 smithi006 bash[33918]: cluster 2023-08-28T02:10:47.222450+0000 mgr.y (mgr.14901) 7565 : cluster [DBG] pgmap v7572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:48.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:48 smithi006 bash[27292]: cluster 2023-08-28T02:10:47.222450+0000 mgr.y (mgr.14901) 7565 : cluster [DBG] pgmap v7572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:48 smithi130 bash[27579]: cluster 2023-08-28T02:10:47.222450+0000 mgr.y (mgr.14901) 7565 : cluster [DBG] pgmap v7572: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:48.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:48 smithi130 bash[51302]: debug 2023-08-28T02:10:48.350+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:48.352215+0000) 2023-08-28T02:10:49.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:49 smithi130 bash[51302]: debug 2023-08-28T02:10:49.350+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:49.352491+0000) 2023-08-28T02:10:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:50 smithi130 bash[27579]: cluster 2023-08-28T02:10:49.223514+0000 mgr.y (mgr.14901) 7566 : cluster [DBG] pgmap v7573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:50 smithi130 bash[27579]: audit 2023-08-28T02:10:49.536967+0000 mon.a (mon.0) 4866 : audit [DBG] from='client.? 172.21.15.6:0/975430942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:50.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:50 smithi130 bash[51302]: debug 2023-08-28T02:10:50.350+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:50.352802+0000) 2023-08-28T02:10:50.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:50 smithi006 bash[27292]: cluster 2023-08-28T02:10:49.223514+0000 mgr.y (mgr.14901) 7566 : cluster [DBG] pgmap v7573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:50 smithi006 bash[27292]: audit 2023-08-28T02:10:49.536967+0000 mon.a (mon.0) 4866 : audit [DBG] from='client.? 172.21.15.6:0/975430942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:50.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:50 smithi006 bash[33918]: cluster 2023-08-28T02:10:49.223514+0000 mgr.y (mgr.14901) 7566 : cluster [DBG] pgmap v7573: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:50.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:50 smithi006 bash[33918]: audit 2023-08-28T02:10:49.536967+0000 mon.a (mon.0) 4866 : audit [DBG] from='client.? 172.21.15.6:0/975430942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:10:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:10:50] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T02:10:51.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:51 smithi130 bash[51302]: debug 2023-08-28T02:10:51.350+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:51.353096+0000) 2023-08-28T02:10:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:10:51.226Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:10:51.228Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:52 smithi130 bash[27579]: cluster 2023-08-28T02:10:51.224189+0000 mgr.y (mgr.14901) 7567 : cluster [DBG] pgmap v7574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:52 smithi130 bash[27579]: audit 2023-08-28T02:10:51.948911+0000 mon.a (mon.0) 4867 : audit [DBG] from='client.? 172.21.15.6:0/2513668052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:52.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:52 smithi130 bash[51302]: debug 2023-08-28T02:10:52.350+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:52.353432+0000) 2023-08-28T02:10:52.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:52 smithi006 bash[33918]: cluster 2023-08-28T02:10:51.224189+0000 mgr.y (mgr.14901) 7567 : cluster [DBG] pgmap v7574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:52.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:52 smithi006 bash[33918]: audit 2023-08-28T02:10:51.948911+0000 mon.a (mon.0) 4867 : audit [DBG] from='client.? 172.21.15.6:0/2513668052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:52.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:52 smithi006 bash[27292]: cluster 2023-08-28T02:10:51.224189+0000 mgr.y (mgr.14901) 7567 : cluster [DBG] pgmap v7574: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:52.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:52 smithi006 bash[27292]: audit 2023-08-28T02:10:51.948911+0000 mon.a (mon.0) 4867 : audit [DBG] from='client.? 172.21.15.6:0/2513668052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:10:52.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:10:52.797Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:53.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:53 smithi130 bash[51302]: debug 2023-08-28T02:10:53.350+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:53.353674+0000) 2023-08-28T02:10:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:53 smithi130 bash[27579]: cluster 2023-08-28T02:10:53.224888+0000 mgr.y (mgr.14901) 7568 : cluster [DBG] pgmap v7575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:54.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:53 smithi006 bash[27292]: cluster 2023-08-28T02:10:53.224888+0000 mgr.y (mgr.14901) 7568 : cluster [DBG] pgmap v7575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:54.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:53 smithi006 bash[33918]: cluster 2023-08-28T02:10:53.224888+0000 mgr.y (mgr.14901) 7568 : cluster [DBG] pgmap v7575: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:54.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:54 smithi130 bash[51302]: debug 2023-08-28T02:10:54.350+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:54.353929+0000) 2023-08-28T02:10:54.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:54 smithi006 bash[33918]: audit 2023-08-28T02:10:54.354464+0000 mon.c (mon.2) 2976 : audit [DBG] from='client.? 172.21.15.6:0/1809903057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:54.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:54 smithi006 bash[27292]: audit 2023-08-28T02:10:54.354464+0000 mon.c (mon.2) 2976 : audit [DBG] from='client.? 172.21.15.6:0/1809903057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:10:54.671Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:10:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:10:54.674Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:10:55.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:54 smithi130 bash[27579]: audit 2023-08-28T02:10:54.354464+0000 mon.c (mon.2) 2976 : audit [DBG] from='client.? 172.21.15.6:0/1809903057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:55.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:55 smithi130 bash[51302]: debug 2023-08-28T02:10:55.350+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:55.354198+0000) 2023-08-28T02:10:56.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:55 smithi130 bash[27579]: cluster 2023-08-28T02:10:55.226026+0000 mgr.y (mgr.14901) 7569 : cluster [DBG] pgmap v7576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:56.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:55 smithi006 bash[27292]: cluster 2023-08-28T02:10:55.226026+0000 mgr.y (mgr.14901) 7569 : cluster [DBG] pgmap v7576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:56.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:55 smithi006 bash[33918]: cluster 2023-08-28T02:10:55.226026+0000 mgr.y (mgr.14901) 7569 : cluster [DBG] pgmap v7576: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:56.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:56 smithi130 bash[51302]: debug 2023-08-28T02:10:56.350+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:56.354471+0000) 2023-08-28T02:10:57.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:56 smithi130 bash[27579]: audit 2023-08-28T02:10:56.771616+0000 mon.a (mon.0) 4868 : audit [DBG] from='client.? 172.21.15.6:0/4256805877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:57.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:56 smithi006 bash[27292]: audit 2023-08-28T02:10:56.771616+0000 mon.a (mon.0) 4868 : audit [DBG] from='client.? 172.21.15.6:0/4256805877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:57.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:56 smithi006 bash[33918]: audit 2023-08-28T02:10:56.771616+0000 mon.a (mon.0) 4868 : audit [DBG] from='client.? 172.21.15.6:0/4256805877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:57.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:57 smithi130 bash[51302]: debug 2023-08-28T02:10:57.354+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:57.354739+0000) 2023-08-28T02:10:58.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:57 smithi130 bash[27579]: cluster 2023-08-28T02:10:57.226747+0000 mgr.y (mgr.14901) 7570 : cluster [DBG] pgmap v7577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:58.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:57 smithi006 bash[33918]: cluster 2023-08-28T02:10:57.226747+0000 mgr.y (mgr.14901) 7570 : cluster [DBG] pgmap v7577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:58.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:57 smithi006 bash[27292]: cluster 2023-08-28T02:10:57.226747+0000 mgr.y (mgr.14901) 7570 : cluster [DBG] pgmap v7577: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:10:58.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:58 smithi130 bash[51302]: debug 2023-08-28T02:10:58.354+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:58.355026+0000) 2023-08-28T02:10:59.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:10:59 smithi130 bash[27579]: audit 2023-08-28T02:10:59.191153+0000 mon.a (mon.0) 4869 : audit [DBG] from='client.? 172.21.15.6:0/3055805063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:59.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:10:59 smithi130 bash[51302]: debug 2023-08-28T02:10:59.354+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:59.355256+0000) 2023-08-28T02:10:59.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:10:59 smithi006 bash[33918]: audit 2023-08-28T02:10:59.191153+0000 mon.a (mon.0) 4869 : audit [DBG] from='client.? 172.21.15.6:0/3055805063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:59.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:10:59 smithi006 bash[27292]: audit 2023-08-28T02:10:59.191153+0000 mon.a (mon.0) 4869 : audit [DBG] from='client.? 172.21.15.6:0/3055805063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:00.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:00 smithi130 bash[51302]: debug 2023-08-28T02:11:00.354+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:00.355484+0000) 2023-08-28T02:11:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:00 smithi130 bash[27579]: cluster 2023-08-28T02:10:59.227825+0000 mgr.y (mgr.14901) 7571 : cluster [DBG] pgmap v7578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:00 smithi130 bash[27579]: audit 2023-08-28T02:10:59.980756+0000 mon.b (mon.1) 837 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:11:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:00 smithi006 bash[33918]: cluster 2023-08-28T02:10:59.227825+0000 mgr.y (mgr.14901) 7571 : cluster [DBG] pgmap v7578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:00 smithi006 bash[33918]: audit 2023-08-28T02:10:59.980756+0000 mon.b (mon.1) 837 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:11:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:00 smithi006 bash[27292]: cluster 2023-08-28T02:10:59.227825+0000 mgr.y (mgr.14901) 7571 : cluster [DBG] pgmap v7578: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:00 smithi006 bash[27292]: audit 2023-08-28T02:10:59.980756+0000 mon.b (mon.1) 837 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:11:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:11:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:11:00] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T02:11:01.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:01 smithi130 bash[51302]: debug 2023-08-28T02:11:01.354+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:01.355803+0000) 2023-08-28T02:11:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:11:01.227Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:11:01.229Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:02 smithi130 bash[27579]: cluster 2023-08-28T02:11:01.228624+0000 mgr.y (mgr.14901) 7572 : cluster [DBG] pgmap v7579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:02 smithi130 bash[27579]: audit 2023-08-28T02:11:01.602723+0000 mon.c (mon.2) 2977 : audit [DBG] from='client.? 172.21.15.6:0/116500614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:02.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:02 smithi130 bash[51302]: debug 2023-08-28T02:11:02.354+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:02.356016+0000) 2023-08-28T02:11:02.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:02 smithi006 bash[33918]: cluster 2023-08-28T02:11:01.228624+0000 mgr.y (mgr.14901) 7572 : cluster [DBG] pgmap v7579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:02.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:02 smithi006 bash[33918]: audit 2023-08-28T02:11:01.602723+0000 mon.c (mon.2) 2977 : audit [DBG] from='client.? 172.21.15.6:0/116500614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:02.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:02 smithi006 bash[27292]: cluster 2023-08-28T02:11:01.228624+0000 mgr.y (mgr.14901) 7572 : cluster [DBG] pgmap v7579: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:02.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:02 smithi006 bash[27292]: audit 2023-08-28T02:11:01.602723+0000 mon.c (mon.2) 2977 : audit [DBG] from='client.? 172.21.15.6:0/116500614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:11:02.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:11:02.797Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:03.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:03 smithi130 bash[51302]: debug 2023-08-28T02:11:03.354+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:03.356274+0000) 2023-08-28T02:11:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:03 smithi130 bash[27579]: cluster 2023-08-28T02:11:03.229340+0000 mgr.y (mgr.14901) 7573 : cluster [DBG] pgmap v7580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:04.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:03 smithi006 bash[27292]: cluster 2023-08-28T02:11:03.229340+0000 mgr.y (mgr.14901) 7573 : cluster [DBG] pgmap v7580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:04.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:03 smithi006 bash[33918]: cluster 2023-08-28T02:11:03.229340+0000 mgr.y (mgr.14901) 7573 : cluster [DBG] pgmap v7580: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:04.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:04 smithi130 bash[51302]: debug 2023-08-28T02:11:04.354+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:04.356463+0000) 2023-08-28T02:11:04.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:04 smithi006 bash[27292]: audit 2023-08-28T02:11:04.017958+0000 mon.a (mon.0) 4870 : audit [DBG] from='client.? 172.21.15.6:0/3730811510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:04.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:04 smithi006 bash[33918]: audit 2023-08-28T02:11:04.017958+0000 mon.a (mon.0) 4870 : audit [DBG] from='client.? 172.21.15.6:0/3730811510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:11:04.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:11:04.674Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:05.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:04 smithi130 bash[27579]: audit 2023-08-28T02:11:04.017958+0000 mon.a (mon.0) 4870 : audit [DBG] from='client.? 172.21.15.6:0/3730811510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:05.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:05 smithi130 bash[51302]: debug 2023-08-28T02:11:05.354+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:05.356656+0000) 2023-08-28T02:11:06.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:05 smithi130 bash[27579]: cluster 2023-08-28T02:11:05.229958+0000 mgr.y (mgr.14901) 7574 : cluster [DBG] pgmap v7581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:06.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:05 smithi006 bash[33918]: cluster 2023-08-28T02:11:05.229958+0000 mgr.y (mgr.14901) 7574 : cluster [DBG] pgmap v7581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:06.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:05 smithi006 bash[27292]: cluster 2023-08-28T02:11:05.229958+0000 mgr.y (mgr.14901) 7574 : cluster [DBG] pgmap v7581: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:06.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:06 smithi130 bash[51302]: debug 2023-08-28T02:11:06.354+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:06.356887+0000) 2023-08-28T02:11:07.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:06 smithi006 bash[33918]: audit 2023-08-28T02:11:05.937354+0000 mon.a (mon.0) 4871 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:11:07.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:06 smithi006 bash[33918]: audit 2023-08-28T02:11:06.372112+0000 mon.a (mon.0) 4872 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:11:07.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:06 smithi006 bash[33918]: audit 2023-08-28T02:11:06.435144+0000 mon.c (mon.2) 2978 : audit [DBG] from='client.? 172.21.15.6:0/2536066533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:07.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:06 smithi006 bash[33918]: audit 2023-08-28T02:11:06.640603+0000 mon.b (mon.1) 838 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:07.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:06 smithi006 bash[33918]: audit 2023-08-28T02:11:06.643056+0000 mon.b (mon.1) 839 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:11:07.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:06 smithi006 bash[33918]: audit 2023-08-28T02:11:06.651231+0000 mon.a (mon.0) 4873 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:11:07.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:06 smithi006 bash[27292]: audit 2023-08-28T02:11:05.937354+0000 mon.a (mon.0) 4871 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:11:07.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:06 smithi006 bash[27292]: audit 2023-08-28T02:11:06.372112+0000 mon.a (mon.0) 4872 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:11:07.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:06 smithi006 bash[27292]: audit 2023-08-28T02:11:06.435144+0000 mon.c (mon.2) 2978 : audit [DBG] from='client.? 172.21.15.6:0/2536066533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:06 smithi006 bash[27292]: audit 2023-08-28T02:11:06.640603+0000 mon.b (mon.1) 838 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:06 smithi006 bash[27292]: audit 2023-08-28T02:11:06.643056+0000 mon.b (mon.1) 839 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:11:07.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:06 smithi006 bash[27292]: audit 2023-08-28T02:11:06.651231+0000 mon.a (mon.0) 4873 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:11:07.355 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:06 smithi130 bash[27579]: audit 2023-08-28T02:11:05.937354+0000 mon.a (mon.0) 4871 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:11:07.355 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:06 smithi130 bash[27579]: audit 2023-08-28T02:11:06.372112+0000 mon.a (mon.0) 4872 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:11:07.356 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:06 smithi130 bash[27579]: audit 2023-08-28T02:11:06.435144+0000 mon.c (mon.2) 2978 : audit [DBG] from='client.? 172.21.15.6:0/2536066533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:07.356 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:06 smithi130 bash[27579]: audit 2023-08-28T02:11:06.640603+0000 mon.b (mon.1) 838 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:07.356 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:06 smithi130 bash[27579]: audit 2023-08-28T02:11:06.643056+0000 mon.b (mon.1) 839 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:11:07.356 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:06 smithi130 bash[27579]: audit 2023-08-28T02:11:06.651231+0000 mon.a (mon.0) 4873 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:11:07.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:07 smithi130 bash[51302]: debug 2023-08-28T02:11:07.354+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:07.357161+0000) 2023-08-28T02:11:08.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:07 smithi006 bash[33918]: cluster 2023-08-28T02:11:07.230598+0000 mgr.y (mgr.14901) 7575 : cluster [DBG] pgmap v7582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:08.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:07 smithi006 bash[27292]: cluster 2023-08-28T02:11:07.230598+0000 mgr.y (mgr.14901) 7575 : cluster [DBG] pgmap v7582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:08.355 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:07 smithi130 bash[27579]: cluster 2023-08-28T02:11:07.230598+0000 mgr.y (mgr.14901) 7575 : cluster [DBG] pgmap v7582: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:08.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:08 smithi130 bash[51302]: debug 2023-08-28T02:11:08.354+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:08.357433+0000) 2023-08-28T02:11:09.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:08 smithi006 bash[33918]: audit 2023-08-28T02:11:08.846171+0000 mon.a (mon.0) 4874 : audit [DBG] from='client.? 172.21.15.6:0/1066065642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:09.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:08 smithi006 bash[27292]: audit 2023-08-28T02:11:08.846171+0000 mon.a (mon.0) 4874 : audit [DBG] from='client.? 172.21.15.6:0/1066065642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:09.355 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:08 smithi130 bash[27579]: audit 2023-08-28T02:11:08.846171+0000 mon.a (mon.0) 4874 : audit [DBG] from='client.? 172.21.15.6:0/1066065642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:09.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:09 smithi130 bash[51302]: debug 2023-08-28T02:11:09.354+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:09.357642+0000) 2023-08-28T02:11:10.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:09 smithi006 bash[27292]: cluster 2023-08-28T02:11:09.231683+0000 mgr.y (mgr.14901) 7576 : cluster [DBG] pgmap v7583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:10.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:09 smithi006 bash[33918]: cluster 2023-08-28T02:11:09.231683+0000 mgr.y (mgr.14901) 7576 : cluster [DBG] pgmap v7583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:10.356 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:09 smithi130 bash[27579]: cluster 2023-08-28T02:11:09.231683+0000 mgr.y (mgr.14901) 7576 : cluster [DBG] pgmap v7583: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:10.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:10 smithi130 bash[51302]: debug 2023-08-28T02:11:10.354+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:10.357949+0000) 2023-08-28T02:11:11.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:11:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:11:10] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T02:11:11.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:11 smithi130 bash[27579]: audit 2023-08-28T02:11:11.262158+0000 mon.c (mon.2) 2979 : audit [DBG] from='client.? 172.21.15.6:0/1308791668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:11.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:11 smithi130 bash[51302]: debug 2023-08-28T02:11:11.354+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:11.358232+0000) 2023-08-28T02:11:11.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:11 smithi006 bash[33918]: audit 2023-08-28T02:11:11.262158+0000 mon.c (mon.2) 2979 : audit [DBG] from='client.? 172.21.15.6:0/1308791668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:11:11.228Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:11:11.230Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:11.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:11 smithi006 bash[27292]: audit 2023-08-28T02:11:11.262158+0000 mon.c (mon.2) 2979 : audit [DBG] from='client.? 172.21.15.6:0/1308791668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:12 smithi130 bash[27579]: cluster 2023-08-28T02:11:11.232360+0000 mgr.y (mgr.14901) 7577 : cluster [DBG] pgmap v7584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:12.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:12 smithi130 bash[51302]: debug 2023-08-28T02:11:12.354+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:12.358505+0000) 2023-08-28T02:11:12.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:12 smithi006 bash[33918]: cluster 2023-08-28T02:11:11.232360+0000 mgr.y (mgr.14901) 7577 : cluster [DBG] pgmap v7584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:12.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:12 smithi006 bash[27292]: cluster 2023-08-28T02:11:11.232360+0000 mgr.y (mgr.14901) 7577 : cluster [DBG] pgmap v7584: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:11:12.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:11:12.799Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:13.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:13 smithi130 bash[51302]: debug 2023-08-28T02:11:13.354+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:13.358785+0000) 2023-08-28T02:11:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:13 smithi130 bash[27579]: cluster 2023-08-28T02:11:13.233099+0000 mgr.y (mgr.14901) 7578 : cluster [DBG] pgmap v7585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:13 smithi130 bash[27579]: audit 2023-08-28T02:11:13.682691+0000 mon.c (mon.2) 2980 : audit [DBG] from='client.? 172.21.15.6:0/884667846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:14.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:13 smithi006 bash[33918]: cluster 2023-08-28T02:11:13.233099+0000 mgr.y (mgr.14901) 7578 : cluster [DBG] pgmap v7585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:14.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:13 smithi006 bash[33918]: audit 2023-08-28T02:11:13.682691+0000 mon.c (mon.2) 2980 : audit [DBG] from='client.? 172.21.15.6:0/884667846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:13 smithi006 bash[27292]: cluster 2023-08-28T02:11:13.233099+0000 mgr.y (mgr.14901) 7578 : cluster [DBG] pgmap v7585: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:13 smithi006 bash[27292]: audit 2023-08-28T02:11:13.682691+0000 mon.c (mon.2) 2980 : audit [DBG] from='client.? 172.21.15.6:0/884667846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:14.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:14 smithi130 bash[51302]: debug 2023-08-28T02:11:14.358+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:14.359003+0000) 2023-08-28T02:11:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:11:14.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:11:14.675Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:15.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:15 smithi130 bash[51302]: debug 2023-08-28T02:11:15.358+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:15.359209+0000) 2023-08-28T02:11:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:16 smithi130 bash[27579]: cluster 2023-08-28T02:11:15.234143+0000 mgr.y (mgr.14901) 7579 : cluster [DBG] pgmap v7586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:16 smithi130 bash[27579]: audit 2023-08-28T02:11:16.113713+0000 mon.c (mon.2) 2981 : audit [DBG] from='client.? 172.21.15.6:0/419973429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:16.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:16 smithi130 bash[51302]: debug 2023-08-28T02:11:16.358+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:16.359477+0000) 2023-08-28T02:11:16.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:16 smithi006 bash[27292]: cluster 2023-08-28T02:11:15.234143+0000 mgr.y (mgr.14901) 7579 : cluster [DBG] pgmap v7586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:16.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:16 smithi006 bash[27292]: audit 2023-08-28T02:11:16.113713+0000 mon.c (mon.2) 2981 : audit [DBG] from='client.? 172.21.15.6:0/419973429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:16 smithi006 bash[33918]: cluster 2023-08-28T02:11:15.234143+0000 mgr.y (mgr.14901) 7579 : cluster [DBG] pgmap v7586: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:16 smithi006 bash[33918]: audit 2023-08-28T02:11:16.113713+0000 mon.c (mon.2) 2981 : audit [DBG] from='client.? 172.21.15.6:0/419973429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:17.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:17 smithi130 bash[51302]: debug 2023-08-28T02:11:17.358+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:17.359773+0000) 2023-08-28T02:11:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:18 smithi130 bash[27579]: cluster 2023-08-28T02:11:17.234797+0000 mgr.y (mgr.14901) 7580 : cluster [DBG] pgmap v7587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:18.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:18 smithi130 bash[51302]: debug 2023-08-28T02:11:18.358+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:18.360075+0000) 2023-08-28T02:11:18.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:18 smithi006 bash[33918]: cluster 2023-08-28T02:11:17.234797+0000 mgr.y (mgr.14901) 7580 : cluster [DBG] pgmap v7587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:18.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:18 smithi006 bash[27292]: cluster 2023-08-28T02:11:17.234797+0000 mgr.y (mgr.14901) 7580 : cluster [DBG] pgmap v7587: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:19.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:19 smithi130 bash[27579]: audit 2023-08-28T02:11:18.529740+0000 mon.a (mon.0) 4875 : audit [DBG] from='client.? 172.21.15.6:0/224331793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:19.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:19 smithi130 bash[51302]: debug 2023-08-28T02:11:19.358+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:19.360368+0000) 2023-08-28T02:11:19.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:19 smithi006 bash[33918]: audit 2023-08-28T02:11:18.529740+0000 mon.a (mon.0) 4875 : audit [DBG] from='client.? 172.21.15.6:0/224331793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:19.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:19 smithi006 bash[27292]: audit 2023-08-28T02:11:18.529740+0000 mon.a (mon.0) 4875 : audit [DBG] from='client.? 172.21.15.6:0/224331793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:20 smithi130 bash[27579]: cluster 2023-08-28T02:11:19.235915+0000 mgr.y (mgr.14901) 7581 : cluster [DBG] pgmap v7588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:20.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:20 smithi130 bash[51302]: debug 2023-08-28T02:11:20.358+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:20.360641+0000) 2023-08-28T02:11:20.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:20 smithi006 bash[27292]: cluster 2023-08-28T02:11:19.235915+0000 mgr.y (mgr.14901) 7581 : cluster [DBG] pgmap v7588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:20.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:20 smithi006 bash[33918]: cluster 2023-08-28T02:11:19.235915+0000 mgr.y (mgr.14901) 7581 : cluster [DBG] pgmap v7588: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:11:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:11:20] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T02:11:21.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:21 smithi130 bash[27579]: audit 2023-08-28T02:11:20.944709+0000 mon.c (mon.2) 2982 : audit [DBG] from='client.? 172.21.15.6:0/963887696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:21.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:21 smithi130 bash[51302]: debug 2023-08-28T02:11:21.358+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:21.360915+0000) 2023-08-28T02:11:21.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:21 smithi006 bash[27292]: audit 2023-08-28T02:11:20.944709+0000 mon.c (mon.2) 2982 : audit [DBG] from='client.? 172.21.15.6:0/963887696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:21.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:21 smithi006 bash[33918]: audit 2023-08-28T02:11:20.944709+0000 mon.c (mon.2) 2982 : audit [DBG] from='client.? 172.21.15.6:0/963887696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:11:21.229Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:11:21.231Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:22 smithi130 bash[27579]: cluster 2023-08-28T02:11:21.236599+0000 mgr.y (mgr.14901) 7582 : cluster [DBG] pgmap v7589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:22.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:22 smithi130 bash[51302]: debug 2023-08-28T02:11:22.358+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:22.361166+0000) 2023-08-28T02:11:22.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:22 smithi006 bash[33918]: cluster 2023-08-28T02:11:21.236599+0000 mgr.y (mgr.14901) 7582 : cluster [DBG] pgmap v7589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:22.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:22 smithi006 bash[27292]: cluster 2023-08-28T02:11:21.236599+0000 mgr.y (mgr.14901) 7582 : cluster [DBG] pgmap v7589: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:11:22.798Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:11:22.799Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:23.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:23 smithi130 bash[51302]: debug 2023-08-28T02:11:23.358+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:23.361381+0000) 2023-08-28T02:11:24.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:23 smithi130 bash[27579]: cluster 2023-08-28T02:11:23.237259+0000 mgr.y (mgr.14901) 7583 : cluster [DBG] pgmap v7590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:24.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:23 smithi130 bash[27579]: audit 2023-08-28T02:11:23.356017+0000 mon.a (mon.0) 4876 : audit [DBG] from='client.? 172.21.15.6:0/2404501670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:24.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:23 smithi006 bash[33918]: cluster 2023-08-28T02:11:23.237259+0000 mgr.y (mgr.14901) 7583 : cluster [DBG] pgmap v7590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:24.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:23 smithi006 bash[33918]: audit 2023-08-28T02:11:23.356017+0000 mon.a (mon.0) 4876 : audit [DBG] from='client.? 172.21.15.6:0/2404501670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:23 smithi006 bash[27292]: cluster 2023-08-28T02:11:23.237259+0000 mgr.y (mgr.14901) 7583 : cluster [DBG] pgmap v7590: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:23 smithi006 bash[27292]: audit 2023-08-28T02:11:23.356017+0000 mon.a (mon.0) 4876 : audit [DBG] from='client.? 172.21.15.6:0/2404501670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:24.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:24 smithi130 bash[51302]: debug 2023-08-28T02:11:24.358+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:24.361666+0000) 2023-08-28T02:11:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:11:24.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:11:24.676Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:25.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:25 smithi130 bash[51302]: debug 2023-08-28T02:11:25.358+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:25.361916+0000) 2023-08-28T02:11:26.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:26 smithi130 bash[51302]: debug 2023-08-28T02:11:26.358+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:26.362175+0000) 2023-08-28T02:11:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:26 smithi130 bash[27579]: cluster 2023-08-28T02:11:25.238352+0000 mgr.y (mgr.14901) 7584 : cluster [DBG] pgmap v7591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:26 smithi130 bash[27579]: audit 2023-08-28T02:11:25.774538+0000 mon.a (mon.0) 4877 : audit [DBG] from='client.? 172.21.15.6:0/3595510931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:26.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:26 smithi006 bash[27292]: cluster 2023-08-28T02:11:25.238352+0000 mgr.y (mgr.14901) 7584 : cluster [DBG] pgmap v7591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:26.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:26 smithi006 bash[27292]: audit 2023-08-28T02:11:25.774538+0000 mon.a (mon.0) 4877 : audit [DBG] from='client.? 172.21.15.6:0/3595510931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:26 smithi006 bash[33918]: cluster 2023-08-28T02:11:25.238352+0000 mgr.y (mgr.14901) 7584 : cluster [DBG] pgmap v7591: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:26 smithi006 bash[33918]: audit 2023-08-28T02:11:25.774538+0000 mon.a (mon.0) 4877 : audit [DBG] from='client.? 172.21.15.6:0/3595510931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:27.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:27 smithi130 bash[51302]: debug 2023-08-28T02:11:27.362+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:27.362452+0000) 2023-08-28T02:11:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:28 smithi130 bash[27579]: cluster 2023-08-28T02:11:27.239000+0000 mgr.y (mgr.14901) 7585 : cluster [DBG] pgmap v7592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:28 smithi130 bash[27579]: audit 2023-08-28T02:11:28.183660+0000 mon.c (mon.2) 2983 : audit [DBG] from='client.? 172.21.15.6:0/1611573319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:28.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:28 smithi130 bash[51302]: debug 2023-08-28T02:11:28.358+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:28.362763+0000) 2023-08-28T02:11:28.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:28 smithi006 bash[27292]: cluster 2023-08-28T02:11:27.239000+0000 mgr.y (mgr.14901) 7585 : cluster [DBG] pgmap v7592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:28 smithi006 bash[27292]: audit 2023-08-28T02:11:28.183660+0000 mon.c (mon.2) 2983 : audit [DBG] from='client.? 172.21.15.6:0/1611573319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:28 smithi006 bash[33918]: cluster 2023-08-28T02:11:27.239000+0000 mgr.y (mgr.14901) 7585 : cluster [DBG] pgmap v7592: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:28 smithi006 bash[33918]: audit 2023-08-28T02:11:28.183660+0000 mon.c (mon.2) 2983 : audit [DBG] from='client.? 172.21.15.6:0/1611573319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:29.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:29 smithi130 bash[51302]: debug 2023-08-28T02:11:29.361+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:29.362964+0000) 2023-08-28T02:11:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:30 smithi130 bash[27579]: cluster 2023-08-28T02:11:29.240100+0000 mgr.y (mgr.14901) 7586 : cluster [DBG] pgmap v7593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:30.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:30 smithi130 bash[51302]: debug 2023-08-28T02:11:30.361+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:30.363206+0000) 2023-08-28T02:11:30.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:30 smithi006 bash[27292]: cluster 2023-08-28T02:11:29.240100+0000 mgr.y (mgr.14901) 7586 : cluster [DBG] pgmap v7593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:30 smithi006 bash[33918]: cluster 2023-08-28T02:11:29.240100+0000 mgr.y (mgr.14901) 7586 : cluster [DBG] pgmap v7593: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:31.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:11:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:11:30] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:11:31.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:31 smithi130 bash[27579]: audit 2023-08-28T02:11:30.603558+0000 mon.c (mon.2) 2984 : audit [DBG] from='client.? 172.21.15.6:0/4088566407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:31.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:31 smithi130 bash[51302]: debug 2023-08-28T02:11:31.361+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:31.363502+0000) 2023-08-28T02:11:31.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:31 smithi006 bash[27292]: audit 2023-08-28T02:11:30.603558+0000 mon.c (mon.2) 2984 : audit [DBG] from='client.? 172.21.15.6:0/4088566407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:31.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:31 smithi006 bash[33918]: audit 2023-08-28T02:11:30.603558+0000 mon.c (mon.2) 2984 : audit [DBG] from='client.? 172.21.15.6:0/4088566407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:11:31.230Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:11:31.232Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:32 smithi130 bash[27579]: cluster 2023-08-28T02:11:31.240774+0000 mgr.y (mgr.14901) 7587 : cluster [DBG] pgmap v7594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:32.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:32 smithi130 bash[51302]: debug 2023-08-28T02:11:32.361+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:32.363765+0000) 2023-08-28T02:11:32.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:32 smithi006 bash[27292]: cluster 2023-08-28T02:11:31.240774+0000 mgr.y (mgr.14901) 7587 : cluster [DBG] pgmap v7594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:32.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:32 smithi006 bash[33918]: cluster 2023-08-28T02:11:31.240774+0000 mgr.y (mgr.14901) 7587 : cluster [DBG] pgmap v7594: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:11:32.798Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:11:32.800Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:33.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:33 smithi130 bash[51302]: debug 2023-08-28T02:11:33.361+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:33.364038+0000) 2023-08-28T02:11:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:33 smithi130 bash[27579]: audit 2023-08-28T02:11:33.015168+0000 mon.a (mon.0) 4878 : audit [DBG] from='client.? 172.21.15.6:0/4287626950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:33 smithi130 bash[27579]: cluster 2023-08-28T02:11:33.241428+0000 mgr.y (mgr.14901) 7588 : cluster [DBG] pgmap v7595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:34.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:33 smithi006 bash[27292]: audit 2023-08-28T02:11:33.015168+0000 mon.a (mon.0) 4878 : audit [DBG] from='client.? 172.21.15.6:0/4287626950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:34.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:33 smithi006 bash[27292]: cluster 2023-08-28T02:11:33.241428+0000 mgr.y (mgr.14901) 7588 : cluster [DBG] pgmap v7595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:33 smithi006 bash[33918]: audit 2023-08-28T02:11:33.015168+0000 mon.a (mon.0) 4878 : audit [DBG] from='client.? 172.21.15.6:0/4287626950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:33 smithi006 bash[33918]: cluster 2023-08-28T02:11:33.241428+0000 mgr.y (mgr.14901) 7588 : cluster [DBG] pgmap v7595: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:34.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:34 smithi130 bash[51302]: debug 2023-08-28T02:11:34.361+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:34.364316+0000) 2023-08-28T02:11:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:11:34.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:11:34.677Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:35.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:35 smithi130 bash[51302]: debug 2023-08-28T02:11:35.361+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:35.364643+0000) 2023-08-28T02:11:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:36 smithi130 bash[27579]: cluster 2023-08-28T02:11:35.242532+0000 mgr.y (mgr.14901) 7589 : cluster [DBG] pgmap v7596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:36 smithi130 bash[27579]: audit 2023-08-28T02:11:35.430921+0000 mon.c (mon.2) 2985 : audit [DBG] from='client.? 172.21.15.6:0/1271196714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:36.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:36 smithi130 bash[51302]: debug 2023-08-28T02:11:36.361+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:36.364952+0000) 2023-08-28T02:11:36.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:36 smithi006 bash[33918]: cluster 2023-08-28T02:11:35.242532+0000 mgr.y (mgr.14901) 7589 : cluster [DBG] pgmap v7596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:36.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:36 smithi006 bash[33918]: audit 2023-08-28T02:11:35.430921+0000 mon.c (mon.2) 2985 : audit [DBG] from='client.? 172.21.15.6:0/1271196714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:36.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:36 smithi006 bash[27292]: cluster 2023-08-28T02:11:35.242532+0000 mgr.y (mgr.14901) 7589 : cluster [DBG] pgmap v7596: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:36.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:36 smithi006 bash[27292]: audit 2023-08-28T02:11:35.430921+0000 mon.c (mon.2) 2985 : audit [DBG] from='client.? 172.21.15.6:0/1271196714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:37.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:37 smithi130 bash[51302]: debug 2023-08-28T02:11:37.361+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:37.365222+0000) 2023-08-28T02:11:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:38 smithi130 bash[27579]: cluster 2023-08-28T02:11:37.243239+0000 mgr.y (mgr.14901) 7590 : cluster [DBG] pgmap v7597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:38 smithi130 bash[27579]: audit 2023-08-28T02:11:37.850817+0000 mon.a (mon.0) 4879 : audit [DBG] from='client.? 172.21.15.6:0/3246159931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:38.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:38 smithi130 bash[51302]: debug 2023-08-28T02:11:38.361+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:38.365576+0000) 2023-08-28T02:11:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:38 smithi006 bash[33918]: cluster 2023-08-28T02:11:37.243239+0000 mgr.y (mgr.14901) 7590 : cluster [DBG] pgmap v7597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:38.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:38 smithi006 bash[33918]: audit 2023-08-28T02:11:37.850817+0000 mon.a (mon.0) 4879 : audit [DBG] from='client.? 172.21.15.6:0/3246159931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:38 smithi006 bash[27292]: cluster 2023-08-28T02:11:37.243239+0000 mgr.y (mgr.14901) 7590 : cluster [DBG] pgmap v7597: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:38 smithi006 bash[27292]: audit 2023-08-28T02:11:37.850817+0000 mon.a (mon.0) 4879 : audit [DBG] from='client.? 172.21.15.6:0/3246159931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:39.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:39 smithi130 bash[51302]: debug 2023-08-28T02:11:39.361+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:39.365844+0000) 2023-08-28T02:11:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:40 smithi130 bash[27579]: cluster 2023-08-28T02:11:39.244335+0000 mgr.y (mgr.14901) 7591 : cluster [DBG] pgmap v7598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:40 smithi130 bash[27579]: audit 2023-08-28T02:11:40.265820+0000 mon.c (mon.2) 2986 : audit [DBG] from='client.? 172.21.15.6:0/979643252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:40.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:40 smithi130 bash[51302]: debug 2023-08-28T02:11:40.361+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:40.366181+0000) 2023-08-28T02:11:40.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:40 smithi006 bash[33918]: cluster 2023-08-28T02:11:39.244335+0000 mgr.y (mgr.14901) 7591 : cluster [DBG] pgmap v7598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:40 smithi006 bash[33918]: audit 2023-08-28T02:11:40.265820+0000 mon.c (mon.2) 2986 : audit [DBG] from='client.? 172.21.15.6:0/979643252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:40 smithi006 bash[27292]: cluster 2023-08-28T02:11:39.244335+0000 mgr.y (mgr.14901) 7591 : cluster [DBG] pgmap v7598: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:40 smithi006 bash[27292]: audit 2023-08-28T02:11:40.265820+0000 mon.c (mon.2) 2986 : audit [DBG] from='client.? 172.21.15.6:0/979643252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:11:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:11:40] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:11:41.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:41 smithi130 bash[51302]: debug 2023-08-28T02:11:41.361+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:41.366539+0000) 2023-08-28T02:11:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:11:41.230Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:11:41.232Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:42.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:42 smithi130 bash[51302]: debug 2023-08-28T02:11:42.361+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:42.366775+0000) 2023-08-28T02:11:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:42 smithi130 bash[27579]: cluster 2023-08-28T02:11:41.245094+0000 mgr.y (mgr.14901) 7592 : cluster [DBG] pgmap v7599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:42.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:42 smithi006 bash[27292]: cluster 2023-08-28T02:11:41.245094+0000 mgr.y (mgr.14901) 7592 : cluster [DBG] pgmap v7599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:42.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:42 smithi006 bash[33918]: cluster 2023-08-28T02:11:41.245094+0000 mgr.y (mgr.14901) 7592 : cluster [DBG] pgmap v7599: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:11:42.799Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:11:42.801Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:43.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:43 smithi130 bash[27579]: audit 2023-08-28T02:11:42.687487+0000 mon.c (mon.2) 2987 : audit [DBG] from='client.? 172.21.15.6:0/819317598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:43.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:43 smithi130 bash[51302]: debug 2023-08-28T02:11:43.365+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:43.367106+0000) 2023-08-28T02:11:43.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:43 smithi006 bash[27292]: audit 2023-08-28T02:11:42.687487+0000 mon.c (mon.2) 2987 : audit [DBG] from='client.? 172.21.15.6:0/819317598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:43.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:43 smithi006 bash[33918]: audit 2023-08-28T02:11:42.687487+0000 mon.c (mon.2) 2987 : audit [DBG] from='client.? 172.21.15.6:0/819317598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:44 smithi130 bash[27579]: cluster 2023-08-28T02:11:43.245737+0000 mgr.y (mgr.14901) 7593 : cluster [DBG] pgmap v7600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:44.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:44 smithi130 bash[51302]: debug 2023-08-28T02:11:44.365+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:44.367414+0000) 2023-08-28T02:11:44.676 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:44 smithi006 bash[33918]: cluster 2023-08-28T02:11:43.245737+0000 mgr.y (mgr.14901) 7593 : cluster [DBG] pgmap v7600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:44.677 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:44 smithi006 bash[27292]: cluster 2023-08-28T02:11:43.245737+0000 mgr.y (mgr.14901) 7593 : cluster [DBG] pgmap v7600: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:11:44.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:11:44.678Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:45.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:45 smithi130 bash[27579]: audit 2023-08-28T02:11:45.102445+0000 mon.c (mon.2) 2988 : audit [DBG] from='client.? 172.21.15.6:0/1671220766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:45.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:45 smithi130 bash[51302]: debug 2023-08-28T02:11:45.365+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:45.367691+0000) 2023-08-28T02:11:45.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:45 smithi006 bash[27292]: audit 2023-08-28T02:11:45.102445+0000 mon.c (mon.2) 2988 : audit [DBG] from='client.? 172.21.15.6:0/1671220766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:45.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:45 smithi006 bash[33918]: audit 2023-08-28T02:11:45.102445+0000 mon.c (mon.2) 2988 : audit [DBG] from='client.? 172.21.15.6:0/1671220766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:46 smithi130 bash[27579]: cluster 2023-08-28T02:11:45.246849+0000 mgr.y (mgr.14901) 7594 : cluster [DBG] pgmap v7601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:46.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:46 smithi130 bash[51302]: debug 2023-08-28T02:11:46.365+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:46.367982+0000) 2023-08-28T02:11:46.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:46 smithi006 bash[33918]: cluster 2023-08-28T02:11:45.246849+0000 mgr.y (mgr.14901) 7594 : cluster [DBG] pgmap v7601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:46.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:46 smithi006 bash[27292]: cluster 2023-08-28T02:11:45.246849+0000 mgr.y (mgr.14901) 7594 : cluster [DBG] pgmap v7601: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:47.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:47 smithi130 bash[51302]: debug 2023-08-28T02:11:47.365+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:47.368247+0000) 2023-08-28T02:11:48.618 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:48 smithi130 bash[51302]: debug 2023-08-28T02:11:48.365+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:48.368510+0000) 2023-08-28T02:11:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:48 smithi130 bash[27579]: cluster 2023-08-28T02:11:47.247266+0000 mgr.y (mgr.14901) 7595 : cluster [DBG] pgmap v7602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:48.618 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:48 smithi130 bash[27579]: audit 2023-08-28T02:11:47.519709+0000 mon.a (mon.0) 4880 : audit [DBG] from='client.? 172.21.15.6:0/1861648834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:48.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:48 smithi006 bash[33918]: cluster 2023-08-28T02:11:47.247266+0000 mgr.y (mgr.14901) 7595 : cluster [DBG] pgmap v7602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:48.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:48 smithi006 bash[33918]: audit 2023-08-28T02:11:47.519709+0000 mon.a (mon.0) 4880 : audit [DBG] from='client.? 172.21.15.6:0/1861648834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:48 smithi006 bash[27292]: cluster 2023-08-28T02:11:47.247266+0000 mgr.y (mgr.14901) 7595 : cluster [DBG] pgmap v7602: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:48 smithi006 bash[27292]: audit 2023-08-28T02:11:47.519709+0000 mon.a (mon.0) 4880 : audit [DBG] from='client.? 172.21.15.6:0/1861648834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:49.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:49 smithi130 bash[51302]: debug 2023-08-28T02:11:49.365+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:49.368812+0000) 2023-08-28T02:11:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:50 smithi130 bash[27579]: cluster 2023-08-28T02:11:49.248296+0000 mgr.y (mgr.14901) 7596 : cluster [DBG] pgmap v7603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:50 smithi130 bash[27579]: audit 2023-08-28T02:11:49.937076+0000 mon.c (mon.2) 2989 : audit [DBG] from='client.? 172.21.15.6:0/836447395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:50.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:50 smithi130 bash[51302]: debug 2023-08-28T02:11:50.365+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:50.369102+0000) 2023-08-28T02:11:50.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:50 smithi006 bash[33918]: cluster 2023-08-28T02:11:49.248296+0000 mgr.y (mgr.14901) 7596 : cluster [DBG] pgmap v7603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:50.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:50 smithi006 bash[33918]: audit 2023-08-28T02:11:49.937076+0000 mon.c (mon.2) 2989 : audit [DBG] from='client.? 172.21.15.6:0/836447395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:50 smithi006 bash[27292]: cluster 2023-08-28T02:11:49.248296+0000 mgr.y (mgr.14901) 7596 : cluster [DBG] pgmap v7603: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:50 smithi006 bash[27292]: audit 2023-08-28T02:11:49.937076+0000 mon.c (mon.2) 2989 : audit [DBG] from='client.? 172.21.15.6:0/836447395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:11:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:11:50] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T02:11:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:11:51.232Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:11:51.234Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:51.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:51 smithi130 bash[51302]: debug 2023-08-28T02:11:51.365+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:51.369409+0000) 2023-08-28T02:11:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:52 smithi130 bash[27579]: cluster 2023-08-28T02:11:51.249087+0000 mgr.y (mgr.14901) 7597 : cluster [DBG] pgmap v7604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:52.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:52 smithi130 bash[51302]: debug 2023-08-28T02:11:52.365+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:52.369682+0000) 2023-08-28T02:11:52.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:52 smithi006 bash[27292]: cluster 2023-08-28T02:11:51.249087+0000 mgr.y (mgr.14901) 7597 : cluster [DBG] pgmap v7604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:52.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:52 smithi006 bash[33918]: cluster 2023-08-28T02:11:51.249087+0000 mgr.y (mgr.14901) 7597 : cluster [DBG] pgmap v7604: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:11:52.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:11:52.802Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:53.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:53 smithi130 bash[27579]: audit 2023-08-28T02:11:52.357297+0000 mon.a (mon.0) 4881 : audit [DBG] from='client.? 172.21.15.6:0/244373872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:53.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:53 smithi130 bash[51302]: debug 2023-08-28T02:11:53.365+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:53.369967+0000) 2023-08-28T02:11:53.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:53 smithi006 bash[33918]: audit 2023-08-28T02:11:52.357297+0000 mon.a (mon.0) 4881 : audit [DBG] from='client.? 172.21.15.6:0/244373872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:53.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:53 smithi006 bash[27292]: audit 2023-08-28T02:11:52.357297+0000 mon.a (mon.0) 4881 : audit [DBG] from='client.? 172.21.15.6:0/244373872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:54 smithi130 bash[27579]: cluster 2023-08-28T02:11:53.249775+0000 mgr.y (mgr.14901) 7598 : cluster [DBG] pgmap v7605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:54.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:54 smithi130 bash[51302]: debug 2023-08-28T02:11:54.365+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:54.370244+0000) 2023-08-28T02:11:54.676 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:54 smithi006 bash[27292]: cluster 2023-08-28T02:11:53.249775+0000 mgr.y (mgr.14901) 7598 : cluster [DBG] pgmap v7605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:54.676 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:54 smithi006 bash[33918]: cluster 2023-08-28T02:11:53.249775+0000 mgr.y (mgr.14901) 7598 : cluster [DBG] pgmap v7605: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:11:54.676Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:11:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:11:54.678Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:11:55.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:55 smithi130 bash[27579]: audit 2023-08-28T02:11:54.768250+0000 mon.c (mon.2) 2990 : audit [DBG] from='client.? 172.21.15.6:0/1805711223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:55.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:55 smithi130 bash[51302]: debug 2023-08-28T02:11:55.365+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:55.370495+0000) 2023-08-28T02:11:55.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:55 smithi006 bash[27292]: audit 2023-08-28T02:11:54.768250+0000 mon.c (mon.2) 2990 : audit [DBG] from='client.? 172.21.15.6:0/1805711223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:55.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:55 smithi006 bash[33918]: audit 2023-08-28T02:11:54.768250+0000 mon.c (mon.2) 2990 : audit [DBG] from='client.? 172.21.15.6:0/1805711223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:56.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:56 smithi006 bash[27292]: cluster 2023-08-28T02:11:55.250991+0000 mgr.y (mgr.14901) 7599 : cluster [DBG] pgmap v7606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:56.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:56 smithi006 bash[33918]: cluster 2023-08-28T02:11:55.250991+0000 mgr.y (mgr.14901) 7599 : cluster [DBG] pgmap v7606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:56 smithi130 bash[27579]: cluster 2023-08-28T02:11:55.250991+0000 mgr.y (mgr.14901) 7599 : cluster [DBG] pgmap v7606: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:56.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:56 smithi130 bash[51302]: debug 2023-08-28T02:11:56.365+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:56.370682+0000) 2023-08-28T02:11:57.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:57 smithi006 bash[33918]: audit 2023-08-28T02:11:57.183862+0000 mon.c (mon.2) 2991 : audit [DBG] from='client.? 172.21.15.6:0/1862963727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:57.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:57 smithi006 bash[27292]: audit 2023-08-28T02:11:57.183862+0000 mon.c (mon.2) 2991 : audit [DBG] from='client.? 172.21.15.6:0/1862963727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:57.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:57 smithi130 bash[27579]: audit 2023-08-28T02:11:57.183862+0000 mon.c (mon.2) 2991 : audit [DBG] from='client.? 172.21.15.6:0/1862963727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:57.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:57 smithi130 bash[51302]: debug 2023-08-28T02:11:57.369+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:57.370951+0000) 2023-08-28T02:11:58.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:11:58 smithi006 bash[27292]: cluster 2023-08-28T02:11:57.251785+0000 mgr.y (mgr.14901) 7600 : cluster [DBG] pgmap v7607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:58.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:11:58 smithi006 bash[33918]: cluster 2023-08-28T02:11:57.251785+0000 mgr.y (mgr.14901) 7600 : cluster [DBG] pgmap v7607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:58.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:11:58 smithi130 bash[27579]: cluster 2023-08-28T02:11:57.251785+0000 mgr.y (mgr.14901) 7600 : cluster [DBG] pgmap v7607: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:11:58.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:58 smithi130 bash[51302]: debug 2023-08-28T02:11:58.369+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:58.371227+0000) 2023-08-28T02:11:59.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:11:59 smithi130 bash[51302]: debug 2023-08-28T02:11:59.369+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:59.371449+0000) 2023-08-28T02:12:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:00 smithi006 bash[33918]: cluster 2023-08-28T02:11:59.253089+0000 mgr.y (mgr.14901) 7601 : cluster [DBG] pgmap v7608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:00 smithi006 bash[33918]: audit 2023-08-28T02:11:59.596988+0000 mon.a (mon.0) 4882 : audit [DBG] from='client.? 172.21.15.6:0/2753094025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:00 smithi006 bash[27292]: cluster 2023-08-28T02:11:59.253089+0000 mgr.y (mgr.14901) 7601 : cluster [DBG] pgmap v7608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:00 smithi006 bash[27292]: audit 2023-08-28T02:11:59.596988+0000 mon.a (mon.0) 4882 : audit [DBG] from='client.? 172.21.15.6:0/2753094025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:00 smithi130 bash[27579]: cluster 2023-08-28T02:11:59.253089+0000 mgr.y (mgr.14901) 7601 : cluster [DBG] pgmap v7608: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:00 smithi130 bash[27579]: audit 2023-08-28T02:11:59.596988+0000 mon.a (mon.0) 4882 : audit [DBG] from='client.? 172.21.15.6:0/2753094025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:00.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:00 smithi130 bash[51302]: debug 2023-08-28T02:12:00.369+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:00.371713+0000) 2023-08-28T02:12:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:12:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:12:00] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:12:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:12:01.233Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:12:01.234Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:01.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:01 smithi130 bash[51302]: debug 2023-08-28T02:12:01.369+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:01.371997+0000) 2023-08-28T02:12:02.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:02 smithi006 bash[33918]: cluster 2023-08-28T02:12:01.253775+0000 mgr.y (mgr.14901) 7602 : cluster [DBG] pgmap v7609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:02.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:02 smithi006 bash[33918]: audit 2023-08-28T02:12:02.018158+0000 mon.c (mon.2) 2992 : audit [DBG] from='client.? 172.21.15.6:0/3723671044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:02.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:02 smithi006 bash[27292]: cluster 2023-08-28T02:12:01.253775+0000 mgr.y (mgr.14901) 7602 : cluster [DBG] pgmap v7609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:02.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:02 smithi006 bash[27292]: audit 2023-08-28T02:12:02.018158+0000 mon.c (mon.2) 2992 : audit [DBG] from='client.? 172.21.15.6:0/3723671044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:02.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:02 smithi130 bash[51302]: debug 2023-08-28T02:12:02.369+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:02.372264+0000) 2023-08-28T02:12:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:02 smithi130 bash[27579]: cluster 2023-08-28T02:12:01.253775+0000 mgr.y (mgr.14901) 7602 : cluster [DBG] pgmap v7609: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:02 smithi130 bash[27579]: audit 2023-08-28T02:12:02.018158+0000 mon.c (mon.2) 2992 : audit [DBG] from='client.? 172.21.15.6:0/3723671044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:12:02.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:12:02.802Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:03.741 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:03 smithi130 bash[51302]: debug 2023-08-28T02:12:03.369+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:03.372567+0000) 2023-08-28T02:12:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:03 smithi130 bash[27579]: cluster 2023-08-28T02:12:03.254434+0000 mgr.y (mgr.14901) 7603 : cluster [DBG] pgmap v7610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:04.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:03 smithi006 bash[33918]: cluster 2023-08-28T02:12:03.254434+0000 mgr.y (mgr.14901) 7603 : cluster [DBG] pgmap v7610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:04.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:03 smithi006 bash[27292]: cluster 2023-08-28T02:12:03.254434+0000 mgr.y (mgr.14901) 7603 : cluster [DBG] pgmap v7610: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:04.744 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:04 smithi130 bash[51302]: debug 2023-08-28T02:12:04.369+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:04.372858+0000) 2023-08-28T02:12:04.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:04 smithi006 bash[27292]: audit 2023-08-28T02:12:04.444828+0000 mon.a (mon.0) 4883 : audit [DBG] from='client.? 172.21.15.6:0/982889802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:04.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:04 smithi006 bash[33918]: audit 2023-08-28T02:12:04.444828+0000 mon.a (mon.0) 4883 : audit [DBG] from='client.? 172.21.15.6:0/982889802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:12:04.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:12:04.679Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:05.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:04 smithi130 bash[27579]: audit 2023-08-28T02:12:04.444828+0000 mon.a (mon.0) 4883 : audit [DBG] from='client.? 172.21.15.6:0/982889802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:05.753 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:05 smithi130 bash[51302]: debug 2023-08-28T02:12:05.369+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:05.373136+0000) 2023-08-28T02:12:06.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:05 smithi130 bash[27579]: cluster 2023-08-28T02:12:05.255584+0000 mgr.y (mgr.14901) 7604 : cluster [DBG] pgmap v7611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:06.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:05 smithi006 bash[33918]: cluster 2023-08-28T02:12:05.255584+0000 mgr.y (mgr.14901) 7604 : cluster [DBG] pgmap v7611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:06.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:05 smithi006 bash[27292]: cluster 2023-08-28T02:12:05.255584+0000 mgr.y (mgr.14901) 7604 : cluster [DBG] pgmap v7611: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:06.756 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:06 smithi130 bash[51302]: debug 2023-08-28T02:12:06.369+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:06.373377+0000) 2023-08-28T02:12:07.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:06 smithi130 bash[27579]: audit 2023-08-28T02:12:06.662356+0000 mon.b (mon.1) 840 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:12:07.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:06 smithi006 bash[33918]: audit 2023-08-28T02:12:06.662356+0000 mon.b (mon.1) 840 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:12:07.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:06 smithi006 bash[27292]: audit 2023-08-28T02:12:06.662356+0000 mon.b (mon.1) 840 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:12:07.759 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:07 smithi130 bash[51302]: debug 2023-08-28T02:12:07.369+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:07.373675+0000) 2023-08-28T02:12:08.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:07 smithi130 bash[27579]: audit 2023-08-28T02:12:06.873464+0000 mon.a (mon.0) 4884 : audit [DBG] from='client.? 172.21.15.6:0/223091044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:08.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:07 smithi130 bash[27579]: audit 2023-08-28T02:12:06.934452+0000 mon.a (mon.0) 4885 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:12:08.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:07 smithi130 bash[27579]: audit 2023-08-28T02:12:06.941231+0000 mon.a (mon.0) 4886 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:12:08.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:07 smithi130 bash[27579]: cluster 2023-08-28T02:12:07.256365+0000 mgr.y (mgr.14901) 7605 : cluster [DBG] pgmap v7612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:08.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:07 smithi006 bash[33918]: audit 2023-08-28T02:12:06.873464+0000 mon.a (mon.0) 4884 : audit [DBG] from='client.? 172.21.15.6:0/223091044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:08.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:07 smithi006 bash[33918]: audit 2023-08-28T02:12:06.934452+0000 mon.a (mon.0) 4885 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:12:08.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:07 smithi006 bash[33918]: audit 2023-08-28T02:12:06.941231+0000 mon.a (mon.0) 4886 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:12:08.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:07 smithi006 bash[33918]: cluster 2023-08-28T02:12:07.256365+0000 mgr.y (mgr.14901) 7605 : cluster [DBG] pgmap v7612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:08.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:07 smithi006 bash[27292]: audit 2023-08-28T02:12:06.873464+0000 mon.a (mon.0) 4884 : audit [DBG] from='client.? 172.21.15.6:0/223091044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:08.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:07 smithi006 bash[27292]: audit 2023-08-28T02:12:06.934452+0000 mon.a (mon.0) 4885 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:12:08.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:07 smithi006 bash[27292]: audit 2023-08-28T02:12:06.941231+0000 mon.a (mon.0) 4886 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:12:08.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:07 smithi006 bash[27292]: cluster 2023-08-28T02:12:07.256365+0000 mgr.y (mgr.14901) 7605 : cluster [DBG] pgmap v7612: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:08.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:08 smithi130 bash[51302]: debug 2023-08-28T02:12:08.369+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:08.373943+0000) 2023-08-28T02:12:09.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:09 smithi130 bash[27579]: audit 2023-08-28T02:12:09.295143+0000 mon.a (mon.0) 4887 : audit [DBG] from='client.? 172.21.15.6:0/2738493706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:09.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:09 smithi130 bash[51302]: debug 2023-08-28T02:12:09.373+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:09.374229+0000) 2023-08-28T02:12:09.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:09 smithi006 bash[33918]: audit 2023-08-28T02:12:09.295143+0000 mon.a (mon.0) 4887 : audit [DBG] from='client.? 172.21.15.6:0/2738493706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:09.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:09 smithi006 bash[27292]: audit 2023-08-28T02:12:09.295143+0000 mon.a (mon.0) 4887 : audit [DBG] from='client.? 172.21.15.6:0/2738493706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:10 smithi130 bash[27579]: cluster 2023-08-28T02:12:09.257507+0000 mgr.y (mgr.14901) 7606 : cluster [DBG] pgmap v7613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:10.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:10 smithi130 bash[51302]: debug 2023-08-28T02:12:10.369+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:10.374539+0000) 2023-08-28T02:12:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:10 smithi006 bash[33918]: cluster 2023-08-28T02:12:09.257507+0000 mgr.y (mgr.14901) 7606 : cluster [DBG] pgmap v7613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:11.710 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:10 smithi006 bash[27292]: cluster 2023-08-28T02:12:09.257507+0000 mgr.y (mgr.14901) 7606 : cluster [DBG] pgmap v7613: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:11.711 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:12:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:12:10] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:12:11.736 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:12:11.233Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:11.736 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:12:11.235Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:11.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:11 smithi130 bash[51302]: debug 2023-08-28T02:12:11.369+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:11.374775+0000) 2023-08-28T02:12:12.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:12 smithi130 bash[51302]: debug 2023-08-28T02:12:12.373+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:12.375016+0000) 2023-08-28T02:12:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:12 smithi130 bash[27579]: cluster 2023-08-28T02:12:11.258132+0000 mgr.y (mgr.14901) 7607 : cluster [DBG] pgmap v7614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:12.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:12 smithi130 bash[27579]: audit 2023-08-28T02:12:11.709559+0000 mon.a (mon.0) 4888 : audit [DBG] from='client.? 172.21.15.6:0/1187782178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:12.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:12 smithi006 bash[27292]: cluster 2023-08-28T02:12:11.258132+0000 mgr.y (mgr.14901) 7607 : cluster [DBG] pgmap v7614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:12.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:12 smithi006 bash[27292]: audit 2023-08-28T02:12:11.709559+0000 mon.a (mon.0) 4888 : audit [DBG] from='client.? 172.21.15.6:0/1187782178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:12.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:12 smithi006 bash[33918]: cluster 2023-08-28T02:12:11.258132+0000 mgr.y (mgr.14901) 7607 : cluster [DBG] pgmap v7614: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:12.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:12 smithi006 bash[33918]: audit 2023-08-28T02:12:11.709559+0000 mon.a (mon.0) 4888 : audit [DBG] from='client.? 172.21.15.6:0/1187782178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:12:12.801Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:12:12.804Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:13.842 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:13 smithi130 bash[51302]: debug 2023-08-28T02:12:13.373+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:13.375288+0000) 2023-08-28T02:12:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:13 smithi130 bash[27579]: audit 2023-08-28T02:12:12.841919+0000 mon.a (mon.0) 4889 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:12:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:13 smithi130 bash[27579]: cluster 2023-08-28T02:12:13.258846+0000 mgr.y (mgr.14901) 7608 : cluster [DBG] pgmap v7615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:13 smithi130 bash[27579]: audit 2023-08-28T02:12:13.264136+0000 mon.a (mon.0) 4890 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:12:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:13 smithi130 bash[27579]: audit 2023-08-28T02:12:13.561983+0000 mon.b (mon.1) 841 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:12:14.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:13 smithi130 bash[27579]: audit 2023-08-28T02:12:13.564258+0000 mon.b (mon.1) 842 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:12:14.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:13 smithi130 bash[27579]: audit 2023-08-28T02:12:13.571315+0000 mon.a (mon.0) 4891 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:12:14.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:13 smithi006 bash[27292]: audit 2023-08-28T02:12:12.841919+0000 mon.a (mon.0) 4889 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:12:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:13 smithi006 bash[27292]: cluster 2023-08-28T02:12:13.258846+0000 mgr.y (mgr.14901) 7608 : cluster [DBG] pgmap v7615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:13 smithi006 bash[27292]: audit 2023-08-28T02:12:13.264136+0000 mon.a (mon.0) 4890 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:12:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:13 smithi006 bash[27292]: audit 2023-08-28T02:12:13.561983+0000 mon.b (mon.1) 841 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:12:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:13 smithi006 bash[27292]: audit 2023-08-28T02:12:13.564258+0000 mon.b (mon.1) 842 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:12:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:13 smithi006 bash[27292]: audit 2023-08-28T02:12:13.571315+0000 mon.a (mon.0) 4891 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:12:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:13 smithi006 bash[33918]: audit 2023-08-28T02:12:12.841919+0000 mon.a (mon.0) 4889 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:12:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:13 smithi006 bash[33918]: cluster 2023-08-28T02:12:13.258846+0000 mgr.y (mgr.14901) 7608 : cluster [DBG] pgmap v7615: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:13 smithi006 bash[33918]: audit 2023-08-28T02:12:13.264136+0000 mon.a (mon.0) 4890 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:12:14.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:13 smithi006 bash[33918]: audit 2023-08-28T02:12:13.561983+0000 mon.b (mon.1) 841 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:12:14.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:13 smithi006 bash[33918]: audit 2023-08-28T02:12:13.564258+0000 mon.b (mon.1) 842 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:12:14.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:13 smithi006 bash[33918]: audit 2023-08-28T02:12:13.571315+0000 mon.a (mon.0) 4891 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:12:14.845 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:14 smithi130 bash[51302]: debug 2023-08-28T02:12:14.373+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:14.375513+0000) 2023-08-28T02:12:14.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:14 smithi006 bash[27292]: audit 2023-08-28T02:12:14.123742+0000 mon.c (mon.2) 2993 : audit [DBG] from='client.? 172.21.15.6:0/1347724253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:12:14.678Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:12:14.681Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:14.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:14 smithi006 bash[33918]: audit 2023-08-28T02:12:14.123742+0000 mon.c (mon.2) 2993 : audit [DBG] from='client.? 172.21.15.6:0/1347724253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:15.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:14 smithi130 bash[27579]: audit 2023-08-28T02:12:14.123742+0000 mon.c (mon.2) 2993 : audit [DBG] from='client.? 172.21.15.6:0/1347724253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:15.852 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:15 smithi130 bash[51302]: debug 2023-08-28T02:12:15.373+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:15.375839+0000) 2023-08-28T02:12:16.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:15 smithi130 bash[27579]: cluster 2023-08-28T02:12:15.260018+0000 mgr.y (mgr.14901) 7609 : cluster [DBG] pgmap v7616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:16.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:15 smithi006 bash[27292]: cluster 2023-08-28T02:12:15.260018+0000 mgr.y (mgr.14901) 7609 : cluster [DBG] pgmap v7616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:16.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:15 smithi006 bash[33918]: cluster 2023-08-28T02:12:15.260018+0000 mgr.y (mgr.14901) 7609 : cluster [DBG] pgmap v7616: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:16.855 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:16 smithi130 bash[51302]: debug 2023-08-28T02:12:16.373+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:16.376146+0000) 2023-08-28T02:12:17.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:16 smithi130 bash[27579]: audit 2023-08-28T02:12:16.534990+0000 mon.c (mon.2) 2994 : audit [DBG] from='client.? 172.21.15.6:0/3180440429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:17.684 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:16 smithi006 bash[27292]: audit 2023-08-28T02:12:16.534990+0000 mon.c (mon.2) 2994 : audit [DBG] from='client.? 172.21.15.6:0/3180440429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:17.684 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:16 smithi006 bash[33918]: audit 2023-08-28T02:12:16.534990+0000 mon.c (mon.2) 2994 : audit [DBG] from='client.? 172.21.15.6:0/3180440429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:17.862 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:17 smithi130 bash[51302]: debug 2023-08-28T02:12:17.373+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:17.376408+0000) 2023-08-28T02:12:18.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:17 smithi130 bash[27579]: cluster 2023-08-28T02:12:17.260712+0000 mgr.y (mgr.14901) 7610 : cluster [DBG] pgmap v7617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:18.664 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:17 smithi006 bash[33918]: cluster 2023-08-28T02:12:17.260712+0000 mgr.y (mgr.14901) 7610 : cluster [DBG] pgmap v7617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:18.664 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:17 smithi006 bash[27292]: cluster 2023-08-28T02:12:17.260712+0000 mgr.y (mgr.14901) 7610 : cluster [DBG] pgmap v7617: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:18.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:18 smithi130 bash[51302]: debug 2023-08-28T02:12:18.373+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:18.376677+0000) 2023-08-28T02:12:19.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:19 smithi130 bash[27579]: audit 2023-08-28T02:12:18.950651+0000 mon.c (mon.2) 2995 : audit [DBG] from='client.? 172.21.15.6:0/3106724808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:19.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:19 smithi006 bash[33918]: audit 2023-08-28T02:12:18.950651+0000 mon.c (mon.2) 2995 : audit [DBG] from='client.? 172.21.15.6:0/3106724808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:19.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:19 smithi006 bash[27292]: audit 2023-08-28T02:12:18.950651+0000 mon.c (mon.2) 2995 : audit [DBG] from='client.? 172.21.15.6:0/3106724808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:19.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:19 smithi130 bash[51302]: debug 2023-08-28T02:12:19.373+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:19.376895+0000) 2023-08-28T02:12:20.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:20 smithi130 bash[27579]: cluster 2023-08-28T02:12:19.261850+0000 mgr.y (mgr.14901) 7611 : cluster [DBG] pgmap v7618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:20.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:20 smithi006 bash[27292]: cluster 2023-08-28T02:12:19.261850+0000 mgr.y (mgr.14901) 7611 : cluster [DBG] pgmap v7618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:20.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:20 smithi006 bash[33918]: cluster 2023-08-28T02:12:19.261850+0000 mgr.y (mgr.14901) 7611 : cluster [DBG] pgmap v7618: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:20.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:20 smithi130 bash[51302]: debug 2023-08-28T02:12:20.373+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:20.377138+0000) 2023-08-28T02:12:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:12:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:12:20] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:12:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:12:21.234Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:12:21.236Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:21.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:21 smithi130 bash[51302]: debug 2023-08-28T02:12:21.373+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:21.377371+0000) 2023-08-28T02:12:22.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:22 smithi130 bash[51302]: debug 2023-08-28T02:12:22.373+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:22.377580+0000) 2023-08-28T02:12:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:22 smithi130 bash[27579]: cluster 2023-08-28T02:12:21.262507+0000 mgr.y (mgr.14901) 7612 : cluster [DBG] pgmap v7619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:22 smithi130 bash[27579]: audit 2023-08-28T02:12:21.364656+0000 mon.a (mon.0) 4892 : audit [DBG] from='client.? 172.21.15.6:0/287887223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:22.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:22 smithi006 bash[27292]: cluster 2023-08-28T02:12:21.262507+0000 mgr.y (mgr.14901) 7612 : cluster [DBG] pgmap v7619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:22.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:22 smithi006 bash[27292]: audit 2023-08-28T02:12:21.364656+0000 mon.a (mon.0) 4892 : audit [DBG] from='client.? 172.21.15.6:0/287887223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:22.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:22 smithi006 bash[33918]: cluster 2023-08-28T02:12:21.262507+0000 mgr.y (mgr.14901) 7612 : cluster [DBG] pgmap v7619: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:22.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:22 smithi006 bash[33918]: audit 2023-08-28T02:12:21.364656+0000 mon.a (mon.0) 4892 : audit [DBG] from='client.? 172.21.15.6:0/287887223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:12:22.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:12:22.804Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:23.744 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:23 smithi130 bash[51302]: debug 2023-08-28T02:12:23.373+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:23.377800+0000) 2023-08-28T02:12:24.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:23 smithi130 bash[27579]: cluster 2023-08-28T02:12:23.263182+0000 mgr.y (mgr.14901) 7613 : cluster [DBG] pgmap v7620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:23 smithi006 bash[27292]: cluster 2023-08-28T02:12:23.263182+0000 mgr.y (mgr.14901) 7613 : cluster [DBG] pgmap v7620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:24.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:23 smithi006 bash[33918]: cluster 2023-08-28T02:12:23.263182+0000 mgr.y (mgr.14901) 7613 : cluster [DBG] pgmap v7620: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:24.746 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:24 smithi130 bash[51302]: debug 2023-08-28T02:12:24.373+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:24.378115+0000) 2023-08-28T02:12:24.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:24 smithi006 bash[33918]: audit 2023-08-28T02:12:23.776488+0000 mon.a (mon.0) 4893 : audit [DBG] from='client.? 172.21.15.6:0/2039740565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:24.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:24 smithi006 bash[27292]: audit 2023-08-28T02:12:23.776488+0000 mon.a (mon.0) 4893 : audit [DBG] from='client.? 172.21.15.6:0/2039740565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:12:24.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:12:24.682Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:25.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:24 smithi130 bash[27579]: audit 2023-08-28T02:12:23.776488+0000 mon.a (mon.0) 4893 : audit [DBG] from='client.? 172.21.15.6:0/2039740565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:25.757 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:25 smithi130 bash[51302]: debug 2023-08-28T02:12:25.372+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:25.378423+0000) 2023-08-28T02:12:26.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:25 smithi130 bash[27579]: cluster 2023-08-28T02:12:25.264270+0000 mgr.y (mgr.14901) 7614 : cluster [DBG] pgmap v7621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:26.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:25 smithi006 bash[27292]: cluster 2023-08-28T02:12:25.264270+0000 mgr.y (mgr.14901) 7614 : cluster [DBG] pgmap v7621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:26.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:25 smithi006 bash[33918]: cluster 2023-08-28T02:12:25.264270+0000 mgr.y (mgr.14901) 7614 : cluster [DBG] pgmap v7621: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:26.760 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:26 smithi130 bash[51302]: debug 2023-08-28T02:12:26.372+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:26.378702+0000) 2023-08-28T02:12:27.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:26 smithi130 bash[27579]: audit 2023-08-28T02:12:26.192691+0000 mon.a (mon.0) 4894 : audit [DBG] from='client.? 172.21.15.6:0/2336135172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:27.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:26 smithi006 bash[27292]: audit 2023-08-28T02:12:26.192691+0000 mon.a (mon.0) 4894 : audit [DBG] from='client.? 172.21.15.6:0/2336135172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:27.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:26 smithi006 bash[33918]: audit 2023-08-28T02:12:26.192691+0000 mon.a (mon.0) 4894 : audit [DBG] from='client.? 172.21.15.6:0/2336135172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:27.765 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:27 smithi130 bash[51302]: debug 2023-08-28T02:12:27.376+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:27.379012+0000) 2023-08-28T02:12:28.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:27 smithi130 bash[27579]: cluster 2023-08-28T02:12:27.265018+0000 mgr.y (mgr.14901) 7615 : cluster [DBG] pgmap v7622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:28.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:27 smithi006 bash[33918]: cluster 2023-08-28T02:12:27.265018+0000 mgr.y (mgr.14901) 7615 : cluster [DBG] pgmap v7622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:28.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:27 smithi006 bash[27292]: cluster 2023-08-28T02:12:27.265018+0000 mgr.y (mgr.14901) 7615 : cluster [DBG] pgmap v7622: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:28.770 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:28 smithi130 bash[51302]: debug 2023-08-28T02:12:28.376+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:28.379275+0000) 2023-08-28T02:12:29.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:28 smithi130 bash[27579]: audit 2023-08-28T02:12:28.602910+0000 mon.a (mon.0) 4895 : audit [DBG] from='client.? 172.21.15.6:0/2210179563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:29.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:28 smithi006 bash[33918]: audit 2023-08-28T02:12:28.602910+0000 mon.a (mon.0) 4895 : audit [DBG] from='client.? 172.21.15.6:0/2210179563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:29.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:28 smithi006 bash[27292]: audit 2023-08-28T02:12:28.602910+0000 mon.a (mon.0) 4895 : audit [DBG] from='client.? 172.21.15.6:0/2210179563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:29.779 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:29 smithi130 bash[51302]: debug 2023-08-28T02:12:29.376+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:29.379502+0000) 2023-08-28T02:12:30.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:29 smithi130 bash[27579]: cluster 2023-08-28T02:12:29.266089+0000 mgr.y (mgr.14901) 7616 : cluster [DBG] pgmap v7623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:30.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:29 smithi006 bash[33918]: cluster 2023-08-28T02:12:29.266089+0000 mgr.y (mgr.14901) 7616 : cluster [DBG] pgmap v7623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:30.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:29 smithi006 bash[27292]: cluster 2023-08-28T02:12:29.266089+0000 mgr.y (mgr.14901) 7616 : cluster [DBG] pgmap v7623: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:30.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:30 smithi130 bash[51302]: debug 2023-08-28T02:12:30.376+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:30.379794+0000) 2023-08-28T02:12:31.072 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:12:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:12:30] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:12:31.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:31 smithi130 bash[27579]: audit 2023-08-28T02:12:31.019277+0000 mon.a (mon.0) 4896 : audit [DBG] from='client.? 172.21.15.6:0/913033662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:31.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:31 smithi006 bash[33918]: audit 2023-08-28T02:12:31.019277+0000 mon.a (mon.0) 4896 : audit [DBG] from='client.? 172.21.15.6:0/913033662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:31.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:31 smithi006 bash[27292]: audit 2023-08-28T02:12:31.019277+0000 mon.a (mon.0) 4896 : audit [DBG] from='client.? 172.21.15.6:0/913033662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:12:31.235Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:12:31.237Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:31.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:31 smithi130 bash[51302]: debug 2023-08-28T02:12:31.376+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:31.380076+0000) 2023-08-28T02:12:32.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:32 smithi130 bash[27579]: cluster 2023-08-28T02:12:31.266751+0000 mgr.y (mgr.14901) 7617 : cluster [DBG] pgmap v7624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:32.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:32 smithi006 bash[33918]: cluster 2023-08-28T02:12:31.266751+0000 mgr.y (mgr.14901) 7617 : cluster [DBG] pgmap v7624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:32.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:32 smithi006 bash[27292]: cluster 2023-08-28T02:12:31.266751+0000 mgr.y (mgr.14901) 7617 : cluster [DBG] pgmap v7624: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:32.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:32 smithi130 bash[51302]: debug 2023-08-28T02:12:32.376+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:32.380348+0000) 2023-08-28T02:12:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:12:32.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:12:32.805Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:33.746 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:33 smithi130 bash[51302]: debug 2023-08-28T02:12:33.376+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:33.380630+0000) 2023-08-28T02:12:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:33 smithi130 bash[27579]: cluster 2023-08-28T02:12:33.267398+0000 mgr.y (mgr.14901) 7618 : cluster [DBG] pgmap v7625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:33 smithi130 bash[27579]: audit 2023-08-28T02:12:33.445288+0000 mon.c (mon.2) 2996 : audit [DBG] from='client.? 172.21.15.6:0/2476375864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:34.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:33 smithi006 bash[27292]: cluster 2023-08-28T02:12:33.267398+0000 mgr.y (mgr.14901) 7618 : cluster [DBG] pgmap v7625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:33 smithi006 bash[27292]: audit 2023-08-28T02:12:33.445288+0000 mon.c (mon.2) 2996 : audit [DBG] from='client.? 172.21.15.6:0/2476375864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:33 smithi006 bash[33918]: cluster 2023-08-28T02:12:33.267398+0000 mgr.y (mgr.14901) 7618 : cluster [DBG] pgmap v7625: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:33 smithi006 bash[33918]: audit 2023-08-28T02:12:33.445288+0000 mon.c (mon.2) 2996 : audit [DBG] from='client.? 172.21.15.6:0/2476375864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:34.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:34 smithi130 bash[51302]: debug 2023-08-28T02:12:34.376+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:34.380910+0000) 2023-08-28T02:12:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:12:34.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:12:34.683Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:35.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:35 smithi130 bash[51302]: debug 2023-08-28T02:12:35.376+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:35.381187+0000) 2023-08-28T02:12:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:36 smithi130 bash[27579]: cluster 2023-08-28T02:12:35.268588+0000 mgr.y (mgr.14901) 7619 : cluster [DBG] pgmap v7626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:36 smithi130 bash[27579]: audit 2023-08-28T02:12:35.854490+0000 mon.a (mon.0) 4897 : audit [DBG] from='client.? 172.21.15.6:0/1032938013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:36.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:36 smithi130 bash[51302]: debug 2023-08-28T02:12:36.376+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:36.381402+0000) 2023-08-28T02:12:36.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:36 smithi006 bash[27292]: cluster 2023-08-28T02:12:35.268588+0000 mgr.y (mgr.14901) 7619 : cluster [DBG] pgmap v7626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:36.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:36 smithi006 bash[27292]: audit 2023-08-28T02:12:35.854490+0000 mon.a (mon.0) 4897 : audit [DBG] from='client.? 172.21.15.6:0/1032938013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:36.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:36 smithi006 bash[33918]: cluster 2023-08-28T02:12:35.268588+0000 mgr.y (mgr.14901) 7619 : cluster [DBG] pgmap v7626: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:36.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:36 smithi006 bash[33918]: audit 2023-08-28T02:12:35.854490+0000 mon.a (mon.0) 4897 : audit [DBG] from='client.? 172.21.15.6:0/1032938013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:37.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:37 smithi130 bash[51302]: debug 2023-08-28T02:12:37.376+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:37.381670+0000) 2023-08-28T02:12:38.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:38 smithi130 bash[51302]: debug 2023-08-28T02:12:38.376+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:38.381990+0000) 2023-08-28T02:12:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:38 smithi130 bash[27579]: cluster 2023-08-28T02:12:37.269440+0000 mgr.y (mgr.14901) 7620 : cluster [DBG] pgmap v7627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:38 smithi130 bash[27579]: audit 2023-08-28T02:12:38.269460+0000 mon.c (mon.2) 2997 : audit [DBG] from='client.? 172.21.15.6:0/32390260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:38.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:38 smithi006 bash[27292]: cluster 2023-08-28T02:12:37.269440+0000 mgr.y (mgr.14901) 7620 : cluster [DBG] pgmap v7627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:38 smithi006 bash[27292]: audit 2023-08-28T02:12:38.269460+0000 mon.c (mon.2) 2997 : audit [DBG] from='client.? 172.21.15.6:0/32390260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:38.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:38 smithi006 bash[33918]: cluster 2023-08-28T02:12:37.269440+0000 mgr.y (mgr.14901) 7620 : cluster [DBG] pgmap v7627: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:38.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:38 smithi006 bash[33918]: audit 2023-08-28T02:12:38.269460+0000 mon.c (mon.2) 2997 : audit [DBG] from='client.? 172.21.15.6:0/32390260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:39.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:39 smithi130 bash[51302]: debug 2023-08-28T02:12:39.376+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:39.382243+0000) 2023-08-28T02:12:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:40 smithi130 bash[27579]: cluster 2023-08-28T02:12:39.270559+0000 mgr.y (mgr.14901) 7621 : cluster [DBG] pgmap v7628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:40.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:40 smithi130 bash[51302]: debug 2023-08-28T02:12:40.376+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:40.382512+0000) 2023-08-28T02:12:40.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:40 smithi006 bash[33918]: cluster 2023-08-28T02:12:39.270559+0000 mgr.y (mgr.14901) 7621 : cluster [DBG] pgmap v7628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:40.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:40 smithi006 bash[27292]: cluster 2023-08-28T02:12:39.270559+0000 mgr.y (mgr.14901) 7621 : cluster [DBG] pgmap v7628: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:12:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:12:40] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:12:41.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:41 smithi130 bash[27579]: audit 2023-08-28T02:12:40.697223+0000 mon.c (mon.2) 2998 : audit [DBG] from='client.? 172.21.15.6:0/2864410156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:41.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:41 smithi130 bash[51302]: debug 2023-08-28T02:12:41.376+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:41.382822+0000) 2023-08-28T02:12:41.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:41 smithi006 bash[27292]: audit 2023-08-28T02:12:40.697223+0000 mon.c (mon.2) 2998 : audit [DBG] from='client.? 172.21.15.6:0/2864410156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:41.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:41 smithi006 bash[33918]: audit 2023-08-28T02:12:40.697223+0000 mon.c (mon.2) 2998 : audit [DBG] from='client.? 172.21.15.6:0/2864410156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:12:41.236Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:12:41.238Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:42 smithi130 bash[27579]: cluster 2023-08-28T02:12:41.271242+0000 mgr.y (mgr.14901) 7622 : cluster [DBG] pgmap v7629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:42.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:42 smithi130 bash[51302]: debug 2023-08-28T02:12:42.380+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:42.383041+0000) 2023-08-28T02:12:42.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:42 smithi006 bash[27292]: cluster 2023-08-28T02:12:41.271242+0000 mgr.y (mgr.14901) 7622 : cluster [DBG] pgmap v7629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:42.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:42 smithi006 bash[33918]: cluster 2023-08-28T02:12:41.271242+0000 mgr.y (mgr.14901) 7622 : cluster [DBG] pgmap v7629: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:12:42.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:12:42.805Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:43.753 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:43 smithi130 bash[51302]: debug 2023-08-28T02:12:43.380+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:43.383264+0000) 2023-08-28T02:12:44.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:43 smithi130 bash[27579]: audit 2023-08-28T02:12:43.110062+0000 mon.c (mon.2) 2999 : audit [DBG] from='client.? 172.21.15.6:0/754000837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:44.123 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:43 smithi130 bash[27579]: cluster 2023-08-28T02:12:43.271903+0000 mgr.y (mgr.14901) 7623 : cluster [DBG] pgmap v7630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:44.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:43 smithi006 bash[33918]: audit 2023-08-28T02:12:43.110062+0000 mon.c (mon.2) 2999 : audit [DBG] from='client.? 172.21.15.6:0/754000837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:44.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:43 smithi006 bash[33918]: cluster 2023-08-28T02:12:43.271903+0000 mgr.y (mgr.14901) 7623 : cluster [DBG] pgmap v7630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:44.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:43 smithi006 bash[27292]: audit 2023-08-28T02:12:43.110062+0000 mon.c (mon.2) 2999 : audit [DBG] from='client.? 172.21.15.6:0/754000837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:44.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:43 smithi006 bash[27292]: cluster 2023-08-28T02:12:43.271903+0000 mgr.y (mgr.14901) 7623 : cluster [DBG] pgmap v7630: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:44.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:44 smithi130 bash[51302]: debug 2023-08-28T02:12:44.380+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:44.383598+0000) 2023-08-28T02:12:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:12:44.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:12:44.683Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:45.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:45 smithi130 bash[51302]: debug 2023-08-28T02:12:45.380+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:45.383853+0000) 2023-08-28T02:12:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:46 smithi130 bash[27579]: cluster 2023-08-28T02:12:45.272958+0000 mgr.y (mgr.14901) 7624 : cluster [DBG] pgmap v7631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:46 smithi130 bash[27579]: audit 2023-08-28T02:12:45.522686+0000 mon.c (mon.2) 3000 : audit [DBG] from='client.? 172.21.15.6:0/776305088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:46.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:46 smithi130 bash[51302]: debug 2023-08-28T02:12:46.380+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:46.384141+0000) 2023-08-28T02:12:46.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:46 smithi006 bash[27292]: cluster 2023-08-28T02:12:45.272958+0000 mgr.y (mgr.14901) 7624 : cluster [DBG] pgmap v7631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:46.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:46 smithi006 bash[27292]: audit 2023-08-28T02:12:45.522686+0000 mon.c (mon.2) 3000 : audit [DBG] from='client.? 172.21.15.6:0/776305088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:46.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:46 smithi006 bash[33918]: cluster 2023-08-28T02:12:45.272958+0000 mgr.y (mgr.14901) 7624 : cluster [DBG] pgmap v7631: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:46.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:46 smithi006 bash[33918]: audit 2023-08-28T02:12:45.522686+0000 mon.c (mon.2) 3000 : audit [DBG] from='client.? 172.21.15.6:0/776305088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:47.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:47 smithi130 bash[51302]: debug 2023-08-28T02:12:47.380+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:47.384408+0000) 2023-08-28T02:12:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:48 smithi130 bash[27579]: cluster 2023-08-28T02:12:47.273545+0000 mgr.y (mgr.14901) 7625 : cluster [DBG] pgmap v7632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:48 smithi130 bash[27579]: audit 2023-08-28T02:12:47.937033+0000 mon.c (mon.2) 3001 : audit [DBG] from='client.? 172.21.15.6:0/1676101771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:48.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:48 smithi130 bash[51302]: debug 2023-08-28T02:12:48.380+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:48.384745+0000) 2023-08-28T02:12:48.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:48 smithi006 bash[33918]: cluster 2023-08-28T02:12:47.273545+0000 mgr.y (mgr.14901) 7625 : cluster [DBG] pgmap v7632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:48.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:48 smithi006 bash[33918]: audit 2023-08-28T02:12:47.937033+0000 mon.c (mon.2) 3001 : audit [DBG] from='client.? 172.21.15.6:0/1676101771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:48 smithi006 bash[27292]: cluster 2023-08-28T02:12:47.273545+0000 mgr.y (mgr.14901) 7625 : cluster [DBG] pgmap v7632: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:48 smithi006 bash[27292]: audit 2023-08-28T02:12:47.937033+0000 mon.c (mon.2) 3001 : audit [DBG] from='client.? 172.21.15.6:0/1676101771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:49.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:49 smithi130 bash[51302]: debug 2023-08-28T02:12:49.384+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:49.385034+0000) 2023-08-28T02:12:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:50 smithi130 bash[27579]: cluster 2023-08-28T02:12:49.274630+0000 mgr.y (mgr.14901) 7626 : cluster [DBG] pgmap v7633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:50.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:50 smithi130 bash[51302]: debug 2023-08-28T02:12:50.380+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:50.385317+0000) 2023-08-28T02:12:50.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:50 smithi006 bash[27292]: cluster 2023-08-28T02:12:49.274630+0000 mgr.y (mgr.14901) 7626 : cluster [DBG] pgmap v7633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:50.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:50 smithi006 bash[33918]: cluster 2023-08-28T02:12:49.274630+0000 mgr.y (mgr.14901) 7626 : cluster [DBG] pgmap v7633: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:12:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:12:50] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:12:51.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:51 smithi130 bash[27579]: audit 2023-08-28T02:12:50.354863+0000 mon.c (mon.2) 3002 : audit [DBG] from='client.? 172.21.15.6:0/3779435216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:51.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:51 smithi130 bash[51302]: debug 2023-08-28T02:12:51.380+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:51.385587+0000) 2023-08-28T02:12:51.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:51 smithi006 bash[27292]: audit 2023-08-28T02:12:50.354863+0000 mon.c (mon.2) 3002 : audit [DBG] from='client.? 172.21.15.6:0/3779435216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:51.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:51 smithi006 bash[33918]: audit 2023-08-28T02:12:50.354863+0000 mon.c (mon.2) 3002 : audit [DBG] from='client.? 172.21.15.6:0/3779435216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:12:51.237Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:12:51.239Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:52 smithi130 bash[27579]: cluster 2023-08-28T02:12:51.275287+0000 mgr.y (mgr.14901) 7627 : cluster [DBG] pgmap v7634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:52.706 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:52 smithi130 bash[51302]: debug 2023-08-28T02:12:52.380+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:52.385899+0000) 2023-08-28T02:12:52.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:52 smithi006 bash[33918]: cluster 2023-08-28T02:12:51.275287+0000 mgr.y (mgr.14901) 7627 : cluster [DBG] pgmap v7634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:52.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:52 smithi006 bash[27292]: cluster 2023-08-28T02:12:51.275287+0000 mgr.y (mgr.14901) 7627 : cluster [DBG] pgmap v7634: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:12:52.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:12:52.806Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:53.747 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:53 smithi130 bash[51302]: debug 2023-08-28T02:12:53.380+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:53.386129+0000) 2023-08-28T02:12:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:53 smithi130 bash[27579]: audit 2023-08-28T02:12:52.768990+0000 mon.c (mon.2) 3003 : audit [DBG] from='client.? 172.21.15.6:0/1881598506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:53 smithi130 bash[27579]: cluster 2023-08-28T02:12:53.275955+0000 mgr.y (mgr.14901) 7628 : cluster [DBG] pgmap v7635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:54.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:53 smithi006 bash[33918]: audit 2023-08-28T02:12:52.768990+0000 mon.c (mon.2) 3003 : audit [DBG] from='client.? 172.21.15.6:0/1881598506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:54.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:53 smithi006 bash[33918]: cluster 2023-08-28T02:12:53.275955+0000 mgr.y (mgr.14901) 7628 : cluster [DBG] pgmap v7635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:54.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:53 smithi006 bash[27292]: audit 2023-08-28T02:12:52.768990+0000 mon.c (mon.2) 3003 : audit [DBG] from='client.? 172.21.15.6:0/1881598506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:54.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:53 smithi006 bash[27292]: cluster 2023-08-28T02:12:53.275955+0000 mgr.y (mgr.14901) 7628 : cluster [DBG] pgmap v7635: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:54.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:54 smithi130 bash[51302]: debug 2023-08-28T02:12:54.380+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:54.386432+0000) 2023-08-28T02:12:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:12:54.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:12:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:12:54.684Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:12:55.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:55 smithi130 bash[51302]: debug 2023-08-28T02:12:55.384+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:55.386693+0000) 2023-08-28T02:12:55.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:55 smithi130 bash[27579]: audit 2023-08-28T02:12:55.185280+0000 mon.a (mon.0) 4898 : audit [DBG] from='client.? 172.21.15.6:0/5900205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:55.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:55 smithi006 bash[27292]: audit 2023-08-28T02:12:55.185280+0000 mon.a (mon.0) 4898 : audit [DBG] from='client.? 172.21.15.6:0/5900205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:55.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:55 smithi006 bash[33918]: audit 2023-08-28T02:12:55.185280+0000 mon.a (mon.0) 4898 : audit [DBG] from='client.? 172.21.15.6:0/5900205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:56 smithi130 bash[27579]: cluster 2023-08-28T02:12:55.277097+0000 mgr.y (mgr.14901) 7629 : cluster [DBG] pgmap v7636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:56.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:56 smithi130 bash[51302]: debug 2023-08-28T02:12:56.384+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:56.386993+0000) 2023-08-28T02:12:56.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:56 smithi006 bash[33918]: cluster 2023-08-28T02:12:55.277097+0000 mgr.y (mgr.14901) 7629 : cluster [DBG] pgmap v7636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:56.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:56 smithi006 bash[27292]: cluster 2023-08-28T02:12:55.277097+0000 mgr.y (mgr.14901) 7629 : cluster [DBG] pgmap v7636: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:57.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:57 smithi130 bash[51302]: debug 2023-08-28T02:12:57.384+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:57.387214+0000) 2023-08-28T02:12:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:58 smithi130 bash[27579]: cluster 2023-08-28T02:12:57.277994+0000 mgr.y (mgr.14901) 7630 : cluster [DBG] pgmap v7637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:12:58 smithi130 bash[27579]: audit 2023-08-28T02:12:57.593119+0000 mon.a (mon.0) 4899 : audit [DBG] from='client.? 172.21.15.6:0/3354087555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:58.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:58 smithi130 bash[51302]: debug 2023-08-28T02:12:58.384+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:58.387524+0000) 2023-08-28T02:12:58.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:58 smithi006 bash[33918]: cluster 2023-08-28T02:12:57.277994+0000 mgr.y (mgr.14901) 7630 : cluster [DBG] pgmap v7637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:58.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:12:58 smithi006 bash[33918]: audit 2023-08-28T02:12:57.593119+0000 mon.a (mon.0) 4899 : audit [DBG] from='client.? 172.21.15.6:0/3354087555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:58.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:58 smithi006 bash[27292]: cluster 2023-08-28T02:12:57.277994+0000 mgr.y (mgr.14901) 7630 : cluster [DBG] pgmap v7637: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:12:58.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:12:58 smithi006 bash[27292]: audit 2023-08-28T02:12:57.593119+0000 mon.a (mon.0) 4899 : audit [DBG] from='client.? 172.21.15.6:0/3354087555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:59.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:12:59 smithi130 bash[51302]: debug 2023-08-28T02:12:59.384+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:59.387804+0000) 2023-08-28T02:13:00.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:00 smithi130 bash[51302]: debug 2023-08-28T02:13:00.384+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:00.388104+0000) 2023-08-28T02:13:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:00 smithi130 bash[27579]: cluster 2023-08-28T02:12:59.279198+0000 mgr.y (mgr.14901) 7631 : cluster [DBG] pgmap v7638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:00 smithi130 bash[27579]: audit 2023-08-28T02:13:00.005253+0000 mon.a (mon.0) 4900 : audit [DBG] from='client.? 172.21.15.6:0/706853575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:00 smithi006 bash[27292]: cluster 2023-08-28T02:12:59.279198+0000 mgr.y (mgr.14901) 7631 : cluster [DBG] pgmap v7638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:00 smithi006 bash[27292]: audit 2023-08-28T02:13:00.005253+0000 mon.a (mon.0) 4900 : audit [DBG] from='client.? 172.21.15.6:0/706853575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:00 smithi006 bash[33918]: cluster 2023-08-28T02:12:59.279198+0000 mgr.y (mgr.14901) 7631 : cluster [DBG] pgmap v7638: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:00 smithi006 bash[33918]: audit 2023-08-28T02:13:00.005253+0000 mon.a (mon.0) 4900 : audit [DBG] from='client.? 172.21.15.6:0/706853575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:13:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:13:00] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T02:13:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:13:01.238Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:13:01.240Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:01.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:01 smithi130 bash[51302]: debug 2023-08-28T02:13:01.384+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:01.388409+0000) 2023-08-28T02:13:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:02 smithi130 bash[27579]: cluster 2023-08-28T02:13:01.279932+0000 mgr.y (mgr.14901) 7632 : cluster [DBG] pgmap v7639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:02.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:02 smithi130 bash[51302]: debug 2023-08-28T02:13:02.384+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:02.388620+0000) 2023-08-28T02:13:02.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:02 smithi006 bash[27292]: cluster 2023-08-28T02:13:01.279932+0000 mgr.y (mgr.14901) 7632 : cluster [DBG] pgmap v7639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:02.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:02 smithi006 bash[33918]: cluster 2023-08-28T02:13:01.279932+0000 mgr.y (mgr.14901) 7632 : cluster [DBG] pgmap v7639: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:13:02.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:13:02.807Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:03.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:03 smithi130 bash[27579]: audit 2023-08-28T02:13:02.429409+0000 mon.a (mon.0) 4901 : audit [DBG] from='client.? 172.21.15.6:0/662730958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:03.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:03 smithi130 bash[51302]: debug 2023-08-28T02:13:03.384+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:03.388928+0000) 2023-08-28T02:13:03.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:03 smithi006 bash[33918]: audit 2023-08-28T02:13:02.429409+0000 mon.a (mon.0) 4901 : audit [DBG] from='client.? 172.21.15.6:0/662730958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:03.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:03 smithi006 bash[27292]: audit 2023-08-28T02:13:02.429409+0000 mon.a (mon.0) 4901 : audit [DBG] from='client.? 172.21.15.6:0/662730958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:04 smithi130 bash[27579]: cluster 2023-08-28T02:13:03.280665+0000 mgr.y (mgr.14901) 7633 : cluster [DBG] pgmap v7640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:04.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:04 smithi130 bash[51302]: debug 2023-08-28T02:13:04.384+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:04.389203+0000) 2023-08-28T02:13:04.682 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:04 smithi006 bash[33918]: cluster 2023-08-28T02:13:03.280665+0000 mgr.y (mgr.14901) 7633 : cluster [DBG] pgmap v7640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:04.682 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:04 smithi006 bash[27292]: cluster 2023-08-28T02:13:03.280665+0000 mgr.y (mgr.14901) 7633 : cluster [DBG] pgmap v7640: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:13:04.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:13:04.684Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:05.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:05 smithi130 bash[27579]: audit 2023-08-28T02:13:04.847184+0000 mon.c (mon.2) 3004 : audit [DBG] from='client.? 172.21.15.6:0/3181305572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:05.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:05 smithi130 bash[51302]: debug 2023-08-28T02:13:05.384+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:05.389477+0000) 2023-08-28T02:13:05.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:05 smithi006 bash[33918]: audit 2023-08-28T02:13:04.847184+0000 mon.c (mon.2) 3004 : audit [DBG] from='client.? 172.21.15.6:0/3181305572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:05.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:05 smithi006 bash[27292]: audit 2023-08-28T02:13:04.847184+0000 mon.c (mon.2) 3004 : audit [DBG] from='client.? 172.21.15.6:0/3181305572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:06 smithi006 bash[33918]: cluster 2023-08-28T02:13:05.281830+0000 mgr.y (mgr.14901) 7634 : cluster [DBG] pgmap v7641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:06 smithi006 bash[27292]: cluster 2023-08-28T02:13:05.281830+0000 mgr.y (mgr.14901) 7634 : cluster [DBG] pgmap v7641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:06.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:06 smithi130 bash[27579]: cluster 2023-08-28T02:13:05.281830+0000 mgr.y (mgr.14901) 7634 : cluster [DBG] pgmap v7641: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:06.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:06 smithi130 bash[51302]: debug 2023-08-28T02:13:06.384+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:06.389745+0000) 2023-08-28T02:13:07.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:07 smithi006 bash[27292]: audit 2023-08-28T02:13:07.266216+0000 mon.a (mon.0) 4902 : audit [DBG] from='client.? 172.21.15.6:0/1588911620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:07.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:07 smithi006 bash[33918]: audit 2023-08-28T02:13:07.266216+0000 mon.a (mon.0) 4902 : audit [DBG] from='client.? 172.21.15.6:0/1588911620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:07.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:07 smithi130 bash[27579]: audit 2023-08-28T02:13:07.266216+0000 mon.a (mon.0) 4902 : audit [DBG] from='client.? 172.21.15.6:0/1588911620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:07.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:07 smithi130 bash[51302]: debug 2023-08-28T02:13:07.384+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:07.390055+0000) 2023-08-28T02:13:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:08 smithi006 bash[33918]: cluster 2023-08-28T02:13:07.282387+0000 mgr.y (mgr.14901) 7635 : cluster [DBG] pgmap v7642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:08 smithi006 bash[27292]: cluster 2023-08-28T02:13:07.282387+0000 mgr.y (mgr.14901) 7635 : cluster [DBG] pgmap v7642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:08 smithi130 bash[27579]: cluster 2023-08-28T02:13:07.282387+0000 mgr.y (mgr.14901) 7635 : cluster [DBG] pgmap v7642: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:08.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:08 smithi130 bash[51302]: debug 2023-08-28T02:13:08.384+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:08.390337+0000) 2023-08-28T02:13:09.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:09 smithi130 bash[51302]: debug 2023-08-28T02:13:09.384+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:09.390550+0000) 2023-08-28T02:13:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:10 smithi006 bash[33918]: cluster 2023-08-28T02:13:09.283423+0000 mgr.y (mgr.14901) 7636 : cluster [DBG] pgmap v7643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:10 smithi006 bash[33918]: audit 2023-08-28T02:13:09.678795+0000 mon.c (mon.2) 3005 : audit [DBG] from='client.? 172.21.15.6:0/2938471123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:10 smithi006 bash[27292]: cluster 2023-08-28T02:13:09.283423+0000 mgr.y (mgr.14901) 7636 : cluster [DBG] pgmap v7643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:10 smithi006 bash[27292]: audit 2023-08-28T02:13:09.678795+0000 mon.c (mon.2) 3005 : audit [DBG] from='client.? 172.21.15.6:0/2938471123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:10.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:10 smithi130 bash[27579]: cluster 2023-08-28T02:13:09.283423+0000 mgr.y (mgr.14901) 7636 : cluster [DBG] pgmap v7643: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:10.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:10 smithi130 bash[27579]: audit 2023-08-28T02:13:09.678795+0000 mon.c (mon.2) 3005 : audit [DBG] from='client.? 172.21.15.6:0/2938471123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:10.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:10 smithi130 bash[51302]: debug 2023-08-28T02:13:10.384+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:10.390815+0000) 2023-08-28T02:13:11.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:13:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:13:10] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T02:13:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:13:11.239Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:13:11.241Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:11.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:11 smithi130 bash[51302]: debug 2023-08-28T02:13:11.388+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:11.391090+0000) 2023-08-28T02:13:12.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:12 smithi006 bash[33918]: cluster 2023-08-28T02:13:11.284103+0000 mgr.y (mgr.14901) 7637 : cluster [DBG] pgmap v7644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:12.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:12 smithi006 bash[33918]: audit 2023-08-28T02:13:12.090503+0000 mon.a (mon.0) 4903 : audit [DBG] from='client.? 172.21.15.6:0/175073684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:12.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:12 smithi006 bash[27292]: cluster 2023-08-28T02:13:11.284103+0000 mgr.y (mgr.14901) 7637 : cluster [DBG] pgmap v7644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:12.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:12 smithi006 bash[27292]: audit 2023-08-28T02:13:12.090503+0000 mon.a (mon.0) 4903 : audit [DBG] from='client.? 172.21.15.6:0/175073684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:12 smithi130 bash[27579]: cluster 2023-08-28T02:13:11.284103+0000 mgr.y (mgr.14901) 7637 : cluster [DBG] pgmap v7644: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:12 smithi130 bash[27579]: audit 2023-08-28T02:13:12.090503+0000 mon.a (mon.0) 4903 : audit [DBG] from='client.? 172.21.15.6:0/175073684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:12.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:12 smithi130 bash[51302]: debug 2023-08-28T02:13:12.388+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:12.391308+0000) 2023-08-28T02:13:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:13:12.806Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:13:12.810Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:13.767 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:13 smithi130 bash[51302]: debug 2023-08-28T02:13:13.388+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:13.391604+0000) 2023-08-28T02:13:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:13 smithi130 bash[27579]: cluster 2023-08-28T02:13:13.284768+0000 mgr.y (mgr.14901) 7638 : cluster [DBG] pgmap v7645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:13 smithi130 bash[27579]: audit 2023-08-28T02:13:13.582947+0000 mon.b (mon.1) 843 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:13:14.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:13 smithi006 bash[33918]: cluster 2023-08-28T02:13:13.284768+0000 mgr.y (mgr.14901) 7638 : cluster [DBG] pgmap v7645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:13 smithi006 bash[33918]: audit 2023-08-28T02:13:13.582947+0000 mon.b (mon.1) 843 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:13:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:13 smithi006 bash[27292]: cluster 2023-08-28T02:13:13.284768+0000 mgr.y (mgr.14901) 7638 : cluster [DBG] pgmap v7645: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:13 smithi006 bash[27292]: audit 2023-08-28T02:13:13.582947+0000 mon.b (mon.1) 843 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:13:14.770 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:14 smithi130 bash[51302]: debug 2023-08-28T02:13:14.388+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:14.391923+0000) 2023-08-28T02:13:14.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:14 smithi006 bash[27292]: audit 2023-08-28T02:13:14.509454+0000 mon.c (mon.2) 3006 : audit [DBG] from='client.? 172.21.15.6:0/2050730118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:14.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:14 smithi006 bash[33918]: audit 2023-08-28T02:13:14.509454+0000 mon.c (mon.2) 3006 : audit [DBG] from='client.? 172.21.15.6:0/2050730118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:13:14.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:13:14.685Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:15.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:14 smithi130 bash[27579]: audit 2023-08-28T02:13:14.509454+0000 mon.c (mon.2) 3006 : audit [DBG] from='client.? 172.21.15.6:0/2050730118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:15.780 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:15 smithi130 bash[51302]: debug 2023-08-28T02:13:15.388+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:15.392192+0000) 2023-08-28T02:13:16.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:15 smithi130 bash[27579]: cluster 2023-08-28T02:13:15.285856+0000 mgr.y (mgr.14901) 7639 : cluster [DBG] pgmap v7646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:16.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:15 smithi006 bash[27292]: cluster 2023-08-28T02:13:15.285856+0000 mgr.y (mgr.14901) 7639 : cluster [DBG] pgmap v7646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:16.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:15 smithi006 bash[33918]: cluster 2023-08-28T02:13:15.285856+0000 mgr.y (mgr.14901) 7639 : cluster [DBG] pgmap v7646: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:16.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:16 smithi130 bash[51302]: debug 2023-08-28T02:13:16.388+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:16.392417+0000) 2023-08-28T02:13:17.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:16 smithi130 bash[27579]: audit 2023-08-28T02:13:16.935835+0000 mon.c (mon.2) 3007 : audit [DBG] from='client.? 172.21.15.6:0/2566829616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:17.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:16 smithi006 bash[33918]: audit 2023-08-28T02:13:16.935835+0000 mon.c (mon.2) 3007 : audit [DBG] from='client.? 172.21.15.6:0/2566829616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:17.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:16 smithi006 bash[27292]: audit 2023-08-28T02:13:16.935835+0000 mon.c (mon.2) 3007 : audit [DBG] from='client.? 172.21.15.6:0/2566829616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:17.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:17 smithi130 bash[51302]: debug 2023-08-28T02:13:17.388+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:17.392697+0000) 2023-08-28T02:13:18.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:17 smithi130 bash[27579]: cluster 2023-08-28T02:13:17.286478+0000 mgr.y (mgr.14901) 7640 : cluster [DBG] pgmap v7647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:18.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:17 smithi006 bash[33918]: cluster 2023-08-28T02:13:17.286478+0000 mgr.y (mgr.14901) 7640 : cluster [DBG] pgmap v7647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:18.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:17 smithi006 bash[27292]: cluster 2023-08-28T02:13:17.286478+0000 mgr.y (mgr.14901) 7640 : cluster [DBG] pgmap v7647: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:18.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:18 smithi130 bash[51302]: debug 2023-08-28T02:13:18.388+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:18.392993+0000) 2023-08-28T02:13:19.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:19 smithi130 bash[51302]: debug 2023-08-28T02:13:19.388+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:19.393178+0000) 2023-08-28T02:13:20.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:20 smithi130 bash[51302]: debug 2023-08-28T02:13:20.387+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:20.393428+0000) 2023-08-28T02:13:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:20 smithi130 bash[27579]: cluster 2023-08-28T02:13:19.287161+0000 mgr.y (mgr.14901) 7641 : cluster [DBG] pgmap v7648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:20 smithi130 bash[27579]: audit 2023-08-28T02:13:19.385719+0000 mon.a (mon.0) 4904 : audit [DBG] from='client.? 172.21.15.6:0/1981145300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:20 smithi130 bash[27579]: audit 2023-08-28T02:13:19.483431+0000 mon.a (mon.0) 4905 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:13:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:20 smithi130 bash[27579]: audit 2023-08-28T02:13:19.934429+0000 mon.a (mon.0) 4906 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:13:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:20 smithi130 bash[27579]: audit 2023-08-28T02:13:20.206461+0000 mon.b (mon.1) 844 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:13:20.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:20 smithi130 bash[27579]: audit 2023-08-28T02:13:20.208779+0000 mon.b (mon.1) 845 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:13:20.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:20 smithi130 bash[27579]: audit 2023-08-28T02:13:20.218529+0000 mon.a (mon.0) 4907 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:13:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:20 smithi006 bash[33918]: cluster 2023-08-28T02:13:19.287161+0000 mgr.y (mgr.14901) 7641 : cluster [DBG] pgmap v7648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:20 smithi006 bash[33918]: audit 2023-08-28T02:13:19.385719+0000 mon.a (mon.0) 4904 : audit [DBG] from='client.? 172.21.15.6:0/1981145300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:20 smithi006 bash[33918]: audit 2023-08-28T02:13:19.483431+0000 mon.a (mon.0) 4905 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:13:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:20 smithi006 bash[33918]: audit 2023-08-28T02:13:19.934429+0000 mon.a (mon.0) 4906 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:13:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:20 smithi006 bash[33918]: audit 2023-08-28T02:13:20.206461+0000 mon.b (mon.1) 844 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:13:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:20 smithi006 bash[33918]: audit 2023-08-28T02:13:20.208779+0000 mon.b (mon.1) 845 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:13:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:20 smithi006 bash[33918]: audit 2023-08-28T02:13:20.218529+0000 mon.a (mon.0) 4907 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:13:20.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:20 smithi006 bash[27292]: cluster 2023-08-28T02:13:19.287161+0000 mgr.y (mgr.14901) 7641 : cluster [DBG] pgmap v7648: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:20.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:20 smithi006 bash[27292]: audit 2023-08-28T02:13:19.385719+0000 mon.a (mon.0) 4904 : audit [DBG] from='client.? 172.21.15.6:0/1981145300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:20 smithi006 bash[27292]: audit 2023-08-28T02:13:19.483431+0000 mon.a (mon.0) 4905 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:13:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:20 smithi006 bash[27292]: audit 2023-08-28T02:13:19.934429+0000 mon.a (mon.0) 4906 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:13:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:20 smithi006 bash[27292]: audit 2023-08-28T02:13:20.206461+0000 mon.b (mon.1) 844 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:13:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:20 smithi006 bash[27292]: audit 2023-08-28T02:13:20.208779+0000 mon.b (mon.1) 845 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:13:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:20 smithi006 bash[27292]: audit 2023-08-28T02:13:20.218529+0000 mon.a (mon.0) 4907 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:13:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:13:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:13:20] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T02:13:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:13:21.239Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:13:21.241Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:21.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:21 smithi130 bash[51302]: debug 2023-08-28T02:13:21.387+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:21.393737+0000) 2023-08-28T02:13:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:22 smithi130 bash[27579]: cluster 2023-08-28T02:13:21.287777+0000 mgr.y (mgr.14901) 7642 : cluster [DBG] pgmap v7649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:22 smithi130 bash[27579]: audit 2023-08-28T02:13:21.799447+0000 mon.a (mon.0) 4908 : audit [DBG] from='client.? 172.21.15.6:0/2354748519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:22.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:22 smithi130 bash[51302]: debug 2023-08-28T02:13:22.387+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:22.394018+0000) 2023-08-28T02:13:22.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:22 smithi006 bash[27292]: cluster 2023-08-28T02:13:21.287777+0000 mgr.y (mgr.14901) 7642 : cluster [DBG] pgmap v7649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:22.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:22 smithi006 bash[27292]: audit 2023-08-28T02:13:21.799447+0000 mon.a (mon.0) 4908 : audit [DBG] from='client.? 172.21.15.6:0/2354748519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:22.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:22 smithi006 bash[33918]: cluster 2023-08-28T02:13:21.287777+0000 mgr.y (mgr.14901) 7642 : cluster [DBG] pgmap v7649: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:22.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:22 smithi006 bash[33918]: audit 2023-08-28T02:13:21.799447+0000 mon.a (mon.0) 4908 : audit [DBG] from='client.? 172.21.15.6:0/2354748519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:13:22.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:13:22.809Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:23.767 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:23 smithi130 bash[51302]: debug 2023-08-28T02:13:23.387+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:23.394307+0000) 2023-08-28T02:13:24.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:23 smithi130 bash[27579]: cluster 2023-08-28T02:13:23.288442+0000 mgr.y (mgr.14901) 7643 : cluster [DBG] pgmap v7650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:24.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:23 smithi006 bash[33918]: cluster 2023-08-28T02:13:23.288442+0000 mgr.y (mgr.14901) 7643 : cluster [DBG] pgmap v7650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:23 smithi006 bash[27292]: cluster 2023-08-28T02:13:23.288442+0000 mgr.y (mgr.14901) 7643 : cluster [DBG] pgmap v7650: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:24.770 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:24 smithi130 bash[51302]: debug 2023-08-28T02:13:24.387+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:24.394556+0000) 2023-08-28T02:13:24.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:24 smithi006 bash[27292]: audit 2023-08-28T02:13:24.213157+0000 mon.a (mon.0) 4909 : audit [DBG] from='client.? 172.21.15.6:0/1723165515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:24.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:24 smithi006 bash[33918]: audit 2023-08-28T02:13:24.213157+0000 mon.a (mon.0) 4909 : audit [DBG] from='client.? 172.21.15.6:0/1723165515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:13:24.684Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:13:24.686Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:25.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:24 smithi130 bash[27579]: audit 2023-08-28T02:13:24.213157+0000 mon.a (mon.0) 4909 : audit [DBG] from='client.? 172.21.15.6:0/1723165515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:25.778 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:25 smithi130 bash[51302]: debug 2023-08-28T02:13:25.391+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:25.394908+0000) 2023-08-28T02:13:26.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:25 smithi130 bash[27579]: cluster 2023-08-28T02:13:25.289586+0000 mgr.y (mgr.14901) 7644 : cluster [DBG] pgmap v7651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:26.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:25 smithi006 bash[27292]: cluster 2023-08-28T02:13:25.289586+0000 mgr.y (mgr.14901) 7644 : cluster [DBG] pgmap v7651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:26.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:25 smithi006 bash[33918]: cluster 2023-08-28T02:13:25.289586+0000 mgr.y (mgr.14901) 7644 : cluster [DBG] pgmap v7651: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:26.781 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:26 smithi130 bash[51302]: debug 2023-08-28T02:13:26.391+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:26.395183+0000) 2023-08-28T02:13:27.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:26 smithi130 bash[27579]: audit 2023-08-28T02:13:26.630401+0000 mon.c (mon.2) 3008 : audit [DBG] from='client.? 172.21.15.6:0/3746641299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:27.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:26 smithi006 bash[27292]: audit 2023-08-28T02:13:26.630401+0000 mon.c (mon.2) 3008 : audit [DBG] from='client.? 172.21.15.6:0/3746641299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:27.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:26 smithi006 bash[33918]: audit 2023-08-28T02:13:26.630401+0000 mon.c (mon.2) 3008 : audit [DBG] from='client.? 172.21.15.6:0/3746641299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:27.791 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:27 smithi130 bash[51302]: debug 2023-08-28T02:13:27.391+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:27.395476+0000) 2023-08-28T02:13:28.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:27 smithi130 bash[27579]: cluster 2023-08-28T02:13:27.290249+0000 mgr.y (mgr.14901) 7645 : cluster [DBG] pgmap v7652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:28.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:27 smithi006 bash[33918]: cluster 2023-08-28T02:13:27.290249+0000 mgr.y (mgr.14901) 7645 : cluster [DBG] pgmap v7652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:28.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:27 smithi006 bash[27292]: cluster 2023-08-28T02:13:27.290249+0000 mgr.y (mgr.14901) 7645 : cluster [DBG] pgmap v7652: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:28.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:28 smithi130 bash[51302]: debug 2023-08-28T02:13:28.391+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:28.395746+0000) 2023-08-28T02:13:29.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:29 smithi130 bash[27579]: audit 2023-08-28T02:13:29.056275+0000 mon.c (mon.2) 3009 : audit [DBG] from='client.? 172.21.15.6:0/3868478829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:29.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:29 smithi006 bash[33918]: audit 2023-08-28T02:13:29.056275+0000 mon.c (mon.2) 3009 : audit [DBG] from='client.? 172.21.15.6:0/3868478829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:29.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:29 smithi006 bash[27292]: audit 2023-08-28T02:13:29.056275+0000 mon.c (mon.2) 3009 : audit [DBG] from='client.? 172.21.15.6:0/3868478829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:29.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:29 smithi130 bash[51302]: debug 2023-08-28T02:13:29.391+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:29.396069+0000) 2023-08-28T02:13:30.394 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:30 smithi130 bash[27579]: cluster 2023-08-28T02:13:29.291321+0000 mgr.y (mgr.14901) 7646 : cluster [DBG] pgmap v7653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:30.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:30 smithi006 bash[33918]: cluster 2023-08-28T02:13:29.291321+0000 mgr.y (mgr.14901) 7646 : cluster [DBG] pgmap v7653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:30.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:30 smithi006 bash[27292]: cluster 2023-08-28T02:13:29.291321+0000 mgr.y (mgr.14901) 7646 : cluster [DBG] pgmap v7653: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:30.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:30 smithi130 bash[51302]: debug 2023-08-28T02:13:30.395+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:30.396375+0000) 2023-08-28T02:13:31.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:13:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:13:30] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T02:13:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:13:31.240Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:13:31.242Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:31.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:31 smithi130 bash[51302]: debug 2023-08-28T02:13:31.391+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:31.396644+0000) 2023-08-28T02:13:32.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:32 smithi130 bash[51302]: debug 2023-08-28T02:13:32.391+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:32.396906+0000) 2023-08-28T02:13:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:32 smithi130 bash[27579]: cluster 2023-08-28T02:13:31.291892+0000 mgr.y (mgr.14901) 7647 : cluster [DBG] pgmap v7654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:32 smithi130 bash[27579]: audit 2023-08-28T02:13:31.472333+0000 mon.c (mon.2) 3010 : audit [DBG] from='client.? 172.21.15.6:0/836695994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:32.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:32 smithi006 bash[33918]: cluster 2023-08-28T02:13:31.291892+0000 mgr.y (mgr.14901) 7647 : cluster [DBG] pgmap v7654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:32.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:32 smithi006 bash[33918]: audit 2023-08-28T02:13:31.472333+0000 mon.c (mon.2) 3010 : audit [DBG] from='client.? 172.21.15.6:0/836695994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:32.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:32 smithi006 bash[27292]: cluster 2023-08-28T02:13:31.291892+0000 mgr.y (mgr.14901) 7647 : cluster [DBG] pgmap v7654: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:32.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:32 smithi006 bash[27292]: audit 2023-08-28T02:13:31.472333+0000 mon.c (mon.2) 3010 : audit [DBG] from='client.? 172.21.15.6:0/836695994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:13:32.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:13:32.810Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:33.769 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:33 smithi130 bash[51302]: debug 2023-08-28T02:13:33.395+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:33.397172+0000) 2023-08-28T02:13:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:33 smithi130 bash[27579]: cluster 2023-08-28T02:13:33.292468+0000 mgr.y (mgr.14901) 7648 : cluster [DBG] pgmap v7655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:34.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:33 smithi006 bash[33918]: cluster 2023-08-28T02:13:33.292468+0000 mgr.y (mgr.14901) 7648 : cluster [DBG] pgmap v7655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:33 smithi006 bash[27292]: cluster 2023-08-28T02:13:33.292468+0000 mgr.y (mgr.14901) 7648 : cluster [DBG] pgmap v7655: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:34.772 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:34 smithi130 bash[51302]: debug 2023-08-28T02:13:34.395+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:34.397511+0000) 2023-08-28T02:13:34.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:34 smithi006 bash[33918]: audit 2023-08-28T02:13:33.881050+0000 mon.c (mon.2) 3011 : audit [DBG] from='client.? 172.21.15.6:0/2038183452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:34.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:34 smithi006 bash[27292]: audit 2023-08-28T02:13:33.881050+0000 mon.c (mon.2) 3011 : audit [DBG] from='client.? 172.21.15.6:0/2038183452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:13:34.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:13:34.687Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:35.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:34 smithi130 bash[27579]: audit 2023-08-28T02:13:33.881050+0000 mon.c (mon.2) 3011 : audit [DBG] from='client.? 172.21.15.6:0/2038183452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:35.779 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:35 smithi130 bash[51302]: debug 2023-08-28T02:13:35.391+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:35.397796+0000) 2023-08-28T02:13:36.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:35 smithi130 bash[27579]: cluster 2023-08-28T02:13:35.293601+0000 mgr.y (mgr.14901) 7649 : cluster [DBG] pgmap v7656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:36.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:35 smithi006 bash[33918]: cluster 2023-08-28T02:13:35.293601+0000 mgr.y (mgr.14901) 7649 : cluster [DBG] pgmap v7656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:36.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:35 smithi006 bash[27292]: cluster 2023-08-28T02:13:35.293601+0000 mgr.y (mgr.14901) 7649 : cluster [DBG] pgmap v7656: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:36.783 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:36 smithi130 bash[51302]: debug 2023-08-28T02:13:36.395+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:36.398073+0000) 2023-08-28T02:13:37.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:36 smithi130 bash[27579]: audit 2023-08-28T02:13:36.298389+0000 mon.c (mon.2) 3012 : audit [DBG] from='client.? 172.21.15.6:0/915667645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:37.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:36 smithi006 bash[27292]: audit 2023-08-28T02:13:36.298389+0000 mon.c (mon.2) 3012 : audit [DBG] from='client.? 172.21.15.6:0/915667645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:37.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:36 smithi006 bash[33918]: audit 2023-08-28T02:13:36.298389+0000 mon.c (mon.2) 3012 : audit [DBG] from='client.? 172.21.15.6:0/915667645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:37.789 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:37 smithi130 bash[51302]: debug 2023-08-28T02:13:37.395+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:37.398376+0000) 2023-08-28T02:13:38.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:37 smithi130 bash[27579]: cluster 2023-08-28T02:13:37.294270+0000 mgr.y (mgr.14901) 7650 : cluster [DBG] pgmap v7657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:38.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:37 smithi006 bash[33918]: cluster 2023-08-28T02:13:37.294270+0000 mgr.y (mgr.14901) 7650 : cluster [DBG] pgmap v7657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:38.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:37 smithi006 bash[27292]: cluster 2023-08-28T02:13:37.294270+0000 mgr.y (mgr.14901) 7650 : cluster [DBG] pgmap v7657: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:38.792 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:38 smithi130 bash[51302]: debug 2023-08-28T02:13:38.395+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:38.398675+0000) 2023-08-28T02:13:39.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:38 smithi130 bash[27579]: audit 2023-08-28T02:13:38.715107+0000 mon.c (mon.2) 3013 : audit [DBG] from='client.? 172.21.15.6:0/325127222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:39.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:38 smithi006 bash[33918]: audit 2023-08-28T02:13:38.715107+0000 mon.c (mon.2) 3013 : audit [DBG] from='client.? 172.21.15.6:0/325127222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:39.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:38 smithi006 bash[27292]: audit 2023-08-28T02:13:38.715107+0000 mon.c (mon.2) 3013 : audit [DBG] from='client.? 172.21.15.6:0/325127222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:39.803 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:39 smithi130 bash[51302]: debug 2023-08-28T02:13:39.395+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:39.398976+0000) 2023-08-28T02:13:40.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:39 smithi130 bash[27579]: cluster 2023-08-28T02:13:39.295458+0000 mgr.y (mgr.14901) 7651 : cluster [DBG] pgmap v7658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:40.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:39 smithi006 bash[33918]: cluster 2023-08-28T02:13:39.295458+0000 mgr.y (mgr.14901) 7651 : cluster [DBG] pgmap v7658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:40.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:39 smithi006 bash[27292]: cluster 2023-08-28T02:13:39.295458+0000 mgr.y (mgr.14901) 7651 : cluster [DBG] pgmap v7658: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:40.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:40 smithi130 bash[51302]: debug 2023-08-28T02:13:40.395+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:40.399199+0000) 2023-08-28T02:13:41.178 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:13:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:13:40] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T02:13:41.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:41 smithi006 bash[33918]: audit 2023-08-28T02:13:41.124917+0000 mon.a (mon.0) 4910 : audit [DBG] from='client.? 172.21.15.6:0/3879285490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:41.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:41 smithi006 bash[27292]: audit 2023-08-28T02:13:41.124917+0000 mon.a (mon.0) 4910 : audit [DBG] from='client.? 172.21.15.6:0/3879285490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:13:41.241Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:41.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:13:41.243Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:41.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:41 smithi130 bash[27579]: audit 2023-08-28T02:13:41.124917+0000 mon.a (mon.0) 4910 : audit [DBG] from='client.? 172.21.15.6:0/3879285490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:41.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:41 smithi130 bash[51302]: debug 2023-08-28T02:13:41.395+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:41.399439+0000) 2023-08-28T02:13:42.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:42 smithi006 bash[33918]: cluster 2023-08-28T02:13:41.296160+0000 mgr.y (mgr.14901) 7652 : cluster [DBG] pgmap v7659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:42.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:42 smithi006 bash[27292]: cluster 2023-08-28T02:13:41.296160+0000 mgr.y (mgr.14901) 7652 : cluster [DBG] pgmap v7659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:42.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:42 smithi130 bash[27579]: cluster 2023-08-28T02:13:41.296160+0000 mgr.y (mgr.14901) 7652 : cluster [DBG] pgmap v7659: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:42.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:42 smithi130 bash[51302]: debug 2023-08-28T02:13:42.395+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:42.399747+0000) 2023-08-28T02:13:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:13:42.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:13:42.810Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:43.776 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:43 smithi130 bash[51302]: debug 2023-08-28T02:13:43.395+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:43.400020+0000) 2023-08-28T02:13:44.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:43 smithi130 bash[27579]: cluster 2023-08-28T02:13:43.296913+0000 mgr.y (mgr.14901) 7653 : cluster [DBG] pgmap v7660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:44.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:43 smithi130 bash[27579]: audit 2023-08-28T02:13:43.536469+0000 mon.a (mon.0) 4911 : audit [DBG] from='client.? 172.21.15.6:0/3823213478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:44.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:43 smithi006 bash[33918]: cluster 2023-08-28T02:13:43.296913+0000 mgr.y (mgr.14901) 7653 : cluster [DBG] pgmap v7660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:44.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:43 smithi006 bash[33918]: audit 2023-08-28T02:13:43.536469+0000 mon.a (mon.0) 4911 : audit [DBG] from='client.? 172.21.15.6:0/3823213478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:44.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:43 smithi006 bash[27292]: cluster 2023-08-28T02:13:43.296913+0000 mgr.y (mgr.14901) 7653 : cluster [DBG] pgmap v7660: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:44.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:43 smithi006 bash[27292]: audit 2023-08-28T02:13:43.536469+0000 mon.a (mon.0) 4911 : audit [DBG] from='client.? 172.21.15.6:0/3823213478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:44.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:44 smithi130 bash[51302]: debug 2023-08-28T02:13:44.395+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:44.400331+0000) 2023-08-28T02:13:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:13:44.686Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:13:44.688Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:45.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:45 smithi130 bash[51302]: debug 2023-08-28T02:13:45.395+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:45.400615+0000) 2023-08-28T02:13:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:46 smithi130 bash[27579]: cluster 2023-08-28T02:13:45.298061+0000 mgr.y (mgr.14901) 7654 : cluster [DBG] pgmap v7661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:46 smithi130 bash[27579]: audit 2023-08-28T02:13:45.947053+0000 mon.a (mon.0) 4912 : audit [DBG] from='client.? 172.21.15.6:0/1219705933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:46.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:46 smithi130 bash[51302]: debug 2023-08-28T02:13:46.399+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:46.400837+0000) 2023-08-28T02:13:46.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:46 smithi006 bash[33918]: cluster 2023-08-28T02:13:45.298061+0000 mgr.y (mgr.14901) 7654 : cluster [DBG] pgmap v7661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:46.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:46 smithi006 bash[33918]: audit 2023-08-28T02:13:45.947053+0000 mon.a (mon.0) 4912 : audit [DBG] from='client.? 172.21.15.6:0/1219705933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:46.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:46 smithi006 bash[27292]: cluster 2023-08-28T02:13:45.298061+0000 mgr.y (mgr.14901) 7654 : cluster [DBG] pgmap v7661: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:46.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:46 smithi006 bash[27292]: audit 2023-08-28T02:13:45.947053+0000 mon.a (mon.0) 4912 : audit [DBG] from='client.? 172.21.15.6:0/1219705933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:47.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:47 smithi130 bash[51302]: debug 2023-08-28T02:13:47.395+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:47.401096+0000) 2023-08-28T02:13:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:48 smithi130 bash[27579]: cluster 2023-08-28T02:13:47.298714+0000 mgr.y (mgr.14901) 7655 : cluster [DBG] pgmap v7662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:48.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:48 smithi130 bash[51302]: debug 2023-08-28T02:13:48.395+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:48.401372+0000) 2023-08-28T02:13:48.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:48 smithi006 bash[33918]: cluster 2023-08-28T02:13:47.298714+0000 mgr.y (mgr.14901) 7655 : cluster [DBG] pgmap v7662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:48 smithi006 bash[27292]: cluster 2023-08-28T02:13:47.298714+0000 mgr.y (mgr.14901) 7655 : cluster [DBG] pgmap v7662: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:49.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:49 smithi130 bash[27579]: audit 2023-08-28T02:13:48.357271+0000 mon.c (mon.2) 3014 : audit [DBG] from='client.? 172.21.15.6:0/3201081619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:49.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:49 smithi130 bash[51302]: debug 2023-08-28T02:13:49.395+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:49.401668+0000) 2023-08-28T02:13:49.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:49 smithi006 bash[33918]: audit 2023-08-28T02:13:48.357271+0000 mon.c (mon.2) 3014 : audit [DBG] from='client.? 172.21.15.6:0/3201081619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:49.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:49 smithi006 bash[27292]: audit 2023-08-28T02:13:48.357271+0000 mon.c (mon.2) 3014 : audit [DBG] from='client.? 172.21.15.6:0/3201081619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:50.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:50 smithi006 bash[27292]: cluster 2023-08-28T02:13:49.299836+0000 mgr.y (mgr.14901) 7656 : cluster [DBG] pgmap v7663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:50.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:50 smithi006 bash[33918]: cluster 2023-08-28T02:13:49.299836+0000 mgr.y (mgr.14901) 7656 : cluster [DBG] pgmap v7663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:50.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:50 smithi130 bash[27579]: cluster 2023-08-28T02:13:49.299836+0000 mgr.y (mgr.14901) 7656 : cluster [DBG] pgmap v7663: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:50.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:50 smithi130 bash[51302]: debug 2023-08-28T02:13:50.395+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:50.401949+0000) 2023-08-28T02:13:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:13:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:13:50] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T02:13:51.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:51 smithi006 bash[27292]: audit 2023-08-28T02:13:50.780042+0000 mon.c (mon.2) 3015 : audit [DBG] from='client.? 172.21.15.6:0/2828129405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:51.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:51 smithi006 bash[33918]: audit 2023-08-28T02:13:50.780042+0000 mon.c (mon.2) 3015 : audit [DBG] from='client.? 172.21.15.6:0/2828129405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:13:51.243Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:13:51.244Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:51.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:51 smithi130 bash[27579]: audit 2023-08-28T02:13:50.780042+0000 mon.c (mon.2) 3015 : audit [DBG] from='client.? 172.21.15.6:0/2828129405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:51.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:51 smithi130 bash[51302]: debug 2023-08-28T02:13:51.399+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:51.402247+0000) 2023-08-28T02:13:52.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:52 smithi006 bash[33918]: cluster 2023-08-28T02:13:51.300500+0000 mgr.y (mgr.14901) 7657 : cluster [DBG] pgmap v7664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:52.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:52 smithi006 bash[27292]: cluster 2023-08-28T02:13:51.300500+0000 mgr.y (mgr.14901) 7657 : cluster [DBG] pgmap v7664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:52.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:52 smithi130 bash[27579]: cluster 2023-08-28T02:13:51.300500+0000 mgr.y (mgr.14901) 7657 : cluster [DBG] pgmap v7664: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:52.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:52 smithi130 bash[51302]: debug 2023-08-28T02:13:52.395+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:52.402532+0000) 2023-08-28T02:13:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:13:52.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:13:52.811Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:53.776 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:53 smithi130 bash[51302]: debug 2023-08-28T02:13:53.399+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:53.402825+0000) 2023-08-28T02:13:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:53 smithi130 bash[27579]: audit 2023-08-28T02:13:53.194027+0000 mon.c (mon.2) 3016 : audit [DBG] from='client.? 172.21.15.6:0/3152663888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:53 smithi130 bash[27579]: cluster 2023-08-28T02:13:53.301224+0000 mgr.y (mgr.14901) 7658 : cluster [DBG] pgmap v7665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:54.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:53 smithi006 bash[27292]: audit 2023-08-28T02:13:53.194027+0000 mon.c (mon.2) 3016 : audit [DBG] from='client.? 172.21.15.6:0/3152663888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:54.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:53 smithi006 bash[27292]: cluster 2023-08-28T02:13:53.301224+0000 mgr.y (mgr.14901) 7658 : cluster [DBG] pgmap v7665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:54.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:53 smithi006 bash[33918]: audit 2023-08-28T02:13:53.194027+0000 mon.c (mon.2) 3016 : audit [DBG] from='client.? 172.21.15.6:0/3152663888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:54.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:53 smithi006 bash[33918]: cluster 2023-08-28T02:13:53.301224+0000 mgr.y (mgr.14901) 7658 : cluster [DBG] pgmap v7665: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:54.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:54 smithi130 bash[51302]: debug 2023-08-28T02:13:54.399+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:54.403141+0000) 2023-08-28T02:13:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:13:54.687Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:13:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:13:54.689Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:13:55.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:55 smithi130 bash[51302]: debug 2023-08-28T02:13:55.399+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:55.403446+0000) 2023-08-28T02:13:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:56 smithi130 bash[27579]: cluster 2023-08-28T02:13:55.302008+0000 mgr.y (mgr.14901) 7659 : cluster [DBG] pgmap v7666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:56.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:56 smithi130 bash[27579]: audit 2023-08-28T02:13:55.607389+0000 mon.c (mon.2) 3017 : audit [DBG] from='client.? 172.21.15.6:0/4148993486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:56.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:56 smithi130 bash[51302]: debug 2023-08-28T02:13:56.399+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:56.403666+0000) 2023-08-28T02:13:56.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:56 smithi006 bash[27292]: cluster 2023-08-28T02:13:55.302008+0000 mgr.y (mgr.14901) 7659 : cluster [DBG] pgmap v7666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:56.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:56 smithi006 bash[27292]: audit 2023-08-28T02:13:55.607389+0000 mon.c (mon.2) 3017 : audit [DBG] from='client.? 172.21.15.6:0/4148993486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:56.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:56 smithi006 bash[33918]: cluster 2023-08-28T02:13:55.302008+0000 mgr.y (mgr.14901) 7659 : cluster [DBG] pgmap v7666: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:56.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:56 smithi006 bash[33918]: audit 2023-08-28T02:13:55.607389+0000 mon.c (mon.2) 3017 : audit [DBG] from='client.? 172.21.15.6:0/4148993486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:57.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:57 smithi130 bash[51302]: debug 2023-08-28T02:13:57.399+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:57.403948+0000) 2023-08-28T02:13:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:58 smithi130 bash[27579]: cluster 2023-08-28T02:13:57.302646+0000 mgr.y (mgr.14901) 7660 : cluster [DBG] pgmap v7667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:13:58 smithi130 bash[27579]: audit 2023-08-28T02:13:58.024682+0000 mon.c (mon.2) 3018 : audit [DBG] from='client.? 172.21.15.6:0/292491679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:58.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:58 smithi130 bash[51302]: debug 2023-08-28T02:13:58.399+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:58.404219+0000) 2023-08-28T02:13:58.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:58 smithi006 bash[27292]: cluster 2023-08-28T02:13:57.302646+0000 mgr.y (mgr.14901) 7660 : cluster [DBG] pgmap v7667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:58.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:13:58 smithi006 bash[27292]: audit 2023-08-28T02:13:58.024682+0000 mon.c (mon.2) 3018 : audit [DBG] from='client.? 172.21.15.6:0/292491679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:58.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:58 smithi006 bash[33918]: cluster 2023-08-28T02:13:57.302646+0000 mgr.y (mgr.14901) 7660 : cluster [DBG] pgmap v7667: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:13:58.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:13:58 smithi006 bash[33918]: audit 2023-08-28T02:13:58.024682+0000 mon.c (mon.2) 3018 : audit [DBG] from='client.? 172.21.15.6:0/292491679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:59.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:13:59 smithi130 bash[51302]: debug 2023-08-28T02:13:59.399+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:59.404559+0000) 2023-08-28T02:14:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:00 smithi006 bash[27292]: cluster 2023-08-28T02:13:59.303742+0000 mgr.y (mgr.14901) 7661 : cluster [DBG] pgmap v7668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:00 smithi006 bash[33918]: cluster 2023-08-28T02:13:59.303742+0000 mgr.y (mgr.14901) 7661 : cluster [DBG] pgmap v7668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:00 smithi130 bash[27579]: cluster 2023-08-28T02:13:59.303742+0000 mgr.y (mgr.14901) 7661 : cluster [DBG] pgmap v7668: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:00.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:00 smithi130 bash[51302]: debug 2023-08-28T02:14:00.399+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:00.404836+0000) 2023-08-28T02:14:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:14:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:14:00] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T02:14:01.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:01 smithi006 bash[27292]: audit 2023-08-28T02:14:00.448003+0000 mon.a (mon.0) 4913 : audit [DBG] from='client.? 172.21.15.6:0/1737216310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:01.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:01 smithi006 bash[33918]: audit 2023-08-28T02:14:00.448003+0000 mon.a (mon.0) 4913 : audit [DBG] from='client.? 172.21.15.6:0/1737216310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:01.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:14:01.244Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:01.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:14:01.246Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:01.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:01 smithi130 bash[27579]: audit 2023-08-28T02:14:00.448003+0000 mon.a (mon.0) 4913 : audit [DBG] from='client.? 172.21.15.6:0/1737216310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:01.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:01 smithi130 bash[51302]: debug 2023-08-28T02:14:01.399+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:01.405115+0000) 2023-08-28T02:14:02.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:02 smithi006 bash[33918]: cluster 2023-08-28T02:14:01.304533+0000 mgr.y (mgr.14901) 7662 : cluster [DBG] pgmap v7669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:02.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:02 smithi006 bash[27292]: cluster 2023-08-28T02:14:01.304533+0000 mgr.y (mgr.14901) 7662 : cluster [DBG] pgmap v7669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:02 smithi130 bash[27579]: cluster 2023-08-28T02:14:01.304533+0000 mgr.y (mgr.14901) 7662 : cluster [DBG] pgmap v7669: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:02.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:02 smithi130 bash[51302]: debug 2023-08-28T02:14:02.399+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:02.405395+0000) 2023-08-28T02:14:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:14:02.809Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:14:02.811Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:03.778 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:03 smithi130 bash[51302]: debug 2023-08-28T02:14:03.403+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:03.405675+0000) 2023-08-28T02:14:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:03 smithi130 bash[27579]: audit 2023-08-28T02:14:02.860989+0000 mon.a (mon.0) 4914 : audit [DBG] from='client.? 172.21.15.6:0/4292746591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:03 smithi130 bash[27579]: cluster 2023-08-28T02:14:03.305311+0000 mgr.y (mgr.14901) 7663 : cluster [DBG] pgmap v7670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:04.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:03 smithi006 bash[27292]: audit 2023-08-28T02:14:02.860989+0000 mon.a (mon.0) 4914 : audit [DBG] from='client.? 172.21.15.6:0/4292746591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:04.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:03 smithi006 bash[27292]: cluster 2023-08-28T02:14:03.305311+0000 mgr.y (mgr.14901) 7663 : cluster [DBG] pgmap v7670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:04.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:03 smithi006 bash[33918]: audit 2023-08-28T02:14:02.860989+0000 mon.a (mon.0) 4914 : audit [DBG] from='client.? 172.21.15.6:0/4292746591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:04.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:03 smithi006 bash[33918]: cluster 2023-08-28T02:14:03.305311+0000 mgr.y (mgr.14901) 7663 : cluster [DBG] pgmap v7670: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:04.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:04 smithi130 bash[51302]: debug 2023-08-28T02:14:04.399+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:04.405984+0000) 2023-08-28T02:14:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:14:04.688Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:14:04.690Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:05.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:05 smithi130 bash[27579]: audit 2023-08-28T02:14:05.271182+0000 mon.c (mon.2) 3019 : audit [DBG] from='client.? 172.21.15.6:0/1325204344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:05.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:05 smithi130 bash[51302]: debug 2023-08-28T02:14:05.403+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:05.406281+0000) 2023-08-28T02:14:05.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:05 smithi006 bash[33918]: audit 2023-08-28T02:14:05.271182+0000 mon.c (mon.2) 3019 : audit [DBG] from='client.? 172.21.15.6:0/1325204344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:05.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:05 smithi006 bash[27292]: audit 2023-08-28T02:14:05.271182+0000 mon.c (mon.2) 3019 : audit [DBG] from='client.? 172.21.15.6:0/1325204344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:06 smithi130 bash[27579]: cluster 2023-08-28T02:14:05.306418+0000 mgr.y (mgr.14901) 7664 : cluster [DBG] pgmap v7671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:06.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:06 smithi130 bash[51302]: debug 2023-08-28T02:14:06.403+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:06.406555+0000) 2023-08-28T02:14:06.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:06 smithi006 bash[33918]: cluster 2023-08-28T02:14:05.306418+0000 mgr.y (mgr.14901) 7664 : cluster [DBG] pgmap v7671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:06 smithi006 bash[27292]: cluster 2023-08-28T02:14:05.306418+0000 mgr.y (mgr.14901) 7664 : cluster [DBG] pgmap v7671: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:07.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:07 smithi130 bash[51302]: debug 2023-08-28T02:14:07.403+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:07.406908+0000) 2023-08-28T02:14:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:08 smithi006 bash[33918]: cluster 2023-08-28T02:14:07.307052+0000 mgr.y (mgr.14901) 7665 : cluster [DBG] pgmap v7672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:08.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:08 smithi006 bash[33918]: audit 2023-08-28T02:14:07.685893+0000 mon.c (mon.2) 3020 : audit [DBG] from='client.? 172.21.15.6:0/972072224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:08 smithi006 bash[27292]: cluster 2023-08-28T02:14:07.307052+0000 mgr.y (mgr.14901) 7665 : cluster [DBG] pgmap v7672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:08 smithi006 bash[27292]: audit 2023-08-28T02:14:07.685893+0000 mon.c (mon.2) 3020 : audit [DBG] from='client.? 172.21.15.6:0/972072224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:08.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:08 smithi130 bash[51302]: debug 2023-08-28T02:14:08.403+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:08.407224+0000) 2023-08-28T02:14:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:08 smithi130 bash[27579]: cluster 2023-08-28T02:14:07.307052+0000 mgr.y (mgr.14901) 7665 : cluster [DBG] pgmap v7672: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:08 smithi130 bash[27579]: audit 2023-08-28T02:14:07.685893+0000 mon.c (mon.2) 3020 : audit [DBG] from='client.? 172.21.15.6:0/972072224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:09.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:09 smithi130 bash[51302]: debug 2023-08-28T02:14:09.403+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:09.407553+0000) 2023-08-28T02:14:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:10 smithi006 bash[33918]: cluster 2023-08-28T02:14:09.308142+0000 mgr.y (mgr.14901) 7666 : cluster [DBG] pgmap v7673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:10.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:10 smithi006 bash[33918]: audit 2023-08-28T02:14:10.105211+0000 mon.c (mon.2) 3021 : audit [DBG] from='client.? 172.21.15.6:0/3504780884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:10 smithi006 bash[27292]: cluster 2023-08-28T02:14:09.308142+0000 mgr.y (mgr.14901) 7666 : cluster [DBG] pgmap v7673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:10 smithi006 bash[27292]: audit 2023-08-28T02:14:10.105211+0000 mon.c (mon.2) 3021 : audit [DBG] from='client.? 172.21.15.6:0/3504780884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:10.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:10 smithi130 bash[51302]: debug 2023-08-28T02:14:10.403+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:10.407848+0000) 2023-08-28T02:14:10.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:10 smithi130 bash[27579]: cluster 2023-08-28T02:14:09.308142+0000 mgr.y (mgr.14901) 7666 : cluster [DBG] pgmap v7673: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:10.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:10 smithi130 bash[27579]: audit 2023-08-28T02:14:10.105211+0000 mon.c (mon.2) 3021 : audit [DBG] from='client.? 172.21.15.6:0/3504780884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:11.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:14:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:14:10] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T02:14:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:14:11.245Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:14:11.247Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:11.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:11 smithi130 bash[51302]: debug 2023-08-28T02:14:11.403+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:11.408160+0000) 2023-08-28T02:14:12.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:12 smithi006 bash[33918]: cluster 2023-08-28T02:14:11.308869+0000 mgr.y (mgr.14901) 7667 : cluster [DBG] pgmap v7674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:12.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:12 smithi006 bash[27292]: cluster 2023-08-28T02:14:11.308869+0000 mgr.y (mgr.14901) 7667 : cluster [DBG] pgmap v7674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:12 smithi130 bash[27579]: cluster 2023-08-28T02:14:11.308869+0000 mgr.y (mgr.14901) 7667 : cluster [DBG] pgmap v7674: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:12.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:12 smithi130 bash[51302]: debug 2023-08-28T02:14:12.403+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:12.408453+0000) 2023-08-28T02:14:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:14:12.810Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:14:12.812Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:13.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:13 smithi006 bash[27292]: audit 2023-08-28T02:14:12.520897+0000 mon.c (mon.2) 3022 : audit [DBG] from='client.? 172.21.15.6:0/3014961429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:13.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:13 smithi006 bash[33918]: audit 2023-08-28T02:14:12.520897+0000 mon.c (mon.2) 3022 : audit [DBG] from='client.? 172.21.15.6:0/3014961429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:13.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:13 smithi130 bash[27579]: audit 2023-08-28T02:14:12.520897+0000 mon.c (mon.2) 3022 : audit [DBG] from='client.? 172.21.15.6:0/3014961429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:13.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:13 smithi130 bash[51302]: debug 2023-08-28T02:14:13.403+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:13.408814+0000) 2023-08-28T02:14:14.689 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:14 smithi006 bash[27292]: cluster 2023-08-28T02:14:13.309587+0000 mgr.y (mgr.14901) 7668 : cluster [DBG] pgmap v7675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:14.689 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:14 smithi006 bash[33918]: cluster 2023-08-28T02:14:13.309587+0000 mgr.y (mgr.14901) 7668 : cluster [DBG] pgmap v7675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:14.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:14 smithi130 bash[27579]: cluster 2023-08-28T02:14:13.309587+0000 mgr.y (mgr.14901) 7668 : cluster [DBG] pgmap v7675: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:14.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:14 smithi130 bash[51302]: debug 2023-08-28T02:14:14.403+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:14.409116+0000) 2023-08-28T02:14:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:14:14.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:14:14.691Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:15.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:15 smithi006 bash[27292]: audit 2023-08-28T02:14:14.950202+0000 mon.a (mon.0) 4915 : audit [DBG] from='client.? 172.21.15.6:0/1125050447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:15.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:15 smithi006 bash[33918]: audit 2023-08-28T02:14:14.950202+0000 mon.a (mon.0) 4915 : audit [DBG] from='client.? 172.21.15.6:0/1125050447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:15.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:15 smithi130 bash[27579]: audit 2023-08-28T02:14:14.950202+0000 mon.a (mon.0) 4915 : audit [DBG] from='client.? 172.21.15.6:0/1125050447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:15.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:15 smithi130 bash[51302]: debug 2023-08-28T02:14:15.403+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:15.409382+0000) 2023-08-28T02:14:16.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:16 smithi006 bash[33918]: cluster 2023-08-28T02:14:15.310733+0000 mgr.y (mgr.14901) 7669 : cluster [DBG] pgmap v7676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:16 smithi006 bash[27292]: cluster 2023-08-28T02:14:15.310733+0000 mgr.y (mgr.14901) 7669 : cluster [DBG] pgmap v7676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:16.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:16 smithi130 bash[27579]: cluster 2023-08-28T02:14:15.310733+0000 mgr.y (mgr.14901) 7669 : cluster [DBG] pgmap v7676: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:16.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:16 smithi130 bash[51302]: debug 2023-08-28T02:14:16.402+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:16.409622+0000) 2023-08-28T02:14:17.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:17 smithi006 bash[33918]: audit 2023-08-28T02:14:17.363358+0000 mon.c (mon.2) 3023 : audit [DBG] from='client.? 172.21.15.6:0/1313072713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:17.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:17 smithi006 bash[27292]: audit 2023-08-28T02:14:17.363358+0000 mon.c (mon.2) 3023 : audit [DBG] from='client.? 172.21.15.6:0/1313072713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:17.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:17 smithi130 bash[27579]: audit 2023-08-28T02:14:17.363358+0000 mon.c (mon.2) 3023 : audit [DBG] from='client.? 172.21.15.6:0/1313072713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:17.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:17 smithi130 bash[51302]: debug 2023-08-28T02:14:17.402+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:17.409924+0000) 2023-08-28T02:14:18.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:18 smithi006 bash[33918]: cluster 2023-08-28T02:14:17.311244+0000 mgr.y (mgr.14901) 7670 : cluster [DBG] pgmap v7677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:18.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:18 smithi006 bash[27292]: cluster 2023-08-28T02:14:17.311244+0000 mgr.y (mgr.14901) 7670 : cluster [DBG] pgmap v7677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:18.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:18 smithi130 bash[27579]: cluster 2023-08-28T02:14:17.311244+0000 mgr.y (mgr.14901) 7670 : cluster [DBG] pgmap v7677: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:18.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:18 smithi130 bash[51302]: debug 2023-08-28T02:14:18.402+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:18.410150+0000) 2023-08-28T02:14:19.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:19 smithi130 bash[51302]: debug 2023-08-28T02:14:19.402+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:19.410364+0000) 2023-08-28T02:14:20.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:20 smithi006 bash[33918]: cluster 2023-08-28T02:14:19.312303+0000 mgr.y (mgr.14901) 7671 : cluster [DBG] pgmap v7678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:20 smithi006 bash[33918]: audit 2023-08-28T02:14:19.777408+0000 mon.c (mon.2) 3024 : audit [DBG] from='client.? 172.21.15.6:0/894661787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:20 smithi006 bash[33918]: audit 2023-08-28T02:14:20.230790+0000 mon.b (mon.1) 846 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:14:20.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:20 smithi006 bash[27292]: cluster 2023-08-28T02:14:19.312303+0000 mgr.y (mgr.14901) 7671 : cluster [DBG] pgmap v7678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:20.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:20 smithi006 bash[27292]: audit 2023-08-28T02:14:19.777408+0000 mon.c (mon.2) 3024 : audit [DBG] from='client.? 172.21.15.6:0/894661787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:20.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:20 smithi006 bash[27292]: audit 2023-08-28T02:14:20.230790+0000 mon.b (mon.1) 846 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:14:20.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:20 smithi130 bash[51302]: debug 2023-08-28T02:14:20.402+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:20.410575+0000) 2023-08-28T02:14:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:20 smithi130 bash[27579]: cluster 2023-08-28T02:14:19.312303+0000 mgr.y (mgr.14901) 7671 : cluster [DBG] pgmap v7678: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:20 smithi130 bash[27579]: audit 2023-08-28T02:14:19.777408+0000 mon.c (mon.2) 3024 : audit [DBG] from='client.? 172.21.15.6:0/894661787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:20 smithi130 bash[27579]: audit 2023-08-28T02:14:20.230790+0000 mon.b (mon.1) 846 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:14:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:14:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:14:20] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:14:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:14:21.245Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:14:21.247Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:21.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:21 smithi130 bash[51302]: debug 2023-08-28T02:14:21.406+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:21.410872+0000) 2023-08-28T02:14:22.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:22 smithi006 bash[27292]: cluster 2023-08-28T02:14:21.313002+0000 mgr.y (mgr.14901) 7672 : cluster [DBG] pgmap v7679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:22.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:22 smithi006 bash[27292]: audit 2023-08-28T02:14:22.194237+0000 mon.c (mon.2) 3025 : audit [DBG] from='client.? 172.21.15.6:0/3113951760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:22.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:22 smithi006 bash[33918]: cluster 2023-08-28T02:14:21.313002+0000 mgr.y (mgr.14901) 7672 : cluster [DBG] pgmap v7679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:22.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:22 smithi006 bash[33918]: audit 2023-08-28T02:14:22.194237+0000 mon.c (mon.2) 3025 : audit [DBG] from='client.? 172.21.15.6:0/3113951760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:22.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:22 smithi130 bash[51302]: debug 2023-08-28T02:14:22.406+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:22.411149+0000) 2023-08-28T02:14:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:22 smithi130 bash[27579]: cluster 2023-08-28T02:14:21.313002+0000 mgr.y (mgr.14901) 7672 : cluster [DBG] pgmap v7679: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:22 smithi130 bash[27579]: audit 2023-08-28T02:14:22.194237+0000 mon.c (mon.2) 3025 : audit [DBG] from='client.? 172.21.15.6:0/3113951760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:14:22.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:14:22.813Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:23.795 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:23 smithi130 bash[51302]: debug 2023-08-28T02:14:23.410+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:23.411465+0000) 2023-08-28T02:14:24.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:23 smithi130 bash[27579]: cluster 2023-08-28T02:14:23.313656+0000 mgr.y (mgr.14901) 7673 : cluster [DBG] pgmap v7680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:23 smithi006 bash[27292]: cluster 2023-08-28T02:14:23.313656+0000 mgr.y (mgr.14901) 7673 : cluster [DBG] pgmap v7680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:24.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:23 smithi006 bash[33918]: cluster 2023-08-28T02:14:23.313656+0000 mgr.y (mgr.14901) 7673 : cluster [DBG] pgmap v7680: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:24.800 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:24 smithi130 bash[51302]: debug 2023-08-28T02:14:24.410+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:24.411798+0000) 2023-08-28T02:14:24.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:24 smithi006 bash[27292]: audit 2023-08-28T02:14:24.613354+0000 mon.c (mon.2) 3026 : audit [DBG] from='client.? 172.21.15.6:0/2554508094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:24.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:24 smithi006 bash[33918]: audit 2023-08-28T02:14:24.613354+0000 mon.c (mon.2) 3026 : audit [DBG] from='client.? 172.21.15.6:0/2554508094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:24.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:14:24.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:24.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:14:24.692Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:25.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:24 smithi130 bash[27579]: audit 2023-08-28T02:14:24.613354+0000 mon.c (mon.2) 3026 : audit [DBG] from='client.? 172.21.15.6:0/2554508094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:25.800 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:25 smithi130 bash[51302]: debug 2023-08-28T02:14:25.410+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:25.411990+0000) 2023-08-28T02:14:26.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:25 smithi130 bash[27579]: cluster 2023-08-28T02:14:25.314892+0000 mgr.y (mgr.14901) 7674 : cluster [DBG] pgmap v7681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:26.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:25 smithi006 bash[27292]: cluster 2023-08-28T02:14:25.314892+0000 mgr.y (mgr.14901) 7674 : cluster [DBG] pgmap v7681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:26.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:25 smithi006 bash[33918]: cluster 2023-08-28T02:14:25.314892+0000 mgr.y (mgr.14901) 7674 : cluster [DBG] pgmap v7681: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:26.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:26 smithi130 bash[51302]: debug 2023-08-28T02:14:26.410+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:26.412199+0000) 2023-08-28T02:14:27.411 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:27 smithi130 bash[27579]: audit 2023-08-28T02:14:26.149623+0000 mon.a (mon.0) 4916 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:14:27.411 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:27 smithi130 bash[27579]: audit 2023-08-28T02:14:26.581302+0000 mon.a (mon.0) 4917 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:14:27.411 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:27 smithi130 bash[27579]: audit 2023-08-28T02:14:26.881937+0000 mon.b (mon.1) 847 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:14:27.411 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:27 smithi130 bash[27579]: audit 2023-08-28T02:14:26.883762+0000 mon.b (mon.1) 848 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:14:27.411 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:27 smithi130 bash[27579]: audit 2023-08-28T02:14:26.890388+0000 mon.a (mon.0) 4918 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:14:27.412 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:27 smithi130 bash[27579]: audit 2023-08-28T02:14:27.031785+0000 mon.a (mon.0) 4919 : audit [DBG] from='client.? 172.21.15.6:0/2575093265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:27.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:27 smithi006 bash[33918]: audit 2023-08-28T02:14:26.149623+0000 mon.a (mon.0) 4916 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:14:27.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:27 smithi006 bash[33918]: audit 2023-08-28T02:14:26.581302+0000 mon.a (mon.0) 4917 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:14:27.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:27 smithi006 bash[33918]: audit 2023-08-28T02:14:26.881937+0000 mon.b (mon.1) 847 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:14:27.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:27 smithi006 bash[33918]: audit 2023-08-28T02:14:26.883762+0000 mon.b (mon.1) 848 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:14:27.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:27 smithi006 bash[33918]: audit 2023-08-28T02:14:26.890388+0000 mon.a (mon.0) 4918 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:14:27.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:27 smithi006 bash[33918]: audit 2023-08-28T02:14:27.031785+0000 mon.a (mon.0) 4919 : audit [DBG] from='client.? 172.21.15.6:0/2575093265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:27.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:27 smithi006 bash[27292]: audit 2023-08-28T02:14:26.149623+0000 mon.a (mon.0) 4916 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:14:27.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:27 smithi006 bash[27292]: audit 2023-08-28T02:14:26.581302+0000 mon.a (mon.0) 4917 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:14:27.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:27 smithi006 bash[27292]: audit 2023-08-28T02:14:26.881937+0000 mon.b (mon.1) 847 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:14:27.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:27 smithi006 bash[27292]: audit 2023-08-28T02:14:26.883762+0000 mon.b (mon.1) 848 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:14:27.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:27 smithi006 bash[27292]: audit 2023-08-28T02:14:26.890388+0000 mon.a (mon.0) 4918 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:14:27.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:27 smithi006 bash[27292]: audit 2023-08-28T02:14:27.031785+0000 mon.a (mon.0) 4919 : audit [DBG] from='client.? 172.21.15.6:0/2575093265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:27.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:27 smithi130 bash[51302]: debug 2023-08-28T02:14:27.410+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:27.412452+0000) 2023-08-28T02:14:28.411 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:28 smithi130 bash[27579]: cluster 2023-08-28T02:14:27.315712+0000 mgr.y (mgr.14901) 7675 : cluster [DBG] pgmap v7682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:28.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:28 smithi006 bash[33918]: cluster 2023-08-28T02:14:27.315712+0000 mgr.y (mgr.14901) 7675 : cluster [DBG] pgmap v7682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:28.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:28 smithi006 bash[27292]: cluster 2023-08-28T02:14:27.315712+0000 mgr.y (mgr.14901) 7675 : cluster [DBG] pgmap v7682: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:28.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:28 smithi130 bash[51302]: debug 2023-08-28T02:14:28.410+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:28.412746+0000) 2023-08-28T02:14:29.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:29 smithi130 bash[51302]: debug 2023-08-28T02:14:29.410+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:29.413102+0000) 2023-08-28T02:14:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:30 smithi006 bash[33918]: cluster 2023-08-28T02:14:29.316956+0000 mgr.y (mgr.14901) 7676 : cluster [DBG] pgmap v7683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:30 smithi006 bash[33918]: audit 2023-08-28T02:14:29.453225+0000 mon.c (mon.2) 3027 : audit [DBG] from='client.? 172.21.15.6:0/1271966617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:30 smithi006 bash[27292]: cluster 2023-08-28T02:14:29.316956+0000 mgr.y (mgr.14901) 7676 : cluster [DBG] pgmap v7683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:30 smithi006 bash[27292]: audit 2023-08-28T02:14:29.453225+0000 mon.c (mon.2) 3027 : audit [DBG] from='client.? 172.21.15.6:0/1271966617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:30 smithi130 bash[27579]: cluster 2023-08-28T02:14:29.316956+0000 mgr.y (mgr.14901) 7676 : cluster [DBG] pgmap v7683: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:30 smithi130 bash[27579]: audit 2023-08-28T02:14:29.453225+0000 mon.c (mon.2) 3027 : audit [DBG] from='client.? 172.21.15.6:0/1271966617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:30.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:30 smithi130 bash[51302]: debug 2023-08-28T02:14:30.410+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:30.413396+0000) 2023-08-28T02:14:31.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:14:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:14:30] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:14:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:14:31.246Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:14:31.248Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:31.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:31 smithi130 bash[51302]: debug 2023-08-28T02:14:31.410+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:31.413684+0000) 2023-08-28T02:14:32.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:32 smithi006 bash[33918]: cluster 2023-08-28T02:14:31.317616+0000 mgr.y (mgr.14901) 7677 : cluster [DBG] pgmap v7684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:32.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:32 smithi006 bash[33918]: audit 2023-08-28T02:14:31.882718+0000 mon.a (mon.0) 4920 : audit [DBG] from='client.? 172.21.15.6:0/2840606426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:32.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:32 smithi006 bash[27292]: cluster 2023-08-28T02:14:31.317616+0000 mgr.y (mgr.14901) 7677 : cluster [DBG] pgmap v7684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:32.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:32 smithi006 bash[27292]: audit 2023-08-28T02:14:31.882718+0000 mon.a (mon.0) 4920 : audit [DBG] from='client.? 172.21.15.6:0/2840606426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:32.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:32 smithi130 bash[51302]: debug 2023-08-28T02:14:32.410+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:32.414001+0000) 2023-08-28T02:14:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:32 smithi130 bash[27579]: cluster 2023-08-28T02:14:31.317616+0000 mgr.y (mgr.14901) 7677 : cluster [DBG] pgmap v7684: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:32 smithi130 bash[27579]: audit 2023-08-28T02:14:31.882718+0000 mon.a (mon.0) 4920 : audit [DBG] from='client.? 172.21.15.6:0/2840606426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:14:32.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:14:32.813Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:33.801 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:33 smithi130 bash[51302]: debug 2023-08-28T02:14:33.410+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:33.414289+0000) 2023-08-28T02:14:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:33 smithi130 bash[27579]: cluster 2023-08-28T02:14:33.318292+0000 mgr.y (mgr.14901) 7678 : cluster [DBG] pgmap v7685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:34.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:33 smithi006 bash[33918]: cluster 2023-08-28T02:14:33.318292+0000 mgr.y (mgr.14901) 7678 : cluster [DBG] pgmap v7685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:33 smithi006 bash[27292]: cluster 2023-08-28T02:14:33.318292+0000 mgr.y (mgr.14901) 7678 : cluster [DBG] pgmap v7685: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:34.804 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:34 smithi130 bash[51302]: debug 2023-08-28T02:14:34.410+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:34.414573+0000) 2023-08-28T02:14:34.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:34 smithi006 bash[27292]: audit 2023-08-28T02:14:34.299254+0000 mon.a (mon.0) 4921 : audit [DBG] from='client.? 172.21.15.6:0/186044285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:34.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:34 smithi006 bash[33918]: audit 2023-08-28T02:14:34.299254+0000 mon.a (mon.0) 4921 : audit [DBG] from='client.? 172.21.15.6:0/186044285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:14:34.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:14:34.692Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:35.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:34 smithi130 bash[27579]: audit 2023-08-28T02:14:34.299254+0000 mon.a (mon.0) 4921 : audit [DBG] from='client.? 172.21.15.6:0/186044285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:35.811 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:35 smithi130 bash[51302]: debug 2023-08-28T02:14:35.414+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:35.414919+0000) 2023-08-28T02:14:36.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:35 smithi130 bash[27579]: cluster 2023-08-28T02:14:35.319454+0000 mgr.y (mgr.14901) 7679 : cluster [DBG] pgmap v7686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:36.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:35 smithi006 bash[33918]: cluster 2023-08-28T02:14:35.319454+0000 mgr.y (mgr.14901) 7679 : cluster [DBG] pgmap v7686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:36.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:35 smithi006 bash[27292]: cluster 2023-08-28T02:14:35.319454+0000 mgr.y (mgr.14901) 7679 : cluster [DBG] pgmap v7686: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:36.814 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:36 smithi130 bash[51302]: debug 2023-08-28T02:14:36.414+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:36.415193+0000) 2023-08-28T02:14:37.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:36 smithi130 bash[27579]: audit 2023-08-28T02:14:36.712470+0000 mon.a (mon.0) 4922 : audit [DBG] from='client.? 172.21.15.6:0/1129948330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:37.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:36 smithi006 bash[33918]: audit 2023-08-28T02:14:36.712470+0000 mon.a (mon.0) 4922 : audit [DBG] from='client.? 172.21.15.6:0/1129948330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:37.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:36 smithi006 bash[27292]: audit 2023-08-28T02:14:36.712470+0000 mon.a (mon.0) 4922 : audit [DBG] from='client.? 172.21.15.6:0/1129948330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:37.823 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:37 smithi130 bash[51302]: debug 2023-08-28T02:14:37.414+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:37.415460+0000) 2023-08-28T02:14:38.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:37 smithi130 bash[27579]: cluster 2023-08-28T02:14:37.320141+0000 mgr.y (mgr.14901) 7680 : cluster [DBG] pgmap v7687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:38.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:37 smithi006 bash[33918]: cluster 2023-08-28T02:14:37.320141+0000 mgr.y (mgr.14901) 7680 : cluster [DBG] pgmap v7687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:38.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:37 smithi006 bash[27292]: cluster 2023-08-28T02:14:37.320141+0000 mgr.y (mgr.14901) 7680 : cluster [DBG] pgmap v7687: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:38.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:38 smithi130 bash[51302]: debug 2023-08-28T02:14:38.414+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:38.415769+0000) 2023-08-28T02:14:39.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:39 smithi006 bash[27292]: audit 2023-08-28T02:14:39.126245+0000 mon.a (mon.0) 4923 : audit [DBG] from='client.? 172.21.15.6:0/695770953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:39.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:39 smithi006 bash[33918]: audit 2023-08-28T02:14:39.126245+0000 mon.a (mon.0) 4923 : audit [DBG] from='client.? 172.21.15.6:0/695770953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:39.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:39 smithi130 bash[27579]: audit 2023-08-28T02:14:39.126245+0000 mon.a (mon.0) 4923 : audit [DBG] from='client.? 172.21.15.6:0/695770953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:39.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:39 smithi130 bash[51302]: debug 2023-08-28T02:14:39.414+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:39.416074+0000) 2023-08-28T02:14:40.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:40 smithi006 bash[27292]: cluster 2023-08-28T02:14:39.321313+0000 mgr.y (mgr.14901) 7681 : cluster [DBG] pgmap v7688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:40 smithi006 bash[33918]: cluster 2023-08-28T02:14:39.321313+0000 mgr.y (mgr.14901) 7681 : cluster [DBG] pgmap v7688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:40 smithi130 bash[27579]: cluster 2023-08-28T02:14:39.321313+0000 mgr.y (mgr.14901) 7681 : cluster [DBG] pgmap v7688: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:40.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:40 smithi130 bash[51302]: debug 2023-08-28T02:14:40.414+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:40.416340+0000) 2023-08-28T02:14:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:14:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:14:40] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:14:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:14:41.246Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:14:41.248Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:41.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:41 smithi130 bash[51302]: debug 2023-08-28T02:14:41.414+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:41.416686+0000) 2023-08-28T02:14:42.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:42 smithi006 bash[33918]: cluster 2023-08-28T02:14:41.321974+0000 mgr.y (mgr.14901) 7682 : cluster [DBG] pgmap v7689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:42.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:42 smithi006 bash[33918]: audit 2023-08-28T02:14:41.531568+0000 mon.c (mon.2) 3028 : audit [DBG] from='client.? 172.21.15.6:0/986853280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:42.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:42 smithi006 bash[27292]: cluster 2023-08-28T02:14:41.321974+0000 mgr.y (mgr.14901) 7682 : cluster [DBG] pgmap v7689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:42.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:42 smithi006 bash[27292]: audit 2023-08-28T02:14:41.531568+0000 mon.c (mon.2) 3028 : audit [DBG] from='client.? 172.21.15.6:0/986853280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:42.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:42 smithi130 bash[27579]: cluster 2023-08-28T02:14:41.321974+0000 mgr.y (mgr.14901) 7682 : cluster [DBG] pgmap v7689: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:42.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:42 smithi130 bash[27579]: audit 2023-08-28T02:14:41.531568+0000 mon.c (mon.2) 3028 : audit [DBG] from='client.? 172.21.15.6:0/986853280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:42.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:42 smithi130 bash[51302]: debug 2023-08-28T02:14:42.414+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:42.416954+0000) 2023-08-28T02:14:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:14:42.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:14:42.814Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:43.802 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:43 smithi130 bash[51302]: debug 2023-08-28T02:14:43.414+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:43.417221+0000) 2023-08-28T02:14:44.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:43 smithi130 bash[27579]: cluster 2023-08-28T02:14:43.322674+0000 mgr.y (mgr.14901) 7683 : cluster [DBG] pgmap v7690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:44.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:43 smithi006 bash[27292]: cluster 2023-08-28T02:14:43.322674+0000 mgr.y (mgr.14901) 7683 : cluster [DBG] pgmap v7690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:44.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:43 smithi006 bash[33918]: cluster 2023-08-28T02:14:43.322674+0000 mgr.y (mgr.14901) 7683 : cluster [DBG] pgmap v7690: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:44.805 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:44 smithi130 bash[51302]: debug 2023-08-28T02:14:44.414+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:44.417502+0000) 2023-08-28T02:14:44.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:44 smithi006 bash[27292]: audit 2023-08-28T02:14:43.949763+0000 mon.c (mon.2) 3029 : audit [DBG] from='client.? 172.21.15.6:0/3634290589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:44.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:44 smithi006 bash[33918]: audit 2023-08-28T02:14:43.949763+0000 mon.c (mon.2) 3029 : audit [DBG] from='client.? 172.21.15.6:0/3634290589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:14:44.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:14:44.693Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:45.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:44 smithi130 bash[27579]: audit 2023-08-28T02:14:43.949763+0000 mon.c (mon.2) 3029 : audit [DBG] from='client.? 172.21.15.6:0/3634290589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:45.813 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:45 smithi130 bash[51302]: debug 2023-08-28T02:14:45.414+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:45.417863+0000) 2023-08-28T02:14:46.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:45 smithi130 bash[27579]: cluster 2023-08-28T02:14:45.323926+0000 mgr.y (mgr.14901) 7684 : cluster [DBG] pgmap v7691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:46.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:45 smithi006 bash[27292]: cluster 2023-08-28T02:14:45.323926+0000 mgr.y (mgr.14901) 7684 : cluster [DBG] pgmap v7691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:46.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:45 smithi006 bash[33918]: cluster 2023-08-28T02:14:45.323926+0000 mgr.y (mgr.14901) 7684 : cluster [DBG] pgmap v7691: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:46.816 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:46 smithi130 bash[51302]: debug 2023-08-28T02:14:46.414+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:46.418178+0000) 2023-08-28T02:14:47.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:46 smithi130 bash[27579]: audit 2023-08-28T02:14:46.361083+0000 mon.a (mon.0) 4924 : audit [DBG] from='client.? 172.21.15.6:0/524919714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:47.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:46 smithi006 bash[33918]: audit 2023-08-28T02:14:46.361083+0000 mon.a (mon.0) 4924 : audit [DBG] from='client.? 172.21.15.6:0/524919714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:47.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:46 smithi006 bash[27292]: audit 2023-08-28T02:14:46.361083+0000 mon.a (mon.0) 4924 : audit [DBG] from='client.? 172.21.15.6:0/524919714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:47.823 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:47 smithi130 bash[51302]: debug 2023-08-28T02:14:47.414+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:47.418450+0000) 2023-08-28T02:14:48.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:47 smithi130 bash[27579]: cluster 2023-08-28T02:14:47.324635+0000 mgr.y (mgr.14901) 7685 : cluster [DBG] pgmap v7692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:48.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:47 smithi006 bash[33918]: cluster 2023-08-28T02:14:47.324635+0000 mgr.y (mgr.14901) 7685 : cluster [DBG] pgmap v7692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:48.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:47 smithi006 bash[27292]: cluster 2023-08-28T02:14:47.324635+0000 mgr.y (mgr.14901) 7685 : cluster [DBG] pgmap v7692: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:48.826 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:48 smithi130 bash[51302]: debug 2023-08-28T02:14:48.418+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:48.418760+0000) 2023-08-28T02:14:49.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:48 smithi130 bash[27579]: audit 2023-08-28T02:14:48.768834+0000 mon.a (mon.0) 4925 : audit [DBG] from='client.? 172.21.15.6:0/3921130200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:49.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:48 smithi006 bash[27292]: audit 2023-08-28T02:14:48.768834+0000 mon.a (mon.0) 4925 : audit [DBG] from='client.? 172.21.15.6:0/3921130200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:49.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:48 smithi006 bash[33918]: audit 2023-08-28T02:14:48.768834+0000 mon.a (mon.0) 4925 : audit [DBG] from='client.? 172.21.15.6:0/3921130200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:49.835 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:49 smithi130 bash[51302]: debug 2023-08-28T02:14:49.418+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:49.419052+0000) 2023-08-28T02:14:50.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:49 smithi130 bash[27579]: cluster 2023-08-28T02:14:49.325773+0000 mgr.y (mgr.14901) 7686 : cluster [DBG] pgmap v7693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:50.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:49 smithi006 bash[27292]: cluster 2023-08-28T02:14:49.325773+0000 mgr.y (mgr.14901) 7686 : cluster [DBG] pgmap v7693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:50.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:49 smithi006 bash[33918]: cluster 2023-08-28T02:14:49.325773+0000 mgr.y (mgr.14901) 7686 : cluster [DBG] pgmap v7693: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:50.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:50 smithi130 bash[51302]: debug 2023-08-28T02:14:50.418+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:50.419377+0000) 2023-08-28T02:14:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:14:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:14:50] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T02:14:51.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:51 smithi130 bash[27579]: audit 2023-08-28T02:14:51.185281+0000 mon.a (mon.0) 4926 : audit [DBG] from='client.? 172.21.15.6:0/1761758508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:51.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:51 smithi130 bash[51302]: debug 2023-08-28T02:14:51.418+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:51.419799+0000) 2023-08-28T02:14:51.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:51 smithi006 bash[27292]: audit 2023-08-28T02:14:51.185281+0000 mon.a (mon.0) 4926 : audit [DBG] from='client.? 172.21.15.6:0/1761758508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:14:51.247Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:14:51.248Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:51.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:51 smithi006 bash[33918]: audit 2023-08-28T02:14:51.185281+0000 mon.a (mon.0) 4926 : audit [DBG] from='client.? 172.21.15.6:0/1761758508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:52.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:52 smithi130 bash[27579]: cluster 2023-08-28T02:14:51.326462+0000 mgr.y (mgr.14901) 7687 : cluster [DBG] pgmap v7694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:52.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:52 smithi130 bash[51302]: debug 2023-08-28T02:14:52.418+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:52.420135+0000) 2023-08-28T02:14:52.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:52 smithi006 bash[33918]: cluster 2023-08-28T02:14:51.326462+0000 mgr.y (mgr.14901) 7687 : cluster [DBG] pgmap v7694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:52.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:52 smithi006 bash[27292]: cluster 2023-08-28T02:14:51.326462+0000 mgr.y (mgr.14901) 7687 : cluster [DBG] pgmap v7694: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:14:52.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:14:52.815Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:53.803 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:53 smithi130 bash[51302]: debug 2023-08-28T02:14:53.418+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:53.420501+0000) 2023-08-28T02:14:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:53 smithi130 bash[27579]: cluster 2023-08-28T02:14:53.326895+0000 mgr.y (mgr.14901) 7688 : cluster [DBG] pgmap v7695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:53 smithi130 bash[27579]: audit 2023-08-28T02:14:53.598567+0000 mon.c (mon.2) 3030 : audit [DBG] from='client.? 172.21.15.6:0/1692161651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:54.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:53 smithi006 bash[33918]: cluster 2023-08-28T02:14:53.326895+0000 mgr.y (mgr.14901) 7688 : cluster [DBG] pgmap v7695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:54.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:53 smithi006 bash[33918]: audit 2023-08-28T02:14:53.598567+0000 mon.c (mon.2) 3030 : audit [DBG] from='client.? 172.21.15.6:0/1692161651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:54.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:53 smithi006 bash[27292]: cluster 2023-08-28T02:14:53.326895+0000 mgr.y (mgr.14901) 7688 : cluster [DBG] pgmap v7695: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:54.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:53 smithi006 bash[27292]: audit 2023-08-28T02:14:53.598567+0000 mon.c (mon.2) 3030 : audit [DBG] from='client.? 172.21.15.6:0/1692161651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:54.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:54 smithi130 bash[51302]: debug 2023-08-28T02:14:54.418+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:54.420834+0000) 2023-08-28T02:14:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:14:54.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:14:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:14:54.694Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:14:55.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:55 smithi130 bash[51302]: debug 2023-08-28T02:14:55.418+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:55.421175+0000) 2023-08-28T02:14:56.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:56 smithi006 bash[33918]: cluster 2023-08-28T02:14:55.328018+0000 mgr.y (mgr.14901) 7689 : cluster [DBG] pgmap v7696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:56.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:56 smithi006 bash[33918]: audit 2023-08-28T02:14:56.022884+0000 mon.a (mon.0) 4927 : audit [DBG] from='client.? 172.21.15.6:0/2769408752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:56.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:56 smithi006 bash[27292]: cluster 2023-08-28T02:14:55.328018+0000 mgr.y (mgr.14901) 7689 : cluster [DBG] pgmap v7696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:56.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:56 smithi006 bash[27292]: audit 2023-08-28T02:14:56.022884+0000 mon.a (mon.0) 4927 : audit [DBG] from='client.? 172.21.15.6:0/2769408752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:56 smithi130 bash[27579]: cluster 2023-08-28T02:14:55.328018+0000 mgr.y (mgr.14901) 7689 : cluster [DBG] pgmap v7696: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:56 smithi130 bash[27579]: audit 2023-08-28T02:14:56.022884+0000 mon.a (mon.0) 4927 : audit [DBG] from='client.? 172.21.15.6:0/2769408752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:56.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:56 smithi130 bash[51302]: debug 2023-08-28T02:14:56.418+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:56.421531+0000) 2023-08-28T02:14:57.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:57 smithi130 bash[51302]: debug 2023-08-28T02:14:57.418+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:57.421846+0000) 2023-08-28T02:14:58.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:58 smithi006 bash[33918]: cluster 2023-08-28T02:14:57.328811+0000 mgr.y (mgr.14901) 7690 : cluster [DBG] pgmap v7697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:58.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:58 smithi006 bash[27292]: cluster 2023-08-28T02:14:57.328811+0000 mgr.y (mgr.14901) 7690 : cluster [DBG] pgmap v7697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:58.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:58 smithi130 bash[27579]: cluster 2023-08-28T02:14:57.328811+0000 mgr.y (mgr.14901) 7690 : cluster [DBG] pgmap v7697: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:14:58.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:58 smithi130 bash[51302]: debug 2023-08-28T02:14:58.418+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:58.422168+0000) 2023-08-28T02:14:59.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:14:59 smithi006 bash[33918]: audit 2023-08-28T02:14:58.438080+0000 mon.c (mon.2) 3031 : audit [DBG] from='client.? 172.21.15.6:0/2454613731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:59.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:14:59 smithi006 bash[27292]: audit 2023-08-28T02:14:58.438080+0000 mon.c (mon.2) 3031 : audit [DBG] from='client.? 172.21.15.6:0/2454613731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:59.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:14:59 smithi130 bash[27579]: audit 2023-08-28T02:14:58.438080+0000 mon.c (mon.2) 3031 : audit [DBG] from='client.? 172.21.15.6:0/2454613731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:59.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:14:59 smithi130 bash[51302]: debug 2023-08-28T02:14:59.418+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:59.422480+0000) 2023-08-28T02:15:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:00 smithi006 bash[33918]: cluster 2023-08-28T02:14:59.329993+0000 mgr.y (mgr.14901) 7691 : cluster [DBG] pgmap v7698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:00 smithi006 bash[27292]: cluster 2023-08-28T02:14:59.329993+0000 mgr.y (mgr.14901) 7691 : cluster [DBG] pgmap v7698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:00 smithi130 bash[27579]: cluster 2023-08-28T02:14:59.329993+0000 mgr.y (mgr.14901) 7691 : cluster [DBG] pgmap v7698: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:00.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:00 smithi130 bash[51302]: debug 2023-08-28T02:15:00.422+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:00.422882+0000) 2023-08-28T02:15:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:15:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:15:00] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T02:15:01.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:01 smithi006 bash[27292]: audit 2023-08-28T02:15:00.854686+0000 mon.a (mon.0) 4928 : audit [DBG] from='client.? 172.21.15.6:0/2699409282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:01.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:01 smithi006 bash[33918]: audit 2023-08-28T02:15:00.854686+0000 mon.a (mon.0) 4928 : audit [DBG] from='client.? 172.21.15.6:0/2699409282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:15:01.247Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:15:01.249Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:01.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:01 smithi130 bash[27579]: audit 2023-08-28T02:15:00.854686+0000 mon.a (mon.0) 4928 : audit [DBG] from='client.? 172.21.15.6:0/2699409282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:01.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:01 smithi130 bash[51302]: debug 2023-08-28T02:15:01.422+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:01.423161+0000) 2023-08-28T02:15:02.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:02 smithi006 bash[27292]: cluster 2023-08-28T02:15:01.330740+0000 mgr.y (mgr.14901) 7692 : cluster [DBG] pgmap v7699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:02.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:02 smithi006 bash[33918]: cluster 2023-08-28T02:15:01.330740+0000 mgr.y (mgr.14901) 7692 : cluster [DBG] pgmap v7699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:02 smithi130 bash[27579]: cluster 2023-08-28T02:15:01.330740+0000 mgr.y (mgr.14901) 7692 : cluster [DBG] pgmap v7699: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:02.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:02 smithi130 bash[51302]: debug 2023-08-28T02:15:02.422+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:02.423411+0000) 2023-08-28T02:15:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:15:02.814Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:15:02.816Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:03.801 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:03 smithi130 bash[51302]: debug 2023-08-28T02:15:03.422+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:03.423743+0000) 2023-08-28T02:15:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:03 smithi130 bash[27579]: audit 2023-08-28T02:15:03.271961+0000 mon.a (mon.0) 4929 : audit [DBG] from='client.? 172.21.15.6:0/401769747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:03 smithi130 bash[27579]: cluster 2023-08-28T02:15:03.331446+0000 mgr.y (mgr.14901) 7693 : cluster [DBG] pgmap v7700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:04.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:03 smithi006 bash[27292]: audit 2023-08-28T02:15:03.271961+0000 mon.a (mon.0) 4929 : audit [DBG] from='client.? 172.21.15.6:0/401769747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:04.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:03 smithi006 bash[27292]: cluster 2023-08-28T02:15:03.331446+0000 mgr.y (mgr.14901) 7693 : cluster [DBG] pgmap v7700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:04.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:03 smithi006 bash[33918]: audit 2023-08-28T02:15:03.271961+0000 mon.a (mon.0) 4929 : audit [DBG] from='client.? 172.21.15.6:0/401769747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:04.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:03 smithi006 bash[33918]: cluster 2023-08-28T02:15:03.331446+0000 mgr.y (mgr.14901) 7693 : cluster [DBG] pgmap v7700: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:04.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:04 smithi130 bash[51302]: debug 2023-08-28T02:15:04.422+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:04.424058+0000) 2023-08-28T02:15:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:15:04.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:15:04.695Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:05.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:05 smithi130 bash[51302]: debug 2023-08-28T02:15:05.422+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:05.424376+0000) 2023-08-28T02:15:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:06 smithi006 bash[27292]: cluster 2023-08-28T02:15:05.332573+0000 mgr.y (mgr.14901) 7694 : cluster [DBG] pgmap v7701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:06 smithi006 bash[27292]: audit 2023-08-28T02:15:05.679175+0000 mon.c (mon.2) 3032 : audit [DBG] from='client.? 172.21.15.6:0/61958761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:06 smithi006 bash[33918]: cluster 2023-08-28T02:15:05.332573+0000 mgr.y (mgr.14901) 7694 : cluster [DBG] pgmap v7701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:06.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:06 smithi006 bash[33918]: audit 2023-08-28T02:15:05.679175+0000 mon.c (mon.2) 3032 : audit [DBG] from='client.? 172.21.15.6:0/61958761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:06.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:06 smithi130 bash[27579]: cluster 2023-08-28T02:15:05.332573+0000 mgr.y (mgr.14901) 7694 : cluster [DBG] pgmap v7701: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:06.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:06 smithi130 bash[27579]: audit 2023-08-28T02:15:05.679175+0000 mon.c (mon.2) 3032 : audit [DBG] from='client.? 172.21.15.6:0/61958761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:06.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:06 smithi130 bash[51302]: debug 2023-08-28T02:15:06.422+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:06.424688+0000) 2023-08-28T02:15:07.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:07 smithi130 bash[51302]: debug 2023-08-28T02:15:07.422+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:07.425022+0000) 2023-08-28T02:15:08.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:08 smithi006 bash[27292]: cluster 2023-08-28T02:15:07.333303+0000 mgr.y (mgr.14901) 7695 : cluster [DBG] pgmap v7702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:08 smithi006 bash[27292]: audit 2023-08-28T02:15:08.084167+0000 mon.a (mon.0) 4930 : audit [DBG] from='client.? 172.21.15.6:0/1600521734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:08.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:08 smithi006 bash[33918]: cluster 2023-08-28T02:15:07.333303+0000 mgr.y (mgr.14901) 7695 : cluster [DBG] pgmap v7702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:08.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:08 smithi006 bash[33918]: audit 2023-08-28T02:15:08.084167+0000 mon.a (mon.0) 4930 : audit [DBG] from='client.? 172.21.15.6:0/1600521734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:08 smithi130 bash[27579]: cluster 2023-08-28T02:15:07.333303+0000 mgr.y (mgr.14901) 7695 : cluster [DBG] pgmap v7702: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:08 smithi130 bash[27579]: audit 2023-08-28T02:15:08.084167+0000 mon.a (mon.0) 4930 : audit [DBG] from='client.? 172.21.15.6:0/1600521734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:08.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:08 smithi130 bash[51302]: debug 2023-08-28T02:15:08.422+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:08.425361+0000) 2023-08-28T02:15:09.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:09 smithi130 bash[51302]: debug 2023-08-28T02:15:09.422+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:09.425683+0000) 2023-08-28T02:15:10.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:10 smithi006 bash[33918]: cluster 2023-08-28T02:15:09.334342+0000 mgr.y (mgr.14901) 7696 : cluster [DBG] pgmap v7703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:10 smithi006 bash[27292]: cluster 2023-08-28T02:15:09.334342+0000 mgr.y (mgr.14901) 7696 : cluster [DBG] pgmap v7703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:10.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:10 smithi130 bash[27579]: cluster 2023-08-28T02:15:09.334342+0000 mgr.y (mgr.14901) 7696 : cluster [DBG] pgmap v7703: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:10.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:10 smithi130 bash[51302]: debug 2023-08-28T02:15:10.422+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:10.426029+0000) 2023-08-28T02:15:11.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:15:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:15:10] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T02:15:11.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:11 smithi006 bash[33918]: audit 2023-08-28T02:15:10.513665+0000 mon.c (mon.2) 3033 : audit [DBG] from='client.? 172.21.15.6:0/263375768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:11.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:11 smithi006 bash[27292]: audit 2023-08-28T02:15:10.513665+0000 mon.c (mon.2) 3033 : audit [DBG] from='client.? 172.21.15.6:0/263375768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:15:11.248Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:15:11.250Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:11.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:11 smithi130 bash[27579]: audit 2023-08-28T02:15:10.513665+0000 mon.c (mon.2) 3033 : audit [DBG] from='client.? 172.21.15.6:0/263375768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:11.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:11 smithi130 bash[51302]: debug 2023-08-28T02:15:11.421+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:11.426390+0000) 2023-08-28T02:15:12.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:12 smithi006 bash[27292]: cluster 2023-08-28T02:15:11.335003+0000 mgr.y (mgr.14901) 7697 : cluster [DBG] pgmap v7704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:12.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:12 smithi006 bash[33918]: cluster 2023-08-28T02:15:11.335003+0000 mgr.y (mgr.14901) 7697 : cluster [DBG] pgmap v7704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:12 smithi130 bash[27579]: cluster 2023-08-28T02:15:11.335003+0000 mgr.y (mgr.14901) 7697 : cluster [DBG] pgmap v7704: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:12.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:12 smithi130 bash[51302]: debug 2023-08-28T02:15:12.421+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:12.426691+0000) 2023-08-28T02:15:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:15:12.815Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:15:12.816Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:13.809 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:13 smithi130 bash[51302]: debug 2023-08-28T02:15:13.425+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:13.427022+0000) 2023-08-28T02:15:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:13 smithi130 bash[27579]: audit 2023-08-28T02:15:12.935595+0000 mon.a (mon.0) 4931 : audit [DBG] from='client.? 172.21.15.6:0/1436246165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:13 smithi130 bash[27579]: cluster 2023-08-28T02:15:13.335713+0000 mgr.y (mgr.14901) 7698 : cluster [DBG] pgmap v7705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:14.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:13 smithi006 bash[27292]: audit 2023-08-28T02:15:12.935595+0000 mon.a (mon.0) 4931 : audit [DBG] from='client.? 172.21.15.6:0/1436246165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:13 smithi006 bash[27292]: cluster 2023-08-28T02:15:13.335713+0000 mgr.y (mgr.14901) 7698 : cluster [DBG] pgmap v7705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:13 smithi006 bash[33918]: audit 2023-08-28T02:15:12.935595+0000 mon.a (mon.0) 4931 : audit [DBG] from='client.? 172.21.15.6:0/1436246165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:13 smithi006 bash[33918]: cluster 2023-08-28T02:15:13.335713+0000 mgr.y (mgr.14901) 7698 : cluster [DBG] pgmap v7705: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:14.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:14 smithi130 bash[51302]: debug 2023-08-28T02:15:14.425+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:14.427377+0000) 2023-08-28T02:15:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:15:14.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:15:14.696Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:15.670 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:15 smithi130 bash[27579]: audit 2023-08-28T02:15:15.347747+0000 mon.a (mon.0) 4932 : audit [DBG] from='client.? 172.21.15.6:0/1803073263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:15.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:15 smithi006 bash[33918]: audit 2023-08-28T02:15:15.347747+0000 mon.a (mon.0) 4932 : audit [DBG] from='client.? 172.21.15.6:0/1803073263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:15.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:15 smithi006 bash[27292]: audit 2023-08-28T02:15:15.347747+0000 mon.a (mon.0) 4932 : audit [DBG] from='client.? 172.21.15.6:0/1803073263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:16.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:15 smithi130 bash[51302]: debug 2023-08-28T02:15:15.425+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:15.427673+0000) 2023-08-28T02:15:16.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:16 smithi006 bash[33918]: cluster 2023-08-28T02:15:15.336788+0000 mgr.y (mgr.14901) 7699 : cluster [DBG] pgmap v7706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:16 smithi006 bash[27292]: cluster 2023-08-28T02:15:15.336788+0000 mgr.y (mgr.14901) 7699 : cluster [DBG] pgmap v7706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:16.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:16 smithi130 bash[27579]: cluster 2023-08-28T02:15:15.336788+0000 mgr.y (mgr.14901) 7699 : cluster [DBG] pgmap v7706: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:16.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:16 smithi130 bash[51302]: debug 2023-08-28T02:15:16.425+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:16.428026+0000) 2023-08-28T02:15:17.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:17 smithi130 bash[51302]: debug 2023-08-28T02:15:17.425+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:17.428380+0000) 2023-08-28T02:15:18.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:18 smithi006 bash[33918]: cluster 2023-08-28T02:15:17.337516+0000 mgr.y (mgr.14901) 7700 : cluster [DBG] pgmap v7707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:18.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:18 smithi006 bash[33918]: audit 2023-08-28T02:15:17.758170+0000 mon.c (mon.2) 3034 : audit [DBG] from='client.? 172.21.15.6:0/1243057647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:18.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:18 smithi006 bash[27292]: cluster 2023-08-28T02:15:17.337516+0000 mgr.y (mgr.14901) 7700 : cluster [DBG] pgmap v7707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:18.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:18 smithi006 bash[27292]: audit 2023-08-28T02:15:17.758170+0000 mon.c (mon.2) 3034 : audit [DBG] from='client.? 172.21.15.6:0/1243057647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:18.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:18 smithi130 bash[27579]: cluster 2023-08-28T02:15:17.337516+0000 mgr.y (mgr.14901) 7700 : cluster [DBG] pgmap v7707: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:18.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:18 smithi130 bash[27579]: audit 2023-08-28T02:15:17.758170+0000 mon.c (mon.2) 3034 : audit [DBG] from='client.? 172.21.15.6:0/1243057647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:18.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:18 smithi130 bash[51302]: debug 2023-08-28T02:15:18.425+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:18.428722+0000) 2023-08-28T02:15:19.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:19 smithi130 bash[51302]: debug 2023-08-28T02:15:19.425+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:19.429040+0000) 2023-08-28T02:15:20.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:20 smithi006 bash[33918]: cluster 2023-08-28T02:15:19.338486+0000 mgr.y (mgr.14901) 7701 : cluster [DBG] pgmap v7708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:20.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:20 smithi006 bash[33918]: audit 2023-08-28T02:15:20.187678+0000 mon.c (mon.2) 3035 : audit [DBG] from='client.? 172.21.15.6:0/2898792195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:20.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:20 smithi006 bash[27292]: cluster 2023-08-28T02:15:19.338486+0000 mgr.y (mgr.14901) 7701 : cluster [DBG] pgmap v7708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:20.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:20 smithi006 bash[27292]: audit 2023-08-28T02:15:20.187678+0000 mon.c (mon.2) 3035 : audit [DBG] from='client.? 172.21.15.6:0/2898792195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:20.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:20 smithi130 bash[51302]: debug 2023-08-28T02:15:20.425+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:20.429409+0000) 2023-08-28T02:15:20.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:20 smithi130 bash[27579]: cluster 2023-08-28T02:15:19.338486+0000 mgr.y (mgr.14901) 7701 : cluster [DBG] pgmap v7708: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:20.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:20 smithi130 bash[27579]: audit 2023-08-28T02:15:20.187678+0000 mon.c (mon.2) 3035 : audit [DBG] from='client.? 172.21.15.6:0/2898792195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:15:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:15:20] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-28T02:15:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:15:21.249Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:15:21.252Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:21.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:21 smithi130 bash[51302]: debug 2023-08-28T02:15:21.425+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:21.429689+0000) 2023-08-28T02:15:22.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:22 smithi006 bash[27292]: cluster 2023-08-28T02:15:21.339200+0000 mgr.y (mgr.14901) 7702 : cluster [DBG] pgmap v7709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:22.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:22 smithi006 bash[33918]: cluster 2023-08-28T02:15:21.339200+0000 mgr.y (mgr.14901) 7702 : cluster [DBG] pgmap v7709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:22 smithi130 bash[27579]: cluster 2023-08-28T02:15:21.339200+0000 mgr.y (mgr.14901) 7702 : cluster [DBG] pgmap v7709: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:22.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:22 smithi130 bash[51302]: debug 2023-08-28T02:15:22.425+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:22.429955+0000) 2023-08-28T02:15:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:15:22.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:15:22.818Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:23.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:23 smithi006 bash[27292]: audit 2023-08-28T02:15:22.607976+0000 mon.c (mon.2) 3036 : audit [DBG] from='client.? 172.21.15.6:0/3733027050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:23.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:23 smithi006 bash[33918]: audit 2023-08-28T02:15:22.607976+0000 mon.c (mon.2) 3036 : audit [DBG] from='client.? 172.21.15.6:0/3733027050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:23.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:23 smithi130 bash[27579]: audit 2023-08-28T02:15:22.607976+0000 mon.c (mon.2) 3036 : audit [DBG] from='client.? 172.21.15.6:0/3733027050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:23.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:23 smithi130 bash[51302]: debug 2023-08-28T02:15:23.425+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:23.430223+0000) 2023-08-28T02:15:24.694 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:24 smithi006 bash[33918]: cluster 2023-08-28T02:15:23.339901+0000 mgr.y (mgr.14901) 7703 : cluster [DBG] pgmap v7710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:24.695 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:24 smithi006 bash[27292]: cluster 2023-08-28T02:15:23.339901+0000 mgr.y (mgr.14901) 7703 : cluster [DBG] pgmap v7710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:24.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:24 smithi130 bash[27579]: cluster 2023-08-28T02:15:23.339901+0000 mgr.y (mgr.14901) 7703 : cluster [DBG] pgmap v7710: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:24.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:24 smithi130 bash[51302]: debug 2023-08-28T02:15:24.425+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:24.430482+0000) 2023-08-28T02:15:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:15:24.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:15:24.696Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:25.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:25 smithi006 bash[33918]: audit 2023-08-28T02:15:25.015974+0000 mon.a (mon.0) 4933 : audit [DBG] from='client.? 172.21.15.6:0/638938646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:25.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:25 smithi006 bash[27292]: audit 2023-08-28T02:15:25.015974+0000 mon.a (mon.0) 4933 : audit [DBG] from='client.? 172.21.15.6:0/638938646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:25.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:25 smithi130 bash[27579]: audit 2023-08-28T02:15:25.015974+0000 mon.a (mon.0) 4933 : audit [DBG] from='client.? 172.21.15.6:0/638938646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:25.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:25 smithi130 bash[51302]: debug 2023-08-28T02:15:25.425+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:25.430785+0000) 2023-08-28T02:15:26.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:26 smithi006 bash[33918]: cluster 2023-08-28T02:15:25.341080+0000 mgr.y (mgr.14901) 7704 : cluster [DBG] pgmap v7711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:26 smithi006 bash[27292]: cluster 2023-08-28T02:15:25.341080+0000 mgr.y (mgr.14901) 7704 : cluster [DBG] pgmap v7711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:26.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:26 smithi130 bash[27579]: cluster 2023-08-28T02:15:25.341080+0000 mgr.y (mgr.14901) 7704 : cluster [DBG] pgmap v7711: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:26.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:26 smithi130 bash[51302]: debug 2023-08-28T02:15:26.429+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:26.431030+0000) 2023-08-28T02:15:27.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:27 smithi006 bash[33918]: audit 2023-08-28T02:15:26.899521+0000 mon.b (mon.1) 849 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:27.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:27 smithi006 bash[33918]: audit 2023-08-28T02:15:27.428345+0000 mon.a (mon.0) 4934 : audit [DBG] from='client.? 172.21.15.6:0/2000891686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:27.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:27 smithi006 bash[27292]: audit 2023-08-28T02:15:26.899521+0000 mon.b (mon.1) 849 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:27.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:27 smithi006 bash[27292]: audit 2023-08-28T02:15:27.428345+0000 mon.a (mon.0) 4934 : audit [DBG] from='client.? 172.21.15.6:0/2000891686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:27.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:27 smithi130 bash[27579]: audit 2023-08-28T02:15:26.899521+0000 mon.b (mon.1) 849 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:27.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:27 smithi130 bash[27579]: audit 2023-08-28T02:15:27.428345+0000 mon.a (mon.0) 4934 : audit [DBG] from='client.? 172.21.15.6:0/2000891686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:27.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:27 smithi130 bash[51302]: debug 2023-08-28T02:15:27.429+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:27.431294+0000) 2023-08-28T02:15:28.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:28 smithi006 bash[33918]: cluster 2023-08-28T02:15:27.341721+0000 mgr.y (mgr.14901) 7705 : cluster [DBG] pgmap v7712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:28 smithi006 bash[27292]: cluster 2023-08-28T02:15:27.341721+0000 mgr.y (mgr.14901) 7705 : cluster [DBG] pgmap v7712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:28.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:28 smithi130 bash[27579]: cluster 2023-08-28T02:15:27.341721+0000 mgr.y (mgr.14901) 7705 : cluster [DBG] pgmap v7712: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:28.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:28 smithi130 bash[51302]: debug 2023-08-28T02:15:28.429+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:28.431574+0000) 2023-08-28T02:15:29.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:29 smithi130 bash[51302]: debug 2023-08-28T02:15:29.429+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:29.431887+0000) 2023-08-28T02:15:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:30 smithi006 bash[27292]: cluster 2023-08-28T02:15:29.342959+0000 mgr.y (mgr.14901) 7706 : cluster [DBG] pgmap v7713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:30 smithi006 bash[27292]: audit 2023-08-28T02:15:29.852102+0000 mon.a (mon.0) 4935 : audit [DBG] from='client.? 172.21.15.6:0/3660370982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:30 smithi006 bash[33918]: cluster 2023-08-28T02:15:29.342959+0000 mgr.y (mgr.14901) 7706 : cluster [DBG] pgmap v7713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:30 smithi006 bash[33918]: audit 2023-08-28T02:15:29.852102+0000 mon.a (mon.0) 4935 : audit [DBG] from='client.? 172.21.15.6:0/3660370982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:30.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:30 smithi130 bash[51302]: debug 2023-08-28T02:15:30.429+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:30.432224+0000) 2023-08-28T02:15:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:30 smithi130 bash[27579]: cluster 2023-08-28T02:15:29.342959+0000 mgr.y (mgr.14901) 7706 : cluster [DBG] pgmap v7713: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:30 smithi130 bash[27579]: audit 2023-08-28T02:15:29.852102+0000 mon.a (mon.0) 4935 : audit [DBG] from='client.? 172.21.15.6:0/3660370982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:31.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:15:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:15:30] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-28T02:15:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:15:31.251Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:15:31.253Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:31.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:31 smithi130 bash[51302]: debug 2023-08-28T02:15:31.429+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:31.432535+0000) 2023-08-28T02:15:32.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:32 smithi006 bash[27292]: cluster 2023-08-28T02:15:31.343445+0000 mgr.y (mgr.14901) 7707 : cluster [DBG] pgmap v7714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:32.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:32 smithi006 bash[27292]: audit 2023-08-28T02:15:32.278996+0000 mon.a (mon.0) 4936 : audit [DBG] from='client.? 172.21.15.6:0/2621024599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:32.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:32 smithi006 bash[33918]: cluster 2023-08-28T02:15:31.343445+0000 mgr.y (mgr.14901) 7707 : cluster [DBG] pgmap v7714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:32.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:32 smithi006 bash[33918]: audit 2023-08-28T02:15:32.278996+0000 mon.a (mon.0) 4936 : audit [DBG] from='client.? 172.21.15.6:0/2621024599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:32 smithi130 bash[27579]: cluster 2023-08-28T02:15:31.343445+0000 mgr.y (mgr.14901) 7707 : cluster [DBG] pgmap v7714: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:32 smithi130 bash[27579]: audit 2023-08-28T02:15:32.278996+0000 mon.a (mon.0) 4936 : audit [DBG] from='client.? 172.21.15.6:0/2621024599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:32.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:32 smithi130 bash[51302]: debug 2023-08-28T02:15:32.429+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:32.432792+0000) 2023-08-28T02:15:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:15:32.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:15:32.818Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:33.786 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:33 smithi130 bash[51302]: debug 2023-08-28T02:15:33.429+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:33.433024+0000) 2023-08-28T02:15:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:33 smithi130 bash[27579]: audit 2023-08-28T02:15:32.786483+0000 mon.a (mon.0) 4937 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:15:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:33 smithi130 bash[27579]: audit 2023-08-28T02:15:33.145318+0000 mon.a (mon.0) 4938 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:15:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:33 smithi130 bash[27579]: cluster 2023-08-28T02:15:33.344037+0000 mgr.y (mgr.14901) 7708 : cluster [DBG] pgmap v7715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:33 smithi130 bash[27579]: audit 2023-08-28T02:15:33.423358+0000 mon.b (mon.1) 850 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:33 smithi130 bash[27579]: audit 2023-08-28T02:15:33.425900+0000 mon.b (mon.1) 851 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:15:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:33 smithi130 bash[27579]: audit 2023-08-28T02:15:33.434231+0000 mon.a (mon.0) 4939 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:15:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:33 smithi006 bash[27292]: audit 2023-08-28T02:15:32.786483+0000 mon.a (mon.0) 4937 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:15:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:33 smithi006 bash[27292]: audit 2023-08-28T02:15:33.145318+0000 mon.a (mon.0) 4938 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:15:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:33 smithi006 bash[27292]: cluster 2023-08-28T02:15:33.344037+0000 mgr.y (mgr.14901) 7708 : cluster [DBG] pgmap v7715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:33 smithi006 bash[27292]: audit 2023-08-28T02:15:33.423358+0000 mon.b (mon.1) 850 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:33 smithi006 bash[27292]: audit 2023-08-28T02:15:33.425900+0000 mon.b (mon.1) 851 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:15:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:33 smithi006 bash[27292]: audit 2023-08-28T02:15:33.434231+0000 mon.a (mon.0) 4939 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:15:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:33 smithi006 bash[33918]: audit 2023-08-28T02:15:32.786483+0000 mon.a (mon.0) 4937 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:15:34.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:33 smithi006 bash[33918]: audit 2023-08-28T02:15:33.145318+0000 mon.a (mon.0) 4938 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:15:34.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:33 smithi006 bash[33918]: cluster 2023-08-28T02:15:33.344037+0000 mgr.y (mgr.14901) 7708 : cluster [DBG] pgmap v7715: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:34.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:33 smithi006 bash[33918]: audit 2023-08-28T02:15:33.423358+0000 mon.b (mon.1) 850 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:34.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:33 smithi006 bash[33918]: audit 2023-08-28T02:15:33.425900+0000 mon.b (mon.1) 851 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:15:34.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:33 smithi006 bash[33918]: audit 2023-08-28T02:15:33.434231+0000 mon.a (mon.0) 4939 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:15:34.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:34 smithi130 bash[51302]: debug 2023-08-28T02:15:34.429+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:34.433383+0000) 2023-08-28T02:15:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:15:34.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:15:34.697Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:35.432 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:35 smithi130 bash[27579]: audit 2023-08-28T02:15:34.703933+0000 mon.c (mon.2) 3037 : audit [DBG] from='client.? 172.21.15.6:0/30085061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:35.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:35 smithi006 bash[27292]: audit 2023-08-28T02:15:34.703933+0000 mon.c (mon.2) 3037 : audit [DBG] from='client.? 172.21.15.6:0/30085061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:35.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:35 smithi006 bash[33918]: audit 2023-08-28T02:15:34.703933+0000 mon.c (mon.2) 3037 : audit [DBG] from='client.? 172.21.15.6:0/30085061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:35.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:35 smithi130 bash[51302]: debug 2023-08-28T02:15:35.429+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:35.433664+0000) 2023-08-28T02:15:36.432 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:36 smithi130 bash[27579]: cluster 2023-08-28T02:15:35.345203+0000 mgr.y (mgr.14901) 7709 : cluster [DBG] pgmap v7716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:36.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:36 smithi006 bash[27292]: cluster 2023-08-28T02:15:35.345203+0000 mgr.y (mgr.14901) 7709 : cluster [DBG] pgmap v7716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:36.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:36 smithi006 bash[33918]: cluster 2023-08-28T02:15:35.345203+0000 mgr.y (mgr.14901) 7709 : cluster [DBG] pgmap v7716: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:36.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:36 smithi130 bash[51302]: debug 2023-08-28T02:15:36.429+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:36.433934+0000) 2023-08-28T02:15:37.432 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:37 smithi130 bash[27579]: audit 2023-08-28T02:15:37.117942+0000 mon.c (mon.2) 3038 : audit [DBG] from='client.? 172.21.15.6:0/2232384422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:37.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:37 smithi006 bash[33918]: audit 2023-08-28T02:15:37.117942+0000 mon.c (mon.2) 3038 : audit [DBG] from='client.? 172.21.15.6:0/2232384422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:37.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:37 smithi006 bash[27292]: audit 2023-08-28T02:15:37.117942+0000 mon.c (mon.2) 3038 : audit [DBG] from='client.? 172.21.15.6:0/2232384422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:37.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:37 smithi130 bash[51302]: debug 2023-08-28T02:15:37.433+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:37.434206+0000) 2023-08-28T02:15:38.433 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:38 smithi130 bash[27579]: cluster 2023-08-28T02:15:37.345852+0000 mgr.y (mgr.14901) 7710 : cluster [DBG] pgmap v7717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:38.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:38 smithi006 bash[33918]: cluster 2023-08-28T02:15:37.345852+0000 mgr.y (mgr.14901) 7710 : cluster [DBG] pgmap v7717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:38.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:38 smithi006 bash[27292]: cluster 2023-08-28T02:15:37.345852+0000 mgr.y (mgr.14901) 7710 : cluster [DBG] pgmap v7717: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:38.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:38 smithi130 bash[51302]: debug 2023-08-28T02:15:38.433+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:38.434527+0000) 2023-08-28T02:15:39.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:39 smithi130 bash[51302]: debug 2023-08-28T02:15:39.433+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:39.434811+0000) 2023-08-28T02:15:40.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:40 smithi006 bash[33918]: cluster 2023-08-28T02:15:39.346606+0000 mgr.y (mgr.14901) 7711 : cluster [DBG] pgmap v7718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:40 smithi006 bash[33918]: audit 2023-08-28T02:15:39.540521+0000 mon.a (mon.0) 4940 : audit [DBG] from='client.? 172.21.15.6:0/2771803843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:40 smithi006 bash[27292]: cluster 2023-08-28T02:15:39.346606+0000 mgr.y (mgr.14901) 7711 : cluster [DBG] pgmap v7718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:40 smithi006 bash[27292]: audit 2023-08-28T02:15:39.540521+0000 mon.a (mon.0) 4940 : audit [DBG] from='client.? 172.21.15.6:0/2771803843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:40.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:40 smithi130 bash[27579]: cluster 2023-08-28T02:15:39.346606+0000 mgr.y (mgr.14901) 7711 : cluster [DBG] pgmap v7718: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:40.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:40 smithi130 bash[27579]: audit 2023-08-28T02:15:39.540521+0000 mon.a (mon.0) 4940 : audit [DBG] from='client.? 172.21.15.6:0/2771803843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:40.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:40 smithi130 bash[51302]: debug 2023-08-28T02:15:40.433+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:40.435121+0000) 2023-08-28T02:15:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:15:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:15:40] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-28T02:15:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:15:41.252Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:15:41.254Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:41.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:41 smithi130 bash[51302]: debug 2023-08-28T02:15:41.433+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:41.435422+0000) 2023-08-28T02:15:42.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:42 smithi006 bash[33918]: cluster 2023-08-28T02:15:41.347245+0000 mgr.y (mgr.14901) 7712 : cluster [DBG] pgmap v7719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:42.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:42 smithi006 bash[33918]: audit 2023-08-28T02:15:41.964201+0000 mon.c (mon.2) 3039 : audit [DBG] from='client.? 172.21.15.6:0/3580819536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:42.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:42 smithi006 bash[27292]: cluster 2023-08-28T02:15:41.347245+0000 mgr.y (mgr.14901) 7712 : cluster [DBG] pgmap v7719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:42.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:42 smithi006 bash[27292]: audit 2023-08-28T02:15:41.964201+0000 mon.c (mon.2) 3039 : audit [DBG] from='client.? 172.21.15.6:0/3580819536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:42.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:42 smithi130 bash[27579]: cluster 2023-08-28T02:15:41.347245+0000 mgr.y (mgr.14901) 7712 : cluster [DBG] pgmap v7719: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:42.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:42 smithi130 bash[27579]: audit 2023-08-28T02:15:41.964201+0000 mon.c (mon.2) 3039 : audit [DBG] from='client.? 172.21.15.6:0/3580819536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:42.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:42 smithi130 bash[51302]: debug 2023-08-28T02:15:42.433+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:42.435697+0000) 2023-08-28T02:15:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:15:42.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:15:42.819Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:43.815 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:43 smithi130 bash[51302]: debug 2023-08-28T02:15:43.433+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:43.435945+0000) 2023-08-28T02:15:44.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:43 smithi130 bash[27579]: cluster 2023-08-28T02:15:43.347902+0000 mgr.y (mgr.14901) 7713 : cluster [DBG] pgmap v7720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:44.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:43 smithi006 bash[27292]: cluster 2023-08-28T02:15:43.347902+0000 mgr.y (mgr.14901) 7713 : cluster [DBG] pgmap v7720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:44.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:43 smithi006 bash[33918]: cluster 2023-08-28T02:15:43.347902+0000 mgr.y (mgr.14901) 7713 : cluster [DBG] pgmap v7720: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:44.818 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:44 smithi130 bash[51302]: debug 2023-08-28T02:15:44.433+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:44.436172+0000) 2023-08-28T02:15:44.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:44 smithi006 bash[27292]: audit 2023-08-28T02:15:44.378819+0000 mon.c (mon.2) 3040 : audit [DBG] from='client.? 172.21.15.6:0/2643852392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:44.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:44 smithi006 bash[33918]: audit 2023-08-28T02:15:44.378819+0000 mon.c (mon.2) 3040 : audit [DBG] from='client.? 172.21.15.6:0/2643852392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:15:44.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:15:44.697Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:45.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:44 smithi130 bash[27579]: audit 2023-08-28T02:15:44.378819+0000 mon.c (mon.2) 3040 : audit [DBG] from='client.? 172.21.15.6:0/2643852392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:45.829 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:45 smithi130 bash[51302]: debug 2023-08-28T02:15:45.433+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:45.436462+0000) 2023-08-28T02:15:46.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:45 smithi130 bash[27579]: cluster 2023-08-28T02:15:45.349041+0000 mgr.y (mgr.14901) 7714 : cluster [DBG] pgmap v7721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:46.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:45 smithi006 bash[33918]: cluster 2023-08-28T02:15:45.349041+0000 mgr.y (mgr.14901) 7714 : cluster [DBG] pgmap v7721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:46.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:45 smithi006 bash[27292]: cluster 2023-08-28T02:15:45.349041+0000 mgr.y (mgr.14901) 7714 : cluster [DBG] pgmap v7721: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:46.832 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:46 smithi130 bash[51302]: debug 2023-08-28T02:15:46.433+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:46.436770+0000) 2023-08-28T02:15:47.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:46 smithi130 bash[27579]: audit 2023-08-28T02:15:46.795771+0000 mon.c (mon.2) 3041 : audit [DBG] from='client.? 172.21.15.6:0/3456393594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:47.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:46 smithi006 bash[33918]: audit 2023-08-28T02:15:46.795771+0000 mon.c (mon.2) 3041 : audit [DBG] from='client.? 172.21.15.6:0/3456393594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:47.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:46 smithi006 bash[27292]: audit 2023-08-28T02:15:46.795771+0000 mon.c (mon.2) 3041 : audit [DBG] from='client.? 172.21.15.6:0/3456393594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:47.840 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:47 smithi130 bash[51302]: debug 2023-08-28T02:15:47.433+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:47.437031+0000) 2023-08-28T02:15:48.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:47 smithi130 bash[27579]: cluster 2023-08-28T02:15:47.349819+0000 mgr.y (mgr.14901) 7715 : cluster [DBG] pgmap v7722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:48.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:47 smithi006 bash[33918]: cluster 2023-08-28T02:15:47.349819+0000 mgr.y (mgr.14901) 7715 : cluster [DBG] pgmap v7722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:48.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:47 smithi006 bash[27292]: cluster 2023-08-28T02:15:47.349819+0000 mgr.y (mgr.14901) 7715 : cluster [DBG] pgmap v7722: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:48.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:48 smithi130 bash[51302]: debug 2023-08-28T02:15:48.433+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:48.437297+0000) 2023-08-28T02:15:49.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:49 smithi130 bash[27579]: audit 2023-08-28T02:15:49.212304+0000 mon.c (mon.2) 3042 : audit [DBG] from='client.? 172.21.15.6:0/745845211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:49.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:49 smithi130 bash[51302]: debug 2023-08-28T02:15:49.433+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:49.437578+0000) 2023-08-28T02:15:49.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:49 smithi006 bash[33918]: audit 2023-08-28T02:15:49.212304+0000 mon.c (mon.2) 3042 : audit [DBG] from='client.? 172.21.15.6:0/745845211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:49.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:49 smithi006 bash[27292]: audit 2023-08-28T02:15:49.212304+0000 mon.c (mon.2) 3042 : audit [DBG] from='client.? 172.21.15.6:0/745845211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:50 smithi130 bash[27579]: cluster 2023-08-28T02:15:49.351020+0000 mgr.y (mgr.14901) 7716 : cluster [DBG] pgmap v7723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:50.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:50 smithi130 bash[51302]: debug 2023-08-28T02:15:50.437+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:50.437860+0000) 2023-08-28T02:15:50.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:50 smithi006 bash[27292]: cluster 2023-08-28T02:15:49.351020+0000 mgr.y (mgr.14901) 7716 : cluster [DBG] pgmap v7723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:50.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:50 smithi006 bash[33918]: cluster 2023-08-28T02:15:49.351020+0000 mgr.y (mgr.14901) 7716 : cluster [DBG] pgmap v7723: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:15:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:15:50] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T02:15:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:15:51.252Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:15:51.255Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:51.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:51 smithi130 bash[51302]: debug 2023-08-28T02:15:51.433+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:51.438243+0000) 2023-08-28T02:15:52.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:52 smithi006 bash[33918]: cluster 2023-08-28T02:15:51.351693+0000 mgr.y (mgr.14901) 7717 : cluster [DBG] pgmap v7724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:52.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:52 smithi006 bash[33918]: audit 2023-08-28T02:15:51.627190+0000 mon.a (mon.0) 4941 : audit [DBG] from='client.? 172.21.15.6:0/2305471012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:52.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:52 smithi006 bash[27292]: cluster 2023-08-28T02:15:51.351693+0000 mgr.y (mgr.14901) 7717 : cluster [DBG] pgmap v7724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:52.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:52 smithi006 bash[27292]: audit 2023-08-28T02:15:51.627190+0000 mon.a (mon.0) 4941 : audit [DBG] from='client.? 172.21.15.6:0/2305471012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:52.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:52 smithi130 bash[51302]: debug 2023-08-28T02:15:52.433+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:52.438583+0000) 2023-08-28T02:15:52.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:52 smithi130 bash[27579]: cluster 2023-08-28T02:15:51.351693+0000 mgr.y (mgr.14901) 7717 : cluster [DBG] pgmap v7724: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:52.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:52 smithi130 bash[27579]: audit 2023-08-28T02:15:51.627190+0000 mon.a (mon.0) 4941 : audit [DBG] from='client.? 172.21.15.6:0/2305471012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:15:52.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:15:52.819Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:53.818 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:53 smithi130 bash[51302]: debug 2023-08-28T02:15:53.437+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:53.438943+0000) 2023-08-28T02:15:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:53 smithi130 bash[27579]: cluster 2023-08-28T02:15:53.352380+0000 mgr.y (mgr.14901) 7718 : cluster [DBG] pgmap v7725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:54.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:53 smithi006 bash[33918]: cluster 2023-08-28T02:15:53.352380+0000 mgr.y (mgr.14901) 7718 : cluster [DBG] pgmap v7725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:54.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:53 smithi006 bash[27292]: cluster 2023-08-28T02:15:53.352380+0000 mgr.y (mgr.14901) 7718 : cluster [DBG] pgmap v7725: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:54.821 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:54 smithi130 bash[51302]: debug 2023-08-28T02:15:54.437+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:54.439205+0000) 2023-08-28T02:15:54.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:54 smithi006 bash[27292]: audit 2023-08-28T02:15:54.038775+0000 mon.a (mon.0) 4942 : audit [DBG] from='client.? 172.21.15.6:0/2589106682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:54.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:54 smithi006 bash[33918]: audit 2023-08-28T02:15:54.038775+0000 mon.a (mon.0) 4942 : audit [DBG] from='client.? 172.21.15.6:0/2589106682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:15:54.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:15:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:15:54.698Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:15:55.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:54 smithi130 bash[27579]: audit 2023-08-28T02:15:54.038775+0000 mon.a (mon.0) 4942 : audit [DBG] from='client.? 172.21.15.6:0/2589106682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:55.828 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:55 smithi130 bash[51302]: debug 2023-08-28T02:15:55.437+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:55.439572+0000) 2023-08-28T02:15:56.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:55 smithi130 bash[27579]: cluster 2023-08-28T02:15:55.353463+0000 mgr.y (mgr.14901) 7719 : cluster [DBG] pgmap v7726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:56.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:55 smithi006 bash[27292]: cluster 2023-08-28T02:15:55.353463+0000 mgr.y (mgr.14901) 7719 : cluster [DBG] pgmap v7726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:56.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:55 smithi006 bash[33918]: cluster 2023-08-28T02:15:55.353463+0000 mgr.y (mgr.14901) 7719 : cluster [DBG] pgmap v7726: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:56.832 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:56 smithi130 bash[51302]: debug 2023-08-28T02:15:56.437+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:56.439894+0000) 2023-08-28T02:15:57.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:56 smithi130 bash[27579]: audit 2023-08-28T02:15:56.451600+0000 mon.a (mon.0) 4943 : audit [DBG] from='client.? 172.21.15.6:0/3634424462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:57.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:56 smithi006 bash[33918]: audit 2023-08-28T02:15:56.451600+0000 mon.a (mon.0) 4943 : audit [DBG] from='client.? 172.21.15.6:0/3634424462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:57.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:56 smithi006 bash[27292]: audit 2023-08-28T02:15:56.451600+0000 mon.a (mon.0) 4943 : audit [DBG] from='client.? 172.21.15.6:0/3634424462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:57.838 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:57 smithi130 bash[51302]: debug 2023-08-28T02:15:57.437+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:57.440359+0000) 2023-08-28T02:15:58.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:57 smithi130 bash[27579]: cluster 2023-08-28T02:15:57.354190+0000 mgr.y (mgr.14901) 7720 : cluster [DBG] pgmap v7727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:58.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:57 smithi006 bash[33918]: cluster 2023-08-28T02:15:57.354190+0000 mgr.y (mgr.14901) 7720 : cluster [DBG] pgmap v7727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:58.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:57 smithi006 bash[27292]: cluster 2023-08-28T02:15:57.354190+0000 mgr.y (mgr.14901) 7720 : cluster [DBG] pgmap v7727: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:15:58.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:58 smithi130 bash[51302]: debug 2023-08-28T02:15:58.437+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:58.440648+0000) 2023-08-28T02:15:59.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:58 smithi006 bash[33918]: audit 2023-08-28T02:15:58.868201+0000 mon.a (mon.0) 4944 : audit [DBG] from='client.? 172.21.15.6:0/1017617695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:59.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:58 smithi006 bash[27292]: audit 2023-08-28T02:15:58.868201+0000 mon.a (mon.0) 4944 : audit [DBG] from='client.? 172.21.15.6:0/1017617695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:59.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:58 smithi130 bash[27579]: audit 2023-08-28T02:15:58.868201+0000 mon.a (mon.0) 4944 : audit [DBG] from='client.? 172.21.15.6:0/1017617695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:59.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:15:59 smithi130 bash[51302]: debug 2023-08-28T02:15:59.437+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:59.440928+0000) 2023-08-28T02:16:00.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:15:59 smithi006 bash[33918]: cluster 2023-08-28T02:15:59.355345+0000 mgr.y (mgr.14901) 7721 : cluster [DBG] pgmap v7728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:00.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:15:59 smithi006 bash[27292]: cluster 2023-08-28T02:15:59.355345+0000 mgr.y (mgr.14901) 7721 : cluster [DBG] pgmap v7728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:00.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:15:59 smithi130 bash[27579]: cluster 2023-08-28T02:15:59.355345+0000 mgr.y (mgr.14901) 7721 : cluster [DBG] pgmap v7728: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:00.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:00 smithi130 bash[51302]: debug 2023-08-28T02:16:00.437+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:00.441208+0000) 2023-08-28T02:16:01.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:16:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:16:00] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:16:01.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:01 smithi130 bash[27579]: audit 2023-08-28T02:16:01.281977+0000 mon.a (mon.0) 4945 : audit [DBG] from='client.? 172.21.15.6:0/3395239073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:01.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:01 smithi130 bash[51302]: debug 2023-08-28T02:16:01.437+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:01.441562+0000) 2023-08-28T02:16:01.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:01 smithi006 bash[27292]: audit 2023-08-28T02:16:01.281977+0000 mon.a (mon.0) 4945 : audit [DBG] from='client.? 172.21.15.6:0/3395239073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:01.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:01 smithi006 bash[33918]: audit 2023-08-28T02:16:01.281977+0000 mon.a (mon.0) 4945 : audit [DBG] from='client.? 172.21.15.6:0/3395239073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:01.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:16:01.253Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:01.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:16:01.254Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:02.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:02 smithi130 bash[27579]: cluster 2023-08-28T02:16:01.355961+0000 mgr.y (mgr.14901) 7722 : cluster [DBG] pgmap v7729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:02.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:02 smithi130 bash[51302]: debug 2023-08-28T02:16:02.437+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:02.441806+0000) 2023-08-28T02:16:02.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:02 smithi006 bash[33918]: cluster 2023-08-28T02:16:01.355961+0000 mgr.y (mgr.14901) 7722 : cluster [DBG] pgmap v7729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:02.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:02 smithi006 bash[27292]: cluster 2023-08-28T02:16:01.355961+0000 mgr.y (mgr.14901) 7722 : cluster [DBG] pgmap v7729: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:16:02.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:16:02.819Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:03.823 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:03 smithi130 bash[51302]: debug 2023-08-28T02:16:03.437+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:03.442121+0000) 2023-08-28T02:16:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:03 smithi130 bash[27579]: cluster 2023-08-28T02:16:03.356620+0000 mgr.y (mgr.14901) 7723 : cluster [DBG] pgmap v7730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:03 smithi130 bash[27579]: audit 2023-08-28T02:16:03.690536+0000 mon.c (mon.2) 3043 : audit [DBG] from='client.? 172.21.15.6:0/3123750859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:04.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:03 smithi006 bash[27292]: cluster 2023-08-28T02:16:03.356620+0000 mgr.y (mgr.14901) 7723 : cluster [DBG] pgmap v7730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:04.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:03 smithi006 bash[27292]: audit 2023-08-28T02:16:03.690536+0000 mon.c (mon.2) 3043 : audit [DBG] from='client.? 172.21.15.6:0/3123750859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:04.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:03 smithi006 bash[33918]: cluster 2023-08-28T02:16:03.356620+0000 mgr.y (mgr.14901) 7723 : cluster [DBG] pgmap v7730: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:04.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:03 smithi006 bash[33918]: audit 2023-08-28T02:16:03.690536+0000 mon.c (mon.2) 3043 : audit [DBG] from='client.? 172.21.15.6:0/3123750859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:04.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:04 smithi130 bash[51302]: debug 2023-08-28T02:16:04.437+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:04.442389+0000) 2023-08-28T02:16:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:16:04.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:16:04.697Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:05.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:05 smithi130 bash[51302]: debug 2023-08-28T02:16:05.437+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:05.442714+0000) 2023-08-28T02:16:06.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:06 smithi006 bash[33918]: cluster 2023-08-28T02:16:05.357779+0000 mgr.y (mgr.14901) 7724 : cluster [DBG] pgmap v7731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:06 smithi006 bash[33918]: audit 2023-08-28T02:16:06.099611+0000 mon.c (mon.2) 3044 : audit [DBG] from='client.? 172.21.15.6:0/3813135253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:06 smithi006 bash[27292]: cluster 2023-08-28T02:16:05.357779+0000 mgr.y (mgr.14901) 7724 : cluster [DBG] pgmap v7731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:06 smithi006 bash[27292]: audit 2023-08-28T02:16:06.099611+0000 mon.c (mon.2) 3044 : audit [DBG] from='client.? 172.21.15.6:0/3813135253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:06.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:06 smithi130 bash[27579]: cluster 2023-08-28T02:16:05.357779+0000 mgr.y (mgr.14901) 7724 : cluster [DBG] pgmap v7731: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:06.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:06 smithi130 bash[27579]: audit 2023-08-28T02:16:06.099611+0000 mon.c (mon.2) 3044 : audit [DBG] from='client.? 172.21.15.6:0/3813135253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:06.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:06 smithi130 bash[51302]: debug 2023-08-28T02:16:06.441+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:06.443012+0000) 2023-08-28T02:16:07.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:07 smithi130 bash[51302]: debug 2023-08-28T02:16:07.440+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:07.443287+0000) 2023-08-28T02:16:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:08 smithi006 bash[33918]: cluster 2023-08-28T02:16:07.358381+0000 mgr.y (mgr.14901) 7725 : cluster [DBG] pgmap v7732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:08 smithi006 bash[27292]: cluster 2023-08-28T02:16:07.358381+0000 mgr.y (mgr.14901) 7725 : cluster [DBG] pgmap v7732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:08 smithi130 bash[27579]: cluster 2023-08-28T02:16:07.358381+0000 mgr.y (mgr.14901) 7725 : cluster [DBG] pgmap v7732: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:08.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:08 smithi130 bash[51302]: debug 2023-08-28T02:16:08.440+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:08.443608+0000) 2023-08-28T02:16:09.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:09 smithi006 bash[33918]: audit 2023-08-28T02:16:08.523965+0000 mon.a (mon.0) 4946 : audit [DBG] from='client.? 172.21.15.6:0/932658530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:09.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:09 smithi006 bash[27292]: audit 2023-08-28T02:16:08.523965+0000 mon.a (mon.0) 4946 : audit [DBG] from='client.? 172.21.15.6:0/932658530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:09.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:09 smithi130 bash[51302]: debug 2023-08-28T02:16:09.440+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:09.443933+0000) 2023-08-28T02:16:09.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:09 smithi130 bash[27579]: audit 2023-08-28T02:16:08.523965+0000 mon.a (mon.0) 4946 : audit [DBG] from='client.? 172.21.15.6:0/932658530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:10 smithi006 bash[33918]: cluster 2023-08-28T02:16:09.359461+0000 mgr.y (mgr.14901) 7726 : cluster [DBG] pgmap v7733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:10 smithi006 bash[27292]: cluster 2023-08-28T02:16:09.359461+0000 mgr.y (mgr.14901) 7726 : cluster [DBG] pgmap v7733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:10.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:10 smithi130 bash[51302]: debug 2023-08-28T02:16:10.440+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:10.444146+0000) 2023-08-28T02:16:10.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:10 smithi130 bash[27579]: cluster 2023-08-28T02:16:09.359461+0000 mgr.y (mgr.14901) 7726 : cluster [DBG] pgmap v7733: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:11.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:16:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:16:10] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:16:11.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:11 smithi006 bash[27292]: audit 2023-08-28T02:16:10.940188+0000 mon.a (mon.0) 4947 : audit [DBG] from='client.? 172.21.15.6:0/514350084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:11.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:11 smithi006 bash[33918]: audit 2023-08-28T02:16:10.940188+0000 mon.a (mon.0) 4947 : audit [DBG] from='client.? 172.21.15.6:0/514350084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:16:11.253Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:16:11.256Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:11.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:11 smithi130 bash[27579]: audit 2023-08-28T02:16:10.940188+0000 mon.a (mon.0) 4947 : audit [DBG] from='client.? 172.21.15.6:0/514350084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:11.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:11 smithi130 bash[51302]: debug 2023-08-28T02:16:11.440+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:11.444356+0000) 2023-08-28T02:16:12.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:12 smithi006 bash[33918]: cluster 2023-08-28T02:16:11.360287+0000 mgr.y (mgr.14901) 7727 : cluster [DBG] pgmap v7734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:12.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:12 smithi006 bash[27292]: cluster 2023-08-28T02:16:11.360287+0000 mgr.y (mgr.14901) 7727 : cluster [DBG] pgmap v7734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:12 smithi130 bash[27579]: cluster 2023-08-28T02:16:11.360287+0000 mgr.y (mgr.14901) 7727 : cluster [DBG] pgmap v7734: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:12.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:12 smithi130 bash[51302]: debug 2023-08-28T02:16:12.440+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:12.444660+0000) 2023-08-28T02:16:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:16:12.819Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:16:12.821Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:13.825 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:13 smithi130 bash[51302]: debug 2023-08-28T02:16:13.440+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:13.444921+0000) 2023-08-28T02:16:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:13 smithi130 bash[27579]: audit 2023-08-28T02:16:13.356139+0000 mon.a (mon.0) 4948 : audit [DBG] from='client.? 172.21.15.6:0/3894690216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:13 smithi130 bash[27579]: cluster 2023-08-28T02:16:13.360863+0000 mgr.y (mgr.14901) 7728 : cluster [DBG] pgmap v7735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:14.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:13 smithi006 bash[33918]: audit 2023-08-28T02:16:13.356139+0000 mon.a (mon.0) 4948 : audit [DBG] from='client.? 172.21.15.6:0/3894690216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:13 smithi006 bash[33918]: cluster 2023-08-28T02:16:13.360863+0000 mgr.y (mgr.14901) 7728 : cluster [DBG] pgmap v7735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:13 smithi006 bash[27292]: audit 2023-08-28T02:16:13.356139+0000 mon.a (mon.0) 4948 : audit [DBG] from='client.? 172.21.15.6:0/3894690216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:13 smithi006 bash[27292]: cluster 2023-08-28T02:16:13.360863+0000 mgr.y (mgr.14901) 7728 : cluster [DBG] pgmap v7735: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:14.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:14 smithi130 bash[51302]: debug 2023-08-28T02:16:14.444+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:14.445244+0000) 2023-08-28T02:16:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:16:14.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:16:14.698Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:15.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:15 smithi130 bash[51302]: debug 2023-08-28T02:16:15.440+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:15.445542+0000) 2023-08-28T02:16:16.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:16 smithi006 bash[33918]: cluster 2023-08-28T02:16:15.361937+0000 mgr.y (mgr.14901) 7729 : cluster [DBG] pgmap v7736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:16 smithi006 bash[33918]: audit 2023-08-28T02:16:15.777041+0000 mon.a (mon.0) 4949 : audit [DBG] from='client.? 172.21.15.6:0/2747355578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:16 smithi006 bash[27292]: cluster 2023-08-28T02:16:15.361937+0000 mgr.y (mgr.14901) 7729 : cluster [DBG] pgmap v7736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:16 smithi006 bash[27292]: audit 2023-08-28T02:16:15.777041+0000 mon.a (mon.0) 4949 : audit [DBG] from='client.? 172.21.15.6:0/2747355578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:16.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:16 smithi130 bash[51302]: debug 2023-08-28T02:16:16.440+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:16.445861+0000) 2023-08-28T02:16:16.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:16 smithi130 bash[27579]: cluster 2023-08-28T02:16:15.361937+0000 mgr.y (mgr.14901) 7729 : cluster [DBG] pgmap v7736: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:16.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:16 smithi130 bash[27579]: audit 2023-08-28T02:16:15.777041+0000 mon.a (mon.0) 4949 : audit [DBG] from='client.? 172.21.15.6:0/2747355578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:17.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:17 smithi130 bash[51302]: debug 2023-08-28T02:16:17.444+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:17.446125+0000) 2023-08-28T02:16:18.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:18 smithi006 bash[33918]: cluster 2023-08-28T02:16:17.362584+0000 mgr.y (mgr.14901) 7730 : cluster [DBG] pgmap v7737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:18.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:18 smithi006 bash[33918]: audit 2023-08-28T02:16:18.194267+0000 mon.c (mon.2) 3045 : audit [DBG] from='client.? 172.21.15.6:0/4099904333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:18.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:18 smithi006 bash[27292]: cluster 2023-08-28T02:16:17.362584+0000 mgr.y (mgr.14901) 7730 : cluster [DBG] pgmap v7737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:18.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:18 smithi006 bash[27292]: audit 2023-08-28T02:16:18.194267+0000 mon.c (mon.2) 3045 : audit [DBG] from='client.? 172.21.15.6:0/4099904333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:18.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:18 smithi130 bash[27579]: cluster 2023-08-28T02:16:17.362584+0000 mgr.y (mgr.14901) 7730 : cluster [DBG] pgmap v7737: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:18.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:18 smithi130 bash[27579]: audit 2023-08-28T02:16:18.194267+0000 mon.c (mon.2) 3045 : audit [DBG] from='client.? 172.21.15.6:0/4099904333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:18.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:18 smithi130 bash[51302]: debug 2023-08-28T02:16:18.440+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:18.446385+0000) 2023-08-28T02:16:19.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:19 smithi130 bash[51302]: debug 2023-08-28T02:16:19.444+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:19.446664+0000) 2023-08-28T02:16:20.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:20 smithi006 bash[33918]: cluster 2023-08-28T02:16:19.363675+0000 mgr.y (mgr.14901) 7731 : cluster [DBG] pgmap v7738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:20.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:20 smithi006 bash[27292]: cluster 2023-08-28T02:16:19.363675+0000 mgr.y (mgr.14901) 7731 : cluster [DBG] pgmap v7738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:20 smithi130 bash[27579]: cluster 2023-08-28T02:16:19.363675+0000 mgr.y (mgr.14901) 7731 : cluster [DBG] pgmap v7738: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:20.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:20 smithi130 bash[51302]: debug 2023-08-28T02:16:20.444+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:20.446984+0000) 2023-08-28T02:16:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:16:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:16:20] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-28T02:16:21.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:21 smithi006 bash[33918]: audit 2023-08-28T02:16:20.615636+0000 mon.a (mon.0) 4950 : audit [DBG] from='client.? 172.21.15.6:0/972435574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:16:21.255Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:16:21.257Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:21.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:21 smithi006 bash[27292]: audit 2023-08-28T02:16:20.615636+0000 mon.a (mon.0) 4950 : audit [DBG] from='client.? 172.21.15.6:0/972435574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:21.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:21 smithi130 bash[51302]: debug 2023-08-28T02:16:21.444+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:21.447200+0000) 2023-08-28T02:16:21.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:21 smithi130 bash[27579]: audit 2023-08-28T02:16:20.615636+0000 mon.a (mon.0) 4950 : audit [DBG] from='client.? 172.21.15.6:0/972435574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:22.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:22 smithi006 bash[33918]: cluster 2023-08-28T02:16:21.364355+0000 mgr.y (mgr.14901) 7732 : cluster [DBG] pgmap v7739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:22.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:22 smithi006 bash[27292]: cluster 2023-08-28T02:16:21.364355+0000 mgr.y (mgr.14901) 7732 : cluster [DBG] pgmap v7739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:22.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:22 smithi130 bash[51302]: debug 2023-08-28T02:16:22.444+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:22.447415+0000) 2023-08-28T02:16:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:22 smithi130 bash[27579]: cluster 2023-08-28T02:16:21.364355+0000 mgr.y (mgr.14901) 7732 : cluster [DBG] pgmap v7739: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:16:22.820Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:16:22.822Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:23.830 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:23 smithi130 bash[51302]: debug 2023-08-28T02:16:23.444+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:23.447707+0000) 2023-08-28T02:16:24.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:23 smithi130 bash[27579]: audit 2023-08-28T02:16:23.023925+0000 mon.a (mon.0) 4951 : audit [DBG] from='client.? 172.21.15.6:0/1896231494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:24.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:23 smithi130 bash[27579]: cluster 2023-08-28T02:16:23.365116+0000 mgr.y (mgr.14901) 7733 : cluster [DBG] pgmap v7740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:24.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:23 smithi006 bash[33918]: audit 2023-08-28T02:16:23.023925+0000 mon.a (mon.0) 4951 : audit [DBG] from='client.? 172.21.15.6:0/1896231494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:24.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:23 smithi006 bash[33918]: cluster 2023-08-28T02:16:23.365116+0000 mgr.y (mgr.14901) 7733 : cluster [DBG] pgmap v7740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:23 smithi006 bash[27292]: audit 2023-08-28T02:16:23.023925+0000 mon.a (mon.0) 4951 : audit [DBG] from='client.? 172.21.15.6:0/1896231494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:23 smithi006 bash[27292]: cluster 2023-08-28T02:16:23.365116+0000 mgr.y (mgr.14901) 7733 : cluster [DBG] pgmap v7740: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:24.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:24 smithi130 bash[51302]: debug 2023-08-28T02:16:24.444+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:24.447986+0000) 2023-08-28T02:16:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:16:24.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:16:24.700Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:25.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:25 smithi130 bash[51302]: debug 2023-08-28T02:16:25.444+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:25.448243+0000) 2023-08-28T02:16:26.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:26 smithi006 bash[33918]: cluster 2023-08-28T02:16:25.365974+0000 mgr.y (mgr.14901) 7734 : cluster [DBG] pgmap v7741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:26 smithi006 bash[33918]: audit 2023-08-28T02:16:25.446503+0000 mon.a (mon.0) 4952 : audit [DBG] from='client.? 172.21.15.6:0/1095116438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:26 smithi006 bash[27292]: cluster 2023-08-28T02:16:25.365974+0000 mgr.y (mgr.14901) 7734 : cluster [DBG] pgmap v7741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:26 smithi006 bash[27292]: audit 2023-08-28T02:16:25.446503+0000 mon.a (mon.0) 4952 : audit [DBG] from='client.? 172.21.15.6:0/1095116438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:26.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:26 smithi130 bash[27579]: cluster 2023-08-28T02:16:25.365974+0000 mgr.y (mgr.14901) 7734 : cluster [DBG] pgmap v7741: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:26.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:26 smithi130 bash[27579]: audit 2023-08-28T02:16:25.446503+0000 mon.a (mon.0) 4952 : audit [DBG] from='client.? 172.21.15.6:0/1095116438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:26.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:26 smithi130 bash[51302]: debug 2023-08-28T02:16:26.444+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:26.448555+0000) 2023-08-28T02:16:27.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:27 smithi130 bash[51302]: debug 2023-08-28T02:16:27.448+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:27.448816+0000) 2023-08-28T02:16:28.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:28 smithi006 bash[33918]: cluster 2023-08-28T02:16:27.366784+0000 mgr.y (mgr.14901) 7735 : cluster [DBG] pgmap v7742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:28 smithi006 bash[33918]: audit 2023-08-28T02:16:27.866933+0000 mon.c (mon.2) 3046 : audit [DBG] from='client.? 172.21.15.6:0/2882718946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:28 smithi006 bash[27292]: cluster 2023-08-28T02:16:27.366784+0000 mgr.y (mgr.14901) 7735 : cluster [DBG] pgmap v7742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:28 smithi006 bash[27292]: audit 2023-08-28T02:16:27.866933+0000 mon.c (mon.2) 3046 : audit [DBG] from='client.? 172.21.15.6:0/2882718946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:28.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:28 smithi130 bash[27579]: cluster 2023-08-28T02:16:27.366784+0000 mgr.y (mgr.14901) 7735 : cluster [DBG] pgmap v7742: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:28.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:28 smithi130 bash[27579]: audit 2023-08-28T02:16:27.866933+0000 mon.c (mon.2) 3046 : audit [DBG] from='client.? 172.21.15.6:0/2882718946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:28.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:28 smithi130 bash[51302]: debug 2023-08-28T02:16:28.444+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:28.449082+0000) 2023-08-28T02:16:29.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:29 smithi130 bash[51302]: debug 2023-08-28T02:16:29.444+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:29.449302+0000) 2023-08-28T02:16:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:30 smithi006 bash[33918]: cluster 2023-08-28T02:16:29.368041+0000 mgr.y (mgr.14901) 7736 : cluster [DBG] pgmap v7743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:30 smithi006 bash[33918]: audit 2023-08-28T02:16:30.274530+0000 mon.a (mon.0) 4953 : audit [DBG] from='client.? 172.21.15.6:0/2121003556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:30 smithi006 bash[27292]: cluster 2023-08-28T02:16:29.368041+0000 mgr.y (mgr.14901) 7736 : cluster [DBG] pgmap v7743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:30 smithi006 bash[27292]: audit 2023-08-28T02:16:30.274530+0000 mon.a (mon.0) 4953 : audit [DBG] from='client.? 172.21.15.6:0/2121003556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:30 smithi130 bash[27579]: cluster 2023-08-28T02:16:29.368041+0000 mgr.y (mgr.14901) 7736 : cluster [DBG] pgmap v7743: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:30 smithi130 bash[27579]: audit 2023-08-28T02:16:30.274530+0000 mon.a (mon.0) 4953 : audit [DBG] from='client.? 172.21.15.6:0/2121003556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:30.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:30 smithi130 bash[51302]: debug 2023-08-28T02:16:30.444+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:30.449564+0000) 2023-08-28T02:16:31.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:16:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:16:30] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:16:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:16:31.256Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:16:31.258Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:31.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:31 smithi130 bash[51302]: debug 2023-08-28T02:16:31.444+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:31.449841+0000) 2023-08-28T02:16:32.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:32 smithi006 bash[33918]: cluster 2023-08-28T02:16:31.368888+0000 mgr.y (mgr.14901) 7737 : cluster [DBG] pgmap v7744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:32.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:32 smithi006 bash[27292]: cluster 2023-08-28T02:16:31.368888+0000 mgr.y (mgr.14901) 7737 : cluster [DBG] pgmap v7744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:32 smithi130 bash[27579]: cluster 2023-08-28T02:16:31.368888+0000 mgr.y (mgr.14901) 7737 : cluster [DBG] pgmap v7744: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:32.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:32 smithi130 bash[51302]: debug 2023-08-28T02:16:32.444+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:32.450084+0000) 2023-08-28T02:16:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:16:32.820Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:16:32.822Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:33.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:33 smithi006 bash[33918]: audit 2023-08-28T02:16:32.692617+0000 mon.a (mon.0) 4954 : audit [DBG] from='client.? 172.21.15.6:0/1832445821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:33.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:33 smithi006 bash[27292]: audit 2023-08-28T02:16:32.692617+0000 mon.a (mon.0) 4954 : audit [DBG] from='client.? 172.21.15.6:0/1832445821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:33.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:33 smithi130 bash[27579]: audit 2023-08-28T02:16:32.692617+0000 mon.a (mon.0) 4954 : audit [DBG] from='client.? 172.21.15.6:0/1832445821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:33.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:33 smithi130 bash[51302]: debug 2023-08-28T02:16:33.444+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:33.450371+0000) 2023-08-28T02:16:34.698 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:34 smithi006 bash[27292]: cluster 2023-08-28T02:16:33.369660+0000 mgr.y (mgr.14901) 7738 : cluster [DBG] pgmap v7745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:34.698 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:34 smithi006 bash[27292]: audit 2023-08-28T02:16:33.448222+0000 mon.b (mon.1) 852 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:16:34.699 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:34 smithi006 bash[33918]: cluster 2023-08-28T02:16:33.369660+0000 mgr.y (mgr.14901) 7738 : cluster [DBG] pgmap v7745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:34.699 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:34 smithi006 bash[33918]: audit 2023-08-28T02:16:33.448222+0000 mon.b (mon.1) 852 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:16:34.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:34 smithi130 bash[27579]: cluster 2023-08-28T02:16:33.369660+0000 mgr.y (mgr.14901) 7738 : cluster [DBG] pgmap v7745: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:34.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:34 smithi130 bash[27579]: audit 2023-08-28T02:16:33.448222+0000 mon.b (mon.1) 852 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:16:34.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:34 smithi130 bash[51302]: debug 2023-08-28T02:16:34.444+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:34.450590+0000) 2023-08-28T02:16:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:16:34.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:16:34.700Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:35.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:35 smithi006 bash[27292]: audit 2023-08-28T02:16:35.111724+0000 mon.c (mon.2) 3047 : audit [DBG] from='client.? 172.21.15.6:0/3863650798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:35.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:35 smithi006 bash[33918]: audit 2023-08-28T02:16:35.111724+0000 mon.c (mon.2) 3047 : audit [DBG] from='client.? 172.21.15.6:0/3863650798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:35.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:35 smithi130 bash[27579]: audit 2023-08-28T02:16:35.111724+0000 mon.c (mon.2) 3047 : audit [DBG] from='client.? 172.21.15.6:0/3863650798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:35.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:35 smithi130 bash[51302]: debug 2023-08-28T02:16:35.448+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:35.450884+0000) 2023-08-28T02:16:36.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:36 smithi006 bash[33918]: cluster 2023-08-28T02:16:35.370853+0000 mgr.y (mgr.14901) 7739 : cluster [DBG] pgmap v7746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:36.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:36 smithi006 bash[27292]: cluster 2023-08-28T02:16:35.370853+0000 mgr.y (mgr.14901) 7739 : cluster [DBG] pgmap v7746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:36.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:36 smithi130 bash[27579]: cluster 2023-08-28T02:16:35.370853+0000 mgr.y (mgr.14901) 7739 : cluster [DBG] pgmap v7746: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:36.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:36 smithi130 bash[51302]: debug 2023-08-28T02:16:36.448+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:36.451168+0000) 2023-08-28T02:16:37.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:37 smithi130 bash[51302]: debug 2023-08-28T02:16:37.448+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:37.451459+0000) 2023-08-28T02:16:38.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:38 smithi130 bash[27579]: cluster 2023-08-28T02:16:37.371487+0000 mgr.y (mgr.14901) 7740 : cluster [DBG] pgmap v7747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:38.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:38 smithi130 bash[27579]: audit 2023-08-28T02:16:37.527064+0000 mon.a (mon.0) 4955 : audit [DBG] from='client.? 172.21.15.6:0/2067941657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:38.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:38 smithi130 bash[51302]: debug 2023-08-28T02:16:38.448+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:38.451725+0000) 2023-08-28T02:16:38.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:38 smithi006 bash[27292]: cluster 2023-08-28T02:16:37.371487+0000 mgr.y (mgr.14901) 7740 : cluster [DBG] pgmap v7747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:38.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:38 smithi006 bash[27292]: audit 2023-08-28T02:16:37.527064+0000 mon.a (mon.0) 4955 : audit [DBG] from='client.? 172.21.15.6:0/2067941657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:38.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:38 smithi006 bash[33918]: cluster 2023-08-28T02:16:37.371487+0000 mgr.y (mgr.14901) 7740 : cluster [DBG] pgmap v7747: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:38.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:38 smithi006 bash[33918]: audit 2023-08-28T02:16:37.527064+0000 mon.a (mon.0) 4955 : audit [DBG] from='client.? 172.21.15.6:0/2067941657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:39.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:39 smithi130 bash[51302]: debug 2023-08-28T02:16:39.448+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:39.452073+0000) 2023-08-28T02:16:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:40 smithi006 bash[33918]: cluster 2023-08-28T02:16:39.372594+0000 mgr.y (mgr.14901) 7741 : cluster [DBG] pgmap v7748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:40 smithi006 bash[33918]: audit 2023-08-28T02:16:39.402424+0000 mon.a (mon.0) 4956 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:16:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:40 smithi006 bash[33918]: audit 2023-08-28T02:16:39.775849+0000 mon.a (mon.0) 4957 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:16:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:40 smithi006 bash[33918]: audit 2023-08-28T02:16:39.934269+0000 mon.c (mon.2) 3048 : audit [DBG] from='client.? 172.21.15.6:0/1768942174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:40 smithi006 bash[33918]: audit 2023-08-28T02:16:40.078801+0000 mon.b (mon.1) 853 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:16:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:40 smithi006 bash[33918]: audit 2023-08-28T02:16:40.080657+0000 mon.b (mon.1) 854 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:16:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:40 smithi006 bash[33918]: audit 2023-08-28T02:16:40.087789+0000 mon.a (mon.0) 4958 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:16:40.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:40 smithi006 bash[27292]: cluster 2023-08-28T02:16:39.372594+0000 mgr.y (mgr.14901) 7741 : cluster [DBG] pgmap v7748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:40.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:40 smithi006 bash[27292]: audit 2023-08-28T02:16:39.402424+0000 mon.a (mon.0) 4956 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:16:40.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:40 smithi006 bash[27292]: audit 2023-08-28T02:16:39.775849+0000 mon.a (mon.0) 4957 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:16:40.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:40 smithi006 bash[27292]: audit 2023-08-28T02:16:39.934269+0000 mon.c (mon.2) 3048 : audit [DBG] from='client.? 172.21.15.6:0/1768942174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:40.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:40 smithi006 bash[27292]: audit 2023-08-28T02:16:40.078801+0000 mon.b (mon.1) 853 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:16:40.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:40 smithi006 bash[27292]: audit 2023-08-28T02:16:40.080657+0000 mon.b (mon.1) 854 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:16:40.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:40 smithi006 bash[27292]: audit 2023-08-28T02:16:40.087789+0000 mon.a (mon.0) 4958 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:16:40.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:40 smithi130 bash[51302]: debug 2023-08-28T02:16:40.448+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:40.452315+0000) 2023-08-28T02:16:40.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:40 smithi130 bash[27579]: cluster 2023-08-28T02:16:39.372594+0000 mgr.y (mgr.14901) 7741 : cluster [DBG] pgmap v7748: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:40.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:40 smithi130 bash[27579]: audit 2023-08-28T02:16:39.402424+0000 mon.a (mon.0) 4956 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:16:40.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:40 smithi130 bash[27579]: audit 2023-08-28T02:16:39.775849+0000 mon.a (mon.0) 4957 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:16:40.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:40 smithi130 bash[27579]: audit 2023-08-28T02:16:39.934269+0000 mon.c (mon.2) 3048 : audit [DBG] from='client.? 172.21.15.6:0/1768942174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:40.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:40 smithi130 bash[27579]: audit 2023-08-28T02:16:40.078801+0000 mon.b (mon.1) 853 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:16:40.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:40 smithi130 bash[27579]: audit 2023-08-28T02:16:40.080657+0000 mon.b (mon.1) 854 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:16:40.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:40 smithi130 bash[27579]: audit 2023-08-28T02:16:40.087789+0000 mon.a (mon.0) 4958 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:16:41.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:16:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:16:40] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:16:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:16:41.256Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:16:41.258Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:41.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:41 smithi130 bash[51302]: debug 2023-08-28T02:16:41.448+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:41.452617+0000) 2023-08-28T02:16:42.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:42 smithi006 bash[33918]: cluster 2023-08-28T02:16:41.373285+0000 mgr.y (mgr.14901) 7742 : cluster [DBG] pgmap v7749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:42.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:42 smithi006 bash[33918]: audit 2023-08-28T02:16:42.345037+0000 mon.c (mon.2) 3049 : audit [DBG] from='client.? 172.21.15.6:0/591206150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:42.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:42 smithi006 bash[27292]: cluster 2023-08-28T02:16:41.373285+0000 mgr.y (mgr.14901) 7742 : cluster [DBG] pgmap v7749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:42.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:42 smithi006 bash[27292]: audit 2023-08-28T02:16:42.345037+0000 mon.c (mon.2) 3049 : audit [DBG] from='client.? 172.21.15.6:0/591206150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:42.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:42 smithi130 bash[51302]: debug 2023-08-28T02:16:42.448+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:42.452876+0000) 2023-08-28T02:16:42.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:42 smithi130 bash[27579]: cluster 2023-08-28T02:16:41.373285+0000 mgr.y (mgr.14901) 7742 : cluster [DBG] pgmap v7749: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:42.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:42 smithi130 bash[27579]: audit 2023-08-28T02:16:42.345037+0000 mon.c (mon.2) 3049 : audit [DBG] from='client.? 172.21.15.6:0/591206150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:16:42.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:16:42.823Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:43.833 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:43 smithi130 bash[51302]: debug 2023-08-28T02:16:43.452+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:43.453201+0000) 2023-08-28T02:16:44.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:43 smithi130 bash[27579]: cluster 2023-08-28T02:16:43.373937+0000 mgr.y (mgr.14901) 7743 : cluster [DBG] pgmap v7750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:44.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:43 smithi006 bash[27292]: cluster 2023-08-28T02:16:43.373937+0000 mgr.y (mgr.14901) 7743 : cluster [DBG] pgmap v7750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:44.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:43 smithi006 bash[33918]: cluster 2023-08-28T02:16:43.373937+0000 mgr.y (mgr.14901) 7743 : cluster [DBG] pgmap v7750: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:44.835 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:44 smithi130 bash[51302]: debug 2023-08-28T02:16:44.448+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:44.453526+0000) 2023-08-28T02:16:44.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:44 smithi006 bash[33918]: audit 2023-08-28T02:16:44.761410+0000 mon.a (mon.0) 4959 : audit [DBG] from='client.? 172.21.15.6:0/2819987380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:44.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:44 smithi006 bash[27292]: audit 2023-08-28T02:16:44.761410+0000 mon.a (mon.0) 4959 : audit [DBG] from='client.? 172.21.15.6:0/2819987380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:16:44.699Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:16:44.701Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:45.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:44 smithi130 bash[27579]: audit 2023-08-28T02:16:44.761410+0000 mon.a (mon.0) 4959 : audit [DBG] from='client.? 172.21.15.6:0/2819987380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:45.843 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:45 smithi130 bash[51302]: debug 2023-08-28T02:16:45.452+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:45.453837+0000) 2023-08-28T02:16:46.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:45 smithi130 bash[27579]: cluster 2023-08-28T02:16:45.375062+0000 mgr.y (mgr.14901) 7744 : cluster [DBG] pgmap v7751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:46.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:45 smithi006 bash[27292]: cluster 2023-08-28T02:16:45.375062+0000 mgr.y (mgr.14901) 7744 : cluster [DBG] pgmap v7751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:46.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:45 smithi006 bash[33918]: cluster 2023-08-28T02:16:45.375062+0000 mgr.y (mgr.14901) 7744 : cluster [DBG] pgmap v7751: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:46.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:46 smithi130 bash[51302]: debug 2023-08-28T02:16:46.452+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:46.454131+0000) 2023-08-28T02:16:47.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:47 smithi130 bash[27579]: audit 2023-08-28T02:16:47.179842+0000 mon.c (mon.2) 3050 : audit [DBG] from='client.? 172.21.15.6:0/1507905733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:47.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:47 smithi130 bash[51302]: debug 2023-08-28T02:16:47.448+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:47.454406+0000) 2023-08-28T02:16:47.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:47 smithi006 bash[33918]: audit 2023-08-28T02:16:47.179842+0000 mon.c (mon.2) 3050 : audit [DBG] from='client.? 172.21.15.6:0/1507905733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:47.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:47 smithi006 bash[27292]: audit 2023-08-28T02:16:47.179842+0000 mon.c (mon.2) 3050 : audit [DBG] from='client.? 172.21.15.6:0/1507905733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:48 smithi130 bash[27579]: cluster 2023-08-28T02:16:47.375718+0000 mgr.y (mgr.14901) 7745 : cluster [DBG] pgmap v7752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:48.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:48 smithi130 bash[51302]: debug 2023-08-28T02:16:48.452+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:48.454710+0000) 2023-08-28T02:16:48.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:48 smithi006 bash[27292]: cluster 2023-08-28T02:16:47.375718+0000 mgr.y (mgr.14901) 7745 : cluster [DBG] pgmap v7752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:48.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:48 smithi006 bash[33918]: cluster 2023-08-28T02:16:47.375718+0000 mgr.y (mgr.14901) 7745 : cluster [DBG] pgmap v7752: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:49.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:49 smithi130 bash[51302]: debug 2023-08-28T02:16:49.452+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:49.454993+0000) 2023-08-28T02:16:50.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:50 smithi006 bash[33918]: cluster 2023-08-28T02:16:49.376789+0000 mgr.y (mgr.14901) 7746 : cluster [DBG] pgmap v7753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:50.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:50 smithi006 bash[33918]: audit 2023-08-28T02:16:49.592583+0000 mon.a (mon.0) 4960 : audit [DBG] from='client.? 172.21.15.6:0/745815737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:50 smithi006 bash[27292]: cluster 2023-08-28T02:16:49.376789+0000 mgr.y (mgr.14901) 7746 : cluster [DBG] pgmap v7753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:50 smithi006 bash[27292]: audit 2023-08-28T02:16:49.592583+0000 mon.a (mon.0) 4960 : audit [DBG] from='client.? 172.21.15.6:0/745815737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:50.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:50 smithi130 bash[27579]: cluster 2023-08-28T02:16:49.376789+0000 mgr.y (mgr.14901) 7746 : cluster [DBG] pgmap v7753: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:50.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:50 smithi130 bash[27579]: audit 2023-08-28T02:16:49.592583+0000 mon.a (mon.0) 4960 : audit [DBG] from='client.? 172.21.15.6:0/745815737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:50.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:50 smithi130 bash[51302]: debug 2023-08-28T02:16:50.452+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:50.455319+0000) 2023-08-28T02:16:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:16:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:16:50] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T02:16:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:16:51.257Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:16:51.259Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:51.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:51 smithi130 bash[51302]: debug 2023-08-28T02:16:51.452+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:51.455700+0000) 2023-08-28T02:16:52.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:52 smithi006 bash[33918]: cluster 2023-08-28T02:16:51.377440+0000 mgr.y (mgr.14901) 7747 : cluster [DBG] pgmap v7754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:52.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:52 smithi006 bash[33918]: audit 2023-08-28T02:16:52.011979+0000 mon.c (mon.2) 3051 : audit [DBG] from='client.? 172.21.15.6:0/879393493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:52.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:52 smithi006 bash[27292]: cluster 2023-08-28T02:16:51.377440+0000 mgr.y (mgr.14901) 7747 : cluster [DBG] pgmap v7754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:52.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:52 smithi006 bash[27292]: audit 2023-08-28T02:16:52.011979+0000 mon.c (mon.2) 3051 : audit [DBG] from='client.? 172.21.15.6:0/879393493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:52.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:52 smithi130 bash[27579]: cluster 2023-08-28T02:16:51.377440+0000 mgr.y (mgr.14901) 7747 : cluster [DBG] pgmap v7754: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:52.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:52 smithi130 bash[27579]: audit 2023-08-28T02:16:52.011979+0000 mon.c (mon.2) 3051 : audit [DBG] from='client.? 172.21.15.6:0/879393493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:52.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:52 smithi130 bash[51302]: debug 2023-08-28T02:16:52.452+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:52.455929+0000) 2023-08-28T02:16:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:16:52.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:16:52.824Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:53.836 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:53 smithi130 bash[51302]: debug 2023-08-28T02:16:53.452+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:53.456236+0000) 2023-08-28T02:16:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:53 smithi130 bash[27579]: cluster 2023-08-28T02:16:53.378233+0000 mgr.y (mgr.14901) 7748 : cluster [DBG] pgmap v7755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:54.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:53 smithi006 bash[33918]: cluster 2023-08-28T02:16:53.378233+0000 mgr.y (mgr.14901) 7748 : cluster [DBG] pgmap v7755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:54.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:53 smithi006 bash[27292]: cluster 2023-08-28T02:16:53.378233+0000 mgr.y (mgr.14901) 7748 : cluster [DBG] pgmap v7755: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:54.839 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:54 smithi130 bash[51302]: debug 2023-08-28T02:16:54.452+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:54.456486+0000) 2023-08-28T02:16:54.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:54 smithi006 bash[33918]: audit 2023-08-28T02:16:54.430711+0000 mon.c (mon.2) 3052 : audit [DBG] from='client.? 172.21.15.6:0/4008639040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:54.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:54 smithi006 bash[27292]: audit 2023-08-28T02:16:54.430711+0000 mon.c (mon.2) 3052 : audit [DBG] from='client.? 172.21.15.6:0/4008639040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:16:54.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:16:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:16:54.702Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:16:55.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:54 smithi130 bash[27579]: audit 2023-08-28T02:16:54.430711+0000 mon.c (mon.2) 3052 : audit [DBG] from='client.? 172.21.15.6:0/4008639040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:55.847 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:55 smithi130 bash[51302]: debug 2023-08-28T02:16:55.456+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:55.456766+0000) 2023-08-28T02:16:56.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:55 smithi130 bash[27579]: cluster 2023-08-28T02:16:55.379533+0000 mgr.y (mgr.14901) 7749 : cluster [DBG] pgmap v7756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:56.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:55 smithi006 bash[33918]: cluster 2023-08-28T02:16:55.379533+0000 mgr.y (mgr.14901) 7749 : cluster [DBG] pgmap v7756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:56.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:55 smithi006 bash[27292]: cluster 2023-08-28T02:16:55.379533+0000 mgr.y (mgr.14901) 7749 : cluster [DBG] pgmap v7756: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:56.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:56 smithi130 bash[51302]: debug 2023-08-28T02:16:56.456+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:56.457050+0000) 2023-08-28T02:16:57.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:56 smithi006 bash[33918]: audit 2023-08-28T02:16:56.852246+0000 mon.c (mon.2) 3053 : audit [DBG] from='client.? 172.21.15.6:0/3201850314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:57.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:56 smithi006 bash[27292]: audit 2023-08-28T02:16:56.852246+0000 mon.c (mon.2) 3053 : audit [DBG] from='client.? 172.21.15.6:0/3201850314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:57.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:56 smithi130 bash[27579]: audit 2023-08-28T02:16:56.852246+0000 mon.c (mon.2) 3053 : audit [DBG] from='client.? 172.21.15.6:0/3201850314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:57.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:57 smithi130 bash[51302]: debug 2023-08-28T02:16:57.452+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:57.457331+0000) 2023-08-28T02:16:58.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:57 smithi006 bash[27292]: cluster 2023-08-28T02:16:57.380215+0000 mgr.y (mgr.14901) 7750 : cluster [DBG] pgmap v7757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:58.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:57 smithi006 bash[33918]: cluster 2023-08-28T02:16:57.380215+0000 mgr.y (mgr.14901) 7750 : cluster [DBG] pgmap v7757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:58.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:57 smithi130 bash[27579]: cluster 2023-08-28T02:16:57.380215+0000 mgr.y (mgr.14901) 7750 : cluster [DBG] pgmap v7757: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:16:58.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:58 smithi130 bash[51302]: debug 2023-08-28T02:16:58.456+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:58.457598+0000) 2023-08-28T02:16:59.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:16:59 smithi130 bash[27579]: audit 2023-08-28T02:16:59.278368+0000 mon.c (mon.2) 3054 : audit [DBG] from='client.? 172.21.15.6:0/3042124131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:59.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:16:59 smithi130 bash[51302]: debug 2023-08-28T02:16:59.452+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:59.457902+0000) 2023-08-28T02:16:59.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:16:59 smithi006 bash[33918]: audit 2023-08-28T02:16:59.278368+0000 mon.c (mon.2) 3054 : audit [DBG] from='client.? 172.21.15.6:0/3042124131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:59.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:16:59 smithi006 bash[27292]: audit 2023-08-28T02:16:59.278368+0000 mon.c (mon.2) 3054 : audit [DBG] from='client.? 172.21.15.6:0/3042124131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:00 smithi130 bash[27579]: cluster 2023-08-28T02:16:59.381365+0000 mgr.y (mgr.14901) 7751 : cluster [DBG] pgmap v7758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:00.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:00 smithi130 bash[51302]: debug 2023-08-28T02:17:00.452+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:00.458204+0000) 2023-08-28T02:17:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:00 smithi006 bash[33918]: cluster 2023-08-28T02:16:59.381365+0000 mgr.y (mgr.14901) 7751 : cluster [DBG] pgmap v7758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:00 smithi006 bash[27292]: cluster 2023-08-28T02:16:59.381365+0000 mgr.y (mgr.14901) 7751 : cluster [DBG] pgmap v7758: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:17:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:17:00] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:17:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:17:01.258Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:17:01.260Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:01.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:01 smithi130 bash[51302]: debug 2023-08-28T02:17:01.452+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:01.458531+0000) 2023-08-28T02:17:02.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:02 smithi006 bash[33918]: cluster 2023-08-28T02:17:01.382001+0000 mgr.y (mgr.14901) 7752 : cluster [DBG] pgmap v7759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:02.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:02 smithi006 bash[33918]: audit 2023-08-28T02:17:01.698380+0000 mon.c (mon.2) 3055 : audit [DBG] from='client.? 172.21.15.6:0/3567721999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:02.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:02 smithi006 bash[27292]: cluster 2023-08-28T02:17:01.382001+0000 mgr.y (mgr.14901) 7752 : cluster [DBG] pgmap v7759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:02.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:02 smithi006 bash[27292]: audit 2023-08-28T02:17:01.698380+0000 mon.c (mon.2) 3055 : audit [DBG] from='client.? 172.21.15.6:0/3567721999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:02 smithi130 bash[27579]: cluster 2023-08-28T02:17:01.382001+0000 mgr.y (mgr.14901) 7752 : cluster [DBG] pgmap v7759: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:02 smithi130 bash[27579]: audit 2023-08-28T02:17:01.698380+0000 mon.c (mon.2) 3055 : audit [DBG] from='client.? 172.21.15.6:0/3567721999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:02.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:02 smithi130 bash[51302]: debug 2023-08-28T02:17:02.455+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:02.458820+0000) 2023-08-28T02:17:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:17:02.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:17:02.824Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:03.837 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:03 smithi130 bash[51302]: debug 2023-08-28T02:17:03.455+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:03.459149+0000) 2023-08-28T02:17:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:03 smithi130 bash[27579]: cluster 2023-08-28T02:17:03.382661+0000 mgr.y (mgr.14901) 7753 : cluster [DBG] pgmap v7760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:04.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:03 smithi006 bash[33918]: cluster 2023-08-28T02:17:03.382661+0000 mgr.y (mgr.14901) 7753 : cluster [DBG] pgmap v7760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:04.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:03 smithi006 bash[27292]: cluster 2023-08-28T02:17:03.382661+0000 mgr.y (mgr.14901) 7753 : cluster [DBG] pgmap v7760: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:04.840 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:04 smithi130 bash[51302]: debug 2023-08-28T02:17:04.455+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:04.459460+0000) 2023-08-28T02:17:04.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:04 smithi006 bash[33918]: audit 2023-08-28T02:17:04.116255+0000 mon.c (mon.2) 3056 : audit [DBG] from='client.? 172.21.15.6:0/993679494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:17:04.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:17:04.702Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:04.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:04 smithi006 bash[27292]: audit 2023-08-28T02:17:04.116255+0000 mon.c (mon.2) 3056 : audit [DBG] from='client.? 172.21.15.6:0/993679494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:05.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:04 smithi130 bash[27579]: audit 2023-08-28T02:17:04.116255+0000 mon.c (mon.2) 3056 : audit [DBG] from='client.? 172.21.15.6:0/993679494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:05.849 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:05 smithi130 bash[51302]: debug 2023-08-28T02:17:05.455+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:05.459768+0000) 2023-08-28T02:17:06.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:05 smithi130 bash[27579]: cluster 2023-08-28T02:17:05.383920+0000 mgr.y (mgr.14901) 7754 : cluster [DBG] pgmap v7761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:06.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:05 smithi006 bash[27292]: cluster 2023-08-28T02:17:05.383920+0000 mgr.y (mgr.14901) 7754 : cluster [DBG] pgmap v7761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:06.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:05 smithi006 bash[33918]: cluster 2023-08-28T02:17:05.383920+0000 mgr.y (mgr.14901) 7754 : cluster [DBG] pgmap v7761: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:06.851 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:06 smithi130 bash[51302]: debug 2023-08-28T02:17:06.455+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:06.460045+0000) 2023-08-28T02:17:07.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:06 smithi130 bash[27579]: audit 2023-08-28T02:17:06.527047+0000 mon.a (mon.0) 4961 : audit [DBG] from='client.? 172.21.15.6:0/3612661800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:07.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:06 smithi006 bash[27292]: audit 2023-08-28T02:17:06.527047+0000 mon.a (mon.0) 4961 : audit [DBG] from='client.? 172.21.15.6:0/3612661800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:07.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:06 smithi006 bash[33918]: audit 2023-08-28T02:17:06.527047+0000 mon.a (mon.0) 4961 : audit [DBG] from='client.? 172.21.15.6:0/3612661800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:07.859 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:07 smithi130 bash[51302]: debug 2023-08-28T02:17:07.459+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:07.460314+0000) 2023-08-28T02:17:08.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:07 smithi130 bash[27579]: cluster 2023-08-28T02:17:07.384625+0000 mgr.y (mgr.14901) 7755 : cluster [DBG] pgmap v7762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:08.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:07 smithi006 bash[27292]: cluster 2023-08-28T02:17:07.384625+0000 mgr.y (mgr.14901) 7755 : cluster [DBG] pgmap v7762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:08.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:07 smithi006 bash[33918]: cluster 2023-08-28T02:17:07.384625+0000 mgr.y (mgr.14901) 7755 : cluster [DBG] pgmap v7762: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:08.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:08 smithi130 bash[51302]: debug 2023-08-28T02:17:08.455+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:08.460585+0000) 2023-08-28T02:17:09.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:08 smithi130 bash[27579]: audit 2023-08-28T02:17:08.938261+0000 mon.c (mon.2) 3057 : audit [DBG] from='client.? 172.21.15.6:0/3097701602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:09.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:08 smithi006 bash[33918]: audit 2023-08-28T02:17:08.938261+0000 mon.c (mon.2) 3057 : audit [DBG] from='client.? 172.21.15.6:0/3097701602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:09.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:08 smithi006 bash[27292]: audit 2023-08-28T02:17:08.938261+0000 mon.c (mon.2) 3057 : audit [DBG] from='client.? 172.21.15.6:0/3097701602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:09.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:09 smithi130 bash[51302]: debug 2023-08-28T02:17:09.459+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:09.460871+0000) 2023-08-28T02:17:10.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:10 smithi130 bash[27579]: cluster 2023-08-28T02:17:09.385691+0000 mgr.y (mgr.14901) 7756 : cluster [DBG] pgmap v7763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:10.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:10 smithi006 bash[33918]: cluster 2023-08-28T02:17:09.385691+0000 mgr.y (mgr.14901) 7756 : cluster [DBG] pgmap v7763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:10.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:10 smithi006 bash[27292]: cluster 2023-08-28T02:17:09.385691+0000 mgr.y (mgr.14901) 7756 : cluster [DBG] pgmap v7763: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:10.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:10 smithi130 bash[51302]: debug 2023-08-28T02:17:10.455+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:10.461167+0000) 2023-08-28T02:17:11.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:17:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:17:10] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:17:11.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:11 smithi006 bash[27292]: audit 2023-08-28T02:17:11.359314+0000 mon.a (mon.0) 4962 : audit [DBG] from='client.? 172.21.15.6:0/624893770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:11.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:11 smithi006 bash[33918]: audit 2023-08-28T02:17:11.359314+0000 mon.a (mon.0) 4962 : audit [DBG] from='client.? 172.21.15.6:0/624893770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:17:11.259Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:17:11.261Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:11.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:11 smithi130 bash[27579]: audit 2023-08-28T02:17:11.359314+0000 mon.a (mon.0) 4962 : audit [DBG] from='client.? 172.21.15.6:0/624893770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:11.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:11 smithi130 bash[51302]: debug 2023-08-28T02:17:11.455+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:11.461464+0000) 2023-08-28T02:17:12.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:12 smithi006 bash[27292]: cluster 2023-08-28T02:17:11.386333+0000 mgr.y (mgr.14901) 7757 : cluster [DBG] pgmap v7764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:12.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:12 smithi006 bash[33918]: cluster 2023-08-28T02:17:11.386333+0000 mgr.y (mgr.14901) 7757 : cluster [DBG] pgmap v7764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:12 smithi130 bash[27579]: cluster 2023-08-28T02:17:11.386333+0000 mgr.y (mgr.14901) 7757 : cluster [DBG] pgmap v7764: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:12.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:12 smithi130 bash[51302]: debug 2023-08-28T02:17:12.455+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:12.461775+0000) 2023-08-28T02:17:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:17:12.823Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:17:12.826Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:13.842 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:13 smithi130 bash[51302]: debug 2023-08-28T02:17:13.455+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:13.462122+0000) 2023-08-28T02:17:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:13 smithi130 bash[27579]: cluster 2023-08-28T02:17:13.386899+0000 mgr.y (mgr.14901) 7758 : cluster [DBG] pgmap v7765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:13 smithi130 bash[27579]: audit 2023-08-28T02:17:13.772410+0000 mon.a (mon.0) 4963 : audit [DBG] from='client.? 172.21.15.6:0/3244982165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:14.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:13 smithi006 bash[33918]: cluster 2023-08-28T02:17:13.386899+0000 mgr.y (mgr.14901) 7758 : cluster [DBG] pgmap v7765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:13 smithi006 bash[33918]: audit 2023-08-28T02:17:13.772410+0000 mon.a (mon.0) 4963 : audit [DBG] from='client.? 172.21.15.6:0/3244982165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:13 smithi006 bash[27292]: cluster 2023-08-28T02:17:13.386899+0000 mgr.y (mgr.14901) 7758 : cluster [DBG] pgmap v7765: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:13 smithi006 bash[27292]: audit 2023-08-28T02:17:13.772410+0000 mon.a (mon.0) 4963 : audit [DBG] from='client.? 172.21.15.6:0/3244982165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:14.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:14 smithi130 bash[51302]: debug 2023-08-28T02:17:14.455+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:14.462445+0000) 2023-08-28T02:17:14.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:17:14.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:17:14.703Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:15.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:15 smithi130 bash[51302]: debug 2023-08-28T02:17:15.455+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:15.462735+0000) 2023-08-28T02:17:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:16 smithi006 bash[33918]: cluster 2023-08-28T02:17:15.388013+0000 mgr.y (mgr.14901) 7759 : cluster [DBG] pgmap v7766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:16 smithi006 bash[33918]: audit 2023-08-28T02:17:16.187107+0000 mon.c (mon.2) 3058 : audit [DBG] from='client.? 172.21.15.6:0/2398336806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:16 smithi006 bash[27292]: cluster 2023-08-28T02:17:15.388013+0000 mgr.y (mgr.14901) 7759 : cluster [DBG] pgmap v7766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:16 smithi006 bash[27292]: audit 2023-08-28T02:17:16.187107+0000 mon.c (mon.2) 3058 : audit [DBG] from='client.? 172.21.15.6:0/2398336806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:16.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:16 smithi130 bash[51302]: debug 2023-08-28T02:17:16.459+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:16.463006+0000) 2023-08-28T02:17:16.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:16 smithi130 bash[27579]: cluster 2023-08-28T02:17:15.388013+0000 mgr.y (mgr.14901) 7759 : cluster [DBG] pgmap v7766: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:16.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:16 smithi130 bash[27579]: audit 2023-08-28T02:17:16.187107+0000 mon.c (mon.2) 3058 : audit [DBG] from='client.? 172.21.15.6:0/2398336806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:17.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:17 smithi130 bash[51302]: debug 2023-08-28T02:17:17.459+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:17.463213+0000) 2023-08-28T02:17:18.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:18 smithi006 bash[27292]: cluster 2023-08-28T02:17:17.388664+0000 mgr.y (mgr.14901) 7760 : cluster [DBG] pgmap v7767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:18.728 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:18 smithi006 bash[33918]: cluster 2023-08-28T02:17:17.388664+0000 mgr.y (mgr.14901) 7760 : cluster [DBG] pgmap v7767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:18.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:18 smithi130 bash[27579]: cluster 2023-08-28T02:17:17.388664+0000 mgr.y (mgr.14901) 7760 : cluster [DBG] pgmap v7767: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:18.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:18 smithi130 bash[51302]: debug 2023-08-28T02:17:18.459+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:18.463405+0000) 2023-08-28T02:17:19.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:19 smithi006 bash[27292]: audit 2023-08-28T02:17:18.603851+0000 mon.a (mon.0) 4964 : audit [DBG] from='client.? 172.21.15.6:0/2420921076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:19.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:19 smithi006 bash[33918]: audit 2023-08-28T02:17:18.603851+0000 mon.a (mon.0) 4964 : audit [DBG] from='client.? 172.21.15.6:0/2420921076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:19.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:19 smithi130 bash[27579]: audit 2023-08-28T02:17:18.603851+0000 mon.a (mon.0) 4964 : audit [DBG] from='client.? 172.21.15.6:0/2420921076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:19.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:19 smithi130 bash[51302]: debug 2023-08-28T02:17:19.459+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:19.463662+0000) 2023-08-28T02:17:20.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:20 smithi006 bash[33918]: cluster 2023-08-28T02:17:19.389793+0000 mgr.y (mgr.14901) 7761 : cluster [DBG] pgmap v7768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:20.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:20 smithi006 bash[27292]: cluster 2023-08-28T02:17:19.389793+0000 mgr.y (mgr.14901) 7761 : cluster [DBG] pgmap v7768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:20 smithi130 bash[27579]: cluster 2023-08-28T02:17:19.389793+0000 mgr.y (mgr.14901) 7761 : cluster [DBG] pgmap v7768: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:20.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:20 smithi130 bash[51302]: debug 2023-08-28T02:17:20.459+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:20.463920+0000) 2023-08-28T02:17:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:17:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:17:20] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:17:21.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:21 smithi006 bash[33918]: audit 2023-08-28T02:17:21.031524+0000 mon.a (mon.0) 4965 : audit [DBG] from='client.? 172.21.15.6:0/3127344549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:21.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:21 smithi006 bash[27292]: audit 2023-08-28T02:17:21.031524+0000 mon.a (mon.0) 4965 : audit [DBG] from='client.? 172.21.15.6:0/3127344549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:17:21.260Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:17:21.263Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:21.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:21 smithi130 bash[27579]: audit 2023-08-28T02:17:21.031524+0000 mon.a (mon.0) 4965 : audit [DBG] from='client.? 172.21.15.6:0/3127344549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:21.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:21 smithi130 bash[51302]: debug 2023-08-28T02:17:21.459+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:21.464220+0000) 2023-08-28T02:17:22.825 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:22 smithi006 bash[27292]: cluster 2023-08-28T02:17:21.390447+0000 mgr.y (mgr.14901) 7762 : cluster [DBG] pgmap v7769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:22.825 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:22 smithi006 bash[33918]: cluster 2023-08-28T02:17:21.390447+0000 mgr.y (mgr.14901) 7762 : cluster [DBG] pgmap v7769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:22 smithi130 bash[27579]: cluster 2023-08-28T02:17:21.390447+0000 mgr.y (mgr.14901) 7762 : cluster [DBG] pgmap v7769: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:22.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:22 smithi130 bash[51302]: debug 2023-08-28T02:17:22.459+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:22.464444+0000) 2023-08-28T02:17:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:17:22.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:17:22.826Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:23.844 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:23 smithi130 bash[51302]: debug 2023-08-28T02:17:23.459+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:23.464713+0000) 2023-08-28T02:17:24.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:23 smithi130 bash[27579]: cluster 2023-08-28T02:17:23.391075+0000 mgr.y (mgr.14901) 7763 : cluster [DBG] pgmap v7770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:24.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:23 smithi130 bash[27579]: audit 2023-08-28T02:17:23.444385+0000 mon.c (mon.2) 3059 : audit [DBG] from='client.? 172.21.15.6:0/4004270217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:23 smithi006 bash[27292]: cluster 2023-08-28T02:17:23.391075+0000 mgr.y (mgr.14901) 7763 : cluster [DBG] pgmap v7770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:23 smithi006 bash[27292]: audit 2023-08-28T02:17:23.444385+0000 mon.c (mon.2) 3059 : audit [DBG] from='client.? 172.21.15.6:0/4004270217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:24.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:23 smithi006 bash[33918]: cluster 2023-08-28T02:17:23.391075+0000 mgr.y (mgr.14901) 7763 : cluster [DBG] pgmap v7770: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:24.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:23 smithi006 bash[33918]: audit 2023-08-28T02:17:23.444385+0000 mon.c (mon.2) 3059 : audit [DBG] from='client.? 172.21.15.6:0/4004270217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:24.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:24 smithi130 bash[51302]: debug 2023-08-28T02:17:24.459+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:24.464962+0000) 2023-08-28T02:17:24.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:17:24.702Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:17:24.703Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:25.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:25 smithi130 bash[51302]: debug 2023-08-28T02:17:25.459+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:25.465227+0000) 2023-08-28T02:17:26.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:26 smithi006 bash[33918]: cluster 2023-08-28T02:17:25.392318+0000 mgr.y (mgr.14901) 7764 : cluster [DBG] pgmap v7771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:26.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:26 smithi006 bash[33918]: audit 2023-08-28T02:17:25.863119+0000 mon.c (mon.2) 3060 : audit [DBG] from='client.? 172.21.15.6:0/3542633558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:26 smithi006 bash[27292]: cluster 2023-08-28T02:17:25.392318+0000 mgr.y (mgr.14901) 7764 : cluster [DBG] pgmap v7771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:26 smithi006 bash[27292]: audit 2023-08-28T02:17:25.863119+0000 mon.c (mon.2) 3060 : audit [DBG] from='client.? 172.21.15.6:0/3542633558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:26.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:26 smithi130 bash[27579]: cluster 2023-08-28T02:17:25.392318+0000 mgr.y (mgr.14901) 7764 : cluster [DBG] pgmap v7771: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:26.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:26 smithi130 bash[27579]: audit 2023-08-28T02:17:25.863119+0000 mon.c (mon.2) 3060 : audit [DBG] from='client.? 172.21.15.6:0/3542633558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:26.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:26 smithi130 bash[51302]: debug 2023-08-28T02:17:26.459+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:26.465492+0000) 2023-08-28T02:17:27.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:27 smithi130 bash[51302]: debug 2023-08-28T02:17:27.459+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:27.465758+0000) 2023-08-28T02:17:28.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:28 smithi006 bash[33918]: cluster 2023-08-28T02:17:27.393015+0000 mgr.y (mgr.14901) 7765 : cluster [DBG] pgmap v7772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:28 smithi006 bash[33918]: audit 2023-08-28T02:17:28.280792+0000 mon.c (mon.2) 3061 : audit [DBG] from='client.? 172.21.15.6:0/1875607159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:28 smithi006 bash[27292]: cluster 2023-08-28T02:17:27.393015+0000 mgr.y (mgr.14901) 7765 : cluster [DBG] pgmap v7772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:28 smithi006 bash[27292]: audit 2023-08-28T02:17:28.280792+0000 mon.c (mon.2) 3061 : audit [DBG] from='client.? 172.21.15.6:0/1875607159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:28.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:28 smithi130 bash[27579]: cluster 2023-08-28T02:17:27.393015+0000 mgr.y (mgr.14901) 7765 : cluster [DBG] pgmap v7772: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:28.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:28 smithi130 bash[27579]: audit 2023-08-28T02:17:28.280792+0000 mon.c (mon.2) 3061 : audit [DBG] from='client.? 172.21.15.6:0/1875607159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:28.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:28 smithi130 bash[51302]: debug 2023-08-28T02:17:28.459+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:28.466027+0000) 2023-08-28T02:17:29.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:29 smithi130 bash[51302]: debug 2023-08-28T02:17:29.459+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:29.466277+0000) 2023-08-28T02:17:30.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:30 smithi006 bash[27292]: cluster 2023-08-28T02:17:29.394024+0000 mgr.y (mgr.14901) 7766 : cluster [DBG] pgmap v7773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:30 smithi006 bash[33918]: cluster 2023-08-28T02:17:29.394024+0000 mgr.y (mgr.14901) 7766 : cluster [DBG] pgmap v7773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:30 smithi130 bash[27579]: cluster 2023-08-28T02:17:29.394024+0000 mgr.y (mgr.14901) 7766 : cluster [DBG] pgmap v7773: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:30.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:30 smithi130 bash[51302]: debug 2023-08-28T02:17:30.459+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:30.466528+0000) 2023-08-28T02:17:31.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:17:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:17:30] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-28T02:17:31.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:31 smithi006 bash[27292]: audit 2023-08-28T02:17:30.715153+0000 mon.c (mon.2) 3062 : audit [DBG] from='client.? 172.21.15.6:0/1471966247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:31.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:31 smithi006 bash[33918]: audit 2023-08-28T02:17:30.715153+0000 mon.c (mon.2) 3062 : audit [DBG] from='client.? 172.21.15.6:0/1471966247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:17:31.261Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:17:31.264Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:31.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:31 smithi130 bash[27579]: audit 2023-08-28T02:17:30.715153+0000 mon.c (mon.2) 3062 : audit [DBG] from='client.? 172.21.15.6:0/1471966247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:31.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:31 smithi130 bash[51302]: debug 2023-08-28T02:17:31.459+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:31.466828+0000) 2023-08-28T02:17:32.825 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:32 smithi006 bash[27292]: cluster 2023-08-28T02:17:31.394743+0000 mgr.y (mgr.14901) 7767 : cluster [DBG] pgmap v7774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:32.826 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:32 smithi006 bash[33918]: cluster 2023-08-28T02:17:31.394743+0000 mgr.y (mgr.14901) 7767 : cluster [DBG] pgmap v7774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:32 smithi130 bash[27579]: cluster 2023-08-28T02:17:31.394743+0000 mgr.y (mgr.14901) 7767 : cluster [DBG] pgmap v7774: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:32.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:32 smithi130 bash[51302]: debug 2023-08-28T02:17:32.463+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:32.467087+0000) 2023-08-28T02:17:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:17:32.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:17:32.828Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:33.846 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:33 smithi130 bash[51302]: debug 2023-08-28T02:17:33.463+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:33.467362+0000) 2023-08-28T02:17:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:33 smithi130 bash[27579]: audit 2023-08-28T02:17:33.134963+0000 mon.a (mon.0) 4966 : audit [DBG] from='client.? 172.21.15.6:0/4058616275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:33 smithi130 bash[27579]: cluster 2023-08-28T02:17:33.395381+0000 mgr.y (mgr.14901) 7768 : cluster [DBG] pgmap v7775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:34.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:33 smithi006 bash[33918]: audit 2023-08-28T02:17:33.134963+0000 mon.a (mon.0) 4966 : audit [DBG] from='client.? 172.21.15.6:0/4058616275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:33 smithi006 bash[33918]: cluster 2023-08-28T02:17:33.395381+0000 mgr.y (mgr.14901) 7768 : cluster [DBG] pgmap v7775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:33 smithi006 bash[27292]: audit 2023-08-28T02:17:33.134963+0000 mon.a (mon.0) 4966 : audit [DBG] from='client.? 172.21.15.6:0/4058616275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:33 smithi006 bash[27292]: cluster 2023-08-28T02:17:33.395381+0000 mgr.y (mgr.14901) 7768 : cluster [DBG] pgmap v7775: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:34.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:34 smithi130 bash[51302]: debug 2023-08-28T02:17:34.463+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:34.467640+0000) 2023-08-28T02:17:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:17:34.702Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:17:34.705Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:35.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:35 smithi130 bash[51302]: debug 2023-08-28T02:17:35.463+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:35.467925+0000) 2023-08-28T02:17:36.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:36 smithi006 bash[27292]: cluster 2023-08-28T02:17:35.396460+0000 mgr.y (mgr.14901) 7769 : cluster [DBG] pgmap v7776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:36.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:36 smithi006 bash[27292]: audit 2023-08-28T02:17:35.546798+0000 mon.c (mon.2) 3063 : audit [DBG] from='client.? 172.21.15.6:0/2180086929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:36.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:36 smithi006 bash[33918]: cluster 2023-08-28T02:17:35.396460+0000 mgr.y (mgr.14901) 7769 : cluster [DBG] pgmap v7776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:36.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:36 smithi006 bash[33918]: audit 2023-08-28T02:17:35.546798+0000 mon.c (mon.2) 3063 : audit [DBG] from='client.? 172.21.15.6:0/2180086929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:36.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:36 smithi130 bash[27579]: cluster 2023-08-28T02:17:35.396460+0000 mgr.y (mgr.14901) 7769 : cluster [DBG] pgmap v7776: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:36.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:36 smithi130 bash[27579]: audit 2023-08-28T02:17:35.546798+0000 mon.c (mon.2) 3063 : audit [DBG] from='client.? 172.21.15.6:0/2180086929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:36.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:36 smithi130 bash[51302]: debug 2023-08-28T02:17:36.463+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:36.468235+0000) 2023-08-28T02:17:37.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:37 smithi130 bash[51302]: debug 2023-08-28T02:17:37.463+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:37.468536+0000) 2023-08-28T02:17:38.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:38 smithi006 bash[27292]: cluster 2023-08-28T02:17:37.397151+0000 mgr.y (mgr.14901) 7770 : cluster [DBG] pgmap v7777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:38 smithi006 bash[27292]: audit 2023-08-28T02:17:37.959417+0000 mon.c (mon.2) 3064 : audit [DBG] from='client.? 172.21.15.6:0/1590511719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:38.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:38 smithi006 bash[33918]: cluster 2023-08-28T02:17:37.397151+0000 mgr.y (mgr.14901) 7770 : cluster [DBG] pgmap v7777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:38.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:38 smithi006 bash[33918]: audit 2023-08-28T02:17:37.959417+0000 mon.c (mon.2) 3064 : audit [DBG] from='client.? 172.21.15.6:0/1590511719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:38.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:38 smithi130 bash[27579]: cluster 2023-08-28T02:17:37.397151+0000 mgr.y (mgr.14901) 7770 : cluster [DBG] pgmap v7777: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:38.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:38 smithi130 bash[27579]: audit 2023-08-28T02:17:37.959417+0000 mon.c (mon.2) 3064 : audit [DBG] from='client.? 172.21.15.6:0/1590511719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:38.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:38 smithi130 bash[51302]: debug 2023-08-28T02:17:38.463+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:38.468784+0000) 2023-08-28T02:17:39.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:39 smithi130 bash[51302]: debug 2023-08-28T02:17:39.463+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:39.469059+0000) 2023-08-28T02:17:40.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:40 smithi006 bash[33918]: cluster 2023-08-28T02:17:39.398191+0000 mgr.y (mgr.14901) 7771 : cluster [DBG] pgmap v7778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:40 smithi006 bash[33918]: audit 2023-08-28T02:17:40.100839+0000 mon.b (mon.1) 855 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:40 smithi006 bash[33918]: audit 2023-08-28T02:17:40.384985+0000 mon.c (mon.2) 3065 : audit [DBG] from='client.? 172.21.15.6:0/3195003303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:40 smithi006 bash[27292]: cluster 2023-08-28T02:17:39.398191+0000 mgr.y (mgr.14901) 7771 : cluster [DBG] pgmap v7778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:40 smithi006 bash[27292]: audit 2023-08-28T02:17:40.100839+0000 mon.b (mon.1) 855 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:40 smithi006 bash[27292]: audit 2023-08-28T02:17:40.384985+0000 mon.c (mon.2) 3065 : audit [DBG] from='client.? 172.21.15.6:0/3195003303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:40.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:40 smithi130 bash[51302]: debug 2023-08-28T02:17:40.463+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:40.469297+0000) 2023-08-28T02:17:40.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:40 smithi130 bash[27579]: cluster 2023-08-28T02:17:39.398191+0000 mgr.y (mgr.14901) 7771 : cluster [DBG] pgmap v7778: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:40.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:40 smithi130 bash[27579]: audit 2023-08-28T02:17:40.100839+0000 mon.b (mon.1) 855 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:40.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:40 smithi130 bash[27579]: audit 2023-08-28T02:17:40.384985+0000 mon.c (mon.2) 3065 : audit [DBG] from='client.? 172.21.15.6:0/3195003303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:17:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:17:40] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-28T02:17:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:17:41.262Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:17:41.264Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:41.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:41 smithi130 bash[51302]: debug 2023-08-28T02:17:41.463+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:41.469585+0000) 2023-08-28T02:17:42.826 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:42 smithi006 bash[27292]: cluster 2023-08-28T02:17:41.398830+0000 mgr.y (mgr.14901) 7772 : cluster [DBG] pgmap v7779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:42.826 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:42 smithi006 bash[33918]: cluster 2023-08-28T02:17:41.398830+0000 mgr.y (mgr.14901) 7772 : cluster [DBG] pgmap v7779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:42.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:42 smithi130 bash[27579]: cluster 2023-08-28T02:17:41.398830+0000 mgr.y (mgr.14901) 7772 : cluster [DBG] pgmap v7779: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:42.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:42 smithi130 bash[51302]: debug 2023-08-28T02:17:42.463+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:42.469832+0000) 2023-08-28T02:17:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:17:42.826Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:17:42.827Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:43.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:43 smithi130 bash[27579]: audit 2023-08-28T02:17:42.804628+0000 mon.c (mon.2) 3066 : audit [DBG] from='client.? 172.21.15.6:0/1082139864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:43.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:43 smithi130 bash[51302]: debug 2023-08-28T02:17:43.463+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:43.470083+0000) 2023-08-28T02:17:43.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:43 smithi006 bash[27292]: audit 2023-08-28T02:17:42.804628+0000 mon.c (mon.2) 3066 : audit [DBG] from='client.? 172.21.15.6:0/1082139864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:43.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:43 smithi006 bash[33918]: audit 2023-08-28T02:17:42.804628+0000 mon.c (mon.2) 3066 : audit [DBG] from='client.? 172.21.15.6:0/1082139864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:44.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:44 smithi130 bash[27579]: cluster 2023-08-28T02:17:43.399394+0000 mgr.y (mgr.14901) 7773 : cluster [DBG] pgmap v7780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:44.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:44 smithi130 bash[51302]: debug 2023-08-28T02:17:44.463+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:44.470307+0000) 2023-08-28T02:17:44.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:44 smithi006 bash[27292]: cluster 2023-08-28T02:17:43.399394+0000 mgr.y (mgr.14901) 7773 : cluster [DBG] pgmap v7780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:44.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:44 smithi006 bash[33918]: cluster 2023-08-28T02:17:43.399394+0000 mgr.y (mgr.14901) 7773 : cluster [DBG] pgmap v7780: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:17:44.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:17:44.706Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:45.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:45 smithi130 bash[27579]: audit 2023-08-28T02:17:45.233435+0000 mon.c (mon.2) 3067 : audit [DBG] from='client.? 172.21.15.6:0/1471203239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:45.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:45 smithi130 bash[51302]: debug 2023-08-28T02:17:45.463+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:45.470520+0000) 2023-08-28T02:17:45.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:45 smithi006 bash[27292]: audit 2023-08-28T02:17:45.233435+0000 mon.c (mon.2) 3067 : audit [DBG] from='client.? 172.21.15.6:0/1471203239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:45.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:45 smithi006 bash[33918]: audit 2023-08-28T02:17:45.233435+0000 mon.c (mon.2) 3067 : audit [DBG] from='client.? 172.21.15.6:0/1471203239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:46.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:46 smithi130 bash[51302]: debug 2023-08-28T02:17:46.463+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:46.470786+0000) 2023-08-28T02:17:46.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:46 smithi130 bash[27579]: cluster 2023-08-28T02:17:45.400535+0000 mgr.y (mgr.14901) 7774 : cluster [DBG] pgmap v7781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:46.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:46 smithi130 bash[27579]: audit 2023-08-28T02:17:45.967680+0000 mon.a (mon.0) 4967 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:17:46.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:46 smithi006 bash[27292]: cluster 2023-08-28T02:17:45.400535+0000 mgr.y (mgr.14901) 7774 : cluster [DBG] pgmap v7781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:46.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:46 smithi006 bash[27292]: audit 2023-08-28T02:17:45.967680+0000 mon.a (mon.0) 4967 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:17:46.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:46 smithi006 bash[33918]: cluster 2023-08-28T02:17:45.400535+0000 mgr.y (mgr.14901) 7774 : cluster [DBG] pgmap v7781: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:46.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:46 smithi006 bash[33918]: audit 2023-08-28T02:17:45.967680+0000 mon.a (mon.0) 4967 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:17:47.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:47 smithi130 bash[51302]: debug 2023-08-28T02:17:47.467+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:47.471062+0000) 2023-08-28T02:17:47.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:47 smithi130 bash[27579]: audit 2023-08-28T02:17:46.510243+0000 mon.a (mon.0) 4968 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:17:47.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:47 smithi130 bash[27579]: audit 2023-08-28T02:17:46.779714+0000 mon.b (mon.1) 856 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:47.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:47 smithi130 bash[27579]: audit 2023-08-28T02:17:46.782154+0000 mon.b (mon.1) 857 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:17:47.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:47 smithi130 bash[27579]: audit 2023-08-28T02:17:46.792703+0000 mon.a (mon.0) 4969 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:17:47.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:47 smithi006 bash[27292]: audit 2023-08-28T02:17:46.510243+0000 mon.a (mon.0) 4968 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:17:47.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:47 smithi006 bash[27292]: audit 2023-08-28T02:17:46.779714+0000 mon.b (mon.1) 856 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:47.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:47 smithi006 bash[27292]: audit 2023-08-28T02:17:46.782154+0000 mon.b (mon.1) 857 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:17:47.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:47 smithi006 bash[27292]: audit 2023-08-28T02:17:46.792703+0000 mon.a (mon.0) 4969 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:17:47.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:47 smithi006 bash[33918]: audit 2023-08-28T02:17:46.510243+0000 mon.a (mon.0) 4968 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:17:47.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:47 smithi006 bash[33918]: audit 2023-08-28T02:17:46.779714+0000 mon.b (mon.1) 856 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:47.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:47 smithi006 bash[33918]: audit 2023-08-28T02:17:46.782154+0000 mon.b (mon.1) 857 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:17:47.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:47 smithi006 bash[33918]: audit 2023-08-28T02:17:46.792703+0000 mon.a (mon.0) 4969 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:17:48.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:48 smithi130 bash[27579]: cluster 2023-08-28T02:17:47.401226+0000 mgr.y (mgr.14901) 7775 : cluster [DBG] pgmap v7782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:48.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:48 smithi130 bash[27579]: audit 2023-08-28T02:17:47.652889+0000 mon.c (mon.2) 3068 : audit [DBG] from='client.? 172.21.15.6:0/3430781910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:48.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:48 smithi130 bash[51302]: debug 2023-08-28T02:17:48.467+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:48.471316+0000) 2023-08-28T02:17:48.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:48 smithi006 bash[27292]: cluster 2023-08-28T02:17:47.401226+0000 mgr.y (mgr.14901) 7775 : cluster [DBG] pgmap v7782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:48.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:48 smithi006 bash[27292]: audit 2023-08-28T02:17:47.652889+0000 mon.c (mon.2) 3068 : audit [DBG] from='client.? 172.21.15.6:0/3430781910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:48.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:48 smithi006 bash[33918]: cluster 2023-08-28T02:17:47.401226+0000 mgr.y (mgr.14901) 7775 : cluster [DBG] pgmap v7782: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:48.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:48 smithi006 bash[33918]: audit 2023-08-28T02:17:47.652889+0000 mon.c (mon.2) 3068 : audit [DBG] from='client.? 172.21.15.6:0/3430781910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:49.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:49 smithi130 bash[51302]: debug 2023-08-28T02:17:49.471+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:49.471590+0000) 2023-08-28T02:17:50.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:50 smithi130 bash[27579]: cluster 2023-08-28T02:17:49.402266+0000 mgr.y (mgr.14901) 7776 : cluster [DBG] pgmap v7783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:50.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:50 smithi130 bash[27579]: audit 2023-08-28T02:17:50.065665+0000 mon.c (mon.2) 3069 : audit [DBG] from='client.? 172.21.15.6:0/1745826598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:50.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:50 smithi130 bash[51302]: debug 2023-08-28T02:17:50.467+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:50.471940+0000) 2023-08-28T02:17:50.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:17:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:17:50] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:17:50.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:50 smithi006 bash[27292]: cluster 2023-08-28T02:17:49.402266+0000 mgr.y (mgr.14901) 7776 : cluster [DBG] pgmap v7783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:50.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:50 smithi006 bash[27292]: audit 2023-08-28T02:17:50.065665+0000 mon.c (mon.2) 3069 : audit [DBG] from='client.? 172.21.15.6:0/1745826598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:50.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:50 smithi006 bash[33918]: cluster 2023-08-28T02:17:49.402266+0000 mgr.y (mgr.14901) 7776 : cluster [DBG] pgmap v7783: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:50.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:50 smithi006 bash[33918]: audit 2023-08-28T02:17:50.065665+0000 mon.c (mon.2) 3069 : audit [DBG] from='client.? 172.21.15.6:0/1745826598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:17:51.262Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:17:51.264Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:51.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:51 smithi130 bash[51302]: debug 2023-08-28T02:17:51.471+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:51.472227+0000) 2023-08-28T02:17:52.827 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:52 smithi006 bash[27292]: cluster 2023-08-28T02:17:51.402937+0000 mgr.y (mgr.14901) 7777 : cluster [DBG] pgmap v7784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:52.828 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:52 smithi006 bash[27292]: audit 2023-08-28T02:17:52.469785+0000 mon.a (mon.0) 4970 : audit [DBG] from='client.? 172.21.15.6:0/2335472638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:52.828 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:52 smithi006 bash[33918]: cluster 2023-08-28T02:17:51.402937+0000 mgr.y (mgr.14901) 7777 : cluster [DBG] pgmap v7784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:52.828 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:52 smithi006 bash[33918]: audit 2023-08-28T02:17:52.469785+0000 mon.a (mon.0) 4970 : audit [DBG] from='client.? 172.21.15.6:0/2335472638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:52.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:52 smithi130 bash[51302]: debug 2023-08-28T02:17:52.467+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:52.472506+0000) 2023-08-28T02:17:52.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:52 smithi130 bash[27579]: cluster 2023-08-28T02:17:51.402937+0000 mgr.y (mgr.14901) 7777 : cluster [DBG] pgmap v7784: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:52.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:52 smithi130 bash[27579]: audit 2023-08-28T02:17:52.469785+0000 mon.a (mon.0) 4970 : audit [DBG] from='client.? 172.21.15.6:0/2335472638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:17:52.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:17:52.829Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:53.850 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:53 smithi130 bash[51302]: debug 2023-08-28T02:17:53.467+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:53.472784+0000) 2023-08-28T02:17:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:53 smithi130 bash[27579]: cluster 2023-08-28T02:17:53.403592+0000 mgr.y (mgr.14901) 7778 : cluster [DBG] pgmap v7785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:54.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:53 smithi006 bash[33918]: cluster 2023-08-28T02:17:53.403592+0000 mgr.y (mgr.14901) 7778 : cluster [DBG] pgmap v7785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:54.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:53 smithi006 bash[27292]: cluster 2023-08-28T02:17:53.403592+0000 mgr.y (mgr.14901) 7778 : cluster [DBG] pgmap v7785: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:54.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:54 smithi130 bash[51302]: debug 2023-08-28T02:17:54.471+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:54.473096+0000) 2023-08-28T02:17:54.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:54 smithi006 bash[27292]: audit 2023-08-28T02:17:54.885513+0000 mon.a (mon.0) 4971 : audit [DBG] from='client.? 172.21.15.6:0/162339242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:54.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:54 smithi006 bash[33918]: audit 2023-08-28T02:17:54.885513+0000 mon.a (mon.0) 4971 : audit [DBG] from='client.? 172.21.15.6:0/162339242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:17:54.705Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:17:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:17:54.706Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:17:55.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:54 smithi130 bash[27579]: audit 2023-08-28T02:17:54.885513+0000 mon.a (mon.0) 4971 : audit [DBG] from='client.? 172.21.15.6:0/162339242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:55.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:55 smithi130 bash[51302]: debug 2023-08-28T02:17:55.471+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:55.473376+0000) 2023-08-28T02:17:56.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:55 smithi006 bash[27292]: cluster 2023-08-28T02:17:55.404712+0000 mgr.y (mgr.14901) 7779 : cluster [DBG] pgmap v7786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:56.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:55 smithi006 bash[33918]: cluster 2023-08-28T02:17:55.404712+0000 mgr.y (mgr.14901) 7779 : cluster [DBG] pgmap v7786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:56.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:55 smithi130 bash[27579]: cluster 2023-08-28T02:17:55.404712+0000 mgr.y (mgr.14901) 7779 : cluster [DBG] pgmap v7786: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:56.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:56 smithi130 bash[51302]: debug 2023-08-28T02:17:56.471+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:56.473663+0000) 2023-08-28T02:17:57.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:57 smithi130 bash[27579]: audit 2023-08-28T02:17:57.303218+0000 mon.c (mon.2) 3070 : audit [DBG] from='client.? 172.21.15.6:0/2638928357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:57.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:57 smithi130 bash[51302]: debug 2023-08-28T02:17:57.467+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:57.473950+0000) 2023-08-28T02:17:57.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:57 smithi006 bash[33918]: audit 2023-08-28T02:17:57.303218+0000 mon.c (mon.2) 3070 : audit [DBG] from='client.? 172.21.15.6:0/2638928357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:57.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:57 smithi006 bash[27292]: audit 2023-08-28T02:17:57.303218+0000 mon.c (mon.2) 3070 : audit [DBG] from='client.? 172.21.15.6:0/2638928357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:58.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:17:58 smithi006 bash[27292]: cluster 2023-08-28T02:17:57.405453+0000 mgr.y (mgr.14901) 7780 : cluster [DBG] pgmap v7787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:58.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:17:58 smithi006 bash[33918]: cluster 2023-08-28T02:17:57.405453+0000 mgr.y (mgr.14901) 7780 : cluster [DBG] pgmap v7787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:58.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:58 smithi130 bash[51302]: debug 2023-08-28T02:17:58.466+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:58.474165+0000) 2023-08-28T02:17:58.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:17:58 smithi130 bash[27579]: cluster 2023-08-28T02:17:57.405453+0000 mgr.y (mgr.14901) 7780 : cluster [DBG] pgmap v7787: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:17:59.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:17:59 smithi130 bash[51302]: debug 2023-08-28T02:17:59.466+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:59.474463+0000) 2023-08-28T02:18:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:00 smithi006 bash[33918]: cluster 2023-08-28T02:17:59.406418+0000 mgr.y (mgr.14901) 7781 : cluster [DBG] pgmap v7788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:00 smithi006 bash[33918]: audit 2023-08-28T02:17:59.711608+0000 mon.a (mon.0) 4972 : audit [DBG] from='client.? 172.21.15.6:0/2926789116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:00 smithi006 bash[27292]: cluster 2023-08-28T02:17:59.406418+0000 mgr.y (mgr.14901) 7781 : cluster [DBG] pgmap v7788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:00 smithi006 bash[27292]: audit 2023-08-28T02:17:59.711608+0000 mon.a (mon.0) 4972 : audit [DBG] from='client.? 172.21.15.6:0/2926789116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:00 smithi130 bash[27579]: cluster 2023-08-28T02:17:59.406418+0000 mgr.y (mgr.14901) 7781 : cluster [DBG] pgmap v7788: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:00 smithi130 bash[27579]: audit 2023-08-28T02:17:59.711608+0000 mon.a (mon.0) 4972 : audit [DBG] from='client.? 172.21.15.6:0/2926789116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:00.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:00 smithi130 bash[51302]: debug 2023-08-28T02:18:00.466+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:00.474738+0000) 2023-08-28T02:18:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:18:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:18:00] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:18:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:18:01.263Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:18:01.265Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:01.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:01 smithi130 bash[51302]: debug 2023-08-28T02:18:01.470+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:01.475011+0000) 2023-08-28T02:18:02.828 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:02 smithi006 bash[33918]: cluster 2023-08-28T02:18:01.407084+0000 mgr.y (mgr.14901) 7782 : cluster [DBG] pgmap v7789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:02.828 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:02 smithi006 bash[33918]: audit 2023-08-28T02:18:02.128188+0000 mon.a (mon.0) 4973 : audit [DBG] from='client.? 172.21.15.6:0/3125539156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:02.828 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:02 smithi006 bash[27292]: cluster 2023-08-28T02:18:01.407084+0000 mgr.y (mgr.14901) 7782 : cluster [DBG] pgmap v7789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:02.828 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:02 smithi006 bash[27292]: audit 2023-08-28T02:18:02.128188+0000 mon.a (mon.0) 4973 : audit [DBG] from='client.? 172.21.15.6:0/3125539156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:02 smithi130 bash[27579]: cluster 2023-08-28T02:18:01.407084+0000 mgr.y (mgr.14901) 7782 : cluster [DBG] pgmap v7789: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:03.028 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:02 smithi130 bash[27579]: audit 2023-08-28T02:18:02.128188+0000 mon.a (mon.0) 4973 : audit [DBG] from='client.? 172.21.15.6:0/3125539156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:03.029 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:02 smithi130 bash[51302]: debug 2023-08-28T02:18:02.470+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:02.475247+0000) 2023-08-28T02:18:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:18:02.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:18:02.829Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:03.854 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:03 smithi130 bash[51302]: debug 2023-08-28T02:18:03.470+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:03.475445+0000) 2023-08-28T02:18:04.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:03 smithi130 bash[27579]: cluster 2023-08-28T02:18:03.407766+0000 mgr.y (mgr.14901) 7783 : cluster [DBG] pgmap v7790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:04.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:03 smithi006 bash[33918]: cluster 2023-08-28T02:18:03.407766+0000 mgr.y (mgr.14901) 7783 : cluster [DBG] pgmap v7790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:04.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:03 smithi006 bash[27292]: cluster 2023-08-28T02:18:03.407766+0000 mgr.y (mgr.14901) 7783 : cluster [DBG] pgmap v7790: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:04.857 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:04 smithi130 bash[51302]: debug 2023-08-28T02:18:04.474+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:04.475700+0000) 2023-08-28T02:18:04.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:04 smithi006 bash[27292]: audit 2023-08-28T02:18:04.543104+0000 mon.a (mon.0) 4974 : audit [DBG] from='client.? 172.21.15.6:0/188409674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:18:04.705Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:18:04.707Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:04.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:04 smithi006 bash[33918]: audit 2023-08-28T02:18:04.543104+0000 mon.a (mon.0) 4974 : audit [DBG] from='client.? 172.21.15.6:0/188409674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:05.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:04 smithi130 bash[27579]: audit 2023-08-28T02:18:04.543104+0000 mon.a (mon.0) 4974 : audit [DBG] from='client.? 172.21.15.6:0/188409674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:05.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:05 smithi130 bash[51302]: debug 2023-08-28T02:18:05.474+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:05.475997+0000) 2023-08-28T02:18:06.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:05 smithi006 bash[33918]: cluster 2023-08-28T02:18:05.408919+0000 mgr.y (mgr.14901) 7784 : cluster [DBG] pgmap v7791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:06.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:05 smithi006 bash[27292]: cluster 2023-08-28T02:18:05.408919+0000 mgr.y (mgr.14901) 7784 : cluster [DBG] pgmap v7791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:06.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:05 smithi130 bash[27579]: cluster 2023-08-28T02:18:05.408919+0000 mgr.y (mgr.14901) 7784 : cluster [DBG] pgmap v7791: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:06.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:06 smithi130 bash[51302]: debug 2023-08-28T02:18:06.474+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:06.476266+0000) 2023-08-28T02:18:07.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:07 smithi130 bash[27579]: audit 2023-08-28T02:18:06.955594+0000 mon.a (mon.0) 4975 : audit [DBG] from='client.? 172.21.15.6:0/264038452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:07.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:07 smithi006 bash[27292]: audit 2023-08-28T02:18:06.955594+0000 mon.a (mon.0) 4975 : audit [DBG] from='client.? 172.21.15.6:0/264038452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:07.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:07 smithi006 bash[33918]: audit 2023-08-28T02:18:06.955594+0000 mon.a (mon.0) 4975 : audit [DBG] from='client.? 172.21.15.6:0/264038452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:07.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:07 smithi130 bash[51302]: debug 2023-08-28T02:18:07.474+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:07.476589+0000) 2023-08-28T02:18:08.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:08 smithi130 bash[27579]: cluster 2023-08-28T02:18:07.409649+0000 mgr.y (mgr.14901) 7785 : cluster [DBG] pgmap v7792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:08.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:08 smithi006 bash[33918]: cluster 2023-08-28T02:18:07.409649+0000 mgr.y (mgr.14901) 7785 : cluster [DBG] pgmap v7792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:08.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:08 smithi006 bash[27292]: cluster 2023-08-28T02:18:07.409649+0000 mgr.y (mgr.14901) 7785 : cluster [DBG] pgmap v7792: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:08.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:08 smithi130 bash[51302]: debug 2023-08-28T02:18:08.474+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:08.476916+0000) 2023-08-28T02:18:09.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:09 smithi006 bash[33918]: audit 2023-08-28T02:18:09.376974+0000 mon.a (mon.0) 4976 : audit [DBG] from='client.? 172.21.15.6:0/2001162542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:09.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:09 smithi006 bash[27292]: audit 2023-08-28T02:18:09.376974+0000 mon.a (mon.0) 4976 : audit [DBG] from='client.? 172.21.15.6:0/2001162542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:09.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:09 smithi130 bash[27579]: audit 2023-08-28T02:18:09.376974+0000 mon.a (mon.0) 4976 : audit [DBG] from='client.? 172.21.15.6:0/2001162542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:09.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:09 smithi130 bash[51302]: debug 2023-08-28T02:18:09.474+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:09.477162+0000) 2023-08-28T02:18:10.752 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:10 smithi006 bash[33918]: cluster 2023-08-28T02:18:09.410802+0000 mgr.y (mgr.14901) 7786 : cluster [DBG] pgmap v7793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:10.752 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:10 smithi006 bash[27292]: cluster 2023-08-28T02:18:09.410802+0000 mgr.y (mgr.14901) 7786 : cluster [DBG] pgmap v7793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:10.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:10 smithi130 bash[27579]: cluster 2023-08-28T02:18:09.410802+0000 mgr.y (mgr.14901) 7786 : cluster [DBG] pgmap v7793: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:10.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:10 smithi130 bash[51302]: debug 2023-08-28T02:18:10.474+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:10.477423+0000) 2023-08-28T02:18:11.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:18:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:18:10] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:18:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:18:11.264Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:18:11.266Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:11.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:11 smithi130 bash[51302]: debug 2023-08-28T02:18:11.474+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:11.477707+0000) 2023-08-28T02:18:12.829 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:12 smithi006 bash[33918]: cluster 2023-08-28T02:18:11.411461+0000 mgr.y (mgr.14901) 7787 : cluster [DBG] pgmap v7794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:12.829 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:12 smithi006 bash[33918]: audit 2023-08-28T02:18:11.814408+0000 mon.a (mon.0) 4977 : audit [DBG] from='client.? 172.21.15.6:0/465919657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:12.830 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:12 smithi006 bash[27292]: cluster 2023-08-28T02:18:11.411461+0000 mgr.y (mgr.14901) 7787 : cluster [DBG] pgmap v7794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:12.830 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:12 smithi006 bash[27292]: audit 2023-08-28T02:18:11.814408+0000 mon.a (mon.0) 4977 : audit [DBG] from='client.? 172.21.15.6:0/465919657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:12 smithi130 bash[27579]: cluster 2023-08-28T02:18:11.411461+0000 mgr.y (mgr.14901) 7787 : cluster [DBG] pgmap v7794: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:12 smithi130 bash[27579]: audit 2023-08-28T02:18:11.814408+0000 mon.a (mon.0) 4977 : audit [DBG] from='client.? 172.21.15.6:0/465919657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:12.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:12 smithi130 bash[51302]: debug 2023-08-28T02:18:12.474+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:12.477959+0000) 2023-08-28T02:18:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:18:12.829Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:18:12.831Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:13.856 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:13 smithi130 bash[51302]: debug 2023-08-28T02:18:13.474+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:13.478249+0000) 2023-08-28T02:18:14.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:13 smithi130 bash[27579]: cluster 2023-08-28T02:18:13.412117+0000 mgr.y (mgr.14901) 7788 : cluster [DBG] pgmap v7795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:14.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:13 smithi006 bash[33918]: cluster 2023-08-28T02:18:13.412117+0000 mgr.y (mgr.14901) 7788 : cluster [DBG] pgmap v7795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:13 smithi006 bash[27292]: cluster 2023-08-28T02:18:13.412117+0000 mgr.y (mgr.14901) 7788 : cluster [DBG] pgmap v7795: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:14.857 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:14 smithi130 bash[51302]: debug 2023-08-28T02:18:14.474+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:14.478530+0000) 2023-08-28T02:18:14.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:14 smithi006 bash[33918]: audit 2023-08-28T02:18:14.240041+0000 mon.c (mon.2) 3071 : audit [DBG] from='client.? 172.21.15.6:0/2962776608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:14.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:14 smithi006 bash[27292]: audit 2023-08-28T02:18:14.240041+0000 mon.c (mon.2) 3071 : audit [DBG] from='client.? 172.21.15.6:0/2962776608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:18:14.705Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:18:14.707Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:15.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:14 smithi130 bash[27579]: audit 2023-08-28T02:18:14.240041+0000 mon.c (mon.2) 3071 : audit [DBG] from='client.? 172.21.15.6:0/2962776608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:15.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:15 smithi130 bash[51302]: debug 2023-08-28T02:18:15.478+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:15.478843+0000) 2023-08-28T02:18:16.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:15 smithi006 bash[27292]: cluster 2023-08-28T02:18:15.413273+0000 mgr.y (mgr.14901) 7789 : cluster [DBG] pgmap v7796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:16.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:15 smithi006 bash[33918]: cluster 2023-08-28T02:18:15.413273+0000 mgr.y (mgr.14901) 7789 : cluster [DBG] pgmap v7796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:16.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:15 smithi130 bash[27579]: cluster 2023-08-28T02:18:15.413273+0000 mgr.y (mgr.14901) 7789 : cluster [DBG] pgmap v7796: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:16.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:16 smithi130 bash[51302]: debug 2023-08-28T02:18:16.478+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:16.479193+0000) 2023-08-28T02:18:17.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:16 smithi006 bash[33918]: audit 2023-08-28T02:18:16.662290+0000 mon.a (mon.0) 4978 : audit [DBG] from='client.? 172.21.15.6:0/529053645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:17.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:16 smithi006 bash[27292]: audit 2023-08-28T02:18:16.662290+0000 mon.a (mon.0) 4978 : audit [DBG] from='client.? 172.21.15.6:0/529053645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:17.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:16 smithi130 bash[27579]: audit 2023-08-28T02:18:16.662290+0000 mon.a (mon.0) 4978 : audit [DBG] from='client.? 172.21.15.6:0/529053645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:17.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:17 smithi130 bash[51302]: debug 2023-08-28T02:18:17.478+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:17.479461+0000) 2023-08-28T02:18:18.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:17 smithi006 bash[27292]: cluster 2023-08-28T02:18:17.413868+0000 mgr.y (mgr.14901) 7790 : cluster [DBG] pgmap v7797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:18.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:17 smithi006 bash[33918]: cluster 2023-08-28T02:18:17.413868+0000 mgr.y (mgr.14901) 7790 : cluster [DBG] pgmap v7797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:18.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:17 smithi130 bash[27579]: cluster 2023-08-28T02:18:17.413868+0000 mgr.y (mgr.14901) 7790 : cluster [DBG] pgmap v7797: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:18.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:18 smithi130 bash[51302]: debug 2023-08-28T02:18:18.478+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:18.479776+0000) 2023-08-28T02:18:19.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:19 smithi006 bash[33918]: audit 2023-08-28T02:18:19.079446+0000 mon.a (mon.0) 4979 : audit [DBG] from='client.? 172.21.15.6:0/2776011415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:19.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:19 smithi006 bash[27292]: audit 2023-08-28T02:18:19.079446+0000 mon.a (mon.0) 4979 : audit [DBG] from='client.? 172.21.15.6:0/2776011415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:19.478 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:19 smithi130 bash[27579]: audit 2023-08-28T02:18:19.079446+0000 mon.a (mon.0) 4979 : audit [DBG] from='client.? 172.21.15.6:0/2776011415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:19.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:19 smithi130 bash[51302]: debug 2023-08-28T02:18:19.478+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:19.480066+0000) 2023-08-28T02:18:20.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:20 smithi006 bash[33918]: cluster 2023-08-28T02:18:19.415157+0000 mgr.y (mgr.14901) 7791 : cluster [DBG] pgmap v7798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:20.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:20 smithi006 bash[27292]: cluster 2023-08-28T02:18:19.415157+0000 mgr.y (mgr.14901) 7791 : cluster [DBG] pgmap v7798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:20.479 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:20 smithi130 bash[27579]: cluster 2023-08-28T02:18:19.415157+0000 mgr.y (mgr.14901) 7791 : cluster [DBG] pgmap v7798: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:20.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:20 smithi130 bash[51302]: debug 2023-08-28T02:18:20.478+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:20.480404+0000) 2023-08-28T02:18:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:18:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:18:20] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:18:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:18:21.265Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:18:21.267Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:21.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:21 smithi130 bash[51302]: debug 2023-08-28T02:18:21.478+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:21.480627+0000) 2023-08-28T02:18:22.829 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:22 smithi006 bash[27292]: cluster 2023-08-28T02:18:21.415831+0000 mgr.y (mgr.14901) 7792 : cluster [DBG] pgmap v7799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:22.830 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:22 smithi006 bash[27292]: audit 2023-08-28T02:18:21.503651+0000 mon.c (mon.2) 3072 : audit [DBG] from='client.? 172.21.15.6:0/2803994519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:22.830 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:22 smithi006 bash[33918]: cluster 2023-08-28T02:18:21.415831+0000 mgr.y (mgr.14901) 7792 : cluster [DBG] pgmap v7799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:22.830 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:22 smithi006 bash[33918]: audit 2023-08-28T02:18:21.503651+0000 mon.c (mon.2) 3072 : audit [DBG] from='client.? 172.21.15.6:0/2803994519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:22 smithi130 bash[27579]: cluster 2023-08-28T02:18:21.415831+0000 mgr.y (mgr.14901) 7792 : cluster [DBG] pgmap v7799: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:22 smithi130 bash[27579]: audit 2023-08-28T02:18:21.503651+0000 mon.c (mon.2) 3072 : audit [DBG] from='client.? 172.21.15.6:0/2803994519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:22.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:22 smithi130 bash[51302]: debug 2023-08-28T02:18:22.478+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:22.480856+0000) 2023-08-28T02:18:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:18:22.829Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:18:22.831Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:23.856 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:23 smithi130 bash[51302]: debug 2023-08-28T02:18:23.478+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:23.481115+0000) 2023-08-28T02:18:24.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:23 smithi130 bash[27579]: cluster 2023-08-28T02:18:23.416510+0000 mgr.y (mgr.14901) 7793 : cluster [DBG] pgmap v7800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:24.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:23 smithi006 bash[33918]: cluster 2023-08-28T02:18:23.416510+0000 mgr.y (mgr.14901) 7793 : cluster [DBG] pgmap v7800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:23 smithi006 bash[27292]: cluster 2023-08-28T02:18:23.416510+0000 mgr.y (mgr.14901) 7793 : cluster [DBG] pgmap v7800: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:24.859 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:24 smithi130 bash[51302]: debug 2023-08-28T02:18:24.478+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:24.481394+0000) 2023-08-28T02:18:24.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:24 smithi006 bash[27292]: audit 2023-08-28T02:18:23.919035+0000 mon.a (mon.0) 4980 : audit [DBG] from='client.? 172.21.15.6:0/2425437840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:24.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:24 smithi006 bash[33918]: audit 2023-08-28T02:18:23.919035+0000 mon.a (mon.0) 4980 : audit [DBG] from='client.? 172.21.15.6:0/2425437840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:18:24.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:18:24.708Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:25.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:24 smithi130 bash[27579]: audit 2023-08-28T02:18:23.919035+0000 mon.a (mon.0) 4980 : audit [DBG] from='client.? 172.21.15.6:0/2425437840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:25.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:25 smithi130 bash[51302]: debug 2023-08-28T02:18:25.478+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:25.481675+0000) 2023-08-28T02:18:26.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:25 smithi006 bash[33918]: cluster 2023-08-28T02:18:25.417667+0000 mgr.y (mgr.14901) 7794 : cluster [DBG] pgmap v7801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:26.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:25 smithi006 bash[27292]: cluster 2023-08-28T02:18:25.417667+0000 mgr.y (mgr.14901) 7794 : cluster [DBG] pgmap v7801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:26.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:25 smithi130 bash[27579]: cluster 2023-08-28T02:18:25.417667+0000 mgr.y (mgr.14901) 7794 : cluster [DBG] pgmap v7801: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:26.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:26 smithi130 bash[51302]: debug 2023-08-28T02:18:26.478+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:26.481991+0000) 2023-08-28T02:18:27.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:26 smithi006 bash[27292]: audit 2023-08-28T02:18:26.327893+0000 mon.a (mon.0) 4981 : audit [DBG] from='client.? 172.21.15.6:0/2957751676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:27.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:26 smithi006 bash[33918]: audit 2023-08-28T02:18:26.327893+0000 mon.a (mon.0) 4981 : audit [DBG] from='client.? 172.21.15.6:0/2957751676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:27.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:26 smithi130 bash[27579]: audit 2023-08-28T02:18:26.327893+0000 mon.a (mon.0) 4981 : audit [DBG] from='client.? 172.21.15.6:0/2957751676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:27.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:27 smithi130 bash[51302]: debug 2023-08-28T02:18:27.478+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:27.482285+0000) 2023-08-28T02:18:28.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:27 smithi006 bash[27292]: cluster 2023-08-28T02:18:27.418332+0000 mgr.y (mgr.14901) 7795 : cluster [DBG] pgmap v7802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:28.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:27 smithi006 bash[33918]: cluster 2023-08-28T02:18:27.418332+0000 mgr.y (mgr.14901) 7795 : cluster [DBG] pgmap v7802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:28.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:27 smithi130 bash[27579]: cluster 2023-08-28T02:18:27.418332+0000 mgr.y (mgr.14901) 7795 : cluster [DBG] pgmap v7802: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:28.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:28 smithi130 bash[51302]: debug 2023-08-28T02:18:28.478+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:28.482555+0000) 2023-08-28T02:18:29.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:28 smithi006 bash[27292]: audit 2023-08-28T02:18:28.753164+0000 mon.a (mon.0) 4982 : audit [DBG] from='client.? 172.21.15.6:0/2472024684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:29.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:28 smithi006 bash[33918]: audit 2023-08-28T02:18:28.753164+0000 mon.a (mon.0) 4982 : audit [DBG] from='client.? 172.21.15.6:0/2472024684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:29.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:28 smithi130 bash[27579]: audit 2023-08-28T02:18:28.753164+0000 mon.a (mon.0) 4982 : audit [DBG] from='client.? 172.21.15.6:0/2472024684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:29.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:29 smithi130 bash[51302]: debug 2023-08-28T02:18:29.482+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:29.482882+0000) 2023-08-28T02:18:30.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:29 smithi006 bash[33918]: cluster 2023-08-28T02:18:29.419438+0000 mgr.y (mgr.14901) 7796 : cluster [DBG] pgmap v7803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:30.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:29 smithi006 bash[27292]: cluster 2023-08-28T02:18:29.419438+0000 mgr.y (mgr.14901) 7796 : cluster [DBG] pgmap v7803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:30.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:29 smithi130 bash[27579]: cluster 2023-08-28T02:18:29.419438+0000 mgr.y (mgr.14901) 7796 : cluster [DBG] pgmap v7803: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:30.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:30 smithi130 bash[51302]: debug 2023-08-28T02:18:30.482+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:30.483160+0000) 2023-08-28T02:18:31.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:18:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:18:30] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:18:31.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:31 smithi130 bash[27579]: audit 2023-08-28T02:18:31.181880+0000 mon.a (mon.0) 4983 : audit [DBG] from='client.? 172.21.15.6:0/800044129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:31.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:31 smithi130 bash[51302]: debug 2023-08-28T02:18:31.482+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:31.483462+0000) 2023-08-28T02:18:31.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:31 smithi006 bash[27292]: audit 2023-08-28T02:18:31.181880+0000 mon.a (mon.0) 4983 : audit [DBG] from='client.? 172.21.15.6:0/800044129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:31.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:31 smithi006 bash[33918]: audit 2023-08-28T02:18:31.181880+0000 mon.a (mon.0) 4983 : audit [DBG] from='client.? 172.21.15.6:0/800044129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:18:31.266Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:18:31.267Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:32.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:32 smithi130 bash[51302]: debug 2023-08-28T02:18:32.482+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:32.483743+0000) 2023-08-28T02:18:32.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:32 smithi130 bash[27579]: cluster 2023-08-28T02:18:31.420129+0000 mgr.y (mgr.14901) 7797 : cluster [DBG] pgmap v7804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:32.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:32 smithi006 bash[33918]: cluster 2023-08-28T02:18:31.420129+0000 mgr.y (mgr.14901) 7797 : cluster [DBG] pgmap v7804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:32.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:32 smithi006 bash[27292]: cluster 2023-08-28T02:18:31.420129+0000 mgr.y (mgr.14901) 7797 : cluster [DBG] pgmap v7804: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:18:32.830Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:18:32.832Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:33.856 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:33 smithi130 bash[51302]: debug 2023-08-28T02:18:33.482+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:33.484027+0000) 2023-08-28T02:18:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:33 smithi130 bash[27579]: cluster 2023-08-28T02:18:33.420749+0000 mgr.y (mgr.14901) 7798 : cluster [DBG] pgmap v7805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:33 smithi130 bash[27579]: audit 2023-08-28T02:18:33.595013+0000 mon.a (mon.0) 4984 : audit [DBG] from='client.? 172.21.15.6:0/3425807649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:34.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:33 smithi006 bash[33918]: cluster 2023-08-28T02:18:33.420749+0000 mgr.y (mgr.14901) 7798 : cluster [DBG] pgmap v7805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:33 smithi006 bash[33918]: audit 2023-08-28T02:18:33.595013+0000 mon.a (mon.0) 4984 : audit [DBG] from='client.? 172.21.15.6:0/3425807649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:33 smithi006 bash[27292]: cluster 2023-08-28T02:18:33.420749+0000 mgr.y (mgr.14901) 7798 : cluster [DBG] pgmap v7805: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:33 smithi006 bash[27292]: audit 2023-08-28T02:18:33.595013+0000 mon.a (mon.0) 4984 : audit [DBG] from='client.? 172.21.15.6:0/3425807649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:34.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:34 smithi130 bash[51302]: debug 2023-08-28T02:18:34.482+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:34.484378+0000) 2023-08-28T02:18:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:18:34.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:34.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:18:34.708Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:35.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:35 smithi130 bash[51302]: debug 2023-08-28T02:18:35.482+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:35.484662+0000) 2023-08-28T02:18:36.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:36 smithi130 bash[27579]: cluster 2023-08-28T02:18:35.421896+0000 mgr.y (mgr.14901) 7799 : cluster [DBG] pgmap v7806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:36.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:36 smithi130 bash[27579]: audit 2023-08-28T02:18:36.008150+0000 mon.c (mon.2) 3073 : audit [DBG] from='client.? 172.21.15.6:0/1961267026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:36.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:36 smithi130 bash[51302]: debug 2023-08-28T02:18:36.482+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:36.484907+0000) 2023-08-28T02:18:36.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:36 smithi006 bash[33918]: cluster 2023-08-28T02:18:35.421896+0000 mgr.y (mgr.14901) 7799 : cluster [DBG] pgmap v7806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:36.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:36 smithi006 bash[33918]: audit 2023-08-28T02:18:36.008150+0000 mon.c (mon.2) 3073 : audit [DBG] from='client.? 172.21.15.6:0/1961267026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:36.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:36 smithi006 bash[27292]: cluster 2023-08-28T02:18:35.421896+0000 mgr.y (mgr.14901) 7799 : cluster [DBG] pgmap v7806: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:36.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:36 smithi006 bash[27292]: audit 2023-08-28T02:18:36.008150+0000 mon.c (mon.2) 3073 : audit [DBG] from='client.? 172.21.15.6:0/1961267026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:37.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:37 smithi130 bash[51302]: debug 2023-08-28T02:18:37.482+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:37.485192+0000) 2023-08-28T02:18:38.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:38 smithi130 bash[27579]: cluster 2023-08-28T02:18:37.422539+0000 mgr.y (mgr.14901) 7800 : cluster [DBG] pgmap v7807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:38.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:38 smithi130 bash[27579]: audit 2023-08-28T02:18:38.430899+0000 mon.c (mon.2) 3074 : audit [DBG] from='client.? 172.21.15.6:0/241368206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:38.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:38 smithi130 bash[51302]: debug 2023-08-28T02:18:38.482+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:38.485490+0000) 2023-08-28T02:18:38.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:38 smithi006 bash[33918]: cluster 2023-08-28T02:18:37.422539+0000 mgr.y (mgr.14901) 7800 : cluster [DBG] pgmap v7807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:38.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:38 smithi006 bash[33918]: audit 2023-08-28T02:18:38.430899+0000 mon.c (mon.2) 3074 : audit [DBG] from='client.? 172.21.15.6:0/241368206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:38 smithi006 bash[27292]: cluster 2023-08-28T02:18:37.422539+0000 mgr.y (mgr.14901) 7800 : cluster [DBG] pgmap v7807: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:38.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:38 smithi006 bash[27292]: audit 2023-08-28T02:18:38.430899+0000 mon.c (mon.2) 3074 : audit [DBG] from='client.? 172.21.15.6:0/241368206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:39.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:39 smithi130 bash[51302]: debug 2023-08-28T02:18:39.482+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:39.485795+0000) 2023-08-28T02:18:40.751 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:40 smithi006 bash[27292]: cluster 2023-08-28T02:18:39.423632+0000 mgr.y (mgr.14901) 7801 : cluster [DBG] pgmap v7808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:40.752 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:40 smithi006 bash[33918]: cluster 2023-08-28T02:18:39.423632+0000 mgr.y (mgr.14901) 7801 : cluster [DBG] pgmap v7808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:40.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:40 smithi130 bash[27579]: cluster 2023-08-28T02:18:39.423632+0000 mgr.y (mgr.14901) 7801 : cluster [DBG] pgmap v7808: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:40.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:40 smithi130 bash[51302]: debug 2023-08-28T02:18:40.482+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:40.486074+0000) 2023-08-28T02:18:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:18:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:18:40] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:18:41.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:41 smithi006 bash[27292]: audit 2023-08-28T02:18:40.850263+0000 mon.c (mon.2) 3075 : audit [DBG] from='client.? 172.21.15.6:0/1973280353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:18:41.267Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:18:41.268Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:41.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:41 smithi006 bash[33918]: audit 2023-08-28T02:18:40.850263+0000 mon.c (mon.2) 3075 : audit [DBG] from='client.? 172.21.15.6:0/1973280353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:41.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:41 smithi130 bash[27579]: audit 2023-08-28T02:18:40.850263+0000 mon.c (mon.2) 3075 : audit [DBG] from='client.? 172.21.15.6:0/1973280353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:41.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:41 smithi130 bash[51302]: debug 2023-08-28T02:18:41.486+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:41.486368+0000) 2023-08-28T02:18:42.831 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:42 smithi006 bash[27292]: cluster 2023-08-28T02:18:41.424441+0000 mgr.y (mgr.14901) 7802 : cluster [DBG] pgmap v7809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:42.831 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:42 smithi006 bash[33918]: cluster 2023-08-28T02:18:41.424441+0000 mgr.y (mgr.14901) 7802 : cluster [DBG] pgmap v7809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:42.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:42 smithi130 bash[27579]: cluster 2023-08-28T02:18:41.424441+0000 mgr.y (mgr.14901) 7802 : cluster [DBG] pgmap v7809: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:42.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:42 smithi130 bash[51302]: debug 2023-08-28T02:18:42.482+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:42.486649+0000) 2023-08-28T02:18:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:18:42.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:18:42.833Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:43.863 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:43 smithi130 bash[51302]: debug 2023-08-28T02:18:43.486+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:43.486961+0000) 2023-08-28T02:18:44.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:43 smithi130 bash[27579]: audit 2023-08-28T02:18:43.264815+0000 mon.a (mon.0) 4985 : audit [DBG] from='client.? 172.21.15.6:0/1099302132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:44.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:43 smithi130 bash[27579]: cluster 2023-08-28T02:18:43.425193+0000 mgr.y (mgr.14901) 7803 : cluster [DBG] pgmap v7810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:44.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:43 smithi006 bash[27292]: audit 2023-08-28T02:18:43.264815+0000 mon.a (mon.0) 4985 : audit [DBG] from='client.? 172.21.15.6:0/1099302132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:44.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:43 smithi006 bash[27292]: cluster 2023-08-28T02:18:43.425193+0000 mgr.y (mgr.14901) 7803 : cluster [DBG] pgmap v7810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:44.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:43 smithi006 bash[33918]: audit 2023-08-28T02:18:43.264815+0000 mon.a (mon.0) 4985 : audit [DBG] from='client.? 172.21.15.6:0/1099302132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:44.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:43 smithi006 bash[33918]: cluster 2023-08-28T02:18:43.425193+0000 mgr.y (mgr.14901) 7803 : cluster [DBG] pgmap v7810: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:44.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:44 smithi130 bash[51302]: debug 2023-08-28T02:18:44.486+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:44.487208+0000) 2023-08-28T02:18:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:18:44.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:18:44.709Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:45.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:45 smithi130 bash[51302]: debug 2023-08-28T02:18:45.486+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:45.487455+0000) 2023-08-28T02:18:46.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:46 smithi130 bash[27579]: cluster 2023-08-28T02:18:45.426262+0000 mgr.y (mgr.14901) 7804 : cluster [DBG] pgmap v7811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:46.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:46 smithi130 bash[27579]: audit 2023-08-28T02:18:45.671788+0000 mon.c (mon.2) 3076 : audit [DBG] from='client.? 172.21.15.6:0/1458898834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:46.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:46 smithi130 bash[51302]: debug 2023-08-28T02:18:46.486+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:46.487677+0000) 2023-08-28T02:18:46.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:46 smithi006 bash[27292]: cluster 2023-08-28T02:18:45.426262+0000 mgr.y (mgr.14901) 7804 : cluster [DBG] pgmap v7811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:46.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:46 smithi006 bash[27292]: audit 2023-08-28T02:18:45.671788+0000 mon.c (mon.2) 3076 : audit [DBG] from='client.? 172.21.15.6:0/1458898834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:46.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:46 smithi006 bash[33918]: cluster 2023-08-28T02:18:45.426262+0000 mgr.y (mgr.14901) 7804 : cluster [DBG] pgmap v7811: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:46.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:46 smithi006 bash[33918]: audit 2023-08-28T02:18:45.671788+0000 mon.c (mon.2) 3076 : audit [DBG] from='client.? 172.21.15.6:0/1458898834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:47.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:47 smithi130 bash[51302]: debug 2023-08-28T02:18:47.486+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:47.487891+0000) 2023-08-28T02:18:47.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:47 smithi130 bash[27579]: audit 2023-08-28T02:18:46.805081+0000 mon.b (mon.1) 858 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:18:47.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:47 smithi006 bash[33918]: audit 2023-08-28T02:18:46.805081+0000 mon.b (mon.1) 858 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:18:47.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:47 smithi006 bash[27292]: audit 2023-08-28T02:18:46.805081+0000 mon.b (mon.1) 858 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:18:48.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:48 smithi130 bash[51302]: debug 2023-08-28T02:18:48.486+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:48.488229+0000) 2023-08-28T02:18:48.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:48 smithi130 bash[27579]: cluster 2023-08-28T02:18:47.426917+0000 mgr.y (mgr.14901) 7805 : cluster [DBG] pgmap v7812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:48.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:48 smithi130 bash[27579]: audit 2023-08-28T02:18:48.093983+0000 mon.c (mon.2) 3077 : audit [DBG] from='client.? 172.21.15.6:0/2404303610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:48.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:48 smithi006 bash[33918]: cluster 2023-08-28T02:18:47.426917+0000 mgr.y (mgr.14901) 7805 : cluster [DBG] pgmap v7812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:48.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:48 smithi006 bash[33918]: audit 2023-08-28T02:18:48.093983+0000 mon.c (mon.2) 3077 : audit [DBG] from='client.? 172.21.15.6:0/2404303610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:48.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:48 smithi006 bash[27292]: cluster 2023-08-28T02:18:47.426917+0000 mgr.y (mgr.14901) 7805 : cluster [DBG] pgmap v7812: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:48.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:48 smithi006 bash[27292]: audit 2023-08-28T02:18:48.093983+0000 mon.c (mon.2) 3077 : audit [DBG] from='client.? 172.21.15.6:0/2404303610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:49.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:49 smithi130 bash[51302]: debug 2023-08-28T02:18:49.486+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:49.488511+0000) 2023-08-28T02:18:50.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:50 smithi130 bash[27579]: cluster 2023-08-28T02:18:49.427936+0000 mgr.y (mgr.14901) 7806 : cluster [DBG] pgmap v7813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:50.901 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:50 smithi130 bash[51302]: debug 2023-08-28T02:18:50.486+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:50.488801+0000) 2023-08-28T02:18:50.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:18:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:18:50] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:18:50.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:50 smithi006 bash[33918]: cluster 2023-08-28T02:18:49.427936+0000 mgr.y (mgr.14901) 7806 : cluster [DBG] pgmap v7813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:50.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:50 smithi006 bash[27292]: cluster 2023-08-28T02:18:49.427936+0000 mgr.y (mgr.14901) 7806 : cluster [DBG] pgmap v7813: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:51.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:51 smithi006 bash[33918]: audit 2023-08-28T02:18:50.517365+0000 mon.a (mon.0) 4986 : audit [DBG] from='client.? 172.21.15.6:0/1085609164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:51.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:51 smithi006 bash[27292]: audit 2023-08-28T02:18:50.517365+0000 mon.a (mon.0) 4986 : audit [DBG] from='client.? 172.21.15.6:0/1085609164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:18:51.268Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:51.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:18:51.269Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:51.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:51 smithi130 bash[27579]: audit 2023-08-28T02:18:50.517365+0000 mon.a (mon.0) 4986 : audit [DBG] from='client.? 172.21.15.6:0/1085609164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:51.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:51 smithi130 bash[51302]: debug 2023-08-28T02:18:51.486+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:51.489108+0000) 2023-08-28T02:18:52.831 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:52 smithi006 bash[27292]: cluster 2023-08-28T02:18:51.428634+0000 mgr.y (mgr.14901) 7807 : cluster [DBG] pgmap v7814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:52.832 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:52 smithi006 bash[33918]: cluster 2023-08-28T02:18:51.428634+0000 mgr.y (mgr.14901) 7807 : cluster [DBG] pgmap v7814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:52.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:52 smithi130 bash[27579]: cluster 2023-08-28T02:18:51.428634+0000 mgr.y (mgr.14901) 7807 : cluster [DBG] pgmap v7814: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:52.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:52 smithi130 bash[51302]: debug 2023-08-28T02:18:52.486+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:52.489430+0000) 2023-08-28T02:18:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:18:52.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:18:52.833Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:53.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:53 smithi130 bash[51302]: debug 2023-08-28T02:18:53.485+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:53.489756+0000) 2023-08-28T02:18:53.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:53 smithi130 bash[27579]: audit 2023-08-28T02:18:52.702177+0000 mon.a (mon.0) 4987 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:18:53.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:53 smithi130 bash[27579]: audit 2023-08-28T02:18:52.980733+0000 mon.c (mon.2) 3078 : audit [DBG] from='client.? 172.21.15.6:0/616924293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:53.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:53 smithi130 bash[27579]: audit 2023-08-28T02:18:53.135523+0000 mon.a (mon.0) 4988 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:18:53.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:53 smithi130 bash[27579]: audit 2023-08-28T02:18:53.435211+0000 mon.b (mon.1) 859 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:18:53.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:53 smithi130 bash[27579]: audit 2023-08-28T02:18:53.437496+0000 mon.b (mon.1) 860 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:18:53.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:53 smithi130 bash[27579]: audit 2023-08-28T02:18:53.444348+0000 mon.a (mon.0) 4989 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:18:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:53 smithi006 bash[33918]: audit 2023-08-28T02:18:52.702177+0000 mon.a (mon.0) 4987 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:18:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:53 smithi006 bash[33918]: audit 2023-08-28T02:18:52.980733+0000 mon.c (mon.2) 3078 : audit [DBG] from='client.? 172.21.15.6:0/616924293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:53 smithi006 bash[33918]: audit 2023-08-28T02:18:53.135523+0000 mon.a (mon.0) 4988 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:18:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:53 smithi006 bash[33918]: audit 2023-08-28T02:18:53.435211+0000 mon.b (mon.1) 859 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:18:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:53 smithi006 bash[33918]: audit 2023-08-28T02:18:53.437496+0000 mon.b (mon.1) 860 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:18:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:53 smithi006 bash[33918]: audit 2023-08-28T02:18:53.444348+0000 mon.a (mon.0) 4989 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:18:53.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:53 smithi006 bash[27292]: audit 2023-08-28T02:18:52.702177+0000 mon.a (mon.0) 4987 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:18:53.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:53 smithi006 bash[27292]: audit 2023-08-28T02:18:52.980733+0000 mon.c (mon.2) 3078 : audit [DBG] from='client.? 172.21.15.6:0/616924293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:53.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:53 smithi006 bash[27292]: audit 2023-08-28T02:18:53.135523+0000 mon.a (mon.0) 4988 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:18:53.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:53 smithi006 bash[27292]: audit 2023-08-28T02:18:53.435211+0000 mon.b (mon.1) 859 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:18:53.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:53 smithi006 bash[27292]: audit 2023-08-28T02:18:53.437496+0000 mon.b (mon.1) 860 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:18:53.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:53 smithi006 bash[27292]: audit 2023-08-28T02:18:53.444348+0000 mon.a (mon.0) 4989 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:18:54.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:54 smithi130 bash[27579]: cluster 2023-08-28T02:18:53.429317+0000 mgr.y (mgr.14901) 7808 : cluster [DBG] pgmap v7815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:54.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:54 smithi130 bash[51302]: debug 2023-08-28T02:18:54.485+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:54.490065+0000) 2023-08-28T02:18:54.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:54 smithi006 bash[33918]: cluster 2023-08-28T02:18:53.429317+0000 mgr.y (mgr.14901) 7808 : cluster [DBG] pgmap v7815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:54.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:54 smithi006 bash[27292]: cluster 2023-08-28T02:18:53.429317+0000 mgr.y (mgr.14901) 7808 : cluster [DBG] pgmap v7815: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:18:54.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:18:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:18:54.710Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:18:55.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:55 smithi130 bash[27579]: audit 2023-08-28T02:18:55.403004+0000 mon.a (mon.0) 4990 : audit [DBG] from='client.? 172.21.15.6:0/4201821612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:55.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:55 smithi130 bash[51302]: debug 2023-08-28T02:18:55.489+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:55.490432+0000) 2023-08-28T02:18:55.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:55 smithi006 bash[33918]: audit 2023-08-28T02:18:55.403004+0000 mon.a (mon.0) 4990 : audit [DBG] from='client.? 172.21.15.6:0/4201821612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:55.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:55 smithi006 bash[27292]: audit 2023-08-28T02:18:55.403004+0000 mon.a (mon.0) 4990 : audit [DBG] from='client.? 172.21.15.6:0/4201821612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:56 smithi130 bash[27579]: cluster 2023-08-28T02:18:55.430386+0000 mgr.y (mgr.14901) 7809 : cluster [DBG] pgmap v7816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:56.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:56 smithi130 bash[51302]: debug 2023-08-28T02:18:56.485+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:56.490710+0000) 2023-08-28T02:18:57.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:56 smithi006 bash[33918]: cluster 2023-08-28T02:18:55.430386+0000 mgr.y (mgr.14901) 7809 : cluster [DBG] pgmap v7816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:57.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:56 smithi006 bash[27292]: cluster 2023-08-28T02:18:55.430386+0000 mgr.y (mgr.14901) 7809 : cluster [DBG] pgmap v7816: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:57.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:57 smithi130 bash[51302]: debug 2023-08-28T02:18:57.489+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:57.490997+0000) 2023-08-28T02:18:58.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:58 smithi130 bash[27579]: cluster 2023-08-28T02:18:57.431027+0000 mgr.y (mgr.14901) 7810 : cluster [DBG] pgmap v7817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:58.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:18:58 smithi130 bash[27579]: audit 2023-08-28T02:18:57.819445+0000 mon.a (mon.0) 4991 : audit [DBG] from='client.? 172.21.15.6:0/3250324066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:58.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:58 smithi130 bash[51302]: debug 2023-08-28T02:18:58.489+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:58.491328+0000) 2023-08-28T02:18:59.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:58 smithi006 bash[33918]: cluster 2023-08-28T02:18:57.431027+0000 mgr.y (mgr.14901) 7810 : cluster [DBG] pgmap v7817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:59.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:18:58 smithi006 bash[33918]: audit 2023-08-28T02:18:57.819445+0000 mon.a (mon.0) 4991 : audit [DBG] from='client.? 172.21.15.6:0/3250324066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:59.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:58 smithi006 bash[27292]: cluster 2023-08-28T02:18:57.431027+0000 mgr.y (mgr.14901) 7810 : cluster [DBG] pgmap v7817: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:18:59.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:18:58 smithi006 bash[27292]: audit 2023-08-28T02:18:57.819445+0000 mon.a (mon.0) 4991 : audit [DBG] from='client.? 172.21.15.6:0/3250324066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:59.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:18:59 smithi130 bash[51302]: debug 2023-08-28T02:18:59.489+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:59.491645+0000) 2023-08-28T02:19:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:00 smithi130 bash[27579]: cluster 2023-08-28T02:18:59.432119+0000 mgr.y (mgr.14901) 7811 : cluster [DBG] pgmap v7818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:00 smithi130 bash[27579]: audit 2023-08-28T02:19:00.231778+0000 mon.c (mon.2) 3079 : audit [DBG] from='client.? 172.21.15.6:0/778823263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:00.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:00 smithi130 bash[51302]: debug 2023-08-28T02:19:00.489+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:00.491929+0000) 2023-08-28T02:19:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:19:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:19:00] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:19:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:00 smithi006 bash[27292]: cluster 2023-08-28T02:18:59.432119+0000 mgr.y (mgr.14901) 7811 : cluster [DBG] pgmap v7818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:00 smithi006 bash[27292]: audit 2023-08-28T02:19:00.231778+0000 mon.c (mon.2) 3079 : audit [DBG] from='client.? 172.21.15.6:0/778823263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:01.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:00 smithi006 bash[33918]: cluster 2023-08-28T02:18:59.432119+0000 mgr.y (mgr.14901) 7811 : cluster [DBG] pgmap v7818: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:01.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:00 smithi006 bash[33918]: audit 2023-08-28T02:19:00.231778+0000 mon.c (mon.2) 3079 : audit [DBG] from='client.? 172.21.15.6:0/778823263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:19:01.269Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:19:01.271Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:01.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:01 smithi130 bash[51302]: debug 2023-08-28T02:19:01.489+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:01.492297+0000) 2023-08-28T02:19:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:02 smithi130 bash[27579]: cluster 2023-08-28T02:19:01.432754+0000 mgr.y (mgr.14901) 7812 : cluster [DBG] pgmap v7819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:02.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:02 smithi130 bash[27579]: audit 2023-08-28T02:19:02.651718+0000 mon.a (mon.0) 4992 : audit [DBG] from='client.? 172.21.15.6:0/2828713666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:02.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:02 smithi130 bash[51302]: debug 2023-08-28T02:19:02.489+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:02.492583+0000) 2023-08-28T02:19:03.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:02 smithi006 bash[27292]: cluster 2023-08-28T02:19:01.432754+0000 mgr.y (mgr.14901) 7812 : cluster [DBG] pgmap v7819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:03.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:02 smithi006 bash[27292]: audit 2023-08-28T02:19:02.651718+0000 mon.a (mon.0) 4992 : audit [DBG] from='client.? 172.21.15.6:0/2828713666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:03.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:02 smithi006 bash[33918]: cluster 2023-08-28T02:19:01.432754+0000 mgr.y (mgr.14901) 7812 : cluster [DBG] pgmap v7819: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:03.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:02 smithi006 bash[33918]: audit 2023-08-28T02:19:02.651718+0000 mon.a (mon.0) 4992 : audit [DBG] from='client.? 172.21.15.6:0/2828713666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:19:02.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:19:02.834Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:03.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:03 smithi130 bash[51302]: debug 2023-08-28T02:19:03.489+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:03.492856+0000) 2023-08-28T02:19:04.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:03 smithi006 bash[27292]: cluster 2023-08-28T02:19:03.433369+0000 mgr.y (mgr.14901) 7813 : cluster [DBG] pgmap v7820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:04.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:03 smithi006 bash[33918]: cluster 2023-08-28T02:19:03.433369+0000 mgr.y (mgr.14901) 7813 : cluster [DBG] pgmap v7820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:03 smithi130 bash[27579]: cluster 2023-08-28T02:19:03.433369+0000 mgr.y (mgr.14901) 7813 : cluster [DBG] pgmap v7820: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:04.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:04 smithi130 bash[51302]: debug 2023-08-28T02:19:04.489+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:04.493120+0000) 2023-08-28T02:19:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:19:04.709Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:04.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:19:04.711Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:05.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:05 smithi006 bash[27292]: audit 2023-08-28T02:19:05.070209+0000 mon.a (mon.0) 4993 : audit [DBG] from='client.? 172.21.15.6:0/2040393430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:05.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:05 smithi006 bash[33918]: audit 2023-08-28T02:19:05.070209+0000 mon.a (mon.0) 4993 : audit [DBG] from='client.? 172.21.15.6:0/2040393430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:05.512 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:05 smithi130 bash[27579]: audit 2023-08-28T02:19:05.070209+0000 mon.a (mon.0) 4993 : audit [DBG] from='client.? 172.21.15.6:0/2040393430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:05.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:05 smithi130 bash[51302]: debug 2023-08-28T02:19:05.489+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:05.493407+0000) 2023-08-28T02:19:06.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:06 smithi006 bash[33918]: cluster 2023-08-28T02:19:05.434620+0000 mgr.y (mgr.14901) 7814 : cluster [DBG] pgmap v7821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:06.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:06 smithi006 bash[27292]: cluster 2023-08-28T02:19:05.434620+0000 mgr.y (mgr.14901) 7814 : cluster [DBG] pgmap v7821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:06.492 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:06 smithi130 bash[27579]: cluster 2023-08-28T02:19:05.434620+0000 mgr.y (mgr.14901) 7814 : cluster [DBG] pgmap v7821: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:06.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:06 smithi130 bash[51302]: debug 2023-08-28T02:19:06.489+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:06.493636+0000) 2023-08-28T02:19:07.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:07 smithi130 bash[27579]: audit 2023-08-28T02:19:07.485000+0000 mon.c (mon.2) 3080 : audit [DBG] from='client.? 172.21.15.6:0/117621916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:07.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:07 smithi130 bash[51302]: debug 2023-08-28T02:19:07.489+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:07.493874+0000) 2023-08-28T02:19:07.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:07 smithi006 bash[33918]: audit 2023-08-28T02:19:07.485000+0000 mon.c (mon.2) 3080 : audit [DBG] from='client.? 172.21.15.6:0/117621916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:07.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:07 smithi006 bash[27292]: audit 2023-08-28T02:19:07.485000+0000 mon.c (mon.2) 3080 : audit [DBG] from='client.? 172.21.15.6:0/117621916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:08 smithi130 bash[27579]: cluster 2023-08-28T02:19:07.435279+0000 mgr.y (mgr.14901) 7815 : cluster [DBG] pgmap v7822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:08.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:08 smithi130 bash[51302]: debug 2023-08-28T02:19:08.493+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:08.494172+0000) 2023-08-28T02:19:08.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:08 smithi006 bash[27292]: cluster 2023-08-28T02:19:07.435279+0000 mgr.y (mgr.14901) 7815 : cluster [DBG] pgmap v7822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:08.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:08 smithi006 bash[33918]: cluster 2023-08-28T02:19:07.435279+0000 mgr.y (mgr.14901) 7815 : cluster [DBG] pgmap v7822: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:09.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:09 smithi130 bash[51302]: debug 2023-08-28T02:19:09.493+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:09.494469+0000) 2023-08-28T02:19:10.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:10 smithi130 bash[27579]: cluster 2023-08-28T02:19:09.436467+0000 mgr.y (mgr.14901) 7816 : cluster [DBG] pgmap v7823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:10.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:10 smithi130 bash[27579]: audit 2023-08-28T02:19:09.899020+0000 mon.c (mon.2) 3081 : audit [DBG] from='client.? 172.21.15.6:0/1496296079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:10.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:10 smithi130 bash[51302]: debug 2023-08-28T02:19:10.493+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:10.494744+0000) 2023-08-28T02:19:10.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:19:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:19:10] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:19:10.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:10 smithi006 bash[33918]: cluster 2023-08-28T02:19:09.436467+0000 mgr.y (mgr.14901) 7816 : cluster [DBG] pgmap v7823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:10.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:10 smithi006 bash[33918]: audit 2023-08-28T02:19:09.899020+0000 mon.c (mon.2) 3081 : audit [DBG] from='client.? 172.21.15.6:0/1496296079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:10.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:10 smithi006 bash[27292]: cluster 2023-08-28T02:19:09.436467+0000 mgr.y (mgr.14901) 7816 : cluster [DBG] pgmap v7823: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:10.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:10 smithi006 bash[27292]: audit 2023-08-28T02:19:09.899020+0000 mon.c (mon.2) 3081 : audit [DBG] from='client.? 172.21.15.6:0/1496296079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:19:11.269Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:19:11.271Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:11.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:11 smithi130 bash[51302]: debug 2023-08-28T02:19:11.493+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:11.495062+0000) 2023-08-28T02:19:12.833 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:12 smithi006 bash[27292]: cluster 2023-08-28T02:19:11.437325+0000 mgr.y (mgr.14901) 7817 : cluster [DBG] pgmap v7824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:12.833 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:12 smithi006 bash[27292]: audit 2023-08-28T02:19:12.307006+0000 mon.a (mon.0) 4994 : audit [DBG] from='client.? 172.21.15.6:0/3618687353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:12.834 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:12 smithi006 bash[33918]: cluster 2023-08-28T02:19:11.437325+0000 mgr.y (mgr.14901) 7817 : cluster [DBG] pgmap v7824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:12.834 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:12 smithi006 bash[33918]: audit 2023-08-28T02:19:12.307006+0000 mon.a (mon.0) 4994 : audit [DBG] from='client.? 172.21.15.6:0/3618687353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:12 smithi130 bash[27579]: cluster 2023-08-28T02:19:11.437325+0000 mgr.y (mgr.14901) 7817 : cluster [DBG] pgmap v7824: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:12 smithi130 bash[27579]: audit 2023-08-28T02:19:12.307006+0000 mon.a (mon.0) 4994 : audit [DBG] from='client.? 172.21.15.6:0/3618687353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:12.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:12 smithi130 bash[51302]: debug 2023-08-28T02:19:12.493+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:12.495438+0000) 2023-08-28T02:19:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:19:12.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:19:12.834Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:13.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:13 smithi130 bash[51302]: debug 2023-08-28T02:19:13.493+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:13.495710+0000) 2023-08-28T02:19:14.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:13 smithi006 bash[27292]: cluster 2023-08-28T02:19:13.438000+0000 mgr.y (mgr.14901) 7818 : cluster [DBG] pgmap v7825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:13 smithi006 bash[33918]: cluster 2023-08-28T02:19:13.438000+0000 mgr.y (mgr.14901) 7818 : cluster [DBG] pgmap v7825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:14.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:13 smithi130 bash[27579]: cluster 2023-08-28T02:19:13.438000+0000 mgr.y (mgr.14901) 7818 : cluster [DBG] pgmap v7825: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:14.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:14 smithi130 bash[51302]: debug 2023-08-28T02:19:14.493+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:14.496026+0000) 2023-08-28T02:19:14.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:14 smithi006 bash[27292]: audit 2023-08-28T02:19:14.736126+0000 mon.a (mon.0) 4995 : audit [DBG] from='client.? 172.21.15.6:0/2219153475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:14.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:14 smithi006 bash[33918]: audit 2023-08-28T02:19:14.736126+0000 mon.a (mon.0) 4995 : audit [DBG] from='client.? 172.21.15.6:0/2219153475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:19:14.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:19:14.712Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:15.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:14 smithi130 bash[27579]: audit 2023-08-28T02:19:14.736126+0000 mon.a (mon.0) 4995 : audit [DBG] from='client.? 172.21.15.6:0/2219153475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:15.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:15 smithi130 bash[51302]: debug 2023-08-28T02:19:15.493+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:15.496325+0000) 2023-08-28T02:19:16.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:15 smithi006 bash[27292]: cluster 2023-08-28T02:19:15.439202+0000 mgr.y (mgr.14901) 7819 : cluster [DBG] pgmap v7826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:16.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:15 smithi006 bash[33918]: cluster 2023-08-28T02:19:15.439202+0000 mgr.y (mgr.14901) 7819 : cluster [DBG] pgmap v7826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:16.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:15 smithi130 bash[27579]: cluster 2023-08-28T02:19:15.439202+0000 mgr.y (mgr.14901) 7819 : cluster [DBG] pgmap v7826: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:16.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:16 smithi130 bash[51302]: debug 2023-08-28T02:19:16.493+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:16.496612+0000) 2023-08-28T02:19:17.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:17 smithi006 bash[27292]: audit 2023-08-28T02:19:17.147024+0000 mon.a (mon.0) 4996 : audit [DBG] from='client.? 172.21.15.6:0/4291352741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:17.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:17 smithi006 bash[33918]: audit 2023-08-28T02:19:17.147024+0000 mon.a (mon.0) 4996 : audit [DBG] from='client.? 172.21.15.6:0/4291352741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:17.495 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:17 smithi130 bash[27579]: audit 2023-08-28T02:19:17.147024+0000 mon.a (mon.0) 4996 : audit [DBG] from='client.? 172.21.15.6:0/4291352741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:17.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:17 smithi130 bash[51302]: debug 2023-08-28T02:19:17.493+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:17.496901+0000) 2023-08-28T02:19:18.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:18 smithi006 bash[33918]: cluster 2023-08-28T02:19:17.439891+0000 mgr.y (mgr.14901) 7820 : cluster [DBG] pgmap v7827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:18.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:18 smithi006 bash[27292]: cluster 2023-08-28T02:19:17.439891+0000 mgr.y (mgr.14901) 7820 : cluster [DBG] pgmap v7827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:18.495 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:18 smithi130 bash[27579]: cluster 2023-08-28T02:19:17.439891+0000 mgr.y (mgr.14901) 7820 : cluster [DBG] pgmap v7827: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:18.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:18 smithi130 bash[51302]: debug 2023-08-28T02:19:18.493+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:18.497192+0000) 2023-08-28T02:19:19.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:19 smithi130 bash[51302]: debug 2023-08-28T02:19:19.493+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:19.497373+0000) 2023-08-28T02:19:20.752 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:20 smithi006 bash[33918]: cluster 2023-08-28T02:19:19.440990+0000 mgr.y (mgr.14901) 7821 : cluster [DBG] pgmap v7828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:20.753 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:20 smithi006 bash[33918]: audit 2023-08-28T02:19:19.571758+0000 mon.c (mon.2) 3082 : audit [DBG] from='client.? 172.21.15.6:0/3927277081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:20.753 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:20 smithi006 bash[27292]: cluster 2023-08-28T02:19:19.440990+0000 mgr.y (mgr.14901) 7821 : cluster [DBG] pgmap v7828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:20.753 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:20 smithi006 bash[27292]: audit 2023-08-28T02:19:19.571758+0000 mon.c (mon.2) 3082 : audit [DBG] from='client.? 172.21.15.6:0/3927277081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:20 smithi130 bash[27579]: cluster 2023-08-28T02:19:19.440990+0000 mgr.y (mgr.14901) 7821 : cluster [DBG] pgmap v7828: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:20 smithi130 bash[27579]: audit 2023-08-28T02:19:19.571758+0000 mon.c (mon.2) 3082 : audit [DBG] from='client.? 172.21.15.6:0/3927277081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:20.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:20 smithi130 bash[51302]: debug 2023-08-28T02:19:20.493+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:20.497600+0000) 2023-08-28T02:19:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:19:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:19:20] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:19:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:19:21.270Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:19:21.272Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:21.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:21 smithi130 bash[51302]: debug 2023-08-28T02:19:21.497+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:21.497964+0000) 2023-08-28T02:19:22.833 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:22 smithi006 bash[33918]: cluster 2023-08-28T02:19:21.441656+0000 mgr.y (mgr.14901) 7822 : cluster [DBG] pgmap v7829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:22.833 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:22 smithi006 bash[33918]: audit 2023-08-28T02:19:21.986871+0000 mon.a (mon.0) 4997 : audit [DBG] from='client.? 172.21.15.6:0/1778117830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:22.834 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:22 smithi006 bash[27292]: cluster 2023-08-28T02:19:21.441656+0000 mgr.y (mgr.14901) 7822 : cluster [DBG] pgmap v7829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:22.834 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:22 smithi006 bash[27292]: audit 2023-08-28T02:19:21.986871+0000 mon.a (mon.0) 4997 : audit [DBG] from='client.? 172.21.15.6:0/1778117830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:22 smithi130 bash[27579]: cluster 2023-08-28T02:19:21.441656+0000 mgr.y (mgr.14901) 7822 : cluster [DBG] pgmap v7829: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:22 smithi130 bash[27579]: audit 2023-08-28T02:19:21.986871+0000 mon.a (mon.0) 4997 : audit [DBG] from='client.? 172.21.15.6:0/1778117830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:22.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:22 smithi130 bash[51302]: debug 2023-08-28T02:19:22.493+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:22.498259+0000) 2023-08-28T02:19:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:19:22.833Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:19:22.835Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:23.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:23 smithi130 bash[51302]: debug 2023-08-28T02:19:23.493+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:23.498529+0000) 2023-08-28T02:19:24.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:23 smithi006 bash[27292]: cluster 2023-08-28T02:19:23.442382+0000 mgr.y (mgr.14901) 7823 : cluster [DBG] pgmap v7830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:24.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:23 smithi006 bash[33918]: cluster 2023-08-28T02:19:23.442382+0000 mgr.y (mgr.14901) 7823 : cluster [DBG] pgmap v7830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:23 smithi130 bash[27579]: cluster 2023-08-28T02:19:23.442382+0000 mgr.y (mgr.14901) 7823 : cluster [DBG] pgmap v7830: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:24.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:24 smithi130 bash[51302]: debug 2023-08-28T02:19:24.493+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:24.498811+0000) 2023-08-28T02:19:24.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:24 smithi006 bash[27292]: audit 2023-08-28T02:19:24.400331+0000 mon.a (mon.0) 4998 : audit [DBG] from='client.? 172.21.15.6:0/1073700750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:24.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:24 smithi006 bash[33918]: audit 2023-08-28T02:19:24.400331+0000 mon.a (mon.0) 4998 : audit [DBG] from='client.? 172.21.15.6:0/1073700750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:19:24.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:19:24.714Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:25.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:24 smithi130 bash[27579]: audit 2023-08-28T02:19:24.400331+0000 mon.a (mon.0) 4998 : audit [DBG] from='client.? 172.21.15.6:0/1073700750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:25.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:25 smithi130 bash[51302]: debug 2023-08-28T02:19:25.497+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:25.499073+0000) 2023-08-28T02:19:26.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:25 smithi006 bash[33918]: cluster 2023-08-28T02:19:25.443457+0000 mgr.y (mgr.14901) 7824 : cluster [DBG] pgmap v7831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:26.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:25 smithi006 bash[27292]: cluster 2023-08-28T02:19:25.443457+0000 mgr.y (mgr.14901) 7824 : cluster [DBG] pgmap v7831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:26.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:25 smithi130 bash[27579]: cluster 2023-08-28T02:19:25.443457+0000 mgr.y (mgr.14901) 7824 : cluster [DBG] pgmap v7831: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:26.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:26 smithi130 bash[51302]: debug 2023-08-28T02:19:26.497+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:26.499321+0000) 2023-08-28T02:19:27.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:26 smithi006 bash[33918]: audit 2023-08-28T02:19:26.829600+0000 mon.a (mon.0) 4999 : audit [DBG] from='client.? 172.21.15.6:0/3557851627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:27.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:26 smithi006 bash[27292]: audit 2023-08-28T02:19:26.829600+0000 mon.a (mon.0) 4999 : audit [DBG] from='client.? 172.21.15.6:0/3557851627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:27.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:26 smithi130 bash[27579]: audit 2023-08-28T02:19:26.829600+0000 mon.a (mon.0) 4999 : audit [DBG] from='client.? 172.21.15.6:0/3557851627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:27.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:27 smithi130 bash[51302]: debug 2023-08-28T02:19:27.497+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:27.499631+0000) 2023-08-28T02:19:28.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:27 smithi006 bash[33918]: cluster 2023-08-28T02:19:27.444129+0000 mgr.y (mgr.14901) 7825 : cluster [DBG] pgmap v7832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:28.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:27 smithi006 bash[27292]: cluster 2023-08-28T02:19:27.444129+0000 mgr.y (mgr.14901) 7825 : cluster [DBG] pgmap v7832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:28.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:27 smithi130 bash[27579]: cluster 2023-08-28T02:19:27.444129+0000 mgr.y (mgr.14901) 7825 : cluster [DBG] pgmap v7832: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:28.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:28 smithi130 bash[51302]: debug 2023-08-28T02:19:28.497+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:28.499906+0000) 2023-08-28T02:19:29.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:29 smithi130 bash[27579]: audit 2023-08-28T02:19:29.247795+0000 mon.c (mon.2) 3083 : audit [DBG] from='client.? 172.21.15.6:0/995436655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:29.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:29 smithi130 bash[51302]: debug 2023-08-28T02:19:29.497+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:29.500167+0000) 2023-08-28T02:19:29.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:29 smithi006 bash[33918]: audit 2023-08-28T02:19:29.247795+0000 mon.c (mon.2) 3083 : audit [DBG] from='client.? 172.21.15.6:0/995436655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:29.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:29 smithi006 bash[27292]: audit 2023-08-28T02:19:29.247795+0000 mon.c (mon.2) 3083 : audit [DBG] from='client.? 172.21.15.6:0/995436655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:30 smithi130 bash[27579]: cluster 2023-08-28T02:19:29.445305+0000 mgr.y (mgr.14901) 7826 : cluster [DBG] pgmap v7833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:30.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:30 smithi130 bash[51302]: debug 2023-08-28T02:19:30.497+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:30.500454+0000) 2023-08-28T02:19:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:30 smithi006 bash[33918]: cluster 2023-08-28T02:19:29.445305+0000 mgr.y (mgr.14901) 7826 : cluster [DBG] pgmap v7833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:30 smithi006 bash[27292]: cluster 2023-08-28T02:19:29.445305+0000 mgr.y (mgr.14901) 7826 : cluster [DBG] pgmap v7833: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:31.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:19:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:19:30] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:19:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:19:31.271Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:31.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:19:31.273Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:31.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:31 smithi130 bash[51302]: debug 2023-08-28T02:19:31.497+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:31.500724+0000) 2023-08-28T02:19:32.834 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:32 smithi006 bash[33918]: cluster 2023-08-28T02:19:31.445751+0000 mgr.y (mgr.14901) 7827 : cluster [DBG] pgmap v7834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:32.835 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:32 smithi006 bash[33918]: audit 2023-08-28T02:19:31.646761+0000 mon.c (mon.2) 3084 : audit [DBG] from='client.? 172.21.15.6:0/1738707585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:32.835 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:32 smithi006 bash[27292]: cluster 2023-08-28T02:19:31.445751+0000 mgr.y (mgr.14901) 7827 : cluster [DBG] pgmap v7834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:32.835 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:32 smithi006 bash[27292]: audit 2023-08-28T02:19:31.646761+0000 mon.c (mon.2) 3084 : audit [DBG] from='client.? 172.21.15.6:0/1738707585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:32 smithi130 bash[27579]: cluster 2023-08-28T02:19:31.445751+0000 mgr.y (mgr.14901) 7827 : cluster [DBG] pgmap v7834: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:32 smithi130 bash[27579]: audit 2023-08-28T02:19:31.646761+0000 mon.c (mon.2) 3084 : audit [DBG] from='client.? 172.21.15.6:0/1738707585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:32.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:32 smithi130 bash[51302]: debug 2023-08-28T02:19:32.497+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:32.500932+0000) 2023-08-28T02:19:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:19:32.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:19:32.836Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:33.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:33 smithi130 bash[51302]: debug 2023-08-28T02:19:33.497+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:33.501239+0000) 2023-08-28T02:19:34.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:33 smithi006 bash[33918]: cluster 2023-08-28T02:19:33.446373+0000 mgr.y (mgr.14901) 7828 : cluster [DBG] pgmap v7835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:33 smithi006 bash[27292]: cluster 2023-08-28T02:19:33.446373+0000 mgr.y (mgr.14901) 7828 : cluster [DBG] pgmap v7835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:33 smithi130 bash[27579]: cluster 2023-08-28T02:19:33.446373+0000 mgr.y (mgr.14901) 7828 : cluster [DBG] pgmap v7835: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:34.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:34 smithi130 bash[51302]: debug 2023-08-28T02:19:34.497+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:34.501560+0000) 2023-08-28T02:19:34.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:34 smithi006 bash[33918]: audit 2023-08-28T02:19:34.066455+0000 mon.c (mon.2) 3085 : audit [DBG] from='client.? 172.21.15.6:0/892927665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:19:34.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:34.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:19:34.714Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:34.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:34 smithi006 bash[27292]: audit 2023-08-28T02:19:34.066455+0000 mon.c (mon.2) 3085 : audit [DBG] from='client.? 172.21.15.6:0/892927665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:35.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:34 smithi130 bash[27579]: audit 2023-08-28T02:19:34.066455+0000 mon.c (mon.2) 3085 : audit [DBG] from='client.? 172.21.15.6:0/892927665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:35.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:35 smithi130 bash[51302]: debug 2023-08-28T02:19:35.497+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:35.501880+0000) 2023-08-28T02:19:36.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:35 smithi006 bash[27292]: cluster 2023-08-28T02:19:35.447605+0000 mgr.y (mgr.14901) 7829 : cluster [DBG] pgmap v7836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:36.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:35 smithi006 bash[33918]: cluster 2023-08-28T02:19:35.447605+0000 mgr.y (mgr.14901) 7829 : cluster [DBG] pgmap v7836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:36.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:35 smithi130 bash[27579]: cluster 2023-08-28T02:19:35.447605+0000 mgr.y (mgr.14901) 7829 : cluster [DBG] pgmap v7836: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:36.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:36 smithi130 bash[51302]: debug 2023-08-28T02:19:36.497+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:36.502161+0000) 2023-08-28T02:19:37.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:36 smithi006 bash[33918]: audit 2023-08-28T02:19:36.484771+0000 mon.c (mon.2) 3086 : audit [DBG] from='client.? 172.21.15.6:0/767653420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:37.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:36 smithi006 bash[27292]: audit 2023-08-28T02:19:36.484771+0000 mon.c (mon.2) 3086 : audit [DBG] from='client.? 172.21.15.6:0/767653420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:37.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:36 smithi130 bash[27579]: audit 2023-08-28T02:19:36.484771+0000 mon.c (mon.2) 3086 : audit [DBG] from='client.? 172.21.15.6:0/767653420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:37.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:37 smithi130 bash[51302]: debug 2023-08-28T02:19:37.497+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:37.502465+0000) 2023-08-28T02:19:38.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:37 smithi006 bash[27292]: cluster 2023-08-28T02:19:37.448301+0000 mgr.y (mgr.14901) 7830 : cluster [DBG] pgmap v7837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:38.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:37 smithi006 bash[33918]: cluster 2023-08-28T02:19:37.448301+0000 mgr.y (mgr.14901) 7830 : cluster [DBG] pgmap v7837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:38.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:37 smithi130 bash[27579]: cluster 2023-08-28T02:19:37.448301+0000 mgr.y (mgr.14901) 7830 : cluster [DBG] pgmap v7837: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:38.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:38 smithi130 bash[51302]: debug 2023-08-28T02:19:38.497+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:38.502759+0000) 2023-08-28T02:19:39.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:38 smithi006 bash[33918]: audit 2023-08-28T02:19:38.899410+0000 mon.a (mon.0) 5000 : audit [DBG] from='client.? 172.21.15.6:0/1452988195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:39.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:38 smithi006 bash[27292]: audit 2023-08-28T02:19:38.899410+0000 mon.a (mon.0) 5000 : audit [DBG] from='client.? 172.21.15.6:0/1452988195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:39.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:38 smithi130 bash[27579]: audit 2023-08-28T02:19:38.899410+0000 mon.a (mon.0) 5000 : audit [DBG] from='client.? 172.21.15.6:0/1452988195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:39.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:39 smithi130 bash[51302]: debug 2023-08-28T02:19:39.501+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:39.502987+0000) 2023-08-28T02:19:40.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:39 smithi006 bash[33918]: cluster 2023-08-28T02:19:39.449421+0000 mgr.y (mgr.14901) 7831 : cluster [DBG] pgmap v7838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:40.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:39 smithi006 bash[27292]: cluster 2023-08-28T02:19:39.449421+0000 mgr.y (mgr.14901) 7831 : cluster [DBG] pgmap v7838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:40.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:39 smithi130 bash[27579]: cluster 2023-08-28T02:19:39.449421+0000 mgr.y (mgr.14901) 7831 : cluster [DBG] pgmap v7838: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:40.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:40 smithi130 bash[51302]: debug 2023-08-28T02:19:40.501+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:40.503296+0000) 2023-08-28T02:19:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:19:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:19:40] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:19:41.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:41 smithi006 bash[27292]: audit 2023-08-28T02:19:41.314088+0000 mon.a (mon.0) 5001 : audit [DBG] from='client.? 172.21.15.6:0/2754756153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:41.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:41 smithi006 bash[33918]: audit 2023-08-28T02:19:41.314088+0000 mon.a (mon.0) 5001 : audit [DBG] from='client.? 172.21.15.6:0/2754756153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:19:41.271Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:19:41.273Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:41.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:41 smithi130 bash[27579]: audit 2023-08-28T02:19:41.314088+0000 mon.a (mon.0) 5001 : audit [DBG] from='client.? 172.21.15.6:0/2754756153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:41.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:41 smithi130 bash[51302]: debug 2023-08-28T02:19:41.501+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:41.503642+0000) 2023-08-28T02:19:42.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:42 smithi006 bash[33918]: cluster 2023-08-28T02:19:41.450114+0000 mgr.y (mgr.14901) 7832 : cluster [DBG] pgmap v7839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:42.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:42 smithi006 bash[27292]: cluster 2023-08-28T02:19:41.450114+0000 mgr.y (mgr.14901) 7832 : cluster [DBG] pgmap v7839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:42.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:42 smithi130 bash[27579]: cluster 2023-08-28T02:19:41.450114+0000 mgr.y (mgr.14901) 7832 : cluster [DBG] pgmap v7839: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:42.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:42 smithi130 bash[51302]: debug 2023-08-28T02:19:42.501+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:42.503903+0000) 2023-08-28T02:19:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:19:42.835Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:19:42.837Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:43.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:43 smithi130 bash[51302]: debug 2023-08-28T02:19:43.501+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:43.504158+0000) 2023-08-28T02:19:44.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:43 smithi006 bash[33918]: cluster 2023-08-28T02:19:43.450709+0000 mgr.y (mgr.14901) 7833 : cluster [DBG] pgmap v7840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:44.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:43 smithi006 bash[33918]: audit 2023-08-28T02:19:43.725696+0000 mon.a (mon.0) 5002 : audit [DBG] from='client.? 172.21.15.6:0/4202314310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:44.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:43 smithi006 bash[27292]: cluster 2023-08-28T02:19:43.450709+0000 mgr.y (mgr.14901) 7833 : cluster [DBG] pgmap v7840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:44.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:43 smithi006 bash[27292]: audit 2023-08-28T02:19:43.725696+0000 mon.a (mon.0) 5002 : audit [DBG] from='client.? 172.21.15.6:0/4202314310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:44.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:43 smithi130 bash[27579]: cluster 2023-08-28T02:19:43.450709+0000 mgr.y (mgr.14901) 7833 : cluster [DBG] pgmap v7840: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:44.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:43 smithi130 bash[27579]: audit 2023-08-28T02:19:43.725696+0000 mon.a (mon.0) 5002 : audit [DBG] from='client.? 172.21.15.6:0/4202314310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:44.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:44 smithi130 bash[51302]: debug 2023-08-28T02:19:44.501+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:44.504499+0000) 2023-08-28T02:19:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:19:44.713Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:19:44.714Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:45.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:45 smithi130 bash[51302]: debug 2023-08-28T02:19:45.501+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:45.504763+0000) 2023-08-28T02:19:46.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:46 smithi130 bash[51302]: debug 2023-08-28T02:19:46.501+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:46.505022+0000) 2023-08-28T02:19:46.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:46 smithi130 bash[27579]: cluster 2023-08-28T02:19:45.451802+0000 mgr.y (mgr.14901) 7834 : cluster [DBG] pgmap v7841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:46.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:46 smithi130 bash[27579]: audit 2023-08-28T02:19:46.146427+0000 mon.c (mon.2) 3087 : audit [DBG] from='client.? 172.21.15.6:0/1889451584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:46.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:46 smithi006 bash[33918]: cluster 2023-08-28T02:19:45.451802+0000 mgr.y (mgr.14901) 7834 : cluster [DBG] pgmap v7841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:46.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:46 smithi006 bash[33918]: audit 2023-08-28T02:19:46.146427+0000 mon.c (mon.2) 3087 : audit [DBG] from='client.? 172.21.15.6:0/1889451584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:46.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:46 smithi006 bash[27292]: cluster 2023-08-28T02:19:45.451802+0000 mgr.y (mgr.14901) 7834 : cluster [DBG] pgmap v7841: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:46.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:46 smithi006 bash[27292]: audit 2023-08-28T02:19:46.146427+0000 mon.c (mon.2) 3087 : audit [DBG] from='client.? 172.21.15.6:0/1889451584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:47.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:47 smithi130 bash[51302]: debug 2023-08-28T02:19:47.500+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:47.505311+0000) 2023-08-28T02:19:48.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:48 smithi130 bash[27579]: cluster 2023-08-28T02:19:47.452469+0000 mgr.y (mgr.14901) 7835 : cluster [DBG] pgmap v7842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:48.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:48 smithi130 bash[51302]: debug 2023-08-28T02:19:48.500+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:48.505556+0000) 2023-08-28T02:19:48.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:48 smithi006 bash[33918]: cluster 2023-08-28T02:19:47.452469+0000 mgr.y (mgr.14901) 7835 : cluster [DBG] pgmap v7842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:48.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:48 smithi006 bash[27292]: cluster 2023-08-28T02:19:47.452469+0000 mgr.y (mgr.14901) 7835 : cluster [DBG] pgmap v7842: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:49.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:49 smithi130 bash[27579]: audit 2023-08-28T02:19:48.557137+0000 mon.c (mon.2) 3088 : audit [DBG] from='client.? 172.21.15.6:0/703158711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:49.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:49 smithi130 bash[51302]: debug 2023-08-28T02:19:49.500+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:49.505863+0000) 2023-08-28T02:19:49.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:49 smithi006 bash[27292]: audit 2023-08-28T02:19:48.557137+0000 mon.c (mon.2) 3088 : audit [DBG] from='client.? 172.21.15.6:0/703158711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:49.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:49 smithi006 bash[33918]: audit 2023-08-28T02:19:48.557137+0000 mon.c (mon.2) 3088 : audit [DBG] from='client.? 172.21.15.6:0/703158711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:50.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:50 smithi130 bash[27579]: cluster 2023-08-28T02:19:49.453720+0000 mgr.y (mgr.14901) 7836 : cluster [DBG] pgmap v7843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:50.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:50 smithi130 bash[51302]: debug 2023-08-28T02:19:50.500+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:50.506157+0000) 2023-08-28T02:19:50.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:19:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:19:50] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T02:19:50.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:50 smithi006 bash[33918]: cluster 2023-08-28T02:19:49.453720+0000 mgr.y (mgr.14901) 7836 : cluster [DBG] pgmap v7843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:50.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:50 smithi006 bash[27292]: cluster 2023-08-28T02:19:49.453720+0000 mgr.y (mgr.14901) 7836 : cluster [DBG] pgmap v7843: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:51.532 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:19:51.272Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:51.532 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:19:51.274Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:51.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:51 smithi130 bash[27579]: audit 2023-08-28T02:19:50.980773+0000 mon.a (mon.0) 5003 : audit [DBG] from='client.? 172.21.15.6:0/2524099919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:51.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:51 smithi130 bash[51302]: debug 2023-08-28T02:19:51.500+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:51.506485+0000) 2023-08-28T02:19:51.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:51 smithi006 bash[33918]: audit 2023-08-28T02:19:50.980773+0000 mon.a (mon.0) 5003 : audit [DBG] from='client.? 172.21.15.6:0/2524099919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:51.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:51 smithi006 bash[27292]: audit 2023-08-28T02:19:50.980773+0000 mon.a (mon.0) 5003 : audit [DBG] from='client.? 172.21.15.6:0/2524099919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:52.836 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:52 smithi006 bash[27292]: cluster 2023-08-28T02:19:51.454427+0000 mgr.y (mgr.14901) 7837 : cluster [DBG] pgmap v7844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:52.836 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:52 smithi006 bash[33918]: cluster 2023-08-28T02:19:51.454427+0000 mgr.y (mgr.14901) 7837 : cluster [DBG] pgmap v7844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:52.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:52 smithi130 bash[27579]: cluster 2023-08-28T02:19:51.454427+0000 mgr.y (mgr.14901) 7837 : cluster [DBG] pgmap v7844: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:52.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:52 smithi130 bash[51302]: debug 2023-08-28T02:19:52.500+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:52.506714+0000) 2023-08-28T02:19:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:19:52.835Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:19:52.837Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:53.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:53 smithi130 bash[51302]: debug 2023-08-28T02:19:53.504+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:53.507010+0000) 2023-08-28T02:19:54.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:53 smithi006 bash[33918]: audit 2023-08-28T02:19:53.389751+0000 mon.a (mon.0) 5004 : audit [DBG] from='client.? 172.21.15.6:0/1150903073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:54.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:53 smithi006 bash[33918]: cluster 2023-08-28T02:19:53.455101+0000 mgr.y (mgr.14901) 7838 : cluster [DBG] pgmap v7845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:54.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:53 smithi006 bash[33918]: audit 2023-08-28T02:19:53.457497+0000 mon.b (mon.1) 861 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:19:54.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:53 smithi006 bash[27292]: audit 2023-08-28T02:19:53.389751+0000 mon.a (mon.0) 5004 : audit [DBG] from='client.? 172.21.15.6:0/1150903073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:54.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:53 smithi006 bash[27292]: cluster 2023-08-28T02:19:53.455101+0000 mgr.y (mgr.14901) 7838 : cluster [DBG] pgmap v7845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:54.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:53 smithi006 bash[27292]: audit 2023-08-28T02:19:53.457497+0000 mon.b (mon.1) 861 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:19:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:53 smithi130 bash[27579]: audit 2023-08-28T02:19:53.389751+0000 mon.a (mon.0) 5004 : audit [DBG] from='client.? 172.21.15.6:0/1150903073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:53 smithi130 bash[27579]: cluster 2023-08-28T02:19:53.455101+0000 mgr.y (mgr.14901) 7838 : cluster [DBG] pgmap v7845: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:53 smithi130 bash[27579]: audit 2023-08-28T02:19:53.457497+0000 mon.b (mon.1) 861 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:19:54.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:54 smithi130 bash[51302]: debug 2023-08-28T02:19:54.504+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:54.507307+0000) 2023-08-28T02:19:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:19:54.713Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:19:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:19:54.716Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:19:55.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:55 smithi130 bash[51302]: debug 2023-08-28T02:19:55.504+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:55.507605+0000) 2023-08-28T02:19:56.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:56 smithi130 bash[51302]: debug 2023-08-28T02:19:56.504+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:56.507898+0000) 2023-08-28T02:19:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:56 smithi130 bash[27579]: cluster 2023-08-28T02:19:55.456216+0000 mgr.y (mgr.14901) 7839 : cluster [DBG] pgmap v7846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:56.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:56 smithi130 bash[27579]: audit 2023-08-28T02:19:55.816726+0000 mon.a (mon.0) 5005 : audit [DBG] from='client.? 172.21.15.6:0/3320304247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:56.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:56 smithi006 bash[33918]: cluster 2023-08-28T02:19:55.456216+0000 mgr.y (mgr.14901) 7839 : cluster [DBG] pgmap v7846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:56.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:56 smithi006 bash[33918]: audit 2023-08-28T02:19:55.816726+0000 mon.a (mon.0) 5005 : audit [DBG] from='client.? 172.21.15.6:0/3320304247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:56.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:56 smithi006 bash[27292]: cluster 2023-08-28T02:19:55.456216+0000 mgr.y (mgr.14901) 7839 : cluster [DBG] pgmap v7846: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:56.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:56 smithi006 bash[27292]: audit 2023-08-28T02:19:55.816726+0000 mon.a (mon.0) 5005 : audit [DBG] from='client.? 172.21.15.6:0/3320304247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:57.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:57 smithi130 bash[51302]: debug 2023-08-28T02:19:57.504+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:57.508171+0000) 2023-08-28T02:19:58.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:58 smithi130 bash[27579]: cluster 2023-08-28T02:19:57.456959+0000 mgr.y (mgr.14901) 7840 : cluster [DBG] pgmap v7847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:58.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:19:58 smithi130 bash[27579]: audit 2023-08-28T02:19:58.227544+0000 mon.c (mon.2) 3089 : audit [DBG] from='client.? 172.21.15.6:0/3996436965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:58.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:58 smithi130 bash[51302]: debug 2023-08-28T02:19:58.504+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:58.508440+0000) 2023-08-28T02:19:58.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:58 smithi006 bash[33918]: cluster 2023-08-28T02:19:57.456959+0000 mgr.y (mgr.14901) 7840 : cluster [DBG] pgmap v7847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:58.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:19:58 smithi006 bash[33918]: audit 2023-08-28T02:19:58.227544+0000 mon.c (mon.2) 3089 : audit [DBG] from='client.? 172.21.15.6:0/3996436965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:58.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:58 smithi006 bash[27292]: cluster 2023-08-28T02:19:57.456959+0000 mgr.y (mgr.14901) 7840 : cluster [DBG] pgmap v7847: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:19:58.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:19:58 smithi006 bash[27292]: audit 2023-08-28T02:19:58.227544+0000 mon.c (mon.2) 3089 : audit [DBG] from='client.? 172.21.15.6:0/3996436965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:59.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:19:59 smithi130 bash[51302]: debug 2023-08-28T02:19:59.504+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:59.508655+0000) 2023-08-28T02:20:00.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:00 smithi006 bash[33918]: audit 2023-08-28T02:19:59.376376+0000 mon.a (mon.0) 5006 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:20:00.717 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:00 smithi006 bash[33918]: cluster 2023-08-28T02:19:59.458036+0000 mgr.y (mgr.14901) 7841 : cluster [DBG] pgmap v7848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:00 smithi006 bash[33918]: audit 2023-08-28T02:19:59.796140+0000 mon.a (mon.0) 5007 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:20:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:00 smithi006 bash[33918]: cluster 2023-08-28T02:20:00.000131+0000 mon.a (mon.0) 5008 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:20:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:00 smithi006 bash[33918]: audit 2023-08-28T02:20:00.087126+0000 mon.b (mon.1) 862 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:20:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:00 smithi006 bash[33918]: audit 2023-08-28T02:20:00.088917+0000 mon.b (mon.1) 863 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:20:00.718 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:00 smithi006 bash[33918]: audit 2023-08-28T02:20:00.095262+0000 mon.a (mon.0) 5009 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:20:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:00 smithi006 bash[27292]: audit 2023-08-28T02:19:59.376376+0000 mon.a (mon.0) 5006 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:20:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:00 smithi006 bash[27292]: cluster 2023-08-28T02:19:59.458036+0000 mgr.y (mgr.14901) 7841 : cluster [DBG] pgmap v7848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:00.718 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:00 smithi006 bash[27292]: audit 2023-08-28T02:19:59.796140+0000 mon.a (mon.0) 5007 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:20:00.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:00 smithi006 bash[27292]: cluster 2023-08-28T02:20:00.000131+0000 mon.a (mon.0) 5008 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:20:00.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:00 smithi006 bash[27292]: audit 2023-08-28T02:20:00.087126+0000 mon.b (mon.1) 862 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:20:00.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:00 smithi006 bash[27292]: audit 2023-08-28T02:20:00.088917+0000 mon.b (mon.1) 863 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:20:00.719 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:00 smithi006 bash[27292]: audit 2023-08-28T02:20:00.095262+0000 mon.a (mon.0) 5009 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:20:00.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:00 smithi130 bash[51302]: debug 2023-08-28T02:20:00.508+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:00.508903+0000) 2023-08-28T02:20:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:00 smithi130 bash[27579]: audit 2023-08-28T02:19:59.376376+0000 mon.a (mon.0) 5006 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:20:00.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:00 smithi130 bash[27579]: cluster 2023-08-28T02:19:59.458036+0000 mgr.y (mgr.14901) 7841 : cluster [DBG] pgmap v7848: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:00.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:00 smithi130 bash[27579]: audit 2023-08-28T02:19:59.796140+0000 mon.a (mon.0) 5007 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:20:00.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:00 smithi130 bash[27579]: cluster 2023-08-28T02:20:00.000131+0000 mon.a (mon.0) 5008 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:20:00.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:00 smithi130 bash[27579]: audit 2023-08-28T02:20:00.087126+0000 mon.b (mon.1) 862 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:20:00.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:00 smithi130 bash[27579]: audit 2023-08-28T02:20:00.088917+0000 mon.b (mon.1) 863 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:20:00.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:00 smithi130 bash[27579]: audit 2023-08-28T02:20:00.095262+0000 mon.a (mon.0) 5009 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:20:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:20:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:20:00] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:20:01.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:01 smithi006 bash[27292]: audit 2023-08-28T02:20:00.642953+0000 mon.a (mon.0) 5010 : audit [DBG] from='client.? 172.21.15.6:0/603169506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:01.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:01 smithi006 bash[33918]: audit 2023-08-28T02:20:00.642953+0000 mon.a (mon.0) 5010 : audit [DBG] from='client.? 172.21.15.6:0/603169506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:20:01.273Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:20:01.275Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:01.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:01 smithi130 bash[27579]: audit 2023-08-28T02:20:00.642953+0000 mon.a (mon.0) 5010 : audit [DBG] from='client.? 172.21.15.6:0/603169506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:01.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:01 smithi130 bash[51302]: debug 2023-08-28T02:20:01.504+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:01.509201+0000) 2023-08-28T02:20:02.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:02 smithi006 bash[33918]: cluster 2023-08-28T02:20:01.458814+0000 mgr.y (mgr.14901) 7842 : cluster [DBG] pgmap v7849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:02.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:02 smithi006 bash[27292]: cluster 2023-08-28T02:20:01.458814+0000 mgr.y (mgr.14901) 7842 : cluster [DBG] pgmap v7849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:02 smithi130 bash[27579]: cluster 2023-08-28T02:20:01.458814+0000 mgr.y (mgr.14901) 7842 : cluster [DBG] pgmap v7849: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:02.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:02 smithi130 bash[51302]: debug 2023-08-28T02:20:02.504+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:02.509482+0000) 2023-08-28T02:20:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:20:02.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:20:02.838Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:03.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:03 smithi130 bash[51302]: debug 2023-08-28T02:20:03.504+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:03.509760+0000) 2023-08-28T02:20:04.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:03 smithi006 bash[27292]: audit 2023-08-28T02:20:03.069491+0000 mon.a (mon.0) 5011 : audit [DBG] from='client.? 172.21.15.6:0/1567980422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:04.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:03 smithi006 bash[27292]: cluster 2023-08-28T02:20:03.459664+0000 mgr.y (mgr.14901) 7843 : cluster [DBG] pgmap v7850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:04.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:03 smithi006 bash[33918]: audit 2023-08-28T02:20:03.069491+0000 mon.a (mon.0) 5011 : audit [DBG] from='client.? 172.21.15.6:0/1567980422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:04.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:03 smithi006 bash[33918]: cluster 2023-08-28T02:20:03.459664+0000 mgr.y (mgr.14901) 7843 : cluster [DBG] pgmap v7850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:03 smithi130 bash[27579]: audit 2023-08-28T02:20:03.069491+0000 mon.a (mon.0) 5011 : audit [DBG] from='client.? 172.21.15.6:0/1567980422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:03 smithi130 bash[27579]: cluster 2023-08-28T02:20:03.459664+0000 mgr.y (mgr.14901) 7843 : cluster [DBG] pgmap v7850: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:04.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:04 smithi130 bash[51302]: debug 2023-08-28T02:20:04.504+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:04.510024+0000) 2023-08-28T02:20:05.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:20:04.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:05.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:20:04.717Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:05.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:05 smithi130 bash[27579]: audit 2023-08-28T02:20:05.489388+0000 mon.a (mon.0) 5012 : audit [DBG] from='client.? 172.21.15.6:0/1998432007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:05.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:05 smithi130 bash[51302]: debug 2023-08-28T02:20:05.504+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:05.510300+0000) 2023-08-28T02:20:05.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:05 smithi006 bash[27292]: audit 2023-08-28T02:20:05.489388+0000 mon.a (mon.0) 5012 : audit [DBG] from='client.? 172.21.15.6:0/1998432007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:05.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:05 smithi006 bash[33918]: audit 2023-08-28T02:20:05.489388+0000 mon.a (mon.0) 5012 : audit [DBG] from='client.? 172.21.15.6:0/1998432007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:06.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:06 smithi130 bash[27579]: cluster 2023-08-28T02:20:05.460949+0000 mgr.y (mgr.14901) 7844 : cluster [DBG] pgmap v7851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:06.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:06 smithi130 bash[51302]: debug 2023-08-28T02:20:06.504+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:06.510522+0000) 2023-08-28T02:20:06.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:06 smithi006 bash[27292]: cluster 2023-08-28T02:20:05.460949+0000 mgr.y (mgr.14901) 7844 : cluster [DBG] pgmap v7851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:06.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:06 smithi006 bash[33918]: cluster 2023-08-28T02:20:05.460949+0000 mgr.y (mgr.14901) 7844 : cluster [DBG] pgmap v7851: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:07.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:07 smithi130 bash[51302]: debug 2023-08-28T02:20:07.504+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:07.510814+0000) 2023-08-28T02:20:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:08 smithi130 bash[27579]: cluster 2023-08-28T02:20:07.461726+0000 mgr.y (mgr.14901) 7845 : cluster [DBG] pgmap v7852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:08 smithi130 bash[27579]: audit 2023-08-28T02:20:07.917201+0000 mon.c (mon.2) 3090 : audit [DBG] from='client.? 172.21.15.6:0/1386509198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:08.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:08 smithi130 bash[51302]: debug 2023-08-28T02:20:08.508+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:08.511142+0000) 2023-08-28T02:20:08.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:08 smithi006 bash[27292]: cluster 2023-08-28T02:20:07.461726+0000 mgr.y (mgr.14901) 7845 : cluster [DBG] pgmap v7852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:08.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:08 smithi006 bash[27292]: audit 2023-08-28T02:20:07.917201+0000 mon.c (mon.2) 3090 : audit [DBG] from='client.? 172.21.15.6:0/1386509198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:08.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:08 smithi006 bash[33918]: cluster 2023-08-28T02:20:07.461726+0000 mgr.y (mgr.14901) 7845 : cluster [DBG] pgmap v7852: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:08.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:08 smithi006 bash[33918]: audit 2023-08-28T02:20:07.917201+0000 mon.c (mon.2) 3090 : audit [DBG] from='client.? 172.21.15.6:0/1386509198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:09.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:09 smithi130 bash[51302]: debug 2023-08-28T02:20:09.508+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:09.511420+0000) 2023-08-28T02:20:10.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:10 smithi130 bash[27579]: cluster 2023-08-28T02:20:09.462852+0000 mgr.y (mgr.14901) 7846 : cluster [DBG] pgmap v7853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:10.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:10 smithi130 bash[27579]: audit 2023-08-28T02:20:10.329491+0000 mon.c (mon.2) 3091 : audit [DBG] from='client.? 172.21.15.6:0/2691468848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:10.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:10 smithi130 bash[51302]: debug 2023-08-28T02:20:10.508+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:10.511704+0000) 2023-08-28T02:20:10.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:20:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:20:10] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:20:10.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:10 smithi006 bash[27292]: cluster 2023-08-28T02:20:09.462852+0000 mgr.y (mgr.14901) 7846 : cluster [DBG] pgmap v7853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:10.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:10 smithi006 bash[27292]: audit 2023-08-28T02:20:10.329491+0000 mon.c (mon.2) 3091 : audit [DBG] from='client.? 172.21.15.6:0/2691468848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:10.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:10 smithi006 bash[33918]: cluster 2023-08-28T02:20:09.462852+0000 mgr.y (mgr.14901) 7846 : cluster [DBG] pgmap v7853: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:10.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:10 smithi006 bash[33918]: audit 2023-08-28T02:20:10.329491+0000 mon.c (mon.2) 3091 : audit [DBG] from='client.? 172.21.15.6:0/2691468848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:20:11.274Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:20:11.276Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:11.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:11 smithi130 bash[51302]: debug 2023-08-28T02:20:11.508+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:11.512009+0000) 2023-08-28T02:20:12.837 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:12 smithi006 bash[27292]: cluster 2023-08-28T02:20:11.463667+0000 mgr.y (mgr.14901) 7847 : cluster [DBG] pgmap v7854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:12.838 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:12 smithi006 bash[33918]: cluster 2023-08-28T02:20:11.463667+0000 mgr.y (mgr.14901) 7847 : cluster [DBG] pgmap v7854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:12 smithi130 bash[27579]: cluster 2023-08-28T02:20:11.463667+0000 mgr.y (mgr.14901) 7847 : cluster [DBG] pgmap v7854: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:12.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:12 smithi130 bash[51302]: debug 2023-08-28T02:20:12.508+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:12.512219+0000) 2023-08-28T02:20:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:20:12.837Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:20:12.839Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:13.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:13 smithi130 bash[51302]: debug 2023-08-28T02:20:13.508+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:13.512534+0000) 2023-08-28T02:20:13.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:13 smithi130 bash[27579]: audit 2023-08-28T02:20:12.749777+0000 mon.c (mon.2) 3092 : audit [DBG] from='client.? 172.21.15.6:0/4135488655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:13.967 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:13 smithi006 bash[27292]: audit 2023-08-28T02:20:12.749777+0000 mon.c (mon.2) 3092 : audit [DBG] from='client.? 172.21.15.6:0/4135488655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:13.967 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:13 smithi006 bash[33918]: audit 2023-08-28T02:20:12.749777+0000 mon.c (mon.2) 3092 : audit [DBG] from='client.? 172.21.15.6:0/4135488655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:14.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:14 smithi130 bash[27579]: cluster 2023-08-28T02:20:13.464388+0000 mgr.y (mgr.14901) 7848 : cluster [DBG] pgmap v7855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:14.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:14 smithi130 bash[51302]: debug 2023-08-28T02:20:14.512+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:14.512816+0000) 2023-08-28T02:20:14.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:14 smithi006 bash[33918]: cluster 2023-08-28T02:20:13.464388+0000 mgr.y (mgr.14901) 7848 : cluster [DBG] pgmap v7855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:20:14.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:20:14.717Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:14.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:14 smithi006 bash[27292]: cluster 2023-08-28T02:20:13.464388+0000 mgr.y (mgr.14901) 7848 : cluster [DBG] pgmap v7855: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:15.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:15 smithi130 bash[27579]: audit 2023-08-28T02:20:15.170351+0000 mon.c (mon.2) 3093 : audit [DBG] from='client.? 172.21.15.6:0/1786523669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:15.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:15 smithi130 bash[51302]: debug 2023-08-28T02:20:15.512+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:15.513123+0000) 2023-08-28T02:20:15.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:15 smithi006 bash[27292]: audit 2023-08-28T02:20:15.170351+0000 mon.c (mon.2) 3093 : audit [DBG] from='client.? 172.21.15.6:0/1786523669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:15.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:15 smithi006 bash[33918]: audit 2023-08-28T02:20:15.170351+0000 mon.c (mon.2) 3093 : audit [DBG] from='client.? 172.21.15.6:0/1786523669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:16.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:16 smithi130 bash[27579]: cluster 2023-08-28T02:20:15.465563+0000 mgr.y (mgr.14901) 7849 : cluster [DBG] pgmap v7856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:16.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:16 smithi130 bash[51302]: debug 2023-08-28T02:20:16.508+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:16.513402+0000) 2023-08-28T02:20:16.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:16 smithi006 bash[27292]: cluster 2023-08-28T02:20:15.465563+0000 mgr.y (mgr.14901) 7849 : cluster [DBG] pgmap v7856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:16.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:16 smithi006 bash[33918]: cluster 2023-08-28T02:20:15.465563+0000 mgr.y (mgr.14901) 7849 : cluster [DBG] pgmap v7856: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:17.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:17 smithi130 bash[27579]: audit 2023-08-28T02:20:17.579929+0000 mon.a (mon.0) 5013 : audit [DBG] from='client.? 172.21.15.6:0/4259118547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:17.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:17 smithi130 bash[51302]: debug 2023-08-28T02:20:17.508+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:17.513690+0000) 2023-08-28T02:20:17.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:17 smithi006 bash[27292]: audit 2023-08-28T02:20:17.579929+0000 mon.a (mon.0) 5013 : audit [DBG] from='client.? 172.21.15.6:0/4259118547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:17.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:17 smithi006 bash[33918]: audit 2023-08-28T02:20:17.579929+0000 mon.a (mon.0) 5013 : audit [DBG] from='client.? 172.21.15.6:0/4259118547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:18.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:18 smithi130 bash[27579]: cluster 2023-08-28T02:20:17.466173+0000 mgr.y (mgr.14901) 7850 : cluster [DBG] pgmap v7857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:18.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:18 smithi130 bash[51302]: debug 2023-08-28T02:20:18.512+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:18.513998+0000) 2023-08-28T02:20:18.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:18 smithi006 bash[27292]: cluster 2023-08-28T02:20:17.466173+0000 mgr.y (mgr.14901) 7850 : cluster [DBG] pgmap v7857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:18.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:18 smithi006 bash[33918]: cluster 2023-08-28T02:20:17.466173+0000 mgr.y (mgr.14901) 7850 : cluster [DBG] pgmap v7857: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:19.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:19 smithi130 bash[51302]: debug 2023-08-28T02:20:19.508+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:19.514279+0000) 2023-08-28T02:20:20.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:20 smithi130 bash[51302]: debug 2023-08-28T02:20:20.512+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:20.514582+0000) 2023-08-28T02:20:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:20 smithi130 bash[27579]: cluster 2023-08-28T02:20:19.467225+0000 mgr.y (mgr.14901) 7851 : cluster [DBG] pgmap v7858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:20 smithi130 bash[27579]: audit 2023-08-28T02:20:19.994613+0000 mon.a (mon.0) 5014 : audit [DBG] from='client.? 172.21.15.6:0/3017940947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:20.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:20:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:20:20] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T02:20:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:20 smithi006 bash[27292]: cluster 2023-08-28T02:20:19.467225+0000 mgr.y (mgr.14901) 7851 : cluster [DBG] pgmap v7858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:20 smithi006 bash[27292]: audit 2023-08-28T02:20:19.994613+0000 mon.a (mon.0) 5014 : audit [DBG] from='client.? 172.21.15.6:0/3017940947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:20.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:20 smithi006 bash[33918]: cluster 2023-08-28T02:20:19.467225+0000 mgr.y (mgr.14901) 7851 : cluster [DBG] pgmap v7858: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:20.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:20 smithi006 bash[33918]: audit 2023-08-28T02:20:19.994613+0000 mon.a (mon.0) 5014 : audit [DBG] from='client.? 172.21.15.6:0/3017940947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:20:21.275Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:20:21.277Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:21.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:21 smithi130 bash[51302]: debug 2023-08-28T02:20:21.512+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:21.515002+0000) 2023-08-28T02:20:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:22 smithi130 bash[27579]: cluster 2023-08-28T02:20:21.467931+0000 mgr.y (mgr.14901) 7852 : cluster [DBG] pgmap v7859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:22 smithi130 bash[27579]: audit 2023-08-28T02:20:22.410993+0000 mon.c (mon.2) 3094 : audit [DBG] from='client.? 172.21.15.6:0/813819627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:22.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:22 smithi130 bash[51302]: debug 2023-08-28T02:20:22.512+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:22.515214+0000) 2023-08-28T02:20:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:20:22.838Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:20:22.839Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:22.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:22 smithi006 bash[27292]: cluster 2023-08-28T02:20:21.467931+0000 mgr.y (mgr.14901) 7852 : cluster [DBG] pgmap v7859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:22.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:22 smithi006 bash[27292]: audit 2023-08-28T02:20:22.410993+0000 mon.c (mon.2) 3094 : audit [DBG] from='client.? 172.21.15.6:0/813819627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:22.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:22 smithi006 bash[33918]: cluster 2023-08-28T02:20:21.467931+0000 mgr.y (mgr.14901) 7852 : cluster [DBG] pgmap v7859: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:22.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:22 smithi006 bash[33918]: audit 2023-08-28T02:20:22.410993+0000 mon.c (mon.2) 3094 : audit [DBG] from='client.? 172.21.15.6:0/813819627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:23.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:23 smithi130 bash[51302]: debug 2023-08-28T02:20:23.512+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:23.515446+0000) 2023-08-28T02:20:24.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:23 smithi006 bash[33918]: cluster 2023-08-28T02:20:23.468625+0000 mgr.y (mgr.14901) 7853 : cluster [DBG] pgmap v7860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:23 smithi006 bash[27292]: cluster 2023-08-28T02:20:23.468625+0000 mgr.y (mgr.14901) 7853 : cluster [DBG] pgmap v7860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:23 smithi130 bash[27579]: cluster 2023-08-28T02:20:23.468625+0000 mgr.y (mgr.14901) 7853 : cluster [DBG] pgmap v7860: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:24.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:24 smithi130 bash[51302]: debug 2023-08-28T02:20:24.512+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:24.515743+0000) 2023-08-28T02:20:25.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:24 smithi006 bash[33918]: audit 2023-08-28T02:20:24.827274+0000 mon.a (mon.0) 5015 : audit [DBG] from='client.? 172.21.15.6:0/2319343236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:25.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:20:24.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:25.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:20:24.717Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:25.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:24 smithi006 bash[27292]: audit 2023-08-28T02:20:24.827274+0000 mon.a (mon.0) 5015 : audit [DBG] from='client.? 172.21.15.6:0/2319343236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:25.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:24 smithi130 bash[27579]: audit 2023-08-28T02:20:24.827274+0000 mon.a (mon.0) 5015 : audit [DBG] from='client.? 172.21.15.6:0/2319343236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:25.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:25 smithi130 bash[51302]: debug 2023-08-28T02:20:25.512+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:25.515989+0000) 2023-08-28T02:20:26.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:25 smithi006 bash[27292]: cluster 2023-08-28T02:20:25.469782+0000 mgr.y (mgr.14901) 7854 : cluster [DBG] pgmap v7861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:26.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:25 smithi006 bash[33918]: cluster 2023-08-28T02:20:25.469782+0000 mgr.y (mgr.14901) 7854 : cluster [DBG] pgmap v7861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:26.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:25 smithi130 bash[27579]: cluster 2023-08-28T02:20:25.469782+0000 mgr.y (mgr.14901) 7854 : cluster [DBG] pgmap v7861: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:26.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:26 smithi130 bash[51302]: debug 2023-08-28T02:20:26.512+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:26.516219+0000) 2023-08-28T02:20:27.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:27 smithi130 bash[27579]: audit 2023-08-28T02:20:27.250081+0000 mon.a (mon.0) 5016 : audit [DBG] from='client.? 172.21.15.6:0/2094616087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:27.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:27 smithi130 bash[51302]: debug 2023-08-28T02:20:27.512+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:27.516477+0000) 2023-08-28T02:20:27.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:27 smithi006 bash[33918]: audit 2023-08-28T02:20:27.250081+0000 mon.a (mon.0) 5016 : audit [DBG] from='client.? 172.21.15.6:0/2094616087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:27.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:27 smithi006 bash[27292]: audit 2023-08-28T02:20:27.250081+0000 mon.a (mon.0) 5016 : audit [DBG] from='client.? 172.21.15.6:0/2094616087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:28 smithi130 bash[27579]: cluster 2023-08-28T02:20:27.470437+0000 mgr.y (mgr.14901) 7855 : cluster [DBG] pgmap v7862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:28.615 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:28 smithi130 bash[51302]: debug 2023-08-28T02:20:28.516+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:28.516752+0000) 2023-08-28T02:20:28.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:28 smithi006 bash[33918]: cluster 2023-08-28T02:20:27.470437+0000 mgr.y (mgr.14901) 7855 : cluster [DBG] pgmap v7862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:28.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:28 smithi006 bash[27292]: cluster 2023-08-28T02:20:27.470437+0000 mgr.y (mgr.14901) 7855 : cluster [DBG] pgmap v7862: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:29.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:29 smithi130 bash[51302]: debug 2023-08-28T02:20:29.512+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:29.517037+0000) 2023-08-28T02:20:30.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:30 smithi130 bash[51302]: debug 2023-08-28T02:20:30.512+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:30.517295+0000) 2023-08-28T02:20:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:30 smithi130 bash[27579]: cluster 2023-08-28T02:20:29.471541+0000 mgr.y (mgr.14901) 7856 : cluster [DBG] pgmap v7863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:30.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:30 smithi130 bash[27579]: audit 2023-08-28T02:20:29.665747+0000 mon.a (mon.0) 5017 : audit [DBG] from='client.? 172.21.15.6:0/1971788467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:30.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:20:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:20:30] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:20:30.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:30 smithi006 bash[27292]: cluster 2023-08-28T02:20:29.471541+0000 mgr.y (mgr.14901) 7856 : cluster [DBG] pgmap v7863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:30.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:30 smithi006 bash[27292]: audit 2023-08-28T02:20:29.665747+0000 mon.a (mon.0) 5017 : audit [DBG] from='client.? 172.21.15.6:0/1971788467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:30.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:30 smithi006 bash[33918]: cluster 2023-08-28T02:20:29.471541+0000 mgr.y (mgr.14901) 7856 : cluster [DBG] pgmap v7863: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:30.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:30 smithi006 bash[33918]: audit 2023-08-28T02:20:29.665747+0000 mon.a (mon.0) 5017 : audit [DBG] from='client.? 172.21.15.6:0/1971788467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:20:31.276Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:20:31.278Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:31.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:31 smithi130 bash[51302]: debug 2023-08-28T02:20:31.512+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:31.517580+0000) 2023-08-28T02:20:32.838 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:32 smithi006 bash[27292]: cluster 2023-08-28T02:20:31.472222+0000 mgr.y (mgr.14901) 7857 : cluster [DBG] pgmap v7864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:32.838 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:32 smithi006 bash[27292]: audit 2023-08-28T02:20:32.075688+0000 mon.a (mon.0) 5018 : audit [DBG] from='client.? 172.21.15.6:0/462991415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:32.839 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:32 smithi006 bash[33918]: cluster 2023-08-28T02:20:31.472222+0000 mgr.y (mgr.14901) 7857 : cluster [DBG] pgmap v7864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:32.839 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:32 smithi006 bash[33918]: audit 2023-08-28T02:20:32.075688+0000 mon.a (mon.0) 5018 : audit [DBG] from='client.? 172.21.15.6:0/462991415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:32.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:32 smithi130 bash[51302]: debug 2023-08-28T02:20:32.512+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:32.517937+0000) 2023-08-28T02:20:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:32 smithi130 bash[27579]: cluster 2023-08-28T02:20:31.472222+0000 mgr.y (mgr.14901) 7857 : cluster [DBG] pgmap v7864: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:32.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:32 smithi130 bash[27579]: audit 2023-08-28T02:20:32.075688+0000 mon.a (mon.0) 5018 : audit [DBG] from='client.? 172.21.15.6:0/462991415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:20:32.838Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:20:32.840Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:33.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:33 smithi130 bash[51302]: debug 2023-08-28T02:20:33.512+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:33.518173+0000) 2023-08-28T02:20:34.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:33 smithi006 bash[27292]: cluster 2023-08-28T02:20:33.472920+0000 mgr.y (mgr.14901) 7858 : cluster [DBG] pgmap v7865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:34.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:33 smithi006 bash[33918]: cluster 2023-08-28T02:20:33.472920+0000 mgr.y (mgr.14901) 7858 : cluster [DBG] pgmap v7865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:33 smithi130 bash[27579]: cluster 2023-08-28T02:20:33.472920+0000 mgr.y (mgr.14901) 7858 : cluster [DBG] pgmap v7865: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:34.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:34 smithi130 bash[51302]: debug 2023-08-28T02:20:34.512+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:34.518510+0000) 2023-08-28T02:20:35.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:34 smithi006 bash[33918]: audit 2023-08-28T02:20:34.502680+0000 mon.a (mon.0) 5019 : audit [DBG] from='client.? 172.21.15.6:0/960721499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:35.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:34 smithi006 bash[27292]: audit 2023-08-28T02:20:34.502680+0000 mon.a (mon.0) 5019 : audit [DBG] from='client.? 172.21.15.6:0/960721499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:35.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:20:34.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:35.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:20:34.718Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:35.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:34 smithi130 bash[27579]: audit 2023-08-28T02:20:34.502680+0000 mon.a (mon.0) 5019 : audit [DBG] from='client.? 172.21.15.6:0/960721499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:35.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:35 smithi130 bash[51302]: debug 2023-08-28T02:20:35.516+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:35.518833+0000) 2023-08-28T02:20:36.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:35 smithi006 bash[27292]: cluster 2023-08-28T02:20:35.474051+0000 mgr.y (mgr.14901) 7859 : cluster [DBG] pgmap v7866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:36.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:35 smithi006 bash[33918]: cluster 2023-08-28T02:20:35.474051+0000 mgr.y (mgr.14901) 7859 : cluster [DBG] pgmap v7866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:36.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:35 smithi130 bash[27579]: cluster 2023-08-28T02:20:35.474051+0000 mgr.y (mgr.14901) 7859 : cluster [DBG] pgmap v7866: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:36.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:36 smithi130 bash[51302]: debug 2023-08-28T02:20:36.516+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:36.519232+0000) 2023-08-28T02:20:37.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:36 smithi006 bash[33918]: audit 2023-08-28T02:20:36.914229+0000 mon.a (mon.0) 5020 : audit [DBG] from='client.? 172.21.15.6:0/333346636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:37.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:36 smithi006 bash[27292]: audit 2023-08-28T02:20:36.914229+0000 mon.a (mon.0) 5020 : audit [DBG] from='client.? 172.21.15.6:0/333346636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:37.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:36 smithi130 bash[27579]: audit 2023-08-28T02:20:36.914229+0000 mon.a (mon.0) 5020 : audit [DBG] from='client.? 172.21.15.6:0/333346636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:37.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:37 smithi130 bash[51302]: debug 2023-08-28T02:20:37.516+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:37.519501+0000) 2023-08-28T02:20:38.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:37 smithi006 bash[27292]: cluster 2023-08-28T02:20:37.474849+0000 mgr.y (mgr.14901) 7860 : cluster [DBG] pgmap v7867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:38.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:37 smithi006 bash[33918]: cluster 2023-08-28T02:20:37.474849+0000 mgr.y (mgr.14901) 7860 : cluster [DBG] pgmap v7867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:38.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:37 smithi130 bash[27579]: cluster 2023-08-28T02:20:37.474849+0000 mgr.y (mgr.14901) 7860 : cluster [DBG] pgmap v7867: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:38.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:38 smithi130 bash[51302]: debug 2023-08-28T02:20:38.516+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:38.519833+0000) 2023-08-28T02:20:39.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:39 smithi006 bash[33918]: audit 2023-08-28T02:20:39.333331+0000 mon.a (mon.0) 5021 : audit [DBG] from='client.? 172.21.15.6:0/1821446809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:39.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:39 smithi006 bash[27292]: audit 2023-08-28T02:20:39.333331+0000 mon.a (mon.0) 5021 : audit [DBG] from='client.? 172.21.15.6:0/1821446809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:39.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:39 smithi130 bash[27579]: audit 2023-08-28T02:20:39.333331+0000 mon.a (mon.0) 5021 : audit [DBG] from='client.? 172.21.15.6:0/1821446809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:39.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:39 smithi130 bash[51302]: debug 2023-08-28T02:20:39.516+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:39.520128+0000) 2023-08-28T02:20:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:40 smithi006 bash[27292]: cluster 2023-08-28T02:20:39.476001+0000 mgr.y (mgr.14901) 7861 : cluster [DBG] pgmap v7868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:40 smithi006 bash[33918]: cluster 2023-08-28T02:20:39.476001+0000 mgr.y (mgr.14901) 7861 : cluster [DBG] pgmap v7868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:40.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:40 smithi130 bash[27579]: cluster 2023-08-28T02:20:39.476001+0000 mgr.y (mgr.14901) 7861 : cluster [DBG] pgmap v7868: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:40.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:40 smithi130 bash[51302]: debug 2023-08-28T02:20:40.516+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:40.520443+0000) 2023-08-28T02:20:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:20:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:20:40] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:20:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:20:41.277Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:20:41.278Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:41.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:41 smithi130 bash[51302]: debug 2023-08-28T02:20:41.520+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:41.520743+0000) 2023-08-28T02:20:42.840 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:42 smithi006 bash[33918]: cluster 2023-08-28T02:20:41.476569+0000 mgr.y (mgr.14901) 7862 : cluster [DBG] pgmap v7869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:42.840 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:42 smithi006 bash[33918]: audit 2023-08-28T02:20:41.751593+0000 mon.a (mon.0) 5022 : audit [DBG] from='client.? 172.21.15.6:0/1963668763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:42.840 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:42 smithi006 bash[27292]: cluster 2023-08-28T02:20:41.476569+0000 mgr.y (mgr.14901) 7862 : cluster [DBG] pgmap v7869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:42.840 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:42 smithi006 bash[27292]: audit 2023-08-28T02:20:41.751593+0000 mon.a (mon.0) 5022 : audit [DBG] from='client.? 172.21.15.6:0/1963668763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:42.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:42 smithi130 bash[51302]: debug 2023-08-28T02:20:42.515+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:42.520995+0000) 2023-08-28T02:20:42.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:42 smithi130 bash[27579]: cluster 2023-08-28T02:20:41.476569+0000 mgr.y (mgr.14901) 7862 : cluster [DBG] pgmap v7869: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:42.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:42 smithi130 bash[27579]: audit 2023-08-28T02:20:41.751593+0000 mon.a (mon.0) 5022 : audit [DBG] from='client.? 172.21.15.6:0/1963668763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:20:42.839Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:20:42.843Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:43.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:43 smithi130 bash[51302]: debug 2023-08-28T02:20:43.515+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:43.521254+0000) 2023-08-28T02:20:44.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:43 smithi006 bash[33918]: cluster 2023-08-28T02:20:43.477201+0000 mgr.y (mgr.14901) 7863 : cluster [DBG] pgmap v7870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:44.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:43 smithi006 bash[27292]: cluster 2023-08-28T02:20:43.477201+0000 mgr.y (mgr.14901) 7863 : cluster [DBG] pgmap v7870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:44.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:43 smithi130 bash[27579]: cluster 2023-08-28T02:20:43.477201+0000 mgr.y (mgr.14901) 7863 : cluster [DBG] pgmap v7870: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:44.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:44 smithi130 bash[51302]: debug 2023-08-28T02:20:44.515+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:44.521520+0000) 2023-08-28T02:20:45.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:44 smithi006 bash[33918]: audit 2023-08-28T02:20:44.172139+0000 mon.c (mon.2) 3095 : audit [DBG] from='client.? 172.21.15.6:0/2339936806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:45.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:44 smithi006 bash[27292]: audit 2023-08-28T02:20:44.172139+0000 mon.c (mon.2) 3095 : audit [DBG] from='client.? 172.21.15.6:0/2339936806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:45.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:20:44.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:45.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:20:44.718Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:45.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:44 smithi130 bash[27579]: audit 2023-08-28T02:20:44.172139+0000 mon.c (mon.2) 3095 : audit [DBG] from='client.? 172.21.15.6:0/2339936806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:45.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:45 smithi130 bash[51302]: debug 2023-08-28T02:20:45.515+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:45.521775+0000) 2023-08-28T02:20:46.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:45 smithi006 bash[27292]: cluster 2023-08-28T02:20:45.478332+0000 mgr.y (mgr.14901) 7864 : cluster [DBG] pgmap v7871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:46.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:45 smithi006 bash[33918]: cluster 2023-08-28T02:20:45.478332+0000 mgr.y (mgr.14901) 7864 : cluster [DBG] pgmap v7871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:46.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:45 smithi130 bash[27579]: cluster 2023-08-28T02:20:45.478332+0000 mgr.y (mgr.14901) 7864 : cluster [DBG] pgmap v7871: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:46.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:46 smithi130 bash[51302]: debug 2023-08-28T02:20:46.519+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:46.522055+0000) 2023-08-28T02:20:47.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:46 smithi006 bash[33918]: audit 2023-08-28T02:20:46.583948+0000 mon.c (mon.2) 3096 : audit [DBG] from='client.? 172.21.15.6:0/2496661427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:47.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:46 smithi006 bash[27292]: audit 2023-08-28T02:20:46.583948+0000 mon.c (mon.2) 3096 : audit [DBG] from='client.? 172.21.15.6:0/2496661427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:47.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:46 smithi130 bash[27579]: audit 2023-08-28T02:20:46.583948+0000 mon.c (mon.2) 3096 : audit [DBG] from='client.? 172.21.15.6:0/2496661427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:47.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:47 smithi130 bash[51302]: debug 2023-08-28T02:20:47.515+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:47.522337+0000) 2023-08-28T02:20:48.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:47 smithi006 bash[27292]: cluster 2023-08-28T02:20:47.478983+0000 mgr.y (mgr.14901) 7865 : cluster [DBG] pgmap v7872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:48.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:47 smithi006 bash[33918]: cluster 2023-08-28T02:20:47.478983+0000 mgr.y (mgr.14901) 7865 : cluster [DBG] pgmap v7872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:48.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:47 smithi130 bash[27579]: cluster 2023-08-28T02:20:47.478983+0000 mgr.y (mgr.14901) 7865 : cluster [DBG] pgmap v7872: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:48.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:48 smithi130 bash[51302]: debug 2023-08-28T02:20:48.515+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:48.522618+0000) 2023-08-28T02:20:49.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:49 smithi130 bash[27579]: audit 2023-08-28T02:20:49.001870+0000 mon.c (mon.2) 3097 : audit [DBG] from='client.? 172.21.15.6:0/261444390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:49.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:49 smithi006 bash[27292]: audit 2023-08-28T02:20:49.001870+0000 mon.c (mon.2) 3097 : audit [DBG] from='client.? 172.21.15.6:0/261444390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:49.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:49 smithi006 bash[33918]: audit 2023-08-28T02:20:49.001870+0000 mon.c (mon.2) 3097 : audit [DBG] from='client.? 172.21.15.6:0/261444390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:49.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:49 smithi130 bash[51302]: debug 2023-08-28T02:20:49.519+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:49.522970+0000) 2023-08-28T02:20:50.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:50 smithi130 bash[27579]: cluster 2023-08-28T02:20:49.480076+0000 mgr.y (mgr.14901) 7866 : cluster [DBG] pgmap v7873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:50.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:50 smithi006 bash[27292]: cluster 2023-08-28T02:20:49.480076+0000 mgr.y (mgr.14901) 7866 : cluster [DBG] pgmap v7873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:50.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:50 smithi006 bash[33918]: cluster 2023-08-28T02:20:49.480076+0000 mgr.y (mgr.14901) 7866 : cluster [DBG] pgmap v7873: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:50.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:50 smithi130 bash[51302]: debug 2023-08-28T02:20:50.519+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:50.523242+0000) 2023-08-28T02:20:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:20:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:20:50] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-28T02:20:51.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:51 smithi006 bash[27292]: audit 2023-08-28T02:20:51.413894+0000 mon.c (mon.2) 3098 : audit [DBG] from='client.? 172.21.15.6:0/38799413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:51.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:51 smithi006 bash[33918]: audit 2023-08-28T02:20:51.413894+0000 mon.c (mon.2) 3098 : audit [DBG] from='client.? 172.21.15.6:0/38799413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:20:51.278Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:20:51.280Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:51.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:51 smithi130 bash[27579]: audit 2023-08-28T02:20:51.413894+0000 mon.c (mon.2) 3098 : audit [DBG] from='client.? 172.21.15.6:0/38799413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:51.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:51 smithi130 bash[51302]: debug 2023-08-28T02:20:51.519+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:51.523543+0000) 2023-08-28T02:20:52.841 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:52 smithi006 bash[33918]: cluster 2023-08-28T02:20:51.480625+0000 mgr.y (mgr.14901) 7867 : cluster [DBG] pgmap v7874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:52.841 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:52 smithi006 bash[27292]: cluster 2023-08-28T02:20:51.480625+0000 mgr.y (mgr.14901) 7867 : cluster [DBG] pgmap v7874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:52.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:52 smithi130 bash[27579]: cluster 2023-08-28T02:20:51.480625+0000 mgr.y (mgr.14901) 7867 : cluster [DBG] pgmap v7874: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:52.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:52 smithi130 bash[51302]: debug 2023-08-28T02:20:52.519+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:52.523837+0000) 2023-08-28T02:20:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:20:52.840Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:20:52.842Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:53.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:53 smithi130 bash[51302]: debug 2023-08-28T02:20:53.523+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:53.524155+0000) 2023-08-28T02:20:54.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:53 smithi006 bash[33918]: cluster 2023-08-28T02:20:53.481291+0000 mgr.y (mgr.14901) 7868 : cluster [DBG] pgmap v7875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:54.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:53 smithi006 bash[33918]: audit 2023-08-28T02:20:53.833498+0000 mon.a (mon.0) 5023 : audit [DBG] from='client.? 172.21.15.6:0/542130855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:54.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:53 smithi006 bash[27292]: cluster 2023-08-28T02:20:53.481291+0000 mgr.y (mgr.14901) 7868 : cluster [DBG] pgmap v7875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:54.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:53 smithi006 bash[27292]: audit 2023-08-28T02:20:53.833498+0000 mon.a (mon.0) 5023 : audit [DBG] from='client.? 172.21.15.6:0/542130855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:53 smithi130 bash[27579]: cluster 2023-08-28T02:20:53.481291+0000 mgr.y (mgr.14901) 7868 : cluster [DBG] pgmap v7875: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:53 smithi130 bash[27579]: audit 2023-08-28T02:20:53.833498+0000 mon.a (mon.0) 5023 : audit [DBG] from='client.? 172.21.15.6:0/542130855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:54.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:54 smithi130 bash[51302]: debug 2023-08-28T02:20:54.523+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:54.524424+0000) 2023-08-28T02:20:55.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:20:54.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:55.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:20:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:20:54.719Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:20:55.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:55 smithi130 bash[51302]: debug 2023-08-28T02:20:55.523+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:55.524728+0000) 2023-08-28T02:20:56.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:56 smithi130 bash[51302]: debug 2023-08-28T02:20:56.519+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:56.525030+0000) 2023-08-28T02:20:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:56 smithi130 bash[27579]: cluster 2023-08-28T02:20:55.482386+0000 mgr.y (mgr.14901) 7869 : cluster [DBG] pgmap v7876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:56 smithi130 bash[27579]: audit 2023-08-28T02:20:56.252214+0000 mon.c (mon.2) 3099 : audit [DBG] from='client.? 172.21.15.6:0/582258638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:56.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:56 smithi006 bash[27292]: cluster 2023-08-28T02:20:55.482386+0000 mgr.y (mgr.14901) 7869 : cluster [DBG] pgmap v7876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:56.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:56 smithi006 bash[27292]: audit 2023-08-28T02:20:56.252214+0000 mon.c (mon.2) 3099 : audit [DBG] from='client.? 172.21.15.6:0/582258638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:56.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:56 smithi006 bash[33918]: cluster 2023-08-28T02:20:55.482386+0000 mgr.y (mgr.14901) 7869 : cluster [DBG] pgmap v7876: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:56.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:56 smithi006 bash[33918]: audit 2023-08-28T02:20:56.252214+0000 mon.c (mon.2) 3099 : audit [DBG] from='client.? 172.21.15.6:0/582258638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:57.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:57 smithi130 bash[51302]: debug 2023-08-28T02:20:57.523+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:57.525329+0000) 2023-08-28T02:20:58.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:58 smithi130 bash[27579]: cluster 2023-08-28T02:20:57.483023+0000 mgr.y (mgr.14901) 7870 : cluster [DBG] pgmap v7877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:58.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:58 smithi130 bash[51302]: debug 2023-08-28T02:20:58.523+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:58.525601+0000) 2023-08-28T02:20:58.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:58 smithi006 bash[33918]: cluster 2023-08-28T02:20:57.483023+0000 mgr.y (mgr.14901) 7870 : cluster [DBG] pgmap v7877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:58.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:58 smithi006 bash[27292]: cluster 2023-08-28T02:20:57.483023+0000 mgr.y (mgr.14901) 7870 : cluster [DBG] pgmap v7877: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:20:59.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:20:59 smithi130 bash[27579]: audit 2023-08-28T02:20:58.666760+0000 mon.a (mon.0) 5024 : audit [DBG] from='client.? 172.21.15.6:0/177475632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:59.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:20:59 smithi130 bash[51302]: debug 2023-08-28T02:20:59.523+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:59.525884+0000) 2023-08-28T02:20:59.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:20:59 smithi006 bash[27292]: audit 2023-08-28T02:20:58.666760+0000 mon.a (mon.0) 5024 : audit [DBG] from='client.? 172.21.15.6:0/177475632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:59.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:20:59 smithi006 bash[33918]: audit 2023-08-28T02:20:58.666760+0000 mon.a (mon.0) 5024 : audit [DBG] from='client.? 172.21.15.6:0/177475632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:00.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:00 smithi130 bash[51302]: debug 2023-08-28T02:21:00.519+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:00.526166+0000) 2023-08-28T02:21:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:00 smithi130 bash[27579]: cluster 2023-08-28T02:20:59.484131+0000 mgr.y (mgr.14901) 7871 : cluster [DBG] pgmap v7878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:00 smithi130 bash[27579]: audit 2023-08-28T02:21:00.106295+0000 mon.b (mon.1) 864 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:21:00.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:00 smithi006 bash[27292]: cluster 2023-08-28T02:20:59.484131+0000 mgr.y (mgr.14901) 7871 : cluster [DBG] pgmap v7878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:00.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:00 smithi006 bash[27292]: audit 2023-08-28T02:21:00.106295+0000 mon.b (mon.1) 864 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:21:00.965 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:21:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:21:00] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:21:00.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:00 smithi006 bash[33918]: cluster 2023-08-28T02:20:59.484131+0000 mgr.y (mgr.14901) 7871 : cluster [DBG] pgmap v7878: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:00.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:00 smithi006 bash[33918]: audit 2023-08-28T02:21:00.106295+0000 mon.b (mon.1) 864 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:21:01.563 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:21:01.279Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:01.563 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:21:01.281Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:01.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:01 smithi130 bash[27579]: audit 2023-08-28T02:21:01.085315+0000 mon.a (mon.0) 5025 : audit [DBG] from='client.? 172.21.15.6:0/2719919181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:01.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:01 smithi130 bash[51302]: debug 2023-08-28T02:21:01.523+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:01.526466+0000) 2023-08-28T02:21:01.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:01 smithi006 bash[27292]: audit 2023-08-28T02:21:01.085315+0000 mon.a (mon.0) 5025 : audit [DBG] from='client.? 172.21.15.6:0/2719919181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:01.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:01 smithi006 bash[33918]: audit 2023-08-28T02:21:01.085315+0000 mon.a (mon.0) 5025 : audit [DBG] from='client.? 172.21.15.6:0/2719919181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:02.840 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:02 smithi006 bash[33918]: cluster 2023-08-28T02:21:01.484789+0000 mgr.y (mgr.14901) 7872 : cluster [DBG] pgmap v7879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:02.841 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:02 smithi006 bash[27292]: cluster 2023-08-28T02:21:01.484789+0000 mgr.y (mgr.14901) 7872 : cluster [DBG] pgmap v7879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:02 smithi130 bash[27579]: cluster 2023-08-28T02:21:01.484789+0000 mgr.y (mgr.14901) 7872 : cluster [DBG] pgmap v7879: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:02.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:02 smithi130 bash[51302]: debug 2023-08-28T02:21:02.519+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:02.526768+0000) 2023-08-28T02:21:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:21:02.840Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:21:02.842Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:03.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:03 smithi130 bash[51302]: debug 2023-08-28T02:21:03.523+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:03.527062+0000) 2023-08-28T02:21:04.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:03 smithi006 bash[33918]: cluster 2023-08-28T02:21:03.485443+0000 mgr.y (mgr.14901) 7873 : cluster [DBG] pgmap v7880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:04.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:03 smithi006 bash[33918]: audit 2023-08-28T02:21:03.508475+0000 mon.c (mon.2) 3100 : audit [DBG] from='client.? 172.21.15.6:0/2494985907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:04.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:03 smithi006 bash[27292]: cluster 2023-08-28T02:21:03.485443+0000 mgr.y (mgr.14901) 7873 : cluster [DBG] pgmap v7880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:04.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:03 smithi006 bash[27292]: audit 2023-08-28T02:21:03.508475+0000 mon.c (mon.2) 3100 : audit [DBG] from='client.? 172.21.15.6:0/2494985907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:04.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:03 smithi130 bash[27579]: cluster 2023-08-28T02:21:03.485443+0000 mgr.y (mgr.14901) 7873 : cluster [DBG] pgmap v7880: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:04.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:03 smithi130 bash[27579]: audit 2023-08-28T02:21:03.508475+0000 mon.c (mon.2) 3100 : audit [DBG] from='client.? 172.21.15.6:0/2494985907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:04.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:04 smithi130 bash[51302]: debug 2023-08-28T02:21:04.523+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:04.527353+0000) 2023-08-28T02:21:05.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:21:04.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:05.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:21:04.721Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:05.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:05 smithi130 bash[51302]: debug 2023-08-28T02:21:05.523+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:05.527603+0000) 2023-08-28T02:21:06.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:06 smithi130 bash[51302]: debug 2023-08-28T02:21:06.523+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:06.527833+0000) 2023-08-28T02:21:06.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:06 smithi130 bash[27579]: cluster 2023-08-28T02:21:05.486491+0000 mgr.y (mgr.14901) 7874 : cluster [DBG] pgmap v7881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:06.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:06 smithi130 bash[27579]: audit 2023-08-28T02:21:05.956015+0000 mon.c (mon.2) 3101 : audit [DBG] from='client.? 172.21.15.6:0/758646464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:06.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:06 smithi130 bash[27579]: audit 2023-08-28T02:21:05.973897+0000 mon.a (mon.0) 5026 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:21:06.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:06 smithi130 bash[27579]: audit 2023-08-28T02:21:06.453848+0000 mon.a (mon.0) 5027 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:21:06.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:06 smithi006 bash[27292]: cluster 2023-08-28T02:21:05.486491+0000 mgr.y (mgr.14901) 7874 : cluster [DBG] pgmap v7881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:06.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:06 smithi006 bash[27292]: audit 2023-08-28T02:21:05.956015+0000 mon.c (mon.2) 3101 : audit [DBG] from='client.? 172.21.15.6:0/758646464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:06.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:06 smithi006 bash[27292]: audit 2023-08-28T02:21:05.973897+0000 mon.a (mon.0) 5026 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:21:06.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:06 smithi006 bash[27292]: audit 2023-08-28T02:21:06.453848+0000 mon.a (mon.0) 5027 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:21:06.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:06 smithi006 bash[33918]: cluster 2023-08-28T02:21:05.486491+0000 mgr.y (mgr.14901) 7874 : cluster [DBG] pgmap v7881: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:06.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:06 smithi006 bash[33918]: audit 2023-08-28T02:21:05.956015+0000 mon.c (mon.2) 3101 : audit [DBG] from='client.? 172.21.15.6:0/758646464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:06.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:06 smithi006 bash[33918]: audit 2023-08-28T02:21:05.973897+0000 mon.a (mon.0) 5026 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:21:06.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:06 smithi006 bash[33918]: audit 2023-08-28T02:21:06.453848+0000 mon.a (mon.0) 5027 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:21:07.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:07 smithi130 bash[51302]: debug 2023-08-28T02:21:07.523+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:07.528097+0000) 2023-08-28T02:21:07.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:07 smithi130 bash[27579]: audit 2023-08-28T02:21:06.751809+0000 mon.b (mon.1) 865 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:21:07.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:07 smithi130 bash[27579]: audit 2023-08-28T02:21:06.754058+0000 mon.b (mon.1) 866 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:21:07.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:07 smithi130 bash[27579]: audit 2023-08-28T02:21:06.761860+0000 mon.a (mon.0) 5028 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:21:07.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:07 smithi006 bash[33918]: audit 2023-08-28T02:21:06.751809+0000 mon.b (mon.1) 865 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:21:07.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:07 smithi006 bash[33918]: audit 2023-08-28T02:21:06.754058+0000 mon.b (mon.1) 866 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:21:07.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:07 smithi006 bash[33918]: audit 2023-08-28T02:21:06.761860+0000 mon.a (mon.0) 5028 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:21:07.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:07 smithi006 bash[27292]: audit 2023-08-28T02:21:06.751809+0000 mon.b (mon.1) 865 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:21:07.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:07 smithi006 bash[27292]: audit 2023-08-28T02:21:06.754058+0000 mon.b (mon.1) 866 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:21:07.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:07 smithi006 bash[27292]: audit 2023-08-28T02:21:06.761860+0000 mon.a (mon.0) 5028 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:21:08.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:08 smithi130 bash[51302]: debug 2023-08-28T02:21:08.527+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:08.528391+0000) 2023-08-28T02:21:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:08 smithi130 bash[27579]: cluster 2023-08-28T02:21:07.487109+0000 mgr.y (mgr.14901) 7875 : cluster [DBG] pgmap v7882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:08 smithi130 bash[27579]: audit 2023-08-28T02:21:08.362729+0000 mon.c (mon.2) 3102 : audit [DBG] from='client.? 172.21.15.6:0/3522809888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:09.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:08 smithi006 bash[33918]: cluster 2023-08-28T02:21:07.487109+0000 mgr.y (mgr.14901) 7875 : cluster [DBG] pgmap v7882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:09.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:08 smithi006 bash[33918]: audit 2023-08-28T02:21:08.362729+0000 mon.c (mon.2) 3102 : audit [DBG] from='client.? 172.21.15.6:0/3522809888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:09.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:08 smithi006 bash[27292]: cluster 2023-08-28T02:21:07.487109+0000 mgr.y (mgr.14901) 7875 : cluster [DBG] pgmap v7882: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:09.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:08 smithi006 bash[27292]: audit 2023-08-28T02:21:08.362729+0000 mon.c (mon.2) 3102 : audit [DBG] from='client.? 172.21.15.6:0/3522809888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:09.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:09 smithi130 bash[51302]: debug 2023-08-28T02:21:09.523+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:09.528678+0000) 2023-08-28T02:21:10.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:10 smithi130 bash[27579]: cluster 2023-08-28T02:21:09.488200+0000 mgr.y (mgr.14901) 7876 : cluster [DBG] pgmap v7883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:10.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:10 smithi130 bash[51302]: debug 2023-08-28T02:21:10.523+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:10.529001+0000) 2023-08-28T02:21:11.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:21:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:21:10] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:21:11.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:10 smithi006 bash[33918]: cluster 2023-08-28T02:21:09.488200+0000 mgr.y (mgr.14901) 7876 : cluster [DBG] pgmap v7883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:11.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:10 smithi006 bash[27292]: cluster 2023-08-28T02:21:09.488200+0000 mgr.y (mgr.14901) 7876 : cluster [DBG] pgmap v7883: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:21:11.280Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:21:11.282Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:11.778 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:11 smithi130 bash[51302]: debug 2023-08-28T02:21:11.523+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:11.529318+0000) 2023-08-28T02:21:12.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:11 smithi130 bash[27579]: audit 2023-08-28T02:21:10.788339+0000 mon.c (mon.2) 3103 : audit [DBG] from='client.? 172.21.15.6:0/1752655629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:12.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:11 smithi006 bash[33918]: audit 2023-08-28T02:21:10.788339+0000 mon.c (mon.2) 3103 : audit [DBG] from='client.? 172.21.15.6:0/1752655629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:12.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:11 smithi006 bash[27292]: audit 2023-08-28T02:21:10.788339+0000 mon.c (mon.2) 3103 : audit [DBG] from='client.? 172.21.15.6:0/1752655629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:12.781 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:12 smithi130 bash[51302]: debug 2023-08-28T02:21:12.523+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:12.529591+0000) 2023-08-28T02:21:13.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:12 smithi130 bash[27579]: cluster 2023-08-28T02:21:11.488888+0000 mgr.y (mgr.14901) 7877 : cluster [DBG] pgmap v7884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:13.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:12 smithi006 bash[27292]: cluster 2023-08-28T02:21:11.488888+0000 mgr.y (mgr.14901) 7877 : cluster [DBG] pgmap v7884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:12 smithi006 bash[33918]: cluster 2023-08-28T02:21:11.488888+0000 mgr.y (mgr.14901) 7877 : cluster [DBG] pgmap v7884: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:21:12.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:21:12.843Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:13.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:13 smithi130 bash[51302]: debug 2023-08-28T02:21:13.527+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:13.529915+0000) 2023-08-28T02:21:14.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:13 smithi006 bash[27292]: audit 2023-08-28T02:21:13.198301+0000 mon.a (mon.0) 5029 : audit [DBG] from='client.? 172.21.15.6:0/788932439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:13 smithi006 bash[27292]: cluster 2023-08-28T02:21:13.489552+0000 mgr.y (mgr.14901) 7878 : cluster [DBG] pgmap v7885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:13 smithi006 bash[33918]: audit 2023-08-28T02:21:13.198301+0000 mon.a (mon.0) 5029 : audit [DBG] from='client.? 172.21.15.6:0/788932439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:13 smithi006 bash[33918]: cluster 2023-08-28T02:21:13.489552+0000 mgr.y (mgr.14901) 7878 : cluster [DBG] pgmap v7885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:14.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:13 smithi130 bash[27579]: audit 2023-08-28T02:21:13.198301+0000 mon.a (mon.0) 5029 : audit [DBG] from='client.? 172.21.15.6:0/788932439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:14.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:13 smithi130 bash[27579]: cluster 2023-08-28T02:21:13.489552+0000 mgr.y (mgr.14901) 7878 : cluster [DBG] pgmap v7885: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:14.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:14 smithi130 bash[51302]: debug 2023-08-28T02:21:14.527+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:14.530232+0000) 2023-08-28T02:21:15.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:21:14.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:15.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:21:14.721Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:15.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:15 smithi130 bash[51302]: debug 2023-08-28T02:21:15.527+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:15.530503+0000) 2023-08-28T02:21:16.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:16 smithi130 bash[51302]: debug 2023-08-28T02:21:16.527+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:16.530804+0000) 2023-08-28T02:21:16.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:16 smithi130 bash[27579]: cluster 2023-08-28T02:21:15.490623+0000 mgr.y (mgr.14901) 7879 : cluster [DBG] pgmap v7886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:16.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:16 smithi130 bash[27579]: audit 2023-08-28T02:21:15.610343+0000 mon.c (mon.2) 3104 : audit [DBG] from='client.? 172.21.15.6:0/2261892101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:16.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:16 smithi006 bash[27292]: cluster 2023-08-28T02:21:15.490623+0000 mgr.y (mgr.14901) 7879 : cluster [DBG] pgmap v7886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:16.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:16 smithi006 bash[27292]: audit 2023-08-28T02:21:15.610343+0000 mon.c (mon.2) 3104 : audit [DBG] from='client.? 172.21.15.6:0/2261892101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:16.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:16 smithi006 bash[33918]: cluster 2023-08-28T02:21:15.490623+0000 mgr.y (mgr.14901) 7879 : cluster [DBG] pgmap v7886: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:16.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:16 smithi006 bash[33918]: audit 2023-08-28T02:21:15.610343+0000 mon.c (mon.2) 3104 : audit [DBG] from='client.? 172.21.15.6:0/2261892101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:17.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:17 smithi130 bash[51302]: debug 2023-08-28T02:21:17.527+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:17.531075+0000) 2023-08-28T02:21:18.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:18 smithi130 bash[51302]: debug 2023-08-28T02:21:18.527+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:18.531379+0000) 2023-08-28T02:21:18.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:18 smithi130 bash[27579]: cluster 2023-08-28T02:21:17.491246+0000 mgr.y (mgr.14901) 7880 : cluster [DBG] pgmap v7887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:18.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:18 smithi130 bash[27579]: audit 2023-08-28T02:21:18.023705+0000 mon.a (mon.0) 5030 : audit [DBG] from='client.? 172.21.15.6:0/2899593875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:18.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:18 smithi006 bash[27292]: cluster 2023-08-28T02:21:17.491246+0000 mgr.y (mgr.14901) 7880 : cluster [DBG] pgmap v7887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:18.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:18 smithi006 bash[27292]: audit 2023-08-28T02:21:18.023705+0000 mon.a (mon.0) 5030 : audit [DBG] from='client.? 172.21.15.6:0/2899593875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:18.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:18 smithi006 bash[33918]: cluster 2023-08-28T02:21:17.491246+0000 mgr.y (mgr.14901) 7880 : cluster [DBG] pgmap v7887: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:18.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:18 smithi006 bash[33918]: audit 2023-08-28T02:21:18.023705+0000 mon.a (mon.0) 5030 : audit [DBG] from='client.? 172.21.15.6:0/2899593875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:19.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:19 smithi130 bash[51302]: debug 2023-08-28T02:21:19.531+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:19.531667+0000) 2023-08-28T02:21:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:20 smithi130 bash[27579]: cluster 2023-08-28T02:21:19.492354+0000 mgr.y (mgr.14901) 7881 : cluster [DBG] pgmap v7888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:20 smithi130 bash[27579]: audit 2023-08-28T02:21:20.452935+0000 mon.c (mon.2) 3105 : audit [DBG] from='client.? 172.21.15.6:0/902926368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:20.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:20 smithi130 bash[51302]: debug 2023-08-28T02:21:20.527+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:20.531990+0000) 2023-08-28T02:21:20.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:21:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:21:20] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T02:21:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:20 smithi006 bash[27292]: cluster 2023-08-28T02:21:19.492354+0000 mgr.y (mgr.14901) 7881 : cluster [DBG] pgmap v7888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:20 smithi006 bash[27292]: audit 2023-08-28T02:21:20.452935+0000 mon.c (mon.2) 3105 : audit [DBG] from='client.? 172.21.15.6:0/902926368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:20.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:20 smithi006 bash[33918]: cluster 2023-08-28T02:21:19.492354+0000 mgr.y (mgr.14901) 7881 : cluster [DBG] pgmap v7888: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:20.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:20 smithi006 bash[33918]: audit 2023-08-28T02:21:20.452935+0000 mon.c (mon.2) 3105 : audit [DBG] from='client.? 172.21.15.6:0/902926368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:21:21.281Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:21:21.283Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:21.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:21 smithi130 bash[51302]: debug 2023-08-28T02:21:21.531+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:21.532275+0000) 2023-08-28T02:21:22.842 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:22 smithi006 bash[27292]: cluster 2023-08-28T02:21:21.493035+0000 mgr.y (mgr.14901) 7882 : cluster [DBG] pgmap v7889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:22.843 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:22 smithi006 bash[33918]: cluster 2023-08-28T02:21:21.493035+0000 mgr.y (mgr.14901) 7882 : cluster [DBG] pgmap v7889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:22 smithi130 bash[27579]: cluster 2023-08-28T02:21:21.493035+0000 mgr.y (mgr.14901) 7882 : cluster [DBG] pgmap v7889: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:22.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:22 smithi130 bash[51302]: debug 2023-08-28T02:21:22.527+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:22.532575+0000) 2023-08-28T02:21:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:21:22.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:21:22.843Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:23.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:23 smithi130 bash[27579]: audit 2023-08-28T02:21:22.870965+0000 mon.a (mon.0) 5031 : audit [DBG] from='client.? 172.21.15.6:0/495622095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:23.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:23 smithi130 bash[51302]: debug 2023-08-28T02:21:23.531+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:23.532874+0000) 2023-08-28T02:21:23.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:23 smithi006 bash[33918]: audit 2023-08-28T02:21:22.870965+0000 mon.a (mon.0) 5031 : audit [DBG] from='client.? 172.21.15.6:0/495622095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:23.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:23 smithi006 bash[27292]: audit 2023-08-28T02:21:22.870965+0000 mon.a (mon.0) 5031 : audit [DBG] from='client.? 172.21.15.6:0/495622095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:24.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:24 smithi130 bash[51302]: debug 2023-08-28T02:21:24.531+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:24.533143+0000) 2023-08-28T02:21:24.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:24 smithi130 bash[27579]: cluster 2023-08-28T02:21:23.493689+0000 mgr.y (mgr.14901) 7883 : cluster [DBG] pgmap v7890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:24.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:24 smithi006 bash[27292]: cluster 2023-08-28T02:21:23.493689+0000 mgr.y (mgr.14901) 7883 : cluster [DBG] pgmap v7890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:24.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:24 smithi006 bash[33918]: cluster 2023-08-28T02:21:23.493689+0000 mgr.y (mgr.14901) 7883 : cluster [DBG] pgmap v7890: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:21:24.720Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:24.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:21:24.722Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:25.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:25 smithi130 bash[27579]: audit 2023-08-28T02:21:25.281612+0000 mon.c (mon.2) 3106 : audit [DBG] from='client.? 172.21.15.6:0/785688663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:25.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:25 smithi130 bash[51302]: debug 2023-08-28T02:21:25.527+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:25.533413+0000) 2023-08-28T02:21:25.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:25 smithi006 bash[33918]: audit 2023-08-28T02:21:25.281612+0000 mon.c (mon.2) 3106 : audit [DBG] from='client.? 172.21.15.6:0/785688663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:25.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:25 smithi006 bash[27292]: audit 2023-08-28T02:21:25.281612+0000 mon.c (mon.2) 3106 : audit [DBG] from='client.? 172.21.15.6:0/785688663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:26.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:26 smithi130 bash[51302]: debug 2023-08-28T02:21:26.531+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:26.533724+0000) 2023-08-28T02:21:26.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:26 smithi130 bash[27579]: cluster 2023-08-28T02:21:25.494824+0000 mgr.y (mgr.14901) 7884 : cluster [DBG] pgmap v7891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:26.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:26 smithi006 bash[27292]: cluster 2023-08-28T02:21:25.494824+0000 mgr.y (mgr.14901) 7884 : cluster [DBG] pgmap v7891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:26.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:26 smithi006 bash[33918]: cluster 2023-08-28T02:21:25.494824+0000 mgr.y (mgr.14901) 7884 : cluster [DBG] pgmap v7891: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:27.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:27 smithi130 bash[51302]: debug 2023-08-28T02:21:27.531+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:27.533999+0000) 2023-08-28T02:21:28.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:28 smithi130 bash[27579]: cluster 2023-08-28T02:21:27.495280+0000 mgr.y (mgr.14901) 7885 : cluster [DBG] pgmap v7892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:28.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:28 smithi130 bash[27579]: audit 2023-08-28T02:21:27.700546+0000 mon.c (mon.2) 3107 : audit [DBG] from='client.? 172.21.15.6:0/1481986844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:28.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:28 smithi130 bash[51302]: debug 2023-08-28T02:21:28.531+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:28.534275+0000) 2023-08-28T02:21:28.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:28 smithi006 bash[33918]: cluster 2023-08-28T02:21:27.495280+0000 mgr.y (mgr.14901) 7885 : cluster [DBG] pgmap v7892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:28.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:28 smithi006 bash[33918]: audit 2023-08-28T02:21:27.700546+0000 mon.c (mon.2) 3107 : audit [DBG] from='client.? 172.21.15.6:0/1481986844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:28.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:28 smithi006 bash[27292]: cluster 2023-08-28T02:21:27.495280+0000 mgr.y (mgr.14901) 7885 : cluster [DBG] pgmap v7892: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:28.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:28 smithi006 bash[27292]: audit 2023-08-28T02:21:27.700546+0000 mon.c (mon.2) 3107 : audit [DBG] from='client.? 172.21.15.6:0/1481986844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:29.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:29 smithi130 bash[51302]: debug 2023-08-28T02:21:29.531+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:29.534624+0000) 2023-08-28T02:21:30.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:30 smithi130 bash[51302]: debug 2023-08-28T02:21:30.531+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:30.534949+0000) 2023-08-28T02:21:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:30 smithi130 bash[27579]: cluster 2023-08-28T02:21:29.496391+0000 mgr.y (mgr.14901) 7886 : cluster [DBG] pgmap v7893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:30 smithi130 bash[27579]: audit 2023-08-28T02:21:30.110972+0000 mon.a (mon.0) 5032 : audit [DBG] from='client.? 172.21.15.6:0/3614154323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:30.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:21:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:21:30] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:21:30.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:30 smithi006 bash[33918]: cluster 2023-08-28T02:21:29.496391+0000 mgr.y (mgr.14901) 7886 : cluster [DBG] pgmap v7893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:30.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:30 smithi006 bash[33918]: audit 2023-08-28T02:21:30.110972+0000 mon.a (mon.0) 5032 : audit [DBG] from='client.? 172.21.15.6:0/3614154323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:30.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:30 smithi006 bash[27292]: cluster 2023-08-28T02:21:29.496391+0000 mgr.y (mgr.14901) 7886 : cluster [DBG] pgmap v7893: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:30.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:30 smithi006 bash[27292]: audit 2023-08-28T02:21:30.110972+0000 mon.a (mon.0) 5032 : audit [DBG] from='client.? 172.21.15.6:0/3614154323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:21:31.281Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:21:31.283Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:31.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:31 smithi130 bash[51302]: debug 2023-08-28T02:21:31.535+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:31.535255+0000) 2023-08-28T02:21:32.842 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:32 smithi006 bash[33918]: cluster 2023-08-28T02:21:31.497082+0000 mgr.y (mgr.14901) 7887 : cluster [DBG] pgmap v7894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:32.843 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:32 smithi006 bash[33918]: audit 2023-08-28T02:21:32.517870+0000 mon.c (mon.2) 3108 : audit [DBG] from='client.? 172.21.15.6:0/3099538564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:32.843 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:32 smithi006 bash[27292]: cluster 2023-08-28T02:21:31.497082+0000 mgr.y (mgr.14901) 7887 : cluster [DBG] pgmap v7894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:32.843 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:32 smithi006 bash[27292]: audit 2023-08-28T02:21:32.517870+0000 mon.c (mon.2) 3108 : audit [DBG] from='client.? 172.21.15.6:0/3099538564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:32.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:32 smithi130 bash[51302]: debug 2023-08-28T02:21:32.531+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:32.535477+0000) 2023-08-28T02:21:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:32 smithi130 bash[27579]: cluster 2023-08-28T02:21:31.497082+0000 mgr.y (mgr.14901) 7887 : cluster [DBG] pgmap v7894: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:32.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:32 smithi130 bash[27579]: audit 2023-08-28T02:21:32.517870+0000 mon.c (mon.2) 3108 : audit [DBG] from='client.? 172.21.15.6:0/3099538564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:21:32.842Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:21:32.844Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:33.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:33 smithi130 bash[51302]: debug 2023-08-28T02:21:33.531+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:33.535791+0000) 2023-08-28T02:21:34.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:33 smithi006 bash[33918]: cluster 2023-08-28T02:21:33.497726+0000 mgr.y (mgr.14901) 7888 : cluster [DBG] pgmap v7895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:33 smithi006 bash[27292]: cluster 2023-08-28T02:21:33.497726+0000 mgr.y (mgr.14901) 7888 : cluster [DBG] pgmap v7895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:33 smithi130 bash[27579]: cluster 2023-08-28T02:21:33.497726+0000 mgr.y (mgr.14901) 7888 : cluster [DBG] pgmap v7895: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:34.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:34 smithi130 bash[51302]: debug 2023-08-28T02:21:34.531+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:34.536060+0000) 2023-08-28T02:21:34.993 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:21:34.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:34.993 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:21:34.723Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:35.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:34 smithi130 bash[27579]: audit 2023-08-28T02:21:34.938477+0000 mon.a (mon.0) 5033 : audit [DBG] from='client.? 172.21.15.6:0/2608119663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:35.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:34 smithi006 bash[27292]: audit 2023-08-28T02:21:34.938477+0000 mon.a (mon.0) 5033 : audit [DBG] from='client.? 172.21.15.6:0/2608119663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:35.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:34 smithi006 bash[33918]: audit 2023-08-28T02:21:34.938477+0000 mon.a (mon.0) 5033 : audit [DBG] from='client.? 172.21.15.6:0/2608119663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:35.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:35 smithi130 bash[51302]: debug 2023-08-28T02:21:35.531+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:35.536320+0000) 2023-08-28T02:21:36.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:36 smithi130 bash[27579]: cluster 2023-08-28T02:21:35.498836+0000 mgr.y (mgr.14901) 7889 : cluster [DBG] pgmap v7896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:36.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:36 smithi006 bash[27292]: cluster 2023-08-28T02:21:35.498836+0000 mgr.y (mgr.14901) 7889 : cluster [DBG] pgmap v7896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:36.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:36 smithi006 bash[33918]: cluster 2023-08-28T02:21:35.498836+0000 mgr.y (mgr.14901) 7889 : cluster [DBG] pgmap v7896: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:36.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:36 smithi130 bash[51302]: debug 2023-08-28T02:21:36.534+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:36.536621+0000) 2023-08-28T02:21:37.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:37 smithi006 bash[33918]: audit 2023-08-28T02:21:37.346066+0000 mon.c (mon.2) 3109 : audit [DBG] from='client.? 172.21.15.6:0/2251032972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:37.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:37 smithi006 bash[27292]: audit 2023-08-28T02:21:37.346066+0000 mon.c (mon.2) 3109 : audit [DBG] from='client.? 172.21.15.6:0/2251032972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:37.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:37 smithi130 bash[27579]: audit 2023-08-28T02:21:37.346066+0000 mon.c (mon.2) 3109 : audit [DBG] from='client.? 172.21.15.6:0/2251032972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:37.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:37 smithi130 bash[51302]: debug 2023-08-28T02:21:37.530+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:37.536910+0000) 2023-08-28T02:21:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:38 smithi006 bash[33918]: cluster 2023-08-28T02:21:37.499460+0000 mgr.y (mgr.14901) 7890 : cluster [DBG] pgmap v7897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:38 smithi006 bash[27292]: cluster 2023-08-28T02:21:37.499460+0000 mgr.y (mgr.14901) 7890 : cluster [DBG] pgmap v7897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:38.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:38 smithi130 bash[27579]: cluster 2023-08-28T02:21:37.499460+0000 mgr.y (mgr.14901) 7890 : cluster [DBG] pgmap v7897: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:38.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:38 smithi130 bash[51302]: debug 2023-08-28T02:21:38.530+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:38.537230+0000) 2023-08-28T02:21:39.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:39 smithi130 bash[51302]: debug 2023-08-28T02:21:39.534+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:39.537563+0000) 2023-08-28T02:21:40.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:40 smithi130 bash[27579]: cluster 2023-08-28T02:21:39.500414+0000 mgr.y (mgr.14901) 7891 : cluster [DBG] pgmap v7898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:40.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:40 smithi130 bash[27579]: audit 2023-08-28T02:21:39.759538+0000 mon.a (mon.0) 5034 : audit [DBG] from='client.? 172.21.15.6:0/227983981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:40.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:40 smithi130 bash[51302]: debug 2023-08-28T02:21:40.534+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:40.537838+0000) 2023-08-28T02:21:40.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:21:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:21:40] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:21:40.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:40 smithi006 bash[33918]: cluster 2023-08-28T02:21:39.500414+0000 mgr.y (mgr.14901) 7891 : cluster [DBG] pgmap v7898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:40.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:40 smithi006 bash[33918]: audit 2023-08-28T02:21:39.759538+0000 mon.a (mon.0) 5034 : audit [DBG] from='client.? 172.21.15.6:0/227983981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:40.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:40 smithi006 bash[27292]: cluster 2023-08-28T02:21:39.500414+0000 mgr.y (mgr.14901) 7891 : cluster [DBG] pgmap v7898: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:40.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:40 smithi006 bash[27292]: audit 2023-08-28T02:21:39.759538+0000 mon.a (mon.0) 5034 : audit [DBG] from='client.? 172.21.15.6:0/227983981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:21:41.282Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:21:41.284Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:41.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:41 smithi130 bash[51302]: debug 2023-08-28T02:21:41.530+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:41.538147+0000) 2023-08-28T02:21:42.843 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:42 smithi006 bash[27292]: cluster 2023-08-28T02:21:41.501112+0000 mgr.y (mgr.14901) 7892 : cluster [DBG] pgmap v7899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:42.843 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:42 smithi006 bash[27292]: audit 2023-08-28T02:21:42.172868+0000 mon.c (mon.2) 3110 : audit [DBG] from='client.? 172.21.15.6:0/2280589141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:42.843 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:42 smithi006 bash[33918]: cluster 2023-08-28T02:21:41.501112+0000 mgr.y (mgr.14901) 7892 : cluster [DBG] pgmap v7899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:42.843 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:42 smithi006 bash[33918]: audit 2023-08-28T02:21:42.172868+0000 mon.c (mon.2) 3110 : audit [DBG] from='client.? 172.21.15.6:0/2280589141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:42.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:42 smithi130 bash[27579]: cluster 2023-08-28T02:21:41.501112+0000 mgr.y (mgr.14901) 7892 : cluster [DBG] pgmap v7899: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:42.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:42 smithi130 bash[27579]: audit 2023-08-28T02:21:42.172868+0000 mon.c (mon.2) 3110 : audit [DBG] from='client.? 172.21.15.6:0/2280589141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:42.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:42 smithi130 bash[51302]: debug 2023-08-28T02:21:42.534+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:42.538430+0000) 2023-08-28T02:21:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:21:42.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:21:42.845Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:43.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:43 smithi130 bash[51302]: debug 2023-08-28T02:21:43.534+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:43.538710+0000) 2023-08-28T02:21:44.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:43 smithi006 bash[33918]: cluster 2023-08-28T02:21:43.501816+0000 mgr.y (mgr.14901) 7893 : cluster [DBG] pgmap v7900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:44.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:43 smithi006 bash[27292]: cluster 2023-08-28T02:21:43.501816+0000 mgr.y (mgr.14901) 7893 : cluster [DBG] pgmap v7900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:44.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:43 smithi130 bash[27579]: cluster 2023-08-28T02:21:43.501816+0000 mgr.y (mgr.14901) 7893 : cluster [DBG] pgmap v7900: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:44.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:44 smithi130 bash[51302]: debug 2023-08-28T02:21:44.538+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:44.538996+0000) 2023-08-28T02:21:45.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:44 smithi006 bash[27292]: audit 2023-08-28T02:21:44.582118+0000 mon.a (mon.0) 5035 : audit [DBG] from='client.? 172.21.15.6:0/1543044665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:45.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:44 smithi006 bash[33918]: audit 2023-08-28T02:21:44.582118+0000 mon.a (mon.0) 5035 : audit [DBG] from='client.? 172.21.15.6:0/1543044665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:45.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:21:44.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:45.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:21:44.724Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:45.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:44 smithi130 bash[27579]: audit 2023-08-28T02:21:44.582118+0000 mon.a (mon.0) 5035 : audit [DBG] from='client.? 172.21.15.6:0/1543044665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:45.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:45 smithi130 bash[51302]: debug 2023-08-28T02:21:45.538+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:45.539218+0000) 2023-08-28T02:21:46.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:45 smithi006 bash[33918]: cluster 2023-08-28T02:21:45.502960+0000 mgr.y (mgr.14901) 7894 : cluster [DBG] pgmap v7901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:46.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:45 smithi006 bash[27292]: cluster 2023-08-28T02:21:45.502960+0000 mgr.y (mgr.14901) 7894 : cluster [DBG] pgmap v7901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:46.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:45 smithi130 bash[27579]: cluster 2023-08-28T02:21:45.502960+0000 mgr.y (mgr.14901) 7894 : cluster [DBG] pgmap v7901: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:46.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:46 smithi130 bash[51302]: debug 2023-08-28T02:21:46.538+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:46.539544+0000) 2023-08-28T02:21:47.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:47 smithi130 bash[27579]: audit 2023-08-28T02:21:46.997323+0000 mon.a (mon.0) 5036 : audit [DBG] from='client.? 172.21.15.6:0/3410393529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:47.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:47 smithi006 bash[27292]: audit 2023-08-28T02:21:46.997323+0000 mon.a (mon.0) 5036 : audit [DBG] from='client.? 172.21.15.6:0/3410393529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:47.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:47 smithi006 bash[33918]: audit 2023-08-28T02:21:46.997323+0000 mon.a (mon.0) 5036 : audit [DBG] from='client.? 172.21.15.6:0/3410393529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:47.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:47 smithi130 bash[51302]: debug 2023-08-28T02:21:47.538+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:47.539855+0000) 2023-08-28T02:21:48.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:48 smithi130 bash[27579]: cluster 2023-08-28T02:21:47.503662+0000 mgr.y (mgr.14901) 7895 : cluster [DBG] pgmap v7902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:48.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:48 smithi006 bash[33918]: cluster 2023-08-28T02:21:47.503662+0000 mgr.y (mgr.14901) 7895 : cluster [DBG] pgmap v7902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:48.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:48 smithi006 bash[27292]: cluster 2023-08-28T02:21:47.503662+0000 mgr.y (mgr.14901) 7895 : cluster [DBG] pgmap v7902: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:48.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:48 smithi130 bash[51302]: debug 2023-08-28T02:21:48.538+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:48.540130+0000) 2023-08-28T02:21:49.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:49 smithi006 bash[27292]: audit 2023-08-28T02:21:49.409793+0000 mon.c (mon.2) 3111 : audit [DBG] from='client.? 172.21.15.6:0/1778351095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:49.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:49 smithi006 bash[33918]: audit 2023-08-28T02:21:49.409793+0000 mon.c (mon.2) 3111 : audit [DBG] from='client.? 172.21.15.6:0/1778351095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:49.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:49 smithi130 bash[27579]: audit 2023-08-28T02:21:49.409793+0000 mon.c (mon.2) 3111 : audit [DBG] from='client.? 172.21.15.6:0/1778351095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:49.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:49 smithi130 bash[51302]: debug 2023-08-28T02:21:49.538+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:49.540403+0000) 2023-08-28T02:21:50.751 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:50 smithi006 bash[33918]: cluster 2023-08-28T02:21:49.504825+0000 mgr.y (mgr.14901) 7896 : cluster [DBG] pgmap v7903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:50.751 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:50 smithi006 bash[27292]: cluster 2023-08-28T02:21:49.504825+0000 mgr.y (mgr.14901) 7896 : cluster [DBG] pgmap v7903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:50.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:50 smithi130 bash[27579]: cluster 2023-08-28T02:21:49.504825+0000 mgr.y (mgr.14901) 7896 : cluster [DBG] pgmap v7903: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:50.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:50 smithi130 bash[51302]: debug 2023-08-28T02:21:50.538+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:50.540700+0000) 2023-08-28T02:21:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:21:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:21:50] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:21:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:21:51.283Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:21:51.285Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:51.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:51 smithi130 bash[51302]: debug 2023-08-28T02:21:51.538+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:51.541013+0000) 2023-08-28T02:21:52.844 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:52 smithi006 bash[27292]: cluster 2023-08-28T02:21:51.505499+0000 mgr.y (mgr.14901) 7897 : cluster [DBG] pgmap v7904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:52.844 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:52 smithi006 bash[27292]: audit 2023-08-28T02:21:51.826498+0000 mon.c (mon.2) 3112 : audit [DBG] from='client.? 172.21.15.6:0/424547968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:52.844 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:52 smithi006 bash[33918]: cluster 2023-08-28T02:21:51.505499+0000 mgr.y (mgr.14901) 7897 : cluster [DBG] pgmap v7904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:52.844 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:52 smithi006 bash[33918]: audit 2023-08-28T02:21:51.826498+0000 mon.c (mon.2) 3112 : audit [DBG] from='client.? 172.21.15.6:0/424547968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:52.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:52 smithi130 bash[51302]: debug 2023-08-28T02:21:52.538+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:52.541295+0000) 2023-08-28T02:21:52.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:52 smithi130 bash[27579]: cluster 2023-08-28T02:21:51.505499+0000 mgr.y (mgr.14901) 7897 : cluster [DBG] pgmap v7904: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:52.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:52 smithi130 bash[27579]: audit 2023-08-28T02:21:51.826498+0000 mon.c (mon.2) 3112 : audit [DBG] from='client.? 172.21.15.6:0/424547968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:21:52.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:21:52.845Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:53.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:53 smithi130 bash[51302]: debug 2023-08-28T02:21:53.538+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:53.541578+0000) 2023-08-28T02:21:54.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:53 smithi006 bash[27292]: cluster 2023-08-28T02:21:53.506185+0000 mgr.y (mgr.14901) 7898 : cluster [DBG] pgmap v7905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:54.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:53 smithi006 bash[33918]: cluster 2023-08-28T02:21:53.506185+0000 mgr.y (mgr.14901) 7898 : cluster [DBG] pgmap v7905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:53 smithi130 bash[27579]: cluster 2023-08-28T02:21:53.506185+0000 mgr.y (mgr.14901) 7898 : cluster [DBG] pgmap v7905: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:54.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:54 smithi130 bash[51302]: debug 2023-08-28T02:21:54.538+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:54.541917+0000) 2023-08-28T02:21:55.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:54 smithi006 bash[33918]: audit 2023-08-28T02:21:54.251815+0000 mon.c (mon.2) 3113 : audit [DBG] from='client.? 172.21.15.6:0/801626456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:55.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:21:54.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:55.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:21:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:21:54.725Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:21:55.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:54 smithi006 bash[27292]: audit 2023-08-28T02:21:54.251815+0000 mon.c (mon.2) 3113 : audit [DBG] from='client.? 172.21.15.6:0/801626456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:55.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:54 smithi130 bash[27579]: audit 2023-08-28T02:21:54.251815+0000 mon.c (mon.2) 3113 : audit [DBG] from='client.? 172.21.15.6:0/801626456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:55.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:55 smithi130 bash[51302]: debug 2023-08-28T02:21:55.538+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:55.542209+0000) 2023-08-28T02:21:56.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:55 smithi006 bash[33918]: cluster 2023-08-28T02:21:55.507308+0000 mgr.y (mgr.14901) 7899 : cluster [DBG] pgmap v7906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:56.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:55 smithi006 bash[27292]: cluster 2023-08-28T02:21:55.507308+0000 mgr.y (mgr.14901) 7899 : cluster [DBG] pgmap v7906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:56.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:55 smithi130 bash[27579]: cluster 2023-08-28T02:21:55.507308+0000 mgr.y (mgr.14901) 7899 : cluster [DBG] pgmap v7906: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:56.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:56 smithi130 bash[51302]: debug 2023-08-28T02:21:56.538+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:56.542524+0000) 2023-08-28T02:21:57.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:56 smithi006 bash[33918]: audit 2023-08-28T02:21:56.664526+0000 mon.a (mon.0) 5037 : audit [DBG] from='client.? 172.21.15.6:0/2825980271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:57.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:56 smithi006 bash[27292]: audit 2023-08-28T02:21:56.664526+0000 mon.a (mon.0) 5037 : audit [DBG] from='client.? 172.21.15.6:0/2825980271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:57.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:56 smithi130 bash[27579]: audit 2023-08-28T02:21:56.664526+0000 mon.a (mon.0) 5037 : audit [DBG] from='client.? 172.21.15.6:0/2825980271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:57.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:57 smithi130 bash[51302]: debug 2023-08-28T02:21:57.542+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:57.542818+0000) 2023-08-28T02:21:58.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:57 smithi006 bash[33918]: cluster 2023-08-28T02:21:57.507979+0000 mgr.y (mgr.14901) 7900 : cluster [DBG] pgmap v7907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:58.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:57 smithi006 bash[27292]: cluster 2023-08-28T02:21:57.507979+0000 mgr.y (mgr.14901) 7900 : cluster [DBG] pgmap v7907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:58.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:57 smithi130 bash[27579]: cluster 2023-08-28T02:21:57.507979+0000 mgr.y (mgr.14901) 7900 : cluster [DBG] pgmap v7907: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:21:58.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:58 smithi130 bash[51302]: debug 2023-08-28T02:21:58.542+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:58.543101+0000) 2023-08-28T02:21:59.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:21:59 smithi006 bash[33918]: audit 2023-08-28T02:21:59.095977+0000 mon.a (mon.0) 5038 : audit [DBG] from='client.? 172.21.15.6:0/613210115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:59.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:21:59 smithi006 bash[27292]: audit 2023-08-28T02:21:59.095977+0000 mon.a (mon.0) 5038 : audit [DBG] from='client.? 172.21.15.6:0/613210115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:59.542 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:21:59 smithi130 bash[27579]: audit 2023-08-28T02:21:59.095977+0000 mon.a (mon.0) 5038 : audit [DBG] from='client.? 172.21.15.6:0/613210115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:59.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:21:59 smithi130 bash[51302]: debug 2023-08-28T02:21:59.542+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:59.543406+0000) 2023-08-28T02:22:00.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:00 smithi006 bash[33918]: cluster 2023-08-28T02:21:59.509149+0000 mgr.y (mgr.14901) 7901 : cluster [DBG] pgmap v7908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:00.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:00 smithi006 bash[27292]: cluster 2023-08-28T02:21:59.509149+0000 mgr.y (mgr.14901) 7901 : cluster [DBG] pgmap v7908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:00.542 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:00 smithi130 bash[27579]: cluster 2023-08-28T02:21:59.509149+0000 mgr.y (mgr.14901) 7901 : cluster [DBG] pgmap v7908: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:00.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:00 smithi130 bash[51302]: debug 2023-08-28T02:22:00.542+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:00.543698+0000) 2023-08-28T02:22:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:22:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:22:00] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:22:01.571 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:22:01.283Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:01.571 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:22:01.285Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:01.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:01 smithi130 bash[27579]: audit 2023-08-28T02:22:01.515788+0000 mon.c (mon.2) 3114 : audit [DBG] from='client.? 172.21.15.6:0/677660251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:01.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:01 smithi130 bash[51302]: debug 2023-08-28T02:22:01.542+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:01.543963+0000) 2023-08-28T02:22:01.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:01 smithi006 bash[33918]: audit 2023-08-28T02:22:01.515788+0000 mon.c (mon.2) 3114 : audit [DBG] from='client.? 172.21.15.6:0/677660251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:01.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:01 smithi006 bash[27292]: audit 2023-08-28T02:22:01.515788+0000 mon.c (mon.2) 3114 : audit [DBG] from='client.? 172.21.15.6:0/677660251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:02.845 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:02 smithi006 bash[33918]: cluster 2023-08-28T02:22:01.509740+0000 mgr.y (mgr.14901) 7902 : cluster [DBG] pgmap v7909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:02.845 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:02 smithi006 bash[27292]: cluster 2023-08-28T02:22:01.509740+0000 mgr.y (mgr.14901) 7902 : cluster [DBG] pgmap v7909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:02 smithi130 bash[27579]: cluster 2023-08-28T02:22:01.509740+0000 mgr.y (mgr.14901) 7902 : cluster [DBG] pgmap v7909: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:02.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:02 smithi130 bash[51302]: debug 2023-08-28T02:22:02.542+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:02.544244+0000) 2023-08-28T02:22:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:22:02.845Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:22:02.847Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:03.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:03 smithi130 bash[51302]: debug 2023-08-28T02:22:03.542+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:03.544515+0000) 2023-08-28T02:22:04.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:03 smithi006 bash[27292]: cluster 2023-08-28T02:22:03.510419+0000 mgr.y (mgr.14901) 7903 : cluster [DBG] pgmap v7910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:04.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:03 smithi006 bash[33918]: cluster 2023-08-28T02:22:03.510419+0000 mgr.y (mgr.14901) 7903 : cluster [DBG] pgmap v7910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:03 smithi130 bash[27579]: cluster 2023-08-28T02:22:03.510419+0000 mgr.y (mgr.14901) 7903 : cluster [DBG] pgmap v7910: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:04.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:04 smithi130 bash[51302]: debug 2023-08-28T02:22:04.542+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:04.544794+0000) 2023-08-28T02:22:05.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:04 smithi006 bash[33918]: audit 2023-08-28T02:22:03.928065+0000 mon.a (mon.0) 5039 : audit [DBG] from='client.? 172.21.15.6:0/734702659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:05.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:04 smithi006 bash[27292]: audit 2023-08-28T02:22:03.928065+0000 mon.a (mon.0) 5039 : audit [DBG] from='client.? 172.21.15.6:0/734702659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:05.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:22:04.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:05.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:22:04.726Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:05.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:04 smithi130 bash[27579]: audit 2023-08-28T02:22:03.928065+0000 mon.a (mon.0) 5039 : audit [DBG] from='client.? 172.21.15.6:0/734702659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:05.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:05 smithi130 bash[51302]: debug 2023-08-28T02:22:05.542+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:05.545074+0000) 2023-08-28T02:22:06.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:05 smithi006 bash[27292]: cluster 2023-08-28T02:22:05.511541+0000 mgr.y (mgr.14901) 7904 : cluster [DBG] pgmap v7911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:06.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:05 smithi006 bash[33918]: cluster 2023-08-28T02:22:05.511541+0000 mgr.y (mgr.14901) 7904 : cluster [DBG] pgmap v7911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:06.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:05 smithi130 bash[27579]: cluster 2023-08-28T02:22:05.511541+0000 mgr.y (mgr.14901) 7904 : cluster [DBG] pgmap v7911: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:06.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:06 smithi130 bash[51302]: debug 2023-08-28T02:22:06.542+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:06.545356+0000) 2023-08-28T02:22:07.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:06 smithi006 bash[27292]: audit 2023-08-28T02:22:06.344483+0000 mon.c (mon.2) 3115 : audit [DBG] from='client.? 172.21.15.6:0/352477685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:07.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:06 smithi006 bash[27292]: audit 2023-08-28T02:22:06.774900+0000 mon.b (mon.1) 867 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:22:07.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:06 smithi006 bash[33918]: audit 2023-08-28T02:22:06.344483+0000 mon.c (mon.2) 3115 : audit [DBG] from='client.? 172.21.15.6:0/352477685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:07.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:06 smithi006 bash[33918]: audit 2023-08-28T02:22:06.774900+0000 mon.b (mon.1) 867 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:22:07.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:06 smithi130 bash[27579]: audit 2023-08-28T02:22:06.344483+0000 mon.c (mon.2) 3115 : audit [DBG] from='client.? 172.21.15.6:0/352477685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:07.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:06 smithi130 bash[27579]: audit 2023-08-28T02:22:06.774900+0000 mon.b (mon.1) 867 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:22:07.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:07 smithi130 bash[51302]: debug 2023-08-28T02:22:07.542+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:07.545665+0000) 2023-08-28T02:22:08.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:07 smithi006 bash[27292]: cluster 2023-08-28T02:22:07.512201+0000 mgr.y (mgr.14901) 7905 : cluster [DBG] pgmap v7912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:08.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:07 smithi006 bash[33918]: cluster 2023-08-28T02:22:07.512201+0000 mgr.y (mgr.14901) 7905 : cluster [DBG] pgmap v7912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:08.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:07 smithi130 bash[27579]: cluster 2023-08-28T02:22:07.512201+0000 mgr.y (mgr.14901) 7905 : cluster [DBG] pgmap v7912: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:08.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:08 smithi130 bash[51302]: debug 2023-08-28T02:22:08.542+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:08.545973+0000) 2023-08-28T02:22:09.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:08 smithi006 bash[27292]: audit 2023-08-28T02:22:08.755016+0000 mon.a (mon.0) 5040 : audit [DBG] from='client.? 172.21.15.6:0/3086008134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:09.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:08 smithi006 bash[33918]: audit 2023-08-28T02:22:08.755016+0000 mon.a (mon.0) 5040 : audit [DBG] from='client.? 172.21.15.6:0/3086008134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:09.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:08 smithi130 bash[27579]: audit 2023-08-28T02:22:08.755016+0000 mon.a (mon.0) 5040 : audit [DBG] from='client.? 172.21.15.6:0/3086008134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:09.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:09 smithi130 bash[51302]: debug 2023-08-28T02:22:09.542+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:09.546188+0000) 2023-08-28T02:22:10.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:09 smithi006 bash[27292]: cluster 2023-08-28T02:22:09.513315+0000 mgr.y (mgr.14901) 7906 : cluster [DBG] pgmap v7913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:10.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:09 smithi006 bash[33918]: cluster 2023-08-28T02:22:09.513315+0000 mgr.y (mgr.14901) 7906 : cluster [DBG] pgmap v7913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:10.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:09 smithi130 bash[27579]: cluster 2023-08-28T02:22:09.513315+0000 mgr.y (mgr.14901) 7906 : cluster [DBG] pgmap v7913: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:10.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:10 smithi130 bash[51302]: debug 2023-08-28T02:22:10.542+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:10.546450+0000) 2023-08-28T02:22:11.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:22:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:22:10] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:22:11.545 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:11 smithi130 bash[27579]: audit 2023-08-28T02:22:11.184946+0000 mon.c (mon.2) 3116 : audit [DBG] from='client.? 172.21.15.6:0/2848584916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:11.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:11 smithi006 bash[27292]: audit 2023-08-28T02:22:11.184946+0000 mon.c (mon.2) 3116 : audit [DBG] from='client.? 172.21.15.6:0/2848584916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:11.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:11 smithi006 bash[33918]: audit 2023-08-28T02:22:11.184946+0000 mon.c (mon.2) 3116 : audit [DBG] from='client.? 172.21.15.6:0/2848584916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:22:11.284Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:22:11.287Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:11.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:11 smithi130 bash[51302]: debug 2023-08-28T02:22:11.546+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:11.546775+0000) 2023-08-28T02:22:12.545 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:12 smithi130 bash[27579]: cluster 2023-08-28T02:22:11.513999+0000 mgr.y (mgr.14901) 7907 : cluster [DBG] pgmap v7914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:12.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:12 smithi006 bash[33918]: cluster 2023-08-28T02:22:11.513999+0000 mgr.y (mgr.14901) 7907 : cluster [DBG] pgmap v7914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:12.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:12 smithi006 bash[27292]: cluster 2023-08-28T02:22:11.513999+0000 mgr.y (mgr.14901) 7907 : cluster [DBG] pgmap v7914: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:12.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:12 smithi130 bash[51302]: debug 2023-08-28T02:22:12.546+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:12.547050+0000) 2023-08-28T02:22:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:22:12.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:22:12.847Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:13.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:13 smithi130 bash[27579]: audit 2023-08-28T02:22:12.665125+0000 mon.a (mon.0) 5041 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:22:13.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:13 smithi130 bash[27579]: audit 2023-08-28T02:22:13.052415+0000 mon.a (mon.0) 5042 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:22:13.868 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:13 smithi130 bash[27579]: audit 2023-08-28T02:22:13.601507+0000 mon.c (mon.2) 3117 : audit [DBG] from='client.? 172.21.15.6:0/850011204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:13.868 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:13 smithi130 bash[51302]: debug 2023-08-28T02:22:13.546+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:13.547338+0000) 2023-08-28T02:22:13.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:13 smithi006 bash[27292]: audit 2023-08-28T02:22:12.665125+0000 mon.a (mon.0) 5041 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:22:13.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:13 smithi006 bash[27292]: audit 2023-08-28T02:22:13.052415+0000 mon.a (mon.0) 5042 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:22:13.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:13 smithi006 bash[27292]: audit 2023-08-28T02:22:13.601507+0000 mon.c (mon.2) 3117 : audit [DBG] from='client.? 172.21.15.6:0/850011204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:13.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:13 smithi006 bash[33918]: audit 2023-08-28T02:22:12.665125+0000 mon.a (mon.0) 5041 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:22:13.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:13 smithi006 bash[33918]: audit 2023-08-28T02:22:13.052415+0000 mon.a (mon.0) 5042 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:22:13.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:13 smithi006 bash[33918]: audit 2023-08-28T02:22:13.601507+0000 mon.c (mon.2) 3117 : audit [DBG] from='client.? 172.21.15.6:0/850011204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:14.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:14 smithi130 bash[27579]: cluster 2023-08-28T02:22:13.514600+0000 mgr.y (mgr.14901) 7908 : cluster [DBG] pgmap v7915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:14.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:14 smithi130 bash[51302]: debug 2023-08-28T02:22:14.546+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:14.547614+0000) 2023-08-28T02:22:14.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:14 smithi006 bash[27292]: cluster 2023-08-28T02:22:13.514600+0000 mgr.y (mgr.14901) 7908 : cluster [DBG] pgmap v7915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:14.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:14 smithi006 bash[33918]: cluster 2023-08-28T02:22:13.514600+0000 mgr.y (mgr.14901) 7908 : cluster [DBG] pgmap v7915: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:22:14.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:14.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:22:14.726Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:15.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:15 smithi130 bash[51302]: debug 2023-08-28T02:22:15.546+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:15.547914+0000) 2023-08-28T02:22:16.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:16 smithi130 bash[51302]: debug 2023-08-28T02:22:16.546+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:16.548184+0000) 2023-08-28T02:22:16.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:16 smithi130 bash[27579]: cluster 2023-08-28T02:22:15.515659+0000 mgr.y (mgr.14901) 7909 : cluster [DBG] pgmap v7916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:16.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:16 smithi130 bash[27579]: audit 2023-08-28T02:22:16.030577+0000 mon.a (mon.0) 5043 : audit [DBG] from='client.? 172.21.15.6:0/868678922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:16.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:16 smithi006 bash[33918]: cluster 2023-08-28T02:22:15.515659+0000 mgr.y (mgr.14901) 7909 : cluster [DBG] pgmap v7916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:16.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:16 smithi006 bash[33918]: audit 2023-08-28T02:22:16.030577+0000 mon.a (mon.0) 5043 : audit [DBG] from='client.? 172.21.15.6:0/868678922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:16.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:16 smithi006 bash[27292]: cluster 2023-08-28T02:22:15.515659+0000 mgr.y (mgr.14901) 7909 : cluster [DBG] pgmap v7916: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:16.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:16 smithi006 bash[27292]: audit 2023-08-28T02:22:16.030577+0000 mon.a (mon.0) 5043 : audit [DBG] from='client.? 172.21.15.6:0/868678922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:17.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:17 smithi130 bash[51302]: debug 2023-08-28T02:22:17.546+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:17.548448+0000) 2023-08-28T02:22:18.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:18 smithi130 bash[51302]: debug 2023-08-28T02:22:18.546+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:18.548730+0000) 2023-08-28T02:22:18.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:18 smithi130 bash[27579]: cluster 2023-08-28T02:22:17.516376+0000 mgr.y (mgr.14901) 7910 : cluster [DBG] pgmap v7917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:18.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:18 smithi130 bash[27579]: audit 2023-08-28T02:22:18.447682+0000 mon.c (mon.2) 3118 : audit [DBG] from='client.? 172.21.15.6:0/3418151046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:18.929 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:18 smithi006 bash[33918]: cluster 2023-08-28T02:22:17.516376+0000 mgr.y (mgr.14901) 7910 : cluster [DBG] pgmap v7917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:18.930 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:18 smithi006 bash[33918]: audit 2023-08-28T02:22:18.447682+0000 mon.c (mon.2) 3118 : audit [DBG] from='client.? 172.21.15.6:0/3418151046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:18.930 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:18 smithi006 bash[27292]: cluster 2023-08-28T02:22:17.516376+0000 mgr.y (mgr.14901) 7910 : cluster [DBG] pgmap v7917: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:18.930 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:18 smithi006 bash[27292]: audit 2023-08-28T02:22:18.447682+0000 mon.c (mon.2) 3118 : audit [DBG] from='client.? 172.21.15.6:0/3418151046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:19.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:19 smithi130 bash[51302]: debug 2023-08-28T02:22:19.546+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:19.549000+0000) 2023-08-28T02:22:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:20 smithi130 bash[27579]: cluster 2023-08-28T02:22:19.517150+0000 mgr.y (mgr.14901) 7911 : cluster [DBG] pgmap v7918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:20.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:20 smithi130 bash[51302]: debug 2023-08-28T02:22:20.546+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:20.549284+0000) 2023-08-28T02:22:20.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:20 smithi006 bash[33918]: cluster 2023-08-28T02:22:19.517150+0000 mgr.y (mgr.14901) 7911 : cluster [DBG] pgmap v7918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:20 smithi006 bash[27292]: cluster 2023-08-28T02:22:19.517150+0000 mgr.y (mgr.14901) 7911 : cluster [DBG] pgmap v7918: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:20.965 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:22:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:22:20] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T02:22:21.688 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:22:21.285Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:21.688 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:22:21.287Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:21.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:21 smithi130 bash[27579]: audit 2023-08-28T02:22:20.863200+0000 mon.a (mon.0) 5044 : audit [DBG] from='client.? 172.21.15.6:0/3044072200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:21.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:21 smithi130 bash[51302]: debug 2023-08-28T02:22:21.546+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:21.549595+0000) 2023-08-28T02:22:21.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:21 smithi006 bash[27292]: audit 2023-08-28T02:22:20.863200+0000 mon.a (mon.0) 5044 : audit [DBG] from='client.? 172.21.15.6:0/3044072200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:21.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:21 smithi006 bash[33918]: audit 2023-08-28T02:22:20.863200+0000 mon.a (mon.0) 5044 : audit [DBG] from='client.? 172.21.15.6:0/3044072200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:22 smithi130 bash[27579]: cluster 2023-08-28T02:22:21.517822+0000 mgr.y (mgr.14901) 7912 : cluster [DBG] pgmap v7919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:22.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:22 smithi130 bash[51302]: debug 2023-08-28T02:22:22.546+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:22.549917+0000) 2023-08-28T02:22:22.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:22:22.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:22:22.848Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:22.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:22 smithi006 bash[33918]: cluster 2023-08-28T02:22:21.517822+0000 mgr.y (mgr.14901) 7912 : cluster [DBG] pgmap v7919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:22.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:22 smithi006 bash[27292]: cluster 2023-08-28T02:22:21.517822+0000 mgr.y (mgr.14901) 7912 : cluster [DBG] pgmap v7919: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:23.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:23 smithi130 bash[51302]: debug 2023-08-28T02:22:23.546+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:23.550229+0000) 2023-08-28T02:22:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:24 smithi130 bash[27579]: audit 2023-08-28T02:22:23.064368+0000 mon.a (mon.0) 5045 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:22:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:24 smithi130 bash[27579]: audit 2023-08-28T02:22:23.285781+0000 mon.c (mon.2) 3119 : audit [DBG] from='client.? 172.21.15.6:0/4261997387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:24.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:24 smithi130 bash[27579]: audit 2023-08-28T02:22:23.358027+0000 mon.b (mon.1) 868 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:22:24.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:24 smithi130 bash[27579]: audit 2023-08-28T02:22:23.359705+0000 mon.b (mon.1) 869 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:22:24.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:24 smithi130 bash[27579]: audit 2023-08-28T02:22:23.369168+0000 mon.a (mon.0) 5046 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:22:24.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:24 smithi130 bash[27579]: cluster 2023-08-28T02:22:23.518459+0000 mgr.y (mgr.14901) 7913 : cluster [DBG] pgmap v7920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:24 smithi006 bash[27292]: audit 2023-08-28T02:22:23.064368+0000 mon.a (mon.0) 5045 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:22:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:24 smithi006 bash[27292]: audit 2023-08-28T02:22:23.285781+0000 mon.c (mon.2) 3119 : audit [DBG] from='client.? 172.21.15.6:0/4261997387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:24 smithi006 bash[27292]: audit 2023-08-28T02:22:23.358027+0000 mon.b (mon.1) 868 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:22:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:24 smithi006 bash[27292]: audit 2023-08-28T02:22:23.359705+0000 mon.b (mon.1) 869 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:22:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:24 smithi006 bash[27292]: audit 2023-08-28T02:22:23.369168+0000 mon.a (mon.0) 5046 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:22:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:24 smithi006 bash[27292]: cluster 2023-08-28T02:22:23.518459+0000 mgr.y (mgr.14901) 7913 : cluster [DBG] pgmap v7920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:24.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:24 smithi006 bash[33918]: audit 2023-08-28T02:22:23.064368+0000 mon.a (mon.0) 5045 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:22:24.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:24 smithi006 bash[33918]: audit 2023-08-28T02:22:23.285781+0000 mon.c (mon.2) 3119 : audit [DBG] from='client.? 172.21.15.6:0/4261997387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:24.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:24 smithi006 bash[33918]: audit 2023-08-28T02:22:23.358027+0000 mon.b (mon.1) 868 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:22:24.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:24 smithi006 bash[33918]: audit 2023-08-28T02:22:23.359705+0000 mon.b (mon.1) 869 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:22:24.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:24 smithi006 bash[33918]: audit 2023-08-28T02:22:23.369168+0000 mon.a (mon.0) 5046 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:22:24.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:24 smithi006 bash[33918]: cluster 2023-08-28T02:22:23.518459+0000 mgr.y (mgr.14901) 7913 : cluster [DBG] pgmap v7920: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:24.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:24 smithi130 bash[51302]: debug 2023-08-28T02:22:24.550+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:24.550556+0000) 2023-08-28T02:22:25.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:22:24.725Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:25.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:22:24.727Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:25.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:25 smithi130 bash[51302]: debug 2023-08-28T02:22:25.550+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:25.550813+0000) 2023-08-28T02:22:26.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:26 smithi130 bash[27579]: cluster 2023-08-28T02:22:25.519607+0000 mgr.y (mgr.14901) 7914 : cluster [DBG] pgmap v7921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:26.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:26 smithi130 bash[27579]: audit 2023-08-28T02:22:25.702980+0000 mon.c (mon.2) 3120 : audit [DBG] from='client.? 172.21.15.6:0/3463888652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:26.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:26 smithi130 bash[51302]: debug 2023-08-28T02:22:26.550+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:26.551075+0000) 2023-08-28T02:22:26.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:26 smithi006 bash[27292]: cluster 2023-08-28T02:22:25.519607+0000 mgr.y (mgr.14901) 7914 : cluster [DBG] pgmap v7921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:26.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:26 smithi006 bash[27292]: audit 2023-08-28T02:22:25.702980+0000 mon.c (mon.2) 3120 : audit [DBG] from='client.? 172.21.15.6:0/3463888652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:26.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:26 smithi006 bash[33918]: cluster 2023-08-28T02:22:25.519607+0000 mgr.y (mgr.14901) 7914 : cluster [DBG] pgmap v7921: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:26.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:26 smithi006 bash[33918]: audit 2023-08-28T02:22:25.702980+0000 mon.c (mon.2) 3120 : audit [DBG] from='client.? 172.21.15.6:0/3463888652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:27.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:27 smithi130 bash[51302]: debug 2023-08-28T02:22:27.550+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:27.551335+0000) 2023-08-28T02:22:28.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:28 smithi130 bash[27579]: cluster 2023-08-28T02:22:27.520406+0000 mgr.y (mgr.14901) 7915 : cluster [DBG] pgmap v7922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:28.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:28 smithi130 bash[27579]: audit 2023-08-28T02:22:28.122498+0000 mon.c (mon.2) 3121 : audit [DBG] from='client.? 172.21.15.6:0/2133691286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:28.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:28 smithi130 bash[51302]: debug 2023-08-28T02:22:28.550+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:28.551609+0000) 2023-08-28T02:22:28.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:28 smithi006 bash[33918]: cluster 2023-08-28T02:22:27.520406+0000 mgr.y (mgr.14901) 7915 : cluster [DBG] pgmap v7922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:28.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:28 smithi006 bash[33918]: audit 2023-08-28T02:22:28.122498+0000 mon.c (mon.2) 3121 : audit [DBG] from='client.? 172.21.15.6:0/2133691286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:28.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:28 smithi006 bash[27292]: cluster 2023-08-28T02:22:27.520406+0000 mgr.y (mgr.14901) 7915 : cluster [DBG] pgmap v7922: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:28.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:28 smithi006 bash[27292]: audit 2023-08-28T02:22:28.122498+0000 mon.c (mon.2) 3121 : audit [DBG] from='client.? 172.21.15.6:0/2133691286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:29.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:29 smithi130 bash[51302]: debug 2023-08-28T02:22:29.550+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:29.551894+0000) 2023-08-28T02:22:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:30 smithi130 bash[27579]: cluster 2023-08-28T02:22:29.521567+0000 mgr.y (mgr.14901) 7916 : cluster [DBG] pgmap v7923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:30.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:30 smithi130 bash[27579]: audit 2023-08-28T02:22:30.538286+0000 mon.c (mon.2) 3122 : audit [DBG] from='client.? 172.21.15.6:0/547290394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:30.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:30 smithi130 bash[51302]: debug 2023-08-28T02:22:30.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:30.552224+0000) 2023-08-28T02:22:30.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:22:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:22:30] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-28T02:22:30.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:30 smithi006 bash[33918]: cluster 2023-08-28T02:22:29.521567+0000 mgr.y (mgr.14901) 7916 : cluster [DBG] pgmap v7923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:30.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:30 smithi006 bash[33918]: audit 2023-08-28T02:22:30.538286+0000 mon.c (mon.2) 3122 : audit [DBG] from='client.? 172.21.15.6:0/547290394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:30.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:30 smithi006 bash[27292]: cluster 2023-08-28T02:22:29.521567+0000 mgr.y (mgr.14901) 7916 : cluster [DBG] pgmap v7923: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:30.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:30 smithi006 bash[27292]: audit 2023-08-28T02:22:30.538286+0000 mon.c (mon.2) 3122 : audit [DBG] from='client.? 172.21.15.6:0/547290394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:22:31.286Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:22:31.288Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:31.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:31 smithi130 bash[51302]: debug 2023-08-28T02:22:31.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:31.552549+0000) 2023-08-28T02:22:32.848 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:32 smithi006 bash[27292]: cluster 2023-08-28T02:22:31.522243+0000 mgr.y (mgr.14901) 7917 : cluster [DBG] pgmap v7924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:32.848 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:32 smithi006 bash[33918]: cluster 2023-08-28T02:22:31.522243+0000 mgr.y (mgr.14901) 7917 : cluster [DBG] pgmap v7924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:32 smithi130 bash[27579]: cluster 2023-08-28T02:22:31.522243+0000 mgr.y (mgr.14901) 7917 : cluster [DBG] pgmap v7924: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:32.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:32 smithi130 bash[51302]: debug 2023-08-28T02:22:32.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:32.552837+0000) 2023-08-28T02:22:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:22:32.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:22:32.849Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:33.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:33 smithi130 bash[51302]: debug 2023-08-28T02:22:33.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:33.553088+0000) 2023-08-28T02:22:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:33 smithi006 bash[33918]: audit 2023-08-28T02:22:32.958535+0000 mon.c (mon.2) 3123 : audit [DBG] from='client.? 172.21.15.6:0/1349713514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:33 smithi006 bash[33918]: cluster 2023-08-28T02:22:33.522866+0000 mgr.y (mgr.14901) 7918 : cluster [DBG] pgmap v7925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:33 smithi006 bash[27292]: audit 2023-08-28T02:22:32.958535+0000 mon.c (mon.2) 3123 : audit [DBG] from='client.? 172.21.15.6:0/1349713514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:33 smithi006 bash[27292]: cluster 2023-08-28T02:22:33.522866+0000 mgr.y (mgr.14901) 7918 : cluster [DBG] pgmap v7925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:33 smithi130 bash[27579]: audit 2023-08-28T02:22:32.958535+0000 mon.c (mon.2) 3123 : audit [DBG] from='client.? 172.21.15.6:0/1349713514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:33 smithi130 bash[27579]: cluster 2023-08-28T02:22:33.522866+0000 mgr.y (mgr.14901) 7918 : cluster [DBG] pgmap v7925: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:34.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:34 smithi130 bash[51302]: debug 2023-08-28T02:22:34.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:34.553401+0000) 2023-08-28T02:22:35.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:22:34.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:35.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:22:34.728Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:35.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:35 smithi006 bash[33918]: audit 2023-08-28T02:22:35.385623+0000 mon.a (mon.0) 5047 : audit [DBG] from='client.? 172.21.15.6:0/3930288104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:35.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:35 smithi006 bash[27292]: audit 2023-08-28T02:22:35.385623+0000 mon.a (mon.0) 5047 : audit [DBG] from='client.? 172.21.15.6:0/3930288104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:35.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:35 smithi130 bash[27579]: audit 2023-08-28T02:22:35.385623+0000 mon.a (mon.0) 5047 : audit [DBG] from='client.? 172.21.15.6:0/3930288104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:35.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:35 smithi130 bash[51302]: debug 2023-08-28T02:22:35.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:35.553715+0000) 2023-08-28T02:22:36.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:36 smithi006 bash[27292]: cluster 2023-08-28T02:22:35.523999+0000 mgr.y (mgr.14901) 7919 : cluster [DBG] pgmap v7926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:36.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:36 smithi006 bash[33918]: cluster 2023-08-28T02:22:35.523999+0000 mgr.y (mgr.14901) 7919 : cluster [DBG] pgmap v7926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:36.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:36 smithi130 bash[27579]: cluster 2023-08-28T02:22:35.523999+0000 mgr.y (mgr.14901) 7919 : cluster [DBG] pgmap v7926: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:36.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:36 smithi130 bash[51302]: debug 2023-08-28T02:22:36.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:36.554015+0000) 2023-08-28T02:22:37.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:37 smithi130 bash[51302]: debug 2023-08-28T02:22:37.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:37.554299+0000) 2023-08-28T02:22:38.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:38 smithi130 bash[27579]: cluster 2023-08-28T02:22:37.524613+0000 mgr.y (mgr.14901) 7920 : cluster [DBG] pgmap v7927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:38.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:38 smithi130 bash[27579]: audit 2023-08-28T02:22:37.794478+0000 mon.c (mon.2) 3124 : audit [DBG] from='client.? 172.21.15.6:0/3803967955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:38.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:38 smithi130 bash[51302]: debug 2023-08-28T02:22:38.549+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:38.554561+0000) 2023-08-28T02:22:38.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:38 smithi006 bash[33918]: cluster 2023-08-28T02:22:37.524613+0000 mgr.y (mgr.14901) 7920 : cluster [DBG] pgmap v7927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:38.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:38 smithi006 bash[33918]: audit 2023-08-28T02:22:37.794478+0000 mon.c (mon.2) 3124 : audit [DBG] from='client.? 172.21.15.6:0/3803967955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:38.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:38 smithi006 bash[27292]: cluster 2023-08-28T02:22:37.524613+0000 mgr.y (mgr.14901) 7920 : cluster [DBG] pgmap v7927: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:38.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:38 smithi006 bash[27292]: audit 2023-08-28T02:22:37.794478+0000 mon.c (mon.2) 3124 : audit [DBG] from='client.? 172.21.15.6:0/3803967955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:39.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:39 smithi130 bash[51302]: debug 2023-08-28T02:22:39.553+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:39.554901+0000) 2023-08-28T02:22:40.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:40 smithi130 bash[51302]: debug 2023-08-28T02:22:40.553+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:40.555123+0000) 2023-08-28T02:22:40.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:40 smithi130 bash[27579]: cluster 2023-08-28T02:22:39.525720+0000 mgr.y (mgr.14901) 7921 : cluster [DBG] pgmap v7928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:40.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:40 smithi130 bash[27579]: audit 2023-08-28T02:22:40.206115+0000 mon.c (mon.2) 3125 : audit [DBG] from='client.? 172.21.15.6:0/2373442742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:40.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:22:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:22:40] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-28T02:22:40.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:40 smithi006 bash[27292]: cluster 2023-08-28T02:22:39.525720+0000 mgr.y (mgr.14901) 7921 : cluster [DBG] pgmap v7928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:40.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:40 smithi006 bash[27292]: audit 2023-08-28T02:22:40.206115+0000 mon.c (mon.2) 3125 : audit [DBG] from='client.? 172.21.15.6:0/2373442742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:40.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:40 smithi006 bash[33918]: cluster 2023-08-28T02:22:39.525720+0000 mgr.y (mgr.14901) 7921 : cluster [DBG] pgmap v7928: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:40.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:40 smithi006 bash[33918]: audit 2023-08-28T02:22:40.206115+0000 mon.c (mon.2) 3125 : audit [DBG] from='client.? 172.21.15.6:0/2373442742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:22:41.287Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:22:41.289Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:41.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:41 smithi130 bash[51302]: debug 2023-08-28T02:22:41.553+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:41.555410+0000) 2023-08-28T02:22:42.847 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:42 smithi006 bash[33918]: cluster 2023-08-28T02:22:41.526436+0000 mgr.y (mgr.14901) 7922 : cluster [DBG] pgmap v7929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:42.848 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:42 smithi006 bash[27292]: cluster 2023-08-28T02:22:41.526436+0000 mgr.y (mgr.14901) 7922 : cluster [DBG] pgmap v7929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:42.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:42 smithi130 bash[27579]: cluster 2023-08-28T02:22:41.526436+0000 mgr.y (mgr.14901) 7922 : cluster [DBG] pgmap v7929: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:42.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:42 smithi130 bash[51302]: debug 2023-08-28T02:22:42.553+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:42.555680+0000) 2023-08-28T02:22:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:22:42.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:22:42.849Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:43.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:43 smithi130 bash[27579]: audit 2023-08-28T02:22:42.623229+0000 mon.a (mon.0) 5048 : audit [DBG] from='client.? 172.21.15.6:0/2666155230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:43.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:43 smithi130 bash[51302]: debug 2023-08-28T02:22:43.553+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:43.555966+0000) 2023-08-28T02:22:43.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:43 smithi006 bash[33918]: audit 2023-08-28T02:22:42.623229+0000 mon.a (mon.0) 5048 : audit [DBG] from='client.? 172.21.15.6:0/2666155230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:43.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:43 smithi006 bash[27292]: audit 2023-08-28T02:22:42.623229+0000 mon.a (mon.0) 5048 : audit [DBG] from='client.? 172.21.15.6:0/2666155230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:44.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:44 smithi130 bash[27579]: cluster 2023-08-28T02:22:43.527263+0000 mgr.y (mgr.14901) 7923 : cluster [DBG] pgmap v7930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:44.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:44 smithi130 bash[51302]: debug 2023-08-28T02:22:44.553+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:44.556251+0000) 2023-08-28T02:22:44.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:22:44.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:22:44.728Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:44.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:44 smithi006 bash[27292]: cluster 2023-08-28T02:22:43.527263+0000 mgr.y (mgr.14901) 7923 : cluster [DBG] pgmap v7930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:44.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:44 smithi006 bash[33918]: cluster 2023-08-28T02:22:43.527263+0000 mgr.y (mgr.14901) 7923 : cluster [DBG] pgmap v7930: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:45.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:45 smithi130 bash[27579]: audit 2023-08-28T02:22:45.044492+0000 mon.c (mon.2) 3126 : audit [DBG] from='client.? 172.21.15.6:0/213675990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:45.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:45 smithi130 bash[51302]: debug 2023-08-28T02:22:45.553+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:45.556521+0000) 2023-08-28T02:22:45.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:45 smithi006 bash[27292]: audit 2023-08-28T02:22:45.044492+0000 mon.c (mon.2) 3126 : audit [DBG] from='client.? 172.21.15.6:0/213675990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:45.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:45 smithi006 bash[33918]: audit 2023-08-28T02:22:45.044492+0000 mon.c (mon.2) 3126 : audit [DBG] from='client.? 172.21.15.6:0/213675990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:46.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:46 smithi130 bash[51302]: debug 2023-08-28T02:22:46.553+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:46.556789+0000) 2023-08-28T02:22:46.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:46 smithi130 bash[27579]: cluster 2023-08-28T02:22:45.528431+0000 mgr.y (mgr.14901) 7924 : cluster [DBG] pgmap v7931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:46.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:46 smithi006 bash[27292]: cluster 2023-08-28T02:22:45.528431+0000 mgr.y (mgr.14901) 7924 : cluster [DBG] pgmap v7931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:46.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:46 smithi006 bash[33918]: cluster 2023-08-28T02:22:45.528431+0000 mgr.y (mgr.14901) 7924 : cluster [DBG] pgmap v7931: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:47.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:47 smithi130 bash[27579]: audit 2023-08-28T02:22:47.460450+0000 mon.c (mon.2) 3127 : audit [DBG] from='client.? 172.21.15.6:0/3701575516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:47.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:47 smithi130 bash[51302]: debug 2023-08-28T02:22:47.553+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:47.557085+0000) 2023-08-28T02:22:47.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:47 smithi006 bash[27292]: audit 2023-08-28T02:22:47.460450+0000 mon.c (mon.2) 3127 : audit [DBG] from='client.? 172.21.15.6:0/3701575516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:47.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:47 smithi006 bash[33918]: audit 2023-08-28T02:22:47.460450+0000 mon.c (mon.2) 3127 : audit [DBG] from='client.? 172.21.15.6:0/3701575516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:48.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:48 smithi130 bash[27579]: cluster 2023-08-28T02:22:47.529092+0000 mgr.y (mgr.14901) 7925 : cluster [DBG] pgmap v7932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:48.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:48 smithi130 bash[51302]: debug 2023-08-28T02:22:48.553+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:48.557439+0000) 2023-08-28T02:22:48.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:48 smithi006 bash[33918]: cluster 2023-08-28T02:22:47.529092+0000 mgr.y (mgr.14901) 7925 : cluster [DBG] pgmap v7932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:48.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:48 smithi006 bash[27292]: cluster 2023-08-28T02:22:47.529092+0000 mgr.y (mgr.14901) 7925 : cluster [DBG] pgmap v7932: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:49.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:49 smithi130 bash[51302]: debug 2023-08-28T02:22:49.553+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:49.557725+0000) 2023-08-28T02:22:50.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:50 smithi130 bash[51302]: debug 2023-08-28T02:22:50.557+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:50.558198+0000) 2023-08-28T02:22:50.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:50 smithi130 bash[27579]: cluster 2023-08-28T02:22:49.530142+0000 mgr.y (mgr.14901) 7926 : cluster [DBG] pgmap v7933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:50.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:50 smithi130 bash[27579]: audit 2023-08-28T02:22:49.878067+0000 mon.c (mon.2) 3128 : audit [DBG] from='client.? 172.21.15.6:0/3497659101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:50.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:22:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:22:50] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T02:22:50.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:50 smithi006 bash[33918]: cluster 2023-08-28T02:22:49.530142+0000 mgr.y (mgr.14901) 7926 : cluster [DBG] pgmap v7933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:50.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:50 smithi006 bash[33918]: audit 2023-08-28T02:22:49.878067+0000 mon.c (mon.2) 3128 : audit [DBG] from='client.? 172.21.15.6:0/3497659101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:50.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:50 smithi006 bash[27292]: cluster 2023-08-28T02:22:49.530142+0000 mgr.y (mgr.14901) 7926 : cluster [DBG] pgmap v7933: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:50.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:50 smithi006 bash[27292]: audit 2023-08-28T02:22:49.878067+0000 mon.c (mon.2) 3128 : audit [DBG] from='client.? 172.21.15.6:0/3497659101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:22:51.287Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:22:51.289Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:51.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:51 smithi130 bash[51302]: debug 2023-08-28T02:22:51.557+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:51.558551+0000) 2023-08-28T02:22:52.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:52 smithi130 bash[51302]: debug 2023-08-28T02:22:52.553+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:52.558780+0000) 2023-08-28T02:22:52.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:52 smithi130 bash[27579]: cluster 2023-08-28T02:22:51.530870+0000 mgr.y (mgr.14901) 7927 : cluster [DBG] pgmap v7934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:52.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:52 smithi130 bash[27579]: audit 2023-08-28T02:22:52.297351+0000 mon.a (mon.0) 5049 : audit [DBG] from='client.? 172.21.15.6:0/1015205000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:52.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:52 smithi006 bash[33918]: cluster 2023-08-28T02:22:51.530870+0000 mgr.y (mgr.14901) 7927 : cluster [DBG] pgmap v7934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:52.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:52 smithi006 bash[33918]: audit 2023-08-28T02:22:52.297351+0000 mon.a (mon.0) 5049 : audit [DBG] from='client.? 172.21.15.6:0/1015205000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:22:52.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:22:52.849Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:52.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:52 smithi006 bash[27292]: cluster 2023-08-28T02:22:51.530870+0000 mgr.y (mgr.14901) 7927 : cluster [DBG] pgmap v7934: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:52.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:52 smithi006 bash[27292]: audit 2023-08-28T02:22:52.297351+0000 mon.a (mon.0) 5049 : audit [DBG] from='client.? 172.21.15.6:0/1015205000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:53.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:53 smithi130 bash[51302]: debug 2023-08-28T02:22:53.557+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:53.559051+0000) 2023-08-28T02:22:54.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:53 smithi006 bash[33918]: cluster 2023-08-28T02:22:53.531568+0000 mgr.y (mgr.14901) 7928 : cluster [DBG] pgmap v7935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:54.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:53 smithi006 bash[27292]: cluster 2023-08-28T02:22:53.531568+0000 mgr.y (mgr.14901) 7928 : cluster [DBG] pgmap v7935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:53 smithi130 bash[27579]: cluster 2023-08-28T02:22:53.531568+0000 mgr.y (mgr.14901) 7928 : cluster [DBG] pgmap v7935: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:54.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:54 smithi130 bash[51302]: debug 2023-08-28T02:22:54.557+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:54.559369+0000) 2023-08-28T02:22:55.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:54 smithi006 bash[27292]: audit 2023-08-28T02:22:54.717491+0000 mon.a (mon.0) 5050 : audit [DBG] from='client.? 172.21.15.6:0/3945818943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:55.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:54 smithi006 bash[33918]: audit 2023-08-28T02:22:54.717491+0000 mon.a (mon.0) 5050 : audit [DBG] from='client.? 172.21.15.6:0/3945818943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:55.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:22:54.727Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:55.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:22:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:22:54.729Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:22:55.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:54 smithi130 bash[27579]: audit 2023-08-28T02:22:54.717491+0000 mon.a (mon.0) 5050 : audit [DBG] from='client.? 172.21.15.6:0/3945818943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:55.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:55 smithi130 bash[51302]: debug 2023-08-28T02:22:55.557+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:55.559633+0000) 2023-08-28T02:22:56.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:55 smithi006 bash[27292]: cluster 2023-08-28T02:22:55.532673+0000 mgr.y (mgr.14901) 7929 : cluster [DBG] pgmap v7936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:56.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:55 smithi006 bash[33918]: cluster 2023-08-28T02:22:55.532673+0000 mgr.y (mgr.14901) 7929 : cluster [DBG] pgmap v7936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:56.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:55 smithi130 bash[27579]: cluster 2023-08-28T02:22:55.532673+0000 mgr.y (mgr.14901) 7929 : cluster [DBG] pgmap v7936: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:56.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:56 smithi130 bash[51302]: debug 2023-08-28T02:22:56.557+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:56.559963+0000) 2023-08-28T02:22:57.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:57 smithi006 bash[27292]: audit 2023-08-28T02:22:57.135259+0000 mon.a (mon.0) 5051 : audit [DBG] from='client.? 172.21.15.6:0/569194253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:57.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:57 smithi006 bash[33918]: audit 2023-08-28T02:22:57.135259+0000 mon.a (mon.0) 5051 : audit [DBG] from='client.? 172.21.15.6:0/569194253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:57.559 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:57 smithi130 bash[27579]: audit 2023-08-28T02:22:57.135259+0000 mon.a (mon.0) 5051 : audit [DBG] from='client.? 172.21.15.6:0/569194253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:57.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:57 smithi130 bash[51302]: debug 2023-08-28T02:22:57.557+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:57.560270+0000) 2023-08-28T02:22:58.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:58 smithi006 bash[33918]: cluster 2023-08-28T02:22:57.533364+0000 mgr.y (mgr.14901) 7930 : cluster [DBG] pgmap v7937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:58.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:58 smithi006 bash[27292]: cluster 2023-08-28T02:22:57.533364+0000 mgr.y (mgr.14901) 7930 : cluster [DBG] pgmap v7937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:58.559 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:58 smithi130 bash[27579]: cluster 2023-08-28T02:22:57.533364+0000 mgr.y (mgr.14901) 7930 : cluster [DBG] pgmap v7937: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:22:58.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:58 smithi130 bash[51302]: debug 2023-08-28T02:22:58.557+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:58.560539+0000) 2023-08-28T02:22:59.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:22:59 smithi130 bash[27579]: audit 2023-08-28T02:22:59.548878+0000 mon.c (mon.2) 3129 : audit [DBG] from='client.? 172.21.15.6:0/3868676022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:59.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:22:59 smithi130 bash[51302]: debug 2023-08-28T02:22:59.557+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:59.560821+0000) 2023-08-28T02:22:59.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:22:59 smithi006 bash[27292]: audit 2023-08-28T02:22:59.548878+0000 mon.c (mon.2) 3129 : audit [DBG] from='client.? 172.21.15.6:0/3868676022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:59.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:22:59 smithi006 bash[33918]: audit 2023-08-28T02:22:59.548878+0000 mon.c (mon.2) 3129 : audit [DBG] from='client.? 172.21.15.6:0/3868676022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:00 smithi130 bash[27579]: cluster 2023-08-28T02:22:59.534543+0000 mgr.y (mgr.14901) 7931 : cluster [DBG] pgmap v7938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:00.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:00 smithi130 bash[51302]: debug 2023-08-28T02:23:00.557+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:00.561112+0000) 2023-08-28T02:23:00.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:00 smithi006 bash[27292]: cluster 2023-08-28T02:22:59.534543+0000 mgr.y (mgr.14901) 7931 : cluster [DBG] pgmap v7938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:00.965 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:23:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:23:00] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-28T02:23:00.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:00 smithi006 bash[33918]: cluster 2023-08-28T02:22:59.534543+0000 mgr.y (mgr.14901) 7931 : cluster [DBG] pgmap v7938: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:23:01.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:23:01.290Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:01.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:01 smithi130 bash[51302]: debug 2023-08-28T02:23:01.557+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:01.561404+0000) 2023-08-28T02:23:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:02 smithi130 bash[27579]: cluster 2023-08-28T02:23:01.535217+0000 mgr.y (mgr.14901) 7932 : cluster [DBG] pgmap v7939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:02 smithi130 bash[27579]: audit 2023-08-28T02:23:01.966615+0000 mon.a (mon.0) 5052 : audit [DBG] from='client.? 172.21.15.6:0/1602590586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:02.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:02 smithi130 bash[51302]: debug 2023-08-28T02:23:02.557+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:02.561714+0000) 2023-08-28T02:23:02.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:02 smithi006 bash[27292]: cluster 2023-08-28T02:23:01.535217+0000 mgr.y (mgr.14901) 7932 : cluster [DBG] pgmap v7939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:02.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:02 smithi006 bash[27292]: audit 2023-08-28T02:23:01.966615+0000 mon.a (mon.0) 5052 : audit [DBG] from='client.? 172.21.15.6:0/1602590586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:02.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:02 smithi006 bash[33918]: cluster 2023-08-28T02:23:01.535217+0000 mgr.y (mgr.14901) 7932 : cluster [DBG] pgmap v7939: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:02.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:02 smithi006 bash[33918]: audit 2023-08-28T02:23:01.966615+0000 mon.a (mon.0) 5052 : audit [DBG] from='client.? 172.21.15.6:0/1602590586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:23:02.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:23:02.850Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:03.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:03 smithi130 bash[51302]: debug 2023-08-28T02:23:03.557+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:03.561987+0000) 2023-08-28T02:23:04.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:03 smithi006 bash[27292]: cluster 2023-08-28T02:23:03.535923+0000 mgr.y (mgr.14901) 7933 : cluster [DBG] pgmap v7940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:04.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:03 smithi006 bash[33918]: cluster 2023-08-28T02:23:03.535923+0000 mgr.y (mgr.14901) 7933 : cluster [DBG] pgmap v7940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:03 smithi130 bash[27579]: cluster 2023-08-28T02:23:03.535923+0000 mgr.y (mgr.14901) 7933 : cluster [DBG] pgmap v7940: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:04.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:04 smithi130 bash[51302]: debug 2023-08-28T02:23:04.561+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:04.562320+0000) 2023-08-28T02:23:05.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:04 smithi006 bash[33918]: audit 2023-08-28T02:23:04.388474+0000 mon.a (mon.0) 5053 : audit [DBG] from='client.? 172.21.15.6:0/3608257368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:05.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:23:04.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:05.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:23:04.730Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:05.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:04 smithi006 bash[27292]: audit 2023-08-28T02:23:04.388474+0000 mon.a (mon.0) 5053 : audit [DBG] from='client.? 172.21.15.6:0/3608257368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:05.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:04 smithi130 bash[27579]: audit 2023-08-28T02:23:04.388474+0000 mon.a (mon.0) 5053 : audit [DBG] from='client.? 172.21.15.6:0/3608257368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:05.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:05 smithi130 bash[51302]: debug 2023-08-28T02:23:05.557+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:05.562628+0000) 2023-08-28T02:23:06.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:05 smithi006 bash[33918]: cluster 2023-08-28T02:23:05.537193+0000 mgr.y (mgr.14901) 7934 : cluster [DBG] pgmap v7941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:06.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:05 smithi006 bash[27292]: cluster 2023-08-28T02:23:05.537193+0000 mgr.y (mgr.14901) 7934 : cluster [DBG] pgmap v7941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:06.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:05 smithi130 bash[27579]: cluster 2023-08-28T02:23:05.537193+0000 mgr.y (mgr.14901) 7934 : cluster [DBG] pgmap v7941: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:06.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:06 smithi130 bash[51302]: debug 2023-08-28T02:23:06.561+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:06.562970+0000) 2023-08-28T02:23:07.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:06 smithi006 bash[33918]: audit 2023-08-28T02:23:06.797065+0000 mon.c (mon.2) 3130 : audit [DBG] from='client.? 172.21.15.6:0/2090215087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:07.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:06 smithi006 bash[27292]: audit 2023-08-28T02:23:06.797065+0000 mon.c (mon.2) 3130 : audit [DBG] from='client.? 172.21.15.6:0/2090215087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:07.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:06 smithi130 bash[27579]: audit 2023-08-28T02:23:06.797065+0000 mon.c (mon.2) 3130 : audit [DBG] from='client.? 172.21.15.6:0/2090215087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:07.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:07 smithi130 bash[51302]: debug 2023-08-28T02:23:07.561+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:07.563211+0000) 2023-08-28T02:23:08.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:07 smithi130 bash[27579]: cluster 2023-08-28T02:23:07.537865+0000 mgr.y (mgr.14901) 7935 : cluster [DBG] pgmap v7942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:08.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:07 smithi006 bash[27292]: cluster 2023-08-28T02:23:07.537865+0000 mgr.y (mgr.14901) 7935 : cluster [DBG] pgmap v7942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:08.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:07 smithi006 bash[33918]: cluster 2023-08-28T02:23:07.537865+0000 mgr.y (mgr.14901) 7935 : cluster [DBG] pgmap v7942: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:08.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:08 smithi130 bash[51302]: debug 2023-08-28T02:23:08.561+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:08.563550+0000) 2023-08-28T02:23:09.562 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:09 smithi130 bash[27579]: audit 2023-08-28T02:23:09.212753+0000 mon.c (mon.2) 3131 : audit [DBG] from='client.? 172.21.15.6:0/600203444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:09.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:09 smithi006 bash[33918]: audit 2023-08-28T02:23:09.212753+0000 mon.c (mon.2) 3131 : audit [DBG] from='client.? 172.21.15.6:0/600203444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:09.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:09 smithi006 bash[27292]: audit 2023-08-28T02:23:09.212753+0000 mon.c (mon.2) 3131 : audit [DBG] from='client.? 172.21.15.6:0/600203444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:09.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:09 smithi130 bash[51302]: debug 2023-08-28T02:23:09.561+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:09.563850+0000) 2023-08-28T02:23:10.562 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:10 smithi130 bash[27579]: cluster 2023-08-28T02:23:09.538998+0000 mgr.y (mgr.14901) 7936 : cluster [DBG] pgmap v7943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:10.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:10 smithi006 bash[33918]: cluster 2023-08-28T02:23:09.538998+0000 mgr.y (mgr.14901) 7936 : cluster [DBG] pgmap v7943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:10.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:10 smithi006 bash[27292]: cluster 2023-08-28T02:23:09.538998+0000 mgr.y (mgr.14901) 7936 : cluster [DBG] pgmap v7943: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:10.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:10 smithi130 bash[51302]: debug 2023-08-28T02:23:10.561+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:10.564129+0000) 2023-08-28T02:23:11.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:23:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:23:10] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-28T02:23:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:23:11.288Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:23:11.290Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:11.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:11 smithi130 bash[51302]: debug 2023-08-28T02:23:11.561+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:11.564403+0000) 2023-08-28T02:23:12.849 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:12 smithi006 bash[27292]: cluster 2023-08-28T02:23:11.539639+0000 mgr.y (mgr.14901) 7937 : cluster [DBG] pgmap v7944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:12.849 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:12 smithi006 bash[27292]: audit 2023-08-28T02:23:11.630833+0000 mon.c (mon.2) 3132 : audit [DBG] from='client.? 172.21.15.6:0/2222681375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:12.850 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:12 smithi006 bash[33918]: cluster 2023-08-28T02:23:11.539639+0000 mgr.y (mgr.14901) 7937 : cluster [DBG] pgmap v7944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:12.850 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:12 smithi006 bash[33918]: audit 2023-08-28T02:23:11.630833+0000 mon.c (mon.2) 3132 : audit [DBG] from='client.? 172.21.15.6:0/2222681375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:12.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:12 smithi130 bash[51302]: debug 2023-08-28T02:23:12.561+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:12.564609+0000) 2023-08-28T02:23:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:12 smithi130 bash[27579]: cluster 2023-08-28T02:23:11.539639+0000 mgr.y (mgr.14901) 7937 : cluster [DBG] pgmap v7944: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:12 smithi130 bash[27579]: audit 2023-08-28T02:23:11.630833+0000 mon.c (mon.2) 3132 : audit [DBG] from='client.? 172.21.15.6:0/2222681375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:23:12.849Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:23:12.851Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:13.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:13 smithi130 bash[51302]: debug 2023-08-28T02:23:13.561+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:13.564936+0000) 2023-08-28T02:23:14.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:13 smithi006 bash[27292]: cluster 2023-08-28T02:23:13.540310+0000 mgr.y (mgr.14901) 7938 : cluster [DBG] pgmap v7945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:13 smithi006 bash[33918]: cluster 2023-08-28T02:23:13.540310+0000 mgr.y (mgr.14901) 7938 : cluster [DBG] pgmap v7945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:14.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:13 smithi130 bash[27579]: cluster 2023-08-28T02:23:13.540310+0000 mgr.y (mgr.14901) 7938 : cluster [DBG] pgmap v7945: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:14.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:14 smithi130 bash[51302]: debug 2023-08-28T02:23:14.561+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:14.565271+0000) 2023-08-28T02:23:15.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:14 smithi006 bash[27292]: audit 2023-08-28T02:23:14.058458+0000 mon.c (mon.2) 3133 : audit [DBG] from='client.? 172.21.15.6:0/3644371628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:15.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:14 smithi006 bash[33918]: audit 2023-08-28T02:23:14.058458+0000 mon.c (mon.2) 3133 : audit [DBG] from='client.? 172.21.15.6:0/3644371628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:15.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:23:14.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:15.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:23:14.731Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:15.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:14 smithi130 bash[27579]: audit 2023-08-28T02:23:14.058458+0000 mon.c (mon.2) 3133 : audit [DBG] from='client.? 172.21.15.6:0/3644371628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:15.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:15 smithi130 bash[51302]: debug 2023-08-28T02:23:15.565+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:15.565708+0000) 2023-08-28T02:23:16.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:15 smithi006 bash[33918]: cluster 2023-08-28T02:23:15.541597+0000 mgr.y (mgr.14901) 7939 : cluster [DBG] pgmap v7946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:16.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:15 smithi006 bash[27292]: cluster 2023-08-28T02:23:15.541597+0000 mgr.y (mgr.14901) 7939 : cluster [DBG] pgmap v7946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:16.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:15 smithi130 bash[27579]: cluster 2023-08-28T02:23:15.541597+0000 mgr.y (mgr.14901) 7939 : cluster [DBG] pgmap v7946: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:16.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:16 smithi130 bash[51302]: debug 2023-08-28T02:23:16.561+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:16.565965+0000) 2023-08-28T02:23:17.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:16 smithi130 bash[27579]: audit 2023-08-28T02:23:16.470650+0000 mon.c (mon.2) 3134 : audit [DBG] from='client.? 172.21.15.6:0/3492677437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:17.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:16 smithi006 bash[33918]: audit 2023-08-28T02:23:16.470650+0000 mon.c (mon.2) 3134 : audit [DBG] from='client.? 172.21.15.6:0/3492677437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:17.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:16 smithi006 bash[27292]: audit 2023-08-28T02:23:16.470650+0000 mon.c (mon.2) 3134 : audit [DBG] from='client.? 172.21.15.6:0/3492677437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:17.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:17 smithi130 bash[51302]: debug 2023-08-28T02:23:17.561+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:17.566230+0000) 2023-08-28T02:23:18.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:17 smithi130 bash[27579]: cluster 2023-08-28T02:23:17.542275+0000 mgr.y (mgr.14901) 7940 : cluster [DBG] pgmap v7947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:18.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:17 smithi006 bash[33918]: cluster 2023-08-28T02:23:17.542275+0000 mgr.y (mgr.14901) 7940 : cluster [DBG] pgmap v7947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:18.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:17 smithi006 bash[27292]: cluster 2023-08-28T02:23:17.542275+0000 mgr.y (mgr.14901) 7940 : cluster [DBG] pgmap v7947: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:18.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:18 smithi130 bash[51302]: debug 2023-08-28T02:23:18.565+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:18.566500+0000) 2023-08-28T02:23:19.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:18 smithi130 bash[27579]: audit 2023-08-28T02:23:18.889002+0000 mon.c (mon.2) 3135 : audit [DBG] from='client.? 172.21.15.6:0/1774957198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:19.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:18 smithi006 bash[27292]: audit 2023-08-28T02:23:18.889002+0000 mon.c (mon.2) 3135 : audit [DBG] from='client.? 172.21.15.6:0/1774957198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:19.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:18 smithi006 bash[33918]: audit 2023-08-28T02:23:18.889002+0000 mon.c (mon.2) 3135 : audit [DBG] from='client.? 172.21.15.6:0/1774957198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:19.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:19 smithi130 bash[51302]: debug 2023-08-28T02:23:19.565+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:19.566793+0000) 2023-08-28T02:23:20.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:19 smithi130 bash[27579]: cluster 2023-08-28T02:23:19.543336+0000 mgr.y (mgr.14901) 7941 : cluster [DBG] pgmap v7948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:20.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:19 smithi006 bash[27292]: cluster 2023-08-28T02:23:19.543336+0000 mgr.y (mgr.14901) 7941 : cluster [DBG] pgmap v7948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:20.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:19 smithi006 bash[33918]: cluster 2023-08-28T02:23:19.543336+0000 mgr.y (mgr.14901) 7941 : cluster [DBG] pgmap v7948: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:20.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:20 smithi130 bash[51302]: debug 2023-08-28T02:23:20.565+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:20.567116+0000) 2023-08-28T02:23:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:23:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:23:20] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-28T02:23:21.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:21 smithi130 bash[27579]: audit 2023-08-28T02:23:21.296886+0000 mon.c (mon.2) 3136 : audit [DBG] from='client.? 172.21.15.6:0/2583689440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:21.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:21 smithi130 bash[51302]: debug 2023-08-28T02:23:21.565+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:21.567407+0000) 2023-08-28T02:23:21.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:21 smithi006 bash[27292]: audit 2023-08-28T02:23:21.296886+0000 mon.c (mon.2) 3136 : audit [DBG] from='client.? 172.21.15.6:0/2583689440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:23:21.289Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:23:21.290Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:21.716 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:21 smithi006 bash[33918]: audit 2023-08-28T02:23:21.296886+0000 mon.c (mon.2) 3136 : audit [DBG] from='client.? 172.21.15.6:0/2583689440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:22 smithi130 bash[27579]: cluster 2023-08-28T02:23:21.544016+0000 mgr.y (mgr.14901) 7942 : cluster [DBG] pgmap v7949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:22.616 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:22 smithi130 bash[51302]: debug 2023-08-28T02:23:22.565+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:22.567682+0000) 2023-08-28T02:23:22.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:22 smithi006 bash[33918]: cluster 2023-08-28T02:23:21.544016+0000 mgr.y (mgr.14901) 7942 : cluster [DBG] pgmap v7949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:22.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:22 smithi006 bash[27292]: cluster 2023-08-28T02:23:21.544016+0000 mgr.y (mgr.14901) 7942 : cluster [DBG] pgmap v7949: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:23:22.850Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:23:22.852Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:23.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:23 smithi130 bash[51302]: debug 2023-08-28T02:23:23.565+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:23.567881+0000) 2023-08-28T02:23:24.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:23 smithi006 bash[33918]: audit 2023-08-28T02:23:23.379285+0000 mon.b (mon.1) 870 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:23:24.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:23 smithi006 bash[33918]: cluster 2023-08-28T02:23:23.544663+0000 mgr.y (mgr.14901) 7943 : cluster [DBG] pgmap v7950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:24.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:23 smithi006 bash[33918]: audit 2023-08-28T02:23:23.657313+0000 mon.a (mon.0) 5054 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:23:24.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:23 smithi006 bash[33918]: audit 2023-08-28T02:23:23.693224+0000 mon.a (mon.0) 5055 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:23:24.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:23 smithi006 bash[33918]: audit 2023-08-28T02:23:23.787023+0000 mon.c (mon.2) 3137 : audit [DBG] from='client.? 172.21.15.6:0/1107151213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:23 smithi006 bash[27292]: audit 2023-08-28T02:23:23.379285+0000 mon.b (mon.1) 870 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:23:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:23 smithi006 bash[27292]: cluster 2023-08-28T02:23:23.544663+0000 mgr.y (mgr.14901) 7943 : cluster [DBG] pgmap v7950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:24.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:23 smithi006 bash[27292]: audit 2023-08-28T02:23:23.657313+0000 mon.a (mon.0) 5054 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:23:24.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:23 smithi006 bash[27292]: audit 2023-08-28T02:23:23.693224+0000 mon.a (mon.0) 5055 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:23:24.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:23 smithi006 bash[27292]: audit 2023-08-28T02:23:23.787023+0000 mon.c (mon.2) 3137 : audit [DBG] from='client.? 172.21.15.6:0/1107151213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:23 smithi130 bash[27579]: audit 2023-08-28T02:23:23.379285+0000 mon.b (mon.1) 870 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:23:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:23 smithi130 bash[27579]: cluster 2023-08-28T02:23:23.544663+0000 mgr.y (mgr.14901) 7943 : cluster [DBG] pgmap v7950: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:24.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:23 smithi130 bash[27579]: audit 2023-08-28T02:23:23.657313+0000 mon.a (mon.0) 5054 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:23:24.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:23 smithi130 bash[27579]: audit 2023-08-28T02:23:23.693224+0000 mon.a (mon.0) 5055 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:23:24.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:23 smithi130 bash[27579]: audit 2023-08-28T02:23:23.787023+0000 mon.c (mon.2) 3137 : audit [DBG] from='client.? 172.21.15.6:0/1107151213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:24.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:24 smithi130 bash[51302]: debug 2023-08-28T02:23:24.565+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:24.568144+0000) 2023-08-28T02:23:25.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:23:24.730Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:25.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:23:24.731Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:25.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:25 smithi130 bash[51302]: debug 2023-08-28T02:23:25.564+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:25.568439+0000) 2023-08-28T02:23:26.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:26 smithi130 bash[27579]: cluster 2023-08-28T02:23:25.545793+0000 mgr.y (mgr.14901) 7944 : cluster [DBG] pgmap v7951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:26.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:26 smithi130 bash[27579]: audit 2023-08-28T02:23:26.204814+0000 mon.a (mon.0) 5056 : audit [DBG] from='client.? 172.21.15.6:0/374360217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:26.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:26 smithi130 bash[51302]: debug 2023-08-28T02:23:26.564+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:26.568711+0000) 2023-08-28T02:23:26.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:26 smithi006 bash[33918]: cluster 2023-08-28T02:23:25.545793+0000 mgr.y (mgr.14901) 7944 : cluster [DBG] pgmap v7951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:26.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:26 smithi006 bash[33918]: audit 2023-08-28T02:23:26.204814+0000 mon.a (mon.0) 5056 : audit [DBG] from='client.? 172.21.15.6:0/374360217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:26.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:26 smithi006 bash[27292]: cluster 2023-08-28T02:23:25.545793+0000 mgr.y (mgr.14901) 7944 : cluster [DBG] pgmap v7951: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:26.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:26 smithi006 bash[27292]: audit 2023-08-28T02:23:26.204814+0000 mon.a (mon.0) 5056 : audit [DBG] from='client.? 172.21.15.6:0/374360217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:27.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:27 smithi130 bash[51302]: debug 2023-08-28T02:23:27.564+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:27.568973+0000) 2023-08-28T02:23:28.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:28 smithi130 bash[27579]: cluster 2023-08-28T02:23:27.546441+0000 mgr.y (mgr.14901) 7945 : cluster [DBG] pgmap v7952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:28.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:28 smithi130 bash[51302]: debug 2023-08-28T02:23:28.564+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:28.569165+0000) 2023-08-28T02:23:28.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:28 smithi006 bash[27292]: cluster 2023-08-28T02:23:27.546441+0000 mgr.y (mgr.14901) 7945 : cluster [DBG] pgmap v7952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:28.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:28 smithi006 bash[33918]: cluster 2023-08-28T02:23:27.546441+0000 mgr.y (mgr.14901) 7945 : cluster [DBG] pgmap v7952: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:29.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:29 smithi130 bash[27579]: audit 2023-08-28T02:23:28.625222+0000 mon.a (mon.0) 5057 : audit [DBG] from='client.? 172.21.15.6:0/4144465003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:29.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:29 smithi130 bash[27579]: audit 2023-08-28T02:23:29.578777+0000 mon.a (mon.0) 5058 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:23:29.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:29 smithi130 bash[51302]: debug 2023-08-28T02:23:29.564+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:29.569376+0000) 2023-08-28T02:23:29.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:29 smithi006 bash[33918]: audit 2023-08-28T02:23:28.625222+0000 mon.a (mon.0) 5057 : audit [DBG] from='client.? 172.21.15.6:0/4144465003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:29.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:29 smithi006 bash[33918]: audit 2023-08-28T02:23:29.578777+0000 mon.a (mon.0) 5058 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:23:29.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:29 smithi006 bash[27292]: audit 2023-08-28T02:23:28.625222+0000 mon.a (mon.0) 5057 : audit [DBG] from='client.? 172.21.15.6:0/4144465003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:29.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:29 smithi006 bash[27292]: audit 2023-08-28T02:23:29.578777+0000 mon.a (mon.0) 5058 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:23:30.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:30 smithi130 bash[51302]: debug 2023-08-28T02:23:30.564+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:30.569584+0000) 2023-08-28T02:23:30.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:30 smithi130 bash[27579]: cluster 2023-08-28T02:23:29.547565+0000 mgr.y (mgr.14901) 7946 : cluster [DBG] pgmap v7953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:30.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:30 smithi130 bash[27579]: audit 2023-08-28T02:23:29.962799+0000 mon.a (mon.0) 5059 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:23:30.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:30 smithi130 bash[27579]: audit 2023-08-28T02:23:30.231965+0000 mon.b (mon.1) 871 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:23:30.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:30 smithi130 bash[27579]: audit 2023-08-28T02:23:30.234258+0000 mon.b (mon.1) 872 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:23:30.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:30 smithi130 bash[27579]: audit 2023-08-28T02:23:30.243334+0000 mon.a (mon.0) 5060 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:23:30.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:30 smithi006 bash[27292]: cluster 2023-08-28T02:23:29.547565+0000 mgr.y (mgr.14901) 7946 : cluster [DBG] pgmap v7953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:30.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:30 smithi006 bash[27292]: audit 2023-08-28T02:23:29.962799+0000 mon.a (mon.0) 5059 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:23:30.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:30 smithi006 bash[27292]: audit 2023-08-28T02:23:30.231965+0000 mon.b (mon.1) 871 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:23:30.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:30 smithi006 bash[27292]: audit 2023-08-28T02:23:30.234258+0000 mon.b (mon.1) 872 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:23:30.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:30 smithi006 bash[27292]: audit 2023-08-28T02:23:30.243334+0000 mon.a (mon.0) 5060 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:23:30.965 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:23:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:23:30] "GET /metrics HTTP/1.1" 200 193115 "" "Prometheus/2.33.4" 2023-08-28T02:23:30.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:30 smithi006 bash[33918]: cluster 2023-08-28T02:23:29.547565+0000 mgr.y (mgr.14901) 7946 : cluster [DBG] pgmap v7953: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:30.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:30 smithi006 bash[33918]: audit 2023-08-28T02:23:29.962799+0000 mon.a (mon.0) 5059 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:23:30.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:30 smithi006 bash[33918]: audit 2023-08-28T02:23:30.231965+0000 mon.b (mon.1) 871 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:23:30.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:30 smithi006 bash[33918]: audit 2023-08-28T02:23:30.234258+0000 mon.b (mon.1) 872 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:23:30.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:30 smithi006 bash[33918]: audit 2023-08-28T02:23:30.243334+0000 mon.a (mon.0) 5060 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:23:31.629 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:23:31.289Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:31.629 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:23:31.291Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:31.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:31 smithi130 bash[27579]: audit 2023-08-28T02:23:31.037550+0000 mon.c (mon.2) 3138 : audit [DBG] from='client.? 172.21.15.6:0/1894873308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:31.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:31 smithi130 bash[51302]: debug 2023-08-28T02:23:31.564+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:31.569864+0000) 2023-08-28T02:23:31.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:31 smithi006 bash[27292]: audit 2023-08-28T02:23:31.037550+0000 mon.c (mon.2) 3138 : audit [DBG] from='client.? 172.21.15.6:0/1894873308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:31.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:31 smithi006 bash[33918]: audit 2023-08-28T02:23:31.037550+0000 mon.c (mon.2) 3138 : audit [DBG] from='client.? 172.21.15.6:0/1894873308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:32.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:32 smithi130 bash[51302]: debug 2023-08-28T02:23:32.564+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:32.570126+0000) 2023-08-28T02:23:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:32 smithi130 bash[27579]: cluster 2023-08-28T02:23:31.548307+0000 mgr.y (mgr.14901) 7947 : cluster [DBG] pgmap v7954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:32.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:32 smithi006 bash[27292]: cluster 2023-08-28T02:23:31.548307+0000 mgr.y (mgr.14901) 7947 : cluster [DBG] pgmap v7954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:32.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:32 smithi006 bash[33918]: cluster 2023-08-28T02:23:31.548307+0000 mgr.y (mgr.14901) 7947 : cluster [DBG] pgmap v7954: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:23:32.850Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:23:32.852Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:33.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:33 smithi130 bash[51302]: debug 2023-08-28T02:23:33.564+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:33.570410+0000) 2023-08-28T02:23:34.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:33 smithi006 bash[27292]: audit 2023-08-28T02:23:33.453372+0000 mon.c (mon.2) 3139 : audit [DBG] from='client.? 172.21.15.6:0/2079214766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:33 smithi006 bash[27292]: cluster 2023-08-28T02:23:33.549014+0000 mgr.y (mgr.14901) 7948 : cluster [DBG] pgmap v7955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:33 smithi006 bash[33918]: audit 2023-08-28T02:23:33.453372+0000 mon.c (mon.2) 3139 : audit [DBG] from='client.? 172.21.15.6:0/2079214766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:33 smithi006 bash[33918]: cluster 2023-08-28T02:23:33.549014+0000 mgr.y (mgr.14901) 7948 : cluster [DBG] pgmap v7955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:33 smithi130 bash[27579]: audit 2023-08-28T02:23:33.453372+0000 mon.c (mon.2) 3139 : audit [DBG] from='client.? 172.21.15.6:0/2079214766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:33 smithi130 bash[27579]: cluster 2023-08-28T02:23:33.549014+0000 mgr.y (mgr.14901) 7948 : cluster [DBG] pgmap v7955: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:34.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:34 smithi130 bash[51302]: debug 2023-08-28T02:23:34.564+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:34.570746+0000) 2023-08-28T02:23:35.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:23:34.730Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:35.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:23:34.733Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:35.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:35 smithi130 bash[51302]: debug 2023-08-28T02:23:35.568+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:35.571039+0000) 2023-08-28T02:23:36.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:36 smithi130 bash[27579]: cluster 2023-08-28T02:23:35.550272+0000 mgr.y (mgr.14901) 7949 : cluster [DBG] pgmap v7956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:36.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:36 smithi130 bash[27579]: audit 2023-08-28T02:23:35.866683+0000 mon.c (mon.2) 3140 : audit [DBG] from='client.? 172.21.15.6:0/3384165043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:36.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:36 smithi130 bash[51302]: debug 2023-08-28T02:23:36.568+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:36.571331+0000) 2023-08-28T02:23:36.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:36 smithi006 bash[33918]: cluster 2023-08-28T02:23:35.550272+0000 mgr.y (mgr.14901) 7949 : cluster [DBG] pgmap v7956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:36.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:36 smithi006 bash[33918]: audit 2023-08-28T02:23:35.866683+0000 mon.c (mon.2) 3140 : audit [DBG] from='client.? 172.21.15.6:0/3384165043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:36.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:36 smithi006 bash[27292]: cluster 2023-08-28T02:23:35.550272+0000 mgr.y (mgr.14901) 7949 : cluster [DBG] pgmap v7956: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:36.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:36 smithi006 bash[27292]: audit 2023-08-28T02:23:35.866683+0000 mon.c (mon.2) 3140 : audit [DBG] from='client.? 172.21.15.6:0/3384165043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:37.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:37 smithi130 bash[51302]: debug 2023-08-28T02:23:37.568+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:37.571590+0000) 2023-08-28T02:23:38.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:38 smithi130 bash[27579]: cluster 2023-08-28T02:23:37.550941+0000 mgr.y (mgr.14901) 7950 : cluster [DBG] pgmap v7957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:38.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:38 smithi130 bash[27579]: audit 2023-08-28T02:23:38.280331+0000 mon.c (mon.2) 3141 : audit [DBG] from='client.? 172.21.15.6:0/1864899612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:38.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:38 smithi130 bash[51302]: debug 2023-08-28T02:23:38.568+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:38.571874+0000) 2023-08-28T02:23:38.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:38 smithi006 bash[33918]: cluster 2023-08-28T02:23:37.550941+0000 mgr.y (mgr.14901) 7950 : cluster [DBG] pgmap v7957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:38.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:38 smithi006 bash[33918]: audit 2023-08-28T02:23:38.280331+0000 mon.c (mon.2) 3141 : audit [DBG] from='client.? 172.21.15.6:0/1864899612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:38.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:38 smithi006 bash[27292]: cluster 2023-08-28T02:23:37.550941+0000 mgr.y (mgr.14901) 7950 : cluster [DBG] pgmap v7957: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:38.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:38 smithi006 bash[27292]: audit 2023-08-28T02:23:38.280331+0000 mon.c (mon.2) 3141 : audit [DBG] from='client.? 172.21.15.6:0/1864899612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:39.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:39 smithi130 bash[51302]: debug 2023-08-28T02:23:39.568+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:39.572118+0000) 2023-08-28T02:23:40.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:40 smithi130 bash[27579]: cluster 2023-08-28T02:23:39.551991+0000 mgr.y (mgr.14901) 7951 : cluster [DBG] pgmap v7958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:40.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:40 smithi130 bash[51302]: debug 2023-08-28T02:23:40.568+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:40.572409+0000) 2023-08-28T02:23:40.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:23:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:23:40] "GET /metrics HTTP/1.1" 200 193115 "" "Prometheus/2.33.4" 2023-08-28T02:23:40.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:40 smithi006 bash[33918]: cluster 2023-08-28T02:23:39.551991+0000 mgr.y (mgr.14901) 7951 : cluster [DBG] pgmap v7958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:40.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:40 smithi006 bash[27292]: cluster 2023-08-28T02:23:39.551991+0000 mgr.y (mgr.14901) 7951 : cluster [DBG] pgmap v7958: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:41.628 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:23:41.290Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:41.629 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:23:41.292Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:41.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:41 smithi130 bash[27579]: audit 2023-08-28T02:23:40.697554+0000 mon.a (mon.0) 5061 : audit [DBG] from='client.? 172.21.15.6:0/481091480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:41.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:41 smithi130 bash[51302]: debug 2023-08-28T02:23:41.568+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:41.572752+0000) 2023-08-28T02:23:41.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:41 smithi006 bash[33918]: audit 2023-08-28T02:23:40.697554+0000 mon.a (mon.0) 5061 : audit [DBG] from='client.? 172.21.15.6:0/481091480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:41.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:41 smithi006 bash[27292]: audit 2023-08-28T02:23:40.697554+0000 mon.a (mon.0) 5061 : audit [DBG] from='client.? 172.21.15.6:0/481091480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:42.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:42 smithi130 bash[27579]: cluster 2023-08-28T02:23:41.552672+0000 mgr.y (mgr.14901) 7952 : cluster [DBG] pgmap v7959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:42.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:42 smithi130 bash[51302]: debug 2023-08-28T02:23:42.568+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:42.573015+0000) 2023-08-28T02:23:42.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:42 smithi006 bash[33918]: cluster 2023-08-28T02:23:41.552672+0000 mgr.y (mgr.14901) 7952 : cluster [DBG] pgmap v7959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:23:42.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:23:42.853Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:42.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:42 smithi006 bash[27292]: cluster 2023-08-28T02:23:41.552672+0000 mgr.y (mgr.14901) 7952 : cluster [DBG] pgmap v7959: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:43.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:43 smithi130 bash[51302]: debug 2023-08-28T02:23:43.572+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:43.573333+0000) 2023-08-28T02:23:44.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:43 smithi006 bash[33918]: audit 2023-08-28T02:23:43.110052+0000 mon.c (mon.2) 3142 : audit [DBG] from='client.? 172.21.15.6:0/562204097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:44.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:43 smithi006 bash[33918]: cluster 2023-08-28T02:23:43.553390+0000 mgr.y (mgr.14901) 7953 : cluster [DBG] pgmap v7960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:44.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:43 smithi006 bash[27292]: audit 2023-08-28T02:23:43.110052+0000 mon.c (mon.2) 3142 : audit [DBG] from='client.? 172.21.15.6:0/562204097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:44.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:43 smithi006 bash[27292]: cluster 2023-08-28T02:23:43.553390+0000 mgr.y (mgr.14901) 7953 : cluster [DBG] pgmap v7960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:44.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:43 smithi130 bash[27579]: audit 2023-08-28T02:23:43.110052+0000 mon.c (mon.2) 3142 : audit [DBG] from='client.? 172.21.15.6:0/562204097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:44.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:43 smithi130 bash[27579]: cluster 2023-08-28T02:23:43.553390+0000 mgr.y (mgr.14901) 7953 : cluster [DBG] pgmap v7960: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:44.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:44 smithi130 bash[51302]: debug 2023-08-28T02:23:44.568+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:44.573722+0000) 2023-08-28T02:23:45.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:23:44.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:45.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:23:44.734Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:45.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:45 smithi130 bash[27579]: audit 2023-08-28T02:23:45.519137+0000 mon.c (mon.2) 3143 : audit [DBG] from='client.? 172.21.15.6:0/1062532490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:45.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:45 smithi130 bash[51302]: debug 2023-08-28T02:23:45.568+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:45.573969+0000) 2023-08-28T02:23:45.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:45 smithi006 bash[27292]: audit 2023-08-28T02:23:45.519137+0000 mon.c (mon.2) 3143 : audit [DBG] from='client.? 172.21.15.6:0/1062532490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:45.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:45 smithi006 bash[33918]: audit 2023-08-28T02:23:45.519137+0000 mon.c (mon.2) 3143 : audit [DBG] from='client.? 172.21.15.6:0/1062532490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:46.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:46 smithi130 bash[27579]: cluster 2023-08-28T02:23:45.554546+0000 mgr.y (mgr.14901) 7954 : cluster [DBG] pgmap v7961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:46.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:46 smithi130 bash[51302]: debug 2023-08-28T02:23:46.568+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:46.574303+0000) 2023-08-28T02:23:46.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:46 smithi006 bash[27292]: cluster 2023-08-28T02:23:45.554546+0000 mgr.y (mgr.14901) 7954 : cluster [DBG] pgmap v7961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:46.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:46 smithi006 bash[33918]: cluster 2023-08-28T02:23:45.554546+0000 mgr.y (mgr.14901) 7954 : cluster [DBG] pgmap v7961: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:47.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:47 smithi130 bash[51302]: debug 2023-08-28T02:23:47.572+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:47.574657+0000) 2023-08-28T02:23:48.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:48 smithi130 bash[51302]: debug 2023-08-28T02:23:48.572+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:48.574976+0000) 2023-08-28T02:23:48.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:48 smithi130 bash[27579]: cluster 2023-08-28T02:23:47.555200+0000 mgr.y (mgr.14901) 7955 : cluster [DBG] pgmap v7962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:48.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:48 smithi130 bash[27579]: audit 2023-08-28T02:23:47.936398+0000 mon.a (mon.0) 5062 : audit [DBG] from='client.? 172.21.15.6:0/3545823599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:48.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:48 smithi006 bash[33918]: cluster 2023-08-28T02:23:47.555200+0000 mgr.y (mgr.14901) 7955 : cluster [DBG] pgmap v7962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:48.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:48 smithi006 bash[33918]: audit 2023-08-28T02:23:47.936398+0000 mon.a (mon.0) 5062 : audit [DBG] from='client.? 172.21.15.6:0/3545823599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:48.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:48 smithi006 bash[27292]: cluster 2023-08-28T02:23:47.555200+0000 mgr.y (mgr.14901) 7955 : cluster [DBG] pgmap v7962: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:48.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:48 smithi006 bash[27292]: audit 2023-08-28T02:23:47.936398+0000 mon.a (mon.0) 5062 : audit [DBG] from='client.? 172.21.15.6:0/3545823599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:49.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:49 smithi130 bash[51302]: debug 2023-08-28T02:23:49.572+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:49.575322+0000) 2023-08-28T02:23:50.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:50 smithi130 bash[27579]: cluster 2023-08-28T02:23:49.556356+0000 mgr.y (mgr.14901) 7956 : cluster [DBG] pgmap v7963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:50.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:50 smithi130 bash[27579]: audit 2023-08-28T02:23:50.356231+0000 mon.c (mon.2) 3144 : audit [DBG] from='client.? 172.21.15.6:0/1292047456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:50.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:50 smithi130 bash[51302]: debug 2023-08-28T02:23:50.572+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:50.575632+0000) 2023-08-28T02:23:50.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:23:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:23:50] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T02:23:50.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:50 smithi006 bash[27292]: cluster 2023-08-28T02:23:49.556356+0000 mgr.y (mgr.14901) 7956 : cluster [DBG] pgmap v7963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:50.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:50 smithi006 bash[27292]: audit 2023-08-28T02:23:50.356231+0000 mon.c (mon.2) 3144 : audit [DBG] from='client.? 172.21.15.6:0/1292047456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:50.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:50 smithi006 bash[33918]: cluster 2023-08-28T02:23:49.556356+0000 mgr.y (mgr.14901) 7956 : cluster [DBG] pgmap v7963: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:50.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:50 smithi006 bash[33918]: audit 2023-08-28T02:23:50.356231+0000 mon.c (mon.2) 3144 : audit [DBG] from='client.? 172.21.15.6:0/1292047456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:23:51.291Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:23:51.293Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:51.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:51 smithi130 bash[51302]: debug 2023-08-28T02:23:51.572+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:51.576026+0000) 2023-08-28T02:23:52.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:52 smithi130 bash[27579]: cluster 2023-08-28T02:23:51.557095+0000 mgr.y (mgr.14901) 7957 : cluster [DBG] pgmap v7964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:52.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:52 smithi130 bash[51302]: debug 2023-08-28T02:23:52.572+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:52.576341+0000) 2023-08-28T02:23:52.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:52 smithi006 bash[27292]: cluster 2023-08-28T02:23:51.557095+0000 mgr.y (mgr.14901) 7957 : cluster [DBG] pgmap v7964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:52.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:52 smithi006 bash[33918]: cluster 2023-08-28T02:23:51.557095+0000 mgr.y (mgr.14901) 7957 : cluster [DBG] pgmap v7964: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:23:52.852Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:23:52.854Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:53.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:53 smithi130 bash[27579]: audit 2023-08-28T02:23:52.773027+0000 mon.a (mon.0) 5063 : audit [DBG] from='client.? 172.21.15.6:0/1436816295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:53.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:53 smithi130 bash[51302]: debug 2023-08-28T02:23:53.576+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:53.576673+0000) 2023-08-28T02:23:53.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:53 smithi006 bash[27292]: audit 2023-08-28T02:23:52.773027+0000 mon.a (mon.0) 5063 : audit [DBG] from='client.? 172.21.15.6:0/1436816295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:53.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:53 smithi006 bash[33918]: audit 2023-08-28T02:23:52.773027+0000 mon.a (mon.0) 5063 : audit [DBG] from='client.? 172.21.15.6:0/1436816295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:54.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:54 smithi130 bash[27579]: cluster 2023-08-28T02:23:53.557769+0000 mgr.y (mgr.14901) 7958 : cluster [DBG] pgmap v7965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:54.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:54 smithi130 bash[51302]: debug 2023-08-28T02:23:54.572+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:54.576974+0000) 2023-08-28T02:23:54.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:54 smithi006 bash[27292]: cluster 2023-08-28T02:23:53.557769+0000 mgr.y (mgr.14901) 7958 : cluster [DBG] pgmap v7965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:54.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:54 smithi006 bash[33918]: cluster 2023-08-28T02:23:53.557769+0000 mgr.y (mgr.14901) 7958 : cluster [DBG] pgmap v7965: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:23:54.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:23:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:23:54.735Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:23:55.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:55 smithi130 bash[27579]: audit 2023-08-28T02:23:55.197533+0000 mon.c (mon.2) 3145 : audit [DBG] from='client.? 172.21.15.6:0/350315412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:55.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:55 smithi130 bash[51302]: debug 2023-08-28T02:23:55.572+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:55.577316+0000) 2023-08-28T02:23:55.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:55 smithi006 bash[27292]: audit 2023-08-28T02:23:55.197533+0000 mon.c (mon.2) 3145 : audit [DBG] from='client.? 172.21.15.6:0/350315412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:55.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:55 smithi006 bash[33918]: audit 2023-08-28T02:23:55.197533+0000 mon.c (mon.2) 3145 : audit [DBG] from='client.? 172.21.15.6:0/350315412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:56 smithi130 bash[27579]: cluster 2023-08-28T02:23:55.558927+0000 mgr.y (mgr.14901) 7959 : cluster [DBG] pgmap v7966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:56.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:56 smithi130 bash[51302]: debug 2023-08-28T02:23:56.572+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:56.577592+0000) 2023-08-28T02:23:56.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:56 smithi006 bash[27292]: cluster 2023-08-28T02:23:55.558927+0000 mgr.y (mgr.14901) 7959 : cluster [DBG] pgmap v7966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:56.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:56 smithi006 bash[33918]: cluster 2023-08-28T02:23:55.558927+0000 mgr.y (mgr.14901) 7959 : cluster [DBG] pgmap v7966: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:57.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:57 smithi130 bash[27579]: audit 2023-08-28T02:23:57.605347+0000 mon.c (mon.2) 3146 : audit [DBG] from='client.? 172.21.15.6:0/1269803223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:57.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:57 smithi130 bash[51302]: debug 2023-08-28T02:23:57.572+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:57.577860+0000) 2023-08-28T02:23:57.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:57 smithi006 bash[27292]: audit 2023-08-28T02:23:57.605347+0000 mon.c (mon.2) 3146 : audit [DBG] from='client.? 172.21.15.6:0/1269803223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:57.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:57 smithi006 bash[33918]: audit 2023-08-28T02:23:57.605347+0000 mon.c (mon.2) 3146 : audit [DBG] from='client.? 172.21.15.6:0/1269803223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:58.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:23:58 smithi130 bash[27579]: cluster 2023-08-28T02:23:57.559543+0000 mgr.y (mgr.14901) 7960 : cluster [DBG] pgmap v7967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:58.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:58 smithi130 bash[51302]: debug 2023-08-28T02:23:58.576+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:58.578165+0000) 2023-08-28T02:23:58.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:23:58 smithi006 bash[27292]: cluster 2023-08-28T02:23:57.559543+0000 mgr.y (mgr.14901) 7960 : cluster [DBG] pgmap v7967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:58.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:23:58 smithi006 bash[33918]: cluster 2023-08-28T02:23:57.559543+0000 mgr.y (mgr.14901) 7960 : cluster [DBG] pgmap v7967: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:23:59.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:23:59 smithi130 bash[51302]: debug 2023-08-28T02:23:59.572+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:59.578415+0000) 2023-08-28T02:24:00.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:00 smithi130 bash[51302]: debug 2023-08-28T02:24:00.572+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:00.578641+0000) 2023-08-28T02:24:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:00 smithi130 bash[27579]: cluster 2023-08-28T02:23:59.560638+0000 mgr.y (mgr.14901) 7961 : cluster [DBG] pgmap v7968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:00.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:00 smithi130 bash[27579]: audit 2023-08-28T02:24:00.016735+0000 mon.c (mon.2) 3147 : audit [DBG] from='client.? 172.21.15.6:0/4221448850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:00.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:24:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:24:00] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:24:00.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:00 smithi006 bash[27292]: cluster 2023-08-28T02:23:59.560638+0000 mgr.y (mgr.14901) 7961 : cluster [DBG] pgmap v7968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:00.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:00 smithi006 bash[27292]: audit 2023-08-28T02:24:00.016735+0000 mon.c (mon.2) 3147 : audit [DBG] from='client.? 172.21.15.6:0/4221448850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:00.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:00 smithi006 bash[33918]: cluster 2023-08-28T02:23:59.560638+0000 mgr.y (mgr.14901) 7961 : cluster [DBG] pgmap v7968: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:00.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:00 smithi006 bash[33918]: audit 2023-08-28T02:24:00.016735+0000 mon.c (mon.2) 3147 : audit [DBG] from='client.? 172.21.15.6:0/4221448850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:24:01.292Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:24:01.294Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:01.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:01 smithi130 bash[51302]: debug 2023-08-28T02:24:01.576+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:01.578997+0000) 2023-08-28T02:24:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:02 smithi130 bash[27579]: cluster 2023-08-28T02:24:01.561297+0000 mgr.y (mgr.14901) 7962 : cluster [DBG] pgmap v7969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:02 smithi130 bash[27579]: audit 2023-08-28T02:24:02.428299+0000 mon.c (mon.2) 3148 : audit [DBG] from='client.? 172.21.15.6:0/2687726240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:02.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:02 smithi130 bash[51302]: debug 2023-08-28T02:24:02.576+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:02.579272+0000) 2023-08-28T02:24:02.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:24:02.852Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:24:02.854Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:02.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:02 smithi006 bash[27292]: cluster 2023-08-28T02:24:01.561297+0000 mgr.y (mgr.14901) 7962 : cluster [DBG] pgmap v7969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:02.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:02 smithi006 bash[27292]: audit 2023-08-28T02:24:02.428299+0000 mon.c (mon.2) 3148 : audit [DBG] from='client.? 172.21.15.6:0/2687726240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:02.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:02 smithi006 bash[33918]: cluster 2023-08-28T02:24:01.561297+0000 mgr.y (mgr.14901) 7962 : cluster [DBG] pgmap v7969: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:02.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:02 smithi006 bash[33918]: audit 2023-08-28T02:24:02.428299+0000 mon.c (mon.2) 3148 : audit [DBG] from='client.? 172.21.15.6:0/2687726240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:03.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:03 smithi130 bash[51302]: debug 2023-08-28T02:24:03.576+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:03.579572+0000) 2023-08-28T02:24:04.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:03 smithi006 bash[33918]: cluster 2023-08-28T02:24:03.562077+0000 mgr.y (mgr.14901) 7963 : cluster [DBG] pgmap v7970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:04.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:03 smithi006 bash[27292]: cluster 2023-08-28T02:24:03.562077+0000 mgr.y (mgr.14901) 7963 : cluster [DBG] pgmap v7970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:03 smithi130 bash[27579]: cluster 2023-08-28T02:24:03.562077+0000 mgr.y (mgr.14901) 7963 : cluster [DBG] pgmap v7970: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:04.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:04 smithi130 bash[51302]: debug 2023-08-28T02:24:04.576+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:04.579846+0000) 2023-08-28T02:24:05.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:04 smithi006 bash[33918]: audit 2023-08-28T02:24:04.837250+0000 mon.a (mon.0) 5064 : audit [DBG] from='client.? 172.21.15.6:0/3912834264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:05.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:04 smithi006 bash[27292]: audit 2023-08-28T02:24:04.837250+0000 mon.a (mon.0) 5064 : audit [DBG] from='client.? 172.21.15.6:0/3912834264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:05.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:24:04.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:05.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:24:04.736Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:05.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:04 smithi130 bash[27579]: audit 2023-08-28T02:24:04.837250+0000 mon.a (mon.0) 5064 : audit [DBG] from='client.? 172.21.15.6:0/3912834264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:05.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:05 smithi130 bash[51302]: debug 2023-08-28T02:24:05.576+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:05.580118+0000) 2023-08-28T02:24:06.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:05 smithi130 bash[27579]: cluster 2023-08-28T02:24:05.563236+0000 mgr.y (mgr.14901) 7964 : cluster [DBG] pgmap v7971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:06.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:05 smithi006 bash[33918]: cluster 2023-08-28T02:24:05.563236+0000 mgr.y (mgr.14901) 7964 : cluster [DBG] pgmap v7971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:06.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:05 smithi006 bash[27292]: cluster 2023-08-28T02:24:05.563236+0000 mgr.y (mgr.14901) 7964 : cluster [DBG] pgmap v7971: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:06.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:06 smithi130 bash[51302]: debug 2023-08-28T02:24:06.576+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:06.580377+0000) 2023-08-28T02:24:07.579 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:07 smithi130 bash[27579]: audit 2023-08-28T02:24:07.253496+0000 mon.c (mon.2) 3149 : audit [DBG] from='client.? 172.21.15.6:0/2760368218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:07.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:07 smithi006 bash[33918]: audit 2023-08-28T02:24:07.253496+0000 mon.c (mon.2) 3149 : audit [DBG] from='client.? 172.21.15.6:0/2760368218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:07.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:07 smithi006 bash[27292]: audit 2023-08-28T02:24:07.253496+0000 mon.c (mon.2) 3149 : audit [DBG] from='client.? 172.21.15.6:0/2760368218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:07.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:07 smithi130 bash[51302]: debug 2023-08-28T02:24:07.576+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:07.580663+0000) 2023-08-28T02:24:08.579 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:08 smithi130 bash[27579]: cluster 2023-08-28T02:24:07.563913+0000 mgr.y (mgr.14901) 7965 : cluster [DBG] pgmap v7972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:08 smithi006 bash[33918]: cluster 2023-08-28T02:24:07.563913+0000 mgr.y (mgr.14901) 7965 : cluster [DBG] pgmap v7972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:08 smithi006 bash[27292]: cluster 2023-08-28T02:24:07.563913+0000 mgr.y (mgr.14901) 7965 : cluster [DBG] pgmap v7972: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:08.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:08 smithi130 bash[51302]: debug 2023-08-28T02:24:08.580+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:08.580976+0000) 2023-08-28T02:24:09.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:09 smithi130 bash[51302]: debug 2023-08-28T02:24:09.576+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:09.581242+0000) 2023-08-28T02:24:10.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:10 smithi130 bash[51302]: debug 2023-08-28T02:24:10.580+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:10.581595+0000) 2023-08-28T02:24:10.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:10 smithi130 bash[27579]: cluster 2023-08-28T02:24:09.565043+0000 mgr.y (mgr.14901) 7966 : cluster [DBG] pgmap v7973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:10.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:10 smithi130 bash[27579]: audit 2023-08-28T02:24:09.671577+0000 mon.a (mon.0) 5065 : audit [DBG] from='client.? 172.21.15.6:0/1413062931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:10.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:24:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:24:10] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:24:10.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:10 smithi006 bash[27292]: cluster 2023-08-28T02:24:09.565043+0000 mgr.y (mgr.14901) 7966 : cluster [DBG] pgmap v7973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:10.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:10 smithi006 bash[27292]: audit 2023-08-28T02:24:09.671577+0000 mon.a (mon.0) 5065 : audit [DBG] from='client.? 172.21.15.6:0/1413062931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:10.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:10 smithi006 bash[33918]: cluster 2023-08-28T02:24:09.565043+0000 mgr.y (mgr.14901) 7966 : cluster [DBG] pgmap v7973: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:10.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:10 smithi006 bash[33918]: audit 2023-08-28T02:24:09.671577+0000 mon.a (mon.0) 5065 : audit [DBG] from='client.? 172.21.15.6:0/1413062931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:24:11.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:24:11.295Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:11.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:11 smithi130 bash[51302]: debug 2023-08-28T02:24:11.576+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:11.581895+0000) 2023-08-28T02:24:12.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:12 smithi130 bash[51302]: debug 2023-08-28T02:24:12.576+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:12.582110+0000) 2023-08-28T02:24:12.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:12 smithi130 bash[27579]: cluster 2023-08-28T02:24:11.565667+0000 mgr.y (mgr.14901) 7967 : cluster [DBG] pgmap v7974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:12.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:12 smithi130 bash[27579]: audit 2023-08-28T02:24:12.093012+0000 mon.a (mon.0) 5066 : audit [DBG] from='client.? 172.21.15.6:0/3637492648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:12.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:12 smithi006 bash[27292]: cluster 2023-08-28T02:24:11.565667+0000 mgr.y (mgr.14901) 7967 : cluster [DBG] pgmap v7974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:12.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:12 smithi006 bash[27292]: audit 2023-08-28T02:24:12.093012+0000 mon.a (mon.0) 5066 : audit [DBG] from='client.? 172.21.15.6:0/3637492648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:12.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:12 smithi006 bash[33918]: cluster 2023-08-28T02:24:11.565667+0000 mgr.y (mgr.14901) 7967 : cluster [DBG] pgmap v7974: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:12.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:12 smithi006 bash[33918]: audit 2023-08-28T02:24:12.093012+0000 mon.a (mon.0) 5066 : audit [DBG] from='client.? 172.21.15.6:0/3637492648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:24:12.854Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:12.966 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:24:12.856Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:13.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:13 smithi130 bash[51302]: debug 2023-08-28T02:24:13.580+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:13.582430+0000) 2023-08-28T02:24:14.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:13 smithi130 bash[27579]: cluster 2023-08-28T02:24:13.566352+0000 mgr.y (mgr.14901) 7968 : cluster [DBG] pgmap v7975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:14.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:13 smithi006 bash[27292]: cluster 2023-08-28T02:24:13.566352+0000 mgr.y (mgr.14901) 7968 : cluster [DBG] pgmap v7975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:13 smithi006 bash[33918]: cluster 2023-08-28T02:24:13.566352+0000 mgr.y (mgr.14901) 7968 : cluster [DBG] pgmap v7975: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:14.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:14 smithi130 bash[51302]: debug 2023-08-28T02:24:14.580+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:14.582701+0000) 2023-08-28T02:24:15.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:14 smithi006 bash[27292]: audit 2023-08-28T02:24:14.502580+0000 mon.c (mon.2) 3150 : audit [DBG] from='client.? 172.21.15.6:0/1606502295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:15.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:14 smithi006 bash[33918]: audit 2023-08-28T02:24:14.502580+0000 mon.c (mon.2) 3150 : audit [DBG] from='client.? 172.21.15.6:0/1606502295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:15.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:24:14.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:15.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:24:14.737Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:15.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:14 smithi130 bash[27579]: audit 2023-08-28T02:24:14.502580+0000 mon.c (mon.2) 3150 : audit [DBG] from='client.? 172.21.15.6:0/1606502295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:15.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:15 smithi130 bash[51302]: debug 2023-08-28T02:24:15.580+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:15.582970+0000) 2023-08-28T02:24:16.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:15 smithi130 bash[27579]: cluster 2023-08-28T02:24:15.567509+0000 mgr.y (mgr.14901) 7969 : cluster [DBG] pgmap v7976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:16.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:15 smithi006 bash[27292]: cluster 2023-08-28T02:24:15.567509+0000 mgr.y (mgr.14901) 7969 : cluster [DBG] pgmap v7976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:16.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:15 smithi006 bash[33918]: cluster 2023-08-28T02:24:15.567509+0000 mgr.y (mgr.14901) 7969 : cluster [DBG] pgmap v7976: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:16.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:16 smithi130 bash[51302]: debug 2023-08-28T02:24:16.580+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:16.583251+0000) 2023-08-28T02:24:17.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:16 smithi130 bash[27579]: audit 2023-08-28T02:24:16.917001+0000 mon.c (mon.2) 3151 : audit [DBG] from='client.? 172.21.15.6:0/124975101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:17.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:16 smithi006 bash[27292]: audit 2023-08-28T02:24:16.917001+0000 mon.c (mon.2) 3151 : audit [DBG] from='client.? 172.21.15.6:0/124975101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:17.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:16 smithi006 bash[33918]: audit 2023-08-28T02:24:16.917001+0000 mon.c (mon.2) 3151 : audit [DBG] from='client.? 172.21.15.6:0/124975101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:17.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:17 smithi130 bash[51302]: debug 2023-08-28T02:24:17.580+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:17.583557+0000) 2023-08-28T02:24:18.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:17 smithi130 bash[27579]: cluster 2023-08-28T02:24:17.568149+0000 mgr.y (mgr.14901) 7970 : cluster [DBG] pgmap v7977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:18.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:17 smithi006 bash[27292]: cluster 2023-08-28T02:24:17.568149+0000 mgr.y (mgr.14901) 7970 : cluster [DBG] pgmap v7977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:18.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:17 smithi006 bash[33918]: cluster 2023-08-28T02:24:17.568149+0000 mgr.y (mgr.14901) 7970 : cluster [DBG] pgmap v7977: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:18.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:18 smithi130 bash[51302]: debug 2023-08-28T02:24:18.580+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:18.583824+0000) 2023-08-28T02:24:19.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:19 smithi006 bash[33918]: audit 2023-08-28T02:24:19.331887+0000 mon.c (mon.2) 3152 : audit [DBG] from='client.? 172.21.15.6:0/2318179371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:19.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:19 smithi006 bash[27292]: audit 2023-08-28T02:24:19.331887+0000 mon.c (mon.2) 3152 : audit [DBG] from='client.? 172.21.15.6:0/2318179371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:19.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:19 smithi130 bash[27579]: audit 2023-08-28T02:24:19.331887+0000 mon.c (mon.2) 3152 : audit [DBG] from='client.? 172.21.15.6:0/2318179371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:19.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:19 smithi130 bash[51302]: debug 2023-08-28T02:24:19.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:19.584139+0000) 2023-08-28T02:24:20.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:20 smithi006 bash[33918]: cluster 2023-08-28T02:24:19.569352+0000 mgr.y (mgr.14901) 7971 : cluster [DBG] pgmap v7978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:20.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:20 smithi006 bash[27292]: cluster 2023-08-28T02:24:19.569352+0000 mgr.y (mgr.14901) 7971 : cluster [DBG] pgmap v7978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:20 smithi130 bash[27579]: cluster 2023-08-28T02:24:19.569352+0000 mgr.y (mgr.14901) 7971 : cluster [DBG] pgmap v7978: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:20.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:20 smithi130 bash[51302]: debug 2023-08-28T02:24:20.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:20.584420+0000) 2023-08-28T02:24:21.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:24:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:24:20] "GET /metrics HTTP/1.1" 200 193115 "" "Prometheus/2.33.4" 2023-08-28T02:24:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:24:21.293Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:24:21.295Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:21.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:21 smithi130 bash[51302]: debug 2023-08-28T02:24:21.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:21.584718+0000) 2023-08-28T02:24:22.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:22 smithi130 bash[51302]: debug 2023-08-28T02:24:22.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:22.585008+0000) 2023-08-28T02:24:22.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:22 smithi130 bash[27579]: cluster 2023-08-28T02:24:21.570008+0000 mgr.y (mgr.14901) 7972 : cluster [DBG] pgmap v7979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:22.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:22 smithi130 bash[27579]: audit 2023-08-28T02:24:21.759935+0000 mon.c (mon.2) 3153 : audit [DBG] from='client.? 172.21.15.6:0/1613776380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:22.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:22 smithi006 bash[33918]: cluster 2023-08-28T02:24:21.570008+0000 mgr.y (mgr.14901) 7972 : cluster [DBG] pgmap v7979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:22.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:22 smithi006 bash[33918]: audit 2023-08-28T02:24:21.759935+0000 mon.c (mon.2) 3153 : audit [DBG] from='client.? 172.21.15.6:0/1613776380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:22.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:22 smithi006 bash[27292]: cluster 2023-08-28T02:24:21.570008+0000 mgr.y (mgr.14901) 7972 : cluster [DBG] pgmap v7979: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:22.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:22 smithi006 bash[27292]: audit 2023-08-28T02:24:21.759935+0000 mon.c (mon.2) 3153 : audit [DBG] from='client.? 172.21.15.6:0/1613776380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:24:22.854Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:24:22.856Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:23.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:23 smithi130 bash[51302]: debug 2023-08-28T02:24:23.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:23.585361+0000) 2023-08-28T02:24:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:23 smithi130 bash[27579]: cluster 2023-08-28T02:24:23.570777+0000 mgr.y (mgr.14901) 7973 : cluster [DBG] pgmap v7980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:24.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:23 smithi006 bash[33918]: cluster 2023-08-28T02:24:23.570777+0000 mgr.y (mgr.14901) 7973 : cluster [DBG] pgmap v7980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:23 smithi006 bash[27292]: cluster 2023-08-28T02:24:23.570777+0000 mgr.y (mgr.14901) 7973 : cluster [DBG] pgmap v7980: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:24.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:24 smithi130 bash[51302]: debug 2023-08-28T02:24:24.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:24.585639+0000) 2023-08-28T02:24:25.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:24 smithi006 bash[27292]: audit 2023-08-28T02:24:24.178804+0000 mon.a (mon.0) 5067 : audit [DBG] from='client.? 172.21.15.6:0/1701480789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:25.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:24 smithi006 bash[33918]: audit 2023-08-28T02:24:24.178804+0000 mon.a (mon.0) 5067 : audit [DBG] from='client.? 172.21.15.6:0/1701480789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:25.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:24:24.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:25.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:24:24.738Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:25.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:24 smithi130 bash[27579]: audit 2023-08-28T02:24:24.178804+0000 mon.a (mon.0) 5067 : audit [DBG] from='client.? 172.21.15.6:0/1701480789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:25.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:25 smithi130 bash[51302]: debug 2023-08-28T02:24:25.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:25.585954+0000) 2023-08-28T02:24:26.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:25 smithi130 bash[27579]: cluster 2023-08-28T02:24:25.572049+0000 mgr.y (mgr.14901) 7974 : cluster [DBG] pgmap v7981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:26.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:25 smithi006 bash[27292]: cluster 2023-08-28T02:24:25.572049+0000 mgr.y (mgr.14901) 7974 : cluster [DBG] pgmap v7981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:26.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:25 smithi006 bash[33918]: cluster 2023-08-28T02:24:25.572049+0000 mgr.y (mgr.14901) 7974 : cluster [DBG] pgmap v7981: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:26.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:26 smithi130 bash[51302]: debug 2023-08-28T02:24:26.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:26.586317+0000) 2023-08-28T02:24:27.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:26 smithi130 bash[27579]: audit 2023-08-28T02:24:26.590363+0000 mon.c (mon.2) 3154 : audit [DBG] from='client.? 172.21.15.6:0/1693766278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:27.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:26 smithi006 bash[33918]: audit 2023-08-28T02:24:26.590363+0000 mon.c (mon.2) 3154 : audit [DBG] from='client.? 172.21.15.6:0/1693766278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:27.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:26 smithi006 bash[27292]: audit 2023-08-28T02:24:26.590363+0000 mon.c (mon.2) 3154 : audit [DBG] from='client.? 172.21.15.6:0/1693766278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:27.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:27 smithi130 bash[51302]: debug 2023-08-28T02:24:27.579+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:27.586603+0000) 2023-08-28T02:24:28.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:27 smithi130 bash[27579]: cluster 2023-08-28T02:24:27.572671+0000 mgr.y (mgr.14901) 7975 : cluster [DBG] pgmap v7982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:28.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:27 smithi006 bash[33918]: cluster 2023-08-28T02:24:27.572671+0000 mgr.y (mgr.14901) 7975 : cluster [DBG] pgmap v7982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:28.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:27 smithi006 bash[27292]: cluster 2023-08-28T02:24:27.572671+0000 mgr.y (mgr.14901) 7975 : cluster [DBG] pgmap v7982: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:28.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:28 smithi130 bash[51302]: debug 2023-08-28T02:24:28.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:28.586901+0000) 2023-08-28T02:24:29.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:29 smithi130 bash[27579]: audit 2023-08-28T02:24:29.009445+0000 mon.a (mon.0) 5068 : audit [DBG] from='client.? 172.21.15.6:0/2632527414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:29.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:29 smithi006 bash[27292]: audit 2023-08-28T02:24:29.009445+0000 mon.a (mon.0) 5068 : audit [DBG] from='client.? 172.21.15.6:0/2632527414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:29.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:29 smithi006 bash[33918]: audit 2023-08-28T02:24:29.009445+0000 mon.a (mon.0) 5068 : audit [DBG] from='client.? 172.21.15.6:0/2632527414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:29.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:29 smithi130 bash[51302]: debug 2023-08-28T02:24:29.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:29.587182+0000) 2023-08-28T02:24:30.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:30 smithi130 bash[27579]: cluster 2023-08-28T02:24:29.573779+0000 mgr.y (mgr.14901) 7976 : cluster [DBG] pgmap v7983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:30.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:30 smithi006 bash[27292]: cluster 2023-08-28T02:24:29.573779+0000 mgr.y (mgr.14901) 7976 : cluster [DBG] pgmap v7983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:30.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:30 smithi006 bash[33918]: cluster 2023-08-28T02:24:29.573779+0000 mgr.y (mgr.14901) 7976 : cluster [DBG] pgmap v7983: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:30.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:30 smithi130 bash[51302]: debug 2023-08-28T02:24:30.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:30.587487+0000) 2023-08-28T02:24:31.073 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:24:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:24:30] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:24:31.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:31 smithi130 bash[27579]: audit 2023-08-28T02:24:30.255163+0000 mon.b (mon.1) 873 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:24:31.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:31 smithi006 bash[27292]: audit 2023-08-28T02:24:30.255163+0000 mon.b (mon.1) 873 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:24:31.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:31 smithi006 bash[33918]: audit 2023-08-28T02:24:30.255163+0000 mon.b (mon.1) 873 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:24:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:24:31.294Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:31.465 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:24:31.296Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:31.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:31 smithi130 bash[51302]: debug 2023-08-28T02:24:31.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:31.587787+0000) 2023-08-28T02:24:32.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:32 smithi130 bash[27579]: audit 2023-08-28T02:24:31.433365+0000 mon.c (mon.2) 3155 : audit [DBG] from='client.? 172.21.15.6:0/739030054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:32.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:32 smithi130 bash[27579]: cluster 2023-08-28T02:24:31.574529+0000 mgr.y (mgr.14901) 7977 : cluster [DBG] pgmap v7984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:32.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:32 smithi006 bash[27292]: audit 2023-08-28T02:24:31.433365+0000 mon.c (mon.2) 3155 : audit [DBG] from='client.? 172.21.15.6:0/739030054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:32.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:32 smithi006 bash[27292]: cluster 2023-08-28T02:24:31.574529+0000 mgr.y (mgr.14901) 7977 : cluster [DBG] pgmap v7984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:32.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:32 smithi006 bash[33918]: audit 2023-08-28T02:24:31.433365+0000 mon.c (mon.2) 3155 : audit [DBG] from='client.? 172.21.15.6:0/739030054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:32.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:32 smithi006 bash[33918]: cluster 2023-08-28T02:24:31.574529+0000 mgr.y (mgr.14901) 7977 : cluster [DBG] pgmap v7984: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:32.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:32 smithi130 bash[51302]: debug 2023-08-28T02:24:32.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:32.588016+0000) 2023-08-28T02:24:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:24:32.855Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:24:32.857Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:33.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:33 smithi130 bash[51302]: debug 2023-08-28T02:24:33.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:33.588281+0000) 2023-08-28T02:24:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:33 smithi130 bash[27579]: cluster 2023-08-28T02:24:33.575257+0000 mgr.y (mgr.14901) 7978 : cluster [DBG] pgmap v7985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:33 smithi130 bash[27579]: audit 2023-08-28T02:24:33.855217+0000 mon.c (mon.2) 3156 : audit [DBG] from='client.? 172.21.15.6:0/1839969545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:34.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:33 smithi006 bash[33918]: cluster 2023-08-28T02:24:33.575257+0000 mgr.y (mgr.14901) 7978 : cluster [DBG] pgmap v7985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:33 smithi006 bash[33918]: audit 2023-08-28T02:24:33.855217+0000 mon.c (mon.2) 3156 : audit [DBG] from='client.? 172.21.15.6:0/1839969545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:34.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:33 smithi006 bash[27292]: cluster 2023-08-28T02:24:33.575257+0000 mgr.y (mgr.14901) 7978 : cluster [DBG] pgmap v7985: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:34.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:33 smithi006 bash[27292]: audit 2023-08-28T02:24:33.855217+0000 mon.c (mon.2) 3156 : audit [DBG] from='client.? 172.21.15.6:0/1839969545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:34.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:34 smithi130 bash[51302]: debug 2023-08-28T02:24:34.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:34.588561+0000) 2023-08-28T02:24:35.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:24:34.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:35.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:24:34.738Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:35.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:35 smithi130 bash[51302]: debug 2023-08-28T02:24:35.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:35.588767+0000) 2023-08-28T02:24:36.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:36 smithi130 bash[27579]: cluster 2023-08-28T02:24:35.576271+0000 mgr.y (mgr.14901) 7979 : cluster [DBG] pgmap v7986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:36.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:36 smithi130 bash[27579]: audit 2023-08-28T02:24:36.147671+0000 mon.a (mon.0) 5069 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:24:36.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:36 smithi130 bash[27579]: audit 2023-08-28T02:24:36.338900+0000 mon.c (mon.2) 3157 : audit [DBG] from='client.? 172.21.15.6:0/1265283964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:36.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:36 smithi130 bash[27579]: audit 2023-08-28T02:24:36.596633+0000 mon.a (mon.0) 5070 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:24:36.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:36 smithi130 bash[51302]: debug 2023-08-28T02:24:36.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:36.589039+0000) 2023-08-28T02:24:36.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:36 smithi006 bash[33918]: cluster 2023-08-28T02:24:35.576271+0000 mgr.y (mgr.14901) 7979 : cluster [DBG] pgmap v7986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:36.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:36 smithi006 bash[33918]: audit 2023-08-28T02:24:36.147671+0000 mon.a (mon.0) 5069 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:24:36.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:36 smithi006 bash[33918]: audit 2023-08-28T02:24:36.338900+0000 mon.c (mon.2) 3157 : audit [DBG] from='client.? 172.21.15.6:0/1265283964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:36.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:36 smithi006 bash[33918]: audit 2023-08-28T02:24:36.596633+0000 mon.a (mon.0) 5070 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:24:36.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:36 smithi006 bash[27292]: cluster 2023-08-28T02:24:35.576271+0000 mgr.y (mgr.14901) 7979 : cluster [DBG] pgmap v7986: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:36.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:36 smithi006 bash[27292]: audit 2023-08-28T02:24:36.147671+0000 mon.a (mon.0) 5069 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:24:36.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:36 smithi006 bash[27292]: audit 2023-08-28T02:24:36.338900+0000 mon.c (mon.2) 3157 : audit [DBG] from='client.? 172.21.15.6:0/1265283964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:36.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:36 smithi006 bash[27292]: audit 2023-08-28T02:24:36.596633+0000 mon.a (mon.0) 5070 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:24:37.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:37 smithi130 bash[51302]: debug 2023-08-28T02:24:37.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:37.589318+0000) 2023-08-28T02:24:37.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:37 smithi130 bash[27579]: audit 2023-08-28T02:24:36.867593+0000 mon.b (mon.1) 874 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:24:37.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:37 smithi130 bash[27579]: audit 2023-08-28T02:24:36.869545+0000 mon.b (mon.1) 875 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:24:37.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:37 smithi130 bash[27579]: audit 2023-08-28T02:24:36.876661+0000 mon.a (mon.0) 5071 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:24:37.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:37 smithi006 bash[27292]: audit 2023-08-28T02:24:36.867593+0000 mon.b (mon.1) 874 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:24:37.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:37 smithi006 bash[27292]: audit 2023-08-28T02:24:36.869545+0000 mon.b (mon.1) 875 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:24:37.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:37 smithi006 bash[27292]: audit 2023-08-28T02:24:36.876661+0000 mon.a (mon.0) 5071 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:24:37.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:37 smithi006 bash[33918]: audit 2023-08-28T02:24:36.867593+0000 mon.b (mon.1) 874 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:24:37.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:37 smithi006 bash[33918]: audit 2023-08-28T02:24:36.869545+0000 mon.b (mon.1) 875 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:24:37.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:37 smithi006 bash[33918]: audit 2023-08-28T02:24:36.876661+0000 mon.a (mon.0) 5071 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:24:38.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:38 smithi130 bash[27579]: cluster 2023-08-28T02:24:37.577056+0000 mgr.y (mgr.14901) 7980 : cluster [DBG] pgmap v7987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:38.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:38 smithi130 bash[51302]: debug 2023-08-28T02:24:38.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:38.589625+0000) 2023-08-28T02:24:38.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:38 smithi006 bash[27292]: cluster 2023-08-28T02:24:37.577056+0000 mgr.y (mgr.14901) 7980 : cluster [DBG] pgmap v7987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:38.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:38 smithi006 bash[33918]: cluster 2023-08-28T02:24:37.577056+0000 mgr.y (mgr.14901) 7980 : cluster [DBG] pgmap v7987: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:39.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:39 smithi130 bash[27579]: audit 2023-08-28T02:24:38.769728+0000 mon.a (mon.0) 5072 : audit [DBG] from='client.? 172.21.15.6:0/3092005403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:39.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:39 smithi130 bash[51302]: debug 2023-08-28T02:24:39.587+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:39.589897+0000) 2023-08-28T02:24:39.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:39 smithi006 bash[27292]: audit 2023-08-28T02:24:38.769728+0000 mon.a (mon.0) 5072 : audit [DBG] from='client.? 172.21.15.6:0/3092005403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:39.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:39 smithi006 bash[33918]: audit 2023-08-28T02:24:38.769728+0000 mon.a (mon.0) 5072 : audit [DBG] from='client.? 172.21.15.6:0/3092005403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:40.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:40 smithi130 bash[27579]: cluster 2023-08-28T02:24:39.578144+0000 mgr.y (mgr.14901) 7981 : cluster [DBG] pgmap v7988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:40.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:40 smithi130 bash[51302]: debug 2023-08-28T02:24:40.587+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:40.590184+0000) 2023-08-28T02:24:40.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:24:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:24:40] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:24:40.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:40 smithi006 bash[27292]: cluster 2023-08-28T02:24:39.578144+0000 mgr.y (mgr.14901) 7981 : cluster [DBG] pgmap v7988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:40.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:40 smithi006 bash[33918]: cluster 2023-08-28T02:24:39.578144+0000 mgr.y (mgr.14901) 7981 : cluster [DBG] pgmap v7988: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:41.658 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:24:41.295Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:41.658 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:24:41.297Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:41.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:41 smithi130 bash[51302]: debug 2023-08-28T02:24:41.587+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:41.590471+0000) 2023-08-28T02:24:41.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:41 smithi130 bash[27579]: audit 2023-08-28T02:24:41.190041+0000 mon.c (mon.2) 3158 : audit [DBG] from='client.? 172.21.15.6:0/1362167557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:41.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:41 smithi006 bash[27292]: audit 2023-08-28T02:24:41.190041+0000 mon.c (mon.2) 3158 : audit [DBG] from='client.? 172.21.15.6:0/1362167557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:41.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:41 smithi006 bash[33918]: audit 2023-08-28T02:24:41.190041+0000 mon.c (mon.2) 3158 : audit [DBG] from='client.? 172.21.15.6:0/1362167557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:42.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:42 smithi130 bash[27579]: cluster 2023-08-28T02:24:41.578824+0000 mgr.y (mgr.14901) 7982 : cluster [DBG] pgmap v7989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:42.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:42 smithi130 bash[51302]: debug 2023-08-28T02:24:42.583+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:42.590737+0000) 2023-08-28T02:24:42.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:42 smithi006 bash[27292]: cluster 2023-08-28T02:24:41.578824+0000 mgr.y (mgr.14901) 7982 : cluster [DBG] pgmap v7989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:42.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:42 smithi006 bash[33918]: cluster 2023-08-28T02:24:41.578824+0000 mgr.y (mgr.14901) 7982 : cluster [DBG] pgmap v7989: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:24:42.856Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:24:42.858Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:43.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:43 smithi130 bash[51302]: debug 2023-08-28T02:24:43.587+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:43.591016+0000) 2023-08-28T02:24:44.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:43 smithi130 bash[27579]: cluster 2023-08-28T02:24:43.579404+0000 mgr.y (mgr.14901) 7983 : cluster [DBG] pgmap v7990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:44.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:43 smithi130 bash[27579]: audit 2023-08-28T02:24:43.609805+0000 mon.c (mon.2) 3159 : audit [DBG] from='client.? 172.21.15.6:0/2010064761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:44.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:43 smithi006 bash[27292]: cluster 2023-08-28T02:24:43.579404+0000 mgr.y (mgr.14901) 7983 : cluster [DBG] pgmap v7990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:44.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:43 smithi006 bash[27292]: audit 2023-08-28T02:24:43.609805+0000 mon.c (mon.2) 3159 : audit [DBG] from='client.? 172.21.15.6:0/2010064761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:43 smithi006 bash[33918]: cluster 2023-08-28T02:24:43.579404+0000 mgr.y (mgr.14901) 7983 : cluster [DBG] pgmap v7990: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:43 smithi006 bash[33918]: audit 2023-08-28T02:24:43.609805+0000 mon.c (mon.2) 3159 : audit [DBG] from='client.? 172.21.15.6:0/2010064761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:44.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:44 smithi130 bash[51302]: debug 2023-08-28T02:24:44.587+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:44.591237+0000) 2023-08-28T02:24:45.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:24:44.737Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:45.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:24:44.739Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:45.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:45 smithi130 bash[51302]: debug 2023-08-28T02:24:45.587+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:45.591517+0000) 2023-08-28T02:24:46.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:46 smithi130 bash[51302]: debug 2023-08-28T02:24:46.591+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:46.591826+0000) 2023-08-28T02:24:46.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:46 smithi130 bash[27579]: cluster 2023-08-28T02:24:45.580504+0000 mgr.y (mgr.14901) 7984 : cluster [DBG] pgmap v7991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:46.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:46 smithi130 bash[27579]: audit 2023-08-28T02:24:46.026441+0000 mon.c (mon.2) 3160 : audit [DBG] from='client.? 172.21.15.6:0/2667491067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:46.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:46 smithi006 bash[33918]: cluster 2023-08-28T02:24:45.580504+0000 mgr.y (mgr.14901) 7984 : cluster [DBG] pgmap v7991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:46.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:46 smithi006 bash[33918]: audit 2023-08-28T02:24:46.026441+0000 mon.c (mon.2) 3160 : audit [DBG] from='client.? 172.21.15.6:0/2667491067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:46.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:46 smithi006 bash[27292]: cluster 2023-08-28T02:24:45.580504+0000 mgr.y (mgr.14901) 7984 : cluster [DBG] pgmap v7991: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:46.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:46 smithi006 bash[27292]: audit 2023-08-28T02:24:46.026441+0000 mon.c (mon.2) 3160 : audit [DBG] from='client.? 172.21.15.6:0/2667491067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:47.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:47 smithi130 bash[51302]: debug 2023-08-28T02:24:47.587+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:47.592069+0000) 2023-08-28T02:24:48.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:48 smithi130 bash[51302]: debug 2023-08-28T02:24:48.587+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:48.592299+0000) 2023-08-28T02:24:48.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:48 smithi130 bash[27579]: cluster 2023-08-28T02:24:47.581207+0000 mgr.y (mgr.14901) 7985 : cluster [DBG] pgmap v7992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:48.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:48 smithi130 bash[27579]: audit 2023-08-28T02:24:48.441418+0000 mon.c (mon.2) 3161 : audit [DBG] from='client.? 172.21.15.6:0/2541580034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:48.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:48 smithi006 bash[27292]: cluster 2023-08-28T02:24:47.581207+0000 mgr.y (mgr.14901) 7985 : cluster [DBG] pgmap v7992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:48.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:48 smithi006 bash[27292]: audit 2023-08-28T02:24:48.441418+0000 mon.c (mon.2) 3161 : audit [DBG] from='client.? 172.21.15.6:0/2541580034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:48.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:48 smithi006 bash[33918]: cluster 2023-08-28T02:24:47.581207+0000 mgr.y (mgr.14901) 7985 : cluster [DBG] pgmap v7992: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:48.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:48 smithi006 bash[33918]: audit 2023-08-28T02:24:48.441418+0000 mon.c (mon.2) 3161 : audit [DBG] from='client.? 172.21.15.6:0/2541580034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:49.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:49 smithi130 bash[51302]: debug 2023-08-28T02:24:49.587+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:49.592567+0000) 2023-08-28T02:24:50.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:50 smithi130 bash[27579]: cluster 2023-08-28T02:24:49.582388+0000 mgr.y (mgr.14901) 7986 : cluster [DBG] pgmap v7993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:50.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:50 smithi130 bash[51302]: debug 2023-08-28T02:24:50.587+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:50.592874+0000) 2023-08-28T02:24:50.965 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:24:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:24:50] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:24:50.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:50 smithi006 bash[33918]: cluster 2023-08-28T02:24:49.582388+0000 mgr.y (mgr.14901) 7986 : cluster [DBG] pgmap v7993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:50.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:50 smithi006 bash[27292]: cluster 2023-08-28T02:24:49.582388+0000 mgr.y (mgr.14901) 7986 : cluster [DBG] pgmap v7993: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:51.655 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:24:51.296Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:51.655 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:24:51.298Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:51.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:51 smithi130 bash[27579]: audit 2023-08-28T02:24:50.864102+0000 mon.c (mon.2) 3162 : audit [DBG] from='client.? 172.21.15.6:0/1998684882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:51.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:51 smithi130 bash[51302]: debug 2023-08-28T02:24:51.587+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:51.593161+0000) 2023-08-28T02:24:51.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:51 smithi006 bash[33918]: audit 2023-08-28T02:24:50.864102+0000 mon.c (mon.2) 3162 : audit [DBG] from='client.? 172.21.15.6:0/1998684882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:51.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:51 smithi006 bash[27292]: audit 2023-08-28T02:24:50.864102+0000 mon.c (mon.2) 3162 : audit [DBG] from='client.? 172.21.15.6:0/1998684882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:52.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:52 smithi130 bash[27579]: cluster 2023-08-28T02:24:51.583075+0000 mgr.y (mgr.14901) 7987 : cluster [DBG] pgmap v7994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:52.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:52 smithi130 bash[51302]: debug 2023-08-28T02:24:52.587+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:52.593457+0000) 2023-08-28T02:24:52.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:52 smithi006 bash[33918]: cluster 2023-08-28T02:24:51.583075+0000 mgr.y (mgr.14901) 7987 : cluster [DBG] pgmap v7994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:52.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:52 smithi006 bash[27292]: cluster 2023-08-28T02:24:51.583075+0000 mgr.y (mgr.14901) 7987 : cluster [DBG] pgmap v7994: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:24:52.857Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:24:52.859Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:53.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:53 smithi130 bash[51302]: debug 2023-08-28T02:24:53.591+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:53.593788+0000) 2023-08-28T02:24:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:53 smithi130 bash[27579]: audit 2023-08-28T02:24:53.285088+0000 mon.a (mon.0) 5073 : audit [DBG] from='client.? 172.21.15.6:0/3315204112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:53 smithi130 bash[27579]: cluster 2023-08-28T02:24:53.583753+0000 mgr.y (mgr.14901) 7988 : cluster [DBG] pgmap v7995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:54.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:53 smithi006 bash[27292]: audit 2023-08-28T02:24:53.285088+0000 mon.a (mon.0) 5073 : audit [DBG] from='client.? 172.21.15.6:0/3315204112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:54.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:53 smithi006 bash[27292]: cluster 2023-08-28T02:24:53.583753+0000 mgr.y (mgr.14901) 7988 : cluster [DBG] pgmap v7995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:53 smithi006 bash[33918]: audit 2023-08-28T02:24:53.285088+0000 mon.a (mon.0) 5073 : audit [DBG] from='client.? 172.21.15.6:0/3315204112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:53 smithi006 bash[33918]: cluster 2023-08-28T02:24:53.583753+0000 mgr.y (mgr.14901) 7988 : cluster [DBG] pgmap v7995: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:54.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:54 smithi130 bash[51302]: debug 2023-08-28T02:24:54.587+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:54.594059+0000) 2023-08-28T02:24:55.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:24:54.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:55.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:24:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:24:54.740Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:24:55.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:55 smithi130 bash[51302]: debug 2023-08-28T02:24:55.587+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:55.594322+0000) 2023-08-28T02:24:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:56 smithi130 bash[27579]: cluster 2023-08-28T02:24:55.584876+0000 mgr.y (mgr.14901) 7989 : cluster [DBG] pgmap v7996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:56 smithi130 bash[27579]: audit 2023-08-28T02:24:55.712316+0000 mon.c (mon.2) 3163 : audit [DBG] from='client.? 172.21.15.6:0/2596023002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:56.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:56 smithi130 bash[51302]: debug 2023-08-28T02:24:56.587+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:56.594629+0000) 2023-08-28T02:24:56.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:56 smithi006 bash[33918]: cluster 2023-08-28T02:24:55.584876+0000 mgr.y (mgr.14901) 7989 : cluster [DBG] pgmap v7996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:56.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:56 smithi006 bash[33918]: audit 2023-08-28T02:24:55.712316+0000 mon.c (mon.2) 3163 : audit [DBG] from='client.? 172.21.15.6:0/2596023002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:56.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:56 smithi006 bash[27292]: cluster 2023-08-28T02:24:55.584876+0000 mgr.y (mgr.14901) 7989 : cluster [DBG] pgmap v7996: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:56.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:56 smithi006 bash[27292]: audit 2023-08-28T02:24:55.712316+0000 mon.c (mon.2) 3163 : audit [DBG] from='client.? 172.21.15.6:0/2596023002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:57.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:57 smithi130 bash[51302]: debug 2023-08-28T02:24:57.591+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:57.594936+0000) 2023-08-28T02:24:58.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:58 smithi130 bash[27579]: cluster 2023-08-28T02:24:57.585476+0000 mgr.y (mgr.14901) 7990 : cluster [DBG] pgmap v7997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:58.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:24:58 smithi130 bash[27579]: audit 2023-08-28T02:24:58.124880+0000 mon.a (mon.0) 5074 : audit [DBG] from='client.? 172.21.15.6:0/1933302796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:58.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:58 smithi130 bash[51302]: debug 2023-08-28T02:24:58.591+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:58.595230+0000) 2023-08-28T02:24:58.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:58 smithi006 bash[33918]: cluster 2023-08-28T02:24:57.585476+0000 mgr.y (mgr.14901) 7990 : cluster [DBG] pgmap v7997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:58.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:24:58 smithi006 bash[33918]: audit 2023-08-28T02:24:58.124880+0000 mon.a (mon.0) 5074 : audit [DBG] from='client.? 172.21.15.6:0/1933302796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:58.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:58 smithi006 bash[27292]: cluster 2023-08-28T02:24:57.585476+0000 mgr.y (mgr.14901) 7990 : cluster [DBG] pgmap v7997: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:24:58.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:24:58 smithi006 bash[27292]: audit 2023-08-28T02:24:58.124880+0000 mon.a (mon.0) 5074 : audit [DBG] from='client.? 172.21.15.6:0/1933302796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:59.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:24:59 smithi130 bash[51302]: debug 2023-08-28T02:24:59.591+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:59.595457+0000) 2023-08-28T02:25:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:00 smithi130 bash[27579]: cluster 2023-08-28T02:24:59.586589+0000 mgr.y (mgr.14901) 7991 : cluster [DBG] pgmap v7998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:00 smithi130 bash[27579]: audit 2023-08-28T02:25:00.548981+0000 mon.c (mon.2) 3164 : audit [DBG] from='client.? 172.21.15.6:0/454227752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:00.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:00 smithi130 bash[51302]: debug 2023-08-28T02:25:00.591+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:00.595738+0000) 2023-08-28T02:25:00.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:25:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:25:00] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:25:00.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:00 smithi006 bash[27292]: cluster 2023-08-28T02:24:59.586589+0000 mgr.y (mgr.14901) 7991 : cluster [DBG] pgmap v7998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:00.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:00 smithi006 bash[27292]: audit 2023-08-28T02:25:00.548981+0000 mon.c (mon.2) 3164 : audit [DBG] from='client.? 172.21.15.6:0/454227752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:00.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:00 smithi006 bash[33918]: cluster 2023-08-28T02:24:59.586589+0000 mgr.y (mgr.14901) 7991 : cluster [DBG] pgmap v7998: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:00.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:00 smithi006 bash[33918]: audit 2023-08-28T02:25:00.548981+0000 mon.c (mon.2) 3164 : audit [DBG] from='client.? 172.21.15.6:0/454227752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:25:01.296Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:25:01.298Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:01.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:01 smithi130 bash[51302]: debug 2023-08-28T02:25:01.595+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:01.596027+0000) 2023-08-28T02:25:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:02 smithi130 bash[27579]: cluster 2023-08-28T02:25:01.587254+0000 mgr.y (mgr.14901) 7992 : cluster [DBG] pgmap v7999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:02.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:02 smithi130 bash[51302]: debug 2023-08-28T02:25:02.595+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:02.596303+0000) 2023-08-28T02:25:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:25:02.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:25:02.859Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:02.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:02 smithi006 bash[27292]: cluster 2023-08-28T02:25:01.587254+0000 mgr.y (mgr.14901) 7992 : cluster [DBG] pgmap v7999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:02.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:02 smithi006 bash[33918]: cluster 2023-08-28T02:25:01.587254+0000 mgr.y (mgr.14901) 7992 : cluster [DBG] pgmap v7999: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:03.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:03 smithi130 bash[27579]: audit 2023-08-28T02:25:02.969615+0000 mon.a (mon.0) 5075 : audit [DBG] from='client.? 172.21.15.6:0/1994115672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:03.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:03 smithi130 bash[51302]: debug 2023-08-28T02:25:03.591+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:03.596609+0000) 2023-08-28T02:25:03.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:03 smithi006 bash[33918]: audit 2023-08-28T02:25:02.969615+0000 mon.a (mon.0) 5075 : audit [DBG] from='client.? 172.21.15.6:0/1994115672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:03.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:03 smithi006 bash[27292]: audit 2023-08-28T02:25:02.969615+0000 mon.a (mon.0) 5075 : audit [DBG] from='client.? 172.21.15.6:0/1994115672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:04.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:04 smithi130 bash[27579]: cluster 2023-08-28T02:25:03.587915+0000 mgr.y (mgr.14901) 7993 : cluster [DBG] pgmap v8000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:04.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:04 smithi130 bash[51302]: debug 2023-08-28T02:25:04.595+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:04.596899+0000) 2023-08-28T02:25:04.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:04 smithi006 bash[27292]: cluster 2023-08-28T02:25:03.587915+0000 mgr.y (mgr.14901) 7993 : cluster [DBG] pgmap v8000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:04.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:04 smithi006 bash[33918]: cluster 2023-08-28T02:25:03.587915+0000 mgr.y (mgr.14901) 7993 : cluster [DBG] pgmap v8000: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:25:04.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:04.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:25:04.741Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:05.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:05 smithi130 bash[27579]: audit 2023-08-28T02:25:05.382462+0000 mon.c (mon.2) 3165 : audit [DBG] from='client.? 172.21.15.6:0/2548661285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:05.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:05 smithi130 bash[51302]: debug 2023-08-28T02:25:05.591+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:05.597189+0000) 2023-08-28T02:25:05.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:05 smithi006 bash[33918]: audit 2023-08-28T02:25:05.382462+0000 mon.c (mon.2) 3165 : audit [DBG] from='client.? 172.21.15.6:0/2548661285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:05.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:05 smithi006 bash[27292]: audit 2023-08-28T02:25:05.382462+0000 mon.c (mon.2) 3165 : audit [DBG] from='client.? 172.21.15.6:0/2548661285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:06.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:06 smithi130 bash[27579]: cluster 2023-08-28T02:25:05.589036+0000 mgr.y (mgr.14901) 7994 : cluster [DBG] pgmap v8001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:06.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:06 smithi130 bash[51302]: debug 2023-08-28T02:25:06.591+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:06.597446+0000) 2023-08-28T02:25:06.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:06 smithi006 bash[27292]: cluster 2023-08-28T02:25:05.589036+0000 mgr.y (mgr.14901) 7994 : cluster [DBG] pgmap v8001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:06.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:06 smithi006 bash[33918]: cluster 2023-08-28T02:25:05.589036+0000 mgr.y (mgr.14901) 7994 : cluster [DBG] pgmap v8001: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:07.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:07 smithi130 bash[51302]: debug 2023-08-28T02:25:07.591+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:07.597725+0000) 2023-08-28T02:25:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:08 smithi130 bash[27579]: cluster 2023-08-28T02:25:07.589671+0000 mgr.y (mgr.14901) 7995 : cluster [DBG] pgmap v8002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:08 smithi130 bash[27579]: audit 2023-08-28T02:25:07.796212+0000 mon.c (mon.2) 3166 : audit [DBG] from='client.? 172.21.15.6:0/2602149113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:08.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:08 smithi130 bash[51302]: debug 2023-08-28T02:25:08.595+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:08.598016+0000) 2023-08-28T02:25:08.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:08 smithi006 bash[27292]: cluster 2023-08-28T02:25:07.589671+0000 mgr.y (mgr.14901) 7995 : cluster [DBG] pgmap v8002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:08.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:08 smithi006 bash[27292]: audit 2023-08-28T02:25:07.796212+0000 mon.c (mon.2) 3166 : audit [DBG] from='client.? 172.21.15.6:0/2602149113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:08.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:08 smithi006 bash[33918]: cluster 2023-08-28T02:25:07.589671+0000 mgr.y (mgr.14901) 7995 : cluster [DBG] pgmap v8002: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:08.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:08 smithi006 bash[33918]: audit 2023-08-28T02:25:07.796212+0000 mon.c (mon.2) 3166 : audit [DBG] from='client.? 172.21.15.6:0/2602149113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:09.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:09 smithi130 bash[51302]: debug 2023-08-28T02:25:09.591+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:09.598325+0000) 2023-08-28T02:25:10.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:10 smithi130 bash[51302]: debug 2023-08-28T02:25:10.595+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:10.598623+0000) 2023-08-28T02:25:10.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:10 smithi130 bash[27579]: cluster 2023-08-28T02:25:09.590749+0000 mgr.y (mgr.14901) 7996 : cluster [DBG] pgmap v8003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:10.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:10 smithi130 bash[27579]: audit 2023-08-28T02:25:10.207875+0000 mon.a (mon.0) 5076 : audit [DBG] from='client.? 172.21.15.6:0/1197988122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:10.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:25:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:25:10] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:25:10.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:10 smithi006 bash[27292]: cluster 2023-08-28T02:25:09.590749+0000 mgr.y (mgr.14901) 7996 : cluster [DBG] pgmap v8003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:10.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:10 smithi006 bash[27292]: audit 2023-08-28T02:25:10.207875+0000 mon.a (mon.0) 5076 : audit [DBG] from='client.? 172.21.15.6:0/1197988122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:10.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:10 smithi006 bash[33918]: cluster 2023-08-28T02:25:09.590749+0000 mgr.y (mgr.14901) 7996 : cluster [DBG] pgmap v8003: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:10.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:10 smithi006 bash[33918]: audit 2023-08-28T02:25:10.207875+0000 mon.a (mon.0) 5076 : audit [DBG] from='client.? 172.21.15.6:0/1197988122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:25:11.297Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:25:11.299Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:11.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:11 smithi130 bash[51302]: debug 2023-08-28T02:25:11.595+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:11.598957+0000) 2023-08-28T02:25:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:12 smithi130 bash[27579]: cluster 2023-08-28T02:25:11.591383+0000 mgr.y (mgr.14901) 7997 : cluster [DBG] pgmap v8004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:12 smithi130 bash[27579]: audit 2023-08-28T02:25:12.620324+0000 mon.c (mon.2) 3167 : audit [DBG] from='client.? 172.21.15.6:0/4237964366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:12.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:12 smithi130 bash[51302]: debug 2023-08-28T02:25:12.595+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:12.599173+0000) 2023-08-28T02:25:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:25:12.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:12.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:25:12.861Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:12.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:12 smithi006 bash[27292]: cluster 2023-08-28T02:25:11.591383+0000 mgr.y (mgr.14901) 7997 : cluster [DBG] pgmap v8004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:12.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:12 smithi006 bash[27292]: audit 2023-08-28T02:25:12.620324+0000 mon.c (mon.2) 3167 : audit [DBG] from='client.? 172.21.15.6:0/4237964366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:12.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:12 smithi006 bash[33918]: cluster 2023-08-28T02:25:11.591383+0000 mgr.y (mgr.14901) 7997 : cluster [DBG] pgmap v8004: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:12.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:12 smithi006 bash[33918]: audit 2023-08-28T02:25:12.620324+0000 mon.c (mon.2) 3167 : audit [DBG] from='client.? 172.21.15.6:0/4237964366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:13.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:13 smithi130 bash[51302]: debug 2023-08-28T02:25:13.594+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:13.599391+0000) 2023-08-28T02:25:14.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:13 smithi130 bash[27579]: cluster 2023-08-28T02:25:13.592028+0000 mgr.y (mgr.14901) 7998 : cluster [DBG] pgmap v8005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:14.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:13 smithi006 bash[27292]: cluster 2023-08-28T02:25:13.592028+0000 mgr.y (mgr.14901) 7998 : cluster [DBG] pgmap v8005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:13 smithi006 bash[33918]: cluster 2023-08-28T02:25:13.592028+0000 mgr.y (mgr.14901) 7998 : cluster [DBG] pgmap v8005: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:14.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:14 smithi130 bash[51302]: debug 2023-08-28T02:25:14.594+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:14.599695+0000) 2023-08-28T02:25:15.087 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:25:14.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:15.087 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:25:14.742Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:15.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:15 smithi130 bash[27579]: audit 2023-08-28T02:25:15.033453+0000 mon.c (mon.2) 3168 : audit [DBG] from='client.? 172.21.15.6:0/900329007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:15.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:15 smithi006 bash[27292]: audit 2023-08-28T02:25:15.033453+0000 mon.c (mon.2) 3168 : audit [DBG] from='client.? 172.21.15.6:0/900329007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:15.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:15 smithi006 bash[33918]: audit 2023-08-28T02:25:15.033453+0000 mon.c (mon.2) 3168 : audit [DBG] from='client.? 172.21.15.6:0/900329007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:15.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:15 smithi130 bash[51302]: debug 2023-08-28T02:25:15.594+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:15.599981+0000) 2023-08-28T02:25:16.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:16 smithi130 bash[27579]: cluster 2023-08-28T02:25:15.593144+0000 mgr.y (mgr.14901) 7999 : cluster [DBG] pgmap v8006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:16.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:16 smithi006 bash[27292]: cluster 2023-08-28T02:25:15.593144+0000 mgr.y (mgr.14901) 7999 : cluster [DBG] pgmap v8006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:16.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:16 smithi006 bash[33918]: cluster 2023-08-28T02:25:15.593144+0000 mgr.y (mgr.14901) 7999 : cluster [DBG] pgmap v8006: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:16.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:16 smithi130 bash[51302]: debug 2023-08-28T02:25:16.598+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:16.600290+0000) 2023-08-28T02:25:17.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:17 smithi130 bash[27579]: audit 2023-08-28T02:25:17.438378+0000 mon.c (mon.2) 3169 : audit [DBG] from='client.? 172.21.15.6:0/3019170162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:17.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:17 smithi130 bash[51302]: debug 2023-08-28T02:25:17.594+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:17.600542+0000) 2023-08-28T02:25:17.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:17 smithi006 bash[27292]: audit 2023-08-28T02:25:17.438378+0000 mon.c (mon.2) 3169 : audit [DBG] from='client.? 172.21.15.6:0/3019170162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:17.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:17 smithi006 bash[33918]: audit 2023-08-28T02:25:17.438378+0000 mon.c (mon.2) 3169 : audit [DBG] from='client.? 172.21.15.6:0/3019170162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:18.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:18 smithi130 bash[27579]: cluster 2023-08-28T02:25:17.593755+0000 mgr.y (mgr.14901) 8000 : cluster [DBG] pgmap v8007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:18.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:18 smithi130 bash[51302]: debug 2023-08-28T02:25:18.594+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:18.600832+0000) 2023-08-28T02:25:18.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:18 smithi006 bash[27292]: cluster 2023-08-28T02:25:17.593755+0000 mgr.y (mgr.14901) 8000 : cluster [DBG] pgmap v8007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:18.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:18 smithi006 bash[33918]: cluster 2023-08-28T02:25:17.593755+0000 mgr.y (mgr.14901) 8000 : cluster [DBG] pgmap v8007: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:19.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:19 smithi130 bash[51302]: debug 2023-08-28T02:25:19.598+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:19.601092+0000) 2023-08-28T02:25:20.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:20 smithi130 bash[51302]: debug 2023-08-28T02:25:20.594+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:20.601426+0000) 2023-08-28T02:25:20.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:20 smithi130 bash[27579]: cluster 2023-08-28T02:25:19.594746+0000 mgr.y (mgr.14901) 8001 : cluster [DBG] pgmap v8008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:20.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:20 smithi130 bash[27579]: audit 2023-08-28T02:25:19.852925+0000 mon.a (mon.0) 5077 : audit [DBG] from='client.? 172.21.15.6:0/3877982440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:20.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:25:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:25:20] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T02:25:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:20 smithi006 bash[27292]: cluster 2023-08-28T02:25:19.594746+0000 mgr.y (mgr.14901) 8001 : cluster [DBG] pgmap v8008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:20 smithi006 bash[27292]: audit 2023-08-28T02:25:19.852925+0000 mon.a (mon.0) 5077 : audit [DBG] from='client.? 172.21.15.6:0/3877982440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:20.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:20 smithi006 bash[33918]: cluster 2023-08-28T02:25:19.594746+0000 mgr.y (mgr.14901) 8001 : cluster [DBG] pgmap v8008: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:20.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:20 smithi006 bash[33918]: audit 2023-08-28T02:25:19.852925+0000 mon.a (mon.0) 5077 : audit [DBG] from='client.? 172.21.15.6:0/3877982440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:25:21.298Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:25:21.300Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:21.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:21 smithi130 bash[51302]: debug 2023-08-28T02:25:21.598+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:21.601702+0000) 2023-08-28T02:25:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:22 smithi130 bash[27579]: cluster 2023-08-28T02:25:21.595391+0000 mgr.y (mgr.14901) 8002 : cluster [DBG] pgmap v8009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:22.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:22 smithi130 bash[27579]: audit 2023-08-28T02:25:22.264789+0000 mon.c (mon.2) 3170 : audit [DBG] from='client.? 172.21.15.6:0/632409671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:22.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:22 smithi130 bash[51302]: debug 2023-08-28T02:25:22.598+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:22.601970+0000) 2023-08-28T02:25:22.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:22 smithi006 bash[33918]: cluster 2023-08-28T02:25:21.595391+0000 mgr.y (mgr.14901) 8002 : cluster [DBG] pgmap v8009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:22.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:22 smithi006 bash[33918]: audit 2023-08-28T02:25:22.264789+0000 mon.c (mon.2) 3170 : audit [DBG] from='client.? 172.21.15.6:0/632409671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:25:22.859Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:25:22.862Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:22.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:22 smithi006 bash[27292]: cluster 2023-08-28T02:25:21.595391+0000 mgr.y (mgr.14901) 8002 : cluster [DBG] pgmap v8009: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:22.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:22 smithi006 bash[27292]: audit 2023-08-28T02:25:22.264789+0000 mon.c (mon.2) 3170 : audit [DBG] from='client.? 172.21.15.6:0/632409671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:23.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:23 smithi130 bash[51302]: debug 2023-08-28T02:25:23.598+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:23.602281+0000) 2023-08-28T02:25:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:23 smithi130 bash[27579]: cluster 2023-08-28T02:25:23.595992+0000 mgr.y (mgr.14901) 8003 : cluster [DBG] pgmap v8010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:24.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:23 smithi006 bash[33918]: cluster 2023-08-28T02:25:23.595992+0000 mgr.y (mgr.14901) 8003 : cluster [DBG] pgmap v8010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:23 smithi006 bash[27292]: cluster 2023-08-28T02:25:23.595992+0000 mgr.y (mgr.14901) 8003 : cluster [DBG] pgmap v8010: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:24.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:24 smithi130 bash[51302]: debug 2023-08-28T02:25:24.598+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:24.602617+0000) 2023-08-28T02:25:25.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:24 smithi006 bash[33918]: audit 2023-08-28T02:25:24.683741+0000 mon.c (mon.2) 3171 : audit [DBG] from='client.? 172.21.15.6:0/4267420930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:25.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:24 smithi006 bash[27292]: audit 2023-08-28T02:25:24.683741+0000 mon.c (mon.2) 3171 : audit [DBG] from='client.? 172.21.15.6:0/4267420930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:25.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:25:24.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:25.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:25:24.743Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:25.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:24 smithi130 bash[27579]: audit 2023-08-28T02:25:24.683741+0000 mon.c (mon.2) 3171 : audit [DBG] from='client.? 172.21.15.6:0/4267420930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:25.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:25 smithi130 bash[51302]: debug 2023-08-28T02:25:25.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:25.602908+0000) 2023-08-28T02:25:26.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:25 smithi130 bash[27579]: cluster 2023-08-28T02:25:25.597233+0000 mgr.y (mgr.14901) 8004 : cluster [DBG] pgmap v8011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:26.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:25 smithi006 bash[27292]: cluster 2023-08-28T02:25:25.597233+0000 mgr.y (mgr.14901) 8004 : cluster [DBG] pgmap v8011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:26.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:25 smithi006 bash[33918]: cluster 2023-08-28T02:25:25.597233+0000 mgr.y (mgr.14901) 8004 : cluster [DBG] pgmap v8011: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:26.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:26 smithi130 bash[51302]: debug 2023-08-28T02:25:26.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:26.603182+0000) 2023-08-28T02:25:27.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:27 smithi006 bash[27292]: audit 2023-08-28T02:25:27.112658+0000 mon.a (mon.0) 5078 : audit [DBG] from='client.? 172.21.15.6:0/1132603089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:27.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:27 smithi006 bash[33918]: audit 2023-08-28T02:25:27.112658+0000 mon.a (mon.0) 5078 : audit [DBG] from='client.? 172.21.15.6:0/1132603089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:27.602 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:27 smithi130 bash[27579]: audit 2023-08-28T02:25:27.112658+0000 mon.a (mon.0) 5078 : audit [DBG] from='client.? 172.21.15.6:0/1132603089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:27.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:27 smithi130 bash[51302]: debug 2023-08-28T02:25:27.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:27.603385+0000) 2023-08-28T02:25:28.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:28 smithi006 bash[27292]: cluster 2023-08-28T02:25:27.597922+0000 mgr.y (mgr.14901) 8005 : cluster [DBG] pgmap v8012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:28.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:28 smithi006 bash[33918]: cluster 2023-08-28T02:25:27.597922+0000 mgr.y (mgr.14901) 8005 : cluster [DBG] pgmap v8012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:28.602 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:28 smithi130 bash[27579]: cluster 2023-08-28T02:25:27.597922+0000 mgr.y (mgr.14901) 8005 : cluster [DBG] pgmap v8012: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:28.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:28 smithi130 bash[51302]: debug 2023-08-28T02:25:28.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:28.603652+0000) 2023-08-28T02:25:29.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:29 smithi130 bash[27579]: audit 2023-08-28T02:25:29.528215+0000 mon.c (mon.2) 3172 : audit [DBG] from='client.? 172.21.15.6:0/1620890445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:29.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:29 smithi130 bash[51302]: debug 2023-08-28T02:25:29.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:29.604003+0000) 2023-08-28T02:25:29.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:29 smithi006 bash[27292]: audit 2023-08-28T02:25:29.528215+0000 mon.c (mon.2) 3172 : audit [DBG] from='client.? 172.21.15.6:0/1620890445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:29.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:29 smithi006 bash[33918]: audit 2023-08-28T02:25:29.528215+0000 mon.c (mon.2) 3172 : audit [DBG] from='client.? 172.21.15.6:0/1620890445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:30.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:30 smithi130 bash[51302]: debug 2023-08-28T02:25:30.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:30.604314+0000) 2023-08-28T02:25:30.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:30 smithi130 bash[27579]: cluster 2023-08-28T02:25:29.598905+0000 mgr.y (mgr.14901) 8006 : cluster [DBG] pgmap v8013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:30.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:25:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:25:30] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:25:30.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:30 smithi006 bash[33918]: cluster 2023-08-28T02:25:29.598905+0000 mgr.y (mgr.14901) 8006 : cluster [DBG] pgmap v8013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:30.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:30 smithi006 bash[27292]: cluster 2023-08-28T02:25:29.598905+0000 mgr.y (mgr.14901) 8006 : cluster [DBG] pgmap v8013: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:25:31.299Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:25:31.300Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:31.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:31 smithi130 bash[51302]: debug 2023-08-28T02:25:31.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:31.604594+0000) 2023-08-28T02:25:32.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:32 smithi130 bash[51302]: debug 2023-08-28T02:25:32.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:32.604899+0000) 2023-08-28T02:25:32.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:32 smithi130 bash[27579]: cluster 2023-08-28T02:25:31.599499+0000 mgr.y (mgr.14901) 8007 : cluster [DBG] pgmap v8014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:32.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:32 smithi130 bash[27579]: audit 2023-08-28T02:25:31.942043+0000 mon.c (mon.2) 3173 : audit [DBG] from='client.? 172.21.15.6:0/1968569207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:32.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:32 smithi006 bash[33918]: cluster 2023-08-28T02:25:31.599499+0000 mgr.y (mgr.14901) 8007 : cluster [DBG] pgmap v8014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:32.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:32 smithi006 bash[33918]: audit 2023-08-28T02:25:31.942043+0000 mon.c (mon.2) 3173 : audit [DBG] from='client.? 172.21.15.6:0/1968569207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:25:32.860Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:25:32.862Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:32.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:32 smithi006 bash[27292]: cluster 2023-08-28T02:25:31.599499+0000 mgr.y (mgr.14901) 8007 : cluster [DBG] pgmap v8014: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:32.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:32 smithi006 bash[27292]: audit 2023-08-28T02:25:31.942043+0000 mon.c (mon.2) 3173 : audit [DBG] from='client.? 172.21.15.6:0/1968569207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:33.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:33 smithi130 bash[51302]: debug 2023-08-28T02:25:33.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:33.605166+0000) 2023-08-28T02:25:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:33 smithi130 bash[27579]: cluster 2023-08-28T02:25:33.600210+0000 mgr.y (mgr.14901) 8008 : cluster [DBG] pgmap v8015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:34.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:33 smithi006 bash[27292]: cluster 2023-08-28T02:25:33.600210+0000 mgr.y (mgr.14901) 8008 : cluster [DBG] pgmap v8015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:33 smithi006 bash[33918]: cluster 2023-08-28T02:25:33.600210+0000 mgr.y (mgr.14901) 8008 : cluster [DBG] pgmap v8015: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:34.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:34 smithi130 bash[51302]: debug 2023-08-28T02:25:34.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:34.605461+0000) 2023-08-28T02:25:35.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:34 smithi006 bash[27292]: audit 2023-08-28T02:25:34.353298+0000 mon.c (mon.2) 3174 : audit [DBG] from='client.? 172.21.15.6:0/1582448371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:35.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:34 smithi006 bash[33918]: audit 2023-08-28T02:25:34.353298+0000 mon.c (mon.2) 3174 : audit [DBG] from='client.? 172.21.15.6:0/1582448371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:35.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:25:34.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:35.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:25:34.744Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:35.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:34 smithi130 bash[27579]: audit 2023-08-28T02:25:34.353298+0000 mon.c (mon.2) 3174 : audit [DBG] from='client.? 172.21.15.6:0/1582448371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:35.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:35 smithi130 bash[51302]: debug 2023-08-28T02:25:35.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:35.605735+0000) 2023-08-28T02:25:36.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:35 smithi130 bash[27579]: cluster 2023-08-28T02:25:35.601457+0000 mgr.y (mgr.14901) 8009 : cluster [DBG] pgmap v8016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:36.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:35 smithi006 bash[33918]: cluster 2023-08-28T02:25:35.601457+0000 mgr.y (mgr.14901) 8009 : cluster [DBG] pgmap v8016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:36.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:35 smithi006 bash[27292]: cluster 2023-08-28T02:25:35.601457+0000 mgr.y (mgr.14901) 8009 : cluster [DBG] pgmap v8016: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:36.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:36 smithi130 bash[51302]: debug 2023-08-28T02:25:36.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:36.606023+0000) 2023-08-28T02:25:37.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:37 smithi130 bash[27579]: audit 2023-08-28T02:25:36.772156+0000 mon.c (mon.2) 3175 : audit [DBG] from='client.? 172.21.15.6:0/4133000945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:37.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:37 smithi130 bash[27579]: audit 2023-08-28T02:25:36.888730+0000 mon.b (mon.1) 876 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:25:37.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:36 smithi006 bash[33918]: audit 2023-08-28T02:25:36.772156+0000 mon.c (mon.2) 3175 : audit [DBG] from='client.? 172.21.15.6:0/4133000945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:37.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:36 smithi006 bash[33918]: audit 2023-08-28T02:25:36.888730+0000 mon.b (mon.1) 876 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:25:37.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:36 smithi006 bash[27292]: audit 2023-08-28T02:25:36.772156+0000 mon.c (mon.2) 3175 : audit [DBG] from='client.? 172.21.15.6:0/4133000945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:37.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:36 smithi006 bash[27292]: audit 2023-08-28T02:25:36.888730+0000 mon.b (mon.1) 876 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:25:37.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:37 smithi130 bash[51302]: debug 2023-08-28T02:25:37.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:37.606342+0000) 2023-08-28T02:25:38.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:38 smithi130 bash[27579]: cluster 2023-08-28T02:25:37.602182+0000 mgr.y (mgr.14901) 8010 : cluster [DBG] pgmap v8017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:38.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:38 smithi006 bash[27292]: cluster 2023-08-28T02:25:37.602182+0000 mgr.y (mgr.14901) 8010 : cluster [DBG] pgmap v8017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:38.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:38 smithi006 bash[33918]: cluster 2023-08-28T02:25:37.602182+0000 mgr.y (mgr.14901) 8010 : cluster [DBG] pgmap v8017: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:38.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:38 smithi130 bash[51302]: debug 2023-08-28T02:25:38.602+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:38.606678+0000) 2023-08-28T02:25:39.605 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:39 smithi130 bash[27579]: audit 2023-08-28T02:25:39.187178+0000 mon.c (mon.2) 3176 : audit [DBG] from='client.? 172.21.15.6:0/3127823688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:39.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:39 smithi006 bash[33918]: audit 2023-08-28T02:25:39.187178+0000 mon.c (mon.2) 3176 : audit [DBG] from='client.? 172.21.15.6:0/3127823688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:39.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:39 smithi006 bash[27292]: audit 2023-08-28T02:25:39.187178+0000 mon.c (mon.2) 3176 : audit [DBG] from='client.? 172.21.15.6:0/3127823688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:39.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:39 smithi130 bash[51302]: debug 2023-08-28T02:25:39.606+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:39.606984+0000) 2023-08-28T02:25:40.606 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:40 smithi130 bash[27579]: cluster 2023-08-28T02:25:39.603290+0000 mgr.y (mgr.14901) 8011 : cluster [DBG] pgmap v8018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:40.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:40 smithi006 bash[33918]: cluster 2023-08-28T02:25:39.603290+0000 mgr.y (mgr.14901) 8011 : cluster [DBG] pgmap v8018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:40 smithi006 bash[27292]: cluster 2023-08-28T02:25:39.603290+0000 mgr.y (mgr.14901) 8011 : cluster [DBG] pgmap v8018: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:40.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:40 smithi130 bash[51302]: debug 2023-08-28T02:25:40.606+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:40.607261+0000) 2023-08-28T02:25:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:25:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:25:40] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:25:41.655 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:25:41.299Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:41.656 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:25:41.301Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:41.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:41 smithi130 bash[27579]: audit 2023-08-28T02:25:41.602682+0000 mon.a (mon.0) 5079 : audit [DBG] from='client.? 172.21.15.6:0/605198541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:41.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:41 smithi130 bash[51302]: debug 2023-08-28T02:25:41.606+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:41.607502+0000) 2023-08-28T02:25:41.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:41 smithi006 bash[33918]: audit 2023-08-28T02:25:41.602682+0000 mon.a (mon.0) 5079 : audit [DBG] from='client.? 172.21.15.6:0/605198541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:41.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:41 smithi006 bash[27292]: audit 2023-08-28T02:25:41.602682+0000 mon.a (mon.0) 5079 : audit [DBG] from='client.? 172.21.15.6:0/605198541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:42.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:42 smithi130 bash[27579]: cluster 2023-08-28T02:25:41.603912+0000 mgr.y (mgr.14901) 8012 : cluster [DBG] pgmap v8019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:42.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:42 smithi130 bash[51302]: debug 2023-08-28T02:25:42.606+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:42.607726+0000) 2023-08-28T02:25:42.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:42 smithi006 bash[27292]: cluster 2023-08-28T02:25:41.603912+0000 mgr.y (mgr.14901) 8012 : cluster [DBG] pgmap v8019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:42.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:42 smithi006 bash[33918]: cluster 2023-08-28T02:25:41.603912+0000 mgr.y (mgr.14901) 8012 : cluster [DBG] pgmap v8019: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:25:42.860Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:25:42.862Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:43.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:43 smithi130 bash[27579]: audit 2023-08-28T02:25:42.801793+0000 mon.a (mon.0) 5080 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:25:43.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:43 smithi130 bash[27579]: audit 2023-08-28T02:25:43.170809+0000 mon.a (mon.0) 5081 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:25:43.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:43 smithi130 bash[51302]: debug 2023-08-28T02:25:43.606+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:43.608011+0000) 2023-08-28T02:25:44.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:43 smithi006 bash[33918]: audit 2023-08-28T02:25:42.801793+0000 mon.a (mon.0) 5080 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:25:44.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:43 smithi006 bash[33918]: audit 2023-08-28T02:25:43.170809+0000 mon.a (mon.0) 5081 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:25:44.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:43 smithi006 bash[27292]: audit 2023-08-28T02:25:42.801793+0000 mon.a (mon.0) 5080 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:25:44.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:43 smithi006 bash[27292]: audit 2023-08-28T02:25:43.170809+0000 mon.a (mon.0) 5081 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:25:44.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:44 smithi130 bash[51302]: debug 2023-08-28T02:25:44.606+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:44.608295+0000) 2023-08-28T02:25:44.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:44 smithi130 bash[27579]: cluster 2023-08-28T02:25:43.604575+0000 mgr.y (mgr.14901) 8013 : cluster [DBG] pgmap v8020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:44.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:44 smithi130 bash[27579]: audit 2023-08-28T02:25:44.022626+0000 mon.a (mon.0) 5082 : audit [DBG] from='client.? 172.21.15.6:0/2058009388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:45.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:44 smithi006 bash[33918]: cluster 2023-08-28T02:25:43.604575+0000 mgr.y (mgr.14901) 8013 : cluster [DBG] pgmap v8020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:45.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:44 smithi006 bash[33918]: audit 2023-08-28T02:25:44.022626+0000 mon.a (mon.0) 5082 : audit [DBG] from='client.? 172.21.15.6:0/2058009388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:45.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:44 smithi006 bash[27292]: cluster 2023-08-28T02:25:43.604575+0000 mgr.y (mgr.14901) 8013 : cluster [DBG] pgmap v8020: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:45.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:44 smithi006 bash[27292]: audit 2023-08-28T02:25:44.022626+0000 mon.a (mon.0) 5082 : audit [DBG] from='client.? 172.21.15.6:0/2058009388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:45.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:25:44.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:45.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:25:44.745Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:45.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:45 smithi130 bash[51302]: debug 2023-08-28T02:25:45.606+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:45.608546+0000) 2023-08-28T02:25:46.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:46 smithi130 bash[27579]: cluster 2023-08-28T02:25:45.605736+0000 mgr.y (mgr.14901) 8014 : cluster [DBG] pgmap v8021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:46.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:46 smithi130 bash[27579]: audit 2023-08-28T02:25:46.435192+0000 mon.a (mon.0) 5083 : audit [DBG] from='client.? 172.21.15.6:0/786394145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:46.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:46 smithi130 bash[51302]: debug 2023-08-28T02:25:46.606+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:46.608840+0000) 2023-08-28T02:25:47.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:46 smithi006 bash[33918]: cluster 2023-08-28T02:25:45.605736+0000 mgr.y (mgr.14901) 8014 : cluster [DBG] pgmap v8021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:47.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:46 smithi006 bash[33918]: audit 2023-08-28T02:25:46.435192+0000 mon.a (mon.0) 5083 : audit [DBG] from='client.? 172.21.15.6:0/786394145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:47.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:46 smithi006 bash[27292]: cluster 2023-08-28T02:25:45.605736+0000 mgr.y (mgr.14901) 8014 : cluster [DBG] pgmap v8021: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:47.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:46 smithi006 bash[27292]: audit 2023-08-28T02:25:46.435192+0000 mon.a (mon.0) 5083 : audit [DBG] from='client.? 172.21.15.6:0/786394145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:47.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:47 smithi130 bash[51302]: debug 2023-08-28T02:25:47.606+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:47.609095+0000) 2023-08-28T02:25:48.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:48 smithi130 bash[27579]: cluster 2023-08-28T02:25:47.606409+0000 mgr.y (mgr.14901) 8015 : cluster [DBG] pgmap v8022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:48.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:48 smithi130 bash[51302]: debug 2023-08-28T02:25:48.606+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:48.609361+0000) 2023-08-28T02:25:49.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:48 smithi006 bash[27292]: cluster 2023-08-28T02:25:47.606409+0000 mgr.y (mgr.14901) 8015 : cluster [DBG] pgmap v8022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:49.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:48 smithi006 bash[33918]: cluster 2023-08-28T02:25:47.606409+0000 mgr.y (mgr.14901) 8015 : cluster [DBG] pgmap v8022: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:49.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:49 smithi130 bash[27579]: audit 2023-08-28T02:25:48.847688+0000 mon.c (mon.2) 3177 : audit [DBG] from='client.? 172.21.15.6:0/2992959747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:49.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:49 smithi130 bash[51302]: debug 2023-08-28T02:25:49.606+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:49.609585+0000) 2023-08-28T02:25:50.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:49 smithi006 bash[33918]: audit 2023-08-28T02:25:48.847688+0000 mon.c (mon.2) 3177 : audit [DBG] from='client.? 172.21.15.6:0/2992959747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:50.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:49 smithi006 bash[27292]: audit 2023-08-28T02:25:48.847688+0000 mon.c (mon.2) 3177 : audit [DBG] from='client.? 172.21.15.6:0/2992959747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:50.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:50 smithi130 bash[51302]: debug 2023-08-28T02:25:50.606+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:50.609872+0000) 2023-08-28T02:25:50.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:50 smithi130 bash[27579]: cluster 2023-08-28T02:25:49.607617+0000 mgr.y (mgr.14901) 8016 : cluster [DBG] pgmap v8023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:25:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:25:50] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T02:25:51.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:50 smithi006 bash[33918]: cluster 2023-08-28T02:25:49.607617+0000 mgr.y (mgr.14901) 8016 : cluster [DBG] pgmap v8023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:51.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:50 smithi006 bash[27292]: cluster 2023-08-28T02:25:49.607617+0000 mgr.y (mgr.14901) 8016 : cluster [DBG] pgmap v8023: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:25:51.299Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:25:51.301Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:51.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:51 smithi130 bash[27579]: audit 2023-08-28T02:25:51.255456+0000 mon.c (mon.2) 3178 : audit [DBG] from='client.? 172.21.15.6:0/668093972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:51.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:51 smithi130 bash[51302]: debug 2023-08-28T02:25:51.606+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:51.610178+0000) 2023-08-28T02:25:52.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:51 smithi006 bash[27292]: audit 2023-08-28T02:25:51.255456+0000 mon.c (mon.2) 3178 : audit [DBG] from='client.? 172.21.15.6:0/668093972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:52.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:51 smithi006 bash[33918]: audit 2023-08-28T02:25:51.255456+0000 mon.c (mon.2) 3178 : audit [DBG] from='client.? 172.21.15.6:0/668093972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:52.838 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:52 smithi130 bash[51302]: debug 2023-08-28T02:25:52.606+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:52.610453+0000) 2023-08-28T02:25:53.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:52 smithi130 bash[27579]: cluster 2023-08-28T02:25:51.608400+0000 mgr.y (mgr.14901) 8017 : cluster [DBG] pgmap v8024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:53.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:52 smithi006 bash[33918]: cluster 2023-08-28T02:25:51.608400+0000 mgr.y (mgr.14901) 8017 : cluster [DBG] pgmap v8024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:53.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:52 smithi006 bash[27292]: cluster 2023-08-28T02:25:51.608400+0000 mgr.y (mgr.14901) 8017 : cluster [DBG] pgmap v8024: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:25:52.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:25:52.863Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:53.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:53 smithi130 bash[51302]: debug 2023-08-28T02:25:53.606+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:53.610763+0000) 2023-08-28T02:25:54.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:54 smithi130 bash[51302]: debug 2023-08-28T02:25:54.610+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:54.611025+0000) 2023-08-28T02:25:54.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:54 smithi130 bash[27579]: audit 2023-08-28T02:25:53.534761+0000 mon.a (mon.0) 5084 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:25:54.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:54 smithi130 bash[27579]: cluster 2023-08-28T02:25:53.609051+0000 mgr.y (mgr.14901) 8018 : cluster [DBG] pgmap v8025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:54.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:54 smithi130 bash[27579]: audit 2023-08-28T02:25:53.675859+0000 mon.c (mon.2) 3179 : audit [DBG] from='client.? 172.21.15.6:0/2208597468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:54 smithi130 bash[27579]: audit 2023-08-28T02:25:53.825418+0000 mon.b (mon.1) 877 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:25:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:54 smithi130 bash[27579]: audit 2023-08-28T02:25:53.827109+0000 mon.b (mon.1) 878 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:25:54.867 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:54 smithi130 bash[27579]: audit 2023-08-28T02:25:53.834883+0000 mon.a (mon.0) 5085 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:25:54.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:25:54.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:54.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:25:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:25:54.746Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:25:54.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:54 smithi006 bash[33918]: audit 2023-08-28T02:25:53.534761+0000 mon.a (mon.0) 5084 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:25:54.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:54 smithi006 bash[33918]: cluster 2023-08-28T02:25:53.609051+0000 mgr.y (mgr.14901) 8018 : cluster [DBG] pgmap v8025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:54.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:54 smithi006 bash[33918]: audit 2023-08-28T02:25:53.675859+0000 mon.c (mon.2) 3179 : audit [DBG] from='client.? 172.21.15.6:0/2208597468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:54.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:54 smithi006 bash[33918]: audit 2023-08-28T02:25:53.825418+0000 mon.b (mon.1) 877 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:25:54.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:54 smithi006 bash[33918]: audit 2023-08-28T02:25:53.827109+0000 mon.b (mon.1) 878 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:25:54.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:54 smithi006 bash[33918]: audit 2023-08-28T02:25:53.834883+0000 mon.a (mon.0) 5085 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:25:54.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:54 smithi006 bash[27292]: audit 2023-08-28T02:25:53.534761+0000 mon.a (mon.0) 5084 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:25:54.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:54 smithi006 bash[27292]: cluster 2023-08-28T02:25:53.609051+0000 mgr.y (mgr.14901) 8018 : cluster [DBG] pgmap v8025: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:54.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:54 smithi006 bash[27292]: audit 2023-08-28T02:25:53.675859+0000 mon.c (mon.2) 3179 : audit [DBG] from='client.? 172.21.15.6:0/2208597468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:54.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:54 smithi006 bash[27292]: audit 2023-08-28T02:25:53.825418+0000 mon.b (mon.1) 877 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:25:54.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:54 smithi006 bash[27292]: audit 2023-08-28T02:25:53.827109+0000 mon.b (mon.1) 878 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:25:54.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:54 smithi006 bash[27292]: audit 2023-08-28T02:25:53.834883+0000 mon.a (mon.0) 5085 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:25:55.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:55 smithi130 bash[51302]: debug 2023-08-28T02:25:55.610+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:55.611237+0000) 2023-08-28T02:25:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:56 smithi130 bash[27579]: cluster 2023-08-28T02:25:55.610139+0000 mgr.y (mgr.14901) 8019 : cluster [DBG] pgmap v8026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:56.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:56 smithi130 bash[27579]: audit 2023-08-28T02:25:56.093989+0000 mon.a (mon.0) 5086 : audit [DBG] from='client.? 172.21.15.6:0/1962029799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:56.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:56 smithi130 bash[51302]: debug 2023-08-28T02:25:56.610+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:56.611456+0000) 2023-08-28T02:25:56.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:56 smithi006 bash[33918]: cluster 2023-08-28T02:25:55.610139+0000 mgr.y (mgr.14901) 8019 : cluster [DBG] pgmap v8026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:56.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:56 smithi006 bash[33918]: audit 2023-08-28T02:25:56.093989+0000 mon.a (mon.0) 5086 : audit [DBG] from='client.? 172.21.15.6:0/1962029799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:56.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:56 smithi006 bash[27292]: cluster 2023-08-28T02:25:55.610139+0000 mgr.y (mgr.14901) 8019 : cluster [DBG] pgmap v8026: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:56.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:56 smithi006 bash[27292]: audit 2023-08-28T02:25:56.093989+0000 mon.a (mon.0) 5086 : audit [DBG] from='client.? 172.21.15.6:0/1962029799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:57.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:57 smithi130 bash[51302]: debug 2023-08-28T02:25:57.610+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:57.611743+0000) 2023-08-28T02:25:58.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:58 smithi130 bash[27579]: cluster 2023-08-28T02:25:57.610792+0000 mgr.y (mgr.14901) 8020 : cluster [DBG] pgmap v8027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:58.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:25:58 smithi130 bash[27579]: audit 2023-08-28T02:25:58.508234+0000 mon.a (mon.0) 5087 : audit [DBG] from='client.? 172.21.15.6:0/851303508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:58.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:58 smithi130 bash[51302]: debug 2023-08-28T02:25:58.610+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:58.612017+0000) 2023-08-28T02:25:58.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:58 smithi006 bash[33918]: cluster 2023-08-28T02:25:57.610792+0000 mgr.y (mgr.14901) 8020 : cluster [DBG] pgmap v8027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:58.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:25:58 smithi006 bash[33918]: audit 2023-08-28T02:25:58.508234+0000 mon.a (mon.0) 5087 : audit [DBG] from='client.? 172.21.15.6:0/851303508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:58.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:58 smithi006 bash[27292]: cluster 2023-08-28T02:25:57.610792+0000 mgr.y (mgr.14901) 8020 : cluster [DBG] pgmap v8027: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:25:58.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:25:58 smithi006 bash[27292]: audit 2023-08-28T02:25:58.508234+0000 mon.a (mon.0) 5087 : audit [DBG] from='client.? 172.21.15.6:0/851303508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:59.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:25:59 smithi130 bash[51302]: debug 2023-08-28T02:25:59.610+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:59.612343+0000) 2023-08-28T02:26:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:00 smithi130 bash[27579]: cluster 2023-08-28T02:25:59.611950+0000 mgr.y (mgr.14901) 8021 : cluster [DBG] pgmap v8028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:00.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:00 smithi130 bash[51302]: debug 2023-08-28T02:26:00.610+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:00.612606+0000) 2023-08-28T02:26:00.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:00 smithi006 bash[27292]: cluster 2023-08-28T02:25:59.611950+0000 mgr.y (mgr.14901) 8021 : cluster [DBG] pgmap v8028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:00.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:26:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:26:00] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T02:26:00.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:00 smithi006 bash[33918]: cluster 2023-08-28T02:25:59.611950+0000 mgr.y (mgr.14901) 8021 : cluster [DBG] pgmap v8028: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:01.685 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:26:01.300Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:01.685 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:26:01.302Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:01.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:01 smithi130 bash[27579]: audit 2023-08-28T02:26:00.925396+0000 mon.c (mon.2) 3180 : audit [DBG] from='client.? 172.21.15.6:0/2910702282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:01.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:01 smithi130 bash[51302]: debug 2023-08-28T02:26:01.610+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:01.612895+0000) 2023-08-28T02:26:01.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:01 smithi006 bash[27292]: audit 2023-08-28T02:26:00.925396+0000 mon.c (mon.2) 3180 : audit [DBG] from='client.? 172.21.15.6:0/2910702282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:01.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:01 smithi006 bash[33918]: audit 2023-08-28T02:26:00.925396+0000 mon.c (mon.2) 3180 : audit [DBG] from='client.? 172.21.15.6:0/2910702282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:02 smithi130 bash[27579]: cluster 2023-08-28T02:26:01.612625+0000 mgr.y (mgr.14901) 8022 : cluster [DBG] pgmap v8029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:02.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:02 smithi130 bash[51302]: debug 2023-08-28T02:26:02.610+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:02.613189+0000) 2023-08-28T02:26:02.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:02 smithi006 bash[27292]: cluster 2023-08-28T02:26:01.612625+0000 mgr.y (mgr.14901) 8022 : cluster [DBG] pgmap v8029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:02.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:02 smithi006 bash[33918]: cluster 2023-08-28T02:26:01.612625+0000 mgr.y (mgr.14901) 8022 : cluster [DBG] pgmap v8029: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:26:02.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:02.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:26:02.863Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:03.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:03 smithi130 bash[51302]: debug 2023-08-28T02:26:03.610+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:03.613455+0000) 2023-08-28T02:26:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:04 smithi130 bash[27579]: audit 2023-08-28T02:26:03.336503+0000 mon.a (mon.0) 5088 : audit [DBG] from='client.? 172.21.15.6:0/1465109818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:04 smithi130 bash[27579]: cluster 2023-08-28T02:26:03.613369+0000 mgr.y (mgr.14901) 8023 : cluster [DBG] pgmap v8030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:04.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:04 smithi006 bash[33918]: audit 2023-08-28T02:26:03.336503+0000 mon.a (mon.0) 5088 : audit [DBG] from='client.? 172.21.15.6:0/1465109818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:04.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:04 smithi006 bash[33918]: cluster 2023-08-28T02:26:03.613369+0000 mgr.y (mgr.14901) 8023 : cluster [DBG] pgmap v8030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:04 smithi006 bash[27292]: audit 2023-08-28T02:26:03.336503+0000 mon.a (mon.0) 5088 : audit [DBG] from='client.? 172.21.15.6:0/1465109818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:04 smithi006 bash[27292]: cluster 2023-08-28T02:26:03.613369+0000 mgr.y (mgr.14901) 8023 : cluster [DBG] pgmap v8030: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:04.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:04 smithi130 bash[51302]: debug 2023-08-28T02:26:04.610+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:04.613720+0000) 2023-08-28T02:26:05.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:26:04.744Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:05.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:26:04.746Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:05.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:05 smithi130 bash[51302]: debug 2023-08-28T02:26:05.610+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:05.614030+0000) 2023-08-28T02:26:06.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:06 smithi130 bash[27579]: cluster 2023-08-28T02:26:05.614427+0000 mgr.y (mgr.14901) 8024 : cluster [DBG] pgmap v8031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:06.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:06 smithi130 bash[27579]: audit 2023-08-28T02:26:05.749670+0000 mon.a (mon.0) 5089 : audit [DBG] from='client.? 172.21.15.6:0/3898039057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:06.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:06 smithi130 bash[51302]: debug 2023-08-28T02:26:06.610+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:06.614296+0000) 2023-08-28T02:26:06.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:06 smithi006 bash[33918]: cluster 2023-08-28T02:26:05.614427+0000 mgr.y (mgr.14901) 8024 : cluster [DBG] pgmap v8031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:06.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:06 smithi006 bash[33918]: audit 2023-08-28T02:26:05.749670+0000 mon.a (mon.0) 5089 : audit [DBG] from='client.? 172.21.15.6:0/3898039057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:06.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:06 smithi006 bash[27292]: cluster 2023-08-28T02:26:05.614427+0000 mgr.y (mgr.14901) 8024 : cluster [DBG] pgmap v8031: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:06.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:06 smithi006 bash[27292]: audit 2023-08-28T02:26:05.749670+0000 mon.a (mon.0) 5089 : audit [DBG] from='client.? 172.21.15.6:0/3898039057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:07.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:07 smithi130 bash[51302]: debug 2023-08-28T02:26:07.610+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:07.614568+0000) 2023-08-28T02:26:08.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:08 smithi130 bash[51302]: debug 2023-08-28T02:26:08.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:08.614848+0000) 2023-08-28T02:26:08.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:08 smithi130 bash[27579]: cluster 2023-08-28T02:26:07.615179+0000 mgr.y (mgr.14901) 8025 : cluster [DBG] pgmap v8032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:08.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:08 smithi130 bash[27579]: audit 2023-08-28T02:26:08.168243+0000 mon.a (mon.0) 5090 : audit [DBG] from='client.? 172.21.15.6:0/1986728087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:08.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:08 smithi006 bash[33918]: cluster 2023-08-28T02:26:07.615179+0000 mgr.y (mgr.14901) 8025 : cluster [DBG] pgmap v8032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:08.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:08 smithi006 bash[33918]: audit 2023-08-28T02:26:08.168243+0000 mon.a (mon.0) 5090 : audit [DBG] from='client.? 172.21.15.6:0/1986728087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:08.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:08 smithi006 bash[27292]: cluster 2023-08-28T02:26:07.615179+0000 mgr.y (mgr.14901) 8025 : cluster [DBG] pgmap v8032: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:08.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:08 smithi006 bash[27292]: audit 2023-08-28T02:26:08.168243+0000 mon.a (mon.0) 5090 : audit [DBG] from='client.? 172.21.15.6:0/1986728087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:09.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:09 smithi130 bash[51302]: debug 2023-08-28T02:26:09.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:09.615161+0000) 2023-08-28T02:26:10.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:10 smithi130 bash[27579]: cluster 2023-08-28T02:26:09.616260+0000 mgr.y (mgr.14901) 8026 : cluster [DBG] pgmap v8033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:10.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:10 smithi130 bash[27579]: audit 2023-08-28T02:26:10.593674+0000 mon.c (mon.2) 3181 : audit [DBG] from='client.? 172.21.15.6:0/2040449452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:10.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:10 smithi130 bash[51302]: debug 2023-08-28T02:26:10.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:10.615557+0000) 2023-08-28T02:26:10.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:26:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:26:10] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T02:26:10.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:10 smithi006 bash[33918]: cluster 2023-08-28T02:26:09.616260+0000 mgr.y (mgr.14901) 8026 : cluster [DBG] pgmap v8033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:10.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:10 smithi006 bash[33918]: audit 2023-08-28T02:26:10.593674+0000 mon.c (mon.2) 3181 : audit [DBG] from='client.? 172.21.15.6:0/2040449452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:10.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:10 smithi006 bash[27292]: cluster 2023-08-28T02:26:09.616260+0000 mgr.y (mgr.14901) 8026 : cluster [DBG] pgmap v8033: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:10.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:10 smithi006 bash[27292]: audit 2023-08-28T02:26:10.593674+0000 mon.c (mon.2) 3181 : audit [DBG] from='client.? 172.21.15.6:0/2040449452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:26:11.302Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:26:11.303Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:11.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:11 smithi130 bash[51302]: debug 2023-08-28T02:26:11.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:11.615924+0000) 2023-08-28T02:26:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:12 smithi130 bash[27579]: cluster 2023-08-28T02:26:11.616938+0000 mgr.y (mgr.14901) 8027 : cluster [DBG] pgmap v8034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:12.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:12 smithi130 bash[51302]: debug 2023-08-28T02:26:12.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:12.616167+0000) 2023-08-28T02:26:12.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:12 smithi006 bash[27292]: cluster 2023-08-28T02:26:11.616938+0000 mgr.y (mgr.14901) 8027 : cluster [DBG] pgmap v8034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:26:12.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:12.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:26:12.863Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:12.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:12 smithi006 bash[33918]: cluster 2023-08-28T02:26:11.616938+0000 mgr.y (mgr.14901) 8027 : cluster [DBG] pgmap v8034: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:13.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:13 smithi130 bash[51302]: debug 2023-08-28T02:26:13.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:13.616511+0000) 2023-08-28T02:26:14.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:14 smithi130 bash[27579]: audit 2023-08-28T02:26:13.012368+0000 mon.c (mon.2) 3182 : audit [DBG] from='client.? 172.21.15.6:0/1081158233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:14.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:14 smithi130 bash[27579]: cluster 2023-08-28T02:26:13.617537+0000 mgr.y (mgr.14901) 8028 : cluster [DBG] pgmap v8035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:14.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:14 smithi006 bash[27292]: audit 2023-08-28T02:26:13.012368+0000 mon.c (mon.2) 3182 : audit [DBG] from='client.? 172.21.15.6:0/1081158233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:14 smithi006 bash[27292]: cluster 2023-08-28T02:26:13.617537+0000 mgr.y (mgr.14901) 8028 : cluster [DBG] pgmap v8035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:14 smithi006 bash[33918]: audit 2023-08-28T02:26:13.012368+0000 mon.c (mon.2) 3182 : audit [DBG] from='client.? 172.21.15.6:0/1081158233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:14 smithi006 bash[33918]: cluster 2023-08-28T02:26:13.617537+0000 mgr.y (mgr.14901) 8028 : cluster [DBG] pgmap v8035: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:14 smithi130 bash[51302]: debug 2023-08-28T02:26:14.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:14.616837+0000) 2023-08-28T02:26:15.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:26:14.744Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:15.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:26:14.746Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:15.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:15 smithi130 bash[27579]: audit 2023-08-28T02:26:15.427564+0000 mon.a (mon.0) 5091 : audit [DBG] from='client.? 172.21.15.6:0/573740088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:15.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:15 smithi130 bash[51302]: debug 2023-08-28T02:26:15.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:15.617135+0000) 2023-08-28T02:26:15.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:15 smithi006 bash[27292]: audit 2023-08-28T02:26:15.427564+0000 mon.a (mon.0) 5091 : audit [DBG] from='client.? 172.21.15.6:0/573740088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:15.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:15 smithi006 bash[33918]: audit 2023-08-28T02:26:15.427564+0000 mon.a (mon.0) 5091 : audit [DBG] from='client.? 172.21.15.6:0/573740088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:16.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:16 smithi130 bash[27579]: cluster 2023-08-28T02:26:15.618668+0000 mgr.y (mgr.14901) 8029 : cluster [DBG] pgmap v8036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:16.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:16 smithi130 bash[51302]: debug 2023-08-28T02:26:16.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:16.617418+0000) 2023-08-28T02:26:16.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:16 smithi006 bash[27292]: cluster 2023-08-28T02:26:15.618668+0000 mgr.y (mgr.14901) 8029 : cluster [DBG] pgmap v8036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:16.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:16 smithi006 bash[33918]: cluster 2023-08-28T02:26:15.618668+0000 mgr.y (mgr.14901) 8029 : cluster [DBG] pgmap v8036: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:18.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:17 smithi130 bash[51302]: debug 2023-08-28T02:26:17.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:17.617704+0000) 2023-08-28T02:26:18.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:18 smithi006 bash[27292]: cluster 2023-08-28T02:26:17.619154+0000 mgr.y (mgr.14901) 8030 : cluster [DBG] pgmap v8037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:18.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:18 smithi006 bash[27292]: audit 2023-08-28T02:26:17.838723+0000 mon.c (mon.2) 3183 : audit [DBG] from='client.? 172.21.15.6:0/2303524320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:18.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:18 smithi006 bash[33918]: cluster 2023-08-28T02:26:17.619154+0000 mgr.y (mgr.14901) 8030 : cluster [DBG] pgmap v8037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:18.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:18 smithi006 bash[33918]: audit 2023-08-28T02:26:17.838723+0000 mon.c (mon.2) 3183 : audit [DBG] from='client.? 172.21.15.6:0/2303524320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:19.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:18 smithi130 bash[51302]: debug 2023-08-28T02:26:18.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:18.617936+0000) 2023-08-28T02:26:19.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:18 smithi130 bash[27579]: cluster 2023-08-28T02:26:17.619154+0000 mgr.y (mgr.14901) 8030 : cluster [DBG] pgmap v8037: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:19.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:18 smithi130 bash[27579]: audit 2023-08-28T02:26:17.838723+0000 mon.c (mon.2) 3183 : audit [DBG] from='client.? 172.21.15.6:0/2303524320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:19 smithi130 bash[51302]: debug 2023-08-28T02:26:19.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:19.618221+0000) 2023-08-28T02:26:20.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:26:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:26:20] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:26:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:20 smithi006 bash[27292]: cluster 2023-08-28T02:26:19.620331+0000 mgr.y (mgr.14901) 8031 : cluster [DBG] pgmap v8038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:20 smithi006 bash[27292]: audit 2023-08-28T02:26:20.252317+0000 mon.a (mon.0) 5092 : audit [DBG] from='client.? 172.21.15.6:0/999851896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:20.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:20 smithi006 bash[33918]: cluster 2023-08-28T02:26:19.620331+0000 mgr.y (mgr.14901) 8031 : cluster [DBG] pgmap v8038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:20.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:20 smithi006 bash[33918]: audit 2023-08-28T02:26:20.252317+0000 mon.a (mon.0) 5092 : audit [DBG] from='client.? 172.21.15.6:0/999851896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:21.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:20 smithi130 bash[51302]: debug 2023-08-28T02:26:20.613+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:20.618495+0000) 2023-08-28T02:26:21.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:20 smithi130 bash[27579]: cluster 2023-08-28T02:26:19.620331+0000 mgr.y (mgr.14901) 8031 : cluster [DBG] pgmap v8038: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:21.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:20 smithi130 bash[27579]: audit 2023-08-28T02:26:20.252317+0000 mon.a (mon.0) 5092 : audit [DBG] from='client.? 172.21.15.6:0/999851896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:26:21.302Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:26:21.304Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:22.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:21 smithi130 bash[51302]: debug 2023-08-28T02:26:21.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:21.618786+0000) 2023-08-28T02:26:22.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:22 smithi006 bash[27292]: cluster 2023-08-28T02:26:21.621193+0000 mgr.y (mgr.14901) 8032 : cluster [DBG] pgmap v8039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:22.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:22 smithi006 bash[27292]: audit 2023-08-28T02:26:22.667582+0000 mon.c (mon.2) 3184 : audit [DBG] from='client.? 172.21.15.6:0/2919810828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:26:22.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:26:22.864Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:22.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:22 smithi006 bash[33918]: cluster 2023-08-28T02:26:21.621193+0000 mgr.y (mgr.14901) 8032 : cluster [DBG] pgmap v8039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:22.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:22 smithi006 bash[33918]: audit 2023-08-28T02:26:22.667582+0000 mon.c (mon.2) 3184 : audit [DBG] from='client.? 172.21.15.6:0/2919810828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:23.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:22 smithi130 bash[27579]: cluster 2023-08-28T02:26:21.621193+0000 mgr.y (mgr.14901) 8032 : cluster [DBG] pgmap v8039: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:23.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:22 smithi130 bash[27579]: audit 2023-08-28T02:26:22.667582+0000 mon.c (mon.2) 3184 : audit [DBG] from='client.? 172.21.15.6:0/2919810828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:23.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:22 smithi130 bash[51302]: debug 2023-08-28T02:26:22.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:22.619031+0000) 2023-08-28T02:26:24.014 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:23 smithi130 bash[51302]: debug 2023-08-28T02:26:23.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:23.619325+0000) 2023-08-28T02:26:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:24 smithi130 bash[27579]: cluster 2023-08-28T02:26:23.622037+0000 mgr.y (mgr.14901) 8033 : cluster [DBG] pgmap v8040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:24.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:24 smithi006 bash[33918]: cluster 2023-08-28T02:26:23.622037+0000 mgr.y (mgr.14901) 8033 : cluster [DBG] pgmap v8040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:24 smithi006 bash[27292]: cluster 2023-08-28T02:26:23.622037+0000 mgr.y (mgr.14901) 8033 : cluster [DBG] pgmap v8040: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:25.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:24 smithi130 bash[51302]: debug 2023-08-28T02:26:24.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:24.619582+0000) 2023-08-28T02:26:25.133 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:26:24.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:25.133 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:26:24.748Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:25.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:25 smithi006 bash[27292]: audit 2023-08-28T02:26:25.079814+0000 mon.a (mon.0) 5093 : audit [DBG] from='client.? 172.21.15.6:0/4021363204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:25.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:25 smithi006 bash[33918]: audit 2023-08-28T02:26:25.079814+0000 mon.a (mon.0) 5093 : audit [DBG] from='client.? 172.21.15.6:0/4021363204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:25.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:25 smithi130 bash[27579]: audit 2023-08-28T02:26:25.079814+0000 mon.a (mon.0) 5093 : audit [DBG] from='client.? 172.21.15.6:0/4021363204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:26.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:25 smithi130 bash[51302]: debug 2023-08-28T02:26:25.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:25.619864+0000) 2023-08-28T02:26:26.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:26 smithi006 bash[27292]: cluster 2023-08-28T02:26:25.623193+0000 mgr.y (mgr.14901) 8034 : cluster [DBG] pgmap v8041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:26.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:26 smithi006 bash[33918]: cluster 2023-08-28T02:26:25.623193+0000 mgr.y (mgr.14901) 8034 : cluster [DBG] pgmap v8041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:26 smithi130 bash[27579]: cluster 2023-08-28T02:26:25.623193+0000 mgr.y (mgr.14901) 8034 : cluster [DBG] pgmap v8041: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:27.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:26 smithi130 bash[51302]: debug 2023-08-28T02:26:26.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:26.620085+0000) 2023-08-28T02:26:27.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:27 smithi130 bash[27579]: audit 2023-08-28T02:26:27.497923+0000 mon.a (mon.0) 5094 : audit [DBG] from='client.? 172.21.15.6:0/833203508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:27.867 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:27 smithi130 bash[51302]: debug 2023-08-28T02:26:27.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:27.620347+0000) 2023-08-28T02:26:27.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:27 smithi006 bash[27292]: audit 2023-08-28T02:26:27.497923+0000 mon.a (mon.0) 5094 : audit [DBG] from='client.? 172.21.15.6:0/833203508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:27.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:27 smithi006 bash[33918]: audit 2023-08-28T02:26:27.497923+0000 mon.a (mon.0) 5094 : audit [DBG] from='client.? 172.21.15.6:0/833203508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:28.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:28 smithi130 bash[51302]: debug 2023-08-28T02:26:28.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:28.620645+0000) 2023-08-28T02:26:28.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:28 smithi130 bash[27579]: cluster 2023-08-28T02:26:27.623975+0000 mgr.y (mgr.14901) 8035 : cluster [DBG] pgmap v8042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:28.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:28 smithi006 bash[27292]: cluster 2023-08-28T02:26:27.623975+0000 mgr.y (mgr.14901) 8035 : cluster [DBG] pgmap v8042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:28.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:28 smithi006 bash[33918]: cluster 2023-08-28T02:26:27.623975+0000 mgr.y (mgr.14901) 8035 : cluster [DBG] pgmap v8042: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:30.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:29 smithi130 bash[51302]: debug 2023-08-28T02:26:29.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:29.620964+0000) 2023-08-28T02:26:30.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:26:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:26:30] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T02:26:30.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:30 smithi006 bash[27292]: cluster 2023-08-28T02:26:29.625267+0000 mgr.y (mgr.14901) 8036 : cluster [DBG] pgmap v8043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:30.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:30 smithi006 bash[27292]: audit 2023-08-28T02:26:29.917297+0000 mon.a (mon.0) 5095 : audit [DBG] from='client.? 172.21.15.6:0/1155326626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:30.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:30 smithi006 bash[33918]: cluster 2023-08-28T02:26:29.625267+0000 mgr.y (mgr.14901) 8036 : cluster [DBG] pgmap v8043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:30.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:30 smithi006 bash[33918]: audit 2023-08-28T02:26:29.917297+0000 mon.a (mon.0) 5095 : audit [DBG] from='client.? 172.21.15.6:0/1155326626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:31.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:30 smithi130 bash[27579]: cluster 2023-08-28T02:26:29.625267+0000 mgr.y (mgr.14901) 8036 : cluster [DBG] pgmap v8043: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:31.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:30 smithi130 bash[27579]: audit 2023-08-28T02:26:29.917297+0000 mon.a (mon.0) 5095 : audit [DBG] from='client.? 172.21.15.6:0/1155326626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:31.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:30 smithi130 bash[51302]: debug 2023-08-28T02:26:30.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:30.621280+0000) 2023-08-28T02:26:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:26:31.303Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:26:31.305Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:32.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:31 smithi130 bash[51302]: debug 2023-08-28T02:26:31.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:31.621601+0000) 2023-08-28T02:26:32.964 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:26:32.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:32.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:26:32.864Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:32.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:32 smithi006 bash[27292]: cluster 2023-08-28T02:26:31.625959+0000 mgr.y (mgr.14901) 8037 : cluster [DBG] pgmap v8044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:32.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:32 smithi006 bash[27292]: audit 2023-08-28T02:26:32.334506+0000 mon.a (mon.0) 5096 : audit [DBG] from='client.? 172.21.15.6:0/2938273528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:32.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:32 smithi006 bash[33918]: cluster 2023-08-28T02:26:31.625959+0000 mgr.y (mgr.14901) 8037 : cluster [DBG] pgmap v8044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:32.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:32 smithi006 bash[33918]: audit 2023-08-28T02:26:32.334506+0000 mon.a (mon.0) 5096 : audit [DBG] from='client.? 172.21.15.6:0/2938273528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:33.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:32 smithi130 bash[27579]: cluster 2023-08-28T02:26:31.625959+0000 mgr.y (mgr.14901) 8037 : cluster [DBG] pgmap v8044: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:33.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:32 smithi130 bash[27579]: audit 2023-08-28T02:26:32.334506+0000 mon.a (mon.0) 5096 : audit [DBG] from='client.? 172.21.15.6:0/2938273528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:33.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:32 smithi130 bash[51302]: debug 2023-08-28T02:26:32.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:32.621884+0000) 2023-08-28T02:26:34.015 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:33 smithi130 bash[51302]: debug 2023-08-28T02:26:33.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:33.622171+0000) 2023-08-28T02:26:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:34 smithi130 bash[27579]: cluster 2023-08-28T02:26:33.626633+0000 mgr.y (mgr.14901) 8038 : cluster [DBG] pgmap v8045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:34.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:34 smithi006 bash[27292]: cluster 2023-08-28T02:26:33.626633+0000 mgr.y (mgr.14901) 8038 : cluster [DBG] pgmap v8045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:34 smithi006 bash[33918]: cluster 2023-08-28T02:26:33.626633+0000 mgr.y (mgr.14901) 8038 : cluster [DBG] pgmap v8045: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:35.018 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:26:34.746Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:35.018 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:26:34.747Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:35.018 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:34 smithi130 bash[51302]: debug 2023-08-28T02:26:34.617+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:34.622442+0000) 2023-08-28T02:26:35.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:35 smithi130 bash[27579]: audit 2023-08-28T02:26:34.753555+0000 mon.a (mon.0) 5097 : audit [DBG] from='client.? 172.21.15.6:0/3782117095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:35.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:35 smithi006 bash[27292]: audit 2023-08-28T02:26:34.753555+0000 mon.a (mon.0) 5097 : audit [DBG] from='client.? 172.21.15.6:0/3782117095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:35.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:35 smithi006 bash[33918]: audit 2023-08-28T02:26:34.753555+0000 mon.a (mon.0) 5097 : audit [DBG] from='client.? 172.21.15.6:0/3782117095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:36.027 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:35 smithi130 bash[51302]: debug 2023-08-28T02:26:35.621+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:35.622730+0000) 2023-08-28T02:26:36.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:36 smithi130 bash[27579]: cluster 2023-08-28T02:26:35.627754+0000 mgr.y (mgr.14901) 8039 : cluster [DBG] pgmap v8046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:36.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:36 smithi006 bash[27292]: cluster 2023-08-28T02:26:35.627754+0000 mgr.y (mgr.14901) 8039 : cluster [DBG] pgmap v8046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:36.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:36 smithi006 bash[33918]: cluster 2023-08-28T02:26:35.627754+0000 mgr.y (mgr.14901) 8039 : cluster [DBG] pgmap v8046: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:37.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:36 smithi130 bash[51302]: debug 2023-08-28T02:26:36.621+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:36.623001+0000) 2023-08-28T02:26:37.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:37 smithi130 bash[27579]: audit 2023-08-28T02:26:37.173804+0000 mon.c (mon.2) 3185 : audit [DBG] from='client.? 172.21.15.6:0/1487974041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:37.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:37 smithi006 bash[27292]: audit 2023-08-28T02:26:37.173804+0000 mon.c (mon.2) 3185 : audit [DBG] from='client.? 172.21.15.6:0/1487974041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:37.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:37 smithi006 bash[33918]: audit 2023-08-28T02:26:37.173804+0000 mon.c (mon.2) 3185 : audit [DBG] from='client.? 172.21.15.6:0/1487974041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:37 smithi130 bash[51302]: debug 2023-08-28T02:26:37.621+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:37.623204+0000) 2023-08-28T02:26:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:38 smithi130 bash[27579]: cluster 2023-08-28T02:26:37.628411+0000 mgr.y (mgr.14901) 8040 : cluster [DBG] pgmap v8047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:38.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:38 smithi006 bash[27292]: cluster 2023-08-28T02:26:37.628411+0000 mgr.y (mgr.14901) 8040 : cluster [DBG] pgmap v8047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:38.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:38 smithi006 bash[33918]: cluster 2023-08-28T02:26:37.628411+0000 mgr.y (mgr.14901) 8040 : cluster [DBG] pgmap v8047: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:39.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:38 smithi130 bash[51302]: debug 2023-08-28T02:26:38.621+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:38.623430+0000) 2023-08-28T02:26:39.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:39 smithi006 bash[27292]: audit 2023-08-28T02:26:39.589135+0000 mon.a (mon.0) 5098 : audit [DBG] from='client.? 172.21.15.6:0/2672745685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:39.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:39 smithi006 bash[33918]: audit 2023-08-28T02:26:39.589135+0000 mon.a (mon.0) 5098 : audit [DBG] from='client.? 172.21.15.6:0/2672745685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:40.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:39 smithi130 bash[27579]: audit 2023-08-28T02:26:39.589135+0000 mon.a (mon.0) 5098 : audit [DBG] from='client.? 172.21.15.6:0/2672745685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:40.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:39 smithi130 bash[51302]: debug 2023-08-28T02:26:39.621+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:39.623746+0000) 2023-08-28T02:26:40.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:26:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:26:40] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T02:26:40.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:40 smithi006 bash[33918]: cluster 2023-08-28T02:26:39.629496+0000 mgr.y (mgr.14901) 8041 : cluster [DBG] pgmap v8048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:40.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:40 smithi006 bash[27292]: cluster 2023-08-28T02:26:39.629496+0000 mgr.y (mgr.14901) 8041 : cluster [DBG] pgmap v8048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:41.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:40 smithi130 bash[27579]: cluster 2023-08-28T02:26:39.629496+0000 mgr.y (mgr.14901) 8041 : cluster [DBG] pgmap v8048: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:41.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:40 smithi130 bash[51302]: debug 2023-08-28T02:26:40.621+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:40.624008+0000) 2023-08-28T02:26:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:26:41.304Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:26:41.305Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:41 smithi130 bash[51302]: debug 2023-08-28T02:26:41.621+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:41.624324+0000) 2023-08-28T02:26:42.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:42 smithi006 bash[27292]: cluster 2023-08-28T02:26:41.630004+0000 mgr.y (mgr.14901) 8042 : cluster [DBG] pgmap v8049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:42.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:42 smithi006 bash[27292]: audit 2023-08-28T02:26:42.014548+0000 mon.a (mon.0) 5099 : audit [DBG] from='client.? 172.21.15.6:0/3138507433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:42.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:42 smithi006 bash[33918]: cluster 2023-08-28T02:26:41.630004+0000 mgr.y (mgr.14901) 8042 : cluster [DBG] pgmap v8049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:42.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:42 smithi006 bash[33918]: audit 2023-08-28T02:26:42.014548+0000 mon.a (mon.0) 5099 : audit [DBG] from='client.? 172.21.15.6:0/3138507433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:26:42.863Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:26:42.865Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:43.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:42 smithi130 bash[27579]: cluster 2023-08-28T02:26:41.630004+0000 mgr.y (mgr.14901) 8042 : cluster [DBG] pgmap v8049: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:43.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:42 smithi130 bash[27579]: audit 2023-08-28T02:26:42.014548+0000 mon.a (mon.0) 5099 : audit [DBG] from='client.? 172.21.15.6:0/3138507433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:43.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:42 smithi130 bash[51302]: debug 2023-08-28T02:26:42.621+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:42.624611+0000) 2023-08-28T02:26:44.017 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:43 smithi130 bash[51302]: debug 2023-08-28T02:26:43.621+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:43.624911+0000) 2023-08-28T02:26:44.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:44 smithi130 bash[27579]: cluster 2023-08-28T02:26:43.630646+0000 mgr.y (mgr.14901) 8043 : cluster [DBG] pgmap v8050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:44.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:44 smithi006 bash[33918]: cluster 2023-08-28T02:26:43.630646+0000 mgr.y (mgr.14901) 8043 : cluster [DBG] pgmap v8050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:44.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:44 smithi006 bash[27292]: cluster 2023-08-28T02:26:43.630646+0000 mgr.y (mgr.14901) 8043 : cluster [DBG] pgmap v8050: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:45.021 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:26:44.746Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:45.022 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:26:44.748Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:45.022 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:44 smithi130 bash[51302]: debug 2023-08-28T02:26:44.621+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:44.625204+0000) 2023-08-28T02:26:45.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:45 smithi130 bash[27579]: audit 2023-08-28T02:26:44.426442+0000 mon.c (mon.2) 3186 : audit [DBG] from='client.? 172.21.15.6:0/2020455541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:45.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:45 smithi006 bash[33918]: audit 2023-08-28T02:26:44.426442+0000 mon.c (mon.2) 3186 : audit [DBG] from='client.? 172.21.15.6:0/2020455541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:45.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:45 smithi006 bash[27292]: audit 2023-08-28T02:26:44.426442+0000 mon.c (mon.2) 3186 : audit [DBG] from='client.? 172.21.15.6:0/2020455541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:46.031 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:45 smithi130 bash[51302]: debug 2023-08-28T02:26:45.621+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:45.625481+0000) 2023-08-28T02:26:46.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:46 smithi130 bash[27579]: cluster 2023-08-28T02:26:45.631761+0000 mgr.y (mgr.14901) 8044 : cluster [DBG] pgmap v8051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:46.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:46 smithi006 bash[27292]: cluster 2023-08-28T02:26:45.631761+0000 mgr.y (mgr.14901) 8044 : cluster [DBG] pgmap v8051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:46.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:46 smithi006 bash[33918]: cluster 2023-08-28T02:26:45.631761+0000 mgr.y (mgr.14901) 8044 : cluster [DBG] pgmap v8051: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:47.034 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:46 smithi130 bash[51302]: debug 2023-08-28T02:26:46.621+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:46.625716+0000) 2023-08-28T02:26:47.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:47 smithi130 bash[27579]: audit 2023-08-28T02:26:46.856233+0000 mon.c (mon.2) 3187 : audit [DBG] from='client.? 172.21.15.6:0/3700522018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:47.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:47 smithi006 bash[33918]: audit 2023-08-28T02:26:46.856233+0000 mon.c (mon.2) 3187 : audit [DBG] from='client.? 172.21.15.6:0/3700522018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:47.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:47 smithi006 bash[27292]: audit 2023-08-28T02:26:46.856233+0000 mon.c (mon.2) 3187 : audit [DBG] from='client.? 172.21.15.6:0/3700522018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:48.041 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:47 smithi130 bash[51302]: debug 2023-08-28T02:26:47.621+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:47.626004+0000) 2023-08-28T02:26:48.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:48 smithi130 bash[27579]: cluster 2023-08-28T02:26:47.632380+0000 mgr.y (mgr.14901) 8045 : cluster [DBG] pgmap v8052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:48.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:48 smithi006 bash[27292]: cluster 2023-08-28T02:26:47.632380+0000 mgr.y (mgr.14901) 8045 : cluster [DBG] pgmap v8052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:48.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:48 smithi006 bash[33918]: cluster 2023-08-28T02:26:47.632380+0000 mgr.y (mgr.14901) 8045 : cluster [DBG] pgmap v8052: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:49.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:48 smithi130 bash[51302]: debug 2023-08-28T02:26:48.621+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:48.626302+0000) 2023-08-28T02:26:49.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:49 smithi130 bash[27579]: audit 2023-08-28T02:26:49.264211+0000 mon.c (mon.2) 3188 : audit [DBG] from='client.? 172.21.15.6:0/3225030714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:49.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:49 smithi006 bash[27292]: audit 2023-08-28T02:26:49.264211+0000 mon.c (mon.2) 3188 : audit [DBG] from='client.? 172.21.15.6:0/3225030714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:49.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:49 smithi006 bash[33918]: audit 2023-08-28T02:26:49.264211+0000 mon.c (mon.2) 3188 : audit [DBG] from='client.? 172.21.15.6:0/3225030714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:50.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:49 smithi130 bash[51302]: debug 2023-08-28T02:26:49.621+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:49.626595+0000) 2023-08-28T02:26:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:50 smithi130 bash[27579]: cluster 2023-08-28T02:26:49.633552+0000 mgr.y (mgr.14901) 8046 : cluster [DBG] pgmap v8053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:50.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:50 smithi006 bash[33918]: cluster 2023-08-28T02:26:49.633552+0000 mgr.y (mgr.14901) 8046 : cluster [DBG] pgmap v8053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:50 smithi006 bash[27292]: cluster 2023-08-28T02:26:49.633552+0000 mgr.y (mgr.14901) 8046 : cluster [DBG] pgmap v8053: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:51.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:50 smithi130 bash[51302]: debug 2023-08-28T02:26:50.625+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:50.626905+0000) 2023-08-28T02:26:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:26:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:26:50] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:26:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:26:51.305Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:26:51.307Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:52.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:51 smithi130 bash[27579]: audit 2023-08-28T02:26:51.677897+0000 mon.a (mon.0) 5100 : audit [DBG] from='client.? 172.21.15.6:0/2450520124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:52.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:51 smithi130 bash[51302]: debug 2023-08-28T02:26:51.625+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:51.627132+0000) 2023-08-28T02:26:52.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:51 smithi006 bash[27292]: audit 2023-08-28T02:26:51.677897+0000 mon.a (mon.0) 5100 : audit [DBG] from='client.? 172.21.15.6:0/2450520124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:52.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:51 smithi006 bash[33918]: audit 2023-08-28T02:26:51.677897+0000 mon.a (mon.0) 5100 : audit [DBG] from='client.? 172.21.15.6:0/2450520124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:53.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:52 smithi130 bash[27579]: cluster 2023-08-28T02:26:51.634121+0000 mgr.y (mgr.14901) 8047 : cluster [DBG] pgmap v8054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:53.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:52 smithi130 bash[51302]: debug 2023-08-28T02:26:52.625+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:52.627392+0000) 2023-08-28T02:26:53.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:52 smithi006 bash[27292]: cluster 2023-08-28T02:26:51.634121+0000 mgr.y (mgr.14901) 8047 : cluster [DBG] pgmap v8054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:53.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:52 smithi006 bash[33918]: cluster 2023-08-28T02:26:51.634121+0000 mgr.y (mgr.14901) 8047 : cluster [DBG] pgmap v8054: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:26:52.864Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:26:52.865Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:54.018 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:53 smithi130 bash[51302]: debug 2023-08-28T02:26:53.625+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:53.627690+0000) 2023-08-28T02:26:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:54 smithi130 bash[27579]: cluster 2023-08-28T02:26:53.634771+0000 mgr.y (mgr.14901) 8048 : cluster [DBG] pgmap v8055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:54 smithi130 bash[27579]: audit 2023-08-28T02:26:53.845329+0000 mon.b (mon.1) 879 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:26:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:54 smithi006 bash[33918]: cluster 2023-08-28T02:26:53.634771+0000 mgr.y (mgr.14901) 8048 : cluster [DBG] pgmap v8055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:54 smithi006 bash[33918]: audit 2023-08-28T02:26:53.845329+0000 mon.b (mon.1) 879 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:26:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:54 smithi006 bash[27292]: cluster 2023-08-28T02:26:53.634771+0000 mgr.y (mgr.14901) 8048 : cluster [DBG] pgmap v8055: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:54 smithi006 bash[27292]: audit 2023-08-28T02:26:53.845329+0000 mon.b (mon.1) 879 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:26:55.021 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:26:54.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:55.021 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:26:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:26:54.749Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:26:55.022 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:54 smithi130 bash[51302]: debug 2023-08-28T02:26:54.625+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:54.627879+0000) 2023-08-28T02:26:55.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:55 smithi130 bash[27579]: audit 2023-08-28T02:26:54.103474+0000 mon.a (mon.0) 5101 : audit [DBG] from='client.? 172.21.15.6:0/2916706767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:55.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:55 smithi006 bash[27292]: audit 2023-08-28T02:26:54.103474+0000 mon.a (mon.0) 5101 : audit [DBG] from='client.? 172.21.15.6:0/2916706767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:55.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:55 smithi006 bash[33918]: audit 2023-08-28T02:26:54.103474+0000 mon.a (mon.0) 5101 : audit [DBG] from='client.? 172.21.15.6:0/2916706767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:56.031 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:55 smithi130 bash[51302]: debug 2023-08-28T02:26:55.625+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:55.628151+0000) 2023-08-28T02:26:56.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:56 smithi130 bash[27579]: cluster 2023-08-28T02:26:55.635994+0000 mgr.y (mgr.14901) 8049 : cluster [DBG] pgmap v8056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:56.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:56 smithi006 bash[27292]: cluster 2023-08-28T02:26:55.635994+0000 mgr.y (mgr.14901) 8049 : cluster [DBG] pgmap v8056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:56.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:56 smithi006 bash[33918]: cluster 2023-08-28T02:26:55.635994+0000 mgr.y (mgr.14901) 8049 : cluster [DBG] pgmap v8056: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:57.034 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:56 smithi130 bash[51302]: debug 2023-08-28T02:26:56.625+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:56.628382+0000) 2023-08-28T02:26:57.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:57 smithi130 bash[27579]: audit 2023-08-28T02:26:56.527733+0000 mon.c (mon.2) 3189 : audit [DBG] from='client.? 172.21.15.6:0/40282689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:57.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:57 smithi006 bash[27292]: audit 2023-08-28T02:26:56.527733+0000 mon.c (mon.2) 3189 : audit [DBG] from='client.? 172.21.15.6:0/40282689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:57.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:57 smithi006 bash[33918]: audit 2023-08-28T02:26:56.527733+0000 mon.c (mon.2) 3189 : audit [DBG] from='client.? 172.21.15.6:0/40282689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:58.039 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:57 smithi130 bash[51302]: debug 2023-08-28T02:26:57.625+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:57.628678+0000) 2023-08-28T02:26:58.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:58 smithi130 bash[27579]: cluster 2023-08-28T02:26:57.636823+0000 mgr.y (mgr.14901) 8050 : cluster [DBG] pgmap v8057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:58.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:58 smithi006 bash[27292]: cluster 2023-08-28T02:26:57.636823+0000 mgr.y (mgr.14901) 8050 : cluster [DBG] pgmap v8057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:58.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:58 smithi006 bash[33918]: cluster 2023-08-28T02:26:57.636823+0000 mgr.y (mgr.14901) 8050 : cluster [DBG] pgmap v8057: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:26:59.041 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:58 smithi130 bash[51302]: debug 2023-08-28T02:26:58.625+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:58.628957+0000) 2023-08-28T02:26:59.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:26:59 smithi130 bash[27579]: audit 2023-08-28T02:26:58.951490+0000 mon.c (mon.2) 3190 : audit [DBG] from='client.? 172.21.15.6:0/1024014452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:59.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:26:59 smithi006 bash[33918]: audit 2023-08-28T02:26:58.951490+0000 mon.c (mon.2) 3190 : audit [DBG] from='client.? 172.21.15.6:0/1024014452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:59.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:26:59 smithi006 bash[27292]: audit 2023-08-28T02:26:58.951490+0000 mon.c (mon.2) 3190 : audit [DBG] from='client.? 172.21.15.6:0/1024014452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:26:59 smithi130 bash[51302]: debug 2023-08-28T02:26:59.625+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:59.629170+0000) 2023-08-28T02:27:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:00 smithi130 bash[51302]: debug 2023-08-28T02:27:00.629+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:00.629423+0000) 2023-08-28T02:27:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:00 smithi130 bash[27579]: cluster 2023-08-28T02:26:59.637811+0000 mgr.y (mgr.14901) 8051 : cluster [DBG] pgmap v8058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:00 smithi130 bash[27579]: audit 2023-08-28T02:26:59.728967+0000 mon.a (mon.0) 5102 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:27:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:00 smithi130 bash[27579]: audit 2023-08-28T02:27:00.134876+0000 mon.a (mon.0) 5103 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:27:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:00 smithi130 bash[27579]: audit 2023-08-28T02:27:00.428157+0000 mon.b (mon.1) 880 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:27:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:00 smithi130 bash[27579]: audit 2023-08-28T02:27:00.430559+0000 mon.b (mon.1) 881 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:27:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:00 smithi130 bash[27579]: audit 2023-08-28T02:27:00.440055+0000 mon.a (mon.0) 5104 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:27:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:27:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:27:00] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T02:27:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:00 smithi006 bash[27292]: cluster 2023-08-28T02:26:59.637811+0000 mgr.y (mgr.14901) 8051 : cluster [DBG] pgmap v8058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:00 smithi006 bash[27292]: audit 2023-08-28T02:26:59.728967+0000 mon.a (mon.0) 5102 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:27:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:00 smithi006 bash[27292]: audit 2023-08-28T02:27:00.134876+0000 mon.a (mon.0) 5103 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:27:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:00 smithi006 bash[27292]: audit 2023-08-28T02:27:00.428157+0000 mon.b (mon.1) 880 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:27:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:00 smithi006 bash[27292]: audit 2023-08-28T02:27:00.430559+0000 mon.b (mon.1) 881 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:27:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:00 smithi006 bash[27292]: audit 2023-08-28T02:27:00.440055+0000 mon.a (mon.0) 5104 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:27:01.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:00 smithi006 bash[33918]: cluster 2023-08-28T02:26:59.637811+0000 mgr.y (mgr.14901) 8051 : cluster [DBG] pgmap v8058: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:01.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:00 smithi006 bash[33918]: audit 2023-08-28T02:26:59.728967+0000 mon.a (mon.0) 5102 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:27:01.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:00 smithi006 bash[33918]: audit 2023-08-28T02:27:00.134876+0000 mon.a (mon.0) 5103 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:27:01.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:00 smithi006 bash[33918]: audit 2023-08-28T02:27:00.428157+0000 mon.b (mon.1) 880 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:27:01.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:00 smithi006 bash[33918]: audit 2023-08-28T02:27:00.430559+0000 mon.b (mon.1) 881 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:27:01.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:00 smithi006 bash[33918]: audit 2023-08-28T02:27:00.440055+0000 mon.a (mon.0) 5104 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:27:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:27:01.305Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:27:01.307Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:02.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:01 smithi130 bash[27579]: audit 2023-08-28T02:27:01.363177+0000 mon.a (mon.0) 5105 : audit [DBG] from='client.? 172.21.15.6:0/1017884045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:02.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:01 smithi130 bash[51302]: debug 2023-08-28T02:27:01.625+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:01.629713+0000) 2023-08-28T02:27:02.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:01 smithi006 bash[33918]: audit 2023-08-28T02:27:01.363177+0000 mon.a (mon.0) 5105 : audit [DBG] from='client.? 172.21.15.6:0/1017884045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:02.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:01 smithi006 bash[27292]: audit 2023-08-28T02:27:01.363177+0000 mon.a (mon.0) 5105 : audit [DBG] from='client.? 172.21.15.6:0/1017884045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:03.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:02 smithi130 bash[27579]: cluster 2023-08-28T02:27:01.638418+0000 mgr.y (mgr.14901) 8052 : cluster [DBG] pgmap v8059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:03.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:02 smithi130 bash[51302]: debug 2023-08-28T02:27:02.624+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:02.629936+0000) 2023-08-28T02:27:03.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:02 smithi006 bash[33918]: cluster 2023-08-28T02:27:01.638418+0000 mgr.y (mgr.14901) 8052 : cluster [DBG] pgmap v8059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:03.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:02 smithi006 bash[27292]: cluster 2023-08-28T02:27:01.638418+0000 mgr.y (mgr.14901) 8052 : cluster [DBG] pgmap v8059: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:27:02.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:27:02.867Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:04.022 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:03 smithi130 bash[51302]: debug 2023-08-28T02:27:03.624+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:03.630195+0000) 2023-08-28T02:27:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:04 smithi130 bash[27579]: cluster 2023-08-28T02:27:03.639151+0000 mgr.y (mgr.14901) 8053 : cluster [DBG] pgmap v8060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:04 smithi130 bash[27579]: audit 2023-08-28T02:27:03.779703+0000 mon.c (mon.2) 3191 : audit [DBG] from='client.? 172.21.15.6:0/956518535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:04.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:04 smithi006 bash[33918]: cluster 2023-08-28T02:27:03.639151+0000 mgr.y (mgr.14901) 8053 : cluster [DBG] pgmap v8060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:04.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:04 smithi006 bash[33918]: audit 2023-08-28T02:27:03.779703+0000 mon.c (mon.2) 3191 : audit [DBG] from='client.? 172.21.15.6:0/956518535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:04 smithi006 bash[27292]: cluster 2023-08-28T02:27:03.639151+0000 mgr.y (mgr.14901) 8053 : cluster [DBG] pgmap v8060: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:04 smithi006 bash[27292]: audit 2023-08-28T02:27:03.779703+0000 mon.c (mon.2) 3191 : audit [DBG] from='client.? 172.21.15.6:0/956518535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:05.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:04 smithi130 bash[51302]: debug 2023-08-28T02:27:04.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:04.630465+0000) 2023-08-28T02:27:05.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:27:04.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:05.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:27:04.750Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:06.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:05 smithi130 bash[51302]: debug 2023-08-28T02:27:05.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:05.630740+0000) 2023-08-28T02:27:06.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:06 smithi006 bash[33918]: cluster 2023-08-28T02:27:05.640450+0000 mgr.y (mgr.14901) 8054 : cluster [DBG] pgmap v8061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:06.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:06 smithi006 bash[33918]: audit 2023-08-28T02:27:06.187319+0000 mon.c (mon.2) 3192 : audit [DBG] from='client.? 172.21.15.6:0/553165979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:06.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:06 smithi006 bash[27292]: cluster 2023-08-28T02:27:05.640450+0000 mgr.y (mgr.14901) 8054 : cluster [DBG] pgmap v8061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:06.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:06 smithi006 bash[27292]: audit 2023-08-28T02:27:06.187319+0000 mon.c (mon.2) 3192 : audit [DBG] from='client.? 172.21.15.6:0/553165979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:07.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:06 smithi130 bash[27579]: cluster 2023-08-28T02:27:05.640450+0000 mgr.y (mgr.14901) 8054 : cluster [DBG] pgmap v8061: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:07.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:06 smithi130 bash[27579]: audit 2023-08-28T02:27:06.187319+0000 mon.c (mon.2) 3192 : audit [DBG] from='client.? 172.21.15.6:0/553165979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:07.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:06 smithi130 bash[51302]: debug 2023-08-28T02:27:06.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:06.631020+0000) 2023-08-28T02:27:08.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:07 smithi130 bash[51302]: debug 2023-08-28T02:27:07.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:07.631280+0000) 2023-08-28T02:27:08.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:08 smithi006 bash[27292]: cluster 2023-08-28T02:27:07.641172+0000 mgr.y (mgr.14901) 8055 : cluster [DBG] pgmap v8062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:08.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:08 smithi006 bash[27292]: audit 2023-08-28T02:27:08.599641+0000 mon.c (mon.2) 3193 : audit [DBG] from='client.? 172.21.15.6:0/377355456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:08.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:08 smithi006 bash[33918]: cluster 2023-08-28T02:27:07.641172+0000 mgr.y (mgr.14901) 8055 : cluster [DBG] pgmap v8062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:08.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:08 smithi006 bash[33918]: audit 2023-08-28T02:27:08.599641+0000 mon.c (mon.2) 3193 : audit [DBG] from='client.? 172.21.15.6:0/377355456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:09.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:08 smithi130 bash[51302]: debug 2023-08-28T02:27:08.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:08.631544+0000) 2023-08-28T02:27:09.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:08 smithi130 bash[27579]: cluster 2023-08-28T02:27:07.641172+0000 mgr.y (mgr.14901) 8055 : cluster [DBG] pgmap v8062: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:09.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:08 smithi130 bash[27579]: audit 2023-08-28T02:27:08.599641+0000 mon.c (mon.2) 3193 : audit [DBG] from='client.? 172.21.15.6:0/377355456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:10.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:09 smithi130 bash[51302]: debug 2023-08-28T02:27:09.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:09.631853+0000) 2023-08-28T02:27:10.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:10 smithi006 bash[27292]: cluster 2023-08-28T02:27:09.642324+0000 mgr.y (mgr.14901) 8056 : cluster [DBG] pgmap v8063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:10.965 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:27:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:27:10] "GET /metrics HTTP/1.1" 200 193127 "" "Prometheus/2.33.4" 2023-08-28T02:27:10.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:10 smithi006 bash[33918]: cluster 2023-08-28T02:27:09.642324+0000 mgr.y (mgr.14901) 8056 : cluster [DBG] pgmap v8063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:11.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:10 smithi130 bash[27579]: cluster 2023-08-28T02:27:09.642324+0000 mgr.y (mgr.14901) 8056 : cluster [DBG] pgmap v8063: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:11.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:10 smithi130 bash[51302]: debug 2023-08-28T02:27:10.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:10.632066+0000) 2023-08-28T02:27:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:27:11.305Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:27:11.307Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:12.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:11 smithi130 bash[27579]: audit 2023-08-28T02:27:11.020257+0000 mon.a (mon.0) 5106 : audit [DBG] from='client.? 172.21.15.6:0/2452066252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:12.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:11 smithi130 bash[51302]: debug 2023-08-28T02:27:11.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:11.632353+0000) 2023-08-28T02:27:12.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:11 smithi006 bash[27292]: audit 2023-08-28T02:27:11.020257+0000 mon.a (mon.0) 5106 : audit [DBG] from='client.? 172.21.15.6:0/2452066252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:12.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:11 smithi006 bash[33918]: audit 2023-08-28T02:27:11.020257+0000 mon.a (mon.0) 5106 : audit [DBG] from='client.? 172.21.15.6:0/2452066252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:13.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:12 smithi130 bash[27579]: cluster 2023-08-28T02:27:11.643009+0000 mgr.y (mgr.14901) 8057 : cluster [DBG] pgmap v8064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:13.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:12 smithi130 bash[51302]: debug 2023-08-28T02:27:12.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:12.632662+0000) 2023-08-28T02:27:13.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:12 smithi006 bash[27292]: cluster 2023-08-28T02:27:11.643009+0000 mgr.y (mgr.14901) 8057 : cluster [DBG] pgmap v8064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:12 smithi006 bash[33918]: cluster 2023-08-28T02:27:11.643009+0000 mgr.y (mgr.14901) 8057 : cluster [DBG] pgmap v8064: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:27:12.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:13.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:27:12.867Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:14.025 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:13 smithi130 bash[51302]: debug 2023-08-28T02:27:13.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:13.632971+0000) 2023-08-28T02:27:14.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:14 smithi130 bash[27579]: audit 2023-08-28T02:27:13.439648+0000 mon.c (mon.2) 3194 : audit [DBG] from='client.? 172.21.15.6:0/4183919864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:14.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:14 smithi130 bash[27579]: cluster 2023-08-28T02:27:13.643688+0000 mgr.y (mgr.14901) 8058 : cluster [DBG] pgmap v8065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:14 smithi006 bash[27292]: audit 2023-08-28T02:27:13.439648+0000 mon.c (mon.2) 3194 : audit [DBG] from='client.? 172.21.15.6:0/4183919864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:14 smithi006 bash[27292]: cluster 2023-08-28T02:27:13.643688+0000 mgr.y (mgr.14901) 8058 : cluster [DBG] pgmap v8065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:14 smithi006 bash[33918]: audit 2023-08-28T02:27:13.439648+0000 mon.c (mon.2) 3194 : audit [DBG] from='client.? 172.21.15.6:0/4183919864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:14.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:14 smithi006 bash[33918]: cluster 2023-08-28T02:27:13.643688+0000 mgr.y (mgr.14901) 8058 : cluster [DBG] pgmap v8065: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:14 smithi130 bash[51302]: debug 2023-08-28T02:27:14.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:14.633241+0000) 2023-08-28T02:27:15.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:27:14.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:15.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:27:14.750Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:16.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:15 smithi130 bash[51302]: debug 2023-08-28T02:27:15.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:15.633532+0000) 2023-08-28T02:27:16.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:16 smithi006 bash[33918]: cluster 2023-08-28T02:27:15.644443+0000 mgr.y (mgr.14901) 8059 : cluster [DBG] pgmap v8066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:16.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:16 smithi006 bash[33918]: audit 2023-08-28T02:27:15.861271+0000 mon.a (mon.0) 5107 : audit [DBG] from='client.? 172.21.15.6:0/2757698906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:16.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:16 smithi006 bash[27292]: cluster 2023-08-28T02:27:15.644443+0000 mgr.y (mgr.14901) 8059 : cluster [DBG] pgmap v8066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:16.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:16 smithi006 bash[27292]: audit 2023-08-28T02:27:15.861271+0000 mon.a (mon.0) 5107 : audit [DBG] from='client.? 172.21.15.6:0/2757698906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:17.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:16 smithi130 bash[51302]: debug 2023-08-28T02:27:16.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:16.633801+0000) 2023-08-28T02:27:17.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:16 smithi130 bash[27579]: cluster 2023-08-28T02:27:15.644443+0000 mgr.y (mgr.14901) 8059 : cluster [DBG] pgmap v8066: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:17.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:16 smithi130 bash[27579]: audit 2023-08-28T02:27:15.861271+0000 mon.a (mon.0) 5107 : audit [DBG] from='client.? 172.21.15.6:0/2757698906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:18.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:17 smithi130 bash[51302]: debug 2023-08-28T02:27:17.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:17.634070+0000) 2023-08-28T02:27:18.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:18 smithi006 bash[27292]: cluster 2023-08-28T02:27:17.645087+0000 mgr.y (mgr.14901) 8060 : cluster [DBG] pgmap v8067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:18.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:18 smithi006 bash[27292]: audit 2023-08-28T02:27:18.277663+0000 mon.a (mon.0) 5108 : audit [DBG] from='client.? 172.21.15.6:0/945465237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:18.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:18 smithi006 bash[33918]: cluster 2023-08-28T02:27:17.645087+0000 mgr.y (mgr.14901) 8060 : cluster [DBG] pgmap v8067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:18.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:18 smithi006 bash[33918]: audit 2023-08-28T02:27:18.277663+0000 mon.a (mon.0) 5108 : audit [DBG] from='client.? 172.21.15.6:0/945465237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:19.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:18 smithi130 bash[51302]: debug 2023-08-28T02:27:18.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:18.634318+0000) 2023-08-28T02:27:19.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:18 smithi130 bash[27579]: cluster 2023-08-28T02:27:17.645087+0000 mgr.y (mgr.14901) 8060 : cluster [DBG] pgmap v8067: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:19.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:18 smithi130 bash[27579]: audit 2023-08-28T02:27:18.277663+0000 mon.a (mon.0) 5108 : audit [DBG] from='client.? 172.21.15.6:0/945465237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:19 smithi130 bash[51302]: debug 2023-08-28T02:27:19.628+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:19.634603+0000) 2023-08-28T02:27:20.965 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:27:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:27:20] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:27:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:20 smithi006 bash[27292]: cluster 2023-08-28T02:27:19.646113+0000 mgr.y (mgr.14901) 8061 : cluster [DBG] pgmap v8068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:20 smithi006 bash[27292]: audit 2023-08-28T02:27:20.688131+0000 mon.c (mon.2) 3195 : audit [DBG] from='client.? 172.21.15.6:0/1144423473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:20.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:20 smithi006 bash[33918]: cluster 2023-08-28T02:27:19.646113+0000 mgr.y (mgr.14901) 8061 : cluster [DBG] pgmap v8068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:20.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:20 smithi006 bash[33918]: audit 2023-08-28T02:27:20.688131+0000 mon.c (mon.2) 3195 : audit [DBG] from='client.? 172.21.15.6:0/1144423473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:21.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:20 smithi130 bash[51302]: debug 2023-08-28T02:27:20.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:20.634907+0000) 2023-08-28T02:27:21.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:20 smithi130 bash[27579]: cluster 2023-08-28T02:27:19.646113+0000 mgr.y (mgr.14901) 8061 : cluster [DBG] pgmap v8068: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:21.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:20 smithi130 bash[27579]: audit 2023-08-28T02:27:20.688131+0000 mon.c (mon.2) 3195 : audit [DBG] from='client.? 172.21.15.6:0/1144423473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:27:21.306Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:27:21.308Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:22.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:21 smithi130 bash[51302]: debug 2023-08-28T02:27:21.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:21.635181+0000) 2023-08-28T02:27:23.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:22 smithi130 bash[27579]: cluster 2023-08-28T02:27:21.646782+0000 mgr.y (mgr.14901) 8062 : cluster [DBG] pgmap v8069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:23.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:22 smithi130 bash[51302]: debug 2023-08-28T02:27:22.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:22.635495+0000) 2023-08-28T02:27:23.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:22 smithi006 bash[33918]: cluster 2023-08-28T02:27:21.646782+0000 mgr.y (mgr.14901) 8062 : cluster [DBG] pgmap v8069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:23.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:22 smithi006 bash[27292]: cluster 2023-08-28T02:27:21.646782+0000 mgr.y (mgr.14901) 8062 : cluster [DBG] pgmap v8069: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:27:22.866Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:27:22.868Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:24.030 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:23 smithi130 bash[51302]: debug 2023-08-28T02:27:23.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:23.635800+0000) 2023-08-28T02:27:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:24 smithi130 bash[27579]: audit 2023-08-28T02:27:23.103177+0000 mon.a (mon.0) 5109 : audit [DBG] from='client.? 172.21.15.6:0/1859523224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:24 smithi130 bash[27579]: cluster 2023-08-28T02:27:23.647477+0000 mgr.y (mgr.14901) 8063 : cluster [DBG] pgmap v8070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:24.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:24 smithi006 bash[27292]: audit 2023-08-28T02:27:23.103177+0000 mon.a (mon.0) 5109 : audit [DBG] from='client.? 172.21.15.6:0/1859523224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:24 smithi006 bash[27292]: cluster 2023-08-28T02:27:23.647477+0000 mgr.y (mgr.14901) 8063 : cluster [DBG] pgmap v8070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:24.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:24 smithi006 bash[33918]: audit 2023-08-28T02:27:23.103177+0000 mon.a (mon.0) 5109 : audit [DBG] from='client.? 172.21.15.6:0/1859523224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:24.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:24 smithi006 bash[33918]: cluster 2023-08-28T02:27:23.647477+0000 mgr.y (mgr.14901) 8063 : cluster [DBG] pgmap v8070: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:25.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:24 smithi130 bash[51302]: debug 2023-08-28T02:27:24.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:24.636114+0000) 2023-08-28T02:27:25.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:27:24.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:25.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:27:24.752Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:25.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:25 smithi130 bash[27579]: audit 2023-08-28T02:27:25.521677+0000 mon.c (mon.2) 3196 : audit [DBG] from='client.? 172.21.15.6:0/4058931455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:25.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:25 smithi130 bash[51302]: debug 2023-08-28T02:27:25.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:25.636379+0000) 2023-08-28T02:27:25.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:25 smithi006 bash[27292]: audit 2023-08-28T02:27:25.521677+0000 mon.c (mon.2) 3196 : audit [DBG] from='client.? 172.21.15.6:0/4058931455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:25.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:25 smithi006 bash[33918]: audit 2023-08-28T02:27:25.521677+0000 mon.c (mon.2) 3196 : audit [DBG] from='client.? 172.21.15.6:0/4058931455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:26.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:26 smithi130 bash[27579]: cluster 2023-08-28T02:27:25.648592+0000 mgr.y (mgr.14901) 8064 : cluster [DBG] pgmap v8071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:26.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:26 smithi130 bash[51302]: debug 2023-08-28T02:27:26.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:26.636655+0000) 2023-08-28T02:27:26.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:26 smithi006 bash[27292]: cluster 2023-08-28T02:27:25.648592+0000 mgr.y (mgr.14901) 8064 : cluster [DBG] pgmap v8071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:26.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:26 smithi006 bash[33918]: cluster 2023-08-28T02:27:25.648592+0000 mgr.y (mgr.14901) 8064 : cluster [DBG] pgmap v8071: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:28.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:27 smithi130 bash[51302]: debug 2023-08-28T02:27:27.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:27.636895+0000) 2023-08-28T02:27:28.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:28 smithi006 bash[33918]: cluster 2023-08-28T02:27:27.649385+0000 mgr.y (mgr.14901) 8065 : cluster [DBG] pgmap v8072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:28.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:28 smithi006 bash[33918]: audit 2023-08-28T02:27:27.936512+0000 mon.a (mon.0) 5110 : audit [DBG] from='client.? 172.21.15.6:0/940207324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:28.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:28 smithi006 bash[27292]: cluster 2023-08-28T02:27:27.649385+0000 mgr.y (mgr.14901) 8065 : cluster [DBG] pgmap v8072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:28.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:28 smithi006 bash[27292]: audit 2023-08-28T02:27:27.936512+0000 mon.a (mon.0) 5110 : audit [DBG] from='client.? 172.21.15.6:0/940207324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:28 smithi130 bash[51302]: debug 2023-08-28T02:27:28.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:28.637138+0000) 2023-08-28T02:27:29.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:28 smithi130 bash[27579]: cluster 2023-08-28T02:27:27.649385+0000 mgr.y (mgr.14901) 8065 : cluster [DBG] pgmap v8072: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:29.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:28 smithi130 bash[27579]: audit 2023-08-28T02:27:27.936512+0000 mon.a (mon.0) 5110 : audit [DBG] from='client.? 172.21.15.6:0/940207324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:30.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:29 smithi130 bash[51302]: debug 2023-08-28T02:27:29.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:29.637438+0000) 2023-08-28T02:27:30.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:27:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:27:30] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:27:30.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:30 smithi006 bash[27292]: cluster 2023-08-28T02:27:29.650519+0000 mgr.y (mgr.14901) 8066 : cluster [DBG] pgmap v8073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:30.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:30 smithi006 bash[27292]: audit 2023-08-28T02:27:30.356813+0000 mon.a (mon.0) 5111 : audit [DBG] from='client.? 172.21.15.6:0/4091326902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:30.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:30 smithi006 bash[33918]: cluster 2023-08-28T02:27:29.650519+0000 mgr.y (mgr.14901) 8066 : cluster [DBG] pgmap v8073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:30.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:30 smithi006 bash[33918]: audit 2023-08-28T02:27:30.356813+0000 mon.a (mon.0) 5111 : audit [DBG] from='client.? 172.21.15.6:0/4091326902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:31.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:30 smithi130 bash[27579]: cluster 2023-08-28T02:27:29.650519+0000 mgr.y (mgr.14901) 8066 : cluster [DBG] pgmap v8073: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:31.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:30 smithi130 bash[27579]: audit 2023-08-28T02:27:30.356813+0000 mon.a (mon.0) 5111 : audit [DBG] from='client.? 172.21.15.6:0/4091326902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:31.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:30 smithi130 bash[51302]: debug 2023-08-28T02:27:30.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:30.637723+0000) 2023-08-28T02:27:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:27:31.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:27:31.309Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:32.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:31 smithi130 bash[51302]: debug 2023-08-28T02:27:31.636+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:31.638011+0000) 2023-08-28T02:27:33.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:32 smithi130 bash[27579]: cluster 2023-08-28T02:27:31.651129+0000 mgr.y (mgr.14901) 8067 : cluster [DBG] pgmap v8074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:33.164 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:32 smithi130 bash[51302]: debug 2023-08-28T02:27:32.636+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:32.638279+0000) 2023-08-28T02:27:33.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:32 smithi006 bash[33918]: cluster 2023-08-28T02:27:31.651129+0000 mgr.y (mgr.14901) 8067 : cluster [DBG] pgmap v8074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:33.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:32 smithi006 bash[27292]: cluster 2023-08-28T02:27:31.651129+0000 mgr.y (mgr.14901) 8067 : cluster [DBG] pgmap v8074: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:33.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:27:32.867Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:33.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:27:32.869Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:34.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:33 smithi130 bash[27579]: audit 2023-08-28T02:27:32.776033+0000 mon.a (mon.0) 5112 : audit [DBG] from='client.? 172.21.15.6:0/1390079773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:34.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:33 smithi130 bash[51302]: debug 2023-08-28T02:27:33.632+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:33.638557+0000) 2023-08-28T02:27:34.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:33 smithi006 bash[33918]: audit 2023-08-28T02:27:32.776033+0000 mon.a (mon.0) 5112 : audit [DBG] from='client.? 172.21.15.6:0/1390079773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:33 smithi006 bash[27292]: audit 2023-08-28T02:27:32.776033+0000 mon.a (mon.0) 5112 : audit [DBG] from='client.? 172.21.15.6:0/1390079773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:35.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:34 smithi130 bash[27579]: cluster 2023-08-28T02:27:33.651800+0000 mgr.y (mgr.14901) 8068 : cluster [DBG] pgmap v8075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:35.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:34 smithi130 bash[51302]: debug 2023-08-28T02:27:34.636+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:34.638917+0000) 2023-08-28T02:27:35.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:34 smithi006 bash[27292]: cluster 2023-08-28T02:27:33.651800+0000 mgr.y (mgr.14901) 8068 : cluster [DBG] pgmap v8075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:35.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:34 smithi006 bash[33918]: cluster 2023-08-28T02:27:33.651800+0000 mgr.y (mgr.14901) 8068 : cluster [DBG] pgmap v8075: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:35.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:27:34.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:35.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:27:34.753Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:36.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:35 smithi130 bash[27579]: audit 2023-08-28T02:27:35.201020+0000 mon.a (mon.0) 5113 : audit [DBG] from='client.? 172.21.15.6:0/2937794684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:36.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:35 smithi130 bash[51302]: debug 2023-08-28T02:27:35.636+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:35.639140+0000) 2023-08-28T02:27:36.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:35 smithi006 bash[27292]: audit 2023-08-28T02:27:35.201020+0000 mon.a (mon.0) 5113 : audit [DBG] from='client.? 172.21.15.6:0/2937794684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:36.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:35 smithi006 bash[33918]: audit 2023-08-28T02:27:35.201020+0000 mon.a (mon.0) 5113 : audit [DBG] from='client.? 172.21.15.6:0/2937794684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:37.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:36 smithi130 bash[27579]: cluster 2023-08-28T02:27:35.652970+0000 mgr.y (mgr.14901) 8069 : cluster [DBG] pgmap v8076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:37.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:36 smithi130 bash[51302]: debug 2023-08-28T02:27:36.636+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:36.639418+0000) 2023-08-28T02:27:37.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:36 smithi006 bash[27292]: cluster 2023-08-28T02:27:35.652970+0000 mgr.y (mgr.14901) 8069 : cluster [DBG] pgmap v8076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:37.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:36 smithi006 bash[33918]: cluster 2023-08-28T02:27:35.652970+0000 mgr.y (mgr.14901) 8069 : cluster [DBG] pgmap v8076: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:38.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:37 smithi130 bash[27579]: audit 2023-08-28T02:27:37.613260+0000 mon.a (mon.0) 5114 : audit [DBG] from='client.? 172.21.15.6:0/4168156270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:37 smithi130 bash[51302]: debug 2023-08-28T02:27:37.636+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:37.639686+0000) 2023-08-28T02:27:38.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:37 smithi006 bash[27292]: audit 2023-08-28T02:27:37.613260+0000 mon.a (mon.0) 5114 : audit [DBG] from='client.? 172.21.15.6:0/4168156270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:38.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:37 smithi006 bash[33918]: audit 2023-08-28T02:27:37.613260+0000 mon.a (mon.0) 5114 : audit [DBG] from='client.? 172.21.15.6:0/4168156270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:39.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:38 smithi130 bash[27579]: cluster 2023-08-28T02:27:37.653673+0000 mgr.y (mgr.14901) 8070 : cluster [DBG] pgmap v8077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:39.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:38 smithi130 bash[51302]: debug 2023-08-28T02:27:38.636+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:38.640000+0000) 2023-08-28T02:27:39.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:38 smithi006 bash[33918]: cluster 2023-08-28T02:27:37.653673+0000 mgr.y (mgr.14901) 8070 : cluster [DBG] pgmap v8077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:39.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:38 smithi006 bash[27292]: cluster 2023-08-28T02:27:37.653673+0000 mgr.y (mgr.14901) 8070 : cluster [DBG] pgmap v8077: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:40.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:39 smithi130 bash[51302]: debug 2023-08-28T02:27:39.636+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:39.640274+0000) 2023-08-28T02:27:41.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:40 smithi130 bash[27579]: cluster 2023-08-28T02:27:39.654532+0000 mgr.y (mgr.14901) 8071 : cluster [DBG] pgmap v8078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:41.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:40 smithi130 bash[27579]: audit 2023-08-28T02:27:40.025522+0000 mon.c (mon.2) 3197 : audit [DBG] from='client.? 172.21.15.6:0/4129625132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:41.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:40 smithi130 bash[51302]: debug 2023-08-28T02:27:40.636+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:40.640570+0000) 2023-08-28T02:27:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:27:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:27:40] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:27:41.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:40 smithi006 bash[27292]: cluster 2023-08-28T02:27:39.654532+0000 mgr.y (mgr.14901) 8071 : cluster [DBG] pgmap v8078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:41.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:40 smithi006 bash[27292]: audit 2023-08-28T02:27:40.025522+0000 mon.c (mon.2) 3197 : audit [DBG] from='client.? 172.21.15.6:0/4129625132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:41.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:40 smithi006 bash[33918]: cluster 2023-08-28T02:27:39.654532+0000 mgr.y (mgr.14901) 8071 : cluster [DBG] pgmap v8078: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:41.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:40 smithi006 bash[33918]: audit 2023-08-28T02:27:40.025522+0000 mon.c (mon.2) 3197 : audit [DBG] from='client.? 172.21.15.6:0/4129625132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:27:41.307Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:27:41.310Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:41 smithi130 bash[51302]: debug 2023-08-28T02:27:41.640+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:41.640891+0000) 2023-08-28T02:27:43.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:42 smithi130 bash[27579]: cluster 2023-08-28T02:27:41.655174+0000 mgr.y (mgr.14901) 8072 : cluster [DBG] pgmap v8079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:43.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:42 smithi130 bash[27579]: audit 2023-08-28T02:27:42.435664+0000 mon.a (mon.0) 5115 : audit [DBG] from='client.? 172.21.15.6:0/2731874182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:43.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:42 smithi130 bash[51302]: debug 2023-08-28T02:27:42.636+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:42.641166+0000) 2023-08-28T02:27:43.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:42 smithi006 bash[33918]: cluster 2023-08-28T02:27:41.655174+0000 mgr.y (mgr.14901) 8072 : cluster [DBG] pgmap v8079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:43.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:42 smithi006 bash[33918]: audit 2023-08-28T02:27:42.435664+0000 mon.a (mon.0) 5115 : audit [DBG] from='client.? 172.21.15.6:0/2731874182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:43.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:42 smithi006 bash[27292]: cluster 2023-08-28T02:27:41.655174+0000 mgr.y (mgr.14901) 8072 : cluster [DBG] pgmap v8079: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:43.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:42 smithi006 bash[27292]: audit 2023-08-28T02:27:42.435664+0000 mon.a (mon.0) 5115 : audit [DBG] from='client.? 172.21.15.6:0/2731874182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:27:42.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:27:42.870Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:44.035 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:43 smithi130 bash[51302]: debug 2023-08-28T02:27:43.636+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:43.641367+0000) 2023-08-28T02:27:44.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:44 smithi130 bash[27579]: cluster 2023-08-28T02:27:43.655821+0000 mgr.y (mgr.14901) 8073 : cluster [DBG] pgmap v8080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:44.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:44 smithi006 bash[27292]: cluster 2023-08-28T02:27:43.655821+0000 mgr.y (mgr.14901) 8073 : cluster [DBG] pgmap v8080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:44 smithi006 bash[33918]: cluster 2023-08-28T02:27:43.655821+0000 mgr.y (mgr.14901) 8073 : cluster [DBG] pgmap v8080: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:45.038 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:27:44.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:45.038 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:27:44.752Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:45.038 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:44 smithi130 bash[51302]: debug 2023-08-28T02:27:44.636+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:44.641651+0000) 2023-08-28T02:27:45.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:45 smithi130 bash[27579]: audit 2023-08-28T02:27:44.855137+0000 mon.c (mon.2) 3198 : audit [DBG] from='client.? 172.21.15.6:0/2195556094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:45.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:45 smithi006 bash[33918]: audit 2023-08-28T02:27:44.855137+0000 mon.c (mon.2) 3198 : audit [DBG] from='client.? 172.21.15.6:0/2195556094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:45.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:45 smithi006 bash[27292]: audit 2023-08-28T02:27:44.855137+0000 mon.c (mon.2) 3198 : audit [DBG] from='client.? 172.21.15.6:0/2195556094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:46.047 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:45 smithi130 bash[51302]: debug 2023-08-28T02:27:45.636+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:45.641925+0000) 2023-08-28T02:27:46.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:46 smithi130 bash[27579]: cluster 2023-08-28T02:27:45.657050+0000 mgr.y (mgr.14901) 8074 : cluster [DBG] pgmap v8081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:46.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:46 smithi006 bash[33918]: cluster 2023-08-28T02:27:45.657050+0000 mgr.y (mgr.14901) 8074 : cluster [DBG] pgmap v8081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:46.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:46 smithi006 bash[27292]: cluster 2023-08-28T02:27:45.657050+0000 mgr.y (mgr.14901) 8074 : cluster [DBG] pgmap v8081: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:46 smithi130 bash[51302]: debug 2023-08-28T02:27:46.636+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:46.642188+0000) 2023-08-28T02:27:47.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:47 smithi130 bash[27579]: audit 2023-08-28T02:27:47.267918+0000 mon.a (mon.0) 5116 : audit [DBG] from='client.? 172.21.15.6:0/2580508729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:47.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:47 smithi006 bash[33918]: audit 2023-08-28T02:27:47.267918+0000 mon.a (mon.0) 5116 : audit [DBG] from='client.? 172.21.15.6:0/2580508729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:47.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:47 smithi006 bash[27292]: audit 2023-08-28T02:27:47.267918+0000 mon.a (mon.0) 5116 : audit [DBG] from='client.? 172.21.15.6:0/2580508729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:47 smithi130 bash[51302]: debug 2023-08-28T02:27:47.636+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:47.642442+0000) 2023-08-28T02:27:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:48 smithi130 bash[27579]: cluster 2023-08-28T02:27:47.657747+0000 mgr.y (mgr.14901) 8075 : cluster [DBG] pgmap v8082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:48.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:48 smithi006 bash[33918]: cluster 2023-08-28T02:27:47.657747+0000 mgr.y (mgr.14901) 8075 : cluster [DBG] pgmap v8082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:48.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:48 smithi006 bash[27292]: cluster 2023-08-28T02:27:47.657747+0000 mgr.y (mgr.14901) 8075 : cluster [DBG] pgmap v8082: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:49.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:48 smithi130 bash[51302]: debug 2023-08-28T02:27:48.636+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:48.642713+0000) 2023-08-28T02:27:50.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:49 smithi130 bash[27579]: audit 2023-08-28T02:27:49.685555+0000 mon.a (mon.0) 5117 : audit [DBG] from='client.? 172.21.15.6:0/2829296146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:50.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:49 smithi130 bash[51302]: debug 2023-08-28T02:27:49.640+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:49.642983+0000) 2023-08-28T02:27:50.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:49 smithi006 bash[33918]: audit 2023-08-28T02:27:49.685555+0000 mon.a (mon.0) 5117 : audit [DBG] from='client.? 172.21.15.6:0/2829296146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:50.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:49 smithi006 bash[27292]: audit 2023-08-28T02:27:49.685555+0000 mon.a (mon.0) 5117 : audit [DBG] from='client.? 172.21.15.6:0/2829296146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:51.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:50 smithi130 bash[27579]: cluster 2023-08-28T02:27:49.658848+0000 mgr.y (mgr.14901) 8076 : cluster [DBG] pgmap v8083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:51.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:50 smithi130 bash[51302]: debug 2023-08-28T02:27:50.640+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:50.643242+0000) 2023-08-28T02:27:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:27:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:27:50] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:27:51.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:50 smithi006 bash[33918]: cluster 2023-08-28T02:27:49.658848+0000 mgr.y (mgr.14901) 8076 : cluster [DBG] pgmap v8083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:51.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:50 smithi006 bash[27292]: cluster 2023-08-28T02:27:49.658848+0000 mgr.y (mgr.14901) 8076 : cluster [DBG] pgmap v8083: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:27:51.308Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:27:51.310Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:52.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:51 smithi130 bash[51302]: debug 2023-08-28T02:27:51.640+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:51.643495+0000) 2023-08-28T02:27:53.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:52 smithi130 bash[51302]: debug 2023-08-28T02:27:52.640+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:52.643782+0000) 2023-08-28T02:27:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:52 smithi130 bash[27579]: cluster 2023-08-28T02:27:51.659529+0000 mgr.y (mgr.14901) 8077 : cluster [DBG] pgmap v8084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:52 smithi130 bash[27579]: audit 2023-08-28T02:27:52.101932+0000 mon.c (mon.2) 3199 : audit [DBG] from='client.? 172.21.15.6:0/110695842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:53.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:52 smithi006 bash[27292]: cluster 2023-08-28T02:27:51.659529+0000 mgr.y (mgr.14901) 8077 : cluster [DBG] pgmap v8084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:53.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:52 smithi006 bash[27292]: audit 2023-08-28T02:27:52.101932+0000 mon.c (mon.2) 3199 : audit [DBG] from='client.? 172.21.15.6:0/110695842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:53.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:52 smithi006 bash[33918]: cluster 2023-08-28T02:27:51.659529+0000 mgr.y (mgr.14901) 8077 : cluster [DBG] pgmap v8084: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:53.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:52 smithi006 bash[33918]: audit 2023-08-28T02:27:52.101932+0000 mon.c (mon.2) 3199 : audit [DBG] from='client.? 172.21.15.6:0/110695842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:27:52.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:27:52.870Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:54.040 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:53 smithi130 bash[51302]: debug 2023-08-28T02:27:53.640+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:53.644059+0000) 2023-08-28T02:27:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:54 smithi130 bash[27579]: cluster 2023-08-28T02:27:53.660209+0000 mgr.y (mgr.14901) 8078 : cluster [DBG] pgmap v8085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:54.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:54 smithi006 bash[33918]: cluster 2023-08-28T02:27:53.660209+0000 mgr.y (mgr.14901) 8078 : cluster [DBG] pgmap v8085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:54 smithi006 bash[27292]: cluster 2023-08-28T02:27:53.660209+0000 mgr.y (mgr.14901) 8078 : cluster [DBG] pgmap v8085: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:55.041 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:27:54.752Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:55.041 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:27:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:27:54.755Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:27:55.042 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:54 smithi130 bash[51302]: debug 2023-08-28T02:27:54.640+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:54.644375+0000) 2023-08-28T02:27:55.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:55 smithi130 bash[27579]: audit 2023-08-28T02:27:54.514724+0000 mon.a (mon.0) 5118 : audit [DBG] from='client.? 172.21.15.6:0/3949500182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:55.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:55 smithi006 bash[27292]: audit 2023-08-28T02:27:54.514724+0000 mon.a (mon.0) 5118 : audit [DBG] from='client.? 172.21.15.6:0/3949500182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:55.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:55 smithi006 bash[33918]: audit 2023-08-28T02:27:54.514724+0000 mon.a (mon.0) 5118 : audit [DBG] from='client.? 172.21.15.6:0/3949500182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:56.056 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:55 smithi130 bash[51302]: debug 2023-08-28T02:27:55.640+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:55.644790+0000) 2023-08-28T02:27:56.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:56 smithi130 bash[27579]: cluster 2023-08-28T02:27:55.661470+0000 mgr.y (mgr.14901) 8079 : cluster [DBG] pgmap v8086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:56.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:56 smithi006 bash[27292]: cluster 2023-08-28T02:27:55.661470+0000 mgr.y (mgr.14901) 8079 : cluster [DBG] pgmap v8086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:56.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:56 smithi006 bash[33918]: cluster 2023-08-28T02:27:55.661470+0000 mgr.y (mgr.14901) 8079 : cluster [DBG] pgmap v8086: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:57.060 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:56 smithi130 bash[51302]: debug 2023-08-28T02:27:56.639+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:56.645072+0000) 2023-08-28T02:27:57.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:57 smithi130 bash[27579]: audit 2023-08-28T02:27:56.921273+0000 mon.c (mon.2) 3200 : audit [DBG] from='client.? 172.21.15.6:0/1256637879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:57.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:57 smithi006 bash[27292]: audit 2023-08-28T02:27:56.921273+0000 mon.c (mon.2) 3200 : audit [DBG] from='client.? 172.21.15.6:0/1256637879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:57.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:57 smithi006 bash[33918]: audit 2023-08-28T02:27:56.921273+0000 mon.c (mon.2) 3200 : audit [DBG] from='client.? 172.21.15.6:0/1256637879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:58.069 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:57 smithi130 bash[51302]: debug 2023-08-28T02:27:57.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:57.645335+0000) 2023-08-28T02:27:58.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:58 smithi130 bash[27579]: cluster 2023-08-28T02:27:57.662148+0000 mgr.y (mgr.14901) 8080 : cluster [DBG] pgmap v8087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:58.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:58 smithi006 bash[27292]: cluster 2023-08-28T02:27:57.662148+0000 mgr.y (mgr.14901) 8080 : cluster [DBG] pgmap v8087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:58.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:58 smithi006 bash[33918]: cluster 2023-08-28T02:27:57.662148+0000 mgr.y (mgr.14901) 8080 : cluster [DBG] pgmap v8087: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:27:59.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:58 smithi130 bash[51302]: debug 2023-08-28T02:27:58.639+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:58.645591+0000) 2023-08-28T02:27:59.644 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:27:59 smithi130 bash[27579]: audit 2023-08-28T02:27:59.340110+0000 mon.c (mon.2) 3201 : audit [DBG] from='client.? 172.21.15.6:0/1772665366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:59.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:27:59 smithi006 bash[27292]: audit 2023-08-28T02:27:59.340110+0000 mon.c (mon.2) 3201 : audit [DBG] from='client.? 172.21.15.6:0/1772665366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:59.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:27:59 smithi006 bash[33918]: audit 2023-08-28T02:27:59.340110+0000 mon.c (mon.2) 3201 : audit [DBG] from='client.? 172.21.15.6:0/1772665366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:27:59 smithi130 bash[51302]: debug 2023-08-28T02:27:59.639+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:59.645810+0000) 2023-08-28T02:28:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:00 smithi006 bash[33918]: cluster 2023-08-28T02:27:59.663288+0000 mgr.y (mgr.14901) 8081 : cluster [DBG] pgmap v8088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:00 smithi006 bash[27292]: cluster 2023-08-28T02:27:59.663288+0000 mgr.y (mgr.14901) 8081 : cluster [DBG] pgmap v8088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:00.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:00 smithi130 bash[27579]: cluster 2023-08-28T02:27:59.663288+0000 mgr.y (mgr.14901) 8081 : cluster [DBG] pgmap v8088: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:00.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:00 smithi130 bash[51302]: debug 2023-08-28T02:28:00.639+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:00.646014+0000) 2023-08-28T02:28:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:28:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:28:00] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T02:28:01.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:01 smithi006 bash[33918]: audit 2023-08-28T02:28:00.451361+0000 mon.b (mon.1) 882 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:28:01.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:01 smithi006 bash[27292]: audit 2023-08-28T02:28:00.451361+0000 mon.b (mon.1) 882 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:28:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:28:01.308Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:28:01.310Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:01.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:01 smithi130 bash[27579]: audit 2023-08-28T02:28:00.451361+0000 mon.b (mon.1) 882 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:28:01.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:01 smithi130 bash[51302]: debug 2023-08-28T02:28:01.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:01.646315+0000) 2023-08-28T02:28:02.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:02 smithi006 bash[27292]: cluster 2023-08-28T02:28:01.663927+0000 mgr.y (mgr.14901) 8082 : cluster [DBG] pgmap v8089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:02.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:02 smithi006 bash[27292]: audit 2023-08-28T02:28:01.762858+0000 mon.a (mon.0) 5119 : audit [DBG] from='client.? 172.21.15.6:0/3378060809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:02.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:02 smithi006 bash[33918]: cluster 2023-08-28T02:28:01.663927+0000 mgr.y (mgr.14901) 8082 : cluster [DBG] pgmap v8089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:02.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:02 smithi006 bash[33918]: audit 2023-08-28T02:28:01.762858+0000 mon.a (mon.0) 5119 : audit [DBG] from='client.? 172.21.15.6:0/3378060809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:02 smithi130 bash[27579]: cluster 2023-08-28T02:28:01.663927+0000 mgr.y (mgr.14901) 8082 : cluster [DBG] pgmap v8089: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:02.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:02 smithi130 bash[27579]: audit 2023-08-28T02:28:01.762858+0000 mon.a (mon.0) 5119 : audit [DBG] from='client.? 172.21.15.6:0/3378060809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:02.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:02 smithi130 bash[51302]: debug 2023-08-28T02:28:02.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:02.646637+0000) 2023-08-28T02:28:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:28:02.869Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:28:02.871Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:04.041 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:03 smithi130 bash[51302]: debug 2023-08-28T02:28:03.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:03.646932+0000) 2023-08-28T02:28:04.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:04 smithi130 bash[27579]: cluster 2023-08-28T02:28:03.664617+0000 mgr.y (mgr.14901) 8083 : cluster [DBG] pgmap v8090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:04 smithi006 bash[27292]: cluster 2023-08-28T02:28:03.664617+0000 mgr.y (mgr.14901) 8083 : cluster [DBG] pgmap v8090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:04.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:04 smithi006 bash[33918]: cluster 2023-08-28T02:28:03.664617+0000 mgr.y (mgr.14901) 8083 : cluster [DBG] pgmap v8090: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:05.043 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:28:04.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:05.044 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:28:04.754Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:05.044 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:04 smithi130 bash[51302]: debug 2023-08-28T02:28:04.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:04.647193+0000) 2023-08-28T02:28:05.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:05 smithi130 bash[27579]: audit 2023-08-28T02:28:04.182444+0000 mon.c (mon.2) 3202 : audit [DBG] from='client.? 172.21.15.6:0/9367304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:05.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:05 smithi006 bash[33918]: audit 2023-08-28T02:28:04.182444+0000 mon.c (mon.2) 3202 : audit [DBG] from='client.? 172.21.15.6:0/9367304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:05.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:05 smithi006 bash[27292]: audit 2023-08-28T02:28:04.182444+0000 mon.c (mon.2) 3202 : audit [DBG] from='client.? 172.21.15.6:0/9367304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:06.051 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:05 smithi130 bash[51302]: debug 2023-08-28T02:28:05.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:05.647460+0000) 2023-08-28T02:28:06.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:06 smithi130 bash[27579]: cluster 2023-08-28T02:28:05.665591+0000 mgr.y (mgr.14901) 8084 : cluster [DBG] pgmap v8091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:06.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:06 smithi006 bash[27292]: cluster 2023-08-28T02:28:05.665591+0000 mgr.y (mgr.14901) 8084 : cluster [DBG] pgmap v8091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:06.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:06 smithi006 bash[33918]: cluster 2023-08-28T02:28:05.665591+0000 mgr.y (mgr.14901) 8084 : cluster [DBG] pgmap v8091: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:07.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:06 smithi130 bash[51302]: debug 2023-08-28T02:28:06.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:06.647733+0000) 2023-08-28T02:28:07.647 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:07 smithi130 bash[27579]: audit 2023-08-28T02:28:06.390289+0000 mon.a (mon.0) 5120 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:28:07.647 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:07 smithi130 bash[27579]: audit 2023-08-28T02:28:06.613500+0000 mon.c (mon.2) 3203 : audit [DBG] from='client.? 172.21.15.6:0/698809818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:07.647 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:07 smithi130 bash[27579]: audit 2023-08-28T02:28:06.781579+0000 mon.a (mon.0) 5121 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:28:07.647 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:07 smithi130 bash[27579]: audit 2023-08-28T02:28:07.085418+0000 mon.b (mon.1) 883 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:28:07.647 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:07 smithi130 bash[27579]: audit 2023-08-28T02:28:07.087809+0000 mon.b (mon.1) 884 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:28:07.647 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:07 smithi130 bash[27579]: audit 2023-08-28T02:28:07.096717+0000 mon.a (mon.0) 5122 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:28:07.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:07 smithi006 bash[33918]: audit 2023-08-28T02:28:06.390289+0000 mon.a (mon.0) 5120 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:28:07.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:07 smithi006 bash[33918]: audit 2023-08-28T02:28:06.613500+0000 mon.c (mon.2) 3203 : audit [DBG] from='client.? 172.21.15.6:0/698809818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:07.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:07 smithi006 bash[33918]: audit 2023-08-28T02:28:06.781579+0000 mon.a (mon.0) 5121 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:28:07.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:07 smithi006 bash[33918]: audit 2023-08-28T02:28:07.085418+0000 mon.b (mon.1) 883 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:28:07.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:07 smithi006 bash[33918]: audit 2023-08-28T02:28:07.087809+0000 mon.b (mon.1) 884 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:28:07.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:07 smithi006 bash[33918]: audit 2023-08-28T02:28:07.096717+0000 mon.a (mon.0) 5122 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:28:07.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:07 smithi006 bash[27292]: audit 2023-08-28T02:28:06.390289+0000 mon.a (mon.0) 5120 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:28:07.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:07 smithi006 bash[27292]: audit 2023-08-28T02:28:06.613500+0000 mon.c (mon.2) 3203 : audit [DBG] from='client.? 172.21.15.6:0/698809818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:07.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:07 smithi006 bash[27292]: audit 2023-08-28T02:28:06.781579+0000 mon.a (mon.0) 5121 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:28:07.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:07 smithi006 bash[27292]: audit 2023-08-28T02:28:07.085418+0000 mon.b (mon.1) 883 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:28:07.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:07 smithi006 bash[27292]: audit 2023-08-28T02:28:07.087809+0000 mon.b (mon.1) 884 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:28:07.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:07 smithi006 bash[27292]: audit 2023-08-28T02:28:07.096717+0000 mon.a (mon.0) 5122 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:28:08.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:07 smithi130 bash[51302]: debug 2023-08-28T02:28:07.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:07.648003+0000) 2023-08-28T02:28:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:08 smithi006 bash[33918]: cluster 2023-08-28T02:28:07.666228+0000 mgr.y (mgr.14901) 8085 : cluster [DBG] pgmap v8092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:08 smithi006 bash[27292]: cluster 2023-08-28T02:28:07.666228+0000 mgr.y (mgr.14901) 8085 : cluster [DBG] pgmap v8092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:08 smithi130 bash[27579]: cluster 2023-08-28T02:28:07.666228+0000 mgr.y (mgr.14901) 8085 : cluster [DBG] pgmap v8092: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:08.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:08 smithi130 bash[51302]: debug 2023-08-28T02:28:08.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:08.648289+0000) 2023-08-28T02:28:09.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:09 smithi006 bash[27292]: audit 2023-08-28T02:28:09.031328+0000 mon.c (mon.2) 3204 : audit [DBG] from='client.? 172.21.15.6:0/410622919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:09.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:09 smithi006 bash[33918]: audit 2023-08-28T02:28:09.031328+0000 mon.c (mon.2) 3204 : audit [DBG] from='client.? 172.21.15.6:0/410622919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:09.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:09 smithi130 bash[27579]: audit 2023-08-28T02:28:09.031328+0000 mon.c (mon.2) 3204 : audit [DBG] from='client.? 172.21.15.6:0/410622919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:09.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:09 smithi130 bash[51302]: debug 2023-08-28T02:28:09.647+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:09.648597+0000) 2023-08-28T02:28:10.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:10 smithi006 bash[27292]: cluster 2023-08-28T02:28:09.667420+0000 mgr.y (mgr.14901) 8086 : cluster [DBG] pgmap v8093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:10.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:10 smithi006 bash[33918]: cluster 2023-08-28T02:28:09.667420+0000 mgr.y (mgr.14901) 8086 : cluster [DBG] pgmap v8093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:10.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:10 smithi130 bash[27579]: cluster 2023-08-28T02:28:09.667420+0000 mgr.y (mgr.14901) 8086 : cluster [DBG] pgmap v8093: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:10.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:10 smithi130 bash[51302]: debug 2023-08-28T02:28:10.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:10.648903+0000) 2023-08-28T02:28:11.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:28:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:28:10] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T02:28:11.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:11 smithi006 bash[27292]: audit 2023-08-28T02:28:11.446660+0000 mon.c (mon.2) 3205 : audit [DBG] from='client.? 172.21.15.6:0/1183605172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:11.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:11 smithi006 bash[33918]: audit 2023-08-28T02:28:11.446660+0000 mon.c (mon.2) 3205 : audit [DBG] from='client.? 172.21.15.6:0/1183605172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:28:11.309Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:11.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:28:11.311Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:11.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:11 smithi130 bash[27579]: audit 2023-08-28T02:28:11.446660+0000 mon.c (mon.2) 3205 : audit [DBG] from='client.? 172.21.15.6:0/1183605172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:11.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:11 smithi130 bash[51302]: debug 2023-08-28T02:28:11.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:11.649250+0000) 2023-08-28T02:28:12.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:12 smithi130 bash[27579]: cluster 2023-08-28T02:28:11.668190+0000 mgr.y (mgr.14901) 8087 : cluster [DBG] pgmap v8094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:12.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:12 smithi130 bash[51302]: debug 2023-08-28T02:28:12.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:12.649526+0000) 2023-08-28T02:28:12.870 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:12 smithi006 bash[33918]: cluster 2023-08-28T02:28:11.668190+0000 mgr.y (mgr.14901) 8087 : cluster [DBG] pgmap v8094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:12.870 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:12 smithi006 bash[27292]: cluster 2023-08-28T02:28:11.668190+0000 mgr.y (mgr.14901) 8087 : cluster [DBG] pgmap v8094: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:28:12.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:28:12.871Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:14.044 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:13 smithi130 bash[51302]: debug 2023-08-28T02:28:13.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:13.649792+0000) 2023-08-28T02:28:14.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:14 smithi130 bash[27579]: cluster 2023-08-28T02:28:13.668697+0000 mgr.y (mgr.14901) 8088 : cluster [DBG] pgmap v8095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:14.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:14 smithi130 bash[27579]: audit 2023-08-28T02:28:13.866917+0000 mon.a (mon.0) 5123 : audit [DBG] from='client.? 172.21.15.6:0/1394357666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:14.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:14 smithi006 bash[33918]: cluster 2023-08-28T02:28:13.668697+0000 mgr.y (mgr.14901) 8088 : cluster [DBG] pgmap v8095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:14 smithi006 bash[33918]: audit 2023-08-28T02:28:13.866917+0000 mon.a (mon.0) 5123 : audit [DBG] from='client.? 172.21.15.6:0/1394357666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:14 smithi006 bash[27292]: cluster 2023-08-28T02:28:13.668697+0000 mgr.y (mgr.14901) 8088 : cluster [DBG] pgmap v8095: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:14 smithi006 bash[27292]: audit 2023-08-28T02:28:13.866917+0000 mon.a (mon.0) 5123 : audit [DBG] from='client.? 172.21.15.6:0/1394357666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:14 smithi130 bash[51302]: debug 2023-08-28T02:28:14.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:14.650105+0000) 2023-08-28T02:28:15.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:28:14.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:15.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:28:14.756Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:16.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:15 smithi130 bash[51302]: debug 2023-08-28T02:28:15.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:15.650426+0000) 2023-08-28T02:28:17.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:16 smithi130 bash[51302]: debug 2023-08-28T02:28:16.643+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:16.650729+0000) 2023-08-28T02:28:17.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:16 smithi130 bash[27579]: cluster 2023-08-28T02:28:15.669734+0000 mgr.y (mgr.14901) 8089 : cluster [DBG] pgmap v8096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:17.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:16 smithi130 bash[27579]: audit 2023-08-28T02:28:16.291927+0000 mon.a (mon.0) 5124 : audit [DBG] from='client.? 172.21.15.6:0/2606307116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:17.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:16 smithi006 bash[27292]: cluster 2023-08-28T02:28:15.669734+0000 mgr.y (mgr.14901) 8089 : cluster [DBG] pgmap v8096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:17.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:16 smithi006 bash[27292]: audit 2023-08-28T02:28:16.291927+0000 mon.a (mon.0) 5124 : audit [DBG] from='client.? 172.21.15.6:0/2606307116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:17.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:16 smithi006 bash[33918]: cluster 2023-08-28T02:28:15.669734+0000 mgr.y (mgr.14901) 8089 : cluster [DBG] pgmap v8096: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:17.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:16 smithi006 bash[33918]: audit 2023-08-28T02:28:16.291927+0000 mon.a (mon.0) 5124 : audit [DBG] from='client.? 172.21.15.6:0/2606307116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:18.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:17 smithi130 bash[51302]: debug 2023-08-28T02:28:17.647+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:17.651032+0000) 2023-08-28T02:28:19.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:18 smithi130 bash[51302]: debug 2023-08-28T02:28:18.647+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:18.651337+0000) 2023-08-28T02:28:19.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:18 smithi130 bash[27579]: cluster 2023-08-28T02:28:17.670504+0000 mgr.y (mgr.14901) 8090 : cluster [DBG] pgmap v8097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:19.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:18 smithi130 bash[27579]: audit 2023-08-28T02:28:18.706112+0000 mon.c (mon.2) 3206 : audit [DBG] from='client.? 172.21.15.6:0/397709331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:19.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:18 smithi006 bash[27292]: cluster 2023-08-28T02:28:17.670504+0000 mgr.y (mgr.14901) 8090 : cluster [DBG] pgmap v8097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:19.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:18 smithi006 bash[27292]: audit 2023-08-28T02:28:18.706112+0000 mon.c (mon.2) 3206 : audit [DBG] from='client.? 172.21.15.6:0/397709331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:19.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:18 smithi006 bash[33918]: cluster 2023-08-28T02:28:17.670504+0000 mgr.y (mgr.14901) 8090 : cluster [DBG] pgmap v8097: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:19.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:18 smithi006 bash[33918]: audit 2023-08-28T02:28:18.706112+0000 mon.c (mon.2) 3206 : audit [DBG] from='client.? 172.21.15.6:0/397709331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:19 smithi130 bash[51302]: debug 2023-08-28T02:28:19.647+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:19.651625+0000) 2023-08-28T02:28:21.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:20 smithi130 bash[27579]: cluster 2023-08-28T02:28:19.671751+0000 mgr.y (mgr.14901) 8091 : cluster [DBG] pgmap v8098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:21.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:20 smithi130 bash[51302]: debug 2023-08-28T02:28:20.647+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:20.651920+0000) 2023-08-28T02:28:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:28:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:28:20] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T02:28:21.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:20 smithi006 bash[33918]: cluster 2023-08-28T02:28:19.671751+0000 mgr.y (mgr.14901) 8091 : cluster [DBG] pgmap v8098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:21.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:20 smithi006 bash[27292]: cluster 2023-08-28T02:28:19.671751+0000 mgr.y (mgr.14901) 8091 : cluster [DBG] pgmap v8098: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:28:21.310Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:28:21.312Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:22.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:21 smithi130 bash[27579]: audit 2023-08-28T02:28:21.133002+0000 mon.c (mon.2) 3207 : audit [DBG] from='client.? 172.21.15.6:0/1581249894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:22.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:21 smithi130 bash[51302]: debug 2023-08-28T02:28:21.651+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:21.652219+0000) 2023-08-28T02:28:22.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:21 smithi006 bash[27292]: audit 2023-08-28T02:28:21.133002+0000 mon.c (mon.2) 3207 : audit [DBG] from='client.? 172.21.15.6:0/1581249894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:22.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:21 smithi006 bash[33918]: audit 2023-08-28T02:28:21.133002+0000 mon.c (mon.2) 3207 : audit [DBG] from='client.? 172.21.15.6:0/1581249894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:23.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:22 smithi130 bash[27579]: cluster 2023-08-28T02:28:21.672600+0000 mgr.y (mgr.14901) 8092 : cluster [DBG] pgmap v8099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:23.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:22 smithi130 bash[51302]: debug 2023-08-28T02:28:22.647+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:22.652481+0000) 2023-08-28T02:28:23.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:22 smithi006 bash[33918]: cluster 2023-08-28T02:28:21.672600+0000 mgr.y (mgr.14901) 8092 : cluster [DBG] pgmap v8099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:28:22.871Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:28:22.872Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:23.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:22 smithi006 bash[27292]: cluster 2023-08-28T02:28:21.672600+0000 mgr.y (mgr.14901) 8092 : cluster [DBG] pgmap v8099: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:24.047 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:23 smithi130 bash[51302]: debug 2023-08-28T02:28:23.647+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:23.652796+0000) 2023-08-28T02:28:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:24 smithi130 bash[27579]: audit 2023-08-28T02:28:23.553172+0000 mon.a (mon.0) 5125 : audit [DBG] from='client.? 172.21.15.6:0/1565573908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:24 smithi130 bash[27579]: cluster 2023-08-28T02:28:23.673406+0000 mgr.y (mgr.14901) 8093 : cluster [DBG] pgmap v8100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:24 smithi006 bash[27292]: audit 2023-08-28T02:28:23.553172+0000 mon.a (mon.0) 5125 : audit [DBG] from='client.? 172.21.15.6:0/1565573908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:24 smithi006 bash[27292]: cluster 2023-08-28T02:28:23.673406+0000 mgr.y (mgr.14901) 8093 : cluster [DBG] pgmap v8100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:24.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:24 smithi006 bash[33918]: audit 2023-08-28T02:28:23.553172+0000 mon.a (mon.0) 5125 : audit [DBG] from='client.? 172.21.15.6:0/1565573908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:24.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:24 smithi006 bash[33918]: cluster 2023-08-28T02:28:23.673406+0000 mgr.y (mgr.14901) 8093 : cluster [DBG] pgmap v8100: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:25.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:24 smithi130 bash[51302]: debug 2023-08-28T02:28:24.647+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:24.653107+0000) 2023-08-28T02:28:25.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:28:24.755Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:25.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:28:24.757Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:26.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:25 smithi130 bash[51302]: debug 2023-08-28T02:28:25.647+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:25.653366+0000) 2023-08-28T02:28:27.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:26 smithi130 bash[27579]: cluster 2023-08-28T02:28:25.674538+0000 mgr.y (mgr.14901) 8094 : cluster [DBG] pgmap v8101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:27.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:26 smithi130 bash[27579]: audit 2023-08-28T02:28:25.966687+0000 mon.c (mon.2) 3208 : audit [DBG] from='client.? 172.21.15.6:0/3106280455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:27.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:26 smithi130 bash[51302]: debug 2023-08-28T02:28:26.651+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:26.653668+0000) 2023-08-28T02:28:27.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:26 smithi006 bash[33918]: cluster 2023-08-28T02:28:25.674538+0000 mgr.y (mgr.14901) 8094 : cluster [DBG] pgmap v8101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:27.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:26 smithi006 bash[33918]: audit 2023-08-28T02:28:25.966687+0000 mon.c (mon.2) 3208 : audit [DBG] from='client.? 172.21.15.6:0/3106280455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:27.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:26 smithi006 bash[27292]: cluster 2023-08-28T02:28:25.674538+0000 mgr.y (mgr.14901) 8094 : cluster [DBG] pgmap v8101: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:27.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:26 smithi006 bash[27292]: audit 2023-08-28T02:28:25.966687+0000 mon.c (mon.2) 3208 : audit [DBG] from='client.? 172.21.15.6:0/3106280455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:28.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:27 smithi130 bash[51302]: debug 2023-08-28T02:28:27.647+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:27.653943+0000) 2023-08-28T02:28:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:28 smithi130 bash[51302]: debug 2023-08-28T02:28:28.647+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:28.654248+0000) 2023-08-28T02:28:29.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:28 smithi130 bash[27579]: cluster 2023-08-28T02:28:27.675203+0000 mgr.y (mgr.14901) 8095 : cluster [DBG] pgmap v8102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:29.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:28 smithi130 bash[27579]: audit 2023-08-28T02:28:28.374713+0000 mon.a (mon.0) 5126 : audit [DBG] from='client.? 172.21.15.6:0/3609013483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:29.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:28 smithi006 bash[27292]: cluster 2023-08-28T02:28:27.675203+0000 mgr.y (mgr.14901) 8095 : cluster [DBG] pgmap v8102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:29.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:28 smithi006 bash[27292]: audit 2023-08-28T02:28:28.374713+0000 mon.a (mon.0) 5126 : audit [DBG] from='client.? 172.21.15.6:0/3609013483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:29.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:28 smithi006 bash[33918]: cluster 2023-08-28T02:28:27.675203+0000 mgr.y (mgr.14901) 8095 : cluster [DBG] pgmap v8102: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:29.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:28 smithi006 bash[33918]: audit 2023-08-28T02:28:28.374713+0000 mon.a (mon.0) 5126 : audit [DBG] from='client.? 172.21.15.6:0/3609013483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:30.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:29 smithi130 bash[51302]: debug 2023-08-28T02:28:29.647+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:29.654521+0000) 2023-08-28T02:28:31.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:30 smithi130 bash[27579]: cluster 2023-08-28T02:28:29.676310+0000 mgr.y (mgr.14901) 8096 : cluster [DBG] pgmap v8103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:31.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:30 smithi130 bash[51302]: debug 2023-08-28T02:28:30.651+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:30.654812+0000) 2023-08-28T02:28:31.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:30 smithi006 bash[27292]: cluster 2023-08-28T02:28:29.676310+0000 mgr.y (mgr.14901) 8096 : cluster [DBG] pgmap v8103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:31.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:28:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:28:30] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:28:31.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:30 smithi006 bash[33918]: cluster 2023-08-28T02:28:29.676310+0000 mgr.y (mgr.14901) 8096 : cluster [DBG] pgmap v8103: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:28:31.311Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:28:31.313Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:32.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:31 smithi130 bash[27579]: audit 2023-08-28T02:28:30.796240+0000 mon.c (mon.2) 3209 : audit [DBG] from='client.? 172.21.15.6:0/3129491845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:32.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:31 smithi130 bash[51302]: debug 2023-08-28T02:28:31.651+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:31.655145+0000) 2023-08-28T02:28:32.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:31 smithi006 bash[27292]: audit 2023-08-28T02:28:30.796240+0000 mon.c (mon.2) 3209 : audit [DBG] from='client.? 172.21.15.6:0/3129491845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:32.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:31 smithi006 bash[33918]: audit 2023-08-28T02:28:30.796240+0000 mon.c (mon.2) 3209 : audit [DBG] from='client.? 172.21.15.6:0/3129491845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:33.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:32 smithi130 bash[27579]: cluster 2023-08-28T02:28:31.677008+0000 mgr.y (mgr.14901) 8097 : cluster [DBG] pgmap v8104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:33.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:32 smithi130 bash[51302]: debug 2023-08-28T02:28:32.651+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:32.655411+0000) 2023-08-28T02:28:33.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:32 smithi006 bash[33918]: cluster 2023-08-28T02:28:31.677008+0000 mgr.y (mgr.14901) 8097 : cluster [DBG] pgmap v8104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:28:32.872Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:28:32.874Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:33.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:32 smithi006 bash[27292]: cluster 2023-08-28T02:28:31.677008+0000 mgr.y (mgr.14901) 8097 : cluster [DBG] pgmap v8104: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:34.048 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:33 smithi130 bash[51302]: debug 2023-08-28T02:28:33.655+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:33.655684+0000) 2023-08-28T02:28:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:34 smithi130 bash[27579]: audit 2023-08-28T02:28:33.209958+0000 mon.a (mon.0) 5127 : audit [DBG] from='client.? 172.21.15.6:0/1000352567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:34 smithi130 bash[27579]: cluster 2023-08-28T02:28:33.677688+0000 mgr.y (mgr.14901) 8098 : cluster [DBG] pgmap v8105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:34.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:34 smithi006 bash[33918]: audit 2023-08-28T02:28:33.209958+0000 mon.a (mon.0) 5127 : audit [DBG] from='client.? 172.21.15.6:0/1000352567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:34 smithi006 bash[33918]: cluster 2023-08-28T02:28:33.677688+0000 mgr.y (mgr.14901) 8098 : cluster [DBG] pgmap v8105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:34.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:34 smithi006 bash[27292]: audit 2023-08-28T02:28:33.209958+0000 mon.a (mon.0) 5127 : audit [DBG] from='client.? 172.21.15.6:0/1000352567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:34.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:34 smithi006 bash[27292]: cluster 2023-08-28T02:28:33.677688+0000 mgr.y (mgr.14901) 8098 : cluster [DBG] pgmap v8105: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:35.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:34 smithi130 bash[51302]: debug 2023-08-28T02:28:34.651+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:34.655919+0000) 2023-08-28T02:28:35.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:28:34.755Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:35.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:28:34.757Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:35.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:35 smithi006 bash[27292]: audit 2023-08-28T02:28:35.626113+0000 mon.a (mon.0) 5128 : audit [DBG] from='client.? 172.21.15.6:0/199851114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:35.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:35 smithi006 bash[33918]: audit 2023-08-28T02:28:35.626113+0000 mon.a (mon.0) 5128 : audit [DBG] from='client.? 172.21.15.6:0/199851114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:36.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:35 smithi130 bash[27579]: audit 2023-08-28T02:28:35.626113+0000 mon.a (mon.0) 5128 : audit [DBG] from='client.? 172.21.15.6:0/199851114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:36.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:35 smithi130 bash[51302]: debug 2023-08-28T02:28:35.651+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:35.656200+0000) 2023-08-28T02:28:36.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:36 smithi006 bash[27292]: cluster 2023-08-28T02:28:35.678882+0000 mgr.y (mgr.14901) 8099 : cluster [DBG] pgmap v8106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:36.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:36 smithi006 bash[33918]: cluster 2023-08-28T02:28:35.678882+0000 mgr.y (mgr.14901) 8099 : cluster [DBG] pgmap v8106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:37.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:36 smithi130 bash[27579]: cluster 2023-08-28T02:28:35.678882+0000 mgr.y (mgr.14901) 8099 : cluster [DBG] pgmap v8106: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:37.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:36 smithi130 bash[51302]: debug 2023-08-28T02:28:36.651+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:36.656468+0000) 2023-08-28T02:28:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:37 smithi130 bash[51302]: debug 2023-08-28T02:28:37.651+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:37.656741+0000) 2023-08-28T02:28:38.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:38 smithi006 bash[27292]: cluster 2023-08-28T02:28:37.679595+0000 mgr.y (mgr.14901) 8100 : cluster [DBG] pgmap v8107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:38.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:38 smithi006 bash[27292]: audit 2023-08-28T02:28:38.047115+0000 mon.c (mon.2) 3210 : audit [DBG] from='client.? 172.21.15.6:0/2108396492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:38.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:38 smithi006 bash[33918]: cluster 2023-08-28T02:28:37.679595+0000 mgr.y (mgr.14901) 8100 : cluster [DBG] pgmap v8107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:38.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:38 smithi006 bash[33918]: audit 2023-08-28T02:28:38.047115+0000 mon.c (mon.2) 3210 : audit [DBG] from='client.? 172.21.15.6:0/2108396492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:39.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:38 smithi130 bash[27579]: cluster 2023-08-28T02:28:37.679595+0000 mgr.y (mgr.14901) 8100 : cluster [DBG] pgmap v8107: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:39.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:38 smithi130 bash[27579]: audit 2023-08-28T02:28:38.047115+0000 mon.c (mon.2) 3210 : audit [DBG] from='client.? 172.21.15.6:0/2108396492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:39.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:38 smithi130 bash[51302]: debug 2023-08-28T02:28:38.651+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:38.657013+0000) 2023-08-28T02:28:40.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:39 smithi130 bash[51302]: debug 2023-08-28T02:28:39.651+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:39.657285+0000) 2023-08-28T02:28:40.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:28:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:28:40] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:28:40.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:40 smithi006 bash[27292]: cluster 2023-08-28T02:28:39.680694+0000 mgr.y (mgr.14901) 8101 : cluster [DBG] pgmap v8108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:40.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:40 smithi006 bash[27292]: audit 2023-08-28T02:28:40.469706+0000 mon.a (mon.0) 5129 : audit [DBG] from='client.? 172.21.15.6:0/1011835707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:40.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:40 smithi006 bash[33918]: cluster 2023-08-28T02:28:39.680694+0000 mgr.y (mgr.14901) 8101 : cluster [DBG] pgmap v8108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:40.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:40 smithi006 bash[33918]: audit 2023-08-28T02:28:40.469706+0000 mon.a (mon.0) 5129 : audit [DBG] from='client.? 172.21.15.6:0/1011835707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:41.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:40 smithi130 bash[27579]: cluster 2023-08-28T02:28:39.680694+0000 mgr.y (mgr.14901) 8101 : cluster [DBG] pgmap v8108: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:41.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:40 smithi130 bash[27579]: audit 2023-08-28T02:28:40.469706+0000 mon.a (mon.0) 5129 : audit [DBG] from='client.? 172.21.15.6:0/1011835707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:41.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:40 smithi130 bash[51302]: debug 2023-08-28T02:28:40.651+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:40.657560+0000) 2023-08-28T02:28:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:28:41.312Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:28:41.314Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:41 smithi130 bash[51302]: debug 2023-08-28T02:28:41.651+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:41.657879+0000) 2023-08-28T02:28:42.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:42 smithi006 bash[27292]: cluster 2023-08-28T02:28:41.681368+0000 mgr.y (mgr.14901) 8102 : cluster [DBG] pgmap v8109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:42.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:42 smithi006 bash[33918]: cluster 2023-08-28T02:28:41.681368+0000 mgr.y (mgr.14901) 8102 : cluster [DBG] pgmap v8109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:28:42.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:42.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:28:42.874Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:43.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:42 smithi130 bash[27579]: cluster 2023-08-28T02:28:41.681368+0000 mgr.y (mgr.14901) 8102 : cluster [DBG] pgmap v8109: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:43.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:42 smithi130 bash[51302]: debug 2023-08-28T02:28:42.651+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:42.658152+0000) 2023-08-28T02:28:43.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:43 smithi006 bash[27292]: audit 2023-08-28T02:28:42.893475+0000 mon.a (mon.0) 5130 : audit [DBG] from='client.? 172.21.15.6:0/534248199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:43.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:43 smithi006 bash[33918]: audit 2023-08-28T02:28:42.893475+0000 mon.a (mon.0) 5130 : audit [DBG] from='client.? 172.21.15.6:0/534248199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:44.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:43 smithi130 bash[27579]: audit 2023-08-28T02:28:42.893475+0000 mon.a (mon.0) 5130 : audit [DBG] from='client.? 172.21.15.6:0/534248199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:44.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:43 smithi130 bash[51302]: debug 2023-08-28T02:28:43.651+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:43.658425+0000) 2023-08-28T02:28:44.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:44 smithi006 bash[27292]: cluster 2023-08-28T02:28:43.682030+0000 mgr.y (mgr.14901) 8103 : cluster [DBG] pgmap v8110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:44.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:44 smithi006 bash[33918]: cluster 2023-08-28T02:28:43.682030+0000 mgr.y (mgr.14901) 8103 : cluster [DBG] pgmap v8110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:28:44.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:44.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:28:44.758Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:45.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:44 smithi130 bash[27579]: cluster 2023-08-28T02:28:43.682030+0000 mgr.y (mgr.14901) 8103 : cluster [DBG] pgmap v8110: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:45.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:44 smithi130 bash[51302]: debug 2023-08-28T02:28:44.651+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:44.658710+0000) 2023-08-28T02:28:46.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:45 smithi130 bash[27579]: audit 2023-08-28T02:28:45.308289+0000 mon.a (mon.0) 5131 : audit [DBG] from='client.? 172.21.15.6:0/345119377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:46.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:45 smithi130 bash[51302]: debug 2023-08-28T02:28:45.655+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:45.658984+0000) 2023-08-28T02:28:46.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:45 smithi006 bash[27292]: audit 2023-08-28T02:28:45.308289+0000 mon.a (mon.0) 5131 : audit [DBG] from='client.? 172.21.15.6:0/345119377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:46.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:45 smithi006 bash[33918]: audit 2023-08-28T02:28:45.308289+0000 mon.a (mon.0) 5131 : audit [DBG] from='client.? 172.21.15.6:0/345119377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:47.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:46 smithi130 bash[27579]: cluster 2023-08-28T02:28:45.683167+0000 mgr.y (mgr.14901) 8104 : cluster [DBG] pgmap v8111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:47.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:46 smithi130 bash[51302]: debug 2023-08-28T02:28:46.655+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:46.659242+0000) 2023-08-28T02:28:47.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:46 smithi006 bash[27292]: cluster 2023-08-28T02:28:45.683167+0000 mgr.y (mgr.14901) 8104 : cluster [DBG] pgmap v8111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:47.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:46 smithi006 bash[33918]: cluster 2023-08-28T02:28:45.683167+0000 mgr.y (mgr.14901) 8104 : cluster [DBG] pgmap v8111: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:47 smithi130 bash[51302]: debug 2023-08-28T02:28:47.659+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:47.659511+0000) 2023-08-28T02:28:49.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:48 smithi130 bash[51302]: debug 2023-08-28T02:28:48.655+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:48.659813+0000) 2023-08-28T02:28:49.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:48 smithi130 bash[27579]: cluster 2023-08-28T02:28:47.683888+0000 mgr.y (mgr.14901) 8105 : cluster [DBG] pgmap v8112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:49.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:48 smithi130 bash[27579]: audit 2023-08-28T02:28:47.730473+0000 mon.a (mon.0) 5132 : audit [DBG] from='client.? 172.21.15.6:0/3316817663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:49.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:48 smithi006 bash[27292]: cluster 2023-08-28T02:28:47.683888+0000 mgr.y (mgr.14901) 8105 : cluster [DBG] pgmap v8112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:49.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:48 smithi006 bash[27292]: audit 2023-08-28T02:28:47.730473+0000 mon.a (mon.0) 5132 : audit [DBG] from='client.? 172.21.15.6:0/3316817663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:49.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:48 smithi006 bash[33918]: cluster 2023-08-28T02:28:47.683888+0000 mgr.y (mgr.14901) 8105 : cluster [DBG] pgmap v8112: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:49.218 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:48 smithi006 bash[33918]: audit 2023-08-28T02:28:47.730473+0000 mon.a (mon.0) 5132 : audit [DBG] from='client.? 172.21.15.6:0/3316817663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:50.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:49 smithi130 bash[51302]: debug 2023-08-28T02:28:49.655+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:49.660131+0000) 2023-08-28T02:28:51.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:50 smithi130 bash[51302]: debug 2023-08-28T02:28:50.655+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:50.660452+0000) 2023-08-28T02:28:51.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:50 smithi130 bash[27579]: cluster 2023-08-28T02:28:49.685056+0000 mgr.y (mgr.14901) 8106 : cluster [DBG] pgmap v8113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:51.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:50 smithi130 bash[27579]: audit 2023-08-28T02:28:50.138126+0000 mon.c (mon.2) 3211 : audit [DBG] from='client.? 172.21.15.6:0/130888806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:28:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:28:50] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:28:51.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:50 smithi006 bash[27292]: cluster 2023-08-28T02:28:49.685056+0000 mgr.y (mgr.14901) 8106 : cluster [DBG] pgmap v8113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:51.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:50 smithi006 bash[27292]: audit 2023-08-28T02:28:50.138126+0000 mon.c (mon.2) 3211 : audit [DBG] from='client.? 172.21.15.6:0/130888806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:51.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:50 smithi006 bash[33918]: cluster 2023-08-28T02:28:49.685056+0000 mgr.y (mgr.14901) 8106 : cluster [DBG] pgmap v8113: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:51.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:50 smithi006 bash[33918]: audit 2023-08-28T02:28:50.138126+0000 mon.c (mon.2) 3211 : audit [DBG] from='client.? 172.21.15.6:0/130888806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:28:51.312Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:28:51.314Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:52.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:51 smithi130 bash[51302]: debug 2023-08-28T02:28:51.658+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:51.660769+0000) 2023-08-28T02:28:53.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:52 smithi130 bash[27579]: cluster 2023-08-28T02:28:51.685726+0000 mgr.y (mgr.14901) 8107 : cluster [DBG] pgmap v8114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:52 smithi130 bash[27579]: audit 2023-08-28T02:28:52.557930+0000 mon.a (mon.0) 5133 : audit [DBG] from='client.? 172.21.15.6:0/912856270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:53.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:52 smithi130 bash[51302]: debug 2023-08-28T02:28:52.654+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:52.661037+0000) 2023-08-28T02:28:53.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:52 smithi006 bash[27292]: cluster 2023-08-28T02:28:51.685726+0000 mgr.y (mgr.14901) 8107 : cluster [DBG] pgmap v8114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:53.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:52 smithi006 bash[27292]: audit 2023-08-28T02:28:52.557930+0000 mon.a (mon.0) 5133 : audit [DBG] from='client.? 172.21.15.6:0/912856270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:53.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:52 smithi006 bash[33918]: cluster 2023-08-28T02:28:51.685726+0000 mgr.y (mgr.14901) 8107 : cluster [DBG] pgmap v8114: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:53.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:52 smithi006 bash[33918]: audit 2023-08-28T02:28:52.557930+0000 mon.a (mon.0) 5133 : audit [DBG] from='client.? 172.21.15.6:0/912856270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:28:52.874Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:28:52.876Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:54.053 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:53 smithi130 bash[51302]: debug 2023-08-28T02:28:53.658+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:53.661352+0000) 2023-08-28T02:28:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:54 smithi130 bash[27579]: cluster 2023-08-28T02:28:53.686369+0000 mgr.y (mgr.14901) 8108 : cluster [DBG] pgmap v8115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:54 smithi006 bash[27292]: cluster 2023-08-28T02:28:53.686369+0000 mgr.y (mgr.14901) 8108 : cluster [DBG] pgmap v8115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:54 smithi006 bash[33918]: cluster 2023-08-28T02:28:53.686369+0000 mgr.y (mgr.14901) 8108 : cluster [DBG] pgmap v8115: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:55.055 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:28:54.757Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:55.055 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:28:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:28:54.760Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:28:55.056 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:54 smithi130 bash[51302]: debug 2023-08-28T02:28:54.654+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:54.661635+0000) 2023-08-28T02:28:55.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:55 smithi130 bash[27579]: audit 2023-08-28T02:28:54.969208+0000 mon.a (mon.0) 5134 : audit [DBG] from='client.? 172.21.15.6:0/1646040005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:55.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:55 smithi006 bash[33918]: audit 2023-08-28T02:28:54.969208+0000 mon.a (mon.0) 5134 : audit [DBG] from='client.? 172.21.15.6:0/1646040005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:55.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:55 smithi006 bash[27292]: audit 2023-08-28T02:28:54.969208+0000 mon.a (mon.0) 5134 : audit [DBG] from='client.? 172.21.15.6:0/1646040005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:56.065 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:55 smithi130 bash[51302]: debug 2023-08-28T02:28:55.654+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:55.661930+0000) 2023-08-28T02:28:56.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:56 smithi130 bash[27579]: cluster 2023-08-28T02:28:55.687599+0000 mgr.y (mgr.14901) 8109 : cluster [DBG] pgmap v8116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:56.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:56 smithi006 bash[33918]: cluster 2023-08-28T02:28:55.687599+0000 mgr.y (mgr.14901) 8109 : cluster [DBG] pgmap v8116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:56.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:56 smithi006 bash[27292]: cluster 2023-08-28T02:28:55.687599+0000 mgr.y (mgr.14901) 8109 : cluster [DBG] pgmap v8116: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:57.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:56 smithi130 bash[51302]: debug 2023-08-28T02:28:56.658+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:56.662202+0000) 2023-08-28T02:28:57.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:57 smithi006 bash[33918]: audit 2023-08-28T02:28:57.392092+0000 mon.c (mon.2) 3212 : audit [DBG] from='client.? 172.21.15.6:0/1260369574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:57.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:57 smithi006 bash[27292]: audit 2023-08-28T02:28:57.392092+0000 mon.c (mon.2) 3212 : audit [DBG] from='client.? 172.21.15.6:0/1260369574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:57.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:57 smithi130 bash[27579]: audit 2023-08-28T02:28:57.392092+0000 mon.c (mon.2) 3212 : audit [DBG] from='client.? 172.21.15.6:0/1260369574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:57.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:57 smithi130 bash[51302]: debug 2023-08-28T02:28:57.658+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:57.662463+0000) 2023-08-28T02:28:58.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:28:58 smithi006 bash[33918]: cluster 2023-08-28T02:28:57.688422+0000 mgr.y (mgr.14901) 8110 : cluster [DBG] pgmap v8117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:58.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:28:58 smithi006 bash[27292]: cluster 2023-08-28T02:28:57.688422+0000 mgr.y (mgr.14901) 8110 : cluster [DBG] pgmap v8117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:58.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:28:58 smithi130 bash[27579]: cluster 2023-08-28T02:28:57.688422+0000 mgr.y (mgr.14901) 8110 : cluster [DBG] pgmap v8117: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:28:58.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:58 smithi130 bash[51302]: debug 2023-08-28T02:28:58.654+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:58.662714+0000) 2023-08-28T02:29:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:28:59 smithi130 bash[51302]: debug 2023-08-28T02:28:59.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:59.662975+0000) 2023-08-28T02:29:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:00 smithi130 bash[51302]: debug 2023-08-28T02:29:00.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:00.663245+0000) 2023-08-28T02:29:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:00 smithi130 bash[27579]: cluster 2023-08-28T02:28:59.689586+0000 mgr.y (mgr.14901) 8111 : cluster [DBG] pgmap v8118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:00 smithi130 bash[27579]: audit 2023-08-28T02:28:59.804981+0000 mon.a (mon.0) 5135 : audit [DBG] from='client.? 172.21.15.6:0/15924695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:29:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:29:00] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:29:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:00 smithi006 bash[27292]: cluster 2023-08-28T02:28:59.689586+0000 mgr.y (mgr.14901) 8111 : cluster [DBG] pgmap v8118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:00 smithi006 bash[27292]: audit 2023-08-28T02:28:59.804981+0000 mon.a (mon.0) 5135 : audit [DBG] from='client.? 172.21.15.6:0/15924695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:01.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:00 smithi006 bash[33918]: cluster 2023-08-28T02:28:59.689586+0000 mgr.y (mgr.14901) 8111 : cluster [DBG] pgmap v8118: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:01.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:00 smithi006 bash[33918]: audit 2023-08-28T02:28:59.804981+0000 mon.a (mon.0) 5135 : audit [DBG] from='client.? 172.21.15.6:0/15924695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:29:01.314Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:29:01.316Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:02.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:01 smithi130 bash[51302]: debug 2023-08-28T02:29:01.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:01.663529+0000) 2023-08-28T02:29:03.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:02 smithi130 bash[51302]: debug 2023-08-28T02:29:02.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:02.663823+0000) 2023-08-28T02:29:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:02 smithi130 bash[27579]: cluster 2023-08-28T02:29:01.690243+0000 mgr.y (mgr.14901) 8112 : cluster [DBG] pgmap v8119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:02 smithi130 bash[27579]: audit 2023-08-28T02:29:02.223068+0000 mon.a (mon.0) 5136 : audit [DBG] from='client.? 172.21.15.6:0/3285878245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:03.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:02 smithi006 bash[33918]: cluster 2023-08-28T02:29:01.690243+0000 mgr.y (mgr.14901) 8112 : cluster [DBG] pgmap v8119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:03.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:02 smithi006 bash[33918]: audit 2023-08-28T02:29:02.223068+0000 mon.a (mon.0) 5136 : audit [DBG] from='client.? 172.21.15.6:0/3285878245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:03.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:02 smithi006 bash[27292]: cluster 2023-08-28T02:29:01.690243+0000 mgr.y (mgr.14901) 8112 : cluster [DBG] pgmap v8119: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:03.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:02 smithi006 bash[27292]: audit 2023-08-28T02:29:02.223068+0000 mon.a (mon.0) 5136 : audit [DBG] from='client.? 172.21.15.6:0/3285878245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:29:02.875Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:29:02.877Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:04.055 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:03 smithi130 bash[51302]: debug 2023-08-28T02:29:03.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:03.664105+0000) 2023-08-28T02:29:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:04 smithi130 bash[27579]: cluster 2023-08-28T02:29:03.690920+0000 mgr.y (mgr.14901) 8113 : cluster [DBG] pgmap v8120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:04.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:04 smithi006 bash[33918]: cluster 2023-08-28T02:29:03.690920+0000 mgr.y (mgr.14901) 8113 : cluster [DBG] pgmap v8120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:04 smithi006 bash[27292]: cluster 2023-08-28T02:29:03.690920+0000 mgr.y (mgr.14901) 8113 : cluster [DBG] pgmap v8120: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:05.057 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:29:04.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:05.057 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:29:04.760Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:05.058 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:04 smithi130 bash[51302]: debug 2023-08-28T02:29:04.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:04.664429+0000) 2023-08-28T02:29:05.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:05 smithi130 bash[27579]: audit 2023-08-28T02:29:04.639043+0000 mon.c (mon.2) 3213 : audit [DBG] from='client.? 172.21.15.6:0/3298630988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:05.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:05 smithi006 bash[27292]: audit 2023-08-28T02:29:04.639043+0000 mon.c (mon.2) 3213 : audit [DBG] from='client.? 172.21.15.6:0/3298630988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:05.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:05 smithi006 bash[33918]: audit 2023-08-28T02:29:04.639043+0000 mon.c (mon.2) 3213 : audit [DBG] from='client.? 172.21.15.6:0/3298630988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:06.070 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:05 smithi130 bash[51302]: debug 2023-08-28T02:29:05.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:05.664774+0000) 2023-08-28T02:29:06.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:06 smithi130 bash[27579]: cluster 2023-08-28T02:29:05.692210+0000 mgr.y (mgr.14901) 8114 : cluster [DBG] pgmap v8121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:06.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:06 smithi006 bash[33918]: cluster 2023-08-28T02:29:05.692210+0000 mgr.y (mgr.14901) 8114 : cluster [DBG] pgmap v8121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:06.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:06 smithi006 bash[27292]: cluster 2023-08-28T02:29:05.692210+0000 mgr.y (mgr.14901) 8114 : cluster [DBG] pgmap v8121: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:07.072 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:06 smithi130 bash[51302]: debug 2023-08-28T02:29:06.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:06.665067+0000) 2023-08-28T02:29:07.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:07 smithi130 bash[27579]: audit 2023-08-28T02:29:07.053755+0000 mon.a (mon.0) 5137 : audit [DBG] from='client.? 172.21.15.6:0/2666973154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:07.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:07 smithi006 bash[33918]: audit 2023-08-28T02:29:07.053755+0000 mon.a (mon.0) 5137 : audit [DBG] from='client.? 172.21.15.6:0/2666973154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:07 smithi006 bash[27292]: audit 2023-08-28T02:29:07.053755+0000 mon.a (mon.0) 5137 : audit [DBG] from='client.? 172.21.15.6:0/2666973154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:08.080 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:07 smithi130 bash[51302]: debug 2023-08-28T02:29:07.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:07.665357+0000) 2023-08-28T02:29:08.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:08 smithi130 bash[27579]: audit 2023-08-28T02:29:07.109264+0000 mon.b (mon.1) 885 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:29:08.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:08 smithi130 bash[27579]: cluster 2023-08-28T02:29:07.692953+0000 mgr.y (mgr.14901) 8115 : cluster [DBG] pgmap v8122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:08.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:08 smithi006 bash[33918]: audit 2023-08-28T02:29:07.109264+0000 mon.b (mon.1) 885 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:29:08.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:08 smithi006 bash[33918]: cluster 2023-08-28T02:29:07.692953+0000 mgr.y (mgr.14901) 8115 : cluster [DBG] pgmap v8122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:08.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:08 smithi006 bash[27292]: audit 2023-08-28T02:29:07.109264+0000 mon.b (mon.1) 885 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:29:08.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:08 smithi006 bash[27292]: cluster 2023-08-28T02:29:07.692953+0000 mgr.y (mgr.14901) 8115 : cluster [DBG] pgmap v8122: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:09.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:08 smithi130 bash[51302]: debug 2023-08-28T02:29:08.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:08.665658+0000) 2023-08-28T02:29:09.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:09 smithi130 bash[27579]: audit 2023-08-28T02:29:09.481300+0000 mon.c (mon.2) 3214 : audit [DBG] from='client.? 172.21.15.6:0/2381790466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:09.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:09 smithi130 bash[51302]: debug 2023-08-28T02:29:09.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:09.665941+0000) 2023-08-28T02:29:09.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:09 smithi006 bash[33918]: audit 2023-08-28T02:29:09.481300+0000 mon.c (mon.2) 3214 : audit [DBG] from='client.? 172.21.15.6:0/2381790466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:09.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:09 smithi006 bash[27292]: audit 2023-08-28T02:29:09.481300+0000 mon.c (mon.2) 3214 : audit [DBG] from='client.? 172.21.15.6:0/2381790466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:10.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:10 smithi130 bash[27579]: cluster 2023-08-28T02:29:09.694053+0000 mgr.y (mgr.14901) 8116 : cluster [DBG] pgmap v8123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:10.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:10 smithi130 bash[51302]: debug 2023-08-28T02:29:10.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:10.666230+0000) 2023-08-28T02:29:10.965 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:29:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:29:10] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:29:10.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:10 smithi006 bash[33918]: cluster 2023-08-28T02:29:09.694053+0000 mgr.y (mgr.14901) 8116 : cluster [DBG] pgmap v8123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:10.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:10 smithi006 bash[27292]: cluster 2023-08-28T02:29:09.694053+0000 mgr.y (mgr.14901) 8116 : cluster [DBG] pgmap v8123: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:29:11.314Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:29:11.316Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:12.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:11 smithi130 bash[51302]: debug 2023-08-28T02:29:11.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:11.666527+0000) 2023-08-28T02:29:13.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:12 smithi130 bash[27579]: cluster 2023-08-28T02:29:11.694693+0000 mgr.y (mgr.14901) 8117 : cluster [DBG] pgmap v8124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:13.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:12 smithi130 bash[27579]: audit 2023-08-28T02:29:11.904747+0000 mon.c (mon.2) 3215 : audit [DBG] from='client.? 172.21.15.6:0/991217787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:13.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:12 smithi130 bash[51302]: debug 2023-08-28T02:29:12.662+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:12.666815+0000) 2023-08-28T02:29:13.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:12 smithi006 bash[27292]: cluster 2023-08-28T02:29:11.694693+0000 mgr.y (mgr.14901) 8117 : cluster [DBG] pgmap v8124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:13.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:12 smithi006 bash[27292]: audit 2023-08-28T02:29:11.904747+0000 mon.c (mon.2) 3215 : audit [DBG] from='client.? 172.21.15.6:0/991217787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:12 smithi006 bash[33918]: cluster 2023-08-28T02:29:11.694693+0000 mgr.y (mgr.14901) 8117 : cluster [DBG] pgmap v8124: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:12 smithi006 bash[33918]: audit 2023-08-28T02:29:11.904747+0000 mon.c (mon.2) 3215 : audit [DBG] from='client.? 172.21.15.6:0/991217787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:29:12.875Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:29:12.877Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:14.016 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:13 smithi130 bash[51302]: debug 2023-08-28T02:29:13.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:13.667100+0000) 2023-08-28T02:29:14.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:14 smithi130 bash[27579]: audit 2023-08-28T02:29:13.016509+0000 mon.a (mon.0) 5138 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:29:14.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:14 smithi130 bash[27579]: audit 2023-08-28T02:29:13.389527+0000 mon.a (mon.0) 5139 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:29:14.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:14 smithi130 bash[27579]: audit 2023-08-28T02:29:13.681717+0000 mon.b (mon.1) 886 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:29:14.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:14 smithi130 bash[27579]: audit 2023-08-28T02:29:13.683763+0000 mon.b (mon.1) 887 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:29:14.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:14 smithi130 bash[27579]: audit 2023-08-28T02:29:13.691565+0000 mon.a (mon.0) 5140 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:29:14.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:14 smithi130 bash[27579]: cluster 2023-08-28T02:29:13.695262+0000 mgr.y (mgr.14901) 8118 : cluster [DBG] pgmap v8125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:14 smithi006 bash[27292]: audit 2023-08-28T02:29:13.016509+0000 mon.a (mon.0) 5138 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:29:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:14 smithi006 bash[27292]: audit 2023-08-28T02:29:13.389527+0000 mon.a (mon.0) 5139 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:29:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:14 smithi006 bash[27292]: audit 2023-08-28T02:29:13.681717+0000 mon.b (mon.1) 886 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:29:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:14 smithi006 bash[27292]: audit 2023-08-28T02:29:13.683763+0000 mon.b (mon.1) 887 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:29:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:14 smithi006 bash[27292]: audit 2023-08-28T02:29:13.691565+0000 mon.a (mon.0) 5140 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:29:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:14 smithi006 bash[27292]: cluster 2023-08-28T02:29:13.695262+0000 mgr.y (mgr.14901) 8118 : cluster [DBG] pgmap v8125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:14 smithi006 bash[33918]: audit 2023-08-28T02:29:13.016509+0000 mon.a (mon.0) 5138 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:29:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:14 smithi006 bash[33918]: audit 2023-08-28T02:29:13.389527+0000 mon.a (mon.0) 5139 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:29:14.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:14 smithi006 bash[33918]: audit 2023-08-28T02:29:13.681717+0000 mon.b (mon.1) 886 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:29:14.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:14 smithi006 bash[33918]: audit 2023-08-28T02:29:13.683763+0000 mon.b (mon.1) 887 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:29:14.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:14 smithi006 bash[33918]: audit 2023-08-28T02:29:13.691565+0000 mon.a (mon.0) 5140 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:29:14.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:14 smithi006 bash[33918]: cluster 2023-08-28T02:29:13.695262+0000 mgr.y (mgr.14901) 8118 : cluster [DBG] pgmap v8125: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:15.018 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:29:14.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:15.018 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:29:14.760Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:15.019 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:14 smithi130 bash[51302]: debug 2023-08-28T02:29:14.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:14.667548+0000) 2023-08-28T02:29:15.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:15 smithi130 bash[27579]: audit 2023-08-28T02:29:14.324739+0000 mon.a (mon.0) 5141 : audit [DBG] from='client.? 172.21.15.6:0/77531172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:15.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:15 smithi006 bash[27292]: audit 2023-08-28T02:29:14.324739+0000 mon.a (mon.0) 5141 : audit [DBG] from='client.? 172.21.15.6:0/77531172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:15.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:15 smithi006 bash[33918]: audit 2023-08-28T02:29:14.324739+0000 mon.a (mon.0) 5141 : audit [DBG] from='client.? 172.21.15.6:0/77531172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:16.027 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:15 smithi130 bash[51302]: debug 2023-08-28T02:29:15.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:15.667837+0000) 2023-08-28T02:29:16.367 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:16 smithi130 bash[27579]: cluster 2023-08-28T02:29:15.696376+0000 mgr.y (mgr.14901) 8119 : cluster [DBG] pgmap v8126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:16.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:16 smithi006 bash[27292]: cluster 2023-08-28T02:29:15.696376+0000 mgr.y (mgr.14901) 8119 : cluster [DBG] pgmap v8126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:16.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:16 smithi006 bash[33918]: cluster 2023-08-28T02:29:15.696376+0000 mgr.y (mgr.14901) 8119 : cluster [DBG] pgmap v8126: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:17.030 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:16 smithi130 bash[51302]: debug 2023-08-28T02:29:16.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:16.668122+0000) 2023-08-28T02:29:17.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:17 smithi130 bash[27579]: audit 2023-08-28T02:29:16.745517+0000 mon.c (mon.2) 3216 : audit [DBG] from='client.? 172.21.15.6:0/3375895640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:17.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:17 smithi006 bash[27292]: audit 2023-08-28T02:29:16.745517+0000 mon.c (mon.2) 3216 : audit [DBG] from='client.? 172.21.15.6:0/3375895640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:17.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:17 smithi006 bash[33918]: audit 2023-08-28T02:29:16.745517+0000 mon.c (mon.2) 3216 : audit [DBG] from='client.? 172.21.15.6:0/3375895640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:18.037 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:17 smithi130 bash[51302]: debug 2023-08-28T02:29:17.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:17.668383+0000) 2023-08-28T02:29:18.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:18 smithi130 bash[27579]: cluster 2023-08-28T02:29:17.697093+0000 mgr.y (mgr.14901) 8120 : cluster [DBG] pgmap v8127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:18.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:18 smithi006 bash[27292]: cluster 2023-08-28T02:29:17.697093+0000 mgr.y (mgr.14901) 8120 : cluster [DBG] pgmap v8127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:18.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:18 smithi006 bash[33918]: cluster 2023-08-28T02:29:17.697093+0000 mgr.y (mgr.14901) 8120 : cluster [DBG] pgmap v8127: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:19.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:18 smithi130 bash[51302]: debug 2023-08-28T02:29:18.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:18.668657+0000) 2023-08-28T02:29:19.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:19 smithi130 bash[27579]: audit 2023-08-28T02:29:19.164959+0000 mon.c (mon.2) 3217 : audit [DBG] from='client.? 172.21.15.6:0/2914589936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:19.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:19 smithi006 bash[33918]: audit 2023-08-28T02:29:19.164959+0000 mon.c (mon.2) 3217 : audit [DBG] from='client.? 172.21.15.6:0/2914589936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:19.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:19 smithi006 bash[27292]: audit 2023-08-28T02:29:19.164959+0000 mon.c (mon.2) 3217 : audit [DBG] from='client.? 172.21.15.6:0/2914589936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:19 smithi130 bash[51302]: debug 2023-08-28T02:29:19.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:19.669005+0000) 2023-08-28T02:29:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:20 smithi130 bash[27579]: cluster 2023-08-28T02:29:19.698193+0000 mgr.y (mgr.14901) 8121 : cluster [DBG] pgmap v8128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:20.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:20 smithi006 bash[33918]: cluster 2023-08-28T02:29:19.698193+0000 mgr.y (mgr.14901) 8121 : cluster [DBG] pgmap v8128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:20.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:20 smithi006 bash[27292]: cluster 2023-08-28T02:29:19.698193+0000 mgr.y (mgr.14901) 8121 : cluster [DBG] pgmap v8128: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:21.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:20 smithi130 bash[51302]: debug 2023-08-28T02:29:20.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:20.669285+0000) 2023-08-28T02:29:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:29:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:29:20] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T02:29:21.636 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:29:21.315Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:21.636 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:29:21.316Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:21.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:21 smithi006 bash[33918]: audit 2023-08-28T02:29:21.582347+0000 mon.c (mon.2) 3218 : audit [DBG] from='client.? 172.21.15.6:0/1262166948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:21.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:21 smithi006 bash[27292]: audit 2023-08-28T02:29:21.582347+0000 mon.c (mon.2) 3218 : audit [DBG] from='client.? 172.21.15.6:0/1262166948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:22.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:21 smithi130 bash[27579]: audit 2023-08-28T02:29:21.582347+0000 mon.c (mon.2) 3218 : audit [DBG] from='client.? 172.21.15.6:0/1262166948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:22.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:21 smithi130 bash[51302]: debug 2023-08-28T02:29:21.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:21.669557+0000) 2023-08-28T02:29:22.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:22 smithi006 bash[27292]: cluster 2023-08-28T02:29:21.698858+0000 mgr.y (mgr.14901) 8122 : cluster [DBG] pgmap v8129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:22.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:22 smithi006 bash[33918]: cluster 2023-08-28T02:29:21.698858+0000 mgr.y (mgr.14901) 8122 : cluster [DBG] pgmap v8129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:29:22.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:22.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:29:22.878Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:23.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:22 smithi130 bash[27579]: cluster 2023-08-28T02:29:21.698858+0000 mgr.y (mgr.14901) 8122 : cluster [DBG] pgmap v8129: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:23.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:22 smithi130 bash[51302]: debug 2023-08-28T02:29:22.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:22.669846+0000) 2023-08-28T02:29:24.060 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:23 smithi130 bash[51302]: debug 2023-08-28T02:29:23.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:23.670132+0000) 2023-08-28T02:29:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:24 smithi130 bash[27579]: cluster 2023-08-28T02:29:23.699350+0000 mgr.y (mgr.14901) 8123 : cluster [DBG] pgmap v8130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:24 smithi130 bash[27579]: audit 2023-08-28T02:29:23.987866+0000 mon.a (mon.0) 5142 : audit [DBG] from='client.? 172.21.15.6:0/927471779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:24.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:24 smithi006 bash[33918]: cluster 2023-08-28T02:29:23.699350+0000 mgr.y (mgr.14901) 8123 : cluster [DBG] pgmap v8130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:24.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:24 smithi006 bash[33918]: audit 2023-08-28T02:29:23.987866+0000 mon.a (mon.0) 5142 : audit [DBG] from='client.? 172.21.15.6:0/927471779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:24 smithi006 bash[27292]: cluster 2023-08-28T02:29:23.699350+0000 mgr.y (mgr.14901) 8123 : cluster [DBG] pgmap v8130: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:24 smithi006 bash[27292]: audit 2023-08-28T02:29:23.987866+0000 mon.a (mon.0) 5142 : audit [DBG] from='client.? 172.21.15.6:0/927471779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:25.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:24 smithi130 bash[51302]: debug 2023-08-28T02:29:24.666+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:24.670435+0000) 2023-08-28T02:29:25.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:29:24.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:25.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:29:24.761Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:26.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:25 smithi130 bash[51302]: debug 2023-08-28T02:29:25.670+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:25.670724+0000) 2023-08-28T02:29:27.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:26 smithi130 bash[51302]: debug 2023-08-28T02:29:26.670+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:26.671046+0000) 2023-08-28T02:29:27.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:26 smithi130 bash[27579]: cluster 2023-08-28T02:29:25.700381+0000 mgr.y (mgr.14901) 8124 : cluster [DBG] pgmap v8131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:27.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:26 smithi130 bash[27579]: audit 2023-08-28T02:29:26.394857+0000 mon.c (mon.2) 3219 : audit [DBG] from='client.? 172.21.15.6:0/4139531446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:27.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:26 smithi006 bash[33918]: cluster 2023-08-28T02:29:25.700381+0000 mgr.y (mgr.14901) 8124 : cluster [DBG] pgmap v8131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:27.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:26 smithi006 bash[33918]: audit 2023-08-28T02:29:26.394857+0000 mon.c (mon.2) 3219 : audit [DBG] from='client.? 172.21.15.6:0/4139531446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:27.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:26 smithi006 bash[27292]: cluster 2023-08-28T02:29:25.700381+0000 mgr.y (mgr.14901) 8124 : cluster [DBG] pgmap v8131: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:27.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:26 smithi006 bash[27292]: audit 2023-08-28T02:29:26.394857+0000 mon.c (mon.2) 3219 : audit [DBG] from='client.? 172.21.15.6:0/4139531446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:28.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:27 smithi130 bash[51302]: debug 2023-08-28T02:29:27.670+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:27.671300+0000) 2023-08-28T02:29:29.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:28 smithi130 bash[27579]: cluster 2023-08-28T02:29:27.701203+0000 mgr.y (mgr.14901) 8125 : cluster [DBG] pgmap v8132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:29.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:28 smithi130 bash[51302]: debug 2023-08-28T02:29:28.670+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:28.671580+0000) 2023-08-28T02:29:29.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:28 smithi006 bash[27292]: cluster 2023-08-28T02:29:27.701203+0000 mgr.y (mgr.14901) 8125 : cluster [DBG] pgmap v8132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:29.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:28 smithi006 bash[33918]: cluster 2023-08-28T02:29:27.701203+0000 mgr.y (mgr.14901) 8125 : cluster [DBG] pgmap v8132: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:30.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:29 smithi130 bash[27579]: audit 2023-08-28T02:29:28.818036+0000 mon.a (mon.0) 5143 : audit [DBG] from='client.? 172.21.15.6:0/2836189536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:30.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:29 smithi130 bash[51302]: debug 2023-08-28T02:29:29.670+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:29.671846+0000) 2023-08-28T02:29:30.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:29 smithi006 bash[33918]: audit 2023-08-28T02:29:28.818036+0000 mon.a (mon.0) 5143 : audit [DBG] from='client.? 172.21.15.6:0/2836189536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:30.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:29 smithi006 bash[27292]: audit 2023-08-28T02:29:28.818036+0000 mon.a (mon.0) 5143 : audit [DBG] from='client.? 172.21.15.6:0/2836189536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:31.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:30 smithi130 bash[27579]: cluster 2023-08-28T02:29:29.702415+0000 mgr.y (mgr.14901) 8126 : cluster [DBG] pgmap v8133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:31.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:30 smithi130 bash[51302]: debug 2023-08-28T02:29:30.670+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:30.672130+0000) 2023-08-28T02:29:31.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:29:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:29:30] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:29:31.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:30 smithi006 bash[33918]: cluster 2023-08-28T02:29:29.702415+0000 mgr.y (mgr.14901) 8126 : cluster [DBG] pgmap v8133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:31.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:30 smithi006 bash[27292]: cluster 2023-08-28T02:29:29.702415+0000 mgr.y (mgr.14901) 8126 : cluster [DBG] pgmap v8133: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:29:31.315Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:29:31.318Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:32.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:31 smithi130 bash[27579]: audit 2023-08-28T02:29:31.232882+0000 mon.c (mon.2) 3220 : audit [DBG] from='client.? 172.21.15.6:0/2564772529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:32.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:31 smithi130 bash[51302]: debug 2023-08-28T02:29:31.670+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:31.672436+0000) 2023-08-28T02:29:32.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:31 smithi006 bash[27292]: audit 2023-08-28T02:29:31.232882+0000 mon.c (mon.2) 3220 : audit [DBG] from='client.? 172.21.15.6:0/2564772529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:32.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:31 smithi006 bash[33918]: audit 2023-08-28T02:29:31.232882+0000 mon.c (mon.2) 3220 : audit [DBG] from='client.? 172.21.15.6:0/2564772529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:33.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:32 smithi130 bash[27579]: cluster 2023-08-28T02:29:31.703213+0000 mgr.y (mgr.14901) 8127 : cluster [DBG] pgmap v8134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:33.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:32 smithi130 bash[51302]: debug 2023-08-28T02:29:32.670+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:32.672699+0000) 2023-08-28T02:29:33.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:32 smithi006 bash[33918]: cluster 2023-08-28T02:29:31.703213+0000 mgr.y (mgr.14901) 8127 : cluster [DBG] pgmap v8134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:29:32.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:29:32.878Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:33.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:32 smithi006 bash[27292]: cluster 2023-08-28T02:29:31.703213+0000 mgr.y (mgr.14901) 8127 : cluster [DBG] pgmap v8134: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:34.064 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:33 smithi130 bash[51302]: debug 2023-08-28T02:29:33.670+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:33.673044+0000) 2023-08-28T02:29:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:34 smithi130 bash[27579]: audit 2023-08-28T02:29:33.650152+0000 mon.a (mon.0) 5144 : audit [DBG] from='client.? 172.21.15.6:0/3703392289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:34 smithi130 bash[27579]: cluster 2023-08-28T02:29:33.704041+0000 mgr.y (mgr.14901) 8128 : cluster [DBG] pgmap v8135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:34.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:34 smithi006 bash[27292]: audit 2023-08-28T02:29:33.650152+0000 mon.a (mon.0) 5144 : audit [DBG] from='client.? 172.21.15.6:0/3703392289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:34.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:34 smithi006 bash[27292]: cluster 2023-08-28T02:29:33.704041+0000 mgr.y (mgr.14901) 8128 : cluster [DBG] pgmap v8135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:34 smithi006 bash[33918]: audit 2023-08-28T02:29:33.650152+0000 mon.a (mon.0) 5144 : audit [DBG] from='client.? 172.21.15.6:0/3703392289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:34 smithi006 bash[33918]: cluster 2023-08-28T02:29:33.704041+0000 mgr.y (mgr.14901) 8128 : cluster [DBG] pgmap v8135: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:35.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:34 smithi130 bash[51302]: debug 2023-08-28T02:29:34.670+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:34.673334+0000) 2023-08-28T02:29:35.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:29:34.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:35.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:29:34.762Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:36.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:35 smithi130 bash[51302]: debug 2023-08-28T02:29:35.670+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:35.673635+0000) 2023-08-28T02:29:37.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:36 smithi130 bash[27579]: cluster 2023-08-28T02:29:35.705389+0000 mgr.y (mgr.14901) 8129 : cluster [DBG] pgmap v8136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:37.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:36 smithi130 bash[27579]: audit 2023-08-28T02:29:36.065920+0000 mon.c (mon.2) 3221 : audit [DBG] from='client.? 172.21.15.6:0/2442187571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:37.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:36 smithi130 bash[51302]: debug 2023-08-28T02:29:36.670+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:36.673905+0000) 2023-08-28T02:29:37.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:36 smithi006 bash[27292]: cluster 2023-08-28T02:29:35.705389+0000 mgr.y (mgr.14901) 8129 : cluster [DBG] pgmap v8136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:37.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:36 smithi006 bash[27292]: audit 2023-08-28T02:29:36.065920+0000 mon.c (mon.2) 3221 : audit [DBG] from='client.? 172.21.15.6:0/2442187571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:37.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:36 smithi006 bash[33918]: cluster 2023-08-28T02:29:35.705389+0000 mgr.y (mgr.14901) 8129 : cluster [DBG] pgmap v8136: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:37.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:36 smithi006 bash[33918]: audit 2023-08-28T02:29:36.065920+0000 mon.c (mon.2) 3221 : audit [DBG] from='client.? 172.21.15.6:0/2442187571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:37 smithi130 bash[51302]: debug 2023-08-28T02:29:37.670+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:37.674172+0000) 2023-08-28T02:29:39.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:38 smithi130 bash[27579]: cluster 2023-08-28T02:29:37.706223+0000 mgr.y (mgr.14901) 8130 : cluster [DBG] pgmap v8137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:39.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:38 smithi130 bash[27579]: audit 2023-08-28T02:29:38.481187+0000 mon.c (mon.2) 3222 : audit [DBG] from='client.? 172.21.15.6:0/147284051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:39.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:38 smithi130 bash[51302]: debug 2023-08-28T02:29:38.670+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:38.674519+0000) 2023-08-28T02:29:39.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:38 smithi006 bash[27292]: cluster 2023-08-28T02:29:37.706223+0000 mgr.y (mgr.14901) 8130 : cluster [DBG] pgmap v8137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:39.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:38 smithi006 bash[27292]: audit 2023-08-28T02:29:38.481187+0000 mon.c (mon.2) 3222 : audit [DBG] from='client.? 172.21.15.6:0/147284051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:39.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:38 smithi006 bash[33918]: cluster 2023-08-28T02:29:37.706223+0000 mgr.y (mgr.14901) 8130 : cluster [DBG] pgmap v8137: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:39.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:38 smithi006 bash[33918]: audit 2023-08-28T02:29:38.481187+0000 mon.c (mon.2) 3222 : audit [DBG] from='client.? 172.21.15.6:0/147284051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:40.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:39 smithi130 bash[51302]: debug 2023-08-28T02:29:39.674+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:39.674821+0000) 2023-08-28T02:29:41.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:40 smithi130 bash[27579]: cluster 2023-08-28T02:29:39.707519+0000 mgr.y (mgr.14901) 8131 : cluster [DBG] pgmap v8138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:41.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:40 smithi130 bash[51302]: debug 2023-08-28T02:29:40.674+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:40.675172+0000) 2023-08-28T02:29:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:29:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:29:40] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:29:41.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:40 smithi006 bash[33918]: cluster 2023-08-28T02:29:39.707519+0000 mgr.y (mgr.14901) 8131 : cluster [DBG] pgmap v8138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:41.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:40 smithi006 bash[27292]: cluster 2023-08-28T02:29:39.707519+0000 mgr.y (mgr.14901) 8131 : cluster [DBG] pgmap v8138: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:29:41.316Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:29:41.318Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:42.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:41 smithi130 bash[27579]: audit 2023-08-28T02:29:40.898048+0000 mon.c (mon.2) 3223 : audit [DBG] from='client.? 172.21.15.6:0/847624894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:42.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:41 smithi130 bash[51302]: debug 2023-08-28T02:29:41.674+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:41.675412+0000) 2023-08-28T02:29:42.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:41 smithi006 bash[27292]: audit 2023-08-28T02:29:40.898048+0000 mon.c (mon.2) 3223 : audit [DBG] from='client.? 172.21.15.6:0/847624894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:42.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:41 smithi006 bash[33918]: audit 2023-08-28T02:29:40.898048+0000 mon.c (mon.2) 3223 : audit [DBG] from='client.? 172.21.15.6:0/847624894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:43.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:42 smithi130 bash[51302]: debug 2023-08-28T02:29:42.674+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:42.675656+0000) 2023-08-28T02:29:43.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:42 smithi130 bash[27579]: cluster 2023-08-28T02:29:41.708351+0000 mgr.y (mgr.14901) 8132 : cluster [DBG] pgmap v8139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:43.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:42 smithi006 bash[33918]: cluster 2023-08-28T02:29:41.708351+0000 mgr.y (mgr.14901) 8132 : cluster [DBG] pgmap v8139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:43.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:42 smithi006 bash[27292]: cluster 2023-08-28T02:29:41.708351+0000 mgr.y (mgr.14901) 8132 : cluster [DBG] pgmap v8139: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:29:42.878Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:29:42.880Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:44.078 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:43 smithi130 bash[51302]: debug 2023-08-28T02:29:43.674+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:43.675956+0000) 2023-08-28T02:29:44.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:44 smithi130 bash[27579]: audit 2023-08-28T02:29:43.315230+0000 mon.c (mon.2) 3224 : audit [DBG] from='client.? 172.21.15.6:0/3595384377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:44.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:44 smithi130 bash[27579]: cluster 2023-08-28T02:29:43.709082+0000 mgr.y (mgr.14901) 8133 : cluster [DBG] pgmap v8140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:44.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:44 smithi006 bash[27292]: audit 2023-08-28T02:29:43.315230+0000 mon.c (mon.2) 3224 : audit [DBG] from='client.? 172.21.15.6:0/3595384377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:44.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:44 smithi006 bash[27292]: cluster 2023-08-28T02:29:43.709082+0000 mgr.y (mgr.14901) 8133 : cluster [DBG] pgmap v8140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:44 smithi006 bash[33918]: audit 2023-08-28T02:29:43.315230+0000 mon.c (mon.2) 3224 : audit [DBG] from='client.? 172.21.15.6:0/3595384377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:44 smithi006 bash[33918]: cluster 2023-08-28T02:29:43.709082+0000 mgr.y (mgr.14901) 8133 : cluster [DBG] pgmap v8140: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:45.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:44 smithi130 bash[51302]: debug 2023-08-28T02:29:44.674+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:44.676228+0000) 2023-08-28T02:29:45.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:29:44.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:45.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:29:44.762Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:46.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:45 smithi130 bash[27579]: audit 2023-08-28T02:29:45.728908+0000 mon.c (mon.2) 3225 : audit [DBG] from='client.? 172.21.15.6:0/1245056430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:46.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:45 smithi130 bash[51302]: debug 2023-08-28T02:29:45.673+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:45.676476+0000) 2023-08-28T02:29:46.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:45 smithi006 bash[27292]: audit 2023-08-28T02:29:45.728908+0000 mon.c (mon.2) 3225 : audit [DBG] from='client.? 172.21.15.6:0/1245056430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:46.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:45 smithi006 bash[33918]: audit 2023-08-28T02:29:45.728908+0000 mon.c (mon.2) 3225 : audit [DBG] from='client.? 172.21.15.6:0/1245056430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:47.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:46 smithi130 bash[27579]: cluster 2023-08-28T02:29:45.710516+0000 mgr.y (mgr.14901) 8134 : cluster [DBG] pgmap v8141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:47.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:46 smithi130 bash[51302]: debug 2023-08-28T02:29:46.673+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:46.676819+0000) 2023-08-28T02:29:47.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:46 smithi006 bash[27292]: cluster 2023-08-28T02:29:45.710516+0000 mgr.y (mgr.14901) 8134 : cluster [DBG] pgmap v8141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:47.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:46 smithi006 bash[33918]: cluster 2023-08-28T02:29:45.710516+0000 mgr.y (mgr.14901) 8134 : cluster [DBG] pgmap v8141: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:47 smithi130 bash[51302]: debug 2023-08-28T02:29:47.673+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:47.677078+0000) 2023-08-28T02:29:49.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:48 smithi130 bash[27579]: cluster 2023-08-28T02:29:47.711230+0000 mgr.y (mgr.14901) 8135 : cluster [DBG] pgmap v8142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:49.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:48 smithi130 bash[27579]: audit 2023-08-28T02:29:48.142187+0000 mon.a (mon.0) 5145 : audit [DBG] from='client.? 172.21.15.6:0/3099736813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:49.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:48 smithi130 bash[51302]: debug 2023-08-28T02:29:48.673+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:48.677335+0000) 2023-08-28T02:29:49.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:48 smithi006 bash[33918]: cluster 2023-08-28T02:29:47.711230+0000 mgr.y (mgr.14901) 8135 : cluster [DBG] pgmap v8142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:49.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:48 smithi006 bash[33918]: audit 2023-08-28T02:29:48.142187+0000 mon.a (mon.0) 5145 : audit [DBG] from='client.? 172.21.15.6:0/3099736813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:49.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:48 smithi006 bash[27292]: cluster 2023-08-28T02:29:47.711230+0000 mgr.y (mgr.14901) 8135 : cluster [DBG] pgmap v8142: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:49.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:48 smithi006 bash[27292]: audit 2023-08-28T02:29:48.142187+0000 mon.a (mon.0) 5145 : audit [DBG] from='client.? 172.21.15.6:0/3099736813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:50.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:49 smithi130 bash[51302]: debug 2023-08-28T02:29:49.673+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:49.677592+0000) 2023-08-28T02:29:51.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:50 smithi130 bash[51302]: debug 2023-08-28T02:29:50.673+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:50.677884+0000) 2023-08-28T02:29:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:50 smithi130 bash[27579]: cluster 2023-08-28T02:29:49.712352+0000 mgr.y (mgr.14901) 8136 : cluster [DBG] pgmap v8143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:51.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:50 smithi130 bash[27579]: audit 2023-08-28T02:29:50.567896+0000 mon.c (mon.2) 3226 : audit [DBG] from='client.? 172.21.15.6:0/203962491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:51.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:50 smithi006 bash[27292]: cluster 2023-08-28T02:29:49.712352+0000 mgr.y (mgr.14901) 8136 : cluster [DBG] pgmap v8143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:51.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:50 smithi006 bash[27292]: audit 2023-08-28T02:29:50.567896+0000 mon.c (mon.2) 3226 : audit [DBG] from='client.? 172.21.15.6:0/203962491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:51.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:29:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:29:50] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T02:29:51.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:50 smithi006 bash[33918]: cluster 2023-08-28T02:29:49.712352+0000 mgr.y (mgr.14901) 8136 : cluster [DBG] pgmap v8143: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:51.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:50 smithi006 bash[33918]: audit 2023-08-28T02:29:50.567896+0000 mon.c (mon.2) 3226 : audit [DBG] from='client.? 172.21.15.6:0/203962491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:29:51.317Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:29:51.318Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:52.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:51 smithi130 bash[51302]: debug 2023-08-28T02:29:51.673+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:51.678199+0000) 2023-08-28T02:29:53.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:52 smithi130 bash[51302]: debug 2023-08-28T02:29:52.673+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:52.678509+0000) 2023-08-28T02:29:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:52 smithi130 bash[27579]: cluster 2023-08-28T02:29:51.713059+0000 mgr.y (mgr.14901) 8137 : cluster [DBG] pgmap v8144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:53.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:52 smithi006 bash[27292]: cluster 2023-08-28T02:29:51.713059+0000 mgr.y (mgr.14901) 8137 : cluster [DBG] pgmap v8144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:53.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:52 smithi006 bash[33918]: cluster 2023-08-28T02:29:51.713059+0000 mgr.y (mgr.14901) 8137 : cluster [DBG] pgmap v8144: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:29:52.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:29:52.881Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:54.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:53 smithi130 bash[27579]: audit 2023-08-28T02:29:52.983807+0000 mon.a (mon.0) 5146 : audit [DBG] from='client.? 172.21.15.6:0/3622621533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:54.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:53 smithi130 bash[51302]: debug 2023-08-28T02:29:53.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:53.678773+0000) 2023-08-28T02:29:54.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:53 smithi006 bash[33918]: audit 2023-08-28T02:29:52.983807+0000 mon.a (mon.0) 5146 : audit [DBG] from='client.? 172.21.15.6:0/3622621533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:54.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:53 smithi006 bash[27292]: audit 2023-08-28T02:29:52.983807+0000 mon.a (mon.0) 5146 : audit [DBG] from='client.? 172.21.15.6:0/3622621533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:55.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:54 smithi130 bash[27579]: cluster 2023-08-28T02:29:53.713757+0000 mgr.y (mgr.14901) 8138 : cluster [DBG] pgmap v8145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:55.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:54 smithi130 bash[51302]: debug 2023-08-28T02:29:54.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:54.679054+0000) 2023-08-28T02:29:55.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:54 smithi006 bash[33918]: cluster 2023-08-28T02:29:53.713757+0000 mgr.y (mgr.14901) 8138 : cluster [DBG] pgmap v8145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:55.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:54 smithi006 bash[27292]: cluster 2023-08-28T02:29:53.713757+0000 mgr.y (mgr.14901) 8138 : cluster [DBG] pgmap v8145: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:55.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:29:54.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:55.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:29:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:29:54.763Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:29:56.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:55 smithi130 bash[27579]: audit 2023-08-28T02:29:55.406805+0000 mon.c (mon.2) 3227 : audit [DBG] from='client.? 172.21.15.6:0/1035773903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:56.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:55 smithi130 bash[51302]: debug 2023-08-28T02:29:55.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:55.679265+0000) 2023-08-28T02:29:56.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:55 smithi006 bash[33918]: audit 2023-08-28T02:29:55.406805+0000 mon.c (mon.2) 3227 : audit [DBG] from='client.? 172.21.15.6:0/1035773903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:56.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:55 smithi006 bash[27292]: audit 2023-08-28T02:29:55.406805+0000 mon.c (mon.2) 3227 : audit [DBG] from='client.? 172.21.15.6:0/1035773903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:57.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:56 smithi130 bash[27579]: cluster 2023-08-28T02:29:55.714919+0000 mgr.y (mgr.14901) 8139 : cluster [DBG] pgmap v8146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:57.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:56 smithi130 bash[51302]: debug 2023-08-28T02:29:56.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:56.679489+0000) 2023-08-28T02:29:57.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:56 smithi006 bash[27292]: cluster 2023-08-28T02:29:55.714919+0000 mgr.y (mgr.14901) 8139 : cluster [DBG] pgmap v8146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:57.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:56 smithi006 bash[33918]: cluster 2023-08-28T02:29:55.714919+0000 mgr.y (mgr.14901) 8139 : cluster [DBG] pgmap v8146: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:58.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:57 smithi130 bash[51302]: debug 2023-08-28T02:29:57.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:57.679737+0000) 2023-08-28T02:29:59.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:58 smithi130 bash[27579]: cluster 2023-08-28T02:29:57.715560+0000 mgr.y (mgr.14901) 8140 : cluster [DBG] pgmap v8147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:59.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:29:58 smithi130 bash[27579]: audit 2023-08-28T02:29:57.829708+0000 mon.a (mon.0) 5147 : audit [DBG] from='client.? 172.21.15.6:0/3410038005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:59.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:58 smithi130 bash[51302]: debug 2023-08-28T02:29:58.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:58.679985+0000) 2023-08-28T02:29:59.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:58 smithi006 bash[33918]: cluster 2023-08-28T02:29:57.715560+0000 mgr.y (mgr.14901) 8140 : cluster [DBG] pgmap v8147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:59.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:29:58 smithi006 bash[33918]: audit 2023-08-28T02:29:57.829708+0000 mon.a (mon.0) 5147 : audit [DBG] from='client.? 172.21.15.6:0/3410038005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:59.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:58 smithi006 bash[27292]: cluster 2023-08-28T02:29:57.715560+0000 mgr.y (mgr.14901) 8140 : cluster [DBG] pgmap v8147: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:29:59.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:29:58 smithi006 bash[27292]: audit 2023-08-28T02:29:57.829708+0000 mon.a (mon.0) 5147 : audit [DBG] from='client.? 172.21.15.6:0/3410038005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:29:59 smithi130 bash[51302]: debug 2023-08-28T02:29:59.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:59.680282+0000) 2023-08-28T02:30:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:00 smithi130 bash[51302]: debug 2023-08-28T02:30:00.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:00.680545+0000) 2023-08-28T02:30:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:00 smithi130 bash[27579]: cluster 2023-08-28T02:29:59.716997+0000 mgr.y (mgr.14901) 8141 : cluster [DBG] pgmap v8148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:00 smithi130 bash[27579]: cluster 2023-08-28T02:30:00.000119+0000 mon.a (mon.0) 5148 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:30:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:00 smithi130 bash[27579]: audit 2023-08-28T02:30:00.246669+0000 mon.c (mon.2) 3228 : audit [DBG] from='client.? 172.21.15.6:0/3192695711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:30:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:30:00] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:30:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:00 smithi006 bash[27292]: cluster 2023-08-28T02:29:59.716997+0000 mgr.y (mgr.14901) 8141 : cluster [DBG] pgmap v8148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:00 smithi006 bash[27292]: cluster 2023-08-28T02:30:00.000119+0000 mon.a (mon.0) 5148 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:30:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:00 smithi006 bash[27292]: audit 2023-08-28T02:30:00.246669+0000 mon.c (mon.2) 3228 : audit [DBG] from='client.? 172.21.15.6:0/3192695711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:01.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:00 smithi006 bash[33918]: cluster 2023-08-28T02:29:59.716997+0000 mgr.y (mgr.14901) 8141 : cluster [DBG] pgmap v8148: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:01.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:00 smithi006 bash[33918]: cluster 2023-08-28T02:30:00.000119+0000 mon.a (mon.0) 5148 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:30:01.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:00 smithi006 bash[33918]: audit 2023-08-28T02:30:00.246669+0000 mon.c (mon.2) 3228 : audit [DBG] from='client.? 172.21.15.6:0/3192695711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:30:01.318Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:30:01.320Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:02.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:01 smithi130 bash[51302]: debug 2023-08-28T02:30:01.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:01.680823+0000) 2023-08-28T02:30:03.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:02 smithi130 bash[27579]: cluster 2023-08-28T02:30:01.717651+0000 mgr.y (mgr.14901) 8142 : cluster [DBG] pgmap v8149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:02 smithi130 bash[27579]: audit 2023-08-28T02:30:02.656545+0000 mon.c (mon.2) 3229 : audit [DBG] from='client.? 172.21.15.6:0/2558542189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:03.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:02 smithi130 bash[51302]: debug 2023-08-28T02:30:02.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:02.681140+0000) 2023-08-28T02:30:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:30:02.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:30:02.881Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:03.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:02 smithi006 bash[27292]: cluster 2023-08-28T02:30:01.717651+0000 mgr.y (mgr.14901) 8142 : cluster [DBG] pgmap v8149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:03.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:02 smithi006 bash[27292]: audit 2023-08-28T02:30:02.656545+0000 mon.c (mon.2) 3229 : audit [DBG] from='client.? 172.21.15.6:0/2558542189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:03.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:02 smithi006 bash[33918]: cluster 2023-08-28T02:30:01.717651+0000 mgr.y (mgr.14901) 8142 : cluster [DBG] pgmap v8149: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:03.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:02 smithi006 bash[33918]: audit 2023-08-28T02:30:02.656545+0000 mon.c (mon.2) 3229 : audit [DBG] from='client.? 172.21.15.6:0/2558542189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:04.076 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:03 smithi130 bash[51302]: debug 2023-08-28T02:30:03.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:03.681446+0000) 2023-08-28T02:30:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:04 smithi130 bash[27579]: cluster 2023-08-28T02:30:03.718344+0000 mgr.y (mgr.14901) 8143 : cluster [DBG] pgmap v8150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:04.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:04 smithi006 bash[27292]: cluster 2023-08-28T02:30:03.718344+0000 mgr.y (mgr.14901) 8143 : cluster [DBG] pgmap v8150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:04.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:04 smithi006 bash[33918]: cluster 2023-08-28T02:30:03.718344+0000 mgr.y (mgr.14901) 8143 : cluster [DBG] pgmap v8150: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:05.080 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:30:04.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:05.080 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:30:04.763Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:05.080 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:04 smithi130 bash[51302]: debug 2023-08-28T02:30:04.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:04.681710+0000) 2023-08-28T02:30:05.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:05 smithi130 bash[27579]: audit 2023-08-28T02:30:05.063586+0000 mon.c (mon.2) 3230 : audit [DBG] from='client.? 172.21.15.6:0/2272300947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:05.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:05 smithi006 bash[33918]: audit 2023-08-28T02:30:05.063586+0000 mon.c (mon.2) 3230 : audit [DBG] from='client.? 172.21.15.6:0/2272300947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:05.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:05 smithi006 bash[27292]: audit 2023-08-28T02:30:05.063586+0000 mon.c (mon.2) 3230 : audit [DBG] from='client.? 172.21.15.6:0/2272300947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:06.089 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:05 smithi130 bash[51302]: debug 2023-08-28T02:30:05.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:05.681970+0000) 2023-08-28T02:30:06.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:06 smithi130 bash[27579]: cluster 2023-08-28T02:30:05.719461+0000 mgr.y (mgr.14901) 8144 : cluster [DBG] pgmap v8151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:06.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:06 smithi006 bash[27292]: cluster 2023-08-28T02:30:05.719461+0000 mgr.y (mgr.14901) 8144 : cluster [DBG] pgmap v8151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:06.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:06 smithi006 bash[33918]: cluster 2023-08-28T02:30:05.719461+0000 mgr.y (mgr.14901) 8144 : cluster [DBG] pgmap v8151: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:07.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:06 smithi130 bash[51302]: debug 2023-08-28T02:30:06.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:06.682191+0000) 2023-08-28T02:30:07.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:07 smithi130 bash[27579]: audit 2023-08-28T02:30:07.486690+0000 mon.c (mon.2) 3231 : audit [DBG] from='client.? 172.21.15.6:0/2837753352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:07.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:07 smithi130 bash[51302]: debug 2023-08-28T02:30:07.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:07.682455+0000) 2023-08-28T02:30:07.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:07 smithi006 bash[27292]: audit 2023-08-28T02:30:07.486690+0000 mon.c (mon.2) 3231 : audit [DBG] from='client.? 172.21.15.6:0/2837753352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:07.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:07 smithi006 bash[33918]: audit 2023-08-28T02:30:07.486690+0000 mon.c (mon.2) 3231 : audit [DBG] from='client.? 172.21.15.6:0/2837753352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:08.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:08 smithi130 bash[27579]: cluster 2023-08-28T02:30:07.720150+0000 mgr.y (mgr.14901) 8145 : cluster [DBG] pgmap v8152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:08.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:08 smithi130 bash[51302]: debug 2023-08-28T02:30:08.677+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:08.682675+0000) 2023-08-28T02:30:08.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:08 smithi006 bash[33918]: cluster 2023-08-28T02:30:07.720150+0000 mgr.y (mgr.14901) 8145 : cluster [DBG] pgmap v8152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:08.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:08 smithi006 bash[27292]: cluster 2023-08-28T02:30:07.720150+0000 mgr.y (mgr.14901) 8145 : cluster [DBG] pgmap v8152: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:10.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:09 smithi130 bash[51302]: debug 2023-08-28T02:30:09.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:09.682926+0000) 2023-08-28T02:30:11.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:10 smithi130 bash[51302]: debug 2023-08-28T02:30:10.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:10.683168+0000) 2023-08-28T02:30:11.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:10 smithi130 bash[27579]: cluster 2023-08-28T02:30:09.721242+0000 mgr.y (mgr.14901) 8146 : cluster [DBG] pgmap v8153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:11.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:10 smithi130 bash[27579]: audit 2023-08-28T02:30:09.895461+0000 mon.c (mon.2) 3232 : audit [DBG] from='client.? 172.21.15.6:0/1833805821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:11.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:30:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:30:10] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:30:11.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:10 smithi006 bash[27292]: cluster 2023-08-28T02:30:09.721242+0000 mgr.y (mgr.14901) 8146 : cluster [DBG] pgmap v8153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:11.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:10 smithi006 bash[27292]: audit 2023-08-28T02:30:09.895461+0000 mon.c (mon.2) 3232 : audit [DBG] from='client.? 172.21.15.6:0/1833805821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:11.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:10 smithi006 bash[33918]: cluster 2023-08-28T02:30:09.721242+0000 mgr.y (mgr.14901) 8146 : cluster [DBG] pgmap v8153: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:11.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:10 smithi006 bash[33918]: audit 2023-08-28T02:30:09.895461+0000 mon.c (mon.2) 3232 : audit [DBG] from='client.? 172.21.15.6:0/1833805821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:30:11.318Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:30:11.320Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:12.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:11 smithi130 bash[51302]: debug 2023-08-28T02:30:11.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:11.683424+0000) 2023-08-28T02:30:13.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:12 smithi130 bash[51302]: debug 2023-08-28T02:30:12.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:12.683730+0000) 2023-08-28T02:30:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:12 smithi130 bash[27579]: cluster 2023-08-28T02:30:11.721891+0000 mgr.y (mgr.14901) 8147 : cluster [DBG] pgmap v8154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:12 smithi130 bash[27579]: audit 2023-08-28T02:30:12.309058+0000 mon.a (mon.0) 5149 : audit [DBG] from='client.? 172.21.15.6:0/320039483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:13.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:12 smithi006 bash[27292]: cluster 2023-08-28T02:30:11.721891+0000 mgr.y (mgr.14901) 8147 : cluster [DBG] pgmap v8154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:13.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:12 smithi006 bash[27292]: audit 2023-08-28T02:30:12.309058+0000 mon.a (mon.0) 5149 : audit [DBG] from='client.? 172.21.15.6:0/320039483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:12 smithi006 bash[33918]: cluster 2023-08-28T02:30:11.721891+0000 mgr.y (mgr.14901) 8147 : cluster [DBG] pgmap v8154: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:12 smithi006 bash[33918]: audit 2023-08-28T02:30:12.309058+0000 mon.a (mon.0) 5149 : audit [DBG] from='client.? 172.21.15.6:0/320039483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:30:12.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:30:12.883Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:14.078 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:13 smithi130 bash[51302]: debug 2023-08-28T02:30:13.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:13.684023+0000) 2023-08-28T02:30:14.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:14 smithi130 bash[27579]: audit 2023-08-28T02:30:13.702016+0000 mon.b (mon.1) 888 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:30:14.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:14 smithi130 bash[27579]: cluster 2023-08-28T02:30:13.722471+0000 mgr.y (mgr.14901) 8148 : cluster [DBG] pgmap v8155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:14.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:14 smithi006 bash[33918]: audit 2023-08-28T02:30:13.702016+0000 mon.b (mon.1) 888 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:30:14.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:14 smithi006 bash[33918]: cluster 2023-08-28T02:30:13.722471+0000 mgr.y (mgr.14901) 8148 : cluster [DBG] pgmap v8155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:14 smithi006 bash[27292]: audit 2023-08-28T02:30:13.702016+0000 mon.b (mon.1) 888 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:30:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:14 smithi006 bash[27292]: cluster 2023-08-28T02:30:13.722471+0000 mgr.y (mgr.14901) 8148 : cluster [DBG] pgmap v8155: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:15.080 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:30:14.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:15.081 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:30:14.764Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:15.081 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:14 smithi130 bash[51302]: debug 2023-08-28T02:30:14.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:14.684281+0000) 2023-08-28T02:30:15.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:15 smithi130 bash[27579]: audit 2023-08-28T02:30:14.729073+0000 mon.a (mon.0) 5150 : audit [DBG] from='client.? 172.21.15.6:0/2254064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:15.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:15 smithi006 bash[33918]: audit 2023-08-28T02:30:14.729073+0000 mon.a (mon.0) 5150 : audit [DBG] from='client.? 172.21.15.6:0/2254064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:15.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:15 smithi006 bash[27292]: audit 2023-08-28T02:30:14.729073+0000 mon.a (mon.0) 5150 : audit [DBG] from='client.? 172.21.15.6:0/2254064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:16.088 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:15 smithi130 bash[51302]: debug 2023-08-28T02:30:15.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:15.684493+0000) 2023-08-28T02:30:16.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:16 smithi130 bash[27579]: cluster 2023-08-28T02:30:15.723628+0000 mgr.y (mgr.14901) 8149 : cluster [DBG] pgmap v8156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:16.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:16 smithi006 bash[27292]: cluster 2023-08-28T02:30:15.723628+0000 mgr.y (mgr.14901) 8149 : cluster [DBG] pgmap v8156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:16.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:16 smithi006 bash[33918]: cluster 2023-08-28T02:30:15.723628+0000 mgr.y (mgr.14901) 8149 : cluster [DBG] pgmap v8156: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:17.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:16 smithi130 bash[51302]: debug 2023-08-28T02:30:16.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:16.684759+0000) 2023-08-28T02:30:17.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:17 smithi006 bash[33918]: audit 2023-08-28T02:30:17.153958+0000 mon.a (mon.0) 5151 : audit [DBG] from='client.? 172.21.15.6:0/1862687352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:17.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:17 smithi006 bash[27292]: audit 2023-08-28T02:30:17.153958+0000 mon.a (mon.0) 5151 : audit [DBG] from='client.? 172.21.15.6:0/1862687352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:17.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:17 smithi130 bash[27579]: audit 2023-08-28T02:30:17.153958+0000 mon.a (mon.0) 5151 : audit [DBG] from='client.? 172.21.15.6:0/1862687352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:18.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:17 smithi130 bash[51302]: debug 2023-08-28T02:30:17.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:17.685031+0000) 2023-08-28T02:30:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:18 smithi130 bash[27579]: cluster 2023-08-28T02:30:17.724294+0000 mgr.y (mgr.14901) 8150 : cluster [DBG] pgmap v8157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:18 smithi006 bash[27292]: cluster 2023-08-28T02:30:17.724294+0000 mgr.y (mgr.14901) 8150 : cluster [DBG] pgmap v8157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:18.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:18 smithi006 bash[33918]: cluster 2023-08-28T02:30:17.724294+0000 mgr.y (mgr.14901) 8150 : cluster [DBG] pgmap v8157: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:19.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:18 smithi130 bash[51302]: debug 2023-08-28T02:30:18.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:18.685322+0000) 2023-08-28T02:30:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:19 smithi130 bash[51302]: debug 2023-08-28T02:30:19.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:19.685644+0000) 2023-08-28T02:30:20.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:20 smithi130 bash[51302]: debug 2023-08-28T02:30:20.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:20.685932+0000) 2023-08-28T02:30:20.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:20 smithi130 bash[27579]: audit 2023-08-28T02:30:19.607408+0000 mon.a (mon.0) 5152 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:30:20.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:20 smithi130 bash[27579]: audit 2023-08-28T02:30:19.621070+0000 mon.c (mon.2) 3233 : audit [DBG] from='client.? 172.21.15.6:0/2446173813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:20.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:20 smithi130 bash[27579]: cluster 2023-08-28T02:30:19.725429+0000 mgr.y (mgr.14901) 8151 : cluster [DBG] pgmap v8158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:20.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:20 smithi130 bash[27579]: audit 2023-08-28T02:30:20.013973+0000 mon.a (mon.0) 5153 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:30:20.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:20 smithi130 bash[27579]: audit 2023-08-28T02:30:20.278804+0000 mon.b (mon.1) 889 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:30:20.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:20 smithi130 bash[27579]: audit 2023-08-28T02:30:20.281062+0000 mon.b (mon.1) 890 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:30:20.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:20 smithi130 bash[27579]: audit 2023-08-28T02:30:20.288797+0000 mon.a (mon.0) 5154 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:30:20.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:30:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:30:20] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T02:30:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:20 smithi006 bash[27292]: audit 2023-08-28T02:30:19.607408+0000 mon.a (mon.0) 5152 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:30:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:20 smithi006 bash[27292]: audit 2023-08-28T02:30:19.621070+0000 mon.c (mon.2) 3233 : audit [DBG] from='client.? 172.21.15.6:0/2446173813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:20 smithi006 bash[27292]: cluster 2023-08-28T02:30:19.725429+0000 mgr.y (mgr.14901) 8151 : cluster [DBG] pgmap v8158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:20 smithi006 bash[27292]: audit 2023-08-28T02:30:20.013973+0000 mon.a (mon.0) 5153 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:30:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:20 smithi006 bash[27292]: audit 2023-08-28T02:30:20.278804+0000 mon.b (mon.1) 889 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:30:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:20 smithi006 bash[27292]: audit 2023-08-28T02:30:20.281062+0000 mon.b (mon.1) 890 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:30:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:20 smithi006 bash[27292]: audit 2023-08-28T02:30:20.288797+0000 mon.a (mon.0) 5154 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:30:20.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:20 smithi006 bash[33918]: audit 2023-08-28T02:30:19.607408+0000 mon.a (mon.0) 5152 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:30:20.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:20 smithi006 bash[33918]: audit 2023-08-28T02:30:19.621070+0000 mon.c (mon.2) 3233 : audit [DBG] from='client.? 172.21.15.6:0/2446173813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:20.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:20 smithi006 bash[33918]: cluster 2023-08-28T02:30:19.725429+0000 mgr.y (mgr.14901) 8151 : cluster [DBG] pgmap v8158: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:20.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:20 smithi006 bash[33918]: audit 2023-08-28T02:30:20.013973+0000 mon.a (mon.0) 5153 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:30:20.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:20 smithi006 bash[33918]: audit 2023-08-28T02:30:20.278804+0000 mon.b (mon.1) 889 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:30:20.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:20 smithi006 bash[33918]: audit 2023-08-28T02:30:20.281062+0000 mon.b (mon.1) 890 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:30:20.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:20 smithi006 bash[33918]: audit 2023-08-28T02:30:20.288797+0000 mon.a (mon.0) 5154 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:30:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:30:21.318Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:30:21.320Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:22.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:21 smithi130 bash[51302]: debug 2023-08-28T02:30:21.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:21.686214+0000) 2023-08-28T02:30:23.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:22 smithi130 bash[51302]: debug 2023-08-28T02:30:22.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:22.686460+0000) 2023-08-28T02:30:23.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:22 smithi130 bash[27579]: cluster 2023-08-28T02:30:21.726090+0000 mgr.y (mgr.14901) 8152 : cluster [DBG] pgmap v8159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:23.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:22 smithi130 bash[27579]: audit 2023-08-28T02:30:22.038874+0000 mon.c (mon.2) 3234 : audit [DBG] from='client.? 172.21.15.6:0/3823357849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:23.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:22 smithi006 bash[33918]: cluster 2023-08-28T02:30:21.726090+0000 mgr.y (mgr.14901) 8152 : cluster [DBG] pgmap v8159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:23.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:22 smithi006 bash[33918]: audit 2023-08-28T02:30:22.038874+0000 mon.c (mon.2) 3234 : audit [DBG] from='client.? 172.21.15.6:0/3823357849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:23.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:22 smithi006 bash[27292]: cluster 2023-08-28T02:30:21.726090+0000 mgr.y (mgr.14901) 8152 : cluster [DBG] pgmap v8159: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:23.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:22 smithi006 bash[27292]: audit 2023-08-28T02:30:22.038874+0000 mon.c (mon.2) 3234 : audit [DBG] from='client.? 172.21.15.6:0/3823357849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:30:22.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:30:22.884Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:24.080 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:23 smithi130 bash[51302]: debug 2023-08-28T02:30:23.681+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:23.686789+0000) 2023-08-28T02:30:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:24 smithi130 bash[27579]: cluster 2023-08-28T02:30:23.726783+0000 mgr.y (mgr.14901) 8153 : cluster [DBG] pgmap v8160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:24.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:24 smithi006 bash[27292]: cluster 2023-08-28T02:30:23.726783+0000 mgr.y (mgr.14901) 8153 : cluster [DBG] pgmap v8160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:24.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:24 smithi006 bash[33918]: cluster 2023-08-28T02:30:23.726783+0000 mgr.y (mgr.14901) 8153 : cluster [DBG] pgmap v8160: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:25.082 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:30:24.763Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:25.083 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:30:24.765Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:25.083 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:24 smithi130 bash[51302]: debug 2023-08-28T02:30:24.685+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:24.687125+0000) 2023-08-28T02:30:25.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:25 smithi130 bash[27579]: audit 2023-08-28T02:30:24.461282+0000 mon.a (mon.0) 5155 : audit [DBG] from='client.? 172.21.15.6:0/1394985254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:25.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:25 smithi006 bash[33918]: audit 2023-08-28T02:30:24.461282+0000 mon.a (mon.0) 5155 : audit [DBG] from='client.? 172.21.15.6:0/1394985254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:25.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:25 smithi006 bash[27292]: audit 2023-08-28T02:30:24.461282+0000 mon.a (mon.0) 5155 : audit [DBG] from='client.? 172.21.15.6:0/1394985254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:26.095 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:25 smithi130 bash[51302]: debug 2023-08-28T02:30:25.685+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:25.687460+0000) 2023-08-28T02:30:26.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:26 smithi130 bash[27579]: cluster 2023-08-28T02:30:25.727897+0000 mgr.y (mgr.14901) 8154 : cluster [DBG] pgmap v8161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:26.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:26 smithi006 bash[33918]: cluster 2023-08-28T02:30:25.727897+0000 mgr.y (mgr.14901) 8154 : cluster [DBG] pgmap v8161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:26.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:26 smithi006 bash[27292]: cluster 2023-08-28T02:30:25.727897+0000 mgr.y (mgr.14901) 8154 : cluster [DBG] pgmap v8161: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:27.098 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:26 smithi130 bash[51302]: debug 2023-08-28T02:30:26.685+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:26.687748+0000) 2023-08-28T02:30:27.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:27 smithi130 bash[27579]: audit 2023-08-28T02:30:26.864763+0000 mon.c (mon.2) 3235 : audit [DBG] from='client.? 172.21.15.6:0/2215168332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:27.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:27 smithi006 bash[27292]: audit 2023-08-28T02:30:26.864763+0000 mon.c (mon.2) 3235 : audit [DBG] from='client.? 172.21.15.6:0/2215168332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:27.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:27 smithi006 bash[33918]: audit 2023-08-28T02:30:26.864763+0000 mon.c (mon.2) 3235 : audit [DBG] from='client.? 172.21.15.6:0/2215168332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:28.104 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:27 smithi130 bash[51302]: debug 2023-08-28T02:30:27.685+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:27.687994+0000) 2023-08-28T02:30:28.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:28 smithi130 bash[27579]: cluster 2023-08-28T02:30:27.728558+0000 mgr.y (mgr.14901) 8155 : cluster [DBG] pgmap v8162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:28.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:28 smithi006 bash[27292]: cluster 2023-08-28T02:30:27.728558+0000 mgr.y (mgr.14901) 8155 : cluster [DBG] pgmap v8162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:28.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:28 smithi006 bash[33918]: cluster 2023-08-28T02:30:27.728558+0000 mgr.y (mgr.14901) 8155 : cluster [DBG] pgmap v8162: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:28 smithi130 bash[51302]: debug 2023-08-28T02:30:28.685+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:28.688311+0000) 2023-08-28T02:30:29.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:29 smithi130 bash[27579]: audit 2023-08-28T02:30:29.273196+0000 mon.a (mon.0) 5156 : audit [DBG] from='client.? 172.21.15.6:0/3460831387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:29.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:29 smithi006 bash[27292]: audit 2023-08-28T02:30:29.273196+0000 mon.a (mon.0) 5156 : audit [DBG] from='client.? 172.21.15.6:0/3460831387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:29.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:29 smithi006 bash[33918]: audit 2023-08-28T02:30:29.273196+0000 mon.a (mon.0) 5156 : audit [DBG] from='client.? 172.21.15.6:0/3460831387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:30.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:29 smithi130 bash[51302]: debug 2023-08-28T02:30:29.685+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:29.688639+0000) 2023-08-28T02:30:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:30 smithi130 bash[27579]: cluster 2023-08-28T02:30:29.729738+0000 mgr.y (mgr.14901) 8156 : cluster [DBG] pgmap v8163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:30.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:30 smithi006 bash[27292]: cluster 2023-08-28T02:30:29.729738+0000 mgr.y (mgr.14901) 8156 : cluster [DBG] pgmap v8163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:30.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:30 smithi006 bash[33918]: cluster 2023-08-28T02:30:29.729738+0000 mgr.y (mgr.14901) 8156 : cluster [DBG] pgmap v8163: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:31.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:30 smithi130 bash[51302]: debug 2023-08-28T02:30:30.685+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:30.688952+0000) 2023-08-28T02:30:31.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:30:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:30:30] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:30:31.606 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:30:31.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:31.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:31 smithi006 bash[27292]: audit 2023-08-28T02:30:31.674132+0000 mon.c (mon.2) 3236 : audit [DBG] from='client.? 172.21.15.6:0/3419548195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:31.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:30:31.321Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:31.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:31 smithi006 bash[33918]: audit 2023-08-28T02:30:31.674132+0000 mon.c (mon.2) 3236 : audit [DBG] from='client.? 172.21.15.6:0/3419548195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:32.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:31 smithi130 bash[27579]: audit 2023-08-28T02:30:31.674132+0000 mon.c (mon.2) 3236 : audit [DBG] from='client.? 172.21.15.6:0/3419548195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:32.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:31 smithi130 bash[51302]: debug 2023-08-28T02:30:31.685+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:31.689327+0000) 2023-08-28T02:30:33.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:32 smithi130 bash[27579]: cluster 2023-08-28T02:30:31.730305+0000 mgr.y (mgr.14901) 8157 : cluster [DBG] pgmap v8164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:33.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:32 smithi130 bash[51302]: debug 2023-08-28T02:30:32.685+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:32.689656+0000) 2023-08-28T02:30:33.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:32 smithi006 bash[33918]: cluster 2023-08-28T02:30:31.730305+0000 mgr.y (mgr.14901) 8157 : cluster [DBG] pgmap v8164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:33.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:32 smithi006 bash[27292]: cluster 2023-08-28T02:30:31.730305+0000 mgr.y (mgr.14901) 8157 : cluster [DBG] pgmap v8164: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:30:32.883Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:30:32.884Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:34.082 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:33 smithi130 bash[51302]: debug 2023-08-28T02:30:33.685+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:33.689955+0000) 2023-08-28T02:30:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:34 smithi130 bash[27579]: cluster 2023-08-28T02:30:33.730941+0000 mgr.y (mgr.14901) 8158 : cluster [DBG] pgmap v8165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:34.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:34 smithi006 bash[33918]: cluster 2023-08-28T02:30:33.730941+0000 mgr.y (mgr.14901) 8158 : cluster [DBG] pgmap v8165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:34.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:34 smithi006 bash[27292]: cluster 2023-08-28T02:30:33.730941+0000 mgr.y (mgr.14901) 8158 : cluster [DBG] pgmap v8165: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:35.084 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:30:34.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:35.085 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:30:34.766Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:35.085 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:34 smithi130 bash[51302]: debug 2023-08-28T02:30:34.685+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:34.690320+0000) 2023-08-28T02:30:35.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:35 smithi130 bash[27579]: audit 2023-08-28T02:30:34.098330+0000 mon.c (mon.2) 3237 : audit [DBG] from='client.? 172.21.15.6:0/2441224789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:35.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:35 smithi006 bash[33918]: audit 2023-08-28T02:30:34.098330+0000 mon.c (mon.2) 3237 : audit [DBG] from='client.? 172.21.15.6:0/2441224789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:35.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:35 smithi006 bash[27292]: audit 2023-08-28T02:30:34.098330+0000 mon.c (mon.2) 3237 : audit [DBG] from='client.? 172.21.15.6:0/2441224789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:36.092 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:35 smithi130 bash[51302]: debug 2023-08-28T02:30:35.685+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:35.690598+0000) 2023-08-28T02:30:36.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:36 smithi130 bash[27579]: cluster 2023-08-28T02:30:35.732190+0000 mgr.y (mgr.14901) 8159 : cluster [DBG] pgmap v8166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:36.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:36 smithi006 bash[27292]: cluster 2023-08-28T02:30:35.732190+0000 mgr.y (mgr.14901) 8159 : cluster [DBG] pgmap v8166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:36.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:36 smithi006 bash[33918]: cluster 2023-08-28T02:30:35.732190+0000 mgr.y (mgr.14901) 8159 : cluster [DBG] pgmap v8166: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:37.096 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:36 smithi130 bash[51302]: debug 2023-08-28T02:30:36.689+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:36.690893+0000) 2023-08-28T02:30:37.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:37 smithi130 bash[27579]: audit 2023-08-28T02:30:36.512863+0000 mon.c (mon.2) 3238 : audit [DBG] from='client.? 172.21.15.6:0/983744520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:37.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:37 smithi006 bash[33918]: audit 2023-08-28T02:30:36.512863+0000 mon.c (mon.2) 3238 : audit [DBG] from='client.? 172.21.15.6:0/983744520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:37.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:37 smithi006 bash[27292]: audit 2023-08-28T02:30:36.512863+0000 mon.c (mon.2) 3238 : audit [DBG] from='client.? 172.21.15.6:0/983744520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:38.102 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:37 smithi130 bash[51302]: debug 2023-08-28T02:30:37.689+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:37.691157+0000) 2023-08-28T02:30:38.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:38 smithi130 bash[27579]: cluster 2023-08-28T02:30:37.733036+0000 mgr.y (mgr.14901) 8160 : cluster [DBG] pgmap v8167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:38.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:38 smithi006 bash[27292]: cluster 2023-08-28T02:30:37.733036+0000 mgr.y (mgr.14901) 8160 : cluster [DBG] pgmap v8167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:38.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:38 smithi006 bash[33918]: cluster 2023-08-28T02:30:37.733036+0000 mgr.y (mgr.14901) 8160 : cluster [DBG] pgmap v8167: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:39.104 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:38 smithi130 bash[51302]: debug 2023-08-28T02:30:38.689+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:38.691402+0000) 2023-08-28T02:30:39.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:39 smithi130 bash[27579]: audit 2023-08-28T02:30:38.932653+0000 mon.c (mon.2) 3239 : audit [DBG] from='client.? 172.21.15.6:0/1705723669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:39.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:39 smithi006 bash[27292]: audit 2023-08-28T02:30:38.932653+0000 mon.c (mon.2) 3239 : audit [DBG] from='client.? 172.21.15.6:0/1705723669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:39.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:39 smithi006 bash[33918]: audit 2023-08-28T02:30:38.932653+0000 mon.c (mon.2) 3239 : audit [DBG] from='client.? 172.21.15.6:0/1705723669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:40.112 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:39 smithi130 bash[51302]: debug 2023-08-28T02:30:39.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:39.691683+0000) 2023-08-28T02:30:40.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:40 smithi130 bash[27579]: cluster 2023-08-28T02:30:39.734184+0000 mgr.y (mgr.14901) 8161 : cluster [DBG] pgmap v8168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:40.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:40 smithi006 bash[33918]: cluster 2023-08-28T02:30:39.734184+0000 mgr.y (mgr.14901) 8161 : cluster [DBG] pgmap v8168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:40.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:40 smithi006 bash[27292]: cluster 2023-08-28T02:30:39.734184+0000 mgr.y (mgr.14901) 8161 : cluster [DBG] pgmap v8168: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:41.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:40 smithi130 bash[51302]: debug 2023-08-28T02:30:40.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:40.692027+0000) 2023-08-28T02:30:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:30:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:30:40] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:30:41.691 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:41 smithi130 bash[27579]: audit 2023-08-28T02:30:41.351666+0000 mon.c (mon.2) 3240 : audit [DBG] from='client.? 172.21.15.6:0/3259859730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:41.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:41 smithi006 bash[33918]: audit 2023-08-28T02:30:41.351666+0000 mon.c (mon.2) 3240 : audit [DBG] from='client.? 172.21.15.6:0/3259859730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:41.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:41 smithi006 bash[27292]: audit 2023-08-28T02:30:41.351666+0000 mon.c (mon.2) 3240 : audit [DBG] from='client.? 172.21.15.6:0/3259859730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:30:41.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:30:41.321Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:41 smithi130 bash[51302]: debug 2023-08-28T02:30:41.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:41.692351+0000) 2023-08-28T02:30:42.691 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:42 smithi130 bash[27579]: cluster 2023-08-28T02:30:41.734843+0000 mgr.y (mgr.14901) 8162 : cluster [DBG] pgmap v8169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:42.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:42 smithi006 bash[33918]: cluster 2023-08-28T02:30:41.734843+0000 mgr.y (mgr.14901) 8162 : cluster [DBG] pgmap v8169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:42.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:42 smithi006 bash[27292]: cluster 2023-08-28T02:30:41.734843+0000 mgr.y (mgr.14901) 8162 : cluster [DBG] pgmap v8169: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:43.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:42 smithi130 bash[51302]: debug 2023-08-28T02:30:42.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:42.692678+0000) 2023-08-28T02:30:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:30:42.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:30:42.886Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:44.084 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:43 smithi130 bash[51302]: debug 2023-08-28T02:30:43.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:43.692967+0000) 2023-08-28T02:30:44.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:44 smithi130 bash[27579]: cluster 2023-08-28T02:30:43.735431+0000 mgr.y (mgr.14901) 8163 : cluster [DBG] pgmap v8170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:44.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:44 smithi130 bash[27579]: audit 2023-08-28T02:30:43.763235+0000 mon.a (mon.0) 5157 : audit [DBG] from='client.? 172.21.15.6:0/3207320974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:44.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:44 smithi006 bash[27292]: cluster 2023-08-28T02:30:43.735431+0000 mgr.y (mgr.14901) 8163 : cluster [DBG] pgmap v8170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:44.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:44 smithi006 bash[27292]: audit 2023-08-28T02:30:43.763235+0000 mon.a (mon.0) 5157 : audit [DBG] from='client.? 172.21.15.6:0/3207320974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:44 smithi006 bash[33918]: cluster 2023-08-28T02:30:43.735431+0000 mgr.y (mgr.14901) 8163 : cluster [DBG] pgmap v8170: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:44 smithi006 bash[33918]: audit 2023-08-28T02:30:43.763235+0000 mon.a (mon.0) 5157 : audit [DBG] from='client.? 172.21.15.6:0/3207320974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:45.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:44 smithi130 bash[51302]: debug 2023-08-28T02:30:44.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:44.693265+0000) 2023-08-28T02:30:45.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:30:44.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:45.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:30:44.766Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:45 smithi130 bash[51302]: debug 2023-08-28T02:30:45.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:45.693528+0000) 2023-08-28T02:30:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:46 smithi130 bash[51302]: debug 2023-08-28T02:30:46.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:46.693790+0000) 2023-08-28T02:30:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:46 smithi130 bash[27579]: cluster 2023-08-28T02:30:45.736525+0000 mgr.y (mgr.14901) 8164 : cluster [DBG] pgmap v8171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:46 smithi130 bash[27579]: audit 2023-08-28T02:30:46.174881+0000 mon.c (mon.2) 3241 : audit [DBG] from='client.? 172.21.15.6:0/1623810494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:47.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:46 smithi006 bash[33918]: cluster 2023-08-28T02:30:45.736525+0000 mgr.y (mgr.14901) 8164 : cluster [DBG] pgmap v8171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:47.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:46 smithi006 bash[33918]: audit 2023-08-28T02:30:46.174881+0000 mon.c (mon.2) 3241 : audit [DBG] from='client.? 172.21.15.6:0/1623810494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:47.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:46 smithi006 bash[27292]: cluster 2023-08-28T02:30:45.736525+0000 mgr.y (mgr.14901) 8164 : cluster [DBG] pgmap v8171: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:47.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:46 smithi006 bash[27292]: audit 2023-08-28T02:30:46.174881+0000 mon.c (mon.2) 3241 : audit [DBG] from='client.? 172.21.15.6:0/1623810494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:47 smithi130 bash[51302]: debug 2023-08-28T02:30:47.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:47.694102+0000) 2023-08-28T02:30:49.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:48 smithi130 bash[51302]: debug 2023-08-28T02:30:48.688+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:48.694382+0000) 2023-08-28T02:30:49.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:48 smithi130 bash[27579]: cluster 2023-08-28T02:30:47.737221+0000 mgr.y (mgr.14901) 8165 : cluster [DBG] pgmap v8172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:49.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:48 smithi130 bash[27579]: audit 2023-08-28T02:30:48.588784+0000 mon.a (mon.0) 5158 : audit [DBG] from='client.? 172.21.15.6:0/2038671138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:49.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:48 smithi006 bash[33918]: cluster 2023-08-28T02:30:47.737221+0000 mgr.y (mgr.14901) 8165 : cluster [DBG] pgmap v8172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:49.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:48 smithi006 bash[33918]: audit 2023-08-28T02:30:48.588784+0000 mon.a (mon.0) 5158 : audit [DBG] from='client.? 172.21.15.6:0/2038671138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:49.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:48 smithi006 bash[27292]: cluster 2023-08-28T02:30:47.737221+0000 mgr.y (mgr.14901) 8165 : cluster [DBG] pgmap v8172: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:49.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:48 smithi006 bash[27292]: audit 2023-08-28T02:30:48.588784+0000 mon.a (mon.0) 5158 : audit [DBG] from='client.? 172.21.15.6:0/2038671138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:50.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:49 smithi130 bash[51302]: debug 2023-08-28T02:30:49.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:49.694666+0000) 2023-08-28T02:30:51.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:50 smithi130 bash[27579]: cluster 2023-08-28T02:30:49.738317+0000 mgr.y (mgr.14901) 8166 : cluster [DBG] pgmap v8173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:51.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:50 smithi130 bash[51302]: debug 2023-08-28T02:30:50.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:50.694983+0000) 2023-08-28T02:30:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:30:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:30:50] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T02:30:51.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:50 smithi006 bash[33918]: cluster 2023-08-28T02:30:49.738317+0000 mgr.y (mgr.14901) 8166 : cluster [DBG] pgmap v8173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:51.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:50 smithi006 bash[27292]: cluster 2023-08-28T02:30:49.738317+0000 mgr.y (mgr.14901) 8166 : cluster [DBG] pgmap v8173: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:30:51.319Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:30:51.321Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:52.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:51 smithi130 bash[27579]: audit 2023-08-28T02:30:51.008485+0000 mon.c (mon.2) 3242 : audit [DBG] from='client.? 172.21.15.6:0/2004045829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:52.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:51 smithi130 bash[51302]: debug 2023-08-28T02:30:51.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:51.695272+0000) 2023-08-28T02:30:52.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:51 smithi006 bash[27292]: audit 2023-08-28T02:30:51.008485+0000 mon.c (mon.2) 3242 : audit [DBG] from='client.? 172.21.15.6:0/2004045829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:52.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:51 smithi006 bash[33918]: audit 2023-08-28T02:30:51.008485+0000 mon.c (mon.2) 3242 : audit [DBG] from='client.? 172.21.15.6:0/2004045829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:53.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:52 smithi130 bash[27579]: cluster 2023-08-28T02:30:51.739033+0000 mgr.y (mgr.14901) 8167 : cluster [DBG] pgmap v8174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:53.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:52 smithi130 bash[51302]: debug 2023-08-28T02:30:52.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:52.695484+0000) 2023-08-28T02:30:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:30:52.885Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:30:52.887Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:53.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:52 smithi006 bash[33918]: cluster 2023-08-28T02:30:51.739033+0000 mgr.y (mgr.14901) 8167 : cluster [DBG] pgmap v8174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:53.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:52 smithi006 bash[27292]: cluster 2023-08-28T02:30:51.739033+0000 mgr.y (mgr.14901) 8167 : cluster [DBG] pgmap v8174: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:54.088 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:53 smithi130 bash[51302]: debug 2023-08-28T02:30:53.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:53.695758+0000) 2023-08-28T02:30:54.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:54 smithi130 bash[27579]: audit 2023-08-28T02:30:53.435292+0000 mon.a (mon.0) 5159 : audit [DBG] from='client.? 172.21.15.6:0/3944235681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:54.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:54 smithi130 bash[27579]: cluster 2023-08-28T02:30:53.739750+0000 mgr.y (mgr.14901) 8168 : cluster [DBG] pgmap v8175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:54.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:54 smithi006 bash[33918]: audit 2023-08-28T02:30:53.435292+0000 mon.a (mon.0) 5159 : audit [DBG] from='client.? 172.21.15.6:0/3944235681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:54.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:54 smithi006 bash[33918]: cluster 2023-08-28T02:30:53.739750+0000 mgr.y (mgr.14901) 8168 : cluster [DBG] pgmap v8175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:54 smithi006 bash[27292]: audit 2023-08-28T02:30:53.435292+0000 mon.a (mon.0) 5159 : audit [DBG] from='client.? 172.21.15.6:0/3944235681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:54 smithi006 bash[27292]: cluster 2023-08-28T02:30:53.739750+0000 mgr.y (mgr.14901) 8168 : cluster [DBG] pgmap v8175: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:55.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:54 smithi130 bash[51302]: debug 2023-08-28T02:30:54.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:54.696037+0000) 2023-08-28T02:30:55.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:30:54.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:55.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:30:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:30:54.767Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:30:56.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:55 smithi130 bash[51302]: debug 2023-08-28T02:30:55.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:55.696394+0000) 2023-08-28T02:30:57.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:56 smithi130 bash[51302]: debug 2023-08-28T02:30:56.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:56.696687+0000) 2023-08-28T02:30:57.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:56 smithi130 bash[27579]: cluster 2023-08-28T02:30:55.740501+0000 mgr.y (mgr.14901) 8169 : cluster [DBG] pgmap v8176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:57.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:56 smithi130 bash[27579]: audit 2023-08-28T02:30:55.851576+0000 mon.c (mon.2) 3243 : audit [DBG] from='client.? 172.21.15.6:0/750432889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:57.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:56 smithi006 bash[33918]: cluster 2023-08-28T02:30:55.740501+0000 mgr.y (mgr.14901) 8169 : cluster [DBG] pgmap v8176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:57.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:56 smithi006 bash[33918]: audit 2023-08-28T02:30:55.851576+0000 mon.c (mon.2) 3243 : audit [DBG] from='client.? 172.21.15.6:0/750432889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:57.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:56 smithi006 bash[27292]: cluster 2023-08-28T02:30:55.740501+0000 mgr.y (mgr.14901) 8169 : cluster [DBG] pgmap v8176: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:57.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:56 smithi006 bash[27292]: audit 2023-08-28T02:30:55.851576+0000 mon.c (mon.2) 3243 : audit [DBG] from='client.? 172.21.15.6:0/750432889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:58.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:57 smithi130 bash[51302]: debug 2023-08-28T02:30:57.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:57.696995+0000) 2023-08-28T02:30:59.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:58 smithi130 bash[51302]: debug 2023-08-28T02:30:58.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:58.697292+0000) 2023-08-28T02:30:59.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:58 smithi130 bash[27579]: cluster 2023-08-28T02:30:57.741160+0000 mgr.y (mgr.14901) 8170 : cluster [DBG] pgmap v8177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:59.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:30:58 smithi130 bash[27579]: audit 2023-08-28T02:30:58.265811+0000 mon.c (mon.2) 3244 : audit [DBG] from='client.? 172.21.15.6:0/1839196933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:59.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:58 smithi006 bash[33918]: cluster 2023-08-28T02:30:57.741160+0000 mgr.y (mgr.14901) 8170 : cluster [DBG] pgmap v8177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:59.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:30:58 smithi006 bash[33918]: audit 2023-08-28T02:30:58.265811+0000 mon.c (mon.2) 3244 : audit [DBG] from='client.? 172.21.15.6:0/1839196933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:59.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:58 smithi006 bash[27292]: cluster 2023-08-28T02:30:57.741160+0000 mgr.y (mgr.14901) 8170 : cluster [DBG] pgmap v8177: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:30:59.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:30:58 smithi006 bash[27292]: audit 2023-08-28T02:30:58.265811+0000 mon.c (mon.2) 3244 : audit [DBG] from='client.? 172.21.15.6:0/1839196933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:30:59 smithi130 bash[51302]: debug 2023-08-28T02:30:59.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:59.697555+0000) 2023-08-28T02:31:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:00 smithi130 bash[51302]: debug 2023-08-28T02:31:00.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:00.697849+0000) 2023-08-28T02:31:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:00 smithi130 bash[27579]: cluster 2023-08-28T02:30:59.742223+0000 mgr.y (mgr.14901) 8171 : cluster [DBG] pgmap v8178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:00 smithi130 bash[27579]: audit 2023-08-28T02:31:00.683051+0000 mon.a (mon.0) 5160 : audit [DBG] from='client.? 172.21.15.6:0/3106171946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:31:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:31:00] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:31:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:00 smithi006 bash[27292]: cluster 2023-08-28T02:30:59.742223+0000 mgr.y (mgr.14901) 8171 : cluster [DBG] pgmap v8178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:00 smithi006 bash[27292]: audit 2023-08-28T02:31:00.683051+0000 mon.a (mon.0) 5160 : audit [DBG] from='client.? 172.21.15.6:0/3106171946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:01.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:00 smithi006 bash[33918]: cluster 2023-08-28T02:30:59.742223+0000 mgr.y (mgr.14901) 8171 : cluster [DBG] pgmap v8178: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:01.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:00 smithi006 bash[33918]: audit 2023-08-28T02:31:00.683051+0000 mon.a (mon.0) 5160 : audit [DBG] from='client.? 172.21.15.6:0/3106171946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:31:01.320Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:31:01.322Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:02.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:01 smithi130 bash[51302]: debug 2023-08-28T02:31:01.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:01.698147+0000) 2023-08-28T02:31:03.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:02 smithi130 bash[27579]: cluster 2023-08-28T02:31:01.742891+0000 mgr.y (mgr.14901) 8172 : cluster [DBG] pgmap v8179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:03.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:02 smithi130 bash[51302]: debug 2023-08-28T02:31:02.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:02.698424+0000) 2023-08-28T02:31:03.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:02 smithi006 bash[27292]: cluster 2023-08-28T02:31:01.742891+0000 mgr.y (mgr.14901) 8172 : cluster [DBG] pgmap v8179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:03.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:02 smithi006 bash[33918]: cluster 2023-08-28T02:31:01.742891+0000 mgr.y (mgr.14901) 8172 : cluster [DBG] pgmap v8179: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:31:02.886Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:31:02.888Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:04.092 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:03 smithi130 bash[51302]: debug 2023-08-28T02:31:03.692+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:03.698691+0000) 2023-08-28T02:31:04.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:04 smithi130 bash[27579]: audit 2023-08-28T02:31:03.094289+0000 mon.a (mon.0) 5161 : audit [DBG] from='client.? 172.21.15.6:0/1445115233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:04.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:04 smithi130 bash[27579]: cluster 2023-08-28T02:31:03.743563+0000 mgr.y (mgr.14901) 8173 : cluster [DBG] pgmap v8180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:04.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:04 smithi006 bash[33918]: audit 2023-08-28T02:31:03.094289+0000 mon.a (mon.0) 5161 : audit [DBG] from='client.? 172.21.15.6:0/1445115233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:04.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:04 smithi006 bash[33918]: cluster 2023-08-28T02:31:03.743563+0000 mgr.y (mgr.14901) 8173 : cluster [DBG] pgmap v8180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:04 smithi006 bash[27292]: audit 2023-08-28T02:31:03.094289+0000 mon.a (mon.0) 5161 : audit [DBG] from='client.? 172.21.15.6:0/1445115233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:04 smithi006 bash[27292]: cluster 2023-08-28T02:31:03.743563+0000 mgr.y (mgr.14901) 8173 : cluster [DBG] pgmap v8180: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:05.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:04 smithi130 bash[51302]: debug 2023-08-28T02:31:04.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:04.699021+0000) 2023-08-28T02:31:05.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:31:04.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:05.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:31:04.766Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:05.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:05 smithi130 bash[27579]: audit 2023-08-28T02:31:05.510771+0000 mon.a (mon.0) 5162 : audit [DBG] from='client.? 172.21.15.6:0/2992052034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:05.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:05 smithi130 bash[51302]: debug 2023-08-28T02:31:05.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:05.699340+0000) 2023-08-28T02:31:05.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:05 smithi006 bash[27292]: audit 2023-08-28T02:31:05.510771+0000 mon.a (mon.0) 5162 : audit [DBG] from='client.? 172.21.15.6:0/2992052034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:05.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:05 smithi006 bash[33918]: audit 2023-08-28T02:31:05.510771+0000 mon.a (mon.0) 5162 : audit [DBG] from='client.? 172.21.15.6:0/2992052034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:06.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:06 smithi130 bash[27579]: cluster 2023-08-28T02:31:05.744737+0000 mgr.y (mgr.14901) 8174 : cluster [DBG] pgmap v8181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:06.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:06 smithi130 bash[51302]: debug 2023-08-28T02:31:06.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:06.699620+0000) 2023-08-28T02:31:06.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:06 smithi006 bash[27292]: cluster 2023-08-28T02:31:05.744737+0000 mgr.y (mgr.14901) 8174 : cluster [DBG] pgmap v8181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:06.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:06 smithi006 bash[33918]: cluster 2023-08-28T02:31:05.744737+0000 mgr.y (mgr.14901) 8174 : cluster [DBG] pgmap v8181: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:08.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:07 smithi130 bash[51302]: debug 2023-08-28T02:31:07.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:07.699896+0000) 2023-08-28T02:31:09.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:08 smithi130 bash[27579]: cluster 2023-08-28T02:31:07.745464+0000 mgr.y (mgr.14901) 8175 : cluster [DBG] pgmap v8182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:09.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:08 smithi130 bash[27579]: audit 2023-08-28T02:31:07.925750+0000 mon.c (mon.2) 3245 : audit [DBG] from='client.? 172.21.15.6:0/3597739591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:09.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:08 smithi130 bash[51302]: debug 2023-08-28T02:31:08.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:08.700167+0000) 2023-08-28T02:31:09.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:08 smithi006 bash[27292]: cluster 2023-08-28T02:31:07.745464+0000 mgr.y (mgr.14901) 8175 : cluster [DBG] pgmap v8182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:09.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:08 smithi006 bash[27292]: audit 2023-08-28T02:31:07.925750+0000 mon.c (mon.2) 3245 : audit [DBG] from='client.? 172.21.15.6:0/3597739591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:09.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:08 smithi006 bash[33918]: cluster 2023-08-28T02:31:07.745464+0000 mgr.y (mgr.14901) 8175 : cluster [DBG] pgmap v8182: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:09.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:08 smithi006 bash[33918]: audit 2023-08-28T02:31:07.925750+0000 mon.c (mon.2) 3245 : audit [DBG] from='client.? 172.21.15.6:0/3597739591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:10.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:09 smithi130 bash[51302]: debug 2023-08-28T02:31:09.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:09.700417+0000) 2023-08-28T02:31:11.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:10 smithi130 bash[51302]: debug 2023-08-28T02:31:10.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:10.700699+0000) 2023-08-28T02:31:11.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:10 smithi130 bash[27579]: cluster 2023-08-28T02:31:09.746623+0000 mgr.y (mgr.14901) 8176 : cluster [DBG] pgmap v8183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:11.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:10 smithi130 bash[27579]: audit 2023-08-28T02:31:10.353511+0000 mon.c (mon.2) 3246 : audit [DBG] from='client.? 172.21.15.6:0/3588907191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:11.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:31:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:31:10] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:31:11.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:10 smithi006 bash[27292]: cluster 2023-08-28T02:31:09.746623+0000 mgr.y (mgr.14901) 8176 : cluster [DBG] pgmap v8183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:11.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:10 smithi006 bash[27292]: audit 2023-08-28T02:31:10.353511+0000 mon.c (mon.2) 3246 : audit [DBG] from='client.? 172.21.15.6:0/3588907191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:11.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:10 smithi006 bash[33918]: cluster 2023-08-28T02:31:09.746623+0000 mgr.y (mgr.14901) 8176 : cluster [DBG] pgmap v8183: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:11.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:10 smithi006 bash[33918]: audit 2023-08-28T02:31:10.353511+0000 mon.c (mon.2) 3246 : audit [DBG] from='client.? 172.21.15.6:0/3588907191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:31:11.321Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:31:11.323Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:12.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:11 smithi130 bash[51302]: debug 2023-08-28T02:31:11.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:11.701038+0000) 2023-08-28T02:31:13.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:12 smithi130 bash[27579]: cluster 2023-08-28T02:31:11.747290+0000 mgr.y (mgr.14901) 8177 : cluster [DBG] pgmap v8184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:12 smithi130 bash[27579]: audit 2023-08-28T02:31:12.768830+0000 mon.c (mon.2) 3247 : audit [DBG] from='client.? 172.21.15.6:0/1256411344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:13.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:12 smithi130 bash[51302]: debug 2023-08-28T02:31:12.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:12.701379+0000) 2023-08-28T02:31:13.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:12 smithi006 bash[33918]: cluster 2023-08-28T02:31:11.747290+0000 mgr.y (mgr.14901) 8177 : cluster [DBG] pgmap v8184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:12 smithi006 bash[33918]: audit 2023-08-28T02:31:12.768830+0000 mon.c (mon.2) 3247 : audit [DBG] from='client.? 172.21.15.6:0/1256411344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:31:12.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:31:12.889Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:13.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:12 smithi006 bash[27292]: cluster 2023-08-28T02:31:11.747290+0000 mgr.y (mgr.14901) 8177 : cluster [DBG] pgmap v8184: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:13.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:12 smithi006 bash[27292]: audit 2023-08-28T02:31:12.768830+0000 mon.c (mon.2) 3247 : audit [DBG] from='client.? 172.21.15.6:0/1256411344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:14.092 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:13 smithi130 bash[51302]: debug 2023-08-28T02:31:13.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:13.701657+0000) 2023-08-28T02:31:14.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:14 smithi130 bash[27579]: cluster 2023-08-28T02:31:13.747801+0000 mgr.y (mgr.14901) 8178 : cluster [DBG] pgmap v8185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:14.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:14 smithi006 bash[33918]: cluster 2023-08-28T02:31:13.747801+0000 mgr.y (mgr.14901) 8178 : cluster [DBG] pgmap v8185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:14 smithi006 bash[27292]: cluster 2023-08-28T02:31:13.747801+0000 mgr.y (mgr.14901) 8178 : cluster [DBG] pgmap v8185: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:14 smithi130 bash[51302]: debug 2023-08-28T02:31:14.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:14.701954+0000) 2023-08-28T02:31:15.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:31:14.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:15.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:31:14.767Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:15.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:15 smithi130 bash[27579]: audit 2023-08-28T02:31:15.181202+0000 mon.a (mon.0) 5163 : audit [DBG] from='client.? 172.21.15.6:0/2474900501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:15.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:15 smithi006 bash[27292]: audit 2023-08-28T02:31:15.181202+0000 mon.a (mon.0) 5163 : audit [DBG] from='client.? 172.21.15.6:0/2474900501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:15.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:15 smithi006 bash[33918]: audit 2023-08-28T02:31:15.181202+0000 mon.a (mon.0) 5163 : audit [DBG] from='client.? 172.21.15.6:0/2474900501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:16.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:15 smithi130 bash[51302]: debug 2023-08-28T02:31:15.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:15.702236+0000) 2023-08-28T02:31:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:16 smithi130 bash[27579]: cluster 2023-08-28T02:31:15.748908+0000 mgr.y (mgr.14901) 8179 : cluster [DBG] pgmap v8186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:16.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:16 smithi006 bash[33918]: cluster 2023-08-28T02:31:15.748908+0000 mgr.y (mgr.14901) 8179 : cluster [DBG] pgmap v8186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:16.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:16 smithi006 bash[27292]: cluster 2023-08-28T02:31:15.748908+0000 mgr.y (mgr.14901) 8179 : cluster [DBG] pgmap v8186: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:17.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:16 smithi130 bash[51302]: debug 2023-08-28T02:31:16.696+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:16.702513+0000) 2023-08-28T02:31:17.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:17 smithi006 bash[27292]: audit 2023-08-28T02:31:17.593670+0000 mon.a (mon.0) 5164 : audit [DBG] from='client.? 172.21.15.6:0/2224995587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:17.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:17 smithi006 bash[33918]: audit 2023-08-28T02:31:17.593670+0000 mon.a (mon.0) 5164 : audit [DBG] from='client.? 172.21.15.6:0/2224995587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:18.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:17 smithi130 bash[27579]: audit 2023-08-28T02:31:17.593670+0000 mon.a (mon.0) 5164 : audit [DBG] from='client.? 172.21.15.6:0/2224995587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:18.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:17 smithi130 bash[51302]: debug 2023-08-28T02:31:17.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:17.702819+0000) 2023-08-28T02:31:18.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:18 smithi006 bash[33918]: cluster 2023-08-28T02:31:17.749588+0000 mgr.y (mgr.14901) 8180 : cluster [DBG] pgmap v8187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:18.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:18 smithi006 bash[27292]: cluster 2023-08-28T02:31:17.749588+0000 mgr.y (mgr.14901) 8180 : cluster [DBG] pgmap v8187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:19.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:18 smithi130 bash[27579]: cluster 2023-08-28T02:31:17.749588+0000 mgr.y (mgr.14901) 8180 : cluster [DBG] pgmap v8187: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:19.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:18 smithi130 bash[51302]: debug 2023-08-28T02:31:18.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:18.703070+0000) 2023-08-28T02:31:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:19 smithi130 bash[51302]: debug 2023-08-28T02:31:19.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:19.703362+0000) 2023-08-28T02:31:21.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:20 smithi130 bash[27579]: cluster 2023-08-28T02:31:19.750497+0000 mgr.y (mgr.14901) 8181 : cluster [DBG] pgmap v8188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:21.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:20 smithi130 bash[27579]: audit 2023-08-28T02:31:20.007600+0000 mon.a (mon.0) 5165 : audit [DBG] from='client.? 172.21.15.6:0/4286383134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:21.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:20 smithi130 bash[27579]: audit 2023-08-28T02:31:20.301374+0000 mon.b (mon.1) 891 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:31:21.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:20 smithi130 bash[51302]: debug 2023-08-28T02:31:20.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:20.703606+0000) 2023-08-28T02:31:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:31:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:31:20] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:31:21.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:20 smithi006 bash[33918]: cluster 2023-08-28T02:31:19.750497+0000 mgr.y (mgr.14901) 8181 : cluster [DBG] pgmap v8188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:21.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:20 smithi006 bash[33918]: audit 2023-08-28T02:31:20.007600+0000 mon.a (mon.0) 5165 : audit [DBG] from='client.? 172.21.15.6:0/4286383134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:21.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:20 smithi006 bash[33918]: audit 2023-08-28T02:31:20.301374+0000 mon.b (mon.1) 891 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:31:21.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:20 smithi006 bash[27292]: cluster 2023-08-28T02:31:19.750497+0000 mgr.y (mgr.14901) 8181 : cluster [DBG] pgmap v8188: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:21.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:20 smithi006 bash[27292]: audit 2023-08-28T02:31:20.007600+0000 mon.a (mon.0) 5165 : audit [DBG] from='client.? 172.21.15.6:0/4286383134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:21.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:20 smithi006 bash[27292]: audit 2023-08-28T02:31:20.301374+0000 mon.b (mon.1) 891 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:31:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:31:21.322Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:31:21.324Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:22.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:21 smithi130 bash[51302]: debug 2023-08-28T02:31:21.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:21.703925+0000) 2023-08-28T02:31:23.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:22 smithi130 bash[27579]: cluster 2023-08-28T02:31:21.751173+0000 mgr.y (mgr.14901) 8182 : cluster [DBG] pgmap v8189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:23.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:22 smithi130 bash[27579]: audit 2023-08-28T02:31:22.420706+0000 mon.c (mon.2) 3248 : audit [DBG] from='client.? 172.21.15.6:0/3884190166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:23.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:22 smithi130 bash[51302]: debug 2023-08-28T02:31:22.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:22.704203+0000) 2023-08-28T02:31:23.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:22 smithi006 bash[27292]: cluster 2023-08-28T02:31:21.751173+0000 mgr.y (mgr.14901) 8182 : cluster [DBG] pgmap v8189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:23.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:22 smithi006 bash[27292]: audit 2023-08-28T02:31:22.420706+0000 mon.c (mon.2) 3248 : audit [DBG] from='client.? 172.21.15.6:0/3884190166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:23.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:22 smithi006 bash[33918]: cluster 2023-08-28T02:31:21.751173+0000 mgr.y (mgr.14901) 8182 : cluster [DBG] pgmap v8189: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:23.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:22 smithi006 bash[33918]: audit 2023-08-28T02:31:22.420706+0000 mon.c (mon.2) 3248 : audit [DBG] from='client.? 172.21.15.6:0/3884190166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:23.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:31:22.888Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:23.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:31:22.890Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:24.097 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:23 smithi130 bash[51302]: debug 2023-08-28T02:31:23.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:23.704535+0000) 2023-08-28T02:31:24.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:24 smithi130 bash[27579]: cluster 2023-08-28T02:31:23.751847+0000 mgr.y (mgr.14901) 8183 : cluster [DBG] pgmap v8190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:24.695 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:24 smithi006 bash[27292]: cluster 2023-08-28T02:31:23.751847+0000 mgr.y (mgr.14901) 8183 : cluster [DBG] pgmap v8190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:24.696 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:24 smithi006 bash[33918]: cluster 2023-08-28T02:31:23.751847+0000 mgr.y (mgr.14901) 8183 : cluster [DBG] pgmap v8190: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:25.099 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:31:24.766Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:25.100 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:31:24.768Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:25.100 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:24 smithi130 bash[51302]: debug 2023-08-28T02:31:24.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:24.704751+0000) 2023-08-28T02:31:25.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:25 smithi130 bash[27579]: audit 2023-08-28T02:31:24.854835+0000 mon.c (mon.2) 3249 : audit [DBG] from='client.? 172.21.15.6:0/3266352535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:25.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:25 smithi006 bash[33918]: audit 2023-08-28T02:31:24.854835+0000 mon.c (mon.2) 3249 : audit [DBG] from='client.? 172.21.15.6:0/3266352535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:25.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:25 smithi006 bash[27292]: audit 2023-08-28T02:31:24.854835+0000 mon.c (mon.2) 3249 : audit [DBG] from='client.? 172.21.15.6:0/3266352535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:26.110 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:25 smithi130 bash[51302]: debug 2023-08-28T02:31:25.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:25.705018+0000) 2023-08-28T02:31:26.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:26 smithi130 bash[27579]: cluster 2023-08-28T02:31:25.752607+0000 mgr.y (mgr.14901) 8184 : cluster [DBG] pgmap v8191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:26.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:26 smithi006 bash[33918]: cluster 2023-08-28T02:31:25.752607+0000 mgr.y (mgr.14901) 8184 : cluster [DBG] pgmap v8191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:26.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:26 smithi006 bash[27292]: cluster 2023-08-28T02:31:25.752607+0000 mgr.y (mgr.14901) 8184 : cluster [DBG] pgmap v8191: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:27.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:26 smithi130 bash[51302]: debug 2023-08-28T02:31:26.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:26.705266+0000) 2023-08-28T02:31:27.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:27 smithi130 bash[27579]: audit 2023-08-28T02:31:26.241152+0000 mon.a (mon.0) 5166 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:31:27.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:27 smithi130 bash[27579]: audit 2023-08-28T02:31:26.654346+0000 mon.a (mon.0) 5167 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:31:27.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:27 smithi130 bash[27579]: audit 2023-08-28T02:31:26.947937+0000 mon.b (mon.1) 892 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:31:27.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:27 smithi130 bash[27579]: audit 2023-08-28T02:31:26.949768+0000 mon.b (mon.1) 893 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:31:27.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:27 smithi130 bash[27579]: audit 2023-08-28T02:31:26.958849+0000 mon.a (mon.0) 5168 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:31:27.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:27 smithi006 bash[27292]: audit 2023-08-28T02:31:26.241152+0000 mon.a (mon.0) 5166 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:31:27.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:27 smithi006 bash[27292]: audit 2023-08-28T02:31:26.654346+0000 mon.a (mon.0) 5167 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:31:27.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:27 smithi006 bash[27292]: audit 2023-08-28T02:31:26.947937+0000 mon.b (mon.1) 892 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:31:27.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:27 smithi006 bash[27292]: audit 2023-08-28T02:31:26.949768+0000 mon.b (mon.1) 893 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:31:27.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:27 smithi006 bash[27292]: audit 2023-08-28T02:31:26.958849+0000 mon.a (mon.0) 5168 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:31:27.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:27 smithi006 bash[33918]: audit 2023-08-28T02:31:26.241152+0000 mon.a (mon.0) 5166 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:31:27.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:27 smithi006 bash[33918]: audit 2023-08-28T02:31:26.654346+0000 mon.a (mon.0) 5167 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:31:27.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:27 smithi006 bash[33918]: audit 2023-08-28T02:31:26.947937+0000 mon.b (mon.1) 892 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:31:27.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:27 smithi006 bash[33918]: audit 2023-08-28T02:31:26.949768+0000 mon.b (mon.1) 893 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:31:27.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:27 smithi006 bash[33918]: audit 2023-08-28T02:31:26.958849+0000 mon.a (mon.0) 5168 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:31:28.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:27 smithi130 bash[51302]: debug 2023-08-28T02:31:27.704+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:27.705540+0000) 2023-08-28T02:31:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:28 smithi130 bash[27579]: audit 2023-08-28T02:31:27.275276+0000 mon.c (mon.2) 3250 : audit [DBG] from='client.? 172.21.15.6:0/1898103187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:28.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:28 smithi130 bash[27579]: cluster 2023-08-28T02:31:27.753273+0000 mgr.y (mgr.14901) 8185 : cluster [DBG] pgmap v8192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:28 smithi006 bash[33918]: audit 2023-08-28T02:31:27.275276+0000 mon.c (mon.2) 3250 : audit [DBG] from='client.? 172.21.15.6:0/1898103187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:28.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:28 smithi006 bash[33918]: cluster 2023-08-28T02:31:27.753273+0000 mgr.y (mgr.14901) 8185 : cluster [DBG] pgmap v8192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:28.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:28 smithi006 bash[27292]: audit 2023-08-28T02:31:27.275276+0000 mon.c (mon.2) 3250 : audit [DBG] from='client.? 172.21.15.6:0/1898103187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:28.716 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:28 smithi006 bash[27292]: cluster 2023-08-28T02:31:27.753273+0000 mgr.y (mgr.14901) 8185 : cluster [DBG] pgmap v8192: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:28 smithi130 bash[51302]: debug 2023-08-28T02:31:28.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:28.705805+0000) 2023-08-28T02:31:30.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:29 smithi130 bash[27579]: audit 2023-08-28T02:31:29.697872+0000 mon.a (mon.0) 5169 : audit [DBG] from='client.? 172.21.15.6:0/2846678712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:30.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:29 smithi130 bash[51302]: debug 2023-08-28T02:31:29.704+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:29.706042+0000) 2023-08-28T02:31:30.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:29 smithi006 bash[33918]: audit 2023-08-28T02:31:29.697872+0000 mon.a (mon.0) 5169 : audit [DBG] from='client.? 172.21.15.6:0/2846678712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:30.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:29 smithi006 bash[27292]: audit 2023-08-28T02:31:29.697872+0000 mon.a (mon.0) 5169 : audit [DBG] from='client.? 172.21.15.6:0/2846678712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:31.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:30 smithi130 bash[27579]: cluster 2023-08-28T02:31:29.754438+0000 mgr.y (mgr.14901) 8186 : cluster [DBG] pgmap v8193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:31.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:30 smithi130 bash[51302]: debug 2023-08-28T02:31:30.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:30.706341+0000) 2023-08-28T02:31:31.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:31:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:31:30] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:31:31.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:30 smithi006 bash[33918]: cluster 2023-08-28T02:31:29.754438+0000 mgr.y (mgr.14901) 8186 : cluster [DBG] pgmap v8193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:31.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:30 smithi006 bash[27292]: cluster 2023-08-28T02:31:29.754438+0000 mgr.y (mgr.14901) 8186 : cluster [DBG] pgmap v8193: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:31:31.323Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:31:31.325Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:32.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:31 smithi130 bash[51302]: debug 2023-08-28T02:31:31.700+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:31.706648+0000) 2023-08-28T02:31:33.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:32 smithi130 bash[51302]: debug 2023-08-28T02:31:32.704+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:32.706931+0000) 2023-08-28T02:31:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:32 smithi130 bash[27579]: cluster 2023-08-28T02:31:31.755146+0000 mgr.y (mgr.14901) 8187 : cluster [DBG] pgmap v8194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:32 smithi130 bash[27579]: audit 2023-08-28T02:31:32.116648+0000 mon.a (mon.0) 5170 : audit [DBG] from='client.? 172.21.15.6:0/3496470336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:33.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:32 smithi006 bash[27292]: cluster 2023-08-28T02:31:31.755146+0000 mgr.y (mgr.14901) 8187 : cluster [DBG] pgmap v8194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:33.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:32 smithi006 bash[27292]: audit 2023-08-28T02:31:32.116648+0000 mon.a (mon.0) 5170 : audit [DBG] from='client.? 172.21.15.6:0/3496470336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:33.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:32 smithi006 bash[33918]: cluster 2023-08-28T02:31:31.755146+0000 mgr.y (mgr.14901) 8187 : cluster [DBG] pgmap v8194: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:33.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:32 smithi006 bash[33918]: audit 2023-08-28T02:31:32.116648+0000 mon.a (mon.0) 5170 : audit [DBG] from='client.? 172.21.15.6:0/3496470336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:31:32.889Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:33.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:31:32.891Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:34.097 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:33 smithi130 bash[51302]: debug 2023-08-28T02:31:33.704+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:33.707149+0000) 2023-08-28T02:31:34.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:34 smithi130 bash[27579]: cluster 2023-08-28T02:31:33.755799+0000 mgr.y (mgr.14901) 8188 : cluster [DBG] pgmap v8195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:34.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:34 smithi006 bash[33918]: cluster 2023-08-28T02:31:33.755799+0000 mgr.y (mgr.14901) 8188 : cluster [DBG] pgmap v8195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:34.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:34 smithi006 bash[27292]: cluster 2023-08-28T02:31:33.755799+0000 mgr.y (mgr.14901) 8188 : cluster [DBG] pgmap v8195: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:35.100 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:31:34.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:35.100 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:31:34.768Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:35.100 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:34 smithi130 bash[51302]: debug 2023-08-28T02:31:34.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:34.707402+0000) 2023-08-28T02:31:35.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:35 smithi130 bash[27579]: audit 2023-08-28T02:31:34.526350+0000 mon.a (mon.0) 5171 : audit [DBG] from='client.? 172.21.15.6:0/759278682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:35.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:35 smithi006 bash[27292]: audit 2023-08-28T02:31:34.526350+0000 mon.a (mon.0) 5171 : audit [DBG] from='client.? 172.21.15.6:0/759278682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:35.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:35 smithi006 bash[33918]: audit 2023-08-28T02:31:34.526350+0000 mon.a (mon.0) 5171 : audit [DBG] from='client.? 172.21.15.6:0/759278682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:36.109 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:35 smithi130 bash[51302]: debug 2023-08-28T02:31:35.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:35.707621+0000) 2023-08-28T02:31:36.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:36 smithi130 bash[27579]: cluster 2023-08-28T02:31:35.756956+0000 mgr.y (mgr.14901) 8189 : cluster [DBG] pgmap v8196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:36.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:36 smithi006 bash[33918]: cluster 2023-08-28T02:31:35.756956+0000 mgr.y (mgr.14901) 8189 : cluster [DBG] pgmap v8196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:36.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:36 smithi006 bash[27292]: cluster 2023-08-28T02:31:35.756956+0000 mgr.y (mgr.14901) 8189 : cluster [DBG] pgmap v8196: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:37.112 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:36 smithi130 bash[51302]: debug 2023-08-28T02:31:36.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:36.707938+0000) 2023-08-28T02:31:37.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:37 smithi130 bash[27579]: audit 2023-08-28T02:31:36.945880+0000 mon.c (mon.2) 3251 : audit [DBG] from='client.? 172.21.15.6:0/2157349256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:37.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:37 smithi006 bash[33918]: audit 2023-08-28T02:31:36.945880+0000 mon.c (mon.2) 3251 : audit [DBG] from='client.? 172.21.15.6:0/2157349256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:37.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:37 smithi006 bash[27292]: audit 2023-08-28T02:31:36.945880+0000 mon.c (mon.2) 3251 : audit [DBG] from='client.? 172.21.15.6:0/2157349256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:37 smithi130 bash[51302]: debug 2023-08-28T02:31:37.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:37.708232+0000) 2023-08-28T02:31:38.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:38 smithi006 bash[27292]: cluster 2023-08-28T02:31:37.757545+0000 mgr.y (mgr.14901) 8190 : cluster [DBG] pgmap v8197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:38.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:38 smithi006 bash[33918]: cluster 2023-08-28T02:31:37.757545+0000 mgr.y (mgr.14901) 8190 : cluster [DBG] pgmap v8197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:38 smithi130 bash[27579]: cluster 2023-08-28T02:31:37.757545+0000 mgr.y (mgr.14901) 8190 : cluster [DBG] pgmap v8197: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:39.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:38 smithi130 bash[51302]: debug 2023-08-28T02:31:38.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:38.708517+0000) 2023-08-28T02:31:39.707 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:39 smithi130 bash[27579]: audit 2023-08-28T02:31:39.352325+0000 mon.c (mon.2) 3252 : audit [DBG] from='client.? 172.21.15.6:0/4098520911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:39.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:39 smithi006 bash[33918]: audit 2023-08-28T02:31:39.352325+0000 mon.c (mon.2) 3252 : audit [DBG] from='client.? 172.21.15.6:0/4098520911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:39.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:39 smithi006 bash[27292]: audit 2023-08-28T02:31:39.352325+0000 mon.c (mon.2) 3252 : audit [DBG] from='client.? 172.21.15.6:0/4098520911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:40.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:39 smithi130 bash[51302]: debug 2023-08-28T02:31:39.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:39.708829+0000) 2023-08-28T02:31:40.708 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:40 smithi130 bash[27579]: cluster 2023-08-28T02:31:39.758636+0000 mgr.y (mgr.14901) 8191 : cluster [DBG] pgmap v8198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:40.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:40 smithi006 bash[33918]: cluster 2023-08-28T02:31:39.758636+0000 mgr.y (mgr.14901) 8191 : cluster [DBG] pgmap v8198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:40.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:40 smithi006 bash[27292]: cluster 2023-08-28T02:31:39.758636+0000 mgr.y (mgr.14901) 8191 : cluster [DBG] pgmap v8198: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:41.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:40 smithi130 bash[51302]: debug 2023-08-28T02:31:40.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:40.709110+0000) 2023-08-28T02:31:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:31:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:31:40] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:31:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:31:41.325Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:31:41.326Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:42.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:41 smithi130 bash[27579]: audit 2023-08-28T02:31:41.760420+0000 mon.c (mon.2) 3253 : audit [DBG] from='client.? 172.21.15.6:0/3116569497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:42.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:41 smithi130 bash[51302]: debug 2023-08-28T02:31:41.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:41.709403+0000) 2023-08-28T02:31:42.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:41 smithi006 bash[27292]: audit 2023-08-28T02:31:41.760420+0000 mon.c (mon.2) 3253 : audit [DBG] from='client.? 172.21.15.6:0/3116569497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:42.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:41 smithi006 bash[33918]: audit 2023-08-28T02:31:41.760420+0000 mon.c (mon.2) 3253 : audit [DBG] from='client.? 172.21.15.6:0/3116569497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:43.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:42 smithi130 bash[27579]: cluster 2023-08-28T02:31:41.759330+0000 mgr.y (mgr.14901) 8192 : cluster [DBG] pgmap v8199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:43.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:42 smithi130 bash[51302]: debug 2023-08-28T02:31:42.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:42.709632+0000) 2023-08-28T02:31:43.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:42 smithi006 bash[27292]: cluster 2023-08-28T02:31:41.759330+0000 mgr.y (mgr.14901) 8192 : cluster [DBG] pgmap v8199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:43.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:42 smithi006 bash[33918]: cluster 2023-08-28T02:31:41.759330+0000 mgr.y (mgr.14901) 8192 : cluster [DBG] pgmap v8199: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:31:42.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:31:42.892Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:44.101 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:43 smithi130 bash[51302]: debug 2023-08-28T02:31:43.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:43.709887+0000) 2023-08-28T02:31:44.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:44 smithi130 bash[27579]: cluster 2023-08-28T02:31:43.759998+0000 mgr.y (mgr.14901) 8193 : cluster [DBG] pgmap v8200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:44 smithi006 bash[33918]: cluster 2023-08-28T02:31:43.759998+0000 mgr.y (mgr.14901) 8193 : cluster [DBG] pgmap v8200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:44.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:44 smithi006 bash[27292]: cluster 2023-08-28T02:31:43.759998+0000 mgr.y (mgr.14901) 8193 : cluster [DBG] pgmap v8200: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:45.103 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:31:44.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:45.103 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:31:44.770Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:45.104 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:44 smithi130 bash[51302]: debug 2023-08-28T02:31:44.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:44.710211+0000) 2023-08-28T02:31:45.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:45 smithi130 bash[27579]: audit 2023-08-28T02:31:44.181785+0000 mon.c (mon.2) 3254 : audit [DBG] from='client.? 172.21.15.6:0/476076412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:45.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:45 smithi006 bash[27292]: audit 2023-08-28T02:31:44.181785+0000 mon.c (mon.2) 3254 : audit [DBG] from='client.? 172.21.15.6:0/476076412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:45.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:45 smithi006 bash[33918]: audit 2023-08-28T02:31:44.181785+0000 mon.c (mon.2) 3254 : audit [DBG] from='client.? 172.21.15.6:0/476076412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:46.114 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:45 smithi130 bash[51302]: debug 2023-08-28T02:31:45.703+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:45.710489+0000) 2023-08-28T02:31:46.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:46 smithi130 bash[27579]: cluster 2023-08-28T02:31:45.761130+0000 mgr.y (mgr.14901) 8194 : cluster [DBG] pgmap v8201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:46.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:46 smithi006 bash[33918]: cluster 2023-08-28T02:31:45.761130+0000 mgr.y (mgr.14901) 8194 : cluster [DBG] pgmap v8201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:46.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:46 smithi006 bash[27292]: cluster 2023-08-28T02:31:45.761130+0000 mgr.y (mgr.14901) 8194 : cluster [DBG] pgmap v8201: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:46 smithi130 bash[51302]: debug 2023-08-28T02:31:46.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:46.710763+0000) 2023-08-28T02:31:47.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:47 smithi006 bash[27292]: audit 2023-08-28T02:31:46.603093+0000 mon.a (mon.0) 5172 : audit [DBG] from='client.? 172.21.15.6:0/1175709570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:47.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:47 smithi006 bash[33918]: audit 2023-08-28T02:31:46.603093+0000 mon.a (mon.0) 5172 : audit [DBG] from='client.? 172.21.15.6:0/1175709570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:47.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:47 smithi130 bash[27579]: audit 2023-08-28T02:31:46.603093+0000 mon.a (mon.0) 5172 : audit [DBG] from='client.? 172.21.15.6:0/1175709570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:47 smithi130 bash[51302]: debug 2023-08-28T02:31:47.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:47.711025+0000) 2023-08-28T02:31:48.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:48 smithi006 bash[33918]: cluster 2023-08-28T02:31:47.761785+0000 mgr.y (mgr.14901) 8195 : cluster [DBG] pgmap v8202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:48.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:48 smithi006 bash[27292]: cluster 2023-08-28T02:31:47.761785+0000 mgr.y (mgr.14901) 8195 : cluster [DBG] pgmap v8202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:48 smithi130 bash[27579]: cluster 2023-08-28T02:31:47.761785+0000 mgr.y (mgr.14901) 8195 : cluster [DBG] pgmap v8202: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:49.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:48 smithi130 bash[51302]: debug 2023-08-28T02:31:48.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:48.711282+0000) 2023-08-28T02:31:49.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:49 smithi006 bash[27292]: audit 2023-08-28T02:31:49.024855+0000 mon.c (mon.2) 3255 : audit [DBG] from='client.? 172.21.15.6:0/3306649888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:49.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:49 smithi006 bash[33918]: audit 2023-08-28T02:31:49.024855+0000 mon.c (mon.2) 3255 : audit [DBG] from='client.? 172.21.15.6:0/3306649888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:49.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:49 smithi130 bash[27579]: audit 2023-08-28T02:31:49.024855+0000 mon.c (mon.2) 3255 : audit [DBG] from='client.? 172.21.15.6:0/3306649888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:50.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:49 smithi130 bash[51302]: debug 2023-08-28T02:31:49.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:49.711503+0000) 2023-08-28T02:31:50.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:50 smithi006 bash[33918]: cluster 2023-08-28T02:31:49.762824+0000 mgr.y (mgr.14901) 8196 : cluster [DBG] pgmap v8203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:50.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:50 smithi006 bash[27292]: cluster 2023-08-28T02:31:49.762824+0000 mgr.y (mgr.14901) 8196 : cluster [DBG] pgmap v8203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:50 smithi130 bash[27579]: cluster 2023-08-28T02:31:49.762824+0000 mgr.y (mgr.14901) 8196 : cluster [DBG] pgmap v8203: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:51.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:50 smithi130 bash[51302]: debug 2023-08-28T02:31:50.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:50.711778+0000) 2023-08-28T02:31:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:31:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:31:50] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:31:51.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:51 smithi006 bash[33918]: audit 2023-08-28T02:31:51.443098+0000 mon.a (mon.0) 5173 : audit [DBG] from='client.? 172.21.15.6:0/112151001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:51.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:51 smithi006 bash[27292]: audit 2023-08-28T02:31:51.443098+0000 mon.a (mon.0) 5173 : audit [DBG] from='client.? 172.21.15.6:0/112151001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:31:51.325Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:31:51.327Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:51.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:51 smithi130 bash[27579]: audit 2023-08-28T02:31:51.443098+0000 mon.a (mon.0) 5173 : audit [DBG] from='client.? 172.21.15.6:0/112151001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:51.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:51 smithi130 bash[51302]: debug 2023-08-28T02:31:51.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:51.712124+0000) 2023-08-28T02:31:52.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:52 smithi130 bash[27579]: cluster 2023-08-28T02:31:51.763487+0000 mgr.y (mgr.14901) 8197 : cluster [DBG] pgmap v8204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:52.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:52 smithi130 bash[51302]: debug 2023-08-28T02:31:52.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:52.712378+0000) 2023-08-28T02:31:52.890 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:52 smithi006 bash[27292]: cluster 2023-08-28T02:31:51.763487+0000 mgr.y (mgr.14901) 8197 : cluster [DBG] pgmap v8204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:52.891 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:52 smithi006 bash[33918]: cluster 2023-08-28T02:31:51.763487+0000 mgr.y (mgr.14901) 8197 : cluster [DBG] pgmap v8204: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:53.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:31:52.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:31:52.892Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:54.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:53 smithi130 bash[51302]: debug 2023-08-28T02:31:53.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:53.712646+0000) 2023-08-28T02:31:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:54 smithi006 bash[33918]: cluster 2023-08-28T02:31:53.764246+0000 mgr.y (mgr.14901) 8198 : cluster [DBG] pgmap v8205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:54 smithi006 bash[33918]: audit 2023-08-28T02:31:53.855912+0000 mon.a (mon.0) 5174 : audit [DBG] from='client.? 172.21.15.6:0/2909379199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:54 smithi006 bash[27292]: cluster 2023-08-28T02:31:53.764246+0000 mgr.y (mgr.14901) 8198 : cluster [DBG] pgmap v8205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:54 smithi006 bash[27292]: audit 2023-08-28T02:31:53.855912+0000 mon.a (mon.0) 5174 : audit [DBG] from='client.? 172.21.15.6:0/2909379199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:54 smithi130 bash[27579]: cluster 2023-08-28T02:31:53.764246+0000 mgr.y (mgr.14901) 8198 : cluster [DBG] pgmap v8205: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:54 smithi130 bash[27579]: audit 2023-08-28T02:31:53.855912+0000 mon.a (mon.0) 5174 : audit [DBG] from='client.? 172.21.15.6:0/2909379199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:55.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:54 smithi130 bash[51302]: debug 2023-08-28T02:31:54.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:54.712939+0000) 2023-08-28T02:31:55.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:31:54.768Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:55.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:31:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:31:54.771Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:31:56.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:55 smithi130 bash[51302]: debug 2023-08-28T02:31:55.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:55.713229+0000) 2023-08-28T02:31:57.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:56 smithi130 bash[27579]: cluster 2023-08-28T02:31:55.765399+0000 mgr.y (mgr.14901) 8199 : cluster [DBG] pgmap v8206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:57.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:56 smithi130 bash[27579]: audit 2023-08-28T02:31:56.277965+0000 mon.a (mon.0) 5175 : audit [DBG] from='client.? 172.21.15.6:0/3832245464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:57.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:56 smithi130 bash[51302]: debug 2023-08-28T02:31:56.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:56.713522+0000) 2023-08-28T02:31:57.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:56 smithi006 bash[27292]: cluster 2023-08-28T02:31:55.765399+0000 mgr.y (mgr.14901) 8199 : cluster [DBG] pgmap v8206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:57.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:56 smithi006 bash[27292]: audit 2023-08-28T02:31:56.277965+0000 mon.a (mon.0) 5175 : audit [DBG] from='client.? 172.21.15.6:0/3832245464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:57.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:56 smithi006 bash[33918]: cluster 2023-08-28T02:31:55.765399+0000 mgr.y (mgr.14901) 8199 : cluster [DBG] pgmap v8206: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:57.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:56 smithi006 bash[33918]: audit 2023-08-28T02:31:56.277965+0000 mon.a (mon.0) 5175 : audit [DBG] from='client.? 172.21.15.6:0/3832245464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:58.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:57 smithi130 bash[51302]: debug 2023-08-28T02:31:57.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:57.713784+0000) 2023-08-28T02:31:59.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:58 smithi130 bash[27579]: cluster 2023-08-28T02:31:57.766174+0000 mgr.y (mgr.14901) 8200 : cluster [DBG] pgmap v8207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:59.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:31:58 smithi130 bash[27579]: audit 2023-08-28T02:31:58.691834+0000 mon.c (mon.2) 3256 : audit [DBG] from='client.? 172.21.15.6:0/3663034095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:59.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:58 smithi130 bash[51302]: debug 2023-08-28T02:31:58.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:58.714058+0000) 2023-08-28T02:31:59.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:58 smithi006 bash[33918]: cluster 2023-08-28T02:31:57.766174+0000 mgr.y (mgr.14901) 8200 : cluster [DBG] pgmap v8207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:59.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:31:58 smithi006 bash[33918]: audit 2023-08-28T02:31:58.691834+0000 mon.c (mon.2) 3256 : audit [DBG] from='client.? 172.21.15.6:0/3663034095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:59.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:58 smithi006 bash[27292]: cluster 2023-08-28T02:31:57.766174+0000 mgr.y (mgr.14901) 8200 : cluster [DBG] pgmap v8207: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:31:59.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:31:58 smithi006 bash[27292]: audit 2023-08-28T02:31:58.691834+0000 mon.c (mon.2) 3256 : audit [DBG] from='client.? 172.21.15.6:0/3663034095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:31:59 smithi130 bash[51302]: debug 2023-08-28T02:31:59.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:59.714391+0000) 2023-08-28T02:32:01.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:00 smithi130 bash[27579]: cluster 2023-08-28T02:31:59.767444+0000 mgr.y (mgr.14901) 8201 : cluster [DBG] pgmap v8208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:01.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:00 smithi130 bash[51302]: debug 2023-08-28T02:32:00.707+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:00.714700+0000) 2023-08-28T02:32:01.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:00 smithi006 bash[27292]: cluster 2023-08-28T02:31:59.767444+0000 mgr.y (mgr.14901) 8201 : cluster [DBG] pgmap v8208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:01.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:32:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:32:00] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:32:01.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:00 smithi006 bash[33918]: cluster 2023-08-28T02:31:59.767444+0000 mgr.y (mgr.14901) 8201 : cluster [DBG] pgmap v8208: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:32:01.326Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:32:01.327Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:02.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:01 smithi130 bash[27579]: audit 2023-08-28T02:32:01.111246+0000 mon.c (mon.2) 3257 : audit [DBG] from='client.? 172.21.15.6:0/1317773807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:02.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:01 smithi130 bash[51302]: debug 2023-08-28T02:32:01.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:01.715010+0000) 2023-08-28T02:32:02.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:01 smithi006 bash[33918]: audit 2023-08-28T02:32:01.111246+0000 mon.c (mon.2) 3257 : audit [DBG] from='client.? 172.21.15.6:0/1317773807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:02.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:01 smithi006 bash[27292]: audit 2023-08-28T02:32:01.111246+0000 mon.c (mon.2) 3257 : audit [DBG] from='client.? 172.21.15.6:0/1317773807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:03.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:02 smithi130 bash[27579]: cluster 2023-08-28T02:32:01.768126+0000 mgr.y (mgr.14901) 8202 : cluster [DBG] pgmap v8209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:03.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:02 smithi130 bash[51302]: debug 2023-08-28T02:32:02.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:02.715286+0000) 2023-08-28T02:32:03.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:02 smithi006 bash[27292]: cluster 2023-08-28T02:32:01.768126+0000 mgr.y (mgr.14901) 8202 : cluster [DBG] pgmap v8209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:03.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:02 smithi006 bash[33918]: cluster 2023-08-28T02:32:01.768126+0000 mgr.y (mgr.14901) 8202 : cluster [DBG] pgmap v8209: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:32:02.891Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:32:02.893Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:04.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:03 smithi130 bash[51302]: debug 2023-08-28T02:32:03.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:03.715505+0000) 2023-08-28T02:32:04.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:04 smithi006 bash[33918]: audit 2023-08-28T02:32:03.526678+0000 mon.a (mon.0) 5176 : audit [DBG] from='client.? 172.21.15.6:0/531447762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:04.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:04 smithi006 bash[33918]: cluster 2023-08-28T02:32:03.768809+0000 mgr.y (mgr.14901) 8203 : cluster [DBG] pgmap v8210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:04 smithi006 bash[27292]: audit 2023-08-28T02:32:03.526678+0000 mon.a (mon.0) 5176 : audit [DBG] from='client.? 172.21.15.6:0/531447762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:04 smithi006 bash[27292]: cluster 2023-08-28T02:32:03.768809+0000 mgr.y (mgr.14901) 8203 : cluster [DBG] pgmap v8210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:04 smithi130 bash[27579]: audit 2023-08-28T02:32:03.526678+0000 mon.a (mon.0) 5176 : audit [DBG] from='client.? 172.21.15.6:0/531447762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:04 smithi130 bash[27579]: cluster 2023-08-28T02:32:03.768809+0000 mgr.y (mgr.14901) 8203 : cluster [DBG] pgmap v8210: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:05.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:04 smithi130 bash[51302]: debug 2023-08-28T02:32:04.715+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:04.715798+0000) 2023-08-28T02:32:05.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:32:04.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:05.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:32:04.771Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:06.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:05 smithi130 bash[51302]: debug 2023-08-28T02:32:05.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:05.716085+0000) 2023-08-28T02:32:07.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:06 smithi130 bash[27579]: cluster 2023-08-28T02:32:05.769886+0000 mgr.y (mgr.14901) 8204 : cluster [DBG] pgmap v8211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:07.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:06 smithi130 bash[27579]: audit 2023-08-28T02:32:05.948162+0000 mon.c (mon.2) 3258 : audit [DBG] from='client.? 172.21.15.6:0/2067662018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:07.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:06 smithi130 bash[51302]: debug 2023-08-28T02:32:06.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:06.716330+0000) 2023-08-28T02:32:07.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:06 smithi006 bash[33918]: cluster 2023-08-28T02:32:05.769886+0000 mgr.y (mgr.14901) 8204 : cluster [DBG] pgmap v8211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:07.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:06 smithi006 bash[33918]: audit 2023-08-28T02:32:05.948162+0000 mon.c (mon.2) 3258 : audit [DBG] from='client.? 172.21.15.6:0/2067662018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:07.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:06 smithi006 bash[27292]: cluster 2023-08-28T02:32:05.769886+0000 mgr.y (mgr.14901) 8204 : cluster [DBG] pgmap v8211: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:07.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:06 smithi006 bash[27292]: audit 2023-08-28T02:32:05.948162+0000 mon.c (mon.2) 3258 : audit [DBG] from='client.? 172.21.15.6:0/2067662018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:08.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:07 smithi130 bash[51302]: debug 2023-08-28T02:32:07.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:07.716612+0000) 2023-08-28T02:32:09.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:08 smithi130 bash[27579]: cluster 2023-08-28T02:32:07.770536+0000 mgr.y (mgr.14901) 8205 : cluster [DBG] pgmap v8212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:09.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:08 smithi130 bash[27579]: audit 2023-08-28T02:32:08.371921+0000 mon.a (mon.0) 5177 : audit [DBG] from='client.? 172.21.15.6:0/2834108312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:09.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:08 smithi130 bash[51302]: debug 2023-08-28T02:32:08.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:08.716930+0000) 2023-08-28T02:32:09.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:08 smithi006 bash[33918]: cluster 2023-08-28T02:32:07.770536+0000 mgr.y (mgr.14901) 8205 : cluster [DBG] pgmap v8212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:09.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:08 smithi006 bash[33918]: audit 2023-08-28T02:32:08.371921+0000 mon.a (mon.0) 5177 : audit [DBG] from='client.? 172.21.15.6:0/2834108312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:09.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:08 smithi006 bash[27292]: cluster 2023-08-28T02:32:07.770536+0000 mgr.y (mgr.14901) 8205 : cluster [DBG] pgmap v8212: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:09.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:08 smithi006 bash[27292]: audit 2023-08-28T02:32:08.371921+0000 mon.a (mon.0) 5177 : audit [DBG] from='client.? 172.21.15.6:0/2834108312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:10.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:09 smithi130 bash[51302]: debug 2023-08-28T02:32:09.715+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:09.717192+0000) 2023-08-28T02:32:11.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:10 smithi130 bash[51302]: debug 2023-08-28T02:32:10.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:10.717505+0000) 2023-08-28T02:32:11.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:10 smithi130 bash[27579]: cluster 2023-08-28T02:32:09.771539+0000 mgr.y (mgr.14901) 8206 : cluster [DBG] pgmap v8213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:11.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:10 smithi130 bash[27579]: audit 2023-08-28T02:32:10.789023+0000 mon.c (mon.2) 3259 : audit [DBG] from='client.? 172.21.15.6:0/2985451254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:11.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:32:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:32:10] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:32:11.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:10 smithi006 bash[33918]: cluster 2023-08-28T02:32:09.771539+0000 mgr.y (mgr.14901) 8206 : cluster [DBG] pgmap v8213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:11.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:10 smithi006 bash[33918]: audit 2023-08-28T02:32:10.789023+0000 mon.c (mon.2) 3259 : audit [DBG] from='client.? 172.21.15.6:0/2985451254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:11.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:10 smithi006 bash[27292]: cluster 2023-08-28T02:32:09.771539+0000 mgr.y (mgr.14901) 8206 : cluster [DBG] pgmap v8213: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:11.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:10 smithi006 bash[27292]: audit 2023-08-28T02:32:10.789023+0000 mon.c (mon.2) 3259 : audit [DBG] from='client.? 172.21.15.6:0/2985451254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:32:11.327Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:32:11.329Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:12.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:11 smithi130 bash[51302]: debug 2023-08-28T02:32:11.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:11.717861+0000) 2023-08-28T02:32:13.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:12 smithi130 bash[51302]: debug 2023-08-28T02:32:12.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:12.718138+0000) 2023-08-28T02:32:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:12 smithi130 bash[27579]: cluster 2023-08-28T02:32:11.772220+0000 mgr.y (mgr.14901) 8207 : cluster [DBG] pgmap v8214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:32:12.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:32:12.894Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:13.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:12 smithi006 bash[27292]: cluster 2023-08-28T02:32:11.772220+0000 mgr.y (mgr.14901) 8207 : cluster [DBG] pgmap v8214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:12 smithi006 bash[33918]: cluster 2023-08-28T02:32:11.772220+0000 mgr.y (mgr.14901) 8207 : cluster [DBG] pgmap v8214: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:14.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:13 smithi130 bash[51302]: debug 2023-08-28T02:32:13.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:13.718459+0000) 2023-08-28T02:32:14.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:14 smithi006 bash[33918]: audit 2023-08-28T02:32:13.209188+0000 mon.c (mon.2) 3260 : audit [DBG] from='client.? 172.21.15.6:0/4105333483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:14.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:14 smithi006 bash[33918]: cluster 2023-08-28T02:32:13.772881+0000 mgr.y (mgr.14901) 8208 : cluster [DBG] pgmap v8215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:14 smithi006 bash[27292]: audit 2023-08-28T02:32:13.209188+0000 mon.c (mon.2) 3260 : audit [DBG] from='client.? 172.21.15.6:0/4105333483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:14 smithi006 bash[27292]: cluster 2023-08-28T02:32:13.772881+0000 mgr.y (mgr.14901) 8208 : cluster [DBG] pgmap v8215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:14 smithi130 bash[27579]: audit 2023-08-28T02:32:13.209188+0000 mon.c (mon.2) 3260 : audit [DBG] from='client.? 172.21.15.6:0/4105333483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:14 smithi130 bash[27579]: cluster 2023-08-28T02:32:13.772881+0000 mgr.y (mgr.14901) 8208 : cluster [DBG] pgmap v8215: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:14 smithi130 bash[51302]: debug 2023-08-28T02:32:14.711+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:14.718737+0000) 2023-08-28T02:32:15.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:32:14.770Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:15.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:32:14.772Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:15.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:15 smithi006 bash[27292]: audit 2023-08-28T02:32:15.632261+0000 mon.c (mon.2) 3261 : audit [DBG] from='client.? 172.21.15.6:0/2416473086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:15.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:15 smithi006 bash[33918]: audit 2023-08-28T02:32:15.632261+0000 mon.c (mon.2) 3261 : audit [DBG] from='client.? 172.21.15.6:0/2416473086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:16.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:15 smithi130 bash[27579]: audit 2023-08-28T02:32:15.632261+0000 mon.c (mon.2) 3261 : audit [DBG] from='client.? 172.21.15.6:0/2416473086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:16.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:15 smithi130 bash[51302]: debug 2023-08-28T02:32:15.715+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:15.719056+0000) 2023-08-28T02:32:16.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:16 smithi006 bash[33918]: cluster 2023-08-28T02:32:15.773826+0000 mgr.y (mgr.14901) 8209 : cluster [DBG] pgmap v8216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:16.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:16 smithi006 bash[27292]: cluster 2023-08-28T02:32:15.773826+0000 mgr.y (mgr.14901) 8209 : cluster [DBG] pgmap v8216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:17.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:16 smithi130 bash[27579]: cluster 2023-08-28T02:32:15.773826+0000 mgr.y (mgr.14901) 8209 : cluster [DBG] pgmap v8216: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:17.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:16 smithi130 bash[51302]: debug 2023-08-28T02:32:16.715+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:16.719316+0000) 2023-08-28T02:32:18.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:17 smithi130 bash[51302]: debug 2023-08-28T02:32:17.719+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:17.719594+0000) 2023-08-28T02:32:19.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:18 smithi130 bash[27579]: cluster 2023-08-28T02:32:17.774430+0000 mgr.y (mgr.14901) 8210 : cluster [DBG] pgmap v8217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:19.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:18 smithi130 bash[27579]: audit 2023-08-28T02:32:18.047516+0000 mon.c (mon.2) 3262 : audit [DBG] from='client.? 172.21.15.6:0/4159428844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:19.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:18 smithi130 bash[51302]: debug 2023-08-28T02:32:18.719+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:18.719910+0000) 2023-08-28T02:32:19.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:18 smithi006 bash[27292]: cluster 2023-08-28T02:32:17.774430+0000 mgr.y (mgr.14901) 8210 : cluster [DBG] pgmap v8217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:19.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:18 smithi006 bash[27292]: audit 2023-08-28T02:32:18.047516+0000 mon.c (mon.2) 3262 : audit [DBG] from='client.? 172.21.15.6:0/4159428844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:19.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:18 smithi006 bash[33918]: cluster 2023-08-28T02:32:17.774430+0000 mgr.y (mgr.14901) 8210 : cluster [DBG] pgmap v8217: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:19.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:18 smithi006 bash[33918]: audit 2023-08-28T02:32:18.047516+0000 mon.c (mon.2) 3262 : audit [DBG] from='client.? 172.21.15.6:0/4159428844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:19 smithi130 bash[51302]: debug 2023-08-28T02:32:19.715+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:19.720206+0000) 2023-08-28T02:32:21.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:20 smithi130 bash[27579]: cluster 2023-08-28T02:32:19.775570+0000 mgr.y (mgr.14901) 8211 : cluster [DBG] pgmap v8218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:21.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:20 smithi130 bash[27579]: audit 2023-08-28T02:32:20.477690+0000 mon.a (mon.0) 5178 : audit [DBG] from='client.? 172.21.15.6:0/3877963061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:21.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:20 smithi130 bash[51302]: debug 2023-08-28T02:32:20.715+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:20.720481+0000) 2023-08-28T02:32:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:32:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:32:20] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T02:32:21.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:20 smithi006 bash[27292]: cluster 2023-08-28T02:32:19.775570+0000 mgr.y (mgr.14901) 8211 : cluster [DBG] pgmap v8218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:21.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:20 smithi006 bash[27292]: audit 2023-08-28T02:32:20.477690+0000 mon.a (mon.0) 5178 : audit [DBG] from='client.? 172.21.15.6:0/3877963061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:21.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:20 smithi006 bash[33918]: cluster 2023-08-28T02:32:19.775570+0000 mgr.y (mgr.14901) 8211 : cluster [DBG] pgmap v8218: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:21.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:20 smithi006 bash[33918]: audit 2023-08-28T02:32:20.477690+0000 mon.a (mon.0) 5178 : audit [DBG] from='client.? 172.21.15.6:0/3877963061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:32:21.328Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:32:21.330Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:22.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:21 smithi130 bash[51302]: debug 2023-08-28T02:32:21.719+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:21.720770+0000) 2023-08-28T02:32:23.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:22 smithi130 bash[51302]: debug 2023-08-28T02:32:22.715+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:22.721070+0000) 2023-08-28T02:32:23.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:22 smithi130 bash[27579]: cluster 2023-08-28T02:32:21.776229+0000 mgr.y (mgr.14901) 8212 : cluster [DBG] pgmap v8219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:23.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:22 smithi006 bash[27292]: cluster 2023-08-28T02:32:21.776229+0000 mgr.y (mgr.14901) 8212 : cluster [DBG] pgmap v8219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:23.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:22 smithi006 bash[33918]: cluster 2023-08-28T02:32:21.776229+0000 mgr.y (mgr.14901) 8212 : cluster [DBG] pgmap v8219: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:32:22.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:32:22.893Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:24.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:23 smithi130 bash[27579]: audit 2023-08-28T02:32:22.890414+0000 mon.a (mon.0) 5179 : audit [DBG] from='client.? 172.21.15.6:0/1042112358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:24.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:23 smithi130 bash[51302]: debug 2023-08-28T02:32:23.715+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:23.721338+0000) 2023-08-28T02:32:24.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:23 smithi006 bash[27292]: audit 2023-08-28T02:32:22.890414+0000 mon.a (mon.0) 5179 : audit [DBG] from='client.? 172.21.15.6:0/1042112358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:24.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:23 smithi006 bash[33918]: audit 2023-08-28T02:32:22.890414+0000 mon.a (mon.0) 5179 : audit [DBG] from='client.? 172.21.15.6:0/1042112358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:25.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:24 smithi130 bash[27579]: cluster 2023-08-28T02:32:23.776903+0000 mgr.y (mgr.14901) 8213 : cluster [DBG] pgmap v8220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:25.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:24 smithi130 bash[51302]: debug 2023-08-28T02:32:24.715+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:24.721801+0000) 2023-08-28T02:32:25.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:24 smithi006 bash[27292]: cluster 2023-08-28T02:32:23.776903+0000 mgr.y (mgr.14901) 8213 : cluster [DBG] pgmap v8220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:25.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:24 smithi006 bash[33918]: cluster 2023-08-28T02:32:23.776903+0000 mgr.y (mgr.14901) 8213 : cluster [DBG] pgmap v8220: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:25.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:32:24.770Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:25.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:32:24.773Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:26.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:25 smithi130 bash[27579]: audit 2023-08-28T02:32:25.301838+0000 mon.a (mon.0) 5180 : audit [DBG] from='client.? 172.21.15.6:0/2065261343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:26.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:25 smithi130 bash[51302]: debug 2023-08-28T02:32:25.719+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:25.722075+0000) 2023-08-28T02:32:26.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:25 smithi006 bash[33918]: audit 2023-08-28T02:32:25.301838+0000 mon.a (mon.0) 5180 : audit [DBG] from='client.? 172.21.15.6:0/2065261343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:26.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:25 smithi006 bash[27292]: audit 2023-08-28T02:32:25.301838+0000 mon.a (mon.0) 5180 : audit [DBG] from='client.? 172.21.15.6:0/2065261343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:27.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:26 smithi130 bash[51302]: debug 2023-08-28T02:32:26.715+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:26.722342+0000) 2023-08-28T02:32:27.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:26 smithi130 bash[27579]: cluster 2023-08-28T02:32:25.778033+0000 mgr.y (mgr.14901) 8214 : cluster [DBG] pgmap v8221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:27.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:26 smithi006 bash[27292]: cluster 2023-08-28T02:32:25.778033+0000 mgr.y (mgr.14901) 8214 : cluster [DBG] pgmap v8221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:27.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:26 smithi006 bash[33918]: cluster 2023-08-28T02:32:25.778033+0000 mgr.y (mgr.14901) 8214 : cluster [DBG] pgmap v8221: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:28.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:27 smithi130 bash[27579]: audit 2023-08-28T02:32:26.969281+0000 mon.b (mon.1) 894 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:32:28.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:27 smithi130 bash[27579]: audit 2023-08-28T02:32:27.712297+0000 mon.a (mon.0) 5181 : audit [DBG] from='client.? 172.21.15.6:0/3167157624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:28.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:27 smithi130 bash[51302]: debug 2023-08-28T02:32:27.715+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:27.722613+0000) 2023-08-28T02:32:28.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:27 smithi006 bash[27292]: audit 2023-08-28T02:32:26.969281+0000 mon.b (mon.1) 894 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:32:28.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:27 smithi006 bash[27292]: audit 2023-08-28T02:32:27.712297+0000 mon.a (mon.0) 5181 : audit [DBG] from='client.? 172.21.15.6:0/3167157624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:28.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:27 smithi006 bash[33918]: audit 2023-08-28T02:32:26.969281+0000 mon.b (mon.1) 894 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:32:28.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:27 smithi006 bash[33918]: audit 2023-08-28T02:32:27.712297+0000 mon.a (mon.0) 5181 : audit [DBG] from='client.? 172.21.15.6:0/3167157624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:29.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:28 smithi130 bash[27579]: cluster 2023-08-28T02:32:27.778689+0000 mgr.y (mgr.14901) 8215 : cluster [DBG] pgmap v8222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:29.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:28 smithi130 bash[51302]: debug 2023-08-28T02:32:28.718+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:28.722939+0000) 2023-08-28T02:32:29.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:28 smithi006 bash[27292]: cluster 2023-08-28T02:32:27.778689+0000 mgr.y (mgr.14901) 8215 : cluster [DBG] pgmap v8222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:29.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:28 smithi006 bash[33918]: cluster 2023-08-28T02:32:27.778689+0000 mgr.y (mgr.14901) 8215 : cluster [DBG] pgmap v8222: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:30.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:29 smithi130 bash[51302]: debug 2023-08-28T02:32:29.718+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:29.723191+0000) 2023-08-28T02:32:31.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:30 smithi130 bash[27579]: cluster 2023-08-28T02:32:29.783002+0000 mgr.y (mgr.14901) 8216 : cluster [DBG] pgmap v8223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:31.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:30 smithi130 bash[27579]: audit 2023-08-28T02:32:30.129595+0000 mon.a (mon.0) 5182 : audit [DBG] from='client.? 172.21.15.6:0/3567519156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:31.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:30 smithi130 bash[51302]: debug 2023-08-28T02:32:30.718+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:30.723417+0000) 2023-08-28T02:32:31.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:32:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:32:30] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-28T02:32:31.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:30 smithi006 bash[27292]: cluster 2023-08-28T02:32:29.783002+0000 mgr.y (mgr.14901) 8216 : cluster [DBG] pgmap v8223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:31.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:30 smithi006 bash[27292]: audit 2023-08-28T02:32:30.129595+0000 mon.a (mon.0) 5182 : audit [DBG] from='client.? 172.21.15.6:0/3567519156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:31.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:30 smithi006 bash[33918]: cluster 2023-08-28T02:32:29.783002+0000 mgr.y (mgr.14901) 8216 : cluster [DBG] pgmap v8223: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:31.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:30 smithi006 bash[33918]: audit 2023-08-28T02:32:30.129595+0000 mon.a (mon.0) 5182 : audit [DBG] from='client.? 172.21.15.6:0/3567519156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:32:31.329Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:32:31.331Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:32.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:31 smithi130 bash[51302]: debug 2023-08-28T02:32:31.718+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:31.723722+0000) 2023-08-28T02:32:33.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:32 smithi130 bash[27579]: cluster 2023-08-28T02:32:31.783656+0000 mgr.y (mgr.14901) 8217 : cluster [DBG] pgmap v8224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:32 smithi130 bash[27579]: audit 2023-08-28T02:32:32.590137+0000 mon.a (mon.0) 5183 : audit [DBG] from='client.? 172.21.15.6:0/1793902115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:33.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:32 smithi130 bash[51302]: debug 2023-08-28T02:32:32.718+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:32.724017+0000) 2023-08-28T02:32:33.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:32 smithi006 bash[27292]: cluster 2023-08-28T02:32:31.783656+0000 mgr.y (mgr.14901) 8217 : cluster [DBG] pgmap v8224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:33.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:32 smithi006 bash[27292]: audit 2023-08-28T02:32:32.590137+0000 mon.a (mon.0) 5183 : audit [DBG] from='client.? 172.21.15.6:0/1793902115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:33.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:32 smithi006 bash[33918]: cluster 2023-08-28T02:32:31.783656+0000 mgr.y (mgr.14901) 8217 : cluster [DBG] pgmap v8224: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:33.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:32 smithi006 bash[33918]: audit 2023-08-28T02:32:32.590137+0000 mon.a (mon.0) 5183 : audit [DBG] from='client.? 172.21.15.6:0/1793902115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:32:32.893Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:32:32.894Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:34.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:33 smithi130 bash[51302]: debug 2023-08-28T02:32:33.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:33.724308+0000) 2023-08-28T02:32:34.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:33 smithi130 bash[27579]: audit 2023-08-28T02:32:32.879959+0000 mon.a (mon.0) 5184 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:32:34.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:33 smithi130 bash[27579]: audit 2023-08-28T02:32:33.257721+0000 mon.a (mon.0) 5185 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:32:34.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:33 smithi130 bash[27579]: audit 2023-08-28T02:32:33.527921+0000 mon.b (mon.1) 895 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:32:34.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:33 smithi130 bash[27579]: audit 2023-08-28T02:32:33.530218+0000 mon.b (mon.1) 896 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:32:34.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:33 smithi130 bash[27579]: audit 2023-08-28T02:32:33.537770+0000 mon.a (mon.0) 5186 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:32:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:33 smithi006 bash[33918]: audit 2023-08-28T02:32:32.879959+0000 mon.a (mon.0) 5184 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:32:34.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:33 smithi006 bash[33918]: audit 2023-08-28T02:32:33.257721+0000 mon.a (mon.0) 5185 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:32:34.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:33 smithi006 bash[33918]: audit 2023-08-28T02:32:33.527921+0000 mon.b (mon.1) 895 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:32:34.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:33 smithi006 bash[33918]: audit 2023-08-28T02:32:33.530218+0000 mon.b (mon.1) 896 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:32:34.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:33 smithi006 bash[33918]: audit 2023-08-28T02:32:33.537770+0000 mon.a (mon.0) 5186 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:32:34.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:33 smithi006 bash[27292]: audit 2023-08-28T02:32:32.879959+0000 mon.a (mon.0) 5184 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:32:34.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:33 smithi006 bash[27292]: audit 2023-08-28T02:32:33.257721+0000 mon.a (mon.0) 5185 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:32:34.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:33 smithi006 bash[27292]: audit 2023-08-28T02:32:33.527921+0000 mon.b (mon.1) 895 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:32:34.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:33 smithi006 bash[27292]: audit 2023-08-28T02:32:33.530218+0000 mon.b (mon.1) 896 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:32:34.217 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:33 smithi006 bash[27292]: audit 2023-08-28T02:32:33.537770+0000 mon.a (mon.0) 5186 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:32:35.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:34 smithi130 bash[27579]: cluster 2023-08-28T02:32:33.784320+0000 mgr.y (mgr.14901) 8218 : cluster [DBG] pgmap v8225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:35.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:34 smithi130 bash[51302]: debug 2023-08-28T02:32:34.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:34.724528+0000) 2023-08-28T02:32:35.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:34 smithi006 bash[33918]: cluster 2023-08-28T02:32:33.784320+0000 mgr.y (mgr.14901) 8218 : cluster [DBG] pgmap v8225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:35.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:34 smithi006 bash[27292]: cluster 2023-08-28T02:32:33.784320+0000 mgr.y (mgr.14901) 8218 : cluster [DBG] pgmap v8225: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:35.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:32:34.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:35.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:32:34.773Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:36.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:35 smithi130 bash[27579]: audit 2023-08-28T02:32:35.015672+0000 mon.c (mon.2) 3263 : audit [DBG] from='client.? 172.21.15.6:0/902350042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:36.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:35 smithi130 bash[51302]: debug 2023-08-28T02:32:35.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:35.724800+0000) 2023-08-28T02:32:36.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:35 smithi006 bash[33918]: audit 2023-08-28T02:32:35.015672+0000 mon.c (mon.2) 3263 : audit [DBG] from='client.? 172.21.15.6:0/902350042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:36.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:35 smithi006 bash[27292]: audit 2023-08-28T02:32:35.015672+0000 mon.c (mon.2) 3263 : audit [DBG] from='client.? 172.21.15.6:0/902350042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:37.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:36 smithi130 bash[27579]: cluster 2023-08-28T02:32:35.785404+0000 mgr.y (mgr.14901) 8219 : cluster [DBG] pgmap v8226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:37.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:36 smithi130 bash[51302]: debug 2023-08-28T02:32:36.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:36.725113+0000) 2023-08-28T02:32:37.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:36 smithi006 bash[27292]: cluster 2023-08-28T02:32:35.785404+0000 mgr.y (mgr.14901) 8219 : cluster [DBG] pgmap v8226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:37.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:36 smithi006 bash[33918]: cluster 2023-08-28T02:32:35.785404+0000 mgr.y (mgr.14901) 8219 : cluster [DBG] pgmap v8226: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:38.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:37 smithi130 bash[27579]: audit 2023-08-28T02:32:37.436167+0000 mon.c (mon.2) 3264 : audit [DBG] from='client.? 172.21.15.6:0/1005627875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:38.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:37 smithi130 bash[51302]: debug 2023-08-28T02:32:37.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:37.725405+0000) 2023-08-28T02:32:38.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:37 smithi006 bash[33918]: audit 2023-08-28T02:32:37.436167+0000 mon.c (mon.2) 3264 : audit [DBG] from='client.? 172.21.15.6:0/1005627875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:38.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:37 smithi006 bash[27292]: audit 2023-08-28T02:32:37.436167+0000 mon.c (mon.2) 3264 : audit [DBG] from='client.? 172.21.15.6:0/1005627875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:39.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:38 smithi130 bash[27579]: cluster 2023-08-28T02:32:37.786009+0000 mgr.y (mgr.14901) 8220 : cluster [DBG] pgmap v8227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:39.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:38 smithi130 bash[51302]: debug 2023-08-28T02:32:38.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:38.725721+0000) 2023-08-28T02:32:39.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:38 smithi006 bash[33918]: cluster 2023-08-28T02:32:37.786009+0000 mgr.y (mgr.14901) 8220 : cluster [DBG] pgmap v8227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:39.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:38 smithi006 bash[27292]: cluster 2023-08-28T02:32:37.786009+0000 mgr.y (mgr.14901) 8220 : cluster [DBG] pgmap v8227: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:40.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:39 smithi130 bash[27579]: audit 2023-08-28T02:32:39.845294+0000 mon.c (mon.2) 3265 : audit [DBG] from='client.? 172.21.15.6:0/2819916931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:40.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:39 smithi130 bash[51302]: debug 2023-08-28T02:32:39.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:39.726030+0000) 2023-08-28T02:32:40.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:39 smithi006 bash[33918]: audit 2023-08-28T02:32:39.845294+0000 mon.c (mon.2) 3265 : audit [DBG] from='client.? 172.21.15.6:0/2819916931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:40.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:39 smithi006 bash[27292]: audit 2023-08-28T02:32:39.845294+0000 mon.c (mon.2) 3265 : audit [DBG] from='client.? 172.21.15.6:0/2819916931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:41.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:40 smithi130 bash[27579]: cluster 2023-08-28T02:32:39.787080+0000 mgr.y (mgr.14901) 8221 : cluster [DBG] pgmap v8228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:41.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:40 smithi130 bash[51302]: debug 2023-08-28T02:32:40.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:40.726306+0000) 2023-08-28T02:32:41.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:32:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:32:40] "GET /metrics HTTP/1.1" 200 193116 "" "Prometheus/2.33.4" 2023-08-28T02:32:41.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:40 smithi006 bash[33918]: cluster 2023-08-28T02:32:39.787080+0000 mgr.y (mgr.14901) 8221 : cluster [DBG] pgmap v8228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:41.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:40 smithi006 bash[27292]: cluster 2023-08-28T02:32:39.787080+0000 mgr.y (mgr.14901) 8221 : cluster [DBG] pgmap v8228: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:32:41.329Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:32:41.331Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:41 smithi130 bash[51302]: debug 2023-08-28T02:32:41.722+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:41.726667+0000) 2023-08-28T02:32:43.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:42 smithi130 bash[51302]: debug 2023-08-28T02:32:42.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:42.726962+0000) 2023-08-28T02:32:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:42 smithi130 bash[27579]: cluster 2023-08-28T02:32:41.787746+0000 mgr.y (mgr.14901) 8222 : cluster [DBG] pgmap v8229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:42 smithi130 bash[27579]: audit 2023-08-28T02:32:42.262396+0000 mon.c (mon.2) 3266 : audit [DBG] from='client.? 172.21.15.6:0/1645095692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:43.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:42 smithi006 bash[27292]: cluster 2023-08-28T02:32:41.787746+0000 mgr.y (mgr.14901) 8222 : cluster [DBG] pgmap v8229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:43.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:42 smithi006 bash[27292]: audit 2023-08-28T02:32:42.262396+0000 mon.c (mon.2) 3266 : audit [DBG] from='client.? 172.21.15.6:0/1645095692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:43.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:42 smithi006 bash[33918]: cluster 2023-08-28T02:32:41.787746+0000 mgr.y (mgr.14901) 8222 : cluster [DBG] pgmap v8229: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:43.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:42 smithi006 bash[33918]: audit 2023-08-28T02:32:42.262396+0000 mon.c (mon.2) 3266 : audit [DBG] from='client.? 172.21.15.6:0/1645095692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:43.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:32:42.893Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:43.216 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:32:42.895Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:44.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:43 smithi130 bash[51302]: debug 2023-08-28T02:32:43.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:43.727263+0000) 2023-08-28T02:32:44.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:44 smithi006 bash[27292]: cluster 2023-08-28T02:32:43.788400+0000 mgr.y (mgr.14901) 8223 : cluster [DBG] pgmap v8230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:44 smithi006 bash[33918]: cluster 2023-08-28T02:32:43.788400+0000 mgr.y (mgr.14901) 8223 : cluster [DBG] pgmap v8230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:44 smithi130 bash[27579]: cluster 2023-08-28T02:32:43.788400+0000 mgr.y (mgr.14901) 8223 : cluster [DBG] pgmap v8230: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:45.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:44 smithi130 bash[51302]: debug 2023-08-28T02:32:44.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:44.727589+0000) 2023-08-28T02:32:45.130 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:32:44.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:45.130 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:32:44.773Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:45.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:45 smithi006 bash[33918]: audit 2023-08-28T02:32:44.672372+0000 mon.c (mon.2) 3267 : audit [DBG] from='client.? 172.21.15.6:0/1959265329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:45.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:45 smithi006 bash[27292]: audit 2023-08-28T02:32:44.672372+0000 mon.c (mon.2) 3267 : audit [DBG] from='client.? 172.21.15.6:0/1959265329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:45.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:45 smithi130 bash[27579]: audit 2023-08-28T02:32:44.672372+0000 mon.c (mon.2) 3267 : audit [DBG] from='client.? 172.21.15.6:0/1959265329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:45 smithi130 bash[51302]: debug 2023-08-28T02:32:45.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:45.727863+0000) 2023-08-28T02:32:46.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:46 smithi006 bash[27292]: cluster 2023-08-28T02:32:45.789488+0000 mgr.y (mgr.14901) 8224 : cluster [DBG] pgmap v8231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:46.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:46 smithi006 bash[33918]: cluster 2023-08-28T02:32:45.789488+0000 mgr.y (mgr.14901) 8224 : cluster [DBG] pgmap v8231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:46 smithi130 bash[27579]: cluster 2023-08-28T02:32:45.789488+0000 mgr.y (mgr.14901) 8224 : cluster [DBG] pgmap v8231: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:46 smithi130 bash[51302]: debug 2023-08-28T02:32:46.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:46.728191+0000) 2023-08-28T02:32:47.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:47 smithi006 bash[33918]: audit 2023-08-28T02:32:47.084518+0000 mon.a (mon.0) 5187 : audit [DBG] from='client.? 172.21.15.6:0/3942656635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:47.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:47 smithi006 bash[27292]: audit 2023-08-28T02:32:47.084518+0000 mon.a (mon.0) 5187 : audit [DBG] from='client.? 172.21.15.6:0/3942656635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:47.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:47 smithi130 bash[27579]: audit 2023-08-28T02:32:47.084518+0000 mon.a (mon.0) 5187 : audit [DBG] from='client.? 172.21.15.6:0/3942656635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:47 smithi130 bash[51302]: debug 2023-08-28T02:32:47.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:47.728433+0000) 2023-08-28T02:32:48.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:48 smithi006 bash[33918]: cluster 2023-08-28T02:32:47.790111+0000 mgr.y (mgr.14901) 8225 : cluster [DBG] pgmap v8232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:48.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:48 smithi006 bash[27292]: cluster 2023-08-28T02:32:47.790111+0000 mgr.y (mgr.14901) 8225 : cluster [DBG] pgmap v8232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:48 smithi130 bash[27579]: cluster 2023-08-28T02:32:47.790111+0000 mgr.y (mgr.14901) 8225 : cluster [DBG] pgmap v8232: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:49.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:48 smithi130 bash[51302]: debug 2023-08-28T02:32:48.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:48.728740+0000) 2023-08-28T02:32:49.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:49 smithi130 bash[51302]: debug 2023-08-28T02:32:49.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:49.728999+0000) 2023-08-28T02:32:49.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:49 smithi130 bash[27579]: audit 2023-08-28T02:32:49.500134+0000 mon.a (mon.0) 5188 : audit [DBG] from='client.? 172.21.15.6:0/974623717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:49.966 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:49 smithi006 bash[27292]: audit 2023-08-28T02:32:49.500134+0000 mon.a (mon.0) 5188 : audit [DBG] from='client.? 172.21.15.6:0/974623717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:49.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:49 smithi006 bash[33918]: audit 2023-08-28T02:32:49.500134+0000 mon.a (mon.0) 5188 : audit [DBG] from='client.? 172.21.15.6:0/974623717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:50.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:50 smithi130 bash[27579]: cluster 2023-08-28T02:32:49.791248+0000 mgr.y (mgr.14901) 8226 : cluster [DBG] pgmap v8233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:50.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:50 smithi130 bash[51302]: debug 2023-08-28T02:32:50.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:50.729283+0000) 2023-08-28T02:32:50.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:50 smithi006 bash[27292]: cluster 2023-08-28T02:32:49.791248+0000 mgr.y (mgr.14901) 8226 : cluster [DBG] pgmap v8233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:50.965 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:32:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:32:50] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:32:50.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:50 smithi006 bash[33918]: cluster 2023-08-28T02:32:49.791248+0000 mgr.y (mgr.14901) 8226 : cluster [DBG] pgmap v8233: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:32:51.330Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:32:51.332Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:52.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:51 smithi130 bash[51302]: debug 2023-08-28T02:32:51.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:51.729561+0000) 2023-08-28T02:32:53.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:52 smithi130 bash[51302]: debug 2023-08-28T02:32:52.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:52.729850+0000) 2023-08-28T02:32:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:52 smithi130 bash[27579]: cluster 2023-08-28T02:32:51.791890+0000 mgr.y (mgr.14901) 8227 : cluster [DBG] pgmap v8234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:52 smithi130 bash[27579]: audit 2023-08-28T02:32:51.919835+0000 mon.c (mon.2) 3268 : audit [DBG] from='client.? 172.21.15.6:0/2565115891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:53.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:52 smithi006 bash[27292]: cluster 2023-08-28T02:32:51.791890+0000 mgr.y (mgr.14901) 8227 : cluster [DBG] pgmap v8234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:53.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:52 smithi006 bash[27292]: audit 2023-08-28T02:32:51.919835+0000 mon.c (mon.2) 3268 : audit [DBG] from='client.? 172.21.15.6:0/2565115891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:53.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:52 smithi006 bash[33918]: cluster 2023-08-28T02:32:51.791890+0000 mgr.y (mgr.14901) 8227 : cluster [DBG] pgmap v8234: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:53.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:52 smithi006 bash[33918]: audit 2023-08-28T02:32:51.919835+0000 mon.c (mon.2) 3268 : audit [DBG] from='client.? 172.21.15.6:0/2565115891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:32:52.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:32:52.896Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:54.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:53 smithi130 bash[51302]: debug 2023-08-28T02:32:53.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:53.730128+0000) 2023-08-28T02:32:54.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:54 smithi006 bash[27292]: cluster 2023-08-28T02:32:53.792672+0000 mgr.y (mgr.14901) 8228 : cluster [DBG] pgmap v8235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:54 smithi006 bash[33918]: cluster 2023-08-28T02:32:53.792672+0000 mgr.y (mgr.14901) 8228 : cluster [DBG] pgmap v8235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:54 smithi130 bash[27579]: cluster 2023-08-28T02:32:53.792672+0000 mgr.y (mgr.14901) 8228 : cluster [DBG] pgmap v8235: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:55.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:54 smithi130 bash[51302]: debug 2023-08-28T02:32:54.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:54.730424+0000) 2023-08-28T02:32:55.134 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:32:54.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:55.134 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:32:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:32:54.774Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:32:55.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:55 smithi006 bash[27292]: audit 2023-08-28T02:32:54.331964+0000 mon.a (mon.0) 5189 : audit [DBG] from='client.? 172.21.15.6:0/2186165138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:55.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:55 smithi006 bash[33918]: audit 2023-08-28T02:32:54.331964+0000 mon.a (mon.0) 5189 : audit [DBG] from='client.? 172.21.15.6:0/2186165138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:55.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:55 smithi130 bash[27579]: audit 2023-08-28T02:32:54.331964+0000 mon.a (mon.0) 5189 : audit [DBG] from='client.? 172.21.15.6:0/2186165138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:56.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:55 smithi130 bash[51302]: debug 2023-08-28T02:32:55.726+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:55.730756+0000) 2023-08-28T02:32:56.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:56 smithi006 bash[27292]: cluster 2023-08-28T02:32:55.793774+0000 mgr.y (mgr.14901) 8229 : cluster [DBG] pgmap v8236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:56.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:56 smithi006 bash[33918]: cluster 2023-08-28T02:32:55.793774+0000 mgr.y (mgr.14901) 8229 : cluster [DBG] pgmap v8236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:56 smithi130 bash[27579]: cluster 2023-08-28T02:32:55.793774+0000 mgr.y (mgr.14901) 8229 : cluster [DBG] pgmap v8236: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:57.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:56 smithi130 bash[51302]: debug 2023-08-28T02:32:56.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:56.731038+0000) 2023-08-28T02:32:57.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:57 smithi006 bash[27292]: audit 2023-08-28T02:32:56.749245+0000 mon.a (mon.0) 5190 : audit [DBG] from='client.? 172.21.15.6:0/1679866130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:57.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:57 smithi006 bash[33918]: audit 2023-08-28T02:32:56.749245+0000 mon.a (mon.0) 5190 : audit [DBG] from='client.? 172.21.15.6:0/1679866130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:57.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:57 smithi130 bash[27579]: audit 2023-08-28T02:32:56.749245+0000 mon.a (mon.0) 5190 : audit [DBG] from='client.? 172.21.15.6:0/1679866130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:58.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:57 smithi130 bash[51302]: debug 2023-08-28T02:32:57.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:57.731266+0000) 2023-08-28T02:32:58.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:58 smithi006 bash[33918]: cluster 2023-08-28T02:32:57.794398+0000 mgr.y (mgr.14901) 8230 : cluster [DBG] pgmap v8237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:58.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:58 smithi006 bash[27292]: cluster 2023-08-28T02:32:57.794398+0000 mgr.y (mgr.14901) 8230 : cluster [DBG] pgmap v8237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:58 smithi130 bash[27579]: cluster 2023-08-28T02:32:57.794398+0000 mgr.y (mgr.14901) 8230 : cluster [DBG] pgmap v8237: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:32:59.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:58 smithi130 bash[51302]: debug 2023-08-28T02:32:58.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:58.731497+0000) 2023-08-28T02:32:59.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:32:59 smithi130 bash[27579]: audit 2023-08-28T02:32:59.163930+0000 mon.c (mon.2) 3269 : audit [DBG] from='client.? 172.21.15.6:0/1301214101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:59.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:32:59 smithi006 bash[27292]: audit 2023-08-28T02:32:59.163930+0000 mon.c (mon.2) 3269 : audit [DBG] from='client.? 172.21.15.6:0/1301214101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:59.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:32:59 smithi006 bash[33918]: audit 2023-08-28T02:32:59.163930+0000 mon.c (mon.2) 3269 : audit [DBG] from='client.? 172.21.15.6:0/1301214101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:32:59 smithi130 bash[51302]: debug 2023-08-28T02:32:59.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:59.731811+0000) 2023-08-28T02:33:00.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:00 smithi130 bash[27579]: cluster 2023-08-28T02:32:59.795485+0000 mgr.y (mgr.14901) 8231 : cluster [DBG] pgmap v8238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:00.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:00 smithi006 bash[27292]: cluster 2023-08-28T02:32:59.795485+0000 mgr.y (mgr.14901) 8231 : cluster [DBG] pgmap v8238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:00 smithi006 bash[33918]: cluster 2023-08-28T02:32:59.795485+0000 mgr.y (mgr.14901) 8231 : cluster [DBG] pgmap v8238: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:00 smithi130 bash[51302]: debug 2023-08-28T02:33:00.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:00.732076+0000) 2023-08-28T02:33:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:33:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:33:00] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:33:01.623 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:33:01.330Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:01.624 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:33:01.332Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:01.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:01 smithi006 bash[33918]: audit 2023-08-28T02:33:01.570350+0000 mon.a (mon.0) 5191 : audit [DBG] from='client.? 172.21.15.6:0/127141054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:01.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:01 smithi006 bash[27292]: audit 2023-08-28T02:33:01.570350+0000 mon.a (mon.0) 5191 : audit [DBG] from='client.? 172.21.15.6:0/127141054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:02.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:01 smithi130 bash[27579]: audit 2023-08-28T02:33:01.570350+0000 mon.a (mon.0) 5191 : audit [DBG] from='client.? 172.21.15.6:0/127141054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:02.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:01 smithi130 bash[51302]: debug 2023-08-28T02:33:01.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:01.732430+0000) 2023-08-28T02:33:02.894 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:02 smithi006 bash[27292]: cluster 2023-08-28T02:33:01.796138+0000 mgr.y (mgr.14901) 8232 : cluster [DBG] pgmap v8239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:02.895 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:02 smithi006 bash[33918]: cluster 2023-08-28T02:33:01.796138+0000 mgr.y (mgr.14901) 8232 : cluster [DBG] pgmap v8239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:03.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:02 smithi130 bash[27579]: cluster 2023-08-28T02:33:01.796138+0000 mgr.y (mgr.14901) 8232 : cluster [DBG] pgmap v8239: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:03.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:02 smithi130 bash[51302]: debug 2023-08-28T02:33:02.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:02.732695+0000) 2023-08-28T02:33:03.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:33:02.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:33:02.896Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:04.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:03 smithi130 bash[51302]: debug 2023-08-28T02:33:03.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:03.732971+0000) 2023-08-28T02:33:04.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:04 smithi006 bash[27292]: cluster 2023-08-28T02:33:03.796753+0000 mgr.y (mgr.14901) 8233 : cluster [DBG] pgmap v8240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:04 smithi006 bash[27292]: audit 2023-08-28T02:33:03.993675+0000 mon.a (mon.0) 5192 : audit [DBG] from='client.? 172.21.15.6:0/797101542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:04.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:04 smithi006 bash[33918]: cluster 2023-08-28T02:33:03.796753+0000 mgr.y (mgr.14901) 8233 : cluster [DBG] pgmap v8240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:04.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:04 smithi006 bash[33918]: audit 2023-08-28T02:33:03.993675+0000 mon.a (mon.0) 5192 : audit [DBG] from='client.? 172.21.15.6:0/797101542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:04 smithi130 bash[27579]: cluster 2023-08-28T02:33:03.796753+0000 mgr.y (mgr.14901) 8233 : cluster [DBG] pgmap v8240: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:04 smithi130 bash[27579]: audit 2023-08-28T02:33:03.993675+0000 mon.a (mon.0) 5192 : audit [DBG] from='client.? 172.21.15.6:0/797101542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:05.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:04 smithi130 bash[51302]: debug 2023-08-28T02:33:04.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:04.733251+0000) 2023-08-28T02:33:05.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:33:04.773Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:05.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:33:04.775Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:06.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:05 smithi130 bash[51302]: debug 2023-08-28T02:33:05.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:05.733614+0000) 2023-08-28T02:33:07.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:06 smithi130 bash[51302]: debug 2023-08-28T02:33:06.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:06.733890+0000) 2023-08-28T02:33:07.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:06 smithi130 bash[27579]: cluster 2023-08-28T02:33:05.797831+0000 mgr.y (mgr.14901) 8234 : cluster [DBG] pgmap v8241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:07.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:06 smithi130 bash[27579]: audit 2023-08-28T02:33:06.400089+0000 mon.c (mon.2) 3270 : audit [DBG] from='client.? 172.21.15.6:0/3960764970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:07.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:06 smithi006 bash[27292]: cluster 2023-08-28T02:33:05.797831+0000 mgr.y (mgr.14901) 8234 : cluster [DBG] pgmap v8241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:07.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:06 smithi006 bash[27292]: audit 2023-08-28T02:33:06.400089+0000 mon.c (mon.2) 3270 : audit [DBG] from='client.? 172.21.15.6:0/3960764970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:07.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:06 smithi006 bash[33918]: cluster 2023-08-28T02:33:05.797831+0000 mgr.y (mgr.14901) 8234 : cluster [DBG] pgmap v8241: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:07.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:06 smithi006 bash[33918]: audit 2023-08-28T02:33:06.400089+0000 mon.c (mon.2) 3270 : audit [DBG] from='client.? 172.21.15.6:0/3960764970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:08.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:07 smithi130 bash[51302]: debug 2023-08-28T02:33:07.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:07.734172+0000) 2023-08-28T02:33:09.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:08 smithi130 bash[27579]: cluster 2023-08-28T02:33:07.798474+0000 mgr.y (mgr.14901) 8235 : cluster [DBG] pgmap v8242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:09.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:08 smithi130 bash[27579]: audit 2023-08-28T02:33:08.809151+0000 mon.a (mon.0) 5193 : audit [DBG] from='client.? 172.21.15.6:0/2928902178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:09.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:08 smithi130 bash[51302]: debug 2023-08-28T02:33:08.730+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:08.734431+0000) 2023-08-28T02:33:09.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:08 smithi006 bash[33918]: cluster 2023-08-28T02:33:07.798474+0000 mgr.y (mgr.14901) 8235 : cluster [DBG] pgmap v8242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:09.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:08 smithi006 bash[33918]: audit 2023-08-28T02:33:08.809151+0000 mon.a (mon.0) 5193 : audit [DBG] from='client.? 172.21.15.6:0/2928902178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:09.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:08 smithi006 bash[27292]: cluster 2023-08-28T02:33:07.798474+0000 mgr.y (mgr.14901) 8235 : cluster [DBG] pgmap v8242: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:09.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:08 smithi006 bash[27292]: audit 2023-08-28T02:33:08.809151+0000 mon.a (mon.0) 5193 : audit [DBG] from='client.? 172.21.15.6:0/2928902178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:10.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:09 smithi130 bash[51302]: debug 2023-08-28T02:33:09.734+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:09.734710+0000) 2023-08-28T02:33:11.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:10 smithi130 bash[27579]: cluster 2023-08-28T02:33:09.799563+0000 mgr.y (mgr.14901) 8236 : cluster [DBG] pgmap v8243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:11.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:10 smithi130 bash[51302]: debug 2023-08-28T02:33:10.734+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:10.734999+0000) 2023-08-28T02:33:11.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:10 smithi006 bash[27292]: cluster 2023-08-28T02:33:09.799563+0000 mgr.y (mgr.14901) 8236 : cluster [DBG] pgmap v8243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:11.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:33:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:33:10] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:33:11.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:10 smithi006 bash[33918]: cluster 2023-08-28T02:33:09.799563+0000 mgr.y (mgr.14901) 8236 : cluster [DBG] pgmap v8243: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:33:11.331Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:33:11.333Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:12.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:11 smithi130 bash[27579]: audit 2023-08-28T02:33:11.229283+0000 mon.c (mon.2) 3271 : audit [DBG] from='client.? 172.21.15.6:0/2183280014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:12.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:11 smithi130 bash[51302]: debug 2023-08-28T02:33:11.734+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:11.735317+0000) 2023-08-28T02:33:12.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:11 smithi006 bash[27292]: audit 2023-08-28T02:33:11.229283+0000 mon.c (mon.2) 3271 : audit [DBG] from='client.? 172.21.15.6:0/2183280014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:12.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:11 smithi006 bash[33918]: audit 2023-08-28T02:33:11.229283+0000 mon.c (mon.2) 3271 : audit [DBG] from='client.? 172.21.15.6:0/2183280014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:13.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:12 smithi130 bash[27579]: cluster 2023-08-28T02:33:11.800228+0000 mgr.y (mgr.14901) 8237 : cluster [DBG] pgmap v8244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:13.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:12 smithi130 bash[51302]: debug 2023-08-28T02:33:12.734+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:12.735531+0000) 2023-08-28T02:33:13.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:12 smithi006 bash[33918]: cluster 2023-08-28T02:33:11.800228+0000 mgr.y (mgr.14901) 8237 : cluster [DBG] pgmap v8244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:13.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:12 smithi006 bash[27292]: cluster 2023-08-28T02:33:11.800228+0000 mgr.y (mgr.14901) 8237 : cluster [DBG] pgmap v8244: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:33:12.895Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:33:12.897Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:14.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:13 smithi130 bash[51302]: debug 2023-08-28T02:33:13.734+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:13.735839+0000) 2023-08-28T02:33:14.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:14 smithi006 bash[27292]: audit 2023-08-28T02:33:13.637641+0000 mon.a (mon.0) 5194 : audit [DBG] from='client.? 172.21.15.6:0/3605047610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:14 smithi006 bash[27292]: cluster 2023-08-28T02:33:13.800943+0000 mgr.y (mgr.14901) 8238 : cluster [DBG] pgmap v8245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:14 smithi006 bash[33918]: audit 2023-08-28T02:33:13.637641+0000 mon.a (mon.0) 5194 : audit [DBG] from='client.? 172.21.15.6:0/3605047610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:14 smithi006 bash[33918]: cluster 2023-08-28T02:33:13.800943+0000 mgr.y (mgr.14901) 8238 : cluster [DBG] pgmap v8245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:14 smithi130 bash[27579]: audit 2023-08-28T02:33:13.637641+0000 mon.a (mon.0) 5194 : audit [DBG] from='client.? 172.21.15.6:0/3605047610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:14 smithi130 bash[27579]: cluster 2023-08-28T02:33:13.800943+0000 mgr.y (mgr.14901) 8238 : cluster [DBG] pgmap v8245: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:14 smithi130 bash[51302]: debug 2023-08-28T02:33:14.734+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:14.736124+0000) 2023-08-28T02:33:15.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:33:14.773Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:15.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:33:14.775Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:16.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:15 smithi130 bash[51302]: debug 2023-08-28T02:33:15.734+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:15.736432+0000) 2023-08-28T02:33:17.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:16 smithi130 bash[27579]: cluster 2023-08-28T02:33:15.801973+0000 mgr.y (mgr.14901) 8239 : cluster [DBG] pgmap v8246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:17.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:16 smithi130 bash[27579]: audit 2023-08-28T02:33:16.050373+0000 mon.c (mon.2) 3272 : audit [DBG] from='client.? 172.21.15.6:0/906410096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:17.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:16 smithi130 bash[51302]: debug 2023-08-28T02:33:16.734+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:16.736719+0000) 2023-08-28T02:33:17.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:16 smithi006 bash[27292]: cluster 2023-08-28T02:33:15.801973+0000 mgr.y (mgr.14901) 8239 : cluster [DBG] pgmap v8246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:17.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:16 smithi006 bash[27292]: audit 2023-08-28T02:33:16.050373+0000 mon.c (mon.2) 3272 : audit [DBG] from='client.? 172.21.15.6:0/906410096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:17.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:16 smithi006 bash[33918]: cluster 2023-08-28T02:33:15.801973+0000 mgr.y (mgr.14901) 8239 : cluster [DBG] pgmap v8246: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:17.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:16 smithi006 bash[33918]: audit 2023-08-28T02:33:16.050373+0000 mon.c (mon.2) 3272 : audit [DBG] from='client.? 172.21.15.6:0/906410096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:18.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:17 smithi130 bash[51302]: debug 2023-08-28T02:33:17.734+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:17.736924+0000) 2023-08-28T02:33:19.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:18 smithi130 bash[27579]: cluster 2023-08-28T02:33:17.802615+0000 mgr.y (mgr.14901) 8240 : cluster [DBG] pgmap v8247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:19.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:18 smithi130 bash[27579]: audit 2023-08-28T02:33:18.468806+0000 mon.a (mon.0) 5195 : audit [DBG] from='client.? 172.21.15.6:0/3556027299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:19.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:18 smithi130 bash[51302]: debug 2023-08-28T02:33:18.734+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:18.737182+0000) 2023-08-28T02:33:19.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:18 smithi006 bash[27292]: cluster 2023-08-28T02:33:17.802615+0000 mgr.y (mgr.14901) 8240 : cluster [DBG] pgmap v8247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:19.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:18 smithi006 bash[27292]: audit 2023-08-28T02:33:18.468806+0000 mon.a (mon.0) 5195 : audit [DBG] from='client.? 172.21.15.6:0/3556027299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:19.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:18 smithi006 bash[33918]: cluster 2023-08-28T02:33:17.802615+0000 mgr.y (mgr.14901) 8240 : cluster [DBG] pgmap v8247: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:19.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:18 smithi006 bash[33918]: audit 2023-08-28T02:33:18.468806+0000 mon.a (mon.0) 5195 : audit [DBG] from='client.? 172.21.15.6:0/3556027299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:19 smithi130 bash[51302]: debug 2023-08-28T02:33:19.734+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:19.737478+0000) 2023-08-28T02:33:21.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:20 smithi130 bash[27579]: cluster 2023-08-28T02:33:19.803658+0000 mgr.y (mgr.14901) 8241 : cluster [DBG] pgmap v8248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:21.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:20 smithi130 bash[51302]: debug 2023-08-28T02:33:20.734+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:20.737808+0000) 2023-08-28T02:33:21.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:20 smithi006 bash[27292]: cluster 2023-08-28T02:33:19.803658+0000 mgr.y (mgr.14901) 8241 : cluster [DBG] pgmap v8248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:21.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:33:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:33:20] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:33:21.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:20 smithi006 bash[33918]: cluster 2023-08-28T02:33:19.803658+0000 mgr.y (mgr.14901) 8241 : cluster [DBG] pgmap v8248: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:33:21.332Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:33:21.334Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:22.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:21 smithi130 bash[27579]: audit 2023-08-28T02:33:20.885436+0000 mon.a (mon.0) 5196 : audit [DBG] from='client.? 172.21.15.6:0/3399420643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:22.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:21 smithi130 bash[51302]: debug 2023-08-28T02:33:21.734+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:21.738147+0000) 2023-08-28T02:33:22.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:21 smithi006 bash[33918]: audit 2023-08-28T02:33:20.885436+0000 mon.a (mon.0) 5196 : audit [DBG] from='client.? 172.21.15.6:0/3399420643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:22.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:21 smithi006 bash[27292]: audit 2023-08-28T02:33:20.885436+0000 mon.a (mon.0) 5196 : audit [DBG] from='client.? 172.21.15.6:0/3399420643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:22 smithi130 bash[27579]: cluster 2023-08-28T02:33:21.804290+0000 mgr.y (mgr.14901) 8242 : cluster [DBG] pgmap v8249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:23.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:22 smithi130 bash[51302]: debug 2023-08-28T02:33:22.733+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:22.738427+0000) 2023-08-28T02:33:23.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:22 smithi006 bash[33918]: cluster 2023-08-28T02:33:21.804290+0000 mgr.y (mgr.14901) 8242 : cluster [DBG] pgmap v8249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:23.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:22 smithi006 bash[27292]: cluster 2023-08-28T02:33:21.804290+0000 mgr.y (mgr.14901) 8242 : cluster [DBG] pgmap v8249: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:33:22.895Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:33:22.897Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:24.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:23 smithi130 bash[51302]: debug 2023-08-28T02:33:23.733+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:23.738697+0000) 2023-08-28T02:33:24.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:24 smithi006 bash[33918]: audit 2023-08-28T02:33:23.301538+0000 mon.a (mon.0) 5197 : audit [DBG] from='client.? 172.21.15.6:0/122325313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:24.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:24 smithi006 bash[33918]: cluster 2023-08-28T02:33:23.804923+0000 mgr.y (mgr.14901) 8243 : cluster [DBG] pgmap v8250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:24 smithi006 bash[27292]: audit 2023-08-28T02:33:23.301538+0000 mon.a (mon.0) 5197 : audit [DBG] from='client.? 172.21.15.6:0/122325313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:24 smithi006 bash[27292]: cluster 2023-08-28T02:33:23.804923+0000 mgr.y (mgr.14901) 8243 : cluster [DBG] pgmap v8250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:24 smithi130 bash[27579]: audit 2023-08-28T02:33:23.301538+0000 mon.a (mon.0) 5197 : audit [DBG] from='client.? 172.21.15.6:0/122325313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:24.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:24 smithi130 bash[27579]: cluster 2023-08-28T02:33:23.804923+0000 mgr.y (mgr.14901) 8243 : cluster [DBG] pgmap v8250: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:25.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:24 smithi130 bash[51302]: debug 2023-08-28T02:33:24.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:24.739012+0000) 2023-08-28T02:33:25.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:33:24.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:25.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:33:24.776Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:26.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:25 smithi130 bash[27579]: audit 2023-08-28T02:33:25.716988+0000 mon.a (mon.0) 5198 : audit [DBG] from='client.? 172.21.15.6:0/3103861330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:26.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:25 smithi130 bash[51302]: debug 2023-08-28T02:33:25.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:25.739288+0000) 2023-08-28T02:33:26.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:25 smithi006 bash[33918]: audit 2023-08-28T02:33:25.716988+0000 mon.a (mon.0) 5198 : audit [DBG] from='client.? 172.21.15.6:0/3103861330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:26.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:25 smithi006 bash[27292]: audit 2023-08-28T02:33:25.716988+0000 mon.a (mon.0) 5198 : audit [DBG] from='client.? 172.21.15.6:0/3103861330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:27.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:26 smithi130 bash[27579]: cluster 2023-08-28T02:33:25.805963+0000 mgr.y (mgr.14901) 8244 : cluster [DBG] pgmap v8251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:27.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:26 smithi130 bash[51302]: debug 2023-08-28T02:33:26.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:26.739614+0000) 2023-08-28T02:33:27.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:26 smithi006 bash[27292]: cluster 2023-08-28T02:33:25.805963+0000 mgr.y (mgr.14901) 8244 : cluster [DBG] pgmap v8251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:27.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:26 smithi006 bash[33918]: cluster 2023-08-28T02:33:25.805963+0000 mgr.y (mgr.14901) 8244 : cluster [DBG] pgmap v8251: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:28.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:27 smithi130 bash[51302]: debug 2023-08-28T02:33:27.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:27.739883+0000) 2023-08-28T02:33:29.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:28 smithi130 bash[27579]: cluster 2023-08-28T02:33:27.806573+0000 mgr.y (mgr.14901) 8245 : cluster [DBG] pgmap v8252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:29.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:28 smithi130 bash[27579]: audit 2023-08-28T02:33:28.131192+0000 mon.c (mon.2) 3273 : audit [DBG] from='client.? 172.21.15.6:0/1484849730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:29.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:28 smithi130 bash[51302]: debug 2023-08-28T02:33:28.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:28.740155+0000) 2023-08-28T02:33:29.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:28 smithi006 bash[27292]: cluster 2023-08-28T02:33:27.806573+0000 mgr.y (mgr.14901) 8245 : cluster [DBG] pgmap v8252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:29.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:28 smithi006 bash[27292]: audit 2023-08-28T02:33:28.131192+0000 mon.c (mon.2) 3273 : audit [DBG] from='client.? 172.21.15.6:0/1484849730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:29.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:28 smithi006 bash[33918]: cluster 2023-08-28T02:33:27.806573+0000 mgr.y (mgr.14901) 8245 : cluster [DBG] pgmap v8252: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:29.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:28 smithi006 bash[33918]: audit 2023-08-28T02:33:28.131192+0000 mon.c (mon.2) 3273 : audit [DBG] from='client.? 172.21.15.6:0/1484849730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:30.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:29 smithi130 bash[51302]: debug 2023-08-28T02:33:29.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:29.740436+0000) 2023-08-28T02:33:31.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:30 smithi130 bash[51302]: debug 2023-08-28T02:33:30.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:30.740751+0000) 2023-08-28T02:33:31.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:30 smithi130 bash[27579]: cluster 2023-08-28T02:33:29.807642+0000 mgr.y (mgr.14901) 8246 : cluster [DBG] pgmap v8253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:31.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:30 smithi130 bash[27579]: audit 2023-08-28T02:33:30.547212+0000 mon.a (mon.0) 5199 : audit [DBG] from='client.? 172.21.15.6:0/3935412144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:31.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:33:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:33:30] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:33:31.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:30 smithi006 bash[27292]: cluster 2023-08-28T02:33:29.807642+0000 mgr.y (mgr.14901) 8246 : cluster [DBG] pgmap v8253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:31.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:30 smithi006 bash[27292]: audit 2023-08-28T02:33:30.547212+0000 mon.a (mon.0) 5199 : audit [DBG] from='client.? 172.21.15.6:0/3935412144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:31.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:30 smithi006 bash[33918]: cluster 2023-08-28T02:33:29.807642+0000 mgr.y (mgr.14901) 8246 : cluster [DBG] pgmap v8253: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:31.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:30 smithi006 bash[33918]: audit 2023-08-28T02:33:30.547212+0000 mon.a (mon.0) 5199 : audit [DBG] from='client.? 172.21.15.6:0/3935412144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:33:31.333Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:33:31.334Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:32.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:31 smithi130 bash[51302]: debug 2023-08-28T02:33:31.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:31.741059+0000) 2023-08-28T02:33:33.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:32 smithi130 bash[27579]: cluster 2023-08-28T02:33:31.808252+0000 mgr.y (mgr.14901) 8247 : cluster [DBG] pgmap v8254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:33.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:32 smithi130 bash[51302]: debug 2023-08-28T02:33:32.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:32.741307+0000) 2023-08-28T02:33:33.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:32 smithi006 bash[27292]: cluster 2023-08-28T02:33:31.808252+0000 mgr.y (mgr.14901) 8247 : cluster [DBG] pgmap v8254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:33.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:32 smithi006 bash[33918]: cluster 2023-08-28T02:33:31.808252+0000 mgr.y (mgr.14901) 8247 : cluster [DBG] pgmap v8254: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:33:32.896Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:33:32.898Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:34.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:33 smithi130 bash[51302]: debug 2023-08-28T02:33:33.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:33.741541+0000) 2023-08-28T02:33:34.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:33 smithi130 bash[27579]: audit 2023-08-28T02:33:32.961504+0000 mon.a (mon.0) 5200 : audit [DBG] from='client.? 172.21.15.6:0/620515514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:34.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:33 smithi130 bash[27579]: audit 2023-08-28T02:33:33.549588+0000 mon.b (mon.1) 897 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:33:34.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:33 smithi130 bash[27579]: audit 2023-08-28T02:33:33.829889+0000 mon.a (mon.0) 5201 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:33:34.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:33 smithi130 bash[27579]: audit 2023-08-28T02:33:33.837266+0000 mon.a (mon.0) 5202 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:33:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:33 smithi006 bash[33918]: audit 2023-08-28T02:33:32.961504+0000 mon.a (mon.0) 5200 : audit [DBG] from='client.? 172.21.15.6:0/620515514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:33 smithi006 bash[33918]: audit 2023-08-28T02:33:33.549588+0000 mon.b (mon.1) 897 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:33:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:33 smithi006 bash[33918]: audit 2023-08-28T02:33:33.829889+0000 mon.a (mon.0) 5201 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:33:34.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:33 smithi006 bash[33918]: audit 2023-08-28T02:33:33.837266+0000 mon.a (mon.0) 5202 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:33:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:33 smithi006 bash[27292]: audit 2023-08-28T02:33:32.961504+0000 mon.a (mon.0) 5200 : audit [DBG] from='client.? 172.21.15.6:0/620515514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:33 smithi006 bash[27292]: audit 2023-08-28T02:33:33.549588+0000 mon.b (mon.1) 897 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:33:34.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:33 smithi006 bash[27292]: audit 2023-08-28T02:33:33.829889+0000 mon.a (mon.0) 5201 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:33:34.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:33 smithi006 bash[27292]: audit 2023-08-28T02:33:33.837266+0000 mon.a (mon.0) 5202 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:33:35.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:34 smithi130 bash[27579]: cluster 2023-08-28T02:33:33.808894+0000 mgr.y (mgr.14901) 8248 : cluster [DBG] pgmap v8255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:35.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:34 smithi130 bash[51302]: debug 2023-08-28T02:33:34.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:34.741837+0000) 2023-08-28T02:33:35.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:34 smithi006 bash[33918]: cluster 2023-08-28T02:33:33.808894+0000 mgr.y (mgr.14901) 8248 : cluster [DBG] pgmap v8255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:35.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:33:34.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:35.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:33:34.777Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:35.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:34 smithi006 bash[27292]: cluster 2023-08-28T02:33:33.808894+0000 mgr.y (mgr.14901) 8248 : cluster [DBG] pgmap v8255: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:36.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:35 smithi130 bash[27579]: audit 2023-08-28T02:33:35.372025+0000 mon.c (mon.2) 3274 : audit [DBG] from='client.? 172.21.15.6:0/1414752521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:36.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:35 smithi130 bash[51302]: debug 2023-08-28T02:33:35.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:35.742156+0000) 2023-08-28T02:33:36.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:35 smithi006 bash[27292]: audit 2023-08-28T02:33:35.372025+0000 mon.c (mon.2) 3274 : audit [DBG] from='client.? 172.21.15.6:0/1414752521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:36.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:35 smithi006 bash[33918]: audit 2023-08-28T02:33:35.372025+0000 mon.c (mon.2) 3274 : audit [DBG] from='client.? 172.21.15.6:0/1414752521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:37.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:36 smithi130 bash[27579]: cluster 2023-08-28T02:33:35.809945+0000 mgr.y (mgr.14901) 8249 : cluster [DBG] pgmap v8256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:37.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:36 smithi130 bash[51302]: debug 2023-08-28T02:33:36.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:36.742387+0000) 2023-08-28T02:33:37.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:36 smithi006 bash[27292]: cluster 2023-08-28T02:33:35.809945+0000 mgr.y (mgr.14901) 8249 : cluster [DBG] pgmap v8256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:37.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:36 smithi006 bash[33918]: cluster 2023-08-28T02:33:35.809945+0000 mgr.y (mgr.14901) 8249 : cluster [DBG] pgmap v8256: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:37 smithi130 bash[51302]: debug 2023-08-28T02:33:37.737+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:37.742684+0000) 2023-08-28T02:33:38.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:37 smithi130 bash[27579]: audit 2023-08-28T02:33:37.782679+0000 mon.a (mon.0) 5203 : audit [DBG] from='client.? 172.21.15.6:0/1326651313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:38.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:37 smithi006 bash[33918]: audit 2023-08-28T02:33:37.782679+0000 mon.a (mon.0) 5203 : audit [DBG] from='client.? 172.21.15.6:0/1326651313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:38.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:37 smithi006 bash[27292]: audit 2023-08-28T02:33:37.782679+0000 mon.a (mon.0) 5203 : audit [DBG] from='client.? 172.21.15.6:0/1326651313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:39.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:38 smithi130 bash[27579]: cluster 2023-08-28T02:33:37.810529+0000 mgr.y (mgr.14901) 8250 : cluster [DBG] pgmap v8257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:39.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:38 smithi130 bash[51302]: debug 2023-08-28T02:33:38.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:38.742984+0000) 2023-08-28T02:33:39.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:38 smithi006 bash[27292]: cluster 2023-08-28T02:33:37.810529+0000 mgr.y (mgr.14901) 8250 : cluster [DBG] pgmap v8257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:39.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:38 smithi006 bash[33918]: cluster 2023-08-28T02:33:37.810529+0000 mgr.y (mgr.14901) 8250 : cluster [DBG] pgmap v8257: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:40.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:39 smithi130 bash[51302]: debug 2023-08-28T02:33:39.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:39.743226+0000) 2023-08-28T02:33:41.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:40 smithi130 bash[51302]: debug 2023-08-28T02:33:40.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:40.743514+0000) 2023-08-28T02:33:41.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:40 smithi130 bash[27579]: audit 2023-08-28T02:33:39.790088+0000 mon.a (mon.0) 5204 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:33:41.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:40 smithi130 bash[27579]: cluster 2023-08-28T02:33:39.811528+0000 mgr.y (mgr.14901) 8251 : cluster [DBG] pgmap v8258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:41.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:40 smithi130 bash[27579]: audit 2023-08-28T02:33:40.190327+0000 mon.a (mon.0) 5205 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:33:41.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:40 smithi130 bash[27579]: audit 2023-08-28T02:33:40.262740+0000 mon.c (mon.2) 3275 : audit [DBG] from='client.? 172.21.15.6:0/1455838498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:41.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:40 smithi130 bash[27579]: audit 2023-08-28T02:33:40.461990+0000 mon.b (mon.1) 898 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:33:41.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:40 smithi130 bash[27579]: audit 2023-08-28T02:33:40.464307+0000 mon.b (mon.1) 899 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:33:41.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:40 smithi130 bash[27579]: audit 2023-08-28T02:33:40.472766+0000 mon.a (mon.0) 5206 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:33:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:33:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:33:40] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:33:41.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:40 smithi006 bash[27292]: audit 2023-08-28T02:33:39.790088+0000 mon.a (mon.0) 5204 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:33:41.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:40 smithi006 bash[27292]: cluster 2023-08-28T02:33:39.811528+0000 mgr.y (mgr.14901) 8251 : cluster [DBG] pgmap v8258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:41.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:40 smithi006 bash[27292]: audit 2023-08-28T02:33:40.190327+0000 mon.a (mon.0) 5205 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:33:41.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:40 smithi006 bash[27292]: audit 2023-08-28T02:33:40.262740+0000 mon.c (mon.2) 3275 : audit [DBG] from='client.? 172.21.15.6:0/1455838498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:41.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:40 smithi006 bash[27292]: audit 2023-08-28T02:33:40.461990+0000 mon.b (mon.1) 898 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:33:41.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:40 smithi006 bash[27292]: audit 2023-08-28T02:33:40.464307+0000 mon.b (mon.1) 899 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:33:41.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:40 smithi006 bash[27292]: audit 2023-08-28T02:33:40.472766+0000 mon.a (mon.0) 5206 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:33:41.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:40 smithi006 bash[33918]: audit 2023-08-28T02:33:39.790088+0000 mon.a (mon.0) 5204 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:33:41.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:40 smithi006 bash[33918]: cluster 2023-08-28T02:33:39.811528+0000 mgr.y (mgr.14901) 8251 : cluster [DBG] pgmap v8258: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:41.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:40 smithi006 bash[33918]: audit 2023-08-28T02:33:40.190327+0000 mon.a (mon.0) 5205 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:33:41.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:40 smithi006 bash[33918]: audit 2023-08-28T02:33:40.262740+0000 mon.c (mon.2) 3275 : audit [DBG] from='client.? 172.21.15.6:0/1455838498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:41.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:40 smithi006 bash[33918]: audit 2023-08-28T02:33:40.461990+0000 mon.b (mon.1) 898 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:33:41.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:40 smithi006 bash[33918]: audit 2023-08-28T02:33:40.464307+0000 mon.b (mon.1) 899 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:33:41.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:40 smithi006 bash[33918]: audit 2023-08-28T02:33:40.472766+0000 mon.a (mon.0) 5206 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:33:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:33:41.333Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:33:41.336Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:41 smithi130 bash[51302]: debug 2023-08-28T02:33:41.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:41.743840+0000) 2023-08-28T02:33:43.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:42 smithi130 bash[51302]: debug 2023-08-28T02:33:42.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:42.744122+0000) 2023-08-28T02:33:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:42 smithi130 bash[27579]: cluster 2023-08-28T02:33:41.812271+0000 mgr.y (mgr.14901) 8252 : cluster [DBG] pgmap v8259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:42 smithi130 bash[27579]: audit 2023-08-28T02:33:42.676559+0000 mon.a (mon.0) 5207 : audit [DBG] from='client.? 172.21.15.6:0/1648366572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:43.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:42 smithi006 bash[33918]: cluster 2023-08-28T02:33:41.812271+0000 mgr.y (mgr.14901) 8252 : cluster [DBG] pgmap v8259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:43.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:42 smithi006 bash[33918]: audit 2023-08-28T02:33:42.676559+0000 mon.a (mon.0) 5207 : audit [DBG] from='client.? 172.21.15.6:0/1648366572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:43.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:42 smithi006 bash[27292]: cluster 2023-08-28T02:33:41.812271+0000 mgr.y (mgr.14901) 8252 : cluster [DBG] pgmap v8259: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:43.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:42 smithi006 bash[27292]: audit 2023-08-28T02:33:42.676559+0000 mon.a (mon.0) 5207 : audit [DBG] from='client.? 172.21.15.6:0/1648366572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:33:42.896Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:33:42.899Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:44.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:43 smithi130 bash[51302]: debug 2023-08-28T02:33:43.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:43.744402+0000) 2023-08-28T02:33:44.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:44 smithi006 bash[33918]: cluster 2023-08-28T02:33:43.812956+0000 mgr.y (mgr.14901) 8253 : cluster [DBG] pgmap v8260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:44.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:44 smithi006 bash[27292]: cluster 2023-08-28T02:33:43.812956+0000 mgr.y (mgr.14901) 8253 : cluster [DBG] pgmap v8260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:44 smithi130 bash[27579]: cluster 2023-08-28T02:33:43.812956+0000 mgr.y (mgr.14901) 8253 : cluster [DBG] pgmap v8260: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:45.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:44 smithi130 bash[51302]: debug 2023-08-28T02:33:44.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:44.744690+0000) 2023-08-28T02:33:45.146 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:33:44.776Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:45.146 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:33:44.778Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:45.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:45 smithi006 bash[27292]: audit 2023-08-28T02:33:45.087785+0000 mon.a (mon.0) 5208 : audit [DBG] from='client.? 172.21.15.6:0/1108433934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:45.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:45 smithi006 bash[33918]: audit 2023-08-28T02:33:45.087785+0000 mon.a (mon.0) 5208 : audit [DBG] from='client.? 172.21.15.6:0/1108433934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:45.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:45 smithi130 bash[27579]: audit 2023-08-28T02:33:45.087785+0000 mon.a (mon.0) 5208 : audit [DBG] from='client.? 172.21.15.6:0/1108433934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:45 smithi130 bash[51302]: debug 2023-08-28T02:33:45.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:45.745005+0000) 2023-08-28T02:33:46.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:46 smithi006 bash[27292]: cluster 2023-08-28T02:33:45.813938+0000 mgr.y (mgr.14901) 8254 : cluster [DBG] pgmap v8261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:46.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:46 smithi006 bash[33918]: cluster 2023-08-28T02:33:45.813938+0000 mgr.y (mgr.14901) 8254 : cluster [DBG] pgmap v8261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:46 smithi130 bash[27579]: cluster 2023-08-28T02:33:45.813938+0000 mgr.y (mgr.14901) 8254 : cluster [DBG] pgmap v8261: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:46 smithi130 bash[51302]: debug 2023-08-28T02:33:46.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:46.745222+0000) 2023-08-28T02:33:47.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:47 smithi130 bash[27579]: audit 2023-08-28T02:33:47.506015+0000 mon.c (mon.2) 3276 : audit [DBG] from='client.? 172.21.15.6:0/11797663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:47.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:47 smithi130 bash[51302]: debug 2023-08-28T02:33:47.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:47.745516+0000) 2023-08-28T02:33:47.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:47 smithi006 bash[33918]: audit 2023-08-28T02:33:47.506015+0000 mon.c (mon.2) 3276 : audit [DBG] from='client.? 172.21.15.6:0/11797663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:47.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:47 smithi006 bash[27292]: audit 2023-08-28T02:33:47.506015+0000 mon.c (mon.2) 3276 : audit [DBG] from='client.? 172.21.15.6:0/11797663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:48.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:48 smithi130 bash[27579]: cluster 2023-08-28T02:33:47.814642+0000 mgr.y (mgr.14901) 8255 : cluster [DBG] pgmap v8262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:48.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:48 smithi130 bash[51302]: debug 2023-08-28T02:33:48.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:48.745779+0000) 2023-08-28T02:33:48.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:48 smithi006 bash[33918]: cluster 2023-08-28T02:33:47.814642+0000 mgr.y (mgr.14901) 8255 : cluster [DBG] pgmap v8262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:48.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:48 smithi006 bash[27292]: cluster 2023-08-28T02:33:47.814642+0000 mgr.y (mgr.14901) 8255 : cluster [DBG] pgmap v8262: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:50.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:49 smithi130 bash[51302]: debug 2023-08-28T02:33:49.741+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:49.746045+0000) 2023-08-28T02:33:51.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:50 smithi130 bash[51302]: debug 2023-08-28T02:33:50.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:50.746332+0000) 2023-08-28T02:33:51.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:50 smithi130 bash[27579]: cluster 2023-08-28T02:33:49.815721+0000 mgr.y (mgr.14901) 8256 : cluster [DBG] pgmap v8263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:51.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:50 smithi130 bash[27579]: audit 2023-08-28T02:33:49.927255+0000 mon.a (mon.0) 5209 : audit [DBG] from='client.? 172.21.15.6:0/1942405356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:33:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:33:50] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:33:51.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:50 smithi006 bash[27292]: cluster 2023-08-28T02:33:49.815721+0000 mgr.y (mgr.14901) 8256 : cluster [DBG] pgmap v8263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:51.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:50 smithi006 bash[27292]: audit 2023-08-28T02:33:49.927255+0000 mon.a (mon.0) 5209 : audit [DBG] from='client.? 172.21.15.6:0/1942405356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:51.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:50 smithi006 bash[33918]: cluster 2023-08-28T02:33:49.815721+0000 mgr.y (mgr.14901) 8256 : cluster [DBG] pgmap v8263: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:51.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:50 smithi006 bash[33918]: audit 2023-08-28T02:33:49.927255+0000 mon.a (mon.0) 5209 : audit [DBG] from='client.? 172.21.15.6:0/1942405356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:33:51.334Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:33:51.336Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:52.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:51 smithi130 bash[51302]: debug 2023-08-28T02:33:51.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:51.746644+0000) 2023-08-28T02:33:53.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:52 smithi130 bash[27579]: cluster 2023-08-28T02:33:51.816393+0000 mgr.y (mgr.14901) 8257 : cluster [DBG] pgmap v8264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:52 smithi130 bash[27579]: audit 2023-08-28T02:33:52.341004+0000 mon.c (mon.2) 3277 : audit [DBG] from='client.? 172.21.15.6:0/129451523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:53.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:52 smithi130 bash[51302]: debug 2023-08-28T02:33:52.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:52.746936+0000) 2023-08-28T02:33:53.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:52 smithi006 bash[33918]: cluster 2023-08-28T02:33:51.816393+0000 mgr.y (mgr.14901) 8257 : cluster [DBG] pgmap v8264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:53.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:52 smithi006 bash[33918]: audit 2023-08-28T02:33:52.341004+0000 mon.c (mon.2) 3277 : audit [DBG] from='client.? 172.21.15.6:0/129451523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:53.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:52 smithi006 bash[27292]: cluster 2023-08-28T02:33:51.816393+0000 mgr.y (mgr.14901) 8257 : cluster [DBG] pgmap v8264: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:53.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:52 smithi006 bash[27292]: audit 2023-08-28T02:33:52.341004+0000 mon.c (mon.2) 3277 : audit [DBG] from='client.? 172.21.15.6:0/129451523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:33:52.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:33:52.899Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:54.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:53 smithi130 bash[51302]: debug 2023-08-28T02:33:53.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:53.747220+0000) 2023-08-28T02:33:54.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:54 smithi006 bash[27292]: cluster 2023-08-28T02:33:53.817069+0000 mgr.y (mgr.14901) 8258 : cluster [DBG] pgmap v8265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:54 smithi006 bash[33918]: cluster 2023-08-28T02:33:53.817069+0000 mgr.y (mgr.14901) 8258 : cluster [DBG] pgmap v8265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:54 smithi130 bash[27579]: cluster 2023-08-28T02:33:53.817069+0000 mgr.y (mgr.14901) 8258 : cluster [DBG] pgmap v8265: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:55.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:54 smithi130 bash[51302]: debug 2023-08-28T02:33:54.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:54.747495+0000) 2023-08-28T02:33:55.150 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:33:54.776Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:55.150 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:33:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:33:54.778Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:33:55.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:55 smithi006 bash[27292]: audit 2023-08-28T02:33:54.754975+0000 mon.a (mon.0) 5210 : audit [DBG] from='client.? 172.21.15.6:0/2894688246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:55.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:55 smithi006 bash[33918]: audit 2023-08-28T02:33:54.754975+0000 mon.a (mon.0) 5210 : audit [DBG] from='client.? 172.21.15.6:0/2894688246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:55.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:55 smithi130 bash[27579]: audit 2023-08-28T02:33:54.754975+0000 mon.a (mon.0) 5210 : audit [DBG] from='client.? 172.21.15.6:0/2894688246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:56.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:55 smithi130 bash[51302]: debug 2023-08-28T02:33:55.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:55.747767+0000) 2023-08-28T02:33:56.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:56 smithi006 bash[33918]: cluster 2023-08-28T02:33:55.818203+0000 mgr.y (mgr.14901) 8259 : cluster [DBG] pgmap v8266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:56.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:56 smithi006 bash[27292]: cluster 2023-08-28T02:33:55.818203+0000 mgr.y (mgr.14901) 8259 : cluster [DBG] pgmap v8266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:56 smithi130 bash[27579]: cluster 2023-08-28T02:33:55.818203+0000 mgr.y (mgr.14901) 8259 : cluster [DBG] pgmap v8266: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:57.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:56 smithi130 bash[51302]: debug 2023-08-28T02:33:56.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:56.747998+0000) 2023-08-28T02:33:57.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:57 smithi130 bash[27579]: audit 2023-08-28T02:33:57.170025+0000 mon.c (mon.2) 3278 : audit [DBG] from='client.? 172.21.15.6:0/2650025722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:57.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:57 smithi006 bash[27292]: audit 2023-08-28T02:33:57.170025+0000 mon.c (mon.2) 3278 : audit [DBG] from='client.? 172.21.15.6:0/2650025722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:57.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:57 smithi006 bash[33918]: audit 2023-08-28T02:33:57.170025+0000 mon.c (mon.2) 3278 : audit [DBG] from='client.? 172.21.15.6:0/2650025722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:58.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:57 smithi130 bash[51302]: debug 2023-08-28T02:33:57.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:57.748284+0000) 2023-08-28T02:33:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:58 smithi130 bash[27579]: cluster 2023-08-28T02:33:57.818868+0000 mgr.y (mgr.14901) 8260 : cluster [DBG] pgmap v8267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:58.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:58 smithi006 bash[33918]: cluster 2023-08-28T02:33:57.818868+0000 mgr.y (mgr.14901) 8260 : cluster [DBG] pgmap v8267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:58.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:58 smithi006 bash[27292]: cluster 2023-08-28T02:33:57.818868+0000 mgr.y (mgr.14901) 8260 : cluster [DBG] pgmap v8267: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:33:59.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:58 smithi130 bash[51302]: debug 2023-08-28T02:33:58.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:58.748548+0000) 2023-08-28T02:33:59.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:33:59 smithi006 bash[27292]: audit 2023-08-28T02:33:59.589700+0000 mon.a (mon.0) 5211 : audit [DBG] from='client.? 172.21.15.6:0/3852306968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:59.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:33:59 smithi006 bash[33918]: audit 2023-08-28T02:33:59.589700+0000 mon.a (mon.0) 5211 : audit [DBG] from='client.? 172.21.15.6:0/3852306968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:00.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:33:59 smithi130 bash[27579]: audit 2023-08-28T02:33:59.589700+0000 mon.a (mon.0) 5211 : audit [DBG] from='client.? 172.21.15.6:0/3852306968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:00.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:33:59 smithi130 bash[51302]: debug 2023-08-28T02:33:59.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:59.748816+0000) 2023-08-28T02:34:00.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:00 smithi006 bash[27292]: cluster 2023-08-28T02:33:59.820027+0000 mgr.y (mgr.14901) 8261 : cluster [DBG] pgmap v8268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:00.965 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:34:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:34:00] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:34:00.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:00 smithi006 bash[33918]: cluster 2023-08-28T02:33:59.820027+0000 mgr.y (mgr.14901) 8261 : cluster [DBG] pgmap v8268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:01.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:00 smithi130 bash[27579]: cluster 2023-08-28T02:33:59.820027+0000 mgr.y (mgr.14901) 8261 : cluster [DBG] pgmap v8268: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:01.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:00 smithi130 bash[51302]: debug 2023-08-28T02:34:00.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:00.749123+0000) 2023-08-28T02:34:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:34:01.334Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:34:01.337Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:02.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:01 smithi130 bash[51302]: debug 2023-08-28T02:34:01.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:01.749536+0000) 2023-08-28T02:34:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:02 smithi130 bash[27579]: cluster 2023-08-28T02:34:01.820672+0000 mgr.y (mgr.14901) 8262 : cluster [DBG] pgmap v8269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:02 smithi130 bash[27579]: audit 2023-08-28T02:34:02.008282+0000 mon.a (mon.0) 5212 : audit [DBG] from='client.? 172.21.15.6:0/69936470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:03.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:02 smithi130 bash[51302]: debug 2023-08-28T02:34:02.749+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:02.749868+0000) 2023-08-28T02:34:03.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:02 smithi006 bash[27292]: cluster 2023-08-28T02:34:01.820672+0000 mgr.y (mgr.14901) 8262 : cluster [DBG] pgmap v8269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:03.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:02 smithi006 bash[27292]: audit 2023-08-28T02:34:02.008282+0000 mon.a (mon.0) 5212 : audit [DBG] from='client.? 172.21.15.6:0/69936470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:03.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:02 smithi006 bash[33918]: cluster 2023-08-28T02:34:01.820672+0000 mgr.y (mgr.14901) 8262 : cluster [DBG] pgmap v8269: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:03.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:02 smithi006 bash[33918]: audit 2023-08-28T02:34:02.008282+0000 mon.a (mon.0) 5212 : audit [DBG] from='client.? 172.21.15.6:0/69936470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:34:02.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:34:02.899Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:04.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:03 smithi130 bash[51302]: debug 2023-08-28T02:34:03.749+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:03.750202+0000) 2023-08-28T02:34:04.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:04 smithi006 bash[33918]: cluster 2023-08-28T02:34:03.821335+0000 mgr.y (mgr.14901) 8263 : cluster [DBG] pgmap v8270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:04 smithi006 bash[27292]: cluster 2023-08-28T02:34:03.821335+0000 mgr.y (mgr.14901) 8263 : cluster [DBG] pgmap v8270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:04 smithi130 bash[27579]: cluster 2023-08-28T02:34:03.821335+0000 mgr.y (mgr.14901) 8263 : cluster [DBG] pgmap v8270: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:05.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:04 smithi130 bash[51302]: debug 2023-08-28T02:34:04.745+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:04.750470+0000) 2023-08-28T02:34:05.151 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:34:04.777Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:05.151 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:34:04.779Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:05.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:05 smithi006 bash[27292]: audit 2023-08-28T02:34:04.429071+0000 mon.c (mon.2) 3279 : audit [DBG] from='client.? 172.21.15.6:0/2158124491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:05.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:05 smithi006 bash[33918]: audit 2023-08-28T02:34:04.429071+0000 mon.c (mon.2) 3279 : audit [DBG] from='client.? 172.21.15.6:0/2158124491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:05.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:05 smithi130 bash[27579]: audit 2023-08-28T02:34:04.429071+0000 mon.c (mon.2) 3279 : audit [DBG] from='client.? 172.21.15.6:0/2158124491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:06.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:05 smithi130 bash[51302]: debug 2023-08-28T02:34:05.749+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:05.750754+0000) 2023-08-28T02:34:06.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:06 smithi006 bash[27292]: cluster 2023-08-28T02:34:05.822459+0000 mgr.y (mgr.14901) 8264 : cluster [DBG] pgmap v8271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:06.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:06 smithi006 bash[33918]: cluster 2023-08-28T02:34:05.822459+0000 mgr.y (mgr.14901) 8264 : cluster [DBG] pgmap v8271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:06 smithi130 bash[27579]: cluster 2023-08-28T02:34:05.822459+0000 mgr.y (mgr.14901) 8264 : cluster [DBG] pgmap v8271: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:07.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:06 smithi130 bash[51302]: debug 2023-08-28T02:34:06.749+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:06.751044+0000) 2023-08-28T02:34:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:07 smithi006 bash[27292]: audit 2023-08-28T02:34:06.847427+0000 mon.c (mon.2) 3280 : audit [DBG] from='client.? 172.21.15.6:0/3822159159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:07.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:07 smithi006 bash[33918]: audit 2023-08-28T02:34:06.847427+0000 mon.c (mon.2) 3280 : audit [DBG] from='client.? 172.21.15.6:0/3822159159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:07.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:07 smithi130 bash[27579]: audit 2023-08-28T02:34:06.847427+0000 mon.c (mon.2) 3280 : audit [DBG] from='client.? 172.21.15.6:0/3822159159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:08.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:07 smithi130 bash[51302]: debug 2023-08-28T02:34:07.749+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:07.751262+0000) 2023-08-28T02:34:08.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:08 smithi006 bash[33918]: cluster 2023-08-28T02:34:07.823119+0000 mgr.y (mgr.14901) 8265 : cluster [DBG] pgmap v8272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:08.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:08 smithi006 bash[27292]: cluster 2023-08-28T02:34:07.823119+0000 mgr.y (mgr.14901) 8265 : cluster [DBG] pgmap v8272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:08 smithi130 bash[27579]: cluster 2023-08-28T02:34:07.823119+0000 mgr.y (mgr.14901) 8265 : cluster [DBG] pgmap v8272: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:09.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:08 smithi130 bash[51302]: debug 2023-08-28T02:34:08.749+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:08.751557+0000) 2023-08-28T02:34:09.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:09 smithi130 bash[27579]: audit 2023-08-28T02:34:09.257330+0000 mon.c (mon.2) 3281 : audit [DBG] from='client.? 172.21.15.6:0/2558011898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:09.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:09 smithi006 bash[27292]: audit 2023-08-28T02:34:09.257330+0000 mon.c (mon.2) 3281 : audit [DBG] from='client.? 172.21.15.6:0/2558011898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:09.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:09 smithi006 bash[33918]: audit 2023-08-28T02:34:09.257330+0000 mon.c (mon.2) 3281 : audit [DBG] from='client.? 172.21.15.6:0/2558011898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:10.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:09 smithi130 bash[51302]: debug 2023-08-28T02:34:09.749+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:09.751827+0000) 2023-08-28T02:34:10.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:10 smithi130 bash[27579]: cluster 2023-08-28T02:34:09.824249+0000 mgr.y (mgr.14901) 8266 : cluster [DBG] pgmap v8273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:10.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:10 smithi006 bash[27292]: cluster 2023-08-28T02:34:09.824249+0000 mgr.y (mgr.14901) 8266 : cluster [DBG] pgmap v8273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:10.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:10 smithi006 bash[33918]: cluster 2023-08-28T02:34:09.824249+0000 mgr.y (mgr.14901) 8266 : cluster [DBG] pgmap v8273: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:11.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:10 smithi130 bash[51302]: debug 2023-08-28T02:34:10.749+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:10.752140+0000) 2023-08-28T02:34:11.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:34:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:34:10] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:34:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:34:11.335Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:34:11.338Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:12.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:11 smithi130 bash[27579]: audit 2023-08-28T02:34:11.672951+0000 mon.a (mon.0) 5213 : audit [DBG] from='client.? 172.21.15.6:0/3018747320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:12.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:11 smithi130 bash[51302]: debug 2023-08-28T02:34:11.749+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:11.752432+0000) 2023-08-28T02:34:12.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:11 smithi006 bash[33918]: audit 2023-08-28T02:34:11.672951+0000 mon.a (mon.0) 5213 : audit [DBG] from='client.? 172.21.15.6:0/3018747320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:12.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:11 smithi006 bash[27292]: audit 2023-08-28T02:34:11.672951+0000 mon.a (mon.0) 5213 : audit [DBG] from='client.? 172.21.15.6:0/3018747320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:13.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:12 smithi130 bash[51302]: debug 2023-08-28T02:34:12.749+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:12.752693+0000) 2023-08-28T02:34:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:12 smithi130 bash[27579]: cluster 2023-08-28T02:34:11.824919+0000 mgr.y (mgr.14901) 8267 : cluster [DBG] pgmap v8274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:13.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:12 smithi006 bash[33918]: cluster 2023-08-28T02:34:11.824919+0000 mgr.y (mgr.14901) 8267 : cluster [DBG] pgmap v8274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:13.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:12 smithi006 bash[27292]: cluster 2023-08-28T02:34:11.824919+0000 mgr.y (mgr.14901) 8267 : cluster [DBG] pgmap v8274: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:34:12.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:34:12.900Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:14.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:13 smithi130 bash[51302]: debug 2023-08-28T02:34:13.749+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:13.753033+0000) 2023-08-28T02:34:14.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:14 smithi006 bash[27292]: cluster 2023-08-28T02:34:13.825350+0000 mgr.y (mgr.14901) 8268 : cluster [DBG] pgmap v8275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:14.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:14 smithi006 bash[27292]: audit 2023-08-28T02:34:14.095766+0000 mon.c (mon.2) 3282 : audit [DBG] from='client.? 172.21.15.6:0/482086542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:14 smithi006 bash[33918]: cluster 2023-08-28T02:34:13.825350+0000 mgr.y (mgr.14901) 8268 : cluster [DBG] pgmap v8275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:14 smithi006 bash[33918]: audit 2023-08-28T02:34:14.095766+0000 mon.c (mon.2) 3282 : audit [DBG] from='client.? 172.21.15.6:0/482086542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:14 smithi130 bash[27579]: cluster 2023-08-28T02:34:13.825350+0000 mgr.y (mgr.14901) 8268 : cluster [DBG] pgmap v8275: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:14.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:14 smithi130 bash[27579]: audit 2023-08-28T02:34:14.095766+0000 mon.c (mon.2) 3282 : audit [DBG] from='client.? 172.21.15.6:0/482086542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:14 smithi130 bash[51302]: debug 2023-08-28T02:34:14.753+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:14.753364+0000) 2023-08-28T02:34:15.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:34:14.777Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:15.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:34:14.779Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:16.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:15 smithi130 bash[51302]: debug 2023-08-28T02:34:15.749+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:15.753573+0000) 2023-08-28T02:34:17.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:16 smithi130 bash[51302]: debug 2023-08-28T02:34:16.749+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:16.753839+0000) 2023-08-28T02:34:17.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:16 smithi130 bash[27579]: cluster 2023-08-28T02:34:15.826409+0000 mgr.y (mgr.14901) 8269 : cluster [DBG] pgmap v8276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:17.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:16 smithi130 bash[27579]: audit 2023-08-28T02:34:16.505825+0000 mon.a (mon.0) 5214 : audit [DBG] from='client.? 172.21.15.6:0/8743927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:17.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:16 smithi006 bash[27292]: cluster 2023-08-28T02:34:15.826409+0000 mgr.y (mgr.14901) 8269 : cluster [DBG] pgmap v8276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:17.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:16 smithi006 bash[27292]: audit 2023-08-28T02:34:16.505825+0000 mon.a (mon.0) 5214 : audit [DBG] from='client.? 172.21.15.6:0/8743927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:17.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:16 smithi006 bash[33918]: cluster 2023-08-28T02:34:15.826409+0000 mgr.y (mgr.14901) 8269 : cluster [DBG] pgmap v8276: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:17.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:16 smithi006 bash[33918]: audit 2023-08-28T02:34:16.505825+0000 mon.a (mon.0) 5214 : audit [DBG] from='client.? 172.21.15.6:0/8743927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:18.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:17 smithi130 bash[51302]: debug 2023-08-28T02:34:17.748+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:17.754090+0000) 2023-08-28T02:34:19.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:18 smithi130 bash[27579]: cluster 2023-08-28T02:34:17.827173+0000 mgr.y (mgr.14901) 8270 : cluster [DBG] pgmap v8277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:19.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:18 smithi130 bash[51302]: debug 2023-08-28T02:34:18.748+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:18.754382+0000) 2023-08-28T02:34:19.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:18 smithi006 bash[27292]: cluster 2023-08-28T02:34:17.827173+0000 mgr.y (mgr.14901) 8270 : cluster [DBG] pgmap v8277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:19.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:18 smithi006 bash[33918]: cluster 2023-08-28T02:34:17.827173+0000 mgr.y (mgr.14901) 8270 : cluster [DBG] pgmap v8277: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:20.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:19 smithi130 bash[27579]: audit 2023-08-28T02:34:18.922333+0000 mon.c (mon.2) 3283 : audit [DBG] from='client.? 172.21.15.6:0/3009741084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:20.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:19 smithi130 bash[51302]: debug 2023-08-28T02:34:19.748+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:19.754655+0000) 2023-08-28T02:34:20.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:19 smithi006 bash[33918]: audit 2023-08-28T02:34:18.922333+0000 mon.c (mon.2) 3283 : audit [DBG] from='client.? 172.21.15.6:0/3009741084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:20.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:19 smithi006 bash[27292]: audit 2023-08-28T02:34:18.922333+0000 mon.c (mon.2) 3283 : audit [DBG] from='client.? 172.21.15.6:0/3009741084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:21.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:20 smithi130 bash[27579]: cluster 2023-08-28T02:34:19.828497+0000 mgr.y (mgr.14901) 8271 : cluster [DBG] pgmap v8278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:21.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:20 smithi130 bash[51302]: debug 2023-08-28T02:34:20.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:20.754958+0000) 2023-08-28T02:34:21.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:20 smithi006 bash[33918]: cluster 2023-08-28T02:34:19.828497+0000 mgr.y (mgr.14901) 8271 : cluster [DBG] pgmap v8278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:21.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:20 smithi006 bash[27292]: cluster 2023-08-28T02:34:19.828497+0000 mgr.y (mgr.14901) 8271 : cluster [DBG] pgmap v8278: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:21.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:34:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:34:20] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:34:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:34:21.336Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:34:21.337Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:22.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:21 smithi130 bash[27579]: audit 2023-08-28T02:34:21.331018+0000 mon.c (mon.2) 3284 : audit [DBG] from='client.? 172.21.15.6:0/3312253531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:22.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:21 smithi130 bash[51302]: debug 2023-08-28T02:34:21.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:21.755270+0000) 2023-08-28T02:34:22.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:21 smithi006 bash[33918]: audit 2023-08-28T02:34:21.331018+0000 mon.c (mon.2) 3284 : audit [DBG] from='client.? 172.21.15.6:0/3312253531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:22.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:21 smithi006 bash[27292]: audit 2023-08-28T02:34:21.331018+0000 mon.c (mon.2) 3284 : audit [DBG] from='client.? 172.21.15.6:0/3312253531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:23.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:22 smithi130 bash[27579]: cluster 2023-08-28T02:34:21.829251+0000 mgr.y (mgr.14901) 8272 : cluster [DBG] pgmap v8279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:23.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:22 smithi130 bash[51302]: debug 2023-08-28T02:34:22.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:22.755575+0000) 2023-08-28T02:34:23.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:22 smithi006 bash[33918]: cluster 2023-08-28T02:34:21.829251+0000 mgr.y (mgr.14901) 8272 : cluster [DBG] pgmap v8279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:23.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:22 smithi006 bash[27292]: cluster 2023-08-28T02:34:21.829251+0000 mgr.y (mgr.14901) 8272 : cluster [DBG] pgmap v8279: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:34:22.899Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:34:22.901Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:24.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:23 smithi130 bash[51302]: debug 2023-08-28T02:34:23.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:23.755808+0000) 2023-08-28T02:34:24.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:24 smithi006 bash[33918]: audit 2023-08-28T02:34:23.746117+0000 mon.c (mon.2) 3285 : audit [DBG] from='client.? 172.21.15.6:0/2994436336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:24.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:24 smithi006 bash[33918]: cluster 2023-08-28T02:34:23.829879+0000 mgr.y (mgr.14901) 8273 : cluster [DBG] pgmap v8280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:24 smithi006 bash[27292]: audit 2023-08-28T02:34:23.746117+0000 mon.c (mon.2) 3285 : audit [DBG] from='client.? 172.21.15.6:0/2994436336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:24 smithi006 bash[27292]: cluster 2023-08-28T02:34:23.829879+0000 mgr.y (mgr.14901) 8273 : cluster [DBG] pgmap v8280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:24 smithi130 bash[27579]: audit 2023-08-28T02:34:23.746117+0000 mon.c (mon.2) 3285 : audit [DBG] from='client.? 172.21.15.6:0/2994436336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:24 smithi130 bash[27579]: cluster 2023-08-28T02:34:23.829879+0000 mgr.y (mgr.14901) 8273 : cluster [DBG] pgmap v8280: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:25.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:24 smithi130 bash[51302]: debug 2023-08-28T02:34:24.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:24.756111+0000) 2023-08-28T02:34:25.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:34:24.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:25.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:34:24.780Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:26.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:25 smithi130 bash[51302]: debug 2023-08-28T02:34:25.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:25.756362+0000) 2023-08-28T02:34:27.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:26 smithi130 bash[27579]: cluster 2023-08-28T02:34:25.830673+0000 mgr.y (mgr.14901) 8274 : cluster [DBG] pgmap v8281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:27.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:26 smithi130 bash[27579]: audit 2023-08-28T02:34:26.162927+0000 mon.c (mon.2) 3286 : audit [DBG] from='client.? 172.21.15.6:0/4262393274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:27.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:26 smithi130 bash[51302]: debug 2023-08-28T02:34:26.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:26.756704+0000) 2023-08-28T02:34:27.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:26 smithi006 bash[33918]: cluster 2023-08-28T02:34:25.830673+0000 mgr.y (mgr.14901) 8274 : cluster [DBG] pgmap v8281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:27.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:26 smithi006 bash[33918]: audit 2023-08-28T02:34:26.162927+0000 mon.c (mon.2) 3286 : audit [DBG] from='client.? 172.21.15.6:0/4262393274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:27.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:26 smithi006 bash[27292]: cluster 2023-08-28T02:34:25.830673+0000 mgr.y (mgr.14901) 8274 : cluster [DBG] pgmap v8281: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:27.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:26 smithi006 bash[27292]: audit 2023-08-28T02:34:26.162927+0000 mon.c (mon.2) 3286 : audit [DBG] from='client.? 172.21.15.6:0/4262393274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:28.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:27 smithi130 bash[51302]: debug 2023-08-28T02:34:27.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:27.756952+0000) 2023-08-28T02:34:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:28 smithi130 bash[51302]: debug 2023-08-28T02:34:28.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:28.757223+0000) 2023-08-28T02:34:29.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:28 smithi130 bash[27579]: cluster 2023-08-28T02:34:27.831416+0000 mgr.y (mgr.14901) 8275 : cluster [DBG] pgmap v8282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:29.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:28 smithi130 bash[27579]: audit 2023-08-28T02:34:28.574711+0000 mon.c (mon.2) 3287 : audit [DBG] from='client.? 172.21.15.6:0/2674809204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:29.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:28 smithi006 bash[33918]: cluster 2023-08-28T02:34:27.831416+0000 mgr.y (mgr.14901) 8275 : cluster [DBG] pgmap v8282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:29.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:28 smithi006 bash[33918]: audit 2023-08-28T02:34:28.574711+0000 mon.c (mon.2) 3287 : audit [DBG] from='client.? 172.21.15.6:0/2674809204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:29.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:28 smithi006 bash[27292]: cluster 2023-08-28T02:34:27.831416+0000 mgr.y (mgr.14901) 8275 : cluster [DBG] pgmap v8282: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:29.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:28 smithi006 bash[27292]: audit 2023-08-28T02:34:28.574711+0000 mon.c (mon.2) 3287 : audit [DBG] from='client.? 172.21.15.6:0/2674809204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:30.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:29 smithi130 bash[51302]: debug 2023-08-28T02:34:29.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:29.757535+0000) 2023-08-28T02:34:31.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:30 smithi130 bash[27579]: cluster 2023-08-28T02:34:29.832575+0000 mgr.y (mgr.14901) 8276 : cluster [DBG] pgmap v8283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:31.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:30 smithi130 bash[51302]: debug 2023-08-28T02:34:30.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:30.757777+0000) 2023-08-28T02:34:31.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:30 smithi006 bash[27292]: cluster 2023-08-28T02:34:29.832575+0000 mgr.y (mgr.14901) 8276 : cluster [DBG] pgmap v8283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:31.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:34:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:34:30] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:34:31.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:30 smithi006 bash[33918]: cluster 2023-08-28T02:34:29.832575+0000 mgr.y (mgr.14901) 8276 : cluster [DBG] pgmap v8283: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:34:31.336Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:34:31.338Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:32.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:31 smithi130 bash[27579]: audit 2023-08-28T02:34:30.997756+0000 mon.c (mon.2) 3288 : audit [DBG] from='client.? 172.21.15.6:0/475159751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:32.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:31 smithi130 bash[51302]: debug 2023-08-28T02:34:31.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:31.758097+0000) 2023-08-28T02:34:32.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:31 smithi006 bash[27292]: audit 2023-08-28T02:34:30.997756+0000 mon.c (mon.2) 3288 : audit [DBG] from='client.? 172.21.15.6:0/475159751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:32.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:31 smithi006 bash[33918]: audit 2023-08-28T02:34:30.997756+0000 mon.c (mon.2) 3288 : audit [DBG] from='client.? 172.21.15.6:0/475159751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:33.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:32 smithi130 bash[27579]: cluster 2023-08-28T02:34:31.833276+0000 mgr.y (mgr.14901) 8277 : cluster [DBG] pgmap v8284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:33.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:32 smithi130 bash[51302]: debug 2023-08-28T02:34:32.752+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:32.758298+0000) 2023-08-28T02:34:33.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:32 smithi006 bash[33918]: cluster 2023-08-28T02:34:31.833276+0000 mgr.y (mgr.14901) 8277 : cluster [DBG] pgmap v8284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:33.469 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:34:32.900Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:33.470 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:34:32.901Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:33.470 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:32 smithi006 bash[27292]: cluster 2023-08-28T02:34:31.833276+0000 mgr.y (mgr.14901) 8277 : cluster [DBG] pgmap v8284: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:34.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:33 smithi130 bash[51302]: debug 2023-08-28T02:34:33.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:33.758606+0000) 2023-08-28T02:34:34.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:34 smithi006 bash[27292]: audit 2023-08-28T02:34:33.401320+0000 mon.a (mon.0) 5215 : audit [DBG] from='client.? 172.21.15.6:0/2176034765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:34.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:34 smithi006 bash[27292]: cluster 2023-08-28T02:34:33.833944+0000 mgr.y (mgr.14901) 8278 : cluster [DBG] pgmap v8285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:34 smithi006 bash[33918]: audit 2023-08-28T02:34:33.401320+0000 mon.a (mon.0) 5215 : audit [DBG] from='client.? 172.21.15.6:0/2176034765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:34 smithi006 bash[33918]: cluster 2023-08-28T02:34:33.833944+0000 mgr.y (mgr.14901) 8278 : cluster [DBG] pgmap v8285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:34 smithi130 bash[27579]: audit 2023-08-28T02:34:33.401320+0000 mon.a (mon.0) 5215 : audit [DBG] from='client.? 172.21.15.6:0/2176034765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:34 smithi130 bash[27579]: cluster 2023-08-28T02:34:33.833944+0000 mgr.y (mgr.14901) 8278 : cluster [DBG] pgmap v8285: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:35.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:34 smithi130 bash[51302]: debug 2023-08-28T02:34:34.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:34.758951+0000) 2023-08-28T02:34:35.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:34:34.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:35.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:34:34.780Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:36.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:35 smithi130 bash[27579]: audit 2023-08-28T02:34:35.818853+0000 mon.a (mon.0) 5216 : audit [DBG] from='client.? 172.21.15.6:0/1171129655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:36.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:35 smithi130 bash[51302]: debug 2023-08-28T02:34:35.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:35.759267+0000) 2023-08-28T02:34:36.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:35 smithi006 bash[33918]: audit 2023-08-28T02:34:35.818853+0000 mon.a (mon.0) 5216 : audit [DBG] from='client.? 172.21.15.6:0/1171129655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:36.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:35 smithi006 bash[27292]: audit 2023-08-28T02:34:35.818853+0000 mon.a (mon.0) 5216 : audit [DBG] from='client.? 172.21.15.6:0/1171129655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:37.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:36 smithi130 bash[27579]: cluster 2023-08-28T02:34:35.834935+0000 mgr.y (mgr.14901) 8279 : cluster [DBG] pgmap v8286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:37.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:36 smithi130 bash[51302]: debug 2023-08-28T02:34:36.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:36.759544+0000) 2023-08-28T02:34:37.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:36 smithi006 bash[33918]: cluster 2023-08-28T02:34:35.834935+0000 mgr.y (mgr.14901) 8279 : cluster [DBG] pgmap v8286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:37.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:36 smithi006 bash[27292]: cluster 2023-08-28T02:34:35.834935+0000 mgr.y (mgr.14901) 8279 : cluster [DBG] pgmap v8286: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:37 smithi130 bash[51302]: debug 2023-08-28T02:34:37.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:37.759807+0000) 2023-08-28T02:34:39.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:38 smithi130 bash[51302]: debug 2023-08-28T02:34:38.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:38.760088+0000) 2023-08-28T02:34:39.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:38 smithi130 bash[27579]: cluster 2023-08-28T02:34:37.835583+0000 mgr.y (mgr.14901) 8280 : cluster [DBG] pgmap v8287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:39.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:38 smithi130 bash[27579]: audit 2023-08-28T02:34:38.232324+0000 mon.a (mon.0) 5217 : audit [DBG] from='client.? 172.21.15.6:0/2707269891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:39.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:38 smithi006 bash[27292]: cluster 2023-08-28T02:34:37.835583+0000 mgr.y (mgr.14901) 8280 : cluster [DBG] pgmap v8287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:39.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:38 smithi006 bash[27292]: audit 2023-08-28T02:34:38.232324+0000 mon.a (mon.0) 5217 : audit [DBG] from='client.? 172.21.15.6:0/2707269891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:39.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:38 smithi006 bash[33918]: cluster 2023-08-28T02:34:37.835583+0000 mgr.y (mgr.14901) 8280 : cluster [DBG] pgmap v8287: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:39.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:38 smithi006 bash[33918]: audit 2023-08-28T02:34:38.232324+0000 mon.a (mon.0) 5217 : audit [DBG] from='client.? 172.21.15.6:0/2707269891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:39.865 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:39 smithi130 bash[51302]: debug 2023-08-28T02:34:39.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:39.760285+0000) 2023-08-28T02:34:41.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:40 smithi130 bash[51302]: debug 2023-08-28T02:34:40.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:40.760535+0000) 2023-08-28T02:34:41.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:40 smithi130 bash[27579]: cluster 2023-08-28T02:34:39.836638+0000 mgr.y (mgr.14901) 8281 : cluster [DBG] pgmap v8288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:41.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:40 smithi130 bash[27579]: audit 2023-08-28T02:34:40.484111+0000 mon.b (mon.1) 900 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:34:41.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:40 smithi130 bash[27579]: audit 2023-08-28T02:34:40.654518+0000 mon.c (mon.2) 3289 : audit [DBG] from='client.? 172.21.15.6:0/1964873542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:34:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:34:40] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:34:41.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:40 smithi006 bash[33918]: cluster 2023-08-28T02:34:39.836638+0000 mgr.y (mgr.14901) 8281 : cluster [DBG] pgmap v8288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:41.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:40 smithi006 bash[33918]: audit 2023-08-28T02:34:40.484111+0000 mon.b (mon.1) 900 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:34:41.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:40 smithi006 bash[33918]: audit 2023-08-28T02:34:40.654518+0000 mon.c (mon.2) 3289 : audit [DBG] from='client.? 172.21.15.6:0/1964873542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:41.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:40 smithi006 bash[27292]: cluster 2023-08-28T02:34:39.836638+0000 mgr.y (mgr.14901) 8281 : cluster [DBG] pgmap v8288: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:41.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:40 smithi006 bash[27292]: audit 2023-08-28T02:34:40.484111+0000 mon.b (mon.1) 900 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:34:41.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:40 smithi006 bash[27292]: audit 2023-08-28T02:34:40.654518+0000 mon.c (mon.2) 3289 : audit [DBG] from='client.? 172.21.15.6:0/1964873542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:34:41.336Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:34:41.338Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:41 smithi130 bash[51302]: debug 2023-08-28T02:34:41.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:41.760834+0000) 2023-08-28T02:34:43.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:42 smithi130 bash[27579]: cluster 2023-08-28T02:34:41.837324+0000 mgr.y (mgr.14901) 8282 : cluster [DBG] pgmap v8289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:43.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:42 smithi130 bash[51302]: debug 2023-08-28T02:34:42.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:42.761129+0000) 2023-08-28T02:34:43.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:42 smithi006 bash[27292]: cluster 2023-08-28T02:34:41.837324+0000 mgr.y (mgr.14901) 8282 : cluster [DBG] pgmap v8289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:43.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:42 smithi006 bash[33918]: cluster 2023-08-28T02:34:41.837324+0000 mgr.y (mgr.14901) 8282 : cluster [DBG] pgmap v8289: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:34:42.900Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:34:42.902Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:44.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:43 smithi130 bash[27579]: audit 2023-08-28T02:34:43.077561+0000 mon.a (mon.0) 5218 : audit [DBG] from='client.? 172.21.15.6:0/2859608743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:44.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:43 smithi130 bash[51302]: debug 2023-08-28T02:34:43.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:43.761329+0000) 2023-08-28T02:34:44.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:43 smithi006 bash[27292]: audit 2023-08-28T02:34:43.077561+0000 mon.a (mon.0) 5218 : audit [DBG] from='client.? 172.21.15.6:0/2859608743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:44.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:43 smithi006 bash[33918]: audit 2023-08-28T02:34:43.077561+0000 mon.a (mon.0) 5218 : audit [DBG] from='client.? 172.21.15.6:0/2859608743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:45.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:44 smithi130 bash[27579]: cluster 2023-08-28T02:34:43.838130+0000 mgr.y (mgr.14901) 8283 : cluster [DBG] pgmap v8290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:45.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:44 smithi130 bash[51302]: debug 2023-08-28T02:34:44.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:44.761589+0000) 2023-08-28T02:34:45.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:44 smithi006 bash[27292]: cluster 2023-08-28T02:34:43.838130+0000 mgr.y (mgr.14901) 8283 : cluster [DBG] pgmap v8290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:45.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:44 smithi006 bash[33918]: cluster 2023-08-28T02:34:43.838130+0000 mgr.y (mgr.14901) 8283 : cluster [DBG] pgmap v8290: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:45.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:34:44.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:45.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:34:44.781Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:46.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:45 smithi130 bash[27579]: audit 2023-08-28T02:34:45.545072+0000 mon.a (mon.0) 5219 : audit [DBG] from='client.? 172.21.15.6:0/2694934656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:46.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:45 smithi130 bash[51302]: debug 2023-08-28T02:34:45.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:45.761826+0000) 2023-08-28T02:34:46.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:45 smithi006 bash[27292]: audit 2023-08-28T02:34:45.545072+0000 mon.a (mon.0) 5219 : audit [DBG] from='client.? 172.21.15.6:0/2694934656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:46.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:45 smithi006 bash[33918]: audit 2023-08-28T02:34:45.545072+0000 mon.a (mon.0) 5219 : audit [DBG] from='client.? 172.21.15.6:0/2694934656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:46 smithi130 bash[51302]: debug 2023-08-28T02:34:46.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:46.762067+0000) 2023-08-28T02:34:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:46 smithi130 bash[27579]: cluster 2023-08-28T02:34:45.839138+0000 mgr.y (mgr.14901) 8284 : cluster [DBG] pgmap v8291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:46 smithi130 bash[27579]: audit 2023-08-28T02:34:46.381273+0000 mon.a (mon.0) 5220 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:34:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:46 smithi130 bash[27579]: audit 2023-08-28T02:34:46.836142+0000 mon.a (mon.0) 5221 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:34:47.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:46 smithi006 bash[27292]: cluster 2023-08-28T02:34:45.839138+0000 mgr.y (mgr.14901) 8284 : cluster [DBG] pgmap v8291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:47.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:46 smithi006 bash[27292]: audit 2023-08-28T02:34:46.381273+0000 mon.a (mon.0) 5220 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:34:47.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:46 smithi006 bash[27292]: audit 2023-08-28T02:34:46.836142+0000 mon.a (mon.0) 5221 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:34:47.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:46 smithi006 bash[33918]: cluster 2023-08-28T02:34:45.839138+0000 mgr.y (mgr.14901) 8284 : cluster [DBG] pgmap v8291: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:47.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:46 smithi006 bash[33918]: audit 2023-08-28T02:34:46.381273+0000 mon.a (mon.0) 5220 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:34:47.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:46 smithi006 bash[33918]: audit 2023-08-28T02:34:46.836142+0000 mon.a (mon.0) 5221 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:34:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:47 smithi130 bash[51302]: debug 2023-08-28T02:34:47.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:47.762258+0000) 2023-08-28T02:34:48.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:47 smithi130 bash[27579]: audit 2023-08-28T02:34:47.124525+0000 mon.b (mon.1) 901 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:34:48.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:47 smithi130 bash[27579]: audit 2023-08-28T02:34:47.126746+0000 mon.b (mon.1) 902 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:34:48.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:47 smithi130 bash[27579]: audit 2023-08-28T02:34:47.136090+0000 mon.a (mon.0) 5222 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:34:48.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:47 smithi006 bash[27292]: audit 2023-08-28T02:34:47.124525+0000 mon.b (mon.1) 901 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:34:48.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:47 smithi006 bash[27292]: audit 2023-08-28T02:34:47.126746+0000 mon.b (mon.1) 902 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:34:48.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:47 smithi006 bash[27292]: audit 2023-08-28T02:34:47.136090+0000 mon.a (mon.0) 5222 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:34:48.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:47 smithi006 bash[33918]: audit 2023-08-28T02:34:47.124525+0000 mon.b (mon.1) 901 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:34:48.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:47 smithi006 bash[33918]: audit 2023-08-28T02:34:47.126746+0000 mon.b (mon.1) 902 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:34:48.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:47 smithi006 bash[33918]: audit 2023-08-28T02:34:47.136090+0000 mon.a (mon.0) 5222 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:34:49.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:48 smithi130 bash[27579]: cluster 2023-08-28T02:34:47.839799+0000 mgr.y (mgr.14901) 8285 : cluster [DBG] pgmap v8292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:49.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:48 smithi130 bash[27579]: audit 2023-08-28T02:34:47.969246+0000 mon.c (mon.2) 3290 : audit [DBG] from='client.? 172.21.15.6:0/1870118221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:49.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:48 smithi130 bash[51302]: debug 2023-08-28T02:34:48.756+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:48.762519+0000) 2023-08-28T02:34:49.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:48 smithi006 bash[27292]: cluster 2023-08-28T02:34:47.839799+0000 mgr.y (mgr.14901) 8285 : cluster [DBG] pgmap v8292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:49.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:48 smithi006 bash[27292]: audit 2023-08-28T02:34:47.969246+0000 mon.c (mon.2) 3290 : audit [DBG] from='client.? 172.21.15.6:0/1870118221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:49.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:48 smithi006 bash[33918]: cluster 2023-08-28T02:34:47.839799+0000 mgr.y (mgr.14901) 8285 : cluster [DBG] pgmap v8292: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:49.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:48 smithi006 bash[33918]: audit 2023-08-28T02:34:47.969246+0000 mon.c (mon.2) 3290 : audit [DBG] from='client.? 172.21.15.6:0/1870118221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:50.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:49 smithi130 bash[51302]: debug 2023-08-28T02:34:49.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:49.762845+0000) 2023-08-28T02:34:51.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:50 smithi130 bash[51302]: debug 2023-08-28T02:34:50.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:50.763136+0000) 2023-08-28T02:34:51.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:50 smithi130 bash[27579]: cluster 2023-08-28T02:34:49.840985+0000 mgr.y (mgr.14901) 8286 : cluster [DBG] pgmap v8293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:51.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:50 smithi130 bash[27579]: audit 2023-08-28T02:34:50.393694+0000 mon.a (mon.0) 5223 : audit [DBG] from='client.? 172.21.15.6:0/3274718830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:34:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:34:50] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T02:34:51.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:50 smithi006 bash[27292]: cluster 2023-08-28T02:34:49.840985+0000 mgr.y (mgr.14901) 8286 : cluster [DBG] pgmap v8293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:51.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:50 smithi006 bash[27292]: audit 2023-08-28T02:34:50.393694+0000 mon.a (mon.0) 5223 : audit [DBG] from='client.? 172.21.15.6:0/3274718830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:51.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:50 smithi006 bash[33918]: cluster 2023-08-28T02:34:49.840985+0000 mgr.y (mgr.14901) 8286 : cluster [DBG] pgmap v8293: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:51.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:50 smithi006 bash[33918]: audit 2023-08-28T02:34:50.393694+0000 mon.a (mon.0) 5223 : audit [DBG] from='client.? 172.21.15.6:0/3274718830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:34:51.337Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:34:51.339Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:52.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:51 smithi130 bash[51302]: debug 2023-08-28T02:34:51.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:51.763461+0000) 2023-08-28T02:34:53.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:52 smithi130 bash[51302]: debug 2023-08-28T02:34:52.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:52.763826+0000) 2023-08-28T02:34:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:52 smithi130 bash[27579]: cluster 2023-08-28T02:34:51.841729+0000 mgr.y (mgr.14901) 8287 : cluster [DBG] pgmap v8294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:52 smithi130 bash[27579]: audit 2023-08-28T02:34:52.805251+0000 mon.a (mon.0) 5224 : audit [DBG] from='client.? 172.21.15.6:0/2259645396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:53.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:52 smithi006 bash[33918]: cluster 2023-08-28T02:34:51.841729+0000 mgr.y (mgr.14901) 8287 : cluster [DBG] pgmap v8294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:53.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:52 smithi006 bash[33918]: audit 2023-08-28T02:34:52.805251+0000 mon.a (mon.0) 5224 : audit [DBG] from='client.? 172.21.15.6:0/2259645396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:53.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:52 smithi006 bash[27292]: cluster 2023-08-28T02:34:51.841729+0000 mgr.y (mgr.14901) 8287 : cluster [DBG] pgmap v8294: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:53.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:52 smithi006 bash[27292]: audit 2023-08-28T02:34:52.805251+0000 mon.a (mon.0) 5224 : audit [DBG] from='client.? 172.21.15.6:0/2259645396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:34:52.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:34:52.904Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:54.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:53 smithi130 bash[51302]: debug 2023-08-28T02:34:53.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:53.764189+0000) 2023-08-28T02:34:54.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:54 smithi006 bash[33918]: cluster 2023-08-28T02:34:53.842469+0000 mgr.y (mgr.14901) 8288 : cluster [DBG] pgmap v8295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:54 smithi006 bash[27292]: cluster 2023-08-28T02:34:53.842469+0000 mgr.y (mgr.14901) 8288 : cluster [DBG] pgmap v8295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:54 smithi130 bash[27579]: cluster 2023-08-28T02:34:53.842469+0000 mgr.y (mgr.14901) 8288 : cluster [DBG] pgmap v8295: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:55.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:54 smithi130 bash[51302]: debug 2023-08-28T02:34:54.764+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:54.764655+0000) 2023-08-28T02:34:55.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:34:54.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:55.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:34:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:34:54.781Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:34:55.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:55 smithi130 bash[27579]: audit 2023-08-28T02:34:55.208684+0000 mon.c (mon.2) 3291 : audit [DBG] from='client.? 172.21.15.6:0/2430169802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:55.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:55 smithi006 bash[33918]: audit 2023-08-28T02:34:55.208684+0000 mon.c (mon.2) 3291 : audit [DBG] from='client.? 172.21.15.6:0/2430169802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:55.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:55 smithi006 bash[27292]: audit 2023-08-28T02:34:55.208684+0000 mon.c (mon.2) 3291 : audit [DBG] from='client.? 172.21.15.6:0/2430169802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:56.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:55 smithi130 bash[51302]: debug 2023-08-28T02:34:55.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:55.764931+0000) 2023-08-28T02:34:56.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:56 smithi130 bash[27579]: cluster 2023-08-28T02:34:55.843613+0000 mgr.y (mgr.14901) 8289 : cluster [DBG] pgmap v8296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:56.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:56 smithi006 bash[27292]: cluster 2023-08-28T02:34:55.843613+0000 mgr.y (mgr.14901) 8289 : cluster [DBG] pgmap v8296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:56.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:56 smithi006 bash[33918]: cluster 2023-08-28T02:34:55.843613+0000 mgr.y (mgr.14901) 8289 : cluster [DBG] pgmap v8296: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:57.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:56 smithi130 bash[51302]: debug 2023-08-28T02:34:56.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:56.765222+0000) 2023-08-28T02:34:57.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:57 smithi006 bash[33918]: audit 2023-08-28T02:34:57.623679+0000 mon.a (mon.0) 5225 : audit [DBG] from='client.? 172.21.15.6:0/886188719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:57.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:57 smithi006 bash[27292]: audit 2023-08-28T02:34:57.623679+0000 mon.a (mon.0) 5225 : audit [DBG] from='client.? 172.21.15.6:0/886188719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:58.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:57 smithi130 bash[27579]: audit 2023-08-28T02:34:57.623679+0000 mon.a (mon.0) 5225 : audit [DBG] from='client.? 172.21.15.6:0/886188719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:58.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:57 smithi130 bash[51302]: debug 2023-08-28T02:34:57.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:57.765484+0000) 2023-08-28T02:34:58.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:34:58 smithi006 bash[33918]: cluster 2023-08-28T02:34:57.844310+0000 mgr.y (mgr.14901) 8290 : cluster [DBG] pgmap v8297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:58.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:34:58 smithi006 bash[27292]: cluster 2023-08-28T02:34:57.844310+0000 mgr.y (mgr.14901) 8290 : cluster [DBG] pgmap v8297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:59.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:34:58 smithi130 bash[27579]: cluster 2023-08-28T02:34:57.844310+0000 mgr.y (mgr.14901) 8290 : cluster [DBG] pgmap v8297: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:34:59.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:58 smithi130 bash[51302]: debug 2023-08-28T02:34:58.764+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:58.765747+0000) 2023-08-28T02:35:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:34:59 smithi130 bash[51302]: debug 2023-08-28T02:34:59.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:59.766012+0000) 2023-08-28T02:35:01.081 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:00 smithi130 bash[51302]: debug 2023-08-28T02:35:00.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:00.766256+0000) 2023-08-28T02:35:01.081 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:00 smithi130 bash[27579]: cluster 2023-08-28T02:34:59.845493+0000 mgr.y (mgr.14901) 8291 : cluster [DBG] pgmap v8298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:01.082 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:00 smithi130 bash[27579]: audit 2023-08-28T02:35:00.038212+0000 mon.a (mon.0) 5226 : audit [DBG] from='client.? 172.21.15.6:0/2723719567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:35:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:35:00] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:35:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:00 smithi006 bash[27292]: cluster 2023-08-28T02:34:59.845493+0000 mgr.y (mgr.14901) 8291 : cluster [DBG] pgmap v8298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:00 smithi006 bash[27292]: audit 2023-08-28T02:35:00.038212+0000 mon.a (mon.0) 5226 : audit [DBG] from='client.? 172.21.15.6:0/2723719567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:01.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:00 smithi006 bash[33918]: cluster 2023-08-28T02:34:59.845493+0000 mgr.y (mgr.14901) 8291 : cluster [DBG] pgmap v8298: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:01.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:00 smithi006 bash[33918]: audit 2023-08-28T02:35:00.038212+0000 mon.a (mon.0) 5226 : audit [DBG] from='client.? 172.21.15.6:0/2723719567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:35:01.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:35:01.340Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:02.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:01 smithi130 bash[51302]: debug 2023-08-28T02:35:01.760+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:01.766590+0000) 2023-08-28T02:35:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:02 smithi130 bash[27579]: cluster 2023-08-28T02:35:01.846131+0000 mgr.y (mgr.14901) 8292 : cluster [DBG] pgmap v8299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:02 smithi130 bash[27579]: audit 2023-08-28T02:35:02.445702+0000 mon.a (mon.0) 5227 : audit [DBG] from='client.? 172.21.15.6:0/3992177920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:03.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:02 smithi130 bash[51302]: debug 2023-08-28T02:35:02.764+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:02.766937+0000) 2023-08-28T02:35:03.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:02 smithi006 bash[33918]: cluster 2023-08-28T02:35:01.846131+0000 mgr.y (mgr.14901) 8292 : cluster [DBG] pgmap v8299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:03.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:02 smithi006 bash[33918]: audit 2023-08-28T02:35:02.445702+0000 mon.a (mon.0) 5227 : audit [DBG] from='client.? 172.21.15.6:0/3992177920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:35:02.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:35:02.903Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:03.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:02 smithi006 bash[27292]: cluster 2023-08-28T02:35:01.846131+0000 mgr.y (mgr.14901) 8292 : cluster [DBG] pgmap v8299: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:03.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:02 smithi006 bash[27292]: audit 2023-08-28T02:35:02.445702+0000 mon.a (mon.0) 5227 : audit [DBG] from='client.? 172.21.15.6:0/3992177920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:04.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:03 smithi130 bash[51302]: debug 2023-08-28T02:35:03.764+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:03.767218+0000) 2023-08-28T02:35:04.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:04 smithi006 bash[27292]: cluster 2023-08-28T02:35:03.846788+0000 mgr.y (mgr.14901) 8293 : cluster [DBG] pgmap v8300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:04.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:04 smithi006 bash[33918]: cluster 2023-08-28T02:35:03.846788+0000 mgr.y (mgr.14901) 8293 : cluster [DBG] pgmap v8300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:04 smithi130 bash[27579]: cluster 2023-08-28T02:35:03.846788+0000 mgr.y (mgr.14901) 8293 : cluster [DBG] pgmap v8300: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:05.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:04 smithi130 bash[51302]: debug 2023-08-28T02:35:04.764+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:04.767500+0000) 2023-08-28T02:35:05.171 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:35:04.780Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:05.171 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:35:04.782Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:05.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:05 smithi006 bash[33918]: audit 2023-08-28T02:35:04.857096+0000 mon.c (mon.2) 3292 : audit [DBG] from='client.? 172.21.15.6:0/193864805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:05.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:05 smithi006 bash[27292]: audit 2023-08-28T02:35:04.857096+0000 mon.c (mon.2) 3292 : audit [DBG] from='client.? 172.21.15.6:0/193864805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:05.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:05 smithi130 bash[27579]: audit 2023-08-28T02:35:04.857096+0000 mon.c (mon.2) 3292 : audit [DBG] from='client.? 172.21.15.6:0/193864805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:06.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:05 smithi130 bash[51302]: debug 2023-08-28T02:35:05.764+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:05.767727+0000) 2023-08-28T02:35:06.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:06 smithi006 bash[27292]: cluster 2023-08-28T02:35:05.847921+0000 mgr.y (mgr.14901) 8294 : cluster [DBG] pgmap v8301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:06.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:06 smithi006 bash[33918]: cluster 2023-08-28T02:35:05.847921+0000 mgr.y (mgr.14901) 8294 : cluster [DBG] pgmap v8301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:06.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:06 smithi130 bash[27579]: cluster 2023-08-28T02:35:05.847921+0000 mgr.y (mgr.14901) 8294 : cluster [DBG] pgmap v8301: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:07.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:06 smithi130 bash[51302]: debug 2023-08-28T02:35:06.764+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:06.768001+0000) 2023-08-28T02:35:07.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:07 smithi130 bash[27579]: audit 2023-08-28T02:35:07.270526+0000 mon.a (mon.0) 5228 : audit [DBG] from='client.? 172.21.15.6:0/1318303021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:07.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:07 smithi006 bash[33918]: audit 2023-08-28T02:35:07.270526+0000 mon.a (mon.0) 5228 : audit [DBG] from='client.? 172.21.15.6:0/1318303021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:07.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:07 smithi006 bash[27292]: audit 2023-08-28T02:35:07.270526+0000 mon.a (mon.0) 5228 : audit [DBG] from='client.? 172.21.15.6:0/1318303021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:08.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:07 smithi130 bash[51302]: debug 2023-08-28T02:35:07.768+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:07.768289+0000) 2023-08-28T02:35:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:08 smithi130 bash[27579]: cluster 2023-08-28T02:35:07.848588+0000 mgr.y (mgr.14901) 8295 : cluster [DBG] pgmap v8302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:08.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:08 smithi006 bash[27292]: cluster 2023-08-28T02:35:07.848588+0000 mgr.y (mgr.14901) 8295 : cluster [DBG] pgmap v8302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:08.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:08 smithi006 bash[33918]: cluster 2023-08-28T02:35:07.848588+0000 mgr.y (mgr.14901) 8295 : cluster [DBG] pgmap v8302: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:09.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:08 smithi130 bash[51302]: debug 2023-08-28T02:35:08.764+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:08.768615+0000) 2023-08-28T02:35:10.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:09 smithi130 bash[27579]: audit 2023-08-28T02:35:09.690321+0000 mon.c (mon.2) 3293 : audit [DBG] from='client.? 172.21.15.6:0/3235518938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:10.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:09 smithi130 bash[51302]: debug 2023-08-28T02:35:09.764+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:09.768895+0000) 2023-08-28T02:35:10.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:09 smithi006 bash[27292]: audit 2023-08-28T02:35:09.690321+0000 mon.c (mon.2) 3293 : audit [DBG] from='client.? 172.21.15.6:0/3235518938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:10.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:09 smithi006 bash[33918]: audit 2023-08-28T02:35:09.690321+0000 mon.c (mon.2) 3293 : audit [DBG] from='client.? 172.21.15.6:0/3235518938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:11.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:10 smithi130 bash[27579]: cluster 2023-08-28T02:35:09.849750+0000 mgr.y (mgr.14901) 8296 : cluster [DBG] pgmap v8303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:11.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:10 smithi130 bash[51302]: debug 2023-08-28T02:35:10.764+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:10.769207+0000) 2023-08-28T02:35:11.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:35:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:35:10] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:35:11.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:10 smithi006 bash[33918]: cluster 2023-08-28T02:35:09.849750+0000 mgr.y (mgr.14901) 8296 : cluster [DBG] pgmap v8303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:11.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:10 smithi006 bash[27292]: cluster 2023-08-28T02:35:09.849750+0000 mgr.y (mgr.14901) 8296 : cluster [DBG] pgmap v8303: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:35:11.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:35:11.340Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:12.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:11 smithi130 bash[51302]: debug 2023-08-28T02:35:11.763+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:11.769521+0000) 2023-08-28T02:35:13.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:12 smithi130 bash[51302]: debug 2023-08-28T02:35:12.763+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:12.769793+0000) 2023-08-28T02:35:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:12 smithi130 bash[27579]: cluster 2023-08-28T02:35:11.850325+0000 mgr.y (mgr.14901) 8297 : cluster [DBG] pgmap v8304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:12 smithi130 bash[27579]: audit 2023-08-28T02:35:12.112745+0000 mon.c (mon.2) 3294 : audit [DBG] from='client.? 172.21.15.6:0/2128160928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:13.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:12 smithi006 bash[27292]: cluster 2023-08-28T02:35:11.850325+0000 mgr.y (mgr.14901) 8297 : cluster [DBG] pgmap v8304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:13.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:12 smithi006 bash[27292]: audit 2023-08-28T02:35:12.112745+0000 mon.c (mon.2) 3294 : audit [DBG] from='client.? 172.21.15.6:0/2128160928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:12 smithi006 bash[33918]: cluster 2023-08-28T02:35:11.850325+0000 mgr.y (mgr.14901) 8297 : cluster [DBG] pgmap v8304: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:12 smithi006 bash[33918]: audit 2023-08-28T02:35:12.112745+0000 mon.c (mon.2) 3294 : audit [DBG] from='client.? 172.21.15.6:0/2128160928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:35:12.902Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:35:12.904Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:14.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:13 smithi130 bash[51302]: debug 2023-08-28T02:35:13.763+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:13.770081+0000) 2023-08-28T02:35:14.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:14 smithi006 bash[33918]: cluster 2023-08-28T02:35:13.850907+0000 mgr.y (mgr.14901) 8298 : cluster [DBG] pgmap v8305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:14 smithi006 bash[27292]: cluster 2023-08-28T02:35:13.850907+0000 mgr.y (mgr.14901) 8298 : cluster [DBG] pgmap v8305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:14 smithi130 bash[27579]: cluster 2023-08-28T02:35:13.850907+0000 mgr.y (mgr.14901) 8298 : cluster [DBG] pgmap v8305: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:14 smithi130 bash[51302]: debug 2023-08-28T02:35:14.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:14.770329+0000) 2023-08-28T02:35:15.170 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:35:14.780Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:15.171 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:35:14.783Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:15.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:15 smithi006 bash[33918]: audit 2023-08-28T02:35:14.538892+0000 mon.c (mon.2) 3295 : audit [DBG] from='client.? 172.21.15.6:0/2124317425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:15.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:15 smithi006 bash[27292]: audit 2023-08-28T02:35:14.538892+0000 mon.c (mon.2) 3295 : audit [DBG] from='client.? 172.21.15.6:0/2124317425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:15.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:15 smithi130 bash[27579]: audit 2023-08-28T02:35:14.538892+0000 mon.c (mon.2) 3295 : audit [DBG] from='client.? 172.21.15.6:0/2124317425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:16.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:15 smithi130 bash[51302]: debug 2023-08-28T02:35:15.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:15.770612+0000) 2023-08-28T02:35:16.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:16 smithi006 bash[33918]: cluster 2023-08-28T02:35:15.852006+0000 mgr.y (mgr.14901) 8299 : cluster [DBG] pgmap v8306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:16.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:16 smithi006 bash[27292]: cluster 2023-08-28T02:35:15.852006+0000 mgr.y (mgr.14901) 8299 : cluster [DBG] pgmap v8306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:16 smithi130 bash[27579]: cluster 2023-08-28T02:35:15.852006+0000 mgr.y (mgr.14901) 8299 : cluster [DBG] pgmap v8306: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:17.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:16 smithi130 bash[51302]: debug 2023-08-28T02:35:16.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:16.770922+0000) 2023-08-28T02:35:17.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:17 smithi006 bash[27292]: audit 2023-08-28T02:35:16.947617+0000 mon.a (mon.0) 5229 : audit [DBG] from='client.? 172.21.15.6:0/2442724400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:17.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:17 smithi006 bash[33918]: audit 2023-08-28T02:35:16.947617+0000 mon.a (mon.0) 5229 : audit [DBG] from='client.? 172.21.15.6:0/2442724400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:17.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:17 smithi130 bash[27579]: audit 2023-08-28T02:35:16.947617+0000 mon.a (mon.0) 5229 : audit [DBG] from='client.? 172.21.15.6:0/2442724400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:18.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:17 smithi130 bash[51302]: debug 2023-08-28T02:35:17.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:17.771209+0000) 2023-08-28T02:35:18.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:18 smithi006 bash[27292]: cluster 2023-08-28T02:35:17.852650+0000 mgr.y (mgr.14901) 8300 : cluster [DBG] pgmap v8307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:18.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:18 smithi006 bash[33918]: cluster 2023-08-28T02:35:17.852650+0000 mgr.y (mgr.14901) 8300 : cluster [DBG] pgmap v8307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:18.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:18 smithi130 bash[27579]: cluster 2023-08-28T02:35:17.852650+0000 mgr.y (mgr.14901) 8300 : cluster [DBG] pgmap v8307: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:19.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:18 smithi130 bash[51302]: debug 2023-08-28T02:35:18.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:18.771454+0000) 2023-08-28T02:35:19.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:19 smithi006 bash[27292]: audit 2023-08-28T02:35:19.362156+0000 mon.c (mon.2) 3296 : audit [DBG] from='client.? 172.21.15.6:0/3043238578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:19.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:19 smithi006 bash[33918]: audit 2023-08-28T02:35:19.362156+0000 mon.c (mon.2) 3296 : audit [DBG] from='client.? 172.21.15.6:0/3043238578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:19.770 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:19 smithi130 bash[27579]: audit 2023-08-28T02:35:19.362156+0000 mon.c (mon.2) 3296 : audit [DBG] from='client.? 172.21.15.6:0/3043238578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:19 smithi130 bash[51302]: debug 2023-08-28T02:35:19.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:19.771711+0000) 2023-08-28T02:35:20.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:20 smithi006 bash[33918]: cluster 2023-08-28T02:35:19.854219+0000 mgr.y (mgr.14901) 8301 : cluster [DBG] pgmap v8308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:20.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:20 smithi006 bash[27292]: cluster 2023-08-28T02:35:19.854219+0000 mgr.y (mgr.14901) 8301 : cluster [DBG] pgmap v8308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:20.770 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:20 smithi130 bash[27579]: cluster 2023-08-28T02:35:19.854219+0000 mgr.y (mgr.14901) 8301 : cluster [DBG] pgmap v8308: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:21.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:20 smithi130 bash[51302]: debug 2023-08-28T02:35:20.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:20.771964+0000) 2023-08-28T02:35:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:35:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:35:20] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:35:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:35:21.338Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:35:21.341Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:22.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:21 smithi130 bash[27579]: audit 2023-08-28T02:35:21.780454+0000 mon.a (mon.0) 5230 : audit [DBG] from='client.? 172.21.15.6:0/436585667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:22.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:21 smithi130 bash[51302]: debug 2023-08-28T02:35:21.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:21.772238+0000) 2023-08-28T02:35:22.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:21 smithi006 bash[27292]: audit 2023-08-28T02:35:21.780454+0000 mon.a (mon.0) 5230 : audit [DBG] from='client.? 172.21.15.6:0/436585667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:22.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:21 smithi006 bash[33918]: audit 2023-08-28T02:35:21.780454+0000 mon.a (mon.0) 5230 : audit [DBG] from='client.? 172.21.15.6:0/436585667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:23.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:22 smithi130 bash[51302]: debug 2023-08-28T02:35:22.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:22.772510+0000) 2023-08-28T02:35:23.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:22 smithi130 bash[27579]: cluster 2023-08-28T02:35:21.854903+0000 mgr.y (mgr.14901) 8302 : cluster [DBG] pgmap v8309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:23.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:22 smithi006 bash[27292]: cluster 2023-08-28T02:35:21.854903+0000 mgr.y (mgr.14901) 8302 : cluster [DBG] pgmap v8309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:23.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:22 smithi006 bash[33918]: cluster 2023-08-28T02:35:21.854903+0000 mgr.y (mgr.14901) 8302 : cluster [DBG] pgmap v8309: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:35:22.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:35:22.904Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:24.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:23 smithi130 bash[51302]: debug 2023-08-28T02:35:23.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:23.772827+0000) 2023-08-28T02:35:24.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:24 smithi006 bash[27292]: cluster 2023-08-28T02:35:23.855525+0000 mgr.y (mgr.14901) 8303 : cluster [DBG] pgmap v8310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:24.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:24 smithi006 bash[33918]: cluster 2023-08-28T02:35:23.855525+0000 mgr.y (mgr.14901) 8303 : cluster [DBG] pgmap v8310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:24 smithi130 bash[27579]: cluster 2023-08-28T02:35:23.855525+0000 mgr.y (mgr.14901) 8303 : cluster [DBG] pgmap v8310: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:25.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:24 smithi130 bash[51302]: debug 2023-08-28T02:35:24.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:24.773141+0000) 2023-08-28T02:35:25.173 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:35:24.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:25.174 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:35:24.783Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:25.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:25 smithi006 bash[27292]: audit 2023-08-28T02:35:24.199700+0000 mon.c (mon.2) 3297 : audit [DBG] from='client.? 172.21.15.6:0/3252123729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:25.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:25 smithi006 bash[33918]: audit 2023-08-28T02:35:24.199700+0000 mon.c (mon.2) 3297 : audit [DBG] from='client.? 172.21.15.6:0/3252123729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:25.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:25 smithi130 bash[27579]: audit 2023-08-28T02:35:24.199700+0000 mon.c (mon.2) 3297 : audit [DBG] from='client.? 172.21.15.6:0/3252123729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:26.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:25 smithi130 bash[51302]: debug 2023-08-28T02:35:25.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:25.773407+0000) 2023-08-28T02:35:26.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:26 smithi006 bash[33918]: cluster 2023-08-28T02:35:25.856624+0000 mgr.y (mgr.14901) 8304 : cluster [DBG] pgmap v8311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:26.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:26 smithi006 bash[27292]: cluster 2023-08-28T02:35:25.856624+0000 mgr.y (mgr.14901) 8304 : cluster [DBG] pgmap v8311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:26 smithi130 bash[27579]: cluster 2023-08-28T02:35:25.856624+0000 mgr.y (mgr.14901) 8304 : cluster [DBG] pgmap v8311: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:27.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:26 smithi130 bash[51302]: debug 2023-08-28T02:35:26.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:26.773709+0000) 2023-08-28T02:35:27.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:27 smithi006 bash[27292]: audit 2023-08-28T02:35:26.613503+0000 mon.a (mon.0) 5231 : audit [DBG] from='client.? 172.21.15.6:0/4259334611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:27.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:27 smithi006 bash[33918]: audit 2023-08-28T02:35:26.613503+0000 mon.a (mon.0) 5231 : audit [DBG] from='client.? 172.21.15.6:0/4259334611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:27.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:27 smithi130 bash[27579]: audit 2023-08-28T02:35:26.613503+0000 mon.a (mon.0) 5231 : audit [DBG] from='client.? 172.21.15.6:0/4259334611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:28.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:27 smithi130 bash[51302]: debug 2023-08-28T02:35:27.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:27.773974+0000) 2023-08-28T02:35:28.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:28 smithi006 bash[33918]: cluster 2023-08-28T02:35:27.857296+0000 mgr.y (mgr.14901) 8305 : cluster [DBG] pgmap v8312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:28.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:28 smithi006 bash[27292]: cluster 2023-08-28T02:35:27.857296+0000 mgr.y (mgr.14901) 8305 : cluster [DBG] pgmap v8312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:28 smithi130 bash[27579]: cluster 2023-08-28T02:35:27.857296+0000 mgr.y (mgr.14901) 8305 : cluster [DBG] pgmap v8312: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:28 smithi130 bash[51302]: debug 2023-08-28T02:35:28.767+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:28.774264+0000) 2023-08-28T02:35:29.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:29 smithi006 bash[33918]: audit 2023-08-28T02:35:29.025992+0000 mon.c (mon.2) 3298 : audit [DBG] from='client.? 172.21.15.6:0/2681894619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:29.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:29 smithi006 bash[27292]: audit 2023-08-28T02:35:29.025992+0000 mon.c (mon.2) 3298 : audit [DBG] from='client.? 172.21.15.6:0/2681894619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:29.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:29 smithi130 bash[27579]: audit 2023-08-28T02:35:29.025992+0000 mon.c (mon.2) 3298 : audit [DBG] from='client.? 172.21.15.6:0/2681894619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:30.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:29 smithi130 bash[51302]: debug 2023-08-28T02:35:29.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:29.774583+0000) 2023-08-28T02:35:30.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:30 smithi006 bash[33918]: cluster 2023-08-28T02:35:29.858689+0000 mgr.y (mgr.14901) 8306 : cluster [DBG] pgmap v8313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:30.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:30 smithi006 bash[27292]: cluster 2023-08-28T02:35:29.858689+0000 mgr.y (mgr.14901) 8306 : cluster [DBG] pgmap v8313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:30 smithi130 bash[27579]: cluster 2023-08-28T02:35:29.858689+0000 mgr.y (mgr.14901) 8306 : cluster [DBG] pgmap v8313: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:31.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:30 smithi130 bash[51302]: debug 2023-08-28T02:35:30.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:30.774918+0000) 2023-08-28T02:35:31.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:35:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:35:30] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:35:31.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:31 smithi006 bash[33918]: audit 2023-08-28T02:35:31.448706+0000 mon.a (mon.0) 5232 : audit [DBG] from='client.? 172.21.15.6:0/1701600637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:31.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:31 smithi006 bash[27292]: audit 2023-08-28T02:35:31.448706+0000 mon.a (mon.0) 5232 : audit [DBG] from='client.? 172.21.15.6:0/1701600637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:35:31.340Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:35:31.341Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:31.774 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:31 smithi130 bash[27579]: audit 2023-08-28T02:35:31.448706+0000 mon.a (mon.0) 5232 : audit [DBG] from='client.? 172.21.15.6:0/1701600637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:32.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:31 smithi130 bash[51302]: debug 2023-08-28T02:35:31.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:31.775166+0000) 2023-08-28T02:35:32.774 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:32 smithi130 bash[27579]: cluster 2023-08-28T02:35:31.859419+0000 mgr.y (mgr.14901) 8307 : cluster [DBG] pgmap v8314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:32.903 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:32 smithi006 bash[27292]: cluster 2023-08-28T02:35:31.859419+0000 mgr.y (mgr.14901) 8307 : cluster [DBG] pgmap v8314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:32.904 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:32 smithi006 bash[33918]: cluster 2023-08-28T02:35:31.859419+0000 mgr.y (mgr.14901) 8307 : cluster [DBG] pgmap v8314: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:33.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:32 smithi130 bash[51302]: debug 2023-08-28T02:35:32.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:32.775423+0000) 2023-08-28T02:35:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:35:32.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:35:32.906Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:34.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:33 smithi130 bash[51302]: debug 2023-08-28T02:35:33.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:33.775715+0000) 2023-08-28T02:35:34.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:34 smithi006 bash[27292]: cluster 2023-08-28T02:35:33.860028+0000 mgr.y (mgr.14901) 8308 : cluster [DBG] pgmap v8315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:34.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:34 smithi006 bash[27292]: audit 2023-08-28T02:35:33.861347+0000 mon.c (mon.2) 3299 : audit [DBG] from='client.? 172.21.15.6:0/1593004888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:34 smithi006 bash[33918]: cluster 2023-08-28T02:35:33.860028+0000 mgr.y (mgr.14901) 8308 : cluster [DBG] pgmap v8315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:34 smithi006 bash[33918]: audit 2023-08-28T02:35:33.861347+0000 mon.c (mon.2) 3299 : audit [DBG] from='client.? 172.21.15.6:0/1593004888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:34.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:34 smithi130 bash[27579]: cluster 2023-08-28T02:35:33.860028+0000 mgr.y (mgr.14901) 8308 : cluster [DBG] pgmap v8315: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:34.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:34 smithi130 bash[27579]: audit 2023-08-28T02:35:33.861347+0000 mon.c (mon.2) 3299 : audit [DBG] from='client.? 172.21.15.6:0/1593004888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:35.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:34 smithi130 bash[51302]: debug 2023-08-28T02:35:34.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:34.776004+0000) 2023-08-28T02:35:35.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:35:34.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:35.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:35:34.784Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:36.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:35 smithi130 bash[51302]: debug 2023-08-28T02:35:35.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:35.776293+0000) 2023-08-28T02:35:37.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:36 smithi130 bash[27579]: cluster 2023-08-28T02:35:35.861185+0000 mgr.y (mgr.14901) 8309 : cluster [DBG] pgmap v8316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:37.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:36 smithi130 bash[27579]: audit 2023-08-28T02:35:36.271264+0000 mon.a (mon.0) 5233 : audit [DBG] from='client.? 172.21.15.6:0/87049698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:37.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:36 smithi130 bash[51302]: debug 2023-08-28T02:35:36.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:36.776572+0000) 2023-08-28T02:35:37.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:36 smithi006 bash[33918]: cluster 2023-08-28T02:35:35.861185+0000 mgr.y (mgr.14901) 8309 : cluster [DBG] pgmap v8316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:37.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:36 smithi006 bash[33918]: audit 2023-08-28T02:35:36.271264+0000 mon.a (mon.0) 5233 : audit [DBG] from='client.? 172.21.15.6:0/87049698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:37.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:36 smithi006 bash[27292]: cluster 2023-08-28T02:35:35.861185+0000 mgr.y (mgr.14901) 8309 : cluster [DBG] pgmap v8316: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:37.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:36 smithi006 bash[27292]: audit 2023-08-28T02:35:36.271264+0000 mon.a (mon.0) 5233 : audit [DBG] from='client.? 172.21.15.6:0/87049698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:37 smithi130 bash[51302]: debug 2023-08-28T02:35:37.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:37.776825+0000) 2023-08-28T02:35:39.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:38 smithi130 bash[51302]: debug 2023-08-28T02:35:38.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:38.777114+0000) 2023-08-28T02:35:39.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:38 smithi130 bash[27579]: cluster 2023-08-28T02:35:37.861977+0000 mgr.y (mgr.14901) 8310 : cluster [DBG] pgmap v8317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:39.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:38 smithi130 bash[27579]: audit 2023-08-28T02:35:38.682655+0000 mon.c (mon.2) 3300 : audit [DBG] from='client.? 172.21.15.6:0/269060034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:39.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:38 smithi006 bash[33918]: cluster 2023-08-28T02:35:37.861977+0000 mgr.y (mgr.14901) 8310 : cluster [DBG] pgmap v8317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:39.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:38 smithi006 bash[33918]: audit 2023-08-28T02:35:38.682655+0000 mon.c (mon.2) 3300 : audit [DBG] from='client.? 172.21.15.6:0/269060034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:39.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:38 smithi006 bash[27292]: cluster 2023-08-28T02:35:37.861977+0000 mgr.y (mgr.14901) 8310 : cluster [DBG] pgmap v8317: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:39.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:38 smithi006 bash[27292]: audit 2023-08-28T02:35:38.682655+0000 mon.c (mon.2) 3300 : audit [DBG] from='client.? 172.21.15.6:0/269060034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:40.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:39 smithi130 bash[51302]: debug 2023-08-28T02:35:39.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:39.777381+0000) 2023-08-28T02:35:41.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:40 smithi130 bash[27579]: cluster 2023-08-28T02:35:39.863245+0000 mgr.y (mgr.14901) 8311 : cluster [DBG] pgmap v8318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:41.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:40 smithi130 bash[51302]: debug 2023-08-28T02:35:40.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:40.777661+0000) 2023-08-28T02:35:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:35:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:35:40] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:35:41.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:40 smithi006 bash[33918]: cluster 2023-08-28T02:35:39.863245+0000 mgr.y (mgr.14901) 8311 : cluster [DBG] pgmap v8318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:41.218 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:40 smithi006 bash[27292]: cluster 2023-08-28T02:35:39.863245+0000 mgr.y (mgr.14901) 8311 : cluster [DBG] pgmap v8318: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:35:41.341Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:35:41.342Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:42.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:41 smithi130 bash[27579]: audit 2023-08-28T02:35:41.101066+0000 mon.a (mon.0) 5234 : audit [DBG] from='client.? 172.21.15.6:0/1719736429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:42.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:41 smithi130 bash[51302]: debug 2023-08-28T02:35:41.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:41.777942+0000) 2023-08-28T02:35:42.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:41 smithi006 bash[27292]: audit 2023-08-28T02:35:41.101066+0000 mon.a (mon.0) 5234 : audit [DBG] from='client.? 172.21.15.6:0/1719736429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:42.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:41 smithi006 bash[33918]: audit 2023-08-28T02:35:41.101066+0000 mon.a (mon.0) 5234 : audit [DBG] from='client.? 172.21.15.6:0/1719736429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:42 smithi130 bash[27579]: cluster 2023-08-28T02:35:41.863971+0000 mgr.y (mgr.14901) 8312 : cluster [DBG] pgmap v8319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:43.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:42 smithi130 bash[51302]: debug 2023-08-28T02:35:42.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:42.778241+0000) 2023-08-28T02:35:43.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:42 smithi006 bash[27292]: cluster 2023-08-28T02:35:41.863971+0000 mgr.y (mgr.14901) 8312 : cluster [DBG] pgmap v8319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:43.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:42 smithi006 bash[33918]: cluster 2023-08-28T02:35:41.863971+0000 mgr.y (mgr.14901) 8312 : cluster [DBG] pgmap v8319: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:35:42.904Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:35:42.906Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:44.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:43 smithi130 bash[51302]: debug 2023-08-28T02:35:43.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:43.778507+0000) 2023-08-28T02:35:44.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:44 smithi006 bash[33918]: audit 2023-08-28T02:35:43.516140+0000 mon.c (mon.2) 3301 : audit [DBG] from='client.? 172.21.15.6:0/1888838382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:44 smithi006 bash[33918]: cluster 2023-08-28T02:35:43.864516+0000 mgr.y (mgr.14901) 8313 : cluster [DBG] pgmap v8320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:44.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:44 smithi006 bash[27292]: audit 2023-08-28T02:35:43.516140+0000 mon.c (mon.2) 3301 : audit [DBG] from='client.? 172.21.15.6:0/1888838382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:44.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:44 smithi006 bash[27292]: cluster 2023-08-28T02:35:43.864516+0000 mgr.y (mgr.14901) 8313 : cluster [DBG] pgmap v8320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:44 smithi130 bash[27579]: audit 2023-08-28T02:35:43.516140+0000 mon.c (mon.2) 3301 : audit [DBG] from='client.? 172.21.15.6:0/1888838382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:44.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:44 smithi130 bash[27579]: cluster 2023-08-28T02:35:43.864516+0000 mgr.y (mgr.14901) 8313 : cluster [DBG] pgmap v8320: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:45.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:44 smithi130 bash[51302]: debug 2023-08-28T02:35:44.771+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:44.778804+0000) 2023-08-28T02:35:45.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:35:44.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:45.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:35:44.785Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:45 smithi130 bash[51302]: debug 2023-08-28T02:35:45.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:45.779100+0000) 2023-08-28T02:35:47.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:46 smithi130 bash[27579]: cluster 2023-08-28T02:35:45.865598+0000 mgr.y (mgr.14901) 8314 : cluster [DBG] pgmap v8321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:46 smithi130 bash[27579]: audit 2023-08-28T02:35:45.928035+0000 mon.c (mon.2) 3302 : audit [DBG] from='client.? 172.21.15.6:0/2214269697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:47.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:46 smithi130 bash[51302]: debug 2023-08-28T02:35:46.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:46.779326+0000) 2023-08-28T02:35:47.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:46 smithi006 bash[27292]: cluster 2023-08-28T02:35:45.865598+0000 mgr.y (mgr.14901) 8314 : cluster [DBG] pgmap v8321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:47.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:46 smithi006 bash[27292]: audit 2023-08-28T02:35:45.928035+0000 mon.c (mon.2) 3302 : audit [DBG] from='client.? 172.21.15.6:0/2214269697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:47.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:46 smithi006 bash[33918]: cluster 2023-08-28T02:35:45.865598+0000 mgr.y (mgr.14901) 8314 : cluster [DBG] pgmap v8321: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:47.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:46 smithi006 bash[33918]: audit 2023-08-28T02:35:45.928035+0000 mon.c (mon.2) 3302 : audit [DBG] from='client.? 172.21.15.6:0/2214269697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:48.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:47 smithi130 bash[27579]: audit 2023-08-28T02:35:47.147967+0000 mon.b (mon.1) 903 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:35:48.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:47 smithi130 bash[51302]: debug 2023-08-28T02:35:47.779+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:47.779580+0000) 2023-08-28T02:35:48.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:47 smithi006 bash[27292]: audit 2023-08-28T02:35:47.147967+0000 mon.b (mon.1) 903 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:35:48.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:47 smithi006 bash[33918]: audit 2023-08-28T02:35:47.147967+0000 mon.b (mon.1) 903 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:35:49.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:48 smithi130 bash[27579]: cluster 2023-08-28T02:35:47.866255+0000 mgr.y (mgr.14901) 8315 : cluster [DBG] pgmap v8322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:49.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:48 smithi130 bash[27579]: audit 2023-08-28T02:35:48.351999+0000 mon.c (mon.2) 3303 : audit [DBG] from='client.? 172.21.15.6:0/583702210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:49.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:48 smithi130 bash[51302]: debug 2023-08-28T02:35:48.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:48.779819+0000) 2023-08-28T02:35:49.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:48 smithi006 bash[33918]: cluster 2023-08-28T02:35:47.866255+0000 mgr.y (mgr.14901) 8315 : cluster [DBG] pgmap v8322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:49.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:48 smithi006 bash[33918]: audit 2023-08-28T02:35:48.351999+0000 mon.c (mon.2) 3303 : audit [DBG] from='client.? 172.21.15.6:0/583702210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:49.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:48 smithi006 bash[27292]: cluster 2023-08-28T02:35:47.866255+0000 mgr.y (mgr.14901) 8315 : cluster [DBG] pgmap v8322: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:49.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:48 smithi006 bash[27292]: audit 2023-08-28T02:35:48.351999+0000 mon.c (mon.2) 3303 : audit [DBG] from='client.? 172.21.15.6:0/583702210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:50.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:49 smithi130 bash[51302]: debug 2023-08-28T02:35:49.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:49.780253+0000) 2023-08-28T02:35:51.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:50 smithi130 bash[27579]: cluster 2023-08-28T02:35:49.867428+0000 mgr.y (mgr.14901) 8316 : cluster [DBG] pgmap v8323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:51.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:50 smithi130 bash[27579]: audit 2023-08-28T02:35:50.776367+0000 mon.a (mon.0) 5235 : audit [DBG] from='client.? 172.21.15.6:0/2707067682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:51.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:50 smithi130 bash[51302]: debug 2023-08-28T02:35:50.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:50.780486+0000) 2023-08-28T02:35:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:35:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:35:50] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T02:35:51.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:50 smithi006 bash[27292]: cluster 2023-08-28T02:35:49.867428+0000 mgr.y (mgr.14901) 8316 : cluster [DBG] pgmap v8323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:51.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:50 smithi006 bash[27292]: audit 2023-08-28T02:35:50.776367+0000 mon.a (mon.0) 5235 : audit [DBG] from='client.? 172.21.15.6:0/2707067682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:51.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:50 smithi006 bash[33918]: cluster 2023-08-28T02:35:49.867428+0000 mgr.y (mgr.14901) 8316 : cluster [DBG] pgmap v8323: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:51.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:50 smithi006 bash[33918]: audit 2023-08-28T02:35:50.776367+0000 mon.a (mon.0) 5235 : audit [DBG] from='client.? 172.21.15.6:0/2707067682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:35:51.341Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:35:51.343Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:52.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:51 smithi130 bash[51302]: debug 2023-08-28T02:35:51.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:51.780780+0000) 2023-08-28T02:35:53.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:52 smithi130 bash[27579]: cluster 2023-08-28T02:35:51.868090+0000 mgr.y (mgr.14901) 8317 : cluster [DBG] pgmap v8324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:53.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:52 smithi130 bash[51302]: debug 2023-08-28T02:35:52.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:52.781055+0000) 2023-08-28T02:35:53.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:52 smithi006 bash[27292]: cluster 2023-08-28T02:35:51.868090+0000 mgr.y (mgr.14901) 8317 : cluster [DBG] pgmap v8324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:53.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:52 smithi006 bash[33918]: cluster 2023-08-28T02:35:51.868090+0000 mgr.y (mgr.14901) 8317 : cluster [DBG] pgmap v8324: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:35:52.905Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:35:52.907Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:54.070 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:53 smithi130 bash[51302]: debug 2023-08-28T02:35:53.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:53.781328+0000) 2023-08-28T02:35:54.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:54 smithi130 bash[27579]: audit 2023-08-28T02:35:53.070677+0000 mon.a (mon.0) 5236 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:35:54.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:54 smithi130 bash[27579]: audit 2023-08-28T02:35:53.244916+0000 mon.c (mon.2) 3304 : audit [DBG] from='client.? 172.21.15.6:0/3741018615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:54.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:54 smithi130 bash[27579]: audit 2023-08-28T02:35:53.449083+0000 mon.a (mon.0) 5237 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:35:54.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:54 smithi130 bash[27579]: audit 2023-08-28T02:35:53.715145+0000 mon.b (mon.1) 904 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:35:54.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:54 smithi130 bash[27579]: audit 2023-08-28T02:35:53.717509+0000 mon.b (mon.1) 905 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:35:54.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:54 smithi130 bash[27579]: audit 2023-08-28T02:35:53.726700+0000 mon.a (mon.0) 5238 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:35:54.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:54 smithi130 bash[27579]: cluster 2023-08-28T02:35:53.868722+0000 mgr.y (mgr.14901) 8318 : cluster [DBG] pgmap v8325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:54 smithi006 bash[27292]: audit 2023-08-28T02:35:53.070677+0000 mon.a (mon.0) 5236 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:35:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:54 smithi006 bash[27292]: audit 2023-08-28T02:35:53.244916+0000 mon.c (mon.2) 3304 : audit [DBG] from='client.? 172.21.15.6:0/3741018615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:54 smithi006 bash[27292]: audit 2023-08-28T02:35:53.449083+0000 mon.a (mon.0) 5237 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:35:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:54 smithi006 bash[27292]: audit 2023-08-28T02:35:53.715145+0000 mon.b (mon.1) 904 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:35:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:54 smithi006 bash[27292]: audit 2023-08-28T02:35:53.717509+0000 mon.b (mon.1) 905 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:35:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:54 smithi006 bash[27292]: audit 2023-08-28T02:35:53.726700+0000 mon.a (mon.0) 5238 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:35:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:54 smithi006 bash[27292]: cluster 2023-08-28T02:35:53.868722+0000 mgr.y (mgr.14901) 8318 : cluster [DBG] pgmap v8325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:54 smithi006 bash[33918]: audit 2023-08-28T02:35:53.070677+0000 mon.a (mon.0) 5236 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:35:54.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:54 smithi006 bash[33918]: audit 2023-08-28T02:35:53.244916+0000 mon.c (mon.2) 3304 : audit [DBG] from='client.? 172.21.15.6:0/3741018615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:54.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:54 smithi006 bash[33918]: audit 2023-08-28T02:35:53.449083+0000 mon.a (mon.0) 5237 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:35:54.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:54 smithi006 bash[33918]: audit 2023-08-28T02:35:53.715145+0000 mon.b (mon.1) 904 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:35:54.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:54 smithi006 bash[33918]: audit 2023-08-28T02:35:53.717509+0000 mon.b (mon.1) 905 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:35:54.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:54 smithi006 bash[33918]: audit 2023-08-28T02:35:53.726700+0000 mon.a (mon.0) 5238 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:35:54.466 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:54 smithi006 bash[33918]: cluster 2023-08-28T02:35:53.868722+0000 mgr.y (mgr.14901) 8318 : cluster [DBG] pgmap v8325: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:55.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:54 smithi130 bash[51302]: debug 2023-08-28T02:35:54.779+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:54.781609+0000) 2023-08-28T02:35:55.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:35:54.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:55.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:35:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:35:54.786Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:35:56.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:55 smithi130 bash[27579]: audit 2023-08-28T02:35:55.659234+0000 mon.c (mon.2) 3305 : audit [DBG] from='client.? 172.21.15.6:0/661195512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:56.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:55 smithi130 bash[51302]: debug 2023-08-28T02:35:55.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:55.781908+0000) 2023-08-28T02:35:56.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:55 smithi006 bash[33918]: audit 2023-08-28T02:35:55.659234+0000 mon.c (mon.2) 3305 : audit [DBG] from='client.? 172.21.15.6:0/661195512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:56.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:55 smithi006 bash[27292]: audit 2023-08-28T02:35:55.659234+0000 mon.c (mon.2) 3305 : audit [DBG] from='client.? 172.21.15.6:0/661195512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:57.122 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:56 smithi130 bash[51302]: debug 2023-08-28T02:35:56.779+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:56.782174+0000) 2023-08-28T02:35:57.122 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:56 smithi130 bash[27579]: cluster 2023-08-28T02:35:55.869852+0000 mgr.y (mgr.14901) 8319 : cluster [DBG] pgmap v8326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:57.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:56 smithi006 bash[33918]: cluster 2023-08-28T02:35:55.869852+0000 mgr.y (mgr.14901) 8319 : cluster [DBG] pgmap v8326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:57.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:56 smithi006 bash[27292]: cluster 2023-08-28T02:35:55.869852+0000 mgr.y (mgr.14901) 8319 : cluster [DBG] pgmap v8326: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:58.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:57 smithi130 bash[51302]: debug 2023-08-28T02:35:57.779+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:57.782439+0000) 2023-08-28T02:35:59.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:58 smithi130 bash[27579]: cluster 2023-08-28T02:35:57.870622+0000 mgr.y (mgr.14901) 8320 : cluster [DBG] pgmap v8327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:59.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:35:58 smithi130 bash[27579]: audit 2023-08-28T02:35:58.077343+0000 mon.c (mon.2) 3306 : audit [DBG] from='client.? 172.21.15.6:0/4149043308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:59.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:58 smithi130 bash[51302]: debug 2023-08-28T02:35:58.775+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:58.782742+0000) 2023-08-28T02:35:59.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:58 smithi006 bash[33918]: cluster 2023-08-28T02:35:57.870622+0000 mgr.y (mgr.14901) 8320 : cluster [DBG] pgmap v8327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:59.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:35:58 smithi006 bash[33918]: audit 2023-08-28T02:35:58.077343+0000 mon.c (mon.2) 3306 : audit [DBG] from='client.? 172.21.15.6:0/4149043308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:59.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:58 smithi006 bash[27292]: cluster 2023-08-28T02:35:57.870622+0000 mgr.y (mgr.14901) 8320 : cluster [DBG] pgmap v8327: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:35:59.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:35:58 smithi006 bash[27292]: audit 2023-08-28T02:35:58.077343+0000 mon.c (mon.2) 3306 : audit [DBG] from='client.? 172.21.15.6:0/4149043308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:35:59 smithi130 bash[51302]: debug 2023-08-28T02:35:59.779+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:59.783065+0000) 2023-08-28T02:36:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:00 smithi130 bash[51302]: debug 2023-08-28T02:36:00.783+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:00.783378+0000) 2023-08-28T02:36:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:00 smithi130 bash[27579]: cluster 2023-08-28T02:35:59.871721+0000 mgr.y (mgr.14901) 8321 : cluster [DBG] pgmap v8328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:00 smithi130 bash[27579]: audit 2023-08-28T02:36:00.496005+0000 mon.a (mon.0) 5239 : audit [DBG] from='client.? 172.21.15.6:0/2969640250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:36:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:36:00] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T02:36:01.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:00 smithi006 bash[33918]: cluster 2023-08-28T02:35:59.871721+0000 mgr.y (mgr.14901) 8321 : cluster [DBG] pgmap v8328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:01.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:00 smithi006 bash[33918]: audit 2023-08-28T02:36:00.496005+0000 mon.a (mon.0) 5239 : audit [DBG] from='client.? 172.21.15.6:0/2969640250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:00 smithi006 bash[27292]: cluster 2023-08-28T02:35:59.871721+0000 mgr.y (mgr.14901) 8321 : cluster [DBG] pgmap v8328: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:00 smithi006 bash[27292]: audit 2023-08-28T02:36:00.496005+0000 mon.a (mon.0) 5239 : audit [DBG] from='client.? 172.21.15.6:0/2969640250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:36:01.342Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:36:01.344Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:02.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:01 smithi130 bash[51302]: debug 2023-08-28T02:36:01.783+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:01.783682+0000) 2023-08-28T02:36:03.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:02 smithi130 bash[51302]: debug 2023-08-28T02:36:02.783+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:02.783978+0000) 2023-08-28T02:36:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:02 smithi130 bash[27579]: cluster 2023-08-28T02:36:01.872294+0000 mgr.y (mgr.14901) 8322 : cluster [DBG] pgmap v8329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:02 smithi130 bash[27579]: audit 2023-08-28T02:36:02.912194+0000 mon.a (mon.0) 5240 : audit [DBG] from='client.? 172.21.15.6:0/3612778199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:03.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:02 smithi006 bash[33918]: cluster 2023-08-28T02:36:01.872294+0000 mgr.y (mgr.14901) 8322 : cluster [DBG] pgmap v8329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:03.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:02 smithi006 bash[33918]: audit 2023-08-28T02:36:02.912194+0000 mon.a (mon.0) 5240 : audit [DBG] from='client.? 172.21.15.6:0/3612778199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:36:02.906Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:36:02.908Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:03.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:02 smithi006 bash[27292]: cluster 2023-08-28T02:36:01.872294+0000 mgr.y (mgr.14901) 8322 : cluster [DBG] pgmap v8329: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:03.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:02 smithi006 bash[27292]: audit 2023-08-28T02:36:02.912194+0000 mon.a (mon.0) 5240 : audit [DBG] from='client.? 172.21.15.6:0/3612778199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:04.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:03 smithi130 bash[51302]: debug 2023-08-28T02:36:03.779+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:03.784310+0000) 2023-08-28T02:36:04.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:04 smithi006 bash[33918]: cluster 2023-08-28T02:36:03.872973+0000 mgr.y (mgr.14901) 8323 : cluster [DBG] pgmap v8330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:04 smithi006 bash[27292]: cluster 2023-08-28T02:36:03.872973+0000 mgr.y (mgr.14901) 8323 : cluster [DBG] pgmap v8330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:04 smithi130 bash[27579]: cluster 2023-08-28T02:36:03.872973+0000 mgr.y (mgr.14901) 8323 : cluster [DBG] pgmap v8330: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:05.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:04 smithi130 bash[51302]: debug 2023-08-28T02:36:04.779+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:04.784600+0000) 2023-08-28T02:36:05.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:36:04.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:05.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:36:04.786Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:05.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:05 smithi006 bash[33918]: audit 2023-08-28T02:36:05.322444+0000 mon.a (mon.0) 5241 : audit [DBG] from='client.? 172.21.15.6:0/2140528548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:05.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:05 smithi006 bash[27292]: audit 2023-08-28T02:36:05.322444+0000 mon.a (mon.0) 5241 : audit [DBG] from='client.? 172.21.15.6:0/2140528548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:05.783 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:05 smithi130 bash[27579]: audit 2023-08-28T02:36:05.322444+0000 mon.a (mon.0) 5241 : audit [DBG] from='client.? 172.21.15.6:0/2140528548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:06.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:05 smithi130 bash[51302]: debug 2023-08-28T02:36:05.783+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:05.784954+0000) 2023-08-28T02:36:06.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:06 smithi006 bash[33918]: cluster 2023-08-28T02:36:05.874063+0000 mgr.y (mgr.14901) 8324 : cluster [DBG] pgmap v8331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:06.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:06 smithi006 bash[27292]: cluster 2023-08-28T02:36:05.874063+0000 mgr.y (mgr.14901) 8324 : cluster [DBG] pgmap v8331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:06.784 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:06 smithi130 bash[27579]: cluster 2023-08-28T02:36:05.874063+0000 mgr.y (mgr.14901) 8324 : cluster [DBG] pgmap v8331: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:07.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:06 smithi130 bash[51302]: debug 2023-08-28T02:36:06.778+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:06.785238+0000) 2023-08-28T02:36:08.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:07 smithi130 bash[27579]: audit 2023-08-28T02:36:07.730553+0000 mon.c (mon.2) 3307 : audit [DBG] from='client.? 172.21.15.6:0/1470870205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:08.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:07 smithi130 bash[51302]: debug 2023-08-28T02:36:07.778+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:07.785513+0000) 2023-08-28T02:36:08.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:07 smithi006 bash[33918]: audit 2023-08-28T02:36:07.730553+0000 mon.c (mon.2) 3307 : audit [DBG] from='client.? 172.21.15.6:0/1470870205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:08.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:07 smithi006 bash[27292]: audit 2023-08-28T02:36:07.730553+0000 mon.c (mon.2) 3307 : audit [DBG] from='client.? 172.21.15.6:0/1470870205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:09.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:08 smithi130 bash[27579]: cluster 2023-08-28T02:36:07.874726+0000 mgr.y (mgr.14901) 8325 : cluster [DBG] pgmap v8332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:09.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:08 smithi130 bash[51302]: debug 2023-08-28T02:36:08.778+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:08.785810+0000) 2023-08-28T02:36:09.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:08 smithi006 bash[33918]: cluster 2023-08-28T02:36:07.874726+0000 mgr.y (mgr.14901) 8325 : cluster [DBG] pgmap v8332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:09.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:08 smithi006 bash[27292]: cluster 2023-08-28T02:36:07.874726+0000 mgr.y (mgr.14901) 8325 : cluster [DBG] pgmap v8332: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:10.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:09 smithi130 bash[51302]: debug 2023-08-28T02:36:09.782+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:09.786090+0000) 2023-08-28T02:36:11.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:10 smithi130 bash[51302]: debug 2023-08-28T02:36:10.778+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:10.786322+0000) 2023-08-28T02:36:11.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:10 smithi130 bash[27579]: cluster 2023-08-28T02:36:09.875885+0000 mgr.y (mgr.14901) 8326 : cluster [DBG] pgmap v8333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:11.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:10 smithi130 bash[27579]: audit 2023-08-28T02:36:10.151423+0000 mon.a (mon.0) 5242 : audit [DBG] from='client.? 172.21.15.6:0/3712902293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:11.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:10 smithi006 bash[27292]: cluster 2023-08-28T02:36:09.875885+0000 mgr.y (mgr.14901) 8326 : cluster [DBG] pgmap v8333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:11.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:10 smithi006 bash[27292]: audit 2023-08-28T02:36:10.151423+0000 mon.a (mon.0) 5242 : audit [DBG] from='client.? 172.21.15.6:0/3712902293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:11.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:36:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:36:10] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T02:36:11.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:10 smithi006 bash[33918]: cluster 2023-08-28T02:36:09.875885+0000 mgr.y (mgr.14901) 8326 : cluster [DBG] pgmap v8333: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:11.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:10 smithi006 bash[33918]: audit 2023-08-28T02:36:10.151423+0000 mon.a (mon.0) 5242 : audit [DBG] from='client.? 172.21.15.6:0/3712902293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:36:11.343Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:36:11.345Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:12.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:11 smithi130 bash[51302]: debug 2023-08-28T02:36:11.778+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:11.786544+0000) 2023-08-28T02:36:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:12 smithi130 bash[27579]: cluster 2023-08-28T02:36:11.876580+0000 mgr.y (mgr.14901) 8327 : cluster [DBG] pgmap v8334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:12 smithi130 bash[27579]: audit 2023-08-28T02:36:12.565217+0000 mon.a (mon.0) 5243 : audit [DBG] from='client.? 172.21.15.6:0/1806043286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:13.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:12 smithi130 bash[51302]: debug 2023-08-28T02:36:12.782+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:12.786808+0000) 2023-08-28T02:36:13.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:12 smithi006 bash[27292]: cluster 2023-08-28T02:36:11.876580+0000 mgr.y (mgr.14901) 8327 : cluster [DBG] pgmap v8334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:13.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:12 smithi006 bash[27292]: audit 2023-08-28T02:36:12.565217+0000 mon.a (mon.0) 5243 : audit [DBG] from='client.? 172.21.15.6:0/1806043286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:12 smithi006 bash[33918]: cluster 2023-08-28T02:36:11.876580+0000 mgr.y (mgr.14901) 8327 : cluster [DBG] pgmap v8334: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:12 smithi006 bash[33918]: audit 2023-08-28T02:36:12.565217+0000 mon.a (mon.0) 5243 : audit [DBG] from='client.? 172.21.15.6:0/1806043286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:36:12.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:36:12.909Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:14.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:13 smithi130 bash[51302]: debug 2023-08-28T02:36:13.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:13.787135+0000) 2023-08-28T02:36:14.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:14 smithi006 bash[27292]: cluster 2023-08-28T02:36:13.877373+0000 mgr.y (mgr.14901) 8328 : cluster [DBG] pgmap v8335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:14.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:14 smithi006 bash[33918]: cluster 2023-08-28T02:36:13.877373+0000 mgr.y (mgr.14901) 8328 : cluster [DBG] pgmap v8335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:14 smithi130 bash[27579]: cluster 2023-08-28T02:36:13.877373+0000 mgr.y (mgr.14901) 8328 : cluster [DBG] pgmap v8335: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:14 smithi130 bash[51302]: debug 2023-08-28T02:36:14.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:14.787349+0000) 2023-08-28T02:36:15.189 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:36:14.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:15.189 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:36:14.787Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:15.191 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:15 smithi006 bash[27292]: audit 2023-08-28T02:36:14.984619+0000 mon.c (mon.2) 3308 : audit [DBG] from='client.? 172.21.15.6:0/2812500298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:15.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:15 smithi006 bash[33918]: audit 2023-08-28T02:36:14.984619+0000 mon.c (mon.2) 3308 : audit [DBG] from='client.? 172.21.15.6:0/2812500298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:15.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:15 smithi130 bash[27579]: audit 2023-08-28T02:36:14.984619+0000 mon.c (mon.2) 3308 : audit [DBG] from='client.? 172.21.15.6:0/2812500298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:16.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:15 smithi130 bash[51302]: debug 2023-08-28T02:36:15.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:15.787579+0000) 2023-08-28T02:36:16.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:16 smithi006 bash[33918]: cluster 2023-08-28T02:36:15.878356+0000 mgr.y (mgr.14901) 8329 : cluster [DBG] pgmap v8336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:16.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:16 smithi006 bash[27292]: cluster 2023-08-28T02:36:15.878356+0000 mgr.y (mgr.14901) 8329 : cluster [DBG] pgmap v8336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:16.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:16 smithi130 bash[27579]: cluster 2023-08-28T02:36:15.878356+0000 mgr.y (mgr.14901) 8329 : cluster [DBG] pgmap v8336: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:17.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:16 smithi130 bash[51302]: debug 2023-08-28T02:36:16.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:16.787866+0000) 2023-08-28T02:36:17.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:17 smithi006 bash[33918]: audit 2023-08-28T02:36:17.396594+0000 mon.a (mon.0) 5244 : audit [DBG] from='client.? 172.21.15.6:0/266224735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:17.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:17 smithi006 bash[27292]: audit 2023-08-28T02:36:17.396594+0000 mon.a (mon.0) 5244 : audit [DBG] from='client.? 172.21.15.6:0/266224735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:17.787 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:17 smithi130 bash[27579]: audit 2023-08-28T02:36:17.396594+0000 mon.a (mon.0) 5244 : audit [DBG] from='client.? 172.21.15.6:0/266224735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:18.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:17 smithi130 bash[51302]: debug 2023-08-28T02:36:17.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:17.788166+0000) 2023-08-28T02:36:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:18 smithi006 bash[27292]: cluster 2023-08-28T02:36:17.878977+0000 mgr.y (mgr.14901) 8330 : cluster [DBG] pgmap v8337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:18.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:18 smithi006 bash[33918]: cluster 2023-08-28T02:36:17.878977+0000 mgr.y (mgr.14901) 8330 : cluster [DBG] pgmap v8337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:18.787 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:18 smithi130 bash[27579]: cluster 2023-08-28T02:36:17.878977+0000 mgr.y (mgr.14901) 8330 : cluster [DBG] pgmap v8337: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:19.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:18 smithi130 bash[51302]: debug 2023-08-28T02:36:18.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:18.788481+0000) 2023-08-28T02:36:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:19 smithi130 bash[51302]: debug 2023-08-28T02:36:19.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:19.788726+0000) 2023-08-28T02:36:20.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:19 smithi130 bash[27579]: audit 2023-08-28T02:36:19.809190+0000 mon.a (mon.0) 5245 : audit [DBG] from='client.? 172.21.15.6:0/160187061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:20.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:19 smithi006 bash[33918]: audit 2023-08-28T02:36:19.809190+0000 mon.a (mon.0) 5245 : audit [DBG] from='client.? 172.21.15.6:0/160187061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:20.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:19 smithi006 bash[27292]: audit 2023-08-28T02:36:19.809190+0000 mon.a (mon.0) 5245 : audit [DBG] from='client.? 172.21.15.6:0/160187061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:21.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:20 smithi130 bash[27579]: cluster 2023-08-28T02:36:19.880058+0000 mgr.y (mgr.14901) 8331 : cluster [DBG] pgmap v8338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:21.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:20 smithi130 bash[51302]: debug 2023-08-28T02:36:20.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:20.789008+0000) 2023-08-28T02:36:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:36:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:36:20] "GET /metrics HTTP/1.1" 200 193124 "" "Prometheus/2.33.4" 2023-08-28T02:36:21.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:20 smithi006 bash[33918]: cluster 2023-08-28T02:36:19.880058+0000 mgr.y (mgr.14901) 8331 : cluster [DBG] pgmap v8338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:21.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:20 smithi006 bash[27292]: cluster 2023-08-28T02:36:19.880058+0000 mgr.y (mgr.14901) 8331 : cluster [DBG] pgmap v8338: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:36:21.343Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:21.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:36:21.345Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:22.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:21 smithi130 bash[51302]: debug 2023-08-28T02:36:21.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:21.789322+0000) 2023-08-28T02:36:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:22 smithi130 bash[27579]: cluster 2023-08-28T02:36:21.880555+0000 mgr.y (mgr.14901) 8332 : cluster [DBG] pgmap v8339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:23.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:22 smithi130 bash[27579]: audit 2023-08-28T02:36:22.228395+0000 mon.a (mon.0) 5246 : audit [DBG] from='client.? 172.21.15.6:0/3315221521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:23.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:22 smithi130 bash[51302]: debug 2023-08-28T02:36:22.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:22.789580+0000) 2023-08-28T02:36:23.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:22 smithi006 bash[27292]: cluster 2023-08-28T02:36:21.880555+0000 mgr.y (mgr.14901) 8332 : cluster [DBG] pgmap v8339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:23.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:22 smithi006 bash[27292]: audit 2023-08-28T02:36:22.228395+0000 mon.a (mon.0) 5246 : audit [DBG] from='client.? 172.21.15.6:0/3315221521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:36:22.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:36:22.909Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:23.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:22 smithi006 bash[33918]: cluster 2023-08-28T02:36:21.880555+0000 mgr.y (mgr.14901) 8332 : cluster [DBG] pgmap v8339: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:23.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:22 smithi006 bash[33918]: audit 2023-08-28T02:36:22.228395+0000 mon.a (mon.0) 5246 : audit [DBG] from='client.? 172.21.15.6:0/3315221521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:24.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:23 smithi130 bash[51302]: debug 2023-08-28T02:36:23.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:23.789845+0000) 2023-08-28T02:36:24.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:24 smithi006 bash[33918]: cluster 2023-08-28T02:36:23.881173+0000 mgr.y (mgr.14901) 8333 : cluster [DBG] pgmap v8340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:24.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:24 smithi006 bash[27292]: cluster 2023-08-28T02:36:23.881173+0000 mgr.y (mgr.14901) 8333 : cluster [DBG] pgmap v8340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:24 smithi130 bash[27579]: cluster 2023-08-28T02:36:23.881173+0000 mgr.y (mgr.14901) 8333 : cluster [DBG] pgmap v8340: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:25.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:24 smithi130 bash[51302]: debug 2023-08-28T02:36:24.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:24.790119+0000) 2023-08-28T02:36:25.190 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:36:24.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:25.190 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:36:24.789Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:25.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:25 smithi006 bash[33918]: audit 2023-08-28T02:36:24.642367+0000 mon.c (mon.2) 3309 : audit [DBG] from='client.? 172.21.15.6:0/407493534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:25.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:25 smithi006 bash[27292]: audit 2023-08-28T02:36:24.642367+0000 mon.c (mon.2) 3309 : audit [DBG] from='client.? 172.21.15.6:0/407493534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:25.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:25 smithi130 bash[27579]: audit 2023-08-28T02:36:24.642367+0000 mon.c (mon.2) 3309 : audit [DBG] from='client.? 172.21.15.6:0/407493534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:26.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:25 smithi130 bash[51302]: debug 2023-08-28T02:36:25.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:25.790392+0000) 2023-08-28T02:36:26.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:26 smithi006 bash[33918]: cluster 2023-08-28T02:36:25.882248+0000 mgr.y (mgr.14901) 8334 : cluster [DBG] pgmap v8341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:26.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:26 smithi006 bash[27292]: cluster 2023-08-28T02:36:25.882248+0000 mgr.y (mgr.14901) 8334 : cluster [DBG] pgmap v8341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:26 smithi130 bash[27579]: cluster 2023-08-28T02:36:25.882248+0000 mgr.y (mgr.14901) 8334 : cluster [DBG] pgmap v8341: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:27.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:26 smithi130 bash[51302]: debug 2023-08-28T02:36:26.786+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:26.790677+0000) 2023-08-28T02:36:27.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:27 smithi006 bash[27292]: audit 2023-08-28T02:36:27.058119+0000 mon.c (mon.2) 3310 : audit [DBG] from='client.? 172.21.15.6:0/315715912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:27.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:27 smithi006 bash[33918]: audit 2023-08-28T02:36:27.058119+0000 mon.c (mon.2) 3310 : audit [DBG] from='client.? 172.21.15.6:0/315715912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:27.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:27 smithi130 bash[27579]: audit 2023-08-28T02:36:27.058119+0000 mon.c (mon.2) 3310 : audit [DBG] from='client.? 172.21.15.6:0/315715912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:28.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:27 smithi130 bash[51302]: debug 2023-08-28T02:36:27.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:27.790986+0000) 2023-08-28T02:36:28.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:28 smithi006 bash[27292]: cluster 2023-08-28T02:36:27.882935+0000 mgr.y (mgr.14901) 8335 : cluster [DBG] pgmap v8342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:28.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:28 smithi006 bash[33918]: cluster 2023-08-28T02:36:27.882935+0000 mgr.y (mgr.14901) 8335 : cluster [DBG] pgmap v8342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:28.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:28 smithi130 bash[27579]: cluster 2023-08-28T02:36:27.882935+0000 mgr.y (mgr.14901) 8335 : cluster [DBG] pgmap v8342: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:28 smithi130 bash[51302]: debug 2023-08-28T02:36:28.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:28.791281+0000) 2023-08-28T02:36:29.790 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:29 smithi130 bash[27579]: audit 2023-08-28T02:36:29.462942+0000 mon.a (mon.0) 5247 : audit [DBG] from='client.? 172.21.15.6:0/84930352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:29.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:29 smithi006 bash[27292]: audit 2023-08-28T02:36:29.462942+0000 mon.a (mon.0) 5247 : audit [DBG] from='client.? 172.21.15.6:0/84930352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:29.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:29 smithi006 bash[33918]: audit 2023-08-28T02:36:29.462942+0000 mon.a (mon.0) 5247 : audit [DBG] from='client.? 172.21.15.6:0/84930352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:30.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:29 smithi130 bash[51302]: debug 2023-08-28T02:36:29.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:29.791559+0000) 2023-08-28T02:36:30.790 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:30 smithi130 bash[27579]: cluster 2023-08-28T02:36:29.884147+0000 mgr.y (mgr.14901) 8336 : cluster [DBG] pgmap v8343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:30.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:36:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:36:30] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T02:36:30.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:30 smithi006 bash[27292]: cluster 2023-08-28T02:36:29.884147+0000 mgr.y (mgr.14901) 8336 : cluster [DBG] pgmap v8343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:30.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:30 smithi006 bash[33918]: cluster 2023-08-28T02:36:29.884147+0000 mgr.y (mgr.14901) 8336 : cluster [DBG] pgmap v8343: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:31.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:30 smithi130 bash[51302]: debug 2023-08-28T02:36:30.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:30.791869+0000) 2023-08-28T02:36:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:36:31.344Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:36:31.346Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:32.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:31 smithi130 bash[27579]: audit 2023-08-28T02:36:31.875464+0000 mon.c (mon.2) 3311 : audit [DBG] from='client.? 172.21.15.6:0/413216984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:32.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:31 smithi130 bash[51302]: debug 2023-08-28T02:36:31.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:31.792169+0000) 2023-08-28T02:36:32.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:31 smithi006 bash[33918]: audit 2023-08-28T02:36:31.875464+0000 mon.c (mon.2) 3311 : audit [DBG] from='client.? 172.21.15.6:0/413216984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:32.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:31 smithi006 bash[27292]: audit 2023-08-28T02:36:31.875464+0000 mon.c (mon.2) 3311 : audit [DBG] from='client.? 172.21.15.6:0/413216984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:33.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:32 smithi130 bash[27579]: cluster 2023-08-28T02:36:31.884801+0000 mgr.y (mgr.14901) 8337 : cluster [DBG] pgmap v8344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:33.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:32 smithi130 bash[51302]: debug 2023-08-28T02:36:32.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:32.792511+0000) 2023-08-28T02:36:33.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:32 smithi006 bash[27292]: cluster 2023-08-28T02:36:31.884801+0000 mgr.y (mgr.14901) 8337 : cluster [DBG] pgmap v8344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:33.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:32 smithi006 bash[33918]: cluster 2023-08-28T02:36:31.884801+0000 mgr.y (mgr.14901) 8337 : cluster [DBG] pgmap v8344: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:36:32.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:36:32.909Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:34.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:33 smithi130 bash[51302]: debug 2023-08-28T02:36:33.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:33.792787+0000) 2023-08-28T02:36:34.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:34 smithi006 bash[27292]: cluster 2023-08-28T02:36:33.885584+0000 mgr.y (mgr.14901) 8338 : cluster [DBG] pgmap v8345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:34.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:34 smithi006 bash[33918]: cluster 2023-08-28T02:36:33.885584+0000 mgr.y (mgr.14901) 8338 : cluster [DBG] pgmap v8345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:34 smithi130 bash[27579]: cluster 2023-08-28T02:36:33.885584+0000 mgr.y (mgr.14901) 8338 : cluster [DBG] pgmap v8345: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:35.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:34 smithi130 bash[51302]: debug 2023-08-28T02:36:34.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:34.793116+0000) 2023-08-28T02:36:35.198 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:36:34.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:35.198 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:36:34.789Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:35.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:35 smithi006 bash[27292]: audit 2023-08-28T02:36:34.287022+0000 mon.a (mon.0) 5248 : audit [DBG] from='client.? 172.21.15.6:0/1715441294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:35.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:35 smithi006 bash[33918]: audit 2023-08-28T02:36:34.287022+0000 mon.a (mon.0) 5248 : audit [DBG] from='client.? 172.21.15.6:0/1715441294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:35.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:35 smithi130 bash[27579]: audit 2023-08-28T02:36:34.287022+0000 mon.a (mon.0) 5248 : audit [DBG] from='client.? 172.21.15.6:0/1715441294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:36.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:35 smithi130 bash[51302]: debug 2023-08-28T02:36:35.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:35.793400+0000) 2023-08-28T02:36:36.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:36 smithi006 bash[33918]: cluster 2023-08-28T02:36:35.886721+0000 mgr.y (mgr.14901) 8339 : cluster [DBG] pgmap v8346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:36.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:36 smithi006 bash[27292]: cluster 2023-08-28T02:36:35.886721+0000 mgr.y (mgr.14901) 8339 : cluster [DBG] pgmap v8346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:36 smithi130 bash[27579]: cluster 2023-08-28T02:36:35.886721+0000 mgr.y (mgr.14901) 8339 : cluster [DBG] pgmap v8346: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:37.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:36 smithi130 bash[51302]: debug 2023-08-28T02:36:36.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:36.793672+0000) 2023-08-28T02:36:37.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:37 smithi006 bash[27292]: audit 2023-08-28T02:36:36.711147+0000 mon.a (mon.0) 5249 : audit [DBG] from='client.? 172.21.15.6:0/2194048830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:37.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:37 smithi006 bash[33918]: audit 2023-08-28T02:36:36.711147+0000 mon.a (mon.0) 5249 : audit [DBG] from='client.? 172.21.15.6:0/2194048830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:37.617 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:37 smithi130 bash[27579]: audit 2023-08-28T02:36:36.711147+0000 mon.a (mon.0) 5249 : audit [DBG] from='client.? 172.21.15.6:0/2194048830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:37 smithi130 bash[51302]: debug 2023-08-28T02:36:37.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:37.793954+0000) 2023-08-28T02:36:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:38 smithi130 bash[27579]: cluster 2023-08-28T02:36:37.887475+0000 mgr.y (mgr.14901) 8340 : cluster [DBG] pgmap v8347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:38 smithi006 bash[33918]: cluster 2023-08-28T02:36:37.887475+0000 mgr.y (mgr.14901) 8340 : cluster [DBG] pgmap v8347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:38.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:38 smithi006 bash[27292]: cluster 2023-08-28T02:36:37.887475+0000 mgr.y (mgr.14901) 8340 : cluster [DBG] pgmap v8347: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:39.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:38 smithi130 bash[51302]: debug 2023-08-28T02:36:38.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:38.794222+0000) 2023-08-28T02:36:39.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:39 smithi130 bash[27579]: audit 2023-08-28T02:36:39.119502+0000 mon.c (mon.2) 3312 : audit [DBG] from='client.? 172.21.15.6:0/217737125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:39.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:39 smithi006 bash[33918]: audit 2023-08-28T02:36:39.119502+0000 mon.c (mon.2) 3312 : audit [DBG] from='client.? 172.21.15.6:0/217737125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:39.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:39 smithi006 bash[27292]: audit 2023-08-28T02:36:39.119502+0000 mon.c (mon.2) 3312 : audit [DBG] from='client.? 172.21.15.6:0/217737125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:40.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:39 smithi130 bash[51302]: debug 2023-08-28T02:36:39.790+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:39.794494+0000) 2023-08-28T02:36:40.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:40 smithi130 bash[27579]: cluster 2023-08-28T02:36:39.888587+0000 mgr.y (mgr.14901) 8341 : cluster [DBG] pgmap v8348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:40.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:40 smithi006 bash[33918]: cluster 2023-08-28T02:36:39.888587+0000 mgr.y (mgr.14901) 8341 : cluster [DBG] pgmap v8348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:40.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:40 smithi006 bash[27292]: cluster 2023-08-28T02:36:39.888587+0000 mgr.y (mgr.14901) 8341 : cluster [DBG] pgmap v8348: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:41.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:40 smithi130 bash[51302]: debug 2023-08-28T02:36:40.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:40.794823+0000) 2023-08-28T02:36:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:36:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:36:40] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T02:36:41.597 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:41 smithi006 bash[27292]: audit 2023-08-28T02:36:41.541901+0000 mon.a (mon.0) 5250 : audit [DBG] from='client.? 172.21.15.6:0/1859313335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:41.597 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:36:41.345Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:41.597 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:36:41.346Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:41.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:41 smithi130 bash[27579]: audit 2023-08-28T02:36:41.541901+0000 mon.a (mon.0) 5250 : audit [DBG] from='client.? 172.21.15.6:0/1859313335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:41.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:41 smithi130 bash[51302]: debug 2023-08-28T02:36:41.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:41.795237+0000) 2023-08-28T02:36:41.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:41 smithi006 bash[33918]: audit 2023-08-28T02:36:41.541901+0000 mon.a (mon.0) 5250 : audit [DBG] from='client.? 172.21.15.6:0/1859313335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:42.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:42 smithi130 bash[27579]: cluster 2023-08-28T02:36:41.889266+0000 mgr.y (mgr.14901) 8342 : cluster [DBG] pgmap v8349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:42.866 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:42 smithi130 bash[51302]: debug 2023-08-28T02:36:42.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:42.795520+0000) 2023-08-28T02:36:42.909 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:42 smithi006 bash[27292]: cluster 2023-08-28T02:36:41.889266+0000 mgr.y (mgr.14901) 8342 : cluster [DBG] pgmap v8349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:42.909 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:42 smithi006 bash[33918]: cluster 2023-08-28T02:36:41.889266+0000 mgr.y (mgr.14901) 8342 : cluster [DBG] pgmap v8349: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:36:42.909Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:36:42.911Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:44.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:43 smithi130 bash[51302]: debug 2023-08-28T02:36:43.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:43.795743+0000) 2023-08-28T02:36:44.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:44 smithi006 bash[27292]: cluster 2023-08-28T02:36:43.890045+0000 mgr.y (mgr.14901) 8343 : cluster [DBG] pgmap v8350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:44.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:44 smithi006 bash[27292]: audit 2023-08-28T02:36:43.998101+0000 mon.c (mon.2) 3313 : audit [DBG] from='client.? 172.21.15.6:0/4220546389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:44 smithi006 bash[33918]: cluster 2023-08-28T02:36:43.890045+0000 mgr.y (mgr.14901) 8343 : cluster [DBG] pgmap v8350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:44.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:44 smithi006 bash[33918]: audit 2023-08-28T02:36:43.998101+0000 mon.c (mon.2) 3313 : audit [DBG] from='client.? 172.21.15.6:0/4220546389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:44 smithi130 bash[27579]: cluster 2023-08-28T02:36:43.890045+0000 mgr.y (mgr.14901) 8343 : cluster [DBG] pgmap v8350: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:44.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:44 smithi130 bash[27579]: audit 2023-08-28T02:36:43.998101+0000 mon.c (mon.2) 3313 : audit [DBG] from='client.? 172.21.15.6:0/4220546389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:45.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:44 smithi130 bash[51302]: debug 2023-08-28T02:36:44.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:44.796024+0000) 2023-08-28T02:36:45.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:36:44.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:45.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:36:44.790Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:45 smithi130 bash[51302]: debug 2023-08-28T02:36:45.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:45.796224+0000) 2023-08-28T02:36:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:46 smithi130 bash[51302]: debug 2023-08-28T02:36:46.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:46.796513+0000) 2023-08-28T02:36:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:46 smithi130 bash[27579]: cluster 2023-08-28T02:36:45.891301+0000 mgr.y (mgr.14901) 8344 : cluster [DBG] pgmap v8351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:46 smithi130 bash[27579]: audit 2023-08-28T02:36:46.413317+0000 mon.a (mon.0) 5251 : audit [DBG] from='client.? 172.21.15.6:0/103451859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:47.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:46 smithi006 bash[27292]: cluster 2023-08-28T02:36:45.891301+0000 mgr.y (mgr.14901) 8344 : cluster [DBG] pgmap v8351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:47.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:46 smithi006 bash[27292]: audit 2023-08-28T02:36:46.413317+0000 mon.a (mon.0) 5251 : audit [DBG] from='client.? 172.21.15.6:0/103451859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:47.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:46 smithi006 bash[33918]: cluster 2023-08-28T02:36:45.891301+0000 mgr.y (mgr.14901) 8344 : cluster [DBG] pgmap v8351: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:47.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:46 smithi006 bash[33918]: audit 2023-08-28T02:36:46.413317+0000 mon.a (mon.0) 5251 : audit [DBG] from='client.? 172.21.15.6:0/103451859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:47 smithi130 bash[51302]: debug 2023-08-28T02:36:47.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:47.796772+0000) 2023-08-28T02:36:49.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:48 smithi130 bash[51302]: debug 2023-08-28T02:36:48.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:48.797103+0000) 2023-08-28T02:36:49.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:48 smithi130 bash[27579]: cluster 2023-08-28T02:36:47.891972+0000 mgr.y (mgr.14901) 8345 : cluster [DBG] pgmap v8352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:49.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:48 smithi130 bash[27579]: audit 2023-08-28T02:36:48.826681+0000 mon.a (mon.0) 5252 : audit [DBG] from='client.? 172.21.15.6:0/1965306199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:49.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:48 smithi006 bash[33918]: cluster 2023-08-28T02:36:47.891972+0000 mgr.y (mgr.14901) 8345 : cluster [DBG] pgmap v8352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:49.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:48 smithi006 bash[33918]: audit 2023-08-28T02:36:48.826681+0000 mon.a (mon.0) 5252 : audit [DBG] from='client.? 172.21.15.6:0/1965306199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:49.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:48 smithi006 bash[27292]: cluster 2023-08-28T02:36:47.891972+0000 mgr.y (mgr.14901) 8345 : cluster [DBG] pgmap v8352: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:49.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:48 smithi006 bash[27292]: audit 2023-08-28T02:36:48.826681+0000 mon.a (mon.0) 5252 : audit [DBG] from='client.? 172.21.15.6:0/1965306199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:50.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:49 smithi130 bash[51302]: debug 2023-08-28T02:36:49.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:49.797381+0000) 2023-08-28T02:36:51.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:50 smithi130 bash[27579]: cluster 2023-08-28T02:36:49.893108+0000 mgr.y (mgr.14901) 8346 : cluster [DBG] pgmap v8353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:51.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:50 smithi130 bash[51302]: debug 2023-08-28T02:36:50.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:50.797655+0000) 2023-08-28T02:36:51.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:50 smithi006 bash[33918]: cluster 2023-08-28T02:36:49.893108+0000 mgr.y (mgr.14901) 8346 : cluster [DBG] pgmap v8353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:36:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:36:50] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:36:51.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:50 smithi006 bash[27292]: cluster 2023-08-28T02:36:49.893108+0000 mgr.y (mgr.14901) 8346 : cluster [DBG] pgmap v8353: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:36:51.345Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:36:51.347Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:52.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:51 smithi130 bash[27579]: audit 2023-08-28T02:36:51.236419+0000 mon.c (mon.2) 3314 : audit [DBG] from='client.? 172.21.15.6:0/1813492794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:52.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:51 smithi130 bash[51302]: debug 2023-08-28T02:36:51.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:51.797940+0000) 2023-08-28T02:36:52.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:51 smithi006 bash[33918]: audit 2023-08-28T02:36:51.236419+0000 mon.c (mon.2) 3314 : audit [DBG] from='client.? 172.21.15.6:0/1813492794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:52.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:51 smithi006 bash[27292]: audit 2023-08-28T02:36:51.236419+0000 mon.c (mon.2) 3314 : audit [DBG] from='client.? 172.21.15.6:0/1813492794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:52 smithi130 bash[27579]: cluster 2023-08-28T02:36:51.893929+0000 mgr.y (mgr.14901) 8347 : cluster [DBG] pgmap v8354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:53.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:52 smithi130 bash[51302]: debug 2023-08-28T02:36:52.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:52.798208+0000) 2023-08-28T02:36:53.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:52 smithi006 bash[33918]: cluster 2023-08-28T02:36:51.893929+0000 mgr.y (mgr.14901) 8347 : cluster [DBG] pgmap v8354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:53.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:52 smithi006 bash[27292]: cluster 2023-08-28T02:36:51.893929+0000 mgr.y (mgr.14901) 8347 : cluster [DBG] pgmap v8354: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:36:52.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:36:52.912Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:54.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:53 smithi130 bash[51302]: debug 2023-08-28T02:36:53.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:53.798467+0000) 2023-08-28T02:36:54.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:54 smithi006 bash[33918]: audit 2023-08-28T02:36:53.646761+0000 mon.a (mon.0) 5253 : audit [DBG] from='client.? 172.21.15.6:0/1718609925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:54 smithi006 bash[33918]: audit 2023-08-28T02:36:53.738776+0000 mon.b (mon.1) 906 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:36:54.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:54 smithi006 bash[33918]: cluster 2023-08-28T02:36:53.894615+0000 mgr.y (mgr.14901) 8348 : cluster [DBG] pgmap v8355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:54 smithi006 bash[27292]: audit 2023-08-28T02:36:53.646761+0000 mon.a (mon.0) 5253 : audit [DBG] from='client.? 172.21.15.6:0/1718609925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:54 smithi006 bash[27292]: audit 2023-08-28T02:36:53.738776+0000 mon.b (mon.1) 906 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:36:54.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:54 smithi006 bash[27292]: cluster 2023-08-28T02:36:53.894615+0000 mgr.y (mgr.14901) 8348 : cluster [DBG] pgmap v8355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:54 smithi130 bash[27579]: audit 2023-08-28T02:36:53.646761+0000 mon.a (mon.0) 5253 : audit [DBG] from='client.? 172.21.15.6:0/1718609925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:54.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:54 smithi130 bash[27579]: audit 2023-08-28T02:36:53.738776+0000 mon.b (mon.1) 906 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:36:54.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:54 smithi130 bash[27579]: cluster 2023-08-28T02:36:53.894615+0000 mgr.y (mgr.14901) 8348 : cluster [DBG] pgmap v8355: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:55.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:54 smithi130 bash[51302]: debug 2023-08-28T02:36:54.794+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:54.798756+0000) 2023-08-28T02:36:55.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:36:54.788Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:55.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:36:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:36:54.790Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:36:56.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:55 smithi130 bash[51302]: debug 2023-08-28T02:36:55.798+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:55.798977+0000) 2023-08-28T02:36:57.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:56 smithi130 bash[27579]: cluster 2023-08-28T02:36:55.895742+0000 mgr.y (mgr.14901) 8349 : cluster [DBG] pgmap v8356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:57.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:56 smithi130 bash[27579]: audit 2023-08-28T02:36:56.072333+0000 mon.a (mon.0) 5254 : audit [DBG] from='client.? 172.21.15.6:0/1050770198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:57.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:56 smithi130 bash[51302]: debug 2023-08-28T02:36:56.798+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:56.799274+0000) 2023-08-28T02:36:57.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:56 smithi006 bash[33918]: cluster 2023-08-28T02:36:55.895742+0000 mgr.y (mgr.14901) 8349 : cluster [DBG] pgmap v8356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:57.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:56 smithi006 bash[33918]: audit 2023-08-28T02:36:56.072333+0000 mon.a (mon.0) 5254 : audit [DBG] from='client.? 172.21.15.6:0/1050770198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:57.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:56 smithi006 bash[27292]: cluster 2023-08-28T02:36:55.895742+0000 mgr.y (mgr.14901) 8349 : cluster [DBG] pgmap v8356: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:57.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:56 smithi006 bash[27292]: audit 2023-08-28T02:36:56.072333+0000 mon.a (mon.0) 5254 : audit [DBG] from='client.? 172.21.15.6:0/1050770198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:58.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:57 smithi130 bash[51302]: debug 2023-08-28T02:36:57.798+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:57.799547+0000) 2023-08-28T02:36:59.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:58 smithi130 bash[27579]: cluster 2023-08-28T02:36:57.896373+0000 mgr.y (mgr.14901) 8350 : cluster [DBG] pgmap v8357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:59.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:36:58 smithi130 bash[27579]: audit 2023-08-28T02:36:58.483083+0000 mon.c (mon.2) 3315 : audit [DBG] from='client.? 172.21.15.6:0/925572199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:59.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:58 smithi130 bash[51302]: debug 2023-08-28T02:36:58.798+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:58.799802+0000) 2023-08-28T02:36:59.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:58 smithi006 bash[33918]: cluster 2023-08-28T02:36:57.896373+0000 mgr.y (mgr.14901) 8350 : cluster [DBG] pgmap v8357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:59.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:36:58 smithi006 bash[33918]: audit 2023-08-28T02:36:58.483083+0000 mon.c (mon.2) 3315 : audit [DBG] from='client.? 172.21.15.6:0/925572199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:59.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:58 smithi006 bash[27292]: cluster 2023-08-28T02:36:57.896373+0000 mgr.y (mgr.14901) 8350 : cluster [DBG] pgmap v8357: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:36:59.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:36:58 smithi006 bash[27292]: audit 2023-08-28T02:36:58.483083+0000 mon.c (mon.2) 3315 : audit [DBG] from='client.? 172.21.15.6:0/925572199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:36:59 smithi130 bash[51302]: debug 2023-08-28T02:36:59.798+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:59.800092+0000) 2023-08-28T02:37:00.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:37:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:37:00] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T02:37:00.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:00 smithi006 bash[27292]: audit 2023-08-28T02:36:59.665741+0000 mon.a (mon.0) 5255 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:37:00.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:00 smithi006 bash[27292]: cluster 2023-08-28T02:36:59.897628+0000 mgr.y (mgr.14901) 8351 : cluster [DBG] pgmap v8358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:00.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:00 smithi006 bash[27292]: audit 2023-08-28T02:37:00.137675+0000 mon.a (mon.0) 5256 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:37:00.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:00 smithi006 bash[27292]: audit 2023-08-28T02:37:00.403684+0000 mon.b (mon.1) 907 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:37:00.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:00 smithi006 bash[27292]: audit 2023-08-28T02:37:00.405928+0000 mon.b (mon.1) 908 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:37:00.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:00 smithi006 bash[27292]: audit 2023-08-28T02:37:00.415196+0000 mon.a (mon.0) 5257 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:37:00.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:00 smithi006 bash[33918]: audit 2023-08-28T02:36:59.665741+0000 mon.a (mon.0) 5255 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:37:00.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:00 smithi006 bash[33918]: cluster 2023-08-28T02:36:59.897628+0000 mgr.y (mgr.14901) 8351 : cluster [DBG] pgmap v8358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:00.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:00 smithi006 bash[33918]: audit 2023-08-28T02:37:00.137675+0000 mon.a (mon.0) 5256 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:37:00.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:00 smithi006 bash[33918]: audit 2023-08-28T02:37:00.403684+0000 mon.b (mon.1) 907 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:37:00.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:00 smithi006 bash[33918]: audit 2023-08-28T02:37:00.405928+0000 mon.b (mon.1) 908 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:37:00.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:00 smithi006 bash[33918]: audit 2023-08-28T02:37:00.415196+0000 mon.a (mon.0) 5257 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:37:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:00 smithi130 bash[51302]: debug 2023-08-28T02:37:00.797+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:00.800360+0000) 2023-08-28T02:37:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:00 smithi130 bash[27579]: audit 2023-08-28T02:36:59.665741+0000 mon.a (mon.0) 5255 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:37:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:00 smithi130 bash[27579]: cluster 2023-08-28T02:36:59.897628+0000 mgr.y (mgr.14901) 8351 : cluster [DBG] pgmap v8358: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:00 smithi130 bash[27579]: audit 2023-08-28T02:37:00.137675+0000 mon.a (mon.0) 5256 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:37:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:00 smithi130 bash[27579]: audit 2023-08-28T02:37:00.403684+0000 mon.b (mon.1) 907 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:37:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:00 smithi130 bash[27579]: audit 2023-08-28T02:37:00.405928+0000 mon.b (mon.1) 908 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:37:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:00 smithi130 bash[27579]: audit 2023-08-28T02:37:00.415196+0000 mon.a (mon.0) 5257 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:37:01.675 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:37:01.346Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:01.675 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:37:01.348Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:01.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:01 smithi006 bash[27292]: audit 2023-08-28T02:37:00.903227+0000 mon.c (mon.2) 3316 : audit [DBG] from='client.? 172.21.15.6:0/1776291848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:01.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:01 smithi006 bash[33918]: audit 2023-08-28T02:37:00.903227+0000 mon.c (mon.2) 3316 : audit [DBG] from='client.? 172.21.15.6:0/1776291848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:02.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:01 smithi130 bash[27579]: audit 2023-08-28T02:37:00.903227+0000 mon.c (mon.2) 3316 : audit [DBG] from='client.? 172.21.15.6:0/1776291848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:02.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:01 smithi130 bash[51302]: debug 2023-08-28T02:37:01.797+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:01.800729+0000) 2023-08-28T02:37:02.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:02 smithi006 bash[33918]: cluster 2023-08-28T02:37:01.898330+0000 mgr.y (mgr.14901) 8352 : cluster [DBG] pgmap v8359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:02.993 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:02 smithi006 bash[27292]: cluster 2023-08-28T02:37:01.898330+0000 mgr.y (mgr.14901) 8352 : cluster [DBG] pgmap v8359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:02.993 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:37:02.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:02.993 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:37:02.912Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:03.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:02 smithi130 bash[27579]: cluster 2023-08-28T02:37:01.898330+0000 mgr.y (mgr.14901) 8352 : cluster [DBG] pgmap v8359: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:03.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:02 smithi130 bash[51302]: debug 2023-08-28T02:37:02.797+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:02.801008+0000) 2023-08-28T02:37:04.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:03 smithi130 bash[51302]: debug 2023-08-28T02:37:03.797+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:03.801308+0000) 2023-08-28T02:37:04.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:04 smithi006 bash[33918]: audit 2023-08-28T02:37:03.317025+0000 mon.a (mon.0) 5258 : audit [DBG] from='client.? 172.21.15.6:0/4030410336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:04.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:04 smithi006 bash[33918]: cluster 2023-08-28T02:37:03.899144+0000 mgr.y (mgr.14901) 8353 : cluster [DBG] pgmap v8360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:04 smithi006 bash[27292]: audit 2023-08-28T02:37:03.317025+0000 mon.a (mon.0) 5258 : audit [DBG] from='client.? 172.21.15.6:0/4030410336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:04.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:04 smithi006 bash[27292]: cluster 2023-08-28T02:37:03.899144+0000 mgr.y (mgr.14901) 8353 : cluster [DBG] pgmap v8360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:04 smithi130 bash[27579]: audit 2023-08-28T02:37:03.317025+0000 mon.a (mon.0) 5258 : audit [DBG] from='client.? 172.21.15.6:0/4030410336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:04.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:04 smithi130 bash[27579]: cluster 2023-08-28T02:37:03.899144+0000 mgr.y (mgr.14901) 8353 : cluster [DBG] pgmap v8360: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:05.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:04 smithi130 bash[51302]: debug 2023-08-28T02:37:04.797+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:04.801584+0000) 2023-08-28T02:37:05.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:37:04.788Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:05.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:37:04.791Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:06.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:05 smithi130 bash[27579]: audit 2023-08-28T02:37:05.732549+0000 mon.a (mon.0) 5259 : audit [DBG] from='client.? 172.21.15.6:0/1807938570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:06.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:05 smithi130 bash[51302]: debug 2023-08-28T02:37:05.797+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:05.801844+0000) 2023-08-28T02:37:06.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:05 smithi006 bash[27292]: audit 2023-08-28T02:37:05.732549+0000 mon.a (mon.0) 5259 : audit [DBG] from='client.? 172.21.15.6:0/1807938570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:06.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:05 smithi006 bash[33918]: audit 2023-08-28T02:37:05.732549+0000 mon.a (mon.0) 5259 : audit [DBG] from='client.? 172.21.15.6:0/1807938570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:07.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:06 smithi130 bash[27579]: cluster 2023-08-28T02:37:05.900369+0000 mgr.y (mgr.14901) 8354 : cluster [DBG] pgmap v8361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:07.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:06 smithi130 bash[51302]: debug 2023-08-28T02:37:06.797+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:06.802022+0000) 2023-08-28T02:37:07.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:06 smithi006 bash[27292]: cluster 2023-08-28T02:37:05.900369+0000 mgr.y (mgr.14901) 8354 : cluster [DBG] pgmap v8361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:07.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:06 smithi006 bash[33918]: cluster 2023-08-28T02:37:05.900369+0000 mgr.y (mgr.14901) 8354 : cluster [DBG] pgmap v8361: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:08.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:07 smithi130 bash[51302]: debug 2023-08-28T02:37:07.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:07.802259+0000) 2023-08-28T02:37:09.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:08 smithi130 bash[51302]: debug 2023-08-28T02:37:08.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:08.802518+0000) 2023-08-28T02:37:09.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:08 smithi130 bash[27579]: cluster 2023-08-28T02:37:07.900921+0000 mgr.y (mgr.14901) 8355 : cluster [DBG] pgmap v8362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:09.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:08 smithi130 bash[27579]: audit 2023-08-28T02:37:08.144458+0000 mon.a (mon.0) 5260 : audit [DBG] from='client.? 172.21.15.6:0/60991876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:09.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:08 smithi006 bash[33918]: cluster 2023-08-28T02:37:07.900921+0000 mgr.y (mgr.14901) 8355 : cluster [DBG] pgmap v8362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:09.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:08 smithi006 bash[33918]: audit 2023-08-28T02:37:08.144458+0000 mon.a (mon.0) 5260 : audit [DBG] from='client.? 172.21.15.6:0/60991876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:09.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:08 smithi006 bash[27292]: cluster 2023-08-28T02:37:07.900921+0000 mgr.y (mgr.14901) 8355 : cluster [DBG] pgmap v8362: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:09.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:08 smithi006 bash[27292]: audit 2023-08-28T02:37:08.144458+0000 mon.a (mon.0) 5260 : audit [DBG] from='client.? 172.21.15.6:0/60991876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:10.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:09 smithi130 bash[51302]: debug 2023-08-28T02:37:09.797+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:09.802777+0000) 2023-08-28T02:37:11.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:10 smithi130 bash[27579]: cluster 2023-08-28T02:37:09.901925+0000 mgr.y (mgr.14901) 8356 : cluster [DBG] pgmap v8363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:11.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:10 smithi130 bash[27579]: audit 2023-08-28T02:37:10.575894+0000 mon.c (mon.2) 3317 : audit [DBG] from='client.? 172.21.15.6:0/3940513445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:11.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:10 smithi130 bash[51302]: debug 2023-08-28T02:37:10.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:10.803000+0000) 2023-08-28T02:37:11.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:37:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:37:10] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T02:37:11.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:10 smithi006 bash[27292]: cluster 2023-08-28T02:37:09.901925+0000 mgr.y (mgr.14901) 8356 : cluster [DBG] pgmap v8363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:11.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:10 smithi006 bash[27292]: audit 2023-08-28T02:37:10.575894+0000 mon.c (mon.2) 3317 : audit [DBG] from='client.? 172.21.15.6:0/3940513445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:11.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:10 smithi006 bash[33918]: cluster 2023-08-28T02:37:09.901925+0000 mgr.y (mgr.14901) 8356 : cluster [DBG] pgmap v8363: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:11.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:10 smithi006 bash[33918]: audit 2023-08-28T02:37:10.575894+0000 mon.c (mon.2) 3317 : audit [DBG] from='client.? 172.21.15.6:0/3940513445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:37:11.346Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:37:11.348Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:12.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:11 smithi130 bash[51302]: debug 2023-08-28T02:37:11.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:11.803249+0000) 2023-08-28T02:37:13.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:12 smithi130 bash[27579]: cluster 2023-08-28T02:37:11.902584+0000 mgr.y (mgr.14901) 8357 : cluster [DBG] pgmap v8364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:13.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:12 smithi130 bash[51302]: debug 2023-08-28T02:37:12.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:12.803526+0000) 2023-08-28T02:37:13.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:12 smithi006 bash[33918]: cluster 2023-08-28T02:37:11.902584+0000 mgr.y (mgr.14901) 8357 : cluster [DBG] pgmap v8364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:13.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:12 smithi006 bash[27292]: cluster 2023-08-28T02:37:11.902584+0000 mgr.y (mgr.14901) 8357 : cluster [DBG] pgmap v8364: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:37:12.912Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:37:12.914Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:14.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:13 smithi130 bash[27579]: audit 2023-08-28T02:37:12.996083+0000 mon.a (mon.0) 5261 : audit [DBG] from='client.? 172.21.15.6:0/3483478975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:14.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:13 smithi130 bash[51302]: debug 2023-08-28T02:37:13.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:13.803914+0000) 2023-08-28T02:37:14.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:13 smithi006 bash[33918]: audit 2023-08-28T02:37:12.996083+0000 mon.a (mon.0) 5261 : audit [DBG] from='client.? 172.21.15.6:0/3483478975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:14.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:13 smithi006 bash[27292]: audit 2023-08-28T02:37:12.996083+0000 mon.a (mon.0) 5261 : audit [DBG] from='client.? 172.21.15.6:0/3483478975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:14 smithi130 bash[51302]: debug 2023-08-28T02:37:14.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:14.804162+0000) 2023-08-28T02:37:15.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:14 smithi130 bash[27579]: cluster 2023-08-28T02:37:13.903228+0000 mgr.y (mgr.14901) 8358 : cluster [DBG] pgmap v8365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:15.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:14 smithi006 bash[33918]: cluster 2023-08-28T02:37:13.903228+0000 mgr.y (mgr.14901) 8358 : cluster [DBG] pgmap v8365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:15.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:14 smithi006 bash[27292]: cluster 2023-08-28T02:37:13.903228+0000 mgr.y (mgr.14901) 8358 : cluster [DBG] pgmap v8365: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:15.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:37:14.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:15.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:37:14.792Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:16.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:15 smithi130 bash[27579]: audit 2023-08-28T02:37:15.410089+0000 mon.c (mon.2) 3318 : audit [DBG] from='client.? 172.21.15.6:0/695555271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:16.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:15 smithi130 bash[51302]: debug 2023-08-28T02:37:15.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:15.804393+0000) 2023-08-28T02:37:16.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:15 smithi006 bash[33918]: audit 2023-08-28T02:37:15.410089+0000 mon.c (mon.2) 3318 : audit [DBG] from='client.? 172.21.15.6:0/695555271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:16.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:15 smithi006 bash[27292]: audit 2023-08-28T02:37:15.410089+0000 mon.c (mon.2) 3318 : audit [DBG] from='client.? 172.21.15.6:0/695555271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:17.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:16 smithi130 bash[27579]: cluster 2023-08-28T02:37:15.904335+0000 mgr.y (mgr.14901) 8359 : cluster [DBG] pgmap v8366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:17.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:16 smithi130 bash[51302]: debug 2023-08-28T02:37:16.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:16.804654+0000) 2023-08-28T02:37:17.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:16 smithi006 bash[33918]: cluster 2023-08-28T02:37:15.904335+0000 mgr.y (mgr.14901) 8359 : cluster [DBG] pgmap v8366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:17.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:16 smithi006 bash[27292]: cluster 2023-08-28T02:37:15.904335+0000 mgr.y (mgr.14901) 8359 : cluster [DBG] pgmap v8366: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:18.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:17 smithi130 bash[27579]: audit 2023-08-28T02:37:17.822418+0000 mon.c (mon.2) 3319 : audit [DBG] from='client.? 172.21.15.6:0/4001045714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:18.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:17 smithi130 bash[27579]: cluster 2023-08-28T02:37:17.905007+0000 mgr.y (mgr.14901) 8360 : cluster [DBG] pgmap v8367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:18.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:17 smithi130 bash[51302]: debug 2023-08-28T02:37:17.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:17.804879+0000) 2023-08-28T02:37:18.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:17 smithi006 bash[33918]: audit 2023-08-28T02:37:17.822418+0000 mon.c (mon.2) 3319 : audit [DBG] from='client.? 172.21.15.6:0/4001045714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:18.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:17 smithi006 bash[33918]: cluster 2023-08-28T02:37:17.905007+0000 mgr.y (mgr.14901) 8360 : cluster [DBG] pgmap v8367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:18.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:17 smithi006 bash[27292]: audit 2023-08-28T02:37:17.822418+0000 mon.c (mon.2) 3319 : audit [DBG] from='client.? 172.21.15.6:0/4001045714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:18.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:17 smithi006 bash[27292]: cluster 2023-08-28T02:37:17.905007+0000 mgr.y (mgr.14901) 8360 : cluster [DBG] pgmap v8367: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:19.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:18 smithi130 bash[51302]: debug 2023-08-28T02:37:18.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:18.805147+0000) 2023-08-28T02:37:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:19 smithi130 bash[51302]: debug 2023-08-28T02:37:19.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:19.805415+0000) 2023-08-28T02:37:21.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:20 smithi130 bash[51302]: debug 2023-08-28T02:37:20.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:20.805752+0000) 2023-08-28T02:37:21.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:20 smithi130 bash[27579]: cluster 2023-08-28T02:37:19.906082+0000 mgr.y (mgr.14901) 8361 : cluster [DBG] pgmap v8368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:21.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:20 smithi130 bash[27579]: audit 2023-08-28T02:37:20.240074+0000 mon.a (mon.0) 5262 : audit [DBG] from='client.? 172.21.15.6:0/3882217408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:37:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:37:20] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T02:37:21.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:20 smithi006 bash[27292]: cluster 2023-08-28T02:37:19.906082+0000 mgr.y (mgr.14901) 8361 : cluster [DBG] pgmap v8368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:21.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:20 smithi006 bash[27292]: audit 2023-08-28T02:37:20.240074+0000 mon.a (mon.0) 5262 : audit [DBG] from='client.? 172.21.15.6:0/3882217408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:21.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:20 smithi006 bash[33918]: cluster 2023-08-28T02:37:19.906082+0000 mgr.y (mgr.14901) 8361 : cluster [DBG] pgmap v8368: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:21.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:20 smithi006 bash[33918]: audit 2023-08-28T02:37:20.240074+0000 mon.a (mon.0) 5262 : audit [DBG] from='client.? 172.21.15.6:0/3882217408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:37:21.347Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:37:21.349Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:22.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:21 smithi130 bash[51302]: debug 2023-08-28T02:37:21.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:21.805976+0000) 2023-08-28T02:37:23.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:22 smithi130 bash[27579]: cluster 2023-08-28T02:37:21.906742+0000 mgr.y (mgr.14901) 8362 : cluster [DBG] pgmap v8369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:23.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:22 smithi130 bash[27579]: audit 2023-08-28T02:37:22.658280+0000 mon.c (mon.2) 3320 : audit [DBG] from='client.? 172.21.15.6:0/3057262910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:23.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:22 smithi130 bash[51302]: debug 2023-08-28T02:37:22.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:22.806230+0000) 2023-08-28T02:37:23.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:22 smithi006 bash[33918]: cluster 2023-08-28T02:37:21.906742+0000 mgr.y (mgr.14901) 8362 : cluster [DBG] pgmap v8369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:23.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:22 smithi006 bash[33918]: audit 2023-08-28T02:37:22.658280+0000 mon.c (mon.2) 3320 : audit [DBG] from='client.? 172.21.15.6:0/3057262910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:23.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:22 smithi006 bash[27292]: cluster 2023-08-28T02:37:21.906742+0000 mgr.y (mgr.14901) 8362 : cluster [DBG] pgmap v8369: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:23.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:22 smithi006 bash[27292]: audit 2023-08-28T02:37:22.658280+0000 mon.c (mon.2) 3320 : audit [DBG] from='client.? 172.21.15.6:0/3057262910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:37:22.912Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:37:22.914Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:24.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:23 smithi130 bash[51302]: debug 2023-08-28T02:37:23.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:23.806488+0000) 2023-08-28T02:37:24.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:24 smithi006 bash[27292]: cluster 2023-08-28T02:37:23.907501+0000 mgr.y (mgr.14901) 8363 : cluster [DBG] pgmap v8370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:24.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:24 smithi006 bash[33918]: cluster 2023-08-28T02:37:23.907501+0000 mgr.y (mgr.14901) 8363 : cluster [DBG] pgmap v8370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:24 smithi130 bash[27579]: cluster 2023-08-28T02:37:23.907501+0000 mgr.y (mgr.14901) 8363 : cluster [DBG] pgmap v8370: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:25.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:24 smithi130 bash[51302]: debug 2023-08-28T02:37:24.801+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:24.806752+0000) 2023-08-28T02:37:25.208 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:37:24.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:25.209 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:37:24.792Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:25.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:25 smithi006 bash[27292]: audit 2023-08-28T02:37:25.081421+0000 mon.c (mon.2) 3321 : audit [DBG] from='client.? 172.21.15.6:0/1116431292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:25.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:25 smithi006 bash[33918]: audit 2023-08-28T02:37:25.081421+0000 mon.c (mon.2) 3321 : audit [DBG] from='client.? 172.21.15.6:0/1116431292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:25.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:25 smithi130 bash[27579]: audit 2023-08-28T02:37:25.081421+0000 mon.c (mon.2) 3321 : audit [DBG] from='client.? 172.21.15.6:0/1116431292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:26.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:25 smithi130 bash[51302]: debug 2023-08-28T02:37:25.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:25.807036+0000) 2023-08-28T02:37:26.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:26 smithi130 bash[27579]: cluster 2023-08-28T02:37:25.908632+0000 mgr.y (mgr.14901) 8364 : cluster [DBG] pgmap v8371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:26.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:26 smithi006 bash[33918]: cluster 2023-08-28T02:37:25.908632+0000 mgr.y (mgr.14901) 8364 : cluster [DBG] pgmap v8371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:26.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:26 smithi006 bash[27292]: cluster 2023-08-28T02:37:25.908632+0000 mgr.y (mgr.14901) 8364 : cluster [DBG] pgmap v8371: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:27.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:26 smithi130 bash[51302]: debug 2023-08-28T02:37:26.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:26.807281+0000) 2023-08-28T02:37:27.806 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:27 smithi130 bash[27579]: audit 2023-08-28T02:37:27.493048+0000 mon.c (mon.2) 3322 : audit [DBG] from='client.? 172.21.15.6:0/3771539021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:27.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:27 smithi006 bash[33918]: audit 2023-08-28T02:37:27.493048+0000 mon.c (mon.2) 3322 : audit [DBG] from='client.? 172.21.15.6:0/3771539021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:27.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:27 smithi006 bash[27292]: audit 2023-08-28T02:37:27.493048+0000 mon.c (mon.2) 3322 : audit [DBG] from='client.? 172.21.15.6:0/3771539021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:28.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:27 smithi130 bash[51302]: debug 2023-08-28T02:37:27.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:27.807518+0000) 2023-08-28T02:37:28.806 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:28 smithi130 bash[27579]: cluster 2023-08-28T02:37:27.909463+0000 mgr.y (mgr.14901) 8365 : cluster [DBG] pgmap v8372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:28.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:28 smithi006 bash[33918]: cluster 2023-08-28T02:37:27.909463+0000 mgr.y (mgr.14901) 8365 : cluster [DBG] pgmap v8372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:28.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:28 smithi006 bash[27292]: cluster 2023-08-28T02:37:27.909463+0000 mgr.y (mgr.14901) 8365 : cluster [DBG] pgmap v8372: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:28 smithi130 bash[51302]: debug 2023-08-28T02:37:28.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:28.807782+0000) 2023-08-28T02:37:30.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:29 smithi130 bash[27579]: audit 2023-08-28T02:37:29.907653+0000 mon.c (mon.2) 3323 : audit [DBG] from='client.? 172.21.15.6:0/4269080159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:30.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:29 smithi130 bash[51302]: debug 2023-08-28T02:37:29.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:29.808051+0000) 2023-08-28T02:37:30.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:29 smithi006 bash[33918]: audit 2023-08-28T02:37:29.907653+0000 mon.c (mon.2) 3323 : audit [DBG] from='client.? 172.21.15.6:0/4269080159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:30.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:29 smithi006 bash[27292]: audit 2023-08-28T02:37:29.907653+0000 mon.c (mon.2) 3323 : audit [DBG] from='client.? 172.21.15.6:0/4269080159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:31.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:30 smithi130 bash[27579]: cluster 2023-08-28T02:37:29.910679+0000 mgr.y (mgr.14901) 8366 : cluster [DBG] pgmap v8373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:31.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:30 smithi130 bash[51302]: debug 2023-08-28T02:37:30.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:30.808341+0000) 2023-08-28T02:37:31.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:30 smithi006 bash[27292]: cluster 2023-08-28T02:37:29.910679+0000 mgr.y (mgr.14901) 8366 : cluster [DBG] pgmap v8373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:31.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:37:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:37:30] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:37:31.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:30 smithi006 bash[33918]: cluster 2023-08-28T02:37:29.910679+0000 mgr.y (mgr.14901) 8366 : cluster [DBG] pgmap v8373: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:31.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:37:31.348Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:37:31.350Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:32.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:31 smithi130 bash[51302]: debug 2023-08-28T02:37:31.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:31.808631+0000) 2023-08-28T02:37:33.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:32 smithi130 bash[27579]: cluster 2023-08-28T02:37:31.911460+0000 mgr.y (mgr.14901) 8367 : cluster [DBG] pgmap v8374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:33.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:32 smithi130 bash[27579]: audit 2023-08-28T02:37:32.319993+0000 mon.a (mon.0) 5263 : audit [DBG] from='client.? 172.21.15.6:0/3270170271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:33.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:32 smithi130 bash[51302]: debug 2023-08-28T02:37:32.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:32.808896+0000) 2023-08-28T02:37:33.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:32 smithi006 bash[27292]: cluster 2023-08-28T02:37:31.911460+0000 mgr.y (mgr.14901) 8367 : cluster [DBG] pgmap v8374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:33.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:32 smithi006 bash[27292]: audit 2023-08-28T02:37:32.319993+0000 mon.a (mon.0) 5263 : audit [DBG] from='client.? 172.21.15.6:0/3270170271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:37:32.913Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:37:32.915Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:33.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:32 smithi006 bash[33918]: cluster 2023-08-28T02:37:31.911460+0000 mgr.y (mgr.14901) 8367 : cluster [DBG] pgmap v8374: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:33.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:32 smithi006 bash[33918]: audit 2023-08-28T02:37:32.319993+0000 mon.a (mon.0) 5263 : audit [DBG] from='client.? 172.21.15.6:0/3270170271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:34.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:33 smithi130 bash[51302]: debug 2023-08-28T02:37:33.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:33.809236+0000) 2023-08-28T02:37:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:34 smithi130 bash[27579]: cluster 2023-08-28T02:37:33.912165+0000 mgr.y (mgr.14901) 8368 : cluster [DBG] pgmap v8375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:34.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:34 smithi006 bash[33918]: cluster 2023-08-28T02:37:33.912165+0000 mgr.y (mgr.14901) 8368 : cluster [DBG] pgmap v8375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:34.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:34 smithi006 bash[27292]: cluster 2023-08-28T02:37:33.912165+0000 mgr.y (mgr.14901) 8368 : cluster [DBG] pgmap v8375: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:35.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:34 smithi130 bash[51302]: debug 2023-08-28T02:37:34.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:34.809497+0000) 2023-08-28T02:37:35.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:37:34.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:35.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:37:34.792Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:35.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:35 smithi130 bash[27579]: audit 2023-08-28T02:37:34.740147+0000 mon.c (mon.2) 3324 : audit [DBG] from='client.? 172.21.15.6:0/608070587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:35.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:35 smithi006 bash[33918]: audit 2023-08-28T02:37:34.740147+0000 mon.c (mon.2) 3324 : audit [DBG] from='client.? 172.21.15.6:0/608070587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:35.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:35 smithi006 bash[27292]: audit 2023-08-28T02:37:34.740147+0000 mon.c (mon.2) 3324 : audit [DBG] from='client.? 172.21.15.6:0/608070587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:36.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:35 smithi130 bash[51302]: debug 2023-08-28T02:37:35.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:35.809840+0000) 2023-08-28T02:37:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:36 smithi130 bash[27579]: cluster 2023-08-28T02:37:35.913062+0000 mgr.y (mgr.14901) 8369 : cluster [DBG] pgmap v8376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:36.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:36 smithi006 bash[33918]: cluster 2023-08-28T02:37:35.913062+0000 mgr.y (mgr.14901) 8369 : cluster [DBG] pgmap v8376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:36.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:36 smithi006 bash[27292]: cluster 2023-08-28T02:37:35.913062+0000 mgr.y (mgr.14901) 8369 : cluster [DBG] pgmap v8376: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:37.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:36 smithi130 bash[51302]: debug 2023-08-28T02:37:36.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:36.810148+0000) 2023-08-28T02:37:37.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:37 smithi130 bash[27579]: audit 2023-08-28T02:37:37.157632+0000 mon.a (mon.0) 5264 : audit [DBG] from='client.? 172.21.15.6:0/2024945779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:37.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:37 smithi006 bash[27292]: audit 2023-08-28T02:37:37.157632+0000 mon.a (mon.0) 5264 : audit [DBG] from='client.? 172.21.15.6:0/2024945779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:37.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:37 smithi006 bash[33918]: audit 2023-08-28T02:37:37.157632+0000 mon.a (mon.0) 5264 : audit [DBG] from='client.? 172.21.15.6:0/2024945779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:38.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:37 smithi130 bash[51302]: debug 2023-08-28T02:37:37.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:37.810421+0000) 2023-08-28T02:37:38.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:38 smithi130 bash[27579]: cluster 2023-08-28T02:37:37.913686+0000 mgr.y (mgr.14901) 8370 : cluster [DBG] pgmap v8377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:38.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:38 smithi006 bash[27292]: cluster 2023-08-28T02:37:37.913686+0000 mgr.y (mgr.14901) 8370 : cluster [DBG] pgmap v8377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:38.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:38 smithi006 bash[33918]: cluster 2023-08-28T02:37:37.913686+0000 mgr.y (mgr.14901) 8370 : cluster [DBG] pgmap v8377: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:39.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:38 smithi130 bash[51302]: debug 2023-08-28T02:37:38.805+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:38.810719+0000) 2023-08-28T02:37:39.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:39 smithi006 bash[27292]: audit 2023-08-28T02:37:39.567583+0000 mon.a (mon.0) 5265 : audit [DBG] from='client.? 172.21.15.6:0/3962996042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:39.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:39 smithi006 bash[33918]: audit 2023-08-28T02:37:39.567583+0000 mon.a (mon.0) 5265 : audit [DBG] from='client.? 172.21.15.6:0/3962996042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:40.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:39 smithi130 bash[27579]: audit 2023-08-28T02:37:39.567583+0000 mon.a (mon.0) 5265 : audit [DBG] from='client.? 172.21.15.6:0/3962996042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:40.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:39 smithi130 bash[51302]: debug 2023-08-28T02:37:39.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:39.811049+0000) 2023-08-28T02:37:40.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:37:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:37:40] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:37:40.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:40 smithi006 bash[27292]: cluster 2023-08-28T02:37:39.914826+0000 mgr.y (mgr.14901) 8371 : cluster [DBG] pgmap v8378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:40.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:40 smithi006 bash[33918]: cluster 2023-08-28T02:37:39.914826+0000 mgr.y (mgr.14901) 8371 : cluster [DBG] pgmap v8378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:41.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:40 smithi130 bash[27579]: cluster 2023-08-28T02:37:39.914826+0000 mgr.y (mgr.14901) 8371 : cluster [DBG] pgmap v8378: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:41.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:40 smithi130 bash[51302]: debug 2023-08-28T02:37:40.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:40.811288+0000) 2023-08-28T02:37:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:37:41.349Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:37:41.351Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:41 smithi130 bash[51302]: debug 2023-08-28T02:37:41.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:41.811616+0000) 2023-08-28T02:37:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:42 smithi130 bash[27579]: cluster 2023-08-28T02:37:41.915450+0000 mgr.y (mgr.14901) 8372 : cluster [DBG] pgmap v8379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:42 smithi130 bash[27579]: audit 2023-08-28T02:37:41.990267+0000 mon.a (mon.0) 5266 : audit [DBG] from='client.? 172.21.15.6:0/241475867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:43.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:42 smithi130 bash[51302]: debug 2023-08-28T02:37:42.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:42.811839+0000) 2023-08-28T02:37:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:37:42.914Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 9 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:37:42.916Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:43.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:42 smithi006 bash[27292]: cluster 2023-08-28T02:37:41.915450+0000 mgr.y (mgr.14901) 8372 : cluster [DBG] pgmap v8379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:43.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:42 smithi006 bash[27292]: audit 2023-08-28T02:37:41.990267+0000 mon.a (mon.0) 5266 : audit [DBG] from='client.? 172.21.15.6:0/241475867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:43.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:42 smithi006 bash[33918]: cluster 2023-08-28T02:37:41.915450+0000 mgr.y (mgr.14901) 8372 : cluster [DBG] pgmap v8379: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:43.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:42 smithi006 bash[33918]: audit 2023-08-28T02:37:41.990267+0000 mon.a (mon.0) 5266 : audit [DBG] from='client.? 172.21.15.6:0/241475867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:44.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:43 smithi130 bash[51302]: debug 2023-08-28T02:37:43.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:43.812149+0000) 2023-08-28T02:37:44.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:44 smithi130 bash[27579]: cluster 2023-08-28T02:37:43.916215+0000 mgr.y (mgr.14901) 8373 : cluster [DBG] pgmap v8380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:44.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:44 smithi006 bash[27292]: cluster 2023-08-28T02:37:43.916215+0000 mgr.y (mgr.14901) 8373 : cluster [DBG] pgmap v8380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:44.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:44 smithi006 bash[33918]: cluster 2023-08-28T02:37:43.916215+0000 mgr.y (mgr.14901) 8373 : cluster [DBG] pgmap v8380: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:45.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:44 smithi130 bash[51302]: debug 2023-08-28T02:37:44.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:44.812407+0000) 2023-08-28T02:37:45.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:37:44.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:45.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:37:44.793Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:45.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:45 smithi130 bash[27579]: audit 2023-08-28T02:37:44.403717+0000 mon.a (mon.0) 5267 : audit [DBG] from='client.? 172.21.15.6:0/764337225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:45.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:45 smithi006 bash[33918]: audit 2023-08-28T02:37:44.403717+0000 mon.a (mon.0) 5267 : audit [DBG] from='client.? 172.21.15.6:0/764337225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:45.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:45 smithi006 bash[27292]: audit 2023-08-28T02:37:44.403717+0000 mon.a (mon.0) 5267 : audit [DBG] from='client.? 172.21.15.6:0/764337225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:45 smithi130 bash[51302]: debug 2023-08-28T02:37:45.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:45.812688+0000) 2023-08-28T02:37:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:46 smithi130 bash[27579]: cluster 2023-08-28T02:37:45.917397+0000 mgr.y (mgr.14901) 8374 : cluster [DBG] pgmap v8381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:46.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:46 smithi006 bash[27292]: cluster 2023-08-28T02:37:45.917397+0000 mgr.y (mgr.14901) 8374 : cluster [DBG] pgmap v8381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:46.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:46 smithi006 bash[33918]: cluster 2023-08-28T02:37:45.917397+0000 mgr.y (mgr.14901) 8374 : cluster [DBG] pgmap v8381: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:46 smithi130 bash[51302]: debug 2023-08-28T02:37:46.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:46.812970+0000) 2023-08-28T02:37:47.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:47 smithi130 bash[27579]: audit 2023-08-28T02:37:46.809860+0000 mon.a (mon.0) 5268 : audit [DBG] from='client.? 172.21.15.6:0/3948234850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:47.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:47 smithi006 bash[27292]: audit 2023-08-28T02:37:46.809860+0000 mon.a (mon.0) 5268 : audit [DBG] from='client.? 172.21.15.6:0/3948234850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:47.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:47 smithi006 bash[33918]: audit 2023-08-28T02:37:46.809860+0000 mon.a (mon.0) 5268 : audit [DBG] from='client.? 172.21.15.6:0/3948234850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:47 smithi130 bash[51302]: debug 2023-08-28T02:37:47.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:47.813237+0000) 2023-08-28T02:37:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:48 smithi130 bash[27579]: cluster 2023-08-28T02:37:47.918004+0000 mgr.y (mgr.14901) 8375 : cluster [DBG] pgmap v8382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:48.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:48 smithi006 bash[27292]: cluster 2023-08-28T02:37:47.918004+0000 mgr.y (mgr.14901) 8375 : cluster [DBG] pgmap v8382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:48.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:48 smithi006 bash[33918]: cluster 2023-08-28T02:37:47.918004+0000 mgr.y (mgr.14901) 8375 : cluster [DBG] pgmap v8382: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:49.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:48 smithi130 bash[51302]: debug 2023-08-28T02:37:48.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:48.813567+0000) 2023-08-28T02:37:49.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:49 smithi130 bash[27579]: audit 2023-08-28T02:37:49.230792+0000 mon.c (mon.2) 3325 : audit [DBG] from='client.? 172.21.15.6:0/3383495440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:49.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:49 smithi006 bash[33918]: audit 2023-08-28T02:37:49.230792+0000 mon.c (mon.2) 3325 : audit [DBG] from='client.? 172.21.15.6:0/3383495440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:49.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:49 smithi006 bash[27292]: audit 2023-08-28T02:37:49.230792+0000 mon.c (mon.2) 3325 : audit [DBG] from='client.? 172.21.15.6:0/3383495440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:50.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:49 smithi130 bash[51302]: debug 2023-08-28T02:37:49.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:49.813849+0000) 2023-08-28T02:37:50.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:50 smithi130 bash[27579]: cluster 2023-08-28T02:37:49.919165+0000 mgr.y (mgr.14901) 8376 : cluster [DBG] pgmap v8383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:50.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:50 smithi006 bash[33918]: cluster 2023-08-28T02:37:49.919165+0000 mgr.y (mgr.14901) 8376 : cluster [DBG] pgmap v8383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:50.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:50 smithi006 bash[27292]: cluster 2023-08-28T02:37:49.919165+0000 mgr.y (mgr.14901) 8376 : cluster [DBG] pgmap v8383: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:51.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:50 smithi130 bash[51302]: debug 2023-08-28T02:37:50.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:50.814119+0000) 2023-08-28T02:37:51.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:37:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:37:50] "GET /metrics HTTP/1.1" 200 193126 "" "Prometheus/2.33.4" 2023-08-28T02:37:51.695 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:37:51.350Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:51.695 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:37:51.352Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:51.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:51 smithi006 bash[33918]: audit 2023-08-28T02:37:51.642012+0000 mon.a (mon.0) 5269 : audit [DBG] from='client.? 172.21.15.6:0/2061454653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:51.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:51 smithi006 bash[27292]: audit 2023-08-28T02:37:51.642012+0000 mon.a (mon.0) 5269 : audit [DBG] from='client.? 172.21.15.6:0/2061454653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:52.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:51 smithi130 bash[27579]: audit 2023-08-28T02:37:51.642012+0000 mon.a (mon.0) 5269 : audit [DBG] from='client.? 172.21.15.6:0/2061454653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:52.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:51 smithi130 bash[51302]: debug 2023-08-28T02:37:51.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:51.814424+0000) 2023-08-28T02:37:52.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:52 smithi006 bash[27292]: cluster 2023-08-28T02:37:51.919834+0000 mgr.y (mgr.14901) 8377 : cluster [DBG] pgmap v8384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:52.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:52 smithi006 bash[33918]: cluster 2023-08-28T02:37:51.919834+0000 mgr.y (mgr.14901) 8377 : cluster [DBG] pgmap v8384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:37:52.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:52.965 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:37:52.916Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:53.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:52 smithi130 bash[27579]: cluster 2023-08-28T02:37:51.919834+0000 mgr.y (mgr.14901) 8377 : cluster [DBG] pgmap v8384: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:53.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:52 smithi130 bash[51302]: debug 2023-08-28T02:37:52.809+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:52.814684+0000) 2023-08-28T02:37:54.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:53 smithi130 bash[51302]: debug 2023-08-28T02:37:53.813+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:53.814983+0000) 2023-08-28T02:37:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:54 smithi130 bash[27579]: cluster 2023-08-28T02:37:53.920398+0000 mgr.y (mgr.14901) 8378 : cluster [DBG] pgmap v8385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:54.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:54 smithi130 bash[27579]: audit 2023-08-28T02:37:54.059403+0000 mon.c (mon.2) 3326 : audit [DBG] from='client.? 172.21.15.6:0/668892565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:54.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:54 smithi006 bash[33918]: cluster 2023-08-28T02:37:53.920398+0000 mgr.y (mgr.14901) 8378 : cluster [DBG] pgmap v8385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:54.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:54 smithi006 bash[33918]: audit 2023-08-28T02:37:54.059403+0000 mon.c (mon.2) 3326 : audit [DBG] from='client.? 172.21.15.6:0/668892565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:54.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:54 smithi006 bash[27292]: cluster 2023-08-28T02:37:53.920398+0000 mgr.y (mgr.14901) 8378 : cluster [DBG] pgmap v8385: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:54.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:54 smithi006 bash[27292]: audit 2023-08-28T02:37:54.059403+0000 mon.c (mon.2) 3326 : audit [DBG] from='client.? 172.21.15.6:0/668892565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:55.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:54 smithi130 bash[51302]: debug 2023-08-28T02:37:54.813+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:54.815279+0000) 2023-08-28T02:37:55.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:37:54.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:55.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:37:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:37:54.794Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:37:56.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:55 smithi130 bash[51302]: debug 2023-08-28T02:37:55.812+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:55.815475+0000) 2023-08-28T02:37:57.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:56 smithi130 bash[51302]: debug 2023-08-28T02:37:56.812+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:56.815684+0000) 2023-08-28T02:37:57.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:56 smithi130 bash[27579]: cluster 2023-08-28T02:37:55.921503+0000 mgr.y (mgr.14901) 8379 : cluster [DBG] pgmap v8386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:57.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:56 smithi130 bash[27579]: audit 2023-08-28T02:37:56.475194+0000 mon.c (mon.2) 3327 : audit [DBG] from='client.? 172.21.15.6:0/987494936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:57.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:56 smithi006 bash[33918]: cluster 2023-08-28T02:37:55.921503+0000 mgr.y (mgr.14901) 8379 : cluster [DBG] pgmap v8386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:57.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:56 smithi006 bash[33918]: audit 2023-08-28T02:37:56.475194+0000 mon.c (mon.2) 3327 : audit [DBG] from='client.? 172.21.15.6:0/987494936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:57.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:56 smithi006 bash[27292]: cluster 2023-08-28T02:37:55.921503+0000 mgr.y (mgr.14901) 8379 : cluster [DBG] pgmap v8386: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:57.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:56 smithi006 bash[27292]: audit 2023-08-28T02:37:56.475194+0000 mon.c (mon.2) 3327 : audit [DBG] from='client.? 172.21.15.6:0/987494936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:58.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:57 smithi130 bash[51302]: debug 2023-08-28T02:37:57.812+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:57.815947+0000) 2023-08-28T02:37:59.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:58 smithi130 bash[51302]: debug 2023-08-28T02:37:58.812+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:58.816213+0000) 2023-08-28T02:37:59.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:58 smithi130 bash[27579]: cluster 2023-08-28T02:37:57.922360+0000 mgr.y (mgr.14901) 8380 : cluster [DBG] pgmap v8387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:59.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:37:58 smithi130 bash[27579]: audit 2023-08-28T02:37:58.894971+0000 mon.a (mon.0) 5270 : audit [DBG] from='client.? 172.21.15.6:0/2823212925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:59.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:58 smithi006 bash[27292]: cluster 2023-08-28T02:37:57.922360+0000 mgr.y (mgr.14901) 8380 : cluster [DBG] pgmap v8387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:59.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:37:58 smithi006 bash[27292]: audit 2023-08-28T02:37:58.894971+0000 mon.a (mon.0) 5270 : audit [DBG] from='client.? 172.21.15.6:0/2823212925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:59.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:58 smithi006 bash[33918]: cluster 2023-08-28T02:37:57.922360+0000 mgr.y (mgr.14901) 8380 : cluster [DBG] pgmap v8387: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:37:59.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:37:58 smithi006 bash[33918]: audit 2023-08-28T02:37:58.894971+0000 mon.a (mon.0) 5270 : audit [DBG] from='client.? 172.21.15.6:0/2823212925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:37:59 smithi130 bash[51302]: debug 2023-08-28T02:37:59.812+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:59.816495+0000) 2023-08-28T02:38:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:00 smithi130 bash[51302]: debug 2023-08-28T02:38:00.812+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:00.816756+0000) 2023-08-28T02:38:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:00 smithi130 bash[27579]: cluster 2023-08-28T02:37:59.923575+0000 mgr.y (mgr.14901) 8381 : cluster [DBG] pgmap v8388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:00 smithi130 bash[27579]: audit 2023-08-28T02:38:00.426077+0000 mon.b (mon.1) 909 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:38:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:38:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:38:00] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:38:01.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:00 smithi006 bash[33918]: cluster 2023-08-28T02:37:59.923575+0000 mgr.y (mgr.14901) 8381 : cluster [DBG] pgmap v8388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:01.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:00 smithi006 bash[33918]: audit 2023-08-28T02:38:00.426077+0000 mon.b (mon.1) 909 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:38:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:00 smithi006 bash[27292]: cluster 2023-08-28T02:37:59.923575+0000 mgr.y (mgr.14901) 8381 : cluster [DBG] pgmap v8388: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:00 smithi006 bash[27292]: audit 2023-08-28T02:38:00.426077+0000 mon.b (mon.1) 909 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:38:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:38:01.350Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:38:01.353Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:02.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:02 smithi130 bash[27579]: audit 2023-08-28T02:38:01.314298+0000 mon.c (mon.2) 3328 : audit [DBG] from='client.? 172.21.15.6:0/3755520988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:02.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:01 smithi130 bash[51302]: debug 2023-08-28T02:38:01.812+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:01.817072+0000) 2023-08-28T02:38:02.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:02 smithi006 bash[27292]: audit 2023-08-28T02:38:01.314298+0000 mon.c (mon.2) 3328 : audit [DBG] from='client.? 172.21.15.6:0/3755520988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:02.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:02 smithi006 bash[33918]: audit 2023-08-28T02:38:01.314298+0000 mon.c (mon.2) 3328 : audit [DBG] from='client.? 172.21.15.6:0/3755520988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:03 smithi130 bash[27579]: cluster 2023-08-28T02:38:01.924286+0000 mgr.y (mgr.14901) 8382 : cluster [DBG] pgmap v8389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:03.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:02 smithi130 bash[51302]: debug 2023-08-28T02:38:02.812+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:02.817423+0000) 2023-08-28T02:38:03.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:03 smithi006 bash[27292]: cluster 2023-08-28T02:38:01.924286+0000 mgr.y (mgr.14901) 8382 : cluster [DBG] pgmap v8389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:03.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:03 smithi006 bash[33918]: cluster 2023-08-28T02:38:01.924286+0000 mgr.y (mgr.14901) 8382 : cluster [DBG] pgmap v8389: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:38:02.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:38:02.917Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:04.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:03 smithi130 bash[51302]: debug 2023-08-28T02:38:03.812+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:03.817764+0000) 2023-08-28T02:38:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:04 smithi130 bash[27579]: audit 2023-08-28T02:38:03.743559+0000 mon.c (mon.2) 3329 : audit [DBG] from='client.? 172.21.15.6:0/725457293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:04.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:04 smithi130 bash[27579]: cluster 2023-08-28T02:38:03.925016+0000 mgr.y (mgr.14901) 8383 : cluster [DBG] pgmap v8390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:04.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:04 smithi006 bash[27292]: audit 2023-08-28T02:38:03.743559+0000 mon.c (mon.2) 3329 : audit [DBG] from='client.? 172.21.15.6:0/725457293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:04.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:04 smithi006 bash[27292]: cluster 2023-08-28T02:38:03.925016+0000 mgr.y (mgr.14901) 8383 : cluster [DBG] pgmap v8390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:04.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:04 smithi006 bash[33918]: audit 2023-08-28T02:38:03.743559+0000 mon.c (mon.2) 3329 : audit [DBG] from='client.? 172.21.15.6:0/725457293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:04.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:04 smithi006 bash[33918]: cluster 2023-08-28T02:38:03.925016+0000 mgr.y (mgr.14901) 8383 : cluster [DBG] pgmap v8390: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:05.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:04 smithi130 bash[51302]: debug 2023-08-28T02:38:04.812+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:04.818053+0000) 2023-08-28T02:38:05.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:38:04.793Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:05.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:38:04.795Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:06.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:05 smithi130 bash[51302]: debug 2023-08-28T02:38:05.812+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:05.818322+0000) 2023-08-28T02:38:07.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:06 smithi130 bash[51302]: debug 2023-08-28T02:38:06.812+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:06.818602+0000) 2023-08-28T02:38:07.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:06 smithi130 bash[27579]: cluster 2023-08-28T02:38:05.926034+0000 mgr.y (mgr.14901) 8384 : cluster [DBG] pgmap v8391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:07.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:06 smithi130 bash[27579]: audit 2023-08-28T02:38:06.201538+0000 mon.a (mon.0) 5271 : audit [DBG] from='client.? 172.21.15.6:0/1986039635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:07.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:06 smithi130 bash[27579]: audit 2023-08-28T02:38:06.280161+0000 mon.a (mon.0) 5272 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:38:07.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:06 smithi130 bash[27579]: audit 2023-08-28T02:38:06.714796+0000 mon.a (mon.0) 5273 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:38:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:06 smithi006 bash[27292]: cluster 2023-08-28T02:38:05.926034+0000 mgr.y (mgr.14901) 8384 : cluster [DBG] pgmap v8391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:06 smithi006 bash[27292]: audit 2023-08-28T02:38:06.201538+0000 mon.a (mon.0) 5271 : audit [DBG] from='client.? 172.21.15.6:0/1986039635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:06 smithi006 bash[27292]: audit 2023-08-28T02:38:06.280161+0000 mon.a (mon.0) 5272 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:38:07.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:06 smithi006 bash[27292]: audit 2023-08-28T02:38:06.714796+0000 mon.a (mon.0) 5273 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:38:07.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:06 smithi006 bash[33918]: cluster 2023-08-28T02:38:05.926034+0000 mgr.y (mgr.14901) 8384 : cluster [DBG] pgmap v8391: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:07.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:06 smithi006 bash[33918]: audit 2023-08-28T02:38:06.201538+0000 mon.a (mon.0) 5271 : audit [DBG] from='client.? 172.21.15.6:0/1986039635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:07.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:06 smithi006 bash[33918]: audit 2023-08-28T02:38:06.280161+0000 mon.a (mon.0) 5272 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:38:07.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:06 smithi006 bash[33918]: audit 2023-08-28T02:38:06.714796+0000 mon.a (mon.0) 5273 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:38:08.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:07 smithi130 bash[51302]: debug 2023-08-28T02:38:07.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:07.818955+0000) 2023-08-28T02:38:08.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:07 smithi130 bash[27579]: audit 2023-08-28T02:38:07.013315+0000 mon.b (mon.1) 910 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:38:08.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:07 smithi130 bash[27579]: audit 2023-08-28T02:38:07.015545+0000 mon.b (mon.1) 911 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:38:08.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:07 smithi130 bash[27579]: audit 2023-08-28T02:38:07.024804+0000 mon.a (mon.0) 5274 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:38:08.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:07 smithi006 bash[27292]: audit 2023-08-28T02:38:07.013315+0000 mon.b (mon.1) 910 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:38:08.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:07 smithi006 bash[27292]: audit 2023-08-28T02:38:07.015545+0000 mon.b (mon.1) 911 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:38:08.467 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:07 smithi006 bash[27292]: audit 2023-08-28T02:38:07.024804+0000 mon.a (mon.0) 5274 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:38:08.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:07 smithi006 bash[33918]: audit 2023-08-28T02:38:07.013315+0000 mon.b (mon.1) 910 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:38:08.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:07 smithi006 bash[33918]: audit 2023-08-28T02:38:07.015545+0000 mon.b (mon.1) 911 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:38:08.468 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:07 smithi006 bash[33918]: audit 2023-08-28T02:38:07.024804+0000 mon.a (mon.0) 5274 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:38:09.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:08 smithi130 bash[51302]: debug 2023-08-28T02:38:08.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:08.819271+0000) 2023-08-28T02:38:09.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:08 smithi130 bash[27579]: cluster 2023-08-28T02:38:07.926699+0000 mgr.y (mgr.14901) 8385 : cluster [DBG] pgmap v8392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:09.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:08 smithi130 bash[27579]: audit 2023-08-28T02:38:08.613605+0000 mon.c (mon.2) 3330 : audit [DBG] from='client.? 172.21.15.6:0/1115696932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:09.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:08 smithi006 bash[33918]: cluster 2023-08-28T02:38:07.926699+0000 mgr.y (mgr.14901) 8385 : cluster [DBG] pgmap v8392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:09.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:08 smithi006 bash[33918]: audit 2023-08-28T02:38:08.613605+0000 mon.c (mon.2) 3330 : audit [DBG] from='client.? 172.21.15.6:0/1115696932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:09.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:08 smithi006 bash[27292]: cluster 2023-08-28T02:38:07.926699+0000 mgr.y (mgr.14901) 8385 : cluster [DBG] pgmap v8392: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:09.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:08 smithi006 bash[27292]: audit 2023-08-28T02:38:08.613605+0000 mon.c (mon.2) 3330 : audit [DBG] from='client.? 172.21.15.6:0/1115696932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:10.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:09 smithi130 bash[51302]: debug 2023-08-28T02:38:09.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:09.819523+0000) 2023-08-28T02:38:11.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:10 smithi130 bash[27579]: cluster 2023-08-28T02:38:09.927883+0000 mgr.y (mgr.14901) 8386 : cluster [DBG] pgmap v8393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:11.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:10 smithi130 bash[51302]: debug 2023-08-28T02:38:10.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:10.819844+0000) 2023-08-28T02:38:11.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:38:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:38:10] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:38:11.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:10 smithi006 bash[33918]: cluster 2023-08-28T02:38:09.927883+0000 mgr.y (mgr.14901) 8386 : cluster [DBG] pgmap v8393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:11.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:10 smithi006 bash[27292]: cluster 2023-08-28T02:38:09.927883+0000 mgr.y (mgr.14901) 8386 : cluster [DBG] pgmap v8393: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:38:11.351Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:38:11.353Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:12.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:12 smithi130 bash[27579]: audit 2023-08-28T02:38:11.034664+0000 mon.a (mon.0) 5275 : audit [DBG] from='client.? 172.21.15.6:0/4121484313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:12.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:11 smithi130 bash[51302]: debug 2023-08-28T02:38:11.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:11.820189+0000) 2023-08-28T02:38:12.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:12 smithi006 bash[27292]: audit 2023-08-28T02:38:11.034664+0000 mon.a (mon.0) 5275 : audit [DBG] from='client.? 172.21.15.6:0/4121484313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:12.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:12 smithi006 bash[33918]: audit 2023-08-28T02:38:11.034664+0000 mon.a (mon.0) 5275 : audit [DBG] from='client.? 172.21.15.6:0/4121484313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:13.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:13 smithi130 bash[27579]: cluster 2023-08-28T02:38:11.928528+0000 mgr.y (mgr.14901) 8387 : cluster [DBG] pgmap v8394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:13.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:12 smithi130 bash[51302]: debug 2023-08-28T02:38:12.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:12.820519+0000) 2023-08-28T02:38:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:38:12.916Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:38:12.918Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:13 smithi006 bash[33918]: cluster 2023-08-28T02:38:11.928528+0000 mgr.y (mgr.14901) 8387 : cluster [DBG] pgmap v8394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:13.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:13 smithi006 bash[27292]: cluster 2023-08-28T02:38:11.928528+0000 mgr.y (mgr.14901) 8387 : cluster [DBG] pgmap v8394: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:14.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:13 smithi130 bash[51302]: debug 2023-08-28T02:38:13.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:13.820826+0000) 2023-08-28T02:38:14.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:14 smithi130 bash[27579]: audit 2023-08-28T02:38:13.447388+0000 mon.c (mon.2) 3331 : audit [DBG] from='client.? 172.21.15.6:0/2524110647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:14.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:14 smithi130 bash[27579]: cluster 2023-08-28T02:38:13.929214+0000 mgr.y (mgr.14901) 8388 : cluster [DBG] pgmap v8395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:14.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:14 smithi006 bash[33918]: audit 2023-08-28T02:38:13.447388+0000 mon.c (mon.2) 3331 : audit [DBG] from='client.? 172.21.15.6:0/2524110647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:14.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:14 smithi006 bash[33918]: cluster 2023-08-28T02:38:13.929214+0000 mgr.y (mgr.14901) 8388 : cluster [DBG] pgmap v8395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:14.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:14 smithi006 bash[27292]: audit 2023-08-28T02:38:13.447388+0000 mon.c (mon.2) 3331 : audit [DBG] from='client.? 172.21.15.6:0/2524110647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:14.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:14 smithi006 bash[27292]: cluster 2023-08-28T02:38:13.929214+0000 mgr.y (mgr.14901) 8388 : cluster [DBG] pgmap v8395: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:15.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:14 smithi130 bash[51302]: debug 2023-08-28T02:38:14.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:14.821178+0000) 2023-08-28T02:38:15.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:38:14.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:15.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:38:14.796Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:16.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:15 smithi130 bash[27579]: audit 2023-08-28T02:38:15.867209+0000 mon.a (mon.0) 5276 : audit [DBG] from='client.? 172.21.15.6:0/526007061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:16.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:15 smithi130 bash[51302]: debug 2023-08-28T02:38:15.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:15.821465+0000) 2023-08-28T02:38:16.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:15 smithi006 bash[27292]: audit 2023-08-28T02:38:15.867209+0000 mon.a (mon.0) 5276 : audit [DBG] from='client.? 172.21.15.6:0/526007061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:16.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:15 smithi006 bash[33918]: audit 2023-08-28T02:38:15.867209+0000 mon.a (mon.0) 5276 : audit [DBG] from='client.? 172.21.15.6:0/526007061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:17.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:16 smithi130 bash[27579]: cluster 2023-08-28T02:38:15.930055+0000 mgr.y (mgr.14901) 8389 : cluster [DBG] pgmap v8396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:17.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:16 smithi130 bash[51302]: debug 2023-08-28T02:38:16.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:16.821775+0000) 2023-08-28T02:38:17.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:16 smithi006 bash[27292]: cluster 2023-08-28T02:38:15.930055+0000 mgr.y (mgr.14901) 8389 : cluster [DBG] pgmap v8396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:17.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:16 smithi006 bash[33918]: cluster 2023-08-28T02:38:15.930055+0000 mgr.y (mgr.14901) 8389 : cluster [DBG] pgmap v8396: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:18.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:17 smithi130 bash[51302]: debug 2023-08-28T02:38:17.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:17.822083+0000) 2023-08-28T02:38:19.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:18 smithi130 bash[27579]: cluster 2023-08-28T02:38:17.930689+0000 mgr.y (mgr.14901) 8390 : cluster [DBG] pgmap v8397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:19.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:18 smithi130 bash[27579]: audit 2023-08-28T02:38:18.276180+0000 mon.a (mon.0) 5277 : audit [DBG] from='client.? 172.21.15.6:0/831939724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:19.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:18 smithi130 bash[51302]: debug 2023-08-28T02:38:18.816+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:18.822405+0000) 2023-08-28T02:38:19.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:18 smithi006 bash[33918]: cluster 2023-08-28T02:38:17.930689+0000 mgr.y (mgr.14901) 8390 : cluster [DBG] pgmap v8397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:19.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:18 smithi006 bash[33918]: audit 2023-08-28T02:38:18.276180+0000 mon.a (mon.0) 5277 : audit [DBG] from='client.? 172.21.15.6:0/831939724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:19.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:18 smithi006 bash[27292]: cluster 2023-08-28T02:38:17.930689+0000 mgr.y (mgr.14901) 8390 : cluster [DBG] pgmap v8397: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:19.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:18 smithi006 bash[27292]: audit 2023-08-28T02:38:18.276180+0000 mon.a (mon.0) 5277 : audit [DBG] from='client.? 172.21.15.6:0/831939724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:20.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:19 smithi130 bash[51302]: debug 2023-08-28T02:38:19.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:19.822740+0000) 2023-08-28T02:38:21.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:20 smithi130 bash[51302]: debug 2023-08-28T02:38:20.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:20.823080+0000) 2023-08-28T02:38:21.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:20 smithi130 bash[27579]: cluster 2023-08-28T02:38:19.931794+0000 mgr.y (mgr.14901) 8391 : cluster [DBG] pgmap v8398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:21.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:20 smithi130 bash[27579]: audit 2023-08-28T02:38:20.697104+0000 mon.c (mon.2) 3332 : audit [DBG] from='client.? 172.21.15.6:0/3963003023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:38:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:38:20] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T02:38:21.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:20 smithi006 bash[27292]: cluster 2023-08-28T02:38:19.931794+0000 mgr.y (mgr.14901) 8391 : cluster [DBG] pgmap v8398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:21.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:20 smithi006 bash[27292]: audit 2023-08-28T02:38:20.697104+0000 mon.c (mon.2) 3332 : audit [DBG] from='client.? 172.21.15.6:0/3963003023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:21.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:20 smithi006 bash[33918]: cluster 2023-08-28T02:38:19.931794+0000 mgr.y (mgr.14901) 8391 : cluster [DBG] pgmap v8398: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:21.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:20 smithi006 bash[33918]: audit 2023-08-28T02:38:20.697104+0000 mon.c (mon.2) 3332 : audit [DBG] from='client.? 172.21.15.6:0/3963003023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:38:21.352Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:21.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:38:21.354Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:22.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:21 smithi130 bash[51302]: debug 2023-08-28T02:38:21.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:21.823457+0000) 2023-08-28T02:38:23.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:22 smithi130 bash[27579]: cluster 2023-08-28T02:38:21.932487+0000 mgr.y (mgr.14901) 8392 : cluster [DBG] pgmap v8399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:23.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:22 smithi130 bash[51302]: debug 2023-08-28T02:38:22.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:22.823783+0000) 2023-08-28T02:38:23.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:22 smithi006 bash[27292]: cluster 2023-08-28T02:38:21.932487+0000 mgr.y (mgr.14901) 8392 : cluster [DBG] pgmap v8399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:38:22.916Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:38:22.918Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:23.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:22 smithi006 bash[33918]: cluster 2023-08-28T02:38:21.932487+0000 mgr.y (mgr.14901) 8392 : cluster [DBG] pgmap v8399: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:24.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:24 smithi130 bash[27579]: audit 2023-08-28T02:38:23.120353+0000 mon.c (mon.2) 3333 : audit [DBG] from='client.? 172.21.15.6:0/1017995199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:24.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:23 smithi130 bash[51302]: debug 2023-08-28T02:38:23.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:23.824152+0000) 2023-08-28T02:38:24.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:24 smithi006 bash[33918]: audit 2023-08-28T02:38:23.120353+0000 mon.c (mon.2) 3333 : audit [DBG] from='client.? 172.21.15.6:0/1017995199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:24.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:23 smithi006 bash[27292]: audit 2023-08-28T02:38:23.120353+0000 mon.c (mon.2) 3333 : audit [DBG] from='client.? 172.21.15.6:0/1017995199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:25.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:25 smithi130 bash[27579]: cluster 2023-08-28T02:38:23.933222+0000 mgr.y (mgr.14901) 8393 : cluster [DBG] pgmap v8400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:25.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:24 smithi130 bash[51302]: debug 2023-08-28T02:38:24.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:24.824622+0000) 2023-08-28T02:38:25.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:25 smithi006 bash[27292]: cluster 2023-08-28T02:38:23.933222+0000 mgr.y (mgr.14901) 8393 : cluster [DBG] pgmap v8400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:25.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:25 smithi006 bash[33918]: cluster 2023-08-28T02:38:23.933222+0000 mgr.y (mgr.14901) 8393 : cluster [DBG] pgmap v8400: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:25.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:38:24.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:25.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:38:24.797Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:26.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:25 smithi130 bash[51302]: debug 2023-08-28T02:38:25.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:25.824931+0000) 2023-08-28T02:38:26.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:26 smithi130 bash[27579]: audit 2023-08-28T02:38:25.535572+0000 mon.a (mon.0) 5278 : audit [DBG] from='client.? 172.21.15.6:0/442257539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:26.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:26 smithi130 bash[27579]: cluster 2023-08-28T02:38:25.934349+0000 mgr.y (mgr.14901) 8394 : cluster [DBG] pgmap v8401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:26.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:26 smithi006 bash[27292]: audit 2023-08-28T02:38:25.535572+0000 mon.a (mon.0) 5278 : audit [DBG] from='client.? 172.21.15.6:0/442257539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:26.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:26 smithi006 bash[27292]: cluster 2023-08-28T02:38:25.934349+0000 mgr.y (mgr.14901) 8394 : cluster [DBG] pgmap v8401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:26.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:26 smithi006 bash[33918]: audit 2023-08-28T02:38:25.535572+0000 mon.a (mon.0) 5278 : audit [DBG] from='client.? 172.21.15.6:0/442257539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:26.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:26 smithi006 bash[33918]: cluster 2023-08-28T02:38:25.934349+0000 mgr.y (mgr.14901) 8394 : cluster [DBG] pgmap v8401: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:27.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:26 smithi130 bash[51302]: debug 2023-08-28T02:38:26.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:26.825262+0000) 2023-08-28T02:38:28.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:27 smithi130 bash[51302]: debug 2023-08-28T02:38:27.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:27.825519+0000) 2023-08-28T02:38:28.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:28 smithi130 bash[27579]: cluster 2023-08-28T02:38:27.934949+0000 mgr.y (mgr.14901) 8395 : cluster [DBG] pgmap v8402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:28.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:28 smithi130 bash[27579]: audit 2023-08-28T02:38:27.954019+0000 mon.a (mon.0) 5279 : audit [DBG] from='client.? 172.21.15.6:0/4142903810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:28.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:28 smithi006 bash[27292]: cluster 2023-08-28T02:38:27.934949+0000 mgr.y (mgr.14901) 8395 : cluster [DBG] pgmap v8402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:28.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:28 smithi006 bash[27292]: audit 2023-08-28T02:38:27.954019+0000 mon.a (mon.0) 5279 : audit [DBG] from='client.? 172.21.15.6:0/4142903810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:28.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:28 smithi006 bash[33918]: cluster 2023-08-28T02:38:27.934949+0000 mgr.y (mgr.14901) 8395 : cluster [DBG] pgmap v8402: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:28.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:28 smithi006 bash[33918]: audit 2023-08-28T02:38:27.954019+0000 mon.a (mon.0) 5279 : audit [DBG] from='client.? 172.21.15.6:0/4142903810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:28 smithi130 bash[51302]: debug 2023-08-28T02:38:28.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:28.825778+0000) 2023-08-28T02:38:30.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:29 smithi130 bash[51302]: debug 2023-08-28T02:38:29.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:29.826089+0000) 2023-08-28T02:38:31.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:30 smithi130 bash[51302]: debug 2023-08-28T02:38:30.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:30.826331+0000) 2023-08-28T02:38:31.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:30 smithi130 bash[27579]: cluster 2023-08-28T02:38:29.936001+0000 mgr.y (mgr.14901) 8396 : cluster [DBG] pgmap v8403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:31.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:30 smithi130 bash[27579]: audit 2023-08-28T02:38:30.376009+0000 mon.a (mon.0) 5280 : audit [DBG] from='client.? 172.21.15.6:0/2479232532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:31.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:38:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:38:30] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:38:31.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:30 smithi006 bash[27292]: cluster 2023-08-28T02:38:29.936001+0000 mgr.y (mgr.14901) 8396 : cluster [DBG] pgmap v8403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:31.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:30 smithi006 bash[27292]: audit 2023-08-28T02:38:30.376009+0000 mon.a (mon.0) 5280 : audit [DBG] from='client.? 172.21.15.6:0/2479232532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:31.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:30 smithi006 bash[33918]: cluster 2023-08-28T02:38:29.936001+0000 mgr.y (mgr.14901) 8396 : cluster [DBG] pgmap v8403: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:31.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:30 smithi006 bash[33918]: audit 2023-08-28T02:38:30.376009+0000 mon.a (mon.0) 5280 : audit [DBG] from='client.? 172.21.15.6:0/2479232532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:38:31.352Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:31.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:38:31.354Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:32.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:31 smithi130 bash[51302]: debug 2023-08-28T02:38:31.820+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:31.826639+0000) 2023-08-28T02:38:33.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:32 smithi130 bash[51302]: debug 2023-08-28T02:38:32.824+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:32.826949+0000) 2023-08-28T02:38:33.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:33 smithi130 bash[27579]: cluster 2023-08-28T02:38:31.936685+0000 mgr.y (mgr.14901) 8397 : cluster [DBG] pgmap v8404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:33.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:33 smithi130 bash[27579]: audit 2023-08-28T02:38:32.790498+0000 mon.c (mon.2) 3334 : audit [DBG] from='client.? 172.21.15.6:0/2915144686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:33.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:32 smithi006 bash[27292]: cluster 2023-08-28T02:38:31.936685+0000 mgr.y (mgr.14901) 8397 : cluster [DBG] pgmap v8404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:33.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:32 smithi006 bash[27292]: audit 2023-08-28T02:38:32.790498+0000 mon.c (mon.2) 3334 : audit [DBG] from='client.? 172.21.15.6:0/2915144686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:38:32.917Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:38:32.920Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:33.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:33 smithi006 bash[33918]: cluster 2023-08-28T02:38:31.936685+0000 mgr.y (mgr.14901) 8397 : cluster [DBG] pgmap v8404: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:33.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:33 smithi006 bash[33918]: audit 2023-08-28T02:38:32.790498+0000 mon.c (mon.2) 3334 : audit [DBG] from='client.? 172.21.15.6:0/2915144686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:34.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:33 smithi130 bash[51302]: debug 2023-08-28T02:38:33.824+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:33.827258+0000) 2023-08-28T02:38:34.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:34 smithi130 bash[27579]: cluster 2023-08-28T02:38:33.937364+0000 mgr.y (mgr.14901) 8398 : cluster [DBG] pgmap v8405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:34.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:34 smithi006 bash[33918]: cluster 2023-08-28T02:38:33.937364+0000 mgr.y (mgr.14901) 8398 : cluster [DBG] pgmap v8405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:34.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:34 smithi006 bash[27292]: cluster 2023-08-28T02:38:33.937364+0000 mgr.y (mgr.14901) 8398 : cluster [DBG] pgmap v8405: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:35.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:34 smithi130 bash[51302]: debug 2023-08-28T02:38:34.824+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:34.827506+0000) 2023-08-28T02:38:35.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:38:34.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:35.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:38:34.798Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:35.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:35 smithi130 bash[27579]: audit 2023-08-28T02:38:35.204230+0000 mon.c (mon.2) 3335 : audit [DBG] from='client.? 172.21.15.6:0/1145624192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:35.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:35 smithi006 bash[33918]: audit 2023-08-28T02:38:35.204230+0000 mon.c (mon.2) 3335 : audit [DBG] from='client.? 172.21.15.6:0/1145624192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:35.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:35 smithi006 bash[27292]: audit 2023-08-28T02:38:35.204230+0000 mon.c (mon.2) 3335 : audit [DBG] from='client.? 172.21.15.6:0/1145624192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:36.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:35 smithi130 bash[51302]: debug 2023-08-28T02:38:35.824+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:35.827751+0000) 2023-08-28T02:38:36.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:36 smithi130 bash[27579]: cluster 2023-08-28T02:38:35.938481+0000 mgr.y (mgr.14901) 8399 : cluster [DBG] pgmap v8406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:36.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:36 smithi006 bash[33918]: cluster 2023-08-28T02:38:35.938481+0000 mgr.y (mgr.14901) 8399 : cluster [DBG] pgmap v8406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:36.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:36 smithi006 bash[27292]: cluster 2023-08-28T02:38:35.938481+0000 mgr.y (mgr.14901) 8399 : cluster [DBG] pgmap v8406: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:37.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:36 smithi130 bash[51302]: debug 2023-08-28T02:38:36.824+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:36.828042+0000) 2023-08-28T02:38:37.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:37 smithi006 bash[27292]: audit 2023-08-28T02:38:37.628682+0000 mon.a (mon.0) 5281 : audit [DBG] from='client.? 172.21.15.6:0/2989677483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:37.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:37 smithi006 bash[33918]: audit 2023-08-28T02:38:37.628682+0000 mon.a (mon.0) 5281 : audit [DBG] from='client.? 172.21.15.6:0/2989677483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:38.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:37 smithi130 bash[51302]: debug 2023-08-28T02:38:37.824+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:37.828303+0000) 2023-08-28T02:38:38.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:37 smithi130 bash[27579]: audit 2023-08-28T02:38:37.628682+0000 mon.a (mon.0) 5281 : audit [DBG] from='client.? 172.21.15.6:0/2989677483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:38.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:38 smithi006 bash[33918]: cluster 2023-08-28T02:38:37.939155+0000 mgr.y (mgr.14901) 8400 : cluster [DBG] pgmap v8407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:38.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:38 smithi006 bash[27292]: cluster 2023-08-28T02:38:37.939155+0000 mgr.y (mgr.14901) 8400 : cluster [DBG] pgmap v8407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:39.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:38 smithi130 bash[27579]: cluster 2023-08-28T02:38:37.939155+0000 mgr.y (mgr.14901) 8400 : cluster [DBG] pgmap v8407: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:39.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:38 smithi130 bash[51302]: debug 2023-08-28T02:38:38.824+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:38.828598+0000) 2023-08-28T02:38:40.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:39 smithi130 bash[51302]: debug 2023-08-28T02:38:39.828+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:39.828879+0000) 2023-08-28T02:38:41.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:40 smithi130 bash[51302]: debug 2023-08-28T02:38:40.828+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:40.829161+0000) 2023-08-28T02:38:41.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:40 smithi130 bash[27579]: cluster 2023-08-28T02:38:39.940295+0000 mgr.y (mgr.14901) 8401 : cluster [DBG] pgmap v8408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:41.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:40 smithi130 bash[27579]: audit 2023-08-28T02:38:40.053734+0000 mon.a (mon.0) 5282 : audit [DBG] from='client.? 172.21.15.6:0/608166298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:38:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:38:40] "GET /metrics HTTP/1.1" 200 193123 "" "Prometheus/2.33.4" 2023-08-28T02:38:41.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:40 smithi006 bash[27292]: cluster 2023-08-28T02:38:39.940295+0000 mgr.y (mgr.14901) 8401 : cluster [DBG] pgmap v8408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:41.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:40 smithi006 bash[27292]: audit 2023-08-28T02:38:40.053734+0000 mon.a (mon.0) 5282 : audit [DBG] from='client.? 172.21.15.6:0/608166298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:41.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:40 smithi006 bash[33918]: cluster 2023-08-28T02:38:39.940295+0000 mgr.y (mgr.14901) 8401 : cluster [DBG] pgmap v8408: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:41.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:40 smithi006 bash[33918]: audit 2023-08-28T02:38:40.053734+0000 mon.a (mon.0) 5282 : audit [DBG] from='client.? 172.21.15.6:0/608166298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:38:41.353Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:38:41.355Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:42.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:41 smithi130 bash[51302]: debug 2023-08-28T02:38:41.828+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:41.829486+0000) 2023-08-28T02:38:43.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:42 smithi130 bash[51302]: debug 2023-08-28T02:38:42.828+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:42.829798+0000) 2023-08-28T02:38:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:43 smithi130 bash[27579]: cluster 2023-08-28T02:38:41.940995+0000 mgr.y (mgr.14901) 8402 : cluster [DBG] pgmap v8409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:43.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:43 smithi130 bash[27579]: audit 2023-08-28T02:38:42.471810+0000 mon.c (mon.2) 3336 : audit [DBG] from='client.? 172.21.15.6:0/386591466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:38:42.918Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:38:42.920Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:43.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:43 smithi006 bash[27292]: cluster 2023-08-28T02:38:41.940995+0000 mgr.y (mgr.14901) 8402 : cluster [DBG] pgmap v8409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:43.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:43 smithi006 bash[27292]: audit 2023-08-28T02:38:42.471810+0000 mon.c (mon.2) 3336 : audit [DBG] from='client.? 172.21.15.6:0/386591466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:43.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:43 smithi006 bash[33918]: cluster 2023-08-28T02:38:41.940995+0000 mgr.y (mgr.14901) 8402 : cluster [DBG] pgmap v8409: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:43.216 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:43 smithi006 bash[33918]: audit 2023-08-28T02:38:42.471810+0000 mon.c (mon.2) 3336 : audit [DBG] from='client.? 172.21.15.6:0/386591466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:44.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:43 smithi130 bash[51302]: debug 2023-08-28T02:38:43.824+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:43.830152+0000) 2023-08-28T02:38:44.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:44 smithi130 bash[27579]: cluster 2023-08-28T02:38:43.941643+0000 mgr.y (mgr.14901) 8403 : cluster [DBG] pgmap v8410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:44.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:44 smithi006 bash[27292]: cluster 2023-08-28T02:38:43.941643+0000 mgr.y (mgr.14901) 8403 : cluster [DBG] pgmap v8410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:44.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:44 smithi006 bash[33918]: cluster 2023-08-28T02:38:43.941643+0000 mgr.y (mgr.14901) 8403 : cluster [DBG] pgmap v8410: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:45.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:44 smithi130 bash[51302]: debug 2023-08-28T02:38:44.828+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:44.830444+0000) 2023-08-28T02:38:45.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:38:44.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:45.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:38:44.799Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:45.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:45 smithi130 bash[27579]: audit 2023-08-28T02:38:44.892332+0000 mon.c (mon.2) 3337 : audit [DBG] from='client.? 172.21.15.6:0/2123549722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:45.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:45 smithi006 bash[27292]: audit 2023-08-28T02:38:44.892332+0000 mon.c (mon.2) 3337 : audit [DBG] from='client.? 172.21.15.6:0/2123549722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:45.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:45 smithi006 bash[33918]: audit 2023-08-28T02:38:44.892332+0000 mon.c (mon.2) 3337 : audit [DBG] from='client.? 172.21.15.6:0/2123549722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:45 smithi130 bash[51302]: debug 2023-08-28T02:38:45.828+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:45.830729+0000) 2023-08-28T02:38:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:46 smithi130 bash[27579]: cluster 2023-08-28T02:38:45.942882+0000 mgr.y (mgr.14901) 8404 : cluster [DBG] pgmap v8411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:46.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:46 smithi006 bash[27292]: cluster 2023-08-28T02:38:45.942882+0000 mgr.y (mgr.14901) 8404 : cluster [DBG] pgmap v8411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:46.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:46 smithi006 bash[33918]: cluster 2023-08-28T02:38:45.942882+0000 mgr.y (mgr.14901) 8404 : cluster [DBG] pgmap v8411: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:47.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:46 smithi130 bash[51302]: debug 2023-08-28T02:38:46.828+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:46.831013+0000) 2023-08-28T02:38:47.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:47 smithi130 bash[27579]: audit 2023-08-28T02:38:47.301670+0000 mon.a (mon.0) 5283 : audit [DBG] from='client.? 172.21.15.6:0/2962192920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:47.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:47 smithi006 bash[27292]: audit 2023-08-28T02:38:47.301670+0000 mon.a (mon.0) 5283 : audit [DBG] from='client.? 172.21.15.6:0/2962192920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:47.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:47 smithi006 bash[33918]: audit 2023-08-28T02:38:47.301670+0000 mon.a (mon.0) 5283 : audit [DBG] from='client.? 172.21.15.6:0/2962192920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:48.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:47 smithi130 bash[51302]: debug 2023-08-28T02:38:47.828+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:47.831259+0000) 2023-08-28T02:38:48.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:48 smithi130 bash[27579]: cluster 2023-08-28T02:38:47.943575+0000 mgr.y (mgr.14901) 8405 : cluster [DBG] pgmap v8412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:48.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:48 smithi006 bash[27292]: cluster 2023-08-28T02:38:47.943575+0000 mgr.y (mgr.14901) 8405 : cluster [DBG] pgmap v8412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:48.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:48 smithi006 bash[33918]: cluster 2023-08-28T02:38:47.943575+0000 mgr.y (mgr.14901) 8405 : cluster [DBG] pgmap v8412: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:49.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:48 smithi130 bash[51302]: debug 2023-08-28T02:38:48.828+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:48.831479+0000) 2023-08-28T02:38:50.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:49 smithi130 bash[27579]: audit 2023-08-28T02:38:49.710689+0000 mon.a (mon.0) 5284 : audit [DBG] from='client.? 172.21.15.6:0/620751950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:50.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:49 smithi130 bash[51302]: debug 2023-08-28T02:38:49.827+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:49.831745+0000) 2023-08-28T02:38:50.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:49 smithi006 bash[33918]: audit 2023-08-28T02:38:49.710689+0000 mon.a (mon.0) 5284 : audit [DBG] from='client.? 172.21.15.6:0/620751950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:50.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:49 smithi006 bash[27292]: audit 2023-08-28T02:38:49.710689+0000 mon.a (mon.0) 5284 : audit [DBG] from='client.? 172.21.15.6:0/620751950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:51.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:50 smithi130 bash[27579]: cluster 2023-08-28T02:38:49.944713+0000 mgr.y (mgr.14901) 8406 : cluster [DBG] pgmap v8413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:51.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:50 smithi130 bash[51302]: debug 2023-08-28T02:38:50.827+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:50.832003+0000) 2023-08-28T02:38:51.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:50 smithi006 bash[27292]: cluster 2023-08-28T02:38:49.944713+0000 mgr.y (mgr.14901) 8406 : cluster [DBG] pgmap v8413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:51.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:38:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:38:50] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:38:51.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:50 smithi006 bash[33918]: cluster 2023-08-28T02:38:49.944713+0000 mgr.y (mgr.14901) 8406 : cluster [DBG] pgmap v8413: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:38:51.353Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:38:51.356Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:52.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:51 smithi130 bash[51302]: debug 2023-08-28T02:38:51.831+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:51.832282+0000) 2023-08-28T02:38:53.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:52 smithi130 bash[51302]: debug 2023-08-28T02:38:52.827+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:52.832575+0000) 2023-08-28T02:38:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:53 smithi130 bash[27579]: cluster 2023-08-28T02:38:51.945365+0000 mgr.y (mgr.14901) 8407 : cluster [DBG] pgmap v8414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:53.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:53 smithi130 bash[27579]: audit 2023-08-28T02:38:52.128558+0000 mon.a (mon.0) 5285 : audit [DBG] from='client.? 172.21.15.6:0/2098675335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:53.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:53 smithi006 bash[27292]: cluster 2023-08-28T02:38:51.945365+0000 mgr.y (mgr.14901) 8407 : cluster [DBG] pgmap v8414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:53.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:53 smithi006 bash[27292]: audit 2023-08-28T02:38:52.128558+0000 mon.a (mon.0) 5285 : audit [DBG] from='client.? 172.21.15.6:0/2098675335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:53.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:53 smithi006 bash[33918]: cluster 2023-08-28T02:38:51.945365+0000 mgr.y (mgr.14901) 8407 : cluster [DBG] pgmap v8414: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:53.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:53 smithi006 bash[33918]: audit 2023-08-28T02:38:52.128558+0000 mon.a (mon.0) 5285 : audit [DBG] from='client.? 172.21.15.6:0/2098675335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:38:52.919Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:38:52.921Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:54.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:53 smithi130 bash[51302]: debug 2023-08-28T02:38:53.827+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:53.832874+0000) 2023-08-28T02:38:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:54 smithi130 bash[27579]: cluster 2023-08-28T02:38:53.946030+0000 mgr.y (mgr.14901) 8408 : cluster [DBG] pgmap v8415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:54.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:54 smithi006 bash[27292]: cluster 2023-08-28T02:38:53.946030+0000 mgr.y (mgr.14901) 8408 : cluster [DBG] pgmap v8415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:54.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:54 smithi006 bash[33918]: cluster 2023-08-28T02:38:53.946030+0000 mgr.y (mgr.14901) 8408 : cluster [DBG] pgmap v8415: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:55.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:54 smithi130 bash[51302]: debug 2023-08-28T02:38:54.827+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:54.833114+0000) 2023-08-28T02:38:55.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:38:54.798Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:55.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:38:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:38:54.800Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:38:55.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:55 smithi130 bash[27579]: audit 2023-08-28T02:38:54.538914+0000 mon.a (mon.0) 5286 : audit [DBG] from='client.? 172.21.15.6:0/673059242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:55.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:55 smithi006 bash[27292]: audit 2023-08-28T02:38:54.538914+0000 mon.a (mon.0) 5286 : audit [DBG] from='client.? 172.21.15.6:0/673059242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:55.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:55 smithi006 bash[33918]: audit 2023-08-28T02:38:54.538914+0000 mon.a (mon.0) 5286 : audit [DBG] from='client.? 172.21.15.6:0/673059242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:56.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:55 smithi130 bash[51302]: debug 2023-08-28T02:38:55.831+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:55.833458+0000) 2023-08-28T02:38:56.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:56 smithi130 bash[27579]: cluster 2023-08-28T02:38:55.947095+0000 mgr.y (mgr.14901) 8409 : cluster [DBG] pgmap v8416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:56.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:56 smithi006 bash[27292]: cluster 2023-08-28T02:38:55.947095+0000 mgr.y (mgr.14901) 8409 : cluster [DBG] pgmap v8416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:56.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:56 smithi006 bash[33918]: cluster 2023-08-28T02:38:55.947095+0000 mgr.y (mgr.14901) 8409 : cluster [DBG] pgmap v8416: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:57.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:56 smithi130 bash[51302]: debug 2023-08-28T02:38:56.831+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:56.833739+0000) 2023-08-28T02:38:57.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:57 smithi130 bash[27579]: audit 2023-08-28T02:38:56.948523+0000 mon.a (mon.0) 5287 : audit [DBG] from='client.? 172.21.15.6:0/2907573126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:57.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:57 smithi006 bash[27292]: audit 2023-08-28T02:38:56.948523+0000 mon.a (mon.0) 5287 : audit [DBG] from='client.? 172.21.15.6:0/2907573126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:57.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:57 smithi006 bash[33918]: audit 2023-08-28T02:38:56.948523+0000 mon.a (mon.0) 5287 : audit [DBG] from='client.? 172.21.15.6:0/2907573126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:58.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:57 smithi130 bash[51302]: debug 2023-08-28T02:38:57.827+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:57.833983+0000) 2023-08-28T02:38:58.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:58 smithi130 bash[27579]: cluster 2023-08-28T02:38:57.947777+0000 mgr.y (mgr.14901) 8410 : cluster [DBG] pgmap v8417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:58.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:58 smithi006 bash[33918]: cluster 2023-08-28T02:38:57.947777+0000 mgr.y (mgr.14901) 8410 : cluster [DBG] pgmap v8417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:58.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:58 smithi006 bash[27292]: cluster 2023-08-28T02:38:57.947777+0000 mgr.y (mgr.14901) 8410 : cluster [DBG] pgmap v8417: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:38:59.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:58 smithi130 bash[51302]: debug 2023-08-28T02:38:58.827+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:58.834286+0000) 2023-08-28T02:38:59.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:38:59 smithi006 bash[27292]: audit 2023-08-28T02:38:59.373193+0000 mon.c (mon.2) 3338 : audit [DBG] from='client.? 172.21.15.6:0/1096605846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:59.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:38:59 smithi006 bash[33918]: audit 2023-08-28T02:38:59.373193+0000 mon.c (mon.2) 3338 : audit [DBG] from='client.? 172.21.15.6:0/1096605846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:59.833 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:38:59 smithi130 bash[27579]: audit 2023-08-28T02:38:59.373193+0000 mon.c (mon.2) 3338 : audit [DBG] from='client.? 172.21.15.6:0/1096605846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:38:59 smithi130 bash[51302]: debug 2023-08-28T02:38:59.827+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:59.834550+0000) 2023-08-28T02:39:00.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:00 smithi006 bash[33918]: cluster 2023-08-28T02:38:59.949023+0000 mgr.y (mgr.14901) 8411 : cluster [DBG] pgmap v8418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:00.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:00 smithi006 bash[27292]: cluster 2023-08-28T02:38:59.949023+0000 mgr.y (mgr.14901) 8411 : cluster [DBG] pgmap v8418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:00.833 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:00 smithi130 bash[27579]: cluster 2023-08-28T02:38:59.949023+0000 mgr.y (mgr.14901) 8411 : cluster [DBG] pgmap v8418: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:01.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:00 smithi130 bash[51302]: debug 2023-08-28T02:39:00.831+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:00.834835+0000) 2023-08-28T02:39:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:39:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:39:00] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:39:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:39:01.355Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:39:01.356Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:02.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:01 smithi130 bash[27579]: audit 2023-08-28T02:39:01.785295+0000 mon.a (mon.0) 5288 : audit [DBG] from='client.? 172.21.15.6:0/2478120351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:02.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:01 smithi130 bash[51302]: debug 2023-08-28T02:39:01.831+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:01.835122+0000) 2023-08-28T02:39:02.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:01 smithi006 bash[27292]: audit 2023-08-28T02:39:01.785295+0000 mon.a (mon.0) 5288 : audit [DBG] from='client.? 172.21.15.6:0/2478120351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:02.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:01 smithi006 bash[33918]: audit 2023-08-28T02:39:01.785295+0000 mon.a (mon.0) 5288 : audit [DBG] from='client.? 172.21.15.6:0/2478120351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:03.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:02 smithi130 bash[27579]: cluster 2023-08-28T02:39:01.949751+0000 mgr.y (mgr.14901) 8412 : cluster [DBG] pgmap v8419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:03.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:02 smithi130 bash[51302]: debug 2023-08-28T02:39:02.831+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:02.835363+0000) 2023-08-28T02:39:03.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:02 smithi006 bash[27292]: cluster 2023-08-28T02:39:01.949751+0000 mgr.y (mgr.14901) 8412 : cluster [DBG] pgmap v8419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:03.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:02 smithi006 bash[33918]: cluster 2023-08-28T02:39:01.949751+0000 mgr.y (mgr.14901) 8412 : cluster [DBG] pgmap v8419: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:39:02.920Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:39:02.921Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:04.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:03 smithi130 bash[51302]: debug 2023-08-28T02:39:03.831+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:03.835663+0000) 2023-08-28T02:39:04.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:04 smithi130 bash[27579]: cluster 2023-08-28T02:39:03.950350+0000 mgr.y (mgr.14901) 8413 : cluster [DBG] pgmap v8420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:04.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:04 smithi130 bash[27579]: audit 2023-08-28T02:39:04.208654+0000 mon.c (mon.2) 3339 : audit [DBG] from='client.? 172.21.15.6:0/3902858814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:04.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:04 smithi006 bash[27292]: cluster 2023-08-28T02:39:03.950350+0000 mgr.y (mgr.14901) 8413 : cluster [DBG] pgmap v8420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:04.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:04 smithi006 bash[27292]: audit 2023-08-28T02:39:04.208654+0000 mon.c (mon.2) 3339 : audit [DBG] from='client.? 172.21.15.6:0/3902858814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:04.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:04 smithi006 bash[33918]: cluster 2023-08-28T02:39:03.950350+0000 mgr.y (mgr.14901) 8413 : cluster [DBG] pgmap v8420: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:04.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:04 smithi006 bash[33918]: audit 2023-08-28T02:39:04.208654+0000 mon.c (mon.2) 3339 : audit [DBG] from='client.? 172.21.15.6:0/3902858814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:05.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:04 smithi130 bash[51302]: debug 2023-08-28T02:39:04.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:04.835942+0000) 2023-08-28T02:39:05.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:39:04.798Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:05.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:39:04.800Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:06.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:05 smithi130 bash[51302]: debug 2023-08-28T02:39:05.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:05.836273+0000) 2023-08-28T02:39:07.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:07 smithi130 bash[27579]: cluster 2023-08-28T02:39:05.951462+0000 mgr.y (mgr.14901) 8414 : cluster [DBG] pgmap v8421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:07.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:07 smithi130 bash[27579]: audit 2023-08-28T02:39:06.626408+0000 mon.c (mon.2) 3340 : audit [DBG] from='client.? 172.21.15.6:0/3608093182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:07.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:06 smithi130 bash[51302]: debug 2023-08-28T02:39:06.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:06.836555+0000) 2023-08-28T02:39:07.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:07 smithi006 bash[27292]: cluster 2023-08-28T02:39:05.951462+0000 mgr.y (mgr.14901) 8414 : cluster [DBG] pgmap v8421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:07.466 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:07 smithi006 bash[27292]: audit 2023-08-28T02:39:06.626408+0000 mon.c (mon.2) 3340 : audit [DBG] from='client.? 172.21.15.6:0/3608093182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:07.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:07 smithi006 bash[33918]: cluster 2023-08-28T02:39:05.951462+0000 mgr.y (mgr.14901) 8414 : cluster [DBG] pgmap v8421: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:07.467 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:07 smithi006 bash[33918]: audit 2023-08-28T02:39:06.626408+0000 mon.c (mon.2) 3340 : audit [DBG] from='client.? 172.21.15.6:0/3608093182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:08.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:08 smithi130 bash[27579]: audit 2023-08-28T02:39:07.036619+0000 mon.b (mon.1) 912 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:39:08.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:08 smithi130 bash[27579]: cluster 2023-08-28T02:39:07.952129+0000 mgr.y (mgr.14901) 8415 : cluster [DBG] pgmap v8422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:08.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:07 smithi130 bash[51302]: debug 2023-08-28T02:39:07.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:07.836820+0000) 2023-08-28T02:39:08.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:08 smithi006 bash[27292]: audit 2023-08-28T02:39:07.036619+0000 mon.b (mon.1) 912 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:39:08.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:08 smithi006 bash[27292]: cluster 2023-08-28T02:39:07.952129+0000 mgr.y (mgr.14901) 8415 : cluster [DBG] pgmap v8422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:08.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:08 smithi006 bash[33918]: audit 2023-08-28T02:39:07.036619+0000 mon.b (mon.1) 912 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:39:08.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:08 smithi006 bash[33918]: cluster 2023-08-28T02:39:07.952129+0000 mgr.y (mgr.14901) 8415 : cluster [DBG] pgmap v8422: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:09.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:08 smithi130 bash[51302]: debug 2023-08-28T02:39:08.831+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:08.837130+0000) 2023-08-28T02:39:09.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:09 smithi130 bash[27579]: audit 2023-08-28T02:39:09.038490+0000 mon.c (mon.2) 3341 : audit [DBG] from='client.? 172.21.15.6:0/617823627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:09.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:09 smithi006 bash[27292]: audit 2023-08-28T02:39:09.038490+0000 mon.c (mon.2) 3341 : audit [DBG] from='client.? 172.21.15.6:0/617823627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:09.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:09 smithi006 bash[33918]: audit 2023-08-28T02:39:09.038490+0000 mon.c (mon.2) 3341 : audit [DBG] from='client.? 172.21.15.6:0/617823627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:10.119 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:09 smithi130 bash[51302]: debug 2023-08-28T02:39:09.831+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:09.837379+0000) 2023-08-28T02:39:10.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:10 smithi130 bash[27579]: cluster 2023-08-28T02:39:09.953287+0000 mgr.y (mgr.14901) 8416 : cluster [DBG] pgmap v8423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:10.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:10 smithi006 bash[27292]: cluster 2023-08-28T02:39:09.953287+0000 mgr.y (mgr.14901) 8416 : cluster [DBG] pgmap v8423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:10.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:10 smithi006 bash[33918]: cluster 2023-08-28T02:39:09.953287+0000 mgr.y (mgr.14901) 8416 : cluster [DBG] pgmap v8423: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:11.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:10 smithi130 bash[51302]: debug 2023-08-28T02:39:10.831+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:10.837725+0000) 2023-08-28T02:39:11.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:39:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:39:10] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:39:11.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:11 smithi006 bash[27292]: audit 2023-08-28T02:39:11.467397+0000 mon.c (mon.2) 3342 : audit [DBG] from='client.? 172.21.15.6:0/967997527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:11.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:11 smithi006 bash[33918]: audit 2023-08-28T02:39:11.467397+0000 mon.c (mon.2) 3342 : audit [DBG] from='client.? 172.21.15.6:0/967997527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:39:11.356Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:39:11.357Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:11.837 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:11 smithi130 bash[27579]: audit 2023-08-28T02:39:11.467397+0000 mon.c (mon.2) 3342 : audit [DBG] from='client.? 172.21.15.6:0/967997527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:12.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:11 smithi130 bash[51302]: debug 2023-08-28T02:39:11.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:11.838074+0000) 2023-08-28T02:39:12.837 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:12 smithi130 bash[27579]: cluster 2023-08-28T02:39:11.953863+0000 mgr.y (mgr.14901) 8417 : cluster [DBG] pgmap v8424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:12.920 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:12 smithi006 bash[33918]: cluster 2023-08-28T02:39:11.953863+0000 mgr.y (mgr.14901) 8417 : cluster [DBG] pgmap v8424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:12.920 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:12 smithi006 bash[27292]: cluster 2023-08-28T02:39:11.953863+0000 mgr.y (mgr.14901) 8417 : cluster [DBG] pgmap v8424: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:13.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:12 smithi130 bash[51302]: debug 2023-08-28T02:39:12.831+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:12.838292+0000) 2023-08-28T02:39:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:39:12.920Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:13.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:39:12.922Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:14.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:13 smithi130 bash[51302]: debug 2023-08-28T02:39:13.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:13.838574+0000) 2023-08-28T02:39:14.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:13 smithi130 bash[27579]: audit 2023-08-28T02:39:12.918149+0000 mon.a (mon.0) 5289 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:39:14.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:13 smithi130 bash[27579]: audit 2023-08-28T02:39:13.363383+0000 mon.a (mon.0) 5290 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:39:14.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:13 smithi130 bash[27579]: audit 2023-08-28T02:39:13.659428+0000 mon.b (mon.1) 913 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:39:14.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:13 smithi130 bash[27579]: audit 2023-08-28T02:39:13.661381+0000 mon.b (mon.1) 914 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:39:14.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:13 smithi130 bash[27579]: audit 2023-08-28T02:39:13.669444+0000 mon.a (mon.0) 5291 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:39:14.117 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:13 smithi130 bash[27579]: audit 2023-08-28T02:39:13.888056+0000 mon.a (mon.0) 5292 : audit [DBG] from='client.? 172.21.15.6:0/1459735797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:14.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:13 smithi006 bash[33918]: audit 2023-08-28T02:39:12.918149+0000 mon.a (mon.0) 5289 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:39:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:13 smithi006 bash[33918]: audit 2023-08-28T02:39:13.363383+0000 mon.a (mon.0) 5290 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:39:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:13 smithi006 bash[33918]: audit 2023-08-28T02:39:13.659428+0000 mon.b (mon.1) 913 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:39:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:13 smithi006 bash[33918]: audit 2023-08-28T02:39:13.661381+0000 mon.b (mon.1) 914 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:39:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:13 smithi006 bash[33918]: audit 2023-08-28T02:39:13.669444+0000 mon.a (mon.0) 5291 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:39:14.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:13 smithi006 bash[33918]: audit 2023-08-28T02:39:13.888056+0000 mon.a (mon.0) 5292 : audit [DBG] from='client.? 172.21.15.6:0/1459735797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:13 smithi006 bash[27292]: audit 2023-08-28T02:39:12.918149+0000 mon.a (mon.0) 5289 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:39:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:13 smithi006 bash[27292]: audit 2023-08-28T02:39:13.363383+0000 mon.a (mon.0) 5290 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:39:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:13 smithi006 bash[27292]: audit 2023-08-28T02:39:13.659428+0000 mon.b (mon.1) 913 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:39:14.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:13 smithi006 bash[27292]: audit 2023-08-28T02:39:13.661381+0000 mon.b (mon.1) 914 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:39:14.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:13 smithi006 bash[27292]: audit 2023-08-28T02:39:13.669444+0000 mon.a (mon.0) 5291 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:39:14.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:13 smithi006 bash[27292]: audit 2023-08-28T02:39:13.888056+0000 mon.a (mon.0) 5292 : audit [DBG] from='client.? 172.21.15.6:0/1459735797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:15.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:14 smithi130 bash[27579]: cluster 2023-08-28T02:39:13.954472+0000 mgr.y (mgr.14901) 8418 : cluster [DBG] pgmap v8425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:15.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:14 smithi130 bash[51302]: debug 2023-08-28T02:39:14.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:14.838874+0000) 2023-08-28T02:39:15.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:14 smithi006 bash[33918]: cluster 2023-08-28T02:39:13.954472+0000 mgr.y (mgr.14901) 8418 : cluster [DBG] pgmap v8425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:15.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:14 smithi006 bash[27292]: cluster 2023-08-28T02:39:13.954472+0000 mgr.y (mgr.14901) 8418 : cluster [DBG] pgmap v8425: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:15.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:39:14.799Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:15.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:39:14.801Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:16.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:16 smithi130 bash[27579]: cluster 2023-08-28T02:39:15.955490+0000 mgr.y (mgr.14901) 8419 : cluster [DBG] pgmap v8426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:16.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:15 smithi130 bash[51302]: debug 2023-08-28T02:39:15.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:15.839185+0000) 2023-08-28T02:39:16.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:16 smithi006 bash[27292]: cluster 2023-08-28T02:39:15.955490+0000 mgr.y (mgr.14901) 8419 : cluster [DBG] pgmap v8426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:16.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:16 smithi006 bash[33918]: cluster 2023-08-28T02:39:15.955490+0000 mgr.y (mgr.14901) 8419 : cluster [DBG] pgmap v8426: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:17.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:17 smithi130 bash[27579]: audit 2023-08-28T02:39:16.305598+0000 mon.c (mon.2) 3343 : audit [DBG] from='client.? 172.21.15.6:0/2345558100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:17.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:16 smithi130 bash[51302]: debug 2023-08-28T02:39:16.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:16.839463+0000) 2023-08-28T02:39:17.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:17 smithi006 bash[33918]: audit 2023-08-28T02:39:16.305598+0000 mon.c (mon.2) 3343 : audit [DBG] from='client.? 172.21.15.6:0/2345558100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:17.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:17 smithi006 bash[27292]: audit 2023-08-28T02:39:16.305598+0000 mon.c (mon.2) 3343 : audit [DBG] from='client.? 172.21.15.6:0/2345558100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:18.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:18 smithi130 bash[27579]: cluster 2023-08-28T02:39:17.956270+0000 mgr.y (mgr.14901) 8420 : cluster [DBG] pgmap v8427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:18.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:17 smithi130 bash[51302]: debug 2023-08-28T02:39:17.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:17.839765+0000) 2023-08-28T02:39:18.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:18 smithi006 bash[27292]: cluster 2023-08-28T02:39:17.956270+0000 mgr.y (mgr.14901) 8420 : cluster [DBG] pgmap v8427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:18.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:18 smithi006 bash[33918]: cluster 2023-08-28T02:39:17.956270+0000 mgr.y (mgr.14901) 8420 : cluster [DBG] pgmap v8427: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:19.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:18 smithi130 bash[51302]: debug 2023-08-28T02:39:18.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:18.840033+0000) 2023-08-28T02:39:19.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:19 smithi130 bash[27579]: audit 2023-08-28T02:39:18.725279+0000 mon.c (mon.2) 3344 : audit [DBG] from='client.? 172.21.15.6:0/979241269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:19.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:19 smithi006 bash[33918]: audit 2023-08-28T02:39:18.725279+0000 mon.c (mon.2) 3344 : audit [DBG] from='client.? 172.21.15.6:0/979241269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:19.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:19 smithi006 bash[27292]: audit 2023-08-28T02:39:18.725279+0000 mon.c (mon.2) 3344 : audit [DBG] from='client.? 172.21.15.6:0/979241269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:19 smithi130 bash[51302]: debug 2023-08-28T02:39:19.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:19.840311+0000) 2023-08-28T02:39:20.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:20 smithi130 bash[27579]: cluster 2023-08-28T02:39:19.957488+0000 mgr.y (mgr.14901) 8421 : cluster [DBG] pgmap v8428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:20.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:20 smithi006 bash[33918]: cluster 2023-08-28T02:39:19.957488+0000 mgr.y (mgr.14901) 8421 : cluster [DBG] pgmap v8428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:20.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:20 smithi006 bash[27292]: cluster 2023-08-28T02:39:19.957488+0000 mgr.y (mgr.14901) 8421 : cluster [DBG] pgmap v8428: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:21.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:20 smithi130 bash[51302]: debug 2023-08-28T02:39:20.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:20.840588+0000) 2023-08-28T02:39:21.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:39:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:39:20] "GET /metrics HTTP/1.1" 200 193121 "" "Prometheus/2.33.4" 2023-08-28T02:39:21.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:21 smithi130 bash[27579]: audit 2023-08-28T02:39:21.136209+0000 mon.c (mon.2) 3345 : audit [DBG] from='client.? 172.21.15.6:0/1821150966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:21.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:21 smithi006 bash[33918]: audit 2023-08-28T02:39:21.136209+0000 mon.c (mon.2) 3345 : audit [DBG] from='client.? 172.21.15.6:0/1821150966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:21.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:21 smithi006 bash[27292]: audit 2023-08-28T02:39:21.136209+0000 mon.c (mon.2) 3345 : audit [DBG] from='client.? 172.21.15.6:0/1821150966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:39:21.356Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:21.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:39:21.359Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:22.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:21 smithi130 bash[51302]: debug 2023-08-28T02:39:21.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:21.840907+0000) 2023-08-28T02:39:22.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:22 smithi130 bash[27579]: cluster 2023-08-28T02:39:21.958177+0000 mgr.y (mgr.14901) 8422 : cluster [DBG] pgmap v8429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:22.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:22 smithi006 bash[33918]: cluster 2023-08-28T02:39:21.958177+0000 mgr.y (mgr.14901) 8422 : cluster [DBG] pgmap v8429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:22.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:22 smithi006 bash[27292]: cluster 2023-08-28T02:39:21.958177+0000 mgr.y (mgr.14901) 8422 : cluster [DBG] pgmap v8429: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:23.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:22 smithi130 bash[51302]: debug 2023-08-28T02:39:22.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:22.841181+0000) 2023-08-28T02:39:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:39:22.921Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:39:22.923Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:24.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:23 smithi130 bash[51302]: debug 2023-08-28T02:39:23.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:23.841464+0000) 2023-08-28T02:39:24.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:24 smithi130 bash[27579]: audit 2023-08-28T02:39:23.557074+0000 mon.c (mon.2) 3346 : audit [DBG] from='client.? 172.21.15.6:0/2728979940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:24.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:24 smithi130 bash[27579]: cluster 2023-08-28T02:39:23.958793+0000 mgr.y (mgr.14901) 8423 : cluster [DBG] pgmap v8430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:24.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:24 smithi006 bash[33918]: audit 2023-08-28T02:39:23.557074+0000 mon.c (mon.2) 3346 : audit [DBG] from='client.? 172.21.15.6:0/2728979940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:24.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:24 smithi006 bash[33918]: cluster 2023-08-28T02:39:23.958793+0000 mgr.y (mgr.14901) 8423 : cluster [DBG] pgmap v8430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:24.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:24 smithi006 bash[27292]: audit 2023-08-28T02:39:23.557074+0000 mon.c (mon.2) 3346 : audit [DBG] from='client.? 172.21.15.6:0/2728979940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:24.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:24 smithi006 bash[27292]: cluster 2023-08-28T02:39:23.958793+0000 mgr.y (mgr.14901) 8423 : cluster [DBG] pgmap v8430: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:25.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:24 smithi130 bash[51302]: debug 2023-08-28T02:39:24.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:24.841749+0000) 2023-08-28T02:39:25.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:39:24.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:25.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:39:24.802Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:26.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:25 smithi130 bash[51302]: debug 2023-08-28T02:39:25.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:25.842028+0000) 2023-08-28T02:39:26.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:26 smithi130 bash[27579]: cluster 2023-08-28T02:39:25.959851+0000 mgr.y (mgr.14901) 8424 : cluster [DBG] pgmap v8431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:26.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:26 smithi130 bash[27579]: audit 2023-08-28T02:39:25.971615+0000 mon.c (mon.2) 3347 : audit [DBG] from='client.? 172.21.15.6:0/475789731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:26.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:26 smithi006 bash[33918]: cluster 2023-08-28T02:39:25.959851+0000 mgr.y (mgr.14901) 8424 : cluster [DBG] pgmap v8431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:26.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:26 smithi006 bash[33918]: audit 2023-08-28T02:39:25.971615+0000 mon.c (mon.2) 3347 : audit [DBG] from='client.? 172.21.15.6:0/475789731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:26.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:26 smithi006 bash[27292]: cluster 2023-08-28T02:39:25.959851+0000 mgr.y (mgr.14901) 8424 : cluster [DBG] pgmap v8431: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:26.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:26 smithi006 bash[27292]: audit 2023-08-28T02:39:25.971615+0000 mon.c (mon.2) 3347 : audit [DBG] from='client.? 172.21.15.6:0/475789731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:27.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:26 smithi130 bash[51302]: debug 2023-08-28T02:39:26.835+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:26.842306+0000) 2023-08-28T02:39:28.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:28 smithi130 bash[27579]: cluster 2023-08-28T02:39:27.960499+0000 mgr.y (mgr.14901) 8425 : cluster [DBG] pgmap v8432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:28.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:27 smithi130 bash[51302]: debug 2023-08-28T02:39:27.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:27.842552+0000) 2023-08-28T02:39:28.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:28 smithi006 bash[33918]: cluster 2023-08-28T02:39:27.960499+0000 mgr.y (mgr.14901) 8425 : cluster [DBG] pgmap v8432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:28.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:28 smithi006 bash[27292]: cluster 2023-08-28T02:39:27.960499+0000 mgr.y (mgr.14901) 8425 : cluster [DBG] pgmap v8432: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:28 smithi130 bash[51302]: debug 2023-08-28T02:39:28.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:28.842925+0000) 2023-08-28T02:39:29.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:29 smithi130 bash[27579]: audit 2023-08-28T02:39:28.388497+0000 mon.c (mon.2) 3348 : audit [DBG] from='client.? 172.21.15.6:0/3636307602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:29.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:29 smithi006 bash[33918]: audit 2023-08-28T02:39:28.388497+0000 mon.c (mon.2) 3348 : audit [DBG] from='client.? 172.21.15.6:0/3636307602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:29.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:29 smithi006 bash[27292]: audit 2023-08-28T02:39:28.388497+0000 mon.c (mon.2) 3348 : audit [DBG] from='client.? 172.21.15.6:0/3636307602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:30.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:29 smithi130 bash[51302]: debug 2023-08-28T02:39:29.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:29.843149+0000) 2023-08-28T02:39:30.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:30 smithi130 bash[27579]: cluster 2023-08-28T02:39:29.961651+0000 mgr.y (mgr.14901) 8426 : cluster [DBG] pgmap v8433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:30 smithi006 bash[33918]: cluster 2023-08-28T02:39:29.961651+0000 mgr.y (mgr.14901) 8426 : cluster [DBG] pgmap v8433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:30.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:30 smithi006 bash[27292]: cluster 2023-08-28T02:39:29.961651+0000 mgr.y (mgr.14901) 8426 : cluster [DBG] pgmap v8433: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:31.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:30 smithi130 bash[51302]: debug 2023-08-28T02:39:30.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:30.843453+0000) 2023-08-28T02:39:31.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:39:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:39:30] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:39:31.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:31 smithi130 bash[27579]: audit 2023-08-28T02:39:30.808077+0000 mon.a (mon.0) 5293 : audit [DBG] from='client.? 172.21.15.6:0/505444311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:31.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:31 smithi006 bash[33918]: audit 2023-08-28T02:39:30.808077+0000 mon.a (mon.0) 5293 : audit [DBG] from='client.? 172.21.15.6:0/505444311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:31.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:31 smithi006 bash[27292]: audit 2023-08-28T02:39:30.808077+0000 mon.a (mon.0) 5293 : audit [DBG] from='client.? 172.21.15.6:0/505444311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:39:31.357Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:39:31.359Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:32.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:31 smithi130 bash[51302]: debug 2023-08-28T02:39:31.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:31.843736+0000) 2023-08-28T02:39:32.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:32 smithi130 bash[27579]: cluster 2023-08-28T02:39:31.962286+0000 mgr.y (mgr.14901) 8427 : cluster [DBG] pgmap v8434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:32.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:32 smithi006 bash[33918]: cluster 2023-08-28T02:39:31.962286+0000 mgr.y (mgr.14901) 8427 : cluster [DBG] pgmap v8434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:32.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:32 smithi006 bash[27292]: cluster 2023-08-28T02:39:31.962286+0000 mgr.y (mgr.14901) 8427 : cluster [DBG] pgmap v8434: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:33.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:32 smithi130 bash[51302]: debug 2023-08-28T02:39:32.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:32.844028+0000) 2023-08-28T02:39:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:39:32.922Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:33.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:39:32.924Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:33.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:33 smithi130 bash[27579]: audit 2023-08-28T02:39:33.222242+0000 mon.a (mon.0) 5294 : audit [DBG] from='client.? 172.21.15.6:0/532533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:33.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:33 smithi006 bash[33918]: audit 2023-08-28T02:39:33.222242+0000 mon.a (mon.0) 5294 : audit [DBG] from='client.? 172.21.15.6:0/532533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:33.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:33 smithi006 bash[27292]: audit 2023-08-28T02:39:33.222242+0000 mon.a (mon.0) 5294 : audit [DBG] from='client.? 172.21.15.6:0/532533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:34.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:33 smithi130 bash[51302]: debug 2023-08-28T02:39:33.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:33.844311+0000) 2023-08-28T02:39:34.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:34 smithi130 bash[27579]: cluster 2023-08-28T02:39:33.962928+0000 mgr.y (mgr.14901) 8428 : cluster [DBG] pgmap v8435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:34.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:34 smithi006 bash[27292]: cluster 2023-08-28T02:39:33.962928+0000 mgr.y (mgr.14901) 8428 : cluster [DBG] pgmap v8435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:34.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:34 smithi006 bash[33918]: cluster 2023-08-28T02:39:33.962928+0000 mgr.y (mgr.14901) 8428 : cluster [DBG] pgmap v8435: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:35.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:34 smithi130 bash[51302]: debug 2023-08-28T02:39:34.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:34.844646+0000) 2023-08-28T02:39:35.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:39:34.801Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:35.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:39:34.802Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:35.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:35 smithi006 bash[33918]: audit 2023-08-28T02:39:35.634401+0000 mon.c (mon.2) 3349 : audit [DBG] from='client.? 172.21.15.6:0/3400287144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:35.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:35 smithi006 bash[27292]: audit 2023-08-28T02:39:35.634401+0000 mon.c (mon.2) 3349 : audit [DBG] from='client.? 172.21.15.6:0/3400287144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:36.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:35 smithi130 bash[27579]: audit 2023-08-28T02:39:35.634401+0000 mon.c (mon.2) 3349 : audit [DBG] from='client.? 172.21.15.6:0/3400287144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:36.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:35 smithi130 bash[51302]: debug 2023-08-28T02:39:35.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:35.844953+0000) 2023-08-28T02:39:36.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:36 smithi006 bash[27292]: cluster 2023-08-28T02:39:35.963994+0000 mgr.y (mgr.14901) 8429 : cluster [DBG] pgmap v8436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:36.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:36 smithi006 bash[33918]: cluster 2023-08-28T02:39:35.963994+0000 mgr.y (mgr.14901) 8429 : cluster [DBG] pgmap v8436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:37.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:36 smithi130 bash[27579]: cluster 2023-08-28T02:39:35.963994+0000 mgr.y (mgr.14901) 8429 : cluster [DBG] pgmap v8436: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:37.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:36 smithi130 bash[51302]: debug 2023-08-28T02:39:36.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:36.845175+0000) 2023-08-28T02:39:38.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:37 smithi130 bash[51302]: debug 2023-08-28T02:39:37.843+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:37.845415+0000) 2023-08-28T02:39:38.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:38 smithi130 bash[27579]: cluster 2023-08-28T02:39:37.964623+0000 mgr.y (mgr.14901) 8430 : cluster [DBG] pgmap v8437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:38.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:38 smithi130 bash[27579]: audit 2023-08-28T02:39:38.046971+0000 mon.c (mon.2) 3350 : audit [DBG] from='client.? 172.21.15.6:0/72201365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:38.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:38 smithi006 bash[27292]: cluster 2023-08-28T02:39:37.964623+0000 mgr.y (mgr.14901) 8430 : cluster [DBG] pgmap v8437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:38.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:38 smithi006 bash[27292]: audit 2023-08-28T02:39:38.046971+0000 mon.c (mon.2) 3350 : audit [DBG] from='client.? 172.21.15.6:0/72201365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:38.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:38 smithi006 bash[33918]: cluster 2023-08-28T02:39:37.964623+0000 mgr.y (mgr.14901) 8430 : cluster [DBG] pgmap v8437: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:38.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:38 smithi006 bash[33918]: audit 2023-08-28T02:39:38.046971+0000 mon.c (mon.2) 3350 : audit [DBG] from='client.? 172.21.15.6:0/72201365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:39.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:38 smithi130 bash[51302]: debug 2023-08-28T02:39:38.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:38.845665+0000) 2023-08-28T02:39:40.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:40 smithi130 bash[27579]: cluster 2023-08-28T02:39:39.965722+0000 mgr.y (mgr.14901) 8431 : cluster [DBG] pgmap v8438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:40.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:39 smithi130 bash[51302]: debug 2023-08-28T02:39:39.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:39.845947+0000) 2023-08-28T02:39:40.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:40 smithi006 bash[33918]: cluster 2023-08-28T02:39:39.965722+0000 mgr.y (mgr.14901) 8431 : cluster [DBG] pgmap v8438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:40.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:40 smithi006 bash[27292]: cluster 2023-08-28T02:39:39.965722+0000 mgr.y (mgr.14901) 8431 : cluster [DBG] pgmap v8438: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:41.067 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:39:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:39:40] "GET /metrics HTTP/1.1" 200 193119 "" "Prometheus/2.33.4" 2023-08-28T02:39:41.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:41 smithi130 bash[27579]: audit 2023-08-28T02:39:40.468095+0000 mon.a (mon.0) 5295 : audit [DBG] from='client.? 172.21.15.6:0/1363080453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:41.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:40 smithi130 bash[51302]: debug 2023-08-28T02:39:40.839+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:40.846249+0000) 2023-08-28T02:39:41.358 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:41 smithi006 bash[27292]: audit 2023-08-28T02:39:40.468095+0000 mon.a (mon.0) 5295 : audit [DBG] from='client.? 172.21.15.6:0/1363080453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:41.358 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:41 smithi006 bash[33918]: audit 2023-08-28T02:39:40.468095+0000 mon.a (mon.0) 5295 : audit [DBG] from='client.? 172.21.15.6:0/1363080453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:41.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:39:41.358Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:39:41.360Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:42.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:42 smithi130 bash[27579]: cluster 2023-08-28T02:39:41.966400+0000 mgr.y (mgr.14901) 8432 : cluster [DBG] pgmap v8439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:42.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:41 smithi130 bash[51302]: debug 2023-08-28T02:39:41.843+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:41.846584+0000) 2023-08-28T02:39:42.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:42 smithi006 bash[33918]: cluster 2023-08-28T02:39:41.966400+0000 mgr.y (mgr.14901) 8432 : cluster [DBG] pgmap v8439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:42.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:42 smithi006 bash[27292]: cluster 2023-08-28T02:39:41.966400+0000 mgr.y (mgr.14901) 8432 : cluster [DBG] pgmap v8439: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:43.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:43 smithi130 bash[27579]: audit 2023-08-28T02:39:42.882250+0000 mon.c (mon.2) 3351 : audit [DBG] from='client.? 172.21.15.6:0/3635518407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:43.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:42 smithi130 bash[51302]: debug 2023-08-28T02:39:42.843+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:42.846847+0000) 2023-08-28T02:39:43.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:43 smithi006 bash[33918]: audit 2023-08-28T02:39:42.882250+0000 mon.c (mon.2) 3351 : audit [DBG] from='client.? 172.21.15.6:0/3635518407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:39:42.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:39:42.925Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:43.216 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:43 smithi006 bash[27292]: audit 2023-08-28T02:39:42.882250+0000 mon.c (mon.2) 3351 : audit [DBG] from='client.? 172.21.15.6:0/3635518407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:44.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:43 smithi130 bash[51302]: debug 2023-08-28T02:39:43.847+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:43.847167+0000) 2023-08-28T02:39:44.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:44 smithi130 bash[27579]: cluster 2023-08-28T02:39:43.967045+0000 mgr.y (mgr.14901) 8433 : cluster [DBG] pgmap v8440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:44.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:44 smithi006 bash[33918]: cluster 2023-08-28T02:39:43.967045+0000 mgr.y (mgr.14901) 8433 : cluster [DBG] pgmap v8440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:44.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:44 smithi006 bash[27292]: cluster 2023-08-28T02:39:43.967045+0000 mgr.y (mgr.14901) 8433 : cluster [DBG] pgmap v8440: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:45.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:44 smithi130 bash[51302]: debug 2023-08-28T02:39:44.846+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:44.847529+0000) 2023-08-28T02:39:45.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:39:44.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:45.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:39:44.804Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:45.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:45 smithi130 bash[27579]: audit 2023-08-28T02:39:45.299469+0000 mon.a (mon.0) 5296 : audit [DBG] from='client.? 172.21.15.6:0/3561113725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:45.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:45 smithi006 bash[33918]: audit 2023-08-28T02:39:45.299469+0000 mon.a (mon.0) 5296 : audit [DBG] from='client.? 172.21.15.6:0/3561113725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:45.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:45 smithi006 bash[27292]: audit 2023-08-28T02:39:45.299469+0000 mon.a (mon.0) 5296 : audit [DBG] from='client.? 172.21.15.6:0/3561113725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:46.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:45 smithi130 bash[51302]: debug 2023-08-28T02:39:45.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:45.847832+0000) 2023-08-28T02:39:46.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:46 smithi130 bash[27579]: cluster 2023-08-28T02:39:45.968144+0000 mgr.y (mgr.14901) 8434 : cluster [DBG] pgmap v8441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:46.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:46 smithi006 bash[33918]: cluster 2023-08-28T02:39:45.968144+0000 mgr.y (mgr.14901) 8434 : cluster [DBG] pgmap v8441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:46.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:46 smithi006 bash[27292]: cluster 2023-08-28T02:39:45.968144+0000 mgr.y (mgr.14901) 8434 : cluster [DBG] pgmap v8441: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:47.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:46 smithi130 bash[51302]: debug 2023-08-28T02:39:46.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:46.848120+0000) 2023-08-28T02:39:48.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:47 smithi130 bash[27579]: audit 2023-08-28T02:39:47.711647+0000 mon.c (mon.2) 3352 : audit [DBG] from='client.? 172.21.15.6:0/4226025870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:48.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:47 smithi130 bash[51302]: debug 2023-08-28T02:39:47.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:47.848438+0000) 2023-08-28T02:39:48.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:47 smithi006 bash[33918]: audit 2023-08-28T02:39:47.711647+0000 mon.c (mon.2) 3352 : audit [DBG] from='client.? 172.21.15.6:0/4226025870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:48.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:47 smithi006 bash[27292]: audit 2023-08-28T02:39:47.711647+0000 mon.c (mon.2) 3352 : audit [DBG] from='client.? 172.21.15.6:0/4226025870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:49.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:48 smithi130 bash[27579]: cluster 2023-08-28T02:39:47.968816+0000 mgr.y (mgr.14901) 8435 : cluster [DBG] pgmap v8442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:49.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:48 smithi130 bash[51302]: debug 2023-08-28T02:39:48.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:48.848658+0000) 2023-08-28T02:39:49.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:48 smithi006 bash[27292]: cluster 2023-08-28T02:39:47.968816+0000 mgr.y (mgr.14901) 8435 : cluster [DBG] pgmap v8442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:49.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:48 smithi006 bash[33918]: cluster 2023-08-28T02:39:47.968816+0000 mgr.y (mgr.14901) 8435 : cluster [DBG] pgmap v8442: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:50.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:50 smithi130 bash[27579]: cluster 2023-08-28T02:39:49.969881+0000 mgr.y (mgr.14901) 8436 : cluster [DBG] pgmap v8443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:50.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:49 smithi130 bash[51302]: debug 2023-08-28T02:39:49.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:49.848870+0000) 2023-08-28T02:39:50.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:50 smithi006 bash[27292]: cluster 2023-08-28T02:39:49.969881+0000 mgr.y (mgr.14901) 8436 : cluster [DBG] pgmap v8443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:50.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:50 smithi006 bash[33918]: cluster 2023-08-28T02:39:49.969881+0000 mgr.y (mgr.14901) 8436 : cluster [DBG] pgmap v8443: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:51.071 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:39:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:39:50] "GET /metrics HTTP/1.1" 200 193118 "" "Prometheus/2.33.4" 2023-08-28T02:39:51.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:51 smithi130 bash[27579]: audit 2023-08-28T02:39:50.125161+0000 mon.c (mon.2) 3353 : audit [DBG] from='client.? 172.21.15.6:0/738593444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:51.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:50 smithi130 bash[51302]: debug 2023-08-28T02:39:50.842+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:50.849143+0000) 2023-08-28T02:39:51.358 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:51 smithi006 bash[33918]: audit 2023-08-28T02:39:50.125161+0000 mon.c (mon.2) 3353 : audit [DBG] from='client.? 172.21.15.6:0/738593444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:51.359 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:51 smithi006 bash[27292]: audit 2023-08-28T02:39:50.125161+0000 mon.c (mon.2) 3353 : audit [DBG] from='client.? 172.21.15.6:0/738593444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:39:51.358Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:39:51.360Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:52.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:52 smithi130 bash[27579]: cluster 2023-08-28T02:39:51.970518+0000 mgr.y (mgr.14901) 8437 : cluster [DBG] pgmap v8444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:52.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:51 smithi130 bash[51302]: debug 2023-08-28T02:39:51.846+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:51.849462+0000) 2023-08-28T02:39:52.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:52 smithi006 bash[33918]: cluster 2023-08-28T02:39:51.970518+0000 mgr.y (mgr.14901) 8437 : cluster [DBG] pgmap v8444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:52.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:52 smithi006 bash[27292]: cluster 2023-08-28T02:39:51.970518+0000 mgr.y (mgr.14901) 8437 : cluster [DBG] pgmap v8444: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:53.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:53 smithi130 bash[27579]: audit 2023-08-28T02:39:52.548219+0000 mon.a (mon.0) 5297 : audit [DBG] from='client.? 172.21.15.6:0/3854219643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:53.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:52 smithi130 bash[51302]: debug 2023-08-28T02:39:52.846+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:52.849703+0000) 2023-08-28T02:39:53.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:53 smithi006 bash[33918]: audit 2023-08-28T02:39:52.548219+0000 mon.a (mon.0) 5297 : audit [DBG] from='client.? 172.21.15.6:0/3854219643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:53.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:53 smithi006 bash[27292]: audit 2023-08-28T02:39:52.548219+0000 mon.a (mon.0) 5297 : audit [DBG] from='client.? 172.21.15.6:0/3854219643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:39:52.924Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:39:52.926Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:54.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:53 smithi130 bash[51302]: debug 2023-08-28T02:39:53.846+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:53.849926+0000) 2023-08-28T02:39:54.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:54 smithi130 bash[27579]: cluster 2023-08-28T02:39:53.971178+0000 mgr.y (mgr.14901) 8438 : cluster [DBG] pgmap v8445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:54.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:54 smithi006 bash[33918]: cluster 2023-08-28T02:39:53.971178+0000 mgr.y (mgr.14901) 8438 : cluster [DBG] pgmap v8445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:54.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:54 smithi006 bash[27292]: cluster 2023-08-28T02:39:53.971178+0000 mgr.y (mgr.14901) 8438 : cluster [DBG] pgmap v8445: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:55.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:54 smithi130 bash[51302]: debug 2023-08-28T02:39:54.846+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:54.850241+0000) 2023-08-28T02:39:55.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:39:54.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:55.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:39:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:39:54.804Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:39:55.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:55 smithi130 bash[27579]: audit 2023-08-28T02:39:54.964208+0000 mon.c (mon.2) 3354 : audit [DBG] from='client.? 172.21.15.6:0/4073482121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:55.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:55 smithi006 bash[27292]: audit 2023-08-28T02:39:54.964208+0000 mon.c (mon.2) 3354 : audit [DBG] from='client.? 172.21.15.6:0/4073482121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:55.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:55 smithi006 bash[33918]: audit 2023-08-28T02:39:54.964208+0000 mon.c (mon.2) 3354 : audit [DBG] from='client.? 172.21.15.6:0/4073482121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:56.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:55 smithi130 bash[51302]: debug 2023-08-28T02:39:55.846+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:55.850641+0000) 2023-08-28T02:39:56.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:56 smithi130 bash[27579]: cluster 2023-08-28T02:39:55.972402+0000 mgr.y (mgr.14901) 8439 : cluster [DBG] pgmap v8446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:56.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:56 smithi006 bash[33918]: cluster 2023-08-28T02:39:55.972402+0000 mgr.y (mgr.14901) 8439 : cluster [DBG] pgmap v8446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:56.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:56 smithi006 bash[27292]: cluster 2023-08-28T02:39:55.972402+0000 mgr.y (mgr.14901) 8439 : cluster [DBG] pgmap v8446: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:57.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:56 smithi130 bash[51302]: debug 2023-08-28T02:39:56.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:56.850994+0000) 2023-08-28T02:39:57.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:57 smithi006 bash[33918]: audit 2023-08-28T02:39:57.371449+0000 mon.c (mon.2) 3355 : audit [DBG] from='client.? 172.21.15.6:0/3024104416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:57.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:57 smithi006 bash[27292]: audit 2023-08-28T02:39:57.371449+0000 mon.c (mon.2) 3355 : audit [DBG] from='client.? 172.21.15.6:0/3024104416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:57.850 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:57 smithi130 bash[27579]: audit 2023-08-28T02:39:57.371449+0000 mon.c (mon.2) 3355 : audit [DBG] from='client.? 172.21.15.6:0/3024104416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:58.117 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:57 smithi130 bash[51302]: debug 2023-08-28T02:39:57.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:57.851326+0000) 2023-08-28T02:39:58.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:58 smithi006 bash[27292]: cluster 2023-08-28T02:39:57.973148+0000 mgr.y (mgr.14901) 8440 : cluster [DBG] pgmap v8447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:58.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:58 smithi006 bash[33918]: cluster 2023-08-28T02:39:57.973148+0000 mgr.y (mgr.14901) 8440 : cluster [DBG] pgmap v8447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:58.850 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:58 smithi130 bash[27579]: cluster 2023-08-28T02:39:57.973148+0000 mgr.y (mgr.14901) 8440 : cluster [DBG] pgmap v8447: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:39:59.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:58 smithi130 bash[51302]: debug 2023-08-28T02:39:58.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:58.851599+0000) 2023-08-28T02:40:00.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:39:59 smithi130 bash[51302]: debug 2023-08-28T02:39:59.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:59.851841+0000) 2023-08-28T02:40:00.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:39:59 smithi130 bash[27579]: audit 2023-08-28T02:39:59.793420+0000 mon.a (mon.0) 5298 : audit [DBG] from='client.? 172.21.15.6:0/3499283739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:00.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:39:59 smithi006 bash[33918]: audit 2023-08-28T02:39:59.793420+0000 mon.a (mon.0) 5298 : audit [DBG] from='client.? 172.21.15.6:0/3499283739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:00.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:39:59 smithi006 bash[27292]: audit 2023-08-28T02:39:59.793420+0000 mon.a (mon.0) 5298 : audit [DBG] from='client.? 172.21.15.6:0/3499283739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:01.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:00 smithi130 bash[51302]: debug 2023-08-28T02:40:00.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:00.852151+0000) 2023-08-28T02:40:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:00 smithi130 bash[27579]: cluster 2023-08-28T02:39:59.974295+0000 mgr.y (mgr.14901) 8441 : cluster [DBG] pgmap v8448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:01.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:00 smithi130 bash[27579]: cluster 2023-08-28T02:40:00.000153+0000 mon.a (mon.0) 5299 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:40:01.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:40:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:40:00] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T02:40:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:00 smithi006 bash[27292]: cluster 2023-08-28T02:39:59.974295+0000 mgr.y (mgr.14901) 8441 : cluster [DBG] pgmap v8448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:01.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:00 smithi006 bash[27292]: cluster 2023-08-28T02:40:00.000153+0000 mon.a (mon.0) 5299 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:40:01.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:00 smithi006 bash[33918]: cluster 2023-08-28T02:39:59.974295+0000 mgr.y (mgr.14901) 8441 : cluster [DBG] pgmap v8448: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:01.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:00 smithi006 bash[33918]: cluster 2023-08-28T02:40:00.000153+0000 mon.a (mon.0) 5299 : cluster [WRN] overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:40:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:40:01.359Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:40:01.361Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:02.115 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:02 smithi130 bash[27579]: cluster 2023-08-28T02:40:01.974929+0000 mgr.y (mgr.14901) 8442 : cluster [DBG] pgmap v8449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:02.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:01 smithi130 bash[51302]: debug 2023-08-28T02:40:01.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:01.852488+0000) 2023-08-28T02:40:02.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:02 smithi006 bash[33918]: cluster 2023-08-28T02:40:01.974929+0000 mgr.y (mgr.14901) 8442 : cluster [DBG] pgmap v8449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:02.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:02 smithi006 bash[27292]: cluster 2023-08-28T02:40:01.974929+0000 mgr.y (mgr.14901) 8442 : cluster [DBG] pgmap v8449: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:03.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:03 smithi130 bash[27579]: audit 2023-08-28T02:40:02.214745+0000 mon.a (mon.0) 5300 : audit [DBG] from='client.? 172.21.15.6:0/1911341095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:03.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:02 smithi130 bash[51302]: debug 2023-08-28T02:40:02.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:02.852861+0000) 2023-08-28T02:40:03.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:03 smithi006 bash[33918]: audit 2023-08-28T02:40:02.214745+0000 mon.a (mon.0) 5300 : audit [DBG] from='client.? 172.21.15.6:0/1911341095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:03.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:03 smithi006 bash[27292]: audit 2023-08-28T02:40:02.214745+0000 mon.a (mon.0) 5300 : audit [DBG] from='client.? 172.21.15.6:0/1911341095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:40:02.925Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:40:02.927Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:04.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:03 smithi130 bash[51302]: debug 2023-08-28T02:40:03.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:03.853175+0000) 2023-08-28T02:40:04.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:04 smithi130 bash[27579]: cluster 2023-08-28T02:40:03.975572+0000 mgr.y (mgr.14901) 8443 : cluster [DBG] pgmap v8450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:04.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:04 smithi006 bash[33918]: cluster 2023-08-28T02:40:03.975572+0000 mgr.y (mgr.14901) 8443 : cluster [DBG] pgmap v8450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:04.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:04 smithi006 bash[27292]: cluster 2023-08-28T02:40:03.975572+0000 mgr.y (mgr.14901) 8443 : cluster [DBG] pgmap v8450: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:05.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:04 smithi130 bash[51302]: debug 2023-08-28T02:40:04.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:04.853507+0000) 2023-08-28T02:40:05.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:40:04.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:05.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:40:04.805Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:05.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:05 smithi130 bash[27579]: audit 2023-08-28T02:40:04.623038+0000 mon.a (mon.0) 5301 : audit [DBG] from='client.? 172.21.15.6:0/2681668465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:05.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:05 smithi006 bash[27292]: audit 2023-08-28T02:40:04.623038+0000 mon.a (mon.0) 5301 : audit [DBG] from='client.? 172.21.15.6:0/2681668465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:05.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:05 smithi006 bash[33918]: audit 2023-08-28T02:40:04.623038+0000 mon.a (mon.0) 5301 : audit [DBG] from='client.? 172.21.15.6:0/2681668465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:06.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:05 smithi130 bash[51302]: debug 2023-08-28T02:40:05.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:05.853840+0000) 2023-08-28T02:40:06.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:06 smithi130 bash[27579]: cluster 2023-08-28T02:40:05.976726+0000 mgr.y (mgr.14901) 8444 : cluster [DBG] pgmap v8451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:06.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:06 smithi006 bash[27292]: cluster 2023-08-28T02:40:05.976726+0000 mgr.y (mgr.14901) 8444 : cluster [DBG] pgmap v8451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:06.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:06 smithi006 bash[33918]: cluster 2023-08-28T02:40:05.976726+0000 mgr.y (mgr.14901) 8444 : cluster [DBG] pgmap v8451: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:07.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:06 smithi130 bash[51302]: debug 2023-08-28T02:40:06.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:06.854176+0000) 2023-08-28T02:40:07.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:07 smithi130 bash[27579]: audit 2023-08-28T02:40:07.041365+0000 mon.a (mon.0) 5302 : audit [DBG] from='client.? 172.21.15.6:0/4067313999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:07.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:07 smithi006 bash[33918]: audit 2023-08-28T02:40:07.041365+0000 mon.a (mon.0) 5302 : audit [DBG] from='client.? 172.21.15.6:0/4067313999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:07.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:07 smithi006 bash[27292]: audit 2023-08-28T02:40:07.041365+0000 mon.a (mon.0) 5302 : audit [DBG] from='client.? 172.21.15.6:0/4067313999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:08.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:07 smithi130 bash[51302]: debug 2023-08-28T02:40:07.850+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:07.854497+0000) 2023-08-28T02:40:08.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:08 smithi130 bash[27579]: cluster 2023-08-28T02:40:07.977350+0000 mgr.y (mgr.14901) 8445 : cluster [DBG] pgmap v8452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:08.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:08 smithi006 bash[33918]: cluster 2023-08-28T02:40:07.977350+0000 mgr.y (mgr.14901) 8445 : cluster [DBG] pgmap v8452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:08.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:08 smithi006 bash[27292]: cluster 2023-08-28T02:40:07.977350+0000 mgr.y (mgr.14901) 8445 : cluster [DBG] pgmap v8452: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:09.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:08 smithi130 bash[51302]: debug 2023-08-28T02:40:08.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:08.854837+0000) 2023-08-28T02:40:09.854 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:09 smithi130 bash[27579]: audit 2023-08-28T02:40:09.458876+0000 mon.a (mon.0) 5303 : audit [DBG] from='client.? 172.21.15.6:0/4000852201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:09.964 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:09 smithi006 bash[27292]: audit 2023-08-28T02:40:09.458876+0000 mon.a (mon.0) 5303 : audit [DBG] from='client.? 172.21.15.6:0/4000852201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:09.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:09 smithi006 bash[33918]: audit 2023-08-28T02:40:09.458876+0000 mon.a (mon.0) 5303 : audit [DBG] from='client.? 172.21.15.6:0/4000852201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:10.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:09 smithi130 bash[51302]: debug 2023-08-28T02:40:09.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:09.855185+0000) 2023-08-28T02:40:10.854 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:10 smithi130 bash[27579]: cluster 2023-08-28T02:40:09.978441+0000 mgr.y (mgr.14901) 8446 : cluster [DBG] pgmap v8453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:10.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:40:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:40:10] "GET /metrics HTTP/1.1" 200 193122 "" "Prometheus/2.33.4" 2023-08-28T02:40:10.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:10 smithi006 bash[27292]: cluster 2023-08-28T02:40:09.978441+0000 mgr.y (mgr.14901) 8446 : cluster [DBG] pgmap v8453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:10.965 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:10 smithi006 bash[33918]: cluster 2023-08-28T02:40:09.978441+0000 mgr.y (mgr.14901) 8446 : cluster [DBG] pgmap v8453: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:11.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:10 smithi130 bash[51302]: debug 2023-08-28T02:40:10.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:10.855574+0000) 2023-08-28T02:40:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:40:11.360Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:40:11.362Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:12.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:11 smithi130 bash[51302]: debug 2023-08-28T02:40:11.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:11.855928+0000) 2023-08-28T02:40:12.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:11 smithi130 bash[27579]: audit 2023-08-28T02:40:11.877038+0000 mon.a (mon.0) 5304 : audit [DBG] from='client.? 172.21.15.6:0/4237149761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:12.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:11 smithi006 bash[33918]: audit 2023-08-28T02:40:11.877038+0000 mon.a (mon.0) 5304 : audit [DBG] from='client.? 172.21.15.6:0/4237149761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:12.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:11 smithi006 bash[27292]: audit 2023-08-28T02:40:11.877038+0000 mon.a (mon.0) 5304 : audit [DBG] from='client.? 172.21.15.6:0/4237149761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:13.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:12 smithi130 bash[27579]: cluster 2023-08-28T02:40:11.979058+0000 mgr.y (mgr.14901) 8447 : cluster [DBG] pgmap v8454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:13.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:12 smithi130 bash[51302]: debug 2023-08-28T02:40:12.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:12.856281+0000) 2023-08-28T02:40:13.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:12 smithi006 bash[27292]: cluster 2023-08-28T02:40:11.979058+0000 mgr.y (mgr.14901) 8447 : cluster [DBG] pgmap v8454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:12 smithi006 bash[33918]: cluster 2023-08-28T02:40:11.979058+0000 mgr.y (mgr.14901) 8447 : cluster [DBG] pgmap v8454: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:40:12.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:40:12.927Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:14.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:13 smithi130 bash[51302]: debug 2023-08-28T02:40:13.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:13.856584+0000) 2023-08-28T02:40:14.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:14 smithi130 bash[27579]: audit 2023-08-28T02:40:13.681645+0000 mon.b (mon.1) 915 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:40:14.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:14 smithi130 bash[27579]: cluster 2023-08-28T02:40:13.979725+0000 mgr.y (mgr.14901) 8448 : cluster [DBG] pgmap v8455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:14.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:14 smithi006 bash[33918]: audit 2023-08-28T02:40:13.681645+0000 mon.b (mon.1) 915 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:40:14.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:14 smithi006 bash[33918]: cluster 2023-08-28T02:40:13.979725+0000 mgr.y (mgr.14901) 8448 : cluster [DBG] pgmap v8455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:14.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:14 smithi006 bash[27292]: audit 2023-08-28T02:40:13.681645+0000 mon.b (mon.1) 915 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:40:14.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:14 smithi006 bash[27292]: cluster 2023-08-28T02:40:13.979725+0000 mgr.y (mgr.14901) 8448 : cluster [DBG] pgmap v8455: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:15.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:14 smithi130 bash[51302]: debug 2023-08-28T02:40:14.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:14.856924+0000) 2023-08-28T02:40:15.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:14 smithi006 bash[56298]: level=error ts=2023-08-28T02:40:14.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:15.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:14 smithi006 bash[56298]: level=warn ts=2023-08-28T02:40:14.806Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:15.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:15 smithi130 bash[27579]: audit 2023-08-28T02:40:14.293325+0000 mon.c (mon.2) 3356 : audit [DBG] from='client.? 172.21.15.6:0/3946711336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:15.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:15 smithi006 bash[27292]: audit 2023-08-28T02:40:14.293325+0000 mon.c (mon.2) 3356 : audit [DBG] from='client.? 172.21.15.6:0/3946711336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:15.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:15 smithi006 bash[33918]: audit 2023-08-28T02:40:14.293325+0000 mon.c (mon.2) 3356 : audit [DBG] from='client.? 172.21.15.6:0/3946711336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:16.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:15 smithi130 bash[51302]: debug 2023-08-28T02:40:15.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:15.857277+0000) 2023-08-28T02:40:16.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:16 smithi130 bash[27579]: cluster 2023-08-28T02:40:15.980923+0000 mgr.y (mgr.14901) 8449 : cluster [DBG] pgmap v8456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:16.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:16 smithi006 bash[27292]: cluster 2023-08-28T02:40:15.980923+0000 mgr.y (mgr.14901) 8449 : cluster [DBG] pgmap v8456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:16.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:16 smithi006 bash[33918]: cluster 2023-08-28T02:40:15.980923+0000 mgr.y (mgr.14901) 8449 : cluster [DBG] pgmap v8456: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:17.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:16 smithi130 bash[51302]: debug 2023-08-28T02:40:16.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:16.857615+0000) 2023-08-28T02:40:17.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:17 smithi130 bash[27579]: audit 2023-08-28T02:40:16.716212+0000 mon.a (mon.0) 5305 : audit [DBG] from='client.? 172.21.15.6:0/419167584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:17.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:17 smithi006 bash[33918]: audit 2023-08-28T02:40:16.716212+0000 mon.a (mon.0) 5305 : audit [DBG] from='client.? 172.21.15.6:0/419167584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:17.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:17 smithi006 bash[27292]: audit 2023-08-28T02:40:16.716212+0000 mon.a (mon.0) 5305 : audit [DBG] from='client.? 172.21.15.6:0/419167584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:18.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:17 smithi130 bash[51302]: debug 2023-08-28T02:40:17.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:17.857924+0000) 2023-08-28T02:40:18.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:18 smithi130 bash[27579]: cluster 2023-08-28T02:40:17.981562+0000 mgr.y (mgr.14901) 8450 : cluster [DBG] pgmap v8457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:18.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:18 smithi006 bash[33918]: cluster 2023-08-28T02:40:17.981562+0000 mgr.y (mgr.14901) 8450 : cluster [DBG] pgmap v8457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:18.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:18 smithi006 bash[27292]: cluster 2023-08-28T02:40:17.981562+0000 mgr.y (mgr.14901) 8450 : cluster [DBG] pgmap v8457: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:19.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:18 smithi130 bash[51302]: debug 2023-08-28T02:40:18.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:18.858204+0000) 2023-08-28T02:40:19.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:19 smithi130 bash[27579]: audit 2023-08-28T02:40:19.136267+0000 mon.a (mon.0) 5306 : audit [DBG] from='client.? 172.21.15.6:0/308671387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:19.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:19 smithi006 bash[33918]: audit 2023-08-28T02:40:19.136267+0000 mon.a (mon.0) 5306 : audit [DBG] from='client.? 172.21.15.6:0/308671387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:19.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:19 smithi006 bash[27292]: audit 2023-08-28T02:40:19.136267+0000 mon.a (mon.0) 5306 : audit [DBG] from='client.? 172.21.15.6:0/308671387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:20.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:19 smithi130 bash[51302]: debug 2023-08-28T02:40:19.854+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:19.858413+0000) 2023-08-28T02:40:20.858 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:20 smithi130 bash[27579]: audit 2023-08-28T02:40:19.554573+0000 mon.a (mon.0) 5307 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:40:20.858 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:20 smithi130 bash[27579]: cluster 2023-08-28T02:40:19.982759+0000 mgr.y (mgr.14901) 8451 : cluster [DBG] pgmap v8458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:20.858 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:20 smithi130 bash[27579]: audit 2023-08-28T02:40:20.003176+0000 mon.a (mon.0) 5308 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:40:20.858 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:20 smithi130 bash[27579]: audit 2023-08-28T02:40:20.291365+0000 mon.b (mon.1) 916 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:40:20.858 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:20 smithi130 bash[27579]: audit 2023-08-28T02:40:20.293307+0000 mon.b (mon.1) 917 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:40:20.858 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:20 smithi130 bash[27579]: audit 2023-08-28T02:40:20.302101+0000 mon.a (mon.0) 5309 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:40:20.964 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:40:20 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:40:20] "GET /metrics HTTP/1.1" 200 193117 "" "Prometheus/2.33.4" 2023-08-28T02:40:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:20 smithi006 bash[27292]: audit 2023-08-28T02:40:19.554573+0000 mon.a (mon.0) 5307 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:40:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:20 smithi006 bash[27292]: cluster 2023-08-28T02:40:19.982759+0000 mgr.y (mgr.14901) 8451 : cluster [DBG] pgmap v8458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:20 smithi006 bash[27292]: audit 2023-08-28T02:40:20.003176+0000 mon.a (mon.0) 5308 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:40:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:20 smithi006 bash[27292]: audit 2023-08-28T02:40:20.291365+0000 mon.b (mon.1) 916 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:40:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:20 smithi006 bash[27292]: audit 2023-08-28T02:40:20.293307+0000 mon.b (mon.1) 917 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:40:20.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:20 smithi006 bash[27292]: audit 2023-08-28T02:40:20.302101+0000 mon.a (mon.0) 5309 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:40:20.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:20 smithi006 bash[33918]: audit 2023-08-28T02:40:19.554573+0000 mon.a (mon.0) 5307 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:40:20.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:20 smithi006 bash[33918]: cluster 2023-08-28T02:40:19.982759+0000 mgr.y (mgr.14901) 8451 : cluster [DBG] pgmap v8458: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:20.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:20 smithi006 bash[33918]: audit 2023-08-28T02:40:20.003176+0000 mon.a (mon.0) 5308 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:40:20.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:20 smithi006 bash[33918]: audit 2023-08-28T02:40:20.291365+0000 mon.b (mon.1) 916 : audit [DBG] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:40:20.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:20 smithi006 bash[33918]: audit 2023-08-28T02:40:20.293307+0000 mon.b (mon.1) 917 : audit [INF] from='mgr.14901 172.21.15.6:0/4062329202' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:40:20.966 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:20 smithi006 bash[33918]: audit 2023-08-28T02:40:20.302101+0000 mon.a (mon.0) 5309 : audit [INF] from='mgr.14901 ' entity='mgr.y' 2023-08-28T02:40:21.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:20 smithi130 bash[51302]: debug 2023-08-28T02:40:20.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:20.858713+0000) 2023-08-28T02:40:21.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:21 smithi006 bash[33918]: audit 2023-08-28T02:40:21.554980+0000 mon.c (mon.2) 3357 : audit [DBG] from='client.? 172.21.15.6:0/1655889047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:21.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:21 smithi006 bash[27292]: audit 2023-08-28T02:40:21.554980+0000 mon.c (mon.2) 3357 : audit [DBG] from='client.? 172.21.15.6:0/1655889047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:21 smithi006 bash[56298]: level=error ts=2023-08-28T02:40:21.361Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:21.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:21 smithi006 bash[56298]: level=warn ts=2023-08-28T02:40:21.362Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:21.858 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:21 smithi130 bash[27579]: audit 2023-08-28T02:40:21.554980+0000 mon.c (mon.2) 3357 : audit [DBG] from='client.? 172.21.15.6:0/1655889047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:22.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:21 smithi130 bash[51302]: debug 2023-08-28T02:40:21.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:21.859037+0000) 2023-08-28T02:40:22.858 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:22 smithi130 bash[27579]: cluster 2023-08-28T02:40:21.983441+0000 mgr.y (mgr.14901) 8452 : cluster [DBG] pgmap v8459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:22.926 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:22 smithi006 bash[27292]: cluster 2023-08-28T02:40:21.983441+0000 mgr.y (mgr.14901) 8452 : cluster [DBG] pgmap v8459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:22.927 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:22 smithi006 bash[33918]: cluster 2023-08-28T02:40:21.983441+0000 mgr.y (mgr.14901) 8452 : cluster [DBG] pgmap v8459: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:23.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:22 smithi130 bash[51302]: debug 2023-08-28T02:40:22.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:22.859368+0000) 2023-08-28T02:40:23.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:22 smithi006 bash[56298]: level=error ts=2023-08-28T02:40:22.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:23.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:22 smithi006 bash[56298]: level=warn ts=2023-08-28T02:40:22.928Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:24.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:23 smithi130 bash[51302]: debug 2023-08-28T02:40:23.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:23.859701+0000) 2023-08-28T02:40:24.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:24 smithi130 bash[27579]: audit 2023-08-28T02:40:23.977125+0000 mon.c (mon.2) 3358 : audit [DBG] from='client.? 172.21.15.6:0/89725548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:24.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:24 smithi130 bash[27579]: cluster 2023-08-28T02:40:23.983918+0000 mgr.y (mgr.14901) 8453 : cluster [DBG] pgmap v8460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:24.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:24 smithi006 bash[33918]: audit 2023-08-28T02:40:23.977125+0000 mon.c (mon.2) 3358 : audit [DBG] from='client.? 172.21.15.6:0/89725548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:24.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:24 smithi006 bash[33918]: cluster 2023-08-28T02:40:23.983918+0000 mgr.y (mgr.14901) 8453 : cluster [DBG] pgmap v8460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:24.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:24 smithi006 bash[27292]: audit 2023-08-28T02:40:23.977125+0000 mon.c (mon.2) 3358 : audit [DBG] from='client.? 172.21.15.6:0/89725548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:24.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:24 smithi006 bash[27292]: cluster 2023-08-28T02:40:23.983918+0000 mgr.y (mgr.14901) 8453 : cluster [DBG] pgmap v8460: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:25.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:24 smithi130 bash[51302]: debug 2023-08-28T02:40:24.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:24.860036+0000) 2023-08-28T02:40:25.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:24 smithi006 bash[56298]: level=error ts=2023-08-28T02:40:24.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:25.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:24 smithi006 bash[56298]: level=warn ts=2023-08-28T02:40:24.807Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:26.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:26 smithi130 bash[27579]: cluster 2023-08-28T02:40:25.985013+0000 mgr.y (mgr.14901) 8454 : cluster [DBG] pgmap v8461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:26.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:25 smithi130 bash[51302]: debug 2023-08-28T02:40:25.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:25.860349+0000) 2023-08-28T02:40:26.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:26 smithi006 bash[27292]: cluster 2023-08-28T02:40:25.985013+0000 mgr.y (mgr.14901) 8454 : cluster [DBG] pgmap v8461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:26.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:26 smithi006 bash[33918]: cluster 2023-08-28T02:40:25.985013+0000 mgr.y (mgr.14901) 8454 : cluster [DBG] pgmap v8461: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:27.118 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:27 smithi130 bash[27579]: audit 2023-08-28T02:40:26.391352+0000 mon.a (mon.0) 5310 : audit [DBG] from='client.? 172.21.15.6:0/2487991609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:27.118 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:26 smithi130 bash[51302]: debug 2023-08-28T02:40:26.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:26.860634+0000) 2023-08-28T02:40:27.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:27 smithi006 bash[27292]: audit 2023-08-28T02:40:26.391352+0000 mon.a (mon.0) 5310 : audit [DBG] from='client.? 172.21.15.6:0/2487991609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:27.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:27 smithi006 bash[33918]: audit 2023-08-28T02:40:26.391352+0000 mon.a (mon.0) 5310 : audit [DBG] from='client.? 172.21.15.6:0/2487991609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:28.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:27 smithi130 bash[51302]: debug 2023-08-28T02:40:27.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:27.860922+0000) 2023-08-28T02:40:28.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:28 smithi130 bash[27579]: cluster 2023-08-28T02:40:27.985683+0000 mgr.y (mgr.14901) 8455 : cluster [DBG] pgmap v8462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:28.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:28 smithi006 bash[27292]: cluster 2023-08-28T02:40:27.985683+0000 mgr.y (mgr.14901) 8455 : cluster [DBG] pgmap v8462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:28.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:28 smithi006 bash[33918]: cluster 2023-08-28T02:40:27.985683+0000 mgr.y (mgr.14901) 8455 : cluster [DBG] pgmap v8462: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:29.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:28 smithi130 bash[51302]: debug 2023-08-28T02:40:28.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:28.861207+0000) 2023-08-28T02:40:29.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:29 smithi130 bash[27579]: audit 2023-08-28T02:40:28.803841+0000 mon.a (mon.0) 5311 : audit [DBG] from='client.? 172.21.15.6:0/4196456205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:29.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:29 smithi006 bash[27292]: audit 2023-08-28T02:40:28.803841+0000 mon.a (mon.0) 5311 : audit [DBG] from='client.? 172.21.15.6:0/4196456205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:29.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:29 smithi006 bash[33918]: audit 2023-08-28T02:40:28.803841+0000 mon.a (mon.0) 5311 : audit [DBG] from='client.? 172.21.15.6:0/4196456205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:30.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:29 smithi130 bash[51302]: debug 2023-08-28T02:40:29.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:29.861563+0000) 2023-08-28T02:40:30.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:30 smithi130 bash[27579]: cluster 2023-08-28T02:40:29.986825+0000 mgr.y (mgr.14901) 8456 : cluster [DBG] pgmap v8463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:30.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:30 smithi006 bash[33918]: cluster 2023-08-28T02:40:29.986825+0000 mgr.y (mgr.14901) 8456 : cluster [DBG] pgmap v8463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:30.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:30 smithi006 bash[27292]: cluster 2023-08-28T02:40:29.986825+0000 mgr.y (mgr.14901) 8456 : cluster [DBG] pgmap v8463: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:31.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:30 smithi130 bash[51302]: debug 2023-08-28T02:40:30.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:30.861846+0000) 2023-08-28T02:40:31.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:40:30 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:40:30] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T02:40:31.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:31 smithi130 bash[27579]: audit 2023-08-28T02:40:31.228939+0000 mon.a (mon.0) 5312 : audit [DBG] from='client.? 172.21.15.6:0/1761018986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:31.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:31 smithi006 bash[33918]: audit 2023-08-28T02:40:31.228939+0000 mon.a (mon.0) 5312 : audit [DBG] from='client.? 172.21.15.6:0/1761018986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:31.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:31 smithi006 bash[27292]: audit 2023-08-28T02:40:31.228939+0000 mon.a (mon.0) 5312 : audit [DBG] from='client.? 172.21.15.6:0/1761018986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:31.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:31 smithi006 bash[56298]: level=error ts=2023-08-28T02:40:31.361Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:31.716 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:31 smithi006 bash[56298]: level=warn ts=2023-08-28T02:40:31.363Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:32.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:31 smithi130 bash[51302]: debug 2023-08-28T02:40:31.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:31.862081+0000) 2023-08-28T02:40:32.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:32 smithi130 bash[27579]: cluster 2023-08-28T02:40:31.987467+0000 mgr.y (mgr.14901) 8457 : cluster [DBG] pgmap v8464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:32.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:32 smithi006 bash[33918]: cluster 2023-08-28T02:40:31.987467+0000 mgr.y (mgr.14901) 8457 : cluster [DBG] pgmap v8464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:32.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:32 smithi006 bash[27292]: cluster 2023-08-28T02:40:31.987467+0000 mgr.y (mgr.14901) 8457 : cluster [DBG] pgmap v8464: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:33.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:32 smithi130 bash[51302]: debug 2023-08-28T02:40:32.862+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:32.862383+0000) 2023-08-28T02:40:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:32 smithi006 bash[56298]: level=error ts=2023-08-28T02:40:32.927Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:33.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:32 smithi006 bash[56298]: level=warn ts=2023-08-28T02:40:32.928Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:34.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:33 smithi130 bash[51302]: debug 2023-08-28T02:40:33.858+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:33.862672+0000) 2023-08-28T02:40:34.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:34 smithi130 bash[27579]: audit 2023-08-28T02:40:33.640293+0000 mon.c (mon.2) 3359 : audit [DBG] from='client.? 172.21.15.6:0/4167915264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:34.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:34 smithi130 bash[27579]: cluster 2023-08-28T02:40:33.988137+0000 mgr.y (mgr.14901) 8458 : cluster [DBG] pgmap v8465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:34.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:34 smithi006 bash[33918]: audit 2023-08-28T02:40:33.640293+0000 mon.c (mon.2) 3359 : audit [DBG] from='client.? 172.21.15.6:0/4167915264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:34.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:34 smithi006 bash[33918]: cluster 2023-08-28T02:40:33.988137+0000 mgr.y (mgr.14901) 8458 : cluster [DBG] pgmap v8465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:34.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:34 smithi006 bash[27292]: audit 2023-08-28T02:40:33.640293+0000 mon.c (mon.2) 3359 : audit [DBG] from='client.? 172.21.15.6:0/4167915264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:34.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:34 smithi006 bash[27292]: cluster 2023-08-28T02:40:33.988137+0000 mgr.y (mgr.14901) 8458 : cluster [DBG] pgmap v8465: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:35.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:34 smithi130 bash[51302]: debug 2023-08-28T02:40:34.862+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:34.862986+0000) 2023-08-28T02:40:35.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:34 smithi006 bash[56298]: level=error ts=2023-08-28T02:40:34.806Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:35.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:34 smithi006 bash[56298]: level=warn ts=2023-08-28T02:40:34.809Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:36.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:36 smithi130 bash[27579]: cluster 2023-08-28T02:40:35.989224+0000 mgr.y (mgr.14901) 8459 : cluster [DBG] pgmap v8466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:36.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:36 smithi130 bash[27579]: audit 2023-08-28T02:40:36.053163+0000 mon.a (mon.0) 5313 : audit [DBG] from='client.? 172.21.15.6:0/3460069798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:36.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:35 smithi130 bash[51302]: debug 2023-08-28T02:40:35.862+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:35.863256+0000) 2023-08-28T02:40:36.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:36 smithi006 bash[27292]: cluster 2023-08-28T02:40:35.989224+0000 mgr.y (mgr.14901) 8459 : cluster [DBG] pgmap v8466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:36.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:36 smithi006 bash[27292]: audit 2023-08-28T02:40:36.053163+0000 mon.a (mon.0) 5313 : audit [DBG] from='client.? 172.21.15.6:0/3460069798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:36.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:36 smithi006 bash[33918]: cluster 2023-08-28T02:40:35.989224+0000 mgr.y (mgr.14901) 8459 : cluster [DBG] pgmap v8466: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:36.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:36 smithi006 bash[33918]: audit 2023-08-28T02:40:36.053163+0000 mon.a (mon.0) 5313 : audit [DBG] from='client.? 172.21.15.6:0/3460069798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:37.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:36 smithi130 bash[51302]: debug 2023-08-28T02:40:36.862+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:36.863520+0000) 2023-08-28T02:40:38.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:38 smithi130 bash[27579]: cluster 2023-08-28T02:40:37.989867+0000 mgr.y (mgr.14901) 8460 : cluster [DBG] pgmap v8467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:38.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:37 smithi130 bash[51302]: debug 2023-08-28T02:40:37.862+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:37.863841+0000) 2023-08-28T02:40:38.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:38 smithi006 bash[27292]: cluster 2023-08-28T02:40:37.989867+0000 mgr.y (mgr.14901) 8460 : cluster [DBG] pgmap v8467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:38.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:38 smithi006 bash[33918]: cluster 2023-08-28T02:40:37.989867+0000 mgr.y (mgr.14901) 8460 : cluster [DBG] pgmap v8467: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:39.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:38 smithi130 bash[51302]: debug 2023-08-28T02:40:38.862+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:38.864159+0000) 2023-08-28T02:40:39.615 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:39 smithi130 bash[27579]: audit 2023-08-28T02:40:38.477280+0000 mon.a (mon.0) 5314 : audit [DBG] from='client.? 172.21.15.6:0/2192856530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:39.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:39 smithi006 bash[27292]: audit 2023-08-28T02:40:38.477280+0000 mon.a (mon.0) 5314 : audit [DBG] from='client.? 172.21.15.6:0/2192856530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:39.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:39 smithi006 bash[33918]: audit 2023-08-28T02:40:38.477280+0000 mon.a (mon.0) 5314 : audit [DBG] from='client.? 172.21.15.6:0/2192856530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:40.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:39 smithi130 bash[51302]: debug 2023-08-28T02:40:39.861+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:39.864445+0000) 2023-08-28T02:40:40.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:40 smithi130 bash[27579]: cluster 2023-08-28T02:40:39.991099+0000 mgr.y (mgr.14901) 8461 : cluster [DBG] pgmap v8468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:40.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:40 smithi006 bash[27292]: cluster 2023-08-28T02:40:39.991099+0000 mgr.y (mgr.14901) 8461 : cluster [DBG] pgmap v8468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:40.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:40 smithi006 bash[33918]: cluster 2023-08-28T02:40:39.991099+0000 mgr.y (mgr.14901) 8461 : cluster [DBG] pgmap v8468: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:41.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:40 smithi130 bash[51302]: debug 2023-08-28T02:40:40.861+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:40.864738+0000) 2023-08-28T02:40:41.214 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:40:40 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:40:40] "GET /metrics HTTP/1.1" 200 193125 "" "Prometheus/2.33.4" 2023-08-28T02:40:41.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:41 smithi130 bash[27579]: audit 2023-08-28T02:40:40.891535+0000 mon.a (mon.0) 5315 : audit [DBG] from='client.? 172.21.15.6:0/223405357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:41.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:41 smithi006 bash[33918]: audit 2023-08-28T02:40:40.891535+0000 mon.a (mon.0) 5315 : audit [DBG] from='client.? 172.21.15.6:0/223405357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:41.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:41 smithi006 bash[27292]: audit 2023-08-28T02:40:40.891535+0000 mon.a (mon.0) 5315 : audit [DBG] from='client.? 172.21.15.6:0/223405357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:41 smithi006 bash[56298]: level=error ts=2023-08-28T02:40:41.362Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:41.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:41 smithi006 bash[56298]: level=warn ts=2023-08-28T02:40:41.365Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:42.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:41 smithi130 bash[51302]: debug 2023-08-28T02:40:41.861+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:41.865027+0000) 2023-08-28T02:40:42.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:42 smithi130 bash[27579]: cluster 2023-08-28T02:40:41.991820+0000 mgr.y (mgr.14901) 8462 : cluster [DBG] pgmap v8469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:42.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:42 smithi006 bash[27292]: cluster 2023-08-28T02:40:41.991820+0000 mgr.y (mgr.14901) 8462 : cluster [DBG] pgmap v8469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:42.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:42 smithi006 bash[33918]: cluster 2023-08-28T02:40:41.991820+0000 mgr.y (mgr.14901) 8462 : cluster [DBG] pgmap v8469: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:43.115 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:42 smithi130 bash[51302]: debug 2023-08-28T02:40:42.861+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:42.865311+0000) 2023-08-28T02:40:43.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:42 smithi006 bash[56298]: level=error ts=2023-08-28T02:40:42.927Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:43.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:42 smithi006 bash[56298]: level=warn ts=2023-08-28T02:40:42.929Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:44.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:43 smithi130 bash[51302]: debug 2023-08-28T02:40:43.861+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:43.865579+0000) 2023-08-28T02:40:44.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:44 smithi130 bash[27579]: audit 2023-08-28T02:40:43.305650+0000 mon.a (mon.0) 5316 : audit [DBG] from='client.? 172.21.15.6:0/1643644700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:44.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:44 smithi130 bash[27579]: cluster 2023-08-28T02:40:43.992479+0000 mgr.y (mgr.14901) 8463 : cluster [DBG] pgmap v8470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:44.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:44 smithi006 bash[27292]: audit 2023-08-28T02:40:43.305650+0000 mon.a (mon.0) 5316 : audit [DBG] from='client.? 172.21.15.6:0/1643644700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:44.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:44 smithi006 bash[27292]: cluster 2023-08-28T02:40:43.992479+0000 mgr.y (mgr.14901) 8463 : cluster [DBG] pgmap v8470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:44.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:44 smithi006 bash[33918]: audit 2023-08-28T02:40:43.305650+0000 mon.a (mon.0) 5316 : audit [DBG] from='client.? 172.21.15.6:0/1643644700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:44.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:44 smithi006 bash[33918]: cluster 2023-08-28T02:40:43.992479+0000 mgr.y (mgr.14901) 8463 : cluster [DBG] pgmap v8470: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:45.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:44 smithi130 bash[51302]: debug 2023-08-28T02:40:44.861+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:44.865943+0000) 2023-08-28T02:40:45.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:44 smithi006 bash[56298]: level=error ts=2023-08-28T02:40:44.806Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:45.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:44 smithi006 bash[56298]: level=warn ts=2023-08-28T02:40:44.808Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:46.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:45 smithi130 bash[27579]: audit 2023-08-28T02:40:45.720777+0000 mon.a (mon.0) 5317 : audit [DBG] from='client.? 172.21.15.6:0/2388625970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:46.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:45 smithi130 bash[51302]: debug 2023-08-28T02:40:45.861+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:45.866195+0000) 2023-08-28T02:40:46.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:45 smithi006 bash[33918]: audit 2023-08-28T02:40:45.720777+0000 mon.a (mon.0) 5317 : audit [DBG] from='client.? 172.21.15.6:0/2388625970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:46.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:45 smithi006 bash[27292]: audit 2023-08-28T02:40:45.720777+0000 mon.a (mon.0) 5317 : audit [DBG] from='client.? 172.21.15.6:0/2388625970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:47.116 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:46 smithi130 bash[27579]: cluster 2023-08-28T02:40:45.993639+0000 mgr.y (mgr.14901) 8464 : cluster [DBG] pgmap v8471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:47.116 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:46 smithi130 bash[51302]: debug 2023-08-28T02:40:46.861+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:46.866492+0000) 2023-08-28T02:40:47.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:46 smithi006 bash[27292]: cluster 2023-08-28T02:40:45.993639+0000 mgr.y (mgr.14901) 8464 : cluster [DBG] pgmap v8471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:47.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:46 smithi006 bash[33918]: cluster 2023-08-28T02:40:45.993639+0000 mgr.y (mgr.14901) 8464 : cluster [DBG] pgmap v8471: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:48.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:48 smithi130 bash[27579]: cluster 2023-08-28T02:40:47.994268+0000 mgr.y (mgr.14901) 8465 : cluster [DBG] pgmap v8472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:48.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:47 smithi130 bash[51302]: debug 2023-08-28T02:40:47.861+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:47.866762+0000) 2023-08-28T02:40:48.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:48 smithi006 bash[27292]: cluster 2023-08-28T02:40:47.994268+0000 mgr.y (mgr.14901) 8465 : cluster [DBG] pgmap v8472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:48.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:48 smithi006 bash[33918]: cluster 2023-08-28T02:40:47.994268+0000 mgr.y (mgr.14901) 8465 : cluster [DBG] pgmap v8472: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:49.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:49 smithi130 bash[27579]: audit 2023-08-28T02:40:48.134961+0000 mon.a (mon.0) 5318 : audit [DBG] from='client.? 172.21.15.6:0/1296775421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:49.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:48 smithi130 bash[51302]: debug 2023-08-28T02:40:48.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:48.867035+0000) 2023-08-28T02:40:49.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:49 smithi006 bash[27292]: audit 2023-08-28T02:40:48.134961+0000 mon.a (mon.0) 5318 : audit [DBG] from='client.? 172.21.15.6:0/1296775421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:49.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:49 smithi006 bash[33918]: audit 2023-08-28T02:40:48.134961+0000 mon.a (mon.0) 5318 : audit [DBG] from='client.? 172.21.15.6:0/1296775421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:50.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:50 smithi130 bash[27579]: cluster 2023-08-28T02:40:49.995375+0000 mgr.y (mgr.14901) 8466 : cluster [DBG] pgmap v8473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:50.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:49 smithi130 bash[51302]: debug 2023-08-28T02:40:49.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:49.867278+0000) 2023-08-28T02:40:50.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:50 smithi006 bash[27292]: cluster 2023-08-28T02:40:49.995375+0000 mgr.y (mgr.14901) 8466 : cluster [DBG] pgmap v8473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:50.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:50 smithi006 bash[33918]: cluster 2023-08-28T02:40:49.995375+0000 mgr.y (mgr.14901) 8466 : cluster [DBG] pgmap v8473: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:51.094 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:40:50 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:40:50] "GET /metrics HTTP/1.1" 200 193114 "" "Prometheus/2.33.4" 2023-08-28T02:40:51.364 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:51 smithi006 bash[27292]: audit 2023-08-28T02:40:50.561233+0000 mon.a (mon.0) 5319 : audit [DBG] from='client.? 172.21.15.6:0/3999094897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:51.364 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:51 smithi006 bash[33918]: audit 2023-08-28T02:40:50.561233+0000 mon.a (mon.0) 5319 : audit [DBG] from='client.? 172.21.15.6:0/3999094897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:51.365 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:51 smithi130 bash[27579]: audit 2023-08-28T02:40:50.561233+0000 mon.a (mon.0) 5319 : audit [DBG] from='client.? 172.21.15.6:0/3999094897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:51.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:50 smithi130 bash[51302]: debug 2023-08-28T02:40:50.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:50.867503+0000) 2023-08-28T02:40:51.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:51 smithi006 bash[56298]: level=error ts=2023-08-28T02:40:51.363Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:51.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:51 smithi006 bash[56298]: level=warn ts=2023-08-28T02:40:51.366Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:52.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:52 smithi130 bash[27579]: cluster 2023-08-28T02:40:51.996053+0000 mgr.y (mgr.14901) 8467 : cluster [DBG] pgmap v8474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:52.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:51 smithi130 bash[51302]: debug 2023-08-28T02:40:51.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:51.867816+0000) 2023-08-28T02:40:52.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:52 smithi006 bash[33918]: cluster 2023-08-28T02:40:51.996053+0000 mgr.y (mgr.14901) 8467 : cluster [DBG] pgmap v8474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:52.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:52 smithi006 bash[27292]: cluster 2023-08-28T02:40:51.996053+0000 mgr.y (mgr.14901) 8467 : cluster [DBG] pgmap v8474: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:53.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:53 smithi006 bash[33918]: audit 2023-08-28T02:40:52.974132+0000 mon.a (mon.0) 5320 : audit [DBG] from='client.? 172.21.15.6:0/1367012812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:53.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:53 smithi006 bash[27292]: audit 2023-08-28T02:40:52.974132+0000 mon.a (mon.0) 5320 : audit [DBG] from='client.? 172.21.15.6:0/1367012812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:52 smithi006 bash[56298]: level=error ts=2023-08-28T02:40:52.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:53.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:52 smithi006 bash[56298]: level=warn ts=2023-08-28T02:40:52.930Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:53.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:53 smithi130 bash[27579]: audit 2023-08-28T02:40:52.974132+0000 mon.a (mon.0) 5320 : audit [DBG] from='client.? 172.21.15.6:0/1367012812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:53.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:52 smithi130 bash[51302]: debug 2023-08-28T02:40:52.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:52.868076+0000) 2023-08-28T02:40:54.272 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:53 smithi130 bash[51302]: debug 2023-08-28T02:40:53.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:53.868372+0000) 2023-08-28T02:40:54.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:54 smithi130 bash[27579]: cluster 2023-08-28T02:40:53.996711+0000 mgr.y (mgr.14901) 8468 : cluster [DBG] pgmap v8475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:54.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:54 smithi006 bash[27292]: cluster 2023-08-28T02:40:53.996711+0000 mgr.y (mgr.14901) 8468 : cluster [DBG] pgmap v8475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:54.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:54 smithi006 bash[33918]: cluster 2023-08-28T02:40:53.996711+0000 mgr.y (mgr.14901) 8468 : cluster [DBG] pgmap v8475: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:55.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:54 smithi006 bash[56298]: level=error ts=2023-08-28T02:40:54.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:55.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:40:54 smithi006 bash[56298]: level=warn ts=2023-08-28T02:40:54.809Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:40:55.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:54 smithi130 bash[51302]: debug 2023-08-28T02:40:54.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:54.868669+0000) 2023-08-28T02:40:55.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:55 smithi006 bash[33918]: audit 2023-08-28T02:40:55.395292+0000 mon.a (mon.0) 5321 : audit [DBG] from='client.? 172.21.15.6:0/3443848306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:55.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:55 smithi006 bash[27292]: audit 2023-08-28T02:40:55.395292+0000 mon.a (mon.0) 5321 : audit [DBG] from='client.? 172.21.15.6:0/3443848306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:55.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:55 smithi130 bash[27579]: audit 2023-08-28T02:40:55.395292+0000 mon.a (mon.0) 5321 : audit [DBG] from='client.? 172.21.15.6:0/3443848306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:56.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:55 smithi130 bash[51302]: debug 2023-08-28T02:40:55.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:55.868960+0000) 2023-08-28T02:40:56.714 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:56 smithi006 bash[33918]: cluster 2023-08-28T02:40:55.997841+0000 mgr.y (mgr.14901) 8469 : cluster [DBG] pgmap v8476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:56.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:56 smithi006 bash[27292]: cluster 2023-08-28T02:40:55.997841+0000 mgr.y (mgr.14901) 8469 : cluster [DBG] pgmap v8476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:56.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:56 smithi130 bash[27579]: cluster 2023-08-28T02:40:55.997841+0000 mgr.y (mgr.14901) 8469 : cluster [DBG] pgmap v8476: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:57.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:56 smithi130 bash[51302]: debug 2023-08-28T02:40:56.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:56.869225+0000) 2023-08-28T02:40:58.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:57 smithi006 bash[33918]: audit 2023-08-28T02:40:57.817884+0000 mon.a (mon.0) 5322 : audit [DBG] from='client.? 172.21.15.6:0/2706321562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:58.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:57 smithi006 bash[27292]: audit 2023-08-28T02:40:57.817884+0000 mon.a (mon.0) 5322 : audit [DBG] from='client.? 172.21.15.6:0/2706321562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:58.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:57 smithi130 bash[27579]: audit 2023-08-28T02:40:57.817884+0000 mon.a (mon.0) 5322 : audit [DBG] from='client.? 172.21.15.6:0/2706321562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:58.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:57 smithi130 bash[51302]: debug 2023-08-28T02:40:57.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:57.869486+0000) 2023-08-28T02:40:59.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:40:58 smithi006 bash[27292]: cluster 2023-08-28T02:40:57.998506+0000 mgr.y (mgr.14901) 8470 : cluster [DBG] pgmap v8477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:59.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:40:58 smithi006 bash[33918]: cluster 2023-08-28T02:40:57.998506+0000 mgr.y (mgr.14901) 8470 : cluster [DBG] pgmap v8477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:40:59.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:58 smithi130 bash[51302]: debug 2023-08-28T02:40:58.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:58.869777+0000) 2023-08-28T02:40:59.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:40:58 smithi130 bash[27579]: cluster 2023-08-28T02:40:57.998506+0000 mgr.y (mgr.14901) 8470 : cluster [DBG] pgmap v8477: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:00.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:41:00 smithi130 bash[27579]: cluster 2023-08-28T02:40:59.999570+0000 mgr.y (mgr.14901) 8471 : cluster [DBG] pgmap v8478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:00.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:40:59 smithi130 bash[51302]: debug 2023-08-28T02:40:59.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:59.870100+0000) 2023-08-28T02:41:00.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:41:00 smithi006 bash[27292]: cluster 2023-08-28T02:40:59.999570+0000 mgr.y (mgr.14901) 8471 : cluster [DBG] pgmap v8478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:00.465 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:41:00 smithi006 bash[33918]: cluster 2023-08-28T02:40:59.999570+0000 mgr.y (mgr.14901) 8471 : cluster [DBG] pgmap v8478: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:01.083 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:41:00 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:41:00] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T02:41:01.365 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:41:01 smithi006 bash[33918]: audit 2023-08-28T02:41:00.243653+0000 mon.a (mon.0) 5323 : audit [DBG] from='client.? 172.21.15.6:0/3511323490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:01.365 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:41:01 smithi006 bash[27292]: audit 2023-08-28T02:41:00.243653+0000 mon.a (mon.0) 5323 : audit [DBG] from='client.? 172.21.15.6:0/3511323490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:01.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:41:00 smithi130 bash[51302]: debug 2023-08-28T02:41:00.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:00.870400+0000) 2023-08-28T02:41:01.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:41:01 smithi130 bash[27579]: audit 2023-08-28T02:41:00.243653+0000 mon.a (mon.0) 5323 : audit [DBG] from='client.? 172.21.15.6:0/3511323490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:01.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:41:01 smithi006 bash[56298]: level=error ts=2023-08-28T02:41:01.364Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:41:01.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:41:01 smithi006 bash[56298]: level=warn ts=2023-08-28T02:41:01.367Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:41:02.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:41:02 smithi130 bash[27579]: cluster 2023-08-28T02:41:02.000287+0000 mgr.y (mgr.14901) 8472 : cluster [DBG] pgmap v8479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:02.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:41:01 smithi130 bash[51302]: debug 2023-08-28T02:41:01.865+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:01.870692+0000) 2023-08-28T02:41:02.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:41:02 smithi006 bash[33918]: cluster 2023-08-28T02:41:02.000287+0000 mgr.y (mgr.14901) 8472 : cluster [DBG] pgmap v8479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:02.464 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:41:02 smithi006 bash[27292]: cluster 2023-08-28T02:41:02.000287+0000 mgr.y (mgr.14901) 8472 : cluster [DBG] pgmap v8479: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:03.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:41:03 smithi006 bash[27292]: audit 2023-08-28T02:41:02.662463+0000 mon.a (mon.0) 5324 : audit [DBG] from='client.? 172.21.15.6:0/1816435779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:03.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:41:03 smithi006 bash[33918]: audit 2023-08-28T02:41:02.662463+0000 mon.a (mon.0) 5324 : audit [DBG] from='client.? 172.21.15.6:0/1816435779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:41:02 smithi006 bash[56298]: level=error ts=2023-08-28T02:41:02.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:41:03.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:41:02 smithi006 bash[56298]: level=warn ts=2023-08-28T02:41:02.930Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:41:03.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:41:02 smithi130 bash[51302]: debug 2023-08-28T02:41:02.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:02.871018+0000) 2023-08-28T02:41:03.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:41:03 smithi130 bash[27579]: audit 2023-08-28T02:41:02.662463+0000 mon.a (mon.0) 5324 : audit [DBG] from='client.? 172.21.15.6:0/1816435779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:04.275 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:41:03 smithi130 bash[51302]: debug 2023-08-28T02:41:03.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:03.871295+0000) 2023-08-28T02:41:04.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:41:04 smithi130 bash[27579]: cluster 2023-08-28T02:41:04.000978+0000 mgr.y (mgr.14901) 8473 : cluster [DBG] pgmap v8480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:04.715 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:41:04 smithi006 bash[27292]: cluster 2023-08-28T02:41:04.000978+0000 mgr.y (mgr.14901) 8473 : cluster [DBG] pgmap v8480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:04.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:41:04 smithi006 bash[33918]: cluster 2023-08-28T02:41:04.000978+0000 mgr.y (mgr.14901) 8473 : cluster [DBG] pgmap v8480: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:05.214 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:41:04 smithi006 bash[56298]: level=error ts=2023-08-28T02:41:04.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:41:05.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:41:04 smithi006 bash[56298]: level=warn ts=2023-08-28T02:41:04.810Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:41:05.279 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:41:04 smithi130 bash[51302]: debug 2023-08-28T02:41:04.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:04.871646+0000) 2023-08-28T02:41:05.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:41:05 smithi130 bash[27579]: audit 2023-08-28T02:41:05.079733+0000 mon.a (mon.0) 5325 : audit [DBG] from='client.? 172.21.15.6:0/1217726763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:05.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:41:05 smithi006 bash[27292]: audit 2023-08-28T02:41:05.079733+0000 mon.a (mon.0) 5325 : audit [DBG] from='client.? 172.21.15.6:0/1217726763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:05.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:41:05 smithi006 bash[33918]: audit 2023-08-28T02:41:05.079733+0000 mon.a (mon.0) 5325 : audit [DBG] from='client.? 172.21.15.6:0/1217726763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:06.288 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:41:05 smithi130 bash[51302]: debug 2023-08-28T02:41:05.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:05.872116+0000) 2023-08-28T02:41:06.616 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:41:06 smithi130 bash[27579]: cluster 2023-08-28T02:41:06.002074+0000 mgr.y (mgr.14901) 8474 : cluster [DBG] pgmap v8481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:06.714 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:41:06 smithi006 bash[27292]: cluster 2023-08-28T02:41:06.002074+0000 mgr.y (mgr.14901) 8474 : cluster [DBG] pgmap v8481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:06.715 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:41:06 smithi006 bash[33918]: cluster 2023-08-28T02:41:06.002074+0000 mgr.y (mgr.14901) 8474 : cluster [DBG] pgmap v8481: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:07.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:41:06 smithi130 bash[51302]: debug 2023-08-28T02:41:06.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:06.872423+0000) 2023-08-28T02:41:07.865 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:41:07 smithi130 bash[27579]: audit 2023-08-28T02:41:07.485722+0000 mon.a (mon.0) 5326 : audit [DBG] from='client.? 172.21.15.6:0/762239838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:07.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:41:07 smithi006 bash[33918]: audit 2023-08-28T02:41:07.485722+0000 mon.a (mon.0) 5326 : audit [DBG] from='client.? 172.21.15.6:0/762239838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:07.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:41:07 smithi006 bash[27292]: audit 2023-08-28T02:41:07.485722+0000 mon.a (mon.0) 5326 : audit [DBG] from='client.? 172.21.15.6:0/762239838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:08.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:41:07 smithi130 bash[51302]: debug 2023-08-28T02:41:07.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:07.872695+0000) 2023-08-28T02:41:08.866 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:41:08 smithi130 bash[27579]: cluster 2023-08-28T02:41:08.002748+0000 mgr.y (mgr.14901) 8475 : cluster [DBG] pgmap v8482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:08.964 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:41:08 smithi006 bash[33918]: cluster 2023-08-28T02:41:08.002748+0000 mgr.y (mgr.14901) 8475 : cluster [DBG] pgmap v8482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:08.965 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:41:08 smithi006 bash[27292]: cluster 2023-08-28T02:41:08.002748+0000 mgr.y (mgr.14901) 8475 : cluster [DBG] pgmap v8482: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:09.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:41:08 smithi130 bash[51302]: debug 2023-08-28T02:41:08.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:08.872961+0000) 2023-08-28T02:41:10.214 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:41:09 smithi006 bash[33918]: audit 2023-08-28T02:41:09.901753+0000 mon.a (mon.0) 5327 : audit [DBG] from='client.? 172.21.15.6:0/2807050201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:10.215 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:41:09 smithi006 bash[27292]: audit 2023-08-28T02:41:09.901753+0000 mon.a (mon.0) 5327 : audit [DBG] from='client.? 172.21.15.6:0/2807050201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:10.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:41:09 smithi130 bash[27579]: audit 2023-08-28T02:41:09.901753+0000 mon.a (mon.0) 5327 : audit [DBG] from='client.? 172.21.15.6:0/2807050201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:10.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:41:09 smithi130 bash[51302]: debug 2023-08-28T02:41:09.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:09.873210+0000) 2023-08-28T02:41:11.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:41:10 smithi006 bash[27292]: cluster 2023-08-28T02:41:10.003809+0000 mgr.y (mgr.14901) 8476 : cluster [DBG] pgmap v8483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:11.215 INFO:journalctl@ceph.mgr.y.smithi006.stdout:Aug 28 02:41:10 smithi006 bash[27566]: ::ffff:172.21.15.130 - - [28/Aug/2023:02:41:10] "GET /metrics HTTP/1.1" 200 193120 "" "Prometheus/2.33.4" 2023-08-28T02:41:11.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:41:10 smithi006 bash[33918]: cluster 2023-08-28T02:41:10.003809+0000 mgr.y (mgr.14901) 8476 : cluster [DBG] pgmap v8483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:11.365 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:41:10 smithi130 bash[51302]: debug 2023-08-28T02:41:10.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:10.873473+0000) 2023-08-28T02:41:11.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:41:10 smithi130 bash[27579]: cluster 2023-08-28T02:41:10.003809+0000 mgr.y (mgr.14901) 8476 : cluster [DBG] pgmap v8483: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:11.714 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:41:11 smithi006 bash[56298]: level=error ts=2023-08-28T02:41:11.365Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:41:11.715 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:41:11 smithi006 bash[56298]: level=warn ts=2023-08-28T02:41:11.367Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:41:12.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:41:12 smithi130 bash[27579]: cluster 2023-08-28T02:41:12.004380+0000 mgr.y (mgr.14901) 8477 : cluster [DBG] pgmap v8484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:12.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:41:11 smithi130 bash[51302]: debug 2023-08-28T02:41:11.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:11.873791+0000) 2023-08-28T02:41:12.464 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:41:12 smithi006 bash[33918]: cluster 2023-08-28T02:41:12.004380+0000 mgr.y (mgr.14901) 8477 : cluster [DBG] pgmap v8484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:12.465 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:41:12 smithi006 bash[27292]: cluster 2023-08-28T02:41:12.004380+0000 mgr.y (mgr.14901) 8477 : cluster [DBG] pgmap v8484: 73 pgs: 73 active+clean; 3.3 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-08-28T02:41:13.214 INFO:journalctl@ceph.mon.a.smithi006.stdout:Aug 28 02:41:13 smithi006 bash[27292]: audit 2023-08-28T02:41:12.313843+0000 mon.a (mon.0) 5328 : audit [DBG] from='client.? 172.21.15.6:0/2230374452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:13.215 INFO:journalctl@ceph.mon.c.smithi006.stdout:Aug 28 02:41:13 smithi006 bash[33918]: audit 2023-08-28T02:41:12.313843+0000 mon.a (mon.0) 5328 : audit [DBG] from='client.? 172.21.15.6:0/2230374452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:41:12 smithi006 bash[56298]: level=error ts=2023-08-28T02:41:12.929Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:41:13.215 INFO:journalctl@ceph.alertmanager.a.smithi006.stdout:Aug 28 02:41:12 smithi006 bash[56298]: level=warn ts=2023-08-28T02:41:12.931Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2023-08-28T02:41:13.366 INFO:journalctl@ceph.mon.b.smithi130.stdout:Aug 28 02:41:13 smithi130 bash[27579]: audit 2023-08-28T02:41:12.313843+0000 mon.a (mon.0) 5328 : audit [DBG] from='client.? 172.21.15.6:0/2230374452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:13.366 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:41:12 smithi130 bash[51302]: debug 2023-08-28T02:41:12.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:12.874120+0000) 2023-08-28T02:41:14.277 INFO:journalctl@ceph.mgr.x.smithi130.stdout:Aug 28 02:41:13 smithi130 bash[51302]: debug 2023-08-28T02:41:13.869+0000 7f7c8e51a700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:13.874410+0000)